diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu0.log b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu0.log new file mode 100644 index 0000000000000000000000000000000000000000..259bd785d2208aa47629bcd90eba3aaa5b9c9d91 --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu0.log @@ -0,0 +1,28 @@ +[static] gpu=0 layer=0 seed=3234570 start +[static] gpu=0 layer=0 done +[static] gpu=0 layer=4 seed=7234582 start +[static] gpu=0 layer=4 done +[static] gpu=0 layer=8 seed=11234594 start +[static] gpu=0 layer=8 done +[static] gpu=0 layer=12 seed=15234606 start +[static] gpu=0 layer=12 done +[static] gpu=0 layer=16 seed=19234618 start +[static] gpu=0 layer=16 done +[static] gpu=0 layer=20 seed=23234630 start +[static] gpu=0 layer=20 done +[static] gpu=0 layer=24 seed=27234642 start +[static] gpu=0 layer=24 done +[resampled] gpu=0 layer=0 seed=1234567 start +[resampled] gpu=0 layer=0 done +[resampled] gpu=0 layer=4 seed=5234579 start +[resampled] gpu=0 layer=4 done +[resampled] gpu=0 layer=8 seed=9234591 start +[resampled] gpu=0 layer=8 done +[resampled] gpu=0 layer=12 seed=13234603 start +[resampled] gpu=0 layer=12 done +[resampled] gpu=0 layer=16 seed=17234615 start +[resampled] gpu=0 layer=16 done +[resampled] gpu=0 layer=20 seed=21234627 start +[resampled] gpu=0 layer=20 done +[resampled] gpu=0 layer=24 seed=25234639 start +[resampled] gpu=0 layer=24 done diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu1.log b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu1.log new file mode 100644 index 0000000000000000000000000000000000000000..2cdbeae1722ecb85270aa9887f084b16f66f0aa1 --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu1.log @@ -0,0 +1,28 @@ +[static] gpu=1 layer=1 seed=4234573 start +[static] gpu=1 layer=1 done +[static] gpu=1 layer=5 seed=8234585 start +[static] gpu=1 layer=5 done +[static] gpu=1 layer=9 seed=12234597 start +[static] gpu=1 layer=9 done +[static] gpu=1 layer=13 seed=16234609 start +[static] gpu=1 layer=13 done +[static] gpu=1 layer=17 seed=20234621 start +[static] gpu=1 layer=17 done +[static] gpu=1 layer=21 seed=24234633 start +[static] gpu=1 layer=21 done +[static] gpu=1 layer=25 seed=28234645 start +[static] gpu=1 layer=25 done +[resampled] gpu=1 layer=1 seed=2234570 start +[resampled] gpu=1 layer=1 done +[resampled] gpu=1 layer=5 seed=6234582 start +[resampled] gpu=1 layer=5 done +[resampled] gpu=1 layer=9 seed=10234594 start +[resampled] gpu=1 layer=9 done +[resampled] gpu=1 layer=13 seed=14234606 start +[resampled] gpu=1 layer=13 done +[resampled] gpu=1 layer=17 seed=18234618 start +[resampled] gpu=1 layer=17 done +[resampled] gpu=1 layer=21 seed=22234630 start +[resampled] gpu=1 layer=21 done +[resampled] gpu=1 layer=25 seed=26234642 start +[resampled] gpu=1 layer=25 done diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu2.log b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu2.log new file mode 100644 index 0000000000000000000000000000000000000000..4f4e533b2fa6ae816f5b4c0508a715d4b3735039 --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu2.log @@ -0,0 +1,28 @@ +[static] gpu=2 layer=2 seed=5234576 start +[static] gpu=2 layer=2 done +[static] gpu=2 layer=6 seed=9234588 start +[static] gpu=2 layer=6 done +[static] gpu=2 layer=10 seed=13234600 start +[static] gpu=2 layer=10 done +[static] gpu=2 layer=14 seed=17234612 start +[static] gpu=2 layer=14 done +[static] gpu=2 layer=18 seed=21234624 start +[static] gpu=2 layer=18 done +[static] gpu=2 layer=22 seed=25234636 start +[static] gpu=2 layer=22 done +[static] gpu=2 layer=26 seed=29234648 start +[static] gpu=2 layer=26 done +[resampled] gpu=2 layer=2 seed=3234573 start +[resampled] gpu=2 layer=2 done +[resampled] gpu=2 layer=6 seed=7234585 start +[resampled] gpu=2 layer=6 done +[resampled] gpu=2 layer=10 seed=11234597 start +[resampled] gpu=2 layer=10 done +[resampled] gpu=2 layer=14 seed=15234609 start +[resampled] gpu=2 layer=14 done +[resampled] gpu=2 layer=18 seed=19234621 start +[resampled] gpu=2 layer=18 done +[resampled] gpu=2 layer=22 seed=23234633 start +[resampled] gpu=2 layer=22 done +[resampled] gpu=2 layer=26 seed=27234645 start +[resampled] gpu=2 layer=26 done diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu3.log b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu3.log new file mode 100644 index 0000000000000000000000000000000000000000..c5d8eb062e47d7f772f26276bd89dd7174bbd1fc --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/cache_gpu3.log @@ -0,0 +1,28 @@ +[static] gpu=3 layer=3 seed=6234579 start +[static] gpu=3 layer=3 done +[static] gpu=3 layer=7 seed=10234591 start +[static] gpu=3 layer=7 done +[static] gpu=3 layer=11 seed=14234603 start +[static] gpu=3 layer=11 done +[static] gpu=3 layer=15 seed=18234615 start +[static] gpu=3 layer=15 done +[static] gpu=3 layer=19 seed=22234627 start +[static] gpu=3 layer=19 done +[static] gpu=3 layer=23 seed=26234639 start +[static] gpu=3 layer=23 done +[static] gpu=3 layer=27 seed=30234651 start +[static] gpu=3 layer=27 done +[resampled] gpu=3 layer=3 seed=4234576 start +[resampled] gpu=3 layer=3 done +[resampled] gpu=3 layer=7 seed=8234588 start +[resampled] gpu=3 layer=7 done +[resampled] gpu=3 layer=11 seed=12234600 start +[resampled] gpu=3 layer=11 done +[resampled] gpu=3 layer=15 seed=16234612 start +[resampled] gpu=3 layer=15 done +[resampled] gpu=3 layer=19 seed=20234624 start +[resampled] gpu=3 layer=19 done +[resampled] gpu=3 layer=23 seed=24234636 start +[resampled] gpu=3 layer=23 done +[resampled] gpu=3 layer=27 seed=28234648 start +[resampled] gpu=3 layer=27 done diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_resampled_nrb35_seed42.log b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_resampled_nrb35_seed42.log new file mode 100644 index 0000000000000000000000000000000000000000..e2c7eaf11376468a147923f0876e9fc3624e61fb --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_resampled_nrb35_seed42.log @@ -0,0 +1,2282 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 365m resampled 8k nrb35 seed42", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "resampled", + "nrb35", + "seed42" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run n56uphgo +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_164610-n56uphgo +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 365m resampled 8k nrb35 seed42 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/n56uphgo +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0388 | LR: 6.67e-06 | 2543.7ms/step | 7.9 samples/s +Step 20/44000 | Loss: 11.9182 | LR: 1.33e-05 | 1331.8ms/step | 15.0 samples/s +Step 30/44000 | Loss: 11.5407 | LR: 2.00e-05 | 1331.9ms/step | 15.0 samples/s +Step 40/44000 | Loss: 11.0034 | LR: 2.67e-05 | 1333.2ms/step | 15.0 samples/s +Step 50/44000 | Loss: 10.7303 | LR: 3.33e-05 | 1324.1ms/step | 15.1 samples/s +Step 60/44000 | Loss: 10.4882 | LR: 4.00e-05 | 1325.0ms/step | 15.1 samples/s +Step 70/44000 | Loss: 10.2519 | LR: 4.67e-05 | 1327.7ms/step | 15.1 samples/s +Step 80/44000 | Loss: 9.9476 | LR: 5.33e-05 | 1325.2ms/step | 15.1 samples/s +Step 90/44000 | Loss: 9.5837 | LR: 6.00e-05 | 1325.1ms/step | 15.1 samples/s +Step 100/44000 | Loss: 9.1356 | LR: 6.67e-05 | 1325.1ms/step | 15.1 samples/s +Step 110/44000 | Loss: 8.6049 | LR: 7.33e-05 | 1324.4ms/step | 15.1 samples/s +Step 120/44000 | Loss: 8.1508 | LR: 8.00e-05 | 1323.1ms/step | 15.1 samples/s +Step 130/44000 | Loss: 7.7726 | LR: 8.67e-05 | 1325.6ms/step | 15.1 samples/s +Step 140/44000 | Loss: 7.5407 | LR: 9.33e-05 | 1324.2ms/step | 15.1 samples/s +Step 150/44000 | Loss: 7.3512 | LR: 1.00e-04 | 1325.9ms/step | 15.1 samples/s +Step 160/44000 | Loss: 7.2398 | LR: 1.07e-04 | 1328.7ms/step | 15.1 samples/s +Step 170/44000 | Loss: 7.1307 | LR: 1.13e-04 | 1327.7ms/step | 15.1 samples/s +Step 180/44000 | Loss: 7.0303 | LR: 1.20e-04 | 1330.3ms/step | 15.0 samples/s +Step 190/44000 | Loss: 6.9390 | LR: 1.27e-04 | 1327.3ms/step | 15.1 samples/s +Step 200/44000 | Loss: 6.8732 | LR: 1.33e-04 | 1327.5ms/step | 15.1 samples/s +Step 210/44000 | Loss: 6.8132 | LR: 1.40e-04 | 1332.6ms/step | 15.0 samples/s +Step 220/44000 | Loss: 6.7079 | LR: 1.47e-04 | 1334.2ms/step | 15.0 samples/s +Step 230/44000 | Loss: 6.6066 | LR: 1.53e-04 | 1334.0ms/step | 15.0 samples/s +Step 240/44000 | Loss: 6.5432 | LR: 1.60e-04 | 1330.9ms/step | 15.0 samples/s +Step 250/44000 | Loss: 6.4587 | LR: 1.67e-04 | 1332.9ms/step | 15.0 samples/s +Validation at step 250: Loss = 6.4523, Perplexity = 634.15 +Step 260/44000 | Loss: 6.4275 | LR: 1.73e-04 | 4362.7ms/step | 4.6 samples/s +Step 270/44000 | Loss: 6.3513 | LR: 1.80e-04 | 1341.6ms/step | 14.9 samples/s +Step 280/44000 | Loss: 6.3110 | LR: 1.87e-04 | 1339.9ms/step | 14.9 samples/s +Step 290/44000 | Loss: 6.2904 | LR: 1.93e-04 | 1342.8ms/step | 14.9 samples/s +Step 300/44000 | Loss: 6.2143 | LR: 2.00e-04 | 1343.9ms/step | 14.9 samples/s +Step 310/44000 | Loss: 6.1883 | LR: 2.07e-04 | 1344.0ms/step | 14.9 samples/s +Step 320/44000 | Loss: 6.1423 | LR: 2.13e-04 | 1344.0ms/step | 14.9 samples/s +Step 330/44000 | Loss: 6.0779 | LR: 2.20e-04 | 1342.7ms/step | 14.9 samples/s +Step 340/44000 | Loss: 6.0595 | LR: 2.27e-04 | 1341.9ms/step | 14.9 samples/s +Step 350/44000 | Loss: 5.9904 | LR: 2.33e-04 | 1341.3ms/step | 14.9 samples/s +Step 360/44000 | Loss: 5.9508 | LR: 2.40e-04 | 1343.1ms/step | 14.9 samples/s +Step 370/44000 | Loss: 5.9241 | LR: 2.47e-04 | 1343.6ms/step | 14.9 samples/s +Step 380/44000 | Loss: 5.9082 | LR: 2.53e-04 | 1338.2ms/step | 14.9 samples/s +Step 390/44000 | Loss: 5.8181 | LR: 2.60e-04 | 1340.7ms/step | 14.9 samples/s +Step 400/44000 | Loss: 5.8375 | LR: 2.67e-04 | 1341.4ms/step | 14.9 samples/s +Step 410/44000 | Loss: 5.8069 | LR: 2.73e-04 | 1342.4ms/step | 14.9 samples/s +Step 420/44000 | Loss: 5.8039 | LR: 2.80e-04 | 1342.0ms/step | 14.9 samples/s +Step 430/44000 | Loss: 5.7324 | LR: 2.87e-04 | 1341.7ms/step | 14.9 samples/s +Step 440/44000 | Loss: 5.7025 | LR: 2.93e-04 | 1344.8ms/step | 14.9 samples/s +Step 450/44000 | Loss: 5.6658 | LR: 3.00e-04 | 1342.3ms/step | 14.9 samples/s +Step 460/44000 | Loss: 5.6010 | LR: 3.07e-04 | 1343.7ms/step | 14.9 samples/s +Step 470/44000 | Loss: 5.6065 | LR: 3.13e-04 | 1343.2ms/step | 14.9 samples/s +Step 480/44000 | Loss: 5.5527 | LR: 3.20e-04 | 1341.3ms/step | 14.9 samples/s +Step 490/44000 | Loss: 5.5039 | LR: 3.27e-04 | 1344.2ms/step | 14.9 samples/s +Step 500/44000 | Loss: 5.4897 | LR: 3.33e-04 | 1342.0ms/step | 14.9 samples/s +Validation at step 500: Loss = 5.4757, Perplexity = 238.81 +Step 510/44000 | Loss: 5.4656 | LR: 3.40e-04 | 3595.8ms/step | 5.6 samples/s +Step 520/44000 | Loss: 5.4101 | LR: 3.47e-04 | 1338.6ms/step | 14.9 samples/s +Step 530/44000 | Loss: 5.4413 | LR: 3.53e-04 | 1340.2ms/step | 14.9 samples/s +Step 540/44000 | Loss: 5.3543 | LR: 3.60e-04 | 1340.3ms/step | 14.9 samples/s +Step 550/44000 | Loss: 5.3231 | LR: 3.67e-04 | 1340.7ms/step | 14.9 samples/s +Step 560/44000 | Loss: 5.3123 | LR: 3.73e-04 | 1342.9ms/step | 14.9 samples/s +Step 570/44000 | Loss: 5.3071 | LR: 3.80e-04 | 1340.5ms/step | 14.9 samples/s +Step 580/44000 | Loss: 5.2543 | LR: 3.87e-04 | 1339.4ms/step | 14.9 samples/s +Step 590/44000 | Loss: 5.2134 | LR: 3.93e-04 | 1340.1ms/step | 14.9 samples/s +Step 600/44000 | Loss: 5.2003 | LR: 4.00e-04 | 1342.9ms/step | 14.9 samples/s +Step 610/44000 | Loss: 5.1843 | LR: 4.07e-04 | 1340.5ms/step | 14.9 samples/s +Step 620/44000 | Loss: 5.1656 | LR: 4.13e-04 | 1342.2ms/step | 14.9 samples/s +Step 630/44000 | Loss: 5.0790 | LR: 4.20e-04 | 1340.6ms/step | 14.9 samples/s +Step 640/44000 | Loss: 5.0583 | LR: 4.27e-04 | 1339.6ms/step | 14.9 samples/s +Step 650/44000 | Loss: 5.0422 | LR: 4.33e-04 | 1340.2ms/step | 14.9 samples/s +Step 660/44000 | Loss: 5.0317 | LR: 4.40e-04 | 1340.3ms/step | 14.9 samples/s +Step 670/44000 | Loss: 5.0017 | LR: 4.47e-04 | 1341.7ms/step | 14.9 samples/s +Step 680/44000 | Loss: 4.9558 | LR: 4.53e-04 | 1341.4ms/step | 14.9 samples/s +Step 690/44000 | Loss: 4.9355 | LR: 4.60e-04 | 1341.9ms/step | 14.9 samples/s +Step 700/44000 | Loss: 4.9171 | LR: 4.67e-04 | 1339.3ms/step | 14.9 samples/s +Step 710/44000 | Loss: 4.8771 | LR: 4.73e-04 | 1339.0ms/step | 14.9 samples/s +Step 720/44000 | Loss: 4.8265 | LR: 4.80e-04 | 1340.8ms/step | 14.9 samples/s +Step 730/44000 | Loss: 4.8274 | LR: 4.87e-04 | 1340.6ms/step | 14.9 samples/s +Step 740/44000 | Loss: 4.7605 | LR: 4.93e-04 | 1340.4ms/step | 14.9 samples/s +Step 750/44000 | Loss: 4.7850 | LR: 5.00e-04 | 1343.1ms/step | 14.9 samples/s +Validation at step 750: Loss = 4.7816, Perplexity = 119.30 +Step 760/44000 | Loss: 4.7674 | LR: 5.00e-04 | 3588.6ms/step | 5.6 samples/s +Step 770/44000 | Loss: 4.7128 | LR: 5.00e-04 | 1339.2ms/step | 14.9 samples/s +Step 780/44000 | Loss: 4.7008 | LR: 5.00e-04 | 1338.3ms/step | 14.9 samples/s +Step 790/44000 | Loss: 4.6771 | LR: 5.00e-04 | 1339.9ms/step | 14.9 samples/s +Step 800/44000 | Loss: 4.6400 | LR: 5.00e-04 | 1340.3ms/step | 14.9 samples/s +Step 810/44000 | Loss: 4.6209 | LR: 5.00e-04 | 1340.3ms/step | 14.9 samples/s +Step 820/44000 | Loss: 4.6031 | LR: 5.00e-04 | 1341.6ms/step | 14.9 samples/s +Step 830/44000 | Loss: 4.5780 | LR: 5.00e-04 | 1341.7ms/step | 14.9 samples/s +Step 840/44000 | Loss: 4.5499 | LR: 5.00e-04 | 1339.4ms/step | 14.9 samples/s +Step 850/44000 | Loss: 4.5555 | LR: 5.00e-04 | 1337.9ms/step | 14.9 samples/s +Step 860/44000 | Loss: 4.5569 | LR: 5.00e-04 | 1342.8ms/step | 14.9 samples/s +Step 870/44000 | Loss: 4.4837 | LR: 5.00e-04 | 1344.6ms/step | 14.9 samples/s +Step 880/44000 | Loss: 4.4926 | LR: 5.00e-04 | 1341.0ms/step | 14.9 samples/s +Step 890/44000 | Loss: 4.4656 | LR: 5.00e-04 | 1341.0ms/step | 14.9 samples/s +Step 900/44000 | Loss: 4.4469 | LR: 5.00e-04 | 1342.3ms/step | 14.9 samples/s +Step 910/44000 | Loss: 4.4215 | LR: 5.00e-04 | 1340.1ms/step | 14.9 samples/s +Step 920/44000 | Loss: 4.4175 | LR: 5.00e-04 | 1341.1ms/step | 14.9 samples/s +Step 930/44000 | Loss: 4.3884 | LR: 5.00e-04 | 1343.0ms/step | 14.9 samples/s +Step 940/44000 | Loss: 4.3814 | LR: 5.00e-04 | 1340.8ms/step | 14.9 samples/s +Step 950/44000 | Loss: 4.3715 | LR: 5.00e-04 | 1340.0ms/step | 14.9 samples/s +Step 960/44000 | Loss: 4.3289 | LR: 5.00e-04 | 1341.1ms/step | 14.9 samples/s +Step 970/44000 | Loss: 4.3598 | LR: 5.00e-04 | 1339.8ms/step | 14.9 samples/s +Step 980/44000 | Loss: 4.3443 | LR: 5.00e-04 | 1340.5ms/step | 14.9 samples/s +Step 990/44000 | Loss: 4.3173 | LR: 5.00e-04 | 1341.7ms/step | 14.9 samples/s +Step 1000/44000 | Loss: 4.3184 | LR: 5.00e-04 | 1342.6ms/step | 14.9 samples/s +Validation at step 1000: Loss = 4.3429, Perplexity = 76.93 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-1000 +Step 1010/44000 | Loss: 4.2867 | LR: 5.00e-04 | 3916.4ms/step | 5.1 samples/s +Step 1020/44000 | Loss: 4.2830 | LR: 5.00e-04 | 1340.8ms/step | 14.9 samples/s +Step 1030/44000 | Loss: 4.2319 | LR: 5.00e-04 | 1339.8ms/step | 14.9 samples/s +Step 1040/44000 | Loss: 4.2422 | LR: 5.00e-04 | 1338.2ms/step | 14.9 samples/s +Step 1050/44000 | Loss: 4.2404 | LR: 5.00e-04 | 1339.1ms/step | 14.9 samples/s +Step 1060/44000 | Loss: 4.2347 | LR: 5.00e-04 | 1341.9ms/step | 14.9 samples/s +Step 1070/44000 | Loss: 4.2160 | LR: 5.00e-04 | 1339.3ms/step | 14.9 samples/s +Step 1080/44000 | Loss: 4.2063 | LR: 5.00e-04 | 1338.5ms/step | 14.9 samples/s +Step 1090/44000 | Loss: 4.2274 | LR: 5.00e-04 | 1338.9ms/step | 14.9 samples/s +Step 1100/44000 | Loss: 4.1980 | LR: 5.00e-04 | 1340.7ms/step | 14.9 samples/s +Step 1110/44000 | Loss: 4.1806 | LR: 5.00e-04 | 1344.0ms/step | 14.9 samples/s +Step 1120/44000 | Loss: 4.1630 | LR: 5.00e-04 | 1339.8ms/step | 14.9 samples/s +Step 1130/44000 | Loss: 4.1363 | LR: 5.00e-04 | 1339.5ms/step | 14.9 samples/s +Step 1140/44000 | Loss: 4.1605 | LR: 5.00e-04 | 1341.5ms/step | 14.9 samples/s +Step 1150/44000 | Loss: 4.1109 | LR: 5.00e-04 | 1340.0ms/step | 14.9 samples/s +Step 1160/44000 | Loss: 4.1093 | LR: 5.00e-04 | 1338.2ms/step | 14.9 samples/s +Step 1170/44000 | Loss: 4.1182 | LR: 5.00e-04 | 1343.2ms/step | 14.9 samples/s +Step 1180/44000 | Loss: 4.0996 | LR: 5.00e-04 | 1340.9ms/step | 14.9 samples/s +Step 1190/44000 | Loss: 4.1200 | LR: 5.00e-04 | 1340.1ms/step | 14.9 samples/s +Step 1200/44000 | Loss: 4.1153 | LR: 5.00e-04 | 1340.9ms/step | 14.9 samples/s +Step 1210/44000 | Loss: 4.1241 | LR: 5.00e-04 | 1339.8ms/step | 14.9 samples/s +Step 1220/44000 | Loss: 4.0718 | LR: 5.00e-04 | 1340.7ms/step | 14.9 samples/s +Step 1230/44000 | Loss: 4.0637 | LR: 5.00e-04 | 1342.5ms/step | 14.9 samples/s +Step 1240/44000 | Loss: 4.0680 | LR: 5.00e-04 | 1340.1ms/step | 14.9 samples/s +Step 1250/44000 | Loss: 4.0711 | LR: 5.00e-04 | 1342.0ms/step | 14.9 samples/s +Validation at step 1250: Loss = 4.0665, Perplexity = 58.35 +Step 1260/44000 | Loss: 4.0414 | LR: 5.00e-04 | 3589.9ms/step | 5.6 samples/s +Step 1270/44000 | Loss: 4.0409 | LR: 5.00e-04 | 1333.6ms/step | 15.0 samples/s +Step 1280/44000 | Loss: 4.0489 | LR: 5.00e-04 | 1337.3ms/step | 15.0 samples/s +Step 1290/44000 | Loss: 3.9961 | LR: 5.00e-04 | 1336.6ms/step | 15.0 samples/s +Step 1300/44000 | Loss: 4.0479 | LR: 5.00e-04 | 1329.9ms/step | 15.0 samples/s +Step 1310/44000 | Loss: 3.9891 | LR: 5.00e-04 | 1330.3ms/step | 15.0 samples/s +Step 1320/44000 | Loss: 4.0167 | LR: 5.00e-04 | 1328.7ms/step | 15.1 samples/s +Step 1330/44000 | Loss: 4.0197 | LR: 5.00e-04 | 1330.6ms/step | 15.0 samples/s +Step 1340/44000 | Loss: 3.9802 | LR: 5.00e-04 | 1333.4ms/step | 15.0 samples/s +Step 1350/44000 | Loss: 3.9835 | LR: 5.00e-04 | 1332.3ms/step | 15.0 samples/s +Step 1360/44000 | Loss: 3.9779 | LR: 5.00e-04 | 1332.3ms/step | 15.0 samples/s +Step 1370/44000 | Loss: 3.9577 | LR: 5.00e-04 | 1331.9ms/step | 15.0 samples/s +Step 1380/44000 | Loss: 3.9669 | LR: 5.00e-04 | 1331.3ms/step | 15.0 samples/s +Step 1390/44000 | Loss: 3.9685 | LR: 5.00e-04 | 1334.1ms/step | 15.0 samples/s +Step 1400/44000 | Loss: 3.9640 | LR: 5.00e-04 | 1330.8ms/step | 15.0 samples/s +Step 1410/44000 | Loss: 3.9561 | LR: 5.00e-04 | 1331.8ms/step | 15.0 samples/s +Step 1420/44000 | Loss: 3.9461 | LR: 5.00e-04 | 1330.4ms/step | 15.0 samples/s +Step 1430/44000 | Loss: 3.9610 | LR: 5.00e-04 | 1330.9ms/step | 15.0 samples/s +Step 1440/44000 | Loss: 3.9573 | LR: 5.00e-04 | 1329.3ms/step | 15.0 samples/s +Step 1450/44000 | Loss: 3.9423 | LR: 5.00e-04 | 1330.2ms/step | 15.0 samples/s +Step 1460/44000 | Loss: 3.9407 | LR: 5.00e-04 | 1331.3ms/step | 15.0 samples/s +Step 1470/44000 | Loss: 3.9525 | LR: 5.00e-04 | 1331.1ms/step | 15.0 samples/s +Step 1480/44000 | Loss: 3.9473 | LR: 5.00e-04 | 1332.4ms/step | 15.0 samples/s +Step 1490/44000 | Loss: 3.9408 | LR: 5.00e-04 | 1332.9ms/step | 15.0 samples/s +Step 1500/44000 | Loss: 3.9195 | LR: 5.00e-04 | 1334.2ms/step | 15.0 samples/s +Validation at step 1500: Loss = 3.9165, Perplexity = 50.22 +Step 1510/44000 | Loss: 3.9409 | LR: 5.00e-04 | 3580.6ms/step | 5.6 samples/s +Step 1520/44000 | Loss: 3.9351 | LR: 5.00e-04 | 1338.7ms/step | 14.9 samples/s +Step 1530/44000 | Loss: 3.9011 | LR: 5.00e-04 | 1338.3ms/step | 14.9 samples/s +Step 1540/44000 | Loss: 3.9097 | LR: 5.00e-04 | 1332.9ms/step | 15.0 samples/s +Step 1550/44000 | Loss: 3.8721 | LR: 5.00e-04 | 1336.5ms/step | 15.0 samples/s +Step 1560/44000 | Loss: 3.8636 | LR: 5.00e-04 | 1335.0ms/step | 15.0 samples/s +Step 1570/44000 | Loss: 3.8575 | LR: 5.00e-04 | 1338.3ms/step | 14.9 samples/s +Step 1580/44000 | Loss: 3.8778 | LR: 5.00e-04 | 1336.0ms/step | 15.0 samples/s +Step 1590/44000 | Loss: 3.8513 | LR: 5.00e-04 | 1335.1ms/step | 15.0 samples/s +Step 1600/44000 | Loss: 3.8602 | LR: 5.00e-04 | 1336.5ms/step | 15.0 samples/s +Step 1610/44000 | Loss: 3.8398 | LR: 5.00e-04 | 1339.0ms/step | 14.9 samples/s +Step 1620/44000 | Loss: 3.8245 | LR: 5.00e-04 | 1336.2ms/step | 15.0 samples/s +Step 1630/44000 | Loss: 3.8393 | LR: 4.99e-04 | 1338.6ms/step | 14.9 samples/s +Step 1640/44000 | Loss: 3.8424 | LR: 4.99e-04 | 1336.4ms/step | 15.0 samples/s +Step 1650/44000 | Loss: 3.8185 | LR: 4.99e-04 | 1334.3ms/step | 15.0 samples/s +Step 1660/44000 | Loss: 3.8245 | LR: 4.99e-04 | 1336.6ms/step | 15.0 samples/s +Step 1670/44000 | Loss: 3.8149 | LR: 4.99e-04 | 1337.0ms/step | 15.0 samples/s +Step 1680/44000 | Loss: 3.8194 | LR: 4.99e-04 | 1337.3ms/step | 15.0 samples/s +Step 1690/44000 | Loss: 3.8045 | LR: 4.99e-04 | 1336.7ms/step | 15.0 samples/s +Step 1700/44000 | Loss: 3.8465 | LR: 4.99e-04 | 1338.1ms/step | 14.9 samples/s +Step 1710/44000 | Loss: 3.7941 | LR: 4.99e-04 | 1335.6ms/step | 15.0 samples/s +Step 1720/44000 | Loss: 3.8037 | LR: 4.99e-04 | 1335.8ms/step | 15.0 samples/s +Step 1730/44000 | Loss: 3.8043 | LR: 4.99e-04 | 1336.1ms/step | 15.0 samples/s +Step 1740/44000 | Loss: 3.7695 | LR: 4.99e-04 | 1336.4ms/step | 15.0 samples/s +Step 1750/44000 | Loss: 3.7892 | LR: 4.99e-04 | 1340.3ms/step | 14.9 samples/s +Validation at step 1750: Loss = 3.8071, Perplexity = 45.02 +Step 1760/44000 | Loss: 3.7752 | LR: 4.99e-04 | 3579.8ms/step | 5.6 samples/s +Step 1770/44000 | Loss: 3.7725 | LR: 4.99e-04 | 1337.0ms/step | 15.0 samples/s +Step 1780/44000 | Loss: 3.8078 | LR: 4.99e-04 | 1338.4ms/step | 14.9 samples/s +Step 1790/44000 | Loss: 3.7723 | LR: 4.99e-04 | 1335.7ms/step | 15.0 samples/s +Step 1800/44000 | Loss: 3.7714 | LR: 4.99e-04 | 1338.6ms/step | 14.9 samples/s +Step 1810/44000 | Loss: 3.7881 | LR: 4.99e-04 | 1336.8ms/step | 15.0 samples/s +Step 1820/44000 | Loss: 3.7764 | LR: 4.99e-04 | 1337.7ms/step | 15.0 samples/s +Step 1830/44000 | Loss: 3.7646 | LR: 4.99e-04 | 1335.4ms/step | 15.0 samples/s +Step 1840/44000 | Loss: 3.7510 | LR: 4.99e-04 | 1336.9ms/step | 15.0 samples/s +Step 1850/44000 | Loss: 3.7788 | LR: 4.99e-04 | 1337.1ms/step | 15.0 samples/s +Step 1860/44000 | Loss: 3.7636 | LR: 4.99e-04 | 1338.0ms/step | 14.9 samples/s +Step 1870/44000 | Loss: 3.7667 | LR: 4.99e-04 | 1337.3ms/step | 15.0 samples/s +Step 1880/44000 | Loss: 3.7655 | LR: 4.99e-04 | 1335.8ms/step | 15.0 samples/s +Step 1890/44000 | Loss: 3.7587 | LR: 4.99e-04 | 1340.4ms/step | 14.9 samples/s +Step 1900/44000 | Loss: 3.7670 | LR: 4.99e-04 | 1336.7ms/step | 15.0 samples/s +Step 1910/44000 | Loss: 3.7395 | LR: 4.99e-04 | 1337.5ms/step | 15.0 samples/s +Step 1920/44000 | Loss: 3.7579 | LR: 4.99e-04 | 1339.3ms/step | 14.9 samples/s +Step 1930/44000 | Loss: 3.7484 | LR: 4.99e-04 | 1339.6ms/step | 14.9 samples/s +Step 1940/44000 | Loss: 3.7467 | LR: 4.99e-04 | 1337.2ms/step | 15.0 samples/s +Step 1950/44000 | Loss: 3.7429 | LR: 4.99e-04 | 1336.6ms/step | 15.0 samples/s +Step 1960/44000 | Loss: 3.7301 | LR: 4.99e-04 | 1337.2ms/step | 15.0 samples/s +Step 1970/44000 | Loss: 3.7360 | LR: 4.99e-04 | 1339.1ms/step | 14.9 samples/s +Step 1980/44000 | Loss: 3.7123 | LR: 4.99e-04 | 1342.1ms/step | 14.9 samples/s +Step 1990/44000 | Loss: 3.6956 | LR: 4.99e-04 | 1345.8ms/step | 14.9 samples/s +Step 2000/44000 | Loss: 3.7050 | LR: 4.99e-04 | 1336.0ms/step | 15.0 samples/s +Validation at step 2000: Loss = 3.7229, Perplexity = 41.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-2000 +Step 2010/44000 | Loss: 3.6844 | LR: 4.99e-04 | 3898.9ms/step | 5.1 samples/s +Step 2020/44000 | Loss: 3.7424 | LR: 4.99e-04 | 1337.8ms/step | 15.0 samples/s +Step 2030/44000 | Loss: 3.7175 | LR: 4.99e-04 | 1335.3ms/step | 15.0 samples/s +Step 2040/44000 | Loss: 3.7026 | LR: 4.99e-04 | 1334.0ms/step | 15.0 samples/s +Step 2050/44000 | Loss: 3.6931 | LR: 4.99e-04 | 1331.1ms/step | 15.0 samples/s +Step 2060/44000 | Loss: 3.6646 | LR: 4.99e-04 | 1331.2ms/step | 15.0 samples/s +Step 2070/44000 | Loss: 3.6864 | LR: 4.99e-04 | 1333.3ms/step | 15.0 samples/s +Step 2080/44000 | Loss: 3.6554 | LR: 4.99e-04 | 1333.8ms/step | 15.0 samples/s +Step 2090/44000 | Loss: 3.6956 | LR: 4.99e-04 | 1331.3ms/step | 15.0 samples/s +Step 2100/44000 | Loss: 3.6863 | LR: 4.99e-04 | 1332.6ms/step | 15.0 samples/s +Step 2110/44000 | Loss: 3.7051 | LR: 4.99e-04 | 1331.5ms/step | 15.0 samples/s +Step 2120/44000 | Loss: 3.6905 | LR: 4.99e-04 | 1333.9ms/step | 15.0 samples/s +Step 2130/44000 | Loss: 3.6994 | LR: 4.99e-04 | 1333.1ms/step | 15.0 samples/s +Step 2140/44000 | Loss: 3.6494 | LR: 4.99e-04 | 1333.9ms/step | 15.0 samples/s +Step 2150/44000 | Loss: 3.6628 | LR: 4.99e-04 | 1332.8ms/step | 15.0 samples/s +Step 2160/44000 | Loss: 3.6704 | LR: 4.99e-04 | 1334.0ms/step | 15.0 samples/s +Step 2170/44000 | Loss: 3.6727 | LR: 4.99e-04 | 1332.6ms/step | 15.0 samples/s +Step 2180/44000 | Loss: 3.6576 | LR: 4.99e-04 | 1333.0ms/step | 15.0 samples/s +Step 2190/44000 | Loss: 3.6670 | LR: 4.99e-04 | 1333.7ms/step | 15.0 samples/s +Step 2200/44000 | Loss: 3.6398 | LR: 4.99e-04 | 1333.0ms/step | 15.0 samples/s +Step 2210/44000 | Loss: 3.6438 | LR: 4.99e-04 | 1330.9ms/step | 15.0 samples/s +Step 2220/44000 | Loss: 3.6234 | LR: 4.99e-04 | 1332.3ms/step | 15.0 samples/s +Step 2230/44000 | Loss: 3.6666 | LR: 4.99e-04 | 1332.6ms/step | 15.0 samples/s +Step 2240/44000 | Loss: 3.6464 | LR: 4.99e-04 | 1331.5ms/step | 15.0 samples/s +Step 2250/44000 | Loss: 3.6487 | LR: 4.99e-04 | 1331.7ms/step | 15.0 samples/s +Validation at step 2250: Loss = 3.6598, Perplexity = 38.86 +Step 2260/44000 | Loss: 3.6441 | LR: 4.98e-04 | 3571.4ms/step | 5.6 samples/s +Step 2270/44000 | Loss: 3.6301 | LR: 4.98e-04 | 1332.4ms/step | 15.0 samples/s +Step 2280/44000 | Loss: 3.6581 | LR: 4.98e-04 | 1332.2ms/step | 15.0 samples/s +Step 2290/44000 | Loss: 3.6484 | LR: 4.98e-04 | 1334.1ms/step | 15.0 samples/s +Step 2300/44000 | Loss: 3.6274 | LR: 4.98e-04 | 1334.0ms/step | 15.0 samples/s +Step 2310/44000 | Loss: 3.6419 | LR: 4.98e-04 | 1330.8ms/step | 15.0 samples/s +Step 2320/44000 | Loss: 3.6289 | LR: 4.98e-04 | 1337.3ms/step | 15.0 samples/s +Step 2330/44000 | Loss: 3.6327 | LR: 4.98e-04 | 1332.9ms/step | 15.0 samples/s +Step 2340/44000 | Loss: 3.6091 | LR: 4.98e-04 | 1333.5ms/step | 15.0 samples/s +Step 2350/44000 | Loss: 3.6201 | LR: 4.98e-04 | 1336.9ms/step | 15.0 samples/s +Step 2360/44000 | Loss: 3.6027 | LR: 4.98e-04 | 1336.5ms/step | 15.0 samples/s +Step 2370/44000 | Loss: 3.6420 | LR: 4.98e-04 | 1331.7ms/step | 15.0 samples/s +Step 2380/44000 | Loss: 3.6196 | LR: 4.98e-04 | 1331.4ms/step | 15.0 samples/s +Step 2390/44000 | Loss: 3.5861 | LR: 4.98e-04 | 1332.3ms/step | 15.0 samples/s +Step 2400/44000 | Loss: 3.5953 | LR: 4.98e-04 | 1332.7ms/step | 15.0 samples/s +Step 2410/44000 | Loss: 3.6373 | LR: 4.98e-04 | 1330.3ms/step | 15.0 samples/s +Step 2420/44000 | Loss: 3.6452 | LR: 4.98e-04 | 1331.0ms/step | 15.0 samples/s +Step 2430/44000 | Loss: 3.6126 | LR: 4.98e-04 | 1330.1ms/step | 15.0 samples/s +Step 2440/44000 | Loss: 3.5893 | LR: 4.98e-04 | 1332.1ms/step | 15.0 samples/s +Step 2450/44000 | Loss: 3.6181 | LR: 4.98e-04 | 1331.8ms/step | 15.0 samples/s +Step 2460/44000 | Loss: 3.6116 | LR: 4.98e-04 | 1333.2ms/step | 15.0 samples/s +Step 2470/44000 | Loss: 3.5888 | LR: 4.98e-04 | 1327.5ms/step | 15.1 samples/s +Step 2480/44000 | Loss: 3.6200 | LR: 4.98e-04 | 1331.8ms/step | 15.0 samples/s +Step 2490/44000 | Loss: 3.5752 | LR: 4.98e-04 | 1332.2ms/step | 15.0 samples/s +Step 2500/44000 | Loss: 3.5931 | LR: 4.98e-04 | 1331.2ms/step | 15.0 samples/s +Validation at step 2500: Loss = 3.6094, Perplexity = 36.94 +Step 2510/44000 | Loss: 3.5765 | LR: 4.98e-04 | 3571.8ms/step | 5.6 samples/s +Step 2520/44000 | Loss: 3.5950 | LR: 4.98e-04 | 1332.4ms/step | 15.0 samples/s +Step 2530/44000 | Loss: 3.5946 | LR: 4.98e-04 | 1332.0ms/step | 15.0 samples/s +Step 2540/44000 | Loss: 3.6066 | LR: 4.98e-04 | 1325.9ms/step | 15.1 samples/s +Step 2550/44000 | Loss: 3.5942 | LR: 4.98e-04 | 1325.5ms/step | 15.1 samples/s +Step 2560/44000 | Loss: 3.6054 | LR: 4.98e-04 | 1330.1ms/step | 15.0 samples/s +Step 2570/44000 | Loss: 3.5772 | LR: 4.98e-04 | 1330.4ms/step | 15.0 samples/s +Step 2580/44000 | Loss: 3.6160 | LR: 4.98e-04 | 1329.9ms/step | 15.0 samples/s +Step 2590/44000 | Loss: 3.5713 | LR: 4.98e-04 | 1330.0ms/step | 15.0 samples/s +Step 2600/44000 | Loss: 3.5780 | LR: 4.98e-04 | 1335.4ms/step | 15.0 samples/s +Step 2610/44000 | Loss: 3.5742 | LR: 4.98e-04 | 1328.6ms/step | 15.1 samples/s +Step 2620/44000 | Loss: 3.5777 | LR: 4.98e-04 | 1329.2ms/step | 15.0 samples/s +Step 2630/44000 | Loss: 3.5682 | LR: 4.98e-04 | 1333.2ms/step | 15.0 samples/s +Step 2640/44000 | Loss: 3.5817 | LR: 4.98e-04 | 1331.7ms/step | 15.0 samples/s +Step 2650/44000 | Loss: 3.5971 | LR: 4.98e-04 | 1332.4ms/step | 15.0 samples/s +Step 2660/44000 | Loss: 3.5677 | LR: 4.98e-04 | 1333.0ms/step | 15.0 samples/s +Step 2670/44000 | Loss: 3.5589 | LR: 4.98e-04 | 1327.9ms/step | 15.1 samples/s +Step 2680/44000 | Loss: 3.5811 | LR: 4.98e-04 | 1328.3ms/step | 15.1 samples/s +Step 2690/44000 | Loss: 3.5696 | LR: 4.98e-04 | 1330.6ms/step | 15.0 samples/s +Step 2700/44000 | Loss: 3.6021 | LR: 4.97e-04 | 1332.3ms/step | 15.0 samples/s +Step 2710/44000 | Loss: 3.5662 | LR: 4.97e-04 | 1332.0ms/step | 15.0 samples/s +Step 2720/44000 | Loss: 3.5672 | LR: 4.97e-04 | 1330.4ms/step | 15.0 samples/s +Step 2730/44000 | Loss: 3.5619 | LR: 4.97e-04 | 1334.4ms/step | 15.0 samples/s +Step 2740/44000 | Loss: 3.5635 | LR: 4.97e-04 | 1331.3ms/step | 15.0 samples/s +Step 2750/44000 | Loss: 3.5529 | LR: 4.97e-04 | 1332.4ms/step | 15.0 samples/s +Validation at step 2750: Loss = 3.5675, Perplexity = 35.43 +Step 2760/44000 | Loss: 3.5243 | LR: 4.97e-04 | 3575.7ms/step | 5.6 samples/s +Step 2770/44000 | Loss: 3.5493 | LR: 4.97e-04 | 1333.3ms/step | 15.0 samples/s +Step 2780/44000 | Loss: 3.5260 | LR: 4.97e-04 | 1334.7ms/step | 15.0 samples/s +Step 2790/44000 | Loss: 3.5508 | LR: 4.97e-04 | 1335.1ms/step | 15.0 samples/s +Step 2800/44000 | Loss: 3.5447 | LR: 4.97e-04 | 1333.8ms/step | 15.0 samples/s +Step 2810/44000 | Loss: 3.5270 | LR: 4.97e-04 | 1333.2ms/step | 15.0 samples/s +Step 2820/44000 | Loss: 3.5390 | LR: 4.97e-04 | 1337.0ms/step | 15.0 samples/s +Step 2830/44000 | Loss: 3.5168 | LR: 4.97e-04 | 1334.7ms/step | 15.0 samples/s +Step 2840/44000 | Loss: 3.5264 | LR: 4.97e-04 | 1335.1ms/step | 15.0 samples/s +Step 2850/44000 | Loss: 3.5335 | LR: 4.97e-04 | 1338.3ms/step | 14.9 samples/s +Step 2860/44000 | Loss: 3.5276 | LR: 4.97e-04 | 1335.6ms/step | 15.0 samples/s +Step 2870/44000 | Loss: 3.5377 | LR: 4.97e-04 | 1334.5ms/step | 15.0 samples/s +Step 2880/44000 | Loss: 3.4992 | LR: 4.97e-04 | 1341.1ms/step | 14.9 samples/s +Step 2890/44000 | Loss: 3.5443 | LR: 4.97e-04 | 1337.5ms/step | 15.0 samples/s +Step 2900/44000 | Loss: 3.5433 | LR: 4.97e-04 | 1332.5ms/step | 15.0 samples/s +Step 2910/44000 | Loss: 3.5168 | LR: 4.97e-04 | 1335.0ms/step | 15.0 samples/s +Step 2920/44000 | Loss: 3.5096 | LR: 4.97e-04 | 1339.6ms/step | 14.9 samples/s +Step 2930/44000 | Loss: 3.5028 | LR: 4.97e-04 | 1340.0ms/step | 14.9 samples/s +Step 2940/44000 | Loss: 3.5210 | LR: 4.97e-04 | 1338.1ms/step | 14.9 samples/s +Step 2950/44000 | Loss: 3.5210 | LR: 4.97e-04 | 1338.3ms/step | 14.9 samples/s +Step 2960/44000 | Loss: 3.5354 | LR: 4.97e-04 | 1337.8ms/step | 15.0 samples/s +Step 2970/44000 | Loss: 3.5193 | LR: 4.97e-04 | 1338.8ms/step | 14.9 samples/s +Step 2980/44000 | Loss: 3.5204 | LR: 4.97e-04 | 1341.0ms/step | 14.9 samples/s +Step 2990/44000 | Loss: 3.5094 | LR: 4.97e-04 | 1337.3ms/step | 15.0 samples/s +Step 3000/44000 | Loss: 3.5409 | LR: 4.97e-04 | 1338.7ms/step | 14.9 samples/s +Validation at step 3000: Loss = 3.5274, Perplexity = 34.03 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-3000 +Step 3010/44000 | Loss: 3.5191 | LR: 4.97e-04 | 3885.7ms/step | 5.1 samples/s +Step 3020/44000 | Loss: 3.4997 | LR: 4.97e-04 | 1338.5ms/step | 14.9 samples/s +Step 3030/44000 | Loss: 3.5186 | LR: 4.97e-04 | 1342.6ms/step | 14.9 samples/s +Step 3040/44000 | Loss: 3.4793 | LR: 4.97e-04 | 1339.1ms/step | 14.9 samples/s +Step 3050/44000 | Loss: 3.5054 | LR: 4.97e-04 | 1339.6ms/step | 14.9 samples/s +Step 3060/44000 | Loss: 3.4775 | LR: 4.96e-04 | 1339.2ms/step | 14.9 samples/s +Step 3070/44000 | Loss: 3.5240 | LR: 4.96e-04 | 1339.4ms/step | 14.9 samples/s +Step 3080/44000 | Loss: 3.5452 | LR: 4.96e-04 | 1337.5ms/step | 15.0 samples/s +Step 3090/44000 | Loss: 3.5333 | LR: 4.96e-04 | 1337.7ms/step | 15.0 samples/s +Step 3100/44000 | Loss: 3.4920 | LR: 4.96e-04 | 1337.1ms/step | 15.0 samples/s +Step 3110/44000 | Loss: 3.4929 | LR: 4.96e-04 | 1337.3ms/step | 15.0 samples/s +Step 3120/44000 | Loss: 3.5120 | LR: 4.96e-04 | 1337.6ms/step | 15.0 samples/s +Step 3130/44000 | Loss: 3.4806 | LR: 4.96e-04 | 1337.7ms/step | 15.0 samples/s +Step 3140/44000 | Loss: 3.4986 | LR: 4.96e-04 | 1337.0ms/step | 15.0 samples/s +Step 3150/44000 | Loss: 3.4947 | LR: 4.96e-04 | 1334.0ms/step | 15.0 samples/s +Step 3160/44000 | Loss: 3.5163 | LR: 4.96e-04 | 1336.4ms/step | 15.0 samples/s +Step 3170/44000 | Loss: 3.4573 | LR: 4.96e-04 | 1339.2ms/step | 14.9 samples/s +Step 3180/44000 | Loss: 3.5004 | LR: 4.96e-04 | 1335.9ms/step | 15.0 samples/s +Step 3190/44000 | Loss: 3.4878 | LR: 4.96e-04 | 1337.5ms/step | 15.0 samples/s +Step 3200/44000 | Loss: 3.5134 | LR: 4.96e-04 | 1338.9ms/step | 14.9 samples/s +Step 3210/44000 | Loss: 3.4853 | LR: 4.96e-04 | 1338.9ms/step | 14.9 samples/s +Step 3220/44000 | Loss: 3.4885 | LR: 4.96e-04 | 1337.7ms/step | 15.0 samples/s +Step 3230/44000 | Loss: 3.4875 | LR: 4.96e-04 | 1336.4ms/step | 15.0 samples/s +Step 3240/44000 | Loss: 3.4831 | LR: 4.96e-04 | 1339.0ms/step | 14.9 samples/s +Step 3250/44000 | Loss: 3.4877 | LR: 4.96e-04 | 1336.5ms/step | 15.0 samples/s +Validation at step 3250: Loss = 3.4942, Perplexity = 32.92 +Step 3260/44000 | Loss: 3.5048 | LR: 4.96e-04 | 3585.6ms/step | 5.6 samples/s +Step 3270/44000 | Loss: 3.4700 | LR: 4.96e-04 | 1341.9ms/step | 14.9 samples/s +Step 3280/44000 | Loss: 3.4594 | LR: 4.96e-04 | 1338.3ms/step | 14.9 samples/s +Step 3290/44000 | Loss: 3.4796 | LR: 4.96e-04 | 1338.8ms/step | 14.9 samples/s +Step 3300/44000 | Loss: 3.5069 | LR: 4.96e-04 | 1336.3ms/step | 15.0 samples/s +Step 3310/44000 | Loss: 3.4807 | LR: 4.96e-04 | 1336.9ms/step | 15.0 samples/s +Step 3320/44000 | Loss: 3.5036 | LR: 4.96e-04 | 1337.5ms/step | 15.0 samples/s +Step 3330/44000 | Loss: 3.5054 | LR: 4.96e-04 | 1339.2ms/step | 14.9 samples/s +Step 3340/44000 | Loss: 3.4640 | LR: 4.96e-04 | 1336.8ms/step | 15.0 samples/s +Step 3350/44000 | Loss: 3.4784 | LR: 4.96e-04 | 1337.5ms/step | 15.0 samples/s +Step 3360/44000 | Loss: 3.4604 | LR: 4.96e-04 | 1336.1ms/step | 15.0 samples/s +Step 3370/44000 | Loss: 3.4332 | LR: 4.95e-04 | 1335.3ms/step | 15.0 samples/s +Step 3380/44000 | Loss: 3.4797 | LR: 4.95e-04 | 1337.3ms/step | 15.0 samples/s +Step 3390/44000 | Loss: 3.4592 | LR: 4.95e-04 | 1335.6ms/step | 15.0 samples/s +Step 3400/44000 | Loss: 3.4928 | LR: 4.95e-04 | 1340.3ms/step | 14.9 samples/s +Step 3410/44000 | Loss: 3.4665 | LR: 4.95e-04 | 1340.3ms/step | 14.9 samples/s +Step 3420/44000 | Loss: 3.4745 | LR: 4.95e-04 | 1338.1ms/step | 14.9 samples/s +Step 3430/44000 | Loss: 3.4679 | LR: 4.95e-04 | 1336.2ms/step | 15.0 samples/s +Step 3440/44000 | Loss: 3.4589 | LR: 4.95e-04 | 1336.6ms/step | 15.0 samples/s +Step 3450/44000 | Loss: 3.4427 | LR: 4.95e-04 | 1339.4ms/step | 14.9 samples/s +Step 3460/44000 | Loss: 3.4822 | LR: 4.95e-04 | 1337.5ms/step | 15.0 samples/s +Step 3470/44000 | Loss: 3.4532 | LR: 4.95e-04 | 1339.0ms/step | 14.9 samples/s +Step 3480/44000 | Loss: 3.5024 | LR: 4.95e-04 | 1335.9ms/step | 15.0 samples/s +Step 3490/44000 | Loss: 3.4314 | LR: 4.95e-04 | 1335.5ms/step | 15.0 samples/s +Step 3500/44000 | Loss: 3.4385 | LR: 4.95e-04 | 1335.3ms/step | 15.0 samples/s +Validation at step 3500: Loss = 3.4668, Perplexity = 32.04 +Step 3510/44000 | Loss: 3.4715 | LR: 4.95e-04 | 3569.2ms/step | 5.6 samples/s +Step 3520/44000 | Loss: 3.4590 | LR: 4.95e-04 | 1327.3ms/step | 15.1 samples/s +Step 3530/44000 | Loss: 3.4570 | LR: 4.95e-04 | 1328.1ms/step | 15.1 samples/s +Step 3540/44000 | Loss: 3.4644 | LR: 4.95e-04 | 1333.4ms/step | 15.0 samples/s +Step 3550/44000 | Loss: 3.4409 | LR: 4.95e-04 | 1333.2ms/step | 15.0 samples/s +Step 3560/44000 | Loss: 3.4263 | LR: 4.95e-04 | 1332.6ms/step | 15.0 samples/s +Step 3570/44000 | Loss: 3.4520 | LR: 4.95e-04 | 1329.7ms/step | 15.0 samples/s +Step 3580/44000 | Loss: 3.4295 | LR: 4.95e-04 | 1330.5ms/step | 15.0 samples/s +Step 3590/44000 | Loss: 3.4427 | LR: 4.95e-04 | 1333.2ms/step | 15.0 samples/s +Step 3600/44000 | Loss: 3.4314 | LR: 4.95e-04 | 1332.1ms/step | 15.0 samples/s +Step 3610/44000 | Loss: 3.4419 | LR: 4.95e-04 | 1331.1ms/step | 15.0 samples/s +Step 3620/44000 | Loss: 3.4430 | LR: 4.95e-04 | 1333.5ms/step | 15.0 samples/s +Step 3630/44000 | Loss: 3.4325 | LR: 4.95e-04 | 1330.1ms/step | 15.0 samples/s +Step 3640/44000 | Loss: 3.4439 | LR: 4.95e-04 | 1332.1ms/step | 15.0 samples/s +Step 3650/44000 | Loss: 3.4332 | LR: 4.94e-04 | 1331.6ms/step | 15.0 samples/s +Step 3660/44000 | Loss: 3.4327 | LR: 4.94e-04 | 1332.2ms/step | 15.0 samples/s +Step 3670/44000 | Loss: 3.4255 | LR: 4.94e-04 | 1332.5ms/step | 15.0 samples/s +Step 3680/44000 | Loss: 3.4367 | LR: 4.94e-04 | 1333.3ms/step | 15.0 samples/s +Step 3690/44000 | Loss: 3.4318 | LR: 4.94e-04 | 1328.5ms/step | 15.1 samples/s +Step 3700/44000 | Loss: 3.4465 | LR: 4.94e-04 | 1330.2ms/step | 15.0 samples/s +Step 3710/44000 | Loss: 3.4409 | LR: 4.94e-04 | 1328.4ms/step | 15.1 samples/s +Step 3720/44000 | Loss: 3.3959 | LR: 4.94e-04 | 1330.7ms/step | 15.0 samples/s +Step 3730/44000 | Loss: 3.4445 | LR: 4.94e-04 | 1330.6ms/step | 15.0 samples/s +Step 3740/44000 | Loss: 3.4375 | LR: 4.94e-04 | 1329.5ms/step | 15.0 samples/s +Step 3750/44000 | Loss: 3.4448 | LR: 4.94e-04 | 1331.5ms/step | 15.0 samples/s +Validation at step 3750: Loss = 3.4386, Perplexity = 31.14 +Step 3760/44000 | Loss: 3.4241 | LR: 4.94e-04 | 3568.8ms/step | 5.6 samples/s +Step 3770/44000 | Loss: 3.4205 | LR: 4.94e-04 | 1331.0ms/step | 15.0 samples/s +Step 3780/44000 | Loss: 3.4136 | LR: 4.94e-04 | 1334.2ms/step | 15.0 samples/s +Step 3790/44000 | Loss: 3.4368 | LR: 4.94e-04 | 1333.1ms/step | 15.0 samples/s +Step 3800/44000 | Loss: 3.4183 | LR: 4.94e-04 | 1330.6ms/step | 15.0 samples/s +Step 3810/44000 | Loss: 3.4230 | LR: 4.94e-04 | 1333.1ms/step | 15.0 samples/s +Step 3820/44000 | Loss: 3.4294 | LR: 4.94e-04 | 1333.9ms/step | 15.0 samples/s +Step 3830/44000 | Loss: 3.4209 | LR: 4.94e-04 | 1331.3ms/step | 15.0 samples/s +Step 3840/44000 | Loss: 3.4085 | LR: 4.94e-04 | 1332.9ms/step | 15.0 samples/s +Step 3850/44000 | Loss: 3.4170 | LR: 4.94e-04 | 1331.9ms/step | 15.0 samples/s +Step 3860/44000 | Loss: 3.4161 | LR: 4.94e-04 | 1331.5ms/step | 15.0 samples/s +Step 3870/44000 | Loss: 3.4211 | LR: 4.94e-04 | 1331.7ms/step | 15.0 samples/s +Step 3880/44000 | Loss: 3.3906 | LR: 4.94e-04 | 1331.5ms/step | 15.0 samples/s +Step 3890/44000 | Loss: 3.4459 | LR: 4.94e-04 | 1332.9ms/step | 15.0 samples/s +Step 3900/44000 | Loss: 3.4051 | LR: 4.93e-04 | 1332.6ms/step | 15.0 samples/s +Step 3910/44000 | Loss: 3.3961 | LR: 4.93e-04 | 1330.7ms/step | 15.0 samples/s +Step 3920/44000 | Loss: 3.3997 | LR: 4.93e-04 | 1333.6ms/step | 15.0 samples/s +Step 3930/44000 | Loss: 3.4096 | LR: 4.93e-04 | 1328.3ms/step | 15.1 samples/s +Step 3940/44000 | Loss: 3.4100 | LR: 4.93e-04 | 1332.7ms/step | 15.0 samples/s +Step 3950/44000 | Loss: 3.4359 | LR: 4.93e-04 | 1335.8ms/step | 15.0 samples/s +Step 3960/44000 | Loss: 3.4267 | LR: 4.93e-04 | 1334.7ms/step | 15.0 samples/s +Step 3970/44000 | Loss: 3.3805 | LR: 4.93e-04 | 1333.8ms/step | 15.0 samples/s +Step 3980/44000 | Loss: 3.3916 | LR: 4.93e-04 | 1332.8ms/step | 15.0 samples/s +Step 3990/44000 | Loss: 3.4196 | LR: 4.93e-04 | 1330.4ms/step | 15.0 samples/s +Step 4000/44000 | Loss: 3.4022 | LR: 4.93e-04 | 1330.5ms/step | 15.0 samples/s +Validation at step 4000: Loss = 3.4166, Perplexity = 30.47 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-4000 +Step 4010/44000 | Loss: 3.4546 | LR: 4.93e-04 | 3877.1ms/step | 5.2 samples/s +Step 4020/44000 | Loss: 3.4130 | LR: 4.93e-04 | 1337.4ms/step | 15.0 samples/s +Step 4030/44000 | Loss: 3.3904 | LR: 4.93e-04 | 1337.1ms/step | 15.0 samples/s +Step 4040/44000 | Loss: 3.4058 | LR: 4.93e-04 | 1337.1ms/step | 15.0 samples/s +Step 4050/44000 | Loss: 3.4385 | LR: 4.93e-04 | 1337.8ms/step | 14.9 samples/s +Step 4060/44000 | Loss: 3.3869 | LR: 4.93e-04 | 1337.1ms/step | 15.0 samples/s +Step 4070/44000 | Loss: 3.4059 | LR: 4.93e-04 | 1337.5ms/step | 15.0 samples/s +Step 4080/44000 | Loss: 3.4241 | LR: 4.93e-04 | 1335.3ms/step | 15.0 samples/s +Step 4090/44000 | Loss: 3.3771 | LR: 4.93e-04 | 1336.7ms/step | 15.0 samples/s +Step 4100/44000 | Loss: 3.4079 | LR: 4.93e-04 | 1338.7ms/step | 14.9 samples/s +Step 4110/44000 | Loss: 3.3784 | LR: 4.93e-04 | 1339.8ms/step | 14.9 samples/s +Step 4120/44000 | Loss: 3.4066 | LR: 4.93e-04 | 1337.7ms/step | 15.0 samples/s +Step 4130/44000 | Loss: 3.4060 | LR: 4.93e-04 | 1337.7ms/step | 15.0 samples/s +Step 4140/44000 | Loss: 3.3875 | LR: 4.92e-04 | 1337.2ms/step | 15.0 samples/s +Step 4150/44000 | Loss: 3.4343 | LR: 4.92e-04 | 1337.4ms/step | 15.0 samples/s +Step 4160/44000 | Loss: 3.4129 | LR: 4.92e-04 | 1339.4ms/step | 14.9 samples/s +Step 4170/44000 | Loss: 3.3965 | LR: 4.92e-04 | 1339.2ms/step | 14.9 samples/s +Step 4180/44000 | Loss: 3.3770 | LR: 4.92e-04 | 1340.3ms/step | 14.9 samples/s +Step 4190/44000 | Loss: 3.3870 | LR: 4.92e-04 | 1337.7ms/step | 15.0 samples/s +Step 4200/44000 | Loss: 3.3795 | LR: 4.92e-04 | 1337.4ms/step | 15.0 samples/s +Step 4210/44000 | Loss: 3.4115 | LR: 4.92e-04 | 1336.5ms/step | 15.0 samples/s +Step 4220/44000 | Loss: 3.3777 | LR: 4.92e-04 | 1339.8ms/step | 14.9 samples/s +Step 4230/44000 | Loss: 3.4071 | LR: 4.92e-04 | 1339.1ms/step | 14.9 samples/s +Step 4240/44000 | Loss: 3.4702 | LR: 4.92e-04 | 1335.7ms/step | 15.0 samples/s +Step 4250/44000 | Loss: 3.4295 | LR: 4.92e-04 | 1337.6ms/step | 15.0 samples/s +Validation at step 4250: Loss = 3.3979, Perplexity = 29.90 +Step 4260/44000 | Loss: 3.4032 | LR: 4.92e-04 | 3594.4ms/step | 5.6 samples/s +Step 4270/44000 | Loss: 3.3365 | LR: 4.92e-04 | 1337.3ms/step | 15.0 samples/s +Step 4280/44000 | Loss: 3.3599 | LR: 4.92e-04 | 1341.0ms/step | 14.9 samples/s +Step 4290/44000 | Loss: 3.3919 | LR: 4.92e-04 | 1337.6ms/step | 15.0 samples/s +Step 4300/44000 | Loss: 3.3910 | LR: 4.92e-04 | 1336.2ms/step | 15.0 samples/s +Step 4310/44000 | Loss: 3.3873 | LR: 4.92e-04 | 1333.8ms/step | 15.0 samples/s +Step 4320/44000 | Loss: 3.3923 | LR: 4.92e-04 | 1337.6ms/step | 15.0 samples/s +Step 4330/44000 | Loss: 3.3804 | LR: 4.92e-04 | 1336.7ms/step | 15.0 samples/s +Step 4340/44000 | Loss: 3.3928 | LR: 4.92e-04 | 1338.7ms/step | 14.9 samples/s +Step 4350/44000 | Loss: 3.3995 | LR: 4.92e-04 | 1336.4ms/step | 15.0 samples/s +Step 4360/44000 | Loss: 3.3483 | LR: 4.91e-04 | 1339.6ms/step | 14.9 samples/s +Step 4370/44000 | Loss: 3.3672 | LR: 4.91e-04 | 1335.3ms/step | 15.0 samples/s +Step 4380/44000 | Loss: 3.4197 | LR: 4.91e-04 | 1335.6ms/step | 15.0 samples/s +Step 4390/44000 | Loss: 3.4008 | LR: 4.91e-04 | 1334.0ms/step | 15.0 samples/s +Step 4400/44000 | Loss: 3.3997 | LR: 4.91e-04 | 1338.8ms/step | 14.9 samples/s +Step 4410/44000 | Loss: 3.3708 | LR: 4.91e-04 | 1337.0ms/step | 15.0 samples/s +Step 4420/44000 | Loss: 3.3789 | LR: 4.91e-04 | 1338.1ms/step | 14.9 samples/s +Step 4430/44000 | Loss: 3.3811 | LR: 4.91e-04 | 1338.7ms/step | 14.9 samples/s +Step 4440/44000 | Loss: 3.3923 | LR: 4.91e-04 | 1336.9ms/step | 15.0 samples/s +Step 4450/44000 | Loss: 3.3832 | LR: 4.91e-04 | 1334.9ms/step | 15.0 samples/s +Step 4460/44000 | Loss: 3.3815 | LR: 4.91e-04 | 1334.6ms/step | 15.0 samples/s +Step 4470/44000 | Loss: 3.3442 | LR: 4.91e-04 | 1335.8ms/step | 15.0 samples/s +Step 4480/44000 | Loss: 3.3788 | LR: 4.91e-04 | 1335.2ms/step | 15.0 samples/s +Step 4490/44000 | Loss: 3.3845 | LR: 4.91e-04 | 1338.2ms/step | 14.9 samples/s +Step 4500/44000 | Loss: 3.3764 | LR: 4.91e-04 | 1335.9ms/step | 15.0 samples/s +Validation at step 4500: Loss = 3.3802, Perplexity = 29.38 +Step 4510/44000 | Loss: 3.3676 | LR: 4.91e-04 | 3580.2ms/step | 5.6 samples/s +Step 4520/44000 | Loss: 3.3899 | LR: 4.91e-04 | 1337.5ms/step | 15.0 samples/s +Step 4530/44000 | Loss: 3.3894 | LR: 4.91e-04 | 1335.4ms/step | 15.0 samples/s +Step 4540/44000 | Loss: 3.3494 | LR: 4.91e-04 | 1334.1ms/step | 15.0 samples/s +Step 4550/44000 | Loss: 3.3433 | LR: 4.91e-04 | 1338.3ms/step | 14.9 samples/s +Step 4560/44000 | Loss: 3.3509 | LR: 4.90e-04 | 1335.9ms/step | 15.0 samples/s +Step 4570/44000 | Loss: 3.3689 | LR: 4.90e-04 | 1338.9ms/step | 14.9 samples/s +Step 4580/44000 | Loss: 3.3277 | LR: 4.90e-04 | 1337.3ms/step | 15.0 samples/s +Step 4590/44000 | Loss: 3.3292 | LR: 4.90e-04 | 1334.1ms/step | 15.0 samples/s +Step 4600/44000 | Loss: 3.3499 | LR: 4.90e-04 | 1335.6ms/step | 15.0 samples/s +Step 4610/44000 | Loss: 3.3582 | LR: 4.90e-04 | 1337.1ms/step | 15.0 samples/s +Step 4620/44000 | Loss: 3.3477 | LR: 4.90e-04 | 1335.9ms/step | 15.0 samples/s +Step 4630/44000 | Loss: 3.3590 | LR: 4.90e-04 | 1335.9ms/step | 15.0 samples/s +Step 4640/44000 | Loss: 3.3697 | LR: 4.90e-04 | 1336.1ms/step | 15.0 samples/s +Step 4650/44000 | Loss: 3.3582 | LR: 4.90e-04 | 1337.1ms/step | 15.0 samples/s +Step 4660/44000 | Loss: 3.3377 | LR: 4.90e-04 | 1338.0ms/step | 14.9 samples/s +Step 4670/44000 | Loss: 3.3298 | LR: 4.90e-04 | 1339.8ms/step | 14.9 samples/s +Step 4680/44000 | Loss: 3.3530 | LR: 4.90e-04 | 1338.2ms/step | 14.9 samples/s +Step 4690/44000 | Loss: 3.3464 | LR: 4.90e-04 | 1336.2ms/step | 15.0 samples/s +Step 4700/44000 | Loss: 3.3817 | LR: 4.90e-04 | 1336.3ms/step | 15.0 samples/s +Step 4710/44000 | Loss: 3.3562 | LR: 4.90e-04 | 1334.8ms/step | 15.0 samples/s +Step 4720/44000 | Loss: 3.3351 | LR: 4.90e-04 | 1336.8ms/step | 15.0 samples/s +Step 4730/44000 | Loss: 3.3545 | LR: 4.90e-04 | 1336.5ms/step | 15.0 samples/s +Step 4740/44000 | Loss: 3.3452 | LR: 4.90e-04 | 1338.1ms/step | 14.9 samples/s +Step 4750/44000 | Loss: 3.3554 | LR: 4.90e-04 | 1337.4ms/step | 15.0 samples/s +Validation at step 4750: Loss = 3.3610, Perplexity = 28.82 +Step 4760/44000 | Loss: 3.3347 | LR: 4.89e-04 | 3579.4ms/step | 5.6 samples/s +Step 4770/44000 | Loss: 3.3595 | LR: 4.89e-04 | 1340.1ms/step | 14.9 samples/s +Step 4780/44000 | Loss: 3.3548 | LR: 4.89e-04 | 1339.0ms/step | 14.9 samples/s +Step 4790/44000 | Loss: 3.3294 | LR: 4.89e-04 | 1336.6ms/step | 15.0 samples/s +Step 4800/44000 | Loss: 3.3563 | LR: 4.89e-04 | 1333.5ms/step | 15.0 samples/s +Step 4810/44000 | Loss: 3.3531 | LR: 4.89e-04 | 1335.9ms/step | 15.0 samples/s +Step 4820/44000 | Loss: 3.3527 | LR: 4.89e-04 | 1335.9ms/step | 15.0 samples/s +Step 4830/44000 | Loss: 3.3404 | LR: 4.89e-04 | 1333.8ms/step | 15.0 samples/s +Step 4840/44000 | Loss: 3.3534 | LR: 4.89e-04 | 1333.0ms/step | 15.0 samples/s +Step 4850/44000 | Loss: 3.3547 | LR: 4.89e-04 | 1334.0ms/step | 15.0 samples/s +Step 4860/44000 | Loss: 3.3621 | LR: 4.89e-04 | 1333.6ms/step | 15.0 samples/s +Step 4870/44000 | Loss: 3.3264 | LR: 4.89e-04 | 1335.8ms/step | 15.0 samples/s +Step 4880/44000 | Loss: 3.3258 | LR: 4.89e-04 | 1335.6ms/step | 15.0 samples/s +Step 4890/44000 | Loss: 3.3169 | LR: 4.89e-04 | 1334.8ms/step | 15.0 samples/s +Step 4900/44000 | Loss: 3.3005 | LR: 4.89e-04 | 1339.6ms/step | 14.9 samples/s +Step 4910/44000 | Loss: 3.3373 | LR: 4.89e-04 | 1337.6ms/step | 15.0 samples/s +Step 4920/44000 | Loss: 3.3376 | LR: 4.89e-04 | 1336.9ms/step | 15.0 samples/s +Step 4930/44000 | Loss: 3.3222 | LR: 4.89e-04 | 1336.7ms/step | 15.0 samples/s +Step 4940/44000 | Loss: 3.3309 | LR: 4.89e-04 | 1335.4ms/step | 15.0 samples/s +Step 4950/44000 | Loss: 3.3331 | LR: 4.88e-04 | 1333.6ms/step | 15.0 samples/s +Step 4960/44000 | Loss: 3.3219 | LR: 4.88e-04 | 1338.7ms/step | 14.9 samples/s +Step 4970/44000 | Loss: 3.3136 | LR: 4.88e-04 | 1335.7ms/step | 15.0 samples/s +Step 4980/44000 | Loss: 3.3119 | LR: 4.88e-04 | 1336.3ms/step | 15.0 samples/s +Step 4990/44000 | Loss: 3.3380 | LR: 4.88e-04 | 1336.1ms/step | 15.0 samples/s +Step 5000/44000 | Loss: 3.3355 | LR: 4.88e-04 | 1337.2ms/step | 15.0 samples/s +Validation at step 5000: Loss = 3.3452, Perplexity = 28.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-5000 +Step 5010/44000 | Loss: 3.3442 | LR: 4.88e-04 | 3875.2ms/step | 5.2 samples/s +Step 5020/44000 | Loss: 3.3232 | LR: 4.88e-04 | 1333.5ms/step | 15.0 samples/s +Step 5030/44000 | Loss: 3.3305 | LR: 4.88e-04 | 1333.8ms/step | 15.0 samples/s +Step 5040/44000 | Loss: 3.3395 | LR: 4.88e-04 | 1337.6ms/step | 15.0 samples/s +Step 5050/44000 | Loss: 3.3148 | LR: 4.88e-04 | 1336.3ms/step | 15.0 samples/s +Step 5060/44000 | Loss: 3.3446 | LR: 4.88e-04 | 1337.5ms/step | 15.0 samples/s +Step 5070/44000 | Loss: 3.3493 | LR: 4.88e-04 | 1336.8ms/step | 15.0 samples/s +Step 5080/44000 | Loss: 3.3403 | LR: 4.88e-04 | 1338.4ms/step | 14.9 samples/s +Step 5090/44000 | Loss: 3.3123 | LR: 4.88e-04 | 1339.7ms/step | 14.9 samples/s +Step 5100/44000 | Loss: 3.3406 | LR: 4.88e-04 | 1340.1ms/step | 14.9 samples/s +Step 5110/44000 | Loss: 3.3570 | LR: 4.88e-04 | 1338.0ms/step | 14.9 samples/s +Step 5120/44000 | Loss: 3.3559 | LR: 4.88e-04 | 1337.3ms/step | 15.0 samples/s +Step 5130/44000 | Loss: 3.3368 | LR: 4.87e-04 | 1336.1ms/step | 15.0 samples/s +Step 5140/44000 | Loss: 3.3132 | LR: 4.87e-04 | 1336.8ms/step | 15.0 samples/s +Step 5150/44000 | Loss: 3.3571 | LR: 4.87e-04 | 1337.7ms/step | 15.0 samples/s +Step 5160/44000 | Loss: 3.3542 | LR: 4.87e-04 | 1336.0ms/step | 15.0 samples/s +Step 5170/44000 | Loss: 3.3438 | LR: 4.87e-04 | 1338.0ms/step | 14.9 samples/s +Step 5180/44000 | Loss: 3.3017 | LR: 4.87e-04 | 1339.3ms/step | 14.9 samples/s +Step 5190/44000 | Loss: 3.3297 | LR: 4.87e-04 | 1340.9ms/step | 14.9 samples/s +Step 5200/44000 | Loss: 3.2976 | LR: 4.87e-04 | 1338.9ms/step | 14.9 samples/s +Step 5210/44000 | Loss: 3.3330 | LR: 4.87e-04 | 1340.0ms/step | 14.9 samples/s +Step 5220/44000 | Loss: 3.3264 | LR: 4.87e-04 | 1336.9ms/step | 15.0 samples/s +Step 5230/44000 | Loss: 3.2807 | LR: 4.87e-04 | 1338.7ms/step | 14.9 samples/s +Step 5240/44000 | Loss: 3.3108 | LR: 4.87e-04 | 1338.3ms/step | 14.9 samples/s +Step 5250/44000 | Loss: 3.3084 | LR: 4.87e-04 | 1339.7ms/step | 14.9 samples/s +Validation at step 5250: Loss = 3.3293, Perplexity = 27.92 +Step 5260/44000 | Loss: 3.3329 | LR: 4.87e-04 | 3582.2ms/step | 5.6 samples/s +Step 5270/44000 | Loss: 3.2996 | LR: 4.87e-04 | 1340.9ms/step | 14.9 samples/s +Step 5280/44000 | Loss: 3.3281 | LR: 4.87e-04 | 1339.7ms/step | 14.9 samples/s +Step 5290/44000 | Loss: 3.3134 | LR: 4.87e-04 | 1336.2ms/step | 15.0 samples/s +Step 5300/44000 | Loss: 3.2656 | LR: 4.86e-04 | 1338.8ms/step | 14.9 samples/s +Step 5310/44000 | Loss: 3.3088 | LR: 4.86e-04 | 1338.6ms/step | 14.9 samples/s +Step 5320/44000 | Loss: 3.3203 | LR: 4.86e-04 | 1338.3ms/step | 14.9 samples/s +Step 5330/44000 | Loss: 3.3153 | LR: 4.86e-04 | 1338.0ms/step | 14.9 samples/s +Step 5340/44000 | Loss: 3.3273 | LR: 4.86e-04 | 1337.4ms/step | 15.0 samples/s +Step 5350/44000 | Loss: 3.2941 | LR: 4.86e-04 | 1337.1ms/step | 15.0 samples/s +Step 5360/44000 | Loss: 3.3188 | LR: 4.86e-04 | 1338.0ms/step | 14.9 samples/s +Step 5370/44000 | Loss: 3.3178 | LR: 4.86e-04 | 1335.5ms/step | 15.0 samples/s +Step 5380/44000 | Loss: 3.3045 | LR: 4.86e-04 | 1335.5ms/step | 15.0 samples/s +Step 5390/44000 | Loss: 3.3285 | LR: 4.86e-04 | 1333.3ms/step | 15.0 samples/s +Step 5400/44000 | Loss: 3.3334 | LR: 4.86e-04 | 1335.2ms/step | 15.0 samples/s +Step 5410/44000 | Loss: 3.3190 | LR: 4.86e-04 | 1335.1ms/step | 15.0 samples/s +Step 5420/44000 | Loss: 3.3157 | LR: 4.86e-04 | 1340.9ms/step | 14.9 samples/s +Step 5430/44000 | Loss: 3.2995 | LR: 4.86e-04 | 1339.2ms/step | 14.9 samples/s +Step 5440/44000 | Loss: 3.3097 | LR: 4.86e-04 | 1338.6ms/step | 14.9 samples/s +Step 5450/44000 | Loss: 3.3219 | LR: 4.86e-04 | 1339.3ms/step | 14.9 samples/s +Step 5460/44000 | Loss: 3.3380 | LR: 4.86e-04 | 1339.2ms/step | 14.9 samples/s +Step 5470/44000 | Loss: 3.3171 | LR: 4.85e-04 | 1336.3ms/step | 15.0 samples/s +Step 5480/44000 | Loss: 3.2994 | LR: 4.85e-04 | 1337.1ms/step | 15.0 samples/s +Step 5490/44000 | Loss: 3.2864 | LR: 4.85e-04 | 1334.9ms/step | 15.0 samples/s +Step 5500/44000 | Loss: 3.3101 | LR: 4.85e-04 | 1333.2ms/step | 15.0 samples/s +Validation at step 5500: Loss = 3.3151, Perplexity = 27.52 +Step 5510/44000 | Loss: 3.3057 | LR: 4.85e-04 | 3564.3ms/step | 5.6 samples/s +Step 5520/44000 | Loss: 3.3083 | LR: 4.85e-04 | 1332.1ms/step | 15.0 samples/s +Step 5530/44000 | Loss: 3.2782 | LR: 4.85e-04 | 1332.0ms/step | 15.0 samples/s +Step 5540/44000 | Loss: 3.3129 | LR: 4.85e-04 | 1330.4ms/step | 15.0 samples/s +Step 5550/44000 | Loss: 3.2933 | LR: 4.85e-04 | 1328.1ms/step | 15.1 samples/s +Step 5560/44000 | Loss: 3.3042 | LR: 4.85e-04 | 1330.8ms/step | 15.0 samples/s +Step 5570/44000 | Loss: 3.2976 | LR: 4.85e-04 | 1328.2ms/step | 15.1 samples/s +Step 5580/44000 | Loss: 3.2842 | LR: 4.85e-04 | 1327.5ms/step | 15.1 samples/s +Step 5590/44000 | Loss: 3.2801 | LR: 4.85e-04 | 1330.8ms/step | 15.0 samples/s +Step 5600/44000 | Loss: 3.2809 | LR: 4.85e-04 | 1328.4ms/step | 15.1 samples/s +Step 5610/44000 | Loss: 3.3116 | LR: 4.85e-04 | 1329.6ms/step | 15.0 samples/s +Step 5620/44000 | Loss: 3.2944 | LR: 4.85e-04 | 1330.6ms/step | 15.0 samples/s +Step 5630/44000 | Loss: 3.3140 | LR: 4.84e-04 | 1329.2ms/step | 15.0 samples/s +Step 5640/44000 | Loss: 3.2859 | LR: 4.84e-04 | 1330.3ms/step | 15.0 samples/s +Step 5650/44000 | Loss: 3.2837 | LR: 4.84e-04 | 1331.1ms/step | 15.0 samples/s +Step 5660/44000 | Loss: 3.2704 | LR: 4.84e-04 | 1331.2ms/step | 15.0 samples/s +Step 5670/44000 | Loss: 3.2888 | LR: 4.84e-04 | 1334.0ms/step | 15.0 samples/s +Step 5680/44000 | Loss: 3.2726 | LR: 4.84e-04 | 1331.6ms/step | 15.0 samples/s +Step 5690/44000 | Loss: 3.3195 | LR: 4.84e-04 | 1335.2ms/step | 15.0 samples/s +Step 5700/44000 | Loss: 3.3099 | LR: 4.84e-04 | 1337.0ms/step | 15.0 samples/s +Step 5710/44000 | Loss: 3.2571 | LR: 4.84e-04 | 1337.7ms/step | 15.0 samples/s +Step 5720/44000 | Loss: 3.2587 | LR: 4.84e-04 | 1336.3ms/step | 15.0 samples/s +Step 5730/44000 | Loss: 3.2773 | LR: 4.84e-04 | 1336.4ms/step | 15.0 samples/s +Step 5740/44000 | Loss: 3.2795 | LR: 4.84e-04 | 1333.0ms/step | 15.0 samples/s +Step 5750/44000 | Loss: 3.2832 | LR: 4.84e-04 | 1335.6ms/step | 15.0 samples/s +Validation at step 5750: Loss = 3.3022, Perplexity = 27.17 +Step 5760/44000 | Loss: 3.3058 | LR: 4.84e-04 | 3577.4ms/step | 5.6 samples/s +Step 5770/44000 | Loss: 3.3113 | LR: 4.84e-04 | 1337.6ms/step | 15.0 samples/s +Step 5780/44000 | Loss: 3.2882 | LR: 4.83e-04 | 1332.6ms/step | 15.0 samples/s +Step 5790/44000 | Loss: 3.2734 | LR: 4.83e-04 | 1333.7ms/step | 15.0 samples/s +Step 5800/44000 | Loss: 3.2985 | LR: 4.83e-04 | 1334.8ms/step | 15.0 samples/s +Step 5810/44000 | Loss: 3.3133 | LR: 4.83e-04 | 1337.2ms/step | 15.0 samples/s +Step 5820/44000 | Loss: 3.2941 | LR: 4.83e-04 | 1338.4ms/step | 14.9 samples/s +Step 5830/44000 | Loss: 3.2900 | LR: 4.83e-04 | 1336.2ms/step | 15.0 samples/s +Step 5840/44000 | Loss: 3.2837 | LR: 4.83e-04 | 1339.7ms/step | 14.9 samples/s +Step 5850/44000 | Loss: 3.2647 | LR: 4.83e-04 | 1337.2ms/step | 15.0 samples/s +Step 5860/44000 | Loss: 3.2938 | LR: 4.83e-04 | 1335.0ms/step | 15.0 samples/s +Step 5870/44000 | Loss: 3.3002 | LR: 4.83e-04 | 1337.1ms/step | 15.0 samples/s +Step 5880/44000 | Loss: 3.2800 | LR: 4.83e-04 | 1336.2ms/step | 15.0 samples/s +Step 5890/44000 | Loss: 3.2960 | LR: 4.83e-04 | 1337.3ms/step | 15.0 samples/s +Step 5900/44000 | Loss: 3.2835 | LR: 4.83e-04 | 1339.5ms/step | 14.9 samples/s +Step 5910/44000 | Loss: 3.2912 | LR: 4.83e-04 | 1336.2ms/step | 15.0 samples/s +Step 5920/44000 | Loss: 3.3007 | LR: 4.83e-04 | 1339.4ms/step | 14.9 samples/s +Step 5930/44000 | Loss: 3.2523 | LR: 4.83e-04 | 1338.4ms/step | 14.9 samples/s +Step 5940/44000 | Loss: 3.3017 | LR: 4.82e-04 | 1339.5ms/step | 14.9 samples/s +Step 5950/44000 | Loss: 3.2628 | LR: 4.82e-04 | 1338.1ms/step | 14.9 samples/s +Step 5960/44000 | Loss: 3.2534 | LR: 4.82e-04 | 1340.0ms/step | 14.9 samples/s +Step 5970/44000 | Loss: 3.2594 | LR: 4.82e-04 | 1339.9ms/step | 14.9 samples/s +Step 5980/44000 | Loss: 3.2550 | LR: 4.82e-04 | 1336.8ms/step | 15.0 samples/s +Step 5990/44000 | Loss: 3.2541 | LR: 4.82e-04 | 1336.1ms/step | 15.0 samples/s +Step 6000/44000 | Loss: 3.2805 | LR: 4.82e-04 | 1336.9ms/step | 15.0 samples/s +Validation at step 6000: Loss = 3.2894, Perplexity = 26.83 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-6000 +Step 6010/44000 | Loss: 3.3043 | LR: 4.82e-04 | 3883.1ms/step | 5.2 samples/s +Step 6020/44000 | Loss: 3.2644 | LR: 4.82e-04 | 1334.9ms/step | 15.0 samples/s +Step 6030/44000 | Loss: 3.2694 | LR: 4.82e-04 | 1334.7ms/step | 15.0 samples/s +Step 6040/44000 | Loss: 3.2731 | LR: 4.82e-04 | 1331.0ms/step | 15.0 samples/s +Step 6050/44000 | Loss: 3.2765 | LR: 4.82e-04 | 1333.0ms/step | 15.0 samples/s +Step 6060/44000 | Loss: 3.3003 | LR: 4.82e-04 | 1333.3ms/step | 15.0 samples/s +Step 6070/44000 | Loss: 3.2806 | LR: 4.82e-04 | 1331.7ms/step | 15.0 samples/s +Step 6080/44000 | Loss: 3.2734 | LR: 4.81e-04 | 1331.1ms/step | 15.0 samples/s +Step 6090/44000 | Loss: 3.3002 | LR: 4.81e-04 | 1331.6ms/step | 15.0 samples/s +Step 6100/44000 | Loss: 3.2549 | LR: 4.81e-04 | 1333.7ms/step | 15.0 samples/s +Step 6110/44000 | Loss: 3.2698 | LR: 4.81e-04 | 1330.0ms/step | 15.0 samples/s +Step 6120/44000 | Loss: 3.2510 | LR: 4.81e-04 | 1329.6ms/step | 15.0 samples/s +Step 6130/44000 | Loss: 3.2673 | LR: 4.81e-04 | 1329.2ms/step | 15.0 samples/s +Step 6140/44000 | Loss: 3.2542 | LR: 4.81e-04 | 1329.6ms/step | 15.0 samples/s +Step 6150/44000 | Loss: 3.2628 | LR: 4.81e-04 | 1330.3ms/step | 15.0 samples/s +Step 6160/44000 | Loss: 3.2597 | LR: 4.81e-04 | 1330.8ms/step | 15.0 samples/s +Step 6170/44000 | Loss: 3.2771 | LR: 4.81e-04 | 1336.0ms/step | 15.0 samples/s +Step 6180/44000 | Loss: 3.2733 | LR: 4.81e-04 | 1333.0ms/step | 15.0 samples/s +Step 6190/44000 | Loss: 3.2580 | LR: 4.81e-04 | 1336.0ms/step | 15.0 samples/s +Step 6200/44000 | Loss: 3.2731 | LR: 4.81e-04 | 1333.2ms/step | 15.0 samples/s +Step 6210/44000 | Loss: 3.2680 | LR: 4.81e-04 | 1333.9ms/step | 15.0 samples/s +Step 6220/44000 | Loss: 3.3012 | LR: 4.81e-04 | 1331.9ms/step | 15.0 samples/s +Step 6230/44000 | Loss: 3.2650 | LR: 4.80e-04 | 1333.1ms/step | 15.0 samples/s +Step 6240/44000 | Loss: 3.2867 | LR: 4.80e-04 | 1335.4ms/step | 15.0 samples/s +Step 6250/44000 | Loss: 3.2813 | LR: 4.80e-04 | 1332.7ms/step | 15.0 samples/s +Validation at step 6250: Loss = 3.2864, Perplexity = 26.75 +Step 6260/44000 | Loss: 3.2765 | LR: 4.80e-04 | 3572.7ms/step | 5.6 samples/s +Step 6270/44000 | Loss: 3.3016 | LR: 4.80e-04 | 1335.1ms/step | 15.0 samples/s +Step 6280/44000 | Loss: 3.2514 | LR: 4.80e-04 | 1336.3ms/step | 15.0 samples/s +Step 6290/44000 | Loss: 3.2821 | LR: 4.80e-04 | 1334.6ms/step | 15.0 samples/s +Step 6300/44000 | Loss: 3.2586 | LR: 4.80e-04 | 1334.0ms/step | 15.0 samples/s +Step 6310/44000 | Loss: 3.2623 | LR: 4.80e-04 | 1337.9ms/step | 14.9 samples/s +Step 6320/44000 | Loss: 3.2573 | LR: 4.80e-04 | 1336.2ms/step | 15.0 samples/s +Step 6330/44000 | Loss: 3.2647 | LR: 4.80e-04 | 1338.6ms/step | 14.9 samples/s +Step 6340/44000 | Loss: 3.2758 | LR: 4.80e-04 | 1338.6ms/step | 14.9 samples/s +Step 6350/44000 | Loss: 3.2354 | LR: 4.80e-04 | 1333.1ms/step | 15.0 samples/s +Step 6360/44000 | Loss: 3.2891 | LR: 4.80e-04 | 1328.9ms/step | 15.0 samples/s +Step 6370/44000 | Loss: 3.2594 | LR: 4.79e-04 | 1331.7ms/step | 15.0 samples/s +Step 6380/44000 | Loss: 3.2464 | LR: 4.79e-04 | 1328.1ms/step | 15.1 samples/s +Step 6390/44000 | Loss: 3.2386 | LR: 4.79e-04 | 1333.1ms/step | 15.0 samples/s +Step 6400/44000 | Loss: 3.2716 | LR: 4.79e-04 | 1332.6ms/step | 15.0 samples/s +Step 6410/44000 | Loss: 3.2411 | LR: 4.79e-04 | 1331.2ms/step | 15.0 samples/s +Step 6420/44000 | Loss: 3.2715 | LR: 4.79e-04 | 1332.6ms/step | 15.0 samples/s +Step 6430/44000 | Loss: 3.2360 | LR: 4.79e-04 | 1332.6ms/step | 15.0 samples/s +Step 6440/44000 | Loss: 3.2518 | LR: 4.79e-04 | 1331.4ms/step | 15.0 samples/s +Step 6450/44000 | Loss: 3.2490 | LR: 4.79e-04 | 1337.4ms/step | 15.0 samples/s +Step 6460/44000 | Loss: 3.2576 | LR: 4.79e-04 | 1334.8ms/step | 15.0 samples/s +Step 6470/44000 | Loss: 3.2367 | LR: 4.79e-04 | 1336.4ms/step | 15.0 samples/s +Step 6480/44000 | Loss: 3.2516 | LR: 4.79e-04 | 1333.7ms/step | 15.0 samples/s +Step 6490/44000 | Loss: 3.2442 | LR: 4.79e-04 | 1331.5ms/step | 15.0 samples/s +Step 6500/44000 | Loss: 3.2429 | LR: 4.79e-04 | 1329.6ms/step | 15.0 samples/s +Validation at step 6500: Loss = 3.2698, Perplexity = 26.31 +Step 6510/44000 | Loss: 3.2677 | LR: 4.78e-04 | 3574.7ms/step | 5.6 samples/s +Step 6520/44000 | Loss: 3.2736 | LR: 4.78e-04 | 1332.7ms/step | 15.0 samples/s +Step 6530/44000 | Loss: 3.2689 | LR: 4.78e-04 | 1334.2ms/step | 15.0 samples/s +Step 6540/44000 | Loss: 3.2663 | LR: 4.78e-04 | 1336.5ms/step | 15.0 samples/s +Step 6550/44000 | Loss: 3.2670 | LR: 4.78e-04 | 1339.5ms/step | 14.9 samples/s +Step 6560/44000 | Loss: 3.2480 | LR: 4.78e-04 | 1336.3ms/step | 15.0 samples/s +Step 6570/44000 | Loss: 3.2489 | LR: 4.78e-04 | 1333.0ms/step | 15.0 samples/s +Step 6580/44000 | Loss: 3.2889 | LR: 4.78e-04 | 1334.4ms/step | 15.0 samples/s +Step 6590/44000 | Loss: 3.2565 | LR: 4.78e-04 | 1333.1ms/step | 15.0 samples/s +Step 6600/44000 | Loss: 3.2447 | LR: 4.78e-04 | 1329.9ms/step | 15.0 samples/s +Step 6610/44000 | Loss: 3.2102 | LR: 4.78e-04 | 1331.5ms/step | 15.0 samples/s +Step 6620/44000 | Loss: 3.2424 | LR: 4.78e-04 | 1332.1ms/step | 15.0 samples/s +Step 6630/44000 | Loss: 3.2449 | LR: 4.78e-04 | 1333.0ms/step | 15.0 samples/s +Step 6640/44000 | Loss: 3.2358 | LR: 4.77e-04 | 1332.0ms/step | 15.0 samples/s +Step 6650/44000 | Loss: 3.2613 | LR: 4.77e-04 | 1329.2ms/step | 15.0 samples/s +Step 6660/44000 | Loss: 3.2866 | LR: 4.77e-04 | 1328.8ms/step | 15.1 samples/s +Step 6670/44000 | Loss: 3.2344 | LR: 4.77e-04 | 1329.2ms/step | 15.0 samples/s +Step 6680/44000 | Loss: 3.2581 | LR: 4.77e-04 | 1329.9ms/step | 15.0 samples/s +Step 6690/44000 | Loss: 3.2589 | LR: 4.77e-04 | 1331.5ms/step | 15.0 samples/s +Step 6700/44000 | Loss: 3.2601 | LR: 4.77e-04 | 1332.6ms/step | 15.0 samples/s +Step 6710/44000 | Loss: 3.2582 | LR: 4.77e-04 | 1332.2ms/step | 15.0 samples/s +Step 6720/44000 | Loss: 3.2240 | LR: 4.77e-04 | 1331.6ms/step | 15.0 samples/s +Step 6730/44000 | Loss: 3.2564 | LR: 4.77e-04 | 1331.1ms/step | 15.0 samples/s +Step 6740/44000 | Loss: 3.2330 | LR: 4.77e-04 | 1330.8ms/step | 15.0 samples/s +Step 6750/44000 | Loss: 3.2392 | LR: 4.77e-04 | 1331.6ms/step | 15.0 samples/s +Validation at step 6750: Loss = 3.2596, Perplexity = 26.04 +Step 6760/44000 | Loss: 3.2648 | LR: 4.77e-04 | 3560.9ms/step | 5.6 samples/s +Step 6770/44000 | Loss: 3.2619 | LR: 4.76e-04 | 1331.8ms/step | 15.0 samples/s +Step 6780/44000 | Loss: 3.2579 | LR: 4.76e-04 | 1332.8ms/step | 15.0 samples/s +Step 6790/44000 | Loss: 3.2511 | LR: 4.76e-04 | 1330.4ms/step | 15.0 samples/s +Step 6800/44000 | Loss: 3.2038 | LR: 4.76e-04 | 1334.1ms/step | 15.0 samples/s +Step 6810/44000 | Loss: 3.2655 | LR: 4.76e-04 | 1332.1ms/step | 15.0 samples/s +Step 6820/44000 | Loss: 3.2252 | LR: 4.76e-04 | 1330.2ms/step | 15.0 samples/s +Step 6830/44000 | Loss: 3.2469 | LR: 4.76e-04 | 1331.0ms/step | 15.0 samples/s +Step 6840/44000 | Loss: 3.2516 | LR: 4.76e-04 | 1332.1ms/step | 15.0 samples/s +Step 6850/44000 | Loss: 3.2526 | LR: 4.76e-04 | 1332.1ms/step | 15.0 samples/s +Step 6860/44000 | Loss: 3.2508 | LR: 4.76e-04 | 1330.3ms/step | 15.0 samples/s +Step 6870/44000 | Loss: 3.2278 | LR: 4.76e-04 | 1330.6ms/step | 15.0 samples/s +Step 6880/44000 | Loss: 3.2007 | LR: 4.76e-04 | 1331.4ms/step | 15.0 samples/s +Step 6890/44000 | Loss: 3.2558 | LR: 4.76e-04 | 1333.8ms/step | 15.0 samples/s +Step 6900/44000 | Loss: 3.2679 | LR: 4.75e-04 | 1333.6ms/step | 15.0 samples/s +Step 6910/44000 | Loss: 3.2301 | LR: 4.75e-04 | 1331.1ms/step | 15.0 samples/s +Step 6920/44000 | Loss: 3.2496 | LR: 4.75e-04 | 1331.0ms/step | 15.0 samples/s +Step 6930/44000 | Loss: 3.2321 | LR: 4.75e-04 | 1333.6ms/step | 15.0 samples/s +Step 6940/44000 | Loss: 3.2585 | LR: 4.75e-04 | 1336.7ms/step | 15.0 samples/s +Step 6950/44000 | Loss: 3.2566 | LR: 4.75e-04 | 1333.7ms/step | 15.0 samples/s +Step 6960/44000 | Loss: 3.2300 | LR: 4.75e-04 | 1335.0ms/step | 15.0 samples/s +Step 6970/44000 | Loss: 3.2521 | LR: 4.75e-04 | 1335.8ms/step | 15.0 samples/s +Step 6980/44000 | Loss: 3.2198 | LR: 4.75e-04 | 1331.9ms/step | 15.0 samples/s +Step 6990/44000 | Loss: 3.2275 | LR: 4.75e-04 | 1334.4ms/step | 15.0 samples/s +Step 7000/44000 | Loss: 3.2339 | LR: 4.75e-04 | 1331.0ms/step | 15.0 samples/s +Validation at step 7000: Loss = 3.2508, Perplexity = 25.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-7000 +Step 7010/44000 | Loss: 3.2261 | LR: 4.75e-04 | 3870.7ms/step | 5.2 samples/s +Step 7020/44000 | Loss: 3.2282 | LR: 4.75e-04 | 1333.9ms/step | 15.0 samples/s +Step 7030/44000 | Loss: 3.2497 | LR: 4.74e-04 | 1332.2ms/step | 15.0 samples/s +Step 7040/44000 | Loss: 3.2420 | LR: 4.74e-04 | 1332.5ms/step | 15.0 samples/s +Step 7050/44000 | Loss: 3.2664 | LR: 4.74e-04 | 1332.7ms/step | 15.0 samples/s +Step 7060/44000 | Loss: 3.2480 | LR: 4.74e-04 | 1334.0ms/step | 15.0 samples/s +Step 7070/44000 | Loss: 3.2346 | LR: 4.74e-04 | 1332.0ms/step | 15.0 samples/s +Step 7080/44000 | Loss: 3.2262 | LR: 4.74e-04 | 1334.2ms/step | 15.0 samples/s +Step 7090/44000 | Loss: 3.2053 | LR: 4.74e-04 | 1334.0ms/step | 15.0 samples/s +Step 7100/44000 | Loss: 3.2323 | LR: 4.74e-04 | 1335.0ms/step | 15.0 samples/s +Step 7110/44000 | Loss: 3.2064 | LR: 4.74e-04 | 1333.5ms/step | 15.0 samples/s +Step 7120/44000 | Loss: 3.2271 | LR: 4.74e-04 | 1333.7ms/step | 15.0 samples/s +Step 7130/44000 | Loss: 3.2398 | LR: 4.74e-04 | 1330.3ms/step | 15.0 samples/s +Step 7140/44000 | Loss: 3.2593 | LR: 4.74e-04 | 1332.4ms/step | 15.0 samples/s +Step 7150/44000 | Loss: 3.2391 | LR: 4.73e-04 | 1331.8ms/step | 15.0 samples/s +Step 7160/44000 | Loss: 3.2496 | LR: 4.73e-04 | 1335.7ms/step | 15.0 samples/s +Step 7170/44000 | Loss: 3.2507 | LR: 4.73e-04 | 1334.3ms/step | 15.0 samples/s +Step 7180/44000 | Loss: 3.2342 | LR: 4.73e-04 | 1332.9ms/step | 15.0 samples/s +Step 7190/44000 | Loss: 3.1951 | LR: 4.73e-04 | 1332.7ms/step | 15.0 samples/s +Step 7200/44000 | Loss: 3.2484 | LR: 4.73e-04 | 1333.8ms/step | 15.0 samples/s +Step 7210/44000 | Loss: 3.2687 | LR: 4.73e-04 | 1335.5ms/step | 15.0 samples/s +Step 7220/44000 | Loss: 3.2515 | LR: 4.73e-04 | 1332.9ms/step | 15.0 samples/s +Step 7230/44000 | Loss: 3.2474 | LR: 4.73e-04 | 1335.3ms/step | 15.0 samples/s +Step 7240/44000 | Loss: 3.2388 | LR: 4.73e-04 | 1330.6ms/step | 15.0 samples/s +Step 7250/44000 | Loss: 3.2410 | LR: 4.73e-04 | 1332.1ms/step | 15.0 samples/s +Validation at step 7250: Loss = 3.2402, Perplexity = 25.54 +Step 7260/44000 | Loss: 3.2230 | LR: 4.73e-04 | 3576.8ms/step | 5.6 samples/s +Step 7270/44000 | Loss: 3.2326 | LR: 4.72e-04 | 1331.3ms/step | 15.0 samples/s +Step 7280/44000 | Loss: 3.2126 | LR: 4.72e-04 | 1329.6ms/step | 15.0 samples/s +Step 7290/44000 | Loss: 3.2538 | LR: 4.72e-04 | 1332.0ms/step | 15.0 samples/s +Step 7300/44000 | Loss: 3.2368 | LR: 4.72e-04 | 1385.4ms/step | 14.4 samples/s +Step 7310/44000 | Loss: 3.2521 | LR: 4.72e-04 | 1349.7ms/step | 14.8 samples/s +Step 7320/44000 | Loss: 3.2530 | LR: 4.72e-04 | 1352.8ms/step | 14.8 samples/s +Step 7330/44000 | Loss: 3.2466 | LR: 4.72e-04 | 1340.9ms/step | 14.9 samples/s +Step 7340/44000 | Loss: 3.2558 | LR: 4.72e-04 | 1336.0ms/step | 15.0 samples/s +Step 7350/44000 | Loss: 3.2033 | LR: 4.72e-04 | 1330.8ms/step | 15.0 samples/s +Step 7360/44000 | Loss: 3.2116 | LR: 4.72e-04 | 1337.9ms/step | 14.9 samples/s +Step 7370/44000 | Loss: 3.2019 | LR: 4.72e-04 | 1335.5ms/step | 15.0 samples/s +Step 7380/44000 | Loss: 3.2318 | LR: 4.72e-04 | 1333.0ms/step | 15.0 samples/s +Step 7390/44000 | Loss: 3.2372 | LR: 4.71e-04 | 1332.5ms/step | 15.0 samples/s +Step 7400/44000 | Loss: 3.2287 | LR: 4.71e-04 | 1331.3ms/step | 15.0 samples/s +Step 7410/44000 | Loss: 3.2513 | LR: 4.71e-04 | 1331.9ms/step | 15.0 samples/s +Step 7420/44000 | Loss: 3.2549 | LR: 4.71e-04 | 1332.6ms/step | 15.0 samples/s +Step 7430/44000 | Loss: 3.2338 | LR: 4.71e-04 | 1331.5ms/step | 15.0 samples/s +Step 7440/44000 | Loss: 3.2353 | LR: 4.71e-04 | 1331.9ms/step | 15.0 samples/s +Step 7450/44000 | Loss: 3.2495 | LR: 4.71e-04 | 1329.3ms/step | 15.0 samples/s +Step 7460/44000 | Loss: 3.1946 | LR: 4.71e-04 | 1334.4ms/step | 15.0 samples/s +Step 7470/44000 | Loss: 3.2367 | LR: 4.71e-04 | 1333.1ms/step | 15.0 samples/s +Step 7480/44000 | Loss: 3.1912 | LR: 4.71e-04 | 1331.2ms/step | 15.0 samples/s +Step 7490/44000 | Loss: 3.2312 | LR: 4.71e-04 | 1331.5ms/step | 15.0 samples/s +Step 7500/44000 | Loss: 3.1958 | LR: 4.71e-04 | 1330.6ms/step | 15.0 samples/s +Validation at step 7500: Loss = 3.2330, Perplexity = 25.36 +Step 7510/44000 | Loss: 3.2083 | LR: 4.70e-04 | 3569.0ms/step | 5.6 samples/s +Step 7520/44000 | Loss: 3.2226 | LR: 4.70e-04 | 1335.1ms/step | 15.0 samples/s +Step 7530/44000 | Loss: 3.2521 | LR: 4.70e-04 | 1332.3ms/step | 15.0 samples/s +Step 7540/44000 | Loss: 3.2087 | LR: 4.70e-04 | 1332.0ms/step | 15.0 samples/s +Step 7550/44000 | Loss: 3.2144 | LR: 4.70e-04 | 1331.6ms/step | 15.0 samples/s +Step 7560/44000 | Loss: 3.2162 | LR: 4.70e-04 | 1330.4ms/step | 15.0 samples/s +Step 7570/44000 | Loss: 3.2457 | LR: 4.70e-04 | 1331.0ms/step | 15.0 samples/s +Step 7580/44000 | Loss: 3.2026 | LR: 4.70e-04 | 1332.8ms/step | 15.0 samples/s +Step 7590/44000 | Loss: 3.2457 | LR: 4.70e-04 | 1332.8ms/step | 15.0 samples/s +Step 7600/44000 | Loss: 3.2271 | LR: 4.70e-04 | 1332.0ms/step | 15.0 samples/s +Step 7610/44000 | Loss: 3.2447 | LR: 4.70e-04 | 1331.1ms/step | 15.0 samples/s +Step 7620/44000 | Loss: 3.1964 | LR: 4.70e-04 | 1334.0ms/step | 15.0 samples/s +Step 7630/44000 | Loss: 3.2524 | LR: 4.69e-04 | 1328.6ms/step | 15.1 samples/s +Step 7640/44000 | Loss: 3.2108 | LR: 4.69e-04 | 1332.0ms/step | 15.0 samples/s +Step 7650/44000 | Loss: 3.2222 | LR: 4.69e-04 | 1330.1ms/step | 15.0 samples/s +Step 7660/44000 | Loss: 3.2324 | LR: 4.69e-04 | 1328.4ms/step | 15.1 samples/s +Step 7670/44000 | Loss: 3.2126 | LR: 4.69e-04 | 1330.9ms/step | 15.0 samples/s +Step 7680/44000 | Loss: 3.2331 | LR: 4.69e-04 | 1332.2ms/step | 15.0 samples/s +Step 7690/44000 | Loss: 3.1941 | LR: 4.69e-04 | 1331.8ms/step | 15.0 samples/s +Step 7700/44000 | Loss: 3.2192 | LR: 4.69e-04 | 1335.1ms/step | 15.0 samples/s +Step 7710/44000 | Loss: 3.2435 | LR: 4.69e-04 | 1331.0ms/step | 15.0 samples/s +Step 7720/44000 | Loss: 3.2422 | LR: 4.69e-04 | 1331.5ms/step | 15.0 samples/s +Step 7730/44000 | Loss: 3.2367 | LR: 4.69e-04 | 1332.9ms/step | 15.0 samples/s +Step 7740/44000 | Loss: 3.2071 | LR: 4.68e-04 | 1332.2ms/step | 15.0 samples/s +Step 7750/44000 | Loss: 3.1849 | LR: 4.68e-04 | 1330.4ms/step | 15.0 samples/s +Validation at step 7750: Loss = 3.2230, Perplexity = 25.10 +Step 7760/44000 | Loss: 3.1962 | LR: 4.68e-04 | 3568.9ms/step | 5.6 samples/s +Step 7770/44000 | Loss: 3.1985 | LR: 4.68e-04 | 1332.4ms/step | 15.0 samples/s +Step 7780/44000 | Loss: 3.1941 | LR: 4.68e-04 | 1332.6ms/step | 15.0 samples/s +Step 7790/44000 | Loss: 3.2188 | LR: 4.68e-04 | 1331.8ms/step | 15.0 samples/s +Step 7800/44000 | Loss: 3.2081 | LR: 4.68e-04 | 1332.0ms/step | 15.0 samples/s +Step 7810/44000 | Loss: 3.1967 | LR: 4.68e-04 | 1333.7ms/step | 15.0 samples/s +Step 7820/44000 | Loss: 3.2130 | LR: 4.68e-04 | 1328.2ms/step | 15.1 samples/s +Step 7830/44000 | Loss: 3.2276 | LR: 4.68e-04 | 1329.6ms/step | 15.0 samples/s +Step 7840/44000 | Loss: 3.2059 | LR: 4.68e-04 | 1333.3ms/step | 15.0 samples/s +Step 7850/44000 | Loss: 3.1850 | LR: 4.67e-04 | 1327.9ms/step | 15.1 samples/s +Step 7860/44000 | Loss: 3.2047 | LR: 4.67e-04 | 1328.8ms/step | 15.1 samples/s +Step 7870/44000 | Loss: 3.2082 | LR: 4.67e-04 | 1328.3ms/step | 15.1 samples/s +Step 7880/44000 | Loss: 3.1681 | LR: 4.67e-04 | 1332.7ms/step | 15.0 samples/s +Step 7890/44000 | Loss: 3.1874 | LR: 4.67e-04 | 1332.4ms/step | 15.0 samples/s +Step 7900/44000 | Loss: 3.2350 | LR: 4.67e-04 | 1333.4ms/step | 15.0 samples/s +Step 7910/44000 | Loss: 3.1834 | LR: 4.67e-04 | 1333.4ms/step | 15.0 samples/s +Step 7920/44000 | Loss: 3.2201 | LR: 4.67e-04 | 1331.7ms/step | 15.0 samples/s +Step 7930/44000 | Loss: 3.1984 | LR: 4.67e-04 | 1330.1ms/step | 15.0 samples/s +Step 7940/44000 | Loss: 3.2150 | LR: 4.67e-04 | 1331.5ms/step | 15.0 samples/s +Step 7950/44000 | Loss: 3.2082 | LR: 4.67e-04 | 1333.3ms/step | 15.0 samples/s +Step 7960/44000 | Loss: 3.2020 | LR: 4.66e-04 | 1336.0ms/step | 15.0 samples/s +Step 7970/44000 | Loss: 3.1967 | LR: 4.66e-04 | 1333.2ms/step | 15.0 samples/s +Step 7980/44000 | Loss: 3.2241 | LR: 4.66e-04 | 1330.9ms/step | 15.0 samples/s +Step 7990/44000 | Loss: 3.2203 | LR: 4.66e-04 | 1331.9ms/step | 15.0 samples/s +Step 8000/44000 | Loss: 3.1923 | LR: 4.66e-04 | 1331.7ms/step | 15.0 samples/s +Validation at step 8000: Loss = 3.2148, Perplexity = 24.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-8000 +Step 8010/44000 | Loss: 3.1613 | LR: 4.66e-04 | 3885.1ms/step | 5.1 samples/s +Step 8020/44000 | Loss: 3.1834 | LR: 4.66e-04 | 1336.7ms/step | 15.0 samples/s +Step 8030/44000 | Loss: 3.1842 | LR: 4.66e-04 | 1335.9ms/step | 15.0 samples/s +Step 8040/44000 | Loss: 3.2091 | LR: 4.66e-04 | 1341.8ms/step | 14.9 samples/s +Step 8050/44000 | Loss: 3.1920 | LR: 4.66e-04 | 1338.2ms/step | 14.9 samples/s +Step 8060/44000 | Loss: 3.2256 | LR: 4.66e-04 | 1339.3ms/step | 14.9 samples/s +Step 8070/44000 | Loss: 3.1905 | LR: 4.65e-04 | 1340.8ms/step | 14.9 samples/s +Step 8080/44000 | Loss: 3.1811 | LR: 4.65e-04 | 1336.8ms/step | 15.0 samples/s +Step 8090/44000 | Loss: 3.1821 | LR: 4.65e-04 | 1339.5ms/step | 14.9 samples/s +Step 8100/44000 | Loss: 3.2035 | LR: 4.65e-04 | 1341.6ms/step | 14.9 samples/s +Step 8110/44000 | Loss: 3.1999 | LR: 4.65e-04 | 1336.4ms/step | 15.0 samples/s +Step 8120/44000 | Loss: 3.1925 | LR: 4.65e-04 | 1339.1ms/step | 14.9 samples/s +Step 8130/44000 | Loss: 3.1940 | LR: 4.65e-04 | 1337.0ms/step | 15.0 samples/s +Step 8140/44000 | Loss: 3.2283 | LR: 4.65e-04 | 1338.7ms/step | 14.9 samples/s +Step 8150/44000 | Loss: 3.1912 | LR: 4.65e-04 | 1334.7ms/step | 15.0 samples/s +Step 8160/44000 | Loss: 3.1821 | LR: 4.65e-04 | 1336.8ms/step | 15.0 samples/s +Step 8170/44000 | Loss: 3.2079 | LR: 4.65e-04 | 1338.1ms/step | 14.9 samples/s +Step 8180/44000 | Loss: 3.1966 | LR: 4.64e-04 | 1338.0ms/step | 14.9 samples/s +Step 8190/44000 | Loss: 3.1894 | LR: 4.64e-04 | 1339.0ms/step | 14.9 samples/s +Step 8200/44000 | Loss: 3.2153 | LR: 4.64e-04 | 1341.1ms/step | 14.9 samples/s +Step 8210/44000 | Loss: 3.2141 | LR: 4.64e-04 | 1338.2ms/step | 14.9 samples/s +Step 8220/44000 | Loss: 3.2216 | LR: 4.64e-04 | 1341.7ms/step | 14.9 samples/s +Step 8230/44000 | Loss: 3.1820 | LR: 4.64e-04 | 1339.3ms/step | 14.9 samples/s +Step 8240/44000 | Loss: 3.1949 | LR: 4.64e-04 | 1338.1ms/step | 14.9 samples/s +Step 8250/44000 | Loss: 3.1907 | LR: 4.64e-04 | 1338.3ms/step | 14.9 samples/s +Validation at step 8250: Loss = 3.2075, Perplexity = 24.72 +Step 8260/44000 | Loss: 3.1766 | LR: 4.64e-04 | 3587.2ms/step | 5.6 samples/s +Step 8270/44000 | Loss: 3.2205 | LR: 4.64e-04 | 1336.6ms/step | 15.0 samples/s +Step 8280/44000 | Loss: 3.1980 | LR: 4.64e-04 | 1336.1ms/step | 15.0 samples/s +Step 8290/44000 | Loss: 3.2005 | LR: 4.63e-04 | 1339.1ms/step | 14.9 samples/s +Step 8300/44000 | Loss: 3.1567 | LR: 4.63e-04 | 1340.0ms/step | 14.9 samples/s +Step 8310/44000 | Loss: 3.2074 | LR: 4.63e-04 | 1339.3ms/step | 14.9 samples/s +Step 8320/44000 | Loss: 3.1833 | LR: 4.63e-04 | 1339.3ms/step | 14.9 samples/s +Step 8330/44000 | Loss: 3.1895 | LR: 4.63e-04 | 1335.3ms/step | 15.0 samples/s +Step 8340/44000 | Loss: 3.2384 | LR: 4.63e-04 | 1334.7ms/step | 15.0 samples/s +Step 8350/44000 | Loss: 3.1854 | LR: 4.63e-04 | 1336.8ms/step | 15.0 samples/s +Step 8360/44000 | Loss: 3.2087 | LR: 4.63e-04 | 1337.5ms/step | 15.0 samples/s +Step 8370/44000 | Loss: 3.1932 | LR: 4.63e-04 | 1333.9ms/step | 15.0 samples/s +Step 8380/44000 | Loss: 3.2307 | LR: 4.63e-04 | 1335.8ms/step | 15.0 samples/s +Step 8390/44000 | Loss: 3.1661 | LR: 4.62e-04 | 1336.4ms/step | 15.0 samples/s +Step 8400/44000 | Loss: 3.1828 | LR: 4.62e-04 | 1335.9ms/step | 15.0 samples/s +Step 8410/44000 | Loss: 3.1806 | LR: 4.62e-04 | 1336.6ms/step | 15.0 samples/s +Step 8420/44000 | Loss: 3.2208 | LR: 4.62e-04 | 1337.3ms/step | 15.0 samples/s +Step 8430/44000 | Loss: 3.1856 | LR: 4.62e-04 | 1337.3ms/step | 15.0 samples/s +Step 8440/44000 | Loss: 3.2114 | LR: 4.62e-04 | 1335.1ms/step | 15.0 samples/s +Step 8450/44000 | Loss: 3.1631 | LR: 4.62e-04 | 1335.5ms/step | 15.0 samples/s +Step 8460/44000 | Loss: 3.2116 | LR: 4.62e-04 | 1337.4ms/step | 15.0 samples/s +Step 8470/44000 | Loss: 3.1596 | LR: 4.62e-04 | 1335.8ms/step | 15.0 samples/s +Step 8480/44000 | Loss: 3.1854 | LR: 4.62e-04 | 1334.4ms/step | 15.0 samples/s +Step 8490/44000 | Loss: 3.2060 | LR: 4.62e-04 | 1339.5ms/step | 14.9 samples/s +Step 8500/44000 | Loss: 3.2129 | LR: 4.61e-04 | 1335.3ms/step | 15.0 samples/s +Validation at step 8500: Loss = 3.2011, Perplexity = 24.56 +Step 8510/44000 | Loss: 3.2105 | LR: 4.61e-04 | 3579.3ms/step | 5.6 samples/s +Step 8520/44000 | Loss: 3.1820 | LR: 4.61e-04 | 1339.1ms/step | 14.9 samples/s +Step 8530/44000 | Loss: 3.2093 | LR: 4.61e-04 | 1337.6ms/step | 15.0 samples/s +Step 8540/44000 | Loss: 3.1694 | LR: 4.61e-04 | 1338.4ms/step | 14.9 samples/s +Step 8550/44000 | Loss: 3.1889 | LR: 4.61e-04 | 1334.6ms/step | 15.0 samples/s +Step 8560/44000 | Loss: 3.1950 | LR: 4.61e-04 | 1333.6ms/step | 15.0 samples/s +Step 8570/44000 | Loss: 3.1619 | LR: 4.61e-04 | 1334.9ms/step | 15.0 samples/s +Step 8580/44000 | Loss: 3.1933 | LR: 4.61e-04 | 1335.2ms/step | 15.0 samples/s +Step 8590/44000 | Loss: 3.1914 | LR: 4.61e-04 | 1335.4ms/step | 15.0 samples/s +Step 8600/44000 | Loss: 3.1807 | LR: 4.60e-04 | 1337.7ms/step | 15.0 samples/s +Step 8610/44000 | Loss: 3.1593 | LR: 4.60e-04 | 1336.1ms/step | 15.0 samples/s +Step 8620/44000 | Loss: 3.1907 | LR: 4.60e-04 | 1336.6ms/step | 15.0 samples/s +Step 8630/44000 | Loss: 3.1799 | LR: 4.60e-04 | 1336.3ms/step | 15.0 samples/s +Step 8640/44000 | Loss: 3.1855 | LR: 4.60e-04 | 1337.5ms/step | 15.0 samples/s +Step 8650/44000 | Loss: 3.1917 | LR: 4.60e-04 | 1335.6ms/step | 15.0 samples/s +Step 8660/44000 | Loss: 3.1522 | LR: 4.60e-04 | 1337.2ms/step | 15.0 samples/s +Step 8670/44000 | Loss: 3.1941 | LR: 4.60e-04 | 1334.5ms/step | 15.0 samples/s +Step 8680/44000 | Loss: 3.1931 | LR: 4.60e-04 | 1336.9ms/step | 15.0 samples/s +Step 8690/44000 | Loss: 3.1722 | LR: 4.60e-04 | 1332.3ms/step | 15.0 samples/s +Step 8700/44000 | Loss: 3.1498 | LR: 4.59e-04 | 1336.8ms/step | 15.0 samples/s +Step 8710/44000 | Loss: 3.1824 | LR: 4.59e-04 | 1336.3ms/step | 15.0 samples/s +Step 8720/44000 | Loss: 3.1872 | LR: 4.59e-04 | 1337.6ms/step | 15.0 samples/s +Step 8730/44000 | Loss: 3.1883 | LR: 4.59e-04 | 1338.4ms/step | 14.9 samples/s +Step 8740/44000 | Loss: 3.1819 | LR: 4.59e-04 | 1334.7ms/step | 15.0 samples/s +Step 8750/44000 | Loss: 3.1550 | LR: 4.59e-04 | 1336.3ms/step | 15.0 samples/s +Validation at step 8750: Loss = 3.1938, Perplexity = 24.38 +Step 8760/44000 | Loss: 3.1678 | LR: 4.59e-04 | 3586.3ms/step | 5.6 samples/s +Step 8770/44000 | Loss: 3.2074 | LR: 4.59e-04 | 1334.6ms/step | 15.0 samples/s +Step 8780/44000 | Loss: 3.1717 | LR: 4.59e-04 | 1335.7ms/step | 15.0 samples/s +Step 8790/44000 | Loss: 3.1597 | LR: 4.59e-04 | 1336.7ms/step | 15.0 samples/s +Step 8800/44000 | Loss: 3.1549 | LR: 4.58e-04 | 1334.7ms/step | 15.0 samples/s +Step 8810/44000 | Loss: 3.1774 | LR: 4.58e-04 | 1337.2ms/step | 15.0 samples/s +Step 8820/44000 | Loss: 3.1613 | LR: 4.58e-04 | 1336.0ms/step | 15.0 samples/s +Step 8830/44000 | Loss: 3.1666 | LR: 4.58e-04 | 1337.2ms/step | 15.0 samples/s +Step 8840/44000 | Loss: 3.1770 | LR: 4.58e-04 | 1335.9ms/step | 15.0 samples/s +Step 8850/44000 | Loss: 3.1778 | LR: 4.58e-04 | 1336.7ms/step | 15.0 samples/s +Step 8860/44000 | Loss: 3.1965 | LR: 4.58e-04 | 1337.9ms/step | 14.9 samples/s +Step 8870/44000 | Loss: 3.1789 | LR: 4.58e-04 | 1337.0ms/step | 15.0 samples/s +Step 8880/44000 | Loss: 3.1578 | LR: 4.58e-04 | 1336.3ms/step | 15.0 samples/s +Step 8890/44000 | Loss: 3.2033 | LR: 4.58e-04 | 1336.6ms/step | 15.0 samples/s +Step 8900/44000 | Loss: 3.1643 | LR: 4.57e-04 | 1334.0ms/step | 15.0 samples/s +Step 8910/44000 | Loss: 3.1994 | LR: 4.57e-04 | 1336.2ms/step | 15.0 samples/s +Step 8920/44000 | Loss: 3.1792 | LR: 4.57e-04 | 1336.0ms/step | 15.0 samples/s +Step 8930/44000 | Loss: 3.1765 | LR: 4.57e-04 | 1334.3ms/step | 15.0 samples/s +Step 8940/44000 | Loss: 3.1698 | LR: 4.57e-04 | 1336.8ms/step | 15.0 samples/s +Step 8950/44000 | Loss: 3.1758 | LR: 4.57e-04 | 1337.0ms/step | 15.0 samples/s +Step 8960/44000 | Loss: 3.1589 | LR: 4.57e-04 | 1336.8ms/step | 15.0 samples/s +Step 8970/44000 | Loss: 3.1812 | LR: 4.57e-04 | 1335.9ms/step | 15.0 samples/s +Step 8980/44000 | Loss: 3.1806 | LR: 4.57e-04 | 1334.5ms/step | 15.0 samples/s +Step 8990/44000 | Loss: 3.1653 | LR: 4.57e-04 | 1339.0ms/step | 14.9 samples/s +Step 9000/44000 | Loss: 3.1664 | LR: 4.56e-04 | 1339.9ms/step | 14.9 samples/s +Validation at step 9000: Loss = 3.1875, Perplexity = 24.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-9000 +Step 9010/44000 | Loss: 3.1667 | LR: 4.56e-04 | 3882.6ms/step | 5.2 samples/s +Step 9020/44000 | Loss: 3.1853 | LR: 4.56e-04 | 1331.1ms/step | 15.0 samples/s +Step 9030/44000 | Loss: 3.1594 | LR: 4.56e-04 | 1333.9ms/step | 15.0 samples/s +Step 9040/44000 | Loss: 3.1615 | LR: 4.56e-04 | 1332.6ms/step | 15.0 samples/s +Step 9050/44000 | Loss: 3.1632 | LR: 4.56e-04 | 1333.1ms/step | 15.0 samples/s +Step 9060/44000 | Loss: 3.2009 | LR: 4.56e-04 | 1333.8ms/step | 15.0 samples/s +Step 9070/44000 | Loss: 3.1943 | LR: 4.56e-04 | 1331.2ms/step | 15.0 samples/s +Step 9080/44000 | Loss: 3.2175 | LR: 4.56e-04 | 1331.5ms/step | 15.0 samples/s +Step 9090/44000 | Loss: 3.1459 | LR: 4.56e-04 | 1334.7ms/step | 15.0 samples/s +Step 9100/44000 | Loss: 3.1522 | LR: 4.55e-04 | 1331.6ms/step | 15.0 samples/s +Step 9110/44000 | Loss: 3.1492 | LR: 4.55e-04 | 1330.8ms/step | 15.0 samples/s +Step 9120/44000 | Loss: 3.1509 | LR: 4.55e-04 | 1330.8ms/step | 15.0 samples/s +Step 9130/44000 | Loss: 3.1970 | LR: 4.55e-04 | 1335.1ms/step | 15.0 samples/s +Step 9140/44000 | Loss: 3.1452 | LR: 4.55e-04 | 1331.4ms/step | 15.0 samples/s +Step 9150/44000 | Loss: 3.1732 | LR: 4.55e-04 | 1336.6ms/step | 15.0 samples/s +Step 9160/44000 | Loss: 3.1487 | LR: 4.55e-04 | 1336.6ms/step | 15.0 samples/s +Step 9170/44000 | Loss: 3.1311 | LR: 4.55e-04 | 1335.8ms/step | 15.0 samples/s +Step 9180/44000 | Loss: 3.1782 | LR: 4.55e-04 | 1334.0ms/step | 15.0 samples/s +Step 9190/44000 | Loss: 3.1544 | LR: 4.54e-04 | 1334.9ms/step | 15.0 samples/s +Step 9200/44000 | Loss: 3.1817 | LR: 4.54e-04 | 1332.4ms/step | 15.0 samples/s +Step 9210/44000 | Loss: 3.1752 | LR: 4.54e-04 | 1335.0ms/step | 15.0 samples/s +Step 9220/44000 | Loss: 3.1652 | LR: 4.54e-04 | 1331.8ms/step | 15.0 samples/s +Step 9230/44000 | Loss: 3.1666 | LR: 4.54e-04 | 1335.6ms/step | 15.0 samples/s +Step 9240/44000 | Loss: 3.2159 | LR: 4.54e-04 | 1333.4ms/step | 15.0 samples/s +Step 9250/44000 | Loss: 3.1694 | LR: 4.54e-04 | 1333.9ms/step | 15.0 samples/s +Validation at step 9250: Loss = 3.1803, Perplexity = 24.05 +Step 9260/44000 | Loss: 3.1765 | LR: 4.54e-04 | 3569.2ms/step | 5.6 samples/s +Step 9270/44000 | Loss: 3.1730 | LR: 4.54e-04 | 1334.0ms/step | 15.0 samples/s +Step 9280/44000 | Loss: 3.1735 | LR: 4.54e-04 | 1332.9ms/step | 15.0 samples/s +Step 9290/44000 | Loss: 3.1834 | LR: 4.53e-04 | 1332.0ms/step | 15.0 samples/s +Step 9300/44000 | Loss: 3.1439 | LR: 4.53e-04 | 1332.5ms/step | 15.0 samples/s +Step 9310/44000 | Loss: 3.1644 | LR: 4.53e-04 | 1331.8ms/step | 15.0 samples/s +Step 9320/44000 | Loss: 3.1486 | LR: 4.53e-04 | 1334.9ms/step | 15.0 samples/s +Step 9330/44000 | Loss: 3.1685 | LR: 4.53e-04 | 1336.5ms/step | 15.0 samples/s +Step 9340/44000 | Loss: 3.1523 | LR: 4.53e-04 | 1338.1ms/step | 14.9 samples/s +Step 9350/44000 | Loss: 3.1424 | LR: 4.53e-04 | 1337.8ms/step | 14.9 samples/s +Step 9360/44000 | Loss: 3.1690 | LR: 4.53e-04 | 1334.8ms/step | 15.0 samples/s +Step 9370/44000 | Loss: 3.1709 | LR: 4.53e-04 | 1331.5ms/step | 15.0 samples/s +Step 9380/44000 | Loss: 3.1670 | LR: 4.52e-04 | 1334.1ms/step | 15.0 samples/s +Step 9390/44000 | Loss: 3.1804 | LR: 4.52e-04 | 1333.4ms/step | 15.0 samples/s +Step 9400/44000 | Loss: 3.1567 | LR: 4.52e-04 | 1333.5ms/step | 15.0 samples/s +Step 9410/44000 | Loss: 3.1624 | LR: 4.52e-04 | 1333.0ms/step | 15.0 samples/s +Step 9420/44000 | Loss: 3.1721 | LR: 4.52e-04 | 1334.1ms/step | 15.0 samples/s +Step 9430/44000 | Loss: 3.1580 | LR: 4.52e-04 | 1333.0ms/step | 15.0 samples/s +Step 9440/44000 | Loss: 3.1363 | LR: 4.52e-04 | 1332.6ms/step | 15.0 samples/s +Step 9450/44000 | Loss: 3.1596 | LR: 4.52e-04 | 1334.4ms/step | 15.0 samples/s +Step 9460/44000 | Loss: 3.1723 | LR: 4.52e-04 | 1330.5ms/step | 15.0 samples/s +Step 9470/44000 | Loss: 3.1539 | LR: 4.52e-04 | 1328.7ms/step | 15.1 samples/s +Step 9480/44000 | Loss: 3.1505 | LR: 4.51e-04 | 1332.3ms/step | 15.0 samples/s +Step 9490/44000 | Loss: 3.1644 | LR: 4.51e-04 | 1328.1ms/step | 15.1 samples/s +Step 9500/44000 | Loss: 3.1671 | LR: 4.51e-04 | 1331.0ms/step | 15.0 samples/s +Validation at step 9500: Loss = 3.1740, Perplexity = 23.90 +Step 9510/44000 | Loss: 3.1975 | LR: 4.51e-04 | 3571.4ms/step | 5.6 samples/s +Step 9520/44000 | Loss: 3.1499 | LR: 4.51e-04 | 1334.7ms/step | 15.0 samples/s +Step 9530/44000 | Loss: 3.1895 | LR: 4.51e-04 | 1336.1ms/step | 15.0 samples/s +Step 9540/44000 | Loss: 3.1865 | LR: 4.51e-04 | 1332.7ms/step | 15.0 samples/s +Step 9550/44000 | Loss: 3.1730 | LR: 4.51e-04 | 1338.1ms/step | 14.9 samples/s +Step 9560/44000 | Loss: 3.1535 | LR: 4.51e-04 | 1332.8ms/step | 15.0 samples/s +Step 9570/44000 | Loss: 3.1495 | LR: 4.50e-04 | 1330.0ms/step | 15.0 samples/s +Step 9580/44000 | Loss: 3.1789 | LR: 4.50e-04 | 1328.8ms/step | 15.1 samples/s +Step 9590/44000 | Loss: 3.1446 | LR: 4.50e-04 | 1331.0ms/step | 15.0 samples/s +Step 9600/44000 | Loss: 3.1480 | LR: 4.50e-04 | 1332.4ms/step | 15.0 samples/s +Step 9610/44000 | Loss: 3.1495 | LR: 4.50e-04 | 1332.7ms/step | 15.0 samples/s +Step 9620/44000 | Loss: 3.1882 | LR: 4.50e-04 | 1332.3ms/step | 15.0 samples/s +Step 9630/44000 | Loss: 3.1654 | LR: 4.50e-04 | 1339.4ms/step | 14.9 samples/s +Step 9640/44000 | Loss: 3.1615 | LR: 4.50e-04 | 1339.5ms/step | 14.9 samples/s +Step 9650/44000 | Loss: 3.1638 | LR: 4.50e-04 | 1333.0ms/step | 15.0 samples/s +Step 9660/44000 | Loss: 3.1381 | LR: 4.49e-04 | 1330.1ms/step | 15.0 samples/s +Step 9670/44000 | Loss: 3.1613 | LR: 4.49e-04 | 1331.0ms/step | 15.0 samples/s +Step 9680/44000 | Loss: 3.1661 | LR: 4.49e-04 | 1332.8ms/step | 15.0 samples/s +Step 9690/44000 | Loss: 3.1642 | LR: 4.49e-04 | 1329.0ms/step | 15.0 samples/s +Step 9700/44000 | Loss: 3.1673 | LR: 4.49e-04 | 1331.2ms/step | 15.0 samples/s +Step 9710/44000 | Loss: 3.1653 | LR: 4.49e-04 | 1332.1ms/step | 15.0 samples/s +Step 9720/44000 | Loss: 3.1534 | LR: 4.49e-04 | 1331.6ms/step | 15.0 samples/s +Step 9730/44000 | Loss: 3.1690 | LR: 4.49e-04 | 1331.7ms/step | 15.0 samples/s +Step 9740/44000 | Loss: 3.1565 | LR: 4.49e-04 | 1333.1ms/step | 15.0 samples/s +Step 9750/44000 | Loss: 3.1507 | LR: 4.48e-04 | 1331.3ms/step | 15.0 samples/s +Validation at step 9750: Loss = 3.1684, Perplexity = 23.77 +Step 9760/44000 | Loss: 3.1287 | LR: 4.48e-04 | 3565.1ms/step | 5.6 samples/s +Step 9770/44000 | Loss: 3.1294 | LR: 4.48e-04 | 1333.7ms/step | 15.0 samples/s +Step 9780/44000 | Loss: 3.1540 | LR: 4.48e-04 | 1335.1ms/step | 15.0 samples/s +Step 9790/44000 | Loss: 3.1722 | LR: 4.48e-04 | 1333.1ms/step | 15.0 samples/s +Step 9800/44000 | Loss: 3.1549 | LR: 4.48e-04 | 1333.6ms/step | 15.0 samples/s +Step 9810/44000 | Loss: 3.1738 | LR: 4.48e-04 | 1333.1ms/step | 15.0 samples/s +Step 9820/44000 | Loss: 3.1726 | LR: 4.48e-04 | 1331.8ms/step | 15.0 samples/s +Step 9830/44000 | Loss: 3.2003 | LR: 4.48e-04 | 1330.2ms/step | 15.0 samples/s +Step 9840/44000 | Loss: 3.1471 | LR: 4.47e-04 | 1328.5ms/step | 15.1 samples/s +Step 9850/44000 | Loss: 3.1669 | LR: 4.47e-04 | 1332.9ms/step | 15.0 samples/s +Step 9860/44000 | Loss: 3.1265 | LR: 4.47e-04 | 1331.7ms/step | 15.0 samples/s +Step 9870/44000 | Loss: 3.1508 | LR: 4.47e-04 | 1330.0ms/step | 15.0 samples/s +Step 9880/44000 | Loss: 3.1537 | LR: 4.47e-04 | 1334.5ms/step | 15.0 samples/s +Step 9890/44000 | Loss: 3.1364 | LR: 4.47e-04 | 1333.1ms/step | 15.0 samples/s +Step 9900/44000 | Loss: 3.1596 | LR: 4.47e-04 | 1332.4ms/step | 15.0 samples/s +Step 9910/44000 | Loss: 3.1745 | LR: 4.47e-04 | 1331.8ms/step | 15.0 samples/s +Step 9920/44000 | Loss: 3.1402 | LR: 4.47e-04 | 1332.8ms/step | 15.0 samples/s +Step 9930/44000 | Loss: 3.1336 | LR: 4.46e-04 | 1333.4ms/step | 15.0 samples/s +Step 9940/44000 | Loss: 3.1573 | LR: 4.46e-04 | 1336.5ms/step | 15.0 samples/s +Step 9950/44000 | Loss: 3.1393 | LR: 4.46e-04 | 1332.4ms/step | 15.0 samples/s +Step 9960/44000 | Loss: 3.1526 | LR: 4.46e-04 | 1334.4ms/step | 15.0 samples/s +Step 9970/44000 | Loss: 3.1967 | LR: 4.46e-04 | 1331.4ms/step | 15.0 samples/s +Step 9980/44000 | Loss: 3.1469 | LR: 4.46e-04 | 1332.7ms/step | 15.0 samples/s +Step 9990/44000 | Loss: 3.1564 | LR: 4.46e-04 | 1330.4ms/step | 15.0 samples/s +Step 10000/44000 | Loss: 3.1502 | LR: 4.46e-04 | 1332.4ms/step | 15.0 samples/s +Validation at step 10000: Loss = 3.1619, Perplexity = 23.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-10000 +Step 10010/44000 | Loss: 3.1459 | LR: 4.46e-04 | 3871.5ms/step | 5.2 samples/s +Step 10020/44000 | Loss: 3.1738 | LR: 4.45e-04 | 1337.0ms/step | 15.0 samples/s +Step 10030/44000 | Loss: 3.1560 | LR: 4.45e-04 | 1337.3ms/step | 15.0 samples/s +Step 10040/44000 | Loss: 3.1242 | LR: 4.45e-04 | 1337.9ms/step | 14.9 samples/s +Step 10050/44000 | Loss: 3.1343 | LR: 4.45e-04 | 1336.4ms/step | 15.0 samples/s +Step 10060/44000 | Loss: 3.1337 | LR: 4.45e-04 | 1339.5ms/step | 14.9 samples/s +Step 10070/44000 | Loss: 3.1363 | LR: 4.45e-04 | 1334.2ms/step | 15.0 samples/s +Step 10080/44000 | Loss: 3.1450 | LR: 4.45e-04 | 1337.1ms/step | 15.0 samples/s +Step 10090/44000 | Loss: 3.1436 | LR: 4.45e-04 | 1339.2ms/step | 14.9 samples/s +Step 10100/44000 | Loss: 3.1606 | LR: 4.45e-04 | 1335.3ms/step | 15.0 samples/s +Step 10110/44000 | Loss: 3.1681 | LR: 4.44e-04 | 1334.8ms/step | 15.0 samples/s +Step 10120/44000 | Loss: 3.1922 | LR: 4.44e-04 | 1336.7ms/step | 15.0 samples/s +Step 10130/44000 | Loss: 3.1386 | LR: 4.44e-04 | 1336.7ms/step | 15.0 samples/s +Step 10140/44000 | Loss: 3.1979 | LR: 4.44e-04 | 1331.8ms/step | 15.0 samples/s +Step 10150/44000 | Loss: 3.1567 | LR: 4.44e-04 | 1331.1ms/step | 15.0 samples/s +Step 10160/44000 | Loss: 3.1557 | LR: 4.44e-04 | 1331.8ms/step | 15.0 samples/s +Step 10170/44000 | Loss: 3.1257 | LR: 4.44e-04 | 1332.1ms/step | 15.0 samples/s +Step 10180/44000 | Loss: 3.1610 | LR: 4.44e-04 | 1334.9ms/step | 15.0 samples/s +Step 10190/44000 | Loss: 3.1311 | LR: 4.43e-04 | 1328.4ms/step | 15.1 samples/s +Step 10200/44000 | Loss: 3.1352 | LR: 4.43e-04 | 1332.4ms/step | 15.0 samples/s +Step 10210/44000 | Loss: 3.1510 | LR: 4.43e-04 | 1335.1ms/step | 15.0 samples/s +Step 10220/44000 | Loss: 3.1367 | LR: 4.43e-04 | 1332.6ms/step | 15.0 samples/s +Step 10230/44000 | Loss: 3.1598 | LR: 4.43e-04 | 1335.4ms/step | 15.0 samples/s +Step 10240/44000 | Loss: 3.1365 | LR: 4.43e-04 | 1334.1ms/step | 15.0 samples/s +Step 10250/44000 | Loss: 3.1579 | LR: 4.43e-04 | 1338.5ms/step | 14.9 samples/s +Validation at step 10250: Loss = 3.1574, Perplexity = 23.51 +Step 10260/44000 | Loss: 3.1161 | LR: 4.43e-04 | 3593.5ms/step | 5.6 samples/s +Step 10270/44000 | Loss: 3.1036 | LR: 4.43e-04 | 1335.0ms/step | 15.0 samples/s +Step 10280/44000 | Loss: 3.1058 | LR: 4.42e-04 | 1330.5ms/step | 15.0 samples/s +Step 10290/44000 | Loss: 3.1013 | LR: 4.42e-04 | 1330.8ms/step | 15.0 samples/s +Step 10300/44000 | Loss: 3.1334 | LR: 4.42e-04 | 1332.6ms/step | 15.0 samples/s +Step 10310/44000 | Loss: 3.1579 | LR: 4.42e-04 | 1338.4ms/step | 14.9 samples/s +Step 10320/44000 | Loss: 3.1248 | LR: 4.42e-04 | 1337.3ms/step | 15.0 samples/s +Step 10330/44000 | Loss: 3.1654 | LR: 4.42e-04 | 1338.6ms/step | 14.9 samples/s +Step 10340/44000 | Loss: 3.1397 | LR: 4.42e-04 | 1334.2ms/step | 15.0 samples/s +Step 10350/44000 | Loss: 3.1529 | LR: 4.42e-04 | 1336.2ms/step | 15.0 samples/s +Step 10360/44000 | Loss: 3.1253 | LR: 4.42e-04 | 1337.3ms/step | 15.0 samples/s +Step 10370/44000 | Loss: 3.1526 | LR: 4.41e-04 | 1338.4ms/step | 14.9 samples/s +Step 10380/44000 | Loss: 3.1646 | LR: 4.41e-04 | 1337.8ms/step | 15.0 samples/s +Step 10390/44000 | Loss: 3.1339 | LR: 4.41e-04 | 1336.7ms/step | 15.0 samples/s +Step 10400/44000 | Loss: 3.1675 | LR: 4.41e-04 | 1336.1ms/step | 15.0 samples/s +Step 10410/44000 | Loss: 3.1718 | LR: 4.41e-04 | 1333.2ms/step | 15.0 samples/s +Step 10420/44000 | Loss: 3.1649 | LR: 4.41e-04 | 1334.2ms/step | 15.0 samples/s +Step 10430/44000 | Loss: 3.1562 | LR: 4.41e-04 | 1335.9ms/step | 15.0 samples/s +Step 10440/44000 | Loss: 3.1263 | LR: 4.41e-04 | 1334.8ms/step | 15.0 samples/s +Step 10450/44000 | Loss: 3.1387 | LR: 4.40e-04 | 1337.7ms/step | 15.0 samples/s +Step 10460/44000 | Loss: 3.1330 | LR: 4.40e-04 | 1336.7ms/step | 15.0 samples/s +Step 10470/44000 | Loss: 3.1628 | LR: 4.40e-04 | 1333.0ms/step | 15.0 samples/s +Step 10480/44000 | Loss: 3.1315 | LR: 4.40e-04 | 1337.0ms/step | 15.0 samples/s +Step 10490/44000 | Loss: 3.1176 | LR: 4.40e-04 | 1336.9ms/step | 15.0 samples/s +Step 10500/44000 | Loss: 3.1697 | LR: 4.40e-04 | 1338.8ms/step | 14.9 samples/s +Validation at step 10500: Loss = 3.1533, Perplexity = 23.41 +Step 10510/44000 | Loss: 3.1771 | LR: 4.40e-04 | 3579.6ms/step | 5.6 samples/s +Step 10520/44000 | Loss: 3.1161 | LR: 4.40e-04 | 1329.2ms/step | 15.0 samples/s +Step 10530/44000 | Loss: 3.1327 | LR: 4.40e-04 | 1335.3ms/step | 15.0 samples/s +Step 10540/44000 | Loss: 3.1415 | LR: 4.39e-04 | 1334.4ms/step | 15.0 samples/s +Step 10550/44000 | Loss: 3.1325 | LR: 4.39e-04 | 1333.9ms/step | 15.0 samples/s +Step 10560/44000 | Loss: 3.1211 | LR: 4.39e-04 | 1334.9ms/step | 15.0 samples/s +Step 10570/44000 | Loss: 3.1357 | LR: 4.39e-04 | 1337.3ms/step | 15.0 samples/s +Step 10580/44000 | Loss: 3.1523 | LR: 4.39e-04 | 1335.7ms/step | 15.0 samples/s +Step 10590/44000 | Loss: 3.1021 | LR: 4.39e-04 | 1334.0ms/step | 15.0 samples/s +Step 10600/44000 | Loss: 3.1251 | LR: 4.39e-04 | 1336.3ms/step | 15.0 samples/s +Step 10610/44000 | Loss: 3.1140 | LR: 4.39e-04 | 1337.2ms/step | 15.0 samples/s +Step 10620/44000 | Loss: 3.1596 | LR: 4.38e-04 | 1336.0ms/step | 15.0 samples/s +Step 10630/44000 | Loss: 3.1489 | LR: 4.38e-04 | 1337.5ms/step | 15.0 samples/s +Step 10640/44000 | Loss: 3.1285 | LR: 4.38e-04 | 1337.6ms/step | 15.0 samples/s +Step 10650/44000 | Loss: 3.1435 | LR: 4.38e-04 | 1338.8ms/step | 14.9 samples/s +Step 10660/44000 | Loss: 3.1293 | LR: 4.38e-04 | 1332.0ms/step | 15.0 samples/s +Step 10670/44000 | Loss: 3.1341 | LR: 4.38e-04 | 1334.3ms/step | 15.0 samples/s +Step 10680/44000 | Loss: 3.1513 | LR: 4.38e-04 | 1333.5ms/step | 15.0 samples/s +Step 10690/44000 | Loss: 3.1025 | LR: 4.38e-04 | 1337.1ms/step | 15.0 samples/s +Step 10700/44000 | Loss: 3.0751 | LR: 4.37e-04 | 1338.4ms/step | 14.9 samples/s +Step 10710/44000 | Loss: 3.1084 | LR: 4.37e-04 | 1337.8ms/step | 15.0 samples/s +Step 10720/44000 | Loss: 3.0976 | LR: 4.37e-04 | 1337.5ms/step | 15.0 samples/s +Step 10730/44000 | Loss: 3.1087 | LR: 4.37e-04 | 1339.7ms/step | 14.9 samples/s +Step 10740/44000 | Loss: 3.1194 | LR: 4.37e-04 | 1332.2ms/step | 15.0 samples/s +Step 10750/44000 | Loss: 3.1126 | LR: 4.37e-04 | 1332.2ms/step | 15.0 samples/s +Validation at step 10750: Loss = 3.1473, Perplexity = 23.27 +Step 10760/44000 | Loss: 3.1498 | LR: 4.37e-04 | 3576.1ms/step | 5.6 samples/s +Step 10770/44000 | Loss: 3.1514 | LR: 4.37e-04 | 1331.1ms/step | 15.0 samples/s +Step 10780/44000 | Loss: 3.1246 | LR: 4.37e-04 | 1329.1ms/step | 15.0 samples/s +Step 10790/44000 | Loss: 3.1333 | LR: 4.36e-04 | 1329.7ms/step | 15.0 samples/s +Step 10800/44000 | Loss: 3.1186 | LR: 4.36e-04 | 1330.9ms/step | 15.0 samples/s +Step 10810/44000 | Loss: 3.1355 | LR: 4.36e-04 | 1332.3ms/step | 15.0 samples/s +Step 10820/44000 | Loss: 3.1444 | LR: 4.36e-04 | 1332.6ms/step | 15.0 samples/s +Step 10830/44000 | Loss: 3.1277 | LR: 4.36e-04 | 1329.8ms/step | 15.0 samples/s +Step 10840/44000 | Loss: 3.1503 | LR: 4.36e-04 | 1336.1ms/step | 15.0 samples/s +Step 10850/44000 | Loss: 3.1456 | LR: 4.36e-04 | 1334.8ms/step | 15.0 samples/s +Step 10860/44000 | Loss: 3.1659 | LR: 4.36e-04 | 1338.0ms/step | 14.9 samples/s +Step 10870/44000 | Loss: 3.1294 | LR: 4.35e-04 | 1336.8ms/step | 15.0 samples/s +Step 10880/44000 | Loss: 3.1255 | LR: 4.35e-04 | 1336.3ms/step | 15.0 samples/s +Step 10890/44000 | Loss: 3.1321 | LR: 4.35e-04 | 1335.2ms/step | 15.0 samples/s +Step 10900/44000 | Loss: 3.1353 | LR: 4.35e-04 | 1335.5ms/step | 15.0 samples/s +Step 10910/44000 | Loss: 3.1117 | LR: 4.35e-04 | 1333.0ms/step | 15.0 samples/s +Step 10920/44000 | Loss: 3.0974 | LR: 4.35e-04 | 1329.8ms/step | 15.0 samples/s +Step 10930/44000 | Loss: 3.1646 | LR: 4.35e-04 | 1331.3ms/step | 15.0 samples/s +Step 10940/44000 | Loss: 3.1326 | LR: 4.35e-04 | 1333.8ms/step | 15.0 samples/s +Step 10950/44000 | Loss: 3.1126 | LR: 4.34e-04 | 1334.5ms/step | 15.0 samples/s +Step 10960/44000 | Loss: 3.1432 | LR: 4.34e-04 | 1335.0ms/step | 15.0 samples/s +Step 10970/44000 | Loss: 3.1267 | LR: 4.34e-04 | 1336.1ms/step | 15.0 samples/s +Step 10980/44000 | Loss: 3.1180 | LR: 4.34e-04 | 1331.1ms/step | 15.0 samples/s +Step 10990/44000 | Loss: 3.1321 | LR: 4.34e-04 | 1331.3ms/step | 15.0 samples/s +Step 11000/44000 | Loss: 3.1248 | LR: 4.34e-04 | 1332.8ms/step | 15.0 samples/s +Validation at step 11000: Loss = 3.1416, Perplexity = 23.14 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-11000 +Step 11010/44000 | Loss: 3.1396 | LR: 4.34e-04 | 3895.8ms/step | 5.1 samples/s +Step 11020/44000 | Loss: 3.1521 | LR: 4.34e-04 | 1335.8ms/step | 15.0 samples/s +Step 11030/44000 | Loss: 3.0952 | LR: 4.33e-04 | 1335.9ms/step | 15.0 samples/s +Step 11040/44000 | Loss: 3.1386 | LR: 4.33e-04 | 1337.7ms/step | 15.0 samples/s +Step 11050/44000 | Loss: 3.1189 | LR: 4.33e-04 | 1339.0ms/step | 14.9 samples/s +Step 11060/44000 | Loss: 3.1335 | LR: 4.33e-04 | 1342.5ms/step | 14.9 samples/s +Step 11070/44000 | Loss: 3.1264 | LR: 4.33e-04 | 1340.0ms/step | 14.9 samples/s +Step 11080/44000 | Loss: 3.0908 | LR: 4.33e-04 | 1337.7ms/step | 15.0 samples/s +Step 11090/44000 | Loss: 3.1502 | LR: 4.33e-04 | 1339.2ms/step | 14.9 samples/s +Step 11100/44000 | Loss: 3.1242 | LR: 4.33e-04 | 1339.6ms/step | 14.9 samples/s +Step 11110/44000 | Loss: 3.0898 | LR: 4.32e-04 | 1338.7ms/step | 14.9 samples/s +Step 11120/44000 | Loss: 3.1174 | LR: 4.32e-04 | 1340.0ms/step | 14.9 samples/s +Step 11130/44000 | Loss: 3.1146 | LR: 4.32e-04 | 1336.3ms/step | 15.0 samples/s +Step 11140/44000 | Loss: 3.1178 | LR: 4.32e-04 | 1339.8ms/step | 14.9 samples/s +Step 11150/44000 | Loss: 3.1387 | LR: 4.32e-04 | 1337.1ms/step | 15.0 samples/s +Step 11160/44000 | Loss: 3.1203 | LR: 4.32e-04 | 1335.1ms/step | 15.0 samples/s +Step 11170/44000 | Loss: 3.1180 | LR: 4.32e-04 | 1337.5ms/step | 15.0 samples/s +Step 11180/44000 | Loss: 3.1282 | LR: 4.32e-04 | 1340.1ms/step | 14.9 samples/s +Step 11190/44000 | Loss: 3.1256 | LR: 4.31e-04 | 1337.3ms/step | 15.0 samples/s +Step 11200/44000 | Loss: 3.1272 | LR: 4.31e-04 | 1338.7ms/step | 14.9 samples/s +Step 11210/44000 | Loss: 3.1279 | LR: 4.31e-04 | 1339.6ms/step | 14.9 samples/s +Step 11220/44000 | Loss: 3.1027 | LR: 4.31e-04 | 1337.4ms/step | 15.0 samples/s +Step 11230/44000 | Loss: 3.1262 | LR: 4.31e-04 | 1341.4ms/step | 14.9 samples/s +Step 11240/44000 | Loss: 3.1142 | LR: 4.31e-04 | 1341.2ms/step | 14.9 samples/s +Step 11250/44000 | Loss: 3.1205 | LR: 4.31e-04 | 1340.6ms/step | 14.9 samples/s +Validation at step 11250: Loss = 3.1367, Perplexity = 23.03 +Step 11260/44000 | Loss: 3.0962 | LR: 4.31e-04 | 3589.5ms/step | 5.6 samples/s +Step 11270/44000 | Loss: 3.1389 | LR: 4.30e-04 | 1340.3ms/step | 14.9 samples/s +Step 11280/44000 | Loss: 3.1138 | LR: 4.30e-04 | 1337.4ms/step | 15.0 samples/s +Step 11290/44000 | Loss: 3.1238 | LR: 4.30e-04 | 1337.8ms/step | 15.0 samples/s +Step 11300/44000 | Loss: 3.1416 | LR: 4.30e-04 | 1338.2ms/step | 14.9 samples/s +Step 11310/44000 | Loss: 3.1082 | LR: 4.30e-04 | 1335.2ms/step | 15.0 samples/s +Step 11320/44000 | Loss: 3.1217 | LR: 4.30e-04 | 1338.5ms/step | 14.9 samples/s +Step 11330/44000 | Loss: 3.1239 | LR: 4.30e-04 | 1338.2ms/step | 14.9 samples/s +Step 11340/44000 | Loss: 3.0925 | LR: 4.30e-04 | 1339.6ms/step | 14.9 samples/s +Step 11350/44000 | Loss: 3.1213 | LR: 4.29e-04 | 1340.0ms/step | 14.9 samples/s +Step 11360/44000 | Loss: 3.1036 | LR: 4.29e-04 | 1338.3ms/step | 14.9 samples/s +Step 11370/44000 | Loss: 3.1325 | LR: 4.29e-04 | 1335.8ms/step | 15.0 samples/s +Step 11380/44000 | Loss: 3.1335 | LR: 4.29e-04 | 1340.6ms/step | 14.9 samples/s +Step 11390/44000 | Loss: 3.1500 | LR: 4.29e-04 | 1340.8ms/step | 14.9 samples/s +Step 11400/44000 | Loss: 3.1145 | LR: 4.29e-04 | 1339.6ms/step | 14.9 samples/s +Step 11410/44000 | Loss: 3.0834 | LR: 4.29e-04 | 1338.6ms/step | 14.9 samples/s +Step 11420/44000 | Loss: 3.1327 | LR: 4.29e-04 | 1336.3ms/step | 15.0 samples/s +Step 11430/44000 | Loss: 3.1146 | LR: 4.28e-04 | 1335.1ms/step | 15.0 samples/s +Step 11440/44000 | Loss: 3.1387 | LR: 4.28e-04 | 1339.1ms/step | 14.9 samples/s +Step 11450/44000 | Loss: 3.1146 | LR: 4.28e-04 | 1339.3ms/step | 14.9 samples/s +Step 11460/44000 | Loss: 3.1493 | LR: 4.28e-04 | 1339.0ms/step | 14.9 samples/s +Step 11470/44000 | Loss: 3.1225 | LR: 4.28e-04 | 1336.4ms/step | 15.0 samples/s +Step 11480/44000 | Loss: 3.1122 | LR: 4.28e-04 | 1335.4ms/step | 15.0 samples/s +Step 11490/44000 | Loss: 3.1141 | LR: 4.28e-04 | 1336.6ms/step | 15.0 samples/s +Step 11500/44000 | Loss: 3.1169 | LR: 4.28e-04 | 1337.3ms/step | 15.0 samples/s +Validation at step 11500: Loss = 3.1307, Perplexity = 22.89 +Step 11510/44000 | Loss: 3.1313 | LR: 4.27e-04 | 3588.8ms/step | 5.6 samples/s +Step 11520/44000 | Loss: 3.0895 | LR: 4.27e-04 | 1336.8ms/step | 15.0 samples/s +Step 11530/44000 | Loss: 3.1188 | LR: 4.27e-04 | 1334.2ms/step | 15.0 samples/s +Step 11540/44000 | Loss: 3.1188 | LR: 4.27e-04 | 1334.2ms/step | 15.0 samples/s +Step 11550/44000 | Loss: 3.1030 | LR: 4.27e-04 | 1336.4ms/step | 15.0 samples/s +Step 11560/44000 | Loss: 3.1196 | LR: 4.27e-04 | 1336.8ms/step | 15.0 samples/s +Step 11570/44000 | Loss: 3.1194 | LR: 4.27e-04 | 1334.9ms/step | 15.0 samples/s +Step 11580/44000 | Loss: 3.1185 | LR: 4.27e-04 | 1333.8ms/step | 15.0 samples/s +Step 11590/44000 | Loss: 3.1283 | LR: 4.26e-04 | 1336.1ms/step | 15.0 samples/s +Step 11600/44000 | Loss: 3.1134 | LR: 4.26e-04 | 1338.4ms/step | 14.9 samples/s +Step 11610/44000 | Loss: 3.1211 | LR: 4.26e-04 | 1337.8ms/step | 15.0 samples/s +Step 11620/44000 | Loss: 3.1154 | LR: 4.26e-04 | 1337.5ms/step | 15.0 samples/s +Step 11630/44000 | Loss: 3.0948 | LR: 4.26e-04 | 1335.3ms/step | 15.0 samples/s +Step 11640/44000 | Loss: 3.1194 | LR: 4.26e-04 | 1340.1ms/step | 14.9 samples/s +Step 11650/44000 | Loss: 3.1205 | LR: 4.26e-04 | 1338.1ms/step | 14.9 samples/s +Step 11660/44000 | Loss: 3.0900 | LR: 4.26e-04 | 1334.0ms/step | 15.0 samples/s +Step 11670/44000 | Loss: 3.1139 | LR: 4.25e-04 | 1338.1ms/step | 14.9 samples/s +Step 11680/44000 | Loss: 3.0893 | LR: 4.25e-04 | 1336.7ms/step | 15.0 samples/s +Step 11690/44000 | Loss: 3.1462 | LR: 4.25e-04 | 1337.7ms/step | 15.0 samples/s +Step 11700/44000 | Loss: 3.0842 | LR: 4.25e-04 | 1336.3ms/step | 15.0 samples/s +Step 11710/44000 | Loss: 3.1353 | LR: 4.25e-04 | 1337.7ms/step | 15.0 samples/s +Step 11720/44000 | Loss: 3.1262 | LR: 4.25e-04 | 1336.2ms/step | 15.0 samples/s +Step 11730/44000 | Loss: 3.1244 | LR: 4.25e-04 | 1335.9ms/step | 15.0 samples/s +Step 11740/44000 | Loss: 3.1494 | LR: 4.24e-04 | 1334.7ms/step | 15.0 samples/s +Step 11750/44000 | Loss: 3.1110 | LR: 4.24e-04 | 1332.9ms/step | 15.0 samples/s +Validation at step 11750: Loss = 3.1272, Perplexity = 22.81 +Step 11760/44000 | Loss: 3.1465 | LR: 4.24e-04 | 3581.9ms/step | 5.6 samples/s +Step 11770/44000 | Loss: 3.1395 | LR: 4.24e-04 | 1339.7ms/step | 14.9 samples/s +Step 11780/44000 | Loss: 3.1440 | LR: 4.24e-04 | 1331.3ms/step | 15.0 samples/s +Step 11790/44000 | Loss: 3.1084 | LR: 4.24e-04 | 1337.6ms/step | 15.0 samples/s +Step 11800/44000 | Loss: 3.1025 | LR: 4.24e-04 | 1337.7ms/step | 15.0 samples/s +Step 11810/44000 | Loss: 3.0943 | LR: 4.24e-04 | 1336.2ms/step | 15.0 samples/s +Step 11820/44000 | Loss: 3.1221 | LR: 4.23e-04 | 1339.8ms/step | 14.9 samples/s +Step 11830/44000 | Loss: 3.1461 | LR: 4.23e-04 | 1340.0ms/step | 14.9 samples/s +Step 11840/44000 | Loss: 3.0857 | LR: 4.23e-04 | 1339.6ms/step | 14.9 samples/s +Step 11850/44000 | Loss: 3.1273 | LR: 4.23e-04 | 1338.7ms/step | 14.9 samples/s +Step 11860/44000 | Loss: 3.1015 | LR: 4.23e-04 | 1339.4ms/step | 14.9 samples/s +Step 11870/44000 | Loss: 3.1272 | LR: 4.23e-04 | 1337.6ms/step | 15.0 samples/s +Step 11880/44000 | Loss: 3.1115 | LR: 4.23e-04 | 1336.0ms/step | 15.0 samples/s +Step 11890/44000 | Loss: 3.0855 | LR: 4.23e-04 | 1339.1ms/step | 14.9 samples/s +Step 11900/44000 | Loss: 3.1438 | LR: 4.22e-04 | 1338.2ms/step | 14.9 samples/s +Step 11910/44000 | Loss: 3.1147 | LR: 4.22e-04 | 1338.2ms/step | 14.9 samples/s +Step 11920/44000 | Loss: 3.1121 | LR: 4.22e-04 | 1338.8ms/step | 14.9 samples/s +Step 11930/44000 | Loss: 3.1155 | LR: 4.22e-04 | 1338.8ms/step | 14.9 samples/s +Step 11940/44000 | Loss: 3.1023 | LR: 4.22e-04 | 1342.1ms/step | 14.9 samples/s +Step 11950/44000 | Loss: 3.0808 | LR: 4.22e-04 | 1341.8ms/step | 14.9 samples/s +Step 11960/44000 | Loss: 3.0996 | LR: 4.22e-04 | 1338.2ms/step | 14.9 samples/s +Step 11970/44000 | Loss: 3.1284 | LR: 4.21e-04 | 1337.8ms/step | 15.0 samples/s +Step 11980/44000 | Loss: 3.1023 | LR: 4.21e-04 | 1336.4ms/step | 15.0 samples/s +Step 11990/44000 | Loss: 3.0999 | LR: 4.21e-04 | 1338.5ms/step | 14.9 samples/s +Step 12000/44000 | Loss: 3.1178 | LR: 4.21e-04 | 1338.5ms/step | 14.9 samples/s +Validation at step 12000: Loss = 3.1218, Perplexity = 22.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-12000 +Step 12010/44000 | Loss: 3.1021 | LR: 4.21e-04 | 3894.4ms/step | 5.1 samples/s +Step 12020/44000 | Loss: 3.0836 | LR: 4.21e-04 | 1337.4ms/step | 15.0 samples/s +Step 12030/44000 | Loss: 3.1207 | LR: 4.21e-04 | 1337.1ms/step | 15.0 samples/s +Step 12040/44000 | Loss: 3.1235 | LR: 4.21e-04 | 1339.0ms/step | 14.9 samples/s +Step 12050/44000 | Loss: 3.1071 | LR: 4.20e-04 | 1337.6ms/step | 15.0 samples/s +Step 12060/44000 | Loss: 3.1003 | LR: 4.20e-04 | 1341.1ms/step | 14.9 samples/s +Step 12070/44000 | Loss: 3.1410 | LR: 4.20e-04 | 1336.2ms/step | 15.0 samples/s +Step 12080/44000 | Loss: 3.1154 | LR: 4.20e-04 | 1337.7ms/step | 15.0 samples/s +Step 12090/44000 | Loss: 3.1288 | LR: 4.20e-04 | 1336.3ms/step | 15.0 samples/s +Step 12100/44000 | Loss: 3.1198 | LR: 4.20e-04 | 1338.0ms/step | 14.9 samples/s +Step 12110/44000 | Loss: 3.1180 | LR: 4.20e-04 | 1340.3ms/step | 14.9 samples/s +Step 12120/44000 | Loss: 3.1188 | LR: 4.19e-04 | 1339.0ms/step | 14.9 samples/s +Step 12130/44000 | Loss: 3.1330 | LR: 4.19e-04 | 1337.0ms/step | 15.0 samples/s +Step 12140/44000 | Loss: 3.0831 | LR: 4.19e-04 | 1340.4ms/step | 14.9 samples/s +Step 12150/44000 | Loss: 3.0976 | LR: 4.19e-04 | 1339.5ms/step | 14.9 samples/s +Step 12160/44000 | Loss: 3.1299 | LR: 4.19e-04 | 1340.3ms/step | 14.9 samples/s +Step 12170/44000 | Loss: 3.1166 | LR: 4.19e-04 | 1337.8ms/step | 15.0 samples/s +Step 12180/44000 | Loss: 3.0941 | LR: 4.19e-04 | 1340.4ms/step | 14.9 samples/s +Step 12190/44000 | Loss: 3.1086 | LR: 4.19e-04 | 1335.9ms/step | 15.0 samples/s +Step 12200/44000 | Loss: 3.0742 | LR: 4.18e-04 | 1338.2ms/step | 14.9 samples/s +Step 12210/44000 | Loss: 3.1052 | LR: 4.18e-04 | 1335.7ms/step | 15.0 samples/s +Step 12220/44000 | Loss: 3.0803 | LR: 4.18e-04 | 1336.4ms/step | 15.0 samples/s +Step 12230/44000 | Loss: 3.0797 | LR: 4.18e-04 | 1339.0ms/step | 14.9 samples/s +Step 12240/44000 | Loss: 3.1335 | LR: 4.18e-04 | 1340.1ms/step | 14.9 samples/s +Step 12250/44000 | Loss: 3.1159 | LR: 4.18e-04 | 1338.6ms/step | 14.9 samples/s +Validation at step 12250: Loss = 3.1190, Perplexity = 22.62 +Step 12260/44000 | Loss: 3.0928 | LR: 4.18e-04 | 3580.2ms/step | 5.6 samples/s +Step 12270/44000 | Loss: 3.1196 | LR: 4.17e-04 | 1338.9ms/step | 14.9 samples/s +Step 12280/44000 | Loss: 3.0983 | LR: 4.17e-04 | 1336.8ms/step | 15.0 samples/s +Step 12290/44000 | Loss: 3.1150 | LR: 4.17e-04 | 1337.1ms/step | 15.0 samples/s +Step 12300/44000 | Loss: 3.1155 | LR: 4.17e-04 | 1337.1ms/step | 15.0 samples/s +Step 12310/44000 | Loss: 3.1218 | LR: 4.17e-04 | 1337.6ms/step | 15.0 samples/s +Step 12320/44000 | Loss: 3.1006 | LR: 4.17e-04 | 1340.1ms/step | 14.9 samples/s +Step 12330/44000 | Loss: 3.1280 | LR: 4.17e-04 | 1340.4ms/step | 14.9 samples/s +Step 12340/44000 | Loss: 3.1090 | LR: 4.17e-04 | 1337.7ms/step | 15.0 samples/s +Step 12350/44000 | Loss: 3.1245 | LR: 4.16e-04 | 1338.7ms/step | 14.9 samples/s +Step 12360/44000 | Loss: 3.1077 | LR: 4.16e-04 | 1335.7ms/step | 15.0 samples/s +Step 12370/44000 | Loss: 3.1033 | LR: 4.16e-04 | 1338.7ms/step | 14.9 samples/s +Step 12380/44000 | Loss: 3.0755 | LR: 4.16e-04 | 1340.4ms/step | 14.9 samples/s +Step 12390/44000 | Loss: 3.1096 | LR: 4.16e-04 | 1339.5ms/step | 14.9 samples/s +Step 12400/44000 | Loss: 3.0748 | LR: 4.16e-04 | 1334.7ms/step | 15.0 samples/s +Step 12410/44000 | Loss: 3.0984 | LR: 4.16e-04 | 1338.4ms/step | 14.9 samples/s +Step 12420/44000 | Loss: 3.1117 | LR: 4.15e-04 | 1337.5ms/step | 15.0 samples/s +Step 12430/44000 | Loss: 3.0914 | LR: 4.15e-04 | 1336.4ms/step | 15.0 samples/s +Step 12440/44000 | Loss: 3.1059 | LR: 4.15e-04 | 1336.0ms/step | 15.0 samples/s +Step 12450/44000 | Loss: 3.1008 | LR: 4.15e-04 | 1336.0ms/step | 15.0 samples/s +Step 12460/44000 | Loss: 3.1088 | LR: 4.15e-04 | 1335.3ms/step | 15.0 samples/s +Step 12470/44000 | Loss: 3.1011 | LR: 4.15e-04 | 1337.6ms/step | 15.0 samples/s +Step 12480/44000 | Loss: 3.0938 | LR: 4.15e-04 | 1333.8ms/step | 15.0 samples/s +Step 12490/44000 | Loss: 3.0745 | LR: 4.14e-04 | 1336.7ms/step | 15.0 samples/s +Step 12500/44000 | Loss: 3.0691 | LR: 4.14e-04 | 1337.8ms/step | 15.0 samples/s +Validation at step 12500: Loss = 3.1140, Perplexity = 22.51 +Step 12510/44000 | Loss: 3.1256 | LR: 4.14e-04 | 3582.7ms/step | 5.6 samples/s +Step 12520/44000 | Loss: 3.1080 | LR: 4.14e-04 | 1340.4ms/step | 14.9 samples/s +Step 12530/44000 | Loss: 3.1018 | LR: 4.14e-04 | 1337.1ms/step | 15.0 samples/s +Step 12540/44000 | Loss: 3.1013 | LR: 4.14e-04 | 1336.9ms/step | 15.0 samples/s +Step 12550/44000 | Loss: 3.1161 | LR: 4.14e-04 | 1339.5ms/step | 14.9 samples/s +Step 12560/44000 | Loss: 3.0799 | LR: 4.14e-04 | 1339.7ms/step | 14.9 samples/s +Step 12570/44000 | Loss: 3.0936 | LR: 4.13e-04 | 1336.2ms/step | 15.0 samples/s +Step 12580/44000 | Loss: 3.1367 | LR: 4.13e-04 | 1339.0ms/step | 14.9 samples/s +Step 12590/44000 | Loss: 3.0948 | LR: 4.13e-04 | 1337.2ms/step | 15.0 samples/s +Step 12600/44000 | Loss: 3.0999 | LR: 4.13e-04 | 1339.1ms/step | 14.9 samples/s +Step 12610/44000 | Loss: 3.1139 | LR: 4.13e-04 | 1338.0ms/step | 14.9 samples/s +Step 12620/44000 | Loss: 3.1476 | LR: 4.13e-04 | 1338.8ms/step | 14.9 samples/s +Step 12630/44000 | Loss: 3.1093 | LR: 4.13e-04 | 1338.9ms/step | 14.9 samples/s +Step 12640/44000 | Loss: 3.0989 | LR: 4.12e-04 | 1337.1ms/step | 15.0 samples/s +Step 12650/44000 | Loss: 3.1091 | LR: 4.12e-04 | 1335.7ms/step | 15.0 samples/s +Step 12660/44000 | Loss: 3.0976 | LR: 4.12e-04 | 1338.6ms/step | 14.9 samples/s +Step 12670/44000 | Loss: 3.1108 | LR: 4.12e-04 | 1339.6ms/step | 14.9 samples/s +Step 12680/44000 | Loss: 3.1138 | LR: 4.12e-04 | 1336.8ms/step | 15.0 samples/s +Step 12690/44000 | Loss: 3.1037 | LR: 4.12e-04 | 1335.5ms/step | 15.0 samples/s +Step 12700/44000 | Loss: 3.0983 | LR: 4.12e-04 | 1335.8ms/step | 15.0 samples/s +Step 12710/44000 | Loss: 3.0637 | LR: 4.11e-04 | 1337.2ms/step | 15.0 samples/s +Step 12720/44000 | Loss: 3.0619 | LR: 4.11e-04 | 1340.3ms/step | 14.9 samples/s +Step 12730/44000 | Loss: 3.0887 | LR: 4.11e-04 | 1336.4ms/step | 15.0 samples/s +Step 12740/44000 | Loss: 3.1071 | LR: 4.11e-04 | 1336.5ms/step | 15.0 samples/s +Step 12750/44000 | Loss: 3.0688 | LR: 4.11e-04 | 1340.1ms/step | 14.9 samples/s +Validation at step 12750: Loss = 3.1091, Perplexity = 22.40 +Step 12760/44000 | Loss: 3.0954 | LR: 4.11e-04 | 3580.5ms/step | 5.6 samples/s +Step 12770/44000 | Loss: 3.1050 | LR: 4.11e-04 | 1337.2ms/step | 15.0 samples/s +Step 12780/44000 | Loss: 3.0850 | LR: 4.10e-04 | 1335.2ms/step | 15.0 samples/s +Step 12790/44000 | Loss: 3.0903 | LR: 4.10e-04 | 1338.7ms/step | 14.9 samples/s +Step 12800/44000 | Loss: 3.1088 | LR: 4.10e-04 | 1333.5ms/step | 15.0 samples/s +Step 12810/44000 | Loss: 3.1098 | LR: 4.10e-04 | 1337.8ms/step | 14.9 samples/s +Step 12820/44000 | Loss: 3.0859 | LR: 4.10e-04 | 1335.3ms/step | 15.0 samples/s +Step 12830/44000 | Loss: 3.0967 | LR: 4.10e-04 | 1337.3ms/step | 15.0 samples/s +Step 12840/44000 | Loss: 3.0780 | LR: 4.10e-04 | 1338.4ms/step | 14.9 samples/s +Step 12850/44000 | Loss: 3.0963 | LR: 4.09e-04 | 1336.5ms/step | 15.0 samples/s +Step 12860/44000 | Loss: 3.0745 | LR: 4.09e-04 | 1340.1ms/step | 14.9 samples/s +Step 12870/44000 | Loss: 3.1033 | LR: 4.09e-04 | 1339.8ms/step | 14.9 samples/s +Step 12880/44000 | Loss: 3.0643 | LR: 4.09e-04 | 1339.0ms/step | 14.9 samples/s +Step 12890/44000 | Loss: 3.1185 | LR: 4.09e-04 | 1338.5ms/step | 14.9 samples/s +Step 12900/44000 | Loss: 3.0811 | LR: 4.09e-04 | 1342.0ms/step | 14.9 samples/s +Step 12910/44000 | Loss: 3.0960 | LR: 4.09e-04 | 1338.4ms/step | 14.9 samples/s +Step 12920/44000 | Loss: 3.0969 | LR: 4.09e-04 | 1338.6ms/step | 14.9 samples/s +Step 12930/44000 | Loss: 3.0970 | LR: 4.08e-04 | 1343.6ms/step | 14.9 samples/s +Step 12940/44000 | Loss: 3.1130 | LR: 4.08e-04 | 1341.2ms/step | 14.9 samples/s +Step 12950/44000 | Loss: 3.0968 | LR: 4.08e-04 | 1338.2ms/step | 14.9 samples/s +Step 12960/44000 | Loss: 3.1042 | LR: 4.08e-04 | 1339.2ms/step | 14.9 samples/s +Step 12970/44000 | Loss: 3.1125 | LR: 4.08e-04 | 1337.5ms/step | 15.0 samples/s +Step 12980/44000 | Loss: 3.1070 | LR: 4.08e-04 | 1339.1ms/step | 14.9 samples/s +Step 12990/44000 | Loss: 3.1002 | LR: 4.08e-04 | 1340.6ms/step | 14.9 samples/s +Step 13000/44000 | Loss: 3.0852 | LR: 4.07e-04 | 1340.0ms/step | 14.9 samples/s +Validation at step 13000: Loss = 3.1057, Perplexity = 22.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-13000 +Step 13010/44000 | Loss: 3.0930 | LR: 4.07e-04 | 3881.3ms/step | 5.2 samples/s +Step 13020/44000 | Loss: 3.0935 | LR: 4.07e-04 | 1337.3ms/step | 15.0 samples/s +Step 13030/44000 | Loss: 3.1012 | LR: 4.07e-04 | 1339.7ms/step | 14.9 samples/s +Step 13040/44000 | Loss: 3.0743 | LR: 4.07e-04 | 1342.2ms/step | 14.9 samples/s +Step 13050/44000 | Loss: 3.0599 | LR: 4.07e-04 | 1340.2ms/step | 14.9 samples/s +Step 13060/44000 | Loss: 3.1047 | LR: 4.07e-04 | 1338.7ms/step | 14.9 samples/s +Step 13070/44000 | Loss: 3.0981 | LR: 4.06e-04 | 1338.3ms/step | 14.9 samples/s +Step 13080/44000 | Loss: 3.1157 | LR: 4.06e-04 | 1340.1ms/step | 14.9 samples/s +Step 13090/44000 | Loss: 3.0836 | LR: 4.06e-04 | 1338.8ms/step | 14.9 samples/s +Step 13100/44000 | Loss: 3.0865 | LR: 4.06e-04 | 1339.1ms/step | 14.9 samples/s +Step 13110/44000 | Loss: 3.0985 | LR: 4.06e-04 | 1341.0ms/step | 14.9 samples/s +Step 13120/44000 | Loss: 3.1075 | LR: 4.06e-04 | 1338.1ms/step | 14.9 samples/s +Step 13130/44000 | Loss: 3.1081 | LR: 4.06e-04 | 1339.3ms/step | 14.9 samples/s +Step 13140/44000 | Loss: 3.1093 | LR: 4.05e-04 | 1338.6ms/step | 14.9 samples/s +Step 13150/44000 | Loss: 3.0919 | LR: 4.05e-04 | 1338.8ms/step | 14.9 samples/s +Step 13160/44000 | Loss: 3.0840 | LR: 4.05e-04 | 1339.1ms/step | 14.9 samples/s +Step 13170/44000 | Loss: 3.0888 | LR: 4.05e-04 | 1340.9ms/step | 14.9 samples/s +Step 13180/44000 | Loss: 3.1109 | LR: 4.05e-04 | 1336.2ms/step | 15.0 samples/s +Step 13190/44000 | Loss: 3.0866 | LR: 4.05e-04 | 1337.9ms/step | 14.9 samples/s +Step 13200/44000 | Loss: 3.1195 | LR: 4.05e-04 | 1337.1ms/step | 15.0 samples/s +Step 13210/44000 | Loss: 3.0721 | LR: 4.04e-04 | 1338.8ms/step | 14.9 samples/s +Step 13220/44000 | Loss: 3.0766 | LR: 4.04e-04 | 1339.8ms/step | 14.9 samples/s +Step 13230/44000 | Loss: 3.1064 | LR: 4.04e-04 | 1335.8ms/step | 15.0 samples/s +Step 13240/44000 | Loss: 3.0861 | LR: 4.04e-04 | 1335.1ms/step | 15.0 samples/s +Step 13250/44000 | Loss: 3.1020 | LR: 4.04e-04 | 1339.5ms/step | 14.9 samples/s +Validation at step 13250: Loss = 3.1017, Perplexity = 22.24 +Step 13260/44000 | Loss: 3.0939 | LR: 4.04e-04 | 3586.0ms/step | 5.6 samples/s +Step 13270/44000 | Loss: 3.0841 | LR: 4.04e-04 | 1337.7ms/step | 15.0 samples/s +Step 13280/44000 | Loss: 3.1103 | LR: 4.03e-04 | 1337.4ms/step | 15.0 samples/s +Step 13290/44000 | Loss: 3.0972 | LR: 4.03e-04 | 1337.8ms/step | 15.0 samples/s +Step 13300/44000 | Loss: 3.0586 | LR: 4.03e-04 | 1335.5ms/step | 15.0 samples/s +Step 13310/44000 | Loss: 3.0900 | LR: 4.03e-04 | 1335.6ms/step | 15.0 samples/s +Step 13320/44000 | Loss: 3.0729 | LR: 4.03e-04 | 1336.5ms/step | 15.0 samples/s +Step 13330/44000 | Loss: 3.0801 | LR: 4.03e-04 | 1333.6ms/step | 15.0 samples/s +Step 13340/44000 | Loss: 3.0877 | LR: 4.03e-04 | 1338.2ms/step | 14.9 samples/s +Step 13350/44000 | Loss: 3.0750 | LR: 4.02e-04 | 1339.8ms/step | 14.9 samples/s +Step 13360/44000 | Loss: 3.0873 | LR: 4.02e-04 | 1337.0ms/step | 15.0 samples/s +Step 13370/44000 | Loss: 3.1176 | LR: 4.02e-04 | 1337.4ms/step | 15.0 samples/s +Step 13380/44000 | Loss: 3.1103 | LR: 4.02e-04 | 1333.5ms/step | 15.0 samples/s +Step 13390/44000 | Loss: 3.0798 | LR: 4.02e-04 | 1336.8ms/step | 15.0 samples/s +Step 13400/44000 | Loss: 3.0800 | LR: 4.02e-04 | 1336.4ms/step | 15.0 samples/s +Step 13410/44000 | Loss: 3.0740 | LR: 4.02e-04 | 1337.8ms/step | 14.9 samples/s +Step 13420/44000 | Loss: 3.0432 | LR: 4.01e-04 | 1337.3ms/step | 15.0 samples/s +Step 13430/44000 | Loss: 3.0994 | LR: 4.01e-04 | 1336.2ms/step | 15.0 samples/s +Step 13440/44000 | Loss: 3.0898 | LR: 4.01e-04 | 1336.7ms/step | 15.0 samples/s +Step 13450/44000 | Loss: 3.0996 | LR: 4.01e-04 | 1336.9ms/step | 15.0 samples/s +Step 13460/44000 | Loss: 3.0774 | LR: 4.01e-04 | 1336.3ms/step | 15.0 samples/s +Step 13470/44000 | Loss: 3.0896 | LR: 4.01e-04 | 1336.0ms/step | 15.0 samples/s +Step 13480/44000 | Loss: 3.0641 | LR: 4.01e-04 | 1335.5ms/step | 15.0 samples/s +Step 13490/44000 | Loss: 3.0906 | LR: 4.00e-04 | 1334.8ms/step | 15.0 samples/s +Step 13500/44000 | Loss: 3.0924 | LR: 4.00e-04 | 1336.5ms/step | 15.0 samples/s +Validation at step 13500: Loss = 3.0967, Perplexity = 22.12 +Step 13510/44000 | Loss: 3.0785 | LR: 4.00e-04 | 3576.3ms/step | 5.6 samples/s +Step 13520/44000 | Loss: 3.0775 | LR: 4.00e-04 | 1335.7ms/step | 15.0 samples/s +Step 13530/44000 | Loss: 3.0728 | LR: 4.00e-04 | 1336.7ms/step | 15.0 samples/s +Step 13540/44000 | Loss: 3.0645 | LR: 4.00e-04 | 1335.5ms/step | 15.0 samples/s +Step 13550/44000 | Loss: 3.0816 | LR: 4.00e-04 | 1336.0ms/step | 15.0 samples/s +Step 13560/44000 | Loss: 3.0750 | LR: 3.99e-04 | 1336.5ms/step | 15.0 samples/s +Step 13570/44000 | Loss: 3.1108 | LR: 3.99e-04 | 1338.3ms/step | 14.9 samples/s +Step 13580/44000 | Loss: 3.0898 | LR: 3.99e-04 | 1334.0ms/step | 15.0 samples/s +Step 13590/44000 | Loss: 3.0672 | LR: 3.99e-04 | 1335.2ms/step | 15.0 samples/s +Step 13600/44000 | Loss: 3.0780 | LR: 3.99e-04 | 1337.3ms/step | 15.0 samples/s +Step 13610/44000 | Loss: 3.0698 | LR: 3.99e-04 | 1336.5ms/step | 15.0 samples/s +Step 13620/44000 | Loss: 3.1019 | LR: 3.98e-04 | 1334.0ms/step | 15.0 samples/s +Step 13630/44000 | Loss: 3.0922 | LR: 3.98e-04 | 1335.6ms/step | 15.0 samples/s +Step 13640/44000 | Loss: 3.0673 | LR: 3.98e-04 | 1336.7ms/step | 15.0 samples/s +Step 13650/44000 | Loss: 3.0860 | LR: 3.98e-04 | 1338.6ms/step | 14.9 samples/s +Step 13660/44000 | Loss: 3.0735 | LR: 3.98e-04 | 1335.8ms/step | 15.0 samples/s +Step 13670/44000 | Loss: 3.0595 | LR: 3.98e-04 | 1335.9ms/step | 15.0 samples/s +Step 13680/44000 | Loss: 3.0727 | LR: 3.98e-04 | 1336.9ms/step | 15.0 samples/s +Step 13690/44000 | Loss: 3.0910 | LR: 3.97e-04 | 1338.1ms/step | 14.9 samples/s +Step 13700/44000 | Loss: 3.0761 | LR: 3.97e-04 | 1336.8ms/step | 15.0 samples/s +Step 13710/44000 | Loss: 3.0866 | LR: 3.97e-04 | 1338.5ms/step | 14.9 samples/s +Step 13720/44000 | Loss: 3.0702 | LR: 3.97e-04 | 1338.1ms/step | 14.9 samples/s +Step 13730/44000 | Loss: 3.0863 | LR: 3.97e-04 | 1339.1ms/step | 14.9 samples/s +Step 13740/44000 | Loss: 3.0934 | LR: 3.97e-04 | 1337.4ms/step | 15.0 samples/s +Step 13750/44000 | Loss: 3.1235 | LR: 3.97e-04 | 1335.9ms/step | 15.0 samples/s +Validation at step 13750: Loss = 3.0944, Perplexity = 22.07 +Step 13760/44000 | Loss: 3.1128 | LR: 3.96e-04 | 3584.3ms/step | 5.6 samples/s +Step 13770/44000 | Loss: 3.0855 | LR: 3.96e-04 | 1340.4ms/step | 14.9 samples/s +Step 13780/44000 | Loss: 3.0795 | LR: 3.96e-04 | 1339.6ms/step | 14.9 samples/s +Step 13790/44000 | Loss: 3.1150 | LR: 3.96e-04 | 1338.1ms/step | 14.9 samples/s +Step 13800/44000 | Loss: 3.0717 | LR: 3.96e-04 | 1333.8ms/step | 15.0 samples/s +Step 13810/44000 | Loss: 3.0643 | LR: 3.96e-04 | 1335.1ms/step | 15.0 samples/s +Step 13820/44000 | Loss: 3.0787 | LR: 3.96e-04 | 1338.2ms/step | 14.9 samples/s +Step 13830/44000 | Loss: 3.0635 | LR: 3.95e-04 | 1339.5ms/step | 14.9 samples/s +Step 13840/44000 | Loss: 3.0541 | LR: 3.95e-04 | 1342.1ms/step | 14.9 samples/s +Step 13850/44000 | Loss: 3.0741 | LR: 3.95e-04 | 1338.7ms/step | 14.9 samples/s +Step 13860/44000 | Loss: 3.0825 | LR: 3.95e-04 | 1337.2ms/step | 15.0 samples/s +Step 13870/44000 | Loss: 3.0810 | LR: 3.95e-04 | 1338.4ms/step | 14.9 samples/s +Step 13880/44000 | Loss: 3.0979 | LR: 3.95e-04 | 1337.9ms/step | 14.9 samples/s +Step 13890/44000 | Loss: 3.0657 | LR: 3.95e-04 | 1337.9ms/step | 14.9 samples/s +Step 13900/44000 | Loss: 3.1153 | LR: 3.94e-04 | 1339.9ms/step | 14.9 samples/s +Step 13910/44000 | Loss: 3.0884 | LR: 3.94e-04 | 1340.5ms/step | 14.9 samples/s +Step 13920/44000 | Loss: 3.0692 | LR: 3.94e-04 | 1339.9ms/step | 14.9 samples/s +Step 13930/44000 | Loss: 3.0820 | LR: 3.94e-04 | 1337.0ms/step | 15.0 samples/s +Step 13940/44000 | Loss: 3.0618 | LR: 3.94e-04 | 1337.9ms/step | 14.9 samples/s +Step 13950/44000 | Loss: 3.0633 | LR: 3.94e-04 | 1332.1ms/step | 15.0 samples/s +Step 13960/44000 | Loss: 3.0826 | LR: 3.93e-04 | 1335.3ms/step | 15.0 samples/s +Step 13970/44000 | Loss: 3.0670 | LR: 3.93e-04 | 1335.9ms/step | 15.0 samples/s +Step 13980/44000 | Loss: 3.0659 | LR: 3.93e-04 | 1338.3ms/step | 14.9 samples/s +Step 13990/44000 | Loss: 3.0670 | LR: 3.93e-04 | 1339.5ms/step | 14.9 samples/s +Step 14000/44000 | Loss: 3.0867 | LR: 3.93e-04 | 1337.3ms/step | 15.0 samples/s +Validation at step 14000: Loss = 3.0898, Perplexity = 21.97 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-14000 +Step 14010/44000 | Loss: 3.0889 | LR: 3.93e-04 | 3870.0ms/step | 5.2 samples/s +Step 14020/44000 | Loss: 3.0623 | LR: 3.93e-04 | 1333.8ms/step | 15.0 samples/s +Step 14030/44000 | Loss: 3.0319 | LR: 3.92e-04 | 1333.0ms/step | 15.0 samples/s +Step 14040/44000 | Loss: 3.0487 | LR: 3.92e-04 | 1333.4ms/step | 15.0 samples/s +Step 14050/44000 | Loss: 3.0401 | LR: 3.92e-04 | 1333.9ms/step | 15.0 samples/s +Step 14060/44000 | Loss: 3.0975 | LR: 3.92e-04 | 1332.6ms/step | 15.0 samples/s +Step 14070/44000 | Loss: 3.0897 | LR: 3.92e-04 | 1333.0ms/step | 15.0 samples/s +Step 14080/44000 | Loss: 3.0790 | LR: 3.92e-04 | 1332.4ms/step | 15.0 samples/s +Step 14090/44000 | Loss: 3.0625 | LR: 3.92e-04 | 1334.8ms/step | 15.0 samples/s +Step 14100/44000 | Loss: 3.0898 | LR: 3.91e-04 | 1333.1ms/step | 15.0 samples/s +Step 14110/44000 | Loss: 3.0846 | LR: 3.91e-04 | 1333.1ms/step | 15.0 samples/s +Step 14120/44000 | Loss: 3.0768 | LR: 3.91e-04 | 1331.4ms/step | 15.0 samples/s +Step 14130/44000 | Loss: 3.1001 | LR: 3.91e-04 | 1334.3ms/step | 15.0 samples/s +Step 14140/44000 | Loss: 3.0863 | LR: 3.91e-04 | 1331.4ms/step | 15.0 samples/s +Step 14150/44000 | Loss: 3.0574 | LR: 3.91e-04 | 1332.3ms/step | 15.0 samples/s +Step 14160/44000 | Loss: 3.0689 | LR: 3.90e-04 | 1331.8ms/step | 15.0 samples/s +Step 14170/44000 | Loss: 3.0931 | LR: 3.90e-04 | 1331.6ms/step | 15.0 samples/s +Step 14180/44000 | Loss: 3.0814 | LR: 3.90e-04 | 1330.5ms/step | 15.0 samples/s +Step 14190/44000 | Loss: 3.0760 | LR: 3.90e-04 | 1334.7ms/step | 15.0 samples/s +Step 14200/44000 | Loss: 3.0857 | LR: 3.90e-04 | 1331.3ms/step | 15.0 samples/s +Step 14210/44000 | Loss: 3.0704 | LR: 3.90e-04 | 1331.3ms/step | 15.0 samples/s +Step 14220/44000 | Loss: 3.0712 | LR: 3.90e-04 | 1332.5ms/step | 15.0 samples/s +Step 14230/44000 | Loss: 3.0761 | LR: 3.89e-04 | 1334.4ms/step | 15.0 samples/s +Step 14240/44000 | Loss: 3.0569 | LR: 3.89e-04 | 1334.4ms/step | 15.0 samples/s +Step 14250/44000 | Loss: 3.0707 | LR: 3.89e-04 | 1328.9ms/step | 15.0 samples/s +Validation at step 14250: Loss = 3.0863, Perplexity = 21.90 +Step 14260/44000 | Loss: 3.1026 | LR: 3.89e-04 | 3569.3ms/step | 5.6 samples/s +Step 14270/44000 | Loss: 3.0952 | LR: 3.89e-04 | 1330.7ms/step | 15.0 samples/s +Step 14280/44000 | Loss: 3.0930 | LR: 3.89e-04 | 1332.1ms/step | 15.0 samples/s +Step 14290/44000 | Loss: 3.0787 | LR: 3.89e-04 | 1333.8ms/step | 15.0 samples/s +Step 14300/44000 | Loss: 3.0788 | LR: 3.88e-04 | 1328.0ms/step | 15.1 samples/s +Step 14310/44000 | Loss: 3.0761 | LR: 3.88e-04 | 1328.2ms/step | 15.1 samples/s +Step 14320/44000 | Loss: 3.0762 | LR: 3.88e-04 | 1330.8ms/step | 15.0 samples/s +Step 14330/44000 | Loss: 3.0886 | LR: 3.88e-04 | 1333.0ms/step | 15.0 samples/s +Step 14340/44000 | Loss: 3.0859 | LR: 3.88e-04 | 1332.5ms/step | 15.0 samples/s +Step 14350/44000 | Loss: 3.0687 | LR: 3.88e-04 | 1332.0ms/step | 15.0 samples/s +Step 14360/44000 | Loss: 3.0859 | LR: 3.87e-04 | 1331.2ms/step | 15.0 samples/s +Step 14370/44000 | Loss: 3.0819 | LR: 3.87e-04 | 1331.4ms/step | 15.0 samples/s +Step 14380/44000 | Loss: 3.0640 | LR: 3.87e-04 | 1330.3ms/step | 15.0 samples/s +Step 14390/44000 | Loss: 3.0696 | LR: 3.87e-04 | 1330.1ms/step | 15.0 samples/s +Step 14400/44000 | Loss: 3.0722 | LR: 3.87e-04 | 1331.6ms/step | 15.0 samples/s +Step 14410/44000 | Loss: 3.0526 | LR: 3.87e-04 | 1330.9ms/step | 15.0 samples/s +Step 14420/44000 | Loss: 3.0753 | LR: 3.87e-04 | 1328.8ms/step | 15.1 samples/s +Step 14430/44000 | Loss: 3.0423 | LR: 3.86e-04 | 1331.3ms/step | 15.0 samples/s +Step 14440/44000 | Loss: 3.0832 | LR: 3.86e-04 | 1332.4ms/step | 15.0 samples/s +Step 14450/44000 | Loss: 3.0752 | LR: 3.86e-04 | 1333.7ms/step | 15.0 samples/s +Step 14460/44000 | Loss: 3.0859 | LR: 3.86e-04 | 1333.8ms/step | 15.0 samples/s +Step 14470/44000 | Loss: 3.0815 | LR: 3.86e-04 | 1334.2ms/step | 15.0 samples/s +Step 14480/44000 | Loss: 3.0458 | LR: 3.86e-04 | 1332.6ms/step | 15.0 samples/s +Step 14490/44000 | Loss: 3.0592 | LR: 3.85e-04 | 1333.8ms/step | 15.0 samples/s +Step 14500/44000 | Loss: 3.0880 | LR: 3.85e-04 | 1333.0ms/step | 15.0 samples/s +Validation at step 14500: Loss = 3.0824, Perplexity = 21.81 +Step 14510/44000 | Loss: 3.0557 | LR: 3.85e-04 | 3565.5ms/step | 5.6 samples/s +Step 14520/44000 | Loss: 3.0885 | LR: 3.85e-04 | 1331.6ms/step | 15.0 samples/s +Step 14530/44000 | Loss: 3.0677 | LR: 3.85e-04 | 1333.1ms/step | 15.0 samples/s +Step 14540/44000 | Loss: 3.0755 | LR: 3.85e-04 | 1332.3ms/step | 15.0 samples/s +Step 14550/44000 | Loss: 3.0751 | LR: 3.85e-04 | 1330.3ms/step | 15.0 samples/s +Step 14560/44000 | Loss: 3.0830 | LR: 3.84e-04 | 1332.8ms/step | 15.0 samples/s +Step 14570/44000 | Loss: 3.0838 | LR: 3.84e-04 | 1332.9ms/step | 15.0 samples/s +Step 14580/44000 | Loss: 3.0705 | LR: 3.84e-04 | 1331.8ms/step | 15.0 samples/s +Step 14590/44000 | Loss: 3.0818 | LR: 3.84e-04 | 1332.3ms/step | 15.0 samples/s +Step 14600/44000 | Loss: 3.0740 | LR: 3.84e-04 | 1333.1ms/step | 15.0 samples/s +Step 14610/44000 | Loss: 3.0736 | LR: 3.84e-04 | 1332.6ms/step | 15.0 samples/s +Step 14620/44000 | Loss: 3.0851 | LR: 3.83e-04 | 1332.8ms/step | 15.0 samples/s +Step 14630/44000 | Loss: 3.0609 | LR: 3.83e-04 | 1332.3ms/step | 15.0 samples/s +Step 14640/44000 | Loss: 3.0609 | LR: 3.83e-04 | 1330.0ms/step | 15.0 samples/s +Step 14650/44000 | Loss: 3.0708 | LR: 3.83e-04 | 1336.5ms/step | 15.0 samples/s +Step 14660/44000 | Loss: 3.0853 | LR: 3.83e-04 | 1331.8ms/step | 15.0 samples/s +Step 14670/44000 | Loss: 3.0805 | LR: 3.83e-04 | 1334.8ms/step | 15.0 samples/s +Step 14680/44000 | Loss: 3.0454 | LR: 3.83e-04 | 1332.4ms/step | 15.0 samples/s +Step 14690/44000 | Loss: 3.0652 | LR: 3.82e-04 | 1334.9ms/step | 15.0 samples/s +Step 14700/44000 | Loss: 3.0814 | LR: 3.82e-04 | 1328.9ms/step | 15.0 samples/s +Step 14710/44000 | Loss: 3.0708 | LR: 3.82e-04 | 1331.7ms/step | 15.0 samples/s +Step 14720/44000 | Loss: 3.0604 | LR: 3.82e-04 | 1328.5ms/step | 15.1 samples/s +Step 14730/44000 | Loss: 3.0755 | LR: 3.82e-04 | 1331.4ms/step | 15.0 samples/s +Step 14740/44000 | Loss: 3.0650 | LR: 3.82e-04 | 1333.4ms/step | 15.0 samples/s +Step 14750/44000 | Loss: 3.0941 | LR: 3.81e-04 | 1332.2ms/step | 15.0 samples/s +Validation at step 14750: Loss = 3.0790, Perplexity = 21.74 +Step 14760/44000 | Loss: 3.0667 | LR: 3.81e-04 | 3563.8ms/step | 5.6 samples/s +Step 14770/44000 | Loss: 3.0778 | LR: 3.81e-04 | 1329.9ms/step | 15.0 samples/s +Step 14780/44000 | Loss: 3.0876 | LR: 3.81e-04 | 1334.4ms/step | 15.0 samples/s +Step 14790/44000 | Loss: 3.0732 | LR: 3.81e-04 | 1331.9ms/step | 15.0 samples/s +Step 14800/44000 | Loss: 3.0412 | LR: 3.81e-04 | 1332.7ms/step | 15.0 samples/s +Step 14810/44000 | Loss: 3.0599 | LR: 3.81e-04 | 1330.9ms/step | 15.0 samples/s +Step 14820/44000 | Loss: 3.0688 | LR: 3.80e-04 | 1333.0ms/step | 15.0 samples/s +Step 14830/44000 | Loss: 3.0743 | LR: 3.80e-04 | 1332.8ms/step | 15.0 samples/s +Step 14840/44000 | Loss: 3.0616 | LR: 3.80e-04 | 1334.7ms/step | 15.0 samples/s +Step 14850/44000 | Loss: 3.0731 | LR: 3.80e-04 | 1333.5ms/step | 15.0 samples/s +Step 14860/44000 | Loss: 3.0547 | LR: 3.80e-04 | 1334.3ms/step | 15.0 samples/s +Step 14870/44000 | Loss: 3.0569 | LR: 3.80e-04 | 1335.1ms/step | 15.0 samples/s +Step 14880/44000 | Loss: 3.0532 | LR: 3.79e-04 | 1330.0ms/step | 15.0 samples/s +Step 14890/44000 | Loss: 3.0536 | LR: 3.79e-04 | 1332.6ms/step | 15.0 samples/s +Step 14900/44000 | Loss: 3.0609 | LR: 3.79e-04 | 1332.9ms/step | 15.0 samples/s +Step 14910/44000 | Loss: 3.0828 | LR: 3.79e-04 | 1333.2ms/step | 15.0 samples/s +Step 14920/44000 | Loss: 3.0820 | LR: 3.79e-04 | 1334.5ms/step | 15.0 samples/s +Step 14930/44000 | Loss: 3.0860 | LR: 3.79e-04 | 1332.8ms/step | 15.0 samples/s +Step 14940/44000 | Loss: 3.0756 | LR: 3.79e-04 | 1332.6ms/step | 15.0 samples/s +Step 14950/44000 | Loss: 3.1017 | LR: 3.78e-04 | 1331.2ms/step | 15.0 samples/s +Step 14960/44000 | Loss: 3.0617 | LR: 3.78e-04 | 1331.1ms/step | 15.0 samples/s +Step 14970/44000 | Loss: 3.0643 | LR: 3.78e-04 | 1331.3ms/step | 15.0 samples/s +Step 14980/44000 | Loss: 3.0656 | LR: 3.78e-04 | 1334.9ms/step | 15.0 samples/s +Step 14990/44000 | Loss: 3.0531 | LR: 3.78e-04 | 1331.8ms/step | 15.0 samples/s +Step 15000/44000 | Loss: 3.0835 | LR: 3.78e-04 | 1333.7ms/step | 15.0 samples/s +Validation at step 15000: Loss = 3.0761, Perplexity = 21.67 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-15000 +Step 15010/44000 | Loss: 3.0396 | LR: 3.77e-04 | 3883.7ms/step | 5.1 samples/s +Step 15020/44000 | Loss: 3.0408 | LR: 3.77e-04 | 1336.7ms/step | 15.0 samples/s +Step 15030/44000 | Loss: 3.0514 | LR: 3.77e-04 | 1341.2ms/step | 14.9 samples/s +Step 15040/44000 | Loss: 3.0605 | LR: 3.77e-04 | 1342.4ms/step | 14.9 samples/s +Step 15050/44000 | Loss: 3.0702 | LR: 3.77e-04 | 1341.0ms/step | 14.9 samples/s +Step 15060/44000 | Loss: 3.0693 | LR: 3.77e-04 | 1336.8ms/step | 15.0 samples/s +Step 15070/44000 | Loss: 3.0475 | LR: 3.77e-04 | 1341.3ms/step | 14.9 samples/s +Step 15080/44000 | Loss: 3.0914 | LR: 3.76e-04 | 1343.6ms/step | 14.9 samples/s +Step 15090/44000 | Loss: 3.0590 | LR: 3.76e-04 | 1339.1ms/step | 14.9 samples/s +Step 15100/44000 | Loss: 3.0525 | LR: 3.76e-04 | 1337.9ms/step | 14.9 samples/s +Step 15110/44000 | Loss: 3.0653 | LR: 3.76e-04 | 1341.1ms/step | 14.9 samples/s +Step 15120/44000 | Loss: 3.0593 | LR: 3.76e-04 | 1340.0ms/step | 14.9 samples/s +Step 15130/44000 | Loss: 3.0587 | LR: 3.76e-04 | 1341.0ms/step | 14.9 samples/s +Step 15140/44000 | Loss: 3.0670 | LR: 3.75e-04 | 1342.3ms/step | 14.9 samples/s +Step 15150/44000 | Loss: 3.0757 | LR: 3.75e-04 | 1338.8ms/step | 14.9 samples/s +Step 15160/44000 | Loss: 3.0798 | LR: 3.75e-04 | 1339.2ms/step | 14.9 samples/s +Step 15170/44000 | Loss: 3.0594 | LR: 3.75e-04 | 1341.7ms/step | 14.9 samples/s +Step 15180/44000 | Loss: 3.0499 | LR: 3.75e-04 | 1341.3ms/step | 14.9 samples/s +Step 15190/44000 | Loss: 3.0522 | LR: 3.75e-04 | 1339.1ms/step | 14.9 samples/s +Step 15200/44000 | Loss: 3.0702 | LR: 3.74e-04 | 1341.3ms/step | 14.9 samples/s +Step 15210/44000 | Loss: 3.0829 | LR: 3.74e-04 | 1339.3ms/step | 14.9 samples/s +Step 15220/44000 | Loss: 3.0879 | LR: 3.74e-04 | 1340.3ms/step | 14.9 samples/s +Step 15230/44000 | Loss: 3.0721 | LR: 3.74e-04 | 1338.7ms/step | 14.9 samples/s +Step 15240/44000 | Loss: 3.0506 | LR: 3.74e-04 | 1336.3ms/step | 15.0 samples/s +Step 15250/44000 | Loss: 3.0576 | LR: 3.74e-04 | 1337.5ms/step | 15.0 samples/s +Validation at step 15250: Loss = 3.0721, Perplexity = 21.59 +Step 15260/44000 | Loss: 3.0403 | LR: 3.74e-04 | 3582.4ms/step | 5.6 samples/s +Step 15270/44000 | Loss: 3.0624 | LR: 3.73e-04 | 1340.7ms/step | 14.9 samples/s +Step 15280/44000 | Loss: 3.0455 | LR: 3.73e-04 | 1337.5ms/step | 15.0 samples/s +Step 15290/44000 | Loss: 3.0214 | LR: 3.73e-04 | 1336.6ms/step | 15.0 samples/s +Step 15300/44000 | Loss: 3.0570 | LR: 3.73e-04 | 1339.0ms/step | 14.9 samples/s +Step 15310/44000 | Loss: 3.0679 | LR: 3.73e-04 | 1335.5ms/step | 15.0 samples/s +Step 15320/44000 | Loss: 3.0526 | LR: 3.73e-04 | 1339.3ms/step | 14.9 samples/s +Step 15330/44000 | Loss: 3.0671 | LR: 3.72e-04 | 1339.4ms/step | 14.9 samples/s +Step 15340/44000 | Loss: 3.0635 | LR: 3.72e-04 | 1338.2ms/step | 14.9 samples/s +Step 15350/44000 | Loss: 3.0834 | LR: 3.72e-04 | 1339.6ms/step | 14.9 samples/s +Step 15360/44000 | Loss: 3.0782 | LR: 3.72e-04 | 1338.7ms/step | 14.9 samples/s +Step 15370/44000 | Loss: 3.0644 | LR: 3.72e-04 | 1337.5ms/step | 15.0 samples/s +Step 15380/44000 | Loss: 3.0499 | LR: 3.72e-04 | 1337.6ms/step | 15.0 samples/s +Step 15390/44000 | Loss: 3.0872 | LR: 3.71e-04 | 1334.6ms/step | 15.0 samples/s +Step 15400/44000 | Loss: 3.0358 | LR: 3.71e-04 | 1335.1ms/step | 15.0 samples/s +Step 15410/44000 | Loss: 3.0569 | LR: 3.71e-04 | 1334.6ms/step | 15.0 samples/s +Step 15420/44000 | Loss: 3.0742 | LR: 3.71e-04 | 1335.5ms/step | 15.0 samples/s +Step 15430/44000 | Loss: 3.0671 | LR: 3.71e-04 | 1339.2ms/step | 14.9 samples/s +Step 15440/44000 | Loss: 3.0618 | LR: 3.71e-04 | 1337.1ms/step | 15.0 samples/s +Step 15450/44000 | Loss: 3.0767 | LR: 3.71e-04 | 1338.2ms/step | 14.9 samples/s +Step 15460/44000 | Loss: 3.0541 | LR: 3.70e-04 | 1337.7ms/step | 15.0 samples/s +Step 15470/44000 | Loss: 3.0701 | LR: 3.70e-04 | 1334.6ms/step | 15.0 samples/s +Step 15480/44000 | Loss: 3.0230 | LR: 3.70e-04 | 1334.6ms/step | 15.0 samples/s +Step 15490/44000 | Loss: 3.0533 | LR: 3.70e-04 | 1341.3ms/step | 14.9 samples/s +Step 15500/44000 | Loss: 3.0424 | LR: 3.70e-04 | 1338.8ms/step | 14.9 samples/s +Validation at step 15500: Loss = 3.0697, Perplexity = 21.54 +Step 15510/44000 | Loss: 3.0823 | LR: 3.70e-04 | 3585.5ms/step | 5.6 samples/s +Step 15520/44000 | Loss: 3.0681 | LR: 3.69e-04 | 1337.4ms/step | 15.0 samples/s +Step 15530/44000 | Loss: 3.0444 | LR: 3.69e-04 | 1337.1ms/step | 15.0 samples/s +Step 15540/44000 | Loss: 3.0612 | LR: 3.69e-04 | 1339.3ms/step | 14.9 samples/s +Step 15550/44000 | Loss: 3.0767 | LR: 3.69e-04 | 1339.2ms/step | 14.9 samples/s +Step 15560/44000 | Loss: 3.0635 | LR: 3.69e-04 | 1337.8ms/step | 14.9 samples/s +Step 15570/44000 | Loss: 3.0770 | LR: 3.69e-04 | 1336.2ms/step | 15.0 samples/s +Step 15580/44000 | Loss: 3.0402 | LR: 3.68e-04 | 1336.8ms/step | 15.0 samples/s +Step 15590/44000 | Loss: 3.0346 | LR: 3.68e-04 | 1336.6ms/step | 15.0 samples/s +Step 15600/44000 | Loss: 3.0766 | LR: 3.68e-04 | 1336.7ms/step | 15.0 samples/s +Step 15610/44000 | Loss: 3.0420 | LR: 3.68e-04 | 1338.0ms/step | 14.9 samples/s +Step 15620/44000 | Loss: 3.0317 | LR: 3.68e-04 | 1335.2ms/step | 15.0 samples/s +Step 15630/44000 | Loss: 3.0513 | LR: 3.68e-04 | 1335.7ms/step | 15.0 samples/s +Step 15640/44000 | Loss: 3.0493 | LR: 3.67e-04 | 1334.7ms/step | 15.0 samples/s +Step 15650/44000 | Loss: 3.0364 | LR: 3.67e-04 | 1335.7ms/step | 15.0 samples/s +Step 15660/44000 | Loss: 3.0624 | LR: 3.67e-04 | 1337.1ms/step | 15.0 samples/s +Step 15670/44000 | Loss: 3.0469 | LR: 3.67e-04 | 1337.7ms/step | 15.0 samples/s +Step 15680/44000 | Loss: 3.0639 | LR: 3.67e-04 | 1335.6ms/step | 15.0 samples/s +Step 15690/44000 | Loss: 3.0486 | LR: 3.67e-04 | 1336.6ms/step | 15.0 samples/s +Step 15700/44000 | Loss: 3.0264 | LR: 3.67e-04 | 1336.8ms/step | 15.0 samples/s +Step 15710/44000 | Loss: 3.0435 | LR: 3.66e-04 | 1336.9ms/step | 15.0 samples/s +Step 15720/44000 | Loss: 3.0454 | LR: 3.66e-04 | 1335.1ms/step | 15.0 samples/s +Step 15730/44000 | Loss: 3.0644 | LR: 3.66e-04 | 1337.8ms/step | 14.9 samples/s +Step 15740/44000 | Loss: 3.0519 | LR: 3.66e-04 | 1335.5ms/step | 15.0 samples/s +Step 15750/44000 | Loss: 3.0462 | LR: 3.66e-04 | 1336.7ms/step | 15.0 samples/s +Validation at step 15750: Loss = 3.0676, Perplexity = 21.49 +Step 15760/44000 | Loss: 3.0287 | LR: 3.66e-04 | 3572.3ms/step | 5.6 samples/s +Step 15770/44000 | Loss: 3.0829 | LR: 3.65e-04 | 1332.8ms/step | 15.0 samples/s +Step 15780/44000 | Loss: 3.0504 | LR: 3.65e-04 | 1329.5ms/step | 15.0 samples/s +Step 15790/44000 | Loss: 3.0764 | LR: 3.65e-04 | 1330.4ms/step | 15.0 samples/s +Step 15800/44000 | Loss: 3.0318 | LR: 3.65e-04 | 1331.9ms/step | 15.0 samples/s +Step 15810/44000 | Loss: 3.0623 | LR: 3.65e-04 | 1332.2ms/step | 15.0 samples/s +Step 15820/44000 | Loss: 3.0167 | LR: 3.65e-04 | 1330.5ms/step | 15.0 samples/s +Step 15830/44000 | Loss: 3.0467 | LR: 3.64e-04 | 1333.8ms/step | 15.0 samples/s +Step 15840/44000 | Loss: 3.0338 | LR: 3.64e-04 | 1331.5ms/step | 15.0 samples/s +Step 15850/44000 | Loss: 3.0482 | LR: 3.64e-04 | 1333.9ms/step | 15.0 samples/s +Step 15860/44000 | Loss: 3.0528 | LR: 3.64e-04 | 1334.3ms/step | 15.0 samples/s +Step 15870/44000 | Loss: 3.0642 | LR: 3.64e-04 | 1333.4ms/step | 15.0 samples/s +Step 15880/44000 | Loss: 3.0435 | LR: 3.64e-04 | 1335.0ms/step | 15.0 samples/s +Step 15890/44000 | Loss: 2.9896 | LR: 3.63e-04 | 1333.9ms/step | 15.0 samples/s +Step 15900/44000 | Loss: 3.0274 | LR: 3.63e-04 | 1332.8ms/step | 15.0 samples/s +Step 15910/44000 | Loss: 3.0603 | LR: 3.63e-04 | 1335.8ms/step | 15.0 samples/s +Step 15920/44000 | Loss: 3.0604 | LR: 3.63e-04 | 1334.7ms/step | 15.0 samples/s +Step 15930/44000 | Loss: 3.0613 | LR: 3.63e-04 | 1332.6ms/step | 15.0 samples/s +Step 15940/44000 | Loss: 3.0663 | LR: 3.63e-04 | 1329.9ms/step | 15.0 samples/s +Step 15950/44000 | Loss: 3.0610 | LR: 3.62e-04 | 1333.8ms/step | 15.0 samples/s +Step 15960/44000 | Loss: 3.0407 | LR: 3.62e-04 | 1333.4ms/step | 15.0 samples/s +Step 15970/44000 | Loss: 3.0591 | LR: 3.62e-04 | 1331.3ms/step | 15.0 samples/s +Step 15980/44000 | Loss: 3.0518 | LR: 3.62e-04 | 1329.5ms/step | 15.0 samples/s +Step 15990/44000 | Loss: 3.0364 | LR: 3.62e-04 | 1332.0ms/step | 15.0 samples/s +Step 16000/44000 | Loss: 3.0599 | LR: 3.62e-04 | 1331.8ms/step | 15.0 samples/s +Validation at step 16000: Loss = 3.0629, Perplexity = 21.39 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-16000 +Step 16010/44000 | Loss: 3.0625 | LR: 3.62e-04 | 3904.0ms/step | 5.1 samples/s +Step 16020/44000 | Loss: 3.0357 | LR: 3.61e-04 | 1338.3ms/step | 14.9 samples/s +Step 16030/44000 | Loss: 3.0629 | LR: 3.61e-04 | 1338.9ms/step | 14.9 samples/s +Step 16040/44000 | Loss: 3.0515 | LR: 3.61e-04 | 1339.5ms/step | 14.9 samples/s +Step 16050/44000 | Loss: 3.0707 | LR: 3.61e-04 | 1339.2ms/step | 14.9 samples/s +Step 16060/44000 | Loss: 3.0675 | LR: 3.61e-04 | 1338.1ms/step | 14.9 samples/s +Step 16070/44000 | Loss: 3.0670 | LR: 3.61e-04 | 1337.7ms/step | 15.0 samples/s +Step 16080/44000 | Loss: 3.0568 | LR: 3.60e-04 | 1339.8ms/step | 14.9 samples/s +Step 16090/44000 | Loss: 3.0334 | LR: 3.60e-04 | 1339.3ms/step | 14.9 samples/s +Step 16100/44000 | Loss: 3.0603 | LR: 3.60e-04 | 1340.6ms/step | 14.9 samples/s +Step 16110/44000 | Loss: 3.0859 | LR: 3.60e-04 | 1340.7ms/step | 14.9 samples/s +Step 16120/44000 | Loss: 3.0664 | LR: 3.60e-04 | 1337.5ms/step | 15.0 samples/s +Step 16130/44000 | Loss: 3.0590 | LR: 3.60e-04 | 1335.2ms/step | 15.0 samples/s +Step 16140/44000 | Loss: 3.0435 | LR: 3.59e-04 | 1339.1ms/step | 14.9 samples/s +Step 16150/44000 | Loss: 3.0483 | LR: 3.59e-04 | 1339.5ms/step | 14.9 samples/s +Step 16160/44000 | Loss: 3.0171 | LR: 3.59e-04 | 1338.3ms/step | 14.9 samples/s +Step 16170/44000 | Loss: 3.0148 | LR: 3.59e-04 | 1340.0ms/step | 14.9 samples/s +Step 16180/44000 | Loss: 3.0235 | LR: 3.59e-04 | 1340.4ms/step | 14.9 samples/s +Step 16190/44000 | Loss: 3.0332 | LR: 3.59e-04 | 1340.5ms/step | 14.9 samples/s +Step 16200/44000 | Loss: 3.0221 | LR: 3.58e-04 | 1339.9ms/step | 14.9 samples/s +Step 16210/44000 | Loss: 3.0379 | LR: 3.58e-04 | 1336.2ms/step | 15.0 samples/s +Step 16220/44000 | Loss: 3.0216 | LR: 3.58e-04 | 1336.4ms/step | 15.0 samples/s +Step 16230/44000 | Loss: 3.0318 | LR: 3.58e-04 | 1338.4ms/step | 14.9 samples/s +Step 16240/44000 | Loss: 3.0314 | LR: 3.58e-04 | 1338.4ms/step | 14.9 samples/s +Step 16250/44000 | Loss: 3.0523 | LR: 3.58e-04 | 1338.7ms/step | 14.9 samples/s +Validation at step 16250: Loss = 3.0594, Perplexity = 21.32 +Step 16260/44000 | Loss: 3.0713 | LR: 3.57e-04 | 3593.2ms/step | 5.6 samples/s +Step 16270/44000 | Loss: 3.0545 | LR: 3.57e-04 | 1339.6ms/step | 14.9 samples/s +Step 16280/44000 | Loss: 3.0284 | LR: 3.57e-04 | 1337.5ms/step | 15.0 samples/s +Step 16290/44000 | Loss: 3.0225 | LR: 3.57e-04 | 1339.4ms/step | 14.9 samples/s +Step 16300/44000 | Loss: 3.0703 | LR: 3.57e-04 | 1337.6ms/step | 15.0 samples/s +Step 16310/44000 | Loss: 3.0506 | LR: 3.57e-04 | 1340.0ms/step | 14.9 samples/s +Step 16320/44000 | Loss: 3.0422 | LR: 3.56e-04 | 1338.8ms/step | 14.9 samples/s +Step 16330/44000 | Loss: 3.0525 | LR: 3.56e-04 | 1341.0ms/step | 14.9 samples/s +Step 16340/44000 | Loss: 3.0519 | LR: 3.56e-04 | 1339.8ms/step | 14.9 samples/s +Step 16350/44000 | Loss: 3.0719 | LR: 3.56e-04 | 1338.2ms/step | 14.9 samples/s +Step 16360/44000 | Loss: 3.0303 | LR: 3.56e-04 | 1335.9ms/step | 15.0 samples/s +Step 16370/44000 | Loss: 3.0521 | LR: 3.56e-04 | 1337.0ms/step | 15.0 samples/s +Step 16380/44000 | Loss: 3.0355 | LR: 3.55e-04 | 1337.4ms/step | 15.0 samples/s +Step 16390/44000 | Loss: 3.0631 | LR: 3.55e-04 | 1337.8ms/step | 14.9 samples/s +Step 16400/44000 | Loss: 3.0457 | LR: 3.55e-04 | 1339.3ms/step | 14.9 samples/s +Step 16410/44000 | Loss: 3.0570 | LR: 3.55e-04 | 1337.1ms/step | 15.0 samples/s +Step 16420/44000 | Loss: 3.0449 | LR: 3.55e-04 | 1337.8ms/step | 14.9 samples/s +Step 16430/44000 | Loss: 3.0509 | LR: 3.55e-04 | 1333.3ms/step | 15.0 samples/s +Step 16440/44000 | Loss: 3.0549 | LR: 3.54e-04 | 1335.1ms/step | 15.0 samples/s +Step 16450/44000 | Loss: 3.0422 | LR: 3.54e-04 | 1336.9ms/step | 15.0 samples/s +Step 16460/44000 | Loss: 3.0393 | LR: 3.54e-04 | 1337.9ms/step | 14.9 samples/s +Step 16470/44000 | Loss: 3.0282 | LR: 3.54e-04 | 1338.7ms/step | 14.9 samples/s +Step 16480/44000 | Loss: 3.0180 | LR: 3.54e-04 | 1338.5ms/step | 14.9 samples/s +Step 16490/44000 | Loss: 3.0228 | LR: 3.54e-04 | 1338.1ms/step | 14.9 samples/s +Step 16500/44000 | Loss: 3.0488 | LR: 3.53e-04 | 1337.4ms/step | 15.0 samples/s +Validation at step 16500: Loss = 3.0566, Perplexity = 21.25 +Step 16510/44000 | Loss: 3.0531 | LR: 3.53e-04 | 3582.2ms/step | 5.6 samples/s +Step 16520/44000 | Loss: 3.0359 | LR: 3.53e-04 | 1338.2ms/step | 14.9 samples/s +Step 16530/44000 | Loss: 3.0163 | LR: 3.53e-04 | 1338.2ms/step | 14.9 samples/s +Step 16540/44000 | Loss: 3.0361 | LR: 3.53e-04 | 1336.4ms/step | 15.0 samples/s +Step 16550/44000 | Loss: 3.0456 | LR: 3.53e-04 | 1338.8ms/step | 14.9 samples/s +Step 16560/44000 | Loss: 3.0453 | LR: 3.52e-04 | 1337.3ms/step | 15.0 samples/s +Step 16570/44000 | Loss: 3.0546 | LR: 3.52e-04 | 1337.8ms/step | 15.0 samples/s +Step 16580/44000 | Loss: 3.0408 | LR: 3.52e-04 | 1334.4ms/step | 15.0 samples/s +Step 16590/44000 | Loss: 3.0392 | LR: 3.52e-04 | 1336.4ms/step | 15.0 samples/s +Step 16600/44000 | Loss: 3.0395 | LR: 3.52e-04 | 1335.3ms/step | 15.0 samples/s +Step 16610/44000 | Loss: 3.0379 | LR: 3.52e-04 | 1335.5ms/step | 15.0 samples/s +Step 16620/44000 | Loss: 3.0097 | LR: 3.51e-04 | 1336.7ms/step | 15.0 samples/s +Step 16630/44000 | Loss: 3.0402 | LR: 3.51e-04 | 1337.9ms/step | 14.9 samples/s +Step 16640/44000 | Loss: 3.0418 | LR: 3.51e-04 | 1337.8ms/step | 14.9 samples/s +Step 16650/44000 | Loss: 3.0470 | LR: 3.51e-04 | 1339.5ms/step | 14.9 samples/s +Step 16660/44000 | Loss: 3.0511 | LR: 3.51e-04 | 1339.7ms/step | 14.9 samples/s +Step 16670/44000 | Loss: 3.0473 | LR: 3.51e-04 | 1337.8ms/step | 14.9 samples/s +Step 16680/44000 | Loss: 3.0389 | LR: 3.50e-04 | 1338.7ms/step | 14.9 samples/s +Step 16690/44000 | Loss: 3.0514 | LR: 3.50e-04 | 1336.4ms/step | 15.0 samples/s +Step 16700/44000 | Loss: 3.0530 | LR: 3.50e-04 | 1337.6ms/step | 15.0 samples/s +Step 16710/44000 | Loss: 3.0430 | LR: 3.50e-04 | 1339.5ms/step | 14.9 samples/s +Step 16720/44000 | Loss: 3.0544 | LR: 3.50e-04 | 1336.5ms/step | 15.0 samples/s +Step 16730/44000 | Loss: 3.0213 | LR: 3.50e-04 | 1335.4ms/step | 15.0 samples/s +Step 16740/44000 | Loss: 3.0400 | LR: 3.49e-04 | 1338.4ms/step | 14.9 samples/s +Step 16750/44000 | Loss: 3.0189 | LR: 3.49e-04 | 1336.0ms/step | 15.0 samples/s +Validation at step 16750: Loss = 3.0526, Perplexity = 21.17 +Step 16760/44000 | Loss: 3.0419 | LR: 3.49e-04 | 3575.4ms/step | 5.6 samples/s +Step 16770/44000 | Loss: 3.0432 | LR: 3.49e-04 | 1338.7ms/step | 14.9 samples/s +Step 16780/44000 | Loss: 3.0661 | LR: 3.49e-04 | 1336.8ms/step | 15.0 samples/s +Step 16790/44000 | Loss: 3.0127 | LR: 3.49e-04 | 1336.3ms/step | 15.0 samples/s +Step 16800/44000 | Loss: 3.0515 | LR: 3.48e-04 | 1337.0ms/step | 15.0 samples/s +Step 16810/44000 | Loss: 3.0485 | LR: 3.48e-04 | 1336.4ms/step | 15.0 samples/s +Step 16820/44000 | Loss: 3.0428 | LR: 3.48e-04 | 1337.3ms/step | 15.0 samples/s +Step 16830/44000 | Loss: 3.0430 | LR: 3.48e-04 | 1339.0ms/step | 14.9 samples/s +Step 16840/44000 | Loss: 3.0183 | LR: 3.48e-04 | 1340.5ms/step | 14.9 samples/s +Step 16850/44000 | Loss: 3.0070 | LR: 3.48e-04 | 1338.9ms/step | 14.9 samples/s +Step 16860/44000 | Loss: 3.0551 | LR: 3.47e-04 | 1341.2ms/step | 14.9 samples/s +Step 16870/44000 | Loss: 3.0376 | LR: 3.47e-04 | 1341.5ms/step | 14.9 samples/s +Step 16880/44000 | Loss: 3.0647 | LR: 3.47e-04 | 1337.8ms/step | 15.0 samples/s +Step 16890/44000 | Loss: 3.0360 | LR: 3.47e-04 | 1337.9ms/step | 14.9 samples/s +Step 16900/44000 | Loss: 3.0518 | LR: 3.47e-04 | 1336.6ms/step | 15.0 samples/s +Step 16910/44000 | Loss: 3.0750 | LR: 3.47e-04 | 1338.8ms/step | 14.9 samples/s +Step 16920/44000 | Loss: 3.0426 | LR: 3.46e-04 | 1338.6ms/step | 14.9 samples/s +Step 16930/44000 | Loss: 3.0289 | LR: 3.46e-04 | 1338.8ms/step | 14.9 samples/s +Step 16940/44000 | Loss: 3.0549 | LR: 3.46e-04 | 1335.8ms/step | 15.0 samples/s +Step 16950/44000 | Loss: 2.9997 | LR: 3.46e-04 | 1336.7ms/step | 15.0 samples/s +Step 16960/44000 | Loss: 3.0396 | LR: 3.46e-04 | 1337.0ms/step | 15.0 samples/s +Step 16970/44000 | Loss: 3.0287 | LR: 3.46e-04 | 1337.5ms/step | 15.0 samples/s +Step 16980/44000 | Loss: 3.0365 | LR: 3.45e-04 | 1335.8ms/step | 15.0 samples/s +Step 16990/44000 | Loss: 3.0368 | LR: 3.45e-04 | 1337.5ms/step | 15.0 samples/s +Step 17000/44000 | Loss: 3.0462 | LR: 3.45e-04 | 1339.2ms/step | 14.9 samples/s +Validation at step 17000: Loss = 3.0494, Perplexity = 21.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-17000 +Step 17010/44000 | Loss: 3.0423 | LR: 3.45e-04 | 3917.4ms/step | 5.1 samples/s +Step 17020/44000 | Loss: 3.0385 | LR: 3.45e-04 | 1336.8ms/step | 15.0 samples/s +Step 17030/44000 | Loss: 3.0143 | LR: 3.45e-04 | 1336.1ms/step | 15.0 samples/s +Step 17040/44000 | Loss: 3.0581 | LR: 3.44e-04 | 1339.2ms/step | 14.9 samples/s +Step 17050/44000 | Loss: 3.0670 | LR: 3.44e-04 | 1338.1ms/step | 14.9 samples/s +Step 17060/44000 | Loss: 3.0093 | LR: 3.44e-04 | 1336.9ms/step | 15.0 samples/s +Step 17070/44000 | Loss: 2.9938 | LR: 3.44e-04 | 1332.8ms/step | 15.0 samples/s +Step 17080/44000 | Loss: 3.0335 | LR: 3.44e-04 | 1332.7ms/step | 15.0 samples/s +Step 17090/44000 | Loss: 3.0354 | LR: 3.44e-04 | 1334.3ms/step | 15.0 samples/s +Step 17100/44000 | Loss: 3.0585 | LR: 3.43e-04 | 1336.6ms/step | 15.0 samples/s +Step 17110/44000 | Loss: 3.0312 | LR: 3.43e-04 | 1330.9ms/step | 15.0 samples/s +Step 17120/44000 | Loss: 3.0367 | LR: 3.43e-04 | 1333.7ms/step | 15.0 samples/s +Step 17130/44000 | Loss: 3.0347 | LR: 3.43e-04 | 1333.7ms/step | 15.0 samples/s +Step 17140/44000 | Loss: 3.0109 | LR: 3.43e-04 | 1333.9ms/step | 15.0 samples/s +Step 17150/44000 | Loss: 3.0141 | LR: 3.43e-04 | 1332.5ms/step | 15.0 samples/s +Step 17160/44000 | Loss: 3.0274 | LR: 3.42e-04 | 1331.4ms/step | 15.0 samples/s +Step 17170/44000 | Loss: 3.0211 | LR: 3.42e-04 | 1333.2ms/step | 15.0 samples/s +Step 17180/44000 | Loss: 3.0036 | LR: 3.42e-04 | 1333.3ms/step | 15.0 samples/s +Step 17190/44000 | Loss: 3.0358 | LR: 3.42e-04 | 1329.5ms/step | 15.0 samples/s +Step 17200/44000 | Loss: 3.0187 | LR: 3.42e-04 | 1336.2ms/step | 15.0 samples/s +Step 17210/44000 | Loss: 3.0349 | LR: 3.42e-04 | 1335.1ms/step | 15.0 samples/s +Step 17220/44000 | Loss: 2.9948 | LR: 3.41e-04 | 1336.9ms/step | 15.0 samples/s +Step 17230/44000 | Loss: 3.0313 | LR: 3.41e-04 | 1336.0ms/step | 15.0 samples/s +Step 17240/44000 | Loss: 3.0154 | LR: 3.41e-04 | 1335.3ms/step | 15.0 samples/s +Step 17250/44000 | Loss: 3.0283 | LR: 3.41e-04 | 1336.7ms/step | 15.0 samples/s +Validation at step 17250: Loss = 3.0461, Perplexity = 21.03 +Step 17260/44000 | Loss: 3.0080 | LR: 3.41e-04 | 3577.0ms/step | 5.6 samples/s +Step 17270/44000 | Loss: 3.0351 | LR: 3.41e-04 | 1332.6ms/step | 15.0 samples/s +Step 17280/44000 | Loss: 3.0426 | LR: 3.40e-04 | 1332.5ms/step | 15.0 samples/s +Step 17290/44000 | Loss: 3.0560 | LR: 3.40e-04 | 1332.3ms/step | 15.0 samples/s +Step 17300/44000 | Loss: 3.0372 | LR: 3.40e-04 | 1331.0ms/step | 15.0 samples/s +Step 17310/44000 | Loss: 2.9877 | LR: 3.40e-04 | 1331.5ms/step | 15.0 samples/s +Step 17320/44000 | Loss: 3.0561 | LR: 3.40e-04 | 1333.9ms/step | 15.0 samples/s +Step 17330/44000 | Loss: 3.0447 | LR: 3.40e-04 | 1332.7ms/step | 15.0 samples/s +Step 17340/44000 | Loss: 3.0340 | LR: 3.39e-04 | 1333.0ms/step | 15.0 samples/s +Step 17350/44000 | Loss: 3.0332 | LR: 3.39e-04 | 1333.1ms/step | 15.0 samples/s +Step 17360/44000 | Loss: 3.0369 | LR: 3.39e-04 | 1334.9ms/step | 15.0 samples/s +Step 17370/44000 | Loss: 3.0484 | LR: 3.39e-04 | 1337.6ms/step | 15.0 samples/s +Step 17380/44000 | Loss: 3.0147 | LR: 3.39e-04 | 1338.9ms/step | 14.9 samples/s +Step 17390/44000 | Loss: 3.0469 | LR: 3.39e-04 | 1336.1ms/step | 15.0 samples/s +Step 17400/44000 | Loss: 3.0142 | LR: 3.38e-04 | 1334.2ms/step | 15.0 samples/s +Step 17410/44000 | Loss: 3.0728 | LR: 3.38e-04 | 1336.0ms/step | 15.0 samples/s +Step 17420/44000 | Loss: 3.0274 | LR: 3.38e-04 | 1338.9ms/step | 14.9 samples/s +Step 17430/44000 | Loss: 3.0019 | LR: 3.38e-04 | 1334.4ms/step | 15.0 samples/s +Step 17440/44000 | Loss: 3.0237 | LR: 3.38e-04 | 1333.4ms/step | 15.0 samples/s +Step 17450/44000 | Loss: 3.0352 | LR: 3.38e-04 | 1335.4ms/step | 15.0 samples/s +Step 17460/44000 | Loss: 3.0176 | LR: 3.37e-04 | 1330.8ms/step | 15.0 samples/s +Step 17470/44000 | Loss: 3.0419 | LR: 3.37e-04 | 1335.2ms/step | 15.0 samples/s +Step 17480/44000 | Loss: 3.0468 | LR: 3.37e-04 | 1341.0ms/step | 14.9 samples/s +Step 17490/44000 | Loss: 3.0390 | LR: 3.37e-04 | 1335.7ms/step | 15.0 samples/s +Step 17500/44000 | Loss: 3.0189 | LR: 3.37e-04 | 1337.3ms/step | 15.0 samples/s +Validation at step 17500: Loss = 3.0433, Perplexity = 20.97 +Step 17510/44000 | Loss: 3.0365 | LR: 3.37e-04 | 3583.2ms/step | 5.6 samples/s +Step 17520/44000 | Loss: 3.0423 | LR: 3.36e-04 | 1336.3ms/step | 15.0 samples/s +Step 17530/44000 | Loss: 3.0207 | LR: 3.36e-04 | 1336.9ms/step | 15.0 samples/s +Step 17540/44000 | Loss: 3.0162 | LR: 3.36e-04 | 1335.4ms/step | 15.0 samples/s +Step 17550/44000 | Loss: 3.0365 | LR: 3.36e-04 | 1332.9ms/step | 15.0 samples/s +Step 17560/44000 | Loss: 3.0710 | LR: 3.36e-04 | 1335.8ms/step | 15.0 samples/s +Step 17570/44000 | Loss: 3.0100 | LR: 3.35e-04 | 1337.5ms/step | 15.0 samples/s +Step 17580/44000 | Loss: 3.0149 | LR: 3.35e-04 | 1337.2ms/step | 15.0 samples/s +Step 17590/44000 | Loss: 3.0548 | LR: 3.35e-04 | 1336.1ms/step | 15.0 samples/s +Step 17600/44000 | Loss: 2.9883 | LR: 3.35e-04 | 1333.3ms/step | 15.0 samples/s +Step 17610/44000 | Loss: 2.9972 | LR: 3.35e-04 | 1334.9ms/step | 15.0 samples/s +Step 17620/44000 | Loss: 3.0307 | LR: 3.35e-04 | 1332.9ms/step | 15.0 samples/s +Step 17630/44000 | Loss: 3.0143 | LR: 3.34e-04 | 1335.1ms/step | 15.0 samples/s +Step 17640/44000 | Loss: 3.0106 | LR: 3.34e-04 | 1333.3ms/step | 15.0 samples/s +Step 17650/44000 | Loss: 3.0280 | LR: 3.34e-04 | 1333.5ms/step | 15.0 samples/s +Step 17660/44000 | Loss: 3.0111 | LR: 3.34e-04 | 1333.8ms/step | 15.0 samples/s +Step 17670/44000 | Loss: 3.0418 | LR: 3.34e-04 | 1335.1ms/step | 15.0 samples/s +Step 17680/44000 | Loss: 3.0259 | LR: 3.34e-04 | 1336.9ms/step | 15.0 samples/s +Step 17690/44000 | Loss: 3.0155 | LR: 3.33e-04 | 1335.3ms/step | 15.0 samples/s +Step 17700/44000 | Loss: 3.0380 | LR: 3.33e-04 | 1335.9ms/step | 15.0 samples/s +Step 17710/44000 | Loss: 3.0331 | LR: 3.33e-04 | 1337.5ms/step | 15.0 samples/s +Step 17720/44000 | Loss: 3.0216 | LR: 3.33e-04 | 1333.9ms/step | 15.0 samples/s +Step 17730/44000 | Loss: 3.0385 | LR: 3.33e-04 | 1332.1ms/step | 15.0 samples/s +Step 17740/44000 | Loss: 3.0274 | LR: 3.33e-04 | 1335.4ms/step | 15.0 samples/s +Step 17750/44000 | Loss: 3.0307 | LR: 3.32e-04 | 1336.8ms/step | 15.0 samples/s +Validation at step 17750: Loss = 3.0402, Perplexity = 20.91 +Step 17760/44000 | Loss: 3.0250 | LR: 3.32e-04 | 3574.1ms/step | 5.6 samples/s +Step 17770/44000 | Loss: 3.0193 | LR: 3.32e-04 | 1333.8ms/step | 15.0 samples/s +Step 17780/44000 | Loss: 3.0327 | LR: 3.32e-04 | 1332.5ms/step | 15.0 samples/s +Step 17790/44000 | Loss: 3.0720 | LR: 3.32e-04 | 1330.3ms/step | 15.0 samples/s +Step 17800/44000 | Loss: 3.0157 | LR: 3.32e-04 | 1332.5ms/step | 15.0 samples/s +Step 17810/44000 | Loss: 3.0147 | LR: 3.31e-04 | 1335.0ms/step | 15.0 samples/s +Step 17820/44000 | Loss: 3.0474 | LR: 3.31e-04 | 1331.3ms/step | 15.0 samples/s +Step 17830/44000 | Loss: 2.9929 | LR: 3.31e-04 | 1331.2ms/step | 15.0 samples/s +Step 17840/44000 | Loss: 3.0403 | LR: 3.31e-04 | 1332.3ms/step | 15.0 samples/s +Step 17850/44000 | Loss: 3.0392 | LR: 3.31e-04 | 1332.6ms/step | 15.0 samples/s +Step 17860/44000 | Loss: 3.0143 | LR: 3.31e-04 | 1332.1ms/step | 15.0 samples/s +Step 17870/44000 | Loss: 3.0424 | LR: 3.30e-04 | 1328.6ms/step | 15.1 samples/s +Step 17880/44000 | Loss: 3.0519 | LR: 3.30e-04 | 1329.5ms/step | 15.0 samples/s +Step 17890/44000 | Loss: 3.0390 | LR: 3.30e-04 | 1335.1ms/step | 15.0 samples/s +Step 17900/44000 | Loss: 3.0377 | LR: 3.30e-04 | 1332.3ms/step | 15.0 samples/s +Step 17910/44000 | Loss: 3.0716 | LR: 3.30e-04 | 1335.1ms/step | 15.0 samples/s +Step 17920/44000 | Loss: 3.0008 | LR: 3.29e-04 | 1332.6ms/step | 15.0 samples/s +Step 17930/44000 | Loss: 2.9982 | LR: 3.29e-04 | 1332.2ms/step | 15.0 samples/s +Step 17940/44000 | Loss: 3.0027 | LR: 3.29e-04 | 1335.0ms/step | 15.0 samples/s +Step 17950/44000 | Loss: 3.0190 | LR: 3.29e-04 | 1334.2ms/step | 15.0 samples/s +Step 17960/44000 | Loss: 3.0324 | LR: 3.29e-04 | 1337.6ms/step | 15.0 samples/s +Step 17970/44000 | Loss: 3.0126 | LR: 3.29e-04 | 1332.4ms/step | 15.0 samples/s +Step 17980/44000 | Loss: 3.0139 | LR: 3.28e-04 | 1335.3ms/step | 15.0 samples/s +Step 17990/44000 | Loss: 2.9935 | LR: 3.28e-04 | 1334.3ms/step | 15.0 samples/s +Step 18000/44000 | Loss: 3.0145 | LR: 3.28e-04 | 1333.1ms/step | 15.0 samples/s +Validation at step 18000: Loss = 3.0375, Perplexity = 20.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-18000 +Step 18010/44000 | Loss: 3.0021 | LR: 3.28e-04 | 3908.9ms/step | 5.1 samples/s +Step 18020/44000 | Loss: 3.0478 | LR: 3.28e-04 | 1335.1ms/step | 15.0 samples/s +Step 18030/44000 | Loss: 3.0125 | LR: 3.28e-04 | 1332.3ms/step | 15.0 samples/s +Step 18040/44000 | Loss: 3.0170 | LR: 3.27e-04 | 1331.5ms/step | 15.0 samples/s +Step 18050/44000 | Loss: 3.0467 | LR: 3.27e-04 | 1331.7ms/step | 15.0 samples/s +Step 18060/44000 | Loss: 3.0202 | LR: 3.27e-04 | 1333.7ms/step | 15.0 samples/s +Step 18070/44000 | Loss: 3.0184 | LR: 3.27e-04 | 1331.2ms/step | 15.0 samples/s +Step 18080/44000 | Loss: 3.0346 | LR: 3.27e-04 | 1331.4ms/step | 15.0 samples/s +Step 18090/44000 | Loss: 3.0242 | LR: 3.27e-04 | 1333.4ms/step | 15.0 samples/s +Step 18100/44000 | Loss: 3.0268 | LR: 3.26e-04 | 1330.1ms/step | 15.0 samples/s +Step 18110/44000 | Loss: 3.0412 | LR: 3.26e-04 | 1331.0ms/step | 15.0 samples/s +Step 18120/44000 | Loss: 3.0128 | LR: 3.26e-04 | 1331.0ms/step | 15.0 samples/s +Step 18130/44000 | Loss: 3.0185 | LR: 3.26e-04 | 1335.1ms/step | 15.0 samples/s +Step 18140/44000 | Loss: 3.0185 | LR: 3.26e-04 | 1334.0ms/step | 15.0 samples/s +Step 18150/44000 | Loss: 3.0258 | LR: 3.26e-04 | 1332.4ms/step | 15.0 samples/s +Step 18160/44000 | Loss: 3.0411 | LR: 3.25e-04 | 1332.4ms/step | 15.0 samples/s +Step 18170/44000 | Loss: 2.9975 | LR: 3.25e-04 | 1334.4ms/step | 15.0 samples/s +Step 18180/44000 | Loss: 3.0174 | LR: 3.25e-04 | 1333.5ms/step | 15.0 samples/s +Step 18190/44000 | Loss: 3.0369 | LR: 3.25e-04 | 1332.4ms/step | 15.0 samples/s +Step 18200/44000 | Loss: 3.0400 | LR: 3.25e-04 | 1327.1ms/step | 15.1 samples/s +Step 18210/44000 | Loss: 3.0523 | LR: 3.24e-04 | 1329.6ms/step | 15.0 samples/s +Step 18220/44000 | Loss: 3.0148 | LR: 3.24e-04 | 1330.4ms/step | 15.0 samples/s +Step 18230/44000 | Loss: 3.0297 | LR: 3.24e-04 | 1333.4ms/step | 15.0 samples/s +Step 18240/44000 | Loss: 3.0405 | LR: 3.24e-04 | 1335.0ms/step | 15.0 samples/s +Step 18250/44000 | Loss: 3.0011 | LR: 3.24e-04 | 1328.7ms/step | 15.1 samples/s +Validation at step 18250: Loss = 3.0348, Perplexity = 20.80 +Step 18260/44000 | Loss: 3.0290 | LR: 3.24e-04 | 3567.4ms/step | 5.6 samples/s +Step 18270/44000 | Loss: 3.0019 | LR: 3.23e-04 | 1331.4ms/step | 15.0 samples/s +Step 18280/44000 | Loss: 2.9973 | LR: 3.23e-04 | 1333.7ms/step | 15.0 samples/s +Step 18290/44000 | Loss: 3.0294 | LR: 3.23e-04 | 1330.8ms/step | 15.0 samples/s +Step 18300/44000 | Loss: 3.0055 | LR: 3.23e-04 | 1330.4ms/step | 15.0 samples/s +Step 18310/44000 | Loss: 3.0261 | LR: 3.23e-04 | 1332.8ms/step | 15.0 samples/s +Step 18320/44000 | Loss: 3.0355 | LR: 3.23e-04 | 1330.8ms/step | 15.0 samples/s +Step 18330/44000 | Loss: 3.0496 | LR: 3.22e-04 | 1332.4ms/step | 15.0 samples/s +Step 18340/44000 | Loss: 3.0389 | LR: 3.22e-04 | 1332.3ms/step | 15.0 samples/s +Step 18350/44000 | Loss: 3.0151 | LR: 3.22e-04 | 1330.5ms/step | 15.0 samples/s +Step 18360/44000 | Loss: 3.0334 | LR: 3.22e-04 | 1333.9ms/step | 15.0 samples/s +Step 18370/44000 | Loss: 3.0508 | LR: 3.22e-04 | 1330.0ms/step | 15.0 samples/s +Step 18380/44000 | Loss: 3.0238 | LR: 3.22e-04 | 1332.2ms/step | 15.0 samples/s +Step 18390/44000 | Loss: 3.0106 | LR: 3.21e-04 | 1331.3ms/step | 15.0 samples/s +Step 18400/44000 | Loss: 2.9807 | LR: 3.21e-04 | 1331.6ms/step | 15.0 samples/s +Step 18410/44000 | Loss: 3.0305 | LR: 3.21e-04 | 1332.4ms/step | 15.0 samples/s +Step 18420/44000 | Loss: 3.0117 | LR: 3.21e-04 | 1330.0ms/step | 15.0 samples/s +Step 18430/44000 | Loss: 3.0368 | LR: 3.21e-04 | 1328.9ms/step | 15.1 samples/s +Step 18440/44000 | Loss: 3.0093 | LR: 3.20e-04 | 1330.4ms/step | 15.0 samples/s +Step 18450/44000 | Loss: 3.0162 | LR: 3.20e-04 | 1332.3ms/step | 15.0 samples/s +Step 18460/44000 | Loss: 3.0149 | LR: 3.20e-04 | 1329.1ms/step | 15.0 samples/s +Step 18470/44000 | Loss: 3.0451 | LR: 3.20e-04 | 1327.7ms/step | 15.1 samples/s +Step 18480/44000 | Loss: 3.0145 | LR: 3.20e-04 | 1328.0ms/step | 15.1 samples/s +Step 18490/44000 | Loss: 3.0133 | LR: 3.20e-04 | 1329.7ms/step | 15.0 samples/s +Step 18500/44000 | Loss: 2.9973 | LR: 3.19e-04 | 1329.7ms/step | 15.0 samples/s +Validation at step 18500: Loss = 3.0316, Perplexity = 20.73 +Step 18510/44000 | Loss: 3.0217 | LR: 3.19e-04 | 3576.8ms/step | 5.6 samples/s +Step 18520/44000 | Loss: 2.9976 | LR: 3.19e-04 | 1330.6ms/step | 15.0 samples/s +Step 18530/44000 | Loss: 3.0297 | LR: 3.19e-04 | 1333.4ms/step | 15.0 samples/s +Step 18540/44000 | Loss: 2.9927 | LR: 3.19e-04 | 1333.0ms/step | 15.0 samples/s +Step 18550/44000 | Loss: 3.0234 | LR: 3.19e-04 | 1332.9ms/step | 15.0 samples/s +Step 18560/44000 | Loss: 3.0126 | LR: 3.18e-04 | 1332.3ms/step | 15.0 samples/s +Step 18570/44000 | Loss: 3.0094 | LR: 3.18e-04 | 1331.2ms/step | 15.0 samples/s +Step 18580/44000 | Loss: 2.9954 | LR: 3.18e-04 | 1332.5ms/step | 15.0 samples/s +Step 18590/44000 | Loss: 3.0018 | LR: 3.18e-04 | 1333.5ms/step | 15.0 samples/s +Step 18600/44000 | Loss: 3.0116 | LR: 3.18e-04 | 1334.0ms/step | 15.0 samples/s +Step 18610/44000 | Loss: 3.0317 | LR: 3.18e-04 | 1330.4ms/step | 15.0 samples/s +Step 18620/44000 | Loss: 3.0553 | LR: 3.17e-04 | 1331.0ms/step | 15.0 samples/s +Step 18630/44000 | Loss: 3.0450 | LR: 3.17e-04 | 1331.3ms/step | 15.0 samples/s +Step 18640/44000 | Loss: 3.0016 | LR: 3.17e-04 | 1331.3ms/step | 15.0 samples/s +Step 18650/44000 | Loss: 3.0279 | LR: 3.17e-04 | 1328.8ms/step | 15.1 samples/s +Step 18660/44000 | Loss: 3.0503 | LR: 3.17e-04 | 1334.2ms/step | 15.0 samples/s +Step 18670/44000 | Loss: 3.0372 | LR: 3.16e-04 | 1331.4ms/step | 15.0 samples/s +Step 18680/44000 | Loss: 3.0176 | LR: 3.16e-04 | 1331.2ms/step | 15.0 samples/s +Step 18690/44000 | Loss: 3.0091 | LR: 3.16e-04 | 1334.1ms/step | 15.0 samples/s +Step 18700/44000 | Loss: 3.0155 | LR: 3.16e-04 | 1334.3ms/step | 15.0 samples/s +Step 18710/44000 | Loss: 3.0310 | LR: 3.16e-04 | 1333.1ms/step | 15.0 samples/s +Step 18720/44000 | Loss: 3.0334 | LR: 3.16e-04 | 1332.7ms/step | 15.0 samples/s +Step 18730/44000 | Loss: 3.0183 | LR: 3.15e-04 | 1331.1ms/step | 15.0 samples/s +Step 18740/44000 | Loss: 3.0233 | LR: 3.15e-04 | 1329.7ms/step | 15.0 samples/s +Step 18750/44000 | Loss: 3.0199 | LR: 3.15e-04 | 1329.9ms/step | 15.0 samples/s +Validation at step 18750: Loss = 3.0285, Perplexity = 20.67 +Step 18760/44000 | Loss: 3.0417 | LR: 3.15e-04 | 3567.2ms/step | 5.6 samples/s +Step 18770/44000 | Loss: 3.0230 | LR: 3.15e-04 | 1334.9ms/step | 15.0 samples/s +Step 18780/44000 | Loss: 3.0368 | LR: 3.15e-04 | 1333.2ms/step | 15.0 samples/s +Step 18790/44000 | Loss: 3.0202 | LR: 3.14e-04 | 1330.1ms/step | 15.0 samples/s +Step 18800/44000 | Loss: 3.0198 | LR: 3.14e-04 | 1334.1ms/step | 15.0 samples/s +Step 18810/44000 | Loss: 3.0167 | LR: 3.14e-04 | 1332.1ms/step | 15.0 samples/s +Step 18820/44000 | Loss: 3.0086 | LR: 3.14e-04 | 1334.4ms/step | 15.0 samples/s +Step 18830/44000 | Loss: 3.0318 | LR: 3.14e-04 | 1333.4ms/step | 15.0 samples/s +Step 18840/44000 | Loss: 3.0249 | LR: 3.13e-04 | 1333.4ms/step | 15.0 samples/s +Step 18850/44000 | Loss: 3.0097 | LR: 3.13e-04 | 1332.1ms/step | 15.0 samples/s +Step 18860/44000 | Loss: 3.0087 | LR: 3.13e-04 | 1329.9ms/step | 15.0 samples/s +Step 18870/44000 | Loss: 3.0070 | LR: 3.13e-04 | 1334.7ms/step | 15.0 samples/s +Step 18880/44000 | Loss: 3.0217 | LR: 3.13e-04 | 1332.5ms/step | 15.0 samples/s +Step 18890/44000 | Loss: 3.0093 | LR: 3.13e-04 | 1332.0ms/step | 15.0 samples/s +Step 18900/44000 | Loss: 2.9989 | LR: 3.12e-04 | 1332.7ms/step | 15.0 samples/s +Step 18910/44000 | Loss: 2.9982 | LR: 3.12e-04 | 1331.1ms/step | 15.0 samples/s +Step 18920/44000 | Loss: 3.0291 | LR: 3.12e-04 | 1334.1ms/step | 15.0 samples/s +Step 18930/44000 | Loss: 3.0311 | LR: 3.12e-04 | 1331.3ms/step | 15.0 samples/s +Step 18940/44000 | Loss: 2.9995 | LR: 3.12e-04 | 1332.8ms/step | 15.0 samples/s +Step 18950/44000 | Loss: 3.0003 | LR: 3.12e-04 | 1331.9ms/step | 15.0 samples/s +Step 18960/44000 | Loss: 3.0037 | LR: 3.11e-04 | 1332.4ms/step | 15.0 samples/s +Step 18970/44000 | Loss: 3.0385 | LR: 3.11e-04 | 1330.0ms/step | 15.0 samples/s +Step 18980/44000 | Loss: 3.0392 | LR: 3.11e-04 | 1331.4ms/step | 15.0 samples/s +Step 18990/44000 | Loss: 3.0195 | LR: 3.11e-04 | 1333.4ms/step | 15.0 samples/s +Step 19000/44000 | Loss: 3.0326 | LR: 3.11e-04 | 1333.5ms/step | 15.0 samples/s +Validation at step 19000: Loss = 3.0255, Perplexity = 20.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-19000 +Step 19010/44000 | Loss: 3.0276 | LR: 3.11e-04 | 3902.3ms/step | 5.1 samples/s +Step 19020/44000 | Loss: 2.9809 | LR: 3.10e-04 | 1331.6ms/step | 15.0 samples/s +Step 19030/44000 | Loss: 3.0209 | LR: 3.10e-04 | 1333.0ms/step | 15.0 samples/s +Step 19040/44000 | Loss: 3.0328 | LR: 3.10e-04 | 1332.6ms/step | 15.0 samples/s +Step 19050/44000 | Loss: 3.0094 | LR: 3.10e-04 | 1331.8ms/step | 15.0 samples/s +Step 19060/44000 | Loss: 3.0178 | LR: 3.10e-04 | 1334.3ms/step | 15.0 samples/s +Step 19070/44000 | Loss: 3.0114 | LR: 3.09e-04 | 1334.3ms/step | 15.0 samples/s +Step 19080/44000 | Loss: 3.0083 | LR: 3.09e-04 | 1333.9ms/step | 15.0 samples/s +Step 19090/44000 | Loss: 3.0130 | LR: 3.09e-04 | 1334.7ms/step | 15.0 samples/s +Step 19100/44000 | Loss: 2.9822 | LR: 3.09e-04 | 1332.7ms/step | 15.0 samples/s +Step 19110/44000 | Loss: 3.0086 | LR: 3.09e-04 | 1332.7ms/step | 15.0 samples/s +Step 19120/44000 | Loss: 2.9708 | LR: 3.09e-04 | 1333.1ms/step | 15.0 samples/s +Step 19130/44000 | Loss: 3.0550 | LR: 3.08e-04 | 1332.1ms/step | 15.0 samples/s +Step 19140/44000 | Loss: 2.9962 | LR: 3.08e-04 | 1334.2ms/step | 15.0 samples/s +Step 19150/44000 | Loss: 3.0049 | LR: 3.08e-04 | 1332.1ms/step | 15.0 samples/s +Step 19160/44000 | Loss: 3.0213 | LR: 3.08e-04 | 1331.4ms/step | 15.0 samples/s +Step 19170/44000 | Loss: 3.0282 | LR: 3.08e-04 | 1335.4ms/step | 15.0 samples/s +Step 19180/44000 | Loss: 2.9811 | LR: 3.08e-04 | 1328.8ms/step | 15.1 samples/s +Step 19190/44000 | Loss: 3.0085 | LR: 3.07e-04 | 1331.8ms/step | 15.0 samples/s +Step 19200/44000 | Loss: 3.0034 | LR: 3.07e-04 | 1332.0ms/step | 15.0 samples/s +Step 19210/44000 | Loss: 2.9736 | LR: 3.07e-04 | 1332.5ms/step | 15.0 samples/s +Step 19220/44000 | Loss: 3.0276 | LR: 3.07e-04 | 1332.6ms/step | 15.0 samples/s +Step 19230/44000 | Loss: 3.0138 | LR: 3.07e-04 | 1327.7ms/step | 15.1 samples/s +Step 19240/44000 | Loss: 2.9896 | LR: 3.06e-04 | 1330.9ms/step | 15.0 samples/s +Step 19250/44000 | Loss: 3.0214 | LR: 3.06e-04 | 1326.8ms/step | 15.1 samples/s +Validation at step 19250: Loss = 3.0231, Perplexity = 20.55 +Step 19260/44000 | Loss: 3.0068 | LR: 3.06e-04 | 3573.6ms/step | 5.6 samples/s +Step 19270/44000 | Loss: 3.0128 | LR: 3.06e-04 | 1332.8ms/step | 15.0 samples/s +Step 19280/44000 | Loss: 3.0050 | LR: 3.06e-04 | 1332.6ms/step | 15.0 samples/s +Step 19290/44000 | Loss: 3.0084 | LR: 3.06e-04 | 1332.7ms/step | 15.0 samples/s +Step 19300/44000 | Loss: 2.9838 | LR: 3.05e-04 | 1330.9ms/step | 15.0 samples/s +Step 19310/44000 | Loss: 2.9853 | LR: 3.05e-04 | 1331.1ms/step | 15.0 samples/s +Step 19320/44000 | Loss: 3.0240 | LR: 3.05e-04 | 1330.9ms/step | 15.0 samples/s +Step 19330/44000 | Loss: 3.0078 | LR: 3.05e-04 | 1330.0ms/step | 15.0 samples/s +Step 19340/44000 | Loss: 3.0128 | LR: 3.05e-04 | 1327.5ms/step | 15.1 samples/s +Step 19350/44000 | Loss: 2.9973 | LR: 3.04e-04 | 1332.5ms/step | 15.0 samples/s +Step 19360/44000 | Loss: 2.9954 | LR: 3.04e-04 | 1331.7ms/step | 15.0 samples/s +Step 19370/44000 | Loss: 3.0220 | LR: 3.04e-04 | 1330.3ms/step | 15.0 samples/s +Step 19380/44000 | Loss: 3.0068 | LR: 3.04e-04 | 1326.5ms/step | 15.1 samples/s +Step 19390/44000 | Loss: 3.0201 | LR: 3.04e-04 | 1332.0ms/step | 15.0 samples/s +Step 19400/44000 | Loss: 3.0402 | LR: 3.04e-04 | 1333.1ms/step | 15.0 samples/s +Step 19410/44000 | Loss: 2.9807 | LR: 3.03e-04 | 1330.7ms/step | 15.0 samples/s +Step 19420/44000 | Loss: 3.0202 | LR: 3.03e-04 | 1331.4ms/step | 15.0 samples/s +Step 19430/44000 | Loss: 3.0235 | LR: 3.03e-04 | 1331.0ms/step | 15.0 samples/s +Step 19440/44000 | Loss: 3.0121 | LR: 3.03e-04 | 1331.0ms/step | 15.0 samples/s +Step 19450/44000 | Loss: 2.9872 | LR: 3.03e-04 | 1330.6ms/step | 15.0 samples/s +Step 19460/44000 | Loss: 3.0180 | LR: 3.03e-04 | 1331.7ms/step | 15.0 samples/s +Step 19470/44000 | Loss: 3.0225 | LR: 3.02e-04 | 1331.3ms/step | 15.0 samples/s +Step 19480/44000 | Loss: 3.0215 | LR: 3.02e-04 | 1329.7ms/step | 15.0 samples/s +Step 19490/44000 | Loss: 2.9907 | LR: 3.02e-04 | 1331.0ms/step | 15.0 samples/s +Step 19500/44000 | Loss: 2.9888 | LR: 3.02e-04 | 1332.9ms/step | 15.0 samples/s +Validation at step 19500: Loss = 3.0208, Perplexity = 20.51 +Step 19510/44000 | Loss: 2.9905 | LR: 3.02e-04 | 3578.3ms/step | 5.6 samples/s +Step 19520/44000 | Loss: 2.9899 | LR: 3.01e-04 | 1335.1ms/step | 15.0 samples/s +Step 19530/44000 | Loss: 2.9802 | LR: 3.01e-04 | 1336.3ms/step | 15.0 samples/s +Step 19540/44000 | Loss: 3.0014 | LR: 3.01e-04 | 1339.9ms/step | 14.9 samples/s +Step 19550/44000 | Loss: 3.0036 | LR: 3.01e-04 | 1336.0ms/step | 15.0 samples/s +Step 19560/44000 | Loss: 3.0021 | LR: 3.01e-04 | 1335.9ms/step | 15.0 samples/s +Step 19570/44000 | Loss: 3.0115 | LR: 3.01e-04 | 1333.0ms/step | 15.0 samples/s +Step 19580/44000 | Loss: 2.9847 | LR: 3.00e-04 | 1332.6ms/step | 15.0 samples/s +Step 19590/44000 | Loss: 2.9828 | LR: 3.00e-04 | 1334.3ms/step | 15.0 samples/s +Step 19600/44000 | Loss: 3.0410 | LR: 3.00e-04 | 1335.8ms/step | 15.0 samples/s +Step 19610/44000 | Loss: 3.0102 | LR: 3.00e-04 | 1338.4ms/step | 14.9 samples/s +Step 19620/44000 | Loss: 3.0094 | LR: 3.00e-04 | 1334.3ms/step | 15.0 samples/s +Step 19630/44000 | Loss: 3.0076 | LR: 3.00e-04 | 1338.0ms/step | 14.9 samples/s +Step 19640/44000 | Loss: 3.0287 | LR: 2.99e-04 | 1338.3ms/step | 14.9 samples/s +Step 19650/44000 | Loss: 3.0147 | LR: 2.99e-04 | 1336.9ms/step | 15.0 samples/s +Step 19660/44000 | Loss: 3.0308 | LR: 2.99e-04 | 1332.7ms/step | 15.0 samples/s +Step 19670/44000 | Loss: 2.9978 | LR: 2.99e-04 | 1335.8ms/step | 15.0 samples/s +Step 19680/44000 | Loss: 2.9657 | LR: 2.99e-04 | 1338.7ms/step | 14.9 samples/s +Step 19690/44000 | Loss: 3.0277 | LR: 2.98e-04 | 1333.4ms/step | 15.0 samples/s +Step 19700/44000 | Loss: 3.0145 | LR: 2.98e-04 | 1336.7ms/step | 15.0 samples/s +Step 19710/44000 | Loss: 2.9982 | LR: 2.98e-04 | 1337.8ms/step | 14.9 samples/s +Step 19720/44000 | Loss: 3.0093 | LR: 2.98e-04 | 1337.5ms/step | 15.0 samples/s +Step 19730/44000 | Loss: 3.0213 | LR: 2.98e-04 | 1334.9ms/step | 15.0 samples/s +Step 19740/44000 | Loss: 2.9936 | LR: 2.98e-04 | 1329.5ms/step | 15.0 samples/s +Step 19750/44000 | Loss: 3.0296 | LR: 2.97e-04 | 1334.7ms/step | 15.0 samples/s +Validation at step 19750: Loss = 3.0176, Perplexity = 20.44 +Step 19760/44000 | Loss: 3.0156 | LR: 2.97e-04 | 3568.5ms/step | 5.6 samples/s +Step 19770/44000 | Loss: 2.9997 | LR: 2.97e-04 | 1331.6ms/step | 15.0 samples/s +Step 19780/44000 | Loss: 2.9985 | LR: 2.97e-04 | 1330.0ms/step | 15.0 samples/s +Step 19790/44000 | Loss: 3.0228 | LR: 2.97e-04 | 1332.4ms/step | 15.0 samples/s +Step 19800/44000 | Loss: 2.9785 | LR: 2.96e-04 | 1332.1ms/step | 15.0 samples/s +Step 19810/44000 | Loss: 3.0054 | LR: 2.96e-04 | 1335.0ms/step | 15.0 samples/s +Step 19820/44000 | Loss: 3.0087 | LR: 2.96e-04 | 1332.4ms/step | 15.0 samples/s +Step 19830/44000 | Loss: 3.0328 | LR: 2.96e-04 | 1332.6ms/step | 15.0 samples/s +Step 19840/44000 | Loss: 2.9719 | LR: 2.96e-04 | 1331.1ms/step | 15.0 samples/s +Step 19850/44000 | Loss: 3.0003 | LR: 2.96e-04 | 1333.2ms/step | 15.0 samples/s +Step 19860/44000 | Loss: 2.9894 | LR: 2.95e-04 | 1335.3ms/step | 15.0 samples/s +Step 19870/44000 | Loss: 2.9854 | LR: 2.95e-04 | 1333.4ms/step | 15.0 samples/s +Step 19880/44000 | Loss: 3.0200 | LR: 2.95e-04 | 1333.3ms/step | 15.0 samples/s +Step 19890/44000 | Loss: 3.0008 | LR: 2.95e-04 | 1333.5ms/step | 15.0 samples/s +Step 19900/44000 | Loss: 2.9873 | LR: 2.95e-04 | 1336.7ms/step | 15.0 samples/s +Step 19910/44000 | Loss: 2.9741 | LR: 2.95e-04 | 1332.9ms/step | 15.0 samples/s +Step 19920/44000 | Loss: 3.0143 | LR: 2.94e-04 | 1331.1ms/step | 15.0 samples/s +Step 19930/44000 | Loss: 3.0098 | LR: 2.94e-04 | 1331.2ms/step | 15.0 samples/s +Step 19940/44000 | Loss: 3.0048 | LR: 2.94e-04 | 1329.8ms/step | 15.0 samples/s +Step 19950/44000 | Loss: 2.9940 | LR: 2.94e-04 | 1330.8ms/step | 15.0 samples/s +Step 19960/44000 | Loss: 3.0013 | LR: 2.94e-04 | 1333.5ms/step | 15.0 samples/s +Step 19970/44000 | Loss: 3.0026 | LR: 2.93e-04 | 1332.3ms/step | 15.0 samples/s +Step 19980/44000 | Loss: 2.9926 | LR: 2.93e-04 | 1331.8ms/step | 15.0 samples/s +Step 19990/44000 | Loss: 3.0035 | LR: 2.93e-04 | 1332.2ms/step | 15.0 samples/s diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_resampled_nrb35_seed43.log b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_resampled_nrb35_seed43.log new file mode 100644 index 0000000000000000000000000000000000000000..9e2302e9ccc3449013132c2538c8d5039c84384f --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_resampled_nrb35_seed43.log @@ -0,0 +1,2296 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 43, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 365m resampled 8k nrb35 seed43", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "resampled", + "nrb35", + "seed43" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run ryta2atg +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_164610-ryta2atg +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 365m resampled 8k nrb35 seed43 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ryta2atg +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0448 | LR: 6.67e-06 | 2520.5ms/step | 7.9 samples/s +Step 20/44000 | Loss: 11.9526 | LR: 1.33e-05 | 1325.4ms/step | 15.1 samples/s +Step 30/44000 | Loss: 11.6339 | LR: 2.00e-05 | 1325.5ms/step | 15.1 samples/s +Step 40/44000 | Loss: 11.1022 | LR: 2.67e-05 | 1323.5ms/step | 15.1 samples/s +Step 50/44000 | Loss: 10.7686 | LR: 3.33e-05 | 1321.2ms/step | 15.1 samples/s +Step 60/44000 | Loss: 10.5338 | LR: 4.00e-05 | 1322.0ms/step | 15.1 samples/s +Step 70/44000 | Loss: 10.2890 | LR: 4.67e-05 | 1322.3ms/step | 15.1 samples/s +Step 80/44000 | Loss: 9.9866 | LR: 5.33e-05 | 1314.4ms/step | 15.2 samples/s +Step 90/44000 | Loss: 9.6273 | LR: 6.00e-05 | 1327.2ms/step | 15.1 samples/s +Step 100/44000 | Loss: 9.1311 | LR: 6.67e-05 | 1321.8ms/step | 15.1 samples/s +Step 110/44000 | Loss: 8.6240 | LR: 7.33e-05 | 1322.7ms/step | 15.1 samples/s +Step 120/44000 | Loss: 8.1266 | LR: 8.00e-05 | 1327.7ms/step | 15.1 samples/s +Step 130/44000 | Loss: 7.7644 | LR: 8.67e-05 | 1328.2ms/step | 15.1 samples/s +Step 140/44000 | Loss: 7.4902 | LR: 9.33e-05 | 1322.5ms/step | 15.1 samples/s +Step 150/44000 | Loss: 7.3332 | LR: 1.00e-04 | 1325.3ms/step | 15.1 samples/s +Step 160/44000 | Loss: 7.2121 | LR: 1.07e-04 | 1324.9ms/step | 15.1 samples/s +Step 170/44000 | Loss: 7.1136 | LR: 1.13e-04 | 1327.6ms/step | 15.1 samples/s +Step 180/44000 | Loss: 7.0290 | LR: 1.20e-04 | 1326.5ms/step | 15.1 samples/s +Step 190/44000 | Loss: 6.9172 | LR: 1.27e-04 | 1329.2ms/step | 15.0 samples/s +Step 200/44000 | Loss: 6.8204 | LR: 1.33e-04 | 1329.8ms/step | 15.0 samples/s +Step 210/44000 | Loss: 6.7374 | LR: 1.40e-04 | 1330.0ms/step | 15.0 samples/s +Step 220/44000 | Loss: 6.6682 | LR: 1.47e-04 | 1331.1ms/step | 15.0 samples/s +Step 230/44000 | Loss: 6.5948 | LR: 1.53e-04 | 1326.8ms/step | 15.1 samples/s +Step 240/44000 | Loss: 6.5208 | LR: 1.60e-04 | 1329.4ms/step | 15.0 samples/s +Step 250/44000 | Loss: 6.4534 | LR: 1.67e-04 | 1332.8ms/step | 15.0 samples/s +Validation at step 250: Loss = 6.4389, Perplexity = 625.74 +Step 260/44000 | Loss: 6.4197 | LR: 1.73e-04 | 4283.8ms/step | 4.7 samples/s +Step 270/44000 | Loss: 6.3752 | LR: 1.80e-04 | 1338.2ms/step | 14.9 samples/s +Step 280/44000 | Loss: 6.3080 | LR: 1.87e-04 | 1334.0ms/step | 15.0 samples/s +Step 290/44000 | Loss: 6.2566 | LR: 1.93e-04 | 1330.3ms/step | 15.0 samples/s +Step 300/44000 | Loss: 6.1900 | LR: 2.00e-04 | 1330.6ms/step | 15.0 samples/s +Step 310/44000 | Loss: 6.1484 | LR: 2.07e-04 | 1333.2ms/step | 15.0 samples/s +Step 320/44000 | Loss: 6.1211 | LR: 2.13e-04 | 1335.4ms/step | 15.0 samples/s +Step 330/44000 | Loss: 6.0713 | LR: 2.20e-04 | 1335.9ms/step | 15.0 samples/s +Step 340/44000 | Loss: 6.0626 | LR: 2.27e-04 | 1333.8ms/step | 15.0 samples/s +Step 350/44000 | Loss: 6.0118 | LR: 2.33e-04 | 1333.7ms/step | 15.0 samples/s +Step 360/44000 | Loss: 5.9440 | LR: 2.40e-04 | 1333.8ms/step | 15.0 samples/s +Step 370/44000 | Loss: 5.9346 | LR: 2.47e-04 | 1333.9ms/step | 15.0 samples/s +Step 380/44000 | Loss: 5.8950 | LR: 2.53e-04 | 1334.5ms/step | 15.0 samples/s +Step 390/44000 | Loss: 5.8314 | LR: 2.60e-04 | 1336.1ms/step | 15.0 samples/s +Step 400/44000 | Loss: 5.8263 | LR: 2.67e-04 | 1335.6ms/step | 15.0 samples/s +Step 410/44000 | Loss: 5.7647 | LR: 2.73e-04 | 1333.1ms/step | 15.0 samples/s +Step 420/44000 | Loss: 5.7228 | LR: 2.80e-04 | 1334.9ms/step | 15.0 samples/s +Step 430/44000 | Loss: 5.7365 | LR: 2.87e-04 | 1334.3ms/step | 15.0 samples/s +Step 440/44000 | Loss: 5.6801 | LR: 2.93e-04 | 1335.4ms/step | 15.0 samples/s +Step 450/44000 | Loss: 5.6577 | LR: 3.00e-04 | 1331.2ms/step | 15.0 samples/s +Step 460/44000 | Loss: 5.6012 | LR: 3.07e-04 | 1331.1ms/step | 15.0 samples/s +Step 470/44000 | Loss: 5.5648 | LR: 3.13e-04 | 1334.4ms/step | 15.0 samples/s +Step 480/44000 | Loss: 5.5644 | LR: 3.20e-04 | 1336.4ms/step | 15.0 samples/s +Step 490/44000 | Loss: 5.4981 | LR: 3.27e-04 | 1335.9ms/step | 15.0 samples/s +Step 500/44000 | Loss: 5.4634 | LR: 3.33e-04 | 1335.4ms/step | 15.0 samples/s +Validation at step 500: Loss = 5.4682, Perplexity = 237.03 +Step 510/44000 | Loss: 5.4632 | LR: 3.40e-04 | 3582.5ms/step | 5.6 samples/s +Step 520/44000 | Loss: 5.4262 | LR: 3.47e-04 | 1329.9ms/step | 15.0 samples/s +Step 530/44000 | Loss: 5.3791 | LR: 3.53e-04 | 1330.6ms/step | 15.0 samples/s +Step 540/44000 | Loss: 5.3857 | LR: 3.60e-04 | 1329.6ms/step | 15.0 samples/s +Step 550/44000 | Loss: 5.3352 | LR: 3.67e-04 | 1331.2ms/step | 15.0 samples/s +Step 560/44000 | Loss: 5.2908 | LR: 3.73e-04 | 1334.0ms/step | 15.0 samples/s +Step 570/44000 | Loss: 5.2674 | LR: 3.80e-04 | 1333.1ms/step | 15.0 samples/s +Step 580/44000 | Loss: 5.2644 | LR: 3.87e-04 | 1334.2ms/step | 15.0 samples/s +Step 590/44000 | Loss: 5.2260 | LR: 3.93e-04 | 1334.4ms/step | 15.0 samples/s +Step 600/44000 | Loss: 5.2058 | LR: 4.00e-04 | 1333.7ms/step | 15.0 samples/s +Step 610/44000 | Loss: 5.1782 | LR: 4.07e-04 | 1333.1ms/step | 15.0 samples/s +Step 620/44000 | Loss: 5.1447 | LR: 4.13e-04 | 1333.5ms/step | 15.0 samples/s +Step 630/44000 | Loss: 5.0772 | LR: 4.20e-04 | 1332.7ms/step | 15.0 samples/s +Step 640/44000 | Loss: 5.0637 | LR: 4.27e-04 | 1333.7ms/step | 15.0 samples/s +Step 650/44000 | Loss: 5.0522 | LR: 4.33e-04 | 1333.3ms/step | 15.0 samples/s +Step 660/44000 | Loss: 5.0210 | LR: 4.40e-04 | 1329.7ms/step | 15.0 samples/s +Step 670/44000 | Loss: 5.0011 | LR: 4.47e-04 | 1335.9ms/step | 15.0 samples/s +Step 680/44000 | Loss: 4.9476 | LR: 4.53e-04 | 1333.3ms/step | 15.0 samples/s +Step 690/44000 | Loss: 4.9394 | LR: 4.60e-04 | 1333.4ms/step | 15.0 samples/s +Step 700/44000 | Loss: 4.8970 | LR: 4.67e-04 | 1327.6ms/step | 15.1 samples/s +Step 710/44000 | Loss: 4.8573 | LR: 4.73e-04 | 1328.4ms/step | 15.1 samples/s +Step 720/44000 | Loss: 4.8845 | LR: 4.80e-04 | 1326.3ms/step | 15.1 samples/s +Step 730/44000 | Loss: 4.8128 | LR: 4.87e-04 | 1328.5ms/step | 15.1 samples/s +Step 740/44000 | Loss: 4.8223 | LR: 4.93e-04 | 1332.9ms/step | 15.0 samples/s +Step 750/44000 | Loss: 4.7756 | LR: 5.00e-04 | 1327.6ms/step | 15.1 samples/s +Validation at step 750: Loss = 4.7902, Perplexity = 120.32 +Step 760/44000 | Loss: 4.7977 | LR: 5.00e-04 | 3569.8ms/step | 5.6 samples/s +Step 770/44000 | Loss: 4.7156 | LR: 5.00e-04 | 1328.2ms/step | 15.1 samples/s +Step 780/44000 | Loss: 4.7276 | LR: 5.00e-04 | 1326.3ms/step | 15.1 samples/s +Step 790/44000 | Loss: 4.6490 | LR: 5.00e-04 | 1328.8ms/step | 15.1 samples/s +Step 800/44000 | Loss: 4.6087 | LR: 5.00e-04 | 1330.3ms/step | 15.0 samples/s +Step 810/44000 | Loss: 4.6192 | LR: 5.00e-04 | 1331.3ms/step | 15.0 samples/s +Step 820/44000 | Loss: 4.6046 | LR: 5.00e-04 | 1332.3ms/step | 15.0 samples/s +Step 830/44000 | Loss: 4.5709 | LR: 5.00e-04 | 1329.4ms/step | 15.0 samples/s +Step 840/44000 | Loss: 4.5439 | LR: 5.00e-04 | 1330.0ms/step | 15.0 samples/s +Step 850/44000 | Loss: 4.5213 | LR: 5.00e-04 | 1327.9ms/step | 15.1 samples/s +Step 860/44000 | Loss: 4.5342 | LR: 5.00e-04 | 1331.1ms/step | 15.0 samples/s +Step 870/44000 | Loss: 4.4927 | LR: 5.00e-04 | 1334.0ms/step | 15.0 samples/s +Step 880/44000 | Loss: 4.4943 | LR: 5.00e-04 | 1330.1ms/step | 15.0 samples/s +Step 890/44000 | Loss: 4.4823 | LR: 5.00e-04 | 1333.3ms/step | 15.0 samples/s +Step 900/44000 | Loss: 4.4651 | LR: 5.00e-04 | 1331.1ms/step | 15.0 samples/s +Step 910/44000 | Loss: 4.4100 | LR: 5.00e-04 | 1330.9ms/step | 15.0 samples/s +Step 920/44000 | Loss: 4.4394 | LR: 5.00e-04 | 1329.0ms/step | 15.0 samples/s +Step 930/44000 | Loss: 4.3981 | LR: 5.00e-04 | 1332.7ms/step | 15.0 samples/s +Step 940/44000 | Loss: 4.3772 | LR: 5.00e-04 | 1328.9ms/step | 15.1 samples/s +Step 950/44000 | Loss: 4.3805 | LR: 5.00e-04 | 1332.0ms/step | 15.0 samples/s +Step 960/44000 | Loss: 4.3659 | LR: 5.00e-04 | 1330.1ms/step | 15.0 samples/s +Step 970/44000 | Loss: 4.3641 | LR: 5.00e-04 | 1331.1ms/step | 15.0 samples/s +Step 980/44000 | Loss: 4.3245 | LR: 5.00e-04 | 1330.8ms/step | 15.0 samples/s +Step 990/44000 | Loss: 4.3103 | LR: 5.00e-04 | 1332.9ms/step | 15.0 samples/s +Step 1000/44000 | Loss: 4.3125 | LR: 5.00e-04 | 1331.9ms/step | 15.0 samples/s +Validation at step 1000: Loss = 4.2913, Perplexity = 73.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-1000 +Step 1010/44000 | Loss: 4.2956 | LR: 5.00e-04 | 3875.1ms/step | 5.2 samples/s +Step 1020/44000 | Loss: 4.3136 | LR: 5.00e-04 | 1326.4ms/step | 15.1 samples/s +Step 1030/44000 | Loss: 4.2649 | LR: 5.00e-04 | 1327.2ms/step | 15.1 samples/s +Step 1040/44000 | Loss: 4.2702 | LR: 5.00e-04 | 1334.4ms/step | 15.0 samples/s +Step 1050/44000 | Loss: 4.2489 | LR: 5.00e-04 | 1330.4ms/step | 15.0 samples/s +Step 1060/44000 | Loss: 4.2319 | LR: 5.00e-04 | 1328.5ms/step | 15.1 samples/s +Step 1070/44000 | Loss: 4.2094 | LR: 5.00e-04 | 1327.6ms/step | 15.1 samples/s +Step 1080/44000 | Loss: 4.2201 | LR: 5.00e-04 | 1330.0ms/step | 15.0 samples/s +Step 1090/44000 | Loss: 4.2110 | LR: 5.00e-04 | 1328.3ms/step | 15.1 samples/s +Step 1100/44000 | Loss: 4.1852 | LR: 5.00e-04 | 1330.1ms/step | 15.0 samples/s +Step 1110/44000 | Loss: 4.1570 | LR: 5.00e-04 | 1326.8ms/step | 15.1 samples/s +Step 1120/44000 | Loss: 4.1851 | LR: 5.00e-04 | 1328.5ms/step | 15.1 samples/s +Step 1130/44000 | Loss: 4.1744 | LR: 5.00e-04 | 1329.0ms/step | 15.0 samples/s +Step 1140/44000 | Loss: 4.1578 | LR: 5.00e-04 | 1329.8ms/step | 15.0 samples/s +Step 1150/44000 | Loss: 4.1663 | LR: 5.00e-04 | 1328.7ms/step | 15.1 samples/s +Step 1160/44000 | Loss: 4.1383 | LR: 5.00e-04 | 1331.1ms/step | 15.0 samples/s +Step 1170/44000 | Loss: 4.1234 | LR: 5.00e-04 | 1331.0ms/step | 15.0 samples/s +Step 1180/44000 | Loss: 4.1150 | LR: 5.00e-04 | 1331.4ms/step | 15.0 samples/s +Step 1190/44000 | Loss: 4.1364 | LR: 5.00e-04 | 1331.9ms/step | 15.0 samples/s +Step 1200/44000 | Loss: 4.0822 | LR: 5.00e-04 | 1323.4ms/step | 15.1 samples/s +Step 1210/44000 | Loss: 4.1090 | LR: 5.00e-04 | 1331.0ms/step | 15.0 samples/s +Step 1220/44000 | Loss: 4.0898 | LR: 5.00e-04 | 1330.1ms/step | 15.0 samples/s +Step 1230/44000 | Loss: 4.0834 | LR: 5.00e-04 | 1329.1ms/step | 15.0 samples/s +Step 1240/44000 | Loss: 4.0642 | LR: 5.00e-04 | 1326.3ms/step | 15.1 samples/s +Step 1250/44000 | Loss: 4.0189 | LR: 5.00e-04 | 1328.9ms/step | 15.1 samples/s +Validation at step 1250: Loss = 4.0508, Perplexity = 57.44 +Step 1260/44000 | Loss: 4.0671 | LR: 5.00e-04 | 3565.4ms/step | 5.6 samples/s +Step 1270/44000 | Loss: 4.0729 | LR: 5.00e-04 | 1323.2ms/step | 15.1 samples/s +Step 1280/44000 | Loss: 4.0454 | LR: 5.00e-04 | 1325.0ms/step | 15.1 samples/s +Step 1290/44000 | Loss: 4.0118 | LR: 5.00e-04 | 1325.9ms/step | 15.1 samples/s +Step 1300/44000 | Loss: 4.0560 | LR: 5.00e-04 | 1325.4ms/step | 15.1 samples/s +Step 1310/44000 | Loss: 4.0315 | LR: 5.00e-04 | 1324.0ms/step | 15.1 samples/s +Step 1320/44000 | Loss: 4.0214 | LR: 5.00e-04 | 1328.5ms/step | 15.1 samples/s +Step 1330/44000 | Loss: 4.0118 | LR: 5.00e-04 | 1327.5ms/step | 15.1 samples/s +Step 1340/44000 | Loss: 4.0163 | LR: 5.00e-04 | 1325.5ms/step | 15.1 samples/s +Step 1350/44000 | Loss: 4.0163 | LR: 5.00e-04 | 1327.0ms/step | 15.1 samples/s +Step 1360/44000 | Loss: 3.9950 | LR: 5.00e-04 | 1326.4ms/step | 15.1 samples/s +Step 1370/44000 | Loss: 3.9800 | LR: 5.00e-04 | 1326.0ms/step | 15.1 samples/s +Step 1380/44000 | Loss: 3.9684 | LR: 5.00e-04 | 1324.7ms/step | 15.1 samples/s +Step 1390/44000 | Loss: 3.9715 | LR: 5.00e-04 | 1325.3ms/step | 15.1 samples/s +Step 1400/44000 | Loss: 3.9416 | LR: 5.00e-04 | 1327.1ms/step | 15.1 samples/s +Step 1410/44000 | Loss: 3.9486 | LR: 5.00e-04 | 1326.9ms/step | 15.1 samples/s +Step 1420/44000 | Loss: 3.9343 | LR: 5.00e-04 | 1326.1ms/step | 15.1 samples/s +Step 1430/44000 | Loss: 3.9486 | LR: 5.00e-04 | 1326.5ms/step | 15.1 samples/s +Step 1440/44000 | Loss: 3.9166 | LR: 5.00e-04 | 1323.6ms/step | 15.1 samples/s +Step 1450/44000 | Loss: 3.9623 | LR: 5.00e-04 | 1328.4ms/step | 15.1 samples/s +Step 1460/44000 | Loss: 3.9296 | LR: 5.00e-04 | 1326.8ms/step | 15.1 samples/s +Step 1470/44000 | Loss: 3.9314 | LR: 5.00e-04 | 1328.4ms/step | 15.1 samples/s +Step 1480/44000 | Loss: 3.9194 | LR: 5.00e-04 | 1325.2ms/step | 15.1 samples/s +Step 1490/44000 | Loss: 3.9032 | LR: 5.00e-04 | 1326.5ms/step | 15.1 samples/s +Step 1500/44000 | Loss: 3.8955 | LR: 5.00e-04 | 1329.8ms/step | 15.0 samples/s +Validation at step 1500: Loss = 3.8973, Perplexity = 49.27 +Step 1510/44000 | Loss: 3.9130 | LR: 5.00e-04 | 3557.0ms/step | 5.6 samples/s +Step 1520/44000 | Loss: 3.9234 | LR: 5.00e-04 | 1324.1ms/step | 15.1 samples/s +Step 1530/44000 | Loss: 3.9083 | LR: 5.00e-04 | 1326.1ms/step | 15.1 samples/s +Step 1540/44000 | Loss: 3.9078 | LR: 5.00e-04 | 1327.2ms/step | 15.1 samples/s +Step 1550/44000 | Loss: 3.8639 | LR: 5.00e-04 | 1324.6ms/step | 15.1 samples/s +Step 1560/44000 | Loss: 3.8644 | LR: 5.00e-04 | 1327.0ms/step | 15.1 samples/s +Step 1570/44000 | Loss: 3.8896 | LR: 5.00e-04 | 1325.9ms/step | 15.1 samples/s +Step 1580/44000 | Loss: 3.8624 | LR: 5.00e-04 | 1324.0ms/step | 15.1 samples/s +Step 1590/44000 | Loss: 3.8654 | LR: 5.00e-04 | 1325.2ms/step | 15.1 samples/s +Step 1600/44000 | Loss: 3.8878 | LR: 5.00e-04 | 1326.7ms/step | 15.1 samples/s +Step 1610/44000 | Loss: 3.8296 | LR: 5.00e-04 | 1326.6ms/step | 15.1 samples/s +Step 1620/44000 | Loss: 3.8502 | LR: 5.00e-04 | 1324.6ms/step | 15.1 samples/s +Step 1630/44000 | Loss: 3.8383 | LR: 4.99e-04 | 1324.8ms/step | 15.1 samples/s +Step 1640/44000 | Loss: 3.8376 | LR: 4.99e-04 | 1326.3ms/step | 15.1 samples/s +Step 1650/44000 | Loss: 3.8832 | LR: 4.99e-04 | 1325.2ms/step | 15.1 samples/s +Step 1660/44000 | Loss: 3.8364 | LR: 4.99e-04 | 1326.0ms/step | 15.1 samples/s +Step 1670/44000 | Loss: 3.8391 | LR: 4.99e-04 | 1325.7ms/step | 15.1 samples/s +Step 1680/44000 | Loss: 3.8352 | LR: 4.99e-04 | 1324.9ms/step | 15.1 samples/s +Step 1690/44000 | Loss: 3.8338 | LR: 4.99e-04 | 1325.7ms/step | 15.1 samples/s +Step 1700/44000 | Loss: 3.7578 | LR: 4.99e-04 | 1324.6ms/step | 15.1 samples/s +Step 1710/44000 | Loss: 3.8241 | LR: 4.99e-04 | 1325.5ms/step | 15.1 samples/s +Step 1720/44000 | Loss: 3.8086 | LR: 4.99e-04 | 1326.6ms/step | 15.1 samples/s +Step 1730/44000 | Loss: 3.7893 | LR: 4.99e-04 | 1324.8ms/step | 15.1 samples/s +Step 1740/44000 | Loss: 3.8001 | LR: 4.99e-04 | 1325.4ms/step | 15.1 samples/s +Step 1750/44000 | Loss: 3.8046 | LR: 4.99e-04 | 1326.1ms/step | 15.1 samples/s +Validation at step 1750: Loss = 3.7872, Perplexity = 44.13 +Step 1760/44000 | Loss: 3.7881 | LR: 4.99e-04 | 3563.5ms/step | 5.6 samples/s +Step 1770/44000 | Loss: 3.7767 | LR: 4.99e-04 | 1326.9ms/step | 15.1 samples/s +Step 1780/44000 | Loss: 3.7725 | LR: 4.99e-04 | 1324.6ms/step | 15.1 samples/s +Step 1790/44000 | Loss: 3.7875 | LR: 4.99e-04 | 1326.7ms/step | 15.1 samples/s +Step 1800/44000 | Loss: 3.7875 | LR: 4.99e-04 | 1327.5ms/step | 15.1 samples/s +Step 1810/44000 | Loss: 3.7748 | LR: 4.99e-04 | 1326.0ms/step | 15.1 samples/s +Step 1820/44000 | Loss: 3.7746 | LR: 4.99e-04 | 1324.4ms/step | 15.1 samples/s +Step 1830/44000 | Loss: 3.7723 | LR: 4.99e-04 | 1325.4ms/step | 15.1 samples/s +Step 1840/44000 | Loss: 3.7613 | LR: 4.99e-04 | 1324.5ms/step | 15.1 samples/s +Step 1850/44000 | Loss: 3.7591 | LR: 4.99e-04 | 1324.2ms/step | 15.1 samples/s +Step 1860/44000 | Loss: 3.7535 | LR: 4.99e-04 | 1329.9ms/step | 15.0 samples/s +Step 1870/44000 | Loss: 3.7698 | LR: 4.99e-04 | 1324.9ms/step | 15.1 samples/s +Step 1880/44000 | Loss: 3.7628 | LR: 4.99e-04 | 1323.7ms/step | 15.1 samples/s +Step 1890/44000 | Loss: 3.7765 | LR: 4.99e-04 | 1324.2ms/step | 15.1 samples/s +Step 1900/44000 | Loss: 3.7334 | LR: 4.99e-04 | 1324.3ms/step | 15.1 samples/s +Step 1910/44000 | Loss: 3.7485 | LR: 4.99e-04 | 1327.7ms/step | 15.1 samples/s +Step 1920/44000 | Loss: 3.7538 | LR: 4.99e-04 | 1324.9ms/step | 15.1 samples/s +Step 1930/44000 | Loss: 3.7542 | LR: 4.99e-04 | 1324.8ms/step | 15.1 samples/s +Step 1940/44000 | Loss: 3.7361 | LR: 4.99e-04 | 1325.9ms/step | 15.1 samples/s +Step 1950/44000 | Loss: 3.7419 | LR: 4.99e-04 | 1327.8ms/step | 15.1 samples/s +Step 1960/44000 | Loss: 3.7282 | LR: 4.99e-04 | 1328.9ms/step | 15.1 samples/s +Step 1970/44000 | Loss: 3.7213 | LR: 4.99e-04 | 1327.1ms/step | 15.1 samples/s +Step 1980/44000 | Loss: 3.7184 | LR: 4.99e-04 | 1326.9ms/step | 15.1 samples/s +Step 1990/44000 | Loss: 3.7044 | LR: 4.99e-04 | 1327.9ms/step | 15.1 samples/s +Step 2000/44000 | Loss: 3.7224 | LR: 4.99e-04 | 1330.4ms/step | 15.0 samples/s +Validation at step 2000: Loss = 3.7006, Perplexity = 40.47 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-2000 +Step 2010/44000 | Loss: 3.6958 | LR: 4.99e-04 | 3871.0ms/step | 5.2 samples/s +Step 2020/44000 | Loss: 3.7004 | LR: 4.99e-04 | 1324.4ms/step | 15.1 samples/s +Step 2030/44000 | Loss: 3.7050 | LR: 4.99e-04 | 1324.3ms/step | 15.1 samples/s +Step 2040/44000 | Loss: 3.6979 | LR: 4.99e-04 | 1322.2ms/step | 15.1 samples/s +Step 2050/44000 | Loss: 3.7101 | LR: 4.99e-04 | 1326.2ms/step | 15.1 samples/s +Step 2060/44000 | Loss: 3.6868 | LR: 4.99e-04 | 1327.9ms/step | 15.1 samples/s +Step 2070/44000 | Loss: 3.6733 | LR: 4.99e-04 | 1322.9ms/step | 15.1 samples/s +Step 2080/44000 | Loss: 3.6714 | LR: 4.99e-04 | 1326.0ms/step | 15.1 samples/s +Step 2090/44000 | Loss: 3.6704 | LR: 4.99e-04 | 1326.7ms/step | 15.1 samples/s +Step 2100/44000 | Loss: 3.7091 | LR: 4.99e-04 | 1322.8ms/step | 15.1 samples/s +Step 2110/44000 | Loss: 3.6864 | LR: 4.99e-04 | 1322.8ms/step | 15.1 samples/s +Step 2120/44000 | Loss: 3.6969 | LR: 4.99e-04 | 1318.8ms/step | 15.2 samples/s +Step 2130/44000 | Loss: 3.6777 | LR: 4.99e-04 | 1322.8ms/step | 15.1 samples/s +Step 2140/44000 | Loss: 3.7004 | LR: 4.99e-04 | 1323.1ms/step | 15.1 samples/s +Step 2150/44000 | Loss: 3.6694 | LR: 4.99e-04 | 1319.9ms/step | 15.2 samples/s +Step 2160/44000 | Loss: 3.6785 | LR: 4.99e-04 | 1324.6ms/step | 15.1 samples/s +Step 2170/44000 | Loss: 3.6490 | LR: 4.99e-04 | 1319.9ms/step | 15.2 samples/s +Step 2180/44000 | Loss: 3.6541 | LR: 4.99e-04 | 1320.7ms/step | 15.1 samples/s +Step 2190/44000 | Loss: 3.6115 | LR: 4.99e-04 | 1318.9ms/step | 15.2 samples/s +Step 2200/44000 | Loss: 3.6711 | LR: 4.99e-04 | 1321.2ms/step | 15.1 samples/s +Step 2210/44000 | Loss: 3.6566 | LR: 4.99e-04 | 1322.5ms/step | 15.1 samples/s +Step 2220/44000 | Loss: 3.6314 | LR: 4.99e-04 | 1319.8ms/step | 15.2 samples/s +Step 2230/44000 | Loss: 3.6459 | LR: 4.99e-04 | 1322.5ms/step | 15.1 samples/s +Step 2240/44000 | Loss: 3.6454 | LR: 4.99e-04 | 1322.4ms/step | 15.1 samples/s +Step 2250/44000 | Loss: 3.6663 | LR: 4.99e-04 | 1323.7ms/step | 15.1 samples/s +Validation at step 2250: Loss = 3.6345, Perplexity = 37.88 +Step 2260/44000 | Loss: 3.6443 | LR: 4.98e-04 | 3552.4ms/step | 5.6 samples/s +Step 2270/44000 | Loss: 3.6247 | LR: 4.98e-04 | 1323.7ms/step | 15.1 samples/s +Step 2280/44000 | Loss: 3.6484 | LR: 4.98e-04 | 1323.9ms/step | 15.1 samples/s +Step 2290/44000 | Loss: 3.6318 | LR: 4.98e-04 | 1326.3ms/step | 15.1 samples/s +Step 2300/44000 | Loss: 3.6256 | LR: 4.98e-04 | 1322.5ms/step | 15.1 samples/s +Step 2310/44000 | Loss: 3.6239 | LR: 4.98e-04 | 1321.6ms/step | 15.1 samples/s +Step 2320/44000 | Loss: 3.6377 | LR: 4.98e-04 | 1320.5ms/step | 15.1 samples/s +Step 2330/44000 | Loss: 3.6475 | LR: 4.98e-04 | 1325.5ms/step | 15.1 samples/s +Step 2340/44000 | Loss: 3.5902 | LR: 4.98e-04 | 1326.1ms/step | 15.1 samples/s +Step 2350/44000 | Loss: 3.6175 | LR: 4.98e-04 | 1326.9ms/step | 15.1 samples/s +Step 2360/44000 | Loss: 3.6328 | LR: 4.98e-04 | 1324.7ms/step | 15.1 samples/s +Step 2370/44000 | Loss: 3.6092 | LR: 4.98e-04 | 1321.6ms/step | 15.1 samples/s +Step 2380/44000 | Loss: 3.5801 | LR: 4.98e-04 | 1324.8ms/step | 15.1 samples/s +Step 2390/44000 | Loss: 3.6545 | LR: 4.98e-04 | 1322.4ms/step | 15.1 samples/s +Step 2400/44000 | Loss: 3.6026 | LR: 4.98e-04 | 1328.0ms/step | 15.1 samples/s +Step 2410/44000 | Loss: 3.6168 | LR: 4.98e-04 | 1323.2ms/step | 15.1 samples/s +Step 2420/44000 | Loss: 3.6209 | LR: 4.98e-04 | 1325.4ms/step | 15.1 samples/s +Step 2430/44000 | Loss: 3.6312 | LR: 4.98e-04 | 1320.1ms/step | 15.2 samples/s +Step 2440/44000 | Loss: 3.5897 | LR: 4.98e-04 | 1318.7ms/step | 15.2 samples/s +Step 2450/44000 | Loss: 3.6028 | LR: 4.98e-04 | 1319.4ms/step | 15.2 samples/s +Step 2460/44000 | Loss: 3.6085 | LR: 4.98e-04 | 1322.7ms/step | 15.1 samples/s +Step 2470/44000 | Loss: 3.5825 | LR: 4.98e-04 | 1327.0ms/step | 15.1 samples/s +Step 2480/44000 | Loss: 3.5720 | LR: 4.98e-04 | 1327.5ms/step | 15.1 samples/s +Step 2490/44000 | Loss: 3.6320 | LR: 4.98e-04 | 1323.8ms/step | 15.1 samples/s +Step 2500/44000 | Loss: 3.6062 | LR: 4.98e-04 | 1320.5ms/step | 15.1 samples/s +Validation at step 2500: Loss = 3.5834, Perplexity = 35.99 +Step 2510/44000 | Loss: 3.5944 | LR: 4.98e-04 | 3549.0ms/step | 5.6 samples/s +Step 2520/44000 | Loss: 3.5544 | LR: 4.98e-04 | 1323.9ms/step | 15.1 samples/s +Step 2530/44000 | Loss: 3.5835 | LR: 4.98e-04 | 1319.0ms/step | 15.2 samples/s +Step 2540/44000 | Loss: 3.5617 | LR: 4.98e-04 | 1322.8ms/step | 15.1 samples/s +Step 2550/44000 | Loss: 3.6003 | LR: 4.98e-04 | 1323.1ms/step | 15.1 samples/s +Step 2560/44000 | Loss: 3.6126 | LR: 4.98e-04 | 1321.1ms/step | 15.1 samples/s +Step 2570/44000 | Loss: 3.5788 | LR: 4.98e-04 | 1323.3ms/step | 15.1 samples/s +Step 2580/44000 | Loss: 3.5772 | LR: 4.98e-04 | 1321.2ms/step | 15.1 samples/s +Step 2590/44000 | Loss: 3.5781 | LR: 4.98e-04 | 1321.3ms/step | 15.1 samples/s +Step 2600/44000 | Loss: 3.5701 | LR: 4.98e-04 | 1321.9ms/step | 15.1 samples/s +Step 2610/44000 | Loss: 3.5810 | LR: 4.98e-04 | 1319.7ms/step | 15.2 samples/s +Step 2620/44000 | Loss: 3.5695 | LR: 4.98e-04 | 1319.5ms/step | 15.2 samples/s +Step 2630/44000 | Loss: 3.6059 | LR: 4.98e-04 | 1319.5ms/step | 15.2 samples/s +Step 2640/44000 | Loss: 3.5503 | LR: 4.98e-04 | 1327.4ms/step | 15.1 samples/s +Step 2650/44000 | Loss: 3.5712 | LR: 4.98e-04 | 1323.9ms/step | 15.1 samples/s +Step 2660/44000 | Loss: 3.5713 | LR: 4.98e-04 | 1324.1ms/step | 15.1 samples/s +Step 2670/44000 | Loss: 3.5661 | LR: 4.98e-04 | 1323.9ms/step | 15.1 samples/s +Step 2680/44000 | Loss: 3.5805 | LR: 4.98e-04 | 1324.8ms/step | 15.1 samples/s +Step 2690/44000 | Loss: 3.5705 | LR: 4.98e-04 | 1323.7ms/step | 15.1 samples/s +Step 2700/44000 | Loss: 3.5796 | LR: 4.97e-04 | 1325.3ms/step | 15.1 samples/s +Step 2710/44000 | Loss: 3.5860 | LR: 4.97e-04 | 1323.1ms/step | 15.1 samples/s +Step 2720/44000 | Loss: 3.5365 | LR: 4.97e-04 | 1326.1ms/step | 15.1 samples/s +Step 2730/44000 | Loss: 3.5374 | LR: 4.97e-04 | 1323.6ms/step | 15.1 samples/s +Step 2740/44000 | Loss: 3.5505 | LR: 4.97e-04 | 1323.6ms/step | 15.1 samples/s +Step 2750/44000 | Loss: 3.5671 | LR: 4.97e-04 | 1322.8ms/step | 15.1 samples/s +Validation at step 2750: Loss = 3.5383, Perplexity = 34.41 +Step 2760/44000 | Loss: 3.5524 | LR: 4.97e-04 | 3559.4ms/step | 5.6 samples/s +Step 2770/44000 | Loss: 3.5692 | LR: 4.97e-04 | 1323.3ms/step | 15.1 samples/s +Step 2780/44000 | Loss: 3.5347 | LR: 4.97e-04 | 1320.4ms/step | 15.1 samples/s +Step 2790/44000 | Loss: 3.5392 | LR: 4.97e-04 | 1323.9ms/step | 15.1 samples/s +Step 2800/44000 | Loss: 3.5214 | LR: 4.97e-04 | 1324.7ms/step | 15.1 samples/s +Step 2810/44000 | Loss: 3.5462 | LR: 4.97e-04 | 1324.0ms/step | 15.1 samples/s +Step 2820/44000 | Loss: 3.4928 | LR: 4.97e-04 | 1324.4ms/step | 15.1 samples/s +Step 2830/44000 | Loss: 3.5512 | LR: 4.97e-04 | 1319.9ms/step | 15.2 samples/s +Step 2840/44000 | Loss: 3.5716 | LR: 4.97e-04 | 1330.5ms/step | 15.0 samples/s +Step 2850/44000 | Loss: 3.5091 | LR: 4.97e-04 | 1324.6ms/step | 15.1 samples/s +Step 2860/44000 | Loss: 3.5490 | LR: 4.97e-04 | 1325.8ms/step | 15.1 samples/s +Step 2870/44000 | Loss: 3.5490 | LR: 4.97e-04 | 1327.1ms/step | 15.1 samples/s +Step 2880/44000 | Loss: 3.5357 | LR: 4.97e-04 | 1325.8ms/step | 15.1 samples/s +Step 2890/44000 | Loss: 3.5062 | LR: 4.97e-04 | 1324.2ms/step | 15.1 samples/s +Step 2900/44000 | Loss: 3.5015 | LR: 4.97e-04 | 1322.0ms/step | 15.1 samples/s +Step 2910/44000 | Loss: 3.5188 | LR: 4.97e-04 | 1326.9ms/step | 15.1 samples/s +Step 2920/44000 | Loss: 3.5400 | LR: 4.97e-04 | 1326.0ms/step | 15.1 samples/s +Step 2930/44000 | Loss: 3.5394 | LR: 4.97e-04 | 1328.3ms/step | 15.1 samples/s +Step 2940/44000 | Loss: 3.5022 | LR: 4.97e-04 | 1326.0ms/step | 15.1 samples/s +Step 2950/44000 | Loss: 3.5027 | LR: 4.97e-04 | 1328.0ms/step | 15.1 samples/s +Step 2960/44000 | Loss: 3.4712 | LR: 4.97e-04 | 1328.7ms/step | 15.1 samples/s +Step 2970/44000 | Loss: 3.5361 | LR: 4.97e-04 | 1326.0ms/step | 15.1 samples/s +Step 2980/44000 | Loss: 3.5151 | LR: 4.97e-04 | 1327.0ms/step | 15.1 samples/s +Step 2990/44000 | Loss: 3.5409 | LR: 4.97e-04 | 1323.9ms/step | 15.1 samples/s +Step 3000/44000 | Loss: 3.5408 | LR: 4.97e-04 | 1325.6ms/step | 15.1 samples/s +Validation at step 3000: Loss = 3.5011, Perplexity = 33.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-3000 +Step 3010/44000 | Loss: 3.5054 | LR: 4.97e-04 | 3865.4ms/step | 5.2 samples/s +Step 3020/44000 | Loss: 3.5094 | LR: 4.97e-04 | 1321.9ms/step | 15.1 samples/s +Step 3030/44000 | Loss: 3.5389 | LR: 4.97e-04 | 1324.4ms/step | 15.1 samples/s +Step 3040/44000 | Loss: 3.4947 | LR: 4.97e-04 | 1324.6ms/step | 15.1 samples/s +Step 3050/44000 | Loss: 3.5158 | LR: 4.97e-04 | 1326.0ms/step | 15.1 samples/s +Step 3060/44000 | Loss: 3.4985 | LR: 4.96e-04 | 1324.1ms/step | 15.1 samples/s +Step 3070/44000 | Loss: 3.5076 | LR: 4.96e-04 | 1322.6ms/step | 15.1 samples/s +Step 3080/44000 | Loss: 3.4877 | LR: 4.96e-04 | 1325.5ms/step | 15.1 samples/s +Step 3090/44000 | Loss: 3.5020 | LR: 4.96e-04 | 1326.3ms/step | 15.1 samples/s +Step 3100/44000 | Loss: 3.5034 | LR: 4.96e-04 | 1324.8ms/step | 15.1 samples/s +Step 3110/44000 | Loss: 3.4875 | LR: 4.96e-04 | 1323.4ms/step | 15.1 samples/s +Step 3120/44000 | Loss: 3.5447 | LR: 4.96e-04 | 1322.3ms/step | 15.1 samples/s +Step 3130/44000 | Loss: 3.4975 | LR: 4.96e-04 | 1323.7ms/step | 15.1 samples/s +Step 3140/44000 | Loss: 3.4566 | LR: 4.96e-04 | 1325.6ms/step | 15.1 samples/s +Step 3150/44000 | Loss: 3.4981 | LR: 4.96e-04 | 1322.4ms/step | 15.1 samples/s +Step 3160/44000 | Loss: 3.4816 | LR: 4.96e-04 | 1324.9ms/step | 15.1 samples/s +Step 3170/44000 | Loss: 3.4991 | LR: 4.96e-04 | 1325.4ms/step | 15.1 samples/s +Step 3180/44000 | Loss: 3.4898 | LR: 4.96e-04 | 1324.1ms/step | 15.1 samples/s +Step 3190/44000 | Loss: 3.4642 | LR: 4.96e-04 | 1326.9ms/step | 15.1 samples/s +Step 3200/44000 | Loss: 3.5098 | LR: 4.96e-04 | 1324.7ms/step | 15.1 samples/s +Step 3210/44000 | Loss: 3.4862 | LR: 4.96e-04 | 1324.7ms/step | 15.1 samples/s +Step 3220/44000 | Loss: 3.4583 | LR: 4.96e-04 | 1326.8ms/step | 15.1 samples/s +Step 3230/44000 | Loss: 3.4665 | LR: 4.96e-04 | 1326.1ms/step | 15.1 samples/s +Step 3240/44000 | Loss: 3.4905 | LR: 4.96e-04 | 1325.0ms/step | 15.1 samples/s +Step 3250/44000 | Loss: 3.4790 | LR: 4.96e-04 | 1327.3ms/step | 15.1 samples/s +Validation at step 3250: Loss = 3.4679, Perplexity = 32.07 +Step 3260/44000 | Loss: 3.4849 | LR: 4.96e-04 | 3555.5ms/step | 5.6 samples/s +Step 3270/44000 | Loss: 3.4906 | LR: 4.96e-04 | 1324.1ms/step | 15.1 samples/s +Step 3280/44000 | Loss: 3.4541 | LR: 4.96e-04 | 1328.9ms/step | 15.1 samples/s +Step 3290/44000 | Loss: 3.4987 | LR: 4.96e-04 | 1325.7ms/step | 15.1 samples/s +Step 3300/44000 | Loss: 3.4565 | LR: 4.96e-04 | 1326.6ms/step | 15.1 samples/s +Step 3310/44000 | Loss: 3.4597 | LR: 4.96e-04 | 1324.1ms/step | 15.1 samples/s +Step 3320/44000 | Loss: 3.4482 | LR: 4.96e-04 | 1322.0ms/step | 15.1 samples/s +Step 3330/44000 | Loss: 3.4971 | LR: 4.96e-04 | 1325.3ms/step | 15.1 samples/s +Step 3340/44000 | Loss: 3.4466 | LR: 4.96e-04 | 1321.4ms/step | 15.1 samples/s +Step 3350/44000 | Loss: 3.4507 | LR: 4.96e-04 | 1322.5ms/step | 15.1 samples/s +Step 3360/44000 | Loss: 3.4444 | LR: 4.96e-04 | 1324.1ms/step | 15.1 samples/s +Step 3370/44000 | Loss: 3.4614 | LR: 4.95e-04 | 1326.6ms/step | 15.1 samples/s +Step 3380/44000 | Loss: 3.4747 | LR: 4.95e-04 | 1327.5ms/step | 15.1 samples/s +Step 3390/44000 | Loss: 3.5172 | LR: 4.95e-04 | 1324.4ms/step | 15.1 samples/s +Step 3400/44000 | Loss: 3.4440 | LR: 4.95e-04 | 1326.1ms/step | 15.1 samples/s +Step 3410/44000 | Loss: 3.4912 | LR: 4.95e-04 | 1322.1ms/step | 15.1 samples/s +Step 3420/44000 | Loss: 3.4606 | LR: 4.95e-04 | 1328.3ms/step | 15.1 samples/s +Step 3430/44000 | Loss: 3.4650 | LR: 4.95e-04 | 1327.7ms/step | 15.1 samples/s +Step 3440/44000 | Loss: 3.4299 | LR: 4.95e-04 | 1327.3ms/step | 15.1 samples/s +Step 3450/44000 | Loss: 3.4396 | LR: 4.95e-04 | 1328.9ms/step | 15.1 samples/s +Step 3460/44000 | Loss: 3.4856 | LR: 4.95e-04 | 1325.8ms/step | 15.1 samples/s +Step 3470/44000 | Loss: 3.4474 | LR: 4.95e-04 | 1324.3ms/step | 15.1 samples/s +Step 3480/44000 | Loss: 3.4525 | LR: 4.95e-04 | 1323.8ms/step | 15.1 samples/s +Step 3490/44000 | Loss: 3.4571 | LR: 4.95e-04 | 1325.8ms/step | 15.1 samples/s +Step 3500/44000 | Loss: 3.4812 | LR: 4.95e-04 | 1321.3ms/step | 15.1 samples/s +Validation at step 3500: Loss = 3.4429, Perplexity = 31.28 +Step 3510/44000 | Loss: 3.4777 | LR: 4.95e-04 | 3552.0ms/step | 5.6 samples/s +Step 3520/44000 | Loss: 3.4654 | LR: 4.95e-04 | 1323.0ms/step | 15.1 samples/s +Step 3530/44000 | Loss: 3.4502 | LR: 4.95e-04 | 1327.9ms/step | 15.1 samples/s +Step 3540/44000 | Loss: 3.4234 | LR: 4.95e-04 | 1325.4ms/step | 15.1 samples/s +Step 3550/44000 | Loss: 3.4719 | LR: 4.95e-04 | 1325.8ms/step | 15.1 samples/s +Step 3560/44000 | Loss: 3.4470 | LR: 4.95e-04 | 1325.9ms/step | 15.1 samples/s +Step 3570/44000 | Loss: 3.4369 | LR: 4.95e-04 | 1327.1ms/step | 15.1 samples/s +Step 3580/44000 | Loss: 3.4528 | LR: 4.95e-04 | 1326.9ms/step | 15.1 samples/s +Step 3590/44000 | Loss: 3.4301 | LR: 4.95e-04 | 1326.4ms/step | 15.1 samples/s +Step 3600/44000 | Loss: 3.4463 | LR: 4.95e-04 | 1322.0ms/step | 15.1 samples/s +Step 3610/44000 | Loss: 3.3878 | LR: 4.95e-04 | 1323.9ms/step | 15.1 samples/s +Step 3620/44000 | Loss: 3.4447 | LR: 4.95e-04 | 1327.3ms/step | 15.1 samples/s +Step 3630/44000 | Loss: 3.4506 | LR: 4.95e-04 | 1329.4ms/step | 15.0 samples/s +Step 3640/44000 | Loss: 3.4250 | LR: 4.95e-04 | 1326.6ms/step | 15.1 samples/s +Step 3650/44000 | Loss: 3.4425 | LR: 4.94e-04 | 1326.3ms/step | 15.1 samples/s +Step 3660/44000 | Loss: 3.4251 | LR: 4.94e-04 | 1326.9ms/step | 15.1 samples/s +Step 3670/44000 | Loss: 3.4387 | LR: 4.94e-04 | 1325.7ms/step | 15.1 samples/s +Step 3680/44000 | Loss: 3.4441 | LR: 4.94e-04 | 1328.1ms/step | 15.1 samples/s +Step 3690/44000 | Loss: 3.4009 | LR: 4.94e-04 | 1328.4ms/step | 15.1 samples/s +Step 3700/44000 | Loss: 3.4164 | LR: 4.94e-04 | 1326.4ms/step | 15.1 samples/s +Step 3710/44000 | Loss: 3.4444 | LR: 4.94e-04 | 1331.4ms/step | 15.0 samples/s +Step 3720/44000 | Loss: 3.4122 | LR: 4.94e-04 | 1334.1ms/step | 15.0 samples/s +Step 3730/44000 | Loss: 3.4141 | LR: 4.94e-04 | 1329.1ms/step | 15.0 samples/s +Step 3740/44000 | Loss: 3.4359 | LR: 4.94e-04 | 1324.1ms/step | 15.1 samples/s +Step 3750/44000 | Loss: 3.4253 | LR: 4.94e-04 | 1324.0ms/step | 15.1 samples/s +Validation at step 3750: Loss = 3.4150, Perplexity = 30.42 +Step 3760/44000 | Loss: 3.4426 | LR: 4.94e-04 | 3560.9ms/step | 5.6 samples/s +Step 3770/44000 | Loss: 3.4312 | LR: 4.94e-04 | 1327.0ms/step | 15.1 samples/s +Step 3780/44000 | Loss: 3.4113 | LR: 4.94e-04 | 1324.3ms/step | 15.1 samples/s +Step 3790/44000 | Loss: 3.4228 | LR: 4.94e-04 | 1328.4ms/step | 15.1 samples/s +Step 3800/44000 | Loss: 3.4131 | LR: 4.94e-04 | 1325.9ms/step | 15.1 samples/s +Step 3810/44000 | Loss: 3.4198 | LR: 4.94e-04 | 1327.8ms/step | 15.1 samples/s +Step 3820/44000 | Loss: 3.4239 | LR: 4.94e-04 | 1324.2ms/step | 15.1 samples/s +Step 3830/44000 | Loss: 3.4127 | LR: 4.94e-04 | 1327.2ms/step | 15.1 samples/s +Step 3840/44000 | Loss: 3.4209 | LR: 4.94e-04 | 1323.0ms/step | 15.1 samples/s +Step 3850/44000 | Loss: 3.4426 | LR: 4.94e-04 | 1325.7ms/step | 15.1 samples/s +Step 3860/44000 | Loss: 3.4106 | LR: 4.94e-04 | 1325.5ms/step | 15.1 samples/s +Step 3870/44000 | Loss: 3.4146 | LR: 4.94e-04 | 1324.5ms/step | 15.1 samples/s +Step 3880/44000 | Loss: 3.4468 | LR: 4.94e-04 | 1325.3ms/step | 15.1 samples/s +Step 3890/44000 | Loss: 3.3959 | LR: 4.94e-04 | 1320.7ms/step | 15.1 samples/s +Step 3900/44000 | Loss: 3.4214 | LR: 4.93e-04 | 1322.4ms/step | 15.1 samples/s +Step 3910/44000 | Loss: 3.4330 | LR: 4.93e-04 | 1326.7ms/step | 15.1 samples/s +Step 3920/44000 | Loss: 3.3792 | LR: 4.93e-04 | 1325.0ms/step | 15.1 samples/s +Step 3930/44000 | Loss: 3.3980 | LR: 4.93e-04 | 1329.1ms/step | 15.0 samples/s +Step 3940/44000 | Loss: 3.4334 | LR: 4.93e-04 | 1326.7ms/step | 15.1 samples/s +Step 3950/44000 | Loss: 3.4021 | LR: 4.93e-04 | 1325.1ms/step | 15.1 samples/s +Step 3960/44000 | Loss: 3.4098 | LR: 4.93e-04 | 1326.0ms/step | 15.1 samples/s +Step 3970/44000 | Loss: 3.4408 | LR: 4.93e-04 | 1329.3ms/step | 15.0 samples/s +Step 3980/44000 | Loss: 3.3857 | LR: 4.93e-04 | 1324.9ms/step | 15.1 samples/s +Step 3990/44000 | Loss: 3.4126 | LR: 4.93e-04 | 1329.2ms/step | 15.0 samples/s +Step 4000/44000 | Loss: 3.4262 | LR: 4.93e-04 | 1325.9ms/step | 15.1 samples/s +Validation at step 4000: Loss = 3.3925, Perplexity = 29.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-4000 +Step 4010/44000 | Loss: 3.3841 | LR: 4.93e-04 | 3930.1ms/step | 5.1 samples/s +Step 4020/44000 | Loss: 3.3930 | LR: 4.93e-04 | 1325.6ms/step | 15.1 samples/s +Step 4030/44000 | Loss: 3.3812 | LR: 4.93e-04 | 1325.4ms/step | 15.1 samples/s +Step 4040/44000 | Loss: 3.3583 | LR: 4.93e-04 | 1326.1ms/step | 15.1 samples/s +Step 4050/44000 | Loss: 3.3920 | LR: 4.93e-04 | 1325.9ms/step | 15.1 samples/s +Step 4060/44000 | Loss: 3.3705 | LR: 4.93e-04 | 1324.9ms/step | 15.1 samples/s +Step 4070/44000 | Loss: 3.3897 | LR: 4.93e-04 | 1325.4ms/step | 15.1 samples/s +Step 4080/44000 | Loss: 3.3695 | LR: 4.93e-04 | 1321.9ms/step | 15.1 samples/s +Step 4090/44000 | Loss: 3.4083 | LR: 4.93e-04 | 1323.9ms/step | 15.1 samples/s +Step 4100/44000 | Loss: 3.4191 | LR: 4.93e-04 | 1322.6ms/step | 15.1 samples/s +Step 4110/44000 | Loss: 3.3862 | LR: 4.93e-04 | 1321.8ms/step | 15.1 samples/s +Step 4120/44000 | Loss: 3.3896 | LR: 4.93e-04 | 1320.3ms/step | 15.1 samples/s +Step 4130/44000 | Loss: 3.4103 | LR: 4.93e-04 | 1324.0ms/step | 15.1 samples/s +Step 4140/44000 | Loss: 3.3860 | LR: 4.92e-04 | 1319.8ms/step | 15.2 samples/s +Step 4150/44000 | Loss: 3.3975 | LR: 4.92e-04 | 1321.5ms/step | 15.1 samples/s +Step 4160/44000 | Loss: 3.3519 | LR: 4.92e-04 | 1321.9ms/step | 15.1 samples/s +Step 4170/44000 | Loss: 3.3892 | LR: 4.92e-04 | 1323.0ms/step | 15.1 samples/s +Step 4180/44000 | Loss: 3.3745 | LR: 4.92e-04 | 1320.3ms/step | 15.1 samples/s +Step 4190/44000 | Loss: 3.3923 | LR: 4.92e-04 | 1322.0ms/step | 15.1 samples/s +Step 4200/44000 | Loss: 3.3895 | LR: 4.92e-04 | 1322.3ms/step | 15.1 samples/s +Step 4210/44000 | Loss: 3.3812 | LR: 4.92e-04 | 1323.5ms/step | 15.1 samples/s +Step 4220/44000 | Loss: 3.3631 | LR: 4.92e-04 | 1323.1ms/step | 15.1 samples/s +Step 4230/44000 | Loss: 3.3878 | LR: 4.92e-04 | 1323.1ms/step | 15.1 samples/s +Step 4240/44000 | Loss: 3.3787 | LR: 4.92e-04 | 1328.9ms/step | 15.0 samples/s +Step 4250/44000 | Loss: 3.3471 | LR: 4.92e-04 | 1323.0ms/step | 15.1 samples/s +Validation at step 4250: Loss = 3.3709, Perplexity = 29.11 +Step 4260/44000 | Loss: 3.3900 | LR: 4.92e-04 | 3548.1ms/step | 5.6 samples/s +Step 4270/44000 | Loss: 3.3907 | LR: 4.92e-04 | 1326.6ms/step | 15.1 samples/s +Step 4280/44000 | Loss: 3.3629 | LR: 4.92e-04 | 1325.2ms/step | 15.1 samples/s +Step 4290/44000 | Loss: 3.3599 | LR: 4.92e-04 | 1324.8ms/step | 15.1 samples/s +Step 4300/44000 | Loss: 3.4086 | LR: 4.92e-04 | 1324.5ms/step | 15.1 samples/s +Step 4310/44000 | Loss: 3.4039 | LR: 4.92e-04 | 1325.7ms/step | 15.1 samples/s +Step 4320/44000 | Loss: 3.3929 | LR: 4.92e-04 | 1324.8ms/step | 15.1 samples/s +Step 4330/44000 | Loss: 3.3710 | LR: 4.92e-04 | 1321.2ms/step | 15.1 samples/s +Step 4340/44000 | Loss: 3.3572 | LR: 4.92e-04 | 1322.9ms/step | 15.1 samples/s +Step 4350/44000 | Loss: 3.3688 | LR: 4.92e-04 | 1325.0ms/step | 15.1 samples/s +Step 4360/44000 | Loss: 3.3732 | LR: 4.91e-04 | 1321.7ms/step | 15.1 samples/s +Step 4370/44000 | Loss: 3.3643 | LR: 4.91e-04 | 1318.1ms/step | 15.2 samples/s +Step 4380/44000 | Loss: 3.3709 | LR: 4.91e-04 | 1321.9ms/step | 15.1 samples/s +Step 4390/44000 | Loss: 3.3702 | LR: 4.91e-04 | 1320.6ms/step | 15.1 samples/s +Step 4400/44000 | Loss: 3.3393 | LR: 4.91e-04 | 1318.6ms/step | 15.2 samples/s +Step 4410/44000 | Loss: 3.3982 | LR: 4.91e-04 | 1324.2ms/step | 15.1 samples/s +Step 4420/44000 | Loss: 3.3814 | LR: 4.91e-04 | 1328.7ms/step | 15.1 samples/s +Step 4430/44000 | Loss: 3.3417 | LR: 4.91e-04 | 1324.0ms/step | 15.1 samples/s +Step 4440/44000 | Loss: 3.3596 | LR: 4.91e-04 | 1325.7ms/step | 15.1 samples/s +Step 4450/44000 | Loss: 3.3481 | LR: 4.91e-04 | 1324.0ms/step | 15.1 samples/s +Step 4460/44000 | Loss: 3.3718 | LR: 4.91e-04 | 1320.6ms/step | 15.1 samples/s +Step 4470/44000 | Loss: 3.3679 | LR: 4.91e-04 | 1321.1ms/step | 15.1 samples/s +Step 4480/44000 | Loss: 3.3528 | LR: 4.91e-04 | 1323.1ms/step | 15.1 samples/s +Step 4490/44000 | Loss: 3.3858 | LR: 4.91e-04 | 1326.8ms/step | 15.1 samples/s +Step 4500/44000 | Loss: 3.3632 | LR: 4.91e-04 | 1321.1ms/step | 15.1 samples/s +Validation at step 4500: Loss = 3.3569, Perplexity = 28.70 +Step 4510/44000 | Loss: 3.3654 | LR: 4.91e-04 | 3552.6ms/step | 5.6 samples/s +Step 4520/44000 | Loss: 3.3374 | LR: 4.91e-04 | 1323.8ms/step | 15.1 samples/s +Step 4530/44000 | Loss: 3.3535 | LR: 4.91e-04 | 1329.3ms/step | 15.0 samples/s +Step 4540/44000 | Loss: 3.3553 | LR: 4.91e-04 | 1325.0ms/step | 15.1 samples/s +Step 4550/44000 | Loss: 3.3296 | LR: 4.91e-04 | 1326.2ms/step | 15.1 samples/s +Step 4560/44000 | Loss: 3.3572 | LR: 4.90e-04 | 1323.8ms/step | 15.1 samples/s +Step 4570/44000 | Loss: 3.3701 | LR: 4.90e-04 | 1325.1ms/step | 15.1 samples/s +Step 4580/44000 | Loss: 3.3306 | LR: 4.90e-04 | 1326.7ms/step | 15.1 samples/s +Step 4590/44000 | Loss: 3.3544 | LR: 4.90e-04 | 1325.5ms/step | 15.1 samples/s +Step 4600/44000 | Loss: 3.3773 | LR: 4.90e-04 | 1322.3ms/step | 15.1 samples/s +Step 4610/44000 | Loss: 3.3867 | LR: 4.90e-04 | 1327.2ms/step | 15.1 samples/s +Step 4620/44000 | Loss: 3.3518 | LR: 4.90e-04 | 1324.8ms/step | 15.1 samples/s +Step 4630/44000 | Loss: 3.3344 | LR: 4.90e-04 | 1324.3ms/step | 15.1 samples/s +Step 4640/44000 | Loss: 3.3503 | LR: 4.90e-04 | 1326.9ms/step | 15.1 samples/s +Step 4650/44000 | Loss: 3.3811 | LR: 4.90e-04 | 1325.2ms/step | 15.1 samples/s +Step 4660/44000 | Loss: 3.3331 | LR: 4.90e-04 | 1324.6ms/step | 15.1 samples/s +Step 4670/44000 | Loss: 3.3723 | LR: 4.90e-04 | 1324.4ms/step | 15.1 samples/s +Step 4680/44000 | Loss: 3.3352 | LR: 4.90e-04 | 1324.9ms/step | 15.1 samples/s +Step 4690/44000 | Loss: 3.3799 | LR: 4.90e-04 | 1322.7ms/step | 15.1 samples/s +Step 4700/44000 | Loss: 3.3529 | LR: 4.90e-04 | 1324.2ms/step | 15.1 samples/s +Step 4710/44000 | Loss: 3.3320 | LR: 4.90e-04 | 1324.8ms/step | 15.1 samples/s +Step 4720/44000 | Loss: 3.3081 | LR: 4.90e-04 | 1326.4ms/step | 15.1 samples/s +Step 4730/44000 | Loss: 3.3483 | LR: 4.90e-04 | 1323.7ms/step | 15.1 samples/s +Step 4740/44000 | Loss: 3.3603 | LR: 4.90e-04 | 1324.5ms/step | 15.1 samples/s +Step 4750/44000 | Loss: 3.3613 | LR: 4.90e-04 | 1323.6ms/step | 15.1 samples/s +Validation at step 4750: Loss = 3.3339, Perplexity = 28.05 +Step 4760/44000 | Loss: 3.3459 | LR: 4.89e-04 | 3556.4ms/step | 5.6 samples/s +Step 4770/44000 | Loss: 3.3405 | LR: 4.89e-04 | 1328.4ms/step | 15.1 samples/s +Step 4780/44000 | Loss: 3.3710 | LR: 4.89e-04 | 1324.4ms/step | 15.1 samples/s +Step 4790/44000 | Loss: 3.3310 | LR: 4.89e-04 | 1324.9ms/step | 15.1 samples/s +Step 4800/44000 | Loss: 3.3835 | LR: 4.89e-04 | 1326.9ms/step | 15.1 samples/s +Step 4810/44000 | Loss: 3.3297 | LR: 4.89e-04 | 1328.6ms/step | 15.1 samples/s +Step 4820/44000 | Loss: 3.3209 | LR: 4.89e-04 | 1323.7ms/step | 15.1 samples/s +Step 4830/44000 | Loss: 3.3816 | LR: 4.89e-04 | 1327.7ms/step | 15.1 samples/s +Step 4840/44000 | Loss: 3.3537 | LR: 4.89e-04 | 1326.6ms/step | 15.1 samples/s +Step 4850/44000 | Loss: 3.3582 | LR: 4.89e-04 | 1324.5ms/step | 15.1 samples/s +Step 4860/44000 | Loss: 3.3171 | LR: 4.89e-04 | 1325.1ms/step | 15.1 samples/s +Step 4870/44000 | Loss: 3.3434 | LR: 4.89e-04 | 1324.9ms/step | 15.1 samples/s +Step 4880/44000 | Loss: 3.3396 | LR: 4.89e-04 | 1328.2ms/step | 15.1 samples/s +Step 4890/44000 | Loss: 3.3634 | LR: 4.89e-04 | 1323.8ms/step | 15.1 samples/s +Step 4900/44000 | Loss: 3.3400 | LR: 4.89e-04 | 1326.1ms/step | 15.1 samples/s +Step 4910/44000 | Loss: 3.3527 | LR: 4.89e-04 | 1324.5ms/step | 15.1 samples/s +Step 4920/44000 | Loss: 3.2911 | LR: 4.89e-04 | 1324.7ms/step | 15.1 samples/s +Step 4930/44000 | Loss: 3.3202 | LR: 4.89e-04 | 1329.5ms/step | 15.0 samples/s +Step 4940/44000 | Loss: 3.3181 | LR: 4.89e-04 | 1324.2ms/step | 15.1 samples/s +Step 4950/44000 | Loss: 3.3280 | LR: 4.88e-04 | 1330.1ms/step | 15.0 samples/s +Step 4960/44000 | Loss: 3.2955 | LR: 4.88e-04 | 1326.5ms/step | 15.1 samples/s +Step 4970/44000 | Loss: 3.3367 | LR: 4.88e-04 | 1327.9ms/step | 15.1 samples/s +Step 4980/44000 | Loss: 3.3266 | LR: 4.88e-04 | 1328.5ms/step | 15.1 samples/s +Step 4990/44000 | Loss: 3.3159 | LR: 4.88e-04 | 1326.4ms/step | 15.1 samples/s +Step 5000/44000 | Loss: 3.3120 | LR: 4.88e-04 | 1327.3ms/step | 15.1 samples/s +Validation at step 5000: Loss = 3.3192, Perplexity = 27.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-5000 +Step 5010/44000 | Loss: 3.3105 | LR: 4.88e-04 | 3869.8ms/step | 5.2 samples/s +Step 5020/44000 | Loss: 3.3274 | LR: 4.88e-04 | 1330.9ms/step | 15.0 samples/s +Step 5030/44000 | Loss: 3.3561 | LR: 4.88e-04 | 1329.5ms/step | 15.0 samples/s +Step 5040/44000 | Loss: 3.3240 | LR: 4.88e-04 | 1330.1ms/step | 15.0 samples/s +Step 5050/44000 | Loss: 3.3198 | LR: 4.88e-04 | 1334.1ms/step | 15.0 samples/s +Step 5060/44000 | Loss: 3.3291 | LR: 4.88e-04 | 1323.6ms/step | 15.1 samples/s +Step 5070/44000 | Loss: 3.3395 | LR: 4.88e-04 | 1325.8ms/step | 15.1 samples/s +Step 5080/44000 | Loss: 3.3431 | LR: 4.88e-04 | 1325.9ms/step | 15.1 samples/s +Step 5090/44000 | Loss: 3.3504 | LR: 4.88e-04 | 1327.1ms/step | 15.1 samples/s +Step 5100/44000 | Loss: 3.3522 | LR: 4.88e-04 | 1328.4ms/step | 15.1 samples/s +Step 5110/44000 | Loss: 3.3138 | LR: 4.88e-04 | 1328.3ms/step | 15.1 samples/s +Step 5120/44000 | Loss: 3.3224 | LR: 4.88e-04 | 1327.0ms/step | 15.1 samples/s +Step 5130/44000 | Loss: 3.3303 | LR: 4.87e-04 | 1328.5ms/step | 15.1 samples/s +Step 5140/44000 | Loss: 3.3464 | LR: 4.87e-04 | 1329.1ms/step | 15.0 samples/s +Step 5150/44000 | Loss: 3.3147 | LR: 4.87e-04 | 1328.8ms/step | 15.1 samples/s +Step 5160/44000 | Loss: 3.3251 | LR: 4.87e-04 | 1325.9ms/step | 15.1 samples/s +Step 5170/44000 | Loss: 3.2924 | LR: 4.87e-04 | 1328.3ms/step | 15.1 samples/s +Step 5180/44000 | Loss: 3.2904 | LR: 4.87e-04 | 1328.5ms/step | 15.1 samples/s +Step 5190/44000 | Loss: 3.3156 | LR: 4.87e-04 | 1324.4ms/step | 15.1 samples/s +Step 5200/44000 | Loss: 3.2922 | LR: 4.87e-04 | 1325.6ms/step | 15.1 samples/s +Step 5210/44000 | Loss: 3.2842 | LR: 4.87e-04 | 1322.7ms/step | 15.1 samples/s +Step 5220/44000 | Loss: 3.2853 | LR: 4.87e-04 | 1324.3ms/step | 15.1 samples/s +Step 5230/44000 | Loss: 3.3002 | LR: 4.87e-04 | 1323.6ms/step | 15.1 samples/s +Step 5240/44000 | Loss: 3.2846 | LR: 4.87e-04 | 1328.7ms/step | 15.1 samples/s +Step 5250/44000 | Loss: 3.3206 | LR: 4.87e-04 | 1326.5ms/step | 15.1 samples/s +Validation at step 5250: Loss = 3.3059, Perplexity = 27.27 +Step 5260/44000 | Loss: 3.3129 | LR: 4.87e-04 | 3562.2ms/step | 5.6 samples/s +Step 5270/44000 | Loss: 3.3270 | LR: 4.87e-04 | 1329.0ms/step | 15.0 samples/s +Step 5280/44000 | Loss: 3.3118 | LR: 4.87e-04 | 1327.7ms/step | 15.1 samples/s +Step 5290/44000 | Loss: 3.3182 | LR: 4.87e-04 | 1330.3ms/step | 15.0 samples/s +Step 5300/44000 | Loss: 3.2927 | LR: 4.86e-04 | 1328.3ms/step | 15.1 samples/s +Step 5310/44000 | Loss: 3.3150 | LR: 4.86e-04 | 1327.1ms/step | 15.1 samples/s +Step 5320/44000 | Loss: 3.3104 | LR: 4.86e-04 | 1328.2ms/step | 15.1 samples/s +Step 5330/44000 | Loss: 3.2742 | LR: 4.86e-04 | 1323.5ms/step | 15.1 samples/s +Step 5340/44000 | Loss: 3.3038 | LR: 4.86e-04 | 1324.8ms/step | 15.1 samples/s +Step 5350/44000 | Loss: 3.3268 | LR: 4.86e-04 | 1327.5ms/step | 15.1 samples/s +Step 5360/44000 | Loss: 3.3042 | LR: 4.86e-04 | 1329.5ms/step | 15.0 samples/s +Step 5370/44000 | Loss: 3.2820 | LR: 4.86e-04 | 1326.9ms/step | 15.1 samples/s +Step 5380/44000 | Loss: 3.3322 | LR: 4.86e-04 | 1325.9ms/step | 15.1 samples/s +Step 5390/44000 | Loss: 3.3304 | LR: 4.86e-04 | 1329.6ms/step | 15.0 samples/s +Step 5400/44000 | Loss: 3.2924 | LR: 4.86e-04 | 1327.7ms/step | 15.1 samples/s +Step 5410/44000 | Loss: 3.3050 | LR: 4.86e-04 | 1327.4ms/step | 15.1 samples/s +Step 5420/44000 | Loss: 3.2807 | LR: 4.86e-04 | 1325.3ms/step | 15.1 samples/s +Step 5430/44000 | Loss: 3.3046 | LR: 4.86e-04 | 1324.5ms/step | 15.1 samples/s +Step 5440/44000 | Loss: 3.3241 | LR: 4.86e-04 | 1326.3ms/step | 15.1 samples/s +Step 5450/44000 | Loss: 3.3201 | LR: 4.86e-04 | 1328.6ms/step | 15.1 samples/s +Step 5460/44000 | Loss: 3.2871 | LR: 4.86e-04 | 1326.6ms/step | 15.1 samples/s +Step 5470/44000 | Loss: 3.2935 | LR: 4.85e-04 | 1327.6ms/step | 15.1 samples/s +Step 5480/44000 | Loss: 3.3042 | LR: 4.85e-04 | 1327.2ms/step | 15.1 samples/s +Step 5490/44000 | Loss: 3.3008 | LR: 4.85e-04 | 1328.4ms/step | 15.1 samples/s +Step 5500/44000 | Loss: 3.3094 | LR: 4.85e-04 | 1329.2ms/step | 15.0 samples/s +Validation at step 5500: Loss = 3.2912, Perplexity = 26.88 +Step 5510/44000 | Loss: 3.3202 | LR: 4.85e-04 | 3562.0ms/step | 5.6 samples/s +Step 5520/44000 | Loss: 3.3239 | LR: 4.85e-04 | 1324.3ms/step | 15.1 samples/s +Step 5530/44000 | Loss: 3.2868 | LR: 4.85e-04 | 1326.5ms/step | 15.1 samples/s +Step 5540/44000 | Loss: 3.3208 | LR: 4.85e-04 | 1327.9ms/step | 15.1 samples/s +Step 5550/44000 | Loss: 3.3279 | LR: 4.85e-04 | 1325.5ms/step | 15.1 samples/s +Step 5560/44000 | Loss: 3.3038 | LR: 4.85e-04 | 1326.4ms/step | 15.1 samples/s +Step 5570/44000 | Loss: 3.2869 | LR: 4.85e-04 | 1324.9ms/step | 15.1 samples/s +Step 5580/44000 | Loss: 3.2596 | LR: 4.85e-04 | 1323.1ms/step | 15.1 samples/s +Step 5590/44000 | Loss: 3.2989 | LR: 4.85e-04 | 1322.4ms/step | 15.1 samples/s +Step 5600/44000 | Loss: 3.2755 | LR: 4.85e-04 | 1325.2ms/step | 15.1 samples/s +Step 5610/44000 | Loss: 3.3014 | LR: 4.85e-04 | 1326.7ms/step | 15.1 samples/s +Step 5620/44000 | Loss: 3.2785 | LR: 4.85e-04 | 1323.2ms/step | 15.1 samples/s +Step 5630/44000 | Loss: 3.3111 | LR: 4.84e-04 | 1326.3ms/step | 15.1 samples/s +Step 5640/44000 | Loss: 3.3242 | LR: 4.84e-04 | 1325.9ms/step | 15.1 samples/s +Step 5650/44000 | Loss: 3.2891 | LR: 4.84e-04 | 1326.9ms/step | 15.1 samples/s +Step 5660/44000 | Loss: 3.3323 | LR: 4.84e-04 | 1327.0ms/step | 15.1 samples/s +Step 5670/44000 | Loss: 3.2809 | LR: 4.84e-04 | 1326.7ms/step | 15.1 samples/s +Step 5680/44000 | Loss: 3.2829 | LR: 4.84e-04 | 1325.2ms/step | 15.1 samples/s +Step 5690/44000 | Loss: 3.3015 | LR: 4.84e-04 | 1324.6ms/step | 15.1 samples/s +Step 5700/44000 | Loss: 3.3141 | LR: 4.84e-04 | 1325.4ms/step | 15.1 samples/s +Step 5710/44000 | Loss: 3.2914 | LR: 4.84e-04 | 1328.3ms/step | 15.1 samples/s +Step 5720/44000 | Loss: 3.2880 | LR: 4.84e-04 | 1320.8ms/step | 15.1 samples/s +Step 5730/44000 | Loss: 3.2679 | LR: 4.84e-04 | 1319.7ms/step | 15.2 samples/s +Step 5740/44000 | Loss: 3.2827 | LR: 4.84e-04 | 1321.2ms/step | 15.1 samples/s +Step 5750/44000 | Loss: 3.2838 | LR: 4.84e-04 | 1325.5ms/step | 15.1 samples/s +Validation at step 5750: Loss = 3.2782, Perplexity = 26.53 +Step 5760/44000 | Loss: 3.2560 | LR: 4.84e-04 | 3548.9ms/step | 5.6 samples/s +Step 5770/44000 | Loss: 3.2840 | LR: 4.84e-04 | 1323.1ms/step | 15.1 samples/s +Step 5780/44000 | Loss: 3.2712 | LR: 4.83e-04 | 1325.2ms/step | 15.1 samples/s +Step 5790/44000 | Loss: 3.2778 | LR: 4.83e-04 | 1325.1ms/step | 15.1 samples/s +Step 5800/44000 | Loss: 3.2808 | LR: 4.83e-04 | 1324.0ms/step | 15.1 samples/s +Step 5810/44000 | Loss: 3.2855 | LR: 4.83e-04 | 1320.4ms/step | 15.1 samples/s +Step 5820/44000 | Loss: 3.2831 | LR: 4.83e-04 | 1324.8ms/step | 15.1 samples/s +Step 5830/44000 | Loss: 3.2919 | LR: 4.83e-04 | 1320.3ms/step | 15.1 samples/s +Step 5840/44000 | Loss: 3.3093 | LR: 4.83e-04 | 1320.9ms/step | 15.1 samples/s +Step 5850/44000 | Loss: 3.2821 | LR: 4.83e-04 | 1320.9ms/step | 15.1 samples/s +Step 5860/44000 | Loss: 3.2823 | LR: 4.83e-04 | 1322.1ms/step | 15.1 samples/s +Step 5870/44000 | Loss: 3.2467 | LR: 4.83e-04 | 1326.3ms/step | 15.1 samples/s +Step 5880/44000 | Loss: 3.2763 | LR: 4.83e-04 | 1329.5ms/step | 15.0 samples/s +Step 5890/44000 | Loss: 3.2839 | LR: 4.83e-04 | 1326.1ms/step | 15.1 samples/s +Step 5900/44000 | Loss: 3.2799 | LR: 4.83e-04 | 1327.8ms/step | 15.1 samples/s +Step 5910/44000 | Loss: 3.2718 | LR: 4.83e-04 | 1325.0ms/step | 15.1 samples/s +Step 5920/44000 | Loss: 3.3126 | LR: 4.83e-04 | 1325.1ms/step | 15.1 samples/s +Step 5930/44000 | Loss: 3.2567 | LR: 4.83e-04 | 1325.9ms/step | 15.1 samples/s +Step 5940/44000 | Loss: 3.2856 | LR: 4.82e-04 | 1326.4ms/step | 15.1 samples/s +Step 5950/44000 | Loss: 3.2653 | LR: 4.82e-04 | 1327.9ms/step | 15.1 samples/s +Step 5960/44000 | Loss: 3.2996 | LR: 4.82e-04 | 1323.3ms/step | 15.1 samples/s +Step 5970/44000 | Loss: 3.2556 | LR: 4.82e-04 | 1325.8ms/step | 15.1 samples/s +Step 5980/44000 | Loss: 3.2803 | LR: 4.82e-04 | 1329.1ms/step | 15.0 samples/s +Step 5990/44000 | Loss: 3.2634 | LR: 4.82e-04 | 1325.8ms/step | 15.1 samples/s +Step 6000/44000 | Loss: 3.2700 | LR: 4.82e-04 | 1327.5ms/step | 15.1 samples/s +Validation at step 6000: Loss = 3.2669, Perplexity = 26.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-6000 +Step 6010/44000 | Loss: 3.2830 | LR: 4.82e-04 | 3864.5ms/step | 5.2 samples/s +Step 6020/44000 | Loss: 3.2847 | LR: 4.82e-04 | 1327.1ms/step | 15.1 samples/s +Step 6030/44000 | Loss: 3.2968 | LR: 4.82e-04 | 1326.1ms/step | 15.1 samples/s +Step 6040/44000 | Loss: 3.2784 | LR: 4.82e-04 | 1328.0ms/step | 15.1 samples/s +Step 6050/44000 | Loss: 3.2849 | LR: 4.82e-04 | 1328.1ms/step | 15.1 samples/s +Step 6060/44000 | Loss: 3.2761 | LR: 4.82e-04 | 1327.5ms/step | 15.1 samples/s +Step 6070/44000 | Loss: 3.2662 | LR: 4.82e-04 | 1328.0ms/step | 15.1 samples/s +Step 6080/44000 | Loss: 3.2872 | LR: 4.81e-04 | 1326.9ms/step | 15.1 samples/s +Step 6090/44000 | Loss: 3.2808 | LR: 4.81e-04 | 1326.6ms/step | 15.1 samples/s +Step 6100/44000 | Loss: 3.2814 | LR: 4.81e-04 | 1332.8ms/step | 15.0 samples/s +Step 6110/44000 | Loss: 3.2747 | LR: 4.81e-04 | 1329.4ms/step | 15.0 samples/s +Step 6120/44000 | Loss: 3.2682 | LR: 4.81e-04 | 1327.0ms/step | 15.1 samples/s +Step 6130/44000 | Loss: 3.2870 | LR: 4.81e-04 | 1327.2ms/step | 15.1 samples/s +Step 6140/44000 | Loss: 3.2833 | LR: 4.81e-04 | 1329.7ms/step | 15.0 samples/s +Step 6150/44000 | Loss: 3.2760 | LR: 4.81e-04 | 1324.8ms/step | 15.1 samples/s +Step 6160/44000 | Loss: 3.2653 | LR: 4.81e-04 | 1326.2ms/step | 15.1 samples/s +Step 6170/44000 | Loss: 3.2909 | LR: 4.81e-04 | 1328.7ms/step | 15.1 samples/s +Step 6180/44000 | Loss: 3.2825 | LR: 4.81e-04 | 1327.6ms/step | 15.1 samples/s +Step 6190/44000 | Loss: 3.2323 | LR: 4.81e-04 | 1325.7ms/step | 15.1 samples/s +Step 6200/44000 | Loss: 3.2701 | LR: 4.81e-04 | 1322.7ms/step | 15.1 samples/s +Step 6210/44000 | Loss: 3.2925 | LR: 4.81e-04 | 1322.8ms/step | 15.1 samples/s +Step 6220/44000 | Loss: 3.2586 | LR: 4.81e-04 | 1328.1ms/step | 15.1 samples/s +Step 6230/44000 | Loss: 3.2778 | LR: 4.80e-04 | 1328.6ms/step | 15.1 samples/s +Step 6240/44000 | Loss: 3.2871 | LR: 4.80e-04 | 1329.6ms/step | 15.0 samples/s +Step 6250/44000 | Loss: 3.2674 | LR: 4.80e-04 | 1328.9ms/step | 15.0 samples/s +Validation at step 6250: Loss = 3.2549, Perplexity = 25.92 +Step 6260/44000 | Loss: 3.2759 | LR: 4.80e-04 | 3564.3ms/step | 5.6 samples/s +Step 6270/44000 | Loss: 3.2571 | LR: 4.80e-04 | 1326.3ms/step | 15.1 samples/s +Step 6280/44000 | Loss: 3.2709 | LR: 4.80e-04 | 1327.1ms/step | 15.1 samples/s +Step 6290/44000 | Loss: 3.2969 | LR: 4.80e-04 | 1323.4ms/step | 15.1 samples/s +Step 6300/44000 | Loss: 3.2724 | LR: 4.80e-04 | 1327.2ms/step | 15.1 samples/s +Step 6310/44000 | Loss: 3.2351 | LR: 4.80e-04 | 1329.1ms/step | 15.0 samples/s +Step 6320/44000 | Loss: 3.2364 | LR: 4.80e-04 | 1328.2ms/step | 15.1 samples/s +Step 6330/44000 | Loss: 3.2707 | LR: 4.80e-04 | 1321.9ms/step | 15.1 samples/s +Step 6340/44000 | Loss: 3.2654 | LR: 4.80e-04 | 1324.6ms/step | 15.1 samples/s +Step 6350/44000 | Loss: 3.2562 | LR: 4.80e-04 | 1326.8ms/step | 15.1 samples/s +Step 6360/44000 | Loss: 3.2492 | LR: 4.80e-04 | 1327.6ms/step | 15.1 samples/s +Step 6370/44000 | Loss: 3.2389 | LR: 4.79e-04 | 1325.4ms/step | 15.1 samples/s +Step 6380/44000 | Loss: 3.2602 | LR: 4.79e-04 | 1327.2ms/step | 15.1 samples/s +Step 6390/44000 | Loss: 3.2689 | LR: 4.79e-04 | 1326.9ms/step | 15.1 samples/s +Step 6400/44000 | Loss: 3.2626 | LR: 4.79e-04 | 1330.7ms/step | 15.0 samples/s +Step 6410/44000 | Loss: 3.2637 | LR: 4.79e-04 | 1325.1ms/step | 15.1 samples/s +Step 6420/44000 | Loss: 3.2505 | LR: 4.79e-04 | 1330.1ms/step | 15.0 samples/s +Step 6430/44000 | Loss: 3.2940 | LR: 4.79e-04 | 1326.1ms/step | 15.1 samples/s +Step 6440/44000 | Loss: 3.2707 | LR: 4.79e-04 | 1325.8ms/step | 15.1 samples/s +Step 6450/44000 | Loss: 3.2446 | LR: 4.79e-04 | 1327.0ms/step | 15.1 samples/s +Step 6460/44000 | Loss: 3.2472 | LR: 4.79e-04 | 1329.2ms/step | 15.0 samples/s +Step 6470/44000 | Loss: 3.2456 | LR: 4.79e-04 | 1328.4ms/step | 15.1 samples/s +Step 6480/44000 | Loss: 3.2323 | LR: 4.79e-04 | 1327.3ms/step | 15.1 samples/s +Step 6490/44000 | Loss: 3.2270 | LR: 4.79e-04 | 1328.8ms/step | 15.1 samples/s +Step 6500/44000 | Loss: 3.2710 | LR: 4.79e-04 | 1328.2ms/step | 15.1 samples/s +Validation at step 6500: Loss = 3.2472, Perplexity = 25.72 +Step 6510/44000 | Loss: 3.2412 | LR: 4.78e-04 | 3551.8ms/step | 5.6 samples/s +Step 6520/44000 | Loss: 3.2473 | LR: 4.78e-04 | 1324.3ms/step | 15.1 samples/s +Step 6530/44000 | Loss: 3.2687 | LR: 4.78e-04 | 1326.4ms/step | 15.1 samples/s +Step 6540/44000 | Loss: 3.2829 | LR: 4.78e-04 | 1325.3ms/step | 15.1 samples/s +Step 6550/44000 | Loss: 3.2539 | LR: 4.78e-04 | 1329.0ms/step | 15.0 samples/s +Step 6560/44000 | Loss: 3.2474 | LR: 4.78e-04 | 1329.2ms/step | 15.0 samples/s +Step 6570/44000 | Loss: 3.2343 | LR: 4.78e-04 | 1324.5ms/step | 15.1 samples/s +Step 6580/44000 | Loss: 3.2479 | LR: 4.78e-04 | 1327.5ms/step | 15.1 samples/s +Step 6590/44000 | Loss: 3.2617 | LR: 4.78e-04 | 1324.3ms/step | 15.1 samples/s +Step 6600/44000 | Loss: 3.2277 | LR: 4.78e-04 | 1326.6ms/step | 15.1 samples/s +Step 6610/44000 | Loss: 3.2557 | LR: 4.78e-04 | 1329.4ms/step | 15.0 samples/s +Step 6620/44000 | Loss: 3.2451 | LR: 4.78e-04 | 1326.0ms/step | 15.1 samples/s +Step 6630/44000 | Loss: 3.2526 | LR: 4.78e-04 | 1326.0ms/step | 15.1 samples/s +Step 6640/44000 | Loss: 3.2778 | LR: 4.77e-04 | 1327.9ms/step | 15.1 samples/s +Step 6650/44000 | Loss: 3.2871 | LR: 4.77e-04 | 1326.6ms/step | 15.1 samples/s +Step 6660/44000 | Loss: 3.2377 | LR: 4.77e-04 | 1326.8ms/step | 15.1 samples/s +Step 6670/44000 | Loss: 3.2642 | LR: 4.77e-04 | 1328.1ms/step | 15.1 samples/s +Step 6680/44000 | Loss: 3.2478 | LR: 4.77e-04 | 1324.1ms/step | 15.1 samples/s +Step 6690/44000 | Loss: 3.2568 | LR: 4.77e-04 | 1323.0ms/step | 15.1 samples/s +Step 6700/44000 | Loss: 3.2282 | LR: 4.77e-04 | 1326.6ms/step | 15.1 samples/s +Step 6710/44000 | Loss: 3.2522 | LR: 4.77e-04 | 1325.5ms/step | 15.1 samples/s +Step 6720/44000 | Loss: 3.2495 | LR: 4.77e-04 | 1325.5ms/step | 15.1 samples/s +Step 6730/44000 | Loss: 3.2535 | LR: 4.77e-04 | 1321.2ms/step | 15.1 samples/s +Step 6740/44000 | Loss: 3.2754 | LR: 4.77e-04 | 1326.8ms/step | 15.1 samples/s +Step 6750/44000 | Loss: 3.2455 | LR: 4.77e-04 | 1327.1ms/step | 15.1 samples/s +Validation at step 6750: Loss = 3.2353, Perplexity = 25.41 +Step 6760/44000 | Loss: 3.2286 | LR: 4.77e-04 | 3558.9ms/step | 5.6 samples/s +Step 6770/44000 | Loss: 3.2832 | LR: 4.76e-04 | 1322.3ms/step | 15.1 samples/s +Step 6780/44000 | Loss: 3.2570 | LR: 4.76e-04 | 1327.1ms/step | 15.1 samples/s +Step 6790/44000 | Loss: 3.2814 | LR: 4.76e-04 | 1327.0ms/step | 15.1 samples/s +Step 6800/44000 | Loss: 3.2380 | LR: 4.76e-04 | 1327.1ms/step | 15.1 samples/s +Step 6810/44000 | Loss: 3.2528 | LR: 4.76e-04 | 1327.1ms/step | 15.1 samples/s +Step 6820/44000 | Loss: 3.2598 | LR: 4.76e-04 | 1327.0ms/step | 15.1 samples/s +Step 6830/44000 | Loss: 3.2875 | LR: 4.76e-04 | 1324.7ms/step | 15.1 samples/s +Step 6840/44000 | Loss: 3.2469 | LR: 4.76e-04 | 1324.0ms/step | 15.1 samples/s +Step 6850/44000 | Loss: 3.2409 | LR: 4.76e-04 | 1324.0ms/step | 15.1 samples/s +Step 6860/44000 | Loss: 3.2663 | LR: 4.76e-04 | 1327.6ms/step | 15.1 samples/s +Step 6870/44000 | Loss: 3.2812 | LR: 4.76e-04 | 1328.2ms/step | 15.1 samples/s +Step 6880/44000 | Loss: 3.2224 | LR: 4.76e-04 | 1325.4ms/step | 15.1 samples/s +Step 6890/44000 | Loss: 3.2285 | LR: 4.76e-04 | 1327.8ms/step | 15.1 samples/s +Step 6900/44000 | Loss: 3.2294 | LR: 4.75e-04 | 1325.7ms/step | 15.1 samples/s +Step 6910/44000 | Loss: 3.2209 | LR: 4.75e-04 | 1325.3ms/step | 15.1 samples/s +Step 6920/44000 | Loss: 3.2347 | LR: 4.75e-04 | 1326.7ms/step | 15.1 samples/s +Step 6930/44000 | Loss: 3.2525 | LR: 4.75e-04 | 1325.9ms/step | 15.1 samples/s +Step 6940/44000 | Loss: 3.2341 | LR: 4.75e-04 | 1327.5ms/step | 15.1 samples/s +Step 6950/44000 | Loss: 3.2512 | LR: 4.75e-04 | 1324.2ms/step | 15.1 samples/s +Step 6960/44000 | Loss: 3.2069 | LR: 4.75e-04 | 1324.0ms/step | 15.1 samples/s +Step 6970/44000 | Loss: 3.2346 | LR: 4.75e-04 | 1328.2ms/step | 15.1 samples/s +Step 6980/44000 | Loss: 3.2597 | LR: 4.75e-04 | 1331.7ms/step | 15.0 samples/s +Step 6990/44000 | Loss: 3.2240 | LR: 4.75e-04 | 1328.3ms/step | 15.1 samples/s +Step 7000/44000 | Loss: 3.2026 | LR: 4.75e-04 | 1328.6ms/step | 15.1 samples/s +Validation at step 7000: Loss = 3.2249, Perplexity = 25.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-7000 +Step 7010/44000 | Loss: 3.2029 | LR: 4.75e-04 | 3867.9ms/step | 5.2 samples/s +Step 7020/44000 | Loss: 3.2514 | LR: 4.75e-04 | 1329.1ms/step | 15.0 samples/s +Step 7030/44000 | Loss: 3.2222 | LR: 4.74e-04 | 1331.4ms/step | 15.0 samples/s +Step 7040/44000 | Loss: 3.2293 | LR: 4.74e-04 | 1328.0ms/step | 15.1 samples/s +Step 7050/44000 | Loss: 3.2346 | LR: 4.74e-04 | 1329.4ms/step | 15.0 samples/s +Step 7060/44000 | Loss: 3.2300 | LR: 4.74e-04 | 1329.8ms/step | 15.0 samples/s +Step 7070/44000 | Loss: 3.2618 | LR: 4.74e-04 | 1324.1ms/step | 15.1 samples/s +Step 7080/44000 | Loss: 3.2042 | LR: 4.74e-04 | 1328.3ms/step | 15.1 samples/s +Step 7090/44000 | Loss: 3.2361 | LR: 4.74e-04 | 1328.1ms/step | 15.1 samples/s +Step 7100/44000 | Loss: 3.2478 | LR: 4.74e-04 | 1326.3ms/step | 15.1 samples/s +Step 7110/44000 | Loss: 3.2369 | LR: 4.74e-04 | 1325.7ms/step | 15.1 samples/s +Step 7120/44000 | Loss: 3.2646 | LR: 4.74e-04 | 1326.9ms/step | 15.1 samples/s +Step 7130/44000 | Loss: 3.2447 | LR: 4.74e-04 | 1328.2ms/step | 15.1 samples/s +Step 7140/44000 | Loss: 3.2461 | LR: 4.74e-04 | 1328.3ms/step | 15.1 samples/s +Step 7150/44000 | Loss: 3.2088 | LR: 4.73e-04 | 1330.2ms/step | 15.0 samples/s +Step 7160/44000 | Loss: 3.2450 | LR: 4.73e-04 | 1327.8ms/step | 15.1 samples/s +Step 7170/44000 | Loss: 3.2570 | LR: 4.73e-04 | 1327.2ms/step | 15.1 samples/s +Step 7180/44000 | Loss: 3.2626 | LR: 4.73e-04 | 1327.8ms/step | 15.1 samples/s +Step 7190/44000 | Loss: 3.2326 | LR: 4.73e-04 | 1329.3ms/step | 15.0 samples/s +Step 7200/44000 | Loss: 3.2115 | LR: 4.73e-04 | 1328.8ms/step | 15.1 samples/s +Step 7210/44000 | Loss: 3.2179 | LR: 4.73e-04 | 1328.6ms/step | 15.1 samples/s +Step 7220/44000 | Loss: 3.2283 | LR: 4.73e-04 | 1327.1ms/step | 15.1 samples/s +Step 7230/44000 | Loss: 3.2298 | LR: 4.73e-04 | 1327.5ms/step | 15.1 samples/s +Step 7240/44000 | Loss: 3.2479 | LR: 4.73e-04 | 1329.5ms/step | 15.0 samples/s +Step 7250/44000 | Loss: 3.2336 | LR: 4.73e-04 | 1326.9ms/step | 15.1 samples/s +Validation at step 7250: Loss = 3.2189, Perplexity = 25.00 +Step 7260/44000 | Loss: 3.2096 | LR: 4.73e-04 | 3562.2ms/step | 5.6 samples/s +Step 7270/44000 | Loss: 3.2384 | LR: 4.72e-04 | 1327.0ms/step | 15.1 samples/s +Step 7280/44000 | Loss: 3.2499 | LR: 4.72e-04 | 1328.1ms/step | 15.1 samples/s +Step 7290/44000 | Loss: 3.2084 | LR: 4.72e-04 | 1329.1ms/step | 15.0 samples/s +Step 7300/44000 | Loss: 3.2146 | LR: 4.72e-04 | 1329.8ms/step | 15.0 samples/s +Step 7310/44000 | Loss: 3.2281 | LR: 4.72e-04 | 1326.9ms/step | 15.1 samples/s +Step 7320/44000 | Loss: 3.2063 | LR: 4.72e-04 | 1326.8ms/step | 15.1 samples/s +Step 7330/44000 | Loss: 3.2039 | LR: 4.72e-04 | 1324.4ms/step | 15.1 samples/s +Step 7340/44000 | Loss: 3.2204 | LR: 4.72e-04 | 1324.1ms/step | 15.1 samples/s +Step 7350/44000 | Loss: 3.2217 | LR: 4.72e-04 | 1326.8ms/step | 15.1 samples/s +Step 7360/44000 | Loss: 3.2306 | LR: 4.72e-04 | 1326.4ms/step | 15.1 samples/s +Step 7370/44000 | Loss: 3.2123 | LR: 4.72e-04 | 1324.9ms/step | 15.1 samples/s +Step 7380/44000 | Loss: 3.2350 | LR: 4.72e-04 | 1324.4ms/step | 15.1 samples/s +Step 7390/44000 | Loss: 3.2083 | LR: 4.71e-04 | 1330.6ms/step | 15.0 samples/s +Step 7400/44000 | Loss: 3.2187 | LR: 4.71e-04 | 1328.9ms/step | 15.1 samples/s +Step 7410/44000 | Loss: 3.2110 | LR: 4.71e-04 | 1326.9ms/step | 15.1 samples/s +Step 7420/44000 | Loss: 3.2347 | LR: 4.71e-04 | 1325.8ms/step | 15.1 samples/s +Step 7430/44000 | Loss: 3.2408 | LR: 4.71e-04 | 1324.7ms/step | 15.1 samples/s +Step 7440/44000 | Loss: 3.2374 | LR: 4.71e-04 | 1327.5ms/step | 15.1 samples/s +Step 7450/44000 | Loss: 3.2124 | LR: 4.71e-04 | 1329.6ms/step | 15.0 samples/s +Step 7460/44000 | Loss: 3.2147 | LR: 4.71e-04 | 1327.2ms/step | 15.1 samples/s +Step 7470/44000 | Loss: 3.2214 | LR: 4.71e-04 | 1326.6ms/step | 15.1 samples/s +Step 7480/44000 | Loss: 3.2286 | LR: 4.71e-04 | 1326.5ms/step | 15.1 samples/s +Step 7490/44000 | Loss: 3.1985 | LR: 4.71e-04 | 1326.2ms/step | 15.1 samples/s +Step 7500/44000 | Loss: 3.1970 | LR: 4.71e-04 | 1328.2ms/step | 15.1 samples/s +Validation at step 7500: Loss = 3.2092, Perplexity = 24.76 +Step 7510/44000 | Loss: 3.2418 | LR: 4.70e-04 | 3552.3ms/step | 5.6 samples/s +Step 7520/44000 | Loss: 3.2246 | LR: 4.70e-04 | 1327.7ms/step | 15.1 samples/s +Step 7530/44000 | Loss: 3.2195 | LR: 4.70e-04 | 1325.9ms/step | 15.1 samples/s +Step 7540/44000 | Loss: 3.2184 | LR: 4.70e-04 | 1323.2ms/step | 15.1 samples/s +Step 7550/44000 | Loss: 3.2199 | LR: 4.70e-04 | 1325.5ms/step | 15.1 samples/s +Step 7560/44000 | Loss: 3.2212 | LR: 4.70e-04 | 1327.6ms/step | 15.1 samples/s +Step 7570/44000 | Loss: 3.2065 | LR: 4.70e-04 | 1327.7ms/step | 15.1 samples/s +Step 7580/44000 | Loss: 3.2350 | LR: 4.70e-04 | 1328.4ms/step | 15.1 samples/s +Step 7590/44000 | Loss: 3.2034 | LR: 4.70e-04 | 1324.7ms/step | 15.1 samples/s +Step 7600/44000 | Loss: 3.1954 | LR: 4.70e-04 | 1324.3ms/step | 15.1 samples/s +Step 7610/44000 | Loss: 3.2496 | LR: 4.70e-04 | 1322.7ms/step | 15.1 samples/s +Step 7620/44000 | Loss: 3.2234 | LR: 4.70e-04 | 1326.1ms/step | 15.1 samples/s +Step 7630/44000 | Loss: 3.2013 | LR: 4.69e-04 | 1325.7ms/step | 15.1 samples/s +Step 7640/44000 | Loss: 3.2132 | LR: 4.69e-04 | 1327.2ms/step | 15.1 samples/s +Step 7650/44000 | Loss: 3.2114 | LR: 4.69e-04 | 1326.1ms/step | 15.1 samples/s +Step 7660/44000 | Loss: 3.2036 | LR: 4.69e-04 | 1325.8ms/step | 15.1 samples/s +Step 7670/44000 | Loss: 3.1979 | LR: 4.69e-04 | 1324.2ms/step | 15.1 samples/s +Step 7680/44000 | Loss: 3.2148 | LR: 4.69e-04 | 1323.0ms/step | 15.1 samples/s +Step 7690/44000 | Loss: 3.2031 | LR: 4.69e-04 | 1325.2ms/step | 15.1 samples/s +Step 7700/44000 | Loss: 3.1923 | LR: 4.69e-04 | 1323.7ms/step | 15.1 samples/s +Step 7710/44000 | Loss: 3.2190 | LR: 4.69e-04 | 1326.5ms/step | 15.1 samples/s +Step 7720/44000 | Loss: 3.2221 | LR: 4.69e-04 | 1328.4ms/step | 15.1 samples/s +Step 7730/44000 | Loss: 3.2430 | LR: 4.69e-04 | 1328.3ms/step | 15.1 samples/s +Step 7740/44000 | Loss: 3.1882 | LR: 4.68e-04 | 1327.6ms/step | 15.1 samples/s +Step 7750/44000 | Loss: 3.2253 | LR: 4.68e-04 | 1326.6ms/step | 15.1 samples/s +Validation at step 7750: Loss = 3.2018, Perplexity = 24.58 +Step 7760/44000 | Loss: 3.1929 | LR: 4.68e-04 | 3560.6ms/step | 5.6 samples/s +Step 7770/44000 | Loss: 3.2474 | LR: 4.68e-04 | 1325.9ms/step | 15.1 samples/s +Step 7780/44000 | Loss: 3.2264 | LR: 4.68e-04 | 1324.7ms/step | 15.1 samples/s +Step 7790/44000 | Loss: 3.1757 | LR: 4.68e-04 | 1326.1ms/step | 15.1 samples/s +Step 7800/44000 | Loss: 3.2026 | LR: 4.68e-04 | 1327.8ms/step | 15.1 samples/s +Step 7810/44000 | Loss: 3.1866 | LR: 4.68e-04 | 1323.1ms/step | 15.1 samples/s +Step 7820/44000 | Loss: 3.2214 | LR: 4.68e-04 | 1321.5ms/step | 15.1 samples/s +Step 7830/44000 | Loss: 3.2400 | LR: 4.68e-04 | 1323.0ms/step | 15.1 samples/s +Step 7840/44000 | Loss: 3.1856 | LR: 4.68e-04 | 1326.8ms/step | 15.1 samples/s +Step 7850/44000 | Loss: 3.1934 | LR: 4.67e-04 | 1327.4ms/step | 15.1 samples/s +Step 7860/44000 | Loss: 3.2162 | LR: 4.67e-04 | 1327.1ms/step | 15.1 samples/s +Step 7870/44000 | Loss: 3.2049 | LR: 4.67e-04 | 1329.1ms/step | 15.0 samples/s +Step 7880/44000 | Loss: 3.2140 | LR: 4.67e-04 | 1327.3ms/step | 15.1 samples/s +Step 7890/44000 | Loss: 3.2241 | LR: 4.67e-04 | 1327.1ms/step | 15.1 samples/s +Step 7900/44000 | Loss: 3.2014 | LR: 4.67e-04 | 1329.1ms/step | 15.0 samples/s +Step 7910/44000 | Loss: 3.1843 | LR: 4.67e-04 | 1327.7ms/step | 15.1 samples/s +Step 7920/44000 | Loss: 3.2402 | LR: 4.67e-04 | 1327.4ms/step | 15.1 samples/s +Step 7930/44000 | Loss: 3.1755 | LR: 4.67e-04 | 1328.5ms/step | 15.1 samples/s +Step 7940/44000 | Loss: 3.2118 | LR: 4.67e-04 | 1327.0ms/step | 15.1 samples/s +Step 7950/44000 | Loss: 3.1931 | LR: 4.67e-04 | 1325.6ms/step | 15.1 samples/s +Step 7960/44000 | Loss: 3.2004 | LR: 4.66e-04 | 1328.9ms/step | 15.0 samples/s +Step 7970/44000 | Loss: 3.2600 | LR: 4.66e-04 | 1327.5ms/step | 15.1 samples/s +Step 7980/44000 | Loss: 3.2024 | LR: 4.66e-04 | 1327.9ms/step | 15.1 samples/s +Step 7990/44000 | Loss: 3.1822 | LR: 4.66e-04 | 1328.6ms/step | 15.1 samples/s +Step 8000/44000 | Loss: 3.2030 | LR: 4.66e-04 | 1329.5ms/step | 15.0 samples/s +Validation at step 8000: Loss = 3.1933, Perplexity = 24.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-8000 +Step 8010/44000 | Loss: 3.1839 | LR: 4.66e-04 | 3865.5ms/step | 5.2 samples/s +Step 8020/44000 | Loss: 3.2041 | LR: 4.66e-04 | 1326.9ms/step | 15.1 samples/s +Step 8030/44000 | Loss: 3.2345 | LR: 4.66e-04 | 1325.3ms/step | 15.1 samples/s +Step 8040/44000 | Loss: 3.2045 | LR: 4.66e-04 | 1325.6ms/step | 15.1 samples/s +Step 8050/44000 | Loss: 3.2045 | LR: 4.66e-04 | 1326.4ms/step | 15.1 samples/s +Step 8060/44000 | Loss: 3.1621 | LR: 4.66e-04 | 1328.1ms/step | 15.1 samples/s +Step 8070/44000 | Loss: 3.1961 | LR: 4.65e-04 | 1325.3ms/step | 15.1 samples/s +Step 8080/44000 | Loss: 3.2229 | LR: 4.65e-04 | 1326.4ms/step | 15.1 samples/s +Step 8090/44000 | Loss: 3.2063 | LR: 4.65e-04 | 1326.9ms/step | 15.1 samples/s +Step 8100/44000 | Loss: 3.1960 | LR: 4.65e-04 | 1325.3ms/step | 15.1 samples/s +Step 8110/44000 | Loss: 3.2053 | LR: 4.65e-04 | 1332.9ms/step | 15.0 samples/s +Step 8120/44000 | Loss: 3.2086 | LR: 4.65e-04 | 1326.9ms/step | 15.1 samples/s +Step 8130/44000 | Loss: 3.2136 | LR: 4.65e-04 | 1328.3ms/step | 15.1 samples/s +Step 8140/44000 | Loss: 3.1897 | LR: 4.65e-04 | 1329.2ms/step | 15.0 samples/s +Step 8150/44000 | Loss: 3.2000 | LR: 4.65e-04 | 1331.8ms/step | 15.0 samples/s +Step 8160/44000 | Loss: 3.2239 | LR: 4.65e-04 | 1327.0ms/step | 15.1 samples/s +Step 8170/44000 | Loss: 3.2158 | LR: 4.65e-04 | 1328.0ms/step | 15.1 samples/s +Step 8180/44000 | Loss: 3.2091 | LR: 4.64e-04 | 1328.3ms/step | 15.1 samples/s +Step 8190/44000 | Loss: 3.1949 | LR: 4.64e-04 | 1324.4ms/step | 15.1 samples/s +Step 8200/44000 | Loss: 3.2366 | LR: 4.64e-04 | 1323.0ms/step | 15.1 samples/s +Step 8210/44000 | Loss: 3.2025 | LR: 4.64e-04 | 1325.3ms/step | 15.1 samples/s +Step 8220/44000 | Loss: 3.2073 | LR: 4.64e-04 | 1329.2ms/step | 15.0 samples/s +Step 8230/44000 | Loss: 3.2042 | LR: 4.64e-04 | 1330.0ms/step | 15.0 samples/s +Step 8240/44000 | Loss: 3.1960 | LR: 4.64e-04 | 1327.6ms/step | 15.1 samples/s +Step 8250/44000 | Loss: 3.1939 | LR: 4.64e-04 | 1328.6ms/step | 15.1 samples/s +Validation at step 8250: Loss = 3.1842, Perplexity = 24.15 +Step 8260/44000 | Loss: 3.1990 | LR: 4.64e-04 | 3562.2ms/step | 5.6 samples/s +Step 8270/44000 | Loss: 3.1987 | LR: 4.64e-04 | 1330.0ms/step | 15.0 samples/s +Step 8280/44000 | Loss: 3.1861 | LR: 4.64e-04 | 1326.9ms/step | 15.1 samples/s +Step 8290/44000 | Loss: 3.1951 | LR: 4.63e-04 | 1325.4ms/step | 15.1 samples/s +Step 8300/44000 | Loss: 3.2083 | LR: 4.63e-04 | 1327.2ms/step | 15.1 samples/s +Step 8310/44000 | Loss: 3.1913 | LR: 4.63e-04 | 1327.2ms/step | 15.1 samples/s +Step 8320/44000 | Loss: 3.1769 | LR: 4.63e-04 | 1330.0ms/step | 15.0 samples/s +Step 8330/44000 | Loss: 3.2014 | LR: 4.63e-04 | 1329.4ms/step | 15.0 samples/s +Step 8340/44000 | Loss: 3.2174 | LR: 4.63e-04 | 1324.7ms/step | 15.1 samples/s +Step 8350/44000 | Loss: 3.2155 | LR: 4.63e-04 | 1326.6ms/step | 15.1 samples/s +Step 8360/44000 | Loss: 3.1784 | LR: 4.63e-04 | 1326.9ms/step | 15.1 samples/s +Step 8370/44000 | Loss: 3.2011 | LR: 4.63e-04 | 1327.5ms/step | 15.1 samples/s +Step 8380/44000 | Loss: 3.2011 | LR: 4.63e-04 | 1331.0ms/step | 15.0 samples/s +Step 8390/44000 | Loss: 3.2062 | LR: 4.62e-04 | 1325.1ms/step | 15.1 samples/s +Step 8400/44000 | Loss: 3.1818 | LR: 4.62e-04 | 1328.1ms/step | 15.1 samples/s +Step 8410/44000 | Loss: 3.2138 | LR: 4.62e-04 | 1323.8ms/step | 15.1 samples/s +Step 8420/44000 | Loss: 3.1780 | LR: 4.62e-04 | 1325.3ms/step | 15.1 samples/s +Step 8430/44000 | Loss: 3.1843 | LR: 4.62e-04 | 1324.5ms/step | 15.1 samples/s +Step 8440/44000 | Loss: 3.1791 | LR: 4.62e-04 | 1326.0ms/step | 15.1 samples/s +Step 8450/44000 | Loss: 3.1891 | LR: 4.62e-04 | 1322.5ms/step | 15.1 samples/s +Step 8460/44000 | Loss: 3.1953 | LR: 4.62e-04 | 1325.6ms/step | 15.1 samples/s +Step 8470/44000 | Loss: 3.1614 | LR: 4.62e-04 | 1326.3ms/step | 15.1 samples/s +Step 8480/44000 | Loss: 3.1769 | LR: 4.62e-04 | 1325.3ms/step | 15.1 samples/s +Step 8490/44000 | Loss: 3.2170 | LR: 4.62e-04 | 1325.7ms/step | 15.1 samples/s +Step 8500/44000 | Loss: 3.2135 | LR: 4.61e-04 | 1322.2ms/step | 15.1 samples/s +Validation at step 8500: Loss = 3.1773, Perplexity = 23.98 +Step 8510/44000 | Loss: 3.1856 | LR: 4.61e-04 | 3549.5ms/step | 5.6 samples/s +Step 8520/44000 | Loss: 3.2023 | LR: 4.61e-04 | 1329.5ms/step | 15.0 samples/s +Step 8530/44000 | Loss: 3.1940 | LR: 4.61e-04 | 1325.5ms/step | 15.1 samples/s +Step 8540/44000 | Loss: 3.1647 | LR: 4.61e-04 | 1324.7ms/step | 15.1 samples/s +Step 8550/44000 | Loss: 3.1681 | LR: 4.61e-04 | 1328.1ms/step | 15.1 samples/s +Step 8560/44000 | Loss: 3.2087 | LR: 4.61e-04 | 1328.2ms/step | 15.1 samples/s +Step 8570/44000 | Loss: 3.2107 | LR: 4.61e-04 | 1326.1ms/step | 15.1 samples/s +Step 8580/44000 | Loss: 3.2011 | LR: 4.61e-04 | 1324.8ms/step | 15.1 samples/s +Step 8590/44000 | Loss: 3.2090 | LR: 4.61e-04 | 1325.1ms/step | 15.1 samples/s +Step 8600/44000 | Loss: 3.2020 | LR: 4.60e-04 | 1323.8ms/step | 15.1 samples/s +Step 8610/44000 | Loss: 3.1710 | LR: 4.60e-04 | 1323.9ms/step | 15.1 samples/s +Step 8620/44000 | Loss: 3.2075 | LR: 4.60e-04 | 1323.4ms/step | 15.1 samples/s +Step 8630/44000 | Loss: 3.1615 | LR: 4.60e-04 | 1322.5ms/step | 15.1 samples/s +Step 8640/44000 | Loss: 3.1725 | LR: 4.60e-04 | 1325.7ms/step | 15.1 samples/s +Step 8650/44000 | Loss: 3.1922 | LR: 4.60e-04 | 1324.8ms/step | 15.1 samples/s +Step 8660/44000 | Loss: 3.1654 | LR: 4.60e-04 | 1327.4ms/step | 15.1 samples/s +Step 8670/44000 | Loss: 3.1686 | LR: 4.60e-04 | 1326.7ms/step | 15.1 samples/s +Step 8680/44000 | Loss: 3.1338 | LR: 4.60e-04 | 1326.4ms/step | 15.1 samples/s +Step 8690/44000 | Loss: 3.1819 | LR: 4.60e-04 | 1331.6ms/step | 15.0 samples/s +Step 8700/44000 | Loss: 3.1714 | LR: 4.59e-04 | 1330.0ms/step | 15.0 samples/s +Step 8710/44000 | Loss: 3.1808 | LR: 4.59e-04 | 1325.2ms/step | 15.1 samples/s +Step 8720/44000 | Loss: 3.1991 | LR: 4.59e-04 | 1326.2ms/step | 15.1 samples/s +Step 8730/44000 | Loss: 3.2057 | LR: 4.59e-04 | 1324.8ms/step | 15.1 samples/s +Step 8740/44000 | Loss: 3.2009 | LR: 4.59e-04 | 1327.1ms/step | 15.1 samples/s +Step 8750/44000 | Loss: 3.1797 | LR: 4.59e-04 | 1327.0ms/step | 15.1 samples/s +Validation at step 8750: Loss = 3.1709, Perplexity = 23.83 +Step 8760/44000 | Loss: 3.1885 | LR: 4.59e-04 | 3557.0ms/step | 5.6 samples/s +Step 8770/44000 | Loss: 3.1686 | LR: 4.59e-04 | 1325.8ms/step | 15.1 samples/s +Step 8780/44000 | Loss: 3.1840 | LR: 4.59e-04 | 1326.0ms/step | 15.1 samples/s +Step 8790/44000 | Loss: 3.1716 | LR: 4.59e-04 | 1327.7ms/step | 15.1 samples/s +Step 8800/44000 | Loss: 3.1704 | LR: 4.58e-04 | 1328.9ms/step | 15.1 samples/s +Step 8810/44000 | Loss: 3.1973 | LR: 4.58e-04 | 1326.3ms/step | 15.1 samples/s +Step 8820/44000 | Loss: 3.1667 | LR: 4.58e-04 | 1328.6ms/step | 15.1 samples/s +Step 8830/44000 | Loss: 3.1758 | LR: 4.58e-04 | 1328.4ms/step | 15.1 samples/s +Step 8840/44000 | Loss: 3.1713 | LR: 4.58e-04 | 1326.4ms/step | 15.1 samples/s +Step 8850/44000 | Loss: 3.1698 | LR: 4.58e-04 | 1324.3ms/step | 15.1 samples/s +Step 8860/44000 | Loss: 3.1646 | LR: 4.58e-04 | 1327.5ms/step | 15.1 samples/s +Step 8870/44000 | Loss: 3.1805 | LR: 4.58e-04 | 1324.6ms/step | 15.1 samples/s +Step 8880/44000 | Loss: 3.1727 | LR: 4.58e-04 | 1327.9ms/step | 15.1 samples/s +Step 8890/44000 | Loss: 3.1820 | LR: 4.58e-04 | 1327.7ms/step | 15.1 samples/s +Step 8900/44000 | Loss: 3.1584 | LR: 4.57e-04 | 1327.6ms/step | 15.1 samples/s +Step 8910/44000 | Loss: 3.1949 | LR: 4.57e-04 | 1328.1ms/step | 15.1 samples/s +Step 8920/44000 | Loss: 3.1603 | LR: 4.57e-04 | 1328.4ms/step | 15.1 samples/s +Step 8930/44000 | Loss: 3.2015 | LR: 4.57e-04 | 1325.4ms/step | 15.1 samples/s +Step 8940/44000 | Loss: 3.1628 | LR: 4.57e-04 | 1324.7ms/step | 15.1 samples/s +Step 8950/44000 | Loss: 3.1658 | LR: 4.57e-04 | 1330.9ms/step | 15.0 samples/s +Step 8960/44000 | Loss: 3.1797 | LR: 4.57e-04 | 1327.1ms/step | 15.1 samples/s +Step 8970/44000 | Loss: 3.1877 | LR: 4.57e-04 | 1325.7ms/step | 15.1 samples/s +Step 8980/44000 | Loss: 3.1837 | LR: 4.57e-04 | 1325.9ms/step | 15.1 samples/s +Step 8990/44000 | Loss: 3.1603 | LR: 4.57e-04 | 1327.4ms/step | 15.1 samples/s +Step 9000/44000 | Loss: 3.1672 | LR: 4.56e-04 | 1328.8ms/step | 15.1 samples/s +Validation at step 9000: Loss = 3.1644, Perplexity = 23.67 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-9000 +Step 9010/44000 | Loss: 3.1468 | LR: 4.56e-04 | 3874.6ms/step | 5.2 samples/s +Step 9020/44000 | Loss: 3.1495 | LR: 4.56e-04 | 1329.1ms/step | 15.0 samples/s +Step 9030/44000 | Loss: 3.2222 | LR: 4.56e-04 | 1324.4ms/step | 15.1 samples/s +Step 9040/44000 | Loss: 3.1859 | LR: 4.56e-04 | 1333.4ms/step | 15.0 samples/s +Step 9050/44000 | Loss: 3.1602 | LR: 4.56e-04 | 1330.8ms/step | 15.0 samples/s +Step 9060/44000 | Loss: 3.1859 | LR: 4.56e-04 | 1328.5ms/step | 15.1 samples/s +Step 9070/44000 | Loss: 3.1660 | LR: 4.56e-04 | 1325.5ms/step | 15.1 samples/s +Step 9080/44000 | Loss: 3.1464 | LR: 4.56e-04 | 1326.2ms/step | 15.1 samples/s +Step 9090/44000 | Loss: 3.1379 | LR: 4.56e-04 | 1326.2ms/step | 15.1 samples/s +Step 9100/44000 | Loss: 3.1916 | LR: 4.55e-04 | 1327.0ms/step | 15.1 samples/s +Step 9110/44000 | Loss: 3.1808 | LR: 4.55e-04 | 1327.7ms/step | 15.1 samples/s +Step 9120/44000 | Loss: 3.2165 | LR: 4.55e-04 | 1328.3ms/step | 15.1 samples/s +Step 9130/44000 | Loss: 3.1536 | LR: 4.55e-04 | 1327.6ms/step | 15.1 samples/s +Step 9140/44000 | Loss: 3.1793 | LR: 4.55e-04 | 1325.9ms/step | 15.1 samples/s +Step 9150/44000 | Loss: 3.1781 | LR: 4.55e-04 | 1326.3ms/step | 15.1 samples/s +Step 9160/44000 | Loss: 3.1918 | LR: 4.55e-04 | 1327.8ms/step | 15.1 samples/s +Step 9170/44000 | Loss: 3.1893 | LR: 4.55e-04 | 1325.2ms/step | 15.1 samples/s +Step 9180/44000 | Loss: 3.1644 | LR: 4.55e-04 | 1330.0ms/step | 15.0 samples/s +Step 9190/44000 | Loss: 3.1606 | LR: 4.54e-04 | 1327.7ms/step | 15.1 samples/s +Step 9200/44000 | Loss: 3.1825 | LR: 4.54e-04 | 1325.8ms/step | 15.1 samples/s +Step 9210/44000 | Loss: 3.1620 | LR: 4.54e-04 | 1329.8ms/step | 15.0 samples/s +Step 9220/44000 | Loss: 3.1834 | LR: 4.54e-04 | 1329.1ms/step | 15.0 samples/s +Step 9230/44000 | Loss: 3.1738 | LR: 4.54e-04 | 1327.4ms/step | 15.1 samples/s +Step 9240/44000 | Loss: 3.1869 | LR: 4.54e-04 | 1328.1ms/step | 15.1 samples/s +Step 9250/44000 | Loss: 3.1602 | LR: 4.54e-04 | 1326.8ms/step | 15.1 samples/s +Validation at step 9250: Loss = 3.1566, Perplexity = 23.49 +Step 9260/44000 | Loss: 3.1801 | LR: 4.54e-04 | 3568.4ms/step | 5.6 samples/s +Step 9270/44000 | Loss: 3.1460 | LR: 4.54e-04 | 1331.5ms/step | 15.0 samples/s +Step 9280/44000 | Loss: 3.1484 | LR: 4.54e-04 | 1330.0ms/step | 15.0 samples/s +Step 9290/44000 | Loss: 3.1742 | LR: 4.53e-04 | 1330.6ms/step | 15.0 samples/s +Step 9300/44000 | Loss: 3.1625 | LR: 4.53e-04 | 1326.4ms/step | 15.1 samples/s +Step 9310/44000 | Loss: 3.1951 | LR: 4.53e-04 | 1324.1ms/step | 15.1 samples/s +Step 9320/44000 | Loss: 3.1915 | LR: 4.53e-04 | 1324.5ms/step | 15.1 samples/s +Step 9330/44000 | Loss: 3.1445 | LR: 4.53e-04 | 1326.3ms/step | 15.1 samples/s +Step 9340/44000 | Loss: 3.1795 | LR: 4.53e-04 | 1325.8ms/step | 15.1 samples/s +Step 9350/44000 | Loss: 3.1818 | LR: 4.53e-04 | 1326.9ms/step | 15.1 samples/s +Step 9360/44000 | Loss: 3.1763 | LR: 4.53e-04 | 1323.3ms/step | 15.1 samples/s +Step 9370/44000 | Loss: 3.1420 | LR: 4.53e-04 | 1331.9ms/step | 15.0 samples/s +Step 9380/44000 | Loss: 3.1861 | LR: 4.52e-04 | 1323.8ms/step | 15.1 samples/s +Step 9390/44000 | Loss: 3.1542 | LR: 4.52e-04 | 1327.4ms/step | 15.1 samples/s +Step 9400/44000 | Loss: 3.1995 | LR: 4.52e-04 | 1325.8ms/step | 15.1 samples/s +Step 9410/44000 | Loss: 3.1900 | LR: 4.52e-04 | 1326.7ms/step | 15.1 samples/s +Step 9420/44000 | Loss: 3.1440 | LR: 4.52e-04 | 1321.7ms/step | 15.1 samples/s +Step 9430/44000 | Loss: 3.1518 | LR: 4.52e-04 | 1326.8ms/step | 15.1 samples/s +Step 9440/44000 | Loss: 3.1543 | LR: 4.52e-04 | 1329.3ms/step | 15.0 samples/s +Step 9450/44000 | Loss: 3.1472 | LR: 4.52e-04 | 1324.5ms/step | 15.1 samples/s +Step 9460/44000 | Loss: 3.1833 | LR: 4.52e-04 | 1326.5ms/step | 15.1 samples/s +Step 9470/44000 | Loss: 3.1777 | LR: 4.52e-04 | 1325.5ms/step | 15.1 samples/s +Step 9480/44000 | Loss: 3.1590 | LR: 4.51e-04 | 1324.5ms/step | 15.1 samples/s +Step 9490/44000 | Loss: 3.1677 | LR: 4.51e-04 | 1327.8ms/step | 15.1 samples/s +Step 9500/44000 | Loss: 3.1632 | LR: 4.51e-04 | 1326.3ms/step | 15.1 samples/s +Validation at step 9500: Loss = 3.1499, Perplexity = 23.33 +Step 9510/44000 | Loss: 3.1655 | LR: 4.51e-04 | 3546.4ms/step | 5.6 samples/s +Step 9520/44000 | Loss: 3.1687 | LR: 4.51e-04 | 1328.1ms/step | 15.1 samples/s +Step 9530/44000 | Loss: 3.1924 | LR: 4.51e-04 | 1325.5ms/step | 15.1 samples/s +Step 9540/44000 | Loss: 3.1828 | LR: 4.51e-04 | 1325.7ms/step | 15.1 samples/s +Step 9550/44000 | Loss: 3.1573 | LR: 4.51e-04 | 1323.9ms/step | 15.1 samples/s +Step 9560/44000 | Loss: 3.1693 | LR: 4.51e-04 | 1327.7ms/step | 15.1 samples/s +Step 9570/44000 | Loss: 3.1682 | LR: 4.50e-04 | 1327.8ms/step | 15.1 samples/s +Step 9580/44000 | Loss: 3.1966 | LR: 4.50e-04 | 1323.8ms/step | 15.1 samples/s +Step 9590/44000 | Loss: 3.1778 | LR: 4.50e-04 | 1324.7ms/step | 15.1 samples/s +Step 9600/44000 | Loss: 3.1447 | LR: 4.50e-04 | 1327.8ms/step | 15.1 samples/s +Step 9610/44000 | Loss: 3.1762 | LR: 4.50e-04 | 1328.7ms/step | 15.1 samples/s +Step 9620/44000 | Loss: 3.1494 | LR: 4.50e-04 | 1327.0ms/step | 15.1 samples/s +Step 9630/44000 | Loss: 3.1642 | LR: 4.50e-04 | 1329.6ms/step | 15.0 samples/s +Step 9640/44000 | Loss: 3.1456 | LR: 4.50e-04 | 1325.6ms/step | 15.1 samples/s +Step 9650/44000 | Loss: 3.1919 | LR: 4.50e-04 | 1326.7ms/step | 15.1 samples/s +Step 9660/44000 | Loss: 3.1497 | LR: 4.49e-04 | 1326.5ms/step | 15.1 samples/s +Step 9670/44000 | Loss: 3.1781 | LR: 4.49e-04 | 1323.5ms/step | 15.1 samples/s +Step 9680/44000 | Loss: 3.1804 | LR: 4.49e-04 | 1326.0ms/step | 15.1 samples/s +Step 9690/44000 | Loss: 3.1599 | LR: 4.49e-04 | 1327.5ms/step | 15.1 samples/s +Step 9700/44000 | Loss: 3.1737 | LR: 4.49e-04 | 1326.3ms/step | 15.1 samples/s +Step 9710/44000 | Loss: 3.1616 | LR: 4.49e-04 | 1324.2ms/step | 15.1 samples/s +Step 9720/44000 | Loss: 3.1534 | LR: 4.49e-04 | 1328.1ms/step | 15.1 samples/s +Step 9730/44000 | Loss: 3.1564 | LR: 4.49e-04 | 1324.5ms/step | 15.1 samples/s +Step 9740/44000 | Loss: 3.1713 | LR: 4.49e-04 | 1325.0ms/step | 15.1 samples/s +Step 9750/44000 | Loss: 3.1741 | LR: 4.48e-04 | 1328.2ms/step | 15.1 samples/s +Validation at step 9750: Loss = 3.1449, Perplexity = 23.22 +Step 9760/44000 | Loss: 3.1790 | LR: 4.48e-04 | 3557.7ms/step | 5.6 samples/s +Step 9770/44000 | Loss: 3.1537 | LR: 4.48e-04 | 1326.0ms/step | 15.1 samples/s +Step 9780/44000 | Loss: 3.1509 | LR: 4.48e-04 | 1326.6ms/step | 15.1 samples/s +Step 9790/44000 | Loss: 3.1578 | LR: 4.48e-04 | 1330.6ms/step | 15.0 samples/s +Step 9800/44000 | Loss: 3.1733 | LR: 4.48e-04 | 1326.6ms/step | 15.1 samples/s +Step 9810/44000 | Loss: 3.1262 | LR: 4.48e-04 | 1331.6ms/step | 15.0 samples/s +Step 9820/44000 | Loss: 3.1653 | LR: 4.48e-04 | 1324.0ms/step | 15.1 samples/s +Step 9830/44000 | Loss: 3.1701 | LR: 4.48e-04 | 1327.6ms/step | 15.1 samples/s +Step 9840/44000 | Loss: 3.1550 | LR: 4.47e-04 | 1329.9ms/step | 15.0 samples/s +Step 9850/44000 | Loss: 3.1519 | LR: 4.47e-04 | 1326.2ms/step | 15.1 samples/s +Step 9860/44000 | Loss: 3.1504 | LR: 4.47e-04 | 1329.6ms/step | 15.0 samples/s +Step 9870/44000 | Loss: 3.1695 | LR: 4.47e-04 | 1327.7ms/step | 15.1 samples/s +Step 9880/44000 | Loss: 3.1384 | LR: 4.47e-04 | 1324.5ms/step | 15.1 samples/s +Step 9890/44000 | Loss: 3.1625 | LR: 4.47e-04 | 1324.9ms/step | 15.1 samples/s +Step 9900/44000 | Loss: 3.1543 | LR: 4.47e-04 | 1324.0ms/step | 15.1 samples/s +Step 9910/44000 | Loss: 3.1300 | LR: 4.47e-04 | 1324.4ms/step | 15.1 samples/s +Step 9920/44000 | Loss: 3.1670 | LR: 4.47e-04 | 1327.4ms/step | 15.1 samples/s +Step 9930/44000 | Loss: 3.1637 | LR: 4.46e-04 | 1327.9ms/step | 15.1 samples/s +Step 9940/44000 | Loss: 3.1385 | LR: 4.46e-04 | 1326.2ms/step | 15.1 samples/s +Step 9950/44000 | Loss: 3.1838 | LR: 4.46e-04 | 1322.9ms/step | 15.1 samples/s +Step 9960/44000 | Loss: 3.1360 | LR: 4.46e-04 | 1326.0ms/step | 15.1 samples/s +Step 9970/44000 | Loss: 3.1268 | LR: 4.46e-04 | 1328.1ms/step | 15.1 samples/s +Step 9980/44000 | Loss: 3.1632 | LR: 4.46e-04 | 1330.5ms/step | 15.0 samples/s +Step 9990/44000 | Loss: 3.1536 | LR: 4.46e-04 | 1330.6ms/step | 15.0 samples/s +Step 10000/44000 | Loss: 3.1636 | LR: 4.46e-04 | 1329.0ms/step | 15.0 samples/s +Validation at step 10000: Loss = 3.1390, Perplexity = 23.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-10000 +Step 10010/44000 | Loss: 3.1191 | LR: 4.46e-04 | 3884.8ms/step | 5.1 samples/s +Step 10020/44000 | Loss: 3.1653 | LR: 4.45e-04 | 1326.4ms/step | 15.1 samples/s +Step 10030/44000 | Loss: 3.1496 | LR: 4.45e-04 | 1329.4ms/step | 15.0 samples/s +Step 10040/44000 | Loss: 3.1829 | LR: 4.45e-04 | 1327.7ms/step | 15.1 samples/s +Step 10050/44000 | Loss: 3.1604 | LR: 4.45e-04 | 1328.8ms/step | 15.1 samples/s +Step 10060/44000 | Loss: 3.1559 | LR: 4.45e-04 | 1327.3ms/step | 15.1 samples/s +Step 10070/44000 | Loss: 3.1452 | LR: 4.45e-04 | 1320.8ms/step | 15.1 samples/s +Step 10080/44000 | Loss: 3.1470 | LR: 4.45e-04 | 1321.0ms/step | 15.1 samples/s +Step 10090/44000 | Loss: 3.1796 | LR: 4.45e-04 | 1323.5ms/step | 15.1 samples/s +Step 10100/44000 | Loss: 3.1879 | LR: 4.45e-04 | 1318.8ms/step | 15.2 samples/s +Step 10110/44000 | Loss: 3.1455 | LR: 4.44e-04 | 1318.9ms/step | 15.2 samples/s +Step 10120/44000 | Loss: 3.1773 | LR: 4.44e-04 | 1318.6ms/step | 15.2 samples/s +Step 10130/44000 | Loss: 3.1398 | LR: 4.44e-04 | 1322.2ms/step | 15.1 samples/s +Step 10140/44000 | Loss: 3.1653 | LR: 4.44e-04 | 1322.8ms/step | 15.1 samples/s +Step 10150/44000 | Loss: 3.1434 | LR: 4.44e-04 | 1324.6ms/step | 15.1 samples/s +Step 10160/44000 | Loss: 3.1174 | LR: 4.44e-04 | 1323.2ms/step | 15.1 samples/s +Step 10170/44000 | Loss: 3.1636 | LR: 4.44e-04 | 1322.7ms/step | 15.1 samples/s +Step 10180/44000 | Loss: 3.1521 | LR: 4.44e-04 | 1322.6ms/step | 15.1 samples/s +Step 10190/44000 | Loss: 3.1705 | LR: 4.43e-04 | 1324.5ms/step | 15.1 samples/s +Step 10200/44000 | Loss: 3.1599 | LR: 4.43e-04 | 1320.7ms/step | 15.1 samples/s +Step 10210/44000 | Loss: 3.1571 | LR: 4.43e-04 | 1323.2ms/step | 15.1 samples/s +Step 10220/44000 | Loss: 3.1683 | LR: 4.43e-04 | 1323.4ms/step | 15.1 samples/s +Step 10230/44000 | Loss: 3.1592 | LR: 4.43e-04 | 1323.0ms/step | 15.1 samples/s +Step 10240/44000 | Loss: 3.1289 | LR: 4.43e-04 | 1325.3ms/step | 15.1 samples/s +Step 10250/44000 | Loss: 3.1239 | LR: 4.43e-04 | 1322.6ms/step | 15.1 samples/s +Validation at step 10250: Loss = 3.1349, Perplexity = 22.99 +Step 10260/44000 | Loss: 3.1330 | LR: 4.43e-04 | 3550.1ms/step | 5.6 samples/s +Step 10270/44000 | Loss: 3.1590 | LR: 4.43e-04 | 1319.3ms/step | 15.2 samples/s +Step 10280/44000 | Loss: 3.1617 | LR: 4.42e-04 | 1323.4ms/step | 15.1 samples/s +Step 10290/44000 | Loss: 3.1585 | LR: 4.42e-04 | 1324.9ms/step | 15.1 samples/s +Step 10300/44000 | Loss: 3.1710 | LR: 4.42e-04 | 1324.2ms/step | 15.1 samples/s +Step 10310/44000 | Loss: 3.1472 | LR: 4.42e-04 | 1323.8ms/step | 15.1 samples/s +Step 10320/44000 | Loss: 3.1739 | LR: 4.42e-04 | 1323.8ms/step | 15.1 samples/s +Step 10330/44000 | Loss: 3.1404 | LR: 4.42e-04 | 1322.9ms/step | 15.1 samples/s +Step 10340/44000 | Loss: 3.1853 | LR: 4.42e-04 | 1324.0ms/step | 15.1 samples/s +Step 10350/44000 | Loss: 3.1495 | LR: 4.42e-04 | 1318.9ms/step | 15.2 samples/s +Step 10360/44000 | Loss: 3.1779 | LR: 4.42e-04 | 1324.4ms/step | 15.1 samples/s +Step 10370/44000 | Loss: 3.1561 | LR: 4.41e-04 | 1320.1ms/step | 15.1 samples/s +Step 10380/44000 | Loss: 3.1174 | LR: 4.41e-04 | 1319.2ms/step | 15.2 samples/s +Step 10390/44000 | Loss: 3.1593 | LR: 4.41e-04 | 1325.3ms/step | 15.1 samples/s +Step 10400/44000 | Loss: 3.1393 | LR: 4.41e-04 | 1328.6ms/step | 15.1 samples/s +Step 10410/44000 | Loss: 3.1524 | LR: 4.41e-04 | 1325.0ms/step | 15.1 samples/s +Step 10420/44000 | Loss: 3.1718 | LR: 4.41e-04 | 1323.2ms/step | 15.1 samples/s +Step 10430/44000 | Loss: 3.1422 | LR: 4.41e-04 | 1323.2ms/step | 15.1 samples/s +Step 10440/44000 | Loss: 3.1548 | LR: 4.41e-04 | 1321.3ms/step | 15.1 samples/s +Step 10450/44000 | Loss: 3.1148 | LR: 4.40e-04 | 1326.1ms/step | 15.1 samples/s +Step 10460/44000 | Loss: 3.1389 | LR: 4.40e-04 | 1322.9ms/step | 15.1 samples/s +Step 10470/44000 | Loss: 3.1575 | LR: 4.40e-04 | 1322.2ms/step | 15.1 samples/s +Step 10480/44000 | Loss: 3.1511 | LR: 4.40e-04 | 1324.7ms/step | 15.1 samples/s +Step 10490/44000 | Loss: 3.1292 | LR: 4.40e-04 | 1326.7ms/step | 15.1 samples/s +Step 10500/44000 | Loss: 3.1471 | LR: 4.40e-04 | 1330.0ms/step | 15.0 samples/s +Validation at step 10500: Loss = 3.1294, Perplexity = 22.86 +Step 10510/44000 | Loss: 3.1240 | LR: 4.40e-04 | 3555.8ms/step | 5.6 samples/s +Step 10520/44000 | Loss: 3.1131 | LR: 4.40e-04 | 1324.2ms/step | 15.1 samples/s +Step 10530/44000 | Loss: 3.1366 | LR: 4.40e-04 | 1324.4ms/step | 15.1 samples/s +Step 10540/44000 | Loss: 3.1526 | LR: 4.39e-04 | 1323.3ms/step | 15.1 samples/s +Step 10550/44000 | Loss: 3.1405 | LR: 4.39e-04 | 1325.7ms/step | 15.1 samples/s +Step 10560/44000 | Loss: 3.1319 | LR: 4.39e-04 | 1328.1ms/step | 15.1 samples/s +Step 10570/44000 | Loss: 3.1212 | LR: 4.39e-04 | 1328.3ms/step | 15.1 samples/s +Step 10580/44000 | Loss: 3.1472 | LR: 4.39e-04 | 1328.5ms/step | 15.1 samples/s +Step 10590/44000 | Loss: 3.0990 | LR: 4.39e-04 | 1326.7ms/step | 15.1 samples/s +Step 10600/44000 | Loss: 3.1278 | LR: 4.39e-04 | 1324.8ms/step | 15.1 samples/s +Step 10610/44000 | Loss: 3.1181 | LR: 4.39e-04 | 1324.1ms/step | 15.1 samples/s +Step 10620/44000 | Loss: 3.1462 | LR: 4.38e-04 | 1328.6ms/step | 15.1 samples/s +Step 10630/44000 | Loss: 3.1300 | LR: 4.38e-04 | 1322.9ms/step | 15.1 samples/s +Step 10640/44000 | Loss: 3.1281 | LR: 4.38e-04 | 1324.6ms/step | 15.1 samples/s +Step 10650/44000 | Loss: 3.1585 | LR: 4.38e-04 | 1325.2ms/step | 15.1 samples/s +Step 10660/44000 | Loss: 3.1431 | LR: 4.38e-04 | 1327.4ms/step | 15.1 samples/s +Step 10670/44000 | Loss: 3.1382 | LR: 4.38e-04 | 1328.0ms/step | 15.1 samples/s +Step 10680/44000 | Loss: 3.1477 | LR: 4.38e-04 | 1323.8ms/step | 15.1 samples/s +Step 10690/44000 | Loss: 3.1221 | LR: 4.38e-04 | 1327.6ms/step | 15.1 samples/s +Step 10700/44000 | Loss: 3.1446 | LR: 4.37e-04 | 1329.2ms/step | 15.0 samples/s +Step 10710/44000 | Loss: 3.1142 | LR: 4.37e-04 | 1326.7ms/step | 15.1 samples/s +Step 10720/44000 | Loss: 3.1376 | LR: 4.37e-04 | 1326.5ms/step | 15.1 samples/s +Step 10730/44000 | Loss: 3.1142 | LR: 4.37e-04 | 1328.2ms/step | 15.1 samples/s +Step 10740/44000 | Loss: 3.1483 | LR: 4.37e-04 | 1332.5ms/step | 15.0 samples/s +Step 10750/44000 | Loss: 3.1185 | LR: 4.37e-04 | 1326.8ms/step | 15.1 samples/s +Validation at step 10750: Loss = 3.1234, Perplexity = 22.72 +Step 10760/44000 | Loss: 3.1594 | LR: 4.37e-04 | 3557.5ms/step | 5.6 samples/s +Step 10770/44000 | Loss: 3.1399 | LR: 4.37e-04 | 1328.3ms/step | 15.1 samples/s +Step 10780/44000 | Loss: 3.1415 | LR: 4.37e-04 | 1328.0ms/step | 15.1 samples/s +Step 10790/44000 | Loss: 3.1266 | LR: 4.36e-04 | 1328.1ms/step | 15.1 samples/s +Step 10800/44000 | Loss: 3.1785 | LR: 4.36e-04 | 1328.5ms/step | 15.1 samples/s +Step 10810/44000 | Loss: 3.1443 | LR: 4.36e-04 | 1323.0ms/step | 15.1 samples/s +Step 10820/44000 | Loss: 3.1284 | LR: 4.36e-04 | 1321.9ms/step | 15.1 samples/s +Step 10830/44000 | Loss: 3.1373 | LR: 4.36e-04 | 1324.6ms/step | 15.1 samples/s +Step 10840/44000 | Loss: 3.1077 | LR: 4.36e-04 | 1324.7ms/step | 15.1 samples/s +Step 10850/44000 | Loss: 3.1361 | LR: 4.36e-04 | 1327.0ms/step | 15.1 samples/s +Step 10860/44000 | Loss: 3.1433 | LR: 4.36e-04 | 1325.5ms/step | 15.1 samples/s +Step 10870/44000 | Loss: 3.1290 | LR: 4.35e-04 | 1324.5ms/step | 15.1 samples/s +Step 10880/44000 | Loss: 3.1828 | LR: 4.35e-04 | 1323.1ms/step | 15.1 samples/s +Step 10890/44000 | Loss: 3.0748 | LR: 4.35e-04 | 1320.4ms/step | 15.1 samples/s +Step 10900/44000 | Loss: 3.1510 | LR: 4.35e-04 | 1326.2ms/step | 15.1 samples/s +Step 10910/44000 | Loss: 3.1257 | LR: 4.35e-04 | 1330.3ms/step | 15.0 samples/s +Step 10920/44000 | Loss: 3.1083 | LR: 4.35e-04 | 1326.0ms/step | 15.1 samples/s +Step 10930/44000 | Loss: 3.1367 | LR: 4.35e-04 | 1325.1ms/step | 15.1 samples/s +Step 10940/44000 | Loss: 3.1440 | LR: 4.35e-04 | 1327.5ms/step | 15.1 samples/s +Step 10950/44000 | Loss: 3.1455 | LR: 4.34e-04 | 1326.9ms/step | 15.1 samples/s +Step 10960/44000 | Loss: 3.1679 | LR: 4.34e-04 | 1327.0ms/step | 15.1 samples/s +Step 10970/44000 | Loss: 3.1629 | LR: 4.34e-04 | 1329.6ms/step | 15.0 samples/s +Step 10980/44000 | Loss: 3.1420 | LR: 4.34e-04 | 1327.8ms/step | 15.1 samples/s +Step 10990/44000 | Loss: 3.1494 | LR: 4.34e-04 | 1325.4ms/step | 15.1 samples/s +Step 11000/44000 | Loss: 3.1271 | LR: 4.34e-04 | 1328.2ms/step | 15.1 samples/s +Validation at step 11000: Loss = 3.1177, Perplexity = 22.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-11000 +Step 11010/44000 | Loss: 3.1314 | LR: 4.34e-04 | 3869.9ms/step | 5.2 samples/s +Step 11020/44000 | Loss: 3.1404 | LR: 4.34e-04 | 1326.9ms/step | 15.1 samples/s +Step 11030/44000 | Loss: 3.1113 | LR: 4.33e-04 | 1326.6ms/step | 15.1 samples/s +Step 11040/44000 | Loss: 3.1679 | LR: 4.33e-04 | 1329.5ms/step | 15.0 samples/s +Step 11050/44000 | Loss: 3.1332 | LR: 4.33e-04 | 1325.8ms/step | 15.1 samples/s +Step 11060/44000 | Loss: 3.1485 | LR: 4.33e-04 | 1328.1ms/step | 15.1 samples/s +Step 11070/44000 | Loss: 3.1309 | LR: 4.33e-04 | 1329.8ms/step | 15.0 samples/s +Step 11080/44000 | Loss: 3.1275 | LR: 4.33e-04 | 1329.1ms/step | 15.0 samples/s +Step 11090/44000 | Loss: 3.1141 | LR: 4.33e-04 | 1326.0ms/step | 15.1 samples/s +Step 11100/44000 | Loss: 3.1407 | LR: 4.33e-04 | 1325.1ms/step | 15.1 samples/s +Step 11110/44000 | Loss: 3.1176 | LR: 4.32e-04 | 1329.7ms/step | 15.0 samples/s +Step 11120/44000 | Loss: 3.1160 | LR: 4.32e-04 | 1327.4ms/step | 15.1 samples/s +Step 11130/44000 | Loss: 3.1296 | LR: 4.32e-04 | 1329.4ms/step | 15.0 samples/s +Step 11140/44000 | Loss: 3.1132 | LR: 4.32e-04 | 1326.1ms/step | 15.1 samples/s +Step 11150/44000 | Loss: 3.1327 | LR: 4.32e-04 | 1328.8ms/step | 15.1 samples/s +Step 11160/44000 | Loss: 3.1411 | LR: 4.32e-04 | 1327.6ms/step | 15.1 samples/s +Step 11170/44000 | Loss: 3.1247 | LR: 4.32e-04 | 1330.0ms/step | 15.0 samples/s +Step 11180/44000 | Loss: 3.1408 | LR: 4.32e-04 | 1324.1ms/step | 15.1 samples/s +Step 11190/44000 | Loss: 3.1244 | LR: 4.31e-04 | 1326.3ms/step | 15.1 samples/s +Step 11200/44000 | Loss: 3.1414 | LR: 4.31e-04 | 1327.9ms/step | 15.1 samples/s +Step 11210/44000 | Loss: 3.1461 | LR: 4.31e-04 | 1334.7ms/step | 15.0 samples/s +Step 11220/44000 | Loss: 3.1103 | LR: 4.31e-04 | 1331.5ms/step | 15.0 samples/s +Step 11230/44000 | Loss: 3.1175 | LR: 4.31e-04 | 1330.4ms/step | 15.0 samples/s +Step 11240/44000 | Loss: 3.1063 | LR: 4.31e-04 | 1327.0ms/step | 15.1 samples/s +Step 11250/44000 | Loss: 3.1024 | LR: 4.31e-04 | 1327.3ms/step | 15.1 samples/s +Validation at step 11250: Loss = 3.1136, Perplexity = 22.50 +Step 11260/44000 | Loss: 3.1198 | LR: 4.31e-04 | 3561.6ms/step | 5.6 samples/s +Step 11270/44000 | Loss: 3.1272 | LR: 4.30e-04 | 1327.3ms/step | 15.1 samples/s +Step 11280/44000 | Loss: 3.1228 | LR: 4.30e-04 | 1328.3ms/step | 15.1 samples/s +Step 11290/44000 | Loss: 3.1295 | LR: 4.30e-04 | 1326.5ms/step | 15.1 samples/s +Step 11300/44000 | Loss: 3.1018 | LR: 4.30e-04 | 1329.1ms/step | 15.0 samples/s +Step 11310/44000 | Loss: 3.1350 | LR: 4.30e-04 | 1325.3ms/step | 15.1 samples/s +Step 11320/44000 | Loss: 3.1041 | LR: 4.30e-04 | 1326.1ms/step | 15.1 samples/s +Step 11330/44000 | Loss: 3.1524 | LR: 4.30e-04 | 1329.1ms/step | 15.0 samples/s +Step 11340/44000 | Loss: 3.1288 | LR: 4.30e-04 | 1327.3ms/step | 15.1 samples/s +Step 11350/44000 | Loss: 3.1295 | LR: 4.29e-04 | 1326.3ms/step | 15.1 samples/s +Step 11360/44000 | Loss: 3.1465 | LR: 4.29e-04 | 1330.5ms/step | 15.0 samples/s +Step 11370/44000 | Loss: 3.0893 | LR: 4.29e-04 | 1329.2ms/step | 15.0 samples/s +Step 11380/44000 | Loss: 3.1067 | LR: 4.29e-04 | 1329.2ms/step | 15.0 samples/s +Step 11390/44000 | Loss: 3.1052 | LR: 4.29e-04 | 1325.5ms/step | 15.1 samples/s +Step 11400/44000 | Loss: 3.1307 | LR: 4.29e-04 | 1327.0ms/step | 15.1 samples/s +Step 11410/44000 | Loss: 3.1165 | LR: 4.29e-04 | 1326.7ms/step | 15.1 samples/s +Step 11420/44000 | Loss: 3.1164 | LR: 4.29e-04 | 1327.7ms/step | 15.1 samples/s +Step 11430/44000 | Loss: 3.1182 | LR: 4.28e-04 | 1327.0ms/step | 15.1 samples/s +Step 11440/44000 | Loss: 3.1438 | LR: 4.28e-04 | 1327.0ms/step | 15.1 samples/s +Step 11450/44000 | Loss: 3.1103 | LR: 4.28e-04 | 1327.5ms/step | 15.1 samples/s +Step 11460/44000 | Loss: 3.1325 | LR: 4.28e-04 | 1324.8ms/step | 15.1 samples/s +Step 11470/44000 | Loss: 3.1319 | LR: 4.28e-04 | 1326.9ms/step | 15.1 samples/s +Step 11480/44000 | Loss: 3.1549 | LR: 4.28e-04 | 1320.6ms/step | 15.1 samples/s +Step 11490/44000 | Loss: 3.1030 | LR: 4.28e-04 | 1325.7ms/step | 15.1 samples/s +Step 11500/44000 | Loss: 3.1468 | LR: 4.28e-04 | 1325.4ms/step | 15.1 samples/s +Validation at step 11500: Loss = 3.1087, Perplexity = 22.39 +Step 11510/44000 | Loss: 3.1344 | LR: 4.27e-04 | 3548.1ms/step | 5.6 samples/s +Step 11520/44000 | Loss: 3.1427 | LR: 4.27e-04 | 1325.2ms/step | 15.1 samples/s +Step 11530/44000 | Loss: 3.1280 | LR: 4.27e-04 | 1325.2ms/step | 15.1 samples/s +Step 11540/44000 | Loss: 3.1234 | LR: 4.27e-04 | 1328.7ms/step | 15.1 samples/s +Step 11550/44000 | Loss: 3.1092 | LR: 4.27e-04 | 1331.3ms/step | 15.0 samples/s +Step 11560/44000 | Loss: 3.1408 | LR: 4.27e-04 | 1327.5ms/step | 15.1 samples/s +Step 11570/44000 | Loss: 3.1172 | LR: 4.27e-04 | 1325.5ms/step | 15.1 samples/s +Step 11580/44000 | Loss: 3.1378 | LR: 4.27e-04 | 1326.2ms/step | 15.1 samples/s +Step 11590/44000 | Loss: 3.1281 | LR: 4.26e-04 | 1326.5ms/step | 15.1 samples/s +Step 11600/44000 | Loss: 3.1029 | LR: 4.26e-04 | 1323.6ms/step | 15.1 samples/s +Step 11610/44000 | Loss: 3.1436 | LR: 4.26e-04 | 1328.3ms/step | 15.1 samples/s +Step 11620/44000 | Loss: 3.0967 | LR: 4.26e-04 | 1327.5ms/step | 15.1 samples/s +Step 11630/44000 | Loss: 3.1185 | LR: 4.26e-04 | 1321.8ms/step | 15.1 samples/s +Step 11640/44000 | Loss: 3.1530 | LR: 4.26e-04 | 1326.3ms/step | 15.1 samples/s +Step 11650/44000 | Loss: 3.1141 | LR: 4.26e-04 | 1324.3ms/step | 15.1 samples/s +Step 11660/44000 | Loss: 3.1141 | LR: 4.26e-04 | 1328.9ms/step | 15.0 samples/s +Step 11670/44000 | Loss: 3.1167 | LR: 4.25e-04 | 1326.8ms/step | 15.1 samples/s +Step 11680/44000 | Loss: 3.1373 | LR: 4.25e-04 | 1325.9ms/step | 15.1 samples/s +Step 11690/44000 | Loss: 3.1075 | LR: 4.25e-04 | 1329.5ms/step | 15.0 samples/s +Step 11700/44000 | Loss: 3.0979 | LR: 4.25e-04 | 1327.8ms/step | 15.1 samples/s +Step 11710/44000 | Loss: 3.1028 | LR: 4.25e-04 | 1326.9ms/step | 15.1 samples/s +Step 11720/44000 | Loss: 3.1438 | LR: 4.25e-04 | 1326.6ms/step | 15.1 samples/s +Step 11730/44000 | Loss: 3.0856 | LR: 4.25e-04 | 1327.5ms/step | 15.1 samples/s +Step 11740/44000 | Loss: 3.1032 | LR: 4.24e-04 | 1327.4ms/step | 15.1 samples/s +Step 11750/44000 | Loss: 3.0992 | LR: 4.24e-04 | 1326.2ms/step | 15.1 samples/s +Validation at step 11750: Loss = 3.1042, Perplexity = 22.29 +Step 11760/44000 | Loss: 3.1217 | LR: 4.24e-04 | 3554.9ms/step | 5.6 samples/s +Step 11770/44000 | Loss: 3.1030 | LR: 4.24e-04 | 1324.0ms/step | 15.1 samples/s +Step 11780/44000 | Loss: 3.1133 | LR: 4.24e-04 | 1325.1ms/step | 15.1 samples/s +Step 11790/44000 | Loss: 3.1190 | LR: 4.24e-04 | 1323.5ms/step | 15.1 samples/s +Step 11800/44000 | Loss: 3.0880 | LR: 4.24e-04 | 1327.3ms/step | 15.1 samples/s +Step 11810/44000 | Loss: 3.0922 | LR: 4.24e-04 | 1327.2ms/step | 15.1 samples/s +Step 11820/44000 | Loss: 3.1191 | LR: 4.23e-04 | 1327.8ms/step | 15.1 samples/s +Step 11830/44000 | Loss: 3.1399 | LR: 4.23e-04 | 1328.2ms/step | 15.1 samples/s +Step 11840/44000 | Loss: 3.1255 | LR: 4.23e-04 | 1328.0ms/step | 15.1 samples/s +Step 11850/44000 | Loss: 3.1197 | LR: 4.23e-04 | 1327.2ms/step | 15.1 samples/s +Step 11860/44000 | Loss: 3.1093 | LR: 4.23e-04 | 1323.8ms/step | 15.1 samples/s +Step 11870/44000 | Loss: 3.0856 | LR: 4.23e-04 | 1327.8ms/step | 15.1 samples/s +Step 11880/44000 | Loss: 3.1153 | LR: 4.23e-04 | 1326.6ms/step | 15.1 samples/s +Step 11890/44000 | Loss: 3.1075 | LR: 4.23e-04 | 1325.9ms/step | 15.1 samples/s +Step 11900/44000 | Loss: 3.1183 | LR: 4.22e-04 | 1324.1ms/step | 15.1 samples/s +Step 11910/44000 | Loss: 3.0956 | LR: 4.22e-04 | 1326.4ms/step | 15.1 samples/s +Step 11920/44000 | Loss: 3.0738 | LR: 4.22e-04 | 1328.3ms/step | 15.1 samples/s +Step 11930/44000 | Loss: 3.1166 | LR: 4.22e-04 | 1327.2ms/step | 15.1 samples/s +Step 11940/44000 | Loss: 3.1205 | LR: 4.22e-04 | 1325.0ms/step | 15.1 samples/s +Step 11950/44000 | Loss: 3.0878 | LR: 4.22e-04 | 1324.5ms/step | 15.1 samples/s +Step 11960/44000 | Loss: 3.1014 | LR: 4.22e-04 | 1327.8ms/step | 15.1 samples/s +Step 11970/44000 | Loss: 3.1252 | LR: 4.21e-04 | 1328.4ms/step | 15.1 samples/s +Step 11980/44000 | Loss: 3.1294 | LR: 4.21e-04 | 1329.2ms/step | 15.0 samples/s +Step 11990/44000 | Loss: 3.1142 | LR: 4.21e-04 | 1323.4ms/step | 15.1 samples/s +Step 12000/44000 | Loss: 3.1117 | LR: 4.21e-04 | 1329.4ms/step | 15.0 samples/s +Validation at step 12000: Loss = 3.0991, Perplexity = 22.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-12000 +Step 12010/44000 | Loss: 3.1330 | LR: 4.21e-04 | 3864.3ms/step | 5.2 samples/s +Step 12020/44000 | Loss: 3.1323 | LR: 4.21e-04 | 1328.9ms/step | 15.0 samples/s +Step 12030/44000 | Loss: 3.1416 | LR: 4.21e-04 | 1329.9ms/step | 15.0 samples/s +Step 12040/44000 | Loss: 3.0892 | LR: 4.21e-04 | 1329.1ms/step | 15.0 samples/s +Step 12050/44000 | Loss: 3.1215 | LR: 4.20e-04 | 1328.2ms/step | 15.1 samples/s +Step 12060/44000 | Loss: 3.1044 | LR: 4.20e-04 | 1326.7ms/step | 15.1 samples/s +Step 12070/44000 | Loss: 3.1378 | LR: 4.20e-04 | 1332.0ms/step | 15.0 samples/s +Step 12080/44000 | Loss: 3.1247 | LR: 4.20e-04 | 1327.1ms/step | 15.1 samples/s +Step 12090/44000 | Loss: 3.0882 | LR: 4.20e-04 | 1327.2ms/step | 15.1 samples/s +Step 12100/44000 | Loss: 3.0992 | LR: 4.20e-04 | 1331.8ms/step | 15.0 samples/s +Step 12110/44000 | Loss: 3.1188 | LR: 4.20e-04 | 1322.2ms/step | 15.1 samples/s +Step 12120/44000 | Loss: 3.1075 | LR: 4.19e-04 | 1327.0ms/step | 15.1 samples/s +Step 12130/44000 | Loss: 3.0909 | LR: 4.19e-04 | 1328.6ms/step | 15.1 samples/s +Step 12140/44000 | Loss: 3.0839 | LR: 4.19e-04 | 1328.5ms/step | 15.1 samples/s +Step 12150/44000 | Loss: 3.1005 | LR: 4.19e-04 | 1328.0ms/step | 15.1 samples/s +Step 12160/44000 | Loss: 3.0907 | LR: 4.19e-04 | 1324.8ms/step | 15.1 samples/s +Step 12170/44000 | Loss: 3.1171 | LR: 4.19e-04 | 1324.7ms/step | 15.1 samples/s +Step 12180/44000 | Loss: 3.1259 | LR: 4.19e-04 | 1323.2ms/step | 15.1 samples/s +Step 12190/44000 | Loss: 3.1035 | LR: 4.19e-04 | 1332.0ms/step | 15.0 samples/s +Step 12200/44000 | Loss: 3.1284 | LR: 4.18e-04 | 1327.8ms/step | 15.1 samples/s +Step 12210/44000 | Loss: 3.1047 | LR: 4.18e-04 | 1326.8ms/step | 15.1 samples/s +Step 12220/44000 | Loss: 3.0749 | LR: 4.18e-04 | 1326.2ms/step | 15.1 samples/s +Step 12230/44000 | Loss: 3.1019 | LR: 4.18e-04 | 1329.4ms/step | 15.0 samples/s +Step 12240/44000 | Loss: 3.0952 | LR: 4.18e-04 | 1327.3ms/step | 15.1 samples/s +Step 12250/44000 | Loss: 3.1016 | LR: 4.18e-04 | 1331.6ms/step | 15.0 samples/s +Validation at step 12250: Loss = 3.0953, Perplexity = 22.09 +Step 12260/44000 | Loss: 3.0965 | LR: 4.18e-04 | 3562.0ms/step | 5.6 samples/s +Step 12270/44000 | Loss: 3.0800 | LR: 4.17e-04 | 1327.0ms/step | 15.1 samples/s +Step 12280/44000 | Loss: 3.0920 | LR: 4.17e-04 | 1325.6ms/step | 15.1 samples/s +Step 12290/44000 | Loss: 3.0852 | LR: 4.17e-04 | 1329.6ms/step | 15.0 samples/s +Step 12300/44000 | Loss: 3.1075 | LR: 4.17e-04 | 1330.1ms/step | 15.0 samples/s +Step 12310/44000 | Loss: 3.1353 | LR: 4.17e-04 | 1329.1ms/step | 15.0 samples/s +Step 12320/44000 | Loss: 3.0981 | LR: 4.17e-04 | 1328.8ms/step | 15.1 samples/s +Step 12330/44000 | Loss: 3.1138 | LR: 4.17e-04 | 1329.2ms/step | 15.0 samples/s +Step 12340/44000 | Loss: 3.1057 | LR: 4.17e-04 | 1327.1ms/step | 15.1 samples/s +Step 12350/44000 | Loss: 3.0799 | LR: 4.16e-04 | 1326.7ms/step | 15.1 samples/s +Step 12360/44000 | Loss: 3.1094 | LR: 4.16e-04 | 1325.4ms/step | 15.1 samples/s +Step 12370/44000 | Loss: 3.0681 | LR: 4.16e-04 | 1326.2ms/step | 15.1 samples/s +Step 12380/44000 | Loss: 3.1144 | LR: 4.16e-04 | 1325.7ms/step | 15.1 samples/s +Step 12390/44000 | Loss: 3.1303 | LR: 4.16e-04 | 1325.0ms/step | 15.1 samples/s +Step 12400/44000 | Loss: 3.1144 | LR: 4.16e-04 | 1323.4ms/step | 15.1 samples/s +Step 12410/44000 | Loss: 3.1269 | LR: 4.16e-04 | 1323.0ms/step | 15.1 samples/s +Step 12420/44000 | Loss: 3.1107 | LR: 4.15e-04 | 1328.8ms/step | 15.1 samples/s +Step 12430/44000 | Loss: 3.0899 | LR: 4.15e-04 | 1328.5ms/step | 15.1 samples/s +Step 12440/44000 | Loss: 3.1056 | LR: 4.15e-04 | 1326.5ms/step | 15.1 samples/s +Step 12450/44000 | Loss: 3.1296 | LR: 4.15e-04 | 1325.5ms/step | 15.1 samples/s +Step 12460/44000 | Loss: 3.0951 | LR: 4.15e-04 | 1329.8ms/step | 15.0 samples/s +Step 12470/44000 | Loss: 3.0775 | LR: 4.15e-04 | 1328.6ms/step | 15.1 samples/s +Step 12480/44000 | Loss: 3.0913 | LR: 4.15e-04 | 1326.8ms/step | 15.1 samples/s +Step 12490/44000 | Loss: 3.1104 | LR: 4.14e-04 | 1328.7ms/step | 15.1 samples/s +Step 12500/44000 | Loss: 3.1021 | LR: 4.14e-04 | 1328.8ms/step | 15.1 samples/s +Validation at step 12500: Loss = 3.0908, Perplexity = 21.99 +Step 12510/44000 | Loss: 3.0772 | LR: 4.14e-04 | 3553.8ms/step | 5.6 samples/s +Step 12520/44000 | Loss: 3.0718 | LR: 4.14e-04 | 1327.0ms/step | 15.1 samples/s +Step 12530/44000 | Loss: 3.0829 | LR: 4.14e-04 | 1325.8ms/step | 15.1 samples/s +Step 12540/44000 | Loss: 3.0732 | LR: 4.14e-04 | 1327.4ms/step | 15.1 samples/s +Step 12550/44000 | Loss: 3.0678 | LR: 4.14e-04 | 1325.3ms/step | 15.1 samples/s +Step 12560/44000 | Loss: 3.0930 | LR: 4.14e-04 | 1323.3ms/step | 15.1 samples/s +Step 12570/44000 | Loss: 3.1011 | LR: 4.13e-04 | 1325.8ms/step | 15.1 samples/s +Step 12580/44000 | Loss: 3.1127 | LR: 4.13e-04 | 1325.7ms/step | 15.1 samples/s +Step 12590/44000 | Loss: 3.0890 | LR: 4.13e-04 | 1324.7ms/step | 15.1 samples/s +Step 12600/44000 | Loss: 3.1283 | LR: 4.13e-04 | 1326.4ms/step | 15.1 samples/s +Step 12610/44000 | Loss: 3.1080 | LR: 4.13e-04 | 1327.4ms/step | 15.1 samples/s +Step 12620/44000 | Loss: 3.0837 | LR: 4.13e-04 | 1326.2ms/step | 15.1 samples/s +Step 12630/44000 | Loss: 3.1080 | LR: 4.13e-04 | 1327.1ms/step | 15.1 samples/s +Step 12640/44000 | Loss: 3.1109 | LR: 4.12e-04 | 1325.6ms/step | 15.1 samples/s +Step 12650/44000 | Loss: 3.1166 | LR: 4.12e-04 | 1324.7ms/step | 15.1 samples/s +Step 12660/44000 | Loss: 3.0865 | LR: 4.12e-04 | 1325.1ms/step | 15.1 samples/s +Step 12670/44000 | Loss: 3.0959 | LR: 4.12e-04 | 1328.5ms/step | 15.1 samples/s +Step 12680/44000 | Loss: 3.1029 | LR: 4.12e-04 | 1325.5ms/step | 15.1 samples/s +Step 12690/44000 | Loss: 3.1068 | LR: 4.12e-04 | 1327.2ms/step | 15.1 samples/s +Step 12700/44000 | Loss: 3.1171 | LR: 4.12e-04 | 1326.7ms/step | 15.1 samples/s +Step 12710/44000 | Loss: 3.0686 | LR: 4.11e-04 | 1325.7ms/step | 15.1 samples/s +Step 12720/44000 | Loss: 3.0985 | LR: 4.11e-04 | 1325.2ms/step | 15.1 samples/s +Step 12730/44000 | Loss: 3.1328 | LR: 4.11e-04 | 1324.8ms/step | 15.1 samples/s +Step 12740/44000 | Loss: 3.0984 | LR: 4.11e-04 | 1325.6ms/step | 15.1 samples/s +Step 12750/44000 | Loss: 3.0830 | LR: 4.11e-04 | 1327.5ms/step | 15.1 samples/s +Validation at step 12750: Loss = 3.0867, Perplexity = 21.90 +Step 12760/44000 | Loss: 3.1061 | LR: 4.11e-04 | 3554.5ms/step | 5.6 samples/s +Step 12770/44000 | Loss: 3.0958 | LR: 4.11e-04 | 1321.7ms/step | 15.1 samples/s +Step 12780/44000 | Loss: 3.1330 | LR: 4.10e-04 | 1325.8ms/step | 15.1 samples/s +Step 12790/44000 | Loss: 3.0843 | LR: 4.10e-04 | 1328.1ms/step | 15.1 samples/s +Step 12800/44000 | Loss: 3.1101 | LR: 4.10e-04 | 1327.2ms/step | 15.1 samples/s +Step 12810/44000 | Loss: 3.1099 | LR: 4.10e-04 | 1324.3ms/step | 15.1 samples/s +Step 12820/44000 | Loss: 3.0887 | LR: 4.10e-04 | 1321.3ms/step | 15.1 samples/s +Step 12830/44000 | Loss: 3.0954 | LR: 4.10e-04 | 1324.0ms/step | 15.1 samples/s +Step 12840/44000 | Loss: 3.1010 | LR: 4.10e-04 | 1327.8ms/step | 15.1 samples/s +Step 12850/44000 | Loss: 3.0977 | LR: 4.09e-04 | 1326.7ms/step | 15.1 samples/s +Step 12860/44000 | Loss: 3.1023 | LR: 4.09e-04 | 1331.2ms/step | 15.0 samples/s +Step 12870/44000 | Loss: 3.1077 | LR: 4.09e-04 | 1327.2ms/step | 15.1 samples/s +Step 12880/44000 | Loss: 3.0869 | LR: 4.09e-04 | 1325.0ms/step | 15.1 samples/s +Step 12890/44000 | Loss: 3.0943 | LR: 4.09e-04 | 1324.5ms/step | 15.1 samples/s +Step 12900/44000 | Loss: 3.1154 | LR: 4.09e-04 | 1328.0ms/step | 15.1 samples/s +Step 12910/44000 | Loss: 3.1081 | LR: 4.09e-04 | 1325.9ms/step | 15.1 samples/s +Step 12920/44000 | Loss: 3.0841 | LR: 4.09e-04 | 1324.6ms/step | 15.1 samples/s +Step 12930/44000 | Loss: 3.1273 | LR: 4.08e-04 | 1322.1ms/step | 15.1 samples/s +Step 12940/44000 | Loss: 3.0823 | LR: 4.08e-04 | 1323.9ms/step | 15.1 samples/s +Step 12950/44000 | Loss: 3.0926 | LR: 4.08e-04 | 1326.9ms/step | 15.1 samples/s +Step 12960/44000 | Loss: 3.0980 | LR: 4.08e-04 | 1324.9ms/step | 15.1 samples/s +Step 12970/44000 | Loss: 3.0757 | LR: 4.08e-04 | 1323.3ms/step | 15.1 samples/s +Step 12980/44000 | Loss: 3.0637 | LR: 4.08e-04 | 1329.0ms/step | 15.0 samples/s +Step 12990/44000 | Loss: 3.0831 | LR: 4.08e-04 | 1328.8ms/step | 15.1 samples/s +Step 13000/44000 | Loss: 3.0460 | LR: 4.07e-04 | 1327.4ms/step | 15.1 samples/s +Validation at step 13000: Loss = 3.0821, Perplexity = 21.80 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-13000 +Step 13010/44000 | Loss: 3.0740 | LR: 4.07e-04 | 3875.9ms/step | 5.2 samples/s +Step 13020/44000 | Loss: 3.0593 | LR: 4.07e-04 | 1326.1ms/step | 15.1 samples/s +Step 13030/44000 | Loss: 3.0993 | LR: 4.07e-04 | 1327.9ms/step | 15.1 samples/s +Step 13040/44000 | Loss: 3.0978 | LR: 4.07e-04 | 1330.0ms/step | 15.0 samples/s +Step 13050/44000 | Loss: 3.1016 | LR: 4.07e-04 | 1329.6ms/step | 15.0 samples/s +Step 13060/44000 | Loss: 3.1026 | LR: 4.07e-04 | 1324.7ms/step | 15.1 samples/s +Step 13070/44000 | Loss: 3.0659 | LR: 4.06e-04 | 1328.3ms/step | 15.1 samples/s +Step 13080/44000 | Loss: 3.1437 | LR: 4.06e-04 | 1327.5ms/step | 15.1 samples/s +Step 13090/44000 | Loss: 3.0944 | LR: 4.06e-04 | 1327.2ms/step | 15.1 samples/s +Step 13100/44000 | Loss: 3.1004 | LR: 4.06e-04 | 1326.2ms/step | 15.1 samples/s +Step 13110/44000 | Loss: 3.1076 | LR: 4.06e-04 | 1324.1ms/step | 15.1 samples/s +Step 13120/44000 | Loss: 3.0788 | LR: 4.06e-04 | 1324.1ms/step | 15.1 samples/s +Step 13130/44000 | Loss: 3.1075 | LR: 4.06e-04 | 1329.4ms/step | 15.0 samples/s +Step 13140/44000 | Loss: 3.0930 | LR: 4.05e-04 | 1329.8ms/step | 15.0 samples/s +Step 13150/44000 | Loss: 3.0812 | LR: 4.05e-04 | 1326.6ms/step | 15.1 samples/s +Step 13160/44000 | Loss: 3.0659 | LR: 4.05e-04 | 1325.8ms/step | 15.1 samples/s +Step 13170/44000 | Loss: 3.0840 | LR: 4.05e-04 | 1329.9ms/step | 15.0 samples/s +Step 13180/44000 | Loss: 3.0785 | LR: 4.05e-04 | 1330.3ms/step | 15.0 samples/s +Step 13190/44000 | Loss: 3.0830 | LR: 4.05e-04 | 1328.1ms/step | 15.1 samples/s +Step 13200/44000 | Loss: 3.1471 | LR: 4.05e-04 | 1327.9ms/step | 15.1 samples/s +Step 13210/44000 | Loss: 3.1030 | LR: 4.04e-04 | 1327.8ms/step | 15.1 samples/s +Step 13220/44000 | Loss: 3.0947 | LR: 4.04e-04 | 1326.6ms/step | 15.1 samples/s +Step 13230/44000 | Loss: 3.1070 | LR: 4.04e-04 | 1326.4ms/step | 15.1 samples/s +Step 13240/44000 | Loss: 3.1107 | LR: 4.04e-04 | 1328.1ms/step | 15.1 samples/s +Step 13250/44000 | Loss: 3.0836 | LR: 4.04e-04 | 1328.6ms/step | 15.1 samples/s +Validation at step 13250: Loss = 3.0789, Perplexity = 21.74 +Step 13260/44000 | Loss: 3.0810 | LR: 4.04e-04 | 3555.8ms/step | 5.6 samples/s +Step 13270/44000 | Loss: 3.0897 | LR: 4.04e-04 | 1325.3ms/step | 15.1 samples/s +Step 13280/44000 | Loss: 3.1111 | LR: 4.03e-04 | 1324.9ms/step | 15.1 samples/s +Step 13290/44000 | Loss: 3.0694 | LR: 4.03e-04 | 1320.2ms/step | 15.1 samples/s +Step 13300/44000 | Loss: 3.0997 | LR: 4.03e-04 | 1326.3ms/step | 15.1 samples/s +Step 13310/44000 | Loss: 3.0721 | LR: 4.03e-04 | 1326.2ms/step | 15.1 samples/s +Step 13320/44000 | Loss: 3.0978 | LR: 4.03e-04 | 1326.8ms/step | 15.1 samples/s +Step 13330/44000 | Loss: 3.1171 | LR: 4.03e-04 | 1322.4ms/step | 15.1 samples/s +Step 13340/44000 | Loss: 3.1041 | LR: 4.03e-04 | 1326.4ms/step | 15.1 samples/s +Step 13350/44000 | Loss: 3.0827 | LR: 4.02e-04 | 1325.0ms/step | 15.1 samples/s +Step 13360/44000 | Loss: 3.0540 | LR: 4.02e-04 | 1326.2ms/step | 15.1 samples/s +Step 13370/44000 | Loss: 3.1020 | LR: 4.02e-04 | 1330.0ms/step | 15.0 samples/s +Step 13380/44000 | Loss: 3.0671 | LR: 4.02e-04 | 1326.5ms/step | 15.1 samples/s +Step 13390/44000 | Loss: 3.0971 | LR: 4.02e-04 | 1326.1ms/step | 15.1 samples/s +Step 13400/44000 | Loss: 3.0683 | LR: 4.02e-04 | 1323.8ms/step | 15.1 samples/s +Step 13410/44000 | Loss: 3.0916 | LR: 4.02e-04 | 1324.0ms/step | 15.1 samples/s +Step 13420/44000 | Loss: 3.0810 | LR: 4.01e-04 | 1323.3ms/step | 15.1 samples/s +Step 13430/44000 | Loss: 3.0874 | LR: 4.01e-04 | 1324.5ms/step | 15.1 samples/s +Step 13440/44000 | Loss: 3.0633 | LR: 4.01e-04 | 1326.7ms/step | 15.1 samples/s +Step 13450/44000 | Loss: 3.1175 | LR: 4.01e-04 | 1323.2ms/step | 15.1 samples/s +Step 13460/44000 | Loss: 3.0798 | LR: 4.01e-04 | 1322.3ms/step | 15.1 samples/s +Step 13470/44000 | Loss: 3.0757 | LR: 4.01e-04 | 1321.0ms/step | 15.1 samples/s +Step 13480/44000 | Loss: 3.0852 | LR: 4.01e-04 | 1323.3ms/step | 15.1 samples/s +Step 13490/44000 | Loss: 3.0809 | LR: 4.00e-04 | 1321.5ms/step | 15.1 samples/s +Step 13500/44000 | Loss: 3.1186 | LR: 4.00e-04 | 1322.9ms/step | 15.1 samples/s +Validation at step 13500: Loss = 3.0730, Perplexity = 21.61 +Step 13510/44000 | Loss: 3.0600 | LR: 4.00e-04 | 3551.0ms/step | 5.6 samples/s +Step 13520/44000 | Loss: 3.1102 | LR: 4.00e-04 | 1320.6ms/step | 15.1 samples/s +Step 13530/44000 | Loss: 3.0536 | LR: 4.00e-04 | 1327.2ms/step | 15.1 samples/s +Step 13540/44000 | Loss: 3.0870 | LR: 4.00e-04 | 1326.2ms/step | 15.1 samples/s +Step 13550/44000 | Loss: 3.1223 | LR: 4.00e-04 | 1325.4ms/step | 15.1 samples/s +Step 13560/44000 | Loss: 3.0923 | LR: 3.99e-04 | 1325.2ms/step | 15.1 samples/s +Step 13570/44000 | Loss: 3.0670 | LR: 3.99e-04 | 1324.3ms/step | 15.1 samples/s +Step 13580/44000 | Loss: 3.0666 | LR: 3.99e-04 | 1324.4ms/step | 15.1 samples/s +Step 13590/44000 | Loss: 3.0646 | LR: 3.99e-04 | 1325.9ms/step | 15.1 samples/s +Step 13600/44000 | Loss: 3.1118 | LR: 3.99e-04 | 1327.0ms/step | 15.1 samples/s +Step 13610/44000 | Loss: 3.1016 | LR: 3.99e-04 | 1328.4ms/step | 15.1 samples/s +Step 13620/44000 | Loss: 3.0685 | LR: 3.98e-04 | 1324.3ms/step | 15.1 samples/s +Step 13630/44000 | Loss: 3.1071 | LR: 3.98e-04 | 1322.9ms/step | 15.1 samples/s +Step 13640/44000 | Loss: 3.0813 | LR: 3.98e-04 | 1325.2ms/step | 15.1 samples/s +Step 13650/44000 | Loss: 3.0831 | LR: 3.98e-04 | 1321.6ms/step | 15.1 samples/s +Step 13660/44000 | Loss: 3.0651 | LR: 3.98e-04 | 1319.9ms/step | 15.2 samples/s +Step 13670/44000 | Loss: 3.0813 | LR: 3.98e-04 | 1319.6ms/step | 15.2 samples/s +Step 13680/44000 | Loss: 3.1141 | LR: 3.98e-04 | 1324.3ms/step | 15.1 samples/s +Step 13690/44000 | Loss: 3.0863 | LR: 3.97e-04 | 1322.3ms/step | 15.1 samples/s +Step 13700/44000 | Loss: 3.0691 | LR: 3.97e-04 | 1326.4ms/step | 15.1 samples/s +Step 13710/44000 | Loss: 3.0792 | LR: 3.97e-04 | 1325.1ms/step | 15.1 samples/s +Step 13720/44000 | Loss: 3.1090 | LR: 3.97e-04 | 1323.5ms/step | 15.1 samples/s +Step 13730/44000 | Loss: 3.0783 | LR: 3.97e-04 | 1326.6ms/step | 15.1 samples/s +Step 13740/44000 | Loss: 3.1087 | LR: 3.97e-04 | 1327.0ms/step | 15.1 samples/s +Step 13750/44000 | Loss: 3.0989 | LR: 3.97e-04 | 1326.9ms/step | 15.1 samples/s +Validation at step 13750: Loss = 3.0704, Perplexity = 21.55 +Step 13760/44000 | Loss: 3.1100 | LR: 3.96e-04 | 3551.7ms/step | 5.6 samples/s +Step 13770/44000 | Loss: 3.1064 | LR: 3.96e-04 | 1326.3ms/step | 15.1 samples/s +Step 13780/44000 | Loss: 3.0824 | LR: 3.96e-04 | 1322.3ms/step | 15.1 samples/s +Step 13790/44000 | Loss: 3.0679 | LR: 3.96e-04 | 1324.1ms/step | 15.1 samples/s +Step 13800/44000 | Loss: 3.0804 | LR: 3.96e-04 | 1327.7ms/step | 15.1 samples/s +Step 13810/44000 | Loss: 3.1128 | LR: 3.96e-04 | 1329.0ms/step | 15.0 samples/s +Step 13820/44000 | Loss: 3.0981 | LR: 3.96e-04 | 1327.1ms/step | 15.1 samples/s +Step 13830/44000 | Loss: 3.0453 | LR: 3.95e-04 | 1326.2ms/step | 15.1 samples/s +Step 13840/44000 | Loss: 3.0888 | LR: 3.95e-04 | 1326.7ms/step | 15.1 samples/s +Step 13850/44000 | Loss: 3.0453 | LR: 3.95e-04 | 1327.5ms/step | 15.1 samples/s +Step 13860/44000 | Loss: 3.0894 | LR: 3.95e-04 | 1325.4ms/step | 15.1 samples/s +Step 13870/44000 | Loss: 3.0989 | LR: 3.95e-04 | 1326.3ms/step | 15.1 samples/s +Step 13880/44000 | Loss: 3.0670 | LR: 3.95e-04 | 1329.8ms/step | 15.0 samples/s +Step 13890/44000 | Loss: 3.0689 | LR: 3.95e-04 | 1329.4ms/step | 15.0 samples/s +Step 13900/44000 | Loss: 3.0882 | LR: 3.94e-04 | 1327.0ms/step | 15.1 samples/s +Step 13910/44000 | Loss: 3.0904 | LR: 3.94e-04 | 1327.4ms/step | 15.1 samples/s +Step 13920/44000 | Loss: 3.0826 | LR: 3.94e-04 | 1327.1ms/step | 15.1 samples/s +Step 13930/44000 | Loss: 3.0887 | LR: 3.94e-04 | 1325.5ms/step | 15.1 samples/s +Step 13940/44000 | Loss: 3.0745 | LR: 3.94e-04 | 1326.0ms/step | 15.1 samples/s +Step 13950/44000 | Loss: 3.0722 | LR: 3.94e-04 | 1328.6ms/step | 15.1 samples/s +Step 13960/44000 | Loss: 3.0814 | LR: 3.93e-04 | 1327.2ms/step | 15.1 samples/s +Step 13970/44000 | Loss: 3.0699 | LR: 3.93e-04 | 1326.2ms/step | 15.1 samples/s +Step 13980/44000 | Loss: 3.0677 | LR: 3.93e-04 | 1327.0ms/step | 15.1 samples/s +Step 13990/44000 | Loss: 3.0718 | LR: 3.93e-04 | 1325.7ms/step | 15.1 samples/s +Step 14000/44000 | Loss: 3.0986 | LR: 3.93e-04 | 1329.7ms/step | 15.0 samples/s +Validation at step 14000: Loss = 3.0668, Perplexity = 21.47 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-14000 +Step 14010/44000 | Loss: 3.0737 | LR: 3.93e-04 | 3875.4ms/step | 5.2 samples/s +Step 14020/44000 | Loss: 3.0957 | LR: 3.93e-04 | 1327.7ms/step | 15.1 samples/s +Step 14030/44000 | Loss: 3.1021 | LR: 3.92e-04 | 1326.8ms/step | 15.1 samples/s +Step 14040/44000 | Loss: 3.0990 | LR: 3.92e-04 | 1329.3ms/step | 15.0 samples/s +Step 14050/44000 | Loss: 3.0579 | LR: 3.92e-04 | 1325.7ms/step | 15.1 samples/s +Step 14060/44000 | Loss: 3.0690 | LR: 3.92e-04 | 1328.0ms/step | 15.1 samples/s +Step 14070/44000 | Loss: 3.0960 | LR: 3.92e-04 | 1323.8ms/step | 15.1 samples/s +Step 14080/44000 | Loss: 3.0683 | LR: 3.92e-04 | 1328.4ms/step | 15.1 samples/s +Step 14090/44000 | Loss: 3.0830 | LR: 3.92e-04 | 1326.5ms/step | 15.1 samples/s +Step 14100/44000 | Loss: 3.0767 | LR: 3.91e-04 | 1325.8ms/step | 15.1 samples/s +Step 14110/44000 | Loss: 3.0885 | LR: 3.91e-04 | 1326.1ms/step | 15.1 samples/s +Step 14120/44000 | Loss: 3.0820 | LR: 3.91e-04 | 1324.8ms/step | 15.1 samples/s +Step 14130/44000 | Loss: 3.1032 | LR: 3.91e-04 | 1323.4ms/step | 15.1 samples/s +Step 14140/44000 | Loss: 3.0762 | LR: 3.91e-04 | 1326.2ms/step | 15.1 samples/s +Step 14150/44000 | Loss: 3.0610 | LR: 3.91e-04 | 1327.7ms/step | 15.1 samples/s +Step 14160/44000 | Loss: 3.0886 | LR: 3.90e-04 | 1327.3ms/step | 15.1 samples/s +Step 14170/44000 | Loss: 3.0609 | LR: 3.90e-04 | 1327.4ms/step | 15.1 samples/s +Step 14180/44000 | Loss: 3.0901 | LR: 3.90e-04 | 1325.0ms/step | 15.1 samples/s +Step 14190/44000 | Loss: 3.0732 | LR: 3.90e-04 | 1328.2ms/step | 15.1 samples/s +Step 14200/44000 | Loss: 3.0778 | LR: 3.90e-04 | 1326.3ms/step | 15.1 samples/s +Step 14210/44000 | Loss: 3.0961 | LR: 3.90e-04 | 1327.2ms/step | 15.1 samples/s +Step 14220/44000 | Loss: 3.0859 | LR: 3.90e-04 | 1327.3ms/step | 15.1 samples/s +Step 14230/44000 | Loss: 3.0626 | LR: 3.89e-04 | 1326.7ms/step | 15.1 samples/s +Step 14240/44000 | Loss: 3.0738 | LR: 3.89e-04 | 1326.0ms/step | 15.1 samples/s +Step 14250/44000 | Loss: 3.0886 | LR: 3.89e-04 | 1326.9ms/step | 15.1 samples/s +Validation at step 14250: Loss = 3.0629, Perplexity = 21.39 +Step 14260/44000 | Loss: 3.0677 | LR: 3.89e-04 | 3563.8ms/step | 5.6 samples/s +Step 14270/44000 | Loss: 3.0739 | LR: 3.89e-04 | 1327.0ms/step | 15.1 samples/s +Step 14280/44000 | Loss: 3.0649 | LR: 3.89e-04 | 1327.2ms/step | 15.1 samples/s +Step 14290/44000 | Loss: 3.0672 | LR: 3.89e-04 | 1328.6ms/step | 15.1 samples/s +Step 14300/44000 | Loss: 3.0853 | LR: 3.88e-04 | 1329.6ms/step | 15.0 samples/s +Step 14310/44000 | Loss: 3.0758 | LR: 3.88e-04 | 1328.4ms/step | 15.1 samples/s +Step 14320/44000 | Loss: 3.0508 | LR: 3.88e-04 | 1330.3ms/step | 15.0 samples/s +Step 14330/44000 | Loss: 3.1100 | LR: 3.88e-04 | 1329.4ms/step | 15.0 samples/s +Step 14340/44000 | Loss: 3.0816 | LR: 3.88e-04 | 1329.0ms/step | 15.0 samples/s +Step 14350/44000 | Loss: 3.0645 | LR: 3.88e-04 | 1328.3ms/step | 15.1 samples/s +Step 14360/44000 | Loss: 3.0621 | LR: 3.87e-04 | 1325.9ms/step | 15.1 samples/s +Step 14370/44000 | Loss: 3.1050 | LR: 3.87e-04 | 1324.1ms/step | 15.1 samples/s +Step 14380/44000 | Loss: 3.0602 | LR: 3.87e-04 | 1321.8ms/step | 15.1 samples/s +Step 14390/44000 | Loss: 3.1031 | LR: 3.87e-04 | 1323.7ms/step | 15.1 samples/s +Step 14400/44000 | Loss: 3.0754 | LR: 3.87e-04 | 1325.1ms/step | 15.1 samples/s +Step 14410/44000 | Loss: 3.0811 | LR: 3.87e-04 | 1322.6ms/step | 15.1 samples/s +Step 14420/44000 | Loss: 3.0468 | LR: 3.87e-04 | 1320.6ms/step | 15.1 samples/s +Step 14430/44000 | Loss: 3.0842 | LR: 3.86e-04 | 1324.0ms/step | 15.1 samples/s +Step 14440/44000 | Loss: 3.0745 | LR: 3.86e-04 | 1325.2ms/step | 15.1 samples/s +Step 14450/44000 | Loss: 3.0866 | LR: 3.86e-04 | 1325.9ms/step | 15.1 samples/s +Step 14460/44000 | Loss: 3.0740 | LR: 3.86e-04 | 1327.8ms/step | 15.1 samples/s +Step 14470/44000 | Loss: 3.0547 | LR: 3.86e-04 | 1331.4ms/step | 15.0 samples/s +Step 14480/44000 | Loss: 3.0661 | LR: 3.86e-04 | 1327.1ms/step | 15.1 samples/s +Step 14490/44000 | Loss: 3.0653 | LR: 3.85e-04 | 1322.7ms/step | 15.1 samples/s +Step 14500/44000 | Loss: 3.0497 | LR: 3.85e-04 | 1324.7ms/step | 15.1 samples/s +Validation at step 14500: Loss = 3.0598, Perplexity = 21.32 +Step 14510/44000 | Loss: 3.0815 | LR: 3.85e-04 | 3547.5ms/step | 5.6 samples/s +Step 14520/44000 | Loss: 3.0715 | LR: 3.85e-04 | 1325.9ms/step | 15.1 samples/s +Step 14530/44000 | Loss: 3.0717 | LR: 3.85e-04 | 1327.1ms/step | 15.1 samples/s +Step 14540/44000 | Loss: 3.0741 | LR: 3.85e-04 | 1329.5ms/step | 15.0 samples/s +Step 14550/44000 | Loss: 3.0907 | LR: 3.85e-04 | 1327.9ms/step | 15.1 samples/s +Step 14560/44000 | Loss: 3.0590 | LR: 3.84e-04 | 1328.0ms/step | 15.1 samples/s +Step 14570/44000 | Loss: 3.0611 | LR: 3.84e-04 | 1328.0ms/step | 15.1 samples/s +Step 14580/44000 | Loss: 3.0520 | LR: 3.84e-04 | 1326.0ms/step | 15.1 samples/s +Step 14590/44000 | Loss: 3.0189 | LR: 3.84e-04 | 1326.3ms/step | 15.1 samples/s +Step 14600/44000 | Loss: 3.0701 | LR: 3.84e-04 | 1328.9ms/step | 15.0 samples/s +Step 14610/44000 | Loss: 3.0779 | LR: 3.84e-04 | 1328.3ms/step | 15.1 samples/s +Step 14620/44000 | Loss: 3.0883 | LR: 3.83e-04 | 1326.4ms/step | 15.1 samples/s +Step 14630/44000 | Loss: 3.0403 | LR: 3.83e-04 | 1329.4ms/step | 15.0 samples/s +Step 14640/44000 | Loss: 3.0658 | LR: 3.83e-04 | 1324.1ms/step | 15.1 samples/s +Step 14650/44000 | Loss: 3.0540 | LR: 3.83e-04 | 1324.6ms/step | 15.1 samples/s +Step 14660/44000 | Loss: 3.0899 | LR: 3.83e-04 | 1326.9ms/step | 15.1 samples/s +Step 14670/44000 | Loss: 3.0929 | LR: 3.83e-04 | 1327.0ms/step | 15.1 samples/s +Step 14680/44000 | Loss: 3.0730 | LR: 3.83e-04 | 1324.3ms/step | 15.1 samples/s +Step 14690/44000 | Loss: 3.0493 | LR: 3.82e-04 | 1323.1ms/step | 15.1 samples/s +Step 14700/44000 | Loss: 3.0549 | LR: 3.82e-04 | 1326.6ms/step | 15.1 samples/s +Step 14710/44000 | Loss: 3.0692 | LR: 3.82e-04 | 1325.5ms/step | 15.1 samples/s +Step 14720/44000 | Loss: 3.0577 | LR: 3.82e-04 | 1328.7ms/step | 15.1 samples/s +Step 14730/44000 | Loss: 3.0538 | LR: 3.82e-04 | 1327.2ms/step | 15.1 samples/s +Step 14740/44000 | Loss: 3.0886 | LR: 3.82e-04 | 1325.5ms/step | 15.1 samples/s +Step 14750/44000 | Loss: 3.0510 | LR: 3.81e-04 | 1327.0ms/step | 15.1 samples/s +Validation at step 14750: Loss = 3.0579, Perplexity = 21.28 +Step 14760/44000 | Loss: 3.0654 | LR: 3.81e-04 | 3554.7ms/step | 5.6 samples/s +Step 14770/44000 | Loss: 3.0632 | LR: 3.81e-04 | 1324.8ms/step | 15.1 samples/s +Step 14780/44000 | Loss: 3.0538 | LR: 3.81e-04 | 1324.2ms/step | 15.1 samples/s +Step 14790/44000 | Loss: 3.0976 | LR: 3.81e-04 | 1321.4ms/step | 15.1 samples/s +Step 14800/44000 | Loss: 3.0725 | LR: 3.81e-04 | 1323.7ms/step | 15.1 samples/s +Step 14810/44000 | Loss: 3.0787 | LR: 3.81e-04 | 1322.5ms/step | 15.1 samples/s +Step 14820/44000 | Loss: 3.0819 | LR: 3.80e-04 | 1324.2ms/step | 15.1 samples/s +Step 14830/44000 | Loss: 3.0602 | LR: 3.80e-04 | 1323.5ms/step | 15.1 samples/s +Step 14840/44000 | Loss: 3.0765 | LR: 3.80e-04 | 1327.6ms/step | 15.1 samples/s +Step 14850/44000 | Loss: 3.0653 | LR: 3.80e-04 | 1327.3ms/step | 15.1 samples/s +Step 14860/44000 | Loss: 3.0661 | LR: 3.80e-04 | 1325.4ms/step | 15.1 samples/s +Step 14870/44000 | Loss: 3.0784 | LR: 3.80e-04 | 1327.3ms/step | 15.1 samples/s +Step 14880/44000 | Loss: 3.0664 | LR: 3.79e-04 | 1330.2ms/step | 15.0 samples/s +Step 14890/44000 | Loss: 3.0576 | LR: 3.79e-04 | 1328.0ms/step | 15.1 samples/s +Step 14900/44000 | Loss: 3.0674 | LR: 3.79e-04 | 1326.5ms/step | 15.1 samples/s +Step 14910/44000 | Loss: 3.0866 | LR: 3.79e-04 | 1327.9ms/step | 15.1 samples/s +Step 14920/44000 | Loss: 3.0847 | LR: 3.79e-04 | 1327.6ms/step | 15.1 samples/s +Step 14930/44000 | Loss: 3.0607 | LR: 3.79e-04 | 1325.9ms/step | 15.1 samples/s +Step 14940/44000 | Loss: 3.0960 | LR: 3.79e-04 | 1328.1ms/step | 15.1 samples/s +Step 14950/44000 | Loss: 3.0725 | LR: 3.78e-04 | 1327.4ms/step | 15.1 samples/s +Step 14960/44000 | Loss: 3.0655 | LR: 3.78e-04 | 1326.9ms/step | 15.1 samples/s +Step 14970/44000 | Loss: 3.1166 | LR: 3.78e-04 | 1328.8ms/step | 15.1 samples/s +Step 14980/44000 | Loss: 3.0624 | LR: 3.78e-04 | 1326.0ms/step | 15.1 samples/s +Step 14990/44000 | Loss: 3.0500 | LR: 3.78e-04 | 1325.7ms/step | 15.1 samples/s +Step 15000/44000 | Loss: 3.0586 | LR: 3.78e-04 | 1325.7ms/step | 15.1 samples/s +Validation at step 15000: Loss = 3.0518, Perplexity = 21.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-15000 +Step 15010/44000 | Loss: 3.0528 | LR: 3.77e-04 | 3880.7ms/step | 5.2 samples/s +Step 15020/44000 | Loss: 3.0797 | LR: 3.77e-04 | 1326.9ms/step | 15.1 samples/s +Step 15030/44000 | Loss: 3.0533 | LR: 3.77e-04 | 1326.7ms/step | 15.1 samples/s +Step 15040/44000 | Loss: 3.0714 | LR: 3.77e-04 | 1325.9ms/step | 15.1 samples/s +Step 15050/44000 | Loss: 3.0406 | LR: 3.77e-04 | 1329.0ms/step | 15.0 samples/s +Step 15060/44000 | Loss: 3.0650 | LR: 3.77e-04 | 1326.0ms/step | 15.1 samples/s +Step 15070/44000 | Loss: 3.0670 | LR: 3.77e-04 | 1326.1ms/step | 15.1 samples/s +Step 15080/44000 | Loss: 3.0508 | LR: 3.76e-04 | 1329.9ms/step | 15.0 samples/s +Step 15090/44000 | Loss: 3.0721 | LR: 3.76e-04 | 1327.8ms/step | 15.1 samples/s +Step 15100/44000 | Loss: 3.0638 | LR: 3.76e-04 | 1327.9ms/step | 15.1 samples/s +Step 15110/44000 | Loss: 3.0688 | LR: 3.76e-04 | 1330.8ms/step | 15.0 samples/s +Step 15120/44000 | Loss: 3.0765 | LR: 3.76e-04 | 1329.8ms/step | 15.0 samples/s +Step 15130/44000 | Loss: 3.0658 | LR: 3.76e-04 | 1329.8ms/step | 15.0 samples/s +Step 15140/44000 | Loss: 3.0748 | LR: 3.75e-04 | 1326.4ms/step | 15.1 samples/s +Step 15150/44000 | Loss: 3.0526 | LR: 3.75e-04 | 1330.3ms/step | 15.0 samples/s +Step 15160/44000 | Loss: 3.0560 | LR: 3.75e-04 | 1328.8ms/step | 15.1 samples/s +Step 15170/44000 | Loss: 3.0952 | LR: 3.75e-04 | 1326.3ms/step | 15.1 samples/s +Step 15180/44000 | Loss: 3.0320 | LR: 3.75e-04 | 1326.1ms/step | 15.1 samples/s +Step 15190/44000 | Loss: 3.0597 | LR: 3.75e-04 | 1328.7ms/step | 15.1 samples/s +Step 15200/44000 | Loss: 3.0443 | LR: 3.74e-04 | 1328.1ms/step | 15.1 samples/s +Step 15210/44000 | Loss: 3.0763 | LR: 3.74e-04 | 1328.0ms/step | 15.1 samples/s +Step 15220/44000 | Loss: 3.0667 | LR: 3.74e-04 | 1325.6ms/step | 15.1 samples/s +Step 15230/44000 | Loss: 3.0756 | LR: 3.74e-04 | 1329.5ms/step | 15.0 samples/s +Step 15240/44000 | Loss: 3.0533 | LR: 3.74e-04 | 1325.4ms/step | 15.1 samples/s +Step 15250/44000 | Loss: 3.0505 | LR: 3.74e-04 | 1330.1ms/step | 15.0 samples/s +Validation at step 15250: Loss = 3.0484, Perplexity = 21.08 +Step 15260/44000 | Loss: 3.0504 | LR: 3.74e-04 | 3554.4ms/step | 5.6 samples/s +Step 15270/44000 | Loss: 3.0492 | LR: 3.73e-04 | 1327.0ms/step | 15.1 samples/s +Step 15280/44000 | Loss: 3.0488 | LR: 3.73e-04 | 1323.7ms/step | 15.1 samples/s +Step 15290/44000 | Loss: 3.0679 | LR: 3.73e-04 | 1324.7ms/step | 15.1 samples/s +Step 15300/44000 | Loss: 3.0558 | LR: 3.73e-04 | 1329.2ms/step | 15.0 samples/s +Step 15310/44000 | Loss: 3.0676 | LR: 3.73e-04 | 1325.8ms/step | 15.1 samples/s +Step 15320/44000 | Loss: 3.0807 | LR: 3.73e-04 | 1326.8ms/step | 15.1 samples/s +Step 15330/44000 | Loss: 3.0448 | LR: 3.72e-04 | 1322.5ms/step | 15.1 samples/s +Step 15340/44000 | Loss: 3.0752 | LR: 3.72e-04 | 1324.9ms/step | 15.1 samples/s +Step 15350/44000 | Loss: 3.0671 | LR: 3.72e-04 | 1326.4ms/step | 15.1 samples/s +Step 15360/44000 | Loss: 3.0755 | LR: 3.72e-04 | 1320.9ms/step | 15.1 samples/s +Step 15370/44000 | Loss: 3.0476 | LR: 3.72e-04 | 1327.7ms/step | 15.1 samples/s +Step 15380/44000 | Loss: 3.0560 | LR: 3.72e-04 | 1325.3ms/step | 15.1 samples/s +Step 15390/44000 | Loss: 3.0573 | LR: 3.71e-04 | 1326.2ms/step | 15.1 samples/s +Step 15400/44000 | Loss: 3.0587 | LR: 3.71e-04 | 1324.9ms/step | 15.1 samples/s +Step 15410/44000 | Loss: 3.0612 | LR: 3.71e-04 | 1327.6ms/step | 15.1 samples/s +Step 15420/44000 | Loss: 3.0407 | LR: 3.71e-04 | 1327.6ms/step | 15.1 samples/s +Step 15430/44000 | Loss: 3.0657 | LR: 3.71e-04 | 1328.4ms/step | 15.1 samples/s +Step 15440/44000 | Loss: 3.0322 | LR: 3.71e-04 | 1326.7ms/step | 15.1 samples/s +Step 15450/44000 | Loss: 3.0777 | LR: 3.71e-04 | 1328.2ms/step | 15.1 samples/s +Step 15460/44000 | Loss: 3.0610 | LR: 3.70e-04 | 1329.3ms/step | 15.0 samples/s +Step 15470/44000 | Loss: 3.0536 | LR: 3.70e-04 | 1327.2ms/step | 15.1 samples/s +Step 15480/44000 | Loss: 3.0566 | LR: 3.70e-04 | 1325.8ms/step | 15.1 samples/s +Step 15490/44000 | Loss: 3.0518 | LR: 3.70e-04 | 1326.8ms/step | 15.1 samples/s +Step 15500/44000 | Loss: 3.0340 | LR: 3.70e-04 | 1324.0ms/step | 15.1 samples/s +Validation at step 15500: Loss = 3.0455, Perplexity = 21.02 +Step 15510/44000 | Loss: 3.0798 | LR: 3.70e-04 | 3554.8ms/step | 5.6 samples/s +Step 15520/44000 | Loss: 3.0511 | LR: 3.69e-04 | 1325.2ms/step | 15.1 samples/s +Step 15530/44000 | Loss: 3.0464 | LR: 3.69e-04 | 1325.2ms/step | 15.1 samples/s +Step 15540/44000 | Loss: 3.0584 | LR: 3.69e-04 | 1323.9ms/step | 15.1 samples/s +Step 15550/44000 | Loss: 3.0344 | LR: 3.69e-04 | 1325.7ms/step | 15.1 samples/s +Step 15560/44000 | Loss: 3.0455 | LR: 3.69e-04 | 1323.7ms/step | 15.1 samples/s +Step 15570/44000 | Loss: 3.0426 | LR: 3.69e-04 | 1323.6ms/step | 15.1 samples/s +Step 15580/44000 | Loss: 3.0617 | LR: 3.68e-04 | 1324.4ms/step | 15.1 samples/s +Step 15590/44000 | Loss: 3.0517 | LR: 3.68e-04 | 1325.2ms/step | 15.1 samples/s +Step 15600/44000 | Loss: 3.0443 | LR: 3.68e-04 | 1327.7ms/step | 15.1 samples/s +Step 15610/44000 | Loss: 3.0317 | LR: 3.68e-04 | 1324.2ms/step | 15.1 samples/s +Step 15620/44000 | Loss: 3.0530 | LR: 3.68e-04 | 1326.5ms/step | 15.1 samples/s +Step 15630/44000 | Loss: 3.0501 | LR: 3.68e-04 | 1325.8ms/step | 15.1 samples/s +Step 15640/44000 | Loss: 3.0615 | LR: 3.67e-04 | 1323.0ms/step | 15.1 samples/s +Step 15650/44000 | Loss: 3.0515 | LR: 3.67e-04 | 1325.9ms/step | 15.1 samples/s +Step 15660/44000 | Loss: 3.0124 | LR: 3.67e-04 | 1329.1ms/step | 15.0 samples/s +Step 15670/44000 | Loss: 3.0422 | LR: 3.67e-04 | 1329.6ms/step | 15.0 samples/s +Step 15680/44000 | Loss: 3.0376 | LR: 3.67e-04 | 1329.2ms/step | 15.0 samples/s +Step 15690/44000 | Loss: 3.0511 | LR: 3.67e-04 | 1330.3ms/step | 15.0 samples/s +Step 15700/44000 | Loss: 3.0473 | LR: 3.67e-04 | 1328.5ms/step | 15.1 samples/s +Step 15710/44000 | Loss: 3.0344 | LR: 3.66e-04 | 1329.4ms/step | 15.0 samples/s +Step 15720/44000 | Loss: 3.0470 | LR: 3.66e-04 | 1325.0ms/step | 15.1 samples/s +Step 15730/44000 | Loss: 3.0671 | LR: 3.66e-04 | 1327.6ms/step | 15.1 samples/s +Step 15740/44000 | Loss: 3.0649 | LR: 3.66e-04 | 1327.1ms/step | 15.1 samples/s +Step 15750/44000 | Loss: 3.0538 | LR: 3.66e-04 | 1327.9ms/step | 15.1 samples/s +Validation at step 15750: Loss = 3.0430, Perplexity = 20.97 +Step 15760/44000 | Loss: 3.0622 | LR: 3.66e-04 | 3567.0ms/step | 5.6 samples/s +Step 15770/44000 | Loss: 3.0451 | LR: 3.65e-04 | 1329.2ms/step | 15.0 samples/s +Step 15780/44000 | Loss: 3.0480 | LR: 3.65e-04 | 1326.7ms/step | 15.1 samples/s +Step 15790/44000 | Loss: 3.0612 | LR: 3.65e-04 | 1329.3ms/step | 15.0 samples/s +Step 15800/44000 | Loss: 3.0444 | LR: 3.65e-04 | 1324.4ms/step | 15.1 samples/s +Step 15810/44000 | Loss: 3.0710 | LR: 3.65e-04 | 1326.2ms/step | 15.1 samples/s +Step 15820/44000 | Loss: 3.0287 | LR: 3.65e-04 | 1325.2ms/step | 15.1 samples/s +Step 15830/44000 | Loss: 3.0475 | LR: 3.64e-04 | 1322.3ms/step | 15.1 samples/s +Step 15840/44000 | Loss: 3.0385 | LR: 3.64e-04 | 1324.2ms/step | 15.1 samples/s +Step 15850/44000 | Loss: 3.0605 | LR: 3.64e-04 | 1327.4ms/step | 15.1 samples/s +Step 15860/44000 | Loss: 3.0687 | LR: 3.64e-04 | 1326.4ms/step | 15.1 samples/s +Step 15870/44000 | Loss: 3.0511 | LR: 3.64e-04 | 1328.2ms/step | 15.1 samples/s +Step 15880/44000 | Loss: 3.0269 | LR: 3.64e-04 | 1328.8ms/step | 15.1 samples/s +Step 15890/44000 | Loss: 3.0549 | LR: 3.63e-04 | 1328.0ms/step | 15.1 samples/s +Step 15900/44000 | Loss: 3.0311 | LR: 3.63e-04 | 1329.0ms/step | 15.0 samples/s +Step 15910/44000 | Loss: 3.0553 | LR: 3.63e-04 | 1327.7ms/step | 15.1 samples/s +Step 15920/44000 | Loss: 3.0523 | LR: 3.63e-04 | 1326.9ms/step | 15.1 samples/s +Step 15930/44000 | Loss: 3.0543 | LR: 3.63e-04 | 1328.5ms/step | 15.1 samples/s +Step 15940/44000 | Loss: 3.0457 | LR: 3.63e-04 | 1328.3ms/step | 15.1 samples/s +Step 15950/44000 | Loss: 3.0441 | LR: 3.62e-04 | 1328.0ms/step | 15.1 samples/s +Step 15960/44000 | Loss: 3.0787 | LR: 3.62e-04 | 1327.4ms/step | 15.1 samples/s +Step 15970/44000 | Loss: 3.0573 | LR: 3.62e-04 | 1330.8ms/step | 15.0 samples/s +Step 15980/44000 | Loss: 3.0634 | LR: 3.62e-04 | 1328.0ms/step | 15.1 samples/s +Step 15990/44000 | Loss: 3.0330 | LR: 3.62e-04 | 1328.9ms/step | 15.1 samples/s +Step 16000/44000 | Loss: 3.0480 | LR: 3.62e-04 | 1329.4ms/step | 15.0 samples/s +Validation at step 16000: Loss = 3.0389, Perplexity = 20.88 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-16000 +Step 16010/44000 | Loss: 3.0277 | LR: 3.62e-04 | 3867.8ms/step | 5.2 samples/s +Step 16020/44000 | Loss: 3.0230 | LR: 3.61e-04 | 1325.7ms/step | 15.1 samples/s +Step 16030/44000 | Loss: 3.0704 | LR: 3.61e-04 | 1322.9ms/step | 15.1 samples/s +Step 16040/44000 | Loss: 3.0244 | LR: 3.61e-04 | 1324.7ms/step | 15.1 samples/s +Step 16050/44000 | Loss: 3.0556 | LR: 3.61e-04 | 1327.9ms/step | 15.1 samples/s +Step 16060/44000 | Loss: 3.0399 | LR: 3.61e-04 | 1327.5ms/step | 15.1 samples/s +Step 16070/44000 | Loss: 3.0842 | LR: 3.61e-04 | 1326.8ms/step | 15.1 samples/s +Step 16080/44000 | Loss: 3.0345 | LR: 3.60e-04 | 1328.0ms/step | 15.1 samples/s +Step 16090/44000 | Loss: 3.0463 | LR: 3.60e-04 | 1329.3ms/step | 15.0 samples/s +Step 16100/44000 | Loss: 3.0383 | LR: 3.60e-04 | 1328.4ms/step | 15.1 samples/s +Step 16110/44000 | Loss: 3.0341 | LR: 3.60e-04 | 1326.4ms/step | 15.1 samples/s +Step 16120/44000 | Loss: 3.0889 | LR: 3.60e-04 | 1328.5ms/step | 15.1 samples/s +Step 16130/44000 | Loss: 3.0066 | LR: 3.60e-04 | 1327.8ms/step | 15.1 samples/s +Step 16140/44000 | Loss: 3.0271 | LR: 3.59e-04 | 1325.1ms/step | 15.1 samples/s +Step 16150/44000 | Loss: 3.0187 | LR: 3.59e-04 | 1326.4ms/step | 15.1 samples/s +Step 16160/44000 | Loss: 3.0490 | LR: 3.59e-04 | 1326.3ms/step | 15.1 samples/s +Step 16170/44000 | Loss: 3.0668 | LR: 3.59e-04 | 1327.7ms/step | 15.1 samples/s +Step 16180/44000 | Loss: 3.0793 | LR: 3.59e-04 | 1329.9ms/step | 15.0 samples/s +Step 16190/44000 | Loss: 3.0451 | LR: 3.59e-04 | 1326.7ms/step | 15.1 samples/s +Step 16200/44000 | Loss: 3.0547 | LR: 3.58e-04 | 1328.3ms/step | 15.1 samples/s +Step 16210/44000 | Loss: 3.0702 | LR: 3.58e-04 | 1330.3ms/step | 15.0 samples/s +Step 16220/44000 | Loss: 3.0551 | LR: 3.58e-04 | 1328.4ms/step | 15.1 samples/s +Step 16230/44000 | Loss: 3.0835 | LR: 3.58e-04 | 1325.1ms/step | 15.1 samples/s +Step 16240/44000 | Loss: 3.0449 | LR: 3.58e-04 | 1325.2ms/step | 15.1 samples/s +Step 16250/44000 | Loss: 3.0860 | LR: 3.58e-04 | 1323.0ms/step | 15.1 samples/s +Validation at step 16250: Loss = 3.0354, Perplexity = 20.81 +Step 16260/44000 | Loss: 3.0460 | LR: 3.57e-04 | 3554.5ms/step | 5.6 samples/s +Step 16270/44000 | Loss: 3.0275 | LR: 3.57e-04 | 1325.9ms/step | 15.1 samples/s +Step 16280/44000 | Loss: 3.0659 | LR: 3.57e-04 | 1330.6ms/step | 15.0 samples/s +Step 16290/44000 | Loss: 3.0450 | LR: 3.57e-04 | 1328.1ms/step | 15.1 samples/s +Step 16300/44000 | Loss: 3.0440 | LR: 3.57e-04 | 1330.2ms/step | 15.0 samples/s +Step 16310/44000 | Loss: 3.0723 | LR: 3.57e-04 | 1327.3ms/step | 15.1 samples/s +Step 16320/44000 | Loss: 3.0525 | LR: 3.56e-04 | 1326.1ms/step | 15.1 samples/s +Step 16330/44000 | Loss: 3.0256 | LR: 3.56e-04 | 1327.1ms/step | 15.1 samples/s +Step 16340/44000 | Loss: 3.0565 | LR: 3.56e-04 | 1325.5ms/step | 15.1 samples/s +Step 16350/44000 | Loss: 3.0657 | LR: 3.56e-04 | 1323.9ms/step | 15.1 samples/s +Step 16360/44000 | Loss: 3.0544 | LR: 3.56e-04 | 1326.4ms/step | 15.1 samples/s +Step 16370/44000 | Loss: 3.0752 | LR: 3.56e-04 | 1326.0ms/step | 15.1 samples/s +Step 16380/44000 | Loss: 3.0608 | LR: 3.55e-04 | 1326.3ms/step | 15.1 samples/s +Step 16390/44000 | Loss: 3.0494 | LR: 3.55e-04 | 1328.6ms/step | 15.1 samples/s +Step 16400/44000 | Loss: 3.0464 | LR: 3.55e-04 | 1328.1ms/step | 15.1 samples/s +Step 16410/44000 | Loss: 3.0340 | LR: 3.55e-04 | 1327.6ms/step | 15.1 samples/s +Step 16420/44000 | Loss: 3.0488 | LR: 3.55e-04 | 1325.8ms/step | 15.1 samples/s +Step 16430/44000 | Loss: 3.0580 | LR: 3.55e-04 | 1328.0ms/step | 15.1 samples/s +Step 16440/44000 | Loss: 3.0541 | LR: 3.54e-04 | 1325.9ms/step | 15.1 samples/s +Step 16450/44000 | Loss: 3.0468 | LR: 3.54e-04 | 1324.0ms/step | 15.1 samples/s +Step 16460/44000 | Loss: 3.0681 | LR: 3.54e-04 | 1325.2ms/step | 15.1 samples/s +Step 16470/44000 | Loss: 3.0607 | LR: 3.54e-04 | 1328.6ms/step | 15.1 samples/s +Step 16480/44000 | Loss: 3.0236 | LR: 3.54e-04 | 1324.1ms/step | 15.1 samples/s +Step 16490/44000 | Loss: 3.0553 | LR: 3.54e-04 | 1321.9ms/step | 15.1 samples/s +Step 16500/44000 | Loss: 3.0556 | LR: 3.53e-04 | 1321.1ms/step | 15.1 samples/s +Validation at step 16500: Loss = 3.0324, Perplexity = 20.75 +Step 16510/44000 | Loss: 3.0193 | LR: 3.53e-04 | 3553.3ms/step | 5.6 samples/s +Step 16520/44000 | Loss: 3.0163 | LR: 3.53e-04 | 1326.9ms/step | 15.1 samples/s +Step 16530/44000 | Loss: 3.0416 | LR: 3.53e-04 | 1325.8ms/step | 15.1 samples/s +Step 16540/44000 | Loss: 3.0359 | LR: 3.53e-04 | 1321.9ms/step | 15.1 samples/s +Step 16550/44000 | Loss: 3.0356 | LR: 3.53e-04 | 1324.8ms/step | 15.1 samples/s +Step 16560/44000 | Loss: 3.0257 | LR: 3.52e-04 | 1326.8ms/step | 15.1 samples/s +Step 16570/44000 | Loss: 3.0727 | LR: 3.52e-04 | 1326.1ms/step | 15.1 samples/s +Step 16580/44000 | Loss: 3.0109 | LR: 3.52e-04 | 1326.9ms/step | 15.1 samples/s +Step 16590/44000 | Loss: 3.0568 | LR: 3.52e-04 | 1327.1ms/step | 15.1 samples/s +Step 16600/44000 | Loss: 3.0157 | LR: 3.52e-04 | 1325.8ms/step | 15.1 samples/s +Step 16610/44000 | Loss: 3.0668 | LR: 3.52e-04 | 1327.6ms/step | 15.1 samples/s +Step 16620/44000 | Loss: 3.0131 | LR: 3.51e-04 | 1328.3ms/step | 15.1 samples/s +Step 16630/44000 | Loss: 3.0529 | LR: 3.51e-04 | 1324.5ms/step | 15.1 samples/s +Step 16640/44000 | Loss: 3.0504 | LR: 3.51e-04 | 1327.0ms/step | 15.1 samples/s +Step 16650/44000 | Loss: 3.0452 | LR: 3.51e-04 | 1326.8ms/step | 15.1 samples/s +Step 16660/44000 | Loss: 3.0372 | LR: 3.51e-04 | 1325.9ms/step | 15.1 samples/s +Step 16670/44000 | Loss: 3.0099 | LR: 3.51e-04 | 1329.5ms/step | 15.0 samples/s +Step 16680/44000 | Loss: 3.0178 | LR: 3.50e-04 | 1325.7ms/step | 15.1 samples/s +Step 16690/44000 | Loss: 3.0308 | LR: 3.50e-04 | 1322.5ms/step | 15.1 samples/s +Step 16700/44000 | Loss: 3.0132 | LR: 3.50e-04 | 1323.7ms/step | 15.1 samples/s +Step 16710/44000 | Loss: 3.0396 | LR: 3.50e-04 | 1325.4ms/step | 15.1 samples/s +Step 16720/44000 | Loss: 2.9993 | LR: 3.50e-04 | 1326.0ms/step | 15.1 samples/s +Step 16730/44000 | Loss: 3.0608 | LR: 3.50e-04 | 1321.1ms/step | 15.1 samples/s +Step 16740/44000 | Loss: 3.0360 | LR: 3.49e-04 | 1326.2ms/step | 15.1 samples/s +Step 16750/44000 | Loss: 3.0443 | LR: 3.49e-04 | 1324.9ms/step | 15.1 samples/s +Validation at step 16750: Loss = 3.0294, Perplexity = 20.68 +Step 16760/44000 | Loss: 3.0535 | LR: 3.49e-04 | 3557.9ms/step | 5.6 samples/s +Step 16770/44000 | Loss: 3.0620 | LR: 3.49e-04 | 1324.4ms/step | 15.1 samples/s +Step 16780/44000 | Loss: 3.0589 | LR: 3.49e-04 | 1326.6ms/step | 15.1 samples/s +Step 16790/44000 | Loss: 3.0661 | LR: 3.49e-04 | 1327.3ms/step | 15.1 samples/s +Step 16800/44000 | Loss: 3.0488 | LR: 3.48e-04 | 1329.2ms/step | 15.0 samples/s +Step 16810/44000 | Loss: 3.0738 | LR: 3.48e-04 | 1326.8ms/step | 15.1 samples/s +Step 16820/44000 | Loss: 3.0590 | LR: 3.48e-04 | 1326.4ms/step | 15.1 samples/s +Step 16830/44000 | Loss: 3.0512 | LR: 3.48e-04 | 1328.5ms/step | 15.1 samples/s +Step 16840/44000 | Loss: 3.0378 | LR: 3.48e-04 | 1326.7ms/step | 15.1 samples/s +Step 16850/44000 | Loss: 3.0413 | LR: 3.48e-04 | 1323.7ms/step | 15.1 samples/s +Step 16860/44000 | Loss: 3.0332 | LR: 3.47e-04 | 1325.6ms/step | 15.1 samples/s +Step 16870/44000 | Loss: 3.0325 | LR: 3.47e-04 | 1330.4ms/step | 15.0 samples/s +Step 16880/44000 | Loss: 3.0316 | LR: 3.47e-04 | 1328.4ms/step | 15.1 samples/s +Step 16890/44000 | Loss: 3.0466 | LR: 3.47e-04 | 1325.7ms/step | 15.1 samples/s +Step 16900/44000 | Loss: 3.0500 | LR: 3.47e-04 | 1324.9ms/step | 15.1 samples/s +Step 16910/44000 | Loss: 3.0568 | LR: 3.47e-04 | 1325.6ms/step | 15.1 samples/s +Step 16920/44000 | Loss: 3.0414 | LR: 3.46e-04 | 1328.4ms/step | 15.1 samples/s +Step 16930/44000 | Loss: 3.0501 | LR: 3.46e-04 | 1326.1ms/step | 15.1 samples/s +Step 16940/44000 | Loss: 3.0707 | LR: 3.46e-04 | 1329.8ms/step | 15.0 samples/s +Step 16950/44000 | Loss: 3.0637 | LR: 3.46e-04 | 1326.3ms/step | 15.1 samples/s +Step 16960/44000 | Loss: 3.0145 | LR: 3.46e-04 | 1325.1ms/step | 15.1 samples/s +Step 16970/44000 | Loss: 3.0192 | LR: 3.46e-04 | 1327.7ms/step | 15.1 samples/s +Step 16980/44000 | Loss: 3.0440 | LR: 3.45e-04 | 1328.0ms/step | 15.1 samples/s +Step 16990/44000 | Loss: 3.0531 | LR: 3.45e-04 | 1327.7ms/step | 15.1 samples/s +Step 17000/44000 | Loss: 3.0543 | LR: 3.45e-04 | 1329.4ms/step | 15.0 samples/s +Validation at step 17000: Loss = 3.0292, Perplexity = 20.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-17000 +Step 17010/44000 | Loss: 3.0458 | LR: 3.45e-04 | 3875.0ms/step | 5.2 samples/s +Step 17020/44000 | Loss: 3.0364 | LR: 3.45e-04 | 1324.9ms/step | 15.1 samples/s +Step 17030/44000 | Loss: 3.0539 | LR: 3.45e-04 | 1323.9ms/step | 15.1 samples/s +Step 17040/44000 | Loss: 3.0519 | LR: 3.44e-04 | 1325.5ms/step | 15.1 samples/s +Step 17050/44000 | Loss: 3.0736 | LR: 3.44e-04 | 1326.4ms/step | 15.1 samples/s +Step 17060/44000 | Loss: 3.0206 | LR: 3.44e-04 | 1329.0ms/step | 15.0 samples/s +Step 17070/44000 | Loss: 3.0606 | LR: 3.44e-04 | 1329.8ms/step | 15.0 samples/s +Step 17080/44000 | Loss: 3.0749 | LR: 3.44e-04 | 1327.8ms/step | 15.1 samples/s +Step 17090/44000 | Loss: 3.0665 | LR: 3.44e-04 | 1327.8ms/step | 15.1 samples/s +Step 17100/44000 | Loss: 3.0354 | LR: 3.43e-04 | 1328.0ms/step | 15.1 samples/s +Step 17110/44000 | Loss: 3.0175 | LR: 3.43e-04 | 1326.7ms/step | 15.1 samples/s +Step 17120/44000 | Loss: 3.0394 | LR: 3.43e-04 | 1331.1ms/step | 15.0 samples/s +Step 17130/44000 | Loss: 3.0268 | LR: 3.43e-04 | 1327.3ms/step | 15.1 samples/s +Step 17140/44000 | Loss: 3.0438 | LR: 3.43e-04 | 1326.1ms/step | 15.1 samples/s +Step 17150/44000 | Loss: 3.0281 | LR: 3.43e-04 | 1324.8ms/step | 15.1 samples/s +Step 17160/44000 | Loss: 3.0276 | LR: 3.42e-04 | 1327.3ms/step | 15.1 samples/s +Step 17170/44000 | Loss: 3.0498 | LR: 3.42e-04 | 1322.7ms/step | 15.1 samples/s +Step 17180/44000 | Loss: 3.0440 | LR: 3.42e-04 | 1327.7ms/step | 15.1 samples/s +Step 17190/44000 | Loss: 3.0267 | LR: 3.42e-04 | 1327.7ms/step | 15.1 samples/s +Step 17200/44000 | Loss: 3.0473 | LR: 3.42e-04 | 1329.6ms/step | 15.0 samples/s +Step 17210/44000 | Loss: 3.0778 | LR: 3.42e-04 | 1324.8ms/step | 15.1 samples/s +Step 17220/44000 | Loss: 3.0418 | LR: 3.41e-04 | 1326.7ms/step | 15.1 samples/s +Step 17230/44000 | Loss: 3.0365 | LR: 3.41e-04 | 1326.1ms/step | 15.1 samples/s +Step 17240/44000 | Loss: 3.0349 | LR: 3.41e-04 | 1326.1ms/step | 15.1 samples/s +Step 17250/44000 | Loss: 3.0026 | LR: 3.41e-04 | 1327.0ms/step | 15.1 samples/s +Validation at step 17250: Loss = 3.0235, Perplexity = 20.56 +Step 17260/44000 | Loss: 3.0212 | LR: 3.41e-04 | 3551.5ms/step | 5.6 samples/s +Step 17270/44000 | Loss: 3.0225 | LR: 3.41e-04 | 1324.6ms/step | 15.1 samples/s +Step 17280/44000 | Loss: 3.0236 | LR: 3.40e-04 | 1326.3ms/step | 15.1 samples/s +Step 17290/44000 | Loss: 3.0148 | LR: 3.40e-04 | 1328.3ms/step | 15.1 samples/s +Step 17300/44000 | Loss: 3.0106 | LR: 3.40e-04 | 1328.1ms/step | 15.1 samples/s +Step 17310/44000 | Loss: 3.0319 | LR: 3.40e-04 | 1325.0ms/step | 15.1 samples/s +Step 17320/44000 | Loss: 3.0049 | LR: 3.40e-04 | 1326.8ms/step | 15.1 samples/s +Step 17330/44000 | Loss: 3.0310 | LR: 3.40e-04 | 1325.4ms/step | 15.1 samples/s +Step 17340/44000 | Loss: 3.0543 | LR: 3.39e-04 | 1326.1ms/step | 15.1 samples/s +Step 17350/44000 | Loss: 3.0459 | LR: 3.39e-04 | 1328.9ms/step | 15.0 samples/s +Step 17360/44000 | Loss: 3.0630 | LR: 3.39e-04 | 1326.7ms/step | 15.1 samples/s +Step 17370/44000 | Loss: 3.0359 | LR: 3.39e-04 | 1322.8ms/step | 15.1 samples/s +Step 17380/44000 | Loss: 3.0579 | LR: 3.39e-04 | 1325.5ms/step | 15.1 samples/s +Step 17390/44000 | Loss: 3.0108 | LR: 3.39e-04 | 1324.7ms/step | 15.1 samples/s +Step 17400/44000 | Loss: 3.0044 | LR: 3.38e-04 | 1326.2ms/step | 15.1 samples/s +Step 17410/44000 | Loss: 3.0330 | LR: 3.38e-04 | 1326.8ms/step | 15.1 samples/s +Step 17420/44000 | Loss: 3.0551 | LR: 3.38e-04 | 1321.0ms/step | 15.1 samples/s +Step 17430/44000 | Loss: 3.0507 | LR: 3.38e-04 | 1327.2ms/step | 15.1 samples/s +Step 17440/44000 | Loss: 3.0490 | LR: 3.38e-04 | 1325.8ms/step | 15.1 samples/s +Step 17450/44000 | Loss: 3.0307 | LR: 3.38e-04 | 1329.5ms/step | 15.0 samples/s +Step 17460/44000 | Loss: 3.0619 | LR: 3.37e-04 | 1327.6ms/step | 15.1 samples/s +Step 17470/44000 | Loss: 3.0449 | LR: 3.37e-04 | 1329.8ms/step | 15.0 samples/s +Step 17480/44000 | Loss: 3.0307 | LR: 3.37e-04 | 1327.6ms/step | 15.1 samples/s +Step 17490/44000 | Loss: 3.0482 | LR: 3.37e-04 | 1330.1ms/step | 15.0 samples/s +Step 17500/44000 | Loss: 3.0531 | LR: 3.37e-04 | 1332.3ms/step | 15.0 samples/s +Validation at step 17500: Loss = 3.0196, Perplexity = 20.48 +Step 17510/44000 | Loss: 3.0591 | LR: 3.37e-04 | 3564.8ms/step | 5.6 samples/s +Step 17520/44000 | Loss: 3.0542 | LR: 3.36e-04 | 1326.5ms/step | 15.1 samples/s +Step 17530/44000 | Loss: 3.0297 | LR: 3.36e-04 | 1325.4ms/step | 15.1 samples/s +Step 17540/44000 | Loss: 3.0330 | LR: 3.36e-04 | 1327.3ms/step | 15.1 samples/s +Step 17550/44000 | Loss: 3.0366 | LR: 3.36e-04 | 1324.6ms/step | 15.1 samples/s +Step 17560/44000 | Loss: 3.0239 | LR: 3.36e-04 | 1324.1ms/step | 15.1 samples/s +Step 17570/44000 | Loss: 3.0176 | LR: 3.35e-04 | 1325.0ms/step | 15.1 samples/s +Step 17580/44000 | Loss: 3.0482 | LR: 3.35e-04 | 1330.1ms/step | 15.0 samples/s +Step 17590/44000 | Loss: 3.0201 | LR: 3.35e-04 | 1334.4ms/step | 15.0 samples/s +Step 17600/44000 | Loss: 3.0164 | LR: 3.35e-04 | 1327.0ms/step | 15.1 samples/s +Step 17610/44000 | Loss: 3.0669 | LR: 3.35e-04 | 1328.7ms/step | 15.1 samples/s +Step 17620/44000 | Loss: 3.0076 | LR: 3.35e-04 | 1327.2ms/step | 15.1 samples/s +Step 17630/44000 | Loss: 3.0671 | LR: 3.34e-04 | 1326.5ms/step | 15.1 samples/s +Step 17640/44000 | Loss: 3.0322 | LR: 3.34e-04 | 1327.2ms/step | 15.1 samples/s +Step 17650/44000 | Loss: 3.0098 | LR: 3.34e-04 | 1325.5ms/step | 15.1 samples/s +Step 17660/44000 | Loss: 3.0257 | LR: 3.34e-04 | 1326.9ms/step | 15.1 samples/s +Step 17670/44000 | Loss: 3.0336 | LR: 3.34e-04 | 1324.4ms/step | 15.1 samples/s +Step 17680/44000 | Loss: 3.0140 | LR: 3.34e-04 | 1324.4ms/step | 15.1 samples/s +Step 17690/44000 | Loss: 3.0376 | LR: 3.33e-04 | 1325.4ms/step | 15.1 samples/s +Step 17700/44000 | Loss: 3.0417 | LR: 3.33e-04 | 1323.4ms/step | 15.1 samples/s +Step 17710/44000 | Loss: 3.0445 | LR: 3.33e-04 | 1323.0ms/step | 15.1 samples/s +Step 17720/44000 | Loss: 3.0333 | LR: 3.33e-04 | 1326.9ms/step | 15.1 samples/s +Step 17730/44000 | Loss: 3.0102 | LR: 3.33e-04 | 1328.2ms/step | 15.1 samples/s +Step 17740/44000 | Loss: 3.0432 | LR: 3.33e-04 | 1328.2ms/step | 15.1 samples/s +Step 17750/44000 | Loss: 3.0206 | LR: 3.32e-04 | 1328.2ms/step | 15.1 samples/s +Validation at step 17750: Loss = 3.0175, Perplexity = 20.44 +Step 17760/44000 | Loss: 3.0403 | LR: 3.32e-04 | 3556.6ms/step | 5.6 samples/s +Step 17770/44000 | Loss: 3.0310 | LR: 3.32e-04 | 1329.1ms/step | 15.0 samples/s +Step 17780/44000 | Loss: 3.0363 | LR: 3.32e-04 | 1327.9ms/step | 15.1 samples/s +Step 17790/44000 | Loss: 3.0222 | LR: 3.32e-04 | 1327.7ms/step | 15.1 samples/s +Step 17800/44000 | Loss: 3.0455 | LR: 3.32e-04 | 1326.6ms/step | 15.1 samples/s +Step 17810/44000 | Loss: 3.0307 | LR: 3.31e-04 | 1325.5ms/step | 15.1 samples/s +Step 17820/44000 | Loss: 3.0340 | LR: 3.31e-04 | 1331.8ms/step | 15.0 samples/s +Step 17830/44000 | Loss: 3.0041 | LR: 3.31e-04 | 1327.1ms/step | 15.1 samples/s +Step 17840/44000 | Loss: 3.0380 | LR: 3.31e-04 | 1324.8ms/step | 15.1 samples/s +Step 17850/44000 | Loss: 3.0226 | LR: 3.31e-04 | 1319.0ms/step | 15.2 samples/s +Step 17860/44000 | Loss: 3.0444 | LR: 3.31e-04 | 1324.5ms/step | 15.1 samples/s +Step 17870/44000 | Loss: 3.0153 | LR: 3.30e-04 | 1327.4ms/step | 15.1 samples/s +Step 17880/44000 | Loss: 3.0249 | LR: 3.30e-04 | 1327.2ms/step | 15.1 samples/s +Step 17890/44000 | Loss: 3.0500 | LR: 3.30e-04 | 1328.8ms/step | 15.1 samples/s +Step 17900/44000 | Loss: 3.0512 | LR: 3.30e-04 | 1328.7ms/step | 15.1 samples/s +Step 17910/44000 | Loss: 2.9965 | LR: 3.30e-04 | 1327.0ms/step | 15.1 samples/s +Step 17920/44000 | Loss: 3.0040 | LR: 3.29e-04 | 1329.5ms/step | 15.0 samples/s +Step 17930/44000 | Loss: 3.0004 | LR: 3.29e-04 | 1328.1ms/step | 15.1 samples/s +Step 17940/44000 | Loss: 3.0488 | LR: 3.29e-04 | 1329.4ms/step | 15.0 samples/s +Step 17950/44000 | Loss: 3.0285 | LR: 3.29e-04 | 1328.5ms/step | 15.1 samples/s +Step 17960/44000 | Loss: 3.0257 | LR: 3.29e-04 | 1328.6ms/step | 15.1 samples/s +Step 17970/44000 | Loss: 3.0447 | LR: 3.29e-04 | 1331.4ms/step | 15.0 samples/s +Step 17980/44000 | Loss: 3.0257 | LR: 3.28e-04 | 1327.1ms/step | 15.1 samples/s +Step 17990/44000 | Loss: 3.0039 | LR: 3.28e-04 | 1328.3ms/step | 15.1 samples/s +Step 18000/44000 | Loss: 3.0341 | LR: 3.28e-04 | 1325.9ms/step | 15.1 samples/s +Validation at step 18000: Loss = 3.0140, Perplexity = 20.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-18000 +Step 18010/44000 | Loss: 2.9716 | LR: 3.28e-04 | 3891.0ms/step | 5.1 samples/s +Step 18020/44000 | Loss: 3.0214 | LR: 3.28e-04 | 1327.9ms/step | 15.1 samples/s +Step 18030/44000 | Loss: 3.0379 | LR: 3.28e-04 | 1327.2ms/step | 15.1 samples/s +Step 18040/44000 | Loss: 3.0319 | LR: 3.27e-04 | 1326.8ms/step | 15.1 samples/s +Step 18050/44000 | Loss: 3.0152 | LR: 3.27e-04 | 1328.1ms/step | 15.1 samples/s +Step 18060/44000 | Loss: 3.0161 | LR: 3.27e-04 | 1326.0ms/step | 15.1 samples/s +Step 18070/44000 | Loss: 3.0213 | LR: 3.27e-04 | 1326.9ms/step | 15.1 samples/s +Step 18080/44000 | Loss: 2.9892 | LR: 3.27e-04 | 1328.6ms/step | 15.1 samples/s +Step 18090/44000 | Loss: 3.0268 | LR: 3.27e-04 | 1326.8ms/step | 15.1 samples/s +Step 18100/44000 | Loss: 3.0261 | LR: 3.26e-04 | 1328.8ms/step | 15.1 samples/s +Step 18110/44000 | Loss: 3.0199 | LR: 3.26e-04 | 1324.8ms/step | 15.1 samples/s +Step 18120/44000 | Loss: 3.0493 | LR: 3.26e-04 | 1332.4ms/step | 15.0 samples/s +Step 18130/44000 | Loss: 3.0235 | LR: 3.26e-04 | 1330.1ms/step | 15.0 samples/s +Step 18140/44000 | Loss: 3.0032 | LR: 3.26e-04 | 1329.0ms/step | 15.0 samples/s +Step 18150/44000 | Loss: 3.0372 | LR: 3.26e-04 | 1328.6ms/step | 15.1 samples/s +Step 18160/44000 | Loss: 3.0346 | LR: 3.25e-04 | 1329.9ms/step | 15.0 samples/s +Step 18170/44000 | Loss: 3.0447 | LR: 3.25e-04 | 1328.4ms/step | 15.1 samples/s +Step 18180/44000 | Loss: 3.0332 | LR: 3.25e-04 | 1326.6ms/step | 15.1 samples/s +Step 18190/44000 | Loss: 3.0012 | LR: 3.25e-04 | 1323.8ms/step | 15.1 samples/s +Step 18200/44000 | Loss: 2.9915 | LR: 3.25e-04 | 1324.7ms/step | 15.1 samples/s +Step 18210/44000 | Loss: 3.0137 | LR: 3.24e-04 | 1327.0ms/step | 15.1 samples/s +Step 18220/44000 | Loss: 2.9909 | LR: 3.24e-04 | 1330.2ms/step | 15.0 samples/s +Step 18230/44000 | Loss: 3.0200 | LR: 3.24e-04 | 1326.2ms/step | 15.1 samples/s +Step 18240/44000 | Loss: 3.0323 | LR: 3.24e-04 | 1328.6ms/step | 15.1 samples/s +Step 18250/44000 | Loss: 3.0419 | LR: 3.24e-04 | 1328.3ms/step | 15.1 samples/s +Validation at step 18250: Loss = 3.0115, Perplexity = 20.32 +Step 18260/44000 | Loss: 3.0378 | LR: 3.24e-04 | 3565.0ms/step | 5.6 samples/s +Step 18270/44000 | Loss: 3.0436 | LR: 3.23e-04 | 1327.1ms/step | 15.1 samples/s +Step 18280/44000 | Loss: 3.0473 | LR: 3.23e-04 | 1325.3ms/step | 15.1 samples/s +Step 18290/44000 | Loss: 3.0082 | LR: 3.23e-04 | 1324.8ms/step | 15.1 samples/s +Step 18300/44000 | Loss: 3.0058 | LR: 3.23e-04 | 1324.7ms/step | 15.1 samples/s +Step 18310/44000 | Loss: 3.0152 | LR: 3.23e-04 | 1327.4ms/step | 15.1 samples/s +Step 18320/44000 | Loss: 3.0238 | LR: 3.23e-04 | 1328.5ms/step | 15.1 samples/s +Step 18330/44000 | Loss: 3.0211 | LR: 3.22e-04 | 1327.2ms/step | 15.1 samples/s +Step 18340/44000 | Loss: 3.0211 | LR: 3.22e-04 | 1326.2ms/step | 15.1 samples/s +Step 18350/44000 | Loss: 3.0420 | LR: 3.22e-04 | 1326.5ms/step | 15.1 samples/s +Step 18360/44000 | Loss: 3.0071 | LR: 3.22e-04 | 1326.6ms/step | 15.1 samples/s +Step 18370/44000 | Loss: 3.0002 | LR: 3.22e-04 | 1324.6ms/step | 15.1 samples/s +Step 18380/44000 | Loss: 3.0137 | LR: 3.22e-04 | 1323.6ms/step | 15.1 samples/s +Step 18390/44000 | Loss: 3.0177 | LR: 3.21e-04 | 1323.6ms/step | 15.1 samples/s +Step 18400/44000 | Loss: 3.0037 | LR: 3.21e-04 | 1324.7ms/step | 15.1 samples/s +Step 18410/44000 | Loss: 3.0414 | LR: 3.21e-04 | 1326.3ms/step | 15.1 samples/s +Step 18420/44000 | Loss: 3.0272 | LR: 3.21e-04 | 1328.5ms/step | 15.1 samples/s +Step 18430/44000 | Loss: 3.0056 | LR: 3.21e-04 | 1326.7ms/step | 15.1 samples/s +Step 18440/44000 | Loss: 3.0191 | LR: 3.20e-04 | 1327.1ms/step | 15.1 samples/s +Step 18450/44000 | Loss: 3.0291 | LR: 3.20e-04 | 1328.5ms/step | 15.1 samples/s +Step 18460/44000 | Loss: 2.9838 | LR: 3.20e-04 | 1325.3ms/step | 15.1 samples/s +Step 18470/44000 | Loss: 3.0326 | LR: 3.20e-04 | 1324.5ms/step | 15.1 samples/s +Step 18480/44000 | Loss: 3.0249 | LR: 3.20e-04 | 1328.7ms/step | 15.1 samples/s +Step 18490/44000 | Loss: 3.0073 | LR: 3.20e-04 | 1329.1ms/step | 15.0 samples/s +Step 18500/44000 | Loss: 3.0286 | LR: 3.19e-04 | 1322.6ms/step | 15.1 samples/s +Validation at step 18500: Loss = 3.0080, Perplexity = 20.25 +Step 18510/44000 | Loss: 3.0194 | LR: 3.19e-04 | 3551.3ms/step | 5.6 samples/s +Step 18520/44000 | Loss: 3.0334 | LR: 3.19e-04 | 1325.7ms/step | 15.1 samples/s +Step 18530/44000 | Loss: 3.0344 | LR: 3.19e-04 | 1325.3ms/step | 15.1 samples/s +Step 18540/44000 | Loss: 3.0205 | LR: 3.19e-04 | 1323.8ms/step | 15.1 samples/s +Step 18550/44000 | Loss: 3.0003 | LR: 3.19e-04 | 1325.6ms/step | 15.1 samples/s +Step 18560/44000 | Loss: 3.0271 | LR: 3.18e-04 | 1326.7ms/step | 15.1 samples/s +Step 18570/44000 | Loss: 3.0432 | LR: 3.18e-04 | 1324.1ms/step | 15.1 samples/s +Step 18580/44000 | Loss: 3.0223 | LR: 3.18e-04 | 1327.2ms/step | 15.1 samples/s +Step 18590/44000 | Loss: 3.0291 | LR: 3.18e-04 | 1324.9ms/step | 15.1 samples/s +Step 18600/44000 | Loss: 3.0324 | LR: 3.18e-04 | 1325.7ms/step | 15.1 samples/s +Step 18610/44000 | Loss: 3.0374 | LR: 3.18e-04 | 1324.6ms/step | 15.1 samples/s +Step 18620/44000 | Loss: 3.0185 | LR: 3.17e-04 | 1327.0ms/step | 15.1 samples/s +Step 18630/44000 | Loss: 3.0167 | LR: 3.17e-04 | 1325.3ms/step | 15.1 samples/s +Step 18640/44000 | Loss: 3.0036 | LR: 3.17e-04 | 1325.1ms/step | 15.1 samples/s +Step 18650/44000 | Loss: 3.0171 | LR: 3.17e-04 | 1327.5ms/step | 15.1 samples/s +Step 18660/44000 | Loss: 3.0387 | LR: 3.17e-04 | 1324.6ms/step | 15.1 samples/s +Step 18670/44000 | Loss: 3.0319 | LR: 3.16e-04 | 1327.9ms/step | 15.1 samples/s +Step 18680/44000 | Loss: 3.0396 | LR: 3.16e-04 | 1327.0ms/step | 15.1 samples/s +Step 18690/44000 | Loss: 3.0523 | LR: 3.16e-04 | 1324.5ms/step | 15.1 samples/s +Step 18700/44000 | Loss: 3.0430 | LR: 3.16e-04 | 1327.7ms/step | 15.1 samples/s +Step 18710/44000 | Loss: 2.9893 | LR: 3.16e-04 | 1326.3ms/step | 15.1 samples/s +Step 18720/44000 | Loss: 3.0038 | LR: 3.16e-04 | 1328.2ms/step | 15.1 samples/s +Step 18730/44000 | Loss: 2.9938 | LR: 3.15e-04 | 1327.1ms/step | 15.1 samples/s +Step 18740/44000 | Loss: 3.0104 | LR: 3.15e-04 | 1327.7ms/step | 15.1 samples/s +Step 18750/44000 | Loss: 2.9884 | LR: 3.15e-04 | 1357.5ms/step | 14.7 samples/s +Validation at step 18750: Loss = 3.0057, Perplexity = 20.20 +Step 18760/44000 | Loss: 3.0003 | LR: 3.15e-04 | 3570.2ms/step | 5.6 samples/s +Step 18770/44000 | Loss: 3.0080 | LR: 3.15e-04 | 1326.7ms/step | 15.1 samples/s +Step 18780/44000 | Loss: 3.0274 | LR: 3.15e-04 | 1328.0ms/step | 15.1 samples/s +Step 18790/44000 | Loss: 3.0060 | LR: 3.14e-04 | 1326.2ms/step | 15.1 samples/s +Step 18800/44000 | Loss: 3.0333 | LR: 3.14e-04 | 1330.8ms/step | 15.0 samples/s +Step 18810/44000 | Loss: 3.0010 | LR: 3.14e-04 | 1327.5ms/step | 15.1 samples/s +Step 18820/44000 | Loss: 3.0077 | LR: 3.14e-04 | 1327.6ms/step | 15.1 samples/s +Step 18830/44000 | Loss: 3.0274 | LR: 3.14e-04 | 1329.0ms/step | 15.0 samples/s +Step 18840/44000 | Loss: 3.0208 | LR: 3.13e-04 | 1326.9ms/step | 15.1 samples/s +Step 18850/44000 | Loss: 3.0190 | LR: 3.13e-04 | 1325.3ms/step | 15.1 samples/s +Step 18860/44000 | Loss: 3.0058 | LR: 3.13e-04 | 1324.8ms/step | 15.1 samples/s +Step 18870/44000 | Loss: 3.0124 | LR: 3.13e-04 | 1325.5ms/step | 15.1 samples/s +Step 18880/44000 | Loss: 3.0062 | LR: 3.13e-04 | 1325.6ms/step | 15.1 samples/s +Step 18890/44000 | Loss: 3.0201 | LR: 3.13e-04 | 1327.8ms/step | 15.1 samples/s +Step 18900/44000 | Loss: 3.0337 | LR: 3.12e-04 | 1325.0ms/step | 15.1 samples/s +Step 18910/44000 | Loss: 3.0056 | LR: 3.12e-04 | 1328.7ms/step | 15.1 samples/s +Step 18920/44000 | Loss: 3.0195 | LR: 3.12e-04 | 1326.8ms/step | 15.1 samples/s +Step 18930/44000 | Loss: 3.0023 | LR: 3.12e-04 | 1323.7ms/step | 15.1 samples/s +Step 18940/44000 | Loss: 3.0169 | LR: 3.12e-04 | 1330.0ms/step | 15.0 samples/s +Step 18950/44000 | Loss: 3.0285 | LR: 3.12e-04 | 1327.8ms/step | 15.1 samples/s +Step 18960/44000 | Loss: 3.0178 | LR: 3.11e-04 | 1328.8ms/step | 15.1 samples/s +Step 18970/44000 | Loss: 3.0016 | LR: 3.11e-04 | 1325.4ms/step | 15.1 samples/s +Step 18980/44000 | Loss: 3.0067 | LR: 3.11e-04 | 1326.2ms/step | 15.1 samples/s +Step 18990/44000 | Loss: 3.0063 | LR: 3.11e-04 | 1328.6ms/step | 15.1 samples/s +Step 19000/44000 | Loss: 3.0128 | LR: 3.11e-04 | 1329.4ms/step | 15.0 samples/s +Validation at step 19000: Loss = 3.0024, Perplexity = 20.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-19000 +Step 19010/44000 | Loss: 3.0252 | LR: 3.11e-04 | 3869.6ms/step | 5.2 samples/s +Step 19020/44000 | Loss: 3.0036 | LR: 3.10e-04 | 1324.1ms/step | 15.1 samples/s +Step 19030/44000 | Loss: 2.9970 | LR: 3.10e-04 | 1322.8ms/step | 15.1 samples/s +Step 19040/44000 | Loss: 3.0089 | LR: 3.10e-04 | 1326.1ms/step | 15.1 samples/s +Step 19050/44000 | Loss: 3.0141 | LR: 3.10e-04 | 1327.9ms/step | 15.1 samples/s +Step 19060/44000 | Loss: 3.0191 | LR: 3.10e-04 | 1327.2ms/step | 15.1 samples/s +Step 19070/44000 | Loss: 3.0061 | LR: 3.09e-04 | 1329.9ms/step | 15.0 samples/s +Step 19080/44000 | Loss: 3.0064 | LR: 3.09e-04 | 1325.4ms/step | 15.1 samples/s +Step 19090/44000 | Loss: 3.0180 | LR: 3.09e-04 | 1327.9ms/step | 15.1 samples/s +Step 19100/44000 | Loss: 2.9975 | LR: 3.09e-04 | 1326.6ms/step | 15.1 samples/s +Step 19110/44000 | Loss: 3.0460 | LR: 3.09e-04 | 1326.8ms/step | 15.1 samples/s +Step 19120/44000 | Loss: 2.9941 | LR: 3.09e-04 | 1325.9ms/step | 15.1 samples/s +Step 19130/44000 | Loss: 3.0171 | LR: 3.08e-04 | 1326.9ms/step | 15.1 samples/s +Step 19140/44000 | Loss: 3.0174 | LR: 3.08e-04 | 1326.9ms/step | 15.1 samples/s +Step 19150/44000 | Loss: 3.0146 | LR: 3.08e-04 | 1326.7ms/step | 15.1 samples/s +Step 19160/44000 | Loss: 2.9908 | LR: 3.08e-04 | 1326.8ms/step | 15.1 samples/s +Step 19170/44000 | Loss: 3.0433 | LR: 3.08e-04 | 1329.1ms/step | 15.0 samples/s +Step 19180/44000 | Loss: 3.0269 | LR: 3.08e-04 | 1328.4ms/step | 15.1 samples/s +Step 19190/44000 | Loss: 3.0546 | LR: 3.07e-04 | 1327.3ms/step | 15.1 samples/s +Step 19200/44000 | Loss: 3.0215 | LR: 3.07e-04 | 1324.2ms/step | 15.1 samples/s +Step 19210/44000 | Loss: 3.0290 | LR: 3.07e-04 | 1327.4ms/step | 15.1 samples/s +Step 19220/44000 | Loss: 2.9698 | LR: 3.07e-04 | 1329.1ms/step | 15.0 samples/s +Step 19230/44000 | Loss: 3.0518 | LR: 3.07e-04 | 1329.3ms/step | 15.0 samples/s +Step 19240/44000 | Loss: 3.0158 | LR: 3.06e-04 | 1327.4ms/step | 15.1 samples/s +Step 19250/44000 | Loss: 3.0258 | LR: 3.06e-04 | 1329.1ms/step | 15.0 samples/s +Validation at step 19250: Loss = 3.0005, Perplexity = 20.10 +Step 19260/44000 | Loss: 3.0231 | LR: 3.06e-04 | 3557.3ms/step | 5.6 samples/s +Step 19270/44000 | Loss: 3.0135 | LR: 3.06e-04 | 1324.1ms/step | 15.1 samples/s +Step 19280/44000 | Loss: 3.0024 | LR: 3.06e-04 | 1325.6ms/step | 15.1 samples/s +Step 19290/44000 | Loss: 3.0351 | LR: 3.06e-04 | 1330.5ms/step | 15.0 samples/s +Step 19300/44000 | Loss: 2.9600 | LR: 3.05e-04 | 1327.9ms/step | 15.1 samples/s +Step 19310/44000 | Loss: 3.0322 | LR: 3.05e-04 | 1323.9ms/step | 15.1 samples/s +Step 19320/44000 | Loss: 3.0165 | LR: 3.05e-04 | 1325.1ms/step | 15.1 samples/s +Step 19330/44000 | Loss: 3.0060 | LR: 3.05e-04 | 1326.1ms/step | 15.1 samples/s +Step 19340/44000 | Loss: 3.0147 | LR: 3.05e-04 | 1326.3ms/step | 15.1 samples/s +Step 19350/44000 | Loss: 3.0017 | LR: 3.04e-04 | 1323.3ms/step | 15.1 samples/s +Step 19360/44000 | Loss: 2.9971 | LR: 3.04e-04 | 1321.9ms/step | 15.1 samples/s +Step 19370/44000 | Loss: 3.0150 | LR: 3.04e-04 | 1326.0ms/step | 15.1 samples/s +Step 19380/44000 | Loss: 2.9910 | LR: 3.04e-04 | 1322.5ms/step | 15.1 samples/s +Step 19390/44000 | Loss: 2.9893 | LR: 3.04e-04 | 1324.6ms/step | 15.1 samples/s +Step 19400/44000 | Loss: 3.0284 | LR: 3.04e-04 | 1322.8ms/step | 15.1 samples/s +Step 19410/44000 | Loss: 3.0117 | LR: 3.03e-04 | 1325.5ms/step | 15.1 samples/s +Step 19420/44000 | Loss: 2.9847 | LR: 3.03e-04 | 1324.8ms/step | 15.1 samples/s +Step 19430/44000 | Loss: 2.9935 | LR: 3.03e-04 | 1326.7ms/step | 15.1 samples/s +Step 19440/44000 | Loss: 2.9859 | LR: 3.03e-04 | 1328.2ms/step | 15.1 samples/s +Step 19450/44000 | Loss: 3.0268 | LR: 3.03e-04 | 1324.8ms/step | 15.1 samples/s +Step 19460/44000 | Loss: 3.0139 | LR: 3.03e-04 | 1327.2ms/step | 15.1 samples/s +Step 19470/44000 | Loss: 3.0103 | LR: 3.02e-04 | 1326.7ms/step | 15.1 samples/s +Step 19480/44000 | Loss: 3.0207 | LR: 3.02e-04 | 1329.6ms/step | 15.0 samples/s +Step 19490/44000 | Loss: 3.0284 | LR: 3.02e-04 | 1322.8ms/step | 15.1 samples/s +Step 19500/44000 | Loss: 2.9910 | LR: 3.02e-04 | 1324.9ms/step | 15.1 samples/s +Validation at step 19500: Loss = 2.9969, Perplexity = 20.02 +Step 19510/44000 | Loss: 3.0407 | LR: 3.02e-04 | 3553.4ms/step | 5.6 samples/s +Step 19520/44000 | Loss: 3.0388 | LR: 3.01e-04 | 1326.2ms/step | 15.1 samples/s +Step 19530/44000 | Loss: 2.9978 | LR: 3.01e-04 | 1328.1ms/step | 15.1 samples/s +Step 19540/44000 | Loss: 2.9885 | LR: 3.01e-04 | 1327.8ms/step | 15.1 samples/s +Step 19550/44000 | Loss: 3.0251 | LR: 3.01e-04 | 1325.8ms/step | 15.1 samples/s +Step 19560/44000 | Loss: 3.0252 | LR: 3.01e-04 | 1328.5ms/step | 15.1 samples/s +Step 19570/44000 | Loss: 3.0256 | LR: 3.01e-04 | 1325.6ms/step | 15.1 samples/s +Step 19580/44000 | Loss: 3.0166 | LR: 3.00e-04 | 1323.1ms/step | 15.1 samples/s +Step 19590/44000 | Loss: 3.0032 | LR: 3.00e-04 | 1323.1ms/step | 15.1 samples/s +Step 19600/44000 | Loss: 3.0345 | LR: 3.00e-04 | 1323.5ms/step | 15.1 samples/s +Step 19610/44000 | Loss: 3.0044 | LR: 3.00e-04 | 1326.7ms/step | 15.1 samples/s +Step 19620/44000 | Loss: 3.0174 | LR: 3.00e-04 | 1329.9ms/step | 15.0 samples/s +Step 19630/44000 | Loss: 2.9902 | LR: 3.00e-04 | 1333.3ms/step | 15.0 samples/s +Step 19640/44000 | Loss: 2.9874 | LR: 2.99e-04 | 1332.1ms/step | 15.0 samples/s +Step 19650/44000 | Loss: 3.0076 | LR: 2.99e-04 | 1329.6ms/step | 15.0 samples/s +Step 19660/44000 | Loss: 3.0220 | LR: 2.99e-04 | 1324.6ms/step | 15.1 samples/s +Step 19670/44000 | Loss: 3.0397 | LR: 2.99e-04 | 1328.1ms/step | 15.1 samples/s +Step 19680/44000 | Loss: 2.9747 | LR: 2.99e-04 | 1326.8ms/step | 15.1 samples/s +Step 19690/44000 | Loss: 3.0458 | LR: 2.98e-04 | 1325.4ms/step | 15.1 samples/s +Step 19700/44000 | Loss: 2.9990 | LR: 2.98e-04 | 1323.9ms/step | 15.1 samples/s +Step 19710/44000 | Loss: 3.0132 | LR: 2.98e-04 | 1327.0ms/step | 15.1 samples/s +Step 19720/44000 | Loss: 2.9960 | LR: 2.98e-04 | 1324.8ms/step | 15.1 samples/s +Step 19730/44000 | Loss: 3.0077 | LR: 2.98e-04 | 1325.1ms/step | 15.1 samples/s +Step 19740/44000 | Loss: 3.0379 | LR: 2.98e-04 | 1327.6ms/step | 15.1 samples/s +Step 19750/44000 | Loss: 2.9993 | LR: 2.97e-04 | 1326.1ms/step | 15.1 samples/s +Validation at step 19750: Loss = 2.9951, Perplexity = 19.99 +Step 19760/44000 | Loss: 3.0539 | LR: 2.97e-04 | 3560.1ms/step | 5.6 samples/s +Step 19770/44000 | Loss: 3.0414 | LR: 2.97e-04 | 1323.6ms/step | 15.1 samples/s +Step 19780/44000 | Loss: 3.0038 | LR: 2.97e-04 | 1328.4ms/step | 15.1 samples/s +Step 19790/44000 | Loss: 3.0040 | LR: 2.97e-04 | 1327.6ms/step | 15.1 samples/s +Step 19800/44000 | Loss: 3.0388 | LR: 2.96e-04 | 1329.8ms/step | 15.0 samples/s +Step 19810/44000 | Loss: 3.0165 | LR: 2.96e-04 | 1322.4ms/step | 15.1 samples/s +Step 19820/44000 | Loss: 2.9885 | LR: 2.96e-04 | 1326.5ms/step | 15.1 samples/s +Step 19830/44000 | Loss: 3.0157 | LR: 2.96e-04 | 1325.8ms/step | 15.1 samples/s +Step 19840/44000 | Loss: 2.9947 | LR: 2.96e-04 | 1323.5ms/step | 15.1 samples/s +Step 19850/44000 | Loss: 3.0072 | LR: 2.96e-04 | 1325.0ms/step | 15.1 samples/s +Step 19860/44000 | Loss: 3.0008 | LR: 2.95e-04 | 1329.2ms/step | 15.0 samples/s +Step 19870/44000 | Loss: 3.0125 | LR: 2.95e-04 | 1325.3ms/step | 15.1 samples/s +Step 19880/44000 | Loss: 3.0042 | LR: 2.95e-04 | 1328.3ms/step | 15.1 samples/s +Step 19890/44000 | Loss: 3.0067 | LR: 2.95e-04 | 1322.8ms/step | 15.1 samples/s +Step 19900/44000 | Loss: 3.0052 | LR: 2.95e-04 | 1320.0ms/step | 15.2 samples/s +Step 19910/44000 | Loss: 2.9954 | LR: 2.95e-04 | 1322.7ms/step | 15.1 samples/s +Step 19920/44000 | Loss: 3.0041 | LR: 2.94e-04 | 1326.7ms/step | 15.1 samples/s +Step 19930/44000 | Loss: 3.0069 | LR: 2.94e-04 | 1322.1ms/step | 15.1 samples/s +Step 19940/44000 | Loss: 3.0008 | LR: 2.94e-04 | 1324.4ms/step | 15.1 samples/s +Step 19950/44000 | Loss: 3.0164 | LR: 2.94e-04 | 1323.0ms/step | 15.1 samples/s +Step 19960/44000 | Loss: 3.0115 | LR: 2.94e-04 | 1327.1ms/step | 15.1 samples/s +Step 19970/44000 | Loss: 3.0122 | LR: 2.93e-04 | 1328.8ms/step | 15.1 samples/s +Step 19980/44000 | Loss: 3.0283 | LR: 2.93e-04 | 1329.5ms/step | 15.0 samples/s +Step 19990/44000 | Loss: 3.0038 | LR: 2.93e-04 | 1328.0ms/step | 15.1 samples/s +Step 20000/44000 | Loss: 2.9723 | LR: 2.93e-04 | 1330.0ms/step | 15.0 samples/s +Validation at step 20000: Loss = 2.9920, Perplexity = 19.93 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-20000 +Step 20010/44000 | Loss: 2.9996 | LR: 2.93e-04 | 3872.6ms/step | 5.2 samples/s +Step 20020/44000 | Loss: 2.9779 | LR: 2.93e-04 | 1323.4ms/step | 15.1 samples/s +Step 20030/44000 | Loss: 3.0145 | LR: 2.92e-04 | 1322.9ms/step | 15.1 samples/s +Step 20040/44000 | Loss: 3.0129 | LR: 2.92e-04 | 1323.3ms/step | 15.1 samples/s +Step 20050/44000 | Loss: 3.0173 | LR: 2.92e-04 | 1328.8ms/step | 15.1 samples/s +Step 20060/44000 | Loss: 3.0002 | LR: 2.92e-04 | 1328.3ms/step | 15.1 samples/s +Step 20070/44000 | Loss: 3.0028 | LR: 2.92e-04 | 1322.7ms/step | 15.1 samples/s +Step 20080/44000 | Loss: 3.0254 | LR: 2.91e-04 | 1322.0ms/step | 15.1 samples/s +Step 20090/44000 | Loss: 2.9971 | LR: 2.91e-04 | 1321.5ms/step | 15.1 samples/s +Step 20100/44000 | Loss: 3.0286 | LR: 2.91e-04 | 1320.4ms/step | 15.1 samples/s +Step 20110/44000 | Loss: 2.9882 | LR: 2.91e-04 | 1317.8ms/step | 15.2 samples/s diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_static_nrb35_seed42.log b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_static_nrb35_seed42.log new file mode 100644 index 0000000000000000000000000000000000000000..de20caf980266baf03eaf9e9dded4eb38d410b99 --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_static_nrb35_seed42.log @@ -0,0 +1,2293 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 365m static 8k nrb35 seed42", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "static", + "nrb35", + "seed42" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1", + "sparse_mask_seed": 1, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run bsc91ke5 +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_164441-bsc91ke5 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 365m static 8k nrb35 seed42 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/bsc91ke5 +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0399 | LR: 6.67e-06 | 6772.5ms/step | 3.0 samples/s +Step 20/44000 | Loss: 11.9200 | LR: 1.33e-05 | 1308.2ms/step | 15.3 samples/s +Step 30/44000 | Loss: 11.5414 | LR: 2.00e-05 | 1309.5ms/step | 15.3 samples/s +Step 40/44000 | Loss: 11.0036 | LR: 2.67e-05 | 1314.7ms/step | 15.2 samples/s +Step 50/44000 | Loss: 10.7300 | LR: 3.33e-05 | 1307.5ms/step | 15.3 samples/s +Step 60/44000 | Loss: 10.4882 | LR: 4.00e-05 | 1302.6ms/step | 15.4 samples/s +Step 70/44000 | Loss: 10.2518 | LR: 4.67e-05 | 1308.6ms/step | 15.3 samples/s +Step 80/44000 | Loss: 9.9466 | LR: 5.33e-05 | 1309.2ms/step | 15.3 samples/s +Step 90/44000 | Loss: 9.5820 | LR: 6.00e-05 | 1311.7ms/step | 15.2 samples/s +Step 100/44000 | Loss: 9.1267 | LR: 6.67e-05 | 1315.7ms/step | 15.2 samples/s +Step 110/44000 | Loss: 8.5966 | LR: 7.33e-05 | 1319.4ms/step | 15.2 samples/s +Step 120/44000 | Loss: 8.1477 | LR: 8.00e-05 | 1314.8ms/step | 15.2 samples/s +Step 130/44000 | Loss: 7.7674 | LR: 8.67e-05 | 1317.0ms/step | 15.2 samples/s +Step 140/44000 | Loss: 7.5316 | LR: 9.33e-05 | 1320.3ms/step | 15.1 samples/s +Step 150/44000 | Loss: 7.3460 | LR: 1.00e-04 | 1315.7ms/step | 15.2 samples/s +Step 160/44000 | Loss: 7.2472 | LR: 1.07e-04 | 1323.3ms/step | 15.1 samples/s +Step 170/44000 | Loss: 7.1354 | LR: 1.13e-04 | 1326.6ms/step | 15.1 samples/s +Step 180/44000 | Loss: 7.0376 | LR: 1.20e-04 | 1320.9ms/step | 15.1 samples/s +Step 190/44000 | Loss: 6.9318 | LR: 1.27e-04 | 1326.1ms/step | 15.1 samples/s +Step 200/44000 | Loss: 6.8699 | LR: 1.33e-04 | 1329.0ms/step | 15.0 samples/s +Step 210/44000 | Loss: 6.8044 | LR: 1.40e-04 | 1324.3ms/step | 15.1 samples/s +Step 220/44000 | Loss: 6.7011 | LR: 1.47e-04 | 1325.4ms/step | 15.1 samples/s +Step 230/44000 | Loss: 6.5957 | LR: 1.53e-04 | 1331.4ms/step | 15.0 samples/s +Step 240/44000 | Loss: 6.5371 | LR: 1.60e-04 | 1322.4ms/step | 15.1 samples/s +Step 250/44000 | Loss: 6.4556 | LR: 1.67e-04 | 1327.3ms/step | 15.1 samples/s +Validation at step 250: Loss = 6.4400, Perplexity = 626.41 +Step 260/44000 | Loss: 6.4240 | LR: 1.73e-04 | 5793.5ms/step | 3.5 samples/s +Step 270/44000 | Loss: 6.3502 | LR: 1.80e-04 | 1322.9ms/step | 15.1 samples/s +Step 280/44000 | Loss: 6.3135 | LR: 1.87e-04 | 1324.0ms/step | 15.1 samples/s +Step 290/44000 | Loss: 6.2814 | LR: 1.93e-04 | 1328.4ms/step | 15.1 samples/s +Step 300/44000 | Loss: 6.2199 | LR: 2.00e-04 | 1322.3ms/step | 15.1 samples/s +Step 310/44000 | Loss: 6.1857 | LR: 2.07e-04 | 1322.7ms/step | 15.1 samples/s +Step 320/44000 | Loss: 6.1379 | LR: 2.13e-04 | 1335.7ms/step | 15.0 samples/s +Step 330/44000 | Loss: 6.0833 | LR: 2.20e-04 | 1322.3ms/step | 15.1 samples/s +Step 340/44000 | Loss: 6.0657 | LR: 2.27e-04 | 1321.6ms/step | 15.1 samples/s +Step 350/44000 | Loss: 5.9907 | LR: 2.33e-04 | 1336.7ms/step | 15.0 samples/s +Step 360/44000 | Loss: 5.9625 | LR: 2.40e-04 | 1324.0ms/step | 15.1 samples/s +Step 370/44000 | Loss: 5.9402 | LR: 2.47e-04 | 1324.5ms/step | 15.1 samples/s +Step 380/44000 | Loss: 5.9206 | LR: 2.53e-04 | 1338.1ms/step | 14.9 samples/s +Step 390/44000 | Loss: 5.8334 | LR: 2.60e-04 | 1324.8ms/step | 15.1 samples/s +Step 400/44000 | Loss: 5.8436 | LR: 2.67e-04 | 1329.1ms/step | 15.0 samples/s +Step 410/44000 | Loss: 5.8092 | LR: 2.73e-04 | 1339.2ms/step | 14.9 samples/s +Step 420/44000 | Loss: 5.8097 | LR: 2.80e-04 | 1323.4ms/step | 15.1 samples/s +Step 430/44000 | Loss: 5.7360 | LR: 2.87e-04 | 1323.9ms/step | 15.1 samples/s +Step 440/44000 | Loss: 5.7096 | LR: 2.93e-04 | 1336.8ms/step | 15.0 samples/s +Step 450/44000 | Loss: 5.6748 | LR: 3.00e-04 | 1324.8ms/step | 15.1 samples/s +Step 460/44000 | Loss: 5.6083 | LR: 3.07e-04 | 1326.7ms/step | 15.1 samples/s +Step 470/44000 | Loss: 5.6215 | LR: 3.13e-04 | 1335.3ms/step | 15.0 samples/s +Step 480/44000 | Loss: 5.5690 | LR: 3.20e-04 | 1327.0ms/step | 15.1 samples/s +Step 490/44000 | Loss: 5.5223 | LR: 3.27e-04 | 1326.9ms/step | 15.1 samples/s +Step 500/44000 | Loss: 5.5023 | LR: 3.33e-04 | 1337.7ms/step | 15.0 samples/s +Validation at step 500: Loss = 5.4881, Perplexity = 241.79 +Step 510/44000 | Loss: 5.4690 | LR: 3.40e-04 | 3560.2ms/step | 5.6 samples/s +Step 520/44000 | Loss: 5.4186 | LR: 3.47e-04 | 1332.5ms/step | 15.0 samples/s +Step 530/44000 | Loss: 5.4361 | LR: 3.53e-04 | 1325.4ms/step | 15.1 samples/s +Step 540/44000 | Loss: 5.3632 | LR: 3.60e-04 | 1331.2ms/step | 15.0 samples/s +Step 550/44000 | Loss: 5.3352 | LR: 3.67e-04 | 1336.4ms/step | 15.0 samples/s +Step 560/44000 | Loss: 5.3113 | LR: 3.73e-04 | 1327.8ms/step | 15.1 samples/s +Step 570/44000 | Loss: 5.2953 | LR: 3.80e-04 | 1331.5ms/step | 15.0 samples/s +Step 580/44000 | Loss: 5.2644 | LR: 3.87e-04 | 1335.4ms/step | 15.0 samples/s +Step 590/44000 | Loss: 5.2201 | LR: 3.93e-04 | 1325.4ms/step | 15.1 samples/s +Step 600/44000 | Loss: 5.2070 | LR: 4.00e-04 | 1323.6ms/step | 15.1 samples/s +Step 610/44000 | Loss: 5.1763 | LR: 4.07e-04 | 1335.6ms/step | 15.0 samples/s +Step 620/44000 | Loss: 5.1470 | LR: 4.13e-04 | 1327.7ms/step | 15.1 samples/s +Step 630/44000 | Loss: 5.0652 | LR: 4.20e-04 | 1328.2ms/step | 15.1 samples/s +Step 640/44000 | Loss: 5.0590 | LR: 4.27e-04 | 1337.8ms/step | 14.9 samples/s +Step 650/44000 | Loss: 5.0470 | LR: 4.33e-04 | 1327.5ms/step | 15.1 samples/s +Step 660/44000 | Loss: 5.0150 | LR: 4.40e-04 | 1324.9ms/step | 15.1 samples/s +Step 670/44000 | Loss: 4.9823 | LR: 4.47e-04 | 1334.1ms/step | 15.0 samples/s +Step 680/44000 | Loss: 4.9422 | LR: 4.53e-04 | 1327.7ms/step | 15.1 samples/s +Step 690/44000 | Loss: 4.9168 | LR: 4.60e-04 | 1323.1ms/step | 15.1 samples/s +Step 700/44000 | Loss: 4.9123 | LR: 4.67e-04 | 1330.6ms/step | 15.0 samples/s +Step 710/44000 | Loss: 4.8704 | LR: 4.73e-04 | 1323.7ms/step | 15.1 samples/s +Step 720/44000 | Loss: 4.8081 | LR: 4.80e-04 | 1328.7ms/step | 15.1 samples/s +Step 730/44000 | Loss: 4.8067 | LR: 4.87e-04 | 1335.5ms/step | 15.0 samples/s +Step 740/44000 | Loss: 4.7493 | LR: 4.93e-04 | 1323.1ms/step | 15.1 samples/s +Step 750/44000 | Loss: 4.7737 | LR: 5.00e-04 | 1323.5ms/step | 15.1 samples/s +Validation at step 750: Loss = 4.7642, Perplexity = 117.24 +Step 760/44000 | Loss: 4.7478 | LR: 5.00e-04 | 3572.3ms/step | 5.6 samples/s +Step 770/44000 | Loss: 4.6949 | LR: 5.00e-04 | 1337.7ms/step | 15.0 samples/s +Step 780/44000 | Loss: 4.7003 | LR: 5.00e-04 | 1324.6ms/step | 15.1 samples/s +Step 790/44000 | Loss: 4.6657 | LR: 5.00e-04 | 1327.8ms/step | 15.1 samples/s +Step 800/44000 | Loss: 4.6225 | LR: 5.00e-04 | 1333.3ms/step | 15.0 samples/s +Step 810/44000 | Loss: 4.6252 | LR: 5.00e-04 | 1330.3ms/step | 15.0 samples/s +Step 820/44000 | Loss: 4.5899 | LR: 5.00e-04 | 1325.7ms/step | 15.1 samples/s +Step 830/44000 | Loss: 4.5674 | LR: 5.00e-04 | 1337.6ms/step | 15.0 samples/s +Step 840/44000 | Loss: 4.5431 | LR: 5.00e-04 | 1333.4ms/step | 15.0 samples/s +Step 850/44000 | Loss: 4.5360 | LR: 5.00e-04 | 1329.7ms/step | 15.0 samples/s +Step 860/44000 | Loss: 4.5578 | LR: 5.00e-04 | 1337.6ms/step | 15.0 samples/s +Step 870/44000 | Loss: 4.4805 | LR: 5.00e-04 | 1331.2ms/step | 15.0 samples/s +Step 880/44000 | Loss: 4.4902 | LR: 5.00e-04 | 1328.3ms/step | 15.1 samples/s +Step 890/44000 | Loss: 4.4648 | LR: 5.00e-04 | 1339.3ms/step | 14.9 samples/s +Step 900/44000 | Loss: 4.4392 | LR: 5.00e-04 | 1333.7ms/step | 15.0 samples/s +Step 910/44000 | Loss: 4.4108 | LR: 5.00e-04 | 1330.2ms/step | 15.0 samples/s +Step 920/44000 | Loss: 4.4130 | LR: 5.00e-04 | 1338.5ms/step | 14.9 samples/s +Step 930/44000 | Loss: 4.3854 | LR: 5.00e-04 | 1336.0ms/step | 15.0 samples/s +Step 940/44000 | Loss: 4.3780 | LR: 5.00e-04 | 1332.7ms/step | 15.0 samples/s +Step 950/44000 | Loss: 4.3654 | LR: 5.00e-04 | 1339.4ms/step | 14.9 samples/s +Step 960/44000 | Loss: 4.3278 | LR: 5.00e-04 | 1337.2ms/step | 15.0 samples/s +Step 970/44000 | Loss: 4.3598 | LR: 5.00e-04 | 1333.9ms/step | 15.0 samples/s +Step 980/44000 | Loss: 4.3371 | LR: 5.00e-04 | 1338.4ms/step | 14.9 samples/s +Step 990/44000 | Loss: 4.3116 | LR: 5.00e-04 | 1337.7ms/step | 15.0 samples/s +Step 1000/44000 | Loss: 4.3105 | LR: 5.00e-04 | 1329.8ms/step | 15.0 samples/s +Validation at step 1000: Loss = 4.3209, Perplexity = 75.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-1000 +Step 1010/44000 | Loss: 4.2771 | LR: 5.00e-04 | 3884.6ms/step | 5.1 samples/s +Step 1020/44000 | Loss: 4.2810 | LR: 5.00e-04 | 1327.3ms/step | 15.1 samples/s +Step 1030/44000 | Loss: 4.2293 | LR: 5.00e-04 | 1338.6ms/step | 14.9 samples/s +Step 1040/44000 | Loss: 4.2400 | LR: 5.00e-04 | 1331.0ms/step | 15.0 samples/s +Step 1050/44000 | Loss: 4.2378 | LR: 5.00e-04 | 1325.9ms/step | 15.1 samples/s +Step 1060/44000 | Loss: 4.2286 | LR: 5.00e-04 | 1333.9ms/step | 15.0 samples/s +Step 1070/44000 | Loss: 4.2151 | LR: 5.00e-04 | 1329.0ms/step | 15.0 samples/s +Step 1080/44000 | Loss: 4.2027 | LR: 5.00e-04 | 1327.5ms/step | 15.1 samples/s +Step 1090/44000 | Loss: 4.2222 | LR: 5.00e-04 | 1328.2ms/step | 15.1 samples/s +Step 1100/44000 | Loss: 4.1953 | LR: 5.00e-04 | 1329.2ms/step | 15.0 samples/s +Step 1110/44000 | Loss: 4.1783 | LR: 5.00e-04 | 1330.9ms/step | 15.0 samples/s +Step 1120/44000 | Loss: 4.1613 | LR: 5.00e-04 | 1329.7ms/step | 15.0 samples/s +Step 1130/44000 | Loss: 4.1368 | LR: 5.00e-04 | 1332.3ms/step | 15.0 samples/s +Step 1140/44000 | Loss: 4.1590 | LR: 5.00e-04 | 1344.1ms/step | 14.9 samples/s +Step 1150/44000 | Loss: 4.1095 | LR: 5.00e-04 | 1330.9ms/step | 15.0 samples/s +Step 1160/44000 | Loss: 4.1107 | LR: 5.00e-04 | 1330.0ms/step | 15.0 samples/s +Step 1170/44000 | Loss: 4.1167 | LR: 5.00e-04 | 1343.5ms/step | 14.9 samples/s +Step 1180/44000 | Loss: 4.0956 | LR: 5.00e-04 | 1329.1ms/step | 15.0 samples/s +Step 1190/44000 | Loss: 4.1208 | LR: 5.00e-04 | 1331.7ms/step | 15.0 samples/s +Step 1200/44000 | Loss: 4.1158 | LR: 5.00e-04 | 1341.1ms/step | 14.9 samples/s +Step 1210/44000 | Loss: 4.1210 | LR: 5.00e-04 | 1329.2ms/step | 15.0 samples/s +Step 1220/44000 | Loss: 4.0712 | LR: 5.00e-04 | 1329.5ms/step | 15.0 samples/s +Step 1230/44000 | Loss: 4.0628 | LR: 5.00e-04 | 1341.9ms/step | 14.9 samples/s +Step 1240/44000 | Loss: 4.0668 | LR: 5.00e-04 | 1330.7ms/step | 15.0 samples/s +Step 1250/44000 | Loss: 4.0707 | LR: 5.00e-04 | 1332.0ms/step | 15.0 samples/s +Validation at step 1250: Loss = 4.0635, Perplexity = 58.18 +Step 1260/44000 | Loss: 4.0396 | LR: 5.00e-04 | 3576.9ms/step | 5.6 samples/s +Step 1270/44000 | Loss: 4.0381 | LR: 5.00e-04 | 1342.0ms/step | 14.9 samples/s +Step 1280/44000 | Loss: 4.0498 | LR: 5.00e-04 | 1334.1ms/step | 15.0 samples/s +Step 1290/44000 | Loss: 3.9955 | LR: 5.00e-04 | 1334.7ms/step | 15.0 samples/s +Step 1300/44000 | Loss: 4.0450 | LR: 5.00e-04 | 1337.4ms/step | 15.0 samples/s +Step 1310/44000 | Loss: 3.9843 | LR: 5.00e-04 | 1328.5ms/step | 15.1 samples/s +Step 1320/44000 | Loss: 4.0136 | LR: 5.00e-04 | 1322.5ms/step | 15.1 samples/s +Step 1330/44000 | Loss: 4.0178 | LR: 5.00e-04 | 1328.6ms/step | 15.1 samples/s +Step 1340/44000 | Loss: 3.9806 | LR: 5.00e-04 | 1332.4ms/step | 15.0 samples/s +Step 1350/44000 | Loss: 3.9830 | LR: 5.00e-04 | 1328.6ms/step | 15.1 samples/s +Step 1360/44000 | Loss: 3.9768 | LR: 5.00e-04 | 1332.7ms/step | 15.0 samples/s +Step 1370/44000 | Loss: 3.9564 | LR: 5.00e-04 | 1332.0ms/step | 15.0 samples/s +Step 1380/44000 | Loss: 3.9681 | LR: 5.00e-04 | 1325.9ms/step | 15.1 samples/s +Step 1390/44000 | Loss: 3.9696 | LR: 5.00e-04 | 1329.0ms/step | 15.0 samples/s +Step 1400/44000 | Loss: 3.9633 | LR: 5.00e-04 | 1331.2ms/step | 15.0 samples/s +Step 1410/44000 | Loss: 3.9603 | LR: 5.00e-04 | 1325.5ms/step | 15.1 samples/s +Step 1420/44000 | Loss: 3.9498 | LR: 5.00e-04 | 1330.0ms/step | 15.0 samples/s +Step 1430/44000 | Loss: 3.9638 | LR: 5.00e-04 | 1330.2ms/step | 15.0 samples/s +Step 1440/44000 | Loss: 3.9592 | LR: 5.00e-04 | 1326.5ms/step | 15.1 samples/s +Step 1450/44000 | Loss: 3.9449 | LR: 5.00e-04 | 1329.4ms/step | 15.0 samples/s +Step 1460/44000 | Loss: 3.9420 | LR: 5.00e-04 | 1330.5ms/step | 15.0 samples/s +Step 1470/44000 | Loss: 3.9513 | LR: 5.00e-04 | 1327.1ms/step | 15.1 samples/s +Step 1480/44000 | Loss: 3.9422 | LR: 5.00e-04 | 1329.0ms/step | 15.0 samples/s +Step 1490/44000 | Loss: 3.9395 | LR: 5.00e-04 | 1329.2ms/step | 15.0 samples/s +Step 1500/44000 | Loss: 3.9177 | LR: 5.00e-04 | 1322.2ms/step | 15.1 samples/s +Validation at step 1500: Loss = 3.9161, Perplexity = 50.20 +Step 1510/44000 | Loss: 3.9285 | LR: 5.00e-04 | 3567.5ms/step | 5.6 samples/s +Step 1520/44000 | Loss: 3.9289 | LR: 5.00e-04 | 1323.8ms/step | 15.1 samples/s +Step 1530/44000 | Loss: 3.8981 | LR: 5.00e-04 | 1330.0ms/step | 15.0 samples/s +Step 1540/44000 | Loss: 3.9101 | LR: 5.00e-04 | 1320.8ms/step | 15.1 samples/s +Step 1550/44000 | Loss: 3.8731 | LR: 5.00e-04 | 1322.6ms/step | 15.1 samples/s +Step 1560/44000 | Loss: 3.8650 | LR: 5.00e-04 | 1334.1ms/step | 15.0 samples/s +Step 1570/44000 | Loss: 3.8580 | LR: 5.00e-04 | 1323.4ms/step | 15.1 samples/s +Step 1580/44000 | Loss: 3.8792 | LR: 5.00e-04 | 1324.4ms/step | 15.1 samples/s +Step 1590/44000 | Loss: 3.8527 | LR: 5.00e-04 | 1334.0ms/step | 15.0 samples/s +Step 1600/44000 | Loss: 3.8593 | LR: 5.00e-04 | 1326.2ms/step | 15.1 samples/s +Step 1610/44000 | Loss: 3.8402 | LR: 5.00e-04 | 1330.0ms/step | 15.0 samples/s +Step 1620/44000 | Loss: 3.8240 | LR: 5.00e-04 | 1337.9ms/step | 14.9 samples/s +Step 1630/44000 | Loss: 3.8399 | LR: 4.99e-04 | 1326.7ms/step | 15.1 samples/s +Step 1640/44000 | Loss: 3.8436 | LR: 4.99e-04 | 1327.1ms/step | 15.1 samples/s +Step 1650/44000 | Loss: 3.8179 | LR: 4.99e-04 | 1338.2ms/step | 14.9 samples/s +Step 1660/44000 | Loss: 3.8229 | LR: 4.99e-04 | 1326.0ms/step | 15.1 samples/s +Step 1670/44000 | Loss: 3.8147 | LR: 4.99e-04 | 1324.5ms/step | 15.1 samples/s +Step 1680/44000 | Loss: 3.8215 | LR: 4.99e-04 | 1337.1ms/step | 15.0 samples/s +Step 1690/44000 | Loss: 3.8061 | LR: 4.99e-04 | 1327.0ms/step | 15.1 samples/s +Step 1700/44000 | Loss: 3.8453 | LR: 4.99e-04 | 1327.4ms/step | 15.1 samples/s +Step 1710/44000 | Loss: 3.7936 | LR: 4.99e-04 | 1337.2ms/step | 15.0 samples/s +Step 1720/44000 | Loss: 3.8030 | LR: 4.99e-04 | 1326.4ms/step | 15.1 samples/s +Step 1730/44000 | Loss: 3.8054 | LR: 4.99e-04 | 1329.1ms/step | 15.0 samples/s +Step 1740/44000 | Loss: 3.7714 | LR: 4.99e-04 | 1337.4ms/step | 15.0 samples/s +Step 1750/44000 | Loss: 3.7905 | LR: 4.99e-04 | 1327.3ms/step | 15.1 samples/s +Validation at step 1750: Loss = 3.8073, Perplexity = 45.03 +Step 1760/44000 | Loss: 3.7753 | LR: 4.99e-04 | 3563.9ms/step | 5.6 samples/s +Step 1770/44000 | Loss: 3.7739 | LR: 4.99e-04 | 1325.3ms/step | 15.1 samples/s +Step 1780/44000 | Loss: 3.8094 | LR: 4.99e-04 | 1331.6ms/step | 15.0 samples/s +Step 1790/44000 | Loss: 3.7726 | LR: 4.99e-04 | 1327.7ms/step | 15.1 samples/s +Step 1800/44000 | Loss: 3.7701 | LR: 4.99e-04 | 1327.7ms/step | 15.1 samples/s +Step 1810/44000 | Loss: 3.7875 | LR: 4.99e-04 | 1336.7ms/step | 15.0 samples/s +Step 1820/44000 | Loss: 3.7762 | LR: 4.99e-04 | 1330.7ms/step | 15.0 samples/s +Step 1830/44000 | Loss: 3.7635 | LR: 4.99e-04 | 1330.8ms/step | 15.0 samples/s +Step 1840/44000 | Loss: 3.7507 | LR: 4.99e-04 | 1339.4ms/step | 14.9 samples/s +Step 1850/44000 | Loss: 3.7781 | LR: 4.99e-04 | 1331.7ms/step | 15.0 samples/s +Step 1860/44000 | Loss: 3.7626 | LR: 4.99e-04 | 1331.9ms/step | 15.0 samples/s +Step 1870/44000 | Loss: 3.7661 | LR: 4.99e-04 | 1342.0ms/step | 14.9 samples/s +Step 1880/44000 | Loss: 3.7666 | LR: 4.99e-04 | 1328.4ms/step | 15.1 samples/s +Step 1890/44000 | Loss: 3.7602 | LR: 4.99e-04 | 1329.9ms/step | 15.0 samples/s +Step 1900/44000 | Loss: 3.7692 | LR: 4.99e-04 | 1341.0ms/step | 14.9 samples/s +Step 1910/44000 | Loss: 3.7394 | LR: 4.99e-04 | 1328.2ms/step | 15.1 samples/s +Step 1920/44000 | Loss: 3.7593 | LR: 4.99e-04 | 1329.4ms/step | 15.0 samples/s +Step 1930/44000 | Loss: 3.7505 | LR: 4.99e-04 | 1340.1ms/step | 14.9 samples/s +Step 1940/44000 | Loss: 3.7460 | LR: 4.99e-04 | 1334.4ms/step | 15.0 samples/s +Step 1950/44000 | Loss: 3.7431 | LR: 4.99e-04 | 1327.4ms/step | 15.1 samples/s +Step 1960/44000 | Loss: 3.7303 | LR: 4.99e-04 | 1338.1ms/step | 14.9 samples/s +Step 1970/44000 | Loss: 3.7357 | LR: 4.99e-04 | 1329.8ms/step | 15.0 samples/s +Step 1980/44000 | Loss: 3.7112 | LR: 4.99e-04 | 1332.4ms/step | 15.0 samples/s +Step 1990/44000 | Loss: 3.6958 | LR: 4.99e-04 | 1342.0ms/step | 14.9 samples/s +Step 2000/44000 | Loss: 3.7054 | LR: 4.99e-04 | 1332.3ms/step | 15.0 samples/s +Validation at step 2000: Loss = 3.7223, Perplexity = 41.36 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-2000 +Step 2010/44000 | Loss: 3.6838 | LR: 4.99e-04 | 3929.9ms/step | 5.1 samples/s +Step 2020/44000 | Loss: 3.7425 | LR: 4.99e-04 | 1324.6ms/step | 15.1 samples/s +Step 2030/44000 | Loss: 3.7167 | LR: 4.99e-04 | 1331.9ms/step | 15.0 samples/s +Step 2040/44000 | Loss: 3.7036 | LR: 4.99e-04 | 1331.0ms/step | 15.0 samples/s +Step 2050/44000 | Loss: 3.6930 | LR: 4.99e-04 | 1330.9ms/step | 15.0 samples/s +Step 2060/44000 | Loss: 3.6652 | LR: 4.99e-04 | 1329.8ms/step | 15.0 samples/s +Step 2070/44000 | Loss: 3.6857 | LR: 4.99e-04 | 1328.4ms/step | 15.1 samples/s +Step 2080/44000 | Loss: 3.6567 | LR: 4.99e-04 | 1331.1ms/step | 15.0 samples/s +Step 2090/44000 | Loss: 3.6953 | LR: 4.99e-04 | 1330.8ms/step | 15.0 samples/s +Step 2100/44000 | Loss: 3.6870 | LR: 4.99e-04 | 1329.9ms/step | 15.0 samples/s +Step 2110/44000 | Loss: 3.7030 | LR: 4.99e-04 | 1331.0ms/step | 15.0 samples/s +Step 2120/44000 | Loss: 3.6899 | LR: 4.99e-04 | 1329.0ms/step | 15.0 samples/s +Step 2130/44000 | Loss: 3.6983 | LR: 4.99e-04 | 1330.8ms/step | 15.0 samples/s +Step 2140/44000 | Loss: 3.6493 | LR: 4.99e-04 | 1332.5ms/step | 15.0 samples/s +Step 2150/44000 | Loss: 3.6630 | LR: 4.99e-04 | 1332.1ms/step | 15.0 samples/s +Step 2160/44000 | Loss: 3.6709 | LR: 4.99e-04 | 1331.2ms/step | 15.0 samples/s +Step 2170/44000 | Loss: 3.6725 | LR: 4.99e-04 | 1332.4ms/step | 15.0 samples/s +Step 2180/44000 | Loss: 3.6572 | LR: 4.99e-04 | 1334.6ms/step | 15.0 samples/s +Step 2190/44000 | Loss: 3.6650 | LR: 4.99e-04 | 1331.5ms/step | 15.0 samples/s +Step 2200/44000 | Loss: 3.6406 | LR: 4.99e-04 | 1333.5ms/step | 15.0 samples/s +Step 2210/44000 | Loss: 3.6436 | LR: 4.99e-04 | 1335.0ms/step | 15.0 samples/s +Step 2220/44000 | Loss: 3.6228 | LR: 4.99e-04 | 1332.6ms/step | 15.0 samples/s +Step 2230/44000 | Loss: 3.6678 | LR: 4.99e-04 | 1333.3ms/step | 15.0 samples/s +Step 2240/44000 | Loss: 3.6468 | LR: 4.99e-04 | 1332.0ms/step | 15.0 samples/s +Step 2250/44000 | Loss: 3.6484 | LR: 4.99e-04 | 1330.7ms/step | 15.0 samples/s +Validation at step 2250: Loss = 3.6579, Perplexity = 38.78 +Step 2260/44000 | Loss: 3.6436 | LR: 4.98e-04 | 3572.8ms/step | 5.6 samples/s +Step 2270/44000 | Loss: 3.6267 | LR: 4.98e-04 | 1338.4ms/step | 14.9 samples/s +Step 2280/44000 | Loss: 3.6554 | LR: 4.98e-04 | 1332.1ms/step | 15.0 samples/s +Step 2290/44000 | Loss: 3.6486 | LR: 4.98e-04 | 1332.8ms/step | 15.0 samples/s +Step 2300/44000 | Loss: 3.6263 | LR: 4.98e-04 | 1332.7ms/step | 15.0 samples/s +Step 2310/44000 | Loss: 3.6420 | LR: 4.98e-04 | 1330.4ms/step | 15.0 samples/s +Step 2320/44000 | Loss: 3.6284 | LR: 4.98e-04 | 1332.0ms/step | 15.0 samples/s +Step 2330/44000 | Loss: 3.6319 | LR: 4.98e-04 | 1339.9ms/step | 14.9 samples/s +Step 2340/44000 | Loss: 3.6077 | LR: 4.98e-04 | 1325.7ms/step | 15.1 samples/s +Step 2350/44000 | Loss: 3.6183 | LR: 4.98e-04 | 1325.0ms/step | 15.1 samples/s +Step 2360/44000 | Loss: 3.6027 | LR: 4.98e-04 | 1337.8ms/step | 15.0 samples/s +Step 2370/44000 | Loss: 3.6423 | LR: 4.98e-04 | 1323.7ms/step | 15.1 samples/s +Step 2380/44000 | Loss: 3.6187 | LR: 4.98e-04 | 1324.6ms/step | 15.1 samples/s +Step 2390/44000 | Loss: 3.5834 | LR: 4.98e-04 | 1333.8ms/step | 15.0 samples/s +Step 2400/44000 | Loss: 3.5917 | LR: 4.98e-04 | 1324.8ms/step | 15.1 samples/s +Step 2410/44000 | Loss: 3.6387 | LR: 4.98e-04 | 1329.9ms/step | 15.0 samples/s +Step 2420/44000 | Loss: 3.6440 | LR: 4.98e-04 | 1341.1ms/step | 14.9 samples/s +Step 2430/44000 | Loss: 3.6136 | LR: 4.98e-04 | 1325.8ms/step | 15.1 samples/s +Step 2440/44000 | Loss: 3.5889 | LR: 4.98e-04 | 1328.2ms/step | 15.1 samples/s +Step 2450/44000 | Loss: 3.6161 | LR: 4.98e-04 | 1340.0ms/step | 14.9 samples/s +Step 2460/44000 | Loss: 3.6117 | LR: 4.98e-04 | 1331.8ms/step | 15.0 samples/s +Step 2470/44000 | Loss: 3.5891 | LR: 4.98e-04 | 1330.8ms/step | 15.0 samples/s +Step 2480/44000 | Loss: 3.6196 | LR: 4.98e-04 | 1338.4ms/step | 14.9 samples/s +Step 2490/44000 | Loss: 3.5752 | LR: 4.98e-04 | 1328.1ms/step | 15.1 samples/s +Step 2500/44000 | Loss: 3.5922 | LR: 4.98e-04 | 1326.3ms/step | 15.1 samples/s +Validation at step 2500: Loss = 3.6075, Perplexity = 36.87 +Step 2510/44000 | Loss: 3.5757 | LR: 4.98e-04 | 3559.2ms/step | 5.6 samples/s +Step 2520/44000 | Loss: 3.5939 | LR: 4.98e-04 | 1328.0ms/step | 15.1 samples/s +Step 2530/44000 | Loss: 3.5934 | LR: 4.98e-04 | 1324.9ms/step | 15.1 samples/s +Step 2540/44000 | Loss: 3.6053 | LR: 4.98e-04 | 1333.7ms/step | 15.0 samples/s +Step 2550/44000 | Loss: 3.5957 | LR: 4.98e-04 | 1328.9ms/step | 15.0 samples/s +Step 2560/44000 | Loss: 3.6081 | LR: 4.98e-04 | 1331.3ms/step | 15.0 samples/s +Step 2570/44000 | Loss: 3.5767 | LR: 4.98e-04 | 1331.0ms/step | 15.0 samples/s +Step 2580/44000 | Loss: 3.6155 | LR: 4.98e-04 | 1326.2ms/step | 15.1 samples/s +Step 2590/44000 | Loss: 3.5714 | LR: 4.98e-04 | 1326.2ms/step | 15.1 samples/s +Step 2600/44000 | Loss: 3.5784 | LR: 4.98e-04 | 1329.4ms/step | 15.0 samples/s +Step 2610/44000 | Loss: 3.5741 | LR: 4.98e-04 | 1330.8ms/step | 15.0 samples/s +Step 2620/44000 | Loss: 3.5759 | LR: 4.98e-04 | 1327.1ms/step | 15.1 samples/s +Step 2630/44000 | Loss: 3.5656 | LR: 4.98e-04 | 1324.2ms/step | 15.1 samples/s +Step 2640/44000 | Loss: 3.5801 | LR: 4.98e-04 | 1325.2ms/step | 15.1 samples/s +Step 2650/44000 | Loss: 3.5959 | LR: 4.98e-04 | 1338.8ms/step | 14.9 samples/s +Step 2660/44000 | Loss: 3.5655 | LR: 4.98e-04 | 1327.9ms/step | 15.1 samples/s +Step 2670/44000 | Loss: 3.5572 | LR: 4.98e-04 | 1325.8ms/step | 15.1 samples/s +Step 2680/44000 | Loss: 3.5791 | LR: 4.98e-04 | 1322.7ms/step | 15.1 samples/s +Step 2690/44000 | Loss: 3.5683 | LR: 4.98e-04 | 1323.6ms/step | 15.1 samples/s +Step 2700/44000 | Loss: 3.5997 | LR: 4.97e-04 | 1330.1ms/step | 15.0 samples/s +Step 2710/44000 | Loss: 3.5646 | LR: 4.97e-04 | 1325.5ms/step | 15.1 samples/s +Step 2720/44000 | Loss: 3.5654 | LR: 4.97e-04 | 1325.7ms/step | 15.1 samples/s +Step 2730/44000 | Loss: 3.5607 | LR: 4.97e-04 | 1328.1ms/step | 15.1 samples/s +Step 2740/44000 | Loss: 3.5605 | LR: 4.97e-04 | 1327.7ms/step | 15.1 samples/s +Step 2750/44000 | Loss: 3.5510 | LR: 4.97e-04 | 1324.0ms/step | 15.1 samples/s +Validation at step 2750: Loss = 3.5647, Perplexity = 35.33 +Step 2760/44000 | Loss: 3.5235 | LR: 4.97e-04 | 3554.8ms/step | 5.6 samples/s +Step 2770/44000 | Loss: 3.5482 | LR: 4.97e-04 | 1328.2ms/step | 15.1 samples/s +Step 2780/44000 | Loss: 3.5252 | LR: 4.97e-04 | 1324.1ms/step | 15.1 samples/s +Step 2790/44000 | Loss: 3.5507 | LR: 4.97e-04 | 1324.5ms/step | 15.1 samples/s +Step 2800/44000 | Loss: 3.5427 | LR: 4.97e-04 | 1327.2ms/step | 15.1 samples/s +Step 2810/44000 | Loss: 3.5241 | LR: 4.97e-04 | 1323.6ms/step | 15.1 samples/s +Step 2820/44000 | Loss: 3.5372 | LR: 4.97e-04 | 1324.7ms/step | 15.1 samples/s +Step 2830/44000 | Loss: 3.5154 | LR: 4.97e-04 | 1327.4ms/step | 15.1 samples/s +Step 2840/44000 | Loss: 3.5244 | LR: 4.97e-04 | 1331.9ms/step | 15.0 samples/s +Step 2850/44000 | Loss: 3.5334 | LR: 4.97e-04 | 1324.7ms/step | 15.1 samples/s +Step 2860/44000 | Loss: 3.5275 | LR: 4.97e-04 | 1333.0ms/step | 15.0 samples/s +Step 2870/44000 | Loss: 3.5361 | LR: 4.97e-04 | 1324.5ms/step | 15.1 samples/s +Step 2880/44000 | Loss: 3.4968 | LR: 4.97e-04 | 1325.5ms/step | 15.1 samples/s +Step 2890/44000 | Loss: 3.5415 | LR: 4.97e-04 | 1336.4ms/step | 15.0 samples/s +Step 2900/44000 | Loss: 3.5399 | LR: 4.97e-04 | 1325.8ms/step | 15.1 samples/s +Step 2910/44000 | Loss: 3.5122 | LR: 4.97e-04 | 1327.9ms/step | 15.1 samples/s +Step 2920/44000 | Loss: 3.5067 | LR: 4.97e-04 | 1331.5ms/step | 15.0 samples/s +Step 2930/44000 | Loss: 3.5004 | LR: 4.97e-04 | 1328.1ms/step | 15.1 samples/s +Step 2940/44000 | Loss: 3.5212 | LR: 4.97e-04 | 1333.4ms/step | 15.0 samples/s +Step 2950/44000 | Loss: 3.5206 | LR: 4.97e-04 | 1334.1ms/step | 15.0 samples/s +Step 2960/44000 | Loss: 3.5342 | LR: 4.97e-04 | 1330.6ms/step | 15.0 samples/s +Step 2970/44000 | Loss: 3.5182 | LR: 4.97e-04 | 1335.3ms/step | 15.0 samples/s +Step 2980/44000 | Loss: 3.5203 | LR: 4.97e-04 | 1332.9ms/step | 15.0 samples/s +Step 2990/44000 | Loss: 3.5078 | LR: 4.97e-04 | 1327.8ms/step | 15.1 samples/s +Step 3000/44000 | Loss: 3.5388 | LR: 4.97e-04 | 1337.0ms/step | 15.0 samples/s +Validation at step 3000: Loss = 3.5245, Perplexity = 33.94 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-3000 +Step 3010/44000 | Loss: 3.5175 | LR: 4.97e-04 | 3883.7ms/step | 5.1 samples/s +Step 3020/44000 | Loss: 3.4972 | LR: 4.97e-04 | 1328.9ms/step | 15.1 samples/s +Step 3030/44000 | Loss: 3.5166 | LR: 4.97e-04 | 1329.1ms/step | 15.0 samples/s +Step 3040/44000 | Loss: 3.4784 | LR: 4.97e-04 | 1330.7ms/step | 15.0 samples/s +Step 3050/44000 | Loss: 3.5047 | LR: 4.97e-04 | 1328.1ms/step | 15.1 samples/s +Step 3060/44000 | Loss: 3.4750 | LR: 4.96e-04 | 1330.3ms/step | 15.0 samples/s +Step 3070/44000 | Loss: 3.5229 | LR: 4.96e-04 | 1332.7ms/step | 15.0 samples/s +Step 3080/44000 | Loss: 3.5419 | LR: 4.96e-04 | 1333.1ms/step | 15.0 samples/s +Step 3090/44000 | Loss: 3.5316 | LR: 4.96e-04 | 1329.6ms/step | 15.0 samples/s +Step 3100/44000 | Loss: 3.4896 | LR: 4.96e-04 | 1331.0ms/step | 15.0 samples/s +Step 3110/44000 | Loss: 3.4928 | LR: 4.96e-04 | 1332.8ms/step | 15.0 samples/s +Step 3120/44000 | Loss: 3.5123 | LR: 4.96e-04 | 1333.0ms/step | 15.0 samples/s +Step 3130/44000 | Loss: 3.4828 | LR: 4.96e-04 | 1334.6ms/step | 15.0 samples/s +Step 3140/44000 | Loss: 3.4992 | LR: 4.96e-04 | 1334.0ms/step | 15.0 samples/s +Step 3150/44000 | Loss: 3.4971 | LR: 4.96e-04 | 1334.2ms/step | 15.0 samples/s +Step 3160/44000 | Loss: 3.5146 | LR: 4.96e-04 | 1331.6ms/step | 15.0 samples/s +Step 3170/44000 | Loss: 3.4566 | LR: 4.96e-04 | 1333.4ms/step | 15.0 samples/s +Step 3180/44000 | Loss: 3.4977 | LR: 4.96e-04 | 1332.5ms/step | 15.0 samples/s +Step 3190/44000 | Loss: 3.4871 | LR: 4.96e-04 | 1331.5ms/step | 15.0 samples/s +Step 3200/44000 | Loss: 3.5121 | LR: 4.96e-04 | 1330.0ms/step | 15.0 samples/s +Step 3210/44000 | Loss: 3.4843 | LR: 4.96e-04 | 1329.6ms/step | 15.0 samples/s +Step 3220/44000 | Loss: 3.4876 | LR: 4.96e-04 | 1330.5ms/step | 15.0 samples/s +Step 3230/44000 | Loss: 3.4846 | LR: 4.96e-04 | 1329.8ms/step | 15.0 samples/s +Step 3240/44000 | Loss: 3.4822 | LR: 4.96e-04 | 1332.6ms/step | 15.0 samples/s +Step 3250/44000 | Loss: 3.4863 | LR: 4.96e-04 | 1333.3ms/step | 15.0 samples/s +Validation at step 3250: Loss = 3.4924, Perplexity = 32.86 +Step 3260/44000 | Loss: 3.5055 | LR: 4.96e-04 | 3571.4ms/step | 5.6 samples/s +Step 3270/44000 | Loss: 3.4692 | LR: 4.96e-04 | 1334.2ms/step | 15.0 samples/s +Step 3280/44000 | Loss: 3.4571 | LR: 4.96e-04 | 1333.2ms/step | 15.0 samples/s +Step 3290/44000 | Loss: 3.4793 | LR: 4.96e-04 | 1334.5ms/step | 15.0 samples/s +Step 3300/44000 | Loss: 3.5063 | LR: 4.96e-04 | 1330.6ms/step | 15.0 samples/s +Step 3310/44000 | Loss: 3.4794 | LR: 4.96e-04 | 1333.6ms/step | 15.0 samples/s +Step 3320/44000 | Loss: 3.5026 | LR: 4.96e-04 | 1330.8ms/step | 15.0 samples/s +Step 3330/44000 | Loss: 3.5041 | LR: 4.96e-04 | 1331.1ms/step | 15.0 samples/s +Step 3340/44000 | Loss: 3.4619 | LR: 4.96e-04 | 1329.8ms/step | 15.0 samples/s +Step 3350/44000 | Loss: 3.4760 | LR: 4.96e-04 | 1327.0ms/step | 15.1 samples/s +Step 3360/44000 | Loss: 3.4586 | LR: 4.96e-04 | 1329.8ms/step | 15.0 samples/s +Step 3370/44000 | Loss: 3.4336 | LR: 4.95e-04 | 1330.0ms/step | 15.0 samples/s +Step 3380/44000 | Loss: 3.4799 | LR: 4.95e-04 | 1328.8ms/step | 15.1 samples/s +Step 3390/44000 | Loss: 3.4584 | LR: 4.95e-04 | 1327.0ms/step | 15.1 samples/s +Step 3400/44000 | Loss: 3.4906 | LR: 4.95e-04 | 1324.9ms/step | 15.1 samples/s +Step 3410/44000 | Loss: 3.4650 | LR: 4.95e-04 | 1325.5ms/step | 15.1 samples/s +Step 3420/44000 | Loss: 3.4755 | LR: 4.95e-04 | 1326.0ms/step | 15.1 samples/s +Step 3430/44000 | Loss: 3.4667 | LR: 4.95e-04 | 1327.3ms/step | 15.1 samples/s +Step 3440/44000 | Loss: 3.4579 | LR: 4.95e-04 | 1328.8ms/step | 15.1 samples/s +Step 3450/44000 | Loss: 3.4418 | LR: 4.95e-04 | 1329.8ms/step | 15.0 samples/s +Step 3460/44000 | Loss: 3.4778 | LR: 4.95e-04 | 1329.3ms/step | 15.0 samples/s +Step 3470/44000 | Loss: 3.4513 | LR: 4.95e-04 | 1327.4ms/step | 15.1 samples/s +Step 3480/44000 | Loss: 3.4969 | LR: 4.95e-04 | 1330.7ms/step | 15.0 samples/s +Step 3490/44000 | Loss: 3.4305 | LR: 4.95e-04 | 1331.3ms/step | 15.0 samples/s +Step 3500/44000 | Loss: 3.4371 | LR: 4.95e-04 | 1330.3ms/step | 15.0 samples/s +Validation at step 3500: Loss = 3.4659, Perplexity = 32.00 +Step 3510/44000 | Loss: 3.4696 | LR: 4.95e-04 | 3560.6ms/step | 5.6 samples/s +Step 3520/44000 | Loss: 3.4578 | LR: 4.95e-04 | 1329.8ms/step | 15.0 samples/s +Step 3530/44000 | Loss: 3.4550 | LR: 4.95e-04 | 1327.6ms/step | 15.1 samples/s +Step 3540/44000 | Loss: 3.4626 | LR: 4.95e-04 | 1330.9ms/step | 15.0 samples/s +Step 3550/44000 | Loss: 3.4380 | LR: 4.95e-04 | 1330.0ms/step | 15.0 samples/s +Step 3560/44000 | Loss: 3.4234 | LR: 4.95e-04 | 1322.6ms/step | 15.1 samples/s +Step 3570/44000 | Loss: 3.4495 | LR: 4.95e-04 | 1326.9ms/step | 15.1 samples/s +Step 3580/44000 | Loss: 3.4267 | LR: 4.95e-04 | 1330.3ms/step | 15.0 samples/s +Step 3590/44000 | Loss: 3.4390 | LR: 4.95e-04 | 1329.3ms/step | 15.0 samples/s +Step 3600/44000 | Loss: 3.4297 | LR: 4.95e-04 | 1331.4ms/step | 15.0 samples/s +Step 3610/44000 | Loss: 3.4435 | LR: 4.95e-04 | 1328.9ms/step | 15.1 samples/s +Step 3620/44000 | Loss: 3.4423 | LR: 4.95e-04 | 1329.0ms/step | 15.0 samples/s +Step 3630/44000 | Loss: 3.4321 | LR: 4.95e-04 | 1330.2ms/step | 15.0 samples/s +Step 3640/44000 | Loss: 3.4464 | LR: 4.95e-04 | 1327.9ms/step | 15.1 samples/s +Step 3650/44000 | Loss: 3.4329 | LR: 4.94e-04 | 1327.9ms/step | 15.1 samples/s +Step 3660/44000 | Loss: 3.4310 | LR: 4.94e-04 | 1330.9ms/step | 15.0 samples/s +Step 3670/44000 | Loss: 3.4240 | LR: 4.94e-04 | 1328.2ms/step | 15.1 samples/s +Step 3680/44000 | Loss: 3.4357 | LR: 4.94e-04 | 1332.5ms/step | 15.0 samples/s +Step 3690/44000 | Loss: 3.4309 | LR: 4.94e-04 | 1328.0ms/step | 15.1 samples/s +Step 3700/44000 | Loss: 3.4452 | LR: 4.94e-04 | 1331.1ms/step | 15.0 samples/s +Step 3710/44000 | Loss: 3.4390 | LR: 4.94e-04 | 1330.7ms/step | 15.0 samples/s +Step 3720/44000 | Loss: 3.3921 | LR: 4.94e-04 | 1331.2ms/step | 15.0 samples/s +Step 3730/44000 | Loss: 3.4415 | LR: 4.94e-04 | 1331.2ms/step | 15.0 samples/s +Step 3740/44000 | Loss: 3.4373 | LR: 4.94e-04 | 1326.7ms/step | 15.1 samples/s +Step 3750/44000 | Loss: 3.4443 | LR: 4.94e-04 | 1327.1ms/step | 15.1 samples/s +Validation at step 3750: Loss = 3.4372, Perplexity = 31.10 +Step 3760/44000 | Loss: 3.4233 | LR: 4.94e-04 | 3561.1ms/step | 5.6 samples/s +Step 3770/44000 | Loss: 3.4188 | LR: 4.94e-04 | 1329.7ms/step | 15.0 samples/s +Step 3780/44000 | Loss: 3.4131 | LR: 4.94e-04 | 1328.0ms/step | 15.1 samples/s +Step 3790/44000 | Loss: 3.4342 | LR: 4.94e-04 | 1329.1ms/step | 15.0 samples/s +Step 3800/44000 | Loss: 3.4171 | LR: 4.94e-04 | 1327.9ms/step | 15.1 samples/s +Step 3810/44000 | Loss: 3.4218 | LR: 4.94e-04 | 1326.3ms/step | 15.1 samples/s +Step 3820/44000 | Loss: 3.4287 | LR: 4.94e-04 | 1327.6ms/step | 15.1 samples/s +Step 3830/44000 | Loss: 3.4206 | LR: 4.94e-04 | 1325.6ms/step | 15.1 samples/s +Step 3840/44000 | Loss: 3.4080 | LR: 4.94e-04 | 1327.7ms/step | 15.1 samples/s +Step 3850/44000 | Loss: 3.4150 | LR: 4.94e-04 | 1329.3ms/step | 15.0 samples/s +Step 3860/44000 | Loss: 3.4153 | LR: 4.94e-04 | 1328.5ms/step | 15.1 samples/s +Step 3870/44000 | Loss: 3.4203 | LR: 4.94e-04 | 1325.6ms/step | 15.1 samples/s +Step 3880/44000 | Loss: 3.3885 | LR: 4.94e-04 | 1325.0ms/step | 15.1 samples/s +Step 3890/44000 | Loss: 3.4450 | LR: 4.94e-04 | 1323.5ms/step | 15.1 samples/s +Step 3900/44000 | Loss: 3.4018 | LR: 4.93e-04 | 1326.6ms/step | 15.1 samples/s +Step 3910/44000 | Loss: 3.3952 | LR: 4.93e-04 | 1330.0ms/step | 15.0 samples/s +Step 3920/44000 | Loss: 3.3973 | LR: 4.93e-04 | 1329.0ms/step | 15.0 samples/s +Step 3930/44000 | Loss: 3.4091 | LR: 4.93e-04 | 1329.1ms/step | 15.0 samples/s +Step 3940/44000 | Loss: 3.4092 | LR: 4.93e-04 | 1330.8ms/step | 15.0 samples/s +Step 3950/44000 | Loss: 3.4341 | LR: 4.93e-04 | 1330.1ms/step | 15.0 samples/s +Step 3960/44000 | Loss: 3.4271 | LR: 4.93e-04 | 1331.0ms/step | 15.0 samples/s +Step 3970/44000 | Loss: 3.3796 | LR: 4.93e-04 | 1334.6ms/step | 15.0 samples/s +Step 3980/44000 | Loss: 3.3911 | LR: 4.93e-04 | 1333.9ms/step | 15.0 samples/s +Step 3990/44000 | Loss: 3.4184 | LR: 4.93e-04 | 1335.6ms/step | 15.0 samples/s +Step 4000/44000 | Loss: 3.4006 | LR: 4.93e-04 | 1334.4ms/step | 15.0 samples/s +Validation at step 4000: Loss = 3.4143, Perplexity = 30.40 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-4000 +Step 4010/44000 | Loss: 3.4525 | LR: 4.93e-04 | 3876.9ms/step | 5.2 samples/s +Step 4020/44000 | Loss: 3.4127 | LR: 4.93e-04 | 1326.6ms/step | 15.1 samples/s +Step 4030/44000 | Loss: 3.3889 | LR: 4.93e-04 | 1329.9ms/step | 15.0 samples/s +Step 4040/44000 | Loss: 3.4045 | LR: 4.93e-04 | 1329.8ms/step | 15.0 samples/s +Step 4050/44000 | Loss: 3.4380 | LR: 4.93e-04 | 1331.3ms/step | 15.0 samples/s +Step 4060/44000 | Loss: 3.3849 | LR: 4.93e-04 | 1330.1ms/step | 15.0 samples/s +Step 4070/44000 | Loss: 3.4056 | LR: 4.93e-04 | 1331.0ms/step | 15.0 samples/s +Step 4080/44000 | Loss: 3.4230 | LR: 4.93e-04 | 1332.1ms/step | 15.0 samples/s +Step 4090/44000 | Loss: 3.3746 | LR: 4.93e-04 | 1332.4ms/step | 15.0 samples/s +Step 4100/44000 | Loss: 3.4058 | LR: 4.93e-04 | 1331.4ms/step | 15.0 samples/s +Step 4110/44000 | Loss: 3.3770 | LR: 4.93e-04 | 1332.7ms/step | 15.0 samples/s +Step 4120/44000 | Loss: 3.4041 | LR: 4.93e-04 | 1331.7ms/step | 15.0 samples/s +Step 4130/44000 | Loss: 3.4053 | LR: 4.93e-04 | 1332.5ms/step | 15.0 samples/s +Step 4140/44000 | Loss: 3.3860 | LR: 4.92e-04 | 1331.3ms/step | 15.0 samples/s +Step 4150/44000 | Loss: 3.4319 | LR: 4.92e-04 | 1334.9ms/step | 15.0 samples/s +Step 4160/44000 | Loss: 3.4104 | LR: 4.92e-04 | 1333.1ms/step | 15.0 samples/s +Step 4170/44000 | Loss: 3.3942 | LR: 4.92e-04 | 1332.0ms/step | 15.0 samples/s +Step 4180/44000 | Loss: 3.3736 | LR: 4.92e-04 | 1331.7ms/step | 15.0 samples/s +Step 4190/44000 | Loss: 3.3834 | LR: 4.92e-04 | 1332.2ms/step | 15.0 samples/s +Step 4200/44000 | Loss: 3.3757 | LR: 4.92e-04 | 1329.2ms/step | 15.0 samples/s +Step 4210/44000 | Loss: 3.4092 | LR: 4.92e-04 | 1331.0ms/step | 15.0 samples/s +Step 4220/44000 | Loss: 3.3764 | LR: 4.92e-04 | 1333.6ms/step | 15.0 samples/s +Step 4230/44000 | Loss: 3.4037 | LR: 4.92e-04 | 1332.5ms/step | 15.0 samples/s +Step 4240/44000 | Loss: 3.4689 | LR: 4.92e-04 | 1331.8ms/step | 15.0 samples/s +Step 4250/44000 | Loss: 3.4287 | LR: 4.92e-04 | 1333.2ms/step | 15.0 samples/s +Validation at step 4250: Loss = 3.3979, Perplexity = 29.90 +Step 4260/44000 | Loss: 3.4023 | LR: 4.92e-04 | 3570.3ms/step | 5.6 samples/s +Step 4270/44000 | Loss: 3.3368 | LR: 4.92e-04 | 1333.3ms/step | 15.0 samples/s +Step 4280/44000 | Loss: 3.3597 | LR: 4.92e-04 | 1335.1ms/step | 15.0 samples/s +Step 4290/44000 | Loss: 3.3920 | LR: 4.92e-04 | 1330.8ms/step | 15.0 samples/s +Step 4300/44000 | Loss: 3.3827 | LR: 4.92e-04 | 1331.9ms/step | 15.0 samples/s +Step 4310/44000 | Loss: 3.3855 | LR: 4.92e-04 | 1335.5ms/step | 15.0 samples/s +Step 4320/44000 | Loss: 3.3903 | LR: 4.92e-04 | 1335.2ms/step | 15.0 samples/s +Step 4330/44000 | Loss: 3.3761 | LR: 4.92e-04 | 1334.2ms/step | 15.0 samples/s +Step 4340/44000 | Loss: 3.3907 | LR: 4.92e-04 | 1330.3ms/step | 15.0 samples/s +Step 4350/44000 | Loss: 3.3976 | LR: 4.92e-04 | 1332.4ms/step | 15.0 samples/s +Step 4360/44000 | Loss: 3.3459 | LR: 4.91e-04 | 1330.1ms/step | 15.0 samples/s +Step 4370/44000 | Loss: 3.3667 | LR: 4.91e-04 | 1329.1ms/step | 15.0 samples/s +Step 4380/44000 | Loss: 3.4131 | LR: 4.91e-04 | 1330.3ms/step | 15.0 samples/s +Step 4390/44000 | Loss: 3.3987 | LR: 4.91e-04 | 1326.4ms/step | 15.1 samples/s +Step 4400/44000 | Loss: 3.3977 | LR: 4.91e-04 | 1328.4ms/step | 15.1 samples/s +Step 4410/44000 | Loss: 3.3690 | LR: 4.91e-04 | 1329.8ms/step | 15.0 samples/s +Step 4420/44000 | Loss: 3.3764 | LR: 4.91e-04 | 1327.5ms/step | 15.1 samples/s +Step 4430/44000 | Loss: 3.3799 | LR: 4.91e-04 | 1324.8ms/step | 15.1 samples/s +Step 4440/44000 | Loss: 3.3887 | LR: 4.91e-04 | 1326.9ms/step | 15.1 samples/s +Step 4450/44000 | Loss: 3.3802 | LR: 4.91e-04 | 1328.4ms/step | 15.1 samples/s +Step 4460/44000 | Loss: 3.3793 | LR: 4.91e-04 | 1331.8ms/step | 15.0 samples/s +Step 4470/44000 | Loss: 3.3417 | LR: 4.91e-04 | 1331.5ms/step | 15.0 samples/s +Step 4480/44000 | Loss: 3.3772 | LR: 4.91e-04 | 1328.8ms/step | 15.1 samples/s +Step 4490/44000 | Loss: 3.3836 | LR: 4.91e-04 | 1329.8ms/step | 15.0 samples/s +Step 4500/44000 | Loss: 3.3736 | LR: 4.91e-04 | 1325.8ms/step | 15.1 samples/s +Validation at step 4500: Loss = 3.3784, Perplexity = 29.32 +Step 4510/44000 | Loss: 3.3658 | LR: 4.91e-04 | 3560.3ms/step | 5.6 samples/s +Step 4520/44000 | Loss: 3.3849 | LR: 4.91e-04 | 1329.3ms/step | 15.0 samples/s +Step 4530/44000 | Loss: 3.3877 | LR: 4.91e-04 | 1324.9ms/step | 15.1 samples/s +Step 4540/44000 | Loss: 3.3473 | LR: 4.91e-04 | 1325.5ms/step | 15.1 samples/s +Step 4550/44000 | Loss: 3.3399 | LR: 4.91e-04 | 1326.4ms/step | 15.1 samples/s +Step 4560/44000 | Loss: 3.3480 | LR: 4.90e-04 | 1326.1ms/step | 15.1 samples/s +Step 4570/44000 | Loss: 3.3678 | LR: 4.90e-04 | 1322.8ms/step | 15.1 samples/s +Step 4580/44000 | Loss: 3.3259 | LR: 4.90e-04 | 1323.5ms/step | 15.1 samples/s +Step 4590/44000 | Loss: 3.3282 | LR: 4.90e-04 | 1323.6ms/step | 15.1 samples/s +Step 4600/44000 | Loss: 3.3480 | LR: 4.90e-04 | 1327.7ms/step | 15.1 samples/s +Step 4610/44000 | Loss: 3.3562 | LR: 4.90e-04 | 1325.7ms/step | 15.1 samples/s +Step 4620/44000 | Loss: 3.3451 | LR: 4.90e-04 | 1330.7ms/step | 15.0 samples/s +Step 4630/44000 | Loss: 3.3566 | LR: 4.90e-04 | 1330.8ms/step | 15.0 samples/s +Step 4640/44000 | Loss: 3.3670 | LR: 4.90e-04 | 1328.8ms/step | 15.1 samples/s +Step 4650/44000 | Loss: 3.3568 | LR: 4.90e-04 | 1326.8ms/step | 15.1 samples/s +Step 4660/44000 | Loss: 3.3339 | LR: 4.90e-04 | 1325.0ms/step | 15.1 samples/s +Step 4670/44000 | Loss: 3.3277 | LR: 4.90e-04 | 1322.3ms/step | 15.1 samples/s +Step 4680/44000 | Loss: 3.3517 | LR: 4.90e-04 | 1327.0ms/step | 15.1 samples/s +Step 4690/44000 | Loss: 3.3459 | LR: 4.90e-04 | 1329.1ms/step | 15.0 samples/s +Step 4700/44000 | Loss: 3.3797 | LR: 4.90e-04 | 1330.6ms/step | 15.0 samples/s +Step 4710/44000 | Loss: 3.3544 | LR: 4.90e-04 | 1331.2ms/step | 15.0 samples/s +Step 4720/44000 | Loss: 3.3332 | LR: 4.90e-04 | 1329.1ms/step | 15.0 samples/s +Step 4730/44000 | Loss: 3.3533 | LR: 4.90e-04 | 1328.6ms/step | 15.1 samples/s +Step 4740/44000 | Loss: 3.3439 | LR: 4.90e-04 | 1325.9ms/step | 15.1 samples/s +Step 4750/44000 | Loss: 3.3531 | LR: 4.90e-04 | 1323.4ms/step | 15.1 samples/s +Validation at step 4750: Loss = 3.3582, Perplexity = 28.74 +Step 4760/44000 | Loss: 3.3324 | LR: 4.89e-04 | 3558.1ms/step | 5.6 samples/s +Step 4770/44000 | Loss: 3.3563 | LR: 4.89e-04 | 1331.2ms/step | 15.0 samples/s +Step 4780/44000 | Loss: 3.3524 | LR: 4.89e-04 | 1328.2ms/step | 15.1 samples/s +Step 4790/44000 | Loss: 3.3231 | LR: 4.89e-04 | 1326.7ms/step | 15.1 samples/s +Step 4800/44000 | Loss: 3.3489 | LR: 4.89e-04 | 1325.0ms/step | 15.1 samples/s +Step 4810/44000 | Loss: 3.3481 | LR: 4.89e-04 | 1326.2ms/step | 15.1 samples/s +Step 4820/44000 | Loss: 3.3499 | LR: 4.89e-04 | 1324.0ms/step | 15.1 samples/s +Step 4830/44000 | Loss: 3.3378 | LR: 4.89e-04 | 1328.3ms/step | 15.1 samples/s +Step 4840/44000 | Loss: 3.3521 | LR: 4.89e-04 | 1330.1ms/step | 15.0 samples/s +Step 4850/44000 | Loss: 3.3527 | LR: 4.89e-04 | 1328.1ms/step | 15.1 samples/s +Step 4860/44000 | Loss: 3.3597 | LR: 4.89e-04 | 1328.0ms/step | 15.1 samples/s +Step 4870/44000 | Loss: 3.3245 | LR: 4.89e-04 | 1324.5ms/step | 15.1 samples/s +Step 4880/44000 | Loss: 3.3237 | LR: 4.89e-04 | 1327.1ms/step | 15.1 samples/s +Step 4890/44000 | Loss: 3.3159 | LR: 4.89e-04 | 1325.0ms/step | 15.1 samples/s +Step 4900/44000 | Loss: 3.2973 | LR: 4.89e-04 | 1324.5ms/step | 15.1 samples/s +Step 4910/44000 | Loss: 3.3353 | LR: 4.89e-04 | 1328.7ms/step | 15.1 samples/s +Step 4920/44000 | Loss: 3.3357 | LR: 4.89e-04 | 1328.8ms/step | 15.1 samples/s +Step 4930/44000 | Loss: 3.3197 | LR: 4.89e-04 | 1328.4ms/step | 15.1 samples/s +Step 4940/44000 | Loss: 3.3280 | LR: 4.89e-04 | 1331.8ms/step | 15.0 samples/s +Step 4950/44000 | Loss: 3.3311 | LR: 4.88e-04 | 1330.7ms/step | 15.0 samples/s +Step 4960/44000 | Loss: 3.3206 | LR: 4.88e-04 | 1330.2ms/step | 15.0 samples/s +Step 4970/44000 | Loss: 3.3115 | LR: 4.88e-04 | 1330.5ms/step | 15.0 samples/s +Step 4980/44000 | Loss: 3.3087 | LR: 4.88e-04 | 1334.1ms/step | 15.0 samples/s +Step 4990/44000 | Loss: 3.3359 | LR: 4.88e-04 | 1333.0ms/step | 15.0 samples/s +Step 5000/44000 | Loss: 3.3329 | LR: 4.88e-04 | 1333.1ms/step | 15.0 samples/s +Validation at step 5000: Loss = 3.3423, Perplexity = 28.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-5000 +Step 5010/44000 | Loss: 3.3417 | LR: 4.88e-04 | 3878.5ms/step | 5.2 samples/s +Step 5020/44000 | Loss: 3.3215 | LR: 4.88e-04 | 1330.2ms/step | 15.0 samples/s +Step 5030/44000 | Loss: 3.3302 | LR: 4.88e-04 | 1332.6ms/step | 15.0 samples/s +Step 5040/44000 | Loss: 3.3382 | LR: 4.88e-04 | 1330.8ms/step | 15.0 samples/s +Step 5050/44000 | Loss: 3.3126 | LR: 4.88e-04 | 1331.6ms/step | 15.0 samples/s +Step 5060/44000 | Loss: 3.3424 | LR: 4.88e-04 | 1330.5ms/step | 15.0 samples/s +Step 5070/44000 | Loss: 3.3459 | LR: 4.88e-04 | 1324.7ms/step | 15.1 samples/s +Step 5080/44000 | Loss: 3.3390 | LR: 4.88e-04 | 1324.6ms/step | 15.1 samples/s +Step 5090/44000 | Loss: 3.3100 | LR: 4.88e-04 | 1327.4ms/step | 15.1 samples/s +Step 5100/44000 | Loss: 3.3375 | LR: 4.88e-04 | 1330.8ms/step | 15.0 samples/s +Step 5110/44000 | Loss: 3.3555 | LR: 4.88e-04 | 1329.2ms/step | 15.0 samples/s +Step 5120/44000 | Loss: 3.3525 | LR: 4.88e-04 | 1328.0ms/step | 15.1 samples/s +Step 5130/44000 | Loss: 3.3349 | LR: 4.87e-04 | 1329.1ms/step | 15.0 samples/s +Step 5140/44000 | Loss: 3.3095 | LR: 4.87e-04 | 1334.0ms/step | 15.0 samples/s +Step 5150/44000 | Loss: 3.3540 | LR: 4.87e-04 | 1332.3ms/step | 15.0 samples/s +Step 5160/44000 | Loss: 3.3532 | LR: 4.87e-04 | 1330.5ms/step | 15.0 samples/s +Step 5170/44000 | Loss: 3.3419 | LR: 4.87e-04 | 1331.1ms/step | 15.0 samples/s +Step 5180/44000 | Loss: 3.3002 | LR: 4.87e-04 | 1332.4ms/step | 15.0 samples/s +Step 5190/44000 | Loss: 3.3264 | LR: 4.87e-04 | 1333.2ms/step | 15.0 samples/s +Step 5200/44000 | Loss: 3.2947 | LR: 4.87e-04 | 1335.9ms/step | 15.0 samples/s +Step 5210/44000 | Loss: 3.3313 | LR: 4.87e-04 | 1332.6ms/step | 15.0 samples/s +Step 5220/44000 | Loss: 3.3244 | LR: 4.87e-04 | 1332.5ms/step | 15.0 samples/s +Step 5230/44000 | Loss: 3.2782 | LR: 4.87e-04 | 1336.1ms/step | 15.0 samples/s +Step 5240/44000 | Loss: 3.3076 | LR: 4.87e-04 | 1334.2ms/step | 15.0 samples/s +Step 5250/44000 | Loss: 3.3065 | LR: 4.87e-04 | 1333.1ms/step | 15.0 samples/s +Validation at step 5250: Loss = 3.3276, Perplexity = 27.87 +Step 5260/44000 | Loss: 3.3293 | LR: 4.87e-04 | 3575.8ms/step | 5.6 samples/s +Step 5270/44000 | Loss: 3.2985 | LR: 4.87e-04 | 1336.2ms/step | 15.0 samples/s +Step 5280/44000 | Loss: 3.3259 | LR: 4.87e-04 | 1335.1ms/step | 15.0 samples/s +Step 5290/44000 | Loss: 3.3129 | LR: 4.87e-04 | 1332.4ms/step | 15.0 samples/s +Step 5300/44000 | Loss: 3.2638 | LR: 4.86e-04 | 1332.4ms/step | 15.0 samples/s +Step 5310/44000 | Loss: 3.3062 | LR: 4.86e-04 | 1332.9ms/step | 15.0 samples/s +Step 5320/44000 | Loss: 3.3179 | LR: 4.86e-04 | 1334.8ms/step | 15.0 samples/s +Step 5330/44000 | Loss: 3.3140 | LR: 4.86e-04 | 1333.7ms/step | 15.0 samples/s +Step 5340/44000 | Loss: 3.3258 | LR: 4.86e-04 | 1326.0ms/step | 15.1 samples/s +Step 5350/44000 | Loss: 3.2924 | LR: 4.86e-04 | 1328.5ms/step | 15.1 samples/s +Step 5360/44000 | Loss: 3.3170 | LR: 4.86e-04 | 1327.7ms/step | 15.1 samples/s +Step 5370/44000 | Loss: 3.3151 | LR: 4.86e-04 | 1327.6ms/step | 15.1 samples/s +Step 5380/44000 | Loss: 3.3037 | LR: 4.86e-04 | 1324.4ms/step | 15.1 samples/s +Step 5390/44000 | Loss: 3.3259 | LR: 4.86e-04 | 1329.5ms/step | 15.0 samples/s +Step 5400/44000 | Loss: 3.3305 | LR: 4.86e-04 | 1328.6ms/step | 15.1 samples/s +Step 5410/44000 | Loss: 3.3160 | LR: 4.86e-04 | 1327.8ms/step | 15.1 samples/s +Step 5420/44000 | Loss: 3.3124 | LR: 4.86e-04 | 1323.9ms/step | 15.1 samples/s +Step 5430/44000 | Loss: 3.2943 | LR: 4.86e-04 | 1327.9ms/step | 15.1 samples/s +Step 5440/44000 | Loss: 3.3072 | LR: 4.86e-04 | 1326.4ms/step | 15.1 samples/s +Step 5450/44000 | Loss: 3.3185 | LR: 4.86e-04 | 1325.9ms/step | 15.1 samples/s +Step 5460/44000 | Loss: 3.3339 | LR: 4.86e-04 | 1327.4ms/step | 15.1 samples/s +Step 5470/44000 | Loss: 3.3157 | LR: 4.85e-04 | 1328.8ms/step | 15.1 samples/s +Step 5480/44000 | Loss: 3.2968 | LR: 4.85e-04 | 1326.5ms/step | 15.1 samples/s +Step 5490/44000 | Loss: 3.2839 | LR: 4.85e-04 | 1327.7ms/step | 15.1 samples/s +Step 5500/44000 | Loss: 3.3081 | LR: 4.85e-04 | 1329.4ms/step | 15.0 samples/s +Validation at step 5500: Loss = 3.3127, Perplexity = 27.46 +Step 5510/44000 | Loss: 3.3030 | LR: 4.85e-04 | 3563.9ms/step | 5.6 samples/s +Step 5520/44000 | Loss: 3.3062 | LR: 4.85e-04 | 1324.4ms/step | 15.1 samples/s +Step 5530/44000 | Loss: 3.2753 | LR: 4.85e-04 | 1324.8ms/step | 15.1 samples/s +Step 5540/44000 | Loss: 3.3096 | LR: 4.85e-04 | 1326.7ms/step | 15.1 samples/s +Step 5550/44000 | Loss: 3.2914 | LR: 4.85e-04 | 1325.6ms/step | 15.1 samples/s +Step 5560/44000 | Loss: 3.3011 | LR: 4.85e-04 | 1325.0ms/step | 15.1 samples/s +Step 5570/44000 | Loss: 3.2947 | LR: 4.85e-04 | 1327.7ms/step | 15.1 samples/s +Step 5580/44000 | Loss: 3.2829 | LR: 4.85e-04 | 1328.3ms/step | 15.1 samples/s +Step 5590/44000 | Loss: 3.2772 | LR: 4.85e-04 | 1327.1ms/step | 15.1 samples/s +Step 5600/44000 | Loss: 3.2783 | LR: 4.85e-04 | 1329.3ms/step | 15.0 samples/s +Step 5610/44000 | Loss: 3.3086 | LR: 4.85e-04 | 1328.9ms/step | 15.0 samples/s +Step 5620/44000 | Loss: 3.2910 | LR: 4.85e-04 | 1325.9ms/step | 15.1 samples/s +Step 5630/44000 | Loss: 3.3106 | LR: 4.84e-04 | 1325.0ms/step | 15.1 samples/s +Step 5640/44000 | Loss: 3.2828 | LR: 4.84e-04 | 1326.4ms/step | 15.1 samples/s +Step 5650/44000 | Loss: 3.2813 | LR: 4.84e-04 | 1323.0ms/step | 15.1 samples/s +Step 5660/44000 | Loss: 3.2681 | LR: 4.84e-04 | 1322.5ms/step | 15.1 samples/s +Step 5670/44000 | Loss: 3.2863 | LR: 4.84e-04 | 1326.0ms/step | 15.1 samples/s +Step 5680/44000 | Loss: 3.2697 | LR: 4.84e-04 | 1326.5ms/step | 15.1 samples/s +Step 5690/44000 | Loss: 3.3163 | LR: 4.84e-04 | 1326.3ms/step | 15.1 samples/s +Step 5700/44000 | Loss: 3.3074 | LR: 4.84e-04 | 1327.2ms/step | 15.1 samples/s +Step 5710/44000 | Loss: 3.2556 | LR: 4.84e-04 | 1328.4ms/step | 15.1 samples/s +Step 5720/44000 | Loss: 3.2553 | LR: 4.84e-04 | 1327.4ms/step | 15.1 samples/s +Step 5730/44000 | Loss: 3.2738 | LR: 4.84e-04 | 1324.2ms/step | 15.1 samples/s +Step 5740/44000 | Loss: 3.2775 | LR: 4.84e-04 | 1325.8ms/step | 15.1 samples/s +Step 5750/44000 | Loss: 3.2805 | LR: 4.84e-04 | 1325.3ms/step | 15.1 samples/s +Validation at step 5750: Loss = 3.3007, Perplexity = 27.13 +Step 5760/44000 | Loss: 3.3039 | LR: 4.84e-04 | 3543.0ms/step | 5.6 samples/s +Step 5770/44000 | Loss: 3.3087 | LR: 4.84e-04 | 1320.0ms/step | 15.2 samples/s +Step 5780/44000 | Loss: 3.2862 | LR: 4.83e-04 | 1321.9ms/step | 15.1 samples/s +Step 5790/44000 | Loss: 3.2698 | LR: 4.83e-04 | 1324.9ms/step | 15.1 samples/s +Step 5800/44000 | Loss: 3.2956 | LR: 4.83e-04 | 1325.3ms/step | 15.1 samples/s +Step 5810/44000 | Loss: 3.3099 | LR: 4.83e-04 | 1323.3ms/step | 15.1 samples/s +Step 5820/44000 | Loss: 3.2915 | LR: 4.83e-04 | 1324.0ms/step | 15.1 samples/s +Step 5830/44000 | Loss: 3.2875 | LR: 4.83e-04 | 1320.4ms/step | 15.1 samples/s +Step 5840/44000 | Loss: 3.2821 | LR: 4.83e-04 | 1324.5ms/step | 15.1 samples/s +Step 5850/44000 | Loss: 3.2637 | LR: 4.83e-04 | 1323.9ms/step | 15.1 samples/s +Step 5860/44000 | Loss: 3.2907 | LR: 4.83e-04 | 1325.5ms/step | 15.1 samples/s +Step 5870/44000 | Loss: 3.2974 | LR: 4.83e-04 | 1325.3ms/step | 15.1 samples/s +Step 5880/44000 | Loss: 3.2766 | LR: 4.83e-04 | 1326.6ms/step | 15.1 samples/s +Step 5890/44000 | Loss: 3.2909 | LR: 4.83e-04 | 1327.7ms/step | 15.1 samples/s +Step 5900/44000 | Loss: 3.2803 | LR: 4.83e-04 | 1328.4ms/step | 15.1 samples/s +Step 5910/44000 | Loss: 3.2879 | LR: 4.83e-04 | 1328.1ms/step | 15.1 samples/s +Step 5920/44000 | Loss: 3.2957 | LR: 4.83e-04 | 1330.7ms/step | 15.0 samples/s +Step 5930/44000 | Loss: 3.2485 | LR: 4.83e-04 | 1327.5ms/step | 15.1 samples/s +Step 5940/44000 | Loss: 3.2997 | LR: 4.82e-04 | 1328.9ms/step | 15.1 samples/s +Step 5950/44000 | Loss: 3.2596 | LR: 4.82e-04 | 1329.2ms/step | 15.0 samples/s +Step 5960/44000 | Loss: 3.2518 | LR: 4.82e-04 | 1331.3ms/step | 15.0 samples/s +Step 5970/44000 | Loss: 3.2580 | LR: 4.82e-04 | 1333.5ms/step | 15.0 samples/s +Step 5980/44000 | Loss: 3.2527 | LR: 4.82e-04 | 1330.9ms/step | 15.0 samples/s +Step 5990/44000 | Loss: 3.2526 | LR: 4.82e-04 | 1333.0ms/step | 15.0 samples/s +Step 6000/44000 | Loss: 3.2785 | LR: 4.82e-04 | 1333.1ms/step | 15.0 samples/s +Validation at step 6000: Loss = 3.2872, Perplexity = 26.77 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-6000 +Step 6010/44000 | Loss: 3.3001 | LR: 4.82e-04 | 3879.7ms/step | 5.2 samples/s +Step 6020/44000 | Loss: 3.2620 | LR: 4.82e-04 | 1328.5ms/step | 15.1 samples/s +Step 6030/44000 | Loss: 3.2666 | LR: 4.82e-04 | 1326.9ms/step | 15.1 samples/s +Step 6040/44000 | Loss: 3.2691 | LR: 4.82e-04 | 1329.6ms/step | 15.0 samples/s +Step 6050/44000 | Loss: 3.2742 | LR: 4.82e-04 | 1331.8ms/step | 15.0 samples/s +Step 6060/44000 | Loss: 3.2976 | LR: 4.82e-04 | 1334.3ms/step | 15.0 samples/s +Step 6070/44000 | Loss: 3.2770 | LR: 4.82e-04 | 1331.8ms/step | 15.0 samples/s +Step 6080/44000 | Loss: 3.2698 | LR: 4.81e-04 | 1333.8ms/step | 15.0 samples/s +Step 6090/44000 | Loss: 3.2977 | LR: 4.81e-04 | 1334.4ms/step | 15.0 samples/s +Step 6100/44000 | Loss: 3.2512 | LR: 4.81e-04 | 1331.4ms/step | 15.0 samples/s +Step 6110/44000 | Loss: 3.2677 | LR: 4.81e-04 | 1331.5ms/step | 15.0 samples/s +Step 6120/44000 | Loss: 3.2479 | LR: 4.81e-04 | 1331.7ms/step | 15.0 samples/s +Step 6130/44000 | Loss: 3.2653 | LR: 4.81e-04 | 1333.2ms/step | 15.0 samples/s +Step 6140/44000 | Loss: 3.2523 | LR: 4.81e-04 | 1332.8ms/step | 15.0 samples/s +Step 6150/44000 | Loss: 3.2605 | LR: 4.81e-04 | 1332.4ms/step | 15.0 samples/s +Step 6160/44000 | Loss: 3.2581 | LR: 4.81e-04 | 1329.3ms/step | 15.0 samples/s +Step 6170/44000 | Loss: 3.2759 | LR: 4.81e-04 | 1330.7ms/step | 15.0 samples/s +Step 6180/44000 | Loss: 3.2721 | LR: 4.81e-04 | 1333.0ms/step | 15.0 samples/s +Step 6190/44000 | Loss: 3.2570 | LR: 4.81e-04 | 1333.3ms/step | 15.0 samples/s +Step 6200/44000 | Loss: 3.2715 | LR: 4.81e-04 | 1330.4ms/step | 15.0 samples/s +Step 6210/44000 | Loss: 3.2674 | LR: 4.81e-04 | 1332.3ms/step | 15.0 samples/s +Step 6220/44000 | Loss: 3.2989 | LR: 4.81e-04 | 1330.2ms/step | 15.0 samples/s +Step 6230/44000 | Loss: 3.2647 | LR: 4.80e-04 | 1332.7ms/step | 15.0 samples/s +Step 6240/44000 | Loss: 3.2841 | LR: 4.80e-04 | 1333.1ms/step | 15.0 samples/s +Step 6250/44000 | Loss: 3.2795 | LR: 4.80e-04 | 1330.9ms/step | 15.0 samples/s +Validation at step 6250: Loss = 3.2882, Perplexity = 26.79 +Step 6260/44000 | Loss: 3.2769 | LR: 4.80e-04 | 3574.9ms/step | 5.6 samples/s +Step 6270/44000 | Loss: 3.2985 | LR: 4.80e-04 | 1331.4ms/step | 15.0 samples/s +Step 6280/44000 | Loss: 3.2510 | LR: 4.80e-04 | 1335.3ms/step | 15.0 samples/s +Step 6290/44000 | Loss: 3.2805 | LR: 4.80e-04 | 1330.6ms/step | 15.0 samples/s +Step 6300/44000 | Loss: 3.2560 | LR: 4.80e-04 | 1332.2ms/step | 15.0 samples/s +Step 6310/44000 | Loss: 3.2610 | LR: 4.80e-04 | 1332.6ms/step | 15.0 samples/s +Step 6320/44000 | Loss: 3.2541 | LR: 4.80e-04 | 1331.2ms/step | 15.0 samples/s +Step 6330/44000 | Loss: 3.2621 | LR: 4.80e-04 | 1333.6ms/step | 15.0 samples/s +Step 6340/44000 | Loss: 3.2720 | LR: 4.80e-04 | 1333.3ms/step | 15.0 samples/s +Step 6350/44000 | Loss: 3.2330 | LR: 4.80e-04 | 1328.4ms/step | 15.1 samples/s +Step 6360/44000 | Loss: 3.2876 | LR: 4.80e-04 | 1327.3ms/step | 15.1 samples/s +Step 6370/44000 | Loss: 3.2564 | LR: 4.79e-04 | 1329.6ms/step | 15.0 samples/s +Step 6380/44000 | Loss: 3.2437 | LR: 4.79e-04 | 1329.8ms/step | 15.0 samples/s +Step 6390/44000 | Loss: 3.2367 | LR: 4.79e-04 | 1326.6ms/step | 15.1 samples/s +Step 6400/44000 | Loss: 3.2697 | LR: 4.79e-04 | 1327.8ms/step | 15.1 samples/s +Step 6410/44000 | Loss: 3.2390 | LR: 4.79e-04 | 1329.8ms/step | 15.0 samples/s +Step 6420/44000 | Loss: 3.2699 | LR: 4.79e-04 | 1328.9ms/step | 15.1 samples/s +Step 6430/44000 | Loss: 3.2330 | LR: 4.79e-04 | 1328.3ms/step | 15.1 samples/s +Step 6440/44000 | Loss: 3.2483 | LR: 4.79e-04 | 1326.7ms/step | 15.1 samples/s +Step 6450/44000 | Loss: 3.2452 | LR: 4.79e-04 | 1329.1ms/step | 15.0 samples/s +Step 6460/44000 | Loss: 3.2560 | LR: 4.79e-04 | 1329.2ms/step | 15.0 samples/s +Step 6470/44000 | Loss: 3.2350 | LR: 4.79e-04 | 1330.7ms/step | 15.0 samples/s +Step 6480/44000 | Loss: 3.2491 | LR: 4.79e-04 | 1331.1ms/step | 15.0 samples/s +Step 6490/44000 | Loss: 3.2409 | LR: 4.79e-04 | 1332.0ms/step | 15.0 samples/s +Step 6500/44000 | Loss: 3.2393 | LR: 4.79e-04 | 1326.6ms/step | 15.1 samples/s +Validation at step 6500: Loss = 3.2670, Perplexity = 26.23 +Step 6510/44000 | Loss: 3.2660 | LR: 4.78e-04 | 3553.2ms/step | 5.6 samples/s +Step 6520/44000 | Loss: 3.2728 | LR: 4.78e-04 | 1327.8ms/step | 15.1 samples/s +Step 6530/44000 | Loss: 3.2671 | LR: 4.78e-04 | 1329.8ms/step | 15.0 samples/s +Step 6540/44000 | Loss: 3.2646 | LR: 4.78e-04 | 1328.0ms/step | 15.1 samples/s +Step 6550/44000 | Loss: 3.2651 | LR: 4.78e-04 | 1325.6ms/step | 15.1 samples/s +Step 6560/44000 | Loss: 3.2449 | LR: 4.78e-04 | 1329.5ms/step | 15.0 samples/s +Step 6570/44000 | Loss: 3.2474 | LR: 4.78e-04 | 1328.8ms/step | 15.1 samples/s +Step 6580/44000 | Loss: 3.2855 | LR: 4.78e-04 | 1329.5ms/step | 15.0 samples/s +Step 6590/44000 | Loss: 3.2532 | LR: 4.78e-04 | 1327.4ms/step | 15.1 samples/s +Step 6600/44000 | Loss: 3.2432 | LR: 4.78e-04 | 1326.4ms/step | 15.1 samples/s +Step 6610/44000 | Loss: 3.2079 | LR: 4.78e-04 | 1324.2ms/step | 15.1 samples/s +Step 6620/44000 | Loss: 3.2403 | LR: 4.78e-04 | 1327.3ms/step | 15.1 samples/s +Step 6630/44000 | Loss: 3.2440 | LR: 4.78e-04 | 1327.2ms/step | 15.1 samples/s +Step 6640/44000 | Loss: 3.2345 | LR: 4.77e-04 | 1330.1ms/step | 15.0 samples/s +Step 6650/44000 | Loss: 3.2607 | LR: 4.77e-04 | 1330.2ms/step | 15.0 samples/s +Step 6660/44000 | Loss: 3.2859 | LR: 4.77e-04 | 1323.8ms/step | 15.1 samples/s +Step 6670/44000 | Loss: 3.2327 | LR: 4.77e-04 | 1328.4ms/step | 15.1 samples/s +Step 6680/44000 | Loss: 3.2550 | LR: 4.77e-04 | 1330.5ms/step | 15.0 samples/s +Step 6690/44000 | Loss: 3.2578 | LR: 4.77e-04 | 1329.7ms/step | 15.0 samples/s +Step 6700/44000 | Loss: 3.2569 | LR: 4.77e-04 | 1328.5ms/step | 15.1 samples/s +Step 6710/44000 | Loss: 3.2549 | LR: 4.77e-04 | 1326.1ms/step | 15.1 samples/s +Step 6720/44000 | Loss: 3.2214 | LR: 4.77e-04 | 1329.5ms/step | 15.0 samples/s +Step 6730/44000 | Loss: 3.2551 | LR: 4.77e-04 | 1329.7ms/step | 15.0 samples/s +Step 6740/44000 | Loss: 3.2329 | LR: 4.77e-04 | 1325.8ms/step | 15.1 samples/s +Step 6750/44000 | Loss: 3.2374 | LR: 4.77e-04 | 1323.9ms/step | 15.1 samples/s +Validation at step 6750: Loss = 3.2579, Perplexity = 26.00 +Step 6760/44000 | Loss: 3.2624 | LR: 4.77e-04 | 3557.5ms/step | 5.6 samples/s +Step 6770/44000 | Loss: 3.2593 | LR: 4.76e-04 | 1329.3ms/step | 15.0 samples/s +Step 6780/44000 | Loss: 3.2565 | LR: 4.76e-04 | 1328.2ms/step | 15.1 samples/s +Step 6790/44000 | Loss: 3.2491 | LR: 4.76e-04 | 1326.9ms/step | 15.1 samples/s +Step 6800/44000 | Loss: 3.2016 | LR: 4.76e-04 | 1326.2ms/step | 15.1 samples/s +Step 6810/44000 | Loss: 3.2639 | LR: 4.76e-04 | 1324.4ms/step | 15.1 samples/s +Step 6820/44000 | Loss: 3.2231 | LR: 4.76e-04 | 1327.7ms/step | 15.1 samples/s +Step 6830/44000 | Loss: 3.2451 | LR: 4.76e-04 | 1326.2ms/step | 15.1 samples/s +Step 6840/44000 | Loss: 3.2482 | LR: 4.76e-04 | 1326.3ms/step | 15.1 samples/s +Step 6850/44000 | Loss: 3.2503 | LR: 4.76e-04 | 1326.5ms/step | 15.1 samples/s +Step 6860/44000 | Loss: 3.2501 | LR: 4.76e-04 | 1329.9ms/step | 15.0 samples/s +Step 6870/44000 | Loss: 3.2256 | LR: 4.76e-04 | 1327.6ms/step | 15.1 samples/s +Step 6880/44000 | Loss: 3.1988 | LR: 4.76e-04 | 1328.2ms/step | 15.1 samples/s +Step 6890/44000 | Loss: 3.2533 | LR: 4.76e-04 | 1323.7ms/step | 15.1 samples/s +Step 6900/44000 | Loss: 3.2664 | LR: 4.75e-04 | 1325.0ms/step | 15.1 samples/s +Step 6910/44000 | Loss: 3.2290 | LR: 4.75e-04 | 1328.0ms/step | 15.1 samples/s +Step 6920/44000 | Loss: 3.2481 | LR: 4.75e-04 | 1328.7ms/step | 15.1 samples/s +Step 6930/44000 | Loss: 3.2296 | LR: 4.75e-04 | 1328.7ms/step | 15.1 samples/s +Step 6940/44000 | Loss: 3.2561 | LR: 4.75e-04 | 1330.6ms/step | 15.0 samples/s +Step 6950/44000 | Loss: 3.2549 | LR: 4.75e-04 | 1331.0ms/step | 15.0 samples/s +Step 6960/44000 | Loss: 3.2271 | LR: 4.75e-04 | 1330.9ms/step | 15.0 samples/s +Step 6970/44000 | Loss: 3.2506 | LR: 4.75e-04 | 1332.1ms/step | 15.0 samples/s +Step 6980/44000 | Loss: 3.2162 | LR: 4.75e-04 | 1330.5ms/step | 15.0 samples/s +Step 6990/44000 | Loss: 3.2236 | LR: 4.75e-04 | 1331.4ms/step | 15.0 samples/s +Step 7000/44000 | Loss: 3.2329 | LR: 4.75e-04 | 1331.8ms/step | 15.0 samples/s +Validation at step 7000: Loss = 3.2485, Perplexity = 25.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-7000 +Step 7010/44000 | Loss: 3.2237 | LR: 4.75e-04 | 3854.2ms/step | 5.2 samples/s +Step 7020/44000 | Loss: 3.2239 | LR: 4.75e-04 | 1330.1ms/step | 15.0 samples/s +Step 7030/44000 | Loss: 3.2488 | LR: 4.74e-04 | 1329.9ms/step | 15.0 samples/s +Step 7040/44000 | Loss: 3.2387 | LR: 4.74e-04 | 1331.3ms/step | 15.0 samples/s +Step 7050/44000 | Loss: 3.2637 | LR: 4.74e-04 | 1332.5ms/step | 15.0 samples/s +Step 7060/44000 | Loss: 3.2439 | LR: 4.74e-04 | 1331.9ms/step | 15.0 samples/s +Step 7070/44000 | Loss: 3.2322 | LR: 4.74e-04 | 1333.6ms/step | 15.0 samples/s +Step 7080/44000 | Loss: 3.2245 | LR: 4.74e-04 | 1332.7ms/step | 15.0 samples/s +Step 7090/44000 | Loss: 3.2033 | LR: 4.74e-04 | 1331.0ms/step | 15.0 samples/s +Step 7100/44000 | Loss: 3.2295 | LR: 4.74e-04 | 1332.1ms/step | 15.0 samples/s +Step 7110/44000 | Loss: 3.2032 | LR: 4.74e-04 | 1331.4ms/step | 15.0 samples/s +Step 7120/44000 | Loss: 3.2243 | LR: 4.74e-04 | 1332.4ms/step | 15.0 samples/s +Step 7130/44000 | Loss: 3.2380 | LR: 4.74e-04 | 1331.9ms/step | 15.0 samples/s +Step 7140/44000 | Loss: 3.2573 | LR: 4.74e-04 | 1332.5ms/step | 15.0 samples/s +Step 7150/44000 | Loss: 3.2375 | LR: 4.73e-04 | 1333.7ms/step | 15.0 samples/s +Step 7160/44000 | Loss: 3.2478 | LR: 4.73e-04 | 1334.7ms/step | 15.0 samples/s +Step 7170/44000 | Loss: 3.2487 | LR: 4.73e-04 | 1335.4ms/step | 15.0 samples/s +Step 7180/44000 | Loss: 3.2327 | LR: 4.73e-04 | 1333.5ms/step | 15.0 samples/s +Step 7190/44000 | Loss: 3.1928 | LR: 4.73e-04 | 1333.2ms/step | 15.0 samples/s +Step 7200/44000 | Loss: 3.2450 | LR: 4.73e-04 | 1333.1ms/step | 15.0 samples/s +Step 7210/44000 | Loss: 3.2659 | LR: 4.73e-04 | 1332.1ms/step | 15.0 samples/s +Step 7220/44000 | Loss: 3.2496 | LR: 4.73e-04 | 1330.8ms/step | 15.0 samples/s +Step 7230/44000 | Loss: 3.2466 | LR: 4.73e-04 | 1332.0ms/step | 15.0 samples/s +Step 7240/44000 | Loss: 3.2360 | LR: 4.73e-04 | 1334.6ms/step | 15.0 samples/s +Step 7250/44000 | Loss: 3.2400 | LR: 4.73e-04 | 1333.0ms/step | 15.0 samples/s +Validation at step 7250: Loss = 3.2376, Perplexity = 25.47 +Step 7260/44000 | Loss: 3.2217 | LR: 4.73e-04 | 3573.6ms/step | 5.6 samples/s +Step 7270/44000 | Loss: 3.2307 | LR: 4.72e-04 | 1334.5ms/step | 15.0 samples/s +Step 7280/44000 | Loss: 3.2107 | LR: 4.72e-04 | 1336.1ms/step | 15.0 samples/s +Step 7290/44000 | Loss: 3.2525 | LR: 4.72e-04 | 1335.6ms/step | 15.0 samples/s +Step 7300/44000 | Loss: 3.2338 | LR: 4.72e-04 | 1335.7ms/step | 15.0 samples/s +Step 7310/44000 | Loss: 3.2508 | LR: 4.72e-04 | 1337.4ms/step | 15.0 samples/s +Step 7320/44000 | Loss: 3.2508 | LR: 4.72e-04 | 1335.3ms/step | 15.0 samples/s +Step 7330/44000 | Loss: 3.2434 | LR: 4.72e-04 | 1333.9ms/step | 15.0 samples/s +Step 7340/44000 | Loss: 3.2535 | LR: 4.72e-04 | 1331.9ms/step | 15.0 samples/s +Step 7350/44000 | Loss: 3.2021 | LR: 4.72e-04 | 1330.5ms/step | 15.0 samples/s +Step 7360/44000 | Loss: 3.2090 | LR: 4.72e-04 | 1329.5ms/step | 15.0 samples/s +Step 7370/44000 | Loss: 3.1999 | LR: 4.72e-04 | 1326.5ms/step | 15.1 samples/s +Step 7380/44000 | Loss: 3.2293 | LR: 4.72e-04 | 1324.3ms/step | 15.1 samples/s +Step 7390/44000 | Loss: 3.2355 | LR: 4.71e-04 | 1323.9ms/step | 15.1 samples/s +Step 7400/44000 | Loss: 3.2268 | LR: 4.71e-04 | 1326.3ms/step | 15.1 samples/s +Step 7410/44000 | Loss: 3.2499 | LR: 4.71e-04 | 1324.1ms/step | 15.1 samples/s +Step 7420/44000 | Loss: 3.2523 | LR: 4.71e-04 | 1326.6ms/step | 15.1 samples/s +Step 7430/44000 | Loss: 3.2313 | LR: 4.71e-04 | 1326.8ms/step | 15.1 samples/s +Step 7440/44000 | Loss: 3.2321 | LR: 4.71e-04 | 1326.3ms/step | 15.1 samples/s +Step 7450/44000 | Loss: 3.2479 | LR: 4.71e-04 | 1330.6ms/step | 15.0 samples/s +Step 7460/44000 | Loss: 3.1920 | LR: 4.71e-04 | 1328.2ms/step | 15.1 samples/s +Step 7470/44000 | Loss: 3.2347 | LR: 4.71e-04 | 1328.2ms/step | 15.1 samples/s +Step 7480/44000 | Loss: 3.1887 | LR: 4.71e-04 | 1323.0ms/step | 15.1 samples/s +Step 7490/44000 | Loss: 3.2303 | LR: 4.71e-04 | 1322.1ms/step | 15.1 samples/s +Step 7500/44000 | Loss: 3.1925 | LR: 4.71e-04 | 1323.5ms/step | 15.1 samples/s +Validation at step 7500: Loss = 3.2297, Perplexity = 25.27 +Step 7510/44000 | Loss: 3.2049 | LR: 4.70e-04 | 3560.1ms/step | 5.6 samples/s +Step 7520/44000 | Loss: 3.2200 | LR: 4.70e-04 | 1325.3ms/step | 15.1 samples/s +Step 7530/44000 | Loss: 3.2482 | LR: 4.70e-04 | 1326.3ms/step | 15.1 samples/s +Step 7540/44000 | Loss: 3.2070 | LR: 4.70e-04 | 1328.5ms/step | 15.1 samples/s +Step 7550/44000 | Loss: 3.2123 | LR: 4.70e-04 | 1328.1ms/step | 15.1 samples/s +Step 7560/44000 | Loss: 3.2146 | LR: 4.70e-04 | 1328.0ms/step | 15.1 samples/s +Step 7570/44000 | Loss: 3.2445 | LR: 4.70e-04 | 1329.2ms/step | 15.0 samples/s +Step 7580/44000 | Loss: 3.2000 | LR: 4.70e-04 | 1330.3ms/step | 15.0 samples/s +Step 7590/44000 | Loss: 3.2444 | LR: 4.70e-04 | 1330.5ms/step | 15.0 samples/s +Step 7600/44000 | Loss: 3.2250 | LR: 4.70e-04 | 1327.1ms/step | 15.1 samples/s +Step 7610/44000 | Loss: 3.2425 | LR: 4.70e-04 | 1325.9ms/step | 15.1 samples/s +Step 7620/44000 | Loss: 3.1962 | LR: 4.70e-04 | 1325.4ms/step | 15.1 samples/s +Step 7630/44000 | Loss: 3.2507 | LR: 4.69e-04 | 1328.6ms/step | 15.1 samples/s +Step 7640/44000 | Loss: 3.2073 | LR: 4.69e-04 | 1325.3ms/step | 15.1 samples/s +Step 7650/44000 | Loss: 3.2203 | LR: 4.69e-04 | 1328.5ms/step | 15.1 samples/s +Step 7660/44000 | Loss: 3.2299 | LR: 4.69e-04 | 1325.9ms/step | 15.1 samples/s +Step 7670/44000 | Loss: 3.2110 | LR: 4.69e-04 | 1325.3ms/step | 15.1 samples/s +Step 7680/44000 | Loss: 3.2295 | LR: 4.69e-04 | 1323.8ms/step | 15.1 samples/s +Step 7690/44000 | Loss: 3.1933 | LR: 4.69e-04 | 1322.7ms/step | 15.1 samples/s +Step 7700/44000 | Loss: 3.2172 | LR: 4.69e-04 | 1323.1ms/step | 15.1 samples/s +Step 7710/44000 | Loss: 3.2407 | LR: 4.69e-04 | 1325.8ms/step | 15.1 samples/s +Step 7720/44000 | Loss: 3.2406 | LR: 4.69e-04 | 1325.9ms/step | 15.1 samples/s +Step 7730/44000 | Loss: 3.2353 | LR: 4.69e-04 | 1327.3ms/step | 15.1 samples/s +Step 7740/44000 | Loss: 3.2044 | LR: 4.68e-04 | 1330.9ms/step | 15.0 samples/s +Step 7750/44000 | Loss: 3.1827 | LR: 4.68e-04 | 1329.5ms/step | 15.0 samples/s +Validation at step 7750: Loss = 3.2206, Perplexity = 25.04 +Step 7760/44000 | Loss: 3.1926 | LR: 4.68e-04 | 3561.7ms/step | 5.6 samples/s +Step 7770/44000 | Loss: 3.1954 | LR: 4.68e-04 | 1328.8ms/step | 15.1 samples/s +Step 7780/44000 | Loss: 3.1913 | LR: 4.68e-04 | 1329.3ms/step | 15.0 samples/s +Step 7790/44000 | Loss: 3.2166 | LR: 4.68e-04 | 1330.5ms/step | 15.0 samples/s +Step 7800/44000 | Loss: 3.2066 | LR: 4.68e-04 | 1329.4ms/step | 15.0 samples/s +Step 7810/44000 | Loss: 3.1955 | LR: 4.68e-04 | 1329.3ms/step | 15.0 samples/s +Step 7820/44000 | Loss: 3.2099 | LR: 4.68e-04 | 1328.7ms/step | 15.1 samples/s +Step 7830/44000 | Loss: 3.2254 | LR: 4.68e-04 | 1326.9ms/step | 15.1 samples/s +Step 7840/44000 | Loss: 3.2034 | LR: 4.68e-04 | 1330.9ms/step | 15.0 samples/s +Step 7850/44000 | Loss: 3.1856 | LR: 4.67e-04 | 1328.9ms/step | 15.1 samples/s +Step 7860/44000 | Loss: 3.2028 | LR: 4.67e-04 | 1330.6ms/step | 15.0 samples/s +Step 7870/44000 | Loss: 3.2058 | LR: 4.67e-04 | 1329.0ms/step | 15.0 samples/s +Step 7880/44000 | Loss: 3.1660 | LR: 4.67e-04 | 1327.6ms/step | 15.1 samples/s +Step 7890/44000 | Loss: 3.1854 | LR: 4.67e-04 | 1329.2ms/step | 15.0 samples/s +Step 7900/44000 | Loss: 3.2338 | LR: 4.67e-04 | 1327.7ms/step | 15.1 samples/s +Step 7910/44000 | Loss: 3.1819 | LR: 4.67e-04 | 1328.8ms/step | 15.1 samples/s +Step 7920/44000 | Loss: 3.2180 | LR: 4.67e-04 | 1326.7ms/step | 15.1 samples/s +Step 7930/44000 | Loss: 3.1957 | LR: 4.67e-04 | 1328.6ms/step | 15.1 samples/s +Step 7940/44000 | Loss: 3.2128 | LR: 4.67e-04 | 1330.1ms/step | 15.0 samples/s +Step 7950/44000 | Loss: 3.2068 | LR: 4.67e-04 | 1333.9ms/step | 15.0 samples/s +Step 7960/44000 | Loss: 3.2003 | LR: 4.66e-04 | 1331.3ms/step | 15.0 samples/s +Step 7970/44000 | Loss: 3.1950 | LR: 4.66e-04 | 1332.3ms/step | 15.0 samples/s +Step 7980/44000 | Loss: 3.2210 | LR: 4.66e-04 | 1332.7ms/step | 15.0 samples/s +Step 7990/44000 | Loss: 3.2184 | LR: 4.66e-04 | 1333.3ms/step | 15.0 samples/s +Step 8000/44000 | Loss: 3.1902 | LR: 4.66e-04 | 1332.9ms/step | 15.0 samples/s +Validation at step 8000: Loss = 3.2128, Perplexity = 24.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-8000 +Step 8010/44000 | Loss: 3.1598 | LR: 4.66e-04 | 3912.0ms/step | 5.1 samples/s +Step 8020/44000 | Loss: 3.1815 | LR: 4.66e-04 | 1328.6ms/step | 15.1 samples/s +Step 8030/44000 | Loss: 3.1823 | LR: 4.66e-04 | 1330.6ms/step | 15.0 samples/s +Step 8040/44000 | Loss: 3.2073 | LR: 4.66e-04 | 1330.4ms/step | 15.0 samples/s +Step 8050/44000 | Loss: 3.1892 | LR: 4.66e-04 | 1331.3ms/step | 15.0 samples/s +Step 8060/44000 | Loss: 3.2236 | LR: 4.66e-04 | 1334.4ms/step | 15.0 samples/s +Step 8070/44000 | Loss: 3.1881 | LR: 4.65e-04 | 1333.6ms/step | 15.0 samples/s +Step 8080/44000 | Loss: 3.1789 | LR: 4.65e-04 | 1330.7ms/step | 15.0 samples/s +Step 8090/44000 | Loss: 3.1793 | LR: 4.65e-04 | 1332.1ms/step | 15.0 samples/s +Step 8100/44000 | Loss: 3.2031 | LR: 4.65e-04 | 1331.4ms/step | 15.0 samples/s +Step 8110/44000 | Loss: 3.1975 | LR: 4.65e-04 | 1331.7ms/step | 15.0 samples/s +Step 8120/44000 | Loss: 3.1893 | LR: 4.65e-04 | 1329.9ms/step | 15.0 samples/s +Step 8130/44000 | Loss: 3.1914 | LR: 4.65e-04 | 1331.2ms/step | 15.0 samples/s +Step 8140/44000 | Loss: 3.2262 | LR: 4.65e-04 | 1332.7ms/step | 15.0 samples/s +Step 8150/44000 | Loss: 3.1891 | LR: 4.65e-04 | 1331.6ms/step | 15.0 samples/s +Step 8160/44000 | Loss: 3.1813 | LR: 4.65e-04 | 1334.2ms/step | 15.0 samples/s +Step 8170/44000 | Loss: 3.2059 | LR: 4.65e-04 | 1332.2ms/step | 15.0 samples/s +Step 8180/44000 | Loss: 3.1940 | LR: 4.64e-04 | 1334.3ms/step | 15.0 samples/s +Step 8190/44000 | Loss: 3.1880 | LR: 4.64e-04 | 1334.9ms/step | 15.0 samples/s +Step 8200/44000 | Loss: 3.2134 | LR: 4.64e-04 | 1332.5ms/step | 15.0 samples/s +Step 8210/44000 | Loss: 3.2137 | LR: 4.64e-04 | 1332.0ms/step | 15.0 samples/s +Step 8220/44000 | Loss: 3.2189 | LR: 4.64e-04 | 1332.2ms/step | 15.0 samples/s +Step 8230/44000 | Loss: 3.1792 | LR: 4.64e-04 | 1331.4ms/step | 15.0 samples/s +Step 8240/44000 | Loss: 3.1918 | LR: 4.64e-04 | 1328.7ms/step | 15.1 samples/s +Step 8250/44000 | Loss: 3.1900 | LR: 4.64e-04 | 1332.1ms/step | 15.0 samples/s +Validation at step 8250: Loss = 3.2053, Perplexity = 24.66 +Step 8260/44000 | Loss: 3.1742 | LR: 4.64e-04 | 3565.5ms/step | 5.6 samples/s +Step 8270/44000 | Loss: 3.2170 | LR: 4.64e-04 | 1334.1ms/step | 15.0 samples/s +Step 8280/44000 | Loss: 3.1950 | LR: 4.64e-04 | 1332.1ms/step | 15.0 samples/s +Step 8290/44000 | Loss: 3.1990 | LR: 4.63e-04 | 1331.0ms/step | 15.0 samples/s +Step 8300/44000 | Loss: 3.1534 | LR: 4.63e-04 | 1333.4ms/step | 15.0 samples/s +Step 8310/44000 | Loss: 3.2059 | LR: 4.63e-04 | 1333.0ms/step | 15.0 samples/s +Step 8320/44000 | Loss: 3.1821 | LR: 4.63e-04 | 1334.7ms/step | 15.0 samples/s +Step 8330/44000 | Loss: 3.1873 | LR: 4.63e-04 | 1331.9ms/step | 15.0 samples/s +Step 8340/44000 | Loss: 3.2345 | LR: 4.63e-04 | 1333.9ms/step | 15.0 samples/s +Step 8350/44000 | Loss: 3.1820 | LR: 4.63e-04 | 1332.7ms/step | 15.0 samples/s +Step 8360/44000 | Loss: 3.2072 | LR: 4.63e-04 | 1334.1ms/step | 15.0 samples/s +Step 8370/44000 | Loss: 3.1931 | LR: 4.63e-04 | 1333.7ms/step | 15.0 samples/s +Step 8380/44000 | Loss: 3.2310 | LR: 4.63e-04 | 1331.1ms/step | 15.0 samples/s +Step 8390/44000 | Loss: 3.1657 | LR: 4.62e-04 | 1328.7ms/step | 15.1 samples/s +Step 8400/44000 | Loss: 3.1809 | LR: 4.62e-04 | 1323.0ms/step | 15.1 samples/s +Step 8410/44000 | Loss: 3.1785 | LR: 4.62e-04 | 1325.8ms/step | 15.1 samples/s +Step 8420/44000 | Loss: 3.2182 | LR: 4.62e-04 | 1327.1ms/step | 15.1 samples/s +Step 8430/44000 | Loss: 3.1821 | LR: 4.62e-04 | 1327.0ms/step | 15.1 samples/s +Step 8440/44000 | Loss: 3.2081 | LR: 4.62e-04 | 1331.3ms/step | 15.0 samples/s +Step 8450/44000 | Loss: 3.1610 | LR: 4.62e-04 | 1325.9ms/step | 15.1 samples/s +Step 8460/44000 | Loss: 3.2094 | LR: 4.62e-04 | 1324.4ms/step | 15.1 samples/s +Step 8470/44000 | Loss: 3.1568 | LR: 4.62e-04 | 1327.4ms/step | 15.1 samples/s +Step 8480/44000 | Loss: 3.1824 | LR: 4.62e-04 | 1328.4ms/step | 15.1 samples/s +Step 8490/44000 | Loss: 3.2041 | LR: 4.62e-04 | 1325.6ms/step | 15.1 samples/s +Step 8500/44000 | Loss: 3.2099 | LR: 4.61e-04 | 1326.7ms/step | 15.1 samples/s +Validation at step 8500: Loss = 3.1987, Perplexity = 24.50 +Step 8510/44000 | Loss: 3.2081 | LR: 4.61e-04 | 3553.5ms/step | 5.6 samples/s +Step 8520/44000 | Loss: 3.1816 | LR: 4.61e-04 | 1325.7ms/step | 15.1 samples/s +Step 8530/44000 | Loss: 3.2080 | LR: 4.61e-04 | 1328.7ms/step | 15.1 samples/s +Step 8540/44000 | Loss: 3.1674 | LR: 4.61e-04 | 1330.4ms/step | 15.0 samples/s +Step 8550/44000 | Loss: 3.1859 | LR: 4.61e-04 | 1330.8ms/step | 15.0 samples/s +Step 8560/44000 | Loss: 3.1926 | LR: 4.61e-04 | 1330.6ms/step | 15.0 samples/s +Step 8570/44000 | Loss: 3.1595 | LR: 4.61e-04 | 1330.9ms/step | 15.0 samples/s +Step 8580/44000 | Loss: 3.1905 | LR: 4.61e-04 | 1329.5ms/step | 15.0 samples/s +Step 8590/44000 | Loss: 3.1895 | LR: 4.61e-04 | 1327.7ms/step | 15.1 samples/s +Step 8600/44000 | Loss: 3.1792 | LR: 4.60e-04 | 1327.7ms/step | 15.1 samples/s +Step 8610/44000 | Loss: 3.1579 | LR: 4.60e-04 | 1324.2ms/step | 15.1 samples/s +Step 8620/44000 | Loss: 3.1893 | LR: 4.60e-04 | 1324.9ms/step | 15.1 samples/s +Step 8630/44000 | Loss: 3.1784 | LR: 4.60e-04 | 1326.6ms/step | 15.1 samples/s +Step 8640/44000 | Loss: 3.1850 | LR: 4.60e-04 | 1326.8ms/step | 15.1 samples/s +Step 8650/44000 | Loss: 3.1896 | LR: 4.60e-04 | 1327.9ms/step | 15.1 samples/s +Step 8660/44000 | Loss: 3.1510 | LR: 4.60e-04 | 1328.2ms/step | 15.1 samples/s +Step 8670/44000 | Loss: 3.1912 | LR: 4.60e-04 | 1330.8ms/step | 15.0 samples/s +Step 8680/44000 | Loss: 3.1911 | LR: 4.60e-04 | 1331.9ms/step | 15.0 samples/s +Step 8690/44000 | Loss: 3.1701 | LR: 4.60e-04 | 1332.2ms/step | 15.0 samples/s +Step 8700/44000 | Loss: 3.1475 | LR: 4.59e-04 | 1329.5ms/step | 15.0 samples/s +Step 8710/44000 | Loss: 3.1792 | LR: 4.59e-04 | 1329.7ms/step | 15.0 samples/s +Step 8720/44000 | Loss: 3.1857 | LR: 4.59e-04 | 1328.0ms/step | 15.1 samples/s +Step 8730/44000 | Loss: 3.1893 | LR: 4.59e-04 | 1325.2ms/step | 15.1 samples/s +Step 8740/44000 | Loss: 3.1811 | LR: 4.59e-04 | 1327.5ms/step | 15.1 samples/s +Step 8750/44000 | Loss: 3.1557 | LR: 4.59e-04 | 1327.7ms/step | 15.1 samples/s +Validation at step 8750: Loss = 3.1934, Perplexity = 24.37 +Step 8760/44000 | Loss: 3.1678 | LR: 4.59e-04 | 3556.6ms/step | 5.6 samples/s +Step 8770/44000 | Loss: 3.2069 | LR: 4.59e-04 | 1328.2ms/step | 15.1 samples/s +Step 8780/44000 | Loss: 3.1707 | LR: 4.59e-04 | 1328.5ms/step | 15.1 samples/s +Step 8790/44000 | Loss: 3.1592 | LR: 4.59e-04 | 1324.4ms/step | 15.1 samples/s +Step 8800/44000 | Loss: 3.1543 | LR: 4.58e-04 | 1327.4ms/step | 15.1 samples/s +Step 8810/44000 | Loss: 3.1761 | LR: 4.58e-04 | 1325.2ms/step | 15.1 samples/s +Step 8820/44000 | Loss: 3.1592 | LR: 4.58e-04 | 1326.8ms/step | 15.1 samples/s +Step 8830/44000 | Loss: 3.1641 | LR: 4.58e-04 | 1329.9ms/step | 15.0 samples/s +Step 8840/44000 | Loss: 3.1742 | LR: 4.58e-04 | 1330.8ms/step | 15.0 samples/s +Step 8850/44000 | Loss: 3.1742 | LR: 4.58e-04 | 1327.8ms/step | 15.1 samples/s +Step 8860/44000 | Loss: 3.1932 | LR: 4.58e-04 | 1328.4ms/step | 15.1 samples/s +Step 8870/44000 | Loss: 3.1763 | LR: 4.58e-04 | 1332.0ms/step | 15.0 samples/s +Step 8880/44000 | Loss: 3.1558 | LR: 4.58e-04 | 1328.1ms/step | 15.1 samples/s +Step 8890/44000 | Loss: 3.2018 | LR: 4.58e-04 | 1327.6ms/step | 15.1 samples/s +Step 8900/44000 | Loss: 3.1643 | LR: 4.57e-04 | 1331.6ms/step | 15.0 samples/s +Step 8910/44000 | Loss: 3.1983 | LR: 4.57e-04 | 1325.8ms/step | 15.1 samples/s +Step 8920/44000 | Loss: 3.1765 | LR: 4.57e-04 | 1327.7ms/step | 15.1 samples/s +Step 8930/44000 | Loss: 3.1753 | LR: 4.57e-04 | 1328.2ms/step | 15.1 samples/s +Step 8940/44000 | Loss: 3.1676 | LR: 4.57e-04 | 1329.5ms/step | 15.0 samples/s +Step 8950/44000 | Loss: 3.1746 | LR: 4.57e-04 | 1328.9ms/step | 15.1 samples/s +Step 8960/44000 | Loss: 3.1569 | LR: 4.57e-04 | 1333.5ms/step | 15.0 samples/s +Step 8970/44000 | Loss: 3.1791 | LR: 4.57e-04 | 1332.4ms/step | 15.0 samples/s +Step 8980/44000 | Loss: 3.1783 | LR: 4.57e-04 | 1332.9ms/step | 15.0 samples/s +Step 8990/44000 | Loss: 3.1640 | LR: 4.57e-04 | 1329.2ms/step | 15.0 samples/s +Step 9000/44000 | Loss: 3.1647 | LR: 4.56e-04 | 1334.1ms/step | 15.0 samples/s +Validation at step 9000: Loss = 3.1852, Perplexity = 24.17 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-9000 +Step 9010/44000 | Loss: 3.1653 | LR: 4.56e-04 | 3872.6ms/step | 5.2 samples/s +Step 9020/44000 | Loss: 3.1826 | LR: 4.56e-04 | 1326.9ms/step | 15.1 samples/s +Step 9030/44000 | Loss: 3.1562 | LR: 4.56e-04 | 1328.2ms/step | 15.1 samples/s +Step 9040/44000 | Loss: 3.1596 | LR: 4.56e-04 | 1330.4ms/step | 15.0 samples/s +Step 9050/44000 | Loss: 3.1620 | LR: 4.56e-04 | 1330.3ms/step | 15.0 samples/s +Step 9060/44000 | Loss: 3.1975 | LR: 4.56e-04 | 1331.8ms/step | 15.0 samples/s +Step 9070/44000 | Loss: 3.1939 | LR: 4.56e-04 | 1333.9ms/step | 15.0 samples/s +Step 9080/44000 | Loss: 3.2146 | LR: 4.56e-04 | 1332.3ms/step | 15.0 samples/s +Step 9090/44000 | Loss: 3.1447 | LR: 4.56e-04 | 1333.0ms/step | 15.0 samples/s +Step 9100/44000 | Loss: 3.1507 | LR: 4.55e-04 | 1332.0ms/step | 15.0 samples/s +Step 9110/44000 | Loss: 3.1467 | LR: 4.55e-04 | 1329.2ms/step | 15.0 samples/s +Step 9120/44000 | Loss: 3.1495 | LR: 4.55e-04 | 1332.1ms/step | 15.0 samples/s +Step 9130/44000 | Loss: 3.1942 | LR: 4.55e-04 | 1334.5ms/step | 15.0 samples/s +Step 9140/44000 | Loss: 3.1437 | LR: 4.55e-04 | 1334.0ms/step | 15.0 samples/s +Step 9150/44000 | Loss: 3.1713 | LR: 4.55e-04 | 1332.0ms/step | 15.0 samples/s +Step 9160/44000 | Loss: 3.1450 | LR: 4.55e-04 | 1330.5ms/step | 15.0 samples/s +Step 9170/44000 | Loss: 3.1280 | LR: 4.55e-04 | 1336.3ms/step | 15.0 samples/s +Step 9180/44000 | Loss: 3.1772 | LR: 4.55e-04 | 1333.7ms/step | 15.0 samples/s +Step 9190/44000 | Loss: 3.1522 | LR: 4.54e-04 | 1331.8ms/step | 15.0 samples/s +Step 9200/44000 | Loss: 3.1795 | LR: 4.54e-04 | 1333.0ms/step | 15.0 samples/s +Step 9210/44000 | Loss: 3.1739 | LR: 4.54e-04 | 1333.6ms/step | 15.0 samples/s +Step 9220/44000 | Loss: 3.1625 | LR: 4.54e-04 | 1333.2ms/step | 15.0 samples/s +Step 9230/44000 | Loss: 3.1649 | LR: 4.54e-04 | 1333.1ms/step | 15.0 samples/s +Step 9240/44000 | Loss: 3.2142 | LR: 4.54e-04 | 1333.2ms/step | 15.0 samples/s +Step 9250/44000 | Loss: 3.1674 | LR: 4.54e-04 | 1331.3ms/step | 15.0 samples/s +Validation at step 9250: Loss = 3.1780, Perplexity = 24.00 +Step 9260/44000 | Loss: 3.1743 | LR: 4.54e-04 | 3573.5ms/step | 5.6 samples/s +Step 9270/44000 | Loss: 3.1708 | LR: 4.54e-04 | 1332.7ms/step | 15.0 samples/s +Step 9280/44000 | Loss: 3.1715 | LR: 4.54e-04 | 1332.6ms/step | 15.0 samples/s +Step 9290/44000 | Loss: 3.1828 | LR: 4.53e-04 | 1333.6ms/step | 15.0 samples/s +Step 9300/44000 | Loss: 3.1421 | LR: 4.53e-04 | 1336.2ms/step | 15.0 samples/s +Step 9310/44000 | Loss: 3.1630 | LR: 4.53e-04 | 1332.5ms/step | 15.0 samples/s +Step 9320/44000 | Loss: 3.1458 | LR: 4.53e-04 | 1330.5ms/step | 15.0 samples/s +Step 9330/44000 | Loss: 3.1673 | LR: 4.53e-04 | 1331.6ms/step | 15.0 samples/s +Step 9340/44000 | Loss: 3.1493 | LR: 4.53e-04 | 1332.0ms/step | 15.0 samples/s +Step 9350/44000 | Loss: 3.1416 | LR: 4.53e-04 | 1334.5ms/step | 15.0 samples/s +Step 9360/44000 | Loss: 3.1672 | LR: 4.53e-04 | 1332.5ms/step | 15.0 samples/s +Step 9370/44000 | Loss: 3.1685 | LR: 4.53e-04 | 1336.8ms/step | 15.0 samples/s +Step 9380/44000 | Loss: 3.1647 | LR: 4.52e-04 | 1330.3ms/step | 15.0 samples/s +Step 9390/44000 | Loss: 3.1763 | LR: 4.52e-04 | 1329.5ms/step | 15.0 samples/s +Step 9400/44000 | Loss: 3.1544 | LR: 4.52e-04 | 1325.6ms/step | 15.1 samples/s +Step 9410/44000 | Loss: 3.1598 | LR: 4.52e-04 | 1323.5ms/step | 15.1 samples/s +Step 9420/44000 | Loss: 3.1710 | LR: 4.52e-04 | 1324.7ms/step | 15.1 samples/s +Step 9430/44000 | Loss: 3.1551 | LR: 4.52e-04 | 1325.2ms/step | 15.1 samples/s +Step 9440/44000 | Loss: 3.1347 | LR: 4.52e-04 | 1329.3ms/step | 15.0 samples/s +Step 9450/44000 | Loss: 3.1581 | LR: 4.52e-04 | 1332.0ms/step | 15.0 samples/s +Step 9460/44000 | Loss: 3.1700 | LR: 4.52e-04 | 1331.1ms/step | 15.0 samples/s +Step 9470/44000 | Loss: 3.1507 | LR: 4.52e-04 | 1331.9ms/step | 15.0 samples/s +Step 9480/44000 | Loss: 3.1477 | LR: 4.51e-04 | 1329.0ms/step | 15.0 samples/s +Step 9490/44000 | Loss: 3.1624 | LR: 4.51e-04 | 1328.1ms/step | 15.1 samples/s +Step 9500/44000 | Loss: 3.1636 | LR: 4.51e-04 | 1332.4ms/step | 15.0 samples/s +Validation at step 9500: Loss = 3.1722, Perplexity = 23.86 +Step 9510/44000 | Loss: 3.1947 | LR: 4.51e-04 | 3560.0ms/step | 5.6 samples/s +Step 9520/44000 | Loss: 3.1480 | LR: 4.51e-04 | 1325.5ms/step | 15.1 samples/s +Step 9530/44000 | Loss: 3.1874 | LR: 4.51e-04 | 1327.5ms/step | 15.1 samples/s +Step 9540/44000 | Loss: 3.1865 | LR: 4.51e-04 | 1328.0ms/step | 15.1 samples/s +Step 9550/44000 | Loss: 3.1715 | LR: 4.51e-04 | 1331.5ms/step | 15.0 samples/s +Step 9560/44000 | Loss: 3.1524 | LR: 4.51e-04 | 1330.2ms/step | 15.0 samples/s +Step 9570/44000 | Loss: 3.1486 | LR: 4.50e-04 | 1327.2ms/step | 15.1 samples/s +Step 9580/44000 | Loss: 3.1786 | LR: 4.50e-04 | 1325.8ms/step | 15.1 samples/s +Step 9590/44000 | Loss: 3.1434 | LR: 4.50e-04 | 1327.0ms/step | 15.1 samples/s +Step 9600/44000 | Loss: 3.1455 | LR: 4.50e-04 | 1325.6ms/step | 15.1 samples/s +Step 9610/44000 | Loss: 3.1463 | LR: 4.50e-04 | 1329.4ms/step | 15.0 samples/s +Step 9620/44000 | Loss: 3.1863 | LR: 4.50e-04 | 1330.7ms/step | 15.0 samples/s +Step 9630/44000 | Loss: 3.1635 | LR: 4.50e-04 | 1327.8ms/step | 15.1 samples/s +Step 9640/44000 | Loss: 3.1592 | LR: 4.50e-04 | 1328.7ms/step | 15.1 samples/s +Step 9650/44000 | Loss: 3.1625 | LR: 4.50e-04 | 1329.7ms/step | 15.0 samples/s +Step 9660/44000 | Loss: 3.1374 | LR: 4.49e-04 | 1330.3ms/step | 15.0 samples/s +Step 9670/44000 | Loss: 3.1602 | LR: 4.49e-04 | 1327.8ms/step | 15.1 samples/s +Step 9680/44000 | Loss: 3.1650 | LR: 4.49e-04 | 1329.8ms/step | 15.0 samples/s +Step 9690/44000 | Loss: 3.1626 | LR: 4.49e-04 | 1325.9ms/step | 15.1 samples/s +Step 9700/44000 | Loss: 3.1659 | LR: 4.49e-04 | 1325.8ms/step | 15.1 samples/s +Step 9710/44000 | Loss: 3.1645 | LR: 4.49e-04 | 1328.3ms/step | 15.1 samples/s +Step 9720/44000 | Loss: 3.1519 | LR: 4.49e-04 | 1325.1ms/step | 15.1 samples/s +Step 9730/44000 | Loss: 3.1663 | LR: 4.49e-04 | 1327.8ms/step | 15.1 samples/s +Step 9740/44000 | Loss: 3.1542 | LR: 4.49e-04 | 1328.1ms/step | 15.1 samples/s +Step 9750/44000 | Loss: 3.1479 | LR: 4.48e-04 | 1327.7ms/step | 15.1 samples/s +Validation at step 9750: Loss = 3.1663, Perplexity = 23.72 +Step 9760/44000 | Loss: 3.1253 | LR: 4.48e-04 | 3560.2ms/step | 5.6 samples/s +Step 9770/44000 | Loss: 3.1282 | LR: 4.48e-04 | 1328.0ms/step | 15.1 samples/s +Step 9780/44000 | Loss: 3.1539 | LR: 4.48e-04 | 1328.9ms/step | 15.1 samples/s +Step 9790/44000 | Loss: 3.1703 | LR: 4.48e-04 | 1330.5ms/step | 15.0 samples/s +Step 9800/44000 | Loss: 3.1535 | LR: 4.48e-04 | 1329.5ms/step | 15.0 samples/s +Step 9810/44000 | Loss: 3.1712 | LR: 4.48e-04 | 1328.2ms/step | 15.1 samples/s +Step 9820/44000 | Loss: 3.1714 | LR: 4.48e-04 | 1327.8ms/step | 15.1 samples/s +Step 9830/44000 | Loss: 3.1994 | LR: 4.48e-04 | 1327.8ms/step | 15.1 samples/s +Step 9840/44000 | Loss: 3.1449 | LR: 4.47e-04 | 1328.8ms/step | 15.1 samples/s +Step 9850/44000 | Loss: 3.1657 | LR: 4.47e-04 | 1329.1ms/step | 15.0 samples/s +Step 9860/44000 | Loss: 3.1259 | LR: 4.47e-04 | 1329.8ms/step | 15.0 samples/s +Step 9870/44000 | Loss: 3.1495 | LR: 4.47e-04 | 1330.1ms/step | 15.0 samples/s +Step 9880/44000 | Loss: 3.1528 | LR: 4.47e-04 | 1331.2ms/step | 15.0 samples/s +Step 9890/44000 | Loss: 3.1331 | LR: 4.47e-04 | 1326.9ms/step | 15.1 samples/s +Step 9900/44000 | Loss: 3.1577 | LR: 4.47e-04 | 1327.9ms/step | 15.1 samples/s +Step 9910/44000 | Loss: 3.1731 | LR: 4.47e-04 | 1326.3ms/step | 15.1 samples/s +Step 9920/44000 | Loss: 3.1392 | LR: 4.47e-04 | 1329.7ms/step | 15.0 samples/s +Step 9930/44000 | Loss: 3.1336 | LR: 4.46e-04 | 1329.7ms/step | 15.0 samples/s +Step 9940/44000 | Loss: 3.1553 | LR: 4.46e-04 | 1327.0ms/step | 15.1 samples/s +Step 9950/44000 | Loss: 3.1377 | LR: 4.46e-04 | 1326.1ms/step | 15.1 samples/s +Step 9960/44000 | Loss: 3.1507 | LR: 4.46e-04 | 1329.3ms/step | 15.0 samples/s +Step 9970/44000 | Loss: 3.1937 | LR: 4.46e-04 | 1328.3ms/step | 15.1 samples/s +Step 9980/44000 | Loss: 3.1444 | LR: 4.46e-04 | 1331.1ms/step | 15.0 samples/s +Step 9990/44000 | Loss: 3.1540 | LR: 4.46e-04 | 1333.4ms/step | 15.0 samples/s +Step 10000/44000 | Loss: 3.1505 | LR: 4.46e-04 | 1333.2ms/step | 15.0 samples/s +Validation at step 10000: Loss = 3.1605, Perplexity = 23.58 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-10000 +Step 10010/44000 | Loss: 3.1428 | LR: 4.46e-04 | 3877.0ms/step | 5.2 samples/s +Step 10020/44000 | Loss: 3.1718 | LR: 4.45e-04 | 1325.7ms/step | 15.1 samples/s +Step 10030/44000 | Loss: 3.1548 | LR: 4.45e-04 | 1324.8ms/step | 15.1 samples/s +Step 10040/44000 | Loss: 3.1234 | LR: 4.45e-04 | 1326.6ms/step | 15.1 samples/s +Step 10050/44000 | Loss: 3.1329 | LR: 4.45e-04 | 1332.0ms/step | 15.0 samples/s +Step 10060/44000 | Loss: 3.1314 | LR: 4.45e-04 | 1330.5ms/step | 15.0 samples/s +Step 10070/44000 | Loss: 3.1347 | LR: 4.45e-04 | 1330.9ms/step | 15.0 samples/s +Step 10080/44000 | Loss: 3.1437 | LR: 4.45e-04 | 1331.3ms/step | 15.0 samples/s +Step 10090/44000 | Loss: 3.1429 | LR: 4.45e-04 | 1332.0ms/step | 15.0 samples/s +Step 10100/44000 | Loss: 3.1589 | LR: 4.45e-04 | 1330.8ms/step | 15.0 samples/s +Step 10110/44000 | Loss: 3.1672 | LR: 4.44e-04 | 1332.1ms/step | 15.0 samples/s +Step 10120/44000 | Loss: 3.1902 | LR: 4.44e-04 | 1332.1ms/step | 15.0 samples/s +Step 10130/44000 | Loss: 3.1369 | LR: 4.44e-04 | 1332.2ms/step | 15.0 samples/s +Step 10140/44000 | Loss: 3.1951 | LR: 4.44e-04 | 1334.2ms/step | 15.0 samples/s +Step 10150/44000 | Loss: 3.1532 | LR: 4.44e-04 | 1331.5ms/step | 15.0 samples/s +Step 10160/44000 | Loss: 3.1547 | LR: 4.44e-04 | 1333.9ms/step | 15.0 samples/s +Step 10170/44000 | Loss: 3.1251 | LR: 4.44e-04 | 1331.7ms/step | 15.0 samples/s +Step 10180/44000 | Loss: 3.1599 | LR: 4.44e-04 | 1332.4ms/step | 15.0 samples/s +Step 10190/44000 | Loss: 3.1293 | LR: 4.43e-04 | 1332.2ms/step | 15.0 samples/s +Step 10200/44000 | Loss: 3.1321 | LR: 4.43e-04 | 1332.7ms/step | 15.0 samples/s +Step 10210/44000 | Loss: 3.1519 | LR: 4.43e-04 | 1332.4ms/step | 15.0 samples/s +Step 10220/44000 | Loss: 3.1368 | LR: 4.43e-04 | 1332.7ms/step | 15.0 samples/s +Step 10230/44000 | Loss: 3.1581 | LR: 4.43e-04 | 1334.1ms/step | 15.0 samples/s +Step 10240/44000 | Loss: 3.1348 | LR: 4.43e-04 | 1334.1ms/step | 15.0 samples/s +Step 10250/44000 | Loss: 3.1560 | LR: 4.43e-04 | 1332.7ms/step | 15.0 samples/s +Validation at step 10250: Loss = 3.1545, Perplexity = 23.44 +Step 10260/44000 | Loss: 3.1152 | LR: 4.43e-04 | 3575.7ms/step | 5.6 samples/s +Step 10270/44000 | Loss: 3.1031 | LR: 4.43e-04 | 1333.0ms/step | 15.0 samples/s +Step 10280/44000 | Loss: 3.1046 | LR: 4.42e-04 | 1335.9ms/step | 15.0 samples/s +Step 10290/44000 | Loss: 3.0986 | LR: 4.42e-04 | 1332.3ms/step | 15.0 samples/s +Step 10300/44000 | Loss: 3.1322 | LR: 4.42e-04 | 1330.2ms/step | 15.0 samples/s +Step 10310/44000 | Loss: 3.1565 | LR: 4.42e-04 | 1331.5ms/step | 15.0 samples/s +Step 10320/44000 | Loss: 3.1227 | LR: 4.42e-04 | 1332.5ms/step | 15.0 samples/s +Step 10330/44000 | Loss: 3.1657 | LR: 4.42e-04 | 1332.3ms/step | 15.0 samples/s +Step 10340/44000 | Loss: 3.1385 | LR: 4.42e-04 | 1335.1ms/step | 15.0 samples/s +Step 10350/44000 | Loss: 3.1516 | LR: 4.42e-04 | 1334.5ms/step | 15.0 samples/s +Step 10360/44000 | Loss: 3.1232 | LR: 4.42e-04 | 1333.9ms/step | 15.0 samples/s +Step 10370/44000 | Loss: 3.1497 | LR: 4.41e-04 | 1328.6ms/step | 15.1 samples/s +Step 10380/44000 | Loss: 3.1617 | LR: 4.41e-04 | 1326.9ms/step | 15.1 samples/s +Step 10390/44000 | Loss: 3.1311 | LR: 4.41e-04 | 1326.5ms/step | 15.1 samples/s +Step 10400/44000 | Loss: 3.1634 | LR: 4.41e-04 | 1326.5ms/step | 15.1 samples/s +Step 10410/44000 | Loss: 3.1701 | LR: 4.41e-04 | 1329.9ms/step | 15.0 samples/s +Step 10420/44000 | Loss: 3.1634 | LR: 4.41e-04 | 1330.0ms/step | 15.0 samples/s +Step 10430/44000 | Loss: 3.1546 | LR: 4.41e-04 | 1328.6ms/step | 15.1 samples/s +Step 10440/44000 | Loss: 3.1248 | LR: 4.41e-04 | 1330.8ms/step | 15.0 samples/s +Step 10450/44000 | Loss: 3.1374 | LR: 4.40e-04 | 1327.4ms/step | 15.1 samples/s +Step 10460/44000 | Loss: 3.1314 | LR: 4.40e-04 | 1325.3ms/step | 15.1 samples/s +Step 10470/44000 | Loss: 3.1612 | LR: 4.40e-04 | 1325.5ms/step | 15.1 samples/s +Step 10480/44000 | Loss: 3.1297 | LR: 4.40e-04 | 1326.1ms/step | 15.1 samples/s +Step 10490/44000 | Loss: 3.1100 | LR: 4.40e-04 | 1322.4ms/step | 15.1 samples/s +Step 10500/44000 | Loss: 3.1641 | LR: 4.40e-04 | 1323.8ms/step | 15.1 samples/s +Validation at step 10500: Loss = 3.1496, Perplexity = 23.33 +Step 10510/44000 | Loss: 3.1733 | LR: 4.40e-04 | 3560.9ms/step | 5.6 samples/s +Step 10520/44000 | Loss: 3.1127 | LR: 4.40e-04 | 1327.9ms/step | 15.1 samples/s +Step 10530/44000 | Loss: 3.1312 | LR: 4.40e-04 | 1329.7ms/step | 15.0 samples/s +Step 10540/44000 | Loss: 3.1406 | LR: 4.39e-04 | 1328.2ms/step | 15.1 samples/s +Step 10550/44000 | Loss: 3.1319 | LR: 4.39e-04 | 1327.6ms/step | 15.1 samples/s +Step 10560/44000 | Loss: 3.1201 | LR: 4.39e-04 | 1329.9ms/step | 15.0 samples/s +Step 10570/44000 | Loss: 3.1344 | LR: 4.39e-04 | 1332.6ms/step | 15.0 samples/s +Step 10580/44000 | Loss: 3.1508 | LR: 4.39e-04 | 1329.9ms/step | 15.0 samples/s +Step 10590/44000 | Loss: 3.1030 | LR: 4.39e-04 | 1327.9ms/step | 15.1 samples/s +Step 10600/44000 | Loss: 3.1222 | LR: 4.39e-04 | 1327.0ms/step | 15.1 samples/s +Step 10610/44000 | Loss: 3.1129 | LR: 4.39e-04 | 1326.4ms/step | 15.1 samples/s +Step 10620/44000 | Loss: 3.1577 | LR: 4.38e-04 | 1328.4ms/step | 15.1 samples/s +Step 10630/44000 | Loss: 3.1470 | LR: 4.38e-04 | 1329.5ms/step | 15.0 samples/s +Step 10640/44000 | Loss: 3.1286 | LR: 4.38e-04 | 1330.7ms/step | 15.0 samples/s +Step 10650/44000 | Loss: 3.1434 | LR: 4.38e-04 | 1327.4ms/step | 15.1 samples/s +Step 10660/44000 | Loss: 3.1289 | LR: 4.38e-04 | 1328.3ms/step | 15.1 samples/s +Step 10670/44000 | Loss: 3.1321 | LR: 4.38e-04 | 1329.3ms/step | 15.0 samples/s +Step 10680/44000 | Loss: 3.1493 | LR: 4.38e-04 | 1328.8ms/step | 15.1 samples/s +Step 10690/44000 | Loss: 3.0999 | LR: 4.38e-04 | 1328.3ms/step | 15.1 samples/s +Step 10700/44000 | Loss: 3.0734 | LR: 4.37e-04 | 1331.9ms/step | 15.0 samples/s +Step 10710/44000 | Loss: 3.1080 | LR: 4.37e-04 | 1329.4ms/step | 15.0 samples/s +Step 10720/44000 | Loss: 3.0958 | LR: 4.37e-04 | 1331.8ms/step | 15.0 samples/s +Step 10730/44000 | Loss: 3.1075 | LR: 4.37e-04 | 1330.2ms/step | 15.0 samples/s +Step 10740/44000 | Loss: 3.1193 | LR: 4.37e-04 | 1329.1ms/step | 15.0 samples/s +Step 10750/44000 | Loss: 3.1121 | LR: 4.37e-04 | 1324.2ms/step | 15.1 samples/s +Validation at step 10750: Loss = 3.1452, Perplexity = 23.22 +Step 10760/44000 | Loss: 3.1470 | LR: 4.37e-04 | 3548.7ms/step | 5.6 samples/s +Step 10770/44000 | Loss: 3.1493 | LR: 4.37e-04 | 1327.0ms/step | 15.1 samples/s +Step 10780/44000 | Loss: 3.1224 | LR: 4.37e-04 | 1327.3ms/step | 15.1 samples/s +Step 10790/44000 | Loss: 3.1303 | LR: 4.36e-04 | 1332.3ms/step | 15.0 samples/s +Step 10800/44000 | Loss: 3.1166 | LR: 4.36e-04 | 1328.9ms/step | 15.0 samples/s +Step 10810/44000 | Loss: 3.1328 | LR: 4.36e-04 | 1324.4ms/step | 15.1 samples/s +Step 10820/44000 | Loss: 3.1433 | LR: 4.36e-04 | 1326.1ms/step | 15.1 samples/s +Step 10830/44000 | Loss: 3.1246 | LR: 4.36e-04 | 1328.6ms/step | 15.1 samples/s +Step 10840/44000 | Loss: 3.1500 | LR: 4.36e-04 | 1331.6ms/step | 15.0 samples/s +Step 10850/44000 | Loss: 3.1447 | LR: 4.36e-04 | 1330.7ms/step | 15.0 samples/s +Step 10860/44000 | Loss: 3.1646 | LR: 4.36e-04 | 1326.6ms/step | 15.1 samples/s +Step 10870/44000 | Loss: 3.1272 | LR: 4.35e-04 | 1325.9ms/step | 15.1 samples/s +Step 10880/44000 | Loss: 3.1234 | LR: 4.35e-04 | 1326.0ms/step | 15.1 samples/s +Step 10890/44000 | Loss: 3.1294 | LR: 4.35e-04 | 1327.3ms/step | 15.1 samples/s +Step 10900/44000 | Loss: 3.1333 | LR: 4.35e-04 | 1333.6ms/step | 15.0 samples/s +Step 10910/44000 | Loss: 3.1099 | LR: 4.35e-04 | 1330.2ms/step | 15.0 samples/s +Step 10920/44000 | Loss: 3.0957 | LR: 4.35e-04 | 1323.6ms/step | 15.1 samples/s +Step 10930/44000 | Loss: 3.1625 | LR: 4.35e-04 | 1324.6ms/step | 15.1 samples/s +Step 10940/44000 | Loss: 3.1308 | LR: 4.35e-04 | 1329.5ms/step | 15.0 samples/s +Step 10950/44000 | Loss: 3.1123 | LR: 4.34e-04 | 1329.4ms/step | 15.0 samples/s +Step 10960/44000 | Loss: 3.1411 | LR: 4.34e-04 | 1328.3ms/step | 15.1 samples/s +Step 10970/44000 | Loss: 3.1248 | LR: 4.34e-04 | 1327.3ms/step | 15.1 samples/s +Step 10980/44000 | Loss: 3.1168 | LR: 4.34e-04 | 1329.8ms/step | 15.0 samples/s +Step 10990/44000 | Loss: 3.1308 | LR: 4.34e-04 | 1333.2ms/step | 15.0 samples/s +Step 11000/44000 | Loss: 3.1226 | LR: 4.34e-04 | 1332.4ms/step | 15.0 samples/s +Validation at step 11000: Loss = 3.1397, Perplexity = 23.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-11000 +Step 11010/44000 | Loss: 3.1381 | LR: 4.34e-04 | 3877.3ms/step | 5.2 samples/s +Step 11020/44000 | Loss: 3.1480 | LR: 4.34e-04 | 1325.0ms/step | 15.1 samples/s +Step 11030/44000 | Loss: 3.0932 | LR: 4.33e-04 | 1327.0ms/step | 15.1 samples/s +Step 11040/44000 | Loss: 3.1351 | LR: 4.33e-04 | 1329.2ms/step | 15.0 samples/s +Step 11050/44000 | Loss: 3.1168 | LR: 4.33e-04 | 1330.2ms/step | 15.0 samples/s +Step 11060/44000 | Loss: 3.1323 | LR: 4.33e-04 | 1331.7ms/step | 15.0 samples/s +Step 11070/44000 | Loss: 3.1254 | LR: 4.33e-04 | 1333.6ms/step | 15.0 samples/s +Step 11080/44000 | Loss: 3.0899 | LR: 4.33e-04 | 1331.4ms/step | 15.0 samples/s +Step 11090/44000 | Loss: 3.1482 | LR: 4.33e-04 | 1328.0ms/step | 15.1 samples/s +Step 11100/44000 | Loss: 3.1241 | LR: 4.33e-04 | 1324.6ms/step | 15.1 samples/s +Step 11110/44000 | Loss: 3.0885 | LR: 4.32e-04 | 1329.0ms/step | 15.0 samples/s +Step 11120/44000 | Loss: 3.1153 | LR: 4.32e-04 | 1329.2ms/step | 15.0 samples/s +Step 11130/44000 | Loss: 3.1119 | LR: 4.32e-04 | 1333.6ms/step | 15.0 samples/s +Step 11140/44000 | Loss: 3.1164 | LR: 4.32e-04 | 1333.2ms/step | 15.0 samples/s +Step 11150/44000 | Loss: 3.1370 | LR: 4.32e-04 | 1333.2ms/step | 15.0 samples/s +Step 11160/44000 | Loss: 3.1203 | LR: 4.32e-04 | 1333.8ms/step | 15.0 samples/s +Step 11170/44000 | Loss: 3.1178 | LR: 4.32e-04 | 1333.6ms/step | 15.0 samples/s +Step 11180/44000 | Loss: 3.1261 | LR: 4.32e-04 | 1334.3ms/step | 15.0 samples/s +Step 11190/44000 | Loss: 3.1221 | LR: 4.31e-04 | 1333.2ms/step | 15.0 samples/s +Step 11200/44000 | Loss: 3.1246 | LR: 4.31e-04 | 1333.2ms/step | 15.0 samples/s +Step 11210/44000 | Loss: 3.1264 | LR: 4.31e-04 | 1330.8ms/step | 15.0 samples/s +Step 11220/44000 | Loss: 3.1011 | LR: 4.31e-04 | 1335.2ms/step | 15.0 samples/s +Step 11230/44000 | Loss: 3.1239 | LR: 4.31e-04 | 1332.1ms/step | 15.0 samples/s +Step 11240/44000 | Loss: 3.1147 | LR: 4.31e-04 | 1331.6ms/step | 15.0 samples/s +Step 11250/44000 | Loss: 3.1199 | LR: 4.31e-04 | 1333.7ms/step | 15.0 samples/s +Validation at step 11250: Loss = 3.1354, Perplexity = 23.00 +Step 11260/44000 | Loss: 3.0952 | LR: 4.31e-04 | 3579.4ms/step | 5.6 samples/s +Step 11270/44000 | Loss: 3.1366 | LR: 4.30e-04 | 1335.3ms/step | 15.0 samples/s +Step 11280/44000 | Loss: 3.1127 | LR: 4.30e-04 | 1332.8ms/step | 15.0 samples/s +Step 11290/44000 | Loss: 3.1239 | LR: 4.30e-04 | 1332.4ms/step | 15.0 samples/s +Step 11300/44000 | Loss: 3.1413 | LR: 4.30e-04 | 1334.4ms/step | 15.0 samples/s +Step 11310/44000 | Loss: 3.1061 | LR: 4.30e-04 | 1334.2ms/step | 15.0 samples/s +Step 11320/44000 | Loss: 3.1197 | LR: 4.30e-04 | 1335.2ms/step | 15.0 samples/s +Step 11330/44000 | Loss: 3.1210 | LR: 4.30e-04 | 1334.2ms/step | 15.0 samples/s +Step 11340/44000 | Loss: 3.0914 | LR: 4.30e-04 | 1333.9ms/step | 15.0 samples/s +Step 11350/44000 | Loss: 3.1197 | LR: 4.29e-04 | 1334.4ms/step | 15.0 samples/s +Step 11360/44000 | Loss: 3.1030 | LR: 4.29e-04 | 1333.2ms/step | 15.0 samples/s +Step 11370/44000 | Loss: 3.1322 | LR: 4.29e-04 | 1332.8ms/step | 15.0 samples/s +Step 11380/44000 | Loss: 3.1322 | LR: 4.29e-04 | 1329.7ms/step | 15.0 samples/s +Step 11390/44000 | Loss: 3.1491 | LR: 4.29e-04 | 1326.8ms/step | 15.1 samples/s +Step 11400/44000 | Loss: 3.1133 | LR: 4.29e-04 | 1325.5ms/step | 15.1 samples/s +Step 11410/44000 | Loss: 3.0809 | LR: 4.29e-04 | 1327.7ms/step | 15.1 samples/s +Step 11420/44000 | Loss: 3.1319 | LR: 4.29e-04 | 1330.5ms/step | 15.0 samples/s +Step 11430/44000 | Loss: 3.1123 | LR: 4.28e-04 | 1333.1ms/step | 15.0 samples/s +Step 11440/44000 | Loss: 3.1371 | LR: 4.28e-04 | 1330.3ms/step | 15.0 samples/s +Step 11450/44000 | Loss: 3.1127 | LR: 4.28e-04 | 1330.7ms/step | 15.0 samples/s +Step 11460/44000 | Loss: 3.1472 | LR: 4.28e-04 | 1330.1ms/step | 15.0 samples/s +Step 11470/44000 | Loss: 3.1222 | LR: 4.28e-04 | 1331.1ms/step | 15.0 samples/s +Step 11480/44000 | Loss: 3.1113 | LR: 4.28e-04 | 1328.2ms/step | 15.1 samples/s +Step 11490/44000 | Loss: 3.1126 | LR: 4.28e-04 | 1330.7ms/step | 15.0 samples/s +Step 11500/44000 | Loss: 3.1148 | LR: 4.28e-04 | 1327.0ms/step | 15.1 samples/s +Validation at step 11500: Loss = 3.1296, Perplexity = 22.87 +Step 11510/44000 | Loss: 3.1312 | LR: 4.27e-04 | 3562.4ms/step | 5.6 samples/s +Step 11520/44000 | Loss: 3.0886 | LR: 4.27e-04 | 1330.5ms/step | 15.0 samples/s +Step 11530/44000 | Loss: 3.1182 | LR: 4.27e-04 | 1330.1ms/step | 15.0 samples/s +Step 11540/44000 | Loss: 3.1175 | LR: 4.27e-04 | 1331.5ms/step | 15.0 samples/s +Step 11550/44000 | Loss: 3.1008 | LR: 4.27e-04 | 1331.5ms/step | 15.0 samples/s +Step 11560/44000 | Loss: 3.1182 | LR: 4.27e-04 | 1331.4ms/step | 15.0 samples/s +Step 11570/44000 | Loss: 3.1179 | LR: 4.27e-04 | 1328.7ms/step | 15.1 samples/s +Step 11580/44000 | Loss: 3.1174 | LR: 4.27e-04 | 1328.8ms/step | 15.1 samples/s +Step 11590/44000 | Loss: 3.1268 | LR: 4.26e-04 | 1329.7ms/step | 15.0 samples/s +Step 11600/44000 | Loss: 3.1123 | LR: 4.26e-04 | 1326.6ms/step | 15.1 samples/s +Step 11610/44000 | Loss: 3.1196 | LR: 4.26e-04 | 1323.8ms/step | 15.1 samples/s +Step 11620/44000 | Loss: 3.1141 | LR: 4.26e-04 | 1330.8ms/step | 15.0 samples/s +Step 11630/44000 | Loss: 3.0939 | LR: 4.26e-04 | 1327.0ms/step | 15.1 samples/s +Step 11640/44000 | Loss: 3.1181 | LR: 4.26e-04 | 1324.1ms/step | 15.1 samples/s +Step 11650/44000 | Loss: 3.1190 | LR: 4.26e-04 | 1325.0ms/step | 15.1 samples/s +Step 11660/44000 | Loss: 3.0879 | LR: 4.26e-04 | 1328.4ms/step | 15.1 samples/s +Step 11670/44000 | Loss: 3.1121 | LR: 4.25e-04 | 1325.3ms/step | 15.1 samples/s +Step 11680/44000 | Loss: 3.0881 | LR: 4.25e-04 | 1325.9ms/step | 15.1 samples/s +Step 11690/44000 | Loss: 3.1437 | LR: 4.25e-04 | 1326.6ms/step | 15.1 samples/s +Step 11700/44000 | Loss: 3.0818 | LR: 4.25e-04 | 1327.9ms/step | 15.1 samples/s +Step 11710/44000 | Loss: 3.1337 | LR: 4.25e-04 | 1326.1ms/step | 15.1 samples/s +Step 11720/44000 | Loss: 3.1256 | LR: 4.25e-04 | 1327.8ms/step | 15.1 samples/s +Step 11730/44000 | Loss: 3.1229 | LR: 4.25e-04 | 1331.0ms/step | 15.0 samples/s +Step 11740/44000 | Loss: 3.1479 | LR: 4.24e-04 | 1326.9ms/step | 15.1 samples/s +Step 11750/44000 | Loss: 3.1096 | LR: 4.24e-04 | 1328.2ms/step | 15.1 samples/s +Validation at step 11750: Loss = 3.1256, Perplexity = 22.77 +Step 11760/44000 | Loss: 3.1451 | LR: 4.24e-04 | 3556.3ms/step | 5.6 samples/s +Step 11770/44000 | Loss: 3.1379 | LR: 4.24e-04 | 1323.8ms/step | 15.1 samples/s +Step 11780/44000 | Loss: 3.1429 | LR: 4.24e-04 | 1325.1ms/step | 15.1 samples/s +Step 11790/44000 | Loss: 3.1063 | LR: 4.24e-04 | 1327.4ms/step | 15.1 samples/s +Step 11800/44000 | Loss: 3.1004 | LR: 4.24e-04 | 1325.2ms/step | 15.1 samples/s +Step 11810/44000 | Loss: 3.0922 | LR: 4.24e-04 | 1328.0ms/step | 15.1 samples/s +Step 11820/44000 | Loss: 3.1205 | LR: 4.23e-04 | 1327.3ms/step | 15.1 samples/s +Step 11830/44000 | Loss: 3.1446 | LR: 4.23e-04 | 1325.8ms/step | 15.1 samples/s +Step 11840/44000 | Loss: 3.0857 | LR: 4.23e-04 | 1328.8ms/step | 15.1 samples/s +Step 11850/44000 | Loss: 3.1253 | LR: 4.23e-04 | 1326.4ms/step | 15.1 samples/s +Step 11860/44000 | Loss: 3.1003 | LR: 4.23e-04 | 1325.2ms/step | 15.1 samples/s +Step 11870/44000 | Loss: 3.1262 | LR: 4.23e-04 | 1325.1ms/step | 15.1 samples/s +Step 11880/44000 | Loss: 3.1103 | LR: 4.23e-04 | 1328.0ms/step | 15.1 samples/s +Step 11890/44000 | Loss: 3.0835 | LR: 4.23e-04 | 1327.0ms/step | 15.1 samples/s +Step 11900/44000 | Loss: 3.1436 | LR: 4.22e-04 | 1327.0ms/step | 15.1 samples/s +Step 11910/44000 | Loss: 3.1135 | LR: 4.22e-04 | 1330.2ms/step | 15.0 samples/s +Step 11920/44000 | Loss: 3.1111 | LR: 4.22e-04 | 1328.7ms/step | 15.1 samples/s +Step 11930/44000 | Loss: 3.1154 | LR: 4.22e-04 | 1332.0ms/step | 15.0 samples/s +Step 11940/44000 | Loss: 3.1016 | LR: 4.22e-04 | 1329.7ms/step | 15.0 samples/s +Step 11950/44000 | Loss: 3.0806 | LR: 4.22e-04 | 1330.2ms/step | 15.0 samples/s +Step 11960/44000 | Loss: 3.0976 | LR: 4.22e-04 | 1328.0ms/step | 15.1 samples/s +Step 11970/44000 | Loss: 3.1273 | LR: 4.21e-04 | 1331.0ms/step | 15.0 samples/s +Step 11980/44000 | Loss: 3.1018 | LR: 4.21e-04 | 1331.1ms/step | 15.0 samples/s +Step 11990/44000 | Loss: 3.0994 | LR: 4.21e-04 | 1332.6ms/step | 15.0 samples/s +Step 12000/44000 | Loss: 3.1160 | LR: 4.21e-04 | 1335.1ms/step | 15.0 samples/s +Validation at step 12000: Loss = 3.1201, Perplexity = 22.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-12000 +Step 12010/44000 | Loss: 3.1012 | LR: 4.21e-04 | 3873.1ms/step | 5.2 samples/s +Step 12020/44000 | Loss: 3.0823 | LR: 4.21e-04 | 1331.2ms/step | 15.0 samples/s +Step 12030/44000 | Loss: 3.1188 | LR: 4.21e-04 | 1330.6ms/step | 15.0 samples/s +Step 12040/44000 | Loss: 3.1210 | LR: 4.21e-04 | 1329.4ms/step | 15.0 samples/s +Step 12050/44000 | Loss: 3.1057 | LR: 4.20e-04 | 1333.7ms/step | 15.0 samples/s +Step 12060/44000 | Loss: 3.1006 | LR: 4.20e-04 | 1332.3ms/step | 15.0 samples/s +Step 12070/44000 | Loss: 3.1401 | LR: 4.20e-04 | 1331.9ms/step | 15.0 samples/s +Step 12080/44000 | Loss: 3.1126 | LR: 4.20e-04 | 1329.3ms/step | 15.0 samples/s +Step 12090/44000 | Loss: 3.1262 | LR: 4.20e-04 | 1334.1ms/step | 15.0 samples/s +Step 12100/44000 | Loss: 3.1175 | LR: 4.20e-04 | 1329.5ms/step | 15.0 samples/s +Step 12110/44000 | Loss: 3.1163 | LR: 4.20e-04 | 1328.7ms/step | 15.1 samples/s +Step 12120/44000 | Loss: 3.1172 | LR: 4.19e-04 | 1332.2ms/step | 15.0 samples/s +Step 12130/44000 | Loss: 3.1320 | LR: 4.19e-04 | 1332.5ms/step | 15.0 samples/s +Step 12140/44000 | Loss: 3.0813 | LR: 4.19e-04 | 1332.7ms/step | 15.0 samples/s +Step 12150/44000 | Loss: 3.0955 | LR: 4.19e-04 | 1335.1ms/step | 15.0 samples/s +Step 12160/44000 | Loss: 3.1276 | LR: 4.19e-04 | 1331.8ms/step | 15.0 samples/s +Step 12170/44000 | Loss: 3.1140 | LR: 4.19e-04 | 1332.7ms/step | 15.0 samples/s +Step 12180/44000 | Loss: 3.0915 | LR: 4.19e-04 | 1331.3ms/step | 15.0 samples/s +Step 12190/44000 | Loss: 3.1060 | LR: 4.19e-04 | 1334.6ms/step | 15.0 samples/s +Step 12200/44000 | Loss: 3.0723 | LR: 4.18e-04 | 1333.5ms/step | 15.0 samples/s +Step 12210/44000 | Loss: 3.1029 | LR: 4.18e-04 | 1333.2ms/step | 15.0 samples/s +Step 12220/44000 | Loss: 3.0791 | LR: 4.18e-04 | 1334.0ms/step | 15.0 samples/s +Step 12230/44000 | Loss: 3.0783 | LR: 4.18e-04 | 1336.3ms/step | 15.0 samples/s +Step 12240/44000 | Loss: 3.1326 | LR: 4.18e-04 | 1333.2ms/step | 15.0 samples/s +Step 12250/44000 | Loss: 3.1137 | LR: 4.18e-04 | 1333.7ms/step | 15.0 samples/s +Validation at step 12250: Loss = 3.1178, Perplexity = 22.60 +Step 12260/44000 | Loss: 3.0909 | LR: 4.18e-04 | 3568.5ms/step | 5.6 samples/s +Step 12270/44000 | Loss: 3.1182 | LR: 4.17e-04 | 1334.3ms/step | 15.0 samples/s +Step 12280/44000 | Loss: 3.0960 | LR: 4.17e-04 | 1332.5ms/step | 15.0 samples/s +Step 12290/44000 | Loss: 3.1139 | LR: 4.17e-04 | 1334.8ms/step | 15.0 samples/s +Step 12300/44000 | Loss: 3.1143 | LR: 4.17e-04 | 1334.3ms/step | 15.0 samples/s +Step 12310/44000 | Loss: 3.1202 | LR: 4.17e-04 | 1333.0ms/step | 15.0 samples/s +Step 12320/44000 | Loss: 3.0987 | LR: 4.17e-04 | 1332.3ms/step | 15.0 samples/s +Step 12330/44000 | Loss: 3.1255 | LR: 4.17e-04 | 1331.9ms/step | 15.0 samples/s +Step 12340/44000 | Loss: 3.1065 | LR: 4.17e-04 | 1333.1ms/step | 15.0 samples/s +Step 12350/44000 | Loss: 3.1234 | LR: 4.16e-04 | 1330.0ms/step | 15.0 samples/s +Step 12360/44000 | Loss: 3.1061 | LR: 4.16e-04 | 1328.6ms/step | 15.1 samples/s +Step 12370/44000 | Loss: 3.1018 | LR: 4.16e-04 | 1329.4ms/step | 15.0 samples/s +Step 12380/44000 | Loss: 3.0726 | LR: 4.16e-04 | 1330.1ms/step | 15.0 samples/s +Step 12390/44000 | Loss: 3.1081 | LR: 4.16e-04 | 1327.1ms/step | 15.1 samples/s +Step 12400/44000 | Loss: 3.0731 | LR: 4.16e-04 | 1332.3ms/step | 15.0 samples/s +Step 12410/44000 | Loss: 3.0967 | LR: 4.16e-04 | 1332.6ms/step | 15.0 samples/s +Step 12420/44000 | Loss: 3.1114 | LR: 4.15e-04 | 1331.5ms/step | 15.0 samples/s +Step 12430/44000 | Loss: 3.0932 | LR: 4.15e-04 | 1330.2ms/step | 15.0 samples/s +Step 12440/44000 | Loss: 3.1059 | LR: 4.15e-04 | 1327.6ms/step | 15.1 samples/s +Step 12450/44000 | Loss: 3.1002 | LR: 4.15e-04 | 1329.1ms/step | 15.0 samples/s +Step 12460/44000 | Loss: 3.1076 | LR: 4.15e-04 | 1332.7ms/step | 15.0 samples/s +Step 12470/44000 | Loss: 3.0991 | LR: 4.15e-04 | 1330.5ms/step | 15.0 samples/s +Step 12480/44000 | Loss: 3.0922 | LR: 4.15e-04 | 1327.5ms/step | 15.1 samples/s +Step 12490/44000 | Loss: 3.0758 | LR: 4.14e-04 | 1326.8ms/step | 15.1 samples/s +Step 12500/44000 | Loss: 3.0690 | LR: 4.14e-04 | 1330.7ms/step | 15.0 samples/s +Validation at step 12500: Loss = 3.1133, Perplexity = 22.50 +Step 12510/44000 | Loss: 3.1230 | LR: 4.14e-04 | 3559.7ms/step | 5.6 samples/s +Step 12520/44000 | Loss: 3.1050 | LR: 4.14e-04 | 1321.5ms/step | 15.1 samples/s +Step 12530/44000 | Loss: 3.0996 | LR: 4.14e-04 | 1328.3ms/step | 15.1 samples/s +Step 12540/44000 | Loss: 3.0998 | LR: 4.14e-04 | 1328.7ms/step | 15.1 samples/s +Step 12550/44000 | Loss: 3.1153 | LR: 4.14e-04 | 1327.0ms/step | 15.1 samples/s +Step 12560/44000 | Loss: 3.0789 | LR: 4.14e-04 | 1323.3ms/step | 15.1 samples/s +Step 12570/44000 | Loss: 3.0913 | LR: 4.13e-04 | 1321.8ms/step | 15.1 samples/s +Step 12580/44000 | Loss: 3.1353 | LR: 4.13e-04 | 1326.5ms/step | 15.1 samples/s +Step 12590/44000 | Loss: 3.0937 | LR: 4.13e-04 | 1328.5ms/step | 15.1 samples/s +Step 12600/44000 | Loss: 3.0990 | LR: 4.13e-04 | 1329.7ms/step | 15.0 samples/s +Step 12610/44000 | Loss: 3.1130 | LR: 4.13e-04 | 1329.7ms/step | 15.0 samples/s +Step 12620/44000 | Loss: 3.1450 | LR: 4.13e-04 | 1328.6ms/step | 15.1 samples/s +Step 12630/44000 | Loss: 3.1066 | LR: 4.13e-04 | 1329.4ms/step | 15.0 samples/s +Step 12640/44000 | Loss: 3.0975 | LR: 4.12e-04 | 1329.0ms/step | 15.0 samples/s +Step 12650/44000 | Loss: 3.1074 | LR: 4.12e-04 | 1325.1ms/step | 15.1 samples/s +Step 12660/44000 | Loss: 3.0962 | LR: 4.12e-04 | 1328.0ms/step | 15.1 samples/s +Step 12670/44000 | Loss: 3.1089 | LR: 4.12e-04 | 1329.0ms/step | 15.0 samples/s +Step 12680/44000 | Loss: 3.1126 | LR: 4.12e-04 | 1331.1ms/step | 15.0 samples/s +Step 12690/44000 | Loss: 3.1019 | LR: 4.12e-04 | 1329.1ms/step | 15.0 samples/s +Step 12700/44000 | Loss: 3.0956 | LR: 4.12e-04 | 1326.7ms/step | 15.1 samples/s +Step 12710/44000 | Loss: 3.0636 | LR: 4.11e-04 | 1326.3ms/step | 15.1 samples/s +Step 12720/44000 | Loss: 3.0617 | LR: 4.11e-04 | 1329.0ms/step | 15.0 samples/s +Step 12730/44000 | Loss: 3.0884 | LR: 4.11e-04 | 1329.2ms/step | 15.0 samples/s +Step 12740/44000 | Loss: 3.1064 | LR: 4.11e-04 | 1330.1ms/step | 15.0 samples/s +Step 12750/44000 | Loss: 3.0679 | LR: 4.11e-04 | 1328.2ms/step | 15.1 samples/s +Validation at step 12750: Loss = 3.1076, Perplexity = 22.37 +Step 12760/44000 | Loss: 3.0938 | LR: 4.11e-04 | 3552.3ms/step | 5.6 samples/s +Step 12770/44000 | Loss: 3.1032 | LR: 4.11e-04 | 1325.5ms/step | 15.1 samples/s +Step 12780/44000 | Loss: 3.0841 | LR: 4.10e-04 | 1330.3ms/step | 15.0 samples/s +Step 12790/44000 | Loss: 3.0885 | LR: 4.10e-04 | 1331.0ms/step | 15.0 samples/s +Step 12800/44000 | Loss: 3.1078 | LR: 4.10e-04 | 1331.2ms/step | 15.0 samples/s +Step 12810/44000 | Loss: 3.1073 | LR: 4.10e-04 | 1327.6ms/step | 15.1 samples/s +Step 12820/44000 | Loss: 3.0851 | LR: 4.10e-04 | 1326.9ms/step | 15.1 samples/s +Step 12830/44000 | Loss: 3.0969 | LR: 4.10e-04 | 1329.8ms/step | 15.0 samples/s +Step 12840/44000 | Loss: 3.0770 | LR: 4.10e-04 | 1325.1ms/step | 15.1 samples/s +Step 12850/44000 | Loss: 3.0960 | LR: 4.09e-04 | 1324.7ms/step | 15.1 samples/s +Step 12860/44000 | Loss: 3.0726 | LR: 4.09e-04 | 1324.9ms/step | 15.1 samples/s +Step 12870/44000 | Loss: 3.1017 | LR: 4.09e-04 | 1329.5ms/step | 15.0 samples/s +Step 12880/44000 | Loss: 3.0634 | LR: 4.09e-04 | 1327.9ms/step | 15.1 samples/s +Step 12890/44000 | Loss: 3.1163 | LR: 4.09e-04 | 1326.9ms/step | 15.1 samples/s +Step 12900/44000 | Loss: 3.0788 | LR: 4.09e-04 | 1325.1ms/step | 15.1 samples/s +Step 12910/44000 | Loss: 3.0959 | LR: 4.09e-04 | 1325.9ms/step | 15.1 samples/s +Step 12920/44000 | Loss: 3.0950 | LR: 4.09e-04 | 1328.3ms/step | 15.1 samples/s +Step 12930/44000 | Loss: 3.0968 | LR: 4.08e-04 | 1328.4ms/step | 15.1 samples/s +Step 12940/44000 | Loss: 3.1112 | LR: 4.08e-04 | 1333.3ms/step | 15.0 samples/s +Step 12950/44000 | Loss: 3.0958 | LR: 4.08e-04 | 1331.0ms/step | 15.0 samples/s +Step 12960/44000 | Loss: 3.1015 | LR: 4.08e-04 | 1330.2ms/step | 15.0 samples/s +Step 12970/44000 | Loss: 3.1097 | LR: 4.08e-04 | 1335.0ms/step | 15.0 samples/s +Step 12980/44000 | Loss: 3.1053 | LR: 4.08e-04 | 1336.1ms/step | 15.0 samples/s +Step 12990/44000 | Loss: 3.0991 | LR: 4.08e-04 | 1335.6ms/step | 15.0 samples/s +Step 13000/44000 | Loss: 3.0851 | LR: 4.07e-04 | 1336.5ms/step | 15.0 samples/s +Validation at step 13000: Loss = 3.1041, Perplexity = 22.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-13000 +Step 13010/44000 | Loss: 3.0919 | LR: 4.07e-04 | 3878.2ms/step | 5.2 samples/s +Step 13020/44000 | Loss: 3.0919 | LR: 4.07e-04 | 1327.0ms/step | 15.1 samples/s +Step 13030/44000 | Loss: 3.0983 | LR: 4.07e-04 | 1329.7ms/step | 15.0 samples/s +Step 13040/44000 | Loss: 3.0740 | LR: 4.07e-04 | 1332.8ms/step | 15.0 samples/s +Step 13050/44000 | Loss: 3.0587 | LR: 4.07e-04 | 1330.8ms/step | 15.0 samples/s +Step 13060/44000 | Loss: 3.1049 | LR: 4.07e-04 | 1333.5ms/step | 15.0 samples/s +Step 13070/44000 | Loss: 3.0963 | LR: 4.06e-04 | 1334.6ms/step | 15.0 samples/s +Step 13080/44000 | Loss: 3.1134 | LR: 4.06e-04 | 1333.3ms/step | 15.0 samples/s +Step 13090/44000 | Loss: 3.0823 | LR: 4.06e-04 | 1331.0ms/step | 15.0 samples/s +Step 13100/44000 | Loss: 3.0851 | LR: 4.06e-04 | 1329.2ms/step | 15.0 samples/s +Step 13110/44000 | Loss: 3.0984 | LR: 4.06e-04 | 1329.4ms/step | 15.0 samples/s +Step 13120/44000 | Loss: 3.1069 | LR: 4.06e-04 | 1329.0ms/step | 15.0 samples/s +Step 13130/44000 | Loss: 3.1079 | LR: 4.06e-04 | 1335.4ms/step | 15.0 samples/s +Step 13140/44000 | Loss: 3.1080 | LR: 4.05e-04 | 1334.1ms/step | 15.0 samples/s +Step 13150/44000 | Loss: 3.0908 | LR: 4.05e-04 | 1330.7ms/step | 15.0 samples/s +Step 13160/44000 | Loss: 3.0827 | LR: 4.05e-04 | 1330.2ms/step | 15.0 samples/s +Step 13170/44000 | Loss: 3.0871 | LR: 4.05e-04 | 1334.0ms/step | 15.0 samples/s +Step 13180/44000 | Loss: 3.1088 | LR: 4.05e-04 | 1333.2ms/step | 15.0 samples/s +Step 13190/44000 | Loss: 3.0843 | LR: 4.05e-04 | 1332.8ms/step | 15.0 samples/s +Step 13200/44000 | Loss: 3.1175 | LR: 4.05e-04 | 1331.6ms/step | 15.0 samples/s +Step 13210/44000 | Loss: 3.0720 | LR: 4.04e-04 | 1332.7ms/step | 15.0 samples/s +Step 13220/44000 | Loss: 3.0757 | LR: 4.04e-04 | 1333.4ms/step | 15.0 samples/s +Step 13230/44000 | Loss: 3.1043 | LR: 4.04e-04 | 1333.7ms/step | 15.0 samples/s +Step 13240/44000 | Loss: 3.0848 | LR: 4.04e-04 | 1331.7ms/step | 15.0 samples/s +Step 13250/44000 | Loss: 3.1013 | LR: 4.04e-04 | 1333.5ms/step | 15.0 samples/s +Validation at step 13250: Loss = 3.1000, Perplexity = 22.20 +Step 13260/44000 | Loss: 3.0926 | LR: 4.04e-04 | 3571.7ms/step | 5.6 samples/s +Step 13270/44000 | Loss: 3.0837 | LR: 4.04e-04 | 1332.2ms/step | 15.0 samples/s +Step 13280/44000 | Loss: 3.1096 | LR: 4.03e-04 | 1332.6ms/step | 15.0 samples/s +Step 13290/44000 | Loss: 3.0947 | LR: 4.03e-04 | 1333.4ms/step | 15.0 samples/s +Step 13300/44000 | Loss: 3.0579 | LR: 4.03e-04 | 1334.7ms/step | 15.0 samples/s +Step 13310/44000 | Loss: 3.0889 | LR: 4.03e-04 | 1333.7ms/step | 15.0 samples/s +Step 13320/44000 | Loss: 3.0698 | LR: 4.03e-04 | 1334.4ms/step | 15.0 samples/s +Step 13330/44000 | Loss: 3.0788 | LR: 4.03e-04 | 1333.5ms/step | 15.0 samples/s +Step 13340/44000 | Loss: 3.0870 | LR: 4.03e-04 | 1331.3ms/step | 15.0 samples/s +Step 13350/44000 | Loss: 3.0726 | LR: 4.02e-04 | 1331.0ms/step | 15.0 samples/s +Step 13360/44000 | Loss: 3.0866 | LR: 4.02e-04 | 1331.1ms/step | 15.0 samples/s +Step 13370/44000 | Loss: 3.1171 | LR: 4.02e-04 | 1331.7ms/step | 15.0 samples/s +Step 13380/44000 | Loss: 3.1089 | LR: 4.02e-04 | 1329.4ms/step | 15.0 samples/s +Step 13390/44000 | Loss: 3.0784 | LR: 4.02e-04 | 1327.6ms/step | 15.1 samples/s +Step 13400/44000 | Loss: 3.0794 | LR: 4.02e-04 | 1325.0ms/step | 15.1 samples/s +Step 13410/44000 | Loss: 3.0713 | LR: 4.02e-04 | 1322.9ms/step | 15.1 samples/s +Step 13420/44000 | Loss: 3.0420 | LR: 4.01e-04 | 1324.4ms/step | 15.1 samples/s +Step 13430/44000 | Loss: 3.0976 | LR: 4.01e-04 | 1327.5ms/step | 15.1 samples/s +Step 13440/44000 | Loss: 3.0879 | LR: 4.01e-04 | 1327.8ms/step | 15.1 samples/s +Step 13450/44000 | Loss: 3.0981 | LR: 4.01e-04 | 1327.7ms/step | 15.1 samples/s +Step 13460/44000 | Loss: 3.0763 | LR: 4.01e-04 | 1330.0ms/step | 15.0 samples/s +Step 13470/44000 | Loss: 3.0887 | LR: 4.01e-04 | 1329.9ms/step | 15.0 samples/s +Step 13480/44000 | Loss: 3.0634 | LR: 4.01e-04 | 1327.3ms/step | 15.1 samples/s +Step 13490/44000 | Loss: 3.0900 | LR: 4.00e-04 | 1324.8ms/step | 15.1 samples/s +Step 13500/44000 | Loss: 3.0920 | LR: 4.00e-04 | 1328.8ms/step | 15.1 samples/s +Validation at step 13500: Loss = 3.0959, Perplexity = 22.11 +Step 13510/44000 | Loss: 3.0776 | LR: 4.00e-04 | 3550.0ms/step | 5.6 samples/s +Step 13520/44000 | Loss: 3.0749 | LR: 4.00e-04 | 1325.9ms/step | 15.1 samples/s +Step 13530/44000 | Loss: 3.0723 | LR: 4.00e-04 | 1328.5ms/step | 15.1 samples/s +Step 13540/44000 | Loss: 3.0630 | LR: 4.00e-04 | 1326.9ms/step | 15.1 samples/s +Step 13550/44000 | Loss: 3.0813 | LR: 4.00e-04 | 1325.7ms/step | 15.1 samples/s +Step 13560/44000 | Loss: 3.0730 | LR: 3.99e-04 | 1325.3ms/step | 15.1 samples/s +Step 13570/44000 | Loss: 3.1096 | LR: 3.99e-04 | 1326.6ms/step | 15.1 samples/s +Step 13580/44000 | Loss: 3.0878 | LR: 3.99e-04 | 1322.2ms/step | 15.1 samples/s +Step 13590/44000 | Loss: 3.0654 | LR: 3.99e-04 | 1328.3ms/step | 15.1 samples/s +Step 13600/44000 | Loss: 3.0781 | LR: 3.99e-04 | 1330.3ms/step | 15.0 samples/s +Step 13610/44000 | Loss: 3.0685 | LR: 3.99e-04 | 1326.8ms/step | 15.1 samples/s +Step 13620/44000 | Loss: 3.1003 | LR: 3.98e-04 | 1325.8ms/step | 15.1 samples/s +Step 13630/44000 | Loss: 3.0914 | LR: 3.98e-04 | 1328.4ms/step | 15.1 samples/s +Step 13640/44000 | Loss: 3.0670 | LR: 3.98e-04 | 1329.0ms/step | 15.0 samples/s +Step 13650/44000 | Loss: 3.0842 | LR: 3.98e-04 | 1326.8ms/step | 15.1 samples/s +Step 13660/44000 | Loss: 3.0720 | LR: 3.98e-04 | 1326.1ms/step | 15.1 samples/s +Step 13670/44000 | Loss: 3.0581 | LR: 3.98e-04 | 1325.3ms/step | 15.1 samples/s +Step 13680/44000 | Loss: 3.0715 | LR: 3.98e-04 | 1325.2ms/step | 15.1 samples/s +Step 13690/44000 | Loss: 3.0886 | LR: 3.97e-04 | 1328.9ms/step | 15.1 samples/s +Step 13700/44000 | Loss: 3.0746 | LR: 3.97e-04 | 1328.1ms/step | 15.1 samples/s +Step 13710/44000 | Loss: 3.0837 | LR: 3.97e-04 | 1328.6ms/step | 15.1 samples/s +Step 13720/44000 | Loss: 3.0658 | LR: 3.97e-04 | 1324.6ms/step | 15.1 samples/s +Step 13730/44000 | Loss: 3.0849 | LR: 3.97e-04 | 1329.2ms/step | 15.0 samples/s +Step 13740/44000 | Loss: 3.0910 | LR: 3.97e-04 | 1325.8ms/step | 15.1 samples/s +Step 13750/44000 | Loss: 3.1215 | LR: 3.97e-04 | 1329.2ms/step | 15.0 samples/s +Validation at step 13750: Loss = 3.0936, Perplexity = 22.06 +Step 13760/44000 | Loss: 3.1117 | LR: 3.96e-04 | 3553.2ms/step | 5.6 samples/s +Step 13770/44000 | Loss: 3.0814 | LR: 3.96e-04 | 1328.3ms/step | 15.1 samples/s +Step 13780/44000 | Loss: 3.0723 | LR: 3.96e-04 | 1328.0ms/step | 15.1 samples/s +Step 13790/44000 | Loss: 3.1102 | LR: 3.96e-04 | 1330.1ms/step | 15.0 samples/s +Step 13800/44000 | Loss: 3.0689 | LR: 3.96e-04 | 1332.6ms/step | 15.0 samples/s +Step 13810/44000 | Loss: 3.0623 | LR: 3.96e-04 | 1329.9ms/step | 15.0 samples/s +Step 13820/44000 | Loss: 3.0779 | LR: 3.96e-04 | 1328.0ms/step | 15.1 samples/s +Step 13830/44000 | Loss: 3.0621 | LR: 3.95e-04 | 1329.5ms/step | 15.0 samples/s +Step 13840/44000 | Loss: 3.0530 | LR: 3.95e-04 | 1328.8ms/step | 15.1 samples/s +Step 13850/44000 | Loss: 3.0730 | LR: 3.95e-04 | 1329.3ms/step | 15.0 samples/s +Step 13860/44000 | Loss: 3.0798 | LR: 3.95e-04 | 1330.9ms/step | 15.0 samples/s +Step 13870/44000 | Loss: 3.0793 | LR: 3.95e-04 | 1331.5ms/step | 15.0 samples/s +Step 13880/44000 | Loss: 3.0972 | LR: 3.95e-04 | 1329.8ms/step | 15.0 samples/s +Step 13890/44000 | Loss: 3.0636 | LR: 3.95e-04 | 1326.1ms/step | 15.1 samples/s +Step 13900/44000 | Loss: 3.1130 | LR: 3.94e-04 | 1325.1ms/step | 15.1 samples/s +Step 13910/44000 | Loss: 3.0861 | LR: 3.94e-04 | 1327.5ms/step | 15.1 samples/s +Step 13920/44000 | Loss: 3.0661 | LR: 3.94e-04 | 1329.0ms/step | 15.0 samples/s +Step 13930/44000 | Loss: 3.0802 | LR: 3.94e-04 | 1332.2ms/step | 15.0 samples/s +Step 13940/44000 | Loss: 3.0605 | LR: 3.94e-04 | 1334.0ms/step | 15.0 samples/s +Step 13950/44000 | Loss: 3.0615 | LR: 3.94e-04 | 1334.7ms/step | 15.0 samples/s +Step 13960/44000 | Loss: 3.0812 | LR: 3.93e-04 | 1331.6ms/step | 15.0 samples/s +Step 13970/44000 | Loss: 3.0659 | LR: 3.93e-04 | 1330.8ms/step | 15.0 samples/s +Step 13980/44000 | Loss: 3.0659 | LR: 3.93e-04 | 1330.1ms/step | 15.0 samples/s +Step 13990/44000 | Loss: 3.0647 | LR: 3.93e-04 | 1332.0ms/step | 15.0 samples/s +Step 14000/44000 | Loss: 3.0852 | LR: 3.93e-04 | 1334.0ms/step | 15.0 samples/s +Validation at step 14000: Loss = 3.0882, Perplexity = 21.94 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-14000 +Step 14010/44000 | Loss: 3.0880 | LR: 3.93e-04 | 3865.5ms/step | 5.2 samples/s +Step 14020/44000 | Loss: 3.0614 | LR: 3.93e-04 | 1327.8ms/step | 15.1 samples/s +Step 14030/44000 | Loss: 3.0297 | LR: 3.92e-04 | 1329.4ms/step | 15.0 samples/s +Step 14040/44000 | Loss: 3.0478 | LR: 3.92e-04 | 1331.6ms/step | 15.0 samples/s +Step 14050/44000 | Loss: 3.0392 | LR: 3.92e-04 | 1335.8ms/step | 15.0 samples/s +Step 14060/44000 | Loss: 3.0961 | LR: 3.92e-04 | 1336.7ms/step | 15.0 samples/s +Step 14070/44000 | Loss: 3.0883 | LR: 3.92e-04 | 1334.9ms/step | 15.0 samples/s +Step 14080/44000 | Loss: 3.0799 | LR: 3.92e-04 | 1334.3ms/step | 15.0 samples/s +Step 14090/44000 | Loss: 3.0612 | LR: 3.92e-04 | 1335.2ms/step | 15.0 samples/s +Step 14100/44000 | Loss: 3.0895 | LR: 3.91e-04 | 1333.5ms/step | 15.0 samples/s +Step 14110/44000 | Loss: 3.0838 | LR: 3.91e-04 | 1335.2ms/step | 15.0 samples/s +Step 14120/44000 | Loss: 3.0762 | LR: 3.91e-04 | 1333.9ms/step | 15.0 samples/s +Step 14130/44000 | Loss: 3.0987 | LR: 3.91e-04 | 1336.7ms/step | 15.0 samples/s +Step 14140/44000 | Loss: 3.0854 | LR: 3.91e-04 | 1336.2ms/step | 15.0 samples/s +Step 14150/44000 | Loss: 3.0560 | LR: 3.91e-04 | 1335.4ms/step | 15.0 samples/s +Step 14160/44000 | Loss: 3.0680 | LR: 3.90e-04 | 1336.8ms/step | 15.0 samples/s +Step 14170/44000 | Loss: 3.0907 | LR: 3.90e-04 | 1333.3ms/step | 15.0 samples/s +Step 14180/44000 | Loss: 3.0796 | LR: 3.90e-04 | 1336.8ms/step | 15.0 samples/s +Step 14190/44000 | Loss: 3.0745 | LR: 3.90e-04 | 1335.1ms/step | 15.0 samples/s +Step 14200/44000 | Loss: 3.0843 | LR: 3.90e-04 | 1334.8ms/step | 15.0 samples/s +Step 14210/44000 | Loss: 3.0702 | LR: 3.90e-04 | 1336.7ms/step | 15.0 samples/s +Step 14220/44000 | Loss: 3.0692 | LR: 3.90e-04 | 1332.2ms/step | 15.0 samples/s +Step 14230/44000 | Loss: 3.0755 | LR: 3.89e-04 | 1334.1ms/step | 15.0 samples/s +Step 14240/44000 | Loss: 3.0548 | LR: 3.89e-04 | 1333.7ms/step | 15.0 samples/s +Step 14250/44000 | Loss: 3.0694 | LR: 3.89e-04 | 1336.0ms/step | 15.0 samples/s +Validation at step 14250: Loss = 3.0846, Perplexity = 21.86 +Step 14260/44000 | Loss: 3.1011 | LR: 3.89e-04 | 3571.1ms/step | 5.6 samples/s +Step 14270/44000 | Loss: 3.0940 | LR: 3.89e-04 | 1335.1ms/step | 15.0 samples/s +Step 14280/44000 | Loss: 3.0901 | LR: 3.89e-04 | 1335.8ms/step | 15.0 samples/s +Step 14290/44000 | Loss: 3.0785 | LR: 3.89e-04 | 1333.9ms/step | 15.0 samples/s +Step 14300/44000 | Loss: 3.0773 | LR: 3.88e-04 | 1332.7ms/step | 15.0 samples/s +Step 14310/44000 | Loss: 3.0755 | LR: 3.88e-04 | 1336.2ms/step | 15.0 samples/s +Step 14320/44000 | Loss: 3.0726 | LR: 3.88e-04 | 1334.9ms/step | 15.0 samples/s +Step 14330/44000 | Loss: 3.0874 | LR: 3.88e-04 | 1333.2ms/step | 15.0 samples/s +Step 14340/44000 | Loss: 3.0838 | LR: 3.88e-04 | 1328.3ms/step | 15.1 samples/s +Step 14350/44000 | Loss: 3.0684 | LR: 3.88e-04 | 1327.3ms/step | 15.1 samples/s +Step 14360/44000 | Loss: 3.0839 | LR: 3.87e-04 | 1326.8ms/step | 15.1 samples/s +Step 14370/44000 | Loss: 3.0826 | LR: 3.87e-04 | 1325.6ms/step | 15.1 samples/s +Step 14380/44000 | Loss: 3.0638 | LR: 3.87e-04 | 1327.6ms/step | 15.1 samples/s +Step 14390/44000 | Loss: 3.0686 | LR: 3.87e-04 | 1328.8ms/step | 15.1 samples/s +Step 14400/44000 | Loss: 3.0708 | LR: 3.87e-04 | 1327.1ms/step | 15.1 samples/s +Step 14410/44000 | Loss: 3.0504 | LR: 3.87e-04 | 1329.7ms/step | 15.0 samples/s +Step 14420/44000 | Loss: 3.0726 | LR: 3.87e-04 | 1327.4ms/step | 15.1 samples/s +Step 14430/44000 | Loss: 3.0391 | LR: 3.86e-04 | 1323.8ms/step | 15.1 samples/s +Step 14440/44000 | Loss: 3.0824 | LR: 3.86e-04 | 1326.4ms/step | 15.1 samples/s +Step 14450/44000 | Loss: 3.0722 | LR: 3.86e-04 | 1325.1ms/step | 15.1 samples/s +Step 14460/44000 | Loss: 3.0837 | LR: 3.86e-04 | 1326.8ms/step | 15.1 samples/s +Step 14470/44000 | Loss: 3.0795 | LR: 3.86e-04 | 1323.2ms/step | 15.1 samples/s +Step 14480/44000 | Loss: 3.0453 | LR: 3.86e-04 | 1323.5ms/step | 15.1 samples/s +Step 14490/44000 | Loss: 3.0587 | LR: 3.85e-04 | 1326.7ms/step | 15.1 samples/s +Step 14500/44000 | Loss: 3.0851 | LR: 3.85e-04 | 1327.7ms/step | 15.1 samples/s +Validation at step 14500: Loss = 3.0807, Perplexity = 21.77 +Step 14510/44000 | Loss: 3.0538 | LR: 3.85e-04 | 3554.3ms/step | 5.6 samples/s +Step 14520/44000 | Loss: 3.0871 | LR: 3.85e-04 | 1328.6ms/step | 15.1 samples/s +Step 14530/44000 | Loss: 3.0663 | LR: 3.85e-04 | 1326.6ms/step | 15.1 samples/s +Step 14540/44000 | Loss: 3.0740 | LR: 3.85e-04 | 1326.1ms/step | 15.1 samples/s +Step 14550/44000 | Loss: 3.0728 | LR: 3.85e-04 | 1329.6ms/step | 15.0 samples/s +Step 14560/44000 | Loss: 3.0829 | LR: 3.84e-04 | 1326.2ms/step | 15.1 samples/s +Step 14570/44000 | Loss: 3.0814 | LR: 3.84e-04 | 1330.6ms/step | 15.0 samples/s +Step 14580/44000 | Loss: 3.0684 | LR: 3.84e-04 | 1332.8ms/step | 15.0 samples/s +Step 14590/44000 | Loss: 3.0788 | LR: 3.84e-04 | 1329.2ms/step | 15.0 samples/s +Step 14600/44000 | Loss: 3.0728 | LR: 3.84e-04 | 1326.1ms/step | 15.1 samples/s +Step 14610/44000 | Loss: 3.0722 | LR: 3.84e-04 | 1329.6ms/step | 15.0 samples/s +Step 14620/44000 | Loss: 3.0849 | LR: 3.83e-04 | 1332.8ms/step | 15.0 samples/s +Step 14630/44000 | Loss: 3.0598 | LR: 3.83e-04 | 1327.2ms/step | 15.1 samples/s +Step 14640/44000 | Loss: 3.0594 | LR: 3.83e-04 | 1324.8ms/step | 15.1 samples/s +Step 14650/44000 | Loss: 3.0695 | LR: 3.83e-04 | 1326.6ms/step | 15.1 samples/s +Step 14660/44000 | Loss: 3.0842 | LR: 3.83e-04 | 1330.2ms/step | 15.0 samples/s +Step 14670/44000 | Loss: 3.0779 | LR: 3.83e-04 | 1327.3ms/step | 15.1 samples/s +Step 14680/44000 | Loss: 3.0430 | LR: 3.83e-04 | 1324.1ms/step | 15.1 samples/s +Step 14690/44000 | Loss: 3.0631 | LR: 3.82e-04 | 1329.5ms/step | 15.0 samples/s +Step 14700/44000 | Loss: 3.0795 | LR: 3.82e-04 | 1330.1ms/step | 15.0 samples/s +Step 14710/44000 | Loss: 3.0702 | LR: 3.82e-04 | 1330.8ms/step | 15.0 samples/s +Step 14720/44000 | Loss: 3.0602 | LR: 3.82e-04 | 1328.8ms/step | 15.1 samples/s +Step 14730/44000 | Loss: 3.0744 | LR: 3.82e-04 | 1327.7ms/step | 15.1 samples/s +Step 14740/44000 | Loss: 3.0637 | LR: 3.82e-04 | 1328.5ms/step | 15.1 samples/s +Step 14750/44000 | Loss: 3.0932 | LR: 3.81e-04 | 1329.3ms/step | 15.0 samples/s +Validation at step 14750: Loss = 3.0773, Perplexity = 21.70 +Step 14760/44000 | Loss: 3.0643 | LR: 3.81e-04 | 3561.5ms/step | 5.6 samples/s +Step 14770/44000 | Loss: 3.0763 | LR: 3.81e-04 | 1328.2ms/step | 15.1 samples/s +Step 14780/44000 | Loss: 3.0876 | LR: 3.81e-04 | 1328.8ms/step | 15.1 samples/s +Step 14790/44000 | Loss: 3.0705 | LR: 3.81e-04 | 1326.6ms/step | 15.1 samples/s +Step 14800/44000 | Loss: 3.0388 | LR: 3.81e-04 | 1326.7ms/step | 15.1 samples/s +Step 14810/44000 | Loss: 3.0582 | LR: 3.81e-04 | 1328.0ms/step | 15.1 samples/s +Step 14820/44000 | Loss: 3.0665 | LR: 3.80e-04 | 1326.8ms/step | 15.1 samples/s +Step 14830/44000 | Loss: 3.0731 | LR: 3.80e-04 | 1332.4ms/step | 15.0 samples/s +Step 14840/44000 | Loss: 3.0605 | LR: 3.80e-04 | 1331.7ms/step | 15.0 samples/s +Step 14850/44000 | Loss: 3.0708 | LR: 3.80e-04 | 1327.9ms/step | 15.1 samples/s +Step 14860/44000 | Loss: 3.0516 | LR: 3.80e-04 | 1327.9ms/step | 15.1 samples/s +Step 14870/44000 | Loss: 3.0562 | LR: 3.80e-04 | 1328.3ms/step | 15.1 samples/s +Step 14880/44000 | Loss: 3.0530 | LR: 3.79e-04 | 1332.5ms/step | 15.0 samples/s +Step 14890/44000 | Loss: 3.0523 | LR: 3.79e-04 | 1329.7ms/step | 15.0 samples/s +Step 14900/44000 | Loss: 3.0604 | LR: 3.79e-04 | 1329.4ms/step | 15.0 samples/s +Step 14910/44000 | Loss: 3.0817 | LR: 3.79e-04 | 1332.1ms/step | 15.0 samples/s +Step 14920/44000 | Loss: 3.0792 | LR: 3.79e-04 | 1334.5ms/step | 15.0 samples/s +Step 14930/44000 | Loss: 3.0841 | LR: 3.79e-04 | 1337.2ms/step | 15.0 samples/s +Step 14940/44000 | Loss: 3.0751 | LR: 3.79e-04 | 1338.0ms/step | 14.9 samples/s +Step 14950/44000 | Loss: 3.1014 | LR: 3.78e-04 | 1334.2ms/step | 15.0 samples/s +Step 14960/44000 | Loss: 3.0603 | LR: 3.78e-04 | 1334.3ms/step | 15.0 samples/s +Step 14970/44000 | Loss: 3.0630 | LR: 3.78e-04 | 1335.8ms/step | 15.0 samples/s +Step 14980/44000 | Loss: 3.0642 | LR: 3.78e-04 | 1334.3ms/step | 15.0 samples/s +Step 14990/44000 | Loss: 3.0509 | LR: 3.78e-04 | 1335.0ms/step | 15.0 samples/s +Step 15000/44000 | Loss: 3.0829 | LR: 3.78e-04 | 1336.4ms/step | 15.0 samples/s +Validation at step 15000: Loss = 3.0741, Perplexity = 21.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-15000 +Step 15010/44000 | Loss: 3.0393 | LR: 3.77e-04 | 3866.5ms/step | 5.2 samples/s +Step 15020/44000 | Loss: 3.0391 | LR: 3.77e-04 | 1331.7ms/step | 15.0 samples/s +Step 15030/44000 | Loss: 3.0493 | LR: 3.77e-04 | 1329.6ms/step | 15.0 samples/s +Step 15040/44000 | Loss: 3.0584 | LR: 3.77e-04 | 1331.6ms/step | 15.0 samples/s +Step 15050/44000 | Loss: 3.0698 | LR: 3.77e-04 | 1335.2ms/step | 15.0 samples/s +Step 15060/44000 | Loss: 3.0673 | LR: 3.77e-04 | 1331.8ms/step | 15.0 samples/s +Step 15070/44000 | Loss: 3.0452 | LR: 3.77e-04 | 1333.5ms/step | 15.0 samples/s +Step 15080/44000 | Loss: 3.0894 | LR: 3.76e-04 | 1333.1ms/step | 15.0 samples/s +Step 15090/44000 | Loss: 3.0578 | LR: 3.76e-04 | 1332.7ms/step | 15.0 samples/s +Step 15100/44000 | Loss: 3.0497 | LR: 3.76e-04 | 1331.8ms/step | 15.0 samples/s +Step 15110/44000 | Loss: 3.0629 | LR: 3.76e-04 | 1330.4ms/step | 15.0 samples/s +Step 15120/44000 | Loss: 3.0571 | LR: 3.76e-04 | 1332.2ms/step | 15.0 samples/s +Step 15130/44000 | Loss: 3.0561 | LR: 3.76e-04 | 1333.0ms/step | 15.0 samples/s +Step 15140/44000 | Loss: 3.0655 | LR: 3.75e-04 | 1334.6ms/step | 15.0 samples/s +Step 15150/44000 | Loss: 3.0742 | LR: 3.75e-04 | 1330.0ms/step | 15.0 samples/s +Step 15160/44000 | Loss: 3.0783 | LR: 3.75e-04 | 1330.4ms/step | 15.0 samples/s +Step 15170/44000 | Loss: 3.0577 | LR: 3.75e-04 | 1330.7ms/step | 15.0 samples/s +Step 15180/44000 | Loss: 3.0477 | LR: 3.75e-04 | 1331.2ms/step | 15.0 samples/s +Step 15190/44000 | Loss: 3.0506 | LR: 3.75e-04 | 1330.0ms/step | 15.0 samples/s +Step 15200/44000 | Loss: 3.0688 | LR: 3.74e-04 | 1331.3ms/step | 15.0 samples/s +Step 15210/44000 | Loss: 3.0807 | LR: 3.74e-04 | 1330.7ms/step | 15.0 samples/s +Step 15220/44000 | Loss: 3.0873 | LR: 3.74e-04 | 1330.1ms/step | 15.0 samples/s +Step 15230/44000 | Loss: 3.0703 | LR: 3.74e-04 | 1335.7ms/step | 15.0 samples/s +Step 15240/44000 | Loss: 3.0484 | LR: 3.74e-04 | 1333.4ms/step | 15.0 samples/s +Step 15250/44000 | Loss: 3.0570 | LR: 3.74e-04 | 1335.3ms/step | 15.0 samples/s +Validation at step 15250: Loss = 3.0710, Perplexity = 21.56 +Step 15260/44000 | Loss: 3.0393 | LR: 3.74e-04 | 3576.6ms/step | 5.6 samples/s +Step 15270/44000 | Loss: 3.0609 | LR: 3.73e-04 | 1335.6ms/step | 15.0 samples/s +Step 15280/44000 | Loss: 3.0444 | LR: 3.73e-04 | 1336.3ms/step | 15.0 samples/s +Step 15290/44000 | Loss: 3.0204 | LR: 3.73e-04 | 1333.5ms/step | 15.0 samples/s +Step 15300/44000 | Loss: 3.0550 | LR: 3.73e-04 | 1335.5ms/step | 15.0 samples/s +Step 15310/44000 | Loss: 3.0660 | LR: 3.73e-04 | 1331.8ms/step | 15.0 samples/s +Step 15320/44000 | Loss: 3.0521 | LR: 3.73e-04 | 1327.2ms/step | 15.1 samples/s +Step 15330/44000 | Loss: 3.0660 | LR: 3.72e-04 | 1328.6ms/step | 15.1 samples/s +Step 15340/44000 | Loss: 3.0615 | LR: 3.72e-04 | 1332.4ms/step | 15.0 samples/s +Step 15350/44000 | Loss: 3.0809 | LR: 3.72e-04 | 1329.7ms/step | 15.0 samples/s +Step 15360/44000 | Loss: 3.0763 | LR: 3.72e-04 | 1329.6ms/step | 15.0 samples/s +Step 15370/44000 | Loss: 3.0631 | LR: 3.72e-04 | 1333.8ms/step | 15.0 samples/s +Step 15380/44000 | Loss: 3.0490 | LR: 3.72e-04 | 1332.8ms/step | 15.0 samples/s +Step 15390/44000 | Loss: 3.0845 | LR: 3.71e-04 | 1328.4ms/step | 15.1 samples/s +Step 15400/44000 | Loss: 3.0348 | LR: 3.71e-04 | 1327.1ms/step | 15.1 samples/s +Step 15410/44000 | Loss: 3.0556 | LR: 3.71e-04 | 1326.0ms/step | 15.1 samples/s +Step 15420/44000 | Loss: 3.0726 | LR: 3.71e-04 | 1327.1ms/step | 15.1 samples/s +Step 15430/44000 | Loss: 3.0652 | LR: 3.71e-04 | 1329.2ms/step | 15.0 samples/s +Step 15440/44000 | Loss: 3.0602 | LR: 3.71e-04 | 1331.2ms/step | 15.0 samples/s +Step 15450/44000 | Loss: 3.0743 | LR: 3.71e-04 | 1330.6ms/step | 15.0 samples/s +Step 15460/44000 | Loss: 3.0543 | LR: 3.70e-04 | 1329.5ms/step | 15.0 samples/s +Step 15470/44000 | Loss: 3.0684 | LR: 3.70e-04 | 1328.2ms/step | 15.1 samples/s +Step 15480/44000 | Loss: 3.0206 | LR: 3.70e-04 | 1326.1ms/step | 15.1 samples/s +Step 15490/44000 | Loss: 3.0530 | LR: 3.70e-04 | 1325.4ms/step | 15.1 samples/s +Step 15500/44000 | Loss: 3.0404 | LR: 3.70e-04 | 1325.9ms/step | 15.1 samples/s +Validation at step 15500: Loss = 3.0673, Perplexity = 21.48 +Step 15510/44000 | Loss: 3.0792 | LR: 3.70e-04 | 3556.5ms/step | 5.6 samples/s +Step 15520/44000 | Loss: 3.0667 | LR: 3.69e-04 | 1329.8ms/step | 15.0 samples/s +Step 15530/44000 | Loss: 3.0423 | LR: 3.69e-04 | 1328.5ms/step | 15.1 samples/s +Step 15540/44000 | Loss: 3.0605 | LR: 3.69e-04 | 1331.0ms/step | 15.0 samples/s +Step 15550/44000 | Loss: 3.0745 | LR: 3.69e-04 | 1329.4ms/step | 15.0 samples/s +Step 15560/44000 | Loss: 3.0619 | LR: 3.69e-04 | 1328.6ms/step | 15.1 samples/s +Step 15570/44000 | Loss: 3.0766 | LR: 3.69e-04 | 1328.5ms/step | 15.1 samples/s +Step 15580/44000 | Loss: 3.0401 | LR: 3.68e-04 | 1325.8ms/step | 15.1 samples/s +Step 15590/44000 | Loss: 3.0338 | LR: 3.68e-04 | 1328.6ms/step | 15.1 samples/s +Step 15600/44000 | Loss: 3.0753 | LR: 3.68e-04 | 1324.8ms/step | 15.1 samples/s +Step 15610/44000 | Loss: 3.0395 | LR: 3.68e-04 | 1324.5ms/step | 15.1 samples/s +Step 15620/44000 | Loss: 3.0314 | LR: 3.68e-04 | 1326.9ms/step | 15.1 samples/s +Step 15630/44000 | Loss: 3.0498 | LR: 3.68e-04 | 1328.7ms/step | 15.1 samples/s +Step 15640/44000 | Loss: 3.0484 | LR: 3.67e-04 | 1330.4ms/step | 15.0 samples/s +Step 15650/44000 | Loss: 3.0355 | LR: 3.67e-04 | 1332.9ms/step | 15.0 samples/s +Step 15660/44000 | Loss: 3.0623 | LR: 3.67e-04 | 1327.4ms/step | 15.1 samples/s +Step 15670/44000 | Loss: 3.0458 | LR: 3.67e-04 | 1326.0ms/step | 15.1 samples/s +Step 15680/44000 | Loss: 3.0630 | LR: 3.67e-04 | 1327.7ms/step | 15.1 samples/s +Step 15690/44000 | Loss: 3.0474 | LR: 3.67e-04 | 1330.9ms/step | 15.0 samples/s +Step 15700/44000 | Loss: 3.0252 | LR: 3.67e-04 | 1327.9ms/step | 15.1 samples/s +Step 15710/44000 | Loss: 3.0411 | LR: 3.66e-04 | 1327.4ms/step | 15.1 samples/s +Step 15720/44000 | Loss: 3.0435 | LR: 3.66e-04 | 1327.6ms/step | 15.1 samples/s +Step 15730/44000 | Loss: 3.0632 | LR: 3.66e-04 | 1327.1ms/step | 15.1 samples/s +Step 15740/44000 | Loss: 3.0501 | LR: 3.66e-04 | 1323.2ms/step | 15.1 samples/s +Step 15750/44000 | Loss: 3.0443 | LR: 3.66e-04 | 1327.5ms/step | 15.1 samples/s +Validation at step 15750: Loss = 3.0655, Perplexity = 21.45 +Step 15760/44000 | Loss: 3.0259 | LR: 3.66e-04 | 3555.0ms/step | 5.6 samples/s +Step 15770/44000 | Loss: 3.0803 | LR: 3.65e-04 | 1329.4ms/step | 15.0 samples/s +Step 15780/44000 | Loss: 3.0489 | LR: 3.65e-04 | 1330.6ms/step | 15.0 samples/s +Step 15790/44000 | Loss: 3.0744 | LR: 3.65e-04 | 1330.3ms/step | 15.0 samples/s +Step 15800/44000 | Loss: 3.0294 | LR: 3.65e-04 | 1325.7ms/step | 15.1 samples/s +Step 15810/44000 | Loss: 3.0612 | LR: 3.65e-04 | 1322.8ms/step | 15.1 samples/s +Step 15820/44000 | Loss: 3.0162 | LR: 3.65e-04 | 1325.7ms/step | 15.1 samples/s +Step 15830/44000 | Loss: 3.0440 | LR: 3.64e-04 | 1325.8ms/step | 15.1 samples/s +Step 15840/44000 | Loss: 3.0334 | LR: 3.64e-04 | 1331.2ms/step | 15.0 samples/s +Step 15850/44000 | Loss: 3.0472 | LR: 3.64e-04 | 1327.9ms/step | 15.1 samples/s +Step 15860/44000 | Loss: 3.0519 | LR: 3.64e-04 | 1325.2ms/step | 15.1 samples/s +Step 15870/44000 | Loss: 3.0635 | LR: 3.64e-04 | 1325.4ms/step | 15.1 samples/s +Step 15880/44000 | Loss: 3.0415 | LR: 3.64e-04 | 1325.9ms/step | 15.1 samples/s +Step 15890/44000 | Loss: 2.9885 | LR: 3.63e-04 | 1331.8ms/step | 15.0 samples/s +Step 15900/44000 | Loss: 3.0259 | LR: 3.63e-04 | 1332.3ms/step | 15.0 samples/s +Step 15910/44000 | Loss: 3.0590 | LR: 3.63e-04 | 1332.8ms/step | 15.0 samples/s +Step 15920/44000 | Loss: 3.0580 | LR: 3.63e-04 | 1333.8ms/step | 15.0 samples/s +Step 15930/44000 | Loss: 3.0608 | LR: 3.63e-04 | 1332.1ms/step | 15.0 samples/s +Step 15940/44000 | Loss: 3.0645 | LR: 3.63e-04 | 1331.6ms/step | 15.0 samples/s +Step 15950/44000 | Loss: 3.0591 | LR: 3.62e-04 | 1333.7ms/step | 15.0 samples/s +Step 15960/44000 | Loss: 3.0387 | LR: 3.62e-04 | 1334.4ms/step | 15.0 samples/s +Step 15970/44000 | Loss: 3.0571 | LR: 3.62e-04 | 1334.3ms/step | 15.0 samples/s +Step 15980/44000 | Loss: 3.0503 | LR: 3.62e-04 | 1332.6ms/step | 15.0 samples/s +Step 15990/44000 | Loss: 3.0364 | LR: 3.62e-04 | 1332.8ms/step | 15.0 samples/s +Step 16000/44000 | Loss: 3.0581 | LR: 3.62e-04 | 1333.9ms/step | 15.0 samples/s +Validation at step 16000: Loss = 3.0605, Perplexity = 21.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-16000 +Step 16010/44000 | Loss: 3.0604 | LR: 3.62e-04 | 3882.1ms/step | 5.2 samples/s +Step 16020/44000 | Loss: 3.0344 | LR: 3.61e-04 | 1326.6ms/step | 15.1 samples/s +Step 16030/44000 | Loss: 3.0606 | LR: 3.61e-04 | 1331.6ms/step | 15.0 samples/s +Step 16040/44000 | Loss: 3.0505 | LR: 3.61e-04 | 1332.0ms/step | 15.0 samples/s +Step 16050/44000 | Loss: 3.0704 | LR: 3.61e-04 | 1332.1ms/step | 15.0 samples/s +Step 16060/44000 | Loss: 3.0660 | LR: 3.61e-04 | 1332.3ms/step | 15.0 samples/s +Step 16070/44000 | Loss: 3.0658 | LR: 3.61e-04 | 1333.7ms/step | 15.0 samples/s +Step 16080/44000 | Loss: 3.0546 | LR: 3.60e-04 | 1333.0ms/step | 15.0 samples/s +Step 16090/44000 | Loss: 3.0313 | LR: 3.60e-04 | 1334.5ms/step | 15.0 samples/s +Step 16100/44000 | Loss: 3.0579 | LR: 3.60e-04 | 1334.5ms/step | 15.0 samples/s +Step 16110/44000 | Loss: 3.0840 | LR: 3.60e-04 | 1333.2ms/step | 15.0 samples/s +Step 16120/44000 | Loss: 3.0640 | LR: 3.60e-04 | 1327.1ms/step | 15.1 samples/s +Step 16130/44000 | Loss: 3.0566 | LR: 3.60e-04 | 1325.5ms/step | 15.1 samples/s +Step 16140/44000 | Loss: 3.0401 | LR: 3.59e-04 | 1327.4ms/step | 15.1 samples/s +Step 16150/44000 | Loss: 3.0465 | LR: 3.59e-04 | 1330.2ms/step | 15.0 samples/s +Step 16160/44000 | Loss: 3.0161 | LR: 3.59e-04 | 1335.3ms/step | 15.0 samples/s +Step 16170/44000 | Loss: 3.0133 | LR: 3.59e-04 | 1333.8ms/step | 15.0 samples/s +Step 16180/44000 | Loss: 3.0214 | LR: 3.59e-04 | 1332.6ms/step | 15.0 samples/s +Step 16190/44000 | Loss: 3.0315 | LR: 3.59e-04 | 1335.7ms/step | 15.0 samples/s +Step 16200/44000 | Loss: 3.0199 | LR: 3.58e-04 | 1335.9ms/step | 15.0 samples/s +Step 16210/44000 | Loss: 3.0363 | LR: 3.58e-04 | 1335.3ms/step | 15.0 samples/s +Step 16220/44000 | Loss: 3.0210 | LR: 3.58e-04 | 1335.2ms/step | 15.0 samples/s +Step 16230/44000 | Loss: 3.0320 | LR: 3.58e-04 | 1335.5ms/step | 15.0 samples/s +Step 16240/44000 | Loss: 3.0297 | LR: 3.58e-04 | 1336.1ms/step | 15.0 samples/s +Step 16250/44000 | Loss: 3.0502 | LR: 3.58e-04 | 1334.9ms/step | 15.0 samples/s +Validation at step 16250: Loss = 3.0575, Perplexity = 21.27 +Step 16260/44000 | Loss: 3.0698 | LR: 3.57e-04 | 3570.5ms/step | 5.6 samples/s +Step 16270/44000 | Loss: 3.0540 | LR: 3.57e-04 | 1336.9ms/step | 15.0 samples/s +Step 16280/44000 | Loss: 3.0264 | LR: 3.57e-04 | 1332.9ms/step | 15.0 samples/s +Step 16290/44000 | Loss: 3.0220 | LR: 3.57e-04 | 1332.1ms/step | 15.0 samples/s +Step 16300/44000 | Loss: 3.0687 | LR: 3.57e-04 | 1335.7ms/step | 15.0 samples/s +Step 16310/44000 | Loss: 3.0501 | LR: 3.57e-04 | 1332.5ms/step | 15.0 samples/s +Step 16320/44000 | Loss: 3.0407 | LR: 3.56e-04 | 1332.3ms/step | 15.0 samples/s +Step 16330/44000 | Loss: 3.0508 | LR: 3.56e-04 | 1326.0ms/step | 15.1 samples/s +Step 16340/44000 | Loss: 3.0493 | LR: 3.56e-04 | 1328.4ms/step | 15.1 samples/s +Step 16350/44000 | Loss: 3.0708 | LR: 3.56e-04 | 1329.0ms/step | 15.0 samples/s +Step 16360/44000 | Loss: 3.0286 | LR: 3.56e-04 | 1330.4ms/step | 15.0 samples/s +Step 16370/44000 | Loss: 3.0506 | LR: 3.56e-04 | 1329.5ms/step | 15.0 samples/s +Step 16380/44000 | Loss: 3.0330 | LR: 3.55e-04 | 1330.6ms/step | 15.0 samples/s +Step 16390/44000 | Loss: 3.0621 | LR: 3.55e-04 | 1328.5ms/step | 15.1 samples/s +Step 16400/44000 | Loss: 3.0441 | LR: 3.55e-04 | 1328.5ms/step | 15.1 samples/s +Step 16410/44000 | Loss: 3.0557 | LR: 3.55e-04 | 1331.2ms/step | 15.0 samples/s +Step 16420/44000 | Loss: 3.0425 | LR: 3.55e-04 | 1331.4ms/step | 15.0 samples/s +Step 16430/44000 | Loss: 3.0492 | LR: 3.55e-04 | 1330.6ms/step | 15.0 samples/s +Step 16440/44000 | Loss: 3.0532 | LR: 3.54e-04 | 1330.8ms/step | 15.0 samples/s +Step 16450/44000 | Loss: 3.0400 | LR: 3.54e-04 | 1329.8ms/step | 15.0 samples/s +Step 16460/44000 | Loss: 3.0389 | LR: 3.54e-04 | 1326.9ms/step | 15.1 samples/s +Step 16470/44000 | Loss: 3.0270 | LR: 3.54e-04 | 1330.0ms/step | 15.0 samples/s +Step 16480/44000 | Loss: 3.0177 | LR: 3.54e-04 | 1331.4ms/step | 15.0 samples/s +Step 16490/44000 | Loss: 3.0213 | LR: 3.54e-04 | 1330.9ms/step | 15.0 samples/s +Step 16500/44000 | Loss: 3.0479 | LR: 3.53e-04 | 1329.0ms/step | 15.0 samples/s +Validation at step 16500: Loss = 3.0545, Perplexity = 21.21 +Step 16510/44000 | Loss: 3.0511 | LR: 3.53e-04 | 3556.7ms/step | 5.6 samples/s +Step 16520/44000 | Loss: 3.0335 | LR: 3.53e-04 | 1328.8ms/step | 15.1 samples/s +Step 16530/44000 | Loss: 3.0144 | LR: 3.53e-04 | 1327.6ms/step | 15.1 samples/s +Step 16540/44000 | Loss: 3.0335 | LR: 3.53e-04 | 1330.8ms/step | 15.0 samples/s +Step 16550/44000 | Loss: 3.0436 | LR: 3.53e-04 | 1328.7ms/step | 15.1 samples/s +Step 16560/44000 | Loss: 3.0429 | LR: 3.52e-04 | 1329.0ms/step | 15.0 samples/s +Step 16570/44000 | Loss: 3.0521 | LR: 3.52e-04 | 1328.4ms/step | 15.1 samples/s +Step 16580/44000 | Loss: 3.0401 | LR: 3.52e-04 | 1326.1ms/step | 15.1 samples/s +Step 16590/44000 | Loss: 3.0365 | LR: 3.52e-04 | 1328.6ms/step | 15.1 samples/s +Step 16600/44000 | Loss: 3.0371 | LR: 3.52e-04 | 1328.5ms/step | 15.1 samples/s +Step 16610/44000 | Loss: 3.0352 | LR: 3.52e-04 | 1330.0ms/step | 15.0 samples/s +Step 16620/44000 | Loss: 3.0081 | LR: 3.51e-04 | 1328.9ms/step | 15.1 samples/s +Step 16630/44000 | Loss: 3.0379 | LR: 3.51e-04 | 1326.5ms/step | 15.1 samples/s +Step 16640/44000 | Loss: 3.0395 | LR: 3.51e-04 | 1329.2ms/step | 15.0 samples/s +Step 16650/44000 | Loss: 3.0446 | LR: 3.51e-04 | 1329.6ms/step | 15.0 samples/s +Step 16660/44000 | Loss: 3.0498 | LR: 3.51e-04 | 1329.4ms/step | 15.0 samples/s +Step 16670/44000 | Loss: 3.0441 | LR: 3.51e-04 | 1330.7ms/step | 15.0 samples/s +Step 16680/44000 | Loss: 3.0371 | LR: 3.50e-04 | 1329.1ms/step | 15.0 samples/s +Step 16690/44000 | Loss: 3.0505 | LR: 3.50e-04 | 1328.0ms/step | 15.1 samples/s +Step 16700/44000 | Loss: 3.0519 | LR: 3.50e-04 | 1326.9ms/step | 15.1 samples/s +Step 16710/44000 | Loss: 3.0401 | LR: 3.50e-04 | 1325.0ms/step | 15.1 samples/s +Step 16720/44000 | Loss: 3.0538 | LR: 3.50e-04 | 1328.0ms/step | 15.1 samples/s +Step 16730/44000 | Loss: 3.0206 | LR: 3.50e-04 | 1325.8ms/step | 15.1 samples/s +Step 16740/44000 | Loss: 3.0392 | LR: 3.49e-04 | 1326.0ms/step | 15.1 samples/s +Step 16750/44000 | Loss: 3.0171 | LR: 3.49e-04 | 1328.1ms/step | 15.1 samples/s +Validation at step 16750: Loss = 3.0512, Perplexity = 21.14 +Step 16760/44000 | Loss: 3.0399 | LR: 3.49e-04 | 3558.8ms/step | 5.6 samples/s +Step 16770/44000 | Loss: 3.0414 | LR: 3.49e-04 | 1330.2ms/step | 15.0 samples/s +Step 16780/44000 | Loss: 3.0638 | LR: 3.49e-04 | 1328.9ms/step | 15.0 samples/s +Step 16790/44000 | Loss: 3.0113 | LR: 3.49e-04 | 1328.5ms/step | 15.1 samples/s +Step 16800/44000 | Loss: 3.0507 | LR: 3.48e-04 | 1329.6ms/step | 15.0 samples/s +Step 16810/44000 | Loss: 3.0464 | LR: 3.48e-04 | 1326.8ms/step | 15.1 samples/s +Step 16820/44000 | Loss: 3.0427 | LR: 3.48e-04 | 1327.2ms/step | 15.1 samples/s +Step 16830/44000 | Loss: 3.0432 | LR: 3.48e-04 | 1326.7ms/step | 15.1 samples/s +Step 16840/44000 | Loss: 3.0164 | LR: 3.48e-04 | 1325.9ms/step | 15.1 samples/s +Step 16850/44000 | Loss: 3.0067 | LR: 3.48e-04 | 1327.0ms/step | 15.1 samples/s +Step 16860/44000 | Loss: 3.0532 | LR: 3.47e-04 | 1332.0ms/step | 15.0 samples/s +Step 16870/44000 | Loss: 3.0374 | LR: 3.47e-04 | 1331.4ms/step | 15.0 samples/s +Step 16880/44000 | Loss: 3.0624 | LR: 3.47e-04 | 1332.1ms/step | 15.0 samples/s +Step 16890/44000 | Loss: 3.0353 | LR: 3.47e-04 | 1331.5ms/step | 15.0 samples/s +Step 16900/44000 | Loss: 3.0505 | LR: 3.47e-04 | 1332.4ms/step | 15.0 samples/s +Step 16910/44000 | Loss: 3.0735 | LR: 3.47e-04 | 1334.2ms/step | 15.0 samples/s +Step 16920/44000 | Loss: 3.0403 | LR: 3.46e-04 | 1334.9ms/step | 15.0 samples/s +Step 16930/44000 | Loss: 3.0274 | LR: 3.46e-04 | 1331.5ms/step | 15.0 samples/s +Step 16940/44000 | Loss: 3.0535 | LR: 3.46e-04 | 1332.5ms/step | 15.0 samples/s +Step 16950/44000 | Loss: 2.9976 | LR: 3.46e-04 | 1332.0ms/step | 15.0 samples/s +Step 16960/44000 | Loss: 3.0369 | LR: 3.46e-04 | 1331.0ms/step | 15.0 samples/s +Step 16970/44000 | Loss: 3.0262 | LR: 3.46e-04 | 1331.0ms/step | 15.0 samples/s +Step 16980/44000 | Loss: 3.0342 | LR: 3.45e-04 | 1332.1ms/step | 15.0 samples/s +Step 16990/44000 | Loss: 3.0337 | LR: 3.45e-04 | 1333.1ms/step | 15.0 samples/s +Step 17000/44000 | Loss: 3.0439 | LR: 3.45e-04 | 1335.3ms/step | 15.0 samples/s +Validation at step 17000: Loss = 3.0487, Perplexity = 21.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-17000 +Step 17010/44000 | Loss: 3.0401 | LR: 3.45e-04 | 3933.1ms/step | 5.1 samples/s +Step 17020/44000 | Loss: 3.0374 | LR: 3.45e-04 | 1327.2ms/step | 15.1 samples/s +Step 17030/44000 | Loss: 3.0122 | LR: 3.45e-04 | 1327.7ms/step | 15.1 samples/s +Step 17040/44000 | Loss: 3.0565 | LR: 3.44e-04 | 1332.7ms/step | 15.0 samples/s +Step 17050/44000 | Loss: 3.0656 | LR: 3.44e-04 | 1335.0ms/step | 15.0 samples/s +Step 17060/44000 | Loss: 3.0070 | LR: 3.44e-04 | 1333.0ms/step | 15.0 samples/s +Step 17070/44000 | Loss: 2.9921 | LR: 3.44e-04 | 1333.5ms/step | 15.0 samples/s +Step 17080/44000 | Loss: 3.0317 | LR: 3.44e-04 | 1335.9ms/step | 15.0 samples/s +Step 17090/44000 | Loss: 3.0345 | LR: 3.44e-04 | 1334.8ms/step | 15.0 samples/s +Step 17100/44000 | Loss: 3.0566 | LR: 3.43e-04 | 1333.5ms/step | 15.0 samples/s +Step 17110/44000 | Loss: 3.0304 | LR: 3.43e-04 | 1335.8ms/step | 15.0 samples/s +Step 17120/44000 | Loss: 3.0351 | LR: 3.43e-04 | 1329.9ms/step | 15.0 samples/s +Step 17130/44000 | Loss: 3.0331 | LR: 3.43e-04 | 1325.8ms/step | 15.1 samples/s +Step 17140/44000 | Loss: 3.0103 | LR: 3.43e-04 | 1324.5ms/step | 15.1 samples/s +Step 17150/44000 | Loss: 3.0123 | LR: 3.43e-04 | 1331.0ms/step | 15.0 samples/s +Step 17160/44000 | Loss: 3.0260 | LR: 3.42e-04 | 1329.9ms/step | 15.0 samples/s +Step 17170/44000 | Loss: 3.0190 | LR: 3.42e-04 | 1331.1ms/step | 15.0 samples/s +Step 17180/44000 | Loss: 3.0042 | LR: 3.42e-04 | 1331.4ms/step | 15.0 samples/s +Step 17190/44000 | Loss: 3.0350 | LR: 3.42e-04 | 1332.6ms/step | 15.0 samples/s +Step 17200/44000 | Loss: 3.0176 | LR: 3.42e-04 | 1336.5ms/step | 15.0 samples/s +Step 17210/44000 | Loss: 3.0340 | LR: 3.42e-04 | 1335.0ms/step | 15.0 samples/s +Step 17220/44000 | Loss: 2.9937 | LR: 3.41e-04 | 1334.9ms/step | 15.0 samples/s +Step 17230/44000 | Loss: 3.0307 | LR: 3.41e-04 | 1335.4ms/step | 15.0 samples/s +Step 17240/44000 | Loss: 3.0146 | LR: 3.41e-04 | 1334.6ms/step | 15.0 samples/s +Step 17250/44000 | Loss: 3.0271 | LR: 3.41e-04 | 1335.2ms/step | 15.0 samples/s +Validation at step 17250: Loss = 3.0449, Perplexity = 21.01 +Step 17260/44000 | Loss: 3.0065 | LR: 3.41e-04 | 3577.4ms/step | 5.6 samples/s +Step 17270/44000 | Loss: 3.0341 | LR: 3.41e-04 | 1335.9ms/step | 15.0 samples/s +Step 17280/44000 | Loss: 3.0394 | LR: 3.40e-04 | 1329.9ms/step | 15.0 samples/s +Step 17290/44000 | Loss: 3.0536 | LR: 3.40e-04 | 1329.0ms/step | 15.0 samples/s +Step 17300/44000 | Loss: 3.0345 | LR: 3.40e-04 | 1325.3ms/step | 15.1 samples/s +Step 17310/44000 | Loss: 2.9860 | LR: 3.40e-04 | 1327.7ms/step | 15.1 samples/s +Step 17320/44000 | Loss: 3.0542 | LR: 3.40e-04 | 1328.1ms/step | 15.1 samples/s +Step 17330/44000 | Loss: 3.0417 | LR: 3.40e-04 | 1327.4ms/step | 15.1 samples/s +Step 17340/44000 | Loss: 3.0330 | LR: 3.39e-04 | 1329.0ms/step | 15.0 samples/s +Step 17350/44000 | Loss: 3.0319 | LR: 3.39e-04 | 1324.4ms/step | 15.1 samples/s +Step 17360/44000 | Loss: 3.0346 | LR: 3.39e-04 | 1327.6ms/step | 15.1 samples/s +Step 17370/44000 | Loss: 3.0470 | LR: 3.39e-04 | 1329.3ms/step | 15.0 samples/s +Step 17380/44000 | Loss: 3.0132 | LR: 3.39e-04 | 1327.5ms/step | 15.1 samples/s +Step 17390/44000 | Loss: 3.0447 | LR: 3.39e-04 | 1328.6ms/step | 15.1 samples/s +Step 17400/44000 | Loss: 3.0133 | LR: 3.38e-04 | 1328.5ms/step | 15.1 samples/s +Step 17410/44000 | Loss: 3.0705 | LR: 3.38e-04 | 1330.4ms/step | 15.0 samples/s +Step 17420/44000 | Loss: 3.0271 | LR: 3.38e-04 | 1329.9ms/step | 15.0 samples/s +Step 17430/44000 | Loss: 3.0004 | LR: 3.38e-04 | 1332.8ms/step | 15.0 samples/s +Step 17440/44000 | Loss: 3.0219 | LR: 3.38e-04 | 1332.3ms/step | 15.0 samples/s +Step 17450/44000 | Loss: 3.0343 | LR: 3.38e-04 | 1333.9ms/step | 15.0 samples/s +Step 17460/44000 | Loss: 3.0156 | LR: 3.37e-04 | 1333.6ms/step | 15.0 samples/s +Step 17470/44000 | Loss: 3.0409 | LR: 3.37e-04 | 1326.3ms/step | 15.1 samples/s +Step 17480/44000 | Loss: 3.0448 | LR: 3.37e-04 | 1326.9ms/step | 15.1 samples/s +Step 17490/44000 | Loss: 3.0381 | LR: 3.37e-04 | 1324.9ms/step | 15.1 samples/s +Step 17500/44000 | Loss: 3.0174 | LR: 3.37e-04 | 1324.7ms/step | 15.1 samples/s +Validation at step 17500: Loss = 3.0421, Perplexity = 20.95 +Step 17510/44000 | Loss: 3.0336 | LR: 3.37e-04 | 3554.2ms/step | 5.6 samples/s +Step 17520/44000 | Loss: 3.0408 | LR: 3.36e-04 | 1325.6ms/step | 15.1 samples/s +Step 17530/44000 | Loss: 3.0192 | LR: 3.36e-04 | 1326.7ms/step | 15.1 samples/s +Step 17540/44000 | Loss: 3.0150 | LR: 3.36e-04 | 1326.0ms/step | 15.1 samples/s +Step 17550/44000 | Loss: 3.0351 | LR: 3.36e-04 | 1327.1ms/step | 15.1 samples/s +Step 17560/44000 | Loss: 3.0698 | LR: 3.36e-04 | 1327.0ms/step | 15.1 samples/s +Step 17570/44000 | Loss: 3.0090 | LR: 3.35e-04 | 1329.4ms/step | 15.0 samples/s +Step 17580/44000 | Loss: 3.0118 | LR: 3.35e-04 | 1326.8ms/step | 15.1 samples/s +Step 17590/44000 | Loss: 3.0541 | LR: 3.35e-04 | 1329.3ms/step | 15.0 samples/s +Step 17600/44000 | Loss: 2.9875 | LR: 3.35e-04 | 1328.4ms/step | 15.1 samples/s +Step 17610/44000 | Loss: 2.9962 | LR: 3.35e-04 | 1326.0ms/step | 15.1 samples/s +Step 17620/44000 | Loss: 3.0301 | LR: 3.35e-04 | 1324.2ms/step | 15.1 samples/s +Step 17630/44000 | Loss: 3.0133 | LR: 3.34e-04 | 1327.7ms/step | 15.1 samples/s +Step 17640/44000 | Loss: 3.0093 | LR: 3.34e-04 | 1329.1ms/step | 15.0 samples/s +Step 17650/44000 | Loss: 3.0260 | LR: 3.34e-04 | 1325.7ms/step | 15.1 samples/s +Step 17660/44000 | Loss: 3.0094 | LR: 3.34e-04 | 1326.9ms/step | 15.1 samples/s +Step 17670/44000 | Loss: 3.0393 | LR: 3.34e-04 | 1323.5ms/step | 15.1 samples/s +Step 17680/44000 | Loss: 3.0251 | LR: 3.34e-04 | 1323.8ms/step | 15.1 samples/s +Step 17690/44000 | Loss: 3.0137 | LR: 3.33e-04 | 1328.2ms/step | 15.1 samples/s +Step 17700/44000 | Loss: 3.0364 | LR: 3.33e-04 | 1330.5ms/step | 15.0 samples/s +Step 17710/44000 | Loss: 3.0311 | LR: 3.33e-04 | 1330.1ms/step | 15.0 samples/s +Step 17720/44000 | Loss: 3.0202 | LR: 3.33e-04 | 1331.8ms/step | 15.0 samples/s +Step 17730/44000 | Loss: 3.0377 | LR: 3.33e-04 | 1331.7ms/step | 15.0 samples/s +Step 17740/44000 | Loss: 3.0263 | LR: 3.33e-04 | 1334.5ms/step | 15.0 samples/s +Step 17750/44000 | Loss: 3.0286 | LR: 3.32e-04 | 1334.3ms/step | 15.0 samples/s +Validation at step 17750: Loss = 3.0386, Perplexity = 20.88 +Step 17760/44000 | Loss: 3.0216 | LR: 3.32e-04 | 3569.7ms/step | 5.6 samples/s +Step 17770/44000 | Loss: 3.0176 | LR: 3.32e-04 | 1326.5ms/step | 15.1 samples/s +Step 17780/44000 | Loss: 3.0315 | LR: 3.32e-04 | 1327.6ms/step | 15.1 samples/s +Step 17790/44000 | Loss: 3.0707 | LR: 3.32e-04 | 1324.5ms/step | 15.1 samples/s +Step 17800/44000 | Loss: 3.0151 | LR: 3.32e-04 | 1325.9ms/step | 15.1 samples/s +Step 17810/44000 | Loss: 3.0128 | LR: 3.31e-04 | 1327.7ms/step | 15.1 samples/s +Step 17820/44000 | Loss: 3.0454 | LR: 3.31e-04 | 1325.4ms/step | 15.1 samples/s +Step 17830/44000 | Loss: 2.9894 | LR: 3.31e-04 | 1323.0ms/step | 15.1 samples/s +Step 17840/44000 | Loss: 3.0390 | LR: 3.31e-04 | 1328.7ms/step | 15.1 samples/s +Step 17850/44000 | Loss: 3.0384 | LR: 3.31e-04 | 1328.4ms/step | 15.1 samples/s +Step 17860/44000 | Loss: 3.0126 | LR: 3.31e-04 | 1330.7ms/step | 15.0 samples/s +Step 17870/44000 | Loss: 3.0417 | LR: 3.30e-04 | 1328.3ms/step | 15.1 samples/s +Step 17880/44000 | Loss: 3.0505 | LR: 3.30e-04 | 1333.4ms/step | 15.0 samples/s +Step 17890/44000 | Loss: 3.0375 | LR: 3.30e-04 | 1333.8ms/step | 15.0 samples/s +Step 17900/44000 | Loss: 3.0369 | LR: 3.30e-04 | 1334.0ms/step | 15.0 samples/s +Step 17910/44000 | Loss: 3.0711 | LR: 3.30e-04 | 1335.5ms/step | 15.0 samples/s +Step 17920/44000 | Loss: 2.9996 | LR: 3.29e-04 | 1332.1ms/step | 15.0 samples/s +Step 17930/44000 | Loss: 2.9976 | LR: 3.29e-04 | 1331.6ms/step | 15.0 samples/s +Step 17940/44000 | Loss: 3.0010 | LR: 3.29e-04 | 1331.4ms/step | 15.0 samples/s +Step 17950/44000 | Loss: 3.0172 | LR: 3.29e-04 | 1334.9ms/step | 15.0 samples/s +Step 17960/44000 | Loss: 3.0299 | LR: 3.29e-04 | 1330.5ms/step | 15.0 samples/s +Step 17970/44000 | Loss: 3.0110 | LR: 3.29e-04 | 1332.7ms/step | 15.0 samples/s +Step 17980/44000 | Loss: 3.0114 | LR: 3.28e-04 | 1334.2ms/step | 15.0 samples/s +Step 17990/44000 | Loss: 2.9917 | LR: 3.28e-04 | 1333.2ms/step | 15.0 samples/s +Step 18000/44000 | Loss: 3.0138 | LR: 3.28e-04 | 1332.9ms/step | 15.0 samples/s +Validation at step 18000: Loss = 3.0355, Perplexity = 20.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-18000 +Step 18010/44000 | Loss: 3.0007 | LR: 3.28e-04 | 3889.8ms/step | 5.1 samples/s +Step 18020/44000 | Loss: 3.0460 | LR: 3.28e-04 | 1330.8ms/step | 15.0 samples/s +Step 18030/44000 | Loss: 3.0109 | LR: 3.28e-04 | 1332.5ms/step | 15.0 samples/s +Step 18040/44000 | Loss: 3.0159 | LR: 3.27e-04 | 1331.7ms/step | 15.0 samples/s +Step 18050/44000 | Loss: 3.0444 | LR: 3.27e-04 | 1331.0ms/step | 15.0 samples/s +Step 18060/44000 | Loss: 3.0194 | LR: 3.27e-04 | 1332.4ms/step | 15.0 samples/s +Step 18070/44000 | Loss: 3.0172 | LR: 3.27e-04 | 1333.8ms/step | 15.0 samples/s +Step 18080/44000 | Loss: 3.0328 | LR: 3.27e-04 | 1331.6ms/step | 15.0 samples/s +Step 18090/44000 | Loss: 3.0226 | LR: 3.27e-04 | 1333.5ms/step | 15.0 samples/s +Step 18100/44000 | Loss: 3.0258 | LR: 3.26e-04 | 1333.3ms/step | 15.0 samples/s +Step 18110/44000 | Loss: 3.0388 | LR: 3.26e-04 | 1334.5ms/step | 15.0 samples/s +Step 18120/44000 | Loss: 3.0106 | LR: 3.26e-04 | 1333.2ms/step | 15.0 samples/s +Step 18130/44000 | Loss: 3.0183 | LR: 3.26e-04 | 1331.3ms/step | 15.0 samples/s +Step 18140/44000 | Loss: 3.0172 | LR: 3.26e-04 | 1334.0ms/step | 15.0 samples/s +Step 18150/44000 | Loss: 3.0246 | LR: 3.26e-04 | 1332.8ms/step | 15.0 samples/s +Step 18160/44000 | Loss: 3.0404 | LR: 3.25e-04 | 1331.0ms/step | 15.0 samples/s +Step 18170/44000 | Loss: 2.9956 | LR: 3.25e-04 | 1330.9ms/step | 15.0 samples/s +Step 18180/44000 | Loss: 3.0158 | LR: 3.25e-04 | 1333.6ms/step | 15.0 samples/s +Step 18190/44000 | Loss: 3.0346 | LR: 3.25e-04 | 1334.0ms/step | 15.0 samples/s +Step 18200/44000 | Loss: 3.0376 | LR: 3.25e-04 | 1332.5ms/step | 15.0 samples/s +Step 18210/44000 | Loss: 3.0512 | LR: 3.24e-04 | 1331.0ms/step | 15.0 samples/s +Step 18220/44000 | Loss: 3.0140 | LR: 3.24e-04 | 1333.2ms/step | 15.0 samples/s +Step 18230/44000 | Loss: 3.0269 | LR: 3.24e-04 | 1334.5ms/step | 15.0 samples/s +Step 18240/44000 | Loss: 3.0401 | LR: 3.24e-04 | 1331.2ms/step | 15.0 samples/s +Step 18250/44000 | Loss: 3.0017 | LR: 3.24e-04 | 1332.2ms/step | 15.0 samples/s +Validation at step 18250: Loss = 3.0339, Perplexity = 20.78 +Step 18260/44000 | Loss: 3.0291 | LR: 3.24e-04 | 3572.6ms/step | 5.6 samples/s +Step 18270/44000 | Loss: 3.0017 | LR: 3.23e-04 | 1332.5ms/step | 15.0 samples/s +Step 18280/44000 | Loss: 2.9956 | LR: 3.23e-04 | 1328.2ms/step | 15.1 samples/s +Step 18290/44000 | Loss: 3.0279 | LR: 3.23e-04 | 1329.4ms/step | 15.0 samples/s +Step 18300/44000 | Loss: 3.0048 | LR: 3.23e-04 | 1329.2ms/step | 15.0 samples/s +Step 18310/44000 | Loss: 3.0242 | LR: 3.23e-04 | 1328.7ms/step | 15.1 samples/s +Step 18320/44000 | Loss: 3.0345 | LR: 3.23e-04 | 1325.6ms/step | 15.1 samples/s +Step 18330/44000 | Loss: 3.0478 | LR: 3.22e-04 | 1327.7ms/step | 15.1 samples/s +Step 18340/44000 | Loss: 3.0382 | LR: 3.22e-04 | 1328.7ms/step | 15.1 samples/s +Step 18350/44000 | Loss: 3.0122 | LR: 3.22e-04 | 1330.2ms/step | 15.0 samples/s +Step 18360/44000 | Loss: 3.0324 | LR: 3.22e-04 | 1327.0ms/step | 15.1 samples/s +Step 18370/44000 | Loss: 3.0500 | LR: 3.22e-04 | 1329.4ms/step | 15.0 samples/s +Step 18380/44000 | Loss: 3.0235 | LR: 3.22e-04 | 1328.6ms/step | 15.1 samples/s +Step 18390/44000 | Loss: 3.0089 | LR: 3.21e-04 | 1329.6ms/step | 15.0 samples/s +Step 18400/44000 | Loss: 2.9775 | LR: 3.21e-04 | 1328.0ms/step | 15.1 samples/s +Step 18410/44000 | Loss: 3.0283 | LR: 3.21e-04 | 1330.0ms/step | 15.0 samples/s +Step 18420/44000 | Loss: 3.0099 | LR: 3.21e-04 | 1330.5ms/step | 15.0 samples/s +Step 18430/44000 | Loss: 3.0358 | LR: 3.21e-04 | 1331.0ms/step | 15.0 samples/s +Step 18440/44000 | Loss: 3.0074 | LR: 3.20e-04 | 1326.7ms/step | 15.1 samples/s +Step 18450/44000 | Loss: 3.0141 | LR: 3.20e-04 | 1325.0ms/step | 15.1 samples/s +Step 18460/44000 | Loss: 3.0125 | LR: 3.20e-04 | 1328.4ms/step | 15.1 samples/s +Step 18470/44000 | Loss: 3.0441 | LR: 3.20e-04 | 1327.3ms/step | 15.1 samples/s +Step 18480/44000 | Loss: 3.0124 | LR: 3.20e-04 | 1325.3ms/step | 15.1 samples/s +Step 18490/44000 | Loss: 3.0116 | LR: 3.20e-04 | 1329.6ms/step | 15.0 samples/s +Step 18500/44000 | Loss: 2.9961 | LR: 3.19e-04 | 1331.0ms/step | 15.0 samples/s +Validation at step 18500: Loss = 3.0307, Perplexity = 20.71 +Step 18510/44000 | Loss: 3.0211 | LR: 3.19e-04 | 3561.2ms/step | 5.6 samples/s +Step 18520/44000 | Loss: 2.9955 | LR: 3.19e-04 | 1326.3ms/step | 15.1 samples/s +Step 18530/44000 | Loss: 3.0288 | LR: 3.19e-04 | 1324.8ms/step | 15.1 samples/s +Step 18540/44000 | Loss: 2.9908 | LR: 3.19e-04 | 1324.1ms/step | 15.1 samples/s +Step 18550/44000 | Loss: 3.0222 | LR: 3.19e-04 | 1322.3ms/step | 15.1 samples/s +Step 18560/44000 | Loss: 3.0112 | LR: 3.18e-04 | 1324.5ms/step | 15.1 samples/s +Step 18570/44000 | Loss: 3.0084 | LR: 3.18e-04 | 1323.1ms/step | 15.1 samples/s +Step 18580/44000 | Loss: 2.9945 | LR: 3.18e-04 | 1327.1ms/step | 15.1 samples/s +Step 18590/44000 | Loss: 3.0004 | LR: 3.18e-04 | 1325.9ms/step | 15.1 samples/s +Step 18600/44000 | Loss: 3.0109 | LR: 3.18e-04 | 1322.1ms/step | 15.1 samples/s +Step 18610/44000 | Loss: 3.0311 | LR: 3.18e-04 | 1322.4ms/step | 15.1 samples/s +Step 18620/44000 | Loss: 3.0534 | LR: 3.17e-04 | 1325.5ms/step | 15.1 samples/s +Step 18630/44000 | Loss: 3.0425 | LR: 3.17e-04 | 1327.2ms/step | 15.1 samples/s +Step 18640/44000 | Loss: 3.0001 | LR: 3.17e-04 | 1329.6ms/step | 15.0 samples/s +Step 18650/44000 | Loss: 3.0272 | LR: 3.17e-04 | 1331.4ms/step | 15.0 samples/s +Step 18660/44000 | Loss: 3.0490 | LR: 3.17e-04 | 1329.1ms/step | 15.0 samples/s +Step 18670/44000 | Loss: 3.0363 | LR: 3.16e-04 | 1330.1ms/step | 15.0 samples/s +Step 18680/44000 | Loss: 3.0163 | LR: 3.16e-04 | 1326.4ms/step | 15.1 samples/s +Step 18690/44000 | Loss: 3.0079 | LR: 3.16e-04 | 1325.4ms/step | 15.1 samples/s +Step 18700/44000 | Loss: 3.0143 | LR: 3.16e-04 | 1326.5ms/step | 15.1 samples/s +Step 18710/44000 | Loss: 3.0292 | LR: 3.16e-04 | 1329.2ms/step | 15.0 samples/s +Step 18720/44000 | Loss: 3.0320 | LR: 3.16e-04 | 1331.7ms/step | 15.0 samples/s +Step 18730/44000 | Loss: 3.0177 | LR: 3.15e-04 | 1332.3ms/step | 15.0 samples/s +Step 18740/44000 | Loss: 3.0220 | LR: 3.15e-04 | 1330.6ms/step | 15.0 samples/s +Step 18750/44000 | Loss: 3.0182 | LR: 3.15e-04 | 1329.8ms/step | 15.0 samples/s +Validation at step 18750: Loss = 3.0272, Perplexity = 20.64 +Step 18760/44000 | Loss: 3.0421 | LR: 3.15e-04 | 3557.8ms/step | 5.6 samples/s +Step 18770/44000 | Loss: 3.0220 | LR: 3.15e-04 | 1328.5ms/step | 15.1 samples/s +Step 18780/44000 | Loss: 3.0355 | LR: 3.15e-04 | 1330.8ms/step | 15.0 samples/s +Step 18790/44000 | Loss: 3.0185 | LR: 3.14e-04 | 1330.7ms/step | 15.0 samples/s +Step 18800/44000 | Loss: 3.0173 | LR: 3.14e-04 | 1330.7ms/step | 15.0 samples/s +Step 18810/44000 | Loss: 3.0155 | LR: 3.14e-04 | 1332.9ms/step | 15.0 samples/s +Step 18820/44000 | Loss: 3.0080 | LR: 3.14e-04 | 1328.8ms/step | 15.1 samples/s +Step 18830/44000 | Loss: 3.0297 | LR: 3.14e-04 | 1324.2ms/step | 15.1 samples/s +Step 18840/44000 | Loss: 3.0236 | LR: 3.13e-04 | 1324.3ms/step | 15.1 samples/s +Step 18850/44000 | Loss: 3.0083 | LR: 3.13e-04 | 1329.1ms/step | 15.0 samples/s +Step 18860/44000 | Loss: 3.0065 | LR: 3.13e-04 | 1330.9ms/step | 15.0 samples/s +Step 18870/44000 | Loss: 3.0043 | LR: 3.13e-04 | 1329.9ms/step | 15.0 samples/s +Step 18880/44000 | Loss: 3.0201 | LR: 3.13e-04 | 1332.4ms/step | 15.0 samples/s +Step 18890/44000 | Loss: 3.0080 | LR: 3.13e-04 | 1334.8ms/step | 15.0 samples/s +Step 18900/44000 | Loss: 2.9973 | LR: 3.12e-04 | 1331.8ms/step | 15.0 samples/s +Step 18910/44000 | Loss: 2.9955 | LR: 3.12e-04 | 1328.9ms/step | 15.0 samples/s +Step 18920/44000 | Loss: 3.0269 | LR: 3.12e-04 | 1332.4ms/step | 15.0 samples/s +Step 18930/44000 | Loss: 3.0291 | LR: 3.12e-04 | 1332.7ms/step | 15.0 samples/s +Step 18940/44000 | Loss: 2.9991 | LR: 3.12e-04 | 1331.9ms/step | 15.0 samples/s +Step 18950/44000 | Loss: 2.9992 | LR: 3.12e-04 | 1332.8ms/step | 15.0 samples/s +Step 18960/44000 | Loss: 3.0023 | LR: 3.11e-04 | 1331.6ms/step | 15.0 samples/s +Step 18970/44000 | Loss: 3.0368 | LR: 3.11e-04 | 1332.6ms/step | 15.0 samples/s +Step 18980/44000 | Loss: 3.0380 | LR: 3.11e-04 | 1333.9ms/step | 15.0 samples/s +Step 18990/44000 | Loss: 3.0185 | LR: 3.11e-04 | 1332.7ms/step | 15.0 samples/s +Step 19000/44000 | Loss: 3.0309 | LR: 3.11e-04 | 1334.5ms/step | 15.0 samples/s +Validation at step 19000: Loss = 3.0241, Perplexity = 20.57 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-19000 +Step 19010/44000 | Loss: 3.0264 | LR: 3.11e-04 | 3889.8ms/step | 5.1 samples/s +Step 19020/44000 | Loss: 2.9797 | LR: 3.10e-04 | 1326.9ms/step | 15.1 samples/s +Step 19030/44000 | Loss: 3.0197 | LR: 3.10e-04 | 1327.5ms/step | 15.1 samples/s +Step 19040/44000 | Loss: 3.0310 | LR: 3.10e-04 | 1331.0ms/step | 15.0 samples/s +Step 19050/44000 | Loss: 3.0080 | LR: 3.10e-04 | 1331.7ms/step | 15.0 samples/s +Step 19060/44000 | Loss: 3.0164 | LR: 3.10e-04 | 1332.8ms/step | 15.0 samples/s +Step 19070/44000 | Loss: 3.0098 | LR: 3.09e-04 | 1333.0ms/step | 15.0 samples/s +Step 19080/44000 | Loss: 3.0071 | LR: 3.09e-04 | 1332.0ms/step | 15.0 samples/s +Step 19090/44000 | Loss: 3.0112 | LR: 3.09e-04 | 1331.8ms/step | 15.0 samples/s +Step 19100/44000 | Loss: 2.9811 | LR: 3.09e-04 | 1333.8ms/step | 15.0 samples/s +Step 19110/44000 | Loss: 3.0079 | LR: 3.09e-04 | 1333.8ms/step | 15.0 samples/s +Step 19120/44000 | Loss: 2.9694 | LR: 3.09e-04 | 1331.1ms/step | 15.0 samples/s +Step 19130/44000 | Loss: 3.0525 | LR: 3.08e-04 | 1331.6ms/step | 15.0 samples/s +Step 19140/44000 | Loss: 2.9950 | LR: 3.08e-04 | 1333.2ms/step | 15.0 samples/s +Step 19150/44000 | Loss: 3.0030 | LR: 3.08e-04 | 1331.4ms/step | 15.0 samples/s +Step 19160/44000 | Loss: 3.0195 | LR: 3.08e-04 | 1331.8ms/step | 15.0 samples/s +Step 19170/44000 | Loss: 3.0258 | LR: 3.08e-04 | 1331.5ms/step | 15.0 samples/s +Step 19180/44000 | Loss: 2.9788 | LR: 3.08e-04 | 1334.1ms/step | 15.0 samples/s +Step 19190/44000 | Loss: 3.0084 | LR: 3.07e-04 | 1335.1ms/step | 15.0 samples/s +Step 19200/44000 | Loss: 3.0013 | LR: 3.07e-04 | 1333.0ms/step | 15.0 samples/s +Step 19210/44000 | Loss: 2.9725 | LR: 3.07e-04 | 1332.3ms/step | 15.0 samples/s +Step 19220/44000 | Loss: 3.0267 | LR: 3.07e-04 | 1332.8ms/step | 15.0 samples/s +Step 19230/44000 | Loss: 3.0124 | LR: 3.07e-04 | 1332.4ms/step | 15.0 samples/s +Step 19240/44000 | Loss: 2.9885 | LR: 3.06e-04 | 1332.8ms/step | 15.0 samples/s +Step 19250/44000 | Loss: 3.0200 | LR: 3.06e-04 | 1335.6ms/step | 15.0 samples/s +Validation at step 19250: Loss = 3.0217, Perplexity = 20.53 +Step 19260/44000 | Loss: 3.0048 | LR: 3.06e-04 | 3569.5ms/step | 5.6 samples/s +Step 19270/44000 | Loss: 3.0119 | LR: 3.06e-04 | 1333.5ms/step | 15.0 samples/s +Step 19280/44000 | Loss: 3.0039 | LR: 3.06e-04 | 1331.8ms/step | 15.0 samples/s +Step 19290/44000 | Loss: 3.0069 | LR: 3.06e-04 | 1327.3ms/step | 15.1 samples/s +Step 19300/44000 | Loss: 2.9817 | LR: 3.05e-04 | 1324.6ms/step | 15.1 samples/s +Step 19310/44000 | Loss: 2.9828 | LR: 3.05e-04 | 1327.8ms/step | 15.1 samples/s +Step 19320/44000 | Loss: 3.0224 | LR: 3.05e-04 | 1326.0ms/step | 15.1 samples/s +Step 19330/44000 | Loss: 3.0061 | LR: 3.05e-04 | 1325.0ms/step | 15.1 samples/s +Step 19340/44000 | Loss: 3.0117 | LR: 3.05e-04 | 1326.2ms/step | 15.1 samples/s +Step 19350/44000 | Loss: 2.9964 | LR: 3.04e-04 | 1329.6ms/step | 15.0 samples/s +Step 19360/44000 | Loss: 2.9948 | LR: 3.04e-04 | 1331.3ms/step | 15.0 samples/s +Step 19370/44000 | Loss: 3.0199 | LR: 3.04e-04 | 1330.6ms/step | 15.0 samples/s +Step 19380/44000 | Loss: 3.0049 | LR: 3.04e-04 | 1324.4ms/step | 15.1 samples/s +Step 19390/44000 | Loss: 3.0181 | LR: 3.04e-04 | 1324.7ms/step | 15.1 samples/s +Step 19400/44000 | Loss: 3.0384 | LR: 3.04e-04 | 1327.9ms/step | 15.1 samples/s +Step 19410/44000 | Loss: 2.9787 | LR: 3.03e-04 | 1325.9ms/step | 15.1 samples/s +Step 19420/44000 | Loss: 3.0194 | LR: 3.03e-04 | 1326.4ms/step | 15.1 samples/s +Step 19430/44000 | Loss: 3.0215 | LR: 3.03e-04 | 1326.5ms/step | 15.1 samples/s +Step 19440/44000 | Loss: 3.0117 | LR: 3.03e-04 | 1326.5ms/step | 15.1 samples/s +Step 19450/44000 | Loss: 2.9873 | LR: 3.03e-04 | 1326.4ms/step | 15.1 samples/s +Step 19460/44000 | Loss: 3.0155 | LR: 3.03e-04 | 1326.4ms/step | 15.1 samples/s +Step 19470/44000 | Loss: 3.0216 | LR: 3.02e-04 | 1328.1ms/step | 15.1 samples/s +Step 19480/44000 | Loss: 3.0207 | LR: 3.02e-04 | 1324.6ms/step | 15.1 samples/s +Step 19490/44000 | Loss: 2.9888 | LR: 3.02e-04 | 1324.4ms/step | 15.1 samples/s +Step 19500/44000 | Loss: 2.9877 | LR: 3.02e-04 | 1326.6ms/step | 15.1 samples/s +Validation at step 19500: Loss = 3.0197, Perplexity = 20.49 +Step 19510/44000 | Loss: 2.9891 | LR: 3.02e-04 | 3551.6ms/step | 5.6 samples/s +Step 19520/44000 | Loss: 2.9893 | LR: 3.01e-04 | 1324.8ms/step | 15.1 samples/s +Step 19530/44000 | Loss: 2.9778 | LR: 3.01e-04 | 1323.6ms/step | 15.1 samples/s +Step 19540/44000 | Loss: 3.0004 | LR: 3.01e-04 | 1326.4ms/step | 15.1 samples/s +Step 19550/44000 | Loss: 3.0019 | LR: 3.01e-04 | 1325.6ms/step | 15.1 samples/s +Step 19560/44000 | Loss: 3.0009 | LR: 3.01e-04 | 1327.5ms/step | 15.1 samples/s +Step 19570/44000 | Loss: 3.0099 | LR: 3.01e-04 | 1325.7ms/step | 15.1 samples/s +Step 19580/44000 | Loss: 2.9832 | LR: 3.00e-04 | 1321.8ms/step | 15.1 samples/s +Step 19590/44000 | Loss: 2.9809 | LR: 3.00e-04 | 1326.5ms/step | 15.1 samples/s +Step 19600/44000 | Loss: 3.0394 | LR: 3.00e-04 | 1325.5ms/step | 15.1 samples/s +Step 19610/44000 | Loss: 3.0092 | LR: 3.00e-04 | 1325.7ms/step | 15.1 samples/s +Step 19620/44000 | Loss: 3.0081 | LR: 3.00e-04 | 1327.5ms/step | 15.1 samples/s +Step 19630/44000 | Loss: 3.0056 | LR: 3.00e-04 | 1325.3ms/step | 15.1 samples/s +Step 19640/44000 | Loss: 3.0272 | LR: 2.99e-04 | 1329.1ms/step | 15.0 samples/s +Step 19650/44000 | Loss: 3.0145 | LR: 2.99e-04 | 1328.5ms/step | 15.1 samples/s +Step 19660/44000 | Loss: 3.0290 | LR: 2.99e-04 | 1326.0ms/step | 15.1 samples/s +Step 19670/44000 | Loss: 2.9964 | LR: 2.99e-04 | 1327.8ms/step | 15.1 samples/s +Step 19680/44000 | Loss: 2.9635 | LR: 2.99e-04 | 1325.7ms/step | 15.1 samples/s +Step 19690/44000 | Loss: 3.0257 | LR: 2.98e-04 | 1322.7ms/step | 15.1 samples/s +Step 19700/44000 | Loss: 3.0134 | LR: 2.98e-04 | 1325.1ms/step | 15.1 samples/s +Step 19710/44000 | Loss: 2.9968 | LR: 2.98e-04 | 1326.9ms/step | 15.1 samples/s +Step 19720/44000 | Loss: 3.0071 | LR: 2.98e-04 | 1326.0ms/step | 15.1 samples/s +Step 19730/44000 | Loss: 3.0199 | LR: 2.98e-04 | 1327.1ms/step | 15.1 samples/s +Step 19740/44000 | Loss: 2.9919 | LR: 2.98e-04 | 1328.7ms/step | 15.1 samples/s +Step 19750/44000 | Loss: 3.0284 | LR: 2.97e-04 | 1330.9ms/step | 15.0 samples/s +Validation at step 19750: Loss = 3.0164, Perplexity = 20.42 +Step 19760/44000 | Loss: 3.0141 | LR: 2.97e-04 | 3556.5ms/step | 5.6 samples/s +Step 19770/44000 | Loss: 2.9985 | LR: 2.97e-04 | 1328.2ms/step | 15.1 samples/s +Step 19780/44000 | Loss: 2.9977 | LR: 2.97e-04 | 1329.9ms/step | 15.0 samples/s +Step 19790/44000 | Loss: 3.0210 | LR: 2.97e-04 | 1326.7ms/step | 15.1 samples/s +Step 19800/44000 | Loss: 2.9776 | LR: 2.96e-04 | 1328.3ms/step | 15.1 samples/s +Step 19810/44000 | Loss: 3.0037 | LR: 2.96e-04 | 1327.6ms/step | 15.1 samples/s +Step 19820/44000 | Loss: 3.0056 | LR: 2.96e-04 | 1329.2ms/step | 15.0 samples/s +Step 19830/44000 | Loss: 3.0311 | LR: 2.96e-04 | 1328.2ms/step | 15.1 samples/s +Step 19840/44000 | Loss: 2.9710 | LR: 2.96e-04 | 1331.3ms/step | 15.0 samples/s +Step 19850/44000 | Loss: 2.9987 | LR: 2.96e-04 | 1330.4ms/step | 15.0 samples/s +Step 19860/44000 | Loss: 2.9892 | LR: 2.95e-04 | 1328.9ms/step | 15.1 samples/s +Step 19870/44000 | Loss: 2.9845 | LR: 2.95e-04 | 1330.9ms/step | 15.0 samples/s +Step 19880/44000 | Loss: 3.0185 | LR: 2.95e-04 | 1331.8ms/step | 15.0 samples/s +Step 19890/44000 | Loss: 2.9987 | LR: 2.95e-04 | 1332.3ms/step | 15.0 samples/s +Step 19900/44000 | Loss: 2.9871 | LR: 2.95e-04 | 1329.1ms/step | 15.0 samples/s +Step 19910/44000 | Loss: 2.9729 | LR: 2.95e-04 | 1332.2ms/step | 15.0 samples/s +Step 19920/44000 | Loss: 3.0122 | LR: 2.94e-04 | 1331.4ms/step | 15.0 samples/s +Step 19930/44000 | Loss: 3.0090 | LR: 2.94e-04 | 1332.0ms/step | 15.0 samples/s +Step 19940/44000 | Loss: 3.0039 | LR: 2.94e-04 | 1333.8ms/step | 15.0 samples/s +Step 19950/44000 | Loss: 2.9931 | LR: 2.94e-04 | 1334.8ms/step | 15.0 samples/s +Step 19960/44000 | Loss: 2.9995 | LR: 2.94e-04 | 1332.7ms/step | 15.0 samples/s +Step 19970/44000 | Loss: 3.0013 | LR: 2.93e-04 | 1333.7ms/step | 15.0 samples/s +Step 19980/44000 | Loss: 2.9915 | LR: 2.93e-04 | 1334.8ms/step | 15.0 samples/s +Step 19990/44000 | Loss: 3.0028 | LR: 2.93e-04 | 1334.1ms/step | 15.0 samples/s +Step 20000/44000 | Loss: 2.9815 | LR: 2.93e-04 | 1336.9ms/step | 15.0 samples/s +Validation at step 20000: Loss = 3.0139, Perplexity = 20.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-20000 +Step 20010/44000 | Loss: 3.0179 | LR: 2.93e-04 | 3912.8ms/step | 5.1 samples/s +Step 20020/44000 | Loss: 3.0166 | LR: 2.93e-04 | 1330.4ms/step | 15.0 samples/s +Step 20030/44000 | Loss: 3.0085 | LR: 2.92e-04 | 1329.6ms/step | 15.0 samples/s +Step 20040/44000 | Loss: 3.0098 | LR: 2.92e-04 | 1330.3ms/step | 15.0 samples/s +Step 20050/44000 | Loss: 3.0185 | LR: 2.92e-04 | 1333.7ms/step | 15.0 samples/s +Step 20060/44000 | Loss: 3.0154 | LR: 2.92e-04 | 1331.2ms/step | 15.0 samples/s +Step 20070/44000 | Loss: 3.0011 | LR: 2.92e-04 | 1331.8ms/step | 15.0 samples/s +Step 20080/44000 | Loss: 3.0129 | LR: 2.91e-04 | 1329.7ms/step | 15.0 samples/s diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_static_nrb35_seed43.log b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_static_nrb35_seed43.log new file mode 100644 index 0000000000000000000000000000000000000000..3ebf2d92b56a9e6d3fb48ccda949eb43dc5e4e2c --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/l8k365m_static_nrb35_seed43.log @@ -0,0 +1,2294 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 43, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 365m static 8k nrb35 seed43", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "static", + "nrb35", + "seed43" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1", + "sparse_mask_seed": 1, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run ino9n2jx +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_164441-ino9n2jx +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 365m static 8k nrb35 seed43 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ino9n2jx +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0470 | LR: 6.67e-06 | 6774.5ms/step | 3.0 samples/s +Step 20/44000 | Loss: 11.9537 | LR: 1.33e-05 | 1317.6ms/step | 15.2 samples/s +Step 30/44000 | Loss: 11.6337 | LR: 2.00e-05 | 1314.0ms/step | 15.2 samples/s +Step 40/44000 | Loss: 11.1033 | LR: 2.67e-05 | 1313.5ms/step | 15.2 samples/s +Step 50/44000 | Loss: 10.7688 | LR: 3.33e-05 | 1309.1ms/step | 15.3 samples/s +Step 60/44000 | Loss: 10.5343 | LR: 4.00e-05 | 1308.6ms/step | 15.3 samples/s +Step 70/44000 | Loss: 10.2893 | LR: 4.67e-05 | 1305.1ms/step | 15.3 samples/s +Step 80/44000 | Loss: 9.9868 | LR: 5.33e-05 | 1306.7ms/step | 15.3 samples/s +Step 90/44000 | Loss: 9.6281 | LR: 6.00e-05 | 1310.1ms/step | 15.3 samples/s +Step 100/44000 | Loss: 9.1348 | LR: 6.67e-05 | 1311.0ms/step | 15.3 samples/s +Step 110/44000 | Loss: 8.6259 | LR: 7.33e-05 | 1312.5ms/step | 15.2 samples/s +Step 120/44000 | Loss: 8.1271 | LR: 8.00e-05 | 1315.3ms/step | 15.2 samples/s +Step 130/44000 | Loss: 7.7671 | LR: 8.67e-05 | 1313.5ms/step | 15.2 samples/s +Step 140/44000 | Loss: 7.4947 | LR: 9.33e-05 | 1313.9ms/step | 15.2 samples/s +Step 150/44000 | Loss: 7.3371 | LR: 1.00e-04 | 1318.6ms/step | 15.2 samples/s +Step 160/44000 | Loss: 7.2115 | LR: 1.07e-04 | 1319.2ms/step | 15.2 samples/s +Step 170/44000 | Loss: 7.1190 | LR: 1.13e-04 | 1323.0ms/step | 15.1 samples/s +Step 180/44000 | Loss: 7.0414 | LR: 1.20e-04 | 1321.2ms/step | 15.1 samples/s +Step 190/44000 | Loss: 6.9289 | LR: 1.27e-04 | 1322.9ms/step | 15.1 samples/s +Step 200/44000 | Loss: 6.8251 | LR: 1.33e-04 | 1324.1ms/step | 15.1 samples/s +Step 210/44000 | Loss: 6.7331 | LR: 1.40e-04 | 1323.3ms/step | 15.1 samples/s +Step 220/44000 | Loss: 6.6696 | LR: 1.47e-04 | 1325.2ms/step | 15.1 samples/s +Step 230/44000 | Loss: 6.6008 | LR: 1.53e-04 | 1325.6ms/step | 15.1 samples/s +Step 240/44000 | Loss: 6.5295 | LR: 1.60e-04 | 1323.3ms/step | 15.1 samples/s +Step 250/44000 | Loss: 6.4597 | LR: 1.67e-04 | 1326.8ms/step | 15.1 samples/s +Validation at step 250: Loss = 6.4378, Perplexity = 625.02 +Step 260/44000 | Loss: 6.4160 | LR: 1.73e-04 | 5675.3ms/step | 3.5 samples/s +Step 270/44000 | Loss: 6.3608 | LR: 1.80e-04 | 1323.8ms/step | 15.1 samples/s +Step 280/44000 | Loss: 6.3001 | LR: 1.87e-04 | 1322.0ms/step | 15.1 samples/s +Step 290/44000 | Loss: 6.2491 | LR: 1.93e-04 | 1322.4ms/step | 15.1 samples/s +Step 300/44000 | Loss: 6.1835 | LR: 2.00e-04 | 1322.0ms/step | 15.1 samples/s +Step 310/44000 | Loss: 6.1498 | LR: 2.07e-04 | 1325.7ms/step | 15.1 samples/s +Step 320/44000 | Loss: 6.1243 | LR: 2.13e-04 | 1325.6ms/step | 15.1 samples/s +Step 330/44000 | Loss: 6.0735 | LR: 2.20e-04 | 1324.6ms/step | 15.1 samples/s +Step 340/44000 | Loss: 6.0716 | LR: 2.27e-04 | 1324.4ms/step | 15.1 samples/s +Step 350/44000 | Loss: 6.0198 | LR: 2.33e-04 | 1325.6ms/step | 15.1 samples/s +Step 360/44000 | Loss: 5.9495 | LR: 2.40e-04 | 1324.1ms/step | 15.1 samples/s +Step 370/44000 | Loss: 5.9392 | LR: 2.47e-04 | 1324.9ms/step | 15.1 samples/s +Step 380/44000 | Loss: 5.8976 | LR: 2.53e-04 | 1323.6ms/step | 15.1 samples/s +Step 390/44000 | Loss: 5.8355 | LR: 2.60e-04 | 1323.3ms/step | 15.1 samples/s +Step 400/44000 | Loss: 5.8310 | LR: 2.67e-04 | 1327.2ms/step | 15.1 samples/s +Step 410/44000 | Loss: 5.7668 | LR: 2.73e-04 | 1327.3ms/step | 15.1 samples/s +Step 420/44000 | Loss: 5.7333 | LR: 2.80e-04 | 1324.5ms/step | 15.1 samples/s +Step 430/44000 | Loss: 5.7443 | LR: 2.87e-04 | 1323.7ms/step | 15.1 samples/s +Step 440/44000 | Loss: 5.6784 | LR: 2.93e-04 | 1324.1ms/step | 15.1 samples/s +Step 450/44000 | Loss: 5.6608 | LR: 3.00e-04 | 1324.4ms/step | 15.1 samples/s +Step 460/44000 | Loss: 5.6084 | LR: 3.07e-04 | 1323.4ms/step | 15.1 samples/s +Step 470/44000 | Loss: 5.5683 | LR: 3.13e-04 | 1325.2ms/step | 15.1 samples/s +Step 480/44000 | Loss: 5.5602 | LR: 3.20e-04 | 1328.0ms/step | 15.1 samples/s +Step 490/44000 | Loss: 5.4884 | LR: 3.27e-04 | 1328.8ms/step | 15.1 samples/s +Step 500/44000 | Loss: 5.4693 | LR: 3.33e-04 | 1327.1ms/step | 15.1 samples/s +Validation at step 500: Loss = 5.4731, Perplexity = 238.20 +Step 510/44000 | Loss: 5.4628 | LR: 3.40e-04 | 3559.6ms/step | 5.6 samples/s +Step 520/44000 | Loss: 5.4241 | LR: 3.47e-04 | 1327.9ms/step | 15.1 samples/s +Step 530/44000 | Loss: 5.3856 | LR: 3.53e-04 | 1328.8ms/step | 15.1 samples/s +Step 540/44000 | Loss: 5.3822 | LR: 3.60e-04 | 1326.7ms/step | 15.1 samples/s +Step 550/44000 | Loss: 5.3294 | LR: 3.67e-04 | 1327.8ms/step | 15.1 samples/s +Step 560/44000 | Loss: 5.2894 | LR: 3.73e-04 | 1329.7ms/step | 15.0 samples/s +Step 570/44000 | Loss: 5.2647 | LR: 3.80e-04 | 1331.3ms/step | 15.0 samples/s +Step 580/44000 | Loss: 5.2609 | LR: 3.87e-04 | 1328.4ms/step | 15.1 samples/s +Step 590/44000 | Loss: 5.2304 | LR: 3.93e-04 | 1327.1ms/step | 15.1 samples/s +Step 600/44000 | Loss: 5.1906 | LR: 4.00e-04 | 1325.6ms/step | 15.1 samples/s +Step 610/44000 | Loss: 5.1640 | LR: 4.07e-04 | 1325.5ms/step | 15.1 samples/s +Step 620/44000 | Loss: 5.1523 | LR: 4.13e-04 | 1328.1ms/step | 15.1 samples/s +Step 630/44000 | Loss: 5.0753 | LR: 4.20e-04 | 1325.8ms/step | 15.1 samples/s +Step 640/44000 | Loss: 5.0469 | LR: 4.27e-04 | 1328.9ms/step | 15.1 samples/s +Step 650/44000 | Loss: 5.0304 | LR: 4.33e-04 | 1327.0ms/step | 15.1 samples/s +Step 660/44000 | Loss: 4.9970 | LR: 4.40e-04 | 1325.2ms/step | 15.1 samples/s +Step 670/44000 | Loss: 4.9849 | LR: 4.47e-04 | 1327.2ms/step | 15.1 samples/s +Step 680/44000 | Loss: 4.9196 | LR: 4.53e-04 | 1326.8ms/step | 15.1 samples/s +Step 690/44000 | Loss: 4.9095 | LR: 4.60e-04 | 1324.8ms/step | 15.1 samples/s +Step 700/44000 | Loss: 4.8612 | LR: 4.67e-04 | 1326.5ms/step | 15.1 samples/s +Step 710/44000 | Loss: 4.8244 | LR: 4.73e-04 | 1328.0ms/step | 15.1 samples/s +Step 720/44000 | Loss: 4.8602 | LR: 4.80e-04 | 1327.4ms/step | 15.1 samples/s +Step 730/44000 | Loss: 4.7888 | LR: 4.87e-04 | 1326.7ms/step | 15.1 samples/s +Step 740/44000 | Loss: 4.7946 | LR: 4.93e-04 | 1324.6ms/step | 15.1 samples/s +Step 750/44000 | Loss: 4.7608 | LR: 5.00e-04 | 1325.1ms/step | 15.1 samples/s +Validation at step 750: Loss = 4.7634, Perplexity = 117.14 +Step 760/44000 | Loss: 4.7765 | LR: 5.00e-04 | 3552.4ms/step | 5.6 samples/s +Step 770/44000 | Loss: 4.6907 | LR: 5.00e-04 | 1327.9ms/step | 15.1 samples/s +Step 780/44000 | Loss: 4.7062 | LR: 5.00e-04 | 1325.5ms/step | 15.1 samples/s +Step 790/44000 | Loss: 4.6347 | LR: 5.00e-04 | 1323.5ms/step | 15.1 samples/s +Step 800/44000 | Loss: 4.5925 | LR: 5.00e-04 | 1324.4ms/step | 15.1 samples/s +Step 810/44000 | Loss: 4.6020 | LR: 5.00e-04 | 1325.8ms/step | 15.1 samples/s +Step 820/44000 | Loss: 4.5908 | LR: 5.00e-04 | 1328.4ms/step | 15.1 samples/s +Step 830/44000 | Loss: 4.5578 | LR: 5.00e-04 | 1328.6ms/step | 15.1 samples/s +Step 840/44000 | Loss: 4.5341 | LR: 5.00e-04 | 1327.8ms/step | 15.1 samples/s +Step 850/44000 | Loss: 4.5090 | LR: 5.00e-04 | 1328.1ms/step | 15.1 samples/s +Step 860/44000 | Loss: 4.5229 | LR: 5.00e-04 | 1331.9ms/step | 15.0 samples/s +Step 870/44000 | Loss: 4.4784 | LR: 5.00e-04 | 1327.7ms/step | 15.1 samples/s +Step 880/44000 | Loss: 4.4846 | LR: 5.00e-04 | 1328.1ms/step | 15.1 samples/s +Step 890/44000 | Loss: 4.4764 | LR: 5.00e-04 | 1328.5ms/step | 15.1 samples/s +Step 900/44000 | Loss: 4.4552 | LR: 5.00e-04 | 1331.2ms/step | 15.0 samples/s +Step 910/44000 | Loss: 4.4009 | LR: 5.00e-04 | 1329.3ms/step | 15.0 samples/s +Step 920/44000 | Loss: 4.4300 | LR: 5.00e-04 | 1329.2ms/step | 15.0 samples/s +Step 930/44000 | Loss: 4.3914 | LR: 5.00e-04 | 1328.9ms/step | 15.0 samples/s +Step 940/44000 | Loss: 4.3746 | LR: 5.00e-04 | 1328.7ms/step | 15.1 samples/s +Step 950/44000 | Loss: 4.3882 | LR: 5.00e-04 | 1332.7ms/step | 15.0 samples/s +Step 960/44000 | Loss: 4.3622 | LR: 5.00e-04 | 1333.3ms/step | 15.0 samples/s +Step 970/44000 | Loss: 4.3572 | LR: 5.00e-04 | 1334.7ms/step | 15.0 samples/s +Step 980/44000 | Loss: 4.3201 | LR: 5.00e-04 | 1333.4ms/step | 15.0 samples/s +Step 990/44000 | Loss: 4.3107 | LR: 5.00e-04 | 1331.6ms/step | 15.0 samples/s +Step 1000/44000 | Loss: 4.3134 | LR: 5.00e-04 | 1331.0ms/step | 15.0 samples/s +Validation at step 1000: Loss = 4.2889, Perplexity = 72.88 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-1000 +Step 1010/44000 | Loss: 4.2953 | LR: 5.00e-04 | 3889.1ms/step | 5.1 samples/s +Step 1020/44000 | Loss: 4.3095 | LR: 5.00e-04 | 1325.4ms/step | 15.1 samples/s +Step 1030/44000 | Loss: 4.2630 | LR: 5.00e-04 | 1325.7ms/step | 15.1 samples/s +Step 1040/44000 | Loss: 4.2756 | LR: 5.00e-04 | 1328.3ms/step | 15.1 samples/s +Step 1050/44000 | Loss: 4.2479 | LR: 5.00e-04 | 1328.1ms/step | 15.1 samples/s +Step 1060/44000 | Loss: 4.2320 | LR: 5.00e-04 | 1325.6ms/step | 15.1 samples/s +Step 1070/44000 | Loss: 4.2131 | LR: 5.00e-04 | 1329.2ms/step | 15.0 samples/s +Step 1080/44000 | Loss: 4.2208 | LR: 5.00e-04 | 1329.2ms/step | 15.0 samples/s +Step 1090/44000 | Loss: 4.2126 | LR: 5.00e-04 | 1329.4ms/step | 15.0 samples/s +Step 1100/44000 | Loss: 4.1895 | LR: 5.00e-04 | 1329.4ms/step | 15.0 samples/s +Step 1110/44000 | Loss: 4.1609 | LR: 5.00e-04 | 1329.9ms/step | 15.0 samples/s +Step 1120/44000 | Loss: 4.1868 | LR: 5.00e-04 | 1330.6ms/step | 15.0 samples/s +Step 1130/44000 | Loss: 4.1759 | LR: 5.00e-04 | 1331.9ms/step | 15.0 samples/s +Step 1140/44000 | Loss: 4.1654 | LR: 5.00e-04 | 1331.7ms/step | 15.0 samples/s +Step 1150/44000 | Loss: 4.1655 | LR: 5.00e-04 | 1332.7ms/step | 15.0 samples/s +Step 1160/44000 | Loss: 4.1414 | LR: 5.00e-04 | 1330.8ms/step | 15.0 samples/s +Step 1170/44000 | Loss: 4.1254 | LR: 5.00e-04 | 1329.7ms/step | 15.0 samples/s +Step 1180/44000 | Loss: 4.1172 | LR: 5.00e-04 | 1329.5ms/step | 15.0 samples/s +Step 1190/44000 | Loss: 4.1371 | LR: 5.00e-04 | 1328.8ms/step | 15.1 samples/s +Step 1200/44000 | Loss: 4.0846 | LR: 5.00e-04 | 1328.4ms/step | 15.1 samples/s +Step 1210/44000 | Loss: 4.1087 | LR: 5.00e-04 | 1330.6ms/step | 15.0 samples/s +Step 1220/44000 | Loss: 4.0922 | LR: 5.00e-04 | 1330.2ms/step | 15.0 samples/s +Step 1230/44000 | Loss: 4.0860 | LR: 5.00e-04 | 1329.8ms/step | 15.0 samples/s +Step 1240/44000 | Loss: 4.0658 | LR: 5.00e-04 | 1331.4ms/step | 15.0 samples/s +Step 1250/44000 | Loss: 4.0221 | LR: 5.00e-04 | 1331.9ms/step | 15.0 samples/s +Validation at step 1250: Loss = 4.0572, Perplexity = 57.81 +Step 1260/44000 | Loss: 4.0716 | LR: 5.00e-04 | 3567.6ms/step | 5.6 samples/s +Step 1270/44000 | Loss: 4.0741 | LR: 5.00e-04 | 1331.4ms/step | 15.0 samples/s +Step 1280/44000 | Loss: 4.0467 | LR: 5.00e-04 | 1332.3ms/step | 15.0 samples/s +Step 1290/44000 | Loss: 4.0133 | LR: 5.00e-04 | 1331.6ms/step | 15.0 samples/s +Step 1300/44000 | Loss: 4.0554 | LR: 5.00e-04 | 1334.8ms/step | 15.0 samples/s +Step 1310/44000 | Loss: 4.0354 | LR: 5.00e-04 | 1328.9ms/step | 15.1 samples/s +Step 1320/44000 | Loss: 4.0190 | LR: 5.00e-04 | 1320.3ms/step | 15.1 samples/s +Step 1330/44000 | Loss: 4.0115 | LR: 5.00e-04 | 1323.8ms/step | 15.1 samples/s +Step 1340/44000 | Loss: 4.0125 | LR: 5.00e-04 | 1326.5ms/step | 15.1 samples/s +Step 1350/44000 | Loss: 4.0159 | LR: 5.00e-04 | 1326.6ms/step | 15.1 samples/s +Step 1360/44000 | Loss: 3.9942 | LR: 5.00e-04 | 1329.4ms/step | 15.0 samples/s +Step 1370/44000 | Loss: 3.9805 | LR: 5.00e-04 | 1327.8ms/step | 15.1 samples/s +Step 1380/44000 | Loss: 3.9687 | LR: 5.00e-04 | 1327.2ms/step | 15.1 samples/s +Step 1390/44000 | Loss: 3.9699 | LR: 5.00e-04 | 1327.2ms/step | 15.1 samples/s +Step 1400/44000 | Loss: 3.9418 | LR: 5.00e-04 | 1327.9ms/step | 15.1 samples/s +Step 1410/44000 | Loss: 3.9488 | LR: 5.00e-04 | 1326.0ms/step | 15.1 samples/s +Step 1420/44000 | Loss: 3.9324 | LR: 5.00e-04 | 1325.7ms/step | 15.1 samples/s +Step 1430/44000 | Loss: 3.9494 | LR: 5.00e-04 | 1326.1ms/step | 15.1 samples/s +Step 1440/44000 | Loss: 3.9158 | LR: 5.00e-04 | 1323.3ms/step | 15.1 samples/s +Step 1450/44000 | Loss: 3.9614 | LR: 5.00e-04 | 1326.3ms/step | 15.1 samples/s +Step 1460/44000 | Loss: 3.9275 | LR: 5.00e-04 | 1325.0ms/step | 15.1 samples/s +Step 1470/44000 | Loss: 3.9322 | LR: 5.00e-04 | 1328.9ms/step | 15.0 samples/s +Step 1480/44000 | Loss: 3.9170 | LR: 5.00e-04 | 1326.5ms/step | 15.1 samples/s +Step 1490/44000 | Loss: 3.9014 | LR: 5.00e-04 | 1327.3ms/step | 15.1 samples/s +Step 1500/44000 | Loss: 3.8937 | LR: 5.00e-04 | 1323.5ms/step | 15.1 samples/s +Validation at step 1500: Loss = 3.8929, Perplexity = 49.06 +Step 1510/44000 | Loss: 3.9115 | LR: 5.00e-04 | 3552.0ms/step | 5.6 samples/s +Step 1520/44000 | Loss: 3.9217 | LR: 5.00e-04 | 1325.3ms/step | 15.1 samples/s +Step 1530/44000 | Loss: 3.9039 | LR: 5.00e-04 | 1326.1ms/step | 15.1 samples/s +Step 1540/44000 | Loss: 3.9051 | LR: 5.00e-04 | 1326.0ms/step | 15.1 samples/s +Step 1550/44000 | Loss: 3.8641 | LR: 5.00e-04 | 1324.1ms/step | 15.1 samples/s +Step 1560/44000 | Loss: 3.8636 | LR: 5.00e-04 | 1327.5ms/step | 15.1 samples/s +Step 1570/44000 | Loss: 3.8871 | LR: 5.00e-04 | 1329.3ms/step | 15.0 samples/s +Step 1580/44000 | Loss: 3.8622 | LR: 5.00e-04 | 1324.3ms/step | 15.1 samples/s +Step 1590/44000 | Loss: 3.8620 | LR: 5.00e-04 | 1326.5ms/step | 15.1 samples/s +Step 1600/44000 | Loss: 3.8843 | LR: 5.00e-04 | 1326.3ms/step | 15.1 samples/s +Step 1610/44000 | Loss: 3.8267 | LR: 5.00e-04 | 1327.5ms/step | 15.1 samples/s +Step 1620/44000 | Loss: 3.8482 | LR: 5.00e-04 | 1326.9ms/step | 15.1 samples/s +Step 1630/44000 | Loss: 3.8357 | LR: 4.99e-04 | 1326.7ms/step | 15.1 samples/s +Step 1640/44000 | Loss: 3.8327 | LR: 4.99e-04 | 1327.9ms/step | 15.1 samples/s +Step 1650/44000 | Loss: 3.8799 | LR: 4.99e-04 | 1326.2ms/step | 15.1 samples/s +Step 1660/44000 | Loss: 3.8344 | LR: 4.99e-04 | 1324.6ms/step | 15.1 samples/s +Step 1670/44000 | Loss: 3.8375 | LR: 4.99e-04 | 1327.1ms/step | 15.1 samples/s +Step 1680/44000 | Loss: 3.8323 | LR: 4.99e-04 | 1327.0ms/step | 15.1 samples/s +Step 1690/44000 | Loss: 3.8302 | LR: 4.99e-04 | 1329.4ms/step | 15.0 samples/s +Step 1700/44000 | Loss: 3.7567 | LR: 4.99e-04 | 1326.0ms/step | 15.1 samples/s +Step 1710/44000 | Loss: 3.8202 | LR: 4.99e-04 | 1327.8ms/step | 15.1 samples/s +Step 1720/44000 | Loss: 3.8040 | LR: 4.99e-04 | 1326.5ms/step | 15.1 samples/s +Step 1730/44000 | Loss: 3.7851 | LR: 4.99e-04 | 1326.7ms/step | 15.1 samples/s +Step 1740/44000 | Loss: 3.7964 | LR: 4.99e-04 | 1327.4ms/step | 15.1 samples/s +Step 1750/44000 | Loss: 3.8013 | LR: 4.99e-04 | 1324.2ms/step | 15.1 samples/s +Validation at step 1750: Loss = 3.7845, Perplexity = 44.01 +Step 1760/44000 | Loss: 3.7852 | LR: 4.99e-04 | 3553.1ms/step | 5.6 samples/s +Step 1770/44000 | Loss: 3.7749 | LR: 4.99e-04 | 1327.7ms/step | 15.1 samples/s +Step 1780/44000 | Loss: 3.7687 | LR: 4.99e-04 | 1325.3ms/step | 15.1 samples/s +Step 1790/44000 | Loss: 3.7848 | LR: 4.99e-04 | 1326.2ms/step | 15.1 samples/s +Step 1800/44000 | Loss: 3.7841 | LR: 4.99e-04 | 1328.2ms/step | 15.1 samples/s +Step 1810/44000 | Loss: 3.7705 | LR: 4.99e-04 | 1328.5ms/step | 15.1 samples/s +Step 1820/44000 | Loss: 3.7724 | LR: 4.99e-04 | 1329.9ms/step | 15.0 samples/s +Step 1830/44000 | Loss: 3.7687 | LR: 4.99e-04 | 1331.0ms/step | 15.0 samples/s +Step 1840/44000 | Loss: 3.7580 | LR: 4.99e-04 | 1330.5ms/step | 15.0 samples/s +Step 1850/44000 | Loss: 3.7546 | LR: 4.99e-04 | 1331.4ms/step | 15.0 samples/s +Step 1860/44000 | Loss: 3.7492 | LR: 4.99e-04 | 1332.2ms/step | 15.0 samples/s +Step 1870/44000 | Loss: 3.7680 | LR: 4.99e-04 | 1331.0ms/step | 15.0 samples/s +Step 1880/44000 | Loss: 3.7594 | LR: 4.99e-04 | 1331.4ms/step | 15.0 samples/s +Step 1890/44000 | Loss: 3.7718 | LR: 4.99e-04 | 1329.9ms/step | 15.0 samples/s +Step 1900/44000 | Loss: 3.7302 | LR: 4.99e-04 | 1327.3ms/step | 15.1 samples/s +Step 1910/44000 | Loss: 3.7447 | LR: 4.99e-04 | 1329.6ms/step | 15.0 samples/s +Step 1920/44000 | Loss: 3.7500 | LR: 4.99e-04 | 1328.8ms/step | 15.1 samples/s +Step 1930/44000 | Loss: 3.7502 | LR: 4.99e-04 | 1330.4ms/step | 15.0 samples/s +Step 1940/44000 | Loss: 3.7349 | LR: 4.99e-04 | 1330.5ms/step | 15.0 samples/s +Step 1950/44000 | Loss: 3.7395 | LR: 4.99e-04 | 1331.2ms/step | 15.0 samples/s +Step 1960/44000 | Loss: 3.7251 | LR: 4.99e-04 | 1330.6ms/step | 15.0 samples/s +Step 1970/44000 | Loss: 3.7190 | LR: 4.99e-04 | 1330.4ms/step | 15.0 samples/s +Step 1980/44000 | Loss: 3.7155 | LR: 4.99e-04 | 1329.5ms/step | 15.0 samples/s +Step 1990/44000 | Loss: 3.7021 | LR: 4.99e-04 | 1329.7ms/step | 15.0 samples/s +Step 2000/44000 | Loss: 3.7202 | LR: 4.99e-04 | 1331.4ms/step | 15.0 samples/s +Validation at step 2000: Loss = 3.6984, Perplexity = 40.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-2000 +Step 2010/44000 | Loss: 3.6966 | LR: 4.99e-04 | 3881.5ms/step | 5.2 samples/s +Step 2020/44000 | Loss: 3.7002 | LR: 4.99e-04 | 1326.0ms/step | 15.1 samples/s +Step 2030/44000 | Loss: 3.7029 | LR: 4.99e-04 | 1324.0ms/step | 15.1 samples/s +Step 2040/44000 | Loss: 3.6947 | LR: 4.99e-04 | 1327.1ms/step | 15.1 samples/s +Step 2050/44000 | Loss: 3.7067 | LR: 4.99e-04 | 1329.5ms/step | 15.0 samples/s +Step 2060/44000 | Loss: 3.6840 | LR: 4.99e-04 | 1328.4ms/step | 15.1 samples/s +Step 2070/44000 | Loss: 3.6711 | LR: 4.99e-04 | 1329.4ms/step | 15.0 samples/s +Step 2080/44000 | Loss: 3.6710 | LR: 4.99e-04 | 1329.7ms/step | 15.0 samples/s +Step 2090/44000 | Loss: 3.6681 | LR: 4.99e-04 | 1330.0ms/step | 15.0 samples/s +Step 2100/44000 | Loss: 3.7064 | LR: 4.99e-04 | 1328.9ms/step | 15.1 samples/s +Step 2110/44000 | Loss: 3.6823 | LR: 4.99e-04 | 1330.4ms/step | 15.0 samples/s +Step 2120/44000 | Loss: 3.6924 | LR: 4.99e-04 | 1331.2ms/step | 15.0 samples/s +Step 2130/44000 | Loss: 3.6751 | LR: 4.99e-04 | 1331.8ms/step | 15.0 samples/s +Step 2140/44000 | Loss: 3.6977 | LR: 4.99e-04 | 1330.6ms/step | 15.0 samples/s +Step 2150/44000 | Loss: 3.6680 | LR: 4.99e-04 | 1330.3ms/step | 15.0 samples/s +Step 2160/44000 | Loss: 3.6789 | LR: 4.99e-04 | 1332.0ms/step | 15.0 samples/s +Step 2170/44000 | Loss: 3.6459 | LR: 4.99e-04 | 1330.4ms/step | 15.0 samples/s +Step 2180/44000 | Loss: 3.6506 | LR: 4.99e-04 | 1332.4ms/step | 15.0 samples/s +Step 2190/44000 | Loss: 3.6084 | LR: 4.99e-04 | 1330.0ms/step | 15.0 samples/s +Step 2200/44000 | Loss: 3.6678 | LR: 4.99e-04 | 1330.3ms/step | 15.0 samples/s +Step 2210/44000 | Loss: 3.6544 | LR: 4.99e-04 | 1332.1ms/step | 15.0 samples/s +Step 2220/44000 | Loss: 3.6291 | LR: 4.99e-04 | 1330.3ms/step | 15.0 samples/s +Step 2230/44000 | Loss: 3.6427 | LR: 4.99e-04 | 1330.4ms/step | 15.0 samples/s +Step 2240/44000 | Loss: 3.6426 | LR: 4.99e-04 | 1331.5ms/step | 15.0 samples/s +Step 2250/44000 | Loss: 3.6634 | LR: 4.99e-04 | 1333.8ms/step | 15.0 samples/s +Validation at step 2250: Loss = 3.6313, Perplexity = 37.76 +Step 2260/44000 | Loss: 3.6427 | LR: 4.98e-04 | 3564.7ms/step | 5.6 samples/s +Step 2270/44000 | Loss: 3.6227 | LR: 4.98e-04 | 1330.2ms/step | 15.0 samples/s +Step 2280/44000 | Loss: 3.6463 | LR: 4.98e-04 | 1332.2ms/step | 15.0 samples/s +Step 2290/44000 | Loss: 3.6292 | LR: 4.98e-04 | 1332.5ms/step | 15.0 samples/s +Step 2300/44000 | Loss: 3.6223 | LR: 4.98e-04 | 1330.6ms/step | 15.0 samples/s +Step 2310/44000 | Loss: 3.6215 | LR: 4.98e-04 | 1331.0ms/step | 15.0 samples/s +Step 2320/44000 | Loss: 3.6344 | LR: 4.98e-04 | 1331.2ms/step | 15.0 samples/s +Step 2330/44000 | Loss: 3.6446 | LR: 4.98e-04 | 1331.3ms/step | 15.0 samples/s +Step 2340/44000 | Loss: 3.5887 | LR: 4.98e-04 | 1329.7ms/step | 15.0 samples/s +Step 2350/44000 | Loss: 3.6160 | LR: 4.98e-04 | 1326.7ms/step | 15.1 samples/s +Step 2360/44000 | Loss: 3.6316 | LR: 4.98e-04 | 1328.0ms/step | 15.1 samples/s +Step 2370/44000 | Loss: 3.6070 | LR: 4.98e-04 | 1328.5ms/step | 15.1 samples/s +Step 2380/44000 | Loss: 3.5785 | LR: 4.98e-04 | 1326.9ms/step | 15.1 samples/s +Step 2390/44000 | Loss: 3.6517 | LR: 4.98e-04 | 1324.2ms/step | 15.1 samples/s +Step 2400/44000 | Loss: 3.6014 | LR: 4.98e-04 | 1326.0ms/step | 15.1 samples/s +Step 2410/44000 | Loss: 3.6139 | LR: 4.98e-04 | 1329.8ms/step | 15.0 samples/s +Step 2420/44000 | Loss: 3.6178 | LR: 4.98e-04 | 1330.9ms/step | 15.0 samples/s +Step 2430/44000 | Loss: 3.6279 | LR: 4.98e-04 | 1327.9ms/step | 15.1 samples/s +Step 2440/44000 | Loss: 3.5861 | LR: 4.98e-04 | 1327.6ms/step | 15.1 samples/s +Step 2450/44000 | Loss: 3.6012 | LR: 4.98e-04 | 1330.0ms/step | 15.0 samples/s +Step 2460/44000 | Loss: 3.6070 | LR: 4.98e-04 | 1330.5ms/step | 15.0 samples/s +Step 2470/44000 | Loss: 3.5827 | LR: 4.98e-04 | 1329.5ms/step | 15.0 samples/s +Step 2480/44000 | Loss: 3.5710 | LR: 4.98e-04 | 1330.3ms/step | 15.0 samples/s +Step 2490/44000 | Loss: 3.6307 | LR: 4.98e-04 | 1326.0ms/step | 15.1 samples/s +Step 2500/44000 | Loss: 3.6056 | LR: 4.98e-04 | 1326.9ms/step | 15.1 samples/s +Validation at step 2500: Loss = 3.5814, Perplexity = 35.92 +Step 2510/44000 | Loss: 3.5933 | LR: 4.98e-04 | 3557.3ms/step | 5.6 samples/s +Step 2520/44000 | Loss: 3.5519 | LR: 4.98e-04 | 1327.9ms/step | 15.1 samples/s +Step 2530/44000 | Loss: 3.5809 | LR: 4.98e-04 | 1327.9ms/step | 15.1 samples/s +Step 2540/44000 | Loss: 3.5604 | LR: 4.98e-04 | 1327.5ms/step | 15.1 samples/s +Step 2550/44000 | Loss: 3.5981 | LR: 4.98e-04 | 1327.4ms/step | 15.1 samples/s +Step 2560/44000 | Loss: 3.6109 | LR: 4.98e-04 | 1328.9ms/step | 15.1 samples/s +Step 2570/44000 | Loss: 3.5771 | LR: 4.98e-04 | 1331.8ms/step | 15.0 samples/s +Step 2580/44000 | Loss: 3.5757 | LR: 4.98e-04 | 1325.7ms/step | 15.1 samples/s +Step 2590/44000 | Loss: 3.5765 | LR: 4.98e-04 | 1327.1ms/step | 15.1 samples/s +Step 2600/44000 | Loss: 3.5679 | LR: 4.98e-04 | 1327.7ms/step | 15.1 samples/s +Step 2610/44000 | Loss: 3.5795 | LR: 4.98e-04 | 1329.9ms/step | 15.0 samples/s +Step 2620/44000 | Loss: 3.5680 | LR: 4.98e-04 | 1327.4ms/step | 15.1 samples/s +Step 2630/44000 | Loss: 3.6038 | LR: 4.98e-04 | 1326.3ms/step | 15.1 samples/s +Step 2640/44000 | Loss: 3.5481 | LR: 4.98e-04 | 1325.4ms/step | 15.1 samples/s +Step 2650/44000 | Loss: 3.5699 | LR: 4.98e-04 | 1326.2ms/step | 15.1 samples/s +Step 2660/44000 | Loss: 3.5692 | LR: 4.98e-04 | 1327.1ms/step | 15.1 samples/s +Step 2670/44000 | Loss: 3.5663 | LR: 4.98e-04 | 1328.8ms/step | 15.1 samples/s +Step 2680/44000 | Loss: 3.5787 | LR: 4.98e-04 | 1328.9ms/step | 15.0 samples/s +Step 2690/44000 | Loss: 3.5685 | LR: 4.98e-04 | 1325.4ms/step | 15.1 samples/s +Step 2700/44000 | Loss: 3.5773 | LR: 4.97e-04 | 1327.7ms/step | 15.1 samples/s +Step 2710/44000 | Loss: 3.5836 | LR: 4.97e-04 | 1326.1ms/step | 15.1 samples/s +Step 2720/44000 | Loss: 3.5351 | LR: 4.97e-04 | 1329.6ms/step | 15.0 samples/s +Step 2730/44000 | Loss: 3.5333 | LR: 4.97e-04 | 1324.5ms/step | 15.1 samples/s +Step 2740/44000 | Loss: 3.5495 | LR: 4.97e-04 | 1328.8ms/step | 15.1 samples/s +Step 2750/44000 | Loss: 3.5652 | LR: 4.97e-04 | 1327.9ms/step | 15.1 samples/s +Validation at step 2750: Loss = 3.5388, Perplexity = 34.42 +Step 2760/44000 | Loss: 3.5531 | LR: 4.97e-04 | 3558.1ms/step | 5.6 samples/s +Step 2770/44000 | Loss: 3.5675 | LR: 4.97e-04 | 1328.7ms/step | 15.1 samples/s +Step 2780/44000 | Loss: 3.5329 | LR: 4.97e-04 | 1325.7ms/step | 15.1 samples/s +Step 2790/44000 | Loss: 3.5399 | LR: 4.97e-04 | 1326.9ms/step | 15.1 samples/s +Step 2800/44000 | Loss: 3.5194 | LR: 4.97e-04 | 1326.3ms/step | 15.1 samples/s +Step 2810/44000 | Loss: 3.5461 | LR: 4.97e-04 | 1327.1ms/step | 15.1 samples/s +Step 2820/44000 | Loss: 3.4897 | LR: 4.97e-04 | 1326.4ms/step | 15.1 samples/s +Step 2830/44000 | Loss: 3.5491 | LR: 4.97e-04 | 1325.7ms/step | 15.1 samples/s +Step 2840/44000 | Loss: 3.5712 | LR: 4.97e-04 | 1325.3ms/step | 15.1 samples/s +Step 2850/44000 | Loss: 3.5057 | LR: 4.97e-04 | 1321.4ms/step | 15.1 samples/s +Step 2860/44000 | Loss: 3.5450 | LR: 4.97e-04 | 1323.7ms/step | 15.1 samples/s +Step 2870/44000 | Loss: 3.5467 | LR: 4.97e-04 | 1325.5ms/step | 15.1 samples/s +Step 2880/44000 | Loss: 3.5338 | LR: 4.97e-04 | 1326.0ms/step | 15.1 samples/s +Step 2890/44000 | Loss: 3.5038 | LR: 4.97e-04 | 1328.2ms/step | 15.1 samples/s +Step 2900/44000 | Loss: 3.4998 | LR: 4.97e-04 | 1322.9ms/step | 15.1 samples/s +Step 2910/44000 | Loss: 3.5161 | LR: 4.97e-04 | 1323.4ms/step | 15.1 samples/s +Step 2920/44000 | Loss: 3.5385 | LR: 4.97e-04 | 1325.4ms/step | 15.1 samples/s +Step 2930/44000 | Loss: 3.5369 | LR: 4.97e-04 | 1326.3ms/step | 15.1 samples/s +Step 2940/44000 | Loss: 3.5004 | LR: 4.97e-04 | 1330.5ms/step | 15.0 samples/s +Step 2950/44000 | Loss: 3.5013 | LR: 4.97e-04 | 1330.0ms/step | 15.0 samples/s +Step 2960/44000 | Loss: 3.4695 | LR: 4.97e-04 | 1330.7ms/step | 15.0 samples/s +Step 2970/44000 | Loss: 3.5345 | LR: 4.97e-04 | 1332.4ms/step | 15.0 samples/s +Step 2980/44000 | Loss: 3.5132 | LR: 4.97e-04 | 1330.2ms/step | 15.0 samples/s +Step 2990/44000 | Loss: 3.5397 | LR: 4.97e-04 | 1328.7ms/step | 15.1 samples/s +Step 3000/44000 | Loss: 3.5395 | LR: 4.97e-04 | 1329.7ms/step | 15.0 samples/s +Validation at step 3000: Loss = 3.4992, Perplexity = 33.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-3000 +Step 3010/44000 | Loss: 3.5052 | LR: 4.97e-04 | 3868.3ms/step | 5.2 samples/s +Step 3020/44000 | Loss: 3.5074 | LR: 4.97e-04 | 1330.0ms/step | 15.0 samples/s +Step 3030/44000 | Loss: 3.5371 | LR: 4.97e-04 | 1328.2ms/step | 15.1 samples/s +Step 3040/44000 | Loss: 3.4918 | LR: 4.97e-04 | 1330.5ms/step | 15.0 samples/s +Step 3050/44000 | Loss: 3.5132 | LR: 4.97e-04 | 1330.7ms/step | 15.0 samples/s +Step 3060/44000 | Loss: 3.4974 | LR: 4.96e-04 | 1328.9ms/step | 15.1 samples/s +Step 3070/44000 | Loss: 3.5065 | LR: 4.96e-04 | 1331.7ms/step | 15.0 samples/s +Step 3080/44000 | Loss: 3.4864 | LR: 4.96e-04 | 1333.4ms/step | 15.0 samples/s +Step 3090/44000 | Loss: 3.4993 | LR: 4.96e-04 | 1331.9ms/step | 15.0 samples/s +Step 3100/44000 | Loss: 3.5026 | LR: 4.96e-04 | 1332.2ms/step | 15.0 samples/s +Step 3110/44000 | Loss: 3.4860 | LR: 4.96e-04 | 1334.2ms/step | 15.0 samples/s +Step 3120/44000 | Loss: 3.5421 | LR: 4.96e-04 | 1331.0ms/step | 15.0 samples/s +Step 3130/44000 | Loss: 3.4953 | LR: 4.96e-04 | 1332.4ms/step | 15.0 samples/s +Step 3140/44000 | Loss: 3.4554 | LR: 4.96e-04 | 1331.0ms/step | 15.0 samples/s +Step 3150/44000 | Loss: 3.4965 | LR: 4.96e-04 | 1333.0ms/step | 15.0 samples/s +Step 3160/44000 | Loss: 3.4804 | LR: 4.96e-04 | 1332.3ms/step | 15.0 samples/s +Step 3170/44000 | Loss: 3.4985 | LR: 4.96e-04 | 1330.8ms/step | 15.0 samples/s +Step 3180/44000 | Loss: 3.4862 | LR: 4.96e-04 | 1330.1ms/step | 15.0 samples/s +Step 3190/44000 | Loss: 3.4614 | LR: 4.96e-04 | 1331.1ms/step | 15.0 samples/s +Step 3200/44000 | Loss: 3.5077 | LR: 4.96e-04 | 1332.0ms/step | 15.0 samples/s +Step 3210/44000 | Loss: 3.4851 | LR: 4.96e-04 | 1328.4ms/step | 15.1 samples/s +Step 3220/44000 | Loss: 3.4553 | LR: 4.96e-04 | 1331.7ms/step | 15.0 samples/s +Step 3230/44000 | Loss: 3.4665 | LR: 4.96e-04 | 1330.2ms/step | 15.0 samples/s +Step 3240/44000 | Loss: 3.4893 | LR: 4.96e-04 | 1328.3ms/step | 15.1 samples/s +Step 3250/44000 | Loss: 3.4770 | LR: 4.96e-04 | 1330.6ms/step | 15.0 samples/s +Validation at step 3250: Loss = 3.4670, Perplexity = 32.04 +Step 3260/44000 | Loss: 3.4827 | LR: 4.96e-04 | 3566.5ms/step | 5.6 samples/s +Step 3270/44000 | Loss: 3.4877 | LR: 4.96e-04 | 1330.0ms/step | 15.0 samples/s +Step 3280/44000 | Loss: 3.4517 | LR: 4.96e-04 | 1332.0ms/step | 15.0 samples/s +Step 3290/44000 | Loss: 3.4951 | LR: 4.96e-04 | 1331.5ms/step | 15.0 samples/s +Step 3300/44000 | Loss: 3.4539 | LR: 4.96e-04 | 1332.2ms/step | 15.0 samples/s +Step 3310/44000 | Loss: 3.4583 | LR: 4.96e-04 | 1331.0ms/step | 15.0 samples/s +Step 3320/44000 | Loss: 3.4464 | LR: 4.96e-04 | 1331.9ms/step | 15.0 samples/s +Step 3330/44000 | Loss: 3.4952 | LR: 4.96e-04 | 1329.3ms/step | 15.0 samples/s +Step 3340/44000 | Loss: 3.4442 | LR: 4.96e-04 | 1332.3ms/step | 15.0 samples/s +Step 3350/44000 | Loss: 3.4498 | LR: 4.96e-04 | 1330.0ms/step | 15.0 samples/s +Step 3360/44000 | Loss: 3.4433 | LR: 4.96e-04 | 1331.0ms/step | 15.0 samples/s +Step 3370/44000 | Loss: 3.4592 | LR: 4.95e-04 | 1333.6ms/step | 15.0 samples/s +Step 3380/44000 | Loss: 3.4731 | LR: 4.95e-04 | 1328.6ms/step | 15.1 samples/s +Step 3390/44000 | Loss: 3.5162 | LR: 4.95e-04 | 1331.8ms/step | 15.0 samples/s +Step 3400/44000 | Loss: 3.4436 | LR: 4.95e-04 | 1328.8ms/step | 15.1 samples/s +Step 3410/44000 | Loss: 3.4882 | LR: 4.95e-04 | 1325.5ms/step | 15.1 samples/s +Step 3420/44000 | Loss: 3.4587 | LR: 4.95e-04 | 1326.5ms/step | 15.1 samples/s +Step 3430/44000 | Loss: 3.4637 | LR: 4.95e-04 | 1325.6ms/step | 15.1 samples/s +Step 3440/44000 | Loss: 3.4280 | LR: 4.95e-04 | 1330.1ms/step | 15.0 samples/s +Step 3450/44000 | Loss: 3.4377 | LR: 4.95e-04 | 1331.0ms/step | 15.0 samples/s +Step 3460/44000 | Loss: 3.4840 | LR: 4.95e-04 | 1328.4ms/step | 15.1 samples/s +Step 3470/44000 | Loss: 3.4466 | LR: 4.95e-04 | 1328.9ms/step | 15.0 samples/s +Step 3480/44000 | Loss: 3.4526 | LR: 4.95e-04 | 1329.1ms/step | 15.0 samples/s +Step 3490/44000 | Loss: 3.4541 | LR: 4.95e-04 | 1330.3ms/step | 15.0 samples/s +Step 3500/44000 | Loss: 3.4777 | LR: 4.95e-04 | 1331.2ms/step | 15.0 samples/s +Validation at step 3500: Loss = 3.4422, Perplexity = 31.26 +Step 3510/44000 | Loss: 3.4763 | LR: 4.95e-04 | 3561.0ms/step | 5.6 samples/s +Step 3520/44000 | Loss: 3.4633 | LR: 4.95e-04 | 1327.3ms/step | 15.1 samples/s +Step 3530/44000 | Loss: 3.4476 | LR: 4.95e-04 | 1326.8ms/step | 15.1 samples/s +Step 3540/44000 | Loss: 3.4208 | LR: 4.95e-04 | 1329.1ms/step | 15.0 samples/s +Step 3550/44000 | Loss: 3.4710 | LR: 4.95e-04 | 1330.1ms/step | 15.0 samples/s +Step 3560/44000 | Loss: 3.4465 | LR: 4.95e-04 | 1328.3ms/step | 15.1 samples/s +Step 3570/44000 | Loss: 3.4367 | LR: 4.95e-04 | 1327.0ms/step | 15.1 samples/s +Step 3580/44000 | Loss: 3.4504 | LR: 4.95e-04 | 1331.6ms/step | 15.0 samples/s +Step 3590/44000 | Loss: 3.4279 | LR: 4.95e-04 | 1332.2ms/step | 15.0 samples/s +Step 3600/44000 | Loss: 3.4453 | LR: 4.95e-04 | 1331.5ms/step | 15.0 samples/s +Step 3610/44000 | Loss: 3.3863 | LR: 4.95e-04 | 1331.8ms/step | 15.0 samples/s +Step 3620/44000 | Loss: 3.4436 | LR: 4.95e-04 | 1331.1ms/step | 15.0 samples/s +Step 3630/44000 | Loss: 3.4496 | LR: 4.95e-04 | 1332.2ms/step | 15.0 samples/s +Step 3640/44000 | Loss: 3.4235 | LR: 4.95e-04 | 1328.2ms/step | 15.1 samples/s +Step 3650/44000 | Loss: 3.4425 | LR: 4.94e-04 | 1326.9ms/step | 15.1 samples/s +Step 3660/44000 | Loss: 3.4238 | LR: 4.94e-04 | 1330.1ms/step | 15.0 samples/s +Step 3670/44000 | Loss: 3.4369 | LR: 4.94e-04 | 1327.9ms/step | 15.1 samples/s +Step 3680/44000 | Loss: 3.4408 | LR: 4.94e-04 | 1330.2ms/step | 15.0 samples/s +Step 3690/44000 | Loss: 3.3977 | LR: 4.94e-04 | 1329.3ms/step | 15.0 samples/s +Step 3700/44000 | Loss: 3.4155 | LR: 4.94e-04 | 1327.8ms/step | 15.1 samples/s +Step 3710/44000 | Loss: 3.4443 | LR: 4.94e-04 | 1328.5ms/step | 15.1 samples/s +Step 3720/44000 | Loss: 3.4125 | LR: 4.94e-04 | 1331.4ms/step | 15.0 samples/s +Step 3730/44000 | Loss: 3.4118 | LR: 4.94e-04 | 1332.5ms/step | 15.0 samples/s +Step 3740/44000 | Loss: 3.4345 | LR: 4.94e-04 | 1328.4ms/step | 15.1 samples/s +Step 3750/44000 | Loss: 3.4228 | LR: 4.94e-04 | 1328.8ms/step | 15.1 samples/s +Validation at step 3750: Loss = 3.4138, Perplexity = 30.38 +Step 3760/44000 | Loss: 3.4407 | LR: 4.94e-04 | 3554.0ms/step | 5.6 samples/s +Step 3770/44000 | Loss: 3.4302 | LR: 4.94e-04 | 1328.0ms/step | 15.1 samples/s +Step 3780/44000 | Loss: 3.4098 | LR: 4.94e-04 | 1326.4ms/step | 15.1 samples/s +Step 3790/44000 | Loss: 3.4192 | LR: 4.94e-04 | 1328.0ms/step | 15.1 samples/s +Step 3800/44000 | Loss: 3.4110 | LR: 4.94e-04 | 1330.0ms/step | 15.0 samples/s +Step 3810/44000 | Loss: 3.4169 | LR: 4.94e-04 | 1331.6ms/step | 15.0 samples/s +Step 3820/44000 | Loss: 3.4213 | LR: 4.94e-04 | 1327.2ms/step | 15.1 samples/s +Step 3830/44000 | Loss: 3.4122 | LR: 4.94e-04 | 1325.3ms/step | 15.1 samples/s +Step 3840/44000 | Loss: 3.4199 | LR: 4.94e-04 | 1328.2ms/step | 15.1 samples/s +Step 3850/44000 | Loss: 3.4402 | LR: 4.94e-04 | 1329.7ms/step | 15.0 samples/s +Step 3860/44000 | Loss: 3.4099 | LR: 4.94e-04 | 1328.8ms/step | 15.1 samples/s +Step 3870/44000 | Loss: 3.4132 | LR: 4.94e-04 | 1324.6ms/step | 15.1 samples/s +Step 3880/44000 | Loss: 3.4451 | LR: 4.94e-04 | 1327.7ms/step | 15.1 samples/s +Step 3890/44000 | Loss: 3.3946 | LR: 4.94e-04 | 1326.4ms/step | 15.1 samples/s +Step 3900/44000 | Loss: 3.4200 | LR: 4.93e-04 | 1327.9ms/step | 15.1 samples/s +Step 3910/44000 | Loss: 3.4327 | LR: 4.93e-04 | 1328.8ms/step | 15.1 samples/s +Step 3920/44000 | Loss: 3.3784 | LR: 4.93e-04 | 1327.2ms/step | 15.1 samples/s +Step 3930/44000 | Loss: 3.3964 | LR: 4.93e-04 | 1329.7ms/step | 15.0 samples/s +Step 3940/44000 | Loss: 3.4326 | LR: 4.93e-04 | 1329.9ms/step | 15.0 samples/s +Step 3950/44000 | Loss: 3.4008 | LR: 4.93e-04 | 1331.4ms/step | 15.0 samples/s +Step 3960/44000 | Loss: 3.4081 | LR: 4.93e-04 | 1329.5ms/step | 15.0 samples/s +Step 3970/44000 | Loss: 3.4396 | LR: 4.93e-04 | 1333.9ms/step | 15.0 samples/s +Step 3980/44000 | Loss: 3.3836 | LR: 4.93e-04 | 1332.5ms/step | 15.0 samples/s +Step 3990/44000 | Loss: 3.4089 | LR: 4.93e-04 | 1331.7ms/step | 15.0 samples/s +Step 4000/44000 | Loss: 3.4235 | LR: 4.93e-04 | 1331.5ms/step | 15.0 samples/s +Validation at step 4000: Loss = 3.3913, Perplexity = 29.70 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-4000 +Step 4010/44000 | Loss: 3.3833 | LR: 4.93e-04 | 3869.6ms/step | 5.2 samples/s +Step 4020/44000 | Loss: 3.3913 | LR: 4.93e-04 | 1326.8ms/step | 15.1 samples/s +Step 4030/44000 | Loss: 3.3806 | LR: 4.93e-04 | 1329.9ms/step | 15.0 samples/s +Step 4040/44000 | Loss: 3.3576 | LR: 4.93e-04 | 1330.8ms/step | 15.0 samples/s +Step 4050/44000 | Loss: 3.3898 | LR: 4.93e-04 | 1331.8ms/step | 15.0 samples/s +Step 4060/44000 | Loss: 3.3669 | LR: 4.93e-04 | 1332.2ms/step | 15.0 samples/s +Step 4070/44000 | Loss: 3.3866 | LR: 4.93e-04 | 1331.0ms/step | 15.0 samples/s +Step 4080/44000 | Loss: 3.3667 | LR: 4.93e-04 | 1333.4ms/step | 15.0 samples/s +Step 4090/44000 | Loss: 3.4043 | LR: 4.93e-04 | 1331.6ms/step | 15.0 samples/s +Step 4100/44000 | Loss: 3.4173 | LR: 4.93e-04 | 1333.2ms/step | 15.0 samples/s +Step 4110/44000 | Loss: 3.3846 | LR: 4.93e-04 | 1330.8ms/step | 15.0 samples/s +Step 4120/44000 | Loss: 3.3866 | LR: 4.93e-04 | 1329.3ms/step | 15.0 samples/s +Step 4130/44000 | Loss: 3.4071 | LR: 4.93e-04 | 1330.1ms/step | 15.0 samples/s +Step 4140/44000 | Loss: 3.3860 | LR: 4.92e-04 | 1331.3ms/step | 15.0 samples/s +Step 4150/44000 | Loss: 3.3969 | LR: 4.92e-04 | 1331.2ms/step | 15.0 samples/s +Step 4160/44000 | Loss: 3.3501 | LR: 4.92e-04 | 1334.0ms/step | 15.0 samples/s +Step 4170/44000 | Loss: 3.3878 | LR: 4.92e-04 | 1332.9ms/step | 15.0 samples/s +Step 4180/44000 | Loss: 3.3728 | LR: 4.92e-04 | 1332.0ms/step | 15.0 samples/s +Step 4190/44000 | Loss: 3.3910 | LR: 4.92e-04 | 1333.2ms/step | 15.0 samples/s +Step 4200/44000 | Loss: 3.3868 | LR: 4.92e-04 | 1330.2ms/step | 15.0 samples/s +Step 4210/44000 | Loss: 3.3801 | LR: 4.92e-04 | 1328.0ms/step | 15.1 samples/s +Step 4220/44000 | Loss: 3.3601 | LR: 4.92e-04 | 1329.2ms/step | 15.0 samples/s +Step 4230/44000 | Loss: 3.3850 | LR: 4.92e-04 | 1331.1ms/step | 15.0 samples/s +Step 4240/44000 | Loss: 3.3768 | LR: 4.92e-04 | 1331.8ms/step | 15.0 samples/s +Step 4250/44000 | Loss: 3.3457 | LR: 4.92e-04 | 1333.5ms/step | 15.0 samples/s +Validation at step 4250: Loss = 3.3698, Perplexity = 29.07 +Step 4260/44000 | Loss: 3.3874 | LR: 4.92e-04 | 3568.1ms/step | 5.6 samples/s +Step 4270/44000 | Loss: 3.3883 | LR: 4.92e-04 | 1334.0ms/step | 15.0 samples/s +Step 4280/44000 | Loss: 3.3618 | LR: 4.92e-04 | 1333.1ms/step | 15.0 samples/s +Step 4290/44000 | Loss: 3.3587 | LR: 4.92e-04 | 1332.2ms/step | 15.0 samples/s +Step 4300/44000 | Loss: 3.4066 | LR: 4.92e-04 | 1329.9ms/step | 15.0 samples/s +Step 4310/44000 | Loss: 3.4031 | LR: 4.92e-04 | 1332.7ms/step | 15.0 samples/s +Step 4320/44000 | Loss: 3.3897 | LR: 4.92e-04 | 1334.7ms/step | 15.0 samples/s +Step 4330/44000 | Loss: 3.3674 | LR: 4.92e-04 | 1332.2ms/step | 15.0 samples/s +Step 4340/44000 | Loss: 3.3546 | LR: 4.92e-04 | 1331.5ms/step | 15.0 samples/s +Step 4350/44000 | Loss: 3.3667 | LR: 4.92e-04 | 1328.9ms/step | 15.0 samples/s +Step 4360/44000 | Loss: 3.3719 | LR: 4.91e-04 | 1331.6ms/step | 15.0 samples/s +Step 4370/44000 | Loss: 3.3612 | LR: 4.91e-04 | 1330.9ms/step | 15.0 samples/s +Step 4380/44000 | Loss: 3.3681 | LR: 4.91e-04 | 1328.7ms/step | 15.1 samples/s +Step 4390/44000 | Loss: 3.3658 | LR: 4.91e-04 | 1328.6ms/step | 15.1 samples/s +Step 4400/44000 | Loss: 3.3379 | LR: 4.91e-04 | 1327.5ms/step | 15.1 samples/s +Step 4410/44000 | Loss: 3.3885 | LR: 4.91e-04 | 1329.6ms/step | 15.0 samples/s +Step 4420/44000 | Loss: 3.3784 | LR: 4.91e-04 | 1328.5ms/step | 15.1 samples/s +Step 4430/44000 | Loss: 3.3398 | LR: 4.91e-04 | 1327.6ms/step | 15.1 samples/s +Step 4440/44000 | Loss: 3.3574 | LR: 4.91e-04 | 1327.4ms/step | 15.1 samples/s +Step 4450/44000 | Loss: 3.3453 | LR: 4.91e-04 | 1327.0ms/step | 15.1 samples/s +Step 4460/44000 | Loss: 3.3696 | LR: 4.91e-04 | 1325.5ms/step | 15.1 samples/s +Step 4470/44000 | Loss: 3.3662 | LR: 4.91e-04 | 1331.8ms/step | 15.0 samples/s +Step 4480/44000 | Loss: 3.3528 | LR: 4.91e-04 | 1330.1ms/step | 15.0 samples/s +Step 4490/44000 | Loss: 3.3815 | LR: 4.91e-04 | 1329.2ms/step | 15.0 samples/s +Step 4500/44000 | Loss: 3.3567 | LR: 4.91e-04 | 1326.1ms/step | 15.1 samples/s +Validation at step 4500: Loss = 3.3538, Perplexity = 28.61 +Step 4510/44000 | Loss: 3.3653 | LR: 4.91e-04 | 3559.3ms/step | 5.6 samples/s +Step 4520/44000 | Loss: 3.3388 | LR: 4.91e-04 | 1326.6ms/step | 15.1 samples/s +Step 4530/44000 | Loss: 3.3548 | LR: 4.91e-04 | 1328.2ms/step | 15.1 samples/s +Step 4540/44000 | Loss: 3.3544 | LR: 4.91e-04 | 1324.3ms/step | 15.1 samples/s +Step 4550/44000 | Loss: 3.3286 | LR: 4.91e-04 | 1329.0ms/step | 15.0 samples/s +Step 4560/44000 | Loss: 3.3561 | LR: 4.90e-04 | 1326.2ms/step | 15.1 samples/s +Step 4570/44000 | Loss: 3.3680 | LR: 4.90e-04 | 1325.9ms/step | 15.1 samples/s +Step 4580/44000 | Loss: 3.3292 | LR: 4.90e-04 | 1323.8ms/step | 15.1 samples/s +Step 4590/44000 | Loss: 3.3535 | LR: 4.90e-04 | 1326.3ms/step | 15.1 samples/s +Step 4600/44000 | Loss: 3.3744 | LR: 4.90e-04 | 1325.9ms/step | 15.1 samples/s +Step 4610/44000 | Loss: 3.3844 | LR: 4.90e-04 | 1324.5ms/step | 15.1 samples/s +Step 4620/44000 | Loss: 3.3484 | LR: 4.90e-04 | 1327.0ms/step | 15.1 samples/s +Step 4630/44000 | Loss: 3.3338 | LR: 4.90e-04 | 1328.0ms/step | 15.1 samples/s +Step 4640/44000 | Loss: 3.3477 | LR: 4.90e-04 | 1326.5ms/step | 15.1 samples/s +Step 4650/44000 | Loss: 3.3793 | LR: 4.90e-04 | 1327.9ms/step | 15.1 samples/s +Step 4660/44000 | Loss: 3.3318 | LR: 4.90e-04 | 1326.7ms/step | 15.1 samples/s +Step 4670/44000 | Loss: 3.3693 | LR: 4.90e-04 | 1327.2ms/step | 15.1 samples/s +Step 4680/44000 | Loss: 3.3308 | LR: 4.90e-04 | 1326.2ms/step | 15.1 samples/s +Step 4690/44000 | Loss: 3.3664 | LR: 4.90e-04 | 1327.7ms/step | 15.1 samples/s +Step 4700/44000 | Loss: 3.3447 | LR: 4.90e-04 | 1329.1ms/step | 15.0 samples/s +Step 4710/44000 | Loss: 3.3284 | LR: 4.90e-04 | 1329.0ms/step | 15.0 samples/s +Step 4720/44000 | Loss: 3.3077 | LR: 4.90e-04 | 1331.0ms/step | 15.0 samples/s +Step 4730/44000 | Loss: 3.3479 | LR: 4.90e-04 | 1330.1ms/step | 15.0 samples/s +Step 4740/44000 | Loss: 3.3598 | LR: 4.90e-04 | 1329.4ms/step | 15.0 samples/s +Step 4750/44000 | Loss: 3.3581 | LR: 4.90e-04 | 1325.7ms/step | 15.1 samples/s +Validation at step 4750: Loss = 3.3321, Perplexity = 28.00 +Step 4760/44000 | Loss: 3.3444 | LR: 4.89e-04 | 3556.2ms/step | 5.6 samples/s +Step 4770/44000 | Loss: 3.3386 | LR: 4.89e-04 | 1330.7ms/step | 15.0 samples/s +Step 4780/44000 | Loss: 3.3695 | LR: 4.89e-04 | 1330.7ms/step | 15.0 samples/s +Step 4790/44000 | Loss: 3.3300 | LR: 4.89e-04 | 1329.5ms/step | 15.0 samples/s +Step 4800/44000 | Loss: 3.3809 | LR: 4.89e-04 | 1324.9ms/step | 15.1 samples/s +Step 4810/44000 | Loss: 3.3277 | LR: 4.89e-04 | 1327.4ms/step | 15.1 samples/s +Step 4820/44000 | Loss: 3.3191 | LR: 4.89e-04 | 1328.4ms/step | 15.1 samples/s +Step 4830/44000 | Loss: 3.3784 | LR: 4.89e-04 | 1330.0ms/step | 15.0 samples/s +Step 4840/44000 | Loss: 3.3512 | LR: 4.89e-04 | 1326.6ms/step | 15.1 samples/s +Step 4850/44000 | Loss: 3.3559 | LR: 4.89e-04 | 1328.4ms/step | 15.1 samples/s +Step 4860/44000 | Loss: 3.3146 | LR: 4.89e-04 | 1329.3ms/step | 15.0 samples/s +Step 4870/44000 | Loss: 3.3406 | LR: 4.89e-04 | 1326.7ms/step | 15.1 samples/s +Step 4880/44000 | Loss: 3.3375 | LR: 4.89e-04 | 1325.5ms/step | 15.1 samples/s +Step 4890/44000 | Loss: 3.3609 | LR: 4.89e-04 | 1327.7ms/step | 15.1 samples/s +Step 4900/44000 | Loss: 3.3387 | LR: 4.89e-04 | 1325.6ms/step | 15.1 samples/s +Step 4910/44000 | Loss: 3.3500 | LR: 4.89e-04 | 1325.4ms/step | 15.1 samples/s +Step 4920/44000 | Loss: 3.2891 | LR: 4.89e-04 | 1327.7ms/step | 15.1 samples/s +Step 4930/44000 | Loss: 3.3194 | LR: 4.89e-04 | 1326.7ms/step | 15.1 samples/s +Step 4940/44000 | Loss: 3.3163 | LR: 4.89e-04 | 1327.2ms/step | 15.1 samples/s +Step 4950/44000 | Loss: 3.3261 | LR: 4.88e-04 | 1328.2ms/step | 15.1 samples/s +Step 4960/44000 | Loss: 3.2937 | LR: 4.88e-04 | 1329.5ms/step | 15.0 samples/s +Step 4970/44000 | Loss: 3.3342 | LR: 4.88e-04 | 1330.9ms/step | 15.0 samples/s +Step 4980/44000 | Loss: 3.3231 | LR: 4.88e-04 | 1327.9ms/step | 15.1 samples/s +Step 4990/44000 | Loss: 3.3137 | LR: 4.88e-04 | 1332.6ms/step | 15.0 samples/s +Step 5000/44000 | Loss: 3.3089 | LR: 4.88e-04 | 1331.0ms/step | 15.0 samples/s +Validation at step 5000: Loss = 3.3179, Perplexity = 27.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-5000 +Step 5010/44000 | Loss: 3.3074 | LR: 4.88e-04 | 3870.0ms/step | 5.2 samples/s +Step 5020/44000 | Loss: 3.3242 | LR: 4.88e-04 | 1329.4ms/step | 15.0 samples/s +Step 5030/44000 | Loss: 3.3543 | LR: 4.88e-04 | 1330.4ms/step | 15.0 samples/s +Step 5040/44000 | Loss: 3.3209 | LR: 4.88e-04 | 1330.1ms/step | 15.0 samples/s +Step 5050/44000 | Loss: 3.3162 | LR: 4.88e-04 | 1330.0ms/step | 15.0 samples/s +Step 5060/44000 | Loss: 3.3262 | LR: 4.88e-04 | 1331.2ms/step | 15.0 samples/s +Step 5070/44000 | Loss: 3.3360 | LR: 4.88e-04 | 1329.2ms/step | 15.0 samples/s +Step 5080/44000 | Loss: 3.3388 | LR: 4.88e-04 | 1323.5ms/step | 15.1 samples/s +Step 5090/44000 | Loss: 3.3478 | LR: 4.88e-04 | 1325.0ms/step | 15.1 samples/s +Step 5100/44000 | Loss: 3.3503 | LR: 4.88e-04 | 1327.2ms/step | 15.1 samples/s +Step 5110/44000 | Loss: 3.3116 | LR: 4.88e-04 | 1331.2ms/step | 15.0 samples/s +Step 5120/44000 | Loss: 3.3189 | LR: 4.88e-04 | 1332.1ms/step | 15.0 samples/s +Step 5130/44000 | Loss: 3.3283 | LR: 4.87e-04 | 1331.4ms/step | 15.0 samples/s +Step 5140/44000 | Loss: 3.3435 | LR: 4.87e-04 | 1334.0ms/step | 15.0 samples/s +Step 5150/44000 | Loss: 3.3122 | LR: 4.87e-04 | 1334.6ms/step | 15.0 samples/s +Step 5160/44000 | Loss: 3.3220 | LR: 4.87e-04 | 1330.5ms/step | 15.0 samples/s +Step 5170/44000 | Loss: 3.2906 | LR: 4.87e-04 | 1333.0ms/step | 15.0 samples/s +Step 5180/44000 | Loss: 3.2874 | LR: 4.87e-04 | 1329.9ms/step | 15.0 samples/s +Step 5190/44000 | Loss: 3.3128 | LR: 4.87e-04 | 1332.8ms/step | 15.0 samples/s +Step 5200/44000 | Loss: 3.2883 | LR: 4.87e-04 | 1333.7ms/step | 15.0 samples/s +Step 5210/44000 | Loss: 3.2809 | LR: 4.87e-04 | 1332.9ms/step | 15.0 samples/s +Step 5220/44000 | Loss: 3.2836 | LR: 4.87e-04 | 1332.2ms/step | 15.0 samples/s +Step 5230/44000 | Loss: 3.2969 | LR: 4.87e-04 | 1332.4ms/step | 15.0 samples/s +Step 5240/44000 | Loss: 3.2812 | LR: 4.87e-04 | 1333.0ms/step | 15.0 samples/s +Step 5250/44000 | Loss: 3.3181 | LR: 4.87e-04 | 1333.4ms/step | 15.0 samples/s +Validation at step 5250: Loss = 3.3040, Perplexity = 27.22 +Step 5260/44000 | Loss: 3.3087 | LR: 4.87e-04 | 3563.3ms/step | 5.6 samples/s +Step 5270/44000 | Loss: 3.3252 | LR: 4.87e-04 | 1329.1ms/step | 15.0 samples/s +Step 5280/44000 | Loss: 3.3091 | LR: 4.87e-04 | 1330.6ms/step | 15.0 samples/s +Step 5290/44000 | Loss: 3.3140 | LR: 4.87e-04 | 1330.3ms/step | 15.0 samples/s +Step 5300/44000 | Loss: 3.2903 | LR: 4.86e-04 | 1328.7ms/step | 15.1 samples/s +Step 5310/44000 | Loss: 3.3128 | LR: 4.86e-04 | 1331.8ms/step | 15.0 samples/s +Step 5320/44000 | Loss: 3.3084 | LR: 4.86e-04 | 1333.0ms/step | 15.0 samples/s +Step 5330/44000 | Loss: 3.2712 | LR: 4.86e-04 | 1331.3ms/step | 15.0 samples/s +Step 5340/44000 | Loss: 3.2993 | LR: 4.86e-04 | 1331.0ms/step | 15.0 samples/s +Step 5350/44000 | Loss: 3.3224 | LR: 4.86e-04 | 1327.9ms/step | 15.1 samples/s +Step 5360/44000 | Loss: 3.3013 | LR: 4.86e-04 | 1326.3ms/step | 15.1 samples/s +Step 5370/44000 | Loss: 3.2809 | LR: 4.86e-04 | 1324.3ms/step | 15.1 samples/s +Step 5380/44000 | Loss: 3.3301 | LR: 4.86e-04 | 1326.3ms/step | 15.1 samples/s +Step 5390/44000 | Loss: 3.3281 | LR: 4.86e-04 | 1325.5ms/step | 15.1 samples/s +Step 5400/44000 | Loss: 3.2894 | LR: 4.86e-04 | 1327.5ms/step | 15.1 samples/s +Step 5410/44000 | Loss: 3.3026 | LR: 4.86e-04 | 1330.1ms/step | 15.0 samples/s +Step 5420/44000 | Loss: 3.2790 | LR: 4.86e-04 | 1327.3ms/step | 15.1 samples/s +Step 5430/44000 | Loss: 3.3023 | LR: 4.86e-04 | 1324.3ms/step | 15.1 samples/s +Step 5440/44000 | Loss: 3.3223 | LR: 4.86e-04 | 1326.9ms/step | 15.1 samples/s +Step 5450/44000 | Loss: 3.3165 | LR: 4.86e-04 | 1323.0ms/step | 15.1 samples/s +Step 5460/44000 | Loss: 3.2835 | LR: 4.86e-04 | 1327.0ms/step | 15.1 samples/s +Step 5470/44000 | Loss: 3.2909 | LR: 4.85e-04 | 1327.2ms/step | 15.1 samples/s +Step 5480/44000 | Loss: 3.3005 | LR: 4.85e-04 | 1325.6ms/step | 15.1 samples/s +Step 5490/44000 | Loss: 3.2982 | LR: 4.85e-04 | 1323.5ms/step | 15.1 samples/s +Step 5500/44000 | Loss: 3.3057 | LR: 4.85e-04 | 1327.1ms/step | 15.1 samples/s +Validation at step 5500: Loss = 3.2888, Perplexity = 26.81 +Step 5510/44000 | Loss: 3.3178 | LR: 4.85e-04 | 3558.2ms/step | 5.6 samples/s +Step 5520/44000 | Loss: 3.3204 | LR: 4.85e-04 | 1326.7ms/step | 15.1 samples/s +Step 5530/44000 | Loss: 3.2842 | LR: 4.85e-04 | 1324.3ms/step | 15.1 samples/s +Step 5540/44000 | Loss: 3.3172 | LR: 4.85e-04 | 1324.3ms/step | 15.1 samples/s +Step 5550/44000 | Loss: 3.3247 | LR: 4.85e-04 | 1325.8ms/step | 15.1 samples/s +Step 5560/44000 | Loss: 3.3001 | LR: 4.85e-04 | 1328.0ms/step | 15.1 samples/s +Step 5570/44000 | Loss: 3.2830 | LR: 4.85e-04 | 1326.9ms/step | 15.1 samples/s +Step 5580/44000 | Loss: 3.2556 | LR: 4.85e-04 | 1328.3ms/step | 15.1 samples/s +Step 5590/44000 | Loss: 3.2957 | LR: 4.85e-04 | 1327.8ms/step | 15.1 samples/s +Step 5600/44000 | Loss: 3.2737 | LR: 4.85e-04 | 1327.8ms/step | 15.1 samples/s +Step 5610/44000 | Loss: 3.2998 | LR: 4.85e-04 | 1330.6ms/step | 15.0 samples/s +Step 5620/44000 | Loss: 3.2761 | LR: 4.85e-04 | 1330.9ms/step | 15.0 samples/s +Step 5630/44000 | Loss: 3.3078 | LR: 4.84e-04 | 1328.0ms/step | 15.1 samples/s +Step 5640/44000 | Loss: 3.3279 | LR: 4.84e-04 | 1327.3ms/step | 15.1 samples/s +Step 5650/44000 | Loss: 3.2884 | LR: 4.84e-04 | 1327.4ms/step | 15.1 samples/s +Step 5660/44000 | Loss: 3.3292 | LR: 4.84e-04 | 1325.2ms/step | 15.1 samples/s +Step 5670/44000 | Loss: 3.2790 | LR: 4.84e-04 | 1324.9ms/step | 15.1 samples/s +Step 5680/44000 | Loss: 3.2812 | LR: 4.84e-04 | 1329.8ms/step | 15.0 samples/s +Step 5690/44000 | Loss: 3.2990 | LR: 4.84e-04 | 1324.5ms/step | 15.1 samples/s +Step 5700/44000 | Loss: 3.3115 | LR: 4.84e-04 | 1327.7ms/step | 15.1 samples/s +Step 5710/44000 | Loss: 3.2866 | LR: 4.84e-04 | 1325.2ms/step | 15.1 samples/s +Step 5720/44000 | Loss: 3.2844 | LR: 4.84e-04 | 1327.7ms/step | 15.1 samples/s +Step 5730/44000 | Loss: 3.2645 | LR: 4.84e-04 | 1330.0ms/step | 15.0 samples/s +Step 5740/44000 | Loss: 3.2808 | LR: 4.84e-04 | 1327.7ms/step | 15.1 samples/s +Step 5750/44000 | Loss: 3.2806 | LR: 4.84e-04 | 1327.2ms/step | 15.1 samples/s +Validation at step 5750: Loss = 3.2754, Perplexity = 26.45 +Step 5760/44000 | Loss: 3.2547 | LR: 4.84e-04 | 3551.5ms/step | 5.6 samples/s +Step 5770/44000 | Loss: 3.2810 | LR: 4.84e-04 | 1323.9ms/step | 15.1 samples/s +Step 5780/44000 | Loss: 3.2685 | LR: 4.83e-04 | 1326.4ms/step | 15.1 samples/s +Step 5790/44000 | Loss: 3.2767 | LR: 4.83e-04 | 1326.7ms/step | 15.1 samples/s +Step 5800/44000 | Loss: 3.2787 | LR: 4.83e-04 | 1322.9ms/step | 15.1 samples/s +Step 5810/44000 | Loss: 3.2838 | LR: 4.83e-04 | 1326.6ms/step | 15.1 samples/s +Step 5820/44000 | Loss: 3.2800 | LR: 4.83e-04 | 1327.7ms/step | 15.1 samples/s +Step 5830/44000 | Loss: 3.2882 | LR: 4.83e-04 | 1326.3ms/step | 15.1 samples/s +Step 5840/44000 | Loss: 3.3072 | LR: 4.83e-04 | 1325.5ms/step | 15.1 samples/s +Step 5850/44000 | Loss: 3.2799 | LR: 4.83e-04 | 1323.2ms/step | 15.1 samples/s +Step 5860/44000 | Loss: 3.2790 | LR: 4.83e-04 | 1325.1ms/step | 15.1 samples/s +Step 5870/44000 | Loss: 3.2451 | LR: 4.83e-04 | 1327.4ms/step | 15.1 samples/s +Step 5880/44000 | Loss: 3.2753 | LR: 4.83e-04 | 1326.3ms/step | 15.1 samples/s +Step 5890/44000 | Loss: 3.2805 | LR: 4.83e-04 | 1326.6ms/step | 15.1 samples/s +Step 5900/44000 | Loss: 3.2785 | LR: 4.83e-04 | 1328.8ms/step | 15.1 samples/s +Step 5910/44000 | Loss: 3.2695 | LR: 4.83e-04 | 1330.7ms/step | 15.0 samples/s +Step 5920/44000 | Loss: 3.3097 | LR: 4.83e-04 | 1328.1ms/step | 15.1 samples/s +Step 5930/44000 | Loss: 3.2541 | LR: 4.83e-04 | 1329.1ms/step | 15.0 samples/s +Step 5940/44000 | Loss: 3.2826 | LR: 4.82e-04 | 1327.2ms/step | 15.1 samples/s +Step 5950/44000 | Loss: 3.2640 | LR: 4.82e-04 | 1331.0ms/step | 15.0 samples/s +Step 5960/44000 | Loss: 3.2981 | LR: 4.82e-04 | 1331.4ms/step | 15.0 samples/s +Step 5970/44000 | Loss: 3.2535 | LR: 4.82e-04 | 1329.6ms/step | 15.0 samples/s +Step 5980/44000 | Loss: 3.2780 | LR: 4.82e-04 | 1330.7ms/step | 15.0 samples/s +Step 5990/44000 | Loss: 3.2606 | LR: 4.82e-04 | 1333.4ms/step | 15.0 samples/s +Step 6000/44000 | Loss: 3.2679 | LR: 4.82e-04 | 1331.1ms/step | 15.0 samples/s +Validation at step 6000: Loss = 3.2645, Perplexity = 26.17 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-6000 +Step 6010/44000 | Loss: 3.2793 | LR: 4.82e-04 | 3894.3ms/step | 5.1 samples/s +Step 6020/44000 | Loss: 3.2813 | LR: 4.82e-04 | 1328.4ms/step | 15.1 samples/s +Step 6030/44000 | Loss: 3.2936 | LR: 4.82e-04 | 1323.1ms/step | 15.1 samples/s +Step 6040/44000 | Loss: 3.2759 | LR: 4.82e-04 | 1325.8ms/step | 15.1 samples/s +Step 6050/44000 | Loss: 3.2816 | LR: 4.82e-04 | 1329.0ms/step | 15.0 samples/s +Step 6060/44000 | Loss: 3.2731 | LR: 4.82e-04 | 1331.2ms/step | 15.0 samples/s +Step 6070/44000 | Loss: 3.2637 | LR: 4.82e-04 | 1336.2ms/step | 15.0 samples/s +Step 6080/44000 | Loss: 3.2856 | LR: 4.81e-04 | 1331.5ms/step | 15.0 samples/s +Step 6090/44000 | Loss: 3.2772 | LR: 4.81e-04 | 1331.5ms/step | 15.0 samples/s +Step 6100/44000 | Loss: 3.2778 | LR: 4.81e-04 | 1329.0ms/step | 15.0 samples/s +Step 6110/44000 | Loss: 3.2717 | LR: 4.81e-04 | 1331.4ms/step | 15.0 samples/s +Step 6120/44000 | Loss: 3.2662 | LR: 4.81e-04 | 1331.4ms/step | 15.0 samples/s +Step 6130/44000 | Loss: 3.2832 | LR: 4.81e-04 | 1332.1ms/step | 15.0 samples/s +Step 6140/44000 | Loss: 3.2805 | LR: 4.81e-04 | 1333.0ms/step | 15.0 samples/s +Step 6150/44000 | Loss: 3.2731 | LR: 4.81e-04 | 1329.7ms/step | 15.0 samples/s +Step 6160/44000 | Loss: 3.2622 | LR: 4.81e-04 | 1329.2ms/step | 15.0 samples/s +Step 6170/44000 | Loss: 3.2872 | LR: 4.81e-04 | 1331.2ms/step | 15.0 samples/s +Step 6180/44000 | Loss: 3.2794 | LR: 4.81e-04 | 1330.5ms/step | 15.0 samples/s +Step 6190/44000 | Loss: 3.2299 | LR: 4.81e-04 | 1329.3ms/step | 15.0 samples/s +Step 6200/44000 | Loss: 3.2684 | LR: 4.81e-04 | 1329.9ms/step | 15.0 samples/s +Step 6210/44000 | Loss: 3.2874 | LR: 4.81e-04 | 1331.5ms/step | 15.0 samples/s +Step 6220/44000 | Loss: 3.2556 | LR: 4.81e-04 | 1331.0ms/step | 15.0 samples/s +Step 6230/44000 | Loss: 3.2751 | LR: 4.80e-04 | 1331.2ms/step | 15.0 samples/s +Step 6240/44000 | Loss: 3.2846 | LR: 4.80e-04 | 1332.8ms/step | 15.0 samples/s +Step 6250/44000 | Loss: 3.2639 | LR: 4.80e-04 | 1332.4ms/step | 15.0 samples/s +Validation at step 6250: Loss = 3.2527, Perplexity = 25.86 +Step 6260/44000 | Loss: 3.2725 | LR: 4.80e-04 | 3565.7ms/step | 5.6 samples/s +Step 6270/44000 | Loss: 3.2549 | LR: 4.80e-04 | 1332.2ms/step | 15.0 samples/s +Step 6280/44000 | Loss: 3.2688 | LR: 4.80e-04 | 1330.2ms/step | 15.0 samples/s +Step 6290/44000 | Loss: 3.2936 | LR: 4.80e-04 | 1332.8ms/step | 15.0 samples/s +Step 6300/44000 | Loss: 3.2689 | LR: 4.80e-04 | 1332.7ms/step | 15.0 samples/s +Step 6310/44000 | Loss: 3.2317 | LR: 4.80e-04 | 1332.8ms/step | 15.0 samples/s +Step 6320/44000 | Loss: 3.2339 | LR: 4.80e-04 | 1332.5ms/step | 15.0 samples/s +Step 6330/44000 | Loss: 3.2681 | LR: 4.80e-04 | 1330.4ms/step | 15.0 samples/s +Step 6340/44000 | Loss: 3.2626 | LR: 4.80e-04 | 1331.4ms/step | 15.0 samples/s +Step 6350/44000 | Loss: 3.2528 | LR: 4.80e-04 | 1330.0ms/step | 15.0 samples/s +Step 6360/44000 | Loss: 3.2472 | LR: 4.80e-04 | 1330.6ms/step | 15.0 samples/s +Step 6370/44000 | Loss: 3.2370 | LR: 4.79e-04 | 1328.2ms/step | 15.1 samples/s +Step 6380/44000 | Loss: 3.2570 | LR: 4.79e-04 | 1329.3ms/step | 15.0 samples/s +Step 6390/44000 | Loss: 3.2660 | LR: 4.79e-04 | 1326.6ms/step | 15.1 samples/s +Step 6400/44000 | Loss: 3.2584 | LR: 4.79e-04 | 1326.0ms/step | 15.1 samples/s +Step 6410/44000 | Loss: 3.2601 | LR: 4.79e-04 | 1328.1ms/step | 15.1 samples/s +Step 6420/44000 | Loss: 3.2468 | LR: 4.79e-04 | 1329.9ms/step | 15.0 samples/s +Step 6430/44000 | Loss: 3.2915 | LR: 4.79e-04 | 1330.2ms/step | 15.0 samples/s +Step 6440/44000 | Loss: 3.2664 | LR: 4.79e-04 | 1328.7ms/step | 15.1 samples/s +Step 6450/44000 | Loss: 3.2412 | LR: 4.79e-04 | 1325.7ms/step | 15.1 samples/s +Step 6460/44000 | Loss: 3.2440 | LR: 4.79e-04 | 1324.3ms/step | 15.1 samples/s +Step 6470/44000 | Loss: 3.2425 | LR: 4.79e-04 | 1325.0ms/step | 15.1 samples/s +Step 6480/44000 | Loss: 3.2293 | LR: 4.79e-04 | 1333.0ms/step | 15.0 samples/s +Step 6490/44000 | Loss: 3.2256 | LR: 4.79e-04 | 1331.6ms/step | 15.0 samples/s +Step 6500/44000 | Loss: 3.2675 | LR: 4.79e-04 | 1330.0ms/step | 15.0 samples/s +Validation at step 6500: Loss = 3.2446, Perplexity = 25.65 +Step 6510/44000 | Loss: 3.2385 | LR: 4.78e-04 | 3552.6ms/step | 5.6 samples/s +Step 6520/44000 | Loss: 3.2437 | LR: 4.78e-04 | 1328.2ms/step | 15.1 samples/s +Step 6530/44000 | Loss: 3.2643 | LR: 4.78e-04 | 1327.5ms/step | 15.1 samples/s +Step 6540/44000 | Loss: 3.2781 | LR: 4.78e-04 | 1326.1ms/step | 15.1 samples/s +Step 6550/44000 | Loss: 3.2497 | LR: 4.78e-04 | 1327.1ms/step | 15.1 samples/s +Step 6560/44000 | Loss: 3.2450 | LR: 4.78e-04 | 1325.5ms/step | 15.1 samples/s +Step 6570/44000 | Loss: 3.2325 | LR: 4.78e-04 | 1328.8ms/step | 15.1 samples/s +Step 6580/44000 | Loss: 3.2450 | LR: 4.78e-04 | 1328.6ms/step | 15.1 samples/s +Step 6590/44000 | Loss: 3.2594 | LR: 4.78e-04 | 1329.2ms/step | 15.0 samples/s +Step 6600/44000 | Loss: 3.2270 | LR: 4.78e-04 | 1326.8ms/step | 15.1 samples/s +Step 6610/44000 | Loss: 3.2530 | LR: 4.78e-04 | 1324.9ms/step | 15.1 samples/s +Step 6620/44000 | Loss: 3.2436 | LR: 4.78e-04 | 1327.7ms/step | 15.1 samples/s +Step 6630/44000 | Loss: 3.2509 | LR: 4.78e-04 | 1329.5ms/step | 15.0 samples/s +Step 6640/44000 | Loss: 3.2756 | LR: 4.77e-04 | 1324.7ms/step | 15.1 samples/s +Step 6650/44000 | Loss: 3.2857 | LR: 4.77e-04 | 1325.7ms/step | 15.1 samples/s +Step 6660/44000 | Loss: 3.2341 | LR: 4.77e-04 | 1325.3ms/step | 15.1 samples/s +Step 6670/44000 | Loss: 3.2618 | LR: 4.77e-04 | 1326.2ms/step | 15.1 samples/s +Step 6680/44000 | Loss: 3.2442 | LR: 4.77e-04 | 1324.6ms/step | 15.1 samples/s +Step 6690/44000 | Loss: 3.2557 | LR: 4.77e-04 | 1328.2ms/step | 15.1 samples/s +Step 6700/44000 | Loss: 3.2262 | LR: 4.77e-04 | 1330.3ms/step | 15.0 samples/s +Step 6710/44000 | Loss: 3.2497 | LR: 4.77e-04 | 1328.0ms/step | 15.1 samples/s +Step 6720/44000 | Loss: 3.2467 | LR: 4.77e-04 | 1327.2ms/step | 15.1 samples/s +Step 6730/44000 | Loss: 3.2495 | LR: 4.77e-04 | 1329.0ms/step | 15.0 samples/s +Step 6740/44000 | Loss: 3.2721 | LR: 4.77e-04 | 1326.4ms/step | 15.1 samples/s +Step 6750/44000 | Loss: 3.2436 | LR: 4.77e-04 | 1326.3ms/step | 15.1 samples/s +Validation at step 6750: Loss = 3.2330, Perplexity = 25.36 +Step 6760/44000 | Loss: 3.2241 | LR: 4.77e-04 | 3554.7ms/step | 5.6 samples/s +Step 6770/44000 | Loss: 3.2801 | LR: 4.76e-04 | 1329.0ms/step | 15.0 samples/s +Step 6780/44000 | Loss: 3.2537 | LR: 4.76e-04 | 1329.3ms/step | 15.0 samples/s +Step 6790/44000 | Loss: 3.2785 | LR: 4.76e-04 | 1326.3ms/step | 15.1 samples/s +Step 6800/44000 | Loss: 3.2360 | LR: 4.76e-04 | 1327.4ms/step | 15.1 samples/s +Step 6810/44000 | Loss: 3.2498 | LR: 4.76e-04 | 1325.9ms/step | 15.1 samples/s +Step 6820/44000 | Loss: 3.2576 | LR: 4.76e-04 | 1326.4ms/step | 15.1 samples/s +Step 6830/44000 | Loss: 3.2845 | LR: 4.76e-04 | 1325.6ms/step | 15.1 samples/s +Step 6840/44000 | Loss: 3.2444 | LR: 4.76e-04 | 1330.3ms/step | 15.0 samples/s +Step 6850/44000 | Loss: 3.2379 | LR: 4.76e-04 | 1327.0ms/step | 15.1 samples/s +Step 6860/44000 | Loss: 3.2633 | LR: 4.76e-04 | 1325.3ms/step | 15.1 samples/s +Step 6870/44000 | Loss: 3.2789 | LR: 4.76e-04 | 1330.8ms/step | 15.0 samples/s +Step 6880/44000 | Loss: 3.2223 | LR: 4.76e-04 | 1327.5ms/step | 15.1 samples/s +Step 6890/44000 | Loss: 3.2243 | LR: 4.76e-04 | 1327.9ms/step | 15.1 samples/s +Step 6900/44000 | Loss: 3.2249 | LR: 4.75e-04 | 1326.8ms/step | 15.1 samples/s +Step 6910/44000 | Loss: 3.2182 | LR: 4.75e-04 | 1326.7ms/step | 15.1 samples/s +Step 6920/44000 | Loss: 3.2338 | LR: 4.75e-04 | 1328.1ms/step | 15.1 samples/s +Step 6930/44000 | Loss: 3.2501 | LR: 4.75e-04 | 1327.3ms/step | 15.1 samples/s +Step 6940/44000 | Loss: 3.2307 | LR: 4.75e-04 | 1328.5ms/step | 15.1 samples/s +Step 6950/44000 | Loss: 3.2497 | LR: 4.75e-04 | 1330.9ms/step | 15.0 samples/s +Step 6960/44000 | Loss: 3.2038 | LR: 4.75e-04 | 1331.2ms/step | 15.0 samples/s +Step 6970/44000 | Loss: 3.2319 | LR: 4.75e-04 | 1330.0ms/step | 15.0 samples/s +Step 6980/44000 | Loss: 3.2574 | LR: 4.75e-04 | 1331.3ms/step | 15.0 samples/s +Step 6990/44000 | Loss: 3.2213 | LR: 4.75e-04 | 1335.9ms/step | 15.0 samples/s +Step 7000/44000 | Loss: 3.2007 | LR: 4.75e-04 | 1333.2ms/step | 15.0 samples/s +Validation at step 7000: Loss = 3.2227, Perplexity = 25.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-7000 +Step 7010/44000 | Loss: 3.1991 | LR: 4.75e-04 | 3880.9ms/step | 5.2 samples/s +Step 7020/44000 | Loss: 3.2480 | LR: 4.75e-04 | 1330.2ms/step | 15.0 samples/s +Step 7030/44000 | Loss: 3.2189 | LR: 4.74e-04 | 1330.7ms/step | 15.0 samples/s +Step 7040/44000 | Loss: 3.2269 | LR: 4.74e-04 | 1332.7ms/step | 15.0 samples/s +Step 7050/44000 | Loss: 3.2311 | LR: 4.74e-04 | 1330.2ms/step | 15.0 samples/s +Step 7060/44000 | Loss: 3.2277 | LR: 4.74e-04 | 1332.4ms/step | 15.0 samples/s +Step 7070/44000 | Loss: 3.2594 | LR: 4.74e-04 | 1332.8ms/step | 15.0 samples/s +Step 7080/44000 | Loss: 3.2034 | LR: 4.74e-04 | 1330.4ms/step | 15.0 samples/s +Step 7090/44000 | Loss: 3.2322 | LR: 4.74e-04 | 1331.4ms/step | 15.0 samples/s +Step 7100/44000 | Loss: 3.2444 | LR: 4.74e-04 | 1330.8ms/step | 15.0 samples/s +Step 7110/44000 | Loss: 3.2335 | LR: 4.74e-04 | 1331.0ms/step | 15.0 samples/s +Step 7120/44000 | Loss: 3.2618 | LR: 4.74e-04 | 1331.8ms/step | 15.0 samples/s +Step 7130/44000 | Loss: 3.2409 | LR: 4.74e-04 | 1330.1ms/step | 15.0 samples/s +Step 7140/44000 | Loss: 3.2436 | LR: 4.74e-04 | 1330.1ms/step | 15.0 samples/s +Step 7150/44000 | Loss: 3.2062 | LR: 4.73e-04 | 1333.0ms/step | 15.0 samples/s +Step 7160/44000 | Loss: 3.2423 | LR: 4.73e-04 | 1333.6ms/step | 15.0 samples/s +Step 7170/44000 | Loss: 3.2546 | LR: 4.73e-04 | 1332.0ms/step | 15.0 samples/s +Step 7180/44000 | Loss: 3.2576 | LR: 4.73e-04 | 1331.7ms/step | 15.0 samples/s +Step 7190/44000 | Loss: 3.2292 | LR: 4.73e-04 | 1332.3ms/step | 15.0 samples/s +Step 7200/44000 | Loss: 3.2081 | LR: 4.73e-04 | 1332.4ms/step | 15.0 samples/s +Step 7210/44000 | Loss: 3.2166 | LR: 4.73e-04 | 1328.9ms/step | 15.0 samples/s +Step 7220/44000 | Loss: 3.2256 | LR: 4.73e-04 | 1328.1ms/step | 15.1 samples/s +Step 7230/44000 | Loss: 3.2275 | LR: 4.73e-04 | 1330.9ms/step | 15.0 samples/s +Step 7240/44000 | Loss: 3.2446 | LR: 4.73e-04 | 1333.0ms/step | 15.0 samples/s +Step 7250/44000 | Loss: 3.2303 | LR: 4.73e-04 | 1332.0ms/step | 15.0 samples/s +Validation at step 7250: Loss = 3.2156, Perplexity = 24.92 +Step 7260/44000 | Loss: 3.2072 | LR: 4.73e-04 | 3572.9ms/step | 5.6 samples/s +Step 7270/44000 | Loss: 3.2352 | LR: 4.72e-04 | 1333.2ms/step | 15.0 samples/s +Step 7280/44000 | Loss: 3.2473 | LR: 4.72e-04 | 1334.4ms/step | 15.0 samples/s +Step 7290/44000 | Loss: 3.2067 | LR: 4.72e-04 | 1332.0ms/step | 15.0 samples/s +Step 7300/44000 | Loss: 3.2127 | LR: 4.72e-04 | 1333.1ms/step | 15.0 samples/s +Step 7310/44000 | Loss: 3.2242 | LR: 4.72e-04 | 1335.2ms/step | 15.0 samples/s +Step 7320/44000 | Loss: 3.2037 | LR: 4.72e-04 | 1335.0ms/step | 15.0 samples/s +Step 7330/44000 | Loss: 3.2020 | LR: 4.72e-04 | 1332.1ms/step | 15.0 samples/s +Step 7340/44000 | Loss: 3.2188 | LR: 4.72e-04 | 1334.4ms/step | 15.0 samples/s +Step 7350/44000 | Loss: 3.2198 | LR: 4.72e-04 | 1333.7ms/step | 15.0 samples/s +Step 7360/44000 | Loss: 3.2280 | LR: 4.72e-04 | 1330.2ms/step | 15.0 samples/s +Step 7370/44000 | Loss: 3.2108 | LR: 4.72e-04 | 1331.9ms/step | 15.0 samples/s +Step 7380/44000 | Loss: 3.2331 | LR: 4.72e-04 | 1327.0ms/step | 15.1 samples/s +Step 7390/44000 | Loss: 3.2063 | LR: 4.71e-04 | 1325.9ms/step | 15.1 samples/s +Step 7400/44000 | Loss: 3.2163 | LR: 4.71e-04 | 1325.7ms/step | 15.1 samples/s +Step 7410/44000 | Loss: 3.2075 | LR: 4.71e-04 | 1324.5ms/step | 15.1 samples/s +Step 7420/44000 | Loss: 3.2322 | LR: 4.71e-04 | 1324.9ms/step | 15.1 samples/s +Step 7430/44000 | Loss: 3.2395 | LR: 4.71e-04 | 1327.4ms/step | 15.1 samples/s +Step 7440/44000 | Loss: 3.2341 | LR: 4.71e-04 | 1327.0ms/step | 15.1 samples/s +Step 7450/44000 | Loss: 3.2094 | LR: 4.71e-04 | 1330.7ms/step | 15.0 samples/s +Step 7460/44000 | Loss: 3.2113 | LR: 4.71e-04 | 1332.7ms/step | 15.0 samples/s +Step 7470/44000 | Loss: 3.2190 | LR: 4.71e-04 | 1329.6ms/step | 15.0 samples/s +Step 7480/44000 | Loss: 3.2246 | LR: 4.71e-04 | 1326.1ms/step | 15.1 samples/s +Step 7490/44000 | Loss: 3.1945 | LR: 4.71e-04 | 1325.4ms/step | 15.1 samples/s +Step 7500/44000 | Loss: 3.1935 | LR: 4.71e-04 | 1327.6ms/step | 15.1 samples/s +Validation at step 7500: Loss = 3.2075, Perplexity = 24.72 +Step 7510/44000 | Loss: 3.2393 | LR: 4.70e-04 | 3558.5ms/step | 5.6 samples/s +Step 7520/44000 | Loss: 3.2213 | LR: 4.70e-04 | 1327.6ms/step | 15.1 samples/s +Step 7530/44000 | Loss: 3.2164 | LR: 4.70e-04 | 1325.4ms/step | 15.1 samples/s +Step 7540/44000 | Loss: 3.2158 | LR: 4.70e-04 | 1327.8ms/step | 15.1 samples/s +Step 7550/44000 | Loss: 3.2172 | LR: 4.70e-04 | 1328.6ms/step | 15.1 samples/s +Step 7560/44000 | Loss: 3.2177 | LR: 4.70e-04 | 1328.6ms/step | 15.1 samples/s +Step 7570/44000 | Loss: 3.2039 | LR: 4.70e-04 | 1329.4ms/step | 15.0 samples/s +Step 7580/44000 | Loss: 3.2323 | LR: 4.70e-04 | 1332.7ms/step | 15.0 samples/s +Step 7590/44000 | Loss: 3.2005 | LR: 4.70e-04 | 1328.6ms/step | 15.1 samples/s +Step 7600/44000 | Loss: 3.1945 | LR: 4.70e-04 | 1324.6ms/step | 15.1 samples/s +Step 7610/44000 | Loss: 3.2475 | LR: 4.70e-04 | 1329.1ms/step | 15.0 samples/s +Step 7620/44000 | Loss: 3.2202 | LR: 4.70e-04 | 1325.3ms/step | 15.1 samples/s +Step 7630/44000 | Loss: 3.1970 | LR: 4.69e-04 | 1328.3ms/step | 15.1 samples/s +Step 7640/44000 | Loss: 3.2110 | LR: 4.69e-04 | 1328.2ms/step | 15.1 samples/s +Step 7650/44000 | Loss: 3.2088 | LR: 4.69e-04 | 1325.4ms/step | 15.1 samples/s +Step 7660/44000 | Loss: 3.1999 | LR: 4.69e-04 | 1329.3ms/step | 15.0 samples/s +Step 7670/44000 | Loss: 3.1950 | LR: 4.69e-04 | 1329.0ms/step | 15.0 samples/s +Step 7680/44000 | Loss: 3.2127 | LR: 4.69e-04 | 1325.9ms/step | 15.1 samples/s +Step 7690/44000 | Loss: 3.1999 | LR: 4.69e-04 | 1325.7ms/step | 15.1 samples/s +Step 7700/44000 | Loss: 3.1904 | LR: 4.69e-04 | 1325.1ms/step | 15.1 samples/s +Step 7710/44000 | Loss: 3.2153 | LR: 4.69e-04 | 1328.0ms/step | 15.1 samples/s +Step 7720/44000 | Loss: 3.2194 | LR: 4.69e-04 | 1323.2ms/step | 15.1 samples/s +Step 7730/44000 | Loss: 3.2440 | LR: 4.69e-04 | 1327.5ms/step | 15.1 samples/s +Step 7740/44000 | Loss: 3.1839 | LR: 4.68e-04 | 1328.6ms/step | 15.1 samples/s +Step 7750/44000 | Loss: 3.2221 | LR: 4.68e-04 | 1327.9ms/step | 15.1 samples/s +Validation at step 7750: Loss = 3.1989, Perplexity = 24.51 +Step 7760/44000 | Loss: 3.1899 | LR: 4.68e-04 | 3567.8ms/step | 5.6 samples/s +Step 7770/44000 | Loss: 3.2444 | LR: 4.68e-04 | 1326.5ms/step | 15.1 samples/s +Step 7780/44000 | Loss: 3.2230 | LR: 4.68e-04 | 1326.5ms/step | 15.1 samples/s +Step 7790/44000 | Loss: 3.1732 | LR: 4.68e-04 | 1331.1ms/step | 15.0 samples/s +Step 7800/44000 | Loss: 3.1996 | LR: 4.68e-04 | 1331.6ms/step | 15.0 samples/s +Step 7810/44000 | Loss: 3.1842 | LR: 4.68e-04 | 1329.8ms/step | 15.0 samples/s +Step 7820/44000 | Loss: 3.2191 | LR: 4.68e-04 | 1331.0ms/step | 15.0 samples/s +Step 7830/44000 | Loss: 3.2373 | LR: 4.68e-04 | 1327.2ms/step | 15.1 samples/s +Step 7840/44000 | Loss: 3.1824 | LR: 4.68e-04 | 1329.1ms/step | 15.0 samples/s +Step 7850/44000 | Loss: 3.1917 | LR: 4.67e-04 | 1329.0ms/step | 15.0 samples/s +Step 7860/44000 | Loss: 3.2127 | LR: 4.67e-04 | 1329.0ms/step | 15.0 samples/s +Step 7870/44000 | Loss: 3.2018 | LR: 4.67e-04 | 1328.0ms/step | 15.1 samples/s +Step 7880/44000 | Loss: 3.2122 | LR: 4.67e-04 | 1329.7ms/step | 15.0 samples/s +Step 7890/44000 | Loss: 3.2199 | LR: 4.67e-04 | 1328.3ms/step | 15.1 samples/s +Step 7900/44000 | Loss: 3.1990 | LR: 4.67e-04 | 1327.8ms/step | 15.1 samples/s +Step 7910/44000 | Loss: 3.1821 | LR: 4.67e-04 | 1329.8ms/step | 15.0 samples/s +Step 7920/44000 | Loss: 3.2392 | LR: 4.67e-04 | 1328.3ms/step | 15.1 samples/s +Step 7930/44000 | Loss: 3.1722 | LR: 4.67e-04 | 1329.3ms/step | 15.0 samples/s +Step 7940/44000 | Loss: 3.2075 | LR: 4.67e-04 | 1327.5ms/step | 15.1 samples/s +Step 7950/44000 | Loss: 3.1899 | LR: 4.67e-04 | 1325.7ms/step | 15.1 samples/s +Step 7960/44000 | Loss: 3.1993 | LR: 4.66e-04 | 1330.0ms/step | 15.0 samples/s +Step 7970/44000 | Loss: 3.2564 | LR: 4.66e-04 | 1330.7ms/step | 15.0 samples/s +Step 7980/44000 | Loss: 3.1999 | LR: 4.66e-04 | 1331.1ms/step | 15.0 samples/s +Step 7990/44000 | Loss: 3.1799 | LR: 4.66e-04 | 1332.3ms/step | 15.0 samples/s +Step 8000/44000 | Loss: 3.1999 | LR: 4.66e-04 | 1331.6ms/step | 15.0 samples/s +Validation at step 8000: Loss = 3.1902, Perplexity = 24.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-8000 +Step 8010/44000 | Loss: 3.1818 | LR: 4.66e-04 | 3882.4ms/step | 5.2 samples/s +Step 8020/44000 | Loss: 3.2013 | LR: 4.66e-04 | 1332.1ms/step | 15.0 samples/s +Step 8030/44000 | Loss: 3.2330 | LR: 4.66e-04 | 1326.3ms/step | 15.1 samples/s +Step 8040/44000 | Loss: 3.2015 | LR: 4.66e-04 | 1330.2ms/step | 15.0 samples/s +Step 8050/44000 | Loss: 3.2006 | LR: 4.66e-04 | 1328.9ms/step | 15.1 samples/s +Step 8060/44000 | Loss: 3.1587 | LR: 4.66e-04 | 1330.7ms/step | 15.0 samples/s +Step 8070/44000 | Loss: 3.1924 | LR: 4.65e-04 | 1333.2ms/step | 15.0 samples/s +Step 8080/44000 | Loss: 3.2209 | LR: 4.65e-04 | 1332.0ms/step | 15.0 samples/s +Step 8090/44000 | Loss: 3.2030 | LR: 4.65e-04 | 1330.6ms/step | 15.0 samples/s +Step 8100/44000 | Loss: 3.1938 | LR: 4.65e-04 | 1333.4ms/step | 15.0 samples/s +Step 8110/44000 | Loss: 3.2037 | LR: 4.65e-04 | 1330.8ms/step | 15.0 samples/s +Step 8120/44000 | Loss: 3.2066 | LR: 4.65e-04 | 1329.4ms/step | 15.0 samples/s +Step 8130/44000 | Loss: 3.2107 | LR: 4.65e-04 | 1329.7ms/step | 15.0 samples/s +Step 8140/44000 | Loss: 3.1873 | LR: 4.65e-04 | 1327.8ms/step | 15.1 samples/s +Step 8150/44000 | Loss: 3.1966 | LR: 4.65e-04 | 1330.6ms/step | 15.0 samples/s +Step 8160/44000 | Loss: 3.2217 | LR: 4.65e-04 | 1333.8ms/step | 15.0 samples/s +Step 8170/44000 | Loss: 3.2132 | LR: 4.65e-04 | 1332.0ms/step | 15.0 samples/s +Step 8180/44000 | Loss: 3.2051 | LR: 4.64e-04 | 1332.1ms/step | 15.0 samples/s +Step 8190/44000 | Loss: 3.1916 | LR: 4.64e-04 | 1331.0ms/step | 15.0 samples/s +Step 8200/44000 | Loss: 3.2328 | LR: 4.64e-04 | 1331.8ms/step | 15.0 samples/s +Step 8210/44000 | Loss: 3.1998 | LR: 4.64e-04 | 1332.4ms/step | 15.0 samples/s +Step 8220/44000 | Loss: 3.2034 | LR: 4.64e-04 | 1334.3ms/step | 15.0 samples/s +Step 8230/44000 | Loss: 3.2014 | LR: 4.64e-04 | 1329.6ms/step | 15.0 samples/s +Step 8240/44000 | Loss: 3.1929 | LR: 4.64e-04 | 1329.6ms/step | 15.0 samples/s +Step 8250/44000 | Loss: 3.1909 | LR: 4.64e-04 | 1330.6ms/step | 15.0 samples/s +Validation at step 8250: Loss = 3.1833, Perplexity = 24.13 +Step 8260/44000 | Loss: 3.1970 | LR: 4.64e-04 | 3566.4ms/step | 5.6 samples/s +Step 8270/44000 | Loss: 3.1963 | LR: 4.64e-04 | 1331.6ms/step | 15.0 samples/s +Step 8280/44000 | Loss: 3.1829 | LR: 4.64e-04 | 1332.8ms/step | 15.0 samples/s +Step 8290/44000 | Loss: 3.1918 | LR: 4.63e-04 | 1333.3ms/step | 15.0 samples/s +Step 8300/44000 | Loss: 3.2049 | LR: 4.63e-04 | 1331.3ms/step | 15.0 samples/s +Step 8310/44000 | Loss: 3.1878 | LR: 4.63e-04 | 1333.0ms/step | 15.0 samples/s +Step 8320/44000 | Loss: 3.1741 | LR: 4.63e-04 | 1333.3ms/step | 15.0 samples/s +Step 8330/44000 | Loss: 3.1969 | LR: 4.63e-04 | 1332.9ms/step | 15.0 samples/s +Step 8340/44000 | Loss: 3.2139 | LR: 4.63e-04 | 1332.0ms/step | 15.0 samples/s +Step 8350/44000 | Loss: 3.2129 | LR: 4.63e-04 | 1331.0ms/step | 15.0 samples/s +Step 8360/44000 | Loss: 3.1755 | LR: 4.63e-04 | 1329.9ms/step | 15.0 samples/s +Step 8370/44000 | Loss: 3.1987 | LR: 4.63e-04 | 1333.0ms/step | 15.0 samples/s +Step 8380/44000 | Loss: 3.1975 | LR: 4.63e-04 | 1331.2ms/step | 15.0 samples/s +Step 8390/44000 | Loss: 3.2031 | LR: 4.62e-04 | 1331.2ms/step | 15.0 samples/s +Step 8400/44000 | Loss: 3.1782 | LR: 4.62e-04 | 1329.3ms/step | 15.0 samples/s +Step 8410/44000 | Loss: 3.2108 | LR: 4.62e-04 | 1326.8ms/step | 15.1 samples/s +Step 8420/44000 | Loss: 3.1751 | LR: 4.62e-04 | 1328.1ms/step | 15.1 samples/s +Step 8430/44000 | Loss: 3.1831 | LR: 4.62e-04 | 1329.7ms/step | 15.0 samples/s +Step 8440/44000 | Loss: 3.1777 | LR: 4.62e-04 | 1327.0ms/step | 15.1 samples/s +Step 8450/44000 | Loss: 3.1872 | LR: 4.62e-04 | 1325.4ms/step | 15.1 samples/s +Step 8460/44000 | Loss: 3.1932 | LR: 4.62e-04 | 1323.8ms/step | 15.1 samples/s +Step 8470/44000 | Loss: 3.1593 | LR: 4.62e-04 | 1325.9ms/step | 15.1 samples/s +Step 8480/44000 | Loss: 3.1744 | LR: 4.62e-04 | 1327.8ms/step | 15.1 samples/s +Step 8490/44000 | Loss: 3.2134 | LR: 4.62e-04 | 1325.8ms/step | 15.1 samples/s +Step 8500/44000 | Loss: 3.2111 | LR: 4.61e-04 | 1326.0ms/step | 15.1 samples/s +Validation at step 8500: Loss = 3.1756, Perplexity = 23.94 +Step 8510/44000 | Loss: 3.1829 | LR: 4.61e-04 | 3548.0ms/step | 5.6 samples/s +Step 8520/44000 | Loss: 3.1993 | LR: 4.61e-04 | 1327.3ms/step | 15.1 samples/s +Step 8530/44000 | Loss: 3.1915 | LR: 4.61e-04 | 1327.6ms/step | 15.1 samples/s +Step 8540/44000 | Loss: 3.1622 | LR: 4.61e-04 | 1329.3ms/step | 15.0 samples/s +Step 8550/44000 | Loss: 3.1655 | LR: 4.61e-04 | 1331.1ms/step | 15.0 samples/s +Step 8560/44000 | Loss: 3.2064 | LR: 4.61e-04 | 1330.5ms/step | 15.0 samples/s +Step 8570/44000 | Loss: 3.2085 | LR: 4.61e-04 | 1330.1ms/step | 15.0 samples/s +Step 8580/44000 | Loss: 3.1970 | LR: 4.61e-04 | 1329.1ms/step | 15.0 samples/s +Step 8590/44000 | Loss: 3.2051 | LR: 4.61e-04 | 1329.1ms/step | 15.0 samples/s +Step 8600/44000 | Loss: 3.1981 | LR: 4.60e-04 | 1327.5ms/step | 15.1 samples/s +Step 8610/44000 | Loss: 3.1684 | LR: 4.60e-04 | 1325.8ms/step | 15.1 samples/s +Step 8620/44000 | Loss: 3.2044 | LR: 4.60e-04 | 1326.8ms/step | 15.1 samples/s +Step 8630/44000 | Loss: 3.1598 | LR: 4.60e-04 | 1326.5ms/step | 15.1 samples/s +Step 8640/44000 | Loss: 3.1709 | LR: 4.60e-04 | 1325.9ms/step | 15.1 samples/s +Step 8650/44000 | Loss: 3.1896 | LR: 4.60e-04 | 1325.1ms/step | 15.1 samples/s +Step 8660/44000 | Loss: 3.1634 | LR: 4.60e-04 | 1325.4ms/step | 15.1 samples/s +Step 8670/44000 | Loss: 3.1651 | LR: 4.60e-04 | 1333.6ms/step | 15.0 samples/s +Step 8680/44000 | Loss: 3.1304 | LR: 4.60e-04 | 1330.6ms/step | 15.0 samples/s +Step 8690/44000 | Loss: 3.1787 | LR: 4.60e-04 | 1331.9ms/step | 15.0 samples/s +Step 8700/44000 | Loss: 3.1691 | LR: 4.59e-04 | 1330.4ms/step | 15.0 samples/s +Step 8710/44000 | Loss: 3.1783 | LR: 4.59e-04 | 1330.7ms/step | 15.0 samples/s +Step 8720/44000 | Loss: 3.1967 | LR: 4.59e-04 | 1327.6ms/step | 15.1 samples/s +Step 8730/44000 | Loss: 3.2029 | LR: 4.59e-04 | 1327.4ms/step | 15.1 samples/s +Step 8740/44000 | Loss: 3.1979 | LR: 4.59e-04 | 1327.3ms/step | 15.1 samples/s +Step 8750/44000 | Loss: 3.1772 | LR: 4.59e-04 | 1326.1ms/step | 15.1 samples/s +Validation at step 8750: Loss = 3.1688, Perplexity = 23.78 +Step 8760/44000 | Loss: 3.1847 | LR: 4.59e-04 | 3552.2ms/step | 5.6 samples/s +Step 8770/44000 | Loss: 3.1661 | LR: 4.59e-04 | 1330.0ms/step | 15.0 samples/s +Step 8780/44000 | Loss: 3.1818 | LR: 4.59e-04 | 1328.1ms/step | 15.1 samples/s +Step 8790/44000 | Loss: 3.1691 | LR: 4.59e-04 | 1328.6ms/step | 15.1 samples/s +Step 8800/44000 | Loss: 3.1661 | LR: 4.58e-04 | 1329.4ms/step | 15.0 samples/s +Step 8810/44000 | Loss: 3.1961 | LR: 4.58e-04 | 1329.2ms/step | 15.0 samples/s +Step 8820/44000 | Loss: 3.1643 | LR: 4.58e-04 | 1327.5ms/step | 15.1 samples/s +Step 8830/44000 | Loss: 3.1740 | LR: 4.58e-04 | 1329.7ms/step | 15.0 samples/s +Step 8840/44000 | Loss: 3.1695 | LR: 4.58e-04 | 1328.9ms/step | 15.1 samples/s +Step 8850/44000 | Loss: 3.1674 | LR: 4.58e-04 | 1326.3ms/step | 15.1 samples/s +Step 8860/44000 | Loss: 3.1628 | LR: 4.58e-04 | 1328.3ms/step | 15.1 samples/s +Step 8870/44000 | Loss: 3.1771 | LR: 4.58e-04 | 1326.4ms/step | 15.1 samples/s +Step 8880/44000 | Loss: 3.1704 | LR: 4.58e-04 | 1328.8ms/step | 15.1 samples/s +Step 8890/44000 | Loss: 3.1779 | LR: 4.58e-04 | 1328.5ms/step | 15.1 samples/s +Step 8900/44000 | Loss: 3.1564 | LR: 4.57e-04 | 1327.4ms/step | 15.1 samples/s +Step 8910/44000 | Loss: 3.1932 | LR: 4.57e-04 | 1325.6ms/step | 15.1 samples/s +Step 8920/44000 | Loss: 3.1590 | LR: 4.57e-04 | 1329.0ms/step | 15.0 samples/s +Step 8930/44000 | Loss: 3.2004 | LR: 4.57e-04 | 1328.8ms/step | 15.1 samples/s +Step 8940/44000 | Loss: 3.1601 | LR: 4.57e-04 | 1328.6ms/step | 15.1 samples/s +Step 8950/44000 | Loss: 3.1630 | LR: 4.57e-04 | 1326.0ms/step | 15.1 samples/s +Step 8960/44000 | Loss: 3.1768 | LR: 4.57e-04 | 1330.0ms/step | 15.0 samples/s +Step 8970/44000 | Loss: 3.1858 | LR: 4.57e-04 | 1330.5ms/step | 15.0 samples/s +Step 8980/44000 | Loss: 3.1801 | LR: 4.57e-04 | 1328.4ms/step | 15.1 samples/s +Step 8990/44000 | Loss: 3.1597 | LR: 4.57e-04 | 1332.0ms/step | 15.0 samples/s +Step 9000/44000 | Loss: 3.1643 | LR: 4.56e-04 | 1332.6ms/step | 15.0 samples/s +Validation at step 9000: Loss = 3.1622, Perplexity = 23.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-9000 +Step 9010/44000 | Loss: 3.1449 | LR: 4.56e-04 | 3889.1ms/step | 5.1 samples/s +Step 9020/44000 | Loss: 3.1458 | LR: 4.56e-04 | 1329.7ms/step | 15.0 samples/s +Step 9030/44000 | Loss: 3.2212 | LR: 4.56e-04 | 1326.7ms/step | 15.1 samples/s +Step 9040/44000 | Loss: 3.1824 | LR: 4.56e-04 | 1326.7ms/step | 15.1 samples/s +Step 9050/44000 | Loss: 3.1583 | LR: 4.56e-04 | 1329.7ms/step | 15.0 samples/s +Step 9060/44000 | Loss: 3.1825 | LR: 4.56e-04 | 1330.9ms/step | 15.0 samples/s +Step 9070/44000 | Loss: 3.1627 | LR: 4.56e-04 | 1332.3ms/step | 15.0 samples/s +Step 9080/44000 | Loss: 3.1438 | LR: 4.56e-04 | 1332.9ms/step | 15.0 samples/s +Step 9090/44000 | Loss: 3.1350 | LR: 4.56e-04 | 1332.8ms/step | 15.0 samples/s +Step 9100/44000 | Loss: 3.1881 | LR: 4.55e-04 | 1333.2ms/step | 15.0 samples/s +Step 9110/44000 | Loss: 3.1783 | LR: 4.55e-04 | 1332.6ms/step | 15.0 samples/s +Step 9120/44000 | Loss: 3.2139 | LR: 4.55e-04 | 1331.9ms/step | 15.0 samples/s +Step 9130/44000 | Loss: 3.1493 | LR: 4.55e-04 | 1330.8ms/step | 15.0 samples/s +Step 9140/44000 | Loss: 3.1769 | LR: 4.55e-04 | 1330.9ms/step | 15.0 samples/s +Step 9150/44000 | Loss: 3.1759 | LR: 4.55e-04 | 1332.4ms/step | 15.0 samples/s +Step 9160/44000 | Loss: 3.1885 | LR: 4.55e-04 | 1333.6ms/step | 15.0 samples/s +Step 9170/44000 | Loss: 3.1868 | LR: 4.55e-04 | 1334.3ms/step | 15.0 samples/s +Step 9180/44000 | Loss: 3.1617 | LR: 4.55e-04 | 1334.7ms/step | 15.0 samples/s +Step 9190/44000 | Loss: 3.1572 | LR: 4.54e-04 | 1332.4ms/step | 15.0 samples/s +Step 9200/44000 | Loss: 3.1788 | LR: 4.54e-04 | 1334.1ms/step | 15.0 samples/s +Step 9210/44000 | Loss: 3.1614 | LR: 4.54e-04 | 1331.9ms/step | 15.0 samples/s +Step 9220/44000 | Loss: 3.1805 | LR: 4.54e-04 | 1334.7ms/step | 15.0 samples/s +Step 9230/44000 | Loss: 3.1711 | LR: 4.54e-04 | 1333.5ms/step | 15.0 samples/s +Step 9240/44000 | Loss: 3.1831 | LR: 4.54e-04 | 1332.3ms/step | 15.0 samples/s +Step 9250/44000 | Loss: 3.1576 | LR: 4.54e-04 | 1331.1ms/step | 15.0 samples/s +Validation at step 9250: Loss = 3.1546, Perplexity = 23.44 +Step 9260/44000 | Loss: 3.1777 | LR: 4.54e-04 | 3565.3ms/step | 5.6 samples/s +Step 9270/44000 | Loss: 3.1435 | LR: 4.54e-04 | 1332.6ms/step | 15.0 samples/s +Step 9280/44000 | Loss: 3.1450 | LR: 4.54e-04 | 1333.7ms/step | 15.0 samples/s +Step 9290/44000 | Loss: 3.1705 | LR: 4.53e-04 | 1332.4ms/step | 15.0 samples/s +Step 9300/44000 | Loss: 3.1601 | LR: 4.53e-04 | 1333.8ms/step | 15.0 samples/s +Step 9310/44000 | Loss: 3.1911 | LR: 4.53e-04 | 1334.2ms/step | 15.0 samples/s +Step 9320/44000 | Loss: 3.1890 | LR: 4.53e-04 | 1333.8ms/step | 15.0 samples/s +Step 9330/44000 | Loss: 3.1418 | LR: 4.53e-04 | 1330.2ms/step | 15.0 samples/s +Step 9340/44000 | Loss: 3.1771 | LR: 4.53e-04 | 1330.2ms/step | 15.0 samples/s +Step 9350/44000 | Loss: 3.1797 | LR: 4.53e-04 | 1331.2ms/step | 15.0 samples/s +Step 9360/44000 | Loss: 3.1747 | LR: 4.53e-04 | 1331.0ms/step | 15.0 samples/s +Step 9370/44000 | Loss: 3.1384 | LR: 4.53e-04 | 1333.3ms/step | 15.0 samples/s +Step 9380/44000 | Loss: 3.1847 | LR: 4.52e-04 | 1334.2ms/step | 15.0 samples/s +Step 9390/44000 | Loss: 3.1515 | LR: 4.52e-04 | 1330.7ms/step | 15.0 samples/s +Step 9400/44000 | Loss: 3.1975 | LR: 4.52e-04 | 1329.3ms/step | 15.0 samples/s +Step 9410/44000 | Loss: 3.1866 | LR: 4.52e-04 | 1322.5ms/step | 15.1 samples/s +Step 9420/44000 | Loss: 3.1413 | LR: 4.52e-04 | 1325.0ms/step | 15.1 samples/s +Step 9430/44000 | Loss: 3.1508 | LR: 4.52e-04 | 1326.0ms/step | 15.1 samples/s +Step 9440/44000 | Loss: 3.1517 | LR: 4.52e-04 | 1328.3ms/step | 15.1 samples/s +Step 9450/44000 | Loss: 3.1440 | LR: 4.52e-04 | 1329.9ms/step | 15.0 samples/s +Step 9460/44000 | Loss: 3.1806 | LR: 4.52e-04 | 1332.8ms/step | 15.0 samples/s +Step 9470/44000 | Loss: 3.1740 | LR: 4.52e-04 | 1331.1ms/step | 15.0 samples/s +Step 9480/44000 | Loss: 3.1558 | LR: 4.51e-04 | 1327.8ms/step | 15.1 samples/s +Step 9490/44000 | Loss: 3.1639 | LR: 4.51e-04 | 1329.5ms/step | 15.0 samples/s +Step 9500/44000 | Loss: 3.1601 | LR: 4.51e-04 | 1330.3ms/step | 15.0 samples/s +Validation at step 9500: Loss = 3.1480, Perplexity = 23.29 +Step 9510/44000 | Loss: 3.1629 | LR: 4.51e-04 | 3555.2ms/step | 5.6 samples/s +Step 9520/44000 | Loss: 3.1669 | LR: 4.51e-04 | 1330.9ms/step | 15.0 samples/s +Step 9530/44000 | Loss: 3.1878 | LR: 4.51e-04 | 1327.8ms/step | 15.1 samples/s +Step 9540/44000 | Loss: 3.1806 | LR: 4.51e-04 | 1328.6ms/step | 15.1 samples/s +Step 9550/44000 | Loss: 3.1552 | LR: 4.51e-04 | 1328.3ms/step | 15.1 samples/s +Step 9560/44000 | Loss: 3.1660 | LR: 4.51e-04 | 1328.9ms/step | 15.0 samples/s +Step 9570/44000 | Loss: 3.1648 | LR: 4.50e-04 | 1329.4ms/step | 15.0 samples/s +Step 9580/44000 | Loss: 3.1946 | LR: 4.50e-04 | 1326.0ms/step | 15.1 samples/s +Step 9590/44000 | Loss: 3.1754 | LR: 4.50e-04 | 1324.7ms/step | 15.1 samples/s +Step 9600/44000 | Loss: 3.1409 | LR: 4.50e-04 | 1324.0ms/step | 15.1 samples/s +Step 9610/44000 | Loss: 3.1735 | LR: 4.50e-04 | 1327.6ms/step | 15.1 samples/s +Step 9620/44000 | Loss: 3.1469 | LR: 4.50e-04 | 1328.1ms/step | 15.1 samples/s +Step 9630/44000 | Loss: 3.1625 | LR: 4.50e-04 | 1329.9ms/step | 15.0 samples/s +Step 9640/44000 | Loss: 3.1445 | LR: 4.50e-04 | 1331.3ms/step | 15.0 samples/s +Step 9650/44000 | Loss: 3.1899 | LR: 4.50e-04 | 1329.8ms/step | 15.0 samples/s +Step 9660/44000 | Loss: 3.1474 | LR: 4.49e-04 | 1328.4ms/step | 15.1 samples/s +Step 9670/44000 | Loss: 3.1755 | LR: 4.49e-04 | 1327.9ms/step | 15.1 samples/s +Step 9680/44000 | Loss: 3.1778 | LR: 4.49e-04 | 1327.5ms/step | 15.1 samples/s +Step 9690/44000 | Loss: 3.1571 | LR: 4.49e-04 | 1330.2ms/step | 15.0 samples/s +Step 9700/44000 | Loss: 3.1715 | LR: 4.49e-04 | 1327.1ms/step | 15.1 samples/s +Step 9710/44000 | Loss: 3.1577 | LR: 4.49e-04 | 1323.5ms/step | 15.1 samples/s +Step 9720/44000 | Loss: 3.1503 | LR: 4.49e-04 | 1325.1ms/step | 15.1 samples/s +Step 9730/44000 | Loss: 3.1525 | LR: 4.49e-04 | 1328.8ms/step | 15.1 samples/s +Step 9740/44000 | Loss: 3.1685 | LR: 4.49e-04 | 1326.0ms/step | 15.1 samples/s +Step 9750/44000 | Loss: 3.1719 | LR: 4.48e-04 | 1326.3ms/step | 15.1 samples/s +Validation at step 9750: Loss = 3.1434, Perplexity = 23.18 +Step 9760/44000 | Loss: 3.1768 | LR: 4.48e-04 | 3557.0ms/step | 5.6 samples/s +Step 9770/44000 | Loss: 3.1505 | LR: 4.48e-04 | 1328.8ms/step | 15.1 samples/s +Step 9780/44000 | Loss: 3.1488 | LR: 4.48e-04 | 1328.1ms/step | 15.1 samples/s +Step 9790/44000 | Loss: 3.1541 | LR: 4.48e-04 | 1328.1ms/step | 15.1 samples/s +Step 9800/44000 | Loss: 3.1694 | LR: 4.48e-04 | 1329.5ms/step | 15.0 samples/s +Step 9810/44000 | Loss: 3.1224 | LR: 4.48e-04 | 1328.4ms/step | 15.1 samples/s +Step 9820/44000 | Loss: 3.1627 | LR: 4.48e-04 | 1323.8ms/step | 15.1 samples/s +Step 9830/44000 | Loss: 3.1678 | LR: 4.48e-04 | 1328.1ms/step | 15.1 samples/s +Step 9840/44000 | Loss: 3.1525 | LR: 4.47e-04 | 1330.1ms/step | 15.0 samples/s +Step 9850/44000 | Loss: 3.1492 | LR: 4.47e-04 | 1329.0ms/step | 15.0 samples/s +Step 9860/44000 | Loss: 3.1459 | LR: 4.47e-04 | 1332.2ms/step | 15.0 samples/s +Step 9870/44000 | Loss: 3.1675 | LR: 4.47e-04 | 1331.2ms/step | 15.0 samples/s +Step 9880/44000 | Loss: 3.1363 | LR: 4.47e-04 | 1331.4ms/step | 15.0 samples/s +Step 9890/44000 | Loss: 3.1602 | LR: 4.47e-04 | 1330.0ms/step | 15.0 samples/s +Step 9900/44000 | Loss: 3.1523 | LR: 4.47e-04 | 1327.0ms/step | 15.1 samples/s +Step 9910/44000 | Loss: 3.1262 | LR: 4.47e-04 | 1328.7ms/step | 15.1 samples/s +Step 9920/44000 | Loss: 3.1646 | LR: 4.47e-04 | 1326.8ms/step | 15.1 samples/s +Step 9930/44000 | Loss: 3.1610 | LR: 4.46e-04 | 1327.1ms/step | 15.1 samples/s +Step 9940/44000 | Loss: 3.1359 | LR: 4.46e-04 | 1326.9ms/step | 15.1 samples/s +Step 9950/44000 | Loss: 3.1787 | LR: 4.46e-04 | 1327.9ms/step | 15.1 samples/s +Step 9960/44000 | Loss: 3.1324 | LR: 4.46e-04 | 1328.1ms/step | 15.1 samples/s +Step 9970/44000 | Loss: 3.1249 | LR: 4.46e-04 | 1327.1ms/step | 15.1 samples/s +Step 9980/44000 | Loss: 3.1613 | LR: 4.46e-04 | 1330.9ms/step | 15.0 samples/s +Step 9990/44000 | Loss: 3.1517 | LR: 4.46e-04 | 1333.2ms/step | 15.0 samples/s +Step 10000/44000 | Loss: 3.1610 | LR: 4.46e-04 | 1330.9ms/step | 15.0 samples/s +Validation at step 10000: Loss = 3.1368, Perplexity = 23.03 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-10000 +Step 10010/44000 | Loss: 3.1156 | LR: 4.46e-04 | 3884.5ms/step | 5.1 samples/s +Step 10020/44000 | Loss: 3.1625 | LR: 4.45e-04 | 1327.4ms/step | 15.1 samples/s +Step 10030/44000 | Loss: 3.1472 | LR: 4.45e-04 | 1325.9ms/step | 15.1 samples/s +Step 10040/44000 | Loss: 3.1802 | LR: 4.45e-04 | 1328.2ms/step | 15.1 samples/s +Step 10050/44000 | Loss: 3.1586 | LR: 4.45e-04 | 1330.7ms/step | 15.0 samples/s +Step 10060/44000 | Loss: 3.1532 | LR: 4.45e-04 | 1329.3ms/step | 15.0 samples/s +Step 10070/44000 | Loss: 3.1423 | LR: 4.45e-04 | 1331.0ms/step | 15.0 samples/s +Step 10080/44000 | Loss: 3.1417 | LR: 4.45e-04 | 1331.0ms/step | 15.0 samples/s +Step 10090/44000 | Loss: 3.1764 | LR: 4.45e-04 | 1329.7ms/step | 15.0 samples/s +Step 10100/44000 | Loss: 3.1860 | LR: 4.45e-04 | 1327.8ms/step | 15.1 samples/s +Step 10110/44000 | Loss: 3.1429 | LR: 4.44e-04 | 1327.7ms/step | 15.1 samples/s +Step 10120/44000 | Loss: 3.1746 | LR: 4.44e-04 | 1328.5ms/step | 15.1 samples/s +Step 10130/44000 | Loss: 3.1357 | LR: 4.44e-04 | 1330.2ms/step | 15.0 samples/s +Step 10140/44000 | Loss: 3.1619 | LR: 4.44e-04 | 1330.2ms/step | 15.0 samples/s +Step 10150/44000 | Loss: 3.1394 | LR: 4.44e-04 | 1328.0ms/step | 15.1 samples/s +Step 10160/44000 | Loss: 3.1142 | LR: 4.44e-04 | 1329.3ms/step | 15.0 samples/s +Step 10170/44000 | Loss: 3.1607 | LR: 4.44e-04 | 1328.3ms/step | 15.1 samples/s +Step 10180/44000 | Loss: 3.1499 | LR: 4.44e-04 | 1330.4ms/step | 15.0 samples/s +Step 10190/44000 | Loss: 3.1675 | LR: 4.43e-04 | 1328.8ms/step | 15.1 samples/s +Step 10200/44000 | Loss: 3.1567 | LR: 4.43e-04 | 1330.8ms/step | 15.0 samples/s +Step 10210/44000 | Loss: 3.1525 | LR: 4.43e-04 | 1330.3ms/step | 15.0 samples/s +Step 10220/44000 | Loss: 3.1654 | LR: 4.43e-04 | 1333.1ms/step | 15.0 samples/s +Step 10230/44000 | Loss: 3.1559 | LR: 4.43e-04 | 1330.6ms/step | 15.0 samples/s +Step 10240/44000 | Loss: 3.1277 | LR: 4.43e-04 | 1332.8ms/step | 15.0 samples/s +Step 10250/44000 | Loss: 3.1210 | LR: 4.43e-04 | 1331.8ms/step | 15.0 samples/s +Validation at step 10250: Loss = 3.1325, Perplexity = 22.93 +Step 10260/44000 | Loss: 3.1294 | LR: 4.43e-04 | 3565.6ms/step | 5.6 samples/s +Step 10270/44000 | Loss: 3.1568 | LR: 4.43e-04 | 1332.6ms/step | 15.0 samples/s +Step 10280/44000 | Loss: 3.1579 | LR: 4.42e-04 | 1331.9ms/step | 15.0 samples/s +Step 10290/44000 | Loss: 3.1556 | LR: 4.42e-04 | 1334.0ms/step | 15.0 samples/s +Step 10300/44000 | Loss: 3.1676 | LR: 4.42e-04 | 1330.0ms/step | 15.0 samples/s +Step 10310/44000 | Loss: 3.1437 | LR: 4.42e-04 | 1330.0ms/step | 15.0 samples/s +Step 10320/44000 | Loss: 3.1687 | LR: 4.42e-04 | 1331.3ms/step | 15.0 samples/s +Step 10330/44000 | Loss: 3.1370 | LR: 4.42e-04 | 1331.1ms/step | 15.0 samples/s +Step 10340/44000 | Loss: 3.1825 | LR: 4.42e-04 | 1331.4ms/step | 15.0 samples/s +Step 10350/44000 | Loss: 3.1465 | LR: 4.42e-04 | 1332.3ms/step | 15.0 samples/s +Step 10360/44000 | Loss: 3.1742 | LR: 4.42e-04 | 1331.9ms/step | 15.0 samples/s +Step 10370/44000 | Loss: 3.1538 | LR: 4.41e-04 | 1333.8ms/step | 15.0 samples/s +Step 10380/44000 | Loss: 3.1152 | LR: 4.41e-04 | 1329.9ms/step | 15.0 samples/s +Step 10390/44000 | Loss: 3.1563 | LR: 4.41e-04 | 1327.8ms/step | 15.1 samples/s +Step 10400/44000 | Loss: 3.1359 | LR: 4.41e-04 | 1329.2ms/step | 15.0 samples/s +Step 10410/44000 | Loss: 3.1491 | LR: 4.41e-04 | 1329.4ms/step | 15.0 samples/s +Step 10420/44000 | Loss: 3.1672 | LR: 4.41e-04 | 1329.5ms/step | 15.0 samples/s +Step 10430/44000 | Loss: 3.1403 | LR: 4.41e-04 | 1330.2ms/step | 15.0 samples/s +Step 10440/44000 | Loss: 3.1522 | LR: 4.41e-04 | 1330.1ms/step | 15.0 samples/s +Step 10450/44000 | Loss: 3.1119 | LR: 4.40e-04 | 1329.1ms/step | 15.0 samples/s +Step 10460/44000 | Loss: 3.1360 | LR: 4.40e-04 | 1327.3ms/step | 15.1 samples/s +Step 10470/44000 | Loss: 3.1540 | LR: 4.40e-04 | 1327.1ms/step | 15.1 samples/s +Step 10480/44000 | Loss: 3.1494 | LR: 4.40e-04 | 1321.5ms/step | 15.1 samples/s +Step 10490/44000 | Loss: 3.1259 | LR: 4.40e-04 | 1325.5ms/step | 15.1 samples/s +Step 10500/44000 | Loss: 3.1447 | LR: 4.40e-04 | 1328.1ms/step | 15.1 samples/s +Validation at step 10500: Loss = 3.1278, Perplexity = 22.82 +Step 10510/44000 | Loss: 3.1225 | LR: 4.40e-04 | 3556.8ms/step | 5.6 samples/s +Step 10520/44000 | Loss: 3.1103 | LR: 4.40e-04 | 1326.0ms/step | 15.1 samples/s +Step 10530/44000 | Loss: 3.1340 | LR: 4.40e-04 | 1324.9ms/step | 15.1 samples/s +Step 10540/44000 | Loss: 3.1496 | LR: 4.39e-04 | 1328.9ms/step | 15.1 samples/s +Step 10550/44000 | Loss: 3.1385 | LR: 4.39e-04 | 1327.0ms/step | 15.1 samples/s +Step 10560/44000 | Loss: 3.1282 | LR: 4.39e-04 | 1326.5ms/step | 15.1 samples/s +Step 10570/44000 | Loss: 3.1177 | LR: 4.39e-04 | 1326.2ms/step | 15.1 samples/s +Step 10580/44000 | Loss: 3.1447 | LR: 4.39e-04 | 1332.7ms/step | 15.0 samples/s +Step 10590/44000 | Loss: 3.0957 | LR: 4.39e-04 | 1331.1ms/step | 15.0 samples/s +Step 10600/44000 | Loss: 3.1249 | LR: 4.39e-04 | 1327.4ms/step | 15.1 samples/s +Step 10610/44000 | Loss: 3.1160 | LR: 4.39e-04 | 1325.3ms/step | 15.1 samples/s +Step 10620/44000 | Loss: 3.1417 | LR: 4.38e-04 | 1327.5ms/step | 15.1 samples/s +Step 10630/44000 | Loss: 3.1263 | LR: 4.38e-04 | 1327.5ms/step | 15.1 samples/s +Step 10640/44000 | Loss: 3.1263 | LR: 4.38e-04 | 1327.6ms/step | 15.1 samples/s +Step 10650/44000 | Loss: 3.1562 | LR: 4.38e-04 | 1327.6ms/step | 15.1 samples/s +Step 10660/44000 | Loss: 3.1412 | LR: 4.38e-04 | 1327.4ms/step | 15.1 samples/s +Step 10670/44000 | Loss: 3.1353 | LR: 4.38e-04 | 1329.1ms/step | 15.0 samples/s +Step 10680/44000 | Loss: 3.1454 | LR: 4.38e-04 | 1327.4ms/step | 15.1 samples/s +Step 10690/44000 | Loss: 3.1194 | LR: 4.38e-04 | 1329.9ms/step | 15.0 samples/s +Step 10700/44000 | Loss: 3.1419 | LR: 4.37e-04 | 1326.7ms/step | 15.1 samples/s +Step 10710/44000 | Loss: 3.1108 | LR: 4.37e-04 | 1329.6ms/step | 15.0 samples/s +Step 10720/44000 | Loss: 3.1349 | LR: 4.37e-04 | 1328.0ms/step | 15.1 samples/s +Step 10730/44000 | Loss: 3.1109 | LR: 4.37e-04 | 1331.5ms/step | 15.0 samples/s +Step 10740/44000 | Loss: 3.1461 | LR: 4.37e-04 | 1330.4ms/step | 15.0 samples/s +Step 10750/44000 | Loss: 3.1155 | LR: 4.37e-04 | 1331.4ms/step | 15.0 samples/s +Validation at step 10750: Loss = 3.1212, Perplexity = 22.67 +Step 10760/44000 | Loss: 3.1549 | LR: 4.37e-04 | 3550.6ms/step | 5.6 samples/s +Step 10770/44000 | Loss: 3.1374 | LR: 4.37e-04 | 1326.1ms/step | 15.1 samples/s +Step 10780/44000 | Loss: 3.1388 | LR: 4.37e-04 | 1324.6ms/step | 15.1 samples/s +Step 10790/44000 | Loss: 3.1236 | LR: 4.36e-04 | 1330.4ms/step | 15.0 samples/s +Step 10800/44000 | Loss: 3.1754 | LR: 4.36e-04 | 1333.2ms/step | 15.0 samples/s +Step 10810/44000 | Loss: 3.1422 | LR: 4.36e-04 | 1328.9ms/step | 15.1 samples/s +Step 10820/44000 | Loss: 3.1247 | LR: 4.36e-04 | 1324.3ms/step | 15.1 samples/s +Step 10830/44000 | Loss: 3.1342 | LR: 4.36e-04 | 1326.5ms/step | 15.1 samples/s +Step 10840/44000 | Loss: 3.1049 | LR: 4.36e-04 | 1325.1ms/step | 15.1 samples/s +Step 10850/44000 | Loss: 3.1334 | LR: 4.36e-04 | 1327.1ms/step | 15.1 samples/s +Step 10860/44000 | Loss: 3.1400 | LR: 4.36e-04 | 1324.7ms/step | 15.1 samples/s +Step 10870/44000 | Loss: 3.1265 | LR: 4.35e-04 | 1323.8ms/step | 15.1 samples/s +Step 10880/44000 | Loss: 3.1812 | LR: 4.35e-04 | 1323.2ms/step | 15.1 samples/s +Step 10890/44000 | Loss: 3.0726 | LR: 4.35e-04 | 1329.0ms/step | 15.0 samples/s +Step 10900/44000 | Loss: 3.1488 | LR: 4.35e-04 | 1328.5ms/step | 15.1 samples/s +Step 10910/44000 | Loss: 3.1237 | LR: 4.35e-04 | 1329.4ms/step | 15.0 samples/s +Step 10920/44000 | Loss: 3.1067 | LR: 4.35e-04 | 1328.8ms/step | 15.1 samples/s +Step 10930/44000 | Loss: 3.1322 | LR: 4.35e-04 | 1326.8ms/step | 15.1 samples/s +Step 10940/44000 | Loss: 3.1415 | LR: 4.35e-04 | 1326.3ms/step | 15.1 samples/s +Step 10950/44000 | Loss: 3.1429 | LR: 4.34e-04 | 1329.3ms/step | 15.0 samples/s +Step 10960/44000 | Loss: 3.1645 | LR: 4.34e-04 | 1329.2ms/step | 15.0 samples/s +Step 10970/44000 | Loss: 3.1610 | LR: 4.34e-04 | 1330.6ms/step | 15.0 samples/s +Step 10980/44000 | Loss: 3.1400 | LR: 4.34e-04 | 1335.0ms/step | 15.0 samples/s +Step 10990/44000 | Loss: 3.1460 | LR: 4.34e-04 | 1329.5ms/step | 15.0 samples/s +Step 11000/44000 | Loss: 3.1244 | LR: 4.34e-04 | 1332.5ms/step | 15.0 samples/s +Validation at step 11000: Loss = 3.1157, Perplexity = 22.55 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-11000 +Step 11010/44000 | Loss: 3.1283 | LR: 4.34e-04 | 3892.9ms/step | 5.1 samples/s +Step 11020/44000 | Loss: 3.1379 | LR: 4.34e-04 | 1328.2ms/step | 15.1 samples/s +Step 11030/44000 | Loss: 3.1070 | LR: 4.33e-04 | 1326.6ms/step | 15.1 samples/s +Step 11040/44000 | Loss: 3.1653 | LR: 4.33e-04 | 1329.2ms/step | 15.0 samples/s +Step 11050/44000 | Loss: 3.1295 | LR: 4.33e-04 | 1329.4ms/step | 15.0 samples/s +Step 11060/44000 | Loss: 3.1455 | LR: 4.33e-04 | 1336.2ms/step | 15.0 samples/s +Step 11070/44000 | Loss: 3.1280 | LR: 4.33e-04 | 1331.9ms/step | 15.0 samples/s +Step 11080/44000 | Loss: 3.1249 | LR: 4.33e-04 | 1336.4ms/step | 15.0 samples/s +Step 11090/44000 | Loss: 3.1126 | LR: 4.33e-04 | 1330.9ms/step | 15.0 samples/s +Step 11100/44000 | Loss: 3.1369 | LR: 4.33e-04 | 1330.5ms/step | 15.0 samples/s +Step 11110/44000 | Loss: 3.1135 | LR: 4.32e-04 | 1328.2ms/step | 15.1 samples/s +Step 11120/44000 | Loss: 3.1133 | LR: 4.32e-04 | 1334.0ms/step | 15.0 samples/s +Step 11130/44000 | Loss: 3.1288 | LR: 4.32e-04 | 1330.7ms/step | 15.0 samples/s +Step 11140/44000 | Loss: 3.1103 | LR: 4.32e-04 | 1331.2ms/step | 15.0 samples/s +Step 11150/44000 | Loss: 3.1307 | LR: 4.32e-04 | 1333.3ms/step | 15.0 samples/s +Step 11160/44000 | Loss: 3.1402 | LR: 4.32e-04 | 1332.9ms/step | 15.0 samples/s +Step 11170/44000 | Loss: 3.1226 | LR: 4.32e-04 | 1333.1ms/step | 15.0 samples/s +Step 11180/44000 | Loss: 3.1375 | LR: 4.32e-04 | 1333.0ms/step | 15.0 samples/s +Step 11190/44000 | Loss: 3.1213 | LR: 4.31e-04 | 1333.2ms/step | 15.0 samples/s +Step 11200/44000 | Loss: 3.1390 | LR: 4.31e-04 | 1333.9ms/step | 15.0 samples/s +Step 11210/44000 | Loss: 3.1428 | LR: 4.31e-04 | 1333.9ms/step | 15.0 samples/s +Step 11220/44000 | Loss: 3.1060 | LR: 4.31e-04 | 1333.9ms/step | 15.0 samples/s +Step 11230/44000 | Loss: 3.1145 | LR: 4.31e-04 | 1331.9ms/step | 15.0 samples/s +Step 11240/44000 | Loss: 3.1036 | LR: 4.31e-04 | 1332.2ms/step | 15.0 samples/s +Step 11250/44000 | Loss: 3.1009 | LR: 4.31e-04 | 1331.8ms/step | 15.0 samples/s +Validation at step 11250: Loss = 3.1113, Perplexity = 22.45 +Step 11260/44000 | Loss: 3.1171 | LR: 4.31e-04 | 3571.7ms/step | 5.6 samples/s +Step 11270/44000 | Loss: 3.1259 | LR: 4.30e-04 | 1334.9ms/step | 15.0 samples/s +Step 11280/44000 | Loss: 3.1205 | LR: 4.30e-04 | 1332.8ms/step | 15.0 samples/s +Step 11290/44000 | Loss: 3.1274 | LR: 4.30e-04 | 1331.6ms/step | 15.0 samples/s +Step 11300/44000 | Loss: 3.0996 | LR: 4.30e-04 | 1331.6ms/step | 15.0 samples/s +Step 11310/44000 | Loss: 3.1323 | LR: 4.30e-04 | 1330.3ms/step | 15.0 samples/s +Step 11320/44000 | Loss: 3.1013 | LR: 4.30e-04 | 1333.4ms/step | 15.0 samples/s +Step 11330/44000 | Loss: 3.1499 | LR: 4.30e-04 | 1333.0ms/step | 15.0 samples/s +Step 11340/44000 | Loss: 3.1267 | LR: 4.30e-04 | 1334.3ms/step | 15.0 samples/s +Step 11350/44000 | Loss: 3.1279 | LR: 4.29e-04 | 1334.8ms/step | 15.0 samples/s +Step 11360/44000 | Loss: 3.1441 | LR: 4.29e-04 | 1333.0ms/step | 15.0 samples/s +Step 11370/44000 | Loss: 3.0868 | LR: 4.29e-04 | 1331.8ms/step | 15.0 samples/s +Step 11380/44000 | Loss: 3.1035 | LR: 4.29e-04 | 1329.4ms/step | 15.0 samples/s +Step 11390/44000 | Loss: 3.1024 | LR: 4.29e-04 | 1328.1ms/step | 15.1 samples/s +Step 11400/44000 | Loss: 3.1273 | LR: 4.29e-04 | 1329.4ms/step | 15.0 samples/s +Step 11410/44000 | Loss: 3.1133 | LR: 4.29e-04 | 1328.5ms/step | 15.1 samples/s +Step 11420/44000 | Loss: 3.1133 | LR: 4.29e-04 | 1329.6ms/step | 15.0 samples/s +Step 11430/44000 | Loss: 3.1148 | LR: 4.28e-04 | 1330.4ms/step | 15.0 samples/s +Step 11440/44000 | Loss: 3.1411 | LR: 4.28e-04 | 1331.0ms/step | 15.0 samples/s +Step 11450/44000 | Loss: 3.1082 | LR: 4.28e-04 | 1332.1ms/step | 15.0 samples/s +Step 11460/44000 | Loss: 3.1307 | LR: 4.28e-04 | 1330.2ms/step | 15.0 samples/s +Step 11470/44000 | Loss: 3.1286 | LR: 4.28e-04 | 1330.4ms/step | 15.0 samples/s +Step 11480/44000 | Loss: 3.1545 | LR: 4.28e-04 | 1328.4ms/step | 15.1 samples/s +Step 11490/44000 | Loss: 3.1035 | LR: 4.28e-04 | 1328.4ms/step | 15.1 samples/s +Step 11500/44000 | Loss: 3.1447 | LR: 4.28e-04 | 1327.7ms/step | 15.1 samples/s +Validation at step 11500: Loss = 3.1072, Perplexity = 22.36 +Step 11510/44000 | Loss: 3.1326 | LR: 4.27e-04 | 3558.9ms/step | 5.6 samples/s +Step 11520/44000 | Loss: 3.1394 | LR: 4.27e-04 | 1331.1ms/step | 15.0 samples/s +Step 11530/44000 | Loss: 3.1260 | LR: 4.27e-04 | 1331.5ms/step | 15.0 samples/s +Step 11540/44000 | Loss: 3.1203 | LR: 4.27e-04 | 1330.5ms/step | 15.0 samples/s +Step 11550/44000 | Loss: 3.1073 | LR: 4.27e-04 | 1330.5ms/step | 15.0 samples/s +Step 11560/44000 | Loss: 3.1385 | LR: 4.27e-04 | 1331.6ms/step | 15.0 samples/s +Step 11570/44000 | Loss: 3.1157 | LR: 4.27e-04 | 1331.4ms/step | 15.0 samples/s +Step 11580/44000 | Loss: 3.1338 | LR: 4.27e-04 | 1329.0ms/step | 15.0 samples/s +Step 11590/44000 | Loss: 3.1251 | LR: 4.26e-04 | 1330.8ms/step | 15.0 samples/s +Step 11600/44000 | Loss: 3.1008 | LR: 4.26e-04 | 1327.7ms/step | 15.1 samples/s +Step 11610/44000 | Loss: 3.1419 | LR: 4.26e-04 | 1329.1ms/step | 15.0 samples/s +Step 11620/44000 | Loss: 3.0942 | LR: 4.26e-04 | 1326.8ms/step | 15.1 samples/s +Step 11630/44000 | Loss: 3.1157 | LR: 4.26e-04 | 1327.1ms/step | 15.1 samples/s +Step 11640/44000 | Loss: 3.1500 | LR: 4.26e-04 | 1325.5ms/step | 15.1 samples/s +Step 11650/44000 | Loss: 3.1123 | LR: 4.26e-04 | 1325.9ms/step | 15.1 samples/s +Step 11660/44000 | Loss: 3.1109 | LR: 4.26e-04 | 1322.8ms/step | 15.1 samples/s +Step 11670/44000 | Loss: 3.1135 | LR: 4.25e-04 | 1323.4ms/step | 15.1 samples/s +Step 11680/44000 | Loss: 3.1340 | LR: 4.25e-04 | 1325.7ms/step | 15.1 samples/s +Step 11690/44000 | Loss: 3.1050 | LR: 4.25e-04 | 1327.1ms/step | 15.1 samples/s +Step 11700/44000 | Loss: 3.0949 | LR: 4.25e-04 | 1324.5ms/step | 15.1 samples/s +Step 11710/44000 | Loss: 3.1008 | LR: 4.25e-04 | 1325.1ms/step | 15.1 samples/s +Step 11720/44000 | Loss: 3.1404 | LR: 4.25e-04 | 1331.3ms/step | 15.0 samples/s +Step 11730/44000 | Loss: 3.0822 | LR: 4.25e-04 | 1329.5ms/step | 15.0 samples/s +Step 11740/44000 | Loss: 3.0999 | LR: 4.24e-04 | 1326.2ms/step | 15.1 samples/s +Step 11750/44000 | Loss: 3.0962 | LR: 4.24e-04 | 1323.9ms/step | 15.1 samples/s +Validation at step 11750: Loss = 3.1015, Perplexity = 22.23 +Step 11760/44000 | Loss: 3.1184 | LR: 4.24e-04 | 3556.2ms/step | 5.6 samples/s +Step 11770/44000 | Loss: 3.1007 | LR: 4.24e-04 | 1327.2ms/step | 15.1 samples/s +Step 11780/44000 | Loss: 3.1097 | LR: 4.24e-04 | 1325.1ms/step | 15.1 samples/s +Step 11790/44000 | Loss: 3.1171 | LR: 4.24e-04 | 1327.8ms/step | 15.1 samples/s +Step 11800/44000 | Loss: 3.0860 | LR: 4.24e-04 | 1329.5ms/step | 15.0 samples/s +Step 11810/44000 | Loss: 3.0881 | LR: 4.24e-04 | 1326.1ms/step | 15.1 samples/s +Step 11820/44000 | Loss: 3.1167 | LR: 4.23e-04 | 1324.0ms/step | 15.1 samples/s +Step 11830/44000 | Loss: 3.1367 | LR: 4.23e-04 | 1328.5ms/step | 15.1 samples/s +Step 11840/44000 | Loss: 3.1217 | LR: 4.23e-04 | 1327.2ms/step | 15.1 samples/s +Step 11850/44000 | Loss: 3.1154 | LR: 4.23e-04 | 1329.2ms/step | 15.0 samples/s +Step 11860/44000 | Loss: 3.1078 | LR: 4.23e-04 | 1327.7ms/step | 15.1 samples/s +Step 11870/44000 | Loss: 3.0829 | LR: 4.23e-04 | 1325.3ms/step | 15.1 samples/s +Step 11880/44000 | Loss: 3.1119 | LR: 4.23e-04 | 1325.5ms/step | 15.1 samples/s +Step 11890/44000 | Loss: 3.1055 | LR: 4.23e-04 | 1331.1ms/step | 15.0 samples/s +Step 11900/44000 | Loss: 3.1146 | LR: 4.22e-04 | 1329.1ms/step | 15.0 samples/s +Step 11910/44000 | Loss: 3.0929 | LR: 4.22e-04 | 1327.7ms/step | 15.1 samples/s +Step 11920/44000 | Loss: 3.0709 | LR: 4.22e-04 | 1329.9ms/step | 15.0 samples/s +Step 11930/44000 | Loss: 3.1124 | LR: 4.22e-04 | 1330.2ms/step | 15.0 samples/s +Step 11940/44000 | Loss: 3.1171 | LR: 4.22e-04 | 1331.1ms/step | 15.0 samples/s +Step 11950/44000 | Loss: 3.0842 | LR: 4.22e-04 | 1329.2ms/step | 15.0 samples/s +Step 11960/44000 | Loss: 3.0975 | LR: 4.22e-04 | 1329.2ms/step | 15.0 samples/s +Step 11970/44000 | Loss: 3.1224 | LR: 4.21e-04 | 1330.3ms/step | 15.0 samples/s +Step 11980/44000 | Loss: 3.1274 | LR: 4.21e-04 | 1331.3ms/step | 15.0 samples/s +Step 11990/44000 | Loss: 3.1127 | LR: 4.21e-04 | 1330.6ms/step | 15.0 samples/s +Step 12000/44000 | Loss: 3.1090 | LR: 4.21e-04 | 1331.3ms/step | 15.0 samples/s +Validation at step 12000: Loss = 3.0971, Perplexity = 22.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-12000 +Step 12010/44000 | Loss: 3.1307 | LR: 4.21e-04 | 3886.8ms/step | 5.1 samples/s +Step 12020/44000 | Loss: 3.1303 | LR: 4.21e-04 | 1327.3ms/step | 15.1 samples/s +Step 12030/44000 | Loss: 3.1391 | LR: 4.21e-04 | 1328.6ms/step | 15.1 samples/s +Step 12040/44000 | Loss: 3.0853 | LR: 4.21e-04 | 1329.2ms/step | 15.0 samples/s +Step 12050/44000 | Loss: 3.1186 | LR: 4.20e-04 | 1333.8ms/step | 15.0 samples/s +Step 12060/44000 | Loss: 3.1002 | LR: 4.20e-04 | 1332.0ms/step | 15.0 samples/s +Step 12070/44000 | Loss: 3.1354 | LR: 4.20e-04 | 1332.6ms/step | 15.0 samples/s +Step 12080/44000 | Loss: 3.1219 | LR: 4.20e-04 | 1331.7ms/step | 15.0 samples/s +Step 12090/44000 | Loss: 3.0852 | LR: 4.20e-04 | 1334.2ms/step | 15.0 samples/s +Step 12100/44000 | Loss: 3.0964 | LR: 4.20e-04 | 1329.9ms/step | 15.0 samples/s +Step 12110/44000 | Loss: 3.1168 | LR: 4.20e-04 | 1327.2ms/step | 15.1 samples/s +Step 12120/44000 | Loss: 3.1059 | LR: 4.19e-04 | 1329.4ms/step | 15.0 samples/s +Step 12130/44000 | Loss: 3.0896 | LR: 4.19e-04 | 1330.4ms/step | 15.0 samples/s +Step 12140/44000 | Loss: 3.0798 | LR: 4.19e-04 | 1330.7ms/step | 15.0 samples/s +Step 12150/44000 | Loss: 3.0983 | LR: 4.19e-04 | 1333.2ms/step | 15.0 samples/s +Step 12160/44000 | Loss: 3.0877 | LR: 4.19e-04 | 1331.8ms/step | 15.0 samples/s +Step 12170/44000 | Loss: 3.1152 | LR: 4.19e-04 | 1330.0ms/step | 15.0 samples/s +Step 12180/44000 | Loss: 3.1230 | LR: 4.19e-04 | 1328.3ms/step | 15.1 samples/s +Step 12190/44000 | Loss: 3.1005 | LR: 4.19e-04 | 1332.3ms/step | 15.0 samples/s +Step 12200/44000 | Loss: 3.1251 | LR: 4.18e-04 | 1333.2ms/step | 15.0 samples/s +Step 12210/44000 | Loss: 3.1029 | LR: 4.18e-04 | 1332.9ms/step | 15.0 samples/s +Step 12220/44000 | Loss: 3.0729 | LR: 4.18e-04 | 1330.6ms/step | 15.0 samples/s +Step 12230/44000 | Loss: 3.0985 | LR: 4.18e-04 | 1334.2ms/step | 15.0 samples/s +Step 12240/44000 | Loss: 3.0918 | LR: 4.18e-04 | 1333.2ms/step | 15.0 samples/s +Step 12250/44000 | Loss: 3.0993 | LR: 4.18e-04 | 1332.8ms/step | 15.0 samples/s +Validation at step 12250: Loss = 3.0926, Perplexity = 22.03 +Step 12260/44000 | Loss: 3.0934 | LR: 4.18e-04 | 3565.6ms/step | 5.6 samples/s +Step 12270/44000 | Loss: 3.0775 | LR: 4.17e-04 | 1333.5ms/step | 15.0 samples/s +Step 12280/44000 | Loss: 3.0884 | LR: 4.17e-04 | 1335.2ms/step | 15.0 samples/s +Step 12290/44000 | Loss: 3.0825 | LR: 4.17e-04 | 1332.9ms/step | 15.0 samples/s +Step 12300/44000 | Loss: 3.1050 | LR: 4.17e-04 | 1331.0ms/step | 15.0 samples/s +Step 12310/44000 | Loss: 3.1322 | LR: 4.17e-04 | 1332.2ms/step | 15.0 samples/s +Step 12320/44000 | Loss: 3.0948 | LR: 4.17e-04 | 1333.9ms/step | 15.0 samples/s +Step 12330/44000 | Loss: 3.1108 | LR: 4.17e-04 | 1331.9ms/step | 15.0 samples/s +Step 12340/44000 | Loss: 3.1033 | LR: 4.17e-04 | 1329.9ms/step | 15.0 samples/s +Step 12350/44000 | Loss: 3.0775 | LR: 4.16e-04 | 1330.3ms/step | 15.0 samples/s +Step 12360/44000 | Loss: 3.1078 | LR: 4.16e-04 | 1328.8ms/step | 15.1 samples/s +Step 12370/44000 | Loss: 3.0671 | LR: 4.16e-04 | 1327.0ms/step | 15.1 samples/s +Step 12380/44000 | Loss: 3.1110 | LR: 4.16e-04 | 1328.2ms/step | 15.1 samples/s +Step 12390/44000 | Loss: 3.1260 | LR: 4.16e-04 | 1327.0ms/step | 15.1 samples/s +Step 12400/44000 | Loss: 3.1127 | LR: 4.16e-04 | 1329.4ms/step | 15.0 samples/s +Step 12410/44000 | Loss: 3.1247 | LR: 4.16e-04 | 1331.6ms/step | 15.0 samples/s +Step 12420/44000 | Loss: 3.1079 | LR: 4.15e-04 | 1329.8ms/step | 15.0 samples/s +Step 12430/44000 | Loss: 3.0863 | LR: 4.15e-04 | 1328.3ms/step | 15.1 samples/s +Step 12440/44000 | Loss: 3.1040 | LR: 4.15e-04 | 1327.9ms/step | 15.1 samples/s +Step 12450/44000 | Loss: 3.1274 | LR: 4.15e-04 | 1329.9ms/step | 15.0 samples/s +Step 12460/44000 | Loss: 3.0922 | LR: 4.15e-04 | 1329.6ms/step | 15.0 samples/s +Step 12470/44000 | Loss: 3.0753 | LR: 4.15e-04 | 1332.0ms/step | 15.0 samples/s +Step 12480/44000 | Loss: 3.0882 | LR: 4.15e-04 | 1328.5ms/step | 15.1 samples/s +Step 12490/44000 | Loss: 3.1075 | LR: 4.14e-04 | 1325.2ms/step | 15.1 samples/s +Step 12500/44000 | Loss: 3.0991 | LR: 4.14e-04 | 1328.5ms/step | 15.1 samples/s +Validation at step 12500: Loss = 3.0880, Perplexity = 21.93 +Step 12510/44000 | Loss: 3.0740 | LR: 4.14e-04 | 3553.8ms/step | 5.6 samples/s +Step 12520/44000 | Loss: 3.0698 | LR: 4.14e-04 | 1323.0ms/step | 15.1 samples/s +Step 12530/44000 | Loss: 3.0799 | LR: 4.14e-04 | 1326.7ms/step | 15.1 samples/s +Step 12540/44000 | Loss: 3.0702 | LR: 4.14e-04 | 1329.0ms/step | 15.0 samples/s +Step 12550/44000 | Loss: 3.0634 | LR: 4.14e-04 | 1327.5ms/step | 15.1 samples/s +Step 12560/44000 | Loss: 3.0902 | LR: 4.14e-04 | 1328.5ms/step | 15.1 samples/s +Step 12570/44000 | Loss: 3.0983 | LR: 4.13e-04 | 1326.1ms/step | 15.1 samples/s +Step 12580/44000 | Loss: 3.1098 | LR: 4.13e-04 | 1321.9ms/step | 15.1 samples/s +Step 12590/44000 | Loss: 3.0874 | LR: 4.13e-04 | 1326.4ms/step | 15.1 samples/s +Step 12600/44000 | Loss: 3.1268 | LR: 4.13e-04 | 1329.7ms/step | 15.0 samples/s +Step 12610/44000 | Loss: 3.1060 | LR: 4.13e-04 | 1328.7ms/step | 15.1 samples/s +Step 12620/44000 | Loss: 3.0808 | LR: 4.13e-04 | 1327.9ms/step | 15.1 samples/s +Step 12630/44000 | Loss: 3.1043 | LR: 4.13e-04 | 1329.0ms/step | 15.0 samples/s +Step 12640/44000 | Loss: 3.1078 | LR: 4.12e-04 | 1328.4ms/step | 15.1 samples/s +Step 12650/44000 | Loss: 3.1134 | LR: 4.12e-04 | 1329.2ms/step | 15.0 samples/s +Step 12660/44000 | Loss: 3.0840 | LR: 4.12e-04 | 1328.2ms/step | 15.1 samples/s +Step 12670/44000 | Loss: 3.0915 | LR: 4.12e-04 | 1327.6ms/step | 15.1 samples/s +Step 12680/44000 | Loss: 3.0988 | LR: 4.12e-04 | 1330.8ms/step | 15.0 samples/s +Step 12690/44000 | Loss: 3.1036 | LR: 4.12e-04 | 1327.1ms/step | 15.1 samples/s +Step 12700/44000 | Loss: 3.1134 | LR: 4.12e-04 | 1325.6ms/step | 15.1 samples/s +Step 12710/44000 | Loss: 3.0659 | LR: 4.11e-04 | 1328.2ms/step | 15.1 samples/s +Step 12720/44000 | Loss: 3.0948 | LR: 4.11e-04 | 1329.7ms/step | 15.0 samples/s +Step 12730/44000 | Loss: 3.1310 | LR: 4.11e-04 | 1327.3ms/step | 15.1 samples/s +Step 12740/44000 | Loss: 3.0956 | LR: 4.11e-04 | 1328.7ms/step | 15.1 samples/s +Step 12750/44000 | Loss: 3.0813 | LR: 4.11e-04 | 1329.8ms/step | 15.0 samples/s +Validation at step 12750: Loss = 3.0845, Perplexity = 21.86 +Step 12760/44000 | Loss: 3.1039 | LR: 4.11e-04 | 3559.9ms/step | 5.6 samples/s +Step 12770/44000 | Loss: 3.0930 | LR: 4.11e-04 | 1330.9ms/step | 15.0 samples/s +Step 12780/44000 | Loss: 3.1291 | LR: 4.10e-04 | 1326.8ms/step | 15.1 samples/s +Step 12790/44000 | Loss: 3.0834 | LR: 4.10e-04 | 1328.5ms/step | 15.1 samples/s +Step 12800/44000 | Loss: 3.1071 | LR: 4.10e-04 | 1331.5ms/step | 15.0 samples/s +Step 12810/44000 | Loss: 3.1084 | LR: 4.10e-04 | 1328.1ms/step | 15.1 samples/s +Step 12820/44000 | Loss: 3.0863 | LR: 4.10e-04 | 1327.3ms/step | 15.1 samples/s +Step 12830/44000 | Loss: 3.0926 | LR: 4.10e-04 | 1323.9ms/step | 15.1 samples/s +Step 12840/44000 | Loss: 3.0982 | LR: 4.10e-04 | 1328.6ms/step | 15.1 samples/s +Step 12850/44000 | Loss: 3.0941 | LR: 4.09e-04 | 1326.1ms/step | 15.1 samples/s +Step 12860/44000 | Loss: 3.1007 | LR: 4.09e-04 | 1324.3ms/step | 15.1 samples/s +Step 12870/44000 | Loss: 3.1048 | LR: 4.09e-04 | 1325.5ms/step | 15.1 samples/s +Step 12880/44000 | Loss: 3.0838 | LR: 4.09e-04 | 1329.2ms/step | 15.0 samples/s +Step 12890/44000 | Loss: 3.0912 | LR: 4.09e-04 | 1327.4ms/step | 15.1 samples/s +Step 12900/44000 | Loss: 3.1130 | LR: 4.09e-04 | 1327.0ms/step | 15.1 samples/s +Step 12910/44000 | Loss: 3.1058 | LR: 4.09e-04 | 1324.1ms/step | 15.1 samples/s +Step 12920/44000 | Loss: 3.0812 | LR: 4.09e-04 | 1326.6ms/step | 15.1 samples/s +Step 12930/44000 | Loss: 3.1255 | LR: 4.08e-04 | 1328.4ms/step | 15.1 samples/s +Step 12940/44000 | Loss: 3.0805 | LR: 4.08e-04 | 1328.8ms/step | 15.1 samples/s +Step 12950/44000 | Loss: 3.0903 | LR: 4.08e-04 | 1324.8ms/step | 15.1 samples/s +Step 12960/44000 | Loss: 3.0949 | LR: 4.08e-04 | 1329.8ms/step | 15.0 samples/s +Step 12970/44000 | Loss: 3.0726 | LR: 4.08e-04 | 1332.6ms/step | 15.0 samples/s +Step 12980/44000 | Loss: 3.0609 | LR: 4.08e-04 | 1333.4ms/step | 15.0 samples/s +Step 12990/44000 | Loss: 3.0816 | LR: 4.08e-04 | 1334.1ms/step | 15.0 samples/s +Step 13000/44000 | Loss: 3.0422 | LR: 4.07e-04 | 1332.0ms/step | 15.0 samples/s +Validation at step 13000: Loss = 3.0793, Perplexity = 21.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-13000 +Step 13010/44000 | Loss: 3.0706 | LR: 4.07e-04 | 3894.4ms/step | 5.1 samples/s +Step 13020/44000 | Loss: 3.0568 | LR: 4.07e-04 | 1330.7ms/step | 15.0 samples/s +Step 13030/44000 | Loss: 3.0968 | LR: 4.07e-04 | 1329.2ms/step | 15.0 samples/s +Step 13040/44000 | Loss: 3.0955 | LR: 4.07e-04 | 1330.4ms/step | 15.0 samples/s +Step 13050/44000 | Loss: 3.0992 | LR: 4.07e-04 | 1329.5ms/step | 15.0 samples/s +Step 13060/44000 | Loss: 3.0990 | LR: 4.07e-04 | 1330.9ms/step | 15.0 samples/s +Step 13070/44000 | Loss: 3.0639 | LR: 4.06e-04 | 1333.1ms/step | 15.0 samples/s +Step 13080/44000 | Loss: 3.1414 | LR: 4.06e-04 | 1333.3ms/step | 15.0 samples/s +Step 13090/44000 | Loss: 3.0916 | LR: 4.06e-04 | 1335.4ms/step | 15.0 samples/s +Step 13100/44000 | Loss: 3.0969 | LR: 4.06e-04 | 1332.6ms/step | 15.0 samples/s +Step 13110/44000 | Loss: 3.1036 | LR: 4.06e-04 | 1332.0ms/step | 15.0 samples/s +Step 13120/44000 | Loss: 3.0774 | LR: 4.06e-04 | 1330.6ms/step | 15.0 samples/s +Step 13130/44000 | Loss: 3.1043 | LR: 4.06e-04 | 1331.7ms/step | 15.0 samples/s +Step 13140/44000 | Loss: 3.0890 | LR: 4.05e-04 | 1330.2ms/step | 15.0 samples/s +Step 13150/44000 | Loss: 3.0793 | LR: 4.05e-04 | 1333.0ms/step | 15.0 samples/s +Step 13160/44000 | Loss: 3.0637 | LR: 4.05e-04 | 1331.8ms/step | 15.0 samples/s +Step 13170/44000 | Loss: 3.0815 | LR: 4.05e-04 | 1333.4ms/step | 15.0 samples/s +Step 13180/44000 | Loss: 3.0754 | LR: 4.05e-04 | 1333.4ms/step | 15.0 samples/s +Step 13190/44000 | Loss: 3.0814 | LR: 4.05e-04 | 1332.1ms/step | 15.0 samples/s +Step 13200/44000 | Loss: 3.1443 | LR: 4.05e-04 | 1333.2ms/step | 15.0 samples/s +Step 13210/44000 | Loss: 3.1008 | LR: 4.04e-04 | 1332.0ms/step | 15.0 samples/s +Step 13220/44000 | Loss: 3.0923 | LR: 4.04e-04 | 1334.7ms/step | 15.0 samples/s +Step 13230/44000 | Loss: 3.1053 | LR: 4.04e-04 | 1331.7ms/step | 15.0 samples/s +Step 13240/44000 | Loss: 3.1076 | LR: 4.04e-04 | 1332.5ms/step | 15.0 samples/s +Step 13250/44000 | Loss: 3.0809 | LR: 4.04e-04 | 1332.1ms/step | 15.0 samples/s +Validation at step 13250: Loss = 3.0776, Perplexity = 21.71 +Step 13260/44000 | Loss: 3.0784 | LR: 4.04e-04 | 3564.4ms/step | 5.6 samples/s +Step 13270/44000 | Loss: 3.0876 | LR: 4.04e-04 | 1333.5ms/step | 15.0 samples/s +Step 13280/44000 | Loss: 3.1084 | LR: 4.03e-04 | 1332.5ms/step | 15.0 samples/s +Step 13290/44000 | Loss: 3.0672 | LR: 4.03e-04 | 1331.9ms/step | 15.0 samples/s +Step 13300/44000 | Loss: 3.0962 | LR: 4.03e-04 | 1329.9ms/step | 15.0 samples/s +Step 13310/44000 | Loss: 3.0689 | LR: 4.03e-04 | 1328.5ms/step | 15.1 samples/s +Step 13320/44000 | Loss: 3.0952 | LR: 4.03e-04 | 1329.9ms/step | 15.0 samples/s +Step 13330/44000 | Loss: 3.1154 | LR: 4.03e-04 | 1329.4ms/step | 15.0 samples/s +Step 13340/44000 | Loss: 3.1025 | LR: 4.03e-04 | 1330.7ms/step | 15.0 samples/s +Step 13350/44000 | Loss: 3.0809 | LR: 4.02e-04 | 1327.5ms/step | 15.1 samples/s +Step 13360/44000 | Loss: 3.0540 | LR: 4.02e-04 | 1331.0ms/step | 15.0 samples/s +Step 13370/44000 | Loss: 3.0995 | LR: 4.02e-04 | 1330.2ms/step | 15.0 samples/s +Step 13380/44000 | Loss: 3.0653 | LR: 4.02e-04 | 1331.1ms/step | 15.0 samples/s +Step 13390/44000 | Loss: 3.0936 | LR: 4.02e-04 | 1328.9ms/step | 15.0 samples/s +Step 13400/44000 | Loss: 3.0652 | LR: 4.02e-04 | 1328.9ms/step | 15.1 samples/s +Step 13410/44000 | Loss: 3.0886 | LR: 4.02e-04 | 1326.9ms/step | 15.1 samples/s +Step 13420/44000 | Loss: 3.0779 | LR: 4.01e-04 | 1326.8ms/step | 15.1 samples/s +Step 13430/44000 | Loss: 3.0840 | LR: 4.01e-04 | 1327.5ms/step | 15.1 samples/s +Step 13440/44000 | Loss: 3.0595 | LR: 4.01e-04 | 1333.3ms/step | 15.0 samples/s +Step 13450/44000 | Loss: 3.1148 | LR: 4.01e-04 | 1329.0ms/step | 15.0 samples/s +Step 13460/44000 | Loss: 3.0782 | LR: 4.01e-04 | 1329.0ms/step | 15.0 samples/s +Step 13470/44000 | Loss: 3.0724 | LR: 4.01e-04 | 1330.3ms/step | 15.0 samples/s +Step 13480/44000 | Loss: 3.0833 | LR: 4.01e-04 | 1327.5ms/step | 15.1 samples/s +Step 13490/44000 | Loss: 3.0792 | LR: 4.00e-04 | 1328.1ms/step | 15.1 samples/s +Step 13500/44000 | Loss: 3.1164 | LR: 4.00e-04 | 1325.9ms/step | 15.1 samples/s +Validation at step 13500: Loss = 3.0713, Perplexity = 21.57 +Step 13510/44000 | Loss: 3.0566 | LR: 4.00e-04 | 3554.2ms/step | 5.6 samples/s +Step 13520/44000 | Loss: 3.1077 | LR: 4.00e-04 | 1325.1ms/step | 15.1 samples/s +Step 13530/44000 | Loss: 3.0504 | LR: 4.00e-04 | 1327.3ms/step | 15.1 samples/s +Step 13540/44000 | Loss: 3.0842 | LR: 4.00e-04 | 1328.1ms/step | 15.1 samples/s +Step 13550/44000 | Loss: 3.1188 | LR: 4.00e-04 | 1326.7ms/step | 15.1 samples/s +Step 13560/44000 | Loss: 3.0902 | LR: 3.99e-04 | 1327.7ms/step | 15.1 samples/s +Step 13570/44000 | Loss: 3.0630 | LR: 3.99e-04 | 1326.6ms/step | 15.1 samples/s +Step 13580/44000 | Loss: 3.0635 | LR: 3.99e-04 | 1325.8ms/step | 15.1 samples/s +Step 13590/44000 | Loss: 3.0618 | LR: 3.99e-04 | 1325.8ms/step | 15.1 samples/s +Step 13600/44000 | Loss: 3.1084 | LR: 3.99e-04 | 1327.2ms/step | 15.1 samples/s +Step 13610/44000 | Loss: 3.0986 | LR: 3.99e-04 | 1329.4ms/step | 15.0 samples/s +Step 13620/44000 | Loss: 3.0659 | LR: 3.98e-04 | 1328.1ms/step | 15.1 samples/s +Step 13630/44000 | Loss: 3.1042 | LR: 3.98e-04 | 1327.6ms/step | 15.1 samples/s +Step 13640/44000 | Loss: 3.0772 | LR: 3.98e-04 | 1325.5ms/step | 15.1 samples/s +Step 13650/44000 | Loss: 3.0823 | LR: 3.98e-04 | 1325.6ms/step | 15.1 samples/s +Step 13660/44000 | Loss: 3.0627 | LR: 3.98e-04 | 1325.8ms/step | 15.1 samples/s +Step 13670/44000 | Loss: 3.0779 | LR: 3.98e-04 | 1326.5ms/step | 15.1 samples/s +Step 13680/44000 | Loss: 3.1120 | LR: 3.98e-04 | 1324.0ms/step | 15.1 samples/s +Step 13690/44000 | Loss: 3.0824 | LR: 3.97e-04 | 1322.9ms/step | 15.1 samples/s +Step 13700/44000 | Loss: 3.0662 | LR: 3.97e-04 | 1327.4ms/step | 15.1 samples/s +Step 13710/44000 | Loss: 3.0757 | LR: 3.97e-04 | 1330.2ms/step | 15.0 samples/s +Step 13720/44000 | Loss: 3.1049 | LR: 3.97e-04 | 1327.3ms/step | 15.1 samples/s +Step 13730/44000 | Loss: 3.0745 | LR: 3.97e-04 | 1327.7ms/step | 15.1 samples/s +Step 13740/44000 | Loss: 3.1051 | LR: 3.97e-04 | 1326.6ms/step | 15.1 samples/s +Step 13750/44000 | Loss: 3.0968 | LR: 3.97e-04 | 1326.5ms/step | 15.1 samples/s +Validation at step 13750: Loss = 3.0679, Perplexity = 21.50 +Step 13760/44000 | Loss: 3.1070 | LR: 3.96e-04 | 3559.8ms/step | 5.6 samples/s +Step 13770/44000 | Loss: 3.1025 | LR: 3.96e-04 | 1329.1ms/step | 15.0 samples/s +Step 13780/44000 | Loss: 3.0790 | LR: 3.96e-04 | 1328.6ms/step | 15.1 samples/s +Step 13790/44000 | Loss: 3.0655 | LR: 3.96e-04 | 1325.8ms/step | 15.1 samples/s +Step 13800/44000 | Loss: 3.0773 | LR: 3.96e-04 | 1328.5ms/step | 15.1 samples/s +Step 13810/44000 | Loss: 3.1108 | LR: 3.96e-04 | 1331.0ms/step | 15.0 samples/s +Step 13820/44000 | Loss: 3.0953 | LR: 3.96e-04 | 1328.9ms/step | 15.0 samples/s +Step 13830/44000 | Loss: 3.0432 | LR: 3.95e-04 | 1328.5ms/step | 15.1 samples/s +Step 13840/44000 | Loss: 3.0875 | LR: 3.95e-04 | 1328.7ms/step | 15.1 samples/s +Step 13850/44000 | Loss: 3.0444 | LR: 3.95e-04 | 1329.4ms/step | 15.0 samples/s +Step 13860/44000 | Loss: 3.0863 | LR: 3.95e-04 | 1332.1ms/step | 15.0 samples/s +Step 13870/44000 | Loss: 3.0951 | LR: 3.95e-04 | 1332.7ms/step | 15.0 samples/s +Step 13880/44000 | Loss: 3.0649 | LR: 3.95e-04 | 1330.7ms/step | 15.0 samples/s +Step 13890/44000 | Loss: 3.0653 | LR: 3.95e-04 | 1327.8ms/step | 15.1 samples/s +Step 13900/44000 | Loss: 3.0842 | LR: 3.94e-04 | 1324.9ms/step | 15.1 samples/s +Step 13910/44000 | Loss: 3.0877 | LR: 3.94e-04 | 1326.3ms/step | 15.1 samples/s +Step 13920/44000 | Loss: 3.0798 | LR: 3.94e-04 | 1328.5ms/step | 15.1 samples/s +Step 13930/44000 | Loss: 3.0879 | LR: 3.94e-04 | 1331.4ms/step | 15.0 samples/s +Step 13940/44000 | Loss: 3.0713 | LR: 3.94e-04 | 1329.8ms/step | 15.0 samples/s +Step 13950/44000 | Loss: 3.0693 | LR: 3.94e-04 | 1331.9ms/step | 15.0 samples/s +Step 13960/44000 | Loss: 3.0794 | LR: 3.93e-04 | 1333.1ms/step | 15.0 samples/s +Step 13970/44000 | Loss: 3.0665 | LR: 3.93e-04 | 1330.7ms/step | 15.0 samples/s +Step 13980/44000 | Loss: 3.0654 | LR: 3.93e-04 | 1328.2ms/step | 15.1 samples/s +Step 13990/44000 | Loss: 3.0689 | LR: 3.93e-04 | 1329.7ms/step | 15.0 samples/s +Step 14000/44000 | Loss: 3.0964 | LR: 3.93e-04 | 1329.6ms/step | 15.0 samples/s +Validation at step 14000: Loss = 3.0649, Perplexity = 21.43 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-14000 +Step 14010/44000 | Loss: 3.0714 | LR: 3.93e-04 | 3872.5ms/step | 5.2 samples/s +Step 14020/44000 | Loss: 3.0932 | LR: 3.93e-04 | 1328.2ms/step | 15.1 samples/s +Step 14030/44000 | Loss: 3.0984 | LR: 3.92e-04 | 1329.4ms/step | 15.0 samples/s +Step 14040/44000 | Loss: 3.0955 | LR: 3.92e-04 | 1331.7ms/step | 15.0 samples/s +Step 14050/44000 | Loss: 3.0554 | LR: 3.92e-04 | 1329.6ms/step | 15.0 samples/s +Step 14060/44000 | Loss: 3.0658 | LR: 3.92e-04 | 1333.6ms/step | 15.0 samples/s +Step 14070/44000 | Loss: 3.0940 | LR: 3.92e-04 | 1331.5ms/step | 15.0 samples/s +Step 14080/44000 | Loss: 3.0656 | LR: 3.92e-04 | 1333.5ms/step | 15.0 samples/s +Step 14090/44000 | Loss: 3.0805 | LR: 3.92e-04 | 1329.1ms/step | 15.0 samples/s +Step 14100/44000 | Loss: 3.0729 | LR: 3.91e-04 | 1333.3ms/step | 15.0 samples/s +Step 14110/44000 | Loss: 3.0858 | LR: 3.91e-04 | 1330.6ms/step | 15.0 samples/s +Step 14120/44000 | Loss: 3.0801 | LR: 3.91e-04 | 1333.6ms/step | 15.0 samples/s +Step 14130/44000 | Loss: 3.1002 | LR: 3.91e-04 | 1331.9ms/step | 15.0 samples/s +Step 14140/44000 | Loss: 3.0740 | LR: 3.91e-04 | 1334.0ms/step | 15.0 samples/s +Step 14150/44000 | Loss: 3.0583 | LR: 3.91e-04 | 1334.1ms/step | 15.0 samples/s +Step 14160/44000 | Loss: 3.0853 | LR: 3.90e-04 | 1335.4ms/step | 15.0 samples/s +Step 14170/44000 | Loss: 3.0578 | LR: 3.90e-04 | 1331.6ms/step | 15.0 samples/s +Step 14180/44000 | Loss: 3.0873 | LR: 3.90e-04 | 1334.0ms/step | 15.0 samples/s +Step 14190/44000 | Loss: 3.0721 | LR: 3.90e-04 | 1335.0ms/step | 15.0 samples/s +Step 14200/44000 | Loss: 3.0763 | LR: 3.90e-04 | 1334.7ms/step | 15.0 samples/s +Step 14210/44000 | Loss: 3.0943 | LR: 3.90e-04 | 1332.7ms/step | 15.0 samples/s +Step 14220/44000 | Loss: 3.0828 | LR: 3.90e-04 | 1332.1ms/step | 15.0 samples/s +Step 14230/44000 | Loss: 3.0596 | LR: 3.89e-04 | 1336.8ms/step | 15.0 samples/s +Step 14240/44000 | Loss: 3.0705 | LR: 3.89e-04 | 1333.2ms/step | 15.0 samples/s +Step 14250/44000 | Loss: 3.0865 | LR: 3.89e-04 | 1333.7ms/step | 15.0 samples/s +Validation at step 14250: Loss = 3.0605, Perplexity = 21.34 +Step 14260/44000 | Loss: 3.0637 | LR: 3.89e-04 | 3567.4ms/step | 5.6 samples/s +Step 14270/44000 | Loss: 3.0724 | LR: 3.89e-04 | 1330.8ms/step | 15.0 samples/s +Step 14280/44000 | Loss: 3.0607 | LR: 3.89e-04 | 1333.4ms/step | 15.0 samples/s +Step 14290/44000 | Loss: 3.0635 | LR: 3.89e-04 | 1333.5ms/step | 15.0 samples/s +Step 14300/44000 | Loss: 3.0827 | LR: 3.88e-04 | 1333.3ms/step | 15.0 samples/s +Step 14310/44000 | Loss: 3.0745 | LR: 3.88e-04 | 1334.0ms/step | 15.0 samples/s +Step 14320/44000 | Loss: 3.0476 | LR: 3.88e-04 | 1333.8ms/step | 15.0 samples/s +Step 14330/44000 | Loss: 3.1074 | LR: 3.88e-04 | 1333.6ms/step | 15.0 samples/s +Step 14340/44000 | Loss: 3.0781 | LR: 3.88e-04 | 1332.7ms/step | 15.0 samples/s +Step 14350/44000 | Loss: 3.0630 | LR: 3.88e-04 | 1329.5ms/step | 15.0 samples/s +Step 14360/44000 | Loss: 3.0595 | LR: 3.87e-04 | 1329.3ms/step | 15.0 samples/s +Step 14370/44000 | Loss: 3.1015 | LR: 3.87e-04 | 1330.7ms/step | 15.0 samples/s +Step 14380/44000 | Loss: 3.0578 | LR: 3.87e-04 | 1330.3ms/step | 15.0 samples/s +Step 14390/44000 | Loss: 3.0991 | LR: 3.87e-04 | 1327.3ms/step | 15.1 samples/s +Step 14400/44000 | Loss: 3.0748 | LR: 3.87e-04 | 1328.6ms/step | 15.1 samples/s +Step 14410/44000 | Loss: 3.0785 | LR: 3.87e-04 | 1323.9ms/step | 15.1 samples/s +Step 14420/44000 | Loss: 3.0444 | LR: 3.87e-04 | 1328.2ms/step | 15.1 samples/s +Step 14430/44000 | Loss: 3.0810 | LR: 3.86e-04 | 1330.8ms/step | 15.0 samples/s +Step 14440/44000 | Loss: 3.0722 | LR: 3.86e-04 | 1326.2ms/step | 15.1 samples/s +Step 14450/44000 | Loss: 3.0822 | LR: 3.86e-04 | 1328.3ms/step | 15.1 samples/s +Step 14460/44000 | Loss: 3.0709 | LR: 3.86e-04 | 1328.1ms/step | 15.1 samples/s +Step 14470/44000 | Loss: 3.0510 | LR: 3.86e-04 | 1328.1ms/step | 15.1 samples/s +Step 14480/44000 | Loss: 3.0620 | LR: 3.86e-04 | 1325.3ms/step | 15.1 samples/s +Step 14490/44000 | Loss: 3.0632 | LR: 3.85e-04 | 1330.3ms/step | 15.0 samples/s +Step 14500/44000 | Loss: 3.0469 | LR: 3.85e-04 | 1326.4ms/step | 15.1 samples/s +Validation at step 14500: Loss = 3.0574, Perplexity = 21.27 +Step 14510/44000 | Loss: 3.0792 | LR: 3.85e-04 | 3563.5ms/step | 5.6 samples/s +Step 14520/44000 | Loss: 3.0700 | LR: 3.85e-04 | 1330.1ms/step | 15.0 samples/s +Step 14530/44000 | Loss: 3.0695 | LR: 3.85e-04 | 1331.8ms/step | 15.0 samples/s +Step 14540/44000 | Loss: 3.0713 | LR: 3.85e-04 | 1329.2ms/step | 15.0 samples/s +Step 14550/44000 | Loss: 3.0868 | LR: 3.85e-04 | 1328.7ms/step | 15.1 samples/s +Step 14560/44000 | Loss: 3.0564 | LR: 3.84e-04 | 1329.6ms/step | 15.0 samples/s +Step 14570/44000 | Loss: 3.0603 | LR: 3.84e-04 | 1330.6ms/step | 15.0 samples/s +Step 14580/44000 | Loss: 3.0501 | LR: 3.84e-04 | 1329.6ms/step | 15.0 samples/s +Step 14590/44000 | Loss: 3.0162 | LR: 3.84e-04 | 1331.8ms/step | 15.0 samples/s +Step 14600/44000 | Loss: 3.0661 | LR: 3.84e-04 | 1330.0ms/step | 15.0 samples/s +Step 14610/44000 | Loss: 3.0738 | LR: 3.84e-04 | 1329.2ms/step | 15.0 samples/s +Step 14620/44000 | Loss: 3.0857 | LR: 3.83e-04 | 1329.5ms/step | 15.0 samples/s +Step 14630/44000 | Loss: 3.0371 | LR: 3.83e-04 | 1329.1ms/step | 15.0 samples/s +Step 14640/44000 | Loss: 3.0644 | LR: 3.83e-04 | 1327.6ms/step | 15.1 samples/s +Step 14650/44000 | Loss: 3.0513 | LR: 3.83e-04 | 1326.1ms/step | 15.1 samples/s +Step 14660/44000 | Loss: 3.0886 | LR: 3.83e-04 | 1328.2ms/step | 15.1 samples/s +Step 14670/44000 | Loss: 3.0908 | LR: 3.83e-04 | 1328.8ms/step | 15.1 samples/s +Step 14680/44000 | Loss: 3.0701 | LR: 3.83e-04 | 1325.6ms/step | 15.1 samples/s +Step 14690/44000 | Loss: 3.0472 | LR: 3.82e-04 | 1326.4ms/step | 15.1 samples/s +Step 14700/44000 | Loss: 3.0516 | LR: 3.82e-04 | 1329.7ms/step | 15.0 samples/s +Step 14710/44000 | Loss: 3.0661 | LR: 3.82e-04 | 1330.3ms/step | 15.0 samples/s +Step 14720/44000 | Loss: 3.0551 | LR: 3.82e-04 | 1330.6ms/step | 15.0 samples/s +Step 14730/44000 | Loss: 3.0503 | LR: 3.82e-04 | 1329.4ms/step | 15.0 samples/s +Step 14740/44000 | Loss: 3.0865 | LR: 3.82e-04 | 1329.9ms/step | 15.0 samples/s +Step 14750/44000 | Loss: 3.0475 | LR: 3.81e-04 | 1330.9ms/step | 15.0 samples/s +Validation at step 14750: Loss = 3.0559, Perplexity = 21.24 +Step 14760/44000 | Loss: 3.0635 | LR: 3.81e-04 | 3562.9ms/step | 5.6 samples/s +Step 14770/44000 | Loss: 3.0616 | LR: 3.81e-04 | 1332.2ms/step | 15.0 samples/s +Step 14780/44000 | Loss: 3.0514 | LR: 3.81e-04 | 1330.1ms/step | 15.0 samples/s +Step 14790/44000 | Loss: 3.0945 | LR: 3.81e-04 | 1328.2ms/step | 15.1 samples/s +Step 14800/44000 | Loss: 3.0701 | LR: 3.81e-04 | 1328.5ms/step | 15.1 samples/s +Step 14810/44000 | Loss: 3.0757 | LR: 3.81e-04 | 1327.4ms/step | 15.1 samples/s +Step 14820/44000 | Loss: 3.0780 | LR: 3.80e-04 | 1326.7ms/step | 15.1 samples/s +Step 14830/44000 | Loss: 3.0577 | LR: 3.80e-04 | 1329.7ms/step | 15.0 samples/s +Step 14840/44000 | Loss: 3.0740 | LR: 3.80e-04 | 1331.3ms/step | 15.0 samples/s +Step 14850/44000 | Loss: 3.0624 | LR: 3.80e-04 | 1331.5ms/step | 15.0 samples/s +Step 14860/44000 | Loss: 3.0622 | LR: 3.80e-04 | 1329.5ms/step | 15.0 samples/s +Step 14870/44000 | Loss: 3.0767 | LR: 3.80e-04 | 1327.7ms/step | 15.1 samples/s +Step 14880/44000 | Loss: 3.0640 | LR: 3.79e-04 | 1331.3ms/step | 15.0 samples/s +Step 14890/44000 | Loss: 3.0541 | LR: 3.79e-04 | 1332.0ms/step | 15.0 samples/s +Step 14900/44000 | Loss: 3.0647 | LR: 3.79e-04 | 1331.3ms/step | 15.0 samples/s +Step 14910/44000 | Loss: 3.0839 | LR: 3.79e-04 | 1335.0ms/step | 15.0 samples/s +Step 14920/44000 | Loss: 3.0820 | LR: 3.79e-04 | 1333.8ms/step | 15.0 samples/s +Step 14930/44000 | Loss: 3.0579 | LR: 3.79e-04 | 1334.0ms/step | 15.0 samples/s +Step 14940/44000 | Loss: 3.0927 | LR: 3.79e-04 | 1334.2ms/step | 15.0 samples/s +Step 14950/44000 | Loss: 3.0692 | LR: 3.78e-04 | 1334.3ms/step | 15.0 samples/s +Step 14960/44000 | Loss: 3.0632 | LR: 3.78e-04 | 1331.1ms/step | 15.0 samples/s +Step 14970/44000 | Loss: 3.1145 | LR: 3.78e-04 | 1332.1ms/step | 15.0 samples/s +Step 14980/44000 | Loss: 3.0612 | LR: 3.78e-04 | 1334.1ms/step | 15.0 samples/s +Step 14990/44000 | Loss: 3.0488 | LR: 3.78e-04 | 1333.7ms/step | 15.0 samples/s +Step 15000/44000 | Loss: 3.0554 | LR: 3.78e-04 | 1331.3ms/step | 15.0 samples/s +Validation at step 15000: Loss = 3.0492, Perplexity = 21.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-15000 +Step 15010/44000 | Loss: 3.0493 | LR: 3.77e-04 | 3901.7ms/step | 5.1 samples/s +Step 15020/44000 | Loss: 3.0752 | LR: 3.77e-04 | 1332.1ms/step | 15.0 samples/s +Step 15030/44000 | Loss: 3.0503 | LR: 3.77e-04 | 1335.6ms/step | 15.0 samples/s +Step 15040/44000 | Loss: 3.0680 | LR: 3.77e-04 | 1332.7ms/step | 15.0 samples/s +Step 15050/44000 | Loss: 3.0388 | LR: 3.77e-04 | 1332.8ms/step | 15.0 samples/s +Step 15060/44000 | Loss: 3.0633 | LR: 3.77e-04 | 1333.9ms/step | 15.0 samples/s +Step 15070/44000 | Loss: 3.0650 | LR: 3.77e-04 | 1335.9ms/step | 15.0 samples/s +Step 15080/44000 | Loss: 3.0489 | LR: 3.76e-04 | 1333.3ms/step | 15.0 samples/s +Step 15090/44000 | Loss: 3.0696 | LR: 3.76e-04 | 1333.8ms/step | 15.0 samples/s +Step 15100/44000 | Loss: 3.0610 | LR: 3.76e-04 | 1329.0ms/step | 15.0 samples/s +Step 15110/44000 | Loss: 3.0669 | LR: 3.76e-04 | 1331.9ms/step | 15.0 samples/s +Step 15120/44000 | Loss: 3.0727 | LR: 3.76e-04 | 1333.3ms/step | 15.0 samples/s +Step 15130/44000 | Loss: 3.0632 | LR: 3.76e-04 | 1332.3ms/step | 15.0 samples/s +Step 15140/44000 | Loss: 3.0715 | LR: 3.75e-04 | 1335.0ms/step | 15.0 samples/s +Step 15150/44000 | Loss: 3.0501 | LR: 3.75e-04 | 1337.1ms/step | 15.0 samples/s +Step 15160/44000 | Loss: 3.0546 | LR: 3.75e-04 | 1331.1ms/step | 15.0 samples/s +Step 15170/44000 | Loss: 3.0930 | LR: 3.75e-04 | 1333.7ms/step | 15.0 samples/s +Step 15180/44000 | Loss: 3.0294 | LR: 3.75e-04 | 1333.5ms/step | 15.0 samples/s +Step 15190/44000 | Loss: 3.0568 | LR: 3.75e-04 | 1333.3ms/step | 15.0 samples/s +Step 15200/44000 | Loss: 3.0417 | LR: 3.74e-04 | 1332.6ms/step | 15.0 samples/s +Step 15210/44000 | Loss: 3.0732 | LR: 3.74e-04 | 1330.9ms/step | 15.0 samples/s +Step 15220/44000 | Loss: 3.0642 | LR: 3.74e-04 | 1334.8ms/step | 15.0 samples/s +Step 15230/44000 | Loss: 3.0723 | LR: 3.74e-04 | 1334.4ms/step | 15.0 samples/s +Step 15240/44000 | Loss: 3.0496 | LR: 3.74e-04 | 1334.3ms/step | 15.0 samples/s +Step 15250/44000 | Loss: 3.0490 | LR: 3.74e-04 | 1334.3ms/step | 15.0 samples/s +Validation at step 15250: Loss = 3.0467, Perplexity = 21.05 +Step 15260/44000 | Loss: 3.0475 | LR: 3.74e-04 | 3571.5ms/step | 5.6 samples/s +Step 15270/44000 | Loss: 3.0467 | LR: 3.73e-04 | 1334.6ms/step | 15.0 samples/s +Step 15280/44000 | Loss: 3.0471 | LR: 3.73e-04 | 1333.1ms/step | 15.0 samples/s +Step 15290/44000 | Loss: 3.0657 | LR: 3.73e-04 | 1335.1ms/step | 15.0 samples/s +Step 15300/44000 | Loss: 3.0517 | LR: 3.73e-04 | 1336.1ms/step | 15.0 samples/s +Step 15310/44000 | Loss: 3.0643 | LR: 3.73e-04 | 1333.2ms/step | 15.0 samples/s +Step 15320/44000 | Loss: 3.0784 | LR: 3.73e-04 | 1330.7ms/step | 15.0 samples/s +Step 15330/44000 | Loss: 3.0419 | LR: 3.72e-04 | 1328.6ms/step | 15.1 samples/s +Step 15340/44000 | Loss: 3.0731 | LR: 3.72e-04 | 1330.6ms/step | 15.0 samples/s +Step 15350/44000 | Loss: 3.0649 | LR: 3.72e-04 | 1332.3ms/step | 15.0 samples/s +Step 15360/44000 | Loss: 3.0715 | LR: 3.72e-04 | 1330.7ms/step | 15.0 samples/s +Step 15370/44000 | Loss: 3.0448 | LR: 3.72e-04 | 1330.3ms/step | 15.0 samples/s +Step 15380/44000 | Loss: 3.0520 | LR: 3.72e-04 | 1329.3ms/step | 15.0 samples/s +Step 15390/44000 | Loss: 3.0537 | LR: 3.71e-04 | 1333.0ms/step | 15.0 samples/s +Step 15400/44000 | Loss: 3.0556 | LR: 3.71e-04 | 1328.6ms/step | 15.1 samples/s +Step 15410/44000 | Loss: 3.0586 | LR: 3.71e-04 | 1326.2ms/step | 15.1 samples/s +Step 15420/44000 | Loss: 3.0377 | LR: 3.71e-04 | 1330.6ms/step | 15.0 samples/s +Step 15430/44000 | Loss: 3.0622 | LR: 3.71e-04 | 1325.8ms/step | 15.1 samples/s +Step 15440/44000 | Loss: 3.0298 | LR: 3.71e-04 | 1330.7ms/step | 15.0 samples/s +Step 15450/44000 | Loss: 3.0753 | LR: 3.71e-04 | 1330.8ms/step | 15.0 samples/s +Step 15460/44000 | Loss: 3.0584 | LR: 3.70e-04 | 1331.0ms/step | 15.0 samples/s +Step 15470/44000 | Loss: 3.0507 | LR: 3.70e-04 | 1329.7ms/step | 15.0 samples/s +Step 15480/44000 | Loss: 3.0541 | LR: 3.70e-04 | 1330.4ms/step | 15.0 samples/s +Step 15490/44000 | Loss: 3.0491 | LR: 3.70e-04 | 1325.7ms/step | 15.1 samples/s +Step 15500/44000 | Loss: 3.0315 | LR: 3.70e-04 | 1325.3ms/step | 15.1 samples/s +Validation at step 15500: Loss = 3.0430, Perplexity = 20.97 +Step 15510/44000 | Loss: 3.0771 | LR: 3.70e-04 | 3563.3ms/step | 5.6 samples/s +Step 15520/44000 | Loss: 3.0492 | LR: 3.69e-04 | 1327.1ms/step | 15.1 samples/s +Step 15530/44000 | Loss: 3.0414 | LR: 3.69e-04 | 1329.4ms/step | 15.0 samples/s +Step 15540/44000 | Loss: 3.0560 | LR: 3.69e-04 | 1329.6ms/step | 15.0 samples/s +Step 15550/44000 | Loss: 3.0318 | LR: 3.69e-04 | 1332.8ms/step | 15.0 samples/s +Step 15560/44000 | Loss: 3.0413 | LR: 3.69e-04 | 1329.0ms/step | 15.0 samples/s +Step 15570/44000 | Loss: 3.0393 | LR: 3.69e-04 | 1329.6ms/step | 15.0 samples/s +Step 15580/44000 | Loss: 3.0599 | LR: 3.68e-04 | 1332.1ms/step | 15.0 samples/s +Step 15590/44000 | Loss: 3.0491 | LR: 3.68e-04 | 1332.1ms/step | 15.0 samples/s +Step 15600/44000 | Loss: 3.0415 | LR: 3.68e-04 | 1327.0ms/step | 15.1 samples/s +Step 15610/44000 | Loss: 3.0300 | LR: 3.68e-04 | 1326.0ms/step | 15.1 samples/s +Step 15620/44000 | Loss: 3.0514 | LR: 3.68e-04 | 1324.7ms/step | 15.1 samples/s +Step 15630/44000 | Loss: 3.0467 | LR: 3.68e-04 | 1330.6ms/step | 15.0 samples/s +Step 15640/44000 | Loss: 3.0593 | LR: 3.67e-04 | 1331.0ms/step | 15.0 samples/s +Step 15650/44000 | Loss: 3.0492 | LR: 3.67e-04 | 1330.9ms/step | 15.0 samples/s +Step 15660/44000 | Loss: 3.0095 | LR: 3.67e-04 | 1332.8ms/step | 15.0 samples/s +Step 15670/44000 | Loss: 3.0398 | LR: 3.67e-04 | 1327.4ms/step | 15.1 samples/s +Step 15680/44000 | Loss: 3.0338 | LR: 3.67e-04 | 1328.4ms/step | 15.1 samples/s +Step 15690/44000 | Loss: 3.0498 | LR: 3.67e-04 | 1328.9ms/step | 15.0 samples/s +Step 15700/44000 | Loss: 3.0446 | LR: 3.67e-04 | 1331.6ms/step | 15.0 samples/s +Step 15710/44000 | Loss: 3.0320 | LR: 3.66e-04 | 1328.3ms/step | 15.1 samples/s +Step 15720/44000 | Loss: 3.0439 | LR: 3.66e-04 | 1329.8ms/step | 15.0 samples/s +Step 15730/44000 | Loss: 3.0656 | LR: 3.66e-04 | 1327.1ms/step | 15.1 samples/s +Step 15740/44000 | Loss: 3.0610 | LR: 3.66e-04 | 1327.7ms/step | 15.1 samples/s +Step 15750/44000 | Loss: 3.0506 | LR: 3.66e-04 | 1327.2ms/step | 15.1 samples/s +Validation at step 15750: Loss = 3.0410, Perplexity = 20.93 +Step 15760/44000 | Loss: 3.0592 | LR: 3.66e-04 | 3556.0ms/step | 5.6 samples/s +Step 15770/44000 | Loss: 3.0425 | LR: 3.65e-04 | 1326.1ms/step | 15.1 samples/s +Step 15780/44000 | Loss: 3.0457 | LR: 3.65e-04 | 1330.0ms/step | 15.0 samples/s +Step 15790/44000 | Loss: 3.0576 | LR: 3.65e-04 | 1327.7ms/step | 15.1 samples/s +Step 15800/44000 | Loss: 3.0409 | LR: 3.65e-04 | 1328.3ms/step | 15.1 samples/s +Step 15810/44000 | Loss: 3.0684 | LR: 3.65e-04 | 1327.7ms/step | 15.1 samples/s +Step 15820/44000 | Loss: 3.0265 | LR: 3.65e-04 | 1325.9ms/step | 15.1 samples/s +Step 15830/44000 | Loss: 3.0448 | LR: 3.64e-04 | 1324.5ms/step | 15.1 samples/s +Step 15840/44000 | Loss: 3.0354 | LR: 3.64e-04 | 1329.5ms/step | 15.0 samples/s +Step 15850/44000 | Loss: 3.0579 | LR: 3.64e-04 | 1330.2ms/step | 15.0 samples/s +Step 15860/44000 | Loss: 3.0674 | LR: 3.64e-04 | 1326.8ms/step | 15.1 samples/s +Step 15870/44000 | Loss: 3.0480 | LR: 3.64e-04 | 1326.0ms/step | 15.1 samples/s +Step 15880/44000 | Loss: 3.0258 | LR: 3.64e-04 | 1327.1ms/step | 15.1 samples/s +Step 15890/44000 | Loss: 3.0512 | LR: 3.63e-04 | 1331.1ms/step | 15.0 samples/s +Step 15900/44000 | Loss: 3.0286 | LR: 3.63e-04 | 1329.6ms/step | 15.0 samples/s +Step 15910/44000 | Loss: 3.0531 | LR: 3.63e-04 | 1331.8ms/step | 15.0 samples/s +Step 15920/44000 | Loss: 3.0501 | LR: 3.63e-04 | 1332.8ms/step | 15.0 samples/s +Step 15930/44000 | Loss: 3.0521 | LR: 3.63e-04 | 1333.0ms/step | 15.0 samples/s +Step 15940/44000 | Loss: 3.0433 | LR: 3.63e-04 | 1330.5ms/step | 15.0 samples/s +Step 15950/44000 | Loss: 3.0413 | LR: 3.62e-04 | 1331.0ms/step | 15.0 samples/s +Step 15960/44000 | Loss: 3.0755 | LR: 3.62e-04 | 1333.6ms/step | 15.0 samples/s +Step 15970/44000 | Loss: 3.0544 | LR: 3.62e-04 | 1335.4ms/step | 15.0 samples/s +Step 15980/44000 | Loss: 3.0619 | LR: 3.62e-04 | 1331.8ms/step | 15.0 samples/s +Step 15990/44000 | Loss: 3.0308 | LR: 3.62e-04 | 1332.9ms/step | 15.0 samples/s +Step 16000/44000 | Loss: 3.0455 | LR: 3.62e-04 | 1333.8ms/step | 15.0 samples/s +Validation at step 16000: Loss = 3.0373, Perplexity = 20.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-16000 +Step 16010/44000 | Loss: 3.0255 | LR: 3.62e-04 | 3869.4ms/step | 5.2 samples/s +Step 16020/44000 | Loss: 3.0213 | LR: 3.61e-04 | 1327.9ms/step | 15.1 samples/s +Step 16030/44000 | Loss: 3.0676 | LR: 3.61e-04 | 1325.2ms/step | 15.1 samples/s +Step 16040/44000 | Loss: 3.0211 | LR: 3.61e-04 | 1327.0ms/step | 15.1 samples/s +Step 16050/44000 | Loss: 3.0534 | LR: 3.61e-04 | 1332.1ms/step | 15.0 samples/s +Step 16060/44000 | Loss: 3.0374 | LR: 3.61e-04 | 1329.6ms/step | 15.0 samples/s +Step 16070/44000 | Loss: 3.0822 | LR: 3.61e-04 | 1330.9ms/step | 15.0 samples/s +Step 16080/44000 | Loss: 3.0303 | LR: 3.60e-04 | 1331.7ms/step | 15.0 samples/s +Step 16090/44000 | Loss: 3.0439 | LR: 3.60e-04 | 1329.8ms/step | 15.0 samples/s +Step 16100/44000 | Loss: 3.0365 | LR: 3.60e-04 | 1330.4ms/step | 15.0 samples/s +Step 16110/44000 | Loss: 3.0305 | LR: 3.60e-04 | 1333.5ms/step | 15.0 samples/s +Step 16120/44000 | Loss: 3.0866 | LR: 3.60e-04 | 1329.1ms/step | 15.0 samples/s +Step 16130/44000 | Loss: 3.0037 | LR: 3.60e-04 | 1325.7ms/step | 15.1 samples/s +Step 16140/44000 | Loss: 3.0253 | LR: 3.59e-04 | 1329.0ms/step | 15.0 samples/s +Step 16150/44000 | Loss: 3.0154 | LR: 3.59e-04 | 1330.3ms/step | 15.0 samples/s +Step 16160/44000 | Loss: 3.0472 | LR: 3.59e-04 | 1331.9ms/step | 15.0 samples/s +Step 16170/44000 | Loss: 3.0643 | LR: 3.59e-04 | 1332.8ms/step | 15.0 samples/s +Step 16180/44000 | Loss: 3.0762 | LR: 3.59e-04 | 1332.1ms/step | 15.0 samples/s +Step 16190/44000 | Loss: 3.0428 | LR: 3.59e-04 | 1331.6ms/step | 15.0 samples/s +Step 16200/44000 | Loss: 3.0522 | LR: 3.58e-04 | 1333.9ms/step | 15.0 samples/s +Step 16210/44000 | Loss: 3.0673 | LR: 3.58e-04 | 1329.8ms/step | 15.0 samples/s +Step 16220/44000 | Loss: 3.0530 | LR: 3.58e-04 | 1331.2ms/step | 15.0 samples/s +Step 16230/44000 | Loss: 3.0806 | LR: 3.58e-04 | 1333.2ms/step | 15.0 samples/s +Step 16240/44000 | Loss: 3.0411 | LR: 3.58e-04 | 1331.1ms/step | 15.0 samples/s +Step 16250/44000 | Loss: 3.0822 | LR: 3.58e-04 | 1332.1ms/step | 15.0 samples/s +Validation at step 16250: Loss = 3.0331, Perplexity = 20.76 +Step 16260/44000 | Loss: 3.0438 | LR: 3.57e-04 | 3561.0ms/step | 5.6 samples/s +Step 16270/44000 | Loss: 3.0260 | LR: 3.57e-04 | 1333.8ms/step | 15.0 samples/s +Step 16280/44000 | Loss: 3.0644 | LR: 3.57e-04 | 1333.1ms/step | 15.0 samples/s +Step 16290/44000 | Loss: 3.0416 | LR: 3.57e-04 | 1331.4ms/step | 15.0 samples/s +Step 16300/44000 | Loss: 3.0401 | LR: 3.57e-04 | 1330.7ms/step | 15.0 samples/s +Step 16310/44000 | Loss: 3.0687 | LR: 3.57e-04 | 1332.4ms/step | 15.0 samples/s +Step 16320/44000 | Loss: 3.0484 | LR: 3.56e-04 | 1331.2ms/step | 15.0 samples/s +Step 16330/44000 | Loss: 3.0239 | LR: 3.56e-04 | 1330.8ms/step | 15.0 samples/s +Step 16340/44000 | Loss: 3.0531 | LR: 3.56e-04 | 1326.4ms/step | 15.1 samples/s +Step 16350/44000 | Loss: 3.0634 | LR: 3.56e-04 | 1329.4ms/step | 15.0 samples/s +Step 16360/44000 | Loss: 3.0504 | LR: 3.56e-04 | 1330.4ms/step | 15.0 samples/s +Step 16370/44000 | Loss: 3.0723 | LR: 3.56e-04 | 1329.7ms/step | 15.0 samples/s +Step 16380/44000 | Loss: 3.0576 | LR: 3.55e-04 | 1330.4ms/step | 15.0 samples/s +Step 16390/44000 | Loss: 3.0469 | LR: 3.55e-04 | 1330.4ms/step | 15.0 samples/s +Step 16400/44000 | Loss: 3.0445 | LR: 3.55e-04 | 1329.1ms/step | 15.0 samples/s +Step 16410/44000 | Loss: 3.0315 | LR: 3.55e-04 | 1328.1ms/step | 15.1 samples/s +Step 16420/44000 | Loss: 3.0465 | LR: 3.55e-04 | 1330.9ms/step | 15.0 samples/s +Step 16430/44000 | Loss: 3.0559 | LR: 3.55e-04 | 1333.0ms/step | 15.0 samples/s +Step 16440/44000 | Loss: 3.0522 | LR: 3.54e-04 | 1330.7ms/step | 15.0 samples/s +Step 16450/44000 | Loss: 3.0459 | LR: 3.54e-04 | 1331.2ms/step | 15.0 samples/s +Step 16460/44000 | Loss: 3.0643 | LR: 3.54e-04 | 1332.0ms/step | 15.0 samples/s +Step 16470/44000 | Loss: 3.0584 | LR: 3.54e-04 | 1329.2ms/step | 15.0 samples/s +Step 16480/44000 | Loss: 3.0208 | LR: 3.54e-04 | 1330.3ms/step | 15.0 samples/s +Step 16490/44000 | Loss: 3.0529 | LR: 3.54e-04 | 1332.6ms/step | 15.0 samples/s +Step 16500/44000 | Loss: 3.0544 | LR: 3.53e-04 | 1328.7ms/step | 15.1 samples/s +Validation at step 16500: Loss = 3.0301, Perplexity = 20.70 +Step 16510/44000 | Loss: 3.0163 | LR: 3.53e-04 | 3556.4ms/step | 5.6 samples/s +Step 16520/44000 | Loss: 3.0154 | LR: 3.53e-04 | 1331.0ms/step | 15.0 samples/s +Step 16530/44000 | Loss: 3.0401 | LR: 3.53e-04 | 1330.0ms/step | 15.0 samples/s +Step 16540/44000 | Loss: 3.0346 | LR: 3.53e-04 | 1330.4ms/step | 15.0 samples/s +Step 16550/44000 | Loss: 3.0342 | LR: 3.53e-04 | 1330.3ms/step | 15.0 samples/s +Step 16560/44000 | Loss: 3.0235 | LR: 3.52e-04 | 1327.9ms/step | 15.1 samples/s +Step 16570/44000 | Loss: 3.0708 | LR: 3.52e-04 | 1329.9ms/step | 15.0 samples/s +Step 16580/44000 | Loss: 3.0087 | LR: 3.52e-04 | 1328.1ms/step | 15.1 samples/s +Step 16590/44000 | Loss: 3.0539 | LR: 3.52e-04 | 1327.8ms/step | 15.1 samples/s +Step 16600/44000 | Loss: 3.0126 | LR: 3.52e-04 | 1332.4ms/step | 15.0 samples/s +Step 16610/44000 | Loss: 3.0641 | LR: 3.52e-04 | 1332.3ms/step | 15.0 samples/s +Step 16620/44000 | Loss: 3.0091 | LR: 3.51e-04 | 1330.0ms/step | 15.0 samples/s +Step 16630/44000 | Loss: 3.0497 | LR: 3.51e-04 | 1329.9ms/step | 15.0 samples/s +Step 16640/44000 | Loss: 3.0466 | LR: 3.51e-04 | 1328.3ms/step | 15.1 samples/s +Step 16650/44000 | Loss: 3.0421 | LR: 3.51e-04 | 1328.4ms/step | 15.1 samples/s +Step 16660/44000 | Loss: 3.0348 | LR: 3.51e-04 | 1329.7ms/step | 15.0 samples/s +Step 16670/44000 | Loss: 3.0085 | LR: 3.51e-04 | 1332.6ms/step | 15.0 samples/s +Step 16680/44000 | Loss: 3.0154 | LR: 3.50e-04 | 1328.9ms/step | 15.0 samples/s +Step 16690/44000 | Loss: 3.0269 | LR: 3.50e-04 | 1329.9ms/step | 15.0 samples/s +Step 16700/44000 | Loss: 3.0105 | LR: 3.50e-04 | 1328.9ms/step | 15.0 samples/s +Step 16710/44000 | Loss: 3.0374 | LR: 3.50e-04 | 1327.6ms/step | 15.1 samples/s +Step 16720/44000 | Loss: 2.9957 | LR: 3.50e-04 | 1323.8ms/step | 15.1 samples/s +Step 16730/44000 | Loss: 3.0585 | LR: 3.50e-04 | 1328.9ms/step | 15.0 samples/s +Step 16740/44000 | Loss: 3.0333 | LR: 3.49e-04 | 1328.9ms/step | 15.1 samples/s +Step 16750/44000 | Loss: 3.0425 | LR: 3.49e-04 | 1327.1ms/step | 15.1 samples/s +Validation at step 16750: Loss = 3.0270, Perplexity = 20.63 +Step 16760/44000 | Loss: 3.0508 | LR: 3.49e-04 | 3557.5ms/step | 5.6 samples/s +Step 16770/44000 | Loss: 3.0585 | LR: 3.49e-04 | 1329.0ms/step | 15.0 samples/s +Step 16780/44000 | Loss: 3.0551 | LR: 3.49e-04 | 1329.5ms/step | 15.0 samples/s +Step 16790/44000 | Loss: 3.0652 | LR: 3.49e-04 | 1328.5ms/step | 15.1 samples/s +Step 16800/44000 | Loss: 3.0469 | LR: 3.48e-04 | 1329.9ms/step | 15.0 samples/s +Step 16810/44000 | Loss: 3.0715 | LR: 3.48e-04 | 1333.0ms/step | 15.0 samples/s +Step 16820/44000 | Loss: 3.0562 | LR: 3.48e-04 | 1327.2ms/step | 15.1 samples/s +Step 16830/44000 | Loss: 3.0490 | LR: 3.48e-04 | 1328.5ms/step | 15.1 samples/s +Step 16840/44000 | Loss: 3.0355 | LR: 3.48e-04 | 1328.2ms/step | 15.1 samples/s +Step 16850/44000 | Loss: 3.0388 | LR: 3.48e-04 | 1326.8ms/step | 15.1 samples/s +Step 16860/44000 | Loss: 3.0297 | LR: 3.47e-04 | 1330.0ms/step | 15.0 samples/s +Step 16870/44000 | Loss: 3.0304 | LR: 3.47e-04 | 1330.2ms/step | 15.0 samples/s +Step 16880/44000 | Loss: 3.0294 | LR: 3.47e-04 | 1333.0ms/step | 15.0 samples/s +Step 16890/44000 | Loss: 3.0442 | LR: 3.47e-04 | 1332.2ms/step | 15.0 samples/s +Step 16900/44000 | Loss: 3.0464 | LR: 3.47e-04 | 1329.9ms/step | 15.0 samples/s +Step 16910/44000 | Loss: 3.0553 | LR: 3.47e-04 | 1330.4ms/step | 15.0 samples/s +Step 16920/44000 | Loss: 3.0382 | LR: 3.46e-04 | 1332.5ms/step | 15.0 samples/s +Step 16930/44000 | Loss: 3.0475 | LR: 3.46e-04 | 1331.9ms/step | 15.0 samples/s +Step 16940/44000 | Loss: 3.0677 | LR: 3.46e-04 | 1332.8ms/step | 15.0 samples/s +Step 16950/44000 | Loss: 3.0609 | LR: 3.46e-04 | 1331.5ms/step | 15.0 samples/s +Step 16960/44000 | Loss: 3.0099 | LR: 3.46e-04 | 1333.4ms/step | 15.0 samples/s +Step 16970/44000 | Loss: 3.0158 | LR: 3.46e-04 | 1330.7ms/step | 15.0 samples/s +Step 16980/44000 | Loss: 3.0413 | LR: 3.45e-04 | 1331.5ms/step | 15.0 samples/s +Step 16990/44000 | Loss: 3.0495 | LR: 3.45e-04 | 1332.7ms/step | 15.0 samples/s +Step 17000/44000 | Loss: 3.0498 | LR: 3.45e-04 | 1333.6ms/step | 15.0 samples/s +Validation at step 17000: Loss = 3.0272, Perplexity = 20.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-17000 +Step 17010/44000 | Loss: 3.0455 | LR: 3.45e-04 | 3868.9ms/step | 5.2 samples/s +Step 17020/44000 | Loss: 3.0343 | LR: 3.45e-04 | 1330.7ms/step | 15.0 samples/s +Step 17030/44000 | Loss: 3.0502 | LR: 3.45e-04 | 1330.3ms/step | 15.0 samples/s +Step 17040/44000 | Loss: 3.0487 | LR: 3.44e-04 | 1329.2ms/step | 15.0 samples/s +Step 17050/44000 | Loss: 3.0704 | LR: 3.44e-04 | 1333.2ms/step | 15.0 samples/s +Step 17060/44000 | Loss: 3.0183 | LR: 3.44e-04 | 1330.7ms/step | 15.0 samples/s +Step 17070/44000 | Loss: 3.0575 | LR: 3.44e-04 | 1328.5ms/step | 15.1 samples/s +Step 17080/44000 | Loss: 3.0713 | LR: 3.44e-04 | 1331.7ms/step | 15.0 samples/s +Step 17090/44000 | Loss: 3.0638 | LR: 3.44e-04 | 1332.2ms/step | 15.0 samples/s +Step 17100/44000 | Loss: 3.0332 | LR: 3.43e-04 | 1333.5ms/step | 15.0 samples/s +Step 17110/44000 | Loss: 3.0144 | LR: 3.43e-04 | 1331.1ms/step | 15.0 samples/s +Step 17120/44000 | Loss: 3.0369 | LR: 3.43e-04 | 1333.5ms/step | 15.0 samples/s +Step 17130/44000 | Loss: 3.0253 | LR: 3.43e-04 | 1330.0ms/step | 15.0 samples/s +Step 17140/44000 | Loss: 3.0420 | LR: 3.43e-04 | 1329.1ms/step | 15.0 samples/s +Step 17150/44000 | Loss: 3.0264 | LR: 3.43e-04 | 1329.9ms/step | 15.0 samples/s +Step 17160/44000 | Loss: 3.0257 | LR: 3.42e-04 | 1332.4ms/step | 15.0 samples/s +Step 17170/44000 | Loss: 3.0471 | LR: 3.42e-04 | 1332.3ms/step | 15.0 samples/s +Step 17180/44000 | Loss: 3.0399 | LR: 3.42e-04 | 1332.2ms/step | 15.0 samples/s +Step 17190/44000 | Loss: 3.0230 | LR: 3.42e-04 | 1333.4ms/step | 15.0 samples/s +Step 17200/44000 | Loss: 3.0441 | LR: 3.42e-04 | 1331.4ms/step | 15.0 samples/s +Step 17210/44000 | Loss: 3.0743 | LR: 3.42e-04 | 1334.3ms/step | 15.0 samples/s +Step 17220/44000 | Loss: 3.0389 | LR: 3.41e-04 | 1332.0ms/step | 15.0 samples/s +Step 17230/44000 | Loss: 3.0334 | LR: 3.41e-04 | 1333.8ms/step | 15.0 samples/s +Step 17240/44000 | Loss: 3.0324 | LR: 3.41e-04 | 1330.8ms/step | 15.0 samples/s +Step 17250/44000 | Loss: 2.9996 | LR: 3.41e-04 | 1330.7ms/step | 15.0 samples/s +Validation at step 17250: Loss = 3.0207, Perplexity = 20.51 +Step 17260/44000 | Loss: 3.0187 | LR: 3.41e-04 | 3563.5ms/step | 5.6 samples/s +Step 17270/44000 | Loss: 3.0197 | LR: 3.41e-04 | 1332.9ms/step | 15.0 samples/s +Step 17280/44000 | Loss: 3.0195 | LR: 3.40e-04 | 1333.5ms/step | 15.0 samples/s +Step 17290/44000 | Loss: 3.0123 | LR: 3.40e-04 | 1331.3ms/step | 15.0 samples/s +Step 17300/44000 | Loss: 3.0090 | LR: 3.40e-04 | 1328.1ms/step | 15.1 samples/s +Step 17310/44000 | Loss: 3.0284 | LR: 3.40e-04 | 1327.9ms/step | 15.1 samples/s +Step 17320/44000 | Loss: 3.0015 | LR: 3.40e-04 | 1329.7ms/step | 15.0 samples/s +Step 17330/44000 | Loss: 3.0276 | LR: 3.40e-04 | 1329.1ms/step | 15.0 samples/s +Step 17340/44000 | Loss: 3.0515 | LR: 3.39e-04 | 1329.8ms/step | 15.0 samples/s +Step 17350/44000 | Loss: 3.0428 | LR: 3.39e-04 | 1328.7ms/step | 15.1 samples/s +Step 17360/44000 | Loss: 3.0602 | LR: 3.39e-04 | 1328.3ms/step | 15.1 samples/s +Step 17370/44000 | Loss: 3.0319 | LR: 3.39e-04 | 1330.2ms/step | 15.0 samples/s +Step 17380/44000 | Loss: 3.0550 | LR: 3.39e-04 | 1331.2ms/step | 15.0 samples/s +Step 17390/44000 | Loss: 3.0075 | LR: 3.39e-04 | 1330.6ms/step | 15.0 samples/s +Step 17400/44000 | Loss: 3.0024 | LR: 3.38e-04 | 1332.2ms/step | 15.0 samples/s +Step 17410/44000 | Loss: 3.0302 | LR: 3.38e-04 | 1331.2ms/step | 15.0 samples/s +Step 17420/44000 | Loss: 3.0531 | LR: 3.38e-04 | 1331.3ms/step | 15.0 samples/s +Step 17430/44000 | Loss: 3.0486 | LR: 3.38e-04 | 1330.9ms/step | 15.0 samples/s +Step 17440/44000 | Loss: 3.0448 | LR: 3.38e-04 | 1328.7ms/step | 15.1 samples/s +Step 17450/44000 | Loss: 3.0263 | LR: 3.38e-04 | 1327.5ms/step | 15.1 samples/s +Step 17460/44000 | Loss: 3.0616 | LR: 3.37e-04 | 1330.5ms/step | 15.0 samples/s +Step 17470/44000 | Loss: 3.0412 | LR: 3.37e-04 | 1329.0ms/step | 15.0 samples/s +Step 17480/44000 | Loss: 3.0284 | LR: 3.37e-04 | 1327.6ms/step | 15.1 samples/s +Step 17490/44000 | Loss: 3.0456 | LR: 3.37e-04 | 1328.0ms/step | 15.1 samples/s +Step 17500/44000 | Loss: 3.0500 | LR: 3.37e-04 | 1325.8ms/step | 15.1 samples/s +Validation at step 17500: Loss = 3.0173, Perplexity = 20.44 +Step 17510/44000 | Loss: 3.0555 | LR: 3.37e-04 | 3551.6ms/step | 5.6 samples/s +Step 17520/44000 | Loss: 3.0519 | LR: 3.36e-04 | 1328.6ms/step | 15.1 samples/s +Step 17530/44000 | Loss: 3.0265 | LR: 3.36e-04 | 1324.4ms/step | 15.1 samples/s +Step 17540/44000 | Loss: 3.0289 | LR: 3.36e-04 | 1329.1ms/step | 15.0 samples/s +Step 17550/44000 | Loss: 3.0358 | LR: 3.36e-04 | 1325.7ms/step | 15.1 samples/s +Step 17560/44000 | Loss: 3.0213 | LR: 3.36e-04 | 1323.3ms/step | 15.1 samples/s +Step 17570/44000 | Loss: 3.0158 | LR: 3.35e-04 | 1326.3ms/step | 15.1 samples/s +Step 17580/44000 | Loss: 3.0449 | LR: 3.35e-04 | 1330.4ms/step | 15.0 samples/s +Step 17590/44000 | Loss: 3.0167 | LR: 3.35e-04 | 1325.3ms/step | 15.1 samples/s +Step 17600/44000 | Loss: 3.0140 | LR: 3.35e-04 | 1326.0ms/step | 15.1 samples/s +Step 17610/44000 | Loss: 3.0640 | LR: 3.35e-04 | 1326.4ms/step | 15.1 samples/s +Step 17620/44000 | Loss: 3.0048 | LR: 3.35e-04 | 1327.2ms/step | 15.1 samples/s +Step 17630/44000 | Loss: 3.0642 | LR: 3.34e-04 | 1327.1ms/step | 15.1 samples/s +Step 17640/44000 | Loss: 3.0292 | LR: 3.34e-04 | 1327.2ms/step | 15.1 samples/s +Step 17650/44000 | Loss: 3.0068 | LR: 3.34e-04 | 1324.7ms/step | 15.1 samples/s +Step 17660/44000 | Loss: 3.0235 | LR: 3.34e-04 | 1326.4ms/step | 15.1 samples/s +Step 17670/44000 | Loss: 3.0301 | LR: 3.34e-04 | 1328.5ms/step | 15.1 samples/s +Step 17680/44000 | Loss: 3.0113 | LR: 3.34e-04 | 1328.2ms/step | 15.1 samples/s +Step 17690/44000 | Loss: 3.0338 | LR: 3.33e-04 | 1328.1ms/step | 15.1 samples/s +Step 17700/44000 | Loss: 3.0400 | LR: 3.33e-04 | 1330.1ms/step | 15.0 samples/s +Step 17710/44000 | Loss: 3.0410 | LR: 3.33e-04 | 1333.6ms/step | 15.0 samples/s +Step 17720/44000 | Loss: 3.0292 | LR: 3.33e-04 | 1333.2ms/step | 15.0 samples/s +Step 17730/44000 | Loss: 3.0081 | LR: 3.33e-04 | 1329.1ms/step | 15.0 samples/s +Step 17740/44000 | Loss: 3.0404 | LR: 3.33e-04 | 1331.0ms/step | 15.0 samples/s +Step 17750/44000 | Loss: 3.0190 | LR: 3.32e-04 | 1330.6ms/step | 15.0 samples/s +Validation at step 17750: Loss = 3.0150, Perplexity = 20.39 +Step 17760/44000 | Loss: 3.0361 | LR: 3.32e-04 | 3566.1ms/step | 5.6 samples/s +Step 17770/44000 | Loss: 3.0283 | LR: 3.32e-04 | 1329.7ms/step | 15.0 samples/s +Step 17780/44000 | Loss: 3.0332 | LR: 3.32e-04 | 1327.9ms/step | 15.1 samples/s +Step 17790/44000 | Loss: 3.0189 | LR: 3.32e-04 | 1326.5ms/step | 15.1 samples/s +Step 17800/44000 | Loss: 3.0435 | LR: 3.32e-04 | 1329.7ms/step | 15.0 samples/s +Step 17810/44000 | Loss: 3.0286 | LR: 3.31e-04 | 1328.9ms/step | 15.0 samples/s +Step 17820/44000 | Loss: 3.0307 | LR: 3.31e-04 | 1328.8ms/step | 15.1 samples/s +Step 17830/44000 | Loss: 3.0022 | LR: 3.31e-04 | 1326.9ms/step | 15.1 samples/s +Step 17840/44000 | Loss: 3.0353 | LR: 3.31e-04 | 1325.5ms/step | 15.1 samples/s +Step 17850/44000 | Loss: 3.0207 | LR: 3.31e-04 | 1328.7ms/step | 15.1 samples/s +Step 17860/44000 | Loss: 3.0416 | LR: 3.31e-04 | 1330.2ms/step | 15.0 samples/s +Step 17870/44000 | Loss: 3.0128 | LR: 3.30e-04 | 1329.9ms/step | 15.0 samples/s +Step 17880/44000 | Loss: 3.0232 | LR: 3.30e-04 | 1330.4ms/step | 15.0 samples/s +Step 17890/44000 | Loss: 3.0480 | LR: 3.30e-04 | 1333.0ms/step | 15.0 samples/s +Step 17900/44000 | Loss: 3.0491 | LR: 3.30e-04 | 1332.8ms/step | 15.0 samples/s +Step 17910/44000 | Loss: 2.9943 | LR: 3.30e-04 | 1331.8ms/step | 15.0 samples/s +Step 17920/44000 | Loss: 3.0012 | LR: 3.29e-04 | 1333.1ms/step | 15.0 samples/s +Step 17930/44000 | Loss: 2.9974 | LR: 3.29e-04 | 1334.1ms/step | 15.0 samples/s +Step 17940/44000 | Loss: 3.0466 | LR: 3.29e-04 | 1331.7ms/step | 15.0 samples/s +Step 17950/44000 | Loss: 3.0262 | LR: 3.29e-04 | 1333.4ms/step | 15.0 samples/s +Step 17960/44000 | Loss: 3.0233 | LR: 3.29e-04 | 1334.9ms/step | 15.0 samples/s +Step 17970/44000 | Loss: 3.0419 | LR: 3.29e-04 | 1334.6ms/step | 15.0 samples/s +Step 17980/44000 | Loss: 3.0225 | LR: 3.28e-04 | 1331.6ms/step | 15.0 samples/s +Step 17990/44000 | Loss: 3.0018 | LR: 3.28e-04 | 1332.2ms/step | 15.0 samples/s +Step 18000/44000 | Loss: 3.0317 | LR: 3.28e-04 | 1337.4ms/step | 15.0 samples/s +Validation at step 18000: Loss = 3.0110, Perplexity = 20.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-18000 +Step 18010/44000 | Loss: 2.9702 | LR: 3.28e-04 | 3862.7ms/step | 5.2 samples/s +Step 18020/44000 | Loss: 3.0176 | LR: 3.28e-04 | 1332.1ms/step | 15.0 samples/s +Step 18030/44000 | Loss: 3.0354 | LR: 3.28e-04 | 1333.2ms/step | 15.0 samples/s +Step 18040/44000 | Loss: 3.0298 | LR: 3.27e-04 | 1333.9ms/step | 15.0 samples/s +Step 18050/44000 | Loss: 3.0123 | LR: 3.27e-04 | 1333.0ms/step | 15.0 samples/s +Step 18060/44000 | Loss: 3.0118 | LR: 3.27e-04 | 1331.7ms/step | 15.0 samples/s +Step 18070/44000 | Loss: 3.0187 | LR: 3.27e-04 | 1330.4ms/step | 15.0 samples/s +Step 18080/44000 | Loss: 2.9876 | LR: 3.27e-04 | 1331.4ms/step | 15.0 samples/s +Step 18090/44000 | Loss: 3.0245 | LR: 3.27e-04 | 1331.8ms/step | 15.0 samples/s +Step 18100/44000 | Loss: 3.0236 | LR: 3.26e-04 | 1334.5ms/step | 15.0 samples/s +Step 18110/44000 | Loss: 3.0169 | LR: 3.26e-04 | 1333.4ms/step | 15.0 samples/s +Step 18120/44000 | Loss: 3.0471 | LR: 3.26e-04 | 1334.5ms/step | 15.0 samples/s +Step 18130/44000 | Loss: 3.0201 | LR: 3.26e-04 | 1333.2ms/step | 15.0 samples/s +Step 18140/44000 | Loss: 3.0006 | LR: 3.26e-04 | 1331.0ms/step | 15.0 samples/s +Step 18150/44000 | Loss: 3.0348 | LR: 3.26e-04 | 1333.0ms/step | 15.0 samples/s +Step 18160/44000 | Loss: 3.0331 | LR: 3.25e-04 | 1332.2ms/step | 15.0 samples/s +Step 18170/44000 | Loss: 3.0419 | LR: 3.25e-04 | 1335.4ms/step | 15.0 samples/s +Step 18180/44000 | Loss: 3.0298 | LR: 3.25e-04 | 1333.4ms/step | 15.0 samples/s +Step 18190/44000 | Loss: 2.9997 | LR: 3.25e-04 | 1333.8ms/step | 15.0 samples/s +Step 18200/44000 | Loss: 2.9887 | LR: 3.25e-04 | 1331.5ms/step | 15.0 samples/s +Step 18210/44000 | Loss: 3.0120 | LR: 3.24e-04 | 1335.1ms/step | 15.0 samples/s +Step 18220/44000 | Loss: 2.9879 | LR: 3.24e-04 | 1332.2ms/step | 15.0 samples/s +Step 18230/44000 | Loss: 3.0174 | LR: 3.24e-04 | 1331.5ms/step | 15.0 samples/s +Step 18240/44000 | Loss: 3.0295 | LR: 3.24e-04 | 1333.6ms/step | 15.0 samples/s +Step 18250/44000 | Loss: 3.0404 | LR: 3.24e-04 | 1333.9ms/step | 15.0 samples/s +Validation at step 18250: Loss = 3.0094, Perplexity = 20.28 +Step 18260/44000 | Loss: 3.0353 | LR: 3.24e-04 | 3570.9ms/step | 5.6 samples/s +Step 18270/44000 | Loss: 3.0403 | LR: 3.23e-04 | 1333.2ms/step | 15.0 samples/s +Step 18280/44000 | Loss: 3.0455 | LR: 3.23e-04 | 1333.4ms/step | 15.0 samples/s +Step 18290/44000 | Loss: 3.0055 | LR: 3.23e-04 | 1329.2ms/step | 15.0 samples/s +Step 18300/44000 | Loss: 3.0027 | LR: 3.23e-04 | 1326.4ms/step | 15.1 samples/s +Step 18310/44000 | Loss: 3.0136 | LR: 3.23e-04 | 1330.3ms/step | 15.0 samples/s +Step 18320/44000 | Loss: 3.0205 | LR: 3.23e-04 | 1325.5ms/step | 15.1 samples/s +Step 18330/44000 | Loss: 3.0181 | LR: 3.22e-04 | 1322.0ms/step | 15.1 samples/s +Step 18340/44000 | Loss: 3.0192 | LR: 3.22e-04 | 1328.1ms/step | 15.1 samples/s +Step 18350/44000 | Loss: 3.0388 | LR: 3.22e-04 | 1330.4ms/step | 15.0 samples/s +Step 18360/44000 | Loss: 3.0043 | LR: 3.22e-04 | 1327.3ms/step | 15.1 samples/s +Step 18370/44000 | Loss: 2.9968 | LR: 3.22e-04 | 1326.2ms/step | 15.1 samples/s +Step 18380/44000 | Loss: 3.0118 | LR: 3.22e-04 | 1329.3ms/step | 15.0 samples/s +Step 18390/44000 | Loss: 3.0140 | LR: 3.21e-04 | 1329.8ms/step | 15.0 samples/s +Step 18400/44000 | Loss: 3.0009 | LR: 3.21e-04 | 1331.5ms/step | 15.0 samples/s +Step 18410/44000 | Loss: 3.0389 | LR: 3.21e-04 | 1329.1ms/step | 15.0 samples/s +Step 18420/44000 | Loss: 3.0252 | LR: 3.21e-04 | 1329.4ms/step | 15.0 samples/s +Step 18430/44000 | Loss: 3.0033 | LR: 3.21e-04 | 1333.7ms/step | 15.0 samples/s +Step 18440/44000 | Loss: 3.0162 | LR: 3.20e-04 | 1335.1ms/step | 15.0 samples/s +Step 18450/44000 | Loss: 3.0264 | LR: 3.20e-04 | 1332.1ms/step | 15.0 samples/s +Step 18460/44000 | Loss: 2.9815 | LR: 3.20e-04 | 1327.1ms/step | 15.1 samples/s +Step 18470/44000 | Loss: 3.0310 | LR: 3.20e-04 | 1327.3ms/step | 15.1 samples/s +Step 18480/44000 | Loss: 3.0216 | LR: 3.20e-04 | 1328.9ms/step | 15.0 samples/s +Step 18490/44000 | Loss: 3.0049 | LR: 3.20e-04 | 1329.5ms/step | 15.0 samples/s +Step 18500/44000 | Loss: 3.0270 | LR: 3.19e-04 | 1331.8ms/step | 15.0 samples/s +Validation at step 18500: Loss = 3.0058, Perplexity = 20.20 +Step 18510/44000 | Loss: 3.0163 | LR: 3.19e-04 | 3559.8ms/step | 5.6 samples/s +Step 18520/44000 | Loss: 3.0311 | LR: 3.19e-04 | 1325.9ms/step | 15.1 samples/s +Step 18530/44000 | Loss: 3.0317 | LR: 3.19e-04 | 1328.1ms/step | 15.1 samples/s +Step 18540/44000 | Loss: 3.0182 | LR: 3.19e-04 | 1321.4ms/step | 15.1 samples/s +Step 18550/44000 | Loss: 2.9976 | LR: 3.19e-04 | 1322.4ms/step | 15.1 samples/s +Step 18560/44000 | Loss: 3.0253 | LR: 3.18e-04 | 1324.8ms/step | 15.1 samples/s +Step 18570/44000 | Loss: 3.0406 | LR: 3.18e-04 | 1326.2ms/step | 15.1 samples/s +Step 18580/44000 | Loss: 3.0201 | LR: 3.18e-04 | 1325.6ms/step | 15.1 samples/s +Step 18590/44000 | Loss: 3.0275 | LR: 3.18e-04 | 1325.9ms/step | 15.1 samples/s +Step 18600/44000 | Loss: 3.0312 | LR: 3.18e-04 | 1327.6ms/step | 15.1 samples/s +Step 18610/44000 | Loss: 3.0349 | LR: 3.18e-04 | 1327.4ms/step | 15.1 samples/s +Step 18620/44000 | Loss: 3.0168 | LR: 3.17e-04 | 1325.1ms/step | 15.1 samples/s +Step 18630/44000 | Loss: 3.0126 | LR: 3.17e-04 | 1327.9ms/step | 15.1 samples/s +Step 18640/44000 | Loss: 3.0002 | LR: 3.17e-04 | 1328.6ms/step | 15.1 samples/s +Step 18650/44000 | Loss: 3.0134 | LR: 3.17e-04 | 1328.8ms/step | 15.1 samples/s +Step 18660/44000 | Loss: 3.0359 | LR: 3.17e-04 | 1332.0ms/step | 15.0 samples/s +Step 18670/44000 | Loss: 3.0302 | LR: 3.16e-04 | 1330.4ms/step | 15.0 samples/s +Step 18680/44000 | Loss: 3.0364 | LR: 3.16e-04 | 1330.2ms/step | 15.0 samples/s +Step 18690/44000 | Loss: 3.0489 | LR: 3.16e-04 | 1328.1ms/step | 15.1 samples/s +Step 18700/44000 | Loss: 3.0409 | LR: 3.16e-04 | 1328.5ms/step | 15.1 samples/s +Step 18710/44000 | Loss: 2.9857 | LR: 3.16e-04 | 1329.6ms/step | 15.0 samples/s +Step 18720/44000 | Loss: 3.0006 | LR: 3.16e-04 | 1331.6ms/step | 15.0 samples/s +Step 18730/44000 | Loss: 2.9901 | LR: 3.15e-04 | 1330.3ms/step | 15.0 samples/s +Step 18740/44000 | Loss: 3.0064 | LR: 3.15e-04 | 1332.5ms/step | 15.0 samples/s +Step 18750/44000 | Loss: 2.9856 | LR: 3.15e-04 | 1330.9ms/step | 15.0 samples/s +Validation at step 18750: Loss = 3.0029, Perplexity = 20.14 +Step 18760/44000 | Loss: 2.9970 | LR: 3.15e-04 | 3561.0ms/step | 5.6 samples/s +Step 18770/44000 | Loss: 3.0059 | LR: 3.15e-04 | 1328.9ms/step | 15.0 samples/s +Step 18780/44000 | Loss: 3.0250 | LR: 3.15e-04 | 1329.7ms/step | 15.0 samples/s +Step 18790/44000 | Loss: 3.0046 | LR: 3.14e-04 | 1332.4ms/step | 15.0 samples/s +Step 18800/44000 | Loss: 3.0314 | LR: 3.14e-04 | 1330.7ms/step | 15.0 samples/s +Step 18810/44000 | Loss: 2.9975 | LR: 3.14e-04 | 1332.1ms/step | 15.0 samples/s +Step 18820/44000 | Loss: 3.0036 | LR: 3.14e-04 | 1329.8ms/step | 15.0 samples/s +Step 18830/44000 | Loss: 3.0252 | LR: 3.14e-04 | 1329.6ms/step | 15.0 samples/s +Step 18840/44000 | Loss: 3.0181 | LR: 3.13e-04 | 1330.4ms/step | 15.0 samples/s +Step 18850/44000 | Loss: 3.0169 | LR: 3.13e-04 | 1329.8ms/step | 15.0 samples/s +Step 18860/44000 | Loss: 3.0035 | LR: 3.13e-04 | 1329.0ms/step | 15.0 samples/s +Step 18870/44000 | Loss: 3.0099 | LR: 3.13e-04 | 1328.0ms/step | 15.1 samples/s +Step 18880/44000 | Loss: 3.0033 | LR: 3.13e-04 | 1333.7ms/step | 15.0 samples/s +Step 18890/44000 | Loss: 3.0189 | LR: 3.13e-04 | 1332.1ms/step | 15.0 samples/s +Step 18900/44000 | Loss: 3.0306 | LR: 3.12e-04 | 1331.4ms/step | 15.0 samples/s +Step 18910/44000 | Loss: 3.0029 | LR: 3.12e-04 | 1329.4ms/step | 15.0 samples/s +Step 18920/44000 | Loss: 3.0157 | LR: 3.12e-04 | 1330.2ms/step | 15.0 samples/s +Step 18930/44000 | Loss: 2.9996 | LR: 3.12e-04 | 1330.5ms/step | 15.0 samples/s +Step 18940/44000 | Loss: 3.0141 | LR: 3.12e-04 | 1334.5ms/step | 15.0 samples/s +Step 18950/44000 | Loss: 3.0268 | LR: 3.12e-04 | 1332.5ms/step | 15.0 samples/s +Step 18960/44000 | Loss: 3.0155 | LR: 3.11e-04 | 1332.2ms/step | 15.0 samples/s +Step 18970/44000 | Loss: 2.9985 | LR: 3.11e-04 | 1332.9ms/step | 15.0 samples/s +Step 18980/44000 | Loss: 3.0048 | LR: 3.11e-04 | 1332.2ms/step | 15.0 samples/s +Step 18990/44000 | Loss: 3.0028 | LR: 3.11e-04 | 1333.8ms/step | 15.0 samples/s +Step 19000/44000 | Loss: 3.0108 | LR: 3.11e-04 | 1333.0ms/step | 15.0 samples/s +Validation at step 19000: Loss = 2.9998, Perplexity = 20.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-19000 +Step 19010/44000 | Loss: 3.0232 | LR: 3.11e-04 | 3876.8ms/step | 5.2 samples/s +Step 19020/44000 | Loss: 3.0014 | LR: 3.10e-04 | 1333.2ms/step | 15.0 samples/s +Step 19030/44000 | Loss: 2.9938 | LR: 3.10e-04 | 1329.6ms/step | 15.0 samples/s +Step 19040/44000 | Loss: 3.0055 | LR: 3.10e-04 | 1329.7ms/step | 15.0 samples/s +Step 19050/44000 | Loss: 3.0117 | LR: 3.10e-04 | 1331.7ms/step | 15.0 samples/s +Step 19060/44000 | Loss: 3.0172 | LR: 3.10e-04 | 1330.6ms/step | 15.0 samples/s +Step 19070/44000 | Loss: 3.0037 | LR: 3.09e-04 | 1329.2ms/step | 15.0 samples/s +Step 19080/44000 | Loss: 3.0055 | LR: 3.09e-04 | 1331.6ms/step | 15.0 samples/s +Step 19090/44000 | Loss: 3.0157 | LR: 3.09e-04 | 1336.0ms/step | 15.0 samples/s +Step 19100/44000 | Loss: 2.9952 | LR: 3.09e-04 | 1331.1ms/step | 15.0 samples/s +Step 19110/44000 | Loss: 3.0447 | LR: 3.09e-04 | 1332.4ms/step | 15.0 samples/s +Step 19120/44000 | Loss: 2.9917 | LR: 3.09e-04 | 1332.5ms/step | 15.0 samples/s +Step 19130/44000 | Loss: 3.0145 | LR: 3.08e-04 | 1332.6ms/step | 15.0 samples/s +Step 19140/44000 | Loss: 3.0144 | LR: 3.08e-04 | 1329.0ms/step | 15.0 samples/s +Step 19150/44000 | Loss: 3.0117 | LR: 3.08e-04 | 1329.1ms/step | 15.0 samples/s +Step 19160/44000 | Loss: 2.9896 | LR: 3.08e-04 | 1333.2ms/step | 15.0 samples/s +Step 19170/44000 | Loss: 3.0416 | LR: 3.08e-04 | 1336.9ms/step | 15.0 samples/s +Step 19180/44000 | Loss: 3.0257 | LR: 3.08e-04 | 1333.2ms/step | 15.0 samples/s +Step 19190/44000 | Loss: 3.0518 | LR: 3.07e-04 | 1331.1ms/step | 15.0 samples/s +Step 19200/44000 | Loss: 3.0198 | LR: 3.07e-04 | 1337.2ms/step | 15.0 samples/s +Step 19210/44000 | Loss: 3.0278 | LR: 3.07e-04 | 1333.6ms/step | 15.0 samples/s +Step 19220/44000 | Loss: 2.9668 | LR: 3.07e-04 | 1335.2ms/step | 15.0 samples/s +Step 19230/44000 | Loss: 3.0498 | LR: 3.07e-04 | 1331.4ms/step | 15.0 samples/s +Step 19240/44000 | Loss: 3.0129 | LR: 3.06e-04 | 1333.3ms/step | 15.0 samples/s +Step 19250/44000 | Loss: 3.0215 | LR: 3.06e-04 | 1334.5ms/step | 15.0 samples/s +Validation at step 19250: Loss = 2.9973, Perplexity = 20.03 +Step 19260/44000 | Loss: 3.0213 | LR: 3.06e-04 | 3570.1ms/step | 5.6 samples/s +Step 19270/44000 | Loss: 3.0107 | LR: 3.06e-04 | 1332.0ms/step | 15.0 samples/s +Step 19280/44000 | Loss: 2.9995 | LR: 3.06e-04 | 1334.9ms/step | 15.0 samples/s +Step 19290/44000 | Loss: 3.0329 | LR: 3.06e-04 | 1332.2ms/step | 15.0 samples/s +Step 19300/44000 | Loss: 2.9577 | LR: 3.05e-04 | 1329.6ms/step | 15.0 samples/s +Step 19310/44000 | Loss: 3.0310 | LR: 3.05e-04 | 1326.2ms/step | 15.1 samples/s +Step 19320/44000 | Loss: 3.0141 | LR: 3.05e-04 | 1329.0ms/step | 15.0 samples/s +Step 19330/44000 | Loss: 3.0026 | LR: 3.05e-04 | 1330.0ms/step | 15.0 samples/s +Step 19340/44000 | Loss: 3.0109 | LR: 3.05e-04 | 1329.4ms/step | 15.0 samples/s +Step 19350/44000 | Loss: 2.9981 | LR: 3.04e-04 | 1328.9ms/step | 15.1 samples/s +Step 19360/44000 | Loss: 2.9942 | LR: 3.04e-04 | 1330.7ms/step | 15.0 samples/s +Step 19370/44000 | Loss: 3.0131 | LR: 3.04e-04 | 1331.5ms/step | 15.0 samples/s +Step 19380/44000 | Loss: 2.9891 | LR: 3.04e-04 | 1329.8ms/step | 15.0 samples/s +Step 19390/44000 | Loss: 2.9860 | LR: 3.04e-04 | 1325.1ms/step | 15.1 samples/s +Step 19400/44000 | Loss: 3.0260 | LR: 3.04e-04 | 1325.7ms/step | 15.1 samples/s +Step 19410/44000 | Loss: 3.0090 | LR: 3.03e-04 | 1325.1ms/step | 15.1 samples/s +Step 19420/44000 | Loss: 2.9815 | LR: 3.03e-04 | 1328.5ms/step | 15.1 samples/s +Step 19430/44000 | Loss: 2.9914 | LR: 3.03e-04 | 1325.4ms/step | 15.1 samples/s +Step 19440/44000 | Loss: 2.9836 | LR: 3.03e-04 | 1327.2ms/step | 15.1 samples/s +Step 19450/44000 | Loss: 3.0242 | LR: 3.03e-04 | 1326.1ms/step | 15.1 samples/s +Step 19460/44000 | Loss: 3.0097 | LR: 3.03e-04 | 1326.2ms/step | 15.1 samples/s +Step 19470/44000 | Loss: 3.0078 | LR: 3.02e-04 | 1327.9ms/step | 15.1 samples/s +Step 19480/44000 | Loss: 3.0180 | LR: 3.02e-04 | 1328.1ms/step | 15.1 samples/s +Step 19490/44000 | Loss: 3.0252 | LR: 3.02e-04 | 1325.7ms/step | 15.1 samples/s +Step 19500/44000 | Loss: 2.9891 | LR: 3.02e-04 | 1324.6ms/step | 15.1 samples/s +Validation at step 19500: Loss = 2.9945, Perplexity = 19.97 +Step 19510/44000 | Loss: 3.0391 | LR: 3.02e-04 | 3549.4ms/step | 5.6 samples/s +Step 19520/44000 | Loss: 3.0351 | LR: 3.01e-04 | 1328.0ms/step | 15.1 samples/s +Step 19530/44000 | Loss: 2.9943 | LR: 3.01e-04 | 1326.6ms/step | 15.1 samples/s +Step 19540/44000 | Loss: 2.9850 | LR: 3.01e-04 | 1327.2ms/step | 15.1 samples/s +Step 19550/44000 | Loss: 3.0213 | LR: 3.01e-04 | 1328.6ms/step | 15.1 samples/s +Step 19560/44000 | Loss: 3.0219 | LR: 3.01e-04 | 1330.8ms/step | 15.0 samples/s +Step 19570/44000 | Loss: 3.0233 | LR: 3.01e-04 | 1328.0ms/step | 15.1 samples/s +Step 19580/44000 | Loss: 3.0147 | LR: 3.00e-04 | 1326.8ms/step | 15.1 samples/s +Step 19590/44000 | Loss: 2.9990 | LR: 3.00e-04 | 1328.3ms/step | 15.1 samples/s +Step 19600/44000 | Loss: 3.0306 | LR: 3.00e-04 | 1328.5ms/step | 15.1 samples/s +Step 19610/44000 | Loss: 2.9999 | LR: 3.00e-04 | 1326.3ms/step | 15.1 samples/s +Step 19620/44000 | Loss: 3.0150 | LR: 3.00e-04 | 1328.1ms/step | 15.1 samples/s +Step 19630/44000 | Loss: 2.9856 | LR: 3.00e-04 | 1326.7ms/step | 15.1 samples/s +Step 19640/44000 | Loss: 2.9843 | LR: 2.99e-04 | 1325.5ms/step | 15.1 samples/s +Step 19650/44000 | Loss: 3.0050 | LR: 2.99e-04 | 1327.3ms/step | 15.1 samples/s +Step 19660/44000 | Loss: 3.0197 | LR: 2.99e-04 | 1328.8ms/step | 15.1 samples/s +Step 19670/44000 | Loss: 3.0364 | LR: 2.99e-04 | 1333.0ms/step | 15.0 samples/s +Step 19680/44000 | Loss: 2.9720 | LR: 2.99e-04 | 1328.9ms/step | 15.0 samples/s +Step 19690/44000 | Loss: 3.0428 | LR: 2.98e-04 | 1329.5ms/step | 15.0 samples/s +Step 19700/44000 | Loss: 2.9968 | LR: 2.98e-04 | 1326.2ms/step | 15.1 samples/s +Step 19710/44000 | Loss: 3.0110 | LR: 2.98e-04 | 1326.1ms/step | 15.1 samples/s +Step 19720/44000 | Loss: 2.9929 | LR: 2.98e-04 | 1328.9ms/step | 15.1 samples/s +Step 19730/44000 | Loss: 3.0045 | LR: 2.98e-04 | 1327.7ms/step | 15.1 samples/s +Step 19740/44000 | Loss: 3.0361 | LR: 2.98e-04 | 1326.6ms/step | 15.1 samples/s +Step 19750/44000 | Loss: 2.9966 | LR: 2.97e-04 | 1330.2ms/step | 15.0 samples/s +Validation at step 19750: Loss = 2.9928, Perplexity = 19.94 +Step 19760/44000 | Loss: 3.0525 | LR: 2.97e-04 | 3555.9ms/step | 5.6 samples/s +Step 19770/44000 | Loss: 3.0386 | LR: 2.97e-04 | 1328.9ms/step | 15.1 samples/s +Step 19780/44000 | Loss: 3.0012 | LR: 2.97e-04 | 1330.0ms/step | 15.0 samples/s +Step 19790/44000 | Loss: 3.0016 | LR: 2.97e-04 | 1329.6ms/step | 15.0 samples/s +Step 19800/44000 | Loss: 3.0365 | LR: 2.96e-04 | 1330.6ms/step | 15.0 samples/s +Step 19810/44000 | Loss: 3.0148 | LR: 2.96e-04 | 1329.7ms/step | 15.0 samples/s +Step 19820/44000 | Loss: 2.9848 | LR: 2.96e-04 | 1327.7ms/step | 15.1 samples/s +Step 19830/44000 | Loss: 3.0130 | LR: 2.96e-04 | 1332.2ms/step | 15.0 samples/s +Step 19840/44000 | Loss: 2.9908 | LR: 2.96e-04 | 1332.2ms/step | 15.0 samples/s +Step 19850/44000 | Loss: 3.0042 | LR: 2.96e-04 | 1329.9ms/step | 15.0 samples/s +Step 19860/44000 | Loss: 2.9993 | LR: 2.95e-04 | 1331.7ms/step | 15.0 samples/s +Step 19870/44000 | Loss: 3.0123 | LR: 2.95e-04 | 1329.4ms/step | 15.0 samples/s +Step 19880/44000 | Loss: 3.0032 | LR: 2.95e-04 | 1330.2ms/step | 15.0 samples/s +Step 19890/44000 | Loss: 3.0046 | LR: 2.95e-04 | 1333.4ms/step | 15.0 samples/s +Step 19900/44000 | Loss: 3.0027 | LR: 2.95e-04 | 1331.3ms/step | 15.0 samples/s +Step 19910/44000 | Loss: 2.9926 | LR: 2.95e-04 | 1333.6ms/step | 15.0 samples/s +Step 19920/44000 | Loss: 3.0014 | LR: 2.94e-04 | 1332.4ms/step | 15.0 samples/s +Step 19930/44000 | Loss: 3.0057 | LR: 2.94e-04 | 1333.6ms/step | 15.0 samples/s +Step 19940/44000 | Loss: 2.9971 | LR: 2.94e-04 | 1333.1ms/step | 15.0 samples/s +Step 19950/44000 | Loss: 3.0143 | LR: 2.94e-04 | 1333.7ms/step | 15.0 samples/s +Step 19960/44000 | Loss: 3.0089 | LR: 2.94e-04 | 1333.4ms/step | 15.0 samples/s +Step 19970/44000 | Loss: 3.0103 | LR: 2.93e-04 | 1333.6ms/step | 15.0 samples/s +Step 19980/44000 | Loss: 3.0245 | LR: 2.93e-04 | 1333.5ms/step | 15.0 samples/s +Step 19990/44000 | Loss: 3.0016 | LR: 2.93e-04 | 1334.0ms/step | 15.0 samples/s +Step 20000/44000 | Loss: 2.9704 | LR: 2.93e-04 | 1334.3ms/step | 15.0 samples/s +Validation at step 20000: Loss = 2.9897, Perplexity = 19.88 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-20000 +Step 20010/44000 | Loss: 2.9981 | LR: 2.93e-04 | 3875.8ms/step | 5.2 samples/s +Step 20020/44000 | Loss: 2.9755 | LR: 2.93e-04 | 1334.6ms/step | 15.0 samples/s +Step 20030/44000 | Loss: 3.0116 | LR: 2.92e-04 | 1332.8ms/step | 15.0 samples/s +Step 20040/44000 | Loss: 3.0108 | LR: 2.92e-04 | 1331.0ms/step | 15.0 samples/s +Step 20050/44000 | Loss: 3.0145 | LR: 2.92e-04 | 1335.2ms/step | 15.0 samples/s +Step 20060/44000 | Loss: 2.9973 | LR: 2.92e-04 | 1332.6ms/step | 15.0 samples/s +Step 20070/44000 | Loss: 3.0009 | LR: 2.92e-04 | 1333.1ms/step | 15.0 samples/s +Step 20080/44000 | Loss: 3.0227 | LR: 2.91e-04 | 1332.4ms/step | 15.0 samples/s +Step 20090/44000 | Loss: 2.9951 | LR: 2.91e-04 | 1333.3ms/step | 15.0 samples/s diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed42_43/supervisor.log b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/supervisor.log new file mode 100644 index 0000000000000000000000000000000000000000..e98fa869cd01309a8520dab3f3415121b3855207 --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed42_43/supervisor.log @@ -0,0 +1,9 @@ +[supervisor] cache build started 2026-08-21T15:27:26Z +validated static: 28 layers, pool=1 +validated resampled: 28 layers, pool=10000 +[supervisor] caches validated; launching training 2026-08-21T16:44:07Z +[supervisor] launched gpu=0 pid=14850 name=l8k365m_static_nrb35_seed42 +[supervisor] launched gpu=1 pid=14851 name=l8k365m_resampled_nrb35_seed42 +[supervisor] launched gpu=2 pid=14852 name=l8k365m_static_nrb35_seed43 +[supervisor] launched gpu=3 pid=14853 name=l8k365m_resampled_nrb35_seed43 +[supervisor] handoff complete 2026-08-21T16:44:07Z diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed44_47/l8k365m_resampled_nrb35_seed44.log b/epfl_final/train_logs/p1_controls_nrb35_seed44_47/l8k365m_resampled_nrb35_seed44.log new file mode 100644 index 0000000000000000000000000000000000000000..5a563e49c040f3801703b02c56aa7259cefd0493 --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed44_47/l8k365m_resampled_nrb35_seed44.log @@ -0,0 +1,2925 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 44, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 365m resampled 8k nrb35 seed44", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "resampled", + "nrb35", + "seed44" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 27rw10jv +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_181107-27rw10jv +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 365m resampled 8k nrb35 seed44 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/27rw10jv +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0405 | LR: 6.67e-06 | 2509.6ms/step | 8.0 samples/s +Step 20/44000 | Loss: 11.9508 | LR: 1.33e-05 | 1323.6ms/step | 15.1 samples/s +Step 30/44000 | Loss: 11.6160 | LR: 2.00e-05 | 1326.5ms/step | 15.1 samples/s +Step 40/44000 | Loss: 11.0832 | LR: 2.67e-05 | 1325.9ms/step | 15.1 samples/s +Step 50/44000 | Loss: 10.7633 | LR: 3.33e-05 | 1322.9ms/step | 15.1 samples/s +Step 60/44000 | Loss: 10.5122 | LR: 4.00e-05 | 1319.1ms/step | 15.2 samples/s +Step 70/44000 | Loss: 10.2863 | LR: 4.67e-05 | 1322.3ms/step | 15.1 samples/s +Step 80/44000 | Loss: 9.9819 | LR: 5.33e-05 | 1325.8ms/step | 15.1 samples/s +Step 90/44000 | Loss: 9.5976 | LR: 6.00e-05 | 1326.5ms/step | 15.1 samples/s +Step 100/44000 | Loss: 9.1258 | LR: 6.67e-05 | 1323.7ms/step | 15.1 samples/s +Step 110/44000 | Loss: 8.6065 | LR: 7.33e-05 | 1323.9ms/step | 15.1 samples/s +Step 120/44000 | Loss: 8.1207 | LR: 8.00e-05 | 1326.6ms/step | 15.1 samples/s +Step 130/44000 | Loss: 7.7521 | LR: 8.67e-05 | 1324.7ms/step | 15.1 samples/s +Step 140/44000 | Loss: 7.5317 | LR: 9.33e-05 | 1327.1ms/step | 15.1 samples/s +Step 150/44000 | Loss: 7.3208 | LR: 1.00e-04 | 1326.7ms/step | 15.1 samples/s +Step 160/44000 | Loss: 7.2174 | LR: 1.07e-04 | 1328.7ms/step | 15.1 samples/s +Step 170/44000 | Loss: 7.0917 | LR: 1.13e-04 | 1327.3ms/step | 15.1 samples/s +Step 180/44000 | Loss: 7.0122 | LR: 1.20e-04 | 1329.0ms/step | 15.0 samples/s +Step 190/44000 | Loss: 6.9068 | LR: 1.27e-04 | 1327.6ms/step | 15.1 samples/s +Step 200/44000 | Loss: 6.8468 | LR: 1.33e-04 | 1326.7ms/step | 15.1 samples/s +Step 210/44000 | Loss: 6.7568 | LR: 1.40e-04 | 1328.9ms/step | 15.0 samples/s +Step 220/44000 | Loss: 6.6751 | LR: 1.47e-04 | 1327.1ms/step | 15.1 samples/s +Step 230/44000 | Loss: 6.5998 | LR: 1.53e-04 | 1328.6ms/step | 15.1 samples/s +Step 240/44000 | Loss: 6.4917 | LR: 1.60e-04 | 1325.5ms/step | 15.1 samples/s +Step 250/44000 | Loss: 6.4525 | LR: 1.67e-04 | 1329.7ms/step | 15.0 samples/s +Validation at step 250: Loss = 6.4388, Perplexity = 625.65 +Step 260/44000 | Loss: 6.3928 | LR: 1.73e-04 | 4376.5ms/step | 4.6 samples/s +Step 270/44000 | Loss: 6.3395 | LR: 1.80e-04 | 1336.1ms/step | 15.0 samples/s +Step 280/44000 | Loss: 6.2819 | LR: 1.87e-04 | 1330.8ms/step | 15.0 samples/s +Step 290/44000 | Loss: 6.2710 | LR: 1.93e-04 | 1333.4ms/step | 15.0 samples/s +Step 300/44000 | Loss: 6.2249 | LR: 2.00e-04 | 1334.2ms/step | 15.0 samples/s +Step 310/44000 | Loss: 6.1953 | LR: 2.07e-04 | 1332.1ms/step | 15.0 samples/s +Step 320/44000 | Loss: 6.1335 | LR: 2.13e-04 | 1337.7ms/step | 15.0 samples/s +Step 330/44000 | Loss: 6.1101 | LR: 2.20e-04 | 1336.6ms/step | 15.0 samples/s +Step 340/44000 | Loss: 6.0380 | LR: 2.27e-04 | 1335.2ms/step | 15.0 samples/s +Step 350/44000 | Loss: 6.0148 | LR: 2.33e-04 | 1340.0ms/step | 14.9 samples/s +Step 360/44000 | Loss: 5.9499 | LR: 2.40e-04 | 1336.4ms/step | 15.0 samples/s +Step 370/44000 | Loss: 5.9111 | LR: 2.47e-04 | 1338.0ms/step | 14.9 samples/s +Step 380/44000 | Loss: 5.8800 | LR: 2.53e-04 | 1339.1ms/step | 14.9 samples/s +Step 390/44000 | Loss: 5.8360 | LR: 2.60e-04 | 1333.4ms/step | 15.0 samples/s +Step 400/44000 | Loss: 5.8359 | LR: 2.67e-04 | 1336.4ms/step | 15.0 samples/s +Step 410/44000 | Loss: 5.7996 | LR: 2.73e-04 | 1334.9ms/step | 15.0 samples/s +Step 420/44000 | Loss: 5.7568 | LR: 2.80e-04 | 1335.8ms/step | 15.0 samples/s +Step 430/44000 | Loss: 5.7574 | LR: 2.87e-04 | 1333.1ms/step | 15.0 samples/s +Step 440/44000 | Loss: 5.7105 | LR: 2.93e-04 | 1336.9ms/step | 15.0 samples/s +Step 450/44000 | Loss: 5.6391 | LR: 3.00e-04 | 1336.7ms/step | 15.0 samples/s +Step 460/44000 | Loss: 5.6230 | LR: 3.07e-04 | 1340.0ms/step | 14.9 samples/s +Step 470/44000 | Loss: 5.5806 | LR: 3.13e-04 | 1336.2ms/step | 15.0 samples/s +Step 480/44000 | Loss: 5.5230 | LR: 3.20e-04 | 1338.9ms/step | 14.9 samples/s +Step 490/44000 | Loss: 5.5234 | LR: 3.27e-04 | 1329.8ms/step | 15.0 samples/s +Step 500/44000 | Loss: 5.4779 | LR: 3.33e-04 | 1332.7ms/step | 15.0 samples/s +Validation at step 500: Loss = 5.4691, Perplexity = 237.24 +Step 510/44000 | Loss: 5.4465 | LR: 3.40e-04 | 3576.9ms/step | 5.6 samples/s +Step 520/44000 | Loss: 5.4307 | LR: 3.47e-04 | 1330.3ms/step | 15.0 samples/s +Step 530/44000 | Loss: 5.3796 | LR: 3.53e-04 | 1332.9ms/step | 15.0 samples/s +Step 540/44000 | Loss: 5.3553 | LR: 3.60e-04 | 1332.8ms/step | 15.0 samples/s +Step 550/44000 | Loss: 5.3472 | LR: 3.67e-04 | 1336.0ms/step | 15.0 samples/s +Step 560/44000 | Loss: 5.3021 | LR: 3.73e-04 | 1336.9ms/step | 15.0 samples/s +Step 570/44000 | Loss: 5.2552 | LR: 3.80e-04 | 1334.2ms/step | 15.0 samples/s +Step 580/44000 | Loss: 5.2073 | LR: 3.87e-04 | 1337.7ms/step | 15.0 samples/s +Step 590/44000 | Loss: 5.2248 | LR: 3.93e-04 | 1338.2ms/step | 14.9 samples/s +Step 600/44000 | Loss: 5.1777 | LR: 4.00e-04 | 1335.1ms/step | 15.0 samples/s +Step 610/44000 | Loss: 5.1713 | LR: 4.07e-04 | 1330.3ms/step | 15.0 samples/s +Step 620/44000 | Loss: 5.1088 | LR: 4.13e-04 | 1334.1ms/step | 15.0 samples/s +Step 630/44000 | Loss: 5.0833 | LR: 4.20e-04 | 1329.8ms/step | 15.0 samples/s +Step 640/44000 | Loss: 5.0715 | LR: 4.27e-04 | 1334.0ms/step | 15.0 samples/s +Step 650/44000 | Loss: 5.0221 | LR: 4.33e-04 | 1334.3ms/step | 15.0 samples/s +Step 660/44000 | Loss: 4.9696 | LR: 4.40e-04 | 1332.5ms/step | 15.0 samples/s +Step 670/44000 | Loss: 4.9677 | LR: 4.47e-04 | 1333.1ms/step | 15.0 samples/s +Step 680/44000 | Loss: 4.9592 | LR: 4.53e-04 | 1332.0ms/step | 15.0 samples/s +Step 690/44000 | Loss: 4.9109 | LR: 4.60e-04 | 1327.9ms/step | 15.1 samples/s +Step 700/44000 | Loss: 4.8931 | LR: 4.67e-04 | 1332.5ms/step | 15.0 samples/s +Step 710/44000 | Loss: 4.8477 | LR: 4.73e-04 | 1333.6ms/step | 15.0 samples/s +Step 720/44000 | Loss: 4.8535 | LR: 4.80e-04 | 1335.7ms/step | 15.0 samples/s +Step 730/44000 | Loss: 4.8360 | LR: 4.87e-04 | 1332.0ms/step | 15.0 samples/s +Step 740/44000 | Loss: 4.7841 | LR: 4.93e-04 | 1330.7ms/step | 15.0 samples/s +Step 750/44000 | Loss: 4.6985 | LR: 5.00e-04 | 1328.2ms/step | 15.1 samples/s +Validation at step 750: Loss = 4.7494, Perplexity = 115.52 +Step 760/44000 | Loss: 4.7515 | LR: 5.00e-04 | 3577.0ms/step | 5.6 samples/s +Step 770/44000 | Loss: 4.7300 | LR: 5.00e-04 | 1333.7ms/step | 15.0 samples/s +Step 780/44000 | Loss: 4.7058 | LR: 5.00e-04 | 1333.1ms/step | 15.0 samples/s +Step 790/44000 | Loss: 4.6928 | LR: 5.00e-04 | 1331.8ms/step | 15.0 samples/s +Step 800/44000 | Loss: 4.6592 | LR: 5.00e-04 | 1329.0ms/step | 15.0 samples/s +Step 810/44000 | Loss: 4.6218 | LR: 5.00e-04 | 1332.8ms/step | 15.0 samples/s +Step 820/44000 | Loss: 4.6185 | LR: 5.00e-04 | 1332.6ms/step | 15.0 samples/s +Step 830/44000 | Loss: 4.5959 | LR: 5.00e-04 | 1330.7ms/step | 15.0 samples/s +Step 840/44000 | Loss: 4.5416 | LR: 5.00e-04 | 1332.5ms/step | 15.0 samples/s +Step 850/44000 | Loss: 4.5333 | LR: 5.00e-04 | 1335.3ms/step | 15.0 samples/s +Step 860/44000 | Loss: 4.5455 | LR: 5.00e-04 | 1326.6ms/step | 15.1 samples/s +Step 870/44000 | Loss: 4.4883 | LR: 5.00e-04 | 1330.5ms/step | 15.0 samples/s +Step 880/44000 | Loss: 4.4909 | LR: 5.00e-04 | 1331.1ms/step | 15.0 samples/s +Step 890/44000 | Loss: 4.4780 | LR: 5.00e-04 | 1331.1ms/step | 15.0 samples/s +Step 900/44000 | Loss: 4.4495 | LR: 5.00e-04 | 1334.8ms/step | 15.0 samples/s +Step 910/44000 | Loss: 4.4299 | LR: 5.00e-04 | 1340.0ms/step | 14.9 samples/s +Step 920/44000 | Loss: 4.4120 | LR: 5.00e-04 | 1332.2ms/step | 15.0 samples/s +Step 930/44000 | Loss: 4.4068 | LR: 5.00e-04 | 1335.7ms/step | 15.0 samples/s +Step 940/44000 | Loss: 4.4037 | LR: 5.00e-04 | 1334.5ms/step | 15.0 samples/s +Step 950/44000 | Loss: 4.3591 | LR: 5.00e-04 | 1333.4ms/step | 15.0 samples/s +Step 960/44000 | Loss: 4.3968 | LR: 5.00e-04 | 1337.9ms/step | 14.9 samples/s +Step 970/44000 | Loss: 4.3581 | LR: 5.00e-04 | 1330.7ms/step | 15.0 samples/s +Step 980/44000 | Loss: 4.3289 | LR: 5.00e-04 | 1330.6ms/step | 15.0 samples/s +Step 990/44000 | Loss: 4.3317 | LR: 5.00e-04 | 1330.3ms/step | 15.0 samples/s +Step 1000/44000 | Loss: 4.3057 | LR: 5.00e-04 | 1332.9ms/step | 15.0 samples/s +Validation at step 1000: Loss = 4.3092, Perplexity = 74.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-1000 +Step 1010/44000 | Loss: 4.3047 | LR: 5.00e-04 | 3867.7ms/step | 5.2 samples/s +Step 1020/44000 | Loss: 4.2629 | LR: 5.00e-04 | 1335.9ms/step | 15.0 samples/s +Step 1030/44000 | Loss: 4.2780 | LR: 5.00e-04 | 1330.7ms/step | 15.0 samples/s +Step 1040/44000 | Loss: 4.2665 | LR: 5.00e-04 | 1324.8ms/step | 15.1 samples/s +Step 1050/44000 | Loss: 4.2369 | LR: 5.00e-04 | 1332.5ms/step | 15.0 samples/s +Step 1060/44000 | Loss: 4.2269 | LR: 5.00e-04 | 1330.2ms/step | 15.0 samples/s +Step 1070/44000 | Loss: 4.2233 | LR: 5.00e-04 | 1334.2ms/step | 15.0 samples/s +Step 1080/44000 | Loss: 4.1911 | LR: 5.00e-04 | 1325.7ms/step | 15.1 samples/s +Step 1090/44000 | Loss: 4.1968 | LR: 5.00e-04 | 1328.8ms/step | 15.1 samples/s +Step 1100/44000 | Loss: 4.1816 | LR: 5.00e-04 | 1324.4ms/step | 15.1 samples/s +Step 1110/44000 | Loss: 4.1581 | LR: 5.00e-04 | 1321.8ms/step | 15.1 samples/s +Step 1120/44000 | Loss: 4.1651 | LR: 5.00e-04 | 1342.9ms/step | 14.9 samples/s +Step 1130/44000 | Loss: 4.2060 | LR: 5.00e-04 | 1331.1ms/step | 15.0 samples/s +Step 1140/44000 | Loss: 4.1393 | LR: 5.00e-04 | 1328.0ms/step | 15.1 samples/s +Step 1150/44000 | Loss: 4.1152 | LR: 5.00e-04 | 1324.7ms/step | 15.1 samples/s +Step 1160/44000 | Loss: 4.1457 | LR: 5.00e-04 | 1328.1ms/step | 15.1 samples/s +Step 1170/44000 | Loss: 4.1163 | LR: 5.00e-04 | 1323.2ms/step | 15.1 samples/s +Step 1180/44000 | Loss: 4.1589 | LR: 5.00e-04 | 1319.6ms/step | 15.2 samples/s +Step 1190/44000 | Loss: 4.1303 | LR: 5.00e-04 | 1319.5ms/step | 15.2 samples/s +Step 1200/44000 | Loss: 4.0925 | LR: 5.00e-04 | 1318.3ms/step | 15.2 samples/s +Step 1210/44000 | Loss: 4.0870 | LR: 5.00e-04 | 1321.4ms/step | 15.1 samples/s +Step 1220/44000 | Loss: 4.0782 | LR: 5.00e-04 | 1323.6ms/step | 15.1 samples/s +Step 1230/44000 | Loss: 4.0826 | LR: 5.00e-04 | 1325.4ms/step | 15.1 samples/s +Step 1240/44000 | Loss: 4.0609 | LR: 5.00e-04 | 1325.5ms/step | 15.1 samples/s +Step 1250/44000 | Loss: 4.0516 | LR: 5.00e-04 | 1322.2ms/step | 15.1 samples/s +Validation at step 1250: Loss = 4.0694, Perplexity = 58.52 +Step 1260/44000 | Loss: 4.0628 | LR: 5.00e-04 | 3547.4ms/step | 5.6 samples/s +Step 1270/44000 | Loss: 4.0417 | LR: 5.00e-04 | 1321.6ms/step | 15.1 samples/s +Step 1280/44000 | Loss: 4.0679 | LR: 5.00e-04 | 1318.4ms/step | 15.2 samples/s +Step 1290/44000 | Loss: 4.0519 | LR: 5.00e-04 | 1320.6ms/step | 15.1 samples/s +Step 1300/44000 | Loss: 4.0364 | LR: 5.00e-04 | 1326.2ms/step | 15.1 samples/s +Step 1310/44000 | Loss: 4.0425 | LR: 5.00e-04 | 1324.1ms/step | 15.1 samples/s +Step 1320/44000 | Loss: 4.0017 | LR: 5.00e-04 | 1321.4ms/step | 15.1 samples/s +Step 1330/44000 | Loss: 4.0390 | LR: 5.00e-04 | 1319.2ms/step | 15.2 samples/s +Step 1340/44000 | Loss: 3.9868 | LR: 5.00e-04 | 1322.2ms/step | 15.1 samples/s +Step 1350/44000 | Loss: 3.9914 | LR: 5.00e-04 | 1319.3ms/step | 15.2 samples/s +Step 1360/44000 | Loss: 3.9444 | LR: 5.00e-04 | 1319.8ms/step | 15.2 samples/s +Step 1370/44000 | Loss: 3.9824 | LR: 5.00e-04 | 1322.3ms/step | 15.1 samples/s +Step 1380/44000 | Loss: 3.9598 | LR: 5.00e-04 | 1321.7ms/step | 15.1 samples/s +Step 1390/44000 | Loss: 3.9696 | LR: 5.00e-04 | 1321.8ms/step | 15.1 samples/s +Step 1400/44000 | Loss: 3.9655 | LR: 5.00e-04 | 1320.1ms/step | 15.2 samples/s +Step 1410/44000 | Loss: 3.9839 | LR: 5.00e-04 | 1319.8ms/step | 15.2 samples/s +Step 1420/44000 | Loss: 3.9420 | LR: 5.00e-04 | 1322.1ms/step | 15.1 samples/s +Step 1430/44000 | Loss: 3.9527 | LR: 5.00e-04 | 1318.7ms/step | 15.2 samples/s +Step 1440/44000 | Loss: 3.9640 | LR: 5.00e-04 | 1322.4ms/step | 15.1 samples/s +Step 1450/44000 | Loss: 3.9726 | LR: 5.00e-04 | 1323.5ms/step | 15.1 samples/s +Step 1460/44000 | Loss: 3.9790 | LR: 5.00e-04 | 1318.9ms/step | 15.2 samples/s +Step 1470/44000 | Loss: 3.9269 | LR: 5.00e-04 | 1317.9ms/step | 15.2 samples/s +Step 1480/44000 | Loss: 3.9147 | LR: 5.00e-04 | 1322.4ms/step | 15.1 samples/s +Step 1490/44000 | Loss: 3.9244 | LR: 5.00e-04 | 1322.5ms/step | 15.1 samples/s +Step 1500/44000 | Loss: 3.9180 | LR: 5.00e-04 | 1321.5ms/step | 15.1 samples/s +Validation at step 1500: Loss = 3.9168, Perplexity = 50.24 +Step 1510/44000 | Loss: 3.9064 | LR: 5.00e-04 | 3543.5ms/step | 5.6 samples/s +Step 1520/44000 | Loss: 3.8795 | LR: 5.00e-04 | 1320.3ms/step | 15.1 samples/s +Step 1530/44000 | Loss: 3.8969 | LR: 5.00e-04 | 1321.5ms/step | 15.1 samples/s +Step 1540/44000 | Loss: 3.8703 | LR: 5.00e-04 | 1324.6ms/step | 15.1 samples/s +Step 1550/44000 | Loss: 3.8865 | LR: 5.00e-04 | 1324.2ms/step | 15.1 samples/s +Step 1560/44000 | Loss: 3.9097 | LR: 5.00e-04 | 1320.4ms/step | 15.1 samples/s +Step 1570/44000 | Loss: 3.8865 | LR: 5.00e-04 | 1322.1ms/step | 15.1 samples/s +Step 1580/44000 | Loss: 3.8789 | LR: 5.00e-04 | 1320.3ms/step | 15.1 samples/s +Step 1590/44000 | Loss: 3.8973 | LR: 5.00e-04 | 1321.9ms/step | 15.1 samples/s +Step 1600/44000 | Loss: 3.8664 | LR: 5.00e-04 | 1322.6ms/step | 15.1 samples/s +Step 1610/44000 | Loss: 3.8576 | LR: 5.00e-04 | 1319.6ms/step | 15.2 samples/s +Step 1620/44000 | Loss: 3.8569 | LR: 5.00e-04 | 1321.3ms/step | 15.1 samples/s +Step 1630/44000 | Loss: 3.8356 | LR: 4.99e-04 | 1315.0ms/step | 15.2 samples/s +Step 1640/44000 | Loss: 3.8690 | LR: 4.99e-04 | 1315.5ms/step | 15.2 samples/s +Step 1650/44000 | Loss: 3.8486 | LR: 4.99e-04 | 1312.5ms/step | 15.2 samples/s +Step 1660/44000 | Loss: 3.8528 | LR: 4.99e-04 | 1313.7ms/step | 15.2 samples/s +Step 1670/44000 | Loss: 3.8210 | LR: 4.99e-04 | 1314.9ms/step | 15.2 samples/s +Step 1680/44000 | Loss: 3.8478 | LR: 4.99e-04 | 1312.7ms/step | 15.2 samples/s +Step 1690/44000 | Loss: 3.8523 | LR: 4.99e-04 | 1313.9ms/step | 15.2 samples/s +Step 1700/44000 | Loss: 3.8172 | LR: 4.99e-04 | 1318.7ms/step | 15.2 samples/s +Step 1710/44000 | Loss: 3.8315 | LR: 4.99e-04 | 1315.4ms/step | 15.2 samples/s +Step 1720/44000 | Loss: 3.7714 | LR: 4.99e-04 | 1315.7ms/step | 15.2 samples/s +Step 1730/44000 | Loss: 3.8538 | LR: 4.99e-04 | 1315.4ms/step | 15.2 samples/s +Step 1740/44000 | Loss: 3.8008 | LR: 4.99e-04 | 1319.5ms/step | 15.2 samples/s +Step 1750/44000 | Loss: 3.8176 | LR: 4.99e-04 | 1320.3ms/step | 15.1 samples/s +Validation at step 1750: Loss = 3.8096, Perplexity = 45.13 +Step 1760/44000 | Loss: 3.8322 | LR: 4.99e-04 | 3537.8ms/step | 5.7 samples/s +Step 1770/44000 | Loss: 3.8113 | LR: 4.99e-04 | 1320.6ms/step | 15.1 samples/s +Step 1780/44000 | Loss: 3.8151 | LR: 4.99e-04 | 1320.4ms/step | 15.1 samples/s +Step 1790/44000 | Loss: 3.7896 | LR: 4.99e-04 | 1321.3ms/step | 15.1 samples/s +Step 1800/44000 | Loss: 3.7867 | LR: 4.99e-04 | 1320.3ms/step | 15.1 samples/s +Step 1810/44000 | Loss: 3.8151 | LR: 4.99e-04 | 1318.2ms/step | 15.2 samples/s +Step 1820/44000 | Loss: 3.7973 | LR: 4.99e-04 | 1317.6ms/step | 15.2 samples/s +Step 1830/44000 | Loss: 3.7887 | LR: 4.99e-04 | 1320.6ms/step | 15.1 samples/s +Step 1840/44000 | Loss: 3.7742 | LR: 4.99e-04 | 1317.4ms/step | 15.2 samples/s +Step 1850/44000 | Loss: 3.7291 | LR: 4.99e-04 | 1318.0ms/step | 15.2 samples/s +Step 1860/44000 | Loss: 3.7557 | LR: 4.99e-04 | 1319.4ms/step | 15.2 samples/s +Step 1870/44000 | Loss: 3.7849 | LR: 4.99e-04 | 1321.2ms/step | 15.1 samples/s +Step 1880/44000 | Loss: 3.7291 | LR: 4.99e-04 | 1319.5ms/step | 15.2 samples/s +Step 1890/44000 | Loss: 3.7404 | LR: 4.99e-04 | 1320.5ms/step | 15.1 samples/s +Step 1900/44000 | Loss: 3.7423 | LR: 4.99e-04 | 1323.0ms/step | 15.1 samples/s +Step 1910/44000 | Loss: 3.7453 | LR: 4.99e-04 | 1321.0ms/step | 15.1 samples/s +Step 1920/44000 | Loss: 3.7400 | LR: 4.99e-04 | 1321.9ms/step | 15.1 samples/s +Step 1930/44000 | Loss: 3.7810 | LR: 4.99e-04 | 1318.7ms/step | 15.2 samples/s +Step 1940/44000 | Loss: 3.7233 | LR: 4.99e-04 | 1322.2ms/step | 15.1 samples/s +Step 1950/44000 | Loss: 3.7615 | LR: 4.99e-04 | 1316.0ms/step | 15.2 samples/s +Step 1960/44000 | Loss: 3.7256 | LR: 4.99e-04 | 1319.8ms/step | 15.2 samples/s +Step 1970/44000 | Loss: 3.7457 | LR: 4.99e-04 | 1315.2ms/step | 15.2 samples/s +Step 1980/44000 | Loss: 3.7574 | LR: 4.99e-04 | 1314.9ms/step | 15.2 samples/s +Step 1990/44000 | Loss: 3.7141 | LR: 4.99e-04 | 1315.7ms/step | 15.2 samples/s +Step 2000/44000 | Loss: 3.7103 | LR: 4.99e-04 | 1319.0ms/step | 15.2 samples/s +Validation at step 2000: Loss = 3.7276, Perplexity = 41.58 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-2000 +Step 2010/44000 | Loss: 3.7214 | LR: 4.99e-04 | 3852.2ms/step | 5.2 samples/s +Step 2020/44000 | Loss: 3.7312 | LR: 4.99e-04 | 1322.3ms/step | 15.1 samples/s +Step 2030/44000 | Loss: 3.7143 | LR: 4.99e-04 | 1323.2ms/step | 15.1 samples/s +Step 2040/44000 | Loss: 3.7124 | LR: 4.99e-04 | 1322.1ms/step | 15.1 samples/s +Step 2050/44000 | Loss: 3.6954 | LR: 4.99e-04 | 1322.9ms/step | 15.1 samples/s +Step 2060/44000 | Loss: 3.7334 | LR: 4.99e-04 | 1323.3ms/step | 15.1 samples/s +Step 2070/44000 | Loss: 3.7028 | LR: 4.99e-04 | 1325.0ms/step | 15.1 samples/s +Step 2080/44000 | Loss: 3.6941 | LR: 4.99e-04 | 1322.8ms/step | 15.1 samples/s +Step 2090/44000 | Loss: 3.7051 | LR: 4.99e-04 | 1323.8ms/step | 15.1 samples/s +Step 2100/44000 | Loss: 3.7017 | LR: 4.99e-04 | 1320.1ms/step | 15.2 samples/s +Step 2110/44000 | Loss: 3.6789 | LR: 4.99e-04 | 1318.2ms/step | 15.2 samples/s +Step 2120/44000 | Loss: 3.6527 | LR: 4.99e-04 | 1323.1ms/step | 15.1 samples/s +Step 2130/44000 | Loss: 3.6921 | LR: 4.99e-04 | 1323.0ms/step | 15.1 samples/s +Step 2140/44000 | Loss: 3.7209 | LR: 4.99e-04 | 1321.3ms/step | 15.1 samples/s +Step 2150/44000 | Loss: 3.6446 | LR: 4.99e-04 | 1323.9ms/step | 15.1 samples/s +Step 2160/44000 | Loss: 3.7188 | LR: 4.99e-04 | 1324.7ms/step | 15.1 samples/s +Step 2170/44000 | Loss: 3.6896 | LR: 4.99e-04 | 1323.7ms/step | 15.1 samples/s +Step 2180/44000 | Loss: 3.6841 | LR: 4.99e-04 | 1328.1ms/step | 15.1 samples/s +Step 2190/44000 | Loss: 3.6713 | LR: 4.99e-04 | 1320.6ms/step | 15.1 samples/s +Step 2200/44000 | Loss: 3.6242 | LR: 4.99e-04 | 1322.3ms/step | 15.1 samples/s +Step 2210/44000 | Loss: 3.6663 | LR: 4.99e-04 | 1325.1ms/step | 15.1 samples/s +Step 2220/44000 | Loss: 3.7032 | LR: 4.99e-04 | 1319.8ms/step | 15.2 samples/s +Step 2230/44000 | Loss: 3.6537 | LR: 4.99e-04 | 1320.9ms/step | 15.1 samples/s +Step 2240/44000 | Loss: 3.6473 | LR: 4.99e-04 | 1324.5ms/step | 15.1 samples/s +Step 2250/44000 | Loss: 3.6506 | LR: 4.99e-04 | 1325.3ms/step | 15.1 samples/s +Validation at step 2250: Loss = 3.6624, Perplexity = 38.95 +Step 2260/44000 | Loss: 3.6527 | LR: 4.98e-04 | 3550.9ms/step | 5.6 samples/s +Step 2270/44000 | Loss: 3.6643 | LR: 4.98e-04 | 1323.0ms/step | 15.1 samples/s +Step 2280/44000 | Loss: 3.6576 | LR: 4.98e-04 | 1323.7ms/step | 15.1 samples/s +Step 2290/44000 | Loss: 3.6721 | LR: 4.98e-04 | 1324.0ms/step | 15.1 samples/s +Step 2300/44000 | Loss: 3.6486 | LR: 4.98e-04 | 1319.7ms/step | 15.2 samples/s +Step 2310/44000 | Loss: 3.6650 | LR: 4.98e-04 | 1319.6ms/step | 15.2 samples/s +Step 2320/44000 | Loss: 3.6339 | LR: 4.98e-04 | 1323.0ms/step | 15.1 samples/s +Step 2330/44000 | Loss: 3.6397 | LR: 4.98e-04 | 1325.6ms/step | 15.1 samples/s +Step 2340/44000 | Loss: 3.6232 | LR: 4.98e-04 | 1325.4ms/step | 15.1 samples/s +Step 2350/44000 | Loss: 3.6302 | LR: 4.98e-04 | 1322.6ms/step | 15.1 samples/s +Step 2360/44000 | Loss: 3.6630 | LR: 4.98e-04 | 1324.9ms/step | 15.1 samples/s +Step 2370/44000 | Loss: 3.6438 | LR: 4.98e-04 | 1321.0ms/step | 15.1 samples/s +Step 2380/44000 | Loss: 3.6331 | LR: 4.98e-04 | 1323.3ms/step | 15.1 samples/s +Step 2390/44000 | Loss: 3.6241 | LR: 4.98e-04 | 1322.1ms/step | 15.1 samples/s +Step 2400/44000 | Loss: 3.6113 | LR: 4.98e-04 | 1322.4ms/step | 15.1 samples/s +Step 2410/44000 | Loss: 3.6059 | LR: 4.98e-04 | 1322.6ms/step | 15.1 samples/s +Step 2420/44000 | Loss: 3.6323 | LR: 4.98e-04 | 1323.8ms/step | 15.1 samples/s +Step 2430/44000 | Loss: 3.6235 | LR: 4.98e-04 | 1323.0ms/step | 15.1 samples/s +Step 2440/44000 | Loss: 3.6909 | LR: 4.98e-04 | 1326.7ms/step | 15.1 samples/s +Step 2450/44000 | Loss: 3.6025 | LR: 4.98e-04 | 1322.5ms/step | 15.1 samples/s +Step 2460/44000 | Loss: 3.6387 | LR: 4.98e-04 | 1327.4ms/step | 15.1 samples/s +Step 2470/44000 | Loss: 3.6045 | LR: 4.98e-04 | 1328.5ms/step | 15.1 samples/s +Step 2480/44000 | Loss: 3.6118 | LR: 4.98e-04 | 1325.4ms/step | 15.1 samples/s +Step 2490/44000 | Loss: 3.6181 | LR: 4.98e-04 | 1322.7ms/step | 15.1 samples/s +Step 2500/44000 | Loss: 3.5881 | LR: 4.98e-04 | 1325.4ms/step | 15.1 samples/s +Validation at step 2500: Loss = 3.6068, Perplexity = 36.85 +Step 2510/44000 | Loss: 3.6122 | LR: 4.98e-04 | 3549.8ms/step | 5.6 samples/s +Step 2520/44000 | Loss: 3.6216 | LR: 4.98e-04 | 1324.3ms/step | 15.1 samples/s +Step 2530/44000 | Loss: 3.5957 | LR: 4.98e-04 | 1324.8ms/step | 15.1 samples/s +Step 2540/44000 | Loss: 3.6100 | LR: 4.98e-04 | 1323.8ms/step | 15.1 samples/s +Step 2550/44000 | Loss: 3.5967 | LR: 4.98e-04 | 1321.7ms/step | 15.1 samples/s +Step 2560/44000 | Loss: 3.5616 | LR: 4.98e-04 | 1322.4ms/step | 15.1 samples/s +Step 2570/44000 | Loss: 3.6023 | LR: 4.98e-04 | 1327.7ms/step | 15.1 samples/s +Step 2580/44000 | Loss: 3.5853 | LR: 4.98e-04 | 1325.0ms/step | 15.1 samples/s +Step 2590/44000 | Loss: 3.5862 | LR: 4.98e-04 | 1326.4ms/step | 15.1 samples/s +Step 2600/44000 | Loss: 3.6134 | LR: 4.98e-04 | 1323.0ms/step | 15.1 samples/s +Step 2610/44000 | Loss: 3.5661 | LR: 4.98e-04 | 1325.3ms/step | 15.1 samples/s +Step 2620/44000 | Loss: 3.6069 | LR: 4.98e-04 | 1323.6ms/step | 15.1 samples/s +Step 2630/44000 | Loss: 3.5590 | LR: 4.98e-04 | 1322.9ms/step | 15.1 samples/s +Step 2640/44000 | Loss: 3.5800 | LR: 4.98e-04 | 1325.3ms/step | 15.1 samples/s +Step 2650/44000 | Loss: 3.5854 | LR: 4.98e-04 | 1326.0ms/step | 15.1 samples/s +Step 2660/44000 | Loss: 3.5909 | LR: 4.98e-04 | 1327.6ms/step | 15.1 samples/s +Step 2670/44000 | Loss: 3.5757 | LR: 4.98e-04 | 1324.2ms/step | 15.1 samples/s +Step 2680/44000 | Loss: 3.5684 | LR: 4.98e-04 | 1323.1ms/step | 15.1 samples/s +Step 2690/44000 | Loss: 3.5725 | LR: 4.98e-04 | 1322.9ms/step | 15.1 samples/s +Step 2700/44000 | Loss: 3.5676 | LR: 4.97e-04 | 1324.0ms/step | 15.1 samples/s +Step 2710/44000 | Loss: 3.5765 | LR: 4.97e-04 | 1326.6ms/step | 15.1 samples/s +Step 2720/44000 | Loss: 3.5676 | LR: 4.97e-04 | 1321.1ms/step | 15.1 samples/s +Step 2730/44000 | Loss: 3.5803 | LR: 4.97e-04 | 1322.4ms/step | 15.1 samples/s +Step 2740/44000 | Loss: 3.5705 | LR: 4.97e-04 | 1320.6ms/step | 15.1 samples/s +Step 2750/44000 | Loss: 3.5698 | LR: 4.97e-04 | 1320.1ms/step | 15.2 samples/s +Validation at step 2750: Loss = 3.5609, Perplexity = 35.19 +Step 2760/44000 | Loss: 3.5696 | LR: 4.97e-04 | 3549.0ms/step | 5.6 samples/s +Step 2770/44000 | Loss: 3.5816 | LR: 4.97e-04 | 1322.8ms/step | 15.1 samples/s +Step 2780/44000 | Loss: 3.5632 | LR: 4.97e-04 | 1324.7ms/step | 15.1 samples/s +Step 2790/44000 | Loss: 3.5193 | LR: 4.97e-04 | 1325.0ms/step | 15.1 samples/s +Step 2800/44000 | Loss: 3.5490 | LR: 4.97e-04 | 1323.7ms/step | 15.1 samples/s +Step 2810/44000 | Loss: 3.5037 | LR: 4.97e-04 | 1325.9ms/step | 15.1 samples/s +Step 2820/44000 | Loss: 3.5658 | LR: 4.97e-04 | 1325.5ms/step | 15.1 samples/s +Step 2830/44000 | Loss: 3.5448 | LR: 4.97e-04 | 1323.0ms/step | 15.1 samples/s +Step 2840/44000 | Loss: 3.5544 | LR: 4.97e-04 | 1326.7ms/step | 15.1 samples/s +Step 2850/44000 | Loss: 3.5313 | LR: 4.97e-04 | 1323.6ms/step | 15.1 samples/s +Step 2860/44000 | Loss: 3.5987 | LR: 4.97e-04 | 1322.6ms/step | 15.1 samples/s +Step 2870/44000 | Loss: 3.5476 | LR: 4.97e-04 | 1325.6ms/step | 15.1 samples/s +Step 2880/44000 | Loss: 3.5310 | LR: 4.97e-04 | 1325.0ms/step | 15.1 samples/s +Step 2890/44000 | Loss: 3.5285 | LR: 4.97e-04 | 1327.1ms/step | 15.1 samples/s +Step 2900/44000 | Loss: 3.4932 | LR: 4.97e-04 | 1327.8ms/step | 15.1 samples/s +Step 2910/44000 | Loss: 3.5330 | LR: 4.97e-04 | 1329.5ms/step | 15.0 samples/s +Step 2920/44000 | Loss: 3.5079 | LR: 4.97e-04 | 1325.8ms/step | 15.1 samples/s +Step 2930/44000 | Loss: 3.5393 | LR: 4.97e-04 | 1328.7ms/step | 15.1 samples/s +Step 2940/44000 | Loss: 3.5326 | LR: 4.97e-04 | 1319.5ms/step | 15.2 samples/s +Step 2950/44000 | Loss: 3.5190 | LR: 4.97e-04 | 1320.3ms/step | 15.1 samples/s +Step 2960/44000 | Loss: 3.5243 | LR: 4.97e-04 | 1326.9ms/step | 15.1 samples/s +Step 2970/44000 | Loss: 3.5002 | LR: 4.97e-04 | 1322.8ms/step | 15.1 samples/s +Step 2980/44000 | Loss: 3.5237 | LR: 4.97e-04 | 1324.4ms/step | 15.1 samples/s +Step 2990/44000 | Loss: 3.5014 | LR: 4.97e-04 | 1321.3ms/step | 15.1 samples/s +Step 3000/44000 | Loss: 3.5199 | LR: 4.97e-04 | 1324.9ms/step | 15.1 samples/s +Validation at step 3000: Loss = 3.5235, Perplexity = 33.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-3000 +Step 3010/44000 | Loss: 3.5415 | LR: 4.97e-04 | 3860.1ms/step | 5.2 samples/s +Step 3020/44000 | Loss: 3.5038 | LR: 4.97e-04 | 1322.2ms/step | 15.1 samples/s +Step 3030/44000 | Loss: 3.5272 | LR: 4.97e-04 | 1325.1ms/step | 15.1 samples/s +Step 3040/44000 | Loss: 3.5354 | LR: 4.97e-04 | 1318.7ms/step | 15.2 samples/s +Step 3050/44000 | Loss: 3.5070 | LR: 4.97e-04 | 1320.3ms/step | 15.1 samples/s +Step 3060/44000 | Loss: 3.5170 | LR: 4.96e-04 | 1320.3ms/step | 15.1 samples/s +Step 3070/44000 | Loss: 3.5469 | LR: 4.96e-04 | 1318.1ms/step | 15.2 samples/s +Step 3080/44000 | Loss: 3.5155 | LR: 4.96e-04 | 1318.0ms/step | 15.2 samples/s +Step 3090/44000 | Loss: 3.4782 | LR: 4.96e-04 | 1316.7ms/step | 15.2 samples/s +Step 3100/44000 | Loss: 3.4921 | LR: 4.96e-04 | 1318.9ms/step | 15.2 samples/s +Step 3110/44000 | Loss: 3.5222 | LR: 4.96e-04 | 1318.3ms/step | 15.2 samples/s +Step 3120/44000 | Loss: 3.4898 | LR: 4.96e-04 | 1321.2ms/step | 15.1 samples/s +Step 3130/44000 | Loss: 3.5108 | LR: 4.96e-04 | 1320.6ms/step | 15.1 samples/s +Step 3140/44000 | Loss: 3.4855 | LR: 4.96e-04 | 1322.9ms/step | 15.1 samples/s +Step 3150/44000 | Loss: 3.5032 | LR: 4.96e-04 | 1316.6ms/step | 15.2 samples/s +Step 3160/44000 | Loss: 3.5140 | LR: 4.96e-04 | 1322.5ms/step | 15.1 samples/s +Step 3170/44000 | Loss: 3.4911 | LR: 4.96e-04 | 1320.2ms/step | 15.1 samples/s +Step 3180/44000 | Loss: 3.5048 | LR: 4.96e-04 | 1316.1ms/step | 15.2 samples/s +Step 3190/44000 | Loss: 3.5138 | LR: 4.96e-04 | 1318.4ms/step | 15.2 samples/s +Step 3200/44000 | Loss: 3.5005 | LR: 4.96e-04 | 1316.6ms/step | 15.2 samples/s +Step 3210/44000 | Loss: 3.5147 | LR: 4.96e-04 | 1316.9ms/step | 15.2 samples/s +Step 3220/44000 | Loss: 3.4777 | LR: 4.96e-04 | 1319.6ms/step | 15.2 samples/s +Step 3230/44000 | Loss: 3.4924 | LR: 4.96e-04 | 1318.0ms/step | 15.2 samples/s +Step 3240/44000 | Loss: 3.4569 | LR: 4.96e-04 | 1318.7ms/step | 15.2 samples/s +Step 3250/44000 | Loss: 3.5188 | LR: 4.96e-04 | 1318.7ms/step | 15.2 samples/s +Validation at step 3250: Loss = 3.4965, Perplexity = 33.00 +Step 3260/44000 | Loss: 3.4876 | LR: 4.96e-04 | 3536.9ms/step | 5.7 samples/s +Step 3270/44000 | Loss: 3.4867 | LR: 4.96e-04 | 1319.7ms/step | 15.2 samples/s +Step 3280/44000 | Loss: 3.4916 | LR: 4.96e-04 | 1317.4ms/step | 15.2 samples/s +Step 3290/44000 | Loss: 3.4734 | LR: 4.96e-04 | 1319.7ms/step | 15.2 samples/s +Step 3300/44000 | Loss: 3.4987 | LR: 4.96e-04 | 1317.5ms/step | 15.2 samples/s +Step 3310/44000 | Loss: 3.4823 | LR: 4.96e-04 | 1317.9ms/step | 15.2 samples/s +Step 3320/44000 | Loss: 3.4760 | LR: 4.96e-04 | 1317.0ms/step | 15.2 samples/s +Step 3330/44000 | Loss: 3.4886 | LR: 4.96e-04 | 1321.4ms/step | 15.1 samples/s +Step 3340/44000 | Loss: 3.4727 | LR: 4.96e-04 | 1323.5ms/step | 15.1 samples/s +Step 3350/44000 | Loss: 3.4825 | LR: 4.96e-04 | 1324.8ms/step | 15.1 samples/s +Step 3360/44000 | Loss: 3.4684 | LR: 4.96e-04 | 1323.9ms/step | 15.1 samples/s +Step 3370/44000 | Loss: 3.4724 | LR: 4.95e-04 | 1328.4ms/step | 15.1 samples/s +Step 3380/44000 | Loss: 3.5136 | LR: 4.95e-04 | 1323.9ms/step | 15.1 samples/s +Step 3390/44000 | Loss: 3.5104 | LR: 4.95e-04 | 1325.0ms/step | 15.1 samples/s +Step 3400/44000 | Loss: 3.4665 | LR: 4.95e-04 | 1327.6ms/step | 15.1 samples/s +Step 3410/44000 | Loss: 3.4730 | LR: 4.95e-04 | 1325.5ms/step | 15.1 samples/s +Step 3420/44000 | Loss: 3.4839 | LR: 4.95e-04 | 1326.7ms/step | 15.1 samples/s +Step 3430/44000 | Loss: 3.4839 | LR: 4.95e-04 | 1324.7ms/step | 15.1 samples/s +Step 3440/44000 | Loss: 3.4715 | LR: 4.95e-04 | 1321.0ms/step | 15.1 samples/s +Step 3450/44000 | Loss: 3.4765 | LR: 4.95e-04 | 1329.7ms/step | 15.0 samples/s +Step 3460/44000 | Loss: 3.4343 | LR: 4.95e-04 | 1328.9ms/step | 15.1 samples/s +Step 3470/44000 | Loss: 3.4776 | LR: 4.95e-04 | 1326.7ms/step | 15.1 samples/s +Step 3480/44000 | Loss: 3.4528 | LR: 4.95e-04 | 1321.8ms/step | 15.1 samples/s +Step 3490/44000 | Loss: 3.4843 | LR: 4.95e-04 | 1323.2ms/step | 15.1 samples/s +Step 3500/44000 | Loss: 3.4718 | LR: 4.95e-04 | 1325.2ms/step | 15.1 samples/s +Validation at step 3500: Loss = 3.4644, Perplexity = 31.96 +Step 3510/44000 | Loss: 3.4703 | LR: 4.95e-04 | 3546.4ms/step | 5.6 samples/s +Step 3520/44000 | Loss: 3.4433 | LR: 4.95e-04 | 1322.9ms/step | 15.1 samples/s +Step 3530/44000 | Loss: 3.4329 | LR: 4.95e-04 | 1324.8ms/step | 15.1 samples/s +Step 3540/44000 | Loss: 3.4616 | LR: 4.95e-04 | 1327.5ms/step | 15.1 samples/s +Step 3550/44000 | Loss: 3.4651 | LR: 4.95e-04 | 1320.3ms/step | 15.1 samples/s +Step 3560/44000 | Loss: 3.4727 | LR: 4.95e-04 | 1318.1ms/step | 15.2 samples/s +Step 3570/44000 | Loss: 3.4559 | LR: 4.95e-04 | 1321.4ms/step | 15.1 samples/s +Step 3580/44000 | Loss: 3.4737 | LR: 4.95e-04 | 1322.5ms/step | 15.1 samples/s +Step 3590/44000 | Loss: 3.4535 | LR: 4.95e-04 | 1320.6ms/step | 15.1 samples/s +Step 3600/44000 | Loss: 3.4466 | LR: 4.95e-04 | 1320.1ms/step | 15.2 samples/s +Step 3610/44000 | Loss: 3.4344 | LR: 4.95e-04 | 1326.1ms/step | 15.1 samples/s +Step 3620/44000 | Loss: 3.4460 | LR: 4.95e-04 | 1329.2ms/step | 15.0 samples/s +Step 3630/44000 | Loss: 3.4303 | LR: 4.95e-04 | 1330.9ms/step | 15.0 samples/s +Step 3640/44000 | Loss: 3.4217 | LR: 4.95e-04 | 1334.3ms/step | 15.0 samples/s +Step 3650/44000 | Loss: 3.4343 | LR: 4.94e-04 | 1329.1ms/step | 15.0 samples/s +Step 3660/44000 | Loss: 3.4277 | LR: 4.94e-04 | 1330.4ms/step | 15.0 samples/s +Step 3670/44000 | Loss: 3.4643 | LR: 4.94e-04 | 1324.3ms/step | 15.1 samples/s +Step 3680/44000 | Loss: 3.4183 | LR: 4.94e-04 | 1330.8ms/step | 15.0 samples/s +Step 3690/44000 | Loss: 3.4378 | LR: 4.94e-04 | 1332.6ms/step | 15.0 samples/s +Step 3700/44000 | Loss: 3.4463 | LR: 4.94e-04 | 1333.4ms/step | 15.0 samples/s +Step 3710/44000 | Loss: 3.4219 | LR: 4.94e-04 | 1330.4ms/step | 15.0 samples/s +Step 3720/44000 | Loss: 3.4455 | LR: 4.94e-04 | 1331.3ms/step | 15.0 samples/s +Step 3730/44000 | Loss: 3.4441 | LR: 4.94e-04 | 1332.1ms/step | 15.0 samples/s +Step 3740/44000 | Loss: 3.4396 | LR: 4.94e-04 | 1330.2ms/step | 15.0 samples/s +Step 3750/44000 | Loss: 3.4376 | LR: 4.94e-04 | 1331.7ms/step | 15.0 samples/s +Validation at step 3750: Loss = 3.4366, Perplexity = 31.08 +Step 3760/44000 | Loss: 3.4329 | LR: 4.94e-04 | 3557.6ms/step | 5.6 samples/s +Step 3770/44000 | Loss: 3.4318 | LR: 4.94e-04 | 1326.8ms/step | 15.1 samples/s +Step 3780/44000 | Loss: 3.4541 | LR: 4.94e-04 | 1329.8ms/step | 15.0 samples/s +Step 3790/44000 | Loss: 3.4394 | LR: 4.94e-04 | 1333.2ms/step | 15.0 samples/s +Step 3800/44000 | Loss: 3.4471 | LR: 4.94e-04 | 1332.5ms/step | 15.0 samples/s +Step 3810/44000 | Loss: 3.4468 | LR: 4.94e-04 | 1332.1ms/step | 15.0 samples/s +Step 3820/44000 | Loss: 3.4540 | LR: 4.94e-04 | 1330.7ms/step | 15.0 samples/s +Step 3830/44000 | Loss: 3.4484 | LR: 4.94e-04 | 1329.2ms/step | 15.0 samples/s +Step 3840/44000 | Loss: 3.4182 | LR: 4.94e-04 | 1329.9ms/step | 15.0 samples/s +Step 3850/44000 | Loss: 3.4070 | LR: 4.94e-04 | 1328.3ms/step | 15.1 samples/s +Step 3860/44000 | Loss: 3.4390 | LR: 4.94e-04 | 1326.0ms/step | 15.1 samples/s +Step 3870/44000 | Loss: 3.4460 | LR: 4.94e-04 | 1326.5ms/step | 15.1 samples/s +Step 3880/44000 | Loss: 3.4042 | LR: 4.94e-04 | 1330.5ms/step | 15.0 samples/s +Step 3890/44000 | Loss: 3.4284 | LR: 4.94e-04 | 1327.4ms/step | 15.1 samples/s +Step 3900/44000 | Loss: 3.4374 | LR: 4.93e-04 | 1328.9ms/step | 15.0 samples/s +Step 3910/44000 | Loss: 3.4126 | LR: 4.93e-04 | 1326.7ms/step | 15.1 samples/s +Step 3920/44000 | Loss: 3.4152 | LR: 4.93e-04 | 1322.8ms/step | 15.1 samples/s +Step 3930/44000 | Loss: 3.4513 | LR: 4.93e-04 | 1324.0ms/step | 15.1 samples/s +Step 3940/44000 | Loss: 3.3955 | LR: 4.93e-04 | 1323.8ms/step | 15.1 samples/s +Step 3950/44000 | Loss: 3.3979 | LR: 4.93e-04 | 1325.4ms/step | 15.1 samples/s +Step 3960/44000 | Loss: 3.4511 | LR: 4.93e-04 | 1322.7ms/step | 15.1 samples/s +Step 3970/44000 | Loss: 3.4021 | LR: 4.93e-04 | 1319.7ms/step | 15.2 samples/s +Step 3980/44000 | Loss: 3.4217 | LR: 4.93e-04 | 1323.9ms/step | 15.1 samples/s +Step 3990/44000 | Loss: 3.3937 | LR: 4.93e-04 | 1323.6ms/step | 15.1 samples/s +Step 4000/44000 | Loss: 3.4311 | LR: 4.93e-04 | 1322.9ms/step | 15.1 samples/s +Validation at step 4000: Loss = 3.4122, Perplexity = 30.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-4000 +Step 4010/44000 | Loss: 3.3629 | LR: 4.93e-04 | 3840.4ms/step | 5.2 samples/s +Step 4020/44000 | Loss: 3.4194 | LR: 4.93e-04 | 1325.7ms/step | 15.1 samples/s +Step 4030/44000 | Loss: 3.3929 | LR: 4.93e-04 | 1322.7ms/step | 15.1 samples/s +Step 4040/44000 | Loss: 3.4259 | LR: 4.93e-04 | 1327.1ms/step | 15.1 samples/s +Step 4050/44000 | Loss: 3.4018 | LR: 4.93e-04 | 1322.3ms/step | 15.1 samples/s +Step 4060/44000 | Loss: 3.4444 | LR: 4.93e-04 | 1324.4ms/step | 15.1 samples/s +Step 4070/44000 | Loss: 3.4057 | LR: 4.93e-04 | 1321.0ms/step | 15.1 samples/s +Step 4080/44000 | Loss: 3.3943 | LR: 4.93e-04 | 1324.9ms/step | 15.1 samples/s +Step 4090/44000 | Loss: 3.3917 | LR: 4.93e-04 | 1326.1ms/step | 15.1 samples/s +Step 4100/44000 | Loss: 3.4009 | LR: 4.93e-04 | 1325.3ms/step | 15.1 samples/s +Step 4110/44000 | Loss: 3.4054 | LR: 4.93e-04 | 1324.4ms/step | 15.1 samples/s +Step 4120/44000 | Loss: 3.3940 | LR: 4.93e-04 | 1326.5ms/step | 15.1 samples/s +Step 4130/44000 | Loss: 3.4065 | LR: 4.93e-04 | 1323.2ms/step | 15.1 samples/s +Step 4140/44000 | Loss: 3.3913 | LR: 4.92e-04 | 1323.6ms/step | 15.1 samples/s +Step 4150/44000 | Loss: 3.3853 | LR: 4.92e-04 | 1323.5ms/step | 15.1 samples/s +Step 4160/44000 | Loss: 3.4023 | LR: 4.92e-04 | 1323.2ms/step | 15.1 samples/s +Step 4170/44000 | Loss: 3.4115 | LR: 4.92e-04 | 1322.3ms/step | 15.1 samples/s +Step 4180/44000 | Loss: 3.3929 | LR: 4.92e-04 | 1327.4ms/step | 15.1 samples/s +Step 4190/44000 | Loss: 3.3937 | LR: 4.92e-04 | 1324.0ms/step | 15.1 samples/s +Step 4200/44000 | Loss: 3.3845 | LR: 4.92e-04 | 1324.0ms/step | 15.1 samples/s +Step 4210/44000 | Loss: 3.4022 | LR: 4.92e-04 | 1325.3ms/step | 15.1 samples/s +Step 4220/44000 | Loss: 3.3887 | LR: 4.92e-04 | 1323.4ms/step | 15.1 samples/s +Step 4230/44000 | Loss: 3.3898 | LR: 4.92e-04 | 1323.4ms/step | 15.1 samples/s +Step 4240/44000 | Loss: 3.3865 | LR: 4.92e-04 | 1322.1ms/step | 15.1 samples/s +Step 4250/44000 | Loss: 3.3989 | LR: 4.92e-04 | 1325.3ms/step | 15.1 samples/s +Validation at step 4250: Loss = 3.3930, Perplexity = 29.75 +Step 4260/44000 | Loss: 3.4217 | LR: 4.92e-04 | 3559.1ms/step | 5.6 samples/s +Step 4270/44000 | Loss: 3.4040 | LR: 4.92e-04 | 1321.7ms/step | 15.1 samples/s +Step 4280/44000 | Loss: 3.4207 | LR: 4.92e-04 | 1324.9ms/step | 15.1 samples/s +Step 4290/44000 | Loss: 3.3749 | LR: 4.92e-04 | 1325.7ms/step | 15.1 samples/s +Step 4300/44000 | Loss: 3.4070 | LR: 4.92e-04 | 1325.4ms/step | 15.1 samples/s +Step 4310/44000 | Loss: 3.4175 | LR: 4.92e-04 | 1321.1ms/step | 15.1 samples/s +Step 4320/44000 | Loss: 3.4046 | LR: 4.92e-04 | 1328.3ms/step | 15.1 samples/s +Step 4330/44000 | Loss: 3.3627 | LR: 4.92e-04 | 1322.9ms/step | 15.1 samples/s +Step 4340/44000 | Loss: 3.3939 | LR: 4.92e-04 | 1326.5ms/step | 15.1 samples/s +Step 4350/44000 | Loss: 3.4031 | LR: 4.92e-04 | 1323.9ms/step | 15.1 samples/s +Step 4360/44000 | Loss: 3.3746 | LR: 4.91e-04 | 1324.6ms/step | 15.1 samples/s +Step 4370/44000 | Loss: 3.4132 | LR: 4.91e-04 | 1327.8ms/step | 15.1 samples/s +Step 4380/44000 | Loss: 3.3671 | LR: 4.91e-04 | 1324.3ms/step | 15.1 samples/s +Step 4390/44000 | Loss: 3.3961 | LR: 4.91e-04 | 1321.5ms/step | 15.1 samples/s +Step 4400/44000 | Loss: 3.3470 | LR: 4.91e-04 | 1322.7ms/step | 15.1 samples/s +Step 4410/44000 | Loss: 3.3659 | LR: 4.91e-04 | 1321.8ms/step | 15.1 samples/s +Step 4420/44000 | Loss: 3.3656 | LR: 4.91e-04 | 1326.4ms/step | 15.1 samples/s +Step 4430/44000 | Loss: 3.3875 | LR: 4.91e-04 | 1330.0ms/step | 15.0 samples/s +Step 4440/44000 | Loss: 3.4242 | LR: 4.91e-04 | 1323.6ms/step | 15.1 samples/s +Step 4450/44000 | Loss: 3.4181 | LR: 4.91e-04 | 1326.1ms/step | 15.1 samples/s +Step 4460/44000 | Loss: 3.3624 | LR: 4.91e-04 | 1327.8ms/step | 15.1 samples/s +Step 4470/44000 | Loss: 3.3516 | LR: 4.91e-04 | 1321.6ms/step | 15.1 samples/s +Step 4480/44000 | Loss: 3.3868 | LR: 4.91e-04 | 1325.3ms/step | 15.1 samples/s +Step 4490/44000 | Loss: 3.3622 | LR: 4.91e-04 | 1324.5ms/step | 15.1 samples/s +Step 4500/44000 | Loss: 3.3925 | LR: 4.91e-04 | 1330.0ms/step | 15.0 samples/s +Validation at step 4500: Loss = 3.3753, Perplexity = 29.23 +Step 4510/44000 | Loss: 3.3567 | LR: 4.91e-04 | 3552.0ms/step | 5.6 samples/s +Step 4520/44000 | Loss: 3.3685 | LR: 4.91e-04 | 1327.5ms/step | 15.1 samples/s +Step 4530/44000 | Loss: 3.3388 | LR: 4.91e-04 | 1324.8ms/step | 15.1 samples/s +Step 4540/44000 | Loss: 3.3808 | LR: 4.91e-04 | 1324.3ms/step | 15.1 samples/s +Step 4550/44000 | Loss: 3.4053 | LR: 4.91e-04 | 1324.6ms/step | 15.1 samples/s +Step 4560/44000 | Loss: 3.3574 | LR: 4.90e-04 | 1324.0ms/step | 15.1 samples/s +Step 4570/44000 | Loss: 3.3567 | LR: 4.90e-04 | 1324.7ms/step | 15.1 samples/s +Step 4580/44000 | Loss: 3.3618 | LR: 4.90e-04 | 1323.8ms/step | 15.1 samples/s +Step 4590/44000 | Loss: 3.3535 | LR: 4.90e-04 | 1322.7ms/step | 15.1 samples/s +Step 4600/44000 | Loss: 3.3715 | LR: 4.90e-04 | 1323.5ms/step | 15.1 samples/s +Step 4610/44000 | Loss: 3.3710 | LR: 4.90e-04 | 1324.2ms/step | 15.1 samples/s +Step 4620/44000 | Loss: 3.3648 | LR: 4.90e-04 | 1324.3ms/step | 15.1 samples/s +Step 4630/44000 | Loss: 3.3526 | LR: 4.90e-04 | 1325.8ms/step | 15.1 samples/s +Step 4640/44000 | Loss: 3.3704 | LR: 4.90e-04 | 1327.1ms/step | 15.1 samples/s +Step 4650/44000 | Loss: 3.3783 | LR: 4.90e-04 | 1327.6ms/step | 15.1 samples/s +Step 4660/44000 | Loss: 3.3310 | LR: 4.90e-04 | 1332.8ms/step | 15.0 samples/s +Step 4670/44000 | Loss: 3.3608 | LR: 4.90e-04 | 1330.7ms/step | 15.0 samples/s +Step 4680/44000 | Loss: 3.3322 | LR: 4.90e-04 | 1325.7ms/step | 15.1 samples/s +Step 4690/44000 | Loss: 3.3401 | LR: 4.90e-04 | 1322.2ms/step | 15.1 samples/s +Step 4700/44000 | Loss: 3.3431 | LR: 4.90e-04 | 1323.8ms/step | 15.1 samples/s +Step 4710/44000 | Loss: 3.3647 | LR: 4.90e-04 | 1324.3ms/step | 15.1 samples/s +Step 4720/44000 | Loss: 3.3679 | LR: 4.90e-04 | 1328.2ms/step | 15.1 samples/s +Step 4730/44000 | Loss: 3.3526 | LR: 4.90e-04 | 1327.9ms/step | 15.1 samples/s +Step 4740/44000 | Loss: 3.3432 | LR: 4.90e-04 | 1325.4ms/step | 15.1 samples/s +Step 4750/44000 | Loss: 3.3511 | LR: 4.90e-04 | 1327.9ms/step | 15.1 samples/s +Validation at step 4750: Loss = 3.3569, Perplexity = 28.70 +Step 4760/44000 | Loss: 3.3599 | LR: 4.89e-04 | 3553.7ms/step | 5.6 samples/s +Step 4770/44000 | Loss: 3.3766 | LR: 4.89e-04 | 1321.7ms/step | 15.1 samples/s +Step 4780/44000 | Loss: 3.3636 | LR: 4.89e-04 | 1325.2ms/step | 15.1 samples/s +Step 4790/44000 | Loss: 3.3248 | LR: 4.89e-04 | 1320.0ms/step | 15.2 samples/s +Step 4800/44000 | Loss: 3.3467 | LR: 4.89e-04 | 1321.8ms/step | 15.1 samples/s +Step 4810/44000 | Loss: 3.3263 | LR: 4.89e-04 | 1322.9ms/step | 15.1 samples/s +Step 4820/44000 | Loss: 3.3628 | LR: 4.89e-04 | 1323.4ms/step | 15.1 samples/s +Step 4830/44000 | Loss: 3.3521 | LR: 4.89e-04 | 1323.2ms/step | 15.1 samples/s +Step 4840/44000 | Loss: 3.3436 | LR: 4.89e-04 | 1324.3ms/step | 15.1 samples/s +Step 4850/44000 | Loss: 3.3624 | LR: 4.89e-04 | 1325.3ms/step | 15.1 samples/s +Step 4860/44000 | Loss: 3.3263 | LR: 4.89e-04 | 1325.1ms/step | 15.1 samples/s +Step 4870/44000 | Loss: 3.3276 | LR: 4.89e-04 | 1323.3ms/step | 15.1 samples/s +Step 4880/44000 | Loss: 3.3511 | LR: 4.89e-04 | 1325.1ms/step | 15.1 samples/s +Step 4890/44000 | Loss: 3.3600 | LR: 4.89e-04 | 1323.6ms/step | 15.1 samples/s +Step 4900/44000 | Loss: 3.3754 | LR: 4.89e-04 | 1322.6ms/step | 15.1 samples/s +Step 4910/44000 | Loss: 3.3491 | LR: 4.89e-04 | 1327.4ms/step | 15.1 samples/s +Step 4920/44000 | Loss: 3.3370 | LR: 4.89e-04 | 1325.0ms/step | 15.1 samples/s +Step 4930/44000 | Loss: 3.3084 | LR: 4.89e-04 | 1324.2ms/step | 15.1 samples/s +Step 4940/44000 | Loss: 3.3408 | LR: 4.89e-04 | 1327.0ms/step | 15.1 samples/s +Step 4950/44000 | Loss: 3.3218 | LR: 4.88e-04 | 1381.0ms/step | 14.5 samples/s +Step 4960/44000 | Loss: 3.3629 | LR: 4.88e-04 | 1332.6ms/step | 15.0 samples/s +Step 4970/44000 | Loss: 3.3427 | LR: 4.88e-04 | 1333.4ms/step | 15.0 samples/s +Step 4980/44000 | Loss: 3.3542 | LR: 4.88e-04 | 1337.3ms/step | 15.0 samples/s +Step 4990/44000 | Loss: 3.3351 | LR: 4.88e-04 | 1328.6ms/step | 15.1 samples/s +Step 5000/44000 | Loss: 3.3054 | LR: 4.88e-04 | 1326.5ms/step | 15.1 samples/s +Validation at step 5000: Loss = 3.3415, Perplexity = 28.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-5000 +Step 5010/44000 | Loss: 3.3510 | LR: 4.88e-04 | 3857.5ms/step | 5.2 samples/s +Step 5020/44000 | Loss: 3.3018 | LR: 4.88e-04 | 1327.4ms/step | 15.1 samples/s +Step 5030/44000 | Loss: 3.3320 | LR: 4.88e-04 | 1324.8ms/step | 15.1 samples/s +Step 5040/44000 | Loss: 3.3158 | LR: 4.88e-04 | 1324.9ms/step | 15.1 samples/s +Step 5050/44000 | Loss: 3.3515 | LR: 4.88e-04 | 1326.3ms/step | 15.1 samples/s +Step 5060/44000 | Loss: 3.3579 | LR: 4.88e-04 | 1327.7ms/step | 15.1 samples/s +Step 5070/44000 | Loss: 3.3391 | LR: 4.88e-04 | 1323.0ms/step | 15.1 samples/s +Step 5080/44000 | Loss: 3.3306 | LR: 4.88e-04 | 1325.9ms/step | 15.1 samples/s +Step 5090/44000 | Loss: 3.3378 | LR: 4.88e-04 | 1331.8ms/step | 15.0 samples/s +Step 5100/44000 | Loss: 3.3476 | LR: 4.88e-04 | 1326.6ms/step | 15.1 samples/s +Step 5110/44000 | Loss: 3.3436 | LR: 4.88e-04 | 1324.0ms/step | 15.1 samples/s +Step 5120/44000 | Loss: 3.3260 | LR: 4.88e-04 | 1324.3ms/step | 15.1 samples/s +Step 5130/44000 | Loss: 3.3243 | LR: 4.87e-04 | 1324.9ms/step | 15.1 samples/s +Step 5140/44000 | Loss: 3.3390 | LR: 4.87e-04 | 1325.0ms/step | 15.1 samples/s +Step 5150/44000 | Loss: 3.3352 | LR: 4.87e-04 | 1325.7ms/step | 15.1 samples/s +Step 5160/44000 | Loss: 3.3437 | LR: 4.87e-04 | 1326.4ms/step | 15.1 samples/s +Step 5170/44000 | Loss: 3.3596 | LR: 4.87e-04 | 1323.7ms/step | 15.1 samples/s +Step 5180/44000 | Loss: 3.3422 | LR: 4.87e-04 | 1325.4ms/step | 15.1 samples/s +Step 5190/44000 | Loss: 3.3081 | LR: 4.87e-04 | 1324.9ms/step | 15.1 samples/s +Step 5200/44000 | Loss: 3.3009 | LR: 4.87e-04 | 1325.8ms/step | 15.1 samples/s +Step 5210/44000 | Loss: 3.3204 | LR: 4.87e-04 | 1324.2ms/step | 15.1 samples/s +Step 5220/44000 | Loss: 3.3312 | LR: 4.87e-04 | 1324.6ms/step | 15.1 samples/s +Step 5230/44000 | Loss: 3.3019 | LR: 4.87e-04 | 1328.1ms/step | 15.1 samples/s +Step 5240/44000 | Loss: 3.3155 | LR: 4.87e-04 | 1325.4ms/step | 15.1 samples/s +Step 5250/44000 | Loss: 3.3456 | LR: 4.87e-04 | 1324.4ms/step | 15.1 samples/s +Validation at step 5250: Loss = 3.3261, Perplexity = 27.83 +Step 5260/44000 | Loss: 3.3550 | LR: 4.87e-04 | 3548.6ms/step | 5.6 samples/s +Step 5270/44000 | Loss: 3.3421 | LR: 4.87e-04 | 1325.3ms/step | 15.1 samples/s +Step 5280/44000 | Loss: 3.3066 | LR: 4.87e-04 | 1325.3ms/step | 15.1 samples/s +Step 5290/44000 | Loss: 3.3278 | LR: 4.87e-04 | 1324.1ms/step | 15.1 samples/s +Step 5300/44000 | Loss: 3.3161 | LR: 4.86e-04 | 1322.8ms/step | 15.1 samples/s +Step 5310/44000 | Loss: 3.3426 | LR: 4.86e-04 | 1327.5ms/step | 15.1 samples/s +Step 5320/44000 | Loss: 3.3203 | LR: 4.86e-04 | 1324.3ms/step | 15.1 samples/s +Step 5330/44000 | Loss: 3.3261 | LR: 4.86e-04 | 1323.8ms/step | 15.1 samples/s +Step 5340/44000 | Loss: 3.3122 | LR: 4.86e-04 | 1326.4ms/step | 15.1 samples/s +Step 5350/44000 | Loss: 3.3068 | LR: 4.86e-04 | 1323.5ms/step | 15.1 samples/s +Step 5360/44000 | Loss: 3.2978 | LR: 4.86e-04 | 1324.8ms/step | 15.1 samples/s +Step 5370/44000 | Loss: 3.2953 | LR: 4.86e-04 | 1325.0ms/step | 15.1 samples/s +Step 5380/44000 | Loss: 3.3104 | LR: 4.86e-04 | 1322.5ms/step | 15.1 samples/s +Step 5390/44000 | Loss: 3.3069 | LR: 4.86e-04 | 1328.1ms/step | 15.1 samples/s +Step 5400/44000 | Loss: 3.3317 | LR: 4.86e-04 | 1322.2ms/step | 15.1 samples/s +Step 5410/44000 | Loss: 3.3319 | LR: 4.86e-04 | 1326.8ms/step | 15.1 samples/s +Step 5420/44000 | Loss: 3.3107 | LR: 4.86e-04 | 1323.3ms/step | 15.1 samples/s +Step 5430/44000 | Loss: 3.3048 | LR: 4.86e-04 | 1327.1ms/step | 15.1 samples/s +Step 5440/44000 | Loss: 3.3076 | LR: 4.86e-04 | 1328.2ms/step | 15.1 samples/s +Step 5450/44000 | Loss: 3.3027 | LR: 4.86e-04 | 1324.1ms/step | 15.1 samples/s +Step 5460/44000 | Loss: 3.2814 | LR: 4.86e-04 | 1321.8ms/step | 15.1 samples/s +Step 5470/44000 | Loss: 3.2842 | LR: 4.85e-04 | 1327.4ms/step | 15.1 samples/s +Step 5480/44000 | Loss: 3.3045 | LR: 4.85e-04 | 1326.4ms/step | 15.1 samples/s +Step 5490/44000 | Loss: 3.3098 | LR: 4.85e-04 | 1331.1ms/step | 15.0 samples/s +Step 5500/44000 | Loss: 3.3248 | LR: 4.85e-04 | 1323.8ms/step | 15.1 samples/s +Validation at step 5500: Loss = 3.3148, Perplexity = 27.52 +Step 5510/44000 | Loss: 3.3260 | LR: 4.85e-04 | 3548.4ms/step | 5.6 samples/s +Step 5520/44000 | Loss: 3.2915 | LR: 4.85e-04 | 1321.3ms/step | 15.1 samples/s +Step 5530/44000 | Loss: 3.3450 | LR: 4.85e-04 | 1322.9ms/step | 15.1 samples/s +Step 5540/44000 | Loss: 3.2975 | LR: 4.85e-04 | 1324.3ms/step | 15.1 samples/s +Step 5550/44000 | Loss: 3.3044 | LR: 4.85e-04 | 1322.9ms/step | 15.1 samples/s +Step 5560/44000 | Loss: 3.2830 | LR: 4.85e-04 | 1321.8ms/step | 15.1 samples/s +Step 5570/44000 | Loss: 3.3096 | LR: 4.85e-04 | 1324.3ms/step | 15.1 samples/s +Step 5580/44000 | Loss: 3.3041 | LR: 4.85e-04 | 1321.4ms/step | 15.1 samples/s +Step 5590/44000 | Loss: 3.2793 | LR: 4.85e-04 | 1319.9ms/step | 15.2 samples/s +Step 5600/44000 | Loss: 3.3317 | LR: 4.85e-04 | 1320.2ms/step | 15.1 samples/s +Step 5610/44000 | Loss: 3.2870 | LR: 4.85e-04 | 1322.7ms/step | 15.1 samples/s +Step 5620/44000 | Loss: 3.3171 | LR: 4.85e-04 | 1320.8ms/step | 15.1 samples/s +Step 5630/44000 | Loss: 3.2714 | LR: 4.84e-04 | 1320.2ms/step | 15.1 samples/s +Step 5640/44000 | Loss: 3.3116 | LR: 4.84e-04 | 1323.3ms/step | 15.1 samples/s +Step 5650/44000 | Loss: 3.2891 | LR: 4.84e-04 | 1325.8ms/step | 15.1 samples/s +Step 5660/44000 | Loss: 3.2850 | LR: 4.84e-04 | 1325.5ms/step | 15.1 samples/s +Step 5670/44000 | Loss: 3.2968 | LR: 4.84e-04 | 1324.3ms/step | 15.1 samples/s +Step 5680/44000 | Loss: 3.3263 | LR: 4.84e-04 | 1320.8ms/step | 15.1 samples/s +Step 5690/44000 | Loss: 3.2908 | LR: 4.84e-04 | 1321.1ms/step | 15.1 samples/s +Step 5700/44000 | Loss: 3.3018 | LR: 4.84e-04 | 1324.3ms/step | 15.1 samples/s +Step 5710/44000 | Loss: 3.3054 | LR: 4.84e-04 | 1321.0ms/step | 15.1 samples/s +Step 5720/44000 | Loss: 3.3074 | LR: 4.84e-04 | 1321.3ms/step | 15.1 samples/s +Step 5730/44000 | Loss: 3.3263 | LR: 4.84e-04 | 1325.0ms/step | 15.1 samples/s +Step 5740/44000 | Loss: 3.3034 | LR: 4.84e-04 | 1325.5ms/step | 15.1 samples/s +Step 5750/44000 | Loss: 3.2860 | LR: 4.84e-04 | 1324.5ms/step | 15.1 samples/s +Validation at step 5750: Loss = 3.3000, Perplexity = 27.11 +Step 5760/44000 | Loss: 3.3026 | LR: 4.84e-04 | 3553.3ms/step | 5.6 samples/s +Step 5770/44000 | Loss: 3.3110 | LR: 4.84e-04 | 1329.0ms/step | 15.0 samples/s +Step 5780/44000 | Loss: 3.3275 | LR: 4.83e-04 | 1324.5ms/step | 15.1 samples/s +Step 5790/44000 | Loss: 3.2846 | LR: 4.83e-04 | 1322.1ms/step | 15.1 samples/s +Step 5800/44000 | Loss: 3.3035 | LR: 4.83e-04 | 1323.2ms/step | 15.1 samples/s +Step 5810/44000 | Loss: 3.2827 | LR: 4.83e-04 | 1322.2ms/step | 15.1 samples/s +Step 5820/44000 | Loss: 3.2642 | LR: 4.83e-04 | 1320.6ms/step | 15.1 samples/s +Step 5830/44000 | Loss: 3.2700 | LR: 4.83e-04 | 1330.5ms/step | 15.0 samples/s +Step 5840/44000 | Loss: 3.2731 | LR: 4.83e-04 | 1326.1ms/step | 15.1 samples/s +Step 5850/44000 | Loss: 3.2894 | LR: 4.83e-04 | 1324.4ms/step | 15.1 samples/s +Step 5860/44000 | Loss: 3.3061 | LR: 4.83e-04 | 1323.7ms/step | 15.1 samples/s +Step 5870/44000 | Loss: 3.2868 | LR: 4.83e-04 | 1322.5ms/step | 15.1 samples/s +Step 5880/44000 | Loss: 3.2757 | LR: 4.83e-04 | 1324.6ms/step | 15.1 samples/s +Step 5890/44000 | Loss: 3.3018 | LR: 4.83e-04 | 1324.2ms/step | 15.1 samples/s +Step 5900/44000 | Loss: 3.2862 | LR: 4.83e-04 | 1324.7ms/step | 15.1 samples/s +Step 5910/44000 | Loss: 3.2830 | LR: 4.83e-04 | 1323.8ms/step | 15.1 samples/s +Step 5920/44000 | Loss: 3.3280 | LR: 4.83e-04 | 1322.7ms/step | 15.1 samples/s +Step 5930/44000 | Loss: 3.2875 | LR: 4.83e-04 | 1323.3ms/step | 15.1 samples/s +Step 5940/44000 | Loss: 3.2741 | LR: 4.82e-04 | 1329.6ms/step | 15.0 samples/s +Step 5950/44000 | Loss: 3.2969 | LR: 4.82e-04 | 1326.5ms/step | 15.1 samples/s +Step 5960/44000 | Loss: 3.2704 | LR: 4.82e-04 | 1325.0ms/step | 15.1 samples/s +Step 5970/44000 | Loss: 3.2928 | LR: 4.82e-04 | 1327.0ms/step | 15.1 samples/s +Step 5980/44000 | Loss: 3.2847 | LR: 4.82e-04 | 1323.7ms/step | 15.1 samples/s +Step 5990/44000 | Loss: 3.2898 | LR: 4.82e-04 | 1321.4ms/step | 15.1 samples/s +Step 6000/44000 | Loss: 3.2671 | LR: 4.82e-04 | 1326.8ms/step | 15.1 samples/s +Validation at step 6000: Loss = 3.2898, Perplexity = 26.84 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-6000 +Step 6010/44000 | Loss: 3.2593 | LR: 4.82e-04 | 3858.5ms/step | 5.2 samples/s +Step 6020/44000 | Loss: 3.2959 | LR: 4.82e-04 | 1322.3ms/step | 15.1 samples/s +Step 6030/44000 | Loss: 3.2804 | LR: 4.82e-04 | 1322.3ms/step | 15.1 samples/s +Step 6040/44000 | Loss: 3.2934 | LR: 4.82e-04 | 1324.9ms/step | 15.1 samples/s +Step 6050/44000 | Loss: 3.2884 | LR: 4.82e-04 | 1322.3ms/step | 15.1 samples/s +Step 6060/44000 | Loss: 3.2989 | LR: 4.82e-04 | 1321.7ms/step | 15.1 samples/s +Step 6070/44000 | Loss: 3.2949 | LR: 4.82e-04 | 1319.3ms/step | 15.2 samples/s +Step 6080/44000 | Loss: 3.2935 | LR: 4.81e-04 | 1317.2ms/step | 15.2 samples/s +Step 6090/44000 | Loss: 3.3010 | LR: 4.81e-04 | 1315.5ms/step | 15.2 samples/s +Step 6100/44000 | Loss: 3.2919 | LR: 4.81e-04 | 1319.9ms/step | 15.2 samples/s +Step 6110/44000 | Loss: 3.2900 | LR: 4.81e-04 | 1319.2ms/step | 15.2 samples/s +Step 6120/44000 | Loss: 3.2906 | LR: 4.81e-04 | 1318.5ms/step | 15.2 samples/s +Step 6130/44000 | Loss: 3.3214 | LR: 4.81e-04 | 1321.6ms/step | 15.1 samples/s +Step 6140/44000 | Loss: 3.2944 | LR: 4.81e-04 | 1319.9ms/step | 15.2 samples/s +Step 6150/44000 | Loss: 3.2625 | LR: 4.81e-04 | 1320.7ms/step | 15.1 samples/s +Step 6160/44000 | Loss: 3.2713 | LR: 4.81e-04 | 1317.6ms/step | 15.2 samples/s +Step 6170/44000 | Loss: 3.2471 | LR: 4.81e-04 | 1319.1ms/step | 15.2 samples/s +Step 6180/44000 | Loss: 3.2814 | LR: 4.81e-04 | 1322.2ms/step | 15.1 samples/s +Step 6190/44000 | Loss: 3.2771 | LR: 4.81e-04 | 1322.4ms/step | 15.1 samples/s +Step 6200/44000 | Loss: 3.3048 | LR: 4.81e-04 | 1323.5ms/step | 15.1 samples/s +Step 6210/44000 | Loss: 3.2882 | LR: 4.81e-04 | 1324.2ms/step | 15.1 samples/s +Step 6220/44000 | Loss: 3.2762 | LR: 4.81e-04 | 1324.8ms/step | 15.1 samples/s +Step 6230/44000 | Loss: 3.2504 | LR: 4.80e-04 | 1324.2ms/step | 15.1 samples/s +Step 6240/44000 | Loss: 3.2838 | LR: 4.80e-04 | 1322.3ms/step | 15.1 samples/s +Step 6250/44000 | Loss: 3.2431 | LR: 4.80e-04 | 1323.3ms/step | 15.1 samples/s +Validation at step 6250: Loss = 3.2786, Perplexity = 26.54 +Step 6260/44000 | Loss: 3.2656 | LR: 4.80e-04 | 3557.2ms/step | 5.6 samples/s +Step 6270/44000 | Loss: 3.2587 | LR: 4.80e-04 | 1322.2ms/step | 15.1 samples/s +Step 6280/44000 | Loss: 3.2519 | LR: 4.80e-04 | 1324.5ms/step | 15.1 samples/s +Step 6290/44000 | Loss: 3.2626 | LR: 4.80e-04 | 1325.6ms/step | 15.1 samples/s +Step 6300/44000 | Loss: 3.2688 | LR: 4.80e-04 | 1324.1ms/step | 15.1 samples/s +Step 6310/44000 | Loss: 3.2781 | LR: 4.80e-04 | 1323.4ms/step | 15.1 samples/s +Step 6320/44000 | Loss: 3.3283 | LR: 4.80e-04 | 1322.1ms/step | 15.1 samples/s +Step 6330/44000 | Loss: 3.2869 | LR: 4.80e-04 | 1324.1ms/step | 15.1 samples/s +Step 6340/44000 | Loss: 3.2655 | LR: 4.80e-04 | 1321.2ms/step | 15.1 samples/s +Step 6350/44000 | Loss: 3.2777 | LR: 4.80e-04 | 1320.2ms/step | 15.1 samples/s +Step 6360/44000 | Loss: 3.2424 | LR: 4.80e-04 | 1326.3ms/step | 15.1 samples/s +Step 6370/44000 | Loss: 3.2649 | LR: 4.79e-04 | 1328.2ms/step | 15.1 samples/s +Step 6380/44000 | Loss: 3.2589 | LR: 4.79e-04 | 1325.3ms/step | 15.1 samples/s +Step 6390/44000 | Loss: 3.2884 | LR: 4.79e-04 | 1324.2ms/step | 15.1 samples/s +Step 6400/44000 | Loss: 3.2709 | LR: 4.79e-04 | 1325.9ms/step | 15.1 samples/s +Step 6410/44000 | Loss: 3.2634 | LR: 4.79e-04 | 1324.8ms/step | 15.1 samples/s +Step 6420/44000 | Loss: 3.2760 | LR: 4.79e-04 | 1324.2ms/step | 15.1 samples/s +Step 6430/44000 | Loss: 3.2535 | LR: 4.79e-04 | 1320.4ms/step | 15.1 samples/s +Step 6440/44000 | Loss: 3.2431 | LR: 4.79e-04 | 1324.8ms/step | 15.1 samples/s +Step 6450/44000 | Loss: 3.2881 | LR: 4.79e-04 | 1325.6ms/step | 15.1 samples/s +Step 6460/44000 | Loss: 3.2977 | LR: 4.79e-04 | 1327.2ms/step | 15.1 samples/s +Step 6470/44000 | Loss: 3.2177 | LR: 4.79e-04 | 1323.5ms/step | 15.1 samples/s +Step 6480/44000 | Loss: 3.2847 | LR: 4.79e-04 | 1320.6ms/step | 15.1 samples/s +Step 6490/44000 | Loss: 3.2427 | LR: 4.79e-04 | 1324.4ms/step | 15.1 samples/s +Step 6500/44000 | Loss: 3.2708 | LR: 4.79e-04 | 1328.2ms/step | 15.1 samples/s +Validation at step 6500: Loss = 3.2667, Perplexity = 26.22 +Step 6510/44000 | Loss: 3.2960 | LR: 4.78e-04 | 3543.7ms/step | 5.6 samples/s +Step 6520/44000 | Loss: 3.2761 | LR: 4.78e-04 | 1321.3ms/step | 15.1 samples/s +Step 6530/44000 | Loss: 3.2883 | LR: 4.78e-04 | 1321.9ms/step | 15.1 samples/s +Step 6540/44000 | Loss: 3.2569 | LR: 4.78e-04 | 1322.9ms/step | 15.1 samples/s +Step 6550/44000 | Loss: 3.2414 | LR: 4.78e-04 | 1324.4ms/step | 15.1 samples/s +Step 6560/44000 | Loss: 3.2726 | LR: 4.78e-04 | 1320.7ms/step | 15.1 samples/s +Step 6570/44000 | Loss: 3.2718 | LR: 4.78e-04 | 1322.7ms/step | 15.1 samples/s +Step 6580/44000 | Loss: 3.2769 | LR: 4.78e-04 | 1322.0ms/step | 15.1 samples/s +Step 6590/44000 | Loss: 3.2674 | LR: 4.78e-04 | 1322.1ms/step | 15.1 samples/s +Step 6600/44000 | Loss: 3.2896 | LR: 4.78e-04 | 1322.5ms/step | 15.1 samples/s +Step 6610/44000 | Loss: 3.2493 | LR: 4.78e-04 | 1324.5ms/step | 15.1 samples/s +Step 6620/44000 | Loss: 3.2817 | LR: 4.78e-04 | 1327.9ms/step | 15.1 samples/s +Step 6630/44000 | Loss: 3.2407 | LR: 4.78e-04 | 1326.3ms/step | 15.1 samples/s +Step 6640/44000 | Loss: 3.2846 | LR: 4.77e-04 | 1325.2ms/step | 15.1 samples/s +Step 6650/44000 | Loss: 3.2502 | LR: 4.77e-04 | 1323.4ms/step | 15.1 samples/s +Step 6660/44000 | Loss: 3.2474 | LR: 4.77e-04 | 1323.5ms/step | 15.1 samples/s +Step 6670/44000 | Loss: 3.2712 | LR: 4.77e-04 | 1320.7ms/step | 15.1 samples/s +Step 6680/44000 | Loss: 3.2237 | LR: 4.77e-04 | 1325.8ms/step | 15.1 samples/s +Step 6690/44000 | Loss: 3.2770 | LR: 4.77e-04 | 1322.8ms/step | 15.1 samples/s +Step 6700/44000 | Loss: 3.2668 | LR: 4.77e-04 | 1327.9ms/step | 15.1 samples/s +Step 6710/44000 | Loss: 3.2495 | LR: 4.77e-04 | 1324.1ms/step | 15.1 samples/s +Step 6720/44000 | Loss: 3.2150 | LR: 4.77e-04 | 1324.7ms/step | 15.1 samples/s +Step 6730/44000 | Loss: 3.2325 | LR: 4.77e-04 | 1323.9ms/step | 15.1 samples/s +Step 6740/44000 | Loss: 3.2453 | LR: 4.77e-04 | 1323.8ms/step | 15.1 samples/s +Step 6750/44000 | Loss: 3.2426 | LR: 4.77e-04 | 1324.9ms/step | 15.1 samples/s +Validation at step 6750: Loss = 3.2566, Perplexity = 25.96 +Step 6760/44000 | Loss: 3.2685 | LR: 4.77e-04 | 3554.3ms/step | 5.6 samples/s +Step 6770/44000 | Loss: 3.2454 | LR: 4.76e-04 | 1323.5ms/step | 15.1 samples/s +Step 6780/44000 | Loss: 3.2505 | LR: 4.76e-04 | 1325.8ms/step | 15.1 samples/s +Step 6790/44000 | Loss: 3.2436 | LR: 4.76e-04 | 1324.8ms/step | 15.1 samples/s +Step 6800/44000 | Loss: 3.2883 | LR: 4.76e-04 | 1327.6ms/step | 15.1 samples/s +Step 6810/44000 | Loss: 3.2049 | LR: 4.76e-04 | 1327.4ms/step | 15.1 samples/s +Step 6820/44000 | Loss: 3.2595 | LR: 4.76e-04 | 1324.0ms/step | 15.1 samples/s +Step 6830/44000 | Loss: 3.2569 | LR: 4.76e-04 | 1322.9ms/step | 15.1 samples/s +Step 6840/44000 | Loss: 3.2601 | LR: 4.76e-04 | 1325.0ms/step | 15.1 samples/s +Step 6850/44000 | Loss: 3.2326 | LR: 4.76e-04 | 1323.4ms/step | 15.1 samples/s +Step 6860/44000 | Loss: 3.2229 | LR: 4.76e-04 | 1324.9ms/step | 15.1 samples/s +Step 6870/44000 | Loss: 3.2725 | LR: 4.76e-04 | 1321.1ms/step | 15.1 samples/s +Step 6880/44000 | Loss: 3.2724 | LR: 4.76e-04 | 1320.1ms/step | 15.2 samples/s +Step 6890/44000 | Loss: 3.2404 | LR: 4.76e-04 | 1321.2ms/step | 15.1 samples/s +Step 6900/44000 | Loss: 3.2576 | LR: 4.75e-04 | 1323.6ms/step | 15.1 samples/s +Step 6910/44000 | Loss: 3.2418 | LR: 4.75e-04 | 1324.1ms/step | 15.1 samples/s +Step 6920/44000 | Loss: 3.2789 | LR: 4.75e-04 | 1327.3ms/step | 15.1 samples/s +Step 6930/44000 | Loss: 3.2437 | LR: 4.75e-04 | 1325.0ms/step | 15.1 samples/s +Step 6940/44000 | Loss: 3.2452 | LR: 4.75e-04 | 1320.4ms/step | 15.1 samples/s +Step 6950/44000 | Loss: 3.2427 | LR: 4.75e-04 | 1326.5ms/step | 15.1 samples/s +Step 6960/44000 | Loss: 3.2419 | LR: 4.75e-04 | 1325.4ms/step | 15.1 samples/s +Step 6970/44000 | Loss: 3.2407 | LR: 4.75e-04 | 1330.1ms/step | 15.0 samples/s +Step 6980/44000 | Loss: 3.2267 | LR: 4.75e-04 | 1321.5ms/step | 15.1 samples/s +Step 6990/44000 | Loss: 3.2332 | LR: 4.75e-04 | 1322.0ms/step | 15.1 samples/s +Step 7000/44000 | Loss: 3.2605 | LR: 4.75e-04 | 1322.1ms/step | 15.1 samples/s +Validation at step 7000: Loss = 3.2471, Perplexity = 25.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-7000 +Step 7010/44000 | Loss: 3.2652 | LR: 4.75e-04 | 3838.8ms/step | 5.2 samples/s +Step 7020/44000 | Loss: 3.2369 | LR: 4.75e-04 | 1324.9ms/step | 15.1 samples/s +Step 7030/44000 | Loss: 3.2267 | LR: 4.74e-04 | 1321.6ms/step | 15.1 samples/s +Step 7040/44000 | Loss: 3.2227 | LR: 4.74e-04 | 1326.7ms/step | 15.1 samples/s +Step 7050/44000 | Loss: 3.2288 | LR: 4.74e-04 | 1325.7ms/step | 15.1 samples/s +Step 7060/44000 | Loss: 3.2506 | LR: 4.74e-04 | 1327.6ms/step | 15.1 samples/s +Step 7070/44000 | Loss: 3.2407 | LR: 4.74e-04 | 1323.9ms/step | 15.1 samples/s +Step 7080/44000 | Loss: 3.2285 | LR: 4.74e-04 | 1322.2ms/step | 15.1 samples/s +Step 7090/44000 | Loss: 3.2428 | LR: 4.74e-04 | 1322.8ms/step | 15.1 samples/s +Step 7100/44000 | Loss: 3.2288 | LR: 4.74e-04 | 1326.5ms/step | 15.1 samples/s +Step 7110/44000 | Loss: 3.2566 | LR: 4.74e-04 | 1323.7ms/step | 15.1 samples/s +Step 7120/44000 | Loss: 3.2337 | LR: 4.74e-04 | 1326.3ms/step | 15.1 samples/s +Step 7130/44000 | Loss: 3.2632 | LR: 4.74e-04 | 1323.8ms/step | 15.1 samples/s +Step 7140/44000 | Loss: 3.2313 | LR: 4.74e-04 | 1326.0ms/step | 15.1 samples/s +Step 7150/44000 | Loss: 3.2564 | LR: 4.73e-04 | 1323.0ms/step | 15.1 samples/s +Step 7160/44000 | Loss: 3.2463 | LR: 4.73e-04 | 1327.2ms/step | 15.1 samples/s +Step 7170/44000 | Loss: 3.2330 | LR: 4.73e-04 | 1329.0ms/step | 15.0 samples/s +Step 7180/44000 | Loss: 3.2615 | LR: 4.73e-04 | 1321.6ms/step | 15.1 samples/s +Step 7190/44000 | Loss: 3.2228 | LR: 4.73e-04 | 1327.1ms/step | 15.1 samples/s +Step 7200/44000 | Loss: 3.2399 | LR: 4.73e-04 | 1322.8ms/step | 15.1 samples/s +Step 7210/44000 | Loss: 3.1862 | LR: 4.73e-04 | 1322.7ms/step | 15.1 samples/s +Step 7220/44000 | Loss: 3.2302 | LR: 4.73e-04 | 1324.2ms/step | 15.1 samples/s +Step 7230/44000 | Loss: 3.2274 | LR: 4.73e-04 | 1324.9ms/step | 15.1 samples/s +Step 7240/44000 | Loss: 3.2178 | LR: 4.73e-04 | 1324.0ms/step | 15.1 samples/s +Step 7250/44000 | Loss: 3.2130 | LR: 4.73e-04 | 1322.5ms/step | 15.1 samples/s +Validation at step 7250: Loss = 3.2403, Perplexity = 25.54 +Step 7260/44000 | Loss: 3.2226 | LR: 4.73e-04 | 3552.5ms/step | 5.6 samples/s +Step 7270/44000 | Loss: 3.2518 | LR: 4.72e-04 | 1323.4ms/step | 15.1 samples/s +Step 7280/44000 | Loss: 3.2927 | LR: 4.72e-04 | 1324.6ms/step | 15.1 samples/s +Step 7290/44000 | Loss: 3.2171 | LR: 4.72e-04 | 1326.3ms/step | 15.1 samples/s +Step 7300/44000 | Loss: 3.2405 | LR: 4.72e-04 | 1326.5ms/step | 15.1 samples/s +Step 7310/44000 | Loss: 3.2209 | LR: 4.72e-04 | 1322.4ms/step | 15.1 samples/s +Step 7320/44000 | Loss: 3.2288 | LR: 4.72e-04 | 1327.7ms/step | 15.1 samples/s +Step 7330/44000 | Loss: 3.2437 | LR: 4.72e-04 | 1327.1ms/step | 15.1 samples/s +Step 7340/44000 | Loss: 3.2249 | LR: 4.72e-04 | 1320.8ms/step | 15.1 samples/s +Step 7350/44000 | Loss: 3.2412 | LR: 4.72e-04 | 1325.0ms/step | 15.1 samples/s +Step 7360/44000 | Loss: 3.2242 | LR: 4.72e-04 | 1322.4ms/step | 15.1 samples/s +Step 7370/44000 | Loss: 3.2335 | LR: 4.72e-04 | 1321.4ms/step | 15.1 samples/s +Step 7380/44000 | Loss: 3.2227 | LR: 4.72e-04 | 1321.1ms/step | 15.1 samples/s +Step 7390/44000 | Loss: 3.1980 | LR: 4.71e-04 | 1319.7ms/step | 15.2 samples/s +Step 7400/44000 | Loss: 3.2193 | LR: 4.71e-04 | 1320.5ms/step | 15.1 samples/s +Step 7410/44000 | Loss: 3.2269 | LR: 4.71e-04 | 1322.0ms/step | 15.1 samples/s +Step 7420/44000 | Loss: 3.2415 | LR: 4.71e-04 | 1322.2ms/step | 15.1 samples/s +Step 7430/44000 | Loss: 3.2423 | LR: 4.71e-04 | 1324.1ms/step | 15.1 samples/s +Step 7440/44000 | Loss: 3.2383 | LR: 4.71e-04 | 1328.8ms/step | 15.1 samples/s +Step 7450/44000 | Loss: 3.2047 | LR: 4.71e-04 | 1326.7ms/step | 15.1 samples/s +Step 7460/44000 | Loss: 3.2150 | LR: 4.71e-04 | 1321.7ms/step | 15.1 samples/s +Step 7470/44000 | Loss: 3.2510 | LR: 4.71e-04 | 1319.3ms/step | 15.2 samples/s +Step 7480/44000 | Loss: 3.2274 | LR: 4.71e-04 | 1320.6ms/step | 15.1 samples/s +Step 7490/44000 | Loss: 3.2375 | LR: 4.71e-04 | 1324.0ms/step | 15.1 samples/s +Step 7500/44000 | Loss: 3.2547 | LR: 4.71e-04 | 1325.6ms/step | 15.1 samples/s +Validation at step 7500: Loss = 3.2308, Perplexity = 25.30 +Step 7510/44000 | Loss: 3.2412 | LR: 4.70e-04 | 3543.7ms/step | 5.6 samples/s +Step 7520/44000 | Loss: 3.2223 | LR: 4.70e-04 | 1319.1ms/step | 15.2 samples/s +Step 7530/44000 | Loss: 3.2525 | LR: 4.70e-04 | 1318.5ms/step | 15.2 samples/s +Step 7540/44000 | Loss: 3.2060 | LR: 4.70e-04 | 1317.6ms/step | 15.2 samples/s +Step 7550/44000 | Loss: 3.2000 | LR: 4.70e-04 | 1318.4ms/step | 15.2 samples/s +Step 7560/44000 | Loss: 3.2068 | LR: 4.70e-04 | 1319.6ms/step | 15.2 samples/s +Step 7570/44000 | Loss: 3.2105 | LR: 4.70e-04 | 1318.8ms/step | 15.2 samples/s +Step 7580/44000 | Loss: 3.2383 | LR: 4.70e-04 | 1321.5ms/step | 15.1 samples/s +Step 7590/44000 | Loss: 3.2600 | LR: 4.70e-04 | 1320.5ms/step | 15.1 samples/s +Step 7600/44000 | Loss: 3.2076 | LR: 4.70e-04 | 1320.4ms/step | 15.1 samples/s +Step 7610/44000 | Loss: 3.1961 | LR: 4.70e-04 | 1321.1ms/step | 15.1 samples/s +Step 7620/44000 | Loss: 3.2048 | LR: 4.70e-04 | 1322.6ms/step | 15.1 samples/s +Step 7630/44000 | Loss: 3.2197 | LR: 4.69e-04 | 1325.2ms/step | 15.1 samples/s +Step 7640/44000 | Loss: 3.2419 | LR: 4.69e-04 | 1324.0ms/step | 15.1 samples/s +Step 7650/44000 | Loss: 3.2314 | LR: 4.69e-04 | 1323.2ms/step | 15.1 samples/s +Step 7660/44000 | Loss: 3.1881 | LR: 4.69e-04 | 1322.9ms/step | 15.1 samples/s +Step 7670/44000 | Loss: 3.2301 | LR: 4.69e-04 | 1325.1ms/step | 15.1 samples/s +Step 7680/44000 | Loss: 3.2462 | LR: 4.69e-04 | 1325.9ms/step | 15.1 samples/s +Step 7690/44000 | Loss: 3.2193 | LR: 4.69e-04 | 1324.5ms/step | 15.1 samples/s +Step 7700/44000 | Loss: 3.2119 | LR: 4.69e-04 | 1322.4ms/step | 15.1 samples/s +Step 7710/44000 | Loss: 3.2270 | LR: 4.69e-04 | 1323.4ms/step | 15.1 samples/s +Step 7720/44000 | Loss: 3.2277 | LR: 4.69e-04 | 1324.9ms/step | 15.1 samples/s +Step 7730/44000 | Loss: 3.1939 | LR: 4.69e-04 | 1328.2ms/step | 15.1 samples/s +Step 7740/44000 | Loss: 3.2423 | LR: 4.68e-04 | 1325.8ms/step | 15.1 samples/s +Step 7750/44000 | Loss: 3.2506 | LR: 4.68e-04 | 1325.0ms/step | 15.1 samples/s +Validation at step 7750: Loss = 3.2222, Perplexity = 25.08 +Step 7760/44000 | Loss: 3.2173 | LR: 4.68e-04 | 3551.7ms/step | 5.6 samples/s +Step 7770/44000 | Loss: 3.2200 | LR: 4.68e-04 | 1322.5ms/step | 15.1 samples/s +Step 7780/44000 | Loss: 3.2448 | LR: 4.68e-04 | 1322.6ms/step | 15.1 samples/s +Step 7790/44000 | Loss: 3.1901 | LR: 4.68e-04 | 1325.1ms/step | 15.1 samples/s +Step 7800/44000 | Loss: 3.2396 | LR: 4.68e-04 | 1325.5ms/step | 15.1 samples/s +Step 7810/44000 | Loss: 3.2141 | LR: 4.68e-04 | 1323.7ms/step | 15.1 samples/s +Step 7820/44000 | Loss: 3.2571 | LR: 4.68e-04 | 1327.6ms/step | 15.1 samples/s +Step 7830/44000 | Loss: 3.2510 | LR: 4.68e-04 | 1327.7ms/step | 15.1 samples/s +Step 7840/44000 | Loss: 3.2175 | LR: 4.68e-04 | 1325.4ms/step | 15.1 samples/s +Step 7850/44000 | Loss: 3.2206 | LR: 4.67e-04 | 1325.8ms/step | 15.1 samples/s +Step 7860/44000 | Loss: 3.1909 | LR: 4.67e-04 | 1324.7ms/step | 15.1 samples/s +Step 7870/44000 | Loss: 3.2241 | LR: 4.67e-04 | 1324.3ms/step | 15.1 samples/s +Step 7880/44000 | Loss: 3.2200 | LR: 4.67e-04 | 1323.6ms/step | 15.1 samples/s +Step 7890/44000 | Loss: 3.2333 | LR: 4.67e-04 | 1324.2ms/step | 15.1 samples/s +Step 7900/44000 | Loss: 3.2295 | LR: 4.67e-04 | 1325.7ms/step | 15.1 samples/s +Step 7910/44000 | Loss: 3.2225 | LR: 4.67e-04 | 1323.5ms/step | 15.1 samples/s +Step 7920/44000 | Loss: 3.2115 | LR: 4.67e-04 | 1325.4ms/step | 15.1 samples/s +Step 7930/44000 | Loss: 3.1967 | LR: 4.67e-04 | 1325.8ms/step | 15.1 samples/s +Step 7940/44000 | Loss: 3.2459 | LR: 4.67e-04 | 1321.9ms/step | 15.1 samples/s +Step 7950/44000 | Loss: 3.2131 | LR: 4.67e-04 | 1321.6ms/step | 15.1 samples/s +Step 7960/44000 | Loss: 3.2155 | LR: 4.66e-04 | 1323.1ms/step | 15.1 samples/s +Step 7970/44000 | Loss: 3.2114 | LR: 4.66e-04 | 1322.2ms/step | 15.1 samples/s +Step 7980/44000 | Loss: 3.1927 | LR: 4.66e-04 | 1323.6ms/step | 15.1 samples/s +Step 7990/44000 | Loss: 3.2064 | LR: 4.66e-04 | 1323.6ms/step | 15.1 samples/s +Step 8000/44000 | Loss: 3.2559 | LR: 4.66e-04 | 1327.0ms/step | 15.1 samples/s +Validation at step 8000: Loss = 3.2137, Perplexity = 24.87 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-8000 +Step 8010/44000 | Loss: 3.1999 | LR: 4.66e-04 | 3862.7ms/step | 5.2 samples/s +Step 8020/44000 | Loss: 3.2150 | LR: 4.66e-04 | 1323.3ms/step | 15.1 samples/s +Step 8030/44000 | Loss: 3.2233 | LR: 4.66e-04 | 1323.8ms/step | 15.1 samples/s +Step 8040/44000 | Loss: 3.1764 | LR: 4.66e-04 | 1321.9ms/step | 15.1 samples/s +Step 8050/44000 | Loss: 3.2080 | LR: 4.66e-04 | 1323.4ms/step | 15.1 samples/s +Step 8060/44000 | Loss: 3.1868 | LR: 4.66e-04 | 1322.2ms/step | 15.1 samples/s +Step 8070/44000 | Loss: 3.2052 | LR: 4.65e-04 | 1323.5ms/step | 15.1 samples/s +Step 8080/44000 | Loss: 3.2212 | LR: 4.65e-04 | 1323.1ms/step | 15.1 samples/s +Step 8090/44000 | Loss: 3.1837 | LR: 4.65e-04 | 1322.8ms/step | 15.1 samples/s +Step 8100/44000 | Loss: 3.2146 | LR: 4.65e-04 | 1322.9ms/step | 15.1 samples/s +Step 8110/44000 | Loss: 3.2012 | LR: 4.65e-04 | 1323.3ms/step | 15.1 samples/s +Step 8120/44000 | Loss: 3.2052 | LR: 4.65e-04 | 1324.9ms/step | 15.1 samples/s +Step 8130/44000 | Loss: 3.1977 | LR: 4.65e-04 | 1323.5ms/step | 15.1 samples/s +Step 8140/44000 | Loss: 3.2009 | LR: 4.65e-04 | 1325.3ms/step | 15.1 samples/s +Step 8150/44000 | Loss: 3.2357 | LR: 4.65e-04 | 1326.0ms/step | 15.1 samples/s +Step 8160/44000 | Loss: 3.1873 | LR: 4.65e-04 | 1324.5ms/step | 15.1 samples/s +Step 8170/44000 | Loss: 3.1962 | LR: 4.65e-04 | 1327.3ms/step | 15.1 samples/s +Step 8180/44000 | Loss: 3.1979 | LR: 4.64e-04 | 1325.5ms/step | 15.1 samples/s +Step 8190/44000 | Loss: 3.2130 | LR: 4.64e-04 | 1327.2ms/step | 15.1 samples/s +Step 8200/44000 | Loss: 3.2344 | LR: 4.64e-04 | 1323.7ms/step | 15.1 samples/s +Step 8210/44000 | Loss: 3.2015 | LR: 4.64e-04 | 1323.2ms/step | 15.1 samples/s +Step 8220/44000 | Loss: 3.2581 | LR: 4.64e-04 | 1325.9ms/step | 15.1 samples/s +Step 8230/44000 | Loss: 3.1728 | LR: 4.64e-04 | 1327.1ms/step | 15.1 samples/s +Step 8240/44000 | Loss: 3.2193 | LR: 4.64e-04 | 1323.4ms/step | 15.1 samples/s +Step 8250/44000 | Loss: 3.2132 | LR: 4.64e-04 | 1327.0ms/step | 15.1 samples/s +Validation at step 8250: Loss = 3.2070, Perplexity = 24.70 +Step 8260/44000 | Loss: 3.2119 | LR: 4.64e-04 | 3551.8ms/step | 5.6 samples/s +Step 8270/44000 | Loss: 3.2111 | LR: 4.64e-04 | 1324.0ms/step | 15.1 samples/s +Step 8280/44000 | Loss: 3.1991 | LR: 4.64e-04 | 1323.5ms/step | 15.1 samples/s +Step 8290/44000 | Loss: 3.1980 | LR: 4.63e-04 | 1319.6ms/step | 15.2 samples/s +Step 8300/44000 | Loss: 3.1834 | LR: 4.63e-04 | 1323.4ms/step | 15.1 samples/s +Step 8310/44000 | Loss: 3.1984 | LR: 4.63e-04 | 1324.3ms/step | 15.1 samples/s +Step 8320/44000 | Loss: 3.1947 | LR: 4.63e-04 | 1322.9ms/step | 15.1 samples/s +Step 8330/44000 | Loss: 3.1820 | LR: 4.63e-04 | 1325.2ms/step | 15.1 samples/s +Step 8340/44000 | Loss: 3.1836 | LR: 4.63e-04 | 1324.7ms/step | 15.1 samples/s +Step 8350/44000 | Loss: 3.1750 | LR: 4.63e-04 | 1323.9ms/step | 15.1 samples/s +Step 8360/44000 | Loss: 3.1966 | LR: 4.63e-04 | 1323.0ms/step | 15.1 samples/s +Step 8370/44000 | Loss: 3.2079 | LR: 4.63e-04 | 1325.6ms/step | 15.1 samples/s +Step 8380/44000 | Loss: 3.2008 | LR: 4.63e-04 | 1321.1ms/step | 15.1 samples/s +Step 8390/44000 | Loss: 3.1819 | LR: 4.62e-04 | 1321.7ms/step | 15.1 samples/s +Step 8400/44000 | Loss: 3.1949 | LR: 4.62e-04 | 1321.3ms/step | 15.1 samples/s +Step 8410/44000 | Loss: 3.1896 | LR: 4.62e-04 | 1325.3ms/step | 15.1 samples/s +Step 8420/44000 | Loss: 3.1967 | LR: 4.62e-04 | 1326.1ms/step | 15.1 samples/s +Step 8430/44000 | Loss: 3.1977 | LR: 4.62e-04 | 1324.7ms/step | 15.1 samples/s +Step 8440/44000 | Loss: 3.1794 | LR: 4.62e-04 | 1323.4ms/step | 15.1 samples/s +Step 8450/44000 | Loss: 3.2498 | LR: 4.62e-04 | 1325.4ms/step | 15.1 samples/s +Step 8460/44000 | Loss: 3.2119 | LR: 4.62e-04 | 1324.6ms/step | 15.1 samples/s +Step 8470/44000 | Loss: 3.1842 | LR: 4.62e-04 | 1320.9ms/step | 15.1 samples/s +Step 8480/44000 | Loss: 3.1957 | LR: 4.62e-04 | 1324.2ms/step | 15.1 samples/s +Step 8490/44000 | Loss: 3.2220 | LR: 4.62e-04 | 1325.6ms/step | 15.1 samples/s +Step 8500/44000 | Loss: 3.2141 | LR: 4.61e-04 | 1321.1ms/step | 15.1 samples/s +Validation at step 8500: Loss = 3.1988, Perplexity = 24.50 +Step 8510/44000 | Loss: 3.1820 | LR: 4.61e-04 | 3546.3ms/step | 5.6 samples/s +Step 8520/44000 | Loss: 3.2000 | LR: 4.61e-04 | 1328.6ms/step | 15.1 samples/s +Step 8530/44000 | Loss: 3.1931 | LR: 4.61e-04 | 1323.9ms/step | 15.1 samples/s +Step 8540/44000 | Loss: 3.1720 | LR: 4.61e-04 | 1325.2ms/step | 15.1 samples/s +Step 8550/44000 | Loss: 3.2045 | LR: 4.61e-04 | 1324.0ms/step | 15.1 samples/s +Step 8560/44000 | Loss: 3.1739 | LR: 4.61e-04 | 1325.3ms/step | 15.1 samples/s +Step 8570/44000 | Loss: 3.2217 | LR: 4.61e-04 | 1320.9ms/step | 15.1 samples/s +Step 8580/44000 | Loss: 3.1566 | LR: 4.61e-04 | 1323.3ms/step | 15.1 samples/s +Step 8590/44000 | Loss: 3.1746 | LR: 4.61e-04 | 1325.4ms/step | 15.1 samples/s +Step 8600/44000 | Loss: 3.1794 | LR: 4.60e-04 | 1325.4ms/step | 15.1 samples/s +Step 8610/44000 | Loss: 3.2042 | LR: 4.60e-04 | 1324.2ms/step | 15.1 samples/s +Step 8620/44000 | Loss: 3.1739 | LR: 4.60e-04 | 1322.3ms/step | 15.1 samples/s +Step 8630/44000 | Loss: 3.1995 | LR: 4.60e-04 | 1323.6ms/step | 15.1 samples/s +Step 8640/44000 | Loss: 3.2019 | LR: 4.60e-04 | 1325.9ms/step | 15.1 samples/s +Step 8650/44000 | Loss: 3.1764 | LR: 4.60e-04 | 1323.8ms/step | 15.1 samples/s +Step 8660/44000 | Loss: 3.2005 | LR: 4.60e-04 | 1323.4ms/step | 15.1 samples/s +Step 8670/44000 | Loss: 3.1924 | LR: 4.60e-04 | 1321.4ms/step | 15.1 samples/s +Step 8680/44000 | Loss: 3.1857 | LR: 4.60e-04 | 1321.6ms/step | 15.1 samples/s +Step 8690/44000 | Loss: 3.1859 | LR: 4.60e-04 | 1326.1ms/step | 15.1 samples/s +Step 8700/44000 | Loss: 3.1552 | LR: 4.59e-04 | 1326.0ms/step | 15.1 samples/s +Step 8710/44000 | Loss: 3.1690 | LR: 4.59e-04 | 1325.2ms/step | 15.1 samples/s +Step 8720/44000 | Loss: 3.1610 | LR: 4.59e-04 | 1326.2ms/step | 15.1 samples/s +Step 8730/44000 | Loss: 3.1474 | LR: 4.59e-04 | 1333.9ms/step | 15.0 samples/s +Step 8740/44000 | Loss: 3.1979 | LR: 4.59e-04 | 1325.3ms/step | 15.1 samples/s +Step 8750/44000 | Loss: 3.1827 | LR: 4.59e-04 | 1324.6ms/step | 15.1 samples/s +Validation at step 8750: Loss = 3.1931, Perplexity = 24.36 +Step 8760/44000 | Loss: 3.1545 | LR: 4.59e-04 | 3553.4ms/step | 5.6 samples/s +Step 8770/44000 | Loss: 3.1804 | LR: 4.59e-04 | 1320.2ms/step | 15.1 samples/s +Step 8780/44000 | Loss: 3.1867 | LR: 4.59e-04 | 1321.1ms/step | 15.1 samples/s +Step 8790/44000 | Loss: 3.1811 | LR: 4.59e-04 | 1320.8ms/step | 15.1 samples/s +Step 8800/44000 | Loss: 3.1984 | LR: 4.58e-04 | 1320.9ms/step | 15.1 samples/s +Step 8810/44000 | Loss: 3.2112 | LR: 4.58e-04 | 1324.9ms/step | 15.1 samples/s +Step 8820/44000 | Loss: 3.1738 | LR: 4.58e-04 | 1324.1ms/step | 15.1 samples/s +Step 8830/44000 | Loss: 3.1664 | LR: 4.58e-04 | 1319.6ms/step | 15.2 samples/s +Step 8840/44000 | Loss: 3.1966 | LR: 4.58e-04 | 1320.2ms/step | 15.1 samples/s +Step 8850/44000 | Loss: 3.2273 | LR: 4.58e-04 | 1321.5ms/step | 15.1 samples/s +Step 8860/44000 | Loss: 3.1635 | LR: 4.58e-04 | 1319.1ms/step | 15.2 samples/s +Step 8870/44000 | Loss: 3.1832 | LR: 4.58e-04 | 1323.6ms/step | 15.1 samples/s +Step 8880/44000 | Loss: 3.1675 | LR: 4.58e-04 | 1319.4ms/step | 15.2 samples/s +Step 8890/44000 | Loss: 3.1831 | LR: 4.58e-04 | 1320.0ms/step | 15.2 samples/s +Step 8900/44000 | Loss: 3.1966 | LR: 4.57e-04 | 1317.6ms/step | 15.2 samples/s +Step 8910/44000 | Loss: 3.1911 | LR: 4.57e-04 | 1318.5ms/step | 15.2 samples/s +Step 8920/44000 | Loss: 3.1997 | LR: 4.57e-04 | 1317.7ms/step | 15.2 samples/s +Step 8930/44000 | Loss: 3.1575 | LR: 4.57e-04 | 1319.6ms/step | 15.2 samples/s +Step 8940/44000 | Loss: 3.1645 | LR: 4.57e-04 | 1314.8ms/step | 15.2 samples/s +Step 8950/44000 | Loss: 3.1566 | LR: 4.57e-04 | 1318.3ms/step | 15.2 samples/s +Step 8960/44000 | Loss: 3.1808 | LR: 4.57e-04 | 1320.5ms/step | 15.1 samples/s +Step 8970/44000 | Loss: 3.1873 | LR: 4.57e-04 | 1323.0ms/step | 15.1 samples/s +Step 8980/44000 | Loss: 3.1855 | LR: 4.57e-04 | 1322.6ms/step | 15.1 samples/s +Step 8990/44000 | Loss: 3.1713 | LR: 4.57e-04 | 1320.4ms/step | 15.1 samples/s +Step 9000/44000 | Loss: 3.1736 | LR: 4.56e-04 | 1320.3ms/step | 15.1 samples/s +Validation at step 9000: Loss = 3.1865, Perplexity = 24.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-9000 +Step 9010/44000 | Loss: 3.2100 | LR: 4.56e-04 | 3853.0ms/step | 5.2 samples/s +Step 9020/44000 | Loss: 3.2026 | LR: 4.56e-04 | 1320.6ms/step | 15.1 samples/s +Step 9030/44000 | Loss: 3.1894 | LR: 4.56e-04 | 1322.7ms/step | 15.1 samples/s +Step 9040/44000 | Loss: 3.1405 | LR: 4.56e-04 | 1326.1ms/step | 15.1 samples/s +Step 9050/44000 | Loss: 3.1492 | LR: 4.56e-04 | 1324.1ms/step | 15.1 samples/s +Step 9060/44000 | Loss: 3.1785 | LR: 4.56e-04 | 1326.3ms/step | 15.1 samples/s +Step 9070/44000 | Loss: 3.1935 | LR: 4.56e-04 | 1327.7ms/step | 15.1 samples/s +Step 9080/44000 | Loss: 3.1852 | LR: 4.56e-04 | 1326.9ms/step | 15.1 samples/s +Step 9090/44000 | Loss: 3.1872 | LR: 4.56e-04 | 1325.7ms/step | 15.1 samples/s +Step 9100/44000 | Loss: 3.1807 | LR: 4.55e-04 | 1327.0ms/step | 15.1 samples/s +Step 9110/44000 | Loss: 3.1719 | LR: 4.55e-04 | 1324.7ms/step | 15.1 samples/s +Step 9120/44000 | Loss: 3.1687 | LR: 4.55e-04 | 1328.1ms/step | 15.1 samples/s +Step 9130/44000 | Loss: 3.1422 | LR: 4.55e-04 | 1325.9ms/step | 15.1 samples/s +Step 9140/44000 | Loss: 3.1540 | LR: 4.55e-04 | 1324.1ms/step | 15.1 samples/s +Step 9150/44000 | Loss: 3.2044 | LR: 4.55e-04 | 1323.0ms/step | 15.1 samples/s +Step 9160/44000 | Loss: 3.2058 | LR: 4.55e-04 | 1324.4ms/step | 15.1 samples/s +Step 9170/44000 | Loss: 3.1747 | LR: 4.55e-04 | 1326.8ms/step | 15.1 samples/s +Step 9180/44000 | Loss: 3.1643 | LR: 4.55e-04 | 1322.9ms/step | 15.1 samples/s +Step 9190/44000 | Loss: 3.2036 | LR: 4.54e-04 | 1323.7ms/step | 15.1 samples/s +Step 9200/44000 | Loss: 3.1511 | LR: 4.54e-04 | 1324.7ms/step | 15.1 samples/s +Step 9210/44000 | Loss: 3.1878 | LR: 4.54e-04 | 1325.3ms/step | 15.1 samples/s +Step 9220/44000 | Loss: 3.1573 | LR: 4.54e-04 | 1323.8ms/step | 15.1 samples/s +Step 9230/44000 | Loss: 3.1772 | LR: 4.54e-04 | 1321.9ms/step | 15.1 samples/s +Step 9240/44000 | Loss: 3.1712 | LR: 4.54e-04 | 1325.4ms/step | 15.1 samples/s +Step 9250/44000 | Loss: 3.1335 | LR: 4.54e-04 | 1322.8ms/step | 15.1 samples/s +Validation at step 9250: Loss = 3.1804, Perplexity = 24.06 +Step 9260/44000 | Loss: 3.1688 | LR: 4.54e-04 | 3550.4ms/step | 5.6 samples/s +Step 9270/44000 | Loss: 3.1723 | LR: 4.54e-04 | 1323.2ms/step | 15.1 samples/s +Step 9280/44000 | Loss: 3.1518 | LR: 4.54e-04 | 1324.1ms/step | 15.1 samples/s +Step 9290/44000 | Loss: 3.1837 | LR: 4.53e-04 | 1324.0ms/step | 15.1 samples/s +Step 9300/44000 | Loss: 3.1751 | LR: 4.53e-04 | 1325.8ms/step | 15.1 samples/s +Step 9310/44000 | Loss: 3.1645 | LR: 4.53e-04 | 1325.4ms/step | 15.1 samples/s +Step 9320/44000 | Loss: 3.1897 | LR: 4.53e-04 | 1324.3ms/step | 15.1 samples/s +Step 9330/44000 | Loss: 3.1932 | LR: 4.53e-04 | 1324.8ms/step | 15.1 samples/s +Step 9340/44000 | Loss: 3.1699 | LR: 4.53e-04 | 1328.6ms/step | 15.1 samples/s +Step 9350/44000 | Loss: 3.1791 | LR: 4.53e-04 | 1329.6ms/step | 15.0 samples/s +Step 9360/44000 | Loss: 3.1685 | LR: 4.53e-04 | 1325.2ms/step | 15.1 samples/s +Step 9370/44000 | Loss: 3.1641 | LR: 4.53e-04 | 1326.0ms/step | 15.1 samples/s +Step 9380/44000 | Loss: 3.1832 | LR: 4.52e-04 | 1325.5ms/step | 15.1 samples/s +Step 9390/44000 | Loss: 3.1723 | LR: 4.52e-04 | 1326.3ms/step | 15.1 samples/s +Step 9400/44000 | Loss: 3.1713 | LR: 4.52e-04 | 1323.9ms/step | 15.1 samples/s +Step 9410/44000 | Loss: 3.1856 | LR: 4.52e-04 | 1322.8ms/step | 15.1 samples/s +Step 9420/44000 | Loss: 3.1665 | LR: 4.52e-04 | 1320.1ms/step | 15.2 samples/s +Step 9430/44000 | Loss: 3.1413 | LR: 4.52e-04 | 1324.2ms/step | 15.1 samples/s +Step 9440/44000 | Loss: 3.1718 | LR: 4.52e-04 | 1321.6ms/step | 15.1 samples/s +Step 9450/44000 | Loss: 3.1509 | LR: 4.52e-04 | 1317.1ms/step | 15.2 samples/s +Step 9460/44000 | Loss: 3.1870 | LR: 4.52e-04 | 1320.9ms/step | 15.1 samples/s +Step 9470/44000 | Loss: 3.1703 | LR: 4.52e-04 | 1321.4ms/step | 15.1 samples/s +Step 9480/44000 | Loss: 3.1798 | LR: 4.51e-04 | 1320.3ms/step | 15.1 samples/s +Step 9490/44000 | Loss: 3.1679 | LR: 4.51e-04 | 1321.0ms/step | 15.1 samples/s +Step 9500/44000 | Loss: 3.1707 | LR: 4.51e-04 | 1320.2ms/step | 15.1 samples/s +Validation at step 9500: Loss = 3.1743, Perplexity = 23.91 +Step 9510/44000 | Loss: 3.1414 | LR: 4.51e-04 | 3540.5ms/step | 5.6 samples/s +Step 9520/44000 | Loss: 3.1607 | LR: 4.51e-04 | 1323.6ms/step | 15.1 samples/s +Step 9530/44000 | Loss: 3.1747 | LR: 4.51e-04 | 1322.9ms/step | 15.1 samples/s +Step 9540/44000 | Loss: 3.1873 | LR: 4.51e-04 | 1323.9ms/step | 15.1 samples/s +Step 9550/44000 | Loss: 3.1745 | LR: 4.51e-04 | 1325.1ms/step | 15.1 samples/s +Step 9560/44000 | Loss: 3.1782 | LR: 4.51e-04 | 1323.2ms/step | 15.1 samples/s +Step 9570/44000 | Loss: 3.1494 | LR: 4.50e-04 | 1320.6ms/step | 15.1 samples/s +Step 9580/44000 | Loss: 3.1637 | LR: 4.50e-04 | 1324.3ms/step | 15.1 samples/s +Step 9590/44000 | Loss: 3.1762 | LR: 4.50e-04 | 1324.5ms/step | 15.1 samples/s +Step 9600/44000 | Loss: 3.1479 | LR: 4.50e-04 | 1324.6ms/step | 15.1 samples/s +Step 9610/44000 | Loss: 3.1373 | LR: 4.50e-04 | 1325.6ms/step | 15.1 samples/s +Step 9620/44000 | Loss: 3.1684 | LR: 4.50e-04 | 1322.5ms/step | 15.1 samples/s +Step 9630/44000 | Loss: 3.1323 | LR: 4.50e-04 | 1322.8ms/step | 15.1 samples/s +Step 9640/44000 | Loss: 3.1747 | LR: 4.50e-04 | 1322.6ms/step | 15.1 samples/s +Step 9650/44000 | Loss: 3.1737 | LR: 4.50e-04 | 1324.2ms/step | 15.1 samples/s +Step 9660/44000 | Loss: 3.1675 | LR: 4.49e-04 | 1322.4ms/step | 15.1 samples/s +Step 9670/44000 | Loss: 3.1902 | LR: 4.49e-04 | 1323.6ms/step | 15.1 samples/s +Step 9680/44000 | Loss: 3.2027 | LR: 4.49e-04 | 1324.8ms/step | 15.1 samples/s +Step 9690/44000 | Loss: 3.1517 | LR: 4.49e-04 | 1322.1ms/step | 15.1 samples/s +Step 9700/44000 | Loss: 3.1819 | LR: 4.49e-04 | 1324.5ms/step | 15.1 samples/s +Step 9710/44000 | Loss: 3.1882 | LR: 4.49e-04 | 1321.0ms/step | 15.1 samples/s +Step 9720/44000 | Loss: 3.1487 | LR: 4.49e-04 | 1325.1ms/step | 15.1 samples/s +Step 9730/44000 | Loss: 3.1856 | LR: 4.49e-04 | 1324.3ms/step | 15.1 samples/s +Step 9740/44000 | Loss: 3.1212 | LR: 4.49e-04 | 1323.2ms/step | 15.1 samples/s +Step 9750/44000 | Loss: 3.1422 | LR: 4.48e-04 | 1323.9ms/step | 15.1 samples/s +Validation at step 9750: Loss = 3.1662, Perplexity = 23.72 +Step 9760/44000 | Loss: 3.1453 | LR: 4.48e-04 | 3548.7ms/step | 5.6 samples/s +Step 9770/44000 | Loss: 3.1711 | LR: 4.48e-04 | 1324.2ms/step | 15.1 samples/s +Step 9780/44000 | Loss: 3.1778 | LR: 4.48e-04 | 1325.5ms/step | 15.1 samples/s +Step 9790/44000 | Loss: 3.1533 | LR: 4.48e-04 | 1324.8ms/step | 15.1 samples/s +Step 9800/44000 | Loss: 3.1915 | LR: 4.48e-04 | 1324.8ms/step | 15.1 samples/s +Step 9810/44000 | Loss: 3.1795 | LR: 4.48e-04 | 1323.0ms/step | 15.1 samples/s +Step 9820/44000 | Loss: 3.1585 | LR: 4.48e-04 | 1325.7ms/step | 15.1 samples/s +Step 9830/44000 | Loss: 3.1552 | LR: 4.48e-04 | 1324.1ms/step | 15.1 samples/s +Step 9840/44000 | Loss: 3.1427 | LR: 4.47e-04 | 1324.9ms/step | 15.1 samples/s +Step 9850/44000 | Loss: 3.1330 | LR: 4.47e-04 | 1324.4ms/step | 15.1 samples/s +Step 9860/44000 | Loss: 3.1732 | LR: 4.47e-04 | 1324.0ms/step | 15.1 samples/s +Step 9870/44000 | Loss: 3.1439 | LR: 4.47e-04 | 1324.7ms/step | 15.1 samples/s +Step 9880/44000 | Loss: 3.1768 | LR: 4.47e-04 | 1324.2ms/step | 15.1 samples/s +Step 9890/44000 | Loss: 3.1454 | LR: 4.47e-04 | 1324.9ms/step | 15.1 samples/s +Step 9900/44000 | Loss: 3.1693 | LR: 4.47e-04 | 1325.6ms/step | 15.1 samples/s +Step 9910/44000 | Loss: 3.1685 | LR: 4.47e-04 | 1322.6ms/step | 15.1 samples/s +Step 9920/44000 | Loss: 3.1451 | LR: 4.47e-04 | 1325.9ms/step | 15.1 samples/s +Step 9930/44000 | Loss: 3.1561 | LR: 4.46e-04 | 1323.7ms/step | 15.1 samples/s +Step 9940/44000 | Loss: 3.1681 | LR: 4.46e-04 | 1322.7ms/step | 15.1 samples/s +Step 9950/44000 | Loss: 3.1859 | LR: 4.46e-04 | 1324.5ms/step | 15.1 samples/s +Step 9960/44000 | Loss: 3.1646 | LR: 4.46e-04 | 1321.2ms/step | 15.1 samples/s +Step 9970/44000 | Loss: 3.1451 | LR: 4.46e-04 | 1324.1ms/step | 15.1 samples/s +Step 9980/44000 | Loss: 3.1601 | LR: 4.46e-04 | 1325.2ms/step | 15.1 samples/s +Step 9990/44000 | Loss: 3.1340 | LR: 4.46e-04 | 1325.6ms/step | 15.1 samples/s +Step 10000/44000 | Loss: 3.1428 | LR: 4.46e-04 | 1326.3ms/step | 15.1 samples/s +Validation at step 10000: Loss = 3.1606, Perplexity = 23.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-10000 +Step 10010/44000 | Loss: 3.1651 | LR: 4.46e-04 | 3851.1ms/step | 5.2 samples/s +Step 10020/44000 | Loss: 3.1470 | LR: 4.45e-04 | 1323.8ms/step | 15.1 samples/s +Step 10030/44000 | Loss: 3.1257 | LR: 4.45e-04 | 1323.2ms/step | 15.1 samples/s +Step 10040/44000 | Loss: 3.1424 | LR: 4.45e-04 | 1323.3ms/step | 15.1 samples/s +Step 10050/44000 | Loss: 3.1460 | LR: 4.45e-04 | 1324.3ms/step | 15.1 samples/s +Step 10060/44000 | Loss: 3.1716 | LR: 4.45e-04 | 1326.6ms/step | 15.1 samples/s +Step 10070/44000 | Loss: 3.1704 | LR: 4.45e-04 | 1324.1ms/step | 15.1 samples/s +Step 10080/44000 | Loss: 3.1554 | LR: 4.45e-04 | 1323.8ms/step | 15.1 samples/s +Step 10090/44000 | Loss: 3.1270 | LR: 4.45e-04 | 1326.1ms/step | 15.1 samples/s +Step 10100/44000 | Loss: 3.1490 | LR: 4.45e-04 | 1322.7ms/step | 15.1 samples/s +Step 10110/44000 | Loss: 3.1649 | LR: 4.44e-04 | 1324.0ms/step | 15.1 samples/s +Step 10120/44000 | Loss: 3.1293 | LR: 4.44e-04 | 1326.0ms/step | 15.1 samples/s +Step 10130/44000 | Loss: 3.1555 | LR: 4.44e-04 | 1324.6ms/step | 15.1 samples/s +Step 10140/44000 | Loss: 3.1641 | LR: 4.44e-04 | 1325.4ms/step | 15.1 samples/s +Step 10150/44000 | Loss: 3.1457 | LR: 4.44e-04 | 1322.0ms/step | 15.1 samples/s +Step 10160/44000 | Loss: 3.1444 | LR: 4.44e-04 | 1323.4ms/step | 15.1 samples/s +Step 10170/44000 | Loss: 3.1862 | LR: 4.44e-04 | 1326.6ms/step | 15.1 samples/s +Step 10180/44000 | Loss: 3.1692 | LR: 4.44e-04 | 1326.7ms/step | 15.1 samples/s +Step 10190/44000 | Loss: 3.1806 | LR: 4.43e-04 | 1324.7ms/step | 15.1 samples/s +Step 10200/44000 | Loss: 3.1606 | LR: 4.43e-04 | 1322.5ms/step | 15.1 samples/s +Step 10210/44000 | Loss: 3.1523 | LR: 4.43e-04 | 1320.8ms/step | 15.1 samples/s +Step 10220/44000 | Loss: 3.1187 | LR: 4.43e-04 | 1325.6ms/step | 15.1 samples/s +Step 10230/44000 | Loss: 3.1209 | LR: 4.43e-04 | 1324.3ms/step | 15.1 samples/s +Step 10240/44000 | Loss: 3.1722 | LR: 4.43e-04 | 1323.4ms/step | 15.1 samples/s +Step 10250/44000 | Loss: 3.1231 | LR: 4.43e-04 | 1324.0ms/step | 15.1 samples/s +Validation at step 10250: Loss = 3.1560, Perplexity = 23.48 +Step 10260/44000 | Loss: 3.1544 | LR: 4.43e-04 | 3552.0ms/step | 5.6 samples/s +Step 10270/44000 | Loss: 3.1581 | LR: 4.43e-04 | 1325.6ms/step | 15.1 samples/s +Step 10280/44000 | Loss: 3.1604 | LR: 4.42e-04 | 1325.4ms/step | 15.1 samples/s +Step 10290/44000 | Loss: 3.1623 | LR: 4.42e-04 | 1325.0ms/step | 15.1 samples/s +Step 10300/44000 | Loss: 3.1257 | LR: 4.42e-04 | 1324.8ms/step | 15.1 samples/s +Step 10310/44000 | Loss: 3.1455 | LR: 4.42e-04 | 1323.0ms/step | 15.1 samples/s +Step 10320/44000 | Loss: 3.1579 | LR: 4.42e-04 | 1321.1ms/step | 15.1 samples/s +Step 10330/44000 | Loss: 3.1600 | LR: 4.42e-04 | 1323.7ms/step | 15.1 samples/s +Step 10340/44000 | Loss: 3.1507 | LR: 4.42e-04 | 1325.9ms/step | 15.1 samples/s +Step 10350/44000 | Loss: 3.1566 | LR: 4.42e-04 | 1321.5ms/step | 15.1 samples/s +Step 10360/44000 | Loss: 3.1545 | LR: 4.42e-04 | 1324.9ms/step | 15.1 samples/s +Step 10370/44000 | Loss: 3.1505 | LR: 4.41e-04 | 1324.9ms/step | 15.1 samples/s +Step 10380/44000 | Loss: 3.1621 | LR: 4.41e-04 | 1320.4ms/step | 15.1 samples/s +Step 10390/44000 | Loss: 3.1716 | LR: 4.41e-04 | 1321.7ms/step | 15.1 samples/s +Step 10400/44000 | Loss: 3.1678 | LR: 4.41e-04 | 1324.5ms/step | 15.1 samples/s +Step 10410/44000 | Loss: 3.1543 | LR: 4.41e-04 | 1327.7ms/step | 15.1 samples/s +Step 10420/44000 | Loss: 3.1744 | LR: 4.41e-04 | 1325.3ms/step | 15.1 samples/s +Step 10430/44000 | Loss: 3.1455 | LR: 4.41e-04 | 1322.9ms/step | 15.1 samples/s +Step 10440/44000 | Loss: 3.1066 | LR: 4.41e-04 | 1333.2ms/step | 15.0 samples/s +Step 10450/44000 | Loss: 3.1615 | LR: 4.40e-04 | 1325.2ms/step | 15.1 samples/s +Step 10460/44000 | Loss: 3.1641 | LR: 4.40e-04 | 1325.6ms/step | 15.1 samples/s +Step 10470/44000 | Loss: 3.1252 | LR: 4.40e-04 | 1328.1ms/step | 15.1 samples/s +Step 10480/44000 | Loss: 3.1430 | LR: 4.40e-04 | 1326.9ms/step | 15.1 samples/s +Step 10490/44000 | Loss: 3.1464 | LR: 4.40e-04 | 1324.3ms/step | 15.1 samples/s +Step 10500/44000 | Loss: 3.1708 | LR: 4.40e-04 | 1326.9ms/step | 15.1 samples/s +Validation at step 10500: Loss = 3.1502, Perplexity = 23.34 +Step 10510/44000 | Loss: 3.1483 | LR: 4.40e-04 | 3544.4ms/step | 5.6 samples/s +Step 10520/44000 | Loss: 3.1678 | LR: 4.40e-04 | 1326.4ms/step | 15.1 samples/s +Step 10530/44000 | Loss: 3.1659 | LR: 4.40e-04 | 1325.0ms/step | 15.1 samples/s +Step 10540/44000 | Loss: 3.1334 | LR: 4.39e-04 | 1325.7ms/step | 15.1 samples/s +Step 10550/44000 | Loss: 3.1533 | LR: 4.39e-04 | 1326.6ms/step | 15.1 samples/s +Step 10560/44000 | Loss: 3.1231 | LR: 4.39e-04 | 1326.3ms/step | 15.1 samples/s +Step 10570/44000 | Loss: 3.1176 | LR: 4.39e-04 | 1324.3ms/step | 15.1 samples/s +Step 10580/44000 | Loss: 3.1650 | LR: 4.39e-04 | 1323.9ms/step | 15.1 samples/s +Step 10590/44000 | Loss: 3.1321 | LR: 4.39e-04 | 1322.8ms/step | 15.1 samples/s +Step 10600/44000 | Loss: 3.1411 | LR: 4.39e-04 | 1324.4ms/step | 15.1 samples/s +Step 10610/44000 | Loss: 3.1594 | LR: 4.39e-04 | 1322.9ms/step | 15.1 samples/s +Step 10620/44000 | Loss: 3.1302 | LR: 4.38e-04 | 1324.1ms/step | 15.1 samples/s +Step 10630/44000 | Loss: 3.0948 | LR: 4.38e-04 | 1322.0ms/step | 15.1 samples/s +Step 10640/44000 | Loss: 3.1776 | LR: 4.38e-04 | 1327.7ms/step | 15.1 samples/s +Step 10650/44000 | Loss: 3.1650 | LR: 4.38e-04 | 1326.9ms/step | 15.1 samples/s +Step 10660/44000 | Loss: 3.1517 | LR: 4.38e-04 | 1324.6ms/step | 15.1 samples/s +Step 10670/44000 | Loss: 3.1217 | LR: 4.38e-04 | 1325.6ms/step | 15.1 samples/s +Step 10680/44000 | Loss: 3.1703 | LR: 4.38e-04 | 1324.1ms/step | 15.1 samples/s +Step 10690/44000 | Loss: 3.1458 | LR: 4.38e-04 | 1322.4ms/step | 15.1 samples/s +Step 10700/44000 | Loss: 3.1720 | LR: 4.37e-04 | 1323.5ms/step | 15.1 samples/s +Step 10710/44000 | Loss: 3.1443 | LR: 4.37e-04 | 1326.4ms/step | 15.1 samples/s +Step 10720/44000 | Loss: 3.1622 | LR: 4.37e-04 | 1326.9ms/step | 15.1 samples/s +Step 10730/44000 | Loss: 3.1594 | LR: 4.37e-04 | 1322.5ms/step | 15.1 samples/s +Step 10740/44000 | Loss: 3.1671 | LR: 4.37e-04 | 1319.2ms/step | 15.2 samples/s +Step 10750/44000 | Loss: 3.1360 | LR: 4.37e-04 | 1322.6ms/step | 15.1 samples/s +Validation at step 10750: Loss = 3.1456, Perplexity = 23.23 +Step 10760/44000 | Loss: 3.1594 | LR: 4.37e-04 | 3552.9ms/step | 5.6 samples/s +Step 10770/44000 | Loss: 3.1354 | LR: 4.37e-04 | 1327.8ms/step | 15.1 samples/s +Step 10780/44000 | Loss: 3.1278 | LR: 4.37e-04 | 1329.1ms/step | 15.0 samples/s +Step 10790/44000 | Loss: 3.1309 | LR: 4.36e-04 | 1329.4ms/step | 15.0 samples/s +Step 10800/44000 | Loss: 3.1148 | LR: 4.36e-04 | 1327.9ms/step | 15.1 samples/s +Step 10810/44000 | Loss: 3.0993 | LR: 4.36e-04 | 1326.8ms/step | 15.1 samples/s +Step 10820/44000 | Loss: 3.1471 | LR: 4.36e-04 | 1327.2ms/step | 15.1 samples/s +Step 10830/44000 | Loss: 3.1559 | LR: 4.36e-04 | 1325.1ms/step | 15.1 samples/s +Step 10840/44000 | Loss: 3.1427 | LR: 4.36e-04 | 1327.6ms/step | 15.1 samples/s +Step 10850/44000 | Loss: 3.1605 | LR: 4.36e-04 | 1324.9ms/step | 15.1 samples/s +Step 10860/44000 | Loss: 3.1513 | LR: 4.36e-04 | 1320.9ms/step | 15.1 samples/s +Step 10870/44000 | Loss: 3.1091 | LR: 4.35e-04 | 1322.2ms/step | 15.1 samples/s +Step 10880/44000 | Loss: 3.0989 | LR: 4.35e-04 | 1321.3ms/step | 15.1 samples/s +Step 10890/44000 | Loss: 3.1593 | LR: 4.35e-04 | 1324.5ms/step | 15.1 samples/s +Step 10900/44000 | Loss: 3.1636 | LR: 4.35e-04 | 1326.9ms/step | 15.1 samples/s +Step 10910/44000 | Loss: 3.1789 | LR: 4.35e-04 | 1325.5ms/step | 15.1 samples/s +Step 10920/44000 | Loss: 3.1337 | LR: 4.35e-04 | 1322.8ms/step | 15.1 samples/s +Step 10930/44000 | Loss: 3.1478 | LR: 4.35e-04 | 1324.0ms/step | 15.1 samples/s +Step 10940/44000 | Loss: 3.1305 | LR: 4.35e-04 | 1319.6ms/step | 15.2 samples/s +Step 10950/44000 | Loss: 3.1610 | LR: 4.34e-04 | 1325.1ms/step | 15.1 samples/s +Step 10960/44000 | Loss: 3.1404 | LR: 4.34e-04 | 1327.8ms/step | 15.1 samples/s +Step 10970/44000 | Loss: 3.1319 | LR: 4.34e-04 | 1322.5ms/step | 15.1 samples/s +Step 10980/44000 | Loss: 3.1490 | LR: 4.34e-04 | 1323.5ms/step | 15.1 samples/s +Step 10990/44000 | Loss: 3.1226 | LR: 4.34e-04 | 1326.1ms/step | 15.1 samples/s +Step 11000/44000 | Loss: 3.1455 | LR: 4.34e-04 | 1325.9ms/step | 15.1 samples/s +Validation at step 11000: Loss = 3.1403, Perplexity = 23.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-11000 +Step 11010/44000 | Loss: 3.1224 | LR: 4.34e-04 | 3854.1ms/step | 5.2 samples/s +Step 11020/44000 | Loss: 3.1018 | LR: 4.34e-04 | 1320.2ms/step | 15.1 samples/s +Step 11030/44000 | Loss: 3.1425 | LR: 4.33e-04 | 1321.1ms/step | 15.1 samples/s +Step 11040/44000 | Loss: 3.1059 | LR: 4.33e-04 | 1322.0ms/step | 15.1 samples/s +Step 11050/44000 | Loss: 3.1504 | LR: 4.33e-04 | 1322.0ms/step | 15.1 samples/s +Step 11060/44000 | Loss: 3.1504 | LR: 4.33e-04 | 1322.5ms/step | 15.1 samples/s +Step 11070/44000 | Loss: 3.1535 | LR: 4.33e-04 | 1323.3ms/step | 15.1 samples/s +Step 11080/44000 | Loss: 3.1431 | LR: 4.33e-04 | 1322.2ms/step | 15.1 samples/s +Step 11090/44000 | Loss: 3.0981 | LR: 4.33e-04 | 1324.4ms/step | 15.1 samples/s +Step 11100/44000 | Loss: 3.1390 | LR: 4.33e-04 | 1321.5ms/step | 15.1 samples/s +Step 11110/44000 | Loss: 3.1213 | LR: 4.32e-04 | 1322.0ms/step | 15.1 samples/s +Step 11120/44000 | Loss: 3.1358 | LR: 4.32e-04 | 1322.5ms/step | 15.1 samples/s +Step 11130/44000 | Loss: 3.1615 | LR: 4.32e-04 | 1324.8ms/step | 15.1 samples/s +Step 11140/44000 | Loss: 3.1283 | LR: 4.32e-04 | 1321.5ms/step | 15.1 samples/s +Step 11150/44000 | Loss: 3.1287 | LR: 4.32e-04 | 1321.4ms/step | 15.1 samples/s +Step 11160/44000 | Loss: 3.1301 | LR: 4.32e-04 | 1323.7ms/step | 15.1 samples/s +Step 11170/44000 | Loss: 3.1230 | LR: 4.32e-04 | 1325.2ms/step | 15.1 samples/s +Step 11180/44000 | Loss: 3.1270 | LR: 4.32e-04 | 1323.3ms/step | 15.1 samples/s +Step 11190/44000 | Loss: 3.1098 | LR: 4.31e-04 | 1325.0ms/step | 15.1 samples/s +Step 11200/44000 | Loss: 3.1512 | LR: 4.31e-04 | 1323.3ms/step | 15.1 samples/s +Step 11210/44000 | Loss: 3.1864 | LR: 4.31e-04 | 1323.3ms/step | 15.1 samples/s +Step 11220/44000 | Loss: 3.1127 | LR: 4.31e-04 | 1321.5ms/step | 15.1 samples/s +Step 11230/44000 | Loss: 3.1355 | LR: 4.31e-04 | 1320.3ms/step | 15.1 samples/s +Step 11240/44000 | Loss: 3.1219 | LR: 4.31e-04 | 1323.8ms/step | 15.1 samples/s +Step 11250/44000 | Loss: 3.1244 | LR: 4.31e-04 | 1323.9ms/step | 15.1 samples/s +Validation at step 11250: Loss = 3.1353, Perplexity = 23.00 +Step 11260/44000 | Loss: 3.1192 | LR: 4.31e-04 | 3552.1ms/step | 5.6 samples/s +Step 11270/44000 | Loss: 3.1374 | LR: 4.30e-04 | 1321.5ms/step | 15.1 samples/s +Step 11280/44000 | Loss: 3.1146 | LR: 4.30e-04 | 1320.4ms/step | 15.1 samples/s +Step 11290/44000 | Loss: 3.1168 | LR: 4.30e-04 | 1319.6ms/step | 15.2 samples/s +Step 11300/44000 | Loss: 3.1550 | LR: 4.30e-04 | 1321.3ms/step | 15.1 samples/s +Step 11310/44000 | Loss: 3.1345 | LR: 4.30e-04 | 1322.0ms/step | 15.1 samples/s +Step 11320/44000 | Loss: 3.1212 | LR: 4.30e-04 | 1323.5ms/step | 15.1 samples/s +Step 11330/44000 | Loss: 3.1210 | LR: 4.30e-04 | 1324.8ms/step | 15.1 samples/s +Step 11340/44000 | Loss: 3.1202 | LR: 4.30e-04 | 1322.0ms/step | 15.1 samples/s +Step 11350/44000 | Loss: 3.1134 | LR: 4.29e-04 | 1321.9ms/step | 15.1 samples/s +Step 11360/44000 | Loss: 3.1451 | LR: 4.29e-04 | 1323.0ms/step | 15.1 samples/s +Step 11370/44000 | Loss: 3.1229 | LR: 4.29e-04 | 1323.0ms/step | 15.1 samples/s +Step 11380/44000 | Loss: 3.1216 | LR: 4.29e-04 | 1322.1ms/step | 15.1 samples/s +Step 11390/44000 | Loss: 3.1238 | LR: 4.29e-04 | 1327.1ms/step | 15.1 samples/s +Step 11400/44000 | Loss: 3.1481 | LR: 4.29e-04 | 1324.5ms/step | 15.1 samples/s +Step 11410/44000 | Loss: 3.1142 | LR: 4.29e-04 | 1326.4ms/step | 15.1 samples/s +Step 11420/44000 | Loss: 3.1220 | LR: 4.29e-04 | 1324.2ms/step | 15.1 samples/s +Step 11430/44000 | Loss: 3.1388 | LR: 4.28e-04 | 1322.7ms/step | 15.1 samples/s +Step 11440/44000 | Loss: 3.1291 | LR: 4.28e-04 | 1324.5ms/step | 15.1 samples/s +Step 11450/44000 | Loss: 3.1427 | LR: 4.28e-04 | 1320.1ms/step | 15.2 samples/s +Step 11460/44000 | Loss: 3.1367 | LR: 4.28e-04 | 1321.6ms/step | 15.1 samples/s +Step 11470/44000 | Loss: 3.1285 | LR: 4.28e-04 | 1321.6ms/step | 15.1 samples/s +Step 11480/44000 | Loss: 3.1335 | LR: 4.28e-04 | 1325.0ms/step | 15.1 samples/s +Step 11490/44000 | Loss: 3.1082 | LR: 4.28e-04 | 1323.6ms/step | 15.1 samples/s +Step 11500/44000 | Loss: 3.1204 | LR: 4.28e-04 | 1322.7ms/step | 15.1 samples/s +Validation at step 11500: Loss = 3.1301, Perplexity = 22.88 +Step 11510/44000 | Loss: 3.1562 | LR: 4.27e-04 | 3547.6ms/step | 5.6 samples/s +Step 11520/44000 | Loss: 3.1398 | LR: 4.27e-04 | 1323.2ms/step | 15.1 samples/s +Step 11530/44000 | Loss: 3.1416 | LR: 4.27e-04 | 1324.0ms/step | 15.1 samples/s +Step 11540/44000 | Loss: 3.1244 | LR: 4.27e-04 | 1324.3ms/step | 15.1 samples/s +Step 11550/44000 | Loss: 3.1052 | LR: 4.27e-04 | 1323.5ms/step | 15.1 samples/s +Step 11560/44000 | Loss: 3.1488 | LR: 4.27e-04 | 1328.5ms/step | 15.1 samples/s +Step 11570/44000 | Loss: 3.1245 | LR: 4.27e-04 | 1324.6ms/step | 15.1 samples/s +Step 11580/44000 | Loss: 3.1541 | LR: 4.27e-04 | 1323.3ms/step | 15.1 samples/s +Step 11590/44000 | Loss: 3.1562 | LR: 4.26e-04 | 1324.6ms/step | 15.1 samples/s +Step 11600/44000 | Loss: 3.1159 | LR: 4.26e-04 | 1321.5ms/step | 15.1 samples/s +Step 11610/44000 | Loss: 3.1396 | LR: 4.26e-04 | 1318.8ms/step | 15.2 samples/s +Step 11620/44000 | Loss: 3.0909 | LR: 4.26e-04 | 1321.5ms/step | 15.1 samples/s +Step 11630/44000 | Loss: 3.0990 | LR: 4.26e-04 | 1321.6ms/step | 15.1 samples/s +Step 11640/44000 | Loss: 3.1483 | LR: 4.26e-04 | 1324.7ms/step | 15.1 samples/s +Step 11650/44000 | Loss: 3.1059 | LR: 4.26e-04 | 1325.9ms/step | 15.1 samples/s +Step 11660/44000 | Loss: 3.1527 | LR: 4.26e-04 | 1324.6ms/step | 15.1 samples/s +Step 11670/44000 | Loss: 3.1106 | LR: 4.25e-04 | 1323.7ms/step | 15.1 samples/s +Step 11680/44000 | Loss: 3.1273 | LR: 4.25e-04 | 1323.0ms/step | 15.1 samples/s +Step 11690/44000 | Loss: 3.1088 | LR: 4.25e-04 | 1325.0ms/step | 15.1 samples/s +Step 11700/44000 | Loss: 3.1303 | LR: 4.25e-04 | 1326.0ms/step | 15.1 samples/s +Step 11710/44000 | Loss: 3.1321 | LR: 4.25e-04 | 1321.9ms/step | 15.1 samples/s +Step 11720/44000 | Loss: 3.1262 | LR: 4.25e-04 | 1321.1ms/step | 15.1 samples/s +Step 11730/44000 | Loss: 3.1192 | LR: 4.25e-04 | 1322.9ms/step | 15.1 samples/s +Step 11740/44000 | Loss: 3.1111 | LR: 4.24e-04 | 1322.4ms/step | 15.1 samples/s +Step 11750/44000 | Loss: 3.1031 | LR: 4.24e-04 | 1325.1ms/step | 15.1 samples/s +Validation at step 11750: Loss = 3.1249, Perplexity = 22.76 +Step 11760/44000 | Loss: 3.1468 | LR: 4.24e-04 | 3556.8ms/step | 5.6 samples/s +Step 11770/44000 | Loss: 3.1282 | LR: 4.24e-04 | 1325.8ms/step | 15.1 samples/s +Step 11780/44000 | Loss: 3.1297 | LR: 4.24e-04 | 1324.5ms/step | 15.1 samples/s +Step 11790/44000 | Loss: 3.1279 | LR: 4.24e-04 | 1326.0ms/step | 15.1 samples/s +Step 11800/44000 | Loss: 3.1375 | LR: 4.24e-04 | 1322.8ms/step | 15.1 samples/s +Step 11810/44000 | Loss: 3.1279 | LR: 4.24e-04 | 1327.0ms/step | 15.1 samples/s +Step 11820/44000 | Loss: 3.1262 | LR: 4.23e-04 | 1321.8ms/step | 15.1 samples/s +Step 11830/44000 | Loss: 3.1471 | LR: 4.23e-04 | 1323.2ms/step | 15.1 samples/s +Step 11840/44000 | Loss: 3.1217 | LR: 4.23e-04 | 1326.1ms/step | 15.1 samples/s +Step 11850/44000 | Loss: 3.1167 | LR: 4.23e-04 | 1323.2ms/step | 15.1 samples/s +Step 11860/44000 | Loss: 3.1257 | LR: 4.23e-04 | 1326.3ms/step | 15.1 samples/s +Step 11870/44000 | Loss: 3.1017 | LR: 4.23e-04 | 1324.4ms/step | 15.1 samples/s +Step 11880/44000 | Loss: 3.1117 | LR: 4.23e-04 | 1322.3ms/step | 15.1 samples/s +Step 11890/44000 | Loss: 3.1329 | LR: 4.23e-04 | 1325.3ms/step | 15.1 samples/s +Step 11900/44000 | Loss: 3.1024 | LR: 4.22e-04 | 1321.7ms/step | 15.1 samples/s +Step 11910/44000 | Loss: 3.0866 | LR: 4.22e-04 | 1327.2ms/step | 15.1 samples/s +Step 11920/44000 | Loss: 3.1162 | LR: 4.22e-04 | 1322.9ms/step | 15.1 samples/s +Step 11930/44000 | Loss: 3.1331 | LR: 4.22e-04 | 1323.3ms/step | 15.1 samples/s +Step 11940/44000 | Loss: 3.1265 | LR: 4.22e-04 | 1325.7ms/step | 15.1 samples/s +Step 11950/44000 | Loss: 3.1158 | LR: 4.22e-04 | 1326.2ms/step | 15.1 samples/s +Step 11960/44000 | Loss: 3.1494 | LR: 4.22e-04 | 1324.2ms/step | 15.1 samples/s +Step 11970/44000 | Loss: 3.1407 | LR: 4.21e-04 | 1325.6ms/step | 15.1 samples/s +Step 11980/44000 | Loss: 3.1473 | LR: 4.21e-04 | 1326.0ms/step | 15.1 samples/s +Step 11990/44000 | Loss: 3.1334 | LR: 4.21e-04 | 1325.8ms/step | 15.1 samples/s +Step 12000/44000 | Loss: 3.0956 | LR: 4.21e-04 | 1325.2ms/step | 15.1 samples/s +Validation at step 12000: Loss = 3.1200, Perplexity = 22.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-12000 +Step 12010/44000 | Loss: 3.1048 | LR: 4.21e-04 | 3839.6ms/step | 5.2 samples/s +Step 12020/44000 | Loss: 3.1334 | LR: 4.21e-04 | 1327.5ms/step | 15.1 samples/s +Step 12030/44000 | Loss: 3.1191 | LR: 4.21e-04 | 1323.8ms/step | 15.1 samples/s +Step 12040/44000 | Loss: 3.1034 | LR: 4.21e-04 | 1322.7ms/step | 15.1 samples/s +Step 12050/44000 | Loss: 3.1400 | LR: 4.20e-04 | 1321.3ms/step | 15.1 samples/s +Step 12060/44000 | Loss: 3.1105 | LR: 4.20e-04 | 1323.4ms/step | 15.1 samples/s +Step 12070/44000 | Loss: 3.1294 | LR: 4.20e-04 | 1324.0ms/step | 15.1 samples/s +Step 12080/44000 | Loss: 3.1242 | LR: 4.20e-04 | 1324.7ms/step | 15.1 samples/s +Step 12090/44000 | Loss: 3.1147 | LR: 4.20e-04 | 1324.4ms/step | 15.1 samples/s +Step 12100/44000 | Loss: 3.0901 | LR: 4.20e-04 | 1323.3ms/step | 15.1 samples/s +Step 12110/44000 | Loss: 3.1140 | LR: 4.20e-04 | 1325.7ms/step | 15.1 samples/s +Step 12120/44000 | Loss: 3.1099 | LR: 4.19e-04 | 1322.5ms/step | 15.1 samples/s +Step 12130/44000 | Loss: 3.1147 | LR: 4.19e-04 | 1324.1ms/step | 15.1 samples/s +Step 12140/44000 | Loss: 3.1029 | LR: 4.19e-04 | 1323.4ms/step | 15.1 samples/s +Step 12150/44000 | Loss: 3.1034 | LR: 4.19e-04 | 1324.4ms/step | 15.1 samples/s +Step 12160/44000 | Loss: 3.1139 | LR: 4.19e-04 | 1327.1ms/step | 15.1 samples/s +Step 12170/44000 | Loss: 3.1122 | LR: 4.19e-04 | 1323.1ms/step | 15.1 samples/s +Step 12180/44000 | Loss: 3.1140 | LR: 4.19e-04 | 1327.5ms/step | 15.1 samples/s +Step 12190/44000 | Loss: 3.1538 | LR: 4.19e-04 | 1326.5ms/step | 15.1 samples/s +Step 12200/44000 | Loss: 3.1189 | LR: 4.18e-04 | 1322.8ms/step | 15.1 samples/s +Step 12210/44000 | Loss: 3.0888 | LR: 4.18e-04 | 1325.7ms/step | 15.1 samples/s +Step 12220/44000 | Loss: 3.1300 | LR: 4.18e-04 | 1323.1ms/step | 15.1 samples/s +Step 12230/44000 | Loss: 3.1025 | LR: 4.18e-04 | 1325.8ms/step | 15.1 samples/s +Step 12240/44000 | Loss: 3.1016 | LR: 4.18e-04 | 1324.3ms/step | 15.1 samples/s +Step 12250/44000 | Loss: 3.1022 | LR: 4.18e-04 | 1323.1ms/step | 15.1 samples/s +Validation at step 12250: Loss = 3.1166, Perplexity = 22.57 +Step 12260/44000 | Loss: 3.1032 | LR: 4.18e-04 | 3551.5ms/step | 5.6 samples/s +Step 12270/44000 | Loss: 3.1229 | LR: 4.17e-04 | 1325.6ms/step | 15.1 samples/s +Step 12280/44000 | Loss: 3.1084 | LR: 4.17e-04 | 1324.2ms/step | 15.1 samples/s +Step 12290/44000 | Loss: 3.1167 | LR: 4.17e-04 | 1323.6ms/step | 15.1 samples/s +Step 12300/44000 | Loss: 3.1098 | LR: 4.17e-04 | 1324.1ms/step | 15.1 samples/s +Step 12310/44000 | Loss: 3.1207 | LR: 4.17e-04 | 1324.9ms/step | 15.1 samples/s +Step 12320/44000 | Loss: 3.0848 | LR: 4.17e-04 | 1323.2ms/step | 15.1 samples/s +Step 12330/44000 | Loss: 3.1211 | LR: 4.17e-04 | 1321.8ms/step | 15.1 samples/s +Step 12340/44000 | Loss: 3.1106 | LR: 4.17e-04 | 1324.4ms/step | 15.1 samples/s +Step 12350/44000 | Loss: 3.1128 | LR: 4.16e-04 | 1321.3ms/step | 15.1 samples/s +Step 12360/44000 | Loss: 3.1092 | LR: 4.16e-04 | 1325.5ms/step | 15.1 samples/s +Step 12370/44000 | Loss: 3.1062 | LR: 4.16e-04 | 1323.6ms/step | 15.1 samples/s +Step 12380/44000 | Loss: 3.0991 | LR: 4.16e-04 | 1324.9ms/step | 15.1 samples/s +Step 12390/44000 | Loss: 3.1194 | LR: 4.16e-04 | 1324.9ms/step | 15.1 samples/s +Step 12400/44000 | Loss: 3.0909 | LR: 4.16e-04 | 1323.4ms/step | 15.1 samples/s +Step 12410/44000 | Loss: 3.0802 | LR: 4.16e-04 | 1324.6ms/step | 15.1 samples/s +Step 12420/44000 | Loss: 3.1179 | LR: 4.15e-04 | 1323.1ms/step | 15.1 samples/s +Step 12430/44000 | Loss: 3.1318 | LR: 4.15e-04 | 1321.1ms/step | 15.1 samples/s +Step 12440/44000 | Loss: 3.1129 | LR: 4.15e-04 | 1325.8ms/step | 15.1 samples/s +Step 12450/44000 | Loss: 3.0809 | LR: 4.15e-04 | 1325.4ms/step | 15.1 samples/s +Step 12460/44000 | Loss: 3.0983 | LR: 4.15e-04 | 1319.4ms/step | 15.2 samples/s +Step 12470/44000 | Loss: 3.0863 | LR: 4.15e-04 | 1323.6ms/step | 15.1 samples/s +Step 12480/44000 | Loss: 3.1341 | LR: 4.15e-04 | 1323.8ms/step | 15.1 samples/s +Step 12490/44000 | Loss: 3.1242 | LR: 4.14e-04 | 1324.3ms/step | 15.1 samples/s +Step 12500/44000 | Loss: 3.0837 | LR: 4.14e-04 | 1328.0ms/step | 15.1 samples/s +Validation at step 12500: Loss = 3.1114, Perplexity = 22.45 +Step 12510/44000 | Loss: 3.1122 | LR: 4.14e-04 | 3540.2ms/step | 5.6 samples/s +Step 12520/44000 | Loss: 3.1007 | LR: 4.14e-04 | 1321.8ms/step | 15.1 samples/s +Step 12530/44000 | Loss: 3.0828 | LR: 4.14e-04 | 1322.7ms/step | 15.1 samples/s +Step 12540/44000 | Loss: 3.0849 | LR: 4.14e-04 | 1322.7ms/step | 15.1 samples/s +Step 12550/44000 | Loss: 3.0861 | LR: 4.14e-04 | 1321.9ms/step | 15.1 samples/s +Step 12560/44000 | Loss: 3.1126 | LR: 4.14e-04 | 1323.4ms/step | 15.1 samples/s +Step 12570/44000 | Loss: 3.1079 | LR: 4.13e-04 | 1322.8ms/step | 15.1 samples/s +Step 12580/44000 | Loss: 3.1033 | LR: 4.13e-04 | 1323.6ms/step | 15.1 samples/s +Step 12590/44000 | Loss: 3.1037 | LR: 4.13e-04 | 1325.6ms/step | 15.1 samples/s +Step 12600/44000 | Loss: 3.0961 | LR: 4.13e-04 | 1324.3ms/step | 15.1 samples/s +Step 12610/44000 | Loss: 3.1554 | LR: 4.13e-04 | 1324.7ms/step | 15.1 samples/s +Step 12620/44000 | Loss: 3.1109 | LR: 4.13e-04 | 1323.4ms/step | 15.1 samples/s +Step 12630/44000 | Loss: 3.1299 | LR: 4.13e-04 | 1324.5ms/step | 15.1 samples/s +Step 12640/44000 | Loss: 3.0938 | LR: 4.12e-04 | 1324.3ms/step | 15.1 samples/s +Step 12650/44000 | Loss: 3.1018 | LR: 4.12e-04 | 1322.6ms/step | 15.1 samples/s +Step 12660/44000 | Loss: 3.0988 | LR: 4.12e-04 | 1320.0ms/step | 15.2 samples/s +Step 12670/44000 | Loss: 3.1100 | LR: 4.12e-04 | 1323.1ms/step | 15.1 samples/s +Step 12680/44000 | Loss: 3.1019 | LR: 4.12e-04 | 1325.1ms/step | 15.1 samples/s +Step 12690/44000 | Loss: 3.1082 | LR: 4.12e-04 | 1320.5ms/step | 15.1 samples/s +Step 12700/44000 | Loss: 3.1340 | LR: 4.12e-04 | 1325.0ms/step | 15.1 samples/s +Step 12710/44000 | Loss: 3.1006 | LR: 4.11e-04 | 1321.2ms/step | 15.1 samples/s +Step 12720/44000 | Loss: 3.1103 | LR: 4.11e-04 | 1323.5ms/step | 15.1 samples/s +Step 12730/44000 | Loss: 3.0990 | LR: 4.11e-04 | 1323.6ms/step | 15.1 samples/s +Step 12740/44000 | Loss: 3.0934 | LR: 4.11e-04 | 1325.7ms/step | 15.1 samples/s +Step 12750/44000 | Loss: 3.1095 | LR: 4.11e-04 | 1324.1ms/step | 15.1 samples/s +Validation at step 12750: Loss = 3.1078, Perplexity = 22.37 +Step 12760/44000 | Loss: 3.1031 | LR: 4.11e-04 | 3551.9ms/step | 5.6 samples/s +Step 12770/44000 | Loss: 3.0788 | LR: 4.11e-04 | 1324.9ms/step | 15.1 samples/s +Step 12780/44000 | Loss: 3.0765 | LR: 4.10e-04 | 1325.8ms/step | 15.1 samples/s +Step 12790/44000 | Loss: 3.1024 | LR: 4.10e-04 | 1325.4ms/step | 15.1 samples/s +Step 12800/44000 | Loss: 3.1032 | LR: 4.10e-04 | 1326.4ms/step | 15.1 samples/s +Step 12810/44000 | Loss: 3.1401 | LR: 4.10e-04 | 1325.8ms/step | 15.1 samples/s +Step 12820/44000 | Loss: 3.1078 | LR: 4.10e-04 | 1328.5ms/step | 15.1 samples/s +Step 12830/44000 | Loss: 3.1163 | LR: 4.10e-04 | 1325.9ms/step | 15.1 samples/s +Step 12840/44000 | Loss: 3.0941 | LR: 4.10e-04 | 1326.8ms/step | 15.1 samples/s +Step 12850/44000 | Loss: 3.1034 | LR: 4.09e-04 | 1324.3ms/step | 15.1 samples/s +Step 12860/44000 | Loss: 3.1109 | LR: 4.09e-04 | 1324.0ms/step | 15.1 samples/s +Step 12870/44000 | Loss: 3.1161 | LR: 4.09e-04 | 1325.5ms/step | 15.1 samples/s +Step 12880/44000 | Loss: 3.0787 | LR: 4.09e-04 | 1324.2ms/step | 15.1 samples/s +Step 12890/44000 | Loss: 3.1029 | LR: 4.09e-04 | 1326.1ms/step | 15.1 samples/s +Step 12900/44000 | Loss: 3.0981 | LR: 4.09e-04 | 1325.5ms/step | 15.1 samples/s +Step 12910/44000 | Loss: 3.1014 | LR: 4.09e-04 | 1326.0ms/step | 15.1 samples/s +Step 12920/44000 | Loss: 3.0991 | LR: 4.09e-04 | 1324.6ms/step | 15.1 samples/s +Step 12930/44000 | Loss: 3.0907 | LR: 4.08e-04 | 1320.9ms/step | 15.1 samples/s +Step 12940/44000 | Loss: 3.1200 | LR: 4.08e-04 | 1325.5ms/step | 15.1 samples/s +Step 12950/44000 | Loss: 3.1045 | LR: 4.08e-04 | 1326.7ms/step | 15.1 samples/s +Step 12960/44000 | Loss: 3.1421 | LR: 4.08e-04 | 1324.6ms/step | 15.1 samples/s +Step 12970/44000 | Loss: 3.0973 | LR: 4.08e-04 | 1322.9ms/step | 15.1 samples/s +Step 12980/44000 | Loss: 3.1000 | LR: 4.08e-04 | 1324.1ms/step | 15.1 samples/s +Step 12990/44000 | Loss: 3.0852 | LR: 4.08e-04 | 1326.8ms/step | 15.1 samples/s +Step 13000/44000 | Loss: 3.1086 | LR: 4.07e-04 | 1326.8ms/step | 15.1 samples/s +Validation at step 13000: Loss = 3.1035, Perplexity = 22.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-13000 +Step 13010/44000 | Loss: 3.0736 | LR: 4.07e-04 | 3842.1ms/step | 5.2 samples/s +Step 13020/44000 | Loss: 3.0860 | LR: 4.07e-04 | 1323.1ms/step | 15.1 samples/s +Step 13030/44000 | Loss: 3.0857 | LR: 4.07e-04 | 1323.1ms/step | 15.1 samples/s +Step 13040/44000 | Loss: 3.0710 | LR: 4.07e-04 | 1326.3ms/step | 15.1 samples/s +Step 13050/44000 | Loss: 3.0991 | LR: 4.07e-04 | 1325.7ms/step | 15.1 samples/s +Step 13060/44000 | Loss: 3.1104 | LR: 4.07e-04 | 1323.3ms/step | 15.1 samples/s +Step 13070/44000 | Loss: 3.0763 | LR: 4.06e-04 | 1322.8ms/step | 15.1 samples/s +Step 13080/44000 | Loss: 3.0553 | LR: 4.06e-04 | 1324.5ms/step | 15.1 samples/s +Step 13090/44000 | Loss: 3.1089 | LR: 4.06e-04 | 1321.9ms/step | 15.1 samples/s +Step 13100/44000 | Loss: 3.0945 | LR: 4.06e-04 | 1324.7ms/step | 15.1 samples/s +Step 13110/44000 | Loss: 3.0755 | LR: 4.06e-04 | 1322.5ms/step | 15.1 samples/s +Step 13120/44000 | Loss: 3.0628 | LR: 4.06e-04 | 1326.4ms/step | 15.1 samples/s +Step 13130/44000 | Loss: 3.1044 | LR: 4.06e-04 | 1325.8ms/step | 15.1 samples/s +Step 13140/44000 | Loss: 3.1288 | LR: 4.05e-04 | 1327.9ms/step | 15.1 samples/s +Step 13150/44000 | Loss: 3.0953 | LR: 4.05e-04 | 1321.9ms/step | 15.1 samples/s +Step 13160/44000 | Loss: 3.1068 | LR: 4.05e-04 | 1318.5ms/step | 15.2 samples/s +Step 13170/44000 | Loss: 3.0937 | LR: 4.05e-04 | 1318.5ms/step | 15.2 samples/s +Step 13180/44000 | Loss: 3.0977 | LR: 4.05e-04 | 1317.6ms/step | 15.2 samples/s +Step 13190/44000 | Loss: 3.1420 | LR: 4.05e-04 | 1317.5ms/step | 15.2 samples/s +Step 13200/44000 | Loss: 3.0992 | LR: 4.05e-04 | 1316.5ms/step | 15.2 samples/s +Step 13210/44000 | Loss: 3.0964 | LR: 4.04e-04 | 1317.2ms/step | 15.2 samples/s +Step 13220/44000 | Loss: 3.0998 | LR: 4.04e-04 | 1316.8ms/step | 15.2 samples/s +Step 13230/44000 | Loss: 3.0761 | LR: 4.04e-04 | 1316.8ms/step | 15.2 samples/s +Step 13240/44000 | Loss: 3.1060 | LR: 4.04e-04 | 1317.7ms/step | 15.2 samples/s +Step 13250/44000 | Loss: 3.1236 | LR: 4.04e-04 | 1320.0ms/step | 15.2 samples/s +Validation at step 13250: Loss = 3.1003, Perplexity = 22.20 +Step 13260/44000 | Loss: 3.0890 | LR: 4.04e-04 | 3546.1ms/step | 5.6 samples/s +Step 13270/44000 | Loss: 3.1186 | LR: 4.04e-04 | 1318.8ms/step | 15.2 samples/s +Step 13280/44000 | Loss: 3.1099 | LR: 4.03e-04 | 1323.7ms/step | 15.1 samples/s +Step 13290/44000 | Loss: 3.1287 | LR: 4.03e-04 | 1328.0ms/step | 15.1 samples/s +Step 13300/44000 | Loss: 3.0966 | LR: 4.03e-04 | 1323.5ms/step | 15.1 samples/s +Step 13310/44000 | Loss: 3.0846 | LR: 4.03e-04 | 1321.3ms/step | 15.1 samples/s +Step 13320/44000 | Loss: 3.0990 | LR: 4.03e-04 | 1323.8ms/step | 15.1 samples/s +Step 13330/44000 | Loss: 3.1289 | LR: 4.03e-04 | 1317.9ms/step | 15.2 samples/s +Step 13340/44000 | Loss: 3.1337 | LR: 4.03e-04 | 1320.5ms/step | 15.1 samples/s +Step 13350/44000 | Loss: 3.0718 | LR: 4.02e-04 | 1327.2ms/step | 15.1 samples/s +Step 13360/44000 | Loss: 3.0946 | LR: 4.02e-04 | 1324.8ms/step | 15.1 samples/s +Step 13370/44000 | Loss: 3.1003 | LR: 4.02e-04 | 1324.3ms/step | 15.1 samples/s +Step 13380/44000 | Loss: 3.0996 | LR: 4.02e-04 | 1325.3ms/step | 15.1 samples/s +Step 13390/44000 | Loss: 3.0654 | LR: 4.02e-04 | 1323.2ms/step | 15.1 samples/s +Step 13400/44000 | Loss: 3.1187 | LR: 4.02e-04 | 1324.1ms/step | 15.1 samples/s +Step 13410/44000 | Loss: 3.0919 | LR: 4.02e-04 | 1326.7ms/step | 15.1 samples/s +Step 13420/44000 | Loss: 3.0752 | LR: 4.01e-04 | 1325.6ms/step | 15.1 samples/s +Step 13430/44000 | Loss: 3.1098 | LR: 4.01e-04 | 1328.1ms/step | 15.1 samples/s +Step 13440/44000 | Loss: 3.1080 | LR: 4.01e-04 | 1328.4ms/step | 15.1 samples/s +Step 13450/44000 | Loss: 3.0898 | LR: 4.01e-04 | 1324.5ms/step | 15.1 samples/s +Step 13460/44000 | Loss: 3.0886 | LR: 4.01e-04 | 1325.5ms/step | 15.1 samples/s +Step 13470/44000 | Loss: 3.1118 | LR: 4.01e-04 | 1321.2ms/step | 15.1 samples/s +Step 13480/44000 | Loss: 3.0780 | LR: 4.01e-04 | 1326.5ms/step | 15.1 samples/s +Step 13490/44000 | Loss: 3.1009 | LR: 4.00e-04 | 1321.9ms/step | 15.1 samples/s +Step 13500/44000 | Loss: 3.0967 | LR: 4.00e-04 | 1325.7ms/step | 15.1 samples/s +Validation at step 13500: Loss = 3.0960, Perplexity = 22.11 +Step 13510/44000 | Loss: 3.0837 | LR: 4.00e-04 | 3542.7ms/step | 5.6 samples/s +Step 13520/44000 | Loss: 3.0855 | LR: 4.00e-04 | 1323.7ms/step | 15.1 samples/s +Step 13530/44000 | Loss: 3.0813 | LR: 4.00e-04 | 1320.9ms/step | 15.1 samples/s +Step 13540/44000 | Loss: 3.0710 | LR: 4.00e-04 | 1324.4ms/step | 15.1 samples/s +Step 13550/44000 | Loss: 3.0870 | LR: 4.00e-04 | 1324.5ms/step | 15.1 samples/s +Step 13560/44000 | Loss: 3.0935 | LR: 3.99e-04 | 1327.6ms/step | 15.1 samples/s +Step 13570/44000 | Loss: 3.1096 | LR: 3.99e-04 | 1323.2ms/step | 15.1 samples/s +Step 13580/44000 | Loss: 3.1061 | LR: 3.99e-04 | 1322.4ms/step | 15.1 samples/s +Step 13590/44000 | Loss: 3.1099 | LR: 3.99e-04 | 1319.6ms/step | 15.2 samples/s +Step 13600/44000 | Loss: 3.0809 | LR: 3.99e-04 | 1326.1ms/step | 15.1 samples/s +Step 13610/44000 | Loss: 3.0931 | LR: 3.99e-04 | 1328.6ms/step | 15.1 samples/s +Step 13620/44000 | Loss: 3.0932 | LR: 3.98e-04 | 1327.5ms/step | 15.1 samples/s +Step 13630/44000 | Loss: 3.0838 | LR: 3.98e-04 | 1322.9ms/step | 15.1 samples/s +Step 13640/44000 | Loss: 3.0916 | LR: 3.98e-04 | 1324.0ms/step | 15.1 samples/s +Step 13650/44000 | Loss: 3.0790 | LR: 3.98e-04 | 1321.7ms/step | 15.1 samples/s +Step 13660/44000 | Loss: 3.0747 | LR: 3.98e-04 | 1323.9ms/step | 15.1 samples/s +Step 13670/44000 | Loss: 3.1261 | LR: 3.98e-04 | 1325.2ms/step | 15.1 samples/s +Step 13680/44000 | Loss: 3.0719 | LR: 3.98e-04 | 1323.1ms/step | 15.1 samples/s +Step 13690/44000 | Loss: 3.0782 | LR: 3.97e-04 | 1326.3ms/step | 15.1 samples/s +Step 13700/44000 | Loss: 3.1056 | LR: 3.97e-04 | 1325.1ms/step | 15.1 samples/s +Step 13710/44000 | Loss: 3.1100 | LR: 3.97e-04 | 1324.5ms/step | 15.1 samples/s +Step 13720/44000 | Loss: 3.1008 | LR: 3.97e-04 | 1325.9ms/step | 15.1 samples/s +Step 13730/44000 | Loss: 3.1024 | LR: 3.97e-04 | 1325.0ms/step | 15.1 samples/s +Step 13740/44000 | Loss: 3.0498 | LR: 3.97e-04 | 1323.9ms/step | 15.1 samples/s +Step 13750/44000 | Loss: 3.1047 | LR: 3.97e-04 | 1323.0ms/step | 15.1 samples/s +Validation at step 13750: Loss = 3.0924, Perplexity = 22.03 +Step 13760/44000 | Loss: 3.1211 | LR: 3.96e-04 | 3542.0ms/step | 5.6 samples/s +Step 13770/44000 | Loss: 3.0855 | LR: 3.96e-04 | 1324.1ms/step | 15.1 samples/s +Step 13780/44000 | Loss: 3.0985 | LR: 3.96e-04 | 1322.5ms/step | 15.1 samples/s +Step 13790/44000 | Loss: 3.1005 | LR: 3.96e-04 | 1328.5ms/step | 15.1 samples/s +Step 13800/44000 | Loss: 3.0927 | LR: 3.96e-04 | 1332.0ms/step | 15.0 samples/s +Step 13810/44000 | Loss: 3.0806 | LR: 3.96e-04 | 1329.0ms/step | 15.0 samples/s +Step 13820/44000 | Loss: 3.1069 | LR: 3.96e-04 | 1326.2ms/step | 15.1 samples/s +Step 13830/44000 | Loss: 3.0984 | LR: 3.95e-04 | 1331.0ms/step | 15.0 samples/s +Step 13840/44000 | Loss: 3.1009 | LR: 3.95e-04 | 1331.9ms/step | 15.0 samples/s +Step 13850/44000 | Loss: 3.0911 | LR: 3.95e-04 | 1327.0ms/step | 15.1 samples/s +Step 13860/44000 | Loss: 3.0739 | LR: 3.95e-04 | 1329.4ms/step | 15.0 samples/s +Step 13870/44000 | Loss: 3.0619 | LR: 3.95e-04 | 1329.8ms/step | 15.0 samples/s +Step 13880/44000 | Loss: 3.0933 | LR: 3.95e-04 | 1328.0ms/step | 15.1 samples/s +Step 13890/44000 | Loss: 3.0773 | LR: 3.95e-04 | 1333.4ms/step | 15.0 samples/s +Step 13900/44000 | Loss: 3.1030 | LR: 3.94e-04 | 1328.8ms/step | 15.1 samples/s +Step 13910/44000 | Loss: 3.1003 | LR: 3.94e-04 | 1323.4ms/step | 15.1 samples/s +Step 13920/44000 | Loss: 3.0745 | LR: 3.94e-04 | 1328.5ms/step | 15.1 samples/s +Step 13930/44000 | Loss: 3.0782 | LR: 3.94e-04 | 1324.1ms/step | 15.1 samples/s +Step 13940/44000 | Loss: 3.0711 | LR: 3.94e-04 | 1324.7ms/step | 15.1 samples/s +Step 13950/44000 | Loss: 3.0814 | LR: 3.94e-04 | 1327.8ms/step | 15.1 samples/s +Step 13960/44000 | Loss: 3.0834 | LR: 3.93e-04 | 1323.3ms/step | 15.1 samples/s +Step 13970/44000 | Loss: 3.1103 | LR: 3.93e-04 | 1323.6ms/step | 15.1 samples/s +Step 13980/44000 | Loss: 3.0383 | LR: 3.93e-04 | 1322.1ms/step | 15.1 samples/s +Step 13990/44000 | Loss: 3.0826 | LR: 3.93e-04 | 1324.9ms/step | 15.1 samples/s +Step 14000/44000 | Loss: 3.1172 | LR: 3.93e-04 | 1323.1ms/step | 15.1 samples/s +Validation at step 14000: Loss = 3.0882, Perplexity = 21.94 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-14000 +Step 14010/44000 | Loss: 3.0663 | LR: 3.93e-04 | 3853.1ms/step | 5.2 samples/s +Step 14020/44000 | Loss: 3.0818 | LR: 3.93e-04 | 1327.1ms/step | 15.1 samples/s +Step 14030/44000 | Loss: 3.0706 | LR: 3.92e-04 | 1325.7ms/step | 15.1 samples/s +Step 14040/44000 | Loss: 3.0605 | LR: 3.92e-04 | 1321.4ms/step | 15.1 samples/s +Step 14050/44000 | Loss: 3.0662 | LR: 3.92e-04 | 1322.2ms/step | 15.1 samples/s +Step 14060/44000 | Loss: 3.0997 | LR: 3.92e-04 | 1321.5ms/step | 15.1 samples/s +Step 14070/44000 | Loss: 3.0945 | LR: 3.92e-04 | 1320.5ms/step | 15.1 samples/s +Step 14080/44000 | Loss: 3.1204 | LR: 3.92e-04 | 1325.4ms/step | 15.1 samples/s +Step 14090/44000 | Loss: 3.0630 | LR: 3.92e-04 | 1325.1ms/step | 15.1 samples/s +Step 14100/44000 | Loss: 3.1095 | LR: 3.91e-04 | 1325.4ms/step | 15.1 samples/s +Step 14110/44000 | Loss: 3.0770 | LR: 3.91e-04 | 1322.3ms/step | 15.1 samples/s +Step 14120/44000 | Loss: 3.0756 | LR: 3.91e-04 | 1323.3ms/step | 15.1 samples/s +Step 14130/44000 | Loss: 3.0964 | LR: 3.91e-04 | 1326.1ms/step | 15.1 samples/s +Step 14140/44000 | Loss: 3.0900 | LR: 3.91e-04 | 1325.6ms/step | 15.1 samples/s +Step 14150/44000 | Loss: 3.1002 | LR: 3.91e-04 | 1324.4ms/step | 15.1 samples/s +Step 14160/44000 | Loss: 3.0935 | LR: 3.90e-04 | 1326.1ms/step | 15.1 samples/s +Step 14170/44000 | Loss: 3.0996 | LR: 3.90e-04 | 1322.8ms/step | 15.1 samples/s +Step 14180/44000 | Loss: 3.0753 | LR: 3.90e-04 | 1326.4ms/step | 15.1 samples/s +Step 14190/44000 | Loss: 3.1049 | LR: 3.90e-04 | 1327.6ms/step | 15.1 samples/s +Step 14200/44000 | Loss: 3.0817 | LR: 3.90e-04 | 1319.8ms/step | 15.2 samples/s +Step 14210/44000 | Loss: 3.0545 | LR: 3.90e-04 | 1323.1ms/step | 15.1 samples/s +Step 14220/44000 | Loss: 3.0908 | LR: 3.90e-04 | 1328.4ms/step | 15.1 samples/s +Step 14230/44000 | Loss: 3.0799 | LR: 3.89e-04 | 1320.6ms/step | 15.1 samples/s +Step 14240/44000 | Loss: 3.0562 | LR: 3.89e-04 | 1324.2ms/step | 15.1 samples/s +Step 14250/44000 | Loss: 3.0776 | LR: 3.89e-04 | 1323.3ms/step | 15.1 samples/s +Validation at step 14250: Loss = 3.0858, Perplexity = 21.88 +Step 14260/44000 | Loss: 3.0797 | LR: 3.89e-04 | 3550.5ms/step | 5.6 samples/s +Step 14270/44000 | Loss: 3.1025 | LR: 3.89e-04 | 1324.3ms/step | 15.1 samples/s +Step 14280/44000 | Loss: 3.0763 | LR: 3.89e-04 | 1323.1ms/step | 15.1 samples/s +Step 14290/44000 | Loss: 3.1131 | LR: 3.89e-04 | 1324.5ms/step | 15.1 samples/s +Step 14300/44000 | Loss: 3.1120 | LR: 3.88e-04 | 1319.3ms/step | 15.2 samples/s +Step 14310/44000 | Loss: 3.0663 | LR: 3.88e-04 | 1321.4ms/step | 15.1 samples/s +Step 14320/44000 | Loss: 3.0594 | LR: 3.88e-04 | 1320.9ms/step | 15.1 samples/s +Step 14330/44000 | Loss: 3.1100 | LR: 3.88e-04 | 1326.2ms/step | 15.1 samples/s +Step 14340/44000 | Loss: 3.0988 | LR: 3.88e-04 | 1324.8ms/step | 15.1 samples/s +Step 14350/44000 | Loss: 3.0804 | LR: 3.88e-04 | 1325.8ms/step | 15.1 samples/s +Step 14360/44000 | Loss: 3.0864 | LR: 3.87e-04 | 1321.5ms/step | 15.1 samples/s +Step 14370/44000 | Loss: 3.0647 | LR: 3.87e-04 | 1319.9ms/step | 15.2 samples/s +Step 14380/44000 | Loss: 3.0922 | LR: 3.87e-04 | 1324.6ms/step | 15.1 samples/s +Step 14390/44000 | Loss: 3.0943 | LR: 3.87e-04 | 1323.1ms/step | 15.1 samples/s +Step 14400/44000 | Loss: 3.0462 | LR: 3.87e-04 | 1322.0ms/step | 15.1 samples/s +Step 14410/44000 | Loss: 3.1084 | LR: 3.87e-04 | 1322.9ms/step | 15.1 samples/s +Step 14420/44000 | Loss: 3.0841 | LR: 3.87e-04 | 1319.9ms/step | 15.2 samples/s +Step 14430/44000 | Loss: 3.0913 | LR: 3.86e-04 | 1320.4ms/step | 15.1 samples/s +Step 14440/44000 | Loss: 3.0758 | LR: 3.86e-04 | 1324.6ms/step | 15.1 samples/s +Step 14450/44000 | Loss: 3.0914 | LR: 3.86e-04 | 1325.2ms/step | 15.1 samples/s +Step 14460/44000 | Loss: 3.0809 | LR: 3.86e-04 | 1324.4ms/step | 15.1 samples/s +Step 14470/44000 | Loss: 3.0764 | LR: 3.86e-04 | 1326.1ms/step | 15.1 samples/s +Step 14480/44000 | Loss: 3.0645 | LR: 3.86e-04 | 1326.8ms/step | 15.1 samples/s +Step 14490/44000 | Loss: 3.0759 | LR: 3.85e-04 | 1323.0ms/step | 15.1 samples/s +Step 14500/44000 | Loss: 3.0710 | LR: 3.85e-04 | 1324.8ms/step | 15.1 samples/s +Validation at step 14500: Loss = 3.0806, Perplexity = 21.77 +Step 14510/44000 | Loss: 3.0793 | LR: 3.85e-04 | 3547.4ms/step | 5.6 samples/s +Step 14520/44000 | Loss: 3.0813 | LR: 3.85e-04 | 1327.6ms/step | 15.1 samples/s +Step 14530/44000 | Loss: 3.1016 | LR: 3.85e-04 | 1324.4ms/step | 15.1 samples/s +Step 14540/44000 | Loss: 3.0731 | LR: 3.85e-04 | 1320.3ms/step | 15.1 samples/s +Step 14550/44000 | Loss: 3.0857 | LR: 3.85e-04 | 1320.2ms/step | 15.1 samples/s +Step 14560/44000 | Loss: 3.0526 | LR: 3.84e-04 | 1323.2ms/step | 15.1 samples/s +Step 14570/44000 | Loss: 3.0970 | LR: 3.84e-04 | 1323.9ms/step | 15.1 samples/s +Step 14580/44000 | Loss: 3.0784 | LR: 3.84e-04 | 1324.4ms/step | 15.1 samples/s +Step 14590/44000 | Loss: 3.0810 | LR: 3.84e-04 | 1320.2ms/step | 15.1 samples/s +Step 14600/44000 | Loss: 3.0698 | LR: 3.84e-04 | 1321.2ms/step | 15.1 samples/s +Step 14610/44000 | Loss: 3.0278 | LR: 3.84e-04 | 1320.2ms/step | 15.1 samples/s +Step 14620/44000 | Loss: 3.0706 | LR: 3.83e-04 | 1322.2ms/step | 15.1 samples/s +Step 14630/44000 | Loss: 3.0582 | LR: 3.83e-04 | 1322.4ms/step | 15.1 samples/s +Step 14640/44000 | Loss: 3.0480 | LR: 3.83e-04 | 1320.1ms/step | 15.2 samples/s +Step 14650/44000 | Loss: 3.0946 | LR: 3.83e-04 | 1322.6ms/step | 15.1 samples/s +Step 14660/44000 | Loss: 3.0521 | LR: 3.83e-04 | 1324.4ms/step | 15.1 samples/s +Step 14670/44000 | Loss: 3.0729 | LR: 3.83e-04 | 1323.7ms/step | 15.1 samples/s +Step 14680/44000 | Loss: 3.1089 | LR: 3.83e-04 | 1323.1ms/step | 15.1 samples/s +Step 14690/44000 | Loss: 3.1116 | LR: 3.82e-04 | 1321.9ms/step | 15.1 samples/s +Step 14700/44000 | Loss: 3.0774 | LR: 3.82e-04 | 1324.2ms/step | 15.1 samples/s +Step 14710/44000 | Loss: 3.0328 | LR: 3.82e-04 | 1325.4ms/step | 15.1 samples/s +Step 14720/44000 | Loss: 3.0729 | LR: 3.82e-04 | 1324.2ms/step | 15.1 samples/s +Step 14730/44000 | Loss: 3.0778 | LR: 3.82e-04 | 1322.7ms/step | 15.1 samples/s +Step 14740/44000 | Loss: 3.0949 | LR: 3.82e-04 | 1322.6ms/step | 15.1 samples/s +Step 14750/44000 | Loss: 3.0775 | LR: 3.81e-04 | 1322.2ms/step | 15.1 samples/s +Validation at step 14750: Loss = 3.0772, Perplexity = 21.70 +Step 14760/44000 | Loss: 3.0815 | LR: 3.81e-04 | 3542.5ms/step | 5.6 samples/s +Step 14770/44000 | Loss: 3.0755 | LR: 3.81e-04 | 1324.7ms/step | 15.1 samples/s +Step 14780/44000 | Loss: 3.0836 | LR: 3.81e-04 | 1331.2ms/step | 15.0 samples/s +Step 14790/44000 | Loss: 3.0675 | LR: 3.81e-04 | 1323.1ms/step | 15.1 samples/s +Step 14800/44000 | Loss: 3.0708 | LR: 3.81e-04 | 1323.4ms/step | 15.1 samples/s +Step 14810/44000 | Loss: 3.0664 | LR: 3.81e-04 | 1323.8ms/step | 15.1 samples/s +Step 14820/44000 | Loss: 3.0972 | LR: 3.80e-04 | 1323.9ms/step | 15.1 samples/s +Step 14830/44000 | Loss: 3.0544 | LR: 3.80e-04 | 1324.9ms/step | 15.1 samples/s +Step 14840/44000 | Loss: 3.1025 | LR: 3.80e-04 | 1324.6ms/step | 15.1 samples/s +Step 14850/44000 | Loss: 3.0581 | LR: 3.80e-04 | 1323.1ms/step | 15.1 samples/s +Step 14860/44000 | Loss: 3.0757 | LR: 3.80e-04 | 1324.8ms/step | 15.1 samples/s +Step 14870/44000 | Loss: 3.0842 | LR: 3.80e-04 | 1323.1ms/step | 15.1 samples/s +Step 14880/44000 | Loss: 3.0851 | LR: 3.79e-04 | 1326.3ms/step | 15.1 samples/s +Step 14890/44000 | Loss: 3.0675 | LR: 3.79e-04 | 1326.7ms/step | 15.1 samples/s +Step 14900/44000 | Loss: 3.0726 | LR: 3.79e-04 | 1323.9ms/step | 15.1 samples/s +Step 14910/44000 | Loss: 3.0518 | LR: 3.79e-04 | 1321.5ms/step | 15.1 samples/s +Step 14920/44000 | Loss: 3.0964 | LR: 3.79e-04 | 1322.6ms/step | 15.1 samples/s +Step 14930/44000 | Loss: 3.0720 | LR: 3.79e-04 | 1322.6ms/step | 15.1 samples/s +Step 14940/44000 | Loss: 3.0952 | LR: 3.79e-04 | 1321.3ms/step | 15.1 samples/s +Step 14950/44000 | Loss: 3.0656 | LR: 3.78e-04 | 1320.6ms/step | 15.1 samples/s +Step 14960/44000 | Loss: 3.1086 | LR: 3.78e-04 | 1322.2ms/step | 15.1 samples/s +Step 14970/44000 | Loss: 3.0731 | LR: 3.78e-04 | 1320.7ms/step | 15.1 samples/s +Step 14980/44000 | Loss: 3.0821 | LR: 3.78e-04 | 1321.5ms/step | 15.1 samples/s +Step 14990/44000 | Loss: 3.0811 | LR: 3.78e-04 | 1321.7ms/step | 15.1 samples/s +Step 15000/44000 | Loss: 3.0633 | LR: 3.78e-04 | 1325.3ms/step | 15.1 samples/s +Validation at step 15000: Loss = 3.0736, Perplexity = 21.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-15000 +Step 15010/44000 | Loss: 3.0932 | LR: 3.77e-04 | 3838.7ms/step | 5.2 samples/s +Step 15020/44000 | Loss: 3.0826 | LR: 3.77e-04 | 1324.5ms/step | 15.1 samples/s +Step 15030/44000 | Loss: 3.1027 | LR: 3.77e-04 | 1323.3ms/step | 15.1 samples/s +Step 15040/44000 | Loss: 3.0366 | LR: 3.77e-04 | 1325.5ms/step | 15.1 samples/s +Step 15050/44000 | Loss: 3.0436 | LR: 3.77e-04 | 1327.1ms/step | 15.1 samples/s +Step 15060/44000 | Loss: 3.0636 | LR: 3.77e-04 | 1322.6ms/step | 15.1 samples/s +Step 15070/44000 | Loss: 3.0658 | LR: 3.77e-04 | 1324.6ms/step | 15.1 samples/s +Step 15080/44000 | Loss: 3.0574 | LR: 3.76e-04 | 1319.8ms/step | 15.2 samples/s +Step 15090/44000 | Loss: 3.0561 | LR: 3.76e-04 | 1321.3ms/step | 15.1 samples/s +Step 15100/44000 | Loss: 3.0407 | LR: 3.76e-04 | 1320.3ms/step | 15.1 samples/s +Step 15110/44000 | Loss: 3.0743 | LR: 3.76e-04 | 1320.5ms/step | 15.1 samples/s +Step 15120/44000 | Loss: 3.0626 | LR: 3.76e-04 | 1316.9ms/step | 15.2 samples/s +Step 15130/44000 | Loss: 3.0625 | LR: 3.76e-04 | 1317.3ms/step | 15.2 samples/s +Step 15140/44000 | Loss: 3.0525 | LR: 3.75e-04 | 1318.2ms/step | 15.2 samples/s +Step 15150/44000 | Loss: 3.0394 | LR: 3.75e-04 | 1321.1ms/step | 15.1 samples/s +Step 15160/44000 | Loss: 3.0483 | LR: 3.75e-04 | 1321.4ms/step | 15.1 samples/s +Step 15170/44000 | Loss: 3.0548 | LR: 3.75e-04 | 1319.7ms/step | 15.2 samples/s +Step 15180/44000 | Loss: 3.0620 | LR: 3.75e-04 | 1323.5ms/step | 15.1 samples/s +Step 15190/44000 | Loss: 3.0795 | LR: 3.75e-04 | 1323.3ms/step | 15.1 samples/s +Step 15200/44000 | Loss: 3.0771 | LR: 3.74e-04 | 1327.3ms/step | 15.1 samples/s +Step 15210/44000 | Loss: 3.0557 | LR: 3.74e-04 | 1323.5ms/step | 15.1 samples/s +Step 15220/44000 | Loss: 3.0638 | LR: 3.74e-04 | 1323.4ms/step | 15.1 samples/s +Step 15230/44000 | Loss: 3.0525 | LR: 3.74e-04 | 1324.1ms/step | 15.1 samples/s +Step 15240/44000 | Loss: 3.0512 | LR: 3.74e-04 | 1323.6ms/step | 15.1 samples/s +Step 15250/44000 | Loss: 3.0324 | LR: 3.74e-04 | 1322.7ms/step | 15.1 samples/s +Validation at step 15250: Loss = 3.0718, Perplexity = 21.58 +Step 15260/44000 | Loss: 3.0704 | LR: 3.74e-04 | 3545.3ms/step | 5.6 samples/s +Step 15270/44000 | Loss: 3.0755 | LR: 3.73e-04 | 1320.8ms/step | 15.1 samples/s +Step 15280/44000 | Loss: 3.0443 | LR: 3.73e-04 | 1322.2ms/step | 15.1 samples/s +Step 15290/44000 | Loss: 3.0633 | LR: 3.73e-04 | 1324.9ms/step | 15.1 samples/s +Step 15300/44000 | Loss: 3.0765 | LR: 3.73e-04 | 1325.9ms/step | 15.1 samples/s +Step 15310/44000 | Loss: 3.1027 | LR: 3.73e-04 | 1320.7ms/step | 15.1 samples/s +Step 15320/44000 | Loss: 3.0667 | LR: 3.73e-04 | 1324.8ms/step | 15.1 samples/s +Step 15330/44000 | Loss: 3.0752 | LR: 3.72e-04 | 1324.8ms/step | 15.1 samples/s +Step 15340/44000 | Loss: 3.0456 | LR: 3.72e-04 | 1325.5ms/step | 15.1 samples/s +Step 15350/44000 | Loss: 3.0679 | LR: 3.72e-04 | 1323.8ms/step | 15.1 samples/s +Step 15360/44000 | Loss: 3.0513 | LR: 3.72e-04 | 1326.9ms/step | 15.1 samples/s +Step 15370/44000 | Loss: 3.0682 | LR: 3.72e-04 | 1325.5ms/step | 15.1 samples/s +Step 15380/44000 | Loss: 3.0605 | LR: 3.72e-04 | 1321.5ms/step | 15.1 samples/s +Step 15390/44000 | Loss: 3.0631 | LR: 3.71e-04 | 1321.9ms/step | 15.1 samples/s +Step 15400/44000 | Loss: 3.0809 | LR: 3.71e-04 | 1322.4ms/step | 15.1 samples/s +Step 15410/44000 | Loss: 3.0516 | LR: 3.71e-04 | 1326.2ms/step | 15.1 samples/s +Step 15420/44000 | Loss: 3.0851 | LR: 3.71e-04 | 1324.3ms/step | 15.1 samples/s +Step 15430/44000 | Loss: 3.0739 | LR: 3.71e-04 | 1326.9ms/step | 15.1 samples/s +Step 15440/44000 | Loss: 3.0511 | LR: 3.71e-04 | 1322.8ms/step | 15.1 samples/s +Step 15450/44000 | Loss: 3.0648 | LR: 3.71e-04 | 1322.6ms/step | 15.1 samples/s +Step 15460/44000 | Loss: 3.0454 | LR: 3.70e-04 | 1323.4ms/step | 15.1 samples/s +Step 15470/44000 | Loss: 3.0660 | LR: 3.70e-04 | 1324.0ms/step | 15.1 samples/s +Step 15480/44000 | Loss: 3.0345 | LR: 3.70e-04 | 1323.1ms/step | 15.1 samples/s +Step 15490/44000 | Loss: 3.0687 | LR: 3.70e-04 | 1323.9ms/step | 15.1 samples/s +Step 15500/44000 | Loss: 3.0724 | LR: 3.70e-04 | 1323.1ms/step | 15.1 samples/s +Validation at step 15500: Loss = 3.0672, Perplexity = 21.48 +Step 15510/44000 | Loss: 3.0864 | LR: 3.70e-04 | 3544.7ms/step | 5.6 samples/s +Step 15520/44000 | Loss: 3.0791 | LR: 3.69e-04 | 1323.9ms/step | 15.1 samples/s +Step 15530/44000 | Loss: 3.0773 | LR: 3.69e-04 | 1326.2ms/step | 15.1 samples/s +Step 15540/44000 | Loss: 3.0712 | LR: 3.69e-04 | 1322.4ms/step | 15.1 samples/s +Step 15550/44000 | Loss: 3.0752 | LR: 3.69e-04 | 1321.9ms/step | 15.1 samples/s +Step 15560/44000 | Loss: 3.0539 | LR: 3.69e-04 | 1321.8ms/step | 15.1 samples/s +Step 15570/44000 | Loss: 3.0385 | LR: 3.69e-04 | 1325.3ms/step | 15.1 samples/s +Step 15580/44000 | Loss: 3.0899 | LR: 3.68e-04 | 1323.4ms/step | 15.1 samples/s +Step 15590/44000 | Loss: 3.0597 | LR: 3.68e-04 | 1324.7ms/step | 15.1 samples/s +Step 15600/44000 | Loss: 3.0615 | LR: 3.68e-04 | 1325.1ms/step | 15.1 samples/s +Step 15610/44000 | Loss: 3.0500 | LR: 3.68e-04 | 1321.3ms/step | 15.1 samples/s +Step 15620/44000 | Loss: 3.0559 | LR: 3.68e-04 | 1322.9ms/step | 15.1 samples/s +Step 15630/44000 | Loss: 3.0393 | LR: 3.68e-04 | 1323.4ms/step | 15.1 samples/s +Step 15640/44000 | Loss: 3.0497 | LR: 3.67e-04 | 1324.8ms/step | 15.1 samples/s +Step 15650/44000 | Loss: 3.0580 | LR: 3.67e-04 | 1326.6ms/step | 15.1 samples/s +Step 15660/44000 | Loss: 3.0884 | LR: 3.67e-04 | 1325.5ms/step | 15.1 samples/s +Step 15670/44000 | Loss: 3.0601 | LR: 3.67e-04 | 1322.0ms/step | 15.1 samples/s +Step 15680/44000 | Loss: 3.0625 | LR: 3.67e-04 | 1324.3ms/step | 15.1 samples/s +Step 15690/44000 | Loss: 3.0618 | LR: 3.67e-04 | 1323.6ms/step | 15.1 samples/s +Step 15700/44000 | Loss: 3.0846 | LR: 3.67e-04 | 1322.0ms/step | 15.1 samples/s +Step 15710/44000 | Loss: 3.0431 | LR: 3.66e-04 | 1321.1ms/step | 15.1 samples/s +Step 15720/44000 | Loss: 3.0523 | LR: 3.66e-04 | 1323.0ms/step | 15.1 samples/s +Step 15730/44000 | Loss: 3.0766 | LR: 3.66e-04 | 1322.6ms/step | 15.1 samples/s +Step 15740/44000 | Loss: 3.0740 | LR: 3.66e-04 | 1323.2ms/step | 15.1 samples/s +Step 15750/44000 | Loss: 3.0544 | LR: 3.66e-04 | 1317.9ms/step | 15.2 samples/s +Validation at step 15750: Loss = 3.0640, Perplexity = 21.41 +Step 15760/44000 | Loss: 3.0728 | LR: 3.66e-04 | 3544.0ms/step | 5.6 samples/s +Step 15770/44000 | Loss: 3.0728 | LR: 3.65e-04 | 1322.2ms/step | 15.1 samples/s +Step 15780/44000 | Loss: 3.0543 | LR: 3.65e-04 | 1322.8ms/step | 15.1 samples/s +Step 15790/44000 | Loss: 3.0747 | LR: 3.65e-04 | 1324.9ms/step | 15.1 samples/s +Step 15800/44000 | Loss: 3.0444 | LR: 3.65e-04 | 1321.5ms/step | 15.1 samples/s +Step 15810/44000 | Loss: 3.0478 | LR: 3.65e-04 | 1322.5ms/step | 15.1 samples/s +Step 15820/44000 | Loss: 3.0682 | LR: 3.65e-04 | 1323.6ms/step | 15.1 samples/s +Step 15830/44000 | Loss: 3.0695 | LR: 3.64e-04 | 1324.4ms/step | 15.1 samples/s +Step 15840/44000 | Loss: 3.0455 | LR: 3.64e-04 | 1322.5ms/step | 15.1 samples/s +Step 15850/44000 | Loss: 3.0391 | LR: 3.64e-04 | 1322.0ms/step | 15.1 samples/s +Step 15860/44000 | Loss: 3.0708 | LR: 3.64e-04 | 1321.8ms/step | 15.1 samples/s +Step 15870/44000 | Loss: 3.0708 | LR: 3.64e-04 | 1326.3ms/step | 15.1 samples/s +Step 15880/44000 | Loss: 3.0568 | LR: 3.64e-04 | 1324.6ms/step | 15.1 samples/s +Step 15890/44000 | Loss: 3.0607 | LR: 3.63e-04 | 1324.9ms/step | 15.1 samples/s +Step 15900/44000 | Loss: 3.0635 | LR: 3.63e-04 | 1328.4ms/step | 15.1 samples/s +Step 15910/44000 | Loss: 3.0488 | LR: 3.63e-04 | 1324.9ms/step | 15.1 samples/s +Step 15920/44000 | Loss: 3.0538 | LR: 3.63e-04 | 1325.3ms/step | 15.1 samples/s +Step 15930/44000 | Loss: 3.0786 | LR: 3.63e-04 | 1321.6ms/step | 15.1 samples/s +Step 15940/44000 | Loss: 3.0370 | LR: 3.63e-04 | 1323.6ms/step | 15.1 samples/s +Step 15950/44000 | Loss: 3.0615 | LR: 3.62e-04 | 1321.4ms/step | 15.1 samples/s +Step 15960/44000 | Loss: 3.0275 | LR: 3.62e-04 | 1322.5ms/step | 15.1 samples/s +Step 15970/44000 | Loss: 3.0698 | LR: 3.62e-04 | 1325.2ms/step | 15.1 samples/s +Step 15980/44000 | Loss: 3.0579 | LR: 3.62e-04 | 1322.6ms/step | 15.1 samples/s +Step 15990/44000 | Loss: 3.0638 | LR: 3.62e-04 | 1322.5ms/step | 15.1 samples/s +Step 16000/44000 | Loss: 3.0595 | LR: 3.62e-04 | 1325.8ms/step | 15.1 samples/s +Validation at step 16000: Loss = 3.0604, Perplexity = 21.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-16000 +Step 16010/44000 | Loss: 3.0743 | LR: 3.62e-04 | 3844.7ms/step | 5.2 samples/s +Step 16020/44000 | Loss: 3.0609 | LR: 3.61e-04 | 1319.3ms/step | 15.2 samples/s +Step 16030/44000 | Loss: 3.0508 | LR: 3.61e-04 | 1321.0ms/step | 15.1 samples/s +Step 16040/44000 | Loss: 3.0749 | LR: 3.61e-04 | 1322.2ms/step | 15.1 samples/s +Step 16050/44000 | Loss: 3.0657 | LR: 3.61e-04 | 1321.9ms/step | 15.1 samples/s +Step 16060/44000 | Loss: 3.0676 | LR: 3.61e-04 | 1322.5ms/step | 15.1 samples/s +Step 16070/44000 | Loss: 3.0398 | LR: 3.61e-04 | 1322.6ms/step | 15.1 samples/s +Step 16080/44000 | Loss: 3.0499 | LR: 3.60e-04 | 1323.4ms/step | 15.1 samples/s +Step 16090/44000 | Loss: 3.0321 | LR: 3.60e-04 | 1324.9ms/step | 15.1 samples/s +Step 16100/44000 | Loss: 3.0663 | LR: 3.60e-04 | 1324.6ms/step | 15.1 samples/s +Step 16110/44000 | Loss: 3.0584 | LR: 3.60e-04 | 1326.0ms/step | 15.1 samples/s +Step 16120/44000 | Loss: 3.0481 | LR: 3.60e-04 | 1328.4ms/step | 15.1 samples/s +Step 16130/44000 | Loss: 3.0354 | LR: 3.60e-04 | 1323.0ms/step | 15.1 samples/s +Step 16140/44000 | Loss: 3.0395 | LR: 3.59e-04 | 1324.6ms/step | 15.1 samples/s +Step 16150/44000 | Loss: 3.0599 | LR: 3.59e-04 | 1322.3ms/step | 15.1 samples/s +Step 16160/44000 | Loss: 3.0663 | LR: 3.59e-04 | 1326.3ms/step | 15.1 samples/s +Step 16170/44000 | Loss: 3.0707 | LR: 3.59e-04 | 1325.6ms/step | 15.1 samples/s +Step 16180/44000 | Loss: 3.0797 | LR: 3.59e-04 | 1322.9ms/step | 15.1 samples/s +Step 16190/44000 | Loss: 3.0660 | LR: 3.59e-04 | 1323.5ms/step | 15.1 samples/s +Step 16200/44000 | Loss: 3.0488 | LR: 3.58e-04 | 1323.7ms/step | 15.1 samples/s +Step 16210/44000 | Loss: 3.0550 | LR: 3.58e-04 | 1320.7ms/step | 15.1 samples/s +Step 16220/44000 | Loss: 3.0461 | LR: 3.58e-04 | 1323.7ms/step | 15.1 samples/s +Step 16230/44000 | Loss: 3.0845 | LR: 3.58e-04 | 1325.3ms/step | 15.1 samples/s +Step 16240/44000 | Loss: 3.0650 | LR: 3.58e-04 | 1324.3ms/step | 15.1 samples/s +Step 16250/44000 | Loss: 3.0342 | LR: 3.58e-04 | 1325.3ms/step | 15.1 samples/s +Validation at step 16250: Loss = 3.0573, Perplexity = 21.27 +Step 16260/44000 | Loss: 3.0554 | LR: 3.57e-04 | 3550.0ms/step | 5.6 samples/s +Step 16270/44000 | Loss: 3.0213 | LR: 3.57e-04 | 1324.3ms/step | 15.1 samples/s +Step 16280/44000 | Loss: 3.0539 | LR: 3.57e-04 | 1323.1ms/step | 15.1 samples/s +Step 16290/44000 | Loss: 3.0757 | LR: 3.57e-04 | 1327.4ms/step | 15.1 samples/s +Step 16300/44000 | Loss: 3.0483 | LR: 3.57e-04 | 1326.4ms/step | 15.1 samples/s +Step 16310/44000 | Loss: 3.0813 | LR: 3.57e-04 | 1326.6ms/step | 15.1 samples/s +Step 16320/44000 | Loss: 3.0505 | LR: 3.56e-04 | 1325.8ms/step | 15.1 samples/s +Step 16330/44000 | Loss: 3.0694 | LR: 3.56e-04 | 1321.1ms/step | 15.1 samples/s +Step 16340/44000 | Loss: 3.0519 | LR: 3.56e-04 | 1324.2ms/step | 15.1 samples/s +Step 16350/44000 | Loss: 3.0511 | LR: 3.56e-04 | 1322.6ms/step | 15.1 samples/s +Step 16360/44000 | Loss: 3.0223 | LR: 3.56e-04 | 1323.0ms/step | 15.1 samples/s +Step 16370/44000 | Loss: 3.0192 | LR: 3.56e-04 | 1325.9ms/step | 15.1 samples/s +Step 16380/44000 | Loss: 3.0583 | LR: 3.55e-04 | 1322.2ms/step | 15.1 samples/s +Step 16390/44000 | Loss: 3.0406 | LR: 3.55e-04 | 1321.2ms/step | 15.1 samples/s +Step 16400/44000 | Loss: 3.0618 | LR: 3.55e-04 | 1320.5ms/step | 15.1 samples/s +Step 16410/44000 | Loss: 3.0642 | LR: 3.55e-04 | 1326.1ms/step | 15.1 samples/s +Step 16420/44000 | Loss: 3.0571 | LR: 3.55e-04 | 1324.1ms/step | 15.1 samples/s +Step 16430/44000 | Loss: 3.0460 | LR: 3.55e-04 | 1323.5ms/step | 15.1 samples/s +Step 16440/44000 | Loss: 3.0404 | LR: 3.54e-04 | 1322.7ms/step | 15.1 samples/s +Step 16450/44000 | Loss: 3.0807 | LR: 3.54e-04 | 1326.0ms/step | 15.1 samples/s +Step 16460/44000 | Loss: 3.0510 | LR: 3.54e-04 | 1326.1ms/step | 15.1 samples/s +Step 16470/44000 | Loss: 3.0277 | LR: 3.54e-04 | 1323.3ms/step | 15.1 samples/s +Step 16480/44000 | Loss: 3.0612 | LR: 3.54e-04 | 1321.4ms/step | 15.1 samples/s +Step 16490/44000 | Loss: 3.0877 | LR: 3.54e-04 | 1320.4ms/step | 15.1 samples/s +Step 16500/44000 | Loss: 3.0744 | LR: 3.53e-04 | 1323.0ms/step | 15.1 samples/s +Validation at step 16500: Loss = 3.0547, Perplexity = 21.22 +Step 16510/44000 | Loss: 3.0479 | LR: 3.53e-04 | 3549.2ms/step | 5.6 samples/s +Step 16520/44000 | Loss: 3.0612 | LR: 3.53e-04 | 1324.6ms/step | 15.1 samples/s +Step 16530/44000 | Loss: 3.0573 | LR: 3.53e-04 | 1322.3ms/step | 15.1 samples/s +Step 16540/44000 | Loss: 3.0394 | LR: 3.53e-04 | 1323.6ms/step | 15.1 samples/s +Step 16550/44000 | Loss: 3.0276 | LR: 3.53e-04 | 1320.1ms/step | 15.2 samples/s +Step 16560/44000 | Loss: 3.0765 | LR: 3.52e-04 | 1324.5ms/step | 15.1 samples/s +Step 16570/44000 | Loss: 3.0435 | LR: 3.52e-04 | 1322.0ms/step | 15.1 samples/s +Step 16580/44000 | Loss: 3.0547 | LR: 3.52e-04 | 1323.4ms/step | 15.1 samples/s +Step 16590/44000 | Loss: 3.0275 | LR: 3.52e-04 | 1321.8ms/step | 15.1 samples/s +Step 16600/44000 | Loss: 3.0609 | LR: 3.52e-04 | 1323.9ms/step | 15.1 samples/s +Step 16610/44000 | Loss: 3.0504 | LR: 3.52e-04 | 1326.0ms/step | 15.1 samples/s +Step 16620/44000 | Loss: 3.0537 | LR: 3.51e-04 | 1321.3ms/step | 15.1 samples/s +Step 16630/44000 | Loss: 3.0689 | LR: 3.51e-04 | 1323.1ms/step | 15.1 samples/s +Step 16640/44000 | Loss: 3.0348 | LR: 3.51e-04 | 1325.5ms/step | 15.1 samples/s +Step 16650/44000 | Loss: 3.0275 | LR: 3.51e-04 | 1322.5ms/step | 15.1 samples/s +Step 16660/44000 | Loss: 3.0369 | LR: 3.51e-04 | 1322.9ms/step | 15.1 samples/s +Step 16670/44000 | Loss: 3.0457 | LR: 3.51e-04 | 1323.2ms/step | 15.1 samples/s +Step 16680/44000 | Loss: 3.0651 | LR: 3.50e-04 | 1322.4ms/step | 15.1 samples/s +Step 16690/44000 | Loss: 3.0546 | LR: 3.50e-04 | 1321.7ms/step | 15.1 samples/s +Step 16700/44000 | Loss: 3.0346 | LR: 3.50e-04 | 1325.1ms/step | 15.1 samples/s +Step 16710/44000 | Loss: 3.0498 | LR: 3.50e-04 | 1324.5ms/step | 15.1 samples/s +Step 16720/44000 | Loss: 3.0482 | LR: 3.50e-04 | 1319.8ms/step | 15.2 samples/s +Step 16730/44000 | Loss: 3.0764 | LR: 3.50e-04 | 1322.3ms/step | 15.1 samples/s +Step 16740/44000 | Loss: 3.0521 | LR: 3.49e-04 | 1322.3ms/step | 15.1 samples/s +Step 16750/44000 | Loss: 3.0475 | LR: 3.49e-04 | 1323.8ms/step | 15.1 samples/s +Validation at step 16750: Loss = 3.0518, Perplexity = 21.15 +Step 16760/44000 | Loss: 3.0292 | LR: 3.49e-04 | 3544.1ms/step | 5.6 samples/s +Step 16770/44000 | Loss: 3.0393 | LR: 3.49e-04 | 1325.4ms/step | 15.1 samples/s +Step 16780/44000 | Loss: 3.0539 | LR: 3.49e-04 | 1321.1ms/step | 15.1 samples/s +Step 16790/44000 | Loss: 3.0467 | LR: 3.49e-04 | 1322.9ms/step | 15.1 samples/s +Step 16800/44000 | Loss: 3.0897 | LR: 3.48e-04 | 1324.3ms/step | 15.1 samples/s +Step 16810/44000 | Loss: 3.0582 | LR: 3.48e-04 | 1322.4ms/step | 15.1 samples/s +Step 16820/44000 | Loss: 3.0348 | LR: 3.48e-04 | 1322.8ms/step | 15.1 samples/s +Step 16830/44000 | Loss: 3.0543 | LR: 3.48e-04 | 1325.1ms/step | 15.1 samples/s +Step 16840/44000 | Loss: 3.0354 | LR: 3.48e-04 | 1322.7ms/step | 15.1 samples/s +Step 16850/44000 | Loss: 3.0424 | LR: 3.48e-04 | 1322.6ms/step | 15.1 samples/s +Step 16860/44000 | Loss: 3.0626 | LR: 3.47e-04 | 1318.5ms/step | 15.2 samples/s +Step 16870/44000 | Loss: 3.0355 | LR: 3.47e-04 | 1322.8ms/step | 15.1 samples/s +Step 16880/44000 | Loss: 3.0299 | LR: 3.47e-04 | 1323.6ms/step | 15.1 samples/s +Step 16890/44000 | Loss: 3.0013 | LR: 3.47e-04 | 1324.9ms/step | 15.1 samples/s +Step 16900/44000 | Loss: 3.0524 | LR: 3.47e-04 | 1323.5ms/step | 15.1 samples/s +Step 16910/44000 | Loss: 3.0517 | LR: 3.47e-04 | 1324.8ms/step | 15.1 samples/s +Step 16920/44000 | Loss: 3.0367 | LR: 3.46e-04 | 1324.2ms/step | 15.1 samples/s +Step 16930/44000 | Loss: 3.0492 | LR: 3.46e-04 | 1324.8ms/step | 15.1 samples/s +Step 16940/44000 | Loss: 3.0634 | LR: 3.46e-04 | 1323.0ms/step | 15.1 samples/s +Step 16950/44000 | Loss: 3.0368 | LR: 3.46e-04 | 1319.3ms/step | 15.2 samples/s +Step 16960/44000 | Loss: 3.0646 | LR: 3.46e-04 | 1323.0ms/step | 15.1 samples/s +Step 16970/44000 | Loss: 3.0383 | LR: 3.46e-04 | 1323.8ms/step | 15.1 samples/s +Step 16980/44000 | Loss: 3.0395 | LR: 3.45e-04 | 1322.3ms/step | 15.1 samples/s +Step 16990/44000 | Loss: 3.0493 | LR: 3.45e-04 | 1321.4ms/step | 15.1 samples/s +Step 17000/44000 | Loss: 3.0524 | LR: 3.45e-04 | 1320.6ms/step | 15.1 samples/s +Validation at step 17000: Loss = 3.0478, Perplexity = 21.07 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-17000 +Step 17010/44000 | Loss: 3.0648 | LR: 3.45e-04 | 3836.6ms/step | 5.2 samples/s +Step 17020/44000 | Loss: 3.0627 | LR: 3.45e-04 | 1320.0ms/step | 15.2 samples/s +Step 17030/44000 | Loss: 3.0475 | LR: 3.45e-04 | 1320.2ms/step | 15.1 samples/s +Step 17040/44000 | Loss: 3.0337 | LR: 3.44e-04 | 1321.4ms/step | 15.1 samples/s +Step 17050/44000 | Loss: 3.0444 | LR: 3.44e-04 | 1322.2ms/step | 15.1 samples/s +Step 17060/44000 | Loss: 3.0628 | LR: 3.44e-04 | 1323.2ms/step | 15.1 samples/s +Step 17070/44000 | Loss: 3.0587 | LR: 3.44e-04 | 1325.3ms/step | 15.1 samples/s +Step 17080/44000 | Loss: 3.0510 | LR: 3.44e-04 | 1326.5ms/step | 15.1 samples/s +Step 17090/44000 | Loss: 3.0148 | LR: 3.44e-04 | 1322.7ms/step | 15.1 samples/s +Step 17100/44000 | Loss: 3.0423 | LR: 3.43e-04 | 1327.7ms/step | 15.1 samples/s +Step 17110/44000 | Loss: 3.0147 | LR: 3.43e-04 | 1327.1ms/step | 15.1 samples/s +Step 17120/44000 | Loss: 3.0443 | LR: 3.43e-04 | 1325.8ms/step | 15.1 samples/s +Step 17130/44000 | Loss: 3.0921 | LR: 3.43e-04 | 1322.8ms/step | 15.1 samples/s +Step 17140/44000 | Loss: 3.0613 | LR: 3.43e-04 | 1322.2ms/step | 15.1 samples/s +Step 17150/44000 | Loss: 3.0392 | LR: 3.43e-04 | 1325.8ms/step | 15.1 samples/s +Step 17160/44000 | Loss: 3.0185 | LR: 3.42e-04 | 1321.7ms/step | 15.1 samples/s +Step 17170/44000 | Loss: 3.0429 | LR: 3.42e-04 | 1325.7ms/step | 15.1 samples/s +Step 17180/44000 | Loss: 3.0204 | LR: 3.42e-04 | 1323.0ms/step | 15.1 samples/s +Step 17190/44000 | Loss: 3.0407 | LR: 3.42e-04 | 1325.5ms/step | 15.1 samples/s +Step 17200/44000 | Loss: 3.0586 | LR: 3.42e-04 | 1324.4ms/step | 15.1 samples/s +Step 17210/44000 | Loss: 3.0616 | LR: 3.42e-04 | 1322.5ms/step | 15.1 samples/s +Step 17220/44000 | Loss: 3.0333 | LR: 3.41e-04 | 1325.1ms/step | 15.1 samples/s +Step 17230/44000 | Loss: 3.0497 | LR: 3.41e-04 | 1324.4ms/step | 15.1 samples/s +Step 17240/44000 | Loss: 3.0394 | LR: 3.41e-04 | 1320.8ms/step | 15.1 samples/s +Step 17250/44000 | Loss: 3.0360 | LR: 3.41e-04 | 1324.4ms/step | 15.1 samples/s +Validation at step 17250: Loss = 3.0451, Perplexity = 21.01 +Step 17260/44000 | Loss: 3.0439 | LR: 3.41e-04 | 3545.1ms/step | 5.6 samples/s +Step 17270/44000 | Loss: 3.0686 | LR: 3.41e-04 | 1324.7ms/step | 15.1 samples/s +Step 17280/44000 | Loss: 3.0373 | LR: 3.40e-04 | 1324.7ms/step | 15.1 samples/s +Step 17290/44000 | Loss: 2.9889 | LR: 3.40e-04 | 1327.6ms/step | 15.1 samples/s +Step 17300/44000 | Loss: 3.0339 | LR: 3.40e-04 | 1324.0ms/step | 15.1 samples/s +Step 17310/44000 | Loss: 3.0347 | LR: 3.40e-04 | 1324.9ms/step | 15.1 samples/s +Step 17320/44000 | Loss: 3.0211 | LR: 3.40e-04 | 1325.0ms/step | 15.1 samples/s +Step 17330/44000 | Loss: 3.0323 | LR: 3.40e-04 | 1322.2ms/step | 15.1 samples/s +Step 17340/44000 | Loss: 3.0500 | LR: 3.39e-04 | 1325.2ms/step | 15.1 samples/s +Step 17350/44000 | Loss: 3.0482 | LR: 3.39e-04 | 1324.4ms/step | 15.1 samples/s +Step 17360/44000 | Loss: 3.0227 | LR: 3.39e-04 | 1325.1ms/step | 15.1 samples/s +Step 17370/44000 | Loss: 3.0496 | LR: 3.39e-04 | 1324.9ms/step | 15.1 samples/s +Step 17380/44000 | Loss: 3.0584 | LR: 3.39e-04 | 1323.7ms/step | 15.1 samples/s +Step 17390/44000 | Loss: 3.0663 | LR: 3.39e-04 | 1323.7ms/step | 15.1 samples/s +Step 17400/44000 | Loss: 3.0015 | LR: 3.38e-04 | 1325.5ms/step | 15.1 samples/s +Step 17410/44000 | Loss: 3.0571 | LR: 3.38e-04 | 1323.5ms/step | 15.1 samples/s +Step 17420/44000 | Loss: 3.0475 | LR: 3.38e-04 | 1322.0ms/step | 15.1 samples/s +Step 17430/44000 | Loss: 3.0346 | LR: 3.38e-04 | 1323.1ms/step | 15.1 samples/s +Step 17440/44000 | Loss: 3.0280 | LR: 3.38e-04 | 1322.8ms/step | 15.1 samples/s +Step 17450/44000 | Loss: 3.0615 | LR: 3.38e-04 | 1323.2ms/step | 15.1 samples/s +Step 17460/44000 | Loss: 3.0527 | LR: 3.37e-04 | 1324.9ms/step | 15.1 samples/s +Step 17470/44000 | Loss: 3.0319 | LR: 3.37e-04 | 1325.4ms/step | 15.1 samples/s +Step 17480/44000 | Loss: 3.0606 | LR: 3.37e-04 | 1322.5ms/step | 15.1 samples/s +Step 17490/44000 | Loss: 3.0255 | LR: 3.37e-04 | 1322.4ms/step | 15.1 samples/s +Step 17500/44000 | Loss: 3.0487 | LR: 3.37e-04 | 1324.1ms/step | 15.1 samples/s +Validation at step 17500: Loss = 3.0422, Perplexity = 20.95 +Step 17510/44000 | Loss: 3.0311 | LR: 3.37e-04 | 3547.5ms/step | 5.6 samples/s +Step 17520/44000 | Loss: 3.0417 | LR: 3.36e-04 | 1325.4ms/step | 15.1 samples/s +Step 17530/44000 | Loss: 3.0091 | LR: 3.36e-04 | 1324.5ms/step | 15.1 samples/s +Step 17540/44000 | Loss: 3.0282 | LR: 3.36e-04 | 1324.6ms/step | 15.1 samples/s +Step 17550/44000 | Loss: 3.0575 | LR: 3.36e-04 | 1324.9ms/step | 15.1 samples/s +Step 17560/44000 | Loss: 3.0513 | LR: 3.36e-04 | 1323.3ms/step | 15.1 samples/s +Step 17570/44000 | Loss: 3.0232 | LR: 3.35e-04 | 1323.2ms/step | 15.1 samples/s +Step 17580/44000 | Loss: 3.0539 | LR: 3.35e-04 | 1324.3ms/step | 15.1 samples/s +Step 17590/44000 | Loss: 3.0439 | LR: 3.35e-04 | 1325.1ms/step | 15.1 samples/s +Step 17600/44000 | Loss: 3.0401 | LR: 3.35e-04 | 1326.9ms/step | 15.1 samples/s +Step 17610/44000 | Loss: 3.0130 | LR: 3.35e-04 | 1325.7ms/step | 15.1 samples/s +Step 17620/44000 | Loss: 3.0459 | LR: 3.35e-04 | 1321.7ms/step | 15.1 samples/s +Step 17630/44000 | Loss: 3.0073 | LR: 3.34e-04 | 1324.1ms/step | 15.1 samples/s +Step 17640/44000 | Loss: 3.0141 | LR: 3.34e-04 | 1324.0ms/step | 15.1 samples/s +Step 17650/44000 | Loss: 3.0566 | LR: 3.34e-04 | 1322.1ms/step | 15.1 samples/s +Step 17660/44000 | Loss: 3.0330 | LR: 3.34e-04 | 1325.2ms/step | 15.1 samples/s +Step 17670/44000 | Loss: 3.0326 | LR: 3.34e-04 | 1323.0ms/step | 15.1 samples/s +Step 17680/44000 | Loss: 3.0357 | LR: 3.34e-04 | 1323.7ms/step | 15.1 samples/s +Step 17690/44000 | Loss: 3.0513 | LR: 3.33e-04 | 1324.8ms/step | 15.1 samples/s +Step 17700/44000 | Loss: 3.0348 | LR: 3.33e-04 | 1322.6ms/step | 15.1 samples/s +Step 17710/44000 | Loss: 3.0134 | LR: 3.33e-04 | 1323.7ms/step | 15.1 samples/s +Step 17720/44000 | Loss: 3.0395 | LR: 3.33e-04 | 1321.3ms/step | 15.1 samples/s +Step 17730/44000 | Loss: 3.0363 | LR: 3.33e-04 | 1323.5ms/step | 15.1 samples/s +Step 17740/44000 | Loss: 3.0289 | LR: 3.33e-04 | 1327.3ms/step | 15.1 samples/s +Step 17750/44000 | Loss: 3.0452 | LR: 3.32e-04 | 1323.5ms/step | 15.1 samples/s +Validation at step 17750: Loss = 3.0386, Perplexity = 20.88 +Step 17760/44000 | Loss: 3.0547 | LR: 3.32e-04 | 3540.8ms/step | 5.6 samples/s +Step 17770/44000 | Loss: 3.0096 | LR: 3.32e-04 | 1319.5ms/step | 15.2 samples/s +Step 17780/44000 | Loss: 3.0158 | LR: 3.32e-04 | 1322.2ms/step | 15.1 samples/s +Step 17790/44000 | Loss: 3.0501 | LR: 3.32e-04 | 1322.6ms/step | 15.1 samples/s +Step 17800/44000 | Loss: 3.0095 | LR: 3.32e-04 | 1322.8ms/step | 15.1 samples/s +Step 17810/44000 | Loss: 3.0317 | LR: 3.31e-04 | 1321.6ms/step | 15.1 samples/s +Step 17820/44000 | Loss: 3.0118 | LR: 3.31e-04 | 1322.8ms/step | 15.1 samples/s +Step 17830/44000 | Loss: 3.0327 | LR: 3.31e-04 | 1324.3ms/step | 15.1 samples/s +Step 17840/44000 | Loss: 3.0293 | LR: 3.31e-04 | 1322.8ms/step | 15.1 samples/s +Step 17850/44000 | Loss: 3.0784 | LR: 3.31e-04 | 1325.7ms/step | 15.1 samples/s +Step 17860/44000 | Loss: 3.0322 | LR: 3.31e-04 | 1324.4ms/step | 15.1 samples/s +Step 17870/44000 | Loss: 3.0343 | LR: 3.30e-04 | 1325.9ms/step | 15.1 samples/s +Step 17880/44000 | Loss: 3.0214 | LR: 3.30e-04 | 1321.8ms/step | 15.1 samples/s +Step 17890/44000 | Loss: 3.0493 | LR: 3.30e-04 | 1320.9ms/step | 15.1 samples/s +Step 17900/44000 | Loss: 3.0304 | LR: 3.30e-04 | 1322.2ms/step | 15.1 samples/s +Step 17910/44000 | Loss: 3.0332 | LR: 3.30e-04 | 1325.7ms/step | 15.1 samples/s +Step 17920/44000 | Loss: 3.0430 | LR: 3.29e-04 | 1326.1ms/step | 15.1 samples/s +Step 17930/44000 | Loss: 2.9996 | LR: 3.29e-04 | 1323.6ms/step | 15.1 samples/s +Step 17940/44000 | Loss: 3.0277 | LR: 3.29e-04 | 1321.4ms/step | 15.1 samples/s +Step 17950/44000 | Loss: 2.9976 | LR: 3.29e-04 | 1320.3ms/step | 15.1 samples/s +Step 17960/44000 | Loss: 3.0231 | LR: 3.29e-04 | 1325.1ms/step | 15.1 samples/s +Step 17970/44000 | Loss: 3.0430 | LR: 3.29e-04 | 1322.5ms/step | 15.1 samples/s +Step 17980/44000 | Loss: 3.0246 | LR: 3.28e-04 | 1325.1ms/step | 15.1 samples/s +Step 17990/44000 | Loss: 3.0219 | LR: 3.28e-04 | 1324.8ms/step | 15.1 samples/s +Step 18000/44000 | Loss: 3.0680 | LR: 3.28e-04 | 1326.4ms/step | 15.1 samples/s +Validation at step 18000: Loss = 3.0367, Perplexity = 20.84 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-18000 +Step 18010/44000 | Loss: 3.0615 | LR: 3.28e-04 | 3830.8ms/step | 5.2 samples/s +Step 18020/44000 | Loss: 3.0341 | LR: 3.28e-04 | 1320.5ms/step | 15.1 samples/s +Step 18030/44000 | Loss: 2.9965 | LR: 3.28e-04 | 1322.8ms/step | 15.1 samples/s +Step 18040/44000 | Loss: 3.0207 | LR: 3.27e-04 | 1323.1ms/step | 15.1 samples/s +Step 18050/44000 | Loss: 3.0247 | LR: 3.27e-04 | 1321.7ms/step | 15.1 samples/s +Step 18060/44000 | Loss: 3.0393 | LR: 3.27e-04 | 1327.1ms/step | 15.1 samples/s +Step 18070/44000 | Loss: 3.0310 | LR: 3.27e-04 | 1324.5ms/step | 15.1 samples/s +Step 18080/44000 | Loss: 3.0440 | LR: 3.27e-04 | 1323.2ms/step | 15.1 samples/s +Step 18090/44000 | Loss: 3.0556 | LR: 3.27e-04 | 1326.0ms/step | 15.1 samples/s +Step 18100/44000 | Loss: 3.0355 | LR: 3.26e-04 | 1323.5ms/step | 15.1 samples/s +Step 18110/44000 | Loss: 3.0325 | LR: 3.26e-04 | 1325.2ms/step | 15.1 samples/s +Step 18120/44000 | Loss: 3.0167 | LR: 3.26e-04 | 1323.3ms/step | 15.1 samples/s +Step 18130/44000 | Loss: 3.0157 | LR: 3.26e-04 | 1322.7ms/step | 15.1 samples/s +Step 18140/44000 | Loss: 3.0352 | LR: 3.26e-04 | 1327.3ms/step | 15.1 samples/s +Step 18150/44000 | Loss: 3.0099 | LR: 3.26e-04 | 1323.3ms/step | 15.1 samples/s +Step 18160/44000 | Loss: 3.0055 | LR: 3.25e-04 | 1321.9ms/step | 15.1 samples/s +Step 18170/44000 | Loss: 3.0306 | LR: 3.25e-04 | 1322.0ms/step | 15.1 samples/s +Step 18180/44000 | Loss: 3.0446 | LR: 3.25e-04 | 1323.2ms/step | 15.1 samples/s +Step 18190/44000 | Loss: 3.0157 | LR: 3.25e-04 | 1322.5ms/step | 15.1 samples/s +Step 18200/44000 | Loss: 3.0207 | LR: 3.25e-04 | 1325.0ms/step | 15.1 samples/s +Step 18210/44000 | Loss: 3.0265 | LR: 3.24e-04 | 1324.5ms/step | 15.1 samples/s +Step 18220/44000 | Loss: 3.0213 | LR: 3.24e-04 | 1322.7ms/step | 15.1 samples/s +Step 18230/44000 | Loss: 3.0605 | LR: 3.24e-04 | 1325.3ms/step | 15.1 samples/s +Step 18240/44000 | Loss: 3.0241 | LR: 3.24e-04 | 1325.5ms/step | 15.1 samples/s +Step 18250/44000 | Loss: 3.0724 | LR: 3.24e-04 | 1324.1ms/step | 15.1 samples/s +Validation at step 18250: Loss = 3.0333, Perplexity = 20.77 +Step 18260/44000 | Loss: 3.0701 | LR: 3.24e-04 | 3549.6ms/step | 5.6 samples/s +Step 18270/44000 | Loss: 3.0183 | LR: 3.23e-04 | 1322.6ms/step | 15.1 samples/s +Step 18280/44000 | Loss: 3.0285 | LR: 3.23e-04 | 1326.1ms/step | 15.1 samples/s +Step 18290/44000 | Loss: 3.0082 | LR: 3.23e-04 | 1324.7ms/step | 15.1 samples/s +Step 18300/44000 | Loss: 3.0159 | LR: 3.23e-04 | 1326.0ms/step | 15.1 samples/s +Step 18310/44000 | Loss: 3.0008 | LR: 3.23e-04 | 1326.5ms/step | 15.1 samples/s +Step 18320/44000 | Loss: 3.0702 | LR: 3.23e-04 | 1323.8ms/step | 15.1 samples/s +Step 18330/44000 | Loss: 3.0049 | LR: 3.22e-04 | 1325.6ms/step | 15.1 samples/s +Step 18340/44000 | Loss: 3.0383 | LR: 3.22e-04 | 1321.1ms/step | 15.1 samples/s +Step 18350/44000 | Loss: 3.0192 | LR: 3.22e-04 | 1326.3ms/step | 15.1 samples/s +Step 18360/44000 | Loss: 3.0115 | LR: 3.22e-04 | 1324.8ms/step | 15.1 samples/s +Step 18370/44000 | Loss: 3.0320 | LR: 3.22e-04 | 1323.3ms/step | 15.1 samples/s +Step 18380/44000 | Loss: 3.0199 | LR: 3.22e-04 | 1326.9ms/step | 15.1 samples/s +Step 18390/44000 | Loss: 3.0295 | LR: 3.21e-04 | 1326.4ms/step | 15.1 samples/s +Step 18400/44000 | Loss: 3.0641 | LR: 3.21e-04 | 1325.0ms/step | 15.1 samples/s +Step 18410/44000 | Loss: 3.0251 | LR: 3.21e-04 | 1325.7ms/step | 15.1 samples/s +Step 18420/44000 | Loss: 3.0082 | LR: 3.21e-04 | 1323.1ms/step | 15.1 samples/s +Step 18430/44000 | Loss: 3.0016 | LR: 3.21e-04 | 1323.5ms/step | 15.1 samples/s +Step 18440/44000 | Loss: 3.0327 | LR: 3.20e-04 | 1320.5ms/step | 15.1 samples/s +Step 18450/44000 | Loss: 3.0333 | LR: 3.20e-04 | 1324.1ms/step | 15.1 samples/s +Step 18460/44000 | Loss: 3.0180 | LR: 3.20e-04 | 1323.1ms/step | 15.1 samples/s +Step 18470/44000 | Loss: 3.0138 | LR: 3.20e-04 | 1326.2ms/step | 15.1 samples/s +Step 18480/44000 | Loss: 3.0354 | LR: 3.20e-04 | 1326.8ms/step | 15.1 samples/s +Step 18490/44000 | Loss: 3.0149 | LR: 3.20e-04 | 1325.3ms/step | 15.1 samples/s +Step 18500/44000 | Loss: 3.0276 | LR: 3.19e-04 | 1324.4ms/step | 15.1 samples/s +Validation at step 18500: Loss = 3.0309, Perplexity = 20.72 +Step 18510/44000 | Loss: 3.0102 | LR: 3.19e-04 | 3540.8ms/step | 5.6 samples/s +Step 18520/44000 | Loss: 3.0367 | LR: 3.19e-04 | 1321.1ms/step | 15.1 samples/s +Step 18530/44000 | Loss: 3.0225 | LR: 3.19e-04 | 1323.5ms/step | 15.1 samples/s +Step 18540/44000 | Loss: 3.0099 | LR: 3.19e-04 | 1325.1ms/step | 15.1 samples/s +Step 18550/44000 | Loss: 3.0119 | LR: 3.19e-04 | 1325.2ms/step | 15.1 samples/s +Step 18560/44000 | Loss: 3.0525 | LR: 3.18e-04 | 1323.6ms/step | 15.1 samples/s +Step 18570/44000 | Loss: 3.0001 | LR: 3.18e-04 | 1324.9ms/step | 15.1 samples/s +Step 18580/44000 | Loss: 3.0432 | LR: 3.18e-04 | 1328.5ms/step | 15.1 samples/s +Step 18590/44000 | Loss: 3.0096 | LR: 3.18e-04 | 1324.2ms/step | 15.1 samples/s +Step 18600/44000 | Loss: 3.0125 | LR: 3.18e-04 | 1323.8ms/step | 15.1 samples/s +Step 18610/44000 | Loss: 3.0471 | LR: 3.18e-04 | 1324.8ms/step | 15.1 samples/s +Step 18620/44000 | Loss: 3.0198 | LR: 3.17e-04 | 1323.1ms/step | 15.1 samples/s +Step 18630/44000 | Loss: 3.0195 | LR: 3.17e-04 | 1325.1ms/step | 15.1 samples/s +Step 18640/44000 | Loss: 3.0394 | LR: 3.17e-04 | 1324.4ms/step | 15.1 samples/s +Step 18650/44000 | Loss: 3.0258 | LR: 3.17e-04 | 1323.1ms/step | 15.1 samples/s +Step 18660/44000 | Loss: 3.0142 | LR: 3.17e-04 | 1319.4ms/step | 15.2 samples/s +Step 18670/44000 | Loss: 3.0267 | LR: 3.16e-04 | 1324.9ms/step | 15.1 samples/s +Step 18680/44000 | Loss: 3.0231 | LR: 3.16e-04 | 1326.2ms/step | 15.1 samples/s +Step 18690/44000 | Loss: 3.0036 | LR: 3.16e-04 | 1324.9ms/step | 15.1 samples/s +Step 18700/44000 | Loss: 3.0279 | LR: 3.16e-04 | 1324.8ms/step | 15.1 samples/s +Step 18710/44000 | Loss: 3.0117 | LR: 3.16e-04 | 1326.5ms/step | 15.1 samples/s +Step 18720/44000 | Loss: 3.0234 | LR: 3.16e-04 | 1325.2ms/step | 15.1 samples/s +Step 18730/44000 | Loss: 3.0223 | LR: 3.15e-04 | 1323.8ms/step | 15.1 samples/s +Step 18740/44000 | Loss: 3.0091 | LR: 3.15e-04 | 1324.2ms/step | 15.1 samples/s +Step 18750/44000 | Loss: 3.0285 | LR: 3.15e-04 | 1322.4ms/step | 15.1 samples/s +Validation at step 18750: Loss = 3.0270, Perplexity = 20.63 +Step 18760/44000 | Loss: 3.0211 | LR: 3.15e-04 | 3545.3ms/step | 5.6 samples/s +Step 18770/44000 | Loss: 3.0010 | LR: 3.15e-04 | 1321.4ms/step | 15.1 samples/s +Step 18780/44000 | Loss: 3.0014 | LR: 3.15e-04 | 1323.4ms/step | 15.1 samples/s +Step 18790/44000 | Loss: 3.0243 | LR: 3.14e-04 | 1323.6ms/step | 15.1 samples/s +Step 18800/44000 | Loss: 3.0025 | LR: 3.14e-04 | 1323.3ms/step | 15.1 samples/s +Step 18810/44000 | Loss: 3.0255 | LR: 3.14e-04 | 1322.4ms/step | 15.1 samples/s +Step 18820/44000 | Loss: 3.0252 | LR: 3.14e-04 | 1325.0ms/step | 15.1 samples/s +Step 18830/44000 | Loss: 3.0311 | LR: 3.14e-04 | 1323.0ms/step | 15.1 samples/s +Step 18840/44000 | Loss: 3.0251 | LR: 3.13e-04 | 1325.1ms/step | 15.1 samples/s +Step 18850/44000 | Loss: 3.0264 | LR: 3.13e-04 | 1324.9ms/step | 15.1 samples/s +Step 18860/44000 | Loss: 3.0530 | LR: 3.13e-04 | 1323.6ms/step | 15.1 samples/s +Step 18870/44000 | Loss: 3.0253 | LR: 3.13e-04 | 1322.9ms/step | 15.1 samples/s +Step 18880/44000 | Loss: 3.0478 | LR: 3.13e-04 | 1320.8ms/step | 15.1 samples/s +Step 18890/44000 | Loss: 3.0334 | LR: 3.13e-04 | 1320.4ms/step | 15.1 samples/s +Step 18900/44000 | Loss: 3.0313 | LR: 3.12e-04 | 1321.5ms/step | 15.1 samples/s +Step 18910/44000 | Loss: 3.0242 | LR: 3.12e-04 | 1323.5ms/step | 15.1 samples/s +Step 18920/44000 | Loss: 3.0539 | LR: 3.12e-04 | 1326.2ms/step | 15.1 samples/s +Step 18930/44000 | Loss: 3.0286 | LR: 3.12e-04 | 1325.5ms/step | 15.1 samples/s +Step 18940/44000 | Loss: 3.0398 | LR: 3.12e-04 | 1324.6ms/step | 15.1 samples/s +Step 18950/44000 | Loss: 3.0281 | LR: 3.12e-04 | 1324.3ms/step | 15.1 samples/s +Step 18960/44000 | Loss: 3.0540 | LR: 3.11e-04 | 1326.3ms/step | 15.1 samples/s +Step 18970/44000 | Loss: 3.0298 | LR: 3.11e-04 | 1325.3ms/step | 15.1 samples/s +Step 18980/44000 | Loss: 2.9984 | LR: 3.11e-04 | 1324.3ms/step | 15.1 samples/s +Step 18990/44000 | Loss: 3.0260 | LR: 3.11e-04 | 1327.4ms/step | 15.1 samples/s +Step 19000/44000 | Loss: 2.9995 | LR: 3.11e-04 | 1323.7ms/step | 15.1 samples/s +Validation at step 19000: Loss = 3.0247, Perplexity = 20.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-19000 +Step 19010/44000 | Loss: 2.9846 | LR: 3.11e-04 | 3855.9ms/step | 5.2 samples/s +Step 19020/44000 | Loss: 3.0078 | LR: 3.10e-04 | 1324.1ms/step | 15.1 samples/s +Step 19030/44000 | Loss: 3.0325 | LR: 3.10e-04 | 1326.6ms/step | 15.1 samples/s +Step 19040/44000 | Loss: 3.0027 | LR: 3.10e-04 | 1324.5ms/step | 15.1 samples/s +Step 19050/44000 | Loss: 3.0195 | LR: 3.10e-04 | 1323.8ms/step | 15.1 samples/s +Step 19060/44000 | Loss: 3.0485 | LR: 3.10e-04 | 1325.7ms/step | 15.1 samples/s +Step 19070/44000 | Loss: 3.0080 | LR: 3.09e-04 | 1321.1ms/step | 15.1 samples/s +Step 19080/44000 | Loss: 3.0207 | LR: 3.09e-04 | 1322.8ms/step | 15.1 samples/s +Step 19090/44000 | Loss: 2.9723 | LR: 3.09e-04 | 1321.4ms/step | 15.1 samples/s +Step 19100/44000 | Loss: 3.0400 | LR: 3.09e-04 | 1321.0ms/step | 15.1 samples/s +Step 19110/44000 | Loss: 3.0170 | LR: 3.09e-04 | 1323.3ms/step | 15.1 samples/s +Step 19120/44000 | Loss: 3.0477 | LR: 3.09e-04 | 1322.1ms/step | 15.1 samples/s +Step 19130/44000 | Loss: 2.9899 | LR: 3.08e-04 | 1328.3ms/step | 15.1 samples/s +Step 19140/44000 | Loss: 3.0297 | LR: 3.08e-04 | 1323.8ms/step | 15.1 samples/s +Step 19150/44000 | Loss: 2.9901 | LR: 3.08e-04 | 1320.9ms/step | 15.1 samples/s +Step 19160/44000 | Loss: 3.0041 | LR: 3.08e-04 | 1323.7ms/step | 15.1 samples/s +Step 19170/44000 | Loss: 2.9708 | LR: 3.08e-04 | 1323.6ms/step | 15.1 samples/s +Step 19180/44000 | Loss: 2.9875 | LR: 3.08e-04 | 1328.5ms/step | 15.1 samples/s +Step 19190/44000 | Loss: 3.0014 | LR: 3.07e-04 | 1323.3ms/step | 15.1 samples/s +Step 19200/44000 | Loss: 3.0093 | LR: 3.07e-04 | 1322.3ms/step | 15.1 samples/s +Step 19210/44000 | Loss: 3.0077 | LR: 3.07e-04 | 1323.7ms/step | 15.1 samples/s +Step 19220/44000 | Loss: 3.0187 | LR: 3.07e-04 | 1323.5ms/step | 15.1 samples/s +Step 19230/44000 | Loss: 3.0283 | LR: 3.07e-04 | 1324.3ms/step | 15.1 samples/s +Step 19240/44000 | Loss: 3.0230 | LR: 3.06e-04 | 1321.5ms/step | 15.1 samples/s +Step 19250/44000 | Loss: 3.0324 | LR: 3.06e-04 | 1324.0ms/step | 15.1 samples/s +Validation at step 19250: Loss = 3.0222, Perplexity = 20.54 +Step 19260/44000 | Loss: 3.0815 | LR: 3.06e-04 | 3546.1ms/step | 5.6 samples/s +Step 19270/44000 | Loss: 3.0237 | LR: 3.06e-04 | 1322.1ms/step | 15.1 samples/s +Step 19280/44000 | Loss: 3.0385 | LR: 3.06e-04 | 1324.6ms/step | 15.1 samples/s +Step 19290/44000 | Loss: 3.0101 | LR: 3.06e-04 | 1325.9ms/step | 15.1 samples/s +Step 19300/44000 | Loss: 2.9878 | LR: 3.05e-04 | 1326.2ms/step | 15.1 samples/s +Step 19310/44000 | Loss: 3.0151 | LR: 3.05e-04 | 1323.0ms/step | 15.1 samples/s +Step 19320/44000 | Loss: 3.0125 | LR: 3.05e-04 | 1319.8ms/step | 15.2 samples/s +Step 19330/44000 | Loss: 3.0346 | LR: 3.05e-04 | 1320.1ms/step | 15.2 samples/s +Step 19340/44000 | Loss: 3.0117 | LR: 3.05e-04 | 1322.2ms/step | 15.1 samples/s +Step 19350/44000 | Loss: 3.0208 | LR: 3.04e-04 | 1326.0ms/step | 15.1 samples/s +Step 19360/44000 | Loss: 2.9826 | LR: 3.04e-04 | 1326.5ms/step | 15.1 samples/s +Step 19370/44000 | Loss: 3.0186 | LR: 3.04e-04 | 1322.7ms/step | 15.1 samples/s +Step 19380/44000 | Loss: 3.0029 | LR: 3.04e-04 | 1321.1ms/step | 15.1 samples/s +Step 19390/44000 | Loss: 3.0251 | LR: 3.04e-04 | 1324.5ms/step | 15.1 samples/s +Step 19400/44000 | Loss: 3.0176 | LR: 3.04e-04 | 1323.5ms/step | 15.1 samples/s +Step 19410/44000 | Loss: 3.0265 | LR: 3.03e-04 | 1321.6ms/step | 15.1 samples/s +Step 19420/44000 | Loss: 3.0302 | LR: 3.03e-04 | 1325.9ms/step | 15.1 samples/s +Step 19430/44000 | Loss: 3.0117 | LR: 3.03e-04 | 1322.4ms/step | 15.1 samples/s +Step 19440/44000 | Loss: 2.9855 | LR: 3.03e-04 | 1322.1ms/step | 15.1 samples/s +Step 19450/44000 | Loss: 2.9988 | LR: 3.03e-04 | 1324.7ms/step | 15.1 samples/s +Step 19460/44000 | Loss: 3.0200 | LR: 3.03e-04 | 1323.4ms/step | 15.1 samples/s +Step 19470/44000 | Loss: 2.9927 | LR: 3.02e-04 | 1326.0ms/step | 15.1 samples/s +Step 19480/44000 | Loss: 3.0377 | LR: 3.02e-04 | 1323.1ms/step | 15.1 samples/s +Step 19490/44000 | Loss: 2.9906 | LR: 3.02e-04 | 1324.2ms/step | 15.1 samples/s +Step 19500/44000 | Loss: 3.0437 | LR: 3.02e-04 | 1322.5ms/step | 15.1 samples/s +Validation at step 19500: Loss = 3.0187, Perplexity = 20.46 +Step 19510/44000 | Loss: 3.0257 | LR: 3.02e-04 | 3551.7ms/step | 5.6 samples/s +Step 19520/44000 | Loss: 3.0144 | LR: 3.01e-04 | 1328.8ms/step | 15.1 samples/s +Step 19530/44000 | Loss: 3.0152 | LR: 3.01e-04 | 1325.0ms/step | 15.1 samples/s +Step 19540/44000 | Loss: 3.0263 | LR: 3.01e-04 | 1328.6ms/step | 15.1 samples/s +Step 19550/44000 | Loss: 2.9989 | LR: 3.01e-04 | 1325.4ms/step | 15.1 samples/s +Step 19560/44000 | Loss: 3.0086 | LR: 3.01e-04 | 1323.1ms/step | 15.1 samples/s +Step 19570/44000 | Loss: 3.0027 | LR: 3.01e-04 | 1324.3ms/step | 15.1 samples/s +Step 19580/44000 | Loss: 2.9929 | LR: 3.00e-04 | 1324.0ms/step | 15.1 samples/s +Step 19590/44000 | Loss: 3.0349 | LR: 3.00e-04 | 1323.9ms/step | 15.1 samples/s +Step 19600/44000 | Loss: 3.0138 | LR: 3.00e-04 | 1322.8ms/step | 15.1 samples/s +Step 19610/44000 | Loss: 2.9983 | LR: 3.00e-04 | 1325.2ms/step | 15.1 samples/s +Step 19620/44000 | Loss: 3.0326 | LR: 3.00e-04 | 1324.1ms/step | 15.1 samples/s +Step 19630/44000 | Loss: 3.0046 | LR: 3.00e-04 | 1323.7ms/step | 15.1 samples/s +Step 19640/44000 | Loss: 3.0004 | LR: 2.99e-04 | 1323.7ms/step | 15.1 samples/s +Step 19650/44000 | Loss: 2.9937 | LR: 2.99e-04 | 1322.1ms/step | 15.1 samples/s +Step 19660/44000 | Loss: 3.0098 | LR: 2.99e-04 | 1327.8ms/step | 15.1 samples/s +Step 19670/44000 | Loss: 3.0007 | LR: 2.99e-04 | 1322.1ms/step | 15.1 samples/s +Step 19680/44000 | Loss: 3.0266 | LR: 2.99e-04 | 1323.0ms/step | 15.1 samples/s +Step 19690/44000 | Loss: 3.0112 | LR: 2.98e-04 | 1324.7ms/step | 15.1 samples/s +Step 19700/44000 | Loss: 3.0397 | LR: 2.98e-04 | 1323.8ms/step | 15.1 samples/s +Step 19710/44000 | Loss: 2.9952 | LR: 2.98e-04 | 1326.3ms/step | 15.1 samples/s +Step 19720/44000 | Loss: 3.0093 | LR: 2.98e-04 | 1326.4ms/step | 15.1 samples/s +Step 19730/44000 | Loss: 3.0350 | LR: 2.98e-04 | 1324.6ms/step | 15.1 samples/s +Step 19740/44000 | Loss: 2.9982 | LR: 2.98e-04 | 1323.4ms/step | 15.1 samples/s +Step 19750/44000 | Loss: 2.9936 | LR: 2.97e-04 | 1324.8ms/step | 15.1 samples/s +Validation at step 19750: Loss = 3.0186, Perplexity = 20.46 +Step 19760/44000 | Loss: 2.9964 | LR: 2.97e-04 | 3545.6ms/step | 5.6 samples/s +Step 19770/44000 | Loss: 3.0192 | LR: 2.97e-04 | 1326.4ms/step | 15.1 samples/s +Step 19780/44000 | Loss: 3.0314 | LR: 2.97e-04 | 1324.1ms/step | 15.1 samples/s +Step 19790/44000 | Loss: 3.0222 | LR: 2.97e-04 | 1325.7ms/step | 15.1 samples/s +Step 19800/44000 | Loss: 3.0052 | LR: 2.96e-04 | 1324.0ms/step | 15.1 samples/s +Step 19810/44000 | Loss: 3.0052 | LR: 2.96e-04 | 1323.8ms/step | 15.1 samples/s +Step 19820/44000 | Loss: 3.0196 | LR: 2.96e-04 | 1325.1ms/step | 15.1 samples/s +Step 19830/44000 | Loss: 3.0281 | LR: 2.96e-04 | 1323.8ms/step | 15.1 samples/s +Step 19840/44000 | Loss: 3.0069 | LR: 2.96e-04 | 1322.3ms/step | 15.1 samples/s +Step 19850/44000 | Loss: 3.0262 | LR: 2.96e-04 | 1326.1ms/step | 15.1 samples/s +Step 19860/44000 | Loss: 2.9637 | LR: 2.95e-04 | 1321.8ms/step | 15.1 samples/s +Step 19870/44000 | Loss: 3.0181 | LR: 2.95e-04 | 1322.7ms/step | 15.1 samples/s +Step 19880/44000 | Loss: 3.0017 | LR: 2.95e-04 | 1325.0ms/step | 15.1 samples/s +Step 19890/44000 | Loss: 3.0203 | LR: 2.95e-04 | 1322.6ms/step | 15.1 samples/s +Step 19900/44000 | Loss: 3.0092 | LR: 2.95e-04 | 1325.8ms/step | 15.1 samples/s +Step 19910/44000 | Loss: 3.0229 | LR: 2.95e-04 | 1323.3ms/step | 15.1 samples/s +Step 19920/44000 | Loss: 2.9960 | LR: 2.94e-04 | 1324.5ms/step | 15.1 samples/s +Step 19930/44000 | Loss: 2.9929 | LR: 2.94e-04 | 1327.9ms/step | 15.1 samples/s +Step 19940/44000 | Loss: 3.0416 | LR: 2.94e-04 | 1323.5ms/step | 15.1 samples/s +Step 19950/44000 | Loss: 3.0314 | LR: 2.94e-04 | 1322.9ms/step | 15.1 samples/s +Step 19960/44000 | Loss: 3.0053 | LR: 2.94e-04 | 1323.7ms/step | 15.1 samples/s +Step 19970/44000 | Loss: 3.0481 | LR: 2.93e-04 | 1325.3ms/step | 15.1 samples/s +Step 19980/44000 | Loss: 3.0030 | LR: 2.93e-04 | 1324.2ms/step | 15.1 samples/s +Step 19990/44000 | Loss: 3.0053 | LR: 2.93e-04 | 1324.5ms/step | 15.1 samples/s +Step 20000/44000 | Loss: 3.0385 | LR: 2.93e-04 | 1323.1ms/step | 15.1 samples/s +Validation at step 20000: Loss = 3.0132, Perplexity = 20.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-20000 +Step 20010/44000 | Loss: 2.9779 | LR: 2.93e-04 | 3842.1ms/step | 5.2 samples/s +Step 20020/44000 | Loss: 3.0193 | LR: 2.93e-04 | 1315.7ms/step | 15.2 samples/s +Step 20030/44000 | Loss: 3.0085 | LR: 2.92e-04 | 1319.0ms/step | 15.2 samples/s +Step 20040/44000 | Loss: 2.9983 | LR: 2.92e-04 | 1325.1ms/step | 15.1 samples/s +Step 20050/44000 | Loss: 3.0004 | LR: 2.92e-04 | 1322.3ms/step | 15.1 samples/s +Step 20060/44000 | Loss: 3.0397 | LR: 2.92e-04 | 1320.1ms/step | 15.2 samples/s +Step 20070/44000 | Loss: 2.9903 | LR: 2.92e-04 | 1322.9ms/step | 15.1 samples/s +Step 20080/44000 | Loss: 2.9849 | LR: 2.91e-04 | 1328.2ms/step | 15.1 samples/s +Step 20090/44000 | Loss: 2.9874 | LR: 2.91e-04 | 1322.7ms/step | 15.1 samples/s +Step 20100/44000 | Loss: 2.9966 | LR: 2.91e-04 | 1323.9ms/step | 15.1 samples/s +Step 20110/44000 | Loss: 2.9915 | LR: 2.91e-04 | 1324.9ms/step | 15.1 samples/s +Step 20120/44000 | Loss: 2.9974 | LR: 2.91e-04 | 1321.6ms/step | 15.1 samples/s +Step 20130/44000 | Loss: 3.0235 | LR: 2.91e-04 | 1326.8ms/step | 15.1 samples/s +Step 20140/44000 | Loss: 3.0268 | LR: 2.90e-04 | 1324.7ms/step | 15.1 samples/s +Step 20150/44000 | Loss: 2.9998 | LR: 2.90e-04 | 1322.4ms/step | 15.1 samples/s +Step 20160/44000 | Loss: 2.9838 | LR: 2.90e-04 | 1322.7ms/step | 15.1 samples/s +Step 20170/44000 | Loss: 3.0261 | LR: 2.90e-04 | 1321.8ms/step | 15.1 samples/s +Step 20180/44000 | Loss: 3.0135 | LR: 2.90e-04 | 1321.4ms/step | 15.1 samples/s +Step 20190/44000 | Loss: 3.0008 | LR: 2.90e-04 | 1323.4ms/step | 15.1 samples/s +Step 20200/44000 | Loss: 3.0135 | LR: 2.89e-04 | 1320.3ms/step | 15.1 samples/s +Step 20210/44000 | Loss: 3.0103 | LR: 2.89e-04 | 1322.2ms/step | 15.1 samples/s +Step 20220/44000 | Loss: 3.0276 | LR: 2.89e-04 | 1324.2ms/step | 15.1 samples/s +Step 20230/44000 | Loss: 3.0117 | LR: 2.89e-04 | 1323.2ms/step | 15.1 samples/s +Step 20240/44000 | Loss: 3.0130 | LR: 2.89e-04 | 1324.6ms/step | 15.1 samples/s +Step 20250/44000 | Loss: 2.9956 | LR: 2.88e-04 | 1324.1ms/step | 15.1 samples/s +Validation at step 20250: Loss = 3.0108, Perplexity = 20.30 +Step 20260/44000 | Loss: 3.0156 | LR: 2.88e-04 | 3547.1ms/step | 5.6 samples/s +Step 20270/44000 | Loss: 2.9862 | LR: 2.88e-04 | 1323.5ms/step | 15.1 samples/s +Step 20280/44000 | Loss: 3.0065 | LR: 2.88e-04 | 1323.1ms/step | 15.1 samples/s +Step 20290/44000 | Loss: 3.0121 | LR: 2.88e-04 | 1326.8ms/step | 15.1 samples/s +Step 20300/44000 | Loss: 3.0044 | LR: 2.88e-04 | 1322.3ms/step | 15.1 samples/s +Step 20310/44000 | Loss: 3.0200 | LR: 2.87e-04 | 1324.1ms/step | 15.1 samples/s +Step 20320/44000 | Loss: 3.0120 | LR: 2.87e-04 | 1323.1ms/step | 15.1 samples/s +Step 20330/44000 | Loss: 3.0164 | LR: 2.87e-04 | 1328.7ms/step | 15.1 samples/s +Step 20340/44000 | Loss: 2.9858 | LR: 2.87e-04 | 1323.6ms/step | 15.1 samples/s +Step 20350/44000 | Loss: 3.0163 | LR: 2.87e-04 | 1323.5ms/step | 15.1 samples/s +Step 20360/44000 | Loss: 3.0053 | LR: 2.86e-04 | 1325.9ms/step | 15.1 samples/s +Step 20370/44000 | Loss: 3.0207 | LR: 2.86e-04 | 1322.1ms/step | 15.1 samples/s +Step 20380/44000 | Loss: 2.9968 | LR: 2.86e-04 | 1324.2ms/step | 15.1 samples/s +Step 20390/44000 | Loss: 3.0070 | LR: 2.86e-04 | 1322.6ms/step | 15.1 samples/s +Step 20400/44000 | Loss: 3.0226 | LR: 2.86e-04 | 1324.4ms/step | 15.1 samples/s +Step 20410/44000 | Loss: 3.0032 | LR: 2.86e-04 | 1325.0ms/step | 15.1 samples/s +Step 20420/44000 | Loss: 3.0391 | LR: 2.85e-04 | 1324.4ms/step | 15.1 samples/s +Step 20430/44000 | Loss: 2.9734 | LR: 2.85e-04 | 1325.2ms/step | 15.1 samples/s +Step 20440/44000 | Loss: 3.0153 | LR: 2.85e-04 | 1324.5ms/step | 15.1 samples/s +Step 20450/44000 | Loss: 3.0064 | LR: 2.85e-04 | 1324.1ms/step | 15.1 samples/s +Step 20460/44000 | Loss: 3.0158 | LR: 2.85e-04 | 1322.5ms/step | 15.1 samples/s +Step 20470/44000 | Loss: 2.9560 | LR: 2.84e-04 | 1323.4ms/step | 15.1 samples/s +Step 20480/44000 | Loss: 2.9867 | LR: 2.84e-04 | 1322.9ms/step | 15.1 samples/s +Step 20490/44000 | Loss: 3.0367 | LR: 2.84e-04 | 1325.1ms/step | 15.1 samples/s +Step 20500/44000 | Loss: 3.0065 | LR: 2.84e-04 | 1324.7ms/step | 15.1 samples/s +Validation at step 20500: Loss = 3.0081, Perplexity = 20.25 +Step 20510/44000 | Loss: 3.0084 | LR: 2.84e-04 | 3545.4ms/step | 5.6 samples/s +Step 20520/44000 | Loss: 2.9932 | LR: 2.84e-04 | 1326.1ms/step | 15.1 samples/s +Step 20530/44000 | Loss: 2.9962 | LR: 2.83e-04 | 1327.3ms/step | 15.1 samples/s +Step 20540/44000 | Loss: 2.9915 | LR: 2.83e-04 | 1322.0ms/step | 15.1 samples/s +Step 20550/44000 | Loss: 2.9974 | LR: 2.83e-04 | 1319.6ms/step | 15.2 samples/s +Step 20560/44000 | Loss: 2.9952 | LR: 2.83e-04 | 1324.7ms/step | 15.1 samples/s +Step 20570/44000 | Loss: 2.9663 | LR: 2.83e-04 | 1324.4ms/step | 15.1 samples/s +Step 20580/44000 | Loss: 3.0382 | LR: 2.83e-04 | 1322.5ms/step | 15.1 samples/s +Step 20590/44000 | Loss: 2.9955 | LR: 2.82e-04 | 1323.5ms/step | 15.1 samples/s +Step 20600/44000 | Loss: 2.9779 | LR: 2.82e-04 | 1325.2ms/step | 15.1 samples/s +Step 20610/44000 | Loss: 2.9634 | LR: 2.82e-04 | 1322.4ms/step | 15.1 samples/s +Step 20620/44000 | Loss: 3.0084 | LR: 2.82e-04 | 1323.6ms/step | 15.1 samples/s +Step 20630/44000 | Loss: 3.0114 | LR: 2.82e-04 | 1323.6ms/step | 15.1 samples/s +Step 20640/44000 | Loss: 2.9718 | LR: 2.81e-04 | 1325.3ms/step | 15.1 samples/s +Step 20650/44000 | Loss: 2.9729 | LR: 2.81e-04 | 1324.3ms/step | 15.1 samples/s +Step 20660/44000 | Loss: 2.9894 | LR: 2.81e-04 | 1326.0ms/step | 15.1 samples/s +Step 20670/44000 | Loss: 3.0085 | LR: 2.81e-04 | 1323.1ms/step | 15.1 samples/s +Step 20680/44000 | Loss: 3.0187 | LR: 2.81e-04 | 1327.5ms/step | 15.1 samples/s +Step 20690/44000 | Loss: 2.9981 | LR: 2.81e-04 | 1328.9ms/step | 15.0 samples/s +Step 20700/44000 | Loss: 3.0018 | LR: 2.80e-04 | 1327.1ms/step | 15.1 samples/s +Step 20710/44000 | Loss: 3.0284 | LR: 2.80e-04 | 1325.1ms/step | 15.1 samples/s +Step 20720/44000 | Loss: 3.0024 | LR: 2.80e-04 | 1324.2ms/step | 15.1 samples/s +Step 20730/44000 | Loss: 2.9627 | LR: 2.80e-04 | 1327.9ms/step | 15.1 samples/s +Step 20740/44000 | Loss: 2.9838 | LR: 2.80e-04 | 1324.8ms/step | 15.1 samples/s +Step 20750/44000 | Loss: 2.9831 | LR: 2.79e-04 | 1321.2ms/step | 15.1 samples/s +Validation at step 20750: Loss = 3.0061, Perplexity = 20.21 +Step 20760/44000 | Loss: 3.0182 | LR: 2.79e-04 | 3548.8ms/step | 5.6 samples/s +Step 20770/44000 | Loss: 2.9865 | LR: 2.79e-04 | 1320.1ms/step | 15.1 samples/s +Step 20780/44000 | Loss: 2.9946 | LR: 2.79e-04 | 1320.8ms/step | 15.1 samples/s +Step 20790/44000 | Loss: 3.0023 | LR: 2.79e-04 | 1320.0ms/step | 15.2 samples/s +Step 20800/44000 | Loss: 3.0121 | LR: 2.79e-04 | 1319.7ms/step | 15.2 samples/s +Step 20810/44000 | Loss: 3.0071 | LR: 2.78e-04 | 1317.1ms/step | 15.2 samples/s +Step 20820/44000 | Loss: 2.9746 | LR: 2.78e-04 | 1316.5ms/step | 15.2 samples/s +Step 20830/44000 | Loss: 2.9931 | LR: 2.78e-04 | 1322.0ms/step | 15.1 samples/s +Step 20840/44000 | Loss: 2.9780 | LR: 2.78e-04 | 1323.1ms/step | 15.1 samples/s +Step 20850/44000 | Loss: 3.0032 | LR: 2.78e-04 | 1322.5ms/step | 15.1 samples/s +Step 20860/44000 | Loss: 3.0032 | LR: 2.77e-04 | 1324.5ms/step | 15.1 samples/s +Step 20870/44000 | Loss: 2.9716 | LR: 2.77e-04 | 1322.0ms/step | 15.1 samples/s +Step 20880/44000 | Loss: 3.0178 | LR: 2.77e-04 | 1325.6ms/step | 15.1 samples/s +Step 20890/44000 | Loss: 3.0105 | LR: 2.77e-04 | 1319.0ms/step | 15.2 samples/s +Step 20900/44000 | Loss: 2.9940 | LR: 2.77e-04 | 1322.4ms/step | 15.1 samples/s +Step 20910/44000 | Loss: 3.0217 | LR: 2.77e-04 | 1320.9ms/step | 15.1 samples/s +Step 20920/44000 | Loss: 3.0334 | LR: 2.76e-04 | 1325.6ms/step | 15.1 samples/s +Step 20930/44000 | Loss: 3.0300 | LR: 2.76e-04 | 1327.4ms/step | 15.1 samples/s +Step 20940/44000 | Loss: 3.0334 | LR: 2.76e-04 | 1322.9ms/step | 15.1 samples/s +Step 20950/44000 | Loss: 2.9957 | LR: 2.76e-04 | 1325.3ms/step | 15.1 samples/s +Step 20960/44000 | Loss: 3.0226 | LR: 2.76e-04 | 1325.0ms/step | 15.1 samples/s +Step 20970/44000 | Loss: 3.0075 | LR: 2.75e-04 | 1322.6ms/step | 15.1 samples/s +Step 20980/44000 | Loss: 3.0252 | LR: 2.75e-04 | 1322.4ms/step | 15.1 samples/s +Step 20990/44000 | Loss: 2.9837 | LR: 2.75e-04 | 1322.6ms/step | 15.1 samples/s +Step 21000/44000 | Loss: 2.9857 | LR: 2.75e-04 | 1323.2ms/step | 15.1 samples/s +Validation at step 21000: Loss = 3.0029, Perplexity = 20.14 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-21000 +Step 21010/44000 | Loss: 3.0028 | LR: 2.75e-04 | 3838.3ms/step | 5.2 samples/s +Step 21020/44000 | Loss: 3.0036 | LR: 2.75e-04 | 1323.0ms/step | 15.1 samples/s +Step 21030/44000 | Loss: 3.0316 | LR: 2.74e-04 | 1322.9ms/step | 15.1 samples/s +Step 21040/44000 | Loss: 3.0235 | LR: 2.74e-04 | 1323.5ms/step | 15.1 samples/s +Step 21050/44000 | Loss: 2.9950 | LR: 2.74e-04 | 1326.0ms/step | 15.1 samples/s +Step 21060/44000 | Loss: 2.9877 | LR: 2.74e-04 | 1324.5ms/step | 15.1 samples/s +Step 21070/44000 | Loss: 3.0184 | LR: 2.74e-04 | 1329.7ms/step | 15.0 samples/s +Step 21080/44000 | Loss: 2.9984 | LR: 2.73e-04 | 1324.6ms/step | 15.1 samples/s +Step 21090/44000 | Loss: 2.9994 | LR: 2.73e-04 | 1322.9ms/step | 15.1 samples/s +Step 21100/44000 | Loss: 2.9975 | LR: 2.73e-04 | 1323.3ms/step | 15.1 samples/s +Step 21110/44000 | Loss: 2.9868 | LR: 2.73e-04 | 1325.4ms/step | 15.1 samples/s +Step 21120/44000 | Loss: 3.0013 | LR: 2.73e-04 | 1325.9ms/step | 15.1 samples/s +Step 21130/44000 | Loss: 2.9873 | LR: 2.73e-04 | 1323.2ms/step | 15.1 samples/s +Step 21140/44000 | Loss: 2.9956 | LR: 2.72e-04 | 1326.1ms/step | 15.1 samples/s +Step 21150/44000 | Loss: 2.9920 | LR: 2.72e-04 | 1321.0ms/step | 15.1 samples/s +Step 21160/44000 | Loss: 2.9802 | LR: 2.72e-04 | 1321.1ms/step | 15.1 samples/s +Step 21170/44000 | Loss: 3.0163 | LR: 2.72e-04 | 1325.3ms/step | 15.1 samples/s +Step 21180/44000 | Loss: 3.0073 | LR: 2.72e-04 | 1322.5ms/step | 15.1 samples/s +Step 21190/44000 | Loss: 2.9815 | LR: 2.71e-04 | 1323.2ms/step | 15.1 samples/s +Step 21200/44000 | Loss: 2.9897 | LR: 2.71e-04 | 1321.4ms/step | 15.1 samples/s +Step 21210/44000 | Loss: 2.9974 | LR: 2.71e-04 | 1325.6ms/step | 15.1 samples/s +Step 21220/44000 | Loss: 2.9973 | LR: 2.71e-04 | 1325.8ms/step | 15.1 samples/s +Step 21230/44000 | Loss: 2.9599 | LR: 2.71e-04 | 1324.9ms/step | 15.1 samples/s +Step 21240/44000 | Loss: 2.9905 | LR: 2.71e-04 | 1322.4ms/step | 15.1 samples/s +Step 21250/44000 | Loss: 3.0093 | LR: 2.70e-04 | 1325.0ms/step | 15.1 samples/s +Validation at step 21250: Loss = 3.0017, Perplexity = 20.12 +Step 21260/44000 | Loss: 3.0200 | LR: 2.70e-04 | 3546.3ms/step | 5.6 samples/s +Step 21270/44000 | Loss: 3.0328 | LR: 2.70e-04 | 1323.5ms/step | 15.1 samples/s +Step 21280/44000 | Loss: 2.9821 | LR: 2.70e-04 | 1325.7ms/step | 15.1 samples/s +Step 21290/44000 | Loss: 3.0054 | LR: 2.70e-04 | 1325.4ms/step | 15.1 samples/s +Step 21300/44000 | Loss: 3.0135 | LR: 2.70e-04 | 1324.2ms/step | 15.1 samples/s +Step 21310/44000 | Loss: 2.9917 | LR: 2.69e-04 | 1325.2ms/step | 15.1 samples/s +Step 21320/44000 | Loss: 2.9971 | LR: 2.69e-04 | 1323.5ms/step | 15.1 samples/s +Step 21330/44000 | Loss: 3.0040 | LR: 2.69e-04 | 1323.3ms/step | 15.1 samples/s +Step 21340/44000 | Loss: 2.9876 | LR: 2.69e-04 | 1328.5ms/step | 15.1 samples/s +Step 21350/44000 | Loss: 2.9769 | LR: 2.69e-04 | 1321.4ms/step | 15.1 samples/s +Step 21360/44000 | Loss: 2.9740 | LR: 2.68e-04 | 1326.1ms/step | 15.1 samples/s +Step 21370/44000 | Loss: 2.9701 | LR: 2.68e-04 | 1324.9ms/step | 15.1 samples/s +Step 21380/44000 | Loss: 2.9947 | LR: 2.68e-04 | 1326.1ms/step | 15.1 samples/s +Step 21390/44000 | Loss: 2.9992 | LR: 2.68e-04 | 1323.4ms/step | 15.1 samples/s +Step 21400/44000 | Loss: 3.0179 | LR: 2.68e-04 | 1322.9ms/step | 15.1 samples/s +Step 21410/44000 | Loss: 3.0044 | LR: 2.68e-04 | 1329.6ms/step | 15.0 samples/s +Step 21420/44000 | Loss: 2.9495 | LR: 2.67e-04 | 1324.1ms/step | 15.1 samples/s +Step 21430/44000 | Loss: 3.0139 | LR: 2.67e-04 | 1321.5ms/step | 15.1 samples/s +Step 21440/44000 | Loss: 3.0116 | LR: 2.67e-04 | 1327.1ms/step | 15.1 samples/s +Step 21450/44000 | Loss: 3.0319 | LR: 2.67e-04 | 1322.8ms/step | 15.1 samples/s +Step 21460/44000 | Loss: 2.9598 | LR: 2.67e-04 | 1323.1ms/step | 15.1 samples/s +Step 21470/44000 | Loss: 2.9909 | LR: 2.66e-04 | 1319.9ms/step | 15.2 samples/s +Step 21480/44000 | Loss: 3.0002 | LR: 2.66e-04 | 1321.7ms/step | 15.1 samples/s +Step 21490/44000 | Loss: 3.0215 | LR: 2.66e-04 | 1324.7ms/step | 15.1 samples/s +Step 21500/44000 | Loss: 2.9932 | LR: 2.66e-04 | 1326.7ms/step | 15.1 samples/s +Validation at step 21500: Loss = 2.9981, Perplexity = 20.05 +Step 21510/44000 | Loss: 3.0084 | LR: 2.66e-04 | 3544.9ms/step | 5.6 samples/s +Step 21520/44000 | Loss: 2.9878 | LR: 2.66e-04 | 1324.1ms/step | 15.1 samples/s +Step 21530/44000 | Loss: 2.9939 | LR: 2.65e-04 | 1325.5ms/step | 15.1 samples/s +Step 21540/44000 | Loss: 2.9980 | LR: 2.65e-04 | 1326.1ms/step | 15.1 samples/s +Step 21550/44000 | Loss: 3.0271 | LR: 2.65e-04 | 1325.5ms/step | 15.1 samples/s +Step 21560/44000 | Loss: 2.9601 | LR: 2.65e-04 | 1325.7ms/step | 15.1 samples/s +Step 21570/44000 | Loss: 3.0018 | LR: 2.65e-04 | 1323.5ms/step | 15.1 samples/s +Step 21580/44000 | Loss: 2.9818 | LR: 2.64e-04 | 1325.2ms/step | 15.1 samples/s +Step 21590/44000 | Loss: 2.9740 | LR: 2.64e-04 | 1323.4ms/step | 15.1 samples/s +Step 21600/44000 | Loss: 3.0160 | LR: 2.64e-04 | 1323.9ms/step | 15.1 samples/s +Step 21610/44000 | Loss: 2.9905 | LR: 2.64e-04 | 1325.5ms/step | 15.1 samples/s +Step 21620/44000 | Loss: 2.9911 | LR: 2.64e-04 | 1327.6ms/step | 15.1 samples/s +Step 21630/44000 | Loss: 2.9744 | LR: 2.64e-04 | 1325.5ms/step | 15.1 samples/s +Step 21640/44000 | Loss: 2.9896 | LR: 2.63e-04 | 1325.4ms/step | 15.1 samples/s +Step 21650/44000 | Loss: 3.0297 | LR: 2.63e-04 | 1324.0ms/step | 15.1 samples/s +Step 21660/44000 | Loss: 2.9549 | LR: 2.63e-04 | 1326.5ms/step | 15.1 samples/s +Step 21670/44000 | Loss: 2.9777 | LR: 2.63e-04 | 1325.1ms/step | 15.1 samples/s +Step 21680/44000 | Loss: 2.9858 | LR: 2.63e-04 | 1326.0ms/step | 15.1 samples/s +Step 21690/44000 | Loss: 2.9764 | LR: 2.62e-04 | 1322.1ms/step | 15.1 samples/s +Step 21700/44000 | Loss: 2.9979 | LR: 2.62e-04 | 1323.9ms/step | 15.1 samples/s +Step 21710/44000 | Loss: 2.9751 | LR: 2.62e-04 | 1324.0ms/step | 15.1 samples/s +Step 21720/44000 | Loss: 3.0113 | LR: 2.62e-04 | 1324.3ms/step | 15.1 samples/s +Step 21730/44000 | Loss: 2.9471 | LR: 2.62e-04 | 1324.3ms/step | 15.1 samples/s +Step 21740/44000 | Loss: 3.0024 | LR: 2.62e-04 | 1325.4ms/step | 15.1 samples/s +Step 21750/44000 | Loss: 2.9985 | LR: 2.61e-04 | 1327.6ms/step | 15.1 samples/s +Validation at step 21750: Loss = 2.9952, Perplexity = 19.99 +Step 21760/44000 | Loss: 2.9872 | LR: 2.61e-04 | 3545.2ms/step | 5.6 samples/s +Step 21770/44000 | Loss: 3.0013 | LR: 2.61e-04 | 1323.0ms/step | 15.1 samples/s +Step 21780/44000 | Loss: 2.9603 | LR: 2.61e-04 | 1324.2ms/step | 15.1 samples/s +Step 21790/44000 | Loss: 2.9902 | LR: 2.61e-04 | 1326.9ms/step | 15.1 samples/s +Step 21800/44000 | Loss: 2.9903 | LR: 2.60e-04 | 1322.9ms/step | 15.1 samples/s +Step 21810/44000 | Loss: 2.9840 | LR: 2.60e-04 | 1323.0ms/step | 15.1 samples/s +Step 21820/44000 | Loss: 3.0108 | LR: 2.60e-04 | 1324.9ms/step | 15.1 samples/s +Step 21830/44000 | Loss: 2.9926 | LR: 2.60e-04 | 1324.2ms/step | 15.1 samples/s +Step 21840/44000 | Loss: 2.9715 | LR: 2.60e-04 | 1325.4ms/step | 15.1 samples/s +Step 21850/44000 | Loss: 3.0246 | LR: 2.60e-04 | 1325.1ms/step | 15.1 samples/s +Step 21860/44000 | Loss: 2.9844 | LR: 2.59e-04 | 1325.2ms/step | 15.1 samples/s +Step 21870/44000 | Loss: 2.9950 | LR: 2.59e-04 | 1322.6ms/step | 15.1 samples/s +Step 21880/44000 | Loss: 2.9792 | LR: 2.59e-04 | 1321.1ms/step | 15.1 samples/s +Step 21890/44000 | Loss: 3.0303 | LR: 2.59e-04 | 1322.1ms/step | 15.1 samples/s +Step 21900/44000 | Loss: 2.9843 | LR: 2.59e-04 | 1320.5ms/step | 15.1 samples/s +Step 21910/44000 | Loss: 2.9832 | LR: 2.58e-04 | 1321.2ms/step | 15.1 samples/s +Step 21920/44000 | Loss: 2.9735 | LR: 2.58e-04 | 1324.6ms/step | 15.1 samples/s +Step 21930/44000 | Loss: 2.9786 | LR: 2.58e-04 | 1319.9ms/step | 15.2 samples/s +Step 21940/44000 | Loss: 2.9945 | LR: 2.58e-04 | 1325.1ms/step | 15.1 samples/s +Step 21950/44000 | Loss: 2.9988 | LR: 2.58e-04 | 1327.0ms/step | 15.1 samples/s +Step 21960/44000 | Loss: 3.0057 | LR: 2.58e-04 | 1326.7ms/step | 15.1 samples/s +Step 21970/44000 | Loss: 2.9896 | LR: 2.57e-04 | 1326.6ms/step | 15.1 samples/s +Step 21980/44000 | Loss: 2.9799 | LR: 2.57e-04 | 1325.9ms/step | 15.1 samples/s +Step 21990/44000 | Loss: 3.0091 | LR: 2.57e-04 | 1324.9ms/step | 15.1 samples/s +Step 22000/44000 | Loss: 2.9592 | LR: 2.57e-04 | 1322.8ms/step | 15.1 samples/s +Validation at step 22000: Loss = 2.9925, Perplexity = 19.94 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-22000 +Step 22010/44000 | Loss: 2.9800 | LR: 2.57e-04 | 3845.5ms/step | 5.2 samples/s +Step 22020/44000 | Loss: 2.9741 | LR: 2.56e-04 | 1324.0ms/step | 15.1 samples/s +Step 22030/44000 | Loss: 2.9884 | LR: 2.56e-04 | 1326.7ms/step | 15.1 samples/s +Step 22040/44000 | Loss: 2.9570 | LR: 2.56e-04 | 1323.9ms/step | 15.1 samples/s +Step 22050/44000 | Loss: 2.9892 | LR: 2.56e-04 | 1322.2ms/step | 15.1 samples/s +Step 22060/44000 | Loss: 2.9852 | LR: 2.56e-04 | 1325.9ms/step | 15.1 samples/s +Step 22070/44000 | Loss: 2.9933 | LR: 2.56e-04 | 1325.4ms/step | 15.1 samples/s +Step 22080/44000 | Loss: 3.0200 | LR: 2.55e-04 | 1325.0ms/step | 15.1 samples/s +Step 22090/44000 | Loss: 2.9747 | LR: 2.55e-04 | 1322.6ms/step | 15.1 samples/s +Step 22100/44000 | Loss: 2.9882 | LR: 2.55e-04 | 1324.2ms/step | 15.1 samples/s +Step 22110/44000 | Loss: 2.9958 | LR: 2.55e-04 | 1323.6ms/step | 15.1 samples/s +Step 22120/44000 | Loss: 3.0183 | LR: 2.55e-04 | 1324.6ms/step | 15.1 samples/s +Step 22130/44000 | Loss: 2.9755 | LR: 2.54e-04 | 1324.1ms/step | 15.1 samples/s +Step 22140/44000 | Loss: 2.9813 | LR: 2.54e-04 | 1323.1ms/step | 15.1 samples/s +Step 22150/44000 | Loss: 2.9784 | LR: 2.54e-04 | 1328.3ms/step | 15.1 samples/s +Step 22160/44000 | Loss: 3.0079 | LR: 2.54e-04 | 1326.1ms/step | 15.1 samples/s +Step 22170/44000 | Loss: 3.0013 | LR: 2.54e-04 | 1323.8ms/step | 15.1 samples/s +Step 22180/44000 | Loss: 2.9634 | LR: 2.54e-04 | 1326.6ms/step | 15.1 samples/s +Step 22190/44000 | Loss: 2.9868 | LR: 2.53e-04 | 1326.2ms/step | 15.1 samples/s +Step 22200/44000 | Loss: 3.0005 | LR: 2.53e-04 | 1325.6ms/step | 15.1 samples/s +Step 22210/44000 | Loss: 2.9923 | LR: 2.53e-04 | 1324.0ms/step | 15.1 samples/s +Step 22220/44000 | Loss: 2.9852 | LR: 2.53e-04 | 1320.4ms/step | 15.1 samples/s +Step 22230/44000 | Loss: 2.9786 | LR: 2.53e-04 | 1324.7ms/step | 15.1 samples/s +Step 22240/44000 | Loss: 2.9788 | LR: 2.52e-04 | 1322.8ms/step | 15.1 samples/s +Step 22250/44000 | Loss: 2.9755 | LR: 2.52e-04 | 1322.1ms/step | 15.1 samples/s +Validation at step 22250: Loss = 2.9914, Perplexity = 19.91 +Step 22260/44000 | Loss: 2.9780 | LR: 2.52e-04 | 3543.6ms/step | 5.6 samples/s +Step 22270/44000 | Loss: 2.9509 | LR: 2.52e-04 | 1326.5ms/step | 15.1 samples/s +Step 22280/44000 | Loss: 2.9874 | LR: 2.52e-04 | 1322.7ms/step | 15.1 samples/s +Step 22290/44000 | Loss: 2.9888 | LR: 2.52e-04 | 1324.5ms/step | 15.1 samples/s +Step 22300/44000 | Loss: 2.9792 | LR: 2.51e-04 | 1323.2ms/step | 15.1 samples/s +Step 22310/44000 | Loss: 2.9826 | LR: 2.51e-04 | 1325.3ms/step | 15.1 samples/s +Step 22320/44000 | Loss: 2.9849 | LR: 2.51e-04 | 1324.8ms/step | 15.1 samples/s +Step 22330/44000 | Loss: 2.9912 | LR: 2.51e-04 | 1324.0ms/step | 15.1 samples/s +Step 22340/44000 | Loss: 2.9850 | LR: 2.51e-04 | 1323.2ms/step | 15.1 samples/s +Step 22350/44000 | Loss: 3.0097 | LR: 2.50e-04 | 1324.2ms/step | 15.1 samples/s +Step 22360/44000 | Loss: 3.0055 | LR: 2.50e-04 | 1328.1ms/step | 15.1 samples/s +Step 22370/44000 | Loss: 2.9896 | LR: 2.50e-04 | 1324.2ms/step | 15.1 samples/s +Step 22380/44000 | Loss: 2.9809 | LR: 2.50e-04 | 1325.2ms/step | 15.1 samples/s +Step 22390/44000 | Loss: 2.9853 | LR: 2.50e-04 | 1326.9ms/step | 15.1 samples/s +Step 22400/44000 | Loss: 2.9758 | LR: 2.50e-04 | 1324.2ms/step | 15.1 samples/s +Step 22410/44000 | Loss: 2.9831 | LR: 2.49e-04 | 1324.4ms/step | 15.1 samples/s +Step 22420/44000 | Loss: 2.9853 | LR: 2.49e-04 | 1327.7ms/step | 15.1 samples/s +Step 22430/44000 | Loss: 2.9753 | LR: 2.49e-04 | 1326.6ms/step | 15.1 samples/s +Step 22440/44000 | Loss: 2.9791 | LR: 2.49e-04 | 1326.1ms/step | 15.1 samples/s +Step 22450/44000 | Loss: 2.9751 | LR: 2.49e-04 | 1325.6ms/step | 15.1 samples/s +Step 22460/44000 | Loss: 2.9585 | LR: 2.48e-04 | 1323.3ms/step | 15.1 samples/s +Step 22470/44000 | Loss: 2.9877 | LR: 2.48e-04 | 1324.2ms/step | 15.1 samples/s +Step 22480/44000 | Loss: 2.9759 | LR: 2.48e-04 | 1324.9ms/step | 15.1 samples/s +Step 22490/44000 | Loss: 2.9854 | LR: 2.48e-04 | 1323.5ms/step | 15.1 samples/s +Step 22500/44000 | Loss: 2.9819 | LR: 2.48e-04 | 1325.3ms/step | 15.1 samples/s +Validation at step 22500: Loss = 2.9878, Perplexity = 19.84 +Step 22510/44000 | Loss: 2.9761 | LR: 2.48e-04 | 3553.1ms/step | 5.6 samples/s +Step 22520/44000 | Loss: 2.9818 | LR: 2.47e-04 | 1322.4ms/step | 15.1 samples/s +Step 22530/44000 | Loss: 2.9849 | LR: 2.47e-04 | 1324.9ms/step | 15.1 samples/s +Step 22540/44000 | Loss: 3.0011 | LR: 2.47e-04 | 1324.0ms/step | 15.1 samples/s +Step 22550/44000 | Loss: 2.9897 | LR: 2.47e-04 | 1324.9ms/step | 15.1 samples/s +Step 22560/44000 | Loss: 2.9809 | LR: 2.47e-04 | 1324.6ms/step | 15.1 samples/s +Step 22570/44000 | Loss: 2.9779 | LR: 2.46e-04 | 1320.5ms/step | 15.1 samples/s +Step 22580/44000 | Loss: 3.0105 | LR: 2.46e-04 | 1323.9ms/step | 15.1 samples/s +Step 22590/44000 | Loss: 2.9929 | LR: 2.46e-04 | 1322.1ms/step | 15.1 samples/s +Step 22600/44000 | Loss: 3.0014 | LR: 2.46e-04 | 1322.6ms/step | 15.1 samples/s +Step 22610/44000 | Loss: 2.9467 | LR: 2.46e-04 | 1322.2ms/step | 15.1 samples/s +Step 22620/44000 | Loss: 2.9987 | LR: 2.46e-04 | 1322.7ms/step | 15.1 samples/s +Step 22630/44000 | Loss: 2.9736 | LR: 2.45e-04 | 1323.9ms/step | 15.1 samples/s +Step 22640/44000 | Loss: 2.9599 | LR: 2.45e-04 | 1326.6ms/step | 15.1 samples/s +Step 22650/44000 | Loss: 2.9490 | LR: 2.45e-04 | 1317.1ms/step | 15.2 samples/s +Step 22660/44000 | Loss: 2.9753 | LR: 2.45e-04 | 1316.7ms/step | 15.2 samples/s +Step 22670/44000 | Loss: 2.9680 | LR: 2.45e-04 | 1315.7ms/step | 15.2 samples/s +Step 22680/44000 | Loss: 2.9609 | LR: 2.44e-04 | 1317.9ms/step | 15.2 samples/s +Step 22690/44000 | Loss: 2.9688 | LR: 2.44e-04 | 1315.5ms/step | 15.2 samples/s +Step 22700/44000 | Loss: 2.9896 | LR: 2.44e-04 | 1316.6ms/step | 15.2 samples/s +Step 22710/44000 | Loss: 3.0018 | LR: 2.44e-04 | 1317.7ms/step | 15.2 samples/s +Step 22720/44000 | Loss: 2.9437 | LR: 2.44e-04 | 1316.6ms/step | 15.2 samples/s +Step 22730/44000 | Loss: 3.0070 | LR: 2.44e-04 | 1319.3ms/step | 15.2 samples/s +Step 22740/44000 | Loss: 2.9566 | LR: 2.43e-04 | 1319.2ms/step | 15.2 samples/s +Step 22750/44000 | Loss: 2.9803 | LR: 2.43e-04 | 1320.5ms/step | 15.1 samples/s +Validation at step 22750: Loss = 2.9856, Perplexity = 19.80 +Step 22760/44000 | Loss: 2.9601 | LR: 2.43e-04 | 3544.4ms/step | 5.6 samples/s +Step 22770/44000 | Loss: 2.9677 | LR: 2.43e-04 | 1327.6ms/step | 15.1 samples/s +Step 22780/44000 | Loss: 3.0116 | LR: 2.43e-04 | 1323.6ms/step | 15.1 samples/s +Step 22790/44000 | Loss: 2.9692 | LR: 2.42e-04 | 1320.3ms/step | 15.1 samples/s +Step 22800/44000 | Loss: 2.9927 | LR: 2.42e-04 | 1324.4ms/step | 15.1 samples/s +Step 22810/44000 | Loss: 2.9756 | LR: 2.42e-04 | 1322.2ms/step | 15.1 samples/s +Step 22820/44000 | Loss: 2.9742 | LR: 2.42e-04 | 1323.2ms/step | 15.1 samples/s +Step 22830/44000 | Loss: 2.9822 | LR: 2.42e-04 | 1325.3ms/step | 15.1 samples/s +Step 22840/44000 | Loss: 2.9661 | LR: 2.42e-04 | 1325.0ms/step | 15.1 samples/s +Step 22850/44000 | Loss: 2.9413 | LR: 2.41e-04 | 1324.0ms/step | 15.1 samples/s +Step 22860/44000 | Loss: 2.9781 | LR: 2.41e-04 | 1326.0ms/step | 15.1 samples/s +Step 22870/44000 | Loss: 2.9581 | LR: 2.41e-04 | 1323.8ms/step | 15.1 samples/s +Step 22880/44000 | Loss: 2.9479 | LR: 2.41e-04 | 1322.7ms/step | 15.1 samples/s +Step 22890/44000 | Loss: 3.0019 | LR: 2.41e-04 | 1322.8ms/step | 15.1 samples/s +Step 22900/44000 | Loss: 2.9753 | LR: 2.40e-04 | 1324.5ms/step | 15.1 samples/s +Step 22910/44000 | Loss: 2.9724 | LR: 2.40e-04 | 1326.1ms/step | 15.1 samples/s +Step 22920/44000 | Loss: 2.9864 | LR: 2.40e-04 | 1324.4ms/step | 15.1 samples/s +Step 22930/44000 | Loss: 2.9731 | LR: 2.40e-04 | 1325.0ms/step | 15.1 samples/s +Step 22940/44000 | Loss: 2.9788 | LR: 2.40e-04 | 1327.8ms/step | 15.1 samples/s +Step 22950/44000 | Loss: 3.0023 | LR: 2.40e-04 | 1323.2ms/step | 15.1 samples/s +Step 22960/44000 | Loss: 2.9943 | LR: 2.39e-04 | 1324.4ms/step | 15.1 samples/s +Step 22970/44000 | Loss: 2.9757 | LR: 2.39e-04 | 1322.2ms/step | 15.1 samples/s +Step 22980/44000 | Loss: 2.9884 | LR: 2.39e-04 | 1323.4ms/step | 15.1 samples/s +Step 22990/44000 | Loss: 2.9850 | LR: 2.39e-04 | 1323.3ms/step | 15.1 samples/s +Step 23000/44000 | Loss: 2.9551 | LR: 2.39e-04 | 1321.1ms/step | 15.1 samples/s +Validation at step 23000: Loss = 2.9834, Perplexity = 19.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-23000 +Step 23010/44000 | Loss: 2.9439 | LR: 2.38e-04 | 3847.7ms/step | 5.2 samples/s +Step 23020/44000 | Loss: 2.9921 | LR: 2.38e-04 | 1326.8ms/step | 15.1 samples/s +Step 23030/44000 | Loss: 2.9561 | LR: 2.38e-04 | 1321.9ms/step | 15.1 samples/s +Step 23040/44000 | Loss: 2.9899 | LR: 2.38e-04 | 1323.6ms/step | 15.1 samples/s +Step 23050/44000 | Loss: 2.9838 | LR: 2.38e-04 | 1322.6ms/step | 15.1 samples/s +Step 23060/44000 | Loss: 3.0054 | LR: 2.38e-04 | 1324.1ms/step | 15.1 samples/s +Step 23070/44000 | Loss: 2.9476 | LR: 2.37e-04 | 1327.9ms/step | 15.1 samples/s +Step 23080/44000 | Loss: 2.9541 | LR: 2.37e-04 | 1322.7ms/step | 15.1 samples/s +Step 23090/44000 | Loss: 2.9923 | LR: 2.37e-04 | 1323.7ms/step | 15.1 samples/s +Step 23100/44000 | Loss: 2.9768 | LR: 2.37e-04 | 1322.8ms/step | 15.1 samples/s +Step 23110/44000 | Loss: 2.9918 | LR: 2.37e-04 | 1321.4ms/step | 15.1 samples/s +Step 23120/44000 | Loss: 2.9772 | LR: 2.36e-04 | 1323.6ms/step | 15.1 samples/s +Step 23130/44000 | Loss: 2.9657 | LR: 2.36e-04 | 1324.3ms/step | 15.1 samples/s +Step 23140/44000 | Loss: 2.9887 | LR: 2.36e-04 | 1325.7ms/step | 15.1 samples/s +Step 23150/44000 | Loss: 2.9548 | LR: 2.36e-04 | 1331.4ms/step | 15.0 samples/s +Step 23160/44000 | Loss: 2.9366 | LR: 2.36e-04 | 1327.1ms/step | 15.1 samples/s +Step 23170/44000 | Loss: 2.9672 | LR: 2.36e-04 | 1326.9ms/step | 15.1 samples/s +Step 23180/44000 | Loss: 2.9651 | LR: 2.35e-04 | 1319.5ms/step | 15.2 samples/s +Step 23190/44000 | Loss: 3.0103 | LR: 2.35e-04 | 1316.7ms/step | 15.2 samples/s +Step 23200/44000 | Loss: 2.9946 | LR: 2.35e-04 | 1317.0ms/step | 15.2 samples/s +Step 23210/44000 | Loss: 2.9941 | LR: 2.35e-04 | 1322.5ms/step | 15.1 samples/s +Step 23220/44000 | Loss: 2.9651 | LR: 2.35e-04 | 1320.3ms/step | 15.1 samples/s +Step 23230/44000 | Loss: 2.9732 | LR: 2.34e-04 | 1323.2ms/step | 15.1 samples/s +Step 23240/44000 | Loss: 2.9823 | LR: 2.34e-04 | 1324.9ms/step | 15.1 samples/s +Step 23250/44000 | Loss: 2.9681 | LR: 2.34e-04 | 1322.7ms/step | 15.1 samples/s +Validation at step 23250: Loss = 2.9814, Perplexity = 19.72 +Step 23260/44000 | Loss: 2.9722 | LR: 2.34e-04 | 3546.3ms/step | 5.6 samples/s +Step 23270/44000 | Loss: 2.9577 | LR: 2.34e-04 | 1322.8ms/step | 15.1 samples/s +Step 23280/44000 | Loss: 2.9695 | LR: 2.34e-04 | 1325.4ms/step | 15.1 samples/s +Step 23290/44000 | Loss: 3.0128 | LR: 2.33e-04 | 1318.8ms/step | 15.2 samples/s +Step 23300/44000 | Loss: 2.9654 | LR: 2.33e-04 | 1316.9ms/step | 15.2 samples/s +Step 23310/44000 | Loss: 2.9779 | LR: 2.33e-04 | 1322.1ms/step | 15.1 samples/s +Step 23320/44000 | Loss: 3.0083 | LR: 2.33e-04 | 1322.1ms/step | 15.1 samples/s +Step 23330/44000 | Loss: 2.9538 | LR: 2.33e-04 | 1320.9ms/step | 15.1 samples/s +Step 23340/44000 | Loss: 2.9718 | LR: 2.32e-04 | 1322.9ms/step | 15.1 samples/s +Step 23350/44000 | Loss: 2.9612 | LR: 2.32e-04 | 1324.2ms/step | 15.1 samples/s +Step 23360/44000 | Loss: 2.9647 | LR: 2.32e-04 | 1321.4ms/step | 15.1 samples/s +Step 23370/44000 | Loss: 2.9927 | LR: 2.32e-04 | 1324.6ms/step | 15.1 samples/s +Step 23380/44000 | Loss: 2.9773 | LR: 2.32e-04 | 1322.1ms/step | 15.1 samples/s +Step 23390/44000 | Loss: 2.9930 | LR: 2.32e-04 | 1320.6ms/step | 15.1 samples/s +Step 23400/44000 | Loss: 2.9898 | LR: 2.31e-04 | 1325.1ms/step | 15.1 samples/s +Step 23410/44000 | Loss: 2.9981 | LR: 2.31e-04 | 1321.9ms/step | 15.1 samples/s +Step 23420/44000 | Loss: 2.9687 | LR: 2.31e-04 | 1324.8ms/step | 15.1 samples/s +Step 23430/44000 | Loss: 2.9857 | LR: 2.31e-04 | 1317.2ms/step | 15.2 samples/s +Step 23440/44000 | Loss: 3.0129 | LR: 2.31e-04 | 1317.9ms/step | 15.2 samples/s +Step 23450/44000 | Loss: 2.9465 | LR: 2.30e-04 | 1317.6ms/step | 15.2 samples/s +Step 23460/44000 | Loss: 2.9821 | LR: 2.30e-04 | 1319.1ms/step | 15.2 samples/s +Step 23470/44000 | Loss: 2.9862 | LR: 2.30e-04 | 1321.6ms/step | 15.1 samples/s +Step 23480/44000 | Loss: 2.9965 | LR: 2.30e-04 | 1320.6ms/step | 15.1 samples/s +Step 23490/44000 | Loss: 3.0057 | LR: 2.30e-04 | 1322.8ms/step | 15.1 samples/s +Step 23500/44000 | Loss: 2.9652 | LR: 2.30e-04 | 1325.0ms/step | 15.1 samples/s +Validation at step 23500: Loss = 2.9794, Perplexity = 19.68 +Step 23510/44000 | Loss: 2.9843 | LR: 2.29e-04 | 3551.7ms/step | 5.6 samples/s +Step 23520/44000 | Loss: 2.9687 | LR: 2.29e-04 | 1324.5ms/step | 15.1 samples/s +Step 23530/44000 | Loss: 2.9694 | LR: 2.29e-04 | 1325.3ms/step | 15.1 samples/s +Step 23540/44000 | Loss: 3.0001 | LR: 2.29e-04 | 1331.4ms/step | 15.0 samples/s +Step 23550/44000 | Loss: 2.9631 | LR: 2.29e-04 | 1323.7ms/step | 15.1 samples/s +Step 23560/44000 | Loss: 3.0007 | LR: 2.29e-04 | 1323.2ms/step | 15.1 samples/s +Step 23570/44000 | Loss: 2.9482 | LR: 2.28e-04 | 1318.6ms/step | 15.2 samples/s +Step 23580/44000 | Loss: 2.9587 | LR: 2.28e-04 | 1318.2ms/step | 15.2 samples/s +Step 23590/44000 | Loss: 2.9587 | LR: 2.28e-04 | 1321.0ms/step | 15.1 samples/s +Step 23600/44000 | Loss: 2.9851 | LR: 2.28e-04 | 1318.2ms/step | 15.2 samples/s +Step 23610/44000 | Loss: 2.9764 | LR: 2.28e-04 | 1322.4ms/step | 15.1 samples/s +Step 23620/44000 | Loss: 2.9900 | LR: 2.27e-04 | 1321.5ms/step | 15.1 samples/s +Step 23630/44000 | Loss: 2.9964 | LR: 2.27e-04 | 1325.6ms/step | 15.1 samples/s +Step 23640/44000 | Loss: 2.9533 | LR: 2.27e-04 | 1320.9ms/step | 15.1 samples/s +Step 23650/44000 | Loss: 2.9204 | LR: 2.27e-04 | 1325.4ms/step | 15.1 samples/s +Step 23660/44000 | Loss: 2.9679 | LR: 2.27e-04 | 1325.3ms/step | 15.1 samples/s +Step 23670/44000 | Loss: 2.9433 | LR: 2.27e-04 | 1321.7ms/step | 15.1 samples/s +Step 23680/44000 | Loss: 2.9566 | LR: 2.26e-04 | 1322.1ms/step | 15.1 samples/s +Step 23690/44000 | Loss: 2.9358 | LR: 2.26e-04 | 1319.2ms/step | 15.2 samples/s +Step 23700/44000 | Loss: 2.9843 | LR: 2.26e-04 | 1322.3ms/step | 15.1 samples/s +Step 23710/44000 | Loss: 2.9636 | LR: 2.26e-04 | 1324.2ms/step | 15.1 samples/s +Step 23720/44000 | Loss: 2.9847 | LR: 2.26e-04 | 1319.7ms/step | 15.2 samples/s +Step 23730/44000 | Loss: 2.9226 | LR: 2.25e-04 | 1322.9ms/step | 15.1 samples/s +Step 23740/44000 | Loss: 2.9759 | LR: 2.25e-04 | 1322.5ms/step | 15.1 samples/s +Step 23750/44000 | Loss: 2.9364 | LR: 2.25e-04 | 1320.9ms/step | 15.1 samples/s +Validation at step 23750: Loss = 2.9767, Perplexity = 19.62 +Step 23760/44000 | Loss: 2.9721 | LR: 2.25e-04 | 3546.9ms/step | 5.6 samples/s +Step 23770/44000 | Loss: 2.9660 | LR: 2.25e-04 | 1320.5ms/step | 15.1 samples/s +Step 23780/44000 | Loss: 2.9896 | LR: 2.25e-04 | 1325.4ms/step | 15.1 samples/s +Step 23790/44000 | Loss: 2.9944 | LR: 2.24e-04 | 1321.7ms/step | 15.1 samples/s +Step 23800/44000 | Loss: 2.9760 | LR: 2.24e-04 | 1322.1ms/step | 15.1 samples/s +Step 23810/44000 | Loss: 2.9489 | LR: 2.24e-04 | 1327.0ms/step | 15.1 samples/s +Step 23820/44000 | Loss: 2.9613 | LR: 2.24e-04 | 1326.4ms/step | 15.1 samples/s +Step 23830/44000 | Loss: 2.9947 | LR: 2.24e-04 | 1324.4ms/step | 15.1 samples/s +Step 23840/44000 | Loss: 2.9852 | LR: 2.23e-04 | 1324.5ms/step | 15.1 samples/s +Step 23850/44000 | Loss: 2.9510 | LR: 2.23e-04 | 1324.8ms/step | 15.1 samples/s +Step 23860/44000 | Loss: 2.9636 | LR: 2.23e-04 | 1325.5ms/step | 15.1 samples/s +Step 23870/44000 | Loss: 2.9687 | LR: 2.23e-04 | 1324.1ms/step | 15.1 samples/s +Step 23880/44000 | Loss: 3.0064 | LR: 2.23e-04 | 1321.1ms/step | 15.1 samples/s +Step 23890/44000 | Loss: 2.9621 | LR: 2.23e-04 | 1321.6ms/step | 15.1 samples/s +Step 23900/44000 | Loss: 2.9504 | LR: 2.22e-04 | 1320.9ms/step | 15.1 samples/s +Step 23910/44000 | Loss: 2.9654 | LR: 2.22e-04 | 1324.0ms/step | 15.1 samples/s +Step 23920/44000 | Loss: 2.9676 | LR: 2.22e-04 | 1321.1ms/step | 15.1 samples/s +Step 23930/44000 | Loss: 2.9582 | LR: 2.22e-04 | 1321.2ms/step | 15.1 samples/s +Step 23940/44000 | Loss: 2.9571 | LR: 2.22e-04 | 1328.1ms/step | 15.1 samples/s +Step 23950/44000 | Loss: 2.9706 | LR: 2.21e-04 | 1327.8ms/step | 15.1 samples/s +Step 23960/44000 | Loss: 2.9859 | LR: 2.21e-04 | 1328.3ms/step | 15.1 samples/s +Step 23970/44000 | Loss: 2.9832 | LR: 2.21e-04 | 1326.0ms/step | 15.1 samples/s +Step 23980/44000 | Loss: 2.9827 | LR: 2.21e-04 | 1324.5ms/step | 15.1 samples/s +Step 23990/44000 | Loss: 2.9591 | LR: 2.21e-04 | 1320.9ms/step | 15.1 samples/s +Step 24000/44000 | Loss: 2.9630 | LR: 2.21e-04 | 1319.7ms/step | 15.2 samples/s +Validation at step 24000: Loss = 2.9755, Perplexity = 19.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-24000 +Step 24010/44000 | Loss: 2.9740 | LR: 2.20e-04 | 3846.4ms/step | 5.2 samples/s +Step 24020/44000 | Loss: 2.9437 | LR: 2.20e-04 | 1324.5ms/step | 15.1 samples/s +Step 24030/44000 | Loss: 3.0026 | LR: 2.20e-04 | 1322.3ms/step | 15.1 samples/s +Step 24040/44000 | Loss: 2.9431 | LR: 2.20e-04 | 1326.2ms/step | 15.1 samples/s +Step 24050/44000 | Loss: 2.9446 | LR: 2.20e-04 | 1321.4ms/step | 15.1 samples/s +Step 24060/44000 | Loss: 3.0237 | LR: 2.19e-04 | 1316.4ms/step | 15.2 samples/s +Step 24070/44000 | Loss: 2.9757 | LR: 2.19e-04 | 1322.7ms/step | 15.1 samples/s +Step 24080/44000 | Loss: 2.9651 | LR: 2.19e-04 | 1319.4ms/step | 15.2 samples/s +Step 24090/44000 | Loss: 3.0076 | LR: 2.19e-04 | 1315.4ms/step | 15.2 samples/s +Step 24100/44000 | Loss: 2.9714 | LR: 2.19e-04 | 1318.2ms/step | 15.2 samples/s +Step 24110/44000 | Loss: 2.9896 | LR: 2.19e-04 | 1317.6ms/step | 15.2 samples/s +Step 24120/44000 | Loss: 2.9789 | LR: 2.18e-04 | 1317.1ms/step | 15.2 samples/s +Step 24130/44000 | Loss: 2.9986 | LR: 2.18e-04 | 1316.4ms/step | 15.2 samples/s +Step 24140/44000 | Loss: 2.9686 | LR: 2.18e-04 | 1313.9ms/step | 15.2 samples/s +Step 24150/44000 | Loss: 2.9699 | LR: 2.18e-04 | 1314.2ms/step | 15.2 samples/s +Step 24160/44000 | Loss: 2.9709 | LR: 2.18e-04 | 1315.7ms/step | 15.2 samples/s +Step 24170/44000 | Loss: 2.9778 | LR: 2.17e-04 | 1320.1ms/step | 15.2 samples/s +Step 24180/44000 | Loss: 2.9752 | LR: 2.17e-04 | 1322.8ms/step | 15.1 samples/s +Step 24190/44000 | Loss: 2.9896 | LR: 2.17e-04 | 1323.1ms/step | 15.1 samples/s +Step 24200/44000 | Loss: 2.9639 | LR: 2.17e-04 | 1322.9ms/step | 15.1 samples/s +Step 24210/44000 | Loss: 2.9614 | LR: 2.17e-04 | 1328.2ms/step | 15.1 samples/s +Step 24220/44000 | Loss: 2.9737 | LR: 2.17e-04 | 1324.7ms/step | 15.1 samples/s +Step 24230/44000 | Loss: 2.9866 | LR: 2.16e-04 | 1325.6ms/step | 15.1 samples/s +Step 24240/44000 | Loss: 2.9892 | LR: 2.16e-04 | 1324.5ms/step | 15.1 samples/s +Step 24250/44000 | Loss: 2.9922 | LR: 2.16e-04 | 1324.2ms/step | 15.1 samples/s +Validation at step 24250: Loss = 2.9727, Perplexity = 19.54 +Step 24260/44000 | Loss: 2.9613 | LR: 2.16e-04 | 3544.7ms/step | 5.6 samples/s +Step 24270/44000 | Loss: 2.9700 | LR: 2.16e-04 | 1324.6ms/step | 15.1 samples/s +Step 24280/44000 | Loss: 2.9651 | LR: 2.16e-04 | 1322.4ms/step | 15.1 samples/s +Step 24290/44000 | Loss: 2.9715 | LR: 2.15e-04 | 1324.0ms/step | 15.1 samples/s +Step 24300/44000 | Loss: 2.9799 | LR: 2.15e-04 | 1323.7ms/step | 15.1 samples/s +Step 24310/44000 | Loss: 2.9795 | LR: 2.15e-04 | 1324.2ms/step | 15.1 samples/s +Step 24320/44000 | Loss: 2.9632 | LR: 2.15e-04 | 1320.5ms/step | 15.1 samples/s +Step 24330/44000 | Loss: 2.9573 | LR: 2.15e-04 | 1322.9ms/step | 15.1 samples/s +Step 24340/44000 | Loss: 2.9414 | LR: 2.14e-04 | 1323.5ms/step | 15.1 samples/s +Step 24350/44000 | Loss: 2.9931 | LR: 2.14e-04 | 1323.1ms/step | 15.1 samples/s +Step 24360/44000 | Loss: 2.9755 | LR: 2.14e-04 | 1324.5ms/step | 15.1 samples/s +Step 24370/44000 | Loss: 2.9814 | LR: 2.14e-04 | 1326.2ms/step | 15.1 samples/s +Step 24380/44000 | Loss: 2.9581 | LR: 2.14e-04 | 1326.5ms/step | 15.1 samples/s +Step 24390/44000 | Loss: 2.9261 | LR: 2.14e-04 | 1327.4ms/step | 15.1 samples/s +Step 24400/44000 | Loss: 2.9766 | LR: 2.13e-04 | 1323.8ms/step | 15.1 samples/s +Step 24410/44000 | Loss: 2.9591 | LR: 2.13e-04 | 1323.6ms/step | 15.1 samples/s +Step 24420/44000 | Loss: 2.9555 | LR: 2.13e-04 | 1322.3ms/step | 15.1 samples/s +Step 24430/44000 | Loss: 2.9384 | LR: 2.13e-04 | 1323.8ms/step | 15.1 samples/s +Step 24440/44000 | Loss: 2.9400 | LR: 2.13e-04 | 1325.7ms/step | 15.1 samples/s +Step 24450/44000 | Loss: 2.9722 | LR: 2.12e-04 | 1329.0ms/step | 15.0 samples/s +Step 24460/44000 | Loss: 2.9482 | LR: 2.12e-04 | 1325.1ms/step | 15.1 samples/s +Step 24470/44000 | Loss: 2.9478 | LR: 2.12e-04 | 1322.8ms/step | 15.1 samples/s +Step 24480/44000 | Loss: 2.9856 | LR: 2.12e-04 | 1322.8ms/step | 15.1 samples/s +Step 24490/44000 | Loss: 2.9627 | LR: 2.12e-04 | 1323.5ms/step | 15.1 samples/s +Step 24500/44000 | Loss: 2.9652 | LR: 2.12e-04 | 1320.4ms/step | 15.1 samples/s +Validation at step 24500: Loss = 2.9712, Perplexity = 19.52 +Step 24510/44000 | Loss: 2.9966 | LR: 2.11e-04 | 3548.4ms/step | 5.6 samples/s +Step 24520/44000 | Loss: 2.9872 | LR: 2.11e-04 | 1323.3ms/step | 15.1 samples/s +Step 24530/44000 | Loss: 2.9619 | LR: 2.11e-04 | 1326.1ms/step | 15.1 samples/s +Step 24540/44000 | Loss: 2.9442 | LR: 2.11e-04 | 1326.0ms/step | 15.1 samples/s +Step 24550/44000 | Loss: 2.9784 | LR: 2.11e-04 | 1324.7ms/step | 15.1 samples/s +Step 24560/44000 | Loss: 2.9714 | LR: 2.10e-04 | 1321.2ms/step | 15.1 samples/s +Step 24570/44000 | Loss: 2.9556 | LR: 2.10e-04 | 1323.8ms/step | 15.1 samples/s +Step 24580/44000 | Loss: 2.9565 | LR: 2.10e-04 | 1324.0ms/step | 15.1 samples/s +Step 24590/44000 | Loss: 2.9700 | LR: 2.10e-04 | 1324.7ms/step | 15.1 samples/s +Step 24600/44000 | Loss: 2.9603 | LR: 2.10e-04 | 1323.4ms/step | 15.1 samples/s +Step 24610/44000 | Loss: 2.9760 | LR: 2.10e-04 | 1319.7ms/step | 15.2 samples/s +Step 24620/44000 | Loss: 2.9598 | LR: 2.09e-04 | 1317.6ms/step | 15.2 samples/s +Step 24630/44000 | Loss: 2.9253 | LR: 2.09e-04 | 1317.2ms/step | 15.2 samples/s +Step 24640/44000 | Loss: 2.9260 | LR: 2.09e-04 | 1321.9ms/step | 15.1 samples/s +Step 24650/44000 | Loss: 2.9376 | LR: 2.09e-04 | 1321.2ms/step | 15.1 samples/s +Step 24660/44000 | Loss: 2.9992 | LR: 2.09e-04 | 1323.1ms/step | 15.1 samples/s +Step 24670/44000 | Loss: 2.9458 | LR: 2.09e-04 | 1322.7ms/step | 15.1 samples/s +Step 24680/44000 | Loss: 2.9690 | LR: 2.08e-04 | 1321.5ms/step | 15.1 samples/s +Step 24690/44000 | Loss: 2.9492 | LR: 2.08e-04 | 1317.1ms/step | 15.2 samples/s +Step 24700/44000 | Loss: 2.9791 | LR: 2.08e-04 | 1318.9ms/step | 15.2 samples/s +Step 24710/44000 | Loss: 2.9757 | LR: 2.08e-04 | 1321.9ms/step | 15.1 samples/s +Step 24720/44000 | Loss: 2.9680 | LR: 2.08e-04 | 1325.1ms/step | 15.1 samples/s +Step 24730/44000 | Loss: 2.9787 | LR: 2.07e-04 | 1325.4ms/step | 15.1 samples/s +Step 24740/44000 | Loss: 2.9309 | LR: 2.07e-04 | 1322.6ms/step | 15.1 samples/s +Step 24750/44000 | Loss: 2.9359 | LR: 2.07e-04 | 1322.3ms/step | 15.1 samples/s +Validation at step 24750: Loss = 2.9693, Perplexity = 19.48 +Step 24760/44000 | Loss: 2.9561 | LR: 2.07e-04 | 3541.7ms/step | 5.6 samples/s +Step 24770/44000 | Loss: 2.9875 | LR: 2.07e-04 | 1323.4ms/step | 15.1 samples/s +Step 24780/44000 | Loss: 3.0040 | LR: 2.07e-04 | 1326.2ms/step | 15.1 samples/s +Step 24790/44000 | Loss: 2.9595 | LR: 2.06e-04 | 1324.2ms/step | 15.1 samples/s +Step 24800/44000 | Loss: 2.9507 | LR: 2.06e-04 | 1326.2ms/step | 15.1 samples/s +Step 24810/44000 | Loss: 2.9467 | LR: 2.06e-04 | 1322.2ms/step | 15.1 samples/s +Step 24820/44000 | Loss: 2.9817 | LR: 2.06e-04 | 1319.6ms/step | 15.2 samples/s +Step 24830/44000 | Loss: 2.9864 | LR: 2.06e-04 | 1318.1ms/step | 15.2 samples/s +Step 24840/44000 | Loss: 2.9402 | LR: 2.05e-04 | 1317.6ms/step | 15.2 samples/s +Step 24850/44000 | Loss: 3.0061 | LR: 2.05e-04 | 1317.2ms/step | 15.2 samples/s +Step 24860/44000 | Loss: 2.9547 | LR: 2.05e-04 | 1319.8ms/step | 15.2 samples/s +Step 24870/44000 | Loss: 2.9431 | LR: 2.05e-04 | 1319.8ms/step | 15.2 samples/s +Step 24880/44000 | Loss: 2.9895 | LR: 2.05e-04 | 1318.4ms/step | 15.2 samples/s +Step 24890/44000 | Loss: 2.9663 | LR: 2.05e-04 | 1317.9ms/step | 15.2 samples/s +Step 24900/44000 | Loss: 2.9664 | LR: 2.04e-04 | 1319.4ms/step | 15.2 samples/s +Step 24910/44000 | Loss: 2.9662 | LR: 2.04e-04 | 1319.7ms/step | 15.2 samples/s +Step 24920/44000 | Loss: 2.9935 | LR: 2.04e-04 | 1316.6ms/step | 15.2 samples/s +Step 24930/44000 | Loss: 2.9488 | LR: 2.04e-04 | 1317.0ms/step | 15.2 samples/s +Step 24940/44000 | Loss: 2.9484 | LR: 2.04e-04 | 1318.9ms/step | 15.2 samples/s +Step 24950/44000 | Loss: 2.9686 | LR: 2.04e-04 | 1317.0ms/step | 15.2 samples/s +Step 24960/44000 | Loss: 2.9493 | LR: 2.03e-04 | 1321.0ms/step | 15.1 samples/s +Step 24970/44000 | Loss: 2.9766 | LR: 2.03e-04 | 1317.6ms/step | 15.2 samples/s +Step 24980/44000 | Loss: 2.9616 | LR: 2.03e-04 | 1317.2ms/step | 15.2 samples/s +Step 24990/44000 | Loss: 2.9655 | LR: 2.03e-04 | 1315.9ms/step | 15.2 samples/s +Step 25000/44000 | Loss: 2.9655 | LR: 2.03e-04 | 1321.1ms/step | 15.1 samples/s +Validation at step 25000: Loss = 2.9674, Perplexity = 19.44 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-25000 +Step 25010/44000 | Loss: 2.9501 | LR: 2.02e-04 | 3836.5ms/step | 5.2 samples/s +Step 25020/44000 | Loss: 2.9623 | LR: 2.02e-04 | 1327.6ms/step | 15.1 samples/s +Step 25030/44000 | Loss: 2.9512 | LR: 2.02e-04 | 1326.4ms/step | 15.1 samples/s +Step 25040/44000 | Loss: 2.9472 | LR: 2.02e-04 | 1322.3ms/step | 15.1 samples/s +Step 25050/44000 | Loss: 2.9588 | LR: 2.02e-04 | 1325.8ms/step | 15.1 samples/s +Step 25060/44000 | Loss: 2.9497 | LR: 2.02e-04 | 1325.3ms/step | 15.1 samples/s +Step 25070/44000 | Loss: 2.9592 | LR: 2.01e-04 | 1324.0ms/step | 15.1 samples/s +Step 25080/44000 | Loss: 2.9706 | LR: 2.01e-04 | 1326.2ms/step | 15.1 samples/s +Step 25090/44000 | Loss: 2.9963 | LR: 2.01e-04 | 1325.7ms/step | 15.1 samples/s +Step 25100/44000 | Loss: 2.9591 | LR: 2.01e-04 | 1325.2ms/step | 15.1 samples/s +Step 25110/44000 | Loss: 2.9675 | LR: 2.01e-04 | 1324.8ms/step | 15.1 samples/s +Step 25120/44000 | Loss: 2.9897 | LR: 2.00e-04 | 1320.3ms/step | 15.1 samples/s +Step 25130/44000 | Loss: 2.9697 | LR: 2.00e-04 | 1324.7ms/step | 15.1 samples/s +Step 25140/44000 | Loss: 2.9651 | LR: 2.00e-04 | 1322.0ms/step | 15.1 samples/s +Step 25150/44000 | Loss: 2.9719 | LR: 2.00e-04 | 1324.6ms/step | 15.1 samples/s +Step 25160/44000 | Loss: 2.9900 | LR: 2.00e-04 | 1324.7ms/step | 15.1 samples/s +Step 25170/44000 | Loss: 2.9314 | LR: 2.00e-04 | 1322.3ms/step | 15.1 samples/s +Step 25180/44000 | Loss: 2.9543 | LR: 1.99e-04 | 1326.6ms/step | 15.1 samples/s +Step 25190/44000 | Loss: 2.9676 | LR: 1.99e-04 | 1327.5ms/step | 15.1 samples/s +Step 25200/44000 | Loss: 2.9651 | LR: 1.99e-04 | 1323.6ms/step | 15.1 samples/s +Step 25210/44000 | Loss: 2.9813 | LR: 1.99e-04 | 1323.5ms/step | 15.1 samples/s +Step 25220/44000 | Loss: 2.9526 | LR: 1.99e-04 | 1322.7ms/step | 15.1 samples/s +Step 25230/44000 | Loss: 2.9677 | LR: 1.99e-04 | 1325.5ms/step | 15.1 samples/s +Step 25240/44000 | Loss: 2.9515 | LR: 1.98e-04 | 1323.6ms/step | 15.1 samples/s +Step 25250/44000 | Loss: 2.9259 | LR: 1.98e-04 | 1320.8ms/step | 15.1 samples/s +Validation at step 25250: Loss = 2.9655, Perplexity = 19.40 +Step 25260/44000 | Loss: 2.9547 | LR: 1.98e-04 | 3547.5ms/step | 5.6 samples/s +Step 25270/44000 | Loss: 2.9952 | LR: 1.98e-04 | 1324.6ms/step | 15.1 samples/s +Step 25280/44000 | Loss: 2.9560 | LR: 1.98e-04 | 1324.3ms/step | 15.1 samples/s +Step 25290/44000 | Loss: 2.9787 | LR: 1.97e-04 | 1324.9ms/step | 15.1 samples/s +Step 25300/44000 | Loss: 2.9738 | LR: 1.97e-04 | 1324.4ms/step | 15.1 samples/s +Step 25310/44000 | Loss: 2.9794 | LR: 1.97e-04 | 1322.5ms/step | 15.1 samples/s +Step 25320/44000 | Loss: 2.9549 | LR: 1.97e-04 | 1324.5ms/step | 15.1 samples/s +Step 25330/44000 | Loss: 2.9140 | LR: 1.97e-04 | 1325.0ms/step | 15.1 samples/s +Step 25340/44000 | Loss: 2.9693 | LR: 1.97e-04 | 1322.6ms/step | 15.1 samples/s +Step 25350/44000 | Loss: 2.9322 | LR: 1.96e-04 | 1323.0ms/step | 15.1 samples/s +Step 25360/44000 | Loss: 2.9799 | LR: 1.96e-04 | 1323.2ms/step | 15.1 samples/s +Step 25370/44000 | Loss: 2.9705 | LR: 1.96e-04 | 1323.0ms/step | 15.1 samples/s +Step 25380/44000 | Loss: 2.9574 | LR: 1.96e-04 | 1324.1ms/step | 15.1 samples/s +Step 25390/44000 | Loss: 2.9503 | LR: 1.96e-04 | 1324.9ms/step | 15.1 samples/s +Step 25400/44000 | Loss: 2.9623 | LR: 1.96e-04 | 1327.2ms/step | 15.1 samples/s +Step 25410/44000 | Loss: 2.9789 | LR: 1.95e-04 | 1322.2ms/step | 15.1 samples/s +Step 25420/44000 | Loss: 2.9919 | LR: 1.95e-04 | 1323.5ms/step | 15.1 samples/s +Step 25430/44000 | Loss: 2.9495 | LR: 1.95e-04 | 1321.8ms/step | 15.1 samples/s +Step 25440/44000 | Loss: 2.9572 | LR: 1.95e-04 | 1325.7ms/step | 15.1 samples/s +Step 25450/44000 | Loss: 2.9564 | LR: 1.95e-04 | 1323.9ms/step | 15.1 samples/s +Step 25460/44000 | Loss: 2.9707 | LR: 1.94e-04 | 1322.6ms/step | 15.1 samples/s +Step 25470/44000 | Loss: 2.9566 | LR: 1.94e-04 | 1323.6ms/step | 15.1 samples/s +Step 25480/44000 | Loss: 2.9541 | LR: 1.94e-04 | 1321.2ms/step | 15.1 samples/s +Step 25490/44000 | Loss: 2.9519 | LR: 1.94e-04 | 1324.5ms/step | 15.1 samples/s +Step 25500/44000 | Loss: 2.9677 | LR: 1.94e-04 | 1318.8ms/step | 15.2 samples/s +Validation at step 25500: Loss = 2.9638, Perplexity = 19.37 +Step 25510/44000 | Loss: 2.9743 | LR: 1.94e-04 | 3543.8ms/step | 5.6 samples/s +Step 25520/44000 | Loss: 2.9516 | LR: 1.93e-04 | 1324.5ms/step | 15.1 samples/s +Step 25530/44000 | Loss: 2.9885 | LR: 1.93e-04 | 1326.2ms/step | 15.1 samples/s +Step 25540/44000 | Loss: 2.9405 | LR: 1.93e-04 | 1324.5ms/step | 15.1 samples/s +Step 25550/44000 | Loss: 2.9609 | LR: 1.93e-04 | 1321.9ms/step | 15.1 samples/s +Step 25560/44000 | Loss: 2.9266 | LR: 1.93e-04 | 1325.5ms/step | 15.1 samples/s +Step 25570/44000 | Loss: 2.9633 | LR: 1.92e-04 | 1325.8ms/step | 15.1 samples/s +Step 25580/44000 | Loss: 2.9705 | LR: 1.92e-04 | 1322.2ms/step | 15.1 samples/s +Step 25590/44000 | Loss: 2.9479 | LR: 1.92e-04 | 1325.6ms/step | 15.1 samples/s +Step 25600/44000 | Loss: 2.9645 | LR: 1.92e-04 | 1325.2ms/step | 15.1 samples/s +Step 25610/44000 | Loss: 2.9663 | LR: 1.92e-04 | 1322.8ms/step | 15.1 samples/s +Step 25620/44000 | Loss: 2.9695 | LR: 1.92e-04 | 1322.0ms/step | 15.1 samples/s +Step 25630/44000 | Loss: 2.9633 | LR: 1.91e-04 | 1320.1ms/step | 15.1 samples/s +Step 25640/44000 | Loss: 2.9500 | LR: 1.91e-04 | 1325.5ms/step | 15.1 samples/s +Step 25650/44000 | Loss: 2.9330 | LR: 1.91e-04 | 1318.1ms/step | 15.2 samples/s +Step 25660/44000 | Loss: 2.9856 | LR: 1.91e-04 | 1324.4ms/step | 15.1 samples/s +Step 25670/44000 | Loss: 2.9687 | LR: 1.91e-04 | 1324.7ms/step | 15.1 samples/s +Step 25680/44000 | Loss: 2.9490 | LR: 1.91e-04 | 1319.6ms/step | 15.2 samples/s +Step 25690/44000 | Loss: 2.9595 | LR: 1.90e-04 | 1320.8ms/step | 15.1 samples/s +Step 25700/44000 | Loss: 2.9633 | LR: 1.90e-04 | 1324.6ms/step | 15.1 samples/s +Step 25710/44000 | Loss: 2.9334 | LR: 1.90e-04 | 1321.8ms/step | 15.1 samples/s +Step 25720/44000 | Loss: 2.9448 | LR: 1.90e-04 | 1323.5ms/step | 15.1 samples/s +Step 25730/44000 | Loss: 2.9513 | LR: 1.90e-04 | 1322.7ms/step | 15.1 samples/s +Step 25740/44000 | Loss: 2.9448 | LR: 1.89e-04 | 1324.7ms/step | 15.1 samples/s +Step 25750/44000 | Loss: 2.9594 | LR: 1.89e-04 | 1320.3ms/step | 15.1 samples/s +Validation at step 25750: Loss = 2.9624, Perplexity = 19.34 +Step 25760/44000 | Loss: 2.9411 | LR: 1.89e-04 | 3542.5ms/step | 5.6 samples/s +Step 25770/44000 | Loss: 2.9629 | LR: 1.89e-04 | 1323.3ms/step | 15.1 samples/s +Step 25780/44000 | Loss: 2.9743 | LR: 1.89e-04 | 1322.7ms/step | 15.1 samples/s +Step 25790/44000 | Loss: 2.9895 | LR: 1.89e-04 | 1325.1ms/step | 15.1 samples/s +Step 25800/44000 | Loss: 2.9709 | LR: 1.88e-04 | 1322.5ms/step | 15.1 samples/s +Step 25810/44000 | Loss: 2.9312 | LR: 1.88e-04 | 1322.0ms/step | 15.1 samples/s +Step 25820/44000 | Loss: 2.9647 | LR: 1.88e-04 | 1326.4ms/step | 15.1 samples/s +Step 25830/44000 | Loss: 2.9585 | LR: 1.88e-04 | 1322.2ms/step | 15.1 samples/s +Step 25840/44000 | Loss: 2.9650 | LR: 1.88e-04 | 1325.0ms/step | 15.1 samples/s +Step 25850/44000 | Loss: 2.9866 | LR: 1.88e-04 | 1326.0ms/step | 15.1 samples/s +Step 25860/44000 | Loss: 2.9353 | LR: 1.87e-04 | 1327.5ms/step | 15.1 samples/s +Step 25870/44000 | Loss: 2.9661 | LR: 1.87e-04 | 1324.2ms/step | 15.1 samples/s +Step 25880/44000 | Loss: 2.9720 | LR: 1.87e-04 | 1323.9ms/step | 15.1 samples/s +Step 25890/44000 | Loss: 2.9357 | LR: 1.87e-04 | 1325.8ms/step | 15.1 samples/s +Step 25900/44000 | Loss: 2.9224 | LR: 1.87e-04 | 1325.2ms/step | 15.1 samples/s +Step 25910/44000 | Loss: 2.9661 | LR: 1.87e-04 | 1325.2ms/step | 15.1 samples/s +Step 25920/44000 | Loss: 2.9598 | LR: 1.86e-04 | 1326.9ms/step | 15.1 samples/s +Step 25930/44000 | Loss: 2.9379 | LR: 1.86e-04 | 1325.3ms/step | 15.1 samples/s +Step 25940/44000 | Loss: 2.9353 | LR: 1.86e-04 | 1325.8ms/step | 15.1 samples/s +Step 25950/44000 | Loss: 2.9622 | LR: 1.86e-04 | 1326.0ms/step | 15.1 samples/s +Step 25960/44000 | Loss: 2.9538 | LR: 1.86e-04 | 1330.2ms/step | 15.0 samples/s +Step 25970/44000 | Loss: 2.9582 | LR: 1.85e-04 | 1325.1ms/step | 15.1 samples/s +Step 25980/44000 | Loss: 2.9545 | LR: 1.85e-04 | 1322.0ms/step | 15.1 samples/s +Step 25990/44000 | Loss: 2.9379 | LR: 1.85e-04 | 1325.3ms/step | 15.1 samples/s +Step 26000/44000 | Loss: 2.9530 | LR: 1.85e-04 | 1325.6ms/step | 15.1 samples/s +Validation at step 26000: Loss = 2.9607, Perplexity = 19.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-26000 +Step 26010/44000 | Loss: 2.9096 | LR: 1.85e-04 | 3861.3ms/step | 5.2 samples/s +Step 26020/44000 | Loss: 2.9755 | LR: 1.85e-04 | 1325.1ms/step | 15.1 samples/s +Step 26030/44000 | Loss: 2.9360 | LR: 1.84e-04 | 1323.9ms/step | 15.1 samples/s +Step 26040/44000 | Loss: 2.9501 | LR: 1.84e-04 | 1328.4ms/step | 15.1 samples/s +Step 26050/44000 | Loss: 2.9581 | LR: 1.84e-04 | 1325.4ms/step | 15.1 samples/s +Step 26060/44000 | Loss: 2.9618 | LR: 1.84e-04 | 1324.0ms/step | 15.1 samples/s +Step 26070/44000 | Loss: 2.9172 | LR: 1.84e-04 | 1324.7ms/step | 15.1 samples/s +Step 26080/44000 | Loss: 2.9352 | LR: 1.84e-04 | 1323.2ms/step | 15.1 samples/s +Step 26090/44000 | Loss: 2.9655 | LR: 1.83e-04 | 1320.0ms/step | 15.2 samples/s +Step 26100/44000 | Loss: 2.9377 | LR: 1.83e-04 | 1321.4ms/step | 15.1 samples/s diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed44_47/l8k365m_static_nrb35_seed44.log b/epfl_final/train_logs/p1_controls_nrb35_seed44_47/l8k365m_static_nrb35_seed44.log new file mode 100644 index 0000000000000000000000000000000000000000..1092d9b5c7da3cf36f2547caa636dfbd76423f0a --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed44_47/l8k365m_static_nrb35_seed44.log @@ -0,0 +1,2941 @@ +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 44, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 365m static 8k nrb35 seed44", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "static", + "nrb35", + "seed44" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1", + "sparse_mask_seed": 1, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_180856-na2e5dc1 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 365m static 8k nrb35 seed44 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/na2e5dc1 +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0408 | LR: 6.67e-06 | 7555.7ms/step | 2.6 samples/s +Step 20/44000 | Loss: 11.9486 | LR: 1.33e-05 | 1306.0ms/step | 15.3 samples/s +Step 30/44000 | Loss: 11.6185 | LR: 2.00e-05 | 1307.3ms/step | 15.3 samples/s +Step 40/44000 | Loss: 11.0847 | LR: 2.67e-05 | 1309.0ms/step | 15.3 samples/s +Step 50/44000 | Loss: 10.7638 | LR: 3.33e-05 | 1304.0ms/step | 15.3 samples/s +Step 60/44000 | Loss: 10.5126 | LR: 4.00e-05 | 1295.8ms/step | 15.4 samples/s +Step 70/44000 | Loss: 10.2867 | LR: 4.67e-05 | 1297.6ms/step | 15.4 samples/s +Step 80/44000 | Loss: 9.9815 | LR: 5.33e-05 | 1295.3ms/step | 15.4 samples/s +Step 90/44000 | Loss: 9.5968 | LR: 6.00e-05 | 1304.0ms/step | 15.3 samples/s +Step 100/44000 | Loss: 9.1227 | LR: 6.67e-05 | 1305.0ms/step | 15.3 samples/s +Step 110/44000 | Loss: 8.6010 | LR: 7.33e-05 | 1307.7ms/step | 15.3 samples/s +Step 120/44000 | Loss: 8.1120 | LR: 8.00e-05 | 1307.5ms/step | 15.3 samples/s +Step 130/44000 | Loss: 7.7435 | LR: 8.67e-05 | 1307.7ms/step | 15.3 samples/s +Step 140/44000 | Loss: 7.5272 | LR: 9.33e-05 | 1306.1ms/step | 15.3 samples/s +Step 150/44000 | Loss: 7.3197 | LR: 1.00e-04 | 1308.2ms/step | 15.3 samples/s +Step 160/44000 | Loss: 7.2150 | LR: 1.07e-04 | 1309.2ms/step | 15.3 samples/s +Step 170/44000 | Loss: 7.1033 | LR: 1.13e-04 | 1315.3ms/step | 15.2 samples/s +Step 180/44000 | Loss: 7.0201 | LR: 1.20e-04 | 1314.4ms/step | 15.2 samples/s +Step 190/44000 | Loss: 6.9044 | LR: 1.27e-04 | 1314.3ms/step | 15.2 samples/s +Step 200/44000 | Loss: 6.8394 | LR: 1.33e-04 | 1310.4ms/step | 15.3 samples/s +Step 210/44000 | Loss: 6.7521 | LR: 1.40e-04 | 1312.6ms/step | 15.2 samples/s +Step 220/44000 | Loss: 6.6688 | LR: 1.47e-04 | 1313.1ms/step | 15.2 samples/s +Step 230/44000 | Loss: 6.6015 | LR: 1.53e-04 | 1311.6ms/step | 15.2 samples/s +Step 240/44000 | Loss: 6.4993 | LR: 1.60e-04 | 1315.1ms/step | 15.2 samples/s +Step 250/44000 | Loss: 6.4578 | LR: 1.67e-04 | 1314.3ms/step | 15.2 samples/s +Validation at step 250: Loss = 6.4373, Perplexity = 624.74 +Step 260/44000 | Loss: 6.3877 | LR: 1.73e-04 | 5663.6ms/step | 3.5 samples/s +Step 270/44000 | Loss: 6.3575 | LR: 1.80e-04 | 1317.1ms/step | 15.2 samples/s +Step 280/44000 | Loss: 6.2984 | LR: 1.87e-04 | 1317.7ms/step | 15.2 samples/s +Step 290/44000 | Loss: 6.2779 | LR: 1.93e-04 | 1316.4ms/step | 15.2 samples/s +Step 300/44000 | Loss: 6.2188 | LR: 2.00e-04 | 1318.2ms/step | 15.2 samples/s +Step 310/44000 | Loss: 6.1907 | LR: 2.07e-04 | 1319.2ms/step | 15.2 samples/s +Step 320/44000 | Loss: 6.1303 | LR: 2.13e-04 | 1319.1ms/step | 15.2 samples/s +Step 330/44000 | Loss: 6.1043 | LR: 2.20e-04 | 1318.0ms/step | 15.2 samples/s +Step 340/44000 | Loss: 6.0341 | LR: 2.27e-04 | 1319.9ms/step | 15.2 samples/s +Step 350/44000 | Loss: 6.0013 | LR: 2.33e-04 | 1319.5ms/step | 15.2 samples/s +Step 360/44000 | Loss: 5.9434 | LR: 2.40e-04 | 1316.9ms/step | 15.2 samples/s +Step 370/44000 | Loss: 5.9055 | LR: 2.47e-04 | 1314.7ms/step | 15.2 samples/s +Step 380/44000 | Loss: 5.8780 | LR: 2.53e-04 | 1318.1ms/step | 15.2 samples/s +Step 390/44000 | Loss: 5.8329 | LR: 2.60e-04 | 1318.7ms/step | 15.2 samples/s +Step 400/44000 | Loss: 5.8295 | LR: 2.67e-04 | 1316.5ms/step | 15.2 samples/s +Step 410/44000 | Loss: 5.7851 | LR: 2.73e-04 | 1314.9ms/step | 15.2 samples/s +Step 420/44000 | Loss: 5.7446 | LR: 2.80e-04 | 1319.0ms/step | 15.2 samples/s +Step 430/44000 | Loss: 5.7500 | LR: 2.87e-04 | 1318.7ms/step | 15.2 samples/s +Step 440/44000 | Loss: 5.6993 | LR: 2.93e-04 | 1318.3ms/step | 15.2 samples/s +Step 450/44000 | Loss: 5.6358 | LR: 3.00e-04 | 1321.1ms/step | 15.1 samples/s +Step 460/44000 | Loss: 5.6177 | LR: 3.07e-04 | 1316.8ms/step | 15.2 samples/s +Step 470/44000 | Loss: 5.5675 | LR: 3.13e-04 | 1316.7ms/step | 15.2 samples/s +Step 480/44000 | Loss: 5.5182 | LR: 3.20e-04 | 1318.2ms/step | 15.2 samples/s +Step 490/44000 | Loss: 5.5221 | LR: 3.27e-04 | 1318.8ms/step | 15.2 samples/s +Step 500/44000 | Loss: 5.4700 | LR: 3.33e-04 | 1316.1ms/step | 15.2 samples/s +Validation at step 500: Loss = 5.4644, Perplexity = 236.14 +Step 510/44000 | Loss: 5.4384 | LR: 3.40e-04 | 3534.0ms/step | 5.7 samples/s +Step 520/44000 | Loss: 5.4244 | LR: 3.47e-04 | 1317.8ms/step | 15.2 samples/s +Step 530/44000 | Loss: 5.3765 | LR: 3.53e-04 | 1316.6ms/step | 15.2 samples/s +Step 540/44000 | Loss: 5.3592 | LR: 3.60e-04 | 1317.6ms/step | 15.2 samples/s +Step 550/44000 | Loss: 5.3533 | LR: 3.67e-04 | 1318.1ms/step | 15.2 samples/s +Step 560/44000 | Loss: 5.2913 | LR: 3.73e-04 | 1318.0ms/step | 15.2 samples/s +Step 570/44000 | Loss: 5.2477 | LR: 3.80e-04 | 1317.9ms/step | 15.2 samples/s +Step 580/44000 | Loss: 5.2192 | LR: 3.87e-04 | 1318.7ms/step | 15.2 samples/s +Step 590/44000 | Loss: 5.2439 | LR: 3.93e-04 | 1317.1ms/step | 15.2 samples/s +Step 600/44000 | Loss: 5.1751 | LR: 4.00e-04 | 1320.5ms/step | 15.1 samples/s +Step 610/44000 | Loss: 5.1758 | LR: 4.07e-04 | 1325.8ms/step | 15.1 samples/s +Step 620/44000 | Loss: 5.1147 | LR: 4.13e-04 | 1319.4ms/step | 15.2 samples/s +Step 630/44000 | Loss: 5.0797 | LR: 4.20e-04 | 1318.8ms/step | 15.2 samples/s +Step 640/44000 | Loss: 5.0693 | LR: 4.27e-04 | 1319.5ms/step | 15.2 samples/s +Step 650/44000 | Loss: 5.0169 | LR: 4.33e-04 | 1316.2ms/step | 15.2 samples/s +Step 660/44000 | Loss: 4.9684 | LR: 4.40e-04 | 1319.6ms/step | 15.2 samples/s +Step 670/44000 | Loss: 4.9957 | LR: 4.47e-04 | 1323.9ms/step | 15.1 samples/s +Step 680/44000 | Loss: 4.9622 | LR: 4.53e-04 | 1325.8ms/step | 15.1 samples/s +Step 690/44000 | Loss: 4.9029 | LR: 4.60e-04 | 1324.3ms/step | 15.1 samples/s +Step 700/44000 | Loss: 4.8885 | LR: 4.67e-04 | 1326.5ms/step | 15.1 samples/s +Step 710/44000 | Loss: 4.8546 | LR: 4.73e-04 | 1325.4ms/step | 15.1 samples/s +Step 720/44000 | Loss: 4.8514 | LR: 4.80e-04 | 1321.0ms/step | 15.1 samples/s +Step 730/44000 | Loss: 4.8361 | LR: 4.87e-04 | 1322.4ms/step | 15.1 samples/s +Step 740/44000 | Loss: 4.7989 | LR: 4.93e-04 | 1321.6ms/step | 15.1 samples/s +Step 750/44000 | Loss: 4.7038 | LR: 5.00e-04 | 1324.8ms/step | 15.1 samples/s +Validation at step 750: Loss = 4.7508, Perplexity = 115.68 +Step 760/44000 | Loss: 4.7550 | LR: 5.00e-04 | 3541.9ms/step | 5.6 samples/s +Step 770/44000 | Loss: 4.7166 | LR: 5.00e-04 | 1322.1ms/step | 15.1 samples/s +Step 780/44000 | Loss: 4.6857 | LR: 5.00e-04 | 1320.6ms/step | 15.1 samples/s +Step 790/44000 | Loss: 4.6912 | LR: 5.00e-04 | 1324.2ms/step | 15.1 samples/s +Step 800/44000 | Loss: 4.6624 | LR: 5.00e-04 | 1322.1ms/step | 15.1 samples/s +Step 810/44000 | Loss: 4.6223 | LR: 5.00e-04 | 1320.8ms/step | 15.1 samples/s +Step 820/44000 | Loss: 4.6176 | LR: 5.00e-04 | 1321.8ms/step | 15.1 samples/s +Step 830/44000 | Loss: 4.5897 | LR: 5.00e-04 | 1320.6ms/step | 15.1 samples/s +Step 840/44000 | Loss: 4.5340 | LR: 5.00e-04 | 1323.0ms/step | 15.1 samples/s +Step 850/44000 | Loss: 4.5377 | LR: 5.00e-04 | 1324.1ms/step | 15.1 samples/s +Step 860/44000 | Loss: 4.5403 | LR: 5.00e-04 | 1325.4ms/step | 15.1 samples/s +Step 870/44000 | Loss: 4.4835 | LR: 5.00e-04 | 1321.1ms/step | 15.1 samples/s +Step 880/44000 | Loss: 4.4861 | LR: 5.00e-04 | 1322.5ms/step | 15.1 samples/s +Step 890/44000 | Loss: 4.4774 | LR: 5.00e-04 | 1324.2ms/step | 15.1 samples/s +Step 900/44000 | Loss: 4.4465 | LR: 5.00e-04 | 1322.0ms/step | 15.1 samples/s +Step 910/44000 | Loss: 4.4293 | LR: 5.00e-04 | 1323.6ms/step | 15.1 samples/s +Step 920/44000 | Loss: 4.3996 | LR: 5.00e-04 | 1325.2ms/step | 15.1 samples/s +Step 930/44000 | Loss: 4.4014 | LR: 5.00e-04 | 1324.2ms/step | 15.1 samples/s +Step 940/44000 | Loss: 4.4107 | LR: 5.00e-04 | 1321.7ms/step | 15.1 samples/s +Step 950/44000 | Loss: 4.3534 | LR: 5.00e-04 | 1323.6ms/step | 15.1 samples/s +Step 960/44000 | Loss: 4.3919 | LR: 5.00e-04 | 1326.4ms/step | 15.1 samples/s +Step 970/44000 | Loss: 4.3570 | LR: 5.00e-04 | 1326.0ms/step | 15.1 samples/s +Step 980/44000 | Loss: 4.3270 | LR: 5.00e-04 | 1327.2ms/step | 15.1 samples/s +Step 990/44000 | Loss: 4.3306 | LR: 5.00e-04 | 1332.2ms/step | 15.0 samples/s +Step 1000/44000 | Loss: 4.3017 | LR: 5.00e-04 | 1338.4ms/step | 14.9 samples/s +Validation at step 1000: Loss = 4.3083, Perplexity = 74.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-1000 +Step 1010/44000 | Loss: 4.3012 | LR: 5.00e-04 | 3960.3ms/step | 5.1 samples/s +Step 1020/44000 | Loss: 4.2577 | LR: 5.00e-04 | 1333.8ms/step | 15.0 samples/s +Step 1030/44000 | Loss: 4.2719 | LR: 5.00e-04 | 1322.8ms/step | 15.1 samples/s +Step 1040/44000 | Loss: 4.2626 | LR: 5.00e-04 | 1320.7ms/step | 15.1 samples/s +Step 1050/44000 | Loss: 4.2331 | LR: 5.00e-04 | 1321.9ms/step | 15.1 samples/s +Step 1060/44000 | Loss: 4.2252 | LR: 5.00e-04 | 1322.3ms/step | 15.1 samples/s +Step 1070/44000 | Loss: 4.2201 | LR: 5.00e-04 | 1321.9ms/step | 15.1 samples/s +Step 1080/44000 | Loss: 4.1866 | LR: 5.00e-04 | 1323.3ms/step | 15.1 samples/s +Step 1090/44000 | Loss: 4.1935 | LR: 5.00e-04 | 1318.9ms/step | 15.2 samples/s +Step 1100/44000 | Loss: 4.1772 | LR: 5.00e-04 | 1316.7ms/step | 15.2 samples/s +Step 1110/44000 | Loss: 4.1510 | LR: 5.00e-04 | 1316.1ms/step | 15.2 samples/s +Step 1120/44000 | Loss: 4.1618 | LR: 5.00e-04 | 1316.8ms/step | 15.2 samples/s +Step 1130/44000 | Loss: 4.2024 | LR: 5.00e-04 | 1318.1ms/step | 15.2 samples/s +Step 1140/44000 | Loss: 4.1373 | LR: 5.00e-04 | 1316.5ms/step | 15.2 samples/s +Step 1150/44000 | Loss: 4.1098 | LR: 5.00e-04 | 1318.6ms/step | 15.2 samples/s +Step 1160/44000 | Loss: 4.1388 | LR: 5.00e-04 | 1317.8ms/step | 15.2 samples/s +Step 1170/44000 | Loss: 4.1117 | LR: 5.00e-04 | 1318.3ms/step | 15.2 samples/s +Step 1180/44000 | Loss: 4.1528 | LR: 5.00e-04 | 1320.1ms/step | 15.2 samples/s +Step 1190/44000 | Loss: 4.1236 | LR: 5.00e-04 | 1320.0ms/step | 15.2 samples/s +Step 1200/44000 | Loss: 4.0877 | LR: 5.00e-04 | 1319.7ms/step | 15.2 samples/s +Step 1210/44000 | Loss: 4.0814 | LR: 5.00e-04 | 1320.2ms/step | 15.1 samples/s +Step 1220/44000 | Loss: 4.0693 | LR: 5.00e-04 | 1318.2ms/step | 15.2 samples/s +Step 1230/44000 | Loss: 4.0762 | LR: 5.00e-04 | 1321.6ms/step | 15.1 samples/s +Step 1240/44000 | Loss: 4.0558 | LR: 5.00e-04 | 1319.9ms/step | 15.2 samples/s +Step 1250/44000 | Loss: 4.0462 | LR: 5.00e-04 | 1317.8ms/step | 15.2 samples/s +Validation at step 1250: Loss = 4.0662, Perplexity = 58.34 +Step 1260/44000 | Loss: 4.0582 | LR: 5.00e-04 | 3537.7ms/step | 5.7 samples/s +Step 1270/44000 | Loss: 4.0346 | LR: 5.00e-04 | 1321.9ms/step | 15.1 samples/s +Step 1280/44000 | Loss: 4.0621 | LR: 5.00e-04 | 1323.8ms/step | 15.1 samples/s +Step 1290/44000 | Loss: 4.0471 | LR: 5.00e-04 | 1321.1ms/step | 15.1 samples/s +Step 1300/44000 | Loss: 4.0309 | LR: 5.00e-04 | 1319.7ms/step | 15.2 samples/s +Step 1310/44000 | Loss: 4.0367 | LR: 5.00e-04 | 1319.1ms/step | 15.2 samples/s +Step 1320/44000 | Loss: 3.9943 | LR: 5.00e-04 | 1318.8ms/step | 15.2 samples/s +Step 1330/44000 | Loss: 4.0322 | LR: 5.00e-04 | 1321.9ms/step | 15.1 samples/s +Step 1340/44000 | Loss: 3.9810 | LR: 5.00e-04 | 1319.1ms/step | 15.2 samples/s +Step 1350/44000 | Loss: 3.9851 | LR: 5.00e-04 | 1318.0ms/step | 15.2 samples/s +Step 1360/44000 | Loss: 3.9399 | LR: 5.00e-04 | 1318.0ms/step | 15.2 samples/s +Step 1370/44000 | Loss: 3.9757 | LR: 5.00e-04 | 1317.5ms/step | 15.2 samples/s +Step 1380/44000 | Loss: 3.9553 | LR: 5.00e-04 | 1321.2ms/step | 15.1 samples/s +Step 1390/44000 | Loss: 3.9646 | LR: 5.00e-04 | 1318.2ms/step | 15.2 samples/s +Step 1400/44000 | Loss: 3.9593 | LR: 5.00e-04 | 1316.8ms/step | 15.2 samples/s +Step 1410/44000 | Loss: 3.9791 | LR: 5.00e-04 | 1317.0ms/step | 15.2 samples/s +Step 1420/44000 | Loss: 3.9362 | LR: 5.00e-04 | 1320.4ms/step | 15.1 samples/s +Step 1430/44000 | Loss: 3.9469 | LR: 5.00e-04 | 1320.2ms/step | 15.1 samples/s +Step 1440/44000 | Loss: 3.9584 | LR: 5.00e-04 | 1320.5ms/step | 15.1 samples/s +Step 1450/44000 | Loss: 3.9674 | LR: 5.00e-04 | 1320.0ms/step | 15.2 samples/s +Step 1460/44000 | Loss: 3.9675 | LR: 5.00e-04 | 1318.8ms/step | 15.2 samples/s +Step 1470/44000 | Loss: 3.9214 | LR: 5.00e-04 | 1314.9ms/step | 15.2 samples/s +Step 1480/44000 | Loss: 3.9110 | LR: 5.00e-04 | 1317.5ms/step | 15.2 samples/s +Step 1490/44000 | Loss: 3.9203 | LR: 5.00e-04 | 1320.0ms/step | 15.2 samples/s +Step 1500/44000 | Loss: 3.9129 | LR: 5.00e-04 | 1318.7ms/step | 15.2 samples/s +Validation at step 1500: Loss = 3.9114, Perplexity = 49.97 +Step 1510/44000 | Loss: 3.8995 | LR: 5.00e-04 | 3534.2ms/step | 5.7 samples/s +Step 1520/44000 | Loss: 3.8749 | LR: 5.00e-04 | 1320.6ms/step | 15.1 samples/s +Step 1530/44000 | Loss: 3.8915 | LR: 5.00e-04 | 1316.9ms/step | 15.2 samples/s +Step 1540/44000 | Loss: 3.8629 | LR: 5.00e-04 | 1318.5ms/step | 15.2 samples/s +Step 1550/44000 | Loss: 3.8827 | LR: 5.00e-04 | 1320.5ms/step | 15.1 samples/s +Step 1560/44000 | Loss: 3.9047 | LR: 5.00e-04 | 1318.6ms/step | 15.2 samples/s +Step 1570/44000 | Loss: 3.8800 | LR: 5.00e-04 | 1320.8ms/step | 15.1 samples/s +Step 1580/44000 | Loss: 3.8736 | LR: 5.00e-04 | 1322.7ms/step | 15.1 samples/s +Step 1590/44000 | Loss: 3.8915 | LR: 5.00e-04 | 1319.2ms/step | 15.2 samples/s +Step 1600/44000 | Loss: 3.8607 | LR: 5.00e-04 | 1319.9ms/step | 15.2 samples/s +Step 1610/44000 | Loss: 3.8526 | LR: 5.00e-04 | 1320.0ms/step | 15.2 samples/s +Step 1620/44000 | Loss: 3.8509 | LR: 5.00e-04 | 1321.1ms/step | 15.1 samples/s +Step 1630/44000 | Loss: 3.8295 | LR: 4.99e-04 | 1320.0ms/step | 15.2 samples/s +Step 1640/44000 | Loss: 3.8636 | LR: 4.99e-04 | 1318.3ms/step | 15.2 samples/s +Step 1650/44000 | Loss: 3.8446 | LR: 4.99e-04 | 1323.7ms/step | 15.1 samples/s +Step 1660/44000 | Loss: 3.8463 | LR: 4.99e-04 | 1318.9ms/step | 15.2 samples/s +Step 1670/44000 | Loss: 3.8166 | LR: 4.99e-04 | 1316.7ms/step | 15.2 samples/s +Step 1680/44000 | Loss: 3.8419 | LR: 4.99e-04 | 1319.7ms/step | 15.2 samples/s +Step 1690/44000 | Loss: 3.8471 | LR: 4.99e-04 | 1318.1ms/step | 15.2 samples/s +Step 1700/44000 | Loss: 3.8131 | LR: 4.99e-04 | 1316.8ms/step | 15.2 samples/s +Step 1710/44000 | Loss: 3.8259 | LR: 4.99e-04 | 1318.1ms/step | 15.2 samples/s +Step 1720/44000 | Loss: 3.7649 | LR: 4.99e-04 | 1317.7ms/step | 15.2 samples/s +Step 1730/44000 | Loss: 3.8481 | LR: 4.99e-04 | 1318.1ms/step | 15.2 samples/s +Step 1740/44000 | Loss: 3.7937 | LR: 4.99e-04 | 1318.6ms/step | 15.2 samples/s +Step 1750/44000 | Loss: 3.8117 | LR: 4.99e-04 | 1318.6ms/step | 15.2 samples/s +Validation at step 1750: Loss = 3.8059, Perplexity = 44.97 +Step 1760/44000 | Loss: 3.8270 | LR: 4.99e-04 | 3528.8ms/step | 5.7 samples/s +Step 1770/44000 | Loss: 3.8040 | LR: 4.99e-04 | 1316.8ms/step | 15.2 samples/s +Step 1780/44000 | Loss: 3.8102 | LR: 4.99e-04 | 1319.1ms/step | 15.2 samples/s +Step 1790/44000 | Loss: 3.7838 | LR: 4.99e-04 | 1319.3ms/step | 15.2 samples/s +Step 1800/44000 | Loss: 3.7811 | LR: 4.99e-04 | 1319.7ms/step | 15.2 samples/s +Step 1810/44000 | Loss: 3.8090 | LR: 4.99e-04 | 1319.7ms/step | 15.2 samples/s +Step 1820/44000 | Loss: 3.7895 | LR: 4.99e-04 | 1319.8ms/step | 15.2 samples/s +Step 1830/44000 | Loss: 3.7832 | LR: 4.99e-04 | 1320.2ms/step | 15.1 samples/s +Step 1840/44000 | Loss: 3.7686 | LR: 4.99e-04 | 1318.8ms/step | 15.2 samples/s +Step 1850/44000 | Loss: 3.7259 | LR: 4.99e-04 | 1319.8ms/step | 15.2 samples/s +Step 1860/44000 | Loss: 3.7533 | LR: 4.99e-04 | 1319.0ms/step | 15.2 samples/s +Step 1870/44000 | Loss: 3.7792 | LR: 4.99e-04 | 1316.8ms/step | 15.2 samples/s +Step 1880/44000 | Loss: 3.7243 | LR: 4.99e-04 | 1315.0ms/step | 15.2 samples/s +Step 1890/44000 | Loss: 3.7341 | LR: 4.99e-04 | 1319.8ms/step | 15.2 samples/s +Step 1900/44000 | Loss: 3.7381 | LR: 4.99e-04 | 1319.1ms/step | 15.2 samples/s +Step 1910/44000 | Loss: 3.7389 | LR: 4.99e-04 | 1321.2ms/step | 15.1 samples/s +Step 1920/44000 | Loss: 3.7337 | LR: 4.99e-04 | 1321.7ms/step | 15.1 samples/s +Step 1930/44000 | Loss: 3.7745 | LR: 4.99e-04 | 1321.3ms/step | 15.1 samples/s +Step 1940/44000 | Loss: 3.7191 | LR: 4.99e-04 | 1319.5ms/step | 15.2 samples/s +Step 1950/44000 | Loss: 3.7569 | LR: 4.99e-04 | 1318.1ms/step | 15.2 samples/s +Step 1960/44000 | Loss: 3.7213 | LR: 4.99e-04 | 1320.4ms/step | 15.1 samples/s +Step 1970/44000 | Loss: 3.7408 | LR: 4.99e-04 | 1319.0ms/step | 15.2 samples/s +Step 1980/44000 | Loss: 3.7528 | LR: 4.99e-04 | 1320.5ms/step | 15.1 samples/s +Step 1990/44000 | Loss: 3.7099 | LR: 4.99e-04 | 1317.5ms/step | 15.2 samples/s +Step 2000/44000 | Loss: 3.7069 | LR: 4.99e-04 | 1319.0ms/step | 15.2 samples/s +Validation at step 2000: Loss = 3.7242, Perplexity = 41.44 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-2000 +Step 2010/44000 | Loss: 3.7151 | LR: 4.99e-04 | 3861.5ms/step | 5.2 samples/s +Step 2020/44000 | Loss: 3.7268 | LR: 4.99e-04 | 1321.1ms/step | 15.1 samples/s +Step 2030/44000 | Loss: 3.7072 | LR: 4.99e-04 | 1319.1ms/step | 15.2 samples/s +Step 2040/44000 | Loss: 3.7076 | LR: 4.99e-04 | 1321.8ms/step | 15.1 samples/s +Step 2050/44000 | Loss: 3.6914 | LR: 4.99e-04 | 1317.6ms/step | 15.2 samples/s +Step 2060/44000 | Loss: 3.7293 | LR: 4.99e-04 | 1320.2ms/step | 15.1 samples/s +Step 2070/44000 | Loss: 3.6994 | LR: 4.99e-04 | 1318.0ms/step | 15.2 samples/s +Step 2080/44000 | Loss: 3.6923 | LR: 4.99e-04 | 1315.9ms/step | 15.2 samples/s +Step 2090/44000 | Loss: 3.7010 | LR: 4.99e-04 | 1314.8ms/step | 15.2 samples/s +Step 2100/44000 | Loss: 3.6959 | LR: 4.99e-04 | 1317.3ms/step | 15.2 samples/s +Step 2110/44000 | Loss: 3.6744 | LR: 4.99e-04 | 1318.2ms/step | 15.2 samples/s +Step 2120/44000 | Loss: 3.6462 | LR: 4.99e-04 | 1310.3ms/step | 15.3 samples/s +Step 2130/44000 | Loss: 3.6880 | LR: 4.99e-04 | 1311.7ms/step | 15.2 samples/s +Step 2140/44000 | Loss: 3.7161 | LR: 4.99e-04 | 1315.8ms/step | 15.2 samples/s +Step 2150/44000 | Loss: 3.6379 | LR: 4.99e-04 | 1314.8ms/step | 15.2 samples/s +Step 2160/44000 | Loss: 3.7118 | LR: 4.99e-04 | 1316.0ms/step | 15.2 samples/s +Step 2170/44000 | Loss: 3.6839 | LR: 4.99e-04 | 1319.7ms/step | 15.2 samples/s +Step 2180/44000 | Loss: 3.6800 | LR: 4.99e-04 | 1316.4ms/step | 15.2 samples/s +Step 2190/44000 | Loss: 3.6664 | LR: 4.99e-04 | 1317.2ms/step | 15.2 samples/s +Step 2200/44000 | Loss: 3.6191 | LR: 4.99e-04 | 1318.5ms/step | 15.2 samples/s +Step 2210/44000 | Loss: 3.6619 | LR: 4.99e-04 | 1318.2ms/step | 15.2 samples/s +Step 2220/44000 | Loss: 3.6979 | LR: 4.99e-04 | 1318.7ms/step | 15.2 samples/s +Step 2230/44000 | Loss: 3.6482 | LR: 4.99e-04 | 1318.4ms/step | 15.2 samples/s +Step 2240/44000 | Loss: 3.6421 | LR: 4.99e-04 | 1316.9ms/step | 15.2 samples/s +Step 2250/44000 | Loss: 3.6441 | LR: 4.99e-04 | 1319.3ms/step | 15.2 samples/s +Validation at step 2250: Loss = 3.6556, Perplexity = 38.69 +Step 2260/44000 | Loss: 3.6467 | LR: 4.98e-04 | 3529.2ms/step | 5.7 samples/s +Step 2270/44000 | Loss: 3.6587 | LR: 4.98e-04 | 1319.6ms/step | 15.2 samples/s +Step 2280/44000 | Loss: 3.6517 | LR: 4.98e-04 | 1319.1ms/step | 15.2 samples/s +Step 2290/44000 | Loss: 3.6665 | LR: 4.98e-04 | 1318.2ms/step | 15.2 samples/s +Step 2300/44000 | Loss: 3.6428 | LR: 4.98e-04 | 1318.1ms/step | 15.2 samples/s +Step 2310/44000 | Loss: 3.6585 | LR: 4.98e-04 | 1318.5ms/step | 15.2 samples/s +Step 2320/44000 | Loss: 3.6271 | LR: 4.98e-04 | 1317.5ms/step | 15.2 samples/s +Step 2330/44000 | Loss: 3.6314 | LR: 4.98e-04 | 1314.9ms/step | 15.2 samples/s +Step 2340/44000 | Loss: 3.6196 | LR: 4.98e-04 | 1318.7ms/step | 15.2 samples/s +Step 2350/44000 | Loss: 3.6259 | LR: 4.98e-04 | 1316.3ms/step | 15.2 samples/s +Step 2360/44000 | Loss: 3.6580 | LR: 4.98e-04 | 1315.4ms/step | 15.2 samples/s +Step 2370/44000 | Loss: 3.6389 | LR: 4.98e-04 | 1318.6ms/step | 15.2 samples/s +Step 2380/44000 | Loss: 3.6268 | LR: 4.98e-04 | 1316.1ms/step | 15.2 samples/s +Step 2390/44000 | Loss: 3.6186 | LR: 4.98e-04 | 1318.3ms/step | 15.2 samples/s +Step 2400/44000 | Loss: 3.6038 | LR: 4.98e-04 | 1320.3ms/step | 15.1 samples/s +Step 2410/44000 | Loss: 3.6019 | LR: 4.98e-04 | 1319.1ms/step | 15.2 samples/s +Step 2420/44000 | Loss: 3.6286 | LR: 4.98e-04 | 1315.3ms/step | 15.2 samples/s +Step 2430/44000 | Loss: 3.6189 | LR: 4.98e-04 | 1316.5ms/step | 15.2 samples/s +Step 2440/44000 | Loss: 3.6860 | LR: 4.98e-04 | 1318.0ms/step | 15.2 samples/s +Step 2450/44000 | Loss: 3.5976 | LR: 4.98e-04 | 1319.4ms/step | 15.2 samples/s +Step 2460/44000 | Loss: 3.6336 | LR: 4.98e-04 | 1317.8ms/step | 15.2 samples/s +Step 2470/44000 | Loss: 3.5993 | LR: 4.98e-04 | 1317.0ms/step | 15.2 samples/s +Step 2480/44000 | Loss: 3.6072 | LR: 4.98e-04 | 1318.2ms/step | 15.2 samples/s +Step 2490/44000 | Loss: 3.6120 | LR: 4.98e-04 | 1315.2ms/step | 15.2 samples/s +Step 2500/44000 | Loss: 3.5835 | LR: 4.98e-04 | 1320.8ms/step | 15.1 samples/s +Validation at step 2500: Loss = 3.6004, Perplexity = 36.61 +Step 2510/44000 | Loss: 3.6068 | LR: 4.98e-04 | 3523.4ms/step | 5.7 samples/s +Step 2520/44000 | Loss: 3.6169 | LR: 4.98e-04 | 1315.7ms/step | 15.2 samples/s +Step 2530/44000 | Loss: 3.5911 | LR: 4.98e-04 | 1313.2ms/step | 15.2 samples/s +Step 2540/44000 | Loss: 3.6065 | LR: 4.98e-04 | 1314.1ms/step | 15.2 samples/s +Step 2550/44000 | Loss: 3.5935 | LR: 4.98e-04 | 1313.0ms/step | 15.2 samples/s +Step 2560/44000 | Loss: 3.5566 | LR: 4.98e-04 | 1312.8ms/step | 15.2 samples/s +Step 2570/44000 | Loss: 3.5988 | LR: 4.98e-04 | 1316.2ms/step | 15.2 samples/s +Step 2580/44000 | Loss: 3.5794 | LR: 4.98e-04 | 1319.3ms/step | 15.2 samples/s +Step 2590/44000 | Loss: 3.5807 | LR: 4.98e-04 | 1318.5ms/step | 15.2 samples/s +Step 2600/44000 | Loss: 3.6092 | LR: 4.98e-04 | 1315.2ms/step | 15.2 samples/s +Step 2610/44000 | Loss: 3.5599 | LR: 4.98e-04 | 1314.5ms/step | 15.2 samples/s +Step 2620/44000 | Loss: 3.6015 | LR: 4.98e-04 | 1314.1ms/step | 15.2 samples/s +Step 2630/44000 | Loss: 3.5543 | LR: 4.98e-04 | 1314.8ms/step | 15.2 samples/s +Step 2640/44000 | Loss: 3.5761 | LR: 4.98e-04 | 1314.5ms/step | 15.2 samples/s +Step 2650/44000 | Loss: 3.5796 | LR: 4.98e-04 | 1314.0ms/step | 15.2 samples/s +Step 2660/44000 | Loss: 3.5866 | LR: 4.98e-04 | 1315.8ms/step | 15.2 samples/s +Step 2670/44000 | Loss: 3.5715 | LR: 4.98e-04 | 1319.8ms/step | 15.2 samples/s +Step 2680/44000 | Loss: 3.5634 | LR: 4.98e-04 | 1323.3ms/step | 15.1 samples/s +Step 2690/44000 | Loss: 3.5694 | LR: 4.98e-04 | 1314.5ms/step | 15.2 samples/s +Step 2700/44000 | Loss: 3.5634 | LR: 4.97e-04 | 1315.7ms/step | 15.2 samples/s +Step 2710/44000 | Loss: 3.5700 | LR: 4.97e-04 | 1310.6ms/step | 15.3 samples/s +Step 2720/44000 | Loss: 3.5601 | LR: 4.97e-04 | 1316.3ms/step | 15.2 samples/s +Step 2730/44000 | Loss: 3.5731 | LR: 4.97e-04 | 1316.0ms/step | 15.2 samples/s +Step 2740/44000 | Loss: 3.5645 | LR: 4.97e-04 | 1313.7ms/step | 15.2 samples/s +Step 2750/44000 | Loss: 3.5625 | LR: 4.97e-04 | 1313.0ms/step | 15.2 samples/s +Validation at step 2750: Loss = 3.5550, Perplexity = 34.99 +Step 2760/44000 | Loss: 3.5646 | LR: 4.97e-04 | 3520.1ms/step | 5.7 samples/s +Step 2770/44000 | Loss: 3.5784 | LR: 4.97e-04 | 1313.1ms/step | 15.2 samples/s +Step 2780/44000 | Loss: 3.5586 | LR: 4.97e-04 | 1311.2ms/step | 15.3 samples/s +Step 2790/44000 | Loss: 3.5148 | LR: 4.97e-04 | 1310.2ms/step | 15.3 samples/s +Step 2800/44000 | Loss: 3.5446 | LR: 4.97e-04 | 1310.5ms/step | 15.3 samples/s +Step 2810/44000 | Loss: 3.4997 | LR: 4.97e-04 | 1310.8ms/step | 15.3 samples/s +Step 2820/44000 | Loss: 3.5606 | LR: 4.97e-04 | 1311.3ms/step | 15.3 samples/s +Step 2830/44000 | Loss: 3.5404 | LR: 4.97e-04 | 1311.9ms/step | 15.2 samples/s +Step 2840/44000 | Loss: 3.5487 | LR: 4.97e-04 | 1311.0ms/step | 15.3 samples/s +Step 2850/44000 | Loss: 3.5266 | LR: 4.97e-04 | 1312.3ms/step | 15.2 samples/s +Step 2860/44000 | Loss: 3.5934 | LR: 4.97e-04 | 1313.3ms/step | 15.2 samples/s +Step 2870/44000 | Loss: 3.5412 | LR: 4.97e-04 | 1314.0ms/step | 15.2 samples/s +Step 2880/44000 | Loss: 3.5257 | LR: 4.97e-04 | 1314.4ms/step | 15.2 samples/s +Step 2890/44000 | Loss: 3.5248 | LR: 4.97e-04 | 1315.8ms/step | 15.2 samples/s +Step 2900/44000 | Loss: 3.4873 | LR: 4.97e-04 | 1314.0ms/step | 15.2 samples/s +Step 2910/44000 | Loss: 3.5275 | LR: 4.97e-04 | 1313.2ms/step | 15.2 samples/s +Step 2920/44000 | Loss: 3.5036 | LR: 4.97e-04 | 1314.9ms/step | 15.2 samples/s +Step 2930/44000 | Loss: 3.5345 | LR: 4.97e-04 | 1314.3ms/step | 15.2 samples/s +Step 2940/44000 | Loss: 3.5293 | LR: 4.97e-04 | 1314.4ms/step | 15.2 samples/s +Step 2950/44000 | Loss: 3.5125 | LR: 4.97e-04 | 1314.5ms/step | 15.2 samples/s +Step 2960/44000 | Loss: 3.5188 | LR: 4.97e-04 | 1314.7ms/step | 15.2 samples/s +Step 2970/44000 | Loss: 3.4963 | LR: 4.97e-04 | 1314.4ms/step | 15.2 samples/s +Step 2980/44000 | Loss: 3.5183 | LR: 4.97e-04 | 1314.7ms/step | 15.2 samples/s +Step 2990/44000 | Loss: 3.4977 | LR: 4.97e-04 | 1312.1ms/step | 15.2 samples/s +Step 3000/44000 | Loss: 3.5134 | LR: 4.97e-04 | 1315.8ms/step | 15.2 samples/s +Validation at step 3000: Loss = 3.5182, Perplexity = 33.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-3000 +Step 3010/44000 | Loss: 3.5379 | LR: 4.97e-04 | 3844.7ms/step | 5.2 samples/s +Step 3020/44000 | Loss: 3.4997 | LR: 4.97e-04 | 1317.6ms/step | 15.2 samples/s +Step 3030/44000 | Loss: 3.5232 | LR: 4.97e-04 | 1319.9ms/step | 15.2 samples/s +Step 3040/44000 | Loss: 3.5304 | LR: 4.97e-04 | 1319.3ms/step | 15.2 samples/s +Step 3050/44000 | Loss: 3.5033 | LR: 4.97e-04 | 1317.2ms/step | 15.2 samples/s +Step 3060/44000 | Loss: 3.5126 | LR: 4.96e-04 | 1316.7ms/step | 15.2 samples/s +Step 3070/44000 | Loss: 3.5418 | LR: 4.96e-04 | 1316.8ms/step | 15.2 samples/s +Step 3080/44000 | Loss: 3.5106 | LR: 4.96e-04 | 1323.0ms/step | 15.1 samples/s +Step 3090/44000 | Loss: 3.4738 | LR: 4.96e-04 | 1319.7ms/step | 15.2 samples/s +Step 3100/44000 | Loss: 3.4887 | LR: 4.96e-04 | 1319.1ms/step | 15.2 samples/s +Step 3110/44000 | Loss: 3.5161 | LR: 4.96e-04 | 1321.6ms/step | 15.1 samples/s +Step 3120/44000 | Loss: 3.4850 | LR: 4.96e-04 | 1319.1ms/step | 15.2 samples/s +Step 3130/44000 | Loss: 3.5059 | LR: 4.96e-04 | 1320.7ms/step | 15.1 samples/s +Step 3140/44000 | Loss: 3.4814 | LR: 4.96e-04 | 1319.1ms/step | 15.2 samples/s +Step 3150/44000 | Loss: 3.4985 | LR: 4.96e-04 | 1318.5ms/step | 15.2 samples/s +Step 3160/44000 | Loss: 3.5089 | LR: 4.96e-04 | 1321.0ms/step | 15.1 samples/s +Step 3170/44000 | Loss: 3.4850 | LR: 4.96e-04 | 1319.5ms/step | 15.2 samples/s +Step 3180/44000 | Loss: 3.5001 | LR: 4.96e-04 | 1319.3ms/step | 15.2 samples/s +Step 3190/44000 | Loss: 3.5087 | LR: 4.96e-04 | 1319.8ms/step | 15.2 samples/s +Step 3200/44000 | Loss: 3.4962 | LR: 4.96e-04 | 1322.0ms/step | 15.1 samples/s +Step 3210/44000 | Loss: 3.5083 | LR: 4.96e-04 | 1317.7ms/step | 15.2 samples/s +Step 3220/44000 | Loss: 3.4714 | LR: 4.96e-04 | 1316.9ms/step | 15.2 samples/s +Step 3230/44000 | Loss: 3.4877 | LR: 4.96e-04 | 1318.9ms/step | 15.2 samples/s +Step 3240/44000 | Loss: 3.4513 | LR: 4.96e-04 | 1318.3ms/step | 15.2 samples/s +Step 3250/44000 | Loss: 3.5147 | LR: 4.96e-04 | 1318.3ms/step | 15.2 samples/s +Validation at step 3250: Loss = 3.4905, Perplexity = 32.80 +Step 3260/44000 | Loss: 3.4853 | LR: 4.96e-04 | 3529.5ms/step | 5.7 samples/s +Step 3270/44000 | Loss: 3.4820 | LR: 4.96e-04 | 1321.3ms/step | 15.1 samples/s +Step 3280/44000 | Loss: 3.4858 | LR: 4.96e-04 | 1319.6ms/step | 15.2 samples/s +Step 3290/44000 | Loss: 3.4694 | LR: 4.96e-04 | 1315.6ms/step | 15.2 samples/s +Step 3300/44000 | Loss: 3.4941 | LR: 4.96e-04 | 1317.5ms/step | 15.2 samples/s +Step 3310/44000 | Loss: 3.4738 | LR: 4.96e-04 | 1319.9ms/step | 15.2 samples/s +Step 3320/44000 | Loss: 3.4668 | LR: 4.96e-04 | 1319.4ms/step | 15.2 samples/s +Step 3330/44000 | Loss: 3.4809 | LR: 4.96e-04 | 1318.9ms/step | 15.2 samples/s +Step 3340/44000 | Loss: 3.4664 | LR: 4.96e-04 | 1318.3ms/step | 15.2 samples/s +Step 3350/44000 | Loss: 3.4757 | LR: 4.96e-04 | 1318.9ms/step | 15.2 samples/s +Step 3360/44000 | Loss: 3.4624 | LR: 4.96e-04 | 1319.6ms/step | 15.2 samples/s +Step 3370/44000 | Loss: 3.4683 | LR: 4.95e-04 | 1319.4ms/step | 15.2 samples/s +Step 3380/44000 | Loss: 3.5096 | LR: 4.95e-04 | 1321.1ms/step | 15.1 samples/s +Step 3390/44000 | Loss: 3.5060 | LR: 4.95e-04 | 1318.8ms/step | 15.2 samples/s +Step 3400/44000 | Loss: 3.4609 | LR: 4.95e-04 | 1318.6ms/step | 15.2 samples/s +Step 3410/44000 | Loss: 3.4671 | LR: 4.95e-04 | 1321.4ms/step | 15.1 samples/s +Step 3420/44000 | Loss: 3.4780 | LR: 4.95e-04 | 1320.8ms/step | 15.1 samples/s +Step 3430/44000 | Loss: 3.4801 | LR: 4.95e-04 | 1319.3ms/step | 15.2 samples/s +Step 3440/44000 | Loss: 3.4677 | LR: 4.95e-04 | 1321.3ms/step | 15.1 samples/s +Step 3450/44000 | Loss: 3.4705 | LR: 4.95e-04 | 1319.7ms/step | 15.2 samples/s +Step 3460/44000 | Loss: 3.4297 | LR: 4.95e-04 | 1320.2ms/step | 15.1 samples/s +Step 3470/44000 | Loss: 3.4722 | LR: 4.95e-04 | 1320.3ms/step | 15.1 samples/s +Step 3480/44000 | Loss: 3.4486 | LR: 4.95e-04 | 1321.1ms/step | 15.1 samples/s +Step 3490/44000 | Loss: 3.4804 | LR: 4.95e-04 | 1322.2ms/step | 15.1 samples/s +Step 3500/44000 | Loss: 3.4669 | LR: 4.95e-04 | 1320.7ms/step | 15.1 samples/s +Validation at step 3500: Loss = 3.4585, Perplexity = 31.77 +Step 3510/44000 | Loss: 3.4648 | LR: 4.95e-04 | 3534.8ms/step | 5.7 samples/s +Step 3520/44000 | Loss: 3.4384 | LR: 4.95e-04 | 1320.7ms/step | 15.1 samples/s +Step 3530/44000 | Loss: 3.4270 | LR: 4.95e-04 | 1320.8ms/step | 15.1 samples/s +Step 3540/44000 | Loss: 3.4579 | LR: 4.95e-04 | 1318.5ms/step | 15.2 samples/s +Step 3550/44000 | Loss: 3.4608 | LR: 4.95e-04 | 1320.5ms/step | 15.1 samples/s +Step 3560/44000 | Loss: 3.4688 | LR: 4.95e-04 | 1320.4ms/step | 15.1 samples/s +Step 3570/44000 | Loss: 3.4511 | LR: 4.95e-04 | 1319.7ms/step | 15.2 samples/s +Step 3580/44000 | Loss: 3.4690 | LR: 4.95e-04 | 1320.9ms/step | 15.1 samples/s +Step 3590/44000 | Loss: 3.4482 | LR: 4.95e-04 | 1321.6ms/step | 15.1 samples/s +Step 3600/44000 | Loss: 3.4421 | LR: 4.95e-04 | 1319.2ms/step | 15.2 samples/s +Step 3610/44000 | Loss: 3.4286 | LR: 4.95e-04 | 1318.3ms/step | 15.2 samples/s +Step 3620/44000 | Loss: 3.4408 | LR: 4.95e-04 | 1319.0ms/step | 15.2 samples/s +Step 3630/44000 | Loss: 3.4254 | LR: 4.95e-04 | 1318.0ms/step | 15.2 samples/s +Step 3640/44000 | Loss: 3.4190 | LR: 4.95e-04 | 1318.7ms/step | 15.2 samples/s +Step 3650/44000 | Loss: 3.4312 | LR: 4.94e-04 | 1317.7ms/step | 15.2 samples/s +Step 3660/44000 | Loss: 3.4226 | LR: 4.94e-04 | 1318.0ms/step | 15.2 samples/s +Step 3670/44000 | Loss: 3.4599 | LR: 4.94e-04 | 1317.9ms/step | 15.2 samples/s +Step 3680/44000 | Loss: 3.4132 | LR: 4.94e-04 | 1317.7ms/step | 15.2 samples/s +Step 3690/44000 | Loss: 3.4329 | LR: 4.94e-04 | 1316.7ms/step | 15.2 samples/s +Step 3700/44000 | Loss: 3.4415 | LR: 4.94e-04 | 1316.3ms/step | 15.2 samples/s +Step 3710/44000 | Loss: 3.4152 | LR: 4.94e-04 | 1319.0ms/step | 15.2 samples/s +Step 3720/44000 | Loss: 3.4416 | LR: 4.94e-04 | 1316.2ms/step | 15.2 samples/s +Step 3730/44000 | Loss: 3.4393 | LR: 4.94e-04 | 1316.5ms/step | 15.2 samples/s +Step 3740/44000 | Loss: 3.4346 | LR: 4.94e-04 | 1318.7ms/step | 15.2 samples/s +Step 3750/44000 | Loss: 3.4326 | LR: 4.94e-04 | 1321.7ms/step | 15.1 samples/s +Validation at step 3750: Loss = 3.4311, Perplexity = 30.91 +Step 3760/44000 | Loss: 3.4264 | LR: 4.94e-04 | 3535.5ms/step | 5.7 samples/s +Step 3770/44000 | Loss: 3.4279 | LR: 4.94e-04 | 1320.8ms/step | 15.1 samples/s +Step 3780/44000 | Loss: 3.4508 | LR: 4.94e-04 | 1320.0ms/step | 15.2 samples/s +Step 3790/44000 | Loss: 3.4354 | LR: 4.94e-04 | 1319.8ms/step | 15.2 samples/s +Step 3800/44000 | Loss: 3.4431 | LR: 4.94e-04 | 1321.6ms/step | 15.1 samples/s +Step 3810/44000 | Loss: 3.4427 | LR: 4.94e-04 | 1317.3ms/step | 15.2 samples/s +Step 3820/44000 | Loss: 3.4501 | LR: 4.94e-04 | 1320.5ms/step | 15.1 samples/s +Step 3830/44000 | Loss: 3.4439 | LR: 4.94e-04 | 1322.4ms/step | 15.1 samples/s +Step 3840/44000 | Loss: 3.4120 | LR: 4.94e-04 | 1318.5ms/step | 15.2 samples/s +Step 3850/44000 | Loss: 3.4037 | LR: 4.94e-04 | 1320.1ms/step | 15.2 samples/s +Step 3860/44000 | Loss: 3.4352 | LR: 4.94e-04 | 1320.0ms/step | 15.2 samples/s +Step 3870/44000 | Loss: 3.4412 | LR: 4.94e-04 | 1317.6ms/step | 15.2 samples/s +Step 3880/44000 | Loss: 3.4001 | LR: 4.94e-04 | 1319.7ms/step | 15.2 samples/s +Step 3890/44000 | Loss: 3.4230 | LR: 4.94e-04 | 1319.8ms/step | 15.2 samples/s +Step 3900/44000 | Loss: 3.4330 | LR: 4.93e-04 | 1317.1ms/step | 15.2 samples/s +Step 3910/44000 | Loss: 3.4072 | LR: 4.93e-04 | 1316.4ms/step | 15.2 samples/s +Step 3920/44000 | Loss: 3.4095 | LR: 4.93e-04 | 1320.9ms/step | 15.1 samples/s +Step 3930/44000 | Loss: 3.4451 | LR: 4.93e-04 | 1319.3ms/step | 15.2 samples/s +Step 3940/44000 | Loss: 3.3910 | LR: 4.93e-04 | 1319.8ms/step | 15.2 samples/s +Step 3950/44000 | Loss: 3.3923 | LR: 4.93e-04 | 1316.0ms/step | 15.2 samples/s +Step 3960/44000 | Loss: 3.4456 | LR: 4.93e-04 | 1318.2ms/step | 15.2 samples/s +Step 3970/44000 | Loss: 3.3982 | LR: 4.93e-04 | 1318.1ms/step | 15.2 samples/s +Step 3980/44000 | Loss: 3.4172 | LR: 4.93e-04 | 1319.4ms/step | 15.2 samples/s +Step 3990/44000 | Loss: 3.3885 | LR: 4.93e-04 | 1319.8ms/step | 15.2 samples/s +Step 4000/44000 | Loss: 3.4264 | LR: 4.93e-04 | 1318.5ms/step | 15.2 samples/s +Validation at step 4000: Loss = 3.4073, Perplexity = 30.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-4000 +Step 4010/44000 | Loss: 3.3584 | LR: 4.93e-04 | 3847.7ms/step | 5.2 samples/s +Step 4020/44000 | Loss: 3.4150 | LR: 4.93e-04 | 1318.0ms/step | 15.2 samples/s +Step 4030/44000 | Loss: 3.3892 | LR: 4.93e-04 | 1321.9ms/step | 15.1 samples/s +Step 4040/44000 | Loss: 3.4203 | LR: 4.93e-04 | 1316.5ms/step | 15.2 samples/s +Step 4050/44000 | Loss: 3.3972 | LR: 4.93e-04 | 1317.0ms/step | 15.2 samples/s +Step 4060/44000 | Loss: 3.4402 | LR: 4.93e-04 | 1312.8ms/step | 15.2 samples/s +Step 4070/44000 | Loss: 3.4007 | LR: 4.93e-04 | 1313.4ms/step | 15.2 samples/s +Step 4080/44000 | Loss: 3.3901 | LR: 4.93e-04 | 1314.9ms/step | 15.2 samples/s +Step 4090/44000 | Loss: 3.3872 | LR: 4.93e-04 | 1315.4ms/step | 15.2 samples/s +Step 4100/44000 | Loss: 3.3968 | LR: 4.93e-04 | 1314.3ms/step | 15.2 samples/s +Step 4110/44000 | Loss: 3.4007 | LR: 4.93e-04 | 1311.9ms/step | 15.2 samples/s +Step 4120/44000 | Loss: 3.3885 | LR: 4.93e-04 | 1311.1ms/step | 15.3 samples/s +Step 4130/44000 | Loss: 3.4010 | LR: 4.93e-04 | 1314.4ms/step | 15.2 samples/s +Step 4140/44000 | Loss: 3.3866 | LR: 4.92e-04 | 1312.6ms/step | 15.2 samples/s +Step 4150/44000 | Loss: 3.3803 | LR: 4.92e-04 | 1314.8ms/step | 15.2 samples/s +Step 4160/44000 | Loss: 3.3990 | LR: 4.92e-04 | 1314.3ms/step | 15.2 samples/s +Step 4170/44000 | Loss: 3.4061 | LR: 4.92e-04 | 1316.0ms/step | 15.2 samples/s +Step 4180/44000 | Loss: 3.3882 | LR: 4.92e-04 | 1315.0ms/step | 15.2 samples/s +Step 4190/44000 | Loss: 3.3878 | LR: 4.92e-04 | 1316.3ms/step | 15.2 samples/s +Step 4200/44000 | Loss: 3.3783 | LR: 4.92e-04 | 1314.1ms/step | 15.2 samples/s +Step 4210/44000 | Loss: 3.3968 | LR: 4.92e-04 | 1313.8ms/step | 15.2 samples/s +Step 4220/44000 | Loss: 3.3834 | LR: 4.92e-04 | 1311.6ms/step | 15.2 samples/s +Step 4230/44000 | Loss: 3.3857 | LR: 4.92e-04 | 1313.0ms/step | 15.2 samples/s +Step 4240/44000 | Loss: 3.3823 | LR: 4.92e-04 | 1314.9ms/step | 15.2 samples/s +Step 4250/44000 | Loss: 3.3934 | LR: 4.92e-04 | 1312.6ms/step | 15.2 samples/s +Validation at step 4250: Loss = 3.3879, Perplexity = 29.60 +Step 4260/44000 | Loss: 3.4166 | LR: 4.92e-04 | 3521.5ms/step | 5.7 samples/s +Step 4270/44000 | Loss: 3.3983 | LR: 4.92e-04 | 1314.0ms/step | 15.2 samples/s +Step 4280/44000 | Loss: 3.4159 | LR: 4.92e-04 | 1313.1ms/step | 15.2 samples/s +Step 4290/44000 | Loss: 3.3713 | LR: 4.92e-04 | 1314.5ms/step | 15.2 samples/s +Step 4300/44000 | Loss: 3.4021 | LR: 4.92e-04 | 1313.7ms/step | 15.2 samples/s +Step 4310/44000 | Loss: 3.4127 | LR: 4.92e-04 | 1314.5ms/step | 15.2 samples/s +Step 4320/44000 | Loss: 3.3999 | LR: 4.92e-04 | 1315.0ms/step | 15.2 samples/s +Step 4330/44000 | Loss: 3.3577 | LR: 4.92e-04 | 1315.3ms/step | 15.2 samples/s +Step 4340/44000 | Loss: 3.3900 | LR: 4.92e-04 | 1314.8ms/step | 15.2 samples/s +Step 4350/44000 | Loss: 3.3990 | LR: 4.92e-04 | 1312.3ms/step | 15.2 samples/s +Step 4360/44000 | Loss: 3.3692 | LR: 4.91e-04 | 1312.1ms/step | 15.2 samples/s +Step 4370/44000 | Loss: 3.4074 | LR: 4.91e-04 | 1315.3ms/step | 15.2 samples/s +Step 4380/44000 | Loss: 3.3627 | LR: 4.91e-04 | 1312.2ms/step | 15.2 samples/s +Step 4390/44000 | Loss: 3.3903 | LR: 4.91e-04 | 1314.3ms/step | 15.2 samples/s +Step 4400/44000 | Loss: 3.3435 | LR: 4.91e-04 | 1315.7ms/step | 15.2 samples/s +Step 4410/44000 | Loss: 3.3598 | LR: 4.91e-04 | 1315.0ms/step | 15.2 samples/s +Step 4420/44000 | Loss: 3.3612 | LR: 4.91e-04 | 1312.1ms/step | 15.2 samples/s +Step 4430/44000 | Loss: 3.3825 | LR: 4.91e-04 | 1311.8ms/step | 15.2 samples/s +Step 4440/44000 | Loss: 3.4196 | LR: 4.91e-04 | 1317.5ms/step | 15.2 samples/s +Step 4450/44000 | Loss: 3.4110 | LR: 4.91e-04 | 1317.7ms/step | 15.2 samples/s +Step 4460/44000 | Loss: 3.3575 | LR: 4.91e-04 | 1312.2ms/step | 15.2 samples/s +Step 4470/44000 | Loss: 3.3539 | LR: 4.91e-04 | 1315.7ms/step | 15.2 samples/s +Step 4480/44000 | Loss: 3.3862 | LR: 4.91e-04 | 1311.2ms/step | 15.3 samples/s +Step 4490/44000 | Loss: 3.3589 | LR: 4.91e-04 | 1316.1ms/step | 15.2 samples/s +Step 4500/44000 | Loss: 3.3890 | LR: 4.91e-04 | 1315.6ms/step | 15.2 samples/s +Validation at step 4500: Loss = 3.3700, Perplexity = 29.08 +Step 4510/44000 | Loss: 3.3531 | LR: 4.91e-04 | 3513.0ms/step | 5.7 samples/s +Step 4520/44000 | Loss: 3.3634 | LR: 4.91e-04 | 1315.1ms/step | 15.2 samples/s +Step 4530/44000 | Loss: 3.3341 | LR: 4.91e-04 | 1311.7ms/step | 15.2 samples/s +Step 4540/44000 | Loss: 3.3774 | LR: 4.91e-04 | 1311.7ms/step | 15.2 samples/s +Step 4550/44000 | Loss: 3.4007 | LR: 4.91e-04 | 1312.1ms/step | 15.2 samples/s +Step 4560/44000 | Loss: 3.3528 | LR: 4.90e-04 | 1315.6ms/step | 15.2 samples/s +Step 4570/44000 | Loss: 3.3529 | LR: 4.90e-04 | 1315.4ms/step | 15.2 samples/s +Step 4580/44000 | Loss: 3.3565 | LR: 4.90e-04 | 1315.7ms/step | 15.2 samples/s +Step 4590/44000 | Loss: 3.3465 | LR: 4.90e-04 | 1313.9ms/step | 15.2 samples/s +Step 4600/44000 | Loss: 3.3669 | LR: 4.90e-04 | 1313.6ms/step | 15.2 samples/s +Step 4610/44000 | Loss: 3.3672 | LR: 4.90e-04 | 1313.7ms/step | 15.2 samples/s +Step 4620/44000 | Loss: 3.3606 | LR: 4.90e-04 | 1311.6ms/step | 15.2 samples/s +Step 4630/44000 | Loss: 3.3477 | LR: 4.90e-04 | 1314.4ms/step | 15.2 samples/s +Step 4640/44000 | Loss: 3.3638 | LR: 4.90e-04 | 1313.5ms/step | 15.2 samples/s +Step 4650/44000 | Loss: 3.3743 | LR: 4.90e-04 | 1314.7ms/step | 15.2 samples/s +Step 4660/44000 | Loss: 3.3260 | LR: 4.90e-04 | 1312.2ms/step | 15.2 samples/s +Step 4670/44000 | Loss: 3.3578 | LR: 4.90e-04 | 1313.4ms/step | 15.2 samples/s +Step 4680/44000 | Loss: 3.3291 | LR: 4.90e-04 | 1314.1ms/step | 15.2 samples/s +Step 4690/44000 | Loss: 3.3352 | LR: 4.90e-04 | 1314.0ms/step | 15.2 samples/s +Step 4700/44000 | Loss: 3.3403 | LR: 4.90e-04 | 1315.0ms/step | 15.2 samples/s +Step 4710/44000 | Loss: 3.3609 | LR: 4.90e-04 | 1312.1ms/step | 15.2 samples/s +Step 4720/44000 | Loss: 3.3618 | LR: 4.90e-04 | 1314.6ms/step | 15.2 samples/s +Step 4730/44000 | Loss: 3.3487 | LR: 4.90e-04 | 1311.6ms/step | 15.2 samples/s +Step 4740/44000 | Loss: 3.3391 | LR: 4.90e-04 | 1312.3ms/step | 15.2 samples/s +Step 4750/44000 | Loss: 3.3452 | LR: 4.90e-04 | 1311.5ms/step | 15.2 samples/s +Validation at step 4750: Loss = 3.3517, Perplexity = 28.55 +Step 4760/44000 | Loss: 3.3556 | LR: 4.89e-04 | 3516.2ms/step | 5.7 samples/s +Step 4770/44000 | Loss: 3.3725 | LR: 4.89e-04 | 1314.6ms/step | 15.2 samples/s +Step 4780/44000 | Loss: 3.3595 | LR: 4.89e-04 | 1312.6ms/step | 15.2 samples/s +Step 4790/44000 | Loss: 3.3207 | LR: 4.89e-04 | 1314.8ms/step | 15.2 samples/s +Step 4800/44000 | Loss: 3.3435 | LR: 4.89e-04 | 1312.6ms/step | 15.2 samples/s +Step 4810/44000 | Loss: 3.3201 | LR: 4.89e-04 | 1312.6ms/step | 15.2 samples/s +Step 4820/44000 | Loss: 3.3598 | LR: 4.89e-04 | 1313.2ms/step | 15.2 samples/s +Step 4830/44000 | Loss: 3.3483 | LR: 4.89e-04 | 1313.2ms/step | 15.2 samples/s +Step 4840/44000 | Loss: 3.3380 | LR: 4.89e-04 | 1312.8ms/step | 15.2 samples/s +Step 4850/44000 | Loss: 3.3584 | LR: 4.89e-04 | 1313.2ms/step | 15.2 samples/s +Step 4860/44000 | Loss: 3.3216 | LR: 4.89e-04 | 1313.3ms/step | 15.2 samples/s +Step 4870/44000 | Loss: 3.3224 | LR: 4.89e-04 | 1313.4ms/step | 15.2 samples/s +Step 4880/44000 | Loss: 3.3465 | LR: 4.89e-04 | 1313.1ms/step | 15.2 samples/s +Step 4890/44000 | Loss: 3.3548 | LR: 4.89e-04 | 1313.3ms/step | 15.2 samples/s +Step 4900/44000 | Loss: 3.3706 | LR: 4.89e-04 | 1311.0ms/step | 15.3 samples/s +Step 4910/44000 | Loss: 3.3460 | LR: 4.89e-04 | 1313.5ms/step | 15.2 samples/s +Step 4920/44000 | Loss: 3.3316 | LR: 4.89e-04 | 1310.7ms/step | 15.3 samples/s +Step 4930/44000 | Loss: 3.3035 | LR: 4.89e-04 | 1313.3ms/step | 15.2 samples/s +Step 4940/44000 | Loss: 3.3369 | LR: 4.89e-04 | 1313.0ms/step | 15.2 samples/s +Step 4950/44000 | Loss: 3.3175 | LR: 4.88e-04 | 1312.6ms/step | 15.2 samples/s +Step 4960/44000 | Loss: 3.3579 | LR: 4.88e-04 | 1314.7ms/step | 15.2 samples/s +Step 4970/44000 | Loss: 3.3377 | LR: 4.88e-04 | 1312.8ms/step | 15.2 samples/s +Step 4980/44000 | Loss: 3.3487 | LR: 4.88e-04 | 1313.3ms/step | 15.2 samples/s +Step 4990/44000 | Loss: 3.3312 | LR: 4.88e-04 | 1313.9ms/step | 15.2 samples/s +Step 5000/44000 | Loss: 3.3016 | LR: 4.88e-04 | 1312.3ms/step | 15.2 samples/s +Validation at step 5000: Loss = 3.3356, Perplexity = 28.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-5000 +Step 5010/44000 | Loss: 3.3467 | LR: 4.88e-04 | 3849.0ms/step | 5.2 samples/s +Step 5020/44000 | Loss: 3.2990 | LR: 4.88e-04 | 1317.1ms/step | 15.2 samples/s +Step 5030/44000 | Loss: 3.3274 | LR: 4.88e-04 | 1318.7ms/step | 15.2 samples/s +Step 5040/44000 | Loss: 3.3085 | LR: 4.88e-04 | 1319.7ms/step | 15.2 samples/s +Step 5050/44000 | Loss: 3.3447 | LR: 4.88e-04 | 1318.9ms/step | 15.2 samples/s +Step 5060/44000 | Loss: 3.3523 | LR: 4.88e-04 | 1320.9ms/step | 15.1 samples/s +Step 5070/44000 | Loss: 3.3339 | LR: 4.88e-04 | 1316.9ms/step | 15.2 samples/s +Step 5080/44000 | Loss: 3.3261 | LR: 4.88e-04 | 1316.6ms/step | 15.2 samples/s +Step 5090/44000 | Loss: 3.3347 | LR: 4.88e-04 | 1318.0ms/step | 15.2 samples/s +Step 5100/44000 | Loss: 3.3431 | LR: 4.88e-04 | 1319.9ms/step | 15.2 samples/s +Step 5110/44000 | Loss: 3.3391 | LR: 4.88e-04 | 1322.0ms/step | 15.1 samples/s +Step 5120/44000 | Loss: 3.3216 | LR: 4.88e-04 | 1320.9ms/step | 15.1 samples/s +Step 5130/44000 | Loss: 3.3193 | LR: 4.87e-04 | 1319.2ms/step | 15.2 samples/s +Step 5140/44000 | Loss: 3.3355 | LR: 4.87e-04 | 1324.4ms/step | 15.1 samples/s +Step 5150/44000 | Loss: 3.3307 | LR: 4.87e-04 | 1318.8ms/step | 15.2 samples/s +Step 5160/44000 | Loss: 3.3384 | LR: 4.87e-04 | 1318.1ms/step | 15.2 samples/s +Step 5170/44000 | Loss: 3.3530 | LR: 4.87e-04 | 1319.8ms/step | 15.2 samples/s +Step 5180/44000 | Loss: 3.3378 | LR: 4.87e-04 | 1319.7ms/step | 15.2 samples/s +Step 5190/44000 | Loss: 3.3074 | LR: 4.87e-04 | 1319.0ms/step | 15.2 samples/s +Step 5200/44000 | Loss: 3.2966 | LR: 4.87e-04 | 1320.6ms/step | 15.1 samples/s +Step 5210/44000 | Loss: 3.3160 | LR: 4.87e-04 | 1317.9ms/step | 15.2 samples/s +Step 5220/44000 | Loss: 3.3273 | LR: 4.87e-04 | 1320.0ms/step | 15.2 samples/s +Step 5230/44000 | Loss: 3.2980 | LR: 4.87e-04 | 1319.8ms/step | 15.2 samples/s +Step 5240/44000 | Loss: 3.3111 | LR: 4.87e-04 | 1315.5ms/step | 15.2 samples/s +Step 5250/44000 | Loss: 3.3409 | LR: 4.87e-04 | 1317.5ms/step | 15.2 samples/s +Validation at step 5250: Loss = 3.3214, Perplexity = 27.70 +Step 5260/44000 | Loss: 3.3509 | LR: 4.87e-04 | 3530.4ms/step | 5.7 samples/s +Step 5270/44000 | Loss: 3.3369 | LR: 4.87e-04 | 1320.1ms/step | 15.2 samples/s +Step 5280/44000 | Loss: 3.3031 | LR: 4.87e-04 | 1319.8ms/step | 15.2 samples/s +Step 5290/44000 | Loss: 3.3231 | LR: 4.87e-04 | 1318.9ms/step | 15.2 samples/s +Step 5300/44000 | Loss: 3.3119 | LR: 4.86e-04 | 1319.8ms/step | 15.2 samples/s +Step 5310/44000 | Loss: 3.3384 | LR: 4.86e-04 | 1322.4ms/step | 15.1 samples/s +Step 5320/44000 | Loss: 3.3169 | LR: 4.86e-04 | 1319.4ms/step | 15.2 samples/s +Step 5330/44000 | Loss: 3.3204 | LR: 4.86e-04 | 1319.5ms/step | 15.2 samples/s +Step 5340/44000 | Loss: 3.3072 | LR: 4.86e-04 | 1317.0ms/step | 15.2 samples/s +Step 5350/44000 | Loss: 3.3044 | LR: 4.86e-04 | 1320.3ms/step | 15.1 samples/s +Step 5360/44000 | Loss: 3.2940 | LR: 4.86e-04 | 1321.8ms/step | 15.1 samples/s +Step 5370/44000 | Loss: 3.2916 | LR: 4.86e-04 | 1319.2ms/step | 15.2 samples/s +Step 5380/44000 | Loss: 3.3056 | LR: 4.86e-04 | 1318.6ms/step | 15.2 samples/s +Step 5390/44000 | Loss: 3.3016 | LR: 4.86e-04 | 1317.2ms/step | 15.2 samples/s +Step 5400/44000 | Loss: 3.3266 | LR: 4.86e-04 | 1320.2ms/step | 15.1 samples/s +Step 5410/44000 | Loss: 3.3269 | LR: 4.86e-04 | 1319.7ms/step | 15.2 samples/s +Step 5420/44000 | Loss: 3.3067 | LR: 4.86e-04 | 1317.1ms/step | 15.2 samples/s +Step 5430/44000 | Loss: 3.3002 | LR: 4.86e-04 | 1318.9ms/step | 15.2 samples/s +Step 5440/44000 | Loss: 3.3056 | LR: 4.86e-04 | 1319.8ms/step | 15.2 samples/s +Step 5450/44000 | Loss: 3.2993 | LR: 4.86e-04 | 1317.2ms/step | 15.2 samples/s +Step 5460/44000 | Loss: 3.2767 | LR: 4.86e-04 | 1319.9ms/step | 15.2 samples/s +Step 5470/44000 | Loss: 3.2788 | LR: 4.85e-04 | 1319.4ms/step | 15.2 samples/s +Step 5480/44000 | Loss: 3.3000 | LR: 4.85e-04 | 1320.4ms/step | 15.1 samples/s +Step 5490/44000 | Loss: 3.3058 | LR: 4.85e-04 | 1319.6ms/step | 15.2 samples/s +Step 5500/44000 | Loss: 3.3193 | LR: 4.85e-04 | 1320.8ms/step | 15.1 samples/s +Validation at step 5500: Loss = 3.3095, Perplexity = 27.37 +Step 5510/44000 | Loss: 3.3223 | LR: 4.85e-04 | 3529.3ms/step | 5.7 samples/s +Step 5520/44000 | Loss: 3.2870 | LR: 4.85e-04 | 1321.4ms/step | 15.1 samples/s +Step 5530/44000 | Loss: 3.3421 | LR: 4.85e-04 | 1318.8ms/step | 15.2 samples/s +Step 5540/44000 | Loss: 3.2939 | LR: 4.85e-04 | 1318.7ms/step | 15.2 samples/s +Step 5550/44000 | Loss: 3.3009 | LR: 4.85e-04 | 1318.2ms/step | 15.2 samples/s +Step 5560/44000 | Loss: 3.2779 | LR: 4.85e-04 | 1316.0ms/step | 15.2 samples/s +Step 5570/44000 | Loss: 3.3045 | LR: 4.85e-04 | 1318.6ms/step | 15.2 samples/s +Step 5580/44000 | Loss: 3.3010 | LR: 4.85e-04 | 1319.6ms/step | 15.2 samples/s +Step 5590/44000 | Loss: 3.2760 | LR: 4.85e-04 | 1320.3ms/step | 15.1 samples/s +Step 5600/44000 | Loss: 3.3280 | LR: 4.85e-04 | 1320.3ms/step | 15.1 samples/s +Step 5610/44000 | Loss: 3.2827 | LR: 4.85e-04 | 1319.1ms/step | 15.2 samples/s +Step 5620/44000 | Loss: 3.3122 | LR: 4.85e-04 | 1316.0ms/step | 15.2 samples/s +Step 5630/44000 | Loss: 3.2680 | LR: 4.84e-04 | 1314.8ms/step | 15.2 samples/s +Step 5640/44000 | Loss: 3.3052 | LR: 4.84e-04 | 1314.4ms/step | 15.2 samples/s +Step 5650/44000 | Loss: 3.2862 | LR: 4.84e-04 | 1314.8ms/step | 15.2 samples/s +Step 5660/44000 | Loss: 3.2825 | LR: 4.84e-04 | 1316.8ms/step | 15.2 samples/s +Step 5670/44000 | Loss: 3.2927 | LR: 4.84e-04 | 1316.2ms/step | 15.2 samples/s +Step 5680/44000 | Loss: 3.3213 | LR: 4.84e-04 | 1316.4ms/step | 15.2 samples/s +Step 5690/44000 | Loss: 3.2881 | LR: 4.84e-04 | 1318.7ms/step | 15.2 samples/s +Step 5700/44000 | Loss: 3.2989 | LR: 4.84e-04 | 1318.6ms/step | 15.2 samples/s +Step 5710/44000 | Loss: 3.3009 | LR: 4.84e-04 | 1316.4ms/step | 15.2 samples/s +Step 5720/44000 | Loss: 3.3036 | LR: 4.84e-04 | 1317.3ms/step | 15.2 samples/s +Step 5730/44000 | Loss: 3.3214 | LR: 4.84e-04 | 1319.2ms/step | 15.2 samples/s +Step 5740/44000 | Loss: 3.2994 | LR: 4.84e-04 | 1321.5ms/step | 15.1 samples/s +Step 5750/44000 | Loss: 3.2814 | LR: 4.84e-04 | 1318.4ms/step | 15.2 samples/s +Validation at step 5750: Loss = 3.2951, Perplexity = 26.98 +Step 5760/44000 | Loss: 3.2977 | LR: 4.84e-04 | 3534.2ms/step | 5.7 samples/s +Step 5770/44000 | Loss: 3.3053 | LR: 4.84e-04 | 1318.6ms/step | 15.2 samples/s +Step 5780/44000 | Loss: 3.3226 | LR: 4.83e-04 | 1317.9ms/step | 15.2 samples/s +Step 5790/44000 | Loss: 3.2803 | LR: 4.83e-04 | 1319.5ms/step | 15.2 samples/s +Step 5800/44000 | Loss: 3.2980 | LR: 4.83e-04 | 1320.7ms/step | 15.1 samples/s +Step 5810/44000 | Loss: 3.2770 | LR: 4.83e-04 | 1320.9ms/step | 15.1 samples/s +Step 5820/44000 | Loss: 3.2591 | LR: 4.83e-04 | 1318.6ms/step | 15.2 samples/s +Step 5830/44000 | Loss: 3.2668 | LR: 4.83e-04 | 1320.4ms/step | 15.1 samples/s +Step 5840/44000 | Loss: 3.2679 | LR: 4.83e-04 | 1320.1ms/step | 15.1 samples/s +Step 5850/44000 | Loss: 3.2838 | LR: 4.83e-04 | 1317.5ms/step | 15.2 samples/s +Step 5860/44000 | Loss: 3.3016 | LR: 4.83e-04 | 1318.3ms/step | 15.2 samples/s +Step 5870/44000 | Loss: 3.2836 | LR: 4.83e-04 | 1317.0ms/step | 15.2 samples/s +Step 5880/44000 | Loss: 3.2727 | LR: 4.83e-04 | 1321.0ms/step | 15.1 samples/s +Step 5890/44000 | Loss: 3.2968 | LR: 4.83e-04 | 1319.0ms/step | 15.2 samples/s +Step 5900/44000 | Loss: 3.2829 | LR: 4.83e-04 | 1318.3ms/step | 15.2 samples/s +Step 5910/44000 | Loss: 3.2773 | LR: 4.83e-04 | 1320.2ms/step | 15.1 samples/s +Step 5920/44000 | Loss: 3.3217 | LR: 4.83e-04 | 1319.4ms/step | 15.2 samples/s +Step 5930/44000 | Loss: 3.2819 | LR: 4.83e-04 | 1323.5ms/step | 15.1 samples/s +Step 5940/44000 | Loss: 3.2705 | LR: 4.82e-04 | 1319.4ms/step | 15.2 samples/s +Step 5950/44000 | Loss: 3.2905 | LR: 4.82e-04 | 1317.5ms/step | 15.2 samples/s +Step 5960/44000 | Loss: 3.2670 | LR: 4.82e-04 | 1319.3ms/step | 15.2 samples/s +Step 5970/44000 | Loss: 3.2875 | LR: 4.82e-04 | 1318.1ms/step | 15.2 samples/s +Step 5980/44000 | Loss: 3.2802 | LR: 4.82e-04 | 1320.5ms/step | 15.1 samples/s +Step 5990/44000 | Loss: 3.2804 | LR: 4.82e-04 | 1319.5ms/step | 15.2 samples/s +Step 6000/44000 | Loss: 3.2613 | LR: 4.82e-04 | 1319.3ms/step | 15.2 samples/s +Validation at step 6000: Loss = 3.2833, Perplexity = 26.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-6000 +Step 6010/44000 | Loss: 3.2535 | LR: 4.82e-04 | 3856.9ms/step | 5.2 samples/s +Step 6020/44000 | Loss: 3.2922 | LR: 4.82e-04 | 1318.8ms/step | 15.2 samples/s +Step 6030/44000 | Loss: 3.2748 | LR: 4.82e-04 | 1317.8ms/step | 15.2 samples/s +Step 6040/44000 | Loss: 3.2903 | LR: 4.82e-04 | 1319.7ms/step | 15.2 samples/s +Step 6050/44000 | Loss: 3.2844 | LR: 4.82e-04 | 1316.5ms/step | 15.2 samples/s +Step 6060/44000 | Loss: 3.2949 | LR: 4.82e-04 | 1321.2ms/step | 15.1 samples/s +Step 6070/44000 | Loss: 3.2914 | LR: 4.82e-04 | 1320.8ms/step | 15.1 samples/s +Step 6080/44000 | Loss: 3.2894 | LR: 4.81e-04 | 1322.1ms/step | 15.1 samples/s +Step 6090/44000 | Loss: 3.2967 | LR: 4.81e-04 | 1323.4ms/step | 15.1 samples/s +Step 6100/44000 | Loss: 3.2860 | LR: 4.81e-04 | 1319.5ms/step | 15.2 samples/s +Step 6110/44000 | Loss: 3.2852 | LR: 4.81e-04 | 1320.6ms/step | 15.1 samples/s +Step 6120/44000 | Loss: 3.2852 | LR: 4.81e-04 | 1318.3ms/step | 15.2 samples/s +Step 6130/44000 | Loss: 3.3163 | LR: 4.81e-04 | 1321.5ms/step | 15.1 samples/s +Step 6140/44000 | Loss: 3.2881 | LR: 4.81e-04 | 1320.3ms/step | 15.1 samples/s +Step 6150/44000 | Loss: 3.2568 | LR: 4.81e-04 | 1320.4ms/step | 15.1 samples/s +Step 6160/44000 | Loss: 3.2662 | LR: 4.81e-04 | 1318.2ms/step | 15.2 samples/s +Step 6170/44000 | Loss: 3.2419 | LR: 4.81e-04 | 1320.1ms/step | 15.2 samples/s +Step 6180/44000 | Loss: 3.2763 | LR: 4.81e-04 | 1320.0ms/step | 15.2 samples/s +Step 6190/44000 | Loss: 3.2710 | LR: 4.81e-04 | 1319.0ms/step | 15.2 samples/s +Step 6200/44000 | Loss: 3.2989 | LR: 4.81e-04 | 1318.5ms/step | 15.2 samples/s +Step 6210/44000 | Loss: 3.2841 | LR: 4.81e-04 | 1320.6ms/step | 15.1 samples/s +Step 6220/44000 | Loss: 3.2713 | LR: 4.81e-04 | 1317.5ms/step | 15.2 samples/s +Step 6230/44000 | Loss: 3.2451 | LR: 4.80e-04 | 1321.1ms/step | 15.1 samples/s +Step 6240/44000 | Loss: 3.2783 | LR: 4.80e-04 | 1318.7ms/step | 15.2 samples/s +Step 6250/44000 | Loss: 3.2376 | LR: 4.80e-04 | 1319.2ms/step | 15.2 samples/s +Validation at step 6250: Loss = 3.2735, Perplexity = 26.40 +Step 6260/44000 | Loss: 3.2619 | LR: 4.80e-04 | 3528.1ms/step | 5.7 samples/s +Step 6270/44000 | Loss: 3.2542 | LR: 4.80e-04 | 1321.0ms/step | 15.1 samples/s +Step 6280/44000 | Loss: 3.2471 | LR: 4.80e-04 | 1320.4ms/step | 15.1 samples/s +Step 6290/44000 | Loss: 3.2582 | LR: 4.80e-04 | 1316.2ms/step | 15.2 samples/s +Step 6300/44000 | Loss: 3.2627 | LR: 4.80e-04 | 1320.3ms/step | 15.1 samples/s +Step 6310/44000 | Loss: 3.2743 | LR: 4.80e-04 | 1320.9ms/step | 15.1 samples/s +Step 6320/44000 | Loss: 3.3249 | LR: 4.80e-04 | 1318.7ms/step | 15.2 samples/s +Step 6330/44000 | Loss: 3.2828 | LR: 4.80e-04 | 1316.2ms/step | 15.2 samples/s +Step 6340/44000 | Loss: 3.2605 | LR: 4.80e-04 | 1321.4ms/step | 15.1 samples/s +Step 6350/44000 | Loss: 3.2740 | LR: 4.80e-04 | 1318.7ms/step | 15.2 samples/s +Step 6360/44000 | Loss: 3.2390 | LR: 4.80e-04 | 1319.4ms/step | 15.2 samples/s +Step 6370/44000 | Loss: 3.2610 | LR: 4.79e-04 | 1320.3ms/step | 15.1 samples/s +Step 6380/44000 | Loss: 3.2544 | LR: 4.79e-04 | 1320.1ms/step | 15.2 samples/s +Step 6390/44000 | Loss: 3.2827 | LR: 4.79e-04 | 1321.2ms/step | 15.1 samples/s +Step 6400/44000 | Loss: 3.2665 | LR: 4.79e-04 | 1317.9ms/step | 15.2 samples/s +Step 6410/44000 | Loss: 3.2582 | LR: 4.79e-04 | 1319.3ms/step | 15.2 samples/s +Step 6420/44000 | Loss: 3.2709 | LR: 4.79e-04 | 1325.9ms/step | 15.1 samples/s +Step 6430/44000 | Loss: 3.2497 | LR: 4.79e-04 | 1317.8ms/step | 15.2 samples/s +Step 6440/44000 | Loss: 3.2393 | LR: 4.79e-04 | 1318.3ms/step | 15.2 samples/s +Step 6450/44000 | Loss: 3.2837 | LR: 4.79e-04 | 1320.8ms/step | 15.1 samples/s +Step 6460/44000 | Loss: 3.2928 | LR: 4.79e-04 | 1319.5ms/step | 15.2 samples/s +Step 6470/44000 | Loss: 3.2115 | LR: 4.79e-04 | 1318.7ms/step | 15.2 samples/s +Step 6480/44000 | Loss: 3.2804 | LR: 4.79e-04 | 1318.8ms/step | 15.2 samples/s +Step 6490/44000 | Loss: 3.2391 | LR: 4.79e-04 | 1318.4ms/step | 15.2 samples/s +Step 6500/44000 | Loss: 3.2663 | LR: 4.79e-04 | 1317.6ms/step | 15.2 samples/s +Validation at step 6500: Loss = 3.2616, Perplexity = 26.09 +Step 6510/44000 | Loss: 3.2913 | LR: 4.78e-04 | 3525.2ms/step | 5.7 samples/s +Step 6520/44000 | Loss: 3.2715 | LR: 4.78e-04 | 1320.8ms/step | 15.1 samples/s +Step 6530/44000 | Loss: 3.2820 | LR: 4.78e-04 | 1319.8ms/step | 15.2 samples/s +Step 6540/44000 | Loss: 3.2514 | LR: 4.78e-04 | 1319.2ms/step | 15.2 samples/s +Step 6550/44000 | Loss: 3.2356 | LR: 4.78e-04 | 1321.2ms/step | 15.1 samples/s +Step 6560/44000 | Loss: 3.2669 | LR: 4.78e-04 | 1320.1ms/step | 15.2 samples/s +Step 6570/44000 | Loss: 3.2656 | LR: 4.78e-04 | 1319.4ms/step | 15.2 samples/s +Step 6580/44000 | Loss: 3.2714 | LR: 4.78e-04 | 1319.4ms/step | 15.2 samples/s +Step 6590/44000 | Loss: 3.2619 | LR: 4.78e-04 | 1320.6ms/step | 15.1 samples/s +Step 6600/44000 | Loss: 3.2850 | LR: 4.78e-04 | 1317.8ms/step | 15.2 samples/s +Step 6610/44000 | Loss: 3.2460 | LR: 4.78e-04 | 1319.8ms/step | 15.2 samples/s +Step 6620/44000 | Loss: 3.2767 | LR: 4.78e-04 | 1318.9ms/step | 15.2 samples/s +Step 6630/44000 | Loss: 3.2362 | LR: 4.78e-04 | 1320.9ms/step | 15.1 samples/s +Step 6640/44000 | Loss: 3.2801 | LR: 4.77e-04 | 1320.0ms/step | 15.2 samples/s +Step 6650/44000 | Loss: 3.2444 | LR: 4.77e-04 | 1315.6ms/step | 15.2 samples/s +Step 6660/44000 | Loss: 3.2422 | LR: 4.77e-04 | 1317.0ms/step | 15.2 samples/s +Step 6670/44000 | Loss: 3.2659 | LR: 4.77e-04 | 1318.4ms/step | 15.2 samples/s +Step 6680/44000 | Loss: 3.2201 | LR: 4.77e-04 | 1320.0ms/step | 15.2 samples/s +Step 6690/44000 | Loss: 3.2720 | LR: 4.77e-04 | 1320.9ms/step | 15.1 samples/s +Step 6700/44000 | Loss: 3.2626 | LR: 4.77e-04 | 1318.8ms/step | 15.2 samples/s +Step 6710/44000 | Loss: 3.2443 | LR: 4.77e-04 | 1319.6ms/step | 15.2 samples/s +Step 6720/44000 | Loss: 3.2098 | LR: 4.77e-04 | 1318.4ms/step | 15.2 samples/s +Step 6730/44000 | Loss: 3.2269 | LR: 4.77e-04 | 1317.2ms/step | 15.2 samples/s +Step 6740/44000 | Loss: 3.2402 | LR: 4.77e-04 | 1319.0ms/step | 15.2 samples/s +Step 6750/44000 | Loss: 3.2397 | LR: 4.77e-04 | 1316.9ms/step | 15.2 samples/s +Validation at step 6750: Loss = 3.2517, Perplexity = 25.83 +Step 6760/44000 | Loss: 3.2655 | LR: 4.77e-04 | 3531.9ms/step | 5.7 samples/s +Step 6770/44000 | Loss: 3.2416 | LR: 4.76e-04 | 1315.7ms/step | 15.2 samples/s +Step 6780/44000 | Loss: 3.2462 | LR: 4.76e-04 | 1315.9ms/step | 15.2 samples/s +Step 6790/44000 | Loss: 3.2396 | LR: 4.76e-04 | 1317.5ms/step | 15.2 samples/s +Step 6800/44000 | Loss: 3.2838 | LR: 4.76e-04 | 1319.1ms/step | 15.2 samples/s +Step 6810/44000 | Loss: 3.2003 | LR: 4.76e-04 | 1318.6ms/step | 15.2 samples/s +Step 6820/44000 | Loss: 3.2550 | LR: 4.76e-04 | 1319.2ms/step | 15.2 samples/s +Step 6830/44000 | Loss: 3.2514 | LR: 4.76e-04 | 1320.0ms/step | 15.2 samples/s +Step 6840/44000 | Loss: 3.2551 | LR: 4.76e-04 | 1319.8ms/step | 15.2 samples/s +Step 6850/44000 | Loss: 3.2295 | LR: 4.76e-04 | 1319.8ms/step | 15.2 samples/s +Step 6860/44000 | Loss: 3.2182 | LR: 4.76e-04 | 1320.2ms/step | 15.1 samples/s +Step 6870/44000 | Loss: 3.2664 | LR: 4.76e-04 | 1318.0ms/step | 15.2 samples/s +Step 6880/44000 | Loss: 3.2681 | LR: 4.76e-04 | 1318.6ms/step | 15.2 samples/s +Step 6890/44000 | Loss: 3.2367 | LR: 4.76e-04 | 1319.2ms/step | 15.2 samples/s +Step 6900/44000 | Loss: 3.2530 | LR: 4.75e-04 | 1317.9ms/step | 15.2 samples/s +Step 6910/44000 | Loss: 3.2378 | LR: 4.75e-04 | 1318.8ms/step | 15.2 samples/s +Step 6920/44000 | Loss: 3.2755 | LR: 4.75e-04 | 1317.2ms/step | 15.2 samples/s +Step 6930/44000 | Loss: 3.2383 | LR: 4.75e-04 | 1320.7ms/step | 15.1 samples/s +Step 6940/44000 | Loss: 3.2415 | LR: 4.75e-04 | 1320.2ms/step | 15.1 samples/s +Step 6950/44000 | Loss: 3.2378 | LR: 4.75e-04 | 1322.7ms/step | 15.1 samples/s +Step 6960/44000 | Loss: 3.2385 | LR: 4.75e-04 | 1319.4ms/step | 15.2 samples/s +Step 6970/44000 | Loss: 3.2360 | LR: 4.75e-04 | 1316.6ms/step | 15.2 samples/s +Step 6980/44000 | Loss: 3.2210 | LR: 4.75e-04 | 1317.9ms/step | 15.2 samples/s +Step 6990/44000 | Loss: 3.2291 | LR: 4.75e-04 | 1318.5ms/step | 15.2 samples/s +Step 7000/44000 | Loss: 3.2544 | LR: 4.75e-04 | 1318.9ms/step | 15.2 samples/s +Validation at step 7000: Loss = 3.2422, Perplexity = 25.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-7000 +Step 7010/44000 | Loss: 3.2606 | LR: 4.75e-04 | 3847.0ms/step | 5.2 samples/s +Step 7020/44000 | Loss: 3.2334 | LR: 4.75e-04 | 1322.1ms/step | 15.1 samples/s +Step 7030/44000 | Loss: 3.2223 | LR: 4.74e-04 | 1320.6ms/step | 15.1 samples/s +Step 7040/44000 | Loss: 3.2195 | LR: 4.74e-04 | 1318.9ms/step | 15.2 samples/s +Step 7050/44000 | Loss: 3.2239 | LR: 4.74e-04 | 1319.9ms/step | 15.2 samples/s +Step 7060/44000 | Loss: 3.2471 | LR: 4.74e-04 | 1318.2ms/step | 15.2 samples/s +Step 7070/44000 | Loss: 3.2365 | LR: 4.74e-04 | 1319.6ms/step | 15.2 samples/s +Step 7080/44000 | Loss: 3.2245 | LR: 4.74e-04 | 1315.1ms/step | 15.2 samples/s +Step 7090/44000 | Loss: 3.2388 | LR: 4.74e-04 | 1318.5ms/step | 15.2 samples/s +Step 7100/44000 | Loss: 3.2238 | LR: 4.74e-04 | 1320.1ms/step | 15.2 samples/s +Step 7110/44000 | Loss: 3.2517 | LR: 4.74e-04 | 1319.2ms/step | 15.2 samples/s +Step 7120/44000 | Loss: 3.2294 | LR: 4.74e-04 | 1319.6ms/step | 15.2 samples/s +Step 7130/44000 | Loss: 3.2577 | LR: 4.74e-04 | 1318.4ms/step | 15.2 samples/s +Step 7140/44000 | Loss: 3.2266 | LR: 4.74e-04 | 1319.8ms/step | 15.2 samples/s +Step 7150/44000 | Loss: 3.2534 | LR: 4.73e-04 | 1322.0ms/step | 15.1 samples/s +Step 7160/44000 | Loss: 3.2434 | LR: 4.73e-04 | 1320.8ms/step | 15.1 samples/s +Step 7170/44000 | Loss: 3.2290 | LR: 4.73e-04 | 1317.9ms/step | 15.2 samples/s +Step 7180/44000 | Loss: 3.2575 | LR: 4.73e-04 | 1319.1ms/step | 15.2 samples/s +Step 7190/44000 | Loss: 3.2187 | LR: 4.73e-04 | 1319.7ms/step | 15.2 samples/s +Step 7200/44000 | Loss: 3.2363 | LR: 4.73e-04 | 1320.0ms/step | 15.2 samples/s +Step 7210/44000 | Loss: 3.1811 | LR: 4.73e-04 | 1317.2ms/step | 15.2 samples/s +Step 7220/44000 | Loss: 3.2258 | LR: 4.73e-04 | 1318.0ms/step | 15.2 samples/s +Step 7230/44000 | Loss: 3.2231 | LR: 4.73e-04 | 1321.4ms/step | 15.1 samples/s +Step 7240/44000 | Loss: 3.2158 | LR: 4.73e-04 | 1319.5ms/step | 15.2 samples/s +Step 7250/44000 | Loss: 3.2092 | LR: 4.73e-04 | 1320.6ms/step | 15.1 samples/s +Validation at step 7250: Loss = 3.2356, Perplexity = 25.42 +Step 7260/44000 | Loss: 3.2185 | LR: 4.73e-04 | 3535.7ms/step | 5.7 samples/s +Step 7270/44000 | Loss: 3.2473 | LR: 4.72e-04 | 1317.1ms/step | 15.2 samples/s +Step 7280/44000 | Loss: 3.2893 | LR: 4.72e-04 | 1320.4ms/step | 15.1 samples/s +Step 7290/44000 | Loss: 3.2115 | LR: 4.72e-04 | 1318.8ms/step | 15.2 samples/s +Step 7300/44000 | Loss: 3.2365 | LR: 4.72e-04 | 1320.1ms/step | 15.2 samples/s +Step 7310/44000 | Loss: 3.2171 | LR: 4.72e-04 | 1319.9ms/step | 15.2 samples/s +Step 7320/44000 | Loss: 3.2247 | LR: 4.72e-04 | 1317.7ms/step | 15.2 samples/s +Step 7330/44000 | Loss: 3.2399 | LR: 4.72e-04 | 1316.6ms/step | 15.2 samples/s +Step 7340/44000 | Loss: 3.2198 | LR: 4.72e-04 | 1319.9ms/step | 15.2 samples/s +Step 7350/44000 | Loss: 3.2364 | LR: 4.72e-04 | 1318.9ms/step | 15.2 samples/s +Step 7360/44000 | Loss: 3.2197 | LR: 4.72e-04 | 1318.0ms/step | 15.2 samples/s +Step 7370/44000 | Loss: 3.2285 | LR: 4.72e-04 | 1319.5ms/step | 15.2 samples/s +Step 7380/44000 | Loss: 3.2170 | LR: 4.72e-04 | 1318.7ms/step | 15.2 samples/s +Step 7390/44000 | Loss: 3.1937 | LR: 4.71e-04 | 1320.1ms/step | 15.2 samples/s +Step 7400/44000 | Loss: 3.2155 | LR: 4.71e-04 | 1320.2ms/step | 15.1 samples/s +Step 7410/44000 | Loss: 3.2236 | LR: 4.71e-04 | 1319.7ms/step | 15.2 samples/s +Step 7420/44000 | Loss: 3.2377 | LR: 4.71e-04 | 1317.0ms/step | 15.2 samples/s +Step 7430/44000 | Loss: 3.2393 | LR: 4.71e-04 | 1324.6ms/step | 15.1 samples/s +Step 7440/44000 | Loss: 3.2347 | LR: 4.71e-04 | 1318.4ms/step | 15.2 samples/s +Step 7450/44000 | Loss: 3.2015 | LR: 4.71e-04 | 1318.9ms/step | 15.2 samples/s +Step 7460/44000 | Loss: 3.2117 | LR: 4.71e-04 | 1316.7ms/step | 15.2 samples/s +Step 7470/44000 | Loss: 3.2473 | LR: 4.71e-04 | 1317.7ms/step | 15.2 samples/s +Step 7480/44000 | Loss: 3.2221 | LR: 4.71e-04 | 1319.8ms/step | 15.2 samples/s +Step 7490/44000 | Loss: 3.2345 | LR: 4.71e-04 | 1317.4ms/step | 15.2 samples/s +Step 7500/44000 | Loss: 3.2501 | LR: 4.71e-04 | 1317.9ms/step | 15.2 samples/s +Validation at step 7500: Loss = 3.2252, Perplexity = 25.16 +Step 7510/44000 | Loss: 3.2371 | LR: 4.70e-04 | 3534.0ms/step | 5.7 samples/s +Step 7520/44000 | Loss: 3.2173 | LR: 4.70e-04 | 1319.9ms/step | 15.2 samples/s +Step 7530/44000 | Loss: 3.2495 | LR: 4.70e-04 | 1316.7ms/step | 15.2 samples/s +Step 7540/44000 | Loss: 3.2013 | LR: 4.70e-04 | 1318.9ms/step | 15.2 samples/s +Step 7550/44000 | Loss: 3.1973 | LR: 4.70e-04 | 1318.1ms/step | 15.2 samples/s +Step 7560/44000 | Loss: 3.2028 | LR: 4.70e-04 | 1320.6ms/step | 15.1 samples/s +Step 7570/44000 | Loss: 3.2076 | LR: 4.70e-04 | 1319.4ms/step | 15.2 samples/s +Step 7580/44000 | Loss: 3.2345 | LR: 4.70e-04 | 1319.4ms/step | 15.2 samples/s +Step 7590/44000 | Loss: 3.2570 | LR: 4.70e-04 | 1318.4ms/step | 15.2 samples/s +Step 7600/44000 | Loss: 3.2037 | LR: 4.70e-04 | 1318.2ms/step | 15.2 samples/s +Step 7610/44000 | Loss: 3.1917 | LR: 4.70e-04 | 1317.7ms/step | 15.2 samples/s +Step 7620/44000 | Loss: 3.2002 | LR: 4.70e-04 | 1319.1ms/step | 15.2 samples/s +Step 7630/44000 | Loss: 3.2164 | LR: 4.69e-04 | 1319.6ms/step | 15.2 samples/s +Step 7640/44000 | Loss: 3.2379 | LR: 4.69e-04 | 1318.9ms/step | 15.2 samples/s +Step 7650/44000 | Loss: 3.2266 | LR: 4.69e-04 | 1319.3ms/step | 15.2 samples/s +Step 7660/44000 | Loss: 3.1843 | LR: 4.69e-04 | 1318.0ms/step | 15.2 samples/s +Step 7670/44000 | Loss: 3.2253 | LR: 4.69e-04 | 1319.0ms/step | 15.2 samples/s +Step 7680/44000 | Loss: 3.2412 | LR: 4.69e-04 | 1317.4ms/step | 15.2 samples/s +Step 7690/44000 | Loss: 3.2150 | LR: 4.69e-04 | 1318.8ms/step | 15.2 samples/s +Step 7700/44000 | Loss: 3.2059 | LR: 4.69e-04 | 1317.9ms/step | 15.2 samples/s +Step 7710/44000 | Loss: 3.2222 | LR: 4.69e-04 | 1317.7ms/step | 15.2 samples/s +Step 7720/44000 | Loss: 3.2227 | LR: 4.69e-04 | 1318.5ms/step | 15.2 samples/s +Step 7730/44000 | Loss: 3.1896 | LR: 4.69e-04 | 1316.5ms/step | 15.2 samples/s +Step 7740/44000 | Loss: 3.2382 | LR: 4.68e-04 | 1317.5ms/step | 15.2 samples/s +Step 7750/44000 | Loss: 3.2461 | LR: 4.68e-04 | 1318.9ms/step | 15.2 samples/s +Validation at step 7750: Loss = 3.2169, Perplexity = 24.95 +Step 7760/44000 | Loss: 3.2128 | LR: 4.68e-04 | 3529.7ms/step | 5.7 samples/s +Step 7770/44000 | Loss: 3.2170 | LR: 4.68e-04 | 1319.8ms/step | 15.2 samples/s +Step 7780/44000 | Loss: 3.2403 | LR: 4.68e-04 | 1319.4ms/step | 15.2 samples/s +Step 7790/44000 | Loss: 3.1848 | LR: 4.68e-04 | 1319.7ms/step | 15.2 samples/s +Step 7800/44000 | Loss: 3.2363 | LR: 4.68e-04 | 1321.6ms/step | 15.1 samples/s +Step 7810/44000 | Loss: 3.2098 | LR: 4.68e-04 | 1320.8ms/step | 15.1 samples/s +Step 7820/44000 | Loss: 3.2515 | LR: 4.68e-04 | 1324.6ms/step | 15.1 samples/s +Step 7830/44000 | Loss: 3.2456 | LR: 4.68e-04 | 1321.2ms/step | 15.1 samples/s +Step 7840/44000 | Loss: 3.2134 | LR: 4.68e-04 | 1317.8ms/step | 15.2 samples/s +Step 7850/44000 | Loss: 3.2170 | LR: 4.67e-04 | 1322.3ms/step | 15.1 samples/s +Step 7860/44000 | Loss: 3.1877 | LR: 4.67e-04 | 1318.0ms/step | 15.2 samples/s +Step 7870/44000 | Loss: 3.2186 | LR: 4.67e-04 | 1318.3ms/step | 15.2 samples/s +Step 7880/44000 | Loss: 3.2159 | LR: 4.67e-04 | 1318.7ms/step | 15.2 samples/s +Step 7890/44000 | Loss: 3.2276 | LR: 4.67e-04 | 1322.4ms/step | 15.1 samples/s +Step 7900/44000 | Loss: 3.2244 | LR: 4.67e-04 | 1320.1ms/step | 15.2 samples/s +Step 7910/44000 | Loss: 3.2175 | LR: 4.67e-04 | 1318.6ms/step | 15.2 samples/s +Step 7920/44000 | Loss: 3.2066 | LR: 4.67e-04 | 1319.6ms/step | 15.2 samples/s +Step 7930/44000 | Loss: 3.1917 | LR: 4.67e-04 | 1320.8ms/step | 15.1 samples/s +Step 7940/44000 | Loss: 3.2410 | LR: 4.67e-04 | 1321.2ms/step | 15.1 samples/s +Step 7950/44000 | Loss: 3.2081 | LR: 4.67e-04 | 1320.7ms/step | 15.1 samples/s +Step 7960/44000 | Loss: 3.2121 | LR: 4.66e-04 | 1317.6ms/step | 15.2 samples/s +Step 7970/44000 | Loss: 3.2069 | LR: 4.66e-04 | 1320.2ms/step | 15.1 samples/s +Step 7980/44000 | Loss: 3.1881 | LR: 4.66e-04 | 1321.2ms/step | 15.1 samples/s +Step 7990/44000 | Loss: 3.2018 | LR: 4.66e-04 | 1322.1ms/step | 15.1 samples/s +Step 8000/44000 | Loss: 3.2526 | LR: 4.66e-04 | 1316.7ms/step | 15.2 samples/s +Validation at step 8000: Loss = 3.2092, Perplexity = 24.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-8000 +Step 8010/44000 | Loss: 3.1945 | LR: 4.66e-04 | 3860.3ms/step | 5.2 samples/s +Step 8020/44000 | Loss: 3.2096 | LR: 4.66e-04 | 1317.0ms/step | 15.2 samples/s +Step 8030/44000 | Loss: 3.2185 | LR: 4.66e-04 | 1314.6ms/step | 15.2 samples/s +Step 8040/44000 | Loss: 3.1704 | LR: 4.66e-04 | 1317.0ms/step | 15.2 samples/s +Step 8050/44000 | Loss: 3.2036 | LR: 4.66e-04 | 1317.0ms/step | 15.2 samples/s +Step 8060/44000 | Loss: 3.1823 | LR: 4.66e-04 | 1320.8ms/step | 15.1 samples/s +Step 8070/44000 | Loss: 3.2005 | LR: 4.65e-04 | 1316.7ms/step | 15.2 samples/s +Step 8080/44000 | Loss: 3.2182 | LR: 4.65e-04 | 1318.6ms/step | 15.2 samples/s +Step 8090/44000 | Loss: 3.1795 | LR: 4.65e-04 | 1317.8ms/step | 15.2 samples/s +Step 8100/44000 | Loss: 3.2110 | LR: 4.65e-04 | 1319.8ms/step | 15.2 samples/s +Step 8110/44000 | Loss: 3.1982 | LR: 4.65e-04 | 1320.9ms/step | 15.1 samples/s +Step 8120/44000 | Loss: 3.2013 | LR: 4.65e-04 | 1320.0ms/step | 15.2 samples/s +Step 8130/44000 | Loss: 3.1950 | LR: 4.65e-04 | 1319.0ms/step | 15.2 samples/s +Step 8140/44000 | Loss: 3.1974 | LR: 4.65e-04 | 1317.4ms/step | 15.2 samples/s +Step 8150/44000 | Loss: 3.2324 | LR: 4.65e-04 | 1319.5ms/step | 15.2 samples/s +Step 8160/44000 | Loss: 3.1825 | LR: 4.65e-04 | 1318.6ms/step | 15.2 samples/s +Step 8170/44000 | Loss: 3.1916 | LR: 4.65e-04 | 1319.8ms/step | 15.2 samples/s +Step 8180/44000 | Loss: 3.1942 | LR: 4.64e-04 | 1318.5ms/step | 15.2 samples/s +Step 8190/44000 | Loss: 3.2083 | LR: 4.64e-04 | 1316.5ms/step | 15.2 samples/s +Step 8200/44000 | Loss: 3.2304 | LR: 4.64e-04 | 1317.7ms/step | 15.2 samples/s +Step 8210/44000 | Loss: 3.1970 | LR: 4.64e-04 | 1319.3ms/step | 15.2 samples/s +Step 8220/44000 | Loss: 3.2533 | LR: 4.64e-04 | 1319.5ms/step | 15.2 samples/s +Step 8230/44000 | Loss: 3.1675 | LR: 4.64e-04 | 1318.4ms/step | 15.2 samples/s +Step 8240/44000 | Loss: 3.2149 | LR: 4.64e-04 | 1320.0ms/step | 15.2 samples/s +Step 8250/44000 | Loss: 3.2082 | LR: 4.64e-04 | 1319.8ms/step | 15.2 samples/s +Validation at step 8250: Loss = 3.2025, Perplexity = 24.59 +Step 8260/44000 | Loss: 3.2093 | LR: 4.64e-04 | 3538.9ms/step | 5.7 samples/s +Step 8270/44000 | Loss: 3.2080 | LR: 4.64e-04 | 1318.7ms/step | 15.2 samples/s +Step 8280/44000 | Loss: 3.1954 | LR: 4.64e-04 | 1319.3ms/step | 15.2 samples/s +Step 8290/44000 | Loss: 3.1938 | LR: 4.63e-04 | 1320.2ms/step | 15.1 samples/s +Step 8300/44000 | Loss: 3.1781 | LR: 4.63e-04 | 1320.3ms/step | 15.1 samples/s +Step 8310/44000 | Loss: 3.1952 | LR: 4.63e-04 | 1321.2ms/step | 15.1 samples/s +Step 8320/44000 | Loss: 3.1897 | LR: 4.63e-04 | 1321.6ms/step | 15.1 samples/s +Step 8330/44000 | Loss: 3.1788 | LR: 4.63e-04 | 1319.2ms/step | 15.2 samples/s +Step 8340/44000 | Loss: 3.1781 | LR: 4.63e-04 | 1319.1ms/step | 15.2 samples/s +Step 8350/44000 | Loss: 3.1701 | LR: 4.63e-04 | 1319.5ms/step | 15.2 samples/s +Step 8360/44000 | Loss: 3.1921 | LR: 4.63e-04 | 1318.8ms/step | 15.2 samples/s +Step 8370/44000 | Loss: 3.2036 | LR: 4.63e-04 | 1321.9ms/step | 15.1 samples/s +Step 8380/44000 | Loss: 3.1953 | LR: 4.63e-04 | 1321.1ms/step | 15.1 samples/s +Step 8390/44000 | Loss: 3.1778 | LR: 4.62e-04 | 1319.5ms/step | 15.2 samples/s +Step 8400/44000 | Loss: 3.1903 | LR: 4.62e-04 | 1319.7ms/step | 15.2 samples/s +Step 8410/44000 | Loss: 3.1851 | LR: 4.62e-04 | 1318.6ms/step | 15.2 samples/s +Step 8420/44000 | Loss: 3.1923 | LR: 4.62e-04 | 1319.4ms/step | 15.2 samples/s +Step 8430/44000 | Loss: 3.1939 | LR: 4.62e-04 | 1319.6ms/step | 15.2 samples/s +Step 8440/44000 | Loss: 3.1738 | LR: 4.62e-04 | 1321.2ms/step | 15.1 samples/s +Step 8450/44000 | Loss: 3.2463 | LR: 4.62e-04 | 1321.0ms/step | 15.1 samples/s +Step 8460/44000 | Loss: 3.2075 | LR: 4.62e-04 | 1324.2ms/step | 15.1 samples/s +Step 8470/44000 | Loss: 3.1803 | LR: 4.62e-04 | 1320.0ms/step | 15.2 samples/s +Step 8480/44000 | Loss: 3.1920 | LR: 4.62e-04 | 1318.8ms/step | 15.2 samples/s +Step 8490/44000 | Loss: 3.2181 | LR: 4.62e-04 | 1316.9ms/step | 15.2 samples/s +Step 8500/44000 | Loss: 3.2103 | LR: 4.61e-04 | 1318.1ms/step | 15.2 samples/s +Validation at step 8500: Loss = 3.1940, Perplexity = 24.39 +Step 8510/44000 | Loss: 3.1780 | LR: 4.61e-04 | 3532.5ms/step | 5.7 samples/s +Step 8520/44000 | Loss: 3.1933 | LR: 4.61e-04 | 1319.6ms/step | 15.2 samples/s +Step 8530/44000 | Loss: 3.1894 | LR: 4.61e-04 | 1318.4ms/step | 15.2 samples/s +Step 8540/44000 | Loss: 3.1690 | LR: 4.61e-04 | 1320.7ms/step | 15.1 samples/s +Step 8550/44000 | Loss: 3.1998 | LR: 4.61e-04 | 1317.5ms/step | 15.2 samples/s +Step 8560/44000 | Loss: 3.1703 | LR: 4.61e-04 | 1321.0ms/step | 15.1 samples/s +Step 8570/44000 | Loss: 3.2160 | LR: 4.61e-04 | 1319.0ms/step | 15.2 samples/s +Step 8580/44000 | Loss: 3.1512 | LR: 4.61e-04 | 1319.2ms/step | 15.2 samples/s +Step 8590/44000 | Loss: 3.1702 | LR: 4.61e-04 | 1318.3ms/step | 15.2 samples/s +Step 8600/44000 | Loss: 3.1736 | LR: 4.60e-04 | 1319.7ms/step | 15.2 samples/s +Step 8610/44000 | Loss: 3.1933 | LR: 4.60e-04 | 1319.3ms/step | 15.2 samples/s +Step 8620/44000 | Loss: 3.1643 | LR: 4.60e-04 | 1320.2ms/step | 15.1 samples/s +Step 8630/44000 | Loss: 3.1932 | LR: 4.60e-04 | 1321.8ms/step | 15.1 samples/s +Step 8640/44000 | Loss: 3.1951 | LR: 4.60e-04 | 1319.4ms/step | 15.2 samples/s +Step 8650/44000 | Loss: 3.1726 | LR: 4.60e-04 | 1321.4ms/step | 15.1 samples/s +Step 8660/44000 | Loss: 3.1968 | LR: 4.60e-04 | 1319.6ms/step | 15.2 samples/s +Step 8670/44000 | Loss: 3.1896 | LR: 4.60e-04 | 1320.6ms/step | 15.1 samples/s +Step 8680/44000 | Loss: 3.1823 | LR: 4.60e-04 | 1316.1ms/step | 15.2 samples/s +Step 8690/44000 | Loss: 3.1817 | LR: 4.60e-04 | 1318.3ms/step | 15.2 samples/s +Step 8700/44000 | Loss: 3.1499 | LR: 4.59e-04 | 1319.4ms/step | 15.2 samples/s +Step 8710/44000 | Loss: 3.1639 | LR: 4.59e-04 | 1319.5ms/step | 15.2 samples/s +Step 8720/44000 | Loss: 3.1573 | LR: 4.59e-04 | 1320.3ms/step | 15.1 samples/s +Step 8730/44000 | Loss: 3.1432 | LR: 4.59e-04 | 1320.6ms/step | 15.1 samples/s +Step 8740/44000 | Loss: 3.1939 | LR: 4.59e-04 | 1319.6ms/step | 15.2 samples/s +Step 8750/44000 | Loss: 3.1771 | LR: 4.59e-04 | 1319.8ms/step | 15.2 samples/s +Validation at step 8750: Loss = 3.1891, Perplexity = 24.27 +Step 8760/44000 | Loss: 3.1521 | LR: 4.59e-04 | 3529.8ms/step | 5.7 samples/s +Step 8770/44000 | Loss: 3.1759 | LR: 4.59e-04 | 1319.6ms/step | 15.2 samples/s +Step 8780/44000 | Loss: 3.1827 | LR: 4.59e-04 | 1321.6ms/step | 15.1 samples/s +Step 8790/44000 | Loss: 3.1765 | LR: 4.59e-04 | 1322.2ms/step | 15.1 samples/s +Step 8800/44000 | Loss: 3.1946 | LR: 4.58e-04 | 1321.8ms/step | 15.1 samples/s +Step 8810/44000 | Loss: 3.2075 | LR: 4.58e-04 | 1323.0ms/step | 15.1 samples/s +Step 8820/44000 | Loss: 3.1700 | LR: 4.58e-04 | 1324.2ms/step | 15.1 samples/s +Step 8830/44000 | Loss: 3.1626 | LR: 4.58e-04 | 1324.5ms/step | 15.1 samples/s +Step 8840/44000 | Loss: 3.1920 | LR: 4.58e-04 | 1321.1ms/step | 15.1 samples/s +Step 8850/44000 | Loss: 3.2220 | LR: 4.58e-04 | 1322.7ms/step | 15.1 samples/s +Step 8860/44000 | Loss: 3.1576 | LR: 4.58e-04 | 1321.1ms/step | 15.1 samples/s +Step 8870/44000 | Loss: 3.1778 | LR: 4.58e-04 | 1323.8ms/step | 15.1 samples/s +Step 8880/44000 | Loss: 3.1632 | LR: 4.58e-04 | 1319.8ms/step | 15.2 samples/s +Step 8890/44000 | Loss: 3.1785 | LR: 4.58e-04 | 1321.0ms/step | 15.1 samples/s +Step 8900/44000 | Loss: 3.1925 | LR: 4.57e-04 | 1319.5ms/step | 15.2 samples/s +Step 8910/44000 | Loss: 3.1877 | LR: 4.57e-04 | 1318.3ms/step | 15.2 samples/s +Step 8920/44000 | Loss: 3.1955 | LR: 4.57e-04 | 1319.6ms/step | 15.2 samples/s +Step 8930/44000 | Loss: 3.1527 | LR: 4.57e-04 | 1318.1ms/step | 15.2 samples/s +Step 8940/44000 | Loss: 3.1598 | LR: 4.57e-04 | 1319.5ms/step | 15.2 samples/s +Step 8950/44000 | Loss: 3.1528 | LR: 4.57e-04 | 1320.6ms/step | 15.1 samples/s +Step 8960/44000 | Loss: 3.1760 | LR: 4.57e-04 | 1319.2ms/step | 15.2 samples/s +Step 8970/44000 | Loss: 3.1839 | LR: 4.57e-04 | 1318.6ms/step | 15.2 samples/s +Step 8980/44000 | Loss: 3.1808 | LR: 4.57e-04 | 1321.2ms/step | 15.1 samples/s +Step 8990/44000 | Loss: 3.1665 | LR: 4.57e-04 | 1319.1ms/step | 15.2 samples/s +Step 9000/44000 | Loss: 3.1694 | LR: 4.56e-04 | 1320.5ms/step | 15.1 samples/s +Validation at step 9000: Loss = 3.1813, Perplexity = 24.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-9000 +Step 9010/44000 | Loss: 3.2059 | LR: 4.56e-04 | 3854.0ms/step | 5.2 samples/s +Step 9020/44000 | Loss: 3.1982 | LR: 4.56e-04 | 1320.0ms/step | 15.2 samples/s +Step 9030/44000 | Loss: 3.1853 | LR: 4.56e-04 | 1314.4ms/step | 15.2 samples/s +Step 9040/44000 | Loss: 3.1363 | LR: 4.56e-04 | 1318.7ms/step | 15.2 samples/s +Step 9050/44000 | Loss: 3.1458 | LR: 4.56e-04 | 1320.5ms/step | 15.1 samples/s +Step 9060/44000 | Loss: 3.1748 | LR: 4.56e-04 | 1320.1ms/step | 15.1 samples/s +Step 9070/44000 | Loss: 3.1886 | LR: 4.56e-04 | 1318.2ms/step | 15.2 samples/s +Step 9080/44000 | Loss: 3.1808 | LR: 4.56e-04 | 1319.2ms/step | 15.2 samples/s +Step 9090/44000 | Loss: 3.1813 | LR: 4.56e-04 | 1319.9ms/step | 15.2 samples/s +Step 9100/44000 | Loss: 3.1763 | LR: 4.55e-04 | 1319.0ms/step | 15.2 samples/s +Step 9110/44000 | Loss: 3.1676 | LR: 4.55e-04 | 1319.9ms/step | 15.2 samples/s +Step 9120/44000 | Loss: 3.1633 | LR: 4.55e-04 | 1319.9ms/step | 15.2 samples/s +Step 9130/44000 | Loss: 3.1370 | LR: 4.55e-04 | 1320.0ms/step | 15.2 samples/s +Step 9140/44000 | Loss: 3.1499 | LR: 4.55e-04 | 1317.2ms/step | 15.2 samples/s +Step 9150/44000 | Loss: 3.1987 | LR: 4.55e-04 | 1319.6ms/step | 15.2 samples/s +Step 9160/44000 | Loss: 3.2008 | LR: 4.55e-04 | 1320.3ms/step | 15.1 samples/s +Step 9170/44000 | Loss: 3.1710 | LR: 4.55e-04 | 1319.6ms/step | 15.2 samples/s +Step 9180/44000 | Loss: 3.1607 | LR: 4.55e-04 | 1318.2ms/step | 15.2 samples/s +Step 9190/44000 | Loss: 3.1990 | LR: 4.54e-04 | 1319.7ms/step | 15.2 samples/s +Step 9200/44000 | Loss: 3.1476 | LR: 4.54e-04 | 1322.4ms/step | 15.1 samples/s +Step 9210/44000 | Loss: 3.1840 | LR: 4.54e-04 | 1319.8ms/step | 15.2 samples/s +Step 9220/44000 | Loss: 3.1538 | LR: 4.54e-04 | 1319.8ms/step | 15.2 samples/s +Step 9230/44000 | Loss: 3.1726 | LR: 4.54e-04 | 1318.8ms/step | 15.2 samples/s +Step 9240/44000 | Loss: 3.1672 | LR: 4.54e-04 | 1322.4ms/step | 15.1 samples/s +Step 9250/44000 | Loss: 3.1285 | LR: 4.54e-04 | 1319.5ms/step | 15.2 samples/s +Validation at step 9250: Loss = 3.1747, Perplexity = 23.92 +Step 9260/44000 | Loss: 3.1648 | LR: 4.54e-04 | 3531.2ms/step | 5.7 samples/s +Step 9270/44000 | Loss: 3.1691 | LR: 4.54e-04 | 1318.3ms/step | 15.2 samples/s +Step 9280/44000 | Loss: 3.1495 | LR: 4.54e-04 | 1322.3ms/step | 15.1 samples/s +Step 9290/44000 | Loss: 3.1798 | LR: 4.53e-04 | 1319.9ms/step | 15.2 samples/s +Step 9300/44000 | Loss: 3.1699 | LR: 4.53e-04 | 1318.4ms/step | 15.2 samples/s +Step 9310/44000 | Loss: 3.1606 | LR: 4.53e-04 | 1317.8ms/step | 15.2 samples/s +Step 9320/44000 | Loss: 3.1846 | LR: 4.53e-04 | 1316.8ms/step | 15.2 samples/s +Step 9330/44000 | Loss: 3.1882 | LR: 4.53e-04 | 1316.0ms/step | 15.2 samples/s +Step 9340/44000 | Loss: 3.1657 | LR: 4.53e-04 | 1315.5ms/step | 15.2 samples/s +Step 9350/44000 | Loss: 3.1734 | LR: 4.53e-04 | 1317.6ms/step | 15.2 samples/s +Step 9360/44000 | Loss: 3.1649 | LR: 4.53e-04 | 1319.9ms/step | 15.2 samples/s +Step 9370/44000 | Loss: 3.1605 | LR: 4.53e-04 | 1319.5ms/step | 15.2 samples/s +Step 9380/44000 | Loss: 3.1800 | LR: 4.52e-04 | 1320.4ms/step | 15.1 samples/s +Step 9390/44000 | Loss: 3.1679 | LR: 4.52e-04 | 1320.6ms/step | 15.1 samples/s +Step 9400/44000 | Loss: 3.1660 | LR: 4.52e-04 | 1321.0ms/step | 15.1 samples/s +Step 9410/44000 | Loss: 3.1822 | LR: 4.52e-04 | 1318.9ms/step | 15.2 samples/s +Step 9420/44000 | Loss: 3.1624 | LR: 4.52e-04 | 1319.9ms/step | 15.2 samples/s +Step 9430/44000 | Loss: 3.1380 | LR: 4.52e-04 | 1319.7ms/step | 15.2 samples/s +Step 9440/44000 | Loss: 3.1673 | LR: 4.52e-04 | 1319.0ms/step | 15.2 samples/s +Step 9450/44000 | Loss: 3.1465 | LR: 4.52e-04 | 1319.3ms/step | 15.2 samples/s +Step 9460/44000 | Loss: 3.1819 | LR: 4.52e-04 | 1318.8ms/step | 15.2 samples/s +Step 9470/44000 | Loss: 3.1643 | LR: 4.52e-04 | 1317.8ms/step | 15.2 samples/s +Step 9480/44000 | Loss: 3.1743 | LR: 4.51e-04 | 1318.9ms/step | 15.2 samples/s +Step 9490/44000 | Loss: 3.1624 | LR: 4.51e-04 | 1320.5ms/step | 15.1 samples/s +Step 9500/44000 | Loss: 3.1659 | LR: 4.51e-04 | 1318.3ms/step | 15.2 samples/s +Validation at step 9500: Loss = 3.1690, Perplexity = 23.78 +Step 9510/44000 | Loss: 3.1383 | LR: 4.51e-04 | 3531.4ms/step | 5.7 samples/s +Step 9520/44000 | Loss: 3.1558 | LR: 4.51e-04 | 1317.6ms/step | 15.2 samples/s +Step 9530/44000 | Loss: 3.1725 | LR: 4.51e-04 | 1317.6ms/step | 15.2 samples/s +Step 9540/44000 | Loss: 3.1831 | LR: 4.51e-04 | 1316.4ms/step | 15.2 samples/s +Step 9550/44000 | Loss: 3.1706 | LR: 4.51e-04 | 1316.8ms/step | 15.2 samples/s +Step 9560/44000 | Loss: 3.1747 | LR: 4.51e-04 | 1316.0ms/step | 15.2 samples/s +Step 9570/44000 | Loss: 3.1446 | LR: 4.50e-04 | 1316.8ms/step | 15.2 samples/s +Step 9580/44000 | Loss: 3.1579 | LR: 4.50e-04 | 1318.4ms/step | 15.2 samples/s +Step 9590/44000 | Loss: 3.1723 | LR: 4.50e-04 | 1317.4ms/step | 15.2 samples/s +Step 9600/44000 | Loss: 3.1438 | LR: 4.50e-04 | 1318.2ms/step | 15.2 samples/s +Step 9610/44000 | Loss: 3.1336 | LR: 4.50e-04 | 1318.8ms/step | 15.2 samples/s +Step 9620/44000 | Loss: 3.1636 | LR: 4.50e-04 | 1318.6ms/step | 15.2 samples/s +Step 9630/44000 | Loss: 3.1279 | LR: 4.50e-04 | 1316.3ms/step | 15.2 samples/s +Step 9640/44000 | Loss: 3.1699 | LR: 4.50e-04 | 1317.3ms/step | 15.2 samples/s +Step 9650/44000 | Loss: 3.1708 | LR: 4.50e-04 | 1320.2ms/step | 15.1 samples/s +Step 9660/44000 | Loss: 3.1570 | LR: 4.49e-04 | 1319.4ms/step | 15.2 samples/s +Step 9670/44000 | Loss: 3.1866 | LR: 4.49e-04 | 1318.3ms/step | 15.2 samples/s +Step 9680/44000 | Loss: 3.1991 | LR: 4.49e-04 | 1318.1ms/step | 15.2 samples/s +Step 9690/44000 | Loss: 3.1461 | LR: 4.49e-04 | 1318.3ms/step | 15.2 samples/s +Step 9700/44000 | Loss: 3.1776 | LR: 4.49e-04 | 1318.4ms/step | 15.2 samples/s +Step 9710/44000 | Loss: 3.1844 | LR: 4.49e-04 | 1319.5ms/step | 15.2 samples/s +Step 9720/44000 | Loss: 3.1445 | LR: 4.49e-04 | 1318.6ms/step | 15.2 samples/s +Step 9730/44000 | Loss: 3.1805 | LR: 4.49e-04 | 1319.8ms/step | 15.2 samples/s +Step 9740/44000 | Loss: 3.1165 | LR: 4.49e-04 | 1318.0ms/step | 15.2 samples/s +Step 9750/44000 | Loss: 3.1375 | LR: 4.48e-04 | 1317.7ms/step | 15.2 samples/s +Validation at step 9750: Loss = 3.1616, Perplexity = 23.61 +Step 9760/44000 | Loss: 3.1405 | LR: 4.48e-04 | 3529.9ms/step | 5.7 samples/s +Step 9770/44000 | Loss: 3.1667 | LR: 4.48e-04 | 1319.5ms/step | 15.2 samples/s +Step 9780/44000 | Loss: 3.1726 | LR: 4.48e-04 | 1319.5ms/step | 15.2 samples/s +Step 9790/44000 | Loss: 3.1492 | LR: 4.48e-04 | 1321.0ms/step | 15.1 samples/s +Step 9800/44000 | Loss: 3.1882 | LR: 4.48e-04 | 1319.4ms/step | 15.2 samples/s +Step 9810/44000 | Loss: 3.1744 | LR: 4.48e-04 | 1319.7ms/step | 15.2 samples/s +Step 9820/44000 | Loss: 3.1535 | LR: 4.48e-04 | 1319.9ms/step | 15.2 samples/s +Step 9830/44000 | Loss: 3.1502 | LR: 4.48e-04 | 1319.7ms/step | 15.2 samples/s +Step 9840/44000 | Loss: 3.1391 | LR: 4.47e-04 | 1316.9ms/step | 15.2 samples/s +Step 9850/44000 | Loss: 3.1292 | LR: 4.47e-04 | 1319.3ms/step | 15.2 samples/s +Step 9860/44000 | Loss: 3.1688 | LR: 4.47e-04 | 1317.8ms/step | 15.2 samples/s +Step 9870/44000 | Loss: 3.1396 | LR: 4.47e-04 | 1319.7ms/step | 15.2 samples/s +Step 9880/44000 | Loss: 3.1718 | LR: 4.47e-04 | 1319.0ms/step | 15.2 samples/s +Step 9890/44000 | Loss: 3.1408 | LR: 4.47e-04 | 1317.5ms/step | 15.2 samples/s +Step 9900/44000 | Loss: 3.1640 | LR: 4.47e-04 | 1319.0ms/step | 15.2 samples/s +Step 9910/44000 | Loss: 3.1653 | LR: 4.47e-04 | 1319.3ms/step | 15.2 samples/s +Step 9920/44000 | Loss: 3.1419 | LR: 4.47e-04 | 1318.7ms/step | 15.2 samples/s +Step 9930/44000 | Loss: 3.1541 | LR: 4.46e-04 | 1318.3ms/step | 15.2 samples/s +Step 9940/44000 | Loss: 3.1642 | LR: 4.46e-04 | 1317.3ms/step | 15.2 samples/s +Step 9950/44000 | Loss: 3.1842 | LR: 4.46e-04 | 1316.6ms/step | 15.2 samples/s +Step 9960/44000 | Loss: 3.1621 | LR: 4.46e-04 | 1315.6ms/step | 15.2 samples/s +Step 9970/44000 | Loss: 3.1426 | LR: 4.46e-04 | 1317.4ms/step | 15.2 samples/s +Step 9980/44000 | Loss: 3.1569 | LR: 4.46e-04 | 1317.8ms/step | 15.2 samples/s +Step 9990/44000 | Loss: 3.1297 | LR: 4.46e-04 | 1320.8ms/step | 15.1 samples/s +Step 10000/44000 | Loss: 3.1390 | LR: 4.46e-04 | 1320.8ms/step | 15.1 samples/s +Validation at step 10000: Loss = 3.1564, Perplexity = 23.49 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-10000 +Step 10010/44000 | Loss: 3.1593 | LR: 4.46e-04 | 3844.0ms/step | 5.2 samples/s +Step 10020/44000 | Loss: 3.1430 | LR: 4.45e-04 | 1320.7ms/step | 15.1 samples/s +Step 10030/44000 | Loss: 3.1212 | LR: 4.45e-04 | 1316.6ms/step | 15.2 samples/s +Step 10040/44000 | Loss: 3.1393 | LR: 4.45e-04 | 1319.1ms/step | 15.2 samples/s +Step 10050/44000 | Loss: 3.1434 | LR: 4.45e-04 | 1321.2ms/step | 15.1 samples/s +Step 10060/44000 | Loss: 3.1692 | LR: 4.45e-04 | 1320.0ms/step | 15.2 samples/s +Step 10070/44000 | Loss: 3.1666 | LR: 4.45e-04 | 1319.6ms/step | 15.2 samples/s +Step 10080/44000 | Loss: 3.1516 | LR: 4.45e-04 | 1319.8ms/step | 15.2 samples/s +Step 10090/44000 | Loss: 3.1229 | LR: 4.45e-04 | 1318.2ms/step | 15.2 samples/s +Step 10100/44000 | Loss: 3.1445 | LR: 4.45e-04 | 1316.7ms/step | 15.2 samples/s +Step 10110/44000 | Loss: 3.1623 | LR: 4.44e-04 | 1321.4ms/step | 15.1 samples/s +Step 10120/44000 | Loss: 3.1263 | LR: 4.44e-04 | 1320.4ms/step | 15.1 samples/s +Step 10130/44000 | Loss: 3.1499 | LR: 4.44e-04 | 1317.3ms/step | 15.2 samples/s +Step 10140/44000 | Loss: 3.1594 | LR: 4.44e-04 | 1320.1ms/step | 15.2 samples/s +Step 10150/44000 | Loss: 3.1414 | LR: 4.44e-04 | 1319.0ms/step | 15.2 samples/s +Step 10160/44000 | Loss: 3.1405 | LR: 4.44e-04 | 1320.6ms/step | 15.1 samples/s +Step 10170/44000 | Loss: 3.1834 | LR: 4.44e-04 | 1321.1ms/step | 15.1 samples/s +Step 10180/44000 | Loss: 3.1667 | LR: 4.44e-04 | 1319.9ms/step | 15.2 samples/s +Step 10190/44000 | Loss: 3.1776 | LR: 4.43e-04 | 1321.3ms/step | 15.1 samples/s +Step 10200/44000 | Loss: 3.1569 | LR: 4.43e-04 | 1320.4ms/step | 15.1 samples/s +Step 10210/44000 | Loss: 3.1489 | LR: 4.43e-04 | 1320.4ms/step | 15.1 samples/s +Step 10220/44000 | Loss: 3.1142 | LR: 4.43e-04 | 1321.7ms/step | 15.1 samples/s +Step 10230/44000 | Loss: 3.1179 | LR: 4.43e-04 | 1319.9ms/step | 15.2 samples/s +Step 10240/44000 | Loss: 3.1693 | LR: 4.43e-04 | 1320.7ms/step | 15.1 samples/s +Step 10250/44000 | Loss: 3.1201 | LR: 4.43e-04 | 1320.6ms/step | 15.1 samples/s +Validation at step 10250: Loss = 3.1516, Perplexity = 23.37 +Step 10260/44000 | Loss: 3.1520 | LR: 4.43e-04 | 3536.9ms/step | 5.7 samples/s +Step 10270/44000 | Loss: 3.1544 | LR: 4.43e-04 | 1319.9ms/step | 15.2 samples/s +Step 10280/44000 | Loss: 3.1561 | LR: 4.42e-04 | 1319.7ms/step | 15.2 samples/s +Step 10290/44000 | Loss: 3.1583 | LR: 4.42e-04 | 1317.2ms/step | 15.2 samples/s +Step 10300/44000 | Loss: 3.1226 | LR: 4.42e-04 | 1319.2ms/step | 15.2 samples/s +Step 10310/44000 | Loss: 3.1404 | LR: 4.42e-04 | 1318.8ms/step | 15.2 samples/s +Step 10320/44000 | Loss: 3.1546 | LR: 4.42e-04 | 1319.3ms/step | 15.2 samples/s +Step 10330/44000 | Loss: 3.1560 | LR: 4.42e-04 | 1321.0ms/step | 15.1 samples/s +Step 10340/44000 | Loss: 3.1465 | LR: 4.42e-04 | 1319.4ms/step | 15.2 samples/s +Step 10350/44000 | Loss: 3.1524 | LR: 4.42e-04 | 1319.6ms/step | 15.2 samples/s +Step 10360/44000 | Loss: 3.1504 | LR: 4.42e-04 | 1317.7ms/step | 15.2 samples/s +Step 10370/44000 | Loss: 3.1462 | LR: 4.41e-04 | 1321.4ms/step | 15.1 samples/s +Step 10380/44000 | Loss: 3.1584 | LR: 4.41e-04 | 1320.7ms/step | 15.1 samples/s +Step 10390/44000 | Loss: 3.1679 | LR: 4.41e-04 | 1317.8ms/step | 15.2 samples/s +Step 10400/44000 | Loss: 3.1652 | LR: 4.41e-04 | 1320.9ms/step | 15.1 samples/s +Step 10410/44000 | Loss: 3.1512 | LR: 4.41e-04 | 1318.1ms/step | 15.2 samples/s +Step 10420/44000 | Loss: 3.1707 | LR: 4.41e-04 | 1319.1ms/step | 15.2 samples/s +Step 10430/44000 | Loss: 3.1416 | LR: 4.41e-04 | 1320.2ms/step | 15.1 samples/s +Step 10440/44000 | Loss: 3.1037 | LR: 4.41e-04 | 1318.4ms/step | 15.2 samples/s +Step 10450/44000 | Loss: 3.1570 | LR: 4.40e-04 | 1317.8ms/step | 15.2 samples/s +Step 10460/44000 | Loss: 3.1606 | LR: 4.40e-04 | 1317.4ms/step | 15.2 samples/s +Step 10470/44000 | Loss: 3.1200 | LR: 4.40e-04 | 1316.4ms/step | 15.2 samples/s +Step 10480/44000 | Loss: 3.1386 | LR: 4.40e-04 | 1314.7ms/step | 15.2 samples/s +Step 10490/44000 | Loss: 3.1424 | LR: 4.40e-04 | 1316.4ms/step | 15.2 samples/s +Step 10500/44000 | Loss: 3.1675 | LR: 4.40e-04 | 1316.3ms/step | 15.2 samples/s +Validation at step 10500: Loss = 3.1462, Perplexity = 23.25 +Step 10510/44000 | Loss: 3.1459 | LR: 4.40e-04 | 3534.2ms/step | 5.7 samples/s +Step 10520/44000 | Loss: 3.1639 | LR: 4.40e-04 | 1319.1ms/step | 15.2 samples/s +Step 10530/44000 | Loss: 3.1623 | LR: 4.40e-04 | 1319.3ms/step | 15.2 samples/s +Step 10540/44000 | Loss: 3.1287 | LR: 4.39e-04 | 1318.7ms/step | 15.2 samples/s +Step 10550/44000 | Loss: 3.1484 | LR: 4.39e-04 | 1318.3ms/step | 15.2 samples/s +Step 10560/44000 | Loss: 3.1211 | LR: 4.39e-04 | 1319.7ms/step | 15.2 samples/s +Step 10570/44000 | Loss: 3.1138 | LR: 4.39e-04 | 1320.0ms/step | 15.2 samples/s +Step 10580/44000 | Loss: 3.1606 | LR: 4.39e-04 | 1321.1ms/step | 15.1 samples/s +Step 10590/44000 | Loss: 3.1271 | LR: 4.39e-04 | 1318.2ms/step | 15.2 samples/s +Step 10600/44000 | Loss: 3.1389 | LR: 4.39e-04 | 1320.1ms/step | 15.2 samples/s +Step 10610/44000 | Loss: 3.1556 | LR: 4.39e-04 | 1321.8ms/step | 15.1 samples/s +Step 10620/44000 | Loss: 3.1253 | LR: 4.38e-04 | 1318.9ms/step | 15.2 samples/s +Step 10630/44000 | Loss: 3.0910 | LR: 4.38e-04 | 1317.0ms/step | 15.2 samples/s +Step 10640/44000 | Loss: 3.1734 | LR: 4.38e-04 | 1318.5ms/step | 15.2 samples/s +Step 10650/44000 | Loss: 3.1607 | LR: 4.38e-04 | 1319.7ms/step | 15.2 samples/s +Step 10660/44000 | Loss: 3.1475 | LR: 4.38e-04 | 1318.2ms/step | 15.2 samples/s +Step 10670/44000 | Loss: 3.1177 | LR: 4.38e-04 | 1320.0ms/step | 15.2 samples/s +Step 10680/44000 | Loss: 3.1659 | LR: 4.38e-04 | 1321.2ms/step | 15.1 samples/s +Step 10690/44000 | Loss: 3.1428 | LR: 4.38e-04 | 1318.8ms/step | 15.2 samples/s +Step 10700/44000 | Loss: 3.1684 | LR: 4.37e-04 | 1319.7ms/step | 15.2 samples/s +Step 10710/44000 | Loss: 3.1401 | LR: 4.37e-04 | 1319.0ms/step | 15.2 samples/s +Step 10720/44000 | Loss: 3.1569 | LR: 4.37e-04 | 1317.0ms/step | 15.2 samples/s +Step 10730/44000 | Loss: 3.1556 | LR: 4.37e-04 | 1319.5ms/step | 15.2 samples/s +Step 10740/44000 | Loss: 3.1625 | LR: 4.37e-04 | 1319.7ms/step | 15.2 samples/s +Step 10750/44000 | Loss: 3.1332 | LR: 4.37e-04 | 1321.4ms/step | 15.1 samples/s +Validation at step 10750: Loss = 3.1410, Perplexity = 23.13 +Step 10760/44000 | Loss: 3.1552 | LR: 4.37e-04 | 3532.3ms/step | 5.7 samples/s +Step 10770/44000 | Loss: 3.1322 | LR: 4.37e-04 | 1318.7ms/step | 15.2 samples/s +Step 10780/44000 | Loss: 3.1232 | LR: 4.37e-04 | 1320.7ms/step | 15.1 samples/s +Step 10790/44000 | Loss: 3.1287 | LR: 4.36e-04 | 1320.4ms/step | 15.1 samples/s +Step 10800/44000 | Loss: 3.1116 | LR: 4.36e-04 | 1320.2ms/step | 15.1 samples/s +Step 10810/44000 | Loss: 3.0961 | LR: 4.36e-04 | 1319.7ms/step | 15.2 samples/s +Step 10820/44000 | Loss: 3.1428 | LR: 4.36e-04 | 1319.4ms/step | 15.2 samples/s +Step 10830/44000 | Loss: 3.1513 | LR: 4.36e-04 | 1320.7ms/step | 15.1 samples/s +Step 10840/44000 | Loss: 3.1396 | LR: 4.36e-04 | 1319.9ms/step | 15.2 samples/s +Step 10850/44000 | Loss: 3.1553 | LR: 4.36e-04 | 1318.5ms/step | 15.2 samples/s +Step 10860/44000 | Loss: 3.1475 | LR: 4.36e-04 | 1320.0ms/step | 15.2 samples/s +Step 10870/44000 | Loss: 3.1050 | LR: 4.35e-04 | 1323.4ms/step | 15.1 samples/s +Step 10880/44000 | Loss: 3.0952 | LR: 4.35e-04 | 1319.8ms/step | 15.2 samples/s +Step 10890/44000 | Loss: 3.1558 | LR: 4.35e-04 | 1320.4ms/step | 15.1 samples/s +Step 10900/44000 | Loss: 3.1588 | LR: 4.35e-04 | 1322.9ms/step | 15.1 samples/s +Step 10910/44000 | Loss: 3.1749 | LR: 4.35e-04 | 1320.0ms/step | 15.2 samples/s +Step 10920/44000 | Loss: 3.1288 | LR: 4.35e-04 | 1323.6ms/step | 15.1 samples/s +Step 10930/44000 | Loss: 3.1455 | LR: 4.35e-04 | 1322.1ms/step | 15.1 samples/s +Step 10940/44000 | Loss: 3.1270 | LR: 4.35e-04 | 1325.9ms/step | 15.1 samples/s +Step 10950/44000 | Loss: 3.1571 | LR: 4.34e-04 | 1320.2ms/step | 15.1 samples/s +Step 10960/44000 | Loss: 3.1366 | LR: 4.34e-04 | 1321.8ms/step | 15.1 samples/s +Step 10970/44000 | Loss: 3.1284 | LR: 4.34e-04 | 1322.8ms/step | 15.1 samples/s +Step 10980/44000 | Loss: 3.1450 | LR: 4.34e-04 | 1320.5ms/step | 15.1 samples/s +Step 10990/44000 | Loss: 3.1202 | LR: 4.34e-04 | 1319.3ms/step | 15.2 samples/s +Step 11000/44000 | Loss: 3.1403 | LR: 4.34e-04 | 1319.4ms/step | 15.2 samples/s +Validation at step 11000: Loss = 3.1367, Perplexity = 23.03 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-11000 +Step 11010/44000 | Loss: 3.1187 | LR: 4.34e-04 | 3898.4ms/step | 5.1 samples/s +Step 11020/44000 | Loss: 3.0990 | LR: 4.34e-04 | 1316.3ms/step | 15.2 samples/s +Step 11030/44000 | Loss: 3.1381 | LR: 4.33e-04 | 1318.4ms/step | 15.2 samples/s +Step 11040/44000 | Loss: 3.1008 | LR: 4.33e-04 | 1319.9ms/step | 15.2 samples/s +Step 11050/44000 | Loss: 3.1464 | LR: 4.33e-04 | 1314.9ms/step | 15.2 samples/s +Step 11060/44000 | Loss: 3.1469 | LR: 4.33e-04 | 1315.0ms/step | 15.2 samples/s +Step 11070/44000 | Loss: 3.1493 | LR: 4.33e-04 | 1312.6ms/step | 15.2 samples/s +Step 11080/44000 | Loss: 3.1395 | LR: 4.33e-04 | 1313.5ms/step | 15.2 samples/s +Step 11090/44000 | Loss: 3.0942 | LR: 4.33e-04 | 1312.5ms/step | 15.2 samples/s +Step 11100/44000 | Loss: 3.1339 | LR: 4.33e-04 | 1313.2ms/step | 15.2 samples/s +Step 11110/44000 | Loss: 3.1174 | LR: 4.32e-04 | 1314.8ms/step | 15.2 samples/s +Step 11120/44000 | Loss: 3.1320 | LR: 4.32e-04 | 1315.1ms/step | 15.2 samples/s +Step 11130/44000 | Loss: 3.1570 | LR: 4.32e-04 | 1313.2ms/step | 15.2 samples/s +Step 11140/44000 | Loss: 3.1244 | LR: 4.32e-04 | 1318.1ms/step | 15.2 samples/s +Step 11150/44000 | Loss: 3.1253 | LR: 4.32e-04 | 1318.8ms/step | 15.2 samples/s +Step 11160/44000 | Loss: 3.1258 | LR: 4.32e-04 | 1315.9ms/step | 15.2 samples/s +Step 11170/44000 | Loss: 3.1187 | LR: 4.32e-04 | 1314.7ms/step | 15.2 samples/s +Step 11180/44000 | Loss: 3.1226 | LR: 4.32e-04 | 1314.5ms/step | 15.2 samples/s +Step 11190/44000 | Loss: 3.1051 | LR: 4.31e-04 | 1314.9ms/step | 15.2 samples/s +Step 11200/44000 | Loss: 3.1464 | LR: 4.31e-04 | 1314.0ms/step | 15.2 samples/s +Step 11210/44000 | Loss: 3.1835 | LR: 4.31e-04 | 1312.8ms/step | 15.2 samples/s +Step 11220/44000 | Loss: 3.1081 | LR: 4.31e-04 | 1314.1ms/step | 15.2 samples/s +Step 11230/44000 | Loss: 3.1316 | LR: 4.31e-04 | 1312.8ms/step | 15.2 samples/s +Step 11240/44000 | Loss: 3.1180 | LR: 4.31e-04 | 1310.0ms/step | 15.3 samples/s +Step 11250/44000 | Loss: 3.1204 | LR: 4.31e-04 | 1312.4ms/step | 15.2 samples/s +Validation at step 11250: Loss = 3.1308, Perplexity = 22.89 +Step 11260/44000 | Loss: 3.1139 | LR: 4.31e-04 | 3522.6ms/step | 5.7 samples/s +Step 11270/44000 | Loss: 3.1338 | LR: 4.30e-04 | 1313.9ms/step | 15.2 samples/s +Step 11280/44000 | Loss: 3.1114 | LR: 4.30e-04 | 1313.0ms/step | 15.2 samples/s +Step 11290/44000 | Loss: 3.1135 | LR: 4.30e-04 | 1313.1ms/step | 15.2 samples/s +Step 11300/44000 | Loss: 3.1510 | LR: 4.30e-04 | 1313.2ms/step | 15.2 samples/s +Step 11310/44000 | Loss: 3.1295 | LR: 4.30e-04 | 1312.9ms/step | 15.2 samples/s +Step 11320/44000 | Loss: 3.1177 | LR: 4.30e-04 | 1311.9ms/step | 15.2 samples/s +Step 11330/44000 | Loss: 3.1154 | LR: 4.30e-04 | 1314.7ms/step | 15.2 samples/s +Step 11340/44000 | Loss: 3.1153 | LR: 4.30e-04 | 1311.1ms/step | 15.3 samples/s +Step 11350/44000 | Loss: 3.1100 | LR: 4.29e-04 | 1314.4ms/step | 15.2 samples/s +Step 11360/44000 | Loss: 3.1415 | LR: 4.29e-04 | 1313.8ms/step | 15.2 samples/s +Step 11370/44000 | Loss: 3.1190 | LR: 4.29e-04 | 1312.9ms/step | 15.2 samples/s +Step 11380/44000 | Loss: 3.1176 | LR: 4.29e-04 | 1314.9ms/step | 15.2 samples/s +Step 11390/44000 | Loss: 3.1204 | LR: 4.29e-04 | 1311.2ms/step | 15.3 samples/s +Step 11400/44000 | Loss: 3.1441 | LR: 4.29e-04 | 1311.1ms/step | 15.3 samples/s +Step 11410/44000 | Loss: 3.1102 | LR: 4.29e-04 | 1309.8ms/step | 15.3 samples/s +Step 11420/44000 | Loss: 3.1195 | LR: 4.29e-04 | 1311.2ms/step | 15.3 samples/s +Step 11430/44000 | Loss: 3.1352 | LR: 4.28e-04 | 1311.1ms/step | 15.3 samples/s +Step 11440/44000 | Loss: 3.1268 | LR: 4.28e-04 | 1310.9ms/step | 15.3 samples/s +Step 11450/44000 | Loss: 3.1395 | LR: 4.28e-04 | 1311.5ms/step | 15.2 samples/s +Step 11460/44000 | Loss: 3.1331 | LR: 4.28e-04 | 1311.5ms/step | 15.2 samples/s +Step 11470/44000 | Loss: 3.1252 | LR: 4.28e-04 | 1311.1ms/step | 15.3 samples/s +Step 11480/44000 | Loss: 3.1298 | LR: 4.28e-04 | 1312.0ms/step | 15.2 samples/s +Step 11490/44000 | Loss: 3.1040 | LR: 4.28e-04 | 1311.7ms/step | 15.2 samples/s +Step 11500/44000 | Loss: 3.1161 | LR: 4.28e-04 | 1313.0ms/step | 15.2 samples/s +Validation at step 11500: Loss = 3.1251, Perplexity = 22.76 +Step 11510/44000 | Loss: 3.1528 | LR: 4.27e-04 | 3514.8ms/step | 5.7 samples/s +Step 11520/44000 | Loss: 3.1367 | LR: 4.27e-04 | 1312.4ms/step | 15.2 samples/s +Step 11530/44000 | Loss: 3.1373 | LR: 4.27e-04 | 1311.6ms/step | 15.2 samples/s +Step 11540/44000 | Loss: 3.1209 | LR: 4.27e-04 | 1311.3ms/step | 15.3 samples/s +Step 11550/44000 | Loss: 3.1005 | LR: 4.27e-04 | 1311.9ms/step | 15.2 samples/s +Step 11560/44000 | Loss: 3.1442 | LR: 4.27e-04 | 1312.0ms/step | 15.2 samples/s +Step 11570/44000 | Loss: 3.1202 | LR: 4.27e-04 | 1311.4ms/step | 15.3 samples/s +Step 11580/44000 | Loss: 3.1502 | LR: 4.27e-04 | 1316.6ms/step | 15.2 samples/s +Step 11590/44000 | Loss: 3.1510 | LR: 4.26e-04 | 1311.8ms/step | 15.2 samples/s +Step 11600/44000 | Loss: 3.1122 | LR: 4.26e-04 | 1312.9ms/step | 15.2 samples/s +Step 11610/44000 | Loss: 3.1361 | LR: 4.26e-04 | 1312.5ms/step | 15.2 samples/s +Step 11620/44000 | Loss: 3.0873 | LR: 4.26e-04 | 1312.0ms/step | 15.2 samples/s +Step 11630/44000 | Loss: 3.0946 | LR: 4.26e-04 | 1313.0ms/step | 15.2 samples/s +Step 11640/44000 | Loss: 3.1434 | LR: 4.26e-04 | 1313.2ms/step | 15.2 samples/s +Step 11650/44000 | Loss: 3.1017 | LR: 4.26e-04 | 1313.5ms/step | 15.2 samples/s +Step 11660/44000 | Loss: 3.1491 | LR: 4.26e-04 | 1315.2ms/step | 15.2 samples/s +Step 11670/44000 | Loss: 3.1069 | LR: 4.25e-04 | 1311.5ms/step | 15.2 samples/s +Step 11680/44000 | Loss: 3.1238 | LR: 4.25e-04 | 1314.6ms/step | 15.2 samples/s +Step 11690/44000 | Loss: 3.1047 | LR: 4.25e-04 | 1314.9ms/step | 15.2 samples/s +Step 11700/44000 | Loss: 3.1262 | LR: 4.25e-04 | 1313.5ms/step | 15.2 samples/s +Step 11710/44000 | Loss: 3.1279 | LR: 4.25e-04 | 1313.0ms/step | 15.2 samples/s +Step 11720/44000 | Loss: 3.1233 | LR: 4.25e-04 | 1312.3ms/step | 15.2 samples/s +Step 11730/44000 | Loss: 3.1159 | LR: 4.25e-04 | 1312.9ms/step | 15.2 samples/s +Step 11740/44000 | Loss: 3.1093 | LR: 4.24e-04 | 1314.3ms/step | 15.2 samples/s +Step 11750/44000 | Loss: 3.0991 | LR: 4.24e-04 | 1314.6ms/step | 15.2 samples/s +Validation at step 11750: Loss = 3.1204, Perplexity = 22.65 +Step 11760/44000 | Loss: 3.1437 | LR: 4.24e-04 | 3512.6ms/step | 5.7 samples/s +Step 11770/44000 | Loss: 3.1258 | LR: 4.24e-04 | 1310.1ms/step | 15.3 samples/s +Step 11780/44000 | Loss: 3.1267 | LR: 4.24e-04 | 1316.7ms/step | 15.2 samples/s +Step 11790/44000 | Loss: 3.1234 | LR: 4.24e-04 | 1314.0ms/step | 15.2 samples/s +Step 11800/44000 | Loss: 3.1344 | LR: 4.24e-04 | 1314.3ms/step | 15.2 samples/s +Step 11810/44000 | Loss: 3.1241 | LR: 4.24e-04 | 1312.0ms/step | 15.2 samples/s +Step 11820/44000 | Loss: 3.1213 | LR: 4.23e-04 | 1313.8ms/step | 15.2 samples/s +Step 11830/44000 | Loss: 3.1431 | LR: 4.23e-04 | 1313.7ms/step | 15.2 samples/s +Step 11840/44000 | Loss: 3.1179 | LR: 4.23e-04 | 1312.0ms/step | 15.2 samples/s +Step 11850/44000 | Loss: 3.1113 | LR: 4.23e-04 | 1310.3ms/step | 15.3 samples/s +Step 11860/44000 | Loss: 3.1217 | LR: 4.23e-04 | 1311.7ms/step | 15.2 samples/s +Step 11870/44000 | Loss: 3.1002 | LR: 4.23e-04 | 1310.7ms/step | 15.3 samples/s +Step 11880/44000 | Loss: 3.1087 | LR: 4.23e-04 | 1314.4ms/step | 15.2 samples/s +Step 11890/44000 | Loss: 3.1290 | LR: 4.23e-04 | 1312.9ms/step | 15.2 samples/s +Step 11900/44000 | Loss: 3.1025 | LR: 4.22e-04 | 1314.2ms/step | 15.2 samples/s +Step 11910/44000 | Loss: 3.0838 | LR: 4.22e-04 | 1311.7ms/step | 15.2 samples/s +Step 11920/44000 | Loss: 3.1117 | LR: 4.22e-04 | 1313.4ms/step | 15.2 samples/s +Step 11930/44000 | Loss: 3.1300 | LR: 4.22e-04 | 1312.8ms/step | 15.2 samples/s +Step 11940/44000 | Loss: 3.1254 | LR: 4.22e-04 | 1312.7ms/step | 15.2 samples/s +Step 11950/44000 | Loss: 3.1142 | LR: 4.22e-04 | 1312.5ms/step | 15.2 samples/s +Step 11960/44000 | Loss: 3.1468 | LR: 4.22e-04 | 1312.2ms/step | 15.2 samples/s +Step 11970/44000 | Loss: 3.1368 | LR: 4.21e-04 | 1314.8ms/step | 15.2 samples/s +Step 11980/44000 | Loss: 3.1438 | LR: 4.21e-04 | 1314.3ms/step | 15.2 samples/s +Step 11990/44000 | Loss: 3.1283 | LR: 4.21e-04 | 1313.0ms/step | 15.2 samples/s +Step 12000/44000 | Loss: 3.0913 | LR: 4.21e-04 | 1314.4ms/step | 15.2 samples/s +Validation at step 12000: Loss = 3.1155, Perplexity = 22.54 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-12000 +Step 12010/44000 | Loss: 3.0993 | LR: 4.21e-04 | 3839.0ms/step | 5.2 samples/s +Step 12020/44000 | Loss: 3.1286 | LR: 4.21e-04 | 1315.9ms/step | 15.2 samples/s +Step 12030/44000 | Loss: 3.1149 | LR: 4.21e-04 | 1319.3ms/step | 15.2 samples/s +Step 12040/44000 | Loss: 3.0987 | LR: 4.21e-04 | 1318.0ms/step | 15.2 samples/s +Step 12050/44000 | Loss: 3.1366 | LR: 4.20e-04 | 1319.5ms/step | 15.2 samples/s +Step 12060/44000 | Loss: 3.1076 | LR: 4.20e-04 | 1320.4ms/step | 15.1 samples/s +Step 12070/44000 | Loss: 3.1258 | LR: 4.20e-04 | 1317.1ms/step | 15.2 samples/s +Step 12080/44000 | Loss: 3.1192 | LR: 4.20e-04 | 1319.9ms/step | 15.2 samples/s +Step 12090/44000 | Loss: 3.1103 | LR: 4.20e-04 | 1319.7ms/step | 15.2 samples/s +Step 12100/44000 | Loss: 3.0868 | LR: 4.20e-04 | 1319.6ms/step | 15.2 samples/s +Step 12110/44000 | Loss: 3.1112 | LR: 4.20e-04 | 1317.8ms/step | 15.2 samples/s +Step 12120/44000 | Loss: 3.1057 | LR: 4.19e-04 | 1320.0ms/step | 15.2 samples/s +Step 12130/44000 | Loss: 3.1131 | LR: 4.19e-04 | 1319.7ms/step | 15.2 samples/s +Step 12140/44000 | Loss: 3.1017 | LR: 4.19e-04 | 1318.8ms/step | 15.2 samples/s +Step 12150/44000 | Loss: 3.0995 | LR: 4.19e-04 | 1321.2ms/step | 15.1 samples/s +Step 12160/44000 | Loss: 3.1116 | LR: 4.19e-04 | 1320.9ms/step | 15.1 samples/s +Step 12170/44000 | Loss: 3.1084 | LR: 4.19e-04 | 1319.3ms/step | 15.2 samples/s +Step 12180/44000 | Loss: 3.1102 | LR: 4.19e-04 | 1320.2ms/step | 15.1 samples/s +Step 12190/44000 | Loss: 3.1504 | LR: 4.19e-04 | 1318.3ms/step | 15.2 samples/s +Step 12200/44000 | Loss: 3.1142 | LR: 4.18e-04 | 1319.9ms/step | 15.2 samples/s +Step 12210/44000 | Loss: 3.0850 | LR: 4.18e-04 | 1320.3ms/step | 15.1 samples/s +Step 12220/44000 | Loss: 3.1259 | LR: 4.18e-04 | 1321.5ms/step | 15.1 samples/s +Step 12230/44000 | Loss: 3.0990 | LR: 4.18e-04 | 1317.9ms/step | 15.2 samples/s +Step 12240/44000 | Loss: 3.0984 | LR: 4.18e-04 | 1319.4ms/step | 15.2 samples/s +Step 12250/44000 | Loss: 3.0984 | LR: 4.18e-04 | 1320.8ms/step | 15.1 samples/s +Validation at step 12250: Loss = 3.1130, Perplexity = 22.49 +Step 12260/44000 | Loss: 3.0994 | LR: 4.18e-04 | 3533.3ms/step | 5.7 samples/s +Step 12270/44000 | Loss: 3.1185 | LR: 4.17e-04 | 1320.0ms/step | 15.2 samples/s +Step 12280/44000 | Loss: 3.1046 | LR: 4.17e-04 | 1319.8ms/step | 15.2 samples/s +Step 12290/44000 | Loss: 3.1135 | LR: 4.17e-04 | 1319.6ms/step | 15.2 samples/s +Step 12300/44000 | Loss: 3.1054 | LR: 4.17e-04 | 1321.1ms/step | 15.1 samples/s +Step 12310/44000 | Loss: 3.1180 | LR: 4.17e-04 | 1321.9ms/step | 15.1 samples/s +Step 12320/44000 | Loss: 3.0805 | LR: 4.17e-04 | 1320.9ms/step | 15.1 samples/s +Step 12330/44000 | Loss: 3.1178 | LR: 4.17e-04 | 1319.7ms/step | 15.2 samples/s +Step 12340/44000 | Loss: 3.1076 | LR: 4.17e-04 | 1318.9ms/step | 15.2 samples/s +Step 12350/44000 | Loss: 3.1090 | LR: 4.16e-04 | 1320.2ms/step | 15.1 samples/s +Step 12360/44000 | Loss: 3.1051 | LR: 4.16e-04 | 1317.2ms/step | 15.2 samples/s +Step 12370/44000 | Loss: 3.1025 | LR: 4.16e-04 | 1317.6ms/step | 15.2 samples/s +Step 12380/44000 | Loss: 3.0966 | LR: 4.16e-04 | 1320.9ms/step | 15.1 samples/s +Step 12390/44000 | Loss: 3.1160 | LR: 4.16e-04 | 1320.3ms/step | 15.1 samples/s +Step 12400/44000 | Loss: 3.0869 | LR: 4.16e-04 | 1319.8ms/step | 15.2 samples/s +Step 12410/44000 | Loss: 3.0769 | LR: 4.16e-04 | 1317.4ms/step | 15.2 samples/s +Step 12420/44000 | Loss: 3.1145 | LR: 4.15e-04 | 1319.5ms/step | 15.2 samples/s +Step 12430/44000 | Loss: 3.1277 | LR: 4.15e-04 | 1317.5ms/step | 15.2 samples/s +Step 12440/44000 | Loss: 3.1097 | LR: 4.15e-04 | 1318.7ms/step | 15.2 samples/s +Step 12450/44000 | Loss: 3.0768 | LR: 4.15e-04 | 1319.8ms/step | 15.2 samples/s +Step 12460/44000 | Loss: 3.0948 | LR: 4.15e-04 | 1319.9ms/step | 15.2 samples/s +Step 12470/44000 | Loss: 3.0826 | LR: 4.15e-04 | 1318.4ms/step | 15.2 samples/s +Step 12480/44000 | Loss: 3.1292 | LR: 4.15e-04 | 1319.5ms/step | 15.2 samples/s +Step 12490/44000 | Loss: 3.1209 | LR: 4.14e-04 | 1316.6ms/step | 15.2 samples/s +Step 12500/44000 | Loss: 3.0794 | LR: 4.14e-04 | 1319.4ms/step | 15.2 samples/s +Validation at step 12500: Loss = 3.1079, Perplexity = 22.37 +Step 12510/44000 | Loss: 3.1090 | LR: 4.14e-04 | 3526.2ms/step | 5.7 samples/s +Step 12520/44000 | Loss: 3.0972 | LR: 4.14e-04 | 1319.2ms/step | 15.2 samples/s +Step 12530/44000 | Loss: 3.0787 | LR: 4.14e-04 | 1319.3ms/step | 15.2 samples/s +Step 12540/44000 | Loss: 3.0827 | LR: 4.14e-04 | 1320.5ms/step | 15.1 samples/s +Step 12550/44000 | Loss: 3.0816 | LR: 4.14e-04 | 1322.2ms/step | 15.1 samples/s +Step 12560/44000 | Loss: 3.1087 | LR: 4.14e-04 | 1321.1ms/step | 15.1 samples/s +Step 12570/44000 | Loss: 3.1033 | LR: 4.13e-04 | 1319.3ms/step | 15.2 samples/s +Step 12580/44000 | Loss: 3.0998 | LR: 4.13e-04 | 1319.2ms/step | 15.2 samples/s +Step 12590/44000 | Loss: 3.0997 | LR: 4.13e-04 | 1317.8ms/step | 15.2 samples/s +Step 12600/44000 | Loss: 3.0928 | LR: 4.13e-04 | 1320.5ms/step | 15.1 samples/s +Step 12610/44000 | Loss: 3.1500 | LR: 4.13e-04 | 1318.9ms/step | 15.2 samples/s +Step 12620/44000 | Loss: 3.1090 | LR: 4.13e-04 | 1317.0ms/step | 15.2 samples/s +Step 12630/44000 | Loss: 3.1268 | LR: 4.13e-04 | 1317.0ms/step | 15.2 samples/s +Step 12640/44000 | Loss: 3.0893 | LR: 4.12e-04 | 1319.6ms/step | 15.2 samples/s +Step 12650/44000 | Loss: 3.0970 | LR: 4.12e-04 | 1320.6ms/step | 15.1 samples/s +Step 12660/44000 | Loss: 3.0946 | LR: 4.12e-04 | 1318.1ms/step | 15.2 samples/s +Step 12670/44000 | Loss: 3.1063 | LR: 4.12e-04 | 1322.3ms/step | 15.1 samples/s +Step 12680/44000 | Loss: 3.0978 | LR: 4.12e-04 | 1318.9ms/step | 15.2 samples/s +Step 12690/44000 | Loss: 3.1041 | LR: 4.12e-04 | 1318.7ms/step | 15.2 samples/s +Step 12700/44000 | Loss: 3.1306 | LR: 4.12e-04 | 1319.8ms/step | 15.2 samples/s +Step 12710/44000 | Loss: 3.0972 | LR: 4.11e-04 | 1319.4ms/step | 15.2 samples/s +Step 12720/44000 | Loss: 3.1071 | LR: 4.11e-04 | 1320.5ms/step | 15.1 samples/s +Step 12730/44000 | Loss: 3.0953 | LR: 4.11e-04 | 1318.9ms/step | 15.2 samples/s +Step 12740/44000 | Loss: 3.0893 | LR: 4.11e-04 | 1320.1ms/step | 15.2 samples/s +Step 12750/44000 | Loss: 3.1064 | LR: 4.11e-04 | 1321.0ms/step | 15.1 samples/s +Validation at step 12750: Loss = 3.1040, Perplexity = 22.29 +Step 12760/44000 | Loss: 3.0998 | LR: 4.11e-04 | 3530.4ms/step | 5.7 samples/s +Step 12770/44000 | Loss: 3.0746 | LR: 4.11e-04 | 1318.8ms/step | 15.2 samples/s +Step 12780/44000 | Loss: 3.0739 | LR: 4.10e-04 | 1317.5ms/step | 15.2 samples/s +Step 12790/44000 | Loss: 3.0992 | LR: 4.10e-04 | 1320.5ms/step | 15.1 samples/s +Step 12800/44000 | Loss: 3.0994 | LR: 4.10e-04 | 1321.8ms/step | 15.1 samples/s +Step 12810/44000 | Loss: 3.1367 | LR: 4.10e-04 | 1321.4ms/step | 15.1 samples/s +Step 12820/44000 | Loss: 3.1040 | LR: 4.10e-04 | 1319.6ms/step | 15.2 samples/s +Step 12830/44000 | Loss: 3.1136 | LR: 4.10e-04 | 1318.3ms/step | 15.2 samples/s +Step 12840/44000 | Loss: 3.0905 | LR: 4.10e-04 | 1316.1ms/step | 15.2 samples/s +Step 12850/44000 | Loss: 3.1000 | LR: 4.09e-04 | 1318.5ms/step | 15.2 samples/s +Step 12860/44000 | Loss: 3.1063 | LR: 4.09e-04 | 1320.2ms/step | 15.1 samples/s +Step 12870/44000 | Loss: 3.1128 | LR: 4.09e-04 | 1322.0ms/step | 15.1 samples/s +Step 12880/44000 | Loss: 3.0744 | LR: 4.09e-04 | 1318.7ms/step | 15.2 samples/s +Step 12890/44000 | Loss: 3.0995 | LR: 4.09e-04 | 1318.7ms/step | 15.2 samples/s +Step 12900/44000 | Loss: 3.0951 | LR: 4.09e-04 | 1320.4ms/step | 15.1 samples/s +Step 12910/44000 | Loss: 3.0967 | LR: 4.09e-04 | 1320.6ms/step | 15.1 samples/s +Step 12920/44000 | Loss: 3.0956 | LR: 4.09e-04 | 1321.0ms/step | 15.1 samples/s +Step 12930/44000 | Loss: 3.0871 | LR: 4.08e-04 | 1318.8ms/step | 15.2 samples/s +Step 12940/44000 | Loss: 3.1171 | LR: 4.08e-04 | 1321.4ms/step | 15.1 samples/s +Step 12950/44000 | Loss: 3.0989 | LR: 4.08e-04 | 1316.3ms/step | 15.2 samples/s +Step 12960/44000 | Loss: 3.1388 | LR: 4.08e-04 | 1314.8ms/step | 15.2 samples/s +Step 12970/44000 | Loss: 3.0937 | LR: 4.08e-04 | 1321.2ms/step | 15.1 samples/s +Step 12980/44000 | Loss: 3.0970 | LR: 4.08e-04 | 1318.3ms/step | 15.2 samples/s +Step 12990/44000 | Loss: 3.0815 | LR: 4.08e-04 | 1315.1ms/step | 15.2 samples/s +Step 13000/44000 | Loss: 3.1064 | LR: 4.07e-04 | 1317.0ms/step | 15.2 samples/s +Validation at step 13000: Loss = 3.0999, Perplexity = 22.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-13000 +Step 13010/44000 | Loss: 3.0690 | LR: 4.07e-04 | 3851.3ms/step | 5.2 samples/s +Step 13020/44000 | Loss: 3.0829 | LR: 4.07e-04 | 1318.6ms/step | 15.2 samples/s +Step 13030/44000 | Loss: 3.0830 | LR: 4.07e-04 | 1320.5ms/step | 15.1 samples/s +Step 13040/44000 | Loss: 3.0678 | LR: 4.07e-04 | 1319.9ms/step | 15.2 samples/s +Step 13050/44000 | Loss: 3.0960 | LR: 4.07e-04 | 1319.9ms/step | 15.2 samples/s +Step 13060/44000 | Loss: 3.1067 | LR: 4.07e-04 | 1316.2ms/step | 15.2 samples/s +Step 13070/44000 | Loss: 3.0735 | LR: 4.06e-04 | 1320.2ms/step | 15.1 samples/s +Step 13080/44000 | Loss: 3.0511 | LR: 4.06e-04 | 1323.9ms/step | 15.1 samples/s +Step 13090/44000 | Loss: 3.1060 | LR: 4.06e-04 | 1321.5ms/step | 15.1 samples/s +Step 13100/44000 | Loss: 3.0925 | LR: 4.06e-04 | 1318.6ms/step | 15.2 samples/s +Step 13110/44000 | Loss: 3.0728 | LR: 4.06e-04 | 1321.4ms/step | 15.1 samples/s +Step 13120/44000 | Loss: 3.0601 | LR: 4.06e-04 | 1322.2ms/step | 15.1 samples/s +Step 13130/44000 | Loss: 3.0997 | LR: 4.06e-04 | 1318.8ms/step | 15.2 samples/s +Step 13140/44000 | Loss: 3.1256 | LR: 4.05e-04 | 1319.2ms/step | 15.2 samples/s +Step 13150/44000 | Loss: 3.0903 | LR: 4.05e-04 | 1318.2ms/step | 15.2 samples/s +Step 13160/44000 | Loss: 3.1035 | LR: 4.05e-04 | 1317.1ms/step | 15.2 samples/s +Step 13170/44000 | Loss: 3.0899 | LR: 4.05e-04 | 1318.1ms/step | 15.2 samples/s +Step 13180/44000 | Loss: 3.0940 | LR: 4.05e-04 | 1320.2ms/step | 15.1 samples/s +Step 13190/44000 | Loss: 3.1384 | LR: 4.05e-04 | 1319.1ms/step | 15.2 samples/s +Step 13200/44000 | Loss: 3.0964 | LR: 4.05e-04 | 1321.5ms/step | 15.1 samples/s +Step 13210/44000 | Loss: 3.0929 | LR: 4.04e-04 | 1316.9ms/step | 15.2 samples/s +Step 13220/44000 | Loss: 3.0959 | LR: 4.04e-04 | 1318.8ms/step | 15.2 samples/s +Step 13230/44000 | Loss: 3.0719 | LR: 4.04e-04 | 1320.0ms/step | 15.2 samples/s +Step 13240/44000 | Loss: 3.1028 | LR: 4.04e-04 | 1321.0ms/step | 15.1 samples/s +Step 13250/44000 | Loss: 3.1198 | LR: 4.04e-04 | 1320.7ms/step | 15.1 samples/s +Validation at step 13250: Loss = 3.0966, Perplexity = 22.12 +Step 13260/44000 | Loss: 3.0856 | LR: 4.04e-04 | 3532.0ms/step | 5.7 samples/s +Step 13270/44000 | Loss: 3.1163 | LR: 4.04e-04 | 1318.0ms/step | 15.2 samples/s +Step 13280/44000 | Loss: 3.1065 | LR: 4.03e-04 | 1317.6ms/step | 15.2 samples/s +Step 13290/44000 | Loss: 3.1254 | LR: 4.03e-04 | 1316.7ms/step | 15.2 samples/s +Step 13300/44000 | Loss: 3.0939 | LR: 4.03e-04 | 1315.5ms/step | 15.2 samples/s +Step 13310/44000 | Loss: 3.0798 | LR: 4.03e-04 | 1313.8ms/step | 15.2 samples/s +Step 13320/44000 | Loss: 3.0952 | LR: 4.03e-04 | 1314.7ms/step | 15.2 samples/s +Step 13330/44000 | Loss: 3.1268 | LR: 4.03e-04 | 1315.5ms/step | 15.2 samples/s +Step 13340/44000 | Loss: 3.1314 | LR: 4.03e-04 | 1316.0ms/step | 15.2 samples/s +Step 13350/44000 | Loss: 3.0692 | LR: 4.02e-04 | 1314.6ms/step | 15.2 samples/s +Step 13360/44000 | Loss: 3.0917 | LR: 4.02e-04 | 1316.8ms/step | 15.2 samples/s +Step 13370/44000 | Loss: 3.0963 | LR: 4.02e-04 | 1318.6ms/step | 15.2 samples/s +Step 13380/44000 | Loss: 3.0955 | LR: 4.02e-04 | 1317.7ms/step | 15.2 samples/s +Step 13390/44000 | Loss: 3.0614 | LR: 4.02e-04 | 1319.0ms/step | 15.2 samples/s +Step 13400/44000 | Loss: 3.1141 | LR: 4.02e-04 | 1321.4ms/step | 15.1 samples/s +Step 13410/44000 | Loss: 3.0895 | LR: 4.02e-04 | 1321.0ms/step | 15.1 samples/s +Step 13420/44000 | Loss: 3.0719 | LR: 4.01e-04 | 1318.8ms/step | 15.2 samples/s +Step 13430/44000 | Loss: 3.1052 | LR: 4.01e-04 | 1318.6ms/step | 15.2 samples/s +Step 13440/44000 | Loss: 3.1043 | LR: 4.01e-04 | 1320.0ms/step | 15.2 samples/s +Step 13450/44000 | Loss: 3.0867 | LR: 4.01e-04 | 1320.3ms/step | 15.1 samples/s +Step 13460/44000 | Loss: 3.0854 | LR: 4.01e-04 | 1322.3ms/step | 15.1 samples/s +Step 13470/44000 | Loss: 3.1093 | LR: 4.01e-04 | 1320.8ms/step | 15.1 samples/s +Step 13480/44000 | Loss: 3.0739 | LR: 4.01e-04 | 1320.4ms/step | 15.1 samples/s +Step 13490/44000 | Loss: 3.0972 | LR: 4.00e-04 | 1320.5ms/step | 15.1 samples/s +Step 13500/44000 | Loss: 3.0937 | LR: 4.00e-04 | 1320.9ms/step | 15.1 samples/s +Validation at step 13500: Loss = 3.0922, Perplexity = 22.02 +Step 13510/44000 | Loss: 3.0805 | LR: 4.00e-04 | 3527.9ms/step | 5.7 samples/s +Step 13520/44000 | Loss: 3.0818 | LR: 4.00e-04 | 1320.3ms/step | 15.1 samples/s +Step 13530/44000 | Loss: 3.0791 | LR: 4.00e-04 | 1321.6ms/step | 15.1 samples/s +Step 13540/44000 | Loss: 3.0666 | LR: 4.00e-04 | 1321.6ms/step | 15.1 samples/s +Step 13550/44000 | Loss: 3.0821 | LR: 4.00e-04 | 1319.5ms/step | 15.2 samples/s +Step 13560/44000 | Loss: 3.0897 | LR: 3.99e-04 | 1321.5ms/step | 15.1 samples/s +Step 13570/44000 | Loss: 3.1063 | LR: 3.99e-04 | 1318.3ms/step | 15.2 samples/s +Step 13580/44000 | Loss: 3.1021 | LR: 3.99e-04 | 1318.3ms/step | 15.2 samples/s +Step 13590/44000 | Loss: 3.1061 | LR: 3.99e-04 | 1318.0ms/step | 15.2 samples/s +Step 13600/44000 | Loss: 3.0779 | LR: 3.99e-04 | 1323.1ms/step | 15.1 samples/s +Step 13610/44000 | Loss: 3.0892 | LR: 3.99e-04 | 1319.5ms/step | 15.2 samples/s +Step 13620/44000 | Loss: 3.0885 | LR: 3.98e-04 | 1319.2ms/step | 15.2 samples/s +Step 13630/44000 | Loss: 3.0797 | LR: 3.98e-04 | 1317.1ms/step | 15.2 samples/s +Step 13640/44000 | Loss: 3.0873 | LR: 3.98e-04 | 1319.9ms/step | 15.2 samples/s +Step 13650/44000 | Loss: 3.0742 | LR: 3.98e-04 | 1320.0ms/step | 15.2 samples/s +Step 13660/44000 | Loss: 3.0710 | LR: 3.98e-04 | 1322.3ms/step | 15.1 samples/s +Step 13670/44000 | Loss: 3.1232 | LR: 3.98e-04 | 1319.4ms/step | 15.2 samples/s +Step 13680/44000 | Loss: 3.0684 | LR: 3.98e-04 | 1320.3ms/step | 15.1 samples/s +Step 13690/44000 | Loss: 3.0738 | LR: 3.97e-04 | 1320.0ms/step | 15.2 samples/s +Step 13700/44000 | Loss: 3.1016 | LR: 3.97e-04 | 1320.5ms/step | 15.1 samples/s +Step 13710/44000 | Loss: 3.1060 | LR: 3.97e-04 | 1318.0ms/step | 15.2 samples/s +Step 13720/44000 | Loss: 3.0980 | LR: 3.97e-04 | 1320.0ms/step | 15.2 samples/s +Step 13730/44000 | Loss: 3.0981 | LR: 3.97e-04 | 1319.4ms/step | 15.2 samples/s +Step 13740/44000 | Loss: 3.0459 | LR: 3.97e-04 | 1319.2ms/step | 15.2 samples/s +Step 13750/44000 | Loss: 3.1012 | LR: 3.97e-04 | 1320.1ms/step | 15.1 samples/s +Validation at step 13750: Loss = 3.0886, Perplexity = 21.95 +Step 13760/44000 | Loss: 3.1186 | LR: 3.96e-04 | 3538.1ms/step | 5.7 samples/s +Step 13770/44000 | Loss: 3.0822 | LR: 3.96e-04 | 1319.6ms/step | 15.2 samples/s +Step 13780/44000 | Loss: 3.0945 | LR: 3.96e-04 | 1317.7ms/step | 15.2 samples/s +Step 13790/44000 | Loss: 3.0949 | LR: 3.96e-04 | 1318.0ms/step | 15.2 samples/s +Step 13800/44000 | Loss: 3.0892 | LR: 3.96e-04 | 1319.1ms/step | 15.2 samples/s +Step 13810/44000 | Loss: 3.0764 | LR: 3.96e-04 | 1318.7ms/step | 15.2 samples/s +Step 13820/44000 | Loss: 3.1033 | LR: 3.96e-04 | 1318.5ms/step | 15.2 samples/s +Step 13830/44000 | Loss: 3.0945 | LR: 3.95e-04 | 1316.9ms/step | 15.2 samples/s +Step 13840/44000 | Loss: 3.0978 | LR: 3.95e-04 | 1317.4ms/step | 15.2 samples/s +Step 13850/44000 | Loss: 3.0874 | LR: 3.95e-04 | 1319.5ms/step | 15.2 samples/s +Step 13860/44000 | Loss: 3.0710 | LR: 3.95e-04 | 1318.1ms/step | 15.2 samples/s +Step 13870/44000 | Loss: 3.0576 | LR: 3.95e-04 | 1318.9ms/step | 15.2 samples/s +Step 13880/44000 | Loss: 3.0896 | LR: 3.95e-04 | 1317.9ms/step | 15.2 samples/s +Step 13890/44000 | Loss: 3.0733 | LR: 3.95e-04 | 1321.0ms/step | 15.1 samples/s +Step 13900/44000 | Loss: 3.0996 | LR: 3.94e-04 | 1321.9ms/step | 15.1 samples/s +Step 13910/44000 | Loss: 3.0971 | LR: 3.94e-04 | 1322.1ms/step | 15.1 samples/s +Step 13920/44000 | Loss: 3.0705 | LR: 3.94e-04 | 1321.7ms/step | 15.1 samples/s +Step 13930/44000 | Loss: 3.0748 | LR: 3.94e-04 | 1320.5ms/step | 15.1 samples/s +Step 13940/44000 | Loss: 3.0677 | LR: 3.94e-04 | 1320.9ms/step | 15.1 samples/s +Step 13950/44000 | Loss: 3.0782 | LR: 3.94e-04 | 1324.1ms/step | 15.1 samples/s +Step 13960/44000 | Loss: 3.0816 | LR: 3.93e-04 | 1327.6ms/step | 15.1 samples/s +Step 13970/44000 | Loss: 3.1061 | LR: 3.93e-04 | 1324.2ms/step | 15.1 samples/s +Step 13980/44000 | Loss: 3.0347 | LR: 3.93e-04 | 1323.1ms/step | 15.1 samples/s +Step 13990/44000 | Loss: 3.0789 | LR: 3.93e-04 | 1322.1ms/step | 15.1 samples/s +Step 14000/44000 | Loss: 3.1126 | LR: 3.93e-04 | 1318.5ms/step | 15.2 samples/s +Validation at step 14000: Loss = 3.0843, Perplexity = 21.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-14000 +Step 14010/44000 | Loss: 3.0618 | LR: 3.93e-04 | 3858.2ms/step | 5.2 samples/s +Step 14020/44000 | Loss: 3.0792 | LR: 3.93e-04 | 1318.1ms/step | 15.2 samples/s +Step 14030/44000 | Loss: 3.0669 | LR: 3.92e-04 | 1318.6ms/step | 15.2 samples/s +Step 14040/44000 | Loss: 3.0571 | LR: 3.92e-04 | 1319.8ms/step | 15.2 samples/s +Step 14050/44000 | Loss: 3.0623 | LR: 3.92e-04 | 1319.4ms/step | 15.2 samples/s +Step 14060/44000 | Loss: 3.0958 | LR: 3.92e-04 | 1321.0ms/step | 15.1 samples/s +Step 14070/44000 | Loss: 3.0908 | LR: 3.92e-04 | 1320.5ms/step | 15.1 samples/s +Step 14080/44000 | Loss: 3.1161 | LR: 3.92e-04 | 1320.6ms/step | 15.1 samples/s +Step 14090/44000 | Loss: 3.0594 | LR: 3.92e-04 | 1318.8ms/step | 15.2 samples/s +Step 14100/44000 | Loss: 3.1059 | LR: 3.91e-04 | 1319.2ms/step | 15.2 samples/s +Step 14110/44000 | Loss: 3.0742 | LR: 3.91e-04 | 1318.3ms/step | 15.2 samples/s +Step 14120/44000 | Loss: 3.0716 | LR: 3.91e-04 | 1318.8ms/step | 15.2 samples/s +Step 14130/44000 | Loss: 3.0910 | LR: 3.91e-04 | 1317.2ms/step | 15.2 samples/s +Step 14140/44000 | Loss: 3.0859 | LR: 3.91e-04 | 1321.2ms/step | 15.1 samples/s +Step 14150/44000 | Loss: 3.0962 | LR: 3.91e-04 | 1318.5ms/step | 15.2 samples/s +Step 14160/44000 | Loss: 3.0908 | LR: 3.90e-04 | 1318.1ms/step | 15.2 samples/s +Step 14170/44000 | Loss: 3.0975 | LR: 3.90e-04 | 1320.0ms/step | 15.2 samples/s +Step 14180/44000 | Loss: 3.0730 | LR: 3.90e-04 | 1319.8ms/step | 15.2 samples/s +Step 14190/44000 | Loss: 3.1024 | LR: 3.90e-04 | 1316.2ms/step | 15.2 samples/s +Step 14200/44000 | Loss: 3.0784 | LR: 3.90e-04 | 1319.7ms/step | 15.2 samples/s +Step 14210/44000 | Loss: 3.0504 | LR: 3.90e-04 | 1318.8ms/step | 15.2 samples/s +Step 14220/44000 | Loss: 3.0884 | LR: 3.90e-04 | 1318.6ms/step | 15.2 samples/s +Step 14230/44000 | Loss: 3.0756 | LR: 3.89e-04 | 1318.7ms/step | 15.2 samples/s +Step 14240/44000 | Loss: 3.0525 | LR: 3.89e-04 | 1320.5ms/step | 15.1 samples/s +Step 14250/44000 | Loss: 3.0724 | LR: 3.89e-04 | 1319.4ms/step | 15.2 samples/s +Validation at step 14250: Loss = 3.0820, Perplexity = 21.80 +Step 14260/44000 | Loss: 3.0760 | LR: 3.89e-04 | 3533.9ms/step | 5.7 samples/s +Step 14270/44000 | Loss: 3.0976 | LR: 3.89e-04 | 1319.4ms/step | 15.2 samples/s +Step 14280/44000 | Loss: 3.0719 | LR: 3.89e-04 | 1319.0ms/step | 15.2 samples/s +Step 14290/44000 | Loss: 3.1094 | LR: 3.89e-04 | 1320.1ms/step | 15.2 samples/s +Step 14300/44000 | Loss: 3.1089 | LR: 3.88e-04 | 1318.7ms/step | 15.2 samples/s +Step 14310/44000 | Loss: 3.0639 | LR: 3.88e-04 | 1321.3ms/step | 15.1 samples/s +Step 14320/44000 | Loss: 3.0560 | LR: 3.88e-04 | 1315.0ms/step | 15.2 samples/s +Step 14330/44000 | Loss: 3.1073 | LR: 3.88e-04 | 1317.9ms/step | 15.2 samples/s +Step 14340/44000 | Loss: 3.0947 | LR: 3.88e-04 | 1319.2ms/step | 15.2 samples/s +Step 14350/44000 | Loss: 3.0780 | LR: 3.88e-04 | 1319.2ms/step | 15.2 samples/s +Step 14360/44000 | Loss: 3.0828 | LR: 3.87e-04 | 1320.5ms/step | 15.1 samples/s +Step 14370/44000 | Loss: 3.0604 | LR: 3.87e-04 | 1322.0ms/step | 15.1 samples/s +Step 14380/44000 | Loss: 3.0885 | LR: 3.87e-04 | 1320.5ms/step | 15.1 samples/s +Step 14390/44000 | Loss: 3.0900 | LR: 3.87e-04 | 1319.1ms/step | 15.2 samples/s +Step 14400/44000 | Loss: 3.0443 | LR: 3.87e-04 | 1321.0ms/step | 15.1 samples/s +Step 14410/44000 | Loss: 3.1042 | LR: 3.87e-04 | 1319.3ms/step | 15.2 samples/s +Step 14420/44000 | Loss: 3.0790 | LR: 3.87e-04 | 1322.0ms/step | 15.1 samples/s +Step 14430/44000 | Loss: 3.0878 | LR: 3.86e-04 | 1320.4ms/step | 15.1 samples/s +Step 14440/44000 | Loss: 3.0731 | LR: 3.86e-04 | 1321.5ms/step | 15.1 samples/s +Step 14450/44000 | Loss: 3.0884 | LR: 3.86e-04 | 1318.0ms/step | 15.2 samples/s +Step 14460/44000 | Loss: 3.0776 | LR: 3.86e-04 | 1320.3ms/step | 15.1 samples/s +Step 14470/44000 | Loss: 3.0728 | LR: 3.86e-04 | 1317.8ms/step | 15.2 samples/s +Step 14480/44000 | Loss: 3.0626 | LR: 3.86e-04 | 1318.4ms/step | 15.2 samples/s +Step 14490/44000 | Loss: 3.0727 | LR: 3.85e-04 | 1317.8ms/step | 15.2 samples/s +Step 14500/44000 | Loss: 3.0663 | LR: 3.85e-04 | 1317.0ms/step | 15.2 samples/s +Validation at step 14500: Loss = 3.0767, Perplexity = 21.69 +Step 14510/44000 | Loss: 3.0765 | LR: 3.85e-04 | 3536.2ms/step | 5.7 samples/s +Step 14520/44000 | Loss: 3.0772 | LR: 3.85e-04 | 1321.3ms/step | 15.1 samples/s +Step 14530/44000 | Loss: 3.0979 | LR: 3.85e-04 | 1317.8ms/step | 15.2 samples/s +Step 14540/44000 | Loss: 3.0689 | LR: 3.85e-04 | 1319.6ms/step | 15.2 samples/s +Step 14550/44000 | Loss: 3.0809 | LR: 3.85e-04 | 1317.5ms/step | 15.2 samples/s +Step 14560/44000 | Loss: 3.0497 | LR: 3.84e-04 | 1318.7ms/step | 15.2 samples/s +Step 14570/44000 | Loss: 3.0919 | LR: 3.84e-04 | 1320.3ms/step | 15.1 samples/s +Step 14580/44000 | Loss: 3.0745 | LR: 3.84e-04 | 1319.5ms/step | 15.2 samples/s +Step 14590/44000 | Loss: 3.0771 | LR: 3.84e-04 | 1319.6ms/step | 15.2 samples/s +Step 14600/44000 | Loss: 3.0654 | LR: 3.84e-04 | 1320.2ms/step | 15.1 samples/s +Step 14610/44000 | Loss: 3.0238 | LR: 3.84e-04 | 1320.9ms/step | 15.1 samples/s +Step 14620/44000 | Loss: 3.0666 | LR: 3.83e-04 | 1320.5ms/step | 15.1 samples/s +Step 14630/44000 | Loss: 3.0529 | LR: 3.83e-04 | 1318.5ms/step | 15.2 samples/s +Step 14640/44000 | Loss: 3.0445 | LR: 3.83e-04 | 1321.1ms/step | 15.1 samples/s +Step 14650/44000 | Loss: 3.0908 | LR: 3.83e-04 | 1316.7ms/step | 15.2 samples/s +Step 14660/44000 | Loss: 3.0485 | LR: 3.83e-04 | 1316.5ms/step | 15.2 samples/s +Step 14670/44000 | Loss: 3.0691 | LR: 3.83e-04 | 1319.8ms/step | 15.2 samples/s +Step 14680/44000 | Loss: 3.1050 | LR: 3.83e-04 | 1318.6ms/step | 15.2 samples/s +Step 14690/44000 | Loss: 3.1074 | LR: 3.82e-04 | 1320.5ms/step | 15.1 samples/s +Step 14700/44000 | Loss: 3.0743 | LR: 3.82e-04 | 1319.7ms/step | 15.2 samples/s +Step 14710/44000 | Loss: 3.0291 | LR: 3.82e-04 | 1318.8ms/step | 15.2 samples/s +Step 14720/44000 | Loss: 3.0684 | LR: 3.82e-04 | 1317.0ms/step | 15.2 samples/s +Step 14730/44000 | Loss: 3.0744 | LR: 3.82e-04 | 1316.0ms/step | 15.2 samples/s +Step 14740/44000 | Loss: 3.0913 | LR: 3.82e-04 | 1317.5ms/step | 15.2 samples/s +Step 14750/44000 | Loss: 3.0736 | LR: 3.81e-04 | 1316.5ms/step | 15.2 samples/s +Validation at step 14750: Loss = 3.0737, Perplexity = 21.62 +Step 14760/44000 | Loss: 3.0790 | LR: 3.81e-04 | 3536.5ms/step | 5.7 samples/s +Step 14770/44000 | Loss: 3.0711 | LR: 3.81e-04 | 1321.2ms/step | 15.1 samples/s +Step 14780/44000 | Loss: 3.0801 | LR: 3.81e-04 | 1319.1ms/step | 15.2 samples/s +Step 14790/44000 | Loss: 3.0646 | LR: 3.81e-04 | 1318.5ms/step | 15.2 samples/s +Step 14800/44000 | Loss: 3.0684 | LR: 3.81e-04 | 1319.8ms/step | 15.2 samples/s +Step 14810/44000 | Loss: 3.0625 | LR: 3.81e-04 | 1318.8ms/step | 15.2 samples/s +Step 14820/44000 | Loss: 3.0938 | LR: 3.80e-04 | 1319.5ms/step | 15.2 samples/s +Step 14830/44000 | Loss: 3.0516 | LR: 3.80e-04 | 1321.5ms/step | 15.1 samples/s +Step 14840/44000 | Loss: 3.0978 | LR: 3.80e-04 | 1320.0ms/step | 15.2 samples/s +Step 14850/44000 | Loss: 3.0541 | LR: 3.80e-04 | 1317.9ms/step | 15.2 samples/s +Step 14860/44000 | Loss: 3.0712 | LR: 3.80e-04 | 1318.9ms/step | 15.2 samples/s +Step 14870/44000 | Loss: 3.0808 | LR: 3.80e-04 | 1318.1ms/step | 15.2 samples/s +Step 14880/44000 | Loss: 3.0813 | LR: 3.79e-04 | 1317.8ms/step | 15.2 samples/s +Step 14890/44000 | Loss: 3.0644 | LR: 3.79e-04 | 1316.0ms/step | 15.2 samples/s +Step 14900/44000 | Loss: 3.0686 | LR: 3.79e-04 | 1319.0ms/step | 15.2 samples/s +Step 14910/44000 | Loss: 3.0491 | LR: 3.79e-04 | 1319.7ms/step | 15.2 samples/s +Step 14920/44000 | Loss: 3.0916 | LR: 3.79e-04 | 1318.1ms/step | 15.2 samples/s +Step 14930/44000 | Loss: 3.0678 | LR: 3.79e-04 | 1321.5ms/step | 15.1 samples/s +Step 14940/44000 | Loss: 3.0917 | LR: 3.79e-04 | 1321.8ms/step | 15.1 samples/s +Step 14950/44000 | Loss: 3.0616 | LR: 3.78e-04 | 1319.7ms/step | 15.2 samples/s +Step 14960/44000 | Loss: 3.1056 | LR: 3.78e-04 | 1319.0ms/step | 15.2 samples/s +Step 14970/44000 | Loss: 3.0694 | LR: 3.78e-04 | 1317.6ms/step | 15.2 samples/s +Step 14980/44000 | Loss: 3.0774 | LR: 3.78e-04 | 1321.4ms/step | 15.1 samples/s +Step 14990/44000 | Loss: 3.0776 | LR: 3.78e-04 | 1318.3ms/step | 15.2 samples/s +Step 15000/44000 | Loss: 3.0600 | LR: 3.78e-04 | 1321.4ms/step | 15.1 samples/s +Validation at step 15000: Loss = 3.0699, Perplexity = 21.54 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-15000 +Step 15010/44000 | Loss: 3.0983 | LR: 3.77e-04 | 3841.5ms/step | 5.2 samples/s +Step 15020/44000 | Loss: 3.0792 | LR: 3.77e-04 | 1318.9ms/step | 15.2 samples/s +Step 15030/44000 | Loss: 3.0984 | LR: 3.77e-04 | 1319.3ms/step | 15.2 samples/s +Step 15040/44000 | Loss: 3.0320 | LR: 3.77e-04 | 1322.0ms/step | 15.1 samples/s +Step 15050/44000 | Loss: 3.0406 | LR: 3.77e-04 | 1319.6ms/step | 15.2 samples/s +Step 15060/44000 | Loss: 3.0596 | LR: 3.77e-04 | 1320.1ms/step | 15.2 samples/s +Step 15070/44000 | Loss: 3.0613 | LR: 3.77e-04 | 1317.9ms/step | 15.2 samples/s +Step 15080/44000 | Loss: 3.0544 | LR: 3.76e-04 | 1319.0ms/step | 15.2 samples/s +Step 15090/44000 | Loss: 3.0524 | LR: 3.76e-04 | 1318.8ms/step | 15.2 samples/s +Step 15100/44000 | Loss: 3.0358 | LR: 3.76e-04 | 1319.1ms/step | 15.2 samples/s +Step 15110/44000 | Loss: 3.0701 | LR: 3.76e-04 | 1319.5ms/step | 15.2 samples/s +Step 15120/44000 | Loss: 3.0581 | LR: 3.76e-04 | 1321.1ms/step | 15.1 samples/s +Step 15130/44000 | Loss: 3.0591 | LR: 3.76e-04 | 1321.9ms/step | 15.1 samples/s +Step 15140/44000 | Loss: 3.0500 | LR: 3.75e-04 | 1319.0ms/step | 15.2 samples/s +Step 15150/44000 | Loss: 3.0362 | LR: 3.75e-04 | 1319.3ms/step | 15.2 samples/s +Step 15160/44000 | Loss: 3.0433 | LR: 3.75e-04 | 1321.0ms/step | 15.1 samples/s +Step 15170/44000 | Loss: 3.0514 | LR: 3.75e-04 | 1322.2ms/step | 15.1 samples/s +Step 15180/44000 | Loss: 3.0591 | LR: 3.75e-04 | 1317.3ms/step | 15.2 samples/s +Step 15190/44000 | Loss: 3.0771 | LR: 3.75e-04 | 1318.1ms/step | 15.2 samples/s +Step 15200/44000 | Loss: 3.0737 | LR: 3.74e-04 | 1316.1ms/step | 15.2 samples/s +Step 15210/44000 | Loss: 3.0517 | LR: 3.74e-04 | 1314.3ms/step | 15.2 samples/s +Step 15220/44000 | Loss: 3.0605 | LR: 3.74e-04 | 1315.4ms/step | 15.2 samples/s +Step 15230/44000 | Loss: 3.0491 | LR: 3.74e-04 | 1315.5ms/step | 15.2 samples/s +Step 15240/44000 | Loss: 3.0465 | LR: 3.74e-04 | 1317.9ms/step | 15.2 samples/s +Step 15250/44000 | Loss: 3.0286 | LR: 3.74e-04 | 1317.4ms/step | 15.2 samples/s +Validation at step 15250: Loss = 3.0669, Perplexity = 21.48 +Step 15260/44000 | Loss: 3.0654 | LR: 3.74e-04 | 3527.9ms/step | 5.7 samples/s +Step 15270/44000 | Loss: 3.0723 | LR: 3.73e-04 | 1317.9ms/step | 15.2 samples/s +Step 15280/44000 | Loss: 3.0420 | LR: 3.73e-04 | 1316.5ms/step | 15.2 samples/s +Step 15290/44000 | Loss: 3.0597 | LR: 3.73e-04 | 1321.0ms/step | 15.1 samples/s +Step 15300/44000 | Loss: 3.0744 | LR: 3.73e-04 | 1320.9ms/step | 15.1 samples/s +Step 15310/44000 | Loss: 3.0989 | LR: 3.73e-04 | 1320.5ms/step | 15.1 samples/s +Step 15320/44000 | Loss: 3.0638 | LR: 3.73e-04 | 1319.4ms/step | 15.2 samples/s +Step 15330/44000 | Loss: 3.0720 | LR: 3.72e-04 | 1320.3ms/step | 15.1 samples/s +Step 15340/44000 | Loss: 3.0416 | LR: 3.72e-04 | 1321.3ms/step | 15.1 samples/s +Step 15350/44000 | Loss: 3.0651 | LR: 3.72e-04 | 1319.3ms/step | 15.2 samples/s +Step 15360/44000 | Loss: 3.0471 | LR: 3.72e-04 | 1320.1ms/step | 15.1 samples/s +Step 15370/44000 | Loss: 3.0642 | LR: 3.72e-04 | 1317.4ms/step | 15.2 samples/s +Step 15380/44000 | Loss: 3.0562 | LR: 3.72e-04 | 1320.1ms/step | 15.2 samples/s +Step 15390/44000 | Loss: 3.0595 | LR: 3.71e-04 | 1321.4ms/step | 15.1 samples/s +Step 15400/44000 | Loss: 3.0766 | LR: 3.71e-04 | 1319.9ms/step | 15.2 samples/s +Step 15410/44000 | Loss: 3.0489 | LR: 3.71e-04 | 1319.7ms/step | 15.2 samples/s +Step 15420/44000 | Loss: 3.0803 | LR: 3.71e-04 | 1318.3ms/step | 15.2 samples/s +Step 15430/44000 | Loss: 3.0705 | LR: 3.71e-04 | 1319.8ms/step | 15.2 samples/s +Step 15440/44000 | Loss: 3.0467 | LR: 3.71e-04 | 1318.6ms/step | 15.2 samples/s +Step 15450/44000 | Loss: 3.0608 | LR: 3.71e-04 | 1320.8ms/step | 15.1 samples/s +Step 15460/44000 | Loss: 3.0428 | LR: 3.70e-04 | 1318.0ms/step | 15.2 samples/s +Step 15470/44000 | Loss: 3.0641 | LR: 3.70e-04 | 1318.2ms/step | 15.2 samples/s +Step 15480/44000 | Loss: 3.0318 | LR: 3.70e-04 | 1319.5ms/step | 15.2 samples/s +Step 15490/44000 | Loss: 3.0668 | LR: 3.70e-04 | 1320.5ms/step | 15.1 samples/s +Step 15500/44000 | Loss: 3.0697 | LR: 3.70e-04 | 1315.4ms/step | 15.2 samples/s +Validation at step 15500: Loss = 3.0634, Perplexity = 21.40 +Step 15510/44000 | Loss: 3.0825 | LR: 3.70e-04 | 3530.0ms/step | 5.7 samples/s +Step 15520/44000 | Loss: 3.0745 | LR: 3.69e-04 | 1319.2ms/step | 15.2 samples/s +Step 15530/44000 | Loss: 3.0710 | LR: 3.69e-04 | 1317.3ms/step | 15.2 samples/s +Step 15540/44000 | Loss: 3.0670 | LR: 3.69e-04 | 1319.5ms/step | 15.2 samples/s +Step 15550/44000 | Loss: 3.0697 | LR: 3.69e-04 | 1318.4ms/step | 15.2 samples/s +Step 15560/44000 | Loss: 3.0496 | LR: 3.69e-04 | 1318.9ms/step | 15.2 samples/s +Step 15570/44000 | Loss: 3.0349 | LR: 3.69e-04 | 1318.8ms/step | 15.2 samples/s +Step 15580/44000 | Loss: 3.0863 | LR: 3.68e-04 | 1314.8ms/step | 15.2 samples/s +Step 15590/44000 | Loss: 3.0559 | LR: 3.68e-04 | 1315.7ms/step | 15.2 samples/s +Step 15600/44000 | Loss: 3.0587 | LR: 3.68e-04 | 1317.3ms/step | 15.2 samples/s +Step 15610/44000 | Loss: 3.0464 | LR: 3.68e-04 | 1314.1ms/step | 15.2 samples/s +Step 15620/44000 | Loss: 3.0521 | LR: 3.68e-04 | 1314.9ms/step | 15.2 samples/s +Step 15630/44000 | Loss: 3.0360 | LR: 3.68e-04 | 1315.1ms/step | 15.2 samples/s +Step 15640/44000 | Loss: 3.0475 | LR: 3.67e-04 | 1313.7ms/step | 15.2 samples/s +Step 15650/44000 | Loss: 3.0556 | LR: 3.67e-04 | 1313.6ms/step | 15.2 samples/s +Step 15660/44000 | Loss: 3.0856 | LR: 3.67e-04 | 1313.2ms/step | 15.2 samples/s +Step 15670/44000 | Loss: 3.0568 | LR: 3.67e-04 | 1318.2ms/step | 15.2 samples/s +Step 15680/44000 | Loss: 3.0578 | LR: 3.67e-04 | 1316.7ms/step | 15.2 samples/s +Step 15690/44000 | Loss: 3.0587 | LR: 3.67e-04 | 1316.1ms/step | 15.2 samples/s +Step 15700/44000 | Loss: 3.0802 | LR: 3.67e-04 | 1318.0ms/step | 15.2 samples/s +Step 15710/44000 | Loss: 3.0409 | LR: 3.66e-04 | 1320.0ms/step | 15.2 samples/s +Step 15720/44000 | Loss: 3.0492 | LR: 3.66e-04 | 1318.8ms/step | 15.2 samples/s +Step 15730/44000 | Loss: 3.0730 | LR: 3.66e-04 | 1319.4ms/step | 15.2 samples/s +Step 15740/44000 | Loss: 3.0689 | LR: 3.66e-04 | 1320.1ms/step | 15.2 samples/s +Step 15750/44000 | Loss: 3.0508 | LR: 3.66e-04 | 1314.3ms/step | 15.2 samples/s +Validation at step 15750: Loss = 3.0595, Perplexity = 21.32 +Step 15760/44000 | Loss: 3.0690 | LR: 3.66e-04 | 3520.4ms/step | 5.7 samples/s +Step 15770/44000 | Loss: 3.0687 | LR: 3.65e-04 | 1314.8ms/step | 15.2 samples/s +Step 15780/44000 | Loss: 3.0507 | LR: 3.65e-04 | 1316.1ms/step | 15.2 samples/s +Step 15790/44000 | Loss: 3.0705 | LR: 3.65e-04 | 1315.9ms/step | 15.2 samples/s +Step 15800/44000 | Loss: 3.0393 | LR: 3.65e-04 | 1315.0ms/step | 15.2 samples/s +Step 15810/44000 | Loss: 3.0429 | LR: 3.65e-04 | 1313.8ms/step | 15.2 samples/s +Step 15820/44000 | Loss: 3.0635 | LR: 3.65e-04 | 1313.0ms/step | 15.2 samples/s +Step 15830/44000 | Loss: 3.0657 | LR: 3.64e-04 | 1311.9ms/step | 15.2 samples/s +Step 15840/44000 | Loss: 3.0422 | LR: 3.64e-04 | 1312.9ms/step | 15.2 samples/s +Step 15850/44000 | Loss: 3.0356 | LR: 3.64e-04 | 1311.3ms/step | 15.3 samples/s +Step 15860/44000 | Loss: 3.0676 | LR: 3.64e-04 | 1310.0ms/step | 15.3 samples/s +Step 15870/44000 | Loss: 3.0672 | LR: 3.64e-04 | 1312.5ms/step | 15.2 samples/s +Step 15880/44000 | Loss: 3.0537 | LR: 3.64e-04 | 1314.8ms/step | 15.2 samples/s +Step 15890/44000 | Loss: 3.0581 | LR: 3.63e-04 | 1315.5ms/step | 15.2 samples/s +Step 15900/44000 | Loss: 3.0598 | LR: 3.63e-04 | 1312.3ms/step | 15.2 samples/s +Step 15910/44000 | Loss: 3.0444 | LR: 3.63e-04 | 1314.6ms/step | 15.2 samples/s +Step 15920/44000 | Loss: 3.0503 | LR: 3.63e-04 | 1314.6ms/step | 15.2 samples/s +Step 15930/44000 | Loss: 3.0730 | LR: 3.63e-04 | 1314.1ms/step | 15.2 samples/s +Step 15940/44000 | Loss: 3.0327 | LR: 3.63e-04 | 1314.4ms/step | 15.2 samples/s +Step 15950/44000 | Loss: 3.0586 | LR: 3.62e-04 | 1315.0ms/step | 15.2 samples/s +Step 15960/44000 | Loss: 3.0237 | LR: 3.62e-04 | 1316.5ms/step | 15.2 samples/s +Step 15970/44000 | Loss: 3.0653 | LR: 3.62e-04 | 1315.8ms/step | 15.2 samples/s +Step 15980/44000 | Loss: 3.0551 | LR: 3.62e-04 | 1315.7ms/step | 15.2 samples/s +Step 15990/44000 | Loss: 3.0604 | LR: 3.62e-04 | 1314.5ms/step | 15.2 samples/s +Step 16000/44000 | Loss: 3.0559 | LR: 3.62e-04 | 1315.4ms/step | 15.2 samples/s +Validation at step 16000: Loss = 3.0561, Perplexity = 21.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-16000 +Step 16010/44000 | Loss: 3.0703 | LR: 3.62e-04 | 3848.9ms/step | 5.2 samples/s +Step 16020/44000 | Loss: 3.0574 | LR: 3.61e-04 | 1318.2ms/step | 15.2 samples/s +Step 16030/44000 | Loss: 3.0478 | LR: 3.61e-04 | 1319.9ms/step | 15.2 samples/s +Step 16040/44000 | Loss: 3.0706 | LR: 3.61e-04 | 1319.2ms/step | 15.2 samples/s +Step 16050/44000 | Loss: 3.0623 | LR: 3.61e-04 | 1314.6ms/step | 15.2 samples/s +Step 16060/44000 | Loss: 3.0640 | LR: 3.61e-04 | 1316.7ms/step | 15.2 samples/s +Step 16070/44000 | Loss: 3.0357 | LR: 3.61e-04 | 1316.8ms/step | 15.2 samples/s +Step 16080/44000 | Loss: 3.0465 | LR: 3.60e-04 | 1319.5ms/step | 15.2 samples/s +Step 16090/44000 | Loss: 3.0286 | LR: 3.60e-04 | 1319.3ms/step | 15.2 samples/s +Step 16100/44000 | Loss: 3.0625 | LR: 3.60e-04 | 1322.9ms/step | 15.1 samples/s +Step 16110/44000 | Loss: 3.0545 | LR: 3.60e-04 | 1321.0ms/step | 15.1 samples/s +Step 16120/44000 | Loss: 3.0451 | LR: 3.60e-04 | 1320.0ms/step | 15.2 samples/s +Step 16130/44000 | Loss: 3.0318 | LR: 3.60e-04 | 1323.8ms/step | 15.1 samples/s +Step 16140/44000 | Loss: 3.0356 | LR: 3.59e-04 | 1320.3ms/step | 15.1 samples/s +Step 16150/44000 | Loss: 3.0545 | LR: 3.59e-04 | 1320.9ms/step | 15.1 samples/s +Step 16160/44000 | Loss: 3.0625 | LR: 3.59e-04 | 1319.6ms/step | 15.2 samples/s +Step 16170/44000 | Loss: 3.0677 | LR: 3.59e-04 | 1319.4ms/step | 15.2 samples/s +Step 16180/44000 | Loss: 3.0771 | LR: 3.59e-04 | 1319.6ms/step | 15.2 samples/s +Step 16190/44000 | Loss: 3.0634 | LR: 3.59e-04 | 1319.9ms/step | 15.2 samples/s +Step 16200/44000 | Loss: 3.0455 | LR: 3.58e-04 | 1318.5ms/step | 15.2 samples/s +Step 16210/44000 | Loss: 3.0506 | LR: 3.58e-04 | 1318.8ms/step | 15.2 samples/s +Step 16220/44000 | Loss: 3.0425 | LR: 3.58e-04 | 1320.0ms/step | 15.2 samples/s +Step 16230/44000 | Loss: 3.0807 | LR: 3.58e-04 | 1320.5ms/step | 15.1 samples/s +Step 16240/44000 | Loss: 3.0610 | LR: 3.58e-04 | 1321.1ms/step | 15.1 samples/s +Step 16250/44000 | Loss: 3.0314 | LR: 3.58e-04 | 1320.0ms/step | 15.2 samples/s +Validation at step 16250: Loss = 3.0537, Perplexity = 21.19 +Step 16260/44000 | Loss: 3.0534 | LR: 3.57e-04 | 3527.7ms/step | 5.7 samples/s +Step 16270/44000 | Loss: 3.0160 | LR: 3.57e-04 | 1317.9ms/step | 15.2 samples/s +Step 16280/44000 | Loss: 3.0507 | LR: 3.57e-04 | 1318.3ms/step | 15.2 samples/s +Step 16290/44000 | Loss: 3.0722 | LR: 3.57e-04 | 1316.4ms/step | 15.2 samples/s +Step 16300/44000 | Loss: 3.0457 | LR: 3.57e-04 | 1319.2ms/step | 15.2 samples/s +Step 16310/44000 | Loss: 3.0768 | LR: 3.57e-04 | 1317.4ms/step | 15.2 samples/s +Step 16320/44000 | Loss: 3.0479 | LR: 3.56e-04 | 1320.4ms/step | 15.1 samples/s +Step 16330/44000 | Loss: 3.0662 | LR: 3.56e-04 | 1317.7ms/step | 15.2 samples/s +Step 16340/44000 | Loss: 3.0499 | LR: 3.56e-04 | 1316.5ms/step | 15.2 samples/s +Step 16350/44000 | Loss: 3.0479 | LR: 3.56e-04 | 1318.9ms/step | 15.2 samples/s +Step 16360/44000 | Loss: 3.0188 | LR: 3.56e-04 | 1319.6ms/step | 15.2 samples/s +Step 16370/44000 | Loss: 3.0173 | LR: 3.56e-04 | 1318.6ms/step | 15.2 samples/s +Step 16380/44000 | Loss: 3.0556 | LR: 3.55e-04 | 1319.4ms/step | 15.2 samples/s +Step 16390/44000 | Loss: 3.0367 | LR: 3.55e-04 | 1318.7ms/step | 15.2 samples/s +Step 16400/44000 | Loss: 3.0581 | LR: 3.55e-04 | 1322.6ms/step | 15.1 samples/s +Step 16410/44000 | Loss: 3.0605 | LR: 3.55e-04 | 1320.3ms/step | 15.1 samples/s +Step 16420/44000 | Loss: 3.0536 | LR: 3.55e-04 | 1323.5ms/step | 15.1 samples/s +Step 16430/44000 | Loss: 3.0430 | LR: 3.55e-04 | 1319.5ms/step | 15.2 samples/s +Step 16440/44000 | Loss: 3.0363 | LR: 3.54e-04 | 1317.9ms/step | 15.2 samples/s +Step 16450/44000 | Loss: 3.0759 | LR: 3.54e-04 | 1318.1ms/step | 15.2 samples/s +Step 16460/44000 | Loss: 3.0476 | LR: 3.54e-04 | 1317.2ms/step | 15.2 samples/s +Step 16470/44000 | Loss: 3.0242 | LR: 3.54e-04 | 1322.4ms/step | 15.1 samples/s +Step 16480/44000 | Loss: 3.0564 | LR: 3.54e-04 | 1322.7ms/step | 15.1 samples/s +Step 16490/44000 | Loss: 3.0842 | LR: 3.54e-04 | 1320.3ms/step | 15.1 samples/s +Step 16500/44000 | Loss: 3.0708 | LR: 3.53e-04 | 1319.9ms/step | 15.2 samples/s +Validation at step 16500: Loss = 3.0508, Perplexity = 21.13 +Step 16510/44000 | Loss: 3.0429 | LR: 3.53e-04 | 3535.7ms/step | 5.7 samples/s +Step 16520/44000 | Loss: 3.0565 | LR: 3.53e-04 | 1323.4ms/step | 15.1 samples/s +Step 16530/44000 | Loss: 3.0547 | LR: 3.53e-04 | 1318.7ms/step | 15.2 samples/s +Step 16540/44000 | Loss: 3.0363 | LR: 3.53e-04 | 1320.2ms/step | 15.1 samples/s +Step 16550/44000 | Loss: 3.0241 | LR: 3.53e-04 | 1318.3ms/step | 15.2 samples/s +Step 16560/44000 | Loss: 3.0723 | LR: 3.52e-04 | 1318.9ms/step | 15.2 samples/s +Step 16570/44000 | Loss: 3.0397 | LR: 3.52e-04 | 1318.7ms/step | 15.2 samples/s +Step 16580/44000 | Loss: 3.0518 | LR: 3.52e-04 | 1319.1ms/step | 15.2 samples/s +Step 16590/44000 | Loss: 3.0231 | LR: 3.52e-04 | 1317.7ms/step | 15.2 samples/s +Step 16600/44000 | Loss: 3.0573 | LR: 3.52e-04 | 1319.7ms/step | 15.2 samples/s +Step 16610/44000 | Loss: 3.0470 | LR: 3.52e-04 | 1319.4ms/step | 15.2 samples/s +Step 16620/44000 | Loss: 3.0505 | LR: 3.51e-04 | 1318.6ms/step | 15.2 samples/s +Step 16630/44000 | Loss: 3.0652 | LR: 3.51e-04 | 1318.5ms/step | 15.2 samples/s +Step 16640/44000 | Loss: 3.0318 | LR: 3.51e-04 | 1321.4ms/step | 15.1 samples/s +Step 16650/44000 | Loss: 3.0229 | LR: 3.51e-04 | 1320.5ms/step | 15.1 samples/s +Step 16660/44000 | Loss: 3.0334 | LR: 3.51e-04 | 1320.3ms/step | 15.1 samples/s +Step 16670/44000 | Loss: 3.0427 | LR: 3.51e-04 | 1320.6ms/step | 15.1 samples/s +Step 16680/44000 | Loss: 3.0624 | LR: 3.50e-04 | 1317.8ms/step | 15.2 samples/s +Step 16690/44000 | Loss: 3.0505 | LR: 3.50e-04 | 1319.7ms/step | 15.2 samples/s +Step 16700/44000 | Loss: 3.0315 | LR: 3.50e-04 | 1320.9ms/step | 15.1 samples/s +Step 16710/44000 | Loss: 3.0462 | LR: 3.50e-04 | 1321.1ms/step | 15.1 samples/s +Step 16720/44000 | Loss: 3.0446 | LR: 3.50e-04 | 1320.5ms/step | 15.1 samples/s +Step 16730/44000 | Loss: 3.0730 | LR: 3.50e-04 | 1315.4ms/step | 15.2 samples/s +Step 16740/44000 | Loss: 3.0479 | LR: 3.49e-04 | 1320.4ms/step | 15.1 samples/s +Step 16750/44000 | Loss: 3.0444 | LR: 3.49e-04 | 1319.9ms/step | 15.2 samples/s +Validation at step 16750: Loss = 3.0480, Perplexity = 21.07 +Step 16760/44000 | Loss: 3.0253 | LR: 3.49e-04 | 3535.9ms/step | 5.7 samples/s +Step 16770/44000 | Loss: 3.0356 | LR: 3.49e-04 | 1318.7ms/step | 15.2 samples/s +Step 16780/44000 | Loss: 3.0497 | LR: 3.49e-04 | 1322.3ms/step | 15.1 samples/s +Step 16790/44000 | Loss: 3.0430 | LR: 3.49e-04 | 1320.0ms/step | 15.2 samples/s +Step 16800/44000 | Loss: 3.0852 | LR: 3.48e-04 | 1320.7ms/step | 15.1 samples/s +Step 16810/44000 | Loss: 3.0549 | LR: 3.48e-04 | 1319.8ms/step | 15.2 samples/s +Step 16820/44000 | Loss: 3.0301 | LR: 3.48e-04 | 1320.2ms/step | 15.1 samples/s +Step 16830/44000 | Loss: 3.0522 | LR: 3.48e-04 | 1320.9ms/step | 15.1 samples/s +Step 16840/44000 | Loss: 3.0314 | LR: 3.48e-04 | 1320.5ms/step | 15.1 samples/s +Step 16850/44000 | Loss: 3.0384 | LR: 3.48e-04 | 1318.4ms/step | 15.2 samples/s +Step 16860/44000 | Loss: 3.0582 | LR: 3.47e-04 | 1318.4ms/step | 15.2 samples/s +Step 16870/44000 | Loss: 3.0324 | LR: 3.47e-04 | 1317.8ms/step | 15.2 samples/s +Step 16880/44000 | Loss: 3.0252 | LR: 3.47e-04 | 1317.9ms/step | 15.2 samples/s +Step 16890/44000 | Loss: 2.9979 | LR: 3.47e-04 | 1319.0ms/step | 15.2 samples/s +Step 16900/44000 | Loss: 3.0481 | LR: 3.47e-04 | 1319.9ms/step | 15.2 samples/s +Step 16910/44000 | Loss: 3.0475 | LR: 3.47e-04 | 1319.2ms/step | 15.2 samples/s +Step 16920/44000 | Loss: 3.0330 | LR: 3.46e-04 | 1318.7ms/step | 15.2 samples/s +Step 16930/44000 | Loss: 3.0445 | LR: 3.46e-04 | 1319.8ms/step | 15.2 samples/s +Step 16940/44000 | Loss: 3.0585 | LR: 3.46e-04 | 1320.2ms/step | 15.1 samples/s +Step 16950/44000 | Loss: 3.0329 | LR: 3.46e-04 | 1320.0ms/step | 15.2 samples/s +Step 16960/44000 | Loss: 3.0601 | LR: 3.46e-04 | 1317.5ms/step | 15.2 samples/s +Step 16970/44000 | Loss: 3.0335 | LR: 3.46e-04 | 1317.2ms/step | 15.2 samples/s +Step 16980/44000 | Loss: 3.0367 | LR: 3.45e-04 | 1315.1ms/step | 15.2 samples/s +Step 16990/44000 | Loss: 3.0458 | LR: 3.45e-04 | 1315.1ms/step | 15.2 samples/s +Step 17000/44000 | Loss: 3.0484 | LR: 3.45e-04 | 1319.2ms/step | 15.2 samples/s +Validation at step 17000: Loss = 3.0435, Perplexity = 20.98 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-17000 +Step 17010/44000 | Loss: 3.0615 | LR: 3.45e-04 | 3843.2ms/step | 5.2 samples/s +Step 17020/44000 | Loss: 3.0587 | LR: 3.45e-04 | 1316.7ms/step | 15.2 samples/s +Step 17030/44000 | Loss: 3.0441 | LR: 3.45e-04 | 1319.5ms/step | 15.2 samples/s +Step 17040/44000 | Loss: 3.0307 | LR: 3.44e-04 | 1318.6ms/step | 15.2 samples/s +Step 17050/44000 | Loss: 3.0419 | LR: 3.44e-04 | 1320.1ms/step | 15.2 samples/s +Step 17060/44000 | Loss: 3.0602 | LR: 3.44e-04 | 1321.5ms/step | 15.1 samples/s +Step 17070/44000 | Loss: 3.0558 | LR: 3.44e-04 | 1320.8ms/step | 15.1 samples/s +Step 17080/44000 | Loss: 3.0482 | LR: 3.44e-04 | 1320.1ms/step | 15.2 samples/s +Step 17090/44000 | Loss: 3.0108 | LR: 3.44e-04 | 1321.3ms/step | 15.1 samples/s +Step 17100/44000 | Loss: 3.0384 | LR: 3.43e-04 | 1318.3ms/step | 15.2 samples/s +Step 17110/44000 | Loss: 3.0111 | LR: 3.43e-04 | 1318.4ms/step | 15.2 samples/s +Step 17120/44000 | Loss: 3.0405 | LR: 3.43e-04 | 1317.2ms/step | 15.2 samples/s +Step 17130/44000 | Loss: 3.0885 | LR: 3.43e-04 | 1319.0ms/step | 15.2 samples/s +Step 17140/44000 | Loss: 3.0580 | LR: 3.43e-04 | 1317.6ms/step | 15.2 samples/s +Step 17150/44000 | Loss: 3.0358 | LR: 3.43e-04 | 1317.1ms/step | 15.2 samples/s +Step 17160/44000 | Loss: 3.0138 | LR: 3.42e-04 | 1317.6ms/step | 15.2 samples/s +Step 17170/44000 | Loss: 3.0378 | LR: 3.42e-04 | 1318.4ms/step | 15.2 samples/s +Step 17180/44000 | Loss: 3.0167 | LR: 3.42e-04 | 1318.7ms/step | 15.2 samples/s +Step 17190/44000 | Loss: 3.0356 | LR: 3.42e-04 | 1314.5ms/step | 15.2 samples/s +Step 17200/44000 | Loss: 3.0546 | LR: 3.42e-04 | 1316.7ms/step | 15.2 samples/s +Step 17210/44000 | Loss: 3.0571 | LR: 3.42e-04 | 1321.7ms/step | 15.1 samples/s +Step 17220/44000 | Loss: 3.0301 | LR: 3.41e-04 | 1320.3ms/step | 15.1 samples/s +Step 17230/44000 | Loss: 3.0451 | LR: 3.41e-04 | 1319.9ms/step | 15.2 samples/s +Step 17240/44000 | Loss: 3.0357 | LR: 3.41e-04 | 1320.0ms/step | 15.2 samples/s +Step 17250/44000 | Loss: 3.0316 | LR: 3.41e-04 | 1322.7ms/step | 15.1 samples/s +Validation at step 17250: Loss = 3.0418, Perplexity = 20.94 +Step 17260/44000 | Loss: 3.0409 | LR: 3.41e-04 | 3528.3ms/step | 5.7 samples/s +Step 17270/44000 | Loss: 3.0647 | LR: 3.41e-04 | 1319.8ms/step | 15.2 samples/s +Step 17280/44000 | Loss: 3.0343 | LR: 3.40e-04 | 1318.9ms/step | 15.2 samples/s +Step 17290/44000 | Loss: 2.9847 | LR: 3.40e-04 | 1319.2ms/step | 15.2 samples/s +Step 17300/44000 | Loss: 3.0299 | LR: 3.40e-04 | 1315.8ms/step | 15.2 samples/s +Step 17310/44000 | Loss: 3.0313 | LR: 3.40e-04 | 1316.3ms/step | 15.2 samples/s +Step 17320/44000 | Loss: 3.0173 | LR: 3.40e-04 | 1318.4ms/step | 15.2 samples/s +Step 17330/44000 | Loss: 3.0302 | LR: 3.40e-04 | 1319.1ms/step | 15.2 samples/s +Step 17340/44000 | Loss: 3.0477 | LR: 3.39e-04 | 1320.2ms/step | 15.1 samples/s +Step 17350/44000 | Loss: 3.0451 | LR: 3.39e-04 | 1320.4ms/step | 15.1 samples/s +Step 17360/44000 | Loss: 3.0191 | LR: 3.39e-04 | 1318.2ms/step | 15.2 samples/s +Step 17370/44000 | Loss: 3.0467 | LR: 3.39e-04 | 1317.6ms/step | 15.2 samples/s +Step 17380/44000 | Loss: 3.0546 | LR: 3.39e-04 | 1319.2ms/step | 15.2 samples/s +Step 17390/44000 | Loss: 3.0626 | LR: 3.39e-04 | 1318.9ms/step | 15.2 samples/s +Step 17400/44000 | Loss: 2.9978 | LR: 3.38e-04 | 1320.4ms/step | 15.1 samples/s +Step 17410/44000 | Loss: 3.0536 | LR: 3.38e-04 | 1319.6ms/step | 15.2 samples/s +Step 17420/44000 | Loss: 3.0444 | LR: 3.38e-04 | 1320.0ms/step | 15.2 samples/s +Step 17430/44000 | Loss: 3.0303 | LR: 3.38e-04 | 1319.0ms/step | 15.2 samples/s +Step 17440/44000 | Loss: 3.0249 | LR: 3.38e-04 | 1320.1ms/step | 15.2 samples/s +Step 17450/44000 | Loss: 3.0575 | LR: 3.38e-04 | 1318.7ms/step | 15.2 samples/s +Step 17460/44000 | Loss: 3.0480 | LR: 3.37e-04 | 1320.4ms/step | 15.1 samples/s +Step 17470/44000 | Loss: 3.0283 | LR: 3.37e-04 | 1317.3ms/step | 15.2 samples/s +Step 17480/44000 | Loss: 3.0565 | LR: 3.37e-04 | 1318.2ms/step | 15.2 samples/s +Step 17490/44000 | Loss: 3.0221 | LR: 3.37e-04 | 1321.2ms/step | 15.1 samples/s +Step 17500/44000 | Loss: 3.0453 | LR: 3.37e-04 | 1318.7ms/step | 15.2 samples/s +Validation at step 17500: Loss = 3.0384, Perplexity = 20.87 +Step 17510/44000 | Loss: 3.0274 | LR: 3.37e-04 | 3531.6ms/step | 5.7 samples/s +Step 17520/44000 | Loss: 3.0372 | LR: 3.36e-04 | 1318.4ms/step | 15.2 samples/s +Step 17530/44000 | Loss: 3.0063 | LR: 3.36e-04 | 1319.2ms/step | 15.2 samples/s +Step 17540/44000 | Loss: 3.0252 | LR: 3.36e-04 | 1319.5ms/step | 15.2 samples/s +Step 17550/44000 | Loss: 3.0534 | LR: 3.36e-04 | 1318.8ms/step | 15.2 samples/s +Step 17560/44000 | Loss: 3.0480 | LR: 3.36e-04 | 1318.5ms/step | 15.2 samples/s +Step 17570/44000 | Loss: 3.0193 | LR: 3.35e-04 | 1320.4ms/step | 15.1 samples/s +Step 17580/44000 | Loss: 3.0492 | LR: 3.35e-04 | 1318.9ms/step | 15.2 samples/s +Step 17590/44000 | Loss: 3.0411 | LR: 3.35e-04 | 1319.6ms/step | 15.2 samples/s +Step 17600/44000 | Loss: 3.0379 | LR: 3.35e-04 | 1324.5ms/step | 15.1 samples/s +Step 17610/44000 | Loss: 3.0089 | LR: 3.35e-04 | 1320.1ms/step | 15.2 samples/s +Step 17620/44000 | Loss: 3.0422 | LR: 3.35e-04 | 1320.3ms/step | 15.1 samples/s +Step 17630/44000 | Loss: 3.0034 | LR: 3.34e-04 | 1320.3ms/step | 15.1 samples/s +Step 17640/44000 | Loss: 3.0111 | LR: 3.34e-04 | 1319.9ms/step | 15.2 samples/s +Step 17650/44000 | Loss: 3.0526 | LR: 3.34e-04 | 1320.8ms/step | 15.1 samples/s +Step 17660/44000 | Loss: 3.0293 | LR: 3.34e-04 | 1321.5ms/step | 15.1 samples/s +Step 17670/44000 | Loss: 3.0300 | LR: 3.34e-04 | 1319.8ms/step | 15.2 samples/s +Step 17680/44000 | Loss: 3.0333 | LR: 3.34e-04 | 1319.5ms/step | 15.2 samples/s +Step 17690/44000 | Loss: 3.0469 | LR: 3.33e-04 | 1316.8ms/step | 15.2 samples/s +Step 17700/44000 | Loss: 3.0292 | LR: 3.33e-04 | 1319.2ms/step | 15.2 samples/s +Step 17710/44000 | Loss: 3.0105 | LR: 3.33e-04 | 1316.7ms/step | 15.2 samples/s +Step 17720/44000 | Loss: 3.0368 | LR: 3.33e-04 | 1321.3ms/step | 15.1 samples/s +Step 17730/44000 | Loss: 3.0334 | LR: 3.33e-04 | 1316.0ms/step | 15.2 samples/s +Step 17740/44000 | Loss: 3.0250 | LR: 3.33e-04 | 1318.9ms/step | 15.2 samples/s +Step 17750/44000 | Loss: 3.0422 | LR: 3.32e-04 | 1319.9ms/step | 15.2 samples/s +Validation at step 17750: Loss = 3.0357, Perplexity = 20.82 +Step 17760/44000 | Loss: 3.0502 | LR: 3.32e-04 | 3527.4ms/step | 5.7 samples/s +Step 17770/44000 | Loss: 3.0055 | LR: 3.32e-04 | 1320.1ms/step | 15.2 samples/s +Step 17780/44000 | Loss: 3.0121 | LR: 3.32e-04 | 1319.9ms/step | 15.2 samples/s +Step 17790/44000 | Loss: 3.0483 | LR: 3.32e-04 | 1319.4ms/step | 15.2 samples/s +Step 17800/44000 | Loss: 3.0060 | LR: 3.32e-04 | 1317.6ms/step | 15.2 samples/s +Step 17810/44000 | Loss: 3.0285 | LR: 3.31e-04 | 1320.5ms/step | 15.1 samples/s +Step 17820/44000 | Loss: 3.0089 | LR: 3.31e-04 | 1320.8ms/step | 15.1 samples/s +Step 17830/44000 | Loss: 3.0292 | LR: 3.31e-04 | 1317.5ms/step | 15.2 samples/s +Step 17840/44000 | Loss: 3.0262 | LR: 3.31e-04 | 1319.1ms/step | 15.2 samples/s +Step 17850/44000 | Loss: 3.0762 | LR: 3.31e-04 | 1319.2ms/step | 15.2 samples/s +Step 17860/44000 | Loss: 3.0253 | LR: 3.31e-04 | 1320.8ms/step | 15.1 samples/s +Step 17870/44000 | Loss: 3.0299 | LR: 3.30e-04 | 1319.0ms/step | 15.2 samples/s +Step 17880/44000 | Loss: 3.0175 | LR: 3.30e-04 | 1318.7ms/step | 15.2 samples/s +Step 17890/44000 | Loss: 3.0456 | LR: 3.30e-04 | 1321.3ms/step | 15.1 samples/s +Step 17900/44000 | Loss: 3.0269 | LR: 3.30e-04 | 1319.1ms/step | 15.2 samples/s +Step 17910/44000 | Loss: 3.0299 | LR: 3.30e-04 | 1319.8ms/step | 15.2 samples/s +Step 17920/44000 | Loss: 3.0388 | LR: 3.29e-04 | 1321.9ms/step | 15.1 samples/s +Step 17930/44000 | Loss: 2.9968 | LR: 3.29e-04 | 1320.6ms/step | 15.1 samples/s +Step 17940/44000 | Loss: 3.0255 | LR: 3.29e-04 | 1319.6ms/step | 15.2 samples/s +Step 17950/44000 | Loss: 2.9944 | LR: 3.29e-04 | 1317.6ms/step | 15.2 samples/s +Step 17960/44000 | Loss: 3.0198 | LR: 3.29e-04 | 1317.3ms/step | 15.2 samples/s +Step 17970/44000 | Loss: 3.0407 | LR: 3.29e-04 | 1319.2ms/step | 15.2 samples/s +Step 17980/44000 | Loss: 3.0209 | LR: 3.28e-04 | 1320.3ms/step | 15.1 samples/s +Step 17990/44000 | Loss: 3.0177 | LR: 3.28e-04 | 1320.8ms/step | 15.1 samples/s +Step 18000/44000 | Loss: 3.0647 | LR: 3.28e-04 | 1317.9ms/step | 15.2 samples/s +Validation at step 18000: Loss = 3.0318, Perplexity = 20.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-18000 +Step 18010/44000 | Loss: 3.0577 | LR: 3.28e-04 | 3844.5ms/step | 5.2 samples/s +Step 18020/44000 | Loss: 3.0304 | LR: 3.28e-04 | 1312.8ms/step | 15.2 samples/s +Step 18030/44000 | Loss: 2.9924 | LR: 3.28e-04 | 1312.6ms/step | 15.2 samples/s +Step 18040/44000 | Loss: 3.0179 | LR: 3.27e-04 | 1316.0ms/step | 15.2 samples/s +Step 18050/44000 | Loss: 3.0203 | LR: 3.27e-04 | 1315.7ms/step | 15.2 samples/s +Step 18060/44000 | Loss: 3.0340 | LR: 3.27e-04 | 1315.0ms/step | 15.2 samples/s +Step 18070/44000 | Loss: 3.0269 | LR: 3.27e-04 | 1314.8ms/step | 15.2 samples/s +Step 18080/44000 | Loss: 3.0396 | LR: 3.27e-04 | 1315.8ms/step | 15.2 samples/s +Step 18090/44000 | Loss: 3.0512 | LR: 3.27e-04 | 1314.3ms/step | 15.2 samples/s +Step 18100/44000 | Loss: 3.0319 | LR: 3.26e-04 | 1312.9ms/step | 15.2 samples/s +Step 18110/44000 | Loss: 3.0290 | LR: 3.26e-04 | 1314.6ms/step | 15.2 samples/s +Step 18120/44000 | Loss: 3.0117 | LR: 3.26e-04 | 1315.1ms/step | 15.2 samples/s +Step 18130/44000 | Loss: 3.0116 | LR: 3.26e-04 | 1314.3ms/step | 15.2 samples/s +Step 18140/44000 | Loss: 3.0321 | LR: 3.26e-04 | 1315.4ms/step | 15.2 samples/s +Step 18150/44000 | Loss: 3.0054 | LR: 3.26e-04 | 1313.6ms/step | 15.2 samples/s +Step 18160/44000 | Loss: 3.0017 | LR: 3.25e-04 | 1313.1ms/step | 15.2 samples/s +Step 18170/44000 | Loss: 3.0264 | LR: 3.25e-04 | 1313.7ms/step | 15.2 samples/s +Step 18180/44000 | Loss: 3.0415 | LR: 3.25e-04 | 1314.4ms/step | 15.2 samples/s +Step 18190/44000 | Loss: 3.0130 | LR: 3.25e-04 | 1314.2ms/step | 15.2 samples/s +Step 18200/44000 | Loss: 3.0180 | LR: 3.25e-04 | 1314.0ms/step | 15.2 samples/s +Step 18210/44000 | Loss: 3.0191 | LR: 3.24e-04 | 1311.5ms/step | 15.3 samples/s +Step 18220/44000 | Loss: 3.0177 | LR: 3.24e-04 | 1310.3ms/step | 15.3 samples/s +Step 18230/44000 | Loss: 3.0569 | LR: 3.24e-04 | 1311.9ms/step | 15.2 samples/s +Step 18240/44000 | Loss: 3.0206 | LR: 3.24e-04 | 1312.1ms/step | 15.2 samples/s +Step 18250/44000 | Loss: 3.0689 | LR: 3.24e-04 | 1313.8ms/step | 15.2 samples/s +Validation at step 18250: Loss = 3.0291, Perplexity = 20.68 +Step 18260/44000 | Loss: 3.0663 | LR: 3.24e-04 | 3520.5ms/step | 5.7 samples/s +Step 18270/44000 | Loss: 3.0128 | LR: 3.23e-04 | 1315.1ms/step | 15.2 samples/s +Step 18280/44000 | Loss: 3.0254 | LR: 3.23e-04 | 1314.6ms/step | 15.2 samples/s +Step 18290/44000 | Loss: 3.0034 | LR: 3.23e-04 | 1312.7ms/step | 15.2 samples/s +Step 18300/44000 | Loss: 3.0121 | LR: 3.23e-04 | 1312.6ms/step | 15.2 samples/s +Step 18310/44000 | Loss: 2.9972 | LR: 3.23e-04 | 1311.8ms/step | 15.2 samples/s +Step 18320/44000 | Loss: 3.0663 | LR: 3.23e-04 | 1311.2ms/step | 15.3 samples/s +Step 18330/44000 | Loss: 3.0011 | LR: 3.22e-04 | 1316.2ms/step | 15.2 samples/s +Step 18340/44000 | Loss: 3.0349 | LR: 3.22e-04 | 1314.5ms/step | 15.2 samples/s +Step 18350/44000 | Loss: 3.0143 | LR: 3.22e-04 | 1313.8ms/step | 15.2 samples/s +Step 18360/44000 | Loss: 3.0065 | LR: 3.22e-04 | 1313.7ms/step | 15.2 samples/s +Step 18370/44000 | Loss: 3.0285 | LR: 3.22e-04 | 1314.2ms/step | 15.2 samples/s +Step 18380/44000 | Loss: 3.0162 | LR: 3.22e-04 | 1314.4ms/step | 15.2 samples/s +Step 18390/44000 | Loss: 3.0253 | LR: 3.21e-04 | 1312.6ms/step | 15.2 samples/s +Step 18400/44000 | Loss: 3.0610 | LR: 3.21e-04 | 1315.1ms/step | 15.2 samples/s +Step 18410/44000 | Loss: 3.0208 | LR: 3.21e-04 | 1314.8ms/step | 15.2 samples/s +Step 18420/44000 | Loss: 3.0048 | LR: 3.21e-04 | 1313.8ms/step | 15.2 samples/s +Step 18430/44000 | Loss: 2.9978 | LR: 3.21e-04 | 1309.2ms/step | 15.3 samples/s +Step 18440/44000 | Loss: 3.0292 | LR: 3.20e-04 | 1310.9ms/step | 15.3 samples/s +Step 18450/44000 | Loss: 3.0293 | LR: 3.20e-04 | 1313.6ms/step | 15.2 samples/s +Step 18460/44000 | Loss: 3.0151 | LR: 3.20e-04 | 1315.3ms/step | 15.2 samples/s +Step 18470/44000 | Loss: 3.0095 | LR: 3.20e-04 | 1315.5ms/step | 15.2 samples/s +Step 18480/44000 | Loss: 3.0320 | LR: 3.20e-04 | 1314.0ms/step | 15.2 samples/s +Step 18490/44000 | Loss: 3.0108 | LR: 3.20e-04 | 1313.3ms/step | 15.2 samples/s +Step 18500/44000 | Loss: 3.0246 | LR: 3.19e-04 | 1313.4ms/step | 15.2 samples/s +Validation at step 18500: Loss = 3.0267, Perplexity = 20.63 +Step 18510/44000 | Loss: 3.0061 | LR: 3.19e-04 | 3517.8ms/step | 5.7 samples/s +Step 18520/44000 | Loss: 3.0329 | LR: 3.19e-04 | 1313.1ms/step | 15.2 samples/s +Step 18530/44000 | Loss: 3.0189 | LR: 3.19e-04 | 1313.2ms/step | 15.2 samples/s +Step 18540/44000 | Loss: 3.0055 | LR: 3.19e-04 | 1310.8ms/step | 15.3 samples/s +Step 18550/44000 | Loss: 3.0088 | LR: 3.19e-04 | 1312.7ms/step | 15.2 samples/s +Step 18560/44000 | Loss: 3.0490 | LR: 3.18e-04 | 1312.0ms/step | 15.2 samples/s +Step 18570/44000 | Loss: 2.9956 | LR: 3.18e-04 | 1312.9ms/step | 15.2 samples/s +Step 18580/44000 | Loss: 3.0400 | LR: 3.18e-04 | 1315.3ms/step | 15.2 samples/s +Step 18590/44000 | Loss: 3.0052 | LR: 3.18e-04 | 1314.8ms/step | 15.2 samples/s +Step 18600/44000 | Loss: 3.0088 | LR: 3.18e-04 | 1316.7ms/step | 15.2 samples/s +Step 18610/44000 | Loss: 3.0435 | LR: 3.18e-04 | 1311.6ms/step | 15.2 samples/s +Step 18620/44000 | Loss: 3.0150 | LR: 3.17e-04 | 1313.1ms/step | 15.2 samples/s +Step 18630/44000 | Loss: 3.0156 | LR: 3.17e-04 | 1312.9ms/step | 15.2 samples/s +Step 18640/44000 | Loss: 3.0345 | LR: 3.17e-04 | 1313.7ms/step | 15.2 samples/s +Step 18650/44000 | Loss: 3.0216 | LR: 3.17e-04 | 1315.1ms/step | 15.2 samples/s +Step 18660/44000 | Loss: 3.0114 | LR: 3.17e-04 | 1314.1ms/step | 15.2 samples/s +Step 18670/44000 | Loss: 3.0226 | LR: 3.16e-04 | 1314.7ms/step | 15.2 samples/s +Step 18680/44000 | Loss: 3.0192 | LR: 3.16e-04 | 1322.1ms/step | 15.1 samples/s +Step 18690/44000 | Loss: 2.9993 | LR: 3.16e-04 | 1318.7ms/step | 15.2 samples/s +Step 18700/44000 | Loss: 3.0246 | LR: 3.16e-04 | 1322.9ms/step | 15.1 samples/s +Step 18710/44000 | Loss: 3.0078 | LR: 3.16e-04 | 1320.6ms/step | 15.1 samples/s +Step 18720/44000 | Loss: 3.0192 | LR: 3.16e-04 | 1320.1ms/step | 15.2 samples/s +Step 18730/44000 | Loss: 3.0183 | LR: 3.15e-04 | 1321.3ms/step | 15.1 samples/s +Step 18740/44000 | Loss: 3.0049 | LR: 3.15e-04 | 1319.9ms/step | 15.2 samples/s +Step 18750/44000 | Loss: 3.0243 | LR: 3.15e-04 | 1318.1ms/step | 15.2 samples/s +Validation at step 18750: Loss = 3.0236, Perplexity = 20.57 +Step 18760/44000 | Loss: 3.0179 | LR: 3.15e-04 | 3533.0ms/step | 5.7 samples/s +Step 18770/44000 | Loss: 2.9979 | LR: 3.15e-04 | 1319.8ms/step | 15.2 samples/s +Step 18780/44000 | Loss: 2.9980 | LR: 3.15e-04 | 1318.6ms/step | 15.2 samples/s +Step 18790/44000 | Loss: 3.0210 | LR: 3.14e-04 | 1321.7ms/step | 15.1 samples/s +Step 18800/44000 | Loss: 3.0001 | LR: 3.14e-04 | 1319.4ms/step | 15.2 samples/s +Step 18810/44000 | Loss: 3.0226 | LR: 3.14e-04 | 1319.0ms/step | 15.2 samples/s +Step 18820/44000 | Loss: 3.0208 | LR: 3.14e-04 | 1319.3ms/step | 15.2 samples/s +Step 18830/44000 | Loss: 3.0274 | LR: 3.14e-04 | 1320.9ms/step | 15.1 samples/s +Step 18840/44000 | Loss: 3.0214 | LR: 3.13e-04 | 1320.5ms/step | 15.1 samples/s +Step 18850/44000 | Loss: 3.0220 | LR: 3.13e-04 | 1318.7ms/step | 15.2 samples/s +Step 18860/44000 | Loss: 3.0499 | LR: 3.13e-04 | 1319.6ms/step | 15.2 samples/s +Step 18870/44000 | Loss: 3.0220 | LR: 3.13e-04 | 1315.3ms/step | 15.2 samples/s +Step 18880/44000 | Loss: 3.0429 | LR: 3.13e-04 | 1321.0ms/step | 15.1 samples/s +Step 18890/44000 | Loss: 3.0307 | LR: 3.13e-04 | 1318.5ms/step | 15.2 samples/s +Step 18900/44000 | Loss: 3.0285 | LR: 3.12e-04 | 1320.5ms/step | 15.1 samples/s +Step 18910/44000 | Loss: 3.0209 | LR: 3.12e-04 | 1319.3ms/step | 15.2 samples/s +Step 18920/44000 | Loss: 3.0506 | LR: 3.12e-04 | 1320.4ms/step | 15.1 samples/s +Step 18930/44000 | Loss: 3.0240 | LR: 3.12e-04 | 1320.6ms/step | 15.1 samples/s +Step 18940/44000 | Loss: 3.0361 | LR: 3.12e-04 | 1316.6ms/step | 15.2 samples/s +Step 18950/44000 | Loss: 3.0245 | LR: 3.12e-04 | 1317.8ms/step | 15.2 samples/s +Step 18960/44000 | Loss: 3.0514 | LR: 3.11e-04 | 1317.5ms/step | 15.2 samples/s +Step 18970/44000 | Loss: 3.0248 | LR: 3.11e-04 | 1319.1ms/step | 15.2 samples/s +Step 18980/44000 | Loss: 2.9952 | LR: 3.11e-04 | 1318.8ms/step | 15.2 samples/s +Step 18990/44000 | Loss: 3.0225 | LR: 3.11e-04 | 1319.7ms/step | 15.2 samples/s +Step 19000/44000 | Loss: 2.9960 | LR: 3.11e-04 | 1317.5ms/step | 15.2 samples/s +Validation at step 19000: Loss = 3.0207, Perplexity = 20.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-19000 +Step 19010/44000 | Loss: 2.9819 | LR: 3.11e-04 | 3829.1ms/step | 5.2 samples/s +Step 19020/44000 | Loss: 3.0054 | LR: 3.10e-04 | 1314.2ms/step | 15.2 samples/s +Step 19030/44000 | Loss: 3.0286 | LR: 3.10e-04 | 1315.9ms/step | 15.2 samples/s +Step 19040/44000 | Loss: 2.9999 | LR: 3.10e-04 | 1320.2ms/step | 15.1 samples/s +Step 19050/44000 | Loss: 3.0149 | LR: 3.10e-04 | 1320.2ms/step | 15.1 samples/s +Step 19060/44000 | Loss: 3.0456 | LR: 3.10e-04 | 1318.8ms/step | 15.2 samples/s +Step 19070/44000 | Loss: 3.0045 | LR: 3.09e-04 | 1321.7ms/step | 15.1 samples/s +Step 19080/44000 | Loss: 3.0183 | LR: 3.09e-04 | 1320.5ms/step | 15.1 samples/s +Step 19090/44000 | Loss: 2.9699 | LR: 3.09e-04 | 1320.2ms/step | 15.1 samples/s +Step 19100/44000 | Loss: 3.0373 | LR: 3.09e-04 | 1326.8ms/step | 15.1 samples/s +Step 19110/44000 | Loss: 3.0132 | LR: 3.09e-04 | 1321.8ms/step | 15.1 samples/s +Step 19120/44000 | Loss: 3.0439 | LR: 3.09e-04 | 1320.0ms/step | 15.2 samples/s +Step 19130/44000 | Loss: 2.9850 | LR: 3.08e-04 | 1319.4ms/step | 15.2 samples/s +Step 19140/44000 | Loss: 3.0257 | LR: 3.08e-04 | 1323.3ms/step | 15.1 samples/s +Step 19150/44000 | Loss: 2.9866 | LR: 3.08e-04 | 1321.3ms/step | 15.1 samples/s +Step 19160/44000 | Loss: 2.9996 | LR: 3.08e-04 | 1319.2ms/step | 15.2 samples/s +Step 19170/44000 | Loss: 2.9671 | LR: 3.08e-04 | 1315.8ms/step | 15.2 samples/s +Step 19180/44000 | Loss: 2.9844 | LR: 3.08e-04 | 1315.7ms/step | 15.2 samples/s +Step 19190/44000 | Loss: 2.9978 | LR: 3.07e-04 | 1315.6ms/step | 15.2 samples/s +Step 19200/44000 | Loss: 3.0054 | LR: 3.07e-04 | 1318.4ms/step | 15.2 samples/s +Step 19210/44000 | Loss: 3.0036 | LR: 3.07e-04 | 1317.4ms/step | 15.2 samples/s +Step 19220/44000 | Loss: 3.0154 | LR: 3.07e-04 | 1317.4ms/step | 15.2 samples/s +Step 19230/44000 | Loss: 3.0252 | LR: 3.07e-04 | 1319.1ms/step | 15.2 samples/s +Step 19240/44000 | Loss: 3.0194 | LR: 3.06e-04 | 1315.5ms/step | 15.2 samples/s +Step 19250/44000 | Loss: 3.0282 | LR: 3.06e-04 | 1317.5ms/step | 15.2 samples/s +Validation at step 19250: Loss = 3.0179, Perplexity = 20.45 +Step 19260/44000 | Loss: 3.0778 | LR: 3.06e-04 | 3532.9ms/step | 5.7 samples/s +Step 19270/44000 | Loss: 3.0206 | LR: 3.06e-04 | 1319.2ms/step | 15.2 samples/s +Step 19280/44000 | Loss: 3.0344 | LR: 3.06e-04 | 1318.4ms/step | 15.2 samples/s +Step 19290/44000 | Loss: 3.0067 | LR: 3.06e-04 | 1320.8ms/step | 15.1 samples/s +Step 19300/44000 | Loss: 2.9839 | LR: 3.05e-04 | 1319.4ms/step | 15.2 samples/s +Step 19310/44000 | Loss: 3.0104 | LR: 3.05e-04 | 1318.7ms/step | 15.2 samples/s +Step 19320/44000 | Loss: 3.0085 | LR: 3.05e-04 | 1318.2ms/step | 15.2 samples/s +Step 19330/44000 | Loss: 3.0305 | LR: 3.05e-04 | 1317.9ms/step | 15.2 samples/s +Step 19340/44000 | Loss: 3.0079 | LR: 3.05e-04 | 1315.9ms/step | 15.2 samples/s +Step 19350/44000 | Loss: 3.0167 | LR: 3.04e-04 | 1315.9ms/step | 15.2 samples/s +Step 19360/44000 | Loss: 2.9795 | LR: 3.04e-04 | 1316.7ms/step | 15.2 samples/s +Step 19370/44000 | Loss: 3.0152 | LR: 3.04e-04 | 1318.7ms/step | 15.2 samples/s +Step 19380/44000 | Loss: 2.9993 | LR: 3.04e-04 | 1321.1ms/step | 15.1 samples/s +Step 19390/44000 | Loss: 3.0207 | LR: 3.04e-04 | 1318.1ms/step | 15.2 samples/s +Step 19400/44000 | Loss: 3.0143 | LR: 3.04e-04 | 1322.2ms/step | 15.1 samples/s +Step 19410/44000 | Loss: 3.0237 | LR: 3.03e-04 | 1319.2ms/step | 15.2 samples/s +Step 19420/44000 | Loss: 3.0259 | LR: 3.03e-04 | 1320.6ms/step | 15.1 samples/s +Step 19430/44000 | Loss: 3.0084 | LR: 3.03e-04 | 1319.5ms/step | 15.2 samples/s +Step 19440/44000 | Loss: 2.9815 | LR: 3.03e-04 | 1317.7ms/step | 15.2 samples/s +Step 19450/44000 | Loss: 2.9949 | LR: 3.03e-04 | 1319.0ms/step | 15.2 samples/s +Step 19460/44000 | Loss: 3.0151 | LR: 3.03e-04 | 1319.4ms/step | 15.2 samples/s +Step 19470/44000 | Loss: 2.9897 | LR: 3.02e-04 | 1319.6ms/step | 15.2 samples/s +Step 19480/44000 | Loss: 3.0351 | LR: 3.02e-04 | 1320.1ms/step | 15.2 samples/s +Step 19490/44000 | Loss: 2.9870 | LR: 3.02e-04 | 1316.5ms/step | 15.2 samples/s +Step 19500/44000 | Loss: 3.0402 | LR: 3.02e-04 | 1320.2ms/step | 15.1 samples/s +Validation at step 19500: Loss = 3.0148, Perplexity = 20.38 +Step 19510/44000 | Loss: 3.0216 | LR: 3.02e-04 | 3528.8ms/step | 5.7 samples/s +Step 19520/44000 | Loss: 3.0099 | LR: 3.01e-04 | 1319.9ms/step | 15.2 samples/s +Step 19530/44000 | Loss: 3.0119 | LR: 3.01e-04 | 1319.1ms/step | 15.2 samples/s +Step 19540/44000 | Loss: 3.0238 | LR: 3.01e-04 | 1320.3ms/step | 15.1 samples/s +Step 19550/44000 | Loss: 2.9951 | LR: 3.01e-04 | 1319.7ms/step | 15.2 samples/s +Step 19560/44000 | Loss: 3.0044 | LR: 3.01e-04 | 1318.3ms/step | 15.2 samples/s +Step 19570/44000 | Loss: 2.9985 | LR: 3.01e-04 | 1320.2ms/step | 15.1 samples/s +Step 19580/44000 | Loss: 2.9887 | LR: 3.00e-04 | 1317.4ms/step | 15.2 samples/s +Step 19590/44000 | Loss: 3.0296 | LR: 3.00e-04 | 1321.1ms/step | 15.1 samples/s +Step 19600/44000 | Loss: 3.0101 | LR: 3.00e-04 | 1317.4ms/step | 15.2 samples/s +Step 19610/44000 | Loss: 2.9954 | LR: 3.00e-04 | 1318.7ms/step | 15.2 samples/s +Step 19620/44000 | Loss: 3.0291 | LR: 3.00e-04 | 1317.8ms/step | 15.2 samples/s +Step 19630/44000 | Loss: 3.0009 | LR: 3.00e-04 | 1319.8ms/step | 15.2 samples/s +Step 19640/44000 | Loss: 2.9973 | LR: 2.99e-04 | 1322.9ms/step | 15.1 samples/s +Step 19650/44000 | Loss: 2.9901 | LR: 2.99e-04 | 1322.3ms/step | 15.1 samples/s +Step 19660/44000 | Loss: 3.0061 | LR: 2.99e-04 | 1318.0ms/step | 15.2 samples/s +Step 19670/44000 | Loss: 2.9969 | LR: 2.99e-04 | 1321.4ms/step | 15.1 samples/s +Step 19680/44000 | Loss: 3.0238 | LR: 2.99e-04 | 1320.1ms/step | 15.2 samples/s +Step 19690/44000 | Loss: 3.0089 | LR: 2.98e-04 | 1320.6ms/step | 15.1 samples/s +Step 19700/44000 | Loss: 3.0353 | LR: 2.98e-04 | 1319.7ms/step | 15.2 samples/s +Step 19710/44000 | Loss: 2.9902 | LR: 2.98e-04 | 1318.2ms/step | 15.2 samples/s +Step 19720/44000 | Loss: 3.0057 | LR: 2.98e-04 | 1316.3ms/step | 15.2 samples/s +Step 19730/44000 | Loss: 3.0315 | LR: 2.98e-04 | 1315.8ms/step | 15.2 samples/s +Step 19740/44000 | Loss: 2.9955 | LR: 2.98e-04 | 1316.6ms/step | 15.2 samples/s +Step 19750/44000 | Loss: 2.9880 | LR: 2.97e-04 | 1321.6ms/step | 15.1 samples/s +Validation at step 19750: Loss = 3.0132, Perplexity = 20.35 +Step 19760/44000 | Loss: 2.9922 | LR: 2.97e-04 | 3532.5ms/step | 5.7 samples/s +Step 19770/44000 | Loss: 3.0148 | LR: 2.97e-04 | 1318.4ms/step | 15.2 samples/s +Step 19780/44000 | Loss: 3.0272 | LR: 2.97e-04 | 1316.9ms/step | 15.2 samples/s +Step 19790/44000 | Loss: 3.0185 | LR: 2.97e-04 | 1319.8ms/step | 15.2 samples/s +Step 19800/44000 | Loss: 2.9952 | LR: 2.96e-04 | 1318.8ms/step | 15.2 samples/s +Step 19810/44000 | Loss: 3.0011 | LR: 2.96e-04 | 1318.8ms/step | 15.2 samples/s +Step 19820/44000 | Loss: 3.0159 | LR: 2.96e-04 | 1316.2ms/step | 15.2 samples/s +Step 19830/44000 | Loss: 3.0242 | LR: 2.96e-04 | 1317.3ms/step | 15.2 samples/s +Step 19840/44000 | Loss: 3.0027 | LR: 2.96e-04 | 1319.6ms/step | 15.2 samples/s +Step 19850/44000 | Loss: 3.0229 | LR: 2.96e-04 | 1319.2ms/step | 15.2 samples/s +Step 19860/44000 | Loss: 2.9588 | LR: 2.95e-04 | 1320.4ms/step | 15.1 samples/s +Step 19870/44000 | Loss: 3.0140 | LR: 2.95e-04 | 1318.3ms/step | 15.2 samples/s +Step 19880/44000 | Loss: 2.9980 | LR: 2.95e-04 | 1318.7ms/step | 15.2 samples/s +Step 19890/44000 | Loss: 3.0164 | LR: 2.95e-04 | 1316.7ms/step | 15.2 samples/s +Step 19900/44000 | Loss: 3.0054 | LR: 2.95e-04 | 1320.2ms/step | 15.1 samples/s +Step 19910/44000 | Loss: 3.0186 | LR: 2.95e-04 | 1319.2ms/step | 15.2 samples/s +Step 19920/44000 | Loss: 2.9923 | LR: 2.94e-04 | 1317.6ms/step | 15.2 samples/s +Step 19930/44000 | Loss: 2.9880 | LR: 2.94e-04 | 1321.2ms/step | 15.1 samples/s +Step 19940/44000 | Loss: 3.0370 | LR: 2.94e-04 | 1318.7ms/step | 15.2 samples/s +Step 19950/44000 | Loss: 3.0285 | LR: 2.94e-04 | 1320.4ms/step | 15.1 samples/s +Step 19960/44000 | Loss: 3.0020 | LR: 2.94e-04 | 1318.6ms/step | 15.2 samples/s +Step 19970/44000 | Loss: 3.0441 | LR: 2.93e-04 | 1318.9ms/step | 15.2 samples/s +Step 19980/44000 | Loss: 2.9995 | LR: 2.93e-04 | 1320.1ms/step | 15.2 samples/s +Step 19990/44000 | Loss: 3.0016 | LR: 2.93e-04 | 1319.2ms/step | 15.2 samples/s +Step 20000/44000 | Loss: 3.0351 | LR: 2.93e-04 | 1319.4ms/step | 15.2 samples/s +Validation at step 20000: Loss = 3.0097, Perplexity = 20.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-20000 +Step 20010/44000 | Loss: 2.9741 | LR: 2.93e-04 | 3864.7ms/step | 5.2 samples/s +Step 20020/44000 | Loss: 3.0159 | LR: 2.93e-04 | 1318.9ms/step | 15.2 samples/s +Step 20030/44000 | Loss: 3.0045 | LR: 2.92e-04 | 1315.9ms/step | 15.2 samples/s +Step 20040/44000 | Loss: 2.9936 | LR: 2.92e-04 | 1319.0ms/step | 15.2 samples/s +Step 20050/44000 | Loss: 2.9969 | LR: 2.92e-04 | 1321.8ms/step | 15.1 samples/s +Step 20060/44000 | Loss: 3.0349 | LR: 2.92e-04 | 1320.6ms/step | 15.1 samples/s +Step 20070/44000 | Loss: 2.9865 | LR: 2.92e-04 | 1319.5ms/step | 15.2 samples/s +Step 20080/44000 | Loss: 2.9806 | LR: 2.91e-04 | 1319.4ms/step | 15.2 samples/s +Step 20090/44000 | Loss: 2.9836 | LR: 2.91e-04 | 1318.8ms/step | 15.2 samples/s +Step 20100/44000 | Loss: 2.9924 | LR: 2.91e-04 | 1318.7ms/step | 15.2 samples/s +Step 20110/44000 | Loss: 2.9876 | LR: 2.91e-04 | 1318.3ms/step | 15.2 samples/s +Step 20120/44000 | Loss: 2.9932 | LR: 2.91e-04 | 1315.0ms/step | 15.2 samples/s +Step 20130/44000 | Loss: 3.0195 | LR: 2.91e-04 | 1317.1ms/step | 15.2 samples/s +Step 20140/44000 | Loss: 3.0213 | LR: 2.90e-04 | 1315.8ms/step | 15.2 samples/s +Step 20150/44000 | Loss: 2.9965 | LR: 2.90e-04 | 1317.7ms/step | 15.2 samples/s +Step 20160/44000 | Loss: 2.9812 | LR: 2.90e-04 | 1317.6ms/step | 15.2 samples/s +Step 20170/44000 | Loss: 3.0231 | LR: 2.90e-04 | 1318.7ms/step | 15.2 samples/s +Step 20180/44000 | Loss: 3.0097 | LR: 2.90e-04 | 1316.9ms/step | 15.2 samples/s +Step 20190/44000 | Loss: 2.9965 | LR: 2.90e-04 | 1321.4ms/step | 15.1 samples/s +Step 20200/44000 | Loss: 3.0080 | LR: 2.89e-04 | 1320.0ms/step | 15.2 samples/s +Step 20210/44000 | Loss: 3.0072 | LR: 2.89e-04 | 1316.7ms/step | 15.2 samples/s +Step 20220/44000 | Loss: 3.0244 | LR: 2.89e-04 | 1318.3ms/step | 15.2 samples/s +Step 20230/44000 | Loss: 3.0087 | LR: 2.89e-04 | 1319.1ms/step | 15.2 samples/s +Step 20240/44000 | Loss: 3.0091 | LR: 2.89e-04 | 1322.3ms/step | 15.1 samples/s +Step 20250/44000 | Loss: 2.9923 | LR: 2.88e-04 | 1318.8ms/step | 15.2 samples/s +Validation at step 20250: Loss = 3.0072, Perplexity = 20.23 +Step 20260/44000 | Loss: 3.0122 | LR: 2.88e-04 | 3535.6ms/step | 5.7 samples/s +Step 20270/44000 | Loss: 2.9827 | LR: 2.88e-04 | 1320.4ms/step | 15.1 samples/s +Step 20280/44000 | Loss: 3.0025 | LR: 2.88e-04 | 1318.7ms/step | 15.2 samples/s +Step 20290/44000 | Loss: 3.0081 | LR: 2.88e-04 | 1319.8ms/step | 15.2 samples/s +Step 20300/44000 | Loss: 3.0003 | LR: 2.88e-04 | 1320.9ms/step | 15.1 samples/s +Step 20310/44000 | Loss: 3.0170 | LR: 2.87e-04 | 1320.1ms/step | 15.2 samples/s +Step 20320/44000 | Loss: 3.0080 | LR: 2.87e-04 | 1321.4ms/step | 15.1 samples/s +Step 20330/44000 | Loss: 3.0129 | LR: 2.87e-04 | 1319.1ms/step | 15.2 samples/s +Step 20340/44000 | Loss: 2.9819 | LR: 2.87e-04 | 1319.6ms/step | 15.2 samples/s +Step 20350/44000 | Loss: 3.0127 | LR: 2.87e-04 | 1319.3ms/step | 15.2 samples/s +Step 20360/44000 | Loss: 3.0018 | LR: 2.86e-04 | 1320.3ms/step | 15.1 samples/s +Step 20370/44000 | Loss: 3.0181 | LR: 2.86e-04 | 1320.3ms/step | 15.1 samples/s +Step 20380/44000 | Loss: 2.9928 | LR: 2.86e-04 | 1319.5ms/step | 15.2 samples/s +Step 20390/44000 | Loss: 3.0038 | LR: 2.86e-04 | 1316.3ms/step | 15.2 samples/s +Step 20400/44000 | Loss: 3.0193 | LR: 2.86e-04 | 1316.7ms/step | 15.2 samples/s +Step 20410/44000 | Loss: 2.9985 | LR: 2.86e-04 | 1318.3ms/step | 15.2 samples/s +Step 20420/44000 | Loss: 3.0349 | LR: 2.85e-04 | 1318.6ms/step | 15.2 samples/s +Step 20430/44000 | Loss: 2.9696 | LR: 2.85e-04 | 1320.6ms/step | 15.1 samples/s +Step 20440/44000 | Loss: 3.0114 | LR: 2.85e-04 | 1321.8ms/step | 15.1 samples/s +Step 20450/44000 | Loss: 3.0024 | LR: 2.85e-04 | 1320.6ms/step | 15.1 samples/s +Step 20460/44000 | Loss: 3.0118 | LR: 2.85e-04 | 1319.1ms/step | 15.2 samples/s +Step 20470/44000 | Loss: 2.9507 | LR: 2.84e-04 | 1318.3ms/step | 15.2 samples/s +Step 20480/44000 | Loss: 2.9835 | LR: 2.84e-04 | 1317.9ms/step | 15.2 samples/s +Step 20490/44000 | Loss: 3.0323 | LR: 2.84e-04 | 1322.3ms/step | 15.1 samples/s +Step 20500/44000 | Loss: 3.0036 | LR: 2.84e-04 | 1319.8ms/step | 15.2 samples/s +Validation at step 20500: Loss = 3.0048, Perplexity = 20.18 +Step 20510/44000 | Loss: 3.0048 | LR: 2.84e-04 | 3531.1ms/step | 5.7 samples/s +Step 20520/44000 | Loss: 2.9909 | LR: 2.84e-04 | 1319.6ms/step | 15.2 samples/s +Step 20530/44000 | Loss: 2.9909 | LR: 2.83e-04 | 1323.0ms/step | 15.1 samples/s +Step 20540/44000 | Loss: 2.9866 | LR: 2.83e-04 | 1321.6ms/step | 15.1 samples/s +Step 20550/44000 | Loss: 2.9938 | LR: 2.83e-04 | 1321.8ms/step | 15.1 samples/s +Step 20560/44000 | Loss: 2.9917 | LR: 2.83e-04 | 1319.0ms/step | 15.2 samples/s +Step 20570/44000 | Loss: 2.9625 | LR: 2.83e-04 | 1320.8ms/step | 15.1 samples/s +Step 20580/44000 | Loss: 3.0348 | LR: 2.83e-04 | 1317.9ms/step | 15.2 samples/s +Step 20590/44000 | Loss: 2.9915 | LR: 2.82e-04 | 1320.0ms/step | 15.2 samples/s +Step 20600/44000 | Loss: 2.9746 | LR: 2.82e-04 | 1317.9ms/step | 15.2 samples/s +Step 20610/44000 | Loss: 2.9600 | LR: 2.82e-04 | 1319.7ms/step | 15.2 samples/s +Step 20620/44000 | Loss: 3.0056 | LR: 2.82e-04 | 1319.1ms/step | 15.2 samples/s +Step 20630/44000 | Loss: 3.0060 | LR: 2.82e-04 | 1321.8ms/step | 15.1 samples/s +Step 20640/44000 | Loss: 2.9673 | LR: 2.81e-04 | 1318.2ms/step | 15.2 samples/s +Step 20650/44000 | Loss: 2.9693 | LR: 2.81e-04 | 1319.0ms/step | 15.2 samples/s +Step 20660/44000 | Loss: 2.9868 | LR: 2.81e-04 | 1319.9ms/step | 15.2 samples/s +Step 20670/44000 | Loss: 3.0056 | LR: 2.81e-04 | 1320.0ms/step | 15.2 samples/s +Step 20680/44000 | Loss: 3.0149 | LR: 2.81e-04 | 1316.5ms/step | 15.2 samples/s +Step 20690/44000 | Loss: 2.9955 | LR: 2.81e-04 | 1315.1ms/step | 15.2 samples/s +Step 20700/44000 | Loss: 2.9981 | LR: 2.80e-04 | 1318.5ms/step | 15.2 samples/s +Step 20710/44000 | Loss: 3.0246 | LR: 2.80e-04 | 1320.8ms/step | 15.1 samples/s +Step 20720/44000 | Loss: 2.9986 | LR: 2.80e-04 | 1320.0ms/step | 15.2 samples/s +Step 20730/44000 | Loss: 2.9594 | LR: 2.80e-04 | 1317.0ms/step | 15.2 samples/s +Step 20740/44000 | Loss: 2.9811 | LR: 2.80e-04 | 1318.4ms/step | 15.2 samples/s +Step 20750/44000 | Loss: 2.9791 | LR: 2.79e-04 | 1316.3ms/step | 15.2 samples/s +Validation at step 20750: Loss = 3.0016, Perplexity = 20.12 +Step 20760/44000 | Loss: 3.0133 | LR: 2.79e-04 | 3533.4ms/step | 5.7 samples/s +Step 20770/44000 | Loss: 2.9824 | LR: 2.79e-04 | 1320.3ms/step | 15.1 samples/s +Step 20780/44000 | Loss: 2.9913 | LR: 2.79e-04 | 1319.4ms/step | 15.2 samples/s +Step 20790/44000 | Loss: 2.9990 | LR: 2.79e-04 | 1320.2ms/step | 15.1 samples/s +Step 20800/44000 | Loss: 3.0084 | LR: 2.79e-04 | 1320.5ms/step | 15.1 samples/s +Step 20810/44000 | Loss: 3.0044 | LR: 2.78e-04 | 1321.1ms/step | 15.1 samples/s +Step 20820/44000 | Loss: 2.9713 | LR: 2.78e-04 | 1319.9ms/step | 15.2 samples/s +Step 20830/44000 | Loss: 2.9892 | LR: 2.78e-04 | 1327.8ms/step | 15.1 samples/s +Step 20840/44000 | Loss: 2.9741 | LR: 2.78e-04 | 1321.7ms/step | 15.1 samples/s +Step 20850/44000 | Loss: 2.9994 | LR: 2.78e-04 | 1321.6ms/step | 15.1 samples/s +Step 20860/44000 | Loss: 2.9995 | LR: 2.77e-04 | 1324.1ms/step | 15.1 samples/s +Step 20870/44000 | Loss: 2.9695 | LR: 2.77e-04 | 1320.8ms/step | 15.1 samples/s +Step 20880/44000 | Loss: 3.0130 | LR: 2.77e-04 | 1318.7ms/step | 15.2 samples/s +Step 20890/44000 | Loss: 3.0067 | LR: 2.77e-04 | 1319.9ms/step | 15.2 samples/s +Step 20900/44000 | Loss: 2.9898 | LR: 2.77e-04 | 1321.2ms/step | 15.1 samples/s +Step 20910/44000 | Loss: 3.0176 | LR: 2.77e-04 | 1319.3ms/step | 15.2 samples/s +Step 20920/44000 | Loss: 3.0301 | LR: 2.76e-04 | 1318.0ms/step | 15.2 samples/s +Step 20930/44000 | Loss: 3.0262 | LR: 2.76e-04 | 1317.4ms/step | 15.2 samples/s +Step 20940/44000 | Loss: 3.0299 | LR: 2.76e-04 | 1317.7ms/step | 15.2 samples/s +Step 20950/44000 | Loss: 2.9917 | LR: 2.76e-04 | 1317.7ms/step | 15.2 samples/s +Step 20960/44000 | Loss: 3.0191 | LR: 2.76e-04 | 1318.1ms/step | 15.2 samples/s +Step 20970/44000 | Loss: 3.0041 | LR: 2.75e-04 | 1317.5ms/step | 15.2 samples/s +Step 20980/44000 | Loss: 3.0219 | LR: 2.75e-04 | 1317.9ms/step | 15.2 samples/s +Step 20990/44000 | Loss: 2.9802 | LR: 2.75e-04 | 1321.5ms/step | 15.1 samples/s +Step 21000/44000 | Loss: 2.9812 | LR: 2.75e-04 | 1320.9ms/step | 15.1 samples/s +Validation at step 21000: Loss = 2.9987, Perplexity = 20.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-21000 +Step 21010/44000 | Loss: 2.9989 | LR: 2.75e-04 | 3847.0ms/step | 5.2 samples/s +Step 21020/44000 | Loss: 3.0002 | LR: 2.75e-04 | 1319.2ms/step | 15.2 samples/s +Step 21030/44000 | Loss: 3.0285 | LR: 2.74e-04 | 1320.7ms/step | 15.1 samples/s +Step 21040/44000 | Loss: 3.0192 | LR: 2.74e-04 | 1320.8ms/step | 15.1 samples/s +Step 21050/44000 | Loss: 2.9912 | LR: 2.74e-04 | 1320.3ms/step | 15.1 samples/s +Step 21060/44000 | Loss: 2.9839 | LR: 2.74e-04 | 1321.9ms/step | 15.1 samples/s +Step 21070/44000 | Loss: 3.0139 | LR: 2.74e-04 | 1320.2ms/step | 15.1 samples/s +Step 21080/44000 | Loss: 2.9946 | LR: 2.73e-04 | 1320.1ms/step | 15.1 samples/s +Step 21090/44000 | Loss: 2.9969 | LR: 2.73e-04 | 1319.5ms/step | 15.2 samples/s +Step 21100/44000 | Loss: 2.9947 | LR: 2.73e-04 | 1320.0ms/step | 15.2 samples/s +Step 21110/44000 | Loss: 2.9831 | LR: 2.73e-04 | 1318.5ms/step | 15.2 samples/s +Step 21120/44000 | Loss: 2.9982 | LR: 2.73e-04 | 1317.8ms/step | 15.2 samples/s +Step 21130/44000 | Loss: 2.9834 | LR: 2.73e-04 | 1319.0ms/step | 15.2 samples/s +Step 21140/44000 | Loss: 2.9927 | LR: 2.72e-04 | 1319.6ms/step | 15.2 samples/s +Step 21150/44000 | Loss: 2.9896 | LR: 2.72e-04 | 1323.6ms/step | 15.1 samples/s +Step 21160/44000 | Loss: 2.9765 | LR: 2.72e-04 | 1321.0ms/step | 15.1 samples/s +Step 21170/44000 | Loss: 3.0116 | LR: 2.72e-04 | 1317.8ms/step | 15.2 samples/s +Step 21180/44000 | Loss: 3.0043 | LR: 2.72e-04 | 1317.2ms/step | 15.2 samples/s +Step 21190/44000 | Loss: 2.9781 | LR: 2.71e-04 | 1315.6ms/step | 15.2 samples/s +Step 21200/44000 | Loss: 2.9859 | LR: 2.71e-04 | 1317.4ms/step | 15.2 samples/s +Step 21210/44000 | Loss: 2.9948 | LR: 2.71e-04 | 1319.7ms/step | 15.2 samples/s +Step 21220/44000 | Loss: 2.9934 | LR: 2.71e-04 | 1320.2ms/step | 15.1 samples/s +Step 21230/44000 | Loss: 2.9563 | LR: 2.71e-04 | 1321.4ms/step | 15.1 samples/s +Step 21240/44000 | Loss: 2.9879 | LR: 2.71e-04 | 1319.9ms/step | 15.2 samples/s +Step 21250/44000 | Loss: 3.0056 | LR: 2.70e-04 | 1321.8ms/step | 15.1 samples/s +Validation at step 21250: Loss = 2.9970, Perplexity = 20.03 +Step 21260/44000 | Loss: 3.0162 | LR: 2.70e-04 | 3531.4ms/step | 5.7 samples/s +Step 21270/44000 | Loss: 3.0291 | LR: 2.70e-04 | 1323.7ms/step | 15.1 samples/s +Step 21280/44000 | Loss: 2.9788 | LR: 2.70e-04 | 1318.9ms/step | 15.2 samples/s +Step 21290/44000 | Loss: 3.0014 | LR: 2.70e-04 | 1320.6ms/step | 15.1 samples/s +Step 21300/44000 | Loss: 3.0087 | LR: 2.70e-04 | 1322.3ms/step | 15.1 samples/s +Step 21310/44000 | Loss: 2.9876 | LR: 2.69e-04 | 1319.2ms/step | 15.2 samples/s +Step 21320/44000 | Loss: 2.9961 | LR: 2.69e-04 | 1316.2ms/step | 15.2 samples/s +Step 21330/44000 | Loss: 3.0013 | LR: 2.69e-04 | 1319.6ms/step | 15.2 samples/s +Step 21340/44000 | Loss: 2.9842 | LR: 2.69e-04 | 1319.9ms/step | 15.2 samples/s +Step 21350/44000 | Loss: 2.9725 | LR: 2.69e-04 | 1320.0ms/step | 15.2 samples/s +Step 21360/44000 | Loss: 2.9696 | LR: 2.68e-04 | 1318.7ms/step | 15.2 samples/s +Step 21370/44000 | Loss: 2.9664 | LR: 2.68e-04 | 1319.5ms/step | 15.2 samples/s +Step 21380/44000 | Loss: 2.9913 | LR: 2.68e-04 | 1319.5ms/step | 15.2 samples/s +Step 21390/44000 | Loss: 2.9960 | LR: 2.68e-04 | 1320.6ms/step | 15.1 samples/s +Step 21400/44000 | Loss: 3.0157 | LR: 2.68e-04 | 1319.8ms/step | 15.2 samples/s +Step 21410/44000 | Loss: 3.0012 | LR: 2.68e-04 | 1317.6ms/step | 15.2 samples/s +Step 21420/44000 | Loss: 2.9461 | LR: 2.67e-04 | 1318.8ms/step | 15.2 samples/s +Step 21430/44000 | Loss: 3.0092 | LR: 2.67e-04 | 1317.8ms/step | 15.2 samples/s +Step 21440/44000 | Loss: 3.0073 | LR: 2.67e-04 | 1316.7ms/step | 15.2 samples/s +Step 21450/44000 | Loss: 3.0278 | LR: 2.67e-04 | 1318.8ms/step | 15.2 samples/s +Step 21460/44000 | Loss: 2.9556 | LR: 2.67e-04 | 1316.6ms/step | 15.2 samples/s +Step 21470/44000 | Loss: 2.9874 | LR: 2.66e-04 | 1319.3ms/step | 15.2 samples/s +Step 21480/44000 | Loss: 2.9958 | LR: 2.66e-04 | 1318.3ms/step | 15.2 samples/s +Step 21490/44000 | Loss: 3.0170 | LR: 2.66e-04 | 1318.8ms/step | 15.2 samples/s +Step 21500/44000 | Loss: 2.9920 | LR: 2.66e-04 | 1317.0ms/step | 15.2 samples/s +Validation at step 21500: Loss = 2.9940, Perplexity = 19.96 +Step 21510/44000 | Loss: 3.0056 | LR: 2.66e-04 | 3524.2ms/step | 5.7 samples/s +Step 21520/44000 | Loss: 2.9848 | LR: 2.66e-04 | 1319.7ms/step | 15.2 samples/s +Step 21530/44000 | Loss: 2.9905 | LR: 2.65e-04 | 1317.4ms/step | 15.2 samples/s +Step 21540/44000 | Loss: 2.9944 | LR: 2.65e-04 | 1320.4ms/step | 15.1 samples/s +Step 21550/44000 | Loss: 3.0242 | LR: 2.65e-04 | 1320.6ms/step | 15.1 samples/s +Step 21560/44000 | Loss: 2.9567 | LR: 2.65e-04 | 1323.7ms/step | 15.1 samples/s +Step 21570/44000 | Loss: 2.9979 | LR: 2.65e-04 | 1320.8ms/step | 15.1 samples/s +Step 21580/44000 | Loss: 2.9778 | LR: 2.64e-04 | 1321.2ms/step | 15.1 samples/s +Step 21590/44000 | Loss: 2.9713 | LR: 2.64e-04 | 1318.9ms/step | 15.2 samples/s +Step 21600/44000 | Loss: 3.0117 | LR: 2.64e-04 | 1324.3ms/step | 15.1 samples/s +Step 21610/44000 | Loss: 2.9869 | LR: 2.64e-04 | 1317.6ms/step | 15.2 samples/s +Step 21620/44000 | Loss: 2.9873 | LR: 2.64e-04 | 1317.9ms/step | 15.2 samples/s +Step 21630/44000 | Loss: 2.9706 | LR: 2.64e-04 | 1321.0ms/step | 15.1 samples/s +Step 21640/44000 | Loss: 2.9859 | LR: 2.63e-04 | 1318.2ms/step | 15.2 samples/s +Step 21650/44000 | Loss: 3.0261 | LR: 2.63e-04 | 1320.3ms/step | 15.1 samples/s +Step 21660/44000 | Loss: 2.9514 | LR: 2.63e-04 | 1320.0ms/step | 15.2 samples/s +Step 21670/44000 | Loss: 2.9742 | LR: 2.63e-04 | 1317.3ms/step | 15.2 samples/s +Step 21680/44000 | Loss: 2.9813 | LR: 2.63e-04 | 1318.9ms/step | 15.2 samples/s +Step 21690/44000 | Loss: 2.9731 | LR: 2.62e-04 | 1319.6ms/step | 15.2 samples/s +Step 21700/44000 | Loss: 2.9943 | LR: 2.62e-04 | 1320.2ms/step | 15.1 samples/s +Step 21710/44000 | Loss: 2.9722 | LR: 2.62e-04 | 1322.4ms/step | 15.1 samples/s +Step 21720/44000 | Loss: 3.0071 | LR: 2.62e-04 | 1319.7ms/step | 15.2 samples/s +Step 21730/44000 | Loss: 2.9435 | LR: 2.62e-04 | 1318.0ms/step | 15.2 samples/s +Step 21740/44000 | Loss: 2.9978 | LR: 2.62e-04 | 1320.9ms/step | 15.1 samples/s +Step 21750/44000 | Loss: 2.9948 | LR: 2.61e-04 | 1318.6ms/step | 15.2 samples/s +Validation at step 21750: Loss = 2.9909, Perplexity = 19.90 +Step 21760/44000 | Loss: 2.9828 | LR: 2.61e-04 | 3534.3ms/step | 5.7 samples/s +Step 21770/44000 | Loss: 2.9981 | LR: 2.61e-04 | 1321.7ms/step | 15.1 samples/s +Step 21780/44000 | Loss: 2.9571 | LR: 2.61e-04 | 1319.0ms/step | 15.2 samples/s +Step 21790/44000 | Loss: 2.9853 | LR: 2.61e-04 | 1317.0ms/step | 15.2 samples/s +Step 21800/44000 | Loss: 2.9868 | LR: 2.60e-04 | 1319.4ms/step | 15.2 samples/s +Step 21810/44000 | Loss: 2.9799 | LR: 2.60e-04 | 1321.3ms/step | 15.1 samples/s +Step 21820/44000 | Loss: 3.0073 | LR: 2.60e-04 | 1321.4ms/step | 15.1 samples/s +Step 21830/44000 | Loss: 2.9898 | LR: 2.60e-04 | 1322.2ms/step | 15.1 samples/s +Step 21840/44000 | Loss: 2.9671 | LR: 2.60e-04 | 1318.2ms/step | 15.2 samples/s +Step 21850/44000 | Loss: 3.0215 | LR: 2.60e-04 | 1320.8ms/step | 15.1 samples/s +Step 21860/44000 | Loss: 2.9805 | LR: 2.59e-04 | 1319.8ms/step | 15.2 samples/s +Step 21870/44000 | Loss: 2.9926 | LR: 2.59e-04 | 1318.4ms/step | 15.2 samples/s +Step 21880/44000 | Loss: 2.9759 | LR: 2.59e-04 | 1321.0ms/step | 15.1 samples/s +Step 21890/44000 | Loss: 3.0270 | LR: 2.59e-04 | 1321.2ms/step | 15.1 samples/s +Step 21900/44000 | Loss: 2.9808 | LR: 2.59e-04 | 1318.9ms/step | 15.2 samples/s +Step 21910/44000 | Loss: 2.9794 | LR: 2.58e-04 | 1320.8ms/step | 15.1 samples/s +Step 21920/44000 | Loss: 2.9699 | LR: 2.58e-04 | 1321.1ms/step | 15.1 samples/s +Step 21930/44000 | Loss: 2.9744 | LR: 2.58e-04 | 1320.3ms/step | 15.1 samples/s +Step 21940/44000 | Loss: 2.9910 | LR: 2.58e-04 | 1319.6ms/step | 15.2 samples/s +Step 21950/44000 | Loss: 2.9953 | LR: 2.58e-04 | 1318.6ms/step | 15.2 samples/s +Step 21960/44000 | Loss: 3.0018 | LR: 2.58e-04 | 1316.6ms/step | 15.2 samples/s +Step 21970/44000 | Loss: 2.9859 | LR: 2.57e-04 | 1319.4ms/step | 15.2 samples/s +Step 21980/44000 | Loss: 2.9755 | LR: 2.57e-04 | 1318.6ms/step | 15.2 samples/s +Step 21990/44000 | Loss: 3.0061 | LR: 2.57e-04 | 1322.5ms/step | 15.1 samples/s +Step 22000/44000 | Loss: 2.9558 | LR: 2.57e-04 | 1320.9ms/step | 15.1 samples/s +Validation at step 22000: Loss = 2.9886, Perplexity = 19.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-22000 +Step 22010/44000 | Loss: 2.9757 | LR: 2.57e-04 | 3842.8ms/step | 5.2 samples/s +Step 22020/44000 | Loss: 2.9699 | LR: 2.56e-04 | 1317.7ms/step | 15.2 samples/s +Step 22030/44000 | Loss: 2.9848 | LR: 2.56e-04 | 1317.5ms/step | 15.2 samples/s +Step 22040/44000 | Loss: 2.9537 | LR: 2.56e-04 | 1320.6ms/step | 15.1 samples/s +Step 22050/44000 | Loss: 2.9863 | LR: 2.56e-04 | 1319.5ms/step | 15.2 samples/s +Step 22060/44000 | Loss: 2.9815 | LR: 2.56e-04 | 1320.5ms/step | 15.1 samples/s +Step 22070/44000 | Loss: 2.9895 | LR: 2.56e-04 | 1320.6ms/step | 15.1 samples/s +Step 22080/44000 | Loss: 3.0155 | LR: 2.55e-04 | 1321.9ms/step | 15.1 samples/s +Step 22090/44000 | Loss: 2.9714 | LR: 2.55e-04 | 1319.4ms/step | 15.2 samples/s +Step 22100/44000 | Loss: 2.9849 | LR: 2.55e-04 | 1320.8ms/step | 15.1 samples/s +Step 22110/44000 | Loss: 2.9916 | LR: 2.55e-04 | 1321.1ms/step | 15.1 samples/s +Step 22120/44000 | Loss: 3.0159 | LR: 2.55e-04 | 1320.4ms/step | 15.1 samples/s +Step 22130/44000 | Loss: 2.9720 | LR: 2.54e-04 | 1320.2ms/step | 15.1 samples/s +Step 22140/44000 | Loss: 2.9779 | LR: 2.54e-04 | 1320.3ms/step | 15.1 samples/s +Step 22150/44000 | Loss: 2.9746 | LR: 2.54e-04 | 1317.9ms/step | 15.2 samples/s +Step 22160/44000 | Loss: 3.0032 | LR: 2.54e-04 | 1320.9ms/step | 15.1 samples/s +Step 22170/44000 | Loss: 2.9979 | LR: 2.54e-04 | 1320.3ms/step | 15.1 samples/s +Step 22180/44000 | Loss: 2.9596 | LR: 2.54e-04 | 1317.9ms/step | 15.2 samples/s +Step 22190/44000 | Loss: 2.9835 | LR: 2.53e-04 | 1318.2ms/step | 15.2 samples/s +Step 22200/44000 | Loss: 2.9956 | LR: 2.53e-04 | 1314.9ms/step | 15.2 samples/s +Step 22210/44000 | Loss: 2.9890 | LR: 2.53e-04 | 1320.6ms/step | 15.1 samples/s +Step 22220/44000 | Loss: 2.9824 | LR: 2.53e-04 | 1318.1ms/step | 15.2 samples/s +Step 22230/44000 | Loss: 2.9749 | LR: 2.53e-04 | 1317.4ms/step | 15.2 samples/s +Step 22240/44000 | Loss: 2.9755 | LR: 2.52e-04 | 1319.5ms/step | 15.2 samples/s +Step 22250/44000 | Loss: 2.9699 | LR: 2.52e-04 | 1320.2ms/step | 15.1 samples/s +Validation at step 22250: Loss = 2.9868, Perplexity = 19.82 +Step 22260/44000 | Loss: 2.9738 | LR: 2.52e-04 | 3534.5ms/step | 5.7 samples/s +Step 22270/44000 | Loss: 2.9473 | LR: 2.52e-04 | 1316.9ms/step | 15.2 samples/s +Step 22280/44000 | Loss: 2.9828 | LR: 2.52e-04 | 1321.5ms/step | 15.1 samples/s +Step 22290/44000 | Loss: 2.9845 | LR: 2.52e-04 | 1321.0ms/step | 15.1 samples/s +Step 22300/44000 | Loss: 2.9758 | LR: 2.51e-04 | 1320.6ms/step | 15.1 samples/s +Step 22310/44000 | Loss: 2.9791 | LR: 2.51e-04 | 1321.3ms/step | 15.1 samples/s +Step 22320/44000 | Loss: 2.9809 | LR: 2.51e-04 | 1320.7ms/step | 15.1 samples/s +Step 22330/44000 | Loss: 2.9879 | LR: 2.51e-04 | 1319.6ms/step | 15.2 samples/s +Step 22340/44000 | Loss: 2.9818 | LR: 2.51e-04 | 1319.1ms/step | 15.2 samples/s +Step 22350/44000 | Loss: 3.0054 | LR: 2.50e-04 | 1317.2ms/step | 15.2 samples/s +Step 22360/44000 | Loss: 3.0017 | LR: 2.50e-04 | 1316.7ms/step | 15.2 samples/s +Step 22370/44000 | Loss: 2.9862 | LR: 2.50e-04 | 1321.5ms/step | 15.1 samples/s +Step 22380/44000 | Loss: 2.9776 | LR: 2.50e-04 | 1321.6ms/step | 15.1 samples/s +Step 22390/44000 | Loss: 2.9817 | LR: 2.50e-04 | 1320.1ms/step | 15.1 samples/s +Step 22400/44000 | Loss: 2.9726 | LR: 2.50e-04 | 1316.3ms/step | 15.2 samples/s +Step 22410/44000 | Loss: 2.9793 | LR: 2.49e-04 | 1318.7ms/step | 15.2 samples/s +Step 22420/44000 | Loss: 2.9818 | LR: 2.49e-04 | 1322.7ms/step | 15.1 samples/s +Step 22430/44000 | Loss: 2.9725 | LR: 2.49e-04 | 1320.6ms/step | 15.1 samples/s +Step 22440/44000 | Loss: 2.9757 | LR: 2.49e-04 | 1318.2ms/step | 15.2 samples/s +Step 22450/44000 | Loss: 2.9717 | LR: 2.49e-04 | 1322.5ms/step | 15.1 samples/s +Step 22460/44000 | Loss: 2.9556 | LR: 2.48e-04 | 1322.0ms/step | 15.1 samples/s +Step 22470/44000 | Loss: 2.9837 | LR: 2.48e-04 | 1317.5ms/step | 15.2 samples/s +Step 22480/44000 | Loss: 2.9717 | LR: 2.48e-04 | 1321.5ms/step | 15.1 samples/s +Step 22490/44000 | Loss: 2.9815 | LR: 2.48e-04 | 1321.4ms/step | 15.1 samples/s +Step 22500/44000 | Loss: 2.9787 | LR: 2.48e-04 | 1320.4ms/step | 15.1 samples/s +Validation at step 22500: Loss = 2.9842, Perplexity = 19.77 +Step 22510/44000 | Loss: 2.9716 | LR: 2.48e-04 | 3536.5ms/step | 5.7 samples/s +Step 22520/44000 | Loss: 2.9773 | LR: 2.47e-04 | 1322.0ms/step | 15.1 samples/s +Step 22530/44000 | Loss: 2.9815 | LR: 2.47e-04 | 1313.4ms/step | 15.2 samples/s +Step 22540/44000 | Loss: 2.9975 | LR: 2.47e-04 | 1315.9ms/step | 15.2 samples/s +Step 22550/44000 | Loss: 2.9856 | LR: 2.47e-04 | 1319.2ms/step | 15.2 samples/s +Step 22560/44000 | Loss: 2.9772 | LR: 2.47e-04 | 1313.6ms/step | 15.2 samples/s +Step 22570/44000 | Loss: 2.9747 | LR: 2.46e-04 | 1314.1ms/step | 15.2 samples/s +Step 22580/44000 | Loss: 3.0056 | LR: 2.46e-04 | 1317.6ms/step | 15.2 samples/s +Step 22590/44000 | Loss: 2.9894 | LR: 2.46e-04 | 1314.4ms/step | 15.2 samples/s +Step 22600/44000 | Loss: 2.9979 | LR: 2.46e-04 | 1314.7ms/step | 15.2 samples/s +Step 22610/44000 | Loss: 2.9435 | LR: 2.46e-04 | 1313.2ms/step | 15.2 samples/s +Step 22620/44000 | Loss: 2.9951 | LR: 2.46e-04 | 1314.4ms/step | 15.2 samples/s +Step 22630/44000 | Loss: 2.9699 | LR: 2.45e-04 | 1312.8ms/step | 15.2 samples/s +Step 22640/44000 | Loss: 2.9556 | LR: 2.45e-04 | 1313.8ms/step | 15.2 samples/s +Step 22650/44000 | Loss: 2.9452 | LR: 2.45e-04 | 1314.5ms/step | 15.2 samples/s +Step 22660/44000 | Loss: 2.9717 | LR: 2.45e-04 | 1314.8ms/step | 15.2 samples/s +Step 22670/44000 | Loss: 2.9655 | LR: 2.45e-04 | 1312.5ms/step | 15.2 samples/s +Step 22680/44000 | Loss: 2.9571 | LR: 2.44e-04 | 1313.2ms/step | 15.2 samples/s +Step 22690/44000 | Loss: 2.9646 | LR: 2.44e-04 | 1311.8ms/step | 15.2 samples/s +Step 22700/44000 | Loss: 2.9869 | LR: 2.44e-04 | 1312.8ms/step | 15.2 samples/s +Step 22710/44000 | Loss: 2.9970 | LR: 2.44e-04 | 1313.7ms/step | 15.2 samples/s +Step 22720/44000 | Loss: 2.9411 | LR: 2.44e-04 | 1314.3ms/step | 15.2 samples/s +Step 22730/44000 | Loss: 3.0035 | LR: 2.44e-04 | 1314.3ms/step | 15.2 samples/s +Step 22740/44000 | Loss: 2.9534 | LR: 2.43e-04 | 1314.3ms/step | 15.2 samples/s +Step 22750/44000 | Loss: 2.9756 | LR: 2.43e-04 | 1313.8ms/step | 15.2 samples/s +Validation at step 22750: Loss = 2.9815, Perplexity = 19.72 +Step 22760/44000 | Loss: 2.9585 | LR: 2.43e-04 | 3515.2ms/step | 5.7 samples/s +Step 22770/44000 | Loss: 2.9642 | LR: 2.43e-04 | 1315.8ms/step | 15.2 samples/s +Step 22780/44000 | Loss: 3.0086 | LR: 2.43e-04 | 1314.4ms/step | 15.2 samples/s +Step 22790/44000 | Loss: 2.9651 | LR: 2.42e-04 | 1310.7ms/step | 15.3 samples/s +Step 22800/44000 | Loss: 2.9891 | LR: 2.42e-04 | 1309.6ms/step | 15.3 samples/s +Step 22810/44000 | Loss: 2.9712 | LR: 2.42e-04 | 1308.7ms/step | 15.3 samples/s +Step 22820/44000 | Loss: 2.9705 | LR: 2.42e-04 | 1309.5ms/step | 15.3 samples/s +Step 22830/44000 | Loss: 2.9785 | LR: 2.42e-04 | 1310.1ms/step | 15.3 samples/s +Step 22840/44000 | Loss: 2.9640 | LR: 2.42e-04 | 1309.9ms/step | 15.3 samples/s +Step 22850/44000 | Loss: 2.9375 | LR: 2.41e-04 | 1310.1ms/step | 15.3 samples/s +Step 22860/44000 | Loss: 2.9741 | LR: 2.41e-04 | 1310.1ms/step | 15.3 samples/s +Step 22870/44000 | Loss: 2.9541 | LR: 2.41e-04 | 1309.8ms/step | 15.3 samples/s +Step 22880/44000 | Loss: 2.9441 | LR: 2.41e-04 | 1310.0ms/step | 15.3 samples/s +Step 22890/44000 | Loss: 2.9986 | LR: 2.41e-04 | 1312.7ms/step | 15.2 samples/s +Step 22900/44000 | Loss: 2.9714 | LR: 2.40e-04 | 1311.6ms/step | 15.2 samples/s +Step 22910/44000 | Loss: 2.9681 | LR: 2.40e-04 | 1315.8ms/step | 15.2 samples/s +Step 22920/44000 | Loss: 2.9827 | LR: 2.40e-04 | 1314.5ms/step | 15.2 samples/s +Step 22930/44000 | Loss: 2.9691 | LR: 2.40e-04 | 1317.8ms/step | 15.2 samples/s +Step 22940/44000 | Loss: 2.9744 | LR: 2.40e-04 | 1315.1ms/step | 15.2 samples/s +Step 22950/44000 | Loss: 2.9985 | LR: 2.40e-04 | 1313.3ms/step | 15.2 samples/s +Step 22960/44000 | Loss: 2.9899 | LR: 2.39e-04 | 1316.5ms/step | 15.2 samples/s +Step 22970/44000 | Loss: 2.9730 | LR: 2.39e-04 | 1318.4ms/step | 15.2 samples/s +Step 22980/44000 | Loss: 2.9841 | LR: 2.39e-04 | 1313.6ms/step | 15.2 samples/s +Step 22990/44000 | Loss: 2.9801 | LR: 2.39e-04 | 1314.6ms/step | 15.2 samples/s +Step 23000/44000 | Loss: 2.9513 | LR: 2.39e-04 | 1315.3ms/step | 15.2 samples/s +Validation at step 23000: Loss = 2.9793, Perplexity = 19.67 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-23000 +Step 23010/44000 | Loss: 2.9400 | LR: 2.38e-04 | 3844.5ms/step | 5.2 samples/s +Step 23020/44000 | Loss: 2.9883 | LR: 2.38e-04 | 1317.7ms/step | 15.2 samples/s +Step 23030/44000 | Loss: 2.9525 | LR: 2.38e-04 | 1323.2ms/step | 15.1 samples/s +Step 23040/44000 | Loss: 2.9858 | LR: 2.38e-04 | 1321.2ms/step | 15.1 samples/s +Step 23050/44000 | Loss: 2.9812 | LR: 2.38e-04 | 1321.7ms/step | 15.1 samples/s +Step 23060/44000 | Loss: 3.0023 | LR: 2.38e-04 | 1320.7ms/step | 15.1 samples/s +Step 23070/44000 | Loss: 2.9446 | LR: 2.37e-04 | 1318.7ms/step | 15.2 samples/s +Step 23080/44000 | Loss: 2.9490 | LR: 2.37e-04 | 1320.4ms/step | 15.1 samples/s +Step 23090/44000 | Loss: 2.9871 | LR: 2.37e-04 | 1319.9ms/step | 15.2 samples/s +Step 23100/44000 | Loss: 2.9735 | LR: 2.37e-04 | 1323.3ms/step | 15.1 samples/s +Step 23110/44000 | Loss: 2.9879 | LR: 2.37e-04 | 1317.8ms/step | 15.2 samples/s +Step 23120/44000 | Loss: 2.9739 | LR: 2.36e-04 | 1318.7ms/step | 15.2 samples/s +Step 23130/44000 | Loss: 2.9612 | LR: 2.36e-04 | 1320.9ms/step | 15.1 samples/s +Step 23140/44000 | Loss: 2.9848 | LR: 2.36e-04 | 1322.3ms/step | 15.1 samples/s +Step 23150/44000 | Loss: 2.9513 | LR: 2.36e-04 | 1324.3ms/step | 15.1 samples/s +Step 23160/44000 | Loss: 2.9315 | LR: 2.36e-04 | 1320.5ms/step | 15.1 samples/s +Step 23170/44000 | Loss: 2.9630 | LR: 2.36e-04 | 1316.0ms/step | 15.2 samples/s +Step 23180/44000 | Loss: 2.9603 | LR: 2.35e-04 | 1319.7ms/step | 15.2 samples/s +Step 23190/44000 | Loss: 3.0060 | LR: 2.35e-04 | 1324.1ms/step | 15.1 samples/s +Step 23200/44000 | Loss: 2.9911 | LR: 2.35e-04 | 1320.0ms/step | 15.2 samples/s +Step 23210/44000 | Loss: 2.9909 | LR: 2.35e-04 | 1320.7ms/step | 15.1 samples/s +Step 23220/44000 | Loss: 2.9616 | LR: 2.35e-04 | 1321.0ms/step | 15.1 samples/s +Step 23230/44000 | Loss: 2.9692 | LR: 2.34e-04 | 1324.0ms/step | 15.1 samples/s +Step 23240/44000 | Loss: 2.9782 | LR: 2.34e-04 | 1321.4ms/step | 15.1 samples/s +Step 23250/44000 | Loss: 2.9638 | LR: 2.34e-04 | 1321.1ms/step | 15.1 samples/s +Validation at step 23250: Loss = 2.9770, Perplexity = 19.63 +Step 23260/44000 | Loss: 2.9689 | LR: 2.34e-04 | 3534.9ms/step | 5.7 samples/s +Step 23270/44000 | Loss: 2.9546 | LR: 2.34e-04 | 1317.3ms/step | 15.2 samples/s +Step 23280/44000 | Loss: 2.9651 | LR: 2.34e-04 | 1323.6ms/step | 15.1 samples/s +Step 23290/44000 | Loss: 3.0085 | LR: 2.33e-04 | 1320.9ms/step | 15.1 samples/s +Step 23300/44000 | Loss: 2.9617 | LR: 2.33e-04 | 1323.3ms/step | 15.1 samples/s +Step 23310/44000 | Loss: 2.9738 | LR: 2.33e-04 | 1322.4ms/step | 15.1 samples/s +Step 23320/44000 | Loss: 3.0037 | LR: 2.33e-04 | 1319.1ms/step | 15.2 samples/s +Step 23330/44000 | Loss: 2.9497 | LR: 2.33e-04 | 1325.5ms/step | 15.1 samples/s +Step 23340/44000 | Loss: 2.9674 | LR: 2.32e-04 | 1322.5ms/step | 15.1 samples/s +Step 23350/44000 | Loss: 2.9578 | LR: 2.32e-04 | 1322.4ms/step | 15.1 samples/s +Step 23360/44000 | Loss: 2.9597 | LR: 2.32e-04 | 1319.7ms/step | 15.2 samples/s +Step 23370/44000 | Loss: 2.9887 | LR: 2.32e-04 | 1318.1ms/step | 15.2 samples/s +Step 23380/44000 | Loss: 2.9748 | LR: 2.32e-04 | 1319.2ms/step | 15.2 samples/s +Step 23390/44000 | Loss: 2.9889 | LR: 2.32e-04 | 1320.1ms/step | 15.2 samples/s +Step 23400/44000 | Loss: 2.9872 | LR: 2.31e-04 | 1317.9ms/step | 15.2 samples/s +Step 23410/44000 | Loss: 2.9948 | LR: 2.31e-04 | 1319.5ms/step | 15.2 samples/s +Step 23420/44000 | Loss: 2.9654 | LR: 2.31e-04 | 1316.8ms/step | 15.2 samples/s +Step 23430/44000 | Loss: 2.9821 | LR: 2.31e-04 | 1321.4ms/step | 15.1 samples/s +Step 23440/44000 | Loss: 3.0094 | LR: 2.31e-04 | 1320.1ms/step | 15.1 samples/s +Step 23450/44000 | Loss: 2.9433 | LR: 2.30e-04 | 1321.5ms/step | 15.1 samples/s +Step 23460/44000 | Loss: 2.9791 | LR: 2.30e-04 | 1318.8ms/step | 15.2 samples/s +Step 23470/44000 | Loss: 2.9833 | LR: 2.30e-04 | 1323.6ms/step | 15.1 samples/s +Step 23480/44000 | Loss: 2.9915 | LR: 2.30e-04 | 1322.1ms/step | 15.1 samples/s +Step 23490/44000 | Loss: 3.0019 | LR: 2.30e-04 | 1321.2ms/step | 15.1 samples/s +Step 23500/44000 | Loss: 2.9610 | LR: 2.30e-04 | 1320.2ms/step | 15.1 samples/s +Validation at step 23500: Loss = 2.9749, Perplexity = 19.59 +Step 23510/44000 | Loss: 2.9809 | LR: 2.29e-04 | 3529.3ms/step | 5.7 samples/s +Step 23520/44000 | Loss: 2.9650 | LR: 2.29e-04 | 1322.0ms/step | 15.1 samples/s +Step 23530/44000 | Loss: 2.9650 | LR: 2.29e-04 | 1319.1ms/step | 15.2 samples/s +Step 23540/44000 | Loss: 2.9959 | LR: 2.29e-04 | 1322.0ms/step | 15.1 samples/s +Step 23550/44000 | Loss: 2.9599 | LR: 2.29e-04 | 1322.4ms/step | 15.1 samples/s +Step 23560/44000 | Loss: 2.9964 | LR: 2.29e-04 | 1319.5ms/step | 15.2 samples/s +Step 23570/44000 | Loss: 2.9452 | LR: 2.28e-04 | 1320.3ms/step | 15.1 samples/s +Step 23580/44000 | Loss: 2.9561 | LR: 2.28e-04 | 1321.2ms/step | 15.1 samples/s +Step 23590/44000 | Loss: 2.9552 | LR: 2.28e-04 | 1315.3ms/step | 15.2 samples/s +Step 23600/44000 | Loss: 2.9814 | LR: 2.28e-04 | 1322.6ms/step | 15.1 samples/s +Step 23610/44000 | Loss: 2.9722 | LR: 2.28e-04 | 1316.8ms/step | 15.2 samples/s +Step 23620/44000 | Loss: 2.9858 | LR: 2.27e-04 | 1319.7ms/step | 15.2 samples/s +Step 23630/44000 | Loss: 2.9931 | LR: 2.27e-04 | 1318.8ms/step | 15.2 samples/s +Step 23640/44000 | Loss: 2.9496 | LR: 2.27e-04 | 1319.9ms/step | 15.2 samples/s +Step 23650/44000 | Loss: 2.9159 | LR: 2.27e-04 | 1319.7ms/step | 15.2 samples/s +Step 23660/44000 | Loss: 2.9638 | LR: 2.27e-04 | 1315.9ms/step | 15.2 samples/s +Step 23670/44000 | Loss: 2.9397 | LR: 2.27e-04 | 1315.4ms/step | 15.2 samples/s +Step 23680/44000 | Loss: 2.9529 | LR: 2.26e-04 | 1318.9ms/step | 15.2 samples/s +Step 23690/44000 | Loss: 2.9317 | LR: 2.26e-04 | 1317.2ms/step | 15.2 samples/s +Step 23700/44000 | Loss: 2.9809 | LR: 2.26e-04 | 1318.6ms/step | 15.2 samples/s +Step 23710/44000 | Loss: 2.9600 | LR: 2.26e-04 | 1323.2ms/step | 15.1 samples/s +Step 23720/44000 | Loss: 2.9809 | LR: 2.26e-04 | 1317.2ms/step | 15.2 samples/s +Step 23730/44000 | Loss: 2.9199 | LR: 2.25e-04 | 1324.1ms/step | 15.1 samples/s +Step 23740/44000 | Loss: 2.9722 | LR: 2.25e-04 | 1322.1ms/step | 15.1 samples/s +Step 23750/44000 | Loss: 2.9305 | LR: 2.25e-04 | 1317.7ms/step | 15.2 samples/s +Validation at step 23750: Loss = 2.9728, Perplexity = 19.55 +Step 23760/44000 | Loss: 2.9677 | LR: 2.25e-04 | 3528.4ms/step | 5.7 samples/s +Step 23770/44000 | Loss: 2.9626 | LR: 2.25e-04 | 1319.4ms/step | 15.2 samples/s +Step 23780/44000 | Loss: 2.9852 | LR: 2.25e-04 | 1321.6ms/step | 15.1 samples/s +Step 23790/44000 | Loss: 2.9907 | LR: 2.24e-04 | 1324.7ms/step | 15.1 samples/s +Step 23800/44000 | Loss: 2.9723 | LR: 2.24e-04 | 1324.4ms/step | 15.1 samples/s +Step 23810/44000 | Loss: 2.9435 | LR: 2.24e-04 | 1321.5ms/step | 15.1 samples/s +Step 23820/44000 | Loss: 2.9578 | LR: 2.24e-04 | 1318.0ms/step | 15.2 samples/s +Step 23830/44000 | Loss: 2.9912 | LR: 2.24e-04 | 1323.3ms/step | 15.1 samples/s +Step 23840/44000 | Loss: 2.9818 | LR: 2.23e-04 | 1320.4ms/step | 15.1 samples/s +Step 23850/44000 | Loss: 2.9461 | LR: 2.23e-04 | 1322.7ms/step | 15.1 samples/s +Step 23860/44000 | Loss: 2.9601 | LR: 2.23e-04 | 1321.6ms/step | 15.1 samples/s +Step 23870/44000 | Loss: 2.9636 | LR: 2.23e-04 | 1320.4ms/step | 15.1 samples/s +Step 23880/44000 | Loss: 3.0031 | LR: 2.23e-04 | 1317.9ms/step | 15.2 samples/s +Step 23890/44000 | Loss: 2.9595 | LR: 2.23e-04 | 1320.1ms/step | 15.2 samples/s +Step 23900/44000 | Loss: 2.9479 | LR: 2.22e-04 | 1321.4ms/step | 15.1 samples/s +Step 23910/44000 | Loss: 2.9614 | LR: 2.22e-04 | 1321.5ms/step | 15.1 samples/s +Step 23920/44000 | Loss: 2.9627 | LR: 2.22e-04 | 1322.5ms/step | 15.1 samples/s +Step 23930/44000 | Loss: 2.9543 | LR: 2.22e-04 | 1319.8ms/step | 15.2 samples/s +Step 23940/44000 | Loss: 2.9540 | LR: 2.22e-04 | 1320.8ms/step | 15.1 samples/s +Step 23950/44000 | Loss: 2.9658 | LR: 2.21e-04 | 1319.1ms/step | 15.2 samples/s +Step 23960/44000 | Loss: 2.9823 | LR: 2.21e-04 | 1316.6ms/step | 15.2 samples/s +Step 23970/44000 | Loss: 2.9795 | LR: 2.21e-04 | 1319.9ms/step | 15.2 samples/s +Step 23980/44000 | Loss: 2.9791 | LR: 2.21e-04 | 1322.3ms/step | 15.1 samples/s +Step 23990/44000 | Loss: 2.9559 | LR: 2.21e-04 | 1323.1ms/step | 15.1 samples/s +Step 24000/44000 | Loss: 2.9583 | LR: 2.21e-04 | 1319.1ms/step | 15.2 samples/s +Validation at step 24000: Loss = 2.9709, Perplexity = 19.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-24000 +Step 24010/44000 | Loss: 2.9700 | LR: 2.20e-04 | 3841.9ms/step | 5.2 samples/s +Step 24020/44000 | Loss: 2.9394 | LR: 2.20e-04 | 1317.5ms/step | 15.2 samples/s +Step 24030/44000 | Loss: 2.9995 | LR: 2.20e-04 | 1319.2ms/step | 15.2 samples/s +Step 24040/44000 | Loss: 2.9390 | LR: 2.20e-04 | 1319.3ms/step | 15.2 samples/s +Step 24050/44000 | Loss: 2.9418 | LR: 2.20e-04 | 1319.0ms/step | 15.2 samples/s +Step 24060/44000 | Loss: 3.0195 | LR: 2.19e-04 | 1320.3ms/step | 15.1 samples/s +Step 24070/44000 | Loss: 2.9726 | LR: 2.19e-04 | 1322.0ms/step | 15.1 samples/s +Step 24080/44000 | Loss: 2.9608 | LR: 2.19e-04 | 1320.8ms/step | 15.1 samples/s +Step 24090/44000 | Loss: 3.0028 | LR: 2.19e-04 | 1323.5ms/step | 15.1 samples/s +Step 24100/44000 | Loss: 2.9675 | LR: 2.19e-04 | 1318.9ms/step | 15.2 samples/s +Step 24110/44000 | Loss: 2.9863 | LR: 2.19e-04 | 1323.6ms/step | 15.1 samples/s +Step 24120/44000 | Loss: 2.9745 | LR: 2.18e-04 | 1321.8ms/step | 15.1 samples/s +Step 24130/44000 | Loss: 2.9943 | LR: 2.18e-04 | 1322.0ms/step | 15.1 samples/s +Step 24140/44000 | Loss: 2.9648 | LR: 2.18e-04 | 1318.1ms/step | 15.2 samples/s +Step 24150/44000 | Loss: 2.9653 | LR: 2.18e-04 | 1321.4ms/step | 15.1 samples/s +Step 24160/44000 | Loss: 2.9660 | LR: 2.18e-04 | 1322.7ms/step | 15.1 samples/s +Step 24170/44000 | Loss: 2.9743 | LR: 2.17e-04 | 1322.7ms/step | 15.1 samples/s +Step 24180/44000 | Loss: 2.9715 | LR: 2.17e-04 | 1319.2ms/step | 15.2 samples/s +Step 24190/44000 | Loss: 2.9858 | LR: 2.17e-04 | 1322.1ms/step | 15.1 samples/s +Step 24200/44000 | Loss: 2.9597 | LR: 2.17e-04 | 1324.0ms/step | 15.1 samples/s +Step 24210/44000 | Loss: 2.9584 | LR: 2.17e-04 | 1323.5ms/step | 15.1 samples/s +Step 24220/44000 | Loss: 2.9687 | LR: 2.17e-04 | 1322.1ms/step | 15.1 samples/s +Step 24230/44000 | Loss: 2.9827 | LR: 2.16e-04 | 1325.4ms/step | 15.1 samples/s +Step 24240/44000 | Loss: 2.9852 | LR: 2.16e-04 | 1322.3ms/step | 15.1 samples/s +Step 24250/44000 | Loss: 2.9892 | LR: 2.16e-04 | 1323.7ms/step | 15.1 samples/s +Validation at step 24250: Loss = 2.9686, Perplexity = 19.46 +Step 24260/44000 | Loss: 2.9582 | LR: 2.16e-04 | 3536.4ms/step | 5.7 samples/s +Step 24270/44000 | Loss: 2.9654 | LR: 2.16e-04 | 1319.1ms/step | 15.2 samples/s +Step 24280/44000 | Loss: 2.9606 | LR: 2.16e-04 | 1320.0ms/step | 15.2 samples/s +Step 24290/44000 | Loss: 2.9676 | LR: 2.15e-04 | 1321.0ms/step | 15.1 samples/s +Step 24300/44000 | Loss: 2.9763 | LR: 2.15e-04 | 1317.4ms/step | 15.2 samples/s +Step 24310/44000 | Loss: 2.9760 | LR: 2.15e-04 | 1317.1ms/step | 15.2 samples/s +Step 24320/44000 | Loss: 2.9603 | LR: 2.15e-04 | 1318.7ms/step | 15.2 samples/s +Step 24330/44000 | Loss: 2.9542 | LR: 2.15e-04 | 1320.9ms/step | 15.1 samples/s +Step 24340/44000 | Loss: 2.9374 | LR: 2.14e-04 | 1321.6ms/step | 15.1 samples/s +Step 24350/44000 | Loss: 2.9894 | LR: 2.14e-04 | 1320.3ms/step | 15.1 samples/s +Step 24360/44000 | Loss: 2.9718 | LR: 2.14e-04 | 1323.6ms/step | 15.1 samples/s +Step 24370/44000 | Loss: 2.9786 | LR: 2.14e-04 | 1322.0ms/step | 15.1 samples/s +Step 24380/44000 | Loss: 2.9537 | LR: 2.14e-04 | 1318.9ms/step | 15.2 samples/s +Step 24390/44000 | Loss: 2.9231 | LR: 2.14e-04 | 1320.4ms/step | 15.1 samples/s +Step 24400/44000 | Loss: 2.9713 | LR: 2.13e-04 | 1322.0ms/step | 15.1 samples/s +Step 24410/44000 | Loss: 2.9549 | LR: 2.13e-04 | 1320.1ms/step | 15.2 samples/s +Step 24420/44000 | Loss: 2.9524 | LR: 2.13e-04 | 1321.2ms/step | 15.1 samples/s +Step 24430/44000 | Loss: 2.9352 | LR: 2.13e-04 | 1319.4ms/step | 15.2 samples/s +Step 24440/44000 | Loss: 2.9373 | LR: 2.13e-04 | 1319.4ms/step | 15.2 samples/s +Step 24450/44000 | Loss: 2.9691 | LR: 2.12e-04 | 1320.5ms/step | 15.1 samples/s +Step 24460/44000 | Loss: 2.9443 | LR: 2.12e-04 | 1324.5ms/step | 15.1 samples/s +Step 24470/44000 | Loss: 2.9431 | LR: 2.12e-04 | 1320.2ms/step | 15.1 samples/s +Step 24480/44000 | Loss: 2.9821 | LR: 2.12e-04 | 1319.4ms/step | 15.2 samples/s +Step 24490/44000 | Loss: 2.9600 | LR: 2.12e-04 | 1320.6ms/step | 15.1 samples/s +Step 24500/44000 | Loss: 2.9612 | LR: 2.12e-04 | 1317.7ms/step | 15.2 samples/s +Validation at step 24500: Loss = 2.9672, Perplexity = 19.44 +Step 24510/44000 | Loss: 2.9934 | LR: 2.11e-04 | 3537.5ms/step | 5.7 samples/s +Step 24520/44000 | Loss: 2.9837 | LR: 2.11e-04 | 1323.3ms/step | 15.1 samples/s +Step 24530/44000 | Loss: 2.9583 | LR: 2.11e-04 | 1317.8ms/step | 15.2 samples/s +Step 24540/44000 | Loss: 2.9397 | LR: 2.11e-04 | 1318.9ms/step | 15.2 samples/s +Step 24550/44000 | Loss: 2.9748 | LR: 2.11e-04 | 1318.7ms/step | 15.2 samples/s +Step 24560/44000 | Loss: 2.9675 | LR: 2.10e-04 | 1316.5ms/step | 15.2 samples/s +Step 24570/44000 | Loss: 2.9520 | LR: 2.10e-04 | 1317.4ms/step | 15.2 samples/s +Step 24580/44000 | Loss: 2.9546 | LR: 2.10e-04 | 1321.0ms/step | 15.1 samples/s +Step 24590/44000 | Loss: 2.9672 | LR: 2.10e-04 | 1319.5ms/step | 15.2 samples/s +Step 24600/44000 | Loss: 2.9573 | LR: 2.10e-04 | 1315.2ms/step | 15.2 samples/s +Step 24610/44000 | Loss: 2.9720 | LR: 2.10e-04 | 1319.9ms/step | 15.2 samples/s +Step 24620/44000 | Loss: 2.9555 | LR: 2.09e-04 | 1318.7ms/step | 15.2 samples/s +Step 24630/44000 | Loss: 2.9229 | LR: 2.09e-04 | 1319.7ms/step | 15.2 samples/s +Step 24640/44000 | Loss: 2.9220 | LR: 2.09e-04 | 1318.8ms/step | 15.2 samples/s +Step 24650/44000 | Loss: 2.9347 | LR: 2.09e-04 | 1322.6ms/step | 15.1 samples/s +Step 24660/44000 | Loss: 2.9949 | LR: 2.09e-04 | 1321.0ms/step | 15.1 samples/s +Step 24670/44000 | Loss: 2.9421 | LR: 2.09e-04 | 1324.4ms/step | 15.1 samples/s +Step 24680/44000 | Loss: 2.9653 | LR: 2.08e-04 | 1318.4ms/step | 15.2 samples/s +Step 24690/44000 | Loss: 2.9451 | LR: 2.08e-04 | 1321.1ms/step | 15.1 samples/s +Step 24700/44000 | Loss: 2.9754 | LR: 2.08e-04 | 1325.1ms/step | 15.1 samples/s +Step 24710/44000 | Loss: 2.9721 | LR: 2.08e-04 | 1318.9ms/step | 15.2 samples/s +Step 24720/44000 | Loss: 2.9636 | LR: 2.08e-04 | 1319.0ms/step | 15.2 samples/s +Step 24730/44000 | Loss: 2.9755 | LR: 2.07e-04 | 1321.4ms/step | 15.1 samples/s +Step 24740/44000 | Loss: 2.9270 | LR: 2.07e-04 | 1320.6ms/step | 15.1 samples/s +Step 24750/44000 | Loss: 2.9329 | LR: 2.07e-04 | 1319.8ms/step | 15.2 samples/s +Validation at step 24750: Loss = 2.9651, Perplexity = 19.40 +Step 24760/44000 | Loss: 2.9528 | LR: 2.07e-04 | 3529.1ms/step | 5.7 samples/s +Step 24770/44000 | Loss: 2.9846 | LR: 2.07e-04 | 1317.6ms/step | 15.2 samples/s +Step 24780/44000 | Loss: 3.0016 | LR: 2.07e-04 | 1315.5ms/step | 15.2 samples/s +Step 24790/44000 | Loss: 2.9575 | LR: 2.06e-04 | 1317.9ms/step | 15.2 samples/s +Step 24800/44000 | Loss: 2.9474 | LR: 2.06e-04 | 1316.4ms/step | 15.2 samples/s +Step 24810/44000 | Loss: 2.9417 | LR: 2.06e-04 | 1317.4ms/step | 15.2 samples/s +Step 24820/44000 | Loss: 2.9781 | LR: 2.06e-04 | 1318.8ms/step | 15.2 samples/s +Step 24830/44000 | Loss: 2.9834 | LR: 2.06e-04 | 1319.3ms/step | 15.2 samples/s +Step 24840/44000 | Loss: 2.9374 | LR: 2.05e-04 | 1316.9ms/step | 15.2 samples/s +Step 24850/44000 | Loss: 3.0027 | LR: 2.05e-04 | 1318.0ms/step | 15.2 samples/s +Step 24860/44000 | Loss: 2.9507 | LR: 2.05e-04 | 1319.8ms/step | 15.2 samples/s +Step 24870/44000 | Loss: 2.9388 | LR: 2.05e-04 | 1319.0ms/step | 15.2 samples/s +Step 24880/44000 | Loss: 2.9857 | LR: 2.05e-04 | 1318.7ms/step | 15.2 samples/s +Step 24890/44000 | Loss: 2.9620 | LR: 2.05e-04 | 1319.7ms/step | 15.2 samples/s +Step 24900/44000 | Loss: 2.9625 | LR: 2.04e-04 | 1321.3ms/step | 15.1 samples/s +Step 24910/44000 | Loss: 2.9622 | LR: 2.04e-04 | 1318.4ms/step | 15.2 samples/s +Step 24920/44000 | Loss: 2.9900 | LR: 2.04e-04 | 1321.6ms/step | 15.1 samples/s +Step 24930/44000 | Loss: 2.9448 | LR: 2.04e-04 | 1322.1ms/step | 15.1 samples/s +Step 24940/44000 | Loss: 2.9450 | LR: 2.04e-04 | 1321.4ms/step | 15.1 samples/s +Step 24950/44000 | Loss: 2.9646 | LR: 2.04e-04 | 1323.0ms/step | 15.1 samples/s +Step 24960/44000 | Loss: 2.9458 | LR: 2.03e-04 | 1324.0ms/step | 15.1 samples/s +Step 24970/44000 | Loss: 2.9737 | LR: 2.03e-04 | 1320.7ms/step | 15.1 samples/s +Step 24980/44000 | Loss: 2.9577 | LR: 2.03e-04 | 1321.7ms/step | 15.1 samples/s +Step 24990/44000 | Loss: 2.9611 | LR: 2.03e-04 | 1319.3ms/step | 15.2 samples/s +Step 25000/44000 | Loss: 2.9612 | LR: 2.03e-04 | 1322.0ms/step | 15.1 samples/s +Validation at step 25000: Loss = 2.9630, Perplexity = 19.36 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-25000 +Step 25010/44000 | Loss: 2.9470 | LR: 2.02e-04 | 3852.7ms/step | 5.2 samples/s +Step 25020/44000 | Loss: 2.9582 | LR: 2.02e-04 | 1319.9ms/step | 15.2 samples/s +Step 25030/44000 | Loss: 2.9468 | LR: 2.02e-04 | 1323.0ms/step | 15.1 samples/s +Step 25040/44000 | Loss: 2.9431 | LR: 2.02e-04 | 1321.3ms/step | 15.1 samples/s +Step 25050/44000 | Loss: 2.9555 | LR: 2.02e-04 | 1320.4ms/step | 15.1 samples/s +Step 25060/44000 | Loss: 2.9452 | LR: 2.02e-04 | 1317.2ms/step | 15.2 samples/s +Step 25070/44000 | Loss: 2.9545 | LR: 2.01e-04 | 1318.3ms/step | 15.2 samples/s +Step 25080/44000 | Loss: 2.9675 | LR: 2.01e-04 | 1319.8ms/step | 15.2 samples/s +Step 25090/44000 | Loss: 2.9936 | LR: 2.01e-04 | 1320.7ms/step | 15.1 samples/s +Step 25100/44000 | Loss: 2.9559 | LR: 2.01e-04 | 1316.8ms/step | 15.2 samples/s +Step 25110/44000 | Loss: 2.9650 | LR: 2.01e-04 | 1317.3ms/step | 15.2 samples/s +Step 25120/44000 | Loss: 2.9869 | LR: 2.00e-04 | 1317.0ms/step | 15.2 samples/s +Step 25130/44000 | Loss: 2.9660 | LR: 2.00e-04 | 1319.8ms/step | 15.2 samples/s +Step 25140/44000 | Loss: 2.9618 | LR: 2.00e-04 | 1312.8ms/step | 15.2 samples/s +Step 25150/44000 | Loss: 2.9666 | LR: 2.00e-04 | 1320.0ms/step | 15.2 samples/s +Step 25160/44000 | Loss: 2.9867 | LR: 2.00e-04 | 1314.8ms/step | 15.2 samples/s +Step 25170/44000 | Loss: 2.9275 | LR: 2.00e-04 | 1318.7ms/step | 15.2 samples/s +Step 25180/44000 | Loss: 2.9505 | LR: 1.99e-04 | 1314.3ms/step | 15.2 samples/s +Step 25190/44000 | Loss: 2.9649 | LR: 1.99e-04 | 1316.6ms/step | 15.2 samples/s +Step 25200/44000 | Loss: 2.9616 | LR: 1.99e-04 | 1322.3ms/step | 15.1 samples/s +Step 25210/44000 | Loss: 2.9781 | LR: 1.99e-04 | 1317.2ms/step | 15.2 samples/s +Step 25220/44000 | Loss: 2.9490 | LR: 1.99e-04 | 1319.7ms/step | 15.2 samples/s +Step 25230/44000 | Loss: 2.9636 | LR: 1.99e-04 | 1318.5ms/step | 15.2 samples/s +Step 25240/44000 | Loss: 2.9476 | LR: 1.98e-04 | 1319.4ms/step | 15.2 samples/s +Step 25250/44000 | Loss: 2.9224 | LR: 1.98e-04 | 1318.9ms/step | 15.2 samples/s +Validation at step 25250: Loss = 2.9615, Perplexity = 19.33 +Step 25260/44000 | Loss: 2.9510 | LR: 1.98e-04 | 3517.4ms/step | 5.7 samples/s +Step 25270/44000 | Loss: 2.9915 | LR: 1.98e-04 | 1311.7ms/step | 15.2 samples/s +Step 25280/44000 | Loss: 2.9509 | LR: 1.98e-04 | 1313.4ms/step | 15.2 samples/s +Step 25290/44000 | Loss: 2.9746 | LR: 1.97e-04 | 1316.7ms/step | 15.2 samples/s +Step 25300/44000 | Loss: 2.9692 | LR: 1.97e-04 | 1316.9ms/step | 15.2 samples/s +Step 25310/44000 | Loss: 2.9758 | LR: 1.97e-04 | 1316.3ms/step | 15.2 samples/s +Step 25320/44000 | Loss: 2.9507 | LR: 1.97e-04 | 1310.6ms/step | 15.3 samples/s +Step 25330/44000 | Loss: 2.9105 | LR: 1.97e-04 | 1312.4ms/step | 15.2 samples/s +Step 25340/44000 | Loss: 2.9663 | LR: 1.97e-04 | 1314.8ms/step | 15.2 samples/s +Step 25350/44000 | Loss: 2.9289 | LR: 1.96e-04 | 1316.6ms/step | 15.2 samples/s +Step 25360/44000 | Loss: 2.9757 | LR: 1.96e-04 | 1313.1ms/step | 15.2 samples/s +Step 25370/44000 | Loss: 2.9677 | LR: 1.96e-04 | 1315.6ms/step | 15.2 samples/s +Step 25380/44000 | Loss: 2.9537 | LR: 1.96e-04 | 1314.5ms/step | 15.2 samples/s +Step 25390/44000 | Loss: 2.9466 | LR: 1.96e-04 | 1313.7ms/step | 15.2 samples/s +Step 25400/44000 | Loss: 2.9585 | LR: 1.96e-04 | 1311.3ms/step | 15.3 samples/s +Step 25410/44000 | Loss: 2.9753 | LR: 1.95e-04 | 1315.4ms/step | 15.2 samples/s +Step 25420/44000 | Loss: 2.9890 | LR: 1.95e-04 | 1313.5ms/step | 15.2 samples/s +Step 25430/44000 | Loss: 2.9450 | LR: 1.95e-04 | 1315.8ms/step | 15.2 samples/s +Step 25440/44000 | Loss: 2.9529 | LR: 1.95e-04 | 1316.3ms/step | 15.2 samples/s +Step 25450/44000 | Loss: 2.9514 | LR: 1.95e-04 | 1315.4ms/step | 15.2 samples/s +Step 25460/44000 | Loss: 2.9665 | LR: 1.94e-04 | 1312.3ms/step | 15.2 samples/s +Step 25470/44000 | Loss: 2.9535 | LR: 1.94e-04 | 1313.6ms/step | 15.2 samples/s +Step 25480/44000 | Loss: 2.9494 | LR: 1.94e-04 | 1314.9ms/step | 15.2 samples/s +Step 25490/44000 | Loss: 2.9473 | LR: 1.94e-04 | 1317.9ms/step | 15.2 samples/s +Step 25500/44000 | Loss: 2.9637 | LR: 1.94e-04 | 1312.7ms/step | 15.2 samples/s +Validation at step 25500: Loss = 2.9595, Perplexity = 19.29 +Step 25510/44000 | Loss: 2.9709 | LR: 1.94e-04 | 3510.5ms/step | 5.7 samples/s +Step 25520/44000 | Loss: 2.9472 | LR: 1.93e-04 | 1313.8ms/step | 15.2 samples/s +Step 25530/44000 | Loss: 2.9845 | LR: 1.93e-04 | 1318.1ms/step | 15.2 samples/s +Step 25540/44000 | Loss: 2.9362 | LR: 1.93e-04 | 1314.5ms/step | 15.2 samples/s +Step 25550/44000 | Loss: 2.9566 | LR: 1.93e-04 | 1316.9ms/step | 15.2 samples/s +Step 25560/44000 | Loss: 2.9231 | LR: 1.93e-04 | 1312.3ms/step | 15.2 samples/s +Step 25570/44000 | Loss: 2.9595 | LR: 1.92e-04 | 1312.5ms/step | 15.2 samples/s +Step 25580/44000 | Loss: 2.9669 | LR: 1.92e-04 | 1313.3ms/step | 15.2 samples/s +Step 25590/44000 | Loss: 2.9446 | LR: 1.92e-04 | 1318.2ms/step | 15.2 samples/s +Step 25600/44000 | Loss: 2.9613 | LR: 1.92e-04 | 1315.1ms/step | 15.2 samples/s +Step 25610/44000 | Loss: 2.9629 | LR: 1.92e-04 | 1317.8ms/step | 15.2 samples/s +Step 25620/44000 | Loss: 2.9660 | LR: 1.92e-04 | 1314.7ms/step | 15.2 samples/s +Step 25630/44000 | Loss: 2.9613 | LR: 1.91e-04 | 1314.3ms/step | 15.2 samples/s +Step 25640/44000 | Loss: 2.9470 | LR: 1.91e-04 | 1319.1ms/step | 15.2 samples/s +Step 25650/44000 | Loss: 2.9295 | LR: 1.91e-04 | 1315.3ms/step | 15.2 samples/s +Step 25660/44000 | Loss: 2.9811 | LR: 1.91e-04 | 1315.6ms/step | 15.2 samples/s +Step 25670/44000 | Loss: 2.9650 | LR: 1.91e-04 | 1316.6ms/step | 15.2 samples/s +Step 25680/44000 | Loss: 2.9449 | LR: 1.91e-04 | 1313.2ms/step | 15.2 samples/s +Step 25690/44000 | Loss: 2.9556 | LR: 1.90e-04 | 1315.1ms/step | 15.2 samples/s +Step 25700/44000 | Loss: 2.9594 | LR: 1.90e-04 | 1313.9ms/step | 15.2 samples/s +Step 25710/44000 | Loss: 2.9285 | LR: 1.90e-04 | 1312.3ms/step | 15.2 samples/s +Step 25720/44000 | Loss: 2.9424 | LR: 1.90e-04 | 1314.4ms/step | 15.2 samples/s +Step 25730/44000 | Loss: 2.9476 | LR: 1.90e-04 | 1318.3ms/step | 15.2 samples/s +Step 25740/44000 | Loss: 2.9409 | LR: 1.89e-04 | 1315.9ms/step | 15.2 samples/s +Step 25750/44000 | Loss: 2.9557 | LR: 1.89e-04 | 1314.1ms/step | 15.2 samples/s +Validation at step 25750: Loss = 2.9583, Perplexity = 19.26 +Step 25760/44000 | Loss: 2.9381 | LR: 1.89e-04 | 3518.5ms/step | 5.7 samples/s +Step 25770/44000 | Loss: 2.9575 | LR: 1.89e-04 | 1316.3ms/step | 15.2 samples/s +Step 25780/44000 | Loss: 2.9708 | LR: 1.89e-04 | 1313.3ms/step | 15.2 samples/s +Step 25790/44000 | Loss: 2.9856 | LR: 1.89e-04 | 1313.3ms/step | 15.2 samples/s +Step 25800/44000 | Loss: 2.9669 | LR: 1.88e-04 | 1312.7ms/step | 15.2 samples/s +Step 25810/44000 | Loss: 2.9287 | LR: 1.88e-04 | 1313.7ms/step | 15.2 samples/s +Step 25820/44000 | Loss: 2.9609 | LR: 1.88e-04 | 1311.5ms/step | 15.2 samples/s +Step 25830/44000 | Loss: 2.9546 | LR: 1.88e-04 | 1315.4ms/step | 15.2 samples/s +Step 25840/44000 | Loss: 2.9611 | LR: 1.88e-04 | 1310.5ms/step | 15.3 samples/s +Step 25850/44000 | Loss: 2.9835 | LR: 1.88e-04 | 1314.1ms/step | 15.2 samples/s +Step 25860/44000 | Loss: 2.9322 | LR: 1.87e-04 | 1314.8ms/step | 15.2 samples/s +Step 25870/44000 | Loss: 2.9626 | LR: 1.87e-04 | 1313.9ms/step | 15.2 samples/s +Step 25880/44000 | Loss: 2.9681 | LR: 1.87e-04 | 1314.6ms/step | 15.2 samples/s +Step 25890/44000 | Loss: 2.9320 | LR: 1.87e-04 | 1314.0ms/step | 15.2 samples/s +Step 25900/44000 | Loss: 2.9186 | LR: 1.87e-04 | 1314.8ms/step | 15.2 samples/s +Step 25910/44000 | Loss: 2.9611 | LR: 1.87e-04 | 1314.2ms/step | 15.2 samples/s +Step 25920/44000 | Loss: 2.9554 | LR: 1.86e-04 | 1312.7ms/step | 15.2 samples/s +Step 25930/44000 | Loss: 2.9330 | LR: 1.86e-04 | 1314.8ms/step | 15.2 samples/s +Step 25940/44000 | Loss: 2.9313 | LR: 1.86e-04 | 1311.8ms/step | 15.2 samples/s +Step 25950/44000 | Loss: 2.9589 | LR: 1.86e-04 | 1314.9ms/step | 15.2 samples/s +Step 25960/44000 | Loss: 2.9499 | LR: 1.86e-04 | 1314.6ms/step | 15.2 samples/s +Step 25970/44000 | Loss: 2.9548 | LR: 1.85e-04 | 1313.7ms/step | 15.2 samples/s +Step 25980/44000 | Loss: 2.9510 | LR: 1.85e-04 | 1313.1ms/step | 15.2 samples/s +Step 25990/44000 | Loss: 2.9341 | LR: 1.85e-04 | 1313.8ms/step | 15.2 samples/s +Step 26000/44000 | Loss: 2.9495 | LR: 1.85e-04 | 1315.6ms/step | 15.2 samples/s +Validation at step 26000: Loss = 2.9564, Perplexity = 19.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-26000 +Step 26010/44000 | Loss: 2.9052 | LR: 1.85e-04 | 3833.1ms/step | 5.2 samples/s +Step 26020/44000 | Loss: 2.9709 | LR: 1.85e-04 | 1320.5ms/step | 15.1 samples/s +Step 26030/44000 | Loss: 2.9326 | LR: 1.84e-04 | 1320.7ms/step | 15.1 samples/s +Step 26040/44000 | Loss: 2.9465 | LR: 1.84e-04 | 1321.6ms/step | 15.1 samples/s +Step 26050/44000 | Loss: 2.9550 | LR: 1.84e-04 | 1318.6ms/step | 15.2 samples/s +Step 26060/44000 | Loss: 2.9575 | LR: 1.84e-04 | 1321.9ms/step | 15.1 samples/s +Step 26070/44000 | Loss: 2.9139 | LR: 1.84e-04 | 1320.1ms/step | 15.2 samples/s +Step 26080/44000 | Loss: 2.9313 | LR: 1.84e-04 | 1319.5ms/step | 15.2 samples/s +Step 26090/44000 | Loss: 2.9619 | LR: 1.83e-04 | 1321.3ms/step | 15.1 samples/s +Step 26100/44000 | Loss: 2.9343 | LR: 1.83e-04 | 1321.4ms/step | 15.1 samples/s +Step 26110/44000 | Loss: 2.9903 | LR: 1.83e-04 | 1317.6ms/step | 15.2 samples/s +Step 26120/44000 | Loss: 2.9316 | LR: 1.83e-04 | 1320.8ms/step | 15.1 samples/s +Step 26130/44000 | Loss: 2.9473 | LR: 1.83e-04 | 1320.5ms/step | 15.1 samples/s +Step 26140/44000 | Loss: 2.9548 | LR: 1.82e-04 | 1321.5ms/step | 15.1 samples/s +Step 26150/44000 | Loss: 2.8960 | LR: 1.82e-04 | 1322.0ms/step | 15.1 samples/s +Step 26160/44000 | Loss: 2.9612 | LR: 1.82e-04 | 1321.5ms/step | 15.1 samples/s +Step 26170/44000 | Loss: 2.9347 | LR: 1.82e-04 | 1319.4ms/step | 15.2 samples/s +Step 26180/44000 | Loss: 2.9399 | LR: 1.82e-04 | 1321.6ms/step | 15.1 samples/s +Step 26190/44000 | Loss: 2.9452 | LR: 1.82e-04 | 1320.5ms/step | 15.1 samples/s +Step 26200/44000 | Loss: 2.9613 | LR: 1.81e-04 | 1322.3ms/step | 15.1 samples/s +Step 26210/44000 | Loss: 2.9535 | LR: 1.81e-04 | 1319.5ms/step | 15.2 samples/s +Step 26220/44000 | Loss: 2.9577 | LR: 1.81e-04 | 1320.6ms/step | 15.1 samples/s +Step 26230/44000 | Loss: 2.9226 | LR: 1.81e-04 | 1322.4ms/step | 15.1 samples/s +Step 26240/44000 | Loss: 2.9373 | LR: 1.81e-04 | 1321.0ms/step | 15.1 samples/s +Step 26250/44000 | Loss: 2.9208 | LR: 1.81e-04 | 1320.2ms/step | 15.1 samples/s +Validation at step 26250: Loss = 2.9551, Perplexity = 19.20 +Step 26260/44000 | Loss: 2.9552 | LR: 1.80e-04 | 3527.9ms/step | 5.7 samples/s diff --git a/epfl_final/train_logs/p1_controls_nrb35_seed44_47/supervisor.log b/epfl_final/train_logs/p1_controls_nrb35_seed44_47/supervisor.log new file mode 100644 index 0000000000000000000000000000000000000000..46b3d6d9340faa5dce3612a8a1ce2cb9a0e411be --- /dev/null +++ b/epfl_final/train_logs/p1_controls_nrb35_seed44_47/supervisor.log @@ -0,0 +1,3 @@ +[supervisor] launching seed=44 static on gpu0 2026-08-21T18:08:18Z +[supervisor] launching seed=44 resampled on gpu1 2026-08-21T18:08:18Z +[supervisor] waiting for seed=44 pids=657,659 diff --git a/epfl_final/train_logs/p1_ddp5_static_seed45_47/l8k365m_static_nrb35_seed45_a100-40gb-train-5-gpu.log b/epfl_final/train_logs/p1_ddp5_static_seed45_47/l8k365m_static_nrb35_seed45_a100-40gb-train-5-gpu.log new file mode 100644 index 0000000000000000000000000000000000000000..8157b329de1eec0e6a66f6fe4fa6e0a3c4e77701 --- /dev/null +++ b/epfl_final/train_logs/p1_ddp5_static_seed45_47/l8k365m_static_nrb35_seed45_a100-40gb-train-5-gpu.log @@ -0,0 +1,4857 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 2, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 45, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 DDP5 365m static 8k nrb35 seed45", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "static", + "nrb35", + "seed45", + "ddp5", + "a100" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1", + "sparse_mask_seed": 1, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: FalseCheckpoint activations: False + +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) + +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 5 +Gradient accumulation steps: 2 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 302776 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_043614-pq6ov2y3 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 DDP5 365m static 8k nrb35 seed45 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/pq6ov2y3 +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0713 | LR: 6.67e-06 | 12244.6ms/step | 1.6 samples/s +Step 20/44000 | Loss: 11.9775 | LR: 1.33e-05 | 1192.9ms/step | 16.8 samples/s +Step 30/44000 | Loss: 11.6587 | LR: 2.00e-05 | 1275.4ms/step | 15.7 samples/s +Step 40/44000 | Loss: 11.1410 | LR: 2.67e-05 | 1215.4ms/step | 16.5 samples/s +Step 50/44000 | Loss: 10.8469 | LR: 3.33e-05 | 1299.2ms/step | 15.4 samples/s +Step 60/44000 | Loss: 10.5832 | LR: 4.00e-05 | 1209.7ms/step | 16.5 samples/s +Step 70/44000 | Loss: 10.3438 | LR: 4.67e-05 | 1300.9ms/step | 15.4 samples/s +Step 80/44000 | Loss: 10.0389 | LR: 5.33e-05 | 1257.1ms/step | 15.9 samples/s +Step 90/44000 | Loss: 9.6799 | LR: 6.00e-05 | 1258.4ms/step | 15.9 samples/s +Step 100/44000 | Loss: 9.1672 | LR: 6.67e-05 | 1211.0ms/step | 16.5 samples/s +Step 110/44000 | Loss: 8.6568 | LR: 7.33e-05 | 1309.0ms/step | 15.3 samples/s +Step 120/44000 | Loss: 8.1896 | LR: 8.00e-05 | 1310.1ms/step | 15.3 samples/s +Step 130/44000 | Loss: 7.7721 | LR: 8.67e-05 | 1272.7ms/step | 15.7 samples/s +Step 140/44000 | Loss: 7.5350 | LR: 9.33e-05 | 1357.1ms/step | 14.7 samples/s +Step 150/44000 | Loss: 7.3880 | LR: 1.00e-04 | 1347.1ms/step | 14.8 samples/s +Step 160/44000 | Loss: 7.2282 | LR: 1.07e-04 | 1281.6ms/step | 15.6 samples/s +Step 170/44000 | Loss: 7.1463 | LR: 1.13e-04 | 1301.3ms/step | 15.4 samples/s +Step 180/44000 | Loss: 7.0173 | LR: 1.20e-04 | 1257.0ms/step | 15.9 samples/s +Step 190/44000 | Loss: 6.9240 | LR: 1.27e-04 | 1252.7ms/step | 16.0 samples/s +Step 200/44000 | Loss: 6.8139 | LR: 1.33e-04 | 1293.8ms/step | 15.5 samples/s +Step 210/44000 | Loss: 6.7193 | LR: 1.40e-04 | 1281.1ms/step | 15.6 samples/s +Step 220/44000 | Loss: 6.6591 | LR: 1.47e-04 | 1312.6ms/step | 15.2 samples/s +Step 230/44000 | Loss: 6.5984 | LR: 1.53e-04 | 1289.4ms/step | 15.5 samples/s +Step 240/44000 | Loss: 6.5270 | LR: 1.60e-04 | 1305.1ms/step | 15.3 samples/s +Step 250/44000 | Loss: 6.4677 | LR: 1.67e-04 | 1232.0ms/step | 16.2 samples/s +Validation at step 250: Loss = 6.4030, Perplexity = 603.65 +Step 260/44000 | Loss: 6.3739 | LR: 1.73e-04 | 5875.5ms/step | 3.4 samples/s +Step 270/44000 | Loss: 6.3360 | LR: 1.80e-04 | 1333.6ms/step | 15.0 samples/s +Step 280/44000 | Loss: 6.2818 | LR: 1.87e-04 | 1380.6ms/step | 14.5 samples/s +Step 290/44000 | Loss: 6.2212 | LR: 1.93e-04 | 1330.6ms/step | 15.0 samples/s +Step 300/44000 | Loss: 6.2294 | LR: 2.00e-04 | 1320.6ms/step | 15.1 samples/s +Step 310/44000 | Loss: 6.1424 | LR: 2.07e-04 | 1252.8ms/step | 16.0 samples/s +Step 320/44000 | Loss: 6.0748 | LR: 2.13e-04 | 1273.7ms/step | 15.7 samples/s +Step 330/44000 | Loss: 6.0442 | LR: 2.20e-04 | 1296.2ms/step | 15.4 samples/s +Step 340/44000 | Loss: 6.0802 | LR: 2.27e-04 | 1300.0ms/step | 15.4 samples/s +Step 350/44000 | Loss: 6.0050 | LR: 2.33e-04 | 1316.7ms/step | 15.2 samples/s +Step 360/44000 | Loss: 5.9715 | LR: 2.40e-04 | 1273.9ms/step | 15.7 samples/s +Step 370/44000 | Loss: 5.9115 | LR: 2.47e-04 | 1278.0ms/step | 15.6 samples/s +Step 380/44000 | Loss: 5.8516 | LR: 2.53e-04 | 1284.0ms/step | 15.6 samples/s +Step 390/44000 | Loss: 5.9460 | LR: 2.60e-04 | 1294.1ms/step | 15.5 samples/s +Step 400/44000 | Loss: 5.7988 | LR: 2.67e-04 | 1263.1ms/step | 15.8 samples/s +Step 410/44000 | Loss: 5.7554 | LR: 2.73e-04 | 1298.6ms/step | 15.4 samples/s +Step 420/44000 | Loss: 5.7530 | LR: 2.80e-04 | 1284.7ms/step | 15.6 samples/s +Step 430/44000 | Loss: 5.7016 | LR: 2.87e-04 | 1258.2ms/step | 15.9 samples/s +Step 440/44000 | Loss: 5.6476 | LR: 2.93e-04 | 1325.2ms/step | 15.1 samples/s +Step 450/44000 | Loss: 5.6802 | LR: 3.00e-04 | 1304.9ms/step | 15.3 samples/s +Step 460/44000 | Loss: 5.6165 | LR: 3.07e-04 | 1420.2ms/step | 14.1 samples/s +Step 470/44000 | Loss: 5.5602 | LR: 3.13e-04 | 1271.5ms/step | 15.7 samples/s +Step 480/44000 | Loss: 5.5280 | LR: 3.20e-04 | 1270.7ms/step | 15.7 samples/s +Step 490/44000 | Loss: 5.5246 | LR: 3.27e-04 | 1253.0ms/step | 16.0 samples/s +Step 500/44000 | Loss: 5.5126 | LR: 3.33e-04 | 1282.0ms/step | 15.6 samples/s +Validation at step 500: Loss = 5.4402, Perplexity = 230.50 +Step 510/44000 | Loss: 5.4477 | LR: 3.40e-04 | 1746.9ms/step | 11.4 samples/s +Step 520/44000 | Loss: 5.4373 | LR: 3.47e-04 | 1279.2ms/step | 15.6 samples/s +Step 530/44000 | Loss: 5.4279 | LR: 3.53e-04 | 1281.1ms/step | 15.6 samples/s +Step 540/44000 | Loss: 5.3746 | LR: 3.60e-04 | 1315.2ms/step | 15.2 samples/s +Step 550/44000 | Loss: 5.3511 | LR: 3.67e-04 | 1289.3ms/step | 15.5 samples/s +Step 560/44000 | Loss: 5.2826 | LR: 3.73e-04 | 1294.5ms/step | 15.5 samples/s +Step 570/44000 | Loss: 5.2810 | LR: 3.80e-04 | 1258.3ms/step | 15.9 samples/s +Step 580/44000 | Loss: 5.2017 | LR: 3.87e-04 | 1323.9ms/step | 15.1 samples/s +Step 590/44000 | Loss: 5.2231 | LR: 3.93e-04 | 1306.6ms/step | 15.3 samples/s +Step 600/44000 | Loss: 5.1746 | LR: 4.00e-04 | 1294.5ms/step | 15.4 samples/s +Step 610/44000 | Loss: 5.1043 | LR: 4.07e-04 | 1210.8ms/step | 16.5 samples/s +Step 620/44000 | Loss: 5.1168 | LR: 4.13e-04 | 1244.7ms/step | 16.1 samples/s +Step 630/44000 | Loss: 5.0682 | LR: 4.20e-04 | 1428.3ms/step | 14.0 samples/s +Step 640/44000 | Loss: 5.0771 | LR: 4.27e-04 | 1257.3ms/step | 15.9 samples/s +Step 650/44000 | Loss: 5.0472 | LR: 4.33e-04 | 1312.1ms/step | 15.2 samples/s +Step 660/44000 | Loss: 4.9967 | LR: 4.40e-04 | 1265.3ms/step | 15.8 samples/s +Step 670/44000 | Loss: 4.9984 | LR: 4.47e-04 | 1264.3ms/step | 15.8 samples/s +Step 680/44000 | Loss: 4.9845 | LR: 4.53e-04 | 1314.0ms/step | 15.2 samples/s +Step 690/44000 | Loss: 4.9078 | LR: 4.60e-04 | 1236.6ms/step | 16.2 samples/s +Step 700/44000 | Loss: 4.9321 | LR: 4.67e-04 | 1247.4ms/step | 16.0 samples/s +Step 710/44000 | Loss: 4.8508 | LR: 4.73e-04 | 1344.6ms/step | 14.9 samples/s +Step 720/44000 | Loss: 4.8735 | LR: 4.80e-04 | 1239.7ms/step | 16.1 samples/s +Step 730/44000 | Loss: 4.7798 | LR: 4.87e-04 | 1294.2ms/step | 15.5 samples/s +Step 740/44000 | Loss: 4.7891 | LR: 4.93e-04 | 1244.5ms/step | 16.1 samples/s +Step 750/44000 | Loss: 4.7265 | LR: 5.00e-04 | 1313.6ms/step | 15.2 samples/s +Validation at step 750: Loss = 4.7034, Perplexity = 110.32 +Step 760/44000 | Loss: 4.6981 | LR: 5.00e-04 | 1761.5ms/step | 11.4 samples/s +Step 770/44000 | Loss: 4.7245 | LR: 5.00e-04 | 1279.2ms/step | 15.6 samples/s +Step 780/44000 | Loss: 4.7045 | LR: 5.00e-04 | 1298.6ms/step | 15.4 samples/s +Step 790/44000 | Loss: 4.6862 | LR: 5.00e-04 | 1309.9ms/step | 15.3 samples/s +Step 800/44000 | Loss: 4.6523 | LR: 5.00e-04 | 1451.5ms/step | 13.8 samples/s +Step 810/44000 | Loss: 4.5644 | LR: 5.00e-04 | 1263.3ms/step | 15.8 samples/s +Step 820/44000 | Loss: 4.5501 | LR: 5.00e-04 | 1328.5ms/step | 15.1 samples/s +Step 830/44000 | Loss: 4.6437 | LR: 5.00e-04 | 1260.8ms/step | 15.9 samples/s +Step 840/44000 | Loss: 4.5651 | LR: 5.00e-04 | 1289.5ms/step | 15.5 samples/s +Step 850/44000 | Loss: 4.5254 | LR: 5.00e-04 | 1278.7ms/step | 15.6 samples/s +Step 860/44000 | Loss: 4.5349 | LR: 5.00e-04 | 1305.8ms/step | 15.3 samples/s +Step 870/44000 | Loss: 4.5025 | LR: 5.00e-04 | 1290.9ms/step | 15.5 samples/s +Step 880/44000 | Loss: 4.4465 | LR: 5.00e-04 | 1302.9ms/step | 15.3 samples/s +Step 890/44000 | Loss: 4.4396 | LR: 5.00e-04 | 1279.1ms/step | 15.6 samples/s +Step 900/44000 | Loss: 4.3997 | LR: 5.00e-04 | 1278.8ms/step | 15.6 samples/s +Step 910/44000 | Loss: 4.3799 | LR: 5.00e-04 | 1237.2ms/step | 16.2 samples/s +Step 920/44000 | Loss: 4.4326 | LR: 5.00e-04 | 1301.9ms/step | 15.4 samples/s +Step 930/44000 | Loss: 4.4036 | LR: 5.00e-04 | 1300.7ms/step | 15.4 samples/s +Step 940/44000 | Loss: 4.3334 | LR: 5.00e-04 | 1273.9ms/step | 15.7 samples/s +Step 950/44000 | Loss: 4.3813 | LR: 5.00e-04 | 1265.7ms/step | 15.8 samples/s +Step 960/44000 | Loss: 4.3298 | LR: 5.00e-04 | 1252.4ms/step | 16.0 samples/s +Step 970/44000 | Loss: 4.3521 | LR: 5.00e-04 | 1408.9ms/step | 14.2 samples/s +Step 980/44000 | Loss: 4.2569 | LR: 5.00e-04 | 1252.6ms/step | 16.0 samples/s +Step 990/44000 | Loss: 4.3005 | LR: 5.00e-04 | 1342.1ms/step | 14.9 samples/s +Step 1000/44000 | Loss: 4.2341 | LR: 5.00e-04 | 1305.7ms/step | 15.3 samples/s +Validation at step 1000: Loss = 4.2438, Perplexity = 69.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-1000 +Step 1010/44000 | Loss: 4.2474 | LR: 5.00e-04 | 2226.2ms/step | 9.0 samples/s +Step 1020/44000 | Loss: 4.2724 | LR: 5.00e-04 | 1278.0ms/step | 15.6 samples/s +Step 1030/44000 | Loss: 4.2410 | LR: 5.00e-04 | 1271.1ms/step | 15.7 samples/s +Step 1040/44000 | Loss: 4.2816 | LR: 5.00e-04 | 1300.9ms/step | 15.4 samples/s +Step 1050/44000 | Loss: 4.2101 | LR: 5.00e-04 | 1256.5ms/step | 15.9 samples/s +Step 1060/44000 | Loss: 4.2154 | LR: 5.00e-04 | 1278.9ms/step | 15.6 samples/s +Step 1070/44000 | Loss: 4.2397 | LR: 5.00e-04 | 1316.2ms/step | 15.2 samples/s +Step 1080/44000 | Loss: 4.2517 | LR: 5.00e-04 | 1236.2ms/step | 16.2 samples/s +Step 1090/44000 | Loss: 4.1695 | LR: 5.00e-04 | 1271.3ms/step | 15.7 samples/s +Step 1100/44000 | Loss: 4.2177 | LR: 5.00e-04 | 1285.5ms/step | 15.6 samples/s +Step 1110/44000 | Loss: 4.1701 | LR: 5.00e-04 | 1234.7ms/step | 16.2 samples/s +Step 1120/44000 | Loss: 4.2262 | LR: 5.00e-04 | 1338.8ms/step | 14.9 samples/s +Step 1130/44000 | Loss: 4.2158 | LR: 5.00e-04 | 1279.0ms/step | 15.6 samples/s +Step 1140/44000 | Loss: 4.1027 | LR: 5.00e-04 | 1394.0ms/step | 14.3 samples/s +Step 1150/44000 | Loss: 4.1458 | LR: 5.00e-04 | 1269.2ms/step | 15.8 samples/s +Step 1160/44000 | Loss: 4.1164 | LR: 5.00e-04 | 1249.6ms/step | 16.0 samples/s +Step 1170/44000 | Loss: 4.0804 | LR: 5.00e-04 | 1228.3ms/step | 16.3 samples/s +Step 1180/44000 | Loss: 4.1443 | LR: 5.00e-04 | 1279.0ms/step | 15.6 samples/s +Step 1190/44000 | Loss: 4.1624 | LR: 5.00e-04 | 1299.6ms/step | 15.4 samples/s +Step 1200/44000 | Loss: 4.0270 | LR: 5.00e-04 | 1305.1ms/step | 15.3 samples/s +Step 1210/44000 | Loss: 4.0931 | LR: 5.00e-04 | 1268.6ms/step | 15.8 samples/s +Step 1220/44000 | Loss: 4.0813 | LR: 5.00e-04 | 1294.5ms/step | 15.4 samples/s +Step 1230/44000 | Loss: 4.0987 | LR: 5.00e-04 | 1278.7ms/step | 15.6 samples/s +Step 1240/44000 | Loss: 4.0481 | LR: 5.00e-04 | 1294.2ms/step | 15.5 samples/s +Step 1250/44000 | Loss: 4.0683 | LR: 5.00e-04 | 1294.5ms/step | 15.5 samples/s +Validation at step 1250: Loss = 3.9967, Perplexity = 54.42 +Step 1260/44000 | Loss: 4.0188 | LR: 5.00e-04 | 1781.9ms/step | 11.2 samples/s +Step 1270/44000 | Loss: 4.0229 | LR: 5.00e-04 | 1286.8ms/step | 15.5 samples/s +Step 1280/44000 | Loss: 4.0347 | LR: 5.00e-04 | 1275.3ms/step | 15.7 samples/s +Step 1290/44000 | Loss: 4.0874 | LR: 5.00e-04 | 1256.8ms/step | 15.9 samples/s +Step 1300/44000 | Loss: 4.0571 | LR: 5.00e-04 | 1278.8ms/step | 15.6 samples/s +Step 1310/44000 | Loss: 4.0706 | LR: 5.00e-04 | 1423.9ms/step | 14.0 samples/s +Step 1320/44000 | Loss: 3.9993 | LR: 5.00e-04 | 1280.5ms/step | 15.6 samples/s +Step 1330/44000 | Loss: 4.0366 | LR: 5.00e-04 | 1304.9ms/step | 15.3 samples/s +Step 1340/44000 | Loss: 3.9759 | LR: 5.00e-04 | 1317.8ms/step | 15.2 samples/s +Step 1350/44000 | Loss: 4.0438 | LR: 5.00e-04 | 1293.3ms/step | 15.5 samples/s +Step 1360/44000 | Loss: 4.0066 | LR: 5.00e-04 | 1262.0ms/step | 15.8 samples/s +Step 1370/44000 | Loss: 4.0132 | LR: 5.00e-04 | 1289.0ms/step | 15.5 samples/s +Step 1380/44000 | Loss: 4.0097 | LR: 5.00e-04 | 1246.2ms/step | 16.0 samples/s +Step 1390/44000 | Loss: 3.9244 | LR: 5.00e-04 | 1316.9ms/step | 15.2 samples/s +Step 1400/44000 | Loss: 3.9229 | LR: 5.00e-04 | 1257.1ms/step | 15.9 samples/s +Step 1410/44000 | Loss: 3.9162 | LR: 5.00e-04 | 1279.2ms/step | 15.6 samples/s +Step 1420/44000 | Loss: 3.9252 | LR: 5.00e-04 | 1295.1ms/step | 15.4 samples/s +Step 1430/44000 | Loss: 3.8724 | LR: 5.00e-04 | 1322.5ms/step | 15.1 samples/s +Step 1440/44000 | Loss: 3.9763 | LR: 5.00e-04 | 1287.7ms/step | 15.5 samples/s +Step 1450/44000 | Loss: 3.9381 | LR: 5.00e-04 | 1284.7ms/step | 15.6 samples/s +Step 1460/44000 | Loss: 3.8557 | LR: 5.00e-04 | 1273.5ms/step | 15.7 samples/s +Step 1470/44000 | Loss: 3.9341 | LR: 5.00e-04 | 1299.8ms/step | 15.4 samples/s +Step 1480/44000 | Loss: 3.9289 | LR: 5.00e-04 | 1325.8ms/step | 15.1 samples/s +Step 1490/44000 | Loss: 3.9324 | LR: 5.00e-04 | 1358.0ms/step | 14.7 samples/s +Step 1500/44000 | Loss: 3.9276 | LR: 5.00e-04 | 1252.9ms/step | 16.0 samples/s +Validation at step 1500: Loss = 3.8428, Perplexity = 46.66 +Step 1510/44000 | Loss: 3.9598 | LR: 5.00e-04 | 1808.1ms/step | 11.1 samples/s +Step 1520/44000 | Loss: 3.8177 | LR: 5.00e-04 | 1298.1ms/step | 15.4 samples/s +Step 1530/44000 | Loss: 3.8696 | LR: 5.00e-04 | 1251.9ms/step | 16.0 samples/s +Step 1540/44000 | Loss: 3.8515 | LR: 5.00e-04 | 1312.3ms/step | 15.2 samples/s +Step 1550/44000 | Loss: 3.8777 | LR: 5.00e-04 | 1336.9ms/step | 15.0 samples/s +Step 1560/44000 | Loss: 3.9201 | LR: 5.00e-04 | 1284.0ms/step | 15.6 samples/s +Step 1570/44000 | Loss: 3.8645 | LR: 5.00e-04 | 1264.3ms/step | 15.8 samples/s +Step 1580/44000 | Loss: 3.9536 | LR: 5.00e-04 | 1298.1ms/step | 15.4 samples/s +Step 1590/44000 | Loss: 3.8744 | LR: 5.00e-04 | 1247.3ms/step | 16.0 samples/s +Step 1600/44000 | Loss: 3.8573 | LR: 5.00e-04 | 1257.6ms/step | 15.9 samples/s +Step 1610/44000 | Loss: 3.8491 | LR: 5.00e-04 | 1282.1ms/step | 15.6 samples/s +Step 1620/44000 | Loss: 3.9123 | LR: 5.00e-04 | 1272.8ms/step | 15.7 samples/s +Step 1630/44000 | Loss: 3.7998 | LR: 4.99e-04 | 1288.1ms/step | 15.5 samples/s +Step 1640/44000 | Loss: 3.8919 | LR: 4.99e-04 | 1299.8ms/step | 15.4 samples/s +Step 1650/44000 | Loss: 3.8367 | LR: 4.99e-04 | 1346.9ms/step | 14.8 samples/s +Step 1660/44000 | Loss: 3.8202 | LR: 4.99e-04 | 1352.8ms/step | 14.8 samples/s +Step 1670/44000 | Loss: 3.8449 | LR: 4.99e-04 | 1278.6ms/step | 15.6 samples/s +Step 1680/44000 | Loss: 3.8319 | LR: 4.99e-04 | 1289.8ms/step | 15.5 samples/s +Step 1690/44000 | Loss: 3.8620 | LR: 4.99e-04 | 1273.9ms/step | 15.7 samples/s +Step 1700/44000 | Loss: 3.8330 | LR: 4.99e-04 | 1288.7ms/step | 15.5 samples/s +Step 1710/44000 | Loss: 3.7783 | LR: 4.99e-04 | 1263.7ms/step | 15.8 samples/s +Step 1720/44000 | Loss: 3.7634 | LR: 4.99e-04 | 1267.7ms/step | 15.8 samples/s +Step 1730/44000 | Loss: 3.8263 | LR: 4.99e-04 | 1334.8ms/step | 15.0 samples/s +Step 1740/44000 | Loss: 3.7972 | LR: 4.99e-04 | 1284.2ms/step | 15.6 samples/s +Step 1750/44000 | Loss: 3.8067 | LR: 4.99e-04 | 1264.4ms/step | 15.8 samples/s +Validation at step 1750: Loss = 3.7371, Perplexity = 41.97 +Step 1760/44000 | Loss: 3.8199 | LR: 4.99e-04 | 1787.4ms/step | 11.2 samples/s +Step 1770/44000 | Loss: 3.7581 | LR: 4.99e-04 | 1255.9ms/step | 15.9 samples/s +Step 1780/44000 | Loss: 3.7877 | LR: 4.99e-04 | 1297.0ms/step | 15.4 samples/s +Step 1790/44000 | Loss: 3.7728 | LR: 4.99e-04 | 1278.6ms/step | 15.6 samples/s +Step 1800/44000 | Loss: 3.8320 | LR: 4.99e-04 | 1294.7ms/step | 15.4 samples/s +Step 1810/44000 | Loss: 3.7664 | LR: 4.99e-04 | 1224.5ms/step | 16.3 samples/s +Step 1820/44000 | Loss: 3.7864 | LR: 4.99e-04 | 1275.4ms/step | 15.7 samples/s +Step 1830/44000 | Loss: 3.7166 | LR: 4.99e-04 | 1425.9ms/step | 14.0 samples/s +Step 1840/44000 | Loss: 3.7803 | LR: 4.99e-04 | 1325.8ms/step | 15.1 samples/s +Step 1850/44000 | Loss: 3.7771 | LR: 4.99e-04 | 1249.3ms/step | 16.0 samples/s +Step 1860/44000 | Loss: 3.7357 | LR: 4.99e-04 | 1251.9ms/step | 16.0 samples/s +Step 1870/44000 | Loss: 3.7417 | LR: 4.99e-04 | 1253.8ms/step | 16.0 samples/s +Step 1880/44000 | Loss: 3.7383 | LR: 4.99e-04 | 1278.1ms/step | 15.6 samples/s +Step 1890/44000 | Loss: 3.6923 | LR: 4.99e-04 | 1289.7ms/step | 15.5 samples/s +Step 1900/44000 | Loss: 3.7471 | LR: 4.99e-04 | 1237.8ms/step | 16.2 samples/s +Step 1910/44000 | Loss: 3.7990 | LR: 4.99e-04 | 1285.4ms/step | 15.6 samples/s +Step 1920/44000 | Loss: 3.7344 | LR: 4.99e-04 | 1315.7ms/step | 15.2 samples/s +Step 1930/44000 | Loss: 3.7250 | LR: 4.99e-04 | 1287.4ms/step | 15.5 samples/s +Step 1940/44000 | Loss: 3.7180 | LR: 4.99e-04 | 1255.3ms/step | 15.9 samples/s +Step 1950/44000 | Loss: 3.7812 | LR: 4.99e-04 | 1260.4ms/step | 15.9 samples/s +Step 1960/44000 | Loss: 3.7251 | LR: 4.99e-04 | 1250.8ms/step | 16.0 samples/s +Step 1970/44000 | Loss: 3.7708 | LR: 4.99e-04 | 1244.3ms/step | 16.1 samples/s +Step 1980/44000 | Loss: 3.7167 | LR: 4.99e-04 | 1278.9ms/step | 15.6 samples/s +Step 1990/44000 | Loss: 3.7103 | LR: 4.99e-04 | 1279.1ms/step | 15.6 samples/s +Step 2000/44000 | Loss: 3.6966 | LR: 4.99e-04 | 1415.0ms/step | 14.1 samples/s +Validation at step 2000: Loss = 3.6498, Perplexity = 38.47 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-2000 +Step 2010/44000 | Loss: 3.6552 | LR: 4.99e-04 | 2233.1ms/step | 9.0 samples/s +Step 2020/44000 | Loss: 3.7417 | LR: 4.99e-04 | 1331.2ms/step | 15.0 samples/s +Step 2030/44000 | Loss: 3.6411 | LR: 4.99e-04 | 1280.7ms/step | 15.6 samples/s +Step 2040/44000 | Loss: 3.6949 | LR: 4.99e-04 | 1271.7ms/step | 15.7 samples/s +Step 2050/44000 | Loss: 3.6982 | LR: 4.99e-04 | 1289.5ms/step | 15.5 samples/s +Step 2060/44000 | Loss: 3.7107 | LR: 4.99e-04 | 1288.1ms/step | 15.5 samples/s +Step 2070/44000 | Loss: 3.6782 | LR: 4.99e-04 | 1276.9ms/step | 15.7 samples/s +Step 2080/44000 | Loss: 3.6544 | LR: 4.99e-04 | 1356.5ms/step | 14.7 samples/s +Step 2090/44000 | Loss: 3.6346 | LR: 4.99e-04 | 1301.2ms/step | 15.4 samples/s +Step 2100/44000 | Loss: 3.5991 | LR: 4.99e-04 | 1324.4ms/step | 15.1 samples/s +Step 2110/44000 | Loss: 3.7328 | LR: 4.99e-04 | 1305.2ms/step | 15.3 samples/s +Step 2120/44000 | Loss: 3.6868 | LR: 4.99e-04 | 1307.7ms/step | 15.3 samples/s +Step 2130/44000 | Loss: 3.7183 | LR: 4.99e-04 | 1276.2ms/step | 15.7 samples/s +Step 2140/44000 | Loss: 3.6606 | LR: 4.99e-04 | 1279.4ms/step | 15.6 samples/s +Step 2150/44000 | Loss: 3.6384 | LR: 4.99e-04 | 1278.9ms/step | 15.6 samples/s +Step 2160/44000 | Loss: 3.7213 | LR: 4.99e-04 | 1283.9ms/step | 15.6 samples/s +Step 2170/44000 | Loss: 3.6837 | LR: 4.99e-04 | 1431.3ms/step | 14.0 samples/s +Step 2180/44000 | Loss: 3.6651 | LR: 4.99e-04 | 1273.7ms/step | 15.7 samples/s +Step 2190/44000 | Loss: 3.6187 | LR: 4.99e-04 | 1284.1ms/step | 15.6 samples/s +Step 2200/44000 | Loss: 3.6269 | LR: 4.99e-04 | 1300.9ms/step | 15.4 samples/s +Step 2210/44000 | Loss: 3.6620 | LR: 4.99e-04 | 1283.8ms/step | 15.6 samples/s +Step 2220/44000 | Loss: 3.5860 | LR: 4.99e-04 | 1289.6ms/step | 15.5 samples/s +Step 2230/44000 | Loss: 3.6559 | LR: 4.99e-04 | 1281.2ms/step | 15.6 samples/s +Step 2240/44000 | Loss: 3.7274 | LR: 4.99e-04 | 1286.7ms/step | 15.5 samples/s +Step 2250/44000 | Loss: 3.6772 | LR: 4.99e-04 | 1269.3ms/step | 15.8 samples/s +Validation at step 2250: Loss = 3.5885, Perplexity = 36.18 +Step 2260/44000 | Loss: 3.6490 | LR: 4.98e-04 | 1792.7ms/step | 11.2 samples/s +Step 2270/44000 | Loss: 3.6656 | LR: 4.98e-04 | 1249.6ms/step | 16.0 samples/s +Step 2280/44000 | Loss: 3.6413 | LR: 4.98e-04 | 1251.0ms/step | 16.0 samples/s +Step 2290/44000 | Loss: 3.6327 | LR: 4.98e-04 | 1315.2ms/step | 15.2 samples/s +Step 2300/44000 | Loss: 3.5971 | LR: 4.98e-04 | 1288.0ms/step | 15.5 samples/s +Step 2310/44000 | Loss: 3.6109 | LR: 4.98e-04 | 1305.3ms/step | 15.3 samples/s +Step 2320/44000 | Loss: 3.6099 | LR: 4.98e-04 | 1259.9ms/step | 15.9 samples/s +Step 2330/44000 | Loss: 3.7030 | LR: 4.98e-04 | 1274.9ms/step | 15.7 samples/s +Step 2340/44000 | Loss: 3.5918 | LR: 4.98e-04 | 1404.7ms/step | 14.2 samples/s +Step 2350/44000 | Loss: 3.5556 | LR: 4.98e-04 | 1315.6ms/step | 15.2 samples/s +Step 2360/44000 | Loss: 3.6830 | LR: 4.98e-04 | 1316.2ms/step | 15.2 samples/s +Step 2370/44000 | Loss: 3.5295 | LR: 4.98e-04 | 1250.3ms/step | 16.0 samples/s +Step 2380/44000 | Loss: 3.6052 | LR: 4.98e-04 | 1283.6ms/step | 15.6 samples/s +Step 2390/44000 | Loss: 3.6521 | LR: 4.98e-04 | 1312.0ms/step | 15.2 samples/s +Step 2400/44000 | Loss: 3.6041 | LR: 4.98e-04 | 1278.8ms/step | 15.6 samples/s +Step 2410/44000 | Loss: 3.6579 | LR: 4.98e-04 | 1283.1ms/step | 15.6 samples/s +Step 2420/44000 | Loss: 3.6664 | LR: 4.98e-04 | 1305.5ms/step | 15.3 samples/s +Step 2430/44000 | Loss: 3.6693 | LR: 4.98e-04 | 1273.8ms/step | 15.7 samples/s +Step 2440/44000 | Loss: 3.5854 | LR: 4.98e-04 | 1248.1ms/step | 16.0 samples/s +Step 2450/44000 | Loss: 3.6266 | LR: 4.98e-04 | 1271.2ms/step | 15.7 samples/s +Step 2460/44000 | Loss: 3.5872 | LR: 4.98e-04 | 1251.7ms/step | 16.0 samples/s +Step 2470/44000 | Loss: 3.5509 | LR: 4.98e-04 | 1277.3ms/step | 15.7 samples/s +Step 2480/44000 | Loss: 3.5761 | LR: 4.98e-04 | 1278.8ms/step | 15.6 samples/s +Step 2490/44000 | Loss: 3.5627 | LR: 4.98e-04 | 1295.0ms/step | 15.4 samples/s +Step 2500/44000 | Loss: 3.6039 | LR: 4.98e-04 | 1289.5ms/step | 15.5 samples/s +Validation at step 2500: Loss = 3.5334, Perplexity = 34.24 +Step 2510/44000 | Loss: 3.6106 | LR: 4.98e-04 | 1897.3ms/step | 10.5 samples/s +Step 2520/44000 | Loss: 3.5669 | LR: 4.98e-04 | 1299.8ms/step | 15.4 samples/s +Step 2530/44000 | Loss: 3.6091 | LR: 4.98e-04 | 1316.1ms/step | 15.2 samples/s +Step 2540/44000 | Loss: 3.6255 | LR: 4.98e-04 | 1278.4ms/step | 15.6 samples/s +Step 2550/44000 | Loss: 3.5305 | LR: 4.98e-04 | 1276.6ms/step | 15.7 samples/s +Step 2560/44000 | Loss: 3.5938 | LR: 4.98e-04 | 1264.7ms/step | 15.8 samples/s +Step 2570/44000 | Loss: 3.5884 | LR: 4.98e-04 | 1242.9ms/step | 16.1 samples/s +Step 2580/44000 | Loss: 3.5941 | LR: 4.98e-04 | 1300.0ms/step | 15.4 samples/s +Step 2590/44000 | Loss: 3.5994 | LR: 4.98e-04 | 1263.3ms/step | 15.8 samples/s +Step 2600/44000 | Loss: 3.5501 | LR: 4.98e-04 | 1294.2ms/step | 15.5 samples/s +Step 2610/44000 | Loss: 3.5612 | LR: 4.98e-04 | 1275.8ms/step | 15.7 samples/s +Step 2620/44000 | Loss: 3.5885 | LR: 4.98e-04 | 1235.5ms/step | 16.2 samples/s +Step 2630/44000 | Loss: 3.5849 | LR: 4.98e-04 | 1256.1ms/step | 15.9 samples/s +Step 2640/44000 | Loss: 3.5847 | LR: 4.98e-04 | 1268.2ms/step | 15.8 samples/s +Step 2650/44000 | Loss: 3.6249 | LR: 4.98e-04 | 1264.6ms/step | 15.8 samples/s +Step 2660/44000 | Loss: 3.6067 | LR: 4.98e-04 | 1261.0ms/step | 15.9 samples/s +Step 2670/44000 | Loss: 3.5223 | LR: 4.98e-04 | 1269.3ms/step | 15.8 samples/s +Step 2680/44000 | Loss: 3.5869 | LR: 4.98e-04 | 1330.1ms/step | 15.0 samples/s +Step 2690/44000 | Loss: 3.5509 | LR: 4.98e-04 | 1325.7ms/step | 15.1 samples/s +Step 2700/44000 | Loss: 3.5345 | LR: 4.97e-04 | 1252.7ms/step | 16.0 samples/s +Step 2710/44000 | Loss: 3.5639 | LR: 4.97e-04 | 1278.9ms/step | 15.6 samples/s +Step 2720/44000 | Loss: 3.5230 | LR: 4.97e-04 | 1280.4ms/step | 15.6 samples/s +Step 2730/44000 | Loss: 3.5608 | LR: 4.97e-04 | 1276.8ms/step | 15.7 samples/s +Step 2740/44000 | Loss: 3.6164 | LR: 4.97e-04 | 1252.3ms/step | 16.0 samples/s +Step 2750/44000 | Loss: 3.5847 | LR: 4.97e-04 | 1278.9ms/step | 15.6 samples/s +Validation at step 2750: Loss = 3.4891, Perplexity = 32.76 +Step 2760/44000 | Loss: 3.5693 | LR: 4.97e-04 | 1797.6ms/step | 11.1 samples/s +Step 2770/44000 | Loss: 3.5055 | LR: 4.97e-04 | 1262.8ms/step | 15.8 samples/s +Step 2780/44000 | Loss: 3.5598 | LR: 4.97e-04 | 1278.9ms/step | 15.6 samples/s +Step 2790/44000 | Loss: 3.4998 | LR: 4.97e-04 | 1299.7ms/step | 15.4 samples/s +Step 2800/44000 | Loss: 3.5670 | LR: 4.97e-04 | 1326.5ms/step | 15.1 samples/s +Step 2810/44000 | Loss: 3.5085 | LR: 4.97e-04 | 1310.1ms/step | 15.3 samples/s +Step 2820/44000 | Loss: 3.5353 | LR: 4.97e-04 | 1305.5ms/step | 15.3 samples/s +Step 2830/44000 | Loss: 3.5379 | LR: 4.97e-04 | 1309.4ms/step | 15.3 samples/s +Step 2840/44000 | Loss: 3.4909 | LR: 4.97e-04 | 1275.2ms/step | 15.7 samples/s +Step 2850/44000 | Loss: 3.5213 | LR: 4.97e-04 | 1314.6ms/step | 15.2 samples/s +Step 2860/44000 | Loss: 3.5211 | LR: 4.97e-04 | 1343.0ms/step | 14.9 samples/s +Step 2870/44000 | Loss: 3.6182 | LR: 4.97e-04 | 1283.1ms/step | 15.6 samples/s +Step 2880/44000 | Loss: 3.4603 | LR: 4.97e-04 | 1310.2ms/step | 15.3 samples/s +Step 2890/44000 | Loss: 3.4930 | LR: 4.97e-04 | 1295.1ms/step | 15.4 samples/s +Step 2900/44000 | Loss: 3.5163 | LR: 4.97e-04 | 1336.8ms/step | 15.0 samples/s +Step 2910/44000 | Loss: 3.5842 | LR: 4.97e-04 | 1295.5ms/step | 15.4 samples/s +Step 2920/44000 | Loss: 3.5733 | LR: 4.97e-04 | 1287.9ms/step | 15.5 samples/s +Step 2930/44000 | Loss: 3.5771 | LR: 4.97e-04 | 1258.2ms/step | 15.9 samples/s +Step 2940/44000 | Loss: 3.5101 | LR: 4.97e-04 | 1252.9ms/step | 16.0 samples/s +Step 2950/44000 | Loss: 3.4898 | LR: 4.97e-04 | 1304.8ms/step | 15.3 samples/s +Step 2960/44000 | Loss: 3.5693 | LR: 4.97e-04 | 1320.7ms/step | 15.1 samples/s +Step 2970/44000 | Loss: 3.5339 | LR: 4.97e-04 | 1272.8ms/step | 15.7 samples/s +Step 2980/44000 | Loss: 3.5087 | LR: 4.97e-04 | 1270.2ms/step | 15.7 samples/s +Step 2990/44000 | Loss: 3.5232 | LR: 4.97e-04 | 1268.5ms/step | 15.8 samples/s +Step 3000/44000 | Loss: 3.5096 | LR: 4.97e-04 | 1247.6ms/step | 16.0 samples/s +Validation at step 3000: Loss = 3.4522, Perplexity = 31.57 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-3000 +Step 3010/44000 | Loss: 3.5256 | LR: 4.97e-04 | 2227.0ms/step | 9.0 samples/s +Step 3020/44000 | Loss: 3.4988 | LR: 4.97e-04 | 1430.4ms/step | 14.0 samples/s +Step 3030/44000 | Loss: 3.4932 | LR: 4.97e-04 | 1284.3ms/step | 15.6 samples/s +Step 3040/44000 | Loss: 3.4418 | LR: 4.97e-04 | 1330.7ms/step | 15.0 samples/s +Step 3050/44000 | Loss: 3.4420 | LR: 4.97e-04 | 1290.3ms/step | 15.5 samples/s +Step 3060/44000 | Loss: 3.6152 | LR: 4.96e-04 | 1237.4ms/step | 16.2 samples/s +Step 3070/44000 | Loss: 3.5381 | LR: 4.96e-04 | 1277.8ms/step | 15.7 samples/s +Step 3080/44000 | Loss: 3.4487 | LR: 4.96e-04 | 1309.9ms/step | 15.3 samples/s +Step 3090/44000 | Loss: 3.4801 | LR: 4.96e-04 | 1268.6ms/step | 15.8 samples/s +Step 3100/44000 | Loss: 3.5161 | LR: 4.96e-04 | 1294.1ms/step | 15.5 samples/s +Step 3110/44000 | Loss: 3.5472 | LR: 4.96e-04 | 1284.1ms/step | 15.6 samples/s +Step 3120/44000 | Loss: 3.4857 | LR: 4.96e-04 | 1309.9ms/step | 15.3 samples/s +Step 3130/44000 | Loss: 3.5503 | LR: 4.96e-04 | 1309.1ms/step | 15.3 samples/s +Step 3140/44000 | Loss: 3.4966 | LR: 4.96e-04 | 1289.8ms/step | 15.5 samples/s +Step 3150/44000 | Loss: 3.5021 | LR: 4.96e-04 | 1294.7ms/step | 15.4 samples/s +Step 3160/44000 | Loss: 3.4784 | LR: 4.96e-04 | 1310.3ms/step | 15.3 samples/s +Step 3170/44000 | Loss: 3.4238 | LR: 4.96e-04 | 1272.8ms/step | 15.7 samples/s +Step 3180/44000 | Loss: 3.5029 | LR: 4.96e-04 | 1292.3ms/step | 15.5 samples/s +Step 3190/44000 | Loss: 3.4583 | LR: 4.96e-04 | 1323.1ms/step | 15.1 samples/s +Step 3200/44000 | Loss: 3.4386 | LR: 4.96e-04 | 1315.8ms/step | 15.2 samples/s +Step 3210/44000 | Loss: 3.4872 | LR: 4.96e-04 | 1268.3ms/step | 15.8 samples/s +Step 3220/44000 | Loss: 3.5165 | LR: 4.96e-04 | 1294.2ms/step | 15.5 samples/s +Step 3230/44000 | Loss: 3.4595 | LR: 4.96e-04 | 1278.3ms/step | 15.6 samples/s +Step 3240/44000 | Loss: 3.4168 | LR: 4.96e-04 | 1314.5ms/step | 15.2 samples/s +Step 3250/44000 | Loss: 3.4593 | LR: 4.96e-04 | 1272.1ms/step | 15.7 samples/s +Validation at step 3250: Loss = 3.4245, Perplexity = 30.71 +Step 3260/44000 | Loss: 3.5052 | LR: 4.96e-04 | 1811.5ms/step | 11.0 samples/s +Step 3270/44000 | Loss: 3.5559 | LR: 4.96e-04 | 1268.6ms/step | 15.8 samples/s +Step 3280/44000 | Loss: 3.5431 | LR: 4.96e-04 | 1283.2ms/step | 15.6 samples/s +Step 3290/44000 | Loss: 3.4581 | LR: 4.96e-04 | 1305.6ms/step | 15.3 samples/s +Step 3300/44000 | Loss: 3.5308 | LR: 4.96e-04 | 1273.7ms/step | 15.7 samples/s +Step 3310/44000 | Loss: 3.5167 | LR: 4.96e-04 | 1264.7ms/step | 15.8 samples/s +Step 3320/44000 | Loss: 3.4968 | LR: 4.96e-04 | 1333.0ms/step | 15.0 samples/s +Step 3330/44000 | Loss: 3.4734 | LR: 4.96e-04 | 1258.8ms/step | 15.9 samples/s +Step 3340/44000 | Loss: 3.5485 | LR: 4.96e-04 | 1294.7ms/step | 15.4 samples/s +Step 3350/44000 | Loss: 3.4534 | LR: 4.96e-04 | 1310.4ms/step | 15.3 samples/s +Step 3360/44000 | Loss: 3.5601 | LR: 4.96e-04 | 1325.3ms/step | 15.1 samples/s +Step 3370/44000 | Loss: 3.4212 | LR: 4.95e-04 | 1385.9ms/step | 14.4 samples/s +Step 3380/44000 | Loss: 3.4845 | LR: 4.95e-04 | 1310.6ms/step | 15.3 samples/s +Step 3390/44000 | Loss: 3.4636 | LR: 4.95e-04 | 1254.5ms/step | 15.9 samples/s +Step 3400/44000 | Loss: 3.5297 | LR: 4.95e-04 | 1310.6ms/step | 15.3 samples/s +Step 3410/44000 | Loss: 3.4602 | LR: 4.95e-04 | 1299.4ms/step | 15.4 samples/s +Step 3420/44000 | Loss: 3.4726 | LR: 4.95e-04 | 1294.6ms/step | 15.4 samples/s +Step 3430/44000 | Loss: 3.4292 | LR: 4.95e-04 | 1293.2ms/step | 15.5 samples/s +Step 3440/44000 | Loss: 3.4496 | LR: 4.95e-04 | 1316.0ms/step | 15.2 samples/s +Step 3450/44000 | Loss: 3.3899 | LR: 4.95e-04 | 1372.9ms/step | 14.6 samples/s +Step 3460/44000 | Loss: 3.4073 | LR: 4.95e-04 | 1257.3ms/step | 15.9 samples/s +Step 3470/44000 | Loss: 3.4781 | LR: 4.95e-04 | 1310.3ms/step | 15.3 samples/s +Step 3480/44000 | Loss: 3.4642 | LR: 4.95e-04 | 1309.7ms/step | 15.3 samples/s +Step 3490/44000 | Loss: 3.4065 | LR: 4.95e-04 | 1292.1ms/step | 15.5 samples/s +Step 3500/44000 | Loss: 3.4359 | LR: 4.95e-04 | 1274.5ms/step | 15.7 samples/s +Validation at step 3500: Loss = 3.3944, Perplexity = 29.80 +Step 3510/44000 | Loss: 3.4295 | LR: 4.95e-04 | 1810.2ms/step | 11.0 samples/s +Step 3520/44000 | Loss: 3.4205 | LR: 4.95e-04 | 1303.4ms/step | 15.3 samples/s +Step 3530/44000 | Loss: 3.4461 | LR: 4.95e-04 | 1425.2ms/step | 14.0 samples/s +Step 3540/44000 | Loss: 3.4804 | LR: 4.95e-04 | 1326.7ms/step | 15.1 samples/s +Step 3550/44000 | Loss: 3.3793 | LR: 4.95e-04 | 1284.0ms/step | 15.6 samples/s +Step 3560/44000 | Loss: 3.4957 | LR: 4.95e-04 | 1299.6ms/step | 15.4 samples/s +Step 3570/44000 | Loss: 3.4555 | LR: 4.95e-04 | 1310.4ms/step | 15.3 samples/s +Step 3580/44000 | Loss: 3.3704 | LR: 4.95e-04 | 1299.6ms/step | 15.4 samples/s +Step 3590/44000 | Loss: 3.3933 | LR: 4.95e-04 | 1310.0ms/step | 15.3 samples/s +Step 3600/44000 | Loss: 3.4588 | LR: 4.95e-04 | 1289.5ms/step | 15.5 samples/s +Step 3610/44000 | Loss: 3.3863 | LR: 4.95e-04 | 1289.8ms/step | 15.5 samples/s +Step 3620/44000 | Loss: 3.3907 | LR: 4.95e-04 | 1297.3ms/step | 15.4 samples/s +Step 3630/44000 | Loss: 3.4488 | LR: 4.95e-04 | 1286.7ms/step | 15.5 samples/s +Step 3640/44000 | Loss: 3.4449 | LR: 4.95e-04 | 1267.1ms/step | 15.8 samples/s +Step 3650/44000 | Loss: 3.4272 | LR: 4.94e-04 | 1258.3ms/step | 15.9 samples/s +Step 3660/44000 | Loss: 3.3937 | LR: 4.94e-04 | 1320.7ms/step | 15.1 samples/s +Step 3670/44000 | Loss: 3.4382 | LR: 4.94e-04 | 1268.3ms/step | 15.8 samples/s +Step 3680/44000 | Loss: 3.4146 | LR: 4.94e-04 | 1279.6ms/step | 15.6 samples/s +Step 3690/44000 | Loss: 3.4183 | LR: 4.94e-04 | 1319.6ms/step | 15.2 samples/s +Step 3700/44000 | Loss: 3.3990 | LR: 4.94e-04 | 1373.0ms/step | 14.6 samples/s +Step 3710/44000 | Loss: 3.3726 | LR: 4.94e-04 | 1395.7ms/step | 14.3 samples/s +Step 3720/44000 | Loss: 3.4235 | LR: 4.94e-04 | 1291.5ms/step | 15.5 samples/s +Step 3730/44000 | Loss: 3.3930 | LR: 4.94e-04 | 1304.7ms/step | 15.3 samples/s +Step 3740/44000 | Loss: 3.4043 | LR: 4.94e-04 | 1290.0ms/step | 15.5 samples/s +Step 3750/44000 | Loss: 3.4930 | LR: 4.94e-04 | 1314.9ms/step | 15.2 samples/s +Validation at step 3750: Loss = 3.3640, Perplexity = 28.90 +Step 3760/44000 | Loss: 3.3911 | LR: 4.94e-04 | 1769.4ms/step | 11.3 samples/s +Step 3770/44000 | Loss: 3.3782 | LR: 4.94e-04 | 1306.5ms/step | 15.3 samples/s +Step 3780/44000 | Loss: 3.4486 | LR: 4.94e-04 | 1305.1ms/step | 15.3 samples/s +Step 3790/44000 | Loss: 3.3721 | LR: 4.94e-04 | 1288.5ms/step | 15.5 samples/s +Step 3800/44000 | Loss: 3.4408 | LR: 4.94e-04 | 1292.6ms/step | 15.5 samples/s +Step 3810/44000 | Loss: 3.3489 | LR: 4.94e-04 | 1290.5ms/step | 15.5 samples/s +Step 3820/44000 | Loss: 3.3635 | LR: 4.94e-04 | 1246.4ms/step | 16.0 samples/s +Step 3830/44000 | Loss: 3.4488 | LR: 4.94e-04 | 1248.3ms/step | 16.0 samples/s +Step 3840/44000 | Loss: 3.3974 | LR: 4.94e-04 | 1245.6ms/step | 16.1 samples/s +Step 3850/44000 | Loss: 3.4524 | LR: 4.94e-04 | 1299.3ms/step | 15.4 samples/s +Step 3860/44000 | Loss: 3.3998 | LR: 4.94e-04 | 1315.2ms/step | 15.2 samples/s +Step 3870/44000 | Loss: 3.4515 | LR: 4.94e-04 | 1330.5ms/step | 15.0 samples/s +Step 3880/44000 | Loss: 3.3573 | LR: 4.94e-04 | 1351.0ms/step | 14.8 samples/s +Step 3890/44000 | Loss: 3.3580 | LR: 4.94e-04 | 1260.7ms/step | 15.9 samples/s +Step 3900/44000 | Loss: 3.4005 | LR: 4.93e-04 | 1278.4ms/step | 15.6 samples/s +Step 3910/44000 | Loss: 3.4495 | LR: 4.93e-04 | 1276.7ms/step | 15.7 samples/s +Step 3920/44000 | Loss: 3.3228 | LR: 4.93e-04 | 1241.8ms/step | 16.1 samples/s +Step 3930/44000 | Loss: 3.4085 | LR: 4.93e-04 | 1276.7ms/step | 15.7 samples/s +Step 3940/44000 | Loss: 3.3681 | LR: 4.93e-04 | 1254.1ms/step | 15.9 samples/s +Step 3950/44000 | Loss: 3.3449 | LR: 4.93e-04 | 1278.7ms/step | 15.6 samples/s +Step 3960/44000 | Loss: 3.3913 | LR: 4.93e-04 | 1278.2ms/step | 15.6 samples/s +Step 3970/44000 | Loss: 3.3639 | LR: 4.93e-04 | 1298.5ms/step | 15.4 samples/s +Step 3980/44000 | Loss: 3.3943 | LR: 4.93e-04 | 1295.8ms/step | 15.4 samples/s +Step 3990/44000 | Loss: 3.5220 | LR: 4.93e-04 | 1298.9ms/step | 15.4 samples/s +Step 4000/44000 | Loss: 3.3946 | LR: 4.93e-04 | 1292.5ms/step | 15.5 samples/s +Validation at step 4000: Loss = 3.3435, Perplexity = 28.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-4000 +Step 4010/44000 | Loss: 3.3674 | LR: 4.93e-04 | 2184.2ms/step | 9.2 samples/s +Step 4020/44000 | Loss: 3.4206 | LR: 4.93e-04 | 1297.6ms/step | 15.4 samples/s +Step 4030/44000 | Loss: 3.4372 | LR: 4.93e-04 | 1299.5ms/step | 15.4 samples/s +Step 4040/44000 | Loss: 3.3925 | LR: 4.93e-04 | 1362.4ms/step | 14.7 samples/s +Step 4050/44000 | Loss: 3.4085 | LR: 4.93e-04 | 1341.8ms/step | 14.9 samples/s +Step 4060/44000 | Loss: 3.3837 | LR: 4.93e-04 | 1268.0ms/step | 15.8 samples/s +Step 4070/44000 | Loss: 3.4308 | LR: 4.93e-04 | 1297.2ms/step | 15.4 samples/s +Step 4080/44000 | Loss: 3.3973 | LR: 4.93e-04 | 1281.0ms/step | 15.6 samples/s +Step 4090/44000 | Loss: 3.4198 | LR: 4.93e-04 | 1299.4ms/step | 15.4 samples/s +Step 4100/44000 | Loss: 3.4196 | LR: 4.93e-04 | 1283.9ms/step | 15.6 samples/s +Step 4110/44000 | Loss: 3.3732 | LR: 4.93e-04 | 1242.5ms/step | 16.1 samples/s +Step 4120/44000 | Loss: 3.3795 | LR: 4.93e-04 | 1330.4ms/step | 15.0 samples/s +Step 4130/44000 | Loss: 3.4073 | LR: 4.93e-04 | 1331.2ms/step | 15.0 samples/s +Step 4140/44000 | Loss: 3.3697 | LR: 4.92e-04 | 1295.5ms/step | 15.4 samples/s +Step 4150/44000 | Loss: 3.3459 | LR: 4.92e-04 | 1276.0ms/step | 15.7 samples/s +Step 4160/44000 | Loss: 3.4175 | LR: 4.92e-04 | 1314.7ms/step | 15.2 samples/s +Step 4170/44000 | Loss: 3.3936 | LR: 4.92e-04 | 1286.1ms/step | 15.6 samples/s +Step 4180/44000 | Loss: 3.3694 | LR: 4.92e-04 | 1310.7ms/step | 15.3 samples/s +Step 4190/44000 | Loss: 3.3877 | LR: 4.92e-04 | 1284.3ms/step | 15.6 samples/s +Step 4200/44000 | Loss: 3.4335 | LR: 4.92e-04 | 1273.9ms/step | 15.7 samples/s +Step 4210/44000 | Loss: 3.3516 | LR: 4.92e-04 | 1272.6ms/step | 15.7 samples/s +Step 4220/44000 | Loss: 3.3930 | LR: 4.92e-04 | 1442.3ms/step | 13.9 samples/s +Step 4230/44000 | Loss: 3.3740 | LR: 4.92e-04 | 1305.5ms/step | 15.3 samples/s +Step 4240/44000 | Loss: 3.3656 | LR: 4.92e-04 | 1277.4ms/step | 15.7 samples/s +Step 4250/44000 | Loss: 3.3843 | LR: 4.92e-04 | 1304.2ms/step | 15.3 samples/s +Validation at step 4250: Loss = 3.3262, Perplexity = 27.83 +Step 4260/44000 | Loss: 3.4133 | LR: 4.92e-04 | 1755.4ms/step | 11.4 samples/s +Step 4270/44000 | Loss: 3.3499 | LR: 4.92e-04 | 1315.8ms/step | 15.2 samples/s +Step 4280/44000 | Loss: 3.4241 | LR: 4.92e-04 | 1320.0ms/step | 15.2 samples/s +Step 4290/44000 | Loss: 3.4200 | LR: 4.92e-04 | 1302.9ms/step | 15.4 samples/s +Step 4300/44000 | Loss: 3.3642 | LR: 4.92e-04 | 1292.1ms/step | 15.5 samples/s +Step 4310/44000 | Loss: 3.3278 | LR: 4.92e-04 | 1288.6ms/step | 15.5 samples/s +Step 4320/44000 | Loss: 3.4016 | LR: 4.92e-04 | 1299.7ms/step | 15.4 samples/s +Step 4330/44000 | Loss: 3.3385 | LR: 4.92e-04 | 1271.1ms/step | 15.7 samples/s +Step 4340/44000 | Loss: 3.3603 | LR: 4.92e-04 | 1308.5ms/step | 15.3 samples/s +Step 4350/44000 | Loss: 3.3702 | LR: 4.92e-04 | 1315.5ms/step | 15.2 samples/s +Step 4360/44000 | Loss: 3.3952 | LR: 4.91e-04 | 1247.7ms/step | 16.0 samples/s +Step 4370/44000 | Loss: 3.3874 | LR: 4.91e-04 | 1279.1ms/step | 15.6 samples/s +Step 4380/44000 | Loss: 3.3910 | LR: 4.91e-04 | 1315.0ms/step | 15.2 samples/s +Step 4390/44000 | Loss: 3.3643 | LR: 4.91e-04 | 1332.1ms/step | 15.0 samples/s +Step 4400/44000 | Loss: 3.4049 | LR: 4.91e-04 | 1262.8ms/step | 15.8 samples/s +Step 4410/44000 | Loss: 3.3346 | LR: 4.91e-04 | 1344.4ms/step | 14.9 samples/s +Step 4420/44000 | Loss: 3.3463 | LR: 4.91e-04 | 1261.0ms/step | 15.9 samples/s +Step 4430/44000 | Loss: 3.4072 | LR: 4.91e-04 | 1368.0ms/step | 14.6 samples/s +Step 4440/44000 | Loss: 3.3134 | LR: 4.91e-04 | 1291.8ms/step | 15.5 samples/s +Step 4450/44000 | Loss: 3.3484 | LR: 4.91e-04 | 1293.4ms/step | 15.5 samples/s +Step 4460/44000 | Loss: 3.3091 | LR: 4.91e-04 | 1315.6ms/step | 15.2 samples/s +Step 4470/44000 | Loss: 3.4038 | LR: 4.91e-04 | 1232.3ms/step | 16.2 samples/s +Step 4480/44000 | Loss: 3.3695 | LR: 4.91e-04 | 1258.4ms/step | 15.9 samples/s +Step 4490/44000 | Loss: 3.3803 | LR: 4.91e-04 | 1253.1ms/step | 16.0 samples/s +Step 4500/44000 | Loss: 3.3764 | LR: 4.91e-04 | 1253.5ms/step | 16.0 samples/s +Validation at step 4500: Loss = 3.3009, Perplexity = 27.14 +Step 4510/44000 | Loss: 3.4025 | LR: 4.91e-04 | 1761.4ms/step | 11.4 samples/s +Step 4520/44000 | Loss: 3.3926 | LR: 4.91e-04 | 1270.7ms/step | 15.7 samples/s +Step 4530/44000 | Loss: 3.3561 | LR: 4.91e-04 | 1287.7ms/step | 15.5 samples/s +Step 4540/44000 | Loss: 3.3321 | LR: 4.91e-04 | 1294.9ms/step | 15.4 samples/s +Step 4550/44000 | Loss: 3.3004 | LR: 4.91e-04 | 1289.0ms/step | 15.5 samples/s +Step 4560/44000 | Loss: 3.4163 | LR: 4.90e-04 | 1404.6ms/step | 14.2 samples/s +Step 4570/44000 | Loss: 3.3836 | LR: 4.90e-04 | 1284.0ms/step | 15.6 samples/s +Step 4580/44000 | Loss: 3.3480 | LR: 4.90e-04 | 1285.7ms/step | 15.6 samples/s +Step 4590/44000 | Loss: 3.3854 | LR: 4.90e-04 | 1277.1ms/step | 15.7 samples/s +Step 4600/44000 | Loss: 3.3554 | LR: 4.90e-04 | 1281.7ms/step | 15.6 samples/s +Step 4610/44000 | Loss: 3.3685 | LR: 4.90e-04 | 1282.0ms/step | 15.6 samples/s +Step 4620/44000 | Loss: 3.3101 | LR: 4.90e-04 | 1278.1ms/step | 15.6 samples/s +Step 4630/44000 | Loss: 3.3590 | LR: 4.90e-04 | 1301.8ms/step | 15.4 samples/s +Step 4640/44000 | Loss: 3.3663 | LR: 4.90e-04 | 1319.5ms/step | 15.2 samples/s +Step 4650/44000 | Loss: 3.3097 | LR: 4.90e-04 | 1285.5ms/step | 15.6 samples/s +Step 4660/44000 | Loss: 3.2872 | LR: 4.90e-04 | 1293.6ms/step | 15.5 samples/s +Step 4670/44000 | Loss: 3.3986 | LR: 4.90e-04 | 1279.8ms/step | 15.6 samples/s +Step 4680/44000 | Loss: 3.3430 | LR: 4.90e-04 | 1305.1ms/step | 15.3 samples/s +Step 4690/44000 | Loss: 3.3655 | LR: 4.90e-04 | 1258.6ms/step | 15.9 samples/s +Step 4700/44000 | Loss: 3.3614 | LR: 4.90e-04 | 1259.1ms/step | 15.9 samples/s +Step 4710/44000 | Loss: 3.4188 | LR: 4.90e-04 | 1304.2ms/step | 15.3 samples/s +Step 4720/44000 | Loss: 3.3912 | LR: 4.90e-04 | 1272.5ms/step | 15.7 samples/s +Step 4730/44000 | Loss: 3.3323 | LR: 4.90e-04 | 1463.5ms/step | 13.7 samples/s +Step 4740/44000 | Loss: 3.3403 | LR: 4.90e-04 | 1299.8ms/step | 15.4 samples/s +Step 4750/44000 | Loss: 3.4105 | LR: 4.90e-04 | 1294.2ms/step | 15.5 samples/s +Validation at step 4750: Loss = 3.2856, Perplexity = 26.72 +Step 4760/44000 | Loss: 3.3396 | LR: 4.89e-04 | 1743.6ms/step | 11.5 samples/s +Step 4770/44000 | Loss: 3.4017 | LR: 4.89e-04 | 1216.0ms/step | 16.4 samples/s +Step 4780/44000 | Loss: 3.2899 | LR: 4.89e-04 | 1319.8ms/step | 15.2 samples/s +Step 4790/44000 | Loss: 3.3976 | LR: 4.89e-04 | 1313.3ms/step | 15.2 samples/s +Step 4800/44000 | Loss: 3.3339 | LR: 4.89e-04 | 1263.9ms/step | 15.8 samples/s +Step 4810/44000 | Loss: 3.3542 | LR: 4.89e-04 | 1305.3ms/step | 15.3 samples/s +Step 4820/44000 | Loss: 3.3766 | LR: 4.89e-04 | 1252.8ms/step | 16.0 samples/s +Step 4830/44000 | Loss: 3.3428 | LR: 4.89e-04 | 1326.9ms/step | 15.1 samples/s +Step 4840/44000 | Loss: 3.2927 | LR: 4.89e-04 | 1289.5ms/step | 15.5 samples/s +Step 4850/44000 | Loss: 3.3853 | LR: 4.89e-04 | 1295.0ms/step | 15.4 samples/s +Step 4860/44000 | Loss: 3.3424 | LR: 4.89e-04 | 1289.4ms/step | 15.5 samples/s +Step 4870/44000 | Loss: 3.3138 | LR: 4.89e-04 | 1264.3ms/step | 15.8 samples/s +Step 4880/44000 | Loss: 3.3591 | LR: 4.89e-04 | 1286.3ms/step | 15.5 samples/s +Step 4890/44000 | Loss: 3.3579 | LR: 4.89e-04 | 1303.8ms/step | 15.3 samples/s +Step 4900/44000 | Loss: 3.3438 | LR: 4.89e-04 | 1468.1ms/step | 13.6 samples/s +Step 4910/44000 | Loss: 3.3552 | LR: 4.89e-04 | 1263.2ms/step | 15.8 samples/s +Step 4920/44000 | Loss: 3.2759 | LR: 4.89e-04 | 1263.5ms/step | 15.8 samples/s +Step 4930/44000 | Loss: 3.2669 | LR: 4.89e-04 | 1289.8ms/step | 15.5 samples/s +Step 4940/44000 | Loss: 3.3758 | LR: 4.89e-04 | 1300.0ms/step | 15.4 samples/s +Step 4950/44000 | Loss: 3.3135 | LR: 4.88e-04 | 1273.6ms/step | 15.7 samples/s +Step 4960/44000 | Loss: 3.3794 | LR: 4.88e-04 | 1289.6ms/step | 15.5 samples/s +Step 4970/44000 | Loss: 3.3087 | LR: 4.88e-04 | 1302.7ms/step | 15.4 samples/s +Step 4980/44000 | Loss: 3.3623 | LR: 4.88e-04 | 1324.3ms/step | 15.1 samples/s +Step 4990/44000 | Loss: 3.3320 | LR: 4.88e-04 | 1249.9ms/step | 16.0 samples/s +Step 5000/44000 | Loss: 3.3174 | LR: 4.88e-04 | 1248.5ms/step | 16.0 samples/s +Validation at step 5000: Loss = 3.2688, Perplexity = 26.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-5000 +Step 5010/44000 | Loss: 3.2821 | LR: 4.88e-04 | 2236.1ms/step | 8.9 samples/s +Step 5020/44000 | Loss: 3.2884 | LR: 4.88e-04 | 1367.9ms/step | 14.6 samples/s +Step 5030/44000 | Loss: 3.3309 | LR: 4.88e-04 | 1270.9ms/step | 15.7 samples/s +Step 5040/44000 | Loss: 3.3371 | LR: 4.88e-04 | 1308.3ms/step | 15.3 samples/s +Step 5050/44000 | Loss: 3.2988 | LR: 4.88e-04 | 1294.8ms/step | 15.4 samples/s +Step 5060/44000 | Loss: 3.3228 | LR: 4.88e-04 | 1351.5ms/step | 14.8 samples/s +Step 5070/44000 | Loss: 3.2764 | LR: 4.88e-04 | 1361.3ms/step | 14.7 samples/s +Step 5080/44000 | Loss: 3.3487 | LR: 4.88e-04 | 1286.1ms/step | 15.6 samples/s +Step 5090/44000 | Loss: 3.2780 | LR: 4.88e-04 | 1346.7ms/step | 14.9 samples/s +Step 5100/44000 | Loss: 3.3048 | LR: 4.88e-04 | 1268.9ms/step | 15.8 samples/s +Step 5110/44000 | Loss: 3.2566 | LR: 4.88e-04 | 1326.0ms/step | 15.1 samples/s +Step 5120/44000 | Loss: 3.3124 | LR: 4.88e-04 | 1274.5ms/step | 15.7 samples/s +Step 5130/44000 | Loss: 3.3489 | LR: 4.87e-04 | 1314.9ms/step | 15.2 samples/s +Step 5140/44000 | Loss: 3.3296 | LR: 4.87e-04 | 1305.8ms/step | 15.3 samples/s +Step 5150/44000 | Loss: 3.3979 | LR: 4.87e-04 | 1247.5ms/step | 16.0 samples/s +Step 5160/44000 | Loss: 3.3011 | LR: 4.87e-04 | 1284.0ms/step | 15.6 samples/s +Step 5170/44000 | Loss: 3.2452 | LR: 4.87e-04 | 1347.1ms/step | 14.8 samples/s +Step 5180/44000 | Loss: 3.3278 | LR: 4.87e-04 | 1289.5ms/step | 15.5 samples/s +Step 5190/44000 | Loss: 3.3320 | LR: 4.87e-04 | 1212.3ms/step | 16.5 samples/s +Step 5200/44000 | Loss: 3.3127 | LR: 4.87e-04 | 1289.3ms/step | 15.5 samples/s +Step 5210/44000 | Loss: 3.2589 | LR: 4.87e-04 | 1304.2ms/step | 15.3 samples/s +Step 5220/44000 | Loss: 3.3993 | LR: 4.87e-04 | 1303.7ms/step | 15.3 samples/s +Step 5230/44000 | Loss: 3.2919 | LR: 4.87e-04 | 1266.0ms/step | 15.8 samples/s +Step 5240/44000 | Loss: 3.3413 | LR: 4.87e-04 | 1436.0ms/step | 13.9 samples/s +Step 5250/44000 | Loss: 3.2819 | LR: 4.87e-04 | 1305.0ms/step | 15.3 samples/s +Validation at step 5250: Loss = 3.2576, Perplexity = 25.99 +Step 5260/44000 | Loss: 3.3647 | LR: 4.87e-04 | 1762.3ms/step | 11.3 samples/s +Step 5270/44000 | Loss: 3.3108 | LR: 4.87e-04 | 1289.1ms/step | 15.5 samples/s +Step 5280/44000 | Loss: 3.2958 | LR: 4.87e-04 | 1337.4ms/step | 15.0 samples/s +Step 5290/44000 | Loss: 3.3140 | LR: 4.87e-04 | 1276.1ms/step | 15.7 samples/s +Step 5300/44000 | Loss: 3.3509 | LR: 4.86e-04 | 1302.6ms/step | 15.4 samples/s +Step 5310/44000 | Loss: 3.3452 | LR: 4.86e-04 | 1306.6ms/step | 15.3 samples/s +Step 5320/44000 | Loss: 3.3905 | LR: 4.86e-04 | 1325.0ms/step | 15.1 samples/s +Step 5330/44000 | Loss: 3.2965 | LR: 4.86e-04 | 1299.9ms/step | 15.4 samples/s +Step 5340/44000 | Loss: 3.2796 | LR: 4.86e-04 | 1257.9ms/step | 15.9 samples/s +Step 5350/44000 | Loss: 3.3707 | LR: 4.86e-04 | 1272.9ms/step | 15.7 samples/s +Step 5360/44000 | Loss: 3.2833 | LR: 4.86e-04 | 1259.2ms/step | 15.9 samples/s +Step 5370/44000 | Loss: 3.2841 | LR: 4.86e-04 | 1299.9ms/step | 15.4 samples/s +Step 5380/44000 | Loss: 3.3168 | LR: 4.86e-04 | 1300.0ms/step | 15.4 samples/s +Step 5390/44000 | Loss: 3.2739 | LR: 4.86e-04 | 1237.2ms/step | 16.2 samples/s +Step 5400/44000 | Loss: 3.3493 | LR: 4.86e-04 | 1316.2ms/step | 15.2 samples/s +Step 5410/44000 | Loss: 3.2611 | LR: 4.86e-04 | 1405.2ms/step | 14.2 samples/s +Step 5420/44000 | Loss: 3.3759 | LR: 4.86e-04 | 1284.3ms/step | 15.6 samples/s +Step 5430/44000 | Loss: 3.2508 | LR: 4.86e-04 | 1271.7ms/step | 15.7 samples/s +Step 5440/44000 | Loss: 3.3112 | LR: 4.86e-04 | 1255.1ms/step | 15.9 samples/s +Step 5450/44000 | Loss: 3.3075 | LR: 4.86e-04 | 1268.0ms/step | 15.8 samples/s +Step 5460/44000 | Loss: 3.3548 | LR: 4.86e-04 | 1304.0ms/step | 15.3 samples/s +Step 5470/44000 | Loss: 3.3403 | LR: 4.85e-04 | 1260.2ms/step | 15.9 samples/s +Step 5480/44000 | Loss: 3.2894 | LR: 4.85e-04 | 1321.0ms/step | 15.1 samples/s +Step 5490/44000 | Loss: 3.3015 | LR: 4.85e-04 | 1279.1ms/step | 15.6 samples/s +Step 5500/44000 | Loss: 3.2797 | LR: 4.85e-04 | 1299.4ms/step | 15.4 samples/s +Validation at step 5500: Loss = 3.2434, Perplexity = 25.62 +Step 5510/44000 | Loss: 3.2534 | LR: 4.85e-04 | 1797.7ms/step | 11.1 samples/s +Step 5520/44000 | Loss: 3.2980 | LR: 4.85e-04 | 1305.6ms/step | 15.3 samples/s +Step 5530/44000 | Loss: 3.3100 | LR: 4.85e-04 | 1278.7ms/step | 15.6 samples/s +Step 5540/44000 | Loss: 3.2723 | LR: 4.85e-04 | 1289.4ms/step | 15.5 samples/s +Step 5550/44000 | Loss: 3.2751 | LR: 4.85e-04 | 1263.5ms/step | 15.8 samples/s +Step 5560/44000 | Loss: 3.2761 | LR: 4.85e-04 | 1241.7ms/step | 16.1 samples/s +Step 5570/44000 | Loss: 3.2350 | LR: 4.85e-04 | 1247.8ms/step | 16.0 samples/s +Step 5580/44000 | Loss: 3.3289 | LR: 4.85e-04 | 1425.7ms/step | 14.0 samples/s +Step 5590/44000 | Loss: 3.3133 | LR: 4.85e-04 | 1278.1ms/step | 15.6 samples/s +Step 5600/44000 | Loss: 3.3276 | LR: 4.85e-04 | 1248.9ms/step | 16.0 samples/s +Step 5610/44000 | Loss: 3.2782 | LR: 4.85e-04 | 1268.3ms/step | 15.8 samples/s +Step 5620/44000 | Loss: 3.3392 | LR: 4.85e-04 | 1263.5ms/step | 15.8 samples/s +Step 5630/44000 | Loss: 3.2366 | LR: 4.84e-04 | 1258.0ms/step | 15.9 samples/s +Step 5640/44000 | Loss: 3.3100 | LR: 4.84e-04 | 1300.4ms/step | 15.4 samples/s +Step 5650/44000 | Loss: 3.2692 | LR: 4.84e-04 | 1305.3ms/step | 15.3 samples/s +Step 5660/44000 | Loss: 3.2942 | LR: 4.84e-04 | 1304.9ms/step | 15.3 samples/s +Step 5670/44000 | Loss: 3.3147 | LR: 4.84e-04 | 1273.9ms/step | 15.7 samples/s +Step 5680/44000 | Loss: 3.3153 | LR: 4.84e-04 | 1319.9ms/step | 15.2 samples/s +Step 5690/44000 | Loss: 3.3264 | LR: 4.84e-04 | 1294.9ms/step | 15.4 samples/s +Step 5700/44000 | Loss: 3.2829 | LR: 4.84e-04 | 1279.0ms/step | 15.6 samples/s +Step 5710/44000 | Loss: 3.2407 | LR: 4.84e-04 | 1253.2ms/step | 16.0 samples/s +Step 5720/44000 | Loss: 3.2287 | LR: 4.84e-04 | 1249.3ms/step | 16.0 samples/s +Step 5730/44000 | Loss: 3.2280 | LR: 4.84e-04 | 1302.1ms/step | 15.4 samples/s +Step 5740/44000 | Loss: 3.2913 | LR: 4.84e-04 | 1261.1ms/step | 15.9 samples/s +Step 5750/44000 | Loss: 3.3112 | LR: 4.84e-04 | 1338.6ms/step | 14.9 samples/s +Validation at step 5750: Loss = 3.2312, Perplexity = 25.31 +Step 5760/44000 | Loss: 3.2909 | LR: 4.84e-04 | 1834.4ms/step | 10.9 samples/s +Step 5770/44000 | Loss: 3.2498 | LR: 4.84e-04 | 1351.3ms/step | 14.8 samples/s +Step 5780/44000 | Loss: 3.2948 | LR: 4.83e-04 | 1274.1ms/step | 15.7 samples/s +Step 5790/44000 | Loss: 3.2582 | LR: 4.83e-04 | 1257.8ms/step | 15.9 samples/s +Step 5800/44000 | Loss: 3.3966 | LR: 4.83e-04 | 1306.9ms/step | 15.3 samples/s +Step 5810/44000 | Loss: 3.2958 | LR: 4.83e-04 | 1292.1ms/step | 15.5 samples/s +Step 5820/44000 | Loss: 3.2586 | LR: 4.83e-04 | 1252.0ms/step | 16.0 samples/s +Step 5830/44000 | Loss: 3.3219 | LR: 4.83e-04 | 1291.1ms/step | 15.5 samples/s +Step 5840/44000 | Loss: 3.3322 | LR: 4.83e-04 | 1280.6ms/step | 15.6 samples/s +Step 5850/44000 | Loss: 3.2550 | LR: 4.83e-04 | 1291.5ms/step | 15.5 samples/s +Step 5860/44000 | Loss: 3.2861 | LR: 4.83e-04 | 1281.3ms/step | 15.6 samples/s +Step 5870/44000 | Loss: 3.3206 | LR: 4.83e-04 | 1295.5ms/step | 15.4 samples/s +Step 5880/44000 | Loss: 3.2682 | LR: 4.83e-04 | 1295.6ms/step | 15.4 samples/s +Step 5890/44000 | Loss: 3.2662 | LR: 4.83e-04 | 1253.7ms/step | 16.0 samples/s +Step 5900/44000 | Loss: 3.2788 | LR: 4.83e-04 | 1247.0ms/step | 16.0 samples/s +Step 5910/44000 | Loss: 3.2368 | LR: 4.83e-04 | 1289.3ms/step | 15.5 samples/s +Step 5920/44000 | Loss: 3.2580 | LR: 4.83e-04 | 1320.7ms/step | 15.1 samples/s +Step 5930/44000 | Loss: 3.3268 | LR: 4.83e-04 | 1368.2ms/step | 14.6 samples/s +Step 5940/44000 | Loss: 3.2566 | LR: 4.82e-04 | 1247.5ms/step | 16.0 samples/s +Step 5950/44000 | Loss: 3.2901 | LR: 4.82e-04 | 1284.2ms/step | 15.6 samples/s +Step 5960/44000 | Loss: 3.2819 | LR: 4.82e-04 | 1294.8ms/step | 15.4 samples/s +Step 5970/44000 | Loss: 3.2856 | LR: 4.82e-04 | 1284.1ms/step | 15.6 samples/s +Step 5980/44000 | Loss: 3.3225 | LR: 4.82e-04 | 1263.4ms/step | 15.8 samples/s +Step 5990/44000 | Loss: 3.3374 | LR: 4.82e-04 | 1268.6ms/step | 15.8 samples/s +Step 6000/44000 | Loss: 3.2073 | LR: 4.82e-04 | 1257.9ms/step | 15.9 samples/s +Validation at step 6000: Loss = 3.2172, Perplexity = 24.96 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-6000 +Step 6010/44000 | Loss: 3.2785 | LR: 4.82e-04 | 2223.3ms/step | 9.0 samples/s +Step 6020/44000 | Loss: 3.2930 | LR: 4.82e-04 | 1268.8ms/step | 15.8 samples/s +Step 6030/44000 | Loss: 3.2664 | LR: 4.82e-04 | 1285.5ms/step | 15.6 samples/s +Step 6040/44000 | Loss: 3.2959 | LR: 4.82e-04 | 1304.2ms/step | 15.3 samples/s +Step 6050/44000 | Loss: 3.2999 | LR: 4.82e-04 | 1299.5ms/step | 15.4 samples/s +Step 6060/44000 | Loss: 3.2881 | LR: 4.82e-04 | 1299.8ms/step | 15.4 samples/s +Step 6070/44000 | Loss: 3.3648 | LR: 4.82e-04 | 1263.3ms/step | 15.8 samples/s +Step 6080/44000 | Loss: 3.3315 | LR: 4.81e-04 | 1281.8ms/step | 15.6 samples/s +Step 6090/44000 | Loss: 3.2095 | LR: 4.81e-04 | 1401.5ms/step | 14.3 samples/s +Step 6100/44000 | Loss: 3.2510 | LR: 4.81e-04 | 1321.8ms/step | 15.1 samples/s +Step 6110/44000 | Loss: 3.2804 | LR: 4.81e-04 | 1329.2ms/step | 15.0 samples/s +Step 6120/44000 | Loss: 3.2993 | LR: 4.81e-04 | 1304.8ms/step | 15.3 samples/s +Step 6130/44000 | Loss: 3.2664 | LR: 4.81e-04 | 1261.4ms/step | 15.9 samples/s +Step 6140/44000 | Loss: 3.2612 | LR: 4.81e-04 | 1305.7ms/step | 15.3 samples/s +Step 6150/44000 | Loss: 3.2698 | LR: 4.81e-04 | 1275.7ms/step | 15.7 samples/s +Step 6160/44000 | Loss: 3.2804 | LR: 4.81e-04 | 1260.1ms/step | 15.9 samples/s +Step 6170/44000 | Loss: 3.3216 | LR: 4.81e-04 | 1317.6ms/step | 15.2 samples/s +Step 6180/44000 | Loss: 3.3158 | LR: 4.81e-04 | 1317.4ms/step | 15.2 samples/s +Step 6190/44000 | Loss: 3.2889 | LR: 4.81e-04 | 1277.0ms/step | 15.7 samples/s +Step 6200/44000 | Loss: 3.2272 | LR: 4.81e-04 | 1279.4ms/step | 15.6 samples/s +Step 6210/44000 | Loss: 3.2440 | LR: 4.81e-04 | 1274.2ms/step | 15.7 samples/s +Step 6220/44000 | Loss: 3.2653 | LR: 4.81e-04 | 1257.6ms/step | 15.9 samples/s +Step 6230/44000 | Loss: 3.2066 | LR: 4.80e-04 | 1294.6ms/step | 15.4 samples/s +Step 6240/44000 | Loss: 3.2863 | LR: 4.80e-04 | 1296.5ms/step | 15.4 samples/s +Step 6250/44000 | Loss: 3.3199 | LR: 4.80e-04 | 1282.2ms/step | 15.6 samples/s +Validation at step 6250: Loss = 3.2082, Perplexity = 24.73 +Step 6260/44000 | Loss: 3.3121 | LR: 4.80e-04 | 1876.2ms/step | 10.7 samples/s +Step 6270/44000 | Loss: 3.2712 | LR: 4.80e-04 | 1305.2ms/step | 15.3 samples/s +Step 6280/44000 | Loss: 3.2876 | LR: 4.80e-04 | 1351.9ms/step | 14.8 samples/s +Step 6290/44000 | Loss: 3.2771 | LR: 4.80e-04 | 1283.7ms/step | 15.6 samples/s +Step 6300/44000 | Loss: 3.3064 | LR: 4.80e-04 | 1277.4ms/step | 15.7 samples/s +Step 6310/44000 | Loss: 3.1994 | LR: 4.80e-04 | 1286.2ms/step | 15.5 samples/s +Step 6320/44000 | Loss: 3.3180 | LR: 4.80e-04 | 1291.7ms/step | 15.5 samples/s +Step 6330/44000 | Loss: 3.2699 | LR: 4.80e-04 | 1265.0ms/step | 15.8 samples/s +Step 6340/44000 | Loss: 3.2295 | LR: 4.80e-04 | 1247.0ms/step | 16.0 samples/s +Step 6350/44000 | Loss: 3.2570 | LR: 4.80e-04 | 1279.2ms/step | 15.6 samples/s +Step 6360/44000 | Loss: 3.2660 | LR: 4.80e-04 | 1330.6ms/step | 15.0 samples/s +Step 6370/44000 | Loss: 3.2284 | LR: 4.79e-04 | 1299.2ms/step | 15.4 samples/s +Step 6380/44000 | Loss: 3.3279 | LR: 4.79e-04 | 1314.8ms/step | 15.2 samples/s +Step 6390/44000 | Loss: 3.2735 | LR: 4.79e-04 | 1252.1ms/step | 16.0 samples/s +Step 6400/44000 | Loss: 3.3038 | LR: 4.79e-04 | 1308.7ms/step | 15.3 samples/s +Step 6410/44000 | Loss: 3.2148 | LR: 4.79e-04 | 1295.1ms/step | 15.4 samples/s +Step 6420/44000 | Loss: 3.2682 | LR: 4.79e-04 | 1309.7ms/step | 15.3 samples/s +Step 6430/44000 | Loss: 3.2401 | LR: 4.79e-04 | 1320.0ms/step | 15.2 samples/s +Step 6440/44000 | Loss: 3.2796 | LR: 4.79e-04 | 1357.5ms/step | 14.7 samples/s +Step 6450/44000 | Loss: 3.2478 | LR: 4.79e-04 | 1312.4ms/step | 15.2 samples/s +Step 6460/44000 | Loss: 3.2740 | LR: 4.79e-04 | 1328.0ms/step | 15.1 samples/s +Step 6470/44000 | Loss: 3.2031 | LR: 4.79e-04 | 1289.4ms/step | 15.5 samples/s +Step 6480/44000 | Loss: 3.2655 | LR: 4.79e-04 | 1310.3ms/step | 15.3 samples/s +Step 6490/44000 | Loss: 3.2072 | LR: 4.79e-04 | 1299.2ms/step | 15.4 samples/s +Step 6500/44000 | Loss: 3.2688 | LR: 4.79e-04 | 1304.4ms/step | 15.3 samples/s +Validation at step 6500: Loss = 3.1969, Perplexity = 24.46 +Step 6510/44000 | Loss: 3.2235 | LR: 4.78e-04 | 1765.6ms/step | 11.3 samples/s +Step 6520/44000 | Loss: 3.2442 | LR: 4.78e-04 | 1294.9ms/step | 15.4 samples/s +Step 6530/44000 | Loss: 3.2537 | LR: 4.78e-04 | 1308.3ms/step | 15.3 samples/s +Step 6540/44000 | Loss: 3.3227 | LR: 4.78e-04 | 1269.7ms/step | 15.8 samples/s +Step 6550/44000 | Loss: 3.2300 | LR: 4.78e-04 | 1283.5ms/step | 15.6 samples/s +Step 6560/44000 | Loss: 3.2561 | LR: 4.78e-04 | 1347.2ms/step | 14.8 samples/s +Step 6570/44000 | Loss: 3.2478 | LR: 4.78e-04 | 1258.2ms/step | 15.9 samples/s +Step 6580/44000 | Loss: 3.2305 | LR: 4.78e-04 | 1278.9ms/step | 15.6 samples/s +Step 6590/44000 | Loss: 3.2559 | LR: 4.78e-04 | 1279.1ms/step | 15.6 samples/s +Step 6600/44000 | Loss: 3.3068 | LR: 4.78e-04 | 1367.8ms/step | 14.6 samples/s +Step 6610/44000 | Loss: 3.2047 | LR: 4.78e-04 | 1341.9ms/step | 14.9 samples/s +Step 6620/44000 | Loss: 3.3134 | LR: 4.78e-04 | 1275.5ms/step | 15.7 samples/s +Step 6630/44000 | Loss: 3.3018 | LR: 4.78e-04 | 1255.7ms/step | 15.9 samples/s +Step 6640/44000 | Loss: 3.2853 | LR: 4.77e-04 | 1284.1ms/step | 15.6 samples/s +Step 6650/44000 | Loss: 3.1214 | LR: 4.77e-04 | 1257.9ms/step | 15.9 samples/s +Step 6660/44000 | Loss: 3.2869 | LR: 4.77e-04 | 1296.7ms/step | 15.4 samples/s +Step 6670/44000 | Loss: 3.2800 | LR: 4.77e-04 | 1318.7ms/step | 15.2 samples/s +Step 6680/44000 | Loss: 3.2241 | LR: 4.77e-04 | 1309.7ms/step | 15.3 samples/s +Step 6690/44000 | Loss: 3.2473 | LR: 4.77e-04 | 1279.0ms/step | 15.6 samples/s +Step 6700/44000 | Loss: 3.2892 | LR: 4.77e-04 | 1310.0ms/step | 15.3 samples/s +Step 6710/44000 | Loss: 3.3061 | LR: 4.77e-04 | 1294.8ms/step | 15.4 samples/s +Step 6720/44000 | Loss: 3.2503 | LR: 4.77e-04 | 1236.3ms/step | 16.2 samples/s +Step 6730/44000 | Loss: 3.2538 | LR: 4.77e-04 | 1279.9ms/step | 15.6 samples/s +Step 6740/44000 | Loss: 3.2414 | LR: 4.77e-04 | 1241.3ms/step | 16.1 samples/s +Step 6750/44000 | Loss: 3.1939 | LR: 4.77e-04 | 1290.4ms/step | 15.5 samples/s +Validation at step 6750: Loss = 3.1874, Perplexity = 24.22 +Step 6760/44000 | Loss: 3.3170 | LR: 4.77e-04 | 1759.7ms/step | 11.4 samples/s +Step 6770/44000 | Loss: 3.3066 | LR: 4.76e-04 | 1330.5ms/step | 15.0 samples/s +Step 6780/44000 | Loss: 3.2473 | LR: 4.76e-04 | 1368.3ms/step | 14.6 samples/s +Step 6790/44000 | Loss: 3.2463 | LR: 4.76e-04 | 1257.9ms/step | 15.9 samples/s +Step 6800/44000 | Loss: 3.1675 | LR: 4.76e-04 | 1278.3ms/step | 15.6 samples/s +Step 6810/44000 | Loss: 3.2019 | LR: 4.76e-04 | 1263.4ms/step | 15.8 samples/s +Step 6820/44000 | Loss: 3.2206 | LR: 4.76e-04 | 1299.0ms/step | 15.4 samples/s +Step 6830/44000 | Loss: 3.3211 | LR: 4.76e-04 | 1273.5ms/step | 15.7 samples/s +Step 6840/44000 | Loss: 3.2349 | LR: 4.76e-04 | 1289.3ms/step | 15.5 samples/s +Step 6850/44000 | Loss: 3.2133 | LR: 4.76e-04 | 1278.8ms/step | 15.6 samples/s +Step 6860/44000 | Loss: 3.1275 | LR: 4.76e-04 | 1315.9ms/step | 15.2 samples/s +Step 6870/44000 | Loss: 3.2359 | LR: 4.76e-04 | 1315.8ms/step | 15.2 samples/s +Step 6880/44000 | Loss: 3.2576 | LR: 4.76e-04 | 1294.8ms/step | 15.4 samples/s +Step 6890/44000 | Loss: 3.2885 | LR: 4.76e-04 | 1326.0ms/step | 15.1 samples/s +Step 6900/44000 | Loss: 3.2146 | LR: 4.75e-04 | 1260.9ms/step | 15.9 samples/s +Step 6910/44000 | Loss: 3.1745 | LR: 4.75e-04 | 1326.0ms/step | 15.1 samples/s +Step 6920/44000 | Loss: 3.2028 | LR: 4.75e-04 | 1250.1ms/step | 16.0 samples/s +Step 6930/44000 | Loss: 3.2268 | LR: 4.75e-04 | 1242.7ms/step | 16.1 samples/s +Step 6940/44000 | Loss: 3.2003 | LR: 4.75e-04 | 1271.2ms/step | 15.7 samples/s +Step 6950/44000 | Loss: 3.2246 | LR: 4.75e-04 | 1427.0ms/step | 14.0 samples/s +Step 6960/44000 | Loss: 3.2340 | LR: 4.75e-04 | 1310.6ms/step | 15.3 samples/s +Step 6970/44000 | Loss: 3.1740 | LR: 4.75e-04 | 1289.0ms/step | 15.5 samples/s +Step 6980/44000 | Loss: 3.2520 | LR: 4.75e-04 | 1216.0ms/step | 16.4 samples/s +Step 6990/44000 | Loss: 3.2288 | LR: 4.75e-04 | 1268.8ms/step | 15.8 samples/s +Step 7000/44000 | Loss: 3.3091 | LR: 4.75e-04 | 1247.3ms/step | 16.0 samples/s +Validation at step 7000: Loss = 3.1797, Perplexity = 24.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-7000 +Step 7010/44000 | Loss: 3.2718 | LR: 4.75e-04 | 2217.4ms/step | 9.0 samples/s +Step 7020/44000 | Loss: 3.2495 | LR: 4.75e-04 | 1289.7ms/step | 15.5 samples/s +Step 7030/44000 | Loss: 3.2337 | LR: 4.74e-04 | 1289.4ms/step | 15.5 samples/s +Step 7040/44000 | Loss: 3.3202 | LR: 4.74e-04 | 1290.3ms/step | 15.5 samples/s +Step 7050/44000 | Loss: 3.2127 | LR: 4.74e-04 | 1246.5ms/step | 16.0 samples/s +Step 7060/44000 | Loss: 3.2522 | LR: 4.74e-04 | 1284.6ms/step | 15.6 samples/s +Step 7070/44000 | Loss: 3.2978 | LR: 4.74e-04 | 1310.4ms/step | 15.3 samples/s +Step 7080/44000 | Loss: 3.2196 | LR: 4.74e-04 | 1284.5ms/step | 15.6 samples/s +Step 7090/44000 | Loss: 3.2371 | LR: 4.74e-04 | 1296.5ms/step | 15.4 samples/s +Step 7100/44000 | Loss: 3.2281 | LR: 4.74e-04 | 1262.0ms/step | 15.8 samples/s +Step 7110/44000 | Loss: 3.1476 | LR: 4.74e-04 | 1252.4ms/step | 16.0 samples/s +Step 7120/44000 | Loss: 3.2575 | LR: 4.74e-04 | 1404.2ms/step | 14.2 samples/s +Step 7130/44000 | Loss: 3.1440 | LR: 4.74e-04 | 1247.3ms/step | 16.0 samples/s +Step 7140/44000 | Loss: 3.2440 | LR: 4.74e-04 | 1344.8ms/step | 14.9 samples/s +Step 7150/44000 | Loss: 3.0877 | LR: 4.73e-04 | 1327.9ms/step | 15.1 samples/s +Step 7160/44000 | Loss: 3.2769 | LR: 4.73e-04 | 1342.6ms/step | 14.9 samples/s +Step 7170/44000 | Loss: 3.2786 | LR: 4.73e-04 | 1283.8ms/step | 15.6 samples/s +Step 7180/44000 | Loss: 3.2677 | LR: 4.73e-04 | 1294.8ms/step | 15.4 samples/s +Step 7190/44000 | Loss: 3.1726 | LR: 4.73e-04 | 1300.3ms/step | 15.4 samples/s +Step 7200/44000 | Loss: 3.1991 | LR: 4.73e-04 | 1296.6ms/step | 15.4 samples/s +Step 7210/44000 | Loss: 3.1912 | LR: 4.73e-04 | 1303.9ms/step | 15.3 samples/s +Step 7220/44000 | Loss: 3.1945 | LR: 4.73e-04 | 1310.7ms/step | 15.3 samples/s +Step 7230/44000 | Loss: 3.2610 | LR: 4.73e-04 | 1321.2ms/step | 15.1 samples/s +Step 7240/44000 | Loss: 3.2313 | LR: 4.73e-04 | 1261.2ms/step | 15.9 samples/s +Step 7250/44000 | Loss: 3.2138 | LR: 4.73e-04 | 1292.3ms/step | 15.5 samples/s +Validation at step 7250: Loss = 3.1732, Perplexity = 23.88 +Step 7260/44000 | Loss: 3.2425 | LR: 4.73e-04 | 1740.7ms/step | 11.5 samples/s +Step 7270/44000 | Loss: 3.1903 | LR: 4.72e-04 | 1315.8ms/step | 15.2 samples/s +Step 7280/44000 | Loss: 3.1633 | LR: 4.72e-04 | 1278.9ms/step | 15.6 samples/s +Step 7290/44000 | Loss: 3.2522 | LR: 4.72e-04 | 1399.8ms/step | 14.3 samples/s +Step 7300/44000 | Loss: 3.2148 | LR: 4.72e-04 | 1280.9ms/step | 15.6 samples/s +Step 7310/44000 | Loss: 3.2008 | LR: 4.72e-04 | 1308.9ms/step | 15.3 samples/s +Step 7320/44000 | Loss: 3.2578 | LR: 4.72e-04 | 1315.5ms/step | 15.2 samples/s +Step 7330/44000 | Loss: 3.2379 | LR: 4.72e-04 | 1320.7ms/step | 15.1 samples/s +Step 7340/44000 | Loss: 3.1768 | LR: 4.72e-04 | 1253.5ms/step | 16.0 samples/s +Step 7350/44000 | Loss: 3.2220 | LR: 4.72e-04 | 1268.3ms/step | 15.8 samples/s +Step 7360/44000 | Loss: 3.2145 | LR: 4.72e-04 | 1284.1ms/step | 15.6 samples/s +Step 7370/44000 | Loss: 3.1790 | LR: 4.72e-04 | 1268.4ms/step | 15.8 samples/s +Step 7380/44000 | Loss: 3.2689 | LR: 4.72e-04 | 1279.1ms/step | 15.6 samples/s +Step 7390/44000 | Loss: 3.1758 | LR: 4.71e-04 | 1247.6ms/step | 16.0 samples/s +Step 7400/44000 | Loss: 3.1589 | LR: 4.71e-04 | 1300.3ms/step | 15.4 samples/s +Step 7410/44000 | Loss: 3.2415 | LR: 4.71e-04 | 1290.2ms/step | 15.5 samples/s +Step 7420/44000 | Loss: 3.2289 | LR: 4.71e-04 | 1273.2ms/step | 15.7 samples/s +Step 7430/44000 | Loss: 3.2365 | LR: 4.71e-04 | 1272.3ms/step | 15.7 samples/s +Step 7440/44000 | Loss: 3.2505 | LR: 4.71e-04 | 1317.1ms/step | 15.2 samples/s +Step 7450/44000 | Loss: 3.2593 | LR: 4.71e-04 | 1316.1ms/step | 15.2 samples/s +Step 7460/44000 | Loss: 3.2635 | LR: 4.71e-04 | 1465.0ms/step | 13.7 samples/s +Step 7470/44000 | Loss: 3.2615 | LR: 4.71e-04 | 1275.5ms/step | 15.7 samples/s +Step 7480/44000 | Loss: 3.1961 | LR: 4.71e-04 | 1310.4ms/step | 15.3 samples/s +Step 7490/44000 | Loss: 3.2649 | LR: 4.71e-04 | 1262.9ms/step | 15.8 samples/s +Step 7500/44000 | Loss: 3.1714 | LR: 4.71e-04 | 1309.9ms/step | 15.3 samples/s +Validation at step 7500: Loss = 3.1644, Perplexity = 23.68 +Step 7510/44000 | Loss: 3.1602 | LR: 4.70e-04 | 1742.7ms/step | 11.5 samples/s +Step 7520/44000 | Loss: 3.2058 | LR: 4.70e-04 | 1292.3ms/step | 15.5 samples/s +Step 7530/44000 | Loss: 3.1844 | LR: 4.70e-04 | 1247.3ms/step | 16.0 samples/s +Step 7540/44000 | Loss: 3.2522 | LR: 4.70e-04 | 1327.0ms/step | 15.1 samples/s +Step 7550/44000 | Loss: 3.1469 | LR: 4.70e-04 | 1272.1ms/step | 15.7 samples/s +Step 7560/44000 | Loss: 3.2196 | LR: 4.70e-04 | 1336.5ms/step | 15.0 samples/s +Step 7570/44000 | Loss: 3.2460 | LR: 4.70e-04 | 1294.5ms/step | 15.4 samples/s +Step 7580/44000 | Loss: 3.2577 | LR: 4.70e-04 | 1289.2ms/step | 15.5 samples/s +Step 7590/44000 | Loss: 3.3012 | LR: 4.70e-04 | 1263.0ms/step | 15.8 samples/s +Step 7600/44000 | Loss: 3.2173 | LR: 4.70e-04 | 1269.7ms/step | 15.8 samples/s +Step 7610/44000 | Loss: 3.2132 | LR: 4.70e-04 | 1282.7ms/step | 15.6 samples/s +Step 7620/44000 | Loss: 3.2450 | LR: 4.70e-04 | 1292.5ms/step | 15.5 samples/s +Step 7630/44000 | Loss: 3.1395 | LR: 4.69e-04 | 1417.4ms/step | 14.1 samples/s +Step 7640/44000 | Loss: 3.1545 | LR: 4.69e-04 | 1278.4ms/step | 15.6 samples/s +Step 7650/44000 | Loss: 3.2228 | LR: 4.69e-04 | 1300.0ms/step | 15.4 samples/s +Step 7660/44000 | Loss: 3.2332 | LR: 4.69e-04 | 1273.7ms/step | 15.7 samples/s +Step 7670/44000 | Loss: 3.1085 | LR: 4.69e-04 | 1260.5ms/step | 15.9 samples/s +Step 7680/44000 | Loss: 3.1611 | LR: 4.69e-04 | 1233.9ms/step | 16.2 samples/s +Step 7690/44000 | Loss: 3.2674 | LR: 4.69e-04 | 1268.2ms/step | 15.8 samples/s +Step 7700/44000 | Loss: 3.2236 | LR: 4.69e-04 | 1294.4ms/step | 15.5 samples/s +Step 7710/44000 | Loss: 3.1883 | LR: 4.69e-04 | 1265.2ms/step | 15.8 samples/s +Step 7720/44000 | Loss: 3.2142 | LR: 4.69e-04 | 1279.2ms/step | 15.6 samples/s +Step 7730/44000 | Loss: 3.2660 | LR: 4.69e-04 | 1277.9ms/step | 15.7 samples/s +Step 7740/44000 | Loss: 3.1539 | LR: 4.68e-04 | 1260.2ms/step | 15.9 samples/s +Step 7750/44000 | Loss: 3.1756 | LR: 4.68e-04 | 1304.6ms/step | 15.3 samples/s +Validation at step 7750: Loss = 3.1581, Perplexity = 23.53 +Step 7760/44000 | Loss: 3.1996 | LR: 4.68e-04 | 1744.9ms/step | 11.5 samples/s +Step 7770/44000 | Loss: 3.1495 | LR: 4.68e-04 | 1325.5ms/step | 15.1 samples/s +Step 7780/44000 | Loss: 3.1934 | LR: 4.68e-04 | 1278.6ms/step | 15.6 samples/s +Step 7790/44000 | Loss: 3.1417 | LR: 4.68e-04 | 1278.4ms/step | 15.6 samples/s +Step 7800/44000 | Loss: 3.1807 | LR: 4.68e-04 | 1435.7ms/step | 13.9 samples/s +Step 7810/44000 | Loss: 3.2217 | LR: 4.68e-04 | 1304.7ms/step | 15.3 samples/s +Step 7820/44000 | Loss: 3.2863 | LR: 4.68e-04 | 1315.6ms/step | 15.2 samples/s +Step 7830/44000 | Loss: 3.2486 | LR: 4.68e-04 | 1319.6ms/step | 15.2 samples/s +Step 7840/44000 | Loss: 3.1938 | LR: 4.68e-04 | 1302.0ms/step | 15.4 samples/s +Step 7850/44000 | Loss: 3.2114 | LR: 4.67e-04 | 1281.5ms/step | 15.6 samples/s +Step 7860/44000 | Loss: 3.1887 | LR: 4.67e-04 | 1283.7ms/step | 15.6 samples/s +Step 7870/44000 | Loss: 3.1707 | LR: 4.67e-04 | 1289.8ms/step | 15.5 samples/s +Step 7880/44000 | Loss: 3.2260 | LR: 4.67e-04 | 1307.3ms/step | 15.3 samples/s +Step 7890/44000 | Loss: 3.1895 | LR: 4.67e-04 | 1286.7ms/step | 15.5 samples/s +Step 7900/44000 | Loss: 3.2412 | LR: 4.67e-04 | 1289.5ms/step | 15.5 samples/s +Step 7910/44000 | Loss: 3.3009 | LR: 4.67e-04 | 1278.3ms/step | 15.6 samples/s +Step 7920/44000 | Loss: 3.1887 | LR: 4.67e-04 | 1267.9ms/step | 15.8 samples/s +Step 7930/44000 | Loss: 3.2518 | LR: 4.67e-04 | 1263.4ms/step | 15.8 samples/s +Step 7940/44000 | Loss: 3.1961 | LR: 4.67e-04 | 1241.9ms/step | 16.1 samples/s +Step 7950/44000 | Loss: 3.1948 | LR: 4.67e-04 | 1273.5ms/step | 15.7 samples/s +Step 7960/44000 | Loss: 3.1857 | LR: 4.66e-04 | 1288.1ms/step | 15.5 samples/s +Step 7970/44000 | Loss: 3.2368 | LR: 4.66e-04 | 1310.5ms/step | 15.3 samples/s +Step 7980/44000 | Loss: 3.2193 | LR: 4.66e-04 | 1362.5ms/step | 14.7 samples/s +Step 7990/44000 | Loss: 3.1385 | LR: 4.66e-04 | 1229.5ms/step | 16.3 samples/s +Step 8000/44000 | Loss: 3.1767 | LR: 4.66e-04 | 1272.8ms/step | 15.7 samples/s +Validation at step 8000: Loss = 3.1493, Perplexity = 23.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-8000 +Step 8010/44000 | Loss: 3.2285 | LR: 4.66e-04 | 2308.6ms/step | 8.7 samples/s +Step 8020/44000 | Loss: 3.2277 | LR: 4.66e-04 | 1304.5ms/step | 15.3 samples/s +Step 8030/44000 | Loss: 3.1382 | LR: 4.66e-04 | 1252.7ms/step | 16.0 samples/s +Step 8040/44000 | Loss: 3.2160 | LR: 4.66e-04 | 1273.5ms/step | 15.7 samples/s +Step 8050/44000 | Loss: 3.1588 | LR: 4.66e-04 | 1310.3ms/step | 15.3 samples/s +Step 8060/44000 | Loss: 3.1142 | LR: 4.66e-04 | 1299.2ms/step | 15.4 samples/s +Step 8070/44000 | Loss: 3.2423 | LR: 4.65e-04 | 1310.3ms/step | 15.3 samples/s +Step 8080/44000 | Loss: 3.1651 | LR: 4.65e-04 | 1248.1ms/step | 16.0 samples/s +Step 8090/44000 | Loss: 3.2464 | LR: 4.65e-04 | 1277.5ms/step | 15.7 samples/s +Step 8100/44000 | Loss: 3.2578 | LR: 4.65e-04 | 1266.1ms/step | 15.8 samples/s +Step 8110/44000 | Loss: 3.2647 | LR: 4.65e-04 | 1275.3ms/step | 15.7 samples/s +Step 8120/44000 | Loss: 3.1832 | LR: 4.65e-04 | 1263.1ms/step | 15.8 samples/s +Step 8130/44000 | Loss: 3.2335 | LR: 4.65e-04 | 1262.3ms/step | 15.8 samples/s +Step 8140/44000 | Loss: 3.2307 | LR: 4.65e-04 | 1373.4ms/step | 14.6 samples/s +Step 8150/44000 | Loss: 3.2495 | LR: 4.65e-04 | 1258.2ms/step | 15.9 samples/s +Step 8160/44000 | Loss: 3.1778 | LR: 4.65e-04 | 1304.9ms/step | 15.3 samples/s +Step 8170/44000 | Loss: 3.1713 | LR: 4.65e-04 | 1266.4ms/step | 15.8 samples/s +Step 8180/44000 | Loss: 3.2459 | LR: 4.64e-04 | 1246.8ms/step | 16.0 samples/s +Step 8190/44000 | Loss: 3.2451 | LR: 4.64e-04 | 1327.2ms/step | 15.1 samples/s +Step 8200/44000 | Loss: 3.1946 | LR: 4.64e-04 | 1274.5ms/step | 15.7 samples/s +Step 8210/44000 | Loss: 3.2509 | LR: 4.64e-04 | 1284.5ms/step | 15.6 samples/s +Step 8220/44000 | Loss: 3.2133 | LR: 4.64e-04 | 1288.6ms/step | 15.5 samples/s +Step 8230/44000 | Loss: 3.1019 | LR: 4.64e-04 | 1305.8ms/step | 15.3 samples/s +Step 8240/44000 | Loss: 3.2311 | LR: 4.64e-04 | 1242.4ms/step | 16.1 samples/s +Step 8250/44000 | Loss: 3.1649 | LR: 4.64e-04 | 1265.0ms/step | 15.8 samples/s +Validation at step 8250: Loss = 3.1424, Perplexity = 23.16 +Step 8260/44000 | Loss: 3.1953 | LR: 4.64e-04 | 1791.9ms/step | 11.2 samples/s +Step 8270/44000 | Loss: 3.1877 | LR: 4.64e-04 | 1280.9ms/step | 15.6 samples/s +Step 8280/44000 | Loss: 3.1866 | LR: 4.64e-04 | 1257.9ms/step | 15.9 samples/s +Step 8290/44000 | Loss: 3.1439 | LR: 4.63e-04 | 1320.5ms/step | 15.1 samples/s +Step 8300/44000 | Loss: 3.1499 | LR: 4.63e-04 | 1268.1ms/step | 15.8 samples/s +Step 8310/44000 | Loss: 3.2301 | LR: 4.63e-04 | 1388.1ms/step | 14.4 samples/s +Step 8320/44000 | Loss: 3.1827 | LR: 4.63e-04 | 1280.7ms/step | 15.6 samples/s +Step 8330/44000 | Loss: 3.2588 | LR: 4.63e-04 | 1302.8ms/step | 15.4 samples/s +Step 8340/44000 | Loss: 3.1977 | LR: 4.63e-04 | 1278.1ms/step | 15.6 samples/s +Step 8350/44000 | Loss: 3.2274 | LR: 4.63e-04 | 1293.7ms/step | 15.5 samples/s +Step 8360/44000 | Loss: 3.1756 | LR: 4.63e-04 | 1236.0ms/step | 16.2 samples/s +Step 8370/44000 | Loss: 3.1903 | LR: 4.63e-04 | 1257.8ms/step | 15.9 samples/s +Step 8380/44000 | Loss: 3.2060 | LR: 4.63e-04 | 1309.7ms/step | 15.3 samples/s +Step 8390/44000 | Loss: 3.1819 | LR: 4.62e-04 | 1293.5ms/step | 15.5 samples/s +Step 8400/44000 | Loss: 3.1642 | LR: 4.62e-04 | 1265.0ms/step | 15.8 samples/s +Step 8410/44000 | Loss: 3.2326 | LR: 4.62e-04 | 1283.3ms/step | 15.6 samples/s +Step 8420/44000 | Loss: 3.1694 | LR: 4.62e-04 | 1259.6ms/step | 15.9 samples/s +Step 8430/44000 | Loss: 3.1297 | LR: 4.62e-04 | 1262.0ms/step | 15.8 samples/s +Step 8440/44000 | Loss: 3.2226 | LR: 4.62e-04 | 1294.1ms/step | 15.5 samples/s +Step 8450/44000 | Loss: 3.2508 | LR: 4.62e-04 | 1314.0ms/step | 15.2 samples/s +Step 8460/44000 | Loss: 3.2022 | LR: 4.62e-04 | 1330.3ms/step | 15.0 samples/s +Step 8470/44000 | Loss: 3.1785 | LR: 4.62e-04 | 1283.8ms/step | 15.6 samples/s +Step 8480/44000 | Loss: 3.1553 | LR: 4.62e-04 | 1267.5ms/step | 15.8 samples/s +Step 8490/44000 | Loss: 3.2541 | LR: 4.62e-04 | 1393.9ms/step | 14.3 samples/s +Step 8500/44000 | Loss: 3.1840 | LR: 4.61e-04 | 1247.5ms/step | 16.0 samples/s +Validation at step 8500: Loss = 3.1311, Perplexity = 22.90 +Step 8510/44000 | Loss: 3.1858 | LR: 4.61e-04 | 1756.5ms/step | 11.4 samples/s +Step 8520/44000 | Loss: 3.1465 | LR: 4.61e-04 | 1308.2ms/step | 15.3 samples/s +Step 8530/44000 | Loss: 3.2076 | LR: 4.61e-04 | 1336.1ms/step | 15.0 samples/s +Step 8540/44000 | Loss: 3.1379 | LR: 4.61e-04 | 1286.8ms/step | 15.5 samples/s +Step 8550/44000 | Loss: 3.1319 | LR: 4.61e-04 | 1270.6ms/step | 15.7 samples/s +Step 8560/44000 | Loss: 3.1399 | LR: 4.61e-04 | 1304.5ms/step | 15.3 samples/s +Step 8570/44000 | Loss: 3.1216 | LR: 4.61e-04 | 1265.3ms/step | 15.8 samples/s +Step 8580/44000 | Loss: 3.2645 | LR: 4.61e-04 | 1242.4ms/step | 16.1 samples/s +Step 8590/44000 | Loss: 3.2531 | LR: 4.61e-04 | 1310.5ms/step | 15.3 samples/s +Step 8600/44000 | Loss: 3.2941 | LR: 4.60e-04 | 1307.7ms/step | 15.3 samples/s +Step 8610/44000 | Loss: 3.2076 | LR: 4.60e-04 | 1324.5ms/step | 15.1 samples/s +Step 8620/44000 | Loss: 3.2143 | LR: 4.60e-04 | 1283.5ms/step | 15.6 samples/s +Step 8630/44000 | Loss: 3.1989 | LR: 4.60e-04 | 1278.3ms/step | 15.6 samples/s +Step 8640/44000 | Loss: 3.2142 | LR: 4.60e-04 | 1325.2ms/step | 15.1 samples/s +Step 8650/44000 | Loss: 3.2471 | LR: 4.60e-04 | 1268.1ms/step | 15.8 samples/s +Step 8660/44000 | Loss: 3.2083 | LR: 4.60e-04 | 1441.2ms/step | 13.9 samples/s +Step 8670/44000 | Loss: 3.2570 | LR: 4.60e-04 | 1308.4ms/step | 15.3 samples/s +Step 8680/44000 | Loss: 3.1903 | LR: 4.60e-04 | 1272.9ms/step | 15.7 samples/s +Step 8690/44000 | Loss: 3.1669 | LR: 4.60e-04 | 1312.3ms/step | 15.2 samples/s +Step 8700/44000 | Loss: 3.1791 | LR: 4.59e-04 | 1278.8ms/step | 15.6 samples/s +Step 8710/44000 | Loss: 3.2056 | LR: 4.59e-04 | 1315.8ms/step | 15.2 samples/s +Step 8720/44000 | Loss: 3.1929 | LR: 4.59e-04 | 1258.0ms/step | 15.9 samples/s +Step 8730/44000 | Loss: 3.1503 | LR: 4.59e-04 | 1247.4ms/step | 16.0 samples/s +Step 8740/44000 | Loss: 3.1784 | LR: 4.59e-04 | 1280.2ms/step | 15.6 samples/s +Step 8750/44000 | Loss: 3.2267 | LR: 4.59e-04 | 1287.7ms/step | 15.5 samples/s +Validation at step 8750: Loss = 3.1234, Perplexity = 22.72 +Step 8760/44000 | Loss: 3.2568 | LR: 4.59e-04 | 1719.4ms/step | 11.6 samples/s +Step 8770/44000 | Loss: 3.1564 | LR: 4.59e-04 | 1298.9ms/step | 15.4 samples/s +Step 8780/44000 | Loss: 3.1357 | LR: 4.59e-04 | 1303.2ms/step | 15.3 samples/s +Step 8790/44000 | Loss: 3.1820 | LR: 4.59e-04 | 1264.7ms/step | 15.8 samples/s +Step 8800/44000 | Loss: 3.1499 | LR: 4.58e-04 | 1326.1ms/step | 15.1 samples/s +Step 8810/44000 | Loss: 3.1812 | LR: 4.58e-04 | 1299.7ms/step | 15.4 samples/s +Step 8820/44000 | Loss: 3.1913 | LR: 4.58e-04 | 1294.6ms/step | 15.4 samples/s +Step 8830/44000 | Loss: 3.1254 | LR: 4.58e-04 | 1394.5ms/step | 14.3 samples/s +Step 8840/44000 | Loss: 3.2011 | LR: 4.58e-04 | 1273.8ms/step | 15.7 samples/s +Step 8850/44000 | Loss: 3.2291 | LR: 4.58e-04 | 1312.6ms/step | 15.2 samples/s +Step 8860/44000 | Loss: 3.2005 | LR: 4.58e-04 | 1254.9ms/step | 15.9 samples/s +Step 8870/44000 | Loss: 3.1337 | LR: 4.58e-04 | 1289.0ms/step | 15.5 samples/s +Step 8880/44000 | Loss: 3.1153 | LR: 4.58e-04 | 1278.7ms/step | 15.6 samples/s +Step 8890/44000 | Loss: 3.1983 | LR: 4.58e-04 | 1296.6ms/step | 15.4 samples/s +Step 8900/44000 | Loss: 3.1491 | LR: 4.57e-04 | 1281.9ms/step | 15.6 samples/s +Step 8910/44000 | Loss: 3.1308 | LR: 4.57e-04 | 1330.5ms/step | 15.0 samples/s +Step 8920/44000 | Loss: 3.2456 | LR: 4.57e-04 | 1252.7ms/step | 16.0 samples/s +Step 8930/44000 | Loss: 3.1138 | LR: 4.57e-04 | 1273.3ms/step | 15.7 samples/s +Step 8940/44000 | Loss: 3.1737 | LR: 4.57e-04 | 1307.7ms/step | 15.3 samples/s +Step 8950/44000 | Loss: 3.2342 | LR: 4.57e-04 | 1286.1ms/step | 15.6 samples/s +Step 8960/44000 | Loss: 3.2114 | LR: 4.57e-04 | 1269.0ms/step | 15.8 samples/s +Step 8970/44000 | Loss: 3.1659 | LR: 4.57e-04 | 1294.1ms/step | 15.5 samples/s +Step 8980/44000 | Loss: 3.2330 | LR: 4.57e-04 | 1294.2ms/step | 15.5 samples/s +Step 8990/44000 | Loss: 3.1306 | LR: 4.57e-04 | 1293.2ms/step | 15.5 samples/s +Step 9000/44000 | Loss: 3.2127 | LR: 4.56e-04 | 1379.6ms/step | 14.5 samples/s +Validation at step 9000: Loss = 3.1184, Perplexity = 22.61 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-9000 +Step 9010/44000 | Loss: 3.1695 | LR: 4.56e-04 | 2232.2ms/step | 9.0 samples/s +Step 9020/44000 | Loss: 3.1447 | LR: 4.56e-04 | 1252.6ms/step | 16.0 samples/s +Step 9030/44000 | Loss: 3.1819 | LR: 4.56e-04 | 1294.1ms/step | 15.5 samples/s +Step 9040/44000 | Loss: 3.2150 | LR: 4.56e-04 | 1273.4ms/step | 15.7 samples/s +Step 9050/44000 | Loss: 3.0997 | LR: 4.56e-04 | 1288.8ms/step | 15.5 samples/s +Step 9060/44000 | Loss: 3.2130 | LR: 4.56e-04 | 1280.3ms/step | 15.6 samples/s +Step 9070/44000 | Loss: 3.2144 | LR: 4.56e-04 | 1285.1ms/step | 15.6 samples/s +Step 9080/44000 | Loss: 3.1956 | LR: 4.56e-04 | 1291.4ms/step | 15.5 samples/s +Step 9090/44000 | Loss: 3.2272 | LR: 4.56e-04 | 1241.8ms/step | 16.1 samples/s +Step 9100/44000 | Loss: 3.1181 | LR: 4.55e-04 | 1262.9ms/step | 15.8 samples/s +Step 9110/44000 | Loss: 3.1203 | LR: 4.55e-04 | 1277.4ms/step | 15.7 samples/s +Step 9120/44000 | Loss: 3.1617 | LR: 4.55e-04 | 1311.0ms/step | 15.3 samples/s +Step 9130/44000 | Loss: 3.2101 | LR: 4.55e-04 | 1257.4ms/step | 15.9 samples/s +Step 9140/44000 | Loss: 3.1625 | LR: 4.55e-04 | 1294.3ms/step | 15.5 samples/s +Step 9150/44000 | Loss: 3.1089 | LR: 4.55e-04 | 1267.7ms/step | 15.8 samples/s +Step 9160/44000 | Loss: 3.2268 | LR: 4.55e-04 | 1267.8ms/step | 15.8 samples/s +Step 9170/44000 | Loss: 3.1859 | LR: 4.55e-04 | 1395.8ms/step | 14.3 samples/s +Step 9180/44000 | Loss: 3.1449 | LR: 4.55e-04 | 1244.8ms/step | 16.1 samples/s +Step 9190/44000 | Loss: 3.1573 | LR: 4.54e-04 | 1332.6ms/step | 15.0 samples/s +Step 9200/44000 | Loss: 3.1567 | LR: 4.54e-04 | 1284.0ms/step | 15.6 samples/s +Step 9210/44000 | Loss: 3.1626 | LR: 4.54e-04 | 1284.1ms/step | 15.6 samples/s +Step 9220/44000 | Loss: 3.1384 | LR: 4.54e-04 | 1305.0ms/step | 15.3 samples/s +Step 9230/44000 | Loss: 3.1257 | LR: 4.54e-04 | 1304.7ms/step | 15.3 samples/s +Step 9240/44000 | Loss: 3.2202 | LR: 4.54e-04 | 1300.7ms/step | 15.4 samples/s +Step 9250/44000 | Loss: 3.1618 | LR: 4.54e-04 | 1293.3ms/step | 15.5 samples/s +Validation at step 9250: Loss = 3.1155, Perplexity = 22.54 +Step 9260/44000 | Loss: 3.1650 | LR: 4.54e-04 | 1766.1ms/step | 11.3 samples/s +Step 9270/44000 | Loss: 3.1739 | LR: 4.54e-04 | 1289.1ms/step | 15.5 samples/s +Step 9280/44000 | Loss: 3.1625 | LR: 4.54e-04 | 1297.2ms/step | 15.4 samples/s +Step 9290/44000 | Loss: 3.1897 | LR: 4.53e-04 | 1282.8ms/step | 15.6 samples/s +Step 9300/44000 | Loss: 3.1465 | LR: 4.53e-04 | 1334.8ms/step | 15.0 samples/s +Step 9310/44000 | Loss: 3.2129 | LR: 4.53e-04 | 1231.4ms/step | 16.2 samples/s +Step 9320/44000 | Loss: 3.1878 | LR: 4.53e-04 | 1283.2ms/step | 15.6 samples/s +Step 9330/44000 | Loss: 3.1368 | LR: 4.53e-04 | 1300.1ms/step | 15.4 samples/s +Step 9340/44000 | Loss: 3.1024 | LR: 4.53e-04 | 1430.5ms/step | 14.0 samples/s +Step 9350/44000 | Loss: 3.2344 | LR: 4.53e-04 | 1264.3ms/step | 15.8 samples/s +Step 9360/44000 | Loss: 3.1659 | LR: 4.53e-04 | 1245.8ms/step | 16.1 samples/s +Step 9370/44000 | Loss: 3.1902 | LR: 4.53e-04 | 1299.6ms/step | 15.4 samples/s +Step 9380/44000 | Loss: 3.1905 | LR: 4.52e-04 | 1284.1ms/step | 15.6 samples/s +Step 9390/44000 | Loss: 3.2414 | LR: 4.52e-04 | 1262.8ms/step | 15.8 samples/s +Step 9400/44000 | Loss: 3.1213 | LR: 4.52e-04 | 1289.0ms/step | 15.5 samples/s +Step 9410/44000 | Loss: 3.1044 | LR: 4.52e-04 | 1243.3ms/step | 16.1 samples/s +Step 9420/44000 | Loss: 3.1367 | LR: 4.52e-04 | 1262.5ms/step | 15.8 samples/s +Step 9430/44000 | Loss: 3.1487 | LR: 4.52e-04 | 1278.2ms/step | 15.6 samples/s +Step 9440/44000 | Loss: 3.1214 | LR: 4.52e-04 | 1274.1ms/step | 15.7 samples/s +Step 9450/44000 | Loss: 3.1988 | LR: 4.52e-04 | 1268.3ms/step | 15.8 samples/s +Step 9460/44000 | Loss: 3.1372 | LR: 4.52e-04 | 1310.1ms/step | 15.3 samples/s +Step 9470/44000 | Loss: 3.1772 | LR: 4.52e-04 | 1322.2ms/step | 15.1 samples/s +Step 9480/44000 | Loss: 3.1839 | LR: 4.51e-04 | 1293.3ms/step | 15.5 samples/s +Step 9490/44000 | Loss: 3.2246 | LR: 4.51e-04 | 1276.9ms/step | 15.7 samples/s +Step 9500/44000 | Loss: 3.2058 | LR: 4.51e-04 | 1290.6ms/step | 15.5 samples/s +Validation at step 9500: Loss = 3.1066, Perplexity = 22.34 +Step 9510/44000 | Loss: 3.1370 | LR: 4.51e-04 | 1918.8ms/step | 10.4 samples/s +Step 9520/44000 | Loss: 3.1751 | LR: 4.51e-04 | 1247.5ms/step | 16.0 samples/s +Step 9530/44000 | Loss: 3.1158 | LR: 4.51e-04 | 1264.9ms/step | 15.8 samples/s +Step 9540/44000 | Loss: 3.1793 | LR: 4.51e-04 | 1292.2ms/step | 15.5 samples/s +Step 9550/44000 | Loss: 3.1485 | LR: 4.51e-04 | 1283.7ms/step | 15.6 samples/s +Step 9560/44000 | Loss: 3.2092 | LR: 4.51e-04 | 1268.2ms/step | 15.8 samples/s +Step 9570/44000 | Loss: 3.1661 | LR: 4.50e-04 | 1279.0ms/step | 15.6 samples/s +Step 9580/44000 | Loss: 3.1200 | LR: 4.50e-04 | 1252.7ms/step | 16.0 samples/s +Step 9590/44000 | Loss: 3.1331 | LR: 4.50e-04 | 1356.7ms/step | 14.7 samples/s +Step 9600/44000 | Loss: 3.1319 | LR: 4.50e-04 | 1299.6ms/step | 15.4 samples/s +Step 9610/44000 | Loss: 3.1385 | LR: 4.50e-04 | 1272.4ms/step | 15.7 samples/s +Step 9620/44000 | Loss: 3.1977 | LR: 4.50e-04 | 1289.5ms/step | 15.5 samples/s +Step 9630/44000 | Loss: 3.1649 | LR: 4.50e-04 | 1278.6ms/step | 15.6 samples/s +Step 9640/44000 | Loss: 3.1130 | LR: 4.50e-04 | 1236.2ms/step | 16.2 samples/s +Step 9650/44000 | Loss: 3.1025 | LR: 4.50e-04 | 1245.0ms/step | 16.1 samples/s +Step 9660/44000 | Loss: 3.2374 | LR: 4.49e-04 | 1238.3ms/step | 16.2 samples/s +Step 9670/44000 | Loss: 3.1645 | LR: 4.49e-04 | 1284.6ms/step | 15.6 samples/s +Step 9680/44000 | Loss: 3.1646 | LR: 4.49e-04 | 1293.9ms/step | 15.5 samples/s +Step 9690/44000 | Loss: 3.1526 | LR: 4.49e-04 | 1352.0ms/step | 14.8 samples/s +Step 9700/44000 | Loss: 3.2128 | LR: 4.49e-04 | 1295.9ms/step | 15.4 samples/s +Step 9710/44000 | Loss: 3.1310 | LR: 4.49e-04 | 1298.4ms/step | 15.4 samples/s +Step 9720/44000 | Loss: 3.1269 | LR: 4.49e-04 | 1277.8ms/step | 15.7 samples/s +Step 9730/44000 | Loss: 3.1904 | LR: 4.49e-04 | 1294.7ms/step | 15.4 samples/s +Step 9740/44000 | Loss: 3.1320 | LR: 4.49e-04 | 1283.5ms/step | 15.6 samples/s +Step 9750/44000 | Loss: 3.1933 | LR: 4.48e-04 | 1284.2ms/step | 15.6 samples/s +Validation at step 9750: Loss = 3.1030, Perplexity = 22.26 +Step 9760/44000 | Loss: 3.1459 | LR: 4.48e-04 | 1755.4ms/step | 11.4 samples/s +Step 9770/44000 | Loss: 3.1874 | LR: 4.48e-04 | 1299.4ms/step | 15.4 samples/s +Step 9780/44000 | Loss: 3.1282 | LR: 4.48e-04 | 1268.1ms/step | 15.8 samples/s +Step 9790/44000 | Loss: 3.0860 | LR: 4.48e-04 | 1299.6ms/step | 15.4 samples/s +Step 9800/44000 | Loss: 3.1802 | LR: 4.48e-04 | 1283.8ms/step | 15.6 samples/s +Step 9810/44000 | Loss: 3.1384 | LR: 4.48e-04 | 1283.8ms/step | 15.6 samples/s +Step 9820/44000 | Loss: 3.1647 | LR: 4.48e-04 | 1242.1ms/step | 16.1 samples/s +Step 9830/44000 | Loss: 3.1675 | LR: 4.48e-04 | 1273.6ms/step | 15.7 samples/s +Step 9840/44000 | Loss: 3.1620 | LR: 4.47e-04 | 1289.5ms/step | 15.5 samples/s +Step 9850/44000 | Loss: 3.1645 | LR: 4.47e-04 | 1247.1ms/step | 16.0 samples/s +Step 9860/44000 | Loss: 3.2078 | LR: 4.47e-04 | 1379.2ms/step | 14.5 samples/s +Step 9870/44000 | Loss: 3.1964 | LR: 4.47e-04 | 1277.1ms/step | 15.7 samples/s +Step 9880/44000 | Loss: 3.1227 | LR: 4.47e-04 | 1288.6ms/step | 15.5 samples/s +Step 9890/44000 | Loss: 3.1802 | LR: 4.47e-04 | 1238.1ms/step | 16.2 samples/s +Step 9900/44000 | Loss: 3.1054 | LR: 4.47e-04 | 1303.5ms/step | 15.3 samples/s +Step 9910/44000 | Loss: 3.1653 | LR: 4.47e-04 | 1299.1ms/step | 15.4 samples/s +Step 9920/44000 | Loss: 3.1473 | LR: 4.47e-04 | 1297.1ms/step | 15.4 samples/s +Step 9930/44000 | Loss: 3.1473 | LR: 4.46e-04 | 1317.7ms/step | 15.2 samples/s +Step 9940/44000 | Loss: 3.1733 | LR: 4.46e-04 | 1276.4ms/step | 15.7 samples/s +Step 9950/44000 | Loss: 3.1524 | LR: 4.46e-04 | 1284.8ms/step | 15.6 samples/s +Step 9960/44000 | Loss: 3.1607 | LR: 4.46e-04 | 1296.3ms/step | 15.4 samples/s +Step 9970/44000 | Loss: 3.2097 | LR: 4.46e-04 | 1294.6ms/step | 15.4 samples/s +Step 9980/44000 | Loss: 3.0936 | LR: 4.46e-04 | 1278.7ms/step | 15.6 samples/s +Step 9990/44000 | Loss: 3.1404 | LR: 4.46e-04 | 1252.2ms/step | 16.0 samples/s +Step 10000/44000 | Loss: 3.1771 | LR: 4.46e-04 | 1252.7ms/step | 16.0 samples/s +Validation at step 10000: Loss = 3.0970, Perplexity = 22.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-10000 +Step 10010/44000 | Loss: 3.1821 | LR: 4.46e-04 | 2241.0ms/step | 8.9 samples/s +Step 10020/44000 | Loss: 3.2078 | LR: 4.45e-04 | 1390.1ms/step | 14.4 samples/s +Step 10030/44000 | Loss: 3.1570 | LR: 4.45e-04 | 1372.8ms/step | 14.6 samples/s +Step 10040/44000 | Loss: 3.1731 | LR: 4.45e-04 | 1267.9ms/step | 15.8 samples/s +Step 10050/44000 | Loss: 3.2194 | LR: 4.45e-04 | 1283.7ms/step | 15.6 samples/s +Step 10060/44000 | Loss: 3.1149 | LR: 4.45e-04 | 1294.3ms/step | 15.5 samples/s +Step 10070/44000 | Loss: 3.1493 | LR: 4.45e-04 | 1293.7ms/step | 15.5 samples/s +Step 10080/44000 | Loss: 3.1797 | LR: 4.45e-04 | 1311.6ms/step | 15.2 samples/s +Step 10090/44000 | Loss: 3.1361 | LR: 4.45e-04 | 1281.5ms/step | 15.6 samples/s +Step 10100/44000 | Loss: 3.1905 | LR: 4.45e-04 | 1274.1ms/step | 15.7 samples/s +Step 10110/44000 | Loss: 3.1540 | LR: 4.44e-04 | 1299.7ms/step | 15.4 samples/s +Step 10120/44000 | Loss: 3.1397 | LR: 4.44e-04 | 1278.3ms/step | 15.6 samples/s +Step 10130/44000 | Loss: 3.0972 | LR: 4.44e-04 | 1288.7ms/step | 15.5 samples/s +Step 10140/44000 | Loss: 3.1876 | LR: 4.44e-04 | 1278.9ms/step | 15.6 samples/s +Step 10150/44000 | Loss: 3.1333 | LR: 4.44e-04 | 1264.1ms/step | 15.8 samples/s +Step 10160/44000 | Loss: 3.1329 | LR: 4.44e-04 | 1309.8ms/step | 15.3 samples/s +Step 10170/44000 | Loss: 3.0818 | LR: 4.44e-04 | 1273.4ms/step | 15.7 samples/s +Step 10180/44000 | Loss: 3.1179 | LR: 4.44e-04 | 1266.4ms/step | 15.8 samples/s +Step 10190/44000 | Loss: 3.1997 | LR: 4.43e-04 | 1270.7ms/step | 15.7 samples/s +Step 10200/44000 | Loss: 3.1256 | LR: 4.43e-04 | 1445.2ms/step | 13.8 samples/s +Step 10210/44000 | Loss: 3.1111 | LR: 4.43e-04 | 1244.6ms/step | 16.1 samples/s +Step 10220/44000 | Loss: 3.0811 | LR: 4.43e-04 | 1282.7ms/step | 15.6 samples/s +Step 10230/44000 | Loss: 3.1786 | LR: 4.43e-04 | 1283.3ms/step | 15.6 samples/s +Step 10240/44000 | Loss: 3.1454 | LR: 4.43e-04 | 1273.0ms/step | 15.7 samples/s +Step 10250/44000 | Loss: 3.1290 | LR: 4.43e-04 | 1313.7ms/step | 15.2 samples/s +Validation at step 10250: Loss = 3.0911, Perplexity = 22.00 +Step 10260/44000 | Loss: 3.0737 | LR: 4.43e-04 | 1736.8ms/step | 11.5 samples/s +Step 10270/44000 | Loss: 3.1429 | LR: 4.43e-04 | 1338.1ms/step | 14.9 samples/s +Step 10280/44000 | Loss: 3.2140 | LR: 4.42e-04 | 1356.2ms/step | 14.7 samples/s +Step 10290/44000 | Loss: 3.1007 | LR: 4.42e-04 | 1284.2ms/step | 15.6 samples/s +Step 10300/44000 | Loss: 3.1606 | LR: 4.42e-04 | 1300.0ms/step | 15.4 samples/s +Step 10310/44000 | Loss: 3.1644 | LR: 4.42e-04 | 1289.1ms/step | 15.5 samples/s +Step 10320/44000 | Loss: 3.1620 | LR: 4.42e-04 | 1263.5ms/step | 15.8 samples/s +Step 10330/44000 | Loss: 3.1375 | LR: 4.42e-04 | 1258.2ms/step | 15.9 samples/s +Step 10340/44000 | Loss: 3.1369 | LR: 4.42e-04 | 1278.5ms/step | 15.6 samples/s +Step 10350/44000 | Loss: 3.1179 | LR: 4.42e-04 | 1242.3ms/step | 16.1 samples/s +Step 10360/44000 | Loss: 3.1287 | LR: 4.42e-04 | 1273.6ms/step | 15.7 samples/s +Step 10370/44000 | Loss: 3.1400 | LR: 4.41e-04 | 1404.3ms/step | 14.2 samples/s +Step 10380/44000 | Loss: 3.1882 | LR: 4.41e-04 | 1335.9ms/step | 15.0 samples/s +Step 10390/44000 | Loss: 3.1640 | LR: 4.41e-04 | 1277.9ms/step | 15.7 samples/s +Step 10400/44000 | Loss: 3.2258 | LR: 4.41e-04 | 1255.1ms/step | 15.9 samples/s +Step 10410/44000 | Loss: 3.0635 | LR: 4.41e-04 | 1275.7ms/step | 15.7 samples/s +Step 10420/44000 | Loss: 3.1731 | LR: 4.41e-04 | 1301.4ms/step | 15.4 samples/s +Step 10430/44000 | Loss: 3.0960 | LR: 4.41e-04 | 1270.9ms/step | 15.7 samples/s +Step 10440/44000 | Loss: 3.1038 | LR: 4.41e-04 | 1305.6ms/step | 15.3 samples/s +Step 10450/44000 | Loss: 3.0985 | LR: 4.40e-04 | 1257.2ms/step | 15.9 samples/s +Step 10460/44000 | Loss: 3.1972 | LR: 4.40e-04 | 1253.0ms/step | 16.0 samples/s +Step 10470/44000 | Loss: 3.1356 | LR: 4.40e-04 | 1294.7ms/step | 15.4 samples/s +Step 10480/44000 | Loss: 3.1106 | LR: 4.40e-04 | 1299.8ms/step | 15.4 samples/s +Step 10490/44000 | Loss: 3.1532 | LR: 4.40e-04 | 1258.2ms/step | 15.9 samples/s +Step 10500/44000 | Loss: 3.1473 | LR: 4.40e-04 | 1310.2ms/step | 15.3 samples/s +Validation at step 10500: Loss = 3.0901, Perplexity = 21.98 +Step 10510/44000 | Loss: 3.1488 | LR: 4.40e-04 | 1820.8ms/step | 11.0 samples/s +Step 10520/44000 | Loss: 3.1842 | LR: 4.40e-04 | 1300.3ms/step | 15.4 samples/s +Step 10530/44000 | Loss: 3.1153 | LR: 4.40e-04 | 1281.6ms/step | 15.6 samples/s +Step 10540/44000 | Loss: 3.0927 | LR: 4.39e-04 | 1394.2ms/step | 14.3 samples/s +Step 10550/44000 | Loss: 3.2017 | LR: 4.39e-04 | 1297.0ms/step | 15.4 samples/s +Step 10560/44000 | Loss: 3.1045 | LR: 4.39e-04 | 1349.3ms/step | 14.8 samples/s +Step 10570/44000 | Loss: 3.1199 | LR: 4.39e-04 | 1300.1ms/step | 15.4 samples/s +Step 10580/44000 | Loss: 3.0677 | LR: 4.39e-04 | 1282.9ms/step | 15.6 samples/s +Step 10590/44000 | Loss: 3.0971 | LR: 4.39e-04 | 1322.2ms/step | 15.1 samples/s +Step 10600/44000 | Loss: 3.1284 | LR: 4.39e-04 | 1305.1ms/step | 15.3 samples/s +Step 10610/44000 | Loss: 3.0502 | LR: 4.39e-04 | 1305.6ms/step | 15.3 samples/s +Step 10620/44000 | Loss: 3.0496 | LR: 4.38e-04 | 1265.6ms/step | 15.8 samples/s +Step 10630/44000 | Loss: 3.1333 | LR: 4.38e-04 | 1255.3ms/step | 15.9 samples/s +Step 10640/44000 | Loss: 3.1643 | LR: 4.38e-04 | 1294.4ms/step | 15.5 samples/s +Step 10650/44000 | Loss: 3.1875 | LR: 4.38e-04 | 1273.4ms/step | 15.7 samples/s +Step 10660/44000 | Loss: 3.1527 | LR: 4.38e-04 | 1247.9ms/step | 16.0 samples/s +Step 10670/44000 | Loss: 3.1368 | LR: 4.38e-04 | 1273.8ms/step | 15.7 samples/s +Step 10680/44000 | Loss: 3.0744 | LR: 4.38e-04 | 1285.7ms/step | 15.6 samples/s +Step 10690/44000 | Loss: 3.1803 | LR: 4.38e-04 | 1256.3ms/step | 15.9 samples/s +Step 10700/44000 | Loss: 3.1165 | LR: 4.37e-04 | 1243.2ms/step | 16.1 samples/s +Step 10710/44000 | Loss: 3.0826 | LR: 4.37e-04 | 1388.7ms/step | 14.4 samples/s +Step 10720/44000 | Loss: 3.1102 | LR: 4.37e-04 | 1331.2ms/step | 15.0 samples/s +Step 10730/44000 | Loss: 3.1844 | LR: 4.37e-04 | 1257.2ms/step | 15.9 samples/s +Step 10740/44000 | Loss: 3.0787 | LR: 4.37e-04 | 1265.8ms/step | 15.8 samples/s +Step 10750/44000 | Loss: 3.1273 | LR: 4.37e-04 | 1289.6ms/step | 15.5 samples/s +Validation at step 10750: Loss = 3.0813, Perplexity = 21.79 +Step 10760/44000 | Loss: 3.0539 | LR: 4.37e-04 | 1790.5ms/step | 11.2 samples/s +Step 10770/44000 | Loss: 3.1374 | LR: 4.37e-04 | 1262.6ms/step | 15.8 samples/s +Step 10780/44000 | Loss: 3.2052 | LR: 4.37e-04 | 1295.0ms/step | 15.4 samples/s +Step 10790/44000 | Loss: 3.1721 | LR: 4.36e-04 | 1341.8ms/step | 14.9 samples/s +Step 10800/44000 | Loss: 3.1416 | LR: 4.36e-04 | 1263.7ms/step | 15.8 samples/s +Step 10810/44000 | Loss: 3.1367 | LR: 4.36e-04 | 1294.0ms/step | 15.5 samples/s +Step 10820/44000 | Loss: 3.0551 | LR: 4.36e-04 | 1310.6ms/step | 15.3 samples/s +Step 10830/44000 | Loss: 3.0965 | LR: 4.36e-04 | 1309.6ms/step | 15.3 samples/s +Step 10840/44000 | Loss: 3.1251 | LR: 4.36e-04 | 1266.5ms/step | 15.8 samples/s +Step 10850/44000 | Loss: 3.1430 | LR: 4.36e-04 | 1280.7ms/step | 15.6 samples/s +Step 10860/44000 | Loss: 3.1177 | LR: 4.36e-04 | 1299.9ms/step | 15.4 samples/s +Step 10870/44000 | Loss: 3.1215 | LR: 4.35e-04 | 1267.7ms/step | 15.8 samples/s +Step 10880/44000 | Loss: 3.1446 | LR: 4.35e-04 | 1399.3ms/step | 14.3 samples/s +Step 10890/44000 | Loss: 3.1365 | LR: 4.35e-04 | 1318.7ms/step | 15.2 samples/s +Step 10900/44000 | Loss: 3.0878 | LR: 4.35e-04 | 1281.2ms/step | 15.6 samples/s +Step 10910/44000 | Loss: 3.1115 | LR: 4.35e-04 | 1289.3ms/step | 15.5 samples/s +Step 10920/44000 | Loss: 3.1689 | LR: 4.35e-04 | 1273.5ms/step | 15.7 samples/s +Step 10930/44000 | Loss: 3.1703 | LR: 4.35e-04 | 1294.2ms/step | 15.5 samples/s +Step 10940/44000 | Loss: 3.1751 | LR: 4.35e-04 | 1269.5ms/step | 15.8 samples/s +Step 10950/44000 | Loss: 3.1716 | LR: 4.34e-04 | 1314.6ms/step | 15.2 samples/s +Step 10960/44000 | Loss: 3.0483 | LR: 4.34e-04 | 1315.2ms/step | 15.2 samples/s +Step 10970/44000 | Loss: 3.1619 | LR: 4.34e-04 | 1294.2ms/step | 15.5 samples/s +Step 10980/44000 | Loss: 3.0972 | LR: 4.34e-04 | 1272.5ms/step | 15.7 samples/s +Step 10990/44000 | Loss: 3.0513 | LR: 4.34e-04 | 1240.3ms/step | 16.1 samples/s +Step 11000/44000 | Loss: 3.1560 | LR: 4.34e-04 | 1286.6ms/step | 15.5 samples/s +Validation at step 11000: Loss = 3.0769, Perplexity = 21.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-11000 +Step 11010/44000 | Loss: 3.1281 | LR: 4.34e-04 | 2302.6ms/step | 8.7 samples/s +Step 11020/44000 | Loss: 3.1639 | LR: 4.34e-04 | 1253.4ms/step | 16.0 samples/s +Step 11030/44000 | Loss: 3.0691 | LR: 4.33e-04 | 1273.9ms/step | 15.7 samples/s +Step 11040/44000 | Loss: 3.0967 | LR: 4.33e-04 | 1311.4ms/step | 15.3 samples/s +Step 11050/44000 | Loss: 3.1100 | LR: 4.33e-04 | 1454.6ms/step | 13.7 samples/s +Step 11060/44000 | Loss: 3.0863 | LR: 4.33e-04 | 1291.7ms/step | 15.5 samples/s +Step 11070/44000 | Loss: 3.0676 | LR: 4.33e-04 | 1280.0ms/step | 15.6 samples/s +Step 11080/44000 | Loss: 3.0902 | LR: 4.33e-04 | 1296.5ms/step | 15.4 samples/s +Step 11090/44000 | Loss: 3.1579 | LR: 4.33e-04 | 1233.9ms/step | 16.2 samples/s +Step 11100/44000 | Loss: 3.1260 | LR: 4.33e-04 | 1267.3ms/step | 15.8 samples/s +Step 11110/44000 | Loss: 3.1494 | LR: 4.32e-04 | 1289.7ms/step | 15.5 samples/s +Step 11120/44000 | Loss: 3.1154 | LR: 4.32e-04 | 1220.6ms/step | 16.4 samples/s +Step 11130/44000 | Loss: 3.0954 | LR: 4.32e-04 | 1253.5ms/step | 16.0 samples/s +Step 11140/44000 | Loss: 3.1338 | LR: 4.32e-04 | 1242.5ms/step | 16.1 samples/s +Step 11150/44000 | Loss: 3.1167 | LR: 4.32e-04 | 1315.2ms/step | 15.2 samples/s +Step 11160/44000 | Loss: 3.1863 | LR: 4.32e-04 | 1289.2ms/step | 15.5 samples/s +Step 11170/44000 | Loss: 3.1364 | LR: 4.32e-04 | 1283.9ms/step | 15.6 samples/s +Step 11180/44000 | Loss: 3.1045 | LR: 4.32e-04 | 1267.0ms/step | 15.8 samples/s +Step 11190/44000 | Loss: 3.0950 | LR: 4.31e-04 | 1285.3ms/step | 15.6 samples/s +Step 11200/44000 | Loss: 3.1083 | LR: 4.31e-04 | 1289.4ms/step | 15.5 samples/s +Step 11210/44000 | Loss: 3.0704 | LR: 4.31e-04 | 1271.4ms/step | 15.7 samples/s +Step 11220/44000 | Loss: 3.0995 | LR: 4.31e-04 | 1395.9ms/step | 14.3 samples/s +Step 11230/44000 | Loss: 3.1436 | LR: 4.31e-04 | 1352.5ms/step | 14.8 samples/s +Step 11240/44000 | Loss: 3.1022 | LR: 4.31e-04 | 1320.0ms/step | 15.2 samples/s +Step 11250/44000 | Loss: 3.0978 | LR: 4.31e-04 | 1269.3ms/step | 15.8 samples/s +Validation at step 11250: Loss = 3.0714, Perplexity = 21.57 +Step 11260/44000 | Loss: 3.0800 | LR: 4.31e-04 | 1786.0ms/step | 11.2 samples/s +Step 11270/44000 | Loss: 3.1076 | LR: 4.30e-04 | 1331.6ms/step | 15.0 samples/s +Step 11280/44000 | Loss: 3.1189 | LR: 4.30e-04 | 1313.1ms/step | 15.2 samples/s +Step 11290/44000 | Loss: 3.1244 | LR: 4.30e-04 | 1307.2ms/step | 15.3 samples/s +Step 11300/44000 | Loss: 3.1844 | LR: 4.30e-04 | 1311.2ms/step | 15.3 samples/s +Step 11310/44000 | Loss: 3.1114 | LR: 4.30e-04 | 1260.5ms/step | 15.9 samples/s +Step 11320/44000 | Loss: 3.1339 | LR: 4.30e-04 | 1295.8ms/step | 15.4 samples/s +Step 11330/44000 | Loss: 3.0729 | LR: 4.30e-04 | 1298.9ms/step | 15.4 samples/s +Step 11340/44000 | Loss: 3.1447 | LR: 4.30e-04 | 1263.5ms/step | 15.8 samples/s +Step 11350/44000 | Loss: 3.1252 | LR: 4.29e-04 | 1329.9ms/step | 15.0 samples/s +Step 11360/44000 | Loss: 3.1564 | LR: 4.29e-04 | 1258.8ms/step | 15.9 samples/s +Step 11370/44000 | Loss: 3.0959 | LR: 4.29e-04 | 1264.2ms/step | 15.8 samples/s +Step 11380/44000 | Loss: 3.1564 | LR: 4.29e-04 | 1288.4ms/step | 15.5 samples/s +Step 11390/44000 | Loss: 3.1153 | LR: 4.29e-04 | 1383.4ms/step | 14.5 samples/s +Step 11400/44000 | Loss: 3.1536 | LR: 4.29e-04 | 1343.3ms/step | 14.9 samples/s +Step 11410/44000 | Loss: 3.0564 | LR: 4.29e-04 | 1282.6ms/step | 15.6 samples/s +Step 11420/44000 | Loss: 3.1324 | LR: 4.29e-04 | 1274.3ms/step | 15.7 samples/s +Step 11430/44000 | Loss: 3.1181 | LR: 4.28e-04 | 1253.0ms/step | 16.0 samples/s +Step 11440/44000 | Loss: 3.0678 | LR: 4.28e-04 | 1283.3ms/step | 15.6 samples/s +Step 11450/44000 | Loss: 3.0863 | LR: 4.28e-04 | 1285.1ms/step | 15.6 samples/s +Step 11460/44000 | Loss: 3.0977 | LR: 4.28e-04 | 1279.2ms/step | 15.6 samples/s +Step 11470/44000 | Loss: 3.1231 | LR: 4.28e-04 | 1268.4ms/step | 15.8 samples/s +Step 11480/44000 | Loss: 3.1074 | LR: 4.28e-04 | 1253.0ms/step | 16.0 samples/s +Step 11490/44000 | Loss: 3.1426 | LR: 4.28e-04 | 1268.5ms/step | 15.8 samples/s +Step 11500/44000 | Loss: 3.0973 | LR: 4.28e-04 | 1247.7ms/step | 16.0 samples/s +Validation at step 11500: Loss = 3.0685, Perplexity = 21.51 +Step 11510/44000 | Loss: 3.1078 | LR: 4.27e-04 | 1739.6ms/step | 11.5 samples/s +Step 11520/44000 | Loss: 3.1768 | LR: 4.27e-04 | 1248.9ms/step | 16.0 samples/s +Step 11530/44000 | Loss: 3.1887 | LR: 4.27e-04 | 1316.0ms/step | 15.2 samples/s +Step 11540/44000 | Loss: 3.0756 | LR: 4.27e-04 | 1289.8ms/step | 15.5 samples/s +Step 11550/44000 | Loss: 3.0855 | LR: 4.27e-04 | 1258.2ms/step | 15.9 samples/s +Step 11560/44000 | Loss: 3.0675 | LR: 4.27e-04 | 1383.8ms/step | 14.5 samples/s +Step 11570/44000 | Loss: 3.1463 | LR: 4.27e-04 | 1363.0ms/step | 14.7 samples/s +Step 11580/44000 | Loss: 3.1539 | LR: 4.27e-04 | 1252.7ms/step | 16.0 samples/s +Step 11590/44000 | Loss: 3.1795 | LR: 4.26e-04 | 1250.1ms/step | 16.0 samples/s +Step 11600/44000 | Loss: 3.1494 | LR: 4.26e-04 | 1293.1ms/step | 15.5 samples/s +Step 11610/44000 | Loss: 3.1633 | LR: 4.26e-04 | 1292.6ms/step | 15.5 samples/s +Step 11620/44000 | Loss: 3.0511 | LR: 4.26e-04 | 1281.6ms/step | 15.6 samples/s +Step 11630/44000 | Loss: 3.0955 | LR: 4.26e-04 | 1298.9ms/step | 15.4 samples/s +Step 11640/44000 | Loss: 3.0382 | LR: 4.26e-04 | 1280.3ms/step | 15.6 samples/s +Step 11650/44000 | Loss: 3.0528 | LR: 4.26e-04 | 1322.8ms/step | 15.1 samples/s +Step 11660/44000 | Loss: 3.0925 | LR: 4.26e-04 | 1283.5ms/step | 15.6 samples/s +Step 11670/44000 | Loss: 3.1535 | LR: 4.25e-04 | 1272.7ms/step | 15.7 samples/s +Step 11680/44000 | Loss: 3.1218 | LR: 4.25e-04 | 1315.9ms/step | 15.2 samples/s +Step 11690/44000 | Loss: 3.1611 | LR: 4.25e-04 | 1268.3ms/step | 15.8 samples/s +Step 11700/44000 | Loss: 3.1945 | LR: 4.25e-04 | 1289.8ms/step | 15.5 samples/s +Step 11710/44000 | Loss: 3.0931 | LR: 4.25e-04 | 1264.1ms/step | 15.8 samples/s +Step 11720/44000 | Loss: 3.0034 | LR: 4.25e-04 | 1280.7ms/step | 15.6 samples/s +Step 11730/44000 | Loss: 3.0796 | LR: 4.25e-04 | 1266.3ms/step | 15.8 samples/s +Step 11740/44000 | Loss: 3.1459 | LR: 4.24e-04 | 1421.0ms/step | 14.1 samples/s +Step 11750/44000 | Loss: 3.1645 | LR: 4.24e-04 | 1314.1ms/step | 15.2 samples/s +Validation at step 11750: Loss = 3.0676, Perplexity = 21.49 +Step 11760/44000 | Loss: 3.1710 | LR: 4.24e-04 | 1804.2ms/step | 11.1 samples/s +Step 11770/44000 | Loss: 3.0862 | LR: 4.24e-04 | 1372.6ms/step | 14.6 samples/s +Step 11780/44000 | Loss: 3.0634 | LR: 4.24e-04 | 1305.8ms/step | 15.3 samples/s +Step 11790/44000 | Loss: 3.1396 | LR: 4.24e-04 | 1315.7ms/step | 15.2 samples/s +Step 11800/44000 | Loss: 3.0794 | LR: 4.24e-04 | 1332.2ms/step | 15.0 samples/s +Step 11810/44000 | Loss: 3.0893 | LR: 4.24e-04 | 1459.0ms/step | 13.7 samples/s +Step 11820/44000 | Loss: 3.1575 | LR: 4.23e-04 | 1532.8ms/step | 13.0 samples/s +Step 11830/44000 | Loss: 3.0979 | LR: 4.23e-04 | 1532.4ms/step | 13.1 samples/s +Step 11840/44000 | Loss: 3.1631 | LR: 4.23e-04 | 1556.5ms/step | 12.8 samples/s +Step 11850/44000 | Loss: 3.1029 | LR: 4.23e-04 | 1508.5ms/step | 13.3 samples/s +Step 11860/44000 | Loss: 3.1471 | LR: 4.23e-04 | 1537.5ms/step | 13.0 samples/s +Step 11870/44000 | Loss: 3.1211 | LR: 4.23e-04 | 1485.7ms/step | 13.5 samples/s +Step 11880/44000 | Loss: 3.1337 | LR: 4.23e-04 | 1545.2ms/step | 12.9 samples/s +Step 11890/44000 | Loss: 3.1098 | LR: 4.23e-04 | 1766.6ms/step | 11.3 samples/s +Step 11900/44000 | Loss: 3.1010 | LR: 4.22e-04 | 1599.1ms/step | 12.5 samples/s +Step 11910/44000 | Loss: 3.1138 | LR: 4.22e-04 | 1664.6ms/step | 12.0 samples/s +Step 11920/44000 | Loss: 3.1301 | LR: 4.22e-04 | 1523.0ms/step | 13.1 samples/s +Step 11930/44000 | Loss: 3.1093 | LR: 4.22e-04 | 1591.9ms/step | 12.6 samples/s +Step 11940/44000 | Loss: 3.1000 | LR: 4.22e-04 | 1528.3ms/step | 13.1 samples/s +Step 11950/44000 | Loss: 3.1129 | LR: 4.22e-04 | 1541.4ms/step | 13.0 samples/s +Step 11960/44000 | Loss: 3.0695 | LR: 4.22e-04 | 1530.6ms/step | 13.1 samples/s +Step 11970/44000 | Loss: 3.1087 | LR: 4.21e-04 | 1593.7ms/step | 12.5 samples/s +Step 11980/44000 | Loss: 3.0942 | LR: 4.21e-04 | 1543.7ms/step | 13.0 samples/s +Step 11990/44000 | Loss: 3.1631 | LR: 4.21e-04 | 1474.1ms/step | 13.6 samples/s +Step 12000/44000 | Loss: 3.0738 | LR: 4.21e-04 | 1567.2ms/step | 12.8 samples/s +Validation at step 12000: Loss = 3.0598, Perplexity = 21.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-12000 +Step 12010/44000 | Loss: 3.0986 | LR: 4.21e-04 | 2728.5ms/step | 7.3 samples/s +Step 12020/44000 | Loss: 3.1202 | LR: 4.21e-04 | 1634.8ms/step | 12.2 samples/s +Step 12030/44000 | Loss: 3.1320 | LR: 4.21e-04 | 1756.8ms/step | 11.4 samples/s +Step 12040/44000 | Loss: 3.1317 | LR: 4.21e-04 | 1583.6ms/step | 12.6 samples/s +Step 12050/44000 | Loss: 3.0621 | LR: 4.20e-04 | 1606.8ms/step | 12.4 samples/s +Step 12060/44000 | Loss: 3.0552 | LR: 4.20e-04 | 1580.0ms/step | 12.7 samples/s +Step 12070/44000 | Loss: 3.0385 | LR: 4.20e-04 | 1532.0ms/step | 13.1 samples/s +Step 12080/44000 | Loss: 3.1311 | LR: 4.20e-04 | 1542.9ms/step | 13.0 samples/s +Step 12090/44000 | Loss: 3.1406 | LR: 4.20e-04 | 1499.9ms/step | 13.3 samples/s +Step 12100/44000 | Loss: 3.0853 | LR: 4.20e-04 | 1516.9ms/step | 13.2 samples/s +Step 12110/44000 | Loss: 3.1017 | LR: 4.20e-04 | 1577.7ms/step | 12.7 samples/s +Step 12120/44000 | Loss: 3.1186 | LR: 4.19e-04 | 1565.1ms/step | 12.8 samples/s +Step 12130/44000 | Loss: 3.0393 | LR: 4.19e-04 | 1566.7ms/step | 12.8 samples/s +Step 12140/44000 | Loss: 3.0340 | LR: 4.19e-04 | 1550.0ms/step | 12.9 samples/s +Step 12150/44000 | Loss: 3.1017 | LR: 4.19e-04 | 1539.7ms/step | 13.0 samples/s +Step 12160/44000 | Loss: 3.1609 | LR: 4.19e-04 | 1561.9ms/step | 12.8 samples/s +Step 12170/44000 | Loss: 3.0560 | LR: 4.19e-04 | 1790.0ms/step | 11.2 samples/s +Step 12180/44000 | Loss: 3.1437 | LR: 4.19e-04 | 1552.5ms/step | 12.9 samples/s +Step 12190/44000 | Loss: 3.0870 | LR: 4.19e-04 | 1580.1ms/step | 12.7 samples/s +Step 12200/44000 | Loss: 3.1063 | LR: 4.18e-04 | 1559.2ms/step | 12.8 samples/s +Step 12210/44000 | Loss: 3.1502 | LR: 4.18e-04 | 1544.0ms/step | 13.0 samples/s +Step 12220/44000 | Loss: 3.1118 | LR: 4.18e-04 | 1531.4ms/step | 13.1 samples/s +Step 12230/44000 | Loss: 3.0746 | LR: 4.18e-04 | 1551.8ms/step | 12.9 samples/s +Step 12240/44000 | Loss: 3.1380 | LR: 4.18e-04 | 1628.2ms/step | 12.3 samples/s +Step 12250/44000 | Loss: 3.1076 | LR: 4.18e-04 | 1565.0ms/step | 12.8 samples/s +Validation at step 12250: Loss = 3.0545, Perplexity = 21.21 +Step 12260/44000 | Loss: 3.1175 | LR: 4.18e-04 | 2164.3ms/step | 9.2 samples/s +Step 12270/44000 | Loss: 3.1266 | LR: 4.17e-04 | 1420.6ms/step | 14.1 samples/s +Step 12280/44000 | Loss: 3.0817 | LR: 4.17e-04 | 1530.7ms/step | 13.1 samples/s +Step 12290/44000 | Loss: 3.1449 | LR: 4.17e-04 | 1514.6ms/step | 13.2 samples/s +Step 12300/44000 | Loss: 3.1877 | LR: 4.17e-04 | 1517.8ms/step | 13.2 samples/s +Step 12310/44000 | Loss: 3.1324 | LR: 4.17e-04 | 1805.1ms/step | 11.1 samples/s +Step 12320/44000 | Loss: 3.1140 | LR: 4.17e-04 | 1571.1ms/step | 12.7 samples/s +Step 12330/44000 | Loss: 3.1196 | LR: 4.17e-04 | 1545.0ms/step | 12.9 samples/s +Step 12340/44000 | Loss: 3.0839 | LR: 4.17e-04 | 1588.6ms/step | 12.6 samples/s +Step 12350/44000 | Loss: 3.0459 | LR: 4.16e-04 | 1476.6ms/step | 13.5 samples/s +Step 12360/44000 | Loss: 3.0727 | LR: 4.16e-04 | 1508.5ms/step | 13.3 samples/s +Step 12370/44000 | Loss: 3.1621 | LR: 4.16e-04 | 1599.2ms/step | 12.5 samples/s +Step 12380/44000 | Loss: 3.0738 | LR: 4.16e-04 | 1592.8ms/step | 12.6 samples/s +Step 12390/44000 | Loss: 3.0987 | LR: 4.16e-04 | 1563.9ms/step | 12.8 samples/s +Step 12400/44000 | Loss: 3.0955 | LR: 4.16e-04 | 1532.5ms/step | 13.1 samples/s +Step 12410/44000 | Loss: 3.0671 | LR: 4.16e-04 | 1584.9ms/step | 12.6 samples/s +Step 12420/44000 | Loss: 3.0892 | LR: 4.15e-04 | 1580.9ms/step | 12.7 samples/s +Step 12430/44000 | Loss: 2.9888 | LR: 4.15e-04 | 1558.5ms/step | 12.8 samples/s +Step 12440/44000 | Loss: 3.1357 | LR: 4.15e-04 | 1526.7ms/step | 13.1 samples/s +Step 12450/44000 | Loss: 3.0688 | LR: 4.15e-04 | 1671.3ms/step | 12.0 samples/s +Step 12460/44000 | Loss: 3.0385 | LR: 4.15e-04 | 1647.6ms/step | 12.1 samples/s +Step 12470/44000 | Loss: 3.1050 | LR: 4.15e-04 | 1615.0ms/step | 12.4 samples/s +Step 12480/44000 | Loss: 3.0969 | LR: 4.15e-04 | 1601.9ms/step | 12.5 samples/s +Step 12490/44000 | Loss: 3.1380 | LR: 4.14e-04 | 1581.3ms/step | 12.6 samples/s +Step 12500/44000 | Loss: 3.1116 | LR: 4.14e-04 | 1532.2ms/step | 13.1 samples/s +Validation at step 12500: Loss = 3.0505, Perplexity = 21.12 +Step 12510/44000 | Loss: 3.0988 | LR: 4.14e-04 | 2175.3ms/step | 9.2 samples/s +Step 12520/44000 | Loss: 3.1003 | LR: 4.14e-04 | 1535.0ms/step | 13.0 samples/s +Step 12530/44000 | Loss: 3.1140 | LR: 4.14e-04 | 1546.3ms/step | 12.9 samples/s +Step 12540/44000 | Loss: 3.0791 | LR: 4.14e-04 | 1559.9ms/step | 12.8 samples/s +Step 12550/44000 | Loss: 3.1151 | LR: 4.14e-04 | 1485.3ms/step | 13.5 samples/s +Step 12560/44000 | Loss: 3.1031 | LR: 4.14e-04 | 1577.8ms/step | 12.7 samples/s +Step 12570/44000 | Loss: 3.0989 | LR: 4.13e-04 | 1543.0ms/step | 13.0 samples/s +Step 12580/44000 | Loss: 3.0943 | LR: 4.13e-04 | 1548.2ms/step | 12.9 samples/s +Step 12590/44000 | Loss: 3.0489 | LR: 4.13e-04 | 1646.1ms/step | 12.2 samples/s +Step 12600/44000 | Loss: 3.1192 | LR: 4.13e-04 | 1622.1ms/step | 12.3 samples/s +Step 12610/44000 | Loss: 3.1066 | LR: 4.13e-04 | 1568.0ms/step | 12.8 samples/s +Step 12620/44000 | Loss: 3.1191 | LR: 4.13e-04 | 1578.2ms/step | 12.7 samples/s +Step 12630/44000 | Loss: 3.0680 | LR: 4.13e-04 | 1599.0ms/step | 12.5 samples/s +Step 12640/44000 | Loss: 3.1474 | LR: 4.12e-04 | 1550.4ms/step | 12.9 samples/s +Step 12650/44000 | Loss: 3.0330 | LR: 4.12e-04 | 1528.2ms/step | 13.1 samples/s +Step 12660/44000 | Loss: 3.1323 | LR: 4.12e-04 | 1594.2ms/step | 12.5 samples/s +Step 12670/44000 | Loss: 3.0965 | LR: 4.12e-04 | 1515.8ms/step | 13.2 samples/s +Step 12680/44000 | Loss: 3.1366 | LR: 4.12e-04 | 1545.9ms/step | 12.9 samples/s +Step 12690/44000 | Loss: 3.1091 | LR: 4.12e-04 | 1541.0ms/step | 13.0 samples/s +Step 12700/44000 | Loss: 3.0549 | LR: 4.12e-04 | 1564.3ms/step | 12.8 samples/s +Step 12710/44000 | Loss: 3.0722 | LR: 4.11e-04 | 1553.2ms/step | 12.9 samples/s +Step 12720/44000 | Loss: 3.0855 | LR: 4.11e-04 | 1525.4ms/step | 13.1 samples/s +Step 12730/44000 | Loss: 3.0721 | LR: 4.11e-04 | 1521.2ms/step | 13.1 samples/s +Step 12740/44000 | Loss: 3.0733 | LR: 4.11e-04 | 1540.2ms/step | 13.0 samples/s +Step 12750/44000 | Loss: 3.0505 | LR: 4.11e-04 | 1576.7ms/step | 12.7 samples/s +Validation at step 12750: Loss = 3.0472, Perplexity = 21.06 +Step 12760/44000 | Loss: 3.0008 | LR: 4.11e-04 | 2143.6ms/step | 9.3 samples/s +Step 12770/44000 | Loss: 3.1015 | LR: 4.11e-04 | 1556.4ms/step | 12.9 samples/s +Step 12780/44000 | Loss: 3.0903 | LR: 4.10e-04 | 1583.1ms/step | 12.6 samples/s +Step 12790/44000 | Loss: 3.0095 | LR: 4.10e-04 | 1577.4ms/step | 12.7 samples/s +Step 12800/44000 | Loss: 3.1122 | LR: 4.10e-04 | 1613.9ms/step | 12.4 samples/s +Step 12810/44000 | Loss: 3.1248 | LR: 4.10e-04 | 1511.5ms/step | 13.2 samples/s +Step 12820/44000 | Loss: 3.0684 | LR: 4.10e-04 | 1573.1ms/step | 12.7 samples/s +Step 12830/44000 | Loss: 3.1181 | LR: 4.10e-04 | 1579.8ms/step | 12.7 samples/s +Step 12840/44000 | Loss: 3.1079 | LR: 4.10e-04 | 1550.8ms/step | 12.9 samples/s +Step 12850/44000 | Loss: 3.1295 | LR: 4.09e-04 | 1528.6ms/step | 13.1 samples/s +Step 12860/44000 | Loss: 3.0613 | LR: 4.09e-04 | 1520.6ms/step | 13.2 samples/s +Step 12870/44000 | Loss: 3.0547 | LR: 4.09e-04 | 1600.5ms/step | 12.5 samples/s +Step 12880/44000 | Loss: 3.0970 | LR: 4.09e-04 | 1771.8ms/step | 11.3 samples/s +Step 12890/44000 | Loss: 3.1255 | LR: 4.09e-04 | 1572.5ms/step | 12.7 samples/s +Step 12900/44000 | Loss: 3.0724 | LR: 4.09e-04 | 1576.0ms/step | 12.7 samples/s +Step 12910/44000 | Loss: 3.1207 | LR: 4.09e-04 | 1525.7ms/step | 13.1 samples/s +Step 12920/44000 | Loss: 3.0596 | LR: 4.09e-04 | 1535.1ms/step | 13.0 samples/s +Step 12930/44000 | Loss: 3.1000 | LR: 4.08e-04 | 1555.9ms/step | 12.9 samples/s +Step 12940/44000 | Loss: 3.1315 | LR: 4.08e-04 | 1560.0ms/step | 12.8 samples/s +Step 12950/44000 | Loss: 3.1363 | LR: 4.08e-04 | 1511.0ms/step | 13.2 samples/s +Step 12960/44000 | Loss: 3.0967 | LR: 4.08e-04 | 1551.5ms/step | 12.9 samples/s +Step 12970/44000 | Loss: 3.0751 | LR: 4.08e-04 | 1499.4ms/step | 13.3 samples/s +Step 12980/44000 | Loss: 3.0773 | LR: 4.08e-04 | 1555.4ms/step | 12.9 samples/s +Step 12990/44000 | Loss: 3.0669 | LR: 4.08e-04 | 1525.0ms/step | 13.1 samples/s +Step 13000/44000 | Loss: 3.1291 | LR: 4.07e-04 | 1554.9ms/step | 12.9 samples/s +Validation at step 13000: Loss = 3.0436, Perplexity = 20.98 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-13000 +Step 13010/44000 | Loss: 3.1046 | LR: 4.07e-04 | 2977.9ms/step | 6.7 samples/s +Step 13020/44000 | Loss: 3.1390 | LR: 4.07e-04 | 1650.8ms/step | 12.1 samples/s +Step 13030/44000 | Loss: 3.1265 | LR: 4.07e-04 | 1663.2ms/step | 12.0 samples/s +Step 13040/44000 | Loss: 3.0804 | LR: 4.07e-04 | 1571.8ms/step | 12.7 samples/s +Step 13050/44000 | Loss: 3.1212 | LR: 4.07e-04 | 1562.9ms/step | 12.8 samples/s +Step 13060/44000 | Loss: 3.0641 | LR: 4.07e-04 | 1586.9ms/step | 12.6 samples/s +Step 13070/44000 | Loss: 3.1229 | LR: 4.06e-04 | 1559.8ms/step | 12.8 samples/s +Step 13080/44000 | Loss: 3.0563 | LR: 4.06e-04 | 1567.2ms/step | 12.8 samples/s +Step 13090/44000 | Loss: 3.0793 | LR: 4.06e-04 | 1635.0ms/step | 12.2 samples/s +Step 13100/44000 | Loss: 3.1526 | LR: 4.06e-04 | 1591.1ms/step | 12.6 samples/s +Step 13110/44000 | Loss: 3.0383 | LR: 4.06e-04 | 1619.3ms/step | 12.4 samples/s +Step 13120/44000 | Loss: 3.0707 | LR: 4.06e-04 | 1602.6ms/step | 12.5 samples/s +Step 13130/44000 | Loss: 3.0450 | LR: 4.06e-04 | 1530.5ms/step | 13.1 samples/s +Step 13140/44000 | Loss: 3.0383 | LR: 4.05e-04 | 1528.2ms/step | 13.1 samples/s +Step 13150/44000 | Loss: 3.1332 | LR: 4.05e-04 | 1686.1ms/step | 11.9 samples/s +Step 13160/44000 | Loss: 3.1213 | LR: 4.05e-04 | 1663.3ms/step | 12.0 samples/s +Step 13170/44000 | Loss: 3.1745 | LR: 4.05e-04 | 1559.8ms/step | 12.8 samples/s +Step 13180/44000 | Loss: 3.1197 | LR: 4.05e-04 | 1589.3ms/step | 12.6 samples/s +Step 13190/44000 | Loss: 3.1149 | LR: 4.05e-04 | 1398.6ms/step | 14.3 samples/s +Step 13200/44000 | Loss: 3.0359 | LR: 4.05e-04 | 1519.9ms/step | 13.2 samples/s +Step 13210/44000 | Loss: 3.0631 | LR: 4.04e-04 | 1556.8ms/step | 12.8 samples/s +Step 13220/44000 | Loss: 3.0891 | LR: 4.04e-04 | 1603.4ms/step | 12.5 samples/s +Step 13230/44000 | Loss: 3.0533 | LR: 4.04e-04 | 1578.1ms/step | 12.7 samples/s +Step 13240/44000 | Loss: 3.1133 | LR: 4.04e-04 | 1557.7ms/step | 12.8 samples/s +Step 13250/44000 | Loss: 3.0697 | LR: 4.04e-04 | 1569.7ms/step | 12.7 samples/s +Validation at step 13250: Loss = 3.0380, Perplexity = 20.86 +Step 13260/44000 | Loss: 3.0825 | LR: 4.04e-04 | 2107.2ms/step | 9.5 samples/s +Step 13270/44000 | Loss: 3.0899 | LR: 4.04e-04 | 1519.7ms/step | 13.2 samples/s +Step 13280/44000 | Loss: 3.0926 | LR: 4.03e-04 | 1567.1ms/step | 12.8 samples/s +Step 13290/44000 | Loss: 3.0909 | LR: 4.03e-04 | 1686.1ms/step | 11.9 samples/s +Step 13300/44000 | Loss: 3.1020 | LR: 4.03e-04 | 1658.4ms/step | 12.1 samples/s +Step 13310/44000 | Loss: 3.0329 | LR: 4.03e-04 | 1554.1ms/step | 12.9 samples/s +Step 13320/44000 | Loss: 3.0681 | LR: 4.03e-04 | 1499.3ms/step | 13.3 samples/s +Step 13330/44000 | Loss: 3.1725 | LR: 4.03e-04 | 1544.6ms/step | 12.9 samples/s +Step 13340/44000 | Loss: 3.2051 | LR: 4.03e-04 | 1569.7ms/step | 12.7 samples/s +Step 13350/44000 | Loss: 3.1351 | LR: 4.02e-04 | 1538.9ms/step | 13.0 samples/s +Step 13360/44000 | Loss: 3.0539 | LR: 4.02e-04 | 1591.3ms/step | 12.6 samples/s +Step 13370/44000 | Loss: 3.1650 | LR: 4.02e-04 | 1572.8ms/step | 12.7 samples/s +Step 13380/44000 | Loss: 3.1082 | LR: 4.02e-04 | 1590.1ms/step | 12.6 samples/s +Step 13390/44000 | Loss: 3.0444 | LR: 4.02e-04 | 1551.7ms/step | 12.9 samples/s +Step 13400/44000 | Loss: 3.1210 | LR: 4.02e-04 | 1456.9ms/step | 13.7 samples/s +Step 13410/44000 | Loss: 3.1133 | LR: 4.02e-04 | 1529.6ms/step | 13.1 samples/s +Step 13420/44000 | Loss: 3.0941 | LR: 4.01e-04 | 1483.9ms/step | 13.5 samples/s +Step 13430/44000 | Loss: 3.1160 | LR: 4.01e-04 | 1535.1ms/step | 13.0 samples/s +Step 13440/44000 | Loss: 3.0858 | LR: 4.01e-04 | 1766.2ms/step | 11.3 samples/s +Step 13450/44000 | Loss: 3.1462 | LR: 4.01e-04 | 1535.9ms/step | 13.0 samples/s +Step 13460/44000 | Loss: 3.0612 | LR: 4.01e-04 | 1551.5ms/step | 12.9 samples/s +Step 13470/44000 | Loss: 3.0843 | LR: 4.01e-04 | 1556.5ms/step | 12.8 samples/s +Step 13480/44000 | Loss: 3.1458 | LR: 4.01e-04 | 1546.3ms/step | 12.9 samples/s +Step 13490/44000 | Loss: 3.0590 | LR: 4.00e-04 | 1548.6ms/step | 12.9 samples/s +Step 13500/44000 | Loss: 3.0428 | LR: 4.00e-04 | 1549.4ms/step | 12.9 samples/s +Validation at step 13500: Loss = 3.0343, Perplexity = 20.79 +Step 13510/44000 | Loss: 3.0918 | LR: 4.00e-04 | 2121.8ms/step | 9.4 samples/s +Step 13520/44000 | Loss: 3.0755 | LR: 4.00e-04 | 1564.2ms/step | 12.8 samples/s +Step 13530/44000 | Loss: 3.1326 | LR: 4.00e-04 | 1546.2ms/step | 12.9 samples/s +Step 13540/44000 | Loss: 3.1306 | LR: 4.00e-04 | 1558.1ms/step | 12.8 samples/s +Step 13550/44000 | Loss: 3.1151 | LR: 4.00e-04 | 1543.8ms/step | 13.0 samples/s +Step 13560/44000 | Loss: 3.0458 | LR: 3.99e-04 | 1542.4ms/step | 13.0 samples/s +Step 13570/44000 | Loss: 3.1445 | LR: 3.99e-04 | 1561.0ms/step | 12.8 samples/s +Step 13580/44000 | Loss: 3.0796 | LR: 3.99e-04 | 1804.1ms/step | 11.1 samples/s +Step 13590/44000 | Loss: 3.0694 | LR: 3.99e-04 | 1515.0ms/step | 13.2 samples/s +Step 13600/44000 | Loss: 3.1049 | LR: 3.99e-04 | 1570.5ms/step | 12.7 samples/s +Step 13610/44000 | Loss: 3.0060 | LR: 3.99e-04 | 1579.5ms/step | 12.7 samples/s +Step 13620/44000 | Loss: 3.1679 | LR: 3.98e-04 | 1590.0ms/step | 12.6 samples/s +Step 13630/44000 | Loss: 3.0473 | LR: 3.98e-04 | 1576.8ms/step | 12.7 samples/s +Step 13640/44000 | Loss: 3.1028 | LR: 3.98e-04 | 1545.7ms/step | 12.9 samples/s +Step 13650/44000 | Loss: 3.1331 | LR: 3.98e-04 | 1464.9ms/step | 13.7 samples/s +Step 13660/44000 | Loss: 3.0944 | LR: 3.98e-04 | 1445.4ms/step | 13.8 samples/s +Step 13670/44000 | Loss: 3.0786 | LR: 3.98e-04 | 1582.3ms/step | 12.6 samples/s +Step 13680/44000 | Loss: 3.1007 | LR: 3.98e-04 | 1515.7ms/step | 13.2 samples/s +Step 13690/44000 | Loss: 3.0188 | LR: 3.97e-04 | 1544.5ms/step | 12.9 samples/s +Step 13700/44000 | Loss: 3.1027 | LR: 3.97e-04 | 1548.3ms/step | 12.9 samples/s +Step 13710/44000 | Loss: 3.0741 | LR: 3.97e-04 | 1577.4ms/step | 12.7 samples/s +Step 13720/44000 | Loss: 3.0661 | LR: 3.97e-04 | 1708.8ms/step | 11.7 samples/s +Step 13730/44000 | Loss: 3.0788 | LR: 3.97e-04 | 1624.8ms/step | 12.3 samples/s +Step 13740/44000 | Loss: 3.0745 | LR: 3.97e-04 | 1526.8ms/step | 13.1 samples/s +Step 13750/44000 | Loss: 3.0970 | LR: 3.97e-04 | 1529.6ms/step | 13.1 samples/s +Validation at step 13750: Loss = 3.0305, Perplexity = 20.71 +Step 13760/44000 | Loss: 3.0000 | LR: 3.96e-04 | 2154.8ms/step | 9.3 samples/s +Step 13770/44000 | Loss: 3.0913 | LR: 3.96e-04 | 1593.5ms/step | 12.6 samples/s +Step 13780/44000 | Loss: 3.1130 | LR: 3.96e-04 | 1547.7ms/step | 12.9 samples/s +Step 13790/44000 | Loss: 3.1619 | LR: 3.96e-04 | 1586.9ms/step | 12.6 samples/s +Step 13800/44000 | Loss: 3.0476 | LR: 3.96e-04 | 1575.6ms/step | 12.7 samples/s +Step 13810/44000 | Loss: 3.0687 | LR: 3.96e-04 | 1525.9ms/step | 13.1 samples/s +Step 13820/44000 | Loss: 3.0497 | LR: 3.96e-04 | 1582.9ms/step | 12.6 samples/s +Step 13830/44000 | Loss: 3.0761 | LR: 3.95e-04 | 1541.1ms/step | 13.0 samples/s +Step 13840/44000 | Loss: 3.0446 | LR: 3.95e-04 | 1512.0ms/step | 13.2 samples/s +Step 13850/44000 | Loss: 3.0836 | LR: 3.95e-04 | 1527.9ms/step | 13.1 samples/s +Step 13860/44000 | Loss: 3.0848 | LR: 3.95e-04 | 1734.0ms/step | 11.5 samples/s +Step 13870/44000 | Loss: 3.1210 | LR: 3.95e-04 | 1574.8ms/step | 12.7 samples/s +Step 13880/44000 | Loss: 3.1242 | LR: 3.95e-04 | 1541.9ms/step | 13.0 samples/s +Step 13890/44000 | Loss: 3.0460 | LR: 3.95e-04 | 1498.5ms/step | 13.3 samples/s +Step 13900/44000 | Loss: 3.0764 | LR: 3.94e-04 | 1537.4ms/step | 13.0 samples/s +Step 13910/44000 | Loss: 3.0899 | LR: 3.94e-04 | 1643.4ms/step | 12.2 samples/s +Step 13920/44000 | Loss: 3.0702 | LR: 3.94e-04 | 1560.4ms/step | 12.8 samples/s +Step 13930/44000 | Loss: 3.0082 | LR: 3.94e-04 | 1552.7ms/step | 12.9 samples/s +Step 13940/44000 | Loss: 3.0745 | LR: 3.94e-04 | 1551.7ms/step | 12.9 samples/s +Step 13950/44000 | Loss: 3.0420 | LR: 3.94e-04 | 1540.3ms/step | 13.0 samples/s +Step 13960/44000 | Loss: 3.0775 | LR: 3.93e-04 | 1552.1ms/step | 12.9 samples/s +Step 13970/44000 | Loss: 3.0528 | LR: 3.93e-04 | 1524.6ms/step | 13.1 samples/s +Step 13980/44000 | Loss: 3.1142 | LR: 3.93e-04 | 1564.5ms/step | 12.8 samples/s +Step 13990/44000 | Loss: 3.0677 | LR: 3.93e-04 | 1591.0ms/step | 12.6 samples/s +Step 14000/44000 | Loss: 2.9986 | LR: 3.93e-04 | 1669.3ms/step | 12.0 samples/s +Validation at step 14000: Loss = 3.0256, Perplexity = 20.61 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-14000 +Step 14010/44000 | Loss: 3.0196 | LR: 3.93e-04 | 2888.3ms/step | 6.9 samples/s +Step 14020/44000 | Loss: 3.0423 | LR: 3.93e-04 | 1566.4ms/step | 12.8 samples/s +Step 14030/44000 | Loss: 3.1324 | LR: 3.92e-04 | 1526.5ms/step | 13.1 samples/s +Step 14040/44000 | Loss: 3.0516 | LR: 3.92e-04 | 1550.7ms/step | 12.9 samples/s +Step 14050/44000 | Loss: 3.0523 | LR: 3.92e-04 | 1615.6ms/step | 12.4 samples/s +Step 14060/44000 | Loss: 3.0701 | LR: 3.92e-04 | 1499.1ms/step | 13.3 samples/s +Step 14070/44000 | Loss: 3.0799 | LR: 3.92e-04 | 1557.3ms/step | 12.8 samples/s +Step 14080/44000 | Loss: 3.1350 | LR: 3.92e-04 | 1610.5ms/step | 12.4 samples/s +Step 14090/44000 | Loss: 3.0806 | LR: 3.92e-04 | 1550.9ms/step | 12.9 samples/s +Step 14100/44000 | Loss: 3.0978 | LR: 3.91e-04 | 1508.5ms/step | 13.3 samples/s +Step 14110/44000 | Loss: 3.0063 | LR: 3.91e-04 | 1430.2ms/step | 14.0 samples/s +Step 14120/44000 | Loss: 3.0770 | LR: 3.91e-04 | 1436.6ms/step | 13.9 samples/s +Step 14130/44000 | Loss: 3.0257 | LR: 3.91e-04 | 1543.2ms/step | 13.0 samples/s +Step 14140/44000 | Loss: 3.0671 | LR: 3.91e-04 | 1763.6ms/step | 11.3 samples/s +Step 14150/44000 | Loss: 3.0562 | LR: 3.91e-04 | 1598.6ms/step | 12.5 samples/s +Step 14160/44000 | Loss: 3.2028 | LR: 3.90e-04 | 1641.0ms/step | 12.2 samples/s +Step 14170/44000 | Loss: 3.0820 | LR: 3.90e-04 | 1587.1ms/step | 12.6 samples/s +Step 14180/44000 | Loss: 3.0102 | LR: 3.90e-04 | 1574.5ms/step | 12.7 samples/s +Step 14190/44000 | Loss: 3.1196 | LR: 3.90e-04 | 1577.1ms/step | 12.7 samples/s +Step 14200/44000 | Loss: 3.1677 | LR: 3.90e-04 | 1577.7ms/step | 12.7 samples/s +Step 14210/44000 | Loss: 3.1037 | LR: 3.90e-04 | 1525.7ms/step | 13.1 samples/s +Step 14220/44000 | Loss: 3.0886 | LR: 3.90e-04 | 1590.9ms/step | 12.6 samples/s +Step 14230/44000 | Loss: 3.1008 | LR: 3.89e-04 | 1519.6ms/step | 13.2 samples/s +Step 14240/44000 | Loss: 3.0391 | LR: 3.89e-04 | 1556.0ms/step | 12.9 samples/s +Step 14250/44000 | Loss: 3.0585 | LR: 3.89e-04 | 1554.5ms/step | 12.9 samples/s +Validation at step 14250: Loss = 3.0229, Perplexity = 20.55 +Step 14260/44000 | Loss: 3.1035 | LR: 3.89e-04 | 2093.3ms/step | 9.6 samples/s +Step 14270/44000 | Loss: 2.9720 | LR: 3.89e-04 | 1517.3ms/step | 13.2 samples/s +Step 14280/44000 | Loss: 3.0304 | LR: 3.89e-04 | 1783.7ms/step | 11.2 samples/s +Step 14290/44000 | Loss: 2.9982 | LR: 3.89e-04 | 1575.2ms/step | 12.7 samples/s +Step 14300/44000 | Loss: 2.9847 | LR: 3.88e-04 | 1553.2ms/step | 12.9 samples/s +Step 14310/44000 | Loss: 3.0376 | LR: 3.88e-04 | 1549.9ms/step | 12.9 samples/s +Step 14320/44000 | Loss: 3.0266 | LR: 3.88e-04 | 1530.9ms/step | 13.1 samples/s +Step 14330/44000 | Loss: 3.1284 | LR: 3.88e-04 | 1515.8ms/step | 13.2 samples/s +Step 14340/44000 | Loss: 3.0751 | LR: 3.88e-04 | 1525.3ms/step | 13.1 samples/s +Step 14350/44000 | Loss: 3.0171 | LR: 3.88e-04 | 1518.6ms/step | 13.2 samples/s +Step 14360/44000 | Loss: 3.0704 | LR: 3.87e-04 | 1548.9ms/step | 12.9 samples/s +Step 14370/44000 | Loss: 3.1114 | LR: 3.87e-04 | 1548.4ms/step | 12.9 samples/s +Step 14380/44000 | Loss: 3.0471 | LR: 3.87e-04 | 1556.2ms/step | 12.9 samples/s +Step 14390/44000 | Loss: 3.0470 | LR: 3.87e-04 | 1488.0ms/step | 13.4 samples/s +Step 14400/44000 | Loss: 3.1183 | LR: 3.87e-04 | 1525.0ms/step | 13.1 samples/s +Step 14410/44000 | Loss: 3.1013 | LR: 3.87e-04 | 1515.1ms/step | 13.2 samples/s +Step 14420/44000 | Loss: 3.0808 | LR: 3.87e-04 | 1588.7ms/step | 12.6 samples/s +Step 14430/44000 | Loss: 3.0016 | LR: 3.86e-04 | 1737.6ms/step | 11.5 samples/s +Step 14440/44000 | Loss: 3.0867 | LR: 3.86e-04 | 1559.7ms/step | 12.8 samples/s +Step 14450/44000 | Loss: 3.0706 | LR: 3.86e-04 | 1525.3ms/step | 13.1 samples/s +Step 14460/44000 | Loss: 3.1018 | LR: 3.86e-04 | 1476.5ms/step | 13.5 samples/s +Step 14470/44000 | Loss: 3.1211 | LR: 3.86e-04 | 1505.3ms/step | 13.3 samples/s +Step 14480/44000 | Loss: 3.0525 | LR: 3.86e-04 | 1498.9ms/step | 13.3 samples/s +Step 14490/44000 | Loss: 3.0614 | LR: 3.85e-04 | 1610.3ms/step | 12.4 samples/s +Step 14500/44000 | Loss: 3.0818 | LR: 3.85e-04 | 1758.1ms/step | 11.4 samples/s +Validation at step 14500: Loss = 3.0200, Perplexity = 20.49 +Step 14510/44000 | Loss: 3.1163 | LR: 3.85e-04 | 2193.1ms/step | 9.1 samples/s +Step 14520/44000 | Loss: 3.1042 | LR: 3.85e-04 | 1571.7ms/step | 12.7 samples/s +Step 14530/44000 | Loss: 3.0889 | LR: 3.85e-04 | 1541.7ms/step | 13.0 samples/s +Step 14540/44000 | Loss: 3.0835 | LR: 3.85e-04 | 1601.2ms/step | 12.5 samples/s +Step 14550/44000 | Loss: 3.0891 | LR: 3.85e-04 | 1586.2ms/step | 12.6 samples/s +Step 14560/44000 | Loss: 3.0817 | LR: 3.84e-04 | 1625.6ms/step | 12.3 samples/s +Step 14570/44000 | Loss: 3.0652 | LR: 3.84e-04 | 1605.7ms/step | 12.5 samples/s +Step 14580/44000 | Loss: 3.0698 | LR: 3.84e-04 | 1405.9ms/step | 14.2 samples/s +Step 14590/44000 | Loss: 3.0500 | LR: 3.84e-04 | 1539.6ms/step | 13.0 samples/s +Step 14600/44000 | Loss: 3.0916 | LR: 3.84e-04 | 1552.9ms/step | 12.9 samples/s +Step 14610/44000 | Loss: 3.0894 | LR: 3.84e-04 | 1550.7ms/step | 12.9 samples/s +Step 14620/44000 | Loss: 3.0947 | LR: 3.83e-04 | 1555.6ms/step | 12.9 samples/s +Step 14630/44000 | Loss: 3.0035 | LR: 3.83e-04 | 1537.7ms/step | 13.0 samples/s +Step 14640/44000 | Loss: 3.0953 | LR: 3.83e-04 | 1561.0ms/step | 12.8 samples/s +Step 14650/44000 | Loss: 2.9829 | LR: 3.83e-04 | 1552.7ms/step | 12.9 samples/s +Step 14660/44000 | Loss: 3.0705 | LR: 3.83e-04 | 1601.2ms/step | 12.5 samples/s +Step 14670/44000 | Loss: 3.0376 | LR: 3.83e-04 | 1574.0ms/step | 12.7 samples/s +Step 14680/44000 | Loss: 3.1083 | LR: 3.83e-04 | 1546.1ms/step | 12.9 samples/s +Step 14690/44000 | Loss: 2.9979 | LR: 3.82e-04 | 1566.2ms/step | 12.8 samples/s +Step 14700/44000 | Loss: 3.0319 | LR: 3.82e-04 | 1536.1ms/step | 13.0 samples/s +Step 14710/44000 | Loss: 3.0440 | LR: 3.82e-04 | 1809.5ms/step | 11.1 samples/s +Step 14720/44000 | Loss: 3.0515 | LR: 3.82e-04 | 1486.4ms/step | 13.5 samples/s +Step 14730/44000 | Loss: 3.0616 | LR: 3.82e-04 | 1587.1ms/step | 12.6 samples/s +Step 14740/44000 | Loss: 3.0444 | LR: 3.82e-04 | 1615.5ms/step | 12.4 samples/s +Step 14750/44000 | Loss: 3.0724 | LR: 3.81e-04 | 1578.1ms/step | 12.7 samples/s +Validation at step 14750: Loss = 3.0163, Perplexity = 20.41 +Step 14760/44000 | Loss: 3.0505 | LR: 3.81e-04 | 2158.9ms/step | 9.3 samples/s +Step 14770/44000 | Loss: 3.1021 | LR: 3.81e-04 | 1525.0ms/step | 13.1 samples/s +Step 14780/44000 | Loss: 3.1255 | LR: 3.81e-04 | 1563.9ms/step | 12.8 samples/s +Step 14790/44000 | Loss: 3.0657 | LR: 3.81e-04 | 1546.9ms/step | 12.9 samples/s +Step 14800/44000 | Loss: 3.0770 | LR: 3.81e-04 | 1550.5ms/step | 12.9 samples/s +Step 14810/44000 | Loss: 3.1018 | LR: 3.81e-04 | 1621.4ms/step | 12.3 samples/s +Step 14820/44000 | Loss: 3.0226 | LR: 3.80e-04 | 1547.2ms/step | 12.9 samples/s +Step 14830/44000 | Loss: 3.0018 | LR: 3.80e-04 | 1567.0ms/step | 12.8 samples/s +Step 14840/44000 | Loss: 3.0443 | LR: 3.80e-04 | 1545.5ms/step | 12.9 samples/s +Step 14850/44000 | Loss: 3.0826 | LR: 3.80e-04 | 1846.2ms/step | 10.8 samples/s +Step 14860/44000 | Loss: 3.0529 | LR: 3.80e-04 | 1474.3ms/step | 13.6 samples/s +Step 14870/44000 | Loss: 3.0717 | LR: 3.80e-04 | 1507.3ms/step | 13.3 samples/s +Step 14880/44000 | Loss: 3.0803 | LR: 3.79e-04 | 1551.9ms/step | 12.9 samples/s +Step 14890/44000 | Loss: 3.0435 | LR: 3.79e-04 | 1578.4ms/step | 12.7 samples/s +Step 14900/44000 | Loss: 2.9978 | LR: 3.79e-04 | 1551.8ms/step | 12.9 samples/s +Step 14910/44000 | Loss: 3.0546 | LR: 3.79e-04 | 1541.1ms/step | 13.0 samples/s +Step 14920/44000 | Loss: 3.0367 | LR: 3.79e-04 | 1552.3ms/step | 12.9 samples/s +Step 14930/44000 | Loss: 3.1012 | LR: 3.79e-04 | 1515.1ms/step | 13.2 samples/s +Step 14940/44000 | Loss: 3.0849 | LR: 3.79e-04 | 1561.1ms/step | 12.8 samples/s +Step 14950/44000 | Loss: 3.1046 | LR: 3.78e-04 | 1568.1ms/step | 12.8 samples/s +Step 14960/44000 | Loss: 3.0606 | LR: 3.78e-04 | 1510.8ms/step | 13.2 samples/s +Step 14970/44000 | Loss: 3.0484 | LR: 3.78e-04 | 1550.9ms/step | 12.9 samples/s +Step 14980/44000 | Loss: 3.0400 | LR: 3.78e-04 | 1522.2ms/step | 13.1 samples/s +Step 14990/44000 | Loss: 3.0769 | LR: 3.78e-04 | 1679.8ms/step | 11.9 samples/s +Step 15000/44000 | Loss: 3.0941 | LR: 3.78e-04 | 1625.2ms/step | 12.3 samples/s +Validation at step 15000: Loss = 3.0140, Perplexity = 20.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-15000 +Step 15010/44000 | Loss: 3.0736 | LR: 3.77e-04 | 2757.3ms/step | 7.3 samples/s +Step 15020/44000 | Loss: 3.0679 | LR: 3.77e-04 | 1504.8ms/step | 13.3 samples/s +Step 15030/44000 | Loss: 3.1009 | LR: 3.77e-04 | 1473.2ms/step | 13.6 samples/s +Step 15040/44000 | Loss: 3.0447 | LR: 3.77e-04 | 1545.5ms/step | 12.9 samples/s +Step 15050/44000 | Loss: 3.1040 | LR: 3.77e-04 | 1587.4ms/step | 12.6 samples/s +Step 15060/44000 | Loss: 3.0706 | LR: 3.77e-04 | 1540.8ms/step | 13.0 samples/s +Step 15070/44000 | Loss: 3.0558 | LR: 3.77e-04 | 1591.1ms/step | 12.6 samples/s +Step 15080/44000 | Loss: 3.0926 | LR: 3.76e-04 | 1589.0ms/step | 12.6 samples/s +Step 15090/44000 | Loss: 3.0767 | LR: 3.76e-04 | 1554.2ms/step | 12.9 samples/s +Step 15100/44000 | Loss: 3.0600 | LR: 3.76e-04 | 1616.2ms/step | 12.4 samples/s +Step 15110/44000 | Loss: 3.0274 | LR: 3.76e-04 | 1567.0ms/step | 12.8 samples/s +Step 15120/44000 | Loss: 3.0851 | LR: 3.76e-04 | 1534.0ms/step | 13.0 samples/s +Step 15130/44000 | Loss: 3.0612 | LR: 3.76e-04 | 1811.1ms/step | 11.0 samples/s +Step 15140/44000 | Loss: 3.0251 | LR: 3.75e-04 | 1591.8ms/step | 12.6 samples/s +Step 15150/44000 | Loss: 3.0842 | LR: 3.75e-04 | 1598.7ms/step | 12.5 samples/s +Step 15160/44000 | Loss: 3.0460 | LR: 3.75e-04 | 1543.4ms/step | 13.0 samples/s +Step 15170/44000 | Loss: 3.0344 | LR: 3.75e-04 | 1492.9ms/step | 13.4 samples/s +Step 15180/44000 | Loss: 3.0377 | LR: 3.75e-04 | 1590.0ms/step | 12.6 samples/s +Step 15190/44000 | Loss: 3.0914 | LR: 3.75e-04 | 1578.4ms/step | 12.7 samples/s +Step 15200/44000 | Loss: 3.1117 | LR: 3.74e-04 | 1539.3ms/step | 13.0 samples/s +Step 15210/44000 | Loss: 3.0286 | LR: 3.74e-04 | 1542.9ms/step | 13.0 samples/s +Step 15220/44000 | Loss: 3.0619 | LR: 3.74e-04 | 1541.2ms/step | 13.0 samples/s +Step 15230/44000 | Loss: 3.0428 | LR: 3.74e-04 | 1533.3ms/step | 13.0 samples/s +Step 15240/44000 | Loss: 3.1402 | LR: 3.74e-04 | 1559.4ms/step | 12.8 samples/s +Step 15250/44000 | Loss: 3.0239 | LR: 3.74e-04 | 1572.4ms/step | 12.7 samples/s +Validation at step 15250: Loss = 3.0110, Perplexity = 20.31 +Step 15260/44000 | Loss: 3.0403 | LR: 3.74e-04 | 2233.3ms/step | 9.0 samples/s +Step 15270/44000 | Loss: 3.0525 | LR: 3.73e-04 | 1803.4ms/step | 11.1 samples/s +Step 15280/44000 | Loss: 3.0296 | LR: 3.73e-04 | 1656.2ms/step | 12.1 samples/s +Step 15290/44000 | Loss: 3.0081 | LR: 3.73e-04 | 1516.7ms/step | 13.2 samples/s +Step 15300/44000 | Loss: 3.0877 | LR: 3.73e-04 | 1628.2ms/step | 12.3 samples/s +Step 15310/44000 | Loss: 2.9915 | LR: 3.73e-04 | 1609.1ms/step | 12.4 samples/s +Step 15320/44000 | Loss: 3.0660 | LR: 3.73e-04 | 1596.2ms/step | 12.5 samples/s +Step 15330/44000 | Loss: 3.0378 | LR: 3.72e-04 | 1553.8ms/step | 12.9 samples/s +Step 15340/44000 | Loss: 3.1005 | LR: 3.72e-04 | 1542.2ms/step | 13.0 samples/s +Step 15350/44000 | Loss: 3.0085 | LR: 3.72e-04 | 1537.1ms/step | 13.0 samples/s +Step 15360/44000 | Loss: 3.0820 | LR: 3.72e-04 | 1606.8ms/step | 12.4 samples/s +Step 15370/44000 | Loss: 3.0310 | LR: 3.72e-04 | 1558.4ms/step | 12.8 samples/s +Step 15380/44000 | Loss: 3.0278 | LR: 3.72e-04 | 1509.7ms/step | 13.2 samples/s +Step 15390/44000 | Loss: 3.1335 | LR: 3.71e-04 | 1545.4ms/step | 12.9 samples/s +Step 15400/44000 | Loss: 3.0418 | LR: 3.71e-04 | 1525.4ms/step | 13.1 samples/s +Step 15410/44000 | Loss: 2.9982 | LR: 3.71e-04 | 1784.6ms/step | 11.2 samples/s +Step 15420/44000 | Loss: 3.0172 | LR: 3.71e-04 | 1557.4ms/step | 12.8 samples/s +Step 15430/44000 | Loss: 3.1040 | LR: 3.71e-04 | 1596.0ms/step | 12.5 samples/s +Step 15440/44000 | Loss: 3.0052 | LR: 3.71e-04 | 1581.1ms/step | 12.6 samples/s +Step 15450/44000 | Loss: 3.1051 | LR: 3.71e-04 | 1541.8ms/step | 13.0 samples/s +Step 15460/44000 | Loss: 3.0842 | LR: 3.70e-04 | 1551.0ms/step | 12.9 samples/s +Step 15470/44000 | Loss: 3.0543 | LR: 3.70e-04 | 1544.5ms/step | 12.9 samples/s +Step 15480/44000 | Loss: 3.0386 | LR: 3.70e-04 | 1541.2ms/step | 13.0 samples/s +Step 15490/44000 | Loss: 3.0889 | LR: 3.70e-04 | 1337.8ms/step | 15.0 samples/s +Step 15500/44000 | Loss: 3.1079 | LR: 3.70e-04 | 1525.6ms/step | 13.1 samples/s +Validation at step 15500: Loss = 3.0079, Perplexity = 20.24 +Step 15510/44000 | Loss: 3.0456 | LR: 3.70e-04 | 2106.6ms/step | 9.5 samples/s +Step 15520/44000 | Loss: 3.0764 | LR: 3.69e-04 | 1567.1ms/step | 12.8 samples/s +Step 15530/44000 | Loss: 3.0421 | LR: 3.69e-04 | 1562.2ms/step | 12.8 samples/s +Step 15540/44000 | Loss: 3.1105 | LR: 3.69e-04 | 1514.8ms/step | 13.2 samples/s +Step 15550/44000 | Loss: 3.0226 | LR: 3.69e-04 | 1750.4ms/step | 11.4 samples/s +Step 15560/44000 | Loss: 3.0449 | LR: 3.69e-04 | 1535.9ms/step | 13.0 samples/s +Step 15570/44000 | Loss: 3.0651 | LR: 3.69e-04 | 1470.5ms/step | 13.6 samples/s +Step 15580/44000 | Loss: 3.0495 | LR: 3.68e-04 | 1516.6ms/step | 13.2 samples/s +Step 15590/44000 | Loss: 3.0533 | LR: 3.68e-04 | 1535.6ms/step | 13.0 samples/s +Step 15600/44000 | Loss: 3.1023 | LR: 3.68e-04 | 1537.1ms/step | 13.0 samples/s +Step 15610/44000 | Loss: 3.0813 | LR: 3.68e-04 | 1535.5ms/step | 13.0 samples/s +Step 15620/44000 | Loss: 3.0314 | LR: 3.68e-04 | 1582.4ms/step | 12.6 samples/s +Step 15630/44000 | Loss: 2.9879 | LR: 3.68e-04 | 1505.0ms/step | 13.3 samples/s +Step 15640/44000 | Loss: 3.0161 | LR: 3.67e-04 | 1550.9ms/step | 12.9 samples/s +Step 15650/44000 | Loss: 3.0830 | LR: 3.67e-04 | 1605.1ms/step | 12.5 samples/s +Step 15660/44000 | Loss: 3.0062 | LR: 3.67e-04 | 1491.1ms/step | 13.4 samples/s +Step 15670/44000 | Loss: 3.0376 | LR: 3.67e-04 | 1505.1ms/step | 13.3 samples/s +Step 15680/44000 | Loss: 3.0868 | LR: 3.67e-04 | 1524.7ms/step | 13.1 samples/s +Step 15690/44000 | Loss: 3.0600 | LR: 3.67e-04 | 1666.3ms/step | 12.0 samples/s +Step 15700/44000 | Loss: 3.0390 | LR: 3.67e-04 | 1723.2ms/step | 11.6 samples/s +Step 15710/44000 | Loss: 3.1077 | LR: 3.66e-04 | 1530.5ms/step | 13.1 samples/s +Step 15720/44000 | Loss: 3.0984 | LR: 3.66e-04 | 1546.8ms/step | 12.9 samples/s +Step 15730/44000 | Loss: 2.9834 | LR: 3.66e-04 | 1585.4ms/step | 12.6 samples/s +Step 15740/44000 | Loss: 3.0361 | LR: 3.66e-04 | 1541.6ms/step | 13.0 samples/s +Step 15750/44000 | Loss: 3.0328 | LR: 3.66e-04 | 1532.3ms/step | 13.1 samples/s +Validation at step 15750: Loss = 3.0043, Perplexity = 20.17 +Step 15760/44000 | Loss: 3.0516 | LR: 3.66e-04 | 2168.3ms/step | 9.2 samples/s +Step 15770/44000 | Loss: 3.0458 | LR: 3.65e-04 | 1548.0ms/step | 12.9 samples/s +Step 15780/44000 | Loss: 3.0561 | LR: 3.65e-04 | 1568.6ms/step | 12.8 samples/s +Step 15790/44000 | Loss: 3.0705 | LR: 3.65e-04 | 1564.7ms/step | 12.8 samples/s +Step 15800/44000 | Loss: 3.0869 | LR: 3.65e-04 | 1550.3ms/step | 12.9 samples/s +Step 15810/44000 | Loss: 3.0283 | LR: 3.65e-04 | 1523.4ms/step | 13.1 samples/s +Step 15820/44000 | Loss: 3.0975 | LR: 3.65e-04 | 1521.0ms/step | 13.1 samples/s +Step 15830/44000 | Loss: 3.0720 | LR: 3.64e-04 | 1625.8ms/step | 12.3 samples/s +Step 15840/44000 | Loss: 3.0796 | LR: 3.64e-04 | 1670.9ms/step | 12.0 samples/s +Step 15850/44000 | Loss: 3.0791 | LR: 3.64e-04 | 1562.2ms/step | 12.8 samples/s +Step 15860/44000 | Loss: 3.0839 | LR: 3.64e-04 | 1509.5ms/step | 13.2 samples/s +Step 15870/44000 | Loss: 3.1054 | LR: 3.64e-04 | 1551.9ms/step | 12.9 samples/s +Step 15880/44000 | Loss: 3.0187 | LR: 3.64e-04 | 1530.7ms/step | 13.1 samples/s +Step 15890/44000 | Loss: 2.9999 | LR: 3.63e-04 | 1514.6ms/step | 13.2 samples/s +Step 15900/44000 | Loss: 3.1119 | LR: 3.63e-04 | 1544.5ms/step | 12.9 samples/s +Step 15910/44000 | Loss: 3.0806 | LR: 3.63e-04 | 1527.9ms/step | 13.1 samples/s +Step 15920/44000 | Loss: 3.0242 | LR: 3.63e-04 | 1556.7ms/step | 12.8 samples/s +Step 15930/44000 | Loss: 3.0968 | LR: 3.63e-04 | 1546.2ms/step | 12.9 samples/s +Step 15940/44000 | Loss: 3.0194 | LR: 3.63e-04 | 1556.8ms/step | 12.8 samples/s +Step 15950/44000 | Loss: 3.0102 | LR: 3.62e-04 | 1427.6ms/step | 14.0 samples/s +Step 15960/44000 | Loss: 3.0307 | LR: 3.62e-04 | 1386.5ms/step | 14.4 samples/s +Step 15970/44000 | Loss: 3.0430 | LR: 3.62e-04 | 1594.0ms/step | 12.5 samples/s +Step 15980/44000 | Loss: 3.0082 | LR: 3.62e-04 | 1745.9ms/step | 11.5 samples/s +Step 15990/44000 | Loss: 3.0715 | LR: 3.62e-04 | 1591.9ms/step | 12.6 samples/s +Step 16000/44000 | Loss: 2.9877 | LR: 3.62e-04 | 1532.4ms/step | 13.1 samples/s +Validation at step 16000: Loss = 3.0013, Perplexity = 20.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-16000 +Step 16010/44000 | Loss: 3.0636 | LR: 3.62e-04 | 2824.5ms/step | 7.1 samples/s +Step 16020/44000 | Loss: 3.0573 | LR: 3.61e-04 | 1570.5ms/step | 12.7 samples/s +Step 16030/44000 | Loss: 3.0044 | LR: 3.61e-04 | 1508.8ms/step | 13.3 samples/s +Step 16040/44000 | Loss: 3.1749 | LR: 3.61e-04 | 1611.1ms/step | 12.4 samples/s +Step 16050/44000 | Loss: 3.0456 | LR: 3.61e-04 | 1623.5ms/step | 12.3 samples/s +Step 16060/44000 | Loss: 3.0988 | LR: 3.61e-04 | 1591.9ms/step | 12.6 samples/s +Step 16070/44000 | Loss: 3.0840 | LR: 3.61e-04 | 1551.8ms/step | 12.9 samples/s +Step 16080/44000 | Loss: 3.0115 | LR: 3.60e-04 | 1518.1ms/step | 13.2 samples/s +Step 16090/44000 | Loss: 3.0096 | LR: 3.60e-04 | 1494.8ms/step | 13.4 samples/s +Step 16100/44000 | Loss: 3.0410 | LR: 3.60e-04 | 1542.9ms/step | 13.0 samples/s +Step 16110/44000 | Loss: 3.0203 | LR: 3.60e-04 | 1673.0ms/step | 12.0 samples/s +Step 16120/44000 | Loss: 3.0098 | LR: 3.60e-04 | 1662.2ms/step | 12.0 samples/s +Step 16130/44000 | Loss: 3.0820 | LR: 3.60e-04 | 1556.7ms/step | 12.8 samples/s +Step 16140/44000 | Loss: 2.9988 | LR: 3.59e-04 | 1557.0ms/step | 12.8 samples/s +Step 16150/44000 | Loss: 3.0664 | LR: 3.59e-04 | 1523.6ms/step | 13.1 samples/s +Step 16160/44000 | Loss: 3.0274 | LR: 3.59e-04 | 1490.3ms/step | 13.4 samples/s +Step 16170/44000 | Loss: 3.0531 | LR: 3.59e-04 | 1468.0ms/step | 13.6 samples/s +Step 16180/44000 | Loss: 3.0819 | LR: 3.59e-04 | 1588.3ms/step | 12.6 samples/s +Step 16190/44000 | Loss: 3.1540 | LR: 3.59e-04 | 1504.5ms/step | 13.3 samples/s +Step 16200/44000 | Loss: 3.0150 | LR: 3.58e-04 | 1553.2ms/step | 12.9 samples/s +Step 16210/44000 | Loss: 3.0383 | LR: 3.58e-04 | 1570.9ms/step | 12.7 samples/s +Step 16220/44000 | Loss: 3.0366 | LR: 3.58e-04 | 1572.3ms/step | 12.7 samples/s +Step 16230/44000 | Loss: 2.9625 | LR: 3.58e-04 | 1520.4ms/step | 13.2 samples/s +Step 16240/44000 | Loss: 3.1000 | LR: 3.58e-04 | 1515.2ms/step | 13.2 samples/s +Step 16250/44000 | Loss: 3.0805 | LR: 3.58e-04 | 1569.5ms/step | 12.7 samples/s +Validation at step 16250: Loss = 2.9988, Perplexity = 20.06 +Step 16260/44000 | Loss: 2.9952 | LR: 3.57e-04 | 2353.1ms/step | 8.5 samples/s +Step 16270/44000 | Loss: 3.0567 | LR: 3.57e-04 | 1562.4ms/step | 12.8 samples/s +Step 16280/44000 | Loss: 2.9861 | LR: 3.57e-04 | 1529.7ms/step | 13.1 samples/s +Step 16290/44000 | Loss: 2.9933 | LR: 3.57e-04 | 1546.1ms/step | 12.9 samples/s +Step 16300/44000 | Loss: 3.0901 | LR: 3.57e-04 | 1527.5ms/step | 13.1 samples/s +Step 16310/44000 | Loss: 3.0415 | LR: 3.57e-04 | 1560.5ms/step | 12.8 samples/s +Step 16320/44000 | Loss: 3.0469 | LR: 3.56e-04 | 1530.4ms/step | 13.1 samples/s +Step 16330/44000 | Loss: 2.9751 | LR: 3.56e-04 | 1580.7ms/step | 12.7 samples/s +Step 16340/44000 | Loss: 3.0217 | LR: 3.56e-04 | 1518.5ms/step | 13.2 samples/s +Step 16350/44000 | Loss: 3.0810 | LR: 3.56e-04 | 1537.4ms/step | 13.0 samples/s +Step 16360/44000 | Loss: 3.0139 | LR: 3.56e-04 | 1563.7ms/step | 12.8 samples/s +Step 16370/44000 | Loss: 3.0215 | LR: 3.56e-04 | 1563.3ms/step | 12.8 samples/s +Step 16380/44000 | Loss: 3.0887 | LR: 3.55e-04 | 1562.1ms/step | 12.8 samples/s +Step 16390/44000 | Loss: 2.9535 | LR: 3.55e-04 | 1521.0ms/step | 13.1 samples/s +Step 16400/44000 | Loss: 3.0481 | LR: 3.55e-04 | 1542.5ms/step | 13.0 samples/s +Step 16410/44000 | Loss: 3.0641 | LR: 3.55e-04 | 1267.2ms/step | 15.8 samples/s +Step 16420/44000 | Loss: 3.0277 | LR: 3.55e-04 | 1312.3ms/step | 15.2 samples/s +Step 16430/44000 | Loss: 3.0764 | LR: 3.55e-04 | 1261.3ms/step | 15.9 samples/s +Step 16440/44000 | Loss: 3.0052 | LR: 3.54e-04 | 1311.5ms/step | 15.2 samples/s +Step 16450/44000 | Loss: 3.0989 | LR: 3.54e-04 | 1291.9ms/step | 15.5 samples/s +Step 16460/44000 | Loss: 3.0361 | LR: 3.54e-04 | 1318.9ms/step | 15.2 samples/s +Step 16470/44000 | Loss: 3.0118 | LR: 3.54e-04 | 1247.6ms/step | 16.0 samples/s +Step 16480/44000 | Loss: 3.0226 | LR: 3.54e-04 | 1324.5ms/step | 15.1 samples/s +Step 16490/44000 | Loss: 3.0492 | LR: 3.54e-04 | 1296.1ms/step | 15.4 samples/s +Step 16500/44000 | Loss: 3.0000 | LR: 3.53e-04 | 1315.7ms/step | 15.2 samples/s +Validation at step 16500: Loss = 2.9957, Perplexity = 20.00 +Step 16510/44000 | Loss: 3.0457 | LR: 3.53e-04 | 1770.8ms/step | 11.3 samples/s +Step 16520/44000 | Loss: 3.0209 | LR: 3.53e-04 | 1263.0ms/step | 15.8 samples/s +Step 16530/44000 | Loss: 2.9746 | LR: 3.53e-04 | 1279.3ms/step | 15.6 samples/s +Step 16540/44000 | Loss: 3.0648 | LR: 3.53e-04 | 1242.3ms/step | 16.1 samples/s +Step 16550/44000 | Loss: 3.0572 | LR: 3.53e-04 | 1287.4ms/step | 15.5 samples/s +Step 16560/44000 | Loss: 3.0186 | LR: 3.52e-04 | 1270.5ms/step | 15.7 samples/s +Step 16570/44000 | Loss: 3.0326 | LR: 3.52e-04 | 1432.9ms/step | 14.0 samples/s +Step 16580/44000 | Loss: 3.0032 | LR: 3.52e-04 | 1294.2ms/step | 15.5 samples/s +Step 16590/44000 | Loss: 3.0256 | LR: 3.52e-04 | 1329.7ms/step | 15.0 samples/s +Step 16600/44000 | Loss: 3.0020 | LR: 3.52e-04 | 1310.9ms/step | 15.3 samples/s +Step 16610/44000 | Loss: 3.0253 | LR: 3.52e-04 | 1320.6ms/step | 15.1 samples/s +Step 16620/44000 | Loss: 3.0379 | LR: 3.51e-04 | 1315.8ms/step | 15.2 samples/s +Step 16630/44000 | Loss: 2.9823 | LR: 3.51e-04 | 1315.8ms/step | 15.2 samples/s +Step 16640/44000 | Loss: 2.9954 | LR: 3.51e-04 | 1315.7ms/step | 15.2 samples/s +Step 16650/44000 | Loss: 3.0819 | LR: 3.51e-04 | 1305.1ms/step | 15.3 samples/s +Step 16660/44000 | Loss: 3.0464 | LR: 3.51e-04 | 1314.2ms/step | 15.2 samples/s +Step 16670/44000 | Loss: 3.0777 | LR: 3.51e-04 | 1306.1ms/step | 15.3 samples/s +Step 16680/44000 | Loss: 3.0137 | LR: 3.50e-04 | 1284.5ms/step | 15.6 samples/s +Step 16690/44000 | Loss: 2.9849 | LR: 3.50e-04 | 1317.2ms/step | 15.2 samples/s +Step 16700/44000 | Loss: 3.0883 | LR: 3.50e-04 | 1340.7ms/step | 14.9 samples/s +Step 16710/44000 | Loss: 3.1009 | LR: 3.50e-04 | 1341.2ms/step | 14.9 samples/s +Step 16720/44000 | Loss: 3.0459 | LR: 3.50e-04 | 1323.1ms/step | 15.1 samples/s +Step 16730/44000 | Loss: 3.0679 | LR: 3.50e-04 | 1297.6ms/step | 15.4 samples/s +Step 16740/44000 | Loss: 3.0451 | LR: 3.49e-04 | 1431.4ms/step | 14.0 samples/s +Step 16750/44000 | Loss: 2.9770 | LR: 3.49e-04 | 1279.1ms/step | 15.6 samples/s +Validation at step 16750: Loss = 2.9923, Perplexity = 19.93 +Step 16760/44000 | Loss: 3.0342 | LR: 3.49e-04 | 1742.6ms/step | 11.5 samples/s +Step 16770/44000 | Loss: 3.0760 | LR: 3.49e-04 | 1282.1ms/step | 15.6 samples/s +Step 16780/44000 | Loss: 3.0327 | LR: 3.49e-04 | 1273.8ms/step | 15.7 samples/s +Step 16790/44000 | Loss: 3.0896 | LR: 3.49e-04 | 1288.1ms/step | 15.5 samples/s +Step 16800/44000 | Loss: 3.0354 | LR: 3.48e-04 | 1302.7ms/step | 15.4 samples/s +Step 16810/44000 | Loss: 3.0555 | LR: 3.48e-04 | 1311.9ms/step | 15.2 samples/s +Step 16820/44000 | Loss: 3.0461 | LR: 3.48e-04 | 1350.5ms/step | 14.8 samples/s +Step 16830/44000 | Loss: 3.0388 | LR: 3.48e-04 | 1326.1ms/step | 15.1 samples/s +Step 16840/44000 | Loss: 2.9961 | LR: 3.48e-04 | 1409.6ms/step | 14.2 samples/s +Step 16850/44000 | Loss: 3.0358 | LR: 3.48e-04 | 1331.9ms/step | 15.0 samples/s +Step 16860/44000 | Loss: 3.0145 | LR: 3.47e-04 | 1315.0ms/step | 15.2 samples/s +Step 16870/44000 | Loss: 3.0673 | LR: 3.47e-04 | 1512.6ms/step | 13.2 samples/s +Step 16880/44000 | Loss: 3.0783 | LR: 3.47e-04 | 1604.5ms/step | 12.5 samples/s +Step 16890/44000 | Loss: 3.0696 | LR: 3.47e-04 | 1608.3ms/step | 12.4 samples/s +Step 16900/44000 | Loss: 3.0683 | LR: 3.47e-04 | 1737.9ms/step | 11.5 samples/s +Step 16910/44000 | Loss: 3.0926 | LR: 3.47e-04 | 1619.5ms/step | 12.3 samples/s +Step 16920/44000 | Loss: 3.0729 | LR: 3.46e-04 | 1541.9ms/step | 13.0 samples/s +Step 16930/44000 | Loss: 3.0578 | LR: 3.46e-04 | 1612.8ms/step | 12.4 samples/s +Step 16940/44000 | Loss: 2.9956 | LR: 3.46e-04 | 1611.7ms/step | 12.4 samples/s +Step 16950/44000 | Loss: 3.0342 | LR: 3.46e-04 | 1540.4ms/step | 13.0 samples/s +Step 16960/44000 | Loss: 2.9836 | LR: 3.46e-04 | 1588.4ms/step | 12.6 samples/s +Step 16970/44000 | Loss: 2.9747 | LR: 3.46e-04 | 1558.8ms/step | 12.8 samples/s +Step 16980/44000 | Loss: 3.0874 | LR: 3.45e-04 | 1533.3ms/step | 13.0 samples/s +Step 16990/44000 | Loss: 3.0831 | LR: 3.45e-04 | 1515.0ms/step | 13.2 samples/s +Step 17000/44000 | Loss: 3.0171 | LR: 3.45e-04 | 1575.5ms/step | 12.7 samples/s +Validation at step 17000: Loss = 2.9876, Perplexity = 19.84 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-17000 +Step 17010/44000 | Loss: 3.0746 | LR: 3.45e-04 | 2776.4ms/step | 7.2 samples/s +Step 17020/44000 | Loss: 3.0353 | LR: 3.45e-04 | 1596.3ms/step | 12.5 samples/s +Step 17030/44000 | Loss: 3.0482 | LR: 3.45e-04 | 1610.4ms/step | 12.4 samples/s +Step 17040/44000 | Loss: 2.9931 | LR: 3.44e-04 | 1772.4ms/step | 11.3 samples/s +Step 17050/44000 | Loss: 3.0120 | LR: 3.44e-04 | 1572.2ms/step | 12.7 samples/s +Step 17060/44000 | Loss: 3.0151 | LR: 3.44e-04 | 1551.3ms/step | 12.9 samples/s +Step 17070/44000 | Loss: 3.0479 | LR: 3.44e-04 | 1577.7ms/step | 12.7 samples/s +Step 17080/44000 | Loss: 2.9968 | LR: 3.44e-04 | 1541.5ms/step | 13.0 samples/s +Step 17090/44000 | Loss: 3.1278 | LR: 3.44e-04 | 1550.8ms/step | 12.9 samples/s +Step 17100/44000 | Loss: 2.9313 | LR: 3.43e-04 | 1571.3ms/step | 12.7 samples/s +Step 17110/44000 | Loss: 3.0332 | LR: 3.43e-04 | 1547.9ms/step | 12.9 samples/s +Step 17120/44000 | Loss: 3.0249 | LR: 3.43e-04 | 1546.7ms/step | 12.9 samples/s +Step 17130/44000 | Loss: 3.0256 | LR: 3.43e-04 | 1540.5ms/step | 13.0 samples/s +Step 17140/44000 | Loss: 2.9250 | LR: 3.43e-04 | 1546.3ms/step | 12.9 samples/s +Step 17150/44000 | Loss: 3.0734 | LR: 3.43e-04 | 1500.0ms/step | 13.3 samples/s +Step 17160/44000 | Loss: 3.0478 | LR: 3.42e-04 | 1516.4ms/step | 13.2 samples/s +Step 17170/44000 | Loss: 3.0409 | LR: 3.42e-04 | 1514.7ms/step | 13.2 samples/s +Step 17180/44000 | Loss: 3.0645 | LR: 3.42e-04 | 1719.5ms/step | 11.6 samples/s +Step 17190/44000 | Loss: 3.0749 | LR: 3.42e-04 | 1568.0ms/step | 12.8 samples/s +Step 17200/44000 | Loss: 3.0056 | LR: 3.42e-04 | 1572.8ms/step | 12.7 samples/s +Step 17210/44000 | Loss: 3.0625 | LR: 3.42e-04 | 1598.9ms/step | 12.5 samples/s +Step 17220/44000 | Loss: 3.0103 | LR: 3.41e-04 | 1558.8ms/step | 12.8 samples/s +Step 17230/44000 | Loss: 3.0750 | LR: 3.41e-04 | 1554.9ms/step | 12.9 samples/s +Step 17240/44000 | Loss: 3.0720 | LR: 3.41e-04 | 1536.8ms/step | 13.0 samples/s +Step 17250/44000 | Loss: 3.0857 | LR: 3.41e-04 | 1545.2ms/step | 12.9 samples/s +Validation at step 17250: Loss = 2.9856, Perplexity = 19.80 +Step 17260/44000 | Loss: 2.9721 | LR: 3.41e-04 | 2155.1ms/step | 9.3 samples/s +Step 17270/44000 | Loss: 3.0533 | LR: 3.41e-04 | 1535.8ms/step | 13.0 samples/s +Step 17280/44000 | Loss: 2.9805 | LR: 3.40e-04 | 1570.7ms/step | 12.7 samples/s +Step 17290/44000 | Loss: 3.0173 | LR: 3.40e-04 | 1548.2ms/step | 12.9 samples/s +Step 17300/44000 | Loss: 3.0538 | LR: 3.40e-04 | 1583.3ms/step | 12.6 samples/s +Step 17310/44000 | Loss: 3.0310 | LR: 3.40e-04 | 1591.6ms/step | 12.6 samples/s +Step 17320/44000 | Loss: 3.0037 | LR: 3.40e-04 | 1652.5ms/step | 12.1 samples/s +Step 17330/44000 | Loss: 2.9803 | LR: 3.40e-04 | 1488.7ms/step | 13.4 samples/s +Step 17340/44000 | Loss: 2.9968 | LR: 3.39e-04 | 1580.6ms/step | 12.7 samples/s +Step 17350/44000 | Loss: 2.9638 | LR: 3.39e-04 | 1575.0ms/step | 12.7 samples/s +Step 17360/44000 | Loss: 3.0459 | LR: 3.39e-04 | 1593.7ms/step | 12.5 samples/s +Step 17370/44000 | Loss: 3.0111 | LR: 3.39e-04 | 1598.1ms/step | 12.5 samples/s +Step 17380/44000 | Loss: 3.0383 | LR: 3.39e-04 | 1637.2ms/step | 12.2 samples/s +Step 17390/44000 | Loss: 3.0557 | LR: 3.39e-04 | 1617.4ms/step | 12.4 samples/s +Step 17400/44000 | Loss: 3.0067 | LR: 3.38e-04 | 1566.4ms/step | 12.8 samples/s +Step 17410/44000 | Loss: 2.9997 | LR: 3.38e-04 | 1588.3ms/step | 12.6 samples/s +Step 17420/44000 | Loss: 3.0978 | LR: 3.38e-04 | 1566.5ms/step | 12.8 samples/s +Step 17430/44000 | Loss: 3.0937 | LR: 3.38e-04 | 1595.6ms/step | 12.5 samples/s +Step 17440/44000 | Loss: 2.9477 | LR: 3.38e-04 | 1639.5ms/step | 12.2 samples/s +Step 17450/44000 | Loss: 3.0355 | LR: 3.38e-04 | 1609.6ms/step | 12.4 samples/s +Step 17460/44000 | Loss: 3.0297 | LR: 3.37e-04 | 1814.8ms/step | 11.0 samples/s +Step 17470/44000 | Loss: 2.9776 | LR: 3.37e-04 | 1593.0ms/step | 12.6 samples/s +Step 17480/44000 | Loss: 3.0594 | LR: 3.37e-04 | 1497.6ms/step | 13.4 samples/s +Step 17490/44000 | Loss: 3.0223 | LR: 3.37e-04 | 1570.9ms/step | 12.7 samples/s +Step 17500/44000 | Loss: 3.0270 | LR: 3.37e-04 | 1519.6ms/step | 13.2 samples/s +Validation at step 17500: Loss = 2.9832, Perplexity = 19.75 +Step 17510/44000 | Loss: 3.0089 | LR: 3.37e-04 | 2227.6ms/step | 9.0 samples/s +Step 17520/44000 | Loss: 3.0508 | LR: 3.36e-04 | 1609.3ms/step | 12.4 samples/s +Step 17530/44000 | Loss: 3.0454 | LR: 3.36e-04 | 1561.3ms/step | 12.8 samples/s +Step 17540/44000 | Loss: 3.0974 | LR: 3.36e-04 | 1519.2ms/step | 13.2 samples/s +Step 17550/44000 | Loss: 2.9887 | LR: 3.36e-04 | 1625.2ms/step | 12.3 samples/s +Step 17560/44000 | Loss: 3.0077 | LR: 3.36e-04 | 1601.4ms/step | 12.5 samples/s +Step 17570/44000 | Loss: 3.0365 | LR: 3.35e-04 | 1604.1ms/step | 12.5 samples/s +Step 17580/44000 | Loss: 3.0235 | LR: 3.35e-04 | 1546.4ms/step | 12.9 samples/s +Step 17590/44000 | Loss: 2.9671 | LR: 3.35e-04 | 1562.1ms/step | 12.8 samples/s +Step 17600/44000 | Loss: 3.0520 | LR: 3.35e-04 | 1785.9ms/step | 11.2 samples/s +Step 17610/44000 | Loss: 3.0222 | LR: 3.35e-04 | 1599.9ms/step | 12.5 samples/s +Step 17620/44000 | Loss: 3.0398 | LR: 3.35e-04 | 1546.5ms/step | 12.9 samples/s +Step 17630/44000 | Loss: 3.0719 | LR: 3.34e-04 | 1556.0ms/step | 12.9 samples/s +Step 17640/44000 | Loss: 3.0356 | LR: 3.34e-04 | 1573.5ms/step | 12.7 samples/s +Step 17650/44000 | Loss: 3.0258 | LR: 3.34e-04 | 1628.1ms/step | 12.3 samples/s +Step 17660/44000 | Loss: 3.0105 | LR: 3.34e-04 | 1624.1ms/step | 12.3 samples/s +Step 17670/44000 | Loss: 3.0276 | LR: 3.34e-04 | 1566.1ms/step | 12.8 samples/s +Step 17680/44000 | Loss: 3.0671 | LR: 3.34e-04 | 1635.6ms/step | 12.2 samples/s +Step 17690/44000 | Loss: 2.9957 | LR: 3.33e-04 | 1570.6ms/step | 12.7 samples/s +Step 17700/44000 | Loss: 3.0483 | LR: 3.33e-04 | 1565.2ms/step | 12.8 samples/s +Step 17710/44000 | Loss: 3.0807 | LR: 3.33e-04 | 1571.1ms/step | 12.7 samples/s +Step 17720/44000 | Loss: 3.0345 | LR: 3.33e-04 | 1625.7ms/step | 12.3 samples/s +Step 17730/44000 | Loss: 3.0522 | LR: 3.33e-04 | 1570.9ms/step | 12.7 samples/s +Step 17740/44000 | Loss: 3.0740 | LR: 3.33e-04 | 1799.7ms/step | 11.1 samples/s +Step 17750/44000 | Loss: 3.0524 | LR: 3.32e-04 | 1533.9ms/step | 13.0 samples/s +Validation at step 17750: Loss = 2.9802, Perplexity = 19.69 +Step 17760/44000 | Loss: 2.9702 | LR: 3.32e-04 | 2207.7ms/step | 9.1 samples/s +Step 17770/44000 | Loss: 3.0085 | LR: 3.32e-04 | 1614.8ms/step | 12.4 samples/s +Step 17780/44000 | Loss: 2.9694 | LR: 3.32e-04 | 1439.9ms/step | 13.9 samples/s +Step 17790/44000 | Loss: 3.0335 | LR: 3.32e-04 | 1505.0ms/step | 13.3 samples/s +Step 17800/44000 | Loss: 3.0413 | LR: 3.32e-04 | 1633.5ms/step | 12.2 samples/s +Step 17810/44000 | Loss: 3.0589 | LR: 3.31e-04 | 1634.1ms/step | 12.2 samples/s +Step 17820/44000 | Loss: 3.0562 | LR: 3.31e-04 | 1700.3ms/step | 11.8 samples/s +Step 17830/44000 | Loss: 3.0768 | LR: 3.31e-04 | 1618.6ms/step | 12.4 samples/s +Step 17840/44000 | Loss: 3.0009 | LR: 3.31e-04 | 1599.0ms/step | 12.5 samples/s +Step 17850/44000 | Loss: 3.0039 | LR: 3.31e-04 | 1540.9ms/step | 13.0 samples/s +Step 17860/44000 | Loss: 2.9690 | LR: 3.31e-04 | 1562.5ms/step | 12.8 samples/s +Step 17870/44000 | Loss: 2.9990 | LR: 3.30e-04 | 1616.8ms/step | 12.4 samples/s +Step 17880/44000 | Loss: 3.0045 | LR: 3.30e-04 | 1713.7ms/step | 11.7 samples/s +Step 17890/44000 | Loss: 3.0786 | LR: 3.30e-04 | 1586.8ms/step | 12.6 samples/s +Step 17900/44000 | Loss: 3.0210 | LR: 3.30e-04 | 1511.8ms/step | 13.2 samples/s +Step 17910/44000 | Loss: 2.9986 | LR: 3.30e-04 | 1531.6ms/step | 13.1 samples/s +Step 17920/44000 | Loss: 3.0345 | LR: 3.29e-04 | 1510.1ms/step | 13.2 samples/s +Step 17930/44000 | Loss: 3.0870 | LR: 3.29e-04 | 1555.6ms/step | 12.9 samples/s +Step 17940/44000 | Loss: 3.0850 | LR: 3.29e-04 | 1638.1ms/step | 12.2 samples/s +Step 17950/44000 | Loss: 3.0060 | LR: 3.29e-04 | 1521.1ms/step | 13.1 samples/s +Step 17960/44000 | Loss: 3.0191 | LR: 3.29e-04 | 1609.8ms/step | 12.4 samples/s +Step 17970/44000 | Loss: 3.0059 | LR: 3.29e-04 | 1608.6ms/step | 12.4 samples/s +Step 17980/44000 | Loss: 2.9639 | LR: 3.28e-04 | 1539.8ms/step | 13.0 samples/s +Step 17990/44000 | Loss: 2.9795 | LR: 3.28e-04 | 1481.5ms/step | 13.5 samples/s +Step 18000/44000 | Loss: 2.9925 | LR: 3.28e-04 | 1506.2ms/step | 13.3 samples/s +Validation at step 18000: Loss = 2.9776, Perplexity = 19.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-18000 +Step 18010/44000 | Loss: 2.9705 | LR: 3.28e-04 | 3018.6ms/step | 6.6 samples/s +Step 18020/44000 | Loss: 3.0282 | LR: 3.28e-04 | 1617.1ms/step | 12.4 samples/s +Step 18030/44000 | Loss: 3.0824 | LR: 3.28e-04 | 1565.7ms/step | 12.8 samples/s +Step 18040/44000 | Loss: 3.0211 | LR: 3.27e-04 | 1539.9ms/step | 13.0 samples/s +Step 18050/44000 | Loss: 3.1310 | LR: 3.27e-04 | 1511.4ms/step | 13.2 samples/s +Step 18060/44000 | Loss: 2.9737 | LR: 3.27e-04 | 1505.9ms/step | 13.3 samples/s +Step 18070/44000 | Loss: 3.0917 | LR: 3.27e-04 | 1585.8ms/step | 12.6 samples/s +Step 18080/44000 | Loss: 3.0446 | LR: 3.27e-04 | 1555.9ms/step | 12.9 samples/s +Step 18090/44000 | Loss: 3.0099 | LR: 3.27e-04 | 1546.6ms/step | 12.9 samples/s +Step 18100/44000 | Loss: 2.9572 | LR: 3.26e-04 | 1551.7ms/step | 12.9 samples/s +Step 18110/44000 | Loss: 3.0011 | LR: 3.26e-04 | 1562.5ms/step | 12.8 samples/s +Step 18120/44000 | Loss: 3.0332 | LR: 3.26e-04 | 1464.2ms/step | 13.7 samples/s +Step 18130/44000 | Loss: 2.9877 | LR: 3.26e-04 | 1509.6ms/step | 13.2 samples/s +Step 18140/44000 | Loss: 3.0835 | LR: 3.26e-04 | 1548.6ms/step | 12.9 samples/s +Step 18150/44000 | Loss: 3.0514 | LR: 3.26e-04 | 1657.5ms/step | 12.1 samples/s +Step 18160/44000 | Loss: 2.9832 | LR: 3.25e-04 | 1755.7ms/step | 11.4 samples/s +Step 18170/44000 | Loss: 3.0009 | LR: 3.25e-04 | 1632.9ms/step | 12.2 samples/s +Step 18180/44000 | Loss: 3.0176 | LR: 3.25e-04 | 1550.4ms/step | 12.9 samples/s +Step 18190/44000 | Loss: 2.9880 | LR: 3.25e-04 | 1579.0ms/step | 12.7 samples/s +Step 18200/44000 | Loss: 3.0189 | LR: 3.25e-04 | 1529.0ms/step | 13.1 samples/s +Step 18210/44000 | Loss: 3.1352 | LR: 3.24e-04 | 1563.7ms/step | 12.8 samples/s +Step 18220/44000 | Loss: 3.0537 | LR: 3.24e-04 | 1632.5ms/step | 12.3 samples/s +Step 18230/44000 | Loss: 2.9857 | LR: 3.24e-04 | 1457.8ms/step | 13.7 samples/s +Step 18240/44000 | Loss: 3.0331 | LR: 3.24e-04 | 1484.7ms/step | 13.5 samples/s +Step 18250/44000 | Loss: 3.0659 | LR: 3.24e-04 | 1572.4ms/step | 12.7 samples/s +Validation at step 18250: Loss = 2.9710, Perplexity = 19.51 +Step 18260/44000 | Loss: 3.0347 | LR: 3.24e-04 | 2156.8ms/step | 9.3 samples/s +Step 18270/44000 | Loss: 3.0414 | LR: 3.23e-04 | 1596.9ms/step | 12.5 samples/s +Step 18280/44000 | Loss: 2.9865 | LR: 3.23e-04 | 1543.4ms/step | 13.0 samples/s +Step 18290/44000 | Loss: 2.9418 | LR: 3.23e-04 | 1661.1ms/step | 12.0 samples/s +Step 18300/44000 | Loss: 3.0029 | LR: 3.23e-04 | 1716.3ms/step | 11.7 samples/s +Step 18310/44000 | Loss: 3.0258 | LR: 3.23e-04 | 1549.3ms/step | 12.9 samples/s +Step 18320/44000 | Loss: 3.0163 | LR: 3.23e-04 | 1595.2ms/step | 12.5 samples/s +Step 18330/44000 | Loss: 2.9789 | LR: 3.22e-04 | 1610.0ms/step | 12.4 samples/s +Step 18340/44000 | Loss: 3.0908 | LR: 3.22e-04 | 1609.1ms/step | 12.4 samples/s +Step 18350/44000 | Loss: 3.0100 | LR: 3.22e-04 | 1567.7ms/step | 12.8 samples/s +Step 18360/44000 | Loss: 3.0010 | LR: 3.22e-04 | 1583.3ms/step | 12.6 samples/s +Step 18370/44000 | Loss: 2.9508 | LR: 3.22e-04 | 1607.2ms/step | 12.4 samples/s +Step 18380/44000 | Loss: 3.0203 | LR: 3.22e-04 | 1542.6ms/step | 13.0 samples/s +Step 18390/44000 | Loss: 3.0363 | LR: 3.21e-04 | 1553.5ms/step | 12.9 samples/s +Step 18400/44000 | Loss: 2.9684 | LR: 3.21e-04 | 1508.6ms/step | 13.3 samples/s +Step 18410/44000 | Loss: 2.9999 | LR: 3.21e-04 | 1566.0ms/step | 12.8 samples/s +Step 18420/44000 | Loss: 3.0044 | LR: 3.21e-04 | 1532.5ms/step | 13.1 samples/s +Step 18430/44000 | Loss: 3.0500 | LR: 3.21e-04 | 1599.8ms/step | 12.5 samples/s +Step 18440/44000 | Loss: 2.9994 | LR: 3.20e-04 | 1835.9ms/step | 10.9 samples/s +Step 18450/44000 | Loss: 2.9516 | LR: 3.20e-04 | 1558.9ms/step | 12.8 samples/s +Step 18460/44000 | Loss: 2.9945 | LR: 3.20e-04 | 1540.3ms/step | 13.0 samples/s +Step 18470/44000 | Loss: 2.9809 | LR: 3.20e-04 | 1493.5ms/step | 13.4 samples/s +Step 18480/44000 | Loss: 3.0337 | LR: 3.20e-04 | 1562.1ms/step | 12.8 samples/s +Step 18490/44000 | Loss: 2.9442 | LR: 3.20e-04 | 1512.6ms/step | 13.2 samples/s +Step 18500/44000 | Loss: 3.0169 | LR: 3.19e-04 | 1621.8ms/step | 12.3 samples/s +Validation at step 18500: Loss = 2.9693, Perplexity = 19.48 +Step 18510/44000 | Loss: 2.9959 | LR: 3.19e-04 | 2192.4ms/step | 9.1 samples/s +Step 18520/44000 | Loss: 2.9609 | LR: 3.19e-04 | 1589.6ms/step | 12.6 samples/s +Step 18530/44000 | Loss: 3.0037 | LR: 3.19e-04 | 1665.5ms/step | 12.0 samples/s +Step 18540/44000 | Loss: 3.0455 | LR: 3.19e-04 | 1544.0ms/step | 13.0 samples/s +Step 18550/44000 | Loss: 2.9683 | LR: 3.19e-04 | 1559.6ms/step | 12.8 samples/s +Step 18560/44000 | Loss: 3.0103 | LR: 3.18e-04 | 1555.7ms/step | 12.9 samples/s +Step 18570/44000 | Loss: 2.9986 | LR: 3.18e-04 | 1736.6ms/step | 11.5 samples/s +Step 18580/44000 | Loss: 2.9944 | LR: 3.18e-04 | 1625.6ms/step | 12.3 samples/s +Step 18590/44000 | Loss: 3.0632 | LR: 3.18e-04 | 1524.2ms/step | 13.1 samples/s +Step 18600/44000 | Loss: 3.0058 | LR: 3.18e-04 | 1558.1ms/step | 12.8 samples/s +Step 18610/44000 | Loss: 3.0760 | LR: 3.18e-04 | 1552.9ms/step | 12.9 samples/s +Step 18620/44000 | Loss: 3.0588 | LR: 3.17e-04 | 1594.3ms/step | 12.5 samples/s +Step 18630/44000 | Loss: 3.0539 | LR: 3.17e-04 | 1540.3ms/step | 13.0 samples/s +Step 18640/44000 | Loss: 3.0262 | LR: 3.17e-04 | 1541.5ms/step | 13.0 samples/s +Step 18650/44000 | Loss: 3.0295 | LR: 3.17e-04 | 1544.7ms/step | 12.9 samples/s +Step 18660/44000 | Loss: 3.0002 | LR: 3.17e-04 | 1559.8ms/step | 12.8 samples/s +Step 18670/44000 | Loss: 3.0431 | LR: 3.16e-04 | 1536.6ms/step | 13.0 samples/s +Step 18680/44000 | Loss: 3.0297 | LR: 3.16e-04 | 1531.1ms/step | 13.1 samples/s +Step 18690/44000 | Loss: 3.0354 | LR: 3.16e-04 | 1477.4ms/step | 13.5 samples/s +Step 18700/44000 | Loss: 2.9222 | LR: 3.16e-04 | 1477.7ms/step | 13.5 samples/s +Step 18710/44000 | Loss: 3.0050 | LR: 3.16e-04 | 1517.4ms/step | 13.2 samples/s +Step 18720/44000 | Loss: 2.9629 | LR: 3.16e-04 | 1767.2ms/step | 11.3 samples/s +Step 18730/44000 | Loss: 3.0412 | LR: 3.15e-04 | 1531.0ms/step | 13.1 samples/s +Step 18740/44000 | Loss: 2.9948 | LR: 3.15e-04 | 1562.2ms/step | 12.8 samples/s +Step 18750/44000 | Loss: 3.0432 | LR: 3.15e-04 | 1520.8ms/step | 13.2 samples/s +Validation at step 18750: Loss = 2.9689, Perplexity = 19.47 +Step 18760/44000 | Loss: 3.0945 | LR: 3.15e-04 | 2202.7ms/step | 9.1 samples/s +Step 18770/44000 | Loss: 3.0806 | LR: 3.15e-04 | 1573.0ms/step | 12.7 samples/s +Step 18780/44000 | Loss: 3.0403 | LR: 3.15e-04 | 1658.7ms/step | 12.1 samples/s +Step 18790/44000 | Loss: 3.0229 | LR: 3.14e-04 | 1578.5ms/step | 12.7 samples/s +Step 18800/44000 | Loss: 2.9190 | LR: 3.14e-04 | 1630.0ms/step | 12.3 samples/s +Step 18810/44000 | Loss: 2.9653 | LR: 3.14e-04 | 1555.4ms/step | 12.9 samples/s +Step 18820/44000 | Loss: 3.0027 | LR: 3.14e-04 | 1515.6ms/step | 13.2 samples/s +Step 18830/44000 | Loss: 3.0929 | LR: 3.14e-04 | 1480.0ms/step | 13.5 samples/s +Step 18840/44000 | Loss: 3.0962 | LR: 3.13e-04 | 1555.8ms/step | 12.9 samples/s +Step 18850/44000 | Loss: 3.0258 | LR: 3.13e-04 | 1520.2ms/step | 13.2 samples/s +Step 18860/44000 | Loss: 3.0001 | LR: 3.13e-04 | 1827.0ms/step | 10.9 samples/s +Step 18870/44000 | Loss: 2.9702 | LR: 3.13e-04 | 1553.4ms/step | 12.9 samples/s +Step 18880/44000 | Loss: 3.0292 | LR: 3.13e-04 | 1556.8ms/step | 12.8 samples/s +Step 18890/44000 | Loss: 3.0626 | LR: 3.13e-04 | 1565.2ms/step | 12.8 samples/s +Step 18900/44000 | Loss: 2.9707 | LR: 3.12e-04 | 1553.4ms/step | 12.9 samples/s +Step 18910/44000 | Loss: 2.9872 | LR: 3.12e-04 | 1536.0ms/step | 13.0 samples/s +Step 18920/44000 | Loss: 3.0404 | LR: 3.12e-04 | 1591.4ms/step | 12.6 samples/s +Step 18930/44000 | Loss: 3.0362 | LR: 3.12e-04 | 1543.8ms/step | 13.0 samples/s +Step 18940/44000 | Loss: 2.9733 | LR: 3.12e-04 | 1533.2ms/step | 13.0 samples/s +Step 18950/44000 | Loss: 2.9826 | LR: 3.12e-04 | 1515.9ms/step | 13.2 samples/s +Step 18960/44000 | Loss: 3.0460 | LR: 3.11e-04 | 1554.8ms/step | 12.9 samples/s +Step 18970/44000 | Loss: 2.9442 | LR: 3.11e-04 | 1582.1ms/step | 12.6 samples/s +Step 18980/44000 | Loss: 3.0457 | LR: 3.11e-04 | 1573.7ms/step | 12.7 samples/s +Step 18990/44000 | Loss: 3.0804 | LR: 3.11e-04 | 1542.1ms/step | 13.0 samples/s +Step 19000/44000 | Loss: 3.0070 | LR: 3.11e-04 | 1707.1ms/step | 11.7 samples/s +Validation at step 19000: Loss = 2.9662, Perplexity = 19.42 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-19000 +Step 19010/44000 | Loss: 3.0222 | LR: 3.11e-04 | 2838.3ms/step | 7.0 samples/s +Step 19020/44000 | Loss: 2.9805 | LR: 3.10e-04 | 1532.3ms/step | 13.1 samples/s +Step 19030/44000 | Loss: 3.0117 | LR: 3.10e-04 | 1541.5ms/step | 13.0 samples/s +Step 19040/44000 | Loss: 2.9774 | LR: 3.10e-04 | 1540.6ms/step | 13.0 samples/s +Step 19050/44000 | Loss: 3.0647 | LR: 3.10e-04 | 1563.8ms/step | 12.8 samples/s +Step 19060/44000 | Loss: 3.0042 | LR: 3.10e-04 | 1609.3ms/step | 12.4 samples/s +Step 19070/44000 | Loss: 2.9791 | LR: 3.09e-04 | 1536.1ms/step | 13.0 samples/s +Step 19080/44000 | Loss: 2.9893 | LR: 3.09e-04 | 1557.1ms/step | 12.8 samples/s +Step 19090/44000 | Loss: 3.0837 | LR: 3.09e-04 | 1588.9ms/step | 12.6 samples/s +Step 19100/44000 | Loss: 2.9540 | LR: 3.09e-04 | 1515.7ms/step | 13.2 samples/s +Step 19110/44000 | Loss: 3.0560 | LR: 3.09e-04 | 1522.2ms/step | 13.1 samples/s +Step 19120/44000 | Loss: 3.0609 | LR: 3.09e-04 | 1493.5ms/step | 13.4 samples/s +Step 19130/44000 | Loss: 3.0492 | LR: 3.08e-04 | 1557.1ms/step | 12.8 samples/s +Step 19140/44000 | Loss: 2.9393 | LR: 3.08e-04 | 1620.2ms/step | 12.3 samples/s +Step 19150/44000 | Loss: 3.0576 | LR: 3.08e-04 | 1510.3ms/step | 13.2 samples/s +Step 19160/44000 | Loss: 3.0402 | LR: 3.08e-04 | 1463.8ms/step | 13.7 samples/s +Step 19170/44000 | Loss: 2.9980 | LR: 3.08e-04 | 1534.8ms/step | 13.0 samples/s +Step 19180/44000 | Loss: 3.0553 | LR: 3.08e-04 | 1530.0ms/step | 13.1 samples/s +Step 19190/44000 | Loss: 3.0440 | LR: 3.07e-04 | 1573.3ms/step | 12.7 samples/s +Step 19200/44000 | Loss: 2.9457 | LR: 3.07e-04 | 1534.5ms/step | 13.0 samples/s +Step 19210/44000 | Loss: 3.0099 | LR: 3.07e-04 | 1600.0ms/step | 12.5 samples/s +Step 19220/44000 | Loss: 3.0308 | LR: 3.07e-04 | 1546.8ms/step | 12.9 samples/s +Step 19230/44000 | Loss: 3.0634 | LR: 3.07e-04 | 1510.1ms/step | 13.2 samples/s +Step 19240/44000 | Loss: 2.9936 | LR: 3.06e-04 | 1494.0ms/step | 13.4 samples/s +Step 19250/44000 | Loss: 3.0785 | LR: 3.06e-04 | 1525.8ms/step | 13.1 samples/s +Validation at step 19250: Loss = 2.9620, Perplexity = 19.34 +Step 19260/44000 | Loss: 2.9915 | LR: 3.06e-04 | 2112.7ms/step | 9.5 samples/s +Step 19270/44000 | Loss: 2.9786 | LR: 3.06e-04 | 1521.3ms/step | 13.1 samples/s +Step 19280/44000 | Loss: 3.0501 | LR: 3.06e-04 | 1758.9ms/step | 11.4 samples/s +Step 19290/44000 | Loss: 2.9938 | LR: 3.06e-04 | 1526.5ms/step | 13.1 samples/s +Step 19300/44000 | Loss: 2.9974 | LR: 3.05e-04 | 1542.7ms/step | 13.0 samples/s +Step 19310/44000 | Loss: 3.0271 | LR: 3.05e-04 | 1520.0ms/step | 13.2 samples/s +Step 19320/44000 | Loss: 2.9676 | LR: 3.05e-04 | 1551.0ms/step | 12.9 samples/s +Step 19330/44000 | Loss: 3.0007 | LR: 3.05e-04 | 1524.6ms/step | 13.1 samples/s +Step 19340/44000 | Loss: 2.9654 | LR: 3.05e-04 | 1548.4ms/step | 12.9 samples/s +Step 19350/44000 | Loss: 3.0147 | LR: 3.04e-04 | 1555.7ms/step | 12.9 samples/s +Step 19360/44000 | Loss: 3.0251 | LR: 3.04e-04 | 1578.0ms/step | 12.7 samples/s +Step 19370/44000 | Loss: 3.0139 | LR: 3.04e-04 | 1545.2ms/step | 12.9 samples/s +Step 19380/44000 | Loss: 2.9796 | LR: 3.04e-04 | 1518.1ms/step | 13.2 samples/s +Step 19390/44000 | Loss: 2.9992 | LR: 3.04e-04 | 1512.5ms/step | 13.2 samples/s +Step 19400/44000 | Loss: 2.9890 | LR: 3.04e-04 | 1505.4ms/step | 13.3 samples/s +Step 19410/44000 | Loss: 2.9694 | LR: 3.03e-04 | 1540.3ms/step | 13.0 samples/s +Step 19420/44000 | Loss: 3.0346 | LR: 3.03e-04 | 1671.7ms/step | 12.0 samples/s +Step 19430/44000 | Loss: 2.9915 | LR: 3.03e-04 | 1677.3ms/step | 11.9 samples/s +Step 19440/44000 | Loss: 3.0484 | LR: 3.03e-04 | 1515.7ms/step | 13.2 samples/s +Step 19450/44000 | Loss: 3.0051 | LR: 3.03e-04 | 1575.6ms/step | 12.7 samples/s +Step 19460/44000 | Loss: 3.0169 | LR: 3.03e-04 | 1571.0ms/step | 12.7 samples/s +Step 19470/44000 | Loss: 3.0301 | LR: 3.02e-04 | 1560.9ms/step | 12.8 samples/s +Step 19480/44000 | Loss: 3.0395 | LR: 3.02e-04 | 1555.9ms/step | 12.9 samples/s +Step 19490/44000 | Loss: 2.9672 | LR: 3.02e-04 | 1583.4ms/step | 12.6 samples/s +Step 19500/44000 | Loss: 2.9918 | LR: 3.02e-04 | 1567.7ms/step | 12.8 samples/s +Validation at step 19500: Loss = 2.9581, Perplexity = 19.26 +Step 19510/44000 | Loss: 2.9895 | LR: 3.02e-04 | 2164.3ms/step | 9.2 samples/s +Step 19520/44000 | Loss: 3.1113 | LR: 3.01e-04 | 1561.2ms/step | 12.8 samples/s +Step 19530/44000 | Loss: 2.9893 | LR: 3.01e-04 | 1559.3ms/step | 12.8 samples/s +Step 19540/44000 | Loss: 3.0396 | LR: 3.01e-04 | 1537.1ms/step | 13.0 samples/s +Step 19550/44000 | Loss: 3.0222 | LR: 3.01e-04 | 1546.5ms/step | 12.9 samples/s +Step 19560/44000 | Loss: 3.0861 | LR: 3.01e-04 | 1718.8ms/step | 11.6 samples/s +Step 19570/44000 | Loss: 2.9828 | LR: 3.01e-04 | 1609.1ms/step | 12.4 samples/s +Step 19580/44000 | Loss: 2.9660 | LR: 3.00e-04 | 1568.9ms/step | 12.7 samples/s +Step 19590/44000 | Loss: 2.9973 | LR: 3.00e-04 | 1528.8ms/step | 13.1 samples/s +Step 19600/44000 | Loss: 3.0334 | LR: 3.00e-04 | 1428.4ms/step | 14.0 samples/s +Step 19610/44000 | Loss: 3.0283 | LR: 3.00e-04 | 1575.2ms/step | 12.7 samples/s +Step 19620/44000 | Loss: 2.9557 | LR: 3.00e-04 | 1445.6ms/step | 13.8 samples/s +Step 19630/44000 | Loss: 3.1028 | LR: 3.00e-04 | 1546.0ms/step | 12.9 samples/s +Step 19640/44000 | Loss: 3.0008 | LR: 2.99e-04 | 1504.1ms/step | 13.3 samples/s +Step 19650/44000 | Loss: 2.9955 | LR: 2.99e-04 | 1613.9ms/step | 12.4 samples/s +Step 19660/44000 | Loss: 3.0340 | LR: 2.99e-04 | 1562.1ms/step | 12.8 samples/s +Step 19670/44000 | Loss: 3.0531 | LR: 2.99e-04 | 1537.5ms/step | 13.0 samples/s +Step 19680/44000 | Loss: 2.9272 | LR: 2.99e-04 | 1530.5ms/step | 13.1 samples/s +Step 19690/44000 | Loss: 3.0066 | LR: 2.98e-04 | 1519.4ms/step | 13.2 samples/s +Step 19700/44000 | Loss: 2.9541 | LR: 2.98e-04 | 1537.8ms/step | 13.0 samples/s +Step 19710/44000 | Loss: 3.0056 | LR: 2.98e-04 | 1728.2ms/step | 11.6 samples/s +Step 19720/44000 | Loss: 2.9653 | LR: 2.98e-04 | 1566.2ms/step | 12.8 samples/s +Step 19730/44000 | Loss: 2.9771 | LR: 2.98e-04 | 1529.4ms/step | 13.1 samples/s +Step 19740/44000 | Loss: 2.9502 | LR: 2.98e-04 | 1532.4ms/step | 13.1 samples/s +Step 19750/44000 | Loss: 3.1064 | LR: 2.97e-04 | 1495.7ms/step | 13.4 samples/s +Validation at step 19750: Loss = 2.9544, Perplexity = 19.19 +Step 19760/44000 | Loss: 3.0317 | LR: 2.97e-04 | 2109.5ms/step | 9.5 samples/s +Step 19770/44000 | Loss: 2.9891 | LR: 2.97e-04 | 1531.2ms/step | 13.1 samples/s +Step 19780/44000 | Loss: 3.0122 | LR: 2.97e-04 | 1473.3ms/step | 13.6 samples/s +Step 19790/44000 | Loss: 2.9654 | LR: 2.97e-04 | 1557.1ms/step | 12.8 samples/s +Step 19800/44000 | Loss: 3.0177 | LR: 2.96e-04 | 1541.6ms/step | 13.0 samples/s +Step 19810/44000 | Loss: 3.0241 | LR: 2.96e-04 | 1570.1ms/step | 12.7 samples/s +Step 19820/44000 | Loss: 3.0670 | LR: 2.96e-04 | 1571.2ms/step | 12.7 samples/s +Step 19830/44000 | Loss: 2.9324 | LR: 2.96e-04 | 1538.2ms/step | 13.0 samples/s +Step 19840/44000 | Loss: 3.0614 | LR: 2.96e-04 | 1527.6ms/step | 13.1 samples/s +Step 19850/44000 | Loss: 2.9544 | LR: 2.96e-04 | 1768.3ms/step | 11.3 samples/s +Step 19860/44000 | Loss: 2.9832 | LR: 2.95e-04 | 1541.5ms/step | 13.0 samples/s +Step 19870/44000 | Loss: 2.9915 | LR: 2.95e-04 | 1552.3ms/step | 12.9 samples/s +Step 19880/44000 | Loss: 2.9574 | LR: 2.95e-04 | 1549.7ms/step | 12.9 samples/s +Step 19890/44000 | Loss: 2.9873 | LR: 2.95e-04 | 1588.4ms/step | 12.6 samples/s +Step 19900/44000 | Loss: 2.9198 | LR: 2.95e-04 | 1546.4ms/step | 12.9 samples/s +Step 19910/44000 | Loss: 2.9640 | LR: 2.95e-04 | 1550.3ms/step | 12.9 samples/s +Step 19920/44000 | Loss: 3.0284 | LR: 2.94e-04 | 1575.8ms/step | 12.7 samples/s +Step 19930/44000 | Loss: 3.0423 | LR: 2.94e-04 | 1572.8ms/step | 12.7 samples/s +Step 19940/44000 | Loss: 3.0830 | LR: 2.94e-04 | 1509.5ms/step | 13.2 samples/s +Step 19950/44000 | Loss: 3.0619 | LR: 2.94e-04 | 1512.1ms/step | 13.2 samples/s +Step 19960/44000 | Loss: 3.0069 | LR: 2.94e-04 | 1517.1ms/step | 13.2 samples/s +Step 19970/44000 | Loss: 2.9579 | LR: 2.93e-04 | 1535.9ms/step | 13.0 samples/s +Step 19980/44000 | Loss: 3.0018 | LR: 2.93e-04 | 1550.0ms/step | 12.9 samples/s +Step 19990/44000 | Loss: 3.0331 | LR: 2.93e-04 | 1599.1ms/step | 12.5 samples/s +Step 20000/44000 | Loss: 2.9196 | LR: 2.93e-04 | 1721.7ms/step | 11.6 samples/s +Validation at step 20000: Loss = 2.9514, Perplexity = 19.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-20000 +Step 20010/44000 | Loss: 3.0248 | LR: 2.93e-04 | 2730.4ms/step | 7.3 samples/s +Step 20020/44000 | Loss: 3.0401 | LR: 2.93e-04 | 1532.0ms/step | 13.1 samples/s +Step 20030/44000 | Loss: 2.9940 | LR: 2.92e-04 | 1560.9ms/step | 12.8 samples/s +Step 20040/44000 | Loss: 3.0406 | LR: 2.92e-04 | 1569.3ms/step | 12.7 samples/s +Step 20050/44000 | Loss: 3.0405 | LR: 2.92e-04 | 1567.2ms/step | 12.8 samples/s +Step 20060/44000 | Loss: 2.9855 | LR: 2.92e-04 | 1572.9ms/step | 12.7 samples/s +Step 20070/44000 | Loss: 2.9564 | LR: 2.92e-04 | 1441.0ms/step | 13.9 samples/s +Step 20080/44000 | Loss: 2.9857 | LR: 2.91e-04 | 1475.6ms/step | 13.6 samples/s +Step 20090/44000 | Loss: 2.9989 | LR: 2.91e-04 | 1524.2ms/step | 13.1 samples/s +Step 20100/44000 | Loss: 3.0163 | LR: 2.91e-04 | 1515.5ms/step | 13.2 samples/s +Step 20110/44000 | Loss: 2.9316 | LR: 2.91e-04 | 1477.9ms/step | 13.5 samples/s +Step 20120/44000 | Loss: 3.0002 | LR: 2.91e-04 | 1499.5ms/step | 13.3 samples/s +Step 20130/44000 | Loss: 2.9721 | LR: 2.91e-04 | 1766.6ms/step | 11.3 samples/s +Step 20140/44000 | Loss: 3.0028 | LR: 2.90e-04 | 1560.4ms/step | 12.8 samples/s +Step 20150/44000 | Loss: 2.9701 | LR: 2.90e-04 | 1528.6ms/step | 13.1 samples/s +Step 20160/44000 | Loss: 2.9075 | LR: 2.90e-04 | 1582.0ms/step | 12.6 samples/s +Step 20170/44000 | Loss: 2.9336 | LR: 2.90e-04 | 1557.0ms/step | 12.8 samples/s +Step 20180/44000 | Loss: 3.0206 | LR: 2.90e-04 | 1543.5ms/step | 13.0 samples/s +Step 20190/44000 | Loss: 2.9932 | LR: 2.90e-04 | 1582.7ms/step | 12.6 samples/s +Step 20200/44000 | Loss: 2.9657 | LR: 2.89e-04 | 1580.7ms/step | 12.7 samples/s +Step 20210/44000 | Loss: 2.9648 | LR: 2.89e-04 | 1528.1ms/step | 13.1 samples/s +Step 20220/44000 | Loss: 3.0465 | LR: 2.89e-04 | 1562.0ms/step | 12.8 samples/s +Step 20230/44000 | Loss: 2.9658 | LR: 2.89e-04 | 1557.8ms/step | 12.8 samples/s +Step 20240/44000 | Loss: 2.9326 | LR: 2.89e-04 | 1534.1ms/step | 13.0 samples/s +Step 20250/44000 | Loss: 3.0106 | LR: 2.88e-04 | 1509.2ms/step | 13.3 samples/s +Validation at step 20250: Loss = 2.9485, Perplexity = 19.08 +Step 20260/44000 | Loss: 2.9510 | LR: 2.88e-04 | 2136.2ms/step | 9.4 samples/s +Step 20270/44000 | Loss: 2.9439 | LR: 2.88e-04 | 1709.2ms/step | 11.7 samples/s +Step 20280/44000 | Loss: 3.0225 | LR: 2.88e-04 | 1615.4ms/step | 12.4 samples/s +Step 20290/44000 | Loss: 3.0354 | LR: 2.88e-04 | 1525.7ms/step | 13.1 samples/s +Step 20300/44000 | Loss: 3.0695 | LR: 2.88e-04 | 1545.1ms/step | 12.9 samples/s +Step 20310/44000 | Loss: 2.9968 | LR: 2.87e-04 | 1595.5ms/step | 12.5 samples/s +Step 20320/44000 | Loss: 2.9909 | LR: 2.87e-04 | 1520.1ms/step | 13.2 samples/s +Step 20330/44000 | Loss: 3.0221 | LR: 2.87e-04 | 1582.3ms/step | 12.6 samples/s +Step 20340/44000 | Loss: 3.0495 | LR: 2.87e-04 | 1563.8ms/step | 12.8 samples/s +Step 20350/44000 | Loss: 3.0344 | LR: 2.87e-04 | 1566.2ms/step | 12.8 samples/s +Step 20360/44000 | Loss: 3.0523 | LR: 2.86e-04 | 1532.2ms/step | 13.1 samples/s +Step 20370/44000 | Loss: 3.0251 | LR: 2.86e-04 | 1570.8ms/step | 12.7 samples/s +Step 20380/44000 | Loss: 3.0818 | LR: 2.86e-04 | 1509.3ms/step | 13.3 samples/s +Step 20390/44000 | Loss: 2.9710 | LR: 2.86e-04 | 1539.1ms/step | 13.0 samples/s +Step 20400/44000 | Loss: 2.9363 | LR: 2.86e-04 | 1496.0ms/step | 13.4 samples/s +Step 20410/44000 | Loss: 2.9317 | LR: 2.86e-04 | 1626.8ms/step | 12.3 samples/s +Step 20420/44000 | Loss: 3.0312 | LR: 2.85e-04 | 1681.8ms/step | 11.9 samples/s +Step 20430/44000 | Loss: 2.9822 | LR: 2.85e-04 | 1620.2ms/step | 12.3 samples/s +Step 20440/44000 | Loss: 2.9563 | LR: 2.85e-04 | 1490.1ms/step | 13.4 samples/s +Step 20450/44000 | Loss: 2.9753 | LR: 2.85e-04 | 1534.2ms/step | 13.0 samples/s +Step 20460/44000 | Loss: 3.0212 | LR: 2.85e-04 | 1532.3ms/step | 13.1 samples/s +Step 20470/44000 | Loss: 2.9630 | LR: 2.84e-04 | 1577.1ms/step | 12.7 samples/s +Step 20480/44000 | Loss: 3.0386 | LR: 2.84e-04 | 1521.8ms/step | 13.1 samples/s +Step 20490/44000 | Loss: 2.9876 | LR: 2.84e-04 | 1591.1ms/step | 12.6 samples/s +Step 20500/44000 | Loss: 3.0073 | LR: 2.84e-04 | 1593.9ms/step | 12.5 samples/s +Validation at step 20500: Loss = 2.9456, Perplexity = 19.02 +Step 20510/44000 | Loss: 2.9551 | LR: 2.84e-04 | 2032.0ms/step | 9.8 samples/s +Step 20520/44000 | Loss: 2.9693 | LR: 2.84e-04 | 1527.9ms/step | 13.1 samples/s +Step 20530/44000 | Loss: 3.0014 | LR: 2.83e-04 | 1532.7ms/step | 13.0 samples/s +Step 20540/44000 | Loss: 2.9136 | LR: 2.83e-04 | 1491.9ms/step | 13.4 samples/s +Step 20550/44000 | Loss: 2.9770 | LR: 2.83e-04 | 1535.2ms/step | 13.0 samples/s +Step 20560/44000 | Loss: 2.9928 | LR: 2.83e-04 | 1762.6ms/step | 11.3 samples/s +Step 20570/44000 | Loss: 2.9680 | LR: 2.83e-04 | 1536.4ms/step | 13.0 samples/s +Step 20580/44000 | Loss: 2.9559 | LR: 2.83e-04 | 1533.7ms/step | 13.0 samples/s +Step 20590/44000 | Loss: 2.9517 | LR: 2.82e-04 | 1548.1ms/step | 12.9 samples/s +Step 20600/44000 | Loss: 3.0459 | LR: 2.82e-04 | 1561.6ms/step | 12.8 samples/s +Step 20610/44000 | Loss: 3.0145 | LR: 2.82e-04 | 1567.6ms/step | 12.8 samples/s +Step 20620/44000 | Loss: 2.9819 | LR: 2.82e-04 | 1559.3ms/step | 12.8 samples/s +Step 20630/44000 | Loss: 3.0154 | LR: 2.82e-04 | 1515.5ms/step | 13.2 samples/s +Step 20640/44000 | Loss: 2.9809 | LR: 2.81e-04 | 1568.5ms/step | 12.8 samples/s +Step 20650/44000 | Loss: 3.0559 | LR: 2.81e-04 | 1539.2ms/step | 13.0 samples/s +Step 20660/44000 | Loss: 2.9631 | LR: 2.81e-04 | 1569.9ms/step | 12.7 samples/s +Step 20670/44000 | Loss: 3.0167 | LR: 2.81e-04 | 1591.4ms/step | 12.6 samples/s +Step 20680/44000 | Loss: 3.0066 | LR: 2.81e-04 | 1574.7ms/step | 12.7 samples/s +Step 20690/44000 | Loss: 3.0716 | LR: 2.81e-04 | 1543.3ms/step | 13.0 samples/s +Step 20700/44000 | Loss: 3.0056 | LR: 2.80e-04 | 1774.7ms/step | 11.3 samples/s +Step 20710/44000 | Loss: 2.9196 | LR: 2.80e-04 | 1619.5ms/step | 12.3 samples/s +Step 20720/44000 | Loss: 3.0186 | LR: 2.80e-04 | 1530.6ms/step | 13.1 samples/s +Step 20730/44000 | Loss: 3.0033 | LR: 2.80e-04 | 1558.5ms/step | 12.8 samples/s +Step 20740/44000 | Loss: 2.9586 | LR: 2.80e-04 | 1600.6ms/step | 12.5 samples/s +Step 20750/44000 | Loss: 3.0383 | LR: 2.79e-04 | 1589.6ms/step | 12.6 samples/s +Validation at step 20750: Loss = 2.9444, Perplexity = 19.00 +Step 20760/44000 | Loss: 2.9569 | LR: 2.79e-04 | 2174.9ms/step | 9.2 samples/s +Step 20770/44000 | Loss: 2.9714 | LR: 2.79e-04 | 1556.0ms/step | 12.9 samples/s +Step 20780/44000 | Loss: 2.9605 | LR: 2.79e-04 | 1612.6ms/step | 12.4 samples/s +Step 20790/44000 | Loss: 3.0217 | LR: 2.79e-04 | 1609.2ms/step | 12.4 samples/s +Step 20800/44000 | Loss: 3.0388 | LR: 2.79e-04 | 1544.7ms/step | 12.9 samples/s +Step 20810/44000 | Loss: 2.9880 | LR: 2.78e-04 | 1512.1ms/step | 13.2 samples/s +Step 20820/44000 | Loss: 2.9887 | LR: 2.78e-04 | 1530.2ms/step | 13.1 samples/s +Step 20830/44000 | Loss: 3.0231 | LR: 2.78e-04 | 1557.3ms/step | 12.8 samples/s +Step 20840/44000 | Loss: 2.9383 | LR: 2.78e-04 | 1843.4ms/step | 10.8 samples/s +Step 20850/44000 | Loss: 2.9765 | LR: 2.78e-04 | 1608.2ms/step | 12.4 samples/s +Step 20860/44000 | Loss: 3.0238 | LR: 2.77e-04 | 1468.7ms/step | 13.6 samples/s +Step 20870/44000 | Loss: 3.0176 | LR: 2.77e-04 | 1575.0ms/step | 12.7 samples/s +Step 20880/44000 | Loss: 2.9641 | LR: 2.77e-04 | 1575.4ms/step | 12.7 samples/s +Step 20890/44000 | Loss: 2.9476 | LR: 2.77e-04 | 1579.8ms/step | 12.7 samples/s +Step 20900/44000 | Loss: 3.0178 | LR: 2.77e-04 | 1563.6ms/step | 12.8 samples/s +Step 20910/44000 | Loss: 2.9749 | LR: 2.77e-04 | 1565.7ms/step | 12.8 samples/s +Step 20920/44000 | Loss: 2.9610 | LR: 2.76e-04 | 1579.9ms/step | 12.7 samples/s +Step 20930/44000 | Loss: 3.0128 | LR: 2.76e-04 | 1529.0ms/step | 13.1 samples/s +Step 20940/44000 | Loss: 2.9515 | LR: 2.76e-04 | 1514.0ms/step | 13.2 samples/s +Step 20950/44000 | Loss: 2.9538 | LR: 2.76e-04 | 1488.7ms/step | 13.4 samples/s +Step 20960/44000 | Loss: 3.0109 | LR: 2.76e-04 | 1529.1ms/step | 13.1 samples/s +Step 20970/44000 | Loss: 2.9571 | LR: 2.75e-04 | 1515.0ms/step | 13.2 samples/s +Step 20980/44000 | Loss: 3.0258 | LR: 2.75e-04 | 1613.8ms/step | 12.4 samples/s +Step 20990/44000 | Loss: 2.9675 | LR: 2.75e-04 | 1599.3ms/step | 12.5 samples/s +Step 21000/44000 | Loss: 2.9783 | LR: 2.75e-04 | 1583.0ms/step | 12.6 samples/s +Validation at step 21000: Loss = 2.9434, Perplexity = 18.98 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-21000 +Step 21010/44000 | Loss: 2.9470 | LR: 2.75e-04 | 2757.3ms/step | 7.3 samples/s +Step 21020/44000 | Loss: 3.0065 | LR: 2.75e-04 | 1582.6ms/step | 12.6 samples/s +Step 21030/44000 | Loss: 2.9631 | LR: 2.74e-04 | 1571.0ms/step | 12.7 samples/s +Step 21040/44000 | Loss: 2.9861 | LR: 2.74e-04 | 1563.3ms/step | 12.8 samples/s +Step 21050/44000 | Loss: 2.9910 | LR: 2.74e-04 | 1591.4ms/step | 12.6 samples/s +Step 21060/44000 | Loss: 3.0411 | LR: 2.74e-04 | 1560.9ms/step | 12.8 samples/s +Step 21070/44000 | Loss: 3.0300 | LR: 2.74e-04 | 1547.7ms/step | 12.9 samples/s +Step 21080/44000 | Loss: 3.0121 | LR: 2.73e-04 | 1613.1ms/step | 12.4 samples/s +Step 21090/44000 | Loss: 2.9685 | LR: 2.73e-04 | 1501.0ms/step | 13.3 samples/s +Step 21100/44000 | Loss: 2.9623 | LR: 2.73e-04 | 1514.7ms/step | 13.2 samples/s +Step 21110/44000 | Loss: 2.9980 | LR: 2.73e-04 | 1536.9ms/step | 13.0 samples/s +Step 21120/44000 | Loss: 2.9728 | LR: 2.73e-04 | 1829.6ms/step | 10.9 samples/s +Step 21130/44000 | Loss: 3.0018 | LR: 2.73e-04 | 1494.2ms/step | 13.4 samples/s +Step 21140/44000 | Loss: 2.9840 | LR: 2.72e-04 | 1551.9ms/step | 12.9 samples/s +Step 21150/44000 | Loss: 2.9834 | LR: 2.72e-04 | 1584.3ms/step | 12.6 samples/s +Step 21160/44000 | Loss: 2.9131 | LR: 2.72e-04 | 1577.5ms/step | 12.7 samples/s +Step 21170/44000 | Loss: 3.0007 | LR: 2.72e-04 | 1550.5ms/step | 12.9 samples/s +Step 21180/44000 | Loss: 3.0044 | LR: 2.72e-04 | 1568.2ms/step | 12.8 samples/s +Step 21190/44000 | Loss: 2.9515 | LR: 2.71e-04 | 1545.4ms/step | 12.9 samples/s +Step 21200/44000 | Loss: 3.0077 | LR: 2.71e-04 | 1570.1ms/step | 12.7 samples/s +Step 21210/44000 | Loss: 2.9253 | LR: 2.71e-04 | 1586.7ms/step | 12.6 samples/s +Step 21220/44000 | Loss: 3.0101 | LR: 2.71e-04 | 1530.5ms/step | 13.1 samples/s +Step 21230/44000 | Loss: 3.0502 | LR: 2.71e-04 | 1517.5ms/step | 13.2 samples/s +Step 21240/44000 | Loss: 2.9899 | LR: 2.71e-04 | 1652.8ms/step | 12.1 samples/s +Step 21250/44000 | Loss: 3.0355 | LR: 2.70e-04 | 1574.1ms/step | 12.7 samples/s +Validation at step 21250: Loss = 2.9386, Perplexity = 18.89 +Step 21260/44000 | Loss: 2.9164 | LR: 2.70e-04 | 2415.8ms/step | 8.3 samples/s +Step 21270/44000 | Loss: 3.0505 | LR: 2.70e-04 | 1559.9ms/step | 12.8 samples/s +Step 21280/44000 | Loss: 2.9841 | LR: 2.70e-04 | 1544.0ms/step | 13.0 samples/s +Step 21290/44000 | Loss: 3.0055 | LR: 2.70e-04 | 1544.3ms/step | 13.0 samples/s +Step 21300/44000 | Loss: 3.0105 | LR: 2.70e-04 | 1522.3ms/step | 13.1 samples/s +Step 21310/44000 | Loss: 2.9276 | LR: 2.69e-04 | 1543.4ms/step | 13.0 samples/s +Step 21320/44000 | Loss: 2.9610 | LR: 2.69e-04 | 1578.0ms/step | 12.7 samples/s +Step 21330/44000 | Loss: 2.9591 | LR: 2.69e-04 | 1562.6ms/step | 12.8 samples/s +Step 21340/44000 | Loss: 2.9600 | LR: 2.69e-04 | 1555.4ms/step | 12.9 samples/s +Step 21350/44000 | Loss: 2.9550 | LR: 2.69e-04 | 1606.0ms/step | 12.5 samples/s +Step 21360/44000 | Loss: 2.9848 | LR: 2.68e-04 | 1544.5ms/step | 12.9 samples/s +Step 21370/44000 | Loss: 2.9482 | LR: 2.68e-04 | 1519.6ms/step | 13.2 samples/s +Step 21380/44000 | Loss: 2.9365 | LR: 2.68e-04 | 1528.4ms/step | 13.1 samples/s +Step 21390/44000 | Loss: 2.9529 | LR: 2.68e-04 | 1546.6ms/step | 12.9 samples/s +Step 21400/44000 | Loss: 3.0608 | LR: 2.68e-04 | 1766.4ms/step | 11.3 samples/s +Step 21410/44000 | Loss: 3.0110 | LR: 2.68e-04 | 1599.4ms/step | 12.5 samples/s +Step 21420/44000 | Loss: 2.9647 | LR: 2.67e-04 | 1530.8ms/step | 13.1 samples/s +Step 21430/44000 | Loss: 2.9953 | LR: 2.67e-04 | 1491.9ms/step | 13.4 samples/s +Step 21440/44000 | Loss: 3.0017 | LR: 2.67e-04 | 1490.5ms/step | 13.4 samples/s +Step 21450/44000 | Loss: 2.9560 | LR: 2.67e-04 | 1614.8ms/step | 12.4 samples/s +Step 21460/44000 | Loss: 2.9970 | LR: 2.67e-04 | 1541.4ms/step | 13.0 samples/s +Step 21470/44000 | Loss: 3.0391 | LR: 2.66e-04 | 1580.3ms/step | 12.7 samples/s +Step 21480/44000 | Loss: 2.9702 | LR: 2.66e-04 | 1518.5ms/step | 13.2 samples/s +Step 21490/44000 | Loss: 2.9758 | LR: 2.66e-04 | 1565.0ms/step | 12.8 samples/s +Step 21500/44000 | Loss: 2.9762 | LR: 2.66e-04 | 1681.8ms/step | 11.9 samples/s +Validation at step 21500: Loss = 2.9378, Perplexity = 18.87 +Step 21510/44000 | Loss: 2.8977 | LR: 2.66e-04 | 2262.1ms/step | 8.8 samples/s +Step 21520/44000 | Loss: 2.9937 | LR: 2.66e-04 | 1538.0ms/step | 13.0 samples/s +Step 21530/44000 | Loss: 3.0445 | LR: 2.65e-04 | 1583.5ms/step | 12.6 samples/s +Step 21540/44000 | Loss: 3.0724 | LR: 2.65e-04 | 1850.4ms/step | 10.8 samples/s +Step 21550/44000 | Loss: 3.0049 | LR: 2.65e-04 | 1572.4ms/step | 12.7 samples/s +Step 21560/44000 | Loss: 2.9499 | LR: 2.65e-04 | 1551.6ms/step | 12.9 samples/s +Step 21570/44000 | Loss: 2.9528 | LR: 2.65e-04 | 1549.3ms/step | 12.9 samples/s +Step 21580/44000 | Loss: 3.0195 | LR: 2.64e-04 | 1596.9ms/step | 12.5 samples/s +Step 21590/44000 | Loss: 2.9847 | LR: 2.64e-04 | 1494.3ms/step | 13.4 samples/s +Step 21600/44000 | Loss: 2.9354 | LR: 2.64e-04 | 1566.3ms/step | 12.8 samples/s +Step 21610/44000 | Loss: 3.0505 | LR: 2.64e-04 | 1562.0ms/step | 12.8 samples/s +Step 21620/44000 | Loss: 3.0140 | LR: 2.64e-04 | 1594.6ms/step | 12.5 samples/s +Step 21630/44000 | Loss: 2.9681 | LR: 2.64e-04 | 1578.1ms/step | 12.7 samples/s +Step 21640/44000 | Loss: 2.9859 | LR: 2.63e-04 | 1568.6ms/step | 12.7 samples/s +Step 21650/44000 | Loss: 2.9831 | LR: 2.63e-04 | 1479.1ms/step | 13.5 samples/s +Step 21660/44000 | Loss: 2.9468 | LR: 2.63e-04 | 1476.7ms/step | 13.5 samples/s +Step 21670/44000 | Loss: 2.9388 | LR: 2.63e-04 | 1543.9ms/step | 13.0 samples/s +Step 21680/44000 | Loss: 3.0069 | LR: 2.63e-04 | 1690.1ms/step | 11.8 samples/s +Step 21690/44000 | Loss: 3.0217 | LR: 2.62e-04 | 1641.6ms/step | 12.2 samples/s +Step 21700/44000 | Loss: 2.9795 | LR: 2.62e-04 | 1566.5ms/step | 12.8 samples/s +Step 21710/44000 | Loss: 2.9944 | LR: 2.62e-04 | 1611.0ms/step | 12.4 samples/s +Step 21720/44000 | Loss: 3.0155 | LR: 2.62e-04 | 1545.2ms/step | 12.9 samples/s +Step 21730/44000 | Loss: 2.9696 | LR: 2.62e-04 | 1524.6ms/step | 13.1 samples/s +Step 21740/44000 | Loss: 3.0624 | LR: 2.62e-04 | 1573.4ms/step | 12.7 samples/s +Step 21750/44000 | Loss: 2.9626 | LR: 2.61e-04 | 1543.5ms/step | 13.0 samples/s +Validation at step 21750: Loss = 2.9348, Perplexity = 18.82 +Step 21760/44000 | Loss: 3.0396 | LR: 2.61e-04 | 2097.1ms/step | 9.5 samples/s +Step 21770/44000 | Loss: 2.9432 | LR: 2.61e-04 | 1533.4ms/step | 13.0 samples/s +Step 21780/44000 | Loss: 2.9453 | LR: 2.61e-04 | 1525.4ms/step | 13.1 samples/s +Step 21790/44000 | Loss: 3.0048 | LR: 2.61e-04 | 1493.8ms/step | 13.4 samples/s +Step 21800/44000 | Loss: 2.9717 | LR: 2.60e-04 | 1536.4ms/step | 13.0 samples/s +Step 21810/44000 | Loss: 2.9651 | LR: 2.60e-04 | 1531.4ms/step | 13.1 samples/s +Step 21820/44000 | Loss: 2.9781 | LR: 2.60e-04 | 1707.0ms/step | 11.7 samples/s +Step 21830/44000 | Loss: 2.9937 | LR: 2.60e-04 | 1648.2ms/step | 12.1 samples/s +Step 21840/44000 | Loss: 2.9517 | LR: 2.60e-04 | 1525.5ms/step | 13.1 samples/s +Step 21850/44000 | Loss: 2.9592 | LR: 2.60e-04 | 1534.1ms/step | 13.0 samples/s +Step 21860/44000 | Loss: 2.8871 | LR: 2.59e-04 | 1593.4ms/step | 12.6 samples/s +Step 21870/44000 | Loss: 2.9899 | LR: 2.59e-04 | 1579.5ms/step | 12.7 samples/s +Step 21880/44000 | Loss: 3.0082 | LR: 2.59e-04 | 1563.9ms/step | 12.8 samples/s +Step 21890/44000 | Loss: 3.0277 | LR: 2.59e-04 | 1544.3ms/step | 13.0 samples/s +Step 21900/44000 | Loss: 3.0430 | LR: 2.59e-04 | 1482.9ms/step | 13.5 samples/s +Step 21910/44000 | Loss: 2.9405 | LR: 2.58e-04 | 1562.3ms/step | 12.8 samples/s +Step 21920/44000 | Loss: 2.9780 | LR: 2.58e-04 | 1567.4ms/step | 12.8 samples/s +Step 21930/44000 | Loss: 2.9715 | LR: 2.58e-04 | 1575.4ms/step | 12.7 samples/s +Step 21940/44000 | Loss: 2.9503 | LR: 2.58e-04 | 1585.8ms/step | 12.6 samples/s +Step 21950/44000 | Loss: 2.9333 | LR: 2.58e-04 | 1520.3ms/step | 13.2 samples/s +Step 21960/44000 | Loss: 2.9501 | LR: 2.58e-04 | 1562.3ms/step | 12.8 samples/s +Step 21970/44000 | Loss: 3.0707 | LR: 2.57e-04 | 1713.3ms/step | 11.7 samples/s +Step 21980/44000 | Loss: 2.9572 | LR: 2.57e-04 | 1531.0ms/step | 13.1 samples/s +Step 21990/44000 | Loss: 2.9956 | LR: 2.57e-04 | 1578.8ms/step | 12.7 samples/s +Step 22000/44000 | Loss: 2.8731 | LR: 2.57e-04 | 1525.9ms/step | 13.1 samples/s +Validation at step 22000: Loss = 2.9329, Perplexity = 18.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-22000 +Step 22010/44000 | Loss: 2.9761 | LR: 2.57e-04 | 2889.6ms/step | 6.9 samples/s +Step 22020/44000 | Loss: 2.9659 | LR: 2.56e-04 | 1613.9ms/step | 12.4 samples/s +Step 22030/44000 | Loss: 3.0395 | LR: 2.56e-04 | 1698.6ms/step | 11.8 samples/s +Step 22040/44000 | Loss: 3.0281 | LR: 2.56e-04 | 1635.3ms/step | 12.2 samples/s +Step 22050/44000 | Loss: 2.9809 | LR: 2.56e-04 | 1667.8ms/step | 12.0 samples/s +Step 22060/44000 | Loss: 3.0338 | LR: 2.56e-04 | 1653.9ms/step | 12.1 samples/s +Step 22070/44000 | Loss: 2.9318 | LR: 2.56e-04 | 1554.5ms/step | 12.9 samples/s +Step 22080/44000 | Loss: 2.9245 | LR: 2.55e-04 | 1566.8ms/step | 12.8 samples/s +Step 22090/44000 | Loss: 2.9111 | LR: 2.55e-04 | 1579.1ms/step | 12.7 samples/s +Step 22100/44000 | Loss: 2.9604 | LR: 2.55e-04 | 1839.9ms/step | 10.9 samples/s +Step 22110/44000 | Loss: 3.0260 | LR: 2.55e-04 | 1604.8ms/step | 12.5 samples/s +Step 22120/44000 | Loss: 3.0365 | LR: 2.55e-04 | 1516.2ms/step | 13.2 samples/s +Step 22130/44000 | Loss: 2.9357 | LR: 2.54e-04 | 1613.6ms/step | 12.4 samples/s +Step 22140/44000 | Loss: 2.9420 | LR: 2.54e-04 | 1579.0ms/step | 12.7 samples/s +Step 22150/44000 | Loss: 2.9759 | LR: 2.54e-04 | 1571.3ms/step | 12.7 samples/s +Step 22160/44000 | Loss: 2.9275 | LR: 2.54e-04 | 1591.6ms/step | 12.6 samples/s +Step 22170/44000 | Loss: 3.0062 | LR: 2.54e-04 | 1548.4ms/step | 12.9 samples/s +Step 22180/44000 | Loss: 3.0095 | LR: 2.54e-04 | 1593.5ms/step | 12.6 samples/s +Step 22190/44000 | Loss: 2.9845 | LR: 2.53e-04 | 1586.6ms/step | 12.6 samples/s +Step 22200/44000 | Loss: 2.9953 | LR: 2.53e-04 | 1616.5ms/step | 12.4 samples/s +Step 22210/44000 | Loss: 2.9876 | LR: 2.53e-04 | 1541.5ms/step | 13.0 samples/s +Step 22220/44000 | Loss: 2.9799 | LR: 2.53e-04 | 1543.8ms/step | 13.0 samples/s +Step 22230/44000 | Loss: 3.0110 | LR: 2.53e-04 | 1589.7ms/step | 12.6 samples/s +Step 22240/44000 | Loss: 2.9001 | LR: 2.52e-04 | 1786.7ms/step | 11.2 samples/s +Step 22250/44000 | Loss: 2.9931 | LR: 2.52e-04 | 1517.4ms/step | 13.2 samples/s +Validation at step 22250: Loss = 2.9301, Perplexity = 18.73 +Step 22260/44000 | Loss: 2.9665 | LR: 2.52e-04 | 2157.8ms/step | 9.3 samples/s +Step 22270/44000 | Loss: 3.0244 | LR: 2.52e-04 | 1548.3ms/step | 12.9 samples/s +Step 22280/44000 | Loss: 2.9493 | LR: 2.52e-04 | 1523.2ms/step | 13.1 samples/s +Step 22290/44000 | Loss: 3.0010 | LR: 2.52e-04 | 1546.7ms/step | 12.9 samples/s +Step 22300/44000 | Loss: 2.9714 | LR: 2.51e-04 | 1570.2ms/step | 12.7 samples/s +Step 22310/44000 | Loss: 2.9885 | LR: 2.51e-04 | 1598.5ms/step | 12.5 samples/s +Step 22320/44000 | Loss: 2.9658 | LR: 2.51e-04 | 1580.7ms/step | 12.7 samples/s +Step 22330/44000 | Loss: 2.9766 | LR: 2.51e-04 | 1599.5ms/step | 12.5 samples/s +Step 22340/44000 | Loss: 2.9692 | LR: 2.51e-04 | 1530.0ms/step | 13.1 samples/s +Step 22350/44000 | Loss: 2.9572 | LR: 2.50e-04 | 1519.8ms/step | 13.2 samples/s +Step 22360/44000 | Loss: 2.9633 | LR: 2.50e-04 | 1530.8ms/step | 13.1 samples/s +Step 22370/44000 | Loss: 3.0769 | LR: 2.50e-04 | 1493.9ms/step | 13.4 samples/s +Step 22380/44000 | Loss: 2.9934 | LR: 2.50e-04 | 1777.1ms/step | 11.3 samples/s +Step 22390/44000 | Loss: 2.9877 | LR: 2.50e-04 | 1583.6ms/step | 12.6 samples/s +Step 22400/44000 | Loss: 3.0398 | LR: 2.50e-04 | 1557.6ms/step | 12.8 samples/s +Step 22410/44000 | Loss: 2.9883 | LR: 2.49e-04 | 1562.0ms/step | 12.8 samples/s +Step 22420/44000 | Loss: 3.0057 | LR: 2.49e-04 | 1530.0ms/step | 13.1 samples/s +Step 22430/44000 | Loss: 2.9960 | LR: 2.49e-04 | 1526.3ms/step | 13.1 samples/s +Step 22440/44000 | Loss: 2.9630 | LR: 2.49e-04 | 1563.4ms/step | 12.8 samples/s +Step 22450/44000 | Loss: 2.9626 | LR: 2.49e-04 | 1597.0ms/step | 12.5 samples/s +Step 22460/44000 | Loss: 2.9522 | LR: 2.48e-04 | 1599.1ms/step | 12.5 samples/s +Step 22470/44000 | Loss: 2.9622 | LR: 2.48e-04 | 1551.4ms/step | 12.9 samples/s +Step 22480/44000 | Loss: 2.9916 | LR: 2.48e-04 | 1595.1ms/step | 12.5 samples/s +Step 22490/44000 | Loss: 2.9360 | LR: 2.48e-04 | 1554.8ms/step | 12.9 samples/s +Step 22500/44000 | Loss: 2.9010 | LR: 2.48e-04 | 1546.0ms/step | 12.9 samples/s +Validation at step 22500: Loss = 2.9269, Perplexity = 18.67 +Step 22510/44000 | Loss: 2.9817 | LR: 2.48e-04 | 2097.1ms/step | 9.5 samples/s +Step 22520/44000 | Loss: 2.9365 | LR: 2.47e-04 | 1773.8ms/step | 11.3 samples/s +Step 22530/44000 | Loss: 2.9532 | LR: 2.47e-04 | 1606.6ms/step | 12.4 samples/s +Step 22540/44000 | Loss: 2.9536 | LR: 2.47e-04 | 1626.1ms/step | 12.3 samples/s +Step 22550/44000 | Loss: 3.0055 | LR: 2.47e-04 | 1536.9ms/step | 13.0 samples/s +Step 22560/44000 | Loss: 2.9694 | LR: 2.47e-04 | 1619.4ms/step | 12.4 samples/s +Step 22570/44000 | Loss: 2.9358 | LR: 2.46e-04 | 1597.0ms/step | 12.5 samples/s +Step 22580/44000 | Loss: 3.0396 | LR: 2.46e-04 | 1598.7ms/step | 12.5 samples/s +Step 22590/44000 | Loss: 2.9980 | LR: 2.46e-04 | 1576.7ms/step | 12.7 samples/s +Step 22600/44000 | Loss: 2.9798 | LR: 2.46e-04 | 1630.6ms/step | 12.3 samples/s +Step 22610/44000 | Loss: 2.9872 | LR: 2.46e-04 | 1558.1ms/step | 12.8 samples/s +Step 22620/44000 | Loss: 2.9329 | LR: 2.46e-04 | 1530.7ms/step | 13.1 samples/s +Step 22630/44000 | Loss: 2.9080 | LR: 2.45e-04 | 1549.4ms/step | 12.9 samples/s +Step 22640/44000 | Loss: 2.9688 | LR: 2.45e-04 | 1535.4ms/step | 13.0 samples/s +Step 22650/44000 | Loss: 2.9518 | LR: 2.45e-04 | 1570.7ms/step | 12.7 samples/s +Step 22660/44000 | Loss: 3.0139 | LR: 2.45e-04 | 1819.4ms/step | 11.0 samples/s +Step 22670/44000 | Loss: 2.9433 | LR: 2.45e-04 | 1492.9ms/step | 13.4 samples/s +Step 22680/44000 | Loss: 3.0134 | LR: 2.44e-04 | 1594.7ms/step | 12.5 samples/s +Step 22690/44000 | Loss: 2.9330 | LR: 2.44e-04 | 1555.4ms/step | 12.9 samples/s +Step 22700/44000 | Loss: 2.9564 | LR: 2.44e-04 | 1630.5ms/step | 12.3 samples/s +Step 22710/44000 | Loss: 2.9949 | LR: 2.44e-04 | 1572.5ms/step | 12.7 samples/s +Step 22720/44000 | Loss: 2.9906 | LR: 2.44e-04 | 1582.7ms/step | 12.6 samples/s +Step 22730/44000 | Loss: 2.9537 | LR: 2.44e-04 | 1567.8ms/step | 12.8 samples/s +Step 22740/44000 | Loss: 2.9345 | LR: 2.43e-04 | 1560.4ms/step | 12.8 samples/s +Step 22750/44000 | Loss: 2.9758 | LR: 2.43e-04 | 1563.5ms/step | 12.8 samples/s +Validation at step 22750: Loss = 2.9249, Perplexity = 18.63 +Step 22760/44000 | Loss: 2.9689 | LR: 2.43e-04 | 2157.5ms/step | 9.3 samples/s +Step 22770/44000 | Loss: 2.9524 | LR: 2.43e-04 | 1562.9ms/step | 12.8 samples/s +Step 22780/44000 | Loss: 2.9803 | LR: 2.43e-04 | 1547.3ms/step | 12.9 samples/s +Step 22790/44000 | Loss: 2.9885 | LR: 2.42e-04 | 1587.8ms/step | 12.6 samples/s +Step 22800/44000 | Loss: 3.0106 | LR: 2.42e-04 | 1800.9ms/step | 11.1 samples/s +Step 22810/44000 | Loss: 3.0022 | LR: 2.42e-04 | 1480.8ms/step | 13.5 samples/s +Step 22820/44000 | Loss: 3.0048 | LR: 2.42e-04 | 1515.0ms/step | 13.2 samples/s +Step 22830/44000 | Loss: 2.9498 | LR: 2.42e-04 | 1546.3ms/step | 12.9 samples/s +Step 22840/44000 | Loss: 2.9049 | LR: 2.42e-04 | 1449.6ms/step | 13.8 samples/s +Step 22850/44000 | Loss: 2.9594 | LR: 2.41e-04 | 1539.2ms/step | 13.0 samples/s +Step 22860/44000 | Loss: 2.9968 | LR: 2.41e-04 | 1559.1ms/step | 12.8 samples/s +Step 22870/44000 | Loss: 2.9796 | LR: 2.41e-04 | 1592.9ms/step | 12.6 samples/s +Step 22880/44000 | Loss: 2.9760 | LR: 2.41e-04 | 1526.5ms/step | 13.1 samples/s +Step 22890/44000 | Loss: 2.9746 | LR: 2.41e-04 | 1538.1ms/step | 13.0 samples/s +Step 22900/44000 | Loss: 3.0037 | LR: 2.40e-04 | 1570.6ms/step | 12.7 samples/s +Step 22910/44000 | Loss: 3.0154 | LR: 2.40e-04 | 1542.7ms/step | 13.0 samples/s +Step 22920/44000 | Loss: 3.0120 | LR: 2.40e-04 | 1549.5ms/step | 12.9 samples/s +Step 22930/44000 | Loss: 2.9604 | LR: 2.40e-04 | 1565.8ms/step | 12.8 samples/s +Step 22940/44000 | Loss: 3.0162 | LR: 2.40e-04 | 1777.0ms/step | 11.3 samples/s +Step 22950/44000 | Loss: 2.9239 | LR: 2.40e-04 | 1582.6ms/step | 12.6 samples/s +Step 22960/44000 | Loss: 2.9655 | LR: 2.39e-04 | 1573.2ms/step | 12.7 samples/s +Step 22970/44000 | Loss: 2.9804 | LR: 2.39e-04 | 1521.2ms/step | 13.1 samples/s +Step 22980/44000 | Loss: 2.9609 | LR: 2.39e-04 | 1577.5ms/step | 12.7 samples/s +Step 22990/44000 | Loss: 2.9595 | LR: 2.39e-04 | 1542.5ms/step | 13.0 samples/s +Step 23000/44000 | Loss: 2.9678 | LR: 2.39e-04 | 1542.7ms/step | 13.0 samples/s +Validation at step 23000: Loss = 2.9217, Perplexity = 18.57 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-23000 +Step 23010/44000 | Loss: 2.9759 | LR: 2.38e-04 | 2756.6ms/step | 7.3 samples/s +Step 23020/44000 | Loss: 2.9556 | LR: 2.38e-04 | 1597.1ms/step | 12.5 samples/s +Step 23030/44000 | Loss: 2.9705 | LR: 2.38e-04 | 1586.9ms/step | 12.6 samples/s +Step 23040/44000 | Loss: 2.9779 | LR: 2.38e-04 | 1548.9ms/step | 12.9 samples/s +Step 23050/44000 | Loss: 2.9644 | LR: 2.38e-04 | 1592.7ms/step | 12.6 samples/s +Step 23060/44000 | Loss: 2.9518 | LR: 2.38e-04 | 1562.3ms/step | 12.8 samples/s +Step 23070/44000 | Loss: 2.9676 | LR: 2.37e-04 | 1626.1ms/step | 12.3 samples/s +Step 23080/44000 | Loss: 2.9916 | LR: 2.37e-04 | 1763.3ms/step | 11.3 samples/s +Step 23090/44000 | Loss: 2.9755 | LR: 2.37e-04 | 1549.1ms/step | 12.9 samples/s +Step 23100/44000 | Loss: 2.9630 | LR: 2.37e-04 | 1507.4ms/step | 13.3 samples/s +Step 23110/44000 | Loss: 2.9804 | LR: 2.37e-04 | 1550.2ms/step | 12.9 samples/s +Step 23120/44000 | Loss: 2.9663 | LR: 2.36e-04 | 1531.5ms/step | 13.1 samples/s +Step 23130/44000 | Loss: 2.9291 | LR: 2.36e-04 | 1564.4ms/step | 12.8 samples/s +Step 23140/44000 | Loss: 2.9937 | LR: 2.36e-04 | 1554.7ms/step | 12.9 samples/s +Step 23150/44000 | Loss: 2.9358 | LR: 2.36e-04 | 1481.4ms/step | 13.5 samples/s +Step 23160/44000 | Loss: 2.9865 | LR: 2.36e-04 | 1598.4ms/step | 12.5 samples/s +Step 23170/44000 | Loss: 2.9645 | LR: 2.36e-04 | 1550.0ms/step | 12.9 samples/s +Step 23180/44000 | Loss: 2.9296 | LR: 2.35e-04 | 1655.8ms/step | 12.1 samples/s +Step 23190/44000 | Loss: 3.0013 | LR: 2.35e-04 | 1543.8ms/step | 13.0 samples/s +Step 23200/44000 | Loss: 2.9997 | LR: 2.35e-04 | 1571.9ms/step | 12.7 samples/s +Step 23210/44000 | Loss: 2.9381 | LR: 2.35e-04 | 1544.0ms/step | 13.0 samples/s +Step 23220/44000 | Loss: 2.9178 | LR: 2.35e-04 | 1771.7ms/step | 11.3 samples/s +Step 23230/44000 | Loss: 3.0380 | LR: 2.34e-04 | 1566.2ms/step | 12.8 samples/s +Step 23240/44000 | Loss: 3.0026 | LR: 2.34e-04 | 1574.1ms/step | 12.7 samples/s +Step 23250/44000 | Loss: 3.0247 | LR: 2.34e-04 | 1577.9ms/step | 12.7 samples/s +Validation at step 23250: Loss = 2.9210, Perplexity = 18.56 +Step 23260/44000 | Loss: 2.9317 | LR: 2.34e-04 | 2287.6ms/step | 8.7 samples/s +Step 23270/44000 | Loss: 3.0166 | LR: 2.34e-04 | 1520.4ms/step | 13.2 samples/s +Step 23280/44000 | Loss: 2.9394 | LR: 2.34e-04 | 1580.0ms/step | 12.7 samples/s +Step 23290/44000 | Loss: 2.9680 | LR: 2.33e-04 | 1490.4ms/step | 13.4 samples/s +Step 23300/44000 | Loss: 2.9092 | LR: 2.33e-04 | 1518.0ms/step | 13.2 samples/s +Step 23310/44000 | Loss: 2.9642 | LR: 2.33e-04 | 1564.3ms/step | 12.8 samples/s +Step 23320/44000 | Loss: 2.9521 | LR: 2.33e-04 | 1584.6ms/step | 12.6 samples/s +Step 23330/44000 | Loss: 3.0173 | LR: 2.33e-04 | 1529.9ms/step | 13.1 samples/s +Step 23340/44000 | Loss: 2.9454 | LR: 2.32e-04 | 1549.6ms/step | 12.9 samples/s +Step 23350/44000 | Loss: 3.0386 | LR: 2.32e-04 | 1528.6ms/step | 13.1 samples/s +Step 23360/44000 | Loss: 2.9514 | LR: 2.32e-04 | 1782.7ms/step | 11.2 samples/s +Step 23370/44000 | Loss: 2.9904 | LR: 2.32e-04 | 1551.1ms/step | 12.9 samples/s +Step 23380/44000 | Loss: 2.8723 | LR: 2.32e-04 | 1547.0ms/step | 12.9 samples/s +Step 23390/44000 | Loss: 2.8620 | LR: 2.32e-04 | 1562.0ms/step | 12.8 samples/s +Step 23400/44000 | Loss: 2.9727 | LR: 2.31e-04 | 1599.3ms/step | 12.5 samples/s +Step 23410/44000 | Loss: 3.0133 | LR: 2.31e-04 | 1550.8ms/step | 12.9 samples/s +Step 23420/44000 | Loss: 2.9614 | LR: 2.31e-04 | 1540.6ms/step | 13.0 samples/s +Step 23430/44000 | Loss: 2.9870 | LR: 2.31e-04 | 1487.0ms/step | 13.4 samples/s +Step 23440/44000 | Loss: 2.9585 | LR: 2.31e-04 | 1539.0ms/step | 13.0 samples/s +Step 23450/44000 | Loss: 3.0228 | LR: 2.30e-04 | 1503.0ms/step | 13.3 samples/s +Step 23460/44000 | Loss: 3.0296 | LR: 2.30e-04 | 1560.5ms/step | 12.8 samples/s +Step 23470/44000 | Loss: 2.9829 | LR: 2.30e-04 | 1557.9ms/step | 12.8 samples/s +Step 23480/44000 | Loss: 2.8932 | LR: 2.30e-04 | 1616.9ms/step | 12.4 samples/s +Step 23490/44000 | Loss: 2.9622 | LR: 2.30e-04 | 1579.3ms/step | 12.7 samples/s +Step 23500/44000 | Loss: 3.0240 | LR: 2.30e-04 | 1735.3ms/step | 11.5 samples/s +Validation at step 23500: Loss = 2.9188, Perplexity = 18.52 +Step 23510/44000 | Loss: 2.9302 | LR: 2.29e-04 | 2310.9ms/step | 8.7 samples/s +Step 23520/44000 | Loss: 2.9494 | LR: 2.29e-04 | 1561.9ms/step | 12.8 samples/s +Step 23530/44000 | Loss: 2.9886 | LR: 2.29e-04 | 1512.5ms/step | 13.2 samples/s +Step 23540/44000 | Loss: 2.9533 | LR: 2.29e-04 | 1526.4ms/step | 13.1 samples/s +Step 23550/44000 | Loss: 2.9545 | LR: 2.29e-04 | 1539.9ms/step | 13.0 samples/s +Step 23560/44000 | Loss: 2.9921 | LR: 2.29e-04 | 1617.1ms/step | 12.4 samples/s +Step 23570/44000 | Loss: 2.9451 | LR: 2.28e-04 | 1562.3ms/step | 12.8 samples/s +Step 23580/44000 | Loss: 2.9631 | LR: 2.28e-04 | 1544.4ms/step | 12.9 samples/s +Step 23590/44000 | Loss: 2.9398 | LR: 2.28e-04 | 1693.2ms/step | 11.8 samples/s +Step 23600/44000 | Loss: 2.9631 | LR: 2.28e-04 | 1555.3ms/step | 12.9 samples/s +Step 23610/44000 | Loss: 3.0001 | LR: 2.28e-04 | 1515.6ms/step | 13.2 samples/s +Step 23620/44000 | Loss: 2.9552 | LR: 2.27e-04 | 1579.6ms/step | 12.7 samples/s +Step 23630/44000 | Loss: 2.9934 | LR: 2.27e-04 | 1585.3ms/step | 12.6 samples/s +Step 23640/44000 | Loss: 2.8952 | LR: 2.27e-04 | 1829.0ms/step | 10.9 samples/s +Step 23650/44000 | Loss: 2.9681 | LR: 2.27e-04 | 1543.6ms/step | 13.0 samples/s +Step 23660/44000 | Loss: 3.0451 | LR: 2.27e-04 | 1568.5ms/step | 12.8 samples/s +Step 23670/44000 | Loss: 2.9439 | LR: 2.27e-04 | 1583.3ms/step | 12.6 samples/s +Step 23680/44000 | Loss: 3.0021 | LR: 2.26e-04 | 1567.1ms/step | 12.8 samples/s +Step 23690/44000 | Loss: 3.0168 | LR: 2.26e-04 | 1570.4ms/step | 12.7 samples/s +Step 23700/44000 | Loss: 3.0213 | LR: 2.26e-04 | 1599.0ms/step | 12.5 samples/s +Step 23710/44000 | Loss: 2.9240 | LR: 2.26e-04 | 1574.1ms/step | 12.7 samples/s +Step 23720/44000 | Loss: 3.0297 | LR: 2.26e-04 | 1522.6ms/step | 13.1 samples/s +Step 23730/44000 | Loss: 2.9799 | LR: 2.25e-04 | 1486.2ms/step | 13.5 samples/s +Step 23740/44000 | Loss: 2.9580 | LR: 2.25e-04 | 1614.8ms/step | 12.4 samples/s +Step 23750/44000 | Loss: 3.0248 | LR: 2.25e-04 | 1483.6ms/step | 13.5 samples/s +Validation at step 23750: Loss = 2.9175, Perplexity = 18.50 +Step 23760/44000 | Loss: 2.8757 | LR: 2.25e-04 | 2101.9ms/step | 9.5 samples/s +Step 23770/44000 | Loss: 2.9648 | LR: 2.25e-04 | 1587.9ms/step | 12.6 samples/s +Step 23780/44000 | Loss: 3.0160 | LR: 2.25e-04 | 1771.5ms/step | 11.3 samples/s +Step 23790/44000 | Loss: 2.9895 | LR: 2.24e-04 | 1529.4ms/step | 13.1 samples/s +Step 23800/44000 | Loss: 2.8942 | LR: 2.24e-04 | 1587.5ms/step | 12.6 samples/s +Step 23810/44000 | Loss: 2.9798 | LR: 2.24e-04 | 1573.3ms/step | 12.7 samples/s +Step 23820/44000 | Loss: 2.9550 | LR: 2.24e-04 | 1612.0ms/step | 12.4 samples/s +Step 23830/44000 | Loss: 2.9666 | LR: 2.24e-04 | 1593.4ms/step | 12.6 samples/s +Step 23840/44000 | Loss: 2.9526 | LR: 2.23e-04 | 1620.5ms/step | 12.3 samples/s +Step 23850/44000 | Loss: 2.9998 | LR: 2.23e-04 | 1533.7ms/step | 13.0 samples/s +Step 23860/44000 | Loss: 2.9584 | LR: 2.23e-04 | 1521.6ms/step | 13.1 samples/s +Step 23870/44000 | Loss: 2.9824 | LR: 2.23e-04 | 1608.9ms/step | 12.4 samples/s +Step 23880/44000 | Loss: 2.9566 | LR: 2.23e-04 | 1598.0ms/step | 12.5 samples/s +Step 23890/44000 | Loss: 2.9508 | LR: 2.23e-04 | 1538.1ms/step | 13.0 samples/s +Step 23900/44000 | Loss: 3.0129 | LR: 2.22e-04 | 1587.8ms/step | 12.6 samples/s +Step 23910/44000 | Loss: 2.9363 | LR: 2.22e-04 | 1531.4ms/step | 13.1 samples/s +Step 23920/44000 | Loss: 3.0250 | LR: 2.22e-04 | 1760.6ms/step | 11.4 samples/s +Step 23930/44000 | Loss: 2.9761 | LR: 2.22e-04 | 1595.7ms/step | 12.5 samples/s +Step 23940/44000 | Loss: 2.9447 | LR: 2.22e-04 | 1539.7ms/step | 13.0 samples/s +Step 23950/44000 | Loss: 2.9099 | LR: 2.21e-04 | 1559.1ms/step | 12.8 samples/s +Step 23960/44000 | Loss: 2.9394 | LR: 2.21e-04 | 1580.2ms/step | 12.7 samples/s +Step 23970/44000 | Loss: 2.9589 | LR: 2.21e-04 | 1588.4ms/step | 12.6 samples/s +Step 23980/44000 | Loss: 2.9363 | LR: 2.21e-04 | 1560.9ms/step | 12.8 samples/s +Step 23990/44000 | Loss: 2.9729 | LR: 2.21e-04 | 1572.8ms/step | 12.7 samples/s +Step 24000/44000 | Loss: 3.0270 | LR: 2.21e-04 | 1549.2ms/step | 12.9 samples/s +Validation at step 24000: Loss = 2.9156, Perplexity = 18.46 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-24000 +Step 24010/44000 | Loss: 2.9979 | LR: 2.20e-04 | 2789.4ms/step | 7.2 samples/s +Step 24020/44000 | Loss: 2.9330 | LR: 2.20e-04 | 1555.1ms/step | 12.9 samples/s +Step 24030/44000 | Loss: 2.9529 | LR: 2.20e-04 | 1564.1ms/step | 12.8 samples/s +Step 24040/44000 | Loss: 2.8931 | LR: 2.20e-04 | 1471.9ms/step | 13.6 samples/s +Step 24050/44000 | Loss: 2.9300 | LR: 2.20e-04 | 1621.2ms/step | 12.3 samples/s +Step 24060/44000 | Loss: 2.9041 | LR: 2.19e-04 | 1757.2ms/step | 11.4 samples/s +Step 24070/44000 | Loss: 2.9818 | LR: 2.19e-04 | 1540.3ms/step | 13.0 samples/s +Step 24080/44000 | Loss: 2.9664 | LR: 2.19e-04 | 1562.3ms/step | 12.8 samples/s +Step 24090/44000 | Loss: 2.9619 | LR: 2.19e-04 | 1520.0ms/step | 13.2 samples/s +Step 24100/44000 | Loss: 2.9688 | LR: 2.19e-04 | 1562.2ms/step | 12.8 samples/s +Step 24110/44000 | Loss: 2.9803 | LR: 2.19e-04 | 1532.8ms/step | 13.0 samples/s +Step 24120/44000 | Loss: 2.9581 | LR: 2.18e-04 | 1577.5ms/step | 12.7 samples/s +Step 24130/44000 | Loss: 2.9749 | LR: 2.18e-04 | 1533.3ms/step | 13.0 samples/s +Step 24140/44000 | Loss: 2.9054 | LR: 2.18e-04 | 1583.2ms/step | 12.6 samples/s +Step 24150/44000 | Loss: 2.9982 | LR: 2.18e-04 | 1588.1ms/step | 12.6 samples/s +Step 24160/44000 | Loss: 2.9059 | LR: 2.18e-04 | 1577.8ms/step | 12.7 samples/s +Step 24170/44000 | Loss: 2.9322 | LR: 2.17e-04 | 1509.5ms/step | 13.2 samples/s +Step 24180/44000 | Loss: 2.9857 | LR: 2.17e-04 | 1449.2ms/step | 13.8 samples/s +Step 24190/44000 | Loss: 2.9539 | LR: 2.17e-04 | 1491.1ms/step | 13.4 samples/s +Step 24200/44000 | Loss: 2.9129 | LR: 2.17e-04 | 1745.5ms/step | 11.5 samples/s +Step 24210/44000 | Loss: 2.9568 | LR: 2.17e-04 | 1488.5ms/step | 13.4 samples/s +Step 24220/44000 | Loss: 2.9110 | LR: 2.17e-04 | 1526.0ms/step | 13.1 samples/s +Step 24230/44000 | Loss: 2.9841 | LR: 2.16e-04 | 1476.9ms/step | 13.5 samples/s +Step 24240/44000 | Loss: 2.9245 | LR: 2.16e-04 | 1519.8ms/step | 13.2 samples/s +Step 24250/44000 | Loss: 2.9892 | LR: 2.16e-04 | 1551.0ms/step | 12.9 samples/s +Validation at step 24250: Loss = 2.9126, Perplexity = 18.40 +Step 24260/44000 | Loss: 2.9637 | LR: 2.16e-04 | 2174.6ms/step | 9.2 samples/s +Step 24270/44000 | Loss: 3.0161 | LR: 2.16e-04 | 1596.6ms/step | 12.5 samples/s +Step 24280/44000 | Loss: 2.9388 | LR: 2.16e-04 | 1563.4ms/step | 12.8 samples/s +Step 24290/44000 | Loss: 3.0407 | LR: 2.15e-04 | 1572.3ms/step | 12.7 samples/s +Step 24300/44000 | Loss: 2.9896 | LR: 2.15e-04 | 1558.1ms/step | 12.8 samples/s +Step 24310/44000 | Loss: 2.8934 | LR: 2.15e-04 | 1597.7ms/step | 12.5 samples/s +Step 24320/44000 | Loss: 2.9472 | LR: 2.15e-04 | 1551.1ms/step | 12.9 samples/s +Step 24330/44000 | Loss: 2.9559 | LR: 2.15e-04 | 1609.0ms/step | 12.4 samples/s +Step 24340/44000 | Loss: 2.9919 | LR: 2.14e-04 | 1787.1ms/step | 11.2 samples/s +Step 24350/44000 | Loss: 2.9751 | LR: 2.14e-04 | 1481.0ms/step | 13.5 samples/s +Step 24360/44000 | Loss: 2.8932 | LR: 2.14e-04 | 1611.2ms/step | 12.4 samples/s +Step 24370/44000 | Loss: 2.9873 | LR: 2.14e-04 | 1544.7ms/step | 12.9 samples/s +Step 24380/44000 | Loss: 2.9282 | LR: 2.14e-04 | 1548.1ms/step | 12.9 samples/s +Step 24390/44000 | Loss: 3.0135 | LR: 2.14e-04 | 1535.6ms/step | 13.0 samples/s +Step 24400/44000 | Loss: 2.9943 | LR: 2.13e-04 | 1562.6ms/step | 12.8 samples/s +Step 24410/44000 | Loss: 2.9594 | LR: 2.13e-04 | 1532.1ms/step | 13.1 samples/s +Step 24420/44000 | Loss: 3.0004 | LR: 2.13e-04 | 1560.8ms/step | 12.8 samples/s +Step 24430/44000 | Loss: 2.8757 | LR: 2.13e-04 | 1540.4ms/step | 13.0 samples/s +Step 24440/44000 | Loss: 2.9081 | LR: 2.13e-04 | 1578.5ms/step | 12.7 samples/s +Step 24450/44000 | Loss: 2.9456 | LR: 2.12e-04 | 1551.2ms/step | 12.9 samples/s +Step 24460/44000 | Loss: 3.0146 | LR: 2.12e-04 | 1533.7ms/step | 13.0 samples/s +Step 24470/44000 | Loss: 2.9052 | LR: 2.12e-04 | 1503.2ms/step | 13.3 samples/s +Step 24480/44000 | Loss: 2.9751 | LR: 2.12e-04 | 1592.8ms/step | 12.6 samples/s +Step 24490/44000 | Loss: 2.8889 | LR: 2.12e-04 | 1666.8ms/step | 12.0 samples/s +Step 24500/44000 | Loss: 2.9901 | LR: 2.12e-04 | 1504.6ms/step | 13.3 samples/s +Validation at step 24500: Loss = 2.9111, Perplexity = 18.38 +Step 24510/44000 | Loss: 2.9498 | LR: 2.11e-04 | 2137.1ms/step | 9.4 samples/s +Step 24520/44000 | Loss: 2.9947 | LR: 2.11e-04 | 1564.3ms/step | 12.8 samples/s +Step 24530/44000 | Loss: 2.9211 | LR: 2.11e-04 | 1562.0ms/step | 12.8 samples/s +Step 24540/44000 | Loss: 2.9030 | LR: 2.11e-04 | 1572.1ms/step | 12.7 samples/s +Step 24550/44000 | Loss: 2.9685 | LR: 2.11e-04 | 1604.5ms/step | 12.5 samples/s +Step 24560/44000 | Loss: 2.9346 | LR: 2.10e-04 | 1524.1ms/step | 13.1 samples/s +Step 24570/44000 | Loss: 3.0037 | LR: 2.10e-04 | 1568.7ms/step | 12.7 samples/s +Step 24580/44000 | Loss: 2.9638 | LR: 2.10e-04 | 1557.1ms/step | 12.8 samples/s +Step 24590/44000 | Loss: 2.9941 | LR: 2.10e-04 | 1529.8ms/step | 13.1 samples/s +Step 24600/44000 | Loss: 2.9757 | LR: 2.10e-04 | 1553.2ms/step | 12.9 samples/s +Step 24610/44000 | Loss: 2.9704 | LR: 2.10e-04 | 1525.6ms/step | 13.1 samples/s +Step 24620/44000 | Loss: 2.9549 | LR: 2.09e-04 | 1687.4ms/step | 11.9 samples/s +Step 24630/44000 | Loss: 2.8738 | LR: 2.09e-04 | 1664.3ms/step | 12.0 samples/s +Step 24640/44000 | Loss: 3.0044 | LR: 2.09e-04 | 1449.4ms/step | 13.8 samples/s +Step 24650/44000 | Loss: 2.9195 | LR: 2.09e-04 | 1546.2ms/step | 12.9 samples/s +Step 24660/44000 | Loss: 2.9674 | LR: 2.09e-04 | 1513.8ms/step | 13.2 samples/s +Step 24670/44000 | Loss: 2.8936 | LR: 2.09e-04 | 1526.5ms/step | 13.1 samples/s +Step 24680/44000 | Loss: 2.9156 | LR: 2.08e-04 | 1431.6ms/step | 14.0 samples/s +Step 24690/44000 | Loss: 2.9368 | LR: 2.08e-04 | 1562.1ms/step | 12.8 samples/s +Step 24700/44000 | Loss: 2.9907 | LR: 2.08e-04 | 1527.5ms/step | 13.1 samples/s +Step 24710/44000 | Loss: 2.9523 | LR: 2.08e-04 | 1530.0ms/step | 13.1 samples/s +Step 24720/44000 | Loss: 2.9564 | LR: 2.08e-04 | 1620.3ms/step | 12.3 samples/s +Step 24730/44000 | Loss: 2.9495 | LR: 2.07e-04 | 1546.7ms/step | 12.9 samples/s +Step 24740/44000 | Loss: 2.9142 | LR: 2.07e-04 | 1539.7ms/step | 13.0 samples/s +Step 24750/44000 | Loss: 2.9258 | LR: 2.07e-04 | 1527.2ms/step | 13.1 samples/s +Validation at step 24750: Loss = 2.9107, Perplexity = 18.37 +Step 24760/44000 | Loss: 3.0000 | LR: 2.07e-04 | 2216.0ms/step | 9.0 samples/s +Step 24770/44000 | Loss: 2.9617 | LR: 2.07e-04 | 1737.1ms/step | 11.5 samples/s +Step 24780/44000 | Loss: 2.9993 | LR: 2.07e-04 | 1587.4ms/step | 12.6 samples/s +Step 24790/44000 | Loss: 3.0040 | LR: 2.06e-04 | 1561.4ms/step | 12.8 samples/s +Step 24800/44000 | Loss: 2.9823 | LR: 2.06e-04 | 1537.6ms/step | 13.0 samples/s +Step 24810/44000 | Loss: 2.9245 | LR: 2.06e-04 | 1560.8ms/step | 12.8 samples/s +Step 24820/44000 | Loss: 2.9577 | LR: 2.06e-04 | 1536.6ms/step | 13.0 samples/s +Step 24830/44000 | Loss: 2.9543 | LR: 2.06e-04 | 1562.2ms/step | 12.8 samples/s +Step 24840/44000 | Loss: 2.8841 | LR: 2.05e-04 | 1528.2ms/step | 13.1 samples/s +Step 24850/44000 | Loss: 2.9851 | LR: 2.05e-04 | 1601.5ms/step | 12.5 samples/s +Step 24860/44000 | Loss: 2.9460 | LR: 2.05e-04 | 1557.0ms/step | 12.8 samples/s +Step 24870/44000 | Loss: 2.9597 | LR: 2.05e-04 | 1603.6ms/step | 12.5 samples/s +Step 24880/44000 | Loss: 2.9618 | LR: 2.05e-04 | 1602.8ms/step | 12.5 samples/s +Step 24890/44000 | Loss: 2.9339 | LR: 2.05e-04 | 1579.1ms/step | 12.7 samples/s +Step 24900/44000 | Loss: 2.9608 | LR: 2.04e-04 | 1530.9ms/step | 13.1 samples/s +Step 24910/44000 | Loss: 2.9409 | LR: 2.04e-04 | 1782.8ms/step | 11.2 samples/s +Step 24920/44000 | Loss: 2.9934 | LR: 2.04e-04 | 1598.5ms/step | 12.5 samples/s +Step 24930/44000 | Loss: 2.9903 | LR: 2.04e-04 | 1562.4ms/step | 12.8 samples/s +Step 24940/44000 | Loss: 2.9157 | LR: 2.04e-04 | 1583.4ms/step | 12.6 samples/s +Step 24950/44000 | Loss: 2.9835 | LR: 2.04e-04 | 1615.8ms/step | 12.4 samples/s +Step 24960/44000 | Loss: 2.9750 | LR: 2.03e-04 | 1632.9ms/step | 12.2 samples/s +Step 24970/44000 | Loss: 3.0161 | LR: 2.03e-04 | 1580.3ms/step | 12.7 samples/s +Step 24980/44000 | Loss: 2.9336 | LR: 2.03e-04 | 1531.0ms/step | 13.1 samples/s +Step 24990/44000 | Loss: 2.8910 | LR: 2.03e-04 | 1558.1ms/step | 12.8 samples/s +Step 25000/44000 | Loss: 2.9195 | LR: 2.03e-04 | 1639.7ms/step | 12.2 samples/s +Validation at step 25000: Loss = 2.9075, Perplexity = 18.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-25000 +Step 25010/44000 | Loss: 2.9857 | LR: 2.02e-04 | 2835.9ms/step | 7.1 samples/s +Step 25020/44000 | Loss: 2.9723 | LR: 2.02e-04 | 1552.5ms/step | 12.9 samples/s +Step 25030/44000 | Loss: 2.9542 | LR: 2.02e-04 | 1509.0ms/step | 13.3 samples/s +Step 25040/44000 | Loss: 3.0029 | LR: 2.02e-04 | 1796.5ms/step | 11.1 samples/s +Step 25050/44000 | Loss: 3.0233 | LR: 2.02e-04 | 1568.3ms/step | 12.8 samples/s +Step 25060/44000 | Loss: 2.9551 | LR: 2.02e-04 | 1524.5ms/step | 13.1 samples/s +Step 25070/44000 | Loss: 2.9800 | LR: 2.01e-04 | 1508.7ms/step | 13.3 samples/s +Step 25080/44000 | Loss: 2.8825 | LR: 2.01e-04 | 1506.1ms/step | 13.3 samples/s +Step 25090/44000 | Loss: 2.9327 | LR: 2.01e-04 | 1516.9ms/step | 13.2 samples/s +Step 25100/44000 | Loss: 2.8865 | LR: 2.01e-04 | 1423.6ms/step | 14.0 samples/s +Step 25110/44000 | Loss: 2.8602 | LR: 2.01e-04 | 1546.3ms/step | 12.9 samples/s +Step 25120/44000 | Loss: 2.9791 | LR: 2.00e-04 | 1503.9ms/step | 13.3 samples/s +Step 25130/44000 | Loss: 2.9442 | LR: 2.00e-04 | 1504.3ms/step | 13.3 samples/s +Step 25140/44000 | Loss: 2.9382 | LR: 2.00e-04 | 1475.4ms/step | 13.6 samples/s +Step 25150/44000 | Loss: 2.9279 | LR: 2.00e-04 | 1518.0ms/step | 13.2 samples/s +Step 25160/44000 | Loss: 2.9844 | LR: 2.00e-04 | 1598.4ms/step | 12.5 samples/s +Step 25170/44000 | Loss: 2.9489 | LR: 2.00e-04 | 1526.1ms/step | 13.1 samples/s +Step 25180/44000 | Loss: 2.9483 | LR: 1.99e-04 | 1550.9ms/step | 12.9 samples/s +Step 25190/44000 | Loss: 2.9149 | LR: 1.99e-04 | 1803.5ms/step | 11.1 samples/s +Step 25200/44000 | Loss: 2.9477 | LR: 1.99e-04 | 1663.4ms/step | 12.0 samples/s +Step 25210/44000 | Loss: 2.9675 | LR: 1.99e-04 | 1561.2ms/step | 12.8 samples/s +Step 25220/44000 | Loss: 2.9553 | LR: 1.99e-04 | 1557.1ms/step | 12.8 samples/s +Step 25230/44000 | Loss: 2.8951 | LR: 1.99e-04 | 1520.3ms/step | 13.2 samples/s +Step 25240/44000 | Loss: 2.9669 | LR: 1.98e-04 | 1611.8ms/step | 12.4 samples/s +Step 25250/44000 | Loss: 2.8939 | LR: 1.98e-04 | 1549.4ms/step | 12.9 samples/s +Validation at step 25250: Loss = 2.9061, Perplexity = 18.28 +Step 25260/44000 | Loss: 2.9523 | LR: 1.98e-04 | 2164.9ms/step | 9.2 samples/s +Step 25270/44000 | Loss: 2.9364 | LR: 1.98e-04 | 1598.8ms/step | 12.5 samples/s +Step 25280/44000 | Loss: 2.9799 | LR: 1.98e-04 | 1618.7ms/step | 12.4 samples/s +Step 25290/44000 | Loss: 2.9618 | LR: 1.97e-04 | 1568.5ms/step | 12.8 samples/s +Step 25300/44000 | Loss: 3.0155 | LR: 1.97e-04 | 1523.3ms/step | 13.1 samples/s +Step 25310/44000 | Loss: 3.0048 | LR: 1.97e-04 | 1540.6ms/step | 13.0 samples/s +Step 25320/44000 | Loss: 2.9199 | LR: 1.97e-04 | 1648.2ms/step | 12.1 samples/s +Step 25330/44000 | Loss: 2.9668 | LR: 1.97e-04 | 1703.8ms/step | 11.7 samples/s +Step 25340/44000 | Loss: 2.9941 | LR: 1.97e-04 | 1515.0ms/step | 13.2 samples/s +Step 25350/44000 | Loss: 2.9670 | LR: 1.96e-04 | 1538.6ms/step | 13.0 samples/s +Step 25360/44000 | Loss: 2.9588 | LR: 1.96e-04 | 1568.9ms/step | 12.7 samples/s +Step 25370/44000 | Loss: 2.9901 | LR: 1.96e-04 | 1565.6ms/step | 12.8 samples/s +Step 25380/44000 | Loss: 2.9417 | LR: 1.96e-04 | 1516.4ms/step | 13.2 samples/s +Step 25390/44000 | Loss: 2.9140 | LR: 1.96e-04 | 1530.5ms/step | 13.1 samples/s +Step 25400/44000 | Loss: 2.9589 | LR: 1.96e-04 | 1510.9ms/step | 13.2 samples/s +Step 25410/44000 | Loss: 3.0021 | LR: 1.95e-04 | 1586.5ms/step | 12.6 samples/s +Step 25420/44000 | Loss: 2.9756 | LR: 1.95e-04 | 1509.8ms/step | 13.2 samples/s +Step 25430/44000 | Loss: 2.9520 | LR: 1.95e-04 | 1530.9ms/step | 13.1 samples/s +Step 25440/44000 | Loss: 2.9690 | LR: 1.95e-04 | 1494.1ms/step | 13.4 samples/s +Step 25450/44000 | Loss: 2.9546 | LR: 1.95e-04 | 1563.1ms/step | 12.8 samples/s +Step 25460/44000 | Loss: 2.9528 | LR: 1.94e-04 | 1601.4ms/step | 12.5 samples/s +Step 25470/44000 | Loss: 2.9584 | LR: 1.94e-04 | 1792.7ms/step | 11.2 samples/s +Step 25480/44000 | Loss: 2.9270 | LR: 1.94e-04 | 1616.9ms/step | 12.4 samples/s +Step 25490/44000 | Loss: 2.9800 | LR: 1.94e-04 | 1577.9ms/step | 12.7 samples/s +Step 25500/44000 | Loss: 2.9399 | LR: 1.94e-04 | 1547.2ms/step | 12.9 samples/s +Validation at step 25500: Loss = 2.9032, Perplexity = 18.23 +Step 25510/44000 | Loss: 2.9551 | LR: 1.94e-04 | 2189.1ms/step | 9.1 samples/s +Step 25520/44000 | Loss: 2.9902 | LR: 1.93e-04 | 1567.5ms/step | 12.8 samples/s +Step 25530/44000 | Loss: 2.9731 | LR: 1.93e-04 | 1606.2ms/step | 12.5 samples/s +Step 25540/44000 | Loss: 3.0088 | LR: 1.93e-04 | 1579.2ms/step | 12.7 samples/s +Step 25550/44000 | Loss: 3.0168 | LR: 1.93e-04 | 1609.6ms/step | 12.4 samples/s +Step 25560/44000 | Loss: 2.9423 | LR: 1.93e-04 | 1505.0ms/step | 13.3 samples/s +Step 25570/44000 | Loss: 2.9396 | LR: 1.92e-04 | 1591.1ms/step | 12.6 samples/s +Step 25580/44000 | Loss: 2.8926 | LR: 1.92e-04 | 1554.0ms/step | 12.9 samples/s +Step 25590/44000 | Loss: 2.9226 | LR: 1.92e-04 | 1540.8ms/step | 13.0 samples/s +Step 25600/44000 | Loss: 2.9595 | LR: 1.92e-04 | 1539.0ms/step | 13.0 samples/s +Step 25610/44000 | Loss: 2.9752 | LR: 1.92e-04 | 1659.0ms/step | 12.1 samples/s +Step 25620/44000 | Loss: 2.8826 | LR: 1.92e-04 | 1562.4ms/step | 12.8 samples/s +Step 25630/44000 | Loss: 2.9339 | LR: 1.91e-04 | 1541.3ms/step | 13.0 samples/s +Step 25640/44000 | Loss: 2.9475 | LR: 1.91e-04 | 1542.7ms/step | 13.0 samples/s +Step 25650/44000 | Loss: 3.0059 | LR: 1.91e-04 | 1576.8ms/step | 12.7 samples/s +Step 25660/44000 | Loss: 2.9632 | LR: 1.91e-04 | 1535.6ms/step | 13.0 samples/s +Step 25670/44000 | Loss: 2.9278 | LR: 1.91e-04 | 1557.6ms/step | 12.8 samples/s +Step 25680/44000 | Loss: 2.9585 | LR: 1.91e-04 | 1499.6ms/step | 13.3 samples/s +Step 25690/44000 | Loss: 2.9439 | LR: 1.90e-04 | 1592.9ms/step | 12.6 samples/s +Step 25700/44000 | Loss: 2.9825 | LR: 1.90e-04 | 1570.8ms/step | 12.7 samples/s +Step 25710/44000 | Loss: 2.9992 | LR: 1.90e-04 | 1532.1ms/step | 13.1 samples/s +Step 25720/44000 | Loss: 2.9953 | LR: 1.90e-04 | 1543.5ms/step | 13.0 samples/s +Step 25730/44000 | Loss: 2.9520 | LR: 1.90e-04 | 1563.9ms/step | 12.8 samples/s +Step 25740/44000 | Loss: 2.8952 | LR: 1.89e-04 | 1529.3ms/step | 13.1 samples/s +Step 25750/44000 | Loss: 2.9420 | LR: 1.89e-04 | 1651.2ms/step | 12.1 samples/s +Validation at step 25750: Loss = 2.9022, Perplexity = 18.21 +Step 25760/44000 | Loss: 2.9653 | LR: 1.89e-04 | 2269.3ms/step | 8.8 samples/s +Step 25770/44000 | Loss: 2.9045 | LR: 1.89e-04 | 1614.0ms/step | 12.4 samples/s +Step 25780/44000 | Loss: 2.9630 | LR: 1.89e-04 | 1575.6ms/step | 12.7 samples/s +Step 25790/44000 | Loss: 2.9148 | LR: 1.89e-04 | 1512.2ms/step | 13.2 samples/s +Step 25800/44000 | Loss: 2.9596 | LR: 1.88e-04 | 1546.8ms/step | 12.9 samples/s +Step 25810/44000 | Loss: 2.8716 | LR: 1.88e-04 | 1593.7ms/step | 12.5 samples/s +Step 25820/44000 | Loss: 2.9625 | LR: 1.88e-04 | 1615.7ms/step | 12.4 samples/s +Step 25830/44000 | Loss: 3.0289 | LR: 1.88e-04 | 1599.1ms/step | 12.5 samples/s +Step 25840/44000 | Loss: 2.9419 | LR: 1.88e-04 | 1598.2ms/step | 12.5 samples/s +Step 25850/44000 | Loss: 2.9657 | LR: 1.88e-04 | 1551.8ms/step | 12.9 samples/s +Step 25860/44000 | Loss: 2.9518 | LR: 1.87e-04 | 1557.2ms/step | 12.8 samples/s +Step 25870/44000 | Loss: 2.9385 | LR: 1.87e-04 | 1574.3ms/step | 12.7 samples/s +Step 25880/44000 | Loss: 2.9736 | LR: 1.87e-04 | 1540.7ms/step | 13.0 samples/s +Step 25890/44000 | Loss: 2.8954 | LR: 1.87e-04 | 1718.9ms/step | 11.6 samples/s +Step 25900/44000 | Loss: 2.9519 | LR: 1.87e-04 | 1567.5ms/step | 12.8 samples/s +Step 25910/44000 | Loss: 3.0142 | LR: 1.87e-04 | 1590.6ms/step | 12.6 samples/s +Step 25920/44000 | Loss: 2.9666 | LR: 1.86e-04 | 1607.3ms/step | 12.4 samples/s +Step 25930/44000 | Loss: 2.9508 | LR: 1.86e-04 | 1556.1ms/step | 12.9 samples/s +Step 25940/44000 | Loss: 2.9635 | LR: 1.86e-04 | 1541.8ms/step | 13.0 samples/s +Step 25950/44000 | Loss: 2.8843 | LR: 1.86e-04 | 1532.1ms/step | 13.1 samples/s +Step 25960/44000 | Loss: 2.9489 | LR: 1.86e-04 | 1546.4ms/step | 12.9 samples/s +Step 25970/44000 | Loss: 2.9188 | LR: 1.85e-04 | 1515.1ms/step | 13.2 samples/s +Step 25980/44000 | Loss: 2.9356 | LR: 1.85e-04 | 1489.0ms/step | 13.4 samples/s +Step 25990/44000 | Loss: 3.0094 | LR: 1.85e-04 | 1561.0ms/step | 12.8 samples/s +Step 26000/44000 | Loss: 2.9138 | LR: 1.85e-04 | 1526.7ms/step | 13.1 samples/s +Validation at step 26000: Loss = 2.9006, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-26000 +Step 26010/44000 | Loss: 2.9545 | LR: 1.85e-04 | 2794.7ms/step | 7.2 samples/s +Step 26020/44000 | Loss: 2.9370 | LR: 1.85e-04 | 1503.2ms/step | 13.3 samples/s +Step 26030/44000 | Loss: 2.9605 | LR: 1.84e-04 | 1679.9ms/step | 11.9 samples/s +Step 26040/44000 | Loss: 3.0037 | LR: 1.84e-04 | 1591.3ms/step | 12.6 samples/s +Step 26050/44000 | Loss: 2.9086 | LR: 1.84e-04 | 1562.6ms/step | 12.8 samples/s +Step 26060/44000 | Loss: 2.9996 | LR: 1.84e-04 | 1557.3ms/step | 12.8 samples/s +Step 26070/44000 | Loss: 2.9119 | LR: 1.84e-04 | 1545.1ms/step | 12.9 samples/s +Step 26080/44000 | Loss: 2.9642 | LR: 1.84e-04 | 1489.5ms/step | 13.4 samples/s +Step 26090/44000 | Loss: 2.9226 | LR: 1.83e-04 | 1596.6ms/step | 12.5 samples/s +Step 26100/44000 | Loss: 2.9851 | LR: 1.83e-04 | 1575.6ms/step | 12.7 samples/s +Step 26110/44000 | Loss: 2.9892 | LR: 1.83e-04 | 1572.6ms/step | 12.7 samples/s +Step 26120/44000 | Loss: 2.9708 | LR: 1.83e-04 | 1551.7ms/step | 12.9 samples/s +Step 26130/44000 | Loss: 3.0198 | LR: 1.83e-04 | 1609.6ms/step | 12.4 samples/s +Step 26140/44000 | Loss: 2.9484 | LR: 1.82e-04 | 1525.6ms/step | 13.1 samples/s +Step 26150/44000 | Loss: 2.9406 | LR: 1.82e-04 | 1495.0ms/step | 13.4 samples/s +Step 26160/44000 | Loss: 2.9412 | LR: 1.82e-04 | 1548.1ms/step | 12.9 samples/s +Step 26170/44000 | Loss: 2.9242 | LR: 1.82e-04 | 1804.6ms/step | 11.1 samples/s +Step 26180/44000 | Loss: 2.9801 | LR: 1.82e-04 | 1571.9ms/step | 12.7 samples/s +Step 26190/44000 | Loss: 3.0065 | LR: 1.82e-04 | 1630.0ms/step | 12.3 samples/s +Step 26200/44000 | Loss: 2.9853 | LR: 1.81e-04 | 1527.6ms/step | 13.1 samples/s +Step 26210/44000 | Loss: 2.9613 | LR: 1.81e-04 | 1536.5ms/step | 13.0 samples/s +Step 26220/44000 | Loss: 2.9798 | LR: 1.81e-04 | 1563.5ms/step | 12.8 samples/s +Step 26230/44000 | Loss: 3.0416 | LR: 1.81e-04 | 1696.9ms/step | 11.8 samples/s +Step 26240/44000 | Loss: 2.9585 | LR: 1.81e-04 | 1553.1ms/step | 12.9 samples/s +Step 26250/44000 | Loss: 2.8924 | LR: 1.81e-04 | 1536.6ms/step | 13.0 samples/s +Validation at step 26250: Loss = 2.8992, Perplexity = 18.16 +Step 26260/44000 | Loss: 2.9387 | LR: 1.80e-04 | 2084.4ms/step | 9.6 samples/s +Step 26270/44000 | Loss: 2.9559 | LR: 1.80e-04 | 1555.2ms/step | 12.9 samples/s +Step 26280/44000 | Loss: 2.9712 | LR: 1.80e-04 | 1532.7ms/step | 13.0 samples/s +Step 26290/44000 | Loss: 2.9608 | LR: 1.80e-04 | 1564.4ms/step | 12.8 samples/s +Step 26300/44000 | Loss: 3.0080 | LR: 1.80e-04 | 1591.5ms/step | 12.6 samples/s +Step 26310/44000 | Loss: 2.8989 | LR: 1.80e-04 | 1777.6ms/step | 11.3 samples/s +Step 26320/44000 | Loss: 2.9040 | LR: 1.79e-04 | 1518.5ms/step | 13.2 samples/s +Step 26330/44000 | Loss: 2.9938 | LR: 1.79e-04 | 1532.1ms/step | 13.1 samples/s +Step 26340/44000 | Loss: 2.9165 | LR: 1.79e-04 | 1516.9ms/step | 13.2 samples/s +Step 26350/44000 | Loss: 2.9097 | LR: 1.79e-04 | 1515.3ms/step | 13.2 samples/s +Step 26360/44000 | Loss: 2.9304 | LR: 1.79e-04 | 1596.0ms/step | 12.5 samples/s +Step 26370/44000 | Loss: 2.9233 | LR: 1.78e-04 | 1570.1ms/step | 12.7 samples/s +Step 26380/44000 | Loss: 2.9879 | LR: 1.78e-04 | 1606.8ms/step | 12.4 samples/s +Step 26390/44000 | Loss: 2.8973 | LR: 1.78e-04 | 1573.0ms/step | 12.7 samples/s +Step 26400/44000 | Loss: 2.9329 | LR: 1.78e-04 | 1587.0ms/step | 12.6 samples/s +Step 26410/44000 | Loss: 2.9707 | LR: 1.78e-04 | 1596.2ms/step | 12.5 samples/s +Step 26420/44000 | Loss: 3.1066 | LR: 1.78e-04 | 1553.7ms/step | 12.9 samples/s +Step 26430/44000 | Loss: 2.9431 | LR: 1.77e-04 | 1571.6ms/step | 12.7 samples/s +Step 26440/44000 | Loss: 2.9851 | LR: 1.77e-04 | 1593.1ms/step | 12.6 samples/s +Step 26450/44000 | Loss: 2.9409 | LR: 1.77e-04 | 1730.3ms/step | 11.6 samples/s +Step 26460/44000 | Loss: 2.9258 | LR: 1.77e-04 | 1599.1ms/step | 12.5 samples/s +Step 26470/44000 | Loss: 2.9049 | LR: 1.77e-04 | 1562.1ms/step | 12.8 samples/s +Step 26480/44000 | Loss: 2.9637 | LR: 1.77e-04 | 1523.7ms/step | 13.1 samples/s +Step 26490/44000 | Loss: 2.9470 | LR: 1.76e-04 | 1441.3ms/step | 13.9 samples/s +Step 26500/44000 | Loss: 2.9651 | LR: 1.76e-04 | 1587.8ms/step | 12.6 samples/s +Validation at step 26500: Loss = 2.8977, Perplexity = 18.13 +Step 26510/44000 | Loss: 2.9818 | LR: 1.76e-04 | 2138.2ms/step | 9.4 samples/s +Step 26520/44000 | Loss: 2.9735 | LR: 1.76e-04 | 1572.7ms/step | 12.7 samples/s +Step 26530/44000 | Loss: 2.9715 | LR: 1.76e-04 | 1563.4ms/step | 12.8 samples/s +Step 26540/44000 | Loss: 2.9804 | LR: 1.76e-04 | 1472.3ms/step | 13.6 samples/s +Step 26550/44000 | Loss: 2.9203 | LR: 1.75e-04 | 1525.8ms/step | 13.1 samples/s +Step 26560/44000 | Loss: 2.9616 | LR: 1.75e-04 | 1504.7ms/step | 13.3 samples/s +Step 26570/44000 | Loss: 2.9003 | LR: 1.75e-04 | 1531.7ms/step | 13.1 samples/s +Step 26580/44000 | Loss: 3.0381 | LR: 1.75e-04 | 1598.6ms/step | 12.5 samples/s +Step 26590/44000 | Loss: 2.9346 | LR: 1.75e-04 | 1775.2ms/step | 11.3 samples/s +Step 26600/44000 | Loss: 2.8413 | LR: 1.74e-04 | 1662.2ms/step | 12.0 samples/s +Step 26610/44000 | Loss: 2.8636 | LR: 1.74e-04 | 1608.9ms/step | 12.4 samples/s +Step 26620/44000 | Loss: 3.0037 | LR: 1.74e-04 | 1533.8ms/step | 13.0 samples/s +Step 26630/44000 | Loss: 2.9871 | LR: 1.74e-04 | 1572.5ms/step | 12.7 samples/s +Step 26640/44000 | Loss: 2.8986 | LR: 1.74e-04 | 1621.8ms/step | 12.3 samples/s +Step 26650/44000 | Loss: 2.9789 | LR: 1.74e-04 | 1577.3ms/step | 12.7 samples/s +Step 26660/44000 | Loss: 2.9642 | LR: 1.73e-04 | 1566.9ms/step | 12.8 samples/s +Step 26670/44000 | Loss: 2.9191 | LR: 1.73e-04 | 1525.3ms/step | 13.1 samples/s +Step 26680/44000 | Loss: 3.0251 | LR: 1.73e-04 | 1538.8ms/step | 13.0 samples/s +Step 26690/44000 | Loss: 2.8708 | LR: 1.73e-04 | 1533.3ms/step | 13.0 samples/s +Step 26700/44000 | Loss: 2.8192 | LR: 1.73e-04 | 1576.4ms/step | 12.7 samples/s +Step 26710/44000 | Loss: 2.9695 | LR: 1.73e-04 | 1548.6ms/step | 12.9 samples/s +Step 26720/44000 | Loss: 2.9956 | LR: 1.72e-04 | 1536.2ms/step | 13.0 samples/s +Step 26730/44000 | Loss: 2.9405 | LR: 1.72e-04 | 1660.2ms/step | 12.0 samples/s +Step 26740/44000 | Loss: 2.9734 | LR: 1.72e-04 | 1679.8ms/step | 11.9 samples/s +Step 26750/44000 | Loss: 2.9466 | LR: 1.72e-04 | 1493.1ms/step | 13.4 samples/s +Validation at step 26750: Loss = 2.8954, Perplexity = 18.09 +Step 26760/44000 | Loss: 2.9040 | LR: 1.72e-04 | 2163.8ms/step | 9.2 samples/s +Step 26770/44000 | Loss: 2.9767 | LR: 1.72e-04 | 1561.6ms/step | 12.8 samples/s +Step 26780/44000 | Loss: 2.9622 | LR: 1.71e-04 | 1558.8ms/step | 12.8 samples/s +Step 26790/44000 | Loss: 2.9281 | LR: 1.71e-04 | 1593.5ms/step | 12.6 samples/s +Step 26800/44000 | Loss: 2.9371 | LR: 1.71e-04 | 1512.6ms/step | 13.2 samples/s +Step 26810/44000 | Loss: 2.9434 | LR: 1.71e-04 | 1595.5ms/step | 12.5 samples/s +Step 26820/44000 | Loss: 2.9464 | LR: 1.71e-04 | 1617.3ms/step | 12.4 samples/s +Step 26830/44000 | Loss: 2.9617 | LR: 1.71e-04 | 1541.1ms/step | 13.0 samples/s +Step 26840/44000 | Loss: 2.9309 | LR: 1.70e-04 | 1501.2ms/step | 13.3 samples/s +Step 26850/44000 | Loss: 2.9500 | LR: 1.70e-04 | 1567.4ms/step | 12.8 samples/s +Step 26860/44000 | Loss: 2.9644 | LR: 1.70e-04 | 1601.8ms/step | 12.5 samples/s +Step 26870/44000 | Loss: 2.9454 | LR: 1.70e-04 | 1629.1ms/step | 12.3 samples/s +Step 26880/44000 | Loss: 2.9082 | LR: 1.70e-04 | 1662.3ms/step | 12.0 samples/s +Step 26890/44000 | Loss: 2.9456 | LR: 1.69e-04 | 1601.4ms/step | 12.5 samples/s +Step 26900/44000 | Loss: 2.9294 | LR: 1.69e-04 | 1546.6ms/step | 12.9 samples/s +Step 26910/44000 | Loss: 2.9130 | LR: 1.69e-04 | 1552.7ms/step | 12.9 samples/s +Step 26920/44000 | Loss: 2.9650 | LR: 1.69e-04 | 1587.2ms/step | 12.6 samples/s +Step 26930/44000 | Loss: 2.9528 | LR: 1.69e-04 | 1573.1ms/step | 12.7 samples/s +Step 26940/44000 | Loss: 2.8829 | LR: 1.69e-04 | 1573.7ms/step | 12.7 samples/s +Step 26950/44000 | Loss: 2.9378 | LR: 1.68e-04 | 1476.3ms/step | 13.5 samples/s +Step 26960/44000 | Loss: 2.9362 | LR: 1.68e-04 | 1614.4ms/step | 12.4 samples/s +Step 26970/44000 | Loss: 2.9891 | LR: 1.68e-04 | 1631.4ms/step | 12.3 samples/s +Step 26980/44000 | Loss: 2.9267 | LR: 1.68e-04 | 1493.5ms/step | 13.4 samples/s +Step 26990/44000 | Loss: 2.9301 | LR: 1.68e-04 | 1509.6ms/step | 13.2 samples/s +Step 27000/44000 | Loss: 2.9833 | LR: 1.68e-04 | 1613.7ms/step | 12.4 samples/s +Validation at step 27000: Loss = 2.8942, Perplexity = 18.07 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-27000 +Step 27010/44000 | Loss: 2.9730 | LR: 1.67e-04 | 2955.6ms/step | 6.8 samples/s +Step 27020/44000 | Loss: 3.0196 | LR: 1.67e-04 | 1541.0ms/step | 13.0 samples/s +Step 27030/44000 | Loss: 2.9375 | LR: 1.67e-04 | 1609.6ms/step | 12.4 samples/s +Step 27040/44000 | Loss: 2.9289 | LR: 1.67e-04 | 1534.8ms/step | 13.0 samples/s +Step 27050/44000 | Loss: 2.9399 | LR: 1.67e-04 | 1587.7ms/step | 12.6 samples/s +Step 27060/44000 | Loss: 2.9410 | LR: 1.67e-04 | 1560.8ms/step | 12.8 samples/s +Step 27070/44000 | Loss: 2.9530 | LR: 1.66e-04 | 1575.9ms/step | 12.7 samples/s +Step 27080/44000 | Loss: 3.0366 | LR: 1.66e-04 | 1649.2ms/step | 12.1 samples/s +Step 27090/44000 | Loss: 2.9549 | LR: 1.66e-04 | 1658.9ms/step | 12.1 samples/s +Step 27100/44000 | Loss: 2.9580 | LR: 1.66e-04 | 1551.8ms/step | 12.9 samples/s +Step 27110/44000 | Loss: 2.9405 | LR: 1.66e-04 | 1605.4ms/step | 12.5 samples/s +Step 27120/44000 | Loss: 2.9278 | LR: 1.66e-04 | 1569.1ms/step | 12.7 samples/s +Step 27130/44000 | Loss: 2.9226 | LR: 1.65e-04 | 1565.8ms/step | 12.8 samples/s +Step 27140/44000 | Loss: 2.8875 | LR: 1.65e-04 | 1580.0ms/step | 12.7 samples/s +Step 27150/44000 | Loss: 2.9069 | LR: 1.65e-04 | 1863.5ms/step | 10.7 samples/s +Step 27160/44000 | Loss: 2.9656 | LR: 1.65e-04 | 1552.0ms/step | 12.9 samples/s +Step 27170/44000 | Loss: 2.9880 | LR: 1.65e-04 | 1619.9ms/step | 12.3 samples/s +Step 27180/44000 | Loss: 2.9258 | LR: 1.65e-04 | 1601.6ms/step | 12.5 samples/s +Step 27190/44000 | Loss: 2.9699 | LR: 1.64e-04 | 1593.0ms/step | 12.6 samples/s +Step 27200/44000 | Loss: 2.9314 | LR: 1.64e-04 | 1589.1ms/step | 12.6 samples/s +Step 27210/44000 | Loss: 2.9134 | LR: 1.64e-04 | 1611.2ms/step | 12.4 samples/s +Step 27220/44000 | Loss: 2.9099 | LR: 1.64e-04 | 1609.0ms/step | 12.4 samples/s +Step 27230/44000 | Loss: 2.8913 | LR: 1.64e-04 | 1608.9ms/step | 12.4 samples/s +Step 27240/44000 | Loss: 2.8946 | LR: 1.63e-04 | 1567.7ms/step | 12.8 samples/s +Step 27250/44000 | Loss: 2.9424 | LR: 1.63e-04 | 1567.5ms/step | 12.8 samples/s +Validation at step 27250: Loss = 2.8931, Perplexity = 18.05 +Step 27260/44000 | Loss: 2.9102 | LR: 1.63e-04 | 2222.8ms/step | 9.0 samples/s +Step 27270/44000 | Loss: 2.9567 | LR: 1.63e-04 | 1629.1ms/step | 12.3 samples/s +Step 27280/44000 | Loss: 2.9136 | LR: 1.63e-04 | 1641.1ms/step | 12.2 samples/s +Step 27290/44000 | Loss: 2.9562 | LR: 1.63e-04 | 1683.2ms/step | 11.9 samples/s +Step 27300/44000 | Loss: 2.9597 | LR: 1.62e-04 | 1514.4ms/step | 13.2 samples/s +Step 27310/44000 | Loss: 3.0280 | LR: 1.62e-04 | 1619.9ms/step | 12.3 samples/s +Step 27320/44000 | Loss: 3.0223 | LR: 1.62e-04 | 1543.2ms/step | 13.0 samples/s +Step 27330/44000 | Loss: 2.9117 | LR: 1.62e-04 | 1632.5ms/step | 12.3 samples/s +Step 27340/44000 | Loss: 2.9193 | LR: 1.62e-04 | 1544.8ms/step | 12.9 samples/s +Step 27350/44000 | Loss: 2.9858 | LR: 1.62e-04 | 1589.7ms/step | 12.6 samples/s +Step 27360/44000 | Loss: 2.9853 | LR: 1.61e-04 | 1609.6ms/step | 12.4 samples/s +Step 27370/44000 | Loss: 2.9953 | LR: 1.61e-04 | 1578.1ms/step | 12.7 samples/s +Step 27380/44000 | Loss: 2.9784 | LR: 1.61e-04 | 1559.6ms/step | 12.8 samples/s +Step 27390/44000 | Loss: 2.9489 | LR: 1.61e-04 | 1557.8ms/step | 12.8 samples/s +Step 27400/44000 | Loss: 2.8836 | LR: 1.61e-04 | 1397.6ms/step | 14.3 samples/s +Step 27410/44000 | Loss: 2.9648 | LR: 1.61e-04 | 1548.9ms/step | 12.9 samples/s +Step 27420/44000 | Loss: 2.9968 | LR: 1.60e-04 | 1581.5ms/step | 12.6 samples/s +Step 27430/44000 | Loss: 2.9557 | LR: 1.60e-04 | 1779.6ms/step | 11.2 samples/s +Step 27440/44000 | Loss: 2.9343 | LR: 1.60e-04 | 1572.6ms/step | 12.7 samples/s +Step 27450/44000 | Loss: 2.9329 | LR: 1.60e-04 | 1489.8ms/step | 13.4 samples/s +Step 27460/44000 | Loss: 2.9491 | LR: 1.60e-04 | 1435.7ms/step | 13.9 samples/s +Step 27470/44000 | Loss: 2.9054 | LR: 1.60e-04 | 1502.1ms/step | 13.3 samples/s +Step 27480/44000 | Loss: 2.9237 | LR: 1.59e-04 | 1556.6ms/step | 12.8 samples/s +Step 27490/44000 | Loss: 2.9196 | LR: 1.59e-04 | 1547.9ms/step | 12.9 samples/s +Step 27500/44000 | Loss: 2.9125 | LR: 1.59e-04 | 1600.5ms/step | 12.5 samples/s +Validation at step 27500: Loss = 2.8921, Perplexity = 18.03 +Step 27510/44000 | Loss: 2.8726 | LR: 1.59e-04 | 2158.3ms/step | 9.3 samples/s +Step 27520/44000 | Loss: 2.9514 | LR: 1.59e-04 | 1687.9ms/step | 11.8 samples/s +Step 27530/44000 | Loss: 2.9899 | LR: 1.59e-04 | 1562.1ms/step | 12.8 samples/s +Step 27540/44000 | Loss: 2.9030 | LR: 1.58e-04 | 1562.1ms/step | 12.8 samples/s +Step 27550/44000 | Loss: 2.9283 | LR: 1.58e-04 | 1546.4ms/step | 12.9 samples/s +Step 27560/44000 | Loss: 2.9630 | LR: 1.58e-04 | 1546.0ms/step | 12.9 samples/s +Step 27570/44000 | Loss: 3.0135 | LR: 1.58e-04 | 1804.0ms/step | 11.1 samples/s +Step 27580/44000 | Loss: 2.8631 | LR: 1.58e-04 | 1540.5ms/step | 13.0 samples/s +Step 27590/44000 | Loss: 2.9811 | LR: 1.58e-04 | 1504.5ms/step | 13.3 samples/s +Step 27600/44000 | Loss: 2.9900 | LR: 1.57e-04 | 1525.3ms/step | 13.1 samples/s +Step 27610/44000 | Loss: 2.9286 | LR: 1.57e-04 | 1535.7ms/step | 13.0 samples/s +Step 27620/44000 | Loss: 2.9840 | LR: 1.57e-04 | 1620.7ms/step | 12.3 samples/s +Step 27630/44000 | Loss: 2.9687 | LR: 1.57e-04 | 1557.7ms/step | 12.8 samples/s +Step 27640/44000 | Loss: 2.9538 | LR: 1.57e-04 | 1543.2ms/step | 13.0 samples/s +Step 27650/44000 | Loss: 3.0376 | LR: 1.57e-04 | 1569.3ms/step | 12.7 samples/s +Step 27660/44000 | Loss: 2.9592 | LR: 1.56e-04 | 1561.1ms/step | 12.8 samples/s +Step 27670/44000 | Loss: 2.8972 | LR: 1.56e-04 | 1557.5ms/step | 12.8 samples/s +Step 27680/44000 | Loss: 2.9265 | LR: 1.56e-04 | 1601.0ms/step | 12.5 samples/s +Step 27690/44000 | Loss: 2.9302 | LR: 1.56e-04 | 1567.2ms/step | 12.8 samples/s +Step 27700/44000 | Loss: 2.9875 | LR: 1.56e-04 | 1521.9ms/step | 13.1 samples/s +Step 27710/44000 | Loss: 2.9359 | LR: 1.56e-04 | 1804.0ms/step | 11.1 samples/s +Step 27720/44000 | Loss: 2.9168 | LR: 1.55e-04 | 1549.8ms/step | 12.9 samples/s +Step 27730/44000 | Loss: 2.9074 | LR: 1.55e-04 | 1563.3ms/step | 12.8 samples/s +Step 27740/44000 | Loss: 2.9428 | LR: 1.55e-04 | 1544.5ms/step | 12.9 samples/s +Step 27750/44000 | Loss: 2.9050 | LR: 1.55e-04 | 1537.6ms/step | 13.0 samples/s +Validation at step 27750: Loss = 2.8903, Perplexity = 18.00 +Step 27760/44000 | Loss: 2.9458 | LR: 1.55e-04 | 2177.4ms/step | 9.2 samples/s +Step 27770/44000 | Loss: 2.9079 | LR: 1.55e-04 | 1550.0ms/step | 12.9 samples/s +Step 27780/44000 | Loss: 2.9322 | LR: 1.54e-04 | 1496.3ms/step | 13.4 samples/s +Step 27790/44000 | Loss: 2.9086 | LR: 1.54e-04 | 1581.8ms/step | 12.6 samples/s +Step 27800/44000 | Loss: 2.9532 | LR: 1.54e-04 | 1561.5ms/step | 12.8 samples/s +Step 27810/44000 | Loss: 2.9405 | LR: 1.54e-04 | 1578.5ms/step | 12.7 samples/s +Step 27820/44000 | Loss: 2.9730 | LR: 1.54e-04 | 1544.3ms/step | 13.0 samples/s +Step 27830/44000 | Loss: 2.9249 | LR: 1.54e-04 | 1514.1ms/step | 13.2 samples/s +Step 27840/44000 | Loss: 2.9341 | LR: 1.53e-04 | 1557.9ms/step | 12.8 samples/s +Step 27850/44000 | Loss: 2.9326 | LR: 1.53e-04 | 1766.3ms/step | 11.3 samples/s +Step 27860/44000 | Loss: 2.9210 | LR: 1.53e-04 | 1457.7ms/step | 13.7 samples/s +Step 27870/44000 | Loss: 2.9781 | LR: 1.53e-04 | 1578.0ms/step | 12.7 samples/s +Step 27880/44000 | Loss: 2.9536 | LR: 1.53e-04 | 1600.6ms/step | 12.5 samples/s +Step 27890/44000 | Loss: 2.9355 | LR: 1.53e-04 | 1542.4ms/step | 13.0 samples/s +Step 27900/44000 | Loss: 2.9141 | LR: 1.52e-04 | 1539.1ms/step | 13.0 samples/s +Step 27910/44000 | Loss: 2.9769 | LR: 1.52e-04 | 1515.8ms/step | 13.2 samples/s +Step 27920/44000 | Loss: 2.9800 | LR: 1.52e-04 | 1488.8ms/step | 13.4 samples/s +Step 27930/44000 | Loss: 2.8681 | LR: 1.52e-04 | 1498.7ms/step | 13.3 samples/s +Step 27940/44000 | Loss: 2.9462 | LR: 1.52e-04 | 1511.4ms/step | 13.2 samples/s +Step 27950/44000 | Loss: 2.8935 | LR: 1.52e-04 | 1583.7ms/step | 12.6 samples/s +Step 27960/44000 | Loss: 2.9486 | LR: 1.51e-04 | 1580.2ms/step | 12.7 samples/s +Step 27970/44000 | Loss: 2.8462 | LR: 1.51e-04 | 1551.6ms/step | 12.9 samples/s +Step 27980/44000 | Loss: 2.9247 | LR: 1.51e-04 | 1556.4ms/step | 12.9 samples/s +Step 27990/44000 | Loss: 2.9573 | LR: 1.51e-04 | 1598.4ms/step | 12.5 samples/s +Step 28000/44000 | Loss: 2.8878 | LR: 1.51e-04 | 1759.0ms/step | 11.4 samples/s +Validation at step 28000: Loss = 2.8892, Perplexity = 17.98 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-28000 +Step 28010/44000 | Loss: 2.9586 | LR: 1.51e-04 | 2744.5ms/step | 7.3 samples/s +Step 28020/44000 | Loss: 2.9810 | LR: 1.50e-04 | 1518.4ms/step | 13.2 samples/s +Step 28030/44000 | Loss: 2.9341 | LR: 1.50e-04 | 1551.4ms/step | 12.9 samples/s +Step 28040/44000 | Loss: 2.9230 | LR: 1.50e-04 | 1556.9ms/step | 12.8 samples/s +Step 28050/44000 | Loss: 2.9687 | LR: 1.50e-04 | 1573.3ms/step | 12.7 samples/s +Step 28060/44000 | Loss: 2.9039 | LR: 1.50e-04 | 1542.5ms/step | 13.0 samples/s +Step 28070/44000 | Loss: 2.9616 | LR: 1.50e-04 | 1544.8ms/step | 12.9 samples/s +Step 28080/44000 | Loss: 2.9724 | LR: 1.49e-04 | 1576.1ms/step | 12.7 samples/s +Step 28090/44000 | Loss: 2.9407 | LR: 1.49e-04 | 1546.1ms/step | 12.9 samples/s +Step 28100/44000 | Loss: 2.9209 | LR: 1.49e-04 | 1541.2ms/step | 13.0 samples/s +Step 28110/44000 | Loss: 2.9189 | LR: 1.49e-04 | 1519.7ms/step | 13.2 samples/s +Step 28120/44000 | Loss: 2.9001 | LR: 1.49e-04 | 1483.5ms/step | 13.5 samples/s +Step 28130/44000 | Loss: 2.8791 | LR: 1.49e-04 | 1697.7ms/step | 11.8 samples/s +Step 28140/44000 | Loss: 3.0166 | LR: 1.48e-04 | 1594.3ms/step | 12.5 samples/s +Step 28150/44000 | Loss: 2.9357 | LR: 1.48e-04 | 1537.6ms/step | 13.0 samples/s +Step 28160/44000 | Loss: 2.9306 | LR: 1.48e-04 | 1542.6ms/step | 13.0 samples/s +Step 28170/44000 | Loss: 2.9145 | LR: 1.48e-04 | 1607.3ms/step | 12.4 samples/s +Step 28180/44000 | Loss: 2.9168 | LR: 1.48e-04 | 1524.6ms/step | 13.1 samples/s +Step 28190/44000 | Loss: 2.9287 | LR: 1.48e-04 | 1535.4ms/step | 13.0 samples/s +Step 28200/44000 | Loss: 2.8871 | LR: 1.47e-04 | 1540.8ms/step | 13.0 samples/s +Step 28210/44000 | Loss: 2.9284 | LR: 1.47e-04 | 1569.8ms/step | 12.7 samples/s +Step 28220/44000 | Loss: 2.9628 | LR: 1.47e-04 | 1531.4ms/step | 13.1 samples/s +Step 28230/44000 | Loss: 2.8869 | LR: 1.47e-04 | 1560.1ms/step | 12.8 samples/s +Step 28240/44000 | Loss: 2.9154 | LR: 1.47e-04 | 1545.8ms/step | 12.9 samples/s +Step 28250/44000 | Loss: 2.8545 | LR: 1.47e-04 | 1561.8ms/step | 12.8 samples/s +Validation at step 28250: Loss = 2.8883, Perplexity = 17.96 +Step 28260/44000 | Loss: 2.9157 | LR: 1.46e-04 | 2069.8ms/step | 9.7 samples/s +Step 28270/44000 | Loss: 2.9466 | LR: 1.46e-04 | 1771.3ms/step | 11.3 samples/s +Step 28280/44000 | Loss: 2.8753 | LR: 1.46e-04 | 1561.3ms/step | 12.8 samples/s +Step 28290/44000 | Loss: 2.9458 | LR: 1.46e-04 | 1559.2ms/step | 12.8 samples/s +Step 28300/44000 | Loss: 2.9470 | LR: 1.46e-04 | 1590.3ms/step | 12.6 samples/s +Step 28310/44000 | Loss: 2.8904 | LR: 1.46e-04 | 1552.1ms/step | 12.9 samples/s +Step 28320/44000 | Loss: 2.8486 | LR: 1.45e-04 | 1470.9ms/step | 13.6 samples/s +Step 28330/44000 | Loss: 2.9031 | LR: 1.45e-04 | 1521.6ms/step | 13.1 samples/s +Step 28340/44000 | Loss: 2.8909 | LR: 1.45e-04 | 1573.4ms/step | 12.7 samples/s +Step 28350/44000 | Loss: 2.9277 | LR: 1.45e-04 | 1532.0ms/step | 13.1 samples/s +Step 28360/44000 | Loss: 2.9547 | LR: 1.45e-04 | 1621.9ms/step | 12.3 samples/s +Step 28370/44000 | Loss: 2.8956 | LR: 1.45e-04 | 1540.3ms/step | 13.0 samples/s +Step 28380/44000 | Loss: 2.9781 | LR: 1.44e-04 | 1495.1ms/step | 13.4 samples/s +Step 28390/44000 | Loss: 2.9554 | LR: 1.44e-04 | 1472.2ms/step | 13.6 samples/s +Step 28400/44000 | Loss: 2.8688 | LR: 1.44e-04 | 1518.9ms/step | 13.2 samples/s +Step 28410/44000 | Loss: 2.9045 | LR: 1.44e-04 | 1550.9ms/step | 12.9 samples/s +Step 28420/44000 | Loss: 2.9017 | LR: 1.44e-04 | 1781.2ms/step | 11.2 samples/s +Step 28430/44000 | Loss: 2.9374 | LR: 1.44e-04 | 1566.7ms/step | 12.8 samples/s +Step 28440/44000 | Loss: 2.9966 | LR: 1.43e-04 | 1503.8ms/step | 13.3 samples/s +Step 28450/44000 | Loss: 2.9391 | LR: 1.43e-04 | 1530.9ms/step | 13.1 samples/s +Step 28460/44000 | Loss: 2.9442 | LR: 1.43e-04 | 1588.0ms/step | 12.6 samples/s +Step 28470/44000 | Loss: 2.9788 | LR: 1.43e-04 | 1515.2ms/step | 13.2 samples/s +Step 28480/44000 | Loss: 2.9492 | LR: 1.43e-04 | 1522.6ms/step | 13.1 samples/s +Step 28490/44000 | Loss: 2.9691 | LR: 1.43e-04 | 1536.3ms/step | 13.0 samples/s +Step 28500/44000 | Loss: 2.9377 | LR: 1.42e-04 | 1578.9ms/step | 12.7 samples/s +Validation at step 28500: Loss = 2.8870, Perplexity = 17.94 +Step 28510/44000 | Loss: 2.8898 | LR: 1.42e-04 | 2125.6ms/step | 9.4 samples/s +Step 28520/44000 | Loss: 2.9220 | LR: 1.42e-04 | 1550.8ms/step | 12.9 samples/s +Step 28530/44000 | Loss: 2.9463 | LR: 1.42e-04 | 1535.2ms/step | 13.0 samples/s +Step 28540/44000 | Loss: 2.8788 | LR: 1.42e-04 | 1545.7ms/step | 12.9 samples/s +Step 28550/44000 | Loss: 2.8940 | LR: 1.42e-04 | 1567.9ms/step | 12.8 samples/s +Step 28560/44000 | Loss: 2.9465 | LR: 1.41e-04 | 1822.3ms/step | 11.0 samples/s +Step 28570/44000 | Loss: 2.8801 | LR: 1.41e-04 | 1546.1ms/step | 12.9 samples/s +Step 28580/44000 | Loss: 2.9238 | LR: 1.41e-04 | 1551.5ms/step | 12.9 samples/s +Step 28590/44000 | Loss: 2.8988 | LR: 1.41e-04 | 1542.8ms/step | 13.0 samples/s +Step 28600/44000 | Loss: 2.9403 | LR: 1.41e-04 | 1527.7ms/step | 13.1 samples/s +Step 28610/44000 | Loss: 2.9274 | LR: 1.41e-04 | 1540.4ms/step | 13.0 samples/s +Step 28620/44000 | Loss: 2.9287 | LR: 1.40e-04 | 1581.9ms/step | 12.6 samples/s +Step 28630/44000 | Loss: 2.8587 | LR: 1.40e-04 | 1556.9ms/step | 12.8 samples/s +Step 28640/44000 | Loss: 2.8974 | LR: 1.40e-04 | 1589.4ms/step | 12.6 samples/s +Step 28650/44000 | Loss: 2.9810 | LR: 1.40e-04 | 1627.6ms/step | 12.3 samples/s +Step 28660/44000 | Loss: 2.9590 | LR: 1.40e-04 | 1571.1ms/step | 12.7 samples/s +Step 28670/44000 | Loss: 2.9736 | LR: 1.40e-04 | 1540.9ms/step | 13.0 samples/s +Step 28680/44000 | Loss: 2.9501 | LR: 1.39e-04 | 1540.8ms/step | 13.0 samples/s +Step 28690/44000 | Loss: 2.8569 | LR: 1.39e-04 | 1544.0ms/step | 13.0 samples/s +Step 28700/44000 | Loss: 2.9445 | LR: 1.39e-04 | 1798.8ms/step | 11.1 samples/s +Step 28710/44000 | Loss: 2.9571 | LR: 1.39e-04 | 1525.1ms/step | 13.1 samples/s +Step 28720/44000 | Loss: 2.9305 | LR: 1.39e-04 | 1524.6ms/step | 13.1 samples/s +Step 28730/44000 | Loss: 2.8634 | LR: 1.39e-04 | 1547.2ms/step | 12.9 samples/s +Step 28740/44000 | Loss: 2.9053 | LR: 1.38e-04 | 1545.3ms/step | 12.9 samples/s +Step 28750/44000 | Loss: 2.9599 | LR: 1.38e-04 | 1569.1ms/step | 12.7 samples/s +Validation at step 28750: Loss = 2.8867, Perplexity = 17.93 +Step 28760/44000 | Loss: 2.8470 | LR: 1.38e-04 | 2131.5ms/step | 9.4 samples/s +Step 28770/44000 | Loss: 2.8990 | LR: 1.38e-04 | 1532.1ms/step | 13.1 samples/s +Step 28780/44000 | Loss: 2.9224 | LR: 1.38e-04 | 1550.7ms/step | 12.9 samples/s +Step 28790/44000 | Loss: 2.9645 | LR: 1.38e-04 | 1460.3ms/step | 13.7 samples/s +Step 28800/44000 | Loss: 2.8989 | LR: 1.38e-04 | 1614.8ms/step | 12.4 samples/s +Step 28810/44000 | Loss: 2.9185 | LR: 1.37e-04 | 1546.1ms/step | 12.9 samples/s +Step 28820/44000 | Loss: 2.9471 | LR: 1.37e-04 | 1563.2ms/step | 12.8 samples/s +Step 28830/44000 | Loss: 2.9623 | LR: 1.37e-04 | 1524.1ms/step | 13.1 samples/s +Step 28840/44000 | Loss: 2.9339 | LR: 1.37e-04 | 1754.0ms/step | 11.4 samples/s +Step 28850/44000 | Loss: 2.9484 | LR: 1.37e-04 | 1506.3ms/step | 13.3 samples/s +Step 28860/44000 | Loss: 2.9211 | LR: 1.37e-04 | 1500.0ms/step | 13.3 samples/s +Step 28870/44000 | Loss: 2.9998 | LR: 1.36e-04 | 1497.4ms/step | 13.4 samples/s +Step 28880/44000 | Loss: 2.8616 | LR: 1.36e-04 | 1500.3ms/step | 13.3 samples/s +Step 28890/44000 | Loss: 2.9175 | LR: 1.36e-04 | 1501.7ms/step | 13.3 samples/s +Step 28900/44000 | Loss: 2.8943 | LR: 1.36e-04 | 1533.1ms/step | 13.0 samples/s +Step 28910/44000 | Loss: 2.8891 | LR: 1.36e-04 | 1556.4ms/step | 12.8 samples/s +Step 28920/44000 | Loss: 2.8809 | LR: 1.36e-04 | 1502.3ms/step | 13.3 samples/s +Step 28930/44000 | Loss: 2.8885 | LR: 1.35e-04 | 1548.4ms/step | 12.9 samples/s +Step 28940/44000 | Loss: 2.9272 | LR: 1.35e-04 | 1516.6ms/step | 13.2 samples/s +Step 28950/44000 | Loss: 2.9204 | LR: 1.35e-04 | 1598.7ms/step | 12.5 samples/s +Step 28960/44000 | Loss: 2.9286 | LR: 1.35e-04 | 1538.6ms/step | 13.0 samples/s +Step 28970/44000 | Loss: 2.9510 | LR: 1.35e-04 | 1530.6ms/step | 13.1 samples/s +Step 28980/44000 | Loss: 2.9237 | LR: 1.35e-04 | 1573.8ms/step | 12.7 samples/s +Step 28990/44000 | Loss: 2.9464 | LR: 1.34e-04 | 1767.0ms/step | 11.3 samples/s +Step 29000/44000 | Loss: 2.9328 | LR: 1.34e-04 | 1529.6ms/step | 13.1 samples/s +Validation at step 29000: Loss = 2.8854, Perplexity = 17.91 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-29000 +Step 29010/44000 | Loss: 2.9164 | LR: 1.34e-04 | 2763.3ms/step | 7.2 samples/s +Step 29020/44000 | Loss: 2.8779 | LR: 1.34e-04 | 1538.9ms/step | 13.0 samples/s +Step 29030/44000 | Loss: 2.9299 | LR: 1.34e-04 | 1517.1ms/step | 13.2 samples/s +Step 29040/44000 | Loss: 2.8955 | LR: 1.34e-04 | 1527.8ms/step | 13.1 samples/s +Step 29050/44000 | Loss: 2.9707 | LR: 1.33e-04 | 1505.2ms/step | 13.3 samples/s +Step 29060/44000 | Loss: 2.8844 | LR: 1.33e-04 | 1565.7ms/step | 12.8 samples/s +Step 29070/44000 | Loss: 3.0062 | LR: 1.33e-04 | 1493.8ms/step | 13.4 samples/s +Step 29080/44000 | Loss: 2.8810 | LR: 1.33e-04 | 1541.6ms/step | 13.0 samples/s +Step 29090/44000 | Loss: 2.9609 | LR: 1.33e-04 | 1509.1ms/step | 13.3 samples/s +Step 29100/44000 | Loss: 2.9471 | LR: 1.33e-04 | 1549.8ms/step | 12.9 samples/s +Step 29110/44000 | Loss: 2.9358 | LR: 1.33e-04 | 1562.1ms/step | 12.8 samples/s +Step 29120/44000 | Loss: 2.9106 | LR: 1.32e-04 | 1697.8ms/step | 11.8 samples/s +Step 29130/44000 | Loss: 2.9448 | LR: 1.32e-04 | 1739.7ms/step | 11.5 samples/s +Step 29140/44000 | Loss: 2.9696 | LR: 1.32e-04 | 1569.0ms/step | 12.7 samples/s +Step 29150/44000 | Loss: 2.8652 | LR: 1.32e-04 | 1572.1ms/step | 12.7 samples/s +Step 29160/44000 | Loss: 2.9794 | LR: 1.32e-04 | 1546.3ms/step | 12.9 samples/s +Step 29170/44000 | Loss: 2.9187 | LR: 1.32e-04 | 1526.7ms/step | 13.1 samples/s +Step 29180/44000 | Loss: 2.8644 | LR: 1.31e-04 | 1486.4ms/step | 13.5 samples/s +Step 29190/44000 | Loss: 2.8838 | LR: 1.31e-04 | 1605.3ms/step | 12.5 samples/s +Step 29200/44000 | Loss: 2.9654 | LR: 1.31e-04 | 1561.6ms/step | 12.8 samples/s +Step 29210/44000 | Loss: 2.9684 | LR: 1.31e-04 | 1629.4ms/step | 12.3 samples/s +Step 29220/44000 | Loss: 2.9374 | LR: 1.31e-04 | 1595.3ms/step | 12.5 samples/s +Step 29230/44000 | Loss: 2.9573 | LR: 1.31e-04 | 1520.0ms/step | 13.2 samples/s +Step 29240/44000 | Loss: 2.9737 | LR: 1.30e-04 | 1552.3ms/step | 12.9 samples/s +Step 29250/44000 | Loss: 2.9614 | LR: 1.30e-04 | 1445.4ms/step | 13.8 samples/s +Validation at step 29250: Loss = 2.8841, Perplexity = 17.89 +Step 29260/44000 | Loss: 2.9767 | LR: 1.30e-04 | 2269.9ms/step | 8.8 samples/s +Step 29270/44000 | Loss: 3.0185 | LR: 1.30e-04 | 1697.6ms/step | 11.8 samples/s +Step 29280/44000 | Loss: 2.9009 | LR: 1.30e-04 | 1599.2ms/step | 12.5 samples/s +Step 29290/44000 | Loss: 2.9240 | LR: 1.30e-04 | 1527.4ms/step | 13.1 samples/s +Step 29300/44000 | Loss: 2.9644 | LR: 1.29e-04 | 1500.3ms/step | 13.3 samples/s +Step 29310/44000 | Loss: 2.8632 | LR: 1.29e-04 | 1531.3ms/step | 13.1 samples/s +Step 29320/44000 | Loss: 3.0065 | LR: 1.29e-04 | 1498.3ms/step | 13.3 samples/s +Step 29330/44000 | Loss: 2.9758 | LR: 1.29e-04 | 1530.3ms/step | 13.1 samples/s +Step 29340/44000 | Loss: 3.0123 | LR: 1.29e-04 | 1498.9ms/step | 13.3 samples/s +Step 29350/44000 | Loss: 2.8913 | LR: 1.29e-04 | 1570.7ms/step | 12.7 samples/s +Step 29360/44000 | Loss: 2.9979 | LR: 1.29e-04 | 1557.6ms/step | 12.8 samples/s +Step 29370/44000 | Loss: 2.9414 | LR: 1.28e-04 | 1568.8ms/step | 12.7 samples/s +Step 29380/44000 | Loss: 2.9301 | LR: 1.28e-04 | 1563.2ms/step | 12.8 samples/s +Step 29390/44000 | Loss: 2.9846 | LR: 1.28e-04 | 1557.5ms/step | 12.8 samples/s +Step 29400/44000 | Loss: 2.9258 | LR: 1.28e-04 | 1597.3ms/step | 12.5 samples/s +Step 29410/44000 | Loss: 2.9232 | LR: 1.28e-04 | 1767.4ms/step | 11.3 samples/s +Step 29420/44000 | Loss: 2.9162 | LR: 1.28e-04 | 1562.4ms/step | 12.8 samples/s +Step 29430/44000 | Loss: 2.9867 | LR: 1.27e-04 | 1525.7ms/step | 13.1 samples/s +Step 29440/44000 | Loss: 2.9745 | LR: 1.27e-04 | 1546.2ms/step | 12.9 samples/s +Step 29450/44000 | Loss: 2.9752 | LR: 1.27e-04 | 1574.0ms/step | 12.7 samples/s +Step 29460/44000 | Loss: 2.8862 | LR: 1.27e-04 | 1556.5ms/step | 12.8 samples/s +Step 29470/44000 | Loss: 2.9682 | LR: 1.27e-04 | 1532.7ms/step | 13.0 samples/s +Step 29480/44000 | Loss: 2.9297 | LR: 1.27e-04 | 1542.2ms/step | 13.0 samples/s +Step 29490/44000 | Loss: 2.9855 | LR: 1.26e-04 | 1591.3ms/step | 12.6 samples/s +Step 29500/44000 | Loss: 2.9411 | LR: 1.26e-04 | 1539.8ms/step | 13.0 samples/s +Validation at step 29500: Loss = 2.8831, Perplexity = 17.87 +Step 29510/44000 | Loss: 2.8996 | LR: 1.26e-04 | 2144.1ms/step | 9.3 samples/s +Step 29520/44000 | Loss: 2.9227 | LR: 1.26e-04 | 1568.8ms/step | 12.7 samples/s +Step 29530/44000 | Loss: 2.9665 | LR: 1.26e-04 | 1554.2ms/step | 12.9 samples/s +Step 29540/44000 | Loss: 2.8743 | LR: 1.26e-04 | 1532.8ms/step | 13.0 samples/s +Step 29550/44000 | Loss: 2.8495 | LR: 1.26e-04 | 1754.6ms/step | 11.4 samples/s +Step 29560/44000 | Loss: 2.9458 | LR: 1.25e-04 | 1536.7ms/step | 13.0 samples/s +Step 29570/44000 | Loss: 2.9413 | LR: 1.25e-04 | 1573.2ms/step | 12.7 samples/s +Step 29580/44000 | Loss: 2.9044 | LR: 1.25e-04 | 1547.3ms/step | 12.9 samples/s +Step 29590/44000 | Loss: 2.9399 | LR: 1.25e-04 | 1541.3ms/step | 13.0 samples/s +Step 29600/44000 | Loss: 2.9262 | LR: 1.25e-04 | 1552.8ms/step | 12.9 samples/s +Step 29610/44000 | Loss: 2.9848 | LR: 1.25e-04 | 1580.3ms/step | 12.7 samples/s +Step 29620/44000 | Loss: 2.8963 | LR: 1.24e-04 | 1525.4ms/step | 13.1 samples/s +Step 29630/44000 | Loss: 2.9209 | LR: 1.24e-04 | 1520.1ms/step | 13.2 samples/s +Step 29640/44000 | Loss: 2.9220 | LR: 1.24e-04 | 1530.2ms/step | 13.1 samples/s +Step 29650/44000 | Loss: 2.8550 | LR: 1.24e-04 | 1509.5ms/step | 13.2 samples/s +Step 29660/44000 | Loss: 2.9586 | LR: 1.24e-04 | 1520.5ms/step | 13.2 samples/s +Step 29670/44000 | Loss: 2.9104 | LR: 1.24e-04 | 1561.1ms/step | 12.8 samples/s +Step 29680/44000 | Loss: 2.9458 | LR: 1.23e-04 | 1546.0ms/step | 12.9 samples/s +Step 29690/44000 | Loss: 2.8895 | LR: 1.23e-04 | 1682.2ms/step | 11.9 samples/s +Step 29700/44000 | Loss: 2.9453 | LR: 1.23e-04 | 1522.9ms/step | 13.1 samples/s +Step 29710/44000 | Loss: 2.9721 | LR: 1.23e-04 | 1449.5ms/step | 13.8 samples/s +Step 29720/44000 | Loss: 2.9368 | LR: 1.23e-04 | 1544.6ms/step | 12.9 samples/s +Step 29730/44000 | Loss: 2.9148 | LR: 1.23e-04 | 1551.0ms/step | 12.9 samples/s +Step 29740/44000 | Loss: 2.9540 | LR: 1.23e-04 | 1575.9ms/step | 12.7 samples/s +Step 29750/44000 | Loss: 2.9604 | LR: 1.22e-04 | 1521.6ms/step | 13.1 samples/s +Validation at step 29750: Loss = 2.8824, Perplexity = 17.86 +Step 29760/44000 | Loss: 2.9194 | LR: 1.22e-04 | 2206.1ms/step | 9.1 samples/s +Step 29770/44000 | Loss: 2.9191 | LR: 1.22e-04 | 1604.4ms/step | 12.5 samples/s +Step 29780/44000 | Loss: 2.9554 | LR: 1.22e-04 | 1483.2ms/step | 13.5 samples/s +Step 29790/44000 | Loss: 2.9285 | LR: 1.22e-04 | 1546.5ms/step | 12.9 samples/s +Step 29800/44000 | Loss: 2.9331 | LR: 1.22e-04 | 1525.0ms/step | 13.1 samples/s +Step 29810/44000 | Loss: 2.8577 | LR: 1.21e-04 | 1514.3ms/step | 13.2 samples/s +Step 29820/44000 | Loss: 2.9343 | LR: 1.21e-04 | 1552.7ms/step | 12.9 samples/s +Step 29830/44000 | Loss: 2.9019 | LR: 1.21e-04 | 1737.2ms/step | 11.5 samples/s +Step 29840/44000 | Loss: 2.8702 | LR: 1.21e-04 | 1638.7ms/step | 12.2 samples/s +Step 29850/44000 | Loss: 2.8994 | LR: 1.21e-04 | 1569.9ms/step | 12.7 samples/s +Step 29860/44000 | Loss: 2.9593 | LR: 1.21e-04 | 1591.9ms/step | 12.6 samples/s +Step 29870/44000 | Loss: 2.9568 | LR: 1.21e-04 | 1500.8ms/step | 13.3 samples/s +Step 29880/44000 | Loss: 2.8922 | LR: 1.20e-04 | 1529.9ms/step | 13.1 samples/s +Step 29890/44000 | Loss: 2.9126 | LR: 1.20e-04 | 1558.5ms/step | 12.8 samples/s +Step 29900/44000 | Loss: 2.8988 | LR: 1.20e-04 | 1561.8ms/step | 12.8 samples/s +Step 29910/44000 | Loss: 2.9096 | LR: 1.20e-04 | 1555.6ms/step | 12.9 samples/s +Step 29920/44000 | Loss: 2.9692 | LR: 1.20e-04 | 1519.8ms/step | 13.2 samples/s +Step 29930/44000 | Loss: 2.9057 | LR: 1.20e-04 | 1538.7ms/step | 13.0 samples/s +Step 29940/44000 | Loss: 2.9383 | LR: 1.19e-04 | 1511.4ms/step | 13.2 samples/s +Step 29950/44000 | Loss: 2.9619 | LR: 1.19e-04 | 1501.2ms/step | 13.3 samples/s +Step 29960/44000 | Loss: 2.9827 | LR: 1.19e-04 | 1505.5ms/step | 13.3 samples/s +Step 29970/44000 | Loss: 2.9638 | LR: 1.19e-04 | 1673.4ms/step | 12.0 samples/s +Step 29980/44000 | Loss: 2.8952 | LR: 1.19e-04 | 1714.2ms/step | 11.7 samples/s +Step 29990/44000 | Loss: 2.8798 | LR: 1.19e-04 | 1635.2ms/step | 12.2 samples/s +Step 30000/44000 | Loss: 2.9433 | LR: 1.19e-04 | 1587.9ms/step | 12.6 samples/s +Validation at step 30000: Loss = 2.8816, Perplexity = 17.84 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-30000 +Step 30010/44000 | Loss: 2.9476 | LR: 1.18e-04 | 2793.5ms/step | 7.2 samples/s +Step 30020/44000 | Loss: 2.9301 | LR: 1.18e-04 | 1593.2ms/step | 12.6 samples/s +Step 30030/44000 | Loss: 2.9882 | LR: 1.18e-04 | 1565.4ms/step | 12.8 samples/s +Step 30040/44000 | Loss: 3.0805 | LR: 1.18e-04 | 1561.8ms/step | 12.8 samples/s +Step 30050/44000 | Loss: 2.9010 | LR: 1.18e-04 | 1558.8ms/step | 12.8 samples/s +Step 30060/44000 | Loss: 2.9952 | LR: 1.18e-04 | 1593.1ms/step | 12.6 samples/s +Step 30070/44000 | Loss: 3.0055 | LR: 1.17e-04 | 1532.7ms/step | 13.0 samples/s +Step 30080/44000 | Loss: 2.8998 | LR: 1.17e-04 | 1513.4ms/step | 13.2 samples/s +Step 30090/44000 | Loss: 2.9531 | LR: 1.17e-04 | 1546.8ms/step | 12.9 samples/s +Step 30100/44000 | Loss: 2.9089 | LR: 1.17e-04 | 1504.1ms/step | 13.3 samples/s +Step 30110/44000 | Loss: 2.9782 | LR: 1.17e-04 | 1793.0ms/step | 11.2 samples/s +Step 30120/44000 | Loss: 2.8876 | LR: 1.17e-04 | 1562.1ms/step | 12.8 samples/s +Step 30130/44000 | Loss: 2.9061 | LR: 1.17e-04 | 1558.1ms/step | 12.8 samples/s +Step 30140/44000 | Loss: 2.9041 | LR: 1.16e-04 | 1564.4ms/step | 12.8 samples/s +Step 30150/44000 | Loss: 2.8704 | LR: 1.16e-04 | 1526.2ms/step | 13.1 samples/s +Step 30160/44000 | Loss: 2.9010 | LR: 1.16e-04 | 1482.7ms/step | 13.5 samples/s +Step 30170/44000 | Loss: 2.9208 | LR: 1.16e-04 | 1503.8ms/step | 13.3 samples/s +Step 30180/44000 | Loss: 2.9455 | LR: 1.16e-04 | 1584.9ms/step | 12.6 samples/s +Step 30190/44000 | Loss: 2.8773 | LR: 1.16e-04 | 1542.4ms/step | 13.0 samples/s +Step 30200/44000 | Loss: 2.9369 | LR: 1.15e-04 | 1535.7ms/step | 13.0 samples/s +Step 30210/44000 | Loss: 2.8922 | LR: 1.15e-04 | 1543.8ms/step | 13.0 samples/s +Step 30220/44000 | Loss: 2.9311 | LR: 1.15e-04 | 1519.7ms/step | 13.2 samples/s +Step 30230/44000 | Loss: 2.9091 | LR: 1.15e-04 | 1564.9ms/step | 12.8 samples/s +Step 30240/44000 | Loss: 2.8864 | LR: 1.15e-04 | 1489.8ms/step | 13.4 samples/s +Step 30250/44000 | Loss: 2.9675 | LR: 1.15e-04 | 1558.0ms/step | 12.8 samples/s +Validation at step 30250: Loss = 2.8799, Perplexity = 17.81 +Step 30260/44000 | Loss: 2.8780 | LR: 1.15e-04 | 2281.0ms/step | 8.8 samples/s +Step 30270/44000 | Loss: 2.9872 | LR: 1.14e-04 | 1569.8ms/step | 12.7 samples/s +Step 30280/44000 | Loss: 2.9035 | LR: 1.14e-04 | 1549.6ms/step | 12.9 samples/s +Step 30290/44000 | Loss: 2.9745 | LR: 1.14e-04 | 1555.2ms/step | 12.9 samples/s +Step 30300/44000 | Loss: 2.9222 | LR: 1.14e-04 | 1560.3ms/step | 12.8 samples/s +Step 30310/44000 | Loss: 2.8625 | LR: 1.14e-04 | 1550.7ms/step | 12.9 samples/s +Step 30320/44000 | Loss: 2.8820 | LR: 1.14e-04 | 1605.2ms/step | 12.5 samples/s +Step 30330/44000 | Loss: 2.9463 | LR: 1.13e-04 | 1542.2ms/step | 13.0 samples/s +Step 30340/44000 | Loss: 2.9139 | LR: 1.13e-04 | 1587.8ms/step | 12.6 samples/s +Step 30350/44000 | Loss: 2.9245 | LR: 1.13e-04 | 1611.3ms/step | 12.4 samples/s +Step 30360/44000 | Loss: 2.9271 | LR: 1.13e-04 | 1552.5ms/step | 12.9 samples/s +Step 30370/44000 | Loss: 2.9099 | LR: 1.13e-04 | 1579.6ms/step | 12.7 samples/s +Step 30380/44000 | Loss: 2.9122 | LR: 1.13e-04 | 1544.4ms/step | 13.0 samples/s +Step 30390/44000 | Loss: 2.9375 | LR: 1.13e-04 | 1653.0ms/step | 12.1 samples/s +Step 30400/44000 | Loss: 3.0164 | LR: 1.12e-04 | 1728.6ms/step | 11.6 samples/s +Step 30410/44000 | Loss: 2.9161 | LR: 1.12e-04 | 1519.4ms/step | 13.2 samples/s +Step 30420/44000 | Loss: 2.9022 | LR: 1.12e-04 | 1589.5ms/step | 12.6 samples/s +Step 30430/44000 | Loss: 2.8944 | LR: 1.12e-04 | 1558.3ms/step | 12.8 samples/s +Step 30440/44000 | Loss: 2.9396 | LR: 1.12e-04 | 1553.4ms/step | 12.9 samples/s +Step 30450/44000 | Loss: 2.9243 | LR: 1.12e-04 | 1621.9ms/step | 12.3 samples/s +Step 30460/44000 | Loss: 2.9101 | LR: 1.11e-04 | 1583.5ms/step | 12.6 samples/s +Step 30470/44000 | Loss: 2.9708 | LR: 1.11e-04 | 1583.1ms/step | 12.6 samples/s +Step 30480/44000 | Loss: 2.9156 | LR: 1.11e-04 | 1538.3ms/step | 13.0 samples/s +Step 30490/44000 | Loss: 2.8859 | LR: 1.11e-04 | 1565.1ms/step | 12.8 samples/s +Step 30500/44000 | Loss: 2.9567 | LR: 1.11e-04 | 1541.5ms/step | 13.0 samples/s +Validation at step 30500: Loss = 2.8793, Perplexity = 17.80 +Step 30510/44000 | Loss: 2.9092 | LR: 1.11e-04 | 2154.9ms/step | 9.3 samples/s +Step 30520/44000 | Loss: 2.8878 | LR: 1.11e-04 | 1572.7ms/step | 12.7 samples/s +Step 30530/44000 | Loss: 2.9453 | LR: 1.10e-04 | 1763.0ms/step | 11.3 samples/s +Step 30540/44000 | Loss: 2.9312 | LR: 1.10e-04 | 1597.4ms/step | 12.5 samples/s +Step 30550/44000 | Loss: 2.9389 | LR: 1.10e-04 | 1616.3ms/step | 12.4 samples/s +Step 30560/44000 | Loss: 2.9455 | LR: 1.10e-04 | 1534.1ms/step | 13.0 samples/s +Step 30570/44000 | Loss: 2.9866 | LR: 1.10e-04 | 1610.3ms/step | 12.4 samples/s +Step 30580/44000 | Loss: 2.8088 | LR: 1.10e-04 | 1599.2ms/step | 12.5 samples/s +Step 30590/44000 | Loss: 3.0551 | LR: 1.10e-04 | 1583.8ms/step | 12.6 samples/s +Step 30600/44000 | Loss: 2.9750 | LR: 1.09e-04 | 1532.8ms/step | 13.0 samples/s +Step 30610/44000 | Loss: 2.8980 | LR: 1.09e-04 | 1554.9ms/step | 12.9 samples/s +Step 30620/44000 | Loss: 2.9320 | LR: 1.09e-04 | 1448.3ms/step | 13.8 samples/s +Step 30630/44000 | Loss: 2.8384 | LR: 1.09e-04 | 1388.9ms/step | 14.4 samples/s +Step 30640/44000 | Loss: 2.9435 | LR: 1.09e-04 | 1420.9ms/step | 14.1 samples/s +Step 30650/44000 | Loss: 2.9808 | LR: 1.09e-04 | 1430.5ms/step | 14.0 samples/s +Step 30660/44000 | Loss: 2.8504 | LR: 1.08e-04 | 1445.1ms/step | 13.8 samples/s +Step 30670/44000 | Loss: 2.8909 | LR: 1.08e-04 | 1390.7ms/step | 14.4 samples/s +Step 30680/44000 | Loss: 2.9385 | LR: 1.08e-04 | 1564.6ms/step | 12.8 samples/s +Step 30690/44000 | Loss: 2.9118 | LR: 1.08e-04 | 1464.3ms/step | 13.7 samples/s +Step 30700/44000 | Loss: 2.9705 | LR: 1.08e-04 | 1390.0ms/step | 14.4 samples/s +Step 30710/44000 | Loss: 2.9374 | LR: 1.08e-04 | 1336.5ms/step | 15.0 samples/s +Step 30720/44000 | Loss: 2.9294 | LR: 1.08e-04 | 1305.4ms/step | 15.3 samples/s +Step 30730/44000 | Loss: 2.9239 | LR: 1.07e-04 | 1326.9ms/step | 15.1 samples/s +Step 30740/44000 | Loss: 2.9377 | LR: 1.07e-04 | 1330.1ms/step | 15.0 samples/s +Step 30750/44000 | Loss: 2.9492 | LR: 1.07e-04 | 1320.8ms/step | 15.1 samples/s +Validation at step 30750: Loss = 2.8785, Perplexity = 17.79 +Step 30760/44000 | Loss: 2.9043 | LR: 1.07e-04 | 1876.4ms/step | 10.7 samples/s +Step 30770/44000 | Loss: 2.9249 | LR: 1.07e-04 | 1330.7ms/step | 15.0 samples/s +Step 30780/44000 | Loss: 2.8177 | LR: 1.07e-04 | 1336.4ms/step | 15.0 samples/s +Step 30790/44000 | Loss: 2.8833 | LR: 1.07e-04 | 1341.8ms/step | 14.9 samples/s +Step 30800/44000 | Loss: 2.9257 | LR: 1.06e-04 | 1375.4ms/step | 14.5 samples/s +Step 30810/44000 | Loss: 2.9624 | LR: 1.06e-04 | 1339.0ms/step | 14.9 samples/s +Step 30820/44000 | Loss: 2.9770 | LR: 1.06e-04 | 1317.6ms/step | 15.2 samples/s +Step 30830/44000 | Loss: 2.8607 | LR: 1.06e-04 | 1296.9ms/step | 15.4 samples/s +Step 30840/44000 | Loss: 2.9585 | LR: 1.06e-04 | 1396.0ms/step | 14.3 samples/s +Step 30850/44000 | Loss: 2.9045 | LR: 1.06e-04 | 1318.9ms/step | 15.2 samples/s +Step 30860/44000 | Loss: 2.9662 | LR: 1.05e-04 | 1339.2ms/step | 14.9 samples/s +Step 30870/44000 | Loss: 2.8851 | LR: 1.05e-04 | 1305.6ms/step | 15.3 samples/s +Step 30880/44000 | Loss: 2.9158 | LR: 1.05e-04 | 1330.9ms/step | 15.0 samples/s +Step 30890/44000 | Loss: 2.9696 | LR: 1.05e-04 | 1320.9ms/step | 15.1 samples/s +Step 30900/44000 | Loss: 2.8967 | LR: 1.05e-04 | 1362.6ms/step | 14.7 samples/s +Step 30910/44000 | Loss: 2.9276 | LR: 1.05e-04 | 1378.2ms/step | 14.5 samples/s +Step 30920/44000 | Loss: 2.9139 | LR: 1.05e-04 | 1347.0ms/step | 14.8 samples/s +Step 30930/44000 | Loss: 2.8959 | LR: 1.04e-04 | 1326.3ms/step | 15.1 samples/s +Step 30940/44000 | Loss: 2.9096 | LR: 1.04e-04 | 1310.5ms/step | 15.3 samples/s +Step 30950/44000 | Loss: 2.9070 | LR: 1.04e-04 | 1325.9ms/step | 15.1 samples/s +Step 30960/44000 | Loss: 3.0248 | LR: 1.04e-04 | 1349.7ms/step | 14.8 samples/s +Step 30970/44000 | Loss: 2.9294 | LR: 1.04e-04 | 1333.5ms/step | 15.0 samples/s +Step 30980/44000 | Loss: 2.9134 | LR: 1.04e-04 | 1319.1ms/step | 15.2 samples/s +Step 30990/44000 | Loss: 2.9271 | LR: 1.04e-04 | 1275.4ms/step | 15.7 samples/s +Step 31000/44000 | Loss: 2.8873 | LR: 1.03e-04 | 1328.6ms/step | 15.1 samples/s +Validation at step 31000: Loss = 2.8782, Perplexity = 17.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-31000 +Step 31010/44000 | Loss: 2.8889 | LR: 1.03e-04 | 2381.9ms/step | 8.4 samples/s +Step 31020/44000 | Loss: 2.8893 | LR: 1.03e-04 | 1357.3ms/step | 14.7 samples/s +Step 31030/44000 | Loss: 2.9765 | LR: 1.03e-04 | 1394.1ms/step | 14.3 samples/s +Step 31040/44000 | Loss: 2.9704 | LR: 1.03e-04 | 1321.3ms/step | 15.1 samples/s +Step 31050/44000 | Loss: 2.9033 | LR: 1.03e-04 | 1392.2ms/step | 14.4 samples/s +Step 31060/44000 | Loss: 2.8858 | LR: 1.03e-04 | 1318.8ms/step | 15.2 samples/s +Step 31070/44000 | Loss: 2.9736 | LR: 1.02e-04 | 1330.4ms/step | 15.0 samples/s +Step 31080/44000 | Loss: 2.8630 | LR: 1.02e-04 | 1325.6ms/step | 15.1 samples/s +Step 31090/44000 | Loss: 2.9547 | LR: 1.02e-04 | 1261.3ms/step | 15.9 samples/s +Step 31100/44000 | Loss: 2.9712 | LR: 1.02e-04 | 1271.0ms/step | 15.7 samples/s +Step 31110/44000 | Loss: 2.9343 | LR: 1.02e-04 | 1320.8ms/step | 15.1 samples/s +Step 31120/44000 | Loss: 2.9672 | LR: 1.02e-04 | 1311.6ms/step | 15.2 samples/s +Step 31130/44000 | Loss: 2.9160 | LR: 1.02e-04 | 1293.4ms/step | 15.5 samples/s +Step 31140/44000 | Loss: 2.9028 | LR: 1.01e-04 | 1252.6ms/step | 16.0 samples/s +Step 31150/44000 | Loss: 2.8480 | LR: 1.01e-04 | 1284.3ms/step | 15.6 samples/s +Step 31160/44000 | Loss: 2.9242 | LR: 1.01e-04 | 1293.9ms/step | 15.5 samples/s +Step 31170/44000 | Loss: 2.9521 | LR: 1.01e-04 | 1248.3ms/step | 16.0 samples/s +Step 31180/44000 | Loss: 2.9578 | LR: 1.01e-04 | 1278.8ms/step | 15.6 samples/s +Step 31190/44000 | Loss: 2.8928 | LR: 1.01e-04 | 1243.8ms/step | 16.1 samples/s +Step 31200/44000 | Loss: 2.8811 | LR: 1.00e-04 | 1256.5ms/step | 15.9 samples/s +Step 31210/44000 | Loss: 2.9101 | LR: 1.00e-04 | 1342.0ms/step | 14.9 samples/s +Step 31220/44000 | Loss: 2.8797 | LR: 1.00e-04 | 1326.2ms/step | 15.1 samples/s +Step 31230/44000 | Loss: 2.9035 | LR: 1.00e-04 | 1334.1ms/step | 15.0 samples/s +Step 31240/44000 | Loss: 3.0311 | LR: 9.99e-05 | 1312.6ms/step | 15.2 samples/s +Step 31250/44000 | Loss: 2.9440 | LR: 9.98e-05 | 1304.9ms/step | 15.3 samples/s +Validation at step 31250: Loss = 2.8774, Perplexity = 17.77 +Step 31260/44000 | Loss: 2.8669 | LR: 9.96e-05 | 1798.2ms/step | 11.1 samples/s +Step 31270/44000 | Loss: 2.9460 | LR: 9.95e-05 | 1289.7ms/step | 15.5 samples/s +Step 31280/44000 | Loss: 2.9266 | LR: 9.93e-05 | 1334.7ms/step | 15.0 samples/s +Step 31290/44000 | Loss: 2.9147 | LR: 9.92e-05 | 1313.0ms/step | 15.2 samples/s +Step 31300/44000 | Loss: 2.9018 | LR: 9.90e-05 | 1294.8ms/step | 15.4 samples/s +Step 31310/44000 | Loss: 2.9321 | LR: 9.89e-05 | 1263.7ms/step | 15.8 samples/s +Step 31320/44000 | Loss: 2.9249 | LR: 9.88e-05 | 1300.0ms/step | 15.4 samples/s +Step 31330/44000 | Loss: 2.9133 | LR: 9.86e-05 | 1284.6ms/step | 15.6 samples/s +Step 31340/44000 | Loss: 2.9131 | LR: 9.85e-05 | 1272.4ms/step | 15.7 samples/s +Step 31350/44000 | Loss: 2.9038 | LR: 9.83e-05 | 1267.9ms/step | 15.8 samples/s +Step 31360/44000 | Loss: 2.9684 | LR: 9.82e-05 | 1275.5ms/step | 15.7 samples/s +Step 31370/44000 | Loss: 2.9374 | LR: 9.80e-05 | 1273.3ms/step | 15.7 samples/s +Step 31380/44000 | Loss: 2.9291 | LR: 9.79e-05 | 1284.8ms/step | 15.6 samples/s +Step 31390/44000 | Loss: 2.9479 | LR: 9.77e-05 | 1256.4ms/step | 15.9 samples/s +Step 31400/44000 | Loss: 2.9157 | LR: 9.76e-05 | 1257.8ms/step | 15.9 samples/s +Step 31410/44000 | Loss: 2.8067 | LR: 9.75e-05 | 1294.4ms/step | 15.5 samples/s +Step 31420/44000 | Loss: 2.9828 | LR: 9.73e-05 | 1258.2ms/step | 15.9 samples/s +Step 31430/44000 | Loss: 2.8963 | LR: 9.72e-05 | 1263.0ms/step | 15.8 samples/s +Step 31440/44000 | Loss: 2.8986 | LR: 9.70e-05 | 1273.8ms/step | 15.7 samples/s +Step 31450/44000 | Loss: 2.8935 | LR: 9.69e-05 | 1313.8ms/step | 15.2 samples/s +Step 31460/44000 | Loss: 2.9303 | LR: 9.67e-05 | 1285.8ms/step | 15.6 samples/s +Step 31470/44000 | Loss: 2.9674 | LR: 9.66e-05 | 1273.2ms/step | 15.7 samples/s +Step 31480/44000 | Loss: 2.9330 | LR: 9.65e-05 | 1294.9ms/step | 15.4 samples/s +Step 31490/44000 | Loss: 2.9210 | LR: 9.63e-05 | 1262.9ms/step | 15.8 samples/s +Step 31500/44000 | Loss: 2.8571 | LR: 9.62e-05 | 1286.5ms/step | 15.5 samples/s +Validation at step 31500: Loss = 2.8772, Perplexity = 17.76 +Step 31510/44000 | Loss: 2.9375 | LR: 9.60e-05 | 1738.2ms/step | 11.5 samples/s +Step 31520/44000 | Loss: 2.9049 | LR: 9.59e-05 | 1350.5ms/step | 14.8 samples/s +Step 31530/44000 | Loss: 2.9371 | LR: 9.57e-05 | 1285.7ms/step | 15.6 samples/s +Step 31540/44000 | Loss: 2.9741 | LR: 9.56e-05 | 1252.4ms/step | 16.0 samples/s +Step 31550/44000 | Loss: 2.9086 | LR: 9.54e-05 | 1258.2ms/step | 15.9 samples/s +Step 31560/44000 | Loss: 2.9130 | LR: 9.53e-05 | 1294.0ms/step | 15.5 samples/s +Step 31570/44000 | Loss: 2.8737 | LR: 9.52e-05 | 1263.8ms/step | 15.8 samples/s +Step 31580/44000 | Loss: 3.0169 | LR: 9.50e-05 | 1339.2ms/step | 14.9 samples/s +Step 31590/44000 | Loss: 3.0011 | LR: 9.49e-05 | 1292.3ms/step | 15.5 samples/s +Step 31600/44000 | Loss: 2.8520 | LR: 9.47e-05 | 1321.2ms/step | 15.1 samples/s +Step 31610/44000 | Loss: 2.9417 | LR: 9.46e-05 | 1268.9ms/step | 15.8 samples/s +Step 31620/44000 | Loss: 2.9114 | LR: 9.45e-05 | 1338.8ms/step | 14.9 samples/s +Step 31630/44000 | Loss: 2.9755 | LR: 9.43e-05 | 1292.4ms/step | 15.5 samples/s +Step 31640/44000 | Loss: 3.0036 | LR: 9.42e-05 | 1315.5ms/step | 15.2 samples/s +Step 31650/44000 | Loss: 2.9520 | LR: 9.40e-05 | 1280.0ms/step | 15.6 samples/s +Step 31660/44000 | Loss: 2.9487 | LR: 9.39e-05 | 1288.4ms/step | 15.5 samples/s +Step 31670/44000 | Loss: 2.9568 | LR: 9.37e-05 | 1302.0ms/step | 15.4 samples/s +Step 31680/44000 | Loss: 2.8964 | LR: 9.36e-05 | 1260.7ms/step | 15.9 samples/s +Step 31690/44000 | Loss: 2.9374 | LR: 9.35e-05 | 1436.6ms/step | 13.9 samples/s +Step 31700/44000 | Loss: 2.9349 | LR: 9.33e-05 | 1289.1ms/step | 15.5 samples/s +Step 31710/44000 | Loss: 2.9503 | LR: 9.32e-05 | 1274.0ms/step | 15.7 samples/s +Step 31720/44000 | Loss: 2.9330 | LR: 9.30e-05 | 1345.6ms/step | 14.9 samples/s +Step 31730/44000 | Loss: 2.8951 | LR: 9.29e-05 | 1287.0ms/step | 15.5 samples/s +Step 31740/44000 | Loss: 2.9262 | LR: 9.28e-05 | 1289.0ms/step | 15.5 samples/s +Step 31750/44000 | Loss: 2.8743 | LR: 9.26e-05 | 1281.7ms/step | 15.6 samples/s +Validation at step 31750: Loss = 2.8765, Perplexity = 17.75 +Step 31760/44000 | Loss: 2.9527 | LR: 9.25e-05 | 1826.4ms/step | 11.0 samples/s +Step 31770/44000 | Loss: 2.9227 | LR: 9.23e-05 | 1342.0ms/step | 14.9 samples/s +Step 31780/44000 | Loss: 2.8657 | LR: 9.22e-05 | 1274.4ms/step | 15.7 samples/s +Step 31790/44000 | Loss: 2.8600 | LR: 9.20e-05 | 1287.7ms/step | 15.5 samples/s +Step 31800/44000 | Loss: 2.9926 | LR: 9.19e-05 | 1315.7ms/step | 15.2 samples/s +Step 31810/44000 | Loss: 2.9721 | LR: 9.18e-05 | 1311.1ms/step | 15.3 samples/s +Step 31820/44000 | Loss: 2.9358 | LR: 9.16e-05 | 1330.4ms/step | 15.0 samples/s +Step 31830/44000 | Loss: 2.9459 | LR: 9.15e-05 | 1294.7ms/step | 15.4 samples/s +Step 31840/44000 | Loss: 2.9160 | LR: 9.13e-05 | 1294.6ms/step | 15.4 samples/s +Step 31850/44000 | Loss: 2.9543 | LR: 9.12e-05 | 1345.2ms/step | 14.9 samples/s +Step 31860/44000 | Loss: 2.9441 | LR: 9.11e-05 | 1470.2ms/step | 13.6 samples/s +Step 31870/44000 | Loss: 2.9153 | LR: 9.09e-05 | 1315.4ms/step | 15.2 samples/s +Step 31880/44000 | Loss: 2.9243 | LR: 9.08e-05 | 1352.5ms/step | 14.8 samples/s +Step 31890/44000 | Loss: 2.8791 | LR: 9.06e-05 | 1305.0ms/step | 15.3 samples/s +Step 31900/44000 | Loss: 2.8911 | LR: 9.05e-05 | 1337.0ms/step | 15.0 samples/s +Step 31910/44000 | Loss: 2.9319 | LR: 9.04e-05 | 1330.7ms/step | 15.0 samples/s +Step 31920/44000 | Loss: 2.9608 | LR: 9.02e-05 | 1265.6ms/step | 15.8 samples/s +Step 31930/44000 | Loss: 2.8439 | LR: 9.01e-05 | 1322.6ms/step | 15.1 samples/s +Step 31940/44000 | Loss: 2.9270 | LR: 8.99e-05 | 1268.2ms/step | 15.8 samples/s +Step 31950/44000 | Loss: 2.9115 | LR: 8.98e-05 | 1284.1ms/step | 15.6 samples/s +Step 31960/44000 | Loss: 3.0062 | LR: 8.97e-05 | 1292.3ms/step | 15.5 samples/s +Step 31970/44000 | Loss: 2.8780 | LR: 8.95e-05 | 1270.2ms/step | 15.7 samples/s +Step 31980/44000 | Loss: 2.9080 | LR: 8.94e-05 | 1263.5ms/step | 15.8 samples/s +Step 31990/44000 | Loss: 2.9182 | LR: 8.92e-05 | 1274.6ms/step | 15.7 samples/s +Step 32000/44000 | Loss: 2.9542 | LR: 8.91e-05 | 1247.0ms/step | 16.0 samples/s +Validation at step 32000: Loss = 2.8757, Perplexity = 17.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-32000 +Step 32010/44000 | Loss: 2.8992 | LR: 8.90e-05 | 2309.1ms/step | 8.7 samples/s +Step 32020/44000 | Loss: 2.9133 | LR: 8.88e-05 | 1365.3ms/step | 14.6 samples/s +Step 32030/44000 | Loss: 2.9908 | LR: 8.87e-05 | 1373.5ms/step | 14.6 samples/s +Step 32040/44000 | Loss: 2.9757 | LR: 8.86e-05 | 1300.1ms/step | 15.4 samples/s +Step 32050/44000 | Loss: 2.8805 | LR: 8.84e-05 | 1315.7ms/step | 15.2 samples/s +Step 32060/44000 | Loss: 2.9505 | LR: 8.83e-05 | 1316.5ms/step | 15.2 samples/s +Step 32070/44000 | Loss: 2.9226 | LR: 8.81e-05 | 1315.9ms/step | 15.2 samples/s +Step 32080/44000 | Loss: 2.9196 | LR: 8.80e-05 | 1296.1ms/step | 15.4 samples/s +Step 32090/44000 | Loss: 2.9160 | LR: 8.79e-05 | 1283.9ms/step | 15.6 samples/s +Step 32100/44000 | Loss: 2.8480 | LR: 8.77e-05 | 1313.4ms/step | 15.2 samples/s +Step 32110/44000 | Loss: 2.9926 | LR: 8.76e-05 | 1276.4ms/step | 15.7 samples/s +Step 32120/44000 | Loss: 2.9120 | LR: 8.74e-05 | 1289.7ms/step | 15.5 samples/s +Step 32130/44000 | Loss: 2.8906 | LR: 8.73e-05 | 1268.9ms/step | 15.8 samples/s +Step 32140/44000 | Loss: 2.8363 | LR: 8.72e-05 | 1305.1ms/step | 15.3 samples/s +Step 32150/44000 | Loss: 2.8033 | LR: 8.70e-05 | 1310.4ms/step | 15.3 samples/s +Step 32160/44000 | Loss: 2.8976 | LR: 8.69e-05 | 1305.4ms/step | 15.3 samples/s +Step 32170/44000 | Loss: 2.9301 | LR: 8.68e-05 | 1273.1ms/step | 15.7 samples/s +Step 32180/44000 | Loss: 2.9057 | LR: 8.66e-05 | 1251.5ms/step | 16.0 samples/s +Step 32190/44000 | Loss: 2.9920 | LR: 8.65e-05 | 1306.9ms/step | 15.3 samples/s +Step 32200/44000 | Loss: 2.8896 | LR: 8.63e-05 | 1420.2ms/step | 14.1 samples/s +Step 32210/44000 | Loss: 2.9411 | LR: 8.62e-05 | 1293.6ms/step | 15.5 samples/s +Step 32220/44000 | Loss: 2.9152 | LR: 8.61e-05 | 1291.4ms/step | 15.5 samples/s +Step 32230/44000 | Loss: 2.9013 | LR: 8.59e-05 | 1332.5ms/step | 15.0 samples/s +Step 32240/44000 | Loss: 2.8880 | LR: 8.58e-05 | 1304.7ms/step | 15.3 samples/s +Step 32250/44000 | Loss: 2.9334 | LR: 8.57e-05 | 1294.2ms/step | 15.5 samples/s +Validation at step 32250: Loss = 2.8753, Perplexity = 17.73 +Step 32260/44000 | Loss: 2.8469 | LR: 8.55e-05 | 1790.4ms/step | 11.2 samples/s +Step 32270/44000 | Loss: 2.9834 | LR: 8.54e-05 | 1294.5ms/step | 15.4 samples/s +Step 32280/44000 | Loss: 2.9531 | LR: 8.53e-05 | 1273.6ms/step | 15.7 samples/s +Step 32290/44000 | Loss: 2.8690 | LR: 8.51e-05 | 1310.8ms/step | 15.3 samples/s +Step 32300/44000 | Loss: 2.9192 | LR: 8.50e-05 | 1299.7ms/step | 15.4 samples/s +Step 32310/44000 | Loss: 2.9047 | LR: 8.48e-05 | 1310.0ms/step | 15.3 samples/s +Step 32320/44000 | Loss: 2.9063 | LR: 8.47e-05 | 1236.2ms/step | 16.2 samples/s +Step 32330/44000 | Loss: 2.8680 | LR: 8.46e-05 | 1242.7ms/step | 16.1 samples/s +Step 32340/44000 | Loss: 2.8642 | LR: 8.44e-05 | 1271.9ms/step | 15.7 samples/s +Step 32350/44000 | Loss: 2.9404 | LR: 8.43e-05 | 1275.1ms/step | 15.7 samples/s +Step 32360/44000 | Loss: 2.9096 | LR: 8.42e-05 | 1341.5ms/step | 14.9 samples/s +Step 32370/44000 | Loss: 2.9032 | LR: 8.40e-05 | 1362.7ms/step | 14.7 samples/s +Step 32380/44000 | Loss: 3.0267 | LR: 8.39e-05 | 1268.0ms/step | 15.8 samples/s +Step 32390/44000 | Loss: 2.9840 | LR: 8.38e-05 | 1284.3ms/step | 15.6 samples/s +Step 32400/44000 | Loss: 2.8838 | LR: 8.36e-05 | 1284.2ms/step | 15.6 samples/s +Step 32410/44000 | Loss: 2.9435 | LR: 8.35e-05 | 1288.1ms/step | 15.5 samples/s +Step 32420/44000 | Loss: 2.9810 | LR: 8.33e-05 | 1260.8ms/step | 15.9 samples/s +Step 32430/44000 | Loss: 2.8548 | LR: 8.32e-05 | 1279.1ms/step | 15.6 samples/s +Step 32440/44000 | Loss: 2.9071 | LR: 8.31e-05 | 1239.4ms/step | 16.1 samples/s +Step 32450/44000 | Loss: 2.8943 | LR: 8.29e-05 | 1277.2ms/step | 15.7 samples/s +Step 32460/44000 | Loss: 2.9156 | LR: 8.28e-05 | 1268.3ms/step | 15.8 samples/s +Step 32470/44000 | Loss: 2.8953 | LR: 8.27e-05 | 1305.6ms/step | 15.3 samples/s +Step 32480/44000 | Loss: 2.8579 | LR: 8.25e-05 | 1305.3ms/step | 15.3 samples/s +Step 32490/44000 | Loss: 2.9089 | LR: 8.24e-05 | 1274.3ms/step | 15.7 samples/s +Step 32500/44000 | Loss: 2.9393 | LR: 8.23e-05 | 1320.6ms/step | 15.1 samples/s +Validation at step 32500: Loss = 2.8745, Perplexity = 17.72 +Step 32510/44000 | Loss: 2.9082 | LR: 8.21e-05 | 1836.2ms/step | 10.9 samples/s +Step 32520/44000 | Loss: 2.9120 | LR: 8.20e-05 | 1256.6ms/step | 15.9 samples/s +Step 32530/44000 | Loss: 2.8648 | LR: 8.19e-05 | 1357.6ms/step | 14.7 samples/s +Step 32540/44000 | Loss: 2.9174 | LR: 8.17e-05 | 1378.9ms/step | 14.5 samples/s +Step 32550/44000 | Loss: 2.8873 | LR: 8.16e-05 | 1278.7ms/step | 15.6 samples/s +Step 32560/44000 | Loss: 2.9119 | LR: 8.15e-05 | 1347.1ms/step | 14.8 samples/s +Step 32570/44000 | Loss: 2.9722 | LR: 8.13e-05 | 1277.1ms/step | 15.7 samples/s +Step 32580/44000 | Loss: 2.9419 | LR: 8.12e-05 | 1321.6ms/step | 15.1 samples/s +Step 32590/44000 | Loss: 3.0331 | LR: 8.11e-05 | 1295.7ms/step | 15.4 samples/s +Step 32600/44000 | Loss: 2.8884 | LR: 8.09e-05 | 1288.7ms/step | 15.5 samples/s +Step 32610/44000 | Loss: 2.9073 | LR: 8.08e-05 | 1284.0ms/step | 15.6 samples/s +Step 32620/44000 | Loss: 2.8285 | LR: 8.07e-05 | 1294.4ms/step | 15.5 samples/s +Step 32630/44000 | Loss: 2.9234 | LR: 8.05e-05 | 1268.1ms/step | 15.8 samples/s +Step 32640/44000 | Loss: 2.9282 | LR: 8.04e-05 | 1336.8ms/step | 15.0 samples/s +Step 32650/44000 | Loss: 2.9045 | LR: 8.03e-05 | 1361.2ms/step | 14.7 samples/s +Step 32660/44000 | Loss: 2.9349 | LR: 8.01e-05 | 1332.8ms/step | 15.0 samples/s +Step 32670/44000 | Loss: 2.8373 | LR: 8.00e-05 | 1316.2ms/step | 15.2 samples/s +Step 32680/44000 | Loss: 2.9495 | LR: 7.99e-05 | 1304.3ms/step | 15.3 samples/s +Step 32690/44000 | Loss: 2.9205 | LR: 7.97e-05 | 1285.0ms/step | 15.6 samples/s +Step 32700/44000 | Loss: 2.8815 | LR: 7.96e-05 | 1330.4ms/step | 15.0 samples/s +Step 32710/44000 | Loss: 2.9127 | LR: 7.95e-05 | 1405.2ms/step | 14.2 samples/s +Step 32720/44000 | Loss: 2.8852 | LR: 7.93e-05 | 1342.3ms/step | 14.9 samples/s +Step 32730/44000 | Loss: 2.9032 | LR: 7.92e-05 | 1331.7ms/step | 15.0 samples/s +Step 32740/44000 | Loss: 2.9178 | LR: 7.91e-05 | 1364.5ms/step | 14.7 samples/s +Step 32750/44000 | Loss: 2.9072 | LR: 7.89e-05 | 1261.6ms/step | 15.9 samples/s +Validation at step 32750: Loss = 2.8742, Perplexity = 17.71 +Step 32760/44000 | Loss: 3.0028 | LR: 7.88e-05 | 1823.4ms/step | 11.0 samples/s +Step 32770/44000 | Loss: 2.9181 | LR: 7.87e-05 | 1310.2ms/step | 15.3 samples/s +Step 32780/44000 | Loss: 2.9344 | LR: 7.85e-05 | 1347.2ms/step | 14.8 samples/s +Step 32790/44000 | Loss: 2.8567 | LR: 7.84e-05 | 1325.5ms/step | 15.1 samples/s +Step 32800/44000 | Loss: 2.8943 | LR: 7.83e-05 | 1288.9ms/step | 15.5 samples/s +Step 32810/44000 | Loss: 2.8553 | LR: 7.81e-05 | 1300.4ms/step | 15.4 samples/s +Step 32820/44000 | Loss: 2.9282 | LR: 7.80e-05 | 1314.7ms/step | 15.2 samples/s +Step 32830/44000 | Loss: 2.9426 | LR: 7.79e-05 | 1292.6ms/step | 15.5 samples/s +Step 32840/44000 | Loss: 2.9699 | LR: 7.77e-05 | 1297.5ms/step | 15.4 samples/s +Step 32850/44000 | Loss: 2.9129 | LR: 7.76e-05 | 1294.7ms/step | 15.4 samples/s +Step 32860/44000 | Loss: 2.8742 | LR: 7.75e-05 | 1273.3ms/step | 15.7 samples/s +Step 32870/44000 | Loss: 2.9059 | LR: 7.73e-05 | 1383.5ms/step | 14.5 samples/s +Step 32880/44000 | Loss: 2.8910 | LR: 7.72e-05 | 1326.0ms/step | 15.1 samples/s +Step 32890/44000 | Loss: 2.9273 | LR: 7.71e-05 | 1319.8ms/step | 15.2 samples/s +Step 32900/44000 | Loss: 2.9381 | LR: 7.70e-05 | 1325.9ms/step | 15.1 samples/s +Step 32910/44000 | Loss: 2.9072 | LR: 7.68e-05 | 1346.1ms/step | 14.9 samples/s +Step 32920/44000 | Loss: 2.9375 | LR: 7.67e-05 | 1381.3ms/step | 14.5 samples/s +Step 32930/44000 | Loss: 2.9651 | LR: 7.66e-05 | 1290.7ms/step | 15.5 samples/s +Step 32940/44000 | Loss: 2.9788 | LR: 7.64e-05 | 1289.4ms/step | 15.5 samples/s +Step 32950/44000 | Loss: 2.9634 | LR: 7.63e-05 | 1289.1ms/step | 15.5 samples/s +Step 32960/44000 | Loss: 2.8884 | LR: 7.62e-05 | 1257.3ms/step | 15.9 samples/s +Step 32970/44000 | Loss: 2.9532 | LR: 7.60e-05 | 1258.2ms/step | 15.9 samples/s +Step 32980/44000 | Loss: 2.9526 | LR: 7.59e-05 | 1263.1ms/step | 15.8 samples/s +Step 32990/44000 | Loss: 2.8644 | LR: 7.58e-05 | 1346.7ms/step | 14.9 samples/s +Step 33000/44000 | Loss: 2.9155 | LR: 7.56e-05 | 1272.3ms/step | 15.7 samples/s +Validation at step 33000: Loss = 2.8737, Perplexity = 17.70 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-33000 +Step 33010/44000 | Loss: 2.9078 | LR: 7.55e-05 | 2242.9ms/step | 8.9 samples/s +Step 33020/44000 | Loss: 2.9298 | LR: 7.54e-05 | 1280.3ms/step | 15.6 samples/s +Step 33030/44000 | Loss: 2.9229 | LR: 7.53e-05 | 1278.9ms/step | 15.6 samples/s +Step 33040/44000 | Loss: 2.9135 | LR: 7.51e-05 | 1433.3ms/step | 14.0 samples/s +Step 33050/44000 | Loss: 2.9442 | LR: 7.50e-05 | 1320.5ms/step | 15.1 samples/s +Step 33060/44000 | Loss: 2.8777 | LR: 7.49e-05 | 1298.4ms/step | 15.4 samples/s +Step 33070/44000 | Loss: 2.9348 | LR: 7.47e-05 | 1241.6ms/step | 16.1 samples/s +Step 33080/44000 | Loss: 2.8629 | LR: 7.46e-05 | 1334.1ms/step | 15.0 samples/s +Step 33090/44000 | Loss: 2.8975 | LR: 7.45e-05 | 1265.1ms/step | 15.8 samples/s +Step 33100/44000 | Loss: 2.9283 | LR: 7.44e-05 | 1274.2ms/step | 15.7 samples/s +Step 33110/44000 | Loss: 2.9554 | LR: 7.42e-05 | 1283.0ms/step | 15.6 samples/s +Step 33120/44000 | Loss: 2.9889 | LR: 7.41e-05 | 1277.2ms/step | 15.7 samples/s +Step 33130/44000 | Loss: 2.8786 | LR: 7.40e-05 | 1275.5ms/step | 15.7 samples/s +Step 33140/44000 | Loss: 2.8400 | LR: 7.38e-05 | 1314.6ms/step | 15.2 samples/s +Step 33150/44000 | Loss: 2.9111 | LR: 7.37e-05 | 1248.8ms/step | 16.0 samples/s +Step 33160/44000 | Loss: 2.9076 | LR: 7.36e-05 | 1315.6ms/step | 15.2 samples/s +Step 33170/44000 | Loss: 2.9296 | LR: 7.34e-05 | 1273.9ms/step | 15.7 samples/s +Step 33180/44000 | Loss: 2.8878 | LR: 7.33e-05 | 1258.0ms/step | 15.9 samples/s +Step 33190/44000 | Loss: 2.9378 | LR: 7.32e-05 | 1264.0ms/step | 15.8 samples/s +Step 33200/44000 | Loss: 2.9216 | LR: 7.31e-05 | 1291.7ms/step | 15.5 samples/s +Step 33210/44000 | Loss: 2.9268 | LR: 7.29e-05 | 1434.4ms/step | 13.9 samples/s +Step 33220/44000 | Loss: 2.8808 | LR: 7.28e-05 | 1289.5ms/step | 15.5 samples/s +Step 33230/44000 | Loss: 2.9729 | LR: 7.27e-05 | 1321.0ms/step | 15.1 samples/s +Step 33240/44000 | Loss: 2.9472 | LR: 7.26e-05 | 1280.9ms/step | 15.6 samples/s +Step 33250/44000 | Loss: 2.9107 | LR: 7.24e-05 | 1319.0ms/step | 15.2 samples/s +Validation at step 33250: Loss = 2.8733, Perplexity = 17.70 +Step 33260/44000 | Loss: 2.9412 | LR: 7.23e-05 | 1746.4ms/step | 11.5 samples/s +Step 33270/44000 | Loss: 2.8507 | LR: 7.22e-05 | 1214.9ms/step | 16.5 samples/s +Step 33280/44000 | Loss: 2.8831 | LR: 7.20e-05 | 1253.0ms/step | 16.0 samples/s +Step 33290/44000 | Loss: 2.8539 | LR: 7.19e-05 | 1280.3ms/step | 15.6 samples/s +Step 33300/44000 | Loss: 2.8517 | LR: 7.18e-05 | 1314.3ms/step | 15.2 samples/s +Step 33310/44000 | Loss: 2.9176 | LR: 7.17e-05 | 1286.0ms/step | 15.6 samples/s +Step 33320/44000 | Loss: 2.9053 | LR: 7.15e-05 | 1251.1ms/step | 16.0 samples/s +Step 33330/44000 | Loss: 2.9741 | LR: 7.14e-05 | 1269.0ms/step | 15.8 samples/s +Step 33340/44000 | Loss: 2.8980 | LR: 7.13e-05 | 1339.1ms/step | 14.9 samples/s +Step 33350/44000 | Loss: 2.9600 | LR: 7.11e-05 | 1276.4ms/step | 15.7 samples/s +Step 33360/44000 | Loss: 2.9136 | LR: 7.10e-05 | 1273.7ms/step | 15.7 samples/s +Step 33370/44000 | Loss: 2.8904 | LR: 7.09e-05 | 1315.9ms/step | 15.2 samples/s +Step 33380/44000 | Loss: 2.9131 | LR: 7.08e-05 | 1409.8ms/step | 14.2 samples/s +Step 33390/44000 | Loss: 2.9535 | LR: 7.06e-05 | 1320.3ms/step | 15.1 samples/s +Step 33400/44000 | Loss: 2.8882 | LR: 7.05e-05 | 1273.6ms/step | 15.7 samples/s +Step 33410/44000 | Loss: 2.9900 | LR: 7.04e-05 | 1268.5ms/step | 15.8 samples/s +Step 33420/44000 | Loss: 2.8804 | LR: 7.03e-05 | 1288.6ms/step | 15.5 samples/s +Step 33430/44000 | Loss: 2.9263 | LR: 7.01e-05 | 1290.5ms/step | 15.5 samples/s +Step 33440/44000 | Loss: 2.9145 | LR: 7.00e-05 | 1299.5ms/step | 15.4 samples/s +Step 33450/44000 | Loss: 2.9497 | LR: 6.99e-05 | 1257.9ms/step | 15.9 samples/s +Step 33460/44000 | Loss: 2.9024 | LR: 6.98e-05 | 1283.6ms/step | 15.6 samples/s +Step 33470/44000 | Loss: 2.9197 | LR: 6.96e-05 | 1273.5ms/step | 15.7 samples/s +Step 33480/44000 | Loss: 2.9544 | LR: 6.95e-05 | 1300.8ms/step | 15.4 samples/s +Step 33490/44000 | Loss: 2.9544 | LR: 6.94e-05 | 1323.5ms/step | 15.1 samples/s +Step 33500/44000 | Loss: 2.8994 | LR: 6.93e-05 | 1264.0ms/step | 15.8 samples/s +Validation at step 33500: Loss = 2.8730, Perplexity = 17.69 +Step 33510/44000 | Loss: 2.9224 | LR: 6.91e-05 | 1776.3ms/step | 11.3 samples/s +Step 33520/44000 | Loss: 2.9059 | LR: 6.90e-05 | 1341.2ms/step | 14.9 samples/s +Step 33530/44000 | Loss: 2.9177 | LR: 6.89e-05 | 1294.2ms/step | 15.5 samples/s +Step 33540/44000 | Loss: 2.9018 | LR: 6.88e-05 | 1284.1ms/step | 15.6 samples/s +Step 33550/44000 | Loss: 2.9037 | LR: 6.86e-05 | 1387.8ms/step | 14.4 samples/s +Step 33560/44000 | Loss: 2.9209 | LR: 6.85e-05 | 1312.2ms/step | 15.2 samples/s +Step 33570/44000 | Loss: 2.8631 | LR: 6.84e-05 | 1302.8ms/step | 15.4 samples/s +Step 33580/44000 | Loss: 2.8788 | LR: 6.83e-05 | 1264.0ms/step | 15.8 samples/s +Step 33590/44000 | Loss: 2.9181 | LR: 6.81e-05 | 1308.8ms/step | 15.3 samples/s +Step 33600/44000 | Loss: 2.9211 | LR: 6.80e-05 | 1273.2ms/step | 15.7 samples/s +Step 33610/44000 | Loss: 2.8741 | LR: 6.79e-05 | 1281.9ms/step | 15.6 samples/s +Step 33620/44000 | Loss: 2.8820 | LR: 6.78e-05 | 1260.4ms/step | 15.9 samples/s +Step 33630/44000 | Loss: 2.9574 | LR: 6.76e-05 | 1329.3ms/step | 15.0 samples/s +Step 33640/44000 | Loss: 2.9041 | LR: 6.75e-05 | 1292.3ms/step | 15.5 samples/s +Step 33650/44000 | Loss: 2.9292 | LR: 6.74e-05 | 1291.3ms/step | 15.5 samples/s +Step 33660/44000 | Loss: 2.9125 | LR: 6.73e-05 | 1273.7ms/step | 15.7 samples/s +Step 33670/44000 | Loss: 2.9488 | LR: 6.71e-05 | 1268.0ms/step | 15.8 samples/s +Step 33680/44000 | Loss: 2.8848 | LR: 6.70e-05 | 1247.5ms/step | 16.0 samples/s +Step 33690/44000 | Loss: 2.9892 | LR: 6.69e-05 | 1257.4ms/step | 15.9 samples/s +Step 33700/44000 | Loss: 2.9491 | LR: 6.68e-05 | 1277.1ms/step | 15.7 samples/s +Step 33710/44000 | Loss: 2.9199 | LR: 6.66e-05 | 1291.0ms/step | 15.5 samples/s +Step 33720/44000 | Loss: 2.9671 | LR: 6.65e-05 | 1331.0ms/step | 15.0 samples/s +Step 33730/44000 | Loss: 2.9631 | LR: 6.64e-05 | 1388.5ms/step | 14.4 samples/s +Step 33740/44000 | Loss: 2.9854 | LR: 6.63e-05 | 1253.2ms/step | 16.0 samples/s +Step 33750/44000 | Loss: 2.9218 | LR: 6.61e-05 | 1257.6ms/step | 15.9 samples/s +Validation at step 33750: Loss = 2.8728, Perplexity = 17.69 +Step 33760/44000 | Loss: 2.9206 | LR: 6.60e-05 | 1771.4ms/step | 11.3 samples/s +Step 33770/44000 | Loss: 2.9573 | LR: 6.59e-05 | 1294.3ms/step | 15.5 samples/s +Step 33780/44000 | Loss: 2.9728 | LR: 6.58e-05 | 1337.4ms/step | 15.0 samples/s +Step 33790/44000 | Loss: 2.9570 | LR: 6.57e-05 | 1340.0ms/step | 14.9 samples/s +Step 33800/44000 | Loss: 3.0158 | LR: 6.55e-05 | 1269.4ms/step | 15.8 samples/s +Step 33810/44000 | Loss: 2.9363 | LR: 6.54e-05 | 1310.0ms/step | 15.3 samples/s +Step 33820/44000 | Loss: 2.9034 | LR: 6.53e-05 | 1280.2ms/step | 15.6 samples/s +Step 33830/44000 | Loss: 2.8738 | LR: 6.52e-05 | 1298.2ms/step | 15.4 samples/s +Step 33840/44000 | Loss: 2.8915 | LR: 6.50e-05 | 1288.8ms/step | 15.5 samples/s +Step 33850/44000 | Loss: 2.9680 | LR: 6.49e-05 | 1284.3ms/step | 15.6 samples/s +Step 33860/44000 | Loss: 2.8864 | LR: 6.48e-05 | 1283.7ms/step | 15.6 samples/s +Step 33870/44000 | Loss: 2.8952 | LR: 6.47e-05 | 1335.4ms/step | 15.0 samples/s +Step 33880/44000 | Loss: 2.8975 | LR: 6.46e-05 | 1329.4ms/step | 15.0 samples/s +Step 33890/44000 | Loss: 2.9348 | LR: 6.44e-05 | 1323.8ms/step | 15.1 samples/s +Step 33900/44000 | Loss: 2.9240 | LR: 6.43e-05 | 1352.1ms/step | 14.8 samples/s +Step 33910/44000 | Loss: 2.9258 | LR: 6.42e-05 | 1297.5ms/step | 15.4 samples/s +Step 33920/44000 | Loss: 2.8854 | LR: 6.41e-05 | 1344.0ms/step | 14.9 samples/s +Step 33930/44000 | Loss: 2.9056 | LR: 6.40e-05 | 1262.6ms/step | 15.8 samples/s +Step 33940/44000 | Loss: 2.9438 | LR: 6.38e-05 | 1315.8ms/step | 15.2 samples/s +Step 33950/44000 | Loss: 2.8765 | LR: 6.37e-05 | 1293.9ms/step | 15.5 samples/s +Step 33960/44000 | Loss: 2.9898 | LR: 6.36e-05 | 1263.6ms/step | 15.8 samples/s +Step 33970/44000 | Loss: 2.9027 | LR: 6.35e-05 | 1289.1ms/step | 15.5 samples/s +Step 33980/44000 | Loss: 2.9843 | LR: 6.33e-05 | 1310.3ms/step | 15.3 samples/s +Step 33990/44000 | Loss: 2.8835 | LR: 6.32e-05 | 1320.1ms/step | 15.2 samples/s +Step 34000/44000 | Loss: 2.9567 | LR: 6.31e-05 | 1310.1ms/step | 15.3 samples/s +Validation at step 34000: Loss = 2.8727, Perplexity = 17.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-34000 +Step 34010/44000 | Loss: 2.8559 | LR: 6.30e-05 | 2244.0ms/step | 8.9 samples/s +Step 34020/44000 | Loss: 2.8399 | LR: 6.29e-05 | 1319.0ms/step | 15.2 samples/s +Step 34030/44000 | Loss: 2.9376 | LR: 6.27e-05 | 1310.4ms/step | 15.3 samples/s +Step 34040/44000 | Loss: 2.9090 | LR: 6.26e-05 | 1311.7ms/step | 15.2 samples/s +Step 34050/44000 | Loss: 2.8905 | LR: 6.25e-05 | 1275.1ms/step | 15.7 samples/s +Step 34060/44000 | Loss: 2.9353 | LR: 6.24e-05 | 1337.4ms/step | 15.0 samples/s +Step 34070/44000 | Loss: 2.9262 | LR: 6.23e-05 | 1318.9ms/step | 15.2 samples/s +Step 34080/44000 | Loss: 2.9858 | LR: 6.21e-05 | 1245.8ms/step | 16.1 samples/s +Step 34090/44000 | Loss: 2.9437 | LR: 6.20e-05 | 1302.8ms/step | 15.4 samples/s +Step 34100/44000 | Loss: 2.8527 | LR: 6.19e-05 | 1254.4ms/step | 15.9 samples/s +Step 34110/44000 | Loss: 2.8688 | LR: 6.18e-05 | 1302.7ms/step | 15.4 samples/s +Step 34120/44000 | Loss: 2.8890 | LR: 6.17e-05 | 1258.1ms/step | 15.9 samples/s +Step 34130/44000 | Loss: 2.9334 | LR: 6.15e-05 | 1307.6ms/step | 15.3 samples/s +Step 34140/44000 | Loss: 2.9474 | LR: 6.14e-05 | 1257.2ms/step | 15.9 samples/s +Step 34150/44000 | Loss: 2.9549 | LR: 6.13e-05 | 1274.2ms/step | 15.7 samples/s +Step 34160/44000 | Loss: 2.9329 | LR: 6.12e-05 | 1242.4ms/step | 16.1 samples/s +Step 34170/44000 | Loss: 2.9289 | LR: 6.11e-05 | 1268.0ms/step | 15.8 samples/s +Step 34180/44000 | Loss: 2.9276 | LR: 6.09e-05 | 1293.8ms/step | 15.5 samples/s +Step 34190/44000 | Loss: 2.9908 | LR: 6.08e-05 | 1237.0ms/step | 16.2 samples/s +Step 34200/44000 | Loss: 2.9345 | LR: 6.07e-05 | 1246.7ms/step | 16.0 samples/s +Step 34210/44000 | Loss: 2.9772 | LR: 6.06e-05 | 1295.5ms/step | 15.4 samples/s +Step 34220/44000 | Loss: 2.8879 | LR: 6.05e-05 | 1282.7ms/step | 15.6 samples/s +Step 34230/44000 | Loss: 2.8430 | LR: 6.04e-05 | 1258.2ms/step | 15.9 samples/s +Step 34240/44000 | Loss: 2.9014 | LR: 6.02e-05 | 1289.5ms/step | 15.5 samples/s +Step 34250/44000 | Loss: 2.9622 | LR: 6.01e-05 | 1247.4ms/step | 16.0 samples/s +Validation at step 34250: Loss = 2.8722, Perplexity = 17.68 +Step 34260/44000 | Loss: 2.8988 | LR: 6.00e-05 | 1776.4ms/step | 11.3 samples/s +Step 34270/44000 | Loss: 2.9211 | LR: 5.99e-05 | 1259.0ms/step | 15.9 samples/s +Step 34280/44000 | Loss: 2.9476 | LR: 5.98e-05 | 1246.0ms/step | 16.1 samples/s +Step 34290/44000 | Loss: 2.9504 | LR: 5.96e-05 | 1294.7ms/step | 15.4 samples/s +Step 34300/44000 | Loss: 2.9178 | LR: 5.95e-05 | 1283.7ms/step | 15.6 samples/s +Step 34310/44000 | Loss: 2.9463 | LR: 5.94e-05 | 1246.9ms/step | 16.0 samples/s +Step 34320/44000 | Loss: 2.9461 | LR: 5.93e-05 | 1280.4ms/step | 15.6 samples/s +Step 34330/44000 | Loss: 2.9202 | LR: 5.92e-05 | 1266.7ms/step | 15.8 samples/s +Step 34340/44000 | Loss: 2.9171 | LR: 5.91e-05 | 1325.5ms/step | 15.1 samples/s +Step 34350/44000 | Loss: 2.8887 | LR: 5.89e-05 | 1305.0ms/step | 15.3 samples/s +Step 34360/44000 | Loss: 2.9083 | LR: 5.88e-05 | 1352.1ms/step | 14.8 samples/s +Step 34370/44000 | Loss: 2.8773 | LR: 5.87e-05 | 1331.4ms/step | 15.0 samples/s +Step 34380/44000 | Loss: 2.9312 | LR: 5.86e-05 | 1336.1ms/step | 15.0 samples/s +Step 34390/44000 | Loss: 2.8665 | LR: 5.85e-05 | 1301.2ms/step | 15.4 samples/s +Step 34400/44000 | Loss: 2.9523 | LR: 5.84e-05 | 1250.4ms/step | 16.0 samples/s +Step 34410/44000 | Loss: 2.9238 | LR: 5.82e-05 | 1330.5ms/step | 15.0 samples/s +Step 34420/44000 | Loss: 2.9412 | LR: 5.81e-05 | 1305.2ms/step | 15.3 samples/s +Step 34430/44000 | Loss: 2.9297 | LR: 5.80e-05 | 1278.2ms/step | 15.6 samples/s +Step 34440/44000 | Loss: 2.8911 | LR: 5.79e-05 | 1256.4ms/step | 15.9 samples/s +Step 34450/44000 | Loss: 2.8851 | LR: 5.78e-05 | 1243.2ms/step | 16.1 samples/s +Step 34460/44000 | Loss: 2.9525 | LR: 5.77e-05 | 1272.4ms/step | 15.7 samples/s +Step 34470/44000 | Loss: 2.8504 | LR: 5.75e-05 | 1284.0ms/step | 15.6 samples/s +Step 34480/44000 | Loss: 2.8979 | LR: 5.74e-05 | 1366.0ms/step | 14.6 samples/s +Step 34490/44000 | Loss: 2.9114 | LR: 5.73e-05 | 1290.7ms/step | 15.5 samples/s +Step 34500/44000 | Loss: 2.9740 | LR: 5.72e-05 | 1273.2ms/step | 15.7 samples/s +Validation at step 34500: Loss = 2.8720, Perplexity = 17.67 +Step 34510/44000 | Loss: 2.8571 | LR: 5.71e-05 | 1758.0ms/step | 11.4 samples/s +Step 34520/44000 | Loss: 2.9438 | LR: 5.70e-05 | 1254.7ms/step | 15.9 samples/s +Step 34530/44000 | Loss: 2.9130 | LR: 5.69e-05 | 1240.5ms/step | 16.1 samples/s +Step 34540/44000 | Loss: 2.9671 | LR: 5.67e-05 | 1263.5ms/step | 15.8 samples/s +Step 34550/44000 | Loss: 2.9178 | LR: 5.66e-05 | 1263.5ms/step | 15.8 samples/s +Step 34560/44000 | Loss: 2.9183 | LR: 5.65e-05 | 1285.8ms/step | 15.6 samples/s +Step 34570/44000 | Loss: 2.9491 | LR: 5.64e-05 | 1213.4ms/step | 16.5 samples/s +Step 34580/44000 | Loss: 2.8449 | LR: 5.63e-05 | 1277.8ms/step | 15.7 samples/s +Step 34590/44000 | Loss: 2.8800 | LR: 5.62e-05 | 1248.1ms/step | 16.0 samples/s +Step 34600/44000 | Loss: 2.8550 | LR: 5.60e-05 | 1242.0ms/step | 16.1 samples/s +Step 34610/44000 | Loss: 2.9325 | LR: 5.59e-05 | 1236.7ms/step | 16.2 samples/s +Step 34620/44000 | Loss: 2.8898 | LR: 5.58e-05 | 1246.4ms/step | 16.0 samples/s +Step 34630/44000 | Loss: 2.9521 | LR: 5.57e-05 | 1273.7ms/step | 15.7 samples/s +Step 34640/44000 | Loss: 2.9261 | LR: 5.56e-05 | 1237.9ms/step | 16.2 samples/s +Step 34650/44000 | Loss: 2.8824 | LR: 5.55e-05 | 1244.7ms/step | 16.1 samples/s +Step 34660/44000 | Loss: 2.9066 | LR: 5.54e-05 | 1283.6ms/step | 15.6 samples/s +Step 34670/44000 | Loss: 2.8679 | LR: 5.52e-05 | 1249.8ms/step | 16.0 samples/s +Step 34680/44000 | Loss: 2.9428 | LR: 5.51e-05 | 1247.3ms/step | 16.0 samples/s +Step 34690/44000 | Loss: 2.9562 | LR: 5.50e-05 | 1268.6ms/step | 15.8 samples/s +Step 34700/44000 | Loss: 2.8707 | LR: 5.49e-05 | 1208.5ms/step | 16.5 samples/s +Step 34710/44000 | Loss: 2.9545 | LR: 5.48e-05 | 1233.5ms/step | 16.2 samples/s +Step 34720/44000 | Loss: 2.9137 | LR: 5.47e-05 | 1280.8ms/step | 15.6 samples/s +Step 34730/44000 | Loss: 2.9838 | LR: 5.46e-05 | 1229.6ms/step | 16.3 samples/s +Step 34740/44000 | Loss: 2.9188 | LR: 5.45e-05 | 1279.5ms/step | 15.6 samples/s +Step 34750/44000 | Loss: 2.9760 | LR: 5.43e-05 | 1273.6ms/step | 15.7 samples/s +Validation at step 34750: Loss = 2.8715, Perplexity = 17.66 +Step 34760/44000 | Loss: 2.9726 | LR: 5.42e-05 | 1761.0ms/step | 11.4 samples/s +Step 34770/44000 | Loss: 2.9076 | LR: 5.41e-05 | 1269.2ms/step | 15.8 samples/s +Step 34780/44000 | Loss: 2.8973 | LR: 5.40e-05 | 1276.3ms/step | 15.7 samples/s +Step 34790/44000 | Loss: 2.8952 | LR: 5.39e-05 | 1249.1ms/step | 16.0 samples/s +Step 34800/44000 | Loss: 2.9409 | LR: 5.38e-05 | 1195.2ms/step | 16.7 samples/s +Step 34810/44000 | Loss: 2.8929 | LR: 5.37e-05 | 1247.5ms/step | 16.0 samples/s +Step 34820/44000 | Loss: 2.9248 | LR: 5.36e-05 | 1259.1ms/step | 15.9 samples/s +Step 34830/44000 | Loss: 2.9655 | LR: 5.34e-05 | 1272.2ms/step | 15.7 samples/s +Step 34840/44000 | Loss: 2.9438 | LR: 5.33e-05 | 1268.5ms/step | 15.8 samples/s +Step 34850/44000 | Loss: 2.9341 | LR: 5.32e-05 | 1240.2ms/step | 16.1 samples/s +Step 34860/44000 | Loss: 2.9540 | LR: 5.31e-05 | 1249.4ms/step | 16.0 samples/s +Step 34870/44000 | Loss: 2.9006 | LR: 5.30e-05 | 1257.7ms/step | 15.9 samples/s +Step 34880/44000 | Loss: 2.7910 | LR: 5.29e-05 | 1248.0ms/step | 16.0 samples/s +Step 34890/44000 | Loss: 2.8999 | LR: 5.28e-05 | 1257.6ms/step | 15.9 samples/s +Step 34900/44000 | Loss: 2.8415 | LR: 5.27e-05 | 1231.6ms/step | 16.2 samples/s +Step 34910/44000 | Loss: 2.8613 | LR: 5.25e-05 | 1232.1ms/step | 16.2 samples/s +Step 34920/44000 | Loss: 2.8934 | LR: 5.24e-05 | 1264.4ms/step | 15.8 samples/s +Step 34930/44000 | Loss: 2.9582 | LR: 5.23e-05 | 1262.6ms/step | 15.8 samples/s +Step 34940/44000 | Loss: 2.8766 | LR: 5.22e-05 | 1305.0ms/step | 15.3 samples/s +Step 34950/44000 | Loss: 2.9498 | LR: 5.21e-05 | 1258.2ms/step | 15.9 samples/s +Step 34960/44000 | Loss: 2.8823 | LR: 5.20e-05 | 1234.9ms/step | 16.2 samples/s +Step 34970/44000 | Loss: 2.9187 | LR: 5.19e-05 | 1217.7ms/step | 16.4 samples/s +Step 34980/44000 | Loss: 2.9280 | LR: 5.18e-05 | 1284.3ms/step | 15.6 samples/s +Step 34990/44000 | Loss: 2.8805 | LR: 5.17e-05 | 1242.0ms/step | 16.1 samples/s +Step 35000/44000 | Loss: 2.9400 | LR: 5.15e-05 | 1258.5ms/step | 15.9 samples/s +Validation at step 35000: Loss = 2.8714, Perplexity = 17.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-35000 +Step 35010/44000 | Loss: 2.8944 | LR: 5.14e-05 | 2229.7ms/step | 9.0 samples/s +Step 35020/44000 | Loss: 2.9331 | LR: 5.13e-05 | 1254.9ms/step | 15.9 samples/s +Step 35030/44000 | Loss: 2.9003 | LR: 5.12e-05 | 1284.7ms/step | 15.6 samples/s +Step 35040/44000 | Loss: 2.9591 | LR: 5.11e-05 | 1320.5ms/step | 15.1 samples/s +Step 35050/44000 | Loss: 2.8859 | LR: 5.10e-05 | 1305.1ms/step | 15.3 samples/s +Step 35060/44000 | Loss: 2.9509 | LR: 5.09e-05 | 1313.5ms/step | 15.2 samples/s +Step 35070/44000 | Loss: 2.8722 | LR: 5.08e-05 | 1249.4ms/step | 16.0 samples/s +Step 35080/44000 | Loss: 2.9544 | LR: 5.07e-05 | 1254.8ms/step | 15.9 samples/s +Step 35090/44000 | Loss: 2.9045 | LR: 5.06e-05 | 1262.2ms/step | 15.8 samples/s +Step 35100/44000 | Loss: 2.8982 | LR: 5.04e-05 | 1284.5ms/step | 15.6 samples/s +Step 35110/44000 | Loss: 2.9395 | LR: 5.03e-05 | 1340.9ms/step | 14.9 samples/s +Step 35120/44000 | Loss: 2.8680 | LR: 5.02e-05 | 1282.7ms/step | 15.6 samples/s +Step 35130/44000 | Loss: 2.9501 | LR: 5.01e-05 | 1187.7ms/step | 16.8 samples/s +Step 35140/44000 | Loss: 2.8768 | LR: 5.00e-05 | 1270.5ms/step | 15.7 samples/s +Step 35150/44000 | Loss: 2.9478 | LR: 4.99e-05 | 1226.2ms/step | 16.3 samples/s +Step 35160/44000 | Loss: 2.9491 | LR: 4.98e-05 | 1255.1ms/step | 15.9 samples/s +Step 35170/44000 | Loss: 2.9362 | LR: 4.97e-05 | 1244.5ms/step | 16.1 samples/s +Step 35180/44000 | Loss: 2.9213 | LR: 4.96e-05 | 1290.0ms/step | 15.5 samples/s +Step 35190/44000 | Loss: 2.9945 | LR: 4.95e-05 | 1257.9ms/step | 15.9 samples/s +Step 35200/44000 | Loss: 2.9181 | LR: 4.94e-05 | 1236.5ms/step | 16.2 samples/s +Step 35210/44000 | Loss: 2.9045 | LR: 4.93e-05 | 1258.8ms/step | 15.9 samples/s +Step 35220/44000 | Loss: 2.8852 | LR: 4.91e-05 | 1293.8ms/step | 15.5 samples/s +Step 35230/44000 | Loss: 2.8882 | LR: 4.90e-05 | 1276.7ms/step | 15.7 samples/s +Step 35240/44000 | Loss: 2.9174 | LR: 4.89e-05 | 1246.1ms/step | 16.0 samples/s +Step 35250/44000 | Loss: 2.9383 | LR: 4.88e-05 | 1276.6ms/step | 15.7 samples/s +Validation at step 35250: Loss = 2.8713, Perplexity = 17.66 +Step 35260/44000 | Loss: 2.9546 | LR: 4.87e-05 | 1713.8ms/step | 11.7 samples/s +Step 35270/44000 | Loss: 2.9576 | LR: 4.86e-05 | 1289.4ms/step | 15.5 samples/s +Step 35280/44000 | Loss: 2.9561 | LR: 4.85e-05 | 1322.2ms/step | 15.1 samples/s +Step 35290/44000 | Loss: 2.8623 | LR: 4.84e-05 | 1268.4ms/step | 15.8 samples/s +Step 35300/44000 | Loss: 2.9724 | LR: 4.83e-05 | 1283.6ms/step | 15.6 samples/s +Step 35310/44000 | Loss: 2.9282 | LR: 4.82e-05 | 1226.3ms/step | 16.3 samples/s +Step 35320/44000 | Loss: 2.9418 | LR: 4.81e-05 | 1295.1ms/step | 15.4 samples/s +Step 35330/44000 | Loss: 2.9291 | LR: 4.80e-05 | 1315.4ms/step | 15.2 samples/s +Step 35340/44000 | Loss: 2.8598 | LR: 4.79e-05 | 1277.4ms/step | 15.7 samples/s +Step 35350/44000 | Loss: 2.9629 | LR: 4.77e-05 | 1319.3ms/step | 15.2 samples/s +Step 35360/44000 | Loss: 2.8889 | LR: 4.76e-05 | 1260.1ms/step | 15.9 samples/s +Step 35370/44000 | Loss: 2.8725 | LR: 4.75e-05 | 1309.8ms/step | 15.3 samples/s +Step 35380/44000 | Loss: 2.8650 | LR: 4.74e-05 | 1273.8ms/step | 15.7 samples/s +Step 35390/44000 | Loss: 2.8879 | LR: 4.73e-05 | 1262.9ms/step | 15.8 samples/s +Step 35400/44000 | Loss: 2.9058 | LR: 4.72e-05 | 1273.1ms/step | 15.7 samples/s +Step 35410/44000 | Loss: 2.9237 | LR: 4.71e-05 | 1249.8ms/step | 16.0 samples/s +Step 35420/44000 | Loss: 2.8661 | LR: 4.70e-05 | 1234.2ms/step | 16.2 samples/s +Step 35430/44000 | Loss: 2.8505 | LR: 4.69e-05 | 1273.7ms/step | 15.7 samples/s +Step 35440/44000 | Loss: 2.9198 | LR: 4.68e-05 | 1257.6ms/step | 15.9 samples/s +Step 35450/44000 | Loss: 2.8959 | LR: 4.67e-05 | 1273.3ms/step | 15.7 samples/s +Step 35460/44000 | Loss: 2.9659 | LR: 4.66e-05 | 1275.9ms/step | 15.7 samples/s +Step 35470/44000 | Loss: 2.8741 | LR: 4.65e-05 | 1265.4ms/step | 15.8 samples/s +Step 35480/44000 | Loss: 2.8931 | LR: 4.64e-05 | 1253.6ms/step | 16.0 samples/s +Step 35490/44000 | Loss: 2.9623 | LR: 4.63e-05 | 1249.3ms/step | 16.0 samples/s +Step 35500/44000 | Loss: 2.8929 | LR: 4.62e-05 | 1301.4ms/step | 15.4 samples/s +Validation at step 35500: Loss = 2.8710, Perplexity = 17.65 +Step 35510/44000 | Loss: 2.8036 | LR: 4.61e-05 | 1739.9ms/step | 11.5 samples/s +Step 35520/44000 | Loss: 2.9164 | LR: 4.59e-05 | 1288.9ms/step | 15.5 samples/s +Step 35530/44000 | Loss: 2.9642 | LR: 4.58e-05 | 1288.6ms/step | 15.5 samples/s +Step 35540/44000 | Loss: 2.9898 | LR: 4.57e-05 | 1278.8ms/step | 15.6 samples/s +Step 35550/44000 | Loss: 2.9382 | LR: 4.56e-05 | 1294.3ms/step | 15.5 samples/s +Step 35560/44000 | Loss: 2.9588 | LR: 4.55e-05 | 1263.7ms/step | 15.8 samples/s +Step 35570/44000 | Loss: 2.9388 | LR: 4.54e-05 | 1252.4ms/step | 16.0 samples/s +Step 35580/44000 | Loss: 2.8829 | LR: 4.53e-05 | 1236.7ms/step | 16.2 samples/s +Step 35590/44000 | Loss: 2.9383 | LR: 4.52e-05 | 1262.8ms/step | 15.8 samples/s +Step 35600/44000 | Loss: 2.8858 | LR: 4.51e-05 | 1254.9ms/step | 15.9 samples/s +Step 35610/44000 | Loss: 2.9137 | LR: 4.50e-05 | 1260.7ms/step | 15.9 samples/s +Step 35620/44000 | Loss: 2.9059 | LR: 4.49e-05 | 1283.9ms/step | 15.6 samples/s +Step 35630/44000 | Loss: 2.9100 | LR: 4.48e-05 | 1315.7ms/step | 15.2 samples/s +Step 35640/44000 | Loss: 2.8949 | LR: 4.47e-05 | 1273.2ms/step | 15.7 samples/s +Step 35650/44000 | Loss: 2.9464 | LR: 4.46e-05 | 1231.5ms/step | 16.2 samples/s +Step 35660/44000 | Loss: 2.9310 | LR: 4.45e-05 | 1247.2ms/step | 16.0 samples/s +Step 35670/44000 | Loss: 2.9238 | LR: 4.44e-05 | 1231.7ms/step | 16.2 samples/s +Step 35680/44000 | Loss: 2.9169 | LR: 4.43e-05 | 1226.6ms/step | 16.3 samples/s +Step 35690/44000 | Loss: 2.9118 | LR: 4.42e-05 | 1220.4ms/step | 16.4 samples/s +Step 35700/44000 | Loss: 2.9267 | LR: 4.41e-05 | 1290.0ms/step | 15.5 samples/s +Step 35710/44000 | Loss: 2.9304 | LR: 4.40e-05 | 1241.0ms/step | 16.1 samples/s +Step 35720/44000 | Loss: 2.9695 | LR: 4.39e-05 | 1258.4ms/step | 15.9 samples/s +Step 35730/44000 | Loss: 2.9115 | LR: 4.38e-05 | 1219.4ms/step | 16.4 samples/s +Step 35740/44000 | Loss: 2.9501 | LR: 4.37e-05 | 1242.1ms/step | 16.1 samples/s +Step 35750/44000 | Loss: 2.9692 | LR: 4.36e-05 | 1281.7ms/step | 15.6 samples/s +Validation at step 35750: Loss = 2.8709, Perplexity = 17.65 +Step 35760/44000 | Loss: 2.9386 | LR: 4.35e-05 | 1685.0ms/step | 11.9 samples/s +Step 35770/44000 | Loss: 2.9043 | LR: 4.34e-05 | 1267.9ms/step | 15.8 samples/s +Step 35780/44000 | Loss: 2.9293 | LR: 4.33e-05 | 1242.7ms/step | 16.1 samples/s +Step 35790/44000 | Loss: 2.9426 | LR: 4.32e-05 | 1273.7ms/step | 15.7 samples/s +Step 35800/44000 | Loss: 2.8910 | LR: 4.31e-05 | 1263.1ms/step | 15.8 samples/s +Step 35810/44000 | Loss: 2.9786 | LR: 4.29e-05 | 1294.3ms/step | 15.5 samples/s +Step 35820/44000 | Loss: 2.9049 | LR: 4.28e-05 | 1256.6ms/step | 15.9 samples/s +Step 35830/44000 | Loss: 2.8693 | LR: 4.27e-05 | 1244.4ms/step | 16.1 samples/s +Step 35840/44000 | Loss: 2.9805 | LR: 4.26e-05 | 1311.1ms/step | 15.3 samples/s +Step 35850/44000 | Loss: 2.8777 | LR: 4.25e-05 | 1298.4ms/step | 15.4 samples/s +Step 35860/44000 | Loss: 2.9849 | LR: 4.24e-05 | 1289.6ms/step | 15.5 samples/s +Step 35870/44000 | Loss: 2.8910 | LR: 4.23e-05 | 1294.6ms/step | 15.4 samples/s +Step 35880/44000 | Loss: 2.8711 | LR: 4.22e-05 | 1258.4ms/step | 15.9 samples/s +Step 35890/44000 | Loss: 2.9135 | LR: 4.21e-05 | 1313.9ms/step | 15.2 samples/s +Step 35900/44000 | Loss: 2.8329 | LR: 4.20e-05 | 1295.8ms/step | 15.4 samples/s +Step 35910/44000 | Loss: 3.0396 | LR: 4.19e-05 | 1271.7ms/step | 15.7 samples/s +Step 35920/44000 | Loss: 2.9727 | LR: 4.18e-05 | 1318.2ms/step | 15.2 samples/s +Step 35930/44000 | Loss: 2.8334 | LR: 4.17e-05 | 1352.3ms/step | 14.8 samples/s +Step 35940/44000 | Loss: 2.8678 | LR: 4.16e-05 | 1311.1ms/step | 15.3 samples/s +Step 35950/44000 | Loss: 2.8491 | LR: 4.15e-05 | 1362.1ms/step | 14.7 samples/s +Step 35960/44000 | Loss: 2.9258 | LR: 4.14e-05 | 1341.4ms/step | 14.9 samples/s +Step 35970/44000 | Loss: 2.9452 | LR: 4.13e-05 | 1332.3ms/step | 15.0 samples/s +Step 35980/44000 | Loss: 2.9725 | LR: 4.12e-05 | 1388.7ms/step | 14.4 samples/s +Step 35990/44000 | Loss: 2.9167 | LR: 4.11e-05 | 1362.7ms/step | 14.7 samples/s +Step 36000/44000 | Loss: 2.9170 | LR: 4.10e-05 | 1347.0ms/step | 14.8 samples/s +Validation at step 36000: Loss = 2.8707, Perplexity = 17.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-36000 +Step 36010/44000 | Loss: 2.8843 | LR: 4.09e-05 | 2338.0ms/step | 8.6 samples/s +Step 36020/44000 | Loss: 2.9042 | LR: 4.08e-05 | 1326.6ms/step | 15.1 samples/s +Step 36030/44000 | Loss: 2.9051 | LR: 4.07e-05 | 1352.5ms/step | 14.8 samples/s +Step 36040/44000 | Loss: 2.9559 | LR: 4.06e-05 | 1308.8ms/step | 15.3 samples/s +Step 36050/44000 | Loss: 2.9526 | LR: 4.05e-05 | 1334.9ms/step | 15.0 samples/s +Step 36060/44000 | Loss: 2.9745 | LR: 4.04e-05 | 1300.1ms/step | 15.4 samples/s +Step 36070/44000 | Loss: 2.9528 | LR: 4.03e-05 | 1311.4ms/step | 15.3 samples/s +Step 36080/44000 | Loss: 2.8751 | LR: 4.02e-05 | 1299.7ms/step | 15.4 samples/s +Step 36090/44000 | Loss: 2.9426 | LR: 4.01e-05 | 1234.0ms/step | 16.2 samples/s +Step 36100/44000 | Loss: 2.9526 | LR: 4.00e-05 | 1329.9ms/step | 15.0 samples/s +Step 36110/44000 | Loss: 2.9115 | LR: 3.99e-05 | 1327.2ms/step | 15.1 samples/s +Step 36120/44000 | Loss: 2.9664 | LR: 3.98e-05 | 1318.4ms/step | 15.2 samples/s +Step 36130/44000 | Loss: 2.8998 | LR: 3.97e-05 | 1294.6ms/step | 15.4 samples/s +Step 36140/44000 | Loss: 2.9586 | LR: 3.97e-05 | 1310.2ms/step | 15.3 samples/s +Step 36150/44000 | Loss: 2.9014 | LR: 3.96e-05 | 1284.0ms/step | 15.6 samples/s +Step 36160/44000 | Loss: 2.8820 | LR: 3.95e-05 | 1222.2ms/step | 16.4 samples/s +Step 36170/44000 | Loss: 2.9401 | LR: 3.94e-05 | 1267.4ms/step | 15.8 samples/s +Step 36180/44000 | Loss: 2.9599 | LR: 3.93e-05 | 1252.5ms/step | 16.0 samples/s +Step 36190/44000 | Loss: 2.8751 | LR: 3.92e-05 | 1263.2ms/step | 15.8 samples/s +Step 36200/44000 | Loss: 2.8991 | LR: 3.91e-05 | 1252.2ms/step | 16.0 samples/s +Step 36210/44000 | Loss: 2.9065 | LR: 3.90e-05 | 1247.2ms/step | 16.0 samples/s +Step 36220/44000 | Loss: 2.8065 | LR: 3.89e-05 | 1289.4ms/step | 15.5 samples/s +Step 36230/44000 | Loss: 2.9236 | LR: 3.88e-05 | 1279.7ms/step | 15.6 samples/s +Step 36240/44000 | Loss: 2.9657 | LR: 3.87e-05 | 1272.2ms/step | 15.7 samples/s +Step 36250/44000 | Loss: 2.9021 | LR: 3.86e-05 | 1231.0ms/step | 16.2 samples/s +Validation at step 36250: Loss = 2.8706, Perplexity = 17.65 +Step 36260/44000 | Loss: 2.9634 | LR: 3.85e-05 | 1750.2ms/step | 11.4 samples/s +Step 36270/44000 | Loss: 2.9063 | LR: 3.84e-05 | 1282.5ms/step | 15.6 samples/s +Step 36280/44000 | Loss: 2.9109 | LR: 3.83e-05 | 1239.8ms/step | 16.1 samples/s +Step 36290/44000 | Loss: 2.8858 | LR: 3.82e-05 | 1307.2ms/step | 15.3 samples/s +Step 36300/44000 | Loss: 2.9120 | LR: 3.81e-05 | 1257.6ms/step | 15.9 samples/s +Step 36310/44000 | Loss: 2.9883 | LR: 3.80e-05 | 1316.9ms/step | 15.2 samples/s +Step 36320/44000 | Loss: 2.9770 | LR: 3.79e-05 | 1297.7ms/step | 15.4 samples/s +Step 36330/44000 | Loss: 2.8951 | LR: 3.78e-05 | 1283.3ms/step | 15.6 samples/s +Step 36340/44000 | Loss: 2.9733 | LR: 3.77e-05 | 1242.1ms/step | 16.1 samples/s +Step 36350/44000 | Loss: 2.9318 | LR: 3.76e-05 | 1262.8ms/step | 15.8 samples/s +Step 36360/44000 | Loss: 2.9047 | LR: 3.75e-05 | 1320.6ms/step | 15.1 samples/s +Step 36370/44000 | Loss: 2.8175 | LR: 3.74e-05 | 1304.3ms/step | 15.3 samples/s +Step 36380/44000 | Loss: 2.9226 | LR: 3.73e-05 | 1284.5ms/step | 15.6 samples/s +Step 36390/44000 | Loss: 2.9047 | LR: 3.72e-05 | 1276.4ms/step | 15.7 samples/s +Step 36400/44000 | Loss: 2.8979 | LR: 3.71e-05 | 1342.8ms/step | 14.9 samples/s +Step 36410/44000 | Loss: 2.9378 | LR: 3.70e-05 | 1353.6ms/step | 14.8 samples/s +Step 36420/44000 | Loss: 2.8936 | LR: 3.69e-05 | 1390.5ms/step | 14.4 samples/s +Step 36430/44000 | Loss: 2.9475 | LR: 3.69e-05 | 1345.4ms/step | 14.9 samples/s +Step 36440/44000 | Loss: 2.8944 | LR: 3.68e-05 | 1430.9ms/step | 14.0 samples/s +Step 36450/44000 | Loss: 2.8908 | LR: 3.67e-05 | 1331.0ms/step | 15.0 samples/s +Step 36460/44000 | Loss: 2.9450 | LR: 3.66e-05 | 1326.2ms/step | 15.1 samples/s +Step 36470/44000 | Loss: 2.8712 | LR: 3.65e-05 | 1344.6ms/step | 14.9 samples/s +Step 36480/44000 | Loss: 2.9742 | LR: 3.64e-05 | 1393.0ms/step | 14.4 samples/s +Step 36490/44000 | Loss: 2.9278 | LR: 3.63e-05 | 1419.3ms/step | 14.1 samples/s +Step 36500/44000 | Loss: 2.9472 | LR: 3.62e-05 | 1326.3ms/step | 15.1 samples/s +Validation at step 36500: Loss = 2.8703, Perplexity = 17.64 +Step 36510/44000 | Loss: 2.9323 | LR: 3.61e-05 | 1856.3ms/step | 10.8 samples/s +Step 36520/44000 | Loss: 3.0091 | LR: 3.60e-05 | 1360.9ms/step | 14.7 samples/s +Step 36530/44000 | Loss: 2.8904 | LR: 3.59e-05 | 1436.0ms/step | 13.9 samples/s +Step 36540/44000 | Loss: 3.0044 | LR: 3.58e-05 | 1373.7ms/step | 14.6 samples/s +Step 36550/44000 | Loss: 2.9324 | LR: 3.57e-05 | 1404.7ms/step | 14.2 samples/s +Step 36560/44000 | Loss: 2.9243 | LR: 3.56e-05 | 1394.5ms/step | 14.3 samples/s +Step 36570/44000 | Loss: 2.9661 | LR: 3.55e-05 | 1393.1ms/step | 14.4 samples/s +Step 36580/44000 | Loss: 2.9596 | LR: 3.54e-05 | 1457.1ms/step | 13.7 samples/s +Step 36590/44000 | Loss: 2.8992 | LR: 3.53e-05 | 1358.4ms/step | 14.7 samples/s +Step 36600/44000 | Loss: 2.8110 | LR: 3.53e-05 | 1359.2ms/step | 14.7 samples/s +Step 36610/44000 | Loss: 2.9654 | LR: 3.52e-05 | 1365.6ms/step | 14.6 samples/s +Step 36620/44000 | Loss: 2.8510 | LR: 3.51e-05 | 1325.7ms/step | 15.1 samples/s +Step 36630/44000 | Loss: 2.8870 | LR: 3.50e-05 | 1288.0ms/step | 15.5 samples/s +Step 36640/44000 | Loss: 2.9089 | LR: 3.49e-05 | 1342.3ms/step | 14.9 samples/s +Step 36650/44000 | Loss: 2.9493 | LR: 3.48e-05 | 1352.7ms/step | 14.8 samples/s +Step 36660/44000 | Loss: 2.9424 | LR: 3.47e-05 | 1337.3ms/step | 15.0 samples/s +Step 36670/44000 | Loss: 2.8768 | LR: 3.46e-05 | 1317.8ms/step | 15.2 samples/s +Step 36680/44000 | Loss: 2.8913 | LR: 3.45e-05 | 1334.1ms/step | 15.0 samples/s +Step 36690/44000 | Loss: 2.9066 | LR: 3.44e-05 | 1331.4ms/step | 15.0 samples/s +Step 36700/44000 | Loss: 2.8802 | LR: 3.43e-05 | 1357.5ms/step | 14.7 samples/s +Step 36710/44000 | Loss: 2.8674 | LR: 3.42e-05 | 1311.0ms/step | 15.3 samples/s +Step 36720/44000 | Loss: 2.8593 | LR: 3.41e-05 | 1298.4ms/step | 15.4 samples/s +Step 36730/44000 | Loss: 2.9713 | LR: 3.41e-05 | 1344.3ms/step | 14.9 samples/s +Step 36740/44000 | Loss: 2.8739 | LR: 3.40e-05 | 1384.0ms/step | 14.5 samples/s +Step 36750/44000 | Loss: 2.9029 | LR: 3.39e-05 | 1321.1ms/step | 15.1 samples/s +Validation at step 36750: Loss = 2.8703, Perplexity = 17.64 +Step 36760/44000 | Loss: 2.8866 | LR: 3.38e-05 | 1817.3ms/step | 11.0 samples/s +Step 36770/44000 | Loss: 3.0053 | LR: 3.37e-05 | 1290.6ms/step | 15.5 samples/s +Step 36780/44000 | Loss: 2.8745 | LR: 3.36e-05 | 1332.1ms/step | 15.0 samples/s +Step 36790/44000 | Loss: 2.9180 | LR: 3.35e-05 | 1325.7ms/step | 15.1 samples/s +Step 36800/44000 | Loss: 2.9395 | LR: 3.34e-05 | 1345.4ms/step | 14.9 samples/s +Step 36810/44000 | Loss: 2.9804 | LR: 3.33e-05 | 1370.0ms/step | 14.6 samples/s +Step 36820/44000 | Loss: 2.9393 | LR: 3.32e-05 | 1311.0ms/step | 15.3 samples/s +Step 36830/44000 | Loss: 2.8770 | LR: 3.31e-05 | 1245.6ms/step | 16.1 samples/s +Step 36840/44000 | Loss: 2.9276 | LR: 3.31e-05 | 1274.1ms/step | 15.7 samples/s +Step 36850/44000 | Loss: 2.9396 | LR: 3.30e-05 | 1265.2ms/step | 15.8 samples/s +Step 36860/44000 | Loss: 2.9594 | LR: 3.29e-05 | 1320.0ms/step | 15.2 samples/s +Step 36870/44000 | Loss: 2.8446 | LR: 3.28e-05 | 1288.7ms/step | 15.5 samples/s +Step 36880/44000 | Loss: 2.9253 | LR: 3.27e-05 | 1281.8ms/step | 15.6 samples/s +Step 36890/44000 | Loss: 2.9179 | LR: 3.26e-05 | 1307.1ms/step | 15.3 samples/s +Step 36900/44000 | Loss: 2.8991 | LR: 3.25e-05 | 1336.2ms/step | 15.0 samples/s +Step 36910/44000 | Loss: 2.9301 | LR: 3.24e-05 | 1312.8ms/step | 15.2 samples/s +Step 36920/44000 | Loss: 2.8864 | LR: 3.23e-05 | 1339.3ms/step | 14.9 samples/s +Step 36930/44000 | Loss: 2.9288 | LR: 3.22e-05 | 1294.5ms/step | 15.4 samples/s +Step 36940/44000 | Loss: 2.8652 | LR: 3.22e-05 | 1315.5ms/step | 15.2 samples/s +Step 36950/44000 | Loss: 2.9198 | LR: 3.21e-05 | 1315.5ms/step | 15.2 samples/s +Step 36960/44000 | Loss: 2.9635 | LR: 3.20e-05 | 1341.4ms/step | 14.9 samples/s +Step 36970/44000 | Loss: 2.8682 | LR: 3.19e-05 | 1315.9ms/step | 15.2 samples/s +Step 36980/44000 | Loss: 2.8904 | LR: 3.18e-05 | 1359.5ms/step | 14.7 samples/s +Step 36990/44000 | Loss: 3.0190 | LR: 3.17e-05 | 1318.8ms/step | 15.2 samples/s +Step 37000/44000 | Loss: 2.8963 | LR: 3.16e-05 | 1322.6ms/step | 15.1 samples/s +Validation at step 37000: Loss = 2.8701, Perplexity = 17.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-37000 +Step 37010/44000 | Loss: 2.9153 | LR: 3.15e-05 | 2339.2ms/step | 8.5 samples/s +Step 37020/44000 | Loss: 2.9580 | LR: 3.15e-05 | 1320.5ms/step | 15.1 samples/s +Step 37030/44000 | Loss: 2.9378 | LR: 3.14e-05 | 1289.3ms/step | 15.5 samples/s +Step 37040/44000 | Loss: 2.9245 | LR: 3.13e-05 | 1307.5ms/step | 15.3 samples/s +Step 37050/44000 | Loss: 2.9438 | LR: 3.12e-05 | 1353.1ms/step | 14.8 samples/s +Step 37060/44000 | Loss: 2.9036 | LR: 3.11e-05 | 1304.0ms/step | 15.3 samples/s +Step 37070/44000 | Loss: 2.9211 | LR: 3.10e-05 | 1291.7ms/step | 15.5 samples/s +Step 37080/44000 | Loss: 2.9509 | LR: 3.09e-05 | 1320.0ms/step | 15.2 samples/s +Step 37090/44000 | Loss: 2.8808 | LR: 3.08e-05 | 1343.2ms/step | 14.9 samples/s +Step 37100/44000 | Loss: 2.8949 | LR: 3.07e-05 | 1293.4ms/step | 15.5 samples/s +Step 37110/44000 | Loss: 2.8761 | LR: 3.07e-05 | 1304.7ms/step | 15.3 samples/s +Step 37120/44000 | Loss: 2.9639 | LR: 3.06e-05 | 1310.6ms/step | 15.3 samples/s +Step 37130/44000 | Loss: 2.8843 | LR: 3.05e-05 | 1305.1ms/step | 15.3 samples/s +Step 37140/44000 | Loss: 2.9618 | LR: 3.04e-05 | 1310.0ms/step | 15.3 samples/s +Step 37150/44000 | Loss: 2.9152 | LR: 3.03e-05 | 1278.9ms/step | 15.6 samples/s +Step 37160/44000 | Loss: 2.9231 | LR: 3.02e-05 | 1307.5ms/step | 15.3 samples/s +Step 37170/44000 | Loss: 2.8851 | LR: 3.01e-05 | 1276.3ms/step | 15.7 samples/s +Step 37180/44000 | Loss: 2.8933 | LR: 3.01e-05 | 1293.5ms/step | 15.5 samples/s +Step 37190/44000 | Loss: 2.9667 | LR: 3.00e-05 | 1321.5ms/step | 15.1 samples/s +Step 37200/44000 | Loss: 2.9347 | LR: 2.99e-05 | 1284.2ms/step | 15.6 samples/s +Step 37210/44000 | Loss: 2.9496 | LR: 2.98e-05 | 1283.8ms/step | 15.6 samples/s +Step 37220/44000 | Loss: 2.9129 | LR: 2.97e-05 | 1320.8ms/step | 15.1 samples/s +Step 37230/44000 | Loss: 2.9054 | LR: 2.96e-05 | 1299.9ms/step | 15.4 samples/s +Step 37240/44000 | Loss: 2.8684 | LR: 2.95e-05 | 1333.8ms/step | 15.0 samples/s +Step 37250/44000 | Loss: 2.9091 | LR: 2.95e-05 | 1276.4ms/step | 15.7 samples/s +Validation at step 37250: Loss = 2.8700, Perplexity = 17.64 +Step 37260/44000 | Loss: 2.9333 | LR: 2.94e-05 | 1809.1ms/step | 11.1 samples/s +Step 37270/44000 | Loss: 2.8647 | LR: 2.93e-05 | 1331.2ms/step | 15.0 samples/s +Step 37280/44000 | Loss: 2.8965 | LR: 2.92e-05 | 1304.8ms/step | 15.3 samples/s +Step 37290/44000 | Loss: 2.9571 | LR: 2.91e-05 | 1284.6ms/step | 15.6 samples/s +Step 37300/44000 | Loss: 2.9390 | LR: 2.90e-05 | 1273.2ms/step | 15.7 samples/s +Step 37310/44000 | Loss: 2.9656 | LR: 2.89e-05 | 1340.3ms/step | 14.9 samples/s +Step 37320/44000 | Loss: 2.9132 | LR: 2.89e-05 | 1317.8ms/step | 15.2 samples/s +Step 37330/44000 | Loss: 2.8705 | LR: 2.88e-05 | 1310.0ms/step | 15.3 samples/s +Step 37340/44000 | Loss: 2.9281 | LR: 2.87e-05 | 1335.4ms/step | 15.0 samples/s +Step 37350/44000 | Loss: 2.8855 | LR: 2.86e-05 | 1354.6ms/step | 14.8 samples/s +Step 37360/44000 | Loss: 2.9188 | LR: 2.85e-05 | 1296.5ms/step | 15.4 samples/s +Step 37370/44000 | Loss: 2.8885 | LR: 2.84e-05 | 1287.4ms/step | 15.5 samples/s +Step 37380/44000 | Loss: 2.8704 | LR: 2.84e-05 | 1352.4ms/step | 14.8 samples/s +Step 37390/44000 | Loss: 2.8906 | LR: 2.83e-05 | 1362.4ms/step | 14.7 samples/s +Step 37400/44000 | Loss: 2.8828 | LR: 2.82e-05 | 1362.7ms/step | 14.7 samples/s +Step 37410/44000 | Loss: 2.8887 | LR: 2.81e-05 | 1387.3ms/step | 14.4 samples/s +Step 37420/44000 | Loss: 2.8455 | LR: 2.80e-05 | 1327.9ms/step | 15.1 samples/s +Step 37430/44000 | Loss: 2.8741 | LR: 2.79e-05 | 1334.9ms/step | 15.0 samples/s +Step 37440/44000 | Loss: 2.9184 | LR: 2.78e-05 | 1326.0ms/step | 15.1 samples/s +Step 37450/44000 | Loss: 2.9475 | LR: 2.78e-05 | 1249.6ms/step | 16.0 samples/s +Step 37460/44000 | Loss: 2.9239 | LR: 2.77e-05 | 1334.4ms/step | 15.0 samples/s +Step 37470/44000 | Loss: 2.9302 | LR: 2.76e-05 | 1333.4ms/step | 15.0 samples/s +Step 37480/44000 | Loss: 2.9899 | LR: 2.75e-05 | 1321.0ms/step | 15.1 samples/s +Step 37490/44000 | Loss: 2.9370 | LR: 2.74e-05 | 1373.2ms/step | 14.6 samples/s +Step 37500/44000 | Loss: 2.9009 | LR: 2.74e-05 | 1316.2ms/step | 15.2 samples/s +Validation at step 37500: Loss = 2.8699, Perplexity = 17.64 +Step 37510/44000 | Loss: 2.9904 | LR: 2.73e-05 | 1882.5ms/step | 10.6 samples/s +Step 37520/44000 | Loss: 2.9055 | LR: 2.72e-05 | 1352.9ms/step | 14.8 samples/s +Step 37530/44000 | Loss: 2.9679 | LR: 2.71e-05 | 1320.8ms/step | 15.1 samples/s +Step 37540/44000 | Loss: 2.8712 | LR: 2.70e-05 | 1361.0ms/step | 14.7 samples/s +Step 37550/44000 | Loss: 2.9954 | LR: 2.69e-05 | 1314.4ms/step | 15.2 samples/s +Step 37560/44000 | Loss: 2.8672 | LR: 2.69e-05 | 1317.1ms/step | 15.2 samples/s +Step 37570/44000 | Loss: 2.9300 | LR: 2.68e-05 | 1290.0ms/step | 15.5 samples/s +Step 37580/44000 | Loss: 2.8609 | LR: 2.67e-05 | 1298.3ms/step | 15.4 samples/s +Step 37590/44000 | Loss: 2.8822 | LR: 2.66e-05 | 1332.1ms/step | 15.0 samples/s +Step 37600/44000 | Loss: 2.9079 | LR: 2.65e-05 | 1289.6ms/step | 15.5 samples/s +Step 37610/44000 | Loss: 2.8833 | LR: 2.65e-05 | 1287.4ms/step | 15.5 samples/s +Step 37620/44000 | Loss: 2.9074 | LR: 2.64e-05 | 1307.8ms/step | 15.3 samples/s +Step 37630/44000 | Loss: 2.9541 | LR: 2.63e-05 | 1316.4ms/step | 15.2 samples/s +Step 37640/44000 | Loss: 2.8709 | LR: 2.62e-05 | 1295.0ms/step | 15.4 samples/s +Step 37650/44000 | Loss: 2.9412 | LR: 2.61e-05 | 1366.9ms/step | 14.6 samples/s +Step 37660/44000 | Loss: 2.8756 | LR: 2.60e-05 | 1317.5ms/step | 15.2 samples/s +Step 37670/44000 | Loss: 2.9246 | LR: 2.60e-05 | 1353.2ms/step | 14.8 samples/s +Step 37680/44000 | Loss: 2.8687 | LR: 2.59e-05 | 1353.2ms/step | 14.8 samples/s +Step 37690/44000 | Loss: 2.8441 | LR: 2.58e-05 | 1372.8ms/step | 14.6 samples/s +Step 37700/44000 | Loss: 2.8873 | LR: 2.57e-05 | 1326.5ms/step | 15.1 samples/s +Step 37710/44000 | Loss: 2.9413 | LR: 2.56e-05 | 1347.5ms/step | 14.8 samples/s +Step 37720/44000 | Loss: 2.9168 | LR: 2.56e-05 | 1268.5ms/step | 15.8 samples/s +Step 37730/44000 | Loss: 2.9178 | LR: 2.55e-05 | 1307.3ms/step | 15.3 samples/s +Step 37740/44000 | Loss: 2.9432 | LR: 2.54e-05 | 1293.2ms/step | 15.5 samples/s +Step 37750/44000 | Loss: 2.9185 | LR: 2.53e-05 | 1279.2ms/step | 15.6 samples/s +Validation at step 37750: Loss = 2.8698, Perplexity = 17.63 +Step 37760/44000 | Loss: 2.8424 | LR: 2.52e-05 | 1782.9ms/step | 11.2 samples/s +Step 37770/44000 | Loss: 2.9061 | LR: 2.52e-05 | 1269.0ms/step | 15.8 samples/s +Step 37780/44000 | Loss: 2.8528 | LR: 2.51e-05 | 1309.1ms/step | 15.3 samples/s +Step 37790/44000 | Loss: 2.8914 | LR: 2.50e-05 | 1280.3ms/step | 15.6 samples/s +Step 37800/44000 | Loss: 2.9271 | LR: 2.49e-05 | 1324.1ms/step | 15.1 samples/s +Step 37810/44000 | Loss: 2.9152 | LR: 2.48e-05 | 1312.3ms/step | 15.2 samples/s +Step 37820/44000 | Loss: 2.9494 | LR: 2.48e-05 | 1372.5ms/step | 14.6 samples/s +Step 37830/44000 | Loss: 2.8736 | LR: 2.47e-05 | 1318.5ms/step | 15.2 samples/s +Step 37840/44000 | Loss: 2.9283 | LR: 2.46e-05 | 1303.4ms/step | 15.3 samples/s +Step 37850/44000 | Loss: 2.8803 | LR: 2.45e-05 | 1340.2ms/step | 14.9 samples/s +Step 37860/44000 | Loss: 2.9888 | LR: 2.45e-05 | 1289.4ms/step | 15.5 samples/s +Step 37870/44000 | Loss: 2.9286 | LR: 2.44e-05 | 1305.3ms/step | 15.3 samples/s +Step 37880/44000 | Loss: 2.8331 | LR: 2.43e-05 | 1306.9ms/step | 15.3 samples/s +Step 37890/44000 | Loss: 2.8773 | LR: 2.42e-05 | 1289.3ms/step | 15.5 samples/s +Step 37900/44000 | Loss: 2.8931 | LR: 2.41e-05 | 1263.1ms/step | 15.8 samples/s +Step 37910/44000 | Loss: 2.8821 | LR: 2.41e-05 | 1305.2ms/step | 15.3 samples/s +Step 37920/44000 | Loss: 2.9521 | LR: 2.40e-05 | 1304.9ms/step | 15.3 samples/s +Step 37930/44000 | Loss: 2.9633 | LR: 2.39e-05 | 1347.6ms/step | 14.8 samples/s +Step 37940/44000 | Loss: 2.9149 | LR: 2.38e-05 | 1320.5ms/step | 15.1 samples/s +Step 37950/44000 | Loss: 2.8847 | LR: 2.38e-05 | 1295.5ms/step | 15.4 samples/s +Step 37960/44000 | Loss: 2.9002 | LR: 2.37e-05 | 1283.6ms/step | 15.6 samples/s +Step 37970/44000 | Loss: 2.9286 | LR: 2.36e-05 | 1327.3ms/step | 15.1 samples/s +Step 37980/44000 | Loss: 2.9666 | LR: 2.35e-05 | 1331.6ms/step | 15.0 samples/s +Step 37990/44000 | Loss: 2.9138 | LR: 2.34e-05 | 1352.8ms/step | 14.8 samples/s +Step 38000/44000 | Loss: 3.0426 | LR: 2.34e-05 | 1280.1ms/step | 15.6 samples/s +Validation at step 38000: Loss = 2.8698, Perplexity = 17.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-38000 +Step 38010/44000 | Loss: 2.8792 | LR: 2.33e-05 | 2300.1ms/step | 8.7 samples/s +Step 38020/44000 | Loss: 2.9088 | LR: 2.32e-05 | 1320.9ms/step | 15.1 samples/s +Step 38030/44000 | Loss: 2.8936 | LR: 2.31e-05 | 1263.1ms/step | 15.8 samples/s +Step 38040/44000 | Loss: 2.9059 | LR: 2.31e-05 | 1300.4ms/step | 15.4 samples/s +Step 38050/44000 | Loss: 2.9181 | LR: 2.30e-05 | 1226.3ms/step | 16.3 samples/s +Step 38060/44000 | Loss: 2.9244 | LR: 2.29e-05 | 1274.7ms/step | 15.7 samples/s +Step 38070/44000 | Loss: 2.9140 | LR: 2.28e-05 | 1304.1ms/step | 15.3 samples/s +Step 38080/44000 | Loss: 2.8922 | LR: 2.28e-05 | 1257.8ms/step | 15.9 samples/s +Step 38090/44000 | Loss: 2.9787 | LR: 2.27e-05 | 1283.4ms/step | 15.6 samples/s +Step 38100/44000 | Loss: 2.9698 | LR: 2.26e-05 | 1263.8ms/step | 15.8 samples/s +Step 38110/44000 | Loss: 2.9906 | LR: 2.25e-05 | 1252.5ms/step | 16.0 samples/s +Step 38120/44000 | Loss: 2.9791 | LR: 2.25e-05 | 1273.8ms/step | 15.7 samples/s +Step 38130/44000 | Loss: 2.9325 | LR: 2.24e-05 | 1279.3ms/step | 15.6 samples/s +Step 38140/44000 | Loss: 2.9148 | LR: 2.23e-05 | 1268.3ms/step | 15.8 samples/s +Step 38150/44000 | Loss: 2.9045 | LR: 2.22e-05 | 1273.5ms/step | 15.7 samples/s +Step 38160/44000 | Loss: 2.9025 | LR: 2.22e-05 | 1310.4ms/step | 15.3 samples/s +Step 38170/44000 | Loss: 2.8849 | LR: 2.21e-05 | 1257.1ms/step | 15.9 samples/s +Step 38180/44000 | Loss: 2.8468 | LR: 2.20e-05 | 1248.7ms/step | 16.0 samples/s +Step 38190/44000 | Loss: 2.8977 | LR: 2.19e-05 | 1278.5ms/step | 15.6 samples/s +Step 38200/44000 | Loss: 2.8740 | LR: 2.19e-05 | 1253.0ms/step | 16.0 samples/s +Step 38210/44000 | Loss: 2.9493 | LR: 2.18e-05 | 1278.7ms/step | 15.6 samples/s +Step 38220/44000 | Loss: 2.9058 | LR: 2.17e-05 | 1241.8ms/step | 16.1 samples/s +Step 38230/44000 | Loss: 2.8901 | LR: 2.16e-05 | 1269.2ms/step | 15.8 samples/s +Step 38240/44000 | Loss: 2.9093 | LR: 2.16e-05 | 1258.0ms/step | 15.9 samples/s +Step 38250/44000 | Loss: 2.8976 | LR: 2.15e-05 | 1242.1ms/step | 16.1 samples/s +Validation at step 38250: Loss = 2.8697, Perplexity = 17.63 +Step 38260/44000 | Loss: 2.8967 | LR: 2.14e-05 | 1761.2ms/step | 11.4 samples/s +Step 38270/44000 | Loss: 2.9510 | LR: 2.13e-05 | 1263.5ms/step | 15.8 samples/s +Step 38280/44000 | Loss: 2.8133 | LR: 2.13e-05 | 1278.9ms/step | 15.6 samples/s +Step 38290/44000 | Loss: 2.9445 | LR: 2.12e-05 | 1252.8ms/step | 16.0 samples/s +Step 38300/44000 | Loss: 2.8953 | LR: 2.11e-05 | 1273.6ms/step | 15.7 samples/s +Step 38310/44000 | Loss: 2.9942 | LR: 2.11e-05 | 1249.1ms/step | 16.0 samples/s +Step 38320/44000 | Loss: 2.9211 | LR: 2.10e-05 | 1303.8ms/step | 15.3 samples/s +Step 38330/44000 | Loss: 2.9150 | LR: 2.09e-05 | 1305.4ms/step | 15.3 samples/s +Step 38340/44000 | Loss: 2.9097 | LR: 2.08e-05 | 1276.5ms/step | 15.7 samples/s +Step 38350/44000 | Loss: 2.9064 | LR: 2.08e-05 | 1300.1ms/step | 15.4 samples/s +Step 38360/44000 | Loss: 2.9478 | LR: 2.07e-05 | 1254.5ms/step | 15.9 samples/s +Step 38370/44000 | Loss: 2.9064 | LR: 2.06e-05 | 1295.7ms/step | 15.4 samples/s +Step 38380/44000 | Loss: 2.9058 | LR: 2.05e-05 | 1309.1ms/step | 15.3 samples/s +Step 38390/44000 | Loss: 2.8824 | LR: 2.05e-05 | 1268.3ms/step | 15.8 samples/s +Step 38400/44000 | Loss: 2.8701 | LR: 2.04e-05 | 1301.5ms/step | 15.4 samples/s +Step 38410/44000 | Loss: 2.9266 | LR: 2.03e-05 | 1328.2ms/step | 15.1 samples/s +Step 38420/44000 | Loss: 2.9641 | LR: 2.03e-05 | 1266.8ms/step | 15.8 samples/s +Step 38430/44000 | Loss: 2.9292 | LR: 2.02e-05 | 1302.9ms/step | 15.4 samples/s +Step 38440/44000 | Loss: 2.9814 | LR: 2.01e-05 | 1277.8ms/step | 15.7 samples/s +Step 38450/44000 | Loss: 2.9575 | LR: 2.00e-05 | 1288.1ms/step | 15.5 samples/s +Step 38460/44000 | Loss: 2.9073 | LR: 2.00e-05 | 1242.6ms/step | 16.1 samples/s +Step 38470/44000 | Loss: 2.9195 | LR: 1.99e-05 | 1291.4ms/step | 15.5 samples/s +Step 38480/44000 | Loss: 2.8881 | LR: 1.98e-05 | 1303.5ms/step | 15.3 samples/s +Step 38490/44000 | Loss: 2.8621 | LR: 1.98e-05 | 1285.1ms/step | 15.6 samples/s +Step 38500/44000 | Loss: 2.8738 | LR: 1.97e-05 | 1278.8ms/step | 15.6 samples/s +Validation at step 38500: Loss = 2.8696, Perplexity = 17.63 +Step 38510/44000 | Loss: 2.9089 | LR: 1.96e-05 | 1829.7ms/step | 10.9 samples/s +Step 38520/44000 | Loss: 2.8865 | LR: 1.95e-05 | 1315.9ms/step | 15.2 samples/s +Step 38530/44000 | Loss: 3.0123 | LR: 1.95e-05 | 1267.6ms/step | 15.8 samples/s +Step 38540/44000 | Loss: 2.8498 | LR: 1.94e-05 | 1290.1ms/step | 15.5 samples/s +Step 38550/44000 | Loss: 2.9981 | LR: 1.93e-05 | 1249.8ms/step | 16.0 samples/s +Step 38560/44000 | Loss: 2.9029 | LR: 1.93e-05 | 1231.1ms/step | 16.2 samples/s +Step 38570/44000 | Loss: 2.9121 | LR: 1.92e-05 | 1271.9ms/step | 15.7 samples/s +Step 38580/44000 | Loss: 2.8987 | LR: 1.91e-05 | 1295.9ms/step | 15.4 samples/s +Step 38590/44000 | Loss: 2.8969 | LR: 1.91e-05 | 1216.6ms/step | 16.4 samples/s +Step 38600/44000 | Loss: 2.9637 | LR: 1.90e-05 | 1294.6ms/step | 15.4 samples/s +Step 38610/44000 | Loss: 2.8837 | LR: 1.89e-05 | 1262.3ms/step | 15.8 samples/s +Step 38620/44000 | Loss: 2.9591 | LR: 1.88e-05 | 1238.9ms/step | 16.1 samples/s +Step 38630/44000 | Loss: 2.9432 | LR: 1.88e-05 | 1287.8ms/step | 15.5 samples/s +Step 38640/44000 | Loss: 2.9692 | LR: 1.87e-05 | 1231.7ms/step | 16.2 samples/s +Step 38650/44000 | Loss: 2.8966 | LR: 1.86e-05 | 1262.9ms/step | 15.8 samples/s +Step 38660/44000 | Loss: 2.8751 | LR: 1.86e-05 | 1269.7ms/step | 15.8 samples/s +Step 38670/44000 | Loss: 2.9759 | LR: 1.85e-05 | 1233.1ms/step | 16.2 samples/s +Step 38680/44000 | Loss: 2.9135 | LR: 1.84e-05 | 1305.7ms/step | 15.3 samples/s +Step 38690/44000 | Loss: 2.9266 | LR: 1.84e-05 | 1218.0ms/step | 16.4 samples/s +Step 38700/44000 | Loss: 2.9278 | LR: 1.83e-05 | 1242.4ms/step | 16.1 samples/s +Step 38710/44000 | Loss: 2.9157 | LR: 1.82e-05 | 1262.8ms/step | 15.8 samples/s +Step 38720/44000 | Loss: 2.8907 | LR: 1.82e-05 | 1257.1ms/step | 15.9 samples/s +Step 38730/44000 | Loss: 2.8774 | LR: 1.81e-05 | 1282.1ms/step | 15.6 samples/s +Step 38740/44000 | Loss: 2.8946 | LR: 1.80e-05 | 1250.5ms/step | 16.0 samples/s +Step 38750/44000 | Loss: 2.8659 | LR: 1.80e-05 | 1269.0ms/step | 15.8 samples/s +Validation at step 38750: Loss = 2.8695, Perplexity = 17.63 +Step 38760/44000 | Loss: 2.8769 | LR: 1.79e-05 | 1802.7ms/step | 11.1 samples/s +Step 38770/44000 | Loss: 2.9024 | LR: 1.78e-05 | 1321.5ms/step | 15.1 samples/s +Step 38780/44000 | Loss: 2.9591 | LR: 1.78e-05 | 1356.6ms/step | 14.7 samples/s +Step 38790/44000 | Loss: 2.8911 | LR: 1.77e-05 | 1320.8ms/step | 15.1 samples/s +Step 38800/44000 | Loss: 2.9117 | LR: 1.76e-05 | 1287.8ms/step | 15.5 samples/s +Step 38810/44000 | Loss: 2.9388 | LR: 1.76e-05 | 1301.2ms/step | 15.4 samples/s +Step 38820/44000 | Loss: 2.8927 | LR: 1.75e-05 | 1295.0ms/step | 15.4 samples/s +Step 38830/44000 | Loss: 2.8703 | LR: 1.74e-05 | 1293.7ms/step | 15.5 samples/s +Step 38840/44000 | Loss: 2.9528 | LR: 1.74e-05 | 1284.3ms/step | 15.6 samples/s +Step 38850/44000 | Loss: 2.9332 | LR: 1.73e-05 | 1322.4ms/step | 15.1 samples/s +Step 38860/44000 | Loss: 2.9530 | LR: 1.72e-05 | 1324.7ms/step | 15.1 samples/s +Step 38870/44000 | Loss: 2.9399 | LR: 1.72e-05 | 1278.6ms/step | 15.6 samples/s +Step 38880/44000 | Loss: 2.9542 | LR: 1.71e-05 | 1290.6ms/step | 15.5 samples/s +Step 38890/44000 | Loss: 2.9119 | LR: 1.70e-05 | 1347.1ms/step | 14.8 samples/s +Step 38900/44000 | Loss: 2.8704 | LR: 1.70e-05 | 1342.3ms/step | 14.9 samples/s +Step 38910/44000 | Loss: 2.8916 | LR: 1.69e-05 | 1289.1ms/step | 15.5 samples/s +Step 38920/44000 | Loss: 2.8317 | LR: 1.68e-05 | 1324.5ms/step | 15.1 samples/s +Step 38930/44000 | Loss: 2.8738 | LR: 1.68e-05 | 1286.1ms/step | 15.6 samples/s +Step 38940/44000 | Loss: 2.9261 | LR: 1.67e-05 | 1288.7ms/step | 15.5 samples/s +Step 38950/44000 | Loss: 2.9128 | LR: 1.66e-05 | 1388.7ms/step | 14.4 samples/s +Step 38960/44000 | Loss: 3.0227 | LR: 1.66e-05 | 1291.5ms/step | 15.5 samples/s +Step 38970/44000 | Loss: 2.8999 | LR: 1.65e-05 | 1315.4ms/step | 15.2 samples/s +Step 38980/44000 | Loss: 2.9348 | LR: 1.64e-05 | 1292.8ms/step | 15.5 samples/s +Step 38990/44000 | Loss: 2.9296 | LR: 1.64e-05 | 1324.1ms/step | 15.1 samples/s +Step 39000/44000 | Loss: 2.9621 | LR: 1.63e-05 | 1292.0ms/step | 15.5 samples/s +Validation at step 39000: Loss = 2.8695, Perplexity = 17.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-39000 +Step 39010/44000 | Loss: 2.8500 | LR: 1.62e-05 | 2305.0ms/step | 8.7 samples/s +Step 39020/44000 | Loss: 2.9245 | LR: 1.62e-05 | 1250.7ms/step | 16.0 samples/s +Step 39030/44000 | Loss: 2.9992 | LR: 1.61e-05 | 1244.2ms/step | 16.1 samples/s +Step 39040/44000 | Loss: 2.8510 | LR: 1.61e-05 | 1358.1ms/step | 14.7 samples/s +Step 39050/44000 | Loss: 2.9137 | LR: 1.60e-05 | 1298.4ms/step | 15.4 samples/s +Step 39060/44000 | Loss: 2.9965 | LR: 1.59e-05 | 1311.8ms/step | 15.2 samples/s +Step 39070/44000 | Loss: 2.9847 | LR: 1.59e-05 | 1378.4ms/step | 14.5 samples/s +Step 39080/44000 | Loss: 2.8983 | LR: 1.58e-05 | 1298.5ms/step | 15.4 samples/s +Step 39090/44000 | Loss: 2.9529 | LR: 1.57e-05 | 1351.9ms/step | 14.8 samples/s +Step 39100/44000 | Loss: 2.8882 | LR: 1.57e-05 | 1297.5ms/step | 15.4 samples/s +Step 39110/44000 | Loss: 2.9260 | LR: 1.56e-05 | 1341.8ms/step | 14.9 samples/s +Step 39120/44000 | Loss: 2.9563 | LR: 1.55e-05 | 1310.1ms/step | 15.3 samples/s +Step 39130/44000 | Loss: 2.9864 | LR: 1.55e-05 | 1289.9ms/step | 15.5 samples/s +Step 39140/44000 | Loss: 2.9171 | LR: 1.54e-05 | 1242.4ms/step | 16.1 samples/s +Step 39150/44000 | Loss: 2.9633 | LR: 1.54e-05 | 1321.1ms/step | 15.1 samples/s +Step 39160/44000 | Loss: 2.8684 | LR: 1.53e-05 | 1315.8ms/step | 15.2 samples/s +Step 39170/44000 | Loss: 2.9657 | LR: 1.52e-05 | 1299.9ms/step | 15.4 samples/s +Step 39180/44000 | Loss: 2.8604 | LR: 1.52e-05 | 1263.3ms/step | 15.8 samples/s +Step 39190/44000 | Loss: 2.9108 | LR: 1.51e-05 | 1352.9ms/step | 14.8 samples/s +Step 39200/44000 | Loss: 2.9229 | LR: 1.50e-05 | 1289.3ms/step | 15.5 samples/s +Step 39210/44000 | Loss: 2.9436 | LR: 1.50e-05 | 1310.6ms/step | 15.3 samples/s +Step 39220/44000 | Loss: 2.9017 | LR: 1.49e-05 | 1247.7ms/step | 16.0 samples/s +Step 39230/44000 | Loss: 2.9045 | LR: 1.49e-05 | 1252.4ms/step | 16.0 samples/s +Step 39240/44000 | Loss: 2.9791 | LR: 1.48e-05 | 1304.6ms/step | 15.3 samples/s +Step 39250/44000 | Loss: 2.9257 | LR: 1.47e-05 | 1274.4ms/step | 15.7 samples/s +Validation at step 39250: Loss = 2.8695, Perplexity = 17.63 +Step 39260/44000 | Loss: 2.8963 | LR: 1.47e-05 | 1765.7ms/step | 11.3 samples/s +Step 39270/44000 | Loss: 2.9255 | LR: 1.46e-05 | 1253.2ms/step | 16.0 samples/s +Step 39280/44000 | Loss: 2.9022 | LR: 1.46e-05 | 1257.7ms/step | 15.9 samples/s +Step 39290/44000 | Loss: 2.8544 | LR: 1.45e-05 | 1279.3ms/step | 15.6 samples/s +Step 39300/44000 | Loss: 2.8852 | LR: 1.44e-05 | 1304.7ms/step | 15.3 samples/s +Step 39310/44000 | Loss: 2.9167 | LR: 1.44e-05 | 1289.5ms/step | 15.5 samples/s +Step 39320/44000 | Loss: 2.9075 | LR: 1.43e-05 | 1330.0ms/step | 15.0 samples/s +Step 39330/44000 | Loss: 2.9201 | LR: 1.42e-05 | 1306.7ms/step | 15.3 samples/s +Step 39340/44000 | Loss: 2.9088 | LR: 1.42e-05 | 1289.5ms/step | 15.5 samples/s +Step 39350/44000 | Loss: 2.8840 | LR: 1.41e-05 | 1268.4ms/step | 15.8 samples/s +Step 39360/44000 | Loss: 2.9083 | LR: 1.41e-05 | 1325.8ms/step | 15.1 samples/s +Step 39370/44000 | Loss: 3.0054 | LR: 1.40e-05 | 1289.1ms/step | 15.5 samples/s +Step 39380/44000 | Loss: 2.8583 | LR: 1.39e-05 | 1263.1ms/step | 15.8 samples/s +Step 39390/44000 | Loss: 2.9579 | LR: 1.39e-05 | 1273.5ms/step | 15.7 samples/s +Step 39400/44000 | Loss: 2.9087 | LR: 1.38e-05 | 1310.3ms/step | 15.3 samples/s +Step 39410/44000 | Loss: 2.9352 | LR: 1.38e-05 | 1269.3ms/step | 15.8 samples/s +Step 39420/44000 | Loss: 2.9148 | LR: 1.37e-05 | 1304.0ms/step | 15.3 samples/s +Step 39430/44000 | Loss: 2.9023 | LR: 1.36e-05 | 1300.0ms/step | 15.4 samples/s +Step 39440/44000 | Loss: 2.9012 | LR: 1.36e-05 | 1304.1ms/step | 15.3 samples/s +Step 39450/44000 | Loss: 2.9003 | LR: 1.35e-05 | 1279.4ms/step | 15.6 samples/s +Step 39460/44000 | Loss: 2.9774 | LR: 1.35e-05 | 1305.0ms/step | 15.3 samples/s +Step 39470/44000 | Loss: 2.9498 | LR: 1.34e-05 | 1283.8ms/step | 15.6 samples/s +Step 39480/44000 | Loss: 2.9703 | LR: 1.34e-05 | 1257.9ms/step | 15.9 samples/s +Step 39490/44000 | Loss: 2.9078 | LR: 1.33e-05 | 1263.0ms/step | 15.8 samples/s +Step 39500/44000 | Loss: 2.9709 | LR: 1.32e-05 | 1210.3ms/step | 16.5 samples/s +Validation at step 39500: Loss = 2.8695, Perplexity = 17.63 +Step 39510/44000 | Loss: 2.8798 | LR: 1.32e-05 | 1735.7ms/step | 11.5 samples/s +Step 39520/44000 | Loss: 2.9339 | LR: 1.31e-05 | 1233.3ms/step | 16.2 samples/s +Step 39530/44000 | Loss: 2.9983 | LR: 1.31e-05 | 1299.6ms/step | 15.4 samples/s +Step 39540/44000 | Loss: 2.9413 | LR: 1.30e-05 | 1256.6ms/step | 15.9 samples/s +Step 39550/44000 | Loss: 2.8756 | LR: 1.29e-05 | 1251.8ms/step | 16.0 samples/s +Step 39560/44000 | Loss: 2.9665 | LR: 1.29e-05 | 1290.9ms/step | 15.5 samples/s +Step 39570/44000 | Loss: 2.8561 | LR: 1.28e-05 | 1295.0ms/step | 15.4 samples/s +Step 39580/44000 | Loss: 2.8791 | LR: 1.28e-05 | 1278.2ms/step | 15.6 samples/s +Step 39590/44000 | Loss: 2.9822 | LR: 1.27e-05 | 1239.0ms/step | 16.1 samples/s +Step 39600/44000 | Loss: 2.8463 | LR: 1.27e-05 | 1237.2ms/step | 16.2 samples/s +Step 39610/44000 | Loss: 2.9229 | LR: 1.26e-05 | 1240.3ms/step | 16.1 samples/s +Step 39620/44000 | Loss: 2.8954 | LR: 1.25e-05 | 1268.8ms/step | 15.8 samples/s +Step 39630/44000 | Loss: 2.9772 | LR: 1.25e-05 | 1296.3ms/step | 15.4 samples/s +Step 39640/44000 | Loss: 2.9602 | LR: 1.24e-05 | 1229.4ms/step | 16.3 samples/s +Step 39650/44000 | Loss: 2.8832 | LR: 1.24e-05 | 1248.0ms/step | 16.0 samples/s +Step 39660/44000 | Loss: 2.9053 | LR: 1.23e-05 | 1279.8ms/step | 15.6 samples/s +Step 39670/44000 | Loss: 3.0152 | LR: 1.23e-05 | 1237.0ms/step | 16.2 samples/s +Step 39680/44000 | Loss: 2.9025 | LR: 1.22e-05 | 1274.3ms/step | 15.7 samples/s +Step 39690/44000 | Loss: 2.9763 | LR: 1.22e-05 | 1310.3ms/step | 15.3 samples/s +Step 39700/44000 | Loss: 2.9039 | LR: 1.21e-05 | 1264.0ms/step | 15.8 samples/s +Step 39710/44000 | Loss: 2.8755 | LR: 1.20e-05 | 1336.6ms/step | 15.0 samples/s +Step 39720/44000 | Loss: 2.9543 | LR: 1.20e-05 | 1255.7ms/step | 15.9 samples/s +Step 39730/44000 | Loss: 2.9141 | LR: 1.19e-05 | 1240.5ms/step | 16.1 samples/s +Step 39740/44000 | Loss: 2.9285 | LR: 1.19e-05 | 1217.4ms/step | 16.4 samples/s +Step 39750/44000 | Loss: 2.8497 | LR: 1.18e-05 | 1269.0ms/step | 15.8 samples/s +Validation at step 39750: Loss = 2.8695, Perplexity = 17.63 +Step 39760/44000 | Loss: 2.9087 | LR: 1.18e-05 | 1721.2ms/step | 11.6 samples/s +Step 39770/44000 | Loss: 2.9232 | LR: 1.17e-05 | 1255.4ms/step | 15.9 samples/s +Step 39780/44000 | Loss: 2.8997 | LR: 1.17e-05 | 1279.0ms/step | 15.6 samples/s +Step 39790/44000 | Loss: 2.8590 | LR: 1.16e-05 | 1260.5ms/step | 15.9 samples/s +Step 39800/44000 | Loss: 2.8997 | LR: 1.15e-05 | 1273.7ms/step | 15.7 samples/s +Step 39810/44000 | Loss: 2.9081 | LR: 1.15e-05 | 1290.9ms/step | 15.5 samples/s +Step 39820/44000 | Loss: 2.8816 | LR: 1.14e-05 | 1279.0ms/step | 15.6 samples/s +Step 39830/44000 | Loss: 2.9197 | LR: 1.14e-05 | 1302.0ms/step | 15.4 samples/s +Step 39840/44000 | Loss: 2.8749 | LR: 1.13e-05 | 1282.6ms/step | 15.6 samples/s +Step 39850/44000 | Loss: 2.8955 | LR: 1.13e-05 | 1269.3ms/step | 15.8 samples/s +Step 39860/44000 | Loss: 2.9371 | LR: 1.12e-05 | 1268.3ms/step | 15.8 samples/s +Step 39870/44000 | Loss: 2.9409 | LR: 1.12e-05 | 1253.0ms/step | 16.0 samples/s +Step 39880/44000 | Loss: 2.9949 | LR: 1.11e-05 | 1314.8ms/step | 15.2 samples/s +Step 39890/44000 | Loss: 2.9242 | LR: 1.11e-05 | 1260.4ms/step | 15.9 samples/s +Step 39900/44000 | Loss: 2.9459 | LR: 1.10e-05 | 1245.7ms/step | 16.1 samples/s +Step 39910/44000 | Loss: 2.8446 | LR: 1.10e-05 | 1232.8ms/step | 16.2 samples/s +Step 39920/44000 | Loss: 2.9425 | LR: 1.09e-05 | 1268.6ms/step | 15.8 samples/s +Step 39930/44000 | Loss: 2.9542 | LR: 1.08e-05 | 1273.7ms/step | 15.7 samples/s +Step 39940/44000 | Loss: 2.9566 | LR: 1.08e-05 | 1273.6ms/step | 15.7 samples/s +Step 39950/44000 | Loss: 2.9065 | LR: 1.07e-05 | 1302.1ms/step | 15.4 samples/s +Step 39960/44000 | Loss: 2.9756 | LR: 1.07e-05 | 1252.1ms/step | 16.0 samples/s +Step 39970/44000 | Loss: 2.9149 | LR: 1.06e-05 | 1278.7ms/step | 15.6 samples/s +Step 39980/44000 | Loss: 2.9048 | LR: 1.06e-05 | 1223.5ms/step | 16.3 samples/s +Step 39990/44000 | Loss: 2.9309 | LR: 1.05e-05 | 1244.2ms/step | 16.1 samples/s +Step 40000/44000 | Loss: 2.8601 | LR: 1.05e-05 | 1258.3ms/step | 15.9 samples/s +Validation at step 40000: Loss = 2.8694, Perplexity = 17.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-40000 +Step 40010/44000 | Loss: 2.9560 | LR: 1.04e-05 | 2190.8ms/step | 9.1 samples/s +Step 40020/44000 | Loss: 2.9301 | LR: 1.04e-05 | 1257.6ms/step | 15.9 samples/s +Step 40030/44000 | Loss: 2.8551 | LR: 1.03e-05 | 1268.6ms/step | 15.8 samples/s +Step 40040/44000 | Loss: 2.8986 | LR: 1.03e-05 | 1231.3ms/step | 16.2 samples/s +Step 40050/44000 | Loss: 2.9092 | LR: 1.02e-05 | 1326.8ms/step | 15.1 samples/s +Step 40060/44000 | Loss: 2.9939 | LR: 1.02e-05 | 1310.5ms/step | 15.3 samples/s +Step 40070/44000 | Loss: 2.8909 | LR: 1.01e-05 | 1265.5ms/step | 15.8 samples/s +Step 40080/44000 | Loss: 2.9051 | LR: 1.01e-05 | 1239.7ms/step | 16.1 samples/s +Step 40090/44000 | Loss: 2.9307 | LR: 1.00e-05 | 1263.3ms/step | 15.8 samples/s +Step 40100/44000 | Loss: 2.9462 | LR: 9.96e-06 | 1320.9ms/step | 15.1 samples/s +Step 40110/44000 | Loss: 2.8787 | LR: 9.91e-06 | 1299.8ms/step | 15.4 samples/s +Step 40120/44000 | Loss: 2.9604 | LR: 9.86e-06 | 1311.3ms/step | 15.3 samples/s +Step 40130/44000 | Loss: 2.9788 | LR: 9.81e-06 | 1251.8ms/step | 16.0 samples/s +Step 40140/44000 | Loss: 2.9151 | LR: 9.76e-06 | 1268.7ms/step | 15.8 samples/s +Step 40150/44000 | Loss: 2.9275 | LR: 9.71e-06 | 1253.2ms/step | 16.0 samples/s +Step 40160/44000 | Loss: 2.8744 | LR: 9.66e-06 | 1320.3ms/step | 15.1 samples/s +Step 40170/44000 | Loss: 2.9654 | LR: 9.61e-06 | 1309.0ms/step | 15.3 samples/s +Step 40180/44000 | Loss: 2.8870 | LR: 9.56e-06 | 1288.2ms/step | 15.5 samples/s +Step 40190/44000 | Loss: 2.9186 | LR: 9.51e-06 | 1271.4ms/step | 15.7 samples/s +Step 40200/44000 | Loss: 2.8978 | LR: 9.46e-06 | 1282.2ms/step | 15.6 samples/s +Step 40210/44000 | Loss: 2.9245 | LR: 9.41e-06 | 1290.3ms/step | 15.5 samples/s +Step 40220/44000 | Loss: 2.9814 | LR: 9.36e-06 | 1257.7ms/step | 15.9 samples/s +Step 40230/44000 | Loss: 2.9704 | LR: 9.32e-06 | 1329.8ms/step | 15.0 samples/s +Step 40240/44000 | Loss: 2.9183 | LR: 9.27e-06 | 1284.0ms/step | 15.6 samples/s +Step 40250/44000 | Loss: 2.8544 | LR: 9.22e-06 | 1259.8ms/step | 15.9 samples/s +Validation at step 40250: Loss = 2.8695, Perplexity = 17.63 +Step 40260/44000 | Loss: 2.9201 | LR: 9.17e-06 | 1754.0ms/step | 11.4 samples/s +Step 40270/44000 | Loss: 2.9593 | LR: 9.12e-06 | 1306.1ms/step | 15.3 samples/s +Step 40280/44000 | Loss: 2.8994 | LR: 9.07e-06 | 1305.0ms/step | 15.3 samples/s +Step 40290/44000 | Loss: 2.9374 | LR: 9.02e-06 | 1278.9ms/step | 15.6 samples/s +Step 40300/44000 | Loss: 2.8807 | LR: 8.97e-06 | 1264.0ms/step | 15.8 samples/s +Step 40310/44000 | Loss: 2.9450 | LR: 8.93e-06 | 1268.7ms/step | 15.8 samples/s +Step 40320/44000 | Loss: 2.9524 | LR: 8.88e-06 | 1273.8ms/step | 15.7 samples/s +Step 40330/44000 | Loss: 2.9042 | LR: 8.83e-06 | 1289.7ms/step | 15.5 samples/s +Step 40340/44000 | Loss: 2.9214 | LR: 8.78e-06 | 1278.8ms/step | 15.6 samples/s +Step 40350/44000 | Loss: 2.9217 | LR: 8.74e-06 | 1237.0ms/step | 16.2 samples/s +Step 40360/44000 | Loss: 2.9129 | LR: 8.69e-06 | 1245.2ms/step | 16.1 samples/s +Step 40370/44000 | Loss: 2.8862 | LR: 8.64e-06 | 1234.6ms/step | 16.2 samples/s +Step 40380/44000 | Loss: 2.9179 | LR: 8.59e-06 | 1294.6ms/step | 15.4 samples/s +Step 40390/44000 | Loss: 2.8936 | LR: 8.55e-06 | 1274.1ms/step | 15.7 samples/s +Step 40400/44000 | Loss: 2.8985 | LR: 8.50e-06 | 1322.3ms/step | 15.1 samples/s +Step 40410/44000 | Loss: 2.9153 | LR: 8.45e-06 | 1282.8ms/step | 15.6 samples/s +Step 40420/44000 | Loss: 2.9290 | LR: 8.41e-06 | 1256.6ms/step | 15.9 samples/s +Step 40430/44000 | Loss: 2.9130 | LR: 8.36e-06 | 1254.6ms/step | 15.9 samples/s +Step 40440/44000 | Loss: 2.9062 | LR: 8.31e-06 | 1260.5ms/step | 15.9 samples/s +Step 40450/44000 | Loss: 2.8870 | LR: 8.27e-06 | 1213.8ms/step | 16.5 samples/s +Step 40460/44000 | Loss: 2.9049 | LR: 8.22e-06 | 1255.4ms/step | 15.9 samples/s +Step 40470/44000 | Loss: 2.9736 | LR: 8.17e-06 | 1292.3ms/step | 15.5 samples/s +Step 40480/44000 | Loss: 2.8659 | LR: 8.13e-06 | 1249.1ms/step | 16.0 samples/s +Step 40490/44000 | Loss: 2.9564 | LR: 8.08e-06 | 1283.4ms/step | 15.6 samples/s +Step 40500/44000 | Loss: 2.9203 | LR: 8.04e-06 | 1253.2ms/step | 16.0 samples/s +Validation at step 40500: Loss = 2.8694, Perplexity = 17.63 +Step 40510/44000 | Loss: 2.8775 | LR: 7.99e-06 | 1712.9ms/step | 11.7 samples/s +Step 40520/44000 | Loss: 2.8462 | LR: 7.94e-06 | 1232.7ms/step | 16.2 samples/s +Step 40530/44000 | Loss: 2.9827 | LR: 7.90e-06 | 1254.8ms/step | 15.9 samples/s +Step 40540/44000 | Loss: 2.9952 | LR: 7.85e-06 | 1280.1ms/step | 15.6 samples/s +Step 40550/44000 | Loss: 2.9382 | LR: 7.81e-06 | 1266.0ms/step | 15.8 samples/s +Step 40560/44000 | Loss: 2.9916 | LR: 7.76e-06 | 1289.2ms/step | 15.5 samples/s +Step 40570/44000 | Loss: 2.9529 | LR: 7.72e-06 | 1288.7ms/step | 15.5 samples/s +Step 40580/44000 | Loss: 2.9411 | LR: 7.67e-06 | 1287.9ms/step | 15.5 samples/s +Step 40590/44000 | Loss: 2.9407 | LR: 7.63e-06 | 1250.5ms/step | 16.0 samples/s +Step 40600/44000 | Loss: 2.9165 | LR: 7.59e-06 | 1263.0ms/step | 15.8 samples/s +Step 40610/44000 | Loss: 2.9089 | LR: 7.54e-06 | 1270.4ms/step | 15.7 samples/s +Step 40620/44000 | Loss: 2.9464 | LR: 7.50e-06 | 1282.4ms/step | 15.6 samples/s +Step 40630/44000 | Loss: 2.7993 | LR: 7.45e-06 | 1284.9ms/step | 15.6 samples/s +Step 40640/44000 | Loss: 2.9090 | LR: 7.41e-06 | 1248.9ms/step | 16.0 samples/s +Step 40650/44000 | Loss: 2.9616 | LR: 7.37e-06 | 1272.9ms/step | 15.7 samples/s +Step 40660/44000 | Loss: 2.8409 | LR: 7.32e-06 | 1242.3ms/step | 16.1 samples/s +Step 40670/44000 | Loss: 2.8936 | LR: 7.28e-06 | 1247.6ms/step | 16.0 samples/s +Step 40680/44000 | Loss: 2.9145 | LR: 7.23e-06 | 1258.7ms/step | 15.9 samples/s +Step 40690/44000 | Loss: 2.9123 | LR: 7.19e-06 | 1253.0ms/step | 16.0 samples/s +Step 40700/44000 | Loss: 2.9892 | LR: 7.15e-06 | 1237.1ms/step | 16.2 samples/s +Step 40710/44000 | Loss: 2.9064 | LR: 7.10e-06 | 1226.0ms/step | 16.3 samples/s +Step 40720/44000 | Loss: 2.9325 | LR: 7.06e-06 | 1263.3ms/step | 15.8 samples/s +Step 40730/44000 | Loss: 2.9298 | LR: 7.02e-06 | 1277.6ms/step | 15.7 samples/s +Step 40740/44000 | Loss: 2.8948 | LR: 6.98e-06 | 1318.0ms/step | 15.2 samples/s +Step 40750/44000 | Loss: 2.9459 | LR: 6.93e-06 | 1290.2ms/step | 15.5 samples/s +Validation at step 40750: Loss = 2.8694, Perplexity = 17.63 +Step 40760/44000 | Loss: 2.8484 | LR: 6.89e-06 | 1739.9ms/step | 11.5 samples/s +Step 40770/44000 | Loss: 2.8611 | LR: 6.85e-06 | 1292.1ms/step | 15.5 samples/s +Step 40780/44000 | Loss: 2.9098 | LR: 6.81e-06 | 1276.8ms/step | 15.7 samples/s +Step 40790/44000 | Loss: 2.9454 | LR: 6.77e-06 | 1284.5ms/step | 15.6 samples/s +Step 40800/44000 | Loss: 2.8684 | LR: 6.72e-06 | 1242.3ms/step | 16.1 samples/s +Step 40810/44000 | Loss: 2.9030 | LR: 6.68e-06 | 1268.2ms/step | 15.8 samples/s +Step 40820/44000 | Loss: 2.9249 | LR: 6.64e-06 | 1357.3ms/step | 14.7 samples/s +Step 40830/44000 | Loss: 2.9269 | LR: 6.60e-06 | 1293.7ms/step | 15.5 samples/s +Step 40840/44000 | Loss: 2.9095 | LR: 6.56e-06 | 1312.2ms/step | 15.2 samples/s +Step 40850/44000 | Loss: 2.8978 | LR: 6.52e-06 | 1295.1ms/step | 15.4 samples/s +Step 40860/44000 | Loss: 2.9150 | LR: 6.47e-06 | 1310.1ms/step | 15.3 samples/s +Step 40870/44000 | Loss: 2.8720 | LR: 6.43e-06 | 1282.1ms/step | 15.6 samples/s +Step 40880/44000 | Loss: 2.9208 | LR: 6.39e-06 | 1281.4ms/step | 15.6 samples/s +Step 40890/44000 | Loss: 2.8878 | LR: 6.35e-06 | 1257.7ms/step | 15.9 samples/s +Step 40900/44000 | Loss: 2.8833 | LR: 6.31e-06 | 1247.8ms/step | 16.0 samples/s +Step 40910/44000 | Loss: 2.9042 | LR: 6.27e-06 | 1258.5ms/step | 15.9 samples/s +Step 40920/44000 | Loss: 2.9791 | LR: 6.23e-06 | 1308.9ms/step | 15.3 samples/s +Step 40930/44000 | Loss: 2.9942 | LR: 6.19e-06 | 1264.7ms/step | 15.8 samples/s +Step 40940/44000 | Loss: 2.8625 | LR: 6.15e-06 | 1262.4ms/step | 15.8 samples/s +Step 40950/44000 | Loss: 2.9083 | LR: 6.11e-06 | 1258.2ms/step | 15.9 samples/s +Step 40960/44000 | Loss: 2.9183 | LR: 6.07e-06 | 1237.6ms/step | 16.2 samples/s +Step 40970/44000 | Loss: 2.9204 | LR: 6.03e-06 | 1267.5ms/step | 15.8 samples/s +Step 40980/44000 | Loss: 2.9534 | LR: 5.99e-06 | 1210.8ms/step | 16.5 samples/s +Step 40990/44000 | Loss: 2.9589 | LR: 5.95e-06 | 1238.2ms/step | 16.2 samples/s +Step 41000/44000 | Loss: 2.9112 | LR: 5.91e-06 | 1300.4ms/step | 15.4 samples/s +Validation at step 41000: Loss = 2.8694, Perplexity = 17.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-41000 +Step 41010/44000 | Loss: 2.9674 | LR: 5.87e-06 | 2255.2ms/step | 8.9 samples/s +Step 41020/44000 | Loss: 2.9660 | LR: 5.83e-06 | 1259.5ms/step | 15.9 samples/s +Step 41030/44000 | Loss: 2.8888 | LR: 5.80e-06 | 1262.8ms/step | 15.8 samples/s +Step 41040/44000 | Loss: 2.8989 | LR: 5.76e-06 | 1257.3ms/step | 15.9 samples/s +Step 41050/44000 | Loss: 2.9371 | LR: 5.72e-06 | 1236.9ms/step | 16.2 samples/s +Step 41060/44000 | Loss: 2.8643 | LR: 5.68e-06 | 1303.8ms/step | 15.3 samples/s +Step 41070/44000 | Loss: 2.8507 | LR: 5.64e-06 | 1264.7ms/step | 15.8 samples/s +Step 41080/44000 | Loss: 2.9930 | LR: 5.60e-06 | 1281.1ms/step | 15.6 samples/s +Step 41090/44000 | Loss: 2.9562 | LR: 5.56e-06 | 1322.5ms/step | 15.1 samples/s +Step 41100/44000 | Loss: 2.8841 | LR: 5.53e-06 | 1285.4ms/step | 15.6 samples/s +Step 41110/44000 | Loss: 2.9249 | LR: 5.49e-06 | 1256.6ms/step | 15.9 samples/s +Step 41120/44000 | Loss: 2.9319 | LR: 5.45e-06 | 1273.0ms/step | 15.7 samples/s +Step 41130/44000 | Loss: 2.9436 | LR: 5.41e-06 | 1265.7ms/step | 15.8 samples/s +Step 41140/44000 | Loss: 2.9315 | LR: 5.38e-06 | 1278.8ms/step | 15.6 samples/s +Step 41150/44000 | Loss: 2.9184 | LR: 5.34e-06 | 1258.4ms/step | 15.9 samples/s +Step 41160/44000 | Loss: 2.8603 | LR: 5.30e-06 | 1286.8ms/step | 15.5 samples/s +Step 41170/44000 | Loss: 2.8920 | LR: 5.26e-06 | 1312.1ms/step | 15.2 samples/s +Step 41180/44000 | Loss: 2.8856 | LR: 5.23e-06 | 1259.0ms/step | 15.9 samples/s +Step 41190/44000 | Loss: 2.8327 | LR: 5.19e-06 | 1263.3ms/step | 15.8 samples/s +Step 41200/44000 | Loss: 2.9499 | LR: 5.15e-06 | 1293.7ms/step | 15.5 samples/s +Step 41210/44000 | Loss: 2.8028 | LR: 5.12e-06 | 1263.5ms/step | 15.8 samples/s +Step 41220/44000 | Loss: 2.8852 | LR: 5.08e-06 | 1268.5ms/step | 15.8 samples/s +Step 41230/44000 | Loss: 2.9286 | LR: 5.04e-06 | 1263.1ms/step | 15.8 samples/s +Step 41240/44000 | Loss: 2.8502 | LR: 5.01e-06 | 1284.6ms/step | 15.6 samples/s +Step 41250/44000 | Loss: 2.9653 | LR: 4.97e-06 | 1268.5ms/step | 15.8 samples/s +Validation at step 41250: Loss = 2.8694, Perplexity = 17.63 +Step 41260/44000 | Loss: 2.9331 | LR: 4.94e-06 | 1766.3ms/step | 11.3 samples/s +Step 41270/44000 | Loss: 2.9216 | LR: 4.90e-06 | 1326.8ms/step | 15.1 samples/s +Step 41280/44000 | Loss: 2.9060 | LR: 4.86e-06 | 1289.2ms/step | 15.5 samples/s +Step 41290/44000 | Loss: 2.9539 | LR: 4.83e-06 | 1258.5ms/step | 15.9 samples/s +Step 41300/44000 | Loss: 2.8767 | LR: 4.79e-06 | 1278.8ms/step | 15.6 samples/s +Step 41310/44000 | Loss: 2.9945 | LR: 4.76e-06 | 1279.1ms/step | 15.6 samples/s +Step 41320/44000 | Loss: 2.8770 | LR: 4.72e-06 | 1284.5ms/step | 15.6 samples/s +Step 41330/44000 | Loss: 2.9422 | LR: 4.69e-06 | 1289.8ms/step | 15.5 samples/s +Step 41340/44000 | Loss: 2.9119 | LR: 4.65e-06 | 1310.2ms/step | 15.3 samples/s +Step 41350/44000 | Loss: 2.8805 | LR: 4.62e-06 | 1252.6ms/step | 16.0 samples/s +Step 41360/44000 | Loss: 2.9044 | LR: 4.58e-06 | 1299.8ms/step | 15.4 samples/s +Step 41370/44000 | Loss: 2.8668 | LR: 4.55e-06 | 1298.2ms/step | 15.4 samples/s +Step 41380/44000 | Loss: 2.9899 | LR: 4.51e-06 | 1282.2ms/step | 15.6 samples/s +Step 41390/44000 | Loss: 2.9487 | LR: 4.48e-06 | 1258.6ms/step | 15.9 samples/s +Step 41400/44000 | Loss: 2.9899 | LR: 4.45e-06 | 1251.6ms/step | 16.0 samples/s +Step 41410/44000 | Loss: 2.9214 | LR: 4.41e-06 | 1288.2ms/step | 15.5 samples/s +Step 41420/44000 | Loss: 2.9489 | LR: 4.38e-06 | 1294.6ms/step | 15.4 samples/s +Step 41430/44000 | Loss: 2.9463 | LR: 4.34e-06 | 1284.2ms/step | 15.6 samples/s +Step 41440/44000 | Loss: 2.9983 | LR: 4.31e-06 | 1336.9ms/step | 15.0 samples/s +Step 41450/44000 | Loss: 2.8866 | LR: 4.28e-06 | 1273.9ms/step | 15.7 samples/s +Step 41460/44000 | Loss: 2.9369 | LR: 4.24e-06 | 1268.4ms/step | 15.8 samples/s +Step 41470/44000 | Loss: 2.9268 | LR: 4.21e-06 | 1268.8ms/step | 15.8 samples/s +Step 41480/44000 | Loss: 2.8719 | LR: 4.18e-06 | 1297.9ms/step | 15.4 samples/s +Step 41490/44000 | Loss: 2.9488 | LR: 4.14e-06 | 1225.7ms/step | 16.3 samples/s +Step 41500/44000 | Loss: 2.9068 | LR: 4.11e-06 | 1276.9ms/step | 15.7 samples/s +Validation at step 41500: Loss = 2.8694, Perplexity = 17.63 +Step 41510/44000 | Loss: 2.9541 | LR: 4.08e-06 | 1767.2ms/step | 11.3 samples/s +Step 41520/44000 | Loss: 2.9543 | LR: 4.05e-06 | 1300.0ms/step | 15.4 samples/s +Step 41530/44000 | Loss: 2.9768 | LR: 4.01e-06 | 1273.8ms/step | 15.7 samples/s +Step 41540/44000 | Loss: 2.8818 | LR: 3.98e-06 | 1295.0ms/step | 15.4 samples/s +Step 41550/44000 | Loss: 2.9346 | LR: 3.95e-06 | 1294.8ms/step | 15.4 samples/s +Step 41560/44000 | Loss: 2.9584 | LR: 3.92e-06 | 1268.6ms/step | 15.8 samples/s +Step 41570/44000 | Loss: 2.9543 | LR: 3.88e-06 | 1269.0ms/step | 15.8 samples/s +Step 41580/44000 | Loss: 2.9112 | LR: 3.85e-06 | 1274.1ms/step | 15.7 samples/s +Step 41590/44000 | Loss: 2.9716 | LR: 3.82e-06 | 1295.4ms/step | 15.4 samples/s +Step 41600/44000 | Loss: 2.8392 | LR: 3.79e-06 | 1263.7ms/step | 15.8 samples/s +Step 41610/44000 | Loss: 2.9287 | LR: 3.76e-06 | 1284.8ms/step | 15.6 samples/s +Step 41620/44000 | Loss: 2.9032 | LR: 3.73e-06 | 1285.5ms/step | 15.6 samples/s +Step 41630/44000 | Loss: 2.9637 | LR: 3.70e-06 | 1294.2ms/step | 15.5 samples/s +Step 41640/44000 | Loss: 2.9189 | LR: 3.66e-06 | 1321.3ms/step | 15.1 samples/s +Step 41650/44000 | Loss: 2.9701 | LR: 3.63e-06 | 1267.2ms/step | 15.8 samples/s +Step 41660/44000 | Loss: 2.9251 | LR: 3.60e-06 | 1264.7ms/step | 15.8 samples/s +Step 41670/44000 | Loss: 2.9119 | LR: 3.57e-06 | 1268.9ms/step | 15.8 samples/s +Step 41680/44000 | Loss: 2.9674 | LR: 3.54e-06 | 1220.3ms/step | 16.4 samples/s +Step 41690/44000 | Loss: 2.9099 | LR: 3.51e-06 | 1270.1ms/step | 15.7 samples/s +Step 41700/44000 | Loss: 2.9351 | LR: 3.48e-06 | 1258.4ms/step | 15.9 samples/s +Step 41710/44000 | Loss: 2.9268 | LR: 3.45e-06 | 1247.4ms/step | 16.0 samples/s +Step 41720/44000 | Loss: 2.9293 | LR: 3.42e-06 | 1269.4ms/step | 15.8 samples/s +Step 41730/44000 | Loss: 2.8994 | LR: 3.39e-06 | 1257.5ms/step | 15.9 samples/s +Step 41740/44000 | Loss: 2.8955 | LR: 3.36e-06 | 1279.4ms/step | 15.6 samples/s +Step 41750/44000 | Loss: 2.9559 | LR: 3.33e-06 | 1221.9ms/step | 16.4 samples/s +Validation at step 41750: Loss = 2.8694, Perplexity = 17.63 +Step 41760/44000 | Loss: 2.9462 | LR: 3.30e-06 | 1730.1ms/step | 11.6 samples/s +Step 41770/44000 | Loss: 2.9490 | LR: 3.27e-06 | 1325.8ms/step | 15.1 samples/s +Step 41780/44000 | Loss: 2.8868 | LR: 3.24e-06 | 1290.5ms/step | 15.5 samples/s +Step 41790/44000 | Loss: 2.9283 | LR: 3.21e-06 | 1283.7ms/step | 15.6 samples/s +Step 41800/44000 | Loss: 2.8925 | LR: 3.19e-06 | 1269.4ms/step | 15.8 samples/s +Step 41810/44000 | Loss: 2.8798 | LR: 3.16e-06 | 1279.5ms/step | 15.6 samples/s +Step 41820/44000 | Loss: 2.9425 | LR: 3.13e-06 | 1283.9ms/step | 15.6 samples/s +Step 41830/44000 | Loss: 2.8673 | LR: 3.10e-06 | 1290.2ms/step | 15.5 samples/s +Step 41840/44000 | Loss: 2.9388 | LR: 3.07e-06 | 1237.8ms/step | 16.2 samples/s +Step 41850/44000 | Loss: 2.9221 | LR: 3.04e-06 | 1279.1ms/step | 15.6 samples/s +Step 41860/44000 | Loss: 2.8804 | LR: 3.01e-06 | 1263.0ms/step | 15.8 samples/s +Step 41870/44000 | Loss: 2.8764 | LR: 2.99e-06 | 1299.3ms/step | 15.4 samples/s +Step 41880/44000 | Loss: 2.8786 | LR: 2.96e-06 | 1290.1ms/step | 15.5 samples/s +Step 41890/44000 | Loss: 2.9092 | LR: 2.93e-06 | 1292.1ms/step | 15.5 samples/s +Step 41900/44000 | Loss: 2.9039 | LR: 2.90e-06 | 1288.8ms/step | 15.5 samples/s +Step 41910/44000 | Loss: 2.9107 | LR: 2.88e-06 | 1337.2ms/step | 15.0 samples/s +Step 41920/44000 | Loss: 2.9343 | LR: 2.85e-06 | 1258.1ms/step | 15.9 samples/s +Step 41930/44000 | Loss: 2.9499 | LR: 2.82e-06 | 1238.0ms/step | 16.2 samples/s +Step 41940/44000 | Loss: 2.8801 | LR: 2.79e-06 | 1289.9ms/step | 15.5 samples/s +Step 41950/44000 | Loss: 3.0070 | LR: 2.77e-06 | 1251.0ms/step | 16.0 samples/s +Step 41960/44000 | Loss: 2.8806 | LR: 2.74e-06 | 1313.6ms/step | 15.2 samples/s +Step 41970/44000 | Loss: 2.9003 | LR: 2.71e-06 | 1315.5ms/step | 15.2 samples/s +Step 41980/44000 | Loss: 2.9261 | LR: 2.69e-06 | 1244.6ms/step | 16.1 samples/s +Step 41990/44000 | Loss: 2.9347 | LR: 2.66e-06 | 1250.4ms/step | 16.0 samples/s +Step 42000/44000 | Loss: 2.9374 | LR: 2.63e-06 | 1305.3ms/step | 15.3 samples/s +Validation at step 42000: Loss = 2.8693, Perplexity = 17.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-42000 +Step 42010/44000 | Loss: 2.9091 | LR: 2.61e-06 | 2186.9ms/step | 9.1 samples/s +Step 42020/44000 | Loss: 2.9130 | LR: 2.58e-06 | 1294.6ms/step | 15.4 samples/s +Step 42030/44000 | Loss: 2.9137 | LR: 2.56e-06 | 1253.2ms/step | 16.0 samples/s +Step 42040/44000 | Loss: 2.9619 | LR: 2.53e-06 | 1260.3ms/step | 15.9 samples/s +Step 42050/44000 | Loss: 2.9877 | LR: 2.50e-06 | 1219.2ms/step | 16.4 samples/s +Step 42060/44000 | Loss: 2.9573 | LR: 2.48e-06 | 1295.3ms/step | 15.4 samples/s +Step 42070/44000 | Loss: 2.9246 | LR: 2.45e-06 | 1304.6ms/step | 15.3 samples/s +Step 42080/44000 | Loss: 2.8871 | LR: 2.43e-06 | 1274.3ms/step | 15.7 samples/s +Step 42090/44000 | Loss: 2.8692 | LR: 2.40e-06 | 1227.1ms/step | 16.3 samples/s +Step 42100/44000 | Loss: 2.9395 | LR: 2.38e-06 | 1247.5ms/step | 16.0 samples/s +Step 42110/44000 | Loss: 2.9107 | LR: 2.35e-06 | 1253.0ms/step | 16.0 samples/s +Step 42120/44000 | Loss: 2.8646 | LR: 2.33e-06 | 1279.7ms/step | 15.6 samples/s +Step 42130/44000 | Loss: 2.9785 | LR: 2.30e-06 | 1284.7ms/step | 15.6 samples/s +Step 42140/44000 | Loss: 2.8916 | LR: 2.28e-06 | 1316.2ms/step | 15.2 samples/s +Step 42150/44000 | Loss: 2.8656 | LR: 2.25e-06 | 1253.3ms/step | 16.0 samples/s +Step 42160/44000 | Loss: 2.9185 | LR: 2.23e-06 | 1285.7ms/step | 15.6 samples/s +Step 42170/44000 | Loss: 2.9282 | LR: 2.21e-06 | 1263.9ms/step | 15.8 samples/s +Step 42180/44000 | Loss: 2.8716 | LR: 2.18e-06 | 1251.9ms/step | 16.0 samples/s +Step 42190/44000 | Loss: 2.9263 | LR: 2.16e-06 | 1326.2ms/step | 15.1 samples/s +Step 42200/44000 | Loss: 2.8966 | LR: 2.13e-06 | 1282.5ms/step | 15.6 samples/s +Step 42210/44000 | Loss: 2.9392 | LR: 2.11e-06 | 1248.4ms/step | 16.0 samples/s +Step 42220/44000 | Loss: 2.8996 | LR: 2.09e-06 | 1269.7ms/step | 15.8 samples/s +Step 42230/44000 | Loss: 3.0170 | LR: 2.06e-06 | 1254.7ms/step | 15.9 samples/s +Step 42240/44000 | Loss: 2.9606 | LR: 2.04e-06 | 1294.8ms/step | 15.4 samples/s +Step 42250/44000 | Loss: 2.9222 | LR: 2.02e-06 | 1247.3ms/step | 16.0 samples/s +Validation at step 42250: Loss = 2.8694, Perplexity = 17.63 +Step 42260/44000 | Loss: 2.9466 | LR: 1.99e-06 | 1771.8ms/step | 11.3 samples/s +Step 42270/44000 | Loss: 2.8742 | LR: 1.97e-06 | 1273.9ms/step | 15.7 samples/s +Step 42280/44000 | Loss: 2.8810 | LR: 1.95e-06 | 1279.0ms/step | 15.6 samples/s +Step 42290/44000 | Loss: 2.9117 | LR: 1.93e-06 | 1258.3ms/step | 15.9 samples/s +Step 42300/44000 | Loss: 2.8870 | LR: 1.90e-06 | 1250.6ms/step | 16.0 samples/s +Step 42310/44000 | Loss: 2.9549 | LR: 1.88e-06 | 1322.9ms/step | 15.1 samples/s +Step 42320/44000 | Loss: 2.9261 | LR: 1.86e-06 | 1241.9ms/step | 16.1 samples/s +Step 42330/44000 | Loss: 2.8835 | LR: 1.84e-06 | 1299.9ms/step | 15.4 samples/s +Step 42340/44000 | Loss: 2.9077 | LR: 1.82e-06 | 1279.3ms/step | 15.6 samples/s +Step 42350/44000 | Loss: 2.8431 | LR: 1.79e-06 | 1295.8ms/step | 15.4 samples/s +Step 42360/44000 | Loss: 2.9424 | LR: 1.77e-06 | 1267.2ms/step | 15.8 samples/s +Step 42370/44000 | Loss: 2.9502 | LR: 1.75e-06 | 1305.3ms/step | 15.3 samples/s +Step 42380/44000 | Loss: 2.9005 | LR: 1.73e-06 | 1294.3ms/step | 15.5 samples/s +Step 42390/44000 | Loss: 2.9867 | LR: 1.71e-06 | 1275.5ms/step | 15.7 samples/s +Step 42400/44000 | Loss: 2.8880 | LR: 1.69e-06 | 1220.2ms/step | 16.4 samples/s +Step 42410/44000 | Loss: 2.8944 | LR: 1.67e-06 | 1269.2ms/step | 15.8 samples/s +Step 42420/44000 | Loss: 2.9163 | LR: 1.64e-06 | 1262.6ms/step | 15.8 samples/s +Step 42430/44000 | Loss: 2.9341 | LR: 1.62e-06 | 1268.9ms/step | 15.8 samples/s +Step 42440/44000 | Loss: 2.9059 | LR: 1.60e-06 | 1279.3ms/step | 15.6 samples/s +Step 42450/44000 | Loss: 2.9026 | LR: 1.58e-06 | 1263.5ms/step | 15.8 samples/s +Step 42460/44000 | Loss: 2.9394 | LR: 1.56e-06 | 1253.4ms/step | 16.0 samples/s +Step 42470/44000 | Loss: 2.8825 | LR: 1.54e-06 | 1289.9ms/step | 15.5 samples/s +Step 42480/44000 | Loss: 2.9425 | LR: 1.52e-06 | 1304.8ms/step | 15.3 samples/s +Step 42490/44000 | Loss: 2.9140 | LR: 1.50e-06 | 1293.7ms/step | 15.5 samples/s +Step 42500/44000 | Loss: 2.8962 | LR: 1.48e-06 | 1249.1ms/step | 16.0 samples/s +Validation at step 42500: Loss = 2.8694, Perplexity = 17.63 +Step 42510/44000 | Loss: 2.9013 | LR: 1.46e-06 | 1677.7ms/step | 11.9 samples/s +Step 42520/44000 | Loss: 2.9523 | LR: 1.44e-06 | 1273.2ms/step | 15.7 samples/s +Step 42530/44000 | Loss: 2.9195 | LR: 1.42e-06 | 1264.1ms/step | 15.8 samples/s +Step 42540/44000 | Loss: 2.9335 | LR: 1.40e-06 | 1240.4ms/step | 16.1 samples/s +Step 42550/44000 | Loss: 2.9239 | LR: 1.39e-06 | 1234.7ms/step | 16.2 samples/s +Step 42560/44000 | Loss: 2.9598 | LR: 1.37e-06 | 1279.2ms/step | 15.6 samples/s +Step 42570/44000 | Loss: 2.9224 | LR: 1.35e-06 | 1293.5ms/step | 15.5 samples/s +Step 42580/44000 | Loss: 2.8309 | LR: 1.33e-06 | 1263.2ms/step | 15.8 samples/s +Step 42590/44000 | Loss: 2.8910 | LR: 1.31e-06 | 1275.5ms/step | 15.7 samples/s +Step 42600/44000 | Loss: 2.9494 | LR: 1.29e-06 | 1284.7ms/step | 15.6 samples/s +Step 42610/44000 | Loss: 2.8573 | LR: 1.27e-06 | 1242.1ms/step | 16.1 samples/s +Step 42620/44000 | Loss: 2.9017 | LR: 1.25e-06 | 1253.2ms/step | 16.0 samples/s +Step 42630/44000 | Loss: 2.9030 | LR: 1.24e-06 | 1242.3ms/step | 16.1 samples/s +Step 42640/44000 | Loss: 2.9110 | LR: 1.22e-06 | 1233.0ms/step | 16.2 samples/s +Step 42650/44000 | Loss: 2.9090 | LR: 1.20e-06 | 1279.5ms/step | 15.6 samples/s +Step 42660/44000 | Loss: 2.9851 | LR: 1.18e-06 | 1289.1ms/step | 15.5 samples/s +Step 42670/44000 | Loss: 2.8798 | LR: 1.17e-06 | 1262.0ms/step | 15.8 samples/s +Step 42680/44000 | Loss: 2.8815 | LR: 1.15e-06 | 1316.2ms/step | 15.2 samples/s +Step 42690/44000 | Loss: 2.9153 | LR: 1.13e-06 | 1256.2ms/step | 15.9 samples/s +Step 42700/44000 | Loss: 2.8917 | LR: 1.11e-06 | 1257.7ms/step | 15.9 samples/s +Step 42710/44000 | Loss: 2.8341 | LR: 1.10e-06 | 1303.0ms/step | 15.3 samples/s +Step 42720/44000 | Loss: 2.9444 | LR: 1.08e-06 | 1266.2ms/step | 15.8 samples/s +Step 42730/44000 | Loss: 2.8927 | LR: 1.06e-06 | 1233.9ms/step | 16.2 samples/s +Step 42740/44000 | Loss: 2.9416 | LR: 1.05e-06 | 1273.9ms/step | 15.7 samples/s +Step 42750/44000 | Loss: 2.9381 | LR: 1.03e-06 | 1236.0ms/step | 16.2 samples/s +Validation at step 42750: Loss = 2.8693, Perplexity = 17.63 +Step 42760/44000 | Loss: 2.9643 | LR: 1.01e-06 | 1740.9ms/step | 11.5 samples/s +Step 42770/44000 | Loss: 2.9708 | LR: 9.97e-07 | 1285.3ms/step | 15.6 samples/s +Step 42780/44000 | Loss: 2.9344 | LR: 9.81e-07 | 1317.5ms/step | 15.2 samples/s +Step 42790/44000 | Loss: 2.8772 | LR: 9.65e-07 | 1265.8ms/step | 15.8 samples/s +Step 42800/44000 | Loss: 2.8679 | LR: 9.49e-07 | 1305.4ms/step | 15.3 samples/s +Step 42810/44000 | Loss: 2.8483 | LR: 9.33e-07 | 1263.0ms/step | 15.8 samples/s +Step 42820/44000 | Loss: 2.9304 | LR: 9.18e-07 | 1311.0ms/step | 15.3 samples/s +Step 42830/44000 | Loss: 2.8958 | LR: 9.02e-07 | 1332.1ms/step | 15.0 samples/s +Step 42840/44000 | Loss: 2.9244 | LR: 8.87e-07 | 1241.7ms/step | 16.1 samples/s +Step 42850/44000 | Loss: 2.9047 | LR: 8.72e-07 | 1270.1ms/step | 15.7 samples/s +Step 42860/44000 | Loss: 2.8925 | LR: 8.57e-07 | 1252.9ms/step | 16.0 samples/s +Step 42870/44000 | Loss: 2.9020 | LR: 8.42e-07 | 1256.9ms/step | 15.9 samples/s +Step 42880/44000 | Loss: 2.9559 | LR: 8.27e-07 | 1294.8ms/step | 15.4 samples/s +Step 42890/44000 | Loss: 2.9642 | LR: 8.12e-07 | 1257.2ms/step | 15.9 samples/s +Step 42900/44000 | Loss: 2.9338 | LR: 7.98e-07 | 1289.8ms/step | 15.5 samples/s +Step 42910/44000 | Loss: 2.9769 | LR: 7.83e-07 | 1289.3ms/step | 15.5 samples/s +Step 42920/44000 | Loss: 2.9436 | LR: 7.69e-07 | 1282.9ms/step | 15.6 samples/s +Step 42930/44000 | Loss: 2.8463 | LR: 7.55e-07 | 1253.2ms/step | 16.0 samples/s +Step 42940/44000 | Loss: 2.9333 | LR: 7.41e-07 | 1284.6ms/step | 15.6 samples/s +Step 42950/44000 | Loss: 2.9108 | LR: 7.27e-07 | 1235.4ms/step | 16.2 samples/s +Step 42960/44000 | Loss: 2.9413 | LR: 7.13e-07 | 1242.3ms/step | 16.1 samples/s +Step 42970/44000 | Loss: 2.8897 | LR: 6.99e-07 | 1254.3ms/step | 15.9 samples/s +Step 42980/44000 | Loss: 2.9660 | LR: 6.86e-07 | 1247.4ms/step | 16.0 samples/s +Step 42990/44000 | Loss: 2.8640 | LR: 6.72e-07 | 1276.1ms/step | 15.7 samples/s +Step 43000/44000 | Loss: 2.8985 | LR: 6.59e-07 | 1249.9ms/step | 16.0 samples/s +Validation at step 43000: Loss = 2.8693, Perplexity = 17.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-43000 +Step 43010/44000 | Loss: 2.8973 | LR: 6.46e-07 | 2322.7ms/step | 8.6 samples/s +Step 43020/44000 | Loss: 2.9707 | LR: 6.33e-07 | 1304.1ms/step | 15.3 samples/s +Step 43030/44000 | Loss: 2.8442 | LR: 6.20e-07 | 1258.2ms/step | 15.9 samples/s +Step 43040/44000 | Loss: 2.9212 | LR: 6.08e-07 | 1257.9ms/step | 15.9 samples/s +Step 43050/44000 | Loss: 2.8612 | LR: 5.95e-07 | 1259.0ms/step | 15.9 samples/s +Step 43060/44000 | Loss: 2.9167 | LR: 5.83e-07 | 1262.5ms/step | 15.8 samples/s +Step 43070/44000 | Loss: 2.8610 | LR: 5.70e-07 | 1268.3ms/step | 15.8 samples/s +Step 43080/44000 | Loss: 2.8813 | LR: 5.58e-07 | 1233.8ms/step | 16.2 samples/s +Step 43090/44000 | Loss: 2.8684 | LR: 5.46e-07 | 1271.1ms/step | 15.7 samples/s +Step 43100/44000 | Loss: 2.8957 | LR: 5.34e-07 | 1284.7ms/step | 15.6 samples/s +Step 43110/44000 | Loss: 2.9269 | LR: 5.22e-07 | 1257.5ms/step | 15.9 samples/s +Step 43120/44000 | Loss: 2.8938 | LR: 5.11e-07 | 1289.7ms/step | 15.5 samples/s +Step 43130/44000 | Loss: 2.8863 | LR: 4.99e-07 | 1288.3ms/step | 15.5 samples/s +Step 43140/44000 | Loss: 2.9075 | LR: 4.88e-07 | 1258.7ms/step | 15.9 samples/s +Step 43150/44000 | Loss: 2.9495 | LR: 4.76e-07 | 1262.2ms/step | 15.8 samples/s +Step 43160/44000 | Loss: 2.9333 | LR: 4.65e-07 | 1285.5ms/step | 15.6 samples/s +Step 43170/44000 | Loss: 2.8730 | LR: 4.54e-07 | 1252.5ms/step | 16.0 samples/s +Step 43180/44000 | Loss: 2.9430 | LR: 4.43e-07 | 1284.0ms/step | 15.6 samples/s +Step 43190/44000 | Loss: 2.9407 | LR: 4.33e-07 | 1283.6ms/step | 15.6 samples/s +Step 43200/44000 | Loss: 2.9253 | LR: 4.22e-07 | 1268.6ms/step | 15.8 samples/s +Step 43210/44000 | Loss: 2.9541 | LR: 4.12e-07 | 1273.0ms/step | 15.7 samples/s +Step 43220/44000 | Loss: 2.8990 | LR: 4.01e-07 | 1279.3ms/step | 15.6 samples/s +Step 43230/44000 | Loss: 2.9462 | LR: 3.91e-07 | 1274.0ms/step | 15.7 samples/s +Step 43240/44000 | Loss: 2.8758 | LR: 3.81e-07 | 1304.5ms/step | 15.3 samples/s +Step 43250/44000 | Loss: 2.8946 | LR: 3.71e-07 | 1252.8ms/step | 16.0 samples/s +Validation at step 43250: Loss = 2.8694, Perplexity = 17.63 +Step 43260/44000 | Loss: 2.8591 | LR: 3.61e-07 | 1745.7ms/step | 11.5 samples/s +Step 43270/44000 | Loss: 2.9452 | LR: 3.51e-07 | 1219.0ms/step | 16.4 samples/s +Step 43280/44000 | Loss: 2.9109 | LR: 3.42e-07 | 1247.9ms/step | 16.0 samples/s +Step 43290/44000 | Loss: 2.9128 | LR: 3.32e-07 | 1301.2ms/step | 15.4 samples/s +Step 43300/44000 | Loss: 2.8752 | LR: 3.23e-07 | 1300.2ms/step | 15.4 samples/s +Step 43310/44000 | Loss: 2.9323 | LR: 3.14e-07 | 1294.3ms/step | 15.5 samples/s +Step 43320/44000 | Loss: 2.9081 | LR: 3.05e-07 | 1273.7ms/step | 15.7 samples/s +Step 43330/44000 | Loss: 2.9514 | LR: 2.96e-07 | 1284.6ms/step | 15.6 samples/s +Step 43340/44000 | Loss: 2.9631 | LR: 2.87e-07 | 1252.4ms/step | 16.0 samples/s +Step 43350/44000 | Loss: 2.9008 | LR: 2.79e-07 | 1341.7ms/step | 14.9 samples/s +Step 43360/44000 | Loss: 2.8856 | LR: 2.70e-07 | 1305.2ms/step | 15.3 samples/s +Step 43370/44000 | Loss: 2.8990 | LR: 2.62e-07 | 1273.4ms/step | 15.7 samples/s +Step 43380/44000 | Loss: 2.9273 | LR: 2.53e-07 | 1297.3ms/step | 15.4 samples/s +Step 43390/44000 | Loss: 2.9134 | LR: 2.45e-07 | 1239.8ms/step | 16.1 samples/s +Step 43400/44000 | Loss: 2.9325 | LR: 2.37e-07 | 1271.4ms/step | 15.7 samples/s +Step 43410/44000 | Loss: 2.9796 | LR: 2.30e-07 | 1266.3ms/step | 15.8 samples/s +Step 43420/44000 | Loss: 2.9291 | LR: 2.22e-07 | 1288.3ms/step | 15.5 samples/s +Step 43430/44000 | Loss: 2.9367 | LR: 2.14e-07 | 1279.6ms/step | 15.6 samples/s +Step 43440/44000 | Loss: 2.9567 | LR: 2.07e-07 | 1274.7ms/step | 15.7 samples/s +Step 43450/44000 | Loss: 2.9248 | LR: 1.99e-07 | 1258.1ms/step | 15.9 samples/s +Step 43460/44000 | Loss: 2.9171 | LR: 1.92e-07 | 1277.8ms/step | 15.7 samples/s +Step 43470/44000 | Loss: 2.8964 | LR: 1.85e-07 | 1258.3ms/step | 15.9 samples/s +Step 43480/44000 | Loss: 2.9587 | LR: 1.78e-07 | 1294.3ms/step | 15.5 samples/s +Step 43490/44000 | Loss: 2.8991 | LR: 1.72e-07 | 1225.3ms/step | 16.3 samples/s +Step 43500/44000 | Loss: 2.9244 | LR: 1.65e-07 | 1237.1ms/step | 16.2 samples/s +Validation at step 43500: Loss = 2.8694, Perplexity = 17.63 +Step 43510/44000 | Loss: 2.8952 | LR: 1.58e-07 | 1742.2ms/step | 11.5 samples/s +Step 43520/44000 | Loss: 2.8815 | LR: 1.52e-07 | 1323.6ms/step | 15.1 samples/s +Step 43530/44000 | Loss: 2.9310 | LR: 1.46e-07 | 1277.2ms/step | 15.7 samples/s +Step 43540/44000 | Loss: 2.9029 | LR: 1.40e-07 | 1293.2ms/step | 15.5 samples/s +Step 43550/44000 | Loss: 2.9361 | LR: 1.34e-07 | 1302.1ms/step | 15.4 samples/s +Step 43560/44000 | Loss: 2.9218 | LR: 1.28e-07 | 1286.8ms/step | 15.5 samples/s +Step 43570/44000 | Loss: 2.8453 | LR: 1.22e-07 | 1298.3ms/step | 15.4 samples/s +Step 43580/44000 | Loss: 2.8964 | LR: 1.16e-07 | 1278.9ms/step | 15.6 samples/s +Step 43590/44000 | Loss: 2.8926 | LR: 1.11e-07 | 1286.5ms/step | 15.5 samples/s +Step 43600/44000 | Loss: 2.9151 | LR: 1.06e-07 | 1268.2ms/step | 15.8 samples/s +Step 43610/44000 | Loss: 2.9762 | LR: 1.00e-07 | 1272.9ms/step | 15.7 samples/s +Step 43620/44000 | Loss: 2.9669 | LR: 9.52e-08 | 1264.8ms/step | 15.8 samples/s +Step 43630/44000 | Loss: 2.8913 | LR: 9.03e-08 | 1276.5ms/step | 15.7 samples/s +Step 43640/44000 | Loss: 2.8392 | LR: 8.55e-08 | 1225.9ms/step | 16.3 samples/s +Step 43650/44000 | Loss: 2.9561 | LR: 8.08e-08 | 1253.5ms/step | 16.0 samples/s +Step 43660/44000 | Loss: 2.8979 | LR: 7.62e-08 | 1218.6ms/step | 16.4 samples/s +Step 43670/44000 | Loss: 2.9389 | LR: 7.18e-08 | 1233.6ms/step | 16.2 samples/s +Step 43680/44000 | Loss: 2.8165 | LR: 6.75e-08 | 1279.5ms/step | 15.6 samples/s +Step 43690/44000 | Loss: 2.9217 | LR: 6.34e-08 | 1279.1ms/step | 15.6 samples/s +Step 43700/44000 | Loss: 2.9156 | LR: 5.94e-08 | 1310.5ms/step | 15.3 samples/s +Step 43710/44000 | Loss: 2.9475 | LR: 5.55e-08 | 1253.2ms/step | 16.0 samples/s +Step 43720/44000 | Loss: 2.9712 | LR: 5.17e-08 | 1248.7ms/step | 16.0 samples/s +Step 43730/44000 | Loss: 2.9880 | LR: 4.81e-08 | 1260.3ms/step | 15.9 samples/s +Step 43740/44000 | Loss: 2.9368 | LR: 4.46e-08 | 1239.5ms/step | 16.1 samples/s +Step 43750/44000 | Loss: 2.8950 | LR: 4.12e-08 | 1258.3ms/step | 15.9 samples/s +Validation at step 43750: Loss = 2.8693, Perplexity = 17.63 +Step 43760/44000 | Loss: 2.9017 | LR: 3.80e-08 | 1782.6ms/step | 11.2 samples/s +Step 43770/44000 | Loss: 2.9206 | LR: 3.49e-08 | 1289.4ms/step | 15.5 samples/s +Step 43780/44000 | Loss: 2.9396 | LR: 3.19e-08 | 1259.6ms/step | 15.9 samples/s +Step 43790/44000 | Loss: 2.9455 | LR: 2.91e-08 | 1272.6ms/step | 15.7 samples/s +Step 43800/44000 | Loss: 2.8318 | LR: 2.64e-08 | 1253.2ms/step | 16.0 samples/s +Step 43810/44000 | Loss: 2.9258 | LR: 2.38e-08 | 1266.1ms/step | 15.8 samples/s +Step 43820/44000 | Loss: 2.9159 | LR: 2.14e-08 | 1291.3ms/step | 15.5 samples/s +Step 43830/44000 | Loss: 2.9055 | LR: 1.91e-08 | 1241.9ms/step | 16.1 samples/s +Step 43840/44000 | Loss: 2.9290 | LR: 1.69e-08 | 1260.4ms/step | 15.9 samples/s +Step 43850/44000 | Loss: 2.9274 | LR: 1.48e-08 | 1224.8ms/step | 16.3 samples/s +Step 43860/44000 | Loss: 2.8300 | LR: 1.29e-08 | 1263.5ms/step | 15.8 samples/s +Step 43870/44000 | Loss: 2.9505 | LR: 1.11e-08 | 1310.9ms/step | 15.3 samples/s +Step 43880/44000 | Loss: 2.9334 | LR: 9.50e-09 | 1310.6ms/step | 15.3 samples/s +Step 43890/44000 | Loss: 2.9480 | LR: 7.98e-09 | 1308.2ms/step | 15.3 samples/s +Step 43900/44000 | Loss: 2.8854 | LR: 6.60e-09 | 1233.8ms/step | 16.2 samples/s +Step 43910/44000 | Loss: 2.9622 | LR: 5.34e-09 | 1282.9ms/step | 15.6 samples/s +Step 43920/44000 | Loss: 2.9354 | LR: 4.22e-09 | 1275.9ms/step | 15.7 samples/s +Step 43930/44000 | Loss: 2.9280 | LR: 3.23e-09 | 1294.7ms/step | 15.4 samples/s +Step 43940/44000 | Loss: 2.9074 | LR: 2.37e-09 | 1256.6ms/step | 15.9 samples/s +Step 43950/44000 | Loss: 2.9444 | LR: 1.65e-09 | 1255.0ms/step | 15.9 samples/s +Step 43960/44000 | Loss: 2.9616 | LR: 1.06e-09 | 1258.8ms/step | 15.9 samples/s +Step 43970/44000 | Loss: 2.8985 | LR: 5.94e-10 | 1305.6ms/step | 15.3 samples/s +Step 43980/44000 | Loss: 2.8931 | LR: 2.64e-10 | 1305.6ms/step | 15.3 samples/s +Step 43990/44000 | Loss: 2.8436 | LR: 6.60e-11 | 1227.1ms/step | 16.3 samples/s +Step 44000/44000 | Loss: 2.8486 | LR: 0.00e+00 | 1243.8ms/step | 16.1 samples/s +Validation at step 44000: Loss = 2.8694, Perplexity = 17.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed45/run_20260822_043537/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▅▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▆▄▄▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▂▁▁▂█▂▃▄▄▃▃▃▃▃▂▃▄▃▃▃▂▂▂▁▂▁▁▁▁▂▂▁▁▁▁▁▁ +wandb: speed/samples_per_sec ▇▇▇▇▇██▇▇█▇▁▅▅▅▃▅▅▅▅▅▄▆▅▅▅▄▅▇▇▇▇██▇▇█▇▇▇ +wandb: speed/steps_per_sec ▆█▇▇▇▇▇▅▇▇▃▄▃▃▄▃▇▃▃▃▄▁▅▃▃▃▃▇▇▇▇█▅▇▆█▇█▇█ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.86939 +wandb: eval/perplexity 17.62618 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 1243.8365 +wandb: speed/samples_per_sec 16.07928 +wandb: speed/steps_per_sec 0.80396 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 DDP5 365m static 8k nrb35 seed45 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/pq6ov2y3 +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_043614-pq6ov2y3/logs +Training complete! diff --git a/epfl_final/train_logs/p1_ddp5_static_seed45_47/l8k365m_static_nrb35_seed46_a100-40gb-train-5-gpu-2.log b/epfl_final/train_logs/p1_ddp5_static_seed45_47/l8k365m_static_nrb35_seed46_a100-40gb-train-5-gpu-2.log new file mode 100644 index 0000000000000000000000000000000000000000..d3c7794e6ce1f4c7dd8d69e6bb189ee48179b9bc --- /dev/null +++ b/epfl_final/train_logs/p1_ddp5_static_seed45_47/l8k365m_static_nrb35_seed46_a100-40gb-train-5-gpu-2.log @@ -0,0 +1,4858 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 2, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 46, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 DDP5 365m static 8k nrb35 seed46", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "static", + "nrb35", + "seed46", + "ddp5", + "a100" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1", + "sparse_mask_seed": 1, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Initializing model from scratch (random weights)... +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Using pre-tokenized dataset (no runtime tokenization) + +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 5 +Gradient accumulation steps: 2 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 302776 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_043615-8naq20vs +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 DDP5 365m static 8k nrb35 seed46 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8naq20vs +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0583 | LR: 6.67e-06 | 13088.8ms/step | 1.5 samples/s +Step 20/44000 | Loss: 11.9528 | LR: 1.33e-05 | 716.5ms/step | 27.9 samples/s +Step 30/44000 | Loss: 11.6154 | LR: 2.00e-05 | 717.7ms/step | 27.9 samples/s +Step 40/44000 | Loss: 11.1284 | LR: 2.67e-05 | 718.0ms/step | 27.9 samples/s +Step 50/44000 | Loss: 10.7535 | LR: 3.33e-05 | 719.3ms/step | 27.8 samples/s +Step 60/44000 | Loss: 10.5348 | LR: 4.00e-05 | 719.1ms/step | 27.8 samples/s +Step 70/44000 | Loss: 10.3128 | LR: 4.67e-05 | 718.4ms/step | 27.8 samples/s +Step 80/44000 | Loss: 10.0058 | LR: 5.33e-05 | 718.5ms/step | 27.8 samples/s +Step 90/44000 | Loss: 9.6081 | LR: 6.00e-05 | 726.9ms/step | 27.5 samples/s +Step 100/44000 | Loss: 9.1352 | LR: 6.67e-05 | 729.7ms/step | 27.4 samples/s +Step 110/44000 | Loss: 8.5967 | LR: 7.33e-05 | 720.9ms/step | 27.7 samples/s +Step 120/44000 | Loss: 8.1520 | LR: 8.00e-05 | 721.1ms/step | 27.7 samples/s +Step 130/44000 | Loss: 7.7512 | LR: 8.67e-05 | 720.0ms/step | 27.8 samples/s +Step 140/44000 | Loss: 7.4905 | LR: 9.33e-05 | 719.4ms/step | 27.8 samples/s +Step 150/44000 | Loss: 7.3014 | LR: 1.00e-04 | 719.9ms/step | 27.8 samples/s +Step 160/44000 | Loss: 7.1746 | LR: 1.07e-04 | 721.1ms/step | 27.7 samples/s +Step 170/44000 | Loss: 7.1504 | LR: 1.13e-04 | 721.0ms/step | 27.7 samples/s +Step 180/44000 | Loss: 6.9549 | LR: 1.20e-04 | 732.8ms/step | 27.3 samples/s +Step 190/44000 | Loss: 7.0023 | LR: 1.27e-04 | 721.0ms/step | 27.7 samples/s +Step 200/44000 | Loss: 6.7778 | LR: 1.33e-04 | 720.7ms/step | 27.7 samples/s +Step 210/44000 | Loss: 6.7896 | LR: 1.40e-04 | 720.0ms/step | 27.8 samples/s +Step 220/44000 | Loss: 6.6650 | LR: 1.47e-04 | 721.0ms/step | 27.7 samples/s +Step 230/44000 | Loss: 6.5848 | LR: 1.53e-04 | 721.8ms/step | 27.7 samples/s +Step 240/44000 | Loss: 6.5379 | LR: 1.60e-04 | 723.2ms/step | 27.7 samples/s +Step 250/44000 | Loss: 6.4701 | LR: 1.67e-04 | 722.5ms/step | 27.7 samples/s +Validation at step 250: Loss = 6.4311, Perplexity = 620.89 +Step 260/44000 | Loss: 6.3943 | LR: 1.73e-04 | 5291.9ms/step | 3.8 samples/s +Step 270/44000 | Loss: 6.3647 | LR: 1.80e-04 | 721.1ms/step | 27.7 samples/s +Step 280/44000 | Loss: 6.3067 | LR: 1.87e-04 | 722.5ms/step | 27.7 samples/s +Step 290/44000 | Loss: 6.2786 | LR: 1.93e-04 | 728.8ms/step | 27.4 samples/s +Step 300/44000 | Loss: 6.1896 | LR: 2.00e-04 | 727.8ms/step | 27.5 samples/s +Step 310/44000 | Loss: 6.2055 | LR: 2.07e-04 | 724.6ms/step | 27.6 samples/s +Step 320/44000 | Loss: 6.1269 | LR: 2.13e-04 | 723.6ms/step | 27.6 samples/s +Step 330/44000 | Loss: 6.0847 | LR: 2.20e-04 | 722.7ms/step | 27.7 samples/s +Step 340/44000 | Loss: 6.0653 | LR: 2.27e-04 | 723.0ms/step | 27.7 samples/s +Step 350/44000 | Loss: 5.9976 | LR: 2.33e-04 | 722.9ms/step | 27.7 samples/s +Step 360/44000 | Loss: 5.9172 | LR: 2.40e-04 | 724.0ms/step | 27.6 samples/s +Step 370/44000 | Loss: 5.9722 | LR: 2.47e-04 | 724.5ms/step | 27.6 samples/s +Step 380/44000 | Loss: 5.8629 | LR: 2.53e-04 | 739.9ms/step | 27.0 samples/s +Step 390/44000 | Loss: 5.8759 | LR: 2.60e-04 | 723.7ms/step | 27.6 samples/s +Step 400/44000 | Loss: 5.7770 | LR: 2.67e-04 | 723.9ms/step | 27.6 samples/s +Step 410/44000 | Loss: 5.8148 | LR: 2.73e-04 | 723.6ms/step | 27.6 samples/s +Step 420/44000 | Loss: 5.7881 | LR: 2.80e-04 | 723.1ms/step | 27.7 samples/s +Step 430/44000 | Loss: 5.6838 | LR: 2.87e-04 | 724.5ms/step | 27.6 samples/s +Step 440/44000 | Loss: 5.6635 | LR: 2.93e-04 | 725.5ms/step | 27.6 samples/s +Step 450/44000 | Loss: 5.6251 | LR: 3.00e-04 | 724.2ms/step | 27.6 samples/s +Step 460/44000 | Loss: 5.6550 | LR: 3.07e-04 | 723.6ms/step | 27.6 samples/s +Step 470/44000 | Loss: 5.5870 | LR: 3.13e-04 | 733.6ms/step | 27.3 samples/s +Step 480/44000 | Loss: 5.5855 | LR: 3.20e-04 | 723.5ms/step | 27.6 samples/s +Step 490/44000 | Loss: 5.5020 | LR: 3.27e-04 | 723.7ms/step | 27.6 samples/s +Step 500/44000 | Loss: 5.4715 | LR: 3.33e-04 | 724.5ms/step | 27.6 samples/s +Validation at step 500: Loss = 5.4472, Perplexity = 232.10 +Step 510/44000 | Loss: 5.4255 | LR: 3.40e-04 | 952.5ms/step | 21.0 samples/s +Step 520/44000 | Loss: 5.4009 | LR: 3.47e-04 | 723.8ms/step | 27.6 samples/s +Step 530/44000 | Loss: 5.3894 | LR: 3.53e-04 | 723.6ms/step | 27.6 samples/s +Step 540/44000 | Loss: 5.4037 | LR: 3.60e-04 | 724.3ms/step | 27.6 samples/s +Step 550/44000 | Loss: 5.3346 | LR: 3.67e-04 | 729.4ms/step | 27.4 samples/s +Step 560/44000 | Loss: 5.2633 | LR: 3.73e-04 | 730.2ms/step | 27.4 samples/s +Step 570/44000 | Loss: 5.2532 | LR: 3.80e-04 | 726.4ms/step | 27.5 samples/s +Step 580/44000 | Loss: 5.2572 | LR: 3.87e-04 | 725.7ms/step | 27.6 samples/s +Step 590/44000 | Loss: 5.2917 | LR: 3.93e-04 | 724.4ms/step | 27.6 samples/s +Step 600/44000 | Loss: 5.1674 | LR: 4.00e-04 | 724.5ms/step | 27.6 samples/s +Step 610/44000 | Loss: 5.2198 | LR: 4.07e-04 | 724.8ms/step | 27.6 samples/s +Step 620/44000 | Loss: 5.1673 | LR: 4.13e-04 | 726.1ms/step | 27.5 samples/s +Step 630/44000 | Loss: 5.0191 | LR: 4.20e-04 | 727.2ms/step | 27.5 samples/s +Step 640/44000 | Loss: 5.0736 | LR: 4.27e-04 | 735.4ms/step | 27.2 samples/s +Step 650/44000 | Loss: 5.0154 | LR: 4.33e-04 | 725.5ms/step | 27.6 samples/s +Step 660/44000 | Loss: 4.9715 | LR: 4.40e-04 | 724.7ms/step | 27.6 samples/s +Step 670/44000 | Loss: 4.9745 | LR: 4.47e-04 | 725.1ms/step | 27.6 samples/s +Step 680/44000 | Loss: 4.9443 | LR: 4.53e-04 | 724.9ms/step | 27.6 samples/s +Step 690/44000 | Loss: 4.8870 | LR: 4.60e-04 | 725.5ms/step | 27.6 samples/s +Step 700/44000 | Loss: 4.9159 | LR: 4.67e-04 | 725.9ms/step | 27.6 samples/s +Step 710/44000 | Loss: 4.8996 | LR: 4.73e-04 | 725.1ms/step | 27.6 samples/s +Step 720/44000 | Loss: 4.8046 | LR: 4.80e-04 | 724.3ms/step | 27.6 samples/s +Step 730/44000 | Loss: 4.8121 | LR: 4.87e-04 | 733.9ms/step | 27.3 samples/s +Step 740/44000 | Loss: 4.7618 | LR: 4.93e-04 | 724.7ms/step | 27.6 samples/s +Step 750/44000 | Loss: 4.8376 | LR: 5.00e-04 | 724.7ms/step | 27.6 samples/s +Validation at step 750: Loss = 4.7205, Perplexity = 112.22 +Step 760/44000 | Loss: 4.7226 | LR: 5.00e-04 | 954.8ms/step | 20.9 samples/s +Step 770/44000 | Loss: 4.6772 | LR: 5.00e-04 | 726.7ms/step | 27.5 samples/s +Step 780/44000 | Loss: 4.6561 | LR: 5.00e-04 | 724.2ms/step | 27.6 samples/s +Step 790/44000 | Loss: 4.6747 | LR: 5.00e-04 | 725.0ms/step | 27.6 samples/s +Step 800/44000 | Loss: 4.6045 | LR: 5.00e-04 | 728.3ms/step | 27.5 samples/s +Step 810/44000 | Loss: 4.6422 | LR: 5.00e-04 | 738.9ms/step | 27.1 samples/s +Step 820/44000 | Loss: 4.5615 | LR: 5.00e-04 | 728.7ms/step | 27.4 samples/s +Step 830/44000 | Loss: 4.5147 | LR: 5.00e-04 | 726.6ms/step | 27.5 samples/s +Step 840/44000 | Loss: 4.4809 | LR: 5.00e-04 | 726.1ms/step | 27.5 samples/s +Step 850/44000 | Loss: 4.5148 | LR: 5.00e-04 | 724.9ms/step | 27.6 samples/s +Step 860/44000 | Loss: 4.5115 | LR: 5.00e-04 | 724.8ms/step | 27.6 samples/s +Step 870/44000 | Loss: 4.4380 | LR: 5.00e-04 | 725.3ms/step | 27.6 samples/s +Step 880/44000 | Loss: 4.5188 | LR: 5.00e-04 | 725.4ms/step | 27.6 samples/s +Step 890/44000 | Loss: 4.4168 | LR: 5.00e-04 | 727.0ms/step | 27.5 samples/s +Step 900/44000 | Loss: 4.4225 | LR: 5.00e-04 | 734.8ms/step | 27.2 samples/s +Step 910/44000 | Loss: 4.4048 | LR: 5.00e-04 | 725.1ms/step | 27.6 samples/s +Step 920/44000 | Loss: 4.4250 | LR: 5.00e-04 | 724.6ms/step | 27.6 samples/s +Step 930/44000 | Loss: 4.4610 | LR: 5.00e-04 | 725.0ms/step | 27.6 samples/s +Step 940/44000 | Loss: 4.3728 | LR: 5.00e-04 | 724.4ms/step | 27.6 samples/s +Step 950/44000 | Loss: 4.3763 | LR: 5.00e-04 | 725.9ms/step | 27.6 samples/s +Step 960/44000 | Loss: 4.3395 | LR: 5.00e-04 | 727.0ms/step | 27.5 samples/s +Step 970/44000 | Loss: 4.3536 | LR: 5.00e-04 | 726.2ms/step | 27.5 samples/s +Step 980/44000 | Loss: 4.3389 | LR: 5.00e-04 | 725.2ms/step | 27.6 samples/s +Step 990/44000 | Loss: 4.3323 | LR: 5.00e-04 | 734.4ms/step | 27.2 samples/s +Step 1000/44000 | Loss: 4.2576 | LR: 5.00e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 1000: Loss = 4.2581, Perplexity = 70.67 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-1000 +Step 1010/44000 | Loss: 4.2550 | LR: 5.00e-04 | 1362.3ms/step | 14.7 samples/s +Step 1020/44000 | Loss: 4.2832 | LR: 5.00e-04 | 727.8ms/step | 27.5 samples/s +Step 1030/44000 | Loss: 4.3114 | LR: 5.00e-04 | 725.7ms/step | 27.6 samples/s +Step 1040/44000 | Loss: 4.3318 | LR: 5.00e-04 | 725.5ms/step | 27.6 samples/s +Step 1050/44000 | Loss: 4.2658 | LR: 5.00e-04 | 725.3ms/step | 27.6 samples/s +Step 1060/44000 | Loss: 4.2461 | LR: 5.00e-04 | 729.7ms/step | 27.4 samples/s +Step 1070/44000 | Loss: 4.1876 | LR: 5.00e-04 | 730.4ms/step | 27.4 samples/s +Step 1080/44000 | Loss: 4.1702 | LR: 5.00e-04 | 726.8ms/step | 27.5 samples/s +Step 1090/44000 | Loss: 4.1763 | LR: 5.00e-04 | 725.6ms/step | 27.6 samples/s +Step 1100/44000 | Loss: 4.1479 | LR: 5.00e-04 | 725.0ms/step | 27.6 samples/s +Step 1110/44000 | Loss: 4.1262 | LR: 5.00e-04 | 725.8ms/step | 27.6 samples/s +Step 1120/44000 | Loss: 4.1851 | LR: 5.00e-04 | 725.0ms/step | 27.6 samples/s +Step 1130/44000 | Loss: 4.1062 | LR: 5.00e-04 | 726.1ms/step | 27.5 samples/s +Step 1140/44000 | Loss: 4.1673 | LR: 5.00e-04 | 726.6ms/step | 27.5 samples/s +Step 1150/44000 | Loss: 4.1380 | LR: 5.00e-04 | 731.0ms/step | 27.4 samples/s +Step 1160/44000 | Loss: 4.1597 | LR: 5.00e-04 | 732.4ms/step | 27.3 samples/s +Step 1170/44000 | Loss: 4.0542 | LR: 5.00e-04 | 725.0ms/step | 27.6 samples/s +Step 1180/44000 | Loss: 4.0706 | LR: 5.00e-04 | 724.5ms/step | 27.6 samples/s +Step 1190/44000 | Loss: 4.0808 | LR: 5.00e-04 | 724.3ms/step | 27.6 samples/s +Step 1200/44000 | Loss: 4.1228 | LR: 5.00e-04 | 725.7ms/step | 27.6 samples/s +Step 1210/44000 | Loss: 4.1557 | LR: 5.00e-04 | 726.9ms/step | 27.5 samples/s +Step 1220/44000 | Loss: 4.0951 | LR: 5.00e-04 | 725.8ms/step | 27.6 samples/s +Step 1230/44000 | Loss: 3.9867 | LR: 5.00e-04 | 724.4ms/step | 27.6 samples/s +Step 1240/44000 | Loss: 4.0650 | LR: 5.00e-04 | 738.5ms/step | 27.1 samples/s +Step 1250/44000 | Loss: 4.0681 | LR: 5.00e-04 | 724.9ms/step | 27.6 samples/s +Validation at step 1250: Loss = 4.0201, Perplexity = 55.71 +Step 1260/44000 | Loss: 4.0808 | LR: 5.00e-04 | 950.4ms/step | 21.0 samples/s +Step 1270/44000 | Loss: 3.9796 | LR: 5.00e-04 | 727.4ms/step | 27.5 samples/s +Step 1280/44000 | Loss: 3.9705 | LR: 5.00e-04 | 726.3ms/step | 27.5 samples/s +Step 1290/44000 | Loss: 4.0394 | LR: 5.00e-04 | 724.9ms/step | 27.6 samples/s +Step 1300/44000 | Loss: 3.9815 | LR: 5.00e-04 | 725.4ms/step | 27.6 samples/s +Step 1310/44000 | Loss: 3.9933 | LR: 5.00e-04 | 725.5ms/step | 27.6 samples/s +Step 1320/44000 | Loss: 3.9576 | LR: 5.00e-04 | 729.6ms/step | 27.4 samples/s +Step 1330/44000 | Loss: 4.0598 | LR: 5.00e-04 | 734.0ms/step | 27.2 samples/s +Step 1340/44000 | Loss: 3.9904 | LR: 5.00e-04 | 726.5ms/step | 27.5 samples/s +Step 1350/44000 | Loss: 3.9638 | LR: 5.00e-04 | 725.0ms/step | 27.6 samples/s +Step 1360/44000 | Loss: 3.9746 | LR: 5.00e-04 | 724.6ms/step | 27.6 samples/s +Step 1370/44000 | Loss: 3.9389 | LR: 5.00e-04 | 724.6ms/step | 27.6 samples/s +Step 1380/44000 | Loss: 3.9862 | LR: 5.00e-04 | 725.0ms/step | 27.6 samples/s +Step 1390/44000 | Loss: 3.9619 | LR: 5.00e-04 | 725.5ms/step | 27.6 samples/s +Step 1400/44000 | Loss: 3.9285 | LR: 5.00e-04 | 726.8ms/step | 27.5 samples/s +Step 1410/44000 | Loss: 3.9555 | LR: 5.00e-04 | 735.5ms/step | 27.2 samples/s +Step 1420/44000 | Loss: 3.9682 | LR: 5.00e-04 | 725.3ms/step | 27.6 samples/s +Step 1430/44000 | Loss: 3.9506 | LR: 5.00e-04 | 725.0ms/step | 27.6 samples/s +Step 1440/44000 | Loss: 3.9704 | LR: 5.00e-04 | 724.7ms/step | 27.6 samples/s +Step 1450/44000 | Loss: 3.9636 | LR: 5.00e-04 | 725.2ms/step | 27.6 samples/s +Step 1460/44000 | Loss: 3.9261 | LR: 5.00e-04 | 726.2ms/step | 27.5 samples/s +Step 1470/44000 | Loss: 3.9435 | LR: 5.00e-04 | 726.7ms/step | 27.5 samples/s +Step 1480/44000 | Loss: 3.8995 | LR: 5.00e-04 | 725.4ms/step | 27.6 samples/s +Step 1490/44000 | Loss: 3.8799 | LR: 5.00e-04 | 724.5ms/step | 27.6 samples/s +Step 1500/44000 | Loss: 3.9614 | LR: 5.00e-04 | 733.8ms/step | 27.3 samples/s +Validation at step 1500: Loss = 3.8758, Perplexity = 48.22 +Step 1510/44000 | Loss: 3.8602 | LR: 5.00e-04 | 950.4ms/step | 21.0 samples/s +Step 1520/44000 | Loss: 3.8742 | LR: 5.00e-04 | 726.9ms/step | 27.5 samples/s +Step 1530/44000 | Loss: 3.8349 | LR: 5.00e-04 | 727.0ms/step | 27.5 samples/s +Step 1540/44000 | Loss: 3.8480 | LR: 5.00e-04 | 726.6ms/step | 27.5 samples/s +Step 1550/44000 | Loss: 3.8661 | LR: 5.00e-04 | 725.2ms/step | 27.6 samples/s +Step 1560/44000 | Loss: 3.8953 | LR: 5.00e-04 | 725.8ms/step | 27.6 samples/s +Step 1570/44000 | Loss: 3.9263 | LR: 5.00e-04 | 725.3ms/step | 27.6 samples/s +Step 1580/44000 | Loss: 3.8169 | LR: 5.00e-04 | 730.3ms/step | 27.4 samples/s +Step 1590/44000 | Loss: 3.8240 | LR: 5.00e-04 | 731.5ms/step | 27.3 samples/s +Step 1600/44000 | Loss: 3.8307 | LR: 5.00e-04 | 726.2ms/step | 27.5 samples/s +Step 1610/44000 | Loss: 3.8728 | LR: 5.00e-04 | 725.5ms/step | 27.6 samples/s +Step 1620/44000 | Loss: 3.8935 | LR: 5.00e-04 | 726.6ms/step | 27.5 samples/s +Step 1630/44000 | Loss: 3.8161 | LR: 4.99e-04 | 724.4ms/step | 27.6 samples/s +Step 1640/44000 | Loss: 3.8814 | LR: 4.99e-04 | 724.8ms/step | 27.6 samples/s +Step 1650/44000 | Loss: 3.8523 | LR: 4.99e-04 | 726.3ms/step | 27.5 samples/s +Step 1660/44000 | Loss: 3.7932 | LR: 4.99e-04 | 727.0ms/step | 27.5 samples/s +Step 1670/44000 | Loss: 3.8382 | LR: 4.99e-04 | 737.2ms/step | 27.1 samples/s +Step 1680/44000 | Loss: 3.8594 | LR: 4.99e-04 | 724.7ms/step | 27.6 samples/s +Step 1690/44000 | Loss: 3.8197 | LR: 4.99e-04 | 724.7ms/step | 27.6 samples/s +Step 1700/44000 | Loss: 3.8317 | LR: 4.99e-04 | 725.0ms/step | 27.6 samples/s +Step 1710/44000 | Loss: 3.8234 | LR: 4.99e-04 | 725.3ms/step | 27.6 samples/s +Step 1720/44000 | Loss: 3.8129 | LR: 4.99e-04 | 727.1ms/step | 27.5 samples/s +Step 1730/44000 | Loss: 3.7753 | LR: 4.99e-04 | 726.2ms/step | 27.5 samples/s +Step 1740/44000 | Loss: 3.7701 | LR: 4.99e-04 | 724.7ms/step | 27.6 samples/s +Step 1750/44000 | Loss: 3.7274 | LR: 4.99e-04 | 724.9ms/step | 27.6 samples/s +Validation at step 1750: Loss = 3.7638, Perplexity = 43.11 +Step 1760/44000 | Loss: 3.7749 | LR: 4.99e-04 | 962.7ms/step | 20.8 samples/s +Step 1770/44000 | Loss: 3.7638 | LR: 4.99e-04 | 724.8ms/step | 27.6 samples/s +Step 1780/44000 | Loss: 3.7639 | LR: 4.99e-04 | 726.8ms/step | 27.5 samples/s +Step 1790/44000 | Loss: 3.8046 | LR: 4.99e-04 | 726.9ms/step | 27.5 samples/s +Step 1800/44000 | Loss: 3.8229 | LR: 4.99e-04 | 725.3ms/step | 27.6 samples/s +Step 1810/44000 | Loss: 3.7429 | LR: 4.99e-04 | 725.2ms/step | 27.6 samples/s +Step 1820/44000 | Loss: 3.7290 | LR: 4.99e-04 | 725.1ms/step | 27.6 samples/s +Step 1830/44000 | Loss: 3.7495 | LR: 4.99e-04 | 725.7ms/step | 27.6 samples/s +Step 1840/44000 | Loss: 3.7942 | LR: 4.99e-04 | 730.6ms/step | 27.4 samples/s +Step 1850/44000 | Loss: 3.7204 | LR: 4.99e-04 | 730.7ms/step | 27.4 samples/s +Step 1860/44000 | Loss: 3.7028 | LR: 4.99e-04 | 725.8ms/step | 27.6 samples/s +Step 1870/44000 | Loss: 3.6865 | LR: 4.99e-04 | 724.4ms/step | 27.6 samples/s +Step 1880/44000 | Loss: 3.7130 | LR: 4.99e-04 | 724.6ms/step | 27.6 samples/s +Step 1890/44000 | Loss: 3.7467 | LR: 4.99e-04 | 724.8ms/step | 27.6 samples/s +Step 1900/44000 | Loss: 3.7498 | LR: 4.99e-04 | 724.9ms/step | 27.6 samples/s +Step 1910/44000 | Loss: 3.7561 | LR: 4.99e-04 | 726.1ms/step | 27.5 samples/s +Step 1920/44000 | Loss: 3.7350 | LR: 4.99e-04 | 727.4ms/step | 27.5 samples/s +Step 1930/44000 | Loss: 3.6302 | LR: 4.99e-04 | 736.8ms/step | 27.1 samples/s +Step 1940/44000 | Loss: 3.6824 | LR: 4.99e-04 | 724.4ms/step | 27.6 samples/s +Step 1950/44000 | Loss: 3.7274 | LR: 4.99e-04 | 724.9ms/step | 27.6 samples/s +Step 1960/44000 | Loss: 3.6956 | LR: 4.99e-04 | 725.6ms/step | 27.6 samples/s +Step 1970/44000 | Loss: 3.7051 | LR: 4.99e-04 | 725.5ms/step | 27.6 samples/s +Step 1980/44000 | Loss: 3.7443 | LR: 4.99e-04 | 726.8ms/step | 27.5 samples/s +Step 1990/44000 | Loss: 3.8385 | LR: 4.99e-04 | 725.6ms/step | 27.6 samples/s +Step 2000/44000 | Loss: 3.7296 | LR: 4.99e-04 | 724.4ms/step | 27.6 samples/s +Validation at step 2000: Loss = 3.6889, Perplexity = 40.00 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-2000 +Step 2010/44000 | Loss: 3.7473 | LR: 4.99e-04 | 1418.2ms/step | 14.1 samples/s +Step 2020/44000 | Loss: 3.6603 | LR: 4.99e-04 | 724.5ms/step | 27.6 samples/s +Step 2030/44000 | Loss: 3.7645 | LR: 4.99e-04 | 726.1ms/step | 27.5 samples/s +Step 2040/44000 | Loss: 3.7028 | LR: 4.99e-04 | 726.5ms/step | 27.5 samples/s +Step 2050/44000 | Loss: 3.6423 | LR: 4.99e-04 | 725.0ms/step | 27.6 samples/s +Step 2060/44000 | Loss: 3.6743 | LR: 4.99e-04 | 725.1ms/step | 27.6 samples/s +Step 2070/44000 | Loss: 3.7014 | LR: 4.99e-04 | 725.3ms/step | 27.6 samples/s +Step 2080/44000 | Loss: 3.6873 | LR: 4.99e-04 | 726.0ms/step | 27.5 samples/s +Step 2090/44000 | Loss: 3.6975 | LR: 4.99e-04 | 733.4ms/step | 27.3 samples/s +Step 2100/44000 | Loss: 3.7116 | LR: 4.99e-04 | 732.0ms/step | 27.3 samples/s +Step 2110/44000 | Loss: 3.6869 | LR: 4.99e-04 | 726.6ms/step | 27.5 samples/s +Step 2120/44000 | Loss: 3.6793 | LR: 4.99e-04 | 724.9ms/step | 27.6 samples/s +Step 2130/44000 | Loss: 3.6418 | LR: 4.99e-04 | 725.1ms/step | 27.6 samples/s +Step 2140/44000 | Loss: 3.7104 | LR: 4.99e-04 | 725.8ms/step | 27.6 samples/s +Step 2150/44000 | Loss: 3.6358 | LR: 4.99e-04 | 724.7ms/step | 27.6 samples/s +Step 2160/44000 | Loss: 3.6788 | LR: 4.99e-04 | 726.0ms/step | 27.5 samples/s +Step 2170/44000 | Loss: 3.6773 | LR: 4.99e-04 | 726.9ms/step | 27.5 samples/s +Step 2180/44000 | Loss: 3.6531 | LR: 4.99e-04 | 729.8ms/step | 27.4 samples/s +Step 2190/44000 | Loss: 3.6991 | LR: 4.99e-04 | 729.3ms/step | 27.4 samples/s +Step 2200/44000 | Loss: 3.6565 | LR: 4.99e-04 | 725.0ms/step | 27.6 samples/s +Step 2210/44000 | Loss: 3.6536 | LR: 4.99e-04 | 724.5ms/step | 27.6 samples/s +Step 2220/44000 | Loss: 3.6874 | LR: 4.99e-04 | 725.3ms/step | 27.6 samples/s +Step 2230/44000 | Loss: 3.6842 | LR: 4.99e-04 | 726.1ms/step | 27.5 samples/s +Step 2240/44000 | Loss: 3.6056 | LR: 4.99e-04 | 726.4ms/step | 27.5 samples/s +Step 2250/44000 | Loss: 3.6250 | LR: 4.99e-04 | 725.0ms/step | 27.6 samples/s +Validation at step 2250: Loss = 3.6247, Perplexity = 37.51 +Step 2260/44000 | Loss: 3.6609 | LR: 4.98e-04 | 948.0ms/step | 21.1 samples/s +Step 2270/44000 | Loss: 3.6581 | LR: 4.98e-04 | 739.4ms/step | 27.1 samples/s +Step 2280/44000 | Loss: 3.6342 | LR: 4.98e-04 | 724.9ms/step | 27.6 samples/s +Step 2290/44000 | Loss: 3.6118 | LR: 4.98e-04 | 727.3ms/step | 27.5 samples/s +Step 2300/44000 | Loss: 3.6477 | LR: 4.98e-04 | 726.9ms/step | 27.5 samples/s +Step 2310/44000 | Loss: 3.6155 | LR: 4.98e-04 | 725.1ms/step | 27.6 samples/s +Step 2320/44000 | Loss: 3.6488 | LR: 4.98e-04 | 724.5ms/step | 27.6 samples/s +Step 2330/44000 | Loss: 3.7156 | LR: 4.98e-04 | 725.3ms/step | 27.6 samples/s +Step 2340/44000 | Loss: 3.5481 | LR: 4.98e-04 | 724.8ms/step | 27.6 samples/s +Step 2350/44000 | Loss: 3.6356 | LR: 4.98e-04 | 731.4ms/step | 27.3 samples/s +Step 2360/44000 | Loss: 3.6538 | LR: 4.98e-04 | 731.7ms/step | 27.3 samples/s +Step 2370/44000 | Loss: 3.6535 | LR: 4.98e-04 | 726.1ms/step | 27.5 samples/s +Step 2380/44000 | Loss: 3.6714 | LR: 4.98e-04 | 725.0ms/step | 27.6 samples/s +Step 2390/44000 | Loss: 3.6261 | LR: 4.98e-04 | 725.3ms/step | 27.6 samples/s +Step 2400/44000 | Loss: 3.6147 | LR: 4.98e-04 | 725.5ms/step | 27.6 samples/s +Step 2410/44000 | Loss: 3.6294 | LR: 4.98e-04 | 725.6ms/step | 27.6 samples/s +Step 2420/44000 | Loss: 3.5841 | LR: 4.98e-04 | 726.5ms/step | 27.5 samples/s +Step 2430/44000 | Loss: 3.6296 | LR: 4.98e-04 | 727.3ms/step | 27.5 samples/s +Step 2440/44000 | Loss: 3.5854 | LR: 4.98e-04 | 734.5ms/step | 27.2 samples/s +Step 2450/44000 | Loss: 3.6536 | LR: 4.98e-04 | 724.8ms/step | 27.6 samples/s +Step 2460/44000 | Loss: 3.5537 | LR: 4.98e-04 | 724.7ms/step | 27.6 samples/s +Step 2470/44000 | Loss: 3.5809 | LR: 4.98e-04 | 724.9ms/step | 27.6 samples/s +Step 2480/44000 | Loss: 3.5882 | LR: 4.98e-04 | 726.0ms/step | 27.5 samples/s +Step 2490/44000 | Loss: 3.6081 | LR: 4.98e-04 | 726.2ms/step | 27.5 samples/s +Step 2500/44000 | Loss: 3.6375 | LR: 4.98e-04 | 725.1ms/step | 27.6 samples/s +Validation at step 2500: Loss = 3.5695, Perplexity = 35.50 +Step 2510/44000 | Loss: 3.6026 | LR: 4.98e-04 | 948.4ms/step | 21.1 samples/s +Step 2520/44000 | Loss: 3.5919 | LR: 4.98e-04 | 728.5ms/step | 27.5 samples/s +Step 2530/44000 | Loss: 3.5405 | LR: 4.98e-04 | 734.9ms/step | 27.2 samples/s +Step 2540/44000 | Loss: 3.6176 | LR: 4.98e-04 | 725.7ms/step | 27.6 samples/s +Step 2550/44000 | Loss: 3.6356 | LR: 4.98e-04 | 727.1ms/step | 27.5 samples/s +Step 2560/44000 | Loss: 3.5552 | LR: 4.98e-04 | 726.6ms/step | 27.5 samples/s +Step 2570/44000 | Loss: 3.5233 | LR: 4.98e-04 | 725.4ms/step | 27.6 samples/s +Step 2580/44000 | Loss: 3.6443 | LR: 4.98e-04 | 725.2ms/step | 27.6 samples/s +Step 2590/44000 | Loss: 3.5666 | LR: 4.98e-04 | 725.1ms/step | 27.6 samples/s +Step 2600/44000 | Loss: 3.5878 | LR: 4.98e-04 | 725.2ms/step | 27.6 samples/s +Step 2610/44000 | Loss: 3.6115 | LR: 4.98e-04 | 731.6ms/step | 27.3 samples/s +Step 2620/44000 | Loss: 3.5789 | LR: 4.98e-04 | 732.7ms/step | 27.3 samples/s +Step 2630/44000 | Loss: 3.5053 | LR: 4.98e-04 | 726.0ms/step | 27.5 samples/s +Step 2640/44000 | Loss: 3.5358 | LR: 4.98e-04 | 725.2ms/step | 27.6 samples/s +Step 2650/44000 | Loss: 3.5975 | LR: 4.98e-04 | 726.5ms/step | 27.5 samples/s +Step 2660/44000 | Loss: 3.4750 | LR: 4.98e-04 | 725.4ms/step | 27.6 samples/s +Step 2670/44000 | Loss: 3.5754 | LR: 4.98e-04 | 726.8ms/step | 27.5 samples/s +Step 2680/44000 | Loss: 3.5624 | LR: 4.98e-04 | 728.4ms/step | 27.5 samples/s +Step 2690/44000 | Loss: 3.5205 | LR: 4.98e-04 | 726.1ms/step | 27.5 samples/s +Step 2700/44000 | Loss: 3.5796 | LR: 4.97e-04 | 736.0ms/step | 27.2 samples/s +Step 2710/44000 | Loss: 3.5728 | LR: 4.97e-04 | 725.1ms/step | 27.6 samples/s +Step 2720/44000 | Loss: 3.4931 | LR: 4.97e-04 | 724.6ms/step | 27.6 samples/s +Step 2730/44000 | Loss: 3.5957 | LR: 4.97e-04 | 724.9ms/step | 27.6 samples/s +Step 2740/44000 | Loss: 3.5635 | LR: 4.97e-04 | 727.0ms/step | 27.5 samples/s +Step 2750/44000 | Loss: 3.5529 | LR: 4.97e-04 | 727.4ms/step | 27.5 samples/s +Validation at step 2750: Loss = 3.5287, Perplexity = 34.08 +Step 2760/44000 | Loss: 3.6124 | LR: 4.97e-04 | 949.5ms/step | 21.1 samples/s +Step 2770/44000 | Loss: 3.5996 | LR: 4.97e-04 | 724.6ms/step | 27.6 samples/s +Step 2780/44000 | Loss: 3.5442 | LR: 4.97e-04 | 731.7ms/step | 27.3 samples/s +Step 2790/44000 | Loss: 3.4996 | LR: 4.97e-04 | 730.0ms/step | 27.4 samples/s +Step 2800/44000 | Loss: 3.5278 | LR: 4.97e-04 | 726.9ms/step | 27.5 samples/s +Step 2810/44000 | Loss: 3.5705 | LR: 4.97e-04 | 727.0ms/step | 27.5 samples/s +Step 2820/44000 | Loss: 3.5520 | LR: 4.97e-04 | 726.1ms/step | 27.5 samples/s +Step 2830/44000 | Loss: 3.5437 | LR: 4.97e-04 | 724.5ms/step | 27.6 samples/s +Step 2840/44000 | Loss: 3.5176 | LR: 4.97e-04 | 725.4ms/step | 27.6 samples/s +Step 2850/44000 | Loss: 3.5995 | LR: 4.97e-04 | 725.5ms/step | 27.6 samples/s +Step 2860/44000 | Loss: 3.5347 | LR: 4.97e-04 | 726.3ms/step | 27.5 samples/s +Step 2870/44000 | Loss: 3.5609 | LR: 4.97e-04 | 737.3ms/step | 27.1 samples/s +Step 2880/44000 | Loss: 3.4916 | LR: 4.97e-04 | 725.8ms/step | 27.6 samples/s +Step 2890/44000 | Loss: 3.5259 | LR: 4.97e-04 | 724.7ms/step | 27.6 samples/s +Step 2900/44000 | Loss: 3.5204 | LR: 4.97e-04 | 725.1ms/step | 27.6 samples/s +Step 2910/44000 | Loss: 3.5354 | LR: 4.97e-04 | 725.6ms/step | 27.6 samples/s +Step 2920/44000 | Loss: 3.5218 | LR: 4.97e-04 | 725.6ms/step | 27.6 samples/s +Step 2930/44000 | Loss: 3.5134 | LR: 4.97e-04 | 726.9ms/step | 27.5 samples/s +Step 2940/44000 | Loss: 3.5836 | LR: 4.97e-04 | 726.7ms/step | 27.5 samples/s +Step 2950/44000 | Loss: 3.4842 | LR: 4.97e-04 | 724.7ms/step | 27.6 samples/s +Step 2960/44000 | Loss: 3.5508 | LR: 4.97e-04 | 736.2ms/step | 27.2 samples/s +Step 2970/44000 | Loss: 3.5172 | LR: 4.97e-04 | 725.6ms/step | 27.6 samples/s +Step 2980/44000 | Loss: 3.5307 | LR: 4.97e-04 | 725.0ms/step | 27.6 samples/s +Step 2990/44000 | Loss: 3.4936 | LR: 4.97e-04 | 725.8ms/step | 27.6 samples/s +Step 3000/44000 | Loss: 3.5845 | LR: 4.97e-04 | 727.1ms/step | 27.5 samples/s +Validation at step 3000: Loss = 3.4923, Perplexity = 32.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-3000 +Step 3010/44000 | Loss: 3.5241 | LR: 4.97e-04 | 1441.7ms/step | 13.9 samples/s +Step 3020/44000 | Loss: 3.4813 | LR: 4.97e-04 | 724.6ms/step | 27.6 samples/s +Step 3030/44000 | Loss: 3.5129 | LR: 4.97e-04 | 725.4ms/step | 27.6 samples/s +Step 3040/44000 | Loss: 3.4798 | LR: 4.97e-04 | 736.4ms/step | 27.2 samples/s +Step 3050/44000 | Loss: 3.5510 | LR: 4.97e-04 | 726.1ms/step | 27.5 samples/s +Step 3060/44000 | Loss: 3.5270 | LR: 4.96e-04 | 726.3ms/step | 27.5 samples/s +Step 3070/44000 | Loss: 3.5117 | LR: 4.96e-04 | 725.2ms/step | 27.6 samples/s +Step 3080/44000 | Loss: 3.4681 | LR: 4.96e-04 | 725.1ms/step | 27.6 samples/s +Step 3090/44000 | Loss: 3.4924 | LR: 4.96e-04 | 724.8ms/step | 27.6 samples/s +Step 3100/44000 | Loss: 3.4913 | LR: 4.96e-04 | 725.1ms/step | 27.6 samples/s +Step 3110/44000 | Loss: 3.5217 | LR: 4.96e-04 | 726.2ms/step | 27.5 samples/s +Step 3120/44000 | Loss: 3.4611 | LR: 4.96e-04 | 732.1ms/step | 27.3 samples/s +Step 3130/44000 | Loss: 3.4781 | LR: 4.96e-04 | 730.2ms/step | 27.4 samples/s +Step 3140/44000 | Loss: 3.5424 | LR: 4.96e-04 | 724.9ms/step | 27.6 samples/s +Step 3150/44000 | Loss: 3.4689 | LR: 4.96e-04 | 724.7ms/step | 27.6 samples/s +Step 3160/44000 | Loss: 3.4232 | LR: 4.96e-04 | 725.7ms/step | 27.6 samples/s +Step 3170/44000 | Loss: 3.4692 | LR: 4.96e-04 | 725.6ms/step | 27.6 samples/s +Step 3180/44000 | Loss: 3.4540 | LR: 4.96e-04 | 726.3ms/step | 27.5 samples/s +Step 3190/44000 | Loss: 3.4704 | LR: 4.96e-04 | 726.8ms/step | 27.5 samples/s +Step 3200/44000 | Loss: 3.3989 | LR: 4.96e-04 | 725.3ms/step | 27.6 samples/s +Step 3210/44000 | Loss: 3.3513 | LR: 4.96e-04 | 734.8ms/step | 27.2 samples/s +Step 3220/44000 | Loss: 3.5019 | LR: 4.96e-04 | 725.4ms/step | 27.6 samples/s +Step 3230/44000 | Loss: 3.4359 | LR: 4.96e-04 | 725.0ms/step | 27.6 samples/s +Step 3240/44000 | Loss: 3.5222 | LR: 4.96e-04 | 726.5ms/step | 27.5 samples/s +Step 3250/44000 | Loss: 3.5565 | LR: 4.96e-04 | 727.5ms/step | 27.5 samples/s +Validation at step 3250: Loss = 3.4589, Perplexity = 31.78 +Step 3260/44000 | Loss: 3.4844 | LR: 4.96e-04 | 953.3ms/step | 21.0 samples/s +Step 3270/44000 | Loss: 3.5518 | LR: 4.96e-04 | 724.8ms/step | 27.6 samples/s +Step 3280/44000 | Loss: 3.5128 | LR: 4.96e-04 | 725.6ms/step | 27.6 samples/s +Step 3290/44000 | Loss: 3.3959 | LR: 4.96e-04 | 730.3ms/step | 27.4 samples/s +Step 3300/44000 | Loss: 3.5186 | LR: 4.96e-04 | 732.9ms/step | 27.3 samples/s +Step 3310/44000 | Loss: 3.5089 | LR: 4.96e-04 | 728.1ms/step | 27.5 samples/s +Step 3320/44000 | Loss: 3.5295 | LR: 4.96e-04 | 726.8ms/step | 27.5 samples/s +Step 3330/44000 | Loss: 3.4551 | LR: 4.96e-04 | 724.9ms/step | 27.6 samples/s +Step 3340/44000 | Loss: 3.4735 | LR: 4.96e-04 | 724.8ms/step | 27.6 samples/s +Step 3350/44000 | Loss: 3.4757 | LR: 4.96e-04 | 725.1ms/step | 27.6 samples/s +Step 3360/44000 | Loss: 3.4385 | LR: 4.96e-04 | 724.7ms/step | 27.6 samples/s +Step 3370/44000 | Loss: 3.4667 | LR: 4.95e-04 | 727.0ms/step | 27.5 samples/s +Step 3380/44000 | Loss: 3.4899 | LR: 4.95e-04 | 731.9ms/step | 27.3 samples/s +Step 3390/44000 | Loss: 3.4485 | LR: 4.95e-04 | 732.0ms/step | 27.3 samples/s +Step 3400/44000 | Loss: 3.5113 | LR: 4.95e-04 | 726.0ms/step | 27.5 samples/s +Step 3410/44000 | Loss: 3.4099 | LR: 4.95e-04 | 725.0ms/step | 27.6 samples/s +Step 3420/44000 | Loss: 3.4866 | LR: 4.95e-04 | 725.1ms/step | 27.6 samples/s +Step 3430/44000 | Loss: 3.4806 | LR: 4.95e-04 | 727.4ms/step | 27.5 samples/s +Step 3440/44000 | Loss: 3.4740 | LR: 4.95e-04 | 730.1ms/step | 27.4 samples/s +Step 3450/44000 | Loss: 3.3820 | LR: 4.95e-04 | 725.7ms/step | 27.6 samples/s +Step 3460/44000 | Loss: 3.5196 | LR: 4.95e-04 | 725.4ms/step | 27.6 samples/s +Step 3470/44000 | Loss: 3.5067 | LR: 4.95e-04 | 736.7ms/step | 27.1 samples/s +Step 3480/44000 | Loss: 3.4437 | LR: 4.95e-04 | 725.2ms/step | 27.6 samples/s +Step 3490/44000 | Loss: 3.3736 | LR: 4.95e-04 | 725.2ms/step | 27.6 samples/s +Step 3500/44000 | Loss: 3.4816 | LR: 4.95e-04 | 727.2ms/step | 27.5 samples/s +Validation at step 3500: Loss = 3.4319, Perplexity = 30.93 +Step 3510/44000 | Loss: 3.5019 | LR: 4.95e-04 | 952.6ms/step | 21.0 samples/s +Step 3520/44000 | Loss: 3.4213 | LR: 4.95e-04 | 724.7ms/step | 27.6 samples/s +Step 3530/44000 | Loss: 3.5006 | LR: 4.95e-04 | 724.9ms/step | 27.6 samples/s +Step 3540/44000 | Loss: 3.5132 | LR: 4.95e-04 | 725.3ms/step | 27.6 samples/s +Step 3550/44000 | Loss: 3.4826 | LR: 4.95e-04 | 729.8ms/step | 27.4 samples/s +Step 3560/44000 | Loss: 3.4122 | LR: 4.95e-04 | 731.4ms/step | 27.3 samples/s +Step 3570/44000 | Loss: 3.4084 | LR: 4.95e-04 | 726.3ms/step | 27.5 samples/s +Step 3580/44000 | Loss: 3.4440 | LR: 4.95e-04 | 724.5ms/step | 27.6 samples/s +Step 3590/44000 | Loss: 3.4855 | LR: 4.95e-04 | 724.7ms/step | 27.6 samples/s +Step 3600/44000 | Loss: 3.4622 | LR: 4.95e-04 | 725.9ms/step | 27.6 samples/s +Step 3610/44000 | Loss: 3.3756 | LR: 4.95e-04 | 725.2ms/step | 27.6 samples/s +Step 3620/44000 | Loss: 3.4123 | LR: 4.95e-04 | 725.5ms/step | 27.6 samples/s +Step 3630/44000 | Loss: 3.3566 | LR: 4.95e-04 | 726.9ms/step | 27.5 samples/s +Step 3640/44000 | Loss: 3.3992 | LR: 4.95e-04 | 736.3ms/step | 27.2 samples/s +Step 3650/44000 | Loss: 3.3586 | LR: 4.94e-04 | 725.2ms/step | 27.6 samples/s +Step 3660/44000 | Loss: 3.3525 | LR: 4.94e-04 | 725.5ms/step | 27.6 samples/s +Step 3670/44000 | Loss: 3.3950 | LR: 4.94e-04 | 725.2ms/step | 27.6 samples/s +Step 3680/44000 | Loss: 3.3864 | LR: 4.94e-04 | 724.9ms/step | 27.6 samples/s +Step 3690/44000 | Loss: 3.4880 | LR: 4.94e-04 | 725.9ms/step | 27.6 samples/s +Step 3700/44000 | Loss: 3.4016 | LR: 4.94e-04 | 726.1ms/step | 27.5 samples/s +Step 3710/44000 | Loss: 3.4534 | LR: 4.94e-04 | 725.9ms/step | 27.6 samples/s +Step 3720/44000 | Loss: 3.4247 | LR: 4.94e-04 | 724.3ms/step | 27.6 samples/s +Step 3730/44000 | Loss: 3.4374 | LR: 4.94e-04 | 738.6ms/step | 27.1 samples/s +Step 3740/44000 | Loss: 3.4227 | LR: 4.94e-04 | 725.2ms/step | 27.6 samples/s +Step 3750/44000 | Loss: 3.3889 | LR: 4.94e-04 | 725.6ms/step | 27.6 samples/s +Validation at step 3750: Loss = 3.4070, Perplexity = 30.17 +Step 3760/44000 | Loss: 3.4600 | LR: 4.94e-04 | 953.5ms/step | 21.0 samples/s +Step 3770/44000 | Loss: 3.4105 | LR: 4.94e-04 | 725.3ms/step | 27.6 samples/s +Step 3780/44000 | Loss: 3.4273 | LR: 4.94e-04 | 724.7ms/step | 27.6 samples/s +Step 3790/44000 | Loss: 3.4622 | LR: 4.94e-04 | 724.9ms/step | 27.6 samples/s +Step 3800/44000 | Loss: 3.3895 | LR: 4.94e-04 | 725.7ms/step | 27.6 samples/s +Step 3810/44000 | Loss: 3.4572 | LR: 4.94e-04 | 730.5ms/step | 27.4 samples/s +Step 3820/44000 | Loss: 3.4223 | LR: 4.94e-04 | 731.8ms/step | 27.3 samples/s +Step 3830/44000 | Loss: 3.5033 | LR: 4.94e-04 | 726.8ms/step | 27.5 samples/s +Step 3840/44000 | Loss: 3.3905 | LR: 4.94e-04 | 725.5ms/step | 27.6 samples/s +Step 3850/44000 | Loss: 3.4250 | LR: 4.94e-04 | 725.0ms/step | 27.6 samples/s +Step 3860/44000 | Loss: 3.4564 | LR: 4.94e-04 | 725.3ms/step | 27.6 samples/s +Step 3870/44000 | Loss: 3.4990 | LR: 4.94e-04 | 725.2ms/step | 27.6 samples/s +Step 3880/44000 | Loss: 3.4545 | LR: 4.94e-04 | 725.7ms/step | 27.6 samples/s +Step 3890/44000 | Loss: 3.4832 | LR: 4.94e-04 | 726.8ms/step | 27.5 samples/s +Step 3900/44000 | Loss: 3.4135 | LR: 4.93e-04 | 735.2ms/step | 27.2 samples/s +Step 3910/44000 | Loss: 3.3284 | LR: 4.93e-04 | 725.2ms/step | 27.6 samples/s +Step 3920/44000 | Loss: 3.4623 | LR: 4.93e-04 | 725.1ms/step | 27.6 samples/s +Step 3930/44000 | Loss: 3.3594 | LR: 4.93e-04 | 725.4ms/step | 27.6 samples/s +Step 3940/44000 | Loss: 3.3784 | LR: 4.93e-04 | 725.2ms/step | 27.6 samples/s +Step 3950/44000 | Loss: 3.4186 | LR: 4.93e-04 | 727.1ms/step | 27.5 samples/s +Step 3960/44000 | Loss: 3.3966 | LR: 4.93e-04 | 726.8ms/step | 27.5 samples/s +Step 3970/44000 | Loss: 3.4182 | LR: 4.93e-04 | 724.9ms/step | 27.6 samples/s +Step 3980/44000 | Loss: 3.3956 | LR: 4.93e-04 | 725.0ms/step | 27.6 samples/s +Step 3990/44000 | Loss: 3.3706 | LR: 4.93e-04 | 738.0ms/step | 27.1 samples/s +Step 4000/44000 | Loss: 3.3719 | LR: 4.93e-04 | 725.1ms/step | 27.6 samples/s +Validation at step 4000: Loss = 3.3843, Perplexity = 29.50 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-4000 +Step 4010/44000 | Loss: 3.2877 | LR: 4.93e-04 | 1355.9ms/step | 14.8 samples/s +Step 4020/44000 | Loss: 3.4182 | LR: 4.93e-04 | 724.9ms/step | 27.6 samples/s +Step 4030/44000 | Loss: 3.3998 | LR: 4.93e-04 | 724.4ms/step | 27.6 samples/s +Step 4040/44000 | Loss: 3.4699 | LR: 4.93e-04 | 724.2ms/step | 27.6 samples/s +Step 4050/44000 | Loss: 3.3644 | LR: 4.93e-04 | 725.1ms/step | 27.6 samples/s +Step 4060/44000 | Loss: 3.3750 | LR: 4.93e-04 | 725.5ms/step | 27.6 samples/s +Step 4070/44000 | Loss: 3.3286 | LR: 4.93e-04 | 737.0ms/step | 27.1 samples/s +Step 4080/44000 | Loss: 3.3813 | LR: 4.93e-04 | 726.5ms/step | 27.5 samples/s +Step 4090/44000 | Loss: 3.4167 | LR: 4.93e-04 | 725.1ms/step | 27.6 samples/s +Step 4100/44000 | Loss: 3.4443 | LR: 4.93e-04 | 724.9ms/step | 27.6 samples/s +Step 4110/44000 | Loss: 3.4231 | LR: 4.93e-04 | 725.2ms/step | 27.6 samples/s +Step 4120/44000 | Loss: 3.4041 | LR: 4.93e-04 | 725.2ms/step | 27.6 samples/s +Step 4130/44000 | Loss: 3.3649 | LR: 4.93e-04 | 726.0ms/step | 27.5 samples/s +Step 4140/44000 | Loss: 3.3602 | LR: 4.92e-04 | 726.3ms/step | 27.5 samples/s +Step 4150/44000 | Loss: 3.3278 | LR: 4.92e-04 | 731.7ms/step | 27.3 samples/s +Step 4160/44000 | Loss: 3.4141 | LR: 4.92e-04 | 732.8ms/step | 27.3 samples/s +Step 4170/44000 | Loss: 3.3864 | LR: 4.92e-04 | 725.1ms/step | 27.6 samples/s +Step 4180/44000 | Loss: 3.4035 | LR: 4.92e-04 | 725.3ms/step | 27.6 samples/s +Step 4190/44000 | Loss: 3.4077 | LR: 4.92e-04 | 725.4ms/step | 27.6 samples/s +Step 4200/44000 | Loss: 3.3293 | LR: 4.92e-04 | 726.8ms/step | 27.5 samples/s +Step 4210/44000 | Loss: 3.3981 | LR: 4.92e-04 | 726.6ms/step | 27.5 samples/s +Step 4220/44000 | Loss: 3.3932 | LR: 4.92e-04 | 725.4ms/step | 27.6 samples/s +Step 4230/44000 | Loss: 3.3802 | LR: 4.92e-04 | 725.2ms/step | 27.6 samples/s +Step 4240/44000 | Loss: 3.4037 | LR: 4.92e-04 | 735.4ms/step | 27.2 samples/s +Step 4250/44000 | Loss: 3.4433 | LR: 4.92e-04 | 724.8ms/step | 27.6 samples/s +Validation at step 4250: Loss = 3.3663, Perplexity = 28.97 +Step 4260/44000 | Loss: 3.4047 | LR: 4.92e-04 | 950.3ms/step | 21.0 samples/s +Step 4270/44000 | Loss: 3.4093 | LR: 4.92e-04 | 726.2ms/step | 27.5 samples/s +Step 4280/44000 | Loss: 3.3966 | LR: 4.92e-04 | 724.9ms/step | 27.6 samples/s +Step 4290/44000 | Loss: 3.3528 | LR: 4.92e-04 | 725.2ms/step | 27.6 samples/s +Step 4300/44000 | Loss: 3.4086 | LR: 4.92e-04 | 725.3ms/step | 27.6 samples/s +Step 4310/44000 | Loss: 3.3031 | LR: 4.92e-04 | 725.0ms/step | 27.6 samples/s +Step 4320/44000 | Loss: 3.3819 | LR: 4.92e-04 | 725.9ms/step | 27.6 samples/s +Step 4330/44000 | Loss: 3.4208 | LR: 4.92e-04 | 736.3ms/step | 27.2 samples/s +Step 4340/44000 | Loss: 3.3902 | LR: 4.92e-04 | 725.8ms/step | 27.6 samples/s +Step 4350/44000 | Loss: 3.4397 | LR: 4.92e-04 | 724.4ms/step | 27.6 samples/s +Step 4360/44000 | Loss: 3.3852 | LR: 4.91e-04 | 724.9ms/step | 27.6 samples/s +Step 4370/44000 | Loss: 3.3041 | LR: 4.91e-04 | 725.0ms/step | 27.6 samples/s +Step 4380/44000 | Loss: 3.3405 | LR: 4.91e-04 | 725.4ms/step | 27.6 samples/s +Step 4390/44000 | Loss: 3.3994 | LR: 4.91e-04 | 727.0ms/step | 27.5 samples/s +Step 4400/44000 | Loss: 3.4051 | LR: 4.91e-04 | 726.7ms/step | 27.5 samples/s +Step 4410/44000 | Loss: 3.4353 | LR: 4.91e-04 | 729.5ms/step | 27.4 samples/s +Step 4420/44000 | Loss: 3.3650 | LR: 4.91e-04 | 731.9ms/step | 27.3 samples/s +Step 4430/44000 | Loss: 3.3511 | LR: 4.91e-04 | 724.9ms/step | 27.6 samples/s +Step 4440/44000 | Loss: 3.3432 | LR: 4.91e-04 | 725.9ms/step | 27.6 samples/s +Step 4450/44000 | Loss: 3.3234 | LR: 4.91e-04 | 726.1ms/step | 27.5 samples/s +Step 4460/44000 | Loss: 3.3327 | LR: 4.91e-04 | 727.5ms/step | 27.5 samples/s +Step 4470/44000 | Loss: 3.3660 | LR: 4.91e-04 | 726.2ms/step | 27.5 samples/s +Step 4480/44000 | Loss: 3.4160 | LR: 4.91e-04 | 724.8ms/step | 27.6 samples/s +Step 4490/44000 | Loss: 3.3746 | LR: 4.91e-04 | 724.8ms/step | 27.6 samples/s +Step 4500/44000 | Loss: 3.3793 | LR: 4.91e-04 | 736.1ms/step | 27.2 samples/s +Validation at step 4500: Loss = 3.3432, Perplexity = 28.31 +Step 4510/44000 | Loss: 3.4066 | LR: 4.91e-04 | 949.3ms/step | 21.1 samples/s +Step 4520/44000 | Loss: 3.4009 | LR: 4.91e-04 | 726.5ms/step | 27.5 samples/s +Step 4530/44000 | Loss: 3.3331 | LR: 4.91e-04 | 726.6ms/step | 27.5 samples/s +Step 4540/44000 | Loss: 3.3731 | LR: 4.91e-04 | 724.8ms/step | 27.6 samples/s +Step 4550/44000 | Loss: 3.4373 | LR: 4.91e-04 | 725.5ms/step | 27.6 samples/s +Step 4560/44000 | Loss: 3.3699 | LR: 4.90e-04 | 726.2ms/step | 27.5 samples/s +Step 4570/44000 | Loss: 3.3627 | LR: 4.90e-04 | 725.3ms/step | 27.6 samples/s +Step 4580/44000 | Loss: 3.3266 | LR: 4.90e-04 | 731.2ms/step | 27.4 samples/s +Step 4590/44000 | Loss: 3.3123 | LR: 4.90e-04 | 731.6ms/step | 27.3 samples/s +Step 4600/44000 | Loss: 3.3910 | LR: 4.90e-04 | 725.5ms/step | 27.6 samples/s +Step 4610/44000 | Loss: 3.3191 | LR: 4.90e-04 | 724.4ms/step | 27.6 samples/s +Step 4620/44000 | Loss: 3.2905 | LR: 4.90e-04 | 725.6ms/step | 27.6 samples/s +Step 4630/44000 | Loss: 3.3624 | LR: 4.90e-04 | 725.5ms/step | 27.6 samples/s +Step 4640/44000 | Loss: 3.3422 | LR: 4.90e-04 | 726.1ms/step | 27.5 samples/s +Step 4650/44000 | Loss: 3.3788 | LR: 4.90e-04 | 726.5ms/step | 27.5 samples/s +Step 4660/44000 | Loss: 3.3262 | LR: 4.90e-04 | 725.6ms/step | 27.6 samples/s +Step 4670/44000 | Loss: 3.3664 | LR: 4.90e-04 | 729.2ms/step | 27.4 samples/s +Step 4680/44000 | Loss: 3.3671 | LR: 4.90e-04 | 729.5ms/step | 27.4 samples/s +Step 4690/44000 | Loss: 3.2813 | LR: 4.90e-04 | 725.0ms/step | 27.6 samples/s +Step 4700/44000 | Loss: 3.4100 | LR: 4.90e-04 | 724.5ms/step | 27.6 samples/s +Step 4710/44000 | Loss: 3.3753 | LR: 4.90e-04 | 725.9ms/step | 27.6 samples/s +Step 4720/44000 | Loss: 3.3212 | LR: 4.90e-04 | 726.2ms/step | 27.5 samples/s +Step 4730/44000 | Loss: 3.3581 | LR: 4.90e-04 | 724.9ms/step | 27.6 samples/s +Step 4740/44000 | Loss: 3.3492 | LR: 4.90e-04 | 724.8ms/step | 27.6 samples/s +Step 4750/44000 | Loss: 3.3196 | LR: 4.90e-04 | 724.3ms/step | 27.6 samples/s +Validation at step 4750: Loss = 3.3344, Perplexity = 28.06 +Step 4760/44000 | Loss: 3.3492 | LR: 4.89e-04 | 964.1ms/step | 20.7 samples/s +Step 4770/44000 | Loss: 3.3462 | LR: 4.89e-04 | 726.4ms/step | 27.5 samples/s +Step 4780/44000 | Loss: 3.3741 | LR: 4.89e-04 | 726.7ms/step | 27.5 samples/s +Step 4790/44000 | Loss: 3.3450 | LR: 4.89e-04 | 725.6ms/step | 27.6 samples/s +Step 4800/44000 | Loss: 3.3224 | LR: 4.89e-04 | 724.9ms/step | 27.6 samples/s +Step 4810/44000 | Loss: 3.3660 | LR: 4.89e-04 | 724.6ms/step | 27.6 samples/s +Step 4820/44000 | Loss: 3.2777 | LR: 4.89e-04 | 724.8ms/step | 27.6 samples/s +Step 4830/44000 | Loss: 3.3531 | LR: 4.89e-04 | 725.1ms/step | 27.6 samples/s +Step 4840/44000 | Loss: 3.3311 | LR: 4.89e-04 | 731.5ms/step | 27.3 samples/s +Step 4850/44000 | Loss: 3.3976 | LR: 4.89e-04 | 731.1ms/step | 27.4 samples/s +Step 4860/44000 | Loss: 3.3524 | LR: 4.89e-04 | 725.6ms/step | 27.6 samples/s +Step 4870/44000 | Loss: 3.3168 | LR: 4.89e-04 | 724.9ms/step | 27.6 samples/s +Step 4880/44000 | Loss: 3.3233 | LR: 4.89e-04 | 725.2ms/step | 27.6 samples/s +Step 4890/44000 | Loss: 3.3803 | LR: 4.89e-04 | 725.6ms/step | 27.6 samples/s +Step 4900/44000 | Loss: 3.2778 | LR: 4.89e-04 | 726.4ms/step | 27.5 samples/s +Step 4910/44000 | Loss: 3.3534 | LR: 4.89e-04 | 726.9ms/step | 27.5 samples/s +Step 4920/44000 | Loss: 3.3876 | LR: 4.89e-04 | 725.2ms/step | 27.6 samples/s +Step 4930/44000 | Loss: 3.3218 | LR: 4.89e-04 | 734.0ms/step | 27.2 samples/s +Step 4940/44000 | Loss: 3.3362 | LR: 4.89e-04 | 724.7ms/step | 27.6 samples/s +Step 4950/44000 | Loss: 3.2987 | LR: 4.88e-04 | 725.1ms/step | 27.6 samples/s +Step 4960/44000 | Loss: 3.3825 | LR: 4.88e-04 | 725.3ms/step | 27.6 samples/s +Step 4970/44000 | Loss: 3.3552 | LR: 4.88e-04 | 726.9ms/step | 27.5 samples/s +Step 4980/44000 | Loss: 3.2711 | LR: 4.88e-04 | 726.4ms/step | 27.5 samples/s +Step 4990/44000 | Loss: 3.3139 | LR: 4.88e-04 | 724.7ms/step | 27.6 samples/s +Step 5000/44000 | Loss: 3.3959 | LR: 4.88e-04 | 725.9ms/step | 27.6 samples/s +Validation at step 5000: Loss = 3.3305, Perplexity = 27.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-5000 +Step 5010/44000 | Loss: 3.3137 | LR: 4.88e-04 | 1372.4ms/step | 14.6 samples/s +Step 5020/44000 | Loss: 3.3376 | LR: 4.88e-04 | 725.7ms/step | 27.6 samples/s +Step 5030/44000 | Loss: 3.3630 | LR: 4.88e-04 | 726.1ms/step | 27.5 samples/s +Step 5040/44000 | Loss: 3.2786 | LR: 4.88e-04 | 725.0ms/step | 27.6 samples/s +Step 5050/44000 | Loss: 3.2459 | LR: 4.88e-04 | 724.3ms/step | 27.6 samples/s +Step 5060/44000 | Loss: 3.3111 | LR: 4.88e-04 | 724.4ms/step | 27.6 samples/s +Step 5070/44000 | Loss: 3.3871 | LR: 4.88e-04 | 725.3ms/step | 27.6 samples/s +Step 5080/44000 | Loss: 3.3466 | LR: 4.88e-04 | 725.0ms/step | 27.6 samples/s +Step 5090/44000 | Loss: 3.3221 | LR: 4.88e-04 | 726.5ms/step | 27.5 samples/s +Step 5100/44000 | Loss: 3.3127 | LR: 4.88e-04 | 736.1ms/step | 27.2 samples/s +Step 5110/44000 | Loss: 3.2835 | LR: 4.88e-04 | 725.2ms/step | 27.6 samples/s +Step 5120/44000 | Loss: 3.2499 | LR: 4.88e-04 | 724.8ms/step | 27.6 samples/s +Step 5130/44000 | Loss: 3.2574 | LR: 4.87e-04 | 725.0ms/step | 27.6 samples/s +Step 5140/44000 | Loss: 3.3733 | LR: 4.87e-04 | 725.2ms/step | 27.6 samples/s +Step 5150/44000 | Loss: 3.3218 | LR: 4.87e-04 | 725.8ms/step | 27.6 samples/s +Step 5160/44000 | Loss: 3.3630 | LR: 4.87e-04 | 725.9ms/step | 27.6 samples/s +Step 5170/44000 | Loss: 3.3253 | LR: 4.87e-04 | 725.0ms/step | 27.6 samples/s +Step 5180/44000 | Loss: 3.4085 | LR: 4.87e-04 | 729.3ms/step | 27.4 samples/s +Step 5190/44000 | Loss: 3.3683 | LR: 4.87e-04 | 728.9ms/step | 27.4 samples/s +Step 5200/44000 | Loss: 3.3427 | LR: 4.87e-04 | 724.5ms/step | 27.6 samples/s +Step 5210/44000 | Loss: 3.3223 | LR: 4.87e-04 | 726.1ms/step | 27.5 samples/s +Step 5220/44000 | Loss: 3.3180 | LR: 4.87e-04 | 726.7ms/step | 27.5 samples/s +Step 5230/44000 | Loss: 3.2304 | LR: 4.87e-04 | 726.5ms/step | 27.5 samples/s +Step 5240/44000 | Loss: 3.3322 | LR: 4.87e-04 | 724.8ms/step | 27.6 samples/s +Step 5250/44000 | Loss: 3.3285 | LR: 4.87e-04 | 725.3ms/step | 27.6 samples/s +Validation at step 5250: Loss = 3.3014, Perplexity = 27.15 +Step 5260/44000 | Loss: 3.2539 | LR: 4.87e-04 | 949.0ms/step | 21.1 samples/s +Step 5270/44000 | Loss: 3.2307 | LR: 4.87e-04 | 737.6ms/step | 27.1 samples/s +Step 5280/44000 | Loss: 3.3573 | LR: 4.87e-04 | 725.9ms/step | 27.6 samples/s +Step 5290/44000 | Loss: 3.3964 | LR: 4.87e-04 | 726.2ms/step | 27.5 samples/s +Step 5300/44000 | Loss: 3.2868 | LR: 4.86e-04 | 725.5ms/step | 27.6 samples/s +Step 5310/44000 | Loss: 3.3369 | LR: 4.86e-04 | 724.6ms/step | 27.6 samples/s +Step 5320/44000 | Loss: 3.3160 | LR: 4.86e-04 | 725.3ms/step | 27.6 samples/s +Step 5330/44000 | Loss: 3.2253 | LR: 4.86e-04 | 724.6ms/step | 27.6 samples/s +Step 5340/44000 | Loss: 3.3094 | LR: 4.86e-04 | 727.0ms/step | 27.5 samples/s +Step 5350/44000 | Loss: 3.3264 | LR: 4.86e-04 | 726.4ms/step | 27.5 samples/s +Step 5360/44000 | Loss: 3.3046 | LR: 4.86e-04 | 738.1ms/step | 27.1 samples/s +Step 5370/44000 | Loss: 3.3001 | LR: 4.86e-04 | 725.5ms/step | 27.6 samples/s +Step 5380/44000 | Loss: 3.2662 | LR: 4.86e-04 | 725.3ms/step | 27.6 samples/s +Step 5390/44000 | Loss: 3.3339 | LR: 4.86e-04 | 725.9ms/step | 27.6 samples/s +Step 5400/44000 | Loss: 3.2854 | LR: 4.86e-04 | 726.3ms/step | 27.5 samples/s +Step 5410/44000 | Loss: 3.2901 | LR: 4.86e-04 | 727.5ms/step | 27.5 samples/s +Step 5420/44000 | Loss: 3.2960 | LR: 4.86e-04 | 726.8ms/step | 27.5 samples/s +Step 5430/44000 | Loss: 3.3181 | LR: 4.86e-04 | 725.4ms/step | 27.6 samples/s +Step 5440/44000 | Loss: 3.2678 | LR: 4.86e-04 | 729.9ms/step | 27.4 samples/s +Step 5450/44000 | Loss: 3.3154 | LR: 4.86e-04 | 730.7ms/step | 27.4 samples/s +Step 5460/44000 | Loss: 3.2985 | LR: 4.86e-04 | 725.6ms/step | 27.6 samples/s +Step 5470/44000 | Loss: 3.3304 | LR: 4.85e-04 | 726.3ms/step | 27.5 samples/s +Step 5480/44000 | Loss: 3.2971 | LR: 4.85e-04 | 726.4ms/step | 27.5 samples/s +Step 5490/44000 | Loss: 3.3045 | LR: 4.85e-04 | 726.3ms/step | 27.5 samples/s +Step 5500/44000 | Loss: 3.2780 | LR: 4.85e-04 | 725.6ms/step | 27.6 samples/s +Validation at step 5500: Loss = 3.2883, Perplexity = 26.80 +Step 5510/44000 | Loss: 3.3303 | LR: 4.85e-04 | 951.5ms/step | 21.0 samples/s +Step 5520/44000 | Loss: 3.3285 | LR: 4.85e-04 | 725.7ms/step | 27.6 samples/s +Step 5530/44000 | Loss: 3.2667 | LR: 4.85e-04 | 738.7ms/step | 27.1 samples/s +Step 5540/44000 | Loss: 3.3188 | LR: 4.85e-04 | 726.6ms/step | 27.5 samples/s +Step 5550/44000 | Loss: 3.3796 | LR: 4.85e-04 | 726.3ms/step | 27.5 samples/s +Step 5560/44000 | Loss: 3.2630 | LR: 4.85e-04 | 725.9ms/step | 27.6 samples/s +Step 5570/44000 | Loss: 3.3191 | LR: 4.85e-04 | 725.6ms/step | 27.6 samples/s +Step 5580/44000 | Loss: 3.3020 | LR: 4.85e-04 | 726.2ms/step | 27.5 samples/s +Step 5590/44000 | Loss: 3.2738 | LR: 4.85e-04 | 726.2ms/step | 27.5 samples/s +Step 5600/44000 | Loss: 3.3321 | LR: 4.85e-04 | 727.7ms/step | 27.5 samples/s +Step 5610/44000 | Loss: 3.3495 | LR: 4.85e-04 | 732.4ms/step | 27.3 samples/s +Step 5620/44000 | Loss: 3.3292 | LR: 4.85e-04 | 730.6ms/step | 27.4 samples/s +Step 5630/44000 | Loss: 3.2545 | LR: 4.84e-04 | 726.0ms/step | 27.5 samples/s +Step 5640/44000 | Loss: 3.2997 | LR: 4.84e-04 | 725.8ms/step | 27.6 samples/s +Step 5650/44000 | Loss: 3.3296 | LR: 4.84e-04 | 726.0ms/step | 27.5 samples/s +Step 5660/44000 | Loss: 3.2791 | LR: 4.84e-04 | 726.6ms/step | 27.5 samples/s +Step 5670/44000 | Loss: 3.2485 | LR: 4.84e-04 | 727.9ms/step | 27.5 samples/s +Step 5680/44000 | Loss: 3.3273 | LR: 4.84e-04 | 727.3ms/step | 27.5 samples/s +Step 5690/44000 | Loss: 3.2632 | LR: 4.84e-04 | 725.4ms/step | 27.6 samples/s +Step 5700/44000 | Loss: 3.2874 | LR: 4.84e-04 | 733.2ms/step | 27.3 samples/s +Step 5710/44000 | Loss: 3.2005 | LR: 4.84e-04 | 732.1ms/step | 27.3 samples/s +Step 5720/44000 | Loss: 3.1850 | LR: 4.84e-04 | 725.7ms/step | 27.6 samples/s +Step 5730/44000 | Loss: 3.2858 | LR: 4.84e-04 | 727.1ms/step | 27.5 samples/s +Step 5740/44000 | Loss: 3.2750 | LR: 4.84e-04 | 727.3ms/step | 27.5 samples/s +Step 5750/44000 | Loss: 3.2264 | LR: 4.84e-04 | 725.5ms/step | 27.6 samples/s +Validation at step 5750: Loss = 3.2734, Perplexity = 26.40 +Step 5760/44000 | Loss: 3.2493 | LR: 4.84e-04 | 949.7ms/step | 21.1 samples/s +Step 5770/44000 | Loss: 3.3451 | LR: 4.84e-04 | 725.6ms/step | 27.6 samples/s +Step 5780/44000 | Loss: 3.2131 | LR: 4.83e-04 | 725.5ms/step | 27.6 samples/s +Step 5790/44000 | Loss: 3.2571 | LR: 4.83e-04 | 736.8ms/step | 27.1 samples/s +Step 5800/44000 | Loss: 3.2594 | LR: 4.83e-04 | 727.2ms/step | 27.5 samples/s +Step 5810/44000 | Loss: 3.2940 | LR: 4.83e-04 | 725.7ms/step | 27.6 samples/s +Step 5820/44000 | Loss: 3.3227 | LR: 4.83e-04 | 725.8ms/step | 27.6 samples/s +Step 5830/44000 | Loss: 3.3234 | LR: 4.83e-04 | 725.5ms/step | 27.6 samples/s +Step 5840/44000 | Loss: 3.2657 | LR: 4.83e-04 | 726.4ms/step | 27.5 samples/s +Step 5850/44000 | Loss: 3.2135 | LR: 4.83e-04 | 727.3ms/step | 27.5 samples/s +Step 5860/44000 | Loss: 3.2867 | LR: 4.83e-04 | 727.1ms/step | 27.5 samples/s +Step 5870/44000 | Loss: 3.2749 | LR: 4.83e-04 | 731.6ms/step | 27.3 samples/s +Step 5880/44000 | Loss: 3.3167 | LR: 4.83e-04 | 730.7ms/step | 27.4 samples/s +Step 5890/44000 | Loss: 3.2904 | LR: 4.83e-04 | 725.8ms/step | 27.6 samples/s +Step 5900/44000 | Loss: 3.2164 | LR: 4.83e-04 | 725.7ms/step | 27.6 samples/s +Step 5910/44000 | Loss: 3.3009 | LR: 4.83e-04 | 727.0ms/step | 27.5 samples/s +Step 5920/44000 | Loss: 3.3490 | LR: 4.83e-04 | 726.5ms/step | 27.5 samples/s +Step 5930/44000 | Loss: 3.2430 | LR: 4.83e-04 | 727.5ms/step | 27.5 samples/s +Step 5940/44000 | Loss: 3.3445 | LR: 4.82e-04 | 725.4ms/step | 27.6 samples/s +Step 5950/44000 | Loss: 3.2797 | LR: 4.82e-04 | 725.2ms/step | 27.6 samples/s +Step 5960/44000 | Loss: 3.2537 | LR: 4.82e-04 | 737.2ms/step | 27.1 samples/s +Step 5970/44000 | Loss: 3.2504 | LR: 4.82e-04 | 724.9ms/step | 27.6 samples/s +Step 5980/44000 | Loss: 3.2887 | LR: 4.82e-04 | 725.7ms/step | 27.6 samples/s +Step 5990/44000 | Loss: 3.2922 | LR: 4.82e-04 | 726.6ms/step | 27.5 samples/s +Step 6000/44000 | Loss: 3.2641 | LR: 4.82e-04 | 726.7ms/step | 27.5 samples/s +Validation at step 6000: Loss = 3.2640, Perplexity = 26.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-6000 +Step 6010/44000 | Loss: 3.2952 | LR: 4.82e-04 | 1440.7ms/step | 13.9 samples/s +Step 6020/44000 | Loss: 3.2453 | LR: 4.82e-04 | 724.9ms/step | 27.6 samples/s +Step 6030/44000 | Loss: 3.3206 | LR: 4.82e-04 | 725.9ms/step | 27.6 samples/s +Step 6040/44000 | Loss: 3.2331 | LR: 4.82e-04 | 740.3ms/step | 27.0 samples/s +Step 6050/44000 | Loss: 3.3375 | LR: 4.82e-04 | 727.8ms/step | 27.5 samples/s +Step 6060/44000 | Loss: 3.2681 | LR: 4.82e-04 | 725.7ms/step | 27.6 samples/s +Step 6070/44000 | Loss: 3.2919 | LR: 4.82e-04 | 725.5ms/step | 27.6 samples/s +Step 6080/44000 | Loss: 3.3033 | LR: 4.81e-04 | 725.6ms/step | 27.6 samples/s +Step 6090/44000 | Loss: 3.2370 | LR: 4.81e-04 | 725.7ms/step | 27.6 samples/s +Step 6100/44000 | Loss: 3.2643 | LR: 4.81e-04 | 726.0ms/step | 27.5 samples/s +Step 6110/44000 | Loss: 3.3012 | LR: 4.81e-04 | 727.1ms/step | 27.5 samples/s +Step 6120/44000 | Loss: 3.2530 | LR: 4.81e-04 | 726.1ms/step | 27.5 samples/s +Step 6130/44000 | Loss: 3.2298 | LR: 4.81e-04 | 734.5ms/step | 27.2 samples/s +Step 6140/44000 | Loss: 3.2668 | LR: 4.81e-04 | 726.1ms/step | 27.5 samples/s +Step 6150/44000 | Loss: 3.2721 | LR: 4.81e-04 | 725.3ms/step | 27.6 samples/s +Step 6160/44000 | Loss: 3.2526 | LR: 4.81e-04 | 725.3ms/step | 27.6 samples/s +Step 6170/44000 | Loss: 3.2943 | LR: 4.81e-04 | 726.7ms/step | 27.5 samples/s +Step 6180/44000 | Loss: 3.2564 | LR: 4.81e-04 | 727.1ms/step | 27.5 samples/s +Step 6190/44000 | Loss: 3.2777 | LR: 4.81e-04 | 726.0ms/step | 27.5 samples/s +Step 6200/44000 | Loss: 3.3037 | LR: 4.81e-04 | 725.5ms/step | 27.6 samples/s +Step 6210/44000 | Loss: 3.3063 | LR: 4.81e-04 | 731.6ms/step | 27.3 samples/s +Step 6220/44000 | Loss: 3.2548 | LR: 4.81e-04 | 733.9ms/step | 27.3 samples/s +Step 6230/44000 | Loss: 3.3008 | LR: 4.80e-04 | 727.2ms/step | 27.5 samples/s +Step 6240/44000 | Loss: 3.2598 | LR: 4.80e-04 | 727.4ms/step | 27.5 samples/s +Step 6250/44000 | Loss: 3.2801 | LR: 4.80e-04 | 726.7ms/step | 27.5 samples/s +Validation at step 6250: Loss = 3.2514, Perplexity = 25.83 +Step 6260/44000 | Loss: 3.2512 | LR: 4.80e-04 | 951.1ms/step | 21.0 samples/s +Step 6270/44000 | Loss: 3.3208 | LR: 4.80e-04 | 725.5ms/step | 27.6 samples/s +Step 6280/44000 | Loss: 3.2493 | LR: 4.80e-04 | 725.0ms/step | 27.6 samples/s +Step 6290/44000 | Loss: 3.2753 | LR: 4.80e-04 | 725.4ms/step | 27.6 samples/s +Step 6300/44000 | Loss: 3.2739 | LR: 4.80e-04 | 736.2ms/step | 27.2 samples/s +Step 6310/44000 | Loss: 3.3439 | LR: 4.80e-04 | 726.3ms/step | 27.5 samples/s +Step 6320/44000 | Loss: 3.2624 | LR: 4.80e-04 | 725.3ms/step | 27.6 samples/s +Step 6330/44000 | Loss: 3.2735 | LR: 4.80e-04 | 725.6ms/step | 27.6 samples/s +Step 6340/44000 | Loss: 3.2677 | LR: 4.80e-04 | 725.6ms/step | 27.6 samples/s +Step 6350/44000 | Loss: 3.1516 | LR: 4.80e-04 | 725.4ms/step | 27.6 samples/s +Step 6360/44000 | Loss: 3.2302 | LR: 4.80e-04 | 726.8ms/step | 27.5 samples/s +Step 6370/44000 | Loss: 3.2450 | LR: 4.79e-04 | 727.1ms/step | 27.5 samples/s +Step 6380/44000 | Loss: 3.2703 | LR: 4.79e-04 | 733.2ms/step | 27.3 samples/s +Step 6390/44000 | Loss: 3.2424 | LR: 4.79e-04 | 731.1ms/step | 27.4 samples/s +Step 6400/44000 | Loss: 3.2751 | LR: 4.79e-04 | 725.7ms/step | 27.6 samples/s +Step 6410/44000 | Loss: 3.2029 | LR: 4.79e-04 | 725.5ms/step | 27.6 samples/s +Step 6420/44000 | Loss: 3.2762 | LR: 4.79e-04 | 725.9ms/step | 27.6 samples/s +Step 6430/44000 | Loss: 3.2095 | LR: 4.79e-04 | 726.8ms/step | 27.5 samples/s +Step 6440/44000 | Loss: 3.2121 | LR: 4.79e-04 | 726.4ms/step | 27.5 samples/s +Step 6450/44000 | Loss: 3.2231 | LR: 4.79e-04 | 725.4ms/step | 27.6 samples/s +Step 6460/44000 | Loss: 3.2457 | LR: 4.79e-04 | 725.7ms/step | 27.6 samples/s +Step 6470/44000 | Loss: 3.3038 | LR: 4.79e-04 | 734.5ms/step | 27.2 samples/s +Step 6480/44000 | Loss: 3.2457 | LR: 4.79e-04 | 725.5ms/step | 27.6 samples/s +Step 6490/44000 | Loss: 3.2526 | LR: 4.79e-04 | 726.9ms/step | 27.5 samples/s +Step 6500/44000 | Loss: 3.2829 | LR: 4.79e-04 | 726.5ms/step | 27.5 samples/s +Validation at step 6500: Loss = 3.2396, Perplexity = 25.52 +Step 6510/44000 | Loss: 3.2271 | LR: 4.78e-04 | 950.0ms/step | 21.1 samples/s +Step 6520/44000 | Loss: 3.2048 | LR: 4.78e-04 | 725.7ms/step | 27.6 samples/s +Step 6530/44000 | Loss: 3.2590 | LR: 4.78e-04 | 725.7ms/step | 27.6 samples/s +Step 6540/44000 | Loss: 3.2815 | LR: 4.78e-04 | 726.0ms/step | 27.5 samples/s +Step 6550/44000 | Loss: 3.2721 | LR: 4.78e-04 | 726.7ms/step | 27.5 samples/s +Step 6560/44000 | Loss: 3.3238 | LR: 4.78e-04 | 737.7ms/step | 27.1 samples/s +Step 6570/44000 | Loss: 3.2430 | LR: 4.78e-04 | 725.3ms/step | 27.6 samples/s +Step 6580/44000 | Loss: 3.3066 | LR: 4.78e-04 | 725.7ms/step | 27.6 samples/s +Step 6590/44000 | Loss: 3.2521 | LR: 4.78e-04 | 725.7ms/step | 27.6 samples/s +Step 6600/44000 | Loss: 3.2375 | LR: 4.78e-04 | 725.6ms/step | 27.6 samples/s +Step 6610/44000 | Loss: 3.3027 | LR: 4.78e-04 | 726.3ms/step | 27.5 samples/s +Step 6620/44000 | Loss: 3.3164 | LR: 4.78e-04 | 727.0ms/step | 27.5 samples/s +Step 6630/44000 | Loss: 3.1966 | LR: 4.78e-04 | 726.6ms/step | 27.5 samples/s +Step 6640/44000 | Loss: 3.2402 | LR: 4.77e-04 | 733.6ms/step | 27.3 samples/s +Step 6650/44000 | Loss: 3.2916 | LR: 4.77e-04 | 730.8ms/step | 27.4 samples/s +Step 6660/44000 | Loss: 3.2695 | LR: 4.77e-04 | 726.0ms/step | 27.5 samples/s +Step 6670/44000 | Loss: 3.2423 | LR: 4.77e-04 | 725.6ms/step | 27.6 samples/s +Step 6680/44000 | Loss: 3.2442 | LR: 4.77e-04 | 727.0ms/step | 27.5 samples/s +Step 6690/44000 | Loss: 3.2603 | LR: 4.77e-04 | 726.9ms/step | 27.5 samples/s +Step 6700/44000 | Loss: 3.2327 | LR: 4.77e-04 | 725.5ms/step | 27.6 samples/s +Step 6710/44000 | Loss: 3.2783 | LR: 4.77e-04 | 725.3ms/step | 27.6 samples/s +Step 6720/44000 | Loss: 3.2271 | LR: 4.77e-04 | 725.3ms/step | 27.6 samples/s +Step 6730/44000 | Loss: 3.2836 | LR: 4.77e-04 | 734.6ms/step | 27.2 samples/s +Step 6740/44000 | Loss: 3.2734 | LR: 4.77e-04 | 726.1ms/step | 27.5 samples/s +Step 6750/44000 | Loss: 3.2648 | LR: 4.77e-04 | 727.6ms/step | 27.5 samples/s +Validation at step 6750: Loss = 3.2327, Perplexity = 25.35 +Step 6760/44000 | Loss: 3.2902 | LR: 4.77e-04 | 952.8ms/step | 21.0 samples/s +Step 6770/44000 | Loss: 3.2542 | LR: 4.76e-04 | 725.5ms/step | 27.6 samples/s +Step 6780/44000 | Loss: 3.2391 | LR: 4.76e-04 | 726.2ms/step | 27.5 samples/s +Step 6790/44000 | Loss: 3.2384 | LR: 4.76e-04 | 726.3ms/step | 27.5 samples/s +Step 6800/44000 | Loss: 3.2597 | LR: 4.76e-04 | 726.2ms/step | 27.5 samples/s +Step 6810/44000 | Loss: 3.2095 | LR: 4.76e-04 | 735.1ms/step | 27.2 samples/s +Step 6820/44000 | Loss: 3.2636 | LR: 4.76e-04 | 732.4ms/step | 27.3 samples/s +Step 6830/44000 | Loss: 3.2784 | LR: 4.76e-04 | 725.9ms/step | 27.6 samples/s +Step 6840/44000 | Loss: 3.2144 | LR: 4.76e-04 | 726.3ms/step | 27.5 samples/s +Step 6850/44000 | Loss: 3.2019 | LR: 4.76e-04 | 725.9ms/step | 27.6 samples/s +Step 6860/44000 | Loss: 3.2349 | LR: 4.76e-04 | 725.5ms/step | 27.6 samples/s +Step 6870/44000 | Loss: 3.2641 | LR: 4.76e-04 | 726.0ms/step | 27.5 samples/s +Step 6880/44000 | Loss: 3.2470 | LR: 4.76e-04 | 726.8ms/step | 27.5 samples/s +Step 6890/44000 | Loss: 3.2680 | LR: 4.76e-04 | 726.1ms/step | 27.5 samples/s +Step 6900/44000 | Loss: 3.2161 | LR: 4.75e-04 | 730.1ms/step | 27.4 samples/s +Step 6910/44000 | Loss: 3.2415 | LR: 4.75e-04 | 730.6ms/step | 27.4 samples/s +Step 6920/44000 | Loss: 3.2161 | LR: 4.75e-04 | 725.6ms/step | 27.6 samples/s +Step 6930/44000 | Loss: 3.1813 | LR: 4.75e-04 | 726.1ms/step | 27.5 samples/s +Step 6940/44000 | Loss: 3.2508 | LR: 4.75e-04 | 728.4ms/step | 27.5 samples/s +Step 6950/44000 | Loss: 3.2940 | LR: 4.75e-04 | 727.0ms/step | 27.5 samples/s +Step 6960/44000 | Loss: 3.1557 | LR: 4.75e-04 | 725.2ms/step | 27.6 samples/s +Step 6970/44000 | Loss: 3.2129 | LR: 4.75e-04 | 725.8ms/step | 27.6 samples/s +Step 6980/44000 | Loss: 3.2758 | LR: 4.75e-04 | 725.3ms/step | 27.6 samples/s +Step 6990/44000 | Loss: 3.3006 | LR: 4.75e-04 | 738.3ms/step | 27.1 samples/s +Step 7000/44000 | Loss: 3.2230 | LR: 4.75e-04 | 727.1ms/step | 27.5 samples/s +Validation at step 7000: Loss = 3.2209, Perplexity = 25.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-7000 +Step 7010/44000 | Loss: 3.2732 | LR: 4.75e-04 | 1417.1ms/step | 14.1 samples/s +Step 7020/44000 | Loss: 3.2421 | LR: 4.75e-04 | 724.9ms/step | 27.6 samples/s +Step 7030/44000 | Loss: 3.2745 | LR: 4.74e-04 | 725.0ms/step | 27.6 samples/s +Step 7040/44000 | Loss: 3.2790 | LR: 4.74e-04 | 725.1ms/step | 27.6 samples/s +Step 7050/44000 | Loss: 3.2211 | LR: 4.74e-04 | 725.7ms/step | 27.6 samples/s +Step 7060/44000 | Loss: 3.2438 | LR: 4.74e-04 | 726.6ms/step | 27.5 samples/s +Step 7070/44000 | Loss: 3.3178 | LR: 4.74e-04 | 736.1ms/step | 27.2 samples/s +Step 7080/44000 | Loss: 3.2471 | LR: 4.74e-04 | 725.7ms/step | 27.6 samples/s +Step 7090/44000 | Loss: 3.1706 | LR: 4.74e-04 | 725.8ms/step | 27.6 samples/s +Step 7100/44000 | Loss: 3.2095 | LR: 4.74e-04 | 725.5ms/step | 27.6 samples/s +Step 7110/44000 | Loss: 3.2485 | LR: 4.74e-04 | 726.0ms/step | 27.5 samples/s +Step 7120/44000 | Loss: 3.2374 | LR: 4.74e-04 | 727.8ms/step | 27.5 samples/s +Step 7130/44000 | Loss: 3.2051 | LR: 4.74e-04 | 727.2ms/step | 27.5 samples/s +Step 7140/44000 | Loss: 3.2125 | LR: 4.74e-04 | 725.9ms/step | 27.6 samples/s +Step 7150/44000 | Loss: 3.3368 | LR: 4.73e-04 | 730.3ms/step | 27.4 samples/s +Step 7160/44000 | Loss: 3.2483 | LR: 4.73e-04 | 733.8ms/step | 27.3 samples/s +Step 7170/44000 | Loss: 3.2162 | LR: 4.73e-04 | 725.5ms/step | 27.6 samples/s +Step 7180/44000 | Loss: 3.1686 | LR: 4.73e-04 | 726.9ms/step | 27.5 samples/s +Step 7190/44000 | Loss: 3.2153 | LR: 4.73e-04 | 727.6ms/step | 27.5 samples/s +Step 7200/44000 | Loss: 3.3053 | LR: 4.73e-04 | 726.6ms/step | 27.5 samples/s +Step 7210/44000 | Loss: 3.1756 | LR: 4.73e-04 | 725.5ms/step | 27.6 samples/s +Step 7220/44000 | Loss: 3.1770 | LR: 4.73e-04 | 725.5ms/step | 27.6 samples/s +Step 7230/44000 | Loss: 3.2449 | LR: 4.73e-04 | 725.9ms/step | 27.6 samples/s +Step 7240/44000 | Loss: 3.2144 | LR: 4.73e-04 | 734.6ms/step | 27.2 samples/s +Step 7250/44000 | Loss: 3.2412 | LR: 4.73e-04 | 726.7ms/step | 27.5 samples/s +Validation at step 7250: Loss = 3.2118, Perplexity = 24.82 +Step 7260/44000 | Loss: 3.2914 | LR: 4.73e-04 | 953.8ms/step | 21.0 samples/s +Step 7270/44000 | Loss: 3.2083 | LR: 4.72e-04 | 725.9ms/step | 27.6 samples/s +Step 7280/44000 | Loss: 3.2362 | LR: 4.72e-04 | 726.0ms/step | 27.5 samples/s +Step 7290/44000 | Loss: 3.2048 | LR: 4.72e-04 | 726.2ms/step | 27.5 samples/s +Step 7300/44000 | Loss: 3.2180 | LR: 4.72e-04 | 726.2ms/step | 27.5 samples/s +Step 7310/44000 | Loss: 3.3054 | LR: 4.72e-04 | 726.8ms/step | 27.5 samples/s +Step 7320/44000 | Loss: 3.1605 | LR: 4.72e-04 | 726.9ms/step | 27.5 samples/s +Step 7330/44000 | Loss: 3.2020 | LR: 4.72e-04 | 738.9ms/step | 27.1 samples/s +Step 7340/44000 | Loss: 3.2325 | LR: 4.72e-04 | 726.6ms/step | 27.5 samples/s +Step 7350/44000 | Loss: 3.2144 | LR: 4.72e-04 | 726.1ms/step | 27.5 samples/s +Step 7360/44000 | Loss: 3.2375 | LR: 4.72e-04 | 725.7ms/step | 27.6 samples/s +Step 7370/44000 | Loss: 3.2319 | LR: 4.72e-04 | 726.4ms/step | 27.5 samples/s +Step 7380/44000 | Loss: 3.2568 | LR: 4.72e-04 | 727.4ms/step | 27.5 samples/s +Step 7390/44000 | Loss: 3.1975 | LR: 4.71e-04 | 727.2ms/step | 27.5 samples/s +Step 7400/44000 | Loss: 3.2012 | LR: 4.71e-04 | 726.8ms/step | 27.5 samples/s +Step 7410/44000 | Loss: 3.1928 | LR: 4.71e-04 | 730.9ms/step | 27.4 samples/s +Step 7420/44000 | Loss: 3.1814 | LR: 4.71e-04 | 731.0ms/step | 27.4 samples/s +Step 7430/44000 | Loss: 3.2428 | LR: 4.71e-04 | 725.9ms/step | 27.6 samples/s +Step 7440/44000 | Loss: 3.2233 | LR: 4.71e-04 | 726.7ms/step | 27.5 samples/s +Step 7450/44000 | Loss: 3.1649 | LR: 4.71e-04 | 727.3ms/step | 27.5 samples/s +Step 7460/44000 | Loss: 3.2669 | LR: 4.71e-04 | 725.8ms/step | 27.6 samples/s +Step 7470/44000 | Loss: 3.2213 | LR: 4.71e-04 | 725.8ms/step | 27.6 samples/s +Step 7480/44000 | Loss: 3.1826 | LR: 4.71e-04 | 725.2ms/step | 27.6 samples/s +Step 7490/44000 | Loss: 3.2922 | LR: 4.71e-04 | 725.8ms/step | 27.6 samples/s +Step 7500/44000 | Loss: 3.1132 | LR: 4.71e-04 | 739.4ms/step | 27.0 samples/s +Validation at step 7500: Loss = 3.2048, Perplexity = 24.65 +Step 7510/44000 | Loss: 3.2261 | LR: 4.70e-04 | 952.9ms/step | 21.0 samples/s +Step 7520/44000 | Loss: 3.2296 | LR: 4.70e-04 | 726.6ms/step | 27.5 samples/s +Step 7530/44000 | Loss: 3.2443 | LR: 4.70e-04 | 725.6ms/step | 27.6 samples/s +Step 7540/44000 | Loss: 3.2364 | LR: 4.70e-04 | 726.1ms/step | 27.5 samples/s +Step 7550/44000 | Loss: 3.1569 | LR: 4.70e-04 | 725.4ms/step | 27.6 samples/s +Step 7560/44000 | Loss: 3.2286 | LR: 4.70e-04 | 725.8ms/step | 27.6 samples/s +Step 7570/44000 | Loss: 3.1916 | LR: 4.70e-04 | 727.1ms/step | 27.5 samples/s +Step 7580/44000 | Loss: 3.2495 | LR: 4.70e-04 | 731.4ms/step | 27.3 samples/s +Step 7590/44000 | Loss: 3.1583 | LR: 4.70e-04 | 730.6ms/step | 27.4 samples/s +Step 7600/44000 | Loss: 3.1791 | LR: 4.70e-04 | 725.8ms/step | 27.6 samples/s +Step 7610/44000 | Loss: 3.2268 | LR: 4.70e-04 | 725.7ms/step | 27.6 samples/s +Step 7620/44000 | Loss: 3.1675 | LR: 4.70e-04 | 726.2ms/step | 27.5 samples/s +Step 7630/44000 | Loss: 3.2035 | LR: 4.69e-04 | 726.5ms/step | 27.5 samples/s +Step 7640/44000 | Loss: 3.0922 | LR: 4.69e-04 | 727.3ms/step | 27.5 samples/s +Step 7650/44000 | Loss: 3.2367 | LR: 4.69e-04 | 726.8ms/step | 27.5 samples/s +Step 7660/44000 | Loss: 3.2337 | LR: 4.69e-04 | 725.4ms/step | 27.6 samples/s +Step 7670/44000 | Loss: 3.1949 | LR: 4.69e-04 | 732.5ms/step | 27.3 samples/s +Step 7680/44000 | Loss: 3.2289 | LR: 4.69e-04 | 730.4ms/step | 27.4 samples/s +Step 7690/44000 | Loss: 3.1938 | LR: 4.69e-04 | 725.7ms/step | 27.6 samples/s +Step 7700/44000 | Loss: 3.1245 | LR: 4.69e-04 | 727.5ms/step | 27.5 samples/s +Step 7710/44000 | Loss: 3.2539 | LR: 4.69e-04 | 726.8ms/step | 27.5 samples/s +Step 7720/44000 | Loss: 3.2368 | LR: 4.69e-04 | 725.6ms/step | 27.6 samples/s +Step 7730/44000 | Loss: 3.2330 | LR: 4.69e-04 | 726.4ms/step | 27.5 samples/s +Step 7740/44000 | Loss: 3.2097 | LR: 4.68e-04 | 726.2ms/step | 27.5 samples/s +Step 7750/44000 | Loss: 3.2130 | LR: 4.68e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 7750: Loss = 3.1947, Perplexity = 24.40 +Step 7760/44000 | Loss: 3.2150 | LR: 4.68e-04 | 960.0ms/step | 20.8 samples/s +Step 7770/44000 | Loss: 3.2292 | LR: 4.68e-04 | 727.1ms/step | 27.5 samples/s +Step 7780/44000 | Loss: 3.2415 | LR: 4.68e-04 | 725.6ms/step | 27.6 samples/s +Step 7790/44000 | Loss: 3.2273 | LR: 4.68e-04 | 726.1ms/step | 27.5 samples/s +Step 7800/44000 | Loss: 3.2862 | LR: 4.68e-04 | 725.9ms/step | 27.6 samples/s +Step 7810/44000 | Loss: 3.2213 | LR: 4.68e-04 | 725.9ms/step | 27.6 samples/s +Step 7820/44000 | Loss: 3.2020 | LR: 4.68e-04 | 727.2ms/step | 27.5 samples/s +Step 7830/44000 | Loss: 3.1795 | LR: 4.68e-04 | 727.8ms/step | 27.5 samples/s +Step 7840/44000 | Loss: 3.2471 | LR: 4.68e-04 | 731.3ms/step | 27.3 samples/s +Step 7850/44000 | Loss: 3.1579 | LR: 4.67e-04 | 733.3ms/step | 27.3 samples/s +Step 7860/44000 | Loss: 3.2490 | LR: 4.67e-04 | 725.7ms/step | 27.6 samples/s +Step 7870/44000 | Loss: 3.1691 | LR: 4.67e-04 | 725.9ms/step | 27.6 samples/s +Step 7880/44000 | Loss: 3.1870 | LR: 4.67e-04 | 726.2ms/step | 27.5 samples/s +Step 7890/44000 | Loss: 3.2364 | LR: 4.67e-04 | 727.1ms/step | 27.5 samples/s +Step 7900/44000 | Loss: 3.2054 | LR: 4.67e-04 | 728.5ms/step | 27.5 samples/s +Step 7910/44000 | Loss: 3.1890 | LR: 4.67e-04 | 726.2ms/step | 27.5 samples/s +Step 7920/44000 | Loss: 3.1641 | LR: 4.67e-04 | 725.5ms/step | 27.6 samples/s +Step 7930/44000 | Loss: 3.1400 | LR: 4.67e-04 | 735.2ms/step | 27.2 samples/s +Step 7940/44000 | Loss: 3.2134 | LR: 4.67e-04 | 725.6ms/step | 27.6 samples/s +Step 7950/44000 | Loss: 3.2311 | LR: 4.67e-04 | 726.2ms/step | 27.5 samples/s +Step 7960/44000 | Loss: 3.2682 | LR: 4.66e-04 | 727.4ms/step | 27.5 samples/s +Step 7970/44000 | Loss: 3.1317 | LR: 4.66e-04 | 727.0ms/step | 27.5 samples/s +Step 7980/44000 | Loss: 3.1930 | LR: 4.66e-04 | 726.1ms/step | 27.5 samples/s +Step 7990/44000 | Loss: 3.1849 | LR: 4.66e-04 | 725.6ms/step | 27.6 samples/s +Step 8000/44000 | Loss: 3.1696 | LR: 4.66e-04 | 726.3ms/step | 27.5 samples/s +Validation at step 8000: Loss = 3.1878, Perplexity = 24.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-8000 +Step 8010/44000 | Loss: 3.2008 | LR: 4.66e-04 | 1424.7ms/step | 14.0 samples/s +Step 8020/44000 | Loss: 3.2211 | LR: 4.66e-04 | 726.7ms/step | 27.5 samples/s +Step 8030/44000 | Loss: 3.1615 | LR: 4.66e-04 | 725.9ms/step | 27.6 samples/s +Step 8040/44000 | Loss: 3.1793 | LR: 4.66e-04 | 725.1ms/step | 27.6 samples/s +Step 8050/44000 | Loss: 3.2328 | LR: 4.66e-04 | 725.5ms/step | 27.6 samples/s +Step 8060/44000 | Loss: 3.2569 | LR: 4.66e-04 | 725.3ms/step | 27.6 samples/s +Step 8070/44000 | Loss: 3.1793 | LR: 4.65e-04 | 726.1ms/step | 27.5 samples/s +Step 8080/44000 | Loss: 3.2138 | LR: 4.65e-04 | 727.9ms/step | 27.5 samples/s +Step 8090/44000 | Loss: 3.1698 | LR: 4.65e-04 | 727.3ms/step | 27.5 samples/s +Step 8100/44000 | Loss: 3.2085 | LR: 4.65e-04 | 738.5ms/step | 27.1 samples/s +Step 8110/44000 | Loss: 3.1723 | LR: 4.65e-04 | 726.3ms/step | 27.5 samples/s +Step 8120/44000 | Loss: 3.1336 | LR: 4.65e-04 | 725.1ms/step | 27.6 samples/s +Step 8130/44000 | Loss: 3.2304 | LR: 4.65e-04 | 725.6ms/step | 27.6 samples/s +Step 8140/44000 | Loss: 3.1991 | LR: 4.65e-04 | 726.4ms/step | 27.5 samples/s +Step 8150/44000 | Loss: 3.2194 | LR: 4.65e-04 | 726.8ms/step | 27.5 samples/s +Step 8160/44000 | Loss: 3.1709 | LR: 4.65e-04 | 726.0ms/step | 27.5 samples/s +Step 8170/44000 | Loss: 3.1971 | LR: 4.65e-04 | 725.9ms/step | 27.6 samples/s +Step 8180/44000 | Loss: 3.1310 | LR: 4.64e-04 | 730.5ms/step | 27.4 samples/s +Step 8190/44000 | Loss: 3.2012 | LR: 4.64e-04 | 730.4ms/step | 27.4 samples/s +Step 8200/44000 | Loss: 3.2173 | LR: 4.64e-04 | 726.3ms/step | 27.5 samples/s +Step 8210/44000 | Loss: 3.2449 | LR: 4.64e-04 | 727.7ms/step | 27.5 samples/s +Step 8220/44000 | Loss: 3.1628 | LR: 4.64e-04 | 726.8ms/step | 27.5 samples/s +Step 8230/44000 | Loss: 3.1745 | LR: 4.64e-04 | 725.7ms/step | 27.6 samples/s +Step 8240/44000 | Loss: 3.2149 | LR: 4.64e-04 | 725.7ms/step | 27.6 samples/s +Step 8250/44000 | Loss: 3.1701 | LR: 4.64e-04 | 725.8ms/step | 27.6 samples/s +Validation at step 8250: Loss = 3.1763, Perplexity = 23.96 +Step 8260/44000 | Loss: 3.1506 | LR: 4.64e-04 | 949.8ms/step | 21.1 samples/s +Step 8270/44000 | Loss: 3.1822 | LR: 4.64e-04 | 737.5ms/step | 27.1 samples/s +Step 8280/44000 | Loss: 3.2019 | LR: 4.64e-04 | 726.2ms/step | 27.5 samples/s +Step 8290/44000 | Loss: 3.2235 | LR: 4.63e-04 | 726.0ms/step | 27.5 samples/s +Step 8300/44000 | Loss: 3.1742 | LR: 4.63e-04 | 726.1ms/step | 27.5 samples/s +Step 8310/44000 | Loss: 3.2081 | LR: 4.63e-04 | 725.9ms/step | 27.6 samples/s +Step 8320/44000 | Loss: 3.2419 | LR: 4.63e-04 | 725.9ms/step | 27.6 samples/s +Step 8330/44000 | Loss: 3.0938 | LR: 4.63e-04 | 727.5ms/step | 27.5 samples/s +Step 8340/44000 | Loss: 3.0238 | LR: 4.63e-04 | 727.4ms/step | 27.5 samples/s +Step 8350/44000 | Loss: 3.2126 | LR: 4.63e-04 | 731.0ms/step | 27.4 samples/s +Step 8360/44000 | Loss: 3.2166 | LR: 4.63e-04 | 730.4ms/step | 27.4 samples/s +Step 8370/44000 | Loss: 3.2116 | LR: 4.63e-04 | 725.3ms/step | 27.6 samples/s +Step 8380/44000 | Loss: 3.2233 | LR: 4.63e-04 | 725.8ms/step | 27.6 samples/s +Step 8390/44000 | Loss: 3.1398 | LR: 4.62e-04 | 726.7ms/step | 27.5 samples/s +Step 8400/44000 | Loss: 3.2086 | LR: 4.62e-04 | 726.8ms/step | 27.5 samples/s +Step 8410/44000 | Loss: 3.2521 | LR: 4.62e-04 | 727.0ms/step | 27.5 samples/s +Step 8420/44000 | Loss: 3.2025 | LR: 4.62e-04 | 726.0ms/step | 27.5 samples/s +Step 8430/44000 | Loss: 3.1943 | LR: 4.62e-04 | 725.5ms/step | 27.6 samples/s +Step 8440/44000 | Loss: 3.1607 | LR: 4.62e-04 | 738.1ms/step | 27.1 samples/s +Step 8450/44000 | Loss: 3.2146 | LR: 4.62e-04 | 725.9ms/step | 27.6 samples/s +Step 8460/44000 | Loss: 3.2086 | LR: 4.62e-04 | 728.3ms/step | 27.5 samples/s +Step 8470/44000 | Loss: 3.2222 | LR: 4.62e-04 | 727.3ms/step | 27.5 samples/s +Step 8480/44000 | Loss: 3.1806 | LR: 4.62e-04 | 725.9ms/step | 27.6 samples/s +Step 8490/44000 | Loss: 3.1891 | LR: 4.62e-04 | 725.7ms/step | 27.6 samples/s +Step 8500/44000 | Loss: 3.1442 | LR: 4.61e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 8500: Loss = 3.1704, Perplexity = 23.82 +Step 8510/44000 | Loss: 3.1157 | LR: 4.61e-04 | 949.9ms/step | 21.1 samples/s +Step 8520/44000 | Loss: 3.1824 | LR: 4.61e-04 | 726.6ms/step | 27.5 samples/s +Step 8530/44000 | Loss: 3.1658 | LR: 4.61e-04 | 738.5ms/step | 27.1 samples/s +Step 8540/44000 | Loss: 3.1079 | LR: 4.61e-04 | 726.3ms/step | 27.5 samples/s +Step 8550/44000 | Loss: 3.1752 | LR: 4.61e-04 | 725.7ms/step | 27.6 samples/s +Step 8560/44000 | Loss: 3.1306 | LR: 4.61e-04 | 725.5ms/step | 27.6 samples/s +Step 8570/44000 | Loss: 3.1427 | LR: 4.61e-04 | 726.0ms/step | 27.5 samples/s +Step 8580/44000 | Loss: 3.2527 | LR: 4.61e-04 | 726.2ms/step | 27.5 samples/s +Step 8590/44000 | Loss: 3.2005 | LR: 4.61e-04 | 727.5ms/step | 27.5 samples/s +Step 8600/44000 | Loss: 3.1488 | LR: 4.60e-04 | 727.6ms/step | 27.5 samples/s +Step 8610/44000 | Loss: 3.2356 | LR: 4.60e-04 | 730.4ms/step | 27.4 samples/s +Step 8620/44000 | Loss: 3.1650 | LR: 4.60e-04 | 730.6ms/step | 27.4 samples/s +Step 8630/44000 | Loss: 3.1742 | LR: 4.60e-04 | 725.7ms/step | 27.6 samples/s +Step 8640/44000 | Loss: 3.1866 | LR: 4.60e-04 | 725.9ms/step | 27.6 samples/s +Step 8650/44000 | Loss: 3.1549 | LR: 4.60e-04 | 727.2ms/step | 27.5 samples/s +Step 8660/44000 | Loss: 3.1738 | LR: 4.60e-04 | 727.3ms/step | 27.5 samples/s +Step 8670/44000 | Loss: 3.1999 | LR: 4.60e-04 | 726.1ms/step | 27.5 samples/s +Step 8680/44000 | Loss: 3.1164 | LR: 4.60e-04 | 726.1ms/step | 27.5 samples/s +Step 8690/44000 | Loss: 3.1873 | LR: 4.60e-04 | 725.7ms/step | 27.6 samples/s +Step 8700/44000 | Loss: 3.1767 | LR: 4.59e-04 | 739.1ms/step | 27.1 samples/s +Step 8710/44000 | Loss: 3.1228 | LR: 4.59e-04 | 727.0ms/step | 27.5 samples/s +Step 8720/44000 | Loss: 3.1387 | LR: 4.59e-04 | 727.4ms/step | 27.5 samples/s +Step 8730/44000 | Loss: 3.1184 | LR: 4.59e-04 | 726.6ms/step | 27.5 samples/s +Step 8740/44000 | Loss: 3.2154 | LR: 4.59e-04 | 725.6ms/step | 27.6 samples/s +Step 8750/44000 | Loss: 3.1199 | LR: 4.59e-04 | 725.4ms/step | 27.6 samples/s +Validation at step 8750: Loss = 3.1602, Perplexity = 23.57 +Step 8760/44000 | Loss: 3.1584 | LR: 4.59e-04 | 949.1ms/step | 21.1 samples/s +Step 8770/44000 | Loss: 3.1064 | LR: 4.59e-04 | 726.0ms/step | 27.5 samples/s +Step 8780/44000 | Loss: 3.1289 | LR: 4.59e-04 | 731.3ms/step | 27.3 samples/s +Step 8790/44000 | Loss: 3.2572 | LR: 4.59e-04 | 731.3ms/step | 27.3 samples/s +Step 8800/44000 | Loss: 3.1299 | LR: 4.58e-04 | 725.7ms/step | 27.6 samples/s +Step 8810/44000 | Loss: 3.2403 | LR: 4.58e-04 | 725.7ms/step | 27.6 samples/s +Step 8820/44000 | Loss: 3.1840 | LR: 4.58e-04 | 725.8ms/step | 27.6 samples/s +Step 8830/44000 | Loss: 3.1628 | LR: 4.58e-04 | 725.8ms/step | 27.6 samples/s +Step 8840/44000 | Loss: 3.2079 | LR: 4.58e-04 | 727.3ms/step | 27.5 samples/s +Step 8850/44000 | Loss: 3.1422 | LR: 4.58e-04 | 726.8ms/step | 27.5 samples/s +Step 8860/44000 | Loss: 3.1659 | LR: 4.58e-04 | 726.4ms/step | 27.5 samples/s +Step 8870/44000 | Loss: 3.1510 | LR: 4.58e-04 | 737.6ms/step | 27.1 samples/s +Step 8880/44000 | Loss: 3.2478 | LR: 4.58e-04 | 726.1ms/step | 27.5 samples/s +Step 8890/44000 | Loss: 3.0969 | LR: 4.58e-04 | 726.3ms/step | 27.5 samples/s +Step 8900/44000 | Loss: 3.1690 | LR: 4.57e-04 | 726.6ms/step | 27.5 samples/s +Step 8910/44000 | Loss: 3.1534 | LR: 4.57e-04 | 727.9ms/step | 27.5 samples/s +Step 8920/44000 | Loss: 3.1873 | LR: 4.57e-04 | 727.4ms/step | 27.5 samples/s +Step 8930/44000 | Loss: 3.1608 | LR: 4.57e-04 | 725.4ms/step | 27.6 samples/s +Step 8940/44000 | Loss: 3.1666 | LR: 4.57e-04 | 725.1ms/step | 27.6 samples/s +Step 8950/44000 | Loss: 3.1785 | LR: 4.57e-04 | 725.8ms/step | 27.6 samples/s +Step 8960/44000 | Loss: 3.1976 | LR: 4.57e-04 | 735.3ms/step | 27.2 samples/s +Step 8970/44000 | Loss: 3.1363 | LR: 4.57e-04 | 726.8ms/step | 27.5 samples/s +Step 8980/44000 | Loss: 3.2642 | LR: 4.57e-04 | 727.0ms/step | 27.5 samples/s +Step 8990/44000 | Loss: 3.1548 | LR: 4.57e-04 | 726.2ms/step | 27.5 samples/s +Step 9000/44000 | Loss: 3.1995 | LR: 4.56e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 9000: Loss = 3.1573, Perplexity = 23.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-9000 +Step 9010/44000 | Loss: 3.1040 | LR: 4.56e-04 | 1362.2ms/step | 14.7 samples/s +Step 9020/44000 | Loss: 3.1439 | LR: 4.56e-04 | 725.4ms/step | 27.6 samples/s +Step 9030/44000 | Loss: 3.2154 | LR: 4.56e-04 | 727.1ms/step | 27.5 samples/s +Step 9040/44000 | Loss: 3.1994 | LR: 4.56e-04 | 735.6ms/step | 27.2 samples/s +Step 9050/44000 | Loss: 3.1729 | LR: 4.56e-04 | 725.1ms/step | 27.6 samples/s +Step 9060/44000 | Loss: 3.1099 | LR: 4.56e-04 | 725.4ms/step | 27.6 samples/s +Step 9070/44000 | Loss: 3.2052 | LR: 4.56e-04 | 725.9ms/step | 27.6 samples/s +Step 9080/44000 | Loss: 3.1591 | LR: 4.56e-04 | 725.3ms/step | 27.6 samples/s +Step 9090/44000 | Loss: 3.1823 | LR: 4.56e-04 | 726.7ms/step | 27.5 samples/s +Step 9100/44000 | Loss: 3.1910 | LR: 4.55e-04 | 727.3ms/step | 27.5 samples/s +Step 9110/44000 | Loss: 3.1821 | LR: 4.55e-04 | 726.1ms/step | 27.5 samples/s +Step 9120/44000 | Loss: 3.1526 | LR: 4.55e-04 | 732.3ms/step | 27.3 samples/s +Step 9130/44000 | Loss: 3.1872 | LR: 4.55e-04 | 732.5ms/step | 27.3 samples/s +Step 9140/44000 | Loss: 3.2375 | LR: 4.55e-04 | 725.4ms/step | 27.6 samples/s +Step 9150/44000 | Loss: 3.1634 | LR: 4.55e-04 | 725.8ms/step | 27.6 samples/s +Step 9160/44000 | Loss: 3.1554 | LR: 4.55e-04 | 727.1ms/step | 27.5 samples/s +Step 9170/44000 | Loss: 3.1643 | LR: 4.55e-04 | 728.4ms/step | 27.5 samples/s +Step 9180/44000 | Loss: 3.1754 | LR: 4.55e-04 | 726.1ms/step | 27.5 samples/s +Step 9190/44000 | Loss: 3.2233 | LR: 4.54e-04 | 725.8ms/step | 27.6 samples/s +Step 9200/44000 | Loss: 3.1371 | LR: 4.54e-04 | 726.6ms/step | 27.5 samples/s +Step 9210/44000 | Loss: 3.2031 | LR: 4.54e-04 | 731.2ms/step | 27.4 samples/s +Step 9220/44000 | Loss: 3.1723 | LR: 4.54e-04 | 733.6ms/step | 27.3 samples/s +Step 9230/44000 | Loss: 3.1356 | LR: 4.54e-04 | 730.0ms/step | 27.4 samples/s +Step 9240/44000 | Loss: 3.1538 | LR: 4.54e-04 | 727.2ms/step | 27.5 samples/s +Step 9250/44000 | Loss: 3.1842 | LR: 4.54e-04 | 726.9ms/step | 27.5 samples/s +Validation at step 9250: Loss = 3.1508, Perplexity = 23.35 +Step 9260/44000 | Loss: 3.1092 | LR: 4.54e-04 | 950.7ms/step | 21.0 samples/s +Step 9270/44000 | Loss: 3.1727 | LR: 4.54e-04 | 726.1ms/step | 27.5 samples/s +Step 9280/44000 | Loss: 3.1861 | LR: 4.54e-04 | 727.2ms/step | 27.5 samples/s +Step 9290/44000 | Loss: 3.1695 | LR: 4.53e-04 | 727.5ms/step | 27.5 samples/s +Step 9300/44000 | Loss: 3.1531 | LR: 4.53e-04 | 738.3ms/step | 27.1 samples/s +Step 9310/44000 | Loss: 3.0907 | LR: 4.53e-04 | 725.8ms/step | 27.6 samples/s +Step 9320/44000 | Loss: 3.1461 | LR: 4.53e-04 | 726.1ms/step | 27.5 samples/s +Step 9330/44000 | Loss: 3.1924 | LR: 4.53e-04 | 726.0ms/step | 27.5 samples/s +Step 9340/44000 | Loss: 3.1813 | LR: 4.53e-04 | 725.9ms/step | 27.6 samples/s +Step 9350/44000 | Loss: 3.2450 | LR: 4.53e-04 | 727.9ms/step | 27.5 samples/s +Step 9360/44000 | Loss: 3.1864 | LR: 4.53e-04 | 727.5ms/step | 27.5 samples/s +Step 9370/44000 | Loss: 3.1857 | LR: 4.53e-04 | 725.8ms/step | 27.6 samples/s +Step 9380/44000 | Loss: 3.1714 | LR: 4.52e-04 | 731.3ms/step | 27.3 samples/s +Step 9390/44000 | Loss: 3.1537 | LR: 4.52e-04 | 731.0ms/step | 27.4 samples/s +Step 9400/44000 | Loss: 3.1875 | LR: 4.52e-04 | 726.6ms/step | 27.5 samples/s +Step 9410/44000 | Loss: 3.1720 | LR: 4.52e-04 | 726.4ms/step | 27.5 samples/s +Step 9420/44000 | Loss: 3.1632 | LR: 4.52e-04 | 727.8ms/step | 27.5 samples/s +Step 9430/44000 | Loss: 3.1354 | LR: 4.52e-04 | 726.6ms/step | 27.5 samples/s +Step 9440/44000 | Loss: 3.1782 | LR: 4.52e-04 | 725.4ms/step | 27.6 samples/s +Step 9450/44000 | Loss: 3.0941 | LR: 4.52e-04 | 725.5ms/step | 27.6 samples/s +Step 9460/44000 | Loss: 3.1266 | LR: 4.52e-04 | 725.7ms/step | 27.6 samples/s +Step 9470/44000 | Loss: 3.1155 | LR: 4.52e-04 | 739.0ms/step | 27.1 samples/s +Step 9480/44000 | Loss: 3.0855 | LR: 4.51e-04 | 727.2ms/step | 27.5 samples/s +Step 9490/44000 | Loss: 3.0791 | LR: 4.51e-04 | 727.2ms/step | 27.5 samples/s +Step 9500/44000 | Loss: 3.1530 | LR: 4.51e-04 | 725.6ms/step | 27.6 samples/s +Validation at step 9500: Loss = 3.1413, Perplexity = 23.13 +Step 9510/44000 | Loss: 3.1391 | LR: 4.51e-04 | 950.1ms/step | 21.1 samples/s +Step 9520/44000 | Loss: 3.1612 | LR: 4.51e-04 | 725.9ms/step | 27.6 samples/s +Step 9530/44000 | Loss: 3.2004 | LR: 4.51e-04 | 726.0ms/step | 27.5 samples/s +Step 9540/44000 | Loss: 3.1358 | LR: 4.51e-04 | 728.0ms/step | 27.5 samples/s +Step 9550/44000 | Loss: 3.1561 | LR: 4.51e-04 | 733.0ms/step | 27.3 samples/s +Step 9560/44000 | Loss: 3.1998 | LR: 4.51e-04 | 733.8ms/step | 27.3 samples/s +Step 9570/44000 | Loss: 3.2192 | LR: 4.50e-04 | 726.3ms/step | 27.5 samples/s +Step 9580/44000 | Loss: 3.1995 | LR: 4.50e-04 | 726.2ms/step | 27.5 samples/s +Step 9590/44000 | Loss: 3.2122 | LR: 4.50e-04 | 725.9ms/step | 27.6 samples/s +Step 9600/44000 | Loss: 3.1452 | LR: 4.50e-04 | 726.2ms/step | 27.5 samples/s +Step 9610/44000 | Loss: 3.1618 | LR: 4.50e-04 | 727.0ms/step | 27.5 samples/s +Step 9620/44000 | Loss: 3.1069 | LR: 4.50e-04 | 726.6ms/step | 27.5 samples/s +Step 9630/44000 | Loss: 3.1072 | LR: 4.50e-04 | 725.5ms/step | 27.6 samples/s +Step 9640/44000 | Loss: 3.1339 | LR: 4.50e-04 | 730.7ms/step | 27.4 samples/s +Step 9650/44000 | Loss: 3.1697 | LR: 4.50e-04 | 730.3ms/step | 27.4 samples/s +Step 9660/44000 | Loss: 3.1746 | LR: 4.49e-04 | 725.7ms/step | 27.6 samples/s +Step 9670/44000 | Loss: 3.1166 | LR: 4.49e-04 | 726.7ms/step | 27.5 samples/s +Step 9680/44000 | Loss: 3.1232 | LR: 4.49e-04 | 727.9ms/step | 27.5 samples/s +Step 9690/44000 | Loss: 3.1759 | LR: 4.49e-04 | 725.3ms/step | 27.6 samples/s +Step 9700/44000 | Loss: 3.1376 | LR: 4.49e-04 | 725.5ms/step | 27.6 samples/s +Step 9710/44000 | Loss: 3.1173 | LR: 4.49e-04 | 725.8ms/step | 27.6 samples/s +Step 9720/44000 | Loss: 3.1026 | LR: 4.49e-04 | 726.4ms/step | 27.5 samples/s +Step 9730/44000 | Loss: 3.1584 | LR: 4.49e-04 | 740.6ms/step | 27.0 samples/s +Step 9740/44000 | Loss: 3.1267 | LR: 4.49e-04 | 727.0ms/step | 27.5 samples/s +Step 9750/44000 | Loss: 3.1151 | LR: 4.48e-04 | 726.2ms/step | 27.5 samples/s +Validation at step 9750: Loss = 3.1369, Perplexity = 23.03 +Step 9760/44000 | Loss: 3.2072 | LR: 4.48e-04 | 949.3ms/step | 21.1 samples/s +Step 9770/44000 | Loss: 3.0980 | LR: 4.48e-04 | 726.0ms/step | 27.5 samples/s +Step 9780/44000 | Loss: 3.1282 | LR: 4.48e-04 | 726.0ms/step | 27.5 samples/s +Step 9790/44000 | Loss: 3.1698 | LR: 4.48e-04 | 727.0ms/step | 27.5 samples/s +Step 9800/44000 | Loss: 3.1510 | LR: 4.48e-04 | 727.3ms/step | 27.5 samples/s +Step 9810/44000 | Loss: 3.1503 | LR: 4.48e-04 | 731.2ms/step | 27.4 samples/s +Step 9820/44000 | Loss: 3.1342 | LR: 4.48e-04 | 730.3ms/step | 27.4 samples/s +Step 9830/44000 | Loss: 3.1237 | LR: 4.48e-04 | 725.6ms/step | 27.6 samples/s +Step 9840/44000 | Loss: 3.2296 | LR: 4.47e-04 | 725.8ms/step | 27.6 samples/s +Step 9850/44000 | Loss: 3.2060 | LR: 4.47e-04 | 728.1ms/step | 27.5 samples/s +Step 9860/44000 | Loss: 3.0550 | LR: 4.47e-04 | 727.1ms/step | 27.5 samples/s +Step 9870/44000 | Loss: 3.1005 | LR: 4.47e-04 | 727.8ms/step | 27.5 samples/s +Step 9880/44000 | Loss: 3.1179 | LR: 4.47e-04 | 726.3ms/step | 27.5 samples/s +Step 9890/44000 | Loss: 3.2069 | LR: 4.47e-04 | 725.3ms/step | 27.6 samples/s +Step 9900/44000 | Loss: 3.1881 | LR: 4.47e-04 | 739.0ms/step | 27.1 samples/s +Step 9910/44000 | Loss: 3.1296 | LR: 4.47e-04 | 725.9ms/step | 27.6 samples/s +Step 9920/44000 | Loss: 3.1965 | LR: 4.47e-04 | 726.6ms/step | 27.5 samples/s +Step 9930/44000 | Loss: 3.1460 | LR: 4.46e-04 | 727.0ms/step | 27.5 samples/s +Step 9940/44000 | Loss: 3.1696 | LR: 4.46e-04 | 726.6ms/step | 27.5 samples/s +Step 9950/44000 | Loss: 3.1700 | LR: 4.46e-04 | 726.7ms/step | 27.5 samples/s +Step 9960/44000 | Loss: 3.1981 | LR: 4.46e-04 | 726.3ms/step | 27.5 samples/s +Step 9970/44000 | Loss: 3.1174 | LR: 4.46e-04 | 726.5ms/step | 27.5 samples/s +Step 9980/44000 | Loss: 3.1415 | LR: 4.46e-04 | 726.1ms/step | 27.5 samples/s +Step 9990/44000 | Loss: 3.1328 | LR: 4.46e-04 | 736.5ms/step | 27.2 samples/s +Step 10000/44000 | Loss: 3.0899 | LR: 4.46e-04 | 727.1ms/step | 27.5 samples/s +Validation at step 10000: Loss = 3.1301, Perplexity = 22.88 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-10000 +Step 10010/44000 | Loss: 3.1370 | LR: 4.46e-04 | 1366.7ms/step | 14.6 samples/s +Step 10020/44000 | Loss: 3.1820 | LR: 4.45e-04 | 725.5ms/step | 27.6 samples/s +Step 10030/44000 | Loss: 3.1357 | LR: 4.45e-04 | 725.2ms/step | 27.6 samples/s +Step 10040/44000 | Loss: 3.1598 | LR: 4.45e-04 | 725.9ms/step | 27.6 samples/s +Step 10050/44000 | Loss: 3.1641 | LR: 4.45e-04 | 726.6ms/step | 27.5 samples/s +Step 10060/44000 | Loss: 3.1056 | LR: 4.45e-04 | 726.2ms/step | 27.5 samples/s +Step 10070/44000 | Loss: 3.1391 | LR: 4.45e-04 | 735.1ms/step | 27.2 samples/s +Step 10080/44000 | Loss: 3.1487 | LR: 4.45e-04 | 725.7ms/step | 27.6 samples/s +Step 10090/44000 | Loss: 3.2119 | LR: 4.45e-04 | 725.4ms/step | 27.6 samples/s +Step 10100/44000 | Loss: 3.1981 | LR: 4.45e-04 | 725.8ms/step | 27.6 samples/s +Step 10110/44000 | Loss: 3.1159 | LR: 4.44e-04 | 727.1ms/step | 27.5 samples/s +Step 10120/44000 | Loss: 3.1115 | LR: 4.44e-04 | 727.6ms/step | 27.5 samples/s +Step 10130/44000 | Loss: 3.1569 | LR: 4.44e-04 | 725.5ms/step | 27.6 samples/s +Step 10140/44000 | Loss: 3.1317 | LR: 4.44e-04 | 726.2ms/step | 27.5 samples/s +Step 10150/44000 | Loss: 3.1694 | LR: 4.44e-04 | 731.0ms/step | 27.4 samples/s +Step 10160/44000 | Loss: 3.1458 | LR: 4.44e-04 | 731.0ms/step | 27.4 samples/s +Step 10170/44000 | Loss: 3.1372 | LR: 4.44e-04 | 726.7ms/step | 27.5 samples/s +Step 10180/44000 | Loss: 3.2032 | LR: 4.44e-04 | 726.5ms/step | 27.5 samples/s +Step 10190/44000 | Loss: 3.1878 | LR: 4.43e-04 | 727.1ms/step | 27.5 samples/s +Step 10200/44000 | Loss: 3.1413 | LR: 4.43e-04 | 726.3ms/step | 27.5 samples/s +Step 10210/44000 | Loss: 3.0710 | LR: 4.43e-04 | 725.8ms/step | 27.6 samples/s +Step 10220/44000 | Loss: 3.1812 | LR: 4.43e-04 | 725.8ms/step | 27.6 samples/s +Step 10230/44000 | Loss: 3.1183 | LR: 4.43e-04 | 726.0ms/step | 27.5 samples/s +Step 10240/44000 | Loss: 3.1501 | LR: 4.43e-04 | 741.0ms/step | 27.0 samples/s +Step 10250/44000 | Loss: 3.2142 | LR: 4.43e-04 | 727.5ms/step | 27.5 samples/s +Validation at step 10250: Loss = 3.1271, Perplexity = 22.81 +Step 10260/44000 | Loss: 3.1515 | LR: 4.43e-04 | 952.2ms/step | 21.0 samples/s +Step 10270/44000 | Loss: 3.0912 | LR: 4.43e-04 | 725.8ms/step | 27.6 samples/s +Step 10280/44000 | Loss: 3.1174 | LR: 4.42e-04 | 725.4ms/step | 27.6 samples/s +Step 10290/44000 | Loss: 3.0795 | LR: 4.42e-04 | 725.9ms/step | 27.6 samples/s +Step 10300/44000 | Loss: 3.0963 | LR: 4.42e-04 | 727.3ms/step | 27.5 samples/s +Step 10310/44000 | Loss: 3.1317 | LR: 4.42e-04 | 728.2ms/step | 27.5 samples/s +Step 10320/44000 | Loss: 3.0533 | LR: 4.42e-04 | 726.2ms/step | 27.5 samples/s +Step 10330/44000 | Loss: 3.0962 | LR: 4.42e-04 | 736.4ms/step | 27.2 samples/s +Step 10340/44000 | Loss: 3.1747 | LR: 4.42e-04 | 725.9ms/step | 27.6 samples/s +Step 10350/44000 | Loss: 3.1362 | LR: 4.42e-04 | 725.8ms/step | 27.6 samples/s +Step 10360/44000 | Loss: 3.2026 | LR: 4.42e-04 | 725.8ms/step | 27.6 samples/s +Step 10370/44000 | Loss: 3.1516 | LR: 4.41e-04 | 727.0ms/step | 27.5 samples/s +Step 10380/44000 | Loss: 3.1472 | LR: 4.41e-04 | 727.1ms/step | 27.5 samples/s +Step 10390/44000 | Loss: 3.1229 | LR: 4.41e-04 | 725.6ms/step | 27.6 samples/s +Step 10400/44000 | Loss: 3.1494 | LR: 4.41e-04 | 726.0ms/step | 27.5 samples/s +Step 10410/44000 | Loss: 3.1468 | LR: 4.41e-04 | 732.9ms/step | 27.3 samples/s +Step 10420/44000 | Loss: 3.1064 | LR: 4.41e-04 | 730.5ms/step | 27.4 samples/s +Step 10430/44000 | Loss: 3.1638 | LR: 4.41e-04 | 727.4ms/step | 27.5 samples/s +Step 10440/44000 | Loss: 3.1795 | LR: 4.41e-04 | 727.0ms/step | 27.5 samples/s +Step 10450/44000 | Loss: 3.0883 | LR: 4.40e-04 | 726.1ms/step | 27.5 samples/s +Step 10460/44000 | Loss: 3.2067 | LR: 4.40e-04 | 726.4ms/step | 27.5 samples/s +Step 10470/44000 | Loss: 3.1453 | LR: 4.40e-04 | 725.6ms/step | 27.6 samples/s +Step 10480/44000 | Loss: 3.1274 | LR: 4.40e-04 | 725.7ms/step | 27.6 samples/s +Step 10490/44000 | Loss: 3.1448 | LR: 4.40e-04 | 726.2ms/step | 27.5 samples/s +Step 10500/44000 | Loss: 3.0924 | LR: 4.40e-04 | 736.0ms/step | 27.2 samples/s +Validation at step 10500: Loss = 3.1196, Perplexity = 22.64 +Step 10510/44000 | Loss: 3.1490 | LR: 4.40e-04 | 952.3ms/step | 21.0 samples/s +Step 10520/44000 | Loss: 3.0772 | LR: 4.40e-04 | 725.9ms/step | 27.6 samples/s +Step 10530/44000 | Loss: 3.1689 | LR: 4.40e-04 | 726.2ms/step | 27.5 samples/s +Step 10540/44000 | Loss: 3.1161 | LR: 4.39e-04 | 726.0ms/step | 27.5 samples/s +Step 10550/44000 | Loss: 3.1172 | LR: 4.39e-04 | 726.3ms/step | 27.5 samples/s +Step 10560/44000 | Loss: 3.1962 | LR: 4.39e-04 | 727.1ms/step | 27.5 samples/s +Step 10570/44000 | Loss: 3.1279 | LR: 4.39e-04 | 727.7ms/step | 27.5 samples/s +Step 10580/44000 | Loss: 3.1041 | LR: 4.39e-04 | 731.7ms/step | 27.3 samples/s +Step 10590/44000 | Loss: 3.1068 | LR: 4.39e-04 | 730.7ms/step | 27.4 samples/s +Step 10600/44000 | Loss: 3.1279 | LR: 4.39e-04 | 725.8ms/step | 27.6 samples/s +Step 10610/44000 | Loss: 3.1479 | LR: 4.39e-04 | 726.2ms/step | 27.5 samples/s +Step 10620/44000 | Loss: 3.1429 | LR: 4.38e-04 | 726.9ms/step | 27.5 samples/s +Step 10630/44000 | Loss: 3.1427 | LR: 4.38e-04 | 727.2ms/step | 27.5 samples/s +Step 10640/44000 | Loss: 3.1623 | LR: 4.38e-04 | 726.4ms/step | 27.5 samples/s +Step 10650/44000 | Loss: 3.1226 | LR: 4.38e-04 | 726.2ms/step | 27.5 samples/s +Step 10660/44000 | Loss: 3.1285 | LR: 4.38e-04 | 725.9ms/step | 27.6 samples/s +Step 10670/44000 | Loss: 3.0603 | LR: 4.38e-04 | 736.8ms/step | 27.1 samples/s +Step 10680/44000 | Loss: 3.2172 | LR: 4.38e-04 | 726.5ms/step | 27.5 samples/s +Step 10690/44000 | Loss: 3.1997 | LR: 4.38e-04 | 728.1ms/step | 27.5 samples/s +Step 10700/44000 | Loss: 3.1840 | LR: 4.37e-04 | 727.2ms/step | 27.5 samples/s +Step 10710/44000 | Loss: 3.0663 | LR: 4.37e-04 | 725.6ms/step | 27.6 samples/s +Step 10720/44000 | Loss: 3.1170 | LR: 4.37e-04 | 725.4ms/step | 27.6 samples/s +Step 10730/44000 | Loss: 3.1057 | LR: 4.37e-04 | 725.8ms/step | 27.6 samples/s +Step 10740/44000 | Loss: 3.1263 | LR: 4.37e-04 | 726.3ms/step | 27.5 samples/s +Step 10750/44000 | Loss: 3.1429 | LR: 4.37e-04 | 727.2ms/step | 27.5 samples/s +Validation at step 10750: Loss = 3.1186, Perplexity = 22.62 +Step 10760/44000 | Loss: 3.1005 | LR: 4.37e-04 | 965.8ms/step | 20.7 samples/s +Step 10770/44000 | Loss: 3.1091 | LR: 4.37e-04 | 725.6ms/step | 27.6 samples/s +Step 10780/44000 | Loss: 3.1559 | LR: 4.37e-04 | 726.4ms/step | 27.5 samples/s +Step 10790/44000 | Loss: 3.1425 | LR: 4.36e-04 | 726.0ms/step | 27.5 samples/s +Step 10800/44000 | Loss: 3.0792 | LR: 4.36e-04 | 726.1ms/step | 27.5 samples/s +Step 10810/44000 | Loss: 3.1056 | LR: 4.36e-04 | 726.6ms/step | 27.5 samples/s +Step 10820/44000 | Loss: 3.1636 | LR: 4.36e-04 | 727.0ms/step | 27.5 samples/s +Step 10830/44000 | Loss: 3.1317 | LR: 4.36e-04 | 726.7ms/step | 27.5 samples/s +Step 10840/44000 | Loss: 3.1480 | LR: 4.36e-04 | 730.3ms/step | 27.4 samples/s +Step 10850/44000 | Loss: 3.1172 | LR: 4.36e-04 | 731.2ms/step | 27.4 samples/s +Step 10860/44000 | Loss: 3.0407 | LR: 4.36e-04 | 726.0ms/step | 27.5 samples/s +Step 10870/44000 | Loss: 3.1387 | LR: 4.35e-04 | 726.5ms/step | 27.5 samples/s +Step 10880/44000 | Loss: 3.1823 | LR: 4.35e-04 | 727.5ms/step | 27.5 samples/s +Step 10890/44000 | Loss: 3.1647 | LR: 4.35e-04 | 727.5ms/step | 27.5 samples/s +Step 10900/44000 | Loss: 3.1357 | LR: 4.35e-04 | 725.4ms/step | 27.6 samples/s +Step 10910/44000 | Loss: 3.0839 | LR: 4.35e-04 | 726.1ms/step | 27.5 samples/s +Step 10920/44000 | Loss: 3.1410 | LR: 4.35e-04 | 725.7ms/step | 27.6 samples/s +Step 10930/44000 | Loss: 3.1565 | LR: 4.35e-04 | 739.1ms/step | 27.1 samples/s +Step 10940/44000 | Loss: 3.1227 | LR: 4.35e-04 | 726.5ms/step | 27.5 samples/s +Step 10950/44000 | Loss: 3.1393 | LR: 4.34e-04 | 727.1ms/step | 27.5 samples/s +Step 10960/44000 | Loss: 3.0894 | LR: 4.34e-04 | 725.8ms/step | 27.6 samples/s +Step 10970/44000 | Loss: 3.1606 | LR: 4.34e-04 | 725.6ms/step | 27.6 samples/s +Step 10980/44000 | Loss: 3.1489 | LR: 4.34e-04 | 726.1ms/step | 27.5 samples/s +Step 10990/44000 | Loss: 3.1303 | LR: 4.34e-04 | 725.6ms/step | 27.6 samples/s +Step 11000/44000 | Loss: 3.0381 | LR: 4.34e-04 | 726.4ms/step | 27.5 samples/s +Validation at step 11000: Loss = 3.1105, Perplexity = 22.43 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-11000 +Step 11010/44000 | Loss: 3.1146 | LR: 4.34e-04 | 1359.6ms/step | 14.7 samples/s +Step 11020/44000 | Loss: 3.1400 | LR: 4.34e-04 | 725.1ms/step | 27.6 samples/s +Step 11030/44000 | Loss: 3.1156 | LR: 4.33e-04 | 725.5ms/step | 27.6 samples/s +Step 11040/44000 | Loss: 3.1361 | LR: 4.33e-04 | 726.1ms/step | 27.5 samples/s +Step 11050/44000 | Loss: 3.0739 | LR: 4.33e-04 | 725.6ms/step | 27.6 samples/s +Step 11060/44000 | Loss: 3.1154 | LR: 4.33e-04 | 727.0ms/step | 27.5 samples/s +Step 11070/44000 | Loss: 3.1220 | LR: 4.33e-04 | 728.3ms/step | 27.5 samples/s +Step 11080/44000 | Loss: 3.1146 | LR: 4.33e-04 | 727.2ms/step | 27.5 samples/s +Step 11090/44000 | Loss: 3.1521 | LR: 4.33e-04 | 725.5ms/step | 27.6 samples/s +Step 11100/44000 | Loss: 3.1066 | LR: 4.33e-04 | 739.8ms/step | 27.0 samples/s +Step 11110/44000 | Loss: 3.0979 | LR: 4.32e-04 | 725.9ms/step | 27.6 samples/s +Step 11120/44000 | Loss: 3.0753 | LR: 4.32e-04 | 725.8ms/step | 27.6 samples/s +Step 11130/44000 | Loss: 3.0947 | LR: 4.32e-04 | 727.1ms/step | 27.5 samples/s +Step 11140/44000 | Loss: 3.1273 | LR: 4.32e-04 | 726.6ms/step | 27.5 samples/s +Step 11150/44000 | Loss: 3.1791 | LR: 4.32e-04 | 725.8ms/step | 27.6 samples/s +Step 11160/44000 | Loss: 3.0878 | LR: 4.32e-04 | 726.0ms/step | 27.5 samples/s +Step 11170/44000 | Loss: 3.1279 | LR: 4.32e-04 | 726.0ms/step | 27.5 samples/s +Step 11180/44000 | Loss: 3.0924 | LR: 4.32e-04 | 730.1ms/step | 27.4 samples/s +Step 11190/44000 | Loss: 3.1583 | LR: 4.31e-04 | 731.8ms/step | 27.3 samples/s +Step 11200/44000 | Loss: 3.1173 | LR: 4.31e-04 | 727.7ms/step | 27.5 samples/s +Step 11210/44000 | Loss: 3.1570 | LR: 4.31e-04 | 726.5ms/step | 27.5 samples/s +Step 11220/44000 | Loss: 3.0482 | LR: 4.31e-04 | 726.1ms/step | 27.5 samples/s +Step 11230/44000 | Loss: 3.1246 | LR: 4.31e-04 | 725.6ms/step | 27.6 samples/s +Step 11240/44000 | Loss: 3.1269 | LR: 4.31e-04 | 725.6ms/step | 27.6 samples/s +Step 11250/44000 | Loss: 3.1278 | LR: 4.31e-04 | 726.4ms/step | 27.5 samples/s +Validation at step 11250: Loss = 3.1059, Perplexity = 22.33 +Step 11260/44000 | Loss: 3.0444 | LR: 4.31e-04 | 955.3ms/step | 20.9 samples/s +Step 11270/44000 | Loss: 3.1498 | LR: 4.30e-04 | 739.9ms/step | 27.0 samples/s +Step 11280/44000 | Loss: 3.0514 | LR: 4.30e-04 | 726.0ms/step | 27.5 samples/s +Step 11290/44000 | Loss: 3.1310 | LR: 4.30e-04 | 726.5ms/step | 27.5 samples/s +Step 11300/44000 | Loss: 3.1259 | LR: 4.30e-04 | 725.7ms/step | 27.6 samples/s +Step 11310/44000 | Loss: 3.1610 | LR: 4.30e-04 | 726.1ms/step | 27.5 samples/s +Step 11320/44000 | Loss: 3.1154 | LR: 4.30e-04 | 727.1ms/step | 27.5 samples/s +Step 11330/44000 | Loss: 3.0713 | LR: 4.30e-04 | 727.1ms/step | 27.5 samples/s +Step 11340/44000 | Loss: 3.1907 | LR: 4.30e-04 | 726.3ms/step | 27.5 samples/s +Step 11350/44000 | Loss: 3.1085 | LR: 4.29e-04 | 730.7ms/step | 27.4 samples/s +Step 11360/44000 | Loss: 3.1107 | LR: 4.29e-04 | 730.7ms/step | 27.4 samples/s +Step 11370/44000 | Loss: 3.1230 | LR: 4.29e-04 | 726.4ms/step | 27.5 samples/s +Step 11380/44000 | Loss: 3.1885 | LR: 4.29e-04 | 727.7ms/step | 27.5 samples/s +Step 11390/44000 | Loss: 3.1198 | LR: 4.29e-04 | 727.6ms/step | 27.5 samples/s +Step 11400/44000 | Loss: 3.1306 | LR: 4.29e-04 | 726.8ms/step | 27.5 samples/s +Step 11410/44000 | Loss: 3.0897 | LR: 4.29e-04 | 725.7ms/step | 27.6 samples/s +Step 11420/44000 | Loss: 3.0802 | LR: 4.29e-04 | 725.8ms/step | 27.6 samples/s +Step 11430/44000 | Loss: 3.0815 | LR: 4.28e-04 | 726.0ms/step | 27.5 samples/s +Step 11440/44000 | Loss: 3.1347 | LR: 4.28e-04 | 730.3ms/step | 27.4 samples/s +Step 11450/44000 | Loss: 3.1547 | LR: 4.28e-04 | 735.5ms/step | 27.2 samples/s +Step 11460/44000 | Loss: 3.1398 | LR: 4.28e-04 | 726.8ms/step | 27.5 samples/s +Step 11470/44000 | Loss: 3.1132 | LR: 4.28e-04 | 725.9ms/step | 27.6 samples/s +Step 11480/44000 | Loss: 3.1249 | LR: 4.28e-04 | 725.9ms/step | 27.6 samples/s +Step 11490/44000 | Loss: 3.0587 | LR: 4.28e-04 | 725.9ms/step | 27.6 samples/s +Step 11500/44000 | Loss: 3.0715 | LR: 4.28e-04 | 725.6ms/step | 27.6 samples/s +Validation at step 11500: Loss = 3.1024, Perplexity = 22.25 +Step 11510/44000 | Loss: 3.0722 | LR: 4.27e-04 | 950.2ms/step | 21.0 samples/s +Step 11520/44000 | Loss: 3.1094 | LR: 4.27e-04 | 727.6ms/step | 27.5 samples/s +Step 11530/44000 | Loss: 3.0713 | LR: 4.27e-04 | 736.2ms/step | 27.2 samples/s +Step 11540/44000 | Loss: 3.0563 | LR: 4.27e-04 | 726.1ms/step | 27.5 samples/s +Step 11550/44000 | Loss: 3.1026 | LR: 4.27e-04 | 725.6ms/step | 27.6 samples/s +Step 11560/44000 | Loss: 3.0827 | LR: 4.27e-04 | 725.7ms/step | 27.6 samples/s +Step 11570/44000 | Loss: 3.1215 | LR: 4.27e-04 | 726.3ms/step | 27.5 samples/s +Step 11580/44000 | Loss: 3.1626 | LR: 4.27e-04 | 728.2ms/step | 27.5 samples/s +Step 11590/44000 | Loss: 3.0565 | LR: 4.26e-04 | 726.8ms/step | 27.5 samples/s +Step 11600/44000 | Loss: 3.1336 | LR: 4.26e-04 | 725.9ms/step | 27.6 samples/s +Step 11610/44000 | Loss: 3.0853 | LR: 4.26e-04 | 730.5ms/step | 27.4 samples/s +Step 11620/44000 | Loss: 3.1366 | LR: 4.26e-04 | 732.1ms/step | 27.3 samples/s +Step 11630/44000 | Loss: 3.0666 | LR: 4.26e-04 | 725.7ms/step | 27.6 samples/s +Step 11640/44000 | Loss: 3.0748 | LR: 4.26e-04 | 726.9ms/step | 27.5 samples/s +Step 11650/44000 | Loss: 3.1553 | LR: 4.26e-04 | 727.1ms/step | 27.5 samples/s +Step 11660/44000 | Loss: 3.1255 | LR: 4.26e-04 | 726.2ms/step | 27.5 samples/s +Step 11670/44000 | Loss: 3.1436 | LR: 4.25e-04 | 725.5ms/step | 27.6 samples/s +Step 11680/44000 | Loss: 3.0961 | LR: 4.25e-04 | 725.6ms/step | 27.6 samples/s +Step 11690/44000 | Loss: 3.1139 | LR: 4.25e-04 | 725.7ms/step | 27.6 samples/s +Step 11700/44000 | Loss: 3.0996 | LR: 4.25e-04 | 739.9ms/step | 27.0 samples/s +Step 11710/44000 | Loss: 3.0928 | LR: 4.25e-04 | 726.9ms/step | 27.5 samples/s +Step 11720/44000 | Loss: 3.0757 | LR: 4.25e-04 | 726.5ms/step | 27.5 samples/s +Step 11730/44000 | Loss: 3.0924 | LR: 4.25e-04 | 726.2ms/step | 27.5 samples/s +Step 11740/44000 | Loss: 3.1035 | LR: 4.24e-04 | 725.9ms/step | 27.6 samples/s +Step 11750/44000 | Loss: 3.1560 | LR: 4.24e-04 | 725.3ms/step | 27.6 samples/s +Validation at step 11750: Loss = 3.0982, Perplexity = 22.16 +Step 11760/44000 | Loss: 3.1227 | LR: 4.24e-04 | 949.9ms/step | 21.1 samples/s +Step 11770/44000 | Loss: 3.1806 | LR: 4.24e-04 | 727.8ms/step | 27.5 samples/s +Step 11780/44000 | Loss: 3.1222 | LR: 4.24e-04 | 727.1ms/step | 27.5 samples/s +Step 11790/44000 | Loss: 3.1435 | LR: 4.24e-04 | 734.8ms/step | 27.2 samples/s +Step 11800/44000 | Loss: 3.1873 | LR: 4.24e-04 | 726.6ms/step | 27.5 samples/s +Step 11810/44000 | Loss: 3.1411 | LR: 4.24e-04 | 725.6ms/step | 27.6 samples/s +Step 11820/44000 | Loss: 3.1752 | LR: 4.23e-04 | 725.6ms/step | 27.6 samples/s +Step 11830/44000 | Loss: 3.0876 | LR: 4.23e-04 | 727.1ms/step | 27.5 samples/s +Step 11840/44000 | Loss: 3.1477 | LR: 4.23e-04 | 726.5ms/step | 27.5 samples/s +Step 11850/44000 | Loss: 3.0784 | LR: 4.23e-04 | 725.9ms/step | 27.6 samples/s +Step 11860/44000 | Loss: 3.0953 | LR: 4.23e-04 | 725.6ms/step | 27.6 samples/s +Step 11870/44000 | Loss: 3.0823 | LR: 4.23e-04 | 733.4ms/step | 27.3 samples/s +Step 11880/44000 | Loss: 3.1223 | LR: 4.23e-04 | 731.9ms/step | 27.3 samples/s +Step 11890/44000 | Loss: 3.1309 | LR: 4.23e-04 | 726.6ms/step | 27.5 samples/s +Step 11900/44000 | Loss: 3.1323 | LR: 4.22e-04 | 727.9ms/step | 27.5 samples/s +Step 11910/44000 | Loss: 3.0705 | LR: 4.22e-04 | 726.5ms/step | 27.5 samples/s +Step 11920/44000 | Loss: 3.1447 | LR: 4.22e-04 | 726.1ms/step | 27.5 samples/s +Step 11930/44000 | Loss: 3.1830 | LR: 4.22e-04 | 725.2ms/step | 27.6 samples/s +Step 11940/44000 | Loss: 3.0979 | LR: 4.22e-04 | 725.7ms/step | 27.6 samples/s +Step 11950/44000 | Loss: 3.1221 | LR: 4.22e-04 | 725.9ms/step | 27.6 samples/s +Step 11960/44000 | Loss: 3.1024 | LR: 4.22e-04 | 736.8ms/step | 27.1 samples/s +Step 11970/44000 | Loss: 3.0518 | LR: 4.21e-04 | 726.9ms/step | 27.5 samples/s +Step 11980/44000 | Loss: 3.1466 | LR: 4.21e-04 | 725.4ms/step | 27.6 samples/s +Step 11990/44000 | Loss: 3.0926 | LR: 4.21e-04 | 725.9ms/step | 27.6 samples/s +Step 12000/44000 | Loss: 3.1230 | LR: 4.21e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 12000: Loss = 3.0944, Perplexity = 22.07 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-12000 +Step 12010/44000 | Loss: 3.1059 | LR: 4.21e-04 | 1368.4ms/step | 14.6 samples/s +Step 12020/44000 | Loss: 3.1218 | LR: 4.21e-04 | 727.5ms/step | 27.5 samples/s +Step 12030/44000 | Loss: 3.1789 | LR: 4.21e-04 | 725.9ms/step | 27.6 samples/s +Step 12040/44000 | Loss: 3.0818 | LR: 4.21e-04 | 738.6ms/step | 27.1 samples/s +Step 12050/44000 | Loss: 3.0870 | LR: 4.20e-04 | 725.6ms/step | 27.6 samples/s +Step 12060/44000 | Loss: 3.0524 | LR: 4.20e-04 | 725.5ms/step | 27.6 samples/s +Step 12070/44000 | Loss: 3.1000 | LR: 4.20e-04 | 726.1ms/step | 27.5 samples/s +Step 12080/44000 | Loss: 3.1001 | LR: 4.20e-04 | 727.9ms/step | 27.5 samples/s +Step 12090/44000 | Loss: 3.1605 | LR: 4.20e-04 | 727.4ms/step | 27.5 samples/s +Step 12100/44000 | Loss: 3.0548 | LR: 4.20e-04 | 726.1ms/step | 27.5 samples/s +Step 12110/44000 | Loss: 3.1600 | LR: 4.20e-04 | 725.5ms/step | 27.6 samples/s +Step 12120/44000 | Loss: 3.1492 | LR: 4.19e-04 | 725.9ms/step | 27.6 samples/s +Step 12130/44000 | Loss: 3.0642 | LR: 4.19e-04 | 734.9ms/step | 27.2 samples/s +Step 12140/44000 | Loss: 3.1112 | LR: 4.19e-04 | 727.1ms/step | 27.5 samples/s +Step 12150/44000 | Loss: 3.0465 | LR: 4.19e-04 | 726.8ms/step | 27.5 samples/s +Step 12160/44000 | Loss: 3.0882 | LR: 4.19e-04 | 726.2ms/step | 27.5 samples/s +Step 12170/44000 | Loss: 3.0419 | LR: 4.19e-04 | 725.4ms/step | 27.6 samples/s +Step 12180/44000 | Loss: 3.1074 | LR: 4.19e-04 | 725.7ms/step | 27.6 samples/s +Step 12190/44000 | Loss: 3.0929 | LR: 4.19e-04 | 725.7ms/step | 27.6 samples/s +Step 12200/44000 | Loss: 3.0506 | LR: 4.18e-04 | 725.1ms/step | 27.6 samples/s +Step 12210/44000 | Loss: 3.0750 | LR: 4.18e-04 | 731.4ms/step | 27.3 samples/s +Step 12220/44000 | Loss: 3.0624 | LR: 4.18e-04 | 732.4ms/step | 27.3 samples/s +Step 12230/44000 | Loss: 3.0267 | LR: 4.18e-04 | 725.4ms/step | 27.6 samples/s +Step 12240/44000 | Loss: 3.0738 | LR: 4.18e-04 | 725.2ms/step | 27.6 samples/s +Step 12250/44000 | Loss: 3.0764 | LR: 4.18e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 12250: Loss = 3.0876, Perplexity = 21.92 +Step 12260/44000 | Loss: 3.1856 | LR: 4.18e-04 | 952.6ms/step | 21.0 samples/s +Step 12270/44000 | Loss: 3.1421 | LR: 4.17e-04 | 728.0ms/step | 27.5 samples/s +Step 12280/44000 | Loss: 3.1236 | LR: 4.17e-04 | 727.3ms/step | 27.5 samples/s +Step 12290/44000 | Loss: 3.0988 | LR: 4.17e-04 | 726.6ms/step | 27.5 samples/s +Step 12300/44000 | Loss: 3.1148 | LR: 4.17e-04 | 734.9ms/step | 27.2 samples/s +Step 12310/44000 | Loss: 3.1434 | LR: 4.17e-04 | 726.4ms/step | 27.5 samples/s +Step 12320/44000 | Loss: 3.0710 | LR: 4.17e-04 | 726.3ms/step | 27.5 samples/s +Step 12330/44000 | Loss: 3.1152 | LR: 4.17e-04 | 726.3ms/step | 27.5 samples/s +Step 12340/44000 | Loss: 3.1376 | LR: 4.17e-04 | 727.1ms/step | 27.5 samples/s +Step 12350/44000 | Loss: 3.0776 | LR: 4.16e-04 | 726.8ms/step | 27.5 samples/s +Step 12360/44000 | Loss: 3.1765 | LR: 4.16e-04 | 725.9ms/step | 27.6 samples/s +Step 12370/44000 | Loss: 3.0914 | LR: 4.16e-04 | 725.6ms/step | 27.6 samples/s +Step 12380/44000 | Loss: 3.1588 | LR: 4.16e-04 | 732.9ms/step | 27.3 samples/s +Step 12390/44000 | Loss: 3.1236 | LR: 4.16e-04 | 730.6ms/step | 27.4 samples/s +Step 12400/44000 | Loss: 3.0819 | LR: 4.16e-04 | 727.0ms/step | 27.5 samples/s +Step 12410/44000 | Loss: 3.1623 | LR: 4.16e-04 | 727.4ms/step | 27.5 samples/s +Step 12420/44000 | Loss: 3.0045 | LR: 4.15e-04 | 725.5ms/step | 27.6 samples/s +Step 12430/44000 | Loss: 3.0895 | LR: 4.15e-04 | 725.8ms/step | 27.6 samples/s +Step 12440/44000 | Loss: 3.1231 | LR: 4.15e-04 | 725.5ms/step | 27.6 samples/s +Step 12450/44000 | Loss: 3.0859 | LR: 4.15e-04 | 725.8ms/step | 27.6 samples/s +Step 12460/44000 | Loss: 3.0910 | LR: 4.15e-04 | 725.8ms/step | 27.6 samples/s +Step 12470/44000 | Loss: 3.0215 | LR: 4.15e-04 | 736.2ms/step | 27.2 samples/s +Step 12480/44000 | Loss: 3.0799 | LR: 4.15e-04 | 726.2ms/step | 27.5 samples/s +Step 12490/44000 | Loss: 3.0931 | LR: 4.14e-04 | 725.6ms/step | 27.6 samples/s +Step 12500/44000 | Loss: 3.1029 | LR: 4.14e-04 | 725.3ms/step | 27.6 samples/s +Validation at step 12500: Loss = 3.0836, Perplexity = 21.84 +Step 12510/44000 | Loss: 3.1210 | LR: 4.14e-04 | 950.7ms/step | 21.0 samples/s +Step 12520/44000 | Loss: 3.0825 | LR: 4.14e-04 | 726.1ms/step | 27.5 samples/s +Step 12530/44000 | Loss: 3.1286 | LR: 4.14e-04 | 727.7ms/step | 27.5 samples/s +Step 12540/44000 | Loss: 3.1155 | LR: 4.14e-04 | 727.4ms/step | 27.5 samples/s +Step 12550/44000 | Loss: 3.1093 | LR: 4.14e-04 | 725.5ms/step | 27.6 samples/s +Step 12560/44000 | Loss: 3.0923 | LR: 4.14e-04 | 738.8ms/step | 27.1 samples/s +Step 12570/44000 | Loss: 3.0781 | LR: 4.13e-04 | 725.3ms/step | 27.6 samples/s +Step 12580/44000 | Loss: 3.0940 | LR: 4.13e-04 | 725.6ms/step | 27.6 samples/s +Step 12590/44000 | Loss: 3.1248 | LR: 4.13e-04 | 726.6ms/step | 27.5 samples/s +Step 12600/44000 | Loss: 3.0776 | LR: 4.13e-04 | 727.6ms/step | 27.5 samples/s +Step 12610/44000 | Loss: 3.0896 | LR: 4.13e-04 | 726.3ms/step | 27.5 samples/s +Step 12620/44000 | Loss: 3.0249 | LR: 4.13e-04 | 725.8ms/step | 27.6 samples/s +Step 12630/44000 | Loss: 3.1125 | LR: 4.13e-04 | 726.2ms/step | 27.5 samples/s +Step 12640/44000 | Loss: 3.1632 | LR: 4.12e-04 | 730.1ms/step | 27.4 samples/s +Step 12650/44000 | Loss: 3.0975 | LR: 4.12e-04 | 730.5ms/step | 27.4 samples/s +Step 12660/44000 | Loss: 3.0742 | LR: 4.12e-04 | 727.3ms/step | 27.5 samples/s +Step 12670/44000 | Loss: 3.0836 | LR: 4.12e-04 | 727.3ms/step | 27.5 samples/s +Step 12680/44000 | Loss: 3.1148 | LR: 4.12e-04 | 725.3ms/step | 27.6 samples/s +Step 12690/44000 | Loss: 3.1220 | LR: 4.12e-04 | 725.3ms/step | 27.6 samples/s +Step 12700/44000 | Loss: 2.9939 | LR: 4.12e-04 | 725.7ms/step | 27.6 samples/s +Step 12710/44000 | Loss: 3.1111 | LR: 4.11e-04 | 725.9ms/step | 27.6 samples/s +Step 12720/44000 | Loss: 3.1026 | LR: 4.11e-04 | 726.7ms/step | 27.5 samples/s +Step 12730/44000 | Loss: 3.1256 | LR: 4.11e-04 | 740.8ms/step | 27.0 samples/s +Step 12740/44000 | Loss: 3.0709 | LR: 4.11e-04 | 725.6ms/step | 27.6 samples/s +Step 12750/44000 | Loss: 3.0648 | LR: 4.11e-04 | 725.3ms/step | 27.6 samples/s +Validation at step 12750: Loss = 3.0791, Perplexity = 21.74 +Step 12760/44000 | Loss: 3.1326 | LR: 4.11e-04 | 952.0ms/step | 21.0 samples/s +Step 12770/44000 | Loss: 3.0725 | LR: 4.11e-04 | 725.8ms/step | 27.6 samples/s +Step 12780/44000 | Loss: 3.0639 | LR: 4.10e-04 | 727.0ms/step | 27.5 samples/s +Step 12790/44000 | Loss: 3.0493 | LR: 4.10e-04 | 727.1ms/step | 27.5 samples/s +Step 12800/44000 | Loss: 3.1556 | LR: 4.10e-04 | 726.2ms/step | 27.5 samples/s +Step 12810/44000 | Loss: 3.1185 | LR: 4.10e-04 | 730.9ms/step | 27.4 samples/s +Step 12820/44000 | Loss: 3.1249 | LR: 4.10e-04 | 730.6ms/step | 27.4 samples/s +Step 12830/44000 | Loss: 3.0782 | LR: 4.10e-04 | 725.8ms/step | 27.6 samples/s +Step 12840/44000 | Loss: 3.1104 | LR: 4.10e-04 | 726.1ms/step | 27.5 samples/s +Step 12850/44000 | Loss: 3.1158 | LR: 4.09e-04 | 726.8ms/step | 27.5 samples/s +Step 12860/44000 | Loss: 3.1710 | LR: 4.09e-04 | 726.8ms/step | 27.5 samples/s +Step 12870/44000 | Loss: 3.1178 | LR: 4.09e-04 | 726.1ms/step | 27.5 samples/s +Step 12880/44000 | Loss: 3.1108 | LR: 4.09e-04 | 725.8ms/step | 27.6 samples/s +Step 12890/44000 | Loss: 3.0899 | LR: 4.09e-04 | 726.0ms/step | 27.5 samples/s +Step 12900/44000 | Loss: 3.1768 | LR: 4.09e-04 | 737.6ms/step | 27.1 samples/s +Step 12910/44000 | Loss: 3.0595 | LR: 4.09e-04 | 729.6ms/step | 27.4 samples/s +Step 12920/44000 | Loss: 3.0488 | LR: 4.09e-04 | 727.4ms/step | 27.5 samples/s +Step 12930/44000 | Loss: 3.1099 | LR: 4.08e-04 | 726.1ms/step | 27.5 samples/s +Step 12940/44000 | Loss: 3.0246 | LR: 4.08e-04 | 725.5ms/step | 27.6 samples/s +Step 12950/44000 | Loss: 3.1105 | LR: 4.08e-04 | 725.4ms/step | 27.6 samples/s +Step 12960/44000 | Loss: 3.0666 | LR: 4.08e-04 | 725.8ms/step | 27.6 samples/s +Step 12970/44000 | Loss: 3.0741 | LR: 4.08e-04 | 725.9ms/step | 27.6 samples/s +Step 12980/44000 | Loss: 3.1014 | LR: 4.08e-04 | 727.2ms/step | 27.5 samples/s +Step 12990/44000 | Loss: 3.1118 | LR: 4.08e-04 | 736.5ms/step | 27.2 samples/s +Step 13000/44000 | Loss: 3.0913 | LR: 4.07e-04 | 726.4ms/step | 27.5 samples/s +Validation at step 13000: Loss = 3.0762, Perplexity = 21.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-13000 +Step 13010/44000 | Loss: 3.1475 | LR: 4.07e-04 | 1381.2ms/step | 14.5 samples/s +Step 13020/44000 | Loss: 3.0743 | LR: 4.07e-04 | 725.6ms/step | 27.6 samples/s +Step 13030/44000 | Loss: 3.0553 | LR: 4.07e-04 | 726.7ms/step | 27.5 samples/s +Step 13040/44000 | Loss: 3.0798 | LR: 4.07e-04 | 727.7ms/step | 27.5 samples/s +Step 13050/44000 | Loss: 3.1254 | LR: 4.07e-04 | 726.2ms/step | 27.5 samples/s +Step 13060/44000 | Loss: 3.0608 | LR: 4.07e-04 | 725.9ms/step | 27.6 samples/s +Step 13070/44000 | Loss: 3.0918 | LR: 4.06e-04 | 739.9ms/step | 27.0 samples/s +Step 13080/44000 | Loss: 3.0411 | LR: 4.06e-04 | 725.8ms/step | 27.6 samples/s +Step 13090/44000 | Loss: 3.1152 | LR: 4.06e-04 | 726.0ms/step | 27.5 samples/s +Step 13100/44000 | Loss: 3.0802 | LR: 4.06e-04 | 727.3ms/step | 27.5 samples/s +Step 13110/44000 | Loss: 3.0907 | LR: 4.06e-04 | 725.9ms/step | 27.6 samples/s +Step 13120/44000 | Loss: 3.0612 | LR: 4.06e-04 | 725.3ms/step | 27.6 samples/s +Step 13130/44000 | Loss: 3.0610 | LR: 4.06e-04 | 725.7ms/step | 27.6 samples/s +Step 13140/44000 | Loss: 3.0217 | LR: 4.05e-04 | 725.8ms/step | 27.6 samples/s +Step 13150/44000 | Loss: 3.0674 | LR: 4.05e-04 | 730.0ms/step | 27.4 samples/s +Step 13160/44000 | Loss: 3.1283 | LR: 4.05e-04 | 731.6ms/step | 27.3 samples/s +Step 13170/44000 | Loss: 3.0816 | LR: 4.05e-04 | 726.9ms/step | 27.5 samples/s +Step 13180/44000 | Loss: 3.0214 | LR: 4.05e-04 | 726.2ms/step | 27.5 samples/s +Step 13190/44000 | Loss: 3.0975 | LR: 4.05e-04 | 726.6ms/step | 27.5 samples/s +Step 13200/44000 | Loss: 3.1076 | LR: 4.05e-04 | 725.7ms/step | 27.6 samples/s +Step 13210/44000 | Loss: 3.0935 | LR: 4.04e-04 | 725.9ms/step | 27.6 samples/s +Step 13220/44000 | Loss: 3.1290 | LR: 4.04e-04 | 725.7ms/step | 27.6 samples/s +Step 13230/44000 | Loss: 3.0973 | LR: 4.04e-04 | 727.7ms/step | 27.5 samples/s +Step 13240/44000 | Loss: 3.0463 | LR: 4.04e-04 | 736.8ms/step | 27.1 samples/s +Step 13250/44000 | Loss: 3.1153 | LR: 4.04e-04 | 728.7ms/step | 27.4 samples/s +Validation at step 13250: Loss = 3.0733, Perplexity = 21.61 +Step 13260/44000 | Loss: 3.0899 | LR: 4.04e-04 | 949.9ms/step | 21.1 samples/s +Step 13270/44000 | Loss: 3.1026 | LR: 4.04e-04 | 725.6ms/step | 27.6 samples/s +Step 13280/44000 | Loss: 3.0593 | LR: 4.03e-04 | 726.5ms/step | 27.5 samples/s +Step 13290/44000 | Loss: 3.1466 | LR: 4.03e-04 | 727.5ms/step | 27.5 samples/s +Step 13300/44000 | Loss: 3.1409 | LR: 4.03e-04 | 727.1ms/step | 27.5 samples/s +Step 13310/44000 | Loss: 3.0970 | LR: 4.03e-04 | 726.3ms/step | 27.5 samples/s +Step 13320/44000 | Loss: 3.1025 | LR: 4.03e-04 | 726.1ms/step | 27.5 samples/s +Step 13330/44000 | Loss: 3.0716 | LR: 4.03e-04 | 735.2ms/step | 27.2 samples/s +Step 13340/44000 | Loss: 3.0686 | LR: 4.03e-04 | 725.6ms/step | 27.6 samples/s +Step 13350/44000 | Loss: 3.1030 | LR: 4.02e-04 | 726.8ms/step | 27.5 samples/s +Step 13360/44000 | Loss: 3.0659 | LR: 4.02e-04 | 726.5ms/step | 27.5 samples/s +Step 13370/44000 | Loss: 3.0951 | LR: 4.02e-04 | 725.8ms/step | 27.6 samples/s +Step 13380/44000 | Loss: 3.0099 | LR: 4.02e-04 | 725.4ms/step | 27.6 samples/s +Step 13390/44000 | Loss: 3.1545 | LR: 4.02e-04 | 725.8ms/step | 27.6 samples/s +Step 13400/44000 | Loss: 3.0564 | LR: 4.02e-04 | 725.7ms/step | 27.6 samples/s +Step 13410/44000 | Loss: 3.0892 | LR: 4.02e-04 | 730.7ms/step | 27.4 samples/s +Step 13420/44000 | Loss: 2.9576 | LR: 4.01e-04 | 733.6ms/step | 27.3 samples/s +Step 13430/44000 | Loss: 3.1058 | LR: 4.01e-04 | 726.1ms/step | 27.5 samples/s +Step 13440/44000 | Loss: 3.1402 | LR: 4.01e-04 | 725.6ms/step | 27.6 samples/s +Step 13450/44000 | Loss: 3.0734 | LR: 4.01e-04 | 725.3ms/step | 27.6 samples/s +Step 13460/44000 | Loss: 3.1235 | LR: 4.01e-04 | 725.5ms/step | 27.6 samples/s +Step 13470/44000 | Loss: 3.0883 | LR: 4.01e-04 | 725.8ms/step | 27.6 samples/s +Step 13480/44000 | Loss: 3.1092 | LR: 4.01e-04 | 726.4ms/step | 27.5 samples/s +Step 13490/44000 | Loss: 3.0867 | LR: 4.00e-04 | 727.5ms/step | 27.5 samples/s +Step 13500/44000 | Loss: 3.1133 | LR: 4.00e-04 | 735.5ms/step | 27.2 samples/s +Validation at step 13500: Loss = 3.0692, Perplexity = 21.52 +Step 13510/44000 | Loss: 3.1293 | LR: 4.00e-04 | 949.6ms/step | 21.1 samples/s +Step 13520/44000 | Loss: 3.1163 | LR: 4.00e-04 | 725.6ms/step | 27.6 samples/s +Step 13530/44000 | Loss: 3.0531 | LR: 4.00e-04 | 725.8ms/step | 27.6 samples/s +Step 13540/44000 | Loss: 3.0761 | LR: 4.00e-04 | 726.9ms/step | 27.5 samples/s +Step 13550/44000 | Loss: 3.0118 | LR: 4.00e-04 | 727.6ms/step | 27.5 samples/s +Step 13560/44000 | Loss: 3.1000 | LR: 3.99e-04 | 725.9ms/step | 27.6 samples/s +Step 13570/44000 | Loss: 3.0242 | LR: 3.99e-04 | 726.1ms/step | 27.5 samples/s +Step 13580/44000 | Loss: 3.1219 | LR: 3.99e-04 | 730.4ms/step | 27.4 samples/s +Step 13590/44000 | Loss: 3.0136 | LR: 3.99e-04 | 731.1ms/step | 27.4 samples/s +Step 13600/44000 | Loss: 3.0758 | LR: 3.99e-04 | 725.7ms/step | 27.6 samples/s +Step 13610/44000 | Loss: 3.1433 | LR: 3.99e-04 | 727.0ms/step | 27.5 samples/s +Step 13620/44000 | Loss: 3.1408 | LR: 3.98e-04 | 726.3ms/step | 27.5 samples/s +Step 13630/44000 | Loss: 3.0741 | LR: 3.98e-04 | 725.4ms/step | 27.6 samples/s +Step 13640/44000 | Loss: 3.0759 | LR: 3.98e-04 | 725.4ms/step | 27.6 samples/s +Step 13650/44000 | Loss: 3.1481 | LR: 3.98e-04 | 725.3ms/step | 27.6 samples/s +Step 13660/44000 | Loss: 2.9998 | LR: 3.98e-04 | 725.5ms/step | 27.6 samples/s +Step 13670/44000 | Loss: 3.0838 | LR: 3.98e-04 | 734.2ms/step | 27.2 samples/s +Step 13680/44000 | Loss: 3.0985 | LR: 3.98e-04 | 732.0ms/step | 27.3 samples/s +Step 13690/44000 | Loss: 3.1040 | LR: 3.97e-04 | 725.5ms/step | 27.6 samples/s +Step 13700/44000 | Loss: 3.0618 | LR: 3.97e-04 | 725.5ms/step | 27.6 samples/s +Step 13710/44000 | Loss: 3.1099 | LR: 3.97e-04 | 725.2ms/step | 27.6 samples/s +Step 13720/44000 | Loss: 2.9989 | LR: 3.97e-04 | 725.5ms/step | 27.6 samples/s +Step 13730/44000 | Loss: 3.1057 | LR: 3.97e-04 | 726.6ms/step | 27.5 samples/s +Step 13740/44000 | Loss: 3.0706 | LR: 3.97e-04 | 727.6ms/step | 27.5 samples/s +Step 13750/44000 | Loss: 3.1100 | LR: 3.97e-04 | 726.7ms/step | 27.5 samples/s +Validation at step 13750: Loss = 3.0658, Perplexity = 21.45 +Step 13760/44000 | Loss: 3.0964 | LR: 3.96e-04 | 959.0ms/step | 20.9 samples/s +Step 13770/44000 | Loss: 3.0690 | LR: 3.96e-04 | 726.0ms/step | 27.5 samples/s +Step 13780/44000 | Loss: 3.0912 | LR: 3.96e-04 | 725.7ms/step | 27.6 samples/s +Step 13790/44000 | Loss: 2.9829 | LR: 3.96e-04 | 726.3ms/step | 27.5 samples/s +Step 13800/44000 | Loss: 3.0655 | LR: 3.96e-04 | 728.0ms/step | 27.5 samples/s +Step 13810/44000 | Loss: 3.0381 | LR: 3.96e-04 | 727.1ms/step | 27.5 samples/s +Step 13820/44000 | Loss: 3.1217 | LR: 3.96e-04 | 725.8ms/step | 27.6 samples/s +Step 13830/44000 | Loss: 3.1748 | LR: 3.95e-04 | 725.3ms/step | 27.6 samples/s +Step 13840/44000 | Loss: 3.0139 | LR: 3.95e-04 | 732.8ms/step | 27.3 samples/s +Step 13850/44000 | Loss: 3.0917 | LR: 3.95e-04 | 732.6ms/step | 27.3 samples/s +Step 13860/44000 | Loss: 3.0806 | LR: 3.95e-04 | 727.9ms/step | 27.5 samples/s +Step 13870/44000 | Loss: 3.0883 | LR: 3.95e-04 | 728.2ms/step | 27.5 samples/s +Step 13880/44000 | Loss: 3.1091 | LR: 3.95e-04 | 725.9ms/step | 27.6 samples/s +Step 13890/44000 | Loss: 3.0692 | LR: 3.95e-04 | 727.0ms/step | 27.5 samples/s +Step 13900/44000 | Loss: 3.0941 | LR: 3.94e-04 | 726.0ms/step | 27.5 samples/s +Step 13910/44000 | Loss: 3.0101 | LR: 3.94e-04 | 725.6ms/step | 27.6 samples/s +Step 13920/44000 | Loss: 3.1124 | LR: 3.94e-04 | 726.3ms/step | 27.5 samples/s +Step 13930/44000 | Loss: 3.0739 | LR: 3.94e-04 | 735.9ms/step | 27.2 samples/s +Step 13940/44000 | Loss: 3.0329 | LR: 3.94e-04 | 726.2ms/step | 27.5 samples/s +Step 13950/44000 | Loss: 3.0318 | LR: 3.94e-04 | 725.9ms/step | 27.6 samples/s +Step 13960/44000 | Loss: 3.1184 | LR: 3.93e-04 | 726.1ms/step | 27.5 samples/s +Step 13970/44000 | Loss: 3.1030 | LR: 3.93e-04 | 726.0ms/step | 27.5 samples/s +Step 13980/44000 | Loss: 3.1034 | LR: 3.93e-04 | 726.4ms/step | 27.5 samples/s +Step 13990/44000 | Loss: 3.0777 | LR: 3.93e-04 | 727.8ms/step | 27.5 samples/s +Step 14000/44000 | Loss: 3.0524 | LR: 3.93e-04 | 727.2ms/step | 27.5 samples/s +Validation at step 14000: Loss = 3.0613, Perplexity = 21.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-14000 +Step 14010/44000 | Loss: 3.0620 | LR: 3.93e-04 | 1409.1ms/step | 14.2 samples/s +Step 14020/44000 | Loss: 3.0214 | LR: 3.93e-04 | 725.5ms/step | 27.6 samples/s +Step 14030/44000 | Loss: 3.1229 | LR: 3.92e-04 | 725.1ms/step | 27.6 samples/s +Step 14040/44000 | Loss: 3.0236 | LR: 3.92e-04 | 726.5ms/step | 27.5 samples/s +Step 14050/44000 | Loss: 3.0974 | LR: 3.92e-04 | 727.5ms/step | 27.5 samples/s +Step 14060/44000 | Loss: 3.1254 | LR: 3.92e-04 | 726.3ms/step | 27.5 samples/s +Step 14070/44000 | Loss: 3.1354 | LR: 3.92e-04 | 725.9ms/step | 27.6 samples/s +Step 14080/44000 | Loss: 3.0552 | LR: 3.92e-04 | 726.6ms/step | 27.5 samples/s +Step 14090/44000 | Loss: 2.9809 | LR: 3.92e-04 | 726.3ms/step | 27.5 samples/s +Step 14100/44000 | Loss: 3.0605 | LR: 3.91e-04 | 735.5ms/step | 27.2 samples/s +Step 14110/44000 | Loss: 3.0188 | LR: 3.91e-04 | 726.7ms/step | 27.5 samples/s +Step 14120/44000 | Loss: 3.0660 | LR: 3.91e-04 | 726.7ms/step | 27.5 samples/s +Step 14130/44000 | Loss: 3.0701 | LR: 3.91e-04 | 725.6ms/step | 27.6 samples/s +Step 14140/44000 | Loss: 3.0433 | LR: 3.91e-04 | 725.9ms/step | 27.6 samples/s +Step 14150/44000 | Loss: 3.0238 | LR: 3.91e-04 | 725.8ms/step | 27.6 samples/s +Step 14160/44000 | Loss: 3.0823 | LR: 3.90e-04 | 725.7ms/step | 27.6 samples/s +Step 14170/44000 | Loss: 3.0302 | LR: 3.90e-04 | 727.2ms/step | 27.5 samples/s +Step 14180/44000 | Loss: 3.0783 | LR: 3.90e-04 | 734.7ms/step | 27.2 samples/s +Step 14190/44000 | Loss: 3.0914 | LR: 3.90e-04 | 730.2ms/step | 27.4 samples/s +Step 14200/44000 | Loss: 3.0728 | LR: 3.90e-04 | 725.4ms/step | 27.6 samples/s +Step 14210/44000 | Loss: 3.0961 | LR: 3.90e-04 | 725.3ms/step | 27.6 samples/s +Step 14220/44000 | Loss: 3.0254 | LR: 3.90e-04 | 725.8ms/step | 27.6 samples/s +Step 14230/44000 | Loss: 3.1410 | LR: 3.89e-04 | 725.8ms/step | 27.6 samples/s +Step 14240/44000 | Loss: 3.0414 | LR: 3.89e-04 | 727.7ms/step | 27.5 samples/s +Step 14250/44000 | Loss: 3.1012 | LR: 3.89e-04 | 726.8ms/step | 27.5 samples/s +Validation at step 14250: Loss = 3.0589, Perplexity = 21.30 +Step 14260/44000 | Loss: 3.0828 | LR: 3.89e-04 | 949.1ms/step | 21.1 samples/s +Step 14270/44000 | Loss: 3.0723 | LR: 3.89e-04 | 736.4ms/step | 27.2 samples/s +Step 14280/44000 | Loss: 3.0688 | LR: 3.89e-04 | 726.2ms/step | 27.5 samples/s +Step 14290/44000 | Loss: 3.1045 | LR: 3.89e-04 | 726.0ms/step | 27.5 samples/s +Step 14300/44000 | Loss: 3.0602 | LR: 3.88e-04 | 727.4ms/step | 27.5 samples/s +Step 14310/44000 | Loss: 3.0379 | LR: 3.88e-04 | 727.1ms/step | 27.5 samples/s +Step 14320/44000 | Loss: 3.0806 | LR: 3.88e-04 | 725.9ms/step | 27.6 samples/s +Step 14330/44000 | Loss: 3.1225 | LR: 3.88e-04 | 726.4ms/step | 27.5 samples/s +Step 14340/44000 | Loss: 3.0654 | LR: 3.88e-04 | 725.7ms/step | 27.6 samples/s +Step 14350/44000 | Loss: 3.0820 | LR: 3.88e-04 | 728.4ms/step | 27.5 samples/s +Step 14360/44000 | Loss: 3.0483 | LR: 3.87e-04 | 737.7ms/step | 27.1 samples/s +Step 14370/44000 | Loss: 3.0448 | LR: 3.87e-04 | 726.8ms/step | 27.5 samples/s +Step 14380/44000 | Loss: 3.1419 | LR: 3.87e-04 | 726.2ms/step | 27.5 samples/s +Step 14390/44000 | Loss: 3.0369 | LR: 3.87e-04 | 725.4ms/step | 27.6 samples/s +Step 14400/44000 | Loss: 3.0317 | LR: 3.87e-04 | 725.8ms/step | 27.6 samples/s +Step 14410/44000 | Loss: 2.9896 | LR: 3.87e-04 | 725.9ms/step | 27.6 samples/s +Step 14420/44000 | Loss: 3.1016 | LR: 3.87e-04 | 725.7ms/step | 27.6 samples/s +Step 14430/44000 | Loss: 3.0750 | LR: 3.86e-04 | 726.4ms/step | 27.5 samples/s +Step 14440/44000 | Loss: 3.0187 | LR: 3.86e-04 | 732.4ms/step | 27.3 samples/s +Step 14450/44000 | Loss: 3.0529 | LR: 3.86e-04 | 730.3ms/step | 27.4 samples/s +Step 14460/44000 | Loss: 3.0420 | LR: 3.86e-04 | 725.6ms/step | 27.6 samples/s +Step 14470/44000 | Loss: 3.0900 | LR: 3.86e-04 | 726.1ms/step | 27.5 samples/s +Step 14480/44000 | Loss: 3.0700 | LR: 3.86e-04 | 725.7ms/step | 27.6 samples/s +Step 14490/44000 | Loss: 3.0854 | LR: 3.85e-04 | 727.0ms/step | 27.5 samples/s +Step 14500/44000 | Loss: 3.0808 | LR: 3.85e-04 | 727.3ms/step | 27.5 samples/s +Validation at step 14500: Loss = 3.0553, Perplexity = 21.23 +Step 14510/44000 | Loss: 3.0503 | LR: 3.85e-04 | 953.1ms/step | 21.0 samples/s +Step 14520/44000 | Loss: 3.0658 | LR: 3.85e-04 | 726.0ms/step | 27.5 samples/s +Step 14530/44000 | Loss: 3.0074 | LR: 3.85e-04 | 738.4ms/step | 27.1 samples/s +Step 14540/44000 | Loss: 3.0215 | LR: 3.85e-04 | 725.9ms/step | 27.6 samples/s +Step 14550/44000 | Loss: 3.0003 | LR: 3.85e-04 | 727.4ms/step | 27.5 samples/s +Step 14560/44000 | Loss: 3.0454 | LR: 3.84e-04 | 727.7ms/step | 27.5 samples/s +Step 14570/44000 | Loss: 3.0254 | LR: 3.84e-04 | 726.7ms/step | 27.5 samples/s +Step 14580/44000 | Loss: 3.1315 | LR: 3.84e-04 | 725.2ms/step | 27.6 samples/s +Step 14590/44000 | Loss: 3.0357 | LR: 3.84e-04 | 724.8ms/step | 27.6 samples/s +Step 14600/44000 | Loss: 3.0619 | LR: 3.84e-04 | 725.4ms/step | 27.6 samples/s +Step 14610/44000 | Loss: 3.0115 | LR: 3.84e-04 | 730.8ms/step | 27.4 samples/s +Step 14620/44000 | Loss: 3.0949 | LR: 3.83e-04 | 732.2ms/step | 27.3 samples/s +Step 14630/44000 | Loss: 2.9818 | LR: 3.83e-04 | 727.3ms/step | 27.5 samples/s +Step 14640/44000 | Loss: 3.0414 | LR: 3.83e-04 | 725.7ms/step | 27.6 samples/s +Step 14650/44000 | Loss: 3.0591 | LR: 3.83e-04 | 725.6ms/step | 27.6 samples/s +Step 14660/44000 | Loss: 3.0469 | LR: 3.83e-04 | 725.5ms/step | 27.6 samples/s +Step 14670/44000 | Loss: 3.0019 | LR: 3.83e-04 | 725.7ms/step | 27.6 samples/s +Step 14680/44000 | Loss: 3.0605 | LR: 3.83e-04 | 726.9ms/step | 27.5 samples/s +Step 14690/44000 | Loss: 3.0688 | LR: 3.82e-04 | 727.4ms/step | 27.5 samples/s +Step 14700/44000 | Loss: 3.0260 | LR: 3.82e-04 | 736.6ms/step | 27.2 samples/s +Step 14710/44000 | Loss: 3.0027 | LR: 3.82e-04 | 726.3ms/step | 27.5 samples/s +Step 14720/44000 | Loss: 3.0192 | LR: 3.82e-04 | 725.6ms/step | 27.6 samples/s +Step 14730/44000 | Loss: 3.0832 | LR: 3.82e-04 | 726.0ms/step | 27.5 samples/s +Step 14740/44000 | Loss: 3.0429 | LR: 3.82e-04 | 727.6ms/step | 27.5 samples/s +Step 14750/44000 | Loss: 3.1343 | LR: 3.81e-04 | 727.6ms/step | 27.5 samples/s +Validation at step 14750: Loss = 3.0500, Perplexity = 21.11 +Step 14760/44000 | Loss: 3.0870 | LR: 3.81e-04 | 952.5ms/step | 21.0 samples/s +Step 14770/44000 | Loss: 3.1128 | LR: 3.81e-04 | 726.0ms/step | 27.5 samples/s +Step 14780/44000 | Loss: 3.0574 | LR: 3.81e-04 | 726.1ms/step | 27.5 samples/s +Step 14790/44000 | Loss: 3.0680 | LR: 3.81e-04 | 735.3ms/step | 27.2 samples/s +Step 14800/44000 | Loss: 3.0265 | LR: 3.81e-04 | 726.0ms/step | 27.5 samples/s +Step 14810/44000 | Loss: 3.0306 | LR: 3.81e-04 | 727.6ms/step | 27.5 samples/s +Step 14820/44000 | Loss: 3.0775 | LR: 3.80e-04 | 726.2ms/step | 27.5 samples/s +Step 14830/44000 | Loss: 3.0444 | LR: 3.80e-04 | 726.1ms/step | 27.5 samples/s +Step 14840/44000 | Loss: 3.0775 | LR: 3.80e-04 | 725.6ms/step | 27.6 samples/s +Step 14850/44000 | Loss: 3.0471 | LR: 3.80e-04 | 725.9ms/step | 27.6 samples/s +Step 14860/44000 | Loss: 3.0190 | LR: 3.80e-04 | 725.7ms/step | 27.6 samples/s +Step 14870/44000 | Loss: 3.0639 | LR: 3.80e-04 | 731.8ms/step | 27.3 samples/s +Step 14880/44000 | Loss: 3.0807 | LR: 3.79e-04 | 731.6ms/step | 27.3 samples/s +Step 14890/44000 | Loss: 3.0689 | LR: 3.79e-04 | 725.7ms/step | 27.6 samples/s +Step 14900/44000 | Loss: 3.0741 | LR: 3.79e-04 | 725.9ms/step | 27.6 samples/s +Step 14910/44000 | Loss: 3.0580 | LR: 3.79e-04 | 725.8ms/step | 27.6 samples/s +Step 14920/44000 | Loss: 3.0422 | LR: 3.79e-04 | 725.6ms/step | 27.6 samples/s +Step 14930/44000 | Loss: 3.0306 | LR: 3.79e-04 | 727.5ms/step | 27.5 samples/s +Step 14940/44000 | Loss: 3.1146 | LR: 3.79e-04 | 727.7ms/step | 27.5 samples/s +Step 14950/44000 | Loss: 3.0520 | LR: 3.78e-04 | 725.8ms/step | 27.6 samples/s +Step 14960/44000 | Loss: 3.0934 | LR: 3.78e-04 | 739.2ms/step | 27.1 samples/s +Step 14970/44000 | Loss: 3.0644 | LR: 3.78e-04 | 725.9ms/step | 27.6 samples/s +Step 14980/44000 | Loss: 3.0765 | LR: 3.78e-04 | 725.6ms/step | 27.6 samples/s +Step 14990/44000 | Loss: 3.0491 | LR: 3.78e-04 | 726.8ms/step | 27.5 samples/s +Step 15000/44000 | Loss: 3.1483 | LR: 3.78e-04 | 727.7ms/step | 27.5 samples/s +Validation at step 15000: Loss = 3.0462, Perplexity = 21.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-15000 +Step 15010/44000 | Loss: 3.0615 | LR: 3.77e-04 | 1402.6ms/step | 14.3 samples/s +Step 15020/44000 | Loss: 3.0358 | LR: 3.77e-04 | 726.1ms/step | 27.5 samples/s +Step 15030/44000 | Loss: 3.0871 | LR: 3.77e-04 | 725.7ms/step | 27.6 samples/s +Step 15040/44000 | Loss: 3.0581 | LR: 3.77e-04 | 738.5ms/step | 27.1 samples/s +Step 15050/44000 | Loss: 3.0956 | LR: 3.77e-04 | 726.6ms/step | 27.5 samples/s +Step 15060/44000 | Loss: 3.0497 | LR: 3.77e-04 | 726.8ms/step | 27.5 samples/s +Step 15070/44000 | Loss: 3.0236 | LR: 3.77e-04 | 726.4ms/step | 27.5 samples/s +Step 15080/44000 | Loss: 2.9806 | LR: 3.76e-04 | 726.7ms/step | 27.5 samples/s +Step 15090/44000 | Loss: 2.9928 | LR: 3.76e-04 | 725.5ms/step | 27.6 samples/s +Step 15100/44000 | Loss: 3.0544 | LR: 3.76e-04 | 725.3ms/step | 27.6 samples/s +Step 15110/44000 | Loss: 3.0086 | LR: 3.76e-04 | 726.6ms/step | 27.5 samples/s +Step 15120/44000 | Loss: 3.0463 | LR: 3.76e-04 | 726.7ms/step | 27.5 samples/s +Step 15130/44000 | Loss: 3.1180 | LR: 3.76e-04 | 734.4ms/step | 27.2 samples/s +Step 15140/44000 | Loss: 2.9819 | LR: 3.75e-04 | 725.8ms/step | 27.6 samples/s +Step 15150/44000 | Loss: 3.0923 | LR: 3.75e-04 | 726.2ms/step | 27.5 samples/s +Step 15160/44000 | Loss: 3.0249 | LR: 3.75e-04 | 726.2ms/step | 27.5 samples/s +Step 15170/44000 | Loss: 3.0492 | LR: 3.75e-04 | 726.4ms/step | 27.5 samples/s +Step 15180/44000 | Loss: 3.0701 | LR: 3.75e-04 | 726.5ms/step | 27.5 samples/s +Step 15190/44000 | Loss: 3.0993 | LR: 3.75e-04 | 725.8ms/step | 27.6 samples/s +Step 15200/44000 | Loss: 3.0577 | LR: 3.74e-04 | 726.2ms/step | 27.5 samples/s +Step 15210/44000 | Loss: 3.0143 | LR: 3.74e-04 | 730.4ms/step | 27.4 samples/s +Step 15220/44000 | Loss: 3.0929 | LR: 3.74e-04 | 730.6ms/step | 27.4 samples/s +Step 15230/44000 | Loss: 3.0649 | LR: 3.74e-04 | 726.3ms/step | 27.5 samples/s +Step 15240/44000 | Loss: 3.0565 | LR: 3.74e-04 | 727.2ms/step | 27.5 samples/s +Step 15250/44000 | Loss: 3.0124 | LR: 3.74e-04 | 725.5ms/step | 27.6 samples/s +Validation at step 15250: Loss = 3.0432, Perplexity = 20.97 +Step 15260/44000 | Loss: 3.0622 | LR: 3.74e-04 | 951.4ms/step | 21.0 samples/s +Step 15270/44000 | Loss: 3.0675 | LR: 3.73e-04 | 725.8ms/step | 27.6 samples/s +Step 15280/44000 | Loss: 3.0529 | LR: 3.73e-04 | 726.1ms/step | 27.5 samples/s +Step 15290/44000 | Loss: 3.0772 | LR: 3.73e-04 | 728.1ms/step | 27.5 samples/s +Step 15300/44000 | Loss: 2.9896 | LR: 3.73e-04 | 737.3ms/step | 27.1 samples/s +Step 15310/44000 | Loss: 3.1180 | LR: 3.73e-04 | 725.8ms/step | 27.6 samples/s +Step 15320/44000 | Loss: 3.1493 | LR: 3.73e-04 | 726.1ms/step | 27.5 samples/s +Step 15330/44000 | Loss: 3.0748 | LR: 3.72e-04 | 726.7ms/step | 27.5 samples/s +Step 15340/44000 | Loss: 3.0654 | LR: 3.72e-04 | 725.9ms/step | 27.6 samples/s +Step 15350/44000 | Loss: 3.0717 | LR: 3.72e-04 | 727.5ms/step | 27.5 samples/s +Step 15360/44000 | Loss: 3.0525 | LR: 3.72e-04 | 727.2ms/step | 27.5 samples/s +Step 15370/44000 | Loss: 3.0247 | LR: 3.72e-04 | 726.9ms/step | 27.5 samples/s +Step 15380/44000 | Loss: 3.0243 | LR: 3.72e-04 | 730.6ms/step | 27.4 samples/s +Step 15390/44000 | Loss: 3.0621 | LR: 3.71e-04 | 731.3ms/step | 27.3 samples/s +Step 15400/44000 | Loss: 3.0349 | LR: 3.71e-04 | 726.4ms/step | 27.5 samples/s +Step 15410/44000 | Loss: 3.0186 | LR: 3.71e-04 | 727.5ms/step | 27.5 samples/s +Step 15420/44000 | Loss: 3.1028 | LR: 3.71e-04 | 727.1ms/step | 27.5 samples/s +Step 15430/44000 | Loss: 3.0522 | LR: 3.71e-04 | 726.3ms/step | 27.5 samples/s +Step 15440/44000 | Loss: 3.0050 | LR: 3.71e-04 | 726.1ms/step | 27.5 samples/s +Step 15450/44000 | Loss: 3.0635 | LR: 3.71e-04 | 725.7ms/step | 27.6 samples/s +Step 15460/44000 | Loss: 3.0215 | LR: 3.70e-04 | 725.5ms/step | 27.6 samples/s +Step 15470/44000 | Loss: 2.9839 | LR: 3.70e-04 | 739.7ms/step | 27.0 samples/s +Step 15480/44000 | Loss: 3.0588 | LR: 3.70e-04 | 727.5ms/step | 27.5 samples/s +Step 15490/44000 | Loss: 3.0593 | LR: 3.70e-04 | 725.9ms/step | 27.6 samples/s +Step 15500/44000 | Loss: 3.1200 | LR: 3.70e-04 | 726.0ms/step | 27.5 samples/s +Validation at step 15500: Loss = 3.0403, Perplexity = 20.91 +Step 15510/44000 | Loss: 3.0335 | LR: 3.70e-04 | 950.3ms/step | 21.0 samples/s +Step 15520/44000 | Loss: 3.1037 | LR: 3.69e-04 | 725.8ms/step | 27.6 samples/s +Step 15530/44000 | Loss: 3.0903 | LR: 3.69e-04 | 727.3ms/step | 27.5 samples/s +Step 15540/44000 | Loss: 3.0752 | LR: 3.69e-04 | 726.8ms/step | 27.5 samples/s +Step 15550/44000 | Loss: 3.0426 | LR: 3.69e-04 | 726.5ms/step | 27.5 samples/s +Step 15560/44000 | Loss: 3.1093 | LR: 3.69e-04 | 735.4ms/step | 27.2 samples/s +Step 15570/44000 | Loss: 3.0502 | LR: 3.69e-04 | 727.6ms/step | 27.5 samples/s +Step 15580/44000 | Loss: 3.0765 | LR: 3.68e-04 | 727.3ms/step | 27.5 samples/s +Step 15590/44000 | Loss: 3.0820 | LR: 3.68e-04 | 729.0ms/step | 27.4 samples/s +Step 15600/44000 | Loss: 3.0219 | LR: 3.68e-04 | 728.4ms/step | 27.5 samples/s +Step 15610/44000 | Loss: 3.0116 | LR: 3.68e-04 | 726.3ms/step | 27.5 samples/s +Step 15620/44000 | Loss: 3.0141 | LR: 3.68e-04 | 726.3ms/step | 27.5 samples/s +Step 15630/44000 | Loss: 3.0783 | LR: 3.68e-04 | 726.2ms/step | 27.5 samples/s +Step 15640/44000 | Loss: 3.1244 | LR: 3.67e-04 | 731.9ms/step | 27.3 samples/s +Step 15650/44000 | Loss: 3.0496 | LR: 3.67e-04 | 731.2ms/step | 27.4 samples/s +Step 15660/44000 | Loss: 3.0521 | LR: 3.67e-04 | 727.8ms/step | 27.5 samples/s +Step 15670/44000 | Loss: 3.0601 | LR: 3.67e-04 | 725.9ms/step | 27.6 samples/s +Step 15680/44000 | Loss: 3.1567 | LR: 3.67e-04 | 725.9ms/step | 27.6 samples/s +Step 15690/44000 | Loss: 3.0599 | LR: 3.67e-04 | 725.6ms/step | 27.6 samples/s +Step 15700/44000 | Loss: 3.0957 | LR: 3.67e-04 | 726.0ms/step | 27.5 samples/s +Step 15710/44000 | Loss: 3.1013 | LR: 3.66e-04 | 726.8ms/step | 27.5 samples/s +Step 15720/44000 | Loss: 3.0060 | LR: 3.66e-04 | 727.6ms/step | 27.5 samples/s +Step 15730/44000 | Loss: 3.0372 | LR: 3.66e-04 | 735.0ms/step | 27.2 samples/s +Step 15740/44000 | Loss: 3.0788 | LR: 3.66e-04 | 725.9ms/step | 27.6 samples/s +Step 15750/44000 | Loss: 3.0407 | LR: 3.66e-04 | 725.4ms/step | 27.6 samples/s +Validation at step 15750: Loss = 3.0390, Perplexity = 20.88 +Step 15760/44000 | Loss: 3.1182 | LR: 3.66e-04 | 951.2ms/step | 21.0 samples/s +Step 15770/44000 | Loss: 3.0546 | LR: 3.65e-04 | 728.2ms/step | 27.5 samples/s +Step 15780/44000 | Loss: 3.0742 | LR: 3.65e-04 | 727.6ms/step | 27.5 samples/s +Step 15790/44000 | Loss: 3.0328 | LR: 3.65e-04 | 726.2ms/step | 27.5 samples/s +Step 15800/44000 | Loss: 3.0523 | LR: 3.65e-04 | 726.5ms/step | 27.5 samples/s +Step 15810/44000 | Loss: 3.0176 | LR: 3.65e-04 | 732.9ms/step | 27.3 samples/s +Step 15820/44000 | Loss: 3.0631 | LR: 3.65e-04 | 731.3ms/step | 27.3 samples/s +Step 15830/44000 | Loss: 3.0511 | LR: 3.64e-04 | 727.4ms/step | 27.5 samples/s +Step 15840/44000 | Loss: 3.0037 | LR: 3.64e-04 | 726.6ms/step | 27.5 samples/s +Step 15850/44000 | Loss: 3.0812 | LR: 3.64e-04 | 725.7ms/step | 27.6 samples/s +Step 15860/44000 | Loss: 3.1420 | LR: 3.64e-04 | 726.0ms/step | 27.6 samples/s +Step 15870/44000 | Loss: 3.0520 | LR: 3.64e-04 | 725.8ms/step | 27.6 samples/s +Step 15880/44000 | Loss: 3.0379 | LR: 3.64e-04 | 725.7ms/step | 27.6 samples/s +Step 15890/44000 | Loss: 3.0966 | LR: 3.63e-04 | 726.8ms/step | 27.5 samples/s +Step 15900/44000 | Loss: 3.0238 | LR: 3.63e-04 | 732.3ms/step | 27.3 samples/s +Step 15910/44000 | Loss: 3.0471 | LR: 3.63e-04 | 730.8ms/step | 27.4 samples/s +Step 15920/44000 | Loss: 3.0640 | LR: 3.63e-04 | 726.1ms/step | 27.5 samples/s +Step 15930/44000 | Loss: 3.0586 | LR: 3.63e-04 | 726.1ms/step | 27.5 samples/s +Step 15940/44000 | Loss: 3.0871 | LR: 3.63e-04 | 725.6ms/step | 27.6 samples/s +Step 15950/44000 | Loss: 3.0033 | LR: 3.62e-04 | 726.6ms/step | 27.5 samples/s +Step 15960/44000 | Loss: 2.9686 | LR: 3.62e-04 | 727.3ms/step | 27.5 samples/s +Step 15970/44000 | Loss: 3.0828 | LR: 3.62e-04 | 725.6ms/step | 27.6 samples/s +Step 15980/44000 | Loss: 3.0374 | LR: 3.62e-04 | 725.3ms/step | 27.6 samples/s +Step 15990/44000 | Loss: 3.0476 | LR: 3.62e-04 | 739.5ms/step | 27.0 samples/s +Step 16000/44000 | Loss: 3.0800 | LR: 3.62e-04 | 725.5ms/step | 27.6 samples/s +Validation at step 16000: Loss = 3.0340, Perplexity = 20.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-16000 +Step 16010/44000 | Loss: 3.0808 | LR: 3.62e-04 | 1468.7ms/step | 13.6 samples/s +Step 16020/44000 | Loss: 3.0732 | LR: 3.61e-04 | 725.4ms/step | 27.6 samples/s +Step 16030/44000 | Loss: 3.0489 | LR: 3.61e-04 | 725.8ms/step | 27.6 samples/s +Step 16040/44000 | Loss: 3.0487 | LR: 3.61e-04 | 725.0ms/step | 27.6 samples/s +Step 16050/44000 | Loss: 3.0213 | LR: 3.61e-04 | 725.3ms/step | 27.6 samples/s +Step 16060/44000 | Loss: 3.0894 | LR: 3.61e-04 | 726.5ms/step | 27.5 samples/s +Step 16070/44000 | Loss: 2.9851 | LR: 3.61e-04 | 738.8ms/step | 27.1 samples/s +Step 16080/44000 | Loss: 3.0326 | LR: 3.60e-04 | 725.4ms/step | 27.6 samples/s +Step 16090/44000 | Loss: 3.0735 | LR: 3.60e-04 | 725.6ms/step | 27.6 samples/s +Step 16100/44000 | Loss: 3.0375 | LR: 3.60e-04 | 725.7ms/step | 27.6 samples/s +Step 16110/44000 | Loss: 3.0233 | LR: 3.60e-04 | 725.3ms/step | 27.6 samples/s +Step 16120/44000 | Loss: 3.0493 | LR: 3.60e-04 | 727.7ms/step | 27.5 samples/s +Step 16130/44000 | Loss: 3.0815 | LR: 3.60e-04 | 726.7ms/step | 27.5 samples/s +Step 16140/44000 | Loss: 3.0567 | LR: 3.59e-04 | 725.4ms/step | 27.6 samples/s +Step 16150/44000 | Loss: 3.0336 | LR: 3.59e-04 | 731.1ms/step | 27.4 samples/s +Step 16160/44000 | Loss: 3.0307 | LR: 3.59e-04 | 730.1ms/step | 27.4 samples/s +Step 16170/44000 | Loss: 3.1513 | LR: 3.59e-04 | 725.5ms/step | 27.6 samples/s +Step 16180/44000 | Loss: 3.0267 | LR: 3.59e-04 | 726.6ms/step | 27.5 samples/s +Step 16190/44000 | Loss: 3.0447 | LR: 3.59e-04 | 726.0ms/step | 27.5 samples/s +Step 16200/44000 | Loss: 3.0582 | LR: 3.58e-04 | 725.7ms/step | 27.6 samples/s +Step 16210/44000 | Loss: 3.0603 | LR: 3.58e-04 | 725.8ms/step | 27.6 samples/s +Step 16220/44000 | Loss: 3.0432 | LR: 3.58e-04 | 725.9ms/step | 27.6 samples/s +Step 16230/44000 | Loss: 3.0314 | LR: 3.58e-04 | 726.1ms/step | 27.5 samples/s +Step 16240/44000 | Loss: 3.0101 | LR: 3.58e-04 | 736.9ms/step | 27.1 samples/s +Step 16250/44000 | Loss: 3.0094 | LR: 3.58e-04 | 725.9ms/step | 27.6 samples/s +Validation at step 16250: Loss = 3.0309, Perplexity = 20.72 +Step 16260/44000 | Loss: 3.0540 | LR: 3.57e-04 | 950.9ms/step | 21.0 samples/s +Step 16270/44000 | Loss: 3.0292 | LR: 3.57e-04 | 725.5ms/step | 27.6 samples/s +Step 16280/44000 | Loss: 3.0746 | LR: 3.57e-04 | 725.6ms/step | 27.6 samples/s +Step 16290/44000 | Loss: 3.0572 | LR: 3.57e-04 | 726.8ms/step | 27.5 samples/s +Step 16300/44000 | Loss: 3.0072 | LR: 3.57e-04 | 727.4ms/step | 27.5 samples/s +Step 16310/44000 | Loss: 2.9983 | LR: 3.57e-04 | 726.2ms/step | 27.5 samples/s +Step 16320/44000 | Loss: 3.0229 | LR: 3.56e-04 | 729.9ms/step | 27.4 samples/s +Step 16330/44000 | Loss: 3.0288 | LR: 3.56e-04 | 734.4ms/step | 27.2 samples/s +Step 16340/44000 | Loss: 3.0475 | LR: 3.56e-04 | 725.6ms/step | 27.6 samples/s +Step 16350/44000 | Loss: 3.0587 | LR: 3.56e-04 | 726.4ms/step | 27.5 samples/s +Step 16360/44000 | Loss: 2.9860 | LR: 3.56e-04 | 726.3ms/step | 27.5 samples/s +Step 16370/44000 | Loss: 3.0019 | LR: 3.56e-04 | 725.4ms/step | 27.6 samples/s +Step 16380/44000 | Loss: 2.9884 | LR: 3.55e-04 | 724.6ms/step | 27.6 samples/s +Step 16390/44000 | Loss: 3.0842 | LR: 3.55e-04 | 724.8ms/step | 27.6 samples/s +Step 16400/44000 | Loss: 3.0550 | LR: 3.55e-04 | 725.1ms/step | 27.6 samples/s +Step 16410/44000 | Loss: 3.0697 | LR: 3.55e-04 | 732.7ms/step | 27.3 samples/s +Step 16420/44000 | Loss: 2.9633 | LR: 3.55e-04 | 730.3ms/step | 27.4 samples/s +Step 16430/44000 | Loss: 3.0421 | LR: 3.55e-04 | 724.8ms/step | 27.6 samples/s +Step 16440/44000 | Loss: 3.0310 | LR: 3.54e-04 | 724.8ms/step | 27.6 samples/s +Step 16450/44000 | Loss: 3.0500 | LR: 3.54e-04 | 724.9ms/step | 27.6 samples/s +Step 16460/44000 | Loss: 3.0769 | LR: 3.54e-04 | 724.9ms/step | 27.6 samples/s +Step 16470/44000 | Loss: 3.0806 | LR: 3.54e-04 | 727.4ms/step | 27.5 samples/s +Step 16480/44000 | Loss: 3.0346 | LR: 3.54e-04 | 725.2ms/step | 27.6 samples/s +Step 16490/44000 | Loss: 3.0140 | LR: 3.54e-04 | 725.0ms/step | 27.6 samples/s +Step 16500/44000 | Loss: 3.0705 | LR: 3.53e-04 | 735.1ms/step | 27.2 samples/s +Validation at step 16500: Loss = 3.0290, Perplexity = 20.68 +Step 16510/44000 | Loss: 3.0513 | LR: 3.53e-04 | 949.1ms/step | 21.1 samples/s +Step 16520/44000 | Loss: 3.0848 | LR: 3.53e-04 | 725.8ms/step | 27.6 samples/s +Step 16530/44000 | Loss: 3.1306 | LR: 3.53e-04 | 731.9ms/step | 27.3 samples/s +Step 16540/44000 | Loss: 3.0030 | LR: 3.53e-04 | 725.0ms/step | 27.6 samples/s +Step 16550/44000 | Loss: 3.0434 | LR: 3.53e-04 | 726.2ms/step | 27.5 samples/s +Step 16560/44000 | Loss: 2.9881 | LR: 3.52e-04 | 725.2ms/step | 27.6 samples/s +Step 16570/44000 | Loss: 3.0457 | LR: 3.52e-04 | 725.4ms/step | 27.6 samples/s +Step 16580/44000 | Loss: 3.0468 | LR: 3.52e-04 | 731.5ms/step | 27.3 samples/s +Step 16590/44000 | Loss: 3.1131 | LR: 3.52e-04 | 735.5ms/step | 27.2 samples/s +Step 16600/44000 | Loss: 3.0487 | LR: 3.52e-04 | 725.8ms/step | 27.6 samples/s +Step 16610/44000 | Loss: 3.0210 | LR: 3.52e-04 | 725.7ms/step | 27.6 samples/s +Step 16620/44000 | Loss: 3.0650 | LR: 3.51e-04 | 726.4ms/step | 27.5 samples/s +Step 16630/44000 | Loss: 2.9682 | LR: 3.51e-04 | 726.1ms/step | 27.5 samples/s +Step 16640/44000 | Loss: 3.0309 | LR: 3.51e-04 | 728.4ms/step | 27.5 samples/s +Step 16650/44000 | Loss: 3.0371 | LR: 3.51e-04 | 726.6ms/step | 27.5 samples/s +Step 16660/44000 | Loss: 3.0777 | LR: 3.51e-04 | 725.9ms/step | 27.6 samples/s +Step 16670/44000 | Loss: 3.0190 | LR: 3.51e-04 | 735.8ms/step | 27.2 samples/s +Step 16680/44000 | Loss: 2.9908 | LR: 3.50e-04 | 725.4ms/step | 27.6 samples/s +Step 16690/44000 | Loss: 3.0991 | LR: 3.50e-04 | 725.8ms/step | 27.6 samples/s +Step 16700/44000 | Loss: 3.0956 | LR: 3.50e-04 | 727.3ms/step | 27.5 samples/s +Step 16710/44000 | Loss: 3.0551 | LR: 3.50e-04 | 726.6ms/step | 27.5 samples/s +Step 16720/44000 | Loss: 2.9840 | LR: 3.50e-04 | 726.5ms/step | 27.5 samples/s +Step 16730/44000 | Loss: 3.0751 | LR: 3.50e-04 | 725.7ms/step | 27.6 samples/s +Step 16740/44000 | Loss: 3.0479 | LR: 3.49e-04 | 725.8ms/step | 27.6 samples/s +Step 16750/44000 | Loss: 3.0295 | LR: 3.49e-04 | 726.0ms/step | 27.5 samples/s +Validation at step 16750: Loss = 3.0244, Perplexity = 20.58 +Step 16760/44000 | Loss: 3.0806 | LR: 3.49e-04 | 967.4ms/step | 20.7 samples/s +Step 16770/44000 | Loss: 3.0344 | LR: 3.49e-04 | 726.7ms/step | 27.5 samples/s +Step 16780/44000 | Loss: 3.0773 | LR: 3.49e-04 | 725.6ms/step | 27.6 samples/s +Step 16790/44000 | Loss: 3.0106 | LR: 3.49e-04 | 726.8ms/step | 27.5 samples/s +Step 16800/44000 | Loss: 3.0472 | LR: 3.48e-04 | 726.3ms/step | 27.5 samples/s +Step 16810/44000 | Loss: 2.9990 | LR: 3.48e-04 | 727.0ms/step | 27.5 samples/s +Step 16820/44000 | Loss: 3.0752 | LR: 3.48e-04 | 727.2ms/step | 27.5 samples/s +Step 16830/44000 | Loss: 3.0434 | LR: 3.48e-04 | 726.2ms/step | 27.5 samples/s +Step 16840/44000 | Loss: 2.9827 | LR: 3.48e-04 | 731.2ms/step | 27.4 samples/s +Step 16850/44000 | Loss: 3.0658 | LR: 3.48e-04 | 731.0ms/step | 27.4 samples/s +Step 16860/44000 | Loss: 3.0713 | LR: 3.47e-04 | 725.8ms/step | 27.6 samples/s +Step 16870/44000 | Loss: 3.0616 | LR: 3.47e-04 | 728.7ms/step | 27.4 samples/s +Step 16880/44000 | Loss: 3.0328 | LR: 3.47e-04 | 726.9ms/step | 27.5 samples/s +Step 16890/44000 | Loss: 2.9815 | LR: 3.47e-04 | 726.0ms/step | 27.5 samples/s +Step 16900/44000 | Loss: 2.9822 | LR: 3.47e-04 | 726.0ms/step | 27.5 samples/s +Step 16910/44000 | Loss: 3.0547 | LR: 3.47e-04 | 726.0ms/step | 27.5 samples/s +Step 16920/44000 | Loss: 3.0271 | LR: 3.46e-04 | 726.1ms/step | 27.5 samples/s +Step 16930/44000 | Loss: 3.0259 | LR: 3.46e-04 | 741.2ms/step | 27.0 samples/s +Step 16940/44000 | Loss: 3.0748 | LR: 3.46e-04 | 725.8ms/step | 27.6 samples/s +Step 16950/44000 | Loss: 3.0009 | LR: 3.46e-04 | 725.8ms/step | 27.6 samples/s +Step 16960/44000 | Loss: 2.9543 | LR: 3.46e-04 | 725.5ms/step | 27.6 samples/s +Step 16970/44000 | Loss: 3.0222 | LR: 3.46e-04 | 725.8ms/step | 27.6 samples/s +Step 16980/44000 | Loss: 3.0045 | LR: 3.45e-04 | 726.2ms/step | 27.5 samples/s +Step 16990/44000 | Loss: 3.0347 | LR: 3.45e-04 | 727.2ms/step | 27.5 samples/s +Step 17000/44000 | Loss: 3.0164 | LR: 3.45e-04 | 726.0ms/step | 27.5 samples/s +Validation at step 17000: Loss = 3.0235, Perplexity = 20.56 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-17000 +Step 17010/44000 | Loss: 2.9506 | LR: 3.45e-04 | 1446.9ms/step | 13.8 samples/s +Step 17020/44000 | Loss: 3.0644 | LR: 3.45e-04 | 725.4ms/step | 27.6 samples/s +Step 17030/44000 | Loss: 3.0312 | LR: 3.45e-04 | 726.6ms/step | 27.5 samples/s +Step 17040/44000 | Loss: 3.0073 | LR: 3.44e-04 | 725.6ms/step | 27.6 samples/s +Step 17050/44000 | Loss: 3.0345 | LR: 3.44e-04 | 725.0ms/step | 27.6 samples/s +Step 17060/44000 | Loss: 3.0859 | LR: 3.44e-04 | 725.6ms/step | 27.6 samples/s +Step 17070/44000 | Loss: 2.9989 | LR: 3.44e-04 | 725.3ms/step | 27.6 samples/s +Step 17080/44000 | Loss: 3.0768 | LR: 3.44e-04 | 726.1ms/step | 27.5 samples/s +Step 17090/44000 | Loss: 2.9583 | LR: 3.44e-04 | 730.2ms/step | 27.4 samples/s +Step 17100/44000 | Loss: 2.9741 | LR: 3.43e-04 | 731.2ms/step | 27.4 samples/s +Step 17110/44000 | Loss: 3.0326 | LR: 3.43e-04 | 725.4ms/step | 27.6 samples/s +Step 17120/44000 | Loss: 3.1018 | LR: 3.43e-04 | 725.3ms/step | 27.6 samples/s +Step 17130/44000 | Loss: 3.0913 | LR: 3.43e-04 | 726.9ms/step | 27.5 samples/s +Step 17140/44000 | Loss: 3.0028 | LR: 3.43e-04 | 725.9ms/step | 27.6 samples/s +Step 17150/44000 | Loss: 2.9989 | LR: 3.43e-04 | 725.2ms/step | 27.6 samples/s +Step 17160/44000 | Loss: 3.0207 | LR: 3.42e-04 | 725.8ms/step | 27.6 samples/s +Step 17170/44000 | Loss: 3.0096 | LR: 3.42e-04 | 725.6ms/step | 27.6 samples/s +Step 17180/44000 | Loss: 3.0470 | LR: 3.42e-04 | 731.2ms/step | 27.4 samples/s +Step 17190/44000 | Loss: 3.0467 | LR: 3.42e-04 | 730.7ms/step | 27.4 samples/s +Step 17200/44000 | Loss: 3.0342 | LR: 3.42e-04 | 725.5ms/step | 27.6 samples/s +Step 17210/44000 | Loss: 2.9824 | LR: 3.42e-04 | 725.5ms/step | 27.6 samples/s +Step 17220/44000 | Loss: 3.0673 | LR: 3.41e-04 | 725.9ms/step | 27.6 samples/s +Step 17230/44000 | Loss: 3.0300 | LR: 3.41e-04 | 726.6ms/step | 27.5 samples/s +Step 17240/44000 | Loss: 2.9577 | LR: 3.41e-04 | 725.4ms/step | 27.6 samples/s +Step 17250/44000 | Loss: 2.9802 | LR: 3.41e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 17250: Loss = 3.0205, Perplexity = 20.50 +Step 17260/44000 | Loss: 3.0531 | LR: 3.41e-04 | 950.4ms/step | 21.0 samples/s +Step 17270/44000 | Loss: 3.0890 | LR: 3.41e-04 | 738.1ms/step | 27.1 samples/s +Step 17280/44000 | Loss: 3.0070 | LR: 3.40e-04 | 726.2ms/step | 27.5 samples/s +Step 17290/44000 | Loss: 2.9794 | LR: 3.40e-04 | 726.2ms/step | 27.5 samples/s +Step 17300/44000 | Loss: 3.0047 | LR: 3.40e-04 | 726.7ms/step | 27.5 samples/s +Step 17310/44000 | Loss: 2.9696 | LR: 3.40e-04 | 726.4ms/step | 27.5 samples/s +Step 17320/44000 | Loss: 2.9596 | LR: 3.40e-04 | 725.2ms/step | 27.6 samples/s +Step 17330/44000 | Loss: 3.0206 | LR: 3.40e-04 | 727.0ms/step | 27.5 samples/s +Step 17340/44000 | Loss: 3.0919 | LR: 3.39e-04 | 726.0ms/step | 27.5 samples/s +Step 17350/44000 | Loss: 2.9586 | LR: 3.39e-04 | 730.7ms/step | 27.4 samples/s +Step 17360/44000 | Loss: 3.0319 | LR: 3.39e-04 | 731.1ms/step | 27.4 samples/s +Step 17370/44000 | Loss: 3.0848 | LR: 3.39e-04 | 725.6ms/step | 27.6 samples/s +Step 17380/44000 | Loss: 2.9416 | LR: 3.39e-04 | 725.9ms/step | 27.6 samples/s +Step 17390/44000 | Loss: 3.0424 | LR: 3.39e-04 | 725.5ms/step | 27.6 samples/s +Step 17400/44000 | Loss: 3.0442 | LR: 3.38e-04 | 725.7ms/step | 27.6 samples/s +Step 17410/44000 | Loss: 3.0656 | LR: 3.38e-04 | 726.0ms/step | 27.5 samples/s +Step 17420/44000 | Loss: 3.0561 | LR: 3.38e-04 | 725.8ms/step | 27.6 samples/s +Step 17430/44000 | Loss: 3.0297 | LR: 3.38e-04 | 725.7ms/step | 27.6 samples/s +Step 17440/44000 | Loss: 3.0514 | LR: 3.38e-04 | 735.2ms/step | 27.2 samples/s +Step 17450/44000 | Loss: 3.0483 | LR: 3.38e-04 | 726.1ms/step | 27.5 samples/s +Step 17460/44000 | Loss: 3.0047 | LR: 3.37e-04 | 725.9ms/step | 27.6 samples/s +Step 17470/44000 | Loss: 2.9556 | LR: 3.37e-04 | 726.3ms/step | 27.5 samples/s +Step 17480/44000 | Loss: 3.0629 | LR: 3.37e-04 | 726.7ms/step | 27.5 samples/s +Step 17490/44000 | Loss: 2.9860 | LR: 3.37e-04 | 725.6ms/step | 27.6 samples/s +Step 17500/44000 | Loss: 3.0146 | LR: 3.37e-04 | 725.8ms/step | 27.6 samples/s +Validation at step 17500: Loss = 3.0168, Perplexity = 20.43 +Step 17510/44000 | Loss: 3.0330 | LR: 3.37e-04 | 950.3ms/step | 21.0 samples/s +Step 17520/44000 | Loss: 3.1260 | LR: 3.36e-04 | 726.6ms/step | 27.5 samples/s +Step 17530/44000 | Loss: 3.0147 | LR: 3.36e-04 | 735.1ms/step | 27.2 samples/s +Step 17540/44000 | Loss: 2.9939 | LR: 3.36e-04 | 726.3ms/step | 27.5 samples/s +Step 17550/44000 | Loss: 3.1027 | LR: 3.36e-04 | 726.8ms/step | 27.5 samples/s +Step 17560/44000 | Loss: 3.0265 | LR: 3.36e-04 | 726.4ms/step | 27.5 samples/s +Step 17570/44000 | Loss: 3.0459 | LR: 3.35e-04 | 726.3ms/step | 27.5 samples/s +Step 17580/44000 | Loss: 3.0417 | LR: 3.35e-04 | 725.3ms/step | 27.6 samples/s +Step 17590/44000 | Loss: 3.0313 | LR: 3.35e-04 | 726.1ms/step | 27.5 samples/s +Step 17600/44000 | Loss: 3.0344 | LR: 3.35e-04 | 725.7ms/step | 27.6 samples/s +Step 17610/44000 | Loss: 2.9979 | LR: 3.35e-04 | 730.1ms/step | 27.4 samples/s +Step 17620/44000 | Loss: 3.0560 | LR: 3.35e-04 | 731.5ms/step | 27.3 samples/s +Step 17630/44000 | Loss: 2.9927 | LR: 3.34e-04 | 725.8ms/step | 27.6 samples/s +Step 17640/44000 | Loss: 2.9994 | LR: 3.34e-04 | 726.0ms/step | 27.5 samples/s +Step 17650/44000 | Loss: 3.0513 | LR: 3.34e-04 | 726.2ms/step | 27.5 samples/s +Step 17660/44000 | Loss: 3.0378 | LR: 3.34e-04 | 726.3ms/step | 27.5 samples/s +Step 17670/44000 | Loss: 3.0826 | LR: 3.34e-04 | 726.0ms/step | 27.5 samples/s +Step 17680/44000 | Loss: 3.0152 | LR: 3.34e-04 | 725.8ms/step | 27.6 samples/s +Step 17690/44000 | Loss: 3.0028 | LR: 3.33e-04 | 726.1ms/step | 27.5 samples/s +Step 17700/44000 | Loss: 2.9703 | LR: 3.33e-04 | 739.2ms/step | 27.1 samples/s +Step 17710/44000 | Loss: 2.9864 | LR: 3.33e-04 | 727.3ms/step | 27.5 samples/s +Step 17720/44000 | Loss: 2.9718 | LR: 3.33e-04 | 727.0ms/step | 27.5 samples/s +Step 17730/44000 | Loss: 2.9788 | LR: 3.33e-04 | 726.6ms/step | 27.5 samples/s +Step 17740/44000 | Loss: 3.0582 | LR: 3.33e-04 | 726.3ms/step | 27.5 samples/s +Step 17750/44000 | Loss: 3.0447 | LR: 3.32e-04 | 726.3ms/step | 27.5 samples/s +Validation at step 17750: Loss = 3.0154, Perplexity = 20.40 +Step 17760/44000 | Loss: 3.0542 | LR: 3.32e-04 | 952.0ms/step | 21.0 samples/s +Step 17770/44000 | Loss: 3.0400 | LR: 3.32e-04 | 726.6ms/step | 27.5 samples/s +Step 17780/44000 | Loss: 3.0890 | LR: 3.32e-04 | 730.8ms/step | 27.4 samples/s +Step 17790/44000 | Loss: 3.0849 | LR: 3.32e-04 | 731.0ms/step | 27.4 samples/s +Step 17800/44000 | Loss: 3.0300 | LR: 3.32e-04 | 726.1ms/step | 27.5 samples/s +Step 17810/44000 | Loss: 3.0423 | LR: 3.31e-04 | 726.1ms/step | 27.5 samples/s +Step 17820/44000 | Loss: 3.0821 | LR: 3.31e-04 | 726.5ms/step | 27.5 samples/s +Step 17830/44000 | Loss: 3.0405 | LR: 3.31e-04 | 726.5ms/step | 27.5 samples/s +Step 17840/44000 | Loss: 3.0599 | LR: 3.31e-04 | 726.2ms/step | 27.5 samples/s +Step 17850/44000 | Loss: 2.9781 | LR: 3.31e-04 | 726.6ms/step | 27.5 samples/s +Step 17860/44000 | Loss: 3.0780 | LR: 3.31e-04 | 726.9ms/step | 27.5 samples/s +Step 17870/44000 | Loss: 3.0047 | LR: 3.30e-04 | 738.0ms/step | 27.1 samples/s +Step 17880/44000 | Loss: 3.0311 | LR: 3.30e-04 | 725.9ms/step | 27.6 samples/s +Step 17890/44000 | Loss: 2.9743 | LR: 3.30e-04 | 726.3ms/step | 27.5 samples/s +Step 17900/44000 | Loss: 3.0651 | LR: 3.30e-04 | 726.1ms/step | 27.5 samples/s +Step 17910/44000 | Loss: 3.0029 | LR: 3.30e-04 | 725.8ms/step | 27.6 samples/s +Step 17920/44000 | Loss: 2.9961 | LR: 3.29e-04 | 725.8ms/step | 27.6 samples/s +Step 17930/44000 | Loss: 2.9570 | LR: 3.29e-04 | 725.6ms/step | 27.6 samples/s +Step 17940/44000 | Loss: 3.0454 | LR: 3.29e-04 | 726.3ms/step | 27.5 samples/s +Step 17950/44000 | Loss: 3.0370 | LR: 3.29e-04 | 725.8ms/step | 27.6 samples/s +Step 17960/44000 | Loss: 3.0604 | LR: 3.29e-04 | 735.0ms/step | 27.2 samples/s +Step 17970/44000 | Loss: 3.0687 | LR: 3.29e-04 | 725.9ms/step | 27.6 samples/s +Step 17980/44000 | Loss: 2.9705 | LR: 3.28e-04 | 725.7ms/step | 27.6 samples/s +Step 17990/44000 | Loss: 2.9821 | LR: 3.28e-04 | 726.0ms/step | 27.5 samples/s +Step 18000/44000 | Loss: 3.0628 | LR: 3.28e-04 | 725.3ms/step | 27.6 samples/s +Validation at step 18000: Loss = 3.0108, Perplexity = 20.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-18000 +Step 18010/44000 | Loss: 3.0024 | LR: 3.28e-04 | 1382.1ms/step | 14.5 samples/s +Step 18020/44000 | Loss: 3.0399 | LR: 3.28e-04 | 725.8ms/step | 27.6 samples/s +Step 18030/44000 | Loss: 2.9791 | LR: 3.28e-04 | 726.1ms/step | 27.5 samples/s +Step 18040/44000 | Loss: 2.9844 | LR: 3.27e-04 | 736.2ms/step | 27.2 samples/s +Step 18050/44000 | Loss: 3.0784 | LR: 3.27e-04 | 725.8ms/step | 27.6 samples/s +Step 18060/44000 | Loss: 3.0704 | LR: 3.27e-04 | 726.0ms/step | 27.5 samples/s +Step 18070/44000 | Loss: 3.0522 | LR: 3.27e-04 | 725.8ms/step | 27.6 samples/s +Step 18080/44000 | Loss: 3.0163 | LR: 3.27e-04 | 725.3ms/step | 27.6 samples/s +Step 18090/44000 | Loss: 3.0055 | LR: 3.27e-04 | 725.9ms/step | 27.6 samples/s +Step 18100/44000 | Loss: 2.9791 | LR: 3.26e-04 | 725.5ms/step | 27.6 samples/s +Step 18110/44000 | Loss: 3.0400 | LR: 3.26e-04 | 725.8ms/step | 27.6 samples/s +Step 18120/44000 | Loss: 3.0644 | LR: 3.26e-04 | 732.1ms/step | 27.3 samples/s +Step 18130/44000 | Loss: 3.0162 | LR: 3.26e-04 | 733.7ms/step | 27.3 samples/s +Step 18140/44000 | Loss: 3.0043 | LR: 3.26e-04 | 725.8ms/step | 27.6 samples/s +Step 18150/44000 | Loss: 2.9837 | LR: 3.26e-04 | 726.1ms/step | 27.5 samples/s +Step 18160/44000 | Loss: 3.0092 | LR: 3.25e-04 | 725.5ms/step | 27.6 samples/s +Step 18170/44000 | Loss: 2.9765 | LR: 3.25e-04 | 725.5ms/step | 27.6 samples/s +Step 18180/44000 | Loss: 2.9502 | LR: 3.25e-04 | 725.8ms/step | 27.6 samples/s +Step 18190/44000 | Loss: 3.0180 | LR: 3.25e-04 | 725.6ms/step | 27.6 samples/s +Step 18200/44000 | Loss: 2.9713 | LR: 3.25e-04 | 725.3ms/step | 27.6 samples/s +Step 18210/44000 | Loss: 2.9611 | LR: 3.24e-04 | 730.1ms/step | 27.4 samples/s +Step 18220/44000 | Loss: 3.0602 | LR: 3.24e-04 | 730.4ms/step | 27.4 samples/s +Step 18230/44000 | Loss: 3.0379 | LR: 3.24e-04 | 725.5ms/step | 27.6 samples/s +Step 18240/44000 | Loss: 2.9681 | LR: 3.24e-04 | 725.3ms/step | 27.6 samples/s +Step 18250/44000 | Loss: 2.9464 | LR: 3.24e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 18250: Loss = 3.0083, Perplexity = 20.25 +Step 18260/44000 | Loss: 2.9764 | LR: 3.24e-04 | 950.7ms/step | 21.0 samples/s +Step 18270/44000 | Loss: 3.0256 | LR: 3.23e-04 | 725.9ms/step | 27.6 samples/s +Step 18280/44000 | Loss: 2.9703 | LR: 3.23e-04 | 726.0ms/step | 27.5 samples/s +Step 18290/44000 | Loss: 2.9649 | LR: 3.23e-04 | 726.1ms/step | 27.5 samples/s +Step 18300/44000 | Loss: 3.0151 | LR: 3.23e-04 | 737.3ms/step | 27.1 samples/s +Step 18310/44000 | Loss: 3.0059 | LR: 3.23e-04 | 726.6ms/step | 27.5 samples/s +Step 18320/44000 | Loss: 3.0605 | LR: 3.23e-04 | 725.6ms/step | 27.6 samples/s +Step 18330/44000 | Loss: 3.0195 | LR: 3.22e-04 | 725.6ms/step | 27.6 samples/s +Step 18340/44000 | Loss: 2.9492 | LR: 3.22e-04 | 725.8ms/step | 27.6 samples/s +Step 18350/44000 | Loss: 3.0238 | LR: 3.22e-04 | 726.7ms/step | 27.5 samples/s +Step 18360/44000 | Loss: 3.0851 | LR: 3.22e-04 | 726.5ms/step | 27.5 samples/s +Step 18370/44000 | Loss: 2.9530 | LR: 3.22e-04 | 726.7ms/step | 27.5 samples/s +Step 18380/44000 | Loss: 3.0022 | LR: 3.22e-04 | 731.9ms/step | 27.3 samples/s +Step 18390/44000 | Loss: 3.0367 | LR: 3.21e-04 | 731.8ms/step | 27.3 samples/s +Step 18400/44000 | Loss: 3.0325 | LR: 3.21e-04 | 726.0ms/step | 27.5 samples/s +Step 18410/44000 | Loss: 2.9709 | LR: 3.21e-04 | 726.8ms/step | 27.5 samples/s +Step 18420/44000 | Loss: 2.9561 | LR: 3.21e-04 | 726.0ms/step | 27.5 samples/s +Step 18430/44000 | Loss: 2.9497 | LR: 3.21e-04 | 726.3ms/step | 27.5 samples/s +Step 18440/44000 | Loss: 3.0335 | LR: 3.20e-04 | 726.8ms/step | 27.5 samples/s +Step 18450/44000 | Loss: 2.9795 | LR: 3.20e-04 | 726.3ms/step | 27.5 samples/s +Step 18460/44000 | Loss: 2.9478 | LR: 3.20e-04 | 725.9ms/step | 27.6 samples/s +Step 18470/44000 | Loss: 3.0272 | LR: 3.20e-04 | 736.5ms/step | 27.2 samples/s +Step 18480/44000 | Loss: 2.9751 | LR: 3.20e-04 | 725.4ms/step | 27.6 samples/s +Step 18490/44000 | Loss: 2.9657 | LR: 3.20e-04 | 725.9ms/step | 27.6 samples/s +Step 18500/44000 | Loss: 3.0210 | LR: 3.19e-04 | 726.2ms/step | 27.5 samples/s +Validation at step 18500: Loss = 3.0058, Perplexity = 20.20 +Step 18510/44000 | Loss: 3.0682 | LR: 3.19e-04 | 949.8ms/step | 21.1 samples/s +Step 18520/44000 | Loss: 3.0587 | LR: 3.19e-04 | 725.8ms/step | 27.6 samples/s +Step 18530/44000 | Loss: 3.0241 | LR: 3.19e-04 | 726.2ms/step | 27.5 samples/s +Step 18540/44000 | Loss: 2.9926 | LR: 3.19e-04 | 726.3ms/step | 27.5 samples/s +Step 18550/44000 | Loss: 2.9589 | LR: 3.19e-04 | 731.9ms/step | 27.3 samples/s +Step 18560/44000 | Loss: 3.0336 | LR: 3.18e-04 | 731.0ms/step | 27.4 samples/s +Step 18570/44000 | Loss: 3.0489 | LR: 3.18e-04 | 725.8ms/step | 27.6 samples/s +Step 18580/44000 | Loss: 2.9753 | LR: 3.18e-04 | 726.5ms/step | 27.5 samples/s +Step 18590/44000 | Loss: 2.9901 | LR: 3.18e-04 | 726.1ms/step | 27.5 samples/s +Step 18600/44000 | Loss: 2.9903 | LR: 3.18e-04 | 726.4ms/step | 27.5 samples/s +Step 18610/44000 | Loss: 3.0169 | LR: 3.18e-04 | 726.1ms/step | 27.5 samples/s +Step 18620/44000 | Loss: 3.0080 | LR: 3.17e-04 | 725.6ms/step | 27.6 samples/s +Step 18630/44000 | Loss: 2.9950 | LR: 3.17e-04 | 725.9ms/step | 27.6 samples/s +Step 18640/44000 | Loss: 2.9818 | LR: 3.17e-04 | 730.8ms/step | 27.4 samples/s +Step 18650/44000 | Loss: 2.9941 | LR: 3.17e-04 | 734.3ms/step | 27.2 samples/s +Step 18660/44000 | Loss: 3.0099 | LR: 3.17e-04 | 726.5ms/step | 27.5 samples/s +Step 18670/44000 | Loss: 3.0287 | LR: 3.16e-04 | 725.5ms/step | 27.6 samples/s +Step 18680/44000 | Loss: 3.0004 | LR: 3.16e-04 | 726.3ms/step | 27.5 samples/s +Step 18690/44000 | Loss: 3.0133 | LR: 3.16e-04 | 725.7ms/step | 27.6 samples/s +Step 18700/44000 | Loss: 3.0326 | LR: 3.16e-04 | 726.0ms/step | 27.6 samples/s +Step 18710/44000 | Loss: 2.9744 | LR: 3.16e-04 | 726.2ms/step | 27.5 samples/s +Step 18720/44000 | Loss: 3.0148 | LR: 3.16e-04 | 725.8ms/step | 27.6 samples/s +Step 18730/44000 | Loss: 2.9461 | LR: 3.15e-04 | 738.6ms/step | 27.1 samples/s +Step 18740/44000 | Loss: 2.9960 | LR: 3.15e-04 | 725.6ms/step | 27.6 samples/s +Step 18750/44000 | Loss: 3.0006 | LR: 3.15e-04 | 726.2ms/step | 27.5 samples/s +Validation at step 18750: Loss = 3.0023, Perplexity = 20.13 +Step 18760/44000 | Loss: 3.0354 | LR: 3.15e-04 | 949.4ms/step | 21.1 samples/s +Step 18770/44000 | Loss: 2.9852 | LR: 3.15e-04 | 726.1ms/step | 27.5 samples/s +Step 18780/44000 | Loss: 3.0288 | LR: 3.15e-04 | 725.7ms/step | 27.6 samples/s +Step 18790/44000 | Loss: 3.0925 | LR: 3.14e-04 | 726.2ms/step | 27.5 samples/s +Step 18800/44000 | Loss: 2.9917 | LR: 3.14e-04 | 726.2ms/step | 27.5 samples/s +Step 18810/44000 | Loss: 3.0576 | LR: 3.14e-04 | 730.6ms/step | 27.4 samples/s +Step 18820/44000 | Loss: 2.9939 | LR: 3.14e-04 | 731.0ms/step | 27.4 samples/s +Step 18830/44000 | Loss: 2.9424 | LR: 3.14e-04 | 725.8ms/step | 27.6 samples/s +Step 18840/44000 | Loss: 2.9609 | LR: 3.13e-04 | 725.6ms/step | 27.6 samples/s +Step 18850/44000 | Loss: 2.9902 | LR: 3.13e-04 | 725.8ms/step | 27.6 samples/s +Step 18860/44000 | Loss: 2.9473 | LR: 3.13e-04 | 726.3ms/step | 27.5 samples/s +Step 18870/44000 | Loss: 3.0294 | LR: 3.13e-04 | 726.0ms/step | 27.5 samples/s +Step 18880/44000 | Loss: 3.0401 | LR: 3.13e-04 | 725.5ms/step | 27.6 samples/s +Step 18890/44000 | Loss: 3.0094 | LR: 3.13e-04 | 726.3ms/step | 27.5 samples/s +Step 18900/44000 | Loss: 2.9640 | LR: 3.12e-04 | 739.0ms/step | 27.1 samples/s +Step 18910/44000 | Loss: 3.0398 | LR: 3.12e-04 | 727.0ms/step | 27.5 samples/s +Step 18920/44000 | Loss: 3.0286 | LR: 3.12e-04 | 726.1ms/step | 27.5 samples/s +Step 18930/44000 | Loss: 2.9604 | LR: 3.12e-04 | 725.9ms/step | 27.6 samples/s +Step 18940/44000 | Loss: 2.9675 | LR: 3.12e-04 | 726.3ms/step | 27.5 samples/s +Step 18950/44000 | Loss: 3.0477 | LR: 3.12e-04 | 726.1ms/step | 27.5 samples/s +Step 18960/44000 | Loss: 2.9650 | LR: 3.11e-04 | 726.2ms/step | 27.5 samples/s +Step 18970/44000 | Loss: 2.9563 | LR: 3.11e-04 | 728.0ms/step | 27.5 samples/s +Step 18980/44000 | Loss: 2.9796 | LR: 3.11e-04 | 728.0ms/step | 27.5 samples/s +Step 18990/44000 | Loss: 3.0043 | LR: 3.11e-04 | 735.9ms/step | 27.2 samples/s +Step 19000/44000 | Loss: 2.9414 | LR: 3.11e-04 | 726.9ms/step | 27.5 samples/s +Validation at step 19000: Loss = 3.0031, Perplexity = 20.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-19000 +Step 19010/44000 | Loss: 3.0727 | LR: 3.11e-04 | 1430.2ms/step | 14.0 samples/s +Step 19020/44000 | Loss: 3.0632 | LR: 3.10e-04 | 725.2ms/step | 27.6 samples/s +Step 19030/44000 | Loss: 3.0302 | LR: 3.10e-04 | 724.9ms/step | 27.6 samples/s +Step 19040/44000 | Loss: 3.0150 | LR: 3.10e-04 | 725.0ms/step | 27.6 samples/s +Step 19050/44000 | Loss: 2.9839 | LR: 3.10e-04 | 724.8ms/step | 27.6 samples/s +Step 19060/44000 | Loss: 3.0111 | LR: 3.10e-04 | 725.4ms/step | 27.6 samples/s +Step 19070/44000 | Loss: 3.0243 | LR: 3.09e-04 | 734.0ms/step | 27.2 samples/s +Step 19080/44000 | Loss: 2.9896 | LR: 3.09e-04 | 725.2ms/step | 27.6 samples/s +Step 19090/44000 | Loss: 3.0393 | LR: 3.09e-04 | 725.4ms/step | 27.6 samples/s +Step 19100/44000 | Loss: 2.9953 | LR: 3.09e-04 | 726.0ms/step | 27.6 samples/s +Step 19110/44000 | Loss: 3.0125 | LR: 3.09e-04 | 725.5ms/step | 27.6 samples/s +Step 19120/44000 | Loss: 3.0274 | LR: 3.09e-04 | 725.0ms/step | 27.6 samples/s +Step 19130/44000 | Loss: 2.9799 | LR: 3.08e-04 | 725.2ms/step | 27.6 samples/s +Step 19140/44000 | Loss: 2.9938 | LR: 3.08e-04 | 725.0ms/step | 27.6 samples/s +Step 19150/44000 | Loss: 2.9691 | LR: 3.08e-04 | 730.1ms/step | 27.4 samples/s +Step 19160/44000 | Loss: 3.0172 | LR: 3.08e-04 | 730.6ms/step | 27.4 samples/s +Step 19170/44000 | Loss: 3.0245 | LR: 3.08e-04 | 725.9ms/step | 27.6 samples/s +Step 19180/44000 | Loss: 2.9901 | LR: 3.08e-04 | 726.1ms/step | 27.5 samples/s +Step 19190/44000 | Loss: 2.9539 | LR: 3.07e-04 | 725.9ms/step | 27.6 samples/s +Step 19200/44000 | Loss: 3.0346 | LR: 3.07e-04 | 725.7ms/step | 27.6 samples/s +Step 19210/44000 | Loss: 3.0063 | LR: 3.07e-04 | 725.8ms/step | 27.6 samples/s +Step 19220/44000 | Loss: 3.0304 | LR: 3.07e-04 | 726.1ms/step | 27.5 samples/s +Step 19230/44000 | Loss: 2.9937 | LR: 3.07e-04 | 726.0ms/step | 27.5 samples/s +Step 19240/44000 | Loss: 3.0344 | LR: 3.06e-04 | 739.6ms/step | 27.0 samples/s +Step 19250/44000 | Loss: 2.9858 | LR: 3.06e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 19250: Loss = 2.9978, Perplexity = 20.04 +Step 19260/44000 | Loss: 3.0446 | LR: 3.06e-04 | 951.0ms/step | 21.0 samples/s +Step 19270/44000 | Loss: 2.9811 | LR: 3.06e-04 | 725.8ms/step | 27.6 samples/s +Step 19280/44000 | Loss: 3.0463 | LR: 3.06e-04 | 727.2ms/step | 27.5 samples/s +Step 19290/44000 | Loss: 2.9596 | LR: 3.06e-04 | 726.6ms/step | 27.5 samples/s +Step 19300/44000 | Loss: 2.9322 | LR: 3.05e-04 | 725.9ms/step | 27.6 samples/s +Step 19310/44000 | Loss: 3.0142 | LR: 3.05e-04 | 725.8ms/step | 27.6 samples/s +Step 19320/44000 | Loss: 3.0077 | LR: 3.05e-04 | 730.3ms/step | 27.4 samples/s +Step 19330/44000 | Loss: 3.0583 | LR: 3.05e-04 | 730.1ms/step | 27.4 samples/s +Step 19340/44000 | Loss: 2.9653 | LR: 3.05e-04 | 725.6ms/step | 27.6 samples/s +Step 19350/44000 | Loss: 2.9561 | LR: 3.04e-04 | 726.2ms/step | 27.5 samples/s +Step 19360/44000 | Loss: 3.0709 | LR: 3.04e-04 | 726.1ms/step | 27.5 samples/s +Step 19370/44000 | Loss: 3.0135 | LR: 3.04e-04 | 726.2ms/step | 27.5 samples/s +Step 19380/44000 | Loss: 2.9864 | LR: 3.04e-04 | 725.8ms/step | 27.6 samples/s +Step 19390/44000 | Loss: 2.9780 | LR: 3.04e-04 | 725.6ms/step | 27.6 samples/s +Step 19400/44000 | Loss: 2.9884 | LR: 3.04e-04 | 725.8ms/step | 27.6 samples/s +Step 19410/44000 | Loss: 3.0650 | LR: 3.03e-04 | 732.4ms/step | 27.3 samples/s +Step 19420/44000 | Loss: 2.9973 | LR: 3.03e-04 | 732.9ms/step | 27.3 samples/s +Step 19430/44000 | Loss: 3.0137 | LR: 3.03e-04 | 725.6ms/step | 27.6 samples/s +Step 19440/44000 | Loss: 2.9510 | LR: 3.03e-04 | 725.2ms/step | 27.6 samples/s +Step 19450/44000 | Loss: 3.0135 | LR: 3.03e-04 | 726.1ms/step | 27.5 samples/s +Step 19460/44000 | Loss: 2.9732 | LR: 3.03e-04 | 725.0ms/step | 27.6 samples/s +Step 19470/44000 | Loss: 2.9928 | LR: 3.02e-04 | 725.7ms/step | 27.6 samples/s +Step 19480/44000 | Loss: 3.0060 | LR: 3.02e-04 | 725.9ms/step | 27.6 samples/s +Step 19490/44000 | Loss: 2.9756 | LR: 3.02e-04 | 725.8ms/step | 27.6 samples/s +Step 19500/44000 | Loss: 2.9935 | LR: 3.02e-04 | 737.3ms/step | 27.1 samples/s +Validation at step 19500: Loss = 2.9963, Perplexity = 20.01 +Step 19510/44000 | Loss: 3.0336 | LR: 3.02e-04 | 949.0ms/step | 21.1 samples/s +Step 19520/44000 | Loss: 2.9517 | LR: 3.01e-04 | 725.9ms/step | 27.6 samples/s +Step 19530/44000 | Loss: 2.9611 | LR: 3.01e-04 | 725.7ms/step | 27.6 samples/s +Step 19540/44000 | Loss: 2.9904 | LR: 3.01e-04 | 725.9ms/step | 27.6 samples/s +Step 19550/44000 | Loss: 3.0264 | LR: 3.01e-04 | 726.2ms/step | 27.5 samples/s +Step 19560/44000 | Loss: 3.0001 | LR: 3.01e-04 | 725.3ms/step | 27.6 samples/s +Step 19570/44000 | Loss: 2.9002 | LR: 3.01e-04 | 724.8ms/step | 27.6 samples/s +Step 19580/44000 | Loss: 3.0239 | LR: 3.00e-04 | 729.7ms/step | 27.4 samples/s +Step 19590/44000 | Loss: 3.0269 | LR: 3.00e-04 | 730.0ms/step | 27.4 samples/s +Step 19600/44000 | Loss: 3.0121 | LR: 3.00e-04 | 725.0ms/step | 27.6 samples/s +Step 19610/44000 | Loss: 2.9634 | LR: 3.00e-04 | 725.6ms/step | 27.6 samples/s +Step 19620/44000 | Loss: 3.0368 | LR: 3.00e-04 | 725.5ms/step | 27.6 samples/s +Step 19630/44000 | Loss: 3.0086 | LR: 3.00e-04 | 725.5ms/step | 27.6 samples/s +Step 19640/44000 | Loss: 3.0147 | LR: 2.99e-04 | 725.4ms/step | 27.6 samples/s +Step 19650/44000 | Loss: 3.0073 | LR: 2.99e-04 | 725.5ms/step | 27.6 samples/s +Step 19660/44000 | Loss: 3.0664 | LR: 2.99e-04 | 725.1ms/step | 27.6 samples/s +Step 19670/44000 | Loss: 3.0981 | LR: 2.99e-04 | 737.5ms/step | 27.1 samples/s +Step 19680/44000 | Loss: 3.0221 | LR: 2.99e-04 | 725.5ms/step | 27.6 samples/s +Step 19690/44000 | Loss: 3.0266 | LR: 2.98e-04 | 725.3ms/step | 27.6 samples/s +Step 19700/44000 | Loss: 3.0217 | LR: 2.98e-04 | 725.1ms/step | 27.6 samples/s +Step 19710/44000 | Loss: 3.0036 | LR: 2.98e-04 | 725.2ms/step | 27.6 samples/s +Step 19720/44000 | Loss: 3.0669 | LR: 2.98e-04 | 725.5ms/step | 27.6 samples/s +Step 19730/44000 | Loss: 2.9089 | LR: 2.98e-04 | 725.5ms/step | 27.6 samples/s +Step 19740/44000 | Loss: 3.0299 | LR: 2.98e-04 | 725.6ms/step | 27.6 samples/s +Step 19750/44000 | Loss: 2.9921 | LR: 2.97e-04 | 725.1ms/step | 27.6 samples/s +Validation at step 19750: Loss = 2.9928, Perplexity = 19.94 +Step 19760/44000 | Loss: 2.9869 | LR: 2.97e-04 | 961.1ms/step | 20.8 samples/s +Step 19770/44000 | Loss: 3.0360 | LR: 2.97e-04 | 725.8ms/step | 27.6 samples/s +Step 19780/44000 | Loss: 3.0184 | LR: 2.97e-04 | 725.0ms/step | 27.6 samples/s +Step 19790/44000 | Loss: 3.0352 | LR: 2.97e-04 | 724.7ms/step | 27.6 samples/s +Step 19800/44000 | Loss: 2.9833 | LR: 2.96e-04 | 725.4ms/step | 27.6 samples/s +Step 19810/44000 | Loss: 2.9929 | LR: 2.96e-04 | 725.4ms/step | 27.6 samples/s +Step 19820/44000 | Loss: 2.9921 | LR: 2.96e-04 | 725.8ms/step | 27.6 samples/s +Step 19830/44000 | Loss: 2.9638 | LR: 2.96e-04 | 726.5ms/step | 27.5 samples/s +Step 19840/44000 | Loss: 2.9118 | LR: 2.96e-04 | 731.2ms/step | 27.4 samples/s +Step 19850/44000 | Loss: 2.9718 | LR: 2.96e-04 | 730.2ms/step | 27.4 samples/s +Step 19860/44000 | Loss: 3.0069 | LR: 2.95e-04 | 725.8ms/step | 27.6 samples/s +Step 19870/44000 | Loss: 2.9914 | LR: 2.95e-04 | 725.6ms/step | 27.6 samples/s +Step 19880/44000 | Loss: 2.9907 | LR: 2.95e-04 | 725.1ms/step | 27.6 samples/s +Step 19890/44000 | Loss: 2.9393 | LR: 2.95e-04 | 725.3ms/step | 27.6 samples/s +Step 19900/44000 | Loss: 2.9538 | LR: 2.95e-04 | 725.6ms/step | 27.6 samples/s +Step 19910/44000 | Loss: 2.9585 | LR: 2.95e-04 | 725.7ms/step | 27.6 samples/s +Step 19920/44000 | Loss: 2.9694 | LR: 2.94e-04 | 725.6ms/step | 27.6 samples/s +Step 19930/44000 | Loss: 3.0251 | LR: 2.94e-04 | 735.5ms/step | 27.2 samples/s +Step 19940/44000 | Loss: 2.9083 | LR: 2.94e-04 | 725.0ms/step | 27.6 samples/s +Step 19950/44000 | Loss: 3.0056 | LR: 2.94e-04 | 725.6ms/step | 27.6 samples/s +Step 19960/44000 | Loss: 3.0414 | LR: 2.94e-04 | 724.9ms/step | 27.6 samples/s +Step 19970/44000 | Loss: 3.0501 | LR: 2.93e-04 | 724.8ms/step | 27.6 samples/s +Step 19980/44000 | Loss: 3.0549 | LR: 2.93e-04 | 724.9ms/step | 27.6 samples/s +Step 19990/44000 | Loss: 2.9188 | LR: 2.93e-04 | 725.4ms/step | 27.6 samples/s +Step 20000/44000 | Loss: 3.0178 | LR: 2.93e-04 | 725.1ms/step | 27.6 samples/s +Validation at step 20000: Loss = 2.9880, Perplexity = 19.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-20000 +Step 20010/44000 | Loss: 3.0003 | LR: 2.93e-04 | 1433.1ms/step | 14.0 samples/s +Step 20020/44000 | Loss: 3.0253 | LR: 2.93e-04 | 725.0ms/step | 27.6 samples/s +Step 20030/44000 | Loss: 2.9422 | LR: 2.92e-04 | 725.1ms/step | 27.6 samples/s +Step 20040/44000 | Loss: 3.0158 | LR: 2.92e-04 | 724.8ms/step | 27.6 samples/s +Step 20050/44000 | Loss: 3.0232 | LR: 2.92e-04 | 725.3ms/step | 27.6 samples/s +Step 20060/44000 | Loss: 2.9497 | LR: 2.92e-04 | 726.9ms/step | 27.5 samples/s +Step 20070/44000 | Loss: 2.9739 | LR: 2.92e-04 | 725.6ms/step | 27.6 samples/s +Step 20080/44000 | Loss: 3.0305 | LR: 2.91e-04 | 725.9ms/step | 27.6 samples/s +Step 20090/44000 | Loss: 3.0211 | LR: 2.91e-04 | 725.4ms/step | 27.6 samples/s +Step 20100/44000 | Loss: 3.0456 | LR: 2.91e-04 | 734.9ms/step | 27.2 samples/s +Step 20110/44000 | Loss: 2.9970 | LR: 2.91e-04 | 725.5ms/step | 27.6 samples/s +Step 20120/44000 | Loss: 2.9690 | LR: 2.91e-04 | 726.0ms/step | 27.5 samples/s +Step 20130/44000 | Loss: 3.0122 | LR: 2.91e-04 | 725.9ms/step | 27.6 samples/s +Step 20140/44000 | Loss: 3.0370 | LR: 2.90e-04 | 725.9ms/step | 27.6 samples/s +Step 20150/44000 | Loss: 2.9332 | LR: 2.90e-04 | 725.4ms/step | 27.6 samples/s +Step 20160/44000 | Loss: 2.9838 | LR: 2.90e-04 | 725.2ms/step | 27.6 samples/s +Step 20170/44000 | Loss: 3.0121 | LR: 2.90e-04 | 725.4ms/step | 27.6 samples/s +Step 20180/44000 | Loss: 3.0169 | LR: 2.90e-04 | 729.9ms/step | 27.4 samples/s +Step 20190/44000 | Loss: 2.9649 | LR: 2.90e-04 | 729.8ms/step | 27.4 samples/s +Step 20200/44000 | Loss: 2.9938 | LR: 2.89e-04 | 724.5ms/step | 27.6 samples/s +Step 20210/44000 | Loss: 2.9822 | LR: 2.89e-04 | 725.4ms/step | 27.6 samples/s +Step 20220/44000 | Loss: 3.0197 | LR: 2.89e-04 | 726.2ms/step | 27.5 samples/s +Step 20230/44000 | Loss: 3.1093 | LR: 2.89e-04 | 725.2ms/step | 27.6 samples/s +Step 20240/44000 | Loss: 3.0473 | LR: 2.89e-04 | 725.2ms/step | 27.6 samples/s +Step 20250/44000 | Loss: 2.9373 | LR: 2.88e-04 | 724.9ms/step | 27.6 samples/s +Validation at step 20250: Loss = 2.9893, Perplexity = 19.87 +Step 20260/44000 | Loss: 3.0457 | LR: 2.88e-04 | 949.4ms/step | 21.1 samples/s +Step 20270/44000 | Loss: 2.9429 | LR: 2.88e-04 | 737.7ms/step | 27.1 samples/s +Step 20280/44000 | Loss: 3.0280 | LR: 2.88e-04 | 725.6ms/step | 27.6 samples/s +Step 20290/44000 | Loss: 2.9453 | LR: 2.88e-04 | 726.3ms/step | 27.5 samples/s +Step 20300/44000 | Loss: 2.9204 | LR: 2.88e-04 | 725.4ms/step | 27.6 samples/s +Step 20310/44000 | Loss: 3.0118 | LR: 2.87e-04 | 725.8ms/step | 27.6 samples/s +Step 20320/44000 | Loss: 3.0472 | LR: 2.87e-04 | 726.2ms/step | 27.5 samples/s +Step 20330/44000 | Loss: 3.0480 | LR: 2.87e-04 | 726.8ms/step | 27.5 samples/s +Step 20340/44000 | Loss: 2.9748 | LR: 2.87e-04 | 726.0ms/step | 27.5 samples/s +Step 20350/44000 | Loss: 3.0232 | LR: 2.87e-04 | 730.8ms/step | 27.4 samples/s +Step 20360/44000 | Loss: 3.0032 | LR: 2.86e-04 | 730.4ms/step | 27.4 samples/s +Step 20370/44000 | Loss: 3.0259 | LR: 2.86e-04 | 725.5ms/step | 27.6 samples/s +Step 20380/44000 | Loss: 2.9526 | LR: 2.86e-04 | 726.1ms/step | 27.5 samples/s +Step 20390/44000 | Loss: 2.9480 | LR: 2.86e-04 | 726.1ms/step | 27.5 samples/s +Step 20400/44000 | Loss: 3.0173 | LR: 2.86e-04 | 725.8ms/step | 27.6 samples/s +Step 20410/44000 | Loss: 2.9650 | LR: 2.86e-04 | 725.9ms/step | 27.6 samples/s +Step 20420/44000 | Loss: 3.0161 | LR: 2.85e-04 | 725.6ms/step | 27.6 samples/s +Step 20430/44000 | Loss: 2.9228 | LR: 2.85e-04 | 725.7ms/step | 27.6 samples/s +Step 20440/44000 | Loss: 3.0080 | LR: 2.85e-04 | 730.8ms/step | 27.4 samples/s +Step 20450/44000 | Loss: 3.0582 | LR: 2.85e-04 | 730.4ms/step | 27.4 samples/s +Step 20460/44000 | Loss: 3.0284 | LR: 2.85e-04 | 725.9ms/step | 27.6 samples/s +Step 20470/44000 | Loss: 3.0395 | LR: 2.84e-04 | 725.4ms/step | 27.6 samples/s +Step 20480/44000 | Loss: 2.9763 | LR: 2.84e-04 | 725.9ms/step | 27.6 samples/s +Step 20490/44000 | Loss: 2.9934 | LR: 2.84e-04 | 725.1ms/step | 27.6 samples/s +Step 20500/44000 | Loss: 3.0086 | LR: 2.84e-04 | 726.1ms/step | 27.5 samples/s +Validation at step 20500: Loss = 2.9815, Perplexity = 19.72 +Step 20510/44000 | Loss: 2.9980 | LR: 2.84e-04 | 951.2ms/step | 21.0 samples/s +Step 20520/44000 | Loss: 3.0202 | LR: 2.84e-04 | 725.3ms/step | 27.6 samples/s +Step 20530/44000 | Loss: 2.9878 | LR: 2.83e-04 | 736.2ms/step | 27.2 samples/s +Step 20540/44000 | Loss: 2.9719 | LR: 2.83e-04 | 725.0ms/step | 27.6 samples/s +Step 20550/44000 | Loss: 3.0249 | LR: 2.83e-04 | 724.8ms/step | 27.6 samples/s +Step 20560/44000 | Loss: 3.0208 | LR: 2.83e-04 | 726.3ms/step | 27.5 samples/s +Step 20570/44000 | Loss: 3.0725 | LR: 2.83e-04 | 725.5ms/step | 27.6 samples/s +Step 20580/44000 | Loss: 2.9727 | LR: 2.83e-04 | 725.2ms/step | 27.6 samples/s +Step 20590/44000 | Loss: 2.9764 | LR: 2.82e-04 | 724.9ms/step | 27.6 samples/s +Step 20600/44000 | Loss: 3.0556 | LR: 2.82e-04 | 724.7ms/step | 27.6 samples/s +Step 20610/44000 | Loss: 2.9846 | LR: 2.82e-04 | 729.6ms/step | 27.4 samples/s +Step 20620/44000 | Loss: 2.9856 | LR: 2.82e-04 | 729.6ms/step | 27.4 samples/s +Step 20630/44000 | Loss: 2.9540 | LR: 2.82e-04 | 724.9ms/step | 27.6 samples/s +Step 20640/44000 | Loss: 3.0112 | LR: 2.81e-04 | 725.0ms/step | 27.6 samples/s +Step 20650/44000 | Loss: 2.9657 | LR: 2.81e-04 | 725.3ms/step | 27.6 samples/s +Step 20660/44000 | Loss: 2.9583 | LR: 2.81e-04 | 725.5ms/step | 27.6 samples/s +Step 20670/44000 | Loss: 3.0034 | LR: 2.81e-04 | 725.0ms/step | 27.6 samples/s +Step 20680/44000 | Loss: 3.0232 | LR: 2.81e-04 | 725.3ms/step | 27.6 samples/s +Step 20690/44000 | Loss: 3.0285 | LR: 2.81e-04 | 725.4ms/step | 27.6 samples/s +Step 20700/44000 | Loss: 3.0451 | LR: 2.80e-04 | 734.9ms/step | 27.2 samples/s +Step 20710/44000 | Loss: 2.9648 | LR: 2.80e-04 | 725.5ms/step | 27.6 samples/s +Step 20720/44000 | Loss: 3.0498 | LR: 2.80e-04 | 725.7ms/step | 27.6 samples/s +Step 20730/44000 | Loss: 2.9460 | LR: 2.80e-04 | 725.7ms/step | 27.6 samples/s +Step 20740/44000 | Loss: 2.9148 | LR: 2.80e-04 | 725.4ms/step | 27.6 samples/s +Step 20750/44000 | Loss: 2.9874 | LR: 2.79e-04 | 725.2ms/step | 27.6 samples/s +Validation at step 20750: Loss = 2.9797, Perplexity = 19.68 +Step 20760/44000 | Loss: 3.0321 | LR: 2.79e-04 | 948.8ms/step | 21.1 samples/s +Step 20770/44000 | Loss: 2.9706 | LR: 2.79e-04 | 724.9ms/step | 27.6 samples/s +Step 20780/44000 | Loss: 2.9909 | LR: 2.79e-04 | 728.7ms/step | 27.4 samples/s +Step 20790/44000 | Loss: 2.9795 | LR: 2.79e-04 | 736.2ms/step | 27.2 samples/s +Step 20800/44000 | Loss: 2.9939 | LR: 2.79e-04 | 725.4ms/step | 27.6 samples/s +Step 20810/44000 | Loss: 2.9982 | LR: 2.78e-04 | 725.8ms/step | 27.6 samples/s +Step 20820/44000 | Loss: 2.9323 | LR: 2.78e-04 | 726.1ms/step | 27.5 samples/s +Step 20830/44000 | Loss: 2.9681 | LR: 2.78e-04 | 726.2ms/step | 27.5 samples/s +Step 20840/44000 | Loss: 3.0652 | LR: 2.78e-04 | 725.9ms/step | 27.6 samples/s +Step 20850/44000 | Loss: 2.9943 | LR: 2.78e-04 | 726.3ms/step | 27.5 samples/s +Step 20860/44000 | Loss: 2.9444 | LR: 2.77e-04 | 726.4ms/step | 27.5 samples/s +Step 20870/44000 | Loss: 2.9432 | LR: 2.77e-04 | 730.2ms/step | 27.4 samples/s +Step 20880/44000 | Loss: 3.0809 | LR: 2.77e-04 | 730.7ms/step | 27.4 samples/s +Step 20890/44000 | Loss: 3.0174 | LR: 2.77e-04 | 726.0ms/step | 27.5 samples/s +Step 20900/44000 | Loss: 3.0032 | LR: 2.77e-04 | 726.6ms/step | 27.5 samples/s +Step 20910/44000 | Loss: 2.9639 | LR: 2.77e-04 | 726.0ms/step | 27.5 samples/s +Step 20920/44000 | Loss: 3.0187 | LR: 2.76e-04 | 726.3ms/step | 27.5 samples/s +Step 20930/44000 | Loss: 2.9731 | LR: 2.76e-04 | 725.7ms/step | 27.6 samples/s +Step 20940/44000 | Loss: 2.9562 | LR: 2.76e-04 | 726.0ms/step | 27.5 samples/s +Step 20950/44000 | Loss: 3.0138 | LR: 2.76e-04 | 725.5ms/step | 27.6 samples/s +Step 20960/44000 | Loss: 2.9745 | LR: 2.76e-04 | 740.6ms/step | 27.0 samples/s +Step 20970/44000 | Loss: 2.9870 | LR: 2.75e-04 | 726.3ms/step | 27.5 samples/s +Step 20980/44000 | Loss: 2.9821 | LR: 2.75e-04 | 727.1ms/step | 27.5 samples/s +Step 20990/44000 | Loss: 3.0122 | LR: 2.75e-04 | 725.6ms/step | 27.6 samples/s +Step 21000/44000 | Loss: 2.9682 | LR: 2.75e-04 | 725.3ms/step | 27.6 samples/s +Validation at step 21000: Loss = 2.9775, Perplexity = 19.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-21000 +Step 21010/44000 | Loss: 3.0021 | LR: 2.75e-04 | 1377.2ms/step | 14.5 samples/s +Step 21020/44000 | Loss: 2.9512 | LR: 2.75e-04 | 724.6ms/step | 27.6 samples/s +Step 21030/44000 | Loss: 2.9942 | LR: 2.74e-04 | 725.8ms/step | 27.6 samples/s +Step 21040/44000 | Loss: 2.9251 | LR: 2.74e-04 | 735.9ms/step | 27.2 samples/s +Step 21050/44000 | Loss: 2.9931 | LR: 2.74e-04 | 726.4ms/step | 27.5 samples/s +Step 21060/44000 | Loss: 3.0120 | LR: 2.74e-04 | 725.7ms/step | 27.6 samples/s +Step 21070/44000 | Loss: 2.9753 | LR: 2.74e-04 | 726.6ms/step | 27.5 samples/s +Step 21080/44000 | Loss: 3.0127 | LR: 2.73e-04 | 727.0ms/step | 27.5 samples/s +Step 21090/44000 | Loss: 2.9247 | LR: 2.73e-04 | 726.2ms/step | 27.5 samples/s +Step 21100/44000 | Loss: 3.0656 | LR: 2.73e-04 | 725.5ms/step | 27.6 samples/s +Step 21110/44000 | Loss: 2.9862 | LR: 2.73e-04 | 725.7ms/step | 27.6 samples/s +Step 21120/44000 | Loss: 3.0165 | LR: 2.73e-04 | 725.7ms/step | 27.6 samples/s +Step 21130/44000 | Loss: 2.9388 | LR: 2.73e-04 | 735.9ms/step | 27.2 samples/s +Step 21140/44000 | Loss: 2.9810 | LR: 2.72e-04 | 726.0ms/step | 27.5 samples/s +Step 21150/44000 | Loss: 2.9599 | LR: 2.72e-04 | 726.3ms/step | 27.5 samples/s +Step 21160/44000 | Loss: 2.9454 | LR: 2.72e-04 | 726.3ms/step | 27.5 samples/s +Step 21170/44000 | Loss: 3.0101 | LR: 2.72e-04 | 725.6ms/step | 27.6 samples/s +Step 21180/44000 | Loss: 2.9537 | LR: 2.72e-04 | 726.2ms/step | 27.5 samples/s +Step 21190/44000 | Loss: 2.9869 | LR: 2.71e-04 | 725.5ms/step | 27.6 samples/s +Step 21200/44000 | Loss: 2.9625 | LR: 2.71e-04 | 725.2ms/step | 27.6 samples/s +Step 21210/44000 | Loss: 2.9701 | LR: 2.71e-04 | 732.6ms/step | 27.3 samples/s +Step 21220/44000 | Loss: 2.9143 | LR: 2.71e-04 | 732.6ms/step | 27.3 samples/s +Step 21230/44000 | Loss: 2.9837 | LR: 2.71e-04 | 725.9ms/step | 27.6 samples/s +Step 21240/44000 | Loss: 2.9818 | LR: 2.71e-04 | 725.8ms/step | 27.6 samples/s +Step 21250/44000 | Loss: 3.0223 | LR: 2.70e-04 | 726.1ms/step | 27.5 samples/s +Validation at step 21250: Loss = 2.9763, Perplexity = 19.62 +Step 21260/44000 | Loss: 2.9220 | LR: 2.70e-04 | 950.1ms/step | 21.1 samples/s +Step 21270/44000 | Loss: 3.0284 | LR: 2.70e-04 | 725.6ms/step | 27.6 samples/s +Step 21280/44000 | Loss: 2.9951 | LR: 2.70e-04 | 725.3ms/step | 27.6 samples/s +Step 21290/44000 | Loss: 3.0444 | LR: 2.70e-04 | 724.9ms/step | 27.6 samples/s +Step 21300/44000 | Loss: 2.9783 | LR: 2.70e-04 | 735.6ms/step | 27.2 samples/s +Step 21310/44000 | Loss: 2.9239 | LR: 2.69e-04 | 726.1ms/step | 27.5 samples/s +Step 21320/44000 | Loss: 2.9279 | LR: 2.69e-04 | 726.4ms/step | 27.5 samples/s +Step 21330/44000 | Loss: 2.9341 | LR: 2.69e-04 | 726.5ms/step | 27.5 samples/s +Step 21340/44000 | Loss: 2.9990 | LR: 2.69e-04 | 726.6ms/step | 27.5 samples/s +Step 21350/44000 | Loss: 3.0242 | LR: 2.69e-04 | 726.6ms/step | 27.5 samples/s +Step 21360/44000 | Loss: 2.9612 | LR: 2.68e-04 | 726.2ms/step | 27.5 samples/s +Step 21370/44000 | Loss: 2.9770 | LR: 2.68e-04 | 726.3ms/step | 27.5 samples/s +Step 21380/44000 | Loss: 2.9704 | LR: 2.68e-04 | 732.4ms/step | 27.3 samples/s +Step 21390/44000 | Loss: 2.9507 | LR: 2.68e-04 | 732.0ms/step | 27.3 samples/s +Step 21400/44000 | Loss: 3.0012 | LR: 2.68e-04 | 726.0ms/step | 27.5 samples/s +Step 21410/44000 | Loss: 2.9943 | LR: 2.68e-04 | 726.2ms/step | 27.5 samples/s +Step 21420/44000 | Loss: 2.9106 | LR: 2.67e-04 | 726.4ms/step | 27.5 samples/s +Step 21430/44000 | Loss: 2.9282 | LR: 2.67e-04 | 726.1ms/step | 27.5 samples/s +Step 21440/44000 | Loss: 3.0200 | LR: 2.67e-04 | 725.8ms/step | 27.6 samples/s +Step 21450/44000 | Loss: 2.9271 | LR: 2.67e-04 | 725.6ms/step | 27.6 samples/s +Step 21460/44000 | Loss: 2.8809 | LR: 2.67e-04 | 725.8ms/step | 27.6 samples/s +Step 21470/44000 | Loss: 3.0643 | LR: 2.66e-04 | 730.0ms/step | 27.4 samples/s +Step 21480/44000 | Loss: 3.0332 | LR: 2.66e-04 | 730.9ms/step | 27.4 samples/s +Step 21490/44000 | Loss: 2.9590 | LR: 2.66e-04 | 726.6ms/step | 27.5 samples/s +Step 21500/44000 | Loss: 2.9669 | LR: 2.66e-04 | 725.8ms/step | 27.6 samples/s +Validation at step 21500: Loss = 2.9749, Perplexity = 19.59 +Step 21510/44000 | Loss: 2.9667 | LR: 2.66e-04 | 949.1ms/step | 21.1 samples/s +Step 21520/44000 | Loss: 2.9965 | LR: 2.66e-04 | 725.4ms/step | 27.6 samples/s +Step 21530/44000 | Loss: 2.9921 | LR: 2.65e-04 | 726.1ms/step | 27.5 samples/s +Step 21540/44000 | Loss: 2.9391 | LR: 2.65e-04 | 726.0ms/step | 27.5 samples/s +Step 21550/44000 | Loss: 2.9886 | LR: 2.65e-04 | 726.3ms/step | 27.5 samples/s +Step 21560/44000 | Loss: 2.9592 | LR: 2.65e-04 | 737.3ms/step | 27.1 samples/s +Step 21570/44000 | Loss: 2.9778 | LR: 2.65e-04 | 726.7ms/step | 27.5 samples/s +Step 21580/44000 | Loss: 2.9741 | LR: 2.64e-04 | 726.2ms/step | 27.5 samples/s +Step 21590/44000 | Loss: 2.9637 | LR: 2.64e-04 | 726.1ms/step | 27.5 samples/s +Step 21600/44000 | Loss: 2.9517 | LR: 2.64e-04 | 725.9ms/step | 27.6 samples/s +Step 21610/44000 | Loss: 3.0572 | LR: 2.64e-04 | 725.2ms/step | 27.6 samples/s +Step 21620/44000 | Loss: 2.9308 | LR: 2.64e-04 | 725.4ms/step | 27.6 samples/s +Step 21630/44000 | Loss: 2.9279 | LR: 2.64e-04 | 725.4ms/step | 27.6 samples/s +Step 21640/44000 | Loss: 3.0085 | LR: 2.63e-04 | 730.4ms/step | 27.4 samples/s +Step 21650/44000 | Loss: 2.9695 | LR: 2.63e-04 | 730.3ms/step | 27.4 samples/s +Step 21660/44000 | Loss: 2.9904 | LR: 2.63e-04 | 726.3ms/step | 27.5 samples/s +Step 21670/44000 | Loss: 3.0085 | LR: 2.63e-04 | 725.8ms/step | 27.6 samples/s +Step 21680/44000 | Loss: 2.9082 | LR: 2.63e-04 | 725.6ms/step | 27.6 samples/s +Step 21690/44000 | Loss: 2.9812 | LR: 2.62e-04 | 725.5ms/step | 27.6 samples/s +Step 21700/44000 | Loss: 3.0312 | LR: 2.62e-04 | 726.4ms/step | 27.5 samples/s +Step 21710/44000 | Loss: 3.0442 | LR: 2.62e-04 | 725.6ms/step | 27.6 samples/s +Step 21720/44000 | Loss: 2.9743 | LR: 2.62e-04 | 725.8ms/step | 27.6 samples/s +Step 21730/44000 | Loss: 2.9354 | LR: 2.62e-04 | 736.8ms/step | 27.1 samples/s +Step 21740/44000 | Loss: 3.0261 | LR: 2.62e-04 | 726.0ms/step | 27.6 samples/s +Step 21750/44000 | Loss: 3.0173 | LR: 2.61e-04 | 725.3ms/step | 27.6 samples/s +Validation at step 21750: Loss = 2.9707, Perplexity = 19.51 +Step 21760/44000 | Loss: 3.0579 | LR: 2.61e-04 | 948.6ms/step | 21.1 samples/s +Step 21770/44000 | Loss: 2.9855 | LR: 2.61e-04 | 725.8ms/step | 27.6 samples/s +Step 21780/44000 | Loss: 2.9820 | LR: 2.61e-04 | 725.5ms/step | 27.6 samples/s +Step 21790/44000 | Loss: 2.9842 | LR: 2.61e-04 | 725.4ms/step | 27.6 samples/s +Step 21800/44000 | Loss: 2.9677 | LR: 2.60e-04 | 725.7ms/step | 27.6 samples/s +Step 21810/44000 | Loss: 3.0006 | LR: 2.60e-04 | 730.5ms/step | 27.4 samples/s +Step 21820/44000 | Loss: 2.9448 | LR: 2.60e-04 | 730.9ms/step | 27.4 samples/s +Step 21830/44000 | Loss: 2.9683 | LR: 2.60e-04 | 725.4ms/step | 27.6 samples/s +Step 21840/44000 | Loss: 2.9185 | LR: 2.60e-04 | 726.3ms/step | 27.5 samples/s +Step 21850/44000 | Loss: 2.9089 | LR: 2.60e-04 | 725.6ms/step | 27.6 samples/s +Step 21860/44000 | Loss: 2.9602 | LR: 2.59e-04 | 725.3ms/step | 27.6 samples/s +Step 21870/44000 | Loss: 2.9661 | LR: 2.59e-04 | 725.2ms/step | 27.6 samples/s +Step 21880/44000 | Loss: 2.9782 | LR: 2.59e-04 | 725.8ms/step | 27.6 samples/s +Step 21890/44000 | Loss: 2.9822 | LR: 2.59e-04 | 725.9ms/step | 27.6 samples/s +Step 21900/44000 | Loss: 2.9602 | LR: 2.59e-04 | 730.2ms/step | 27.4 samples/s +Step 21910/44000 | Loss: 2.9442 | LR: 2.58e-04 | 731.0ms/step | 27.4 samples/s +Step 21920/44000 | Loss: 3.0695 | LR: 2.58e-04 | 725.3ms/step | 27.6 samples/s +Step 21930/44000 | Loss: 2.8845 | LR: 2.58e-04 | 725.6ms/step | 27.6 samples/s +Step 21940/44000 | Loss: 2.9942 | LR: 2.58e-04 | 725.9ms/step | 27.6 samples/s +Step 21950/44000 | Loss: 2.9709 | LR: 2.58e-04 | 725.6ms/step | 27.6 samples/s +Step 21960/44000 | Loss: 2.9899 | LR: 2.58e-04 | 726.4ms/step | 27.5 samples/s +Step 21970/44000 | Loss: 3.0446 | LR: 2.57e-04 | 725.8ms/step | 27.6 samples/s +Step 21980/44000 | Loss: 2.9798 | LR: 2.57e-04 | 725.8ms/step | 27.6 samples/s +Step 21990/44000 | Loss: 2.9370 | LR: 2.57e-04 | 735.2ms/step | 27.2 samples/s +Step 22000/44000 | Loss: 2.9701 | LR: 2.57e-04 | 725.6ms/step | 27.6 samples/s +Validation at step 22000: Loss = 2.9685, Perplexity = 19.46 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-22000 +Step 22010/44000 | Loss: 2.9894 | LR: 2.57e-04 | 1372.0ms/step | 14.6 samples/s +Step 22020/44000 | Loss: 2.9758 | LR: 2.56e-04 | 724.5ms/step | 27.6 samples/s +Step 22030/44000 | Loss: 2.9837 | LR: 2.56e-04 | 725.1ms/step | 27.6 samples/s +Step 22040/44000 | Loss: 2.9830 | LR: 2.56e-04 | 725.6ms/step | 27.6 samples/s +Step 22050/44000 | Loss: 2.9966 | LR: 2.56e-04 | 725.4ms/step | 27.6 samples/s +Step 22060/44000 | Loss: 3.0478 | LR: 2.56e-04 | 726.7ms/step | 27.5 samples/s +Step 22070/44000 | Loss: 2.9501 | LR: 2.56e-04 | 735.3ms/step | 27.2 samples/s +Step 22080/44000 | Loss: 2.9554 | LR: 2.55e-04 | 726.4ms/step | 27.5 samples/s +Step 22090/44000 | Loss: 3.0027 | LR: 2.55e-04 | 726.2ms/step | 27.5 samples/s +Step 22100/44000 | Loss: 2.9903 | LR: 2.55e-04 | 726.0ms/step | 27.5 samples/s +Step 22110/44000 | Loss: 2.9633 | LR: 2.55e-04 | 725.8ms/step | 27.6 samples/s +Step 22120/44000 | Loss: 2.9957 | LR: 2.55e-04 | 725.9ms/step | 27.6 samples/s +Step 22130/44000 | Loss: 2.9767 | LR: 2.54e-04 | 725.8ms/step | 27.6 samples/s +Step 22140/44000 | Loss: 2.9577 | LR: 2.54e-04 | 726.0ms/step | 27.5 samples/s +Step 22150/44000 | Loss: 2.9523 | LR: 2.54e-04 | 725.6ms/step | 27.6 samples/s +Step 22160/44000 | Loss: 2.9773 | LR: 2.54e-04 | 740.1ms/step | 27.0 samples/s +Step 22170/44000 | Loss: 2.9895 | LR: 2.54e-04 | 725.8ms/step | 27.6 samples/s +Step 22180/44000 | Loss: 2.9655 | LR: 2.54e-04 | 725.9ms/step | 27.6 samples/s +Step 22190/44000 | Loss: 2.9651 | LR: 2.53e-04 | 726.1ms/step | 27.5 samples/s +Step 22200/44000 | Loss: 2.9707 | LR: 2.53e-04 | 725.7ms/step | 27.6 samples/s +Step 22210/44000 | Loss: 2.9827 | LR: 2.53e-04 | 725.8ms/step | 27.6 samples/s +Step 22220/44000 | Loss: 2.9405 | LR: 2.53e-04 | 726.2ms/step | 27.5 samples/s +Step 22230/44000 | Loss: 2.9856 | LR: 2.53e-04 | 725.9ms/step | 27.6 samples/s +Step 22240/44000 | Loss: 2.9808 | LR: 2.52e-04 | 730.1ms/step | 27.4 samples/s +Step 22250/44000 | Loss: 2.9773 | LR: 2.52e-04 | 730.0ms/step | 27.4 samples/s +Validation at step 22250: Loss = 2.9657, Perplexity = 19.41 +Step 22260/44000 | Loss: 3.0696 | LR: 2.52e-04 | 949.9ms/step | 21.1 samples/s +Step 22270/44000 | Loss: 3.0111 | LR: 2.52e-04 | 726.3ms/step | 27.5 samples/s +Step 22280/44000 | Loss: 2.9942 | LR: 2.52e-04 | 725.7ms/step | 27.6 samples/s +Step 22290/44000 | Loss: 2.9434 | LR: 2.52e-04 | 725.7ms/step | 27.6 samples/s +Step 22300/44000 | Loss: 2.9874 | LR: 2.51e-04 | 725.7ms/step | 27.6 samples/s +Step 22310/44000 | Loss: 2.9958 | LR: 2.51e-04 | 725.9ms/step | 27.6 samples/s +Step 22320/44000 | Loss: 2.9436 | LR: 2.51e-04 | 725.8ms/step | 27.6 samples/s +Step 22330/44000 | Loss: 3.0041 | LR: 2.51e-04 | 739.2ms/step | 27.1 samples/s +Step 22340/44000 | Loss: 3.0043 | LR: 2.51e-04 | 725.1ms/step | 27.6 samples/s +Step 22350/44000 | Loss: 2.9553 | LR: 2.50e-04 | 725.5ms/step | 27.6 samples/s +Step 22360/44000 | Loss: 2.9588 | LR: 2.50e-04 | 725.8ms/step | 27.6 samples/s +Step 22370/44000 | Loss: 2.9944 | LR: 2.50e-04 | 726.4ms/step | 27.5 samples/s +Step 22380/44000 | Loss: 2.8922 | LR: 2.50e-04 | 727.2ms/step | 27.5 samples/s +Step 22390/44000 | Loss: 2.9612 | LR: 2.50e-04 | 726.7ms/step | 27.5 samples/s +Step 22400/44000 | Loss: 2.9879 | LR: 2.50e-04 | 725.2ms/step | 27.6 samples/s +Step 22410/44000 | Loss: 2.9535 | LR: 2.49e-04 | 730.7ms/step | 27.4 samples/s +Step 22420/44000 | Loss: 2.9655 | LR: 2.49e-04 | 731.0ms/step | 27.4 samples/s +Step 22430/44000 | Loss: 2.9646 | LR: 2.49e-04 | 725.4ms/step | 27.6 samples/s +Step 22440/44000 | Loss: 2.9886 | LR: 2.49e-04 | 726.0ms/step | 27.5 samples/s +Step 22450/44000 | Loss: 2.9346 | LR: 2.49e-04 | 726.0ms/step | 27.5 samples/s +Step 22460/44000 | Loss: 2.9449 | LR: 2.48e-04 | 725.8ms/step | 27.6 samples/s +Step 22470/44000 | Loss: 2.9428 | LR: 2.48e-04 | 726.2ms/step | 27.5 samples/s +Step 22480/44000 | Loss: 2.9593 | LR: 2.48e-04 | 725.8ms/step | 27.6 samples/s +Step 22490/44000 | Loss: 2.9975 | LR: 2.48e-04 | 725.2ms/step | 27.6 samples/s +Step 22500/44000 | Loss: 2.9794 | LR: 2.48e-04 | 731.9ms/step | 27.3 samples/s +Validation at step 22500: Loss = 2.9638, Perplexity = 19.37 +Step 22510/44000 | Loss: 2.9350 | LR: 2.48e-04 | 950.7ms/step | 21.0 samples/s +Step 22520/44000 | Loss: 2.9630 | LR: 2.47e-04 | 725.9ms/step | 27.6 samples/s +Step 22530/44000 | Loss: 2.9661 | LR: 2.47e-04 | 725.9ms/step | 27.6 samples/s +Step 22540/44000 | Loss: 2.9875 | LR: 2.47e-04 | 725.3ms/step | 27.6 samples/s +Step 22550/44000 | Loss: 2.9708 | LR: 2.47e-04 | 726.0ms/step | 27.5 samples/s +Step 22560/44000 | Loss: 2.9179 | LR: 2.47e-04 | 725.7ms/step | 27.6 samples/s +Step 22570/44000 | Loss: 2.9675 | LR: 2.46e-04 | 725.3ms/step | 27.6 samples/s +Step 22580/44000 | Loss: 2.9394 | LR: 2.46e-04 | 726.0ms/step | 27.5 samples/s +Step 22590/44000 | Loss: 2.9200 | LR: 2.46e-04 | 736.9ms/step | 27.1 samples/s +Step 22600/44000 | Loss: 2.9471 | LR: 2.46e-04 | 725.9ms/step | 27.6 samples/s +Step 22610/44000 | Loss: 2.9973 | LR: 2.46e-04 | 725.8ms/step | 27.6 samples/s +Step 22620/44000 | Loss: 2.9826 | LR: 2.46e-04 | 726.2ms/step | 27.5 samples/s +Step 22630/44000 | Loss: 3.0204 | LR: 2.45e-04 | 725.7ms/step | 27.6 samples/s +Step 22640/44000 | Loss: 2.9573 | LR: 2.45e-04 | 725.2ms/step | 27.6 samples/s +Step 22650/44000 | Loss: 2.9582 | LR: 2.45e-04 | 725.5ms/step | 27.6 samples/s +Step 22660/44000 | Loss: 2.9835 | LR: 2.45e-04 | 725.4ms/step | 27.6 samples/s +Step 22670/44000 | Loss: 2.9913 | LR: 2.45e-04 | 729.9ms/step | 27.4 samples/s +Step 22680/44000 | Loss: 2.9480 | LR: 2.44e-04 | 730.0ms/step | 27.4 samples/s +Step 22690/44000 | Loss: 2.9688 | LR: 2.44e-04 | 726.1ms/step | 27.5 samples/s +Step 22700/44000 | Loss: 2.9783 | LR: 2.44e-04 | 726.0ms/step | 27.5 samples/s +Step 22710/44000 | Loss: 3.0154 | LR: 2.44e-04 | 725.8ms/step | 27.6 samples/s +Step 22720/44000 | Loss: 3.0173 | LR: 2.44e-04 | 726.3ms/step | 27.5 samples/s +Step 22730/44000 | Loss: 2.9709 | LR: 2.44e-04 | 726.3ms/step | 27.5 samples/s +Step 22740/44000 | Loss: 2.9785 | LR: 2.43e-04 | 725.9ms/step | 27.6 samples/s +Step 22750/44000 | Loss: 2.9539 | LR: 2.43e-04 | 725.6ms/step | 27.6 samples/s +Validation at step 22750: Loss = 2.9611, Perplexity = 19.32 +Step 22760/44000 | Loss: 2.9782 | LR: 2.43e-04 | 964.5ms/step | 20.7 samples/s +Step 22770/44000 | Loss: 2.9935 | LR: 2.43e-04 | 726.9ms/step | 27.5 samples/s +Step 22780/44000 | Loss: 2.9588 | LR: 2.43e-04 | 726.2ms/step | 27.5 samples/s +Step 22790/44000 | Loss: 2.9655 | LR: 2.42e-04 | 725.5ms/step | 27.6 samples/s +Step 22800/44000 | Loss: 2.9469 | LR: 2.42e-04 | 725.3ms/step | 27.6 samples/s +Step 22810/44000 | Loss: 2.9309 | LR: 2.42e-04 | 725.9ms/step | 27.6 samples/s +Step 22820/44000 | Loss: 3.0011 | LR: 2.42e-04 | 726.8ms/step | 27.5 samples/s +Step 22830/44000 | Loss: 2.9911 | LR: 2.42e-04 | 725.9ms/step | 27.6 samples/s +Step 22840/44000 | Loss: 2.9597 | LR: 2.42e-04 | 730.6ms/step | 27.4 samples/s +Step 22850/44000 | Loss: 2.9797 | LR: 2.41e-04 | 729.9ms/step | 27.4 samples/s +Step 22860/44000 | Loss: 2.9852 | LR: 2.41e-04 | 725.4ms/step | 27.6 samples/s +Step 22870/44000 | Loss: 2.9000 | LR: 2.41e-04 | 725.6ms/step | 27.6 samples/s +Step 22880/44000 | Loss: 2.9281 | LR: 2.41e-04 | 725.2ms/step | 27.6 samples/s +Step 22890/44000 | Loss: 2.9912 | LR: 2.41e-04 | 725.3ms/step | 27.6 samples/s +Step 22900/44000 | Loss: 3.0111 | LR: 2.40e-04 | 725.9ms/step | 27.6 samples/s +Step 22910/44000 | Loss: 2.9853 | LR: 2.40e-04 | 725.7ms/step | 27.6 samples/s +Step 22920/44000 | Loss: 2.9609 | LR: 2.40e-04 | 725.9ms/step | 27.6 samples/s +Step 22930/44000 | Loss: 2.9453 | LR: 2.40e-04 | 730.5ms/step | 27.4 samples/s +Step 22940/44000 | Loss: 2.9716 | LR: 2.40e-04 | 731.1ms/step | 27.4 samples/s +Step 22950/44000 | Loss: 3.0086 | LR: 2.40e-04 | 726.2ms/step | 27.5 samples/s +Step 22960/44000 | Loss: 3.0199 | LR: 2.39e-04 | 725.3ms/step | 27.6 samples/s +Step 22970/44000 | Loss: 2.9898 | LR: 2.39e-04 | 725.4ms/step | 27.6 samples/s +Step 22980/44000 | Loss: 2.9441 | LR: 2.39e-04 | 725.2ms/step | 27.6 samples/s +Step 22990/44000 | Loss: 3.0161 | LR: 2.39e-04 | 725.8ms/step | 27.6 samples/s +Step 23000/44000 | Loss: 2.9720 | LR: 2.39e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 23000: Loss = 2.9603, Perplexity = 19.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-23000 +Step 23010/44000 | Loss: 2.9065 | LR: 2.38e-04 | 1430.5ms/step | 14.0 samples/s +Step 23020/44000 | Loss: 3.0165 | LR: 2.38e-04 | 725.2ms/step | 27.6 samples/s +Step 23030/44000 | Loss: 2.9332 | LR: 2.38e-04 | 725.2ms/step | 27.6 samples/s +Step 23040/44000 | Loss: 2.9655 | LR: 2.38e-04 | 725.3ms/step | 27.6 samples/s +Step 23050/44000 | Loss: 2.9791 | LR: 2.38e-04 | 725.6ms/step | 27.6 samples/s +Step 23060/44000 | Loss: 2.9613 | LR: 2.38e-04 | 725.6ms/step | 27.6 samples/s +Step 23070/44000 | Loss: 2.9392 | LR: 2.37e-04 | 725.4ms/step | 27.6 samples/s +Step 23080/44000 | Loss: 2.9969 | LR: 2.37e-04 | 725.6ms/step | 27.6 samples/s +Step 23090/44000 | Loss: 2.9789 | LR: 2.37e-04 | 725.5ms/step | 27.6 samples/s +Step 23100/44000 | Loss: 2.9877 | LR: 2.37e-04 | 737.2ms/step | 27.1 samples/s +Step 23110/44000 | Loss: 2.9621 | LR: 2.37e-04 | 725.5ms/step | 27.6 samples/s +Step 23120/44000 | Loss: 2.8488 | LR: 2.36e-04 | 725.5ms/step | 27.6 samples/s +Step 23130/44000 | Loss: 3.0168 | LR: 2.36e-04 | 725.4ms/step | 27.6 samples/s +Step 23140/44000 | Loss: 2.9978 | LR: 2.36e-04 | 725.9ms/step | 27.6 samples/s +Step 23150/44000 | Loss: 2.9056 | LR: 2.36e-04 | 725.4ms/step | 27.6 samples/s +Step 23160/44000 | Loss: 2.9847 | LR: 2.36e-04 | 725.8ms/step | 27.6 samples/s +Step 23170/44000 | Loss: 2.9676 | LR: 2.36e-04 | 725.5ms/step | 27.6 samples/s +Step 23180/44000 | Loss: 3.0097 | LR: 2.35e-04 | 730.5ms/step | 27.4 samples/s +Step 23190/44000 | Loss: 2.9415 | LR: 2.35e-04 | 730.6ms/step | 27.4 samples/s +Step 23200/44000 | Loss: 2.9540 | LR: 2.35e-04 | 726.2ms/step | 27.5 samples/s +Step 23210/44000 | Loss: 2.9444 | LR: 2.35e-04 | 726.2ms/step | 27.5 samples/s +Step 23220/44000 | Loss: 2.8898 | LR: 2.35e-04 | 725.7ms/step | 27.6 samples/s +Step 23230/44000 | Loss: 2.9433 | LR: 2.34e-04 | 726.3ms/step | 27.5 samples/s +Step 23240/44000 | Loss: 2.9597 | LR: 2.34e-04 | 725.9ms/step | 27.6 samples/s +Step 23250/44000 | Loss: 2.9761 | LR: 2.34e-04 | 725.5ms/step | 27.6 samples/s +Validation at step 23250: Loss = 2.9564, Perplexity = 19.23 +Step 23260/44000 | Loss: 3.0108 | LR: 2.34e-04 | 949.1ms/step | 21.1 samples/s +Step 23270/44000 | Loss: 2.9071 | LR: 2.34e-04 | 739.7ms/step | 27.0 samples/s +Step 23280/44000 | Loss: 2.9750 | LR: 2.34e-04 | 725.3ms/step | 27.6 samples/s +Step 23290/44000 | Loss: 2.9420 | LR: 2.33e-04 | 725.3ms/step | 27.6 samples/s +Step 23300/44000 | Loss: 2.9829 | LR: 2.33e-04 | 726.0ms/step | 27.5 samples/s +Step 23310/44000 | Loss: 2.9439 | LR: 2.33e-04 | 725.7ms/step | 27.6 samples/s +Step 23320/44000 | Loss: 3.0039 | LR: 2.33e-04 | 725.5ms/step | 27.6 samples/s +Step 23330/44000 | Loss: 2.9068 | LR: 2.33e-04 | 725.7ms/step | 27.6 samples/s +Step 23340/44000 | Loss: 2.9888 | LR: 2.32e-04 | 725.4ms/step | 27.6 samples/s +Step 23350/44000 | Loss: 2.9816 | LR: 2.32e-04 | 725.8ms/step | 27.6 samples/s +Step 23360/44000 | Loss: 2.9722 | LR: 2.32e-04 | 735.6ms/step | 27.2 samples/s +Step 23370/44000 | Loss: 2.9845 | LR: 2.32e-04 | 725.0ms/step | 27.6 samples/s +Step 23380/44000 | Loss: 2.9131 | LR: 2.32e-04 | 725.3ms/step | 27.6 samples/s +Step 23390/44000 | Loss: 2.9799 | LR: 2.32e-04 | 725.4ms/step | 27.6 samples/s +Step 23400/44000 | Loss: 2.9477 | LR: 2.31e-04 | 726.1ms/step | 27.5 samples/s +Step 23410/44000 | Loss: 2.9729 | LR: 2.31e-04 | 725.5ms/step | 27.6 samples/s +Step 23420/44000 | Loss: 2.9112 | LR: 2.31e-04 | 725.7ms/step | 27.6 samples/s +Step 23430/44000 | Loss: 2.9635 | LR: 2.31e-04 | 725.8ms/step | 27.6 samples/s +Step 23440/44000 | Loss: 2.9496 | LR: 2.31e-04 | 730.4ms/step | 27.4 samples/s +Step 23450/44000 | Loss: 2.9589 | LR: 2.30e-04 | 734.3ms/step | 27.2 samples/s +Step 23460/44000 | Loss: 2.9867 | LR: 2.30e-04 | 726.1ms/step | 27.5 samples/s +Step 23470/44000 | Loss: 3.0482 | LR: 2.30e-04 | 725.9ms/step | 27.6 samples/s +Step 23480/44000 | Loss: 3.0211 | LR: 2.30e-04 | 725.4ms/step | 27.6 samples/s +Step 23490/44000 | Loss: 2.9774 | LR: 2.30e-04 | 726.0ms/step | 27.5 samples/s +Step 23500/44000 | Loss: 2.9813 | LR: 2.30e-04 | 726.4ms/step | 27.5 samples/s +Validation at step 23500: Loss = 2.9551, Perplexity = 19.20 +Step 23510/44000 | Loss: 2.9505 | LR: 2.29e-04 | 947.8ms/step | 21.1 samples/s +Step 23520/44000 | Loss: 2.9623 | LR: 2.29e-04 | 725.7ms/step | 27.6 samples/s +Step 23530/44000 | Loss: 2.9473 | LR: 2.29e-04 | 735.1ms/step | 27.2 samples/s +Step 23540/44000 | Loss: 2.9668 | LR: 2.29e-04 | 725.1ms/step | 27.6 samples/s +Step 23550/44000 | Loss: 2.9864 | LR: 2.29e-04 | 726.1ms/step | 27.5 samples/s +Step 23560/44000 | Loss: 2.9429 | LR: 2.29e-04 | 725.9ms/step | 27.6 samples/s +Step 23570/44000 | Loss: 3.0310 | LR: 2.28e-04 | 726.3ms/step | 27.5 samples/s +Step 23580/44000 | Loss: 2.9011 | LR: 2.28e-04 | 726.0ms/step | 27.5 samples/s +Step 23590/44000 | Loss: 2.9354 | LR: 2.28e-04 | 726.2ms/step | 27.5 samples/s +Step 23600/44000 | Loss: 3.0087 | LR: 2.28e-04 | 726.4ms/step | 27.5 samples/s +Step 23610/44000 | Loss: 2.9396 | LR: 2.28e-04 | 726.5ms/step | 27.5 samples/s +Step 23620/44000 | Loss: 2.9368 | LR: 2.27e-04 | 737.7ms/step | 27.1 samples/s +Step 23630/44000 | Loss: 2.9672 | LR: 2.27e-04 | 726.1ms/step | 27.5 samples/s +Step 23640/44000 | Loss: 2.8807 | LR: 2.27e-04 | 726.5ms/step | 27.5 samples/s +Step 23650/44000 | Loss: 2.9414 | LR: 2.27e-04 | 725.9ms/step | 27.6 samples/s +Step 23660/44000 | Loss: 2.9857 | LR: 2.27e-04 | 726.1ms/step | 27.5 samples/s +Step 23670/44000 | Loss: 2.9824 | LR: 2.27e-04 | 726.4ms/step | 27.5 samples/s +Step 23680/44000 | Loss: 2.9475 | LR: 2.26e-04 | 725.5ms/step | 27.6 samples/s +Step 23690/44000 | Loss: 2.9595 | LR: 2.26e-04 | 725.9ms/step | 27.6 samples/s +Step 23700/44000 | Loss: 2.9564 | LR: 2.26e-04 | 731.6ms/step | 27.3 samples/s +Step 23710/44000 | Loss: 2.9850 | LR: 2.26e-04 | 730.6ms/step | 27.4 samples/s +Step 23720/44000 | Loss: 2.9669 | LR: 2.26e-04 | 725.5ms/step | 27.6 samples/s +Step 23730/44000 | Loss: 2.9752 | LR: 2.25e-04 | 725.1ms/step | 27.6 samples/s +Step 23740/44000 | Loss: 2.9835 | LR: 2.25e-04 | 725.5ms/step | 27.6 samples/s +Step 23750/44000 | Loss: 2.9342 | LR: 2.25e-04 | 725.5ms/step | 27.6 samples/s +Validation at step 23750: Loss = 2.9514, Perplexity = 19.13 +Step 23760/44000 | Loss: 2.9947 | LR: 2.25e-04 | 949.6ms/step | 21.1 samples/s +Step 23770/44000 | Loss: 2.8653 | LR: 2.25e-04 | 726.2ms/step | 27.5 samples/s +Step 23780/44000 | Loss: 2.9389 | LR: 2.25e-04 | 725.0ms/step | 27.6 samples/s +Step 23790/44000 | Loss: 2.9388 | LR: 2.24e-04 | 737.5ms/step | 27.1 samples/s +Step 23800/44000 | Loss: 2.8621 | LR: 2.24e-04 | 725.8ms/step | 27.6 samples/s +Step 23810/44000 | Loss: 2.8833 | LR: 2.24e-04 | 726.1ms/step | 27.5 samples/s +Step 23820/44000 | Loss: 2.9239 | LR: 2.24e-04 | 725.8ms/step | 27.6 samples/s +Step 23830/44000 | Loss: 3.0019 | LR: 2.24e-04 | 726.1ms/step | 27.5 samples/s +Step 23840/44000 | Loss: 2.9799 | LR: 2.23e-04 | 725.6ms/step | 27.6 samples/s +Step 23850/44000 | Loss: 2.9264 | LR: 2.23e-04 | 725.6ms/step | 27.6 samples/s +Step 23860/44000 | Loss: 2.9829 | LR: 2.23e-04 | 725.4ms/step | 27.6 samples/s +Step 23870/44000 | Loss: 2.9727 | LR: 2.23e-04 | 732.5ms/step | 27.3 samples/s +Step 23880/44000 | Loss: 2.9725 | LR: 2.23e-04 | 732.3ms/step | 27.3 samples/s +Step 23890/44000 | Loss: 2.9107 | LR: 2.23e-04 | 725.1ms/step | 27.6 samples/s +Step 23900/44000 | Loss: 2.8934 | LR: 2.22e-04 | 726.3ms/step | 27.5 samples/s +Step 23910/44000 | Loss: 2.9611 | LR: 2.22e-04 | 726.6ms/step | 27.5 samples/s +Step 23920/44000 | Loss: 2.9728 | LR: 2.22e-04 | 725.5ms/step | 27.6 samples/s +Step 23930/44000 | Loss: 2.9379 | LR: 2.22e-04 | 725.4ms/step | 27.6 samples/s +Step 23940/44000 | Loss: 2.9740 | LR: 2.22e-04 | 725.3ms/step | 27.6 samples/s +Step 23950/44000 | Loss: 2.9811 | LR: 2.21e-04 | 724.9ms/step | 27.6 samples/s +Step 23960/44000 | Loss: 2.9876 | LR: 2.21e-04 | 734.9ms/step | 27.2 samples/s +Step 23970/44000 | Loss: 3.0072 | LR: 2.21e-04 | 725.5ms/step | 27.6 samples/s +Step 23980/44000 | Loss: 3.0094 | LR: 2.21e-04 | 725.3ms/step | 27.6 samples/s +Step 23990/44000 | Loss: 2.8661 | LR: 2.21e-04 | 726.1ms/step | 27.5 samples/s +Step 24000/44000 | Loss: 2.9726 | LR: 2.21e-04 | 726.2ms/step | 27.5 samples/s +Validation at step 24000: Loss = 2.9508, Perplexity = 19.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-24000 +Step 24010/44000 | Loss: 3.0031 | LR: 2.20e-04 | 1368.1ms/step | 14.6 samples/s +Step 24020/44000 | Loss: 2.9809 | LR: 2.20e-04 | 725.0ms/step | 27.6 samples/s +Step 24030/44000 | Loss: 2.9611 | LR: 2.20e-04 | 725.1ms/step | 27.6 samples/s +Step 24040/44000 | Loss: 2.9577 | LR: 2.20e-04 | 736.1ms/step | 27.2 samples/s +Step 24050/44000 | Loss: 2.9314 | LR: 2.20e-04 | 726.2ms/step | 27.5 samples/s +Step 24060/44000 | Loss: 2.9883 | LR: 2.19e-04 | 725.3ms/step | 27.6 samples/s +Step 24070/44000 | Loss: 2.9902 | LR: 2.19e-04 | 725.3ms/step | 27.6 samples/s +Step 24080/44000 | Loss: 2.9401 | LR: 2.19e-04 | 725.7ms/step | 27.6 samples/s +Step 24090/44000 | Loss: 2.9603 | LR: 2.19e-04 | 726.9ms/step | 27.5 samples/s +Step 24100/44000 | Loss: 2.9453 | LR: 2.19e-04 | 726.8ms/step | 27.5 samples/s +Step 24110/44000 | Loss: 3.0271 | LR: 2.19e-04 | 725.8ms/step | 27.6 samples/s +Step 24120/44000 | Loss: 2.9988 | LR: 2.18e-04 | 726.5ms/step | 27.5 samples/s +Step 24130/44000 | Loss: 2.9499 | LR: 2.18e-04 | 735.2ms/step | 27.2 samples/s +Step 24140/44000 | Loss: 2.9554 | LR: 2.18e-04 | 726.2ms/step | 27.5 samples/s +Step 24150/44000 | Loss: 2.9677 | LR: 2.18e-04 | 725.1ms/step | 27.6 samples/s +Step 24160/44000 | Loss: 2.9990 | LR: 2.18e-04 | 725.7ms/step | 27.6 samples/s +Step 24170/44000 | Loss: 2.9935 | LR: 2.17e-04 | 726.2ms/step | 27.5 samples/s +Step 24180/44000 | Loss: 3.0002 | LR: 2.17e-04 | 725.5ms/step | 27.6 samples/s +Step 24190/44000 | Loss: 2.9426 | LR: 2.17e-04 | 725.5ms/step | 27.6 samples/s +Step 24200/44000 | Loss: 2.8993 | LR: 2.17e-04 | 725.5ms/step | 27.6 samples/s +Step 24210/44000 | Loss: 2.9799 | LR: 2.17e-04 | 732.9ms/step | 27.3 samples/s +Step 24220/44000 | Loss: 2.9689 | LR: 2.17e-04 | 730.4ms/step | 27.4 samples/s +Step 24230/44000 | Loss: 2.9925 | LR: 2.16e-04 | 725.8ms/step | 27.6 samples/s +Step 24240/44000 | Loss: 2.9470 | LR: 2.16e-04 | 725.6ms/step | 27.6 samples/s +Step 24250/44000 | Loss: 2.9375 | LR: 2.16e-04 | 725.2ms/step | 27.6 samples/s +Validation at step 24250: Loss = 2.9481, Perplexity = 19.07 +Step 24260/44000 | Loss: 2.8652 | LR: 2.16e-04 | 949.1ms/step | 21.1 samples/s +Step 24270/44000 | Loss: 2.9102 | LR: 2.16e-04 | 725.5ms/step | 27.6 samples/s +Step 24280/44000 | Loss: 2.9904 | LR: 2.16e-04 | 725.2ms/step | 27.6 samples/s +Step 24290/44000 | Loss: 2.9310 | LR: 2.15e-04 | 725.2ms/step | 27.6 samples/s +Step 24300/44000 | Loss: 3.0033 | LR: 2.15e-04 | 734.8ms/step | 27.2 samples/s +Step 24310/44000 | Loss: 2.9735 | LR: 2.15e-04 | 725.4ms/step | 27.6 samples/s +Step 24320/44000 | Loss: 2.9395 | LR: 2.15e-04 | 725.7ms/step | 27.6 samples/s +Step 24330/44000 | Loss: 2.8849 | LR: 2.15e-04 | 725.5ms/step | 27.6 samples/s +Step 24340/44000 | Loss: 2.9731 | LR: 2.14e-04 | 725.6ms/step | 27.6 samples/s +Step 24350/44000 | Loss: 3.0036 | LR: 2.14e-04 | 725.7ms/step | 27.6 samples/s +Step 24360/44000 | Loss: 2.9583 | LR: 2.14e-04 | 725.6ms/step | 27.6 samples/s +Step 24370/44000 | Loss: 2.9347 | LR: 2.14e-04 | 725.6ms/step | 27.6 samples/s +Step 24380/44000 | Loss: 2.9829 | LR: 2.14e-04 | 725.5ms/step | 27.6 samples/s +Step 24390/44000 | Loss: 2.9717 | LR: 2.14e-04 | 735.2ms/step | 27.2 samples/s +Step 24400/44000 | Loss: 2.9913 | LR: 2.13e-04 | 725.7ms/step | 27.6 samples/s +Step 24410/44000 | Loss: 2.9107 | LR: 2.13e-04 | 725.8ms/step | 27.6 samples/s +Step 24420/44000 | Loss: 2.9463 | LR: 2.13e-04 | 725.2ms/step | 27.6 samples/s +Step 24430/44000 | Loss: 2.9088 | LR: 2.13e-04 | 726.1ms/step | 27.5 samples/s +Step 24440/44000 | Loss: 2.9496 | LR: 2.13e-04 | 725.8ms/step | 27.6 samples/s +Step 24450/44000 | Loss: 2.9580 | LR: 2.12e-04 | 725.9ms/step | 27.6 samples/s +Step 24460/44000 | Loss: 2.9427 | LR: 2.12e-04 | 725.5ms/step | 27.6 samples/s +Step 24470/44000 | Loss: 3.0183 | LR: 2.12e-04 | 733.4ms/step | 27.3 samples/s +Step 24480/44000 | Loss: 2.8624 | LR: 2.12e-04 | 732.2ms/step | 27.3 samples/s +Step 24490/44000 | Loss: 3.0252 | LR: 2.12e-04 | 725.9ms/step | 27.6 samples/s +Step 24500/44000 | Loss: 3.0102 | LR: 2.12e-04 | 726.1ms/step | 27.5 samples/s +Validation at step 24500: Loss = 2.9468, Perplexity = 19.05 +Step 24510/44000 | Loss: 3.0101 | LR: 2.11e-04 | 950.0ms/step | 21.1 samples/s +Step 24520/44000 | Loss: 2.9374 | LR: 2.11e-04 | 726.5ms/step | 27.5 samples/s +Step 24530/44000 | Loss: 2.9523 | LR: 2.11e-04 | 725.1ms/step | 27.6 samples/s +Step 24540/44000 | Loss: 2.9061 | LR: 2.11e-04 | 725.4ms/step | 27.6 samples/s +Step 24550/44000 | Loss: 2.9872 | LR: 2.11e-04 | 725.5ms/step | 27.6 samples/s +Step 24560/44000 | Loss: 2.9160 | LR: 2.10e-04 | 734.7ms/step | 27.2 samples/s +Step 24570/44000 | Loss: 2.9765 | LR: 2.10e-04 | 725.7ms/step | 27.6 samples/s +Step 24580/44000 | Loss: 2.9889 | LR: 2.10e-04 | 726.3ms/step | 27.5 samples/s +Step 24590/44000 | Loss: 2.9384 | LR: 2.10e-04 | 726.3ms/step | 27.5 samples/s +Step 24600/44000 | Loss: 2.9418 | LR: 2.10e-04 | 725.6ms/step | 27.6 samples/s +Step 24610/44000 | Loss: 2.9900 | LR: 2.10e-04 | 726.4ms/step | 27.5 samples/s +Step 24620/44000 | Loss: 3.0281 | LR: 2.09e-04 | 725.5ms/step | 27.6 samples/s +Step 24630/44000 | Loss: 2.9190 | LR: 2.09e-04 | 726.2ms/step | 27.5 samples/s +Step 24640/44000 | Loss: 2.9412 | LR: 2.09e-04 | 726.2ms/step | 27.5 samples/s +Step 24650/44000 | Loss: 2.9739 | LR: 2.09e-04 | 739.4ms/step | 27.1 samples/s +Step 24660/44000 | Loss: 2.9767 | LR: 2.09e-04 | 725.6ms/step | 27.6 samples/s +Step 24670/44000 | Loss: 2.9227 | LR: 2.09e-04 | 726.2ms/step | 27.5 samples/s +Step 24680/44000 | Loss: 2.9356 | LR: 2.08e-04 | 726.1ms/step | 27.5 samples/s +Step 24690/44000 | Loss: 2.9951 | LR: 2.08e-04 | 726.4ms/step | 27.5 samples/s +Step 24700/44000 | Loss: 2.9995 | LR: 2.08e-04 | 725.7ms/step | 27.6 samples/s +Step 24710/44000 | Loss: 2.9209 | LR: 2.08e-04 | 725.3ms/step | 27.6 samples/s +Step 24720/44000 | Loss: 2.9662 | LR: 2.08e-04 | 725.6ms/step | 27.6 samples/s +Step 24730/44000 | Loss: 2.9802 | LR: 2.07e-04 | 730.6ms/step | 27.4 samples/s +Step 24740/44000 | Loss: 2.8881 | LR: 2.07e-04 | 730.0ms/step | 27.4 samples/s +Step 24750/44000 | Loss: 2.9678 | LR: 2.07e-04 | 725.7ms/step | 27.6 samples/s +Validation at step 24750: Loss = 2.9452, Perplexity = 19.01 +Step 24760/44000 | Loss: 2.9508 | LR: 2.07e-04 | 949.8ms/step | 21.1 samples/s +Step 24770/44000 | Loss: 2.9769 | LR: 2.07e-04 | 725.8ms/step | 27.6 samples/s +Step 24780/44000 | Loss: 2.9629 | LR: 2.07e-04 | 725.6ms/step | 27.6 samples/s +Step 24790/44000 | Loss: 2.9626 | LR: 2.06e-04 | 725.5ms/step | 27.6 samples/s +Step 24800/44000 | Loss: 2.9629 | LR: 2.06e-04 | 725.4ms/step | 27.6 samples/s +Step 24810/44000 | Loss: 2.9990 | LR: 2.06e-04 | 726.0ms/step | 27.5 samples/s +Step 24820/44000 | Loss: 2.9126 | LR: 2.06e-04 | 736.2ms/step | 27.2 samples/s +Step 24830/44000 | Loss: 2.9967 | LR: 2.06e-04 | 726.5ms/step | 27.5 samples/s +Step 24840/44000 | Loss: 2.9751 | LR: 2.05e-04 | 725.5ms/step | 27.6 samples/s +Step 24850/44000 | Loss: 2.9614 | LR: 2.05e-04 | 725.3ms/step | 27.6 samples/s +Step 24860/44000 | Loss: 2.9461 | LR: 2.05e-04 | 725.4ms/step | 27.6 samples/s +Step 24870/44000 | Loss: 2.9277 | LR: 2.05e-04 | 726.1ms/step | 27.5 samples/s +Step 24880/44000 | Loss: 2.9616 | LR: 2.05e-04 | 726.1ms/step | 27.5 samples/s +Step 24890/44000 | Loss: 2.9795 | LR: 2.05e-04 | 725.5ms/step | 27.6 samples/s +Step 24900/44000 | Loss: 2.9554 | LR: 2.04e-04 | 732.3ms/step | 27.3 samples/s +Step 24910/44000 | Loss: 2.9679 | LR: 2.04e-04 | 733.4ms/step | 27.3 samples/s +Step 24920/44000 | Loss: 3.0051 | LR: 2.04e-04 | 725.7ms/step | 27.6 samples/s +Step 24930/44000 | Loss: 2.9302 | LR: 2.04e-04 | 726.1ms/step | 27.5 samples/s +Step 24940/44000 | Loss: 2.9098 | LR: 2.04e-04 | 725.4ms/step | 27.6 samples/s +Step 24950/44000 | Loss: 2.9587 | LR: 2.04e-04 | 725.5ms/step | 27.6 samples/s +Step 24960/44000 | Loss: 2.9714 | LR: 2.03e-04 | 725.5ms/step | 27.6 samples/s +Step 24970/44000 | Loss: 2.9703 | LR: 2.03e-04 | 725.0ms/step | 27.6 samples/s +Step 24980/44000 | Loss: 2.9321 | LR: 2.03e-04 | 725.1ms/step | 27.6 samples/s +Step 24990/44000 | Loss: 2.9833 | LR: 2.03e-04 | 735.0ms/step | 27.2 samples/s +Step 25000/44000 | Loss: 2.9741 | LR: 2.03e-04 | 726.0ms/step | 27.5 samples/s +Validation at step 25000: Loss = 2.9426, Perplexity = 18.96 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-25000 +Step 25010/44000 | Loss: 2.9403 | LR: 2.02e-04 | 1378.1ms/step | 14.5 samples/s +Step 25020/44000 | Loss: 3.0487 | LR: 2.02e-04 | 725.3ms/step | 27.6 samples/s +Step 25030/44000 | Loss: 2.9359 | LR: 2.02e-04 | 725.4ms/step | 27.6 samples/s +Step 25040/44000 | Loss: 2.9695 | LR: 2.02e-04 | 726.0ms/step | 27.6 samples/s +Step 25050/44000 | Loss: 2.9781 | LR: 2.02e-04 | 725.6ms/step | 27.6 samples/s +Step 25060/44000 | Loss: 2.9546 | LR: 2.02e-04 | 725.6ms/step | 27.6 samples/s +Step 25070/44000 | Loss: 2.9298 | LR: 2.01e-04 | 735.2ms/step | 27.2 samples/s +Step 25080/44000 | Loss: 2.9646 | LR: 2.01e-04 | 725.6ms/step | 27.6 samples/s +Step 25090/44000 | Loss: 3.0342 | LR: 2.01e-04 | 725.2ms/step | 27.6 samples/s +Step 25100/44000 | Loss: 2.9046 | LR: 2.01e-04 | 725.8ms/step | 27.6 samples/s +Step 25110/44000 | Loss: 2.9842 | LR: 2.01e-04 | 726.5ms/step | 27.5 samples/s +Step 25120/44000 | Loss: 2.9353 | LR: 2.00e-04 | 726.3ms/step | 27.5 samples/s +Step 25130/44000 | Loss: 3.0230 | LR: 2.00e-04 | 725.3ms/step | 27.6 samples/s +Step 25140/44000 | Loss: 2.9698 | LR: 2.00e-04 | 725.7ms/step | 27.6 samples/s +Step 25150/44000 | Loss: 2.9796 | LR: 2.00e-04 | 726.1ms/step | 27.5 samples/s +Step 25160/44000 | Loss: 3.0188 | LR: 2.00e-04 | 740.2ms/step | 27.0 samples/s +Step 25170/44000 | Loss: 2.9088 | LR: 2.00e-04 | 725.4ms/step | 27.6 samples/s +Step 25180/44000 | Loss: 2.9494 | LR: 1.99e-04 | 725.6ms/step | 27.6 samples/s +Step 25190/44000 | Loss: 2.9484 | LR: 1.99e-04 | 726.0ms/step | 27.5 samples/s +Step 25200/44000 | Loss: 2.9291 | LR: 1.99e-04 | 725.6ms/step | 27.6 samples/s +Step 25210/44000 | Loss: 2.9673 | LR: 1.99e-04 | 725.6ms/step | 27.6 samples/s +Step 25220/44000 | Loss: 2.9987 | LR: 1.99e-04 | 725.7ms/step | 27.6 samples/s +Step 25230/44000 | Loss: 2.8927 | LR: 1.99e-04 | 726.0ms/step | 27.5 samples/s +Step 25240/44000 | Loss: 2.9910 | LR: 1.98e-04 | 730.6ms/step | 27.4 samples/s +Step 25250/44000 | Loss: 2.9798 | LR: 1.98e-04 | 730.4ms/step | 27.4 samples/s +Validation at step 25250: Loss = 2.9396, Perplexity = 18.91 +Step 25260/44000 | Loss: 2.9764 | LR: 1.98e-04 | 949.7ms/step | 21.1 samples/s +Step 25270/44000 | Loss: 2.9042 | LR: 1.98e-04 | 726.2ms/step | 27.5 samples/s +Step 25280/44000 | Loss: 2.9308 | LR: 1.98e-04 | 726.1ms/step | 27.5 samples/s +Step 25290/44000 | Loss: 2.9831 | LR: 1.97e-04 | 725.7ms/step | 27.6 samples/s +Step 25300/44000 | Loss: 2.9857 | LR: 1.97e-04 | 725.5ms/step | 27.6 samples/s +Step 25310/44000 | Loss: 2.9022 | LR: 1.97e-04 | 725.7ms/step | 27.6 samples/s +Step 25320/44000 | Loss: 2.9652 | LR: 1.97e-04 | 725.4ms/step | 27.6 samples/s +Step 25330/44000 | Loss: 2.9622 | LR: 1.97e-04 | 738.7ms/step | 27.1 samples/s +Step 25340/44000 | Loss: 2.9883 | LR: 1.97e-04 | 725.5ms/step | 27.6 samples/s +Step 25350/44000 | Loss: 3.0095 | LR: 1.96e-04 | 725.9ms/step | 27.6 samples/s +Step 25360/44000 | Loss: 2.8592 | LR: 1.96e-04 | 725.8ms/step | 27.6 samples/s +Step 25370/44000 | Loss: 2.9775 | LR: 1.96e-04 | 725.4ms/step | 27.6 samples/s +Step 25380/44000 | Loss: 2.8880 | LR: 1.96e-04 | 725.4ms/step | 27.6 samples/s +Step 25390/44000 | Loss: 2.9832 | LR: 1.96e-04 | 725.9ms/step | 27.6 samples/s +Step 25400/44000 | Loss: 2.9895 | LR: 1.96e-04 | 726.1ms/step | 27.5 samples/s +Step 25410/44000 | Loss: 2.9799 | LR: 1.95e-04 | 725.8ms/step | 27.6 samples/s +Step 25420/44000 | Loss: 3.0245 | LR: 1.95e-04 | 735.7ms/step | 27.2 samples/s +Step 25430/44000 | Loss: 2.9062 | LR: 1.95e-04 | 726.3ms/step | 27.5 samples/s +Step 25440/44000 | Loss: 2.9720 | LR: 1.95e-04 | 725.9ms/step | 27.6 samples/s +Step 25450/44000 | Loss: 2.8856 | LR: 1.95e-04 | 725.7ms/step | 27.6 samples/s +Step 25460/44000 | Loss: 3.0255 | LR: 1.94e-04 | 726.1ms/step | 27.5 samples/s +Step 25470/44000 | Loss: 2.9161 | LR: 1.94e-04 | 725.8ms/step | 27.6 samples/s +Step 25480/44000 | Loss: 2.9150 | LR: 1.94e-04 | 726.0ms/step | 27.5 samples/s +Step 25490/44000 | Loss: 2.9210 | LR: 1.94e-04 | 725.9ms/step | 27.6 samples/s +Step 25500/44000 | Loss: 2.9516 | LR: 1.94e-04 | 730.6ms/step | 27.4 samples/s +Validation at step 25500: Loss = 2.9390, Perplexity = 18.90 +Step 25510/44000 | Loss: 2.9468 | LR: 1.94e-04 | 960.3ms/step | 20.8 samples/s +Step 25520/44000 | Loss: 2.9496 | LR: 1.93e-04 | 726.4ms/step | 27.5 samples/s +Step 25530/44000 | Loss: 2.9015 | LR: 1.93e-04 | 726.5ms/step | 27.5 samples/s +Step 25540/44000 | Loss: 2.9288 | LR: 1.93e-04 | 726.0ms/step | 27.5 samples/s +Step 25550/44000 | Loss: 2.9616 | LR: 1.93e-04 | 725.6ms/step | 27.6 samples/s +Step 25560/44000 | Loss: 2.9507 | LR: 1.93e-04 | 725.3ms/step | 27.6 samples/s +Step 25570/44000 | Loss: 2.8899 | LR: 1.92e-04 | 725.1ms/step | 27.6 samples/s +Step 25580/44000 | Loss: 2.8966 | LR: 1.92e-04 | 725.8ms/step | 27.6 samples/s +Step 25590/44000 | Loss: 2.9113 | LR: 1.92e-04 | 735.9ms/step | 27.2 samples/s +Step 25600/44000 | Loss: 2.9806 | LR: 1.92e-04 | 726.5ms/step | 27.5 samples/s +Step 25610/44000 | Loss: 2.9513 | LR: 1.92e-04 | 726.1ms/step | 27.5 samples/s +Step 25620/44000 | Loss: 2.9269 | LR: 1.92e-04 | 726.4ms/step | 27.5 samples/s +Step 25630/44000 | Loss: 3.0001 | LR: 1.91e-04 | 725.8ms/step | 27.6 samples/s +Step 25640/44000 | Loss: 2.9279 | LR: 1.91e-04 | 726.4ms/step | 27.5 samples/s +Step 25650/44000 | Loss: 2.9303 | LR: 1.91e-04 | 726.2ms/step | 27.5 samples/s +Step 25660/44000 | Loss: 3.0382 | LR: 1.91e-04 | 726.2ms/step | 27.5 samples/s +Step 25670/44000 | Loss: 2.9038 | LR: 1.91e-04 | 731.0ms/step | 27.4 samples/s +Step 25680/44000 | Loss: 2.8727 | LR: 1.91e-04 | 733.5ms/step | 27.3 samples/s +Step 25690/44000 | Loss: 2.9257 | LR: 1.90e-04 | 726.0ms/step | 27.5 samples/s +Step 25700/44000 | Loss: 2.9382 | LR: 1.90e-04 | 725.9ms/step | 27.6 samples/s +Step 25710/44000 | Loss: 2.9661 | LR: 1.90e-04 | 725.6ms/step | 27.6 samples/s +Step 25720/44000 | Loss: 2.9126 | LR: 1.90e-04 | 726.4ms/step | 27.5 samples/s +Step 25730/44000 | Loss: 2.9523 | LR: 1.90e-04 | 726.6ms/step | 27.5 samples/s +Step 25740/44000 | Loss: 3.0019 | LR: 1.89e-04 | 726.1ms/step | 27.5 samples/s +Step 25750/44000 | Loss: 2.9722 | LR: 1.89e-04 | 725.8ms/step | 27.6 samples/s +Validation at step 25750: Loss = 2.9371, Perplexity = 18.86 +Step 25760/44000 | Loss: 2.9180 | LR: 1.89e-04 | 958.2ms/step | 20.9 samples/s +Step 25770/44000 | Loss: 2.9488 | LR: 1.89e-04 | 725.7ms/step | 27.6 samples/s +Step 25780/44000 | Loss: 2.9553 | LR: 1.89e-04 | 725.7ms/step | 27.6 samples/s +Step 25790/44000 | Loss: 2.9494 | LR: 1.89e-04 | 725.2ms/step | 27.6 samples/s +Step 25800/44000 | Loss: 2.9056 | LR: 1.88e-04 | 725.8ms/step | 27.6 samples/s +Step 25810/44000 | Loss: 2.9203 | LR: 1.88e-04 | 726.2ms/step | 27.5 samples/s +Step 25820/44000 | Loss: 2.9620 | LR: 1.88e-04 | 725.5ms/step | 27.6 samples/s +Step 25830/44000 | Loss: 2.9002 | LR: 1.88e-04 | 725.9ms/step | 27.6 samples/s +Step 25840/44000 | Loss: 3.0058 | LR: 1.88e-04 | 725.6ms/step | 27.6 samples/s +Step 25850/44000 | Loss: 2.9242 | LR: 1.88e-04 | 735.4ms/step | 27.2 samples/s +Step 25860/44000 | Loss: 2.9089 | LR: 1.87e-04 | 725.8ms/step | 27.6 samples/s +Step 25870/44000 | Loss: 2.9533 | LR: 1.87e-04 | 725.6ms/step | 27.6 samples/s +Step 25880/44000 | Loss: 2.9671 | LR: 1.87e-04 | 725.4ms/step | 27.6 samples/s +Step 25890/44000 | Loss: 2.9657 | LR: 1.87e-04 | 725.7ms/step | 27.6 samples/s +Step 25900/44000 | Loss: 2.8579 | LR: 1.87e-04 | 726.5ms/step | 27.5 samples/s +Step 25910/44000 | Loss: 3.0039 | LR: 1.87e-04 | 725.6ms/step | 27.6 samples/s +Step 25920/44000 | Loss: 3.0146 | LR: 1.86e-04 | 725.9ms/step | 27.6 samples/s +Step 25930/44000 | Loss: 2.9757 | LR: 1.86e-04 | 733.7ms/step | 27.3 samples/s +Step 25940/44000 | Loss: 2.9117 | LR: 1.86e-04 | 732.3ms/step | 27.3 samples/s +Step 25950/44000 | Loss: 2.9841 | LR: 1.86e-04 | 725.6ms/step | 27.6 samples/s +Step 25960/44000 | Loss: 2.9519 | LR: 1.86e-04 | 725.4ms/step | 27.6 samples/s +Step 25970/44000 | Loss: 2.9348 | LR: 1.85e-04 | 725.4ms/step | 27.6 samples/s +Step 25980/44000 | Loss: 2.9812 | LR: 1.85e-04 | 725.5ms/step | 27.6 samples/s +Step 25990/44000 | Loss: 2.9574 | LR: 1.85e-04 | 725.3ms/step | 27.6 samples/s +Step 26000/44000 | Loss: 2.9564 | LR: 1.85e-04 | 725.2ms/step | 27.6 samples/s +Validation at step 26000: Loss = 2.9352, Perplexity = 18.82 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-26000 +Step 26010/44000 | Loss: 2.9767 | LR: 1.85e-04 | 1370.8ms/step | 14.6 samples/s +Step 26020/44000 | Loss: 2.9701 | LR: 1.85e-04 | 729.7ms/step | 27.4 samples/s +Step 26030/44000 | Loss: 2.9864 | LR: 1.84e-04 | 725.1ms/step | 27.6 samples/s +Step 26040/44000 | Loss: 2.9390 | LR: 1.84e-04 | 725.8ms/step | 27.6 samples/s +Step 26050/44000 | Loss: 2.9201 | LR: 1.84e-04 | 725.8ms/step | 27.6 samples/s +Step 26060/44000 | Loss: 2.9797 | LR: 1.84e-04 | 725.3ms/step | 27.6 samples/s +Step 26070/44000 | Loss: 2.9722 | LR: 1.84e-04 | 724.9ms/step | 27.6 samples/s +Step 26080/44000 | Loss: 2.9113 | LR: 1.84e-04 | 725.2ms/step | 27.6 samples/s +Step 26090/44000 | Loss: 2.9611 | LR: 1.83e-04 | 725.5ms/step | 27.6 samples/s +Step 26100/44000 | Loss: 2.9944 | LR: 1.83e-04 | 736.4ms/step | 27.2 samples/s +Step 26110/44000 | Loss: 2.9907 | LR: 1.83e-04 | 725.5ms/step | 27.6 samples/s +Step 26120/44000 | Loss: 2.8949 | LR: 1.83e-04 | 726.4ms/step | 27.5 samples/s +Step 26130/44000 | Loss: 2.9304 | LR: 1.83e-04 | 726.6ms/step | 27.5 samples/s +Step 26140/44000 | Loss: 2.9242 | LR: 1.82e-04 | 726.1ms/step | 27.5 samples/s +Step 26150/44000 | Loss: 2.8849 | LR: 1.82e-04 | 727.2ms/step | 27.5 samples/s +Step 26160/44000 | Loss: 2.9302 | LR: 1.82e-04 | 726.7ms/step | 27.5 samples/s +Step 26170/44000 | Loss: 2.9563 | LR: 1.82e-04 | 726.4ms/step | 27.5 samples/s +Step 26180/44000 | Loss: 3.0238 | LR: 1.82e-04 | 726.0ms/step | 27.5 samples/s +Step 26190/44000 | Loss: 2.9522 | LR: 1.82e-04 | 735.5ms/step | 27.2 samples/s +Step 26200/44000 | Loss: 2.9486 | LR: 1.81e-04 | 726.2ms/step | 27.5 samples/s +Step 26210/44000 | Loss: 2.9458 | LR: 1.81e-04 | 725.8ms/step | 27.6 samples/s +Step 26220/44000 | Loss: 2.9607 | LR: 1.81e-04 | 725.6ms/step | 27.6 samples/s +Step 26230/44000 | Loss: 2.9331 | LR: 1.81e-04 | 726.1ms/step | 27.5 samples/s +Step 26240/44000 | Loss: 3.0222 | LR: 1.81e-04 | 725.5ms/step | 27.6 samples/s +Step 26250/44000 | Loss: 2.9132 | LR: 1.81e-04 | 726.0ms/step | 27.5 samples/s +Validation at step 26250: Loss = 2.9338, Perplexity = 18.80 +Step 26260/44000 | Loss: 2.9494 | LR: 1.80e-04 | 950.2ms/step | 21.0 samples/s +Step 26270/44000 | Loss: 2.8482 | LR: 1.80e-04 | 735.9ms/step | 27.2 samples/s +Step 26280/44000 | Loss: 3.0048 | LR: 1.80e-04 | 726.4ms/step | 27.5 samples/s +Step 26290/44000 | Loss: 2.9162 | LR: 1.80e-04 | 726.6ms/step | 27.5 samples/s +Step 26300/44000 | Loss: 2.9356 | LR: 1.80e-04 | 726.1ms/step | 27.5 samples/s +Step 26310/44000 | Loss: 2.9649 | LR: 1.80e-04 | 725.8ms/step | 27.6 samples/s +Step 26320/44000 | Loss: 2.8745 | LR: 1.79e-04 | 726.0ms/step | 27.5 samples/s +Step 26330/44000 | Loss: 2.9742 | LR: 1.79e-04 | 725.7ms/step | 27.6 samples/s +Step 26340/44000 | Loss: 2.8685 | LR: 1.79e-04 | 726.6ms/step | 27.5 samples/s +Step 26350/44000 | Loss: 2.9200 | LR: 1.79e-04 | 725.5ms/step | 27.6 samples/s +Step 26360/44000 | Loss: 2.8435 | LR: 1.79e-04 | 734.3ms/step | 27.2 samples/s +Step 26370/44000 | Loss: 2.9338 | LR: 1.78e-04 | 725.6ms/step | 27.6 samples/s +Step 26380/44000 | Loss: 2.8645 | LR: 1.78e-04 | 725.2ms/step | 27.6 samples/s +Step 26390/44000 | Loss: 2.9395 | LR: 1.78e-04 | 725.2ms/step | 27.6 samples/s +Step 26400/44000 | Loss: 2.9022 | LR: 1.78e-04 | 725.5ms/step | 27.6 samples/s +Step 26410/44000 | Loss: 2.9138 | LR: 1.78e-04 | 725.1ms/step | 27.6 samples/s +Step 26420/44000 | Loss: 2.9350 | LR: 1.78e-04 | 725.3ms/step | 27.6 samples/s +Step 26430/44000 | Loss: 2.9142 | LR: 1.77e-04 | 725.5ms/step | 27.6 samples/s +Step 26440/44000 | Loss: 2.9539 | LR: 1.77e-04 | 725.6ms/step | 27.6 samples/s +Step 26450/44000 | Loss: 2.9943 | LR: 1.77e-04 | 739.0ms/step | 27.1 samples/s +Step 26460/44000 | Loss: 2.9190 | LR: 1.77e-04 | 724.9ms/step | 27.6 samples/s +Step 26470/44000 | Loss: 2.9463 | LR: 1.77e-04 | 725.2ms/step | 27.6 samples/s +Step 26480/44000 | Loss: 2.9453 | LR: 1.77e-04 | 724.9ms/step | 27.6 samples/s +Step 26490/44000 | Loss: 2.8693 | LR: 1.76e-04 | 725.8ms/step | 27.6 samples/s +Step 26500/44000 | Loss: 2.9375 | LR: 1.76e-04 | 725.5ms/step | 27.6 samples/s +Validation at step 26500: Loss = 2.9334, Perplexity = 18.79 +Step 26510/44000 | Loss: 3.0141 | LR: 1.76e-04 | 949.8ms/step | 21.1 samples/s +Step 26520/44000 | Loss: 2.9057 | LR: 1.76e-04 | 726.0ms/step | 27.5 samples/s +Step 26530/44000 | Loss: 2.9618 | LR: 1.76e-04 | 734.8ms/step | 27.2 samples/s +Step 26540/44000 | Loss: 2.9740 | LR: 1.76e-04 | 725.4ms/step | 27.6 samples/s +Step 26550/44000 | Loss: 2.9159 | LR: 1.75e-04 | 725.8ms/step | 27.6 samples/s +Step 26560/44000 | Loss: 2.9063 | LR: 1.75e-04 | 725.9ms/step | 27.6 samples/s +Step 26570/44000 | Loss: 2.9630 | LR: 1.75e-04 | 725.9ms/step | 27.6 samples/s +Step 26580/44000 | Loss: 2.9633 | LR: 1.75e-04 | 725.8ms/step | 27.6 samples/s +Step 26590/44000 | Loss: 2.9027 | LR: 1.75e-04 | 725.9ms/step | 27.6 samples/s +Step 26600/44000 | Loss: 3.0225 | LR: 1.74e-04 | 725.9ms/step | 27.6 samples/s +Step 26610/44000 | Loss: 2.9779 | LR: 1.74e-04 | 726.2ms/step | 27.5 samples/s +Step 26620/44000 | Loss: 3.0004 | LR: 1.74e-04 | 736.8ms/step | 27.1 samples/s +Step 26630/44000 | Loss: 2.9758 | LR: 1.74e-04 | 725.4ms/step | 27.6 samples/s +Step 26640/44000 | Loss: 2.9755 | LR: 1.74e-04 | 725.7ms/step | 27.6 samples/s +Step 26650/44000 | Loss: 2.8911 | LR: 1.74e-04 | 725.3ms/step | 27.6 samples/s +Step 26660/44000 | Loss: 2.9504 | LR: 1.73e-04 | 725.2ms/step | 27.6 samples/s +Step 26670/44000 | Loss: 2.9082 | LR: 1.73e-04 | 725.0ms/step | 27.6 samples/s +Step 26680/44000 | Loss: 2.9543 | LR: 1.73e-04 | 725.8ms/step | 27.6 samples/s +Step 26690/44000 | Loss: 3.0123 | LR: 1.73e-04 | 725.8ms/step | 27.6 samples/s +Step 26700/44000 | Loss: 2.9735 | LR: 1.73e-04 | 734.1ms/step | 27.2 samples/s +Step 26710/44000 | Loss: 2.9131 | LR: 1.73e-04 | 730.4ms/step | 27.4 samples/s +Step 26720/44000 | Loss: 2.9658 | LR: 1.72e-04 | 725.9ms/step | 27.6 samples/s +Step 26730/44000 | Loss: 3.0014 | LR: 1.72e-04 | 726.4ms/step | 27.5 samples/s +Step 26740/44000 | Loss: 2.9820 | LR: 1.72e-04 | 725.9ms/step | 27.6 samples/s +Step 26750/44000 | Loss: 2.9226 | LR: 1.72e-04 | 726.0ms/step | 27.5 samples/s +Validation at step 26750: Loss = 2.9321, Perplexity = 18.77 +Step 26760/44000 | Loss: 2.9429 | LR: 1.72e-04 | 950.1ms/step | 21.1 samples/s +Step 26770/44000 | Loss: 2.8900 | LR: 1.72e-04 | 725.4ms/step | 27.6 samples/s +Step 26780/44000 | Loss: 2.8673 | LR: 1.71e-04 | 726.0ms/step | 27.5 samples/s +Step 26790/44000 | Loss: 2.9764 | LR: 1.71e-04 | 735.6ms/step | 27.2 samples/s +Step 26800/44000 | Loss: 2.9553 | LR: 1.71e-04 | 725.6ms/step | 27.6 samples/s +Step 26810/44000 | Loss: 2.9020 | LR: 1.71e-04 | 725.7ms/step | 27.6 samples/s +Step 26820/44000 | Loss: 2.8726 | LR: 1.71e-04 | 725.7ms/step | 27.6 samples/s +Step 26830/44000 | Loss: 2.9039 | LR: 1.71e-04 | 725.6ms/step | 27.6 samples/s +Step 26840/44000 | Loss: 2.9093 | LR: 1.70e-04 | 725.7ms/step | 27.6 samples/s +Step 26850/44000 | Loss: 2.9776 | LR: 1.70e-04 | 725.7ms/step | 27.6 samples/s +Step 26860/44000 | Loss: 2.9006 | LR: 1.70e-04 | 725.8ms/step | 27.6 samples/s +Step 26870/44000 | Loss: 2.9752 | LR: 1.70e-04 | 726.5ms/step | 27.5 samples/s +Step 26880/44000 | Loss: 3.0760 | LR: 1.70e-04 | 740.0ms/step | 27.0 samples/s +Step 26890/44000 | Loss: 2.9080 | LR: 1.69e-04 | 726.1ms/step | 27.5 samples/s +Step 26900/44000 | Loss: 2.9242 | LR: 1.69e-04 | 725.8ms/step | 27.6 samples/s +Step 26910/44000 | Loss: 2.9601 | LR: 1.69e-04 | 725.5ms/step | 27.6 samples/s +Step 26920/44000 | Loss: 2.8904 | LR: 1.69e-04 | 725.0ms/step | 27.6 samples/s +Step 26930/44000 | Loss: 2.9097 | LR: 1.69e-04 | 725.6ms/step | 27.6 samples/s +Step 26940/44000 | Loss: 2.9582 | LR: 1.69e-04 | 725.5ms/step | 27.6 samples/s +Step 26950/44000 | Loss: 2.8593 | LR: 1.68e-04 | 726.0ms/step | 27.5 samples/s +Step 26960/44000 | Loss: 2.8934 | LR: 1.68e-04 | 730.4ms/step | 27.4 samples/s +Step 26970/44000 | Loss: 2.9803 | LR: 1.68e-04 | 730.3ms/step | 27.4 samples/s +Step 26980/44000 | Loss: 2.9665 | LR: 1.68e-04 | 725.9ms/step | 27.6 samples/s +Step 26990/44000 | Loss: 2.9114 | LR: 1.68e-04 | 726.0ms/step | 27.5 samples/s +Step 27000/44000 | Loss: 2.9177 | LR: 1.68e-04 | 726.1ms/step | 27.5 samples/s +Validation at step 27000: Loss = 2.9304, Perplexity = 18.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-27000 +Step 27010/44000 | Loss: 2.8992 | LR: 1.67e-04 | 1357.2ms/step | 14.7 samples/s +Step 27020/44000 | Loss: 2.9093 | LR: 1.67e-04 | 725.7ms/step | 27.6 samples/s +Step 27030/44000 | Loss: 3.0018 | LR: 1.67e-04 | 726.0ms/step | 27.5 samples/s +Step 27040/44000 | Loss: 2.9684 | LR: 1.67e-04 | 730.8ms/step | 27.4 samples/s +Step 27050/44000 | Loss: 2.9034 | LR: 1.67e-04 | 734.1ms/step | 27.2 samples/s +Step 27060/44000 | Loss: 2.9637 | LR: 1.67e-04 | 725.5ms/step | 27.6 samples/s +Step 27070/44000 | Loss: 2.9703 | LR: 1.66e-04 | 725.6ms/step | 27.6 samples/s +Step 27080/44000 | Loss: 2.9609 | LR: 1.66e-04 | 726.1ms/step | 27.5 samples/s +Step 27090/44000 | Loss: 3.0325 | LR: 1.66e-04 | 725.1ms/step | 27.6 samples/s +Step 27100/44000 | Loss: 2.8990 | LR: 1.66e-04 | 725.7ms/step | 27.6 samples/s +Step 27110/44000 | Loss: 2.9717 | LR: 1.66e-04 | 726.2ms/step | 27.5 samples/s +Step 27120/44000 | Loss: 2.9441 | LR: 1.66e-04 | 725.9ms/step | 27.6 samples/s +Step 27130/44000 | Loss: 2.9242 | LR: 1.65e-04 | 736.7ms/step | 27.1 samples/s +Step 27140/44000 | Loss: 2.9775 | LR: 1.65e-04 | 725.6ms/step | 27.6 samples/s +Step 27150/44000 | Loss: 2.9680 | LR: 1.65e-04 | 725.9ms/step | 27.6 samples/s +Step 27160/44000 | Loss: 2.9351 | LR: 1.65e-04 | 725.2ms/step | 27.6 samples/s +Step 27170/44000 | Loss: 2.9186 | LR: 1.65e-04 | 726.7ms/step | 27.5 samples/s +Step 27180/44000 | Loss: 3.0010 | LR: 1.65e-04 | 725.8ms/step | 27.6 samples/s +Step 27190/44000 | Loss: 2.8652 | LR: 1.64e-04 | 725.3ms/step | 27.6 samples/s +Step 27200/44000 | Loss: 2.9424 | LR: 1.64e-04 | 725.7ms/step | 27.6 samples/s +Step 27210/44000 | Loss: 2.9266 | LR: 1.64e-04 | 725.6ms/step | 27.6 samples/s +Step 27220/44000 | Loss: 2.9787 | LR: 1.64e-04 | 734.9ms/step | 27.2 samples/s +Step 27230/44000 | Loss: 2.9101 | LR: 1.64e-04 | 725.1ms/step | 27.6 samples/s +Step 27240/44000 | Loss: 2.9681 | LR: 1.63e-04 | 725.5ms/step | 27.6 samples/s +Step 27250/44000 | Loss: 2.9587 | LR: 1.63e-04 | 725.5ms/step | 27.6 samples/s +Validation at step 27250: Loss = 2.9294, Perplexity = 18.72 +Step 27260/44000 | Loss: 2.9268 | LR: 1.63e-04 | 950.5ms/step | 21.0 samples/s +Step 27270/44000 | Loss: 2.9580 | LR: 1.63e-04 | 725.6ms/step | 27.6 samples/s +Step 27280/44000 | Loss: 2.9637 | LR: 1.63e-04 | 725.8ms/step | 27.6 samples/s +Step 27290/44000 | Loss: 2.9555 | LR: 1.63e-04 | 725.2ms/step | 27.6 samples/s +Step 27300/44000 | Loss: 3.0206 | LR: 1.62e-04 | 739.3ms/step | 27.1 samples/s +Step 27310/44000 | Loss: 2.9165 | LR: 1.62e-04 | 725.1ms/step | 27.6 samples/s +Step 27320/44000 | Loss: 2.9797 | LR: 1.62e-04 | 724.9ms/step | 27.6 samples/s +Step 27330/44000 | Loss: 2.8957 | LR: 1.62e-04 | 725.0ms/step | 27.6 samples/s +Step 27340/44000 | Loss: 2.9824 | LR: 1.62e-04 | 725.5ms/step | 27.6 samples/s +Step 27350/44000 | Loss: 2.9024 | LR: 1.62e-04 | 725.2ms/step | 27.6 samples/s +Step 27360/44000 | Loss: 2.9286 | LR: 1.61e-04 | 725.1ms/step | 27.6 samples/s +Step 27370/44000 | Loss: 2.9694 | LR: 1.61e-04 | 725.0ms/step | 27.6 samples/s +Step 27380/44000 | Loss: 3.0040 | LR: 1.61e-04 | 725.4ms/step | 27.6 samples/s +Step 27390/44000 | Loss: 2.9828 | LR: 1.61e-04 | 734.7ms/step | 27.2 samples/s +Step 27400/44000 | Loss: 2.8711 | LR: 1.61e-04 | 725.2ms/step | 27.6 samples/s +Step 27410/44000 | Loss: 2.9325 | LR: 1.61e-04 | 725.4ms/step | 27.6 samples/s +Step 27420/44000 | Loss: 3.0106 | LR: 1.60e-04 | 726.1ms/step | 27.5 samples/s +Step 27430/44000 | Loss: 2.9753 | LR: 1.60e-04 | 725.8ms/step | 27.6 samples/s +Step 27440/44000 | Loss: 2.9085 | LR: 1.60e-04 | 725.3ms/step | 27.6 samples/s +Step 27450/44000 | Loss: 3.0653 | LR: 1.60e-04 | 725.4ms/step | 27.6 samples/s +Step 27460/44000 | Loss: 2.9339 | LR: 1.60e-04 | 725.5ms/step | 27.6 samples/s +Step 27470/44000 | Loss: 2.9085 | LR: 1.60e-04 | 726.4ms/step | 27.5 samples/s +Step 27480/44000 | Loss: 2.9642 | LR: 1.59e-04 | 737.0ms/step | 27.1 samples/s +Step 27490/44000 | Loss: 2.9637 | LR: 1.59e-04 | 725.6ms/step | 27.6 samples/s +Step 27500/44000 | Loss: 2.9323 | LR: 1.59e-04 | 725.8ms/step | 27.6 samples/s +Validation at step 27500: Loss = 2.9286, Perplexity = 18.70 +Step 27510/44000 | Loss: 2.9213 | LR: 1.59e-04 | 949.1ms/step | 21.1 samples/s +Step 27520/44000 | Loss: 2.8891 | LR: 1.59e-04 | 725.6ms/step | 27.6 samples/s +Step 27530/44000 | Loss: 2.9890 | LR: 1.59e-04 | 726.0ms/step | 27.5 samples/s +Step 27540/44000 | Loss: 2.9469 | LR: 1.58e-04 | 726.3ms/step | 27.5 samples/s +Step 27550/44000 | Loss: 2.9427 | LR: 1.58e-04 | 726.5ms/step | 27.5 samples/s +Step 27560/44000 | Loss: 2.9208 | LR: 1.58e-04 | 738.0ms/step | 27.1 samples/s +Step 27570/44000 | Loss: 2.9423 | LR: 1.58e-04 | 727.3ms/step | 27.5 samples/s +Step 27580/44000 | Loss: 2.9515 | LR: 1.58e-04 | 726.0ms/step | 27.5 samples/s +Step 27590/44000 | Loss: 2.9506 | LR: 1.58e-04 | 726.1ms/step | 27.5 samples/s +Step 27600/44000 | Loss: 2.9122 | LR: 1.57e-04 | 726.1ms/step | 27.5 samples/s +Step 27610/44000 | Loss: 2.9723 | LR: 1.57e-04 | 725.8ms/step | 27.6 samples/s +Step 27620/44000 | Loss: 2.9414 | LR: 1.57e-04 | 726.2ms/step | 27.5 samples/s +Step 27630/44000 | Loss: 2.9332 | LR: 1.57e-04 | 726.1ms/step | 27.5 samples/s +Step 27640/44000 | Loss: 2.9212 | LR: 1.57e-04 | 726.1ms/step | 27.5 samples/s +Step 27650/44000 | Loss: 2.9667 | LR: 1.57e-04 | 735.8ms/step | 27.2 samples/s +Step 27660/44000 | Loss: 2.9803 | LR: 1.56e-04 | 726.1ms/step | 27.5 samples/s +Step 27670/44000 | Loss: 2.9121 | LR: 1.56e-04 | 726.4ms/step | 27.5 samples/s +Step 27680/44000 | Loss: 2.9591 | LR: 1.56e-04 | 725.5ms/step | 27.6 samples/s +Step 27690/44000 | Loss: 2.9790 | LR: 1.56e-04 | 726.0ms/step | 27.5 samples/s +Step 27700/44000 | Loss: 2.9940 | LR: 1.56e-04 | 726.4ms/step | 27.5 samples/s +Step 27710/44000 | Loss: 2.9052 | LR: 1.56e-04 | 726.7ms/step | 27.5 samples/s +Step 27720/44000 | Loss: 2.9294 | LR: 1.55e-04 | 726.3ms/step | 27.5 samples/s +Step 27730/44000 | Loss: 2.8424 | LR: 1.55e-04 | 734.0ms/step | 27.2 samples/s +Step 27740/44000 | Loss: 2.9317 | LR: 1.55e-04 | 731.7ms/step | 27.3 samples/s +Step 27750/44000 | Loss: 2.9379 | LR: 1.55e-04 | 726.8ms/step | 27.5 samples/s +Validation at step 27750: Loss = 2.9278, Perplexity = 18.69 +Step 27760/44000 | Loss: 2.9553 | LR: 1.55e-04 | 951.1ms/step | 21.0 samples/s +Step 27770/44000 | Loss: 2.9821 | LR: 1.55e-04 | 726.6ms/step | 27.5 samples/s +Step 27780/44000 | Loss: 2.9545 | LR: 1.54e-04 | 726.7ms/step | 27.5 samples/s +Step 27790/44000 | Loss: 2.9171 | LR: 1.54e-04 | 726.0ms/step | 27.5 samples/s +Step 27800/44000 | Loss: 2.9138 | LR: 1.54e-04 | 726.1ms/step | 27.5 samples/s +Step 27810/44000 | Loss: 2.9260 | LR: 1.54e-04 | 726.0ms/step | 27.5 samples/s +Step 27820/44000 | Loss: 2.8998 | LR: 1.54e-04 | 739.6ms/step | 27.0 samples/s +Step 27830/44000 | Loss: 2.9709 | LR: 1.54e-04 | 725.9ms/step | 27.6 samples/s +Step 27840/44000 | Loss: 2.8593 | LR: 1.53e-04 | 726.2ms/step | 27.5 samples/s +Step 27850/44000 | Loss: 2.8907 | LR: 1.53e-04 | 726.4ms/step | 27.5 samples/s +Step 27860/44000 | Loss: 2.8619 | LR: 1.53e-04 | 726.1ms/step | 27.5 samples/s +Step 27870/44000 | Loss: 2.9857 | LR: 1.53e-04 | 725.8ms/step | 27.6 samples/s +Step 27880/44000 | Loss: 2.9426 | LR: 1.53e-04 | 726.1ms/step | 27.5 samples/s +Step 27890/44000 | Loss: 2.9803 | LR: 1.53e-04 | 726.2ms/step | 27.5 samples/s +Step 27900/44000 | Loss: 2.9162 | LR: 1.52e-04 | 726.6ms/step | 27.5 samples/s +Step 27910/44000 | Loss: 2.9853 | LR: 1.52e-04 | 736.2ms/step | 27.2 samples/s +Step 27920/44000 | Loss: 2.9974 | LR: 1.52e-04 | 726.2ms/step | 27.5 samples/s +Step 27930/44000 | Loss: 2.9273 | LR: 1.52e-04 | 726.0ms/step | 27.5 samples/s +Step 27940/44000 | Loss: 2.9226 | LR: 1.52e-04 | 726.4ms/step | 27.5 samples/s +Step 27950/44000 | Loss: 2.9852 | LR: 1.52e-04 | 726.1ms/step | 27.5 samples/s +Step 27960/44000 | Loss: 2.8969 | LR: 1.51e-04 | 725.6ms/step | 27.6 samples/s +Step 27970/44000 | Loss: 2.9638 | LR: 1.51e-04 | 725.7ms/step | 27.6 samples/s +Step 27980/44000 | Loss: 3.0405 | LR: 1.51e-04 | 725.4ms/step | 27.6 samples/s +Step 27990/44000 | Loss: 2.9512 | LR: 1.51e-04 | 730.6ms/step | 27.4 samples/s +Step 28000/44000 | Loss: 2.9189 | LR: 1.51e-04 | 730.6ms/step | 27.4 samples/s +Validation at step 28000: Loss = 2.9271, Perplexity = 18.67 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-28000 +Step 28010/44000 | Loss: 2.8954 | LR: 1.51e-04 | 1387.5ms/step | 14.4 samples/s +Step 28020/44000 | Loss: 2.9977 | LR: 1.50e-04 | 725.6ms/step | 27.6 samples/s +Step 28030/44000 | Loss: 2.9067 | LR: 1.50e-04 | 725.7ms/step | 27.6 samples/s +Step 28040/44000 | Loss: 2.9527 | LR: 1.50e-04 | 725.2ms/step | 27.6 samples/s +Step 28050/44000 | Loss: 2.9667 | LR: 1.50e-04 | 725.9ms/step | 27.6 samples/s +Step 28060/44000 | Loss: 2.9034 | LR: 1.50e-04 | 725.6ms/step | 27.6 samples/s +Step 28070/44000 | Loss: 2.9872 | LR: 1.50e-04 | 736.7ms/step | 27.1 samples/s +Step 28080/44000 | Loss: 2.9673 | LR: 1.49e-04 | 729.1ms/step | 27.4 samples/s +Step 28090/44000 | Loss: 2.9908 | LR: 1.49e-04 | 725.6ms/step | 27.6 samples/s +Step 28100/44000 | Loss: 2.9711 | LR: 1.49e-04 | 726.0ms/step | 27.5 samples/s +Step 28110/44000 | Loss: 2.9631 | LR: 1.49e-04 | 726.0ms/step | 27.5 samples/s +Step 28120/44000 | Loss: 2.9582 | LR: 1.49e-04 | 726.0ms/step | 27.5 samples/s +Step 28130/44000 | Loss: 2.9367 | LR: 1.49e-04 | 726.6ms/step | 27.5 samples/s +Step 28140/44000 | Loss: 2.9336 | LR: 1.48e-04 | 726.1ms/step | 27.5 samples/s +Step 28150/44000 | Loss: 2.9118 | LR: 1.48e-04 | 725.6ms/step | 27.6 samples/s +Step 28160/44000 | Loss: 2.8978 | LR: 1.48e-04 | 735.2ms/step | 27.2 samples/s +Step 28170/44000 | Loss: 2.8562 | LR: 1.48e-04 | 725.6ms/step | 27.6 samples/s +Step 28180/44000 | Loss: 2.8905 | LR: 1.48e-04 | 725.6ms/step | 27.6 samples/s +Step 28190/44000 | Loss: 2.9259 | LR: 1.48e-04 | 725.9ms/step | 27.6 samples/s +Step 28200/44000 | Loss: 2.9854 | LR: 1.47e-04 | 726.1ms/step | 27.5 samples/s +Step 28210/44000 | Loss: 2.8866 | LR: 1.47e-04 | 725.4ms/step | 27.6 samples/s +Step 28220/44000 | Loss: 2.8634 | LR: 1.47e-04 | 725.9ms/step | 27.6 samples/s +Step 28230/44000 | Loss: 2.9430 | LR: 1.47e-04 | 726.3ms/step | 27.5 samples/s +Step 28240/44000 | Loss: 2.8929 | LR: 1.47e-04 | 726.1ms/step | 27.5 samples/s +Step 28250/44000 | Loss: 2.9198 | LR: 1.47e-04 | 735.4ms/step | 27.2 samples/s +Validation at step 28250: Loss = 2.9256, Perplexity = 18.65 +Step 28260/44000 | Loss: 2.9490 | LR: 1.46e-04 | 949.9ms/step | 21.1 samples/s +Step 28270/44000 | Loss: 2.9266 | LR: 1.46e-04 | 725.7ms/step | 27.6 samples/s +Step 28280/44000 | Loss: 2.9707 | LR: 1.46e-04 | 725.6ms/step | 27.6 samples/s +Step 28290/44000 | Loss: 2.9257 | LR: 1.46e-04 | 726.0ms/step | 27.5 samples/s +Step 28300/44000 | Loss: 2.9851 | LR: 1.46e-04 | 726.4ms/step | 27.5 samples/s +Step 28310/44000 | Loss: 2.9805 | LR: 1.46e-04 | 725.4ms/step | 27.6 samples/s +Step 28320/44000 | Loss: 2.9521 | LR: 1.45e-04 | 725.5ms/step | 27.6 samples/s +Step 28330/44000 | Loss: 2.9967 | LR: 1.45e-04 | 738.5ms/step | 27.1 samples/s +Step 28340/44000 | Loss: 2.8449 | LR: 1.45e-04 | 725.4ms/step | 27.6 samples/s +Step 28350/44000 | Loss: 2.9232 | LR: 1.45e-04 | 725.8ms/step | 27.6 samples/s +Step 28360/44000 | Loss: 2.9658 | LR: 1.45e-04 | 725.5ms/step | 27.6 samples/s +Step 28370/44000 | Loss: 2.9873 | LR: 1.45e-04 | 725.8ms/step | 27.6 samples/s +Step 28380/44000 | Loss: 2.8877 | LR: 1.44e-04 | 725.9ms/step | 27.6 samples/s +Step 28390/44000 | Loss: 2.9268 | LR: 1.44e-04 | 726.0ms/step | 27.5 samples/s +Step 28400/44000 | Loss: 2.9065 | LR: 1.44e-04 | 725.8ms/step | 27.6 samples/s +Step 28410/44000 | Loss: 2.9710 | LR: 1.44e-04 | 725.6ms/step | 27.6 samples/s +Step 28420/44000 | Loss: 3.0097 | LR: 1.44e-04 | 735.6ms/step | 27.2 samples/s +Step 28430/44000 | Loss: 2.9569 | LR: 1.44e-04 | 725.7ms/step | 27.6 samples/s +Step 28440/44000 | Loss: 2.9429 | LR: 1.43e-04 | 725.5ms/step | 27.6 samples/s +Step 28450/44000 | Loss: 2.8940 | LR: 1.43e-04 | 725.8ms/step | 27.6 samples/s +Step 28460/44000 | Loss: 3.0231 | LR: 1.43e-04 | 725.9ms/step | 27.6 samples/s +Step 28470/44000 | Loss: 2.8508 | LR: 1.43e-04 | 726.0ms/step | 27.5 samples/s +Step 28480/44000 | Loss: 2.8898 | LR: 1.43e-04 | 725.8ms/step | 27.6 samples/s +Step 28490/44000 | Loss: 2.8362 | LR: 1.43e-04 | 725.8ms/step | 27.6 samples/s +Step 28500/44000 | Loss: 2.9344 | LR: 1.42e-04 | 726.0ms/step | 27.5 samples/s +Validation at step 28500: Loss = 2.9240, Perplexity = 18.62 +Step 28510/44000 | Loss: 2.9343 | LR: 1.42e-04 | 959.2ms/step | 20.9 samples/s +Step 28520/44000 | Loss: 2.9925 | LR: 1.42e-04 | 725.8ms/step | 27.6 samples/s +Step 28530/44000 | Loss: 2.9019 | LR: 1.42e-04 | 725.6ms/step | 27.6 samples/s +Step 28540/44000 | Loss: 2.9529 | LR: 1.42e-04 | 725.6ms/step | 27.6 samples/s +Step 28550/44000 | Loss: 2.9370 | LR: 1.42e-04 | 726.1ms/step | 27.5 samples/s +Step 28560/44000 | Loss: 2.9064 | LR: 1.41e-04 | 726.1ms/step | 27.5 samples/s +Step 28570/44000 | Loss: 2.9107 | LR: 1.41e-04 | 725.6ms/step | 27.6 samples/s +Step 28580/44000 | Loss: 2.9093 | LR: 1.41e-04 | 726.0ms/step | 27.5 samples/s +Step 28590/44000 | Loss: 2.9582 | LR: 1.41e-04 | 739.8ms/step | 27.0 samples/s +Step 28600/44000 | Loss: 2.8508 | LR: 1.41e-04 | 725.9ms/step | 27.6 samples/s +Step 28610/44000 | Loss: 2.9402 | LR: 1.41e-04 | 726.0ms/step | 27.5 samples/s +Step 28620/44000 | Loss: 2.9429 | LR: 1.40e-04 | 725.6ms/step | 27.6 samples/s +Step 28630/44000 | Loss: 2.8788 | LR: 1.40e-04 | 725.9ms/step | 27.6 samples/s +Step 28640/44000 | Loss: 2.9492 | LR: 1.40e-04 | 726.3ms/step | 27.5 samples/s +Step 28650/44000 | Loss: 2.9369 | LR: 1.40e-04 | 725.2ms/step | 27.6 samples/s +Step 28660/44000 | Loss: 2.9344 | LR: 1.40e-04 | 725.5ms/step | 27.6 samples/s +Step 28670/44000 | Loss: 2.9327 | LR: 1.40e-04 | 725.4ms/step | 27.6 samples/s +Step 28680/44000 | Loss: 2.8972 | LR: 1.39e-04 | 737.0ms/step | 27.1 samples/s +Step 28690/44000 | Loss: 2.9221 | LR: 1.39e-04 | 726.1ms/step | 27.5 samples/s +Step 28700/44000 | Loss: 2.8900 | LR: 1.39e-04 | 726.1ms/step | 27.5 samples/s +Step 28710/44000 | Loss: 2.8976 | LR: 1.39e-04 | 726.4ms/step | 27.5 samples/s +Step 28720/44000 | Loss: 2.9366 | LR: 1.39e-04 | 725.4ms/step | 27.6 samples/s +Step 28730/44000 | Loss: 2.9026 | LR: 1.39e-04 | 726.1ms/step | 27.5 samples/s +Step 28740/44000 | Loss: 2.9784 | LR: 1.38e-04 | 726.1ms/step | 27.5 samples/s +Step 28750/44000 | Loss: 2.8656 | LR: 1.38e-04 | 725.6ms/step | 27.6 samples/s +Validation at step 28750: Loss = 2.9234, Perplexity = 18.60 +Step 28760/44000 | Loss: 2.8528 | LR: 1.38e-04 | 964.9ms/step | 20.7 samples/s +Step 28770/44000 | Loss: 2.8746 | LR: 1.38e-04 | 726.0ms/step | 27.5 samples/s +Step 28780/44000 | Loss: 2.9310 | LR: 1.38e-04 | 727.5ms/step | 27.5 samples/s +Step 28790/44000 | Loss: 2.9711 | LR: 1.38e-04 | 725.7ms/step | 27.6 samples/s +Step 28800/44000 | Loss: 2.8472 | LR: 1.38e-04 | 725.8ms/step | 27.6 samples/s +Step 28810/44000 | Loss: 2.9754 | LR: 1.37e-04 | 725.2ms/step | 27.6 samples/s +Step 28820/44000 | Loss: 2.8927 | LR: 1.37e-04 | 725.6ms/step | 27.6 samples/s +Step 28830/44000 | Loss: 2.9451 | LR: 1.37e-04 | 725.8ms/step | 27.6 samples/s +Step 28840/44000 | Loss: 2.9702 | LR: 1.37e-04 | 725.8ms/step | 27.6 samples/s +Step 28850/44000 | Loss: 2.9364 | LR: 1.37e-04 | 735.4ms/step | 27.2 samples/s +Step 28860/44000 | Loss: 2.9313 | LR: 1.37e-04 | 725.7ms/step | 27.6 samples/s +Step 28870/44000 | Loss: 2.8899 | LR: 1.36e-04 | 725.9ms/step | 27.6 samples/s +Step 28880/44000 | Loss: 2.9045 | LR: 1.36e-04 | 725.7ms/step | 27.6 samples/s +Step 28890/44000 | Loss: 3.0287 | LR: 1.36e-04 | 726.2ms/step | 27.5 samples/s +Step 28900/44000 | Loss: 2.9423 | LR: 1.36e-04 | 725.5ms/step | 27.6 samples/s +Step 28910/44000 | Loss: 2.9486 | LR: 1.36e-04 | 725.6ms/step | 27.6 samples/s +Step 28920/44000 | Loss: 2.9031 | LR: 1.36e-04 | 725.4ms/step | 27.6 samples/s +Step 28930/44000 | Loss: 2.9651 | LR: 1.35e-04 | 725.2ms/step | 27.6 samples/s +Step 28940/44000 | Loss: 2.9031 | LR: 1.35e-04 | 735.5ms/step | 27.2 samples/s +Step 28950/44000 | Loss: 2.9168 | LR: 1.35e-04 | 725.6ms/step | 27.6 samples/s +Step 28960/44000 | Loss: 2.9419 | LR: 1.35e-04 | 726.1ms/step | 27.5 samples/s +Step 28970/44000 | Loss: 2.9238 | LR: 1.35e-04 | 726.1ms/step | 27.5 samples/s +Step 28980/44000 | Loss: 2.9280 | LR: 1.35e-04 | 726.1ms/step | 27.5 samples/s +Step 28990/44000 | Loss: 2.9320 | LR: 1.34e-04 | 725.9ms/step | 27.6 samples/s +Step 29000/44000 | Loss: 2.9265 | LR: 1.34e-04 | 725.9ms/step | 27.6 samples/s +Validation at step 29000: Loss = 2.9219, Perplexity = 18.58 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-29000 +Step 29010/44000 | Loss: 2.9298 | LR: 1.34e-04 | 1387.2ms/step | 14.4 samples/s +Step 29020/44000 | Loss: 2.9260 | LR: 1.34e-04 | 731.5ms/step | 27.3 samples/s +Step 29030/44000 | Loss: 2.9149 | LR: 1.34e-04 | 725.5ms/step | 27.6 samples/s +Step 29040/44000 | Loss: 2.9692 | LR: 1.34e-04 | 725.7ms/step | 27.6 samples/s +Step 29050/44000 | Loss: 2.9899 | LR: 1.33e-04 | 725.9ms/step | 27.6 samples/s +Step 29060/44000 | Loss: 2.9631 | LR: 1.33e-04 | 725.8ms/step | 27.6 samples/s +Step 29070/44000 | Loss: 2.9095 | LR: 1.33e-04 | 725.2ms/step | 27.6 samples/s +Step 29080/44000 | Loss: 2.8968 | LR: 1.33e-04 | 726.2ms/step | 27.5 samples/s +Step 29090/44000 | Loss: 2.8952 | LR: 1.33e-04 | 724.7ms/step | 27.6 samples/s +Step 29100/44000 | Loss: 2.9014 | LR: 1.33e-04 | 734.7ms/step | 27.2 samples/s +Step 29110/44000 | Loss: 2.9428 | LR: 1.33e-04 | 725.0ms/step | 27.6 samples/s +Step 29120/44000 | Loss: 2.9518 | LR: 1.32e-04 | 726.0ms/step | 27.5 samples/s +Step 29130/44000 | Loss: 2.9132 | LR: 1.32e-04 | 725.9ms/step | 27.6 samples/s +Step 29140/44000 | Loss: 2.8871 | LR: 1.32e-04 | 725.7ms/step | 27.6 samples/s +Step 29150/44000 | Loss: 2.9835 | LR: 1.32e-04 | 725.5ms/step | 27.6 samples/s +Step 29160/44000 | Loss: 2.9229 | LR: 1.32e-04 | 725.7ms/step | 27.6 samples/s +Step 29170/44000 | Loss: 2.9418 | LR: 1.32e-04 | 725.8ms/step | 27.6 samples/s +Step 29180/44000 | Loss: 2.9986 | LR: 1.31e-04 | 725.9ms/step | 27.6 samples/s +Step 29190/44000 | Loss: 2.9138 | LR: 1.31e-04 | 735.5ms/step | 27.2 samples/s +Step 29200/44000 | Loss: 2.8417 | LR: 1.31e-04 | 726.5ms/step | 27.5 samples/s +Step 29210/44000 | Loss: 2.9137 | LR: 1.31e-04 | 726.3ms/step | 27.5 samples/s +Step 29220/44000 | Loss: 2.9674 | LR: 1.31e-04 | 725.9ms/step | 27.6 samples/s +Step 29230/44000 | Loss: 2.8870 | LR: 1.31e-04 | 725.8ms/step | 27.6 samples/s +Step 29240/44000 | Loss: 2.9388 | LR: 1.30e-04 | 726.1ms/step | 27.5 samples/s +Step 29250/44000 | Loss: 2.8435 | LR: 1.30e-04 | 725.9ms/step | 27.6 samples/s +Validation at step 29250: Loss = 2.9214, Perplexity = 18.57 +Step 29260/44000 | Loss: 2.9867 | LR: 1.30e-04 | 951.1ms/step | 21.0 samples/s +Step 29270/44000 | Loss: 2.8401 | LR: 1.30e-04 | 731.6ms/step | 27.3 samples/s +Step 29280/44000 | Loss: 2.9986 | LR: 1.30e-04 | 731.5ms/step | 27.3 samples/s +Step 29290/44000 | Loss: 2.9444 | LR: 1.30e-04 | 726.2ms/step | 27.5 samples/s +Step 29300/44000 | Loss: 3.0431 | LR: 1.29e-04 | 726.3ms/step | 27.5 samples/s +Step 29310/44000 | Loss: 2.9540 | LR: 1.29e-04 | 726.0ms/step | 27.5 samples/s +Step 29320/44000 | Loss: 2.8745 | LR: 1.29e-04 | 727.0ms/step | 27.5 samples/s +Step 29330/44000 | Loss: 2.9011 | LR: 1.29e-04 | 726.0ms/step | 27.5 samples/s +Step 29340/44000 | Loss: 2.8590 | LR: 1.29e-04 | 725.9ms/step | 27.6 samples/s +Step 29350/44000 | Loss: 3.0680 | LR: 1.29e-04 | 726.4ms/step | 27.5 samples/s +Step 29360/44000 | Loss: 2.9012 | LR: 1.29e-04 | 735.8ms/step | 27.2 samples/s +Step 29370/44000 | Loss: 2.9788 | LR: 1.28e-04 | 725.5ms/step | 27.6 samples/s +Step 29380/44000 | Loss: 2.9107 | LR: 1.28e-04 | 726.3ms/step | 27.5 samples/s +Step 29390/44000 | Loss: 2.9748 | LR: 1.28e-04 | 726.3ms/step | 27.5 samples/s +Step 29400/44000 | Loss: 3.0054 | LR: 1.28e-04 | 726.1ms/step | 27.5 samples/s +Step 29410/44000 | Loss: 2.8845 | LR: 1.28e-04 | 725.9ms/step | 27.6 samples/s +Step 29420/44000 | Loss: 2.9415 | LR: 1.28e-04 | 726.1ms/step | 27.5 samples/s +Step 29430/44000 | Loss: 2.8562 | LR: 1.27e-04 | 726.3ms/step | 27.5 samples/s +Step 29440/44000 | Loss: 2.9805 | LR: 1.27e-04 | 725.5ms/step | 27.6 samples/s +Step 29450/44000 | Loss: 2.9075 | LR: 1.27e-04 | 738.8ms/step | 27.1 samples/s +Step 29460/44000 | Loss: 2.9913 | LR: 1.27e-04 | 725.7ms/step | 27.6 samples/s +Step 29470/44000 | Loss: 2.8580 | LR: 1.27e-04 | 725.6ms/step | 27.6 samples/s +Step 29480/44000 | Loss: 2.9705 | LR: 1.27e-04 | 726.2ms/step | 27.5 samples/s +Step 29490/44000 | Loss: 2.9723 | LR: 1.26e-04 | 725.8ms/step | 27.6 samples/s +Step 29500/44000 | Loss: 2.9266 | LR: 1.26e-04 | 725.9ms/step | 27.6 samples/s +Validation at step 29500: Loss = 2.9204, Perplexity = 18.55 +Step 29510/44000 | Loss: 2.9358 | LR: 1.26e-04 | 949.7ms/step | 21.1 samples/s +Step 29520/44000 | Loss: 2.9263 | LR: 1.26e-04 | 725.9ms/step | 27.6 samples/s +Step 29530/44000 | Loss: 2.9821 | LR: 1.26e-04 | 731.2ms/step | 27.4 samples/s +Step 29540/44000 | Loss: 2.9348 | LR: 1.26e-04 | 730.7ms/step | 27.4 samples/s +Step 29550/44000 | Loss: 2.9752 | LR: 1.26e-04 | 726.5ms/step | 27.5 samples/s +Step 29560/44000 | Loss: 2.9701 | LR: 1.25e-04 | 725.5ms/step | 27.6 samples/s +Step 29570/44000 | Loss: 2.9449 | LR: 1.25e-04 | 725.9ms/step | 27.6 samples/s +Step 29580/44000 | Loss: 2.8622 | LR: 1.25e-04 | 725.6ms/step | 27.6 samples/s +Step 29590/44000 | Loss: 2.8777 | LR: 1.25e-04 | 725.8ms/step | 27.6 samples/s +Step 29600/44000 | Loss: 2.9741 | LR: 1.25e-04 | 725.6ms/step | 27.6 samples/s +Step 29610/44000 | Loss: 2.9421 | LR: 1.25e-04 | 725.5ms/step | 27.6 samples/s +Step 29620/44000 | Loss: 2.9551 | LR: 1.24e-04 | 738.7ms/step | 27.1 samples/s +Step 29630/44000 | Loss: 2.9780 | LR: 1.24e-04 | 725.5ms/step | 27.6 samples/s +Step 29640/44000 | Loss: 2.9137 | LR: 1.24e-04 | 725.4ms/step | 27.6 samples/s +Step 29650/44000 | Loss: 2.8962 | LR: 1.24e-04 | 725.5ms/step | 27.6 samples/s +Step 29660/44000 | Loss: 2.9273 | LR: 1.24e-04 | 725.7ms/step | 27.6 samples/s +Step 29670/44000 | Loss: 2.9004 | LR: 1.24e-04 | 726.2ms/step | 27.5 samples/s +Step 29680/44000 | Loss: 2.9561 | LR: 1.23e-04 | 725.5ms/step | 27.6 samples/s +Step 29690/44000 | Loss: 2.9830 | LR: 1.23e-04 | 727.1ms/step | 27.5 samples/s +Step 29700/44000 | Loss: 2.8944 | LR: 1.23e-04 | 725.5ms/step | 27.6 samples/s +Step 29710/44000 | Loss: 2.8363 | LR: 1.23e-04 | 735.5ms/step | 27.2 samples/s +Step 29720/44000 | Loss: 2.9293 | LR: 1.23e-04 | 725.7ms/step | 27.6 samples/s +Step 29730/44000 | Loss: 2.9913 | LR: 1.23e-04 | 726.6ms/step | 27.5 samples/s +Step 29740/44000 | Loss: 2.9590 | LR: 1.23e-04 | 726.6ms/step | 27.5 samples/s +Step 29750/44000 | Loss: 2.8875 | LR: 1.22e-04 | 725.0ms/step | 27.6 samples/s +Validation at step 29750: Loss = 2.9196, Perplexity = 18.53 +Step 29760/44000 | Loss: 2.9224 | LR: 1.22e-04 | 950.4ms/step | 21.0 samples/s +Step 29770/44000 | Loss: 2.9168 | LR: 1.22e-04 | 725.7ms/step | 27.6 samples/s +Step 29780/44000 | Loss: 2.9056 | LR: 1.22e-04 | 725.6ms/step | 27.6 samples/s +Step 29790/44000 | Loss: 2.8710 | LR: 1.22e-04 | 739.5ms/step | 27.0 samples/s +Step 29800/44000 | Loss: 2.9142 | LR: 1.22e-04 | 725.9ms/step | 27.6 samples/s +Step 29810/44000 | Loss: 2.9861 | LR: 1.21e-04 | 727.4ms/step | 27.5 samples/s +Step 29820/44000 | Loss: 2.9953 | LR: 1.21e-04 | 725.9ms/step | 27.6 samples/s +Step 29830/44000 | Loss: 2.9418 | LR: 1.21e-04 | 725.6ms/step | 27.6 samples/s +Step 29840/44000 | Loss: 2.9358 | LR: 1.21e-04 | 725.9ms/step | 27.6 samples/s +Step 29850/44000 | Loss: 2.8876 | LR: 1.21e-04 | 726.0ms/step | 27.5 samples/s +Step 29860/44000 | Loss: 2.9602 | LR: 1.21e-04 | 725.5ms/step | 27.6 samples/s +Step 29870/44000 | Loss: 2.9611 | LR: 1.21e-04 | 726.3ms/step | 27.5 samples/s +Step 29880/44000 | Loss: 2.8161 | LR: 1.20e-04 | 734.8ms/step | 27.2 samples/s +Step 29890/44000 | Loss: 2.9348 | LR: 1.20e-04 | 725.6ms/step | 27.6 samples/s +Step 29900/44000 | Loss: 2.9489 | LR: 1.20e-04 | 725.9ms/step | 27.6 samples/s +Step 29910/44000 | Loss: 2.8965 | LR: 1.20e-04 | 727.0ms/step | 27.5 samples/s +Step 29920/44000 | Loss: 2.9100 | LR: 1.20e-04 | 725.6ms/step | 27.6 samples/s +Step 29930/44000 | Loss: 2.9658 | LR: 1.20e-04 | 725.9ms/step | 27.6 samples/s +Step 29940/44000 | Loss: 2.9621 | LR: 1.19e-04 | 726.5ms/step | 27.5 samples/s +Step 29950/44000 | Loss: 2.9949 | LR: 1.19e-04 | 725.3ms/step | 27.6 samples/s +Step 29960/44000 | Loss: 2.8793 | LR: 1.19e-04 | 724.9ms/step | 27.6 samples/s +Step 29970/44000 | Loss: 3.0212 | LR: 1.19e-04 | 735.8ms/step | 27.2 samples/s +Step 29980/44000 | Loss: 2.8333 | LR: 1.19e-04 | 725.7ms/step | 27.6 samples/s +Step 29990/44000 | Loss: 2.9252 | LR: 1.19e-04 | 725.8ms/step | 27.6 samples/s +Step 30000/44000 | Loss: 2.9051 | LR: 1.19e-04 | 726.1ms/step | 27.5 samples/s +Validation at step 30000: Loss = 2.9187, Perplexity = 18.52 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-30000 +Step 30010/44000 | Loss: 2.9000 | LR: 1.18e-04 | 1459.1ms/step | 13.7 samples/s +Step 30020/44000 | Loss: 2.8912 | LR: 1.18e-04 | 725.7ms/step | 27.6 samples/s +Step 30030/44000 | Loss: 2.9414 | LR: 1.18e-04 | 725.8ms/step | 27.6 samples/s +Step 30040/44000 | Loss: 2.9161 | LR: 1.18e-04 | 731.6ms/step | 27.3 samples/s +Step 30050/44000 | Loss: 2.9303 | LR: 1.18e-04 | 730.3ms/step | 27.4 samples/s +Step 30060/44000 | Loss: 2.8635 | LR: 1.18e-04 | 725.9ms/step | 27.6 samples/s +Step 30070/44000 | Loss: 2.9442 | LR: 1.17e-04 | 726.2ms/step | 27.5 samples/s +Step 30080/44000 | Loss: 2.9181 | LR: 1.17e-04 | 726.5ms/step | 27.5 samples/s +Step 30090/44000 | Loss: 2.9150 | LR: 1.17e-04 | 725.9ms/step | 27.6 samples/s +Step 30100/44000 | Loss: 2.9159 | LR: 1.17e-04 | 725.3ms/step | 27.6 samples/s +Step 30110/44000 | Loss: 2.9337 | LR: 1.17e-04 | 725.5ms/step | 27.6 samples/s +Step 30120/44000 | Loss: 2.9511 | LR: 1.17e-04 | 726.2ms/step | 27.5 samples/s +Step 30130/44000 | Loss: 2.9521 | LR: 1.17e-04 | 735.6ms/step | 27.2 samples/s +Step 30140/44000 | Loss: 2.8535 | LR: 1.16e-04 | 725.9ms/step | 27.6 samples/s +Step 30150/44000 | Loss: 2.9312 | LR: 1.16e-04 | 725.5ms/step | 27.6 samples/s +Step 30160/44000 | Loss: 2.9765 | LR: 1.16e-04 | 725.0ms/step | 27.6 samples/s +Step 30170/44000 | Loss: 2.9171 | LR: 1.16e-04 | 725.5ms/step | 27.6 samples/s +Step 30180/44000 | Loss: 2.8964 | LR: 1.16e-04 | 725.5ms/step | 27.6 samples/s +Step 30190/44000 | Loss: 2.8826 | LR: 1.16e-04 | 725.4ms/step | 27.6 samples/s +Step 30200/44000 | Loss: 3.0230 | LR: 1.15e-04 | 726.5ms/step | 27.5 samples/s +Step 30210/44000 | Loss: 2.8933 | LR: 1.15e-04 | 726.0ms/step | 27.5 samples/s +Step 30220/44000 | Loss: 2.9203 | LR: 1.15e-04 | 739.6ms/step | 27.0 samples/s +Step 30230/44000 | Loss: 2.9131 | LR: 1.15e-04 | 726.1ms/step | 27.5 samples/s +Step 30240/44000 | Loss: 2.9413 | LR: 1.15e-04 | 725.8ms/step | 27.6 samples/s +Step 30250/44000 | Loss: 2.9326 | LR: 1.15e-04 | 726.2ms/step | 27.5 samples/s +Validation at step 30250: Loss = 2.9175, Perplexity = 18.49 +Step 30260/44000 | Loss: 2.8698 | LR: 1.15e-04 | 950.9ms/step | 21.0 samples/s +Step 30270/44000 | Loss: 2.9317 | LR: 1.14e-04 | 726.2ms/step | 27.5 samples/s +Step 30280/44000 | Loss: 2.9400 | LR: 1.14e-04 | 726.1ms/step | 27.5 samples/s +Step 30290/44000 | Loss: 2.9373 | LR: 1.14e-04 | 725.7ms/step | 27.6 samples/s +Step 30300/44000 | Loss: 2.9380 | LR: 1.14e-04 | 735.7ms/step | 27.2 samples/s +Step 30310/44000 | Loss: 2.9623 | LR: 1.14e-04 | 725.5ms/step | 27.6 samples/s +Step 30320/44000 | Loss: 2.9872 | LR: 1.14e-04 | 725.9ms/step | 27.6 samples/s +Step 30330/44000 | Loss: 2.8544 | LR: 1.13e-04 | 725.5ms/step | 27.6 samples/s +Step 30340/44000 | Loss: 2.9213 | LR: 1.13e-04 | 725.5ms/step | 27.6 samples/s +Step 30350/44000 | Loss: 2.9317 | LR: 1.13e-04 | 726.0ms/step | 27.5 samples/s +Step 30360/44000 | Loss: 2.9604 | LR: 1.13e-04 | 726.0ms/step | 27.5 samples/s +Step 30370/44000 | Loss: 2.9604 | LR: 1.13e-04 | 726.0ms/step | 27.5 samples/s +Step 30380/44000 | Loss: 2.9454 | LR: 1.13e-04 | 725.9ms/step | 27.6 samples/s +Step 30390/44000 | Loss: 2.9256 | LR: 1.13e-04 | 735.1ms/step | 27.2 samples/s +Step 30400/44000 | Loss: 2.9012 | LR: 1.12e-04 | 725.9ms/step | 27.6 samples/s +Step 30410/44000 | Loss: 2.9024 | LR: 1.12e-04 | 725.6ms/step | 27.6 samples/s +Step 30420/44000 | Loss: 2.9390 | LR: 1.12e-04 | 725.8ms/step | 27.6 samples/s +Step 30430/44000 | Loss: 2.8795 | LR: 1.12e-04 | 725.6ms/step | 27.6 samples/s +Step 30440/44000 | Loss: 2.8876 | LR: 1.12e-04 | 725.3ms/step | 27.6 samples/s +Step 30450/44000 | Loss: 2.9000 | LR: 1.12e-04 | 725.8ms/step | 27.6 samples/s +Step 30460/44000 | Loss: 2.8336 | LR: 1.11e-04 | 725.4ms/step | 27.6 samples/s +Step 30470/44000 | Loss: 2.8999 | LR: 1.11e-04 | 725.6ms/step | 27.6 samples/s +Step 30480/44000 | Loss: 2.8560 | LR: 1.11e-04 | 737.5ms/step | 27.1 samples/s +Step 30490/44000 | Loss: 2.9488 | LR: 1.11e-04 | 725.5ms/step | 27.6 samples/s +Step 30500/44000 | Loss: 2.9169 | LR: 1.11e-04 | 725.9ms/step | 27.6 samples/s +Validation at step 30500: Loss = 2.9166, Perplexity = 18.48 +Step 30510/44000 | Loss: 2.9305 | LR: 1.11e-04 | 950.6ms/step | 21.0 samples/s +Step 30520/44000 | Loss: 2.9727 | LR: 1.11e-04 | 725.7ms/step | 27.6 samples/s +Step 30530/44000 | Loss: 2.9483 | LR: 1.10e-04 | 725.2ms/step | 27.6 samples/s +Step 30540/44000 | Loss: 2.9244 | LR: 1.10e-04 | 725.7ms/step | 27.6 samples/s +Step 30550/44000 | Loss: 2.8802 | LR: 1.10e-04 | 726.0ms/step | 27.5 samples/s +Step 30560/44000 | Loss: 2.9291 | LR: 1.10e-04 | 736.6ms/step | 27.2 samples/s +Step 30570/44000 | Loss: 2.9131 | LR: 1.10e-04 | 725.4ms/step | 27.6 samples/s +Step 30580/44000 | Loss: 2.9114 | LR: 1.10e-04 | 725.7ms/step | 27.6 samples/s +Step 30590/44000 | Loss: 2.9465 | LR: 1.10e-04 | 725.9ms/step | 27.6 samples/s +Step 30600/44000 | Loss: 2.8999 | LR: 1.09e-04 | 725.8ms/step | 27.6 samples/s +Step 30610/44000 | Loss: 2.9740 | LR: 1.09e-04 | 725.7ms/step | 27.6 samples/s +Step 30620/44000 | Loss: 2.9149 | LR: 1.09e-04 | 725.9ms/step | 27.6 samples/s +Step 30630/44000 | Loss: 2.8780 | LR: 1.09e-04 | 726.2ms/step | 27.5 samples/s +Step 30640/44000 | Loss: 2.9486 | LR: 1.09e-04 | 725.6ms/step | 27.6 samples/s +Step 30650/44000 | Loss: 2.8729 | LR: 1.09e-04 | 735.4ms/step | 27.2 samples/s +Step 30660/44000 | Loss: 2.8889 | LR: 1.08e-04 | 726.6ms/step | 27.5 samples/s +Step 30670/44000 | Loss: 2.9621 | LR: 1.08e-04 | 725.8ms/step | 27.6 samples/s +Step 30680/44000 | Loss: 2.9244 | LR: 1.08e-04 | 725.9ms/step | 27.6 samples/s +Step 30690/44000 | Loss: 2.8813 | LR: 1.08e-04 | 725.9ms/step | 27.6 samples/s +Step 30700/44000 | Loss: 2.8508 | LR: 1.08e-04 | 726.3ms/step | 27.5 samples/s +Step 30710/44000 | Loss: 2.9822 | LR: 1.08e-04 | 726.5ms/step | 27.5 samples/s +Step 30720/44000 | Loss: 2.9116 | LR: 1.08e-04 | 726.9ms/step | 27.5 samples/s +Step 30730/44000 | Loss: 2.9678 | LR: 1.07e-04 | 727.1ms/step | 27.5 samples/s +Step 30740/44000 | Loss: 2.9870 | LR: 1.07e-04 | 738.0ms/step | 27.1 samples/s +Step 30750/44000 | Loss: 2.9715 | LR: 1.07e-04 | 725.6ms/step | 27.6 samples/s +Validation at step 30750: Loss = 2.9159, Perplexity = 18.47 +Step 30760/44000 | Loss: 2.9912 | LR: 1.07e-04 | 953.7ms/step | 21.0 samples/s +Step 30770/44000 | Loss: 2.9845 | LR: 1.07e-04 | 727.0ms/step | 27.5 samples/s +Step 30780/44000 | Loss: 2.9305 | LR: 1.07e-04 | 725.9ms/step | 27.6 samples/s +Step 30790/44000 | Loss: 2.9489 | LR: 1.07e-04 | 726.7ms/step | 27.5 samples/s +Step 30800/44000 | Loss: 2.9113 | LR: 1.06e-04 | 725.2ms/step | 27.6 samples/s +Step 30810/44000 | Loss: 2.9590 | LR: 1.06e-04 | 725.5ms/step | 27.6 samples/s +Step 30820/44000 | Loss: 2.8891 | LR: 1.06e-04 | 735.3ms/step | 27.2 samples/s +Step 30830/44000 | Loss: 2.9724 | LR: 1.06e-04 | 726.2ms/step | 27.5 samples/s +Step 30840/44000 | Loss: 2.9314 | LR: 1.06e-04 | 725.7ms/step | 27.6 samples/s +Step 30850/44000 | Loss: 2.9640 | LR: 1.06e-04 | 725.5ms/step | 27.6 samples/s +Step 30860/44000 | Loss: 2.9227 | LR: 1.05e-04 | 726.0ms/step | 27.5 samples/s +Step 30870/44000 | Loss: 2.9759 | LR: 1.05e-04 | 725.7ms/step | 27.6 samples/s +Step 30880/44000 | Loss: 2.9144 | LR: 1.05e-04 | 725.7ms/step | 27.6 samples/s +Step 30890/44000 | Loss: 2.8723 | LR: 1.05e-04 | 725.6ms/step | 27.6 samples/s +Step 30900/44000 | Loss: 2.9202 | LR: 1.05e-04 | 726.4ms/step | 27.5 samples/s +Step 30910/44000 | Loss: 2.8862 | LR: 1.05e-04 | 736.5ms/step | 27.2 samples/s +Step 30920/44000 | Loss: 2.8888 | LR: 1.05e-04 | 725.8ms/step | 27.6 samples/s +Step 30930/44000 | Loss: 2.9314 | LR: 1.04e-04 | 725.7ms/step | 27.6 samples/s +Step 30940/44000 | Loss: 2.8557 | LR: 1.04e-04 | 725.6ms/step | 27.6 samples/s +Step 30950/44000 | Loss: 2.9458 | LR: 1.04e-04 | 725.1ms/step | 27.6 samples/s +Step 30960/44000 | Loss: 2.8959 | LR: 1.04e-04 | 725.5ms/step | 27.6 samples/s +Step 30970/44000 | Loss: 2.9879 | LR: 1.04e-04 | 725.0ms/step | 27.6 samples/s +Step 30980/44000 | Loss: 2.9714 | LR: 1.04e-04 | 725.2ms/step | 27.6 samples/s +Step 30990/44000 | Loss: 2.9266 | LR: 1.04e-04 | 733.8ms/step | 27.3 samples/s +Step 31000/44000 | Loss: 2.8743 | LR: 1.03e-04 | 731.1ms/step | 27.4 samples/s +Validation at step 31000: Loss = 2.9151, Perplexity = 18.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-31000 +Step 31010/44000 | Loss: 2.9383 | LR: 1.03e-04 | 1476.2ms/step | 13.5 samples/s +Step 31020/44000 | Loss: 2.9018 | LR: 1.03e-04 | 725.0ms/step | 27.6 samples/s +Step 31030/44000 | Loss: 2.9317 | LR: 1.03e-04 | 725.2ms/step | 27.6 samples/s +Step 31040/44000 | Loss: 2.9607 | LR: 1.03e-04 | 725.7ms/step | 27.6 samples/s +Step 31050/44000 | Loss: 2.8841 | LR: 1.03e-04 | 725.1ms/step | 27.6 samples/s +Step 31060/44000 | Loss: 2.9032 | LR: 1.03e-04 | 725.4ms/step | 27.6 samples/s +Step 31070/44000 | Loss: 2.9262 | LR: 1.02e-04 | 738.7ms/step | 27.1 samples/s +Step 31080/44000 | Loss: 2.9564 | LR: 1.02e-04 | 725.2ms/step | 27.6 samples/s +Step 31090/44000 | Loss: 2.9141 | LR: 1.02e-04 | 725.3ms/step | 27.6 samples/s +Step 31100/44000 | Loss: 2.9833 | LR: 1.02e-04 | 725.3ms/step | 27.6 samples/s +Step 31110/44000 | Loss: 2.9086 | LR: 1.02e-04 | 725.6ms/step | 27.6 samples/s +Step 31120/44000 | Loss: 2.9489 | LR: 1.02e-04 | 725.0ms/step | 27.6 samples/s +Step 31130/44000 | Loss: 2.9590 | LR: 1.02e-04 | 725.7ms/step | 27.6 samples/s +Step 31140/44000 | Loss: 2.8709 | LR: 1.01e-04 | 725.3ms/step | 27.6 samples/s +Step 31150/44000 | Loss: 2.9477 | LR: 1.01e-04 | 725.9ms/step | 27.6 samples/s +Step 31160/44000 | Loss: 2.8875 | LR: 1.01e-04 | 737.3ms/step | 27.1 samples/s +Step 31170/44000 | Loss: 2.9949 | LR: 1.01e-04 | 725.2ms/step | 27.6 samples/s +Step 31180/44000 | Loss: 2.9497 | LR: 1.01e-04 | 725.6ms/step | 27.6 samples/s +Step 31190/44000 | Loss: 2.9185 | LR: 1.01e-04 | 725.2ms/step | 27.6 samples/s +Step 31200/44000 | Loss: 2.9770 | LR: 1.00e-04 | 725.3ms/step | 27.6 samples/s +Step 31210/44000 | Loss: 2.9165 | LR: 1.00e-04 | 725.0ms/step | 27.6 samples/s +Step 31220/44000 | Loss: 3.0002 | LR: 1.00e-04 | 725.1ms/step | 27.6 samples/s +Step 31230/44000 | Loss: 2.8938 | LR: 1.00e-04 | 724.8ms/step | 27.6 samples/s +Step 31240/44000 | Loss: 2.9213 | LR: 9.99e-05 | 725.2ms/step | 27.6 samples/s +Step 31250/44000 | Loss: 2.9002 | LR: 9.98e-05 | 734.8ms/step | 27.2 samples/s +Validation at step 31250: Loss = 2.9145, Perplexity = 18.44 +Step 31260/44000 | Loss: 2.8943 | LR: 9.96e-05 | 948.3ms/step | 21.1 samples/s +Step 31270/44000 | Loss: 2.9540 | LR: 9.95e-05 | 724.9ms/step | 27.6 samples/s +Step 31280/44000 | Loss: 2.9667 | LR: 9.93e-05 | 725.1ms/step | 27.6 samples/s +Step 31290/44000 | Loss: 2.9464 | LR: 9.92e-05 | 726.1ms/step | 27.5 samples/s +Step 31300/44000 | Loss: 2.8595 | LR: 9.90e-05 | 726.3ms/step | 27.5 samples/s +Step 31310/44000 | Loss: 2.8908 | LR: 9.89e-05 | 725.7ms/step | 27.6 samples/s +Step 31320/44000 | Loss: 2.8816 | LR: 9.88e-05 | 725.1ms/step | 27.6 samples/s +Step 31330/44000 | Loss: 2.8478 | LR: 9.86e-05 | 739.5ms/step | 27.0 samples/s +Step 31340/44000 | Loss: 2.9314 | LR: 9.85e-05 | 725.6ms/step | 27.6 samples/s +Step 31350/44000 | Loss: 2.9150 | LR: 9.83e-05 | 725.6ms/step | 27.6 samples/s +Step 31360/44000 | Loss: 2.9719 | LR: 9.82e-05 | 725.3ms/step | 27.6 samples/s +Step 31370/44000 | Loss: 3.0234 | LR: 9.80e-05 | 724.6ms/step | 27.6 samples/s +Step 31380/44000 | Loss: 2.9264 | LR: 9.79e-05 | 724.7ms/step | 27.6 samples/s +Step 31390/44000 | Loss: 2.9037 | LR: 9.77e-05 | 725.2ms/step | 27.6 samples/s +Step 31400/44000 | Loss: 2.9925 | LR: 9.76e-05 | 725.5ms/step | 27.6 samples/s +Step 31410/44000 | Loss: 2.9347 | LR: 9.75e-05 | 725.7ms/step | 27.6 samples/s +Step 31420/44000 | Loss: 2.9458 | LR: 9.73e-05 | 735.8ms/step | 27.2 samples/s +Step 31430/44000 | Loss: 2.9491 | LR: 9.72e-05 | 726.2ms/step | 27.5 samples/s +Step 31440/44000 | Loss: 2.9001 | LR: 9.70e-05 | 726.3ms/step | 27.5 samples/s +Step 31450/44000 | Loss: 2.9606 | LR: 9.69e-05 | 724.8ms/step | 27.6 samples/s +Step 31460/44000 | Loss: 2.9393 | LR: 9.67e-05 | 725.0ms/step | 27.6 samples/s +Step 31470/44000 | Loss: 2.9220 | LR: 9.66e-05 | 725.6ms/step | 27.6 samples/s +Step 31480/44000 | Loss: 2.9143 | LR: 9.65e-05 | 725.1ms/step | 27.6 samples/s +Step 31490/44000 | Loss: 2.9580 | LR: 9.63e-05 | 725.5ms/step | 27.6 samples/s +Step 31500/44000 | Loss: 2.9047 | LR: 9.62e-05 | 730.6ms/step | 27.4 samples/s +Validation at step 31500: Loss = 2.9140, Perplexity = 18.43 +Step 31510/44000 | Loss: 2.9644 | LR: 9.60e-05 | 955.6ms/step | 20.9 samples/s +Step 31520/44000 | Loss: 2.8935 | LR: 9.59e-05 | 725.6ms/step | 27.6 samples/s +Step 31530/44000 | Loss: 2.9558 | LR: 9.57e-05 | 725.1ms/step | 27.6 samples/s +Step 31540/44000 | Loss: 2.9098 | LR: 9.56e-05 | 725.5ms/step | 27.6 samples/s +Step 31550/44000 | Loss: 2.9989 | LR: 9.54e-05 | 725.9ms/step | 27.6 samples/s +Step 31560/44000 | Loss: 2.8797 | LR: 9.53e-05 | 726.4ms/step | 27.5 samples/s +Step 31570/44000 | Loss: 2.8886 | LR: 9.52e-05 | 726.5ms/step | 27.5 samples/s +Step 31580/44000 | Loss: 2.9659 | LR: 9.50e-05 | 727.0ms/step | 27.5 samples/s +Step 31590/44000 | Loss: 2.9368 | LR: 9.49e-05 | 740.0ms/step | 27.0 samples/s +Step 31600/44000 | Loss: 2.8839 | LR: 9.47e-05 | 726.4ms/step | 27.5 samples/s +Step 31610/44000 | Loss: 2.9673 | LR: 9.46e-05 | 724.9ms/step | 27.6 samples/s +Step 31620/44000 | Loss: 2.9775 | LR: 9.45e-05 | 725.5ms/step | 27.6 samples/s +Step 31630/44000 | Loss: 2.9708 | LR: 9.43e-05 | 726.3ms/step | 27.5 samples/s +Step 31640/44000 | Loss: 2.9238 | LR: 9.42e-05 | 726.5ms/step | 27.5 samples/s +Step 31650/44000 | Loss: 2.8963 | LR: 9.40e-05 | 726.0ms/step | 27.5 samples/s +Step 31660/44000 | Loss: 2.9477 | LR: 9.39e-05 | 725.6ms/step | 27.6 samples/s +Step 31670/44000 | Loss: 2.9270 | LR: 9.37e-05 | 725.5ms/step | 27.6 samples/s +Step 31680/44000 | Loss: 2.9312 | LR: 9.36e-05 | 735.1ms/step | 27.2 samples/s +Step 31690/44000 | Loss: 2.9425 | LR: 9.35e-05 | 725.5ms/step | 27.6 samples/s +Step 31700/44000 | Loss: 2.8156 | LR: 9.33e-05 | 725.9ms/step | 27.6 samples/s +Step 31710/44000 | Loss: 2.9348 | LR: 9.32e-05 | 725.3ms/step | 27.6 samples/s +Step 31720/44000 | Loss: 2.9606 | LR: 9.30e-05 | 725.5ms/step | 27.6 samples/s +Step 31730/44000 | Loss: 2.8831 | LR: 9.29e-05 | 726.3ms/step | 27.5 samples/s +Step 31740/44000 | Loss: 2.9558 | LR: 9.28e-05 | 725.8ms/step | 27.6 samples/s +Step 31750/44000 | Loss: 2.9752 | LR: 9.26e-05 | 725.9ms/step | 27.6 samples/s +Validation at step 31750: Loss = 2.9135, Perplexity = 18.42 +Step 31760/44000 | Loss: 2.9436 | LR: 9.25e-05 | 960.9ms/step | 20.8 samples/s +Step 31770/44000 | Loss: 2.9079 | LR: 9.23e-05 | 725.8ms/step | 27.6 samples/s +Step 31780/44000 | Loss: 2.8737 | LR: 9.22e-05 | 725.6ms/step | 27.6 samples/s +Step 31790/44000 | Loss: 2.8797 | LR: 9.20e-05 | 725.8ms/step | 27.6 samples/s +Step 31800/44000 | Loss: 2.8823 | LR: 9.19e-05 | 726.6ms/step | 27.5 samples/s +Step 31810/44000 | Loss: 2.9214 | LR: 9.18e-05 | 725.3ms/step | 27.6 samples/s +Step 31820/44000 | Loss: 2.8978 | LR: 9.16e-05 | 725.7ms/step | 27.6 samples/s +Step 31830/44000 | Loss: 2.9974 | LR: 9.15e-05 | 726.0ms/step | 27.5 samples/s +Step 31840/44000 | Loss: 2.9410 | LR: 9.13e-05 | 725.9ms/step | 27.6 samples/s +Step 31850/44000 | Loss: 2.8854 | LR: 9.12e-05 | 736.8ms/step | 27.1 samples/s +Step 31860/44000 | Loss: 2.9603 | LR: 9.11e-05 | 725.8ms/step | 27.6 samples/s +Step 31870/44000 | Loss: 2.9494 | LR: 9.09e-05 | 725.9ms/step | 27.6 samples/s +Step 31880/44000 | Loss: 2.9257 | LR: 9.08e-05 | 726.0ms/step | 27.5 samples/s +Step 31890/44000 | Loss: 2.9487 | LR: 9.06e-05 | 725.8ms/step | 27.6 samples/s +Step 31900/44000 | Loss: 2.9483 | LR: 9.05e-05 | 725.5ms/step | 27.6 samples/s +Step 31910/44000 | Loss: 2.9296 | LR: 9.04e-05 | 725.8ms/step | 27.6 samples/s +Step 31920/44000 | Loss: 2.9407 | LR: 9.02e-05 | 725.8ms/step | 27.6 samples/s +Step 31930/44000 | Loss: 2.9163 | LR: 9.01e-05 | 725.6ms/step | 27.6 samples/s +Step 31940/44000 | Loss: 3.0067 | LR: 8.99e-05 | 734.6ms/step | 27.2 samples/s +Step 31950/44000 | Loss: 2.8751 | LR: 8.98e-05 | 725.4ms/step | 27.6 samples/s +Step 31960/44000 | Loss: 2.9417 | LR: 8.97e-05 | 725.4ms/step | 27.6 samples/s +Step 31970/44000 | Loss: 2.9522 | LR: 8.95e-05 | 725.4ms/step | 27.6 samples/s +Step 31980/44000 | Loss: 2.9681 | LR: 8.94e-05 | 725.8ms/step | 27.6 samples/s +Step 31990/44000 | Loss: 2.9657 | LR: 8.92e-05 | 725.9ms/step | 27.6 samples/s +Step 32000/44000 | Loss: 2.8581 | LR: 8.91e-05 | 726.4ms/step | 27.5 samples/s +Validation at step 32000: Loss = 2.9128, Perplexity = 18.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-32000 +Step 32010/44000 | Loss: 2.8710 | LR: 8.90e-05 | 1464.2ms/step | 13.7 samples/s +Step 32020/44000 | Loss: 2.9588 | LR: 8.88e-05 | 731.7ms/step | 27.3 samples/s +Step 32030/44000 | Loss: 2.9203 | LR: 8.87e-05 | 724.9ms/step | 27.6 samples/s +Step 32040/44000 | Loss: 2.8731 | LR: 8.86e-05 | 724.9ms/step | 27.6 samples/s +Step 32050/44000 | Loss: 2.9512 | LR: 8.84e-05 | 725.0ms/step | 27.6 samples/s +Step 32060/44000 | Loss: 2.9624 | LR: 8.83e-05 | 725.4ms/step | 27.6 samples/s +Step 32070/44000 | Loss: 2.8927 | LR: 8.81e-05 | 726.1ms/step | 27.5 samples/s +Step 32080/44000 | Loss: 2.9538 | LR: 8.80e-05 | 725.5ms/step | 27.6 samples/s +Step 32090/44000 | Loss: 2.9493 | LR: 8.79e-05 | 725.9ms/step | 27.6 samples/s +Step 32100/44000 | Loss: 2.8649 | LR: 8.77e-05 | 735.8ms/step | 27.2 samples/s +Step 32110/44000 | Loss: 2.8931 | LR: 8.76e-05 | 726.2ms/step | 27.5 samples/s +Step 32120/44000 | Loss: 2.9866 | LR: 8.74e-05 | 726.1ms/step | 27.5 samples/s +Step 32130/44000 | Loss: 2.9299 | LR: 8.73e-05 | 725.9ms/step | 27.6 samples/s +Step 32140/44000 | Loss: 2.9132 | LR: 8.72e-05 | 726.2ms/step | 27.5 samples/s +Step 32150/44000 | Loss: 2.9582 | LR: 8.70e-05 | 725.9ms/step | 27.6 samples/s +Step 32160/44000 | Loss: 2.9130 | LR: 8.69e-05 | 726.0ms/step | 27.5 samples/s +Step 32170/44000 | Loss: 2.9088 | LR: 8.68e-05 | 726.2ms/step | 27.5 samples/s +Step 32180/44000 | Loss: 2.9356 | LR: 8.66e-05 | 725.9ms/step | 27.6 samples/s +Step 32190/44000 | Loss: 2.9260 | LR: 8.65e-05 | 735.1ms/step | 27.2 samples/s +Step 32200/44000 | Loss: 2.8460 | LR: 8.63e-05 | 725.8ms/step | 27.6 samples/s +Step 32210/44000 | Loss: 2.9057 | LR: 8.62e-05 | 725.1ms/step | 27.6 samples/s +Step 32220/44000 | Loss: 2.9137 | LR: 8.61e-05 | 726.0ms/step | 27.5 samples/s +Step 32230/44000 | Loss: 2.9118 | LR: 8.59e-05 | 726.0ms/step | 27.5 samples/s +Step 32240/44000 | Loss: 2.9563 | LR: 8.58e-05 | 726.8ms/step | 27.5 samples/s +Step 32250/44000 | Loss: 2.9207 | LR: 8.57e-05 | 726.2ms/step | 27.5 samples/s +Validation at step 32250: Loss = 2.9124, Perplexity = 18.40 +Step 32260/44000 | Loss: 2.9378 | LR: 8.55e-05 | 951.7ms/step | 21.0 samples/s +Step 32270/44000 | Loss: 2.9510 | LR: 8.54e-05 | 739.0ms/step | 27.1 samples/s +Step 32280/44000 | Loss: 2.8479 | LR: 8.53e-05 | 728.8ms/step | 27.4 samples/s +Step 32290/44000 | Loss: 2.9097 | LR: 8.51e-05 | 725.8ms/step | 27.6 samples/s +Step 32300/44000 | Loss: 2.9526 | LR: 8.50e-05 | 725.3ms/step | 27.6 samples/s +Step 32310/44000 | Loss: 2.9421 | LR: 8.48e-05 | 725.5ms/step | 27.6 samples/s +Step 32320/44000 | Loss: 2.9139 | LR: 8.47e-05 | 725.9ms/step | 27.6 samples/s +Step 32330/44000 | Loss: 2.8911 | LR: 8.46e-05 | 726.0ms/step | 27.5 samples/s +Step 32340/44000 | Loss: 2.8810 | LR: 8.44e-05 | 725.2ms/step | 27.6 samples/s +Step 32350/44000 | Loss: 2.8690 | LR: 8.43e-05 | 726.2ms/step | 27.5 samples/s +Step 32360/44000 | Loss: 2.9650 | LR: 8.42e-05 | 735.2ms/step | 27.2 samples/s +Step 32370/44000 | Loss: 2.9026 | LR: 8.40e-05 | 725.9ms/step | 27.6 samples/s +Step 32380/44000 | Loss: 2.9733 | LR: 8.39e-05 | 726.5ms/step | 27.5 samples/s +Step 32390/44000 | Loss: 2.9603 | LR: 8.38e-05 | 724.9ms/step | 27.6 samples/s +Step 32400/44000 | Loss: 2.9788 | LR: 8.36e-05 | 726.6ms/step | 27.5 samples/s +Step 32410/44000 | Loss: 2.9600 | LR: 8.35e-05 | 725.6ms/step | 27.6 samples/s +Step 32420/44000 | Loss: 2.9465 | LR: 8.33e-05 | 726.8ms/step | 27.5 samples/s +Step 32430/44000 | Loss: 2.9465 | LR: 8.32e-05 | 725.8ms/step | 27.6 samples/s +Step 32440/44000 | Loss: 3.0227 | LR: 8.31e-05 | 725.3ms/step | 27.6 samples/s +Step 32450/44000 | Loss: 2.8748 | LR: 8.29e-05 | 740.0ms/step | 27.0 samples/s +Step 32460/44000 | Loss: 2.9181 | LR: 8.28e-05 | 725.6ms/step | 27.6 samples/s +Step 32470/44000 | Loss: 2.9180 | LR: 8.27e-05 | 725.2ms/step | 27.6 samples/s +Step 32480/44000 | Loss: 2.9608 | LR: 8.25e-05 | 725.5ms/step | 27.6 samples/s +Step 32490/44000 | Loss: 2.9473 | LR: 8.24e-05 | 725.6ms/step | 27.6 samples/s +Step 32500/44000 | Loss: 2.8685 | LR: 8.23e-05 | 725.6ms/step | 27.6 samples/s +Validation at step 32500: Loss = 2.9120, Perplexity = 18.39 +Step 32510/44000 | Loss: 2.9319 | LR: 8.21e-05 | 949.7ms/step | 21.1 samples/s +Step 32520/44000 | Loss: 2.9244 | LR: 8.20e-05 | 725.9ms/step | 27.6 samples/s +Step 32530/44000 | Loss: 2.8991 | LR: 8.19e-05 | 734.8ms/step | 27.2 samples/s +Step 32540/44000 | Loss: 2.9778 | LR: 8.17e-05 | 725.6ms/step | 27.6 samples/s +Step 32550/44000 | Loss: 2.9057 | LR: 8.16e-05 | 726.5ms/step | 27.5 samples/s +Step 32560/44000 | Loss: 2.9021 | LR: 8.15e-05 | 725.6ms/step | 27.6 samples/s +Step 32570/44000 | Loss: 2.9536 | LR: 8.13e-05 | 725.6ms/step | 27.6 samples/s +Step 32580/44000 | Loss: 2.8908 | LR: 8.12e-05 | 725.9ms/step | 27.6 samples/s +Step 32590/44000 | Loss: 2.9340 | LR: 8.11e-05 | 726.3ms/step | 27.5 samples/s +Step 32600/44000 | Loss: 2.9134 | LR: 8.09e-05 | 726.8ms/step | 27.5 samples/s +Step 32610/44000 | Loss: 2.9405 | LR: 8.08e-05 | 726.4ms/step | 27.5 samples/s +Step 32620/44000 | Loss: 2.9051 | LR: 8.07e-05 | 739.7ms/step | 27.0 samples/s +Step 32630/44000 | Loss: 2.9063 | LR: 8.05e-05 | 725.5ms/step | 27.6 samples/s +Step 32640/44000 | Loss: 2.9194 | LR: 8.04e-05 | 726.8ms/step | 27.5 samples/s +Step 32650/44000 | Loss: 2.9092 | LR: 8.03e-05 | 725.8ms/step | 27.6 samples/s +Step 32660/44000 | Loss: 2.9689 | LR: 8.01e-05 | 726.0ms/step | 27.5 samples/s +Step 32670/44000 | Loss: 2.8958 | LR: 8.00e-05 | 725.4ms/step | 27.6 samples/s +Step 32680/44000 | Loss: 2.9062 | LR: 7.99e-05 | 725.8ms/step | 27.6 samples/s +Step 32690/44000 | Loss: 2.8838 | LR: 7.97e-05 | 725.6ms/step | 27.6 samples/s +Step 32700/44000 | Loss: 2.8817 | LR: 7.96e-05 | 730.1ms/step | 27.4 samples/s +Step 32710/44000 | Loss: 2.8615 | LR: 7.95e-05 | 732.8ms/step | 27.3 samples/s +Step 32720/44000 | Loss: 2.9580 | LR: 7.93e-05 | 725.4ms/step | 27.6 samples/s +Step 32730/44000 | Loss: 2.9284 | LR: 7.92e-05 | 725.5ms/step | 27.6 samples/s +Step 32740/44000 | Loss: 2.8398 | LR: 7.91e-05 | 725.0ms/step | 27.6 samples/s +Step 32750/44000 | Loss: 2.9174 | LR: 7.89e-05 | 725.2ms/step | 27.6 samples/s +Validation at step 32750: Loss = 2.9114, Perplexity = 18.38 +Step 32760/44000 | Loss: 2.8960 | LR: 7.88e-05 | 950.8ms/step | 21.0 samples/s +Step 32770/44000 | Loss: 2.9586 | LR: 7.87e-05 | 725.6ms/step | 27.6 samples/s +Step 32780/44000 | Loss: 2.9689 | LR: 7.85e-05 | 725.7ms/step | 27.6 samples/s +Step 32790/44000 | Loss: 2.9190 | LR: 7.84e-05 | 735.2ms/step | 27.2 samples/s +Step 32800/44000 | Loss: 2.8875 | LR: 7.83e-05 | 726.2ms/step | 27.5 samples/s +Step 32810/44000 | Loss: 2.9304 | LR: 7.81e-05 | 725.8ms/step | 27.6 samples/s +Step 32820/44000 | Loss: 2.8785 | LR: 7.80e-05 | 726.2ms/step | 27.5 samples/s +Step 32830/44000 | Loss: 2.8808 | LR: 7.79e-05 | 726.4ms/step | 27.5 samples/s +Step 32840/44000 | Loss: 2.8712 | LR: 7.77e-05 | 726.2ms/step | 27.5 samples/s +Step 32850/44000 | Loss: 2.9690 | LR: 7.76e-05 | 726.3ms/step | 27.5 samples/s +Step 32860/44000 | Loss: 2.8853 | LR: 7.75e-05 | 725.9ms/step | 27.6 samples/s +Step 32870/44000 | Loss: 2.8941 | LR: 7.73e-05 | 725.8ms/step | 27.6 samples/s +Step 32880/44000 | Loss: 2.8995 | LR: 7.72e-05 | 738.4ms/step | 27.1 samples/s +Step 32890/44000 | Loss: 2.8446 | LR: 7.71e-05 | 725.5ms/step | 27.6 samples/s +Step 32900/44000 | Loss: 2.8929 | LR: 7.70e-05 | 725.9ms/step | 27.6 samples/s +Step 32910/44000 | Loss: 2.8909 | LR: 7.68e-05 | 725.1ms/step | 27.6 samples/s +Step 32920/44000 | Loss: 2.9184 | LR: 7.67e-05 | 725.5ms/step | 27.6 samples/s +Step 32930/44000 | Loss: 2.9330 | LR: 7.66e-05 | 726.5ms/step | 27.5 samples/s +Step 32940/44000 | Loss: 2.9790 | LR: 7.64e-05 | 726.4ms/step | 27.5 samples/s +Step 32950/44000 | Loss: 2.8894 | LR: 7.63e-05 | 725.2ms/step | 27.6 samples/s +Step 32960/44000 | Loss: 2.9525 | LR: 7.62e-05 | 730.4ms/step | 27.4 samples/s +Step 32970/44000 | Loss: 2.9685 | LR: 7.60e-05 | 729.8ms/step | 27.4 samples/s +Step 32980/44000 | Loss: 2.8989 | LR: 7.59e-05 | 725.2ms/step | 27.6 samples/s +Step 32990/44000 | Loss: 2.8763 | LR: 7.58e-05 | 725.3ms/step | 27.6 samples/s +Step 33000/44000 | Loss: 2.9693 | LR: 7.56e-05 | 725.3ms/step | 27.6 samples/s +Validation at step 33000: Loss = 2.9109, Perplexity = 18.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-33000 +Step 33010/44000 | Loss: 2.9439 | LR: 7.55e-05 | 1475.2ms/step | 13.6 samples/s +Step 33020/44000 | Loss: 2.8762 | LR: 7.54e-05 | 724.4ms/step | 27.6 samples/s +Step 33030/44000 | Loss: 3.0151 | LR: 7.53e-05 | 725.5ms/step | 27.6 samples/s +Step 33040/44000 | Loss: 2.9286 | LR: 7.51e-05 | 734.7ms/step | 27.2 samples/s +Step 33050/44000 | Loss: 2.9267 | LR: 7.50e-05 | 725.4ms/step | 27.6 samples/s +Step 33060/44000 | Loss: 2.9166 | LR: 7.49e-05 | 725.9ms/step | 27.6 samples/s +Step 33070/44000 | Loss: 2.9581 | LR: 7.47e-05 | 726.1ms/step | 27.5 samples/s +Step 33080/44000 | Loss: 2.9991 | LR: 7.46e-05 | 725.5ms/step | 27.6 samples/s +Step 33090/44000 | Loss: 2.9000 | LR: 7.45e-05 | 725.7ms/step | 27.6 samples/s +Step 33100/44000 | Loss: 2.9686 | LR: 7.44e-05 | 725.8ms/step | 27.6 samples/s +Step 33110/44000 | Loss: 3.0162 | LR: 7.42e-05 | 726.0ms/step | 27.5 samples/s +Step 33120/44000 | Loss: 2.9422 | LR: 7.41e-05 | 726.2ms/step | 27.5 samples/s +Step 33130/44000 | Loss: 2.9606 | LR: 7.40e-05 | 737.8ms/step | 27.1 samples/s +Step 33140/44000 | Loss: 2.8441 | LR: 7.38e-05 | 726.0ms/step | 27.5 samples/s +Step 33150/44000 | Loss: 2.8875 | LR: 7.37e-05 | 725.9ms/step | 27.6 samples/s +Step 33160/44000 | Loss: 2.9411 | LR: 7.36e-05 | 725.8ms/step | 27.6 samples/s +Step 33170/44000 | Loss: 2.9146 | LR: 7.34e-05 | 725.9ms/step | 27.6 samples/s +Step 33180/44000 | Loss: 2.9440 | LR: 7.33e-05 | 726.6ms/step | 27.5 samples/s +Step 33190/44000 | Loss: 2.9311 | LR: 7.32e-05 | 726.0ms/step | 27.5 samples/s +Step 33200/44000 | Loss: 2.8343 | LR: 7.31e-05 | 726.0ms/step | 27.5 samples/s +Step 33210/44000 | Loss: 2.8897 | LR: 7.29e-05 | 726.1ms/step | 27.5 samples/s +Step 33220/44000 | Loss: 2.9470 | LR: 7.28e-05 | 736.6ms/step | 27.2 samples/s +Step 33230/44000 | Loss: 2.9171 | LR: 7.27e-05 | 725.8ms/step | 27.6 samples/s +Step 33240/44000 | Loss: 2.8880 | LR: 7.26e-05 | 726.3ms/step | 27.5 samples/s +Step 33250/44000 | Loss: 2.9075 | LR: 7.24e-05 | 726.3ms/step | 27.5 samples/s +Validation at step 33250: Loss = 2.9104, Perplexity = 18.36 +Step 33260/44000 | Loss: 2.9181 | LR: 7.23e-05 | 951.1ms/step | 21.0 samples/s +Step 33270/44000 | Loss: 2.9203 | LR: 7.22e-05 | 726.4ms/step | 27.5 samples/s +Step 33280/44000 | Loss: 2.9034 | LR: 7.20e-05 | 726.0ms/step | 27.5 samples/s +Step 33290/44000 | Loss: 2.9357 | LR: 7.19e-05 | 725.6ms/step | 27.6 samples/s +Step 33300/44000 | Loss: 2.9047 | LR: 7.18e-05 | 736.2ms/step | 27.2 samples/s +Step 33310/44000 | Loss: 2.9288 | LR: 7.17e-05 | 725.6ms/step | 27.6 samples/s +Step 33320/44000 | Loss: 2.9010 | LR: 7.15e-05 | 725.5ms/step | 27.6 samples/s +Step 33330/44000 | Loss: 2.9518 | LR: 7.14e-05 | 726.9ms/step | 27.5 samples/s +Step 33340/44000 | Loss: 2.8971 | LR: 7.13e-05 | 728.5ms/step | 27.5 samples/s +Step 33350/44000 | Loss: 2.9277 | LR: 7.11e-05 | 727.6ms/step | 27.5 samples/s +Step 33360/44000 | Loss: 2.8848 | LR: 7.10e-05 | 725.8ms/step | 27.6 samples/s +Step 33370/44000 | Loss: 2.9045 | LR: 7.09e-05 | 725.6ms/step | 27.6 samples/s +Step 33380/44000 | Loss: 2.9311 | LR: 7.08e-05 | 725.5ms/step | 27.6 samples/s +Step 33390/44000 | Loss: 2.8874 | LR: 7.06e-05 | 740.1ms/step | 27.0 samples/s +Step 33400/44000 | Loss: 2.9088 | LR: 7.05e-05 | 725.2ms/step | 27.6 samples/s +Step 33410/44000 | Loss: 2.9051 | LR: 7.04e-05 | 725.1ms/step | 27.6 samples/s +Step 33420/44000 | Loss: 2.8674 | LR: 7.03e-05 | 725.8ms/step | 27.6 samples/s +Step 33430/44000 | Loss: 2.9580 | LR: 7.01e-05 | 726.6ms/step | 27.5 samples/s +Step 33440/44000 | Loss: 2.9079 | LR: 7.00e-05 | 726.5ms/step | 27.5 samples/s +Step 33450/44000 | Loss: 2.9353 | LR: 6.99e-05 | 726.5ms/step | 27.5 samples/s +Step 33460/44000 | Loss: 2.9175 | LR: 6.98e-05 | 726.3ms/step | 27.5 samples/s +Step 33470/44000 | Loss: 2.8956 | LR: 6.96e-05 | 729.8ms/step | 27.4 samples/s +Step 33480/44000 | Loss: 2.9321 | LR: 6.95e-05 | 729.9ms/step | 27.4 samples/s +Step 33490/44000 | Loss: 2.8868 | LR: 6.94e-05 | 725.6ms/step | 27.6 samples/s +Step 33500/44000 | Loss: 2.8917 | LR: 6.93e-05 | 726.1ms/step | 27.5 samples/s +Validation at step 33500: Loss = 2.9104, Perplexity = 18.36 +Step 33510/44000 | Loss: 2.9575 | LR: 6.91e-05 | 949.7ms/step | 21.1 samples/s +Step 33520/44000 | Loss: 2.9049 | LR: 6.90e-05 | 725.3ms/step | 27.6 samples/s +Step 33530/44000 | Loss: 2.8736 | LR: 6.89e-05 | 726.0ms/step | 27.5 samples/s +Step 33540/44000 | Loss: 2.9295 | LR: 6.88e-05 | 725.9ms/step | 27.6 samples/s +Step 33550/44000 | Loss: 2.8849 | LR: 6.86e-05 | 725.1ms/step | 27.6 samples/s +Step 33560/44000 | Loss: 2.8760 | LR: 6.85e-05 | 738.3ms/step | 27.1 samples/s +Step 33570/44000 | Loss: 2.8831 | LR: 6.84e-05 | 725.1ms/step | 27.6 samples/s +Step 33580/44000 | Loss: 2.9447 | LR: 6.83e-05 | 725.4ms/step | 27.6 samples/s +Step 33590/44000 | Loss: 2.9099 | LR: 6.81e-05 | 725.8ms/step | 27.6 samples/s +Step 33600/44000 | Loss: 2.9616 | LR: 6.80e-05 | 725.8ms/step | 27.6 samples/s +Step 33610/44000 | Loss: 2.8601 | LR: 6.79e-05 | 725.5ms/step | 27.6 samples/s +Step 33620/44000 | Loss: 2.8866 | LR: 6.78e-05 | 725.9ms/step | 27.6 samples/s +Step 33630/44000 | Loss: 2.9637 | LR: 6.76e-05 | 726.4ms/step | 27.5 samples/s +Step 33640/44000 | Loss: 3.0277 | LR: 6.75e-05 | 726.4ms/step | 27.5 samples/s +Step 33650/44000 | Loss: 2.9422 | LR: 6.74e-05 | 735.0ms/step | 27.2 samples/s +Step 33660/44000 | Loss: 2.9290 | LR: 6.73e-05 | 726.2ms/step | 27.5 samples/s +Step 33670/44000 | Loss: 2.9003 | LR: 6.71e-05 | 725.8ms/step | 27.6 samples/s +Step 33680/44000 | Loss: 2.8833 | LR: 6.70e-05 | 725.4ms/step | 27.6 samples/s +Step 33690/44000 | Loss: 2.8933 | LR: 6.69e-05 | 726.1ms/step | 27.5 samples/s +Step 33700/44000 | Loss: 2.8816 | LR: 6.68e-05 | 726.0ms/step | 27.5 samples/s +Step 33710/44000 | Loss: 2.9297 | LR: 6.66e-05 | 726.6ms/step | 27.5 samples/s +Step 33720/44000 | Loss: 2.8064 | LR: 6.65e-05 | 725.9ms/step | 27.6 samples/s +Step 33730/44000 | Loss: 2.8750 | LR: 6.64e-05 | 730.4ms/step | 27.4 samples/s +Step 33740/44000 | Loss: 2.8903 | LR: 6.63e-05 | 734.0ms/step | 27.2 samples/s +Step 33750/44000 | Loss: 2.8429 | LR: 6.61e-05 | 725.6ms/step | 27.6 samples/s +Validation at step 33750: Loss = 2.9097, Perplexity = 18.35 +Step 33760/44000 | Loss: 2.8369 | LR: 6.60e-05 | 951.0ms/step | 21.0 samples/s +Step 33770/44000 | Loss: 2.9346 | LR: 6.59e-05 | 725.7ms/step | 27.6 samples/s +Step 33780/44000 | Loss: 2.9196 | LR: 6.58e-05 | 726.2ms/step | 27.5 samples/s +Step 33790/44000 | Loss: 2.8969 | LR: 6.57e-05 | 725.5ms/step | 27.6 samples/s +Step 33800/44000 | Loss: 2.9625 | LR: 6.55e-05 | 726.1ms/step | 27.5 samples/s +Step 33810/44000 | Loss: 2.9429 | LR: 6.54e-05 | 725.8ms/step | 27.6 samples/s +Step 33820/44000 | Loss: 2.9106 | LR: 6.53e-05 | 735.2ms/step | 27.2 samples/s +Step 33830/44000 | Loss: 2.9456 | LR: 6.52e-05 | 725.7ms/step | 27.6 samples/s +Step 33840/44000 | Loss: 2.9459 | LR: 6.50e-05 | 725.8ms/step | 27.6 samples/s +Step 33850/44000 | Loss: 2.9171 | LR: 6.49e-05 | 726.4ms/step | 27.5 samples/s +Step 33860/44000 | Loss: 2.9708 | LR: 6.48e-05 | 725.9ms/step | 27.6 samples/s +Step 33870/44000 | Loss: 2.8950 | LR: 6.47e-05 | 725.9ms/step | 27.6 samples/s +Step 33880/44000 | Loss: 2.9347 | LR: 6.46e-05 | 725.9ms/step | 27.6 samples/s +Step 33890/44000 | Loss: 2.9010 | LR: 6.44e-05 | 725.9ms/step | 27.6 samples/s +Step 33900/44000 | Loss: 2.9163 | LR: 6.43e-05 | 733.8ms/step | 27.3 samples/s +Step 33910/44000 | Loss: 2.9283 | LR: 6.42e-05 | 731.5ms/step | 27.3 samples/s +Step 33920/44000 | Loss: 2.8500 | LR: 6.41e-05 | 725.5ms/step | 27.6 samples/s +Step 33930/44000 | Loss: 2.9351 | LR: 6.40e-05 | 725.5ms/step | 27.6 samples/s +Step 33940/44000 | Loss: 2.9008 | LR: 6.38e-05 | 725.5ms/step | 27.6 samples/s +Step 33950/44000 | Loss: 2.9709 | LR: 6.37e-05 | 725.5ms/step | 27.6 samples/s +Step 33960/44000 | Loss: 2.8697 | LR: 6.36e-05 | 725.6ms/step | 27.6 samples/s +Step 33970/44000 | Loss: 2.9175 | LR: 6.35e-05 | 725.9ms/step | 27.6 samples/s +Step 33980/44000 | Loss: 2.8522 | LR: 6.33e-05 | 725.5ms/step | 27.6 samples/s +Step 33990/44000 | Loss: 2.9987 | LR: 6.32e-05 | 730.4ms/step | 27.4 samples/s +Step 34000/44000 | Loss: 2.9419 | LR: 6.31e-05 | 730.6ms/step | 27.4 samples/s +Validation at step 34000: Loss = 2.9097, Perplexity = 18.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-34000 +Step 34010/44000 | Loss: 2.8796 | LR: 6.30e-05 | 1395.1ms/step | 14.3 samples/s +Step 34020/44000 | Loss: 2.9605 | LR: 6.29e-05 | 725.4ms/step | 27.6 samples/s +Step 34030/44000 | Loss: 2.8941 | LR: 6.27e-05 | 725.4ms/step | 27.6 samples/s +Step 34040/44000 | Loss: 2.8941 | LR: 6.26e-05 | 726.1ms/step | 27.5 samples/s +Step 34050/44000 | Loss: 2.9263 | LR: 6.25e-05 | 726.0ms/step | 27.5 samples/s +Step 34060/44000 | Loss: 2.9255 | LR: 6.24e-05 | 726.0ms/step | 27.5 samples/s +Step 34070/44000 | Loss: 2.8680 | LR: 6.23e-05 | 737.2ms/step | 27.1 samples/s +Step 34080/44000 | Loss: 2.8494 | LR: 6.21e-05 | 725.6ms/step | 27.6 samples/s +Step 34090/44000 | Loss: 2.9461 | LR: 6.20e-05 | 725.8ms/step | 27.6 samples/s +Step 34100/44000 | Loss: 2.9355 | LR: 6.19e-05 | 725.5ms/step | 27.6 samples/s +Step 34110/44000 | Loss: 2.8717 | LR: 6.18e-05 | 725.6ms/step | 27.6 samples/s +Step 34120/44000 | Loss: 2.9130 | LR: 6.17e-05 | 725.3ms/step | 27.6 samples/s +Step 34130/44000 | Loss: 2.9040 | LR: 6.15e-05 | 725.5ms/step | 27.6 samples/s +Step 34140/44000 | Loss: 2.8503 | LR: 6.14e-05 | 725.4ms/step | 27.6 samples/s +Step 34150/44000 | Loss: 2.9460 | LR: 6.13e-05 | 726.2ms/step | 27.5 samples/s +Step 34160/44000 | Loss: 2.9698 | LR: 6.12e-05 | 735.6ms/step | 27.2 samples/s +Step 34170/44000 | Loss: 2.9141 | LR: 6.11e-05 | 725.6ms/step | 27.6 samples/s +Step 34180/44000 | Loss: 2.8804 | LR: 6.09e-05 | 725.7ms/step | 27.6 samples/s +Step 34190/44000 | Loss: 2.9299 | LR: 6.08e-05 | 725.9ms/step | 27.6 samples/s +Step 34200/44000 | Loss: 2.9279 | LR: 6.07e-05 | 725.4ms/step | 27.6 samples/s +Step 34210/44000 | Loss: 2.8974 | LR: 6.06e-05 | 725.8ms/step | 27.6 samples/s +Step 34220/44000 | Loss: 2.9021 | LR: 6.05e-05 | 726.1ms/step | 27.5 samples/s +Step 34230/44000 | Loss: 2.8961 | LR: 6.04e-05 | 725.6ms/step | 27.6 samples/s +Step 34240/44000 | Loss: 2.8669 | LR: 6.02e-05 | 725.7ms/step | 27.6 samples/s +Step 34250/44000 | Loss: 2.9443 | LR: 6.01e-05 | 735.4ms/step | 27.2 samples/s +Validation at step 34250: Loss = 2.9091, Perplexity = 18.34 +Step 34260/44000 | Loss: 2.9241 | LR: 6.00e-05 | 949.3ms/step | 21.1 samples/s +Step 34270/44000 | Loss: 2.9113 | LR: 5.99e-05 | 725.9ms/step | 27.6 samples/s +Step 34280/44000 | Loss: 2.9134 | LR: 5.98e-05 | 725.8ms/step | 27.6 samples/s +Step 34290/44000 | Loss: 2.8423 | LR: 5.96e-05 | 725.5ms/step | 27.6 samples/s +Step 34300/44000 | Loss: 2.9488 | LR: 5.95e-05 | 725.6ms/step | 27.6 samples/s +Step 34310/44000 | Loss: 2.8996 | LR: 5.94e-05 | 726.1ms/step | 27.5 samples/s +Step 34320/44000 | Loss: 2.8553 | LR: 5.93e-05 | 727.0ms/step | 27.5 samples/s +Step 34330/44000 | Loss: 2.9294 | LR: 5.92e-05 | 735.7ms/step | 27.2 samples/s +Step 34340/44000 | Loss: 2.8929 | LR: 5.91e-05 | 726.5ms/step | 27.5 samples/s +Step 34350/44000 | Loss: 2.8920 | LR: 5.89e-05 | 725.7ms/step | 27.6 samples/s +Step 34360/44000 | Loss: 2.9943 | LR: 5.88e-05 | 725.7ms/step | 27.6 samples/s +Step 34370/44000 | Loss: 2.9396 | LR: 5.87e-05 | 726.5ms/step | 27.5 samples/s +Step 34380/44000 | Loss: 3.0004 | LR: 5.86e-05 | 725.8ms/step | 27.6 samples/s +Step 34390/44000 | Loss: 2.8942 | LR: 5.85e-05 | 725.9ms/step | 27.6 samples/s +Step 34400/44000 | Loss: 2.9435 | LR: 5.84e-05 | 725.8ms/step | 27.6 samples/s +Step 34410/44000 | Loss: 2.9815 | LR: 5.82e-05 | 726.1ms/step | 27.5 samples/s +Step 34420/44000 | Loss: 2.9464 | LR: 5.81e-05 | 734.6ms/step | 27.2 samples/s +Step 34430/44000 | Loss: 2.8974 | LR: 5.80e-05 | 726.1ms/step | 27.5 samples/s +Step 34440/44000 | Loss: 2.8820 | LR: 5.79e-05 | 726.2ms/step | 27.5 samples/s +Step 34450/44000 | Loss: 2.9113 | LR: 5.78e-05 | 726.0ms/step | 27.5 samples/s +Step 34460/44000 | Loss: 2.8989 | LR: 5.77e-05 | 726.0ms/step | 27.5 samples/s +Step 34470/44000 | Loss: 2.9352 | LR: 5.75e-05 | 726.0ms/step | 27.5 samples/s +Step 34480/44000 | Loss: 2.9515 | LR: 5.74e-05 | 725.9ms/step | 27.6 samples/s +Step 34490/44000 | Loss: 2.9291 | LR: 5.73e-05 | 725.6ms/step | 27.6 samples/s +Step 34500/44000 | Loss: 2.9301 | LR: 5.72e-05 | 730.5ms/step | 27.4 samples/s +Validation at step 34500: Loss = 2.9089, Perplexity = 18.34 +Step 34510/44000 | Loss: 2.9643 | LR: 5.71e-05 | 954.7ms/step | 20.9 samples/s +Step 34520/44000 | Loss: 2.8774 | LR: 5.70e-05 | 726.2ms/step | 27.5 samples/s +Step 34530/44000 | Loss: 2.9069 | LR: 5.69e-05 | 726.2ms/step | 27.5 samples/s +Step 34540/44000 | Loss: 2.9745 | LR: 5.67e-05 | 726.2ms/step | 27.5 samples/s +Step 34550/44000 | Loss: 2.9264 | LR: 5.66e-05 | 726.0ms/step | 27.5 samples/s +Step 34560/44000 | Loss: 2.9729 | LR: 5.65e-05 | 726.0ms/step | 27.5 samples/s +Step 34570/44000 | Loss: 2.9122 | LR: 5.64e-05 | 726.2ms/step | 27.5 samples/s +Step 34580/44000 | Loss: 2.9540 | LR: 5.63e-05 | 726.0ms/step | 27.5 samples/s +Step 34590/44000 | Loss: 2.9498 | LR: 5.62e-05 | 736.0ms/step | 27.2 samples/s +Step 34600/44000 | Loss: 2.9571 | LR: 5.60e-05 | 725.8ms/step | 27.6 samples/s +Step 34610/44000 | Loss: 2.9426 | LR: 5.59e-05 | 726.2ms/step | 27.5 samples/s +Step 34620/44000 | Loss: 2.8656 | LR: 5.58e-05 | 725.9ms/step | 27.6 samples/s +Step 34630/44000 | Loss: 2.8651 | LR: 5.57e-05 | 726.2ms/step | 27.5 samples/s +Step 34640/44000 | Loss: 2.9330 | LR: 5.56e-05 | 726.4ms/step | 27.5 samples/s +Step 34650/44000 | Loss: 2.8929 | LR: 5.55e-05 | 726.1ms/step | 27.5 samples/s +Step 34660/44000 | Loss: 3.0086 | LR: 5.54e-05 | 725.7ms/step | 27.6 samples/s +Step 34670/44000 | Loss: 2.9131 | LR: 5.52e-05 | 726.4ms/step | 27.5 samples/s +Step 34680/44000 | Loss: 2.9268 | LR: 5.51e-05 | 737.4ms/step | 27.1 samples/s +Step 34690/44000 | Loss: 2.9272 | LR: 5.50e-05 | 726.5ms/step | 27.5 samples/s +Step 34700/44000 | Loss: 2.8939 | LR: 5.49e-05 | 725.9ms/step | 27.6 samples/s +Step 34710/44000 | Loss: 2.8884 | LR: 5.48e-05 | 725.5ms/step | 27.6 samples/s +Step 34720/44000 | Loss: 2.8517 | LR: 5.47e-05 | 726.3ms/step | 27.5 samples/s +Step 34730/44000 | Loss: 2.9241 | LR: 5.46e-05 | 725.6ms/step | 27.6 samples/s +Step 34740/44000 | Loss: 2.9216 | LR: 5.45e-05 | 725.6ms/step | 27.6 samples/s +Step 34750/44000 | Loss: 2.8996 | LR: 5.43e-05 | 726.1ms/step | 27.5 samples/s +Validation at step 34750: Loss = 2.9087, Perplexity = 18.33 +Step 34760/44000 | Loss: 2.8814 | LR: 5.42e-05 | 961.9ms/step | 20.8 samples/s +Step 34770/44000 | Loss: 2.9525 | LR: 5.41e-05 | 726.2ms/step | 27.5 samples/s +Step 34780/44000 | Loss: 2.9163 | LR: 5.40e-05 | 726.0ms/step | 27.5 samples/s +Step 34790/44000 | Loss: 2.9202 | LR: 5.39e-05 | 726.4ms/step | 27.5 samples/s +Step 34800/44000 | Loss: 2.9847 | LR: 5.38e-05 | 726.0ms/step | 27.5 samples/s +Step 34810/44000 | Loss: 2.9834 | LR: 5.37e-05 | 725.3ms/step | 27.6 samples/s +Step 34820/44000 | Loss: 2.8764 | LR: 5.36e-05 | 725.5ms/step | 27.6 samples/s +Step 34830/44000 | Loss: 2.9095 | LR: 5.34e-05 | 725.5ms/step | 27.6 samples/s +Step 34840/44000 | Loss: 2.8919 | LR: 5.33e-05 | 726.0ms/step | 27.5 samples/s +Step 34850/44000 | Loss: 2.9422 | LR: 5.32e-05 | 734.8ms/step | 27.2 samples/s +Step 34860/44000 | Loss: 2.8792 | LR: 5.31e-05 | 726.5ms/step | 27.5 samples/s +Step 34870/44000 | Loss: 2.8877 | LR: 5.30e-05 | 725.6ms/step | 27.6 samples/s +Step 34880/44000 | Loss: 2.9696 | LR: 5.29e-05 | 727.0ms/step | 27.5 samples/s +Step 34890/44000 | Loss: 2.9209 | LR: 5.28e-05 | 726.3ms/step | 27.5 samples/s +Step 34900/44000 | Loss: 2.9130 | LR: 5.27e-05 | 725.6ms/step | 27.6 samples/s +Step 34910/44000 | Loss: 2.8328 | LR: 5.25e-05 | 726.2ms/step | 27.5 samples/s +Step 34920/44000 | Loss: 2.8830 | LR: 5.24e-05 | 727.3ms/step | 27.5 samples/s +Step 34930/44000 | Loss: 2.8547 | LR: 5.23e-05 | 731.2ms/step | 27.4 samples/s +Step 34940/44000 | Loss: 2.9463 | LR: 5.22e-05 | 732.2ms/step | 27.3 samples/s +Step 34950/44000 | Loss: 2.8742 | LR: 5.21e-05 | 726.8ms/step | 27.5 samples/s +Step 34960/44000 | Loss: 2.8645 | LR: 5.20e-05 | 726.4ms/step | 27.5 samples/s +Step 34970/44000 | Loss: 2.8437 | LR: 5.19e-05 | 725.2ms/step | 27.6 samples/s +Step 34980/44000 | Loss: 2.9537 | LR: 5.18e-05 | 726.0ms/step | 27.5 samples/s +Step 34990/44000 | Loss: 2.9664 | LR: 5.17e-05 | 725.2ms/step | 27.6 samples/s +Step 35000/44000 | Loss: 2.8476 | LR: 5.15e-05 | 725.6ms/step | 27.6 samples/s +Validation at step 35000: Loss = 2.9085, Perplexity = 18.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-35000 +Step 35010/44000 | Loss: 2.9000 | LR: 5.14e-05 | 1456.6ms/step | 13.7 samples/s +Step 35020/44000 | Loss: 2.9873 | LR: 5.13e-05 | 729.1ms/step | 27.4 samples/s +Step 35030/44000 | Loss: 2.8593 | LR: 5.12e-05 | 725.2ms/step | 27.6 samples/s +Step 35040/44000 | Loss: 2.9961 | LR: 5.11e-05 | 724.8ms/step | 27.6 samples/s +Step 35050/44000 | Loss: 2.8915 | LR: 5.10e-05 | 724.8ms/step | 27.6 samples/s +Step 35060/44000 | Loss: 2.8994 | LR: 5.09e-05 | 725.0ms/step | 27.6 samples/s +Step 35070/44000 | Loss: 2.9523 | LR: 5.08e-05 | 725.1ms/step | 27.6 samples/s +Step 35080/44000 | Loss: 2.9199 | LR: 5.07e-05 | 725.4ms/step | 27.6 samples/s +Step 35090/44000 | Loss: 2.8609 | LR: 5.06e-05 | 725.3ms/step | 27.6 samples/s +Step 35100/44000 | Loss: 2.9101 | LR: 5.04e-05 | 734.5ms/step | 27.2 samples/s +Step 35110/44000 | Loss: 2.9450 | LR: 5.03e-05 | 725.0ms/step | 27.6 samples/s +Step 35120/44000 | Loss: 2.8889 | LR: 5.02e-05 | 724.7ms/step | 27.6 samples/s +Step 35130/44000 | Loss: 2.9068 | LR: 5.01e-05 | 724.9ms/step | 27.6 samples/s +Step 35140/44000 | Loss: 2.8943 | LR: 5.00e-05 | 725.1ms/step | 27.6 samples/s +Step 35150/44000 | Loss: 2.9166 | LR: 4.99e-05 | 725.0ms/step | 27.6 samples/s +Step 35160/44000 | Loss: 2.9189 | LR: 4.98e-05 | 725.5ms/step | 27.6 samples/s +Step 35170/44000 | Loss: 2.9458 | LR: 4.97e-05 | 725.4ms/step | 27.6 samples/s +Step 35180/44000 | Loss: 2.9464 | LR: 4.96e-05 | 725.5ms/step | 27.6 samples/s +Step 35190/44000 | Loss: 2.8458 | LR: 4.95e-05 | 735.0ms/step | 27.2 samples/s +Step 35200/44000 | Loss: 2.8385 | LR: 4.94e-05 | 725.5ms/step | 27.6 samples/s +Step 35210/44000 | Loss: 2.9080 | LR: 4.93e-05 | 725.2ms/step | 27.6 samples/s +Step 35220/44000 | Loss: 2.9668 | LR: 4.91e-05 | 725.2ms/step | 27.6 samples/s +Step 35230/44000 | Loss: 2.9374 | LR: 4.90e-05 | 725.3ms/step | 27.6 samples/s +Step 35240/44000 | Loss: 2.9500 | LR: 4.89e-05 | 725.0ms/step | 27.6 samples/s +Step 35250/44000 | Loss: 2.8722 | LR: 4.88e-05 | 725.3ms/step | 27.6 samples/s +Validation at step 35250: Loss = 2.9083, Perplexity = 18.33 +Step 35260/44000 | Loss: 2.9645 | LR: 4.87e-05 | 949.7ms/step | 21.1 samples/s +Step 35270/44000 | Loss: 2.9348 | LR: 4.86e-05 | 736.9ms/step | 27.1 samples/s +Step 35280/44000 | Loss: 2.9119 | LR: 4.85e-05 | 725.0ms/step | 27.6 samples/s +Step 35290/44000 | Loss: 2.9450 | LR: 4.84e-05 | 725.7ms/step | 27.6 samples/s +Step 35300/44000 | Loss: 2.9599 | LR: 4.83e-05 | 725.4ms/step | 27.6 samples/s +Step 35310/44000 | Loss: 2.9005 | LR: 4.82e-05 | 725.8ms/step | 27.6 samples/s +Step 35320/44000 | Loss: 2.9185 | LR: 4.81e-05 | 725.8ms/step | 27.6 samples/s +Step 35330/44000 | Loss: 2.9857 | LR: 4.80e-05 | 725.1ms/step | 27.6 samples/s +Step 35340/44000 | Loss: 2.9500 | LR: 4.79e-05 | 725.2ms/step | 27.6 samples/s +Step 35350/44000 | Loss: 2.9616 | LR: 4.77e-05 | 725.0ms/step | 27.6 samples/s +Step 35360/44000 | Loss: 2.8772 | LR: 4.76e-05 | 736.7ms/step | 27.1 samples/s +Step 35370/44000 | Loss: 2.9468 | LR: 4.75e-05 | 725.4ms/step | 27.6 samples/s +Step 35380/44000 | Loss: 2.9378 | LR: 4.74e-05 | 726.0ms/step | 27.5 samples/s +Step 35390/44000 | Loss: 2.9599 | LR: 4.73e-05 | 725.4ms/step | 27.6 samples/s +Step 35400/44000 | Loss: 2.9797 | LR: 4.72e-05 | 726.1ms/step | 27.5 samples/s +Step 35410/44000 | Loss: 2.8930 | LR: 4.71e-05 | 725.7ms/step | 27.6 samples/s +Step 35420/44000 | Loss: 2.9217 | LR: 4.70e-05 | 725.1ms/step | 27.6 samples/s +Step 35430/44000 | Loss: 2.8764 | LR: 4.69e-05 | 725.3ms/step | 27.6 samples/s +Step 35440/44000 | Loss: 2.8847 | LR: 4.68e-05 | 726.6ms/step | 27.5 samples/s +Step 35450/44000 | Loss: 2.9578 | LR: 4.67e-05 | 735.8ms/step | 27.2 samples/s +Step 35460/44000 | Loss: 2.9237 | LR: 4.66e-05 | 725.5ms/step | 27.6 samples/s +Step 35470/44000 | Loss: 2.8844 | LR: 4.65e-05 | 725.6ms/step | 27.6 samples/s +Step 35480/44000 | Loss: 2.9225 | LR: 4.64e-05 | 726.3ms/step | 27.5 samples/s +Step 35490/44000 | Loss: 2.9438 | LR: 4.63e-05 | 725.7ms/step | 27.6 samples/s +Step 35500/44000 | Loss: 2.9442 | LR: 4.62e-05 | 726.6ms/step | 27.5 samples/s +Validation at step 35500: Loss = 2.9080, Perplexity = 18.32 +Step 35510/44000 | Loss: 2.9095 | LR: 4.61e-05 | 951.4ms/step | 21.0 samples/s +Step 35520/44000 | Loss: 2.9613 | LR: 4.59e-05 | 725.6ms/step | 27.6 samples/s +Step 35530/44000 | Loss: 2.9170 | LR: 4.58e-05 | 736.7ms/step | 27.1 samples/s +Step 35540/44000 | Loss: 2.9237 | LR: 4.57e-05 | 725.7ms/step | 27.6 samples/s +Step 35550/44000 | Loss: 2.9438 | LR: 4.56e-05 | 725.8ms/step | 27.6 samples/s +Step 35560/44000 | Loss: 2.9537 | LR: 4.55e-05 | 725.7ms/step | 27.6 samples/s +Step 35570/44000 | Loss: 2.9628 | LR: 4.54e-05 | 725.0ms/step | 27.6 samples/s +Step 35580/44000 | Loss: 2.8480 | LR: 4.53e-05 | 725.9ms/step | 27.6 samples/s +Step 35590/44000 | Loss: 2.9122 | LR: 4.52e-05 | 725.4ms/step | 27.6 samples/s +Step 35600/44000 | Loss: 2.9389 | LR: 4.51e-05 | 726.8ms/step | 27.5 samples/s +Step 35610/44000 | Loss: 2.8944 | LR: 4.50e-05 | 726.3ms/step | 27.5 samples/s +Step 35620/44000 | Loss: 2.9304 | LR: 4.49e-05 | 738.9ms/step | 27.1 samples/s +Step 35630/44000 | Loss: 3.0069 | LR: 4.48e-05 | 726.5ms/step | 27.5 samples/s +Step 35640/44000 | Loss: 2.8956 | LR: 4.47e-05 | 726.6ms/step | 27.5 samples/s +Step 35650/44000 | Loss: 2.9640 | LR: 4.46e-05 | 726.2ms/step | 27.5 samples/s +Step 35660/44000 | Loss: 2.9787 | LR: 4.45e-05 | 727.0ms/step | 27.5 samples/s +Step 35670/44000 | Loss: 2.8828 | LR: 4.44e-05 | 726.8ms/step | 27.5 samples/s +Step 35680/44000 | Loss: 2.8646 | LR: 4.43e-05 | 725.5ms/step | 27.6 samples/s +Step 35690/44000 | Loss: 2.9591 | LR: 4.42e-05 | 727.2ms/step | 27.5 samples/s +Step 35700/44000 | Loss: 2.9517 | LR: 4.41e-05 | 731.7ms/step | 27.3 samples/s +Step 35710/44000 | Loss: 2.8721 | LR: 4.40e-05 | 733.2ms/step | 27.3 samples/s +Step 35720/44000 | Loss: 2.8976 | LR: 4.39e-05 | 726.1ms/step | 27.5 samples/s +Step 35730/44000 | Loss: 2.8898 | LR: 4.38e-05 | 726.7ms/step | 27.5 samples/s +Step 35740/44000 | Loss: 2.8562 | LR: 4.37e-05 | 726.6ms/step | 27.5 samples/s +Step 35750/44000 | Loss: 2.9371 | LR: 4.36e-05 | 726.1ms/step | 27.5 samples/s +Validation at step 35750: Loss = 2.9076, Perplexity = 18.31 +Step 35760/44000 | Loss: 2.8649 | LR: 4.35e-05 | 950.5ms/step | 21.0 samples/s +Step 35770/44000 | Loss: 2.9528 | LR: 4.34e-05 | 726.4ms/step | 27.5 samples/s +Step 35780/44000 | Loss: 2.8512 | LR: 4.33e-05 | 727.2ms/step | 27.5 samples/s +Step 35790/44000 | Loss: 2.9445 | LR: 4.32e-05 | 736.4ms/step | 27.2 samples/s +Step 35800/44000 | Loss: 2.9091 | LR: 4.31e-05 | 727.1ms/step | 27.5 samples/s +Step 35810/44000 | Loss: 2.8825 | LR: 4.29e-05 | 726.1ms/step | 27.5 samples/s +Step 35820/44000 | Loss: 2.9352 | LR: 4.28e-05 | 726.6ms/step | 27.5 samples/s +Step 35830/44000 | Loss: 2.9927 | LR: 4.27e-05 | 726.6ms/step | 27.5 samples/s +Step 35840/44000 | Loss: 2.9832 | LR: 4.26e-05 | 726.0ms/step | 27.5 samples/s +Step 35850/44000 | Loss: 2.9621 | LR: 4.25e-05 | 726.5ms/step | 27.5 samples/s +Step 35860/44000 | Loss: 2.8916 | LR: 4.24e-05 | 725.9ms/step | 27.6 samples/s +Step 35870/44000 | Loss: 2.9929 | LR: 4.23e-05 | 725.8ms/step | 27.6 samples/s +Step 35880/44000 | Loss: 2.9081 | LR: 4.22e-05 | 737.0ms/step | 27.1 samples/s +Step 35890/44000 | Loss: 2.9264 | LR: 4.21e-05 | 726.5ms/step | 27.5 samples/s +Step 35900/44000 | Loss: 2.9295 | LR: 4.20e-05 | 726.6ms/step | 27.5 samples/s +Step 35910/44000 | Loss: 3.0113 | LR: 4.19e-05 | 726.4ms/step | 27.5 samples/s +Step 35920/44000 | Loss: 2.9801 | LR: 4.18e-05 | 726.3ms/step | 27.5 samples/s +Step 35930/44000 | Loss: 2.9291 | LR: 4.17e-05 | 726.2ms/step | 27.5 samples/s +Step 35940/44000 | Loss: 2.9102 | LR: 4.16e-05 | 726.6ms/step | 27.5 samples/s +Step 35950/44000 | Loss: 2.8814 | LR: 4.15e-05 | 725.7ms/step | 27.6 samples/s +Step 35960/44000 | Loss: 2.9543 | LR: 4.14e-05 | 730.2ms/step | 27.4 samples/s +Step 35970/44000 | Loss: 2.8847 | LR: 4.13e-05 | 730.7ms/step | 27.4 samples/s +Step 35980/44000 | Loss: 2.9215 | LR: 4.12e-05 | 726.1ms/step | 27.5 samples/s +Step 35990/44000 | Loss: 2.9522 | LR: 4.11e-05 | 726.1ms/step | 27.5 samples/s +Step 36000/44000 | Loss: 3.0046 | LR: 4.10e-05 | 726.0ms/step | 27.5 samples/s +Validation at step 36000: Loss = 2.9074, Perplexity = 18.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-36000 +Step 36010/44000 | Loss: 2.9714 | LR: 4.09e-05 | 1427.3ms/step | 14.0 samples/s +Step 36020/44000 | Loss: 2.9194 | LR: 4.08e-05 | 725.4ms/step | 27.6 samples/s +Step 36030/44000 | Loss: 2.9619 | LR: 4.07e-05 | 725.1ms/step | 27.6 samples/s +Step 36040/44000 | Loss: 2.8996 | LR: 4.06e-05 | 739.4ms/step | 27.0 samples/s +Step 36050/44000 | Loss: 2.9477 | LR: 4.05e-05 | 726.2ms/step | 27.5 samples/s +Step 36060/44000 | Loss: 2.9345 | LR: 4.04e-05 | 726.3ms/step | 27.5 samples/s +Step 36070/44000 | Loss: 2.9426 | LR: 4.03e-05 | 726.0ms/step | 27.5 samples/s +Step 36080/44000 | Loss: 2.9423 | LR: 4.02e-05 | 725.7ms/step | 27.6 samples/s +Step 36090/44000 | Loss: 2.8886 | LR: 4.01e-05 | 725.4ms/step | 27.6 samples/s +Step 36100/44000 | Loss: 2.8862 | LR: 4.00e-05 | 725.7ms/step | 27.6 samples/s +Step 36110/44000 | Loss: 2.9000 | LR: 3.99e-05 | 726.2ms/step | 27.5 samples/s +Step 36120/44000 | Loss: 2.9068 | LR: 3.98e-05 | 725.5ms/step | 27.6 samples/s +Step 36130/44000 | Loss: 2.9047 | LR: 3.97e-05 | 735.2ms/step | 27.2 samples/s +Step 36140/44000 | Loss: 2.8537 | LR: 3.97e-05 | 726.5ms/step | 27.5 samples/s +Step 36150/44000 | Loss: 2.8347 | LR: 3.96e-05 | 726.6ms/step | 27.5 samples/s +Step 36160/44000 | Loss: 2.8434 | LR: 3.95e-05 | 726.8ms/step | 27.5 samples/s +Step 36170/44000 | Loss: 2.9668 | LR: 3.94e-05 | 726.8ms/step | 27.5 samples/s +Step 36180/44000 | Loss: 2.9184 | LR: 3.93e-05 | 727.1ms/step | 27.5 samples/s +Step 36190/44000 | Loss: 2.9009 | LR: 3.92e-05 | 726.9ms/step | 27.5 samples/s +Step 36200/44000 | Loss: 2.9391 | LR: 3.91e-05 | 726.1ms/step | 27.5 samples/s +Step 36210/44000 | Loss: 2.8753 | LR: 3.90e-05 | 725.5ms/step | 27.6 samples/s +Step 36220/44000 | Loss: 2.8802 | LR: 3.89e-05 | 739.6ms/step | 27.0 samples/s +Step 36230/44000 | Loss: 2.9234 | LR: 3.88e-05 | 725.5ms/step | 27.6 samples/s +Step 36240/44000 | Loss: 2.9202 | LR: 3.87e-05 | 725.8ms/step | 27.6 samples/s +Step 36250/44000 | Loss: 2.8647 | LR: 3.86e-05 | 726.0ms/step | 27.5 samples/s +Validation at step 36250: Loss = 2.9073, Perplexity = 18.31 +Step 36260/44000 | Loss: 2.9422 | LR: 3.85e-05 | 950.4ms/step | 21.0 samples/s +Step 36270/44000 | Loss: 2.9494 | LR: 3.84e-05 | 725.5ms/step | 27.6 samples/s +Step 36280/44000 | Loss: 2.9633 | LR: 3.83e-05 | 725.7ms/step | 27.6 samples/s +Step 36290/44000 | Loss: 2.9681 | LR: 3.82e-05 | 725.7ms/step | 27.6 samples/s +Step 36300/44000 | Loss: 2.9187 | LR: 3.81e-05 | 735.2ms/step | 27.2 samples/s +Step 36310/44000 | Loss: 2.8823 | LR: 3.80e-05 | 726.2ms/step | 27.5 samples/s +Step 36320/44000 | Loss: 2.9480 | LR: 3.79e-05 | 726.0ms/step | 27.5 samples/s +Step 36330/44000 | Loss: 2.8870 | LR: 3.78e-05 | 725.7ms/step | 27.6 samples/s +Step 36340/44000 | Loss: 2.9070 | LR: 3.77e-05 | 725.6ms/step | 27.6 samples/s +Step 36350/44000 | Loss: 2.9048 | LR: 3.76e-05 | 725.5ms/step | 27.6 samples/s +Step 36360/44000 | Loss: 2.9425 | LR: 3.75e-05 | 725.5ms/step | 27.6 samples/s +Step 36370/44000 | Loss: 2.8621 | LR: 3.74e-05 | 726.6ms/step | 27.5 samples/s +Step 36380/44000 | Loss: 2.9513 | LR: 3.73e-05 | 725.6ms/step | 27.6 samples/s +Step 36390/44000 | Loss: 2.9056 | LR: 3.72e-05 | 735.5ms/step | 27.2 samples/s +Step 36400/44000 | Loss: 2.9268 | LR: 3.71e-05 | 726.1ms/step | 27.5 samples/s +Step 36410/44000 | Loss: 2.8950 | LR: 3.70e-05 | 726.2ms/step | 27.5 samples/s +Step 36420/44000 | Loss: 2.9528 | LR: 3.69e-05 | 726.9ms/step | 27.5 samples/s +Step 36430/44000 | Loss: 2.8786 | LR: 3.69e-05 | 725.9ms/step | 27.6 samples/s +Step 36440/44000 | Loss: 2.9451 | LR: 3.68e-05 | 726.7ms/step | 27.5 samples/s +Step 36450/44000 | Loss: 2.8577 | LR: 3.67e-05 | 726.6ms/step | 27.5 samples/s +Step 36460/44000 | Loss: 2.9182 | LR: 3.66e-05 | 726.8ms/step | 27.5 samples/s +Step 36470/44000 | Loss: 2.9013 | LR: 3.65e-05 | 733.6ms/step | 27.3 samples/s +Step 36480/44000 | Loss: 2.9775 | LR: 3.64e-05 | 731.4ms/step | 27.3 samples/s +Step 36490/44000 | Loss: 2.8460 | LR: 3.63e-05 | 726.6ms/step | 27.5 samples/s +Step 36500/44000 | Loss: 2.9140 | LR: 3.62e-05 | 726.9ms/step | 27.5 samples/s +Validation at step 36500: Loss = 2.9070, Perplexity = 18.30 +Step 36510/44000 | Loss: 2.8845 | LR: 3.61e-05 | 951.1ms/step | 21.0 samples/s +Step 36520/44000 | Loss: 2.9289 | LR: 3.60e-05 | 727.2ms/step | 27.5 samples/s +Step 36530/44000 | Loss: 2.9181 | LR: 3.59e-05 | 727.2ms/step | 27.5 samples/s +Step 36540/44000 | Loss: 2.9464 | LR: 3.58e-05 | 726.9ms/step | 27.5 samples/s +Step 36550/44000 | Loss: 2.9217 | LR: 3.57e-05 | 727.5ms/step | 27.5 samples/s +Step 36560/44000 | Loss: 2.8937 | LR: 3.56e-05 | 738.6ms/step | 27.1 samples/s +Step 36570/44000 | Loss: 2.9384 | LR: 3.55e-05 | 726.6ms/step | 27.5 samples/s +Step 36580/44000 | Loss: 2.9045 | LR: 3.54e-05 | 726.3ms/step | 27.5 samples/s +Step 36590/44000 | Loss: 2.8813 | LR: 3.53e-05 | 726.6ms/step | 27.5 samples/s +Step 36600/44000 | Loss: 2.9993 | LR: 3.53e-05 | 726.3ms/step | 27.5 samples/s +Step 36610/44000 | Loss: 2.9878 | LR: 3.52e-05 | 726.3ms/step | 27.5 samples/s +Step 36620/44000 | Loss: 2.8953 | LR: 3.51e-05 | 725.9ms/step | 27.6 samples/s +Step 36630/44000 | Loss: 2.9029 | LR: 3.50e-05 | 726.0ms/step | 27.5 samples/s +Step 36640/44000 | Loss: 3.0011 | LR: 3.49e-05 | 726.5ms/step | 27.5 samples/s +Step 36650/44000 | Loss: 2.9141 | LR: 3.48e-05 | 735.5ms/step | 27.2 samples/s +Step 36660/44000 | Loss: 2.8722 | LR: 3.47e-05 | 726.1ms/step | 27.5 samples/s +Step 36670/44000 | Loss: 2.9020 | LR: 3.46e-05 | 726.8ms/step | 27.5 samples/s +Step 36680/44000 | Loss: 2.8656 | LR: 3.45e-05 | 725.9ms/step | 27.6 samples/s +Step 36690/44000 | Loss: 2.9663 | LR: 3.44e-05 | 726.1ms/step | 27.5 samples/s +Step 36700/44000 | Loss: 2.8932 | LR: 3.43e-05 | 725.6ms/step | 27.6 samples/s +Step 36710/44000 | Loss: 2.9387 | LR: 3.42e-05 | 726.1ms/step | 27.5 samples/s +Step 36720/44000 | Loss: 2.9694 | LR: 3.41e-05 | 725.7ms/step | 27.6 samples/s +Step 36730/44000 | Loss: 2.8861 | LR: 3.41e-05 | 730.2ms/step | 27.4 samples/s +Step 36740/44000 | Loss: 2.9166 | LR: 3.40e-05 | 730.5ms/step | 27.4 samples/s +Step 36750/44000 | Loss: 2.9020 | LR: 3.39e-05 | 725.9ms/step | 27.6 samples/s +Validation at step 36750: Loss = 2.9071, Perplexity = 18.30 +Step 36760/44000 | Loss: 2.9022 | LR: 3.38e-05 | 952.2ms/step | 21.0 samples/s +Step 36770/44000 | Loss: 2.8682 | LR: 3.37e-05 | 726.4ms/step | 27.5 samples/s +Step 36780/44000 | Loss: 2.9699 | LR: 3.36e-05 | 726.0ms/step | 27.5 samples/s +Step 36790/44000 | Loss: 2.8260 | LR: 3.35e-05 | 725.9ms/step | 27.6 samples/s +Step 36800/44000 | Loss: 2.9385 | LR: 3.34e-05 | 726.0ms/step | 27.5 samples/s +Step 36810/44000 | Loss: 2.9125 | LR: 3.33e-05 | 725.9ms/step | 27.6 samples/s +Step 36820/44000 | Loss: 2.9015 | LR: 3.32e-05 | 735.1ms/step | 27.2 samples/s +Step 36830/44000 | Loss: 2.9300 | LR: 3.31e-05 | 725.9ms/step | 27.6 samples/s +Step 36840/44000 | Loss: 3.0436 | LR: 3.31e-05 | 725.9ms/step | 27.6 samples/s +Step 36850/44000 | Loss: 2.9900 | LR: 3.30e-05 | 726.4ms/step | 27.5 samples/s +Step 36860/44000 | Loss: 2.9028 | LR: 3.29e-05 | 725.9ms/step | 27.6 samples/s +Step 36870/44000 | Loss: 2.9270 | LR: 3.28e-05 | 726.1ms/step | 27.5 samples/s +Step 36880/44000 | Loss: 2.9216 | LR: 3.27e-05 | 726.0ms/step | 27.5 samples/s +Step 36890/44000 | Loss: 2.9501 | LR: 3.26e-05 | 726.7ms/step | 27.5 samples/s +Step 36900/44000 | Loss: 2.8911 | LR: 3.25e-05 | 726.6ms/step | 27.5 samples/s +Step 36910/44000 | Loss: 2.9368 | LR: 3.24e-05 | 737.3ms/step | 27.1 samples/s +Step 36920/44000 | Loss: 2.9445 | LR: 3.23e-05 | 725.9ms/step | 27.6 samples/s +Step 36930/44000 | Loss: 2.9458 | LR: 3.22e-05 | 726.0ms/step | 27.5 samples/s +Step 36940/44000 | Loss: 2.8720 | LR: 3.22e-05 | 729.1ms/step | 27.4 samples/s +Step 36950/44000 | Loss: 2.9155 | LR: 3.21e-05 | 729.4ms/step | 27.4 samples/s +Step 36960/44000 | Loss: 2.9381 | LR: 3.20e-05 | 729.1ms/step | 27.4 samples/s +Step 36970/44000 | Loss: 2.8370 | LR: 3.19e-05 | 729.4ms/step | 27.4 samples/s +Step 36980/44000 | Loss: 2.8848 | LR: 3.18e-05 | 729.2ms/step | 27.4 samples/s +Step 36990/44000 | Loss: 2.9433 | LR: 3.17e-05 | 733.0ms/step | 27.3 samples/s +Step 37000/44000 | Loss: 2.9415 | LR: 3.16e-05 | 741.7ms/step | 27.0 samples/s +Validation at step 37000: Loss = 2.9068, Perplexity = 18.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-37000 +Step 37010/44000 | Loss: 2.9189 | LR: 3.15e-05 | 1499.3ms/step | 13.3 samples/s +Step 37020/44000 | Loss: 2.9125 | LR: 3.15e-05 | 728.4ms/step | 27.5 samples/s +Step 37030/44000 | Loss: 2.8814 | LR: 3.14e-05 | 729.5ms/step | 27.4 samples/s +Step 37040/44000 | Loss: 2.8992 | LR: 3.13e-05 | 728.9ms/step | 27.4 samples/s +Step 37050/44000 | Loss: 2.8881 | LR: 3.12e-05 | 728.6ms/step | 27.5 samples/s +Step 37060/44000 | Loss: 2.9422 | LR: 3.11e-05 | 727.1ms/step | 27.5 samples/s +Step 37070/44000 | Loss: 2.9198 | LR: 3.10e-05 | 743.6ms/step | 26.9 samples/s +Step 37080/44000 | Loss: 2.8757 | LR: 3.09e-05 | 726.5ms/step | 27.5 samples/s +Step 37090/44000 | Loss: 2.9509 | LR: 3.08e-05 | 727.6ms/step | 27.5 samples/s +Step 37100/44000 | Loss: 2.9092 | LR: 3.07e-05 | 727.4ms/step | 27.5 samples/s +Step 37110/44000 | Loss: 2.8909 | LR: 3.07e-05 | 726.5ms/step | 27.5 samples/s +Step 37120/44000 | Loss: 3.0078 | LR: 3.06e-05 | 726.4ms/step | 27.5 samples/s +Step 37130/44000 | Loss: 2.8640 | LR: 3.05e-05 | 726.5ms/step | 27.5 samples/s +Step 37140/44000 | Loss: 2.8218 | LR: 3.04e-05 | 726.6ms/step | 27.5 samples/s +Step 37150/44000 | Loss: 2.8788 | LR: 3.03e-05 | 726.2ms/step | 27.5 samples/s +Step 37160/44000 | Loss: 2.8931 | LR: 3.02e-05 | 734.4ms/step | 27.2 samples/s +Step 37170/44000 | Loss: 2.9314 | LR: 3.01e-05 | 726.2ms/step | 27.5 samples/s +Step 37180/44000 | Loss: 2.9044 | LR: 3.01e-05 | 726.9ms/step | 27.5 samples/s +Step 37190/44000 | Loss: 2.8685 | LR: 3.00e-05 | 727.1ms/step | 27.5 samples/s +Step 37200/44000 | Loss: 2.8745 | LR: 2.99e-05 | 726.8ms/step | 27.5 samples/s +Step 37210/44000 | Loss: 2.9597 | LR: 2.98e-05 | 726.5ms/step | 27.5 samples/s +Step 37220/44000 | Loss: 2.8813 | LR: 2.97e-05 | 726.4ms/step | 27.5 samples/s +Step 37230/44000 | Loss: 2.8806 | LR: 2.96e-05 | 726.8ms/step | 27.5 samples/s +Step 37240/44000 | Loss: 2.8954 | LR: 2.95e-05 | 734.6ms/step | 27.2 samples/s +Step 37250/44000 | Loss: 2.9758 | LR: 2.95e-05 | 731.2ms/step | 27.4 samples/s +Validation at step 37250: Loss = 2.9068, Perplexity = 18.30 +Step 37260/44000 | Loss: 2.9467 | LR: 2.94e-05 | 951.0ms/step | 21.0 samples/s +Step 37270/44000 | Loss: 2.9813 | LR: 2.93e-05 | 726.5ms/step | 27.5 samples/s +Step 37280/44000 | Loss: 2.9061 | LR: 2.92e-05 | 726.5ms/step | 27.5 samples/s +Step 37290/44000 | Loss: 2.9386 | LR: 2.91e-05 | 726.5ms/step | 27.5 samples/s +Step 37300/44000 | Loss: 2.9077 | LR: 2.90e-05 | 726.4ms/step | 27.5 samples/s +Step 37310/44000 | Loss: 2.9655 | LR: 2.89e-05 | 727.7ms/step | 27.5 samples/s +Step 37320/44000 | Loss: 2.9122 | LR: 2.89e-05 | 726.5ms/step | 27.5 samples/s +Step 37330/44000 | Loss: 2.9183 | LR: 2.88e-05 | 738.4ms/step | 27.1 samples/s +Step 37340/44000 | Loss: 2.9441 | LR: 2.87e-05 | 726.7ms/step | 27.5 samples/s +Step 37350/44000 | Loss: 2.9353 | LR: 2.86e-05 | 726.2ms/step | 27.5 samples/s +Step 37360/44000 | Loss: 2.9310 | LR: 2.85e-05 | 726.4ms/step | 27.5 samples/s +Step 37370/44000 | Loss: 2.9010 | LR: 2.84e-05 | 726.7ms/step | 27.5 samples/s +Step 37380/44000 | Loss: 2.9580 | LR: 2.84e-05 | 726.6ms/step | 27.5 samples/s +Step 37390/44000 | Loss: 2.8667 | LR: 2.83e-05 | 726.2ms/step | 27.5 samples/s +Step 37400/44000 | Loss: 2.9397 | LR: 2.82e-05 | 727.6ms/step | 27.5 samples/s +Step 37410/44000 | Loss: 2.9554 | LR: 2.81e-05 | 731.3ms/step | 27.3 samples/s +Step 37420/44000 | Loss: 2.9079 | LR: 2.80e-05 | 732.1ms/step | 27.3 samples/s +Step 37430/44000 | Loss: 2.9750 | LR: 2.79e-05 | 726.6ms/step | 27.5 samples/s +Step 37440/44000 | Loss: 2.8879 | LR: 2.78e-05 | 726.5ms/step | 27.5 samples/s +Step 37450/44000 | Loss: 2.9633 | LR: 2.78e-05 | 726.4ms/step | 27.5 samples/s +Step 37460/44000 | Loss: 3.0181 | LR: 2.77e-05 | 726.6ms/step | 27.5 samples/s +Step 37470/44000 | Loss: 2.9396 | LR: 2.76e-05 | 726.2ms/step | 27.5 samples/s +Step 37480/44000 | Loss: 2.8699 | LR: 2.75e-05 | 726.1ms/step | 27.5 samples/s +Step 37490/44000 | Loss: 2.9357 | LR: 2.74e-05 | 727.0ms/step | 27.5 samples/s +Step 37500/44000 | Loss: 2.9041 | LR: 2.74e-05 | 732.1ms/step | 27.3 samples/s +Validation at step 37500: Loss = 2.9066, Perplexity = 18.29 +Step 37510/44000 | Loss: 2.8644 | LR: 2.73e-05 | 957.2ms/step | 20.9 samples/s +Step 37520/44000 | Loss: 2.9379 | LR: 2.72e-05 | 726.3ms/step | 27.5 samples/s +Step 37530/44000 | Loss: 2.8634 | LR: 2.71e-05 | 727.0ms/step | 27.5 samples/s +Step 37540/44000 | Loss: 2.8990 | LR: 2.70e-05 | 726.1ms/step | 27.5 samples/s +Step 37550/44000 | Loss: 2.9582 | LR: 2.69e-05 | 726.5ms/step | 27.5 samples/s +Step 37560/44000 | Loss: 2.8690 | LR: 2.69e-05 | 726.6ms/step | 27.5 samples/s +Step 37570/44000 | Loss: 2.8643 | LR: 2.68e-05 | 726.1ms/step | 27.5 samples/s +Step 37580/44000 | Loss: 2.9317 | LR: 2.67e-05 | 726.4ms/step | 27.5 samples/s +Step 37590/44000 | Loss: 2.9571 | LR: 2.66e-05 | 735.5ms/step | 27.2 samples/s +Step 37600/44000 | Loss: 2.8872 | LR: 2.65e-05 | 726.6ms/step | 27.5 samples/s +Step 37610/44000 | Loss: 2.8951 | LR: 2.65e-05 | 726.3ms/step | 27.5 samples/s +Step 37620/44000 | Loss: 2.8702 | LR: 2.64e-05 | 726.1ms/step | 27.5 samples/s +Step 37630/44000 | Loss: 2.9096 | LR: 2.63e-05 | 726.7ms/step | 27.5 samples/s +Step 37640/44000 | Loss: 2.9317 | LR: 2.62e-05 | 727.7ms/step | 27.5 samples/s +Step 37650/44000 | Loss: 3.0330 | LR: 2.61e-05 | 727.1ms/step | 27.5 samples/s +Step 37660/44000 | Loss: 2.9567 | LR: 2.60e-05 | 726.5ms/step | 27.5 samples/s +Step 37670/44000 | Loss: 2.9250 | LR: 2.60e-05 | 732.9ms/step | 27.3 samples/s +Step 37680/44000 | Loss: 2.9142 | LR: 2.59e-05 | 733.6ms/step | 27.3 samples/s +Step 37690/44000 | Loss: 2.9346 | LR: 2.58e-05 | 726.1ms/step | 27.5 samples/s +Step 37700/44000 | Loss: 2.9100 | LR: 2.57e-05 | 726.1ms/step | 27.5 samples/s +Step 37710/44000 | Loss: 2.9428 | LR: 2.56e-05 | 726.3ms/step | 27.5 samples/s +Step 37720/44000 | Loss: 2.9118 | LR: 2.56e-05 | 726.9ms/step | 27.5 samples/s +Step 37730/44000 | Loss: 2.8889 | LR: 2.55e-05 | 727.0ms/step | 27.5 samples/s +Step 37740/44000 | Loss: 2.8370 | LR: 2.54e-05 | 726.4ms/step | 27.5 samples/s +Step 37750/44000 | Loss: 2.8956 | LR: 2.53e-05 | 726.0ms/step | 27.5 samples/s +Validation at step 37750: Loss = 2.9066, Perplexity = 18.29 +Step 37760/44000 | Loss: 2.8785 | LR: 2.52e-05 | 960.3ms/step | 20.8 samples/s +Step 37770/44000 | Loss: 2.9533 | LR: 2.52e-05 | 727.1ms/step | 27.5 samples/s +Step 37780/44000 | Loss: 2.9311 | LR: 2.51e-05 | 727.4ms/step | 27.5 samples/s +Step 37790/44000 | Loss: 2.8905 | LR: 2.50e-05 | 728.6ms/step | 27.5 samples/s +Step 37800/44000 | Loss: 2.8333 | LR: 2.49e-05 | 726.3ms/step | 27.5 samples/s +Step 37810/44000 | Loss: 2.9124 | LR: 2.48e-05 | 726.5ms/step | 27.5 samples/s +Step 37820/44000 | Loss: 2.8826 | LR: 2.48e-05 | 727.2ms/step | 27.5 samples/s +Step 37830/44000 | Loss: 2.9152 | LR: 2.47e-05 | 726.1ms/step | 27.5 samples/s +Step 37840/44000 | Loss: 2.9516 | LR: 2.46e-05 | 728.6ms/step | 27.5 samples/s +Step 37850/44000 | Loss: 2.9109 | LR: 2.45e-05 | 737.2ms/step | 27.1 samples/s +Step 37860/44000 | Loss: 2.9392 | LR: 2.45e-05 | 726.4ms/step | 27.5 samples/s +Step 37870/44000 | Loss: 2.9007 | LR: 2.44e-05 | 726.6ms/step | 27.5 samples/s +Step 37880/44000 | Loss: 2.9343 | LR: 2.43e-05 | 725.8ms/step | 27.6 samples/s +Step 37890/44000 | Loss: 2.9254 | LR: 2.42e-05 | 725.9ms/step | 27.6 samples/s +Step 37900/44000 | Loss: 2.9020 | LR: 2.41e-05 | 726.6ms/step | 27.5 samples/s +Step 37910/44000 | Loss: 2.9508 | LR: 2.41e-05 | 725.9ms/step | 27.6 samples/s +Step 37920/44000 | Loss: 2.9639 | LR: 2.40e-05 | 725.6ms/step | 27.6 samples/s +Step 37930/44000 | Loss: 2.9742 | LR: 2.39e-05 | 730.6ms/step | 27.4 samples/s +Step 37940/44000 | Loss: 2.9015 | LR: 2.38e-05 | 730.6ms/step | 27.4 samples/s +Step 37950/44000 | Loss: 3.0117 | LR: 2.38e-05 | 726.5ms/step | 27.5 samples/s +Step 37960/44000 | Loss: 2.8507 | LR: 2.37e-05 | 726.1ms/step | 27.5 samples/s +Step 37970/44000 | Loss: 2.9859 | LR: 2.36e-05 | 725.4ms/step | 27.6 samples/s +Step 37980/44000 | Loss: 2.9659 | LR: 2.35e-05 | 725.9ms/step | 27.6 samples/s +Step 37990/44000 | Loss: 2.9006 | LR: 2.34e-05 | 725.6ms/step | 27.6 samples/s +Step 38000/44000 | Loss: 2.8308 | LR: 2.34e-05 | 725.0ms/step | 27.6 samples/s +Validation at step 38000: Loss = 2.9065, Perplexity = 18.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-38000 +Step 38010/44000 | Loss: 2.9597 | LR: 2.33e-05 | 1386.6ms/step | 14.4 samples/s +Step 38020/44000 | Loss: 2.9338 | LR: 2.32e-05 | 725.5ms/step | 27.6 samples/s +Step 38030/44000 | Loss: 2.8783 | LR: 2.31e-05 | 725.1ms/step | 27.6 samples/s +Step 38040/44000 | Loss: 2.8732 | LR: 2.31e-05 | 726.0ms/step | 27.5 samples/s +Step 38050/44000 | Loss: 2.8579 | LR: 2.30e-05 | 725.5ms/step | 27.6 samples/s +Step 38060/44000 | Loss: 2.9048 | LR: 2.29e-05 | 725.5ms/step | 27.6 samples/s +Step 38070/44000 | Loss: 2.8381 | LR: 2.28e-05 | 725.3ms/step | 27.6 samples/s +Step 38080/44000 | Loss: 2.9267 | LR: 2.28e-05 | 725.3ms/step | 27.6 samples/s +Step 38090/44000 | Loss: 2.9437 | LR: 2.27e-05 | 725.5ms/step | 27.6 samples/s +Step 38100/44000 | Loss: 2.9161 | LR: 2.26e-05 | 740.8ms/step | 27.0 samples/s +Step 38110/44000 | Loss: 2.8746 | LR: 2.25e-05 | 725.7ms/step | 27.6 samples/s +Step 38120/44000 | Loss: 2.8818 | LR: 2.25e-05 | 726.0ms/step | 27.5 samples/s +Step 38130/44000 | Loss: 2.9171 | LR: 2.24e-05 | 725.6ms/step | 27.6 samples/s +Step 38140/44000 | Loss: 2.8744 | LR: 2.23e-05 | 726.0ms/step | 27.5 samples/s +Step 38150/44000 | Loss: 2.8939 | LR: 2.22e-05 | 725.6ms/step | 27.6 samples/s +Step 38160/44000 | Loss: 2.8655 | LR: 2.22e-05 | 726.2ms/step | 27.5 samples/s +Step 38170/44000 | Loss: 2.8829 | LR: 2.21e-05 | 726.9ms/step | 27.5 samples/s +Step 38180/44000 | Loss: 2.8557 | LR: 2.20e-05 | 726.5ms/step | 27.5 samples/s +Step 38190/44000 | Loss: 2.9600 | LR: 2.19e-05 | 734.5ms/step | 27.2 samples/s +Step 38200/44000 | Loss: 2.9353 | LR: 2.19e-05 | 725.9ms/step | 27.6 samples/s +Step 38210/44000 | Loss: 2.8883 | LR: 2.18e-05 | 726.6ms/step | 27.5 samples/s +Step 38220/44000 | Loss: 2.9415 | LR: 2.17e-05 | 725.7ms/step | 27.6 samples/s +Step 38230/44000 | Loss: 2.9082 | LR: 2.16e-05 | 725.7ms/step | 27.6 samples/s +Step 38240/44000 | Loss: 2.9462 | LR: 2.16e-05 | 725.9ms/step | 27.6 samples/s +Step 38250/44000 | Loss: 2.9257 | LR: 2.15e-05 | 726.0ms/step | 27.5 samples/s +Validation at step 38250: Loss = 2.9064, Perplexity = 18.29 +Step 38260/44000 | Loss: 2.9012 | LR: 2.14e-05 | 950.5ms/step | 21.0 samples/s +Step 38270/44000 | Loss: 2.8895 | LR: 2.13e-05 | 736.7ms/step | 27.1 samples/s +Step 38280/44000 | Loss: 2.9289 | LR: 2.13e-05 | 725.4ms/step | 27.6 samples/s +Step 38290/44000 | Loss: 2.8854 | LR: 2.12e-05 | 725.3ms/step | 27.6 samples/s +Step 38300/44000 | Loss: 2.8923 | LR: 2.11e-05 | 725.8ms/step | 27.6 samples/s +Step 38310/44000 | Loss: 2.9284 | LR: 2.11e-05 | 726.1ms/step | 27.5 samples/s +Step 38320/44000 | Loss: 2.9083 | LR: 2.10e-05 | 726.1ms/step | 27.5 samples/s +Step 38330/44000 | Loss: 2.9723 | LR: 2.09e-05 | 725.4ms/step | 27.6 samples/s +Step 38340/44000 | Loss: 2.9567 | LR: 2.08e-05 | 725.4ms/step | 27.6 samples/s +Step 38350/44000 | Loss: 2.8970 | LR: 2.08e-05 | 725.8ms/step | 27.6 samples/s +Step 38360/44000 | Loss: 2.9623 | LR: 2.07e-05 | 739.9ms/step | 27.0 samples/s +Step 38370/44000 | Loss: 2.9486 | LR: 2.06e-05 | 726.1ms/step | 27.5 samples/s +Step 38380/44000 | Loss: 2.9205 | LR: 2.05e-05 | 726.6ms/step | 27.5 samples/s +Step 38390/44000 | Loss: 2.8797 | LR: 2.05e-05 | 726.2ms/step | 27.5 samples/s +Step 38400/44000 | Loss: 2.9576 | LR: 2.04e-05 | 726.6ms/step | 27.5 samples/s +Step 38410/44000 | Loss: 3.0086 | LR: 2.03e-05 | 726.6ms/step | 27.5 samples/s +Step 38420/44000 | Loss: 2.9505 | LR: 2.03e-05 | 726.4ms/step | 27.5 samples/s +Step 38430/44000 | Loss: 2.8899 | LR: 2.02e-05 | 726.7ms/step | 27.5 samples/s +Step 38440/44000 | Loss: 2.9340 | LR: 2.01e-05 | 730.2ms/step | 27.4 samples/s +Step 38450/44000 | Loss: 2.8723 | LR: 2.00e-05 | 731.6ms/step | 27.3 samples/s +Step 38460/44000 | Loss: 2.9002 | LR: 2.00e-05 | 726.3ms/step | 27.5 samples/s +Step 38470/44000 | Loss: 2.8827 | LR: 1.99e-05 | 726.7ms/step | 27.5 samples/s +Step 38480/44000 | Loss: 2.8956 | LR: 1.98e-05 | 726.4ms/step | 27.5 samples/s +Step 38490/44000 | Loss: 2.9096 | LR: 1.98e-05 | 725.9ms/step | 27.6 samples/s +Step 38500/44000 | Loss: 2.9226 | LR: 1.97e-05 | 726.1ms/step | 27.5 samples/s +Validation at step 38500: Loss = 2.9064, Perplexity = 18.29 +Step 38510/44000 | Loss: 2.8745 | LR: 1.96e-05 | 949.3ms/step | 21.1 samples/s +Step 38520/44000 | Loss: 2.9220 | LR: 1.95e-05 | 726.1ms/step | 27.5 samples/s +Step 38530/44000 | Loss: 2.9779 | LR: 1.95e-05 | 739.2ms/step | 27.1 samples/s +Step 38540/44000 | Loss: 2.9164 | LR: 1.94e-05 | 725.7ms/step | 27.6 samples/s +Step 38550/44000 | Loss: 2.8623 | LR: 1.93e-05 | 726.0ms/step | 27.5 samples/s +Step 38560/44000 | Loss: 2.9430 | LR: 1.93e-05 | 725.9ms/step | 27.6 samples/s +Step 38570/44000 | Loss: 2.8806 | LR: 1.92e-05 | 725.8ms/step | 27.6 samples/s +Step 38580/44000 | Loss: 2.9495 | LR: 1.91e-05 | 725.7ms/step | 27.6 samples/s +Step 38590/44000 | Loss: 2.9378 | LR: 1.91e-05 | 726.3ms/step | 27.5 samples/s +Step 38600/44000 | Loss: 2.8746 | LR: 1.90e-05 | 725.9ms/step | 27.6 samples/s +Step 38610/44000 | Loss: 2.9441 | LR: 1.89e-05 | 725.8ms/step | 27.6 samples/s +Step 38620/44000 | Loss: 2.9234 | LR: 1.88e-05 | 735.5ms/step | 27.2 samples/s +Step 38630/44000 | Loss: 2.8435 | LR: 1.88e-05 | 726.3ms/step | 27.5 samples/s +Step 38640/44000 | Loss: 2.9629 | LR: 1.87e-05 | 726.1ms/step | 27.5 samples/s +Step 38650/44000 | Loss: 2.9499 | LR: 1.86e-05 | 725.8ms/step | 27.6 samples/s +Step 38660/44000 | Loss: 2.9182 | LR: 1.86e-05 | 725.9ms/step | 27.6 samples/s +Step 38670/44000 | Loss: 2.8658 | LR: 1.85e-05 | 725.7ms/step | 27.6 samples/s +Step 38680/44000 | Loss: 2.9167 | LR: 1.84e-05 | 725.5ms/step | 27.6 samples/s +Step 38690/44000 | Loss: 2.9164 | LR: 1.84e-05 | 725.7ms/step | 27.6 samples/s +Step 38700/44000 | Loss: 2.8689 | LR: 1.83e-05 | 730.3ms/step | 27.4 samples/s +Step 38710/44000 | Loss: 2.9434 | LR: 1.82e-05 | 731.4ms/step | 27.3 samples/s +Step 38720/44000 | Loss: 2.8667 | LR: 1.82e-05 | 725.3ms/step | 27.6 samples/s +Step 38730/44000 | Loss: 2.8910 | LR: 1.81e-05 | 725.1ms/step | 27.6 samples/s +Step 38740/44000 | Loss: 2.8980 | LR: 1.80e-05 | 726.4ms/step | 27.5 samples/s +Step 38750/44000 | Loss: 2.9563 | LR: 1.80e-05 | 725.4ms/step | 27.6 samples/s +Validation at step 38750: Loss = 2.9063, Perplexity = 18.29 +Step 38760/44000 | Loss: 2.8420 | LR: 1.79e-05 | 950.2ms/step | 21.0 samples/s +Step 38770/44000 | Loss: 2.9469 | LR: 1.78e-05 | 726.0ms/step | 27.5 samples/s +Step 38780/44000 | Loss: 3.0377 | LR: 1.78e-05 | 725.6ms/step | 27.6 samples/s +Step 38790/44000 | Loss: 2.8660 | LR: 1.77e-05 | 737.4ms/step | 27.1 samples/s +Step 38800/44000 | Loss: 2.9089 | LR: 1.76e-05 | 725.5ms/step | 27.6 samples/s +Step 38810/44000 | Loss: 2.9085 | LR: 1.76e-05 | 725.5ms/step | 27.6 samples/s +Step 38820/44000 | Loss: 2.9275 | LR: 1.75e-05 | 725.9ms/step | 27.6 samples/s +Step 38830/44000 | Loss: 2.9705 | LR: 1.74e-05 | 725.5ms/step | 27.6 samples/s +Step 38840/44000 | Loss: 2.9331 | LR: 1.74e-05 | 726.3ms/step | 27.5 samples/s +Step 38850/44000 | Loss: 2.9451 | LR: 1.73e-05 | 725.6ms/step | 27.6 samples/s +Step 38860/44000 | Loss: 2.9085 | LR: 1.72e-05 | 726.0ms/step | 27.5 samples/s +Step 38870/44000 | Loss: 2.9099 | LR: 1.72e-05 | 726.1ms/step | 27.5 samples/s +Step 38880/44000 | Loss: 2.8785 | LR: 1.71e-05 | 734.7ms/step | 27.2 samples/s +Step 38890/44000 | Loss: 2.9002 | LR: 1.70e-05 | 725.5ms/step | 27.6 samples/s +Step 38900/44000 | Loss: 2.9468 | LR: 1.70e-05 | 727.0ms/step | 27.5 samples/s +Step 38910/44000 | Loss: 2.9173 | LR: 1.69e-05 | 727.1ms/step | 27.5 samples/s +Step 38920/44000 | Loss: 2.8612 | LR: 1.68e-05 | 725.8ms/step | 27.6 samples/s +Step 38930/44000 | Loss: 2.9422 | LR: 1.68e-05 | 725.2ms/step | 27.6 samples/s +Step 38940/44000 | Loss: 2.9825 | LR: 1.67e-05 | 725.6ms/step | 27.6 samples/s +Step 38950/44000 | Loss: 2.9131 | LR: 1.66e-05 | 725.4ms/step | 27.6 samples/s +Step 38960/44000 | Loss: 2.9703 | LR: 1.66e-05 | 730.6ms/step | 27.4 samples/s +Step 38970/44000 | Loss: 2.9456 | LR: 1.65e-05 | 731.0ms/step | 27.4 samples/s +Step 38980/44000 | Loss: 2.9172 | LR: 1.64e-05 | 726.5ms/step | 27.5 samples/s +Step 38990/44000 | Loss: 2.9818 | LR: 1.64e-05 | 726.1ms/step | 27.5 samples/s +Step 39000/44000 | Loss: 2.9259 | LR: 1.63e-05 | 725.5ms/step | 27.6 samples/s +Validation at step 39000: Loss = 2.9063, Perplexity = 18.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-39000 +Step 39010/44000 | Loss: 2.9640 | LR: 1.62e-05 | 1451.2ms/step | 13.8 samples/s +Step 39020/44000 | Loss: 2.8941 | LR: 1.62e-05 | 724.9ms/step | 27.6 samples/s +Step 39030/44000 | Loss: 2.9009 | LR: 1.61e-05 | 724.8ms/step | 27.6 samples/s +Step 39040/44000 | Loss: 2.8960 | LR: 1.61e-05 | 739.3ms/step | 27.1 samples/s +Step 39050/44000 | Loss: 2.8639 | LR: 1.60e-05 | 725.0ms/step | 27.6 samples/s +Step 39060/44000 | Loss: 2.8854 | LR: 1.59e-05 | 724.8ms/step | 27.6 samples/s +Step 39070/44000 | Loss: 2.9398 | LR: 1.59e-05 | 725.2ms/step | 27.6 samples/s +Step 39080/44000 | Loss: 2.9370 | LR: 1.58e-05 | 724.8ms/step | 27.6 samples/s +Step 39090/44000 | Loss: 2.8713 | LR: 1.57e-05 | 725.0ms/step | 27.6 samples/s +Step 39100/44000 | Loss: 2.8104 | LR: 1.57e-05 | 724.3ms/step | 27.6 samples/s +Step 39110/44000 | Loss: 2.8908 | LR: 1.56e-05 | 724.9ms/step | 27.6 samples/s +Step 39120/44000 | Loss: 2.9025 | LR: 1.55e-05 | 724.2ms/step | 27.6 samples/s +Step 39130/44000 | Loss: 2.9188 | LR: 1.55e-05 | 736.0ms/step | 27.2 samples/s +Step 39140/44000 | Loss: 2.9480 | LR: 1.54e-05 | 725.0ms/step | 27.6 samples/s +Step 39150/44000 | Loss: 2.9316 | LR: 1.54e-05 | 725.3ms/step | 27.6 samples/s +Step 39160/44000 | Loss: 2.9711 | LR: 1.53e-05 | 725.1ms/step | 27.6 samples/s +Step 39170/44000 | Loss: 2.8824 | LR: 1.52e-05 | 725.6ms/step | 27.6 samples/s +Step 39180/44000 | Loss: 2.9215 | LR: 1.52e-05 | 726.1ms/step | 27.5 samples/s +Step 39190/44000 | Loss: 2.9101 | LR: 1.51e-05 | 726.3ms/step | 27.5 samples/s +Step 39200/44000 | Loss: 2.9005 | LR: 1.50e-05 | 726.7ms/step | 27.5 samples/s +Step 39210/44000 | Loss: 2.9521 | LR: 1.50e-05 | 730.8ms/step | 27.4 samples/s +Step 39220/44000 | Loss: 2.9209 | LR: 1.49e-05 | 730.9ms/step | 27.4 samples/s +Step 39230/44000 | Loss: 2.8713 | LR: 1.49e-05 | 726.1ms/step | 27.5 samples/s +Step 39240/44000 | Loss: 2.8935 | LR: 1.48e-05 | 726.5ms/step | 27.5 samples/s +Step 39250/44000 | Loss: 2.8991 | LR: 1.47e-05 | 725.8ms/step | 27.6 samples/s +Validation at step 39250: Loss = 2.9063, Perplexity = 18.29 +Step 39260/44000 | Loss: 2.9194 | LR: 1.47e-05 | 948.2ms/step | 21.1 samples/s +Step 39270/44000 | Loss: 2.8585 | LR: 1.46e-05 | 725.2ms/step | 27.6 samples/s +Step 39280/44000 | Loss: 2.9709 | LR: 1.46e-05 | 726.2ms/step | 27.5 samples/s +Step 39290/44000 | Loss: 2.8657 | LR: 1.45e-05 | 726.2ms/step | 27.5 samples/s +Step 39300/44000 | Loss: 2.9601 | LR: 1.44e-05 | 736.8ms/step | 27.1 samples/s +Step 39310/44000 | Loss: 2.9382 | LR: 1.44e-05 | 727.4ms/step | 27.5 samples/s +Step 39320/44000 | Loss: 2.9310 | LR: 1.43e-05 | 728.0ms/step | 27.5 samples/s +Step 39330/44000 | Loss: 2.9490 | LR: 1.42e-05 | 726.7ms/step | 27.5 samples/s +Step 39340/44000 | Loss: 2.8525 | LR: 1.42e-05 | 726.0ms/step | 27.5 samples/s +Step 39350/44000 | Loss: 2.9799 | LR: 1.41e-05 | 725.8ms/step | 27.6 samples/s +Step 39360/44000 | Loss: 2.8965 | LR: 1.41e-05 | 725.8ms/step | 27.6 samples/s +Step 39370/44000 | Loss: 2.9421 | LR: 1.40e-05 | 725.6ms/step | 27.6 samples/s +Step 39380/44000 | Loss: 2.9573 | LR: 1.39e-05 | 727.2ms/step | 27.5 samples/s +Step 39390/44000 | Loss: 2.8962 | LR: 1.39e-05 | 737.3ms/step | 27.1 samples/s +Step 39400/44000 | Loss: 2.9083 | LR: 1.38e-05 | 727.0ms/step | 27.5 samples/s +Step 39410/44000 | Loss: 2.9055 | LR: 1.38e-05 | 726.2ms/step | 27.5 samples/s +Step 39420/44000 | Loss: 2.9258 | LR: 1.37e-05 | 726.5ms/step | 27.5 samples/s +Step 39430/44000 | Loss: 2.8658 | LR: 1.36e-05 | 727.3ms/step | 27.5 samples/s +Step 39440/44000 | Loss: 2.9602 | LR: 1.36e-05 | 726.6ms/step | 27.5 samples/s +Step 39450/44000 | Loss: 2.9731 | LR: 1.35e-05 | 726.4ms/step | 27.5 samples/s +Step 39460/44000 | Loss: 2.9391 | LR: 1.35e-05 | 726.5ms/step | 27.5 samples/s +Step 39470/44000 | Loss: 2.9180 | LR: 1.34e-05 | 733.1ms/step | 27.3 samples/s +Step 39480/44000 | Loss: 2.9514 | LR: 1.34e-05 | 733.1ms/step | 27.3 samples/s +Step 39490/44000 | Loss: 2.9611 | LR: 1.33e-05 | 727.1ms/step | 27.5 samples/s +Step 39500/44000 | Loss: 2.9186 | LR: 1.32e-05 | 726.5ms/step | 27.5 samples/s +Validation at step 39500: Loss = 2.9063, Perplexity = 18.29 +Step 39510/44000 | Loss: 2.9308 | LR: 1.32e-05 | 951.1ms/step | 21.0 samples/s +Step 39520/44000 | Loss: 2.8951 | LR: 1.31e-05 | 726.3ms/step | 27.5 samples/s +Step 39530/44000 | Loss: 2.8821 | LR: 1.31e-05 | 725.7ms/step | 27.6 samples/s +Step 39540/44000 | Loss: 2.9317 | LR: 1.30e-05 | 725.8ms/step | 27.6 samples/s +Step 39550/44000 | Loss: 2.9211 | LR: 1.29e-05 | 725.7ms/step | 27.6 samples/s +Step 39560/44000 | Loss: 2.9468 | LR: 1.29e-05 | 734.7ms/step | 27.2 samples/s +Step 39570/44000 | Loss: 2.8798 | LR: 1.28e-05 | 726.1ms/step | 27.5 samples/s +Step 39580/44000 | Loss: 2.8698 | LR: 1.28e-05 | 725.7ms/step | 27.6 samples/s +Step 39590/44000 | Loss: 2.8765 | LR: 1.27e-05 | 726.3ms/step | 27.5 samples/s +Step 39600/44000 | Loss: 2.9320 | LR: 1.27e-05 | 726.3ms/step | 27.5 samples/s +Step 39610/44000 | Loss: 2.9493 | LR: 1.26e-05 | 726.3ms/step | 27.5 samples/s +Step 39620/44000 | Loss: 2.9204 | LR: 1.25e-05 | 725.8ms/step | 27.6 samples/s +Step 39630/44000 | Loss: 2.9002 | LR: 1.25e-05 | 725.3ms/step | 27.6 samples/s +Step 39640/44000 | Loss: 2.8898 | LR: 1.24e-05 | 729.2ms/step | 27.4 samples/s +Step 39650/44000 | Loss: 2.8643 | LR: 1.24e-05 | 736.2ms/step | 27.2 samples/s +Step 39660/44000 | Loss: 2.8614 | LR: 1.23e-05 | 726.2ms/step | 27.5 samples/s +Step 39670/44000 | Loss: 2.9249 | LR: 1.23e-05 | 726.4ms/step | 27.5 samples/s +Step 39680/44000 | Loss: 2.8712 | LR: 1.22e-05 | 726.1ms/step | 27.5 samples/s +Step 39690/44000 | Loss: 2.8101 | LR: 1.22e-05 | 726.4ms/step | 27.5 samples/s +Step 39700/44000 | Loss: 2.9456 | LR: 1.21e-05 | 726.2ms/step | 27.5 samples/s +Step 39710/44000 | Loss: 2.8791 | LR: 1.20e-05 | 725.7ms/step | 27.6 samples/s +Step 39720/44000 | Loss: 2.9106 | LR: 1.20e-05 | 726.0ms/step | 27.5 samples/s +Step 39730/44000 | Loss: 2.8898 | LR: 1.19e-05 | 730.2ms/step | 27.4 samples/s +Step 39740/44000 | Loss: 2.8917 | LR: 1.19e-05 | 730.1ms/step | 27.4 samples/s +Step 39750/44000 | Loss: 2.8858 | LR: 1.18e-05 | 725.8ms/step | 27.6 samples/s +Validation at step 39750: Loss = 2.9063, Perplexity = 18.29 +Step 39760/44000 | Loss: 2.8876 | LR: 1.18e-05 | 950.2ms/step | 21.0 samples/s +Step 39770/44000 | Loss: 2.9187 | LR: 1.17e-05 | 725.6ms/step | 27.6 samples/s +Step 39780/44000 | Loss: 2.8926 | LR: 1.17e-05 | 725.4ms/step | 27.6 samples/s +Step 39790/44000 | Loss: 2.9385 | LR: 1.16e-05 | 725.9ms/step | 27.6 samples/s +Step 39800/44000 | Loss: 2.9743 | LR: 1.15e-05 | 725.6ms/step | 27.6 samples/s +Step 39810/44000 | Loss: 2.8734 | LR: 1.15e-05 | 725.9ms/step | 27.6 samples/s +Step 39820/44000 | Loss: 2.8681 | LR: 1.14e-05 | 740.0ms/step | 27.0 samples/s +Step 39830/44000 | Loss: 2.9108 | LR: 1.14e-05 | 726.8ms/step | 27.5 samples/s +Step 39840/44000 | Loss: 2.8895 | LR: 1.13e-05 | 726.1ms/step | 27.5 samples/s +Step 39850/44000 | Loss: 2.8894 | LR: 1.13e-05 | 725.6ms/step | 27.6 samples/s +Step 39860/44000 | Loss: 2.8868 | LR: 1.12e-05 | 725.7ms/step | 27.6 samples/s +Step 39870/44000 | Loss: 2.9311 | LR: 1.12e-05 | 725.9ms/step | 27.6 samples/s +Step 39880/44000 | Loss: 2.9306 | LR: 1.11e-05 | 725.8ms/step | 27.6 samples/s +Step 39890/44000 | Loss: 2.9256 | LR: 1.11e-05 | 725.6ms/step | 27.6 samples/s +Step 39900/44000 | Loss: 2.8894 | LR: 1.10e-05 | 731.4ms/step | 27.3 samples/s +Step 39910/44000 | Loss: 2.9125 | LR: 1.10e-05 | 731.1ms/step | 27.4 samples/s +Step 39920/44000 | Loss: 2.9572 | LR: 1.09e-05 | 726.5ms/step | 27.5 samples/s +Step 39930/44000 | Loss: 2.9623 | LR: 1.08e-05 | 725.7ms/step | 27.6 samples/s +Step 39940/44000 | Loss: 2.8420 | LR: 1.08e-05 | 725.6ms/step | 27.6 samples/s +Step 39950/44000 | Loss: 2.9053 | LR: 1.07e-05 | 725.4ms/step | 27.6 samples/s +Step 39960/44000 | Loss: 2.8835 | LR: 1.07e-05 | 726.1ms/step | 27.5 samples/s +Step 39970/44000 | Loss: 2.9223 | LR: 1.06e-05 | 725.8ms/step | 27.6 samples/s +Step 39980/44000 | Loss: 2.8997 | LR: 1.06e-05 | 725.8ms/step | 27.6 samples/s +Step 39990/44000 | Loss: 2.9213 | LR: 1.05e-05 | 730.2ms/step | 27.4 samples/s +Step 40000/44000 | Loss: 2.9168 | LR: 1.05e-05 | 730.8ms/step | 27.4 samples/s +Validation at step 40000: Loss = 2.9063, Perplexity = 18.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-40000 +Step 40010/44000 | Loss: 2.9405 | LR: 1.04e-05 | 1451.3ms/step | 13.8 samples/s +Step 40020/44000 | Loss: 2.9223 | LR: 1.04e-05 | 725.6ms/step | 27.6 samples/s +Step 40030/44000 | Loss: 2.8944 | LR: 1.03e-05 | 725.5ms/step | 27.6 samples/s +Step 40040/44000 | Loss: 2.9118 | LR: 1.03e-05 | 725.9ms/step | 27.6 samples/s +Step 40050/44000 | Loss: 2.9343 | LR: 1.02e-05 | 725.4ms/step | 27.6 samples/s +Step 40060/44000 | Loss: 2.8818 | LR: 1.02e-05 | 726.0ms/step | 27.5 samples/s +Step 40070/44000 | Loss: 2.9018 | LR: 1.01e-05 | 735.9ms/step | 27.2 samples/s +Step 40080/44000 | Loss: 2.9202 | LR: 1.01e-05 | 726.6ms/step | 27.5 samples/s +Step 40090/44000 | Loss: 2.9386 | LR: 1.00e-05 | 726.2ms/step | 27.5 samples/s +Step 40100/44000 | Loss: 2.8998 | LR: 9.96e-06 | 726.0ms/step | 27.5 samples/s +Step 40110/44000 | Loss: 2.9513 | LR: 9.91e-06 | 726.2ms/step | 27.5 samples/s +Step 40120/44000 | Loss: 2.9167 | LR: 9.86e-06 | 725.5ms/step | 27.6 samples/s +Step 40130/44000 | Loss: 2.8925 | LR: 9.81e-06 | 726.7ms/step | 27.5 samples/s +Step 40140/44000 | Loss: 2.9441 | LR: 9.76e-06 | 726.2ms/step | 27.5 samples/s +Step 40150/44000 | Loss: 2.9171 | LR: 9.71e-06 | 725.7ms/step | 27.6 samples/s +Step 40160/44000 | Loss: 2.9405 | LR: 9.66e-06 | 739.4ms/step | 27.0 samples/s +Step 40170/44000 | Loss: 2.9366 | LR: 9.61e-06 | 727.9ms/step | 27.5 samples/s +Step 40180/44000 | Loss: 2.9740 | LR: 9.56e-06 | 726.7ms/step | 27.5 samples/s +Step 40190/44000 | Loss: 2.8876 | LR: 9.51e-06 | 726.0ms/step | 27.5 samples/s +Step 40200/44000 | Loss: 2.9191 | LR: 9.46e-06 | 726.2ms/step | 27.5 samples/s +Step 40210/44000 | Loss: 2.9297 | LR: 9.41e-06 | 726.0ms/step | 27.5 samples/s +Step 40220/44000 | Loss: 2.9857 | LR: 9.36e-06 | 725.8ms/step | 27.6 samples/s +Step 40230/44000 | Loss: 2.9417 | LR: 9.32e-06 | 725.8ms/step | 27.6 samples/s +Step 40240/44000 | Loss: 2.9344 | LR: 9.27e-06 | 730.1ms/step | 27.4 samples/s +Step 40250/44000 | Loss: 2.9190 | LR: 9.22e-06 | 730.3ms/step | 27.4 samples/s +Validation at step 40250: Loss = 2.9062, Perplexity = 18.29 +Step 40260/44000 | Loss: 2.8355 | LR: 9.17e-06 | 949.7ms/step | 21.1 samples/s +Step 40270/44000 | Loss: 2.9184 | LR: 9.12e-06 | 725.7ms/step | 27.6 samples/s +Step 40280/44000 | Loss: 2.9446 | LR: 9.07e-06 | 725.7ms/step | 27.6 samples/s +Step 40290/44000 | Loss: 2.9138 | LR: 9.02e-06 | 726.0ms/step | 27.6 samples/s +Step 40300/44000 | Loss: 2.8990 | LR: 8.97e-06 | 725.8ms/step | 27.6 samples/s +Step 40310/44000 | Loss: 2.8892 | LR: 8.93e-06 | 726.1ms/step | 27.5 samples/s +Step 40320/44000 | Loss: 2.9648 | LR: 8.88e-06 | 726.0ms/step | 27.5 samples/s +Step 40330/44000 | Loss: 2.9696 | LR: 8.83e-06 | 735.3ms/step | 27.2 samples/s +Step 40340/44000 | Loss: 2.8796 | LR: 8.78e-06 | 725.5ms/step | 27.6 samples/s +Step 40350/44000 | Loss: 2.9431 | LR: 8.74e-06 | 726.2ms/step | 27.5 samples/s +Step 40360/44000 | Loss: 2.9178 | LR: 8.69e-06 | 726.0ms/step | 27.5 samples/s +Step 40370/44000 | Loss: 2.9076 | LR: 8.64e-06 | 726.9ms/step | 27.5 samples/s +Step 40380/44000 | Loss: 2.9328 | LR: 8.59e-06 | 726.3ms/step | 27.5 samples/s +Step 40390/44000 | Loss: 2.9409 | LR: 8.55e-06 | 725.8ms/step | 27.6 samples/s +Step 40400/44000 | Loss: 2.8548 | LR: 8.50e-06 | 726.0ms/step | 27.5 samples/s +Step 40410/44000 | Loss: 2.8690 | LR: 8.45e-06 | 725.8ms/step | 27.6 samples/s +Step 40420/44000 | Loss: 2.9769 | LR: 8.41e-06 | 737.4ms/step | 27.1 samples/s +Step 40430/44000 | Loss: 2.9030 | LR: 8.36e-06 | 726.0ms/step | 27.5 samples/s +Step 40440/44000 | Loss: 2.9047 | LR: 8.31e-06 | 727.0ms/step | 27.5 samples/s +Step 40450/44000 | Loss: 2.8831 | LR: 8.27e-06 | 726.4ms/step | 27.5 samples/s +Step 40460/44000 | Loss: 2.9987 | LR: 8.22e-06 | 726.2ms/step | 27.5 samples/s +Step 40470/44000 | Loss: 2.9354 | LR: 8.17e-06 | 727.0ms/step | 27.5 samples/s +Step 40480/44000 | Loss: 2.8834 | LR: 8.13e-06 | 727.4ms/step | 27.5 samples/s +Step 40490/44000 | Loss: 2.8592 | LR: 8.08e-06 | 726.8ms/step | 27.5 samples/s +Step 40500/44000 | Loss: 2.8682 | LR: 8.04e-06 | 729.9ms/step | 27.4 samples/s +Validation at step 40500: Loss = 2.9063, Perplexity = 18.29 +Step 40510/44000 | Loss: 2.9356 | LR: 7.99e-06 | 956.6ms/step | 20.9 samples/s +Step 40520/44000 | Loss: 2.8444 | LR: 7.94e-06 | 726.2ms/step | 27.5 samples/s +Step 40530/44000 | Loss: 2.9562 | LR: 7.90e-06 | 726.6ms/step | 27.5 samples/s +Step 40540/44000 | Loss: 2.9120 | LR: 7.85e-06 | 726.7ms/step | 27.5 samples/s +Step 40550/44000 | Loss: 2.8920 | LR: 7.81e-06 | 726.1ms/step | 27.5 samples/s +Step 40560/44000 | Loss: 2.8295 | LR: 7.76e-06 | 726.8ms/step | 27.5 samples/s +Step 40570/44000 | Loss: 2.8915 | LR: 7.72e-06 | 727.1ms/step | 27.5 samples/s +Step 40580/44000 | Loss: 2.9036 | LR: 7.67e-06 | 725.8ms/step | 27.6 samples/s +Step 40590/44000 | Loss: 2.9088 | LR: 7.63e-06 | 735.5ms/step | 27.2 samples/s +Step 40600/44000 | Loss: 2.9263 | LR: 7.59e-06 | 727.0ms/step | 27.5 samples/s +Step 40610/44000 | Loss: 2.9867 | LR: 7.54e-06 | 725.8ms/step | 27.6 samples/s +Step 40620/44000 | Loss: 2.9103 | LR: 7.50e-06 | 725.7ms/step | 27.6 samples/s +Step 40630/44000 | Loss: 2.8973 | LR: 7.45e-06 | 725.5ms/step | 27.6 samples/s +Step 40640/44000 | Loss: 2.9218 | LR: 7.41e-06 | 726.0ms/step | 27.5 samples/s +Step 40650/44000 | Loss: 2.9428 | LR: 7.37e-06 | 726.2ms/step | 27.5 samples/s +Step 40660/44000 | Loss: 2.9235 | LR: 7.32e-06 | 726.7ms/step | 27.5 samples/s +Step 40670/44000 | Loss: 2.8708 | LR: 7.28e-06 | 731.8ms/step | 27.3 samples/s +Step 40680/44000 | Loss: 2.9506 | LR: 7.23e-06 | 731.1ms/step | 27.4 samples/s +Step 40690/44000 | Loss: 2.9484 | LR: 7.19e-06 | 725.1ms/step | 27.6 samples/s +Step 40700/44000 | Loss: 2.9069 | LR: 7.15e-06 | 725.6ms/step | 27.6 samples/s +Step 40710/44000 | Loss: 2.9020 | LR: 7.10e-06 | 725.8ms/step | 27.6 samples/s +Step 40720/44000 | Loss: 2.9784 | LR: 7.06e-06 | 725.8ms/step | 27.6 samples/s +Step 40730/44000 | Loss: 2.9142 | LR: 7.02e-06 | 726.1ms/step | 27.5 samples/s +Step 40740/44000 | Loss: 2.9100 | LR: 6.98e-06 | 726.7ms/step | 27.5 samples/s +Step 40750/44000 | Loss: 2.8739 | LR: 6.93e-06 | 726.2ms/step | 27.5 samples/s +Validation at step 40750: Loss = 2.9062, Perplexity = 18.29 +Step 40760/44000 | Loss: 2.9051 | LR: 6.89e-06 | 965.4ms/step | 20.7 samples/s +Step 40770/44000 | Loss: 2.9562 | LR: 6.85e-06 | 726.1ms/step | 27.5 samples/s +Step 40780/44000 | Loss: 2.9288 | LR: 6.81e-06 | 725.7ms/step | 27.6 samples/s +Step 40790/44000 | Loss: 2.9106 | LR: 6.77e-06 | 725.7ms/step | 27.6 samples/s +Step 40800/44000 | Loss: 2.9049 | LR: 6.72e-06 | 725.9ms/step | 27.6 samples/s +Step 40810/44000 | Loss: 3.0159 | LR: 6.68e-06 | 726.9ms/step | 27.5 samples/s +Step 40820/44000 | Loss: 2.9033 | LR: 6.64e-06 | 727.0ms/step | 27.5 samples/s +Step 40830/44000 | Loss: 2.9027 | LR: 6.60e-06 | 727.3ms/step | 27.5 samples/s +Step 40840/44000 | Loss: 2.8604 | LR: 6.56e-06 | 726.8ms/step | 27.5 samples/s +Step 40850/44000 | Loss: 2.9984 | LR: 6.52e-06 | 735.9ms/step | 27.2 samples/s +Step 40860/44000 | Loss: 2.8753 | LR: 6.47e-06 | 726.1ms/step | 27.5 samples/s +Step 40870/44000 | Loss: 2.9086 | LR: 6.43e-06 | 725.9ms/step | 27.6 samples/s +Step 40880/44000 | Loss: 2.9104 | LR: 6.39e-06 | 725.6ms/step | 27.6 samples/s +Step 40890/44000 | Loss: 2.9915 | LR: 6.35e-06 | 725.6ms/step | 27.6 samples/s +Step 40900/44000 | Loss: 2.9657 | LR: 6.31e-06 | 726.4ms/step | 27.5 samples/s +Step 40910/44000 | Loss: 2.8020 | LR: 6.27e-06 | 726.6ms/step | 27.5 samples/s +Step 40920/44000 | Loss: 2.9269 | LR: 6.23e-06 | 726.0ms/step | 27.5 samples/s +Step 40930/44000 | Loss: 3.0038 | LR: 6.19e-06 | 730.3ms/step | 27.4 samples/s +Step 40940/44000 | Loss: 2.9540 | LR: 6.15e-06 | 732.0ms/step | 27.3 samples/s +Step 40950/44000 | Loss: 2.7814 | LR: 6.11e-06 | 725.4ms/step | 27.6 samples/s +Step 40960/44000 | Loss: 2.9224 | LR: 6.07e-06 | 725.8ms/step | 27.6 samples/s +Step 40970/44000 | Loss: 2.9314 | LR: 6.03e-06 | 726.9ms/step | 27.5 samples/s +Step 40980/44000 | Loss: 2.9381 | LR: 5.99e-06 | 726.1ms/step | 27.5 samples/s +Step 40990/44000 | Loss: 2.8503 | LR: 5.95e-06 | 726.9ms/step | 27.5 samples/s +Step 41000/44000 | Loss: 2.8342 | LR: 5.91e-06 | 726.4ms/step | 27.5 samples/s +Validation at step 41000: Loss = 2.9062, Perplexity = 18.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-41000 +Step 41010/44000 | Loss: 2.9179 | LR: 5.87e-06 | 1434.6ms/step | 13.9 samples/s +Step 41020/44000 | Loss: 2.9431 | LR: 5.83e-06 | 725.3ms/step | 27.6 samples/s +Step 41030/44000 | Loss: 2.9140 | LR: 5.80e-06 | 725.3ms/step | 27.6 samples/s +Step 41040/44000 | Loss: 2.8893 | LR: 5.76e-06 | 725.2ms/step | 27.6 samples/s +Step 41050/44000 | Loss: 2.8865 | LR: 5.72e-06 | 725.3ms/step | 27.6 samples/s +Step 41060/44000 | Loss: 2.9147 | LR: 5.68e-06 | 725.4ms/step | 27.6 samples/s +Step 41070/44000 | Loss: 2.9111 | LR: 5.64e-06 | 725.6ms/step | 27.6 samples/s +Step 41080/44000 | Loss: 2.9000 | LR: 5.60e-06 | 725.8ms/step | 27.6 samples/s +Step 41090/44000 | Loss: 2.8238 | LR: 5.56e-06 | 725.3ms/step | 27.6 samples/s +Step 41100/44000 | Loss: 2.8768 | LR: 5.53e-06 | 735.0ms/step | 27.2 samples/s +Step 41110/44000 | Loss: 2.9211 | LR: 5.49e-06 | 725.5ms/step | 27.6 samples/s +Step 41120/44000 | Loss: 2.8871 | LR: 5.45e-06 | 725.3ms/step | 27.6 samples/s +Step 41130/44000 | Loss: 2.9386 | LR: 5.41e-06 | 725.3ms/step | 27.6 samples/s +Step 41140/44000 | Loss: 2.9289 | LR: 5.38e-06 | 725.5ms/step | 27.6 samples/s +Step 41150/44000 | Loss: 2.8857 | LR: 5.34e-06 | 725.7ms/step | 27.6 samples/s +Step 41160/44000 | Loss: 3.0165 | LR: 5.30e-06 | 725.3ms/step | 27.6 samples/s +Step 41170/44000 | Loss: 2.8394 | LR: 5.26e-06 | 725.0ms/step | 27.6 samples/s +Step 41180/44000 | Loss: 2.9618 | LR: 5.23e-06 | 725.5ms/step | 27.6 samples/s +Step 41190/44000 | Loss: 2.9198 | LR: 5.19e-06 | 736.5ms/step | 27.2 samples/s +Step 41200/44000 | Loss: 2.9429 | LR: 5.15e-06 | 725.8ms/step | 27.6 samples/s +Step 41210/44000 | Loss: 2.8923 | LR: 5.12e-06 | 726.0ms/step | 27.5 samples/s +Step 41220/44000 | Loss: 2.9282 | LR: 5.08e-06 | 725.4ms/step | 27.6 samples/s +Step 41230/44000 | Loss: 2.9820 | LR: 5.04e-06 | 725.5ms/step | 27.6 samples/s +Step 41240/44000 | Loss: 2.8719 | LR: 5.01e-06 | 726.0ms/step | 27.5 samples/s +Step 41250/44000 | Loss: 2.8698 | LR: 4.97e-06 | 725.9ms/step | 27.6 samples/s +Validation at step 41250: Loss = 2.9063, Perplexity = 18.29 +Step 41260/44000 | Loss: 2.8673 | LR: 4.94e-06 | 949.6ms/step | 21.1 samples/s +Step 41270/44000 | Loss: 2.9620 | LR: 4.90e-06 | 735.9ms/step | 27.2 samples/s +Step 41280/44000 | Loss: 2.8779 | LR: 4.86e-06 | 725.4ms/step | 27.6 samples/s +Step 41290/44000 | Loss: 2.9273 | LR: 4.83e-06 | 725.7ms/step | 27.6 samples/s +Step 41300/44000 | Loss: 2.9705 | LR: 4.79e-06 | 725.9ms/step | 27.6 samples/s +Step 41310/44000 | Loss: 2.9296 | LR: 4.76e-06 | 726.1ms/step | 27.5 samples/s +Step 41320/44000 | Loss: 2.9494 | LR: 4.72e-06 | 725.5ms/step | 27.6 samples/s +Step 41330/44000 | Loss: 2.8864 | LR: 4.69e-06 | 725.7ms/step | 27.6 samples/s +Step 41340/44000 | Loss: 2.9175 | LR: 4.65e-06 | 725.8ms/step | 27.6 samples/s +Step 41350/44000 | Loss: 2.8846 | LR: 4.62e-06 | 725.4ms/step | 27.6 samples/s +Step 41360/44000 | Loss: 2.8872 | LR: 4.58e-06 | 734.4ms/step | 27.2 samples/s +Step 41370/44000 | Loss: 2.9686 | LR: 4.55e-06 | 724.7ms/step | 27.6 samples/s +Step 41380/44000 | Loss: 2.9327 | LR: 4.51e-06 | 725.3ms/step | 27.6 samples/s +Step 41390/44000 | Loss: 2.9029 | LR: 4.48e-06 | 726.4ms/step | 27.5 samples/s +Step 41400/44000 | Loss: 2.8493 | LR: 4.45e-06 | 725.9ms/step | 27.6 samples/s +Step 41410/44000 | Loss: 2.9414 | LR: 4.41e-06 | 727.2ms/step | 27.5 samples/s +Step 41420/44000 | Loss: 2.9281 | LR: 4.38e-06 | 727.0ms/step | 27.5 samples/s +Step 41430/44000 | Loss: 2.9122 | LR: 4.34e-06 | 726.5ms/step | 27.5 samples/s +Step 41440/44000 | Loss: 2.9027 | LR: 4.31e-06 | 732.1ms/step | 27.3 samples/s +Step 41450/44000 | Loss: 2.9370 | LR: 4.28e-06 | 731.9ms/step | 27.3 samples/s +Step 41460/44000 | Loss: 2.8847 | LR: 4.24e-06 | 726.8ms/step | 27.5 samples/s +Step 41470/44000 | Loss: 2.9784 | LR: 4.21e-06 | 726.7ms/step | 27.5 samples/s +Step 41480/44000 | Loss: 2.9181 | LR: 4.18e-06 | 726.3ms/step | 27.5 samples/s +Step 41490/44000 | Loss: 2.9297 | LR: 4.14e-06 | 726.1ms/step | 27.5 samples/s +Step 41500/44000 | Loss: 2.9482 | LR: 4.11e-06 | 726.2ms/step | 27.5 samples/s +Validation at step 41500: Loss = 2.9062, Perplexity = 18.29 +Step 41510/44000 | Loss: 2.8935 | LR: 4.08e-06 | 951.2ms/step | 21.0 samples/s +Step 41520/44000 | Loss: 2.8855 | LR: 4.05e-06 | 727.1ms/step | 27.5 samples/s +Step 41530/44000 | Loss: 2.8640 | LR: 4.01e-06 | 738.0ms/step | 27.1 samples/s +Step 41540/44000 | Loss: 2.8906 | LR: 3.98e-06 | 726.4ms/step | 27.5 samples/s +Step 41550/44000 | Loss: 2.9133 | LR: 3.95e-06 | 726.6ms/step | 27.5 samples/s +Step 41560/44000 | Loss: 2.9140 | LR: 3.92e-06 | 726.4ms/step | 27.5 samples/s +Step 41570/44000 | Loss: 3.0199 | LR: 3.88e-06 | 726.3ms/step | 27.5 samples/s +Step 41580/44000 | Loss: 2.9086 | LR: 3.85e-06 | 726.1ms/step | 27.5 samples/s +Step 41590/44000 | Loss: 2.8703 | LR: 3.82e-06 | 726.9ms/step | 27.5 samples/s +Step 41600/44000 | Loss: 2.9426 | LR: 3.79e-06 | 726.6ms/step | 27.5 samples/s +Step 41610/44000 | Loss: 2.9647 | LR: 3.76e-06 | 727.0ms/step | 27.5 samples/s +Step 41620/44000 | Loss: 2.8979 | LR: 3.73e-06 | 735.2ms/step | 27.2 samples/s +Step 41630/44000 | Loss: 2.8550 | LR: 3.70e-06 | 726.0ms/step | 27.5 samples/s +Step 41640/44000 | Loss: 2.9047 | LR: 3.66e-06 | 726.6ms/step | 27.5 samples/s +Step 41650/44000 | Loss: 2.8711 | LR: 3.63e-06 | 726.9ms/step | 27.5 samples/s +Step 41660/44000 | Loss: 2.9154 | LR: 3.60e-06 | 727.1ms/step | 27.5 samples/s +Step 41670/44000 | Loss: 2.9368 | LR: 3.57e-06 | 726.2ms/step | 27.5 samples/s +Step 41680/44000 | Loss: 2.9222 | LR: 3.54e-06 | 726.8ms/step | 27.5 samples/s +Step 41690/44000 | Loss: 2.9547 | LR: 3.51e-06 | 726.9ms/step | 27.5 samples/s +Step 41700/44000 | Loss: 2.8856 | LR: 3.48e-06 | 732.1ms/step | 27.3 samples/s +Step 41710/44000 | Loss: 2.9376 | LR: 3.45e-06 | 731.2ms/step | 27.4 samples/s +Step 41720/44000 | Loss: 2.9006 | LR: 3.42e-06 | 726.5ms/step | 27.5 samples/s +Step 41730/44000 | Loss: 2.9033 | LR: 3.39e-06 | 726.1ms/step | 27.5 samples/s +Step 41740/44000 | Loss: 2.9660 | LR: 3.36e-06 | 726.1ms/step | 27.5 samples/s +Step 41750/44000 | Loss: 2.8906 | LR: 3.33e-06 | 726.4ms/step | 27.5 samples/s +Validation at step 41750: Loss = 2.9062, Perplexity = 18.29 +Step 41760/44000 | Loss: 2.8880 | LR: 3.30e-06 | 951.7ms/step | 21.0 samples/s +Step 41770/44000 | Loss: 2.8794 | LR: 3.27e-06 | 728.0ms/step | 27.5 samples/s +Step 41780/44000 | Loss: 2.8914 | LR: 3.24e-06 | 726.0ms/step | 27.5 samples/s +Step 41790/44000 | Loss: 2.8984 | LR: 3.21e-06 | 738.4ms/step | 27.1 samples/s +Step 41800/44000 | Loss: 2.9429 | LR: 3.19e-06 | 725.9ms/step | 27.6 samples/s +Step 41810/44000 | Loss: 2.9254 | LR: 3.16e-06 | 726.4ms/step | 27.5 samples/s +Step 41820/44000 | Loss: 2.9533 | LR: 3.13e-06 | 726.1ms/step | 27.5 samples/s +Step 41830/44000 | Loss: 2.9049 | LR: 3.10e-06 | 728.2ms/step | 27.5 samples/s +Step 41840/44000 | Loss: 2.8704 | LR: 3.07e-06 | 725.6ms/step | 27.6 samples/s +Step 41850/44000 | Loss: 2.8629 | LR: 3.04e-06 | 725.4ms/step | 27.6 samples/s +Step 41860/44000 | Loss: 2.9142 | LR: 3.01e-06 | 725.2ms/step | 27.6 samples/s +Step 41870/44000 | Loss: 2.9548 | LR: 2.99e-06 | 730.1ms/step | 27.4 samples/s +Step 41880/44000 | Loss: 2.8710 | LR: 2.96e-06 | 730.4ms/step | 27.4 samples/s +Step 41890/44000 | Loss: 2.9325 | LR: 2.93e-06 | 726.0ms/step | 27.5 samples/s +Step 41900/44000 | Loss: 2.9583 | LR: 2.90e-06 | 725.9ms/step | 27.6 samples/s +Step 41910/44000 | Loss: 2.9834 | LR: 2.88e-06 | 724.9ms/step | 27.6 samples/s +Step 41920/44000 | Loss: 2.9323 | LR: 2.85e-06 | 725.3ms/step | 27.6 samples/s +Step 41930/44000 | Loss: 2.8765 | LR: 2.82e-06 | 725.5ms/step | 27.6 samples/s +Step 41940/44000 | Loss: 2.8910 | LR: 2.79e-06 | 725.7ms/step | 27.6 samples/s +Step 41950/44000 | Loss: 2.9330 | LR: 2.77e-06 | 725.2ms/step | 27.6 samples/s +Step 41960/44000 | Loss: 2.9036 | LR: 2.74e-06 | 732.9ms/step | 27.3 samples/s +Step 41970/44000 | Loss: 2.9292 | LR: 2.71e-06 | 730.3ms/step | 27.4 samples/s +Step 41980/44000 | Loss: 2.9039 | LR: 2.69e-06 | 726.0ms/step | 27.5 samples/s +Step 41990/44000 | Loss: 2.8694 | LR: 2.66e-06 | 725.4ms/step | 27.6 samples/s +Step 42000/44000 | Loss: 2.9456 | LR: 2.63e-06 | 726.0ms/step | 27.5 samples/s +Validation at step 42000: Loss = 2.9061, Perplexity = 18.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-42000 +Step 42010/44000 | Loss: 2.8869 | LR: 2.61e-06 | 1426.0ms/step | 14.0 samples/s +Step 42020/44000 | Loss: 2.9692 | LR: 2.58e-06 | 728.2ms/step | 27.5 samples/s +Step 42030/44000 | Loss: 2.8896 | LR: 2.56e-06 | 728.1ms/step | 27.5 samples/s +Step 42040/44000 | Loss: 2.9533 | LR: 2.53e-06 | 736.6ms/step | 27.2 samples/s +Step 42050/44000 | Loss: 2.9268 | LR: 2.50e-06 | 727.1ms/step | 27.5 samples/s +Step 42060/44000 | Loss: 2.9268 | LR: 2.48e-06 | 726.3ms/step | 27.5 samples/s +Step 42070/44000 | Loss: 2.9214 | LR: 2.45e-06 | 726.7ms/step | 27.5 samples/s +Step 42080/44000 | Loss: 2.8197 | LR: 2.43e-06 | 726.3ms/step | 27.5 samples/s +Step 42090/44000 | Loss: 2.9297 | LR: 2.40e-06 | 728.2ms/step | 27.5 samples/s +Step 42100/44000 | Loss: 2.9199 | LR: 2.38e-06 | 725.3ms/step | 27.6 samples/s +Step 42110/44000 | Loss: 2.9437 | LR: 2.35e-06 | 725.6ms/step | 27.6 samples/s +Step 42120/44000 | Loss: 2.9285 | LR: 2.33e-06 | 726.0ms/step | 27.5 samples/s +Step 42130/44000 | Loss: 3.0222 | LR: 2.30e-06 | 734.6ms/step | 27.2 samples/s +Step 42140/44000 | Loss: 2.9777 | LR: 2.28e-06 | 726.3ms/step | 27.5 samples/s +Step 42150/44000 | Loss: 2.8930 | LR: 2.25e-06 | 726.5ms/step | 27.5 samples/s +Step 42160/44000 | Loss: 2.9077 | LR: 2.23e-06 | 726.6ms/step | 27.5 samples/s +Step 42170/44000 | Loss: 2.9468 | LR: 2.21e-06 | 725.7ms/step | 27.6 samples/s +Step 42180/44000 | Loss: 2.8846 | LR: 2.18e-06 | 726.2ms/step | 27.5 samples/s +Step 42190/44000 | Loss: 2.9191 | LR: 2.16e-06 | 727.2ms/step | 27.5 samples/s +Step 42200/44000 | Loss: 2.8622 | LR: 2.13e-06 | 726.8ms/step | 27.5 samples/s +Step 42210/44000 | Loss: 2.8371 | LR: 2.11e-06 | 734.3ms/step | 27.2 samples/s +Step 42220/44000 | Loss: 2.8822 | LR: 2.09e-06 | 731.5ms/step | 27.3 samples/s +Step 42230/44000 | Loss: 2.9254 | LR: 2.06e-06 | 726.7ms/step | 27.5 samples/s +Step 42240/44000 | Loss: 2.9199 | LR: 2.04e-06 | 725.4ms/step | 27.6 samples/s +Step 42250/44000 | Loss: 2.9213 | LR: 2.02e-06 | 724.9ms/step | 27.6 samples/s +Validation at step 42250: Loss = 2.9062, Perplexity = 18.29 +Step 42260/44000 | Loss: 2.9220 | LR: 1.99e-06 | 949.8ms/step | 21.1 samples/s +Step 42270/44000 | Loss: 2.8711 | LR: 1.97e-06 | 725.1ms/step | 27.6 samples/s +Step 42280/44000 | Loss: 2.8926 | LR: 1.95e-06 | 725.3ms/step | 27.6 samples/s +Step 42290/44000 | Loss: 2.8829 | LR: 1.93e-06 | 725.3ms/step | 27.6 samples/s +Step 42300/44000 | Loss: 2.9489 | LR: 1.90e-06 | 735.7ms/step | 27.2 samples/s +Step 42310/44000 | Loss: 2.9924 | LR: 1.88e-06 | 726.2ms/step | 27.5 samples/s +Step 42320/44000 | Loss: 2.9368 | LR: 1.86e-06 | 726.4ms/step | 27.5 samples/s +Step 42330/44000 | Loss: 2.8791 | LR: 1.84e-06 | 726.3ms/step | 27.5 samples/s +Step 42340/44000 | Loss: 2.9148 | LR: 1.82e-06 | 726.0ms/step | 27.5 samples/s +Step 42350/44000 | Loss: 2.8689 | LR: 1.79e-06 | 726.4ms/step | 27.5 samples/s +Step 42360/44000 | Loss: 2.8289 | LR: 1.77e-06 | 727.0ms/step | 27.5 samples/s +Step 42370/44000 | Loss: 2.9863 | LR: 1.75e-06 | 726.7ms/step | 27.5 samples/s +Step 42380/44000 | Loss: 2.8731 | LR: 1.73e-06 | 726.8ms/step | 27.5 samples/s +Step 42390/44000 | Loss: 2.9299 | LR: 1.71e-06 | 740.4ms/step | 27.0 samples/s +Step 42400/44000 | Loss: 2.9214 | LR: 1.69e-06 | 726.9ms/step | 27.5 samples/s +Step 42410/44000 | Loss: 2.9670 | LR: 1.67e-06 | 726.6ms/step | 27.5 samples/s +Step 42420/44000 | Loss: 2.9464 | LR: 1.64e-06 | 727.9ms/step | 27.5 samples/s +Step 42430/44000 | Loss: 2.9097 | LR: 1.62e-06 | 725.6ms/step | 27.6 samples/s +Step 42440/44000 | Loss: 2.9036 | LR: 1.60e-06 | 726.3ms/step | 27.5 samples/s +Step 42450/44000 | Loss: 2.8892 | LR: 1.58e-06 | 726.4ms/step | 27.5 samples/s +Step 42460/44000 | Loss: 2.9766 | LR: 1.56e-06 | 725.9ms/step | 27.6 samples/s +Step 42470/44000 | Loss: 2.9731 | LR: 1.54e-06 | 730.0ms/step | 27.4 samples/s +Step 42480/44000 | Loss: 2.8929 | LR: 1.52e-06 | 730.9ms/step | 27.4 samples/s +Step 42490/44000 | Loss: 2.9571 | LR: 1.50e-06 | 726.8ms/step | 27.5 samples/s +Step 42500/44000 | Loss: 2.8745 | LR: 1.48e-06 | 726.1ms/step | 27.5 samples/s +Validation at step 42500: Loss = 2.9062, Perplexity = 18.29 +Step 42510/44000 | Loss: 2.9473 | LR: 1.46e-06 | 950.7ms/step | 21.0 samples/s +Step 42520/44000 | Loss: 2.9240 | LR: 1.44e-06 | 726.3ms/step | 27.5 samples/s +Step 42530/44000 | Loss: 2.8809 | LR: 1.42e-06 | 726.9ms/step | 27.5 samples/s +Step 42540/44000 | Loss: 2.9153 | LR: 1.40e-06 | 726.6ms/step | 27.5 samples/s +Step 42550/44000 | Loss: 2.9515 | LR: 1.39e-06 | 726.0ms/step | 27.5 samples/s +Step 42560/44000 | Loss: 2.9531 | LR: 1.37e-06 | 735.4ms/step | 27.2 samples/s +Step 42570/44000 | Loss: 2.8585 | LR: 1.35e-06 | 726.0ms/step | 27.5 samples/s +Step 42580/44000 | Loss: 2.9450 | LR: 1.33e-06 | 726.1ms/step | 27.5 samples/s +Step 42590/44000 | Loss: 2.9149 | LR: 1.31e-06 | 726.2ms/step | 27.5 samples/s +Step 42600/44000 | Loss: 2.8733 | LR: 1.29e-06 | 726.4ms/step | 27.5 samples/s +Step 42610/44000 | Loss: 2.8628 | LR: 1.27e-06 | 726.8ms/step | 27.5 samples/s +Step 42620/44000 | Loss: 2.9258 | LR: 1.25e-06 | 726.1ms/step | 27.5 samples/s +Step 42630/44000 | Loss: 2.9428 | LR: 1.24e-06 | 726.4ms/step | 27.5 samples/s +Step 42640/44000 | Loss: 2.9249 | LR: 1.22e-06 | 726.1ms/step | 27.5 samples/s +Step 42650/44000 | Loss: 2.8852 | LR: 1.20e-06 | 738.8ms/step | 27.1 samples/s +Step 42660/44000 | Loss: 2.9276 | LR: 1.18e-06 | 726.8ms/step | 27.5 samples/s +Step 42670/44000 | Loss: 2.9000 | LR: 1.17e-06 | 725.8ms/step | 27.6 samples/s +Step 42680/44000 | Loss: 2.9179 | LR: 1.15e-06 | 726.3ms/step | 27.5 samples/s +Step 42690/44000 | Loss: 2.8901 | LR: 1.13e-06 | 726.2ms/step | 27.5 samples/s +Step 42700/44000 | Loss: 2.8675 | LR: 1.11e-06 | 726.2ms/step | 27.5 samples/s +Step 42710/44000 | Loss: 2.8562 | LR: 1.10e-06 | 725.9ms/step | 27.6 samples/s +Step 42720/44000 | Loss: 2.9576 | LR: 1.08e-06 | 725.9ms/step | 27.6 samples/s +Step 42730/44000 | Loss: 2.8435 | LR: 1.06e-06 | 730.7ms/step | 27.4 samples/s +Step 42740/44000 | Loss: 2.9595 | LR: 1.05e-06 | 731.7ms/step | 27.3 samples/s +Step 42750/44000 | Loss: 2.9070 | LR: 1.03e-06 | 725.8ms/step | 27.6 samples/s +Validation at step 42750: Loss = 2.9062, Perplexity = 18.29 +Step 42760/44000 | Loss: 2.9102 | LR: 1.01e-06 | 949.8ms/step | 21.1 samples/s +Step 42770/44000 | Loss: 2.9227 | LR: 9.97e-07 | 726.0ms/step | 27.5 samples/s +Step 42780/44000 | Loss: 2.9382 | LR: 9.81e-07 | 725.7ms/step | 27.6 samples/s +Step 42790/44000 | Loss: 2.8543 | LR: 9.65e-07 | 726.3ms/step | 27.5 samples/s +Step 42800/44000 | Loss: 2.9298 | LR: 9.49e-07 | 726.1ms/step | 27.5 samples/s +Step 42810/44000 | Loss: 2.9497 | LR: 9.33e-07 | 726.0ms/step | 27.5 samples/s +Step 42820/44000 | Loss: 2.9704 | LR: 9.18e-07 | 736.6ms/step | 27.2 samples/s +Step 42830/44000 | Loss: 2.8851 | LR: 9.02e-07 | 726.1ms/step | 27.5 samples/s +Step 42840/44000 | Loss: 2.8755 | LR: 8.87e-07 | 725.9ms/step | 27.6 samples/s +Step 42850/44000 | Loss: 2.9019 | LR: 8.72e-07 | 727.6ms/step | 27.5 samples/s +Step 42860/44000 | Loss: 2.9091 | LR: 8.57e-07 | 726.3ms/step | 27.5 samples/s +Step 42870/44000 | Loss: 2.9599 | LR: 8.42e-07 | 726.5ms/step | 27.5 samples/s +Step 42880/44000 | Loss: 2.9264 | LR: 8.27e-07 | 727.0ms/step | 27.5 samples/s +Step 42890/44000 | Loss: 2.9544 | LR: 8.12e-07 | 726.9ms/step | 27.5 samples/s +Step 42900/44000 | Loss: 2.8446 | LR: 7.98e-07 | 731.6ms/step | 27.3 samples/s +Step 42910/44000 | Loss: 2.8555 | LR: 7.83e-07 | 735.1ms/step | 27.2 samples/s +Step 42920/44000 | Loss: 2.8569 | LR: 7.69e-07 | 726.5ms/step | 27.5 samples/s +Step 42930/44000 | Loss: 2.9067 | LR: 7.55e-07 | 726.4ms/step | 27.5 samples/s +Step 42940/44000 | Loss: 2.9456 | LR: 7.41e-07 | 727.3ms/step | 27.5 samples/s +Step 42950/44000 | Loss: 2.9006 | LR: 7.27e-07 | 726.5ms/step | 27.5 samples/s +Step 42960/44000 | Loss: 2.9265 | LR: 7.13e-07 | 727.5ms/step | 27.5 samples/s +Step 42970/44000 | Loss: 2.8419 | LR: 6.99e-07 | 726.1ms/step | 27.5 samples/s +Step 42980/44000 | Loss: 2.8719 | LR: 6.86e-07 | 726.0ms/step | 27.5 samples/s +Step 42990/44000 | Loss: 2.8998 | LR: 6.72e-07 | 735.4ms/step | 27.2 samples/s +Step 43000/44000 | Loss: 2.9303 | LR: 6.59e-07 | 726.0ms/step | 27.5 samples/s +Validation at step 43000: Loss = 2.9062, Perplexity = 18.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-43000 +Step 43010/44000 | Loss: 2.8419 | LR: 6.46e-07 | 1369.8ms/step | 14.6 samples/s +Step 43020/44000 | Loss: 2.9134 | LR: 6.33e-07 | 726.5ms/step | 27.5 samples/s +Step 43030/44000 | Loss: 2.8823 | LR: 6.20e-07 | 725.9ms/step | 27.6 samples/s +Step 43040/44000 | Loss: 2.9578 | LR: 6.08e-07 | 726.6ms/step | 27.5 samples/s +Step 43050/44000 | Loss: 2.9617 | LR: 5.95e-07 | 725.8ms/step | 27.6 samples/s +Step 43060/44000 | Loss: 2.8685 | LR: 5.83e-07 | 725.9ms/step | 27.6 samples/s +Step 43070/44000 | Loss: 2.9398 | LR: 5.70e-07 | 735.2ms/step | 27.2 samples/s +Step 43080/44000 | Loss: 2.9409 | LR: 5.58e-07 | 726.0ms/step | 27.5 samples/s +Step 43090/44000 | Loss: 2.9030 | LR: 5.46e-07 | 725.6ms/step | 27.6 samples/s +Step 43100/44000 | Loss: 2.8969 | LR: 5.34e-07 | 726.0ms/step | 27.5 samples/s +Step 43110/44000 | Loss: 2.8724 | LR: 5.22e-07 | 726.2ms/step | 27.5 samples/s +Step 43120/44000 | Loss: 2.9113 | LR: 5.11e-07 | 725.7ms/step | 27.6 samples/s +Step 43130/44000 | Loss: 2.9867 | LR: 4.99e-07 | 725.7ms/step | 27.6 samples/s +Step 43140/44000 | Loss: 2.9403 | LR: 4.88e-07 | 726.1ms/step | 27.5 samples/s +Step 43150/44000 | Loss: 2.9081 | LR: 4.76e-07 | 726.4ms/step | 27.5 samples/s +Step 43160/44000 | Loss: 2.9070 | LR: 4.65e-07 | 734.7ms/step | 27.2 samples/s +Step 43170/44000 | Loss: 2.8646 | LR: 4.54e-07 | 727.9ms/step | 27.5 samples/s +Step 43180/44000 | Loss: 2.8965 | LR: 4.43e-07 | 726.5ms/step | 27.5 samples/s +Step 43190/44000 | Loss: 2.9343 | LR: 4.33e-07 | 726.4ms/step | 27.5 samples/s +Step 43200/44000 | Loss: 2.8827 | LR: 4.22e-07 | 726.2ms/step | 27.5 samples/s +Step 43210/44000 | Loss: 2.9852 | LR: 4.12e-07 | 726.5ms/step | 27.5 samples/s +Step 43220/44000 | Loss: 2.9103 | LR: 4.01e-07 | 726.0ms/step | 27.5 samples/s +Step 43230/44000 | Loss: 2.9106 | LR: 3.91e-07 | 726.2ms/step | 27.5 samples/s +Step 43240/44000 | Loss: 2.9721 | LR: 3.81e-07 | 733.9ms/step | 27.3 samples/s +Step 43250/44000 | Loss: 2.9490 | LR: 3.71e-07 | 732.8ms/step | 27.3 samples/s +Validation at step 43250: Loss = 2.9062, Perplexity = 18.29 +Step 43260/44000 | Loss: 2.9521 | LR: 3.61e-07 | 950.4ms/step | 21.0 samples/s +Step 43270/44000 | Loss: 2.9556 | LR: 3.51e-07 | 726.5ms/step | 27.5 samples/s +Step 43280/44000 | Loss: 2.9341 | LR: 3.42e-07 | 726.5ms/step | 27.5 samples/s +Step 43290/44000 | Loss: 2.9147 | LR: 3.32e-07 | 727.1ms/step | 27.5 samples/s +Step 43300/44000 | Loss: 2.9491 | LR: 3.23e-07 | 727.0ms/step | 27.5 samples/s +Step 43310/44000 | Loss: 2.9329 | LR: 3.14e-07 | 726.6ms/step | 27.5 samples/s +Step 43320/44000 | Loss: 2.8220 | LR: 3.05e-07 | 726.0ms/step | 27.5 samples/s +Step 43330/44000 | Loss: 2.8747 | LR: 2.96e-07 | 739.5ms/step | 27.0 samples/s +Step 43340/44000 | Loss: 2.9113 | LR: 2.87e-07 | 726.5ms/step | 27.5 samples/s +Step 43350/44000 | Loss: 2.8921 | LR: 2.79e-07 | 727.3ms/step | 27.5 samples/s +Step 43360/44000 | Loss: 2.8834 | LR: 2.70e-07 | 726.5ms/step | 27.5 samples/s +Step 43370/44000 | Loss: 2.9344 | LR: 2.62e-07 | 725.6ms/step | 27.6 samples/s +Step 43380/44000 | Loss: 2.9248 | LR: 2.53e-07 | 726.4ms/step | 27.5 samples/s +Step 43390/44000 | Loss: 2.9152 | LR: 2.45e-07 | 725.8ms/step | 27.6 samples/s +Step 43400/44000 | Loss: 2.9180 | LR: 2.37e-07 | 725.7ms/step | 27.6 samples/s +Step 43410/44000 | Loss: 2.9593 | LR: 2.30e-07 | 726.0ms/step | 27.5 samples/s +Step 43420/44000 | Loss: 2.8610 | LR: 2.22e-07 | 736.1ms/step | 27.2 samples/s +Step 43430/44000 | Loss: 2.9390 | LR: 2.14e-07 | 726.3ms/step | 27.5 samples/s +Step 43440/44000 | Loss: 2.8691 | LR: 2.07e-07 | 726.1ms/step | 27.5 samples/s +Step 43450/44000 | Loss: 2.8666 | LR: 1.99e-07 | 725.9ms/step | 27.6 samples/s +Step 43460/44000 | Loss: 2.8896 | LR: 1.92e-07 | 725.9ms/step | 27.6 samples/s +Step 43470/44000 | Loss: 2.8585 | LR: 1.85e-07 | 726.4ms/step | 27.5 samples/s +Step 43480/44000 | Loss: 2.9269 | LR: 1.78e-07 | 725.9ms/step | 27.6 samples/s +Step 43490/44000 | Loss: 2.9439 | LR: 1.72e-07 | 726.6ms/step | 27.5 samples/s +Step 43500/44000 | Loss: 2.9190 | LR: 1.65e-07 | 730.4ms/step | 27.4 samples/s +Validation at step 43500: Loss = 2.9062, Perplexity = 18.29 +Step 43510/44000 | Loss: 2.9066 | LR: 1.58e-07 | 955.7ms/step | 20.9 samples/s +Step 43520/44000 | Loss: 2.8738 | LR: 1.52e-07 | 725.9ms/step | 27.6 samples/s +Step 43530/44000 | Loss: 2.9144 | LR: 1.46e-07 | 726.0ms/step | 27.5 samples/s +Step 43540/44000 | Loss: 2.9064 | LR: 1.40e-07 | 725.8ms/step | 27.6 samples/s +Step 43550/44000 | Loss: 2.8908 | LR: 1.34e-07 | 726.1ms/step | 27.5 samples/s +Step 43560/44000 | Loss: 2.9132 | LR: 1.28e-07 | 726.6ms/step | 27.5 samples/s +Step 43570/44000 | Loss: 2.9080 | LR: 1.22e-07 | 726.5ms/step | 27.5 samples/s +Step 43580/44000 | Loss: 2.9184 | LR: 1.16e-07 | 726.0ms/step | 27.5 samples/s +Step 43590/44000 | Loss: 2.9136 | LR: 1.11e-07 | 736.4ms/step | 27.2 samples/s +Step 43600/44000 | Loss: 2.9336 | LR: 1.06e-07 | 726.1ms/step | 27.5 samples/s +Step 43610/44000 | Loss: 2.9034 | LR: 1.00e-07 | 726.0ms/step | 27.5 samples/s +Step 43620/44000 | Loss: 2.9354 | LR: 9.52e-08 | 725.8ms/step | 27.6 samples/s +Step 43630/44000 | Loss: 2.9084 | LR: 9.03e-08 | 725.6ms/step | 27.6 samples/s +Step 43640/44000 | Loss: 2.9122 | LR: 8.55e-08 | 725.8ms/step | 27.6 samples/s +Step 43650/44000 | Loss: 2.8799 | LR: 8.08e-08 | 726.0ms/step | 27.5 samples/s +Step 43660/44000 | Loss: 2.8867 | LR: 7.62e-08 | 725.9ms/step | 27.6 samples/s +Step 43670/44000 | Loss: 2.9418 | LR: 7.18e-08 | 730.5ms/step | 27.4 samples/s +Step 43680/44000 | Loss: 2.8794 | LR: 6.75e-08 | 730.1ms/step | 27.4 samples/s +Step 43690/44000 | Loss: 2.8720 | LR: 6.34e-08 | 726.2ms/step | 27.5 samples/s +Step 43700/44000 | Loss: 2.9420 | LR: 5.94e-08 | 727.3ms/step | 27.5 samples/s +Step 43710/44000 | Loss: 2.9022 | LR: 5.55e-08 | 726.2ms/step | 27.5 samples/s +Step 43720/44000 | Loss: 2.9088 | LR: 5.17e-08 | 726.2ms/step | 27.5 samples/s +Step 43730/44000 | Loss: 2.9213 | LR: 4.81e-08 | 725.9ms/step | 27.6 samples/s +Step 43740/44000 | Loss: 2.8669 | LR: 4.46e-08 | 726.5ms/step | 27.5 samples/s +Step 43750/44000 | Loss: 2.8183 | LR: 4.12e-08 | 726.3ms/step | 27.5 samples/s +Validation at step 43750: Loss = 2.9062, Perplexity = 18.29 +Step 43760/44000 | Loss: 2.8490 | LR: 3.80e-08 | 962.6ms/step | 20.8 samples/s +Step 43770/44000 | Loss: 2.9127 | LR: 3.49e-08 | 725.9ms/step | 27.6 samples/s +Step 43780/44000 | Loss: 2.9891 | LR: 3.19e-08 | 725.8ms/step | 27.6 samples/s +Step 43790/44000 | Loss: 2.8425 | LR: 2.91e-08 | 726.8ms/step | 27.5 samples/s +Step 43800/44000 | Loss: 2.9627 | LR: 2.64e-08 | 728.0ms/step | 27.5 samples/s +Step 43810/44000 | Loss: 2.8922 | LR: 2.38e-08 | 725.9ms/step | 27.6 samples/s +Step 43820/44000 | Loss: 2.9409 | LR: 2.14e-08 | 726.6ms/step | 27.5 samples/s +Step 43830/44000 | Loss: 2.9322 | LR: 1.91e-08 | 726.6ms/step | 27.5 samples/s +Step 43840/44000 | Loss: 2.9542 | LR: 1.69e-08 | 726.3ms/step | 27.5 samples/s +Step 43850/44000 | Loss: 2.8777 | LR: 1.48e-08 | 736.3ms/step | 27.2 samples/s +Step 43860/44000 | Loss: 2.9567 | LR: 1.29e-08 | 726.5ms/step | 27.5 samples/s +Step 43870/44000 | Loss: 2.9652 | LR: 1.11e-08 | 727.0ms/step | 27.5 samples/s +Step 43880/44000 | Loss: 2.9275 | LR: 9.50e-09 | 726.3ms/step | 27.5 samples/s +Step 43890/44000 | Loss: 2.9184 | LR: 7.98e-09 | 726.4ms/step | 27.5 samples/s +Step 43900/44000 | Loss: 2.8964 | LR: 6.60e-09 | 726.3ms/step | 27.5 samples/s +Step 43910/44000 | Loss: 2.9141 | LR: 5.34e-09 | 726.5ms/step | 27.5 samples/s +Step 43920/44000 | Loss: 2.9637 | LR: 4.22e-09 | 726.3ms/step | 27.5 samples/s +Step 43930/44000 | Loss: 2.9266 | LR: 3.23e-09 | 732.7ms/step | 27.3 samples/s +Step 43940/44000 | Loss: 2.8865 | LR: 2.37e-09 | 732.7ms/step | 27.3 samples/s +Step 43950/44000 | Loss: 2.8905 | LR: 1.65e-09 | 726.4ms/step | 27.5 samples/s +Step 43960/44000 | Loss: 2.8715 | LR: 1.06e-09 | 728.5ms/step | 27.5 samples/s +Step 43970/44000 | Loss: 2.9670 | LR: 5.94e-10 | 726.3ms/step | 27.5 samples/s +Step 43980/44000 | Loss: 2.8625 | LR: 2.64e-10 | 727.1ms/step | 27.5 samples/s +Step 43990/44000 | Loss: 2.9032 | LR: 6.60e-11 | 726.8ms/step | 27.5 samples/s +Step 44000/44000 | Loss: 2.8961 | LR: 0.00e+00 | 726.4ms/step | 27.5 samples/s +Validation at step 44000: Loss = 2.9062, Perplexity = 18.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed46/run_20260822_043537/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▅▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▄▃▃▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁█▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec █████████████▁█▇█████████████████▁▁█████ +wandb: speed/steps_per_sec ████▇████▇█▁▁█████▁█▇███▇▁▁█████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.90625 +wandb: eval/perplexity 18.28809 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 726.38793 +wandb: speed/samples_per_sec 27.5335 +wandb: speed/steps_per_sec 1.37667 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 DDP5 365m static 8k nrb35 seed46 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8naq20vs +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_043615-8naq20vs/logs +Training complete! diff --git a/epfl_final/train_logs/p1_ddp5_static_seed45_47/l8k365m_static_nrb35_seed47_a100-train-5-gpu.log b/epfl_final/train_logs/p1_ddp5_static_seed45_47/l8k365m_static_nrb35_seed47_a100-train-5-gpu.log new file mode 100644 index 0000000000000000000000000000000000000000..7a2a37e7a0afc25d6d816421e209e2855c3fd2fe --- /dev/null +++ b/epfl_final/train_logs/p1_ddp5_static_seed45_47/l8k365m_static_nrb35_seed47_a100-train-5-gpu.log @@ -0,0 +1,4859 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 2, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 47, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 DDP5 365m static 8k nrb35 seed47", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "static", + "nrb35", + "seed47", + "ddp5", + "a100" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1", + "sparse_mask_seed": 1, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 5 +Gradient accumulation steps: 2 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 302776 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_043611-xvizh3ih +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 DDP5 365m static 8k nrb35 seed47 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/xvizh3ih +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.1201 | LR: 6.67e-06 | 9280.7ms/step | 2.2 samples/s +Step 20/44000 | Loss: 12.0177 | LR: 1.33e-05 | 706.1ms/step | 28.3 samples/s +Step 30/44000 | Loss: 11.6850 | LR: 2.00e-05 | 705.1ms/step | 28.4 samples/s +Step 40/44000 | Loss: 11.1069 | LR: 2.67e-05 | 722.4ms/step | 27.7 samples/s +Step 50/44000 | Loss: 10.8329 | LR: 3.33e-05 | 703.8ms/step | 28.4 samples/s +Step 60/44000 | Loss: 10.5616 | LR: 4.00e-05 | 704.5ms/step | 28.4 samples/s +Step 70/44000 | Loss: 10.3102 | LR: 4.67e-05 | 704.1ms/step | 28.4 samples/s +Step 80/44000 | Loss: 10.0076 | LR: 5.33e-05 | 703.5ms/step | 28.4 samples/s +Step 90/44000 | Loss: 9.6027 | LR: 6.00e-05 | 705.8ms/step | 28.3 samples/s +Step 100/44000 | Loss: 9.1742 | LR: 6.67e-05 | 706.7ms/step | 28.3 samples/s +Step 110/44000 | Loss: 8.6474 | LR: 7.33e-05 | 705.4ms/step | 28.4 samples/s +Step 120/44000 | Loss: 8.1865 | LR: 8.00e-05 | 706.8ms/step | 28.3 samples/s +Step 130/44000 | Loss: 7.8035 | LR: 8.67e-05 | 722.1ms/step | 27.7 samples/s +Step 140/44000 | Loss: 7.5169 | LR: 9.33e-05 | 705.4ms/step | 28.4 samples/s +Step 150/44000 | Loss: 7.3998 | LR: 1.00e-04 | 706.0ms/step | 28.3 samples/s +Step 160/44000 | Loss: 7.2347 | LR: 1.07e-04 | 706.6ms/step | 28.3 samples/s +Step 170/44000 | Loss: 7.0851 | LR: 1.13e-04 | 708.7ms/step | 28.2 samples/s +Step 180/44000 | Loss: 7.0385 | LR: 1.20e-04 | 707.9ms/step | 28.3 samples/s +Step 190/44000 | Loss: 6.9376 | LR: 1.27e-04 | 706.9ms/step | 28.3 samples/s +Step 200/44000 | Loss: 6.8776 | LR: 1.33e-04 | 707.6ms/step | 28.3 samples/s +Step 210/44000 | Loss: 6.7464 | LR: 1.40e-04 | 707.4ms/step | 28.3 samples/s +Step 220/44000 | Loss: 6.6618 | LR: 1.47e-04 | 725.5ms/step | 27.6 samples/s +Step 230/44000 | Loss: 6.5795 | LR: 1.53e-04 | 709.6ms/step | 28.2 samples/s +Step 240/44000 | Loss: 6.5518 | LR: 1.60e-04 | 709.4ms/step | 28.2 samples/s +Step 250/44000 | Loss: 6.4833 | LR: 1.67e-04 | 710.1ms/step | 28.2 samples/s +Validation at step 250: Loss = 6.4588, Perplexity = 638.32 +Step 260/44000 | Loss: 6.4069 | LR: 1.73e-04 | 4571.1ms/step | 4.4 samples/s +Step 270/44000 | Loss: 6.4113 | LR: 1.80e-04 | 709.2ms/step | 28.2 samples/s +Step 280/44000 | Loss: 6.2709 | LR: 1.87e-04 | 709.3ms/step | 28.2 samples/s +Step 290/44000 | Loss: 6.3572 | LR: 1.93e-04 | 711.2ms/step | 28.1 samples/s +Step 300/44000 | Loss: 6.2559 | LR: 2.00e-04 | 708.7ms/step | 28.2 samples/s +Step 310/44000 | Loss: 6.2077 | LR: 2.07e-04 | 708.0ms/step | 28.2 samples/s +Step 320/44000 | Loss: 6.1400 | LR: 2.13e-04 | 708.0ms/step | 28.2 samples/s +Step 330/44000 | Loss: 6.1180 | LR: 2.20e-04 | 708.9ms/step | 28.2 samples/s +Step 340/44000 | Loss: 6.0428 | LR: 2.27e-04 | 719.6ms/step | 27.8 samples/s +Step 350/44000 | Loss: 6.0603 | LR: 2.33e-04 | 714.3ms/step | 28.0 samples/s +Step 360/44000 | Loss: 5.9264 | LR: 2.40e-04 | 709.3ms/step | 28.2 samples/s +Step 370/44000 | Loss: 5.9770 | LR: 2.47e-04 | 709.4ms/step | 28.2 samples/s +Step 380/44000 | Loss: 5.8851 | LR: 2.53e-04 | 709.2ms/step | 28.2 samples/s +Step 390/44000 | Loss: 5.8235 | LR: 2.60e-04 | 710.2ms/step | 28.2 samples/s +Step 400/44000 | Loss: 5.8796 | LR: 2.67e-04 | 711.6ms/step | 28.1 samples/s +Step 410/44000 | Loss: 5.8162 | LR: 2.73e-04 | 709.8ms/step | 28.2 samples/s +Step 420/44000 | Loss: 5.7395 | LR: 2.80e-04 | 709.9ms/step | 28.2 samples/s +Step 430/44000 | Loss: 5.7395 | LR: 2.87e-04 | 722.4ms/step | 27.7 samples/s +Step 440/44000 | Loss: 5.7008 | LR: 2.93e-04 | 717.3ms/step | 27.9 samples/s +Step 450/44000 | Loss: 5.6914 | LR: 3.00e-04 | 712.5ms/step | 28.1 samples/s +Step 460/44000 | Loss: 5.6297 | LR: 3.07e-04 | 710.9ms/step | 28.1 samples/s +Step 470/44000 | Loss: 5.6154 | LR: 3.13e-04 | 712.6ms/step | 28.1 samples/s +Step 480/44000 | Loss: 5.5714 | LR: 3.20e-04 | 710.4ms/step | 28.2 samples/s +Step 490/44000 | Loss: 5.5372 | LR: 3.27e-04 | 712.1ms/step | 28.1 samples/s +Step 500/44000 | Loss: 5.4235 | LR: 3.33e-04 | 712.6ms/step | 28.1 samples/s +Validation at step 500: Loss = 5.4857, Perplexity = 241.22 +Step 510/44000 | Loss: 5.4343 | LR: 3.40e-04 | 929.7ms/step | 21.5 samples/s +Step 520/44000 | Loss: 5.4303 | LR: 3.47e-04 | 725.4ms/step | 27.6 samples/s +Step 530/44000 | Loss: 5.4149 | LR: 3.53e-04 | 711.4ms/step | 28.1 samples/s +Step 540/44000 | Loss: 5.3742 | LR: 3.60e-04 | 712.1ms/step | 28.1 samples/s +Step 550/44000 | Loss: 5.2831 | LR: 3.67e-04 | 711.5ms/step | 28.1 samples/s +Step 560/44000 | Loss: 5.3568 | LR: 3.73e-04 | 711.2ms/step | 28.1 samples/s +Step 570/44000 | Loss: 5.2573 | LR: 3.80e-04 | 712.2ms/step | 28.1 samples/s +Step 580/44000 | Loss: 5.2247 | LR: 3.87e-04 | 712.3ms/step | 28.1 samples/s +Step 590/44000 | Loss: 5.2288 | LR: 3.93e-04 | 712.2ms/step | 28.1 samples/s +Step 600/44000 | Loss: 5.2678 | LR: 4.00e-04 | 711.9ms/step | 28.1 samples/s +Step 610/44000 | Loss: 5.1854 | LR: 4.07e-04 | 727.3ms/step | 27.5 samples/s +Step 620/44000 | Loss: 5.1410 | LR: 4.13e-04 | 710.6ms/step | 28.1 samples/s +Step 630/44000 | Loss: 5.1152 | LR: 4.20e-04 | 711.6ms/step | 28.1 samples/s +Step 640/44000 | Loss: 5.0764 | LR: 4.27e-04 | 712.7ms/step | 28.1 samples/s +Step 650/44000 | Loss: 5.0583 | LR: 4.33e-04 | 712.0ms/step | 28.1 samples/s +Step 660/44000 | Loss: 4.9470 | LR: 4.40e-04 | 711.3ms/step | 28.1 samples/s +Step 670/44000 | Loss: 4.9464 | LR: 4.47e-04 | 710.6ms/step | 28.1 samples/s +Step 680/44000 | Loss: 4.9646 | LR: 4.53e-04 | 710.9ms/step | 28.1 samples/s +Step 690/44000 | Loss: 4.9272 | LR: 4.60e-04 | 710.8ms/step | 28.1 samples/s +Step 700/44000 | Loss: 4.8988 | LR: 4.67e-04 | 728.5ms/step | 27.5 samples/s +Step 710/44000 | Loss: 4.8618 | LR: 4.73e-04 | 710.4ms/step | 28.2 samples/s +Step 720/44000 | Loss: 4.8534 | LR: 4.80e-04 | 711.4ms/step | 28.1 samples/s +Step 730/44000 | Loss: 4.8011 | LR: 4.87e-04 | 710.8ms/step | 28.1 samples/s +Step 740/44000 | Loss: 4.7838 | LR: 4.93e-04 | 712.0ms/step | 28.1 samples/s +Step 750/44000 | Loss: 4.7328 | LR: 5.00e-04 | 711.5ms/step | 28.1 samples/s +Validation at step 750: Loss = 4.7365, Perplexity = 114.03 +Step 760/44000 | Loss: 4.7186 | LR: 5.00e-04 | 928.1ms/step | 21.5 samples/s +Step 770/44000 | Loss: 4.7116 | LR: 5.00e-04 | 710.2ms/step | 28.2 samples/s +Step 780/44000 | Loss: 4.6690 | LR: 5.00e-04 | 722.6ms/step | 27.7 samples/s +Step 790/44000 | Loss: 4.7105 | LR: 5.00e-04 | 718.0ms/step | 27.9 samples/s +Step 800/44000 | Loss: 4.6207 | LR: 5.00e-04 | 712.7ms/step | 28.1 samples/s +Step 810/44000 | Loss: 4.5882 | LR: 5.00e-04 | 712.6ms/step | 28.1 samples/s +Step 820/44000 | Loss: 4.5758 | LR: 5.00e-04 | 712.1ms/step | 28.1 samples/s +Step 830/44000 | Loss: 4.5799 | LR: 5.00e-04 | 710.5ms/step | 28.1 samples/s +Step 840/44000 | Loss: 4.5653 | LR: 5.00e-04 | 711.2ms/step | 28.1 samples/s +Step 850/44000 | Loss: 4.5138 | LR: 5.00e-04 | 710.3ms/step | 28.2 samples/s +Step 860/44000 | Loss: 4.4806 | LR: 5.00e-04 | 710.9ms/step | 28.1 samples/s +Step 870/44000 | Loss: 4.3705 | LR: 5.00e-04 | 723.8ms/step | 27.6 samples/s +Step 880/44000 | Loss: 4.4814 | LR: 5.00e-04 | 714.8ms/step | 28.0 samples/s +Step 890/44000 | Loss: 4.4391 | LR: 5.00e-04 | 710.9ms/step | 28.1 samples/s +Step 900/44000 | Loss: 4.4331 | LR: 5.00e-04 | 709.9ms/step | 28.2 samples/s +Step 910/44000 | Loss: 4.4630 | LR: 5.00e-04 | 710.7ms/step | 28.1 samples/s +Step 920/44000 | Loss: 4.4270 | LR: 5.00e-04 | 710.5ms/step | 28.1 samples/s +Step 930/44000 | Loss: 4.3836 | LR: 5.00e-04 | 710.2ms/step | 28.2 samples/s +Step 940/44000 | Loss: 4.3531 | LR: 5.00e-04 | 709.7ms/step | 28.2 samples/s +Step 950/44000 | Loss: 4.3367 | LR: 5.00e-04 | 710.1ms/step | 28.2 samples/s +Step 960/44000 | Loss: 4.3452 | LR: 5.00e-04 | 729.6ms/step | 27.4 samples/s +Step 970/44000 | Loss: 4.3451 | LR: 5.00e-04 | 710.0ms/step | 28.2 samples/s +Step 980/44000 | Loss: 4.4006 | LR: 5.00e-04 | 710.9ms/step | 28.1 samples/s +Step 990/44000 | Loss: 4.2521 | LR: 5.00e-04 | 712.9ms/step | 28.1 samples/s +Step 1000/44000 | Loss: 4.3073 | LR: 5.00e-04 | 710.3ms/step | 28.2 samples/s +Validation at step 1000: Loss = 4.2708, Perplexity = 71.58 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-1000 +Step 1010/44000 | Loss: 4.2722 | LR: 5.00e-04 | 1257.2ms/step | 15.9 samples/s +Step 1020/44000 | Loss: 4.2704 | LR: 5.00e-04 | 711.0ms/step | 28.1 samples/s +Step 1030/44000 | Loss: 4.2350 | LR: 5.00e-04 | 710.7ms/step | 28.1 samples/s +Step 1040/44000 | Loss: 4.3010 | LR: 5.00e-04 | 722.4ms/step | 27.7 samples/s +Step 1050/44000 | Loss: 4.1775 | LR: 5.00e-04 | 716.1ms/step | 27.9 samples/s +Step 1060/44000 | Loss: 4.2309 | LR: 5.00e-04 | 710.4ms/step | 28.2 samples/s +Step 1070/44000 | Loss: 4.2119 | LR: 5.00e-04 | 712.3ms/step | 28.1 samples/s +Step 1080/44000 | Loss: 4.1875 | LR: 5.00e-04 | 712.2ms/step | 28.1 samples/s +Step 1090/44000 | Loss: 4.2167 | LR: 5.00e-04 | 712.6ms/step | 28.1 samples/s +Step 1100/44000 | Loss: 4.1740 | LR: 5.00e-04 | 712.9ms/step | 28.1 samples/s +Step 1110/44000 | Loss: 4.1614 | LR: 5.00e-04 | 712.3ms/step | 28.1 samples/s +Step 1120/44000 | Loss: 4.0830 | LR: 5.00e-04 | 711.4ms/step | 28.1 samples/s +Step 1130/44000 | Loss: 4.1896 | LR: 5.00e-04 | 731.7ms/step | 27.3 samples/s +Step 1140/44000 | Loss: 4.0976 | LR: 5.00e-04 | 711.9ms/step | 28.1 samples/s +Step 1150/44000 | Loss: 4.1229 | LR: 5.00e-04 | 711.3ms/step | 28.1 samples/s +Step 1160/44000 | Loss: 4.1374 | LR: 5.00e-04 | 711.6ms/step | 28.1 samples/s +Step 1170/44000 | Loss: 4.0568 | LR: 5.00e-04 | 710.7ms/step | 28.1 samples/s +Step 1180/44000 | Loss: 4.0782 | LR: 5.00e-04 | 710.7ms/step | 28.1 samples/s +Step 1190/44000 | Loss: 4.1217 | LR: 5.00e-04 | 711.7ms/step | 28.1 samples/s +Step 1200/44000 | Loss: 4.0724 | LR: 5.00e-04 | 710.7ms/step | 28.1 samples/s +Step 1210/44000 | Loss: 4.0702 | LR: 5.00e-04 | 711.5ms/step | 28.1 samples/s +Step 1220/44000 | Loss: 4.0964 | LR: 5.00e-04 | 725.3ms/step | 27.6 samples/s +Step 1230/44000 | Loss: 4.1293 | LR: 5.00e-04 | 711.4ms/step | 28.1 samples/s +Step 1240/44000 | Loss: 4.0360 | LR: 5.00e-04 | 710.9ms/step | 28.1 samples/s +Step 1250/44000 | Loss: 4.0524 | LR: 5.00e-04 | 711.2ms/step | 28.1 samples/s +Validation at step 1250: Loss = 4.0462, Perplexity = 57.18 +Step 1260/44000 | Loss: 4.0858 | LR: 5.00e-04 | 930.1ms/step | 21.5 samples/s +Step 1270/44000 | Loss: 4.0858 | LR: 5.00e-04 | 711.4ms/step | 28.1 samples/s +Step 1280/44000 | Loss: 4.0462 | LR: 5.00e-04 | 711.6ms/step | 28.1 samples/s +Step 1290/44000 | Loss: 4.0545 | LR: 5.00e-04 | 711.7ms/step | 28.1 samples/s +Step 1300/44000 | Loss: 4.0136 | LR: 5.00e-04 | 711.4ms/step | 28.1 samples/s +Step 1310/44000 | Loss: 4.0485 | LR: 5.00e-04 | 728.6ms/step | 27.4 samples/s +Step 1320/44000 | Loss: 4.0038 | LR: 5.00e-04 | 711.1ms/step | 28.1 samples/s +Step 1330/44000 | Loss: 3.9867 | LR: 5.00e-04 | 711.2ms/step | 28.1 samples/s +Step 1340/44000 | Loss: 4.0173 | LR: 5.00e-04 | 711.3ms/step | 28.1 samples/s +Step 1350/44000 | Loss: 4.0291 | LR: 5.00e-04 | 711.2ms/step | 28.1 samples/s +Step 1360/44000 | Loss: 3.9543 | LR: 5.00e-04 | 711.8ms/step | 28.1 samples/s +Step 1370/44000 | Loss: 4.0081 | LR: 5.00e-04 | 712.0ms/step | 28.1 samples/s +Step 1380/44000 | Loss: 3.9517 | LR: 5.00e-04 | 712.0ms/step | 28.1 samples/s +Step 1390/44000 | Loss: 3.9806 | LR: 5.00e-04 | 717.7ms/step | 27.9 samples/s +Step 1400/44000 | Loss: 3.9286 | LR: 5.00e-04 | 723.5ms/step | 27.6 samples/s +Step 1410/44000 | Loss: 3.9232 | LR: 5.00e-04 | 712.4ms/step | 28.1 samples/s +Step 1420/44000 | Loss: 3.9989 | LR: 5.00e-04 | 711.8ms/step | 28.1 samples/s +Step 1430/44000 | Loss: 3.9718 | LR: 5.00e-04 | 712.6ms/step | 28.1 samples/s +Step 1440/44000 | Loss: 3.9389 | LR: 5.00e-04 | 711.4ms/step | 28.1 samples/s +Step 1450/44000 | Loss: 3.8730 | LR: 5.00e-04 | 712.2ms/step | 28.1 samples/s +Step 1460/44000 | Loss: 3.8932 | LR: 5.00e-04 | 712.8ms/step | 28.1 samples/s +Step 1470/44000 | Loss: 3.9743 | LR: 5.00e-04 | 711.7ms/step | 28.1 samples/s +Step 1480/44000 | Loss: 3.9330 | LR: 5.00e-04 | 724.5ms/step | 27.6 samples/s +Step 1490/44000 | Loss: 3.9240 | LR: 5.00e-04 | 717.6ms/step | 27.9 samples/s +Step 1500/44000 | Loss: 3.8653 | LR: 5.00e-04 | 713.1ms/step | 28.0 samples/s +Validation at step 1500: Loss = 3.8898, Perplexity = 48.90 +Step 1510/44000 | Loss: 3.8861 | LR: 5.00e-04 | 930.7ms/step | 21.5 samples/s +Step 1520/44000 | Loss: 3.9045 | LR: 5.00e-04 | 713.0ms/step | 28.1 samples/s +Step 1530/44000 | Loss: 3.8387 | LR: 5.00e-04 | 712.0ms/step | 28.1 samples/s +Step 1540/44000 | Loss: 3.9425 | LR: 5.00e-04 | 712.8ms/step | 28.1 samples/s +Step 1550/44000 | Loss: 3.8450 | LR: 5.00e-04 | 710.9ms/step | 28.1 samples/s +Step 1560/44000 | Loss: 3.8854 | LR: 5.00e-04 | 711.8ms/step | 28.1 samples/s +Step 1570/44000 | Loss: 3.8102 | LR: 5.00e-04 | 728.0ms/step | 27.5 samples/s +Step 1580/44000 | Loss: 3.8726 | LR: 5.00e-04 | 711.9ms/step | 28.1 samples/s +Step 1590/44000 | Loss: 3.8090 | LR: 5.00e-04 | 711.0ms/step | 28.1 samples/s +Step 1600/44000 | Loss: 3.8811 | LR: 5.00e-04 | 711.2ms/step | 28.1 samples/s +Step 1610/44000 | Loss: 3.8527 | LR: 5.00e-04 | 710.6ms/step | 28.1 samples/s +Step 1620/44000 | Loss: 3.8567 | LR: 5.00e-04 | 711.0ms/step | 28.1 samples/s +Step 1630/44000 | Loss: 3.8170 | LR: 4.99e-04 | 711.7ms/step | 28.1 samples/s +Step 1640/44000 | Loss: 3.8241 | LR: 4.99e-04 | 712.6ms/step | 28.1 samples/s +Step 1650/44000 | Loss: 3.8424 | LR: 4.99e-04 | 712.3ms/step | 28.1 samples/s +Step 1660/44000 | Loss: 3.8517 | LR: 4.99e-04 | 724.4ms/step | 27.6 samples/s +Step 1670/44000 | Loss: 3.9103 | LR: 4.99e-04 | 711.6ms/step | 28.1 samples/s +Step 1680/44000 | Loss: 3.8023 | LR: 4.99e-04 | 711.9ms/step | 28.1 samples/s +Step 1690/44000 | Loss: 3.8457 | LR: 4.99e-04 | 711.2ms/step | 28.1 samples/s +Step 1700/44000 | Loss: 3.7513 | LR: 4.99e-04 | 711.5ms/step | 28.1 samples/s +Step 1710/44000 | Loss: 3.7525 | LR: 4.99e-04 | 711.0ms/step | 28.1 samples/s +Step 1720/44000 | Loss: 3.7571 | LR: 4.99e-04 | 711.4ms/step | 28.1 samples/s +Step 1730/44000 | Loss: 3.7869 | LR: 4.99e-04 | 713.9ms/step | 28.0 samples/s +Step 1740/44000 | Loss: 3.7443 | LR: 4.99e-04 | 713.2ms/step | 28.0 samples/s +Step 1750/44000 | Loss: 3.8028 | LR: 4.99e-04 | 729.2ms/step | 27.4 samples/s +Validation at step 1750: Loss = 3.7793, Perplexity = 43.78 +Step 1760/44000 | Loss: 3.7412 | LR: 4.99e-04 | 930.1ms/step | 21.5 samples/s +Step 1770/44000 | Loss: 3.7938 | LR: 4.99e-04 | 712.1ms/step | 28.1 samples/s +Step 1780/44000 | Loss: 3.8198 | LR: 4.99e-04 | 712.3ms/step | 28.1 samples/s +Step 1790/44000 | Loss: 3.7737 | LR: 4.99e-04 | 711.9ms/step | 28.1 samples/s +Step 1800/44000 | Loss: 3.7759 | LR: 4.99e-04 | 712.5ms/step | 28.1 samples/s +Step 1810/44000 | Loss: 3.7727 | LR: 4.99e-04 | 712.3ms/step | 28.1 samples/s +Step 1820/44000 | Loss: 3.7948 | LR: 4.99e-04 | 713.0ms/step | 28.1 samples/s +Step 1830/44000 | Loss: 3.7446 | LR: 4.99e-04 | 721.8ms/step | 27.7 samples/s +Step 1840/44000 | Loss: 3.7577 | LR: 4.99e-04 | 716.3ms/step | 27.9 samples/s +Step 1850/44000 | Loss: 3.8339 | LR: 4.99e-04 | 711.9ms/step | 28.1 samples/s +Step 1860/44000 | Loss: 3.8052 | LR: 4.99e-04 | 711.6ms/step | 28.1 samples/s +Step 1870/44000 | Loss: 3.7234 | LR: 4.99e-04 | 712.3ms/step | 28.1 samples/s +Step 1880/44000 | Loss: 3.7213 | LR: 4.99e-04 | 711.4ms/step | 28.1 samples/s +Step 1890/44000 | Loss: 3.7170 | LR: 4.99e-04 | 711.7ms/step | 28.1 samples/s +Step 1900/44000 | Loss: 3.7374 | LR: 4.99e-04 | 711.6ms/step | 28.1 samples/s +Step 1910/44000 | Loss: 3.7240 | LR: 4.99e-04 | 711.8ms/step | 28.1 samples/s +Step 1920/44000 | Loss: 3.7506 | LR: 4.99e-04 | 721.6ms/step | 27.7 samples/s +Step 1930/44000 | Loss: 3.7866 | LR: 4.99e-04 | 719.9ms/step | 27.8 samples/s +Step 1940/44000 | Loss: 3.7519 | LR: 4.99e-04 | 713.4ms/step | 28.0 samples/s +Step 1950/44000 | Loss: 3.7709 | LR: 4.99e-04 | 712.7ms/step | 28.1 samples/s +Step 1960/44000 | Loss: 3.7094 | LR: 4.99e-04 | 712.0ms/step | 28.1 samples/s +Step 1970/44000 | Loss: 3.7227 | LR: 4.99e-04 | 711.8ms/step | 28.1 samples/s +Step 1980/44000 | Loss: 3.7098 | LR: 4.99e-04 | 712.3ms/step | 28.1 samples/s +Step 1990/44000 | Loss: 3.7499 | LR: 4.99e-04 | 711.8ms/step | 28.1 samples/s +Step 2000/44000 | Loss: 3.6846 | LR: 4.99e-04 | 711.8ms/step | 28.1 samples/s +Validation at step 2000: Loss = 3.6961, Perplexity = 40.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-2000 +Step 2010/44000 | Loss: 3.6959 | LR: 4.99e-04 | 1326.4ms/step | 15.1 samples/s +Step 2020/44000 | Loss: 3.7234 | LR: 4.99e-04 | 712.0ms/step | 28.1 samples/s +Step 2030/44000 | Loss: 3.6872 | LR: 4.99e-04 | 712.4ms/step | 28.1 samples/s +Step 2040/44000 | Loss: 3.7627 | LR: 4.99e-04 | 712.3ms/step | 28.1 samples/s +Step 2050/44000 | Loss: 3.6845 | LR: 4.99e-04 | 712.0ms/step | 28.1 samples/s +Step 2060/44000 | Loss: 3.7291 | LR: 4.99e-04 | 712.1ms/step | 28.1 samples/s +Step 2070/44000 | Loss: 3.6265 | LR: 4.99e-04 | 712.9ms/step | 28.1 samples/s +Step 2080/44000 | Loss: 3.6634 | LR: 4.99e-04 | 713.1ms/step | 28.0 samples/s +Step 2090/44000 | Loss: 3.6727 | LR: 4.99e-04 | 728.1ms/step | 27.5 samples/s +Step 2100/44000 | Loss: 3.6622 | LR: 4.99e-04 | 712.3ms/step | 28.1 samples/s +Step 2110/44000 | Loss: 3.6279 | LR: 4.99e-04 | 712.6ms/step | 28.1 samples/s +Step 2120/44000 | Loss: 3.6772 | LR: 4.99e-04 | 712.5ms/step | 28.1 samples/s +Step 2130/44000 | Loss: 3.7045 | LR: 4.99e-04 | 712.7ms/step | 28.1 samples/s +Step 2140/44000 | Loss: 3.6500 | LR: 4.99e-04 | 713.4ms/step | 28.0 samples/s +Step 2150/44000 | Loss: 3.6517 | LR: 4.99e-04 | 712.5ms/step | 28.1 samples/s +Step 2160/44000 | Loss: 3.6439 | LR: 4.99e-04 | 711.7ms/step | 28.1 samples/s +Step 2170/44000 | Loss: 3.6156 | LR: 4.99e-04 | 712.1ms/step | 28.1 samples/s +Step 2180/44000 | Loss: 3.6595 | LR: 4.99e-04 | 728.4ms/step | 27.5 samples/s +Step 2190/44000 | Loss: 3.6828 | LR: 4.99e-04 | 712.8ms/step | 28.1 samples/s +Step 2200/44000 | Loss: 3.6368 | LR: 4.99e-04 | 712.3ms/step | 28.1 samples/s +Step 2210/44000 | Loss: 3.6737 | LR: 4.99e-04 | 712.7ms/step | 28.1 samples/s +Step 2220/44000 | Loss: 3.6241 | LR: 4.99e-04 | 712.4ms/step | 28.1 samples/s +Step 2230/44000 | Loss: 3.6289 | LR: 4.99e-04 | 712.5ms/step | 28.1 samples/s +Step 2240/44000 | Loss: 3.6343 | LR: 4.99e-04 | 712.7ms/step | 28.1 samples/s +Step 2250/44000 | Loss: 3.7378 | LR: 4.99e-04 | 713.1ms/step | 28.0 samples/s +Validation at step 2250: Loss = 3.6281, Perplexity = 37.64 +Step 2260/44000 | Loss: 3.6690 | LR: 4.98e-04 | 931.9ms/step | 21.5 samples/s +Step 2270/44000 | Loss: 3.6882 | LR: 4.98e-04 | 730.0ms/step | 27.4 samples/s +Step 2280/44000 | Loss: 3.5861 | LR: 4.98e-04 | 713.0ms/step | 28.1 samples/s +Step 2290/44000 | Loss: 3.6340 | LR: 4.98e-04 | 714.0ms/step | 28.0 samples/s +Step 2300/44000 | Loss: 3.6453 | LR: 4.98e-04 | 714.1ms/step | 28.0 samples/s +Step 2310/44000 | Loss: 3.5984 | LR: 4.98e-04 | 713.3ms/step | 28.0 samples/s +Step 2320/44000 | Loss: 3.6795 | LR: 4.98e-04 | 712.6ms/step | 28.1 samples/s +Step 2330/44000 | Loss: 3.5965 | LR: 4.98e-04 | 712.7ms/step | 28.1 samples/s +Step 2340/44000 | Loss: 3.6194 | LR: 4.98e-04 | 712.9ms/step | 28.1 samples/s +Step 2350/44000 | Loss: 3.6261 | LR: 4.98e-04 | 720.1ms/step | 27.8 samples/s +Step 2360/44000 | Loss: 3.6861 | LR: 4.98e-04 | 723.9ms/step | 27.6 samples/s +Step 2370/44000 | Loss: 3.6042 | LR: 4.98e-04 | 713.3ms/step | 28.0 samples/s +Step 2380/44000 | Loss: 3.6952 | LR: 4.98e-04 | 713.9ms/step | 28.0 samples/s +Step 2390/44000 | Loss: 3.6621 | LR: 4.98e-04 | 713.4ms/step | 28.0 samples/s +Step 2400/44000 | Loss: 3.7106 | LR: 4.98e-04 | 713.5ms/step | 28.0 samples/s +Step 2410/44000 | Loss: 3.5947 | LR: 4.98e-04 | 712.8ms/step | 28.1 samples/s +Step 2420/44000 | Loss: 3.6654 | LR: 4.98e-04 | 714.2ms/step | 28.0 samples/s +Step 2430/44000 | Loss: 3.5647 | LR: 4.98e-04 | 713.1ms/step | 28.0 samples/s +Step 2440/44000 | Loss: 3.6465 | LR: 4.98e-04 | 724.2ms/step | 27.6 samples/s +Step 2450/44000 | Loss: 3.5752 | LR: 4.98e-04 | 716.6ms/step | 27.9 samples/s +Step 2460/44000 | Loss: 3.5945 | LR: 4.98e-04 | 712.6ms/step | 28.1 samples/s +Step 2470/44000 | Loss: 3.5405 | LR: 4.98e-04 | 713.7ms/step | 28.0 samples/s +Step 2480/44000 | Loss: 3.6350 | LR: 4.98e-04 | 713.0ms/step | 28.0 samples/s +Step 2490/44000 | Loss: 3.5975 | LR: 4.98e-04 | 713.0ms/step | 28.1 samples/s +Step 2500/44000 | Loss: 3.5968 | LR: 4.98e-04 | 712.0ms/step | 28.1 samples/s +Validation at step 2500: Loss = 3.5769, Perplexity = 35.76 +Step 2510/44000 | Loss: 3.5878 | LR: 4.98e-04 | 933.0ms/step | 21.4 samples/s +Step 2520/44000 | Loss: 3.5834 | LR: 4.98e-04 | 713.5ms/step | 28.0 samples/s +Step 2530/44000 | Loss: 3.5696 | LR: 4.98e-04 | 730.8ms/step | 27.4 samples/s +Step 2540/44000 | Loss: 3.5926 | LR: 4.98e-04 | 711.8ms/step | 28.1 samples/s +Step 2550/44000 | Loss: 3.6017 | LR: 4.98e-04 | 713.1ms/step | 28.0 samples/s +Step 2560/44000 | Loss: 3.6021 | LR: 4.98e-04 | 713.5ms/step | 28.0 samples/s +Step 2570/44000 | Loss: 3.5893 | LR: 4.98e-04 | 713.6ms/step | 28.0 samples/s +Step 2580/44000 | Loss: 3.5689 | LR: 4.98e-04 | 713.2ms/step | 28.0 samples/s +Step 2590/44000 | Loss: 3.5341 | LR: 4.98e-04 | 711.7ms/step | 28.1 samples/s +Step 2600/44000 | Loss: 3.5746 | LR: 4.98e-04 | 711.8ms/step | 28.1 samples/s +Step 2610/44000 | Loss: 3.5677 | LR: 4.98e-04 | 712.2ms/step | 28.1 samples/s +Step 2620/44000 | Loss: 3.5385 | LR: 4.98e-04 | 729.0ms/step | 27.4 samples/s +Step 2630/44000 | Loss: 3.5768 | LR: 4.98e-04 | 711.7ms/step | 28.1 samples/s +Step 2640/44000 | Loss: 3.5440 | LR: 4.98e-04 | 711.7ms/step | 28.1 samples/s +Step 2650/44000 | Loss: 3.5294 | LR: 4.98e-04 | 712.5ms/step | 28.1 samples/s +Step 2660/44000 | Loss: 3.6238 | LR: 4.98e-04 | 712.9ms/step | 28.1 samples/s +Step 2670/44000 | Loss: 3.5432 | LR: 4.98e-04 | 712.0ms/step | 28.1 samples/s +Step 2680/44000 | Loss: 3.5367 | LR: 4.98e-04 | 711.9ms/step | 28.1 samples/s +Step 2690/44000 | Loss: 3.5882 | LR: 4.98e-04 | 712.1ms/step | 28.1 samples/s +Step 2700/44000 | Loss: 3.5830 | LR: 4.97e-04 | 711.9ms/step | 28.1 samples/s +Step 2710/44000 | Loss: 3.5180 | LR: 4.97e-04 | 728.0ms/step | 27.5 samples/s +Step 2720/44000 | Loss: 3.5695 | LR: 4.97e-04 | 711.8ms/step | 28.1 samples/s +Step 2730/44000 | Loss: 3.5488 | LR: 4.97e-04 | 711.9ms/step | 28.1 samples/s +Step 2740/44000 | Loss: 3.5787 | LR: 4.97e-04 | 712.2ms/step | 28.1 samples/s +Step 2750/44000 | Loss: 3.5570 | LR: 4.97e-04 | 712.0ms/step | 28.1 samples/s +Validation at step 2750: Loss = 3.5312, Perplexity = 34.17 +Step 2760/44000 | Loss: 3.5438 | LR: 4.97e-04 | 931.8ms/step | 21.5 samples/s +Step 2770/44000 | Loss: 3.6380 | LR: 4.97e-04 | 712.5ms/step | 28.1 samples/s +Step 2780/44000 | Loss: 3.5200 | LR: 4.97e-04 | 712.7ms/step | 28.1 samples/s +Step 2790/44000 | Loss: 3.5624 | LR: 4.97e-04 | 726.0ms/step | 27.5 samples/s +Step 2800/44000 | Loss: 3.5391 | LR: 4.97e-04 | 718.7ms/step | 27.8 samples/s +Step 2810/44000 | Loss: 3.6079 | LR: 4.97e-04 | 713.8ms/step | 28.0 samples/s +Step 2820/44000 | Loss: 3.5109 | LR: 4.97e-04 | 714.4ms/step | 28.0 samples/s +Step 2830/44000 | Loss: 3.5154 | LR: 4.97e-04 | 712.9ms/step | 28.1 samples/s +Step 2840/44000 | Loss: 3.5093 | LR: 4.97e-04 | 712.2ms/step | 28.1 samples/s +Step 2850/44000 | Loss: 3.5053 | LR: 4.97e-04 | 712.5ms/step | 28.1 samples/s +Step 2860/44000 | Loss: 3.5113 | LR: 4.97e-04 | 712.2ms/step | 28.1 samples/s +Step 2870/44000 | Loss: 3.6102 | LR: 4.97e-04 | 712.0ms/step | 28.1 samples/s +Step 2880/44000 | Loss: 3.5800 | LR: 4.97e-04 | 725.9ms/step | 27.6 samples/s +Step 2890/44000 | Loss: 3.5408 | LR: 4.97e-04 | 711.4ms/step | 28.1 samples/s +Step 2900/44000 | Loss: 3.5938 | LR: 4.97e-04 | 712.2ms/step | 28.1 samples/s +Step 2910/44000 | Loss: 3.5336 | LR: 4.97e-04 | 712.9ms/step | 28.1 samples/s +Step 2920/44000 | Loss: 3.5232 | LR: 4.97e-04 | 713.7ms/step | 28.0 samples/s +Step 2930/44000 | Loss: 3.5772 | LR: 4.97e-04 | 713.4ms/step | 28.0 samples/s +Step 2940/44000 | Loss: 3.5013 | LR: 4.97e-04 | 714.0ms/step | 28.0 samples/s +Step 2950/44000 | Loss: 3.5193 | LR: 4.97e-04 | 713.5ms/step | 28.0 samples/s +Step 2960/44000 | Loss: 3.5368 | LR: 4.97e-04 | 713.3ms/step | 28.0 samples/s +Step 2970/44000 | Loss: 3.5237 | LR: 4.97e-04 | 730.4ms/step | 27.4 samples/s +Step 2980/44000 | Loss: 3.4495 | LR: 4.97e-04 | 713.2ms/step | 28.0 samples/s +Step 2990/44000 | Loss: 3.5052 | LR: 4.97e-04 | 713.7ms/step | 28.0 samples/s +Step 3000/44000 | Loss: 3.5268 | LR: 4.97e-04 | 713.7ms/step | 28.0 samples/s +Validation at step 3000: Loss = 3.4965, Perplexity = 33.00 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-3000 +Step 3010/44000 | Loss: 3.4928 | LR: 4.97e-04 | 1278.5ms/step | 15.6 samples/s +Step 3020/44000 | Loss: 3.5260 | LR: 4.97e-04 | 712.4ms/step | 28.1 samples/s +Step 3030/44000 | Loss: 3.4482 | LR: 4.97e-04 | 712.4ms/step | 28.1 samples/s +Step 3040/44000 | Loss: 3.4854 | LR: 4.97e-04 | 711.4ms/step | 28.1 samples/s +Step 3050/44000 | Loss: 3.4723 | LR: 4.97e-04 | 726.4ms/step | 27.5 samples/s +Step 3060/44000 | Loss: 3.4410 | LR: 4.96e-04 | 711.4ms/step | 28.1 samples/s +Step 3070/44000 | Loss: 3.4863 | LR: 4.96e-04 | 712.6ms/step | 28.1 samples/s +Step 3080/44000 | Loss: 3.4815 | LR: 4.96e-04 | 711.5ms/step | 28.1 samples/s +Step 3090/44000 | Loss: 3.5447 | LR: 4.96e-04 | 711.8ms/step | 28.1 samples/s +Step 3100/44000 | Loss: 3.4875 | LR: 4.96e-04 | 712.4ms/step | 28.1 samples/s +Step 3110/44000 | Loss: 3.4517 | LR: 4.96e-04 | 711.8ms/step | 28.1 samples/s +Step 3120/44000 | Loss: 3.4887 | LR: 4.96e-04 | 711.7ms/step | 28.1 samples/s +Step 3130/44000 | Loss: 3.4233 | LR: 4.96e-04 | 711.5ms/step | 28.1 samples/s +Step 3140/44000 | Loss: 3.5298 | LR: 4.96e-04 | 726.1ms/step | 27.5 samples/s +Step 3150/44000 | Loss: 3.4952 | LR: 4.96e-04 | 711.2ms/step | 28.1 samples/s +Step 3160/44000 | Loss: 3.5390 | LR: 4.96e-04 | 711.8ms/step | 28.1 samples/s +Step 3170/44000 | Loss: 3.5016 | LR: 4.96e-04 | 712.1ms/step | 28.1 samples/s +Step 3180/44000 | Loss: 3.4624 | LR: 4.96e-04 | 712.1ms/step | 28.1 samples/s +Step 3190/44000 | Loss: 3.4704 | LR: 4.96e-04 | 712.6ms/step | 28.1 samples/s +Step 3200/44000 | Loss: 3.5095 | LR: 4.96e-04 | 713.4ms/step | 28.0 samples/s +Step 3210/44000 | Loss: 3.5085 | LR: 4.96e-04 | 713.8ms/step | 28.0 samples/s +Step 3220/44000 | Loss: 3.4840 | LR: 4.96e-04 | 711.4ms/step | 28.1 samples/s +Step 3230/44000 | Loss: 3.5046 | LR: 4.96e-04 | 729.6ms/step | 27.4 samples/s +Step 3240/44000 | Loss: 3.4841 | LR: 4.96e-04 | 711.5ms/step | 28.1 samples/s +Step 3250/44000 | Loss: 3.5274 | LR: 4.96e-04 | 710.8ms/step | 28.1 samples/s +Validation at step 3250: Loss = 3.4628, Perplexity = 31.91 +Step 3260/44000 | Loss: 3.3950 | LR: 4.96e-04 | 929.6ms/step | 21.5 samples/s +Step 3270/44000 | Loss: 3.4567 | LR: 4.96e-04 | 711.5ms/step | 28.1 samples/s +Step 3280/44000 | Loss: 3.4207 | LR: 4.96e-04 | 712.6ms/step | 28.1 samples/s +Step 3290/44000 | Loss: 3.5154 | LR: 4.96e-04 | 713.0ms/step | 28.1 samples/s +Step 3300/44000 | Loss: 3.4336 | LR: 4.96e-04 | 714.4ms/step | 28.0 samples/s +Step 3310/44000 | Loss: 3.4358 | LR: 4.96e-04 | 722.0ms/step | 27.7 samples/s +Step 3320/44000 | Loss: 3.4333 | LR: 4.96e-04 | 719.9ms/step | 27.8 samples/s +Step 3330/44000 | Loss: 3.4902 | LR: 4.96e-04 | 713.5ms/step | 28.0 samples/s +Step 3340/44000 | Loss: 3.5208 | LR: 4.96e-04 | 712.6ms/step | 28.1 samples/s +Step 3350/44000 | Loss: 3.4005 | LR: 4.96e-04 | 712.6ms/step | 28.1 samples/s +Step 3360/44000 | Loss: 3.5379 | LR: 4.96e-04 | 712.5ms/step | 28.1 samples/s +Step 3370/44000 | Loss: 3.4504 | LR: 4.95e-04 | 712.4ms/step | 28.1 samples/s +Step 3380/44000 | Loss: 3.4467 | LR: 4.95e-04 | 712.3ms/step | 28.1 samples/s +Step 3390/44000 | Loss: 3.5042 | LR: 4.95e-04 | 713.0ms/step | 28.1 samples/s +Step 3400/44000 | Loss: 3.5154 | LR: 4.95e-04 | 723.4ms/step | 27.6 samples/s +Step 3410/44000 | Loss: 3.4640 | LR: 4.95e-04 | 720.1ms/step | 27.8 samples/s +Step 3420/44000 | Loss: 3.4574 | LR: 4.95e-04 | 711.9ms/step | 28.1 samples/s +Step 3430/44000 | Loss: 3.4668 | LR: 4.95e-04 | 712.6ms/step | 28.1 samples/s +Step 3440/44000 | Loss: 3.4506 | LR: 4.95e-04 | 712.0ms/step | 28.1 samples/s +Step 3450/44000 | Loss: 3.4595 | LR: 4.95e-04 | 710.7ms/step | 28.1 samples/s +Step 3460/44000 | Loss: 3.4394 | LR: 4.95e-04 | 710.7ms/step | 28.1 samples/s +Step 3470/44000 | Loss: 3.4285 | LR: 4.95e-04 | 711.0ms/step | 28.1 samples/s +Step 3480/44000 | Loss: 3.3498 | LR: 4.95e-04 | 711.4ms/step | 28.1 samples/s +Step 3490/44000 | Loss: 3.4653 | LR: 4.95e-04 | 724.0ms/step | 27.6 samples/s +Step 3500/44000 | Loss: 3.4918 | LR: 4.95e-04 | 715.8ms/step | 27.9 samples/s +Validation at step 3500: Loss = 3.4353, Perplexity = 31.04 +Step 3510/44000 | Loss: 3.4336 | LR: 4.95e-04 | 930.3ms/step | 21.5 samples/s +Step 3520/44000 | Loss: 3.4291 | LR: 4.95e-04 | 712.6ms/step | 28.1 samples/s +Step 3530/44000 | Loss: 3.4804 | LR: 4.95e-04 | 713.5ms/step | 28.0 samples/s +Step 3540/44000 | Loss: 3.4506 | LR: 4.95e-04 | 711.0ms/step | 28.1 samples/s +Step 3550/44000 | Loss: 3.4287 | LR: 4.95e-04 | 711.5ms/step | 28.1 samples/s +Step 3560/44000 | Loss: 3.4272 | LR: 4.95e-04 | 712.3ms/step | 28.1 samples/s +Step 3570/44000 | Loss: 3.4584 | LR: 4.95e-04 | 712.5ms/step | 28.1 samples/s +Step 3580/44000 | Loss: 3.5090 | LR: 4.95e-04 | 726.2ms/step | 27.5 samples/s +Step 3590/44000 | Loss: 3.4377 | LR: 4.95e-04 | 712.4ms/step | 28.1 samples/s +Step 3600/44000 | Loss: 3.4247 | LR: 4.95e-04 | 711.4ms/step | 28.1 samples/s +Step 3610/44000 | Loss: 3.4666 | LR: 4.95e-04 | 711.2ms/step | 28.1 samples/s +Step 3620/44000 | Loss: 3.4046 | LR: 4.95e-04 | 710.8ms/step | 28.1 samples/s +Step 3630/44000 | Loss: 3.4392 | LR: 4.95e-04 | 711.7ms/step | 28.1 samples/s +Step 3640/44000 | Loss: 3.4263 | LR: 4.95e-04 | 711.2ms/step | 28.1 samples/s +Step 3650/44000 | Loss: 3.4177 | LR: 4.94e-04 | 711.7ms/step | 28.1 samples/s +Step 3660/44000 | Loss: 3.4637 | LR: 4.94e-04 | 711.3ms/step | 28.1 samples/s +Step 3670/44000 | Loss: 3.4004 | LR: 4.94e-04 | 724.4ms/step | 27.6 samples/s +Step 3680/44000 | Loss: 3.4818 | LR: 4.94e-04 | 711.2ms/step | 28.1 samples/s +Step 3690/44000 | Loss: 3.4511 | LR: 4.94e-04 | 711.0ms/step | 28.1 samples/s +Step 3700/44000 | Loss: 3.3856 | LR: 4.94e-04 | 710.9ms/step | 28.1 samples/s +Step 3710/44000 | Loss: 3.3822 | LR: 4.94e-04 | 711.5ms/step | 28.1 samples/s +Step 3720/44000 | Loss: 3.3913 | LR: 4.94e-04 | 712.0ms/step | 28.1 samples/s +Step 3730/44000 | Loss: 3.3667 | LR: 4.94e-04 | 711.7ms/step | 28.1 samples/s +Step 3740/44000 | Loss: 3.3803 | LR: 4.94e-04 | 711.4ms/step | 28.1 samples/s +Step 3750/44000 | Loss: 3.3998 | LR: 4.94e-04 | 711.5ms/step | 28.1 samples/s +Validation at step 3750: Loss = 3.4083, Perplexity = 30.21 +Step 3760/44000 | Loss: 3.5160 | LR: 4.94e-04 | 943.5ms/step | 21.2 samples/s +Step 3770/44000 | Loss: 3.3829 | LR: 4.94e-04 | 710.6ms/step | 28.1 samples/s +Step 3780/44000 | Loss: 3.4462 | LR: 4.94e-04 | 710.9ms/step | 28.1 samples/s +Step 3790/44000 | Loss: 3.4251 | LR: 4.94e-04 | 711.0ms/step | 28.1 samples/s +Step 3800/44000 | Loss: 3.3617 | LR: 4.94e-04 | 711.4ms/step | 28.1 samples/s +Step 3810/44000 | Loss: 3.4694 | LR: 4.94e-04 | 711.6ms/step | 28.1 samples/s +Step 3820/44000 | Loss: 3.4740 | LR: 4.94e-04 | 712.3ms/step | 28.1 samples/s +Step 3830/44000 | Loss: 3.3732 | LR: 4.94e-04 | 712.0ms/step | 28.1 samples/s +Step 3840/44000 | Loss: 3.4203 | LR: 4.94e-04 | 722.7ms/step | 27.7 samples/s +Step 3850/44000 | Loss: 3.4444 | LR: 4.94e-04 | 715.6ms/step | 27.9 samples/s +Step 3860/44000 | Loss: 3.3338 | LR: 4.94e-04 | 711.3ms/step | 28.1 samples/s +Step 3870/44000 | Loss: 3.3916 | LR: 4.94e-04 | 710.7ms/step | 28.1 samples/s +Step 3880/44000 | Loss: 3.4226 | LR: 4.94e-04 | 710.1ms/step | 28.2 samples/s +Step 3890/44000 | Loss: 3.4635 | LR: 4.94e-04 | 710.9ms/step | 28.1 samples/s +Step 3900/44000 | Loss: 3.3409 | LR: 4.93e-04 | 711.7ms/step | 28.1 samples/s +Step 3910/44000 | Loss: 3.4021 | LR: 4.93e-04 | 711.1ms/step | 28.1 samples/s +Step 3920/44000 | Loss: 3.4019 | LR: 4.93e-04 | 710.4ms/step | 28.2 samples/s +Step 3930/44000 | Loss: 3.3895 | LR: 4.93e-04 | 723.9ms/step | 27.6 samples/s +Step 3940/44000 | Loss: 3.4838 | LR: 4.93e-04 | 711.0ms/step | 28.1 samples/s +Step 3950/44000 | Loss: 3.4073 | LR: 4.93e-04 | 711.3ms/step | 28.1 samples/s +Step 3960/44000 | Loss: 3.4352 | LR: 4.93e-04 | 710.9ms/step | 28.1 samples/s +Step 3970/44000 | Loss: 3.4543 | LR: 4.93e-04 | 711.0ms/step | 28.1 samples/s +Step 3980/44000 | Loss: 3.4519 | LR: 4.93e-04 | 711.9ms/step | 28.1 samples/s +Step 3990/44000 | Loss: 3.3738 | LR: 4.93e-04 | 711.8ms/step | 28.1 samples/s +Step 4000/44000 | Loss: 3.4085 | LR: 4.93e-04 | 711.2ms/step | 28.1 samples/s +Validation at step 4000: Loss = 3.3879, Perplexity = 29.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-4000 +Step 4010/44000 | Loss: 3.3414 | LR: 4.93e-04 | 1272.8ms/step | 15.7 samples/s +Step 4020/44000 | Loss: 3.4689 | LR: 4.93e-04 | 716.4ms/step | 27.9 samples/s +Step 4030/44000 | Loss: 3.3498 | LR: 4.93e-04 | 713.0ms/step | 28.1 samples/s +Step 4040/44000 | Loss: 3.4226 | LR: 4.93e-04 | 710.2ms/step | 28.2 samples/s +Step 4050/44000 | Loss: 3.3888 | LR: 4.93e-04 | 711.0ms/step | 28.1 samples/s +Step 4060/44000 | Loss: 3.4009 | LR: 4.93e-04 | 711.9ms/step | 28.1 samples/s +Step 4070/44000 | Loss: 3.4145 | LR: 4.93e-04 | 711.4ms/step | 28.1 samples/s +Step 4080/44000 | Loss: 3.3916 | LR: 4.93e-04 | 711.0ms/step | 28.1 samples/s +Step 4090/44000 | Loss: 3.3636 | LR: 4.93e-04 | 712.1ms/step | 28.1 samples/s +Step 4100/44000 | Loss: 3.4688 | LR: 4.93e-04 | 728.0ms/step | 27.5 samples/s +Step 4110/44000 | Loss: 3.4214 | LR: 4.93e-04 | 712.0ms/step | 28.1 samples/s +Step 4120/44000 | Loss: 3.4323 | LR: 4.93e-04 | 712.0ms/step | 28.1 samples/s +Step 4130/44000 | Loss: 3.3821 | LR: 4.93e-04 | 712.1ms/step | 28.1 samples/s +Step 4140/44000 | Loss: 3.3771 | LR: 4.92e-04 | 711.5ms/step | 28.1 samples/s +Step 4150/44000 | Loss: 3.4069 | LR: 4.92e-04 | 710.9ms/step | 28.1 samples/s +Step 4160/44000 | Loss: 3.4407 | LR: 4.92e-04 | 710.6ms/step | 28.1 samples/s +Step 4170/44000 | Loss: 3.3519 | LR: 4.92e-04 | 711.0ms/step | 28.1 samples/s +Step 4180/44000 | Loss: 3.3220 | LR: 4.92e-04 | 710.8ms/step | 28.1 samples/s +Step 4190/44000 | Loss: 3.4020 | LR: 4.92e-04 | 728.4ms/step | 27.5 samples/s +Step 4200/44000 | Loss: 3.4250 | LR: 4.92e-04 | 711.4ms/step | 28.1 samples/s +Step 4210/44000 | Loss: 3.3578 | LR: 4.92e-04 | 711.6ms/step | 28.1 samples/s +Step 4220/44000 | Loss: 3.3217 | LR: 4.92e-04 | 712.0ms/step | 28.1 samples/s +Step 4230/44000 | Loss: 3.3467 | LR: 4.92e-04 | 711.3ms/step | 28.1 samples/s +Step 4240/44000 | Loss: 3.4168 | LR: 4.92e-04 | 711.9ms/step | 28.1 samples/s +Step 4250/44000 | Loss: 3.4376 | LR: 4.92e-04 | 713.0ms/step | 28.0 samples/s +Validation at step 4250: Loss = 3.3637, Perplexity = 28.90 +Step 4260/44000 | Loss: 3.3513 | LR: 4.92e-04 | 930.9ms/step | 21.5 samples/s +Step 4270/44000 | Loss: 3.3141 | LR: 4.92e-04 | 711.4ms/step | 28.1 samples/s +Step 4280/44000 | Loss: 3.3848 | LR: 4.92e-04 | 724.8ms/step | 27.6 samples/s +Step 4290/44000 | Loss: 3.3283 | LR: 4.92e-04 | 712.2ms/step | 28.1 samples/s +Step 4300/44000 | Loss: 3.3853 | LR: 4.92e-04 | 713.7ms/step | 28.0 samples/s +Step 4310/44000 | Loss: 3.3301 | LR: 4.92e-04 | 711.1ms/step | 28.1 samples/s +Step 4320/44000 | Loss: 3.3076 | LR: 4.92e-04 | 712.4ms/step | 28.1 samples/s +Step 4330/44000 | Loss: 3.4054 | LR: 4.92e-04 | 711.5ms/step | 28.1 samples/s +Step 4340/44000 | Loss: 3.3805 | LR: 4.92e-04 | 711.2ms/step | 28.1 samples/s +Step 4350/44000 | Loss: 3.3933 | LR: 4.92e-04 | 712.9ms/step | 28.1 samples/s +Step 4360/44000 | Loss: 3.3845 | LR: 4.91e-04 | 715.8ms/step | 27.9 samples/s +Step 4370/44000 | Loss: 3.3152 | LR: 4.91e-04 | 722.1ms/step | 27.7 samples/s +Step 4380/44000 | Loss: 3.3744 | LR: 4.91e-04 | 710.9ms/step | 28.1 samples/s +Step 4390/44000 | Loss: 3.3985 | LR: 4.91e-04 | 709.6ms/step | 28.2 samples/s +Step 4400/44000 | Loss: 3.3770 | LR: 4.91e-04 | 710.9ms/step | 28.1 samples/s +Step 4410/44000 | Loss: 3.3619 | LR: 4.91e-04 | 711.0ms/step | 28.1 samples/s +Step 4420/44000 | Loss: 3.3498 | LR: 4.91e-04 | 710.4ms/step | 28.2 samples/s +Step 4430/44000 | Loss: 3.3208 | LR: 4.91e-04 | 710.4ms/step | 28.2 samples/s +Step 4440/44000 | Loss: 3.3214 | LR: 4.91e-04 | 710.4ms/step | 28.2 samples/s +Step 4450/44000 | Loss: 3.3874 | LR: 4.91e-04 | 722.2ms/step | 27.7 samples/s +Step 4460/44000 | Loss: 3.3814 | LR: 4.91e-04 | 714.9ms/step | 28.0 samples/s +Step 4470/44000 | Loss: 3.3757 | LR: 4.91e-04 | 709.5ms/step | 28.2 samples/s +Step 4480/44000 | Loss: 3.3589 | LR: 4.91e-04 | 710.1ms/step | 28.2 samples/s +Step 4490/44000 | Loss: 3.3431 | LR: 4.91e-04 | 711.6ms/step | 28.1 samples/s +Step 4500/44000 | Loss: 3.3976 | LR: 4.91e-04 | 710.3ms/step | 28.2 samples/s +Validation at step 4500: Loss = 3.3484, Perplexity = 28.46 +Step 4510/44000 | Loss: 3.4494 | LR: 4.91e-04 | 929.7ms/step | 21.5 samples/s +Step 4520/44000 | Loss: 3.3413 | LR: 4.91e-04 | 711.4ms/step | 28.1 samples/s +Step 4530/44000 | Loss: 3.3407 | LR: 4.91e-04 | 710.9ms/step | 28.1 samples/s +Step 4540/44000 | Loss: 3.3528 | LR: 4.91e-04 | 728.5ms/step | 27.5 samples/s +Step 4550/44000 | Loss: 3.3865 | LR: 4.91e-04 | 712.2ms/step | 28.1 samples/s +Step 4560/44000 | Loss: 3.4052 | LR: 4.90e-04 | 712.0ms/step | 28.1 samples/s +Step 4570/44000 | Loss: 3.3867 | LR: 4.90e-04 | 712.2ms/step | 28.1 samples/s +Step 4580/44000 | Loss: 3.2775 | LR: 4.90e-04 | 713.6ms/step | 28.0 samples/s +Step 4590/44000 | Loss: 3.4013 | LR: 4.90e-04 | 712.0ms/step | 28.1 samples/s +Step 4600/44000 | Loss: 3.3845 | LR: 4.90e-04 | 711.1ms/step | 28.1 samples/s +Step 4610/44000 | Loss: 3.4070 | LR: 4.90e-04 | 711.8ms/step | 28.1 samples/s +Step 4620/44000 | Loss: 3.3402 | LR: 4.90e-04 | 711.9ms/step | 28.1 samples/s +Step 4630/44000 | Loss: 3.4057 | LR: 4.90e-04 | 726.0ms/step | 27.5 samples/s +Step 4640/44000 | Loss: 3.3987 | LR: 4.90e-04 | 711.6ms/step | 28.1 samples/s +Step 4650/44000 | Loss: 3.3459 | LR: 4.90e-04 | 712.2ms/step | 28.1 samples/s +Step 4660/44000 | Loss: 3.3901 | LR: 4.90e-04 | 710.5ms/step | 28.1 samples/s +Step 4670/44000 | Loss: 3.3873 | LR: 4.90e-04 | 709.8ms/step | 28.2 samples/s +Step 4680/44000 | Loss: 3.3634 | LR: 4.90e-04 | 709.7ms/step | 28.2 samples/s +Step 4690/44000 | Loss: 3.3854 | LR: 4.90e-04 | 711.5ms/step | 28.1 samples/s +Step 4700/44000 | Loss: 3.3660 | LR: 4.90e-04 | 711.3ms/step | 28.1 samples/s +Step 4710/44000 | Loss: 3.3829 | LR: 4.90e-04 | 712.3ms/step | 28.1 samples/s +Step 4720/44000 | Loss: 3.3674 | LR: 4.90e-04 | 725.3ms/step | 27.6 samples/s +Step 4730/44000 | Loss: 3.3467 | LR: 4.90e-04 | 712.4ms/step | 28.1 samples/s +Step 4740/44000 | Loss: 3.3543 | LR: 4.90e-04 | 712.0ms/step | 28.1 samples/s +Step 4750/44000 | Loss: 3.3359 | LR: 4.90e-04 | 711.8ms/step | 28.1 samples/s +Validation at step 4750: Loss = 3.3276, Perplexity = 27.87 +Step 4760/44000 | Loss: 3.4092 | LR: 4.89e-04 | 929.7ms/step | 21.5 samples/s +Step 4770/44000 | Loss: 3.3637 | LR: 4.89e-04 | 711.2ms/step | 28.1 samples/s +Step 4780/44000 | Loss: 3.3065 | LR: 4.89e-04 | 711.3ms/step | 28.1 samples/s +Step 4790/44000 | Loss: 3.2805 | LR: 4.89e-04 | 712.4ms/step | 28.1 samples/s +Step 4800/44000 | Loss: 3.3230 | LR: 4.89e-04 | 723.4ms/step | 27.6 samples/s +Step 4810/44000 | Loss: 3.3128 | LR: 4.89e-04 | 720.5ms/step | 27.8 samples/s +Step 4820/44000 | Loss: 3.3313 | LR: 4.89e-04 | 712.7ms/step | 28.1 samples/s +Step 4830/44000 | Loss: 3.3097 | LR: 4.89e-04 | 712.5ms/step | 28.1 samples/s +Step 4840/44000 | Loss: 3.2805 | LR: 4.89e-04 | 712.3ms/step | 28.1 samples/s +Step 4850/44000 | Loss: 3.3623 | LR: 4.89e-04 | 711.9ms/step | 28.1 samples/s +Step 4860/44000 | Loss: 3.3832 | LR: 4.89e-04 | 713.5ms/step | 28.0 samples/s +Step 4870/44000 | Loss: 3.3543 | LR: 4.89e-04 | 710.4ms/step | 28.2 samples/s +Step 4880/44000 | Loss: 3.3196 | LR: 4.89e-04 | 711.9ms/step | 28.1 samples/s +Step 4890/44000 | Loss: 3.2978 | LR: 4.89e-04 | 722.9ms/step | 27.7 samples/s +Step 4900/44000 | Loss: 3.3302 | LR: 4.89e-04 | 715.9ms/step | 27.9 samples/s +Step 4910/44000 | Loss: 3.3545 | LR: 4.89e-04 | 712.5ms/step | 28.1 samples/s +Step 4920/44000 | Loss: 3.3562 | LR: 4.89e-04 | 712.0ms/step | 28.1 samples/s +Step 4930/44000 | Loss: 3.3308 | LR: 4.89e-04 | 710.8ms/step | 28.1 samples/s +Step 4940/44000 | Loss: 3.3066 | LR: 4.89e-04 | 712.0ms/step | 28.1 samples/s +Step 4950/44000 | Loss: 3.3346 | LR: 4.88e-04 | 712.3ms/step | 28.1 samples/s +Step 4960/44000 | Loss: 3.3678 | LR: 4.88e-04 | 710.9ms/step | 28.1 samples/s +Step 4970/44000 | Loss: 3.3042 | LR: 4.88e-04 | 711.0ms/step | 28.1 samples/s +Step 4980/44000 | Loss: 3.3808 | LR: 4.88e-04 | 727.4ms/step | 27.5 samples/s +Step 4990/44000 | Loss: 3.3584 | LR: 4.88e-04 | 712.1ms/step | 28.1 samples/s +Step 5000/44000 | Loss: 3.3295 | LR: 4.88e-04 | 713.1ms/step | 28.0 samples/s +Validation at step 5000: Loss = 3.3157, Perplexity = 27.54 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-5000 +Step 5010/44000 | Loss: 3.2787 | LR: 4.88e-04 | 1250.6ms/step | 16.0 samples/s +Step 5020/44000 | Loss: 3.3298 | LR: 4.88e-04 | 710.2ms/step | 28.2 samples/s +Step 5030/44000 | Loss: 3.3798 | LR: 4.88e-04 | 711.5ms/step | 28.1 samples/s +Step 5040/44000 | Loss: 3.3046 | LR: 4.88e-04 | 711.7ms/step | 28.1 samples/s +Step 5050/44000 | Loss: 3.2768 | LR: 4.88e-04 | 711.4ms/step | 28.1 samples/s +Step 5060/44000 | Loss: 3.2926 | LR: 4.88e-04 | 722.6ms/step | 27.7 samples/s +Step 5070/44000 | Loss: 3.3509 | LR: 4.88e-04 | 715.6ms/step | 27.9 samples/s +Step 5080/44000 | Loss: 3.3954 | LR: 4.88e-04 | 712.5ms/step | 28.1 samples/s +Step 5090/44000 | Loss: 3.3729 | LR: 4.88e-04 | 710.7ms/step | 28.1 samples/s +Step 5100/44000 | Loss: 3.3153 | LR: 4.88e-04 | 713.3ms/step | 28.0 samples/s +Step 5110/44000 | Loss: 3.3258 | LR: 4.88e-04 | 711.0ms/step | 28.1 samples/s +Step 5120/44000 | Loss: 3.3249 | LR: 4.88e-04 | 711.4ms/step | 28.1 samples/s +Step 5130/44000 | Loss: 3.2669 | LR: 4.87e-04 | 712.8ms/step | 28.1 samples/s +Step 5140/44000 | Loss: 3.3772 | LR: 4.87e-04 | 712.0ms/step | 28.1 samples/s +Step 5150/44000 | Loss: 3.3702 | LR: 4.87e-04 | 721.8ms/step | 27.7 samples/s +Step 5160/44000 | Loss: 3.3555 | LR: 4.87e-04 | 716.3ms/step | 27.9 samples/s +Step 5170/44000 | Loss: 3.3103 | LR: 4.87e-04 | 711.8ms/step | 28.1 samples/s +Step 5180/44000 | Loss: 3.3051 | LR: 4.87e-04 | 713.5ms/step | 28.0 samples/s +Step 5190/44000 | Loss: 3.3735 | LR: 4.87e-04 | 715.3ms/step | 28.0 samples/s +Step 5200/44000 | Loss: 3.3496 | LR: 4.87e-04 | 714.6ms/step | 28.0 samples/s +Step 5210/44000 | Loss: 3.3737 | LR: 4.87e-04 | 711.5ms/step | 28.1 samples/s +Step 5220/44000 | Loss: 3.3217 | LR: 4.87e-04 | 711.4ms/step | 28.1 samples/s +Step 5230/44000 | Loss: 3.2764 | LR: 4.87e-04 | 712.4ms/step | 28.1 samples/s +Step 5240/44000 | Loss: 3.3562 | LR: 4.87e-04 | 730.0ms/step | 27.4 samples/s +Step 5250/44000 | Loss: 3.3173 | LR: 4.87e-04 | 711.6ms/step | 28.1 samples/s +Validation at step 5250: Loss = 3.3017, Perplexity = 27.16 +Step 5260/44000 | Loss: 3.3564 | LR: 4.87e-04 | 931.8ms/step | 21.5 samples/s +Step 5270/44000 | Loss: 3.3231 | LR: 4.87e-04 | 714.1ms/step | 28.0 samples/s +Step 5280/44000 | Loss: 3.3437 | LR: 4.87e-04 | 712.5ms/step | 28.1 samples/s +Step 5290/44000 | Loss: 3.2941 | LR: 4.87e-04 | 712.5ms/step | 28.1 samples/s +Step 5300/44000 | Loss: 3.3324 | LR: 4.86e-04 | 712.1ms/step | 28.1 samples/s +Step 5310/44000 | Loss: 3.2905 | LR: 4.86e-04 | 711.6ms/step | 28.1 samples/s +Step 5320/44000 | Loss: 3.3093 | LR: 4.86e-04 | 712.3ms/step | 28.1 samples/s +Step 5330/44000 | Loss: 3.3211 | LR: 4.86e-04 | 728.4ms/step | 27.5 samples/s +Step 5340/44000 | Loss: 3.3202 | LR: 4.86e-04 | 712.6ms/step | 28.1 samples/s +Step 5350/44000 | Loss: 3.3070 | LR: 4.86e-04 | 712.5ms/step | 28.1 samples/s +Step 5360/44000 | Loss: 3.2996 | LR: 4.86e-04 | 712.5ms/step | 28.1 samples/s +Step 5370/44000 | Loss: 3.2156 | LR: 4.86e-04 | 712.5ms/step | 28.1 samples/s +Step 5380/44000 | Loss: 3.3326 | LR: 4.86e-04 | 712.1ms/step | 28.1 samples/s +Step 5390/44000 | Loss: 3.2820 | LR: 4.86e-04 | 712.0ms/step | 28.1 samples/s +Step 5400/44000 | Loss: 3.2620 | LR: 4.86e-04 | 712.6ms/step | 28.1 samples/s +Step 5410/44000 | Loss: 3.3563 | LR: 4.86e-04 | 711.9ms/step | 28.1 samples/s +Step 5420/44000 | Loss: 3.2930 | LR: 4.86e-04 | 726.0ms/step | 27.5 samples/s +Step 5430/44000 | Loss: 3.3268 | LR: 4.86e-04 | 711.9ms/step | 28.1 samples/s +Step 5440/44000 | Loss: 3.3558 | LR: 4.86e-04 | 712.5ms/step | 28.1 samples/s +Step 5450/44000 | Loss: 3.2769 | LR: 4.86e-04 | 714.2ms/step | 28.0 samples/s +Step 5460/44000 | Loss: 3.3161 | LR: 4.86e-04 | 712.9ms/step | 28.1 samples/s +Step 5470/44000 | Loss: 3.3217 | LR: 4.85e-04 | 712.1ms/step | 28.1 samples/s +Step 5480/44000 | Loss: 3.3325 | LR: 4.85e-04 | 712.0ms/step | 28.1 samples/s +Step 5490/44000 | Loss: 3.2762 | LR: 4.85e-04 | 712.1ms/step | 28.1 samples/s +Step 5500/44000 | Loss: 3.2757 | LR: 4.85e-04 | 724.1ms/step | 27.6 samples/s +Validation at step 5500: Loss = 3.2859, Perplexity = 26.73 +Step 5510/44000 | Loss: 3.3113 | LR: 4.85e-04 | 938.2ms/step | 21.3 samples/s +Step 5520/44000 | Loss: 3.3184 | LR: 4.85e-04 | 714.3ms/step | 28.0 samples/s +Step 5530/44000 | Loss: 3.3387 | LR: 4.85e-04 | 713.7ms/step | 28.0 samples/s +Step 5540/44000 | Loss: 3.2875 | LR: 4.85e-04 | 712.6ms/step | 28.1 samples/s +Step 5550/44000 | Loss: 3.2941 | LR: 4.85e-04 | 712.7ms/step | 28.1 samples/s +Step 5560/44000 | Loss: 3.2792 | LR: 4.85e-04 | 712.9ms/step | 28.1 samples/s +Step 5570/44000 | Loss: 3.2988 | LR: 4.85e-04 | 713.4ms/step | 28.0 samples/s +Step 5580/44000 | Loss: 3.2863 | LR: 4.85e-04 | 713.4ms/step | 28.0 samples/s +Step 5590/44000 | Loss: 3.3096 | LR: 4.85e-04 | 725.9ms/step | 27.6 samples/s +Step 5600/44000 | Loss: 3.3095 | LR: 4.85e-04 | 713.3ms/step | 28.0 samples/s +Step 5610/44000 | Loss: 3.3095 | LR: 4.85e-04 | 713.1ms/step | 28.0 samples/s +Step 5620/44000 | Loss: 3.2504 | LR: 4.85e-04 | 712.6ms/step | 28.1 samples/s +Step 5630/44000 | Loss: 3.3053 | LR: 4.84e-04 | 712.8ms/step | 28.1 samples/s +Step 5640/44000 | Loss: 3.3242 | LR: 4.84e-04 | 712.0ms/step | 28.1 samples/s +Step 5650/44000 | Loss: 3.3449 | LR: 4.84e-04 | 712.2ms/step | 28.1 samples/s +Step 5660/44000 | Loss: 3.2601 | LR: 4.84e-04 | 712.8ms/step | 28.1 samples/s +Step 5670/44000 | Loss: 3.3142 | LR: 4.84e-04 | 712.0ms/step | 28.1 samples/s +Step 5680/44000 | Loss: 3.3299 | LR: 4.84e-04 | 730.6ms/step | 27.4 samples/s +Step 5690/44000 | Loss: 3.2953 | LR: 4.84e-04 | 712.9ms/step | 28.1 samples/s +Step 5700/44000 | Loss: 3.3167 | LR: 4.84e-04 | 712.9ms/step | 28.1 samples/s +Step 5710/44000 | Loss: 3.2992 | LR: 4.84e-04 | 712.0ms/step | 28.1 samples/s +Step 5720/44000 | Loss: 3.2433 | LR: 4.84e-04 | 711.8ms/step | 28.1 samples/s +Step 5730/44000 | Loss: 3.2221 | LR: 4.84e-04 | 711.9ms/step | 28.1 samples/s +Step 5740/44000 | Loss: 3.2972 | LR: 4.84e-04 | 712.1ms/step | 28.1 samples/s +Step 5750/44000 | Loss: 3.3472 | LR: 4.84e-04 | 712.3ms/step | 28.1 samples/s +Validation at step 5750: Loss = 3.2763, Perplexity = 26.48 +Step 5760/44000 | Loss: 3.2982 | LR: 4.84e-04 | 941.9ms/step | 21.2 samples/s +Step 5770/44000 | Loss: 3.2776 | LR: 4.84e-04 | 717.6ms/step | 27.9 samples/s +Step 5780/44000 | Loss: 3.3151 | LR: 4.83e-04 | 713.5ms/step | 28.0 samples/s +Step 5790/44000 | Loss: 3.2775 | LR: 4.83e-04 | 711.4ms/step | 28.1 samples/s +Step 5800/44000 | Loss: 3.3627 | LR: 4.83e-04 | 711.9ms/step | 28.1 samples/s +Step 5810/44000 | Loss: 3.2949 | LR: 4.83e-04 | 711.9ms/step | 28.1 samples/s +Step 5820/44000 | Loss: 3.2765 | LR: 4.83e-04 | 711.9ms/step | 28.1 samples/s +Step 5830/44000 | Loss: 3.2673 | LR: 4.83e-04 | 711.8ms/step | 28.1 samples/s +Step 5840/44000 | Loss: 3.3273 | LR: 4.83e-04 | 712.1ms/step | 28.1 samples/s +Step 5850/44000 | Loss: 3.3414 | LR: 4.83e-04 | 720.8ms/step | 27.7 samples/s +Step 5860/44000 | Loss: 3.3744 | LR: 4.83e-04 | 717.5ms/step | 27.9 samples/s +Step 5870/44000 | Loss: 3.2989 | LR: 4.83e-04 | 712.0ms/step | 28.1 samples/s +Step 5880/44000 | Loss: 3.2560 | LR: 4.83e-04 | 711.6ms/step | 28.1 samples/s +Step 5890/44000 | Loss: 3.2682 | LR: 4.83e-04 | 711.4ms/step | 28.1 samples/s +Step 5900/44000 | Loss: 3.2703 | LR: 4.83e-04 | 711.5ms/step | 28.1 samples/s +Step 5910/44000 | Loss: 3.2670 | LR: 4.83e-04 | 711.5ms/step | 28.1 samples/s +Step 5920/44000 | Loss: 3.2789 | LR: 4.83e-04 | 711.7ms/step | 28.1 samples/s +Step 5930/44000 | Loss: 3.3713 | LR: 4.83e-04 | 712.5ms/step | 28.1 samples/s +Step 5940/44000 | Loss: 3.2614 | LR: 4.82e-04 | 721.8ms/step | 27.7 samples/s +Step 5950/44000 | Loss: 3.2637 | LR: 4.82e-04 | 720.7ms/step | 27.8 samples/s +Step 5960/44000 | Loss: 3.2838 | LR: 4.82e-04 | 712.6ms/step | 28.1 samples/s +Step 5970/44000 | Loss: 3.2395 | LR: 4.82e-04 | 711.6ms/step | 28.1 samples/s +Step 5980/44000 | Loss: 3.2880 | LR: 4.82e-04 | 713.2ms/step | 28.0 samples/s +Step 5990/44000 | Loss: 3.2797 | LR: 4.82e-04 | 713.4ms/step | 28.0 samples/s +Step 6000/44000 | Loss: 3.2863 | LR: 4.82e-04 | 712.2ms/step | 28.1 samples/s +Validation at step 6000: Loss = 3.2640, Perplexity = 26.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-6000 +Step 6010/44000 | Loss: 3.2219 | LR: 4.82e-04 | 1291.0ms/step | 15.5 samples/s +Step 6020/44000 | Loss: 3.3095 | LR: 4.82e-04 | 721.9ms/step | 27.7 samples/s +Step 6030/44000 | Loss: 3.2368 | LR: 4.82e-04 | 715.1ms/step | 28.0 samples/s +Step 6040/44000 | Loss: 3.2276 | LR: 4.82e-04 | 711.5ms/step | 28.1 samples/s +Step 6050/44000 | Loss: 3.3255 | LR: 4.82e-04 | 711.1ms/step | 28.1 samples/s +Step 6060/44000 | Loss: 3.3251 | LR: 4.82e-04 | 711.6ms/step | 28.1 samples/s +Step 6070/44000 | Loss: 3.2553 | LR: 4.82e-04 | 711.0ms/step | 28.1 samples/s +Step 6080/44000 | Loss: 3.3052 | LR: 4.81e-04 | 711.3ms/step | 28.1 samples/s +Step 6090/44000 | Loss: 3.2634 | LR: 4.81e-04 | 711.9ms/step | 28.1 samples/s +Step 6100/44000 | Loss: 3.2435 | LR: 4.81e-04 | 711.7ms/step | 28.1 samples/s +Step 6110/44000 | Loss: 3.2940 | LR: 4.81e-04 | 723.3ms/step | 27.7 samples/s +Step 6120/44000 | Loss: 3.2271 | LR: 4.81e-04 | 716.2ms/step | 27.9 samples/s +Step 6130/44000 | Loss: 3.2290 | LR: 4.81e-04 | 712.0ms/step | 28.1 samples/s +Step 6140/44000 | Loss: 3.2474 | LR: 4.81e-04 | 711.2ms/step | 28.1 samples/s +Step 6150/44000 | Loss: 3.3109 | LR: 4.81e-04 | 710.3ms/step | 28.2 samples/s +Step 6160/44000 | Loss: 3.2200 | LR: 4.81e-04 | 711.2ms/step | 28.1 samples/s +Step 6170/44000 | Loss: 3.3165 | LR: 4.81e-04 | 711.7ms/step | 28.1 samples/s +Step 6180/44000 | Loss: 3.2506 | LR: 4.81e-04 | 712.0ms/step | 28.1 samples/s +Step 6190/44000 | Loss: 3.2302 | LR: 4.81e-04 | 711.5ms/step | 28.1 samples/s +Step 6200/44000 | Loss: 3.2796 | LR: 4.81e-04 | 726.6ms/step | 27.5 samples/s +Step 6210/44000 | Loss: 3.3116 | LR: 4.81e-04 | 710.5ms/step | 28.1 samples/s +Step 6220/44000 | Loss: 3.2339 | LR: 4.81e-04 | 711.8ms/step | 28.1 samples/s +Step 6230/44000 | Loss: 3.2467 | LR: 4.80e-04 | 711.7ms/step | 28.1 samples/s +Step 6240/44000 | Loss: 3.2496 | LR: 4.80e-04 | 711.4ms/step | 28.1 samples/s +Step 6250/44000 | Loss: 3.2824 | LR: 4.80e-04 | 712.0ms/step | 28.1 samples/s +Validation at step 6250: Loss = 3.2507, Perplexity = 25.81 +Step 6260/44000 | Loss: 3.2236 | LR: 4.80e-04 | 930.4ms/step | 21.5 samples/s +Step 6270/44000 | Loss: 3.2940 | LR: 4.80e-04 | 711.8ms/step | 28.1 samples/s +Step 6280/44000 | Loss: 3.2888 | LR: 4.80e-04 | 712.0ms/step | 28.1 samples/s +Step 6290/44000 | Loss: 3.3028 | LR: 4.80e-04 | 725.8ms/step | 27.6 samples/s +Step 6300/44000 | Loss: 3.3134 | LR: 4.80e-04 | 711.7ms/step | 28.1 samples/s +Step 6310/44000 | Loss: 3.2504 | LR: 4.80e-04 | 712.0ms/step | 28.1 samples/s +Step 6320/44000 | Loss: 3.2724 | LR: 4.80e-04 | 711.9ms/step | 28.1 samples/s +Step 6330/44000 | Loss: 3.3015 | LR: 4.80e-04 | 711.3ms/step | 28.1 samples/s +Step 6340/44000 | Loss: 3.3091 | LR: 4.80e-04 | 712.3ms/step | 28.1 samples/s +Step 6350/44000 | Loss: 3.1738 | LR: 4.80e-04 | 712.2ms/step | 28.1 samples/s +Step 6360/44000 | Loss: 3.1778 | LR: 4.80e-04 | 712.5ms/step | 28.1 samples/s +Step 6370/44000 | Loss: 3.2694 | LR: 4.79e-04 | 712.0ms/step | 28.1 samples/s +Step 6380/44000 | Loss: 3.2915 | LR: 4.79e-04 | 729.6ms/step | 27.4 samples/s +Step 6390/44000 | Loss: 3.3090 | LR: 4.79e-04 | 712.6ms/step | 28.1 samples/s +Step 6400/44000 | Loss: 3.2887 | LR: 4.79e-04 | 712.9ms/step | 28.1 samples/s +Step 6410/44000 | Loss: 3.2736 | LR: 4.79e-04 | 712.4ms/step | 28.1 samples/s +Step 6420/44000 | Loss: 3.2801 | LR: 4.79e-04 | 712.7ms/step | 28.1 samples/s +Step 6430/44000 | Loss: 3.2671 | LR: 4.79e-04 | 712.6ms/step | 28.1 samples/s +Step 6440/44000 | Loss: 3.2078 | LR: 4.79e-04 | 712.8ms/step | 28.1 samples/s +Step 6450/44000 | Loss: 3.2282 | LR: 4.79e-04 | 712.2ms/step | 28.1 samples/s +Step 6460/44000 | Loss: 3.2579 | LR: 4.79e-04 | 721.5ms/step | 27.7 samples/s +Step 6470/44000 | Loss: 3.3255 | LR: 4.79e-04 | 719.3ms/step | 27.8 samples/s +Step 6480/44000 | Loss: 3.2666 | LR: 4.79e-04 | 711.6ms/step | 28.1 samples/s +Step 6490/44000 | Loss: 3.2467 | LR: 4.79e-04 | 712.3ms/step | 28.1 samples/s +Step 6500/44000 | Loss: 3.3352 | LR: 4.79e-04 | 712.1ms/step | 28.1 samples/s +Validation at step 6500: Loss = 3.2403, Perplexity = 25.54 +Step 6510/44000 | Loss: 3.2631 | LR: 4.78e-04 | 930.0ms/step | 21.5 samples/s +Step 6520/44000 | Loss: 3.2415 | LR: 4.78e-04 | 712.3ms/step | 28.1 samples/s +Step 6530/44000 | Loss: 3.2299 | LR: 4.78e-04 | 712.9ms/step | 28.1 samples/s +Step 6540/44000 | Loss: 3.2797 | LR: 4.78e-04 | 712.3ms/step | 28.1 samples/s +Step 6550/44000 | Loss: 3.2454 | LR: 4.78e-04 | 728.4ms/step | 27.5 samples/s +Step 6560/44000 | Loss: 3.3116 | LR: 4.78e-04 | 712.6ms/step | 28.1 samples/s +Step 6570/44000 | Loss: 3.2629 | LR: 4.78e-04 | 711.6ms/step | 28.1 samples/s +Step 6580/44000 | Loss: 3.2648 | LR: 4.78e-04 | 710.7ms/step | 28.1 samples/s +Step 6590/44000 | Loss: 3.2138 | LR: 4.78e-04 | 710.8ms/step | 28.1 samples/s +Step 6600/44000 | Loss: 3.2105 | LR: 4.78e-04 | 711.4ms/step | 28.1 samples/s +Step 6610/44000 | Loss: 3.2495 | LR: 4.78e-04 | 711.8ms/step | 28.1 samples/s +Step 6620/44000 | Loss: 3.2685 | LR: 4.78e-04 | 712.2ms/step | 28.1 samples/s +Step 6630/44000 | Loss: 3.3209 | LR: 4.78e-04 | 712.4ms/step | 28.1 samples/s +Step 6640/44000 | Loss: 3.2767 | LR: 4.77e-04 | 730.5ms/step | 27.4 samples/s +Step 6650/44000 | Loss: 3.2067 | LR: 4.77e-04 | 710.8ms/step | 28.1 samples/s +Step 6660/44000 | Loss: 3.2355 | LR: 4.77e-04 | 711.2ms/step | 28.1 samples/s +Step 6670/44000 | Loss: 3.1789 | LR: 4.77e-04 | 712.1ms/step | 28.1 samples/s +Step 6680/44000 | Loss: 3.2258 | LR: 4.77e-04 | 712.1ms/step | 28.1 samples/s +Step 6690/44000 | Loss: 3.3205 | LR: 4.77e-04 | 712.6ms/step | 28.1 samples/s +Step 6700/44000 | Loss: 3.1803 | LR: 4.77e-04 | 712.3ms/step | 28.1 samples/s +Step 6710/44000 | Loss: 3.2332 | LR: 4.77e-04 | 712.4ms/step | 28.1 samples/s +Step 6720/44000 | Loss: 3.2407 | LR: 4.77e-04 | 712.4ms/step | 28.1 samples/s +Step 6730/44000 | Loss: 3.2607 | LR: 4.77e-04 | 728.5ms/step | 27.5 samples/s +Step 6740/44000 | Loss: 3.2612 | LR: 4.77e-04 | 711.4ms/step | 28.1 samples/s +Step 6750/44000 | Loss: 3.2290 | LR: 4.77e-04 | 711.9ms/step | 28.1 samples/s +Validation at step 6750: Loss = 3.2298, Perplexity = 25.28 +Step 6760/44000 | Loss: 3.2314 | LR: 4.77e-04 | 932.1ms/step | 21.5 samples/s +Step 6770/44000 | Loss: 3.2872 | LR: 4.76e-04 | 712.3ms/step | 28.1 samples/s +Step 6780/44000 | Loss: 3.2922 | LR: 4.76e-04 | 711.7ms/step | 28.1 samples/s +Step 6790/44000 | Loss: 3.2398 | LR: 4.76e-04 | 711.1ms/step | 28.1 samples/s +Step 6800/44000 | Loss: 3.2824 | LR: 4.76e-04 | 711.5ms/step | 28.1 samples/s +Step 6810/44000 | Loss: 3.3112 | LR: 4.76e-04 | 720.6ms/step | 27.8 samples/s +Step 6820/44000 | Loss: 3.1292 | LR: 4.76e-04 | 716.2ms/step | 27.9 samples/s +Step 6830/44000 | Loss: 3.2360 | LR: 4.76e-04 | 712.1ms/step | 28.1 samples/s +Step 6840/44000 | Loss: 3.2035 | LR: 4.76e-04 | 712.5ms/step | 28.1 samples/s +Step 6850/44000 | Loss: 3.2166 | LR: 4.76e-04 | 712.5ms/step | 28.1 samples/s +Step 6860/44000 | Loss: 3.2848 | LR: 4.76e-04 | 712.2ms/step | 28.1 samples/s +Step 6870/44000 | Loss: 3.2405 | LR: 4.76e-04 | 711.7ms/step | 28.1 samples/s +Step 6880/44000 | Loss: 3.2134 | LR: 4.76e-04 | 711.8ms/step | 28.1 samples/s +Step 6890/44000 | Loss: 3.2315 | LR: 4.76e-04 | 711.6ms/step | 28.1 samples/s +Step 6900/44000 | Loss: 3.2528 | LR: 4.75e-04 | 720.3ms/step | 27.8 samples/s +Step 6910/44000 | Loss: 3.1942 | LR: 4.75e-04 | 716.2ms/step | 27.9 samples/s +Step 6920/44000 | Loss: 3.2379 | LR: 4.75e-04 | 711.1ms/step | 28.1 samples/s +Step 6930/44000 | Loss: 3.2105 | LR: 4.75e-04 | 712.3ms/step | 28.1 samples/s +Step 6940/44000 | Loss: 3.2176 | LR: 4.75e-04 | 711.6ms/step | 28.1 samples/s +Step 6950/44000 | Loss: 3.1639 | LR: 4.75e-04 | 711.5ms/step | 28.1 samples/s +Step 6960/44000 | Loss: 3.2931 | LR: 4.75e-04 | 711.9ms/step | 28.1 samples/s +Step 6970/44000 | Loss: 3.2282 | LR: 4.75e-04 | 711.9ms/step | 28.1 samples/s +Step 6980/44000 | Loss: 3.2459 | LR: 4.75e-04 | 711.1ms/step | 28.1 samples/s +Step 6990/44000 | Loss: 3.2574 | LR: 4.75e-04 | 723.7ms/step | 27.6 samples/s +Step 7000/44000 | Loss: 3.2025 | LR: 4.75e-04 | 718.3ms/step | 27.8 samples/s +Validation at step 7000: Loss = 3.2202, Perplexity = 25.03 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-7000 +Step 7010/44000 | Loss: 3.2291 | LR: 4.75e-04 | 1254.0ms/step | 15.9 samples/s +Step 7020/44000 | Loss: 3.1700 | LR: 4.75e-04 | 711.3ms/step | 28.1 samples/s +Step 7030/44000 | Loss: 3.2161 | LR: 4.74e-04 | 714.7ms/step | 28.0 samples/s +Step 7040/44000 | Loss: 3.2268 | LR: 4.74e-04 | 713.3ms/step | 28.0 samples/s +Step 7050/44000 | Loss: 3.2026 | LR: 4.74e-04 | 713.3ms/step | 28.0 samples/s +Step 7060/44000 | Loss: 3.2687 | LR: 4.74e-04 | 712.0ms/step | 28.1 samples/s +Step 7070/44000 | Loss: 3.2237 | LR: 4.74e-04 | 722.6ms/step | 27.7 samples/s +Step 7080/44000 | Loss: 3.1841 | LR: 4.74e-04 | 716.3ms/step | 27.9 samples/s +Step 7090/44000 | Loss: 3.2166 | LR: 4.74e-04 | 712.0ms/step | 28.1 samples/s +Step 7100/44000 | Loss: 3.2519 | LR: 4.74e-04 | 713.0ms/step | 28.1 samples/s +Step 7110/44000 | Loss: 3.2905 | LR: 4.74e-04 | 711.4ms/step | 28.1 samples/s +Step 7120/44000 | Loss: 3.1847 | LR: 4.74e-04 | 713.4ms/step | 28.0 samples/s +Step 7130/44000 | Loss: 3.2817 | LR: 4.74e-04 | 713.5ms/step | 28.0 samples/s +Step 7140/44000 | Loss: 3.2323 | LR: 4.74e-04 | 712.8ms/step | 28.1 samples/s +Step 7150/44000 | Loss: 3.2338 | LR: 4.73e-04 | 712.5ms/step | 28.1 samples/s +Step 7160/44000 | Loss: 3.1408 | LR: 4.73e-04 | 721.6ms/step | 27.7 samples/s +Step 7170/44000 | Loss: 3.2341 | LR: 4.73e-04 | 716.6ms/step | 27.9 samples/s +Step 7180/44000 | Loss: 3.1943 | LR: 4.73e-04 | 712.5ms/step | 28.1 samples/s +Step 7190/44000 | Loss: 3.2503 | LR: 4.73e-04 | 712.5ms/step | 28.1 samples/s +Step 7200/44000 | Loss: 3.2801 | LR: 4.73e-04 | 712.3ms/step | 28.1 samples/s +Step 7210/44000 | Loss: 3.2159 | LR: 4.73e-04 | 713.0ms/step | 28.1 samples/s +Step 7220/44000 | Loss: 3.2479 | LR: 4.73e-04 | 712.8ms/step | 28.1 samples/s +Step 7230/44000 | Loss: 3.2612 | LR: 4.73e-04 | 712.6ms/step | 28.1 samples/s +Step 7240/44000 | Loss: 3.2429 | LR: 4.73e-04 | 712.4ms/step | 28.1 samples/s +Step 7250/44000 | Loss: 3.2312 | LR: 4.73e-04 | 732.5ms/step | 27.3 samples/s +Validation at step 7250: Loss = 3.2118, Perplexity = 24.82 +Step 7260/44000 | Loss: 3.2160 | LR: 4.73e-04 | 931.1ms/step | 21.5 samples/s +Step 7270/44000 | Loss: 3.2485 | LR: 4.72e-04 | 712.5ms/step | 28.1 samples/s +Step 7280/44000 | Loss: 3.2779 | LR: 4.72e-04 | 713.0ms/step | 28.0 samples/s +Step 7290/44000 | Loss: 3.2718 | LR: 4.72e-04 | 713.9ms/step | 28.0 samples/s +Step 7300/44000 | Loss: 3.2796 | LR: 4.72e-04 | 713.5ms/step | 28.0 samples/s +Step 7310/44000 | Loss: 3.1933 | LR: 4.72e-04 | 711.4ms/step | 28.1 samples/s +Step 7320/44000 | Loss: 3.2013 | LR: 4.72e-04 | 711.8ms/step | 28.1 samples/s +Step 7330/44000 | Loss: 3.2303 | LR: 4.72e-04 | 712.3ms/step | 28.1 samples/s +Step 7340/44000 | Loss: 3.2152 | LR: 4.72e-04 | 726.2ms/step | 27.5 samples/s +Step 7350/44000 | Loss: 3.2115 | LR: 4.72e-04 | 712.2ms/step | 28.1 samples/s +Step 7360/44000 | Loss: 3.2362 | LR: 4.72e-04 | 713.2ms/step | 28.0 samples/s +Step 7370/44000 | Loss: 3.2676 | LR: 4.72e-04 | 712.4ms/step | 28.1 samples/s +Step 7380/44000 | Loss: 3.2374 | LR: 4.72e-04 | 711.8ms/step | 28.1 samples/s +Step 7390/44000 | Loss: 3.1823 | LR: 4.71e-04 | 711.5ms/step | 28.1 samples/s +Step 7400/44000 | Loss: 3.1958 | LR: 4.71e-04 | 712.2ms/step | 28.1 samples/s +Step 7410/44000 | Loss: 3.2034 | LR: 4.71e-04 | 711.9ms/step | 28.1 samples/s +Step 7420/44000 | Loss: 3.2423 | LR: 4.71e-04 | 711.6ms/step | 28.1 samples/s +Step 7430/44000 | Loss: 3.2624 | LR: 4.71e-04 | 727.8ms/step | 27.5 samples/s +Step 7440/44000 | Loss: 3.2022 | LR: 4.71e-04 | 711.7ms/step | 28.1 samples/s +Step 7450/44000 | Loss: 3.2942 | LR: 4.71e-04 | 711.7ms/step | 28.1 samples/s +Step 7460/44000 | Loss: 3.1997 | LR: 4.71e-04 | 711.4ms/step | 28.1 samples/s +Step 7470/44000 | Loss: 3.2353 | LR: 4.71e-04 | 711.0ms/step | 28.1 samples/s +Step 7480/44000 | Loss: 3.1679 | LR: 4.71e-04 | 711.6ms/step | 28.1 samples/s +Step 7490/44000 | Loss: 3.1872 | LR: 4.71e-04 | 711.4ms/step | 28.1 samples/s +Step 7500/44000 | Loss: 3.2526 | LR: 4.71e-04 | 711.2ms/step | 28.1 samples/s +Validation at step 7500: Loss = 3.2030, Perplexity = 24.61 +Step 7510/44000 | Loss: 3.2638 | LR: 4.70e-04 | 945.5ms/step | 21.2 samples/s +Step 7520/44000 | Loss: 3.2144 | LR: 4.70e-04 | 711.4ms/step | 28.1 samples/s +Step 7530/44000 | Loss: 3.1825 | LR: 4.70e-04 | 712.2ms/step | 28.1 samples/s +Step 7540/44000 | Loss: 3.2745 | LR: 4.70e-04 | 711.0ms/step | 28.1 samples/s +Step 7550/44000 | Loss: 3.1784 | LR: 4.70e-04 | 714.0ms/step | 28.0 samples/s +Step 7560/44000 | Loss: 3.2242 | LR: 4.70e-04 | 713.0ms/step | 28.1 samples/s +Step 7570/44000 | Loss: 3.2572 | LR: 4.70e-04 | 710.8ms/step | 28.1 samples/s +Step 7580/44000 | Loss: 3.2124 | LR: 4.70e-04 | 711.1ms/step | 28.1 samples/s +Step 7590/44000 | Loss: 3.1863 | LR: 4.70e-04 | 712.4ms/step | 28.1 samples/s +Step 7600/44000 | Loss: 3.2646 | LR: 4.70e-04 | 728.2ms/step | 27.5 samples/s +Step 7610/44000 | Loss: 3.1728 | LR: 4.70e-04 | 713.3ms/step | 28.0 samples/s +Step 7620/44000 | Loss: 3.2156 | LR: 4.70e-04 | 712.5ms/step | 28.1 samples/s +Step 7630/44000 | Loss: 3.1620 | LR: 4.69e-04 | 715.6ms/step | 27.9 samples/s +Step 7640/44000 | Loss: 3.2034 | LR: 4.69e-04 | 711.9ms/step | 28.1 samples/s +Step 7650/44000 | Loss: 3.2874 | LR: 4.69e-04 | 711.2ms/step | 28.1 samples/s +Step 7660/44000 | Loss: 3.2437 | LR: 4.69e-04 | 711.8ms/step | 28.1 samples/s +Step 7670/44000 | Loss: 3.2594 | LR: 4.69e-04 | 712.0ms/step | 28.1 samples/s +Step 7680/44000 | Loss: 3.1905 | LR: 4.69e-04 | 715.3ms/step | 28.0 samples/s +Step 7690/44000 | Loss: 3.2007 | LR: 4.69e-04 | 726.1ms/step | 27.5 samples/s +Step 7700/44000 | Loss: 3.2065 | LR: 4.69e-04 | 713.1ms/step | 28.0 samples/s +Step 7710/44000 | Loss: 3.2421 | LR: 4.69e-04 | 712.1ms/step | 28.1 samples/s +Step 7720/44000 | Loss: 3.1593 | LR: 4.69e-04 | 711.1ms/step | 28.1 samples/s +Step 7730/44000 | Loss: 3.2423 | LR: 4.69e-04 | 711.4ms/step | 28.1 samples/s +Step 7740/44000 | Loss: 3.2107 | LR: 4.68e-04 | 712.0ms/step | 28.1 samples/s +Step 7750/44000 | Loss: 3.2343 | LR: 4.68e-04 | 712.9ms/step | 28.1 samples/s +Validation at step 7750: Loss = 3.1934, Perplexity = 24.37 +Step 7760/44000 | Loss: 3.1469 | LR: 4.68e-04 | 930.9ms/step | 21.5 samples/s +Step 7770/44000 | Loss: 3.1601 | LR: 4.68e-04 | 718.6ms/step | 27.8 samples/s +Step 7780/44000 | Loss: 3.2037 | LR: 4.68e-04 | 722.2ms/step | 27.7 samples/s +Step 7790/44000 | Loss: 3.2138 | LR: 4.68e-04 | 713.1ms/step | 28.0 samples/s +Step 7800/44000 | Loss: 3.1694 | LR: 4.68e-04 | 712.1ms/step | 28.1 samples/s +Step 7810/44000 | Loss: 3.2701 | LR: 4.68e-04 | 711.1ms/step | 28.1 samples/s +Step 7820/44000 | Loss: 3.2257 | LR: 4.68e-04 | 711.4ms/step | 28.1 samples/s +Step 7830/44000 | Loss: 3.1830 | LR: 4.68e-04 | 710.5ms/step | 28.1 samples/s +Step 7840/44000 | Loss: 3.2223 | LR: 4.68e-04 | 711.6ms/step | 28.1 samples/s +Step 7850/44000 | Loss: 3.1971 | LR: 4.67e-04 | 712.0ms/step | 28.1 samples/s +Step 7860/44000 | Loss: 3.2190 | LR: 4.67e-04 | 722.5ms/step | 27.7 samples/s +Step 7870/44000 | Loss: 3.2570 | LR: 4.67e-04 | 717.5ms/step | 27.9 samples/s +Step 7880/44000 | Loss: 3.2110 | LR: 4.67e-04 | 715.1ms/step | 28.0 samples/s +Step 7890/44000 | Loss: 3.1811 | LR: 4.67e-04 | 712.1ms/step | 28.1 samples/s +Step 7900/44000 | Loss: 3.2245 | LR: 4.67e-04 | 712.7ms/step | 28.1 samples/s +Step 7910/44000 | Loss: 3.1870 | LR: 4.67e-04 | 713.3ms/step | 28.0 samples/s +Step 7920/44000 | Loss: 3.2403 | LR: 4.67e-04 | 712.0ms/step | 28.1 samples/s +Step 7930/44000 | Loss: 3.2364 | LR: 4.67e-04 | 711.9ms/step | 28.1 samples/s +Step 7940/44000 | Loss: 3.2141 | LR: 4.67e-04 | 711.9ms/step | 28.1 samples/s +Step 7950/44000 | Loss: 3.1527 | LR: 4.67e-04 | 720.3ms/step | 27.8 samples/s +Step 7960/44000 | Loss: 3.2152 | LR: 4.66e-04 | 715.6ms/step | 27.9 samples/s +Step 7970/44000 | Loss: 3.2159 | LR: 4.66e-04 | 713.0ms/step | 28.1 samples/s +Step 7980/44000 | Loss: 3.2742 | LR: 4.66e-04 | 713.0ms/step | 28.1 samples/s +Step 7990/44000 | Loss: 3.2180 | LR: 4.66e-04 | 713.0ms/step | 28.0 samples/s +Step 8000/44000 | Loss: 3.2004 | LR: 4.66e-04 | 712.2ms/step | 28.1 samples/s +Validation at step 8000: Loss = 3.1876, Perplexity = 24.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-8000 +Step 8010/44000 | Loss: 3.1564 | LR: 4.66e-04 | 1291.2ms/step | 15.5 samples/s +Step 8020/44000 | Loss: 3.1773 | LR: 4.66e-04 | 709.9ms/step | 28.2 samples/s +Step 8030/44000 | Loss: 3.1485 | LR: 4.66e-04 | 720.4ms/step | 27.8 samples/s +Step 8040/44000 | Loss: 3.1921 | LR: 4.66e-04 | 717.0ms/step | 27.9 samples/s +Step 8050/44000 | Loss: 3.1643 | LR: 4.66e-04 | 710.9ms/step | 28.1 samples/s +Step 8060/44000 | Loss: 3.1760 | LR: 4.66e-04 | 710.9ms/step | 28.1 samples/s +Step 8070/44000 | Loss: 3.1711 | LR: 4.65e-04 | 712.3ms/step | 28.1 samples/s +Step 8080/44000 | Loss: 3.2438 | LR: 4.65e-04 | 710.8ms/step | 28.1 samples/s +Step 8090/44000 | Loss: 3.1445 | LR: 4.65e-04 | 710.8ms/step | 28.1 samples/s +Step 8100/44000 | Loss: 3.2005 | LR: 4.65e-04 | 710.7ms/step | 28.1 samples/s +Step 8110/44000 | Loss: 3.1715 | LR: 4.65e-04 | 710.8ms/step | 28.1 samples/s +Step 8120/44000 | Loss: 3.2236 | LR: 4.65e-04 | 722.6ms/step | 27.7 samples/s +Step 8130/44000 | Loss: 3.1800 | LR: 4.65e-04 | 715.3ms/step | 28.0 samples/s +Step 8140/44000 | Loss: 3.2812 | LR: 4.65e-04 | 711.3ms/step | 28.1 samples/s +Step 8150/44000 | Loss: 3.1635 | LR: 4.65e-04 | 714.1ms/step | 28.0 samples/s +Step 8160/44000 | Loss: 3.1457 | LR: 4.65e-04 | 712.3ms/step | 28.1 samples/s +Step 8170/44000 | Loss: 3.1821 | LR: 4.65e-04 | 710.5ms/step | 28.1 samples/s +Step 8180/44000 | Loss: 3.1328 | LR: 4.64e-04 | 711.2ms/step | 28.1 samples/s +Step 8190/44000 | Loss: 3.2341 | LR: 4.64e-04 | 710.4ms/step | 28.2 samples/s +Step 8200/44000 | Loss: 3.2560 | LR: 4.64e-04 | 711.2ms/step | 28.1 samples/s +Step 8210/44000 | Loss: 3.1594 | LR: 4.64e-04 | 728.5ms/step | 27.5 samples/s +Step 8220/44000 | Loss: 3.2011 | LR: 4.64e-04 | 712.2ms/step | 28.1 samples/s +Step 8230/44000 | Loss: 3.2180 | LR: 4.64e-04 | 710.9ms/step | 28.1 samples/s +Step 8240/44000 | Loss: 3.2231 | LR: 4.64e-04 | 711.9ms/step | 28.1 samples/s +Step 8250/44000 | Loss: 3.1741 | LR: 4.64e-04 | 712.5ms/step | 28.1 samples/s +Validation at step 8250: Loss = 3.1815, Perplexity = 24.08 +Step 8260/44000 | Loss: 3.1471 | LR: 4.64e-04 | 929.3ms/step | 21.5 samples/s +Step 8270/44000 | Loss: 3.1902 | LR: 4.64e-04 | 710.4ms/step | 28.2 samples/s +Step 8280/44000 | Loss: 3.1699 | LR: 4.64e-04 | 711.3ms/step | 28.1 samples/s +Step 8290/44000 | Loss: 3.2202 | LR: 4.63e-04 | 712.6ms/step | 28.1 samples/s +Step 8300/44000 | Loss: 3.2703 | LR: 4.63e-04 | 725.1ms/step | 27.6 samples/s +Step 8310/44000 | Loss: 3.1841 | LR: 4.63e-04 | 711.2ms/step | 28.1 samples/s +Step 8320/44000 | Loss: 3.2207 | LR: 4.63e-04 | 711.0ms/step | 28.1 samples/s +Step 8330/44000 | Loss: 3.1238 | LR: 4.63e-04 | 711.5ms/step | 28.1 samples/s +Step 8340/44000 | Loss: 3.1385 | LR: 4.63e-04 | 712.0ms/step | 28.1 samples/s +Step 8350/44000 | Loss: 3.1989 | LR: 4.63e-04 | 711.8ms/step | 28.1 samples/s +Step 8360/44000 | Loss: 3.1717 | LR: 4.63e-04 | 715.0ms/step | 28.0 samples/s +Step 8370/44000 | Loss: 3.1840 | LR: 4.63e-04 | 714.9ms/step | 28.0 samples/s +Step 8380/44000 | Loss: 3.1814 | LR: 4.63e-04 | 715.3ms/step | 28.0 samples/s +Step 8390/44000 | Loss: 3.1854 | LR: 4.62e-04 | 730.5ms/step | 27.4 samples/s +Step 8400/44000 | Loss: 3.2005 | LR: 4.62e-04 | 712.8ms/step | 28.1 samples/s +Step 8410/44000 | Loss: 3.1657 | LR: 4.62e-04 | 711.6ms/step | 28.1 samples/s +Step 8420/44000 | Loss: 3.1630 | LR: 4.62e-04 | 711.2ms/step | 28.1 samples/s +Step 8430/44000 | Loss: 3.1820 | LR: 4.62e-04 | 711.9ms/step | 28.1 samples/s +Step 8440/44000 | Loss: 3.1815 | LR: 4.62e-04 | 711.4ms/step | 28.1 samples/s +Step 8450/44000 | Loss: 3.2455 | LR: 4.62e-04 | 711.4ms/step | 28.1 samples/s +Step 8460/44000 | Loss: 3.2719 | LR: 4.62e-04 | 711.3ms/step | 28.1 samples/s +Step 8470/44000 | Loss: 3.1960 | LR: 4.62e-04 | 715.6ms/step | 27.9 samples/s +Step 8480/44000 | Loss: 3.1699 | LR: 4.62e-04 | 722.8ms/step | 27.7 samples/s +Step 8490/44000 | Loss: 3.1604 | LR: 4.62e-04 | 710.5ms/step | 28.1 samples/s +Step 8500/44000 | Loss: 3.2151 | LR: 4.61e-04 | 709.6ms/step | 28.2 samples/s +Validation at step 8500: Loss = 3.1729, Perplexity = 23.88 +Step 8510/44000 | Loss: 3.1541 | LR: 4.61e-04 | 929.6ms/step | 21.5 samples/s +Step 8520/44000 | Loss: 3.1862 | LR: 4.61e-04 | 710.3ms/step | 28.2 samples/s +Step 8530/44000 | Loss: 3.2610 | LR: 4.61e-04 | 711.4ms/step | 28.1 samples/s +Step 8540/44000 | Loss: 3.1863 | LR: 4.61e-04 | 712.9ms/step | 28.1 samples/s +Step 8550/44000 | Loss: 3.2043 | LR: 4.61e-04 | 713.0ms/step | 28.1 samples/s +Step 8560/44000 | Loss: 3.1291 | LR: 4.61e-04 | 728.3ms/step | 27.5 samples/s +Step 8570/44000 | Loss: 3.1606 | LR: 4.61e-04 | 712.7ms/step | 28.1 samples/s +Step 8580/44000 | Loss: 3.1821 | LR: 4.61e-04 | 711.8ms/step | 28.1 samples/s +Step 8590/44000 | Loss: 3.1975 | LR: 4.61e-04 | 711.9ms/step | 28.1 samples/s +Step 8600/44000 | Loss: 3.1479 | LR: 4.60e-04 | 712.3ms/step | 28.1 samples/s +Step 8610/44000 | Loss: 3.2046 | LR: 4.60e-04 | 712.0ms/step | 28.1 samples/s +Step 8620/44000 | Loss: 3.1575 | LR: 4.60e-04 | 714.3ms/step | 28.0 samples/s +Step 8630/44000 | Loss: 3.1788 | LR: 4.60e-04 | 711.7ms/step | 28.1 samples/s +Step 8640/44000 | Loss: 3.2301 | LR: 4.60e-04 | 711.6ms/step | 28.1 samples/s +Step 8650/44000 | Loss: 3.1917 | LR: 4.60e-04 | 727.1ms/step | 27.5 samples/s +Step 8660/44000 | Loss: 3.1773 | LR: 4.60e-04 | 711.7ms/step | 28.1 samples/s +Step 8670/44000 | Loss: 3.1550 | LR: 4.60e-04 | 711.8ms/step | 28.1 samples/s +Step 8680/44000 | Loss: 3.2108 | LR: 4.60e-04 | 712.5ms/step | 28.1 samples/s +Step 8690/44000 | Loss: 3.1899 | LR: 4.60e-04 | 711.7ms/step | 28.1 samples/s +Step 8700/44000 | Loss: 3.1736 | LR: 4.59e-04 | 711.7ms/step | 28.1 samples/s +Step 8710/44000 | Loss: 3.1624 | LR: 4.59e-04 | 712.5ms/step | 28.1 samples/s +Step 8720/44000 | Loss: 3.1306 | LR: 4.59e-04 | 712.2ms/step | 28.1 samples/s +Step 8730/44000 | Loss: 3.1770 | LR: 4.59e-04 | 711.4ms/step | 28.1 samples/s +Step 8740/44000 | Loss: 3.1736 | LR: 4.59e-04 | 725.2ms/step | 27.6 samples/s +Step 8750/44000 | Loss: 3.2187 | LR: 4.59e-04 | 711.9ms/step | 28.1 samples/s +Validation at step 8750: Loss = 3.1618, Perplexity = 23.61 +Step 8760/44000 | Loss: 3.0954 | LR: 4.59e-04 | 931.3ms/step | 21.5 samples/s +Step 8770/44000 | Loss: 3.2129 | LR: 4.59e-04 | 712.5ms/step | 28.1 samples/s +Step 8780/44000 | Loss: 3.1739 | LR: 4.59e-04 | 712.5ms/step | 28.1 samples/s +Step 8790/44000 | Loss: 3.1736 | LR: 4.59e-04 | 712.3ms/step | 28.1 samples/s +Step 8800/44000 | Loss: 3.1269 | LR: 4.58e-04 | 712.8ms/step | 28.1 samples/s +Step 8810/44000 | Loss: 3.1825 | LR: 4.58e-04 | 712.1ms/step | 28.1 samples/s +Step 8820/44000 | Loss: 3.1936 | LR: 4.58e-04 | 718.5ms/step | 27.8 samples/s +Step 8830/44000 | Loss: 3.2052 | LR: 4.58e-04 | 720.5ms/step | 27.8 samples/s +Step 8840/44000 | Loss: 3.1943 | LR: 4.58e-04 | 713.0ms/step | 28.0 samples/s +Step 8850/44000 | Loss: 3.1052 | LR: 4.58e-04 | 712.1ms/step | 28.1 samples/s +Step 8860/44000 | Loss: 3.1769 | LR: 4.58e-04 | 712.2ms/step | 28.1 samples/s +Step 8870/44000 | Loss: 3.1642 | LR: 4.58e-04 | 712.4ms/step | 28.1 samples/s +Step 8880/44000 | Loss: 3.2344 | LR: 4.58e-04 | 712.8ms/step | 28.1 samples/s +Step 8890/44000 | Loss: 3.2474 | LR: 4.58e-04 | 713.1ms/step | 28.0 samples/s +Step 8900/44000 | Loss: 3.1920 | LR: 4.57e-04 | 713.0ms/step | 28.1 samples/s +Step 8910/44000 | Loss: 3.1484 | LR: 4.57e-04 | 722.0ms/step | 27.7 samples/s +Step 8920/44000 | Loss: 3.1951 | LR: 4.57e-04 | 716.9ms/step | 27.9 samples/s +Step 8930/44000 | Loss: 3.1950 | LR: 4.57e-04 | 713.1ms/step | 28.0 samples/s +Step 8940/44000 | Loss: 3.2069 | LR: 4.57e-04 | 713.1ms/step | 28.0 samples/s +Step 8950/44000 | Loss: 3.0983 | LR: 4.57e-04 | 712.9ms/step | 28.1 samples/s +Step 8960/44000 | Loss: 3.1679 | LR: 4.57e-04 | 711.5ms/step | 28.1 samples/s +Step 8970/44000 | Loss: 3.1177 | LR: 4.57e-04 | 711.4ms/step | 28.1 samples/s +Step 8980/44000 | Loss: 3.1893 | LR: 4.57e-04 | 712.1ms/step | 28.1 samples/s +Step 8990/44000 | Loss: 3.2098 | LR: 4.57e-04 | 711.8ms/step | 28.1 samples/s +Step 9000/44000 | Loss: 3.2021 | LR: 4.56e-04 | 722.3ms/step | 27.7 samples/s +Validation at step 9000: Loss = 3.1561, Perplexity = 23.48 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-9000 +Step 9010/44000 | Loss: 3.1946 | LR: 4.56e-04 | 1344.9ms/step | 14.9 samples/s +Step 9020/44000 | Loss: 3.1707 | LR: 4.56e-04 | 712.4ms/step | 28.1 samples/s +Step 9030/44000 | Loss: 3.1236 | LR: 4.56e-04 | 712.4ms/step | 28.1 samples/s +Step 9040/44000 | Loss: 3.2034 | LR: 4.56e-04 | 712.1ms/step | 28.1 samples/s +Step 9050/44000 | Loss: 3.2095 | LR: 4.56e-04 | 713.1ms/step | 28.0 samples/s +Step 9060/44000 | Loss: 3.2092 | LR: 4.56e-04 | 712.5ms/step | 28.1 samples/s +Step 9070/44000 | Loss: 3.1233 | LR: 4.56e-04 | 712.9ms/step | 28.1 samples/s +Step 9080/44000 | Loss: 3.1849 | LR: 4.56e-04 | 722.1ms/step | 27.7 samples/s +Step 9090/44000 | Loss: 3.2021 | LR: 4.56e-04 | 719.6ms/step | 27.8 samples/s +Step 9100/44000 | Loss: 3.1647 | LR: 4.55e-04 | 712.6ms/step | 28.1 samples/s +Step 9110/44000 | Loss: 3.2025 | LR: 4.55e-04 | 711.7ms/step | 28.1 samples/s +Step 9120/44000 | Loss: 3.1175 | LR: 4.55e-04 | 712.2ms/step | 28.1 samples/s +Step 9130/44000 | Loss: 3.2131 | LR: 4.55e-04 | 712.0ms/step | 28.1 samples/s +Step 9140/44000 | Loss: 3.1590 | LR: 4.55e-04 | 712.3ms/step | 28.1 samples/s +Step 9150/44000 | Loss: 3.2462 | LR: 4.55e-04 | 712.1ms/step | 28.1 samples/s +Step 9160/44000 | Loss: 3.1669 | LR: 4.55e-04 | 712.7ms/step | 28.1 samples/s +Step 9170/44000 | Loss: 3.2079 | LR: 4.55e-04 | 728.2ms/step | 27.5 samples/s +Step 9180/44000 | Loss: 3.1502 | LR: 4.55e-04 | 711.7ms/step | 28.1 samples/s +Step 9190/44000 | Loss: 3.2672 | LR: 4.54e-04 | 711.7ms/step | 28.1 samples/s +Step 9200/44000 | Loss: 3.1337 | LR: 4.54e-04 | 712.2ms/step | 28.1 samples/s +Step 9210/44000 | Loss: 3.1605 | LR: 4.54e-04 | 712.1ms/step | 28.1 samples/s +Step 9220/44000 | Loss: 3.2235 | LR: 4.54e-04 | 711.5ms/step | 28.1 samples/s +Step 9230/44000 | Loss: 3.2153 | LR: 4.54e-04 | 711.7ms/step | 28.1 samples/s +Step 9240/44000 | Loss: 3.2413 | LR: 4.54e-04 | 712.1ms/step | 28.1 samples/s +Step 9250/44000 | Loss: 3.1737 | LR: 4.54e-04 | 712.7ms/step | 28.1 samples/s +Validation at step 9250: Loss = 3.1489, Perplexity = 23.31 +Step 9260/44000 | Loss: 3.2378 | LR: 4.54e-04 | 950.4ms/step | 21.0 samples/s +Step 9270/44000 | Loss: 3.1237 | LR: 4.54e-04 | 713.5ms/step | 28.0 samples/s +Step 9280/44000 | Loss: 3.1359 | LR: 4.54e-04 | 712.4ms/step | 28.1 samples/s +Step 9290/44000 | Loss: 3.2278 | LR: 4.53e-04 | 713.5ms/step | 28.0 samples/s +Step 9300/44000 | Loss: 3.1912 | LR: 4.53e-04 | 713.0ms/step | 28.1 samples/s +Step 9310/44000 | Loss: 3.2116 | LR: 4.53e-04 | 713.8ms/step | 28.0 samples/s +Step 9320/44000 | Loss: 3.1287 | LR: 4.53e-04 | 715.5ms/step | 28.0 samples/s +Step 9330/44000 | Loss: 3.1558 | LR: 4.53e-04 | 714.4ms/step | 28.0 samples/s +Step 9340/44000 | Loss: 3.2270 | LR: 4.53e-04 | 714.5ms/step | 28.0 samples/s +Step 9350/44000 | Loss: 3.1909 | LR: 4.53e-04 | 727.9ms/step | 27.5 samples/s +Step 9360/44000 | Loss: 3.2146 | LR: 4.53e-04 | 713.1ms/step | 28.0 samples/s +Step 9370/44000 | Loss: 3.2565 | LR: 4.53e-04 | 713.1ms/step | 28.0 samples/s +Step 9380/44000 | Loss: 3.2015 | LR: 4.52e-04 | 711.2ms/step | 28.1 samples/s +Step 9390/44000 | Loss: 3.1797 | LR: 4.52e-04 | 711.4ms/step | 28.1 samples/s +Step 9400/44000 | Loss: 3.2238 | LR: 4.52e-04 | 712.6ms/step | 28.1 samples/s +Step 9410/44000 | Loss: 3.2144 | LR: 4.52e-04 | 713.2ms/step | 28.0 samples/s +Step 9420/44000 | Loss: 3.2017 | LR: 4.52e-04 | 712.7ms/step | 28.1 samples/s +Step 9430/44000 | Loss: 3.2319 | LR: 4.52e-04 | 724.5ms/step | 27.6 samples/s +Step 9440/44000 | Loss: 3.0710 | LR: 4.52e-04 | 717.3ms/step | 27.9 samples/s +Step 9450/44000 | Loss: 3.1538 | LR: 4.52e-04 | 712.5ms/step | 28.1 samples/s +Step 9460/44000 | Loss: 3.1617 | LR: 4.52e-04 | 712.4ms/step | 28.1 samples/s +Step 9470/44000 | Loss: 3.2108 | LR: 4.52e-04 | 711.8ms/step | 28.1 samples/s +Step 9480/44000 | Loss: 3.1940 | LR: 4.51e-04 | 712.0ms/step | 28.1 samples/s +Step 9490/44000 | Loss: 3.2209 | LR: 4.51e-04 | 710.7ms/step | 28.1 samples/s +Step 9500/44000 | Loss: 3.1575 | LR: 4.51e-04 | 711.2ms/step | 28.1 samples/s +Validation at step 9500: Loss = 3.1454, Perplexity = 23.23 +Step 9510/44000 | Loss: 3.1364 | LR: 4.51e-04 | 930.2ms/step | 21.5 samples/s +Step 9520/44000 | Loss: 3.1490 | LR: 4.51e-04 | 727.1ms/step | 27.5 samples/s +Step 9530/44000 | Loss: 3.2210 | LR: 4.51e-04 | 711.5ms/step | 28.1 samples/s +Step 9540/44000 | Loss: 3.2049 | LR: 4.51e-04 | 712.3ms/step | 28.1 samples/s +Step 9550/44000 | Loss: 3.1072 | LR: 4.51e-04 | 714.4ms/step | 28.0 samples/s +Step 9560/44000 | Loss: 3.1704 | LR: 4.51e-04 | 713.5ms/step | 28.0 samples/s +Step 9570/44000 | Loss: 3.1380 | LR: 4.50e-04 | 712.6ms/step | 28.1 samples/s +Step 9580/44000 | Loss: 3.2212 | LR: 4.50e-04 | 714.7ms/step | 28.0 samples/s +Step 9590/44000 | Loss: 3.1930 | LR: 4.50e-04 | 714.7ms/step | 28.0 samples/s +Step 9600/44000 | Loss: 3.1341 | LR: 4.50e-04 | 715.5ms/step | 28.0 samples/s +Step 9610/44000 | Loss: 3.1579 | LR: 4.50e-04 | 731.3ms/step | 27.3 samples/s +Step 9620/44000 | Loss: 3.0819 | LR: 4.50e-04 | 715.3ms/step | 28.0 samples/s +Step 9630/44000 | Loss: 3.1054 | LR: 4.50e-04 | 713.7ms/step | 28.0 samples/s +Step 9640/44000 | Loss: 3.0527 | LR: 4.50e-04 | 714.1ms/step | 28.0 samples/s +Step 9650/44000 | Loss: 3.2020 | LR: 4.50e-04 | 713.9ms/step | 28.0 samples/s +Step 9660/44000 | Loss: 3.1715 | LR: 4.49e-04 | 713.9ms/step | 28.0 samples/s +Step 9670/44000 | Loss: 3.1508 | LR: 4.49e-04 | 714.1ms/step | 28.0 samples/s +Step 9680/44000 | Loss: 3.1241 | LR: 4.49e-04 | 712.6ms/step | 28.1 samples/s +Step 9690/44000 | Loss: 3.2587 | LR: 4.49e-04 | 713.0ms/step | 28.1 samples/s +Step 9700/44000 | Loss: 3.1117 | LR: 4.49e-04 | 730.9ms/step | 27.4 samples/s +Step 9710/44000 | Loss: 3.1935 | LR: 4.49e-04 | 712.3ms/step | 28.1 samples/s +Step 9720/44000 | Loss: 3.1498 | LR: 4.49e-04 | 712.0ms/step | 28.1 samples/s +Step 9730/44000 | Loss: 3.1996 | LR: 4.49e-04 | 712.7ms/step | 28.1 samples/s +Step 9740/44000 | Loss: 3.1829 | LR: 4.49e-04 | 712.6ms/step | 28.1 samples/s +Step 9750/44000 | Loss: 3.1687 | LR: 4.48e-04 | 712.8ms/step | 28.1 samples/s +Validation at step 9750: Loss = 3.1384, Perplexity = 23.07 +Step 9760/44000 | Loss: 3.1590 | LR: 4.48e-04 | 930.1ms/step | 21.5 samples/s +Step 9770/44000 | Loss: 3.0927 | LR: 4.48e-04 | 712.6ms/step | 28.1 samples/s +Step 9780/44000 | Loss: 3.2366 | LR: 4.48e-04 | 722.3ms/step | 27.7 samples/s +Step 9790/44000 | Loss: 3.1276 | LR: 4.48e-04 | 717.0ms/step | 27.9 samples/s +Step 9800/44000 | Loss: 3.1448 | LR: 4.48e-04 | 713.1ms/step | 28.0 samples/s +Step 9810/44000 | Loss: 3.1675 | LR: 4.48e-04 | 713.6ms/step | 28.0 samples/s +Step 9820/44000 | Loss: 3.1429 | LR: 4.48e-04 | 712.7ms/step | 28.1 samples/s +Step 9830/44000 | Loss: 3.1913 | LR: 4.48e-04 | 713.4ms/step | 28.0 samples/s +Step 9840/44000 | Loss: 3.1448 | LR: 4.47e-04 | 712.6ms/step | 28.1 samples/s +Step 9850/44000 | Loss: 3.1392 | LR: 4.47e-04 | 712.4ms/step | 28.1 samples/s +Step 9860/44000 | Loss: 3.1991 | LR: 4.47e-04 | 713.1ms/step | 28.0 samples/s +Step 9870/44000 | Loss: 3.1279 | LR: 4.47e-04 | 730.9ms/step | 27.4 samples/s +Step 9880/44000 | Loss: 3.1471 | LR: 4.47e-04 | 714.0ms/step | 28.0 samples/s +Step 9890/44000 | Loss: 3.1338 | LR: 4.47e-04 | 715.4ms/step | 28.0 samples/s +Step 9900/44000 | Loss: 3.1529 | LR: 4.47e-04 | 714.2ms/step | 28.0 samples/s +Step 9910/44000 | Loss: 3.0993 | LR: 4.47e-04 | 713.8ms/step | 28.0 samples/s +Step 9920/44000 | Loss: 3.1383 | LR: 4.47e-04 | 714.1ms/step | 28.0 samples/s +Step 9930/44000 | Loss: 3.1500 | LR: 4.46e-04 | 714.1ms/step | 28.0 samples/s +Step 9940/44000 | Loss: 3.1307 | LR: 4.46e-04 | 713.8ms/step | 28.0 samples/s +Step 9950/44000 | Loss: 3.1842 | LR: 4.46e-04 | 713.4ms/step | 28.0 samples/s +Step 9960/44000 | Loss: 3.1582 | LR: 4.46e-04 | 730.5ms/step | 27.4 samples/s +Step 9970/44000 | Loss: 3.1909 | LR: 4.46e-04 | 714.3ms/step | 28.0 samples/s +Step 9980/44000 | Loss: 3.1473 | LR: 4.46e-04 | 713.8ms/step | 28.0 samples/s +Step 9990/44000 | Loss: 3.1774 | LR: 4.46e-04 | 713.4ms/step | 28.0 samples/s +Step 10000/44000 | Loss: 3.1643 | LR: 4.46e-04 | 713.9ms/step | 28.0 samples/s +Validation at step 10000: Loss = 3.1336, Perplexity = 22.96 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-10000 +Step 10010/44000 | Loss: 3.1607 | LR: 4.46e-04 | 1300.3ms/step | 15.4 samples/s +Step 10020/44000 | Loss: 3.1046 | LR: 4.45e-04 | 713.8ms/step | 28.0 samples/s +Step 10030/44000 | Loss: 3.0822 | LR: 4.45e-04 | 713.4ms/step | 28.0 samples/s +Step 10040/44000 | Loss: 3.1205 | LR: 4.45e-04 | 727.4ms/step | 27.5 samples/s +Step 10050/44000 | Loss: 3.1673 | LR: 4.45e-04 | 714.1ms/step | 28.0 samples/s +Step 10060/44000 | Loss: 3.1747 | LR: 4.45e-04 | 713.2ms/step | 28.0 samples/s +Step 10070/44000 | Loss: 3.1672 | LR: 4.45e-04 | 713.5ms/step | 28.0 samples/s +Step 10080/44000 | Loss: 3.2007 | LR: 4.45e-04 | 712.8ms/step | 28.1 samples/s +Step 10090/44000 | Loss: 3.2116 | LR: 4.45e-04 | 712.8ms/step | 28.1 samples/s +Step 10100/44000 | Loss: 3.1696 | LR: 4.45e-04 | 713.5ms/step | 28.0 samples/s +Step 10110/44000 | Loss: 3.1679 | LR: 4.44e-04 | 712.8ms/step | 28.1 samples/s +Step 10120/44000 | Loss: 3.0920 | LR: 4.44e-04 | 712.4ms/step | 28.1 samples/s +Step 10130/44000 | Loss: 3.1084 | LR: 4.44e-04 | 728.6ms/step | 27.5 samples/s +Step 10140/44000 | Loss: 3.1685 | LR: 4.44e-04 | 712.8ms/step | 28.1 samples/s +Step 10150/44000 | Loss: 3.1984 | LR: 4.44e-04 | 712.8ms/step | 28.1 samples/s +Step 10160/44000 | Loss: 3.1478 | LR: 4.44e-04 | 712.7ms/step | 28.1 samples/s +Step 10170/44000 | Loss: 3.1679 | LR: 4.44e-04 | 712.5ms/step | 28.1 samples/s +Step 10180/44000 | Loss: 3.1610 | LR: 4.44e-04 | 712.5ms/step | 28.1 samples/s +Step 10190/44000 | Loss: 3.1012 | LR: 4.43e-04 | 713.0ms/step | 28.1 samples/s +Step 10200/44000 | Loss: 3.1362 | LR: 4.43e-04 | 711.9ms/step | 28.1 samples/s +Step 10210/44000 | Loss: 3.1610 | LR: 4.43e-04 | 712.1ms/step | 28.1 samples/s +Step 10220/44000 | Loss: 3.0976 | LR: 4.43e-04 | 726.5ms/step | 27.5 samples/s +Step 10230/44000 | Loss: 3.1389 | LR: 4.43e-04 | 713.6ms/step | 28.0 samples/s +Step 10240/44000 | Loss: 3.2532 | LR: 4.43e-04 | 713.7ms/step | 28.0 samples/s +Step 10250/44000 | Loss: 3.1860 | LR: 4.43e-04 | 713.9ms/step | 28.0 samples/s +Validation at step 10250: Loss = 3.1287, Perplexity = 22.84 +Step 10260/44000 | Loss: 3.1290 | LR: 4.43e-04 | 932.9ms/step | 21.4 samples/s +Step 10270/44000 | Loss: 3.1661 | LR: 4.43e-04 | 713.9ms/step | 28.0 samples/s +Step 10280/44000 | Loss: 3.1708 | LR: 4.42e-04 | 714.0ms/step | 28.0 samples/s +Step 10290/44000 | Loss: 3.1166 | LR: 4.42e-04 | 714.4ms/step | 28.0 samples/s +Step 10300/44000 | Loss: 3.0796 | LR: 4.42e-04 | 726.7ms/step | 27.5 samples/s +Step 10310/44000 | Loss: 3.1568 | LR: 4.42e-04 | 716.9ms/step | 27.9 samples/s +Step 10320/44000 | Loss: 3.1360 | LR: 4.42e-04 | 713.1ms/step | 28.0 samples/s +Step 10330/44000 | Loss: 3.1227 | LR: 4.42e-04 | 714.2ms/step | 28.0 samples/s +Step 10340/44000 | Loss: 3.1019 | LR: 4.42e-04 | 714.1ms/step | 28.0 samples/s +Step 10350/44000 | Loss: 3.1133 | LR: 4.42e-04 | 714.8ms/step | 28.0 samples/s +Step 10360/44000 | Loss: 3.1497 | LR: 4.42e-04 | 714.7ms/step | 28.0 samples/s +Step 10370/44000 | Loss: 3.2048 | LR: 4.41e-04 | 714.6ms/step | 28.0 samples/s +Step 10380/44000 | Loss: 3.0615 | LR: 4.41e-04 | 715.5ms/step | 28.0 samples/s +Step 10390/44000 | Loss: 3.1368 | LR: 4.41e-04 | 725.6ms/step | 27.6 samples/s +Step 10400/44000 | Loss: 3.1147 | LR: 4.41e-04 | 718.4ms/step | 27.8 samples/s +Step 10410/44000 | Loss: 3.1175 | LR: 4.41e-04 | 713.1ms/step | 28.0 samples/s +Step 10420/44000 | Loss: 3.1379 | LR: 4.41e-04 | 713.3ms/step | 28.0 samples/s +Step 10430/44000 | Loss: 3.1920 | LR: 4.41e-04 | 713.2ms/step | 28.0 samples/s +Step 10440/44000 | Loss: 3.1223 | LR: 4.41e-04 | 713.1ms/step | 28.0 samples/s +Step 10450/44000 | Loss: 3.1260 | LR: 4.40e-04 | 712.5ms/step | 28.1 samples/s +Step 10460/44000 | Loss: 3.0472 | LR: 4.40e-04 | 712.6ms/step | 28.1 samples/s +Step 10470/44000 | Loss: 3.1068 | LR: 4.40e-04 | 712.6ms/step | 28.1 samples/s +Step 10480/44000 | Loss: 3.0962 | LR: 4.40e-04 | 726.1ms/step | 27.5 samples/s +Step 10490/44000 | Loss: 3.1528 | LR: 4.40e-04 | 712.6ms/step | 28.1 samples/s +Step 10500/44000 | Loss: 3.0967 | LR: 4.40e-04 | 712.4ms/step | 28.1 samples/s +Validation at step 10500: Loss = 3.1212, Perplexity = 22.67 +Step 10510/44000 | Loss: 3.1793 | LR: 4.40e-04 | 932.1ms/step | 21.5 samples/s +Step 10520/44000 | Loss: 3.1462 | LR: 4.40e-04 | 714.1ms/step | 28.0 samples/s +Step 10530/44000 | Loss: 3.2117 | LR: 4.40e-04 | 713.8ms/step | 28.0 samples/s +Step 10540/44000 | Loss: 3.1138 | LR: 4.39e-04 | 713.5ms/step | 28.0 samples/s +Step 10550/44000 | Loss: 3.1337 | LR: 4.39e-04 | 712.8ms/step | 28.1 samples/s +Step 10560/44000 | Loss: 3.1410 | LR: 4.39e-04 | 713.7ms/step | 28.0 samples/s +Step 10570/44000 | Loss: 3.1623 | LR: 4.39e-04 | 729.4ms/step | 27.4 samples/s +Step 10580/44000 | Loss: 3.1338 | LR: 4.39e-04 | 712.7ms/step | 28.1 samples/s +Step 10590/44000 | Loss: 3.1140 | LR: 4.39e-04 | 713.4ms/step | 28.0 samples/s +Step 10600/44000 | Loss: 3.1223 | LR: 4.39e-04 | 713.1ms/step | 28.0 samples/s +Step 10610/44000 | Loss: 3.1145 | LR: 4.39e-04 | 713.2ms/step | 28.0 samples/s +Step 10620/44000 | Loss: 3.1594 | LR: 4.38e-04 | 713.4ms/step | 28.0 samples/s +Step 10630/44000 | Loss: 3.1867 | LR: 4.38e-04 | 713.1ms/step | 28.0 samples/s +Step 10640/44000 | Loss: 3.1740 | LR: 4.38e-04 | 713.1ms/step | 28.0 samples/s +Step 10650/44000 | Loss: 3.1723 | LR: 4.38e-04 | 713.3ms/step | 28.0 samples/s +Step 10660/44000 | Loss: 3.1523 | LR: 4.38e-04 | 725.8ms/step | 27.6 samples/s +Step 10670/44000 | Loss: 3.0895 | LR: 4.38e-04 | 711.0ms/step | 28.1 samples/s +Step 10680/44000 | Loss: 3.0690 | LR: 4.38e-04 | 712.6ms/step | 28.1 samples/s +Step 10690/44000 | Loss: 3.1658 | LR: 4.38e-04 | 713.6ms/step | 28.0 samples/s +Step 10700/44000 | Loss: 3.2119 | LR: 4.37e-04 | 713.9ms/step | 28.0 samples/s +Step 10710/44000 | Loss: 3.1585 | LR: 4.37e-04 | 713.5ms/step | 28.0 samples/s +Step 10720/44000 | Loss: 3.0813 | LR: 4.37e-04 | 713.2ms/step | 28.0 samples/s +Step 10730/44000 | Loss: 3.1002 | LR: 4.37e-04 | 713.1ms/step | 28.0 samples/s +Step 10740/44000 | Loss: 3.1123 | LR: 4.37e-04 | 720.8ms/step | 27.7 samples/s +Step 10750/44000 | Loss: 3.1645 | LR: 4.37e-04 | 721.1ms/step | 27.7 samples/s +Validation at step 10750: Loss = 3.1173, Perplexity = 22.59 +Step 10760/44000 | Loss: 3.1244 | LR: 4.37e-04 | 932.8ms/step | 21.4 samples/s +Step 10770/44000 | Loss: 3.1345 | LR: 4.37e-04 | 714.1ms/step | 28.0 samples/s +Step 10780/44000 | Loss: 3.1258 | LR: 4.37e-04 | 712.3ms/step | 28.1 samples/s +Step 10790/44000 | Loss: 3.1425 | LR: 4.36e-04 | 714.0ms/step | 28.0 samples/s +Step 10800/44000 | Loss: 3.1201 | LR: 4.36e-04 | 713.0ms/step | 28.0 samples/s +Step 10810/44000 | Loss: 3.1302 | LR: 4.36e-04 | 714.3ms/step | 28.0 samples/s +Step 10820/44000 | Loss: 3.0857 | LR: 4.36e-04 | 715.2ms/step | 28.0 samples/s +Step 10830/44000 | Loss: 3.1448 | LR: 4.36e-04 | 731.3ms/step | 27.3 samples/s +Step 10840/44000 | Loss: 3.0687 | LR: 4.36e-04 | 713.3ms/step | 28.0 samples/s +Step 10850/44000 | Loss: 3.1750 | LR: 4.36e-04 | 713.2ms/step | 28.0 samples/s +Step 10860/44000 | Loss: 3.1637 | LR: 4.36e-04 | 711.3ms/step | 28.1 samples/s +Step 10870/44000 | Loss: 3.1290 | LR: 4.35e-04 | 712.8ms/step | 28.1 samples/s +Step 10880/44000 | Loss: 3.0595 | LR: 4.35e-04 | 713.8ms/step | 28.0 samples/s +Step 10890/44000 | Loss: 3.1216 | LR: 4.35e-04 | 711.8ms/step | 28.1 samples/s +Step 10900/44000 | Loss: 3.1094 | LR: 4.35e-04 | 711.0ms/step | 28.1 samples/s +Step 10910/44000 | Loss: 3.1007 | LR: 4.35e-04 | 711.3ms/step | 28.1 samples/s +Step 10920/44000 | Loss: 3.1370 | LR: 4.35e-04 | 727.9ms/step | 27.5 samples/s +Step 10930/44000 | Loss: 3.1639 | LR: 4.35e-04 | 711.5ms/step | 28.1 samples/s +Step 10940/44000 | Loss: 3.1887 | LR: 4.35e-04 | 711.8ms/step | 28.1 samples/s +Step 10950/44000 | Loss: 3.1379 | LR: 4.34e-04 | 712.0ms/step | 28.1 samples/s +Step 10960/44000 | Loss: 3.1044 | LR: 4.34e-04 | 712.1ms/step | 28.1 samples/s +Step 10970/44000 | Loss: 3.0914 | LR: 4.34e-04 | 713.6ms/step | 28.0 samples/s +Step 10980/44000 | Loss: 3.1100 | LR: 4.34e-04 | 712.1ms/step | 28.1 samples/s +Step 10990/44000 | Loss: 3.1337 | LR: 4.34e-04 | 712.7ms/step | 28.1 samples/s +Step 11000/44000 | Loss: 3.1806 | LR: 4.34e-04 | 714.3ms/step | 28.0 samples/s +Validation at step 11000: Loss = 3.1143, Perplexity = 22.52 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-11000 +Step 11010/44000 | Loss: 3.0300 | LR: 4.34e-04 | 1313.3ms/step | 15.2 samples/s +Step 11020/44000 | Loss: 3.1847 | LR: 4.34e-04 | 710.9ms/step | 28.1 samples/s +Step 11030/44000 | Loss: 3.1257 | LR: 4.33e-04 | 711.2ms/step | 28.1 samples/s +Step 11040/44000 | Loss: 3.0634 | LR: 4.33e-04 | 711.6ms/step | 28.1 samples/s +Step 11050/44000 | Loss: 3.1153 | LR: 4.33e-04 | 713.9ms/step | 28.0 samples/s +Step 11060/44000 | Loss: 3.1426 | LR: 4.33e-04 | 713.4ms/step | 28.0 samples/s +Step 11070/44000 | Loss: 3.0937 | LR: 4.33e-04 | 714.5ms/step | 28.0 samples/s +Step 11080/44000 | Loss: 3.1739 | LR: 4.33e-04 | 713.8ms/step | 28.0 samples/s +Step 11090/44000 | Loss: 3.1313 | LR: 4.33e-04 | 731.4ms/step | 27.3 samples/s +Step 11100/44000 | Loss: 3.1185 | LR: 4.33e-04 | 713.3ms/step | 28.0 samples/s +Step 11110/44000 | Loss: 3.1232 | LR: 4.32e-04 | 713.3ms/step | 28.0 samples/s +Step 11120/44000 | Loss: 3.1489 | LR: 4.32e-04 | 712.3ms/step | 28.1 samples/s +Step 11130/44000 | Loss: 3.1558 | LR: 4.32e-04 | 712.7ms/step | 28.1 samples/s +Step 11140/44000 | Loss: 3.1470 | LR: 4.32e-04 | 712.3ms/step | 28.1 samples/s +Step 11150/44000 | Loss: 3.1349 | LR: 4.32e-04 | 714.6ms/step | 28.0 samples/s +Step 11160/44000 | Loss: 3.0895 | LR: 4.32e-04 | 712.9ms/step | 28.1 samples/s +Step 11170/44000 | Loss: 3.1408 | LR: 4.32e-04 | 712.9ms/step | 28.1 samples/s +Step 11180/44000 | Loss: 3.0826 | LR: 4.32e-04 | 726.3ms/step | 27.5 samples/s +Step 11190/44000 | Loss: 3.1218 | LR: 4.31e-04 | 714.2ms/step | 28.0 samples/s +Step 11200/44000 | Loss: 3.0638 | LR: 4.31e-04 | 712.8ms/step | 28.1 samples/s +Step 11210/44000 | Loss: 3.1263 | LR: 4.31e-04 | 712.0ms/step | 28.1 samples/s +Step 11220/44000 | Loss: 3.1561 | LR: 4.31e-04 | 711.4ms/step | 28.1 samples/s +Step 11230/44000 | Loss: 3.0699 | LR: 4.31e-04 | 711.7ms/step | 28.1 samples/s +Step 11240/44000 | Loss: 3.0642 | LR: 4.31e-04 | 711.6ms/step | 28.1 samples/s +Step 11250/44000 | Loss: 3.1464 | LR: 4.31e-04 | 711.4ms/step | 28.1 samples/s +Validation at step 11250: Loss = 3.1071, Perplexity = 22.36 +Step 11260/44000 | Loss: 3.1372 | LR: 4.31e-04 | 943.7ms/step | 21.2 samples/s +Step 11270/44000 | Loss: 3.0799 | LR: 4.30e-04 | 720.0ms/step | 27.8 samples/s +Step 11280/44000 | Loss: 3.1609 | LR: 4.30e-04 | 713.3ms/step | 28.0 samples/s +Step 11290/44000 | Loss: 3.0892 | LR: 4.30e-04 | 713.0ms/step | 28.1 samples/s +Step 11300/44000 | Loss: 3.1486 | LR: 4.30e-04 | 712.7ms/step | 28.1 samples/s +Step 11310/44000 | Loss: 3.0477 | LR: 4.30e-04 | 712.0ms/step | 28.1 samples/s +Step 11320/44000 | Loss: 3.1797 | LR: 4.30e-04 | 712.3ms/step | 28.1 samples/s +Step 11330/44000 | Loss: 3.1185 | LR: 4.30e-04 | 713.0ms/step | 28.1 samples/s +Step 11340/44000 | Loss: 3.1027 | LR: 4.30e-04 | 713.3ms/step | 28.0 samples/s +Step 11350/44000 | Loss: 3.1266 | LR: 4.29e-04 | 729.2ms/step | 27.4 samples/s +Step 11360/44000 | Loss: 3.1176 | LR: 4.29e-04 | 712.5ms/step | 28.1 samples/s +Step 11370/44000 | Loss: 3.1074 | LR: 4.29e-04 | 712.1ms/step | 28.1 samples/s +Step 11380/44000 | Loss: 3.1397 | LR: 4.29e-04 | 712.0ms/step | 28.1 samples/s +Step 11390/44000 | Loss: 3.1546 | LR: 4.29e-04 | 712.3ms/step | 28.1 samples/s +Step 11400/44000 | Loss: 3.1048 | LR: 4.29e-04 | 711.9ms/step | 28.1 samples/s +Step 11410/44000 | Loss: 3.0221 | LR: 4.29e-04 | 713.4ms/step | 28.0 samples/s +Step 11420/44000 | Loss: 3.1390 | LR: 4.29e-04 | 713.2ms/step | 28.0 samples/s +Step 11430/44000 | Loss: 3.1036 | LR: 4.28e-04 | 711.4ms/step | 28.1 samples/s +Step 11440/44000 | Loss: 3.1224 | LR: 4.28e-04 | 726.7ms/step | 27.5 samples/s +Step 11450/44000 | Loss: 3.2127 | LR: 4.28e-04 | 712.5ms/step | 28.1 samples/s +Step 11460/44000 | Loss: 3.0736 | LR: 4.28e-04 | 712.2ms/step | 28.1 samples/s +Step 11470/44000 | Loss: 3.1272 | LR: 4.28e-04 | 711.3ms/step | 28.1 samples/s +Step 11480/44000 | Loss: 3.1200 | LR: 4.28e-04 | 711.5ms/step | 28.1 samples/s +Step 11490/44000 | Loss: 3.0667 | LR: 4.28e-04 | 711.3ms/step | 28.1 samples/s +Step 11500/44000 | Loss: 3.1013 | LR: 4.28e-04 | 711.3ms/step | 28.1 samples/s +Validation at step 11500: Loss = 3.1030, Perplexity = 22.26 +Step 11510/44000 | Loss: 3.1053 | LR: 4.27e-04 | 931.3ms/step | 21.5 samples/s +Step 11520/44000 | Loss: 3.1026 | LR: 4.27e-04 | 711.2ms/step | 28.1 samples/s +Step 11530/44000 | Loss: 3.1598 | LR: 4.27e-04 | 724.5ms/step | 27.6 samples/s +Step 11540/44000 | Loss: 3.1174 | LR: 4.27e-04 | 713.5ms/step | 28.0 samples/s +Step 11550/44000 | Loss: 3.1984 | LR: 4.27e-04 | 716.3ms/step | 27.9 samples/s +Step 11560/44000 | Loss: 3.0423 | LR: 4.27e-04 | 712.4ms/step | 28.1 samples/s +Step 11570/44000 | Loss: 3.1437 | LR: 4.27e-04 | 712.3ms/step | 28.1 samples/s +Step 11580/44000 | Loss: 3.1151 | LR: 4.27e-04 | 713.1ms/step | 28.0 samples/s +Step 11590/44000 | Loss: 3.0783 | LR: 4.26e-04 | 712.5ms/step | 28.1 samples/s +Step 11600/44000 | Loss: 3.1058 | LR: 4.26e-04 | 712.4ms/step | 28.1 samples/s +Step 11610/44000 | Loss: 3.1488 | LR: 4.26e-04 | 712.8ms/step | 28.1 samples/s +Step 11620/44000 | Loss: 3.1331 | LR: 4.26e-04 | 727.9ms/step | 27.5 samples/s +Step 11630/44000 | Loss: 3.1470 | LR: 4.26e-04 | 711.6ms/step | 28.1 samples/s +Step 11640/44000 | Loss: 3.1491 | LR: 4.26e-04 | 711.4ms/step | 28.1 samples/s +Step 11650/44000 | Loss: 3.1471 | LR: 4.26e-04 | 711.2ms/step | 28.1 samples/s +Step 11660/44000 | Loss: 3.1766 | LR: 4.26e-04 | 712.0ms/step | 28.1 samples/s +Step 11670/44000 | Loss: 3.1605 | LR: 4.25e-04 | 712.2ms/step | 28.1 samples/s +Step 11680/44000 | Loss: 3.0777 | LR: 4.25e-04 | 712.6ms/step | 28.1 samples/s +Step 11690/44000 | Loss: 3.1413 | LR: 4.25e-04 | 712.4ms/step | 28.1 samples/s +Step 11700/44000 | Loss: 3.1476 | LR: 4.25e-04 | 722.0ms/step | 27.7 samples/s +Step 11710/44000 | Loss: 3.1220 | LR: 4.25e-04 | 719.8ms/step | 27.8 samples/s +Step 11720/44000 | Loss: 3.0634 | LR: 4.25e-04 | 712.3ms/step | 28.1 samples/s +Step 11730/44000 | Loss: 3.1289 | LR: 4.25e-04 | 712.0ms/step | 28.1 samples/s +Step 11740/44000 | Loss: 3.0660 | LR: 4.24e-04 | 713.2ms/step | 28.0 samples/s +Step 11750/44000 | Loss: 3.2094 | LR: 4.24e-04 | 712.1ms/step | 28.1 samples/s +Validation at step 11750: Loss = 3.0967, Perplexity = 22.13 +Step 11760/44000 | Loss: 3.0595 | LR: 4.24e-04 | 931.6ms/step | 21.5 samples/s +Step 11770/44000 | Loss: 3.0582 | LR: 4.24e-04 | 711.9ms/step | 28.1 samples/s +Step 11780/44000 | Loss: 3.0963 | LR: 4.24e-04 | 712.7ms/step | 28.1 samples/s +Step 11790/44000 | Loss: 3.0855 | LR: 4.24e-04 | 729.6ms/step | 27.4 samples/s +Step 11800/44000 | Loss: 3.1427 | LR: 4.24e-04 | 712.9ms/step | 28.1 samples/s +Step 11810/44000 | Loss: 3.1144 | LR: 4.24e-04 | 712.8ms/step | 28.1 samples/s +Step 11820/44000 | Loss: 3.0453 | LR: 4.23e-04 | 713.0ms/step | 28.0 samples/s +Step 11830/44000 | Loss: 3.1034 | LR: 4.23e-04 | 712.2ms/step | 28.1 samples/s +Step 11840/44000 | Loss: 3.1168 | LR: 4.23e-04 | 712.4ms/step | 28.1 samples/s +Step 11850/44000 | Loss: 3.1406 | LR: 4.23e-04 | 712.4ms/step | 28.1 samples/s +Step 11860/44000 | Loss: 3.1059 | LR: 4.23e-04 | 713.1ms/step | 28.0 samples/s +Step 11870/44000 | Loss: 3.0915 | LR: 4.23e-04 | 713.2ms/step | 28.0 samples/s +Step 11880/44000 | Loss: 3.0813 | LR: 4.23e-04 | 725.8ms/step | 27.6 samples/s +Step 11890/44000 | Loss: 3.1376 | LR: 4.23e-04 | 713.1ms/step | 28.0 samples/s +Step 11900/44000 | Loss: 3.0920 | LR: 4.22e-04 | 712.8ms/step | 28.1 samples/s +Step 11910/44000 | Loss: 3.0692 | LR: 4.22e-04 | 712.4ms/step | 28.1 samples/s +Step 11920/44000 | Loss: 3.0849 | LR: 4.22e-04 | 712.5ms/step | 28.1 samples/s +Step 11930/44000 | Loss: 3.1439 | LR: 4.22e-04 | 713.1ms/step | 28.0 samples/s +Step 11940/44000 | Loss: 3.1090 | LR: 4.22e-04 | 713.1ms/step | 28.0 samples/s +Step 11950/44000 | Loss: 3.1102 | LR: 4.22e-04 | 713.0ms/step | 28.0 samples/s +Step 11960/44000 | Loss: 2.9911 | LR: 4.22e-04 | 713.7ms/step | 28.0 samples/s +Step 11970/44000 | Loss: 3.0961 | LR: 4.21e-04 | 727.8ms/step | 27.5 samples/s +Step 11980/44000 | Loss: 3.1452 | LR: 4.21e-04 | 713.1ms/step | 28.0 samples/s +Step 11990/44000 | Loss: 3.0712 | LR: 4.21e-04 | 711.3ms/step | 28.1 samples/s +Step 12000/44000 | Loss: 3.1197 | LR: 4.21e-04 | 712.5ms/step | 28.1 samples/s +Validation at step 12000: Loss = 3.0921, Perplexity = 22.02 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-12000 +Step 12010/44000 | Loss: 3.1223 | LR: 4.21e-04 | 1337.6ms/step | 15.0 samples/s +Step 12020/44000 | Loss: 3.1350 | LR: 4.21e-04 | 712.9ms/step | 28.1 samples/s +Step 12030/44000 | Loss: 3.1045 | LR: 4.21e-04 | 712.7ms/step | 28.1 samples/s +Step 12040/44000 | Loss: 3.2031 | LR: 4.21e-04 | 712.5ms/step | 28.1 samples/s +Step 12050/44000 | Loss: 3.1065 | LR: 4.20e-04 | 729.3ms/step | 27.4 samples/s +Step 12060/44000 | Loss: 3.1014 | LR: 4.20e-04 | 712.4ms/step | 28.1 samples/s +Step 12070/44000 | Loss: 3.1073 | LR: 4.20e-04 | 712.5ms/step | 28.1 samples/s +Step 12080/44000 | Loss: 3.1517 | LR: 4.20e-04 | 712.7ms/step | 28.1 samples/s +Step 12090/44000 | Loss: 3.0903 | LR: 4.20e-04 | 712.3ms/step | 28.1 samples/s +Step 12100/44000 | Loss: 3.0923 | LR: 4.20e-04 | 712.3ms/step | 28.1 samples/s +Step 12110/44000 | Loss: 3.1589 | LR: 4.20e-04 | 712.3ms/step | 28.1 samples/s +Step 12120/44000 | Loss: 3.1520 | LR: 4.19e-04 | 712.0ms/step | 28.1 samples/s +Step 12130/44000 | Loss: 3.0927 | LR: 4.19e-04 | 711.4ms/step | 28.1 samples/s +Step 12140/44000 | Loss: 3.1439 | LR: 4.19e-04 | 726.6ms/step | 27.5 samples/s +Step 12150/44000 | Loss: 3.1568 | LR: 4.19e-04 | 710.9ms/step | 28.1 samples/s +Step 12160/44000 | Loss: 3.1473 | LR: 4.19e-04 | 712.3ms/step | 28.1 samples/s +Step 12170/44000 | Loss: 3.1308 | LR: 4.19e-04 | 712.7ms/step | 28.1 samples/s +Step 12180/44000 | Loss: 3.2024 | LR: 4.19e-04 | 712.1ms/step | 28.1 samples/s +Step 12190/44000 | Loss: 3.1306 | LR: 4.19e-04 | 712.0ms/step | 28.1 samples/s +Step 12200/44000 | Loss: 3.0255 | LR: 4.18e-04 | 712.2ms/step | 28.1 samples/s +Step 12210/44000 | Loss: 3.1323 | LR: 4.18e-04 | 711.8ms/step | 28.1 samples/s +Step 12220/44000 | Loss: 3.0395 | LR: 4.18e-04 | 712.2ms/step | 28.1 samples/s +Step 12230/44000 | Loss: 3.1104 | LR: 4.18e-04 | 726.9ms/step | 27.5 samples/s +Step 12240/44000 | Loss: 3.1208 | LR: 4.18e-04 | 712.1ms/step | 28.1 samples/s +Step 12250/44000 | Loss: 3.1200 | LR: 4.18e-04 | 711.4ms/step | 28.1 samples/s +Validation at step 12250: Loss = 3.0903, Perplexity = 21.98 +Step 12260/44000 | Loss: 3.0848 | LR: 4.18e-04 | 931.1ms/step | 21.5 samples/s +Step 12270/44000 | Loss: 3.1373 | LR: 4.17e-04 | 712.5ms/step | 28.1 samples/s +Step 12280/44000 | Loss: 3.1075 | LR: 4.17e-04 | 712.2ms/step | 28.1 samples/s +Step 12290/44000 | Loss: 3.0320 | LR: 4.17e-04 | 712.3ms/step | 28.1 samples/s +Step 12300/44000 | Loss: 3.1856 | LR: 4.17e-04 | 711.7ms/step | 28.1 samples/s +Step 12310/44000 | Loss: 3.1254 | LR: 4.17e-04 | 726.7ms/step | 27.5 samples/s +Step 12320/44000 | Loss: 3.1703 | LR: 4.17e-04 | 711.3ms/step | 28.1 samples/s +Step 12330/44000 | Loss: 3.1801 | LR: 4.17e-04 | 712.3ms/step | 28.1 samples/s +Step 12340/44000 | Loss: 3.0719 | LR: 4.17e-04 | 712.1ms/step | 28.1 samples/s +Step 12350/44000 | Loss: 3.0759 | LR: 4.16e-04 | 710.7ms/step | 28.1 samples/s +Step 12360/44000 | Loss: 3.1625 | LR: 4.16e-04 | 711.8ms/step | 28.1 samples/s +Step 12370/44000 | Loss: 3.1702 | LR: 4.16e-04 | 713.0ms/step | 28.1 samples/s +Step 12380/44000 | Loss: 3.1411 | LR: 4.16e-04 | 711.4ms/step | 28.1 samples/s +Step 12390/44000 | Loss: 3.1110 | LR: 4.16e-04 | 711.0ms/step | 28.1 samples/s +Step 12400/44000 | Loss: 3.0991 | LR: 4.16e-04 | 729.1ms/step | 27.4 samples/s +Step 12410/44000 | Loss: 3.0728 | LR: 4.16e-04 | 711.4ms/step | 28.1 samples/s +Step 12420/44000 | Loss: 3.0640 | LR: 4.15e-04 | 711.6ms/step | 28.1 samples/s +Step 12430/44000 | Loss: 3.0504 | LR: 4.15e-04 | 711.5ms/step | 28.1 samples/s +Step 12440/44000 | Loss: 3.1675 | LR: 4.15e-04 | 711.3ms/step | 28.1 samples/s +Step 12450/44000 | Loss: 3.1326 | LR: 4.15e-04 | 711.5ms/step | 28.1 samples/s +Step 12460/44000 | Loss: 3.1347 | LR: 4.15e-04 | 711.4ms/step | 28.1 samples/s +Step 12470/44000 | Loss: 3.0976 | LR: 4.15e-04 | 711.6ms/step | 28.1 samples/s +Step 12480/44000 | Loss: 3.1697 | LR: 4.15e-04 | 711.5ms/step | 28.1 samples/s +Step 12490/44000 | Loss: 3.1051 | LR: 4.14e-04 | 727.9ms/step | 27.5 samples/s +Step 12500/44000 | Loss: 3.1107 | LR: 4.14e-04 | 711.6ms/step | 28.1 samples/s +Validation at step 12500: Loss = 3.0859, Perplexity = 21.89 +Step 12510/44000 | Loss: 3.1082 | LR: 4.14e-04 | 931.0ms/step | 21.5 samples/s +Step 12520/44000 | Loss: 3.1491 | LR: 4.14e-04 | 712.0ms/step | 28.1 samples/s +Step 12530/44000 | Loss: 3.0783 | LR: 4.14e-04 | 711.5ms/step | 28.1 samples/s +Step 12540/44000 | Loss: 3.0966 | LR: 4.14e-04 | 711.7ms/step | 28.1 samples/s +Step 12550/44000 | Loss: 3.0904 | LR: 4.14e-04 | 711.0ms/step | 28.1 samples/s +Step 12560/44000 | Loss: 3.0951 | LR: 4.14e-04 | 713.2ms/step | 28.0 samples/s +Step 12570/44000 | Loss: 3.1382 | LR: 4.13e-04 | 717.7ms/step | 27.9 samples/s +Step 12580/44000 | Loss: 3.1271 | LR: 4.13e-04 | 724.0ms/step | 27.6 samples/s +Step 12590/44000 | Loss: 3.0657 | LR: 4.13e-04 | 713.2ms/step | 28.0 samples/s +Step 12600/44000 | Loss: 3.0915 | LR: 4.13e-04 | 713.0ms/step | 28.1 samples/s +Step 12610/44000 | Loss: 3.1329 | LR: 4.13e-04 | 713.9ms/step | 28.0 samples/s +Step 12620/44000 | Loss: 3.1025 | LR: 4.13e-04 | 712.9ms/step | 28.1 samples/s +Step 12630/44000 | Loss: 3.0559 | LR: 4.13e-04 | 712.9ms/step | 28.1 samples/s +Step 12640/44000 | Loss: 3.0780 | LR: 4.12e-04 | 712.7ms/step | 28.1 samples/s +Step 12650/44000 | Loss: 3.1424 | LR: 4.12e-04 | 712.7ms/step | 28.1 samples/s +Step 12660/44000 | Loss: 3.1010 | LR: 4.12e-04 | 722.0ms/step | 27.7 samples/s +Step 12670/44000 | Loss: 3.0537 | LR: 4.12e-04 | 719.6ms/step | 27.8 samples/s +Step 12680/44000 | Loss: 3.0676 | LR: 4.12e-04 | 713.0ms/step | 28.1 samples/s +Step 12690/44000 | Loss: 3.1342 | LR: 4.12e-04 | 712.6ms/step | 28.1 samples/s +Step 12700/44000 | Loss: 3.0781 | LR: 4.12e-04 | 712.5ms/step | 28.1 samples/s +Step 12710/44000 | Loss: 3.0368 | LR: 4.11e-04 | 712.5ms/step | 28.1 samples/s +Step 12720/44000 | Loss: 3.0875 | LR: 4.11e-04 | 713.0ms/step | 28.1 samples/s +Step 12730/44000 | Loss: 3.1443 | LR: 4.11e-04 | 712.8ms/step | 28.1 samples/s +Step 12740/44000 | Loss: 3.0314 | LR: 4.11e-04 | 712.5ms/step | 28.1 samples/s +Step 12750/44000 | Loss: 3.1082 | LR: 4.11e-04 | 723.1ms/step | 27.7 samples/s +Validation at step 12750: Loss = 3.0837, Perplexity = 21.84 +Step 12760/44000 | Loss: 3.0965 | LR: 4.11e-04 | 937.7ms/step | 21.3 samples/s +Step 12770/44000 | Loss: 3.0910 | LR: 4.11e-04 | 713.2ms/step | 28.0 samples/s +Step 12780/44000 | Loss: 3.1352 | LR: 4.10e-04 | 712.8ms/step | 28.1 samples/s +Step 12790/44000 | Loss: 3.0793 | LR: 4.10e-04 | 712.8ms/step | 28.1 samples/s +Step 12800/44000 | Loss: 3.0513 | LR: 4.10e-04 | 712.7ms/step | 28.1 samples/s +Step 12810/44000 | Loss: 3.0992 | LR: 4.10e-04 | 713.0ms/step | 28.1 samples/s +Step 12820/44000 | Loss: 3.0771 | LR: 4.10e-04 | 713.4ms/step | 28.0 samples/s +Step 12830/44000 | Loss: 3.0697 | LR: 4.10e-04 | 712.5ms/step | 28.1 samples/s +Step 12840/44000 | Loss: 3.0994 | LR: 4.10e-04 | 731.9ms/step | 27.3 samples/s +Step 12850/44000 | Loss: 3.0743 | LR: 4.09e-04 | 713.3ms/step | 28.0 samples/s +Step 12860/44000 | Loss: 3.1106 | LR: 4.09e-04 | 713.0ms/step | 28.1 samples/s +Step 12870/44000 | Loss: 3.0539 | LR: 4.09e-04 | 713.2ms/step | 28.0 samples/s +Step 12880/44000 | Loss: 3.0700 | LR: 4.09e-04 | 712.8ms/step | 28.1 samples/s +Step 12890/44000 | Loss: 3.1470 | LR: 4.09e-04 | 712.4ms/step | 28.1 samples/s +Step 12900/44000 | Loss: 3.2216 | LR: 4.09e-04 | 712.8ms/step | 28.1 samples/s +Step 12910/44000 | Loss: 3.0486 | LR: 4.09e-04 | 712.9ms/step | 28.1 samples/s +Step 12920/44000 | Loss: 3.1611 | LR: 4.09e-04 | 712.3ms/step | 28.1 samples/s +Step 12930/44000 | Loss: 3.0036 | LR: 4.08e-04 | 725.8ms/step | 27.6 samples/s +Step 12940/44000 | Loss: 3.1197 | LR: 4.08e-04 | 712.6ms/step | 28.1 samples/s +Step 12950/44000 | Loss: 3.0702 | LR: 4.08e-04 | 712.7ms/step | 28.1 samples/s +Step 12960/44000 | Loss: 3.0700 | LR: 4.08e-04 | 712.4ms/step | 28.1 samples/s +Step 12970/44000 | Loss: 3.0732 | LR: 4.08e-04 | 712.8ms/step | 28.1 samples/s +Step 12980/44000 | Loss: 3.1279 | LR: 4.08e-04 | 712.8ms/step | 28.1 samples/s +Step 12990/44000 | Loss: 3.1422 | LR: 4.08e-04 | 712.9ms/step | 28.1 samples/s +Step 13000/44000 | Loss: 3.1376 | LR: 4.07e-04 | 712.9ms/step | 28.1 samples/s +Validation at step 13000: Loss = 3.0777, Perplexity = 21.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-13000 +Step 13010/44000 | Loss: 3.1207 | LR: 4.07e-04 | 1279.5ms/step | 15.6 samples/s +Step 13020/44000 | Loss: 3.0710 | LR: 4.07e-04 | 711.8ms/step | 28.1 samples/s +Step 13030/44000 | Loss: 3.0862 | LR: 4.07e-04 | 711.5ms/step | 28.1 samples/s +Step 13040/44000 | Loss: 3.1203 | LR: 4.07e-04 | 711.6ms/step | 28.1 samples/s +Step 13050/44000 | Loss: 3.0814 | LR: 4.07e-04 | 711.3ms/step | 28.1 samples/s +Step 13060/44000 | Loss: 3.1030 | LR: 4.07e-04 | 711.5ms/step | 28.1 samples/s +Step 13070/44000 | Loss: 3.0732 | LR: 4.06e-04 | 712.1ms/step | 28.1 samples/s +Step 13080/44000 | Loss: 3.1604 | LR: 4.06e-04 | 712.3ms/step | 28.1 samples/s +Step 13090/44000 | Loss: 3.0366 | LR: 4.06e-04 | 711.8ms/step | 28.1 samples/s +Step 13100/44000 | Loss: 3.0580 | LR: 4.06e-04 | 728.6ms/step | 27.5 samples/s +Step 13110/44000 | Loss: 3.1290 | LR: 4.06e-04 | 713.7ms/step | 28.0 samples/s +Step 13120/44000 | Loss: 3.1549 | LR: 4.06e-04 | 712.8ms/step | 28.1 samples/s +Step 13130/44000 | Loss: 3.1012 | LR: 4.06e-04 | 713.2ms/step | 28.0 samples/s +Step 13140/44000 | Loss: 3.0820 | LR: 4.05e-04 | 711.9ms/step | 28.1 samples/s +Step 13150/44000 | Loss: 3.0877 | LR: 4.05e-04 | 711.6ms/step | 28.1 samples/s +Step 13160/44000 | Loss: 3.0947 | LR: 4.05e-04 | 710.2ms/step | 28.2 samples/s +Step 13170/44000 | Loss: 3.0978 | LR: 4.05e-04 | 711.5ms/step | 28.1 samples/s +Step 13180/44000 | Loss: 3.0831 | LR: 4.05e-04 | 711.6ms/step | 28.1 samples/s +Step 13190/44000 | Loss: 3.0711 | LR: 4.05e-04 | 724.5ms/step | 27.6 samples/s +Step 13200/44000 | Loss: 3.1319 | LR: 4.05e-04 | 711.9ms/step | 28.1 samples/s +Step 13210/44000 | Loss: 3.1380 | LR: 4.04e-04 | 712.3ms/step | 28.1 samples/s +Step 13220/44000 | Loss: 2.9978 | LR: 4.04e-04 | 711.7ms/step | 28.1 samples/s +Step 13230/44000 | Loss: 3.1223 | LR: 4.04e-04 | 711.4ms/step | 28.1 samples/s +Step 13240/44000 | Loss: 3.0842 | LR: 4.04e-04 | 711.6ms/step | 28.1 samples/s +Step 13250/44000 | Loss: 3.1084 | LR: 4.04e-04 | 714.1ms/step | 28.0 samples/s +Validation at step 13250: Loss = 3.0729, Perplexity = 21.60 +Step 13260/44000 | Loss: 3.1173 | LR: 4.04e-04 | 934.2ms/step | 21.4 samples/s +Step 13270/44000 | Loss: 3.1506 | LR: 4.04e-04 | 730.1ms/step | 27.4 samples/s +Step 13280/44000 | Loss: 3.0973 | LR: 4.03e-04 | 712.8ms/step | 28.1 samples/s +Step 13290/44000 | Loss: 3.1309 | LR: 4.03e-04 | 713.1ms/step | 28.0 samples/s +Step 13300/44000 | Loss: 3.1634 | LR: 4.03e-04 | 712.1ms/step | 28.1 samples/s +Step 13310/44000 | Loss: 3.0352 | LR: 4.03e-04 | 712.2ms/step | 28.1 samples/s +Step 13320/44000 | Loss: 3.0326 | LR: 4.03e-04 | 711.6ms/step | 28.1 samples/s +Step 13330/44000 | Loss: 3.1007 | LR: 4.03e-04 | 711.4ms/step | 28.1 samples/s +Step 13340/44000 | Loss: 3.1337 | LR: 4.03e-04 | 711.6ms/step | 28.1 samples/s +Step 13350/44000 | Loss: 3.0901 | LR: 4.02e-04 | 711.5ms/step | 28.1 samples/s +Step 13360/44000 | Loss: 3.1225 | LR: 4.02e-04 | 726.4ms/step | 27.5 samples/s +Step 13370/44000 | Loss: 3.1269 | LR: 4.02e-04 | 712.2ms/step | 28.1 samples/s +Step 13380/44000 | Loss: 3.1081 | LR: 4.02e-04 | 712.4ms/step | 28.1 samples/s +Step 13390/44000 | Loss: 3.0939 | LR: 4.02e-04 | 713.8ms/step | 28.0 samples/s +Step 13400/44000 | Loss: 3.1207 | LR: 4.02e-04 | 712.2ms/step | 28.1 samples/s +Step 13410/44000 | Loss: 3.0687 | LR: 4.02e-04 | 712.6ms/step | 28.1 samples/s +Step 13420/44000 | Loss: 3.0319 | LR: 4.01e-04 | 712.3ms/step | 28.1 samples/s +Step 13430/44000 | Loss: 3.0669 | LR: 4.01e-04 | 712.9ms/step | 28.1 samples/s +Step 13440/44000 | Loss: 3.0280 | LR: 4.01e-04 | 713.8ms/step | 28.0 samples/s +Step 13450/44000 | Loss: 3.1285 | LR: 4.01e-04 | 725.8ms/step | 27.6 samples/s +Step 13460/44000 | Loss: 3.0849 | LR: 4.01e-04 | 711.3ms/step | 28.1 samples/s +Step 13470/44000 | Loss: 3.1035 | LR: 4.01e-04 | 712.3ms/step | 28.1 samples/s +Step 13480/44000 | Loss: 3.0968 | LR: 4.01e-04 | 712.6ms/step | 28.1 samples/s +Step 13490/44000 | Loss: 3.0828 | LR: 4.00e-04 | 713.3ms/step | 28.0 samples/s +Step 13500/44000 | Loss: 3.0839 | LR: 4.00e-04 | 712.5ms/step | 28.1 samples/s +Validation at step 13500: Loss = 3.0707, Perplexity = 21.56 +Step 13510/44000 | Loss: 3.1673 | LR: 4.00e-04 | 931.3ms/step | 21.5 samples/s +Step 13520/44000 | Loss: 3.1064 | LR: 4.00e-04 | 711.7ms/step | 28.1 samples/s +Step 13530/44000 | Loss: 3.1574 | LR: 4.00e-04 | 716.8ms/step | 27.9 samples/s +Step 13540/44000 | Loss: 3.0852 | LR: 4.00e-04 | 724.8ms/step | 27.6 samples/s +Step 13550/44000 | Loss: 3.0342 | LR: 4.00e-04 | 711.6ms/step | 28.1 samples/s +Step 13560/44000 | Loss: 3.1102 | LR: 3.99e-04 | 711.9ms/step | 28.1 samples/s +Step 13570/44000 | Loss: 3.0943 | LR: 3.99e-04 | 712.5ms/step | 28.1 samples/s +Step 13580/44000 | Loss: 3.0553 | LR: 3.99e-04 | 711.2ms/step | 28.1 samples/s +Step 13590/44000 | Loss: 3.0621 | LR: 3.99e-04 | 711.9ms/step | 28.1 samples/s +Step 13600/44000 | Loss: 3.0818 | LR: 3.99e-04 | 711.2ms/step | 28.1 samples/s +Step 13610/44000 | Loss: 3.0648 | LR: 3.99e-04 | 711.1ms/step | 28.1 samples/s +Step 13620/44000 | Loss: 3.0821 | LR: 3.98e-04 | 723.4ms/step | 27.6 samples/s +Step 13630/44000 | Loss: 3.0849 | LR: 3.98e-04 | 715.1ms/step | 28.0 samples/s +Step 13640/44000 | Loss: 2.9825 | LR: 3.98e-04 | 709.8ms/step | 28.2 samples/s +Step 13650/44000 | Loss: 3.0786 | LR: 3.98e-04 | 710.5ms/step | 28.2 samples/s +Step 13660/44000 | Loss: 3.1187 | LR: 3.98e-04 | 710.5ms/step | 28.1 samples/s +Step 13670/44000 | Loss: 3.1089 | LR: 3.98e-04 | 710.7ms/step | 28.1 samples/s +Step 13680/44000 | Loss: 3.0204 | LR: 3.98e-04 | 710.3ms/step | 28.2 samples/s +Step 13690/44000 | Loss: 3.0293 | LR: 3.97e-04 | 711.3ms/step | 28.1 samples/s +Step 13700/44000 | Loss: 3.1055 | LR: 3.97e-04 | 711.2ms/step | 28.1 samples/s +Step 13710/44000 | Loss: 3.1113 | LR: 3.97e-04 | 722.7ms/step | 27.7 samples/s +Step 13720/44000 | Loss: 3.1118 | LR: 3.97e-04 | 716.1ms/step | 27.9 samples/s +Step 13730/44000 | Loss: 3.1831 | LR: 3.97e-04 | 710.9ms/step | 28.1 samples/s +Step 13740/44000 | Loss: 2.9804 | LR: 3.97e-04 | 711.3ms/step | 28.1 samples/s +Step 13750/44000 | Loss: 3.0904 | LR: 3.97e-04 | 714.1ms/step | 28.0 samples/s +Validation at step 13750: Loss = 3.0645, Perplexity = 21.42 +Step 13760/44000 | Loss: 3.1139 | LR: 3.96e-04 | 931.7ms/step | 21.5 samples/s +Step 13770/44000 | Loss: 3.0656 | LR: 3.96e-04 | 713.9ms/step | 28.0 samples/s +Step 13780/44000 | Loss: 3.1352 | LR: 3.96e-04 | 713.5ms/step | 28.0 samples/s +Step 13790/44000 | Loss: 3.0606 | LR: 3.96e-04 | 712.9ms/step | 28.1 samples/s +Step 13800/44000 | Loss: 3.1113 | LR: 3.96e-04 | 725.8ms/step | 27.6 samples/s +Step 13810/44000 | Loss: 3.1629 | LR: 3.96e-04 | 712.4ms/step | 28.1 samples/s +Step 13820/44000 | Loss: 3.0662 | LR: 3.96e-04 | 712.2ms/step | 28.1 samples/s +Step 13830/44000 | Loss: 3.1810 | LR: 3.95e-04 | 710.1ms/step | 28.2 samples/s +Step 13840/44000 | Loss: 3.0899 | LR: 3.95e-04 | 710.9ms/step | 28.1 samples/s +Step 13850/44000 | Loss: 3.0475 | LR: 3.95e-04 | 711.4ms/step | 28.1 samples/s +Step 13860/44000 | Loss: 3.1128 | LR: 3.95e-04 | 710.7ms/step | 28.1 samples/s +Step 13870/44000 | Loss: 3.0747 | LR: 3.95e-04 | 710.3ms/step | 28.2 samples/s +Step 13880/44000 | Loss: 3.0830 | LR: 3.95e-04 | 711.3ms/step | 28.1 samples/s +Step 13890/44000 | Loss: 3.0127 | LR: 3.95e-04 | 725.8ms/step | 27.6 samples/s +Step 13900/44000 | Loss: 3.1103 | LR: 3.94e-04 | 711.6ms/step | 28.1 samples/s +Step 13910/44000 | Loss: 3.0805 | LR: 3.94e-04 | 711.4ms/step | 28.1 samples/s +Step 13920/44000 | Loss: 3.0542 | LR: 3.94e-04 | 711.5ms/step | 28.1 samples/s +Step 13930/44000 | Loss: 3.0949 | LR: 3.94e-04 | 712.2ms/step | 28.1 samples/s +Step 13940/44000 | Loss: 3.1365 | LR: 3.94e-04 | 712.8ms/step | 28.1 samples/s +Step 13950/44000 | Loss: 3.0455 | LR: 3.94e-04 | 711.0ms/step | 28.1 samples/s +Step 13960/44000 | Loss: 3.0800 | LR: 3.93e-04 | 712.0ms/step | 28.1 samples/s +Step 13970/44000 | Loss: 3.0656 | LR: 3.93e-04 | 712.1ms/step | 28.1 samples/s +Step 13980/44000 | Loss: 3.0489 | LR: 3.93e-04 | 726.2ms/step | 27.5 samples/s +Step 13990/44000 | Loss: 3.1769 | LR: 3.93e-04 | 711.2ms/step | 28.1 samples/s +Step 14000/44000 | Loss: 3.0807 | LR: 3.93e-04 | 713.6ms/step | 28.0 samples/s +Validation at step 14000: Loss = 3.0619, Perplexity = 21.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-14000 +Step 14010/44000 | Loss: 3.1195 | LR: 3.93e-04 | 1256.1ms/step | 15.9 samples/s +Step 14020/44000 | Loss: 3.0816 | LR: 3.93e-04 | 711.7ms/step | 28.1 samples/s +Step 14030/44000 | Loss: 3.0883 | LR: 3.92e-04 | 711.6ms/step | 28.1 samples/s +Step 14040/44000 | Loss: 3.1187 | LR: 3.92e-04 | 712.0ms/step | 28.1 samples/s +Step 14050/44000 | Loss: 3.1721 | LR: 3.92e-04 | 712.0ms/step | 28.1 samples/s +Step 14060/44000 | Loss: 3.1039 | LR: 3.92e-04 | 728.8ms/step | 27.4 samples/s +Step 14070/44000 | Loss: 3.0606 | LR: 3.92e-04 | 711.8ms/step | 28.1 samples/s +Step 14080/44000 | Loss: 3.1199 | LR: 3.92e-04 | 712.3ms/step | 28.1 samples/s +Step 14090/44000 | Loss: 3.1112 | LR: 3.92e-04 | 712.1ms/step | 28.1 samples/s +Step 14100/44000 | Loss: 3.0857 | LR: 3.91e-04 | 711.9ms/step | 28.1 samples/s +Step 14110/44000 | Loss: 3.1189 | LR: 3.91e-04 | 711.7ms/step | 28.1 samples/s +Step 14120/44000 | Loss: 3.0548 | LR: 3.91e-04 | 711.4ms/step | 28.1 samples/s +Step 14130/44000 | Loss: 3.0496 | LR: 3.91e-04 | 712.8ms/step | 28.1 samples/s +Step 14140/44000 | Loss: 3.0625 | LR: 3.91e-04 | 712.2ms/step | 28.1 samples/s +Step 14150/44000 | Loss: 3.0715 | LR: 3.91e-04 | 726.8ms/step | 27.5 samples/s +Step 14160/44000 | Loss: 3.1055 | LR: 3.90e-04 | 713.2ms/step | 28.0 samples/s +Step 14170/44000 | Loss: 3.1090 | LR: 3.90e-04 | 711.5ms/step | 28.1 samples/s +Step 14180/44000 | Loss: 3.0901 | LR: 3.90e-04 | 712.5ms/step | 28.1 samples/s +Step 14190/44000 | Loss: 3.0802 | LR: 3.90e-04 | 712.3ms/step | 28.1 samples/s +Step 14200/44000 | Loss: 3.0999 | LR: 3.90e-04 | 711.7ms/step | 28.1 samples/s +Step 14210/44000 | Loss: 3.0975 | LR: 3.90e-04 | 711.5ms/step | 28.1 samples/s +Step 14220/44000 | Loss: 3.0320 | LR: 3.90e-04 | 711.7ms/step | 28.1 samples/s +Step 14230/44000 | Loss: 3.0916 | LR: 3.89e-04 | 712.7ms/step | 28.1 samples/s +Step 14240/44000 | Loss: 3.1377 | LR: 3.89e-04 | 729.9ms/step | 27.4 samples/s +Step 14250/44000 | Loss: 3.0865 | LR: 3.89e-04 | 711.8ms/step | 28.1 samples/s +Validation at step 14250: Loss = 3.0562, Perplexity = 21.25 +Step 14260/44000 | Loss: 3.0110 | LR: 3.89e-04 | 931.8ms/step | 21.5 samples/s +Step 14270/44000 | Loss: 3.0932 | LR: 3.89e-04 | 714.2ms/step | 28.0 samples/s +Step 14280/44000 | Loss: 3.0509 | LR: 3.89e-04 | 713.0ms/step | 28.0 samples/s +Step 14290/44000 | Loss: 3.0265 | LR: 3.89e-04 | 712.3ms/step | 28.1 samples/s +Step 14300/44000 | Loss: 3.0134 | LR: 3.88e-04 | 712.2ms/step | 28.1 samples/s +Step 14310/44000 | Loss: 3.1369 | LR: 3.88e-04 | 712.3ms/step | 28.1 samples/s +Step 14320/44000 | Loss: 3.1143 | LR: 3.88e-04 | 729.3ms/step | 27.4 samples/s +Step 14330/44000 | Loss: 3.0886 | LR: 3.88e-04 | 711.9ms/step | 28.1 samples/s +Step 14340/44000 | Loss: 3.1201 | LR: 3.88e-04 | 711.7ms/step | 28.1 samples/s +Step 14350/44000 | Loss: 3.0779 | LR: 3.88e-04 | 712.4ms/step | 28.1 samples/s +Step 14360/44000 | Loss: 3.1015 | LR: 3.87e-04 | 711.6ms/step | 28.1 samples/s +Step 14370/44000 | Loss: 3.0596 | LR: 3.87e-04 | 712.9ms/step | 28.1 samples/s +Step 14380/44000 | Loss: 3.0493 | LR: 3.87e-04 | 711.6ms/step | 28.1 samples/s +Step 14390/44000 | Loss: 3.0791 | LR: 3.87e-04 | 711.4ms/step | 28.1 samples/s +Step 14400/44000 | Loss: 3.0688 | LR: 3.87e-04 | 711.4ms/step | 28.1 samples/s +Step 14410/44000 | Loss: 3.1118 | LR: 3.87e-04 | 725.5ms/step | 27.6 samples/s +Step 14420/44000 | Loss: 3.0983 | LR: 3.87e-04 | 711.5ms/step | 28.1 samples/s +Step 14430/44000 | Loss: 3.0409 | LR: 3.86e-04 | 712.2ms/step | 28.1 samples/s +Step 14440/44000 | Loss: 3.0462 | LR: 3.86e-04 | 712.6ms/step | 28.1 samples/s +Step 14450/44000 | Loss: 3.0331 | LR: 3.86e-04 | 714.5ms/step | 28.0 samples/s +Step 14460/44000 | Loss: 3.0413 | LR: 3.86e-04 | 713.4ms/step | 28.0 samples/s +Step 14470/44000 | Loss: 3.0613 | LR: 3.86e-04 | 713.6ms/step | 28.0 samples/s +Step 14480/44000 | Loss: 3.0744 | LR: 3.86e-04 | 712.3ms/step | 28.1 samples/s +Step 14490/44000 | Loss: 3.0457 | LR: 3.85e-04 | 712.1ms/step | 28.1 samples/s +Step 14500/44000 | Loss: 3.0966 | LR: 3.85e-04 | 731.0ms/step | 27.4 samples/s +Validation at step 14500: Loss = 3.0531, Perplexity = 21.18 +Step 14510/44000 | Loss: 3.0036 | LR: 3.85e-04 | 929.6ms/step | 21.5 samples/s +Step 14520/44000 | Loss: 3.0143 | LR: 3.85e-04 | 713.5ms/step | 28.0 samples/s +Step 14530/44000 | Loss: 3.0333 | LR: 3.85e-04 | 712.7ms/step | 28.1 samples/s +Step 14540/44000 | Loss: 3.0430 | LR: 3.85e-04 | 712.5ms/step | 28.1 samples/s +Step 14550/44000 | Loss: 3.0776 | LR: 3.85e-04 | 712.3ms/step | 28.1 samples/s +Step 14560/44000 | Loss: 3.0192 | LR: 3.84e-04 | 711.7ms/step | 28.1 samples/s +Step 14570/44000 | Loss: 3.0568 | LR: 3.84e-04 | 711.2ms/step | 28.1 samples/s +Step 14580/44000 | Loss: 3.1489 | LR: 3.84e-04 | 716.9ms/step | 27.9 samples/s +Step 14590/44000 | Loss: 3.0477 | LR: 3.84e-04 | 723.2ms/step | 27.7 samples/s +Step 14600/44000 | Loss: 3.1562 | LR: 3.84e-04 | 712.0ms/step | 28.1 samples/s +Step 14610/44000 | Loss: 3.0063 | LR: 3.84e-04 | 711.3ms/step | 28.1 samples/s +Step 14620/44000 | Loss: 3.0884 | LR: 3.83e-04 | 710.9ms/step | 28.1 samples/s +Step 14630/44000 | Loss: 3.0237 | LR: 3.83e-04 | 710.7ms/step | 28.1 samples/s +Step 14640/44000 | Loss: 3.0823 | LR: 3.83e-04 | 711.2ms/step | 28.1 samples/s +Step 14650/44000 | Loss: 3.0367 | LR: 3.83e-04 | 711.0ms/step | 28.1 samples/s +Step 14660/44000 | Loss: 3.0386 | LR: 3.83e-04 | 710.9ms/step | 28.1 samples/s +Step 14670/44000 | Loss: 3.0472 | LR: 3.83e-04 | 723.6ms/step | 27.6 samples/s +Step 14680/44000 | Loss: 3.0603 | LR: 3.83e-04 | 716.7ms/step | 27.9 samples/s +Step 14690/44000 | Loss: 3.0656 | LR: 3.82e-04 | 712.6ms/step | 28.1 samples/s +Step 14700/44000 | Loss: 3.0533 | LR: 3.82e-04 | 710.7ms/step | 28.1 samples/s +Step 14710/44000 | Loss: 3.0173 | LR: 3.82e-04 | 710.8ms/step | 28.1 samples/s +Step 14720/44000 | Loss: 3.0725 | LR: 3.82e-04 | 711.4ms/step | 28.1 samples/s +Step 14730/44000 | Loss: 3.0329 | LR: 3.82e-04 | 712.0ms/step | 28.1 samples/s +Step 14740/44000 | Loss: 3.0645 | LR: 3.82e-04 | 710.7ms/step | 28.1 samples/s +Step 14750/44000 | Loss: 3.0475 | LR: 3.81e-04 | 710.7ms/step | 28.1 samples/s +Validation at step 14750: Loss = 3.0486, Perplexity = 21.09 +Step 14760/44000 | Loss: 3.0130 | LR: 3.81e-04 | 943.7ms/step | 21.2 samples/s +Step 14770/44000 | Loss: 3.0891 | LR: 3.81e-04 | 709.6ms/step | 28.2 samples/s +Step 14780/44000 | Loss: 3.1480 | LR: 3.81e-04 | 710.3ms/step | 28.2 samples/s +Step 14790/44000 | Loss: 3.0045 | LR: 3.81e-04 | 710.2ms/step | 28.2 samples/s +Step 14800/44000 | Loss: 3.0584 | LR: 3.81e-04 | 710.8ms/step | 28.1 samples/s +Step 14810/44000 | Loss: 3.0838 | LR: 3.81e-04 | 710.6ms/step | 28.1 samples/s +Step 14820/44000 | Loss: 3.0769 | LR: 3.80e-04 | 711.0ms/step | 28.1 samples/s +Step 14830/44000 | Loss: 3.0441 | LR: 3.80e-04 | 710.9ms/step | 28.1 samples/s +Step 14840/44000 | Loss: 3.0260 | LR: 3.80e-04 | 710.8ms/step | 28.1 samples/s +Step 14850/44000 | Loss: 3.0449 | LR: 3.80e-04 | 728.4ms/step | 27.5 samples/s +Step 14860/44000 | Loss: 3.0578 | LR: 3.80e-04 | 710.7ms/step | 28.1 samples/s +Step 14870/44000 | Loss: 3.0988 | LR: 3.80e-04 | 710.3ms/step | 28.2 samples/s +Step 14880/44000 | Loss: 2.9946 | LR: 3.79e-04 | 710.0ms/step | 28.2 samples/s +Step 14890/44000 | Loss: 3.1078 | LR: 3.79e-04 | 710.6ms/step | 28.1 samples/s +Step 14900/44000 | Loss: 3.0330 | LR: 3.79e-04 | 710.5ms/step | 28.2 samples/s +Step 14910/44000 | Loss: 3.0561 | LR: 3.79e-04 | 710.3ms/step | 28.2 samples/s +Step 14920/44000 | Loss: 3.0146 | LR: 3.79e-04 | 711.3ms/step | 28.1 samples/s +Step 14930/44000 | Loss: 3.0643 | LR: 3.79e-04 | 711.3ms/step | 28.1 samples/s +Step 14940/44000 | Loss: 3.0541 | LR: 3.79e-04 | 728.8ms/step | 27.4 samples/s +Step 14950/44000 | Loss: 3.0799 | LR: 3.78e-04 | 710.4ms/step | 28.2 samples/s +Step 14960/44000 | Loss: 3.1085 | LR: 3.78e-04 | 710.2ms/step | 28.2 samples/s +Step 14970/44000 | Loss: 3.0821 | LR: 3.78e-04 | 710.6ms/step | 28.1 samples/s +Step 14980/44000 | Loss: 3.0008 | LR: 3.78e-04 | 711.1ms/step | 28.1 samples/s +Step 14990/44000 | Loss: 3.0664 | LR: 3.78e-04 | 710.9ms/step | 28.1 samples/s +Step 15000/44000 | Loss: 2.9965 | LR: 3.78e-04 | 710.7ms/step | 28.1 samples/s +Validation at step 15000: Loss = 3.0466, Perplexity = 21.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-15000 +Step 15010/44000 | Loss: 3.0194 | LR: 3.77e-04 | 1304.5ms/step | 15.3 samples/s +Step 15020/44000 | Loss: 2.9829 | LR: 3.77e-04 | 724.4ms/step | 27.6 samples/s +Step 15030/44000 | Loss: 3.0226 | LR: 3.77e-04 | 709.9ms/step | 28.2 samples/s +Step 15040/44000 | Loss: 3.0210 | LR: 3.77e-04 | 709.7ms/step | 28.2 samples/s +Step 15050/44000 | Loss: 3.0616 | LR: 3.77e-04 | 710.6ms/step | 28.1 samples/s +Step 15060/44000 | Loss: 3.0617 | LR: 3.77e-04 | 710.7ms/step | 28.1 samples/s +Step 15070/44000 | Loss: 3.0925 | LR: 3.77e-04 | 710.1ms/step | 28.2 samples/s +Step 15080/44000 | Loss: 3.0561 | LR: 3.76e-04 | 710.3ms/step | 28.2 samples/s +Step 15090/44000 | Loss: 3.0491 | LR: 3.76e-04 | 710.7ms/step | 28.1 samples/s +Step 15100/44000 | Loss: 3.1385 | LR: 3.76e-04 | 711.2ms/step | 28.1 samples/s +Step 15110/44000 | Loss: 3.0744 | LR: 3.76e-04 | 728.8ms/step | 27.4 samples/s +Step 15120/44000 | Loss: 3.1429 | LR: 3.76e-04 | 711.4ms/step | 28.1 samples/s +Step 15130/44000 | Loss: 3.0435 | LR: 3.76e-04 | 710.5ms/step | 28.1 samples/s +Step 15140/44000 | Loss: 3.0621 | LR: 3.75e-04 | 711.5ms/step | 28.1 samples/s +Step 15150/44000 | Loss: 3.1012 | LR: 3.75e-04 | 711.0ms/step | 28.1 samples/s +Step 15160/44000 | Loss: 3.1363 | LR: 3.75e-04 | 711.6ms/step | 28.1 samples/s +Step 15170/44000 | Loss: 3.1117 | LR: 3.75e-04 | 711.3ms/step | 28.1 samples/s +Step 15180/44000 | Loss: 3.0686 | LR: 3.75e-04 | 710.6ms/step | 28.1 samples/s +Step 15190/44000 | Loss: 3.0662 | LR: 3.75e-04 | 711.0ms/step | 28.1 samples/s +Step 15200/44000 | Loss: 3.0764 | LR: 3.74e-04 | 727.3ms/step | 27.5 samples/s +Step 15210/44000 | Loss: 3.0826 | LR: 3.74e-04 | 711.3ms/step | 28.1 samples/s +Step 15220/44000 | Loss: 3.0344 | LR: 3.74e-04 | 710.8ms/step | 28.1 samples/s +Step 15230/44000 | Loss: 2.9869 | LR: 3.74e-04 | 710.9ms/step | 28.1 samples/s +Step 15240/44000 | Loss: 3.0289 | LR: 3.74e-04 | 711.1ms/step | 28.1 samples/s +Step 15250/44000 | Loss: 3.0984 | LR: 3.74e-04 | 711.1ms/step | 28.1 samples/s +Validation at step 15250: Loss = 3.0426, Perplexity = 20.96 +Step 15260/44000 | Loss: 3.0050 | LR: 3.74e-04 | 929.8ms/step | 21.5 samples/s +Step 15270/44000 | Loss: 3.0780 | LR: 3.73e-04 | 712.3ms/step | 28.1 samples/s +Step 15280/44000 | Loss: 3.0577 | LR: 3.73e-04 | 720.1ms/step | 27.8 samples/s +Step 15290/44000 | Loss: 3.0537 | LR: 3.73e-04 | 715.2ms/step | 28.0 samples/s +Step 15300/44000 | Loss: 3.0779 | LR: 3.73e-04 | 711.3ms/step | 28.1 samples/s +Step 15310/44000 | Loss: 3.0407 | LR: 3.73e-04 | 710.1ms/step | 28.2 samples/s +Step 15320/44000 | Loss: 3.0850 | LR: 3.73e-04 | 710.2ms/step | 28.2 samples/s +Step 15330/44000 | Loss: 3.0498 | LR: 3.72e-04 | 709.9ms/step | 28.2 samples/s +Step 15340/44000 | Loss: 3.0745 | LR: 3.72e-04 | 710.7ms/step | 28.1 samples/s +Step 15350/44000 | Loss: 3.0712 | LR: 3.72e-04 | 712.0ms/step | 28.1 samples/s +Step 15360/44000 | Loss: 3.0261 | LR: 3.72e-04 | 711.5ms/step | 28.1 samples/s +Step 15370/44000 | Loss: 2.9743 | LR: 3.72e-04 | 722.7ms/step | 27.7 samples/s +Step 15380/44000 | Loss: 3.0776 | LR: 3.72e-04 | 716.1ms/step | 27.9 samples/s +Step 15390/44000 | Loss: 3.0552 | LR: 3.71e-04 | 711.8ms/step | 28.1 samples/s +Step 15400/44000 | Loss: 3.0906 | LR: 3.71e-04 | 711.6ms/step | 28.1 samples/s +Step 15410/44000 | Loss: 3.1528 | LR: 3.71e-04 | 711.9ms/step | 28.1 samples/s +Step 15420/44000 | Loss: 3.1058 | LR: 3.71e-04 | 711.8ms/step | 28.1 samples/s +Step 15430/44000 | Loss: 3.1285 | LR: 3.71e-04 | 711.7ms/step | 28.1 samples/s +Step 15440/44000 | Loss: 3.0819 | LR: 3.71e-04 | 711.2ms/step | 28.1 samples/s +Step 15450/44000 | Loss: 2.9920 | LR: 3.71e-04 | 710.7ms/step | 28.1 samples/s +Step 15460/44000 | Loss: 3.0261 | LR: 3.70e-04 | 724.3ms/step | 27.6 samples/s +Step 15470/44000 | Loss: 3.1087 | LR: 3.70e-04 | 711.6ms/step | 28.1 samples/s +Step 15480/44000 | Loss: 3.1284 | LR: 3.70e-04 | 711.4ms/step | 28.1 samples/s +Step 15490/44000 | Loss: 3.0764 | LR: 3.70e-04 | 710.3ms/step | 28.2 samples/s +Step 15500/44000 | Loss: 3.1047 | LR: 3.70e-04 | 710.3ms/step | 28.2 samples/s +Validation at step 15500: Loss = 3.0388, Perplexity = 20.88 +Step 15510/44000 | Loss: 3.0721 | LR: 3.70e-04 | 928.7ms/step | 21.5 samples/s +Step 15520/44000 | Loss: 3.0730 | LR: 3.69e-04 | 710.7ms/step | 28.1 samples/s +Step 15530/44000 | Loss: 3.0369 | LR: 3.69e-04 | 712.0ms/step | 28.1 samples/s +Step 15540/44000 | Loss: 3.0095 | LR: 3.69e-04 | 710.2ms/step | 28.2 samples/s +Step 15550/44000 | Loss: 2.9762 | LR: 3.69e-04 | 724.6ms/step | 27.6 samples/s +Step 15560/44000 | Loss: 3.0143 | LR: 3.69e-04 | 710.4ms/step | 28.2 samples/s +Step 15570/44000 | Loss: 3.0472 | LR: 3.69e-04 | 710.5ms/step | 28.2 samples/s +Step 15580/44000 | Loss: 2.9937 | LR: 3.68e-04 | 710.4ms/step | 28.2 samples/s +Step 15590/44000 | Loss: 3.0936 | LR: 3.68e-04 | 710.7ms/step | 28.1 samples/s +Step 15600/44000 | Loss: 3.0716 | LR: 3.68e-04 | 709.9ms/step | 28.2 samples/s +Step 15610/44000 | Loss: 3.0589 | LR: 3.68e-04 | 711.1ms/step | 28.1 samples/s +Step 15620/44000 | Loss: 3.0663 | LR: 3.68e-04 | 711.1ms/step | 28.1 samples/s +Step 15630/44000 | Loss: 3.0588 | LR: 3.68e-04 | 711.2ms/step | 28.1 samples/s +Step 15640/44000 | Loss: 3.0344 | LR: 3.67e-04 | 727.0ms/step | 27.5 samples/s +Step 15650/44000 | Loss: 3.0521 | LR: 3.67e-04 | 710.5ms/step | 28.1 samples/s +Step 15660/44000 | Loss: 3.0391 | LR: 3.67e-04 | 711.3ms/step | 28.1 samples/s +Step 15670/44000 | Loss: 3.0429 | LR: 3.67e-04 | 710.5ms/step | 28.2 samples/s +Step 15680/44000 | Loss: 3.0664 | LR: 3.67e-04 | 711.8ms/step | 28.1 samples/s +Step 15690/44000 | Loss: 3.0378 | LR: 3.67e-04 | 711.3ms/step | 28.1 samples/s +Step 15700/44000 | Loss: 3.0622 | LR: 3.67e-04 | 710.9ms/step | 28.1 samples/s +Step 15710/44000 | Loss: 3.0730 | LR: 3.66e-04 | 709.7ms/step | 28.2 samples/s +Step 15720/44000 | Loss: 3.0407 | LR: 3.66e-04 | 715.7ms/step | 27.9 samples/s +Step 15730/44000 | Loss: 3.0753 | LR: 3.66e-04 | 721.6ms/step | 27.7 samples/s +Step 15740/44000 | Loss: 3.0021 | LR: 3.66e-04 | 710.0ms/step | 28.2 samples/s +Step 15750/44000 | Loss: 3.0222 | LR: 3.66e-04 | 709.9ms/step | 28.2 samples/s +Validation at step 15750: Loss = 3.0371, Perplexity = 20.84 +Step 15760/44000 | Loss: 3.0727 | LR: 3.66e-04 | 927.7ms/step | 21.6 samples/s +Step 15770/44000 | Loss: 3.0085 | LR: 3.65e-04 | 710.6ms/step | 28.1 samples/s +Step 15780/44000 | Loss: 3.0182 | LR: 3.65e-04 | 710.0ms/step | 28.2 samples/s +Step 15790/44000 | Loss: 3.0821 | LR: 3.65e-04 | 710.0ms/step | 28.2 samples/s +Step 15800/44000 | Loss: 2.9710 | LR: 3.65e-04 | 710.5ms/step | 28.2 samples/s +Step 15810/44000 | Loss: 3.0556 | LR: 3.65e-04 | 727.7ms/step | 27.5 samples/s +Step 15820/44000 | Loss: 3.0735 | LR: 3.65e-04 | 710.4ms/step | 28.2 samples/s +Step 15830/44000 | Loss: 3.0085 | LR: 3.64e-04 | 710.4ms/step | 28.2 samples/s +Step 15840/44000 | Loss: 3.0867 | LR: 3.64e-04 | 710.2ms/step | 28.2 samples/s +Step 15850/44000 | Loss: 2.9879 | LR: 3.64e-04 | 710.4ms/step | 28.2 samples/s +Step 15860/44000 | Loss: 3.1092 | LR: 3.64e-04 | 710.6ms/step | 28.1 samples/s +Step 15870/44000 | Loss: 3.0031 | LR: 3.64e-04 | 710.5ms/step | 28.1 samples/s +Step 15880/44000 | Loss: 3.0801 | LR: 3.64e-04 | 711.6ms/step | 28.1 samples/s +Step 15890/44000 | Loss: 3.0443 | LR: 3.63e-04 | 710.8ms/step | 28.1 samples/s +Step 15900/44000 | Loss: 3.0686 | LR: 3.63e-04 | 729.4ms/step | 27.4 samples/s +Step 15910/44000 | Loss: 3.1094 | LR: 3.63e-04 | 710.4ms/step | 28.2 samples/s +Step 15920/44000 | Loss: 3.0222 | LR: 3.63e-04 | 710.9ms/step | 28.1 samples/s +Step 15930/44000 | Loss: 3.0618 | LR: 3.63e-04 | 710.5ms/step | 28.1 samples/s +Step 15940/44000 | Loss: 3.0310 | LR: 3.63e-04 | 711.0ms/step | 28.1 samples/s +Step 15950/44000 | Loss: 3.0682 | LR: 3.62e-04 | 710.1ms/step | 28.2 samples/s +Step 15960/44000 | Loss: 3.0379 | LR: 3.62e-04 | 710.8ms/step | 28.1 samples/s +Step 15970/44000 | Loss: 3.1318 | LR: 3.62e-04 | 710.3ms/step | 28.2 samples/s +Step 15980/44000 | Loss: 3.0743 | LR: 3.62e-04 | 710.8ms/step | 28.1 samples/s +Step 15990/44000 | Loss: 3.0125 | LR: 3.62e-04 | 728.3ms/step | 27.5 samples/s +Step 16000/44000 | Loss: 3.0038 | LR: 3.62e-04 | 710.2ms/step | 28.2 samples/s +Validation at step 16000: Loss = 3.0328, Perplexity = 20.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-16000 +Step 16010/44000 | Loss: 3.0326 | LR: 3.62e-04 | 1267.4ms/step | 15.8 samples/s +Step 16020/44000 | Loss: 3.0280 | LR: 3.61e-04 | 710.4ms/step | 28.2 samples/s +Step 16030/44000 | Loss: 3.0635 | LR: 3.61e-04 | 710.6ms/step | 28.1 samples/s +Step 16040/44000 | Loss: 3.0889 | LR: 3.61e-04 | 710.4ms/step | 28.2 samples/s +Step 16050/44000 | Loss: 3.0388 | LR: 3.61e-04 | 710.7ms/step | 28.1 samples/s +Step 16060/44000 | Loss: 3.0695 | LR: 3.61e-04 | 710.6ms/step | 28.1 samples/s +Step 16070/44000 | Loss: 3.0475 | LR: 3.61e-04 | 723.5ms/step | 27.6 samples/s +Step 16080/44000 | Loss: 3.0316 | LR: 3.60e-04 | 710.3ms/step | 28.2 samples/s +Step 16090/44000 | Loss: 2.9778 | LR: 3.60e-04 | 711.1ms/step | 28.1 samples/s +Step 16100/44000 | Loss: 3.0576 | LR: 3.60e-04 | 711.1ms/step | 28.1 samples/s +Step 16110/44000 | Loss: 2.9865 | LR: 3.60e-04 | 710.4ms/step | 28.2 samples/s +Step 16120/44000 | Loss: 3.0741 | LR: 3.60e-04 | 710.4ms/step | 28.2 samples/s +Step 16130/44000 | Loss: 2.9879 | LR: 3.60e-04 | 709.8ms/step | 28.2 samples/s +Step 16140/44000 | Loss: 3.0567 | LR: 3.59e-04 | 711.1ms/step | 28.1 samples/s +Step 16150/44000 | Loss: 3.0864 | LR: 3.59e-04 | 712.8ms/step | 28.1 samples/s +Step 16160/44000 | Loss: 3.0800 | LR: 3.59e-04 | 724.5ms/step | 27.6 samples/s +Step 16170/44000 | Loss: 3.0658 | LR: 3.59e-04 | 710.9ms/step | 28.1 samples/s +Step 16180/44000 | Loss: 3.0928 | LR: 3.59e-04 | 711.2ms/step | 28.1 samples/s +Step 16190/44000 | Loss: 3.0624 | LR: 3.59e-04 | 711.2ms/step | 28.1 samples/s +Step 16200/44000 | Loss: 3.0573 | LR: 3.58e-04 | 710.9ms/step | 28.1 samples/s +Step 16210/44000 | Loss: 3.0803 | LR: 3.58e-04 | 710.3ms/step | 28.2 samples/s +Step 16220/44000 | Loss: 3.0608 | LR: 3.58e-04 | 710.7ms/step | 28.1 samples/s +Step 16230/44000 | Loss: 3.0405 | LR: 3.58e-04 | 711.0ms/step | 28.1 samples/s +Step 16240/44000 | Loss: 3.0341 | LR: 3.58e-04 | 710.4ms/step | 28.2 samples/s +Step 16250/44000 | Loss: 3.0686 | LR: 3.58e-04 | 724.7ms/step | 27.6 samples/s +Validation at step 16250: Loss = 3.0306, Perplexity = 20.71 +Step 16260/44000 | Loss: 3.0148 | LR: 3.57e-04 | 929.4ms/step | 21.5 samples/s +Step 16270/44000 | Loss: 2.9951 | LR: 3.57e-04 | 710.7ms/step | 28.1 samples/s +Step 16280/44000 | Loss: 3.0465 | LR: 3.57e-04 | 712.0ms/step | 28.1 samples/s +Step 16290/44000 | Loss: 3.1219 | LR: 3.57e-04 | 711.9ms/step | 28.1 samples/s +Step 16300/44000 | Loss: 3.0515 | LR: 3.57e-04 | 711.7ms/step | 28.1 samples/s +Step 16310/44000 | Loss: 3.0047 | LR: 3.57e-04 | 711.4ms/step | 28.1 samples/s +Step 16320/44000 | Loss: 3.0355 | LR: 3.56e-04 | 710.9ms/step | 28.1 samples/s +Step 16330/44000 | Loss: 3.0766 | LR: 3.56e-04 | 719.8ms/step | 27.8 samples/s +Step 16340/44000 | Loss: 3.0026 | LR: 3.56e-04 | 717.4ms/step | 27.9 samples/s +Step 16350/44000 | Loss: 3.0422 | LR: 3.56e-04 | 711.8ms/step | 28.1 samples/s +Step 16360/44000 | Loss: 3.0116 | LR: 3.56e-04 | 710.9ms/step | 28.1 samples/s +Step 16370/44000 | Loss: 3.1033 | LR: 3.56e-04 | 710.9ms/step | 28.1 samples/s +Step 16380/44000 | Loss: 2.9964 | LR: 3.55e-04 | 711.8ms/step | 28.1 samples/s +Step 16390/44000 | Loss: 3.0055 | LR: 3.55e-04 | 713.0ms/step | 28.0 samples/s +Step 16400/44000 | Loss: 3.0615 | LR: 3.55e-04 | 712.5ms/step | 28.1 samples/s +Step 16410/44000 | Loss: 3.0403 | LR: 3.55e-04 | 713.1ms/step | 28.0 samples/s +Step 16420/44000 | Loss: 3.0296 | LR: 3.55e-04 | 725.2ms/step | 27.6 samples/s +Step 16430/44000 | Loss: 3.0733 | LR: 3.55e-04 | 718.4ms/step | 27.8 samples/s +Step 16440/44000 | Loss: 2.9823 | LR: 3.54e-04 | 712.8ms/step | 28.1 samples/s +Step 16450/44000 | Loss: 3.0109 | LR: 3.54e-04 | 712.7ms/step | 28.1 samples/s +Step 16460/44000 | Loss: 3.0805 | LR: 3.54e-04 | 712.4ms/step | 28.1 samples/s +Step 16470/44000 | Loss: 3.0947 | LR: 3.54e-04 | 712.5ms/step | 28.1 samples/s +Step 16480/44000 | Loss: 3.0414 | LR: 3.54e-04 | 712.0ms/step | 28.1 samples/s +Step 16490/44000 | Loss: 3.0153 | LR: 3.54e-04 | 711.6ms/step | 28.1 samples/s +Step 16500/44000 | Loss: 3.1404 | LR: 3.53e-04 | 711.4ms/step | 28.1 samples/s +Validation at step 16500: Loss = 3.0250, Perplexity = 20.59 +Step 16510/44000 | Loss: 3.1050 | LR: 3.53e-04 | 945.5ms/step | 21.2 samples/s +Step 16520/44000 | Loss: 3.0984 | LR: 3.53e-04 | 710.3ms/step | 28.2 samples/s +Step 16530/44000 | Loss: 2.9929 | LR: 3.53e-04 | 710.9ms/step | 28.1 samples/s +Step 16540/44000 | Loss: 3.0430 | LR: 3.53e-04 | 712.3ms/step | 28.1 samples/s +Step 16550/44000 | Loss: 3.0354 | LR: 3.53e-04 | 710.6ms/step | 28.1 samples/s +Step 16560/44000 | Loss: 2.9988 | LR: 3.52e-04 | 711.9ms/step | 28.1 samples/s +Step 16570/44000 | Loss: 3.0452 | LR: 3.52e-04 | 711.6ms/step | 28.1 samples/s +Step 16580/44000 | Loss: 3.1135 | LR: 3.52e-04 | 711.5ms/step | 28.1 samples/s +Step 16590/44000 | Loss: 3.0436 | LR: 3.52e-04 | 711.8ms/step | 28.1 samples/s +Step 16600/44000 | Loss: 3.0582 | LR: 3.52e-04 | 726.1ms/step | 27.5 samples/s +Step 16610/44000 | Loss: 3.0792 | LR: 3.52e-04 | 710.9ms/step | 28.1 samples/s +Step 16620/44000 | Loss: 3.0474 | LR: 3.51e-04 | 710.4ms/step | 28.2 samples/s +Step 16630/44000 | Loss: 3.1174 | LR: 3.51e-04 | 711.0ms/step | 28.1 samples/s +Step 16640/44000 | Loss: 3.0200 | LR: 3.51e-04 | 710.3ms/step | 28.2 samples/s +Step 16650/44000 | Loss: 3.1082 | LR: 3.51e-04 | 710.9ms/step | 28.1 samples/s +Step 16660/44000 | Loss: 3.0844 | LR: 3.51e-04 | 710.8ms/step | 28.1 samples/s +Step 16670/44000 | Loss: 3.0140 | LR: 3.51e-04 | 711.8ms/step | 28.1 samples/s +Step 16680/44000 | Loss: 3.0159 | LR: 3.50e-04 | 711.5ms/step | 28.1 samples/s +Step 16690/44000 | Loss: 2.9976 | LR: 3.50e-04 | 726.0ms/step | 27.5 samples/s +Step 16700/44000 | Loss: 2.9927 | LR: 3.50e-04 | 711.4ms/step | 28.1 samples/s +Step 16710/44000 | Loss: 2.9949 | LR: 3.50e-04 | 711.4ms/step | 28.1 samples/s +Step 16720/44000 | Loss: 2.9984 | LR: 3.50e-04 | 711.5ms/step | 28.1 samples/s +Step 16730/44000 | Loss: 3.0817 | LR: 3.50e-04 | 712.8ms/step | 28.1 samples/s +Step 16740/44000 | Loss: 3.0359 | LR: 3.49e-04 | 711.4ms/step | 28.1 samples/s +Step 16750/44000 | Loss: 3.0291 | LR: 3.49e-04 | 711.3ms/step | 28.1 samples/s +Validation at step 16750: Loss = 3.0213, Perplexity = 20.52 +Step 16760/44000 | Loss: 3.0177 | LR: 3.49e-04 | 929.3ms/step | 21.5 samples/s +Step 16770/44000 | Loss: 3.0487 | LR: 3.49e-04 | 720.7ms/step | 27.7 samples/s +Step 16780/44000 | Loss: 3.0553 | LR: 3.49e-04 | 716.5ms/step | 27.9 samples/s +Step 16790/44000 | Loss: 3.1212 | LR: 3.49e-04 | 712.6ms/step | 28.1 samples/s +Step 16800/44000 | Loss: 3.0462 | LR: 3.48e-04 | 713.2ms/step | 28.0 samples/s +Step 16810/44000 | Loss: 3.0441 | LR: 3.48e-04 | 713.6ms/step | 28.0 samples/s +Step 16820/44000 | Loss: 3.0487 | LR: 3.48e-04 | 712.6ms/step | 28.1 samples/s +Step 16830/44000 | Loss: 3.0747 | LR: 3.48e-04 | 712.6ms/step | 28.1 samples/s +Step 16840/44000 | Loss: 2.9715 | LR: 3.48e-04 | 712.5ms/step | 28.1 samples/s +Step 16850/44000 | Loss: 3.0187 | LR: 3.48e-04 | 711.4ms/step | 28.1 samples/s +Step 16860/44000 | Loss: 3.0535 | LR: 3.47e-04 | 725.1ms/step | 27.6 samples/s +Step 16870/44000 | Loss: 3.0518 | LR: 3.47e-04 | 715.0ms/step | 28.0 samples/s +Step 16880/44000 | Loss: 3.1024 | LR: 3.47e-04 | 711.7ms/step | 28.1 samples/s +Step 16890/44000 | Loss: 3.0323 | LR: 3.47e-04 | 711.0ms/step | 28.1 samples/s +Step 16900/44000 | Loss: 3.0267 | LR: 3.47e-04 | 711.1ms/step | 28.1 samples/s +Step 16910/44000 | Loss: 3.0419 | LR: 3.47e-04 | 710.4ms/step | 28.2 samples/s +Step 16920/44000 | Loss: 3.0256 | LR: 3.46e-04 | 710.9ms/step | 28.1 samples/s +Step 16930/44000 | Loss: 3.0582 | LR: 3.46e-04 | 710.9ms/step | 28.1 samples/s +Step 16940/44000 | Loss: 3.0880 | LR: 3.46e-04 | 711.6ms/step | 28.1 samples/s +Step 16950/44000 | Loss: 3.0127 | LR: 3.46e-04 | 725.7ms/step | 27.6 samples/s +Step 16960/44000 | Loss: 3.0281 | LR: 3.46e-04 | 712.8ms/step | 28.1 samples/s +Step 16970/44000 | Loss: 2.9525 | LR: 3.46e-04 | 712.5ms/step | 28.1 samples/s +Step 16980/44000 | Loss: 3.0273 | LR: 3.45e-04 | 712.4ms/step | 28.1 samples/s +Step 16990/44000 | Loss: 3.0692 | LR: 3.45e-04 | 711.8ms/step | 28.1 samples/s +Step 17000/44000 | Loss: 3.0309 | LR: 3.45e-04 | 712.5ms/step | 28.1 samples/s +Validation at step 17000: Loss = 3.0175, Perplexity = 20.44 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-17000 +Step 17010/44000 | Loss: 3.0726 | LR: 3.45e-04 | 1331.5ms/step | 15.0 samples/s +Step 17020/44000 | Loss: 3.0958 | LR: 3.45e-04 | 712.0ms/step | 28.1 samples/s +Step 17030/44000 | Loss: 3.0358 | LR: 3.45e-04 | 727.9ms/step | 27.5 samples/s +Step 17040/44000 | Loss: 3.0266 | LR: 3.44e-04 | 712.2ms/step | 28.1 samples/s +Step 17050/44000 | Loss: 3.0176 | LR: 3.44e-04 | 710.8ms/step | 28.1 samples/s +Step 17060/44000 | Loss: 3.0062 | LR: 3.44e-04 | 710.5ms/step | 28.1 samples/s +Step 17070/44000 | Loss: 3.0481 | LR: 3.44e-04 | 710.4ms/step | 28.2 samples/s +Step 17080/44000 | Loss: 3.0721 | LR: 3.44e-04 | 709.6ms/step | 28.2 samples/s +Step 17090/44000 | Loss: 2.9669 | LR: 3.44e-04 | 710.6ms/step | 28.1 samples/s +Step 17100/44000 | Loss: 3.1408 | LR: 3.43e-04 | 710.5ms/step | 28.1 samples/s +Step 17110/44000 | Loss: 3.0882 | LR: 3.43e-04 | 710.5ms/step | 28.1 samples/s +Step 17120/44000 | Loss: 3.0412 | LR: 3.43e-04 | 725.4ms/step | 27.6 samples/s +Step 17130/44000 | Loss: 3.0781 | LR: 3.43e-04 | 710.0ms/step | 28.2 samples/s +Step 17140/44000 | Loss: 3.0303 | LR: 3.43e-04 | 710.3ms/step | 28.2 samples/s +Step 17150/44000 | Loss: 3.0170 | LR: 3.43e-04 | 710.6ms/step | 28.1 samples/s +Step 17160/44000 | Loss: 3.0635 | LR: 3.42e-04 | 712.6ms/step | 28.1 samples/s +Step 17170/44000 | Loss: 2.9878 | LR: 3.42e-04 | 710.7ms/step | 28.1 samples/s +Step 17180/44000 | Loss: 3.0286 | LR: 3.42e-04 | 710.4ms/step | 28.2 samples/s +Step 17190/44000 | Loss: 3.0523 | LR: 3.42e-04 | 711.0ms/step | 28.1 samples/s +Step 17200/44000 | Loss: 3.0733 | LR: 3.42e-04 | 710.5ms/step | 28.1 samples/s +Step 17210/44000 | Loss: 3.1257 | LR: 3.42e-04 | 729.4ms/step | 27.4 samples/s +Step 17220/44000 | Loss: 3.0164 | LR: 3.41e-04 | 710.3ms/step | 28.2 samples/s +Step 17230/44000 | Loss: 3.0673 | LR: 3.41e-04 | 711.2ms/step | 28.1 samples/s +Step 17240/44000 | Loss: 3.0351 | LR: 3.41e-04 | 710.7ms/step | 28.1 samples/s +Step 17250/44000 | Loss: 2.9742 | LR: 3.41e-04 | 710.7ms/step | 28.1 samples/s +Validation at step 17250: Loss = 3.0151, Perplexity = 20.39 +Step 17260/44000 | Loss: 3.1039 | LR: 3.41e-04 | 930.5ms/step | 21.5 samples/s +Step 17270/44000 | Loss: 3.0041 | LR: 3.41e-04 | 712.1ms/step | 28.1 samples/s +Step 17280/44000 | Loss: 3.0611 | LR: 3.40e-04 | 710.7ms/step | 28.1 samples/s +Step 17290/44000 | Loss: 3.0268 | LR: 3.40e-04 | 716.9ms/step | 27.9 samples/s +Step 17300/44000 | Loss: 2.9829 | LR: 3.40e-04 | 723.0ms/step | 27.7 samples/s +Step 17310/44000 | Loss: 3.0418 | LR: 3.40e-04 | 711.4ms/step | 28.1 samples/s +Step 17320/44000 | Loss: 2.9854 | LR: 3.40e-04 | 711.2ms/step | 28.1 samples/s +Step 17330/44000 | Loss: 2.9803 | LR: 3.40e-04 | 710.8ms/step | 28.1 samples/s +Step 17340/44000 | Loss: 3.0703 | LR: 3.39e-04 | 710.5ms/step | 28.1 samples/s +Step 17350/44000 | Loss: 3.0538 | LR: 3.39e-04 | 710.4ms/step | 28.2 samples/s +Step 17360/44000 | Loss: 3.0566 | LR: 3.39e-04 | 710.5ms/step | 28.1 samples/s +Step 17370/44000 | Loss: 3.0805 | LR: 3.39e-04 | 710.5ms/step | 28.1 samples/s +Step 17380/44000 | Loss: 3.0325 | LR: 3.39e-04 | 722.0ms/step | 27.7 samples/s +Step 17390/44000 | Loss: 3.1087 | LR: 3.39e-04 | 714.3ms/step | 28.0 samples/s +Step 17400/44000 | Loss: 3.0085 | LR: 3.38e-04 | 711.2ms/step | 28.1 samples/s +Step 17410/44000 | Loss: 3.0473 | LR: 3.38e-04 | 710.6ms/step | 28.1 samples/s +Step 17420/44000 | Loss: 2.9889 | LR: 3.38e-04 | 710.9ms/step | 28.1 samples/s +Step 17430/44000 | Loss: 3.1392 | LR: 3.38e-04 | 710.8ms/step | 28.1 samples/s +Step 17440/44000 | Loss: 3.0263 | LR: 3.38e-04 | 711.0ms/step | 28.1 samples/s +Step 17450/44000 | Loss: 3.0576 | LR: 3.38e-04 | 710.8ms/step | 28.1 samples/s +Step 17460/44000 | Loss: 3.0189 | LR: 3.37e-04 | 711.5ms/step | 28.1 samples/s +Step 17470/44000 | Loss: 3.0403 | LR: 3.37e-04 | 722.3ms/step | 27.7 samples/s +Step 17480/44000 | Loss: 3.0095 | LR: 3.37e-04 | 715.2ms/step | 28.0 samples/s +Step 17490/44000 | Loss: 3.0162 | LR: 3.37e-04 | 711.2ms/step | 28.1 samples/s +Step 17500/44000 | Loss: 3.0456 | LR: 3.37e-04 | 711.0ms/step | 28.1 samples/s +Validation at step 17500: Loss = 3.0140, Perplexity = 20.37 +Step 17510/44000 | Loss: 3.0004 | LR: 3.37e-04 | 930.3ms/step | 21.5 samples/s +Step 17520/44000 | Loss: 3.0798 | LR: 3.36e-04 | 712.1ms/step | 28.1 samples/s +Step 17530/44000 | Loss: 3.0136 | LR: 3.36e-04 | 711.2ms/step | 28.1 samples/s +Step 17540/44000 | Loss: 3.0852 | LR: 3.36e-04 | 712.0ms/step | 28.1 samples/s +Step 17550/44000 | Loss: 3.0187 | LR: 3.36e-04 | 712.2ms/step | 28.1 samples/s +Step 17560/44000 | Loss: 3.0778 | LR: 3.36e-04 | 726.1ms/step | 27.5 samples/s +Step 17570/44000 | Loss: 3.0006 | LR: 3.35e-04 | 710.5ms/step | 28.1 samples/s +Step 17580/44000 | Loss: 3.0116 | LR: 3.35e-04 | 711.1ms/step | 28.1 samples/s +Step 17590/44000 | Loss: 2.9745 | LR: 3.35e-04 | 711.2ms/step | 28.1 samples/s +Step 17600/44000 | Loss: 3.0982 | LR: 3.35e-04 | 710.9ms/step | 28.1 samples/s +Step 17610/44000 | Loss: 3.0262 | LR: 3.35e-04 | 711.6ms/step | 28.1 samples/s +Step 17620/44000 | Loss: 3.0529 | LR: 3.35e-04 | 712.8ms/step | 28.1 samples/s +Step 17630/44000 | Loss: 3.0116 | LR: 3.34e-04 | 712.8ms/step | 28.1 samples/s +Step 17640/44000 | Loss: 3.0591 | LR: 3.34e-04 | 712.6ms/step | 28.1 samples/s +Step 17650/44000 | Loss: 3.0083 | LR: 3.34e-04 | 724.5ms/step | 27.6 samples/s +Step 17660/44000 | Loss: 3.0232 | LR: 3.34e-04 | 712.5ms/step | 28.1 samples/s +Step 17670/44000 | Loss: 2.9632 | LR: 3.34e-04 | 711.8ms/step | 28.1 samples/s +Step 17680/44000 | Loss: 3.0232 | LR: 3.34e-04 | 712.2ms/step | 28.1 samples/s +Step 17690/44000 | Loss: 3.0856 | LR: 3.33e-04 | 712.0ms/step | 28.1 samples/s +Step 17700/44000 | Loss: 3.0122 | LR: 3.33e-04 | 713.2ms/step | 28.0 samples/s +Step 17710/44000 | Loss: 3.0453 | LR: 3.33e-04 | 712.2ms/step | 28.1 samples/s +Step 17720/44000 | Loss: 3.0025 | LR: 3.33e-04 | 712.0ms/step | 28.1 samples/s +Step 17730/44000 | Loss: 3.0238 | LR: 3.33e-04 | 711.2ms/step | 28.1 samples/s +Step 17740/44000 | Loss: 3.0364 | LR: 3.33e-04 | 725.5ms/step | 27.6 samples/s +Step 17750/44000 | Loss: 3.0274 | LR: 3.32e-04 | 711.3ms/step | 28.1 samples/s +Validation at step 17750: Loss = 3.0099, Perplexity = 20.29 +Step 17760/44000 | Loss: 3.0203 | LR: 3.32e-04 | 930.9ms/step | 21.5 samples/s +Step 17770/44000 | Loss: 3.0877 | LR: 3.32e-04 | 712.9ms/step | 28.1 samples/s +Step 17780/44000 | Loss: 2.9560 | LR: 3.32e-04 | 712.3ms/step | 28.1 samples/s +Step 17790/44000 | Loss: 3.0227 | LR: 3.32e-04 | 711.1ms/step | 28.1 samples/s +Step 17800/44000 | Loss: 3.0455 | LR: 3.32e-04 | 712.5ms/step | 28.1 samples/s +Step 17810/44000 | Loss: 3.0672 | LR: 3.31e-04 | 712.1ms/step | 28.1 samples/s +Step 17820/44000 | Loss: 3.0619 | LR: 3.31e-04 | 723.4ms/step | 27.6 samples/s +Step 17830/44000 | Loss: 3.0572 | LR: 3.31e-04 | 719.5ms/step | 27.8 samples/s +Step 17840/44000 | Loss: 2.9851 | LR: 3.31e-04 | 712.3ms/step | 28.1 samples/s +Step 17850/44000 | Loss: 3.0095 | LR: 3.31e-04 | 712.7ms/step | 28.1 samples/s +Step 17860/44000 | Loss: 2.9859 | LR: 3.31e-04 | 712.7ms/step | 28.1 samples/s +Step 17870/44000 | Loss: 3.0025 | LR: 3.30e-04 | 712.1ms/step | 28.1 samples/s +Step 17880/44000 | Loss: 3.0314 | LR: 3.30e-04 | 712.2ms/step | 28.1 samples/s +Step 17890/44000 | Loss: 3.0290 | LR: 3.30e-04 | 711.6ms/step | 28.1 samples/s +Step 17900/44000 | Loss: 3.0164 | LR: 3.30e-04 | 712.0ms/step | 28.1 samples/s +Step 17910/44000 | Loss: 3.0495 | LR: 3.30e-04 | 728.1ms/step | 27.5 samples/s +Step 17920/44000 | Loss: 3.0554 | LR: 3.29e-04 | 710.8ms/step | 28.1 samples/s +Step 17930/44000 | Loss: 3.0722 | LR: 3.29e-04 | 711.4ms/step | 28.1 samples/s +Step 17940/44000 | Loss: 3.0369 | LR: 3.29e-04 | 710.3ms/step | 28.2 samples/s +Step 17950/44000 | Loss: 3.0237 | LR: 3.29e-04 | 710.9ms/step | 28.1 samples/s +Step 17960/44000 | Loss: 3.0555 | LR: 3.29e-04 | 711.2ms/step | 28.1 samples/s +Step 17970/44000 | Loss: 3.0254 | LR: 3.29e-04 | 711.3ms/step | 28.1 samples/s +Step 17980/44000 | Loss: 3.0635 | LR: 3.28e-04 | 711.1ms/step | 28.1 samples/s +Step 17990/44000 | Loss: 3.0541 | LR: 3.28e-04 | 711.6ms/step | 28.1 samples/s +Step 18000/44000 | Loss: 2.9864 | LR: 3.28e-04 | 727.3ms/step | 27.5 samples/s +Validation at step 18000: Loss = 3.0074, Perplexity = 20.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-18000 +Step 18010/44000 | Loss: 3.0385 | LR: 3.28e-04 | 1304.9ms/step | 15.3 samples/s +Step 18020/44000 | Loss: 3.0281 | LR: 3.28e-04 | 711.9ms/step | 28.1 samples/s +Step 18030/44000 | Loss: 3.0761 | LR: 3.28e-04 | 712.7ms/step | 28.1 samples/s +Step 18040/44000 | Loss: 3.0093 | LR: 3.27e-04 | 712.5ms/step | 28.1 samples/s +Step 18050/44000 | Loss: 2.9845 | LR: 3.27e-04 | 712.2ms/step | 28.1 samples/s +Step 18060/44000 | Loss: 3.0724 | LR: 3.27e-04 | 711.7ms/step | 28.1 samples/s +Step 18070/44000 | Loss: 2.9908 | LR: 3.27e-04 | 712.1ms/step | 28.1 samples/s +Step 18080/44000 | Loss: 3.0491 | LR: 3.27e-04 | 729.5ms/step | 27.4 samples/s +Step 18090/44000 | Loss: 3.0859 | LR: 3.27e-04 | 713.7ms/step | 28.0 samples/s +Step 18100/44000 | Loss: 3.0057 | LR: 3.26e-04 | 712.9ms/step | 28.1 samples/s +Step 18110/44000 | Loss: 3.0658 | LR: 3.26e-04 | 712.3ms/step | 28.1 samples/s +Step 18120/44000 | Loss: 3.0705 | LR: 3.26e-04 | 711.9ms/step | 28.1 samples/s +Step 18130/44000 | Loss: 3.0314 | LR: 3.26e-04 | 712.9ms/step | 28.1 samples/s +Step 18140/44000 | Loss: 3.0806 | LR: 3.26e-04 | 713.6ms/step | 28.0 samples/s +Step 18150/44000 | Loss: 3.0297 | LR: 3.26e-04 | 712.7ms/step | 28.1 samples/s +Step 18160/44000 | Loss: 3.0642 | LR: 3.25e-04 | 712.3ms/step | 28.1 samples/s +Step 18170/44000 | Loss: 3.0185 | LR: 3.25e-04 | 725.4ms/step | 27.6 samples/s +Step 18180/44000 | Loss: 3.0785 | LR: 3.25e-04 | 712.9ms/step | 28.1 samples/s +Step 18190/44000 | Loss: 2.9732 | LR: 3.25e-04 | 712.4ms/step | 28.1 samples/s +Step 18200/44000 | Loss: 3.0128 | LR: 3.25e-04 | 711.9ms/step | 28.1 samples/s +Step 18210/44000 | Loss: 3.0575 | LR: 3.24e-04 | 711.8ms/step | 28.1 samples/s +Step 18220/44000 | Loss: 3.0404 | LR: 3.24e-04 | 712.4ms/step | 28.1 samples/s +Step 18230/44000 | Loss: 3.0017 | LR: 3.24e-04 | 712.0ms/step | 28.1 samples/s +Step 18240/44000 | Loss: 2.9622 | LR: 3.24e-04 | 711.9ms/step | 28.1 samples/s +Step 18250/44000 | Loss: 2.9837 | LR: 3.24e-04 | 712.1ms/step | 28.1 samples/s +Validation at step 18250: Loss = 3.0041, Perplexity = 20.17 +Step 18260/44000 | Loss: 3.0144 | LR: 3.24e-04 | 948.7ms/step | 21.1 samples/s +Step 18270/44000 | Loss: 3.0409 | LR: 3.23e-04 | 716.3ms/step | 27.9 samples/s +Step 18280/44000 | Loss: 3.0210 | LR: 3.23e-04 | 712.3ms/step | 28.1 samples/s +Step 18290/44000 | Loss: 3.0707 | LR: 3.23e-04 | 712.0ms/step | 28.1 samples/s +Step 18300/44000 | Loss: 3.0252 | LR: 3.23e-04 | 711.8ms/step | 28.1 samples/s +Step 18310/44000 | Loss: 2.9604 | LR: 3.23e-04 | 712.3ms/step | 28.1 samples/s +Step 18320/44000 | Loss: 3.0697 | LR: 3.23e-04 | 712.9ms/step | 28.1 samples/s +Step 18330/44000 | Loss: 2.9464 | LR: 3.22e-04 | 712.5ms/step | 28.1 samples/s +Step 18340/44000 | Loss: 3.0174 | LR: 3.22e-04 | 721.7ms/step | 27.7 samples/s +Step 18350/44000 | Loss: 3.0126 | LR: 3.22e-04 | 718.5ms/step | 27.8 samples/s +Step 18360/44000 | Loss: 2.9123 | LR: 3.22e-04 | 712.4ms/step | 28.1 samples/s +Step 18370/44000 | Loss: 3.0097 | LR: 3.22e-04 | 712.5ms/step | 28.1 samples/s +Step 18380/44000 | Loss: 3.0408 | LR: 3.22e-04 | 713.4ms/step | 28.0 samples/s +Step 18390/44000 | Loss: 3.0201 | LR: 3.21e-04 | 713.6ms/step | 28.0 samples/s +Step 18400/44000 | Loss: 3.0276 | LR: 3.21e-04 | 713.8ms/step | 28.0 samples/s +Step 18410/44000 | Loss: 3.0593 | LR: 3.21e-04 | 713.9ms/step | 28.0 samples/s +Step 18420/44000 | Loss: 3.0484 | LR: 3.21e-04 | 711.8ms/step | 28.1 samples/s +Step 18430/44000 | Loss: 3.0592 | LR: 3.21e-04 | 721.0ms/step | 27.7 samples/s +Step 18440/44000 | Loss: 3.0022 | LR: 3.20e-04 | 716.3ms/step | 27.9 samples/s +Step 18450/44000 | Loss: 2.9904 | LR: 3.20e-04 | 712.8ms/step | 28.1 samples/s +Step 18460/44000 | Loss: 3.0538 | LR: 3.20e-04 | 713.3ms/step | 28.0 samples/s +Step 18470/44000 | Loss: 2.9873 | LR: 3.20e-04 | 714.6ms/step | 28.0 samples/s +Step 18480/44000 | Loss: 3.0723 | LR: 3.20e-04 | 715.9ms/step | 27.9 samples/s +Step 18490/44000 | Loss: 3.0280 | LR: 3.20e-04 | 714.7ms/step | 28.0 samples/s +Step 18500/44000 | Loss: 3.0084 | LR: 3.19e-04 | 715.1ms/step | 28.0 samples/s +Validation at step 18500: Loss = 3.0003, Perplexity = 20.09 +Step 18510/44000 | Loss: 3.0428 | LR: 3.19e-04 | 933.4ms/step | 21.4 samples/s +Step 18520/44000 | Loss: 3.0569 | LR: 3.19e-04 | 728.3ms/step | 27.5 samples/s +Step 18530/44000 | Loss: 2.9695 | LR: 3.19e-04 | 716.6ms/step | 27.9 samples/s +Step 18540/44000 | Loss: 3.0741 | LR: 3.19e-04 | 715.3ms/step | 28.0 samples/s +Step 18550/44000 | Loss: 3.0597 | LR: 3.19e-04 | 716.1ms/step | 27.9 samples/s +Step 18560/44000 | Loss: 3.1192 | LR: 3.18e-04 | 716.7ms/step | 27.9 samples/s +Step 18570/44000 | Loss: 3.0361 | LR: 3.18e-04 | 715.4ms/step | 28.0 samples/s +Step 18580/44000 | Loss: 2.9820 | LR: 3.18e-04 | 713.7ms/step | 28.0 samples/s +Step 18590/44000 | Loss: 3.0387 | LR: 3.18e-04 | 712.3ms/step | 28.1 samples/s +Step 18600/44000 | Loss: 2.9675 | LR: 3.18e-04 | 713.1ms/step | 28.0 samples/s +Step 18610/44000 | Loss: 3.0124 | LR: 3.18e-04 | 727.9ms/step | 27.5 samples/s +Step 18620/44000 | Loss: 3.0675 | LR: 3.17e-04 | 712.8ms/step | 28.1 samples/s +Step 18630/44000 | Loss: 2.9078 | LR: 3.17e-04 | 713.4ms/step | 28.0 samples/s +Step 18640/44000 | Loss: 2.9842 | LR: 3.17e-04 | 710.4ms/step | 28.2 samples/s +Step 18650/44000 | Loss: 3.0399 | LR: 3.17e-04 | 711.9ms/step | 28.1 samples/s +Step 18660/44000 | Loss: 3.0441 | LR: 3.17e-04 | 711.6ms/step | 28.1 samples/s +Step 18670/44000 | Loss: 2.9799 | LR: 3.16e-04 | 711.5ms/step | 28.1 samples/s +Step 18680/44000 | Loss: 3.0088 | LR: 3.16e-04 | 712.7ms/step | 28.1 samples/s +Step 18690/44000 | Loss: 3.0413 | LR: 3.16e-04 | 713.1ms/step | 28.0 samples/s +Step 18700/44000 | Loss: 2.9504 | LR: 3.16e-04 | 726.1ms/step | 27.5 samples/s +Step 18710/44000 | Loss: 3.0722 | LR: 3.16e-04 | 713.4ms/step | 28.0 samples/s +Step 18720/44000 | Loss: 2.9917 | LR: 3.16e-04 | 712.0ms/step | 28.1 samples/s +Step 18730/44000 | Loss: 3.0217 | LR: 3.15e-04 | 712.2ms/step | 28.1 samples/s +Step 18740/44000 | Loss: 2.9815 | LR: 3.15e-04 | 712.7ms/step | 28.1 samples/s +Step 18750/44000 | Loss: 3.0966 | LR: 3.15e-04 | 714.3ms/step | 28.0 samples/s +Validation at step 18750: Loss = 2.9986, Perplexity = 20.06 +Step 18760/44000 | Loss: 3.0347 | LR: 3.15e-04 | 932.2ms/step | 21.5 samples/s +Step 18770/44000 | Loss: 3.0060 | LR: 3.15e-04 | 711.5ms/step | 28.1 samples/s +Step 18780/44000 | Loss: 3.0438 | LR: 3.15e-04 | 721.3ms/step | 27.7 samples/s +Step 18790/44000 | Loss: 3.0371 | LR: 3.14e-04 | 714.7ms/step | 28.0 samples/s +Step 18800/44000 | Loss: 3.0462 | LR: 3.14e-04 | 710.8ms/step | 28.1 samples/s +Step 18810/44000 | Loss: 2.9954 | LR: 3.14e-04 | 711.0ms/step | 28.1 samples/s +Step 18820/44000 | Loss: 3.0399 | LR: 3.14e-04 | 710.4ms/step | 28.2 samples/s +Step 18830/44000 | Loss: 3.0177 | LR: 3.14e-04 | 710.6ms/step | 28.1 samples/s +Step 18840/44000 | Loss: 3.0755 | LR: 3.13e-04 | 710.5ms/step | 28.1 samples/s +Step 18850/44000 | Loss: 3.0123 | LR: 3.13e-04 | 710.2ms/step | 28.2 samples/s +Step 18860/44000 | Loss: 3.0376 | LR: 3.13e-04 | 710.4ms/step | 28.2 samples/s +Step 18870/44000 | Loss: 3.0607 | LR: 3.13e-04 | 728.4ms/step | 27.5 samples/s +Step 18880/44000 | Loss: 2.9630 | LR: 3.13e-04 | 710.6ms/step | 28.1 samples/s +Step 18890/44000 | Loss: 2.8916 | LR: 3.13e-04 | 710.0ms/step | 28.2 samples/s +Step 18900/44000 | Loss: 2.9671 | LR: 3.12e-04 | 710.1ms/step | 28.2 samples/s +Step 18910/44000 | Loss: 2.9533 | LR: 3.12e-04 | 710.3ms/step | 28.2 samples/s +Step 18920/44000 | Loss: 3.0474 | LR: 3.12e-04 | 710.0ms/step | 28.2 samples/s +Step 18930/44000 | Loss: 3.0441 | LR: 3.12e-04 | 709.7ms/step | 28.2 samples/s +Step 18940/44000 | Loss: 2.9887 | LR: 3.12e-04 | 710.4ms/step | 28.2 samples/s +Step 18950/44000 | Loss: 2.9958 | LR: 3.12e-04 | 710.6ms/step | 28.1 samples/s +Step 18960/44000 | Loss: 2.9705 | LR: 3.11e-04 | 729.1ms/step | 27.4 samples/s +Step 18970/44000 | Loss: 3.0390 | LR: 3.11e-04 | 710.4ms/step | 28.2 samples/s +Step 18980/44000 | Loss: 3.0457 | LR: 3.11e-04 | 710.0ms/step | 28.2 samples/s +Step 18990/44000 | Loss: 3.0391 | LR: 3.11e-04 | 709.9ms/step | 28.2 samples/s +Step 19000/44000 | Loss: 3.0322 | LR: 3.11e-04 | 710.2ms/step | 28.2 samples/s +Validation at step 19000: Loss = 2.9956, Perplexity = 20.00 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-19000 +Step 19010/44000 | Loss: 2.9950 | LR: 3.11e-04 | 1313.2ms/step | 15.2 samples/s +Step 19020/44000 | Loss: 3.0048 | LR: 3.10e-04 | 709.5ms/step | 28.2 samples/s +Step 19030/44000 | Loss: 3.0219 | LR: 3.10e-04 | 709.9ms/step | 28.2 samples/s +Step 19040/44000 | Loss: 3.0383 | LR: 3.10e-04 | 726.8ms/step | 27.5 samples/s +Step 19050/44000 | Loss: 3.0137 | LR: 3.10e-04 | 710.2ms/step | 28.2 samples/s +Step 19060/44000 | Loss: 3.0263 | LR: 3.10e-04 | 709.8ms/step | 28.2 samples/s +Step 19070/44000 | Loss: 3.0908 | LR: 3.09e-04 | 712.9ms/step | 28.1 samples/s +Step 19080/44000 | Loss: 2.9400 | LR: 3.09e-04 | 713.3ms/step | 28.0 samples/s +Step 19090/44000 | Loss: 3.0071 | LR: 3.09e-04 | 711.4ms/step | 28.1 samples/s +Step 19100/44000 | Loss: 2.9467 | LR: 3.09e-04 | 710.9ms/step | 28.1 samples/s +Step 19110/44000 | Loss: 3.0500 | LR: 3.09e-04 | 711.9ms/step | 28.1 samples/s +Step 19120/44000 | Loss: 3.0448 | LR: 3.09e-04 | 710.2ms/step | 28.2 samples/s +Step 19130/44000 | Loss: 3.0675 | LR: 3.08e-04 | 729.3ms/step | 27.4 samples/s +Step 19140/44000 | Loss: 3.0200 | LR: 3.08e-04 | 710.4ms/step | 28.2 samples/s +Step 19150/44000 | Loss: 3.0488 | LR: 3.08e-04 | 711.5ms/step | 28.1 samples/s +Step 19160/44000 | Loss: 2.9791 | LR: 3.08e-04 | 709.8ms/step | 28.2 samples/s +Step 19170/44000 | Loss: 3.0183 | LR: 3.08e-04 | 709.5ms/step | 28.2 samples/s +Step 19180/44000 | Loss: 3.0043 | LR: 3.08e-04 | 710.1ms/step | 28.2 samples/s +Step 19190/44000 | Loss: 2.9818 | LR: 3.07e-04 | 712.0ms/step | 28.1 samples/s +Step 19200/44000 | Loss: 3.0325 | LR: 3.07e-04 | 711.3ms/step | 28.1 samples/s +Step 19210/44000 | Loss: 2.9731 | LR: 3.07e-04 | 709.9ms/step | 28.2 samples/s +Step 19220/44000 | Loss: 3.0708 | LR: 3.07e-04 | 728.9ms/step | 27.4 samples/s +Step 19230/44000 | Loss: 2.9729 | LR: 3.07e-04 | 711.2ms/step | 28.1 samples/s +Step 19240/44000 | Loss: 2.9832 | LR: 3.06e-04 | 711.0ms/step | 28.1 samples/s +Step 19250/44000 | Loss: 3.0282 | LR: 3.06e-04 | 710.6ms/step | 28.1 samples/s +Validation at step 19250: Loss = 2.9939, Perplexity = 19.96 +Step 19260/44000 | Loss: 3.0112 | LR: 3.06e-04 | 928.0ms/step | 21.6 samples/s +Step 19270/44000 | Loss: 2.9495 | LR: 3.06e-04 | 711.2ms/step | 28.1 samples/s +Step 19280/44000 | Loss: 2.8881 | LR: 3.06e-04 | 710.8ms/step | 28.1 samples/s +Step 19290/44000 | Loss: 2.9974 | LR: 3.06e-04 | 710.7ms/step | 28.1 samples/s +Step 19300/44000 | Loss: 3.0298 | LR: 3.05e-04 | 718.1ms/step | 27.8 samples/s +Step 19310/44000 | Loss: 3.0257 | LR: 3.05e-04 | 719.9ms/step | 27.8 samples/s +Step 19320/44000 | Loss: 3.0360 | LR: 3.05e-04 | 711.4ms/step | 28.1 samples/s +Step 19330/44000 | Loss: 3.0331 | LR: 3.05e-04 | 712.1ms/step | 28.1 samples/s +Step 19340/44000 | Loss: 2.9888 | LR: 3.05e-04 | 711.9ms/step | 28.1 samples/s +Step 19350/44000 | Loss: 3.0030 | LR: 3.04e-04 | 712.0ms/step | 28.1 samples/s +Step 19360/44000 | Loss: 3.0202 | LR: 3.04e-04 | 711.9ms/step | 28.1 samples/s +Step 19370/44000 | Loss: 3.0040 | LR: 3.04e-04 | 710.7ms/step | 28.1 samples/s +Step 19380/44000 | Loss: 3.0444 | LR: 3.04e-04 | 711.1ms/step | 28.1 samples/s +Step 19390/44000 | Loss: 2.9418 | LR: 3.04e-04 | 721.3ms/step | 27.7 samples/s +Step 19400/44000 | Loss: 3.0430 | LR: 3.04e-04 | 715.1ms/step | 28.0 samples/s +Step 19410/44000 | Loss: 2.9563 | LR: 3.03e-04 | 711.3ms/step | 28.1 samples/s +Step 19420/44000 | Loss: 3.0172 | LR: 3.03e-04 | 710.6ms/step | 28.1 samples/s +Step 19430/44000 | Loss: 2.9910 | LR: 3.03e-04 | 710.8ms/step | 28.1 samples/s +Step 19440/44000 | Loss: 3.1029 | LR: 3.03e-04 | 709.2ms/step | 28.2 samples/s +Step 19450/44000 | Loss: 3.0113 | LR: 3.03e-04 | 709.3ms/step | 28.2 samples/s +Step 19460/44000 | Loss: 2.9535 | LR: 3.03e-04 | 709.2ms/step | 28.2 samples/s +Step 19470/44000 | Loss: 2.9672 | LR: 3.02e-04 | 709.6ms/step | 28.2 samples/s +Step 19480/44000 | Loss: 3.0727 | LR: 3.02e-04 | 720.5ms/step | 27.8 samples/s +Step 19490/44000 | Loss: 3.0243 | LR: 3.02e-04 | 714.4ms/step | 28.0 samples/s +Step 19500/44000 | Loss: 2.9796 | LR: 3.02e-04 | 709.9ms/step | 28.2 samples/s +Validation at step 19500: Loss = 2.9918, Perplexity = 19.92 +Step 19510/44000 | Loss: 3.0675 | LR: 3.02e-04 | 928.7ms/step | 21.5 samples/s +Step 19520/44000 | Loss: 3.0458 | LR: 3.01e-04 | 710.4ms/step | 28.2 samples/s +Step 19530/44000 | Loss: 2.9911 | LR: 3.01e-04 | 710.6ms/step | 28.1 samples/s +Step 19540/44000 | Loss: 3.0481 | LR: 3.01e-04 | 709.7ms/step | 28.2 samples/s +Step 19550/44000 | Loss: 2.9875 | LR: 3.01e-04 | 709.9ms/step | 28.2 samples/s +Step 19560/44000 | Loss: 3.0308 | LR: 3.01e-04 | 709.9ms/step | 28.2 samples/s +Step 19570/44000 | Loss: 3.0403 | LR: 3.01e-04 | 727.7ms/step | 27.5 samples/s +Step 19580/44000 | Loss: 3.0409 | LR: 3.00e-04 | 710.9ms/step | 28.1 samples/s +Step 19590/44000 | Loss: 2.9984 | LR: 3.00e-04 | 710.1ms/step | 28.2 samples/s +Step 19600/44000 | Loss: 2.9756 | LR: 3.00e-04 | 710.0ms/step | 28.2 samples/s +Step 19610/44000 | Loss: 3.0187 | LR: 3.00e-04 | 710.1ms/step | 28.2 samples/s +Step 19620/44000 | Loss: 3.0214 | LR: 3.00e-04 | 709.3ms/step | 28.2 samples/s +Step 19630/44000 | Loss: 3.0421 | LR: 3.00e-04 | 709.7ms/step | 28.2 samples/s +Step 19640/44000 | Loss: 3.0474 | LR: 2.99e-04 | 711.0ms/step | 28.1 samples/s +Step 19650/44000 | Loss: 3.0182 | LR: 2.99e-04 | 710.2ms/step | 28.2 samples/s +Step 19660/44000 | Loss: 3.0324 | LR: 2.99e-04 | 727.0ms/step | 27.5 samples/s +Step 19670/44000 | Loss: 2.9361 | LR: 2.99e-04 | 709.7ms/step | 28.2 samples/s +Step 19680/44000 | Loss: 3.0099 | LR: 2.99e-04 | 709.3ms/step | 28.2 samples/s +Step 19690/44000 | Loss: 2.9742 | LR: 2.98e-04 | 709.5ms/step | 28.2 samples/s +Step 19700/44000 | Loss: 2.9774 | LR: 2.98e-04 | 709.5ms/step | 28.2 samples/s +Step 19710/44000 | Loss: 2.9387 | LR: 2.98e-04 | 710.0ms/step | 28.2 samples/s +Step 19720/44000 | Loss: 2.9995 | LR: 2.98e-04 | 710.9ms/step | 28.1 samples/s +Step 19730/44000 | Loss: 3.0379 | LR: 2.98e-04 | 709.9ms/step | 28.2 samples/s +Step 19740/44000 | Loss: 2.9800 | LR: 2.98e-04 | 710.8ms/step | 28.1 samples/s +Step 19750/44000 | Loss: 2.9833 | LR: 2.97e-04 | 723.4ms/step | 27.6 samples/s +Validation at step 19750: Loss = 2.9906, Perplexity = 19.90 +Step 19760/44000 | Loss: 3.0804 | LR: 2.97e-04 | 928.5ms/step | 21.5 samples/s +Step 19770/44000 | Loss: 2.9339 | LR: 2.97e-04 | 710.6ms/step | 28.1 samples/s +Step 19780/44000 | Loss: 2.9702 | LR: 2.97e-04 | 710.1ms/step | 28.2 samples/s +Step 19790/44000 | Loss: 3.0036 | LR: 2.97e-04 | 709.7ms/step | 28.2 samples/s +Step 19800/44000 | Loss: 2.9331 | LR: 2.96e-04 | 710.8ms/step | 28.1 samples/s +Step 19810/44000 | Loss: 2.9882 | LR: 2.96e-04 | 710.2ms/step | 28.2 samples/s +Step 19820/44000 | Loss: 3.0293 | LR: 2.96e-04 | 710.3ms/step | 28.2 samples/s +Step 19830/44000 | Loss: 2.9386 | LR: 2.96e-04 | 721.5ms/step | 27.7 samples/s +Step 19840/44000 | Loss: 2.9654 | LR: 2.96e-04 | 715.1ms/step | 28.0 samples/s +Step 19850/44000 | Loss: 3.0062 | LR: 2.96e-04 | 710.7ms/step | 28.1 samples/s +Step 19860/44000 | Loss: 3.0765 | LR: 2.95e-04 | 711.4ms/step | 28.1 samples/s +Step 19870/44000 | Loss: 2.9995 | LR: 2.95e-04 | 710.3ms/step | 28.2 samples/s +Step 19880/44000 | Loss: 3.0648 | LR: 2.95e-04 | 710.4ms/step | 28.2 samples/s +Step 19890/44000 | Loss: 3.0343 | LR: 2.95e-04 | 710.4ms/step | 28.2 samples/s +Step 19900/44000 | Loss: 3.0124 | LR: 2.95e-04 | 710.4ms/step | 28.2 samples/s +Step 19910/44000 | Loss: 3.0687 | LR: 2.95e-04 | 709.8ms/step | 28.2 samples/s +Step 19920/44000 | Loss: 3.0297 | LR: 2.94e-04 | 721.8ms/step | 27.7 samples/s +Step 19930/44000 | Loss: 3.0034 | LR: 2.94e-04 | 714.9ms/step | 28.0 samples/s +Step 19940/44000 | Loss: 2.9980 | LR: 2.94e-04 | 712.0ms/step | 28.1 samples/s +Step 19950/44000 | Loss: 2.9880 | LR: 2.94e-04 | 712.5ms/step | 28.1 samples/s +Step 19960/44000 | Loss: 3.0282 | LR: 2.94e-04 | 710.5ms/step | 28.1 samples/s +Step 19970/44000 | Loss: 2.9560 | LR: 2.93e-04 | 709.6ms/step | 28.2 samples/s +Step 19980/44000 | Loss: 3.0566 | LR: 2.93e-04 | 710.5ms/step | 28.2 samples/s +Step 19990/44000 | Loss: 2.9815 | LR: 2.93e-04 | 710.6ms/step | 28.1 samples/s +Step 20000/44000 | Loss: 2.9388 | LR: 2.93e-04 | 710.3ms/step | 28.2 samples/s +Validation at step 20000: Loss = 2.9853, Perplexity = 19.79 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-20000 +Step 20010/44000 | Loss: 2.9622 | LR: 2.93e-04 | 1304.7ms/step | 15.3 samples/s +Step 20020/44000 | Loss: 2.9812 | LR: 2.93e-04 | 709.9ms/step | 28.2 samples/s +Step 20030/44000 | Loss: 3.0154 | LR: 2.92e-04 | 710.8ms/step | 28.1 samples/s +Step 20040/44000 | Loss: 3.0162 | LR: 2.92e-04 | 711.7ms/step | 28.1 samples/s +Step 20050/44000 | Loss: 3.0066 | LR: 2.92e-04 | 708.9ms/step | 28.2 samples/s +Step 20060/44000 | Loss: 2.9931 | LR: 2.92e-04 | 708.5ms/step | 28.2 samples/s +Step 20070/44000 | Loss: 3.0465 | LR: 2.92e-04 | 709.0ms/step | 28.2 samples/s +Step 20080/44000 | Loss: 2.9960 | LR: 2.91e-04 | 708.3ms/step | 28.2 samples/s +Step 20090/44000 | Loss: 2.9998 | LR: 2.91e-04 | 719.6ms/step | 27.8 samples/s +Step 20100/44000 | Loss: 3.0134 | LR: 2.91e-04 | 713.0ms/step | 28.1 samples/s +Step 20110/44000 | Loss: 2.9775 | LR: 2.91e-04 | 708.5ms/step | 28.2 samples/s +Step 20120/44000 | Loss: 2.9988 | LR: 2.91e-04 | 709.3ms/step | 28.2 samples/s +Step 20130/44000 | Loss: 3.0243 | LR: 2.91e-04 | 709.0ms/step | 28.2 samples/s +Step 20140/44000 | Loss: 3.0300 | LR: 2.90e-04 | 709.3ms/step | 28.2 samples/s +Step 20150/44000 | Loss: 2.9935 | LR: 2.90e-04 | 710.3ms/step | 28.2 samples/s +Step 20160/44000 | Loss: 3.0701 | LR: 2.90e-04 | 710.5ms/step | 28.2 samples/s +Step 20170/44000 | Loss: 3.0518 | LR: 2.90e-04 | 710.7ms/step | 28.1 samples/s +Step 20180/44000 | Loss: 3.0004 | LR: 2.90e-04 | 729.0ms/step | 27.4 samples/s +Step 20190/44000 | Loss: 2.9986 | LR: 2.90e-04 | 708.9ms/step | 28.2 samples/s +Step 20200/44000 | Loss: 3.0229 | LR: 2.89e-04 | 708.4ms/step | 28.2 samples/s +Step 20210/44000 | Loss: 3.0088 | LR: 2.89e-04 | 708.3ms/step | 28.2 samples/s +Step 20220/44000 | Loss: 2.9397 | LR: 2.89e-04 | 709.0ms/step | 28.2 samples/s +Step 20230/44000 | Loss: 3.0722 | LR: 2.89e-04 | 708.7ms/step | 28.2 samples/s +Step 20240/44000 | Loss: 2.9419 | LR: 2.89e-04 | 708.5ms/step | 28.2 samples/s +Step 20250/44000 | Loss: 3.0308 | LR: 2.88e-04 | 708.4ms/step | 28.2 samples/s +Validation at step 20250: Loss = 2.9836, Perplexity = 19.76 +Step 20260/44000 | Loss: 3.0126 | LR: 2.88e-04 | 929.6ms/step | 21.5 samples/s +Step 20270/44000 | Loss: 3.0205 | LR: 2.88e-04 | 722.3ms/step | 27.7 samples/s +Step 20280/44000 | Loss: 3.0142 | LR: 2.88e-04 | 708.8ms/step | 28.2 samples/s +Step 20290/44000 | Loss: 3.0301 | LR: 2.88e-04 | 708.8ms/step | 28.2 samples/s +Step 20300/44000 | Loss: 2.9626 | LR: 2.88e-04 | 709.1ms/step | 28.2 samples/s +Step 20310/44000 | Loss: 2.9928 | LR: 2.87e-04 | 708.9ms/step | 28.2 samples/s +Step 20320/44000 | Loss: 2.9583 | LR: 2.87e-04 | 709.0ms/step | 28.2 samples/s +Step 20330/44000 | Loss: 2.8995 | LR: 2.87e-04 | 708.7ms/step | 28.2 samples/s +Step 20340/44000 | Loss: 2.9869 | LR: 2.87e-04 | 708.7ms/step | 28.2 samples/s +Step 20350/44000 | Loss: 2.9380 | LR: 2.87e-04 | 709.4ms/step | 28.2 samples/s +Step 20360/44000 | Loss: 2.9757 | LR: 2.86e-04 | 728.3ms/step | 27.5 samples/s +Step 20370/44000 | Loss: 3.0521 | LR: 2.86e-04 | 709.5ms/step | 28.2 samples/s +Step 20380/44000 | Loss: 3.0154 | LR: 2.86e-04 | 709.6ms/step | 28.2 samples/s +Step 20390/44000 | Loss: 3.0399 | LR: 2.86e-04 | 709.3ms/step | 28.2 samples/s +Step 20400/44000 | Loss: 3.0388 | LR: 2.86e-04 | 708.5ms/step | 28.2 samples/s +Step 20410/44000 | Loss: 2.9991 | LR: 2.86e-04 | 708.3ms/step | 28.2 samples/s +Step 20420/44000 | Loss: 3.0415 | LR: 2.85e-04 | 709.7ms/step | 28.2 samples/s +Step 20430/44000 | Loss: 2.9693 | LR: 2.85e-04 | 709.0ms/step | 28.2 samples/s +Step 20440/44000 | Loss: 2.9779 | LR: 2.85e-04 | 709.2ms/step | 28.2 samples/s +Step 20450/44000 | Loss: 2.9122 | LR: 2.85e-04 | 725.2ms/step | 27.6 samples/s +Step 20460/44000 | Loss: 2.9862 | LR: 2.85e-04 | 708.9ms/step | 28.2 samples/s +Step 20470/44000 | Loss: 2.9565 | LR: 2.84e-04 | 709.2ms/step | 28.2 samples/s +Step 20480/44000 | Loss: 2.9907 | LR: 2.84e-04 | 708.9ms/step | 28.2 samples/s +Step 20490/44000 | Loss: 3.0182 | LR: 2.84e-04 | 708.6ms/step | 28.2 samples/s +Step 20500/44000 | Loss: 3.0000 | LR: 2.84e-04 | 708.4ms/step | 28.2 samples/s +Validation at step 20500: Loss = 2.9798, Perplexity = 19.68 +Step 20510/44000 | Loss: 2.9879 | LR: 2.84e-04 | 926.2ms/step | 21.6 samples/s +Step 20520/44000 | Loss: 3.0109 | LR: 2.84e-04 | 708.9ms/step | 28.2 samples/s +Step 20530/44000 | Loss: 2.9756 | LR: 2.83e-04 | 721.8ms/step | 27.7 samples/s +Step 20540/44000 | Loss: 2.9549 | LR: 2.83e-04 | 716.0ms/step | 27.9 samples/s +Step 20550/44000 | Loss: 2.9774 | LR: 2.83e-04 | 709.2ms/step | 28.2 samples/s +Step 20560/44000 | Loss: 3.0565 | LR: 2.83e-04 | 709.1ms/step | 28.2 samples/s +Step 20570/44000 | Loss: 3.0187 | LR: 2.83e-04 | 710.4ms/step | 28.2 samples/s +Step 20580/44000 | Loss: 2.9929 | LR: 2.83e-04 | 709.9ms/step | 28.2 samples/s +Step 20590/44000 | Loss: 3.0308 | LR: 2.82e-04 | 710.7ms/step | 28.1 samples/s +Step 20600/44000 | Loss: 2.9803 | LR: 2.82e-04 | 709.7ms/step | 28.2 samples/s +Step 20610/44000 | Loss: 2.9565 | LR: 2.82e-04 | 709.1ms/step | 28.2 samples/s +Step 20620/44000 | Loss: 3.0447 | LR: 2.82e-04 | 724.1ms/step | 27.6 samples/s +Step 20630/44000 | Loss: 2.9656 | LR: 2.82e-04 | 710.2ms/step | 28.2 samples/s +Step 20640/44000 | Loss: 2.9987 | LR: 2.81e-04 | 709.8ms/step | 28.2 samples/s +Step 20650/44000 | Loss: 2.9771 | LR: 2.81e-04 | 709.9ms/step | 28.2 samples/s +Step 20660/44000 | Loss: 3.0349 | LR: 2.81e-04 | 709.4ms/step | 28.2 samples/s +Step 20670/44000 | Loss: 2.9977 | LR: 2.81e-04 | 709.8ms/step | 28.2 samples/s +Step 20680/44000 | Loss: 3.0328 | LR: 2.81e-04 | 710.8ms/step | 28.1 samples/s +Step 20690/44000 | Loss: 2.9872 | LR: 2.81e-04 | 710.9ms/step | 28.1 samples/s +Step 20700/44000 | Loss: 2.9122 | LR: 2.80e-04 | 709.0ms/step | 28.2 samples/s +Step 20710/44000 | Loss: 2.9930 | LR: 2.80e-04 | 724.6ms/step | 27.6 samples/s +Step 20720/44000 | Loss: 2.9694 | LR: 2.80e-04 | 708.7ms/step | 28.2 samples/s +Step 20730/44000 | Loss: 2.9728 | LR: 2.80e-04 | 708.7ms/step | 28.2 samples/s +Step 20740/44000 | Loss: 2.9536 | LR: 2.80e-04 | 709.6ms/step | 28.2 samples/s +Step 20750/44000 | Loss: 2.9682 | LR: 2.79e-04 | 709.3ms/step | 28.2 samples/s +Validation at step 20750: Loss = 2.9775, Perplexity = 19.64 +Step 20760/44000 | Loss: 2.9358 | LR: 2.79e-04 | 927.2ms/step | 21.6 samples/s +Step 20770/44000 | Loss: 2.9786 | LR: 2.79e-04 | 709.4ms/step | 28.2 samples/s +Step 20780/44000 | Loss: 3.0175 | LR: 2.79e-04 | 708.7ms/step | 28.2 samples/s +Step 20790/44000 | Loss: 2.9658 | LR: 2.79e-04 | 708.7ms/step | 28.2 samples/s +Step 20800/44000 | Loss: 3.0107 | LR: 2.79e-04 | 724.0ms/step | 27.6 samples/s +Step 20810/44000 | Loss: 2.9733 | LR: 2.78e-04 | 708.9ms/step | 28.2 samples/s +Step 20820/44000 | Loss: 2.9731 | LR: 2.78e-04 | 709.1ms/step | 28.2 samples/s +Step 20830/44000 | Loss: 2.9952 | LR: 2.78e-04 | 708.7ms/step | 28.2 samples/s +Step 20840/44000 | Loss: 3.0366 | LR: 2.78e-04 | 708.4ms/step | 28.2 samples/s +Step 20850/44000 | Loss: 3.0355 | LR: 2.78e-04 | 709.1ms/step | 28.2 samples/s +Step 20860/44000 | Loss: 3.0937 | LR: 2.77e-04 | 708.3ms/step | 28.2 samples/s +Step 20870/44000 | Loss: 2.9323 | LR: 2.77e-04 | 708.5ms/step | 28.2 samples/s +Step 20880/44000 | Loss: 3.0857 | LR: 2.77e-04 | 710.0ms/step | 28.2 samples/s +Step 20890/44000 | Loss: 2.9529 | LR: 2.77e-04 | 723.2ms/step | 27.7 samples/s +Step 20900/44000 | Loss: 3.0389 | LR: 2.77e-04 | 708.6ms/step | 28.2 samples/s +Step 20910/44000 | Loss: 3.0561 | LR: 2.77e-04 | 708.9ms/step | 28.2 samples/s +Step 20920/44000 | Loss: 3.0217 | LR: 2.76e-04 | 709.0ms/step | 28.2 samples/s +Step 20930/44000 | Loss: 2.9286 | LR: 2.76e-04 | 708.5ms/step | 28.2 samples/s +Step 20940/44000 | Loss: 2.9095 | LR: 2.76e-04 | 709.5ms/step | 28.2 samples/s +Step 20950/44000 | Loss: 3.0519 | LR: 2.76e-04 | 709.0ms/step | 28.2 samples/s +Step 20960/44000 | Loss: 3.0494 | LR: 2.76e-04 | 708.5ms/step | 28.2 samples/s +Step 20970/44000 | Loss: 3.0191 | LR: 2.75e-04 | 714.1ms/step | 28.0 samples/s +Step 20980/44000 | Loss: 3.0165 | LR: 2.75e-04 | 722.5ms/step | 27.7 samples/s +Step 20990/44000 | Loss: 2.9481 | LR: 2.75e-04 | 708.1ms/step | 28.2 samples/s +Step 21000/44000 | Loss: 3.0195 | LR: 2.75e-04 | 708.4ms/step | 28.2 samples/s +Validation at step 21000: Loss = 2.9756, Perplexity = 19.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-21000 +Step 21010/44000 | Loss: 2.9968 | LR: 2.75e-04 | 1266.6ms/step | 15.8 samples/s +Step 21020/44000 | Loss: 2.9911 | LR: 2.75e-04 | 709.5ms/step | 28.2 samples/s +Step 21030/44000 | Loss: 3.0162 | LR: 2.74e-04 | 710.0ms/step | 28.2 samples/s +Step 21040/44000 | Loss: 3.0036 | LR: 2.74e-04 | 709.6ms/step | 28.2 samples/s +Step 21050/44000 | Loss: 3.0052 | LR: 2.74e-04 | 708.6ms/step | 28.2 samples/s +Step 21060/44000 | Loss: 3.0513 | LR: 2.74e-04 | 728.1ms/step | 27.5 samples/s +Step 21070/44000 | Loss: 2.9911 | LR: 2.74e-04 | 711.0ms/step | 28.1 samples/s +Step 21080/44000 | Loss: 2.9896 | LR: 2.73e-04 | 710.1ms/step | 28.2 samples/s +Step 21090/44000 | Loss: 3.0262 | LR: 2.73e-04 | 711.0ms/step | 28.1 samples/s +Step 21100/44000 | Loss: 3.0126 | LR: 2.73e-04 | 711.5ms/step | 28.1 samples/s +Step 21110/44000 | Loss: 3.0084 | LR: 2.73e-04 | 709.2ms/step | 28.2 samples/s +Step 21120/44000 | Loss: 2.9396 | LR: 2.73e-04 | 710.3ms/step | 28.2 samples/s +Step 21130/44000 | Loss: 2.9401 | LR: 2.73e-04 | 709.7ms/step | 28.2 samples/s +Step 21140/44000 | Loss: 2.9961 | LR: 2.72e-04 | 709.8ms/step | 28.2 samples/s +Step 21150/44000 | Loss: 3.0181 | LR: 2.72e-04 | 726.3ms/step | 27.5 samples/s +Step 21160/44000 | Loss: 3.0551 | LR: 2.72e-04 | 708.9ms/step | 28.2 samples/s +Step 21170/44000 | Loss: 2.9538 | LR: 2.72e-04 | 708.7ms/step | 28.2 samples/s +Step 21180/44000 | Loss: 2.9464 | LR: 2.72e-04 | 709.0ms/step | 28.2 samples/s +Step 21190/44000 | Loss: 2.9747 | LR: 2.71e-04 | 709.1ms/step | 28.2 samples/s +Step 21200/44000 | Loss: 3.0139 | LR: 2.71e-04 | 708.6ms/step | 28.2 samples/s +Step 21210/44000 | Loss: 2.9767 | LR: 2.71e-04 | 708.4ms/step | 28.2 samples/s +Step 21220/44000 | Loss: 2.9514 | LR: 2.71e-04 | 708.6ms/step | 28.2 samples/s +Step 21230/44000 | Loss: 3.0340 | LR: 2.71e-04 | 713.7ms/step | 28.0 samples/s +Step 21240/44000 | Loss: 2.9090 | LR: 2.71e-04 | 717.1ms/step | 27.9 samples/s +Step 21250/44000 | Loss: 3.0167 | LR: 2.70e-04 | 708.7ms/step | 28.2 samples/s +Validation at step 21250: Loss = 2.9719, Perplexity = 19.53 +Step 21260/44000 | Loss: 2.9373 | LR: 2.70e-04 | 926.5ms/step | 21.6 samples/s +Step 21270/44000 | Loss: 3.0614 | LR: 2.70e-04 | 709.2ms/step | 28.2 samples/s +Step 21280/44000 | Loss: 2.9911 | LR: 2.70e-04 | 709.5ms/step | 28.2 samples/s +Step 21290/44000 | Loss: 3.0080 | LR: 2.70e-04 | 709.4ms/step | 28.2 samples/s +Step 21300/44000 | Loss: 3.0632 | LR: 2.70e-04 | 709.5ms/step | 28.2 samples/s +Step 21310/44000 | Loss: 3.0537 | LR: 2.69e-04 | 709.9ms/step | 28.2 samples/s +Step 21320/44000 | Loss: 2.9923 | LR: 2.69e-04 | 724.9ms/step | 27.6 samples/s +Step 21330/44000 | Loss: 2.9665 | LR: 2.69e-04 | 708.9ms/step | 28.2 samples/s +Step 21340/44000 | Loss: 3.1171 | LR: 2.69e-04 | 708.6ms/step | 28.2 samples/s +Step 21350/44000 | Loss: 3.0096 | LR: 2.69e-04 | 708.8ms/step | 28.2 samples/s +Step 21360/44000 | Loss: 2.9345 | LR: 2.68e-04 | 708.7ms/step | 28.2 samples/s +Step 21370/44000 | Loss: 3.0099 | LR: 2.68e-04 | 708.9ms/step | 28.2 samples/s +Step 21380/44000 | Loss: 2.9645 | LR: 2.68e-04 | 709.3ms/step | 28.2 samples/s +Step 21390/44000 | Loss: 3.0855 | LR: 2.68e-04 | 709.2ms/step | 28.2 samples/s +Step 21400/44000 | Loss: 2.9906 | LR: 2.68e-04 | 708.8ms/step | 28.2 samples/s +Step 21410/44000 | Loss: 2.9457 | LR: 2.68e-04 | 723.8ms/step | 27.6 samples/s +Step 21420/44000 | Loss: 3.0388 | LR: 2.67e-04 | 708.7ms/step | 28.2 samples/s +Step 21430/44000 | Loss: 3.0200 | LR: 2.67e-04 | 708.9ms/step | 28.2 samples/s +Step 21440/44000 | Loss: 2.9862 | LR: 2.67e-04 | 708.8ms/step | 28.2 samples/s +Step 21450/44000 | Loss: 3.0098 | LR: 2.67e-04 | 709.2ms/step | 28.2 samples/s +Step 21460/44000 | Loss: 2.9855 | LR: 2.67e-04 | 708.9ms/step | 28.2 samples/s +Step 21470/44000 | Loss: 2.9771 | LR: 2.66e-04 | 709.0ms/step | 28.2 samples/s +Step 21480/44000 | Loss: 2.9592 | LR: 2.66e-04 | 709.0ms/step | 28.2 samples/s +Step 21490/44000 | Loss: 3.0175 | LR: 2.66e-04 | 708.9ms/step | 28.2 samples/s +Step 21500/44000 | Loss: 3.0050 | LR: 2.66e-04 | 722.7ms/step | 27.7 samples/s +Validation at step 21500: Loss = 2.9704, Perplexity = 19.50 +Step 21510/44000 | Loss: 2.9908 | LR: 2.66e-04 | 930.0ms/step | 21.5 samples/s +Step 21520/44000 | Loss: 2.9976 | LR: 2.66e-04 | 709.3ms/step | 28.2 samples/s +Step 21530/44000 | Loss: 2.9748 | LR: 2.65e-04 | 709.4ms/step | 28.2 samples/s +Step 21540/44000 | Loss: 2.9838 | LR: 2.65e-04 | 708.9ms/step | 28.2 samples/s +Step 21550/44000 | Loss: 3.0158 | LR: 2.65e-04 | 709.1ms/step | 28.2 samples/s +Step 21560/44000 | Loss: 2.9750 | LR: 2.65e-04 | 709.3ms/step | 28.2 samples/s +Step 21570/44000 | Loss: 2.9606 | LR: 2.65e-04 | 708.4ms/step | 28.2 samples/s +Step 21580/44000 | Loss: 2.9573 | LR: 2.64e-04 | 708.7ms/step | 28.2 samples/s +Step 21590/44000 | Loss: 2.9711 | LR: 2.64e-04 | 725.5ms/step | 27.6 samples/s +Step 21600/44000 | Loss: 2.9724 | LR: 2.64e-04 | 709.2ms/step | 28.2 samples/s +Step 21610/44000 | Loss: 2.9900 | LR: 2.64e-04 | 710.4ms/step | 28.2 samples/s +Step 21620/44000 | Loss: 3.0146 | LR: 2.64e-04 | 709.4ms/step | 28.2 samples/s +Step 21630/44000 | Loss: 2.9801 | LR: 2.64e-04 | 709.2ms/step | 28.2 samples/s +Step 21640/44000 | Loss: 3.0037 | LR: 2.63e-04 | 709.9ms/step | 28.2 samples/s +Step 21650/44000 | Loss: 2.9749 | LR: 2.63e-04 | 708.9ms/step | 28.2 samples/s +Step 21660/44000 | Loss: 3.0234 | LR: 2.63e-04 | 710.5ms/step | 28.2 samples/s +Step 21670/44000 | Loss: 2.9839 | LR: 2.63e-04 | 715.0ms/step | 28.0 samples/s +Step 21680/44000 | Loss: 2.9867 | LR: 2.63e-04 | 721.7ms/step | 27.7 samples/s +Step 21690/44000 | Loss: 2.9446 | LR: 2.62e-04 | 709.2ms/step | 28.2 samples/s +Step 21700/44000 | Loss: 2.9155 | LR: 2.62e-04 | 710.4ms/step | 28.2 samples/s +Step 21710/44000 | Loss: 2.9768 | LR: 2.62e-04 | 710.1ms/step | 28.2 samples/s +Step 21720/44000 | Loss: 3.0437 | LR: 2.62e-04 | 709.1ms/step | 28.2 samples/s +Step 21730/44000 | Loss: 3.0392 | LR: 2.62e-04 | 711.5ms/step | 28.1 samples/s +Step 21740/44000 | Loss: 2.9791 | LR: 2.62e-04 | 710.8ms/step | 28.1 samples/s +Step 21750/44000 | Loss: 2.9644 | LR: 2.61e-04 | 711.7ms/step | 28.1 samples/s +Validation at step 21750: Loss = 2.9668, Perplexity = 19.43 +Step 21760/44000 | Loss: 3.0566 | LR: 2.61e-04 | 945.1ms/step | 21.2 samples/s +Step 21770/44000 | Loss: 2.9727 | LR: 2.61e-04 | 709.3ms/step | 28.2 samples/s +Step 21780/44000 | Loss: 2.9825 | LR: 2.61e-04 | 709.3ms/step | 28.2 samples/s +Step 21790/44000 | Loss: 3.0127 | LR: 2.61e-04 | 708.7ms/step | 28.2 samples/s +Step 21800/44000 | Loss: 3.0055 | LR: 2.60e-04 | 709.3ms/step | 28.2 samples/s +Step 21810/44000 | Loss: 2.9637 | LR: 2.60e-04 | 709.6ms/step | 28.2 samples/s +Step 21820/44000 | Loss: 2.9502 | LR: 2.60e-04 | 711.1ms/step | 28.1 samples/s +Step 21830/44000 | Loss: 2.9800 | LR: 2.60e-04 | 711.5ms/step | 28.1 samples/s +Step 21840/44000 | Loss: 3.0127 | LR: 2.60e-04 | 712.3ms/step | 28.1 samples/s +Step 21850/44000 | Loss: 2.9956 | LR: 2.60e-04 | 727.9ms/step | 27.5 samples/s +Step 21860/44000 | Loss: 2.9959 | LR: 2.59e-04 | 712.0ms/step | 28.1 samples/s +Step 21870/44000 | Loss: 3.0172 | LR: 2.59e-04 | 709.8ms/step | 28.2 samples/s +Step 21880/44000 | Loss: 2.9747 | LR: 2.59e-04 | 709.6ms/step | 28.2 samples/s +Step 21890/44000 | Loss: 2.9653 | LR: 2.59e-04 | 709.0ms/step | 28.2 samples/s +Step 21900/44000 | Loss: 2.9118 | LR: 2.59e-04 | 708.8ms/step | 28.2 samples/s +Step 21910/44000 | Loss: 2.9685 | LR: 2.58e-04 | 708.9ms/step | 28.2 samples/s +Step 21920/44000 | Loss: 3.0233 | LR: 2.58e-04 | 709.6ms/step | 28.2 samples/s +Step 21930/44000 | Loss: 2.9522 | LR: 2.58e-04 | 708.8ms/step | 28.2 samples/s +Step 21940/44000 | Loss: 2.9726 | LR: 2.58e-04 | 727.8ms/step | 27.5 samples/s +Step 21950/44000 | Loss: 2.9614 | LR: 2.58e-04 | 709.0ms/step | 28.2 samples/s +Step 21960/44000 | Loss: 2.9674 | LR: 2.58e-04 | 709.0ms/step | 28.2 samples/s +Step 21970/44000 | Loss: 2.9683 | LR: 2.57e-04 | 709.3ms/step | 28.2 samples/s +Step 21980/44000 | Loss: 2.9904 | LR: 2.57e-04 | 708.8ms/step | 28.2 samples/s +Step 21990/44000 | Loss: 3.0085 | LR: 2.57e-04 | 708.7ms/step | 28.2 samples/s +Step 22000/44000 | Loss: 3.0308 | LR: 2.57e-04 | 708.7ms/step | 28.2 samples/s +Validation at step 22000: Loss = 2.9640, Perplexity = 19.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-22000 +Step 22010/44000 | Loss: 2.9333 | LR: 2.57e-04 | 1328.2ms/step | 15.1 samples/s +Step 22020/44000 | Loss: 2.9913 | LR: 2.56e-04 | 726.5ms/step | 27.5 samples/s +Step 22030/44000 | Loss: 2.9621 | LR: 2.56e-04 | 710.3ms/step | 28.2 samples/s +Step 22040/44000 | Loss: 2.9735 | LR: 2.56e-04 | 710.3ms/step | 28.2 samples/s +Step 22050/44000 | Loss: 3.0394 | LR: 2.56e-04 | 709.0ms/step | 28.2 samples/s +Step 22060/44000 | Loss: 2.9734 | LR: 2.56e-04 | 710.1ms/step | 28.2 samples/s +Step 22070/44000 | Loss: 3.0643 | LR: 2.56e-04 | 709.9ms/step | 28.2 samples/s +Step 22080/44000 | Loss: 2.9235 | LR: 2.55e-04 | 711.1ms/step | 28.1 samples/s +Step 22090/44000 | Loss: 2.9483 | LR: 2.55e-04 | 710.4ms/step | 28.2 samples/s +Step 22100/44000 | Loss: 2.9286 | LR: 2.55e-04 | 710.0ms/step | 28.2 samples/s +Step 22110/44000 | Loss: 3.0615 | LR: 2.55e-04 | 725.0ms/step | 27.6 samples/s +Step 22120/44000 | Loss: 3.0179 | LR: 2.55e-04 | 711.0ms/step | 28.1 samples/s +Step 22130/44000 | Loss: 2.9778 | LR: 2.54e-04 | 710.1ms/step | 28.2 samples/s +Step 22140/44000 | Loss: 2.9348 | LR: 2.54e-04 | 710.2ms/step | 28.2 samples/s +Step 22150/44000 | Loss: 2.9451 | LR: 2.54e-04 | 710.5ms/step | 28.2 samples/s +Step 22160/44000 | Loss: 3.0316 | LR: 2.54e-04 | 709.8ms/step | 28.2 samples/s +Step 22170/44000 | Loss: 2.9761 | LR: 2.54e-04 | 709.5ms/step | 28.2 samples/s +Step 22180/44000 | Loss: 3.0715 | LR: 2.54e-04 | 709.9ms/step | 28.2 samples/s +Step 22190/44000 | Loss: 2.9534 | LR: 2.53e-04 | 710.2ms/step | 28.2 samples/s +Step 22200/44000 | Loss: 2.9709 | LR: 2.53e-04 | 725.7ms/step | 27.6 samples/s +Step 22210/44000 | Loss: 2.9836 | LR: 2.53e-04 | 710.0ms/step | 28.2 samples/s +Step 22220/44000 | Loss: 3.0441 | LR: 2.53e-04 | 709.3ms/step | 28.2 samples/s +Step 22230/44000 | Loss: 3.0393 | LR: 2.53e-04 | 709.9ms/step | 28.2 samples/s +Step 22240/44000 | Loss: 2.9647 | LR: 2.52e-04 | 709.7ms/step | 28.2 samples/s +Step 22250/44000 | Loss: 2.9855 | LR: 2.52e-04 | 709.7ms/step | 28.2 samples/s +Validation at step 22250: Loss = 2.9615, Perplexity = 19.33 +Step 22260/44000 | Loss: 2.9739 | LR: 2.52e-04 | 930.6ms/step | 21.5 samples/s +Step 22270/44000 | Loss: 2.9630 | LR: 2.52e-04 | 710.2ms/step | 28.2 samples/s +Step 22280/44000 | Loss: 2.9957 | LR: 2.52e-04 | 722.0ms/step | 27.7 samples/s +Step 22290/44000 | Loss: 2.9538 | LR: 2.52e-04 | 714.8ms/step | 28.0 samples/s +Step 22300/44000 | Loss: 2.9434 | LR: 2.51e-04 | 710.0ms/step | 28.2 samples/s +Step 22310/44000 | Loss: 2.9560 | LR: 2.51e-04 | 711.1ms/step | 28.1 samples/s +Step 22320/44000 | Loss: 3.0042 | LR: 2.51e-04 | 710.6ms/step | 28.1 samples/s +Step 22330/44000 | Loss: 2.9552 | LR: 2.51e-04 | 710.0ms/step | 28.2 samples/s +Step 22340/44000 | Loss: 2.9228 | LR: 2.51e-04 | 709.5ms/step | 28.2 samples/s +Step 22350/44000 | Loss: 2.9374 | LR: 2.50e-04 | 709.8ms/step | 28.2 samples/s +Step 22360/44000 | Loss: 2.9933 | LR: 2.50e-04 | 709.6ms/step | 28.2 samples/s +Step 22370/44000 | Loss: 2.9286 | LR: 2.50e-04 | 722.1ms/step | 27.7 samples/s +Step 22380/44000 | Loss: 2.9297 | LR: 2.50e-04 | 715.5ms/step | 28.0 samples/s +Step 22390/44000 | Loss: 2.9995 | LR: 2.50e-04 | 711.0ms/step | 28.1 samples/s +Step 22400/44000 | Loss: 2.9223 | LR: 2.50e-04 | 710.7ms/step | 28.1 samples/s +Step 22410/44000 | Loss: 3.0069 | LR: 2.49e-04 | 710.6ms/step | 28.1 samples/s +Step 22420/44000 | Loss: 3.0484 | LR: 2.49e-04 | 711.4ms/step | 28.1 samples/s +Step 22430/44000 | Loss: 3.0138 | LR: 2.49e-04 | 711.2ms/step | 28.1 samples/s +Step 22440/44000 | Loss: 2.9959 | LR: 2.49e-04 | 710.7ms/step | 28.1 samples/s +Step 22450/44000 | Loss: 3.0112 | LR: 2.49e-04 | 710.8ms/step | 28.1 samples/s +Step 22460/44000 | Loss: 2.9935 | LR: 2.48e-04 | 723.3ms/step | 27.7 samples/s +Step 22470/44000 | Loss: 2.9463 | LR: 2.48e-04 | 715.2ms/step | 28.0 samples/s +Step 22480/44000 | Loss: 2.9558 | LR: 2.48e-04 | 710.0ms/step | 28.2 samples/s +Step 22490/44000 | Loss: 2.9488 | LR: 2.48e-04 | 709.9ms/step | 28.2 samples/s +Step 22500/44000 | Loss: 2.9355 | LR: 2.48e-04 | 710.0ms/step | 28.2 samples/s +Validation at step 22500: Loss = 2.9586, Perplexity = 19.27 +Step 22510/44000 | Loss: 3.0194 | LR: 2.48e-04 | 929.2ms/step | 21.5 samples/s +Step 22520/44000 | Loss: 2.9445 | LR: 2.47e-04 | 710.8ms/step | 28.1 samples/s +Step 22530/44000 | Loss: 3.0140 | LR: 2.47e-04 | 710.1ms/step | 28.2 samples/s +Step 22540/44000 | Loss: 2.9789 | LR: 2.47e-04 | 710.6ms/step | 28.1 samples/s +Step 22550/44000 | Loss: 2.9125 | LR: 2.47e-04 | 728.0ms/step | 27.5 samples/s +Step 22560/44000 | Loss: 2.9501 | LR: 2.47e-04 | 710.4ms/step | 28.2 samples/s +Step 22570/44000 | Loss: 2.9475 | LR: 2.46e-04 | 709.8ms/step | 28.2 samples/s +Step 22580/44000 | Loss: 2.9499 | LR: 2.46e-04 | 708.9ms/step | 28.2 samples/s +Step 22590/44000 | Loss: 2.9671 | LR: 2.46e-04 | 709.6ms/step | 28.2 samples/s +Step 22600/44000 | Loss: 2.9177 | LR: 2.46e-04 | 709.5ms/step | 28.2 samples/s +Step 22610/44000 | Loss: 2.9984 | LR: 2.46e-04 | 710.5ms/step | 28.1 samples/s +Step 22620/44000 | Loss: 2.9966 | LR: 2.46e-04 | 711.2ms/step | 28.1 samples/s +Step 22630/44000 | Loss: 2.9665 | LR: 2.45e-04 | 711.1ms/step | 28.1 samples/s +Step 22640/44000 | Loss: 2.9731 | LR: 2.45e-04 | 728.4ms/step | 27.5 samples/s +Step 22650/44000 | Loss: 2.9818 | LR: 2.45e-04 | 711.1ms/step | 28.1 samples/s +Step 22660/44000 | Loss: 2.9633 | LR: 2.45e-04 | 711.6ms/step | 28.1 samples/s +Step 22670/44000 | Loss: 2.9916 | LR: 2.45e-04 | 711.9ms/step | 28.1 samples/s +Step 22680/44000 | Loss: 2.9208 | LR: 2.44e-04 | 712.8ms/step | 28.1 samples/s +Step 22690/44000 | Loss: 2.9468 | LR: 2.44e-04 | 713.1ms/step | 28.0 samples/s +Step 22700/44000 | Loss: 3.0215 | LR: 2.44e-04 | 713.8ms/step | 28.0 samples/s +Step 22710/44000 | Loss: 3.0516 | LR: 2.44e-04 | 713.3ms/step | 28.0 samples/s +Step 22720/44000 | Loss: 3.0693 | LR: 2.44e-04 | 713.0ms/step | 28.0 samples/s +Step 22730/44000 | Loss: 2.9825 | LR: 2.44e-04 | 731.7ms/step | 27.3 samples/s +Step 22740/44000 | Loss: 3.0389 | LR: 2.43e-04 | 713.8ms/step | 28.0 samples/s +Step 22750/44000 | Loss: 2.9601 | LR: 2.43e-04 | 713.1ms/step | 28.0 samples/s +Validation at step 22750: Loss = 2.9570, Perplexity = 19.24 +Step 22760/44000 | Loss: 2.9575 | LR: 2.43e-04 | 935.3ms/step | 21.4 samples/s +Step 22770/44000 | Loss: 2.9933 | LR: 2.43e-04 | 712.4ms/step | 28.1 samples/s +Step 22780/44000 | Loss: 2.9696 | LR: 2.43e-04 | 712.1ms/step | 28.1 samples/s +Step 22790/44000 | Loss: 3.0191 | LR: 2.42e-04 | 709.8ms/step | 28.2 samples/s +Step 22800/44000 | Loss: 3.0245 | LR: 2.42e-04 | 711.4ms/step | 28.1 samples/s +Step 22810/44000 | Loss: 3.0672 | LR: 2.42e-04 | 719.4ms/step | 27.8 samples/s +Step 22820/44000 | Loss: 3.0223 | LR: 2.42e-04 | 714.5ms/step | 28.0 samples/s +Step 22830/44000 | Loss: 2.9681 | LR: 2.42e-04 | 710.1ms/step | 28.2 samples/s +Step 22840/44000 | Loss: 2.9914 | LR: 2.42e-04 | 709.2ms/step | 28.2 samples/s +Step 22850/44000 | Loss: 3.0499 | LR: 2.41e-04 | 709.9ms/step | 28.2 samples/s +Step 22860/44000 | Loss: 2.9411 | LR: 2.41e-04 | 711.2ms/step | 28.1 samples/s +Step 22870/44000 | Loss: 2.9390 | LR: 2.41e-04 | 711.7ms/step | 28.1 samples/s +Step 22880/44000 | Loss: 2.9761 | LR: 2.41e-04 | 711.1ms/step | 28.1 samples/s +Step 22890/44000 | Loss: 2.9790 | LR: 2.41e-04 | 711.8ms/step | 28.1 samples/s +Step 22900/44000 | Loss: 2.9642 | LR: 2.40e-04 | 725.7ms/step | 27.6 samples/s +Step 22910/44000 | Loss: 3.0309 | LR: 2.40e-04 | 712.3ms/step | 28.1 samples/s +Step 22920/44000 | Loss: 3.0321 | LR: 2.40e-04 | 711.7ms/step | 28.1 samples/s +Step 22930/44000 | Loss: 2.8981 | LR: 2.40e-04 | 711.9ms/step | 28.1 samples/s +Step 22940/44000 | Loss: 2.9568 | LR: 2.40e-04 | 712.7ms/step | 28.1 samples/s +Step 22950/44000 | Loss: 3.0025 | LR: 2.40e-04 | 714.1ms/step | 28.0 samples/s +Step 22960/44000 | Loss: 2.9904 | LR: 2.39e-04 | 712.7ms/step | 28.1 samples/s +Step 22970/44000 | Loss: 3.0585 | LR: 2.39e-04 | 712.5ms/step | 28.1 samples/s +Step 22980/44000 | Loss: 2.9213 | LR: 2.39e-04 | 712.5ms/step | 28.1 samples/s +Step 22990/44000 | Loss: 2.9669 | LR: 2.39e-04 | 730.9ms/step | 27.4 samples/s +Step 23000/44000 | Loss: 3.0008 | LR: 2.39e-04 | 713.5ms/step | 28.0 samples/s +Validation at step 23000: Loss = 2.9555, Perplexity = 19.21 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-23000 +Step 23010/44000 | Loss: 2.9538 | LR: 2.38e-04 | 1264.8ms/step | 15.8 samples/s +Step 23020/44000 | Loss: 2.9748 | LR: 2.38e-04 | 712.0ms/step | 28.1 samples/s +Step 23030/44000 | Loss: 2.9956 | LR: 2.38e-04 | 712.4ms/step | 28.1 samples/s +Step 23040/44000 | Loss: 2.9972 | LR: 2.38e-04 | 712.0ms/step | 28.1 samples/s +Step 23050/44000 | Loss: 2.9355 | LR: 2.38e-04 | 711.5ms/step | 28.1 samples/s +Step 23060/44000 | Loss: 2.9687 | LR: 2.38e-04 | 712.5ms/step | 28.1 samples/s +Step 23070/44000 | Loss: 2.9267 | LR: 2.37e-04 | 730.1ms/step | 27.4 samples/s +Step 23080/44000 | Loss: 3.0250 | LR: 2.37e-04 | 712.2ms/step | 28.1 samples/s +Step 23090/44000 | Loss: 2.9492 | LR: 2.37e-04 | 710.3ms/step | 28.2 samples/s +Step 23100/44000 | Loss: 2.9408 | LR: 2.37e-04 | 715.7ms/step | 27.9 samples/s +Step 23110/44000 | Loss: 2.9821 | LR: 2.37e-04 | 712.8ms/step | 28.1 samples/s +Step 23120/44000 | Loss: 2.9786 | LR: 2.36e-04 | 712.9ms/step | 28.1 samples/s +Step 23130/44000 | Loss: 2.9706 | LR: 2.36e-04 | 713.0ms/step | 28.1 samples/s +Step 23140/44000 | Loss: 2.9804 | LR: 2.36e-04 | 712.2ms/step | 28.1 samples/s +Step 23150/44000 | Loss: 2.9821 | LR: 2.36e-04 | 711.8ms/step | 28.1 samples/s +Step 23160/44000 | Loss: 3.0000 | LR: 2.36e-04 | 727.0ms/step | 27.5 samples/s +Step 23170/44000 | Loss: 2.9190 | LR: 2.36e-04 | 712.2ms/step | 28.1 samples/s +Step 23180/44000 | Loss: 3.0165 | LR: 2.35e-04 | 712.6ms/step | 28.1 samples/s +Step 23190/44000 | Loss: 2.9444 | LR: 2.35e-04 | 711.3ms/step | 28.1 samples/s +Step 23200/44000 | Loss: 2.9146 | LR: 2.35e-04 | 711.3ms/step | 28.1 samples/s +Step 23210/44000 | Loss: 2.9580 | LR: 2.35e-04 | 711.6ms/step | 28.1 samples/s +Step 23220/44000 | Loss: 3.0253 | LR: 2.35e-04 | 711.5ms/step | 28.1 samples/s +Step 23230/44000 | Loss: 2.9824 | LR: 2.34e-04 | 712.5ms/step | 28.1 samples/s +Step 23240/44000 | Loss: 2.9405 | LR: 2.34e-04 | 711.3ms/step | 28.1 samples/s +Step 23250/44000 | Loss: 2.9836 | LR: 2.34e-04 | 725.0ms/step | 27.6 samples/s +Validation at step 23250: Loss = 2.9522, Perplexity = 19.15 +Step 23260/44000 | Loss: 2.8868 | LR: 2.34e-04 | 930.0ms/step | 21.5 samples/s +Step 23270/44000 | Loss: 2.9446 | LR: 2.34e-04 | 711.5ms/step | 28.1 samples/s +Step 23280/44000 | Loss: 2.9845 | LR: 2.34e-04 | 710.6ms/step | 28.1 samples/s +Step 23290/44000 | Loss: 2.9649 | LR: 2.33e-04 | 711.1ms/step | 28.1 samples/s +Step 23300/44000 | Loss: 2.9588 | LR: 2.33e-04 | 710.5ms/step | 28.1 samples/s +Step 23310/44000 | Loss: 2.9594 | LR: 2.33e-04 | 710.6ms/step | 28.1 samples/s +Step 23320/44000 | Loss: 2.9917 | LR: 2.33e-04 | 710.8ms/step | 28.1 samples/s +Step 23330/44000 | Loss: 2.9472 | LR: 2.33e-04 | 720.3ms/step | 27.8 samples/s +Step 23340/44000 | Loss: 2.9010 | LR: 2.32e-04 | 717.3ms/step | 27.9 samples/s +Step 23350/44000 | Loss: 3.0197 | LR: 2.32e-04 | 711.0ms/step | 28.1 samples/s +Step 23360/44000 | Loss: 2.9921 | LR: 2.32e-04 | 710.5ms/step | 28.2 samples/s +Step 23370/44000 | Loss: 2.9782 | LR: 2.32e-04 | 711.2ms/step | 28.1 samples/s +Step 23380/44000 | Loss: 2.9335 | LR: 2.32e-04 | 711.8ms/step | 28.1 samples/s +Step 23390/44000 | Loss: 3.0130 | LR: 2.32e-04 | 712.7ms/step | 28.1 samples/s +Step 23400/44000 | Loss: 2.9481 | LR: 2.31e-04 | 710.8ms/step | 28.1 samples/s +Step 23410/44000 | Loss: 2.9363 | LR: 2.31e-04 | 711.1ms/step | 28.1 samples/s +Step 23420/44000 | Loss: 3.0082 | LR: 2.31e-04 | 719.9ms/step | 27.8 samples/s +Step 23430/44000 | Loss: 2.9391 | LR: 2.31e-04 | 714.8ms/step | 28.0 samples/s +Step 23440/44000 | Loss: 3.0711 | LR: 2.31e-04 | 711.2ms/step | 28.1 samples/s +Step 23450/44000 | Loss: 2.9548 | LR: 2.30e-04 | 711.1ms/step | 28.1 samples/s +Step 23460/44000 | Loss: 2.9640 | LR: 2.30e-04 | 711.1ms/step | 28.1 samples/s +Step 23470/44000 | Loss: 2.9880 | LR: 2.30e-04 | 711.1ms/step | 28.1 samples/s +Step 23480/44000 | Loss: 2.9247 | LR: 2.30e-04 | 711.0ms/step | 28.1 samples/s +Step 23490/44000 | Loss: 3.0863 | LR: 2.30e-04 | 710.9ms/step | 28.1 samples/s +Step 23500/44000 | Loss: 2.9188 | LR: 2.30e-04 | 710.3ms/step | 28.2 samples/s +Validation at step 23500: Loss = 2.9513, Perplexity = 19.13 +Step 23510/44000 | Loss: 2.9712 | LR: 2.29e-04 | 942.9ms/step | 21.2 samples/s +Step 23520/44000 | Loss: 3.0118 | LR: 2.29e-04 | 708.6ms/step | 28.2 samples/s +Step 23530/44000 | Loss: 2.9322 | LR: 2.29e-04 | 708.8ms/step | 28.2 samples/s +Step 23540/44000 | Loss: 2.9604 | LR: 2.29e-04 | 709.3ms/step | 28.2 samples/s +Step 23550/44000 | Loss: 3.0149 | LR: 2.29e-04 | 713.9ms/step | 28.0 samples/s +Step 23560/44000 | Loss: 2.9152 | LR: 2.29e-04 | 716.6ms/step | 27.9 samples/s +Step 23570/44000 | Loss: 2.9954 | LR: 2.28e-04 | 717.2ms/step | 27.9 samples/s +Step 23580/44000 | Loss: 2.9762 | LR: 2.28e-04 | 714.6ms/step | 28.0 samples/s +Step 23590/44000 | Loss: 3.0339 | LR: 2.28e-04 | 718.0ms/step | 27.9 samples/s +Step 23600/44000 | Loss: 2.9952 | LR: 2.28e-04 | 730.4ms/step | 27.4 samples/s +Step 23610/44000 | Loss: 2.9338 | LR: 2.28e-04 | 709.4ms/step | 28.2 samples/s +Step 23620/44000 | Loss: 3.0052 | LR: 2.27e-04 | 713.0ms/step | 28.1 samples/s +Step 23630/44000 | Loss: 2.9666 | LR: 2.27e-04 | 713.9ms/step | 28.0 samples/s +Step 23640/44000 | Loss: 2.9130 | LR: 2.27e-04 | 715.5ms/step | 28.0 samples/s +Step 23650/44000 | Loss: 2.9267 | LR: 2.27e-04 | 717.3ms/step | 27.9 samples/s +Step 23660/44000 | Loss: 2.9372 | LR: 2.27e-04 | 716.5ms/step | 27.9 samples/s +Step 23670/44000 | Loss: 2.9710 | LR: 2.27e-04 | 719.2ms/step | 27.8 samples/s +Step 23680/44000 | Loss: 2.9836 | LR: 2.26e-04 | 713.2ms/step | 28.0 samples/s +Step 23690/44000 | Loss: 3.0054 | LR: 2.26e-04 | 725.0ms/step | 27.6 samples/s +Step 23700/44000 | Loss: 2.9288 | LR: 2.26e-04 | 714.8ms/step | 28.0 samples/s +Step 23710/44000 | Loss: 3.0267 | LR: 2.26e-04 | 715.0ms/step | 28.0 samples/s +Step 23720/44000 | Loss: 2.9725 | LR: 2.26e-04 | 712.8ms/step | 28.1 samples/s +Step 23730/44000 | Loss: 2.9791 | LR: 2.25e-04 | 712.1ms/step | 28.1 samples/s +Step 23740/44000 | Loss: 2.9274 | LR: 2.25e-04 | 710.6ms/step | 28.1 samples/s +Step 23750/44000 | Loss: 2.9160 | LR: 2.25e-04 | 710.7ms/step | 28.1 samples/s +Validation at step 23750: Loss = 2.9484, Perplexity = 19.07 +Step 23760/44000 | Loss: 3.0190 | LR: 2.25e-04 | 928.9ms/step | 21.5 samples/s +Step 23770/44000 | Loss: 2.9736 | LR: 2.25e-04 | 720.8ms/step | 27.7 samples/s +Step 23780/44000 | Loss: 2.9696 | LR: 2.25e-04 | 717.7ms/step | 27.9 samples/s +Step 23790/44000 | Loss: 2.9755 | LR: 2.24e-04 | 712.5ms/step | 28.1 samples/s +Step 23800/44000 | Loss: 3.0247 | LR: 2.24e-04 | 714.9ms/step | 28.0 samples/s +Step 23810/44000 | Loss: 3.0079 | LR: 2.24e-04 | 710.6ms/step | 28.1 samples/s +Step 23820/44000 | Loss: 2.9371 | LR: 2.24e-04 | 711.3ms/step | 28.1 samples/s +Step 23830/44000 | Loss: 2.9811 | LR: 2.24e-04 | 711.5ms/step | 28.1 samples/s +Step 23840/44000 | Loss: 2.9301 | LR: 2.23e-04 | 710.8ms/step | 28.1 samples/s +Step 23850/44000 | Loss: 2.9315 | LR: 2.23e-04 | 710.9ms/step | 28.1 samples/s +Step 23860/44000 | Loss: 3.0337 | LR: 2.23e-04 | 726.9ms/step | 27.5 samples/s +Step 23870/44000 | Loss: 3.0081 | LR: 2.23e-04 | 710.8ms/step | 28.1 samples/s +Step 23880/44000 | Loss: 2.9842 | LR: 2.23e-04 | 710.5ms/step | 28.1 samples/s +Step 23890/44000 | Loss: 2.9387 | LR: 2.23e-04 | 710.9ms/step | 28.1 samples/s +Step 23900/44000 | Loss: 2.9016 | LR: 2.22e-04 | 709.2ms/step | 28.2 samples/s +Step 23910/44000 | Loss: 2.9506 | LR: 2.22e-04 | 708.9ms/step | 28.2 samples/s +Step 23920/44000 | Loss: 2.8610 | LR: 2.22e-04 | 711.2ms/step | 28.1 samples/s +Step 23930/44000 | Loss: 2.9674 | LR: 2.22e-04 | 713.3ms/step | 28.0 samples/s +Step 23940/44000 | Loss: 2.9554 | LR: 2.22e-04 | 713.2ms/step | 28.0 samples/s +Step 23950/44000 | Loss: 2.8749 | LR: 2.21e-04 | 726.2ms/step | 27.5 samples/s +Step 23960/44000 | Loss: 2.9493 | LR: 2.21e-04 | 712.7ms/step | 28.1 samples/s +Step 23970/44000 | Loss: 2.9788 | LR: 2.21e-04 | 715.9ms/step | 27.9 samples/s +Step 23980/44000 | Loss: 2.9950 | LR: 2.21e-04 | 714.4ms/step | 28.0 samples/s +Step 23990/44000 | Loss: 2.9721 | LR: 2.21e-04 | 711.7ms/step | 28.1 samples/s +Step 24000/44000 | Loss: 2.9384 | LR: 2.21e-04 | 710.9ms/step | 28.1 samples/s +Validation at step 24000: Loss = 2.9455, Perplexity = 19.02 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-24000 +Step 24010/44000 | Loss: 2.9353 | LR: 2.20e-04 | 1293.3ms/step | 15.5 samples/s +Step 24020/44000 | Loss: 2.9888 | LR: 2.20e-04 | 709.0ms/step | 28.2 samples/s +Step 24030/44000 | Loss: 2.9329 | LR: 2.20e-04 | 727.2ms/step | 27.5 samples/s +Step 24040/44000 | Loss: 2.9620 | LR: 2.20e-04 | 709.6ms/step | 28.2 samples/s +Step 24050/44000 | Loss: 2.9883 | LR: 2.20e-04 | 710.1ms/step | 28.2 samples/s +Step 24060/44000 | Loss: 2.9275 | LR: 2.19e-04 | 709.7ms/step | 28.2 samples/s +Step 24070/44000 | Loss: 3.0020 | LR: 2.19e-04 | 709.5ms/step | 28.2 samples/s +Step 24080/44000 | Loss: 3.0087 | LR: 2.19e-04 | 709.6ms/step | 28.2 samples/s +Step 24090/44000 | Loss: 2.8917 | LR: 2.19e-04 | 710.7ms/step | 28.1 samples/s +Step 24100/44000 | Loss: 2.9141 | LR: 2.19e-04 | 712.3ms/step | 28.1 samples/s +Step 24110/44000 | Loss: 2.9369 | LR: 2.19e-04 | 711.2ms/step | 28.1 samples/s +Step 24120/44000 | Loss: 2.9499 | LR: 2.18e-04 | 727.2ms/step | 27.5 samples/s +Step 24130/44000 | Loss: 2.9226 | LR: 2.18e-04 | 714.0ms/step | 28.0 samples/s +Step 24140/44000 | Loss: 2.9553 | LR: 2.18e-04 | 711.4ms/step | 28.1 samples/s +Step 24150/44000 | Loss: 2.9894 | LR: 2.18e-04 | 711.5ms/step | 28.1 samples/s +Step 24160/44000 | Loss: 2.9390 | LR: 2.18e-04 | 711.1ms/step | 28.1 samples/s +Step 24170/44000 | Loss: 3.0473 | LR: 2.17e-04 | 709.6ms/step | 28.2 samples/s +Step 24180/44000 | Loss: 2.9412 | LR: 2.17e-04 | 709.7ms/step | 28.2 samples/s +Step 24190/44000 | Loss: 2.9634 | LR: 2.17e-04 | 710.5ms/step | 28.2 samples/s +Step 24200/44000 | Loss: 2.9292 | LR: 2.17e-04 | 711.4ms/step | 28.1 samples/s +Step 24210/44000 | Loss: 2.9844 | LR: 2.17e-04 | 726.6ms/step | 27.5 samples/s +Step 24220/44000 | Loss: 2.9438 | LR: 2.17e-04 | 714.1ms/step | 28.0 samples/s +Step 24230/44000 | Loss: 2.9873 | LR: 2.16e-04 | 712.7ms/step | 28.1 samples/s +Step 24240/44000 | Loss: 2.9553 | LR: 2.16e-04 | 710.5ms/step | 28.2 samples/s +Step 24250/44000 | Loss: 2.9456 | LR: 2.16e-04 | 710.7ms/step | 28.1 samples/s +Validation at step 24250: Loss = 2.9437, Perplexity = 18.99 +Step 24260/44000 | Loss: 2.9594 | LR: 2.16e-04 | 931.0ms/step | 21.5 samples/s +Step 24270/44000 | Loss: 2.9624 | LR: 2.16e-04 | 711.0ms/step | 28.1 samples/s +Step 24280/44000 | Loss: 2.9684 | LR: 2.16e-04 | 710.3ms/step | 28.2 samples/s +Step 24290/44000 | Loss: 2.9708 | LR: 2.15e-04 | 711.0ms/step | 28.1 samples/s +Step 24300/44000 | Loss: 2.9785 | LR: 2.15e-04 | 728.0ms/step | 27.5 samples/s +Step 24310/44000 | Loss: 3.0268 | LR: 2.15e-04 | 710.0ms/step | 28.2 samples/s +Step 24320/44000 | Loss: 2.9959 | LR: 2.15e-04 | 710.3ms/step | 28.2 samples/s +Step 24330/44000 | Loss: 2.9358 | LR: 2.15e-04 | 710.2ms/step | 28.2 samples/s +Step 24340/44000 | Loss: 2.9808 | LR: 2.14e-04 | 709.8ms/step | 28.2 samples/s +Step 24350/44000 | Loss: 2.9641 | LR: 2.14e-04 | 710.7ms/step | 28.1 samples/s +Step 24360/44000 | Loss: 2.9424 | LR: 2.14e-04 | 711.9ms/step | 28.1 samples/s +Step 24370/44000 | Loss: 2.9858 | LR: 2.14e-04 | 714.3ms/step | 28.0 samples/s +Step 24380/44000 | Loss: 2.9113 | LR: 2.14e-04 | 726.1ms/step | 27.5 samples/s +Step 24390/44000 | Loss: 2.9236 | LR: 2.14e-04 | 719.1ms/step | 27.8 samples/s +Step 24400/44000 | Loss: 3.0284 | LR: 2.13e-04 | 714.5ms/step | 28.0 samples/s +Step 24410/44000 | Loss: 2.9345 | LR: 2.13e-04 | 711.7ms/step | 28.1 samples/s +Step 24420/44000 | Loss: 2.9449 | LR: 2.13e-04 | 710.0ms/step | 28.2 samples/s +Step 24430/44000 | Loss: 3.0203 | LR: 2.13e-04 | 710.7ms/step | 28.1 samples/s +Step 24440/44000 | Loss: 2.9289 | LR: 2.13e-04 | 711.7ms/step | 28.1 samples/s +Step 24450/44000 | Loss: 2.9527 | LR: 2.12e-04 | 710.7ms/step | 28.1 samples/s +Step 24460/44000 | Loss: 2.9494 | LR: 2.12e-04 | 710.2ms/step | 28.2 samples/s +Step 24470/44000 | Loss: 2.9262 | LR: 2.12e-04 | 722.7ms/step | 27.7 samples/s +Step 24480/44000 | Loss: 2.9606 | LR: 2.12e-04 | 714.7ms/step | 28.0 samples/s +Step 24490/44000 | Loss: 2.9220 | LR: 2.12e-04 | 710.8ms/step | 28.1 samples/s +Step 24500/44000 | Loss: 3.0228 | LR: 2.12e-04 | 710.5ms/step | 28.2 samples/s +Validation at step 24500: Loss = 2.9425, Perplexity = 18.96 +Step 24510/44000 | Loss: 2.9016 | LR: 2.11e-04 | 928.9ms/step | 21.5 samples/s +Step 24520/44000 | Loss: 3.0020 | LR: 2.11e-04 | 711.3ms/step | 28.1 samples/s +Step 24530/44000 | Loss: 2.9430 | LR: 2.11e-04 | 710.3ms/step | 28.2 samples/s +Step 24540/44000 | Loss: 2.9419 | LR: 2.11e-04 | 710.5ms/step | 28.1 samples/s +Step 24550/44000 | Loss: 2.9922 | LR: 2.11e-04 | 710.3ms/step | 28.2 samples/s +Step 24560/44000 | Loss: 2.8968 | LR: 2.10e-04 | 727.2ms/step | 27.5 samples/s +Step 24570/44000 | Loss: 2.9413 | LR: 2.10e-04 | 711.1ms/step | 28.1 samples/s +Step 24580/44000 | Loss: 3.0043 | LR: 2.10e-04 | 710.6ms/step | 28.1 samples/s +Step 24590/44000 | Loss: 2.9541 | LR: 2.10e-04 | 710.7ms/step | 28.1 samples/s +Step 24600/44000 | Loss: 2.9601 | LR: 2.10e-04 | 710.5ms/step | 28.1 samples/s +Step 24610/44000 | Loss: 2.9336 | LR: 2.10e-04 | 711.6ms/step | 28.1 samples/s +Step 24620/44000 | Loss: 2.9583 | LR: 2.09e-04 | 711.3ms/step | 28.1 samples/s +Step 24630/44000 | Loss: 2.9572 | LR: 2.09e-04 | 711.3ms/step | 28.1 samples/s +Step 24640/44000 | Loss: 2.9766 | LR: 2.09e-04 | 711.6ms/step | 28.1 samples/s +Step 24650/44000 | Loss: 3.0263 | LR: 2.09e-04 | 727.2ms/step | 27.5 samples/s +Step 24660/44000 | Loss: 2.9424 | LR: 2.09e-04 | 712.0ms/step | 28.1 samples/s +Step 24670/44000 | Loss: 2.9681 | LR: 2.09e-04 | 711.3ms/step | 28.1 samples/s +Step 24680/44000 | Loss: 3.0084 | LR: 2.08e-04 | 711.6ms/step | 28.1 samples/s +Step 24690/44000 | Loss: 2.9195 | LR: 2.08e-04 | 711.1ms/step | 28.1 samples/s +Step 24700/44000 | Loss: 2.9019 | LR: 2.08e-04 | 711.7ms/step | 28.1 samples/s +Step 24710/44000 | Loss: 2.9287 | LR: 2.08e-04 | 709.8ms/step | 28.2 samples/s +Step 24720/44000 | Loss: 2.9579 | LR: 2.08e-04 | 710.8ms/step | 28.1 samples/s +Step 24730/44000 | Loss: 3.0130 | LR: 2.07e-04 | 710.6ms/step | 28.1 samples/s +Step 24740/44000 | Loss: 3.0224 | LR: 2.07e-04 | 726.9ms/step | 27.5 samples/s +Step 24750/44000 | Loss: 2.9459 | LR: 2.07e-04 | 710.3ms/step | 28.2 samples/s +Validation at step 24750: Loss = 2.9410, Perplexity = 18.93 +Step 24760/44000 | Loss: 2.9227 | LR: 2.07e-04 | 930.5ms/step | 21.5 samples/s +Step 24770/44000 | Loss: 2.9507 | LR: 2.07e-04 | 710.2ms/step | 28.2 samples/s +Step 24780/44000 | Loss: 2.9115 | LR: 2.07e-04 | 709.8ms/step | 28.2 samples/s +Step 24790/44000 | Loss: 2.9278 | LR: 2.06e-04 | 710.1ms/step | 28.2 samples/s +Step 24800/44000 | Loss: 2.9368 | LR: 2.06e-04 | 709.9ms/step | 28.2 samples/s +Step 24810/44000 | Loss: 2.9581 | LR: 2.06e-04 | 710.1ms/step | 28.2 samples/s +Step 24820/44000 | Loss: 2.9660 | LR: 2.06e-04 | 722.2ms/step | 27.7 samples/s +Step 24830/44000 | Loss: 2.9941 | LR: 2.06e-04 | 714.9ms/step | 28.0 samples/s +Step 24840/44000 | Loss: 2.9205 | LR: 2.05e-04 | 710.2ms/step | 28.2 samples/s +Step 24850/44000 | Loss: 2.9409 | LR: 2.05e-04 | 710.8ms/step | 28.1 samples/s +Step 24860/44000 | Loss: 2.9445 | LR: 2.05e-04 | 709.7ms/step | 28.2 samples/s +Step 24870/44000 | Loss: 2.9420 | LR: 2.05e-04 | 710.4ms/step | 28.2 samples/s +Step 24880/44000 | Loss: 3.0009 | LR: 2.05e-04 | 710.2ms/step | 28.2 samples/s +Step 24890/44000 | Loss: 2.9290 | LR: 2.05e-04 | 709.9ms/step | 28.2 samples/s +Step 24900/44000 | Loss: 3.0056 | LR: 2.04e-04 | 710.0ms/step | 28.2 samples/s +Step 24910/44000 | Loss: 3.0007 | LR: 2.04e-04 | 719.6ms/step | 27.8 samples/s +Step 24920/44000 | Loss: 3.0045 | LR: 2.04e-04 | 717.7ms/step | 27.9 samples/s +Step 24930/44000 | Loss: 2.9304 | LR: 2.04e-04 | 709.9ms/step | 28.2 samples/s +Step 24940/44000 | Loss: 2.9983 | LR: 2.04e-04 | 709.9ms/step | 28.2 samples/s +Step 24950/44000 | Loss: 2.9023 | LR: 2.04e-04 | 710.3ms/step | 28.2 samples/s +Step 24960/44000 | Loss: 2.9659 | LR: 2.03e-04 | 710.6ms/step | 28.1 samples/s +Step 24970/44000 | Loss: 2.9527 | LR: 2.03e-04 | 709.9ms/step | 28.2 samples/s +Step 24980/44000 | Loss: 2.9309 | LR: 2.03e-04 | 710.7ms/step | 28.1 samples/s +Step 24990/44000 | Loss: 2.9164 | LR: 2.03e-04 | 710.1ms/step | 28.2 samples/s +Step 25000/44000 | Loss: 2.8638 | LR: 2.03e-04 | 723.6ms/step | 27.6 samples/s +Validation at step 25000: Loss = 2.9393, Perplexity = 18.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-25000 +Step 25010/44000 | Loss: 2.9602 | LR: 2.02e-04 | 1255.6ms/step | 15.9 samples/s +Step 25020/44000 | Loss: 3.0612 | LR: 2.02e-04 | 710.0ms/step | 28.2 samples/s +Step 25030/44000 | Loss: 2.8890 | LR: 2.02e-04 | 709.8ms/step | 28.2 samples/s +Step 25040/44000 | Loss: 2.9164 | LR: 2.02e-04 | 709.5ms/step | 28.2 samples/s +Step 25050/44000 | Loss: 2.9650 | LR: 2.02e-04 | 712.7ms/step | 28.1 samples/s +Step 25060/44000 | Loss: 3.0117 | LR: 2.02e-04 | 711.9ms/step | 28.1 samples/s +Step 25070/44000 | Loss: 2.9726 | LR: 2.01e-04 | 712.4ms/step | 28.1 samples/s +Step 25080/44000 | Loss: 2.9468 | LR: 2.01e-04 | 723.0ms/step | 27.7 samples/s +Step 25090/44000 | Loss: 2.9561 | LR: 2.01e-04 | 718.3ms/step | 27.8 samples/s +Step 25100/44000 | Loss: 2.9622 | LR: 2.01e-04 | 710.6ms/step | 28.1 samples/s +Step 25110/44000 | Loss: 2.9357 | LR: 2.01e-04 | 710.9ms/step | 28.1 samples/s +Step 25120/44000 | Loss: 2.9875 | LR: 2.00e-04 | 710.4ms/step | 28.2 samples/s +Step 25130/44000 | Loss: 2.8901 | LR: 2.00e-04 | 711.3ms/step | 28.1 samples/s +Step 25140/44000 | Loss: 2.9683 | LR: 2.00e-04 | 710.5ms/step | 28.2 samples/s +Step 25150/44000 | Loss: 2.8995 | LR: 2.00e-04 | 711.1ms/step | 28.1 samples/s +Step 25160/44000 | Loss: 2.8802 | LR: 2.00e-04 | 711.3ms/step | 28.1 samples/s +Step 25170/44000 | Loss: 3.0237 | LR: 2.00e-04 | 720.9ms/step | 27.7 samples/s +Step 25180/44000 | Loss: 2.9969 | LR: 1.99e-04 | 716.7ms/step | 27.9 samples/s +Step 25190/44000 | Loss: 2.9339 | LR: 1.99e-04 | 712.3ms/step | 28.1 samples/s +Step 25200/44000 | Loss: 2.9410 | LR: 1.99e-04 | 711.7ms/step | 28.1 samples/s +Step 25210/44000 | Loss: 2.9579 | LR: 1.99e-04 | 711.9ms/step | 28.1 samples/s +Step 25220/44000 | Loss: 2.9248 | LR: 1.99e-04 | 711.5ms/step | 28.1 samples/s +Step 25230/44000 | Loss: 2.9712 | LR: 1.99e-04 | 712.2ms/step | 28.1 samples/s +Step 25240/44000 | Loss: 2.9789 | LR: 1.98e-04 | 713.0ms/step | 28.0 samples/s +Step 25250/44000 | Loss: 2.9286 | LR: 1.98e-04 | 713.0ms/step | 28.0 samples/s +Validation at step 25250: Loss = 2.9373, Perplexity = 18.86 +Step 25260/44000 | Loss: 2.9444 | LR: 1.98e-04 | 947.8ms/step | 21.1 samples/s +Step 25270/44000 | Loss: 2.9533 | LR: 1.98e-04 | 711.5ms/step | 28.1 samples/s +Step 25280/44000 | Loss: 2.8614 | LR: 1.98e-04 | 711.7ms/step | 28.1 samples/s +Step 25290/44000 | Loss: 2.9714 | LR: 1.97e-04 | 710.9ms/step | 28.1 samples/s +Step 25300/44000 | Loss: 2.9795 | LR: 1.97e-04 | 710.8ms/step | 28.1 samples/s +Step 25310/44000 | Loss: 2.9302 | LR: 1.97e-04 | 711.3ms/step | 28.1 samples/s +Step 25320/44000 | Loss: 2.9219 | LR: 1.97e-04 | 710.8ms/step | 28.1 samples/s +Step 25330/44000 | Loss: 2.9195 | LR: 1.97e-04 | 711.4ms/step | 28.1 samples/s +Step 25340/44000 | Loss: 2.9638 | LR: 1.97e-04 | 710.5ms/step | 28.1 samples/s +Step 25350/44000 | Loss: 2.9352 | LR: 1.96e-04 | 727.8ms/step | 27.5 samples/s +Step 25360/44000 | Loss: 2.9579 | LR: 1.96e-04 | 711.5ms/step | 28.1 samples/s +Step 25370/44000 | Loss: 2.9327 | LR: 1.96e-04 | 711.8ms/step | 28.1 samples/s +Step 25380/44000 | Loss: 2.9586 | LR: 1.96e-04 | 711.8ms/step | 28.1 samples/s +Step 25390/44000 | Loss: 2.9326 | LR: 1.96e-04 | 711.8ms/step | 28.1 samples/s +Step 25400/44000 | Loss: 3.0056 | LR: 1.96e-04 | 712.2ms/step | 28.1 samples/s +Step 25410/44000 | Loss: 2.8995 | LR: 1.95e-04 | 712.0ms/step | 28.1 samples/s +Step 25420/44000 | Loss: 2.8731 | LR: 1.95e-04 | 712.4ms/step | 28.1 samples/s +Step 25430/44000 | Loss: 2.9468 | LR: 1.95e-04 | 712.4ms/step | 28.1 samples/s +Step 25440/44000 | Loss: 2.9883 | LR: 1.95e-04 | 728.4ms/step | 27.5 samples/s +Step 25450/44000 | Loss: 2.9277 | LR: 1.95e-04 | 711.4ms/step | 28.1 samples/s +Step 25460/44000 | Loss: 3.0082 | LR: 1.94e-04 | 711.4ms/step | 28.1 samples/s +Step 25470/44000 | Loss: 3.0055 | LR: 1.94e-04 | 711.7ms/step | 28.1 samples/s +Step 25480/44000 | Loss: 2.9198 | LR: 1.94e-04 | 714.9ms/step | 28.0 samples/s +Step 25490/44000 | Loss: 2.9769 | LR: 1.94e-04 | 712.4ms/step | 28.1 samples/s +Step 25500/44000 | Loss: 3.0578 | LR: 1.94e-04 | 711.3ms/step | 28.1 samples/s +Validation at step 25500: Loss = 2.9358, Perplexity = 18.84 +Step 25510/44000 | Loss: 2.9232 | LR: 1.94e-04 | 930.3ms/step | 21.5 samples/s +Step 25520/44000 | Loss: 2.9360 | LR: 1.93e-04 | 726.6ms/step | 27.5 samples/s +Step 25530/44000 | Loss: 2.9654 | LR: 1.93e-04 | 712.1ms/step | 28.1 samples/s +Step 25540/44000 | Loss: 2.9783 | LR: 1.93e-04 | 712.8ms/step | 28.1 samples/s +Step 25550/44000 | Loss: 2.8839 | LR: 1.93e-04 | 711.7ms/step | 28.1 samples/s +Step 25560/44000 | Loss: 2.8754 | LR: 1.93e-04 | 711.6ms/step | 28.1 samples/s +Step 25570/44000 | Loss: 2.8781 | LR: 1.92e-04 | 713.3ms/step | 28.0 samples/s +Step 25580/44000 | Loss: 2.9404 | LR: 1.92e-04 | 713.4ms/step | 28.0 samples/s +Step 25590/44000 | Loss: 2.9976 | LR: 1.92e-04 | 711.5ms/step | 28.1 samples/s +Step 25600/44000 | Loss: 2.9693 | LR: 1.92e-04 | 711.4ms/step | 28.1 samples/s +Step 25610/44000 | Loss: 2.9240 | LR: 1.92e-04 | 725.7ms/step | 27.6 samples/s +Step 25620/44000 | Loss: 2.8970 | LR: 1.92e-04 | 714.0ms/step | 28.0 samples/s +Step 25630/44000 | Loss: 2.9839 | LR: 1.91e-04 | 713.2ms/step | 28.0 samples/s +Step 25640/44000 | Loss: 2.8248 | LR: 1.91e-04 | 713.7ms/step | 28.0 samples/s +Step 25650/44000 | Loss: 2.9147 | LR: 1.91e-04 | 712.8ms/step | 28.1 samples/s +Step 25660/44000 | Loss: 2.9942 | LR: 1.91e-04 | 714.6ms/step | 28.0 samples/s +Step 25670/44000 | Loss: 2.9852 | LR: 1.91e-04 | 713.3ms/step | 28.0 samples/s +Step 25680/44000 | Loss: 2.9440 | LR: 1.91e-04 | 712.3ms/step | 28.1 samples/s +Step 25690/44000 | Loss: 2.9755 | LR: 1.90e-04 | 711.8ms/step | 28.1 samples/s +Step 25700/44000 | Loss: 2.9479 | LR: 1.90e-04 | 728.8ms/step | 27.4 samples/s +Step 25710/44000 | Loss: 2.9314 | LR: 1.90e-04 | 711.9ms/step | 28.1 samples/s +Step 25720/44000 | Loss: 2.9888 | LR: 1.90e-04 | 712.1ms/step | 28.1 samples/s +Step 25730/44000 | Loss: 2.9426 | LR: 1.90e-04 | 712.8ms/step | 28.1 samples/s +Step 25740/44000 | Loss: 2.9174 | LR: 1.89e-04 | 712.3ms/step | 28.1 samples/s +Step 25750/44000 | Loss: 2.9418 | LR: 1.89e-04 | 713.2ms/step | 28.0 samples/s +Validation at step 25750: Loss = 2.9330, Perplexity = 18.78 +Step 25760/44000 | Loss: 2.8919 | LR: 1.89e-04 | 935.2ms/step | 21.4 samples/s +Step 25770/44000 | Loss: 2.9172 | LR: 1.89e-04 | 713.1ms/step | 28.0 samples/s +Step 25780/44000 | Loss: 3.0374 | LR: 1.89e-04 | 716.9ms/step | 27.9 samples/s +Step 25790/44000 | Loss: 3.0085 | LR: 1.89e-04 | 720.9ms/step | 27.7 samples/s +Step 25800/44000 | Loss: 2.9666 | LR: 1.88e-04 | 711.9ms/step | 28.1 samples/s +Step 25810/44000 | Loss: 2.9104 | LR: 1.88e-04 | 713.1ms/step | 28.0 samples/s +Step 25820/44000 | Loss: 2.9377 | LR: 1.88e-04 | 714.9ms/step | 28.0 samples/s +Step 25830/44000 | Loss: 2.9529 | LR: 1.88e-04 | 714.6ms/step | 28.0 samples/s +Step 25840/44000 | Loss: 2.9988 | LR: 1.88e-04 | 715.0ms/step | 28.0 samples/s +Step 25850/44000 | Loss: 2.9484 | LR: 1.88e-04 | 713.7ms/step | 28.0 samples/s +Step 25860/44000 | Loss: 2.9414 | LR: 1.87e-04 | 714.9ms/step | 28.0 samples/s +Step 25870/44000 | Loss: 3.0453 | LR: 1.87e-04 | 726.0ms/step | 27.5 samples/s +Step 25880/44000 | Loss: 3.0222 | LR: 1.87e-04 | 718.9ms/step | 27.8 samples/s +Step 25890/44000 | Loss: 3.0057 | LR: 1.87e-04 | 714.3ms/step | 28.0 samples/s +Step 25900/44000 | Loss: 2.9631 | LR: 1.87e-04 | 714.9ms/step | 28.0 samples/s +Step 25910/44000 | Loss: 2.9742 | LR: 1.87e-04 | 711.0ms/step | 28.1 samples/s +Step 25920/44000 | Loss: 2.9672 | LR: 1.86e-04 | 711.0ms/step | 28.1 samples/s +Step 25930/44000 | Loss: 2.9240 | LR: 1.86e-04 | 711.0ms/step | 28.1 samples/s +Step 25940/44000 | Loss: 2.9889 | LR: 1.86e-04 | 711.9ms/step | 28.1 samples/s +Step 25950/44000 | Loss: 2.9527 | LR: 1.86e-04 | 711.0ms/step | 28.1 samples/s +Step 25960/44000 | Loss: 2.9799 | LR: 1.86e-04 | 721.9ms/step | 27.7 samples/s +Step 25970/44000 | Loss: 2.9129 | LR: 1.85e-04 | 715.6ms/step | 27.9 samples/s +Step 25980/44000 | Loss: 2.9265 | LR: 1.85e-04 | 711.1ms/step | 28.1 samples/s +Step 25990/44000 | Loss: 2.9550 | LR: 1.85e-04 | 711.2ms/step | 28.1 samples/s +Step 26000/44000 | Loss: 2.9807 | LR: 1.85e-04 | 710.8ms/step | 28.1 samples/s +Validation at step 26000: Loss = 2.9316, Perplexity = 18.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-26000 +Step 26010/44000 | Loss: 2.8760 | LR: 1.85e-04 | 1269.4ms/step | 15.8 samples/s +Step 26020/44000 | Loss: 2.9900 | LR: 1.85e-04 | 709.7ms/step | 28.2 samples/s +Step 26030/44000 | Loss: 2.8962 | LR: 1.84e-04 | 711.2ms/step | 28.1 samples/s +Step 26040/44000 | Loss: 2.9470 | LR: 1.84e-04 | 721.2ms/step | 27.7 samples/s +Step 26050/44000 | Loss: 2.9447 | LR: 1.84e-04 | 715.0ms/step | 28.0 samples/s +Step 26060/44000 | Loss: 2.9120 | LR: 1.84e-04 | 712.5ms/step | 28.1 samples/s +Step 26070/44000 | Loss: 2.8992 | LR: 1.84e-04 | 711.9ms/step | 28.1 samples/s +Step 26080/44000 | Loss: 2.9699 | LR: 1.84e-04 | 712.5ms/step | 28.1 samples/s +Step 26090/44000 | Loss: 2.9213 | LR: 1.83e-04 | 712.4ms/step | 28.1 samples/s +Step 26100/44000 | Loss: 2.9292 | LR: 1.83e-04 | 711.2ms/step | 28.1 samples/s +Step 26110/44000 | Loss: 2.8526 | LR: 1.83e-04 | 711.8ms/step | 28.1 samples/s +Step 26120/44000 | Loss: 2.9003 | LR: 1.83e-04 | 711.5ms/step | 28.1 samples/s +Step 26130/44000 | Loss: 2.9065 | LR: 1.83e-04 | 724.4ms/step | 27.6 samples/s +Step 26140/44000 | Loss: 2.9658 | LR: 1.82e-04 | 716.3ms/step | 27.9 samples/s +Step 26150/44000 | Loss: 3.0188 | LR: 1.82e-04 | 711.3ms/step | 28.1 samples/s +Step 26160/44000 | Loss: 2.9558 | LR: 1.82e-04 | 710.9ms/step | 28.1 samples/s +Step 26170/44000 | Loss: 2.9784 | LR: 1.82e-04 | 711.1ms/step | 28.1 samples/s +Step 26180/44000 | Loss: 2.9258 | LR: 1.82e-04 | 711.5ms/step | 28.1 samples/s +Step 26190/44000 | Loss: 2.9516 | LR: 1.82e-04 | 711.3ms/step | 28.1 samples/s +Step 26200/44000 | Loss: 2.9539 | LR: 1.81e-04 | 713.4ms/step | 28.0 samples/s +Step 26210/44000 | Loss: 2.9705 | LR: 1.81e-04 | 711.7ms/step | 28.1 samples/s +Step 26220/44000 | Loss: 2.9809 | LR: 1.81e-04 | 728.7ms/step | 27.4 samples/s +Step 26230/44000 | Loss: 2.9600 | LR: 1.81e-04 | 710.6ms/step | 28.1 samples/s +Step 26240/44000 | Loss: 2.9268 | LR: 1.81e-04 | 710.6ms/step | 28.1 samples/s +Step 26250/44000 | Loss: 2.9886 | LR: 1.81e-04 | 710.9ms/step | 28.1 samples/s +Validation at step 26250: Loss = 2.9305, Perplexity = 18.74 +Step 26260/44000 | Loss: 2.9911 | LR: 1.80e-04 | 929.6ms/step | 21.5 samples/s +Step 26270/44000 | Loss: 2.9686 | LR: 1.80e-04 | 709.8ms/step | 28.2 samples/s +Step 26280/44000 | Loss: 2.9608 | LR: 1.80e-04 | 710.0ms/step | 28.2 samples/s +Step 26290/44000 | Loss: 2.9635 | LR: 1.80e-04 | 710.3ms/step | 28.2 samples/s +Step 26300/44000 | Loss: 2.9533 | LR: 1.80e-04 | 710.4ms/step | 28.2 samples/s +Step 26310/44000 | Loss: 2.8929 | LR: 1.80e-04 | 727.3ms/step | 27.5 samples/s +Step 26320/44000 | Loss: 2.8953 | LR: 1.79e-04 | 712.0ms/step | 28.1 samples/s +Step 26330/44000 | Loss: 2.9233 | LR: 1.79e-04 | 712.6ms/step | 28.1 samples/s +Step 26340/44000 | Loss: 2.8843 | LR: 1.79e-04 | 711.3ms/step | 28.1 samples/s +Step 26350/44000 | Loss: 2.9481 | LR: 1.79e-04 | 711.1ms/step | 28.1 samples/s +Step 26360/44000 | Loss: 2.9693 | LR: 1.79e-04 | 710.3ms/step | 28.2 samples/s +Step 26370/44000 | Loss: 2.9573 | LR: 1.78e-04 | 709.4ms/step | 28.2 samples/s +Step 26380/44000 | Loss: 2.9065 | LR: 1.78e-04 | 710.0ms/step | 28.2 samples/s +Step 26390/44000 | Loss: 2.9074 | LR: 1.78e-04 | 710.4ms/step | 28.2 samples/s +Step 26400/44000 | Loss: 2.9829 | LR: 1.78e-04 | 726.0ms/step | 27.5 samples/s +Step 26410/44000 | Loss: 2.9369 | LR: 1.78e-04 | 711.5ms/step | 28.1 samples/s +Step 26420/44000 | Loss: 2.9417 | LR: 1.78e-04 | 711.6ms/step | 28.1 samples/s +Step 26430/44000 | Loss: 2.9550 | LR: 1.77e-04 | 709.6ms/step | 28.2 samples/s +Step 26440/44000 | Loss: 2.9982 | LR: 1.77e-04 | 711.4ms/step | 28.1 samples/s +Step 26450/44000 | Loss: 3.0020 | LR: 1.77e-04 | 711.1ms/step | 28.1 samples/s +Step 26460/44000 | Loss: 2.9542 | LR: 1.77e-04 | 710.9ms/step | 28.1 samples/s +Step 26470/44000 | Loss: 2.8854 | LR: 1.77e-04 | 711.2ms/step | 28.1 samples/s +Step 26480/44000 | Loss: 3.0115 | LR: 1.77e-04 | 711.7ms/step | 28.1 samples/s +Step 26490/44000 | Loss: 2.9249 | LR: 1.76e-04 | 724.8ms/step | 27.6 samples/s +Step 26500/44000 | Loss: 2.9759 | LR: 1.76e-04 | 710.2ms/step | 28.2 samples/s +Validation at step 26500: Loss = 2.9287, Perplexity = 18.70 +Step 26510/44000 | Loss: 2.8904 | LR: 1.76e-04 | 932.3ms/step | 21.5 samples/s +Step 26520/44000 | Loss: 2.8747 | LR: 1.76e-04 | 710.3ms/step | 28.2 samples/s +Step 26530/44000 | Loss: 2.9307 | LR: 1.76e-04 | 710.4ms/step | 28.2 samples/s +Step 26540/44000 | Loss: 2.9083 | LR: 1.76e-04 | 710.9ms/step | 28.1 samples/s +Step 26550/44000 | Loss: 2.9382 | LR: 1.75e-04 | 710.1ms/step | 28.2 samples/s +Step 26560/44000 | Loss: 2.9114 | LR: 1.75e-04 | 710.8ms/step | 28.1 samples/s +Step 26570/44000 | Loss: 2.9314 | LR: 1.75e-04 | 723.4ms/step | 27.6 samples/s +Step 26580/44000 | Loss: 2.9543 | LR: 1.75e-04 | 709.7ms/step | 28.2 samples/s +Step 26590/44000 | Loss: 2.9316 | LR: 1.75e-04 | 710.6ms/step | 28.1 samples/s +Step 26600/44000 | Loss: 2.9459 | LR: 1.74e-04 | 710.5ms/step | 28.1 samples/s +Step 26610/44000 | Loss: 2.9104 | LR: 1.74e-04 | 710.9ms/step | 28.1 samples/s +Step 26620/44000 | Loss: 2.9564 | LR: 1.74e-04 | 709.4ms/step | 28.2 samples/s +Step 26630/44000 | Loss: 2.9475 | LR: 1.74e-04 | 711.0ms/step | 28.1 samples/s +Step 26640/44000 | Loss: 2.9163 | LR: 1.74e-04 | 710.9ms/step | 28.1 samples/s +Step 26650/44000 | Loss: 2.9569 | LR: 1.74e-04 | 710.1ms/step | 28.2 samples/s +Step 26660/44000 | Loss: 2.9618 | LR: 1.73e-04 | 727.1ms/step | 27.5 samples/s +Step 26670/44000 | Loss: 3.0066 | LR: 1.73e-04 | 710.3ms/step | 28.2 samples/s +Step 26680/44000 | Loss: 2.9468 | LR: 1.73e-04 | 711.1ms/step | 28.1 samples/s +Step 26690/44000 | Loss: 3.0255 | LR: 1.73e-04 | 711.1ms/step | 28.1 samples/s +Step 26700/44000 | Loss: 2.9628 | LR: 1.73e-04 | 711.2ms/step | 28.1 samples/s +Step 26710/44000 | Loss: 2.9772 | LR: 1.73e-04 | 710.3ms/step | 28.2 samples/s +Step 26720/44000 | Loss: 2.9430 | LR: 1.72e-04 | 710.3ms/step | 28.2 samples/s +Step 26730/44000 | Loss: 2.8825 | LR: 1.72e-04 | 710.1ms/step | 28.2 samples/s +Step 26740/44000 | Loss: 2.9874 | LR: 1.72e-04 | 709.8ms/step | 28.2 samples/s +Step 26750/44000 | Loss: 2.9809 | LR: 1.72e-04 | 728.3ms/step | 27.5 samples/s +Validation at step 26750: Loss = 2.9276, Perplexity = 18.68 +Step 26760/44000 | Loss: 2.9339 | LR: 1.72e-04 | 929.8ms/step | 21.5 samples/s +Step 26770/44000 | Loss: 2.9495 | LR: 1.72e-04 | 710.7ms/step | 28.1 samples/s +Step 26780/44000 | Loss: 3.0395 | LR: 1.71e-04 | 710.7ms/step | 28.1 samples/s +Step 26790/44000 | Loss: 2.9717 | LR: 1.71e-04 | 710.8ms/step | 28.1 samples/s +Step 26800/44000 | Loss: 2.9577 | LR: 1.71e-04 | 710.6ms/step | 28.1 samples/s +Step 26810/44000 | Loss: 2.8978 | LR: 1.71e-04 | 710.6ms/step | 28.1 samples/s +Step 26820/44000 | Loss: 2.9166 | LR: 1.71e-04 | 709.8ms/step | 28.2 samples/s +Step 26830/44000 | Loss: 2.8919 | LR: 1.71e-04 | 709.5ms/step | 28.2 samples/s +Step 26840/44000 | Loss: 2.8459 | LR: 1.70e-04 | 727.7ms/step | 27.5 samples/s +Step 26850/44000 | Loss: 2.9219 | LR: 1.70e-04 | 710.2ms/step | 28.2 samples/s +Step 26860/44000 | Loss: 2.9136 | LR: 1.70e-04 | 709.7ms/step | 28.2 samples/s +Step 26870/44000 | Loss: 2.9495 | LR: 1.70e-04 | 709.6ms/step | 28.2 samples/s +Step 26880/44000 | Loss: 2.8695 | LR: 1.70e-04 | 709.8ms/step | 28.2 samples/s +Step 26890/44000 | Loss: 2.9627 | LR: 1.69e-04 | 710.3ms/step | 28.2 samples/s +Step 26900/44000 | Loss: 2.9888 | LR: 1.69e-04 | 709.8ms/step | 28.2 samples/s +Step 26910/44000 | Loss: 2.9551 | LR: 1.69e-04 | 709.9ms/step | 28.2 samples/s +Step 26920/44000 | Loss: 2.9566 | LR: 1.69e-04 | 715.4ms/step | 28.0 samples/s +Step 26930/44000 | Loss: 2.9871 | LR: 1.69e-04 | 720.8ms/step | 27.7 samples/s +Step 26940/44000 | Loss: 2.9446 | LR: 1.69e-04 | 709.8ms/step | 28.2 samples/s +Step 26950/44000 | Loss: 2.9340 | LR: 1.68e-04 | 710.8ms/step | 28.1 samples/s +Step 26960/44000 | Loss: 2.9577 | LR: 1.68e-04 | 709.6ms/step | 28.2 samples/s +Step 26970/44000 | Loss: 2.8984 | LR: 1.68e-04 | 709.8ms/step | 28.2 samples/s +Step 26980/44000 | Loss: 2.9695 | LR: 1.68e-04 | 709.8ms/step | 28.2 samples/s +Step 26990/44000 | Loss: 2.9458 | LR: 1.68e-04 | 709.9ms/step | 28.2 samples/s +Step 27000/44000 | Loss: 3.0236 | LR: 1.68e-04 | 710.3ms/step | 28.2 samples/s +Validation at step 27000: Loss = 2.9265, Perplexity = 18.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-27000 +Step 27010/44000 | Loss: 2.9506 | LR: 1.67e-04 | 1267.0ms/step | 15.8 samples/s +Step 27020/44000 | Loss: 2.9274 | LR: 1.67e-04 | 710.0ms/step | 28.2 samples/s +Step 27030/44000 | Loss: 2.9508 | LR: 1.67e-04 | 710.4ms/step | 28.2 samples/s +Step 27040/44000 | Loss: 2.9482 | LR: 1.67e-04 | 709.5ms/step | 28.2 samples/s +Step 27050/44000 | Loss: 2.9150 | LR: 1.67e-04 | 710.3ms/step | 28.2 samples/s +Step 27060/44000 | Loss: 2.8685 | LR: 1.67e-04 | 709.6ms/step | 28.2 samples/s +Step 27070/44000 | Loss: 2.8885 | LR: 1.66e-04 | 710.8ms/step | 28.1 samples/s +Step 27080/44000 | Loss: 2.9348 | LR: 1.66e-04 | 709.5ms/step | 28.2 samples/s +Step 27090/44000 | Loss: 2.9974 | LR: 1.66e-04 | 711.6ms/step | 28.1 samples/s +Step 27100/44000 | Loss: 3.0058 | LR: 1.66e-04 | 726.2ms/step | 27.5 samples/s +Step 27110/44000 | Loss: 2.9689 | LR: 1.66e-04 | 709.2ms/step | 28.2 samples/s +Step 27120/44000 | Loss: 2.9497 | LR: 1.66e-04 | 709.6ms/step | 28.2 samples/s +Step 27130/44000 | Loss: 2.9000 | LR: 1.65e-04 | 709.4ms/step | 28.2 samples/s +Step 27140/44000 | Loss: 2.8933 | LR: 1.65e-04 | 711.2ms/step | 28.1 samples/s +Step 27150/44000 | Loss: 2.9265 | LR: 1.65e-04 | 710.2ms/step | 28.2 samples/s +Step 27160/44000 | Loss: 3.0143 | LR: 1.65e-04 | 709.9ms/step | 28.2 samples/s +Step 27170/44000 | Loss: 2.9583 | LR: 1.65e-04 | 710.0ms/step | 28.2 samples/s +Step 27180/44000 | Loss: 2.9485 | LR: 1.65e-04 | 717.2ms/step | 27.9 samples/s +Step 27190/44000 | Loss: 3.0021 | LR: 1.64e-04 | 722.2ms/step | 27.7 samples/s +Step 27200/44000 | Loss: 2.8852 | LR: 1.64e-04 | 711.3ms/step | 28.1 samples/s +Step 27210/44000 | Loss: 2.9924 | LR: 1.64e-04 | 711.1ms/step | 28.1 samples/s +Step 27220/44000 | Loss: 2.9299 | LR: 1.64e-04 | 709.5ms/step | 28.2 samples/s +Step 27230/44000 | Loss: 2.9182 | LR: 1.64e-04 | 709.9ms/step | 28.2 samples/s +Step 27240/44000 | Loss: 2.9242 | LR: 1.63e-04 | 710.2ms/step | 28.2 samples/s +Step 27250/44000 | Loss: 2.9381 | LR: 1.63e-04 | 709.9ms/step | 28.2 samples/s +Validation at step 27250: Loss = 2.9247, Perplexity = 18.63 +Step 27260/44000 | Loss: 2.9115 | LR: 1.63e-04 | 930.3ms/step | 21.5 samples/s +Step 27270/44000 | Loss: 2.9609 | LR: 1.63e-04 | 727.7ms/step | 27.5 samples/s +Step 27280/44000 | Loss: 2.9860 | LR: 1.63e-04 | 710.5ms/step | 28.1 samples/s +Step 27290/44000 | Loss: 2.9389 | LR: 1.63e-04 | 709.7ms/step | 28.2 samples/s +Step 27300/44000 | Loss: 3.0295 | LR: 1.62e-04 | 711.7ms/step | 28.1 samples/s +Step 27310/44000 | Loss: 2.9402 | LR: 1.62e-04 | 713.0ms/step | 28.1 samples/s +Step 27320/44000 | Loss: 2.9612 | LR: 1.62e-04 | 712.1ms/step | 28.1 samples/s +Step 27330/44000 | Loss: 2.9784 | LR: 1.62e-04 | 712.2ms/step | 28.1 samples/s +Step 27340/44000 | Loss: 2.9307 | LR: 1.62e-04 | 712.0ms/step | 28.1 samples/s +Step 27350/44000 | Loss: 2.9260 | LR: 1.62e-04 | 711.4ms/step | 28.1 samples/s +Step 27360/44000 | Loss: 2.9631 | LR: 1.61e-04 | 727.3ms/step | 27.5 samples/s +Step 27370/44000 | Loss: 2.9169 | LR: 1.61e-04 | 710.2ms/step | 28.2 samples/s +Step 27380/44000 | Loss: 2.9090 | LR: 1.61e-04 | 710.1ms/step | 28.2 samples/s +Step 27390/44000 | Loss: 2.8895 | LR: 1.61e-04 | 710.5ms/step | 28.1 samples/s +Step 27400/44000 | Loss: 2.9951 | LR: 1.61e-04 | 711.9ms/step | 28.1 samples/s +Step 27410/44000 | Loss: 2.9779 | LR: 1.61e-04 | 710.5ms/step | 28.1 samples/s +Step 27420/44000 | Loss: 2.9684 | LR: 1.60e-04 | 709.7ms/step | 28.2 samples/s +Step 27430/44000 | Loss: 2.9758 | LR: 1.60e-04 | 709.6ms/step | 28.2 samples/s +Step 27440/44000 | Loss: 2.9825 | LR: 1.60e-04 | 709.5ms/step | 28.2 samples/s +Step 27450/44000 | Loss: 2.9698 | LR: 1.60e-04 | 726.6ms/step | 27.5 samples/s +Step 27460/44000 | Loss: 2.9224 | LR: 1.60e-04 | 709.7ms/step | 28.2 samples/s +Step 27470/44000 | Loss: 2.9474 | LR: 1.60e-04 | 710.0ms/step | 28.2 samples/s +Step 27480/44000 | Loss: 2.9364 | LR: 1.59e-04 | 710.5ms/step | 28.1 samples/s +Step 27490/44000 | Loss: 2.9283 | LR: 1.59e-04 | 709.5ms/step | 28.2 samples/s +Step 27500/44000 | Loss: 2.9429 | LR: 1.59e-04 | 709.9ms/step | 28.2 samples/s +Validation at step 27500: Loss = 2.9238, Perplexity = 18.61 +Step 27510/44000 | Loss: 2.9481 | LR: 1.59e-04 | 929.3ms/step | 21.5 samples/s +Step 27520/44000 | Loss: 2.9511 | LR: 1.59e-04 | 710.8ms/step | 28.1 samples/s +Step 27530/44000 | Loss: 2.9761 | LR: 1.59e-04 | 717.9ms/step | 27.9 samples/s +Step 27540/44000 | Loss: 3.0109 | LR: 1.58e-04 | 719.8ms/step | 27.8 samples/s +Step 27550/44000 | Loss: 2.9270 | LR: 1.58e-04 | 711.4ms/step | 28.1 samples/s +Step 27560/44000 | Loss: 2.9349 | LR: 1.58e-04 | 710.7ms/step | 28.1 samples/s +Step 27570/44000 | Loss: 2.8832 | LR: 1.58e-04 | 711.6ms/step | 28.1 samples/s +Step 27580/44000 | Loss: 3.0002 | LR: 1.58e-04 | 710.7ms/step | 28.1 samples/s +Step 27590/44000 | Loss: 2.9481 | LR: 1.58e-04 | 710.1ms/step | 28.2 samples/s +Step 27600/44000 | Loss: 2.9033 | LR: 1.57e-04 | 710.3ms/step | 28.2 samples/s +Step 27610/44000 | Loss: 2.9916 | LR: 1.57e-04 | 710.2ms/step | 28.2 samples/s +Step 27620/44000 | Loss: 2.8612 | LR: 1.57e-04 | 723.0ms/step | 27.7 samples/s +Step 27630/44000 | Loss: 2.9045 | LR: 1.57e-04 | 716.8ms/step | 27.9 samples/s +Step 27640/44000 | Loss: 2.8790 | LR: 1.57e-04 | 711.7ms/step | 28.1 samples/s +Step 27650/44000 | Loss: 2.9161 | LR: 1.57e-04 | 710.9ms/step | 28.1 samples/s +Step 27660/44000 | Loss: 2.9466 | LR: 1.56e-04 | 710.9ms/step | 28.1 samples/s +Step 27670/44000 | Loss: 2.9351 | LR: 1.56e-04 | 710.7ms/step | 28.1 samples/s +Step 27680/44000 | Loss: 2.9219 | LR: 1.56e-04 | 709.7ms/step | 28.2 samples/s +Step 27690/44000 | Loss: 2.9154 | LR: 1.56e-04 | 709.9ms/step | 28.2 samples/s +Step 27700/44000 | Loss: 2.9802 | LR: 1.56e-04 | 709.2ms/step | 28.2 samples/s +Step 27710/44000 | Loss: 2.9371 | LR: 1.56e-04 | 722.1ms/step | 27.7 samples/s +Step 27720/44000 | Loss: 2.8910 | LR: 1.55e-04 | 718.6ms/step | 27.8 samples/s +Step 27730/44000 | Loss: 2.9678 | LR: 1.55e-04 | 710.3ms/step | 28.2 samples/s +Step 27740/44000 | Loss: 2.9791 | LR: 1.55e-04 | 710.2ms/step | 28.2 samples/s +Step 27750/44000 | Loss: 2.9340 | LR: 1.55e-04 | 709.7ms/step | 28.2 samples/s +Validation at step 27750: Loss = 2.9228, Perplexity = 18.59 +Step 27760/44000 | Loss: 2.9386 | LR: 1.55e-04 | 931.1ms/step | 21.5 samples/s +Step 27770/44000 | Loss: 2.9695 | LR: 1.55e-04 | 712.7ms/step | 28.1 samples/s +Step 27780/44000 | Loss: 2.9061 | LR: 1.54e-04 | 712.1ms/step | 28.1 samples/s +Step 27790/44000 | Loss: 2.9340 | LR: 1.54e-04 | 712.6ms/step | 28.1 samples/s +Step 27800/44000 | Loss: 2.9824 | LR: 1.54e-04 | 729.0ms/step | 27.4 samples/s +Step 27810/44000 | Loss: 3.0084 | LR: 1.54e-04 | 712.5ms/step | 28.1 samples/s +Step 27820/44000 | Loss: 2.9698 | LR: 1.54e-04 | 710.7ms/step | 28.1 samples/s +Step 27830/44000 | Loss: 2.9178 | LR: 1.54e-04 | 711.2ms/step | 28.1 samples/s +Step 27840/44000 | Loss: 2.9381 | LR: 1.53e-04 | 711.4ms/step | 28.1 samples/s +Step 27850/44000 | Loss: 2.9372 | LR: 1.53e-04 | 712.1ms/step | 28.1 samples/s +Step 27860/44000 | Loss: 2.9459 | LR: 1.53e-04 | 712.6ms/step | 28.1 samples/s +Step 27870/44000 | Loss: 2.9145 | LR: 1.53e-04 | 709.8ms/step | 28.2 samples/s +Step 27880/44000 | Loss: 2.8777 | LR: 1.53e-04 | 710.0ms/step | 28.2 samples/s +Step 27890/44000 | Loss: 2.9869 | LR: 1.53e-04 | 727.6ms/step | 27.5 samples/s +Step 27900/44000 | Loss: 2.9074 | LR: 1.52e-04 | 710.2ms/step | 28.2 samples/s +Step 27910/44000 | Loss: 2.9879 | LR: 1.52e-04 | 711.2ms/step | 28.1 samples/s +Step 27920/44000 | Loss: 2.9472 | LR: 1.52e-04 | 710.8ms/step | 28.1 samples/s +Step 27930/44000 | Loss: 2.9752 | LR: 1.52e-04 | 710.0ms/step | 28.2 samples/s +Step 27940/44000 | Loss: 2.9592 | LR: 1.52e-04 | 709.3ms/step | 28.2 samples/s +Step 27950/44000 | Loss: 2.9207 | LR: 1.52e-04 | 709.4ms/step | 28.2 samples/s +Step 27960/44000 | Loss: 2.9421 | LR: 1.51e-04 | 709.9ms/step | 28.2 samples/s +Step 27970/44000 | Loss: 2.9598 | LR: 1.51e-04 | 708.2ms/step | 28.2 samples/s +Step 27980/44000 | Loss: 2.9204 | LR: 1.51e-04 | 727.2ms/step | 27.5 samples/s +Step 27990/44000 | Loss: 2.9721 | LR: 1.51e-04 | 711.0ms/step | 28.1 samples/s +Step 28000/44000 | Loss: 2.9218 | LR: 1.51e-04 | 709.4ms/step | 28.2 samples/s +Validation at step 28000: Loss = 2.9215, Perplexity = 18.57 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-28000 +Step 28010/44000 | Loss: 2.8927 | LR: 1.51e-04 | 1261.4ms/step | 15.9 samples/s +Step 28020/44000 | Loss: 2.9846 | LR: 1.50e-04 | 710.0ms/step | 28.2 samples/s +Step 28030/44000 | Loss: 2.9616 | LR: 1.50e-04 | 711.6ms/step | 28.1 samples/s +Step 28040/44000 | Loss: 2.9307 | LR: 1.50e-04 | 711.3ms/step | 28.1 samples/s +Step 28050/44000 | Loss: 2.9172 | LR: 1.50e-04 | 711.1ms/step | 28.1 samples/s +Step 28060/44000 | Loss: 2.9517 | LR: 1.50e-04 | 727.1ms/step | 27.5 samples/s +Step 28070/44000 | Loss: 2.9775 | LR: 1.50e-04 | 710.4ms/step | 28.2 samples/s +Step 28080/44000 | Loss: 2.9467 | LR: 1.49e-04 | 709.5ms/step | 28.2 samples/s +Step 28090/44000 | Loss: 3.0389 | LR: 1.49e-04 | 710.0ms/step | 28.2 samples/s +Step 28100/44000 | Loss: 2.9150 | LR: 1.49e-04 | 711.1ms/step | 28.1 samples/s +Step 28110/44000 | Loss: 2.8737 | LR: 1.49e-04 | 710.4ms/step | 28.2 samples/s +Step 28120/44000 | Loss: 2.8809 | LR: 1.49e-04 | 712.3ms/step | 28.1 samples/s +Step 28130/44000 | Loss: 2.8748 | LR: 1.49e-04 | 711.9ms/step | 28.1 samples/s +Step 28140/44000 | Loss: 2.8656 | LR: 1.48e-04 | 711.5ms/step | 28.1 samples/s +Step 28150/44000 | Loss: 3.0066 | LR: 1.48e-04 | 728.7ms/step | 27.4 samples/s +Step 28160/44000 | Loss: 2.9342 | LR: 1.48e-04 | 712.4ms/step | 28.1 samples/s +Step 28170/44000 | Loss: 2.8997 | LR: 1.48e-04 | 711.4ms/step | 28.1 samples/s +Step 28180/44000 | Loss: 2.9153 | LR: 1.48e-04 | 711.4ms/step | 28.1 samples/s +Step 28190/44000 | Loss: 2.9544 | LR: 1.48e-04 | 711.5ms/step | 28.1 samples/s +Step 28200/44000 | Loss: 2.8999 | LR: 1.47e-04 | 710.8ms/step | 28.1 samples/s +Step 28210/44000 | Loss: 2.9579 | LR: 1.47e-04 | 711.5ms/step | 28.1 samples/s +Step 28220/44000 | Loss: 2.9266 | LR: 1.47e-04 | 710.7ms/step | 28.1 samples/s +Step 28230/44000 | Loss: 2.9066 | LR: 1.47e-04 | 710.9ms/step | 28.1 samples/s +Step 28240/44000 | Loss: 2.9765 | LR: 1.47e-04 | 727.7ms/step | 27.5 samples/s +Step 28250/44000 | Loss: 2.9901 | LR: 1.47e-04 | 712.2ms/step | 28.1 samples/s +Validation at step 28250: Loss = 2.9203, Perplexity = 18.55 +Step 28260/44000 | Loss: 3.0216 | LR: 1.46e-04 | 930.4ms/step | 21.5 samples/s +Step 28270/44000 | Loss: 2.9824 | LR: 1.46e-04 | 711.3ms/step | 28.1 samples/s +Step 28280/44000 | Loss: 2.8955 | LR: 1.46e-04 | 711.6ms/step | 28.1 samples/s +Step 28290/44000 | Loss: 2.9630 | LR: 1.46e-04 | 710.8ms/step | 28.1 samples/s +Step 28300/44000 | Loss: 2.9876 | LR: 1.46e-04 | 710.0ms/step | 28.2 samples/s +Step 28310/44000 | Loss: 2.8426 | LR: 1.46e-04 | 711.3ms/step | 28.1 samples/s +Step 28320/44000 | Loss: 2.9120 | LR: 1.45e-04 | 722.2ms/step | 27.7 samples/s +Step 28330/44000 | Loss: 2.9245 | LR: 1.45e-04 | 716.2ms/step | 27.9 samples/s +Step 28340/44000 | Loss: 2.9638 | LR: 1.45e-04 | 711.5ms/step | 28.1 samples/s +Step 28350/44000 | Loss: 2.9338 | LR: 1.45e-04 | 709.7ms/step | 28.2 samples/s +Step 28360/44000 | Loss: 2.9905 | LR: 1.45e-04 | 711.5ms/step | 28.1 samples/s +Step 28370/44000 | Loss: 2.9606 | LR: 1.45e-04 | 711.6ms/step | 28.1 samples/s +Step 28380/44000 | Loss: 2.9991 | LR: 1.44e-04 | 710.7ms/step | 28.1 samples/s +Step 28390/44000 | Loss: 2.9863 | LR: 1.44e-04 | 709.8ms/step | 28.2 samples/s +Step 28400/44000 | Loss: 3.0297 | LR: 1.44e-04 | 711.2ms/step | 28.1 samples/s +Step 28410/44000 | Loss: 2.9410 | LR: 1.44e-04 | 727.2ms/step | 27.5 samples/s +Step 28420/44000 | Loss: 2.8946 | LR: 1.44e-04 | 712.4ms/step | 28.1 samples/s +Step 28430/44000 | Loss: 2.8989 | LR: 1.44e-04 | 713.0ms/step | 28.1 samples/s +Step 28440/44000 | Loss: 2.9854 | LR: 1.43e-04 | 710.0ms/step | 28.2 samples/s +Step 28450/44000 | Loss: 2.8985 | LR: 1.43e-04 | 709.4ms/step | 28.2 samples/s +Step 28460/44000 | Loss: 2.9740 | LR: 1.43e-04 | 710.8ms/step | 28.1 samples/s +Step 28470/44000 | Loss: 2.9130 | LR: 1.43e-04 | 710.3ms/step | 28.2 samples/s +Step 28480/44000 | Loss: 2.9049 | LR: 1.43e-04 | 710.5ms/step | 28.1 samples/s +Step 28490/44000 | Loss: 2.9665 | LR: 1.43e-04 | 711.0ms/step | 28.1 samples/s +Step 28500/44000 | Loss: 2.9131 | LR: 1.42e-04 | 727.3ms/step | 27.5 samples/s +Validation at step 28500: Loss = 2.9187, Perplexity = 18.52 +Step 28510/44000 | Loss: 2.9341 | LR: 1.42e-04 | 930.5ms/step | 21.5 samples/s +Step 28520/44000 | Loss: 2.9286 | LR: 1.42e-04 | 709.3ms/step | 28.2 samples/s +Step 28530/44000 | Loss: 2.9939 | LR: 1.42e-04 | 711.4ms/step | 28.1 samples/s +Step 28540/44000 | Loss: 2.9444 | LR: 1.42e-04 | 711.6ms/step | 28.1 samples/s +Step 28550/44000 | Loss: 2.9556 | LR: 1.42e-04 | 711.2ms/step | 28.1 samples/s +Step 28560/44000 | Loss: 3.1022 | LR: 1.41e-04 | 711.2ms/step | 28.1 samples/s +Step 28570/44000 | Loss: 2.9174 | LR: 1.41e-04 | 712.1ms/step | 28.1 samples/s +Step 28580/44000 | Loss: 2.9500 | LR: 1.41e-04 | 710.9ms/step | 28.1 samples/s +Step 28590/44000 | Loss: 2.9921 | LR: 1.41e-04 | 726.8ms/step | 27.5 samples/s +Step 28600/44000 | Loss: 2.9793 | LR: 1.41e-04 | 711.6ms/step | 28.1 samples/s +Step 28610/44000 | Loss: 2.9177 | LR: 1.41e-04 | 711.5ms/step | 28.1 samples/s +Step 28620/44000 | Loss: 2.8170 | LR: 1.40e-04 | 711.9ms/step | 28.1 samples/s +Step 28630/44000 | Loss: 3.0206 | LR: 1.40e-04 | 712.6ms/step | 28.1 samples/s +Step 28640/44000 | Loss: 2.9144 | LR: 1.40e-04 | 713.3ms/step | 28.0 samples/s +Step 28650/44000 | Loss: 2.9550 | LR: 1.40e-04 | 712.1ms/step | 28.1 samples/s +Step 28660/44000 | Loss: 2.8548 | LR: 1.40e-04 | 711.9ms/step | 28.1 samples/s +Step 28670/44000 | Loss: 2.9703 | LR: 1.40e-04 | 716.7ms/step | 27.9 samples/s +Step 28680/44000 | Loss: 2.8203 | LR: 1.39e-04 | 722.1ms/step | 27.7 samples/s +Step 28690/44000 | Loss: 2.9079 | LR: 1.39e-04 | 711.7ms/step | 28.1 samples/s +Step 28700/44000 | Loss: 2.9695 | LR: 1.39e-04 | 709.2ms/step | 28.2 samples/s +Step 28710/44000 | Loss: 2.9374 | LR: 1.39e-04 | 711.0ms/step | 28.1 samples/s +Step 28720/44000 | Loss: 2.9358 | LR: 1.39e-04 | 710.6ms/step | 28.1 samples/s +Step 28730/44000 | Loss: 2.9645 | LR: 1.39e-04 | 710.2ms/step | 28.2 samples/s +Step 28740/44000 | Loss: 2.9368 | LR: 1.38e-04 | 713.3ms/step | 28.0 samples/s +Step 28750/44000 | Loss: 2.9182 | LR: 1.38e-04 | 710.2ms/step | 28.2 samples/s +Validation at step 28750: Loss = 2.9180, Perplexity = 18.50 +Step 28760/44000 | Loss: 2.9160 | LR: 1.38e-04 | 943.5ms/step | 21.2 samples/s +Step 28770/44000 | Loss: 2.9338 | LR: 1.38e-04 | 712.1ms/step | 28.1 samples/s +Step 28780/44000 | Loss: 2.9953 | LR: 1.38e-04 | 711.8ms/step | 28.1 samples/s +Step 28790/44000 | Loss: 2.9096 | LR: 1.38e-04 | 711.6ms/step | 28.1 samples/s +Step 28800/44000 | Loss: 2.9665 | LR: 1.38e-04 | 711.3ms/step | 28.1 samples/s +Step 28810/44000 | Loss: 2.8858 | LR: 1.37e-04 | 712.0ms/step | 28.1 samples/s +Step 28820/44000 | Loss: 2.9222 | LR: 1.37e-04 | 711.9ms/step | 28.1 samples/s +Step 28830/44000 | Loss: 2.9473 | LR: 1.37e-04 | 712.5ms/step | 28.1 samples/s +Step 28840/44000 | Loss: 2.9651 | LR: 1.37e-04 | 710.6ms/step | 28.1 samples/s +Step 28850/44000 | Loss: 2.9441 | LR: 1.37e-04 | 727.0ms/step | 27.5 samples/s +Step 28860/44000 | Loss: 2.9187 | LR: 1.37e-04 | 710.2ms/step | 28.2 samples/s +Step 28870/44000 | Loss: 2.9652 | LR: 1.36e-04 | 712.1ms/step | 28.1 samples/s +Step 28880/44000 | Loss: 2.9324 | LR: 1.36e-04 | 712.6ms/step | 28.1 samples/s +Step 28890/44000 | Loss: 2.9009 | LR: 1.36e-04 | 711.2ms/step | 28.1 samples/s +Step 28900/44000 | Loss: 2.9788 | LR: 1.36e-04 | 709.5ms/step | 28.2 samples/s +Step 28910/44000 | Loss: 3.0194 | LR: 1.36e-04 | 710.1ms/step | 28.2 samples/s +Step 28920/44000 | Loss: 2.9032 | LR: 1.36e-04 | 710.0ms/step | 28.2 samples/s +Step 28930/44000 | Loss: 2.9042 | LR: 1.35e-04 | 709.6ms/step | 28.2 samples/s +Step 28940/44000 | Loss: 2.9148 | LR: 1.35e-04 | 726.8ms/step | 27.5 samples/s +Step 28950/44000 | Loss: 2.9828 | LR: 1.35e-04 | 709.7ms/step | 28.2 samples/s +Step 28960/44000 | Loss: 2.9565 | LR: 1.35e-04 | 710.9ms/step | 28.1 samples/s +Step 28970/44000 | Loss: 2.9215 | LR: 1.35e-04 | 710.7ms/step | 28.1 samples/s +Step 28980/44000 | Loss: 2.9141 | LR: 1.35e-04 | 710.4ms/step | 28.2 samples/s +Step 28990/44000 | Loss: 2.9235 | LR: 1.34e-04 | 709.7ms/step | 28.2 samples/s +Step 29000/44000 | Loss: 2.9682 | LR: 1.34e-04 | 709.7ms/step | 28.2 samples/s +Validation at step 29000: Loss = 2.9166, Perplexity = 18.48 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-29000 +Step 29010/44000 | Loss: 2.9131 | LR: 1.34e-04 | 1292.5ms/step | 15.5 samples/s +Step 29020/44000 | Loss: 2.9016 | LR: 1.34e-04 | 724.0ms/step | 27.6 samples/s +Step 29030/44000 | Loss: 2.9107 | LR: 1.34e-04 | 711.1ms/step | 28.1 samples/s +Step 29040/44000 | Loss: 2.9713 | LR: 1.34e-04 | 711.0ms/step | 28.1 samples/s +Step 29050/44000 | Loss: 2.9227 | LR: 1.33e-04 | 710.3ms/step | 28.2 samples/s +Step 29060/44000 | Loss: 2.9247 | LR: 1.33e-04 | 711.2ms/step | 28.1 samples/s +Step 29070/44000 | Loss: 2.9160 | LR: 1.33e-04 | 711.0ms/step | 28.1 samples/s +Step 29080/44000 | Loss: 2.8892 | LR: 1.33e-04 | 711.3ms/step | 28.1 samples/s +Step 29090/44000 | Loss: 2.9654 | LR: 1.33e-04 | 713.8ms/step | 28.0 samples/s +Step 29100/44000 | Loss: 2.9457 | LR: 1.33e-04 | 713.0ms/step | 28.1 samples/s +Step 29110/44000 | Loss: 2.8999 | LR: 1.33e-04 | 724.3ms/step | 27.6 samples/s +Step 29120/44000 | Loss: 2.9505 | LR: 1.32e-04 | 711.9ms/step | 28.1 samples/s +Step 29130/44000 | Loss: 2.8777 | LR: 1.32e-04 | 710.8ms/step | 28.1 samples/s +Step 29140/44000 | Loss: 2.9454 | LR: 1.32e-04 | 710.2ms/step | 28.2 samples/s +Step 29150/44000 | Loss: 2.9400 | LR: 1.32e-04 | 710.6ms/step | 28.1 samples/s +Step 29160/44000 | Loss: 3.0139 | LR: 1.32e-04 | 712.0ms/step | 28.1 samples/s +Step 29170/44000 | Loss: 2.8859 | LR: 1.32e-04 | 710.9ms/step | 28.1 samples/s +Step 29180/44000 | Loss: 2.9097 | LR: 1.31e-04 | 711.2ms/step | 28.1 samples/s +Step 29190/44000 | Loss: 2.8857 | LR: 1.31e-04 | 712.2ms/step | 28.1 samples/s +Step 29200/44000 | Loss: 2.8711 | LR: 1.31e-04 | 725.6ms/step | 27.6 samples/s +Step 29210/44000 | Loss: 2.9090 | LR: 1.31e-04 | 711.6ms/step | 28.1 samples/s +Step 29220/44000 | Loss: 2.9441 | LR: 1.31e-04 | 711.8ms/step | 28.1 samples/s +Step 29230/44000 | Loss: 3.0163 | LR: 1.31e-04 | 711.4ms/step | 28.1 samples/s +Step 29240/44000 | Loss: 2.8902 | LR: 1.30e-04 | 710.7ms/step | 28.1 samples/s +Step 29250/44000 | Loss: 2.9494 | LR: 1.30e-04 | 711.3ms/step | 28.1 samples/s +Validation at step 29250: Loss = 2.9157, Perplexity = 18.46 +Step 29260/44000 | Loss: 2.9442 | LR: 1.30e-04 | 930.9ms/step | 21.5 samples/s +Step 29270/44000 | Loss: 2.9550 | LR: 1.30e-04 | 710.6ms/step | 28.1 samples/s +Step 29280/44000 | Loss: 2.9502 | LR: 1.30e-04 | 719.9ms/step | 27.8 samples/s +Step 29290/44000 | Loss: 2.9258 | LR: 1.30e-04 | 716.0ms/step | 27.9 samples/s +Step 29300/44000 | Loss: 3.0271 | LR: 1.29e-04 | 709.5ms/step | 28.2 samples/s +Step 29310/44000 | Loss: 2.9190 | LR: 1.29e-04 | 711.3ms/step | 28.1 samples/s +Step 29320/44000 | Loss: 2.8879 | LR: 1.29e-04 | 711.6ms/step | 28.1 samples/s +Step 29330/44000 | Loss: 2.8993 | LR: 1.29e-04 | 711.2ms/step | 28.1 samples/s +Step 29340/44000 | Loss: 3.0070 | LR: 1.29e-04 | 712.3ms/step | 28.1 samples/s +Step 29350/44000 | Loss: 2.9633 | LR: 1.29e-04 | 712.3ms/step | 28.1 samples/s +Step 29360/44000 | Loss: 2.9669 | LR: 1.29e-04 | 710.5ms/step | 28.1 samples/s +Step 29370/44000 | Loss: 3.0120 | LR: 1.28e-04 | 718.9ms/step | 27.8 samples/s +Step 29380/44000 | Loss: 2.9933 | LR: 1.28e-04 | 716.5ms/step | 27.9 samples/s +Step 29390/44000 | Loss: 2.9596 | LR: 1.28e-04 | 711.2ms/step | 28.1 samples/s +Step 29400/44000 | Loss: 2.9531 | LR: 1.28e-04 | 710.9ms/step | 28.1 samples/s +Step 29410/44000 | Loss: 3.0113 | LR: 1.28e-04 | 713.4ms/step | 28.0 samples/s +Step 29420/44000 | Loss: 2.9396 | LR: 1.28e-04 | 712.7ms/step | 28.1 samples/s +Step 29430/44000 | Loss: 2.9212 | LR: 1.27e-04 | 713.4ms/step | 28.0 samples/s +Step 29440/44000 | Loss: 2.9154 | LR: 1.27e-04 | 713.4ms/step | 28.0 samples/s +Step 29450/44000 | Loss: 2.9351 | LR: 1.27e-04 | 711.6ms/step | 28.1 samples/s +Step 29460/44000 | Loss: 2.9041 | LR: 1.27e-04 | 729.8ms/step | 27.4 samples/s +Step 29470/44000 | Loss: 2.9270 | LR: 1.27e-04 | 712.2ms/step | 28.1 samples/s +Step 29480/44000 | Loss: 2.8981 | LR: 1.27e-04 | 711.9ms/step | 28.1 samples/s +Step 29490/44000 | Loss: 2.8336 | LR: 1.26e-04 | 712.2ms/step | 28.1 samples/s +Step 29500/44000 | Loss: 2.9886 | LR: 1.26e-04 | 714.3ms/step | 28.0 samples/s +Validation at step 29500: Loss = 2.9147, Perplexity = 18.44 +Step 29510/44000 | Loss: 2.9430 | LR: 1.26e-04 | 930.9ms/step | 21.5 samples/s +Step 29520/44000 | Loss: 2.9018 | LR: 1.26e-04 | 711.8ms/step | 28.1 samples/s +Step 29530/44000 | Loss: 2.9298 | LR: 1.26e-04 | 710.4ms/step | 28.2 samples/s +Step 29540/44000 | Loss: 2.9440 | LR: 1.26e-04 | 711.2ms/step | 28.1 samples/s +Step 29550/44000 | Loss: 2.8948 | LR: 1.26e-04 | 725.8ms/step | 27.6 samples/s +Step 29560/44000 | Loss: 2.9311 | LR: 1.25e-04 | 711.4ms/step | 28.1 samples/s +Step 29570/44000 | Loss: 2.9152 | LR: 1.25e-04 | 714.6ms/step | 28.0 samples/s +Step 29580/44000 | Loss: 2.9155 | LR: 1.25e-04 | 711.4ms/step | 28.1 samples/s +Step 29590/44000 | Loss: 2.8969 | LR: 1.25e-04 | 710.0ms/step | 28.2 samples/s +Step 29600/44000 | Loss: 2.9603 | LR: 1.25e-04 | 710.3ms/step | 28.2 samples/s +Step 29610/44000 | Loss: 2.9373 | LR: 1.25e-04 | 710.0ms/step | 28.2 samples/s +Step 29620/44000 | Loss: 2.9217 | LR: 1.24e-04 | 710.6ms/step | 28.1 samples/s +Step 29630/44000 | Loss: 2.9543 | LR: 1.24e-04 | 711.1ms/step | 28.1 samples/s +Step 29640/44000 | Loss: 3.0245 | LR: 1.24e-04 | 725.7ms/step | 27.6 samples/s +Step 29650/44000 | Loss: 2.9621 | LR: 1.24e-04 | 710.5ms/step | 28.1 samples/s +Step 29660/44000 | Loss: 2.9369 | LR: 1.24e-04 | 711.0ms/step | 28.1 samples/s +Step 29670/44000 | Loss: 2.9775 | LR: 1.24e-04 | 711.7ms/step | 28.1 samples/s +Step 29680/44000 | Loss: 2.9576 | LR: 1.23e-04 | 712.5ms/step | 28.1 samples/s +Step 29690/44000 | Loss: 2.9197 | LR: 1.23e-04 | 711.6ms/step | 28.1 samples/s +Step 29700/44000 | Loss: 2.9436 | LR: 1.23e-04 | 711.3ms/step | 28.1 samples/s +Step 29710/44000 | Loss: 2.9810 | LR: 1.23e-04 | 710.3ms/step | 28.2 samples/s +Step 29720/44000 | Loss: 2.9538 | LR: 1.23e-04 | 710.3ms/step | 28.2 samples/s +Step 29730/44000 | Loss: 2.9435 | LR: 1.23e-04 | 725.3ms/step | 27.6 samples/s +Step 29740/44000 | Loss: 2.9528 | LR: 1.23e-04 | 711.3ms/step | 28.1 samples/s +Step 29750/44000 | Loss: 2.9941 | LR: 1.22e-04 | 711.9ms/step | 28.1 samples/s +Validation at step 29750: Loss = 2.9136, Perplexity = 18.42 +Step 29760/44000 | Loss: 2.9449 | LR: 1.22e-04 | 931.3ms/step | 21.5 samples/s +Step 29770/44000 | Loss: 2.9600 | LR: 1.22e-04 | 711.5ms/step | 28.1 samples/s +Step 29780/44000 | Loss: 2.9274 | LR: 1.22e-04 | 711.0ms/step | 28.1 samples/s +Step 29790/44000 | Loss: 2.9395 | LR: 1.22e-04 | 709.8ms/step | 28.2 samples/s +Step 29800/44000 | Loss: 2.9407 | LR: 1.22e-04 | 709.9ms/step | 28.2 samples/s +Step 29810/44000 | Loss: 3.0261 | LR: 1.21e-04 | 723.9ms/step | 27.6 samples/s +Step 29820/44000 | Loss: 2.9758 | LR: 1.21e-04 | 710.7ms/step | 28.1 samples/s +Step 29830/44000 | Loss: 2.9303 | LR: 1.21e-04 | 710.3ms/step | 28.2 samples/s +Step 29840/44000 | Loss: 2.9553 | LR: 1.21e-04 | 711.4ms/step | 28.1 samples/s +Step 29850/44000 | Loss: 2.9706 | LR: 1.21e-04 | 711.1ms/step | 28.1 samples/s +Step 29860/44000 | Loss: 2.9115 | LR: 1.21e-04 | 711.2ms/step | 28.1 samples/s +Step 29870/44000 | Loss: 2.9451 | LR: 1.21e-04 | 709.9ms/step | 28.2 samples/s +Step 29880/44000 | Loss: 2.9448 | LR: 1.20e-04 | 710.6ms/step | 28.1 samples/s +Step 29890/44000 | Loss: 2.9240 | LR: 1.20e-04 | 709.9ms/step | 28.2 samples/s +Step 29900/44000 | Loss: 2.9386 | LR: 1.20e-04 | 727.0ms/step | 27.5 samples/s +Step 29910/44000 | Loss: 2.8986 | LR: 1.20e-04 | 712.5ms/step | 28.1 samples/s +Step 29920/44000 | Loss: 2.9261 | LR: 1.20e-04 | 713.0ms/step | 28.1 samples/s +Step 29930/44000 | Loss: 2.9248 | LR: 1.20e-04 | 709.8ms/step | 28.2 samples/s +Step 29940/44000 | Loss: 2.8925 | LR: 1.19e-04 | 709.6ms/step | 28.2 samples/s +Step 29950/44000 | Loss: 2.8993 | LR: 1.19e-04 | 710.8ms/step | 28.1 samples/s +Step 29960/44000 | Loss: 2.8902 | LR: 1.19e-04 | 714.1ms/step | 28.0 samples/s +Step 29970/44000 | Loss: 2.8231 | LR: 1.19e-04 | 713.4ms/step | 28.0 samples/s +Step 29980/44000 | Loss: 2.9367 | LR: 1.19e-04 | 713.8ms/step | 28.0 samples/s +Step 29990/44000 | Loss: 2.8976 | LR: 1.19e-04 | 728.2ms/step | 27.5 samples/s +Step 30000/44000 | Loss: 2.9100 | LR: 1.19e-04 | 712.0ms/step | 28.1 samples/s +Validation at step 30000: Loss = 2.9130, Perplexity = 18.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-30000 +Step 30010/44000 | Loss: 2.8581 | LR: 1.18e-04 | 1265.8ms/step | 15.8 samples/s +Step 30020/44000 | Loss: 2.9439 | LR: 1.18e-04 | 709.6ms/step | 28.2 samples/s +Step 30030/44000 | Loss: 2.9662 | LR: 1.18e-04 | 711.6ms/step | 28.1 samples/s +Step 30040/44000 | Loss: 2.8950 | LR: 1.18e-04 | 713.0ms/step | 28.0 samples/s +Step 30050/44000 | Loss: 2.9171 | LR: 1.18e-04 | 713.8ms/step | 28.0 samples/s +Step 30060/44000 | Loss: 2.9428 | LR: 1.18e-04 | 713.9ms/step | 28.0 samples/s +Step 30070/44000 | Loss: 2.9174 | LR: 1.17e-04 | 727.2ms/step | 27.5 samples/s +Step 30080/44000 | Loss: 2.9007 | LR: 1.17e-04 | 713.9ms/step | 28.0 samples/s +Step 30090/44000 | Loss: 2.9550 | LR: 1.17e-04 | 714.0ms/step | 28.0 samples/s +Step 30100/44000 | Loss: 2.9317 | LR: 1.17e-04 | 714.0ms/step | 28.0 samples/s +Step 30110/44000 | Loss: 2.8721 | LR: 1.17e-04 | 714.0ms/step | 28.0 samples/s +Step 30120/44000 | Loss: 2.9217 | LR: 1.17e-04 | 713.0ms/step | 28.0 samples/s +Step 30130/44000 | Loss: 2.9402 | LR: 1.17e-04 | 710.0ms/step | 28.2 samples/s +Step 30140/44000 | Loss: 2.8780 | LR: 1.16e-04 | 710.2ms/step | 28.2 samples/s +Step 30150/44000 | Loss: 2.9703 | LR: 1.16e-04 | 709.1ms/step | 28.2 samples/s +Step 30160/44000 | Loss: 2.9219 | LR: 1.16e-04 | 727.5ms/step | 27.5 samples/s +Step 30170/44000 | Loss: 2.9279 | LR: 1.16e-04 | 710.1ms/step | 28.2 samples/s +Step 30180/44000 | Loss: 2.8564 | LR: 1.16e-04 | 710.5ms/step | 28.1 samples/s +Step 30190/44000 | Loss: 2.9292 | LR: 1.16e-04 | 711.5ms/step | 28.1 samples/s +Step 30200/44000 | Loss: 2.9317 | LR: 1.15e-04 | 709.6ms/step | 28.2 samples/s +Step 30210/44000 | Loss: 2.8221 | LR: 1.15e-04 | 710.1ms/step | 28.2 samples/s +Step 30220/44000 | Loss: 2.9320 | LR: 1.15e-04 | 710.7ms/step | 28.1 samples/s +Step 30230/44000 | Loss: 2.9088 | LR: 1.15e-04 | 710.4ms/step | 28.2 samples/s +Step 30240/44000 | Loss: 2.9139 | LR: 1.15e-04 | 711.9ms/step | 28.1 samples/s +Step 30250/44000 | Loss: 2.9811 | LR: 1.15e-04 | 724.4ms/step | 27.6 samples/s +Validation at step 30250: Loss = 2.9124, Perplexity = 18.40 +Step 30260/44000 | Loss: 2.8912 | LR: 1.15e-04 | 929.9ms/step | 21.5 samples/s +Step 30270/44000 | Loss: 2.9298 | LR: 1.14e-04 | 711.4ms/step | 28.1 samples/s +Step 30280/44000 | Loss: 2.9998 | LR: 1.14e-04 | 711.5ms/step | 28.1 samples/s +Step 30290/44000 | Loss: 2.9504 | LR: 1.14e-04 | 709.6ms/step | 28.2 samples/s +Step 30300/44000 | Loss: 2.9874 | LR: 1.14e-04 | 711.1ms/step | 28.1 samples/s +Step 30310/44000 | Loss: 2.9639 | LR: 1.14e-04 | 711.3ms/step | 28.1 samples/s +Step 30320/44000 | Loss: 2.9580 | LR: 1.14e-04 | 710.8ms/step | 28.1 samples/s +Step 30330/44000 | Loss: 2.9067 | LR: 1.13e-04 | 719.1ms/step | 27.8 samples/s +Step 30340/44000 | Loss: 2.9165 | LR: 1.13e-04 | 715.0ms/step | 28.0 samples/s +Step 30350/44000 | Loss: 2.9583 | LR: 1.13e-04 | 710.2ms/step | 28.2 samples/s +Step 30360/44000 | Loss: 2.8988 | LR: 1.13e-04 | 710.4ms/step | 28.2 samples/s +Step 30370/44000 | Loss: 2.8920 | LR: 1.13e-04 | 711.0ms/step | 28.1 samples/s +Step 30380/44000 | Loss: 2.9415 | LR: 1.13e-04 | 711.5ms/step | 28.1 samples/s +Step 30390/44000 | Loss: 2.9595 | LR: 1.13e-04 | 710.2ms/step | 28.2 samples/s +Step 30400/44000 | Loss: 2.9031 | LR: 1.12e-04 | 710.4ms/step | 28.2 samples/s +Step 30410/44000 | Loss: 2.8951 | LR: 1.12e-04 | 710.5ms/step | 28.1 samples/s +Step 30420/44000 | Loss: 2.9717 | LR: 1.12e-04 | 722.0ms/step | 27.7 samples/s +Step 30430/44000 | Loss: 2.9461 | LR: 1.12e-04 | 713.5ms/step | 28.0 samples/s +Step 30440/44000 | Loss: 2.9331 | LR: 1.12e-04 | 709.2ms/step | 28.2 samples/s +Step 30450/44000 | Loss: 2.9263 | LR: 1.12e-04 | 709.3ms/step | 28.2 samples/s +Step 30460/44000 | Loss: 2.9349 | LR: 1.11e-04 | 710.0ms/step | 28.2 samples/s +Step 30470/44000 | Loss: 2.9085 | LR: 1.11e-04 | 710.0ms/step | 28.2 samples/s +Step 30480/44000 | Loss: 2.9071 | LR: 1.11e-04 | 711.0ms/step | 28.1 samples/s +Step 30490/44000 | Loss: 2.8884 | LR: 1.11e-04 | 710.4ms/step | 28.2 samples/s +Step 30500/44000 | Loss: 2.9513 | LR: 1.11e-04 | 710.9ms/step | 28.1 samples/s +Validation at step 30500: Loss = 2.9112, Perplexity = 18.38 +Step 30510/44000 | Loss: 2.9917 | LR: 1.11e-04 | 946.7ms/step | 21.1 samples/s +Step 30520/44000 | Loss: 2.9128 | LR: 1.11e-04 | 709.9ms/step | 28.2 samples/s +Step 30530/44000 | Loss: 2.9835 | LR: 1.10e-04 | 709.6ms/step | 28.2 samples/s +Step 30540/44000 | Loss: 3.0055 | LR: 1.10e-04 | 709.7ms/step | 28.2 samples/s +Step 30550/44000 | Loss: 2.9137 | LR: 1.10e-04 | 710.2ms/step | 28.2 samples/s +Step 30560/44000 | Loss: 2.9665 | LR: 1.10e-04 | 710.6ms/step | 28.1 samples/s +Step 30570/44000 | Loss: 2.8825 | LR: 1.10e-04 | 712.4ms/step | 28.1 samples/s +Step 30580/44000 | Loss: 2.9109 | LR: 1.10e-04 | 711.0ms/step | 28.1 samples/s +Step 30590/44000 | Loss: 2.9627 | LR: 1.10e-04 | 710.6ms/step | 28.1 samples/s +Step 30600/44000 | Loss: 2.9263 | LR: 1.09e-04 | 727.1ms/step | 27.5 samples/s +Step 30610/44000 | Loss: 2.9159 | LR: 1.09e-04 | 710.8ms/step | 28.1 samples/s +Step 30620/44000 | Loss: 2.8757 | LR: 1.09e-04 | 710.3ms/step | 28.2 samples/s +Step 30630/44000 | Loss: 2.9515 | LR: 1.09e-04 | 710.7ms/step | 28.1 samples/s +Step 30640/44000 | Loss: 2.9288 | LR: 1.09e-04 | 711.2ms/step | 28.1 samples/s +Step 30650/44000 | Loss: 2.9189 | LR: 1.09e-04 | 711.2ms/step | 28.1 samples/s +Step 30660/44000 | Loss: 2.9193 | LR: 1.08e-04 | 710.5ms/step | 28.1 samples/s +Step 30670/44000 | Loss: 2.9123 | LR: 1.08e-04 | 710.6ms/step | 28.1 samples/s +Step 30680/44000 | Loss: 2.9625 | LR: 1.08e-04 | 710.7ms/step | 28.1 samples/s +Step 30690/44000 | Loss: 2.8862 | LR: 1.08e-04 | 729.1ms/step | 27.4 samples/s +Step 30700/44000 | Loss: 2.9357 | LR: 1.08e-04 | 711.3ms/step | 28.1 samples/s +Step 30710/44000 | Loss: 2.8733 | LR: 1.08e-04 | 711.4ms/step | 28.1 samples/s +Step 30720/44000 | Loss: 2.9303 | LR: 1.08e-04 | 710.6ms/step | 28.1 samples/s +Step 30730/44000 | Loss: 3.0777 | LR: 1.07e-04 | 711.3ms/step | 28.1 samples/s +Step 30740/44000 | Loss: 2.9407 | LR: 1.07e-04 | 711.3ms/step | 28.1 samples/s +Step 30750/44000 | Loss: 2.9175 | LR: 1.07e-04 | 711.3ms/step | 28.1 samples/s +Validation at step 30750: Loss = 2.9107, Perplexity = 18.37 +Step 30760/44000 | Loss: 2.9140 | LR: 1.07e-04 | 929.5ms/step | 21.5 samples/s +Step 30770/44000 | Loss: 2.9181 | LR: 1.07e-04 | 723.1ms/step | 27.7 samples/s +Step 30780/44000 | Loss: 2.9005 | LR: 1.07e-04 | 716.5ms/step | 27.9 samples/s +Step 30790/44000 | Loss: 2.9585 | LR: 1.07e-04 | 711.5ms/step | 28.1 samples/s +Step 30800/44000 | Loss: 2.9163 | LR: 1.06e-04 | 711.2ms/step | 28.1 samples/s +Step 30810/44000 | Loss: 2.9329 | LR: 1.06e-04 | 715.2ms/step | 28.0 samples/s +Step 30820/44000 | Loss: 2.9552 | LR: 1.06e-04 | 712.6ms/step | 28.1 samples/s +Step 30830/44000 | Loss: 2.9437 | LR: 1.06e-04 | 712.3ms/step | 28.1 samples/s +Step 30840/44000 | Loss: 2.9539 | LR: 1.06e-04 | 711.7ms/step | 28.1 samples/s +Step 30850/44000 | Loss: 2.8974 | LR: 1.06e-04 | 710.8ms/step | 28.1 samples/s +Step 30860/44000 | Loss: 2.9158 | LR: 1.05e-04 | 721.2ms/step | 27.7 samples/s +Step 30870/44000 | Loss: 2.8880 | LR: 1.05e-04 | 715.5ms/step | 28.0 samples/s +Step 30880/44000 | Loss: 2.9807 | LR: 1.05e-04 | 710.6ms/step | 28.1 samples/s +Step 30890/44000 | Loss: 2.9035 | LR: 1.05e-04 | 710.6ms/step | 28.1 samples/s +Step 30900/44000 | Loss: 2.9852 | LR: 1.05e-04 | 710.7ms/step | 28.1 samples/s +Step 30910/44000 | Loss: 2.9620 | LR: 1.05e-04 | 710.1ms/step | 28.2 samples/s +Step 30920/44000 | Loss: 2.8967 | LR: 1.05e-04 | 710.0ms/step | 28.2 samples/s +Step 30930/44000 | Loss: 2.9391 | LR: 1.04e-04 | 709.6ms/step | 28.2 samples/s +Step 30940/44000 | Loss: 2.9075 | LR: 1.04e-04 | 709.5ms/step | 28.2 samples/s +Step 30950/44000 | Loss: 2.9639 | LR: 1.04e-04 | 725.4ms/step | 27.6 samples/s +Step 30960/44000 | Loss: 2.9137 | LR: 1.04e-04 | 709.6ms/step | 28.2 samples/s +Step 30970/44000 | Loss: 2.8958 | LR: 1.04e-04 | 710.0ms/step | 28.2 samples/s +Step 30980/44000 | Loss: 2.9523 | LR: 1.04e-04 | 710.4ms/step | 28.2 samples/s +Step 30990/44000 | Loss: 2.9167 | LR: 1.04e-04 | 710.8ms/step | 28.1 samples/s +Step 31000/44000 | Loss: 2.8740 | LR: 1.03e-04 | 710.4ms/step | 28.2 samples/s +Validation at step 31000: Loss = 2.9100, Perplexity = 18.36 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-31000 +Step 31010/44000 | Loss: 2.9612 | LR: 1.03e-04 | 1325.5ms/step | 15.1 samples/s +Step 31020/44000 | Loss: 2.9070 | LR: 1.03e-04 | 709.0ms/step | 28.2 samples/s +Step 31030/44000 | Loss: 2.9557 | LR: 1.03e-04 | 721.3ms/step | 27.7 samples/s +Step 31040/44000 | Loss: 2.9050 | LR: 1.03e-04 | 714.2ms/step | 28.0 samples/s +Step 31050/44000 | Loss: 2.9496 | LR: 1.03e-04 | 709.0ms/step | 28.2 samples/s +Step 31060/44000 | Loss: 2.8785 | LR: 1.03e-04 | 709.3ms/step | 28.2 samples/s +Step 31070/44000 | Loss: 2.9550 | LR: 1.02e-04 | 710.2ms/step | 28.2 samples/s +Step 31080/44000 | Loss: 2.8743 | LR: 1.02e-04 | 709.5ms/step | 28.2 samples/s +Step 31090/44000 | Loss: 2.9158 | LR: 1.02e-04 | 710.1ms/step | 28.2 samples/s +Step 31100/44000 | Loss: 2.9117 | LR: 1.02e-04 | 709.7ms/step | 28.2 samples/s +Step 31110/44000 | Loss: 2.9085 | LR: 1.02e-04 | 710.4ms/step | 28.2 samples/s +Step 31120/44000 | Loss: 2.8955 | LR: 1.02e-04 | 725.0ms/step | 27.6 samples/s +Step 31130/44000 | Loss: 2.9581 | LR: 1.02e-04 | 709.3ms/step | 28.2 samples/s +Step 31140/44000 | Loss: 2.9197 | LR: 1.01e-04 | 709.8ms/step | 28.2 samples/s +Step 31150/44000 | Loss: 2.9566 | LR: 1.01e-04 | 712.2ms/step | 28.1 samples/s +Step 31160/44000 | Loss: 2.9618 | LR: 1.01e-04 | 710.6ms/step | 28.1 samples/s +Step 31170/44000 | Loss: 2.8702 | LR: 1.01e-04 | 709.7ms/step | 28.2 samples/s +Step 31180/44000 | Loss: 3.0017 | LR: 1.01e-04 | 709.0ms/step | 28.2 samples/s +Step 31190/44000 | Loss: 2.9053 | LR: 1.01e-04 | 708.8ms/step | 28.2 samples/s +Step 31200/44000 | Loss: 2.9274 | LR: 1.00e-04 | 708.4ms/step | 28.2 samples/s +Step 31210/44000 | Loss: 2.9512 | LR: 1.00e-04 | 723.9ms/step | 27.6 samples/s +Step 31220/44000 | Loss: 2.9121 | LR: 1.00e-04 | 709.9ms/step | 28.2 samples/s +Step 31230/44000 | Loss: 2.9366 | LR: 1.00e-04 | 710.3ms/step | 28.2 samples/s +Step 31240/44000 | Loss: 2.9426 | LR: 9.99e-05 | 709.5ms/step | 28.2 samples/s +Step 31250/44000 | Loss: 2.9388 | LR: 9.98e-05 | 710.0ms/step | 28.2 samples/s +Validation at step 31250: Loss = 2.9093, Perplexity = 18.34 +Step 31260/44000 | Loss: 2.9454 | LR: 9.96e-05 | 927.8ms/step | 21.6 samples/s +Step 31270/44000 | Loss: 2.9811 | LR: 9.95e-05 | 711.0ms/step | 28.1 samples/s +Step 31280/44000 | Loss: 2.8468 | LR: 9.93e-05 | 710.8ms/step | 28.1 samples/s +Step 31290/44000 | Loss: 2.9288 | LR: 9.92e-05 | 711.2ms/step | 28.1 samples/s +Step 31300/44000 | Loss: 2.9438 | LR: 9.90e-05 | 727.6ms/step | 27.5 samples/s +Step 31310/44000 | Loss: 3.0034 | LR: 9.89e-05 | 712.7ms/step | 28.1 samples/s +Step 31320/44000 | Loss: 2.9244 | LR: 9.88e-05 | 710.0ms/step | 28.2 samples/s +Step 31330/44000 | Loss: 2.9743 | LR: 9.86e-05 | 709.8ms/step | 28.2 samples/s +Step 31340/44000 | Loss: 2.9490 | LR: 9.85e-05 | 709.8ms/step | 28.2 samples/s +Step 31350/44000 | Loss: 2.9540 | LR: 9.83e-05 | 709.6ms/step | 28.2 samples/s +Step 31360/44000 | Loss: 2.9269 | LR: 9.82e-05 | 710.0ms/step | 28.2 samples/s +Step 31370/44000 | Loss: 2.9025 | LR: 9.80e-05 | 709.5ms/step | 28.2 samples/s +Step 31380/44000 | Loss: 3.0157 | LR: 9.79e-05 | 709.8ms/step | 28.2 samples/s +Step 31390/44000 | Loss: 2.9096 | LR: 9.77e-05 | 725.4ms/step | 27.6 samples/s +Step 31400/44000 | Loss: 2.9949 | LR: 9.76e-05 | 709.2ms/step | 28.2 samples/s +Step 31410/44000 | Loss: 2.9176 | LR: 9.75e-05 | 709.1ms/step | 28.2 samples/s +Step 31420/44000 | Loss: 2.8942 | LR: 9.73e-05 | 709.4ms/step | 28.2 samples/s +Step 31430/44000 | Loss: 2.9284 | LR: 9.72e-05 | 709.6ms/step | 28.2 samples/s +Step 31440/44000 | Loss: 2.8924 | LR: 9.70e-05 | 710.1ms/step | 28.2 samples/s +Step 31450/44000 | Loss: 2.9743 | LR: 9.69e-05 | 709.2ms/step | 28.2 samples/s +Step 31460/44000 | Loss: 2.9641 | LR: 9.67e-05 | 709.1ms/step | 28.2 samples/s +Step 31470/44000 | Loss: 2.9306 | LR: 9.66e-05 | 714.6ms/step | 28.0 samples/s +Step 31480/44000 | Loss: 2.9261 | LR: 9.65e-05 | 721.9ms/step | 27.7 samples/s +Step 31490/44000 | Loss: 2.9857 | LR: 9.63e-05 | 710.1ms/step | 28.2 samples/s +Step 31500/44000 | Loss: 2.9464 | LR: 9.62e-05 | 710.1ms/step | 28.2 samples/s +Validation at step 31500: Loss = 2.9085, Perplexity = 18.33 +Step 31510/44000 | Loss: 2.9046 | LR: 9.60e-05 | 928.4ms/step | 21.5 samples/s +Step 31520/44000 | Loss: 2.9382 | LR: 9.59e-05 | 709.6ms/step | 28.2 samples/s +Step 31530/44000 | Loss: 2.8877 | LR: 9.57e-05 | 709.6ms/step | 28.2 samples/s +Step 31540/44000 | Loss: 2.9488 | LR: 9.56e-05 | 710.0ms/step | 28.2 samples/s +Step 31550/44000 | Loss: 2.9134 | LR: 9.54e-05 | 710.5ms/step | 28.1 samples/s +Step 31560/44000 | Loss: 3.0020 | LR: 9.53e-05 | 728.2ms/step | 27.5 samples/s +Step 31570/44000 | Loss: 2.8720 | LR: 9.52e-05 | 710.0ms/step | 28.2 samples/s +Step 31580/44000 | Loss: 2.9424 | LR: 9.50e-05 | 709.5ms/step | 28.2 samples/s +Step 31590/44000 | Loss: 2.9280 | LR: 9.49e-05 | 709.6ms/step | 28.2 samples/s +Step 31600/44000 | Loss: 2.9537 | LR: 9.47e-05 | 709.5ms/step | 28.2 samples/s +Step 31610/44000 | Loss: 2.8496 | LR: 9.46e-05 | 709.7ms/step | 28.2 samples/s +Step 31620/44000 | Loss: 2.8877 | LR: 9.45e-05 | 709.7ms/step | 28.2 samples/s +Step 31630/44000 | Loss: 2.9270 | LR: 9.43e-05 | 709.4ms/step | 28.2 samples/s +Step 31640/44000 | Loss: 2.9533 | LR: 9.42e-05 | 709.5ms/step | 28.2 samples/s +Step 31650/44000 | Loss: 2.8895 | LR: 9.40e-05 | 723.3ms/step | 27.6 samples/s +Step 31660/44000 | Loss: 2.9592 | LR: 9.39e-05 | 709.6ms/step | 28.2 samples/s +Step 31670/44000 | Loss: 2.9214 | LR: 9.37e-05 | 709.1ms/step | 28.2 samples/s +Step 31680/44000 | Loss: 2.9358 | LR: 9.36e-05 | 709.6ms/step | 28.2 samples/s +Step 31690/44000 | Loss: 2.8768 | LR: 9.35e-05 | 709.1ms/step | 28.2 samples/s +Step 31700/44000 | Loss: 2.9114 | LR: 9.33e-05 | 709.5ms/step | 28.2 samples/s +Step 31710/44000 | Loss: 2.8850 | LR: 9.32e-05 | 709.7ms/step | 28.2 samples/s +Step 31720/44000 | Loss: 2.9774 | LR: 9.30e-05 | 709.7ms/step | 28.2 samples/s +Step 31730/44000 | Loss: 2.8599 | LR: 9.29e-05 | 709.6ms/step | 28.2 samples/s +Step 31740/44000 | Loss: 2.9531 | LR: 9.28e-05 | 726.2ms/step | 27.5 samples/s +Step 31750/44000 | Loss: 2.9458 | LR: 9.26e-05 | 710.4ms/step | 28.2 samples/s +Validation at step 31750: Loss = 2.9078, Perplexity = 18.32 +Step 31760/44000 | Loss: 2.9314 | LR: 9.25e-05 | 927.2ms/step | 21.6 samples/s +Step 31770/44000 | Loss: 3.0101 | LR: 9.23e-05 | 710.2ms/step | 28.2 samples/s +Step 31780/44000 | Loss: 2.9442 | LR: 9.22e-05 | 709.8ms/step | 28.2 samples/s +Step 31790/44000 | Loss: 2.9095 | LR: 9.20e-05 | 710.4ms/step | 28.2 samples/s +Step 31800/44000 | Loss: 2.8753 | LR: 9.19e-05 | 709.8ms/step | 28.2 samples/s +Step 31810/44000 | Loss: 2.9101 | LR: 9.18e-05 | 710.6ms/step | 28.1 samples/s +Step 31820/44000 | Loss: 2.9447 | LR: 9.16e-05 | 710.4ms/step | 28.2 samples/s +Step 31830/44000 | Loss: 2.9537 | LR: 9.15e-05 | 727.9ms/step | 27.5 samples/s +Step 31840/44000 | Loss: 2.8870 | LR: 9.13e-05 | 709.5ms/step | 28.2 samples/s +Step 31850/44000 | Loss: 2.9356 | LR: 9.12e-05 | 709.6ms/step | 28.2 samples/s +Step 31860/44000 | Loss: 2.9564 | LR: 9.11e-05 | 709.8ms/step | 28.2 samples/s +Step 31870/44000 | Loss: 2.8877 | LR: 9.09e-05 | 709.0ms/step | 28.2 samples/s +Step 31880/44000 | Loss: 2.9247 | LR: 9.08e-05 | 709.8ms/step | 28.2 samples/s +Step 31890/44000 | Loss: 2.9300 | LR: 9.06e-05 | 709.6ms/step | 28.2 samples/s +Step 31900/44000 | Loss: 2.9023 | LR: 9.05e-05 | 709.1ms/step | 28.2 samples/s +Step 31910/44000 | Loss: 3.1000 | LR: 9.04e-05 | 714.2ms/step | 28.0 samples/s +Step 31920/44000 | Loss: 2.9573 | LR: 9.02e-05 | 720.3ms/step | 27.8 samples/s +Step 31930/44000 | Loss: 2.9654 | LR: 9.01e-05 | 711.3ms/step | 28.1 samples/s +Step 31940/44000 | Loss: 2.9087 | LR: 8.99e-05 | 709.5ms/step | 28.2 samples/s +Step 31950/44000 | Loss: 2.9246 | LR: 8.98e-05 | 709.8ms/step | 28.2 samples/s +Step 31960/44000 | Loss: 2.8918 | LR: 8.97e-05 | 709.2ms/step | 28.2 samples/s +Step 31970/44000 | Loss: 2.9094 | LR: 8.95e-05 | 711.4ms/step | 28.1 samples/s +Step 31980/44000 | Loss: 2.9260 | LR: 8.94e-05 | 710.2ms/step | 28.2 samples/s +Step 31990/44000 | Loss: 2.9170 | LR: 8.92e-05 | 710.0ms/step | 28.2 samples/s +Step 32000/44000 | Loss: 2.9181 | LR: 8.91e-05 | 723.2ms/step | 27.7 samples/s +Validation at step 32000: Loss = 2.9071, Perplexity = 18.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-32000 +Step 32010/44000 | Loss: 2.9701 | LR: 8.90e-05 | 1264.8ms/step | 15.8 samples/s +Step 32020/44000 | Loss: 2.8868 | LR: 8.88e-05 | 710.9ms/step | 28.1 samples/s +Step 32030/44000 | Loss: 2.9988 | LR: 8.87e-05 | 711.0ms/step | 28.1 samples/s +Step 32040/44000 | Loss: 2.9941 | LR: 8.86e-05 | 710.1ms/step | 28.2 samples/s +Step 32050/44000 | Loss: 2.9652 | LR: 8.84e-05 | 711.2ms/step | 28.1 samples/s +Step 32060/44000 | Loss: 2.9064 | LR: 8.83e-05 | 711.1ms/step | 28.1 samples/s +Step 32070/44000 | Loss: 2.9142 | LR: 8.81e-05 | 710.4ms/step | 28.2 samples/s +Step 32080/44000 | Loss: 2.8854 | LR: 8.80e-05 | 712.0ms/step | 28.1 samples/s +Step 32090/44000 | Loss: 2.9748 | LR: 8.79e-05 | 729.2ms/step | 27.4 samples/s +Step 32100/44000 | Loss: 2.9251 | LR: 8.77e-05 | 711.4ms/step | 28.1 samples/s +Step 32110/44000 | Loss: 3.0081 | LR: 8.76e-05 | 711.5ms/step | 28.1 samples/s +Step 32120/44000 | Loss: 2.9532 | LR: 8.74e-05 | 711.8ms/step | 28.1 samples/s +Step 32130/44000 | Loss: 2.8419 | LR: 8.73e-05 | 711.2ms/step | 28.1 samples/s +Step 32140/44000 | Loss: 2.9101 | LR: 8.72e-05 | 708.9ms/step | 28.2 samples/s +Step 32150/44000 | Loss: 2.9042 | LR: 8.70e-05 | 709.2ms/step | 28.2 samples/s +Step 32160/44000 | Loss: 2.8787 | LR: 8.69e-05 | 709.2ms/step | 28.2 samples/s +Step 32170/44000 | Loss: 2.8590 | LR: 8.68e-05 | 719.4ms/step | 27.8 samples/s +Step 32180/44000 | Loss: 2.9759 | LR: 8.66e-05 | 714.2ms/step | 28.0 samples/s +Step 32190/44000 | Loss: 2.8565 | LR: 8.65e-05 | 709.1ms/step | 28.2 samples/s +Step 32200/44000 | Loss: 2.8975 | LR: 8.63e-05 | 709.4ms/step | 28.2 samples/s +Step 32210/44000 | Loss: 2.8672 | LR: 8.62e-05 | 710.4ms/step | 28.2 samples/s +Step 32220/44000 | Loss: 2.8736 | LR: 8.61e-05 | 710.8ms/step | 28.1 samples/s +Step 32230/44000 | Loss: 2.8673 | LR: 8.59e-05 | 711.1ms/step | 28.1 samples/s +Step 32240/44000 | Loss: 2.9435 | LR: 8.58e-05 | 711.0ms/step | 28.1 samples/s +Step 32250/44000 | Loss: 2.8967 | LR: 8.57e-05 | 710.6ms/step | 28.1 samples/s +Validation at step 32250: Loss = 2.9066, Perplexity = 18.29 +Step 32260/44000 | Loss: 2.9317 | LR: 8.55e-05 | 946.9ms/step | 21.1 samples/s +Step 32270/44000 | Loss: 2.8260 | LR: 8.54e-05 | 711.1ms/step | 28.1 samples/s +Step 32280/44000 | Loss: 2.9261 | LR: 8.53e-05 | 710.0ms/step | 28.2 samples/s +Step 32290/44000 | Loss: 2.9402 | LR: 8.51e-05 | 709.7ms/step | 28.2 samples/s +Step 32300/44000 | Loss: 2.8866 | LR: 8.50e-05 | 709.3ms/step | 28.2 samples/s +Step 32310/44000 | Loss: 2.9109 | LR: 8.48e-05 | 709.6ms/step | 28.2 samples/s +Step 32320/44000 | Loss: 2.9318 | LR: 8.47e-05 | 710.7ms/step | 28.1 samples/s +Step 32330/44000 | Loss: 2.9188 | LR: 8.46e-05 | 710.7ms/step | 28.1 samples/s +Step 32340/44000 | Loss: 2.9213 | LR: 8.44e-05 | 710.6ms/step | 28.1 samples/s +Step 32350/44000 | Loss: 2.9768 | LR: 8.43e-05 | 726.7ms/step | 27.5 samples/s +Step 32360/44000 | Loss: 2.9293 | LR: 8.42e-05 | 710.3ms/step | 28.2 samples/s +Step 32370/44000 | Loss: 2.9516 | LR: 8.40e-05 | 710.9ms/step | 28.1 samples/s +Step 32380/44000 | Loss: 2.9454 | LR: 8.39e-05 | 710.0ms/step | 28.2 samples/s +Step 32390/44000 | Loss: 2.9285 | LR: 8.38e-05 | 710.3ms/step | 28.2 samples/s +Step 32400/44000 | Loss: 2.9375 | LR: 8.36e-05 | 709.9ms/step | 28.2 samples/s +Step 32410/44000 | Loss: 2.8871 | LR: 8.35e-05 | 709.7ms/step | 28.2 samples/s +Step 32420/44000 | Loss: 2.9059 | LR: 8.33e-05 | 709.6ms/step | 28.2 samples/s +Step 32430/44000 | Loss: 2.9102 | LR: 8.32e-05 | 709.6ms/step | 28.2 samples/s +Step 32440/44000 | Loss: 2.8638 | LR: 8.31e-05 | 725.9ms/step | 27.6 samples/s +Step 32450/44000 | Loss: 2.9488 | LR: 8.29e-05 | 710.2ms/step | 28.2 samples/s +Step 32460/44000 | Loss: 2.9713 | LR: 8.28e-05 | 710.2ms/step | 28.2 samples/s +Step 32470/44000 | Loss: 2.9179 | LR: 8.27e-05 | 709.9ms/step | 28.2 samples/s +Step 32480/44000 | Loss: 2.9355 | LR: 8.25e-05 | 709.7ms/step | 28.2 samples/s +Step 32490/44000 | Loss: 2.9364 | LR: 8.24e-05 | 709.6ms/step | 28.2 samples/s +Step 32500/44000 | Loss: 2.9259 | LR: 8.23e-05 | 709.4ms/step | 28.2 samples/s +Validation at step 32500: Loss = 2.9061, Perplexity = 18.29 +Step 32510/44000 | Loss: 2.9053 | LR: 8.21e-05 | 928.4ms/step | 21.5 samples/s +Step 32520/44000 | Loss: 2.9115 | LR: 8.20e-05 | 719.9ms/step | 27.8 samples/s +Step 32530/44000 | Loss: 2.9663 | LR: 8.19e-05 | 720.7ms/step | 27.8 samples/s +Step 32540/44000 | Loss: 2.9114 | LR: 8.17e-05 | 710.9ms/step | 28.1 samples/s +Step 32550/44000 | Loss: 2.9031 | LR: 8.16e-05 | 711.2ms/step | 28.1 samples/s +Step 32560/44000 | Loss: 2.9882 | LR: 8.15e-05 | 711.4ms/step | 28.1 samples/s +Step 32570/44000 | Loss: 2.8803 | LR: 8.13e-05 | 711.5ms/step | 28.1 samples/s +Step 32580/44000 | Loss: 2.9436 | LR: 8.12e-05 | 711.6ms/step | 28.1 samples/s +Step 32590/44000 | Loss: 2.9186 | LR: 8.11e-05 | 711.1ms/step | 28.1 samples/s +Step 32600/44000 | Loss: 2.8600 | LR: 8.09e-05 | 711.6ms/step | 28.1 samples/s +Step 32610/44000 | Loss: 2.9090 | LR: 8.08e-05 | 723.4ms/step | 27.6 samples/s +Step 32620/44000 | Loss: 2.8940 | LR: 8.07e-05 | 716.0ms/step | 27.9 samples/s +Step 32630/44000 | Loss: 2.8870 | LR: 8.05e-05 | 711.4ms/step | 28.1 samples/s +Step 32640/44000 | Loss: 2.9507 | LR: 8.04e-05 | 711.4ms/step | 28.1 samples/s +Step 32650/44000 | Loss: 2.9593 | LR: 8.03e-05 | 711.0ms/step | 28.1 samples/s +Step 32660/44000 | Loss: 2.8136 | LR: 8.01e-05 | 710.7ms/step | 28.1 samples/s +Step 32670/44000 | Loss: 2.9231 | LR: 8.00e-05 | 711.5ms/step | 28.1 samples/s +Step 32680/44000 | Loss: 2.9148 | LR: 7.99e-05 | 712.7ms/step | 28.1 samples/s +Step 32690/44000 | Loss: 2.9657 | LR: 7.97e-05 | 711.6ms/step | 28.1 samples/s +Step 32700/44000 | Loss: 2.9732 | LR: 7.96e-05 | 722.7ms/step | 27.7 samples/s +Step 32710/44000 | Loss: 2.9769 | LR: 7.95e-05 | 715.6ms/step | 27.9 samples/s +Step 32720/44000 | Loss: 2.9029 | LR: 7.93e-05 | 711.6ms/step | 28.1 samples/s +Step 32730/44000 | Loss: 2.9733 | LR: 7.92e-05 | 711.2ms/step | 28.1 samples/s +Step 32740/44000 | Loss: 2.9128 | LR: 7.91e-05 | 710.7ms/step | 28.1 samples/s +Step 32750/44000 | Loss: 2.8835 | LR: 7.89e-05 | 710.1ms/step | 28.2 samples/s +Validation at step 32750: Loss = 2.9053, Perplexity = 18.27 +Step 32760/44000 | Loss: 2.9091 | LR: 7.88e-05 | 929.0ms/step | 21.5 samples/s +Step 32770/44000 | Loss: 2.8349 | LR: 7.87e-05 | 710.4ms/step | 28.2 samples/s +Step 32780/44000 | Loss: 2.9331 | LR: 7.85e-05 | 710.7ms/step | 28.1 samples/s +Step 32790/44000 | Loss: 2.9433 | LR: 7.84e-05 | 727.3ms/step | 27.5 samples/s +Step 32800/44000 | Loss: 2.9100 | LR: 7.83e-05 | 711.6ms/step | 28.1 samples/s +Step 32810/44000 | Loss: 2.9367 | LR: 7.81e-05 | 712.2ms/step | 28.1 samples/s +Step 32820/44000 | Loss: 2.8739 | LR: 7.80e-05 | 711.7ms/step | 28.1 samples/s +Step 32830/44000 | Loss: 2.9636 | LR: 7.79e-05 | 711.3ms/step | 28.1 samples/s +Step 32840/44000 | Loss: 2.9214 | LR: 7.77e-05 | 711.3ms/step | 28.1 samples/s +Step 32850/44000 | Loss: 2.9137 | LR: 7.76e-05 | 711.0ms/step | 28.1 samples/s +Step 32860/44000 | Loss: 2.9207 | LR: 7.75e-05 | 710.3ms/step | 28.2 samples/s +Step 32870/44000 | Loss: 2.8822 | LR: 7.73e-05 | 710.5ms/step | 28.1 samples/s +Step 32880/44000 | Loss: 2.8905 | LR: 7.72e-05 | 726.2ms/step | 27.5 samples/s +Step 32890/44000 | Loss: 2.9323 | LR: 7.71e-05 | 709.5ms/step | 28.2 samples/s +Step 32900/44000 | Loss: 2.8820 | LR: 7.70e-05 | 709.5ms/step | 28.2 samples/s +Step 32910/44000 | Loss: 2.8330 | LR: 7.68e-05 | 710.1ms/step | 28.2 samples/s +Step 32920/44000 | Loss: 2.9462 | LR: 7.67e-05 | 709.7ms/step | 28.2 samples/s +Step 32930/44000 | Loss: 2.9374 | LR: 7.66e-05 | 711.2ms/step | 28.1 samples/s +Step 32940/44000 | Loss: 2.9530 | LR: 7.64e-05 | 711.6ms/step | 28.1 samples/s +Step 32950/44000 | Loss: 2.8826 | LR: 7.63e-05 | 711.6ms/step | 28.1 samples/s +Step 32960/44000 | Loss: 2.8977 | LR: 7.62e-05 | 711.4ms/step | 28.1 samples/s +Step 32970/44000 | Loss: 2.8491 | LR: 7.60e-05 | 727.4ms/step | 27.5 samples/s +Step 32980/44000 | Loss: 2.8572 | LR: 7.59e-05 | 711.7ms/step | 28.1 samples/s +Step 32990/44000 | Loss: 2.9226 | LR: 7.58e-05 | 710.7ms/step | 28.1 samples/s +Step 33000/44000 | Loss: 2.9328 | LR: 7.56e-05 | 710.2ms/step | 28.2 samples/s +Validation at step 33000: Loss = 2.9050, Perplexity = 18.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-33000 +Step 33010/44000 | Loss: 2.9379 | LR: 7.55e-05 | 1300.4ms/step | 15.4 samples/s +Step 33020/44000 | Loss: 2.9383 | LR: 7.54e-05 | 709.4ms/step | 28.2 samples/s +Step 33030/44000 | Loss: 2.8796 | LR: 7.53e-05 | 710.0ms/step | 28.2 samples/s +Step 33040/44000 | Loss: 2.9146 | LR: 7.51e-05 | 708.9ms/step | 28.2 samples/s +Step 33050/44000 | Loss: 2.9165 | LR: 7.50e-05 | 725.0ms/step | 27.6 samples/s +Step 33060/44000 | Loss: 2.9259 | LR: 7.49e-05 | 709.8ms/step | 28.2 samples/s +Step 33070/44000 | Loss: 2.9676 | LR: 7.47e-05 | 710.7ms/step | 28.1 samples/s +Step 33080/44000 | Loss: 2.9518 | LR: 7.46e-05 | 710.7ms/step | 28.1 samples/s +Step 33090/44000 | Loss: 2.9288 | LR: 7.45e-05 | 710.5ms/step | 28.2 samples/s +Step 33100/44000 | Loss: 2.8912 | LR: 7.44e-05 | 711.3ms/step | 28.1 samples/s +Step 33110/44000 | Loss: 2.9396 | LR: 7.42e-05 | 710.6ms/step | 28.1 samples/s +Step 33120/44000 | Loss: 2.8550 | LR: 7.41e-05 | 710.9ms/step | 28.1 samples/s +Step 33130/44000 | Loss: 2.8799 | LR: 7.40e-05 | 710.4ms/step | 28.2 samples/s +Step 33140/44000 | Loss: 2.9042 | LR: 7.38e-05 | 724.3ms/step | 27.6 samples/s +Step 33150/44000 | Loss: 2.9623 | LR: 7.37e-05 | 711.1ms/step | 28.1 samples/s +Step 33160/44000 | Loss: 2.9365 | LR: 7.36e-05 | 710.3ms/step | 28.2 samples/s +Step 33170/44000 | Loss: 2.8365 | LR: 7.34e-05 | 710.1ms/step | 28.2 samples/s +Step 33180/44000 | Loss: 2.8786 | LR: 7.33e-05 | 709.0ms/step | 28.2 samples/s +Step 33190/44000 | Loss: 2.8937 | LR: 7.32e-05 | 709.8ms/step | 28.2 samples/s +Step 33200/44000 | Loss: 2.9622 | LR: 7.31e-05 | 710.2ms/step | 28.2 samples/s +Step 33210/44000 | Loss: 2.8950 | LR: 7.29e-05 | 711.5ms/step | 28.1 samples/s +Step 33220/44000 | Loss: 3.0119 | LR: 7.28e-05 | 711.1ms/step | 28.1 samples/s +Step 33230/44000 | Loss: 2.9603 | LR: 7.27e-05 | 727.8ms/step | 27.5 samples/s +Step 33240/44000 | Loss: 2.9764 | LR: 7.26e-05 | 711.4ms/step | 28.1 samples/s +Step 33250/44000 | Loss: 2.9156 | LR: 7.24e-05 | 711.2ms/step | 28.1 samples/s +Validation at step 33250: Loss = 2.9046, Perplexity = 18.26 +Step 33260/44000 | Loss: 2.9487 | LR: 7.23e-05 | 931.4ms/step | 21.5 samples/s +Step 33270/44000 | Loss: 2.8892 | LR: 7.22e-05 | 710.5ms/step | 28.1 samples/s +Step 33280/44000 | Loss: 2.9265 | LR: 7.20e-05 | 710.3ms/step | 28.2 samples/s +Step 33290/44000 | Loss: 2.8776 | LR: 7.19e-05 | 710.8ms/step | 28.1 samples/s +Step 33300/44000 | Loss: 2.8853 | LR: 7.18e-05 | 710.0ms/step | 28.2 samples/s +Step 33310/44000 | Loss: 2.9447 | LR: 7.17e-05 | 725.1ms/step | 27.6 samples/s +Step 33320/44000 | Loss: 2.8902 | LR: 7.15e-05 | 710.5ms/step | 28.2 samples/s +Step 33330/44000 | Loss: 2.9741 | LR: 7.14e-05 | 710.6ms/step | 28.1 samples/s +Step 33340/44000 | Loss: 2.8906 | LR: 7.13e-05 | 710.6ms/step | 28.1 samples/s +Step 33350/44000 | Loss: 2.9430 | LR: 7.11e-05 | 710.8ms/step | 28.1 samples/s +Step 33360/44000 | Loss: 2.9166 | LR: 7.10e-05 | 710.7ms/step | 28.1 samples/s +Step 33370/44000 | Loss: 2.8988 | LR: 7.09e-05 | 710.9ms/step | 28.1 samples/s +Step 33380/44000 | Loss: 2.9890 | LR: 7.08e-05 | 710.5ms/step | 28.1 samples/s +Step 33390/44000 | Loss: 2.8763 | LR: 7.06e-05 | 710.5ms/step | 28.2 samples/s +Step 33400/44000 | Loss: 2.9502 | LR: 7.05e-05 | 727.1ms/step | 27.5 samples/s +Step 33410/44000 | Loss: 2.8693 | LR: 7.04e-05 | 710.7ms/step | 28.1 samples/s +Step 33420/44000 | Loss: 2.8839 | LR: 7.03e-05 | 710.0ms/step | 28.2 samples/s +Step 33430/44000 | Loss: 2.9532 | LR: 7.01e-05 | 710.8ms/step | 28.1 samples/s +Step 33440/44000 | Loss: 2.9324 | LR: 7.00e-05 | 710.4ms/step | 28.2 samples/s +Step 33450/44000 | Loss: 2.9693 | LR: 6.99e-05 | 710.5ms/step | 28.1 samples/s +Step 33460/44000 | Loss: 2.9337 | LR: 6.98e-05 | 710.8ms/step | 28.1 samples/s +Step 33470/44000 | Loss: 2.9859 | LR: 6.96e-05 | 710.1ms/step | 28.2 samples/s +Step 33480/44000 | Loss: 2.8687 | LR: 6.95e-05 | 710.2ms/step | 28.2 samples/s +Step 33490/44000 | Loss: 2.9582 | LR: 6.94e-05 | 727.7ms/step | 27.5 samples/s +Step 33500/44000 | Loss: 2.9559 | LR: 6.93e-05 | 710.3ms/step | 28.2 samples/s +Validation at step 33500: Loss = 2.9043, Perplexity = 18.25 +Step 33510/44000 | Loss: 2.8865 | LR: 6.91e-05 | 930.9ms/step | 21.5 samples/s +Step 33520/44000 | Loss: 2.9182 | LR: 6.90e-05 | 710.7ms/step | 28.1 samples/s +Step 33530/44000 | Loss: 2.9012 | LR: 6.89e-05 | 711.7ms/step | 28.1 samples/s +Step 33540/44000 | Loss: 2.9503 | LR: 6.88e-05 | 712.5ms/step | 28.1 samples/s +Step 33550/44000 | Loss: 2.9091 | LR: 6.86e-05 | 711.7ms/step | 28.1 samples/s +Step 33560/44000 | Loss: 2.8877 | LR: 6.85e-05 | 712.2ms/step | 28.1 samples/s +Step 33570/44000 | Loss: 2.9657 | LR: 6.84e-05 | 710.9ms/step | 28.1 samples/s +Step 33580/44000 | Loss: 2.9349 | LR: 6.83e-05 | 725.9ms/step | 27.6 samples/s +Step 33590/44000 | Loss: 2.8685 | LR: 6.81e-05 | 710.7ms/step | 28.1 samples/s +Step 33600/44000 | Loss: 2.9777 | LR: 6.80e-05 | 710.1ms/step | 28.2 samples/s +Step 33610/44000 | Loss: 2.9261 | LR: 6.79e-05 | 710.7ms/step | 28.1 samples/s +Step 33620/44000 | Loss: 2.9128 | LR: 6.78e-05 | 712.9ms/step | 28.1 samples/s +Step 33630/44000 | Loss: 3.0045 | LR: 6.76e-05 | 713.9ms/step | 28.0 samples/s +Step 33640/44000 | Loss: 2.9547 | LR: 6.75e-05 | 713.6ms/step | 28.0 samples/s +Step 33650/44000 | Loss: 2.8487 | LR: 6.74e-05 | 713.8ms/step | 28.0 samples/s +Step 33660/44000 | Loss: 2.9582 | LR: 6.73e-05 | 723.1ms/step | 27.7 samples/s +Step 33670/44000 | Loss: 3.0184 | LR: 6.71e-05 | 719.5ms/step | 27.8 samples/s +Step 33680/44000 | Loss: 2.9707 | LR: 6.70e-05 | 714.5ms/step | 28.0 samples/s +Step 33690/44000 | Loss: 2.9252 | LR: 6.69e-05 | 714.3ms/step | 28.0 samples/s +Step 33700/44000 | Loss: 2.8767 | LR: 6.68e-05 | 714.0ms/step | 28.0 samples/s +Step 33710/44000 | Loss: 2.9214 | LR: 6.66e-05 | 714.1ms/step | 28.0 samples/s +Step 33720/44000 | Loss: 2.8518 | LR: 6.65e-05 | 713.3ms/step | 28.0 samples/s +Step 33730/44000 | Loss: 2.9107 | LR: 6.64e-05 | 710.6ms/step | 28.1 samples/s +Step 33740/44000 | Loss: 2.9037 | LR: 6.63e-05 | 708.6ms/step | 28.2 samples/s +Step 33750/44000 | Loss: 2.9416 | LR: 6.61e-05 | 721.1ms/step | 27.7 samples/s +Validation at step 33750: Loss = 2.9040, Perplexity = 18.25 +Step 33760/44000 | Loss: 2.9641 | LR: 6.60e-05 | 937.3ms/step | 21.3 samples/s +Step 33770/44000 | Loss: 2.9012 | LR: 6.59e-05 | 710.0ms/step | 28.2 samples/s +Step 33780/44000 | Loss: 2.9457 | LR: 6.58e-05 | 710.5ms/step | 28.1 samples/s +Step 33790/44000 | Loss: 2.9218 | LR: 6.57e-05 | 709.9ms/step | 28.2 samples/s +Step 33800/44000 | Loss: 2.9481 | LR: 6.55e-05 | 711.0ms/step | 28.1 samples/s +Step 33810/44000 | Loss: 2.9078 | LR: 6.54e-05 | 710.4ms/step | 28.2 samples/s +Step 33820/44000 | Loss: 2.9607 | LR: 6.53e-05 | 710.7ms/step | 28.1 samples/s +Step 33830/44000 | Loss: 2.9385 | LR: 6.52e-05 | 712.2ms/step | 28.1 samples/s +Step 33840/44000 | Loss: 2.9004 | LR: 6.50e-05 | 723.2ms/step | 27.7 samples/s +Step 33850/44000 | Loss: 2.8180 | LR: 6.49e-05 | 712.1ms/step | 28.1 samples/s +Step 33860/44000 | Loss: 2.8879 | LR: 6.48e-05 | 710.6ms/step | 28.1 samples/s +Step 33870/44000 | Loss: 2.9149 | LR: 6.47e-05 | 709.2ms/step | 28.2 samples/s +Step 33880/44000 | Loss: 2.9728 | LR: 6.46e-05 | 711.3ms/step | 28.1 samples/s +Step 33890/44000 | Loss: 2.8687 | LR: 6.44e-05 | 711.1ms/step | 28.1 samples/s +Step 33900/44000 | Loss: 2.9498 | LR: 6.43e-05 | 710.8ms/step | 28.1 samples/s +Step 33910/44000 | Loss: 2.9765 | LR: 6.42e-05 | 709.7ms/step | 28.2 samples/s +Step 33920/44000 | Loss: 2.9252 | LR: 6.41e-05 | 710.1ms/step | 28.2 samples/s +Step 33930/44000 | Loss: 2.9327 | LR: 6.40e-05 | 725.3ms/step | 27.6 samples/s +Step 33940/44000 | Loss: 2.9470 | LR: 6.38e-05 | 709.7ms/step | 28.2 samples/s +Step 33950/44000 | Loss: 2.9626 | LR: 6.37e-05 | 710.1ms/step | 28.2 samples/s +Step 33960/44000 | Loss: 2.9374 | LR: 6.36e-05 | 709.1ms/step | 28.2 samples/s +Step 33970/44000 | Loss: 2.9440 | LR: 6.35e-05 | 709.9ms/step | 28.2 samples/s +Step 33980/44000 | Loss: 2.9521 | LR: 6.33e-05 | 709.6ms/step | 28.2 samples/s +Step 33990/44000 | Loss: 2.9998 | LR: 6.32e-05 | 711.0ms/step | 28.1 samples/s +Step 34000/44000 | Loss: 2.8902 | LR: 6.31e-05 | 709.6ms/step | 28.2 samples/s +Validation at step 34000: Loss = 2.9032, Perplexity = 18.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-34000 +Step 34010/44000 | Loss: 2.9853 | LR: 6.30e-05 | 1323.5ms/step | 15.1 samples/s +Step 34020/44000 | Loss: 2.9200 | LR: 6.29e-05 | 712.3ms/step | 28.1 samples/s +Step 34030/44000 | Loss: 2.9585 | LR: 6.27e-05 | 711.6ms/step | 28.1 samples/s +Step 34040/44000 | Loss: 2.9385 | LR: 6.26e-05 | 710.3ms/step | 28.2 samples/s +Step 34050/44000 | Loss: 2.9060 | LR: 6.25e-05 | 712.1ms/step | 28.1 samples/s +Step 34060/44000 | Loss: 2.9143 | LR: 6.24e-05 | 713.9ms/step | 28.0 samples/s +Step 34070/44000 | Loss: 2.9609 | LR: 6.23e-05 | 713.3ms/step | 28.0 samples/s +Step 34080/44000 | Loss: 2.9240 | LR: 6.21e-05 | 713.3ms/step | 28.0 samples/s +Step 34090/44000 | Loss: 2.9603 | LR: 6.20e-05 | 713.4ms/step | 28.0 samples/s +Step 34100/44000 | Loss: 3.0201 | LR: 6.19e-05 | 729.6ms/step | 27.4 samples/s +Step 34110/44000 | Loss: 2.9653 | LR: 6.18e-05 | 709.8ms/step | 28.2 samples/s +Step 34120/44000 | Loss: 2.8626 | LR: 6.17e-05 | 709.4ms/step | 28.2 samples/s +Step 34130/44000 | Loss: 2.9460 | LR: 6.15e-05 | 711.3ms/step | 28.1 samples/s +Step 34140/44000 | Loss: 2.9063 | LR: 6.14e-05 | 712.1ms/step | 28.1 samples/s +Step 34150/44000 | Loss: 2.8254 | LR: 6.13e-05 | 711.3ms/step | 28.1 samples/s +Step 34160/44000 | Loss: 3.0230 | LR: 6.12e-05 | 711.4ms/step | 28.1 samples/s +Step 34170/44000 | Loss: 2.8792 | LR: 6.11e-05 | 711.2ms/step | 28.1 samples/s +Step 34180/44000 | Loss: 2.8731 | LR: 6.09e-05 | 711.3ms/step | 28.1 samples/s +Step 34190/44000 | Loss: 2.8490 | LR: 6.08e-05 | 728.3ms/step | 27.5 samples/s +Step 34200/44000 | Loss: 2.9309 | LR: 6.07e-05 | 711.4ms/step | 28.1 samples/s +Step 34210/44000 | Loss: 2.9482 | LR: 6.06e-05 | 711.6ms/step | 28.1 samples/s +Step 34220/44000 | Loss: 2.8940 | LR: 6.05e-05 | 710.2ms/step | 28.2 samples/s +Step 34230/44000 | Loss: 2.8502 | LR: 6.04e-05 | 710.2ms/step | 28.2 samples/s +Step 34240/44000 | Loss: 2.9450 | LR: 6.02e-05 | 710.3ms/step | 28.2 samples/s +Step 34250/44000 | Loss: 2.8433 | LR: 6.01e-05 | 710.1ms/step | 28.2 samples/s +Validation at step 34250: Loss = 2.9032, Perplexity = 18.23 +Step 34260/44000 | Loss: 2.9664 | LR: 6.00e-05 | 931.0ms/step | 21.5 samples/s +Step 34270/44000 | Loss: 2.9232 | LR: 5.99e-05 | 722.5ms/step | 27.7 samples/s +Step 34280/44000 | Loss: 2.9355 | LR: 5.98e-05 | 714.6ms/step | 28.0 samples/s +Step 34290/44000 | Loss: 2.9265 | LR: 5.96e-05 | 709.0ms/step | 28.2 samples/s +Step 34300/44000 | Loss: 2.9221 | LR: 5.95e-05 | 710.4ms/step | 28.2 samples/s +Step 34310/44000 | Loss: 2.9508 | LR: 5.94e-05 | 710.9ms/step | 28.1 samples/s +Step 34320/44000 | Loss: 2.8884 | LR: 5.93e-05 | 712.3ms/step | 28.1 samples/s +Step 34330/44000 | Loss: 2.9350 | LR: 5.92e-05 | 711.5ms/step | 28.1 samples/s +Step 34340/44000 | Loss: 2.9585 | LR: 5.91e-05 | 711.2ms/step | 28.1 samples/s +Step 34350/44000 | Loss: 2.9117 | LR: 5.89e-05 | 710.6ms/step | 28.1 samples/s +Step 34360/44000 | Loss: 2.8394 | LR: 5.88e-05 | 722.4ms/step | 27.7 samples/s +Step 34370/44000 | Loss: 2.9421 | LR: 5.87e-05 | 716.7ms/step | 27.9 samples/s +Step 34380/44000 | Loss: 2.8913 | LR: 5.86e-05 | 710.5ms/step | 28.1 samples/s +Step 34390/44000 | Loss: 2.8891 | LR: 5.85e-05 | 713.8ms/step | 28.0 samples/s +Step 34400/44000 | Loss: 2.8664 | LR: 5.84e-05 | 713.9ms/step | 28.0 samples/s +Step 34410/44000 | Loss: 2.8879 | LR: 5.82e-05 | 713.3ms/step | 28.0 samples/s +Step 34420/44000 | Loss: 2.9147 | LR: 5.81e-05 | 713.5ms/step | 28.0 samples/s +Step 34430/44000 | Loss: 2.9535 | LR: 5.80e-05 | 711.9ms/step | 28.1 samples/s +Step 34440/44000 | Loss: 2.9246 | LR: 5.79e-05 | 710.7ms/step | 28.1 samples/s +Step 34450/44000 | Loss: 2.9572 | LR: 5.78e-05 | 726.5ms/step | 27.5 samples/s +Step 34460/44000 | Loss: 2.9275 | LR: 5.77e-05 | 709.8ms/step | 28.2 samples/s +Step 34470/44000 | Loss: 2.9168 | LR: 5.75e-05 | 709.4ms/step | 28.2 samples/s +Step 34480/44000 | Loss: 2.9638 | LR: 5.74e-05 | 710.3ms/step | 28.2 samples/s +Step 34490/44000 | Loss: 2.9763 | LR: 5.73e-05 | 710.2ms/step | 28.2 samples/s +Step 34500/44000 | Loss: 2.9315 | LR: 5.72e-05 | 710.1ms/step | 28.2 samples/s +Validation at step 34500: Loss = 2.9027, Perplexity = 18.22 +Step 34510/44000 | Loss: 2.9184 | LR: 5.71e-05 | 927.1ms/step | 21.6 samples/s +Step 34520/44000 | Loss: 2.9258 | LR: 5.70e-05 | 707.7ms/step | 28.3 samples/s +Step 34530/44000 | Loss: 2.9094 | LR: 5.69e-05 | 709.5ms/step | 28.2 samples/s +Step 34540/44000 | Loss: 2.8532 | LR: 5.67e-05 | 729.4ms/step | 27.4 samples/s +Step 34550/44000 | Loss: 2.9201 | LR: 5.66e-05 | 709.3ms/step | 28.2 samples/s +Step 34560/44000 | Loss: 2.9493 | LR: 5.65e-05 | 708.9ms/step | 28.2 samples/s +Step 34570/44000 | Loss: 2.9497 | LR: 5.64e-05 | 710.9ms/step | 28.1 samples/s +Step 34580/44000 | Loss: 2.9299 | LR: 5.63e-05 | 709.9ms/step | 28.2 samples/s +Step 34590/44000 | Loss: 2.9136 | LR: 5.62e-05 | 710.3ms/step | 28.2 samples/s +Step 34600/44000 | Loss: 2.8881 | LR: 5.60e-05 | 709.4ms/step | 28.2 samples/s +Step 34610/44000 | Loss: 2.9515 | LR: 5.59e-05 | 709.2ms/step | 28.2 samples/s +Step 34620/44000 | Loss: 2.9783 | LR: 5.58e-05 | 709.2ms/step | 28.2 samples/s +Step 34630/44000 | Loss: 2.8773 | LR: 5.57e-05 | 724.4ms/step | 27.6 samples/s +Step 34640/44000 | Loss: 2.8934 | LR: 5.56e-05 | 710.2ms/step | 28.2 samples/s +Step 34650/44000 | Loss: 2.9793 | LR: 5.55e-05 | 709.6ms/step | 28.2 samples/s +Step 34660/44000 | Loss: 2.9365 | LR: 5.54e-05 | 710.1ms/step | 28.2 samples/s +Step 34670/44000 | Loss: 2.8803 | LR: 5.52e-05 | 710.7ms/step | 28.1 samples/s +Step 34680/44000 | Loss: 2.9522 | LR: 5.51e-05 | 710.8ms/step | 28.1 samples/s +Step 34690/44000 | Loss: 2.9401 | LR: 5.50e-05 | 711.2ms/step | 28.1 samples/s +Step 34700/44000 | Loss: 2.9278 | LR: 5.49e-05 | 710.6ms/step | 28.1 samples/s +Step 34710/44000 | Loss: 2.8796 | LR: 5.48e-05 | 712.0ms/step | 28.1 samples/s +Step 34720/44000 | Loss: 2.9162 | LR: 5.47e-05 | 725.3ms/step | 27.6 samples/s +Step 34730/44000 | Loss: 2.8527 | LR: 5.46e-05 | 710.2ms/step | 28.2 samples/s +Step 34740/44000 | Loss: 2.9680 | LR: 5.45e-05 | 709.8ms/step | 28.2 samples/s +Step 34750/44000 | Loss: 3.0045 | LR: 5.43e-05 | 710.0ms/step | 28.2 samples/s +Validation at step 34750: Loss = 2.9024, Perplexity = 18.22 +Step 34760/44000 | Loss: 2.9666 | LR: 5.42e-05 | 929.5ms/step | 21.5 samples/s +Step 34770/44000 | Loss: 2.9300 | LR: 5.41e-05 | 712.5ms/step | 28.1 samples/s +Step 34780/44000 | Loss: 2.8913 | LR: 5.40e-05 | 710.0ms/step | 28.2 samples/s +Step 34790/44000 | Loss: 2.9067 | LR: 5.39e-05 | 709.6ms/step | 28.2 samples/s +Step 34800/44000 | Loss: 2.9597 | LR: 5.38e-05 | 726.4ms/step | 27.5 samples/s +Step 34810/44000 | Loss: 2.9884 | LR: 5.37e-05 | 710.3ms/step | 28.2 samples/s +Step 34820/44000 | Loss: 2.8289 | LR: 5.36e-05 | 711.3ms/step | 28.1 samples/s +Step 34830/44000 | Loss: 2.9094 | LR: 5.34e-05 | 711.4ms/step | 28.1 samples/s +Step 34840/44000 | Loss: 2.9781 | LR: 5.33e-05 | 711.3ms/step | 28.1 samples/s +Step 34850/44000 | Loss: 2.9201 | LR: 5.32e-05 | 712.1ms/step | 28.1 samples/s +Step 34860/44000 | Loss: 2.9328 | LR: 5.31e-05 | 709.7ms/step | 28.2 samples/s +Step 34870/44000 | Loss: 2.9506 | LR: 5.30e-05 | 709.5ms/step | 28.2 samples/s +Step 34880/44000 | Loss: 2.9303 | LR: 5.29e-05 | 710.1ms/step | 28.2 samples/s +Step 34890/44000 | Loss: 2.9332 | LR: 5.28e-05 | 727.6ms/step | 27.5 samples/s +Step 34900/44000 | Loss: 2.9226 | LR: 5.27e-05 | 710.5ms/step | 28.1 samples/s +Step 34910/44000 | Loss: 2.8891 | LR: 5.25e-05 | 710.2ms/step | 28.2 samples/s +Step 34920/44000 | Loss: 2.8884 | LR: 5.24e-05 | 710.2ms/step | 28.2 samples/s +Step 34930/44000 | Loss: 2.9311 | LR: 5.23e-05 | 710.5ms/step | 28.2 samples/s +Step 34940/44000 | Loss: 2.8445 | LR: 5.22e-05 | 712.6ms/step | 28.1 samples/s +Step 34950/44000 | Loss: 2.9048 | LR: 5.21e-05 | 712.9ms/step | 28.1 samples/s +Step 34960/44000 | Loss: 3.0332 | LR: 5.20e-05 | 710.2ms/step | 28.2 samples/s +Step 34970/44000 | Loss: 2.9328 | LR: 5.19e-05 | 709.7ms/step | 28.2 samples/s +Step 34980/44000 | Loss: 2.9165 | LR: 5.18e-05 | 725.5ms/step | 27.6 samples/s +Step 34990/44000 | Loss: 2.9178 | LR: 5.17e-05 | 713.2ms/step | 28.0 samples/s +Step 35000/44000 | Loss: 2.8985 | LR: 5.15e-05 | 713.6ms/step | 28.0 samples/s +Validation at step 35000: Loss = 2.9021, Perplexity = 18.21 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-35000 +Step 35010/44000 | Loss: 2.8813 | LR: 5.14e-05 | 1344.6ms/step | 14.9 samples/s +Step 35020/44000 | Loss: 2.9037 | LR: 5.13e-05 | 710.4ms/step | 28.2 samples/s +Step 35030/44000 | Loss: 2.9175 | LR: 5.12e-05 | 710.3ms/step | 28.2 samples/s +Step 35040/44000 | Loss: 2.9435 | LR: 5.11e-05 | 710.2ms/step | 28.2 samples/s +Step 35050/44000 | Loss: 2.9417 | LR: 5.10e-05 | 711.2ms/step | 28.1 samples/s +Step 35060/44000 | Loss: 2.9960 | LR: 5.09e-05 | 729.5ms/step | 27.4 samples/s +Step 35070/44000 | Loss: 2.9454 | LR: 5.08e-05 | 710.9ms/step | 28.1 samples/s +Step 35080/44000 | Loss: 2.9533 | LR: 5.07e-05 | 710.6ms/step | 28.1 samples/s +Step 35090/44000 | Loss: 2.8980 | LR: 5.06e-05 | 710.6ms/step | 28.1 samples/s +Step 35100/44000 | Loss: 2.9921 | LR: 5.04e-05 | 710.5ms/step | 28.2 samples/s +Step 35110/44000 | Loss: 2.9392 | LR: 5.03e-05 | 711.3ms/step | 28.1 samples/s +Step 35120/44000 | Loss: 2.9243 | LR: 5.02e-05 | 711.3ms/step | 28.1 samples/s +Step 35130/44000 | Loss: 2.8698 | LR: 5.01e-05 | 711.7ms/step | 28.1 samples/s +Step 35140/44000 | Loss: 2.9632 | LR: 5.00e-05 | 711.6ms/step | 28.1 samples/s +Step 35150/44000 | Loss: 2.9412 | LR: 4.99e-05 | 729.2ms/step | 27.4 samples/s +Step 35160/44000 | Loss: 2.8930 | LR: 4.98e-05 | 710.9ms/step | 28.1 samples/s +Step 35170/44000 | Loss: 2.9810 | LR: 4.97e-05 | 711.6ms/step | 28.1 samples/s +Step 35180/44000 | Loss: 2.9056 | LR: 4.96e-05 | 711.3ms/step | 28.1 samples/s +Step 35190/44000 | Loss: 2.9155 | LR: 4.95e-05 | 711.2ms/step | 28.1 samples/s +Step 35200/44000 | Loss: 2.9299 | LR: 4.94e-05 | 712.3ms/step | 28.1 samples/s +Step 35210/44000 | Loss: 2.9114 | LR: 4.93e-05 | 714.5ms/step | 28.0 samples/s +Step 35220/44000 | Loss: 2.9153 | LR: 4.91e-05 | 714.5ms/step | 28.0 samples/s +Step 35230/44000 | Loss: 2.9891 | LR: 4.90e-05 | 715.1ms/step | 28.0 samples/s +Step 35240/44000 | Loss: 2.8298 | LR: 4.89e-05 | 727.1ms/step | 27.5 samples/s +Step 35250/44000 | Loss: 2.9141 | LR: 4.88e-05 | 715.0ms/step | 28.0 samples/s +Validation at step 35250: Loss = 2.9018, Perplexity = 18.21 +Step 35260/44000 | Loss: 2.9543 | LR: 4.87e-05 | 930.6ms/step | 21.5 samples/s +Step 35270/44000 | Loss: 2.8793 | LR: 4.86e-05 | 710.8ms/step | 28.1 samples/s +Step 35280/44000 | Loss: 2.9955 | LR: 4.85e-05 | 712.5ms/step | 28.1 samples/s +Step 35290/44000 | Loss: 2.9124 | LR: 4.84e-05 | 712.3ms/step | 28.1 samples/s +Step 35300/44000 | Loss: 2.8768 | LR: 4.83e-05 | 712.8ms/step | 28.1 samples/s +Step 35310/44000 | Loss: 2.8451 | LR: 4.82e-05 | 711.8ms/step | 28.1 samples/s +Step 35320/44000 | Loss: 2.9412 | LR: 4.81e-05 | 724.8ms/step | 27.6 samples/s +Step 35330/44000 | Loss: 2.8808 | LR: 4.80e-05 | 717.3ms/step | 27.9 samples/s +Step 35340/44000 | Loss: 2.9304 | LR: 4.79e-05 | 712.5ms/step | 28.1 samples/s +Step 35350/44000 | Loss: 2.8519 | LR: 4.77e-05 | 712.1ms/step | 28.1 samples/s +Step 35360/44000 | Loss: 2.8499 | LR: 4.76e-05 | 712.9ms/step | 28.1 samples/s +Step 35370/44000 | Loss: 2.8490 | LR: 4.75e-05 | 711.2ms/step | 28.1 samples/s +Step 35380/44000 | Loss: 2.9356 | LR: 4.74e-05 | 710.6ms/step | 28.1 samples/s +Step 35390/44000 | Loss: 2.9082 | LR: 4.73e-05 | 710.9ms/step | 28.1 samples/s +Step 35400/44000 | Loss: 2.9364 | LR: 4.72e-05 | 712.1ms/step | 28.1 samples/s +Step 35410/44000 | Loss: 2.9576 | LR: 4.71e-05 | 723.6ms/step | 27.6 samples/s +Step 35420/44000 | Loss: 2.9459 | LR: 4.70e-05 | 717.5ms/step | 27.9 samples/s +Step 35430/44000 | Loss: 2.9154 | LR: 4.69e-05 | 712.8ms/step | 28.1 samples/s +Step 35440/44000 | Loss: 2.9400 | LR: 4.68e-05 | 711.5ms/step | 28.1 samples/s +Step 35450/44000 | Loss: 2.9216 | LR: 4.67e-05 | 712.4ms/step | 28.1 samples/s +Step 35460/44000 | Loss: 2.9364 | LR: 4.66e-05 | 712.7ms/step | 28.1 samples/s +Step 35470/44000 | Loss: 2.8160 | LR: 4.65e-05 | 713.3ms/step | 28.0 samples/s +Step 35480/44000 | Loss: 2.9303 | LR: 4.64e-05 | 712.0ms/step | 28.1 samples/s +Step 35490/44000 | Loss: 2.8907 | LR: 4.63e-05 | 711.7ms/step | 28.1 samples/s +Step 35500/44000 | Loss: 2.9596 | LR: 4.62e-05 | 727.2ms/step | 27.5 samples/s +Validation at step 35500: Loss = 2.9018, Perplexity = 18.21 +Step 35510/44000 | Loss: 2.9554 | LR: 4.61e-05 | 932.0ms/step | 21.5 samples/s +Step 35520/44000 | Loss: 2.8517 | LR: 4.59e-05 | 711.9ms/step | 28.1 samples/s +Step 35530/44000 | Loss: 2.9272 | LR: 4.58e-05 | 711.6ms/step | 28.1 samples/s +Step 35540/44000 | Loss: 2.9544 | LR: 4.57e-05 | 712.1ms/step | 28.1 samples/s +Step 35550/44000 | Loss: 2.9794 | LR: 4.56e-05 | 712.0ms/step | 28.1 samples/s +Step 35560/44000 | Loss: 2.8891 | LR: 4.55e-05 | 713.0ms/step | 28.1 samples/s +Step 35570/44000 | Loss: 2.9493 | LR: 4.54e-05 | 712.4ms/step | 28.1 samples/s +Step 35580/44000 | Loss: 2.9341 | LR: 4.53e-05 | 711.4ms/step | 28.1 samples/s +Step 35590/44000 | Loss: 2.9461 | LR: 4.52e-05 | 725.9ms/step | 27.6 samples/s +Step 35600/44000 | Loss: 2.8954 | LR: 4.51e-05 | 712.2ms/step | 28.1 samples/s +Step 35610/44000 | Loss: 2.9082 | LR: 4.50e-05 | 712.5ms/step | 28.1 samples/s +Step 35620/44000 | Loss: 2.9174 | LR: 4.49e-05 | 711.1ms/step | 28.1 samples/s +Step 35630/44000 | Loss: 2.9016 | LR: 4.48e-05 | 710.0ms/step | 28.2 samples/s +Step 35640/44000 | Loss: 2.8991 | LR: 4.47e-05 | 711.0ms/step | 28.1 samples/s +Step 35650/44000 | Loss: 2.8745 | LR: 4.46e-05 | 710.4ms/step | 28.2 samples/s +Step 35660/44000 | Loss: 2.8622 | LR: 4.45e-05 | 712.1ms/step | 28.1 samples/s +Step 35670/44000 | Loss: 2.8493 | LR: 4.44e-05 | 712.6ms/step | 28.1 samples/s +Step 35680/44000 | Loss: 2.9025 | LR: 4.43e-05 | 730.4ms/step | 27.4 samples/s +Step 35690/44000 | Loss: 2.9211 | LR: 4.42e-05 | 712.9ms/step | 28.1 samples/s +Step 35700/44000 | Loss: 2.8843 | LR: 4.41e-05 | 710.8ms/step | 28.1 samples/s +Step 35710/44000 | Loss: 2.9527 | LR: 4.40e-05 | 711.0ms/step | 28.1 samples/s +Step 35720/44000 | Loss: 2.7990 | LR: 4.39e-05 | 710.4ms/step | 28.2 samples/s +Step 35730/44000 | Loss: 2.9038 | LR: 4.38e-05 | 710.6ms/step | 28.1 samples/s +Step 35740/44000 | Loss: 2.9233 | LR: 4.37e-05 | 711.1ms/step | 28.1 samples/s +Step 35750/44000 | Loss: 2.8111 | LR: 4.36e-05 | 711.4ms/step | 28.1 samples/s +Validation at step 35750: Loss = 2.9014, Perplexity = 18.20 +Step 35760/44000 | Loss: 2.9114 | LR: 4.35e-05 | 947.7ms/step | 21.1 samples/s +Step 35770/44000 | Loss: 2.8534 | LR: 4.34e-05 | 710.9ms/step | 28.1 samples/s +Step 35780/44000 | Loss: 2.9582 | LR: 4.33e-05 | 712.0ms/step | 28.1 samples/s +Step 35790/44000 | Loss: 2.9001 | LR: 4.32e-05 | 713.1ms/step | 28.0 samples/s +Step 35800/44000 | Loss: 2.9128 | LR: 4.31e-05 | 712.7ms/step | 28.1 samples/s +Step 35810/44000 | Loss: 2.9168 | LR: 4.29e-05 | 712.1ms/step | 28.1 samples/s +Step 35820/44000 | Loss: 2.9848 | LR: 4.28e-05 | 712.3ms/step | 28.1 samples/s +Step 35830/44000 | Loss: 2.9033 | LR: 4.27e-05 | 712.0ms/step | 28.1 samples/s +Step 35840/44000 | Loss: 2.8967 | LR: 4.26e-05 | 710.9ms/step | 28.1 samples/s +Step 35850/44000 | Loss: 2.9004 | LR: 4.25e-05 | 727.5ms/step | 27.5 samples/s +Step 35860/44000 | Loss: 2.8890 | LR: 4.24e-05 | 710.5ms/step | 28.1 samples/s +Step 35870/44000 | Loss: 2.8581 | LR: 4.23e-05 | 711.1ms/step | 28.1 samples/s +Step 35880/44000 | Loss: 2.9396 | LR: 4.22e-05 | 710.6ms/step | 28.1 samples/s +Step 35890/44000 | Loss: 2.9416 | LR: 4.21e-05 | 712.6ms/step | 28.1 samples/s +Step 35900/44000 | Loss: 2.9420 | LR: 4.20e-05 | 711.8ms/step | 28.1 samples/s +Step 35910/44000 | Loss: 2.9220 | LR: 4.19e-05 | 711.1ms/step | 28.1 samples/s +Step 35920/44000 | Loss: 2.9141 | LR: 4.18e-05 | 710.4ms/step | 28.2 samples/s +Step 35930/44000 | Loss: 2.8832 | LR: 4.17e-05 | 710.6ms/step | 28.1 samples/s +Step 35940/44000 | Loss: 2.9354 | LR: 4.16e-05 | 729.5ms/step | 27.4 samples/s +Step 35950/44000 | Loss: 2.9140 | LR: 4.15e-05 | 711.8ms/step | 28.1 samples/s +Step 35960/44000 | Loss: 2.9442 | LR: 4.14e-05 | 712.0ms/step | 28.1 samples/s +Step 35970/44000 | Loss: 2.9969 | LR: 4.13e-05 | 711.8ms/step | 28.1 samples/s +Step 35980/44000 | Loss: 2.9748 | LR: 4.12e-05 | 711.3ms/step | 28.1 samples/s +Step 35990/44000 | Loss: 2.9614 | LR: 4.11e-05 | 711.6ms/step | 28.1 samples/s +Step 36000/44000 | Loss: 2.8956 | LR: 4.10e-05 | 711.2ms/step | 28.1 samples/s +Validation at step 36000: Loss = 2.9012, Perplexity = 18.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-36000 +Step 36010/44000 | Loss: 2.9295 | LR: 4.09e-05 | 1317.1ms/step | 15.2 samples/s +Step 36020/44000 | Loss: 2.8480 | LR: 4.08e-05 | 726.3ms/step | 27.5 samples/s +Step 36030/44000 | Loss: 2.9117 | LR: 4.07e-05 | 710.3ms/step | 28.2 samples/s +Step 36040/44000 | Loss: 2.9442 | LR: 4.06e-05 | 711.5ms/step | 28.1 samples/s +Step 36050/44000 | Loss: 2.9543 | LR: 4.05e-05 | 710.3ms/step | 28.2 samples/s +Step 36060/44000 | Loss: 2.9356 | LR: 4.04e-05 | 711.6ms/step | 28.1 samples/s +Step 36070/44000 | Loss: 2.8972 | LR: 4.03e-05 | 717.4ms/step | 27.9 samples/s +Step 36080/44000 | Loss: 2.9374 | LR: 4.02e-05 | 715.2ms/step | 28.0 samples/s +Step 36090/44000 | Loss: 2.9136 | LR: 4.01e-05 | 712.0ms/step | 28.1 samples/s +Step 36100/44000 | Loss: 2.9048 | LR: 4.00e-05 | 709.8ms/step | 28.2 samples/s +Step 36110/44000 | Loss: 2.9120 | LR: 3.99e-05 | 725.9ms/step | 27.6 samples/s +Step 36120/44000 | Loss: 2.9415 | LR: 3.98e-05 | 711.0ms/step | 28.1 samples/s +Step 36130/44000 | Loss: 2.9001 | LR: 3.97e-05 | 710.7ms/step | 28.1 samples/s +Step 36140/44000 | Loss: 2.9605 | LR: 3.97e-05 | 711.4ms/step | 28.1 samples/s +Step 36150/44000 | Loss: 2.9013 | LR: 3.96e-05 | 711.1ms/step | 28.1 samples/s +Step 36160/44000 | Loss: 2.9168 | LR: 3.95e-05 | 712.2ms/step | 28.1 samples/s +Step 36170/44000 | Loss: 2.9020 | LR: 3.94e-05 | 711.7ms/step | 28.1 samples/s +Step 36180/44000 | Loss: 2.8976 | LR: 3.93e-05 | 713.2ms/step | 28.0 samples/s +Step 36190/44000 | Loss: 2.9266 | LR: 3.92e-05 | 714.4ms/step | 28.0 samples/s +Step 36200/44000 | Loss: 2.9169 | LR: 3.91e-05 | 726.0ms/step | 27.5 samples/s +Step 36210/44000 | Loss: 2.8947 | LR: 3.90e-05 | 712.2ms/step | 28.1 samples/s +Step 36220/44000 | Loss: 2.8995 | LR: 3.89e-05 | 711.6ms/step | 28.1 samples/s +Step 36230/44000 | Loss: 2.9280 | LR: 3.88e-05 | 711.3ms/step | 28.1 samples/s +Step 36240/44000 | Loss: 2.9451 | LR: 3.87e-05 | 711.9ms/step | 28.1 samples/s +Step 36250/44000 | Loss: 2.9175 | LR: 3.86e-05 | 712.2ms/step | 28.1 samples/s +Validation at step 36250: Loss = 2.9012, Perplexity = 18.20 +Step 36260/44000 | Loss: 2.9092 | LR: 3.85e-05 | 931.2ms/step | 21.5 samples/s +Step 36270/44000 | Loss: 2.9436 | LR: 3.84e-05 | 710.9ms/step | 28.1 samples/s +Step 36280/44000 | Loss: 2.8969 | LR: 3.83e-05 | 721.5ms/step | 27.7 samples/s +Step 36290/44000 | Loss: 2.9688 | LR: 3.82e-05 | 718.1ms/step | 27.9 samples/s +Step 36300/44000 | Loss: 2.8591 | LR: 3.81e-05 | 711.2ms/step | 28.1 samples/s +Step 36310/44000 | Loss: 2.8957 | LR: 3.80e-05 | 711.0ms/step | 28.1 samples/s +Step 36320/44000 | Loss: 2.9638 | LR: 3.79e-05 | 711.1ms/step | 28.1 samples/s +Step 36330/44000 | Loss: 2.8357 | LR: 3.78e-05 | 713.3ms/step | 28.0 samples/s +Step 36340/44000 | Loss: 2.9014 | LR: 3.77e-05 | 714.0ms/step | 28.0 samples/s +Step 36350/44000 | Loss: 2.8867 | LR: 3.76e-05 | 711.8ms/step | 28.1 samples/s +Step 36360/44000 | Loss: 2.9056 | LR: 3.75e-05 | 711.6ms/step | 28.1 samples/s +Step 36370/44000 | Loss: 2.8878 | LR: 3.74e-05 | 724.1ms/step | 27.6 samples/s +Step 36380/44000 | Loss: 2.9294 | LR: 3.73e-05 | 717.3ms/step | 27.9 samples/s +Step 36390/44000 | Loss: 2.9065 | LR: 3.72e-05 | 710.8ms/step | 28.1 samples/s +Step 36400/44000 | Loss: 3.0065 | LR: 3.71e-05 | 711.5ms/step | 28.1 samples/s +Step 36410/44000 | Loss: 2.8690 | LR: 3.70e-05 | 711.3ms/step | 28.1 samples/s +Step 36420/44000 | Loss: 2.9588 | LR: 3.69e-05 | 711.6ms/step | 28.1 samples/s +Step 36430/44000 | Loss: 2.9943 | LR: 3.69e-05 | 712.1ms/step | 28.1 samples/s +Step 36440/44000 | Loss: 2.9126 | LR: 3.68e-05 | 712.0ms/step | 28.1 samples/s +Step 36450/44000 | Loss: 2.8618 | LR: 3.67e-05 | 712.1ms/step | 28.1 samples/s +Step 36460/44000 | Loss: 2.8789 | LR: 3.66e-05 | 729.5ms/step | 27.4 samples/s +Step 36470/44000 | Loss: 2.9760 | LR: 3.65e-05 | 711.2ms/step | 28.1 samples/s +Step 36480/44000 | Loss: 2.8806 | LR: 3.64e-05 | 711.4ms/step | 28.1 samples/s +Step 36490/44000 | Loss: 2.9468 | LR: 3.63e-05 | 711.8ms/step | 28.1 samples/s +Step 36500/44000 | Loss: 2.8813 | LR: 3.62e-05 | 710.8ms/step | 28.1 samples/s +Validation at step 36500: Loss = 2.9011, Perplexity = 18.19 +Step 36510/44000 | Loss: 2.8920 | LR: 3.61e-05 | 932.5ms/step | 21.4 samples/s +Step 36520/44000 | Loss: 2.8700 | LR: 3.60e-05 | 712.7ms/step | 28.1 samples/s +Step 36530/44000 | Loss: 2.9130 | LR: 3.59e-05 | 712.4ms/step | 28.1 samples/s +Step 36540/44000 | Loss: 2.8975 | LR: 3.58e-05 | 712.7ms/step | 28.1 samples/s +Step 36550/44000 | Loss: 2.9595 | LR: 3.57e-05 | 727.2ms/step | 27.5 samples/s +Step 36560/44000 | Loss: 2.9033 | LR: 3.56e-05 | 712.2ms/step | 28.1 samples/s +Step 36570/44000 | Loss: 2.9613 | LR: 3.55e-05 | 710.9ms/step | 28.1 samples/s +Step 36580/44000 | Loss: 2.9297 | LR: 3.54e-05 | 710.9ms/step | 28.1 samples/s +Step 36590/44000 | Loss: 2.8970 | LR: 3.53e-05 | 710.6ms/step | 28.1 samples/s +Step 36600/44000 | Loss: 2.8599 | LR: 3.53e-05 | 709.8ms/step | 28.2 samples/s +Step 36610/44000 | Loss: 2.9646 | LR: 3.52e-05 | 710.4ms/step | 28.2 samples/s +Step 36620/44000 | Loss: 2.9159 | LR: 3.51e-05 | 710.5ms/step | 28.1 samples/s +Step 36630/44000 | Loss: 2.8692 | LR: 3.50e-05 | 710.2ms/step | 28.2 samples/s +Step 36640/44000 | Loss: 2.9053 | LR: 3.49e-05 | 727.1ms/step | 27.5 samples/s +Step 36650/44000 | Loss: 2.8686 | LR: 3.48e-05 | 710.6ms/step | 28.1 samples/s +Step 36660/44000 | Loss: 2.8699 | LR: 3.47e-05 | 710.4ms/step | 28.2 samples/s +Step 36670/44000 | Loss: 2.9918 | LR: 3.46e-05 | 711.1ms/step | 28.1 samples/s +Step 36680/44000 | Loss: 2.9476 | LR: 3.45e-05 | 712.5ms/step | 28.1 samples/s +Step 36690/44000 | Loss: 2.8499 | LR: 3.44e-05 | 712.8ms/step | 28.1 samples/s +Step 36700/44000 | Loss: 2.8958 | LR: 3.43e-05 | 712.4ms/step | 28.1 samples/s +Step 36710/44000 | Loss: 2.9280 | LR: 3.42e-05 | 711.7ms/step | 28.1 samples/s +Step 36720/44000 | Loss: 2.9189 | LR: 3.41e-05 | 713.0ms/step | 28.1 samples/s +Step 36730/44000 | Loss: 2.8761 | LR: 3.41e-05 | 729.0ms/step | 27.4 samples/s +Step 36740/44000 | Loss: 2.9094 | LR: 3.40e-05 | 711.4ms/step | 28.1 samples/s +Step 36750/44000 | Loss: 2.9140 | LR: 3.39e-05 | 712.1ms/step | 28.1 samples/s +Validation at step 36750: Loss = 2.9009, Perplexity = 18.19 +Step 36760/44000 | Loss: 2.9297 | LR: 3.38e-05 | 932.8ms/step | 21.4 samples/s +Step 36770/44000 | Loss: 2.9401 | LR: 3.37e-05 | 712.7ms/step | 28.1 samples/s +Step 36780/44000 | Loss: 2.9284 | LR: 3.36e-05 | 712.5ms/step | 28.1 samples/s +Step 36790/44000 | Loss: 2.8969 | LR: 3.35e-05 | 712.3ms/step | 28.1 samples/s +Step 36800/44000 | Loss: 2.9116 | LR: 3.34e-05 | 712.1ms/step | 28.1 samples/s +Step 36810/44000 | Loss: 2.9836 | LR: 3.33e-05 | 730.0ms/step | 27.4 samples/s +Step 36820/44000 | Loss: 2.8808 | LR: 3.32e-05 | 711.3ms/step | 28.1 samples/s +Step 36830/44000 | Loss: 2.9642 | LR: 3.31e-05 | 712.8ms/step | 28.1 samples/s +Step 36840/44000 | Loss: 2.8884 | LR: 3.31e-05 | 710.5ms/step | 28.1 samples/s +Step 36850/44000 | Loss: 2.9082 | LR: 3.30e-05 | 710.8ms/step | 28.1 samples/s +Step 36860/44000 | Loss: 2.8943 | LR: 3.29e-05 | 714.0ms/step | 28.0 samples/s +Step 36870/44000 | Loss: 3.0266 | LR: 3.28e-05 | 714.7ms/step | 28.0 samples/s +Step 36880/44000 | Loss: 2.8441 | LR: 3.27e-05 | 713.8ms/step | 28.0 samples/s +Step 36890/44000 | Loss: 3.0006 | LR: 3.26e-05 | 711.0ms/step | 28.1 samples/s +Step 36900/44000 | Loss: 2.9134 | LR: 3.25e-05 | 728.0ms/step | 27.5 samples/s +Step 36910/44000 | Loss: 2.9738 | LR: 3.24e-05 | 712.0ms/step | 28.1 samples/s +Step 36920/44000 | Loss: 2.9579 | LR: 3.23e-05 | 710.8ms/step | 28.1 samples/s +Step 36930/44000 | Loss: 2.9364 | LR: 3.22e-05 | 710.5ms/step | 28.1 samples/s +Step 36940/44000 | Loss: 2.9458 | LR: 3.22e-05 | 710.8ms/step | 28.1 samples/s +Step 36950/44000 | Loss: 2.9133 | LR: 3.21e-05 | 711.8ms/step | 28.1 samples/s +Step 36960/44000 | Loss: 2.8673 | LR: 3.20e-05 | 711.9ms/step | 28.1 samples/s +Step 36970/44000 | Loss: 2.9036 | LR: 3.19e-05 | 711.9ms/step | 28.1 samples/s +Step 36980/44000 | Loss: 2.9191 | LR: 3.18e-05 | 712.7ms/step | 28.1 samples/s +Step 36990/44000 | Loss: 2.8646 | LR: 3.17e-05 | 728.7ms/step | 27.4 samples/s +Step 37000/44000 | Loss: 2.9131 | LR: 3.16e-05 | 711.1ms/step | 28.1 samples/s +Validation at step 37000: Loss = 2.9008, Perplexity = 18.19 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-37000 +Step 37010/44000 | Loss: 2.9283 | LR: 3.15e-05 | 1267.7ms/step | 15.8 samples/s +Step 37020/44000 | Loss: 2.8732 | LR: 3.15e-05 | 713.1ms/step | 28.0 samples/s +Step 37030/44000 | Loss: 2.9334 | LR: 3.14e-05 | 712.8ms/step | 28.1 samples/s +Step 37040/44000 | Loss: 2.9173 | LR: 3.13e-05 | 711.9ms/step | 28.1 samples/s +Step 37050/44000 | Loss: 2.9490 | LR: 3.12e-05 | 711.4ms/step | 28.1 samples/s +Step 37060/44000 | Loss: 2.8948 | LR: 3.11e-05 | 711.2ms/step | 28.1 samples/s +Step 37070/44000 | Loss: 2.8835 | LR: 3.10e-05 | 728.9ms/step | 27.4 samples/s +Step 37080/44000 | Loss: 2.9464 | LR: 3.09e-05 | 710.9ms/step | 28.1 samples/s +Step 37090/44000 | Loss: 2.9325 | LR: 3.08e-05 | 709.8ms/step | 28.2 samples/s +Step 37100/44000 | Loss: 2.8985 | LR: 3.07e-05 | 711.2ms/step | 28.1 samples/s +Step 37110/44000 | Loss: 2.8772 | LR: 3.07e-05 | 711.1ms/step | 28.1 samples/s +Step 37120/44000 | Loss: 2.9608 | LR: 3.06e-05 | 711.2ms/step | 28.1 samples/s +Step 37130/44000 | Loss: 2.9562 | LR: 3.05e-05 | 711.7ms/step | 28.1 samples/s +Step 37140/44000 | Loss: 2.9285 | LR: 3.04e-05 | 712.3ms/step | 28.1 samples/s +Step 37150/44000 | Loss: 2.8759 | LR: 3.03e-05 | 713.9ms/step | 28.0 samples/s +Step 37160/44000 | Loss: 2.9327 | LR: 3.02e-05 | 728.7ms/step | 27.4 samples/s +Step 37170/44000 | Loss: 2.9399 | LR: 3.01e-05 | 711.6ms/step | 28.1 samples/s +Step 37180/44000 | Loss: 2.9240 | LR: 3.01e-05 | 711.6ms/step | 28.1 samples/s +Step 37190/44000 | Loss: 2.9142 | LR: 3.00e-05 | 711.8ms/step | 28.1 samples/s +Step 37200/44000 | Loss: 2.8664 | LR: 2.99e-05 | 711.6ms/step | 28.1 samples/s +Step 37210/44000 | Loss: 2.9979 | LR: 2.98e-05 | 711.7ms/step | 28.1 samples/s +Step 37220/44000 | Loss: 2.9441 | LR: 2.97e-05 | 712.5ms/step | 28.1 samples/s +Step 37230/44000 | Loss: 2.8828 | LR: 2.96e-05 | 711.2ms/step | 28.1 samples/s +Step 37240/44000 | Loss: 3.0011 | LR: 2.95e-05 | 710.7ms/step | 28.1 samples/s +Step 37250/44000 | Loss: 2.9315 | LR: 2.95e-05 | 726.7ms/step | 27.5 samples/s +Validation at step 37250: Loss = 2.9007, Perplexity = 18.19 +Step 37260/44000 | Loss: 2.9558 | LR: 2.94e-05 | 929.2ms/step | 21.5 samples/s +Step 37270/44000 | Loss: 2.9214 | LR: 2.93e-05 | 711.1ms/step | 28.1 samples/s +Step 37280/44000 | Loss: 2.9465 | LR: 2.92e-05 | 711.0ms/step | 28.1 samples/s +Step 37290/44000 | Loss: 2.9294 | LR: 2.91e-05 | 711.0ms/step | 28.1 samples/s +Step 37300/44000 | Loss: 2.8991 | LR: 2.90e-05 | 711.6ms/step | 28.1 samples/s +Step 37310/44000 | Loss: 2.9051 | LR: 2.89e-05 | 711.9ms/step | 28.1 samples/s +Step 37320/44000 | Loss: 2.9026 | LR: 2.89e-05 | 711.8ms/step | 28.1 samples/s +Step 37330/44000 | Loss: 2.8877 | LR: 2.88e-05 | 720.3ms/step | 27.8 samples/s +Step 37340/44000 | Loss: 2.9286 | LR: 2.87e-05 | 715.9ms/step | 27.9 samples/s +Step 37350/44000 | Loss: 2.9741 | LR: 2.86e-05 | 711.7ms/step | 28.1 samples/s +Step 37360/44000 | Loss: 2.8744 | LR: 2.85e-05 | 712.2ms/step | 28.1 samples/s +Step 37370/44000 | Loss: 2.8905 | LR: 2.84e-05 | 710.1ms/step | 28.2 samples/s +Step 37380/44000 | Loss: 2.9017 | LR: 2.84e-05 | 712.5ms/step | 28.1 samples/s +Step 37390/44000 | Loss: 2.9191 | LR: 2.83e-05 | 712.9ms/step | 28.1 samples/s +Step 37400/44000 | Loss: 2.9112 | LR: 2.82e-05 | 712.6ms/step | 28.1 samples/s +Step 37410/44000 | Loss: 2.9082 | LR: 2.81e-05 | 712.6ms/step | 28.1 samples/s +Step 37420/44000 | Loss: 3.0290 | LR: 2.80e-05 | 721.0ms/step | 27.7 samples/s +Step 37430/44000 | Loss: 2.8380 | LR: 2.79e-05 | 718.7ms/step | 27.8 samples/s +Step 37440/44000 | Loss: 2.9709 | LR: 2.78e-05 | 711.7ms/step | 28.1 samples/s +Step 37450/44000 | Loss: 2.8830 | LR: 2.78e-05 | 709.6ms/step | 28.2 samples/s +Step 37460/44000 | Loss: 2.8911 | LR: 2.77e-05 | 710.8ms/step | 28.1 samples/s +Step 37470/44000 | Loss: 2.9429 | LR: 2.76e-05 | 710.1ms/step | 28.2 samples/s +Step 37480/44000 | Loss: 2.9467 | LR: 2.75e-05 | 711.2ms/step | 28.1 samples/s +Step 37490/44000 | Loss: 2.9184 | LR: 2.74e-05 | 711.3ms/step | 28.1 samples/s +Step 37500/44000 | Loss: 2.8891 | LR: 2.74e-05 | 711.8ms/step | 28.1 samples/s +Validation at step 37500: Loss = 2.9006, Perplexity = 18.18 +Step 37510/44000 | Loss: 2.8779 | LR: 2.73e-05 | 947.9ms/step | 21.1 samples/s +Step 37520/44000 | Loss: 2.8875 | LR: 2.72e-05 | 711.2ms/step | 28.1 samples/s +Step 37530/44000 | Loss: 2.9103 | LR: 2.71e-05 | 711.8ms/step | 28.1 samples/s +Step 37540/44000 | Loss: 2.8972 | LR: 2.70e-05 | 711.3ms/step | 28.1 samples/s +Step 37550/44000 | Loss: 2.9193 | LR: 2.69e-05 | 710.9ms/step | 28.1 samples/s +Step 37560/44000 | Loss: 2.9612 | LR: 2.69e-05 | 712.2ms/step | 28.1 samples/s +Step 37570/44000 | Loss: 2.9022 | LR: 2.68e-05 | 713.2ms/step | 28.0 samples/s +Step 37580/44000 | Loss: 2.9207 | LR: 2.67e-05 | 713.4ms/step | 28.0 samples/s +Step 37590/44000 | Loss: 2.9278 | LR: 2.66e-05 | 711.9ms/step | 28.1 samples/s +Step 37600/44000 | Loss: 2.9711 | LR: 2.65e-05 | 726.9ms/step | 27.5 samples/s +Step 37610/44000 | Loss: 2.8897 | LR: 2.65e-05 | 710.7ms/step | 28.1 samples/s +Step 37620/44000 | Loss: 2.8829 | LR: 2.64e-05 | 711.2ms/step | 28.1 samples/s +Step 37630/44000 | Loss: 2.9422 | LR: 2.63e-05 | 710.5ms/step | 28.1 samples/s +Step 37640/44000 | Loss: 2.9510 | LR: 2.62e-05 | 711.5ms/step | 28.1 samples/s +Step 37650/44000 | Loss: 2.9269 | LR: 2.61e-05 | 710.5ms/step | 28.1 samples/s +Step 37660/44000 | Loss: 2.9680 | LR: 2.60e-05 | 711.1ms/step | 28.1 samples/s +Step 37670/44000 | Loss: 2.9250 | LR: 2.60e-05 | 711.0ms/step | 28.1 samples/s +Step 37680/44000 | Loss: 2.8403 | LR: 2.59e-05 | 711.4ms/step | 28.1 samples/s +Step 37690/44000 | Loss: 2.8683 | LR: 2.58e-05 | 729.5ms/step | 27.4 samples/s +Step 37700/44000 | Loss: 2.9764 | LR: 2.57e-05 | 710.8ms/step | 28.1 samples/s +Step 37710/44000 | Loss: 2.9911 | LR: 2.56e-05 | 710.8ms/step | 28.1 samples/s +Step 37720/44000 | Loss: 2.8854 | LR: 2.56e-05 | 711.8ms/step | 28.1 samples/s +Step 37730/44000 | Loss: 2.9382 | LR: 2.55e-05 | 712.2ms/step | 28.1 samples/s +Step 37740/44000 | Loss: 2.8941 | LR: 2.54e-05 | 711.4ms/step | 28.1 samples/s +Step 37750/44000 | Loss: 2.9107 | LR: 2.53e-05 | 711.5ms/step | 28.1 samples/s +Validation at step 37750: Loss = 2.9005, Perplexity = 18.18 +Step 37760/44000 | Loss: 2.9987 | LR: 2.52e-05 | 932.1ms/step | 21.5 samples/s +Step 37770/44000 | Loss: 2.8624 | LR: 2.52e-05 | 720.6ms/step | 27.8 samples/s +Step 37780/44000 | Loss: 2.8929 | LR: 2.51e-05 | 715.8ms/step | 27.9 samples/s +Step 37790/44000 | Loss: 2.9169 | LR: 2.50e-05 | 711.9ms/step | 28.1 samples/s +Step 37800/44000 | Loss: 2.9628 | LR: 2.49e-05 | 712.6ms/step | 28.1 samples/s +Step 37810/44000 | Loss: 2.9890 | LR: 2.48e-05 | 712.7ms/step | 28.1 samples/s +Step 37820/44000 | Loss: 2.9236 | LR: 2.48e-05 | 711.2ms/step | 28.1 samples/s +Step 37830/44000 | Loss: 2.9484 | LR: 2.47e-05 | 712.8ms/step | 28.1 samples/s +Step 37840/44000 | Loss: 2.9079 | LR: 2.46e-05 | 715.3ms/step | 28.0 samples/s +Step 37850/44000 | Loss: 2.9938 | LR: 2.45e-05 | 713.1ms/step | 28.0 samples/s +Step 37860/44000 | Loss: 2.8680 | LR: 2.45e-05 | 730.1ms/step | 27.4 samples/s +Step 37870/44000 | Loss: 2.9404 | LR: 2.44e-05 | 712.4ms/step | 28.1 samples/s +Step 37880/44000 | Loss: 2.8891 | LR: 2.43e-05 | 712.8ms/step | 28.1 samples/s +Step 37890/44000 | Loss: 2.8614 | LR: 2.42e-05 | 713.9ms/step | 28.0 samples/s +Step 37900/44000 | Loss: 2.9294 | LR: 2.41e-05 | 712.0ms/step | 28.1 samples/s +Step 37910/44000 | Loss: 2.8648 | LR: 2.41e-05 | 711.1ms/step | 28.1 samples/s +Step 37920/44000 | Loss: 2.9598 | LR: 2.40e-05 | 711.0ms/step | 28.1 samples/s +Step 37930/44000 | Loss: 2.8910 | LR: 2.39e-05 | 710.7ms/step | 28.1 samples/s +Step 37940/44000 | Loss: 2.8881 | LR: 2.38e-05 | 712.2ms/step | 28.1 samples/s +Step 37950/44000 | Loss: 2.9684 | LR: 2.38e-05 | 726.4ms/step | 27.5 samples/s +Step 37960/44000 | Loss: 2.9266 | LR: 2.37e-05 | 711.2ms/step | 28.1 samples/s +Step 37970/44000 | Loss: 2.8980 | LR: 2.36e-05 | 711.7ms/step | 28.1 samples/s +Step 37980/44000 | Loss: 2.9062 | LR: 2.35e-05 | 711.5ms/step | 28.1 samples/s +Step 37990/44000 | Loss: 2.9112 | LR: 2.34e-05 | 712.7ms/step | 28.1 samples/s +Step 38000/44000 | Loss: 2.9453 | LR: 2.34e-05 | 712.2ms/step | 28.1 samples/s +Validation at step 38000: Loss = 2.9004, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-38000 +Step 38010/44000 | Loss: 2.8893 | LR: 2.33e-05 | 1272.6ms/step | 15.7 samples/s +Step 38020/44000 | Loss: 2.9159 | LR: 2.32e-05 | 710.3ms/step | 28.2 samples/s +Step 38030/44000 | Loss: 2.9258 | LR: 2.31e-05 | 727.6ms/step | 27.5 samples/s +Step 38040/44000 | Loss: 2.8947 | LR: 2.31e-05 | 709.9ms/step | 28.2 samples/s +Step 38050/44000 | Loss: 2.9033 | LR: 2.30e-05 | 709.9ms/step | 28.2 samples/s +Step 38060/44000 | Loss: 2.9471 | LR: 2.29e-05 | 709.9ms/step | 28.2 samples/s +Step 38070/44000 | Loss: 2.9624 | LR: 2.28e-05 | 710.4ms/step | 28.2 samples/s +Step 38080/44000 | Loss: 2.9462 | LR: 2.28e-05 | 708.8ms/step | 28.2 samples/s +Step 38090/44000 | Loss: 2.9160 | LR: 2.27e-05 | 710.3ms/step | 28.2 samples/s +Step 38100/44000 | Loss: 2.9421 | LR: 2.26e-05 | 709.3ms/step | 28.2 samples/s +Step 38110/44000 | Loss: 2.8708 | LR: 2.25e-05 | 710.8ms/step | 28.1 samples/s +Step 38120/44000 | Loss: 2.8847 | LR: 2.25e-05 | 728.5ms/step | 27.5 samples/s +Step 38130/44000 | Loss: 2.8982 | LR: 2.24e-05 | 712.0ms/step | 28.1 samples/s +Step 38140/44000 | Loss: 2.9067 | LR: 2.23e-05 | 710.5ms/step | 28.2 samples/s +Step 38150/44000 | Loss: 2.9395 | LR: 2.22e-05 | 710.5ms/step | 28.1 samples/s +Step 38160/44000 | Loss: 2.8812 | LR: 2.22e-05 | 709.9ms/step | 28.2 samples/s +Step 38170/44000 | Loss: 2.9075 | LR: 2.21e-05 | 708.9ms/step | 28.2 samples/s +Step 38180/44000 | Loss: 2.9178 | LR: 2.20e-05 | 710.3ms/step | 28.2 samples/s +Step 38190/44000 | Loss: 2.9642 | LR: 2.19e-05 | 710.1ms/step | 28.2 samples/s +Step 38200/44000 | Loss: 2.9188 | LR: 2.19e-05 | 709.6ms/step | 28.2 samples/s +Step 38210/44000 | Loss: 2.9236 | LR: 2.18e-05 | 728.7ms/step | 27.4 samples/s +Step 38220/44000 | Loss: 2.9331 | LR: 2.17e-05 | 710.0ms/step | 28.2 samples/s +Step 38230/44000 | Loss: 2.8876 | LR: 2.16e-05 | 709.9ms/step | 28.2 samples/s +Step 38240/44000 | Loss: 2.9398 | LR: 2.16e-05 | 710.5ms/step | 28.1 samples/s +Step 38250/44000 | Loss: 2.8682 | LR: 2.15e-05 | 710.7ms/step | 28.1 samples/s +Validation at step 38250: Loss = 2.9004, Perplexity = 18.18 +Step 38260/44000 | Loss: 2.9106 | LR: 2.14e-05 | 929.9ms/step | 21.5 samples/s +Step 38270/44000 | Loss: 2.9178 | LR: 2.13e-05 | 710.7ms/step | 28.1 samples/s +Step 38280/44000 | Loss: 2.9631 | LR: 2.13e-05 | 708.8ms/step | 28.2 samples/s +Step 38290/44000 | Loss: 2.8730 | LR: 2.12e-05 | 714.9ms/step | 28.0 samples/s +Step 38300/44000 | Loss: 2.9266 | LR: 2.11e-05 | 724.4ms/step | 27.6 samples/s +Step 38310/44000 | Loss: 2.9798 | LR: 2.11e-05 | 710.6ms/step | 28.1 samples/s +Step 38320/44000 | Loss: 2.9727 | LR: 2.10e-05 | 710.9ms/step | 28.1 samples/s +Step 38330/44000 | Loss: 2.9222 | LR: 2.09e-05 | 710.0ms/step | 28.2 samples/s +Step 38340/44000 | Loss: 2.9383 | LR: 2.08e-05 | 710.0ms/step | 28.2 samples/s +Step 38350/44000 | Loss: 2.9391 | LR: 2.08e-05 | 709.7ms/step | 28.2 samples/s +Step 38360/44000 | Loss: 2.8991 | LR: 2.07e-05 | 711.3ms/step | 28.1 samples/s +Step 38370/44000 | Loss: 2.9832 | LR: 2.06e-05 | 711.0ms/step | 28.1 samples/s +Step 38380/44000 | Loss: 2.9213 | LR: 2.05e-05 | 721.3ms/step | 27.7 samples/s +Step 38390/44000 | Loss: 2.9172 | LR: 2.05e-05 | 716.9ms/step | 27.9 samples/s +Step 38400/44000 | Loss: 2.9027 | LR: 2.04e-05 | 711.6ms/step | 28.1 samples/s +Step 38410/44000 | Loss: 2.9381 | LR: 2.03e-05 | 710.5ms/step | 28.2 samples/s +Step 38420/44000 | Loss: 2.8538 | LR: 2.03e-05 | 710.6ms/step | 28.1 samples/s +Step 38430/44000 | Loss: 2.9228 | LR: 2.02e-05 | 709.1ms/step | 28.2 samples/s +Step 38440/44000 | Loss: 2.9660 | LR: 2.01e-05 | 711.1ms/step | 28.1 samples/s +Step 38450/44000 | Loss: 2.9336 | LR: 2.00e-05 | 710.6ms/step | 28.1 samples/s +Step 38460/44000 | Loss: 2.9330 | LR: 2.00e-05 | 709.9ms/step | 28.2 samples/s +Step 38470/44000 | Loss: 2.8876 | LR: 1.99e-05 | 719.2ms/step | 27.8 samples/s +Step 38480/44000 | Loss: 2.9602 | LR: 1.98e-05 | 714.0ms/step | 28.0 samples/s +Step 38490/44000 | Loss: 2.9146 | LR: 1.98e-05 | 709.7ms/step | 28.2 samples/s +Step 38500/44000 | Loss: 2.9233 | LR: 1.97e-05 | 711.1ms/step | 28.1 samples/s +Validation at step 38500: Loss = 2.9002, Perplexity = 18.18 +Step 38510/44000 | Loss: 2.9024 | LR: 1.96e-05 | 932.1ms/step | 21.5 samples/s +Step 38520/44000 | Loss: 2.9541 | LR: 1.95e-05 | 711.8ms/step | 28.1 samples/s +Step 38530/44000 | Loss: 2.8555 | LR: 1.95e-05 | 710.5ms/step | 28.2 samples/s +Step 38540/44000 | Loss: 2.9137 | LR: 1.94e-05 | 711.5ms/step | 28.1 samples/s +Step 38550/44000 | Loss: 2.9071 | LR: 1.93e-05 | 710.5ms/step | 28.1 samples/s +Step 38560/44000 | Loss: 2.9464 | LR: 1.93e-05 | 725.7ms/step | 27.6 samples/s +Step 38570/44000 | Loss: 2.8220 | LR: 1.92e-05 | 711.8ms/step | 28.1 samples/s +Step 38580/44000 | Loss: 2.9253 | LR: 1.91e-05 | 710.7ms/step | 28.1 samples/s +Step 38590/44000 | Loss: 2.9345 | LR: 1.91e-05 | 711.3ms/step | 28.1 samples/s +Step 38600/44000 | Loss: 2.9585 | LR: 1.90e-05 | 710.3ms/step | 28.2 samples/s +Step 38610/44000 | Loss: 2.8801 | LR: 1.89e-05 | 710.3ms/step | 28.2 samples/s +Step 38620/44000 | Loss: 2.9496 | LR: 1.88e-05 | 710.9ms/step | 28.1 samples/s +Step 38630/44000 | Loss: 2.9311 | LR: 1.88e-05 | 710.7ms/step | 28.1 samples/s +Step 38640/44000 | Loss: 2.9715 | LR: 1.87e-05 | 709.5ms/step | 28.2 samples/s +Step 38650/44000 | Loss: 2.9272 | LR: 1.86e-05 | 723.8ms/step | 27.6 samples/s +Step 38660/44000 | Loss: 2.9158 | LR: 1.86e-05 | 709.9ms/step | 28.2 samples/s +Step 38670/44000 | Loss: 2.8410 | LR: 1.85e-05 | 709.5ms/step | 28.2 samples/s +Step 38680/44000 | Loss: 2.9487 | LR: 1.84e-05 | 710.7ms/step | 28.1 samples/s +Step 38690/44000 | Loss: 2.8940 | LR: 1.84e-05 | 710.0ms/step | 28.2 samples/s +Step 38700/44000 | Loss: 2.9504 | LR: 1.83e-05 | 710.5ms/step | 28.1 samples/s +Step 38710/44000 | Loss: 2.8499 | LR: 1.82e-05 | 712.9ms/step | 28.1 samples/s +Step 38720/44000 | Loss: 2.9367 | LR: 1.82e-05 | 711.1ms/step | 28.1 samples/s +Step 38730/44000 | Loss: 2.7715 | LR: 1.81e-05 | 709.6ms/step | 28.2 samples/s +Step 38740/44000 | Loss: 2.8344 | LR: 1.80e-05 | 726.1ms/step | 27.5 samples/s +Step 38750/44000 | Loss: 2.9407 | LR: 1.80e-05 | 710.2ms/step | 28.2 samples/s +Validation at step 38750: Loss = 2.9003, Perplexity = 18.18 +Step 38760/44000 | Loss: 2.9098 | LR: 1.79e-05 | 929.0ms/step | 21.5 samples/s +Step 38770/44000 | Loss: 2.9192 | LR: 1.78e-05 | 709.2ms/step | 28.2 samples/s +Step 38780/44000 | Loss: 2.9473 | LR: 1.78e-05 | 709.0ms/step | 28.2 samples/s +Step 38790/44000 | Loss: 2.9698 | LR: 1.77e-05 | 708.7ms/step | 28.2 samples/s +Step 38800/44000 | Loss: 2.9328 | LR: 1.76e-05 | 709.2ms/step | 28.2 samples/s +Step 38810/44000 | Loss: 2.9146 | LR: 1.76e-05 | 709.0ms/step | 28.2 samples/s +Step 38820/44000 | Loss: 2.8776 | LR: 1.75e-05 | 720.7ms/step | 27.8 samples/s +Step 38830/44000 | Loss: 2.8551 | LR: 1.74e-05 | 716.0ms/step | 27.9 samples/s +Step 38840/44000 | Loss: 2.8779 | LR: 1.74e-05 | 712.2ms/step | 28.1 samples/s +Step 38850/44000 | Loss: 2.9161 | LR: 1.73e-05 | 711.3ms/step | 28.1 samples/s +Step 38860/44000 | Loss: 2.9420 | LR: 1.72e-05 | 709.8ms/step | 28.2 samples/s +Step 38870/44000 | Loss: 2.9316 | LR: 1.72e-05 | 711.7ms/step | 28.1 samples/s +Step 38880/44000 | Loss: 2.9008 | LR: 1.71e-05 | 712.6ms/step | 28.1 samples/s +Step 38890/44000 | Loss: 2.8962 | LR: 1.70e-05 | 713.3ms/step | 28.0 samples/s +Step 38900/44000 | Loss: 2.9562 | LR: 1.70e-05 | 714.7ms/step | 28.0 samples/s +Step 38910/44000 | Loss: 2.9263 | LR: 1.69e-05 | 724.5ms/step | 27.6 samples/s +Step 38920/44000 | Loss: 2.9504 | LR: 1.68e-05 | 717.8ms/step | 27.9 samples/s +Step 38930/44000 | Loss: 2.9133 | LR: 1.68e-05 | 713.8ms/step | 28.0 samples/s +Step 38940/44000 | Loss: 2.8370 | LR: 1.67e-05 | 714.5ms/step | 28.0 samples/s +Step 38950/44000 | Loss: 2.9586 | LR: 1.66e-05 | 714.7ms/step | 28.0 samples/s +Step 38960/44000 | Loss: 2.9552 | LR: 1.66e-05 | 714.0ms/step | 28.0 samples/s +Step 38970/44000 | Loss: 2.9259 | LR: 1.65e-05 | 712.9ms/step | 28.1 samples/s +Step 38980/44000 | Loss: 2.9590 | LR: 1.64e-05 | 714.1ms/step | 28.0 samples/s +Step 38990/44000 | Loss: 2.8609 | LR: 1.64e-05 | 712.6ms/step | 28.1 samples/s +Step 39000/44000 | Loss: 2.9042 | LR: 1.63e-05 | 729.0ms/step | 27.4 samples/s +Validation at step 39000: Loss = 2.9002, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-39000 +Step 39010/44000 | Loss: 2.8688 | LR: 1.62e-05 | 1350.4ms/step | 14.8 samples/s +Step 39020/44000 | Loss: 2.9319 | LR: 1.62e-05 | 711.8ms/step | 28.1 samples/s +Step 39030/44000 | Loss: 2.9504 | LR: 1.61e-05 | 713.3ms/step | 28.0 samples/s +Step 39040/44000 | Loss: 2.9246 | LR: 1.61e-05 | 714.0ms/step | 28.0 samples/s +Step 39050/44000 | Loss: 2.9281 | LR: 1.60e-05 | 712.9ms/step | 28.1 samples/s +Step 39060/44000 | Loss: 2.9466 | LR: 1.59e-05 | 713.6ms/step | 28.0 samples/s +Step 39070/44000 | Loss: 2.8535 | LR: 1.59e-05 | 713.5ms/step | 28.0 samples/s +Step 39080/44000 | Loss: 2.9807 | LR: 1.58e-05 | 729.7ms/step | 27.4 samples/s +Step 39090/44000 | Loss: 2.8989 | LR: 1.57e-05 | 714.1ms/step | 28.0 samples/s +Step 39100/44000 | Loss: 2.8954 | LR: 1.57e-05 | 712.5ms/step | 28.1 samples/s +Step 39110/44000 | Loss: 2.9046 | LR: 1.56e-05 | 710.8ms/step | 28.1 samples/s +Step 39120/44000 | Loss: 2.9222 | LR: 1.55e-05 | 711.7ms/step | 28.1 samples/s +Step 39130/44000 | Loss: 2.9311 | LR: 1.55e-05 | 711.4ms/step | 28.1 samples/s +Step 39140/44000 | Loss: 2.9259 | LR: 1.54e-05 | 711.0ms/step | 28.1 samples/s +Step 39150/44000 | Loss: 2.9562 | LR: 1.54e-05 | 709.6ms/step | 28.2 samples/s +Step 39160/44000 | Loss: 2.9044 | LR: 1.53e-05 | 709.6ms/step | 28.2 samples/s +Step 39170/44000 | Loss: 2.9612 | LR: 1.52e-05 | 725.5ms/step | 27.6 samples/s +Step 39180/44000 | Loss: 2.9068 | LR: 1.52e-05 | 708.8ms/step | 28.2 samples/s +Step 39190/44000 | Loss: 2.9133 | LR: 1.51e-05 | 710.6ms/step | 28.1 samples/s +Step 39200/44000 | Loss: 2.9294 | LR: 1.50e-05 | 711.6ms/step | 28.1 samples/s +Step 39210/44000 | Loss: 2.8614 | LR: 1.50e-05 | 711.0ms/step | 28.1 samples/s +Step 39220/44000 | Loss: 2.9170 | LR: 1.49e-05 | 711.7ms/step | 28.1 samples/s +Step 39230/44000 | Loss: 2.8849 | LR: 1.49e-05 | 711.9ms/step | 28.1 samples/s +Step 39240/44000 | Loss: 2.9125 | LR: 1.48e-05 | 712.8ms/step | 28.1 samples/s +Step 39250/44000 | Loss: 2.9126 | LR: 1.47e-05 | 712.6ms/step | 28.1 samples/s +Validation at step 39250: Loss = 2.9002, Perplexity = 18.18 +Step 39260/44000 | Loss: 2.8811 | LR: 1.47e-05 | 948.9ms/step | 21.1 samples/s +Step 39270/44000 | Loss: 2.9214 | LR: 1.46e-05 | 711.3ms/step | 28.1 samples/s +Step 39280/44000 | Loss: 2.9018 | LR: 1.46e-05 | 711.7ms/step | 28.1 samples/s +Step 39290/44000 | Loss: 2.9358 | LR: 1.45e-05 | 712.5ms/step | 28.1 samples/s +Step 39300/44000 | Loss: 2.9187 | LR: 1.44e-05 | 711.6ms/step | 28.1 samples/s +Step 39310/44000 | Loss: 2.9688 | LR: 1.44e-05 | 712.5ms/step | 28.1 samples/s +Step 39320/44000 | Loss: 2.9018 | LR: 1.43e-05 | 710.2ms/step | 28.2 samples/s +Step 39330/44000 | Loss: 2.9766 | LR: 1.42e-05 | 711.9ms/step | 28.1 samples/s +Step 39340/44000 | Loss: 2.9548 | LR: 1.42e-05 | 716.8ms/step | 27.9 samples/s +Step 39350/44000 | Loss: 2.9700 | LR: 1.41e-05 | 722.6ms/step | 27.7 samples/s +Step 39360/44000 | Loss: 2.9071 | LR: 1.41e-05 | 712.3ms/step | 28.1 samples/s +Step 39370/44000 | Loss: 3.0127 | LR: 1.40e-05 | 711.4ms/step | 28.1 samples/s +Step 39380/44000 | Loss: 2.9551 | LR: 1.39e-05 | 711.1ms/step | 28.1 samples/s +Step 39390/44000 | Loss: 2.9353 | LR: 1.39e-05 | 712.0ms/step | 28.1 samples/s +Step 39400/44000 | Loss: 2.9471 | LR: 1.38e-05 | 711.0ms/step | 28.1 samples/s +Step 39410/44000 | Loss: 2.9430 | LR: 1.38e-05 | 712.4ms/step | 28.1 samples/s +Step 39420/44000 | Loss: 2.9257 | LR: 1.37e-05 | 711.8ms/step | 28.1 samples/s +Step 39430/44000 | Loss: 2.9163 | LR: 1.36e-05 | 722.6ms/step | 27.7 samples/s +Step 39440/44000 | Loss: 2.8747 | LR: 1.36e-05 | 715.1ms/step | 28.0 samples/s +Step 39450/44000 | Loss: 2.9356 | LR: 1.35e-05 | 710.7ms/step | 28.1 samples/s +Step 39460/44000 | Loss: 2.9559 | LR: 1.35e-05 | 711.9ms/step | 28.1 samples/s +Step 39470/44000 | Loss: 2.9299 | LR: 1.34e-05 | 711.7ms/step | 28.1 samples/s +Step 39480/44000 | Loss: 2.9464 | LR: 1.34e-05 | 711.8ms/step | 28.1 samples/s +Step 39490/44000 | Loss: 2.9162 | LR: 1.33e-05 | 710.9ms/step | 28.1 samples/s +Step 39500/44000 | Loss: 2.9573 | LR: 1.32e-05 | 711.8ms/step | 28.1 samples/s +Validation at step 39500: Loss = 2.9002, Perplexity = 18.18 +Step 39510/44000 | Loss: 2.9126 | LR: 1.32e-05 | 933.8ms/step | 21.4 samples/s +Step 39520/44000 | Loss: 2.9045 | LR: 1.31e-05 | 727.5ms/step | 27.5 samples/s +Step 39530/44000 | Loss: 2.9200 | LR: 1.31e-05 | 712.1ms/step | 28.1 samples/s +Step 39540/44000 | Loss: 2.9763 | LR: 1.30e-05 | 711.9ms/step | 28.1 samples/s +Step 39550/44000 | Loss: 2.8906 | LR: 1.29e-05 | 711.9ms/step | 28.1 samples/s +Step 39560/44000 | Loss: 2.9683 | LR: 1.29e-05 | 712.2ms/step | 28.1 samples/s +Step 39570/44000 | Loss: 2.9613 | LR: 1.28e-05 | 712.0ms/step | 28.1 samples/s +Step 39580/44000 | Loss: 2.8659 | LR: 1.28e-05 | 715.5ms/step | 28.0 samples/s +Step 39590/44000 | Loss: 2.9106 | LR: 1.27e-05 | 711.5ms/step | 28.1 samples/s +Step 39600/44000 | Loss: 3.0045 | LR: 1.27e-05 | 712.1ms/step | 28.1 samples/s +Step 39610/44000 | Loss: 2.8615 | LR: 1.26e-05 | 730.1ms/step | 27.4 samples/s +Step 39620/44000 | Loss: 2.9522 | LR: 1.25e-05 | 712.2ms/step | 28.1 samples/s +Step 39630/44000 | Loss: 2.9135 | LR: 1.25e-05 | 711.2ms/step | 28.1 samples/s +Step 39640/44000 | Loss: 2.9000 | LR: 1.24e-05 | 712.4ms/step | 28.1 samples/s +Step 39650/44000 | Loss: 2.8954 | LR: 1.24e-05 | 712.9ms/step | 28.1 samples/s +Step 39660/44000 | Loss: 2.9208 | LR: 1.23e-05 | 712.0ms/step | 28.1 samples/s +Step 39670/44000 | Loss: 2.9637 | LR: 1.23e-05 | 711.2ms/step | 28.1 samples/s +Step 39680/44000 | Loss: 2.9066 | LR: 1.22e-05 | 712.4ms/step | 28.1 samples/s +Step 39690/44000 | Loss: 2.9691 | LR: 1.22e-05 | 710.3ms/step | 28.2 samples/s +Step 39700/44000 | Loss: 2.9232 | LR: 1.21e-05 | 726.5ms/step | 27.5 samples/s +Step 39710/44000 | Loss: 2.9350 | LR: 1.20e-05 | 712.7ms/step | 28.1 samples/s +Step 39720/44000 | Loss: 2.8718 | LR: 1.20e-05 | 712.2ms/step | 28.1 samples/s +Step 39730/44000 | Loss: 2.9280 | LR: 1.19e-05 | 711.9ms/step | 28.1 samples/s +Step 39740/44000 | Loss: 2.9470 | LR: 1.19e-05 | 711.3ms/step | 28.1 samples/s +Step 39750/44000 | Loss: 2.8947 | LR: 1.18e-05 | 711.6ms/step | 28.1 samples/s +Validation at step 39750: Loss = 2.9002, Perplexity = 18.18 +Step 39760/44000 | Loss: 2.9378 | LR: 1.18e-05 | 930.1ms/step | 21.5 samples/s +Step 39770/44000 | Loss: 2.9083 | LR: 1.17e-05 | 712.0ms/step | 28.1 samples/s +Step 39780/44000 | Loss: 2.8737 | LR: 1.17e-05 | 719.9ms/step | 27.8 samples/s +Step 39790/44000 | Loss: 2.9500 | LR: 1.16e-05 | 717.9ms/step | 27.9 samples/s +Step 39800/44000 | Loss: 2.9558 | LR: 1.15e-05 | 711.4ms/step | 28.1 samples/s +Step 39810/44000 | Loss: 2.9948 | LR: 1.15e-05 | 711.4ms/step | 28.1 samples/s +Step 39820/44000 | Loss: 2.8985 | LR: 1.14e-05 | 711.4ms/step | 28.1 samples/s +Step 39830/44000 | Loss: 3.0226 | LR: 1.14e-05 | 712.3ms/step | 28.1 samples/s +Step 39840/44000 | Loss: 2.9091 | LR: 1.13e-05 | 711.4ms/step | 28.1 samples/s +Step 39850/44000 | Loss: 2.8866 | LR: 1.13e-05 | 712.7ms/step | 28.1 samples/s +Step 39860/44000 | Loss: 2.8730 | LR: 1.12e-05 | 709.6ms/step | 28.2 samples/s +Step 39870/44000 | Loss: 2.9878 | LR: 1.12e-05 | 719.0ms/step | 27.8 samples/s +Step 39880/44000 | Loss: 2.8710 | LR: 1.11e-05 | 715.5ms/step | 28.0 samples/s +Step 39890/44000 | Loss: 2.9253 | LR: 1.11e-05 | 709.3ms/step | 28.2 samples/s +Step 39900/44000 | Loss: 2.9133 | LR: 1.10e-05 | 712.6ms/step | 28.1 samples/s +Step 39910/44000 | Loss: 2.8626 | LR: 1.10e-05 | 713.7ms/step | 28.0 samples/s +Step 39920/44000 | Loss: 2.8151 | LR: 1.09e-05 | 713.0ms/step | 28.0 samples/s +Step 39930/44000 | Loss: 2.9513 | LR: 1.08e-05 | 714.6ms/step | 28.0 samples/s +Step 39940/44000 | Loss: 2.9145 | LR: 1.08e-05 | 714.9ms/step | 28.0 samples/s +Step 39950/44000 | Loss: 2.9039 | LR: 1.07e-05 | 714.6ms/step | 28.0 samples/s +Step 39960/44000 | Loss: 2.9029 | LR: 1.07e-05 | 728.3ms/step | 27.5 samples/s +Step 39970/44000 | Loss: 2.9226 | LR: 1.06e-05 | 720.3ms/step | 27.8 samples/s +Step 39980/44000 | Loss: 2.9384 | LR: 1.06e-05 | 716.3ms/step | 27.9 samples/s +Step 39990/44000 | Loss: 2.9622 | LR: 1.05e-05 | 714.8ms/step | 28.0 samples/s +Step 40000/44000 | Loss: 2.8937 | LR: 1.05e-05 | 714.9ms/step | 28.0 samples/s +Validation at step 40000: Loss = 2.9002, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-40000 +Step 40010/44000 | Loss: 2.9372 | LR: 1.04e-05 | 1296.5ms/step | 15.4 samples/s +Step 40020/44000 | Loss: 2.8821 | LR: 1.04e-05 | 709.3ms/step | 28.2 samples/s +Step 40030/44000 | Loss: 2.9474 | LR: 1.03e-05 | 712.3ms/step | 28.1 samples/s +Step 40040/44000 | Loss: 2.8660 | LR: 1.03e-05 | 723.9ms/step | 27.6 samples/s +Step 40050/44000 | Loss: 2.8953 | LR: 1.02e-05 | 716.7ms/step | 27.9 samples/s +Step 40060/44000 | Loss: 2.9093 | LR: 1.02e-05 | 712.2ms/step | 28.1 samples/s +Step 40070/44000 | Loss: 2.9207 | LR: 1.01e-05 | 711.8ms/step | 28.1 samples/s +Step 40080/44000 | Loss: 3.0554 | LR: 1.01e-05 | 713.4ms/step | 28.0 samples/s +Step 40090/44000 | Loss: 2.8816 | LR: 1.00e-05 | 713.0ms/step | 28.0 samples/s +Step 40100/44000 | Loss: 2.9132 | LR: 9.96e-06 | 713.0ms/step | 28.1 samples/s +Step 40110/44000 | Loss: 2.9427 | LR: 9.91e-06 | 713.4ms/step | 28.0 samples/s +Step 40120/44000 | Loss: 2.9186 | LR: 9.86e-06 | 713.8ms/step | 28.0 samples/s +Step 40130/44000 | Loss: 2.9628 | LR: 9.81e-06 | 727.2ms/step | 27.5 samples/s +Step 40140/44000 | Loss: 2.8873 | LR: 9.76e-06 | 713.1ms/step | 28.0 samples/s +Step 40150/44000 | Loss: 2.8600 | LR: 9.71e-06 | 712.7ms/step | 28.1 samples/s +Step 40160/44000 | Loss: 2.9540 | LR: 9.66e-06 | 715.7ms/step | 27.9 samples/s +Step 40170/44000 | Loss: 2.9317 | LR: 9.61e-06 | 712.6ms/step | 28.1 samples/s +Step 40180/44000 | Loss: 2.8465 | LR: 9.56e-06 | 712.2ms/step | 28.1 samples/s +Step 40190/44000 | Loss: 2.9495 | LR: 9.51e-06 | 712.4ms/step | 28.1 samples/s +Step 40200/44000 | Loss: 2.8885 | LR: 9.46e-06 | 712.5ms/step | 28.1 samples/s +Step 40210/44000 | Loss: 2.9319 | LR: 9.41e-06 | 712.8ms/step | 28.1 samples/s +Step 40220/44000 | Loss: 2.8903 | LR: 9.36e-06 | 731.2ms/step | 27.4 samples/s +Step 40230/44000 | Loss: 3.0041 | LR: 9.32e-06 | 712.2ms/step | 28.1 samples/s +Step 40240/44000 | Loss: 2.8938 | LR: 9.27e-06 | 712.3ms/step | 28.1 samples/s +Step 40250/44000 | Loss: 2.8876 | LR: 9.22e-06 | 712.5ms/step | 28.1 samples/s +Validation at step 40250: Loss = 2.9002, Perplexity = 18.18 +Step 40260/44000 | Loss: 2.9325 | LR: 9.17e-06 | 933.9ms/step | 21.4 samples/s +Step 40270/44000 | Loss: 2.9046 | LR: 9.12e-06 | 711.5ms/step | 28.1 samples/s +Step 40280/44000 | Loss: 2.9885 | LR: 9.07e-06 | 712.6ms/step | 28.1 samples/s +Step 40290/44000 | Loss: 2.8807 | LR: 9.02e-06 | 714.0ms/step | 28.0 samples/s +Step 40300/44000 | Loss: 2.8812 | LR: 8.97e-06 | 713.1ms/step | 28.0 samples/s +Step 40310/44000 | Loss: 2.9379 | LR: 8.93e-06 | 724.6ms/step | 27.6 samples/s +Step 40320/44000 | Loss: 2.8883 | LR: 8.88e-06 | 713.1ms/step | 28.0 samples/s +Step 40330/44000 | Loss: 2.9851 | LR: 8.83e-06 | 713.2ms/step | 28.0 samples/s +Step 40340/44000 | Loss: 2.9202 | LR: 8.78e-06 | 713.0ms/step | 28.1 samples/s +Step 40350/44000 | Loss: 2.9220 | LR: 8.74e-06 | 713.1ms/step | 28.0 samples/s +Step 40360/44000 | Loss: 2.9060 | LR: 8.69e-06 | 712.8ms/step | 28.1 samples/s +Step 40370/44000 | Loss: 2.9127 | LR: 8.64e-06 | 712.2ms/step | 28.1 samples/s +Step 40380/44000 | Loss: 2.9141 | LR: 8.59e-06 | 712.6ms/step | 28.1 samples/s +Step 40390/44000 | Loss: 2.9123 | LR: 8.55e-06 | 717.9ms/step | 27.9 samples/s +Step 40400/44000 | Loss: 2.9434 | LR: 8.50e-06 | 723.9ms/step | 27.6 samples/s +Step 40410/44000 | Loss: 2.9672 | LR: 8.45e-06 | 713.8ms/step | 28.0 samples/s +Step 40420/44000 | Loss: 2.9534 | LR: 8.41e-06 | 712.4ms/step | 28.1 samples/s +Step 40430/44000 | Loss: 2.9342 | LR: 8.36e-06 | 712.8ms/step | 28.1 samples/s +Step 40440/44000 | Loss: 2.8882 | LR: 8.31e-06 | 711.7ms/step | 28.1 samples/s +Step 40450/44000 | Loss: 2.9006 | LR: 8.27e-06 | 713.1ms/step | 28.0 samples/s +Step 40460/44000 | Loss: 2.9343 | LR: 8.22e-06 | 712.3ms/step | 28.1 samples/s +Step 40470/44000 | Loss: 2.9721 | LR: 8.17e-06 | 712.4ms/step | 28.1 samples/s +Step 40480/44000 | Loss: 2.8779 | LR: 8.13e-06 | 723.8ms/step | 27.6 samples/s +Step 40490/44000 | Loss: 2.8764 | LR: 8.08e-06 | 715.9ms/step | 27.9 samples/s +Step 40500/44000 | Loss: 2.8731 | LR: 8.04e-06 | 713.0ms/step | 28.0 samples/s +Validation at step 40500: Loss = 2.9001, Perplexity = 18.18 +Step 40510/44000 | Loss: 2.8801 | LR: 7.99e-06 | 932.6ms/step | 21.4 samples/s +Step 40520/44000 | Loss: 2.9317 | LR: 7.94e-06 | 712.5ms/step | 28.1 samples/s +Step 40530/44000 | Loss: 2.9401 | LR: 7.90e-06 | 712.6ms/step | 28.1 samples/s +Step 40540/44000 | Loss: 2.9012 | LR: 7.85e-06 | 712.7ms/step | 28.1 samples/s +Step 40550/44000 | Loss: 2.9830 | LR: 7.81e-06 | 711.8ms/step | 28.1 samples/s +Step 40560/44000 | Loss: 2.9359 | LR: 7.76e-06 | 712.7ms/step | 28.1 samples/s +Step 40570/44000 | Loss: 2.9577 | LR: 7.72e-06 | 726.4ms/step | 27.5 samples/s +Step 40580/44000 | Loss: 2.9423 | LR: 7.67e-06 | 711.9ms/step | 28.1 samples/s +Step 40590/44000 | Loss: 2.8828 | LR: 7.63e-06 | 712.0ms/step | 28.1 samples/s +Step 40600/44000 | Loss: 2.9006 | LR: 7.59e-06 | 712.5ms/step | 28.1 samples/s +Step 40610/44000 | Loss: 2.9884 | LR: 7.54e-06 | 712.4ms/step | 28.1 samples/s +Step 40620/44000 | Loss: 2.9142 | LR: 7.50e-06 | 711.6ms/step | 28.1 samples/s +Step 40630/44000 | Loss: 2.8763 | LR: 7.45e-06 | 711.4ms/step | 28.1 samples/s +Step 40640/44000 | Loss: 2.9243 | LR: 7.41e-06 | 711.9ms/step | 28.1 samples/s +Step 40650/44000 | Loss: 2.8872 | LR: 7.37e-06 | 716.3ms/step | 27.9 samples/s +Step 40660/44000 | Loss: 2.9508 | LR: 7.32e-06 | 733.0ms/step | 27.3 samples/s +Step 40670/44000 | Loss: 2.8600 | LR: 7.28e-06 | 713.5ms/step | 28.0 samples/s +Step 40680/44000 | Loss: 2.8931 | LR: 7.23e-06 | 711.9ms/step | 28.1 samples/s +Step 40690/44000 | Loss: 2.8841 | LR: 7.19e-06 | 712.1ms/step | 28.1 samples/s +Step 40700/44000 | Loss: 2.9737 | LR: 7.15e-06 | 712.3ms/step | 28.1 samples/s +Step 40710/44000 | Loss: 2.8778 | LR: 7.10e-06 | 713.0ms/step | 28.1 samples/s +Step 40720/44000 | Loss: 2.9268 | LR: 7.06e-06 | 711.7ms/step | 28.1 samples/s +Step 40730/44000 | Loss: 2.8878 | LR: 7.02e-06 | 712.1ms/step | 28.1 samples/s +Step 40740/44000 | Loss: 2.9980 | LR: 6.98e-06 | 711.9ms/step | 28.1 samples/s +Step 40750/44000 | Loss: 2.9366 | LR: 6.93e-06 | 728.4ms/step | 27.5 samples/s +Validation at step 40750: Loss = 2.9001, Perplexity = 18.18 +Step 40760/44000 | Loss: 2.8876 | LR: 6.89e-06 | 931.0ms/step | 21.5 samples/s +Step 40770/44000 | Loss: 2.9353 | LR: 6.85e-06 | 711.5ms/step | 28.1 samples/s +Step 40780/44000 | Loss: 2.9262 | LR: 6.81e-06 | 712.3ms/step | 28.1 samples/s +Step 40790/44000 | Loss: 2.9072 | LR: 6.77e-06 | 712.0ms/step | 28.1 samples/s +Step 40800/44000 | Loss: 2.8460 | LR: 6.72e-06 | 711.6ms/step | 28.1 samples/s +Step 40810/44000 | Loss: 2.9008 | LR: 6.68e-06 | 712.2ms/step | 28.1 samples/s +Step 40820/44000 | Loss: 2.9326 | LR: 6.64e-06 | 716.3ms/step | 27.9 samples/s +Step 40830/44000 | Loss: 2.9058 | LR: 6.60e-06 | 726.9ms/step | 27.5 samples/s +Step 40840/44000 | Loss: 2.9255 | LR: 6.56e-06 | 717.9ms/step | 27.9 samples/s +Step 40850/44000 | Loss: 2.8279 | LR: 6.52e-06 | 710.6ms/step | 28.1 samples/s +Step 40860/44000 | Loss: 2.9399 | LR: 6.47e-06 | 713.6ms/step | 28.0 samples/s +Step 40870/44000 | Loss: 2.9222 | LR: 6.43e-06 | 716.5ms/step | 27.9 samples/s +Step 40880/44000 | Loss: 2.9076 | LR: 6.39e-06 | 715.4ms/step | 28.0 samples/s +Step 40890/44000 | Loss: 2.8786 | LR: 6.35e-06 | 714.9ms/step | 28.0 samples/s +Step 40900/44000 | Loss: 2.9243 | LR: 6.31e-06 | 714.9ms/step | 28.0 samples/s +Step 40910/44000 | Loss: 2.9266 | LR: 6.27e-06 | 719.1ms/step | 27.8 samples/s +Step 40920/44000 | Loss: 2.9196 | LR: 6.23e-06 | 730.9ms/step | 27.4 samples/s +Step 40930/44000 | Loss: 2.9496 | LR: 6.19e-06 | 720.0ms/step | 27.8 samples/s +Step 40940/44000 | Loss: 2.9098 | LR: 6.15e-06 | 714.5ms/step | 28.0 samples/s +Step 40950/44000 | Loss: 2.9053 | LR: 6.11e-06 | 715.8ms/step | 27.9 samples/s +Step 40960/44000 | Loss: 2.9667 | LR: 6.07e-06 | 717.3ms/step | 27.9 samples/s +Step 40970/44000 | Loss: 2.8743 | LR: 6.03e-06 | 715.6ms/step | 27.9 samples/s +Step 40980/44000 | Loss: 2.9445 | LR: 5.99e-06 | 717.3ms/step | 27.9 samples/s +Step 40990/44000 | Loss: 2.9241 | LR: 5.95e-06 | 715.3ms/step | 28.0 samples/s +Step 41000/44000 | Loss: 2.9219 | LR: 5.91e-06 | 715.6ms/step | 28.0 samples/s +Validation at step 41000: Loss = 2.9001, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-41000 +Step 41010/44000 | Loss: 2.9039 | LR: 5.87e-06 | 1303.2ms/step | 15.3 samples/s +Step 41020/44000 | Loss: 2.8968 | LR: 5.83e-06 | 715.5ms/step | 28.0 samples/s +Step 41030/44000 | Loss: 2.9100 | LR: 5.80e-06 | 715.9ms/step | 27.9 samples/s +Step 41040/44000 | Loss: 2.9608 | LR: 5.76e-06 | 715.2ms/step | 28.0 samples/s +Step 41050/44000 | Loss: 2.8856 | LR: 5.72e-06 | 713.8ms/step | 28.0 samples/s +Step 41060/44000 | Loss: 2.9017 | LR: 5.68e-06 | 713.0ms/step | 28.1 samples/s +Step 41070/44000 | Loss: 2.9325 | LR: 5.64e-06 | 713.8ms/step | 28.0 samples/s +Step 41080/44000 | Loss: 2.9962 | LR: 5.60e-06 | 715.2ms/step | 28.0 samples/s +Step 41090/44000 | Loss: 2.8885 | LR: 5.56e-06 | 724.6ms/step | 27.6 samples/s +Step 41100/44000 | Loss: 2.8824 | LR: 5.53e-06 | 721.9ms/step | 27.7 samples/s +Step 41110/44000 | Loss: 2.9560 | LR: 5.49e-06 | 718.0ms/step | 27.9 samples/s +Step 41120/44000 | Loss: 2.8257 | LR: 5.45e-06 | 717.7ms/step | 27.9 samples/s +Step 41130/44000 | Loss: 2.8968 | LR: 5.41e-06 | 719.6ms/step | 27.8 samples/s +Step 41140/44000 | Loss: 2.9338 | LR: 5.38e-06 | 714.8ms/step | 28.0 samples/s +Step 41150/44000 | Loss: 2.9490 | LR: 5.34e-06 | 713.8ms/step | 28.0 samples/s +Step 41160/44000 | Loss: 2.8764 | LR: 5.30e-06 | 714.2ms/step | 28.0 samples/s +Step 41170/44000 | Loss: 2.9471 | LR: 5.26e-06 | 715.8ms/step | 27.9 samples/s +Step 41180/44000 | Loss: 2.9263 | LR: 5.23e-06 | 731.8ms/step | 27.3 samples/s +Step 41190/44000 | Loss: 2.9501 | LR: 5.19e-06 | 717.9ms/step | 27.9 samples/s +Step 41200/44000 | Loss: 2.9650 | LR: 5.15e-06 | 715.0ms/step | 28.0 samples/s +Step 41210/44000 | Loss: 2.8958 | LR: 5.12e-06 | 715.0ms/step | 28.0 samples/s +Step 41220/44000 | Loss: 2.9066 | LR: 5.08e-06 | 714.9ms/step | 28.0 samples/s +Step 41230/44000 | Loss: 2.9269 | LR: 5.04e-06 | 715.2ms/step | 28.0 samples/s +Step 41240/44000 | Loss: 2.9434 | LR: 5.01e-06 | 714.8ms/step | 28.0 samples/s +Step 41250/44000 | Loss: 2.9099 | LR: 4.97e-06 | 715.2ms/step | 28.0 samples/s +Validation at step 41250: Loss = 2.9001, Perplexity = 18.18 +Step 41260/44000 | Loss: 2.8454 | LR: 4.94e-06 | 935.7ms/step | 21.4 samples/s +Step 41270/44000 | Loss: 2.8528 | LR: 4.90e-06 | 733.0ms/step | 27.3 samples/s +Step 41280/44000 | Loss: 2.9577 | LR: 4.86e-06 | 717.5ms/step | 27.9 samples/s +Step 41290/44000 | Loss: 2.9212 | LR: 4.83e-06 | 716.9ms/step | 27.9 samples/s +Step 41300/44000 | Loss: 2.8888 | LR: 4.79e-06 | 713.9ms/step | 28.0 samples/s +Step 41310/44000 | Loss: 2.9297 | LR: 4.76e-06 | 714.7ms/step | 28.0 samples/s +Step 41320/44000 | Loss: 2.8987 | LR: 4.72e-06 | 714.5ms/step | 28.0 samples/s +Step 41330/44000 | Loss: 2.9224 | LR: 4.69e-06 | 715.0ms/step | 28.0 samples/s +Step 41340/44000 | Loss: 2.9908 | LR: 4.65e-06 | 714.1ms/step | 28.0 samples/s +Step 41350/44000 | Loss: 2.8985 | LR: 4.62e-06 | 714.5ms/step | 28.0 samples/s +Step 41360/44000 | Loss: 2.9360 | LR: 4.58e-06 | 729.6ms/step | 27.4 samples/s +Step 41370/44000 | Loss: 2.9226 | LR: 4.55e-06 | 715.9ms/step | 27.9 samples/s +Step 41380/44000 | Loss: 2.9013 | LR: 4.51e-06 | 715.6ms/step | 27.9 samples/s +Step 41390/44000 | Loss: 2.9268 | LR: 4.48e-06 | 715.7ms/step | 27.9 samples/s +Step 41400/44000 | Loss: 2.9513 | LR: 4.45e-06 | 716.8ms/step | 27.9 samples/s +Step 41410/44000 | Loss: 2.9669 | LR: 4.41e-06 | 713.8ms/step | 28.0 samples/s +Step 41420/44000 | Loss: 2.9687 | LR: 4.38e-06 | 715.0ms/step | 28.0 samples/s +Step 41430/44000 | Loss: 2.9290 | LR: 4.34e-06 | 714.1ms/step | 28.0 samples/s +Step 41440/44000 | Loss: 2.9466 | LR: 4.31e-06 | 723.5ms/step | 27.6 samples/s +Step 41450/44000 | Loss: 2.9353 | LR: 4.28e-06 | 719.9ms/step | 27.8 samples/s +Step 41460/44000 | Loss: 2.9278 | LR: 4.24e-06 | 715.9ms/step | 27.9 samples/s +Step 41470/44000 | Loss: 2.9558 | LR: 4.21e-06 | 714.4ms/step | 28.0 samples/s +Step 41480/44000 | Loss: 2.9262 | LR: 4.18e-06 | 716.7ms/step | 27.9 samples/s +Step 41490/44000 | Loss: 2.9108 | LR: 4.14e-06 | 716.2ms/step | 27.9 samples/s +Step 41500/44000 | Loss: 2.9621 | LR: 4.11e-06 | 715.9ms/step | 27.9 samples/s +Validation at step 41500: Loss = 2.9002, Perplexity = 18.18 +Step 41510/44000 | Loss: 2.9097 | LR: 4.08e-06 | 937.0ms/step | 21.3 samples/s +Step 41520/44000 | Loss: 2.9294 | LR: 4.05e-06 | 715.8ms/step | 27.9 samples/s +Step 41530/44000 | Loss: 2.9145 | LR: 4.01e-06 | 730.9ms/step | 27.4 samples/s +Step 41540/44000 | Loss: 2.8766 | LR: 3.98e-06 | 715.4ms/step | 28.0 samples/s +Step 41550/44000 | Loss: 2.9429 | LR: 3.95e-06 | 715.6ms/step | 27.9 samples/s +Step 41560/44000 | Loss: 2.8590 | LR: 3.92e-06 | 715.1ms/step | 28.0 samples/s +Step 41570/44000 | Loss: 2.9126 | LR: 3.88e-06 | 714.2ms/step | 28.0 samples/s +Step 41580/44000 | Loss: 2.9370 | LR: 3.85e-06 | 715.6ms/step | 27.9 samples/s +Step 41590/44000 | Loss: 2.8654 | LR: 3.82e-06 | 714.8ms/step | 28.0 samples/s +Step 41600/44000 | Loss: 2.9643 | LR: 3.79e-06 | 716.6ms/step | 27.9 samples/s +Step 41610/44000 | Loss: 2.9285 | LR: 3.76e-06 | 717.1ms/step | 27.9 samples/s +Step 41620/44000 | Loss: 2.9321 | LR: 3.73e-06 | 731.3ms/step | 27.3 samples/s +Step 41630/44000 | Loss: 2.9737 | LR: 3.70e-06 | 715.1ms/step | 28.0 samples/s +Step 41640/44000 | Loss: 2.9344 | LR: 3.66e-06 | 715.2ms/step | 28.0 samples/s +Step 41650/44000 | Loss: 2.9366 | LR: 3.63e-06 | 715.1ms/step | 28.0 samples/s +Step 41660/44000 | Loss: 2.9205 | LR: 3.60e-06 | 717.3ms/step | 27.9 samples/s +Step 41670/44000 | Loss: 2.9124 | LR: 3.57e-06 | 713.9ms/step | 28.0 samples/s +Step 41680/44000 | Loss: 2.8998 | LR: 3.54e-06 | 715.8ms/step | 27.9 samples/s +Step 41690/44000 | Loss: 2.8982 | LR: 3.51e-06 | 715.6ms/step | 27.9 samples/s +Step 41700/44000 | Loss: 2.9188 | LR: 3.48e-06 | 714.8ms/step | 28.0 samples/s +Step 41710/44000 | Loss: 2.9197 | LR: 3.45e-06 | 728.3ms/step | 27.5 samples/s +Step 41720/44000 | Loss: 2.9266 | LR: 3.42e-06 | 715.1ms/step | 28.0 samples/s +Step 41730/44000 | Loss: 2.8884 | LR: 3.39e-06 | 718.4ms/step | 27.8 samples/s +Step 41740/44000 | Loss: 2.9214 | LR: 3.36e-06 | 717.9ms/step | 27.9 samples/s +Step 41750/44000 | Loss: 2.9041 | LR: 3.33e-06 | 715.4ms/step | 28.0 samples/s +Validation at step 41750: Loss = 2.9001, Perplexity = 18.18 +Step 41760/44000 | Loss: 2.9581 | LR: 3.30e-06 | 936.2ms/step | 21.4 samples/s +Step 41770/44000 | Loss: 2.9290 | LR: 3.27e-06 | 716.2ms/step | 27.9 samples/s +Step 41780/44000 | Loss: 2.8892 | LR: 3.24e-06 | 715.5ms/step | 28.0 samples/s +Step 41790/44000 | Loss: 2.9483 | LR: 3.21e-06 | 726.6ms/step | 27.5 samples/s +Step 41800/44000 | Loss: 2.9303 | LR: 3.19e-06 | 723.6ms/step | 27.6 samples/s +Step 41810/44000 | Loss: 2.8503 | LR: 3.16e-06 | 715.4ms/step | 28.0 samples/s +Step 41820/44000 | Loss: 2.9654 | LR: 3.13e-06 | 714.3ms/step | 28.0 samples/s +Step 41830/44000 | Loss: 2.8474 | LR: 3.10e-06 | 715.2ms/step | 28.0 samples/s +Step 41840/44000 | Loss: 2.9694 | LR: 3.07e-06 | 714.5ms/step | 28.0 samples/s +Step 41850/44000 | Loss: 2.9329 | LR: 3.04e-06 | 716.1ms/step | 27.9 samples/s +Step 41860/44000 | Loss: 2.8801 | LR: 3.01e-06 | 714.3ms/step | 28.0 samples/s +Step 41870/44000 | Loss: 2.8944 | LR: 2.99e-06 | 714.6ms/step | 28.0 samples/s +Step 41880/44000 | Loss: 2.9553 | LR: 2.96e-06 | 724.4ms/step | 27.6 samples/s +Step 41890/44000 | Loss: 2.9431 | LR: 2.93e-06 | 719.7ms/step | 27.8 samples/s +Step 41900/44000 | Loss: 2.8626 | LR: 2.90e-06 | 714.8ms/step | 28.0 samples/s +Step 41910/44000 | Loss: 2.9410 | LR: 2.88e-06 | 714.8ms/step | 28.0 samples/s +Step 41920/44000 | Loss: 2.8726 | LR: 2.85e-06 | 716.3ms/step | 27.9 samples/s +Step 41930/44000 | Loss: 2.9653 | LR: 2.82e-06 | 715.3ms/step | 28.0 samples/s +Step 41940/44000 | Loss: 2.9847 | LR: 2.79e-06 | 715.2ms/step | 28.0 samples/s +Step 41950/44000 | Loss: 2.9307 | LR: 2.77e-06 | 715.1ms/step | 28.0 samples/s +Step 41960/44000 | Loss: 2.9579 | LR: 2.74e-06 | 715.7ms/step | 27.9 samples/s +Step 41970/44000 | Loss: 2.9718 | LR: 2.71e-06 | 726.8ms/step | 27.5 samples/s +Step 41980/44000 | Loss: 2.9392 | LR: 2.69e-06 | 716.8ms/step | 27.9 samples/s +Step 41990/44000 | Loss: 2.8915 | LR: 2.66e-06 | 714.5ms/step | 28.0 samples/s +Step 42000/44000 | Loss: 2.9170 | LR: 2.63e-06 | 713.4ms/step | 28.0 samples/s +Validation at step 42000: Loss = 2.9001, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-42000 +Step 42010/44000 | Loss: 2.8713 | LR: 2.61e-06 | 1299.5ms/step | 15.4 samples/s +Step 42020/44000 | Loss: 2.9358 | LR: 2.58e-06 | 714.5ms/step | 28.0 samples/s +Step 42030/44000 | Loss: 2.9018 | LR: 2.56e-06 | 715.1ms/step | 28.0 samples/s +Step 42040/44000 | Loss: 2.8839 | LR: 2.53e-06 | 717.6ms/step | 27.9 samples/s +Step 42050/44000 | Loss: 2.8730 | LR: 2.50e-06 | 727.2ms/step | 27.5 samples/s +Step 42060/44000 | Loss: 2.8937 | LR: 2.48e-06 | 720.4ms/step | 27.8 samples/s +Step 42070/44000 | Loss: 2.9311 | LR: 2.45e-06 | 712.6ms/step | 28.1 samples/s +Step 42080/44000 | Loss: 2.8690 | LR: 2.43e-06 | 710.9ms/step | 28.1 samples/s +Step 42090/44000 | Loss: 2.9581 | LR: 2.40e-06 | 710.7ms/step | 28.1 samples/s +Step 42100/44000 | Loss: 2.8624 | LR: 2.38e-06 | 710.7ms/step | 28.1 samples/s +Step 42110/44000 | Loss: 2.8971 | LR: 2.35e-06 | 710.7ms/step | 28.1 samples/s +Step 42120/44000 | Loss: 2.9665 | LR: 2.33e-06 | 710.6ms/step | 28.1 samples/s +Step 42130/44000 | Loss: 2.9239 | LR: 2.30e-06 | 712.4ms/step | 28.1 samples/s +Step 42140/44000 | Loss: 2.8902 | LR: 2.28e-06 | 721.9ms/step | 27.7 samples/s +Step 42150/44000 | Loss: 2.9118 | LR: 2.25e-06 | 717.6ms/step | 27.9 samples/s +Step 42160/44000 | Loss: 2.9397 | LR: 2.23e-06 | 715.0ms/step | 28.0 samples/s +Step 42170/44000 | Loss: 2.9629 | LR: 2.21e-06 | 723.2ms/step | 27.7 samples/s +Step 42180/44000 | Loss: 2.8619 | LR: 2.18e-06 | 720.4ms/step | 27.8 samples/s +Step 42190/44000 | Loss: 2.9078 | LR: 2.16e-06 | 718.3ms/step | 27.8 samples/s +Step 42200/44000 | Loss: 2.8362 | LR: 2.13e-06 | 717.7ms/step | 27.9 samples/s +Step 42210/44000 | Loss: 2.8906 | LR: 2.11e-06 | 717.9ms/step | 27.9 samples/s +Step 42220/44000 | Loss: 2.8966 | LR: 2.09e-06 | 718.3ms/step | 27.8 samples/s +Step 42230/44000 | Loss: 2.9306 | LR: 2.06e-06 | 732.4ms/step | 27.3 samples/s +Step 42240/44000 | Loss: 2.8854 | LR: 2.04e-06 | 718.0ms/step | 27.9 samples/s +Step 42250/44000 | Loss: 2.9493 | LR: 2.02e-06 | 716.3ms/step | 27.9 samples/s +Validation at step 42250: Loss = 2.9001, Perplexity = 18.18 +Step 42260/44000 | Loss: 2.9460 | LR: 1.99e-06 | 942.2ms/step | 21.2 samples/s +Step 42270/44000 | Loss: 2.9175 | LR: 1.97e-06 | 718.3ms/step | 27.8 samples/s +Step 42280/44000 | Loss: 2.8556 | LR: 1.95e-06 | 718.0ms/step | 27.9 samples/s +Step 42290/44000 | Loss: 2.9028 | LR: 1.93e-06 | 717.9ms/step | 27.9 samples/s +Step 42300/44000 | Loss: 2.9510 | LR: 1.90e-06 | 716.7ms/step | 27.9 samples/s +Step 42310/44000 | Loss: 2.9179 | LR: 1.88e-06 | 717.6ms/step | 27.9 samples/s +Step 42320/44000 | Loss: 2.8831 | LR: 1.86e-06 | 733.1ms/step | 27.3 samples/s +Step 42330/44000 | Loss: 2.9287 | LR: 1.84e-06 | 719.1ms/step | 27.8 samples/s +Step 42340/44000 | Loss: 2.9354 | LR: 1.82e-06 | 717.8ms/step | 27.9 samples/s +Step 42350/44000 | Loss: 2.9011 | LR: 1.79e-06 | 716.3ms/step | 27.9 samples/s +Step 42360/44000 | Loss: 2.9076 | LR: 1.77e-06 | 717.8ms/step | 27.9 samples/s +Step 42370/44000 | Loss: 2.8906 | LR: 1.75e-06 | 718.1ms/step | 27.9 samples/s +Step 42380/44000 | Loss: 2.9312 | LR: 1.73e-06 | 719.4ms/step | 27.8 samples/s +Step 42390/44000 | Loss: 2.8789 | LR: 1.71e-06 | 717.6ms/step | 27.9 samples/s +Step 42400/44000 | Loss: 2.8272 | LR: 1.69e-06 | 726.6ms/step | 27.5 samples/s +Step 42410/44000 | Loss: 2.9206 | LR: 1.67e-06 | 719.9ms/step | 27.8 samples/s +Step 42420/44000 | Loss: 2.9024 | LR: 1.64e-06 | 715.6ms/step | 28.0 samples/s +Step 42430/44000 | Loss: 2.9106 | LR: 1.62e-06 | 720.0ms/step | 27.8 samples/s +Step 42440/44000 | Loss: 2.9202 | LR: 1.60e-06 | 717.9ms/step | 27.9 samples/s +Step 42450/44000 | Loss: 2.9233 | LR: 1.58e-06 | 716.7ms/step | 27.9 samples/s +Step 42460/44000 | Loss: 2.9661 | LR: 1.56e-06 | 715.1ms/step | 28.0 samples/s +Step 42470/44000 | Loss: 2.8503 | LR: 1.54e-06 | 716.2ms/step | 27.9 samples/s +Step 42480/44000 | Loss: 2.9373 | LR: 1.52e-06 | 718.2ms/step | 27.8 samples/s +Step 42490/44000 | Loss: 2.9050 | LR: 1.50e-06 | 726.6ms/step | 27.5 samples/s +Step 42500/44000 | Loss: 2.8664 | LR: 1.48e-06 | 721.4ms/step | 27.7 samples/s +Validation at step 42500: Loss = 2.9001, Perplexity = 18.18 +Step 42510/44000 | Loss: 2.8861 | LR: 1.46e-06 | 935.2ms/step | 21.4 samples/s +Step 42520/44000 | Loss: 2.9437 | LR: 1.44e-06 | 710.2ms/step | 28.2 samples/s +Step 42530/44000 | Loss: 2.9514 | LR: 1.42e-06 | 712.2ms/step | 28.1 samples/s +Step 42540/44000 | Loss: 2.8780 | LR: 1.40e-06 | 713.0ms/step | 28.1 samples/s +Step 42550/44000 | Loss: 2.9322 | LR: 1.39e-06 | 710.7ms/step | 28.1 samples/s +Step 42560/44000 | Loss: 2.9732 | LR: 1.37e-06 | 710.5ms/step | 28.2 samples/s +Step 42570/44000 | Loss: 2.9599 | LR: 1.35e-06 | 711.6ms/step | 28.1 samples/s +Step 42580/44000 | Loss: 2.9200 | LR: 1.33e-06 | 728.7ms/step | 27.4 samples/s +Step 42590/44000 | Loss: 2.9013 | LR: 1.31e-06 | 713.9ms/step | 28.0 samples/s +Step 42600/44000 | Loss: 2.9032 | LR: 1.29e-06 | 715.3ms/step | 28.0 samples/s +Step 42610/44000 | Loss: 2.9408 | LR: 1.27e-06 | 713.9ms/step | 28.0 samples/s +Step 42620/44000 | Loss: 2.9337 | LR: 1.25e-06 | 713.8ms/step | 28.0 samples/s +Step 42630/44000 | Loss: 2.9222 | LR: 1.24e-06 | 714.1ms/step | 28.0 samples/s +Step 42640/44000 | Loss: 2.8911 | LR: 1.22e-06 | 713.6ms/step | 28.0 samples/s +Step 42650/44000 | Loss: 2.8194 | LR: 1.20e-06 | 716.0ms/step | 27.9 samples/s +Step 42660/44000 | Loss: 2.8416 | LR: 1.18e-06 | 715.6ms/step | 27.9 samples/s +Step 42670/44000 | Loss: 2.9043 | LR: 1.17e-06 | 732.6ms/step | 27.3 samples/s +Step 42680/44000 | Loss: 2.8714 | LR: 1.15e-06 | 715.0ms/step | 28.0 samples/s +Step 42690/44000 | Loss: 3.0096 | LR: 1.13e-06 | 714.8ms/step | 28.0 samples/s +Step 42700/44000 | Loss: 2.9666 | LR: 1.11e-06 | 714.6ms/step | 28.0 samples/s +Step 42710/44000 | Loss: 2.9293 | LR: 1.10e-06 | 714.6ms/step | 28.0 samples/s +Step 42720/44000 | Loss: 2.8845 | LR: 1.08e-06 | 715.0ms/step | 28.0 samples/s +Step 42730/44000 | Loss: 2.9108 | LR: 1.06e-06 | 715.7ms/step | 27.9 samples/s +Step 42740/44000 | Loss: 2.9013 | LR: 1.05e-06 | 713.6ms/step | 28.0 samples/s +Step 42750/44000 | Loss: 2.8905 | LR: 1.03e-06 | 715.2ms/step | 28.0 samples/s +Validation at step 42750: Loss = 2.9001, Perplexity = 18.18 +Step 42760/44000 | Loss: 2.8797 | LR: 1.01e-06 | 950.6ms/step | 21.0 samples/s +Step 42770/44000 | Loss: 2.8909 | LR: 9.97e-07 | 714.5ms/step | 28.0 samples/s +Step 42780/44000 | Loss: 2.9002 | LR: 9.81e-07 | 714.8ms/step | 28.0 samples/s +Step 42790/44000 | Loss: 2.9617 | LR: 9.65e-07 | 713.7ms/step | 28.0 samples/s +Step 42800/44000 | Loss: 2.9357 | LR: 9.49e-07 | 714.8ms/step | 28.0 samples/s +Step 42810/44000 | Loss: 2.9038 | LR: 9.33e-07 | 713.2ms/step | 28.0 samples/s +Step 42820/44000 | Loss: 2.9686 | LR: 9.18e-07 | 715.1ms/step | 28.0 samples/s +Step 42830/44000 | Loss: 2.9191 | LR: 9.02e-07 | 714.3ms/step | 28.0 samples/s +Step 42840/44000 | Loss: 2.9225 | LR: 8.87e-07 | 727.0ms/step | 27.5 samples/s +Step 42850/44000 | Loss: 2.9687 | LR: 8.72e-07 | 714.8ms/step | 28.0 samples/s +Step 42860/44000 | Loss: 2.8465 | LR: 8.57e-07 | 714.4ms/step | 28.0 samples/s +Step 42870/44000 | Loss: 2.9670 | LR: 8.42e-07 | 715.4ms/step | 28.0 samples/s +Step 42880/44000 | Loss: 2.8761 | LR: 8.27e-07 | 713.8ms/step | 28.0 samples/s +Step 42890/44000 | Loss: 2.9322 | LR: 8.12e-07 | 714.5ms/step | 28.0 samples/s +Step 42900/44000 | Loss: 2.9142 | LR: 7.98e-07 | 715.4ms/step | 28.0 samples/s +Step 42910/44000 | Loss: 2.8851 | LR: 7.83e-07 | 713.9ms/step | 28.0 samples/s +Step 42920/44000 | Loss: 2.9588 | LR: 7.69e-07 | 714.0ms/step | 28.0 samples/s +Step 42930/44000 | Loss: 2.9209 | LR: 7.55e-07 | 731.7ms/step | 27.3 samples/s +Step 42940/44000 | Loss: 2.9473 | LR: 7.41e-07 | 715.5ms/step | 28.0 samples/s +Step 42950/44000 | Loss: 2.9370 | LR: 7.27e-07 | 715.8ms/step | 27.9 samples/s +Step 42960/44000 | Loss: 2.8977 | LR: 7.13e-07 | 716.1ms/step | 27.9 samples/s +Step 42970/44000 | Loss: 2.8757 | LR: 6.99e-07 | 715.2ms/step | 28.0 samples/s +Step 42980/44000 | Loss: 2.9074 | LR: 6.86e-07 | 714.0ms/step | 28.0 samples/s +Step 42990/44000 | Loss: 2.8928 | LR: 6.72e-07 | 714.3ms/step | 28.0 samples/s +Step 43000/44000 | Loss: 2.9144 | LR: 6.59e-07 | 714.7ms/step | 28.0 samples/s +Validation at step 43000: Loss = 2.9001, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-43000 +Step 43010/44000 | Loss: 2.9275 | LR: 6.46e-07 | 1400.8ms/step | 14.3 samples/s +Step 43020/44000 | Loss: 2.8891 | LR: 6.33e-07 | 713.3ms/step | 28.0 samples/s +Step 43030/44000 | Loss: 2.9301 | LR: 6.20e-07 | 712.6ms/step | 28.1 samples/s +Step 43040/44000 | Loss: 2.8498 | LR: 6.08e-07 | 711.3ms/step | 28.1 samples/s +Step 43050/44000 | Loss: 2.9777 | LR: 5.95e-07 | 711.2ms/step | 28.1 samples/s +Step 43060/44000 | Loss: 2.9454 | LR: 5.83e-07 | 713.0ms/step | 28.0 samples/s +Step 43070/44000 | Loss: 2.9634 | LR: 5.70e-07 | 713.4ms/step | 28.0 samples/s +Step 43080/44000 | Loss: 2.8471 | LR: 5.58e-07 | 714.0ms/step | 28.0 samples/s +Step 43090/44000 | Loss: 2.9438 | LR: 5.46e-07 | 714.9ms/step | 28.0 samples/s +Step 43100/44000 | Loss: 2.9913 | LR: 5.34e-07 | 725.9ms/step | 27.6 samples/s +Step 43110/44000 | Loss: 2.9354 | LR: 5.22e-07 | 713.1ms/step | 28.0 samples/s +Step 43120/44000 | Loss: 2.9107 | LR: 5.11e-07 | 714.0ms/step | 28.0 samples/s +Step 43130/44000 | Loss: 2.9358 | LR: 4.99e-07 | 712.8ms/step | 28.1 samples/s +Step 43140/44000 | Loss: 2.9829 | LR: 4.88e-07 | 712.4ms/step | 28.1 samples/s +Step 43150/44000 | Loss: 2.9424 | LR: 4.76e-07 | 713.9ms/step | 28.0 samples/s +Step 43160/44000 | Loss: 2.9032 | LR: 4.65e-07 | 714.3ms/step | 28.0 samples/s +Step 43170/44000 | Loss: 2.9460 | LR: 4.54e-07 | 712.9ms/step | 28.1 samples/s +Step 43180/44000 | Loss: 2.9203 | LR: 4.43e-07 | 714.0ms/step | 28.0 samples/s +Step 43190/44000 | Loss: 2.9344 | LR: 4.33e-07 | 727.8ms/step | 27.5 samples/s +Step 43200/44000 | Loss: 2.8863 | LR: 4.22e-07 | 712.7ms/step | 28.1 samples/s +Step 43210/44000 | Loss: 2.9607 | LR: 4.12e-07 | 711.3ms/step | 28.1 samples/s +Step 43220/44000 | Loss: 2.9061 | LR: 4.01e-07 | 711.2ms/step | 28.1 samples/s +Step 43230/44000 | Loss: 2.9038 | LR: 3.91e-07 | 711.7ms/step | 28.1 samples/s +Step 43240/44000 | Loss: 2.8859 | LR: 3.81e-07 | 711.2ms/step | 28.1 samples/s +Step 43250/44000 | Loss: 2.8563 | LR: 3.71e-07 | 712.8ms/step | 28.1 samples/s +Validation at step 43250: Loss = 2.9001, Perplexity = 18.18 +Step 43260/44000 | Loss: 2.8881 | LR: 3.61e-07 | 932.0ms/step | 21.5 samples/s +Step 43270/44000 | Loss: 2.9264 | LR: 3.51e-07 | 720.8ms/step | 27.7 samples/s +Step 43280/44000 | Loss: 2.8492 | LR: 3.42e-07 | 724.4ms/step | 27.6 samples/s +Step 43290/44000 | Loss: 2.9174 | LR: 3.32e-07 | 712.1ms/step | 28.1 samples/s +Step 43300/44000 | Loss: 2.8631 | LR: 3.23e-07 | 712.9ms/step | 28.1 samples/s +Step 43310/44000 | Loss: 2.9128 | LR: 3.14e-07 | 712.2ms/step | 28.1 samples/s +Step 43320/44000 | Loss: 2.9466 | LR: 3.05e-07 | 712.6ms/step | 28.1 samples/s +Step 43330/44000 | Loss: 2.9140 | LR: 2.96e-07 | 713.0ms/step | 28.0 samples/s +Step 43340/44000 | Loss: 2.8262 | LR: 2.87e-07 | 712.1ms/step | 28.1 samples/s +Step 43350/44000 | Loss: 2.9273 | LR: 2.79e-07 | 710.9ms/step | 28.1 samples/s +Step 43360/44000 | Loss: 2.8967 | LR: 2.70e-07 | 724.5ms/step | 27.6 samples/s +Step 43370/44000 | Loss: 2.8457 | LR: 2.62e-07 | 717.0ms/step | 27.9 samples/s +Step 43380/44000 | Loss: 2.9200 | LR: 2.53e-07 | 712.0ms/step | 28.1 samples/s +Step 43390/44000 | Loss: 2.9334 | LR: 2.45e-07 | 713.6ms/step | 28.0 samples/s +Step 43400/44000 | Loss: 3.0228 | LR: 2.37e-07 | 712.0ms/step | 28.1 samples/s +Step 43410/44000 | Loss: 2.9408 | LR: 2.30e-07 | 712.9ms/step | 28.1 samples/s +Step 43420/44000 | Loss: 2.9356 | LR: 2.22e-07 | 715.6ms/step | 27.9 samples/s +Step 43430/44000 | Loss: 2.8806 | LR: 2.14e-07 | 712.3ms/step | 28.1 samples/s +Step 43440/44000 | Loss: 2.9466 | LR: 2.07e-07 | 713.0ms/step | 28.0 samples/s +Step 43450/44000 | Loss: 2.8748 | LR: 1.99e-07 | 725.6ms/step | 27.6 samples/s +Step 43460/44000 | Loss: 2.9031 | LR: 1.92e-07 | 718.7ms/step | 27.8 samples/s +Step 43470/44000 | Loss: 2.9424 | LR: 1.85e-07 | 713.8ms/step | 28.0 samples/s +Step 43480/44000 | Loss: 2.9041 | LR: 1.78e-07 | 713.7ms/step | 28.0 samples/s +Step 43490/44000 | Loss: 2.9579 | LR: 1.72e-07 | 713.9ms/step | 28.0 samples/s +Step 43500/44000 | Loss: 2.9861 | LR: 1.65e-07 | 712.6ms/step | 28.1 samples/s +Validation at step 43500: Loss = 2.9001, Perplexity = 18.18 +Step 43510/44000 | Loss: 3.0348 | LR: 1.58e-07 | 933.2ms/step | 21.4 samples/s +Step 43520/44000 | Loss: 2.9515 | LR: 1.52e-07 | 713.5ms/step | 28.0 samples/s +Step 43530/44000 | Loss: 2.8866 | LR: 1.46e-07 | 714.3ms/step | 28.0 samples/s +Step 43540/44000 | Loss: 2.9538 | LR: 1.40e-07 | 729.3ms/step | 27.4 samples/s +Step 43550/44000 | Loss: 2.8804 | LR: 1.34e-07 | 714.9ms/step | 28.0 samples/s +Step 43560/44000 | Loss: 2.8455 | LR: 1.28e-07 | 715.4ms/step | 28.0 samples/s +Step 43570/44000 | Loss: 2.8697 | LR: 1.22e-07 | 714.0ms/step | 28.0 samples/s +Step 43580/44000 | Loss: 2.9407 | LR: 1.16e-07 | 713.6ms/step | 28.0 samples/s +Step 43590/44000 | Loss: 2.9577 | LR: 1.11e-07 | 715.4ms/step | 28.0 samples/s +Step 43600/44000 | Loss: 2.9086 | LR: 1.06e-07 | 715.5ms/step | 28.0 samples/s +Step 43610/44000 | Loss: 2.9321 | LR: 1.00e-07 | 713.5ms/step | 28.0 samples/s +Step 43620/44000 | Loss: 2.8424 | LR: 9.52e-08 | 714.0ms/step | 28.0 samples/s +Step 43630/44000 | Loss: 2.8826 | LR: 9.03e-08 | 731.8ms/step | 27.3 samples/s +Step 43640/44000 | Loss: 2.9464 | LR: 8.55e-08 | 712.6ms/step | 28.1 samples/s +Step 43650/44000 | Loss: 2.9626 | LR: 8.08e-08 | 714.1ms/step | 28.0 samples/s +Step 43660/44000 | Loss: 2.9794 | LR: 7.62e-08 | 713.0ms/step | 28.1 samples/s +Step 43670/44000 | Loss: 2.8929 | LR: 7.18e-08 | 713.0ms/step | 28.1 samples/s +Step 43680/44000 | Loss: 2.8754 | LR: 6.75e-08 | 713.1ms/step | 28.0 samples/s +Step 43690/44000 | Loss: 2.9776 | LR: 6.34e-08 | 713.3ms/step | 28.0 samples/s +Step 43700/44000 | Loss: 2.9697 | LR: 5.94e-08 | 713.6ms/step | 28.0 samples/s +Step 43710/44000 | Loss: 2.8575 | LR: 5.55e-08 | 713.9ms/step | 28.0 samples/s +Step 43720/44000 | Loss: 2.8615 | LR: 5.17e-08 | 729.5ms/step | 27.4 samples/s +Step 43730/44000 | Loss: 2.9060 | LR: 4.81e-08 | 714.6ms/step | 28.0 samples/s +Step 43740/44000 | Loss: 2.8912 | LR: 4.46e-08 | 712.6ms/step | 28.1 samples/s +Step 43750/44000 | Loss: 2.9476 | LR: 4.12e-08 | 712.9ms/step | 28.1 samples/s +Validation at step 43750: Loss = 2.9001, Perplexity = 18.18 +Step 43760/44000 | Loss: 2.9707 | LR: 3.80e-08 | 934.5ms/step | 21.4 samples/s +Step 43770/44000 | Loss: 2.8902 | LR: 3.49e-08 | 715.1ms/step | 28.0 samples/s +Step 43780/44000 | Loss: 2.8848 | LR: 3.19e-08 | 715.3ms/step | 28.0 samples/s +Step 43790/44000 | Loss: 2.9286 | LR: 2.91e-08 | 714.4ms/step | 28.0 samples/s +Step 43800/44000 | Loss: 2.9579 | LR: 2.64e-08 | 724.1ms/step | 27.6 samples/s +Step 43810/44000 | Loss: 2.9277 | LR: 2.38e-08 | 720.4ms/step | 27.8 samples/s +Step 43820/44000 | Loss: 2.9460 | LR: 2.14e-08 | 713.8ms/step | 28.0 samples/s +Step 43830/44000 | Loss: 2.9691 | LR: 1.91e-08 | 712.8ms/step | 28.1 samples/s +Step 43840/44000 | Loss: 2.9630 | LR: 1.69e-08 | 712.6ms/step | 28.1 samples/s +Step 43850/44000 | Loss: 2.8643 | LR: 1.48e-08 | 712.7ms/step | 28.1 samples/s +Step 43860/44000 | Loss: 2.8980 | LR: 1.29e-08 | 713.1ms/step | 28.0 samples/s +Step 43870/44000 | Loss: 2.9317 | LR: 1.11e-08 | 715.8ms/step | 27.9 samples/s +Step 43880/44000 | Loss: 2.9847 | LR: 9.50e-09 | 713.7ms/step | 28.0 samples/s +Step 43890/44000 | Loss: 2.8695 | LR: 7.98e-09 | 725.0ms/step | 27.6 samples/s +Step 43900/44000 | Loss: 2.9152 | LR: 6.60e-09 | 718.0ms/step | 27.9 samples/s +Step 43910/44000 | Loss: 2.8820 | LR: 5.34e-09 | 709.1ms/step | 28.2 samples/s +Step 43920/44000 | Loss: 2.9162 | LR: 4.22e-09 | 709.6ms/step | 28.2 samples/s +Step 43930/44000 | Loss: 2.9272 | LR: 3.23e-09 | 710.2ms/step | 28.2 samples/s +Step 43940/44000 | Loss: 2.9350 | LR: 2.37e-09 | 709.2ms/step | 28.2 samples/s +Step 43950/44000 | Loss: 2.9394 | LR: 1.65e-09 | 709.2ms/step | 28.2 samples/s +Step 43960/44000 | Loss: 2.9203 | LR: 1.06e-09 | 709.4ms/step | 28.2 samples/s +Step 43970/44000 | Loss: 2.9376 | LR: 5.94e-10 | 709.4ms/step | 28.2 samples/s +Step 43980/44000 | Loss: 2.9024 | LR: 2.64e-10 | 718.7ms/step | 27.8 samples/s +Step 43990/44000 | Loss: 2.9611 | LR: 6.60e-11 | 717.7ms/step | 27.9 samples/s +Step 44000/44000 | Loss: 2.9684 | LR: 0.00e+00 | 714.5ms/step | 28.0 samples/s +Validation at step 44000: Loss = 2.9001, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed47/run_20260822_043538/checkpoint-44000 +wandb: updating run metadata +wandb: uploading wandb-summary.json; uploading config.yaml; uploading output.log +wandb: uploading wandb-summary.json; uploading output.log +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▅▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▃▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ███▁████████▁███████████████▇███████████ +wandb: speed/steps_per_sec ██████████████▁█████████████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.90009 +wandb: eval/perplexity 18.1758 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 714.49835 +wandb: speed/samples_per_sec 27.99167 +wandb: speed/steps_per_sec 1.39958 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 DDP5 365m static 8k nrb35 seed47 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/xvizh3ih +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_043611-xvizh3ih/logs +Training complete! diff --git a/epfl_final/train_logs/p1_resume_h1002_20260822/l8k365m_resampled_nrb35_seed44.log b/epfl_final/train_logs/p1_resume_h1002_20260822/l8k365m_resampled_nrb35_seed44.log new file mode 100644 index 0000000000000000000000000000000000000000..2cb7620e907fe98e87fed0948a4c6600e93f27e3 --- /dev/null +++ b/epfl_final/train_logs/p1_resume_h1002_20260822/l8k365m_resampled_nrb35_seed44.log @@ -0,0 +1,2130 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-26000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 44, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 resume 365m resampled 8k nrb35 seed44", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "resampled", + "nrb35", + "seed44", + "resume", + "h1002", + "step26000" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-26000 +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-26000... +Preserving checkpoint LR state for exact resume: current=[0.00018492988117498478], schedule_base=[0.0005] +Resumed from checkpoint at step 26000 (epoch 0, micro_step 260000) +Resuming training from step 26000 +Resuming wandb run 27rw10jv +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 27rw10jv +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_043604-27rw10jv +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run P1 resume 365m resampled 8k nrb35 seed44 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/27rw10jv +Starting training... +Validation at step 26000: Loss = 2.9608, Perplexity = 19.31 +wandb: WARNING Tried to log to step 26000 that is less than the current step 26091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 26001: sparse_stochastic (configured sparse attention) +Step 26010/44000 | Loss: 2.9365 | LR: 1.85e-04 | 48038.3ms/step | 0.4 samples/s +wandb: WARNING Tried to log to step 26010 that is less than the current step 26091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26020/44000 | Loss: 2.9526 | LR: 1.85e-04 | 1325.6ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 26020 that is less than the current step 26091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26030/44000 | Loss: 2.9322 | LR: 1.84e-04 | 1324.4ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 26030 that is less than the current step 26091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26040/44000 | Loss: 2.9586 | LR: 1.84e-04 | 1322.4ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 26040 that is less than the current step 26091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26050/44000 | Loss: 2.9305 | LR: 1.84e-04 | 1325.6ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 26050 that is less than the current step 26091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26060/44000 | Loss: 2.9661 | LR: 1.84e-04 | 1327.6ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 26060 that is less than the current step 26091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26070/44000 | Loss: 2.9621 | LR: 1.84e-04 | 1330.6ms/step | 15.0 samples/s +wandb: WARNING Tried to log to step 26070 that is less than the current step 26091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26080/44000 | Loss: 2.9430 | LR: 1.84e-04 | 1326.8ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 26080 that is less than the current step 26091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26090/44000 | Loss: 2.9560 | LR: 1.83e-04 | 1331.4ms/step | 15.0 samples/s +wandb: WARNING Tried to log to step 26090 that is less than the current step 26091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26100/44000 | Loss: 2.9589 | LR: 1.83e-04 | 1327.6ms/step | 15.1 samples/s +Step 26110/44000 | Loss: 2.9600 | LR: 1.83e-04 | 1327.5ms/step | 15.1 samples/s +Step 26120/44000 | Loss: 2.9479 | LR: 1.83e-04 | 1329.1ms/step | 15.0 samples/s +Step 26130/44000 | Loss: 2.9321 | LR: 1.83e-04 | 1331.2ms/step | 15.0 samples/s +Step 26140/44000 | Loss: 2.9258 | LR: 1.82e-04 | 1331.8ms/step | 15.0 samples/s +Step 26150/44000 | Loss: 2.9907 | LR: 1.82e-04 | 1329.1ms/step | 15.0 samples/s +Step 26160/44000 | Loss: 2.9741 | LR: 1.82e-04 | 1330.1ms/step | 15.0 samples/s +Step 26170/44000 | Loss: 2.9182 | LR: 1.82e-04 | 1327.4ms/step | 15.1 samples/s +Step 26180/44000 | Loss: 2.9337 | LR: 1.82e-04 | 1333.4ms/step | 15.0 samples/s +Step 26190/44000 | Loss: 2.9183 | LR: 1.82e-04 | 1331.6ms/step | 15.0 samples/s +Step 26200/44000 | Loss: 2.9458 | LR: 1.81e-04 | 1325.6ms/step | 15.1 samples/s +Step 26210/44000 | Loss: 2.9278 | LR: 1.81e-04 | 1330.3ms/step | 15.0 samples/s +Step 26220/44000 | Loss: 2.9462 | LR: 1.81e-04 | 1326.9ms/step | 15.1 samples/s +Step 26230/44000 | Loss: 2.9135 | LR: 1.81e-04 | 1329.0ms/step | 15.0 samples/s +Step 26240/44000 | Loss: 2.9464 | LR: 1.81e-04 | 1325.2ms/step | 15.1 samples/s +Step 26250/44000 | Loss: 2.9484 | LR: 1.81e-04 | 1328.8ms/step | 15.1 samples/s +Validation at step 26250: Loss = 2.9593, Perplexity = 19.28 +Step 26260/44000 | Loss: 2.9520 | LR: 1.80e-04 | 3580.4ms/step | 5.6 samples/s +Step 26270/44000 | Loss: 2.9545 | LR: 1.80e-04 | 1334.0ms/step | 15.0 samples/s +Step 26280/44000 | Loss: 2.9565 | LR: 1.80e-04 | 1335.9ms/step | 15.0 samples/s +Step 26290/44000 | Loss: 2.9507 | LR: 1.80e-04 | 1331.5ms/step | 15.0 samples/s +Step 26300/44000 | Loss: 2.9402 | LR: 1.80e-04 | 1333.3ms/step | 15.0 samples/s +Step 26310/44000 | Loss: 2.9186 | LR: 1.80e-04 | 1334.3ms/step | 15.0 samples/s +Step 26320/44000 | Loss: 2.9291 | LR: 1.79e-04 | 1333.7ms/step | 15.0 samples/s +Step 26330/44000 | Loss: 2.9815 | LR: 1.79e-04 | 1339.1ms/step | 14.9 samples/s +Step 26340/44000 | Loss: 2.9417 | LR: 1.79e-04 | 1333.6ms/step | 15.0 samples/s +Step 26350/44000 | Loss: 2.9851 | LR: 1.79e-04 | 1340.1ms/step | 14.9 samples/s +Step 26360/44000 | Loss: 2.9697 | LR: 1.79e-04 | 1335.0ms/step | 15.0 samples/s +Step 26370/44000 | Loss: 2.9712 | LR: 1.78e-04 | 1330.6ms/step | 15.0 samples/s +Step 26380/44000 | Loss: 2.9639 | LR: 1.78e-04 | 1338.6ms/step | 14.9 samples/s +Step 26390/44000 | Loss: 2.9716 | LR: 1.78e-04 | 1333.9ms/step | 15.0 samples/s +Step 26400/44000 | Loss: 2.9750 | LR: 1.78e-04 | 1339.8ms/step | 14.9 samples/s +Step 26410/44000 | Loss: 2.9342 | LR: 1.78e-04 | 1341.1ms/step | 14.9 samples/s +Step 26420/44000 | Loss: 2.9495 | LR: 1.78e-04 | 1338.1ms/step | 14.9 samples/s +Step 26430/44000 | Loss: 2.9285 | LR: 1.77e-04 | 1337.7ms/step | 15.0 samples/s +Step 26440/44000 | Loss: 2.8921 | LR: 1.77e-04 | 1333.6ms/step | 15.0 samples/s +Step 26450/44000 | Loss: 2.9412 | LR: 1.77e-04 | 1333.3ms/step | 15.0 samples/s +Step 26460/44000 | Loss: 2.9465 | LR: 1.77e-04 | 1336.5ms/step | 15.0 samples/s +Step 26470/44000 | Loss: 2.9615 | LR: 1.77e-04 | 1333.5ms/step | 15.0 samples/s +Step 26480/44000 | Loss: 2.9509 | LR: 1.77e-04 | 1335.6ms/step | 15.0 samples/s +Step 26490/44000 | Loss: 2.9617 | LR: 1.76e-04 | 1332.7ms/step | 15.0 samples/s +Step 26500/44000 | Loss: 2.9302 | LR: 1.76e-04 | 1331.7ms/step | 15.0 samples/s +Validation at step 26500: Loss = 2.9576, Perplexity = 19.25 +Step 26510/44000 | Loss: 2.9342 | LR: 1.76e-04 | 3568.3ms/step | 5.6 samples/s +Step 26520/44000 | Loss: 2.9317 | LR: 1.76e-04 | 1331.2ms/step | 15.0 samples/s +Step 26530/44000 | Loss: 2.9313 | LR: 1.76e-04 | 1332.3ms/step | 15.0 samples/s +Step 26540/44000 | Loss: 2.9531 | LR: 1.76e-04 | 1330.1ms/step | 15.0 samples/s +Step 26550/44000 | Loss: 2.9467 | LR: 1.75e-04 | 1332.4ms/step | 15.0 samples/s +Step 26560/44000 | Loss: 2.9412 | LR: 1.75e-04 | 1332.1ms/step | 15.0 samples/s +Step 26570/44000 | Loss: 2.9595 | LR: 1.75e-04 | 1333.5ms/step | 15.0 samples/s +Step 26580/44000 | Loss: 2.9663 | LR: 1.75e-04 | 1334.5ms/step | 15.0 samples/s +Step 26590/44000 | Loss: 2.9503 | LR: 1.75e-04 | 1332.3ms/step | 15.0 samples/s +Step 26600/44000 | Loss: 2.9554 | LR: 1.74e-04 | 1336.6ms/step | 15.0 samples/s +Step 26610/44000 | Loss: 2.9352 | LR: 1.74e-04 | 1330.2ms/step | 15.0 samples/s +Step 26620/44000 | Loss: 2.9508 | LR: 1.74e-04 | 1332.2ms/step | 15.0 samples/s +Step 26630/44000 | Loss: 2.9435 | LR: 1.74e-04 | 1330.1ms/step | 15.0 samples/s +Step 26640/44000 | Loss: 2.9343 | LR: 1.74e-04 | 1332.4ms/step | 15.0 samples/s +Step 26650/44000 | Loss: 2.9801 | LR: 1.74e-04 | 1334.5ms/step | 15.0 samples/s +Step 26660/44000 | Loss: 2.9264 | LR: 1.73e-04 | 1332.6ms/step | 15.0 samples/s +Step 26670/44000 | Loss: 2.9642 | LR: 1.73e-04 | 1332.9ms/step | 15.0 samples/s +Step 26680/44000 | Loss: 2.9223 | LR: 1.73e-04 | 1335.0ms/step | 15.0 samples/s +Step 26690/44000 | Loss: 2.9293 | LR: 1.73e-04 | 1335.2ms/step | 15.0 samples/s +Step 26700/44000 | Loss: 2.9373 | LR: 1.73e-04 | 1329.4ms/step | 15.0 samples/s +Step 26710/44000 | Loss: 2.9592 | LR: 1.73e-04 | 1334.2ms/step | 15.0 samples/s +Step 26720/44000 | Loss: 2.9559 | LR: 1.72e-04 | 1333.2ms/step | 15.0 samples/s +Step 26730/44000 | Loss: 2.9268 | LR: 1.72e-04 | 1334.0ms/step | 15.0 samples/s +Step 26740/44000 | Loss: 2.9556 | LR: 1.72e-04 | 1330.4ms/step | 15.0 samples/s +Step 26750/44000 | Loss: 2.9763 | LR: 1.72e-04 | 1331.9ms/step | 15.0 samples/s +Validation at step 26750: Loss = 2.9564, Perplexity = 19.23 +Step 26760/44000 | Loss: 2.9518 | LR: 1.72e-04 | 3556.3ms/step | 5.6 samples/s +Step 26770/44000 | Loss: 2.9648 | LR: 1.72e-04 | 1326.7ms/step | 15.1 samples/s +Step 26780/44000 | Loss: 2.9519 | LR: 1.71e-04 | 1326.5ms/step | 15.1 samples/s +Step 26790/44000 | Loss: 2.9619 | LR: 1.71e-04 | 1328.1ms/step | 15.1 samples/s +Step 26800/44000 | Loss: 2.9661 | LR: 1.71e-04 | 1333.0ms/step | 15.0 samples/s +Step 26810/44000 | Loss: 2.9553 | LR: 1.71e-04 | 1331.9ms/step | 15.0 samples/s +Step 26820/44000 | Loss: 2.9466 | LR: 1.71e-04 | 1331.8ms/step | 15.0 samples/s +Step 26830/44000 | Loss: 2.9612 | LR: 1.71e-04 | 1330.7ms/step | 15.0 samples/s +Step 26840/44000 | Loss: 2.9221 | LR: 1.70e-04 | 1328.4ms/step | 15.1 samples/s +Step 26850/44000 | Loss: 2.9554 | LR: 1.70e-04 | 1330.7ms/step | 15.0 samples/s +Step 26860/44000 | Loss: 2.9458 | LR: 1.70e-04 | 1328.3ms/step | 15.1 samples/s +Step 26870/44000 | Loss: 2.9707 | LR: 1.70e-04 | 1335.0ms/step | 15.0 samples/s +Step 26880/44000 | Loss: 2.9461 | LR: 1.70e-04 | 1334.3ms/step | 15.0 samples/s +Step 26890/44000 | Loss: 2.9258 | LR: 1.69e-04 | 1331.1ms/step | 15.0 samples/s +Step 26900/44000 | Loss: 2.9349 | LR: 1.69e-04 | 1334.0ms/step | 15.0 samples/s +Step 26910/44000 | Loss: 2.9781 | LR: 1.69e-04 | 1328.2ms/step | 15.1 samples/s +Step 26920/44000 | Loss: 2.9512 | LR: 1.69e-04 | 1325.1ms/step | 15.1 samples/s +Step 26930/44000 | Loss: 2.9725 | LR: 1.69e-04 | 1327.0ms/step | 15.1 samples/s +Step 26940/44000 | Loss: 2.9494 | LR: 1.69e-04 | 1332.6ms/step | 15.0 samples/s +Step 26950/44000 | Loss: 2.9420 | LR: 1.68e-04 | 1331.7ms/step | 15.0 samples/s +Step 26960/44000 | Loss: 2.9317 | LR: 1.68e-04 | 1328.4ms/step | 15.1 samples/s +Step 26970/44000 | Loss: 2.9272 | LR: 1.68e-04 | 1331.6ms/step | 15.0 samples/s +Step 26980/44000 | Loss: 2.9277 | LR: 1.68e-04 | 1330.0ms/step | 15.0 samples/s +Step 26990/44000 | Loss: 2.9302 | LR: 1.68e-04 | 1329.6ms/step | 15.0 samples/s +Step 27000/44000 | Loss: 2.9383 | LR: 1.68e-04 | 1331.5ms/step | 15.0 samples/s +Validation at step 27000: Loss = 2.9553, Perplexity = 19.21 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-27000 +Step 27010/44000 | Loss: 2.9430 | LR: 1.67e-04 | 3852.6ms/step | 5.2 samples/s +Step 27020/44000 | Loss: 2.9362 | LR: 1.67e-04 | 1328.3ms/step | 15.1 samples/s +Step 27030/44000 | Loss: 2.9355 | LR: 1.67e-04 | 1324.4ms/step | 15.1 samples/s +Step 27040/44000 | Loss: 2.9231 | LR: 1.67e-04 | 1328.3ms/step | 15.1 samples/s +Step 27050/44000 | Loss: 2.9562 | LR: 1.67e-04 | 1325.7ms/step | 15.1 samples/s +Step 27060/44000 | Loss: 2.9509 | LR: 1.67e-04 | 1326.6ms/step | 15.1 samples/s +Step 27070/44000 | Loss: 2.9777 | LR: 1.66e-04 | 1327.7ms/step | 15.1 samples/s +Step 27080/44000 | Loss: 2.9380 | LR: 1.66e-04 | 1330.6ms/step | 15.0 samples/s +Step 27090/44000 | Loss: 2.9534 | LR: 1.66e-04 | 1328.0ms/step | 15.1 samples/s +Step 27100/44000 | Loss: 2.9404 | LR: 1.66e-04 | 1330.0ms/step | 15.0 samples/s +Step 27110/44000 | Loss: 2.9357 | LR: 1.66e-04 | 1327.9ms/step | 15.1 samples/s +Step 27120/44000 | Loss: 2.9524 | LR: 1.66e-04 | 1328.2ms/step | 15.1 samples/s +Step 27130/44000 | Loss: 2.9430 | LR: 1.65e-04 | 1328.6ms/step | 15.1 samples/s +Step 27140/44000 | Loss: 2.9607 | LR: 1.65e-04 | 1326.7ms/step | 15.1 samples/s +Step 27150/44000 | Loss: 2.9171 | LR: 1.65e-04 | 1329.4ms/step | 15.0 samples/s +Step 27160/44000 | Loss: 2.9518 | LR: 1.65e-04 | 1330.2ms/step | 15.0 samples/s +Step 27170/44000 | Loss: 2.9500 | LR: 1.65e-04 | 1328.2ms/step | 15.1 samples/s +Step 27180/44000 | Loss: 2.9200 | LR: 1.65e-04 | 1328.6ms/step | 15.1 samples/s +Step 27190/44000 | Loss: 2.9656 | LR: 1.64e-04 | 1328.8ms/step | 15.1 samples/s +Step 27200/44000 | Loss: 2.9452 | LR: 1.64e-04 | 1330.7ms/step | 15.0 samples/s +Step 27210/44000 | Loss: 2.9635 | LR: 1.64e-04 | 1328.9ms/step | 15.0 samples/s +Step 27220/44000 | Loss: 2.9236 | LR: 1.64e-04 | 1329.3ms/step | 15.0 samples/s +Step 27230/44000 | Loss: 2.9497 | LR: 1.64e-04 | 1329.1ms/step | 15.0 samples/s +Step 27240/44000 | Loss: 2.9341 | LR: 1.63e-04 | 1326.5ms/step | 15.1 samples/s +Step 27250/44000 | Loss: 2.9385 | LR: 1.63e-04 | 1330.7ms/step | 15.0 samples/s +Validation at step 27250: Loss = 2.9538, Perplexity = 19.18 +Step 27260/44000 | Loss: 2.9548 | LR: 1.63e-04 | 3551.0ms/step | 5.6 samples/s +Step 27270/44000 | Loss: 2.9694 | LR: 1.63e-04 | 1328.5ms/step | 15.1 samples/s +Step 27280/44000 | Loss: 2.9427 | LR: 1.63e-04 | 1327.6ms/step | 15.1 samples/s +Step 27290/44000 | Loss: 2.9607 | LR: 1.63e-04 | 1324.6ms/step | 15.1 samples/s +Step 27300/44000 | Loss: 2.9266 | LR: 1.62e-04 | 1324.6ms/step | 15.1 samples/s +Step 27310/44000 | Loss: 2.9379 | LR: 1.62e-04 | 1325.8ms/step | 15.1 samples/s +Step 27320/44000 | Loss: 2.9532 | LR: 1.62e-04 | 1324.1ms/step | 15.1 samples/s +Step 27330/44000 | Loss: 2.9447 | LR: 1.62e-04 | 1324.7ms/step | 15.1 samples/s +Step 27340/44000 | Loss: 2.9113 | LR: 1.62e-04 | 1324.9ms/step | 15.1 samples/s +Step 27350/44000 | Loss: 2.9497 | LR: 1.62e-04 | 1325.9ms/step | 15.1 samples/s +Step 27360/44000 | Loss: 2.9332 | LR: 1.61e-04 | 1323.7ms/step | 15.1 samples/s +Step 27370/44000 | Loss: 2.9418 | LR: 1.61e-04 | 1329.3ms/step | 15.0 samples/s +Step 27380/44000 | Loss: 2.9477 | LR: 1.61e-04 | 1330.9ms/step | 15.0 samples/s +Step 27390/44000 | Loss: 2.9373 | LR: 1.61e-04 | 1324.7ms/step | 15.1 samples/s +Step 27400/44000 | Loss: 2.9590 | LR: 1.61e-04 | 1326.2ms/step | 15.1 samples/s +Step 27410/44000 | Loss: 2.9529 | LR: 1.61e-04 | 1328.3ms/step | 15.1 samples/s +Step 27420/44000 | Loss: 2.9481 | LR: 1.60e-04 | 1324.4ms/step | 15.1 samples/s +Step 27430/44000 | Loss: 2.9417 | LR: 1.60e-04 | 1324.7ms/step | 15.1 samples/s +Step 27440/44000 | Loss: 2.9277 | LR: 1.60e-04 | 1327.7ms/step | 15.1 samples/s +Step 27450/44000 | Loss: 2.9405 | LR: 1.60e-04 | 1328.8ms/step | 15.1 samples/s +Step 27460/44000 | Loss: 2.9940 | LR: 1.60e-04 | 1327.7ms/step | 15.1 samples/s +Step 27470/44000 | Loss: 2.9446 | LR: 1.60e-04 | 1327.4ms/step | 15.1 samples/s +Step 27480/44000 | Loss: 2.9443 | LR: 1.59e-04 | 1325.7ms/step | 15.1 samples/s +Step 27490/44000 | Loss: 2.9447 | LR: 1.59e-04 | 1326.7ms/step | 15.1 samples/s +Step 27500/44000 | Loss: 2.9175 | LR: 1.59e-04 | 1328.7ms/step | 15.1 samples/s +Validation at step 27500: Loss = 2.9527, Perplexity = 19.16 +Step 27510/44000 | Loss: 2.9696 | LR: 1.59e-04 | 3555.3ms/step | 5.6 samples/s +Step 27520/44000 | Loss: 2.9489 | LR: 1.59e-04 | 1328.9ms/step | 15.1 samples/s +Step 27530/44000 | Loss: 2.9542 | LR: 1.59e-04 | 1324.6ms/step | 15.1 samples/s +Step 27540/44000 | Loss: 2.9514 | LR: 1.58e-04 | 1325.8ms/step | 15.1 samples/s +Step 27550/44000 | Loss: 2.9379 | LR: 1.58e-04 | 1326.9ms/step | 15.1 samples/s +Step 27560/44000 | Loss: 2.9255 | LR: 1.58e-04 | 1326.0ms/step | 15.1 samples/s +Step 27570/44000 | Loss: 2.9203 | LR: 1.58e-04 | 1325.0ms/step | 15.1 samples/s +Step 27580/44000 | Loss: 2.9754 | LR: 1.58e-04 | 1325.0ms/step | 15.1 samples/s +Step 27590/44000 | Loss: 2.9281 | LR: 1.58e-04 | 1324.8ms/step | 15.1 samples/s +Step 27600/44000 | Loss: 2.9525 | LR: 1.57e-04 | 1324.8ms/step | 15.1 samples/s +Step 27610/44000 | Loss: 2.9579 | LR: 1.57e-04 | 1323.5ms/step | 15.1 samples/s +Step 27620/44000 | Loss: 2.9499 | LR: 1.57e-04 | 1323.2ms/step | 15.1 samples/s +Step 27630/44000 | Loss: 2.9596 | LR: 1.57e-04 | 1324.5ms/step | 15.1 samples/s +Step 27640/44000 | Loss: 2.9500 | LR: 1.57e-04 | 1318.8ms/step | 15.2 samples/s +Step 27650/44000 | Loss: 2.9502 | LR: 1.57e-04 | 1327.6ms/step | 15.1 samples/s +Step 27660/44000 | Loss: 2.9343 | LR: 1.56e-04 | 1316.5ms/step | 15.2 samples/s +Step 27670/44000 | Loss: 2.9337 | LR: 1.56e-04 | 1318.2ms/step | 15.2 samples/s +Step 27680/44000 | Loss: 2.9873 | LR: 1.56e-04 | 1311.6ms/step | 15.2 samples/s +Step 27690/44000 | Loss: 2.9461 | LR: 1.56e-04 | 1313.3ms/step | 15.2 samples/s +Step 27700/44000 | Loss: 2.9251 | LR: 1.56e-04 | 1316.6ms/step | 15.2 samples/s +Step 27710/44000 | Loss: 2.9348 | LR: 1.56e-04 | 1317.6ms/step | 15.2 samples/s +Step 27720/44000 | Loss: 2.9338 | LR: 1.55e-04 | 1318.2ms/step | 15.2 samples/s +Step 27730/44000 | Loss: 2.9450 | LR: 1.55e-04 | 1318.4ms/step | 15.2 samples/s +Step 27740/44000 | Loss: 2.9562 | LR: 1.55e-04 | 1319.4ms/step | 15.2 samples/s +Step 27750/44000 | Loss: 2.9458 | LR: 1.55e-04 | 1323.4ms/step | 15.1 samples/s +Validation at step 27750: Loss = 2.9514, Perplexity = 19.13 +Step 27760/44000 | Loss: 2.9358 | LR: 1.55e-04 | 3544.1ms/step | 5.6 samples/s +Step 27770/44000 | Loss: 2.9460 | LR: 1.55e-04 | 1326.4ms/step | 15.1 samples/s +Step 27780/44000 | Loss: 2.8954 | LR: 1.54e-04 | 1324.9ms/step | 15.1 samples/s +Step 27790/44000 | Loss: 2.9470 | LR: 1.54e-04 | 1325.5ms/step | 15.1 samples/s +Step 27800/44000 | Loss: 2.9218 | LR: 1.54e-04 | 1325.5ms/step | 15.1 samples/s +Step 27810/44000 | Loss: 2.9567 | LR: 1.54e-04 | 1323.3ms/step | 15.1 samples/s +Step 27820/44000 | Loss: 2.9202 | LR: 1.54e-04 | 1322.3ms/step | 15.1 samples/s +Step 27830/44000 | Loss: 2.9229 | LR: 1.54e-04 | 1317.8ms/step | 15.2 samples/s +Step 27840/44000 | Loss: 2.9406 | LR: 1.53e-04 | 1320.3ms/step | 15.1 samples/s +Step 27850/44000 | Loss: 2.9377 | LR: 1.53e-04 | 1322.9ms/step | 15.1 samples/s +Step 27860/44000 | Loss: 2.9630 | LR: 1.53e-04 | 1321.8ms/step | 15.1 samples/s +Step 27870/44000 | Loss: 2.9414 | LR: 1.53e-04 | 1318.5ms/step | 15.2 samples/s +Step 27880/44000 | Loss: 2.9580 | LR: 1.53e-04 | 1324.4ms/step | 15.1 samples/s +Step 27890/44000 | Loss: 2.9221 | LR: 1.53e-04 | 1321.6ms/step | 15.1 samples/s +Step 27900/44000 | Loss: 2.9657 | LR: 1.52e-04 | 1320.8ms/step | 15.1 samples/s +Step 27910/44000 | Loss: 2.9372 | LR: 1.52e-04 | 1320.6ms/step | 15.1 samples/s +Step 27920/44000 | Loss: 2.9219 | LR: 1.52e-04 | 1322.5ms/step | 15.1 samples/s +Step 27930/44000 | Loss: 2.9459 | LR: 1.52e-04 | 1321.3ms/step | 15.1 samples/s +Step 27940/44000 | Loss: 2.9396 | LR: 1.52e-04 | 1320.4ms/step | 15.1 samples/s +Step 27950/44000 | Loss: 2.9620 | LR: 1.52e-04 | 1320.7ms/step | 15.1 samples/s +Step 27960/44000 | Loss: 2.9314 | LR: 1.51e-04 | 1317.3ms/step | 15.2 samples/s +Step 27970/44000 | Loss: 2.9650 | LR: 1.51e-04 | 1319.9ms/step | 15.2 samples/s +Step 27980/44000 | Loss: 2.9348 | LR: 1.51e-04 | 1323.6ms/step | 15.1 samples/s +Step 27990/44000 | Loss: 2.9248 | LR: 1.51e-04 | 1320.9ms/step | 15.1 samples/s +Step 28000/44000 | Loss: 2.9604 | LR: 1.51e-04 | 1318.0ms/step | 15.2 samples/s +Validation at step 28000: Loss = 2.9502, Perplexity = 19.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-28000 +Step 28010/44000 | Loss: 2.9457 | LR: 1.51e-04 | 3814.1ms/step | 5.2 samples/s +Step 28020/44000 | Loss: 2.9391 | LR: 1.50e-04 | 1320.8ms/step | 15.1 samples/s +Step 28030/44000 | Loss: 2.9411 | LR: 1.50e-04 | 1320.8ms/step | 15.1 samples/s +Step 28040/44000 | Loss: 2.9420 | LR: 1.50e-04 | 1321.4ms/step | 15.1 samples/s +Step 28050/44000 | Loss: 2.9413 | LR: 1.50e-04 | 1325.4ms/step | 15.1 samples/s +Step 28060/44000 | Loss: 2.9188 | LR: 1.50e-04 | 1322.7ms/step | 15.1 samples/s +Step 28070/44000 | Loss: 2.9430 | LR: 1.50e-04 | 1327.1ms/step | 15.1 samples/s +Step 28080/44000 | Loss: 2.9336 | LR: 1.49e-04 | 1328.9ms/step | 15.1 samples/s +Step 28090/44000 | Loss: 2.9557 | LR: 1.49e-04 | 1327.5ms/step | 15.1 samples/s +Step 28100/44000 | Loss: 2.9283 | LR: 1.49e-04 | 1325.5ms/step | 15.1 samples/s +Step 28110/44000 | Loss: 2.9652 | LR: 1.49e-04 | 1323.9ms/step | 15.1 samples/s +Step 28120/44000 | Loss: 2.9511 | LR: 1.49e-04 | 1325.9ms/step | 15.1 samples/s +Step 28130/44000 | Loss: 2.9381 | LR: 1.49e-04 | 1327.7ms/step | 15.1 samples/s +Step 28140/44000 | Loss: 2.9250 | LR: 1.48e-04 | 1324.0ms/step | 15.1 samples/s +Step 28150/44000 | Loss: 2.9128 | LR: 1.48e-04 | 1323.7ms/step | 15.1 samples/s +Step 28160/44000 | Loss: 2.9160 | LR: 1.48e-04 | 1321.5ms/step | 15.1 samples/s +Step 28170/44000 | Loss: 2.9485 | LR: 1.48e-04 | 1327.4ms/step | 15.1 samples/s +Step 28180/44000 | Loss: 2.9453 | LR: 1.48e-04 | 1326.6ms/step | 15.1 samples/s +Step 28190/44000 | Loss: 2.9518 | LR: 1.48e-04 | 1326.8ms/step | 15.1 samples/s +Step 28200/44000 | Loss: 2.9510 | LR: 1.47e-04 | 1325.7ms/step | 15.1 samples/s +Step 28210/44000 | Loss: 2.9537 | LR: 1.47e-04 | 1325.4ms/step | 15.1 samples/s +Step 28220/44000 | Loss: 2.9689 | LR: 1.47e-04 | 1329.7ms/step | 15.0 samples/s +Step 28230/44000 | Loss: 2.9362 | LR: 1.47e-04 | 1324.7ms/step | 15.1 samples/s +Step 28240/44000 | Loss: 2.9402 | LR: 1.47e-04 | 1325.4ms/step | 15.1 samples/s +Step 28250/44000 | Loss: 2.9197 | LR: 1.47e-04 | 1324.7ms/step | 15.1 samples/s +Validation at step 28250: Loss = 2.9491, Perplexity = 19.09 +Step 28260/44000 | Loss: 2.9293 | LR: 1.46e-04 | 3549.6ms/step | 5.6 samples/s +Step 28270/44000 | Loss: 2.9386 | LR: 1.46e-04 | 1326.5ms/step | 15.1 samples/s +Step 28280/44000 | Loss: 2.9440 | LR: 1.46e-04 | 1324.4ms/step | 15.1 samples/s +Step 28290/44000 | Loss: 2.9102 | LR: 1.46e-04 | 1325.9ms/step | 15.1 samples/s +Step 28300/44000 | Loss: 2.9010 | LR: 1.46e-04 | 1325.3ms/step | 15.1 samples/s +Step 28310/44000 | Loss: 2.9677 | LR: 1.46e-04 | 1325.8ms/step | 15.1 samples/s +Step 28320/44000 | Loss: 2.9699 | LR: 1.45e-04 | 1328.4ms/step | 15.1 samples/s +Step 28330/44000 | Loss: 2.8971 | LR: 1.45e-04 | 1328.8ms/step | 15.1 samples/s +Step 28340/44000 | Loss: 2.8972 | LR: 1.45e-04 | 1330.2ms/step | 15.0 samples/s +Step 28350/44000 | Loss: 2.9393 | LR: 1.45e-04 | 1329.0ms/step | 15.0 samples/s +Step 28360/44000 | Loss: 2.9448 | LR: 1.45e-04 | 1332.5ms/step | 15.0 samples/s +Step 28370/44000 | Loss: 2.9479 | LR: 1.45e-04 | 1330.3ms/step | 15.0 samples/s +Step 28380/44000 | Loss: 2.9514 | LR: 1.44e-04 | 1325.4ms/step | 15.1 samples/s +Step 28390/44000 | Loss: 2.9178 | LR: 1.44e-04 | 1330.1ms/step | 15.0 samples/s +Step 28400/44000 | Loss: 2.9562 | LR: 1.44e-04 | 1324.5ms/step | 15.1 samples/s +Step 28410/44000 | Loss: 2.9626 | LR: 1.44e-04 | 1325.1ms/step | 15.1 samples/s +Step 28420/44000 | Loss: 2.9480 | LR: 1.44e-04 | 1325.7ms/step | 15.1 samples/s +Step 28430/44000 | Loss: 2.9377 | LR: 1.44e-04 | 1325.2ms/step | 15.1 samples/s +Step 28440/44000 | Loss: 2.9543 | LR: 1.43e-04 | 1324.0ms/step | 15.1 samples/s +Step 28450/44000 | Loss: 2.9567 | LR: 1.43e-04 | 1326.2ms/step | 15.1 samples/s +Step 28460/44000 | Loss: 2.9343 | LR: 1.43e-04 | 1325.0ms/step | 15.1 samples/s +Step 28470/44000 | Loss: 2.9218 | LR: 1.43e-04 | 1326.0ms/step | 15.1 samples/s +Step 28480/44000 | Loss: 2.9228 | LR: 1.43e-04 | 1326.4ms/step | 15.1 samples/s +Step 28490/44000 | Loss: 2.9406 | LR: 1.43e-04 | 1324.7ms/step | 15.1 samples/s +Step 28500/44000 | Loss: 2.9346 | LR: 1.42e-04 | 1323.9ms/step | 15.1 samples/s +Validation at step 28500: Loss = 2.9479, Perplexity = 19.07 +Step 28510/44000 | Loss: 2.9320 | LR: 1.42e-04 | 3540.5ms/step | 5.6 samples/s +Step 28520/44000 | Loss: 2.9621 | LR: 1.42e-04 | 1322.7ms/step | 15.1 samples/s +Step 28530/44000 | Loss: 2.9407 | LR: 1.42e-04 | 1327.3ms/step | 15.1 samples/s +Step 28540/44000 | Loss: 2.9339 | LR: 1.42e-04 | 1325.9ms/step | 15.1 samples/s +Step 28550/44000 | Loss: 2.9513 | LR: 1.42e-04 | 1324.4ms/step | 15.1 samples/s +Step 28560/44000 | Loss: 2.9416 | LR: 1.41e-04 | 1325.8ms/step | 15.1 samples/s +Step 28570/44000 | Loss: 2.9527 | LR: 1.41e-04 | 1324.5ms/step | 15.1 samples/s +Step 28580/44000 | Loss: 2.9530 | LR: 1.41e-04 | 1326.3ms/step | 15.1 samples/s +Step 28590/44000 | Loss: 2.9405 | LR: 1.41e-04 | 1327.4ms/step | 15.1 samples/s +Step 28600/44000 | Loss: 2.9522 | LR: 1.41e-04 | 1323.7ms/step | 15.1 samples/s +Step 28610/44000 | Loss: 2.9649 | LR: 1.41e-04 | 1325.1ms/step | 15.1 samples/s +Step 28620/44000 | Loss: 2.9343 | LR: 1.40e-04 | 1325.3ms/step | 15.1 samples/s +Step 28630/44000 | Loss: 2.9253 | LR: 1.40e-04 | 1325.8ms/step | 15.1 samples/s +Step 28640/44000 | Loss: 2.9407 | LR: 1.40e-04 | 1327.4ms/step | 15.1 samples/s +Step 28650/44000 | Loss: 2.9691 | LR: 1.40e-04 | 1323.9ms/step | 15.1 samples/s +Step 28660/44000 | Loss: 2.9199 | LR: 1.40e-04 | 1323.5ms/step | 15.1 samples/s +Step 28670/44000 | Loss: 2.9258 | LR: 1.40e-04 | 1325.0ms/step | 15.1 samples/s +Step 28680/44000 | Loss: 2.9299 | LR: 1.39e-04 | 1325.8ms/step | 15.1 samples/s +Step 28690/44000 | Loss: 2.9649 | LR: 1.39e-04 | 1327.0ms/step | 15.1 samples/s +Step 28700/44000 | Loss: 2.9437 | LR: 1.39e-04 | 1323.4ms/step | 15.1 samples/s +Step 28710/44000 | Loss: 2.9447 | LR: 1.39e-04 | 1325.5ms/step | 15.1 samples/s +Step 28720/44000 | Loss: 2.9437 | LR: 1.39e-04 | 1324.7ms/step | 15.1 samples/s +Step 28730/44000 | Loss: 2.9339 | LR: 1.39e-04 | 1325.4ms/step | 15.1 samples/s +Step 28740/44000 | Loss: 2.9693 | LR: 1.38e-04 | 1326.4ms/step | 15.1 samples/s +Step 28750/44000 | Loss: 2.9549 | LR: 1.38e-04 | 1328.0ms/step | 15.1 samples/s +Validation at step 28750: Loss = 2.9467, Perplexity = 19.04 +Step 28760/44000 | Loss: 2.9350 | LR: 1.38e-04 | 3542.9ms/step | 5.6 samples/s +Step 28770/44000 | Loss: 2.9604 | LR: 1.38e-04 | 1322.8ms/step | 15.1 samples/s +Step 28780/44000 | Loss: 2.9190 | LR: 1.38e-04 | 1325.1ms/step | 15.1 samples/s +Step 28790/44000 | Loss: 2.9492 | LR: 1.38e-04 | 1324.4ms/step | 15.1 samples/s +Step 28800/44000 | Loss: 2.9500 | LR: 1.38e-04 | 1322.8ms/step | 15.1 samples/s +Step 28810/44000 | Loss: 2.9605 | LR: 1.37e-04 | 1324.8ms/step | 15.1 samples/s +Step 28820/44000 | Loss: 2.9351 | LR: 1.37e-04 | 1322.3ms/step | 15.1 samples/s +Step 28830/44000 | Loss: 2.9344 | LR: 1.37e-04 | 1328.5ms/step | 15.1 samples/s +Step 28840/44000 | Loss: 2.9492 | LR: 1.37e-04 | 1326.4ms/step | 15.1 samples/s +Step 28850/44000 | Loss: 2.9515 | LR: 1.37e-04 | 1321.6ms/step | 15.1 samples/s +Step 28860/44000 | Loss: 2.9328 | LR: 1.37e-04 | 1323.3ms/step | 15.1 samples/s +Step 28870/44000 | Loss: 2.9112 | LR: 1.36e-04 | 1333.0ms/step | 15.0 samples/s +Step 28880/44000 | Loss: 2.9414 | LR: 1.36e-04 | 1329.2ms/step | 15.0 samples/s +Step 28890/44000 | Loss: 2.9514 | LR: 1.36e-04 | 1324.7ms/step | 15.1 samples/s +Step 28900/44000 | Loss: 2.9364 | LR: 1.36e-04 | 1322.8ms/step | 15.1 samples/s +Step 28910/44000 | Loss: 2.9438 | LR: 1.36e-04 | 1323.2ms/step | 15.1 samples/s +Step 28920/44000 | Loss: 2.9194 | LR: 1.36e-04 | 1326.0ms/step | 15.1 samples/s +Step 28930/44000 | Loss: 2.9432 | LR: 1.35e-04 | 1323.9ms/step | 15.1 samples/s +Step 28940/44000 | Loss: 2.9442 | LR: 1.35e-04 | 1322.7ms/step | 15.1 samples/s +Step 28950/44000 | Loss: 2.9235 | LR: 1.35e-04 | 1322.7ms/step | 15.1 samples/s +Step 28960/44000 | Loss: 2.9250 | LR: 1.35e-04 | 1326.7ms/step | 15.1 samples/s +Step 28970/44000 | Loss: 2.9279 | LR: 1.35e-04 | 1323.1ms/step | 15.1 samples/s +Step 28980/44000 | Loss: 2.9539 | LR: 1.35e-04 | 1319.9ms/step | 15.2 samples/s +Step 28990/44000 | Loss: 2.9280 | LR: 1.34e-04 | 1321.0ms/step | 15.1 samples/s +Step 29000/44000 | Loss: 2.9605 | LR: 1.34e-04 | 1323.4ms/step | 15.1 samples/s +Validation at step 29000: Loss = 2.9457, Perplexity = 19.02 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-29000 +Step 29010/44000 | Loss: 2.9444 | LR: 1.34e-04 | 3810.6ms/step | 5.2 samples/s +Step 29020/44000 | Loss: 2.9627 | LR: 1.34e-04 | 1323.8ms/step | 15.1 samples/s +Step 29030/44000 | Loss: 2.9241 | LR: 1.34e-04 | 1322.7ms/step | 15.1 samples/s +Step 29040/44000 | Loss: 2.9134 | LR: 1.34e-04 | 1323.9ms/step | 15.1 samples/s +Step 29050/44000 | Loss: 2.9229 | LR: 1.33e-04 | 1325.2ms/step | 15.1 samples/s +Step 29060/44000 | Loss: 2.9646 | LR: 1.33e-04 | 1324.3ms/step | 15.1 samples/s +Step 29070/44000 | Loss: 2.9463 | LR: 1.33e-04 | 1324.0ms/step | 15.1 samples/s +Step 29080/44000 | Loss: 2.9277 | LR: 1.33e-04 | 1324.5ms/step | 15.1 samples/s +Step 29090/44000 | Loss: 2.9423 | LR: 1.33e-04 | 1327.8ms/step | 15.1 samples/s +Step 29100/44000 | Loss: 2.9349 | LR: 1.33e-04 | 1327.7ms/step | 15.1 samples/s +Step 29110/44000 | Loss: 2.9334 | LR: 1.33e-04 | 1326.5ms/step | 15.1 samples/s +Step 29120/44000 | Loss: 2.9436 | LR: 1.32e-04 | 1324.8ms/step | 15.1 samples/s +Step 29130/44000 | Loss: 2.9383 | LR: 1.32e-04 | 1325.2ms/step | 15.1 samples/s +Step 29140/44000 | Loss: 2.9290 | LR: 1.32e-04 | 1327.2ms/step | 15.1 samples/s +Step 29150/44000 | Loss: 2.9444 | LR: 1.32e-04 | 1323.1ms/step | 15.1 samples/s +Step 29160/44000 | Loss: 2.9187 | LR: 1.32e-04 | 1323.0ms/step | 15.1 samples/s +Step 29170/44000 | Loss: 2.9203 | LR: 1.32e-04 | 1324.7ms/step | 15.1 samples/s +Step 29180/44000 | Loss: 2.9279 | LR: 1.31e-04 | 1324.7ms/step | 15.1 samples/s +Step 29190/44000 | Loss: 2.9369 | LR: 1.31e-04 | 1327.1ms/step | 15.1 samples/s +Step 29200/44000 | Loss: 2.9297 | LR: 1.31e-04 | 1323.2ms/step | 15.1 samples/s +Step 29210/44000 | Loss: 2.9242 | LR: 1.31e-04 | 1325.6ms/step | 15.1 samples/s +Step 29220/44000 | Loss: 2.9269 | LR: 1.31e-04 | 1324.2ms/step | 15.1 samples/s +Step 29230/44000 | Loss: 2.9378 | LR: 1.31e-04 | 1326.0ms/step | 15.1 samples/s +Step 29240/44000 | Loss: 2.9499 | LR: 1.30e-04 | 1319.6ms/step | 15.2 samples/s +Step 29250/44000 | Loss: 2.9418 | LR: 1.30e-04 | 1324.5ms/step | 15.1 samples/s +Validation at step 29250: Loss = 2.9448, Perplexity = 19.01 +Step 29260/44000 | Loss: 2.9274 | LR: 1.30e-04 | 3547.1ms/step | 5.6 samples/s +Step 29270/44000 | Loss: 2.9545 | LR: 1.30e-04 | 1323.8ms/step | 15.1 samples/s +Step 29280/44000 | Loss: 2.9523 | LR: 1.30e-04 | 1322.3ms/step | 15.1 samples/s +Step 29290/44000 | Loss: 2.9444 | LR: 1.30e-04 | 1325.0ms/step | 15.1 samples/s +Step 29300/44000 | Loss: 2.9281 | LR: 1.29e-04 | 1323.2ms/step | 15.1 samples/s +Step 29310/44000 | Loss: 2.9335 | LR: 1.29e-04 | 1327.2ms/step | 15.1 samples/s +Step 29320/44000 | Loss: 2.9385 | LR: 1.29e-04 | 1324.1ms/step | 15.1 samples/s +Step 29330/44000 | Loss: 2.9187 | LR: 1.29e-04 | 1323.5ms/step | 15.1 samples/s +Step 29340/44000 | Loss: 2.9483 | LR: 1.29e-04 | 1325.2ms/step | 15.1 samples/s +Step 29350/44000 | Loss: 2.9606 | LR: 1.29e-04 | 1328.0ms/step | 15.1 samples/s +Step 29360/44000 | Loss: 2.9243 | LR: 1.29e-04 | 1323.2ms/step | 15.1 samples/s +Step 29370/44000 | Loss: 2.9698 | LR: 1.28e-04 | 1323.8ms/step | 15.1 samples/s +Step 29380/44000 | Loss: 2.9197 | LR: 1.28e-04 | 1324.8ms/step | 15.1 samples/s +Step 29390/44000 | Loss: 2.9655 | LR: 1.28e-04 | 1325.2ms/step | 15.1 samples/s +Step 29400/44000 | Loss: 2.9606 | LR: 1.28e-04 | 1327.3ms/step | 15.1 samples/s +Step 29410/44000 | Loss: 2.9391 | LR: 1.28e-04 | 1324.8ms/step | 15.1 samples/s +Step 29420/44000 | Loss: 2.9219 | LR: 1.28e-04 | 1325.7ms/step | 15.1 samples/s +Step 29430/44000 | Loss: 2.9615 | LR: 1.27e-04 | 1324.5ms/step | 15.1 samples/s +Step 29440/44000 | Loss: 2.9223 | LR: 1.27e-04 | 1323.7ms/step | 15.1 samples/s +Step 29450/44000 | Loss: 2.9161 | LR: 1.27e-04 | 1323.3ms/step | 15.1 samples/s +Step 29460/44000 | Loss: 2.9232 | LR: 1.27e-04 | 1322.3ms/step | 15.1 samples/s +Step 29470/44000 | Loss: 2.9443 | LR: 1.27e-04 | 1325.9ms/step | 15.1 samples/s +Step 29480/44000 | Loss: 2.9266 | LR: 1.27e-04 | 1323.0ms/step | 15.1 samples/s +Step 29490/44000 | Loss: 2.9426 | LR: 1.26e-04 | 1326.8ms/step | 15.1 samples/s +Step 29500/44000 | Loss: 2.9475 | LR: 1.26e-04 | 1327.1ms/step | 15.1 samples/s +Validation at step 29500: Loss = 2.9436, Perplexity = 18.98 +Step 29510/44000 | Loss: 2.9222 | LR: 1.26e-04 | 3549.9ms/step | 5.6 samples/s +Step 29520/44000 | Loss: 2.9296 | LR: 1.26e-04 | 1324.9ms/step | 15.1 samples/s +Step 29530/44000 | Loss: 2.9202 | LR: 1.26e-04 | 1323.5ms/step | 15.1 samples/s +Step 29540/44000 | Loss: 2.9658 | LR: 1.26e-04 | 1325.5ms/step | 15.1 samples/s +Step 29550/44000 | Loss: 2.9412 | LR: 1.26e-04 | 1328.2ms/step | 15.1 samples/s +Step 29560/44000 | Loss: 2.9211 | LR: 1.25e-04 | 1322.4ms/step | 15.1 samples/s +Step 29570/44000 | Loss: 2.9447 | LR: 1.25e-04 | 1323.8ms/step | 15.1 samples/s +Step 29580/44000 | Loss: 2.9162 | LR: 1.25e-04 | 1325.5ms/step | 15.1 samples/s +Step 29590/44000 | Loss: 2.9470 | LR: 1.25e-04 | 1324.5ms/step | 15.1 samples/s +Step 29600/44000 | Loss: 2.9276 | LR: 1.25e-04 | 1325.2ms/step | 15.1 samples/s +Step 29610/44000 | Loss: 2.9326 | LR: 1.25e-04 | 1320.8ms/step | 15.1 samples/s +Step 29620/44000 | Loss: 2.9201 | LR: 1.24e-04 | 1318.6ms/step | 15.2 samples/s +Step 29630/44000 | Loss: 2.9225 | LR: 1.24e-04 | 1320.6ms/step | 15.1 samples/s +Step 29640/44000 | Loss: 2.9409 | LR: 1.24e-04 | 1321.7ms/step | 15.1 samples/s +Step 29650/44000 | Loss: 2.9598 | LR: 1.24e-04 | 1320.0ms/step | 15.2 samples/s +Step 29660/44000 | Loss: 2.9379 | LR: 1.24e-04 | 1325.5ms/step | 15.1 samples/s +Step 29670/44000 | Loss: 2.9397 | LR: 1.24e-04 | 1323.6ms/step | 15.1 samples/s +Step 29680/44000 | Loss: 2.9568 | LR: 1.23e-04 | 1327.1ms/step | 15.1 samples/s +Step 29690/44000 | Loss: 2.9315 | LR: 1.23e-04 | 1323.1ms/step | 15.1 samples/s +Step 29700/44000 | Loss: 2.9447 | LR: 1.23e-04 | 1326.9ms/step | 15.1 samples/s +Step 29710/44000 | Loss: 2.9479 | LR: 1.23e-04 | 1325.5ms/step | 15.1 samples/s +Step 29720/44000 | Loss: 2.9533 | LR: 1.23e-04 | 1326.5ms/step | 15.1 samples/s +Step 29730/44000 | Loss: 2.9402 | LR: 1.23e-04 | 1327.3ms/step | 15.1 samples/s +Step 29740/44000 | Loss: 2.9262 | LR: 1.23e-04 | 1326.9ms/step | 15.1 samples/s +Step 29750/44000 | Loss: 2.9530 | LR: 1.22e-04 | 1325.7ms/step | 15.1 samples/s +Validation at step 29750: Loss = 2.9426, Perplexity = 18.97 +Step 29760/44000 | Loss: 2.9376 | LR: 1.22e-04 | 3546.9ms/step | 5.6 samples/s +Step 29770/44000 | Loss: 2.9012 | LR: 1.22e-04 | 1330.9ms/step | 15.0 samples/s +Step 29780/44000 | Loss: 2.9804 | LR: 1.22e-04 | 1329.2ms/step | 15.0 samples/s +Step 29790/44000 | Loss: 2.9237 | LR: 1.22e-04 | 1327.6ms/step | 15.1 samples/s +Step 29800/44000 | Loss: 2.9139 | LR: 1.22e-04 | 1329.8ms/step | 15.0 samples/s +Step 29810/44000 | Loss: 2.9191 | LR: 1.21e-04 | 1329.0ms/step | 15.0 samples/s +Step 29820/44000 | Loss: 2.9292 | LR: 1.21e-04 | 1332.2ms/step | 15.0 samples/s +Step 29830/44000 | Loss: 2.9464 | LR: 1.21e-04 | 1328.4ms/step | 15.1 samples/s +Step 29840/44000 | Loss: 2.9147 | LR: 1.21e-04 | 1327.7ms/step | 15.1 samples/s +Step 29850/44000 | Loss: 2.9173 | LR: 1.21e-04 | 1325.4ms/step | 15.1 samples/s +Step 29860/44000 | Loss: 2.9468 | LR: 1.21e-04 | 1325.6ms/step | 15.1 samples/s +Step 29870/44000 | Loss: 2.9375 | LR: 1.21e-04 | 1326.3ms/step | 15.1 samples/s +Step 29880/44000 | Loss: 2.9405 | LR: 1.20e-04 | 1325.3ms/step | 15.1 samples/s +Step 29890/44000 | Loss: 2.8896 | LR: 1.20e-04 | 1321.9ms/step | 15.1 samples/s +Step 29900/44000 | Loss: 2.9086 | LR: 1.20e-04 | 1323.2ms/step | 15.1 samples/s +Step 29910/44000 | Loss: 2.9374 | LR: 1.20e-04 | 1321.8ms/step | 15.1 samples/s +Step 29920/44000 | Loss: 2.9237 | LR: 1.20e-04 | 1320.7ms/step | 15.1 samples/s +Step 29930/44000 | Loss: 2.9435 | LR: 1.20e-04 | 1321.7ms/step | 15.1 samples/s +Step 29940/44000 | Loss: 2.9148 | LR: 1.19e-04 | 1323.1ms/step | 15.1 samples/s +Step 29950/44000 | Loss: 2.9275 | LR: 1.19e-04 | 1321.8ms/step | 15.1 samples/s +Step 29960/44000 | Loss: 2.9170 | LR: 1.19e-04 | 1325.0ms/step | 15.1 samples/s +Step 29970/44000 | Loss: 2.9533 | LR: 1.19e-04 | 1323.9ms/step | 15.1 samples/s +Step 29980/44000 | Loss: 2.9239 | LR: 1.19e-04 | 1325.3ms/step | 15.1 samples/s +Step 29990/44000 | Loss: 2.9373 | LR: 1.19e-04 | 1321.7ms/step | 15.1 samples/s +Step 30000/44000 | Loss: 2.9059 | LR: 1.19e-04 | 1318.9ms/step | 15.2 samples/s +Validation at step 30000: Loss = 2.9418, Perplexity = 18.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-30000 +Step 30010/44000 | Loss: 2.9312 | LR: 1.18e-04 | 3846.0ms/step | 5.2 samples/s +Step 30020/44000 | Loss: 2.9272 | LR: 1.18e-04 | 1326.5ms/step | 15.1 samples/s +Step 30030/44000 | Loss: 2.9077 | LR: 1.18e-04 | 1324.8ms/step | 15.1 samples/s +Step 30040/44000 | Loss: 2.9395 | LR: 1.18e-04 | 1321.4ms/step | 15.1 samples/s +Step 30050/44000 | Loss: 2.9472 | LR: 1.18e-04 | 1324.3ms/step | 15.1 samples/s +Step 30060/44000 | Loss: 2.9309 | LR: 1.18e-04 | 1325.7ms/step | 15.1 samples/s +Step 30070/44000 | Loss: 2.9212 | LR: 1.17e-04 | 1328.0ms/step | 15.1 samples/s +Step 30080/44000 | Loss: 2.9135 | LR: 1.17e-04 | 1324.3ms/step | 15.1 samples/s +Step 30090/44000 | Loss: 2.9519 | LR: 1.17e-04 | 1325.2ms/step | 15.1 samples/s +Step 30100/44000 | Loss: 2.9120 | LR: 1.17e-04 | 1325.2ms/step | 15.1 samples/s +Step 30110/44000 | Loss: 2.9425 | LR: 1.17e-04 | 1327.1ms/step | 15.1 samples/s +Step 30120/44000 | Loss: 2.9392 | LR: 1.17e-04 | 1323.1ms/step | 15.1 samples/s +Step 30130/44000 | Loss: 2.9493 | LR: 1.17e-04 | 1325.6ms/step | 15.1 samples/s +Step 30140/44000 | Loss: 2.9254 | LR: 1.16e-04 | 1324.2ms/step | 15.1 samples/s +Step 30150/44000 | Loss: 2.9368 | LR: 1.16e-04 | 1324.6ms/step | 15.1 samples/s +Step 30160/44000 | Loss: 2.9435 | LR: 1.16e-04 | 1324.6ms/step | 15.1 samples/s +Step 30170/44000 | Loss: 2.9451 | LR: 1.16e-04 | 1326.2ms/step | 15.1 samples/s +Step 30180/44000 | Loss: 2.9093 | LR: 1.16e-04 | 1324.5ms/step | 15.1 samples/s +Step 30190/44000 | Loss: 2.9425 | LR: 1.16e-04 | 1328.1ms/step | 15.1 samples/s +Step 30200/44000 | Loss: 2.9418 | LR: 1.15e-04 | 1323.8ms/step | 15.1 samples/s +Step 30210/44000 | Loss: 2.9435 | LR: 1.15e-04 | 1321.7ms/step | 15.1 samples/s +Step 30220/44000 | Loss: 2.9392 | LR: 1.15e-04 | 1326.4ms/step | 15.1 samples/s +Step 30230/44000 | Loss: 2.9306 | LR: 1.15e-04 | 1323.1ms/step | 15.1 samples/s +Step 30240/44000 | Loss: 2.9368 | LR: 1.15e-04 | 1326.2ms/step | 15.1 samples/s +Step 30250/44000 | Loss: 2.9469 | LR: 1.15e-04 | 1327.8ms/step | 15.1 samples/s +Validation at step 30250: Loss = 2.9408, Perplexity = 18.93 +Step 30260/44000 | Loss: 2.9319 | LR: 1.15e-04 | 3540.7ms/step | 5.6 samples/s +Step 30270/44000 | Loss: 2.9054 | LR: 1.14e-04 | 1322.0ms/step | 15.1 samples/s +Step 30280/44000 | Loss: 2.9229 | LR: 1.14e-04 | 1322.4ms/step | 15.1 samples/s +Step 30290/44000 | Loss: 2.9632 | LR: 1.14e-04 | 1323.4ms/step | 15.1 samples/s +Step 30300/44000 | Loss: 2.9414 | LR: 1.14e-04 | 1323.0ms/step | 15.1 samples/s +Step 30310/44000 | Loss: 2.9238 | LR: 1.14e-04 | 1325.4ms/step | 15.1 samples/s +Step 30320/44000 | Loss: 2.8961 | LR: 1.14e-04 | 1323.4ms/step | 15.1 samples/s +Step 30330/44000 | Loss: 2.9266 | LR: 1.13e-04 | 1323.9ms/step | 15.1 samples/s +Step 30340/44000 | Loss: 2.9190 | LR: 1.13e-04 | 1324.2ms/step | 15.1 samples/s +Step 30350/44000 | Loss: 2.8867 | LR: 1.13e-04 | 1324.6ms/step | 15.1 samples/s +Step 30360/44000 | Loss: 2.9316 | LR: 1.13e-04 | 1322.3ms/step | 15.1 samples/s +Step 30370/44000 | Loss: 2.9365 | LR: 1.13e-04 | 1326.4ms/step | 15.1 samples/s +Step 30380/44000 | Loss: 2.9256 | LR: 1.13e-04 | 1324.5ms/step | 15.1 samples/s +Step 30390/44000 | Loss: 2.9220 | LR: 1.13e-04 | 1325.5ms/step | 15.1 samples/s +Step 30400/44000 | Loss: 2.9391 | LR: 1.12e-04 | 1326.5ms/step | 15.1 samples/s +Step 30410/44000 | Loss: 2.9058 | LR: 1.12e-04 | 1319.7ms/step | 15.2 samples/s +Step 30420/44000 | Loss: 2.9608 | LR: 1.12e-04 | 1321.1ms/step | 15.1 samples/s +Step 30430/44000 | Loss: 2.9339 | LR: 1.12e-04 | 1323.6ms/step | 15.1 samples/s +Step 30440/44000 | Loss: 2.9133 | LR: 1.12e-04 | 1321.8ms/step | 15.1 samples/s +Step 30450/44000 | Loss: 2.9175 | LR: 1.12e-04 | 1326.0ms/step | 15.1 samples/s +Step 30460/44000 | Loss: 2.8908 | LR: 1.11e-04 | 1324.7ms/step | 15.1 samples/s +Step 30470/44000 | Loss: 2.9278 | LR: 1.11e-04 | 1325.8ms/step | 15.1 samples/s +Step 30480/44000 | Loss: 2.9206 | LR: 1.11e-04 | 1324.6ms/step | 15.1 samples/s +Step 30490/44000 | Loss: 2.9270 | LR: 1.11e-04 | 1323.4ms/step | 15.1 samples/s +Step 30500/44000 | Loss: 2.9322 | LR: 1.11e-04 | 1323.5ms/step | 15.1 samples/s +Validation at step 30500: Loss = 2.9401, Perplexity = 18.92 +Step 30510/44000 | Loss: 2.9470 | LR: 1.11e-04 | 3531.9ms/step | 5.7 samples/s +Step 30520/44000 | Loss: 2.9294 | LR: 1.11e-04 | 1324.9ms/step | 15.1 samples/s +Step 30530/44000 | Loss: 2.9029 | LR: 1.10e-04 | 1325.8ms/step | 15.1 samples/s +Step 30540/44000 | Loss: 2.9596 | LR: 1.10e-04 | 1325.8ms/step | 15.1 samples/s +Step 30550/44000 | Loss: 2.8862 | LR: 1.10e-04 | 1321.9ms/step | 15.1 samples/s +Step 30560/44000 | Loss: 2.9534 | LR: 1.10e-04 | 1324.8ms/step | 15.1 samples/s +Step 30570/44000 | Loss: 2.9462 | LR: 1.10e-04 | 1324.8ms/step | 15.1 samples/s +Step 30580/44000 | Loss: 2.9417 | LR: 1.10e-04 | 1325.3ms/step | 15.1 samples/s +Step 30590/44000 | Loss: 2.9103 | LR: 1.10e-04 | 1324.8ms/step | 15.1 samples/s +Step 30600/44000 | Loss: 2.9421 | LR: 1.09e-04 | 1318.6ms/step | 15.2 samples/s +Step 30610/44000 | Loss: 2.9390 | LR: 1.09e-04 | 1317.2ms/step | 15.2 samples/s +Step 30620/44000 | Loss: 2.9349 | LR: 1.09e-04 | 1318.5ms/step | 15.2 samples/s +Step 30630/44000 | Loss: 2.9365 | LR: 1.09e-04 | 1320.5ms/step | 15.1 samples/s +Step 30640/44000 | Loss: 2.9392 | LR: 1.09e-04 | 1321.4ms/step | 15.1 samples/s +Step 30650/44000 | Loss: 2.9173 | LR: 1.09e-04 | 1323.3ms/step | 15.1 samples/s +Step 30660/44000 | Loss: 2.9368 | LR: 1.08e-04 | 1322.8ms/step | 15.1 samples/s +Step 30670/44000 | Loss: 2.9566 | LR: 1.08e-04 | 1319.6ms/step | 15.2 samples/s +Step 30680/44000 | Loss: 2.9407 | LR: 1.08e-04 | 1317.3ms/step | 15.2 samples/s +Step 30690/44000 | Loss: 2.9014 | LR: 1.08e-04 | 1321.0ms/step | 15.1 samples/s +Step 30700/44000 | Loss: 2.9222 | LR: 1.08e-04 | 1319.9ms/step | 15.2 samples/s +Step 30710/44000 | Loss: 2.9416 | LR: 1.08e-04 | 1317.7ms/step | 15.2 samples/s +Step 30720/44000 | Loss: 2.9722 | LR: 1.08e-04 | 1320.0ms/step | 15.2 samples/s +Step 30730/44000 | Loss: 2.9260 | LR: 1.07e-04 | 1323.9ms/step | 15.1 samples/s +Step 30740/44000 | Loss: 2.9083 | LR: 1.07e-04 | 1323.1ms/step | 15.1 samples/s +Step 30750/44000 | Loss: 2.9379 | LR: 1.07e-04 | 1321.2ms/step | 15.1 samples/s +Validation at step 30750: Loss = 2.9394, Perplexity = 18.90 +Step 30760/44000 | Loss: 2.9592 | LR: 1.07e-04 | 3550.3ms/step | 5.6 samples/s +Step 30770/44000 | Loss: 2.9410 | LR: 1.07e-04 | 1326.0ms/step | 15.1 samples/s +Step 30780/44000 | Loss: 2.9343 | LR: 1.07e-04 | 1324.8ms/step | 15.1 samples/s +Step 30790/44000 | Loss: 2.9647 | LR: 1.07e-04 | 1325.3ms/step | 15.1 samples/s +Step 30800/44000 | Loss: 2.9127 | LR: 1.06e-04 | 1323.9ms/step | 15.1 samples/s +Step 30810/44000 | Loss: 2.9433 | LR: 1.06e-04 | 1324.1ms/step | 15.1 samples/s +Step 30820/44000 | Loss: 2.9427 | LR: 1.06e-04 | 1327.5ms/step | 15.1 samples/s +Step 30830/44000 | Loss: 2.9368 | LR: 1.06e-04 | 1329.0ms/step | 15.0 samples/s +Step 30840/44000 | Loss: 2.9063 | LR: 1.06e-04 | 1326.7ms/step | 15.1 samples/s +Step 30850/44000 | Loss: 2.9209 | LR: 1.06e-04 | 1325.6ms/step | 15.1 samples/s +Step 30860/44000 | Loss: 2.9245 | LR: 1.05e-04 | 1326.1ms/step | 15.1 samples/s +Step 30870/44000 | Loss: 2.9022 | LR: 1.05e-04 | 1326.9ms/step | 15.1 samples/s +Step 30880/44000 | Loss: 2.9151 | LR: 1.05e-04 | 1327.2ms/step | 15.1 samples/s +Step 30890/44000 | Loss: 2.9145 | LR: 1.05e-04 | 1327.9ms/step | 15.1 samples/s +Step 30900/44000 | Loss: 2.9280 | LR: 1.05e-04 | 1323.4ms/step | 15.1 samples/s +Step 30910/44000 | Loss: 2.9548 | LR: 1.05e-04 | 1329.4ms/step | 15.0 samples/s +Step 30920/44000 | Loss: 2.9363 | LR: 1.05e-04 | 1327.9ms/step | 15.1 samples/s +Step 30930/44000 | Loss: 2.9170 | LR: 1.04e-04 | 1326.5ms/step | 15.1 samples/s +Step 30940/44000 | Loss: 2.9514 | LR: 1.04e-04 | 1328.7ms/step | 15.1 samples/s +Step 30950/44000 | Loss: 2.9150 | LR: 1.04e-04 | 1326.9ms/step | 15.1 samples/s +Step 30960/44000 | Loss: 2.9347 | LR: 1.04e-04 | 1328.6ms/step | 15.1 samples/s +Step 30970/44000 | Loss: 2.9314 | LR: 1.04e-04 | 1328.0ms/step | 15.1 samples/s +Step 30980/44000 | Loss: 2.9593 | LR: 1.04e-04 | 1325.2ms/step | 15.1 samples/s +Step 30990/44000 | Loss: 2.9402 | LR: 1.04e-04 | 1326.7ms/step | 15.1 samples/s +Step 31000/44000 | Loss: 2.9233 | LR: 1.03e-04 | 1323.7ms/step | 15.1 samples/s +Validation at step 31000: Loss = 2.9387, Perplexity = 18.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-31000 +Step 31010/44000 | Loss: 2.9028 | LR: 1.03e-04 | 3840.1ms/step | 5.2 samples/s +Step 31020/44000 | Loss: 2.9165 | LR: 1.03e-04 | 1325.2ms/step | 15.1 samples/s +Step 31030/44000 | Loss: 2.9185 | LR: 1.03e-04 | 1323.8ms/step | 15.1 samples/s +Step 31040/44000 | Loss: 2.8839 | LR: 1.03e-04 | 1324.3ms/step | 15.1 samples/s +Step 31050/44000 | Loss: 2.9243 | LR: 1.03e-04 | 1324.8ms/step | 15.1 samples/s +Step 31060/44000 | Loss: 2.9563 | LR: 1.03e-04 | 1322.2ms/step | 15.1 samples/s +Step 31070/44000 | Loss: 2.9349 | LR: 1.02e-04 | 1323.0ms/step | 15.1 samples/s +Step 31080/44000 | Loss: 2.9346 | LR: 1.02e-04 | 1321.4ms/step | 15.1 samples/s +Step 31090/44000 | Loss: 2.9504 | LR: 1.02e-04 | 1327.6ms/step | 15.1 samples/s +Step 31100/44000 | Loss: 2.9326 | LR: 1.02e-04 | 1325.2ms/step | 15.1 samples/s +Step 31110/44000 | Loss: 2.9118 | LR: 1.02e-04 | 1322.9ms/step | 15.1 samples/s +Step 31120/44000 | Loss: 2.9503 | LR: 1.02e-04 | 1325.5ms/step | 15.1 samples/s +Step 31130/44000 | Loss: 2.9082 | LR: 1.02e-04 | 1325.4ms/step | 15.1 samples/s +Step 31140/44000 | Loss: 2.9664 | LR: 1.01e-04 | 1323.9ms/step | 15.1 samples/s +Step 31150/44000 | Loss: 2.9359 | LR: 1.01e-04 | 1326.6ms/step | 15.1 samples/s +Step 31160/44000 | Loss: 2.9074 | LR: 1.01e-04 | 1327.6ms/step | 15.1 samples/s +Step 31170/44000 | Loss: 2.9237 | LR: 1.01e-04 | 1327.3ms/step | 15.1 samples/s +Step 31180/44000 | Loss: 2.9481 | LR: 1.01e-04 | 1322.9ms/step | 15.1 samples/s +Step 31190/44000 | Loss: 2.9350 | LR: 1.01e-04 | 1325.6ms/step | 15.1 samples/s +Step 31200/44000 | Loss: 2.9312 | LR: 1.00e-04 | 1329.1ms/step | 15.0 samples/s +Step 31210/44000 | Loss: 2.9227 | LR: 1.00e-04 | 1326.7ms/step | 15.1 samples/s +Step 31220/44000 | Loss: 2.9369 | LR: 1.00e-04 | 1325.1ms/step | 15.1 samples/s +Step 31230/44000 | Loss: 2.9434 | LR: 1.00e-04 | 1324.3ms/step | 15.1 samples/s +Step 31240/44000 | Loss: 2.9135 | LR: 9.99e-05 | 1324.5ms/step | 15.1 samples/s +Step 31250/44000 | Loss: 2.9072 | LR: 9.98e-05 | 1324.8ms/step | 15.1 samples/s +Validation at step 31250: Loss = 2.9381, Perplexity = 18.88 +Step 31260/44000 | Loss: 2.9530 | LR: 9.96e-05 | 3553.7ms/step | 5.6 samples/s +Step 31270/44000 | Loss: 2.9302 | LR: 9.95e-05 | 1330.0ms/step | 15.0 samples/s +Step 31280/44000 | Loss: 2.9238 | LR: 9.93e-05 | 1325.9ms/step | 15.1 samples/s +Step 31290/44000 | Loss: 2.9408 | LR: 9.92e-05 | 1326.6ms/step | 15.1 samples/s +Step 31300/44000 | Loss: 2.9445 | LR: 9.90e-05 | 1328.2ms/step | 15.1 samples/s +Step 31310/44000 | Loss: 2.9327 | LR: 9.89e-05 | 1327.7ms/step | 15.1 samples/s +Step 31320/44000 | Loss: 2.9243 | LR: 9.88e-05 | 1326.1ms/step | 15.1 samples/s +Step 31330/44000 | Loss: 2.9365 | LR: 9.86e-05 | 1324.2ms/step | 15.1 samples/s +Step 31340/44000 | Loss: 2.9391 | LR: 9.85e-05 | 1323.2ms/step | 15.1 samples/s +Step 31350/44000 | Loss: 2.9427 | LR: 9.83e-05 | 1330.1ms/step | 15.0 samples/s +Step 31360/44000 | Loss: 2.9513 | LR: 9.82e-05 | 1326.9ms/step | 15.1 samples/s +Step 31370/44000 | Loss: 2.9062 | LR: 9.80e-05 | 1324.6ms/step | 15.1 samples/s +Step 31380/44000 | Loss: 2.9192 | LR: 9.79e-05 | 1327.5ms/step | 15.1 samples/s +Step 31390/44000 | Loss: 2.9196 | LR: 9.77e-05 | 1326.8ms/step | 15.1 samples/s +Step 31400/44000 | Loss: 2.9268 | LR: 9.76e-05 | 1328.5ms/step | 15.1 samples/s +Step 31410/44000 | Loss: 2.9437 | LR: 9.75e-05 | 1327.7ms/step | 15.1 samples/s +Step 31420/44000 | Loss: 2.9214 | LR: 9.73e-05 | 1328.5ms/step | 15.1 samples/s +Step 31430/44000 | Loss: 2.9188 | LR: 9.72e-05 | 1325.9ms/step | 15.1 samples/s +Step 31440/44000 | Loss: 2.9325 | LR: 9.70e-05 | 1321.9ms/step | 15.1 samples/s +Step 31450/44000 | Loss: 2.9439 | LR: 9.69e-05 | 1324.2ms/step | 15.1 samples/s +Step 31460/44000 | Loss: 2.8876 | LR: 9.67e-05 | 1324.8ms/step | 15.1 samples/s +Step 31470/44000 | Loss: 2.8890 | LR: 9.66e-05 | 1321.9ms/step | 15.1 samples/s +Step 31480/44000 | Loss: 2.9292 | LR: 9.65e-05 | 1322.5ms/step | 15.1 samples/s +Step 31490/44000 | Loss: 2.9182 | LR: 9.63e-05 | 1320.5ms/step | 15.1 samples/s +Step 31500/44000 | Loss: 2.9345 | LR: 9.62e-05 | 1324.5ms/step | 15.1 samples/s +Validation at step 31500: Loss = 2.9376, Perplexity = 18.87 +Step 31510/44000 | Loss: 2.9131 | LR: 9.60e-05 | 3541.4ms/step | 5.6 samples/s +Step 31520/44000 | Loss: 2.9292 | LR: 9.59e-05 | 1322.1ms/step | 15.1 samples/s +Step 31530/44000 | Loss: 2.9116 | LR: 9.57e-05 | 1324.1ms/step | 15.1 samples/s +Step 31540/44000 | Loss: 2.9213 | LR: 9.56e-05 | 1325.3ms/step | 15.1 samples/s +Step 31550/44000 | Loss: 2.9425 | LR: 9.54e-05 | 1322.6ms/step | 15.1 samples/s +Step 31560/44000 | Loss: 2.9376 | LR: 9.53e-05 | 1328.3ms/step | 15.1 samples/s +Step 31570/44000 | Loss: 2.9228 | LR: 9.52e-05 | 1327.4ms/step | 15.1 samples/s +Step 31580/44000 | Loss: 2.9166 | LR: 9.50e-05 | 1324.8ms/step | 15.1 samples/s +Step 31590/44000 | Loss: 2.9210 | LR: 9.49e-05 | 1325.8ms/step | 15.1 samples/s +Step 31600/44000 | Loss: 2.9066 | LR: 9.47e-05 | 1327.1ms/step | 15.1 samples/s +Step 31610/44000 | Loss: 2.9157 | LR: 9.46e-05 | 1324.8ms/step | 15.1 samples/s +Step 31620/44000 | Loss: 2.9339 | LR: 9.45e-05 | 1327.4ms/step | 15.1 samples/s +Step 31630/44000 | Loss: 2.9434 | LR: 9.43e-05 | 1326.1ms/step | 15.1 samples/s +Step 31640/44000 | Loss: 2.9067 | LR: 9.42e-05 | 1327.0ms/step | 15.1 samples/s +Step 31650/44000 | Loss: 2.9427 | LR: 9.40e-05 | 1329.2ms/step | 15.0 samples/s +Step 31660/44000 | Loss: 2.9372 | LR: 9.39e-05 | 1326.3ms/step | 15.1 samples/s +Step 31670/44000 | Loss: 2.9594 | LR: 9.37e-05 | 1327.6ms/step | 15.1 samples/s +Step 31680/44000 | Loss: 2.9117 | LR: 9.36e-05 | 1322.9ms/step | 15.1 samples/s +Step 31690/44000 | Loss: 2.9253 | LR: 9.35e-05 | 1328.0ms/step | 15.1 samples/s +Step 31700/44000 | Loss: 2.9387 | LR: 9.33e-05 | 1326.2ms/step | 15.1 samples/s +Step 31710/44000 | Loss: 2.9151 | LR: 9.32e-05 | 1323.8ms/step | 15.1 samples/s +Step 31720/44000 | Loss: 2.9453 | LR: 9.30e-05 | 1325.6ms/step | 15.1 samples/s +Step 31730/44000 | Loss: 2.9419 | LR: 9.29e-05 | 1326.1ms/step | 15.1 samples/s +Step 31740/44000 | Loss: 2.9451 | LR: 9.28e-05 | 1327.1ms/step | 15.1 samples/s +Step 31750/44000 | Loss: 2.9196 | LR: 9.26e-05 | 1326.3ms/step | 15.1 samples/s +Validation at step 31750: Loss = 2.9367, Perplexity = 18.85 +Step 31760/44000 | Loss: 2.9181 | LR: 9.25e-05 | 3546.7ms/step | 5.6 samples/s +Step 31770/44000 | Loss: 2.9243 | LR: 9.23e-05 | 1322.1ms/step | 15.1 samples/s +Step 31780/44000 | Loss: 2.9236 | LR: 9.22e-05 | 1325.3ms/step | 15.1 samples/s +Step 31790/44000 | Loss: 2.9003 | LR: 9.20e-05 | 1324.8ms/step | 15.1 samples/s +Step 31800/44000 | Loss: 2.9355 | LR: 9.19e-05 | 1325.3ms/step | 15.1 samples/s +Step 31810/44000 | Loss: 2.9458 | LR: 9.18e-05 | 1322.5ms/step | 15.1 samples/s +Step 31820/44000 | Loss: 2.9264 | LR: 9.16e-05 | 1322.8ms/step | 15.1 samples/s +Step 31830/44000 | Loss: 2.9273 | LR: 9.15e-05 | 1324.5ms/step | 15.1 samples/s +Step 31840/44000 | Loss: 2.9477 | LR: 9.13e-05 | 1325.9ms/step | 15.1 samples/s +Step 31850/44000 | Loss: 2.9326 | LR: 9.12e-05 | 1327.2ms/step | 15.1 samples/s +Step 31860/44000 | Loss: 2.9586 | LR: 9.11e-05 | 1323.6ms/step | 15.1 samples/s +Step 31870/44000 | Loss: 2.9437 | LR: 9.09e-05 | 1322.4ms/step | 15.1 samples/s +Step 31880/44000 | Loss: 2.9170 | LR: 9.08e-05 | 1322.4ms/step | 15.1 samples/s +Step 31890/44000 | Loss: 2.9312 | LR: 9.06e-05 | 1322.6ms/step | 15.1 samples/s +Step 31900/44000 | Loss: 2.9083 | LR: 9.05e-05 | 1325.1ms/step | 15.1 samples/s +Step 31910/44000 | Loss: 2.9463 | LR: 9.04e-05 | 1320.2ms/step | 15.1 samples/s +Step 31920/44000 | Loss: 2.9013 | LR: 9.02e-05 | 1324.6ms/step | 15.1 samples/s +Step 31930/44000 | Loss: 2.9111 | LR: 9.01e-05 | 1323.6ms/step | 15.1 samples/s +Step 31940/44000 | Loss: 2.9438 | LR: 8.99e-05 | 1328.8ms/step | 15.1 samples/s +Step 31950/44000 | Loss: 2.9346 | LR: 8.98e-05 | 1327.9ms/step | 15.1 samples/s +Step 31960/44000 | Loss: 2.9114 | LR: 8.97e-05 | 1324.7ms/step | 15.1 samples/s +Step 31970/44000 | Loss: 2.9108 | LR: 8.95e-05 | 1323.2ms/step | 15.1 samples/s +Step 31980/44000 | Loss: 2.9370 | LR: 8.94e-05 | 1325.1ms/step | 15.1 samples/s +Step 31990/44000 | Loss: 2.9469 | LR: 8.92e-05 | 1322.5ms/step | 15.1 samples/s +Step 32000/44000 | Loss: 2.9184 | LR: 8.91e-05 | 1323.6ms/step | 15.1 samples/s +Validation at step 32000: Loss = 2.9360, Perplexity = 18.84 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-32000 +Step 32010/44000 | Loss: 2.9076 | LR: 8.90e-05 | 3824.9ms/step | 5.2 samples/s +Step 32020/44000 | Loss: 2.8971 | LR: 8.88e-05 | 1328.1ms/step | 15.1 samples/s +Step 32030/44000 | Loss: 2.9508 | LR: 8.87e-05 | 1323.2ms/step | 15.1 samples/s +Step 32040/44000 | Loss: 2.9341 | LR: 8.86e-05 | 1329.0ms/step | 15.0 samples/s +Step 32050/44000 | Loss: 2.9194 | LR: 8.84e-05 | 1323.4ms/step | 15.1 samples/s +Step 32060/44000 | Loss: 2.9409 | LR: 8.83e-05 | 1328.4ms/step | 15.1 samples/s +Step 32070/44000 | Loss: 2.9300 | LR: 8.81e-05 | 1326.7ms/step | 15.1 samples/s +Step 32080/44000 | Loss: 2.9112 | LR: 8.80e-05 | 1327.3ms/step | 15.1 samples/s +Step 32090/44000 | Loss: 2.9298 | LR: 8.79e-05 | 1327.9ms/step | 15.1 samples/s +Step 32100/44000 | Loss: 2.9411 | LR: 8.77e-05 | 1327.1ms/step | 15.1 samples/s +Step 32110/44000 | Loss: 2.9360 | LR: 8.76e-05 | 1327.3ms/step | 15.1 samples/s +Step 32120/44000 | Loss: 2.9283 | LR: 8.74e-05 | 1324.8ms/step | 15.1 samples/s +Step 32130/44000 | Loss: 2.9391 | LR: 8.73e-05 | 1329.7ms/step | 15.0 samples/s +Step 32140/44000 | Loss: 2.9243 | LR: 8.72e-05 | 1327.7ms/step | 15.1 samples/s +Step 32150/44000 | Loss: 2.9214 | LR: 8.70e-05 | 1328.9ms/step | 15.1 samples/s +Step 32160/44000 | Loss: 2.9027 | LR: 8.69e-05 | 1325.0ms/step | 15.1 samples/s +Step 32170/44000 | Loss: 2.9456 | LR: 8.68e-05 | 1323.1ms/step | 15.1 samples/s +Step 32180/44000 | Loss: 2.9370 | LR: 8.66e-05 | 1328.0ms/step | 15.1 samples/s +Step 32190/44000 | Loss: 2.9097 | LR: 8.65e-05 | 1328.5ms/step | 15.1 samples/s +Step 32200/44000 | Loss: 2.9429 | LR: 8.63e-05 | 1323.5ms/step | 15.1 samples/s +Step 32210/44000 | Loss: 2.9480 | LR: 8.62e-05 | 1325.2ms/step | 15.1 samples/s +Step 32220/44000 | Loss: 2.9310 | LR: 8.61e-05 | 1323.5ms/step | 15.1 samples/s +Step 32230/44000 | Loss: 2.9341 | LR: 8.59e-05 | 1327.6ms/step | 15.1 samples/s +Step 32240/44000 | Loss: 2.9574 | LR: 8.58e-05 | 1327.6ms/step | 15.1 samples/s +Step 32250/44000 | Loss: 2.9534 | LR: 8.57e-05 | 1325.5ms/step | 15.1 samples/s +Validation at step 32250: Loss = 2.9358, Perplexity = 18.84 +Step 32260/44000 | Loss: 2.9125 | LR: 8.55e-05 | 3553.2ms/step | 5.6 samples/s +Step 32270/44000 | Loss: 2.9376 | LR: 8.54e-05 | 1330.3ms/step | 15.0 samples/s +Step 32280/44000 | Loss: 2.9266 | LR: 8.53e-05 | 1328.4ms/step | 15.1 samples/s +Step 32290/44000 | Loss: 2.9333 | LR: 8.51e-05 | 1328.0ms/step | 15.1 samples/s +Step 32300/44000 | Loss: 2.9280 | LR: 8.50e-05 | 1325.2ms/step | 15.1 samples/s +Step 32310/44000 | Loss: 2.9194 | LR: 8.48e-05 | 1328.8ms/step | 15.1 samples/s +Step 32320/44000 | Loss: 2.9391 | LR: 8.47e-05 | 1323.4ms/step | 15.1 samples/s +Step 32330/44000 | Loss: 2.9353 | LR: 8.46e-05 | 1326.1ms/step | 15.1 samples/s +Step 32340/44000 | Loss: 2.9090 | LR: 8.44e-05 | 1326.9ms/step | 15.1 samples/s +Step 32350/44000 | Loss: 2.9305 | LR: 8.43e-05 | 1326.0ms/step | 15.1 samples/s +Step 32360/44000 | Loss: 2.9034 | LR: 8.42e-05 | 1328.2ms/step | 15.1 samples/s +Step 32370/44000 | Loss: 2.9253 | LR: 8.40e-05 | 1324.1ms/step | 15.1 samples/s +Step 32380/44000 | Loss: 2.9274 | LR: 8.39e-05 | 1328.2ms/step | 15.1 samples/s +Step 32390/44000 | Loss: 2.9338 | LR: 8.38e-05 | 1329.6ms/step | 15.0 samples/s +Step 32400/44000 | Loss: 2.9207 | LR: 8.36e-05 | 1329.6ms/step | 15.0 samples/s +Step 32410/44000 | Loss: 2.9102 | LR: 8.35e-05 | 1326.4ms/step | 15.1 samples/s +Step 32420/44000 | Loss: 2.9086 | LR: 8.33e-05 | 1324.0ms/step | 15.1 samples/s +Step 32430/44000 | Loss: 2.9589 | LR: 8.32e-05 | 1324.7ms/step | 15.1 samples/s +Step 32440/44000 | Loss: 2.9067 | LR: 8.31e-05 | 1326.4ms/step | 15.1 samples/s +Step 32450/44000 | Loss: 2.9391 | LR: 8.29e-05 | 1325.0ms/step | 15.1 samples/s +Step 32460/44000 | Loss: 2.9285 | LR: 8.28e-05 | 1329.0ms/step | 15.0 samples/s +Step 32470/44000 | Loss: 2.9309 | LR: 8.27e-05 | 1328.2ms/step | 15.1 samples/s +Step 32480/44000 | Loss: 2.9614 | LR: 8.25e-05 | 1327.1ms/step | 15.1 samples/s +Step 32490/44000 | Loss: 2.9754 | LR: 8.24e-05 | 1324.0ms/step | 15.1 samples/s +Step 32500/44000 | Loss: 2.9357 | LR: 8.23e-05 | 1326.0ms/step | 15.1 samples/s +Validation at step 32500: Loss = 2.9352, Perplexity = 18.82 +Step 32510/44000 | Loss: 2.9575 | LR: 8.21e-05 | 3555.4ms/step | 5.6 samples/s +Step 32520/44000 | Loss: 2.9332 | LR: 8.20e-05 | 1321.8ms/step | 15.1 samples/s +Step 32530/44000 | Loss: 2.9279 | LR: 8.19e-05 | 1326.8ms/step | 15.1 samples/s +Step 32540/44000 | Loss: 2.9168 | LR: 8.17e-05 | 1325.1ms/step | 15.1 samples/s +Step 32550/44000 | Loss: 2.9508 | LR: 8.16e-05 | 1327.6ms/step | 15.1 samples/s +Step 32560/44000 | Loss: 2.9217 | LR: 8.15e-05 | 1326.9ms/step | 15.1 samples/s +Step 32570/44000 | Loss: 2.9089 | LR: 8.13e-05 | 1325.4ms/step | 15.1 samples/s +Step 32580/44000 | Loss: 2.9273 | LR: 8.12e-05 | 1328.4ms/step | 15.1 samples/s +Step 32590/44000 | Loss: 2.9348 | LR: 8.11e-05 | 1327.1ms/step | 15.1 samples/s +Step 32600/44000 | Loss: 2.9145 | LR: 8.09e-05 | 1329.0ms/step | 15.0 samples/s +Step 32610/44000 | Loss: 2.9323 | LR: 8.08e-05 | 1326.0ms/step | 15.1 samples/s +Step 32620/44000 | Loss: 2.9115 | LR: 8.07e-05 | 1325.5ms/step | 15.1 samples/s +Step 32630/44000 | Loss: 2.9047 | LR: 8.05e-05 | 1322.8ms/step | 15.1 samples/s +Step 32640/44000 | Loss: 2.9231 | LR: 8.04e-05 | 1323.0ms/step | 15.1 samples/s +Step 32650/44000 | Loss: 2.8946 | LR: 8.03e-05 | 1327.4ms/step | 15.1 samples/s +Step 32660/44000 | Loss: 2.9100 | LR: 8.01e-05 | 1323.4ms/step | 15.1 samples/s +Step 32670/44000 | Loss: 2.9372 | LR: 8.00e-05 | 1327.4ms/step | 15.1 samples/s +Step 32680/44000 | Loss: 2.9298 | LR: 7.99e-05 | 1327.6ms/step | 15.1 samples/s +Step 32690/44000 | Loss: 2.9261 | LR: 7.97e-05 | 1326.4ms/step | 15.1 samples/s +Step 32700/44000 | Loss: 2.9135 | LR: 7.96e-05 | 1323.8ms/step | 15.1 samples/s +Step 32710/44000 | Loss: 2.9333 | LR: 7.95e-05 | 1324.0ms/step | 15.1 samples/s +Step 32720/44000 | Loss: 2.9123 | LR: 7.93e-05 | 1325.8ms/step | 15.1 samples/s +Step 32730/44000 | Loss: 2.9105 | LR: 7.92e-05 | 1324.0ms/step | 15.1 samples/s +Step 32740/44000 | Loss: 2.9194 | LR: 7.91e-05 | 1325.5ms/step | 15.1 samples/s +Step 32750/44000 | Loss: 2.9327 | LR: 7.89e-05 | 1325.5ms/step | 15.1 samples/s +Validation at step 32750: Loss = 2.9348, Perplexity = 18.82 +Step 32760/44000 | Loss: 2.9572 | LR: 7.88e-05 | 3543.0ms/step | 5.6 samples/s +Step 32770/44000 | Loss: 2.9068 | LR: 7.87e-05 | 1325.7ms/step | 15.1 samples/s +Step 32780/44000 | Loss: 2.9382 | LR: 7.85e-05 | 1327.9ms/step | 15.1 samples/s +Step 32790/44000 | Loss: 2.8921 | LR: 7.84e-05 | 1332.3ms/step | 15.0 samples/s +Step 32800/44000 | Loss: 2.9148 | LR: 7.83e-05 | 1329.6ms/step | 15.0 samples/s +Step 32810/44000 | Loss: 2.9137 | LR: 7.81e-05 | 1326.3ms/step | 15.1 samples/s +Step 32820/44000 | Loss: 2.8928 | LR: 7.80e-05 | 1326.9ms/step | 15.1 samples/s +Step 32830/44000 | Loss: 2.9324 | LR: 7.79e-05 | 1327.8ms/step | 15.1 samples/s +Step 32840/44000 | Loss: 2.9424 | LR: 7.77e-05 | 1329.2ms/step | 15.0 samples/s +Step 32850/44000 | Loss: 2.9296 | LR: 7.76e-05 | 1326.6ms/step | 15.1 samples/s +Step 32860/44000 | Loss: 2.9010 | LR: 7.75e-05 | 1329.3ms/step | 15.0 samples/s +Step 32870/44000 | Loss: 2.9396 | LR: 7.73e-05 | 1328.1ms/step | 15.1 samples/s +Step 32880/44000 | Loss: 2.9002 | LR: 7.72e-05 | 1329.3ms/step | 15.0 samples/s +Step 32890/44000 | Loss: 2.9015 | LR: 7.71e-05 | 1327.9ms/step | 15.1 samples/s +Step 32900/44000 | Loss: 2.9093 | LR: 7.70e-05 | 1327.5ms/step | 15.1 samples/s +Step 32910/44000 | Loss: 2.9135 | LR: 7.68e-05 | 1326.3ms/step | 15.1 samples/s +Step 32920/44000 | Loss: 2.9338 | LR: 7.67e-05 | 1325.3ms/step | 15.1 samples/s +Step 32930/44000 | Loss: 2.9258 | LR: 7.66e-05 | 1327.3ms/step | 15.1 samples/s +Step 32940/44000 | Loss: 2.9230 | LR: 7.64e-05 | 1329.1ms/step | 15.0 samples/s +Step 32950/44000 | Loss: 2.9423 | LR: 7.63e-05 | 1329.2ms/step | 15.0 samples/s +Step 32960/44000 | Loss: 2.9650 | LR: 7.62e-05 | 1326.5ms/step | 15.1 samples/s +Step 32970/44000 | Loss: 2.9534 | LR: 7.60e-05 | 1323.8ms/step | 15.1 samples/s +Step 32980/44000 | Loss: 2.9289 | LR: 7.59e-05 | 1324.4ms/step | 15.1 samples/s +Step 32990/44000 | Loss: 2.9528 | LR: 7.58e-05 | 1325.2ms/step | 15.1 samples/s +Step 33000/44000 | Loss: 2.8997 | LR: 7.56e-05 | 1330.5ms/step | 15.0 samples/s +Validation at step 33000: Loss = 2.9344, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-33000 +Step 33010/44000 | Loss: 2.9357 | LR: 7.55e-05 | 3842.7ms/step | 5.2 samples/s +Step 33020/44000 | Loss: 2.9306 | LR: 7.54e-05 | 1328.5ms/step | 15.1 samples/s +Step 33030/44000 | Loss: 2.9145 | LR: 7.53e-05 | 1326.5ms/step | 15.1 samples/s +Step 33040/44000 | Loss: 2.9296 | LR: 7.51e-05 | 1328.6ms/step | 15.1 samples/s +Step 33050/44000 | Loss: 2.9058 | LR: 7.50e-05 | 1325.0ms/step | 15.1 samples/s +Step 33060/44000 | Loss: 2.9527 | LR: 7.49e-05 | 1326.7ms/step | 15.1 samples/s +Step 33070/44000 | Loss: 2.9240 | LR: 7.47e-05 | 1326.8ms/step | 15.1 samples/s +Step 33080/44000 | Loss: 2.9124 | LR: 7.46e-05 | 1323.1ms/step | 15.1 samples/s +Step 33090/44000 | Loss: 2.9623 | LR: 7.45e-05 | 1326.9ms/step | 15.1 samples/s +Step 33100/44000 | Loss: 2.8934 | LR: 7.44e-05 | 1323.6ms/step | 15.1 samples/s +Step 33110/44000 | Loss: 2.9639 | LR: 7.42e-05 | 1321.1ms/step | 15.1 samples/s +Step 33120/44000 | Loss: 2.9242 | LR: 7.41e-05 | 1323.6ms/step | 15.1 samples/s +Step 33130/44000 | Loss: 2.9334 | LR: 7.40e-05 | 1325.7ms/step | 15.1 samples/s +Step 33140/44000 | Loss: 2.9174 | LR: 7.38e-05 | 1326.9ms/step | 15.1 samples/s +Step 33150/44000 | Loss: 2.9266 | LR: 7.37e-05 | 1327.0ms/step | 15.1 samples/s +Step 33160/44000 | Loss: 2.9353 | LR: 7.36e-05 | 1326.0ms/step | 15.1 samples/s +Step 33170/44000 | Loss: 2.9444 | LR: 7.34e-05 | 1324.8ms/step | 15.1 samples/s +Step 33180/44000 | Loss: 2.8968 | LR: 7.33e-05 | 1326.4ms/step | 15.1 samples/s +Step 33190/44000 | Loss: 2.9290 | LR: 7.32e-05 | 1325.8ms/step | 15.1 samples/s +Step 33200/44000 | Loss: 2.9371 | LR: 7.31e-05 | 1324.9ms/step | 15.1 samples/s +Step 33210/44000 | Loss: 2.9387 | LR: 7.29e-05 | 1329.9ms/step | 15.0 samples/s +Step 33220/44000 | Loss: 2.8748 | LR: 7.28e-05 | 1325.6ms/step | 15.1 samples/s +Step 33230/44000 | Loss: 2.9077 | LR: 7.27e-05 | 1328.6ms/step | 15.1 samples/s +Step 33240/44000 | Loss: 2.9302 | LR: 7.26e-05 | 1329.0ms/step | 15.0 samples/s +Step 33250/44000 | Loss: 2.9278 | LR: 7.24e-05 | 1325.3ms/step | 15.1 samples/s +Validation at step 33250: Loss = 2.9337, Perplexity = 18.80 +Step 33260/44000 | Loss: 2.9213 | LR: 7.23e-05 | 3553.6ms/step | 5.6 samples/s +Step 33270/44000 | Loss: 2.9464 | LR: 7.22e-05 | 1328.4ms/step | 15.1 samples/s +Step 33280/44000 | Loss: 2.9571 | LR: 7.20e-05 | 1326.4ms/step | 15.1 samples/s +Step 33290/44000 | Loss: 2.9142 | LR: 7.19e-05 | 1321.2ms/step | 15.1 samples/s +Step 33300/44000 | Loss: 2.9185 | LR: 7.18e-05 | 1321.9ms/step | 15.1 samples/s +Step 33310/44000 | Loss: 2.9265 | LR: 7.17e-05 | 1323.6ms/step | 15.1 samples/s +Step 33320/44000 | Loss: 2.9461 | LR: 7.15e-05 | 1324.8ms/step | 15.1 samples/s +Step 33330/44000 | Loss: 2.9101 | LR: 7.14e-05 | 1324.5ms/step | 15.1 samples/s +Step 33340/44000 | Loss: 2.9431 | LR: 7.13e-05 | 1327.3ms/step | 15.1 samples/s +Step 33350/44000 | Loss: 2.8904 | LR: 7.11e-05 | 1324.2ms/step | 15.1 samples/s +Step 33360/44000 | Loss: 2.9330 | LR: 7.10e-05 | 1320.4ms/step | 15.1 samples/s +Step 33370/44000 | Loss: 2.9432 | LR: 7.09e-05 | 1321.6ms/step | 15.1 samples/s +Step 33380/44000 | Loss: 2.9235 | LR: 7.08e-05 | 1325.1ms/step | 15.1 samples/s +Step 33390/44000 | Loss: 2.9331 | LR: 7.06e-05 | 1322.9ms/step | 15.1 samples/s +Step 33400/44000 | Loss: 2.9312 | LR: 7.05e-05 | 1323.1ms/step | 15.1 samples/s +Step 33410/44000 | Loss: 2.9163 | LR: 7.04e-05 | 1325.9ms/step | 15.1 samples/s +Step 33420/44000 | Loss: 2.9517 | LR: 7.03e-05 | 1323.5ms/step | 15.1 samples/s +Step 33430/44000 | Loss: 2.9087 | LR: 7.01e-05 | 1322.3ms/step | 15.1 samples/s +Step 33440/44000 | Loss: 2.9307 | LR: 7.00e-05 | 1324.1ms/step | 15.1 samples/s +Step 33450/44000 | Loss: 2.9426 | LR: 6.99e-05 | 1323.6ms/step | 15.1 samples/s +Step 33460/44000 | Loss: 2.9144 | LR: 6.98e-05 | 1324.4ms/step | 15.1 samples/s +Step 33470/44000 | Loss: 2.9197 | LR: 6.96e-05 | 1329.0ms/step | 15.0 samples/s +Step 33480/44000 | Loss: 2.9334 | LR: 6.95e-05 | 1326.9ms/step | 15.1 samples/s +Step 33490/44000 | Loss: 2.9409 | LR: 6.94e-05 | 1325.5ms/step | 15.1 samples/s +Step 33500/44000 | Loss: 2.9282 | LR: 6.93e-05 | 1326.9ms/step | 15.1 samples/s +Validation at step 33500: Loss = 2.9335, Perplexity = 18.79 +Step 33510/44000 | Loss: 2.9064 | LR: 6.91e-05 | 3556.8ms/step | 5.6 samples/s +Step 33520/44000 | Loss: 2.9100 | LR: 6.90e-05 | 1327.5ms/step | 15.1 samples/s +Step 33530/44000 | Loss: 2.9133 | LR: 6.89e-05 | 1325.4ms/step | 15.1 samples/s +Step 33540/44000 | Loss: 2.9401 | LR: 6.88e-05 | 1325.8ms/step | 15.1 samples/s +Step 33550/44000 | Loss: 2.9256 | LR: 6.86e-05 | 1326.9ms/step | 15.1 samples/s +Step 33560/44000 | Loss: 2.9051 | LR: 6.85e-05 | 1327.5ms/step | 15.1 samples/s +Step 33570/44000 | Loss: 2.9348 | LR: 6.84e-05 | 1329.1ms/step | 15.0 samples/s +Step 33580/44000 | Loss: 2.9570 | LR: 6.83e-05 | 1325.8ms/step | 15.1 samples/s +Step 33590/44000 | Loss: 2.9153 | LR: 6.81e-05 | 1328.8ms/step | 15.1 samples/s +Step 33600/44000 | Loss: 2.9392 | LR: 6.80e-05 | 1323.0ms/step | 15.1 samples/s +Step 33610/44000 | Loss: 2.9374 | LR: 6.79e-05 | 1326.1ms/step | 15.1 samples/s +Step 33620/44000 | Loss: 2.9267 | LR: 6.78e-05 | 1321.5ms/step | 15.1 samples/s +Step 33630/44000 | Loss: 2.9449 | LR: 6.76e-05 | 1321.0ms/step | 15.1 samples/s +Step 33640/44000 | Loss: 2.9209 | LR: 6.75e-05 | 1324.3ms/step | 15.1 samples/s +Step 33650/44000 | Loss: 2.9285 | LR: 6.74e-05 | 1326.1ms/step | 15.1 samples/s +Step 33660/44000 | Loss: 2.9317 | LR: 6.73e-05 | 1317.9ms/step | 15.2 samples/s +Step 33670/44000 | Loss: 2.9270 | LR: 6.71e-05 | 1324.7ms/step | 15.1 samples/s +Step 33680/44000 | Loss: 2.9178 | LR: 6.70e-05 | 1329.9ms/step | 15.0 samples/s +Step 33690/44000 | Loss: 2.9169 | LR: 6.69e-05 | 1320.3ms/step | 15.1 samples/s +Step 33700/44000 | Loss: 2.9448 | LR: 6.68e-05 | 1319.1ms/step | 15.2 samples/s +Step 33710/44000 | Loss: 2.9099 | LR: 6.66e-05 | 1322.9ms/step | 15.1 samples/s +Step 33720/44000 | Loss: 2.9155 | LR: 6.65e-05 | 1321.7ms/step | 15.1 samples/s +Step 33730/44000 | Loss: 2.9338 | LR: 6.64e-05 | 1323.4ms/step | 15.1 samples/s +Step 33740/44000 | Loss: 2.9296 | LR: 6.63e-05 | 1326.0ms/step | 15.1 samples/s +Step 33750/44000 | Loss: 2.9265 | LR: 6.61e-05 | 1321.6ms/step | 15.1 samples/s +Validation at step 33750: Loss = 2.9329, Perplexity = 18.78 +Step 33760/44000 | Loss: 2.9220 | LR: 6.60e-05 | 3533.0ms/step | 5.7 samples/s +Step 33770/44000 | Loss: 2.8885 | LR: 6.59e-05 | 1318.3ms/step | 15.2 samples/s +Step 33780/44000 | Loss: 2.9373 | LR: 6.58e-05 | 1320.3ms/step | 15.1 samples/s +Step 33790/44000 | Loss: 2.9285 | LR: 6.57e-05 | 1321.7ms/step | 15.1 samples/s +Step 33800/44000 | Loss: 2.8866 | LR: 6.55e-05 | 1319.1ms/step | 15.2 samples/s +Step 33810/44000 | Loss: 2.9156 | LR: 6.54e-05 | 1318.4ms/step | 15.2 samples/s +Step 33820/44000 | Loss: 2.9266 | LR: 6.53e-05 | 1320.0ms/step | 15.2 samples/s +Step 33830/44000 | Loss: 2.9185 | LR: 6.52e-05 | 1323.8ms/step | 15.1 samples/s +Step 33840/44000 | Loss: 2.9357 | LR: 6.50e-05 | 1325.2ms/step | 15.1 samples/s +Step 33850/44000 | Loss: 2.9021 | LR: 6.49e-05 | 1320.9ms/step | 15.1 samples/s +Step 33860/44000 | Loss: 2.9384 | LR: 6.48e-05 | 1322.7ms/step | 15.1 samples/s +Step 33870/44000 | Loss: 2.9312 | LR: 6.47e-05 | 1324.5ms/step | 15.1 samples/s +Step 33880/44000 | Loss: 2.9111 | LR: 6.46e-05 | 1319.2ms/step | 15.2 samples/s +Step 33890/44000 | Loss: 2.9419 | LR: 6.44e-05 | 1323.3ms/step | 15.1 samples/s +Step 33900/44000 | Loss: 2.8839 | LR: 6.43e-05 | 1321.9ms/step | 15.1 samples/s +Step 33910/44000 | Loss: 2.9084 | LR: 6.42e-05 | 1327.0ms/step | 15.1 samples/s +Step 33920/44000 | Loss: 2.9245 | LR: 6.41e-05 | 1324.0ms/step | 15.1 samples/s +Step 33930/44000 | Loss: 2.9223 | LR: 6.40e-05 | 1321.1ms/step | 15.1 samples/s +Step 33940/44000 | Loss: 2.9377 | LR: 6.38e-05 | 1320.6ms/step | 15.1 samples/s +Step 33950/44000 | Loss: 2.9110 | LR: 6.37e-05 | 1321.4ms/step | 15.1 samples/s +Step 33960/44000 | Loss: 2.8946 | LR: 6.36e-05 | 1316.9ms/step | 15.2 samples/s +Step 33970/44000 | Loss: 2.9285 | LR: 6.35e-05 | 1316.0ms/step | 15.2 samples/s +Step 33980/44000 | Loss: 2.9212 | LR: 6.33e-05 | 1318.7ms/step | 15.2 samples/s +Step 33990/44000 | Loss: 2.9104 | LR: 6.32e-05 | 1323.4ms/step | 15.1 samples/s +Step 34000/44000 | Loss: 2.9042 | LR: 6.31e-05 | 1320.1ms/step | 15.1 samples/s +Validation at step 34000: Loss = 2.9328, Perplexity = 18.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-34000 +Step 34010/44000 | Loss: 2.9436 | LR: 6.30e-05 | 3847.8ms/step | 5.2 samples/s +Step 34020/44000 | Loss: 2.9198 | LR: 6.29e-05 | 1323.9ms/step | 15.1 samples/s +Step 34030/44000 | Loss: 2.9165 | LR: 6.27e-05 | 1325.9ms/step | 15.1 samples/s +Step 34040/44000 | Loss: 2.9227 | LR: 6.26e-05 | 1326.7ms/step | 15.1 samples/s +Step 34050/44000 | Loss: 2.9405 | LR: 6.25e-05 | 1324.3ms/step | 15.1 samples/s +Step 34060/44000 | Loss: 2.9034 | LR: 6.24e-05 | 1326.8ms/step | 15.1 samples/s +Step 34070/44000 | Loss: 2.9179 | LR: 6.23e-05 | 1323.7ms/step | 15.1 samples/s +Step 34080/44000 | Loss: 2.9234 | LR: 6.21e-05 | 1328.8ms/step | 15.1 samples/s +Step 34090/44000 | Loss: 2.9003 | LR: 6.20e-05 | 1323.8ms/step | 15.1 samples/s +Step 34100/44000 | Loss: 2.9021 | LR: 6.19e-05 | 1327.5ms/step | 15.1 samples/s +Step 34110/44000 | Loss: 2.9344 | LR: 6.18e-05 | 1328.3ms/step | 15.1 samples/s +Step 34120/44000 | Loss: 2.9398 | LR: 6.17e-05 | 1332.0ms/step | 15.0 samples/s +Step 34130/44000 | Loss: 2.9118 | LR: 6.15e-05 | 1323.9ms/step | 15.1 samples/s +Step 34140/44000 | Loss: 2.9245 | LR: 6.14e-05 | 1330.1ms/step | 15.0 samples/s +Step 34150/44000 | Loss: 2.9472 | LR: 6.13e-05 | 1326.0ms/step | 15.1 samples/s +Step 34160/44000 | Loss: 2.9388 | LR: 6.12e-05 | 1326.8ms/step | 15.1 samples/s +Step 34170/44000 | Loss: 2.9099 | LR: 6.11e-05 | 1318.1ms/step | 15.2 samples/s +Step 34180/44000 | Loss: 2.9137 | LR: 6.09e-05 | 1317.4ms/step | 15.2 samples/s +Step 34190/44000 | Loss: 2.8914 | LR: 6.08e-05 | 1321.2ms/step | 15.1 samples/s +Step 34200/44000 | Loss: 2.8860 | LR: 6.07e-05 | 1324.3ms/step | 15.1 samples/s +Step 34210/44000 | Loss: 2.8943 | LR: 6.06e-05 | 1324.9ms/step | 15.1 samples/s +Step 34220/44000 | Loss: 2.9256 | LR: 6.05e-05 | 1321.4ms/step | 15.1 samples/s +Step 34230/44000 | Loss: 2.9319 | LR: 6.04e-05 | 1321.7ms/step | 15.1 samples/s +Step 34240/44000 | Loss: 2.9410 | LR: 6.02e-05 | 1317.5ms/step | 15.2 samples/s +Step 34250/44000 | Loss: 2.9199 | LR: 6.01e-05 | 1316.7ms/step | 15.2 samples/s +Validation at step 34250: Loss = 2.9321, Perplexity = 18.77 +Step 34260/44000 | Loss: 2.9486 | LR: 6.00e-05 | 3543.3ms/step | 5.6 samples/s +Step 34270/44000 | Loss: 2.8922 | LR: 5.99e-05 | 1320.7ms/step | 15.1 samples/s +Step 34280/44000 | Loss: 2.9292 | LR: 5.98e-05 | 1319.1ms/step | 15.2 samples/s +Step 34290/44000 | Loss: 2.9403 | LR: 5.96e-05 | 1322.5ms/step | 15.1 samples/s +Step 34300/44000 | Loss: 2.9364 | LR: 5.95e-05 | 1321.7ms/step | 15.1 samples/s +Step 34310/44000 | Loss: 2.9322 | LR: 5.94e-05 | 1319.4ms/step | 15.2 samples/s +Step 34320/44000 | Loss: 2.9141 | LR: 5.93e-05 | 1318.4ms/step | 15.2 samples/s +Step 34330/44000 | Loss: 2.9664 | LR: 5.92e-05 | 1319.5ms/step | 15.2 samples/s +Step 34340/44000 | Loss: 2.9246 | LR: 5.91e-05 | 1324.1ms/step | 15.1 samples/s +Step 34350/44000 | Loss: 2.9195 | LR: 5.89e-05 | 1318.3ms/step | 15.2 samples/s +Step 34360/44000 | Loss: 2.9196 | LR: 5.88e-05 | 1318.1ms/step | 15.2 samples/s +Step 34370/44000 | Loss: 2.9367 | LR: 5.87e-05 | 1321.2ms/step | 15.1 samples/s +Step 34380/44000 | Loss: 2.9122 | LR: 5.86e-05 | 1320.6ms/step | 15.1 samples/s +Step 34390/44000 | Loss: 2.9394 | LR: 5.85e-05 | 1321.4ms/step | 15.1 samples/s +Step 34400/44000 | Loss: 2.9262 | LR: 5.84e-05 | 1322.0ms/step | 15.1 samples/s +Step 34410/44000 | Loss: 2.9043 | LR: 5.82e-05 | 1321.2ms/step | 15.1 samples/s +Step 34420/44000 | Loss: 2.8992 | LR: 5.81e-05 | 1319.7ms/step | 15.2 samples/s +Step 34430/44000 | Loss: 2.9423 | LR: 5.80e-05 | 1323.3ms/step | 15.1 samples/s +Step 34440/44000 | Loss: 2.9248 | LR: 5.79e-05 | 1321.6ms/step | 15.1 samples/s +Step 34450/44000 | Loss: 2.9062 | LR: 5.78e-05 | 1324.3ms/step | 15.1 samples/s +Step 34460/44000 | Loss: 2.9208 | LR: 5.77e-05 | 1320.8ms/step | 15.1 samples/s +Step 34470/44000 | Loss: 2.9175 | LR: 5.75e-05 | 1323.4ms/step | 15.1 samples/s +Step 34480/44000 | Loss: 2.9473 | LR: 5.74e-05 | 1322.3ms/step | 15.1 samples/s +Step 34490/44000 | Loss: 2.9236 | LR: 5.73e-05 | 1318.8ms/step | 15.2 samples/s +Step 34500/44000 | Loss: 2.9227 | LR: 5.72e-05 | 1321.5ms/step | 15.1 samples/s +Validation at step 34500: Loss = 2.9319, Perplexity = 18.76 +Step 34510/44000 | Loss: 2.8907 | LR: 5.71e-05 | 3551.2ms/step | 5.6 samples/s +Step 34520/44000 | Loss: 2.9218 | LR: 5.70e-05 | 1330.1ms/step | 15.0 samples/s +Step 34530/44000 | Loss: 2.9208 | LR: 5.69e-05 | 1324.9ms/step | 15.1 samples/s +Step 34540/44000 | Loss: 2.9155 | LR: 5.67e-05 | 1327.4ms/step | 15.1 samples/s +Step 34550/44000 | Loss: 2.9008 | LR: 5.66e-05 | 1328.1ms/step | 15.1 samples/s +Step 34560/44000 | Loss: 2.9504 | LR: 5.65e-05 | 1331.0ms/step | 15.0 samples/s +Step 34570/44000 | Loss: 2.9627 | LR: 5.64e-05 | 1327.1ms/step | 15.1 samples/s +Step 34580/44000 | Loss: 2.9143 | LR: 5.63e-05 | 1330.1ms/step | 15.0 samples/s +Step 34590/44000 | Loss: 2.9116 | LR: 5.62e-05 | 1327.9ms/step | 15.1 samples/s +Step 34600/44000 | Loss: 2.9308 | LR: 5.60e-05 | 1323.7ms/step | 15.1 samples/s +Step 34610/44000 | Loss: 2.9329 | LR: 5.59e-05 | 1326.1ms/step | 15.1 samples/s +Step 34620/44000 | Loss: 2.9028 | LR: 5.58e-05 | 1325.8ms/step | 15.1 samples/s +Step 34630/44000 | Loss: 2.9402 | LR: 5.57e-05 | 1325.1ms/step | 15.1 samples/s +Step 34640/44000 | Loss: 2.8862 | LR: 5.56e-05 | 1325.7ms/step | 15.1 samples/s +Step 34650/44000 | Loss: 2.9232 | LR: 5.55e-05 | 1324.6ms/step | 15.1 samples/s +Step 34660/44000 | Loss: 2.9193 | LR: 5.54e-05 | 1324.9ms/step | 15.1 samples/s +Step 34670/44000 | Loss: 2.9225 | LR: 5.52e-05 | 1325.1ms/step | 15.1 samples/s +Step 34680/44000 | Loss: 2.9130 | LR: 5.51e-05 | 1326.0ms/step | 15.1 samples/s +Step 34690/44000 | Loss: 2.9150 | LR: 5.50e-05 | 1325.0ms/step | 15.1 samples/s +Step 34700/44000 | Loss: 2.9109 | LR: 5.49e-05 | 1325.9ms/step | 15.1 samples/s +Step 34710/44000 | Loss: 2.9046 | LR: 5.48e-05 | 1326.3ms/step | 15.1 samples/s +Step 34720/44000 | Loss: 2.9263 | LR: 5.47e-05 | 1327.3ms/step | 15.1 samples/s +Step 34730/44000 | Loss: 2.9062 | LR: 5.46e-05 | 1327.7ms/step | 15.1 samples/s +Step 34740/44000 | Loss: 2.9598 | LR: 5.45e-05 | 1329.0ms/step | 15.0 samples/s +Step 34750/44000 | Loss: 2.9313 | LR: 5.43e-05 | 1327.6ms/step | 15.1 samples/s +Validation at step 34750: Loss = 2.9317, Perplexity = 18.76 +Step 34760/44000 | Loss: 2.9009 | LR: 5.42e-05 | 3558.4ms/step | 5.6 samples/s +Step 34770/44000 | Loss: 2.9553 | LR: 5.41e-05 | 1325.3ms/step | 15.1 samples/s +Step 34780/44000 | Loss: 2.9241 | LR: 5.40e-05 | 1322.6ms/step | 15.1 samples/s +Step 34790/44000 | Loss: 2.9336 | LR: 5.39e-05 | 1328.7ms/step | 15.1 samples/s +Step 34800/44000 | Loss: 2.9467 | LR: 5.38e-05 | 1328.1ms/step | 15.1 samples/s +Step 34810/44000 | Loss: 2.9473 | LR: 5.37e-05 | 1326.3ms/step | 15.1 samples/s +Step 34820/44000 | Loss: 2.9264 | LR: 5.36e-05 | 1326.7ms/step | 15.1 samples/s +Step 34830/44000 | Loss: 2.9528 | LR: 5.34e-05 | 1325.5ms/step | 15.1 samples/s +Step 34840/44000 | Loss: 2.9324 | LR: 5.33e-05 | 1324.2ms/step | 15.1 samples/s +Step 34850/44000 | Loss: 2.8882 | LR: 5.32e-05 | 1323.8ms/step | 15.1 samples/s +Step 34860/44000 | Loss: 2.9084 | LR: 5.31e-05 | 1325.6ms/step | 15.1 samples/s +Step 34870/44000 | Loss: 2.9015 | LR: 5.30e-05 | 1325.7ms/step | 15.1 samples/s +Step 34880/44000 | Loss: 2.9116 | LR: 5.29e-05 | 1328.4ms/step | 15.1 samples/s +Step 34890/44000 | Loss: 2.8974 | LR: 5.28e-05 | 1324.8ms/step | 15.1 samples/s +Step 34900/44000 | Loss: 2.9340 | LR: 5.27e-05 | 1330.5ms/step | 15.0 samples/s +Step 34910/44000 | Loss: 2.9274 | LR: 5.25e-05 | 1327.6ms/step | 15.1 samples/s +Step 34920/44000 | Loss: 2.9145 | LR: 5.24e-05 | 1325.4ms/step | 15.1 samples/s +Step 34930/44000 | Loss: 2.9234 | LR: 5.23e-05 | 1326.0ms/step | 15.1 samples/s +Step 34940/44000 | Loss: 2.9299 | LR: 5.22e-05 | 1325.1ms/step | 15.1 samples/s +Step 34950/44000 | Loss: 2.9049 | LR: 5.21e-05 | 1326.1ms/step | 15.1 samples/s +Step 34960/44000 | Loss: 2.9490 | LR: 5.20e-05 | 1326.8ms/step | 15.1 samples/s +Step 34970/44000 | Loss: 2.9278 | LR: 5.19e-05 | 1326.3ms/step | 15.1 samples/s +Step 34980/44000 | Loss: 2.9410 | LR: 5.18e-05 | 1327.5ms/step | 15.1 samples/s +Step 34990/44000 | Loss: 2.9226 | LR: 5.17e-05 | 1329.2ms/step | 15.0 samples/s +Step 35000/44000 | Loss: 2.9308 | LR: 5.15e-05 | 1324.1ms/step | 15.1 samples/s +Validation at step 35000: Loss = 2.9316, Perplexity = 18.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-35000 +Step 35010/44000 | Loss: 2.9293 | LR: 5.14e-05 | 3854.5ms/step | 5.2 samples/s +Step 35020/44000 | Loss: 2.9339 | LR: 5.13e-05 | 1323.6ms/step | 15.1 samples/s +Step 35030/44000 | Loss: 2.9090 | LR: 5.12e-05 | 1322.4ms/step | 15.1 samples/s +Step 35040/44000 | Loss: 2.9524 | LR: 5.11e-05 | 1324.7ms/step | 15.1 samples/s +Step 35050/44000 | Loss: 2.9111 | LR: 5.10e-05 | 1330.0ms/step | 15.0 samples/s +Step 35060/44000 | Loss: 2.9030 | LR: 5.09e-05 | 1323.2ms/step | 15.1 samples/s +Step 35070/44000 | Loss: 2.8993 | LR: 5.08e-05 | 1323.6ms/step | 15.1 samples/s +Step 35080/44000 | Loss: 2.9262 | LR: 5.07e-05 | 1325.8ms/step | 15.1 samples/s +Step 35090/44000 | Loss: 2.9183 | LR: 5.06e-05 | 1327.2ms/step | 15.1 samples/s +Step 35100/44000 | Loss: 2.9279 | LR: 5.04e-05 | 1328.0ms/step | 15.1 samples/s +Step 35110/44000 | Loss: 2.9217 | LR: 5.03e-05 | 1323.6ms/step | 15.1 samples/s +Step 35120/44000 | Loss: 2.9288 | LR: 5.02e-05 | 1324.1ms/step | 15.1 samples/s +Step 35130/44000 | Loss: 2.9221 | LR: 5.01e-05 | 1324.8ms/step | 15.1 samples/s +Step 35140/44000 | Loss: 2.9160 | LR: 5.00e-05 | 1326.0ms/step | 15.1 samples/s +Step 35150/44000 | Loss: 2.8927 | LR: 4.99e-05 | 1326.9ms/step | 15.1 samples/s +Step 35160/44000 | Loss: 2.9255 | LR: 4.98e-05 | 1327.7ms/step | 15.1 samples/s +Step 35170/44000 | Loss: 2.9272 | LR: 4.97e-05 | 1323.6ms/step | 15.1 samples/s +Step 35180/44000 | Loss: 2.9287 | LR: 4.96e-05 | 1326.3ms/step | 15.1 samples/s +Step 35190/44000 | Loss: 2.9171 | LR: 4.95e-05 | 1327.7ms/step | 15.1 samples/s +Step 35200/44000 | Loss: 2.9039 | LR: 4.94e-05 | 1326.7ms/step | 15.1 samples/s +Step 35210/44000 | Loss: 2.9428 | LR: 4.93e-05 | 1324.8ms/step | 15.1 samples/s +Step 35220/44000 | Loss: 2.9180 | LR: 4.91e-05 | 1324.4ms/step | 15.1 samples/s +Step 35230/44000 | Loss: 2.9144 | LR: 4.90e-05 | 1328.5ms/step | 15.1 samples/s +Step 35240/44000 | Loss: 2.9190 | LR: 4.89e-05 | 1326.4ms/step | 15.1 samples/s +Step 35250/44000 | Loss: 2.9022 | LR: 4.88e-05 | 1325.2ms/step | 15.1 samples/s +Validation at step 35250: Loss = 2.9313, Perplexity = 18.75 +Step 35260/44000 | Loss: 2.9458 | LR: 4.87e-05 | 3544.0ms/step | 5.6 samples/s +Step 35270/44000 | Loss: 2.9021 | LR: 4.86e-05 | 1325.4ms/step | 15.1 samples/s +Step 35280/44000 | Loss: 2.9315 | LR: 4.85e-05 | 1327.4ms/step | 15.1 samples/s +Step 35290/44000 | Loss: 2.9097 | LR: 4.84e-05 | 1322.1ms/step | 15.1 samples/s +Step 35300/44000 | Loss: 2.8969 | LR: 4.83e-05 | 1328.6ms/step | 15.1 samples/s +Step 35310/44000 | Loss: 2.8886 | LR: 4.82e-05 | 1324.1ms/step | 15.1 samples/s +Step 35320/44000 | Loss: 2.9210 | LR: 4.81e-05 | 1325.5ms/step | 15.1 samples/s +Step 35330/44000 | Loss: 2.9345 | LR: 4.80e-05 | 1321.1ms/step | 15.1 samples/s +Step 35340/44000 | Loss: 2.9093 | LR: 4.79e-05 | 1320.0ms/step | 15.2 samples/s +Step 35350/44000 | Loss: 2.9140 | LR: 4.77e-05 | 1316.6ms/step | 15.2 samples/s +Step 35360/44000 | Loss: 2.9318 | LR: 4.76e-05 | 1318.3ms/step | 15.2 samples/s +Step 35370/44000 | Loss: 2.9313 | LR: 4.75e-05 | 1316.1ms/step | 15.2 samples/s +Step 35380/44000 | Loss: 2.9264 | LR: 4.74e-05 | 1318.6ms/step | 15.2 samples/s +Step 35390/44000 | Loss: 2.9220 | LR: 4.73e-05 | 1321.6ms/step | 15.1 samples/s +Step 35400/44000 | Loss: 2.9231 | LR: 4.72e-05 | 1321.3ms/step | 15.1 samples/s +Step 35410/44000 | Loss: 2.8979 | LR: 4.71e-05 | 1320.9ms/step | 15.1 samples/s +Step 35420/44000 | Loss: 2.9628 | LR: 4.70e-05 | 1324.9ms/step | 15.1 samples/s +Step 35430/44000 | Loss: 2.9246 | LR: 4.69e-05 | 1326.9ms/step | 15.1 samples/s +Step 35440/44000 | Loss: 2.9306 | LR: 4.68e-05 | 1326.8ms/step | 15.1 samples/s +Step 35450/44000 | Loss: 2.9167 | LR: 4.67e-05 | 1328.3ms/step | 15.1 samples/s +Step 35460/44000 | Loss: 2.9161 | LR: 4.66e-05 | 1326.2ms/step | 15.1 samples/s +Step 35470/44000 | Loss: 2.9115 | LR: 4.65e-05 | 1327.4ms/step | 15.1 samples/s +Step 35480/44000 | Loss: 2.9213 | LR: 4.64e-05 | 1323.2ms/step | 15.1 samples/s +Step 35490/44000 | Loss: 2.9274 | LR: 4.63e-05 | 1323.3ms/step | 15.1 samples/s +Step 35500/44000 | Loss: 2.9427 | LR: 4.62e-05 | 1326.7ms/step | 15.1 samples/s +Validation at step 35500: Loss = 2.9310, Perplexity = 18.75 +Step 35510/44000 | Loss: 2.9207 | LR: 4.61e-05 | 3541.2ms/step | 5.6 samples/s +Step 35520/44000 | Loss: 2.9204 | LR: 4.59e-05 | 1316.9ms/step | 15.2 samples/s +Step 35530/44000 | Loss: 2.9581 | LR: 4.58e-05 | 1316.3ms/step | 15.2 samples/s +Step 35540/44000 | Loss: 2.9372 | LR: 4.57e-05 | 1314.8ms/step | 15.2 samples/s +Step 35550/44000 | Loss: 2.9506 | LR: 4.56e-05 | 1319.6ms/step | 15.2 samples/s +Step 35560/44000 | Loss: 2.9086 | LR: 4.55e-05 | 1324.6ms/step | 15.1 samples/s +Step 35570/44000 | Loss: 2.9287 | LR: 4.54e-05 | 1322.4ms/step | 15.1 samples/s +Step 35580/44000 | Loss: 2.9208 | LR: 4.53e-05 | 1321.9ms/step | 15.1 samples/s +Step 35590/44000 | Loss: 2.9410 | LR: 4.52e-05 | 1321.6ms/step | 15.1 samples/s +Step 35600/44000 | Loss: 2.9316 | LR: 4.51e-05 | 1323.6ms/step | 15.1 samples/s +Step 35610/44000 | Loss: 2.8913 | LR: 4.50e-05 | 1323.9ms/step | 15.1 samples/s +Step 35620/44000 | Loss: 2.9421 | LR: 4.49e-05 | 1321.7ms/step | 15.1 samples/s +Step 35630/44000 | Loss: 2.9429 | LR: 4.48e-05 | 1322.0ms/step | 15.1 samples/s +Step 35640/44000 | Loss: 2.9358 | LR: 4.47e-05 | 1313.1ms/step | 15.2 samples/s +Step 35650/44000 | Loss: 2.9194 | LR: 4.46e-05 | 1318.9ms/step | 15.2 samples/s +Step 35660/44000 | Loss: 2.9309 | LR: 4.45e-05 | 1322.8ms/step | 15.1 samples/s +Step 35670/44000 | Loss: 2.9172 | LR: 4.44e-05 | 1321.4ms/step | 15.1 samples/s +Step 35680/44000 | Loss: 2.9146 | LR: 4.43e-05 | 1324.0ms/step | 15.1 samples/s +Step 35690/44000 | Loss: 2.9036 | LR: 4.42e-05 | 1324.5ms/step | 15.1 samples/s +Step 35700/44000 | Loss: 2.9424 | LR: 4.41e-05 | 1324.8ms/step | 15.1 samples/s +Step 35710/44000 | Loss: 2.9098 | LR: 4.40e-05 | 1322.5ms/step | 15.1 samples/s +Step 35720/44000 | Loss: 2.9072 | LR: 4.39e-05 | 1322.6ms/step | 15.1 samples/s +Step 35730/44000 | Loss: 2.9329 | LR: 4.38e-05 | 1324.8ms/step | 15.1 samples/s +Step 35740/44000 | Loss: 2.8893 | LR: 4.37e-05 | 1322.8ms/step | 15.1 samples/s +Step 35750/44000 | Loss: 2.9211 | LR: 4.36e-05 | 1318.1ms/step | 15.2 samples/s +Validation at step 35750: Loss = 2.9313, Perplexity = 18.75 +Step 35760/44000 | Loss: 2.9277 | LR: 4.35e-05 | 3536.3ms/step | 5.7 samples/s +Step 35770/44000 | Loss: 2.9155 | LR: 4.34e-05 | 1319.2ms/step | 15.2 samples/s +Step 35780/44000 | Loss: 2.9266 | LR: 4.33e-05 | 1321.5ms/step | 15.1 samples/s +Step 35790/44000 | Loss: 2.9307 | LR: 4.32e-05 | 1316.7ms/step | 15.2 samples/s +Step 35800/44000 | Loss: 2.9198 | LR: 4.31e-05 | 1320.6ms/step | 15.1 samples/s +Step 35810/44000 | Loss: 2.9194 | LR: 4.29e-05 | 1321.8ms/step | 15.1 samples/s +Step 35820/44000 | Loss: 2.9123 | LR: 4.28e-05 | 1320.5ms/step | 15.1 samples/s +Step 35830/44000 | Loss: 2.9115 | LR: 4.27e-05 | 1323.5ms/step | 15.1 samples/s +Step 35840/44000 | Loss: 2.8977 | LR: 4.26e-05 | 1323.6ms/step | 15.1 samples/s +Step 35850/44000 | Loss: 2.9046 | LR: 4.25e-05 | 1321.3ms/step | 15.1 samples/s +Step 35860/44000 | Loss: 2.9166 | LR: 4.24e-05 | 1324.1ms/step | 15.1 samples/s +Step 35870/44000 | Loss: 2.9525 | LR: 4.23e-05 | 1322.9ms/step | 15.1 samples/s +Step 35880/44000 | Loss: 2.9203 | LR: 4.22e-05 | 1322.8ms/step | 15.1 samples/s +Step 35890/44000 | Loss: 2.9079 | LR: 4.21e-05 | 1320.2ms/step | 15.1 samples/s +Step 35900/44000 | Loss: 2.9063 | LR: 4.20e-05 | 1319.9ms/step | 15.2 samples/s +Step 35910/44000 | Loss: 2.9319 | LR: 4.19e-05 | 1322.8ms/step | 15.1 samples/s +Step 35920/44000 | Loss: 2.9320 | LR: 4.18e-05 | 1324.9ms/step | 15.1 samples/s +Step 35930/44000 | Loss: 2.9263 | LR: 4.17e-05 | 1324.7ms/step | 15.1 samples/s +Step 35940/44000 | Loss: 2.8990 | LR: 4.16e-05 | 1323.4ms/step | 15.1 samples/s +Step 35950/44000 | Loss: 2.9290 | LR: 4.15e-05 | 1324.0ms/step | 15.1 samples/s +Step 35960/44000 | Loss: 2.9320 | LR: 4.14e-05 | 1322.9ms/step | 15.1 samples/s +Step 35970/44000 | Loss: 2.9173 | LR: 4.13e-05 | 1321.1ms/step | 15.1 samples/s +Step 35980/44000 | Loss: 2.9309 | LR: 4.12e-05 | 1321.0ms/step | 15.1 samples/s +Step 35990/44000 | Loss: 2.8906 | LR: 4.11e-05 | 1324.8ms/step | 15.1 samples/s +Step 36000/44000 | Loss: 2.9005 | LR: 4.10e-05 | 1319.7ms/step | 15.2 samples/s +Validation at step 36000: Loss = 2.9308, Perplexity = 18.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-36000 +Step 36010/44000 | Loss: 2.9376 | LR: 4.09e-05 | 3853.3ms/step | 5.2 samples/s +Step 36020/44000 | Loss: 2.9208 | LR: 4.08e-05 | 1324.3ms/step | 15.1 samples/s +Step 36030/44000 | Loss: 2.9034 | LR: 4.07e-05 | 1321.5ms/step | 15.1 samples/s +Step 36040/44000 | Loss: 2.9135 | LR: 4.06e-05 | 1320.8ms/step | 15.1 samples/s +Step 36050/44000 | Loss: 2.9245 | LR: 4.05e-05 | 1323.1ms/step | 15.1 samples/s +Step 36060/44000 | Loss: 2.9200 | LR: 4.04e-05 | 1322.3ms/step | 15.1 samples/s +Step 36070/44000 | Loss: 2.9239 | LR: 4.03e-05 | 1326.8ms/step | 15.1 samples/s +Step 36080/44000 | Loss: 2.9092 | LR: 4.02e-05 | 1322.5ms/step | 15.1 samples/s +Step 36090/44000 | Loss: 2.9116 | LR: 4.01e-05 | 1323.2ms/step | 15.1 samples/s +Step 36100/44000 | Loss: 2.9207 | LR: 4.00e-05 | 1326.0ms/step | 15.1 samples/s +Step 36110/44000 | Loss: 2.9467 | LR: 3.99e-05 | 1325.8ms/step | 15.1 samples/s +Step 36120/44000 | Loss: 2.9409 | LR: 3.98e-05 | 1323.5ms/step | 15.1 samples/s +Step 36130/44000 | Loss: 2.9216 | LR: 3.97e-05 | 1325.7ms/step | 15.1 samples/s +Step 36140/44000 | Loss: 2.9326 | LR: 3.97e-05 | 1324.2ms/step | 15.1 samples/s +Step 36150/44000 | Loss: 2.9563 | LR: 3.96e-05 | 1322.7ms/step | 15.1 samples/s +Step 36160/44000 | Loss: 2.9454 | LR: 3.95e-05 | 1320.9ms/step | 15.1 samples/s +Step 36170/44000 | Loss: 2.9164 | LR: 3.94e-05 | 1327.6ms/step | 15.1 samples/s +Step 36180/44000 | Loss: 2.9608 | LR: 3.93e-05 | 1327.5ms/step | 15.1 samples/s +Step 36190/44000 | Loss: 2.9221 | LR: 3.92e-05 | 1328.4ms/step | 15.1 samples/s +Step 36200/44000 | Loss: 2.9413 | LR: 3.91e-05 | 1331.2ms/step | 15.0 samples/s +Step 36210/44000 | Loss: 2.9238 | LR: 3.90e-05 | 1321.5ms/step | 15.1 samples/s +Step 36220/44000 | Loss: 2.9019 | LR: 3.89e-05 | 1323.1ms/step | 15.1 samples/s +Step 36230/44000 | Loss: 2.9101 | LR: 3.88e-05 | 1321.6ms/step | 15.1 samples/s +Step 36240/44000 | Loss: 2.9236 | LR: 3.87e-05 | 1326.6ms/step | 15.1 samples/s +Step 36250/44000 | Loss: 2.9613 | LR: 3.86e-05 | 1320.1ms/step | 15.2 samples/s +Validation at step 36250: Loss = 2.9303, Perplexity = 18.73 +Step 36260/44000 | Loss: 2.9423 | LR: 3.85e-05 | 3547.6ms/step | 5.6 samples/s +Step 36270/44000 | Loss: 2.9083 | LR: 3.84e-05 | 1319.5ms/step | 15.2 samples/s +Step 36280/44000 | Loss: 2.9080 | LR: 3.83e-05 | 1325.7ms/step | 15.1 samples/s +Step 36290/44000 | Loss: 2.9275 | LR: 3.82e-05 | 1323.7ms/step | 15.1 samples/s +Step 36300/44000 | Loss: 2.9415 | LR: 3.81e-05 | 1325.0ms/step | 15.1 samples/s +Step 36310/44000 | Loss: 2.9191 | LR: 3.80e-05 | 1326.4ms/step | 15.1 samples/s +Step 36320/44000 | Loss: 2.9132 | LR: 3.79e-05 | 1327.8ms/step | 15.1 samples/s +Step 36330/44000 | Loss: 2.9492 | LR: 3.78e-05 | 1325.5ms/step | 15.1 samples/s +Step 36340/44000 | Loss: 2.9281 | LR: 3.77e-05 | 1322.6ms/step | 15.1 samples/s +Step 36350/44000 | Loss: 2.9209 | LR: 3.76e-05 | 1326.4ms/step | 15.1 samples/s +Step 36360/44000 | Loss: 2.9179 | LR: 3.75e-05 | 1327.5ms/step | 15.1 samples/s +Step 36370/44000 | Loss: 2.9649 | LR: 3.74e-05 | 1327.4ms/step | 15.1 samples/s +Step 36380/44000 | Loss: 2.9320 | LR: 3.73e-05 | 1329.4ms/step | 15.0 samples/s +Step 36390/44000 | Loss: 2.9065 | LR: 3.72e-05 | 1322.9ms/step | 15.1 samples/s +Step 36400/44000 | Loss: 2.9290 | LR: 3.71e-05 | 1321.9ms/step | 15.1 samples/s +Step 36410/44000 | Loss: 2.8964 | LR: 3.70e-05 | 1322.4ms/step | 15.1 samples/s +Step 36420/44000 | Loss: 2.9176 | LR: 3.69e-05 | 1327.2ms/step | 15.1 samples/s +Step 36430/44000 | Loss: 2.8736 | LR: 3.69e-05 | 1322.1ms/step | 15.1 samples/s +Step 36440/44000 | Loss: 2.9114 | LR: 3.68e-05 | 1321.4ms/step | 15.1 samples/s +Step 36450/44000 | Loss: 2.9113 | LR: 3.67e-05 | 1320.7ms/step | 15.1 samples/s +Step 36460/44000 | Loss: 2.8925 | LR: 3.66e-05 | 1324.2ms/step | 15.1 samples/s +Step 36470/44000 | Loss: 2.9265 | LR: 3.65e-05 | 1325.8ms/step | 15.1 samples/s +Step 36480/44000 | Loss: 2.9465 | LR: 3.64e-05 | 1330.7ms/step | 15.0 samples/s +Step 36490/44000 | Loss: 2.9052 | LR: 3.63e-05 | 1325.4ms/step | 15.1 samples/s +Step 36500/44000 | Loss: 2.9357 | LR: 3.62e-05 | 1326.4ms/step | 15.1 samples/s +Validation at step 36500: Loss = 2.9303, Perplexity = 18.73 +Step 36510/44000 | Loss: 2.9269 | LR: 3.61e-05 | 3551.0ms/step | 5.6 samples/s +Step 36520/44000 | Loss: 2.9249 | LR: 3.60e-05 | 1326.0ms/step | 15.1 samples/s +Step 36530/44000 | Loss: 2.9416 | LR: 3.59e-05 | 1327.1ms/step | 15.1 samples/s +Step 36540/44000 | Loss: 2.9454 | LR: 3.58e-05 | 1325.7ms/step | 15.1 samples/s +Step 36550/44000 | Loss: 2.9431 | LR: 3.57e-05 | 1323.7ms/step | 15.1 samples/s +Step 36560/44000 | Loss: 2.9134 | LR: 3.56e-05 | 1323.0ms/step | 15.1 samples/s +Step 36570/44000 | Loss: 2.9153 | LR: 3.55e-05 | 1321.2ms/step | 15.1 samples/s +Step 36580/44000 | Loss: 2.8999 | LR: 3.54e-05 | 1324.9ms/step | 15.1 samples/s +Step 36590/44000 | Loss: 2.9407 | LR: 3.53e-05 | 1326.6ms/step | 15.1 samples/s +Step 36600/44000 | Loss: 2.8938 | LR: 3.53e-05 | 1324.2ms/step | 15.1 samples/s +Step 36610/44000 | Loss: 2.9012 | LR: 3.52e-05 | 1323.0ms/step | 15.1 samples/s +Step 36620/44000 | Loss: 2.9382 | LR: 3.51e-05 | 1323.7ms/step | 15.1 samples/s +Step 36630/44000 | Loss: 2.8977 | LR: 3.50e-05 | 1323.6ms/step | 15.1 samples/s +Step 36640/44000 | Loss: 2.8933 | LR: 3.49e-05 | 1323.5ms/step | 15.1 samples/s +Step 36650/44000 | Loss: 2.9133 | LR: 3.48e-05 | 1331.3ms/step | 15.0 samples/s +Step 36660/44000 | Loss: 2.8942 | LR: 3.47e-05 | 1328.2ms/step | 15.1 samples/s +Step 36670/44000 | Loss: 2.8937 | LR: 3.46e-05 | 1326.2ms/step | 15.1 samples/s +Step 36680/44000 | Loss: 2.9458 | LR: 3.45e-05 | 1326.9ms/step | 15.1 samples/s +Step 36690/44000 | Loss: 2.9424 | LR: 3.44e-05 | 1325.5ms/step | 15.1 samples/s +Step 36700/44000 | Loss: 2.9297 | LR: 3.43e-05 | 1320.7ms/step | 15.1 samples/s +Step 36710/44000 | Loss: 2.9454 | LR: 3.42e-05 | 1325.4ms/step | 15.1 samples/s +Step 36720/44000 | Loss: 2.9228 | LR: 3.41e-05 | 1324.6ms/step | 15.1 samples/s +Step 36730/44000 | Loss: 2.9207 | LR: 3.41e-05 | 1326.5ms/step | 15.1 samples/s +Step 36740/44000 | Loss: 2.9194 | LR: 3.40e-05 | 1322.7ms/step | 15.1 samples/s +Step 36750/44000 | Loss: 2.9313 | LR: 3.39e-05 | 1324.5ms/step | 15.1 samples/s +Validation at step 36750: Loss = 2.9303, Perplexity = 18.73 +Step 36760/44000 | Loss: 2.9024 | LR: 3.38e-05 | 3543.0ms/step | 5.6 samples/s +Step 36770/44000 | Loss: 2.9480 | LR: 3.37e-05 | 1324.1ms/step | 15.1 samples/s +Step 36780/44000 | Loss: 2.9316 | LR: 3.36e-05 | 1323.1ms/step | 15.1 samples/s +Step 36790/44000 | Loss: 2.9463 | LR: 3.35e-05 | 1326.3ms/step | 15.1 samples/s +Step 36800/44000 | Loss: 2.9096 | LR: 3.34e-05 | 1327.7ms/step | 15.1 samples/s +Step 36810/44000 | Loss: 2.8999 | LR: 3.33e-05 | 1326.0ms/step | 15.1 samples/s +Step 36820/44000 | Loss: 2.9420 | LR: 3.32e-05 | 1326.3ms/step | 15.1 samples/s +Step 36830/44000 | Loss: 2.9246 | LR: 3.31e-05 | 1325.3ms/step | 15.1 samples/s +Step 36840/44000 | Loss: 2.9197 | LR: 3.31e-05 | 1325.0ms/step | 15.1 samples/s +Step 36850/44000 | Loss: 2.9347 | LR: 3.30e-05 | 1324.1ms/step | 15.1 samples/s +Step 36860/44000 | Loss: 2.9218 | LR: 3.29e-05 | 1325.1ms/step | 15.1 samples/s +Step 36870/44000 | Loss: 2.9334 | LR: 3.28e-05 | 1325.0ms/step | 15.1 samples/s +Step 36880/44000 | Loss: 2.9358 | LR: 3.27e-05 | 1333.0ms/step | 15.0 samples/s +Step 36890/44000 | Loss: 2.9131 | LR: 3.26e-05 | 1325.5ms/step | 15.1 samples/s +Step 36900/44000 | Loss: 2.9030 | LR: 3.25e-05 | 1322.8ms/step | 15.1 samples/s +Step 36910/44000 | Loss: 2.9398 | LR: 3.24e-05 | 1327.3ms/step | 15.1 samples/s +Step 36920/44000 | Loss: 2.9327 | LR: 3.23e-05 | 1325.9ms/step | 15.1 samples/s +Step 36930/44000 | Loss: 2.9081 | LR: 3.22e-05 | 1325.0ms/step | 15.1 samples/s +Step 36940/44000 | Loss: 2.9068 | LR: 3.22e-05 | 1324.4ms/step | 15.1 samples/s +Step 36950/44000 | Loss: 2.8859 | LR: 3.21e-05 | 1324.4ms/step | 15.1 samples/s +Step 36960/44000 | Loss: 2.9186 | LR: 3.20e-05 | 1323.4ms/step | 15.1 samples/s +Step 36970/44000 | Loss: 2.9146 | LR: 3.19e-05 | 1326.4ms/step | 15.1 samples/s +Step 36980/44000 | Loss: 2.8954 | LR: 3.18e-05 | 1326.4ms/step | 15.1 samples/s +Step 36990/44000 | Loss: 2.9260 | LR: 3.17e-05 | 1323.3ms/step | 15.1 samples/s +Step 37000/44000 | Loss: 2.9524 | LR: 3.16e-05 | 1325.0ms/step | 15.1 samples/s +Validation at step 37000: Loss = 2.9303, Perplexity = 18.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-37000 +Step 37010/44000 | Loss: 2.9238 | LR: 3.15e-05 | 3854.1ms/step | 5.2 samples/s +Step 37020/44000 | Loss: 2.9139 | LR: 3.15e-05 | 1322.6ms/step | 15.1 samples/s +Step 37030/44000 | Loss: 2.9047 | LR: 3.14e-05 | 1327.7ms/step | 15.1 samples/s +Step 37040/44000 | Loss: 2.9196 | LR: 3.13e-05 | 1325.7ms/step | 15.1 samples/s +Step 37050/44000 | Loss: 2.9200 | LR: 3.12e-05 | 1321.9ms/step | 15.1 samples/s +Step 37060/44000 | Loss: 2.9421 | LR: 3.11e-05 | 1322.8ms/step | 15.1 samples/s +Step 37070/44000 | Loss: 2.9180 | LR: 3.10e-05 | 1323.5ms/step | 15.1 samples/s +Step 37080/44000 | Loss: 2.9249 | LR: 3.09e-05 | 1325.7ms/step | 15.1 samples/s +Step 37090/44000 | Loss: 2.9312 | LR: 3.08e-05 | 1320.2ms/step | 15.1 samples/s +Step 37100/44000 | Loss: 2.9281 | LR: 3.07e-05 | 1320.2ms/step | 15.1 samples/s +Step 37110/44000 | Loss: 2.9003 | LR: 3.07e-05 | 1315.8ms/step | 15.2 samples/s +Step 37120/44000 | Loss: 2.9247 | LR: 3.06e-05 | 1323.6ms/step | 15.1 samples/s +Step 37130/44000 | Loss: 2.9394 | LR: 3.05e-05 | 1318.2ms/step | 15.2 samples/s +Step 37140/44000 | Loss: 2.9089 | LR: 3.04e-05 | 1320.6ms/step | 15.1 samples/s +Step 37150/44000 | Loss: 2.8959 | LR: 3.03e-05 | 1320.4ms/step | 15.1 samples/s +Step 37160/44000 | Loss: 2.9249 | LR: 3.02e-05 | 1317.7ms/step | 15.2 samples/s +Step 37170/44000 | Loss: 2.9202 | LR: 3.01e-05 | 1318.2ms/step | 15.2 samples/s +Step 37180/44000 | Loss: 2.8966 | LR: 3.01e-05 | 1316.7ms/step | 15.2 samples/s +Step 37190/44000 | Loss: 2.9264 | LR: 3.00e-05 | 1321.3ms/step | 15.1 samples/s +Step 37200/44000 | Loss: 2.9371 | LR: 2.99e-05 | 1315.1ms/step | 15.2 samples/s +Step 37210/44000 | Loss: 2.9140 | LR: 2.98e-05 | 1318.4ms/step | 15.2 samples/s +Step 37220/44000 | Loss: 2.9101 | LR: 2.97e-05 | 1319.6ms/step | 15.2 samples/s +Step 37230/44000 | Loss: 2.9321 | LR: 2.96e-05 | 1316.8ms/step | 15.2 samples/s +Step 37240/44000 | Loss: 2.9452 | LR: 2.95e-05 | 1318.8ms/step | 15.2 samples/s +Step 37250/44000 | Loss: 2.9398 | LR: 2.95e-05 | 1316.6ms/step | 15.2 samples/s +Validation at step 37250: Loss = 2.9301, Perplexity = 18.73 +Step 37260/44000 | Loss: 2.9143 | LR: 2.94e-05 | 3665.6ms/step | 5.5 samples/s +Step 37270/44000 | Loss: 2.9482 | LR: 2.93e-05 | 1330.7ms/step | 15.0 samples/s +Step 37280/44000 | Loss: 2.9234 | LR: 2.92e-05 | 1328.5ms/step | 15.1 samples/s +Step 37290/44000 | Loss: 2.9306 | LR: 2.91e-05 | 1323.5ms/step | 15.1 samples/s +Step 37300/44000 | Loss: 2.9376 | LR: 2.90e-05 | 1320.1ms/step | 15.2 samples/s +Step 37310/44000 | Loss: 2.9101 | LR: 2.89e-05 | 1321.3ms/step | 15.1 samples/s +Step 37320/44000 | Loss: 2.9669 | LR: 2.89e-05 | 1321.5ms/step | 15.1 samples/s +Step 37330/44000 | Loss: 2.9010 | LR: 2.88e-05 | 1320.0ms/step | 15.2 samples/s +Step 37340/44000 | Loss: 2.9205 | LR: 2.87e-05 | 1321.1ms/step | 15.1 samples/s +Step 37350/44000 | Loss: 2.9269 | LR: 2.86e-05 | 1323.2ms/step | 15.1 samples/s +Step 37360/44000 | Loss: 2.9536 | LR: 2.85e-05 | 1327.8ms/step | 15.1 samples/s +Step 37370/44000 | Loss: 2.9129 | LR: 2.84e-05 | 1327.2ms/step | 15.1 samples/s +Step 37380/44000 | Loss: 2.9251 | LR: 2.84e-05 | 1326.5ms/step | 15.1 samples/s +Step 37390/44000 | Loss: 2.9205 | LR: 2.83e-05 | 1327.4ms/step | 15.1 samples/s +Step 37400/44000 | Loss: 2.9389 | LR: 2.82e-05 | 1328.1ms/step | 15.1 samples/s +Step 37410/44000 | Loss: 2.8971 | LR: 2.81e-05 | 1323.9ms/step | 15.1 samples/s +Step 37420/44000 | Loss: 2.9237 | LR: 2.80e-05 | 1324.6ms/step | 15.1 samples/s +Step 37430/44000 | Loss: 2.9290 | LR: 2.79e-05 | 1326.7ms/step | 15.1 samples/s +Step 37440/44000 | Loss: 2.9044 | LR: 2.78e-05 | 1324.2ms/step | 15.1 samples/s +Step 37450/44000 | Loss: 2.9472 | LR: 2.78e-05 | 1324.7ms/step | 15.1 samples/s +Step 37460/44000 | Loss: 2.9132 | LR: 2.77e-05 | 1325.2ms/step | 15.1 samples/s +Step 37470/44000 | Loss: 2.9014 | LR: 2.76e-05 | 1323.9ms/step | 15.1 samples/s +Step 37480/44000 | Loss: 2.9397 | LR: 2.75e-05 | 1324.8ms/step | 15.1 samples/s +Step 37490/44000 | Loss: 2.9213 | LR: 2.74e-05 | 1327.7ms/step | 15.1 samples/s +Step 37500/44000 | Loss: 2.8957 | LR: 2.74e-05 | 1325.2ms/step | 15.1 samples/s +Validation at step 37500: Loss = 2.9300, Perplexity = 18.73 +Step 37510/44000 | Loss: 2.9313 | LR: 2.73e-05 | 3547.9ms/step | 5.6 samples/s +Step 37520/44000 | Loss: 2.9476 | LR: 2.72e-05 | 1325.7ms/step | 15.1 samples/s +Step 37530/44000 | Loss: 2.9277 | LR: 2.71e-05 | 1325.1ms/step | 15.1 samples/s +Step 37540/44000 | Loss: 2.9489 | LR: 2.70e-05 | 1324.9ms/step | 15.1 samples/s +Step 37550/44000 | Loss: 2.9135 | LR: 2.69e-05 | 1324.1ms/step | 15.1 samples/s +Step 37560/44000 | Loss: 2.9219 | LR: 2.69e-05 | 1325.3ms/step | 15.1 samples/s +Step 37570/44000 | Loss: 2.9308 | LR: 2.68e-05 | 1324.1ms/step | 15.1 samples/s +Step 37580/44000 | Loss: 2.9149 | LR: 2.67e-05 | 1322.6ms/step | 15.1 samples/s +Step 37590/44000 | Loss: 2.9279 | LR: 2.66e-05 | 1321.6ms/step | 15.1 samples/s +Step 37600/44000 | Loss: 2.9141 | LR: 2.65e-05 | 1330.3ms/step | 15.0 samples/s +Step 37610/44000 | Loss: 2.9149 | LR: 2.65e-05 | 1326.7ms/step | 15.1 samples/s +Step 37620/44000 | Loss: 2.9246 | LR: 2.64e-05 | 1323.2ms/step | 15.1 samples/s +Step 37630/44000 | Loss: 2.9278 | LR: 2.63e-05 | 1324.3ms/step | 15.1 samples/s +Step 37640/44000 | Loss: 2.9312 | LR: 2.62e-05 | 1326.9ms/step | 15.1 samples/s +Step 37650/44000 | Loss: 2.9326 | LR: 2.61e-05 | 1327.7ms/step | 15.1 samples/s +Step 37660/44000 | Loss: 2.9401 | LR: 2.60e-05 | 1322.9ms/step | 15.1 samples/s +Step 37670/44000 | Loss: 2.8934 | LR: 2.60e-05 | 1326.0ms/step | 15.1 samples/s +Step 37680/44000 | Loss: 2.9378 | LR: 2.59e-05 | 1325.4ms/step | 15.1 samples/s +Step 37690/44000 | Loss: 2.9355 | LR: 2.58e-05 | 1323.7ms/step | 15.1 samples/s +Step 37700/44000 | Loss: 2.9083 | LR: 2.57e-05 | 1325.1ms/step | 15.1 samples/s +Step 37710/44000 | Loss: 2.8956 | LR: 2.56e-05 | 1328.9ms/step | 15.1 samples/s +Step 37720/44000 | Loss: 2.9161 | LR: 2.56e-05 | 1326.5ms/step | 15.1 samples/s +Step 37730/44000 | Loss: 2.9164 | LR: 2.55e-05 | 1325.2ms/step | 15.1 samples/s +Step 37740/44000 | Loss: 2.9488 | LR: 2.54e-05 | 1321.5ms/step | 15.1 samples/s +Step 37750/44000 | Loss: 2.9116 | LR: 2.53e-05 | 1323.1ms/step | 15.1 samples/s +Validation at step 37750: Loss = 2.9297, Perplexity = 18.72 +Step 37760/44000 | Loss: 2.9176 | LR: 2.52e-05 | 3552.3ms/step | 5.6 samples/s +Step 37770/44000 | Loss: 2.9296 | LR: 2.52e-05 | 1326.5ms/step | 15.1 samples/s +Step 37780/44000 | Loss: 2.8932 | LR: 2.51e-05 | 1325.6ms/step | 15.1 samples/s +Step 37790/44000 | Loss: 2.9185 | LR: 2.50e-05 | 1322.7ms/step | 15.1 samples/s +Step 37800/44000 | Loss: 2.9201 | LR: 2.49e-05 | 1332.2ms/step | 15.0 samples/s +Step 37810/44000 | Loss: 2.9094 | LR: 2.48e-05 | 1326.7ms/step | 15.1 samples/s +Step 37820/44000 | Loss: 2.9249 | LR: 2.48e-05 | 1326.6ms/step | 15.1 samples/s +Step 37830/44000 | Loss: 2.9419 | LR: 2.47e-05 | 1326.2ms/step | 15.1 samples/s +Step 37840/44000 | Loss: 2.8858 | LR: 2.46e-05 | 1325.8ms/step | 15.1 samples/s +Step 37850/44000 | Loss: 2.9293 | LR: 2.45e-05 | 1328.3ms/step | 15.1 samples/s +Step 37860/44000 | Loss: 2.8975 | LR: 2.45e-05 | 1329.3ms/step | 15.0 samples/s +Step 37870/44000 | Loss: 2.9126 | LR: 2.44e-05 | 1325.8ms/step | 15.1 samples/s +Step 37880/44000 | Loss: 2.8880 | LR: 2.43e-05 | 1326.4ms/step | 15.1 samples/s +Step 37890/44000 | Loss: 2.9082 | LR: 2.42e-05 | 1326.0ms/step | 15.1 samples/s +Step 37900/44000 | Loss: 2.9399 | LR: 2.41e-05 | 1325.1ms/step | 15.1 samples/s +Step 37910/44000 | Loss: 2.9179 | LR: 2.41e-05 | 1325.8ms/step | 15.1 samples/s +Step 37920/44000 | Loss: 2.9178 | LR: 2.40e-05 | 1326.8ms/step | 15.1 samples/s +Step 37930/44000 | Loss: 2.9290 | LR: 2.39e-05 | 1324.6ms/step | 15.1 samples/s +Step 37940/44000 | Loss: 2.9215 | LR: 2.38e-05 | 1322.0ms/step | 15.1 samples/s +Step 37950/44000 | Loss: 2.8986 | LR: 2.38e-05 | 1324.3ms/step | 15.1 samples/s +Step 37960/44000 | Loss: 2.9554 | LR: 2.37e-05 | 1324.6ms/step | 15.1 samples/s +Step 37970/44000 | Loss: 2.8903 | LR: 2.36e-05 | 1326.8ms/step | 15.1 samples/s +Step 37980/44000 | Loss: 2.8937 | LR: 2.35e-05 | 1327.4ms/step | 15.1 samples/s +Step 37990/44000 | Loss: 2.9001 | LR: 2.34e-05 | 1326.7ms/step | 15.1 samples/s +Step 38000/44000 | Loss: 2.9248 | LR: 2.34e-05 | 1327.5ms/step | 15.1 samples/s +Validation at step 38000: Loss = 2.9300, Perplexity = 18.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-38000 +Step 38010/44000 | Loss: 2.9374 | LR: 2.33e-05 | 3820.6ms/step | 5.2 samples/s +Step 38020/44000 | Loss: 2.9439 | LR: 2.32e-05 | 1322.6ms/step | 15.1 samples/s +Step 38030/44000 | Loss: 2.8971 | LR: 2.31e-05 | 1322.2ms/step | 15.1 samples/s +Step 38040/44000 | Loss: 2.9206 | LR: 2.31e-05 | 1326.9ms/step | 15.1 samples/s +Step 38050/44000 | Loss: 2.9267 | LR: 2.30e-05 | 1325.1ms/step | 15.1 samples/s +Step 38060/44000 | Loss: 2.9411 | LR: 2.29e-05 | 1326.2ms/step | 15.1 samples/s +Step 38070/44000 | Loss: 2.9203 | LR: 2.28e-05 | 1325.8ms/step | 15.1 samples/s +Step 38080/44000 | Loss: 2.8831 | LR: 2.28e-05 | 1327.4ms/step | 15.1 samples/s +Step 38090/44000 | Loss: 2.8969 | LR: 2.27e-05 | 1327.7ms/step | 15.1 samples/s +Step 38100/44000 | Loss: 2.9392 | LR: 2.26e-05 | 1325.3ms/step | 15.1 samples/s +Step 38110/44000 | Loss: 2.8971 | LR: 2.25e-05 | 1325.3ms/step | 15.1 samples/s +Step 38120/44000 | Loss: 2.9232 | LR: 2.25e-05 | 1326.6ms/step | 15.1 samples/s +Step 38130/44000 | Loss: 2.9101 | LR: 2.24e-05 | 1325.9ms/step | 15.1 samples/s +Step 38140/44000 | Loss: 2.8954 | LR: 2.23e-05 | 1326.9ms/step | 15.1 samples/s +Step 38150/44000 | Loss: 2.9262 | LR: 2.22e-05 | 1324.5ms/step | 15.1 samples/s +Step 38160/44000 | Loss: 2.9282 | LR: 2.22e-05 | 1325.0ms/step | 15.1 samples/s +Step 38170/44000 | Loss: 2.9185 | LR: 2.21e-05 | 1327.2ms/step | 15.1 samples/s +Step 38180/44000 | Loss: 2.9403 | LR: 2.20e-05 | 1329.0ms/step | 15.0 samples/s +Step 38190/44000 | Loss: 2.9458 | LR: 2.19e-05 | 1321.8ms/step | 15.1 samples/s +Step 38200/44000 | Loss: 2.8949 | LR: 2.19e-05 | 1324.3ms/step | 15.1 samples/s +Step 38210/44000 | Loss: 2.9060 | LR: 2.18e-05 | 1324.7ms/step | 15.1 samples/s +Step 38220/44000 | Loss: 2.9426 | LR: 2.17e-05 | 1328.5ms/step | 15.1 samples/s +Step 38230/44000 | Loss: 2.9415 | LR: 2.16e-05 | 1322.8ms/step | 15.1 samples/s +Step 38240/44000 | Loss: 2.9526 | LR: 2.16e-05 | 1322.3ms/step | 15.1 samples/s +Step 38250/44000 | Loss: 2.9134 | LR: 2.15e-05 | 1325.3ms/step | 15.1 samples/s +Validation at step 38250: Loss = 2.9297, Perplexity = 18.72 +Step 38260/44000 | Loss: 2.9295 | LR: 2.14e-05 | 3546.7ms/step | 5.6 samples/s +Step 38270/44000 | Loss: 2.9646 | LR: 2.13e-05 | 1325.6ms/step | 15.1 samples/s +Step 38280/44000 | Loss: 2.9359 | LR: 2.13e-05 | 1327.2ms/step | 15.1 samples/s +Step 38290/44000 | Loss: 2.9198 | LR: 2.12e-05 | 1324.0ms/step | 15.1 samples/s +Step 38300/44000 | Loss: 2.9469 | LR: 2.11e-05 | 1324.7ms/step | 15.1 samples/s +Step 38310/44000 | Loss: 2.9497 | LR: 2.11e-05 | 1324.3ms/step | 15.1 samples/s +Step 38320/44000 | Loss: 2.8883 | LR: 2.10e-05 | 1326.1ms/step | 15.1 samples/s +Step 38330/44000 | Loss: 2.9068 | LR: 2.09e-05 | 1324.8ms/step | 15.1 samples/s +Step 38340/44000 | Loss: 2.9331 | LR: 2.08e-05 | 1325.6ms/step | 15.1 samples/s +Step 38350/44000 | Loss: 2.9144 | LR: 2.08e-05 | 1327.2ms/step | 15.1 samples/s +Step 38360/44000 | Loss: 2.9094 | LR: 2.07e-05 | 1327.2ms/step | 15.1 samples/s +Step 38370/44000 | Loss: 2.9096 | LR: 2.06e-05 | 1324.4ms/step | 15.1 samples/s +Step 38380/44000 | Loss: 2.9056 | LR: 2.05e-05 | 1322.4ms/step | 15.1 samples/s +Step 38390/44000 | Loss: 2.9090 | LR: 2.05e-05 | 1324.0ms/step | 15.1 samples/s +Step 38400/44000 | Loss: 2.8879 | LR: 2.04e-05 | 1328.1ms/step | 15.1 samples/s +Step 38410/44000 | Loss: 2.9297 | LR: 2.03e-05 | 1326.7ms/step | 15.1 samples/s +Step 38420/44000 | Loss: 2.8864 | LR: 2.03e-05 | 1325.1ms/step | 15.1 samples/s +Step 38430/44000 | Loss: 2.9013 | LR: 2.02e-05 | 1326.7ms/step | 15.1 samples/s +Step 38440/44000 | Loss: 2.9306 | LR: 2.01e-05 | 1324.9ms/step | 15.1 samples/s +Step 38450/44000 | Loss: 2.9232 | LR: 2.00e-05 | 1325.3ms/step | 15.1 samples/s +Step 38460/44000 | Loss: 2.9534 | LR: 2.00e-05 | 1325.6ms/step | 15.1 samples/s +Step 38470/44000 | Loss: 2.9301 | LR: 1.99e-05 | 1326.0ms/step | 15.1 samples/s +Step 38480/44000 | Loss: 2.9170 | LR: 1.98e-05 | 1324.5ms/step | 15.1 samples/s +Step 38490/44000 | Loss: 2.9139 | LR: 1.98e-05 | 1328.4ms/step | 15.1 samples/s +Step 38500/44000 | Loss: 2.9241 | LR: 1.97e-05 | 1331.7ms/step | 15.0 samples/s +Validation at step 38500: Loss = 2.9298, Perplexity = 18.72 +Step 38510/44000 | Loss: 2.9090 | LR: 1.96e-05 | 3547.1ms/step | 5.6 samples/s +Step 38520/44000 | Loss: 2.9253 | LR: 1.95e-05 | 1326.2ms/step | 15.1 samples/s +Step 38530/44000 | Loss: 2.9104 | LR: 1.95e-05 | 1327.7ms/step | 15.1 samples/s +Step 38540/44000 | Loss: 2.9343 | LR: 1.94e-05 | 1322.9ms/step | 15.1 samples/s +Step 38550/44000 | Loss: 2.9440 | LR: 1.93e-05 | 1327.0ms/step | 15.1 samples/s +Step 38560/44000 | Loss: 2.9198 | LR: 1.93e-05 | 1329.7ms/step | 15.0 samples/s +Step 38570/44000 | Loss: 2.9784 | LR: 1.92e-05 | 1324.3ms/step | 15.1 samples/s +Step 38580/44000 | Loss: 2.9118 | LR: 1.91e-05 | 1356.0ms/step | 14.7 samples/s +Step 38590/44000 | Loss: 2.9030 | LR: 1.91e-05 | 1392.3ms/step | 14.4 samples/s +Step 38600/44000 | Loss: 2.9237 | LR: 1.90e-05 | 1372.9ms/step | 14.6 samples/s +Step 38610/44000 | Loss: 2.9114 | LR: 1.89e-05 | 1345.7ms/step | 14.9 samples/s +Step 38620/44000 | Loss: 2.9242 | LR: 1.88e-05 | 1344.3ms/step | 14.9 samples/s +Step 38630/44000 | Loss: 2.9056 | LR: 1.88e-05 | 1334.9ms/step | 15.0 samples/s +Step 38640/44000 | Loss: 2.9157 | LR: 1.87e-05 | 1329.7ms/step | 15.0 samples/s +Step 38650/44000 | Loss: 2.9277 | LR: 1.86e-05 | 1332.6ms/step | 15.0 samples/s +Step 38660/44000 | Loss: 2.8825 | LR: 1.86e-05 | 1333.9ms/step | 15.0 samples/s +Step 38670/44000 | Loss: 2.9406 | LR: 1.85e-05 | 1329.5ms/step | 15.0 samples/s +Step 38680/44000 | Loss: 2.9150 | LR: 1.84e-05 | 1323.7ms/step | 15.1 samples/s +Step 38690/44000 | Loss: 2.9128 | LR: 1.84e-05 | 1327.3ms/step | 15.1 samples/s +Step 38700/44000 | Loss: 2.9137 | LR: 1.83e-05 | 1324.2ms/step | 15.1 samples/s +Step 38710/44000 | Loss: 2.9351 | LR: 1.82e-05 | 1328.1ms/step | 15.1 samples/s +Step 38720/44000 | Loss: 2.8916 | LR: 1.82e-05 | 1326.9ms/step | 15.1 samples/s +Step 38730/44000 | Loss: 2.9385 | LR: 1.81e-05 | 1323.8ms/step | 15.1 samples/s +Step 38740/44000 | Loss: 2.9316 | LR: 1.80e-05 | 1325.9ms/step | 15.1 samples/s +Step 38750/44000 | Loss: 2.9243 | LR: 1.80e-05 | 1328.8ms/step | 15.1 samples/s +Validation at step 38750: Loss = 2.9298, Perplexity = 18.72 +Step 38760/44000 | Loss: 2.9226 | LR: 1.79e-05 | 3564.2ms/step | 5.6 samples/s +Step 38770/44000 | Loss: 2.9166 | LR: 1.78e-05 | 1323.3ms/step | 15.1 samples/s +Step 38780/44000 | Loss: 2.9131 | LR: 1.78e-05 | 1325.9ms/step | 15.1 samples/s +Step 38790/44000 | Loss: 2.9317 | LR: 1.77e-05 | 1335.4ms/step | 15.0 samples/s +Step 38800/44000 | Loss: 2.9164 | LR: 1.76e-05 | 1325.1ms/step | 15.1 samples/s +Step 38810/44000 | Loss: 2.9013 | LR: 1.76e-05 | 1325.6ms/step | 15.1 samples/s +Step 38820/44000 | Loss: 2.9087 | LR: 1.75e-05 | 1327.9ms/step | 15.1 samples/s +Step 38830/44000 | Loss: 2.9333 | LR: 1.74e-05 | 1325.5ms/step | 15.1 samples/s +Step 38840/44000 | Loss: 2.8996 | LR: 1.74e-05 | 1323.9ms/step | 15.1 samples/s +Step 38850/44000 | Loss: 2.9511 | LR: 1.73e-05 | 1324.7ms/step | 15.1 samples/s +Step 38860/44000 | Loss: 2.9539 | LR: 1.72e-05 | 1324.3ms/step | 15.1 samples/s +Step 38870/44000 | Loss: 2.9424 | LR: 1.72e-05 | 1324.3ms/step | 15.1 samples/s +Step 38880/44000 | Loss: 2.9281 | LR: 1.71e-05 | 1323.3ms/step | 15.1 samples/s +Step 38890/44000 | Loss: 2.9249 | LR: 1.70e-05 | 1326.2ms/step | 15.1 samples/s +Step 38900/44000 | Loss: 2.9744 | LR: 1.70e-05 | 1327.9ms/step | 15.1 samples/s +Step 38910/44000 | Loss: 2.9045 | LR: 1.69e-05 | 1323.1ms/step | 15.1 samples/s +Step 38920/44000 | Loss: 2.9090 | LR: 1.68e-05 | 1324.3ms/step | 15.1 samples/s +Step 38930/44000 | Loss: 2.9140 | LR: 1.68e-05 | 1323.8ms/step | 15.1 samples/s +Step 38940/44000 | Loss: 2.9284 | LR: 1.67e-05 | 1326.1ms/step | 15.1 samples/s +Step 38950/44000 | Loss: 2.9021 | LR: 1.66e-05 | 1327.1ms/step | 15.1 samples/s +Step 38960/44000 | Loss: 2.9190 | LR: 1.66e-05 | 1327.7ms/step | 15.1 samples/s +Step 38970/44000 | Loss: 2.8820 | LR: 1.65e-05 | 1327.4ms/step | 15.1 samples/s +Step 38980/44000 | Loss: 2.9265 | LR: 1.64e-05 | 1326.7ms/step | 15.1 samples/s +Step 38990/44000 | Loss: 2.9297 | LR: 1.64e-05 | 1323.6ms/step | 15.1 samples/s +Step 39000/44000 | Loss: 2.9201 | LR: 1.63e-05 | 1324.5ms/step | 15.1 samples/s +Validation at step 39000: Loss = 2.9297, Perplexity = 18.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-39000 +Step 39010/44000 | Loss: 2.9066 | LR: 1.62e-05 | 3849.2ms/step | 5.2 samples/s +Step 39020/44000 | Loss: 2.9185 | LR: 1.62e-05 | 1324.9ms/step | 15.1 samples/s +Step 39030/44000 | Loss: 2.9073 | LR: 1.61e-05 | 1323.9ms/step | 15.1 samples/s +Step 39040/44000 | Loss: 2.9180 | LR: 1.61e-05 | 1323.5ms/step | 15.1 samples/s +Step 39050/44000 | Loss: 2.9419 | LR: 1.60e-05 | 1328.8ms/step | 15.1 samples/s +Step 39060/44000 | Loss: 2.9319 | LR: 1.59e-05 | 1323.2ms/step | 15.1 samples/s +Step 39070/44000 | Loss: 2.9263 | LR: 1.59e-05 | 1325.8ms/step | 15.1 samples/s +Step 39080/44000 | Loss: 2.9088 | LR: 1.58e-05 | 1325.8ms/step | 15.1 samples/s +Step 39090/44000 | Loss: 2.9089 | LR: 1.57e-05 | 1323.0ms/step | 15.1 samples/s +Step 39100/44000 | Loss: 2.8896 | LR: 1.57e-05 | 1327.6ms/step | 15.1 samples/s +Step 39110/44000 | Loss: 2.9321 | LR: 1.56e-05 | 1325.5ms/step | 15.1 samples/s +Step 39120/44000 | Loss: 2.8986 | LR: 1.55e-05 | 1322.0ms/step | 15.1 samples/s +Step 39130/44000 | Loss: 2.9146 | LR: 1.55e-05 | 1322.3ms/step | 15.1 samples/s +Step 39140/44000 | Loss: 2.9194 | LR: 1.54e-05 | 1322.4ms/step | 15.1 samples/s +Step 39150/44000 | Loss: 2.8875 | LR: 1.54e-05 | 1328.2ms/step | 15.1 samples/s +Step 39160/44000 | Loss: 2.9669 | LR: 1.53e-05 | 1329.5ms/step | 15.0 samples/s +Step 39170/44000 | Loss: 2.9174 | LR: 1.52e-05 | 1328.5ms/step | 15.1 samples/s +Step 39180/44000 | Loss: 2.9297 | LR: 1.52e-05 | 1330.7ms/step | 15.0 samples/s +Step 39190/44000 | Loss: 2.9336 | LR: 1.51e-05 | 1330.4ms/step | 15.0 samples/s +Step 39200/44000 | Loss: 2.9017 | LR: 1.50e-05 | 1326.4ms/step | 15.1 samples/s +Step 39210/44000 | Loss: 2.8989 | LR: 1.50e-05 | 1323.4ms/step | 15.1 samples/s +Step 39220/44000 | Loss: 2.9257 | LR: 1.49e-05 | 1320.6ms/step | 15.1 samples/s +Step 39230/44000 | Loss: 2.9142 | LR: 1.49e-05 | 1322.9ms/step | 15.1 samples/s +Step 39240/44000 | Loss: 2.9173 | LR: 1.48e-05 | 1320.4ms/step | 15.1 samples/s +Step 39250/44000 | Loss: 2.9574 | LR: 1.47e-05 | 1321.3ms/step | 15.1 samples/s +Validation at step 39250: Loss = 2.9295, Perplexity = 18.72 +Step 39260/44000 | Loss: 2.9449 | LR: 1.47e-05 | 3544.1ms/step | 5.6 samples/s +Step 39270/44000 | Loss: 2.9230 | LR: 1.46e-05 | 1321.4ms/step | 15.1 samples/s +Step 39280/44000 | Loss: 2.9426 | LR: 1.46e-05 | 1321.5ms/step | 15.1 samples/s +Step 39290/44000 | Loss: 2.9001 | LR: 1.45e-05 | 1323.6ms/step | 15.1 samples/s +Step 39300/44000 | Loss: 2.9071 | LR: 1.44e-05 | 1324.2ms/step | 15.1 samples/s +Step 39310/44000 | Loss: 2.9194 | LR: 1.44e-05 | 1323.1ms/step | 15.1 samples/s +Step 39320/44000 | Loss: 2.9310 | LR: 1.43e-05 | 1327.9ms/step | 15.1 samples/s +Step 39330/44000 | Loss: 2.9444 | LR: 1.42e-05 | 1324.7ms/step | 15.1 samples/s +Step 39340/44000 | Loss: 2.9320 | LR: 1.42e-05 | 1325.8ms/step | 15.1 samples/s +Step 39350/44000 | Loss: 2.9173 | LR: 1.41e-05 | 1324.5ms/step | 15.1 samples/s +Step 39360/44000 | Loss: 2.9238 | LR: 1.41e-05 | 1326.0ms/step | 15.1 samples/s +Step 39370/44000 | Loss: 2.9552 | LR: 1.40e-05 | 1325.8ms/step | 15.1 samples/s +Step 39380/44000 | Loss: 2.9506 | LR: 1.39e-05 | 1325.5ms/step | 15.1 samples/s +Step 39390/44000 | Loss: 2.9065 | LR: 1.39e-05 | 1327.9ms/step | 15.1 samples/s +Step 39400/44000 | Loss: 2.9403 | LR: 1.38e-05 | 1325.2ms/step | 15.1 samples/s +Step 39410/44000 | Loss: 2.9590 | LR: 1.38e-05 | 1323.9ms/step | 15.1 samples/s +Step 39420/44000 | Loss: 2.9598 | LR: 1.37e-05 | 1324.4ms/step | 15.1 samples/s +Step 39430/44000 | Loss: 2.9181 | LR: 1.36e-05 | 1323.8ms/step | 15.1 samples/s +Step 39440/44000 | Loss: 2.9177 | LR: 1.36e-05 | 1325.4ms/step | 15.1 samples/s +Step 39450/44000 | Loss: 2.9304 | LR: 1.35e-05 | 1325.2ms/step | 15.1 samples/s +Step 39460/44000 | Loss: 2.9149 | LR: 1.35e-05 | 1324.8ms/step | 15.1 samples/s +Step 39470/44000 | Loss: 2.9149 | LR: 1.34e-05 | 1326.9ms/step | 15.1 samples/s +Step 39480/44000 | Loss: 2.9250 | LR: 1.34e-05 | 1329.7ms/step | 15.0 samples/s +Step 39490/44000 | Loss: 2.9244 | LR: 1.33e-05 | 1324.9ms/step | 15.1 samples/s +Step 39500/44000 | Loss: 2.9650 | LR: 1.32e-05 | 1324.1ms/step | 15.1 samples/s +Validation at step 39500: Loss = 2.9294, Perplexity = 18.72 +Step 39510/44000 | Loss: 2.9072 | LR: 1.32e-05 | 3545.9ms/step | 5.6 samples/s +Step 39520/44000 | Loss: 2.9091 | LR: 1.31e-05 | 1324.9ms/step | 15.1 samples/s +Step 39530/44000 | Loss: 2.9069 | LR: 1.31e-05 | 1326.5ms/step | 15.1 samples/s +Step 39540/44000 | Loss: 2.9344 | LR: 1.30e-05 | 1325.1ms/step | 15.1 samples/s +Step 39550/44000 | Loss: 2.9091 | LR: 1.29e-05 | 1324.0ms/step | 15.1 samples/s +Step 39560/44000 | Loss: 2.8990 | LR: 1.29e-05 | 1323.7ms/step | 15.1 samples/s +Step 39570/44000 | Loss: 2.9312 | LR: 1.28e-05 | 1325.9ms/step | 15.1 samples/s +Step 39580/44000 | Loss: 2.9235 | LR: 1.28e-05 | 1327.1ms/step | 15.1 samples/s +Step 39590/44000 | Loss: 2.9304 | LR: 1.27e-05 | 1324.4ms/step | 15.1 samples/s +Step 39600/44000 | Loss: 2.9467 | LR: 1.27e-05 | 1324.6ms/step | 15.1 samples/s +Step 39610/44000 | Loss: 2.8980 | LR: 1.26e-05 | 1326.9ms/step | 15.1 samples/s +Step 39620/44000 | Loss: 2.9079 | LR: 1.25e-05 | 1325.2ms/step | 15.1 samples/s +Step 39630/44000 | Loss: 2.9342 | LR: 1.25e-05 | 1324.8ms/step | 15.1 samples/s +Step 39640/44000 | Loss: 2.9183 | LR: 1.24e-05 | 1380.2ms/step | 14.5 samples/s +Step 39650/44000 | Loss: 2.9217 | LR: 1.24e-05 | 1372.5ms/step | 14.6 samples/s +Step 39660/44000 | Loss: 2.9302 | LR: 1.23e-05 | 1345.0ms/step | 14.9 samples/s +Step 39670/44000 | Loss: 2.9044 | LR: 1.23e-05 | 1342.0ms/step | 14.9 samples/s +Step 39680/44000 | Loss: 2.9330 | LR: 1.22e-05 | 1335.1ms/step | 15.0 samples/s +Step 39690/44000 | Loss: 2.9117 | LR: 1.22e-05 | 1332.2ms/step | 15.0 samples/s +Step 39700/44000 | Loss: 2.9522 | LR: 1.21e-05 | 1334.6ms/step | 15.0 samples/s +Step 39710/44000 | Loss: 2.9123 | LR: 1.20e-05 | 1328.1ms/step | 15.1 samples/s +Step 39720/44000 | Loss: 2.8994 | LR: 1.20e-05 | 1326.2ms/step | 15.1 samples/s +Step 39730/44000 | Loss: 2.9158 | LR: 1.19e-05 | 1329.7ms/step | 15.0 samples/s +Step 39740/44000 | Loss: 2.9131 | LR: 1.19e-05 | 1329.2ms/step | 15.0 samples/s +Step 39750/44000 | Loss: 2.9182 | LR: 1.18e-05 | 1323.3ms/step | 15.1 samples/s +Validation at step 39750: Loss = 2.9296, Perplexity = 18.72 +Step 39760/44000 | Loss: 2.8937 | LR: 1.18e-05 | 3550.8ms/step | 5.6 samples/s +Step 39770/44000 | Loss: 2.9242 | LR: 1.17e-05 | 1328.2ms/step | 15.1 samples/s +Step 39780/44000 | Loss: 2.8936 | LR: 1.17e-05 | 1327.4ms/step | 15.1 samples/s +Step 39790/44000 | Loss: 2.8926 | LR: 1.16e-05 | 1325.7ms/step | 15.1 samples/s +Step 39800/44000 | Loss: 2.9283 | LR: 1.15e-05 | 1326.0ms/step | 15.1 samples/s +Step 39810/44000 | Loss: 2.9252 | LR: 1.15e-05 | 1329.0ms/step | 15.0 samples/s +Step 39820/44000 | Loss: 2.9272 | LR: 1.14e-05 | 1323.6ms/step | 15.1 samples/s +Step 39830/44000 | Loss: 2.9230 | LR: 1.14e-05 | 1325.6ms/step | 15.1 samples/s +Step 39840/44000 | Loss: 2.9126 | LR: 1.13e-05 | 1327.0ms/step | 15.1 samples/s +Step 39850/44000 | Loss: 2.9208 | LR: 1.13e-05 | 1324.5ms/step | 15.1 samples/s +Step 39860/44000 | Loss: 2.9306 | LR: 1.12e-05 | 1326.6ms/step | 15.1 samples/s +Step 39870/44000 | Loss: 2.9189 | LR: 1.12e-05 | 1326.9ms/step | 15.1 samples/s +Step 39880/44000 | Loss: 2.9146 | LR: 1.11e-05 | 1322.3ms/step | 15.1 samples/s +Step 39890/44000 | Loss: 2.9201 | LR: 1.11e-05 | 1331.2ms/step | 15.0 samples/s +Step 39900/44000 | Loss: 2.9309 | LR: 1.10e-05 | 1326.1ms/step | 15.1 samples/s +Step 39910/44000 | Loss: 2.9271 | LR: 1.10e-05 | 1325.7ms/step | 15.1 samples/s +Step 39920/44000 | Loss: 2.9214 | LR: 1.09e-05 | 1326.0ms/step | 15.1 samples/s +Step 39930/44000 | Loss: 2.9257 | LR: 1.08e-05 | 1327.1ms/step | 15.1 samples/s +Step 39940/44000 | Loss: 2.9261 | LR: 1.08e-05 | 1323.3ms/step | 15.1 samples/s +Step 39950/44000 | Loss: 2.9271 | LR: 1.07e-05 | 1326.1ms/step | 15.1 samples/s +Step 39960/44000 | Loss: 2.9070 | LR: 1.07e-05 | 1327.7ms/step | 15.1 samples/s +Step 39970/44000 | Loss: 2.9298 | LR: 1.06e-05 | 1323.4ms/step | 15.1 samples/s +Step 39980/44000 | Loss: 2.9258 | LR: 1.06e-05 | 1325.1ms/step | 15.1 samples/s +Step 39990/44000 | Loss: 2.9399 | LR: 1.05e-05 | 1323.1ms/step | 15.1 samples/s +Step 40000/44000 | Loss: 2.9070 | LR: 1.05e-05 | 1320.8ms/step | 15.1 samples/s +Validation at step 40000: Loss = 2.9295, Perplexity = 18.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-40000 +Step 40010/44000 | Loss: 2.9088 | LR: 1.04e-05 | 3845.9ms/step | 5.2 samples/s +Step 40020/44000 | Loss: 2.9242 | LR: 1.04e-05 | 1325.7ms/step | 15.1 samples/s +Step 40030/44000 | Loss: 2.9135 | LR: 1.03e-05 | 1323.6ms/step | 15.1 samples/s +Step 40040/44000 | Loss: 2.9036 | LR: 1.03e-05 | 1325.9ms/step | 15.1 samples/s +Step 40050/44000 | Loss: 2.9283 | LR: 1.02e-05 | 1319.7ms/step | 15.2 samples/s +Step 40060/44000 | Loss: 2.8967 | LR: 1.02e-05 | 1320.5ms/step | 15.1 samples/s +Step 40070/44000 | Loss: 2.9137 | LR: 1.01e-05 | 1325.2ms/step | 15.1 samples/s +Step 40080/44000 | Loss: 2.9054 | LR: 1.01e-05 | 1322.8ms/step | 15.1 samples/s +Step 40090/44000 | Loss: 2.9008 | LR: 1.00e-05 | 1325.9ms/step | 15.1 samples/s +Step 40100/44000 | Loss: 2.9271 | LR: 9.96e-06 | 1326.5ms/step | 15.1 samples/s +Step 40110/44000 | Loss: 2.9255 | LR: 9.91e-06 | 1325.6ms/step | 15.1 samples/s +Step 40120/44000 | Loss: 2.9095 | LR: 9.86e-06 | 1327.7ms/step | 15.1 samples/s +Step 40130/44000 | Loss: 2.9283 | LR: 9.81e-06 | 1324.4ms/step | 15.1 samples/s +Step 40140/44000 | Loss: 2.9231 | LR: 9.76e-06 | 1326.1ms/step | 15.1 samples/s +Step 40150/44000 | Loss: 2.9266 | LR: 9.71e-06 | 1323.7ms/step | 15.1 samples/s +Step 40160/44000 | Loss: 2.9263 | LR: 9.66e-06 | 1321.4ms/step | 15.1 samples/s +Step 40170/44000 | Loss: 2.9136 | LR: 9.61e-06 | 1325.1ms/step | 15.1 samples/s +Step 40180/44000 | Loss: 2.9244 | LR: 9.56e-06 | 1328.1ms/step | 15.1 samples/s +Step 40190/44000 | Loss: 2.9035 | LR: 9.51e-06 | 1326.3ms/step | 15.1 samples/s +Step 40200/44000 | Loss: 2.9301 | LR: 9.46e-06 | 1321.9ms/step | 15.1 samples/s +Step 40210/44000 | Loss: 2.9505 | LR: 9.41e-06 | 1321.3ms/step | 15.1 samples/s +Step 40220/44000 | Loss: 2.9675 | LR: 9.36e-06 | 1327.2ms/step | 15.1 samples/s +Step 40230/44000 | Loss: 2.9212 | LR: 9.32e-06 | 1323.1ms/step | 15.1 samples/s +Step 40240/44000 | Loss: 2.9006 | LR: 9.27e-06 | 1322.3ms/step | 15.1 samples/s +Step 40250/44000 | Loss: 2.9606 | LR: 9.22e-06 | 1328.8ms/step | 15.1 samples/s +Validation at step 40250: Loss = 2.9295, Perplexity = 18.72 +Step 40260/44000 | Loss: 2.9462 | LR: 9.17e-06 | 3552.4ms/step | 5.6 samples/s +Step 40270/44000 | Loss: 2.9565 | LR: 9.12e-06 | 1319.4ms/step | 15.2 samples/s +Step 40280/44000 | Loss: 2.9347 | LR: 9.07e-06 | 1323.5ms/step | 15.1 samples/s +Step 40290/44000 | Loss: 2.9387 | LR: 9.02e-06 | 1324.7ms/step | 15.1 samples/s +Step 40300/44000 | Loss: 2.9198 | LR: 8.97e-06 | 1327.3ms/step | 15.1 samples/s +Step 40310/44000 | Loss: 2.9140 | LR: 8.93e-06 | 1325.4ms/step | 15.1 samples/s +Step 40320/44000 | Loss: 2.9106 | LR: 8.88e-06 | 1319.4ms/step | 15.2 samples/s +Step 40330/44000 | Loss: 2.9345 | LR: 8.83e-06 | 1319.8ms/step | 15.2 samples/s +Step 40340/44000 | Loss: 2.9407 | LR: 8.78e-06 | 1322.3ms/step | 15.1 samples/s +Step 40350/44000 | Loss: 2.9501 | LR: 8.74e-06 | 1326.1ms/step | 15.1 samples/s +Step 40360/44000 | Loss: 2.8931 | LR: 8.69e-06 | 1327.7ms/step | 15.1 samples/s +Step 40370/44000 | Loss: 2.8941 | LR: 8.64e-06 | 1324.4ms/step | 15.1 samples/s +Step 40380/44000 | Loss: 2.9369 | LR: 8.59e-06 | 1326.5ms/step | 15.1 samples/s +Step 40390/44000 | Loss: 2.9079 | LR: 8.55e-06 | 1332.3ms/step | 15.0 samples/s +Step 40400/44000 | Loss: 2.9357 | LR: 8.50e-06 | 1326.0ms/step | 15.1 samples/s +Step 40410/44000 | Loss: 2.9192 | LR: 8.45e-06 | 1325.5ms/step | 15.1 samples/s +Step 40420/44000 | Loss: 2.9330 | LR: 8.41e-06 | 1325.3ms/step | 15.1 samples/s +Step 40430/44000 | Loss: 2.9366 | LR: 8.36e-06 | 1327.5ms/step | 15.1 samples/s +Step 40440/44000 | Loss: 2.9174 | LR: 8.31e-06 | 1324.3ms/step | 15.1 samples/s +Step 40450/44000 | Loss: 2.9232 | LR: 8.27e-06 | 1325.2ms/step | 15.1 samples/s +Step 40460/44000 | Loss: 2.9026 | LR: 8.22e-06 | 1325.8ms/step | 15.1 samples/s +Step 40470/44000 | Loss: 2.9169 | LR: 8.17e-06 | 1324.5ms/step | 15.1 samples/s +Step 40480/44000 | Loss: 2.9226 | LR: 8.13e-06 | 1326.0ms/step | 15.1 samples/s +Step 40490/44000 | Loss: 2.9236 | LR: 8.08e-06 | 1324.3ms/step | 15.1 samples/s +Step 40500/44000 | Loss: 2.9405 | LR: 8.04e-06 | 1323.3ms/step | 15.1 samples/s +Validation at step 40500: Loss = 2.9294, Perplexity = 18.72 +Step 40510/44000 | Loss: 2.9384 | LR: 7.99e-06 | 3548.2ms/step | 5.6 samples/s +Step 40520/44000 | Loss: 2.9084 | LR: 7.94e-06 | 1327.7ms/step | 15.1 samples/s +Step 40530/44000 | Loss: 2.9102 | LR: 7.90e-06 | 1326.5ms/step | 15.1 samples/s +Step 40540/44000 | Loss: 2.9410 | LR: 7.85e-06 | 1323.5ms/step | 15.1 samples/s +Step 40550/44000 | Loss: 2.8976 | LR: 7.81e-06 | 1326.0ms/step | 15.1 samples/s +Step 40560/44000 | Loss: 2.9213 | LR: 7.76e-06 | 1323.3ms/step | 15.1 samples/s +Step 40570/44000 | Loss: 2.8872 | LR: 7.72e-06 | 1324.7ms/step | 15.1 samples/s +Step 40580/44000 | Loss: 2.9190 | LR: 7.67e-06 | 1327.8ms/step | 15.1 samples/s +Step 40590/44000 | Loss: 2.9224 | LR: 7.63e-06 | 1327.4ms/step | 15.1 samples/s +Step 40600/44000 | Loss: 2.9445 | LR: 7.59e-06 | 1326.6ms/step | 15.1 samples/s +Step 40610/44000 | Loss: 2.9353 | LR: 7.54e-06 | 1328.1ms/step | 15.1 samples/s +Step 40620/44000 | Loss: 2.9259 | LR: 7.50e-06 | 1325.7ms/step | 15.1 samples/s +Step 40630/44000 | Loss: 2.9344 | LR: 7.45e-06 | 1321.8ms/step | 15.1 samples/s +Step 40640/44000 | Loss: 2.9334 | LR: 7.41e-06 | 1326.2ms/step | 15.1 samples/s +Step 40650/44000 | Loss: 2.9295 | LR: 7.37e-06 | 1324.2ms/step | 15.1 samples/s +Step 40660/44000 | Loss: 2.9298 | LR: 7.32e-06 | 1325.8ms/step | 15.1 samples/s +Step 40670/44000 | Loss: 2.8999 | LR: 7.28e-06 | 1323.7ms/step | 15.1 samples/s +Step 40680/44000 | Loss: 2.9216 | LR: 7.23e-06 | 1326.2ms/step | 15.1 samples/s +Step 40690/44000 | Loss: 2.9214 | LR: 7.19e-06 | 1323.6ms/step | 15.1 samples/s +Step 40700/44000 | Loss: 2.9038 | LR: 7.15e-06 | 1325.6ms/step | 15.1 samples/s +Step 40710/44000 | Loss: 2.8782 | LR: 7.10e-06 | 1326.5ms/step | 15.1 samples/s +Step 40720/44000 | Loss: 2.9062 | LR: 7.06e-06 | 1327.8ms/step | 15.1 samples/s +Step 40730/44000 | Loss: 2.9063 | LR: 7.02e-06 | 1321.8ms/step | 15.1 samples/s +Step 40740/44000 | Loss: 2.9460 | LR: 6.98e-06 | 1324.8ms/step | 15.1 samples/s +Step 40750/44000 | Loss: 2.9284 | LR: 6.93e-06 | 1326.1ms/step | 15.1 samples/s +Validation at step 40750: Loss = 2.9296, Perplexity = 18.72 +Step 40760/44000 | Loss: 2.9228 | LR: 6.89e-06 | 3547.8ms/step | 5.6 samples/s +Step 40770/44000 | Loss: 2.9777 | LR: 6.85e-06 | 1326.8ms/step | 15.1 samples/s +Step 40780/44000 | Loss: 2.9261 | LR: 6.81e-06 | 1329.1ms/step | 15.0 samples/s +Step 40790/44000 | Loss: 2.9310 | LR: 6.77e-06 | 1326.0ms/step | 15.1 samples/s +Step 40800/44000 | Loss: 2.9014 | LR: 6.72e-06 | 1324.3ms/step | 15.1 samples/s +Step 40810/44000 | Loss: 2.9079 | LR: 6.68e-06 | 1324.7ms/step | 15.1 samples/s +Step 40820/44000 | Loss: 2.9198 | LR: 6.64e-06 | 1322.8ms/step | 15.1 samples/s +Step 40830/44000 | Loss: 2.9456 | LR: 6.60e-06 | 1329.6ms/step | 15.0 samples/s +Step 40840/44000 | Loss: 2.9274 | LR: 6.56e-06 | 1327.8ms/step | 15.1 samples/s +Step 40850/44000 | Loss: 2.9167 | LR: 6.52e-06 | 1327.9ms/step | 15.1 samples/s +Step 40860/44000 | Loss: 2.8844 | LR: 6.47e-06 | 1323.8ms/step | 15.1 samples/s +Step 40870/44000 | Loss: 2.9160 | LR: 6.43e-06 | 1324.6ms/step | 15.1 samples/s +Step 40880/44000 | Loss: 2.8991 | LR: 6.39e-06 | 1324.5ms/step | 15.1 samples/s +Step 40890/44000 | Loss: 2.9036 | LR: 6.35e-06 | 1324.0ms/step | 15.1 samples/s +Step 40900/44000 | Loss: 2.9213 | LR: 6.31e-06 | 1325.0ms/step | 15.1 samples/s +Step 40910/44000 | Loss: 2.9157 | LR: 6.27e-06 | 1323.7ms/step | 15.1 samples/s +Step 40920/44000 | Loss: 2.9360 | LR: 6.23e-06 | 1325.0ms/step | 15.1 samples/s +Step 40930/44000 | Loss: 2.9528 | LR: 6.19e-06 | 1326.3ms/step | 15.1 samples/s +Step 40940/44000 | Loss: 2.8869 | LR: 6.15e-06 | 1326.5ms/step | 15.1 samples/s +Step 40950/44000 | Loss: 2.9272 | LR: 6.11e-06 | 1324.9ms/step | 15.1 samples/s +Step 40960/44000 | Loss: 2.9348 | LR: 6.07e-06 | 1328.3ms/step | 15.1 samples/s +Step 40970/44000 | Loss: 2.9394 | LR: 6.03e-06 | 1325.0ms/step | 15.1 samples/s +Step 40980/44000 | Loss: 2.9141 | LR: 5.99e-06 | 1325.5ms/step | 15.1 samples/s +Step 40990/44000 | Loss: 2.9303 | LR: 5.95e-06 | 1343.5ms/step | 14.9 samples/s +Step 41000/44000 | Loss: 2.9033 | LR: 5.91e-06 | 1386.5ms/step | 14.4 samples/s +Validation at step 41000: Loss = 2.9296, Perplexity = 18.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-41000 +Step 41010/44000 | Loss: 2.9245 | LR: 5.87e-06 | 3959.0ms/step | 5.1 samples/s +Step 41020/44000 | Loss: 2.9314 | LR: 5.83e-06 | 1359.3ms/step | 14.7 samples/s +Step 41030/44000 | Loss: 2.8999 | LR: 5.80e-06 | 1356.8ms/step | 14.7 samples/s +Step 41040/44000 | Loss: 2.9760 | LR: 5.76e-06 | 1349.9ms/step | 14.8 samples/s +Step 41050/44000 | Loss: 2.9165 | LR: 5.72e-06 | 1341.9ms/step | 14.9 samples/s +Step 41060/44000 | Loss: 2.9086 | LR: 5.68e-06 | 1334.2ms/step | 15.0 samples/s +Step 41070/44000 | Loss: 2.9368 | LR: 5.64e-06 | 1336.7ms/step | 15.0 samples/s +Step 41080/44000 | Loss: 2.9067 | LR: 5.60e-06 | 1330.0ms/step | 15.0 samples/s +Step 41090/44000 | Loss: 2.9102 | LR: 5.56e-06 | 1327.8ms/step | 15.1 samples/s +Step 41100/44000 | Loss: 2.9042 | LR: 5.53e-06 | 1330.2ms/step | 15.0 samples/s +Step 41110/44000 | Loss: 2.9346 | LR: 5.49e-06 | 1326.0ms/step | 15.1 samples/s +Step 41120/44000 | Loss: 2.9133 | LR: 5.45e-06 | 1326.6ms/step | 15.1 samples/s +Step 41130/44000 | Loss: 2.9381 | LR: 5.41e-06 | 1323.5ms/step | 15.1 samples/s +Step 41140/44000 | Loss: 2.9621 | LR: 5.38e-06 | 1382.8ms/step | 14.5 samples/s +Step 41150/44000 | Loss: 2.8950 | LR: 5.34e-06 | 1383.2ms/step | 14.5 samples/s +Step 41160/44000 | Loss: 2.9141 | LR: 5.30e-06 | 1379.7ms/step | 14.5 samples/s +Step 41170/44000 | Loss: 2.9074 | LR: 5.26e-06 | 1367.8ms/step | 14.6 samples/s +Step 41180/44000 | Loss: 2.9374 | LR: 5.23e-06 | 1337.5ms/step | 15.0 samples/s +Step 41190/44000 | Loss: 2.9171 | LR: 5.19e-06 | 1338.1ms/step | 14.9 samples/s +Step 41200/44000 | Loss: 2.8966 | LR: 5.15e-06 | 1340.4ms/step | 14.9 samples/s +Step 41210/44000 | Loss: 2.9080 | LR: 5.12e-06 | 1345.5ms/step | 14.9 samples/s +Step 41220/44000 | Loss: 2.9379 | LR: 5.08e-06 | 1342.8ms/step | 14.9 samples/s +Step 41230/44000 | Loss: 2.9183 | LR: 5.04e-06 | 1329.8ms/step | 15.0 samples/s +Step 41240/44000 | Loss: 2.9368 | LR: 5.01e-06 | 1321.1ms/step | 15.1 samples/s +Step 41250/44000 | Loss: 2.8957 | LR: 4.97e-06 | 1325.0ms/step | 15.1 samples/s +Validation at step 41250: Loss = 2.9295, Perplexity = 18.72 +Step 41260/44000 | Loss: 2.9281 | LR: 4.94e-06 | 3609.2ms/step | 5.5 samples/s +Step 41270/44000 | Loss: 2.8956 | LR: 4.90e-06 | 1329.2ms/step | 15.0 samples/s +Step 41280/44000 | Loss: 2.9293 | LR: 4.86e-06 | 1330.4ms/step | 15.0 samples/s +Step 41290/44000 | Loss: 2.8939 | LR: 4.83e-06 | 1328.0ms/step | 15.1 samples/s +Step 41300/44000 | Loss: 2.9502 | LR: 4.79e-06 | 1323.7ms/step | 15.1 samples/s +Step 41310/44000 | Loss: 2.9288 | LR: 4.76e-06 | 1323.7ms/step | 15.1 samples/s +Step 41320/44000 | Loss: 2.9384 | LR: 4.72e-06 | 1327.4ms/step | 15.1 samples/s +Step 41330/44000 | Loss: 2.9243 | LR: 4.69e-06 | 1321.9ms/step | 15.1 samples/s +Step 41340/44000 | Loss: 2.8992 | LR: 4.65e-06 | 1326.8ms/step | 15.1 samples/s +Step 41350/44000 | Loss: 2.9174 | LR: 4.62e-06 | 1326.1ms/step | 15.1 samples/s +Step 41360/44000 | Loss: 2.9123 | LR: 4.58e-06 | 1323.3ms/step | 15.1 samples/s +Step 41370/44000 | Loss: 2.9065 | LR: 4.55e-06 | 1321.8ms/step | 15.1 samples/s +Step 41380/44000 | Loss: 2.9556 | LR: 4.51e-06 | 1323.8ms/step | 15.1 samples/s +Step 41390/44000 | Loss: 2.9254 | LR: 4.48e-06 | 1326.2ms/step | 15.1 samples/s +Step 41400/44000 | Loss: 2.9312 | LR: 4.45e-06 | 1325.2ms/step | 15.1 samples/s +Step 41410/44000 | Loss: 2.9338 | LR: 4.41e-06 | 1331.3ms/step | 15.0 samples/s +Step 41420/44000 | Loss: 2.9405 | LR: 4.38e-06 | 1329.3ms/step | 15.0 samples/s +Step 41430/44000 | Loss: 2.9064 | LR: 4.34e-06 | 1325.5ms/step | 15.1 samples/s +Step 41440/44000 | Loss: 2.9129 | LR: 4.31e-06 | 1325.1ms/step | 15.1 samples/s +Step 41450/44000 | Loss: 2.9329 | LR: 4.28e-06 | 1325.2ms/step | 15.1 samples/s +Step 41460/44000 | Loss: 2.9121 | LR: 4.24e-06 | 1327.1ms/step | 15.1 samples/s +Step 41470/44000 | Loss: 2.9274 | LR: 4.21e-06 | 1329.5ms/step | 15.0 samples/s +Step 41480/44000 | Loss: 2.9270 | LR: 4.18e-06 | 1328.1ms/step | 15.1 samples/s +Step 41490/44000 | Loss: 2.9229 | LR: 4.14e-06 | 1326.8ms/step | 15.1 samples/s +Step 41500/44000 | Loss: 2.9366 | LR: 4.11e-06 | 1328.0ms/step | 15.1 samples/s +Validation at step 41500: Loss = 2.9295, Perplexity = 18.72 +Step 41510/44000 | Loss: 2.9334 | LR: 4.08e-06 | 3559.1ms/step | 5.6 samples/s +Step 41520/44000 | Loss: 2.8948 | LR: 4.05e-06 | 1329.1ms/step | 15.0 samples/s +Step 41530/44000 | Loss: 2.9150 | LR: 4.01e-06 | 1328.9ms/step | 15.0 samples/s +Step 41540/44000 | Loss: 2.9260 | LR: 3.98e-06 | 1328.6ms/step | 15.1 samples/s +Step 41550/44000 | Loss: 2.8839 | LR: 3.95e-06 | 1330.2ms/step | 15.0 samples/s +Step 41560/44000 | Loss: 2.9270 | LR: 3.92e-06 | 1333.3ms/step | 15.0 samples/s +Step 41570/44000 | Loss: 2.9258 | LR: 3.88e-06 | 1332.5ms/step | 15.0 samples/s +Step 41580/44000 | Loss: 2.9275 | LR: 3.85e-06 | 1329.0ms/step | 15.0 samples/s +Step 41590/44000 | Loss: 2.9245 | LR: 3.82e-06 | 1330.3ms/step | 15.0 samples/s +Step 41600/44000 | Loss: 2.9156 | LR: 3.79e-06 | 1331.3ms/step | 15.0 samples/s +Step 41610/44000 | Loss: 2.9277 | LR: 3.76e-06 | 1329.2ms/step | 15.0 samples/s +Step 41620/44000 | Loss: 2.9130 | LR: 3.73e-06 | 1329.9ms/step | 15.0 samples/s +Step 41630/44000 | Loss: 2.9068 | LR: 3.70e-06 | 1325.4ms/step | 15.1 samples/s +Step 41640/44000 | Loss: 2.8927 | LR: 3.66e-06 | 1323.3ms/step | 15.1 samples/s +Step 41650/44000 | Loss: 2.9001 | LR: 3.63e-06 | 1325.0ms/step | 15.1 samples/s +Step 41660/44000 | Loss: 2.9250 | LR: 3.60e-06 | 1325.1ms/step | 15.1 samples/s +Step 41670/44000 | Loss: 2.9134 | LR: 3.57e-06 | 1327.7ms/step | 15.1 samples/s +Step 41680/44000 | Loss: 2.9387 | LR: 3.54e-06 | 1322.4ms/step | 15.1 samples/s +Step 41690/44000 | Loss: 2.9050 | LR: 3.51e-06 | 1325.1ms/step | 15.1 samples/s +Step 41700/44000 | Loss: 2.9007 | LR: 3.48e-06 | 1329.4ms/step | 15.0 samples/s +Step 41710/44000 | Loss: 2.9188 | LR: 3.45e-06 | 1324.9ms/step | 15.1 samples/s +Step 41720/44000 | Loss: 2.9173 | LR: 3.42e-06 | 1327.3ms/step | 15.1 samples/s +Step 41730/44000 | Loss: 2.9167 | LR: 3.39e-06 | 1326.0ms/step | 15.1 samples/s +Step 41740/44000 | Loss: 2.9387 | LR: 3.36e-06 | 1327.3ms/step | 15.1 samples/s +Step 41750/44000 | Loss: 2.8943 | LR: 3.33e-06 | 1321.6ms/step | 15.1 samples/s +Validation at step 41750: Loss = 2.9294, Perplexity = 18.72 +Step 41760/44000 | Loss: 2.9512 | LR: 3.30e-06 | 3546.7ms/step | 5.6 samples/s +Step 41770/44000 | Loss: 2.8859 | LR: 3.27e-06 | 1323.1ms/step | 15.1 samples/s +Step 41780/44000 | Loss: 2.9104 | LR: 3.24e-06 | 1328.0ms/step | 15.1 samples/s +Step 41790/44000 | Loss: 2.9118 | LR: 3.21e-06 | 1323.6ms/step | 15.1 samples/s +Step 41800/44000 | Loss: 2.9337 | LR: 3.19e-06 | 1326.8ms/step | 15.1 samples/s +Step 41810/44000 | Loss: 2.9236 | LR: 3.16e-06 | 1325.0ms/step | 15.1 samples/s +Step 41820/44000 | Loss: 2.9363 | LR: 3.13e-06 | 1327.2ms/step | 15.1 samples/s +Step 41830/44000 | Loss: 2.9149 | LR: 3.10e-06 | 1327.0ms/step | 15.1 samples/s +Step 41840/44000 | Loss: 2.9348 | LR: 3.07e-06 | 1324.2ms/step | 15.1 samples/s +Step 41850/44000 | Loss: 2.9297 | LR: 3.04e-06 | 1327.1ms/step | 15.1 samples/s +Step 41860/44000 | Loss: 2.9315 | LR: 3.01e-06 | 1327.9ms/step | 15.1 samples/s +Step 41870/44000 | Loss: 2.9040 | LR: 2.99e-06 | 1327.0ms/step | 15.1 samples/s +Step 41880/44000 | Loss: 2.9029 | LR: 2.96e-06 | 1324.7ms/step | 15.1 samples/s +Step 41890/44000 | Loss: 2.9367 | LR: 2.93e-06 | 1328.3ms/step | 15.1 samples/s +Step 41900/44000 | Loss: 2.9293 | LR: 2.90e-06 | 1324.6ms/step | 15.1 samples/s +Step 41910/44000 | Loss: 2.9029 | LR: 2.88e-06 | 1323.3ms/step | 15.1 samples/s +Step 41920/44000 | Loss: 2.8958 | LR: 2.85e-06 | 1323.4ms/step | 15.1 samples/s +Step 41930/44000 | Loss: 2.9524 | LR: 2.82e-06 | 1321.6ms/step | 15.1 samples/s +Step 41940/44000 | Loss: 2.9088 | LR: 2.79e-06 | 1325.7ms/step | 15.1 samples/s +Step 41950/44000 | Loss: 2.8994 | LR: 2.77e-06 | 1325.5ms/step | 15.1 samples/s +Step 41960/44000 | Loss: 2.8998 | LR: 2.74e-06 | 1326.0ms/step | 15.1 samples/s +Step 41970/44000 | Loss: 2.9212 | LR: 2.71e-06 | 1325.7ms/step | 15.1 samples/s +Step 41980/44000 | Loss: 2.9169 | LR: 2.69e-06 | 1326.9ms/step | 15.1 samples/s +Step 41990/44000 | Loss: 2.9536 | LR: 2.66e-06 | 1326.4ms/step | 15.1 samples/s +Step 42000/44000 | Loss: 2.9268 | LR: 2.63e-06 | 1323.0ms/step | 15.1 samples/s +Validation at step 42000: Loss = 2.9297, Perplexity = 18.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-42000 +Step 42010/44000 | Loss: 2.9294 | LR: 2.61e-06 | 3879.9ms/step | 5.2 samples/s +Step 42020/44000 | Loss: 2.9123 | LR: 2.58e-06 | 1325.8ms/step | 15.1 samples/s +Step 42030/44000 | Loss: 2.9302 | LR: 2.56e-06 | 1334.4ms/step | 15.0 samples/s +Step 42040/44000 | Loss: 2.9326 | LR: 2.53e-06 | 1325.8ms/step | 15.1 samples/s +Step 42050/44000 | Loss: 2.9364 | LR: 2.50e-06 | 1330.8ms/step | 15.0 samples/s +Step 42060/44000 | Loss: 2.9131 | LR: 2.48e-06 | 1328.9ms/step | 15.1 samples/s +Step 42070/44000 | Loss: 2.9079 | LR: 2.45e-06 | 1323.8ms/step | 15.1 samples/s +Step 42080/44000 | Loss: 2.9274 | LR: 2.43e-06 | 1324.4ms/step | 15.1 samples/s +Step 42090/44000 | Loss: 2.9445 | LR: 2.40e-06 | 1326.6ms/step | 15.1 samples/s +Step 42100/44000 | Loss: 2.8833 | LR: 2.38e-06 | 1322.6ms/step | 15.1 samples/s +Step 42110/44000 | Loss: 2.9251 | LR: 2.35e-06 | 1323.2ms/step | 15.1 samples/s +Step 42120/44000 | Loss: 2.8903 | LR: 2.33e-06 | 1325.7ms/step | 15.1 samples/s +Step 42130/44000 | Loss: 2.9508 | LR: 2.30e-06 | 1322.0ms/step | 15.1 samples/s +Step 42140/44000 | Loss: 2.8860 | LR: 2.28e-06 | 1327.7ms/step | 15.1 samples/s +Step 42150/44000 | Loss: 2.8998 | LR: 2.25e-06 | 1328.8ms/step | 15.1 samples/s +Step 42160/44000 | Loss: 2.9251 | LR: 2.23e-06 | 1325.4ms/step | 15.1 samples/s +Step 42170/44000 | Loss: 2.8964 | LR: 2.21e-06 | 1325.1ms/step | 15.1 samples/s +Step 42180/44000 | Loss: 2.9055 | LR: 2.18e-06 | 1326.2ms/step | 15.1 samples/s +Step 42190/44000 | Loss: 2.9387 | LR: 2.16e-06 | 1326.9ms/step | 15.1 samples/s +Step 42200/44000 | Loss: 2.9062 | LR: 2.13e-06 | 1323.0ms/step | 15.1 samples/s +Step 42210/44000 | Loss: 2.9397 | LR: 2.11e-06 | 1325.5ms/step | 15.1 samples/s +Step 42220/44000 | Loss: 2.9170 | LR: 2.09e-06 | 1325.7ms/step | 15.1 samples/s +Step 42230/44000 | Loss: 2.8955 | LR: 2.06e-06 | 1328.4ms/step | 15.1 samples/s +Step 42240/44000 | Loss: 2.9359 | LR: 2.04e-06 | 1326.8ms/step | 15.1 samples/s +Step 42250/44000 | Loss: 2.9136 | LR: 2.02e-06 | 1326.1ms/step | 15.1 samples/s +Validation at step 42250: Loss = 2.9294, Perplexity = 18.72 +Step 42260/44000 | Loss: 2.9250 | LR: 1.99e-06 | 3558.0ms/step | 5.6 samples/s +Step 42270/44000 | Loss: 2.9418 | LR: 1.97e-06 | 1320.2ms/step | 15.1 samples/s +Step 42280/44000 | Loss: 2.9374 | LR: 1.95e-06 | 1325.6ms/step | 15.1 samples/s +Step 42290/44000 | Loss: 2.8897 | LR: 1.93e-06 | 1324.8ms/step | 15.1 samples/s +Step 42300/44000 | Loss: 2.9258 | LR: 1.90e-06 | 1326.0ms/step | 15.1 samples/s +Step 42310/44000 | Loss: 2.9187 | LR: 1.88e-06 | 1325.8ms/step | 15.1 samples/s +Step 42320/44000 | Loss: 2.9378 | LR: 1.86e-06 | 1324.9ms/step | 15.1 samples/s +Step 42330/44000 | Loss: 2.9457 | LR: 1.84e-06 | 1324.8ms/step | 15.1 samples/s +Step 42340/44000 | Loss: 2.9194 | LR: 1.82e-06 | 1323.1ms/step | 15.1 samples/s +Step 42350/44000 | Loss: 2.9170 | LR: 1.79e-06 | 1323.7ms/step | 15.1 samples/s +Step 42360/44000 | Loss: 2.9161 | LR: 1.77e-06 | 1327.3ms/step | 15.1 samples/s +Step 42370/44000 | Loss: 2.9265 | LR: 1.75e-06 | 1326.7ms/step | 15.1 samples/s +Step 42380/44000 | Loss: 2.9418 | LR: 1.73e-06 | 1327.2ms/step | 15.1 samples/s +Step 42390/44000 | Loss: 2.9111 | LR: 1.71e-06 | 1326.6ms/step | 15.1 samples/s +Step 42400/44000 | Loss: 2.9187 | LR: 1.69e-06 | 1324.8ms/step | 15.1 samples/s +Step 42410/44000 | Loss: 2.8986 | LR: 1.67e-06 | 1323.3ms/step | 15.1 samples/s +Step 42420/44000 | Loss: 2.9260 | LR: 1.64e-06 | 1326.9ms/step | 15.1 samples/s +Step 42430/44000 | Loss: 2.9425 | LR: 1.62e-06 | 1325.1ms/step | 15.1 samples/s +Step 42440/44000 | Loss: 2.8987 | LR: 1.60e-06 | 1322.6ms/step | 15.1 samples/s +Step 42450/44000 | Loss: 2.9457 | LR: 1.58e-06 | 1319.9ms/step | 15.2 samples/s +Step 42460/44000 | Loss: 2.9270 | LR: 1.56e-06 | 1323.6ms/step | 15.1 samples/s +Step 42470/44000 | Loss: 2.9113 | LR: 1.54e-06 | 1327.6ms/step | 15.1 samples/s +Step 42480/44000 | Loss: 2.9376 | LR: 1.52e-06 | 1325.9ms/step | 15.1 samples/s +Step 42490/44000 | Loss: 2.9031 | LR: 1.50e-06 | 1323.6ms/step | 15.1 samples/s +Step 42500/44000 | Loss: 2.9274 | LR: 1.48e-06 | 1328.5ms/step | 15.1 samples/s +Validation at step 42500: Loss = 2.9294, Perplexity = 18.72 +Step 42510/44000 | Loss: 2.9097 | LR: 1.46e-06 | 3548.9ms/step | 5.6 samples/s +Step 42520/44000 | Loss: 2.9216 | LR: 1.44e-06 | 1322.7ms/step | 15.1 samples/s +Step 42530/44000 | Loss: 2.8898 | LR: 1.42e-06 | 1326.4ms/step | 15.1 samples/s +Step 42540/44000 | Loss: 2.9656 | LR: 1.40e-06 | 1326.7ms/step | 15.1 samples/s +Step 42550/44000 | Loss: 2.9173 | LR: 1.39e-06 | 1326.0ms/step | 15.1 samples/s +Step 42560/44000 | Loss: 2.9346 | LR: 1.37e-06 | 1323.8ms/step | 15.1 samples/s +Step 42570/44000 | Loss: 2.9300 | LR: 1.35e-06 | 1325.7ms/step | 15.1 samples/s +Step 42580/44000 | Loss: 2.9193 | LR: 1.33e-06 | 1323.9ms/step | 15.1 samples/s +Step 42590/44000 | Loss: 2.9277 | LR: 1.31e-06 | 1322.4ms/step | 15.1 samples/s +Step 42600/44000 | Loss: 2.9095 | LR: 1.29e-06 | 1324.8ms/step | 15.1 samples/s +Step 42610/44000 | Loss: 2.9114 | LR: 1.27e-06 | 1320.7ms/step | 15.1 samples/s +Step 42620/44000 | Loss: 2.9490 | LR: 1.25e-06 | 1324.7ms/step | 15.1 samples/s +Step 42630/44000 | Loss: 2.8934 | LR: 1.24e-06 | 1325.3ms/step | 15.1 samples/s +Step 42640/44000 | Loss: 2.9351 | LR: 1.22e-06 | 1322.7ms/step | 15.1 samples/s +Step 42650/44000 | Loss: 2.9334 | LR: 1.20e-06 | 1329.5ms/step | 15.0 samples/s +Step 42660/44000 | Loss: 2.9261 | LR: 1.18e-06 | 1326.2ms/step | 15.1 samples/s +Step 42670/44000 | Loss: 2.9127 | LR: 1.17e-06 | 1321.9ms/step | 15.1 samples/s +Step 42680/44000 | Loss: 2.9196 | LR: 1.15e-06 | 1327.7ms/step | 15.1 samples/s +Step 42690/44000 | Loss: 2.9496 | LR: 1.13e-06 | 1327.3ms/step | 15.1 samples/s +Step 42700/44000 | Loss: 2.8992 | LR: 1.11e-06 | 1324.7ms/step | 15.1 samples/s +Step 42710/44000 | Loss: 2.9185 | LR: 1.10e-06 | 1327.2ms/step | 15.1 samples/s +Step 42720/44000 | Loss: 2.8954 | LR: 1.08e-06 | 1323.4ms/step | 15.1 samples/s +Step 42730/44000 | Loss: 2.9059 | LR: 1.06e-06 | 1327.0ms/step | 15.1 samples/s +Step 42740/44000 | Loss: 2.9406 | LR: 1.05e-06 | 1327.1ms/step | 15.1 samples/s +Step 42750/44000 | Loss: 2.8876 | LR: 1.03e-06 | 1326.9ms/step | 15.1 samples/s +Validation at step 42750: Loss = 2.9292, Perplexity = 18.71 +Step 42760/44000 | Loss: 2.9383 | LR: 1.01e-06 | 3544.2ms/step | 5.6 samples/s +Step 42770/44000 | Loss: 2.9024 | LR: 9.97e-07 | 1322.7ms/step | 15.1 samples/s +Step 42780/44000 | Loss: 2.9067 | LR: 9.81e-07 | 1324.9ms/step | 15.1 samples/s +Step 42790/44000 | Loss: 2.9102 | LR: 9.65e-07 | 1325.1ms/step | 15.1 samples/s +Step 42800/44000 | Loss: 2.9383 | LR: 9.49e-07 | 1325.4ms/step | 15.1 samples/s +Step 42810/44000 | Loss: 2.9281 | LR: 9.33e-07 | 1329.3ms/step | 15.0 samples/s +Step 42820/44000 | Loss: 2.9297 | LR: 9.18e-07 | 1328.7ms/step | 15.1 samples/s +Step 42830/44000 | Loss: 2.9345 | LR: 9.02e-07 | 1322.4ms/step | 15.1 samples/s +Step 42840/44000 | Loss: 2.9193 | LR: 8.87e-07 | 1328.1ms/step | 15.1 samples/s +Step 42850/44000 | Loss: 2.9077 | LR: 8.72e-07 | 1324.2ms/step | 15.1 samples/s +Step 42860/44000 | Loss: 2.9401 | LR: 8.57e-07 | 1323.2ms/step | 15.1 samples/s +Step 42870/44000 | Loss: 2.9265 | LR: 8.42e-07 | 1330.4ms/step | 15.0 samples/s +Step 42880/44000 | Loss: 2.9085 | LR: 8.27e-07 | 1323.8ms/step | 15.1 samples/s +Step 42890/44000 | Loss: 2.9054 | LR: 8.12e-07 | 1323.8ms/step | 15.1 samples/s +Step 42900/44000 | Loss: 2.9134 | LR: 7.98e-07 | 1325.4ms/step | 15.1 samples/s +Step 42910/44000 | Loss: 2.9109 | LR: 7.83e-07 | 1320.1ms/step | 15.2 samples/s +Step 42920/44000 | Loss: 2.9445 | LR: 7.69e-07 | 1323.8ms/step | 15.1 samples/s +Step 42930/44000 | Loss: 2.9233 | LR: 7.55e-07 | 1322.8ms/step | 15.1 samples/s +Step 42940/44000 | Loss: 2.8868 | LR: 7.41e-07 | 1325.0ms/step | 15.1 samples/s +Step 42950/44000 | Loss: 2.9412 | LR: 7.27e-07 | 1324.4ms/step | 15.1 samples/s +Step 42960/44000 | Loss: 2.9396 | LR: 7.13e-07 | 1327.0ms/step | 15.1 samples/s +Step 42970/44000 | Loss: 2.9409 | LR: 6.99e-07 | 1331.5ms/step | 15.0 samples/s +Step 42980/44000 | Loss: 2.9254 | LR: 6.86e-07 | 1326.0ms/step | 15.1 samples/s +Step 42990/44000 | Loss: 2.9253 | LR: 6.72e-07 | 1330.3ms/step | 15.0 samples/s +Step 43000/44000 | Loss: 2.9471 | LR: 6.59e-07 | 1328.0ms/step | 15.1 samples/s +Validation at step 43000: Loss = 2.9297, Perplexity = 18.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-43000 +Step 43010/44000 | Loss: 2.9309 | LR: 6.46e-07 | 3924.9ms/step | 5.1 samples/s +Step 43020/44000 | Loss: 2.9090 | LR: 6.33e-07 | 1337.4ms/step | 15.0 samples/s +Step 43030/44000 | Loss: 2.9087 | LR: 6.20e-07 | 1337.9ms/step | 14.9 samples/s +Step 43040/44000 | Loss: 2.9268 | LR: 6.08e-07 | 1353.2ms/step | 14.8 samples/s +Step 43050/44000 | Loss: 2.8976 | LR: 5.95e-07 | 1363.4ms/step | 14.7 samples/s +Step 43060/44000 | Loss: 2.9338 | LR: 5.83e-07 | 1331.2ms/step | 15.0 samples/s +Step 43070/44000 | Loss: 2.8960 | LR: 5.70e-07 | 1343.9ms/step | 14.9 samples/s +Step 43080/44000 | Loss: 2.9181 | LR: 5.58e-07 | 1365.2ms/step | 14.6 samples/s +Step 43090/44000 | Loss: 2.9182 | LR: 5.46e-07 | 1357.1ms/step | 14.7 samples/s +Step 43100/44000 | Loss: 2.9165 | LR: 5.34e-07 | 1355.9ms/step | 14.7 samples/s +Step 43110/44000 | Loss: 2.9453 | LR: 5.22e-07 | 1353.3ms/step | 14.8 samples/s +Step 43120/44000 | Loss: 2.9203 | LR: 5.11e-07 | 1351.4ms/step | 14.8 samples/s +Step 43130/44000 | Loss: 2.9436 | LR: 4.99e-07 | 1351.8ms/step | 14.8 samples/s +Step 43140/44000 | Loss: 2.9267 | LR: 4.88e-07 | 1356.3ms/step | 14.7 samples/s +Step 43150/44000 | Loss: 2.9304 | LR: 4.76e-07 | 1354.9ms/step | 14.8 samples/s +Step 43160/44000 | Loss: 2.9151 | LR: 4.65e-07 | 1358.9ms/step | 14.7 samples/s +Step 43170/44000 | Loss: 2.9339 | LR: 4.54e-07 | 1354.5ms/step | 14.8 samples/s +Step 43180/44000 | Loss: 2.8950 | LR: 4.43e-07 | 1355.9ms/step | 14.8 samples/s +Step 43190/44000 | Loss: 2.9128 | LR: 4.33e-07 | 1355.8ms/step | 14.8 samples/s +Step 43200/44000 | Loss: 2.9441 | LR: 4.22e-07 | 1345.9ms/step | 14.9 samples/s +Step 43210/44000 | Loss: 2.9236 | LR: 4.12e-07 | 1349.0ms/step | 14.8 samples/s +Step 43220/44000 | Loss: 2.9308 | LR: 4.01e-07 | 1340.4ms/step | 14.9 samples/s +Step 43230/44000 | Loss: 2.8846 | LR: 3.91e-07 | 1336.5ms/step | 15.0 samples/s +Step 43240/44000 | Loss: 2.9394 | LR: 3.81e-07 | 1335.7ms/step | 15.0 samples/s +Step 43250/44000 | Loss: 2.9480 | LR: 3.71e-07 | 1373.1ms/step | 14.6 samples/s +Validation at step 43250: Loss = 2.9293, Perplexity = 18.71 +Step 43260/44000 | Loss: 2.8877 | LR: 3.61e-07 | 3585.0ms/step | 5.6 samples/s +Step 43270/44000 | Loss: 2.8978 | LR: 3.51e-07 | 1331.9ms/step | 15.0 samples/s +Step 43280/44000 | Loss: 2.9027 | LR: 3.42e-07 | 1340.1ms/step | 14.9 samples/s +Step 43290/44000 | Loss: 2.9426 | LR: 3.32e-07 | 1361.9ms/step | 14.7 samples/s +Step 43300/44000 | Loss: 2.9566 | LR: 3.23e-07 | 1346.6ms/step | 14.9 samples/s +Step 43310/44000 | Loss: 2.9465 | LR: 3.14e-07 | 1345.2ms/step | 14.9 samples/s +Step 43320/44000 | Loss: 2.9066 | LR: 3.05e-07 | 1336.9ms/step | 15.0 samples/s +Step 43330/44000 | Loss: 2.9095 | LR: 2.96e-07 | 1336.2ms/step | 15.0 samples/s +Step 43340/44000 | Loss: 2.9157 | LR: 2.87e-07 | 1333.2ms/step | 15.0 samples/s +Step 43350/44000 | Loss: 2.9308 | LR: 2.79e-07 | 1334.7ms/step | 15.0 samples/s +Step 43360/44000 | Loss: 2.9012 | LR: 2.70e-07 | 1330.7ms/step | 15.0 samples/s +Step 43370/44000 | Loss: 2.9128 | LR: 2.62e-07 | 1327.1ms/step | 15.1 samples/s +Step 43380/44000 | Loss: 2.9441 | LR: 2.53e-07 | 1331.7ms/step | 15.0 samples/s +Step 43390/44000 | Loss: 2.9080 | LR: 2.45e-07 | 1326.8ms/step | 15.1 samples/s +Step 43400/44000 | Loss: 2.9384 | LR: 2.37e-07 | 1327.2ms/step | 15.1 samples/s +Step 43410/44000 | Loss: 2.8965 | LR: 2.30e-07 | 1356.9ms/step | 14.7 samples/s +Step 43420/44000 | Loss: 2.9077 | LR: 2.22e-07 | 1374.2ms/step | 14.6 samples/s +Step 43430/44000 | Loss: 2.9179 | LR: 2.14e-07 | 1344.0ms/step | 14.9 samples/s +Step 43440/44000 | Loss: 2.8900 | LR: 2.07e-07 | 1330.1ms/step | 15.0 samples/s +Step 43450/44000 | Loss: 2.9205 | LR: 1.99e-07 | 1329.6ms/step | 15.0 samples/s +Step 43460/44000 | Loss: 2.8938 | LR: 1.92e-07 | 1329.7ms/step | 15.0 samples/s +Step 43470/44000 | Loss: 2.9279 | LR: 1.85e-07 | 1327.5ms/step | 15.1 samples/s +Step 43480/44000 | Loss: 2.9236 | LR: 1.78e-07 | 1329.3ms/step | 15.0 samples/s +Step 43490/44000 | Loss: 2.9071 | LR: 1.72e-07 | 1329.5ms/step | 15.0 samples/s +Step 43500/44000 | Loss: 2.9517 | LR: 1.65e-07 | 1328.4ms/step | 15.1 samples/s +Validation at step 43500: Loss = 2.9296, Perplexity = 18.72 +Step 43510/44000 | Loss: 2.9269 | LR: 1.58e-07 | 3561.6ms/step | 5.6 samples/s +Step 43520/44000 | Loss: 2.9480 | LR: 1.52e-07 | 1327.7ms/step | 15.1 samples/s +Step 43530/44000 | Loss: 2.9092 | LR: 1.46e-07 | 1328.3ms/step | 15.1 samples/s +Step 43540/44000 | Loss: 2.9461 | LR: 1.40e-07 | 1332.6ms/step | 15.0 samples/s +Step 43550/44000 | Loss: 2.9056 | LR: 1.34e-07 | 1330.7ms/step | 15.0 samples/s +Step 43560/44000 | Loss: 2.9071 | LR: 1.28e-07 | 1326.1ms/step | 15.1 samples/s +Step 43570/44000 | Loss: 2.9118 | LR: 1.22e-07 | 1332.2ms/step | 15.0 samples/s +Step 43580/44000 | Loss: 2.9430 | LR: 1.16e-07 | 1326.9ms/step | 15.1 samples/s +Step 43590/44000 | Loss: 2.9410 | LR: 1.11e-07 | 1324.6ms/step | 15.1 samples/s +Step 43600/44000 | Loss: 2.9178 | LR: 1.06e-07 | 1325.8ms/step | 15.1 samples/s +Step 43610/44000 | Loss: 2.9219 | LR: 1.00e-07 | 1324.5ms/step | 15.1 samples/s +Step 43620/44000 | Loss: 2.9228 | LR: 9.52e-08 | 1322.9ms/step | 15.1 samples/s +Step 43630/44000 | Loss: 2.9282 | LR: 9.03e-08 | 1326.5ms/step | 15.1 samples/s +Step 43640/44000 | Loss: 2.9056 | LR: 8.55e-08 | 1326.9ms/step | 15.1 samples/s +Step 43650/44000 | Loss: 2.9111 | LR: 8.08e-08 | 1326.2ms/step | 15.1 samples/s +Step 43660/44000 | Loss: 2.9326 | LR: 7.62e-08 | 1325.8ms/step | 15.1 samples/s +Step 43670/44000 | Loss: 2.9367 | LR: 7.18e-08 | 1325.5ms/step | 15.1 samples/s +Step 43680/44000 | Loss: 2.9327 | LR: 6.75e-08 | 1322.3ms/step | 15.1 samples/s +Step 43690/44000 | Loss: 2.9323 | LR: 6.34e-08 | 1323.4ms/step | 15.1 samples/s +Step 43700/44000 | Loss: 2.9078 | LR: 5.94e-08 | 1325.5ms/step | 15.1 samples/s +Step 43710/44000 | Loss: 2.9415 | LR: 5.55e-08 | 1325.0ms/step | 15.1 samples/s +Step 43720/44000 | Loss: 2.8635 | LR: 5.17e-08 | 1325.4ms/step | 15.1 samples/s +Step 43730/44000 | Loss: 2.9421 | LR: 4.81e-08 | 1327.0ms/step | 15.1 samples/s +Step 43740/44000 | Loss: 2.9156 | LR: 4.46e-08 | 1322.0ms/step | 15.1 samples/s +Step 43750/44000 | Loss: 2.9031 | LR: 4.12e-08 | 1326.1ms/step | 15.1 samples/s +Validation at step 43750: Loss = 2.9296, Perplexity = 18.72 +Step 43760/44000 | Loss: 2.8981 | LR: 3.80e-08 | 3550.7ms/step | 5.6 samples/s +Step 43770/44000 | Loss: 2.9210 | LR: 3.49e-08 | 1322.8ms/step | 15.1 samples/s +Step 43780/44000 | Loss: 2.9098 | LR: 3.19e-08 | 1321.7ms/step | 15.1 samples/s +Step 43790/44000 | Loss: 2.9744 | LR: 2.91e-08 | 1326.3ms/step | 15.1 samples/s +Step 43800/44000 | Loss: 2.9193 | LR: 2.64e-08 | 1323.5ms/step | 15.1 samples/s +Step 43810/44000 | Loss: 2.9313 | LR: 2.38e-08 | 1324.5ms/step | 15.1 samples/s +Step 43820/44000 | Loss: 2.9305 | LR: 2.14e-08 | 1323.0ms/step | 15.1 samples/s +Step 43830/44000 | Loss: 2.9295 | LR: 1.91e-08 | 1327.0ms/step | 15.1 samples/s +Step 43840/44000 | Loss: 2.9118 | LR: 1.69e-08 | 1323.4ms/step | 15.1 samples/s +Step 43850/44000 | Loss: 2.9214 | LR: 1.48e-08 | 1323.9ms/step | 15.1 samples/s +Step 43860/44000 | Loss: 2.9235 | LR: 1.29e-08 | 1325.4ms/step | 15.1 samples/s +Step 43870/44000 | Loss: 2.9354 | LR: 1.11e-08 | 1323.6ms/step | 15.1 samples/s +Step 43880/44000 | Loss: 2.8907 | LR: 9.50e-09 | 1324.2ms/step | 15.1 samples/s +Step 43890/44000 | Loss: 2.9187 | LR: 7.98e-09 | 1328.5ms/step | 15.1 samples/s +Step 43900/44000 | Loss: 2.9185 | LR: 6.60e-09 | 1326.8ms/step | 15.1 samples/s +Step 43910/44000 | Loss: 2.9286 | LR: 5.34e-09 | 1323.4ms/step | 15.1 samples/s +Step 43920/44000 | Loss: 2.8983 | LR: 4.22e-09 | 1321.5ms/step | 15.1 samples/s +Step 43930/44000 | Loss: 2.9197 | LR: 3.23e-09 | 1319.9ms/step | 15.2 samples/s +Step 43940/44000 | Loss: 2.9304 | LR: 2.37e-09 | 1326.1ms/step | 15.1 samples/s +Step 43950/44000 | Loss: 2.9500 | LR: 1.65e-09 | 1326.8ms/step | 15.1 samples/s +Step 43960/44000 | Loss: 2.9553 | LR: 1.06e-09 | 1324.8ms/step | 15.1 samples/s +Step 43970/44000 | Loss: 2.9101 | LR: 5.94e-10 | 1325.8ms/step | 15.1 samples/s +Step 43980/44000 | Loss: 2.9202 | LR: 2.64e-10 | 1324.3ms/step | 15.1 samples/s +Step 43990/44000 | Loss: 2.9246 | LR: 6.60e-11 | 1324.9ms/step | 15.1 samples/s +Step 44000/44000 | Loss: 2.9037 | LR: 0.00e+00 | 1323.7ms/step | 15.1 samples/s +Validation at step 44000: Loss = 2.9293, Perplexity = 18.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading history steps 4399-4399, summary, console lines 1901-1903 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss ██▇▇▇▆▅▅▅▅▄▄▃▃▃▃▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▆▆▅▅▅▄▄▃▃▃▃▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▇▁▁▁▁▁▁▁▁▁▇▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ███▁██████████████████████████████▁█████ +wandb: speed/steps_per_sec ███████████████████████▁████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.92928 +wandb: eval/perplexity 18.71413 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 1323.72015 +wandb: speed/samples_per_sec 15.10893 +wandb: speed/steps_per_sec 0.75545 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 resume 365m resampled 8k nrb35 seed44 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/27rw10jv +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_043604-27rw10jv/logs +Training complete! diff --git a/epfl_final/train_logs/p1_resume_h1002_20260822/l8k365m_static_nrb35_seed44.log b/epfl_final/train_logs/p1_resume_h1002_20260822/l8k365m_static_nrb35_seed44.log new file mode 100644 index 0000000000000000000000000000000000000000..fff2e19628fb86f8e671575abf072eb8cd1eaae4 --- /dev/null +++ b/epfl_final/train_logs/p1_resume_h1002_20260822/l8k365m_static_nrb35_seed44.log @@ -0,0 +1,2145 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-26000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 44, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 resume 365m static 8k nrb35 seed44", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "static", + "nrb35", + "seed44", + "resume", + "h1002", + "step26000" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1", + "sparse_mask_seed": 1, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-26000 +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-26000... +Preserving checkpoint LR state for exact resume: current=[0.00018492988117498478], schedule_base=[0.0005] +Resumed from checkpoint at step 26000 (epoch 0, micro_step 260000) +Resuming training from step 26000 +Resuming wandb run na2e5dc1 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_043357-na2e5dc1 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run P1 resume 365m static 8k nrb35 seed44 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/na2e5dc1 +Starting training... +Validation at step 26000: Loss = 2.9564, Perplexity = 19.23 +wandb: WARNING Tried to log to step 26000 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 26001: sparse_stochastic (configured sparse attention) +Step 26010/44000 | Loss: 2.9306 | LR: 1.85e-04 | 60623.6ms/step | 0.3 samples/s +wandb: WARNING Tried to log to step 26010 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26020/44000 | Loss: 2.9484 | LR: 1.85e-04 | 1312.7ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26020 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26030/44000 | Loss: 2.9276 | LR: 1.84e-04 | 1310.3ms/step | 15.3 samples/s +wandb: WARNING Tried to log to step 26030 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26040/44000 | Loss: 2.9548 | LR: 1.84e-04 | 1310.5ms/step | 15.3 samples/s +wandb: WARNING Tried to log to step 26040 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26050/44000 | Loss: 2.9266 | LR: 1.84e-04 | 1312.6ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26050 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26060/44000 | Loss: 2.9614 | LR: 1.84e-04 | 1317.0ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26060 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26070/44000 | Loss: 2.9579 | LR: 1.84e-04 | 1314.1ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26070 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26080/44000 | Loss: 2.9389 | LR: 1.84e-04 | 1316.5ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26080 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26090/44000 | Loss: 2.9497 | LR: 1.83e-04 | 1317.8ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26090 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26100/44000 | Loss: 2.9558 | LR: 1.83e-04 | 1315.7ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26100 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26110/44000 | Loss: 2.9554 | LR: 1.83e-04 | 1315.0ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26110 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26120/44000 | Loss: 2.9436 | LR: 1.83e-04 | 1316.8ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26120 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26130/44000 | Loss: 2.9283 | LR: 1.83e-04 | 1318.4ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26130 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26140/44000 | Loss: 2.9219 | LR: 1.82e-04 | 1318.4ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26140 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26150/44000 | Loss: 2.9871 | LR: 1.82e-04 | 1317.0ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26150 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26160/44000 | Loss: 2.9690 | LR: 1.82e-04 | 1317.9ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26160 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26170/44000 | Loss: 2.9123 | LR: 1.82e-04 | 1317.0ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26170 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26180/44000 | Loss: 2.9293 | LR: 1.82e-04 | 1318.3ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26180 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26190/44000 | Loss: 2.9142 | LR: 1.82e-04 | 1316.0ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26190 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26200/44000 | Loss: 2.9423 | LR: 1.81e-04 | 1318.3ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26200 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26210/44000 | Loss: 2.9230 | LR: 1.81e-04 | 1318.2ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26210 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26220/44000 | Loss: 2.9412 | LR: 1.81e-04 | 1318.0ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26220 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26230/44000 | Loss: 2.9099 | LR: 1.81e-04 | 1316.3ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26230 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26240/44000 | Loss: 2.9414 | LR: 1.81e-04 | 1315.1ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26240 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26250/44000 | Loss: 2.9430 | LR: 1.81e-04 | 1317.5ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 26250 that is less than the current step 26251. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 26250: Loss = 2.9550, Perplexity = 19.20 +Step 26260/44000 | Loss: 2.9483 | LR: 1.80e-04 | 3531.9ms/step | 5.7 samples/s +Step 26270/44000 | Loss: 2.9499 | LR: 1.80e-04 | 1323.3ms/step | 15.1 samples/s +Step 26280/44000 | Loss: 2.9531 | LR: 1.80e-04 | 1323.4ms/step | 15.1 samples/s +Step 26290/44000 | Loss: 2.9462 | LR: 1.80e-04 | 1323.6ms/step | 15.1 samples/s +Step 26300/44000 | Loss: 2.9351 | LR: 1.80e-04 | 1322.8ms/step | 15.1 samples/s +Step 26310/44000 | Loss: 2.9139 | LR: 1.80e-04 | 1324.0ms/step | 15.1 samples/s +Step 26320/44000 | Loss: 2.9242 | LR: 1.79e-04 | 1325.0ms/step | 15.1 samples/s +Step 26330/44000 | Loss: 2.9781 | LR: 1.79e-04 | 1323.8ms/step | 15.1 samples/s +Step 26340/44000 | Loss: 2.9387 | LR: 1.79e-04 | 1322.9ms/step | 15.1 samples/s +Step 26350/44000 | Loss: 2.9810 | LR: 1.79e-04 | 1324.7ms/step | 15.1 samples/s +Step 26360/44000 | Loss: 2.9655 | LR: 1.79e-04 | 1321.2ms/step | 15.1 samples/s +Step 26370/44000 | Loss: 2.9671 | LR: 1.78e-04 | 1322.8ms/step | 15.1 samples/s +Step 26380/44000 | Loss: 2.9599 | LR: 1.78e-04 | 1321.1ms/step | 15.1 samples/s +Step 26390/44000 | Loss: 2.9678 | LR: 1.78e-04 | 1322.0ms/step | 15.1 samples/s +Step 26400/44000 | Loss: 2.9718 | LR: 1.78e-04 | 1323.5ms/step | 15.1 samples/s +Step 26410/44000 | Loss: 2.9293 | LR: 1.78e-04 | 1323.7ms/step | 15.1 samples/s +Step 26420/44000 | Loss: 2.9465 | LR: 1.78e-04 | 1324.2ms/step | 15.1 samples/s +Step 26430/44000 | Loss: 2.9241 | LR: 1.77e-04 | 1323.7ms/step | 15.1 samples/s +Step 26440/44000 | Loss: 2.8880 | LR: 1.77e-04 | 1325.2ms/step | 15.1 samples/s +Step 26450/44000 | Loss: 2.9372 | LR: 1.77e-04 | 1325.3ms/step | 15.1 samples/s +Step 26460/44000 | Loss: 2.9431 | LR: 1.77e-04 | 1324.1ms/step | 15.1 samples/s +Step 26470/44000 | Loss: 2.9563 | LR: 1.77e-04 | 1324.6ms/step | 15.1 samples/s +Step 26480/44000 | Loss: 2.9479 | LR: 1.77e-04 | 1323.3ms/step | 15.1 samples/s +Step 26490/44000 | Loss: 2.9584 | LR: 1.76e-04 | 1323.7ms/step | 15.1 samples/s +Step 26500/44000 | Loss: 2.9265 | LR: 1.76e-04 | 1323.3ms/step | 15.1 samples/s +Validation at step 26500: Loss = 2.9536, Perplexity = 19.17 +Step 26510/44000 | Loss: 2.9304 | LR: 1.76e-04 | 3534.6ms/step | 5.7 samples/s +Step 26520/44000 | Loss: 2.9269 | LR: 1.76e-04 | 1323.1ms/step | 15.1 samples/s +Step 26530/44000 | Loss: 2.9268 | LR: 1.76e-04 | 1323.9ms/step | 15.1 samples/s +Step 26540/44000 | Loss: 2.9496 | LR: 1.76e-04 | 1321.9ms/step | 15.1 samples/s +Step 26550/44000 | Loss: 2.9424 | LR: 1.75e-04 | 1320.9ms/step | 15.1 samples/s +Step 26560/44000 | Loss: 2.9379 | LR: 1.75e-04 | 1323.4ms/step | 15.1 samples/s +Step 26570/44000 | Loss: 2.9561 | LR: 1.75e-04 | 1322.5ms/step | 15.1 samples/s +Step 26580/44000 | Loss: 2.9624 | LR: 1.75e-04 | 1322.3ms/step | 15.1 samples/s +Step 26590/44000 | Loss: 2.9463 | LR: 1.75e-04 | 1322.1ms/step | 15.1 samples/s +Step 26600/44000 | Loss: 2.9498 | LR: 1.74e-04 | 1319.6ms/step | 15.2 samples/s +Step 26610/44000 | Loss: 2.9314 | LR: 1.74e-04 | 1324.1ms/step | 15.1 samples/s +Step 26620/44000 | Loss: 2.9468 | LR: 1.74e-04 | 1323.0ms/step | 15.1 samples/s +Step 26630/44000 | Loss: 2.9400 | LR: 1.74e-04 | 1322.5ms/step | 15.1 samples/s +Step 26640/44000 | Loss: 2.9292 | LR: 1.74e-04 | 1322.9ms/step | 15.1 samples/s +Step 26650/44000 | Loss: 2.9777 | LR: 1.74e-04 | 1323.3ms/step | 15.1 samples/s +Step 26660/44000 | Loss: 2.9221 | LR: 1.73e-04 | 1320.9ms/step | 15.1 samples/s +Step 26670/44000 | Loss: 2.9599 | LR: 1.73e-04 | 1321.6ms/step | 15.1 samples/s +Step 26680/44000 | Loss: 2.9178 | LR: 1.73e-04 | 1323.5ms/step | 15.1 samples/s +Step 26690/44000 | Loss: 2.9238 | LR: 1.73e-04 | 1322.5ms/step | 15.1 samples/s +Step 26700/44000 | Loss: 2.9326 | LR: 1.73e-04 | 1323.6ms/step | 15.1 samples/s +Step 26710/44000 | Loss: 2.9548 | LR: 1.73e-04 | 1323.5ms/step | 15.1 samples/s +Step 26720/44000 | Loss: 2.9520 | LR: 1.72e-04 | 1320.1ms/step | 15.1 samples/s +Step 26730/44000 | Loss: 2.9235 | LR: 1.72e-04 | 1320.4ms/step | 15.1 samples/s +Step 26740/44000 | Loss: 2.9511 | LR: 1.72e-04 | 1324.0ms/step | 15.1 samples/s +Step 26750/44000 | Loss: 2.9705 | LR: 1.72e-04 | 1322.7ms/step | 15.1 samples/s +Validation at step 26750: Loss = 2.9521, Perplexity = 19.15 +Step 26760/44000 | Loss: 2.9495 | LR: 1.72e-04 | 3531.2ms/step | 5.7 samples/s +Step 26770/44000 | Loss: 2.9599 | LR: 1.72e-04 | 1322.5ms/step | 15.1 samples/s +Step 26780/44000 | Loss: 2.9473 | LR: 1.71e-04 | 1325.3ms/step | 15.1 samples/s +Step 26790/44000 | Loss: 2.9590 | LR: 1.71e-04 | 1324.6ms/step | 15.1 samples/s +Step 26800/44000 | Loss: 2.9616 | LR: 1.71e-04 | 1324.9ms/step | 15.1 samples/s +Step 26810/44000 | Loss: 2.9514 | LR: 1.71e-04 | 1323.3ms/step | 15.1 samples/s +Step 26820/44000 | Loss: 2.9430 | LR: 1.71e-04 | 1324.7ms/step | 15.1 samples/s +Step 26830/44000 | Loss: 2.9583 | LR: 1.71e-04 | 1322.7ms/step | 15.1 samples/s +Step 26840/44000 | Loss: 2.9185 | LR: 1.70e-04 | 1324.6ms/step | 15.1 samples/s +Step 26850/44000 | Loss: 2.9520 | LR: 1.70e-04 | 1322.3ms/step | 15.1 samples/s +Step 26860/44000 | Loss: 2.9413 | LR: 1.70e-04 | 1321.3ms/step | 15.1 samples/s +Step 26870/44000 | Loss: 2.9664 | LR: 1.70e-04 | 1322.9ms/step | 15.1 samples/s +Step 26880/44000 | Loss: 2.9427 | LR: 1.70e-04 | 1322.3ms/step | 15.1 samples/s +Step 26890/44000 | Loss: 2.9212 | LR: 1.69e-04 | 1323.0ms/step | 15.1 samples/s +Step 26900/44000 | Loss: 2.9304 | LR: 1.69e-04 | 1320.8ms/step | 15.1 samples/s +Step 26910/44000 | Loss: 2.9742 | LR: 1.69e-04 | 1323.2ms/step | 15.1 samples/s +Step 26920/44000 | Loss: 2.9468 | LR: 1.69e-04 | 1322.1ms/step | 15.1 samples/s +Step 26930/44000 | Loss: 2.9693 | LR: 1.69e-04 | 1322.8ms/step | 15.1 samples/s +Step 26940/44000 | Loss: 2.9460 | LR: 1.69e-04 | 1322.9ms/step | 15.1 samples/s +Step 26950/44000 | Loss: 2.9370 | LR: 1.68e-04 | 1321.9ms/step | 15.1 samples/s +Step 26960/44000 | Loss: 2.9279 | LR: 1.68e-04 | 1323.3ms/step | 15.1 samples/s +Step 26970/44000 | Loss: 2.9244 | LR: 1.68e-04 | 1321.6ms/step | 15.1 samples/s +Step 26980/44000 | Loss: 2.9238 | LR: 1.68e-04 | 1321.6ms/step | 15.1 samples/s +Step 26990/44000 | Loss: 2.9253 | LR: 1.68e-04 | 1322.7ms/step | 15.1 samples/s +Step 27000/44000 | Loss: 2.9342 | LR: 1.68e-04 | 1323.5ms/step | 15.1 samples/s +Validation at step 27000: Loss = 2.9511, Perplexity = 19.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-27000 +Step 27010/44000 | Loss: 2.9392 | LR: 1.67e-04 | 3848.2ms/step | 5.2 samples/s +Step 27020/44000 | Loss: 2.9331 | LR: 1.67e-04 | 1322.2ms/step | 15.1 samples/s +Step 27030/44000 | Loss: 2.9316 | LR: 1.67e-04 | 1320.5ms/step | 15.1 samples/s +Step 27040/44000 | Loss: 2.9201 | LR: 1.67e-04 | 1321.5ms/step | 15.1 samples/s +Step 27050/44000 | Loss: 2.9518 | LR: 1.67e-04 | 1322.0ms/step | 15.1 samples/s +Step 27060/44000 | Loss: 2.9469 | LR: 1.67e-04 | 1321.5ms/step | 15.1 samples/s +Step 27070/44000 | Loss: 2.9729 | LR: 1.66e-04 | 1321.6ms/step | 15.1 samples/s +Step 27080/44000 | Loss: 2.9346 | LR: 1.66e-04 | 1323.1ms/step | 15.1 samples/s +Step 27090/44000 | Loss: 2.9495 | LR: 1.66e-04 | 1323.5ms/step | 15.1 samples/s +Step 27100/44000 | Loss: 2.9361 | LR: 1.66e-04 | 1324.8ms/step | 15.1 samples/s +Step 27110/44000 | Loss: 2.9310 | LR: 1.66e-04 | 1321.6ms/step | 15.1 samples/s +Step 27120/44000 | Loss: 2.9482 | LR: 1.66e-04 | 1322.9ms/step | 15.1 samples/s +Step 27130/44000 | Loss: 2.9392 | LR: 1.65e-04 | 1323.1ms/step | 15.1 samples/s +Step 27140/44000 | Loss: 2.9562 | LR: 1.65e-04 | 1323.4ms/step | 15.1 samples/s +Step 27150/44000 | Loss: 2.9125 | LR: 1.65e-04 | 1322.0ms/step | 15.1 samples/s +Step 27160/44000 | Loss: 2.9482 | LR: 1.65e-04 | 1323.3ms/step | 15.1 samples/s +Step 27170/44000 | Loss: 2.9453 | LR: 1.65e-04 | 1323.9ms/step | 15.1 samples/s +Step 27180/44000 | Loss: 2.9161 | LR: 1.65e-04 | 1324.0ms/step | 15.1 samples/s +Step 27190/44000 | Loss: 2.9616 | LR: 1.64e-04 | 1326.0ms/step | 15.1 samples/s +Step 27200/44000 | Loss: 2.9420 | LR: 1.64e-04 | 1320.1ms/step | 15.2 samples/s +Step 27210/44000 | Loss: 2.9583 | LR: 1.64e-04 | 1321.1ms/step | 15.1 samples/s +Step 27220/44000 | Loss: 2.9194 | LR: 1.64e-04 | 1322.6ms/step | 15.1 samples/s +Step 27230/44000 | Loss: 2.9457 | LR: 1.64e-04 | 1320.0ms/step | 15.2 samples/s +Step 27240/44000 | Loss: 2.9281 | LR: 1.63e-04 | 1322.7ms/step | 15.1 samples/s +Step 27250/44000 | Loss: 2.9347 | LR: 1.63e-04 | 1323.7ms/step | 15.1 samples/s +Validation at step 27250: Loss = 2.9497, Perplexity = 19.10 +Step 27260/44000 | Loss: 2.9499 | LR: 1.63e-04 | 3537.1ms/step | 5.7 samples/s +Step 27270/44000 | Loss: 2.9653 | LR: 1.63e-04 | 1323.0ms/step | 15.1 samples/s +Step 27280/44000 | Loss: 2.9388 | LR: 1.63e-04 | 1324.1ms/step | 15.1 samples/s +Step 27290/44000 | Loss: 2.9562 | LR: 1.63e-04 | 1323.8ms/step | 15.1 samples/s +Step 27300/44000 | Loss: 2.9227 | LR: 1.62e-04 | 1321.7ms/step | 15.1 samples/s +Step 27310/44000 | Loss: 2.9329 | LR: 1.62e-04 | 1322.2ms/step | 15.1 samples/s +Step 27320/44000 | Loss: 2.9490 | LR: 1.62e-04 | 1322.4ms/step | 15.1 samples/s +Step 27330/44000 | Loss: 2.9398 | LR: 1.62e-04 | 1321.7ms/step | 15.1 samples/s +Step 27340/44000 | Loss: 2.9068 | LR: 1.62e-04 | 1321.6ms/step | 15.1 samples/s +Step 27350/44000 | Loss: 2.9452 | LR: 1.62e-04 | 1322.5ms/step | 15.1 samples/s +Step 27360/44000 | Loss: 2.9300 | LR: 1.61e-04 | 1324.0ms/step | 15.1 samples/s +Step 27370/44000 | Loss: 2.9362 | LR: 1.61e-04 | 1322.7ms/step | 15.1 samples/s +Step 27380/44000 | Loss: 2.9443 | LR: 1.61e-04 | 1323.4ms/step | 15.1 samples/s +Step 27390/44000 | Loss: 2.9324 | LR: 1.61e-04 | 1322.3ms/step | 15.1 samples/s +Step 27400/44000 | Loss: 2.9543 | LR: 1.61e-04 | 1323.2ms/step | 15.1 samples/s +Step 27410/44000 | Loss: 2.9490 | LR: 1.61e-04 | 1325.0ms/step | 15.1 samples/s +Step 27420/44000 | Loss: 2.9417 | LR: 1.60e-04 | 1323.8ms/step | 15.1 samples/s +Step 27430/44000 | Loss: 2.9373 | LR: 1.60e-04 | 1319.6ms/step | 15.2 samples/s +Step 27440/44000 | Loss: 2.9233 | LR: 1.60e-04 | 1320.3ms/step | 15.1 samples/s +Step 27450/44000 | Loss: 2.9358 | LR: 1.60e-04 | 1321.7ms/step | 15.1 samples/s +Step 27460/44000 | Loss: 2.9909 | LR: 1.60e-04 | 1322.7ms/step | 15.1 samples/s +Step 27470/44000 | Loss: 2.9407 | LR: 1.60e-04 | 1323.0ms/step | 15.1 samples/s +Step 27480/44000 | Loss: 2.9405 | LR: 1.59e-04 | 1323.0ms/step | 15.1 samples/s +Step 27490/44000 | Loss: 2.9412 | LR: 1.59e-04 | 1322.5ms/step | 15.1 samples/s +Step 27500/44000 | Loss: 2.9134 | LR: 1.59e-04 | 1325.8ms/step | 15.1 samples/s +Validation at step 27500: Loss = 2.9483, Perplexity = 19.07 +Step 27510/44000 | Loss: 2.9653 | LR: 1.59e-04 | 3532.0ms/step | 5.7 samples/s +Step 27520/44000 | Loss: 2.9445 | LR: 1.59e-04 | 1322.5ms/step | 15.1 samples/s +Step 27530/44000 | Loss: 2.9502 | LR: 1.59e-04 | 1321.1ms/step | 15.1 samples/s +Step 27540/44000 | Loss: 2.9479 | LR: 1.58e-04 | 1322.9ms/step | 15.1 samples/s +Step 27550/44000 | Loss: 2.9338 | LR: 1.58e-04 | 1324.0ms/step | 15.1 samples/s +Step 27560/44000 | Loss: 2.9225 | LR: 1.58e-04 | 1324.8ms/step | 15.1 samples/s +Step 27570/44000 | Loss: 2.9162 | LR: 1.58e-04 | 1322.7ms/step | 15.1 samples/s +Step 27580/44000 | Loss: 2.9716 | LR: 1.58e-04 | 1319.2ms/step | 15.2 samples/s +Step 27590/44000 | Loss: 2.9232 | LR: 1.58e-04 | 1323.7ms/step | 15.1 samples/s +Step 27600/44000 | Loss: 2.9478 | LR: 1.57e-04 | 1322.9ms/step | 15.1 samples/s +Step 27610/44000 | Loss: 2.9519 | LR: 1.57e-04 | 1325.9ms/step | 15.1 samples/s +Step 27620/44000 | Loss: 2.9453 | LR: 1.57e-04 | 1323.4ms/step | 15.1 samples/s +Step 27630/44000 | Loss: 2.9559 | LR: 1.57e-04 | 1322.6ms/step | 15.1 samples/s +Step 27640/44000 | Loss: 2.9468 | LR: 1.57e-04 | 1323.3ms/step | 15.1 samples/s +Step 27650/44000 | Loss: 2.9447 | LR: 1.57e-04 | 1321.5ms/step | 15.1 samples/s +Step 27660/44000 | Loss: 2.9293 | LR: 1.56e-04 | 1323.7ms/step | 15.1 samples/s +Step 27670/44000 | Loss: 2.9297 | LR: 1.56e-04 | 1326.4ms/step | 15.1 samples/s +Step 27680/44000 | Loss: 2.9833 | LR: 1.56e-04 | 1320.8ms/step | 15.1 samples/s +Step 27690/44000 | Loss: 2.9414 | LR: 1.56e-04 | 1322.4ms/step | 15.1 samples/s +Step 27700/44000 | Loss: 2.9218 | LR: 1.56e-04 | 1323.9ms/step | 15.1 samples/s +Step 27710/44000 | Loss: 2.9308 | LR: 1.56e-04 | 1324.5ms/step | 15.1 samples/s +Step 27720/44000 | Loss: 2.9292 | LR: 1.55e-04 | 1323.6ms/step | 15.1 samples/s +Step 27730/44000 | Loss: 2.9413 | LR: 1.55e-04 | 1323.5ms/step | 15.1 samples/s +Step 27740/44000 | Loss: 2.9514 | LR: 1.55e-04 | 1321.6ms/step | 15.1 samples/s +Step 27750/44000 | Loss: 2.9417 | LR: 1.55e-04 | 1324.8ms/step | 15.1 samples/s +Validation at step 27750: Loss = 2.9473, Perplexity = 19.05 +Step 27760/44000 | Loss: 2.9318 | LR: 1.55e-04 | 3533.0ms/step | 5.7 samples/s +Step 27770/44000 | Loss: 2.9418 | LR: 1.55e-04 | 1323.9ms/step | 15.1 samples/s +Step 27780/44000 | Loss: 2.8918 | LR: 1.54e-04 | 1323.2ms/step | 15.1 samples/s +Step 27790/44000 | Loss: 2.9421 | LR: 1.54e-04 | 1323.0ms/step | 15.1 samples/s +Step 27800/44000 | Loss: 2.9179 | LR: 1.54e-04 | 1323.0ms/step | 15.1 samples/s +Step 27810/44000 | Loss: 2.9519 | LR: 1.54e-04 | 1320.6ms/step | 15.1 samples/s +Step 27820/44000 | Loss: 2.9160 | LR: 1.54e-04 | 1323.7ms/step | 15.1 samples/s +Step 27830/44000 | Loss: 2.9191 | LR: 1.54e-04 | 1320.4ms/step | 15.1 samples/s +Step 27840/44000 | Loss: 2.9369 | LR: 1.53e-04 | 1321.2ms/step | 15.1 samples/s +Step 27850/44000 | Loss: 2.9325 | LR: 1.53e-04 | 1320.4ms/step | 15.1 samples/s +Step 27860/44000 | Loss: 2.9584 | LR: 1.53e-04 | 1324.4ms/step | 15.1 samples/s +Step 27870/44000 | Loss: 2.9379 | LR: 1.53e-04 | 1322.4ms/step | 15.1 samples/s +Step 27880/44000 | Loss: 2.9547 | LR: 1.53e-04 | 1320.9ms/step | 15.1 samples/s +Step 27890/44000 | Loss: 2.9182 | LR: 1.53e-04 | 1321.0ms/step | 15.1 samples/s +Step 27900/44000 | Loss: 2.9619 | LR: 1.52e-04 | 1323.0ms/step | 15.1 samples/s +Step 27910/44000 | Loss: 2.9335 | LR: 1.52e-04 | 1323.1ms/step | 15.1 samples/s +Step 27920/44000 | Loss: 2.9173 | LR: 1.52e-04 | 1323.9ms/step | 15.1 samples/s +Step 27930/44000 | Loss: 2.9426 | LR: 1.52e-04 | 1325.6ms/step | 15.1 samples/s +Step 27940/44000 | Loss: 2.9353 | LR: 1.52e-04 | 1324.4ms/step | 15.1 samples/s +Step 27950/44000 | Loss: 2.9582 | LR: 1.52e-04 | 1323.8ms/step | 15.1 samples/s +Step 27960/44000 | Loss: 2.9268 | LR: 1.51e-04 | 1322.9ms/step | 15.1 samples/s +Step 27970/44000 | Loss: 2.9612 | LR: 1.51e-04 | 1323.1ms/step | 15.1 samples/s +Step 27980/44000 | Loss: 2.9303 | LR: 1.51e-04 | 1324.2ms/step | 15.1 samples/s +Step 27990/44000 | Loss: 2.9183 | LR: 1.51e-04 | 1324.2ms/step | 15.1 samples/s +Step 28000/44000 | Loss: 2.9562 | LR: 1.51e-04 | 1325.5ms/step | 15.1 samples/s +Validation at step 28000: Loss = 2.9461, Perplexity = 19.03 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-28000 +Step 28010/44000 | Loss: 2.9413 | LR: 1.51e-04 | 3870.5ms/step | 5.2 samples/s +Step 28020/44000 | Loss: 2.9354 | LR: 1.50e-04 | 1320.8ms/step | 15.1 samples/s +Step 28030/44000 | Loss: 2.9349 | LR: 1.50e-04 | 1323.6ms/step | 15.1 samples/s +Step 28040/44000 | Loss: 2.9366 | LR: 1.50e-04 | 1320.5ms/step | 15.1 samples/s +Step 28050/44000 | Loss: 2.9379 | LR: 1.50e-04 | 1322.1ms/step | 15.1 samples/s +Step 28060/44000 | Loss: 2.9156 | LR: 1.50e-04 | 1324.4ms/step | 15.1 samples/s +Step 28070/44000 | Loss: 2.9395 | LR: 1.50e-04 | 1321.9ms/step | 15.1 samples/s +Step 28080/44000 | Loss: 2.9287 | LR: 1.49e-04 | 1321.9ms/step | 15.1 samples/s +Step 28090/44000 | Loss: 2.9514 | LR: 1.49e-04 | 1322.5ms/step | 15.1 samples/s +Step 28100/44000 | Loss: 2.9240 | LR: 1.49e-04 | 1321.4ms/step | 15.1 samples/s +Step 28110/44000 | Loss: 2.9608 | LR: 1.49e-04 | 1322.4ms/step | 15.1 samples/s +Step 28120/44000 | Loss: 2.9476 | LR: 1.49e-04 | 1320.5ms/step | 15.1 samples/s +Step 28130/44000 | Loss: 2.9332 | LR: 1.49e-04 | 1324.5ms/step | 15.1 samples/s +Step 28140/44000 | Loss: 2.9213 | LR: 1.48e-04 | 1322.6ms/step | 15.1 samples/s +Step 28150/44000 | Loss: 2.9093 | LR: 1.48e-04 | 1323.0ms/step | 15.1 samples/s +Step 28160/44000 | Loss: 2.9118 | LR: 1.48e-04 | 1322.6ms/step | 15.1 samples/s +Step 28170/44000 | Loss: 2.9454 | LR: 1.48e-04 | 1324.6ms/step | 15.1 samples/s +Step 28180/44000 | Loss: 2.9415 | LR: 1.48e-04 | 1320.9ms/step | 15.1 samples/s +Step 28190/44000 | Loss: 2.9474 | LR: 1.48e-04 | 1321.5ms/step | 15.1 samples/s +Step 28200/44000 | Loss: 2.9461 | LR: 1.47e-04 | 1322.1ms/step | 15.1 samples/s +Step 28210/44000 | Loss: 2.9494 | LR: 1.47e-04 | 1322.0ms/step | 15.1 samples/s +Step 28220/44000 | Loss: 2.9641 | LR: 1.47e-04 | 1322.4ms/step | 15.1 samples/s +Step 28230/44000 | Loss: 2.9325 | LR: 1.47e-04 | 1323.2ms/step | 15.1 samples/s +Step 28240/44000 | Loss: 2.9362 | LR: 1.47e-04 | 1324.1ms/step | 15.1 samples/s +Step 28250/44000 | Loss: 2.9153 | LR: 1.47e-04 | 1321.6ms/step | 15.1 samples/s +Validation at step 28250: Loss = 2.9448, Perplexity = 19.01 +Step 28260/44000 | Loss: 2.9257 | LR: 1.46e-04 | 3532.5ms/step | 5.7 samples/s +Step 28270/44000 | Loss: 2.9343 | LR: 1.46e-04 | 1322.9ms/step | 15.1 samples/s +Step 28280/44000 | Loss: 2.9405 | LR: 1.46e-04 | 1320.1ms/step | 15.2 samples/s +Step 28290/44000 | Loss: 2.9055 | LR: 1.46e-04 | 1322.8ms/step | 15.1 samples/s +Step 28300/44000 | Loss: 2.8975 | LR: 1.46e-04 | 1321.9ms/step | 15.1 samples/s +Step 28310/44000 | Loss: 2.9630 | LR: 1.46e-04 | 1323.2ms/step | 15.1 samples/s +Step 28320/44000 | Loss: 2.9649 | LR: 1.45e-04 | 1322.0ms/step | 15.1 samples/s +Step 28330/44000 | Loss: 2.8939 | LR: 1.45e-04 | 1320.5ms/step | 15.1 samples/s +Step 28340/44000 | Loss: 2.8936 | LR: 1.45e-04 | 1326.7ms/step | 15.1 samples/s +Step 28350/44000 | Loss: 2.9353 | LR: 1.45e-04 | 1323.9ms/step | 15.1 samples/s +Step 28360/44000 | Loss: 2.9407 | LR: 1.45e-04 | 1321.8ms/step | 15.1 samples/s +Step 28370/44000 | Loss: 2.9439 | LR: 1.45e-04 | 1323.9ms/step | 15.1 samples/s +Step 28380/44000 | Loss: 2.9469 | LR: 1.44e-04 | 1323.0ms/step | 15.1 samples/s +Step 28390/44000 | Loss: 2.9140 | LR: 1.44e-04 | 1324.1ms/step | 15.1 samples/s +Step 28400/44000 | Loss: 2.9520 | LR: 1.44e-04 | 1326.0ms/step | 15.1 samples/s +Step 28410/44000 | Loss: 2.9581 | LR: 1.44e-04 | 1323.9ms/step | 15.1 samples/s +Step 28420/44000 | Loss: 2.9445 | LR: 1.44e-04 | 1322.0ms/step | 15.1 samples/s +Step 28430/44000 | Loss: 2.9347 | LR: 1.44e-04 | 1323.6ms/step | 15.1 samples/s +Step 28440/44000 | Loss: 2.9511 | LR: 1.43e-04 | 1324.8ms/step | 15.1 samples/s +Step 28450/44000 | Loss: 2.9529 | LR: 1.43e-04 | 1323.5ms/step | 15.1 samples/s +Step 28460/44000 | Loss: 2.9301 | LR: 1.43e-04 | 1321.9ms/step | 15.1 samples/s +Step 28470/44000 | Loss: 2.9184 | LR: 1.43e-04 | 1322.5ms/step | 15.1 samples/s +Step 28480/44000 | Loss: 2.9176 | LR: 1.43e-04 | 1324.5ms/step | 15.1 samples/s +Step 28490/44000 | Loss: 2.9358 | LR: 1.43e-04 | 1322.5ms/step | 15.1 samples/s +Step 28500/44000 | Loss: 2.9299 | LR: 1.42e-04 | 1322.9ms/step | 15.1 samples/s +Validation at step 28500: Loss = 2.9439, Perplexity = 18.99 +Step 28510/44000 | Loss: 2.9279 | LR: 1.42e-04 | 3532.6ms/step | 5.7 samples/s +Step 28520/44000 | Loss: 2.9578 | LR: 1.42e-04 | 1318.8ms/step | 15.2 samples/s +Step 28530/44000 | Loss: 2.9371 | LR: 1.42e-04 | 1323.4ms/step | 15.1 samples/s +Step 28540/44000 | Loss: 2.9292 | LR: 1.42e-04 | 1323.8ms/step | 15.1 samples/s +Step 28550/44000 | Loss: 2.9470 | LR: 1.42e-04 | 1321.1ms/step | 15.1 samples/s +Step 28560/44000 | Loss: 2.9372 | LR: 1.41e-04 | 1322.4ms/step | 15.1 samples/s +Step 28570/44000 | Loss: 2.9479 | LR: 1.41e-04 | 1324.7ms/step | 15.1 samples/s +Step 28580/44000 | Loss: 2.9489 | LR: 1.41e-04 | 1323.9ms/step | 15.1 samples/s +Step 28590/44000 | Loss: 2.9362 | LR: 1.41e-04 | 1322.3ms/step | 15.1 samples/s +Step 28600/44000 | Loss: 2.9487 | LR: 1.41e-04 | 1322.5ms/step | 15.1 samples/s +Step 28610/44000 | Loss: 2.9609 | LR: 1.41e-04 | 1323.0ms/step | 15.1 samples/s +Step 28620/44000 | Loss: 2.9311 | LR: 1.40e-04 | 1322.6ms/step | 15.1 samples/s +Step 28630/44000 | Loss: 2.9204 | LR: 1.40e-04 | 1323.4ms/step | 15.1 samples/s +Step 28640/44000 | Loss: 2.9370 | LR: 1.40e-04 | 1322.4ms/step | 15.1 samples/s +Step 28650/44000 | Loss: 2.9652 | LR: 1.40e-04 | 1323.2ms/step | 15.1 samples/s +Step 28660/44000 | Loss: 2.9150 | LR: 1.40e-04 | 1323.5ms/step | 15.1 samples/s +Step 28670/44000 | Loss: 2.9221 | LR: 1.40e-04 | 1323.1ms/step | 15.1 samples/s +Step 28680/44000 | Loss: 2.9255 | LR: 1.39e-04 | 1326.0ms/step | 15.1 samples/s +Step 28690/44000 | Loss: 2.9603 | LR: 1.39e-04 | 1321.8ms/step | 15.1 samples/s +Step 28700/44000 | Loss: 2.9401 | LR: 1.39e-04 | 1319.6ms/step | 15.2 samples/s +Step 28710/44000 | Loss: 2.9408 | LR: 1.39e-04 | 1322.8ms/step | 15.1 samples/s +Step 28720/44000 | Loss: 2.9394 | LR: 1.39e-04 | 1323.2ms/step | 15.1 samples/s +Step 28730/44000 | Loss: 2.9298 | LR: 1.39e-04 | 1323.9ms/step | 15.1 samples/s +Step 28740/44000 | Loss: 2.9664 | LR: 1.38e-04 | 1323.9ms/step | 15.1 samples/s +Step 28750/44000 | Loss: 2.9502 | LR: 1.38e-04 | 1322.7ms/step | 15.1 samples/s +Validation at step 28750: Loss = 2.9426, Perplexity = 18.96 +Step 28760/44000 | Loss: 2.9304 | LR: 1.38e-04 | 3529.9ms/step | 5.7 samples/s +Step 28770/44000 | Loss: 2.9559 | LR: 1.38e-04 | 1323.8ms/step | 15.1 samples/s +Step 28780/44000 | Loss: 2.9153 | LR: 1.38e-04 | 1322.7ms/step | 15.1 samples/s +Step 28790/44000 | Loss: 2.9442 | LR: 1.38e-04 | 1322.9ms/step | 15.1 samples/s +Step 28800/44000 | Loss: 2.9453 | LR: 1.38e-04 | 1321.4ms/step | 15.1 samples/s +Step 28810/44000 | Loss: 2.9567 | LR: 1.37e-04 | 1323.9ms/step | 15.1 samples/s +Step 28820/44000 | Loss: 2.9300 | LR: 1.37e-04 | 1323.7ms/step | 15.1 samples/s +Step 28830/44000 | Loss: 2.9314 | LR: 1.37e-04 | 1323.9ms/step | 15.1 samples/s +Step 28840/44000 | Loss: 2.9447 | LR: 1.37e-04 | 1323.6ms/step | 15.1 samples/s +Step 28850/44000 | Loss: 2.9475 | LR: 1.37e-04 | 1322.2ms/step | 15.1 samples/s +Step 28860/44000 | Loss: 2.9294 | LR: 1.37e-04 | 1323.6ms/step | 15.1 samples/s +Step 28870/44000 | Loss: 2.9079 | LR: 1.36e-04 | 1322.3ms/step | 15.1 samples/s +Step 28880/44000 | Loss: 2.9380 | LR: 1.36e-04 | 1323.4ms/step | 15.1 samples/s +Step 28890/44000 | Loss: 2.9468 | LR: 1.36e-04 | 1323.5ms/step | 15.1 samples/s +Step 28900/44000 | Loss: 2.9311 | LR: 1.36e-04 | 1324.5ms/step | 15.1 samples/s +Step 28910/44000 | Loss: 2.9394 | LR: 1.36e-04 | 1321.3ms/step | 15.1 samples/s +Step 28920/44000 | Loss: 2.9150 | LR: 1.36e-04 | 1319.0ms/step | 15.2 samples/s +Step 28930/44000 | Loss: 2.9387 | LR: 1.35e-04 | 1320.1ms/step | 15.2 samples/s +Step 28940/44000 | Loss: 2.9400 | LR: 1.35e-04 | 1320.6ms/step | 15.1 samples/s +Step 28950/44000 | Loss: 2.9194 | LR: 1.35e-04 | 1324.6ms/step | 15.1 samples/s +Step 28960/44000 | Loss: 2.9217 | LR: 1.35e-04 | 1320.1ms/step | 15.2 samples/s +Step 28970/44000 | Loss: 2.9236 | LR: 1.35e-04 | 1325.1ms/step | 15.1 samples/s +Step 28980/44000 | Loss: 2.9505 | LR: 1.35e-04 | 1323.1ms/step | 15.1 samples/s +Step 28990/44000 | Loss: 2.9239 | LR: 1.34e-04 | 1323.4ms/step | 15.1 samples/s +Step 29000/44000 | Loss: 2.9552 | LR: 1.34e-04 | 1323.5ms/step | 15.1 samples/s +Validation at step 29000: Loss = 2.9417, Perplexity = 18.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-29000 +Step 29010/44000 | Loss: 2.9411 | LR: 1.34e-04 | 3845.1ms/step | 5.2 samples/s +Step 29020/44000 | Loss: 2.9586 | LR: 1.34e-04 | 1321.5ms/step | 15.1 samples/s +Step 29030/44000 | Loss: 2.9206 | LR: 1.34e-04 | 1321.0ms/step | 15.1 samples/s +Step 29040/44000 | Loss: 2.9098 | LR: 1.34e-04 | 1320.1ms/step | 15.2 samples/s +Step 29050/44000 | Loss: 2.9186 | LR: 1.33e-04 | 1323.3ms/step | 15.1 samples/s +Step 29060/44000 | Loss: 2.9607 | LR: 1.33e-04 | 1322.0ms/step | 15.1 samples/s +Step 29070/44000 | Loss: 2.9430 | LR: 1.33e-04 | 1321.3ms/step | 15.1 samples/s +Step 29080/44000 | Loss: 2.9234 | LR: 1.33e-04 | 1323.8ms/step | 15.1 samples/s +Step 29090/44000 | Loss: 2.9389 | LR: 1.33e-04 | 1325.3ms/step | 15.1 samples/s +Step 29100/44000 | Loss: 2.9303 | LR: 1.33e-04 | 1323.4ms/step | 15.1 samples/s +Step 29110/44000 | Loss: 2.9300 | LR: 1.33e-04 | 1324.6ms/step | 15.1 samples/s +Step 29120/44000 | Loss: 2.9395 | LR: 1.32e-04 | 1323.2ms/step | 15.1 samples/s +Step 29130/44000 | Loss: 2.9352 | LR: 1.32e-04 | 1321.4ms/step | 15.1 samples/s +Step 29140/44000 | Loss: 2.9249 | LR: 1.32e-04 | 1324.7ms/step | 15.1 samples/s +Step 29150/44000 | Loss: 2.9410 | LR: 1.32e-04 | 1321.3ms/step | 15.1 samples/s +Step 29160/44000 | Loss: 2.9153 | LR: 1.32e-04 | 1322.8ms/step | 15.1 samples/s +Step 29170/44000 | Loss: 2.9163 | LR: 1.32e-04 | 1323.4ms/step | 15.1 samples/s +Step 29180/44000 | Loss: 2.9236 | LR: 1.31e-04 | 1324.5ms/step | 15.1 samples/s +Step 29190/44000 | Loss: 2.9325 | LR: 1.31e-04 | 1322.2ms/step | 15.1 samples/s +Step 29200/44000 | Loss: 2.9258 | LR: 1.31e-04 | 1322.7ms/step | 15.1 samples/s +Step 29210/44000 | Loss: 2.9198 | LR: 1.31e-04 | 1325.3ms/step | 15.1 samples/s +Step 29220/44000 | Loss: 2.9221 | LR: 1.31e-04 | 1322.9ms/step | 15.1 samples/s +Step 29230/44000 | Loss: 2.9340 | LR: 1.31e-04 | 1325.7ms/step | 15.1 samples/s +Step 29240/44000 | Loss: 2.9456 | LR: 1.30e-04 | 1323.2ms/step | 15.1 samples/s +Step 29250/44000 | Loss: 2.9386 | LR: 1.30e-04 | 1321.9ms/step | 15.1 samples/s +Validation at step 29250: Loss = 2.9404, Perplexity = 18.92 +Step 29260/44000 | Loss: 2.9227 | LR: 1.30e-04 | 3536.2ms/step | 5.7 samples/s +Step 29270/44000 | Loss: 2.9500 | LR: 1.30e-04 | 1322.6ms/step | 15.1 samples/s +Step 29280/44000 | Loss: 2.9473 | LR: 1.30e-04 | 1324.3ms/step | 15.1 samples/s +Step 29290/44000 | Loss: 2.9389 | LR: 1.30e-04 | 1324.6ms/step | 15.1 samples/s +Step 29300/44000 | Loss: 2.9232 | LR: 1.29e-04 | 1324.5ms/step | 15.1 samples/s +Step 29310/44000 | Loss: 2.9304 | LR: 1.29e-04 | 1322.6ms/step | 15.1 samples/s +Step 29320/44000 | Loss: 2.9335 | LR: 1.29e-04 | 1320.0ms/step | 15.2 samples/s +Step 29330/44000 | Loss: 2.9138 | LR: 1.29e-04 | 1323.3ms/step | 15.1 samples/s +Step 29340/44000 | Loss: 2.9433 | LR: 1.29e-04 | 1324.0ms/step | 15.1 samples/s +Step 29350/44000 | Loss: 2.9563 | LR: 1.29e-04 | 1323.9ms/step | 15.1 samples/s +Step 29360/44000 | Loss: 2.9194 | LR: 1.29e-04 | 1324.6ms/step | 15.1 samples/s +Step 29370/44000 | Loss: 2.9642 | LR: 1.28e-04 | 1323.5ms/step | 15.1 samples/s +Step 29380/44000 | Loss: 2.9152 | LR: 1.28e-04 | 1324.5ms/step | 15.1 samples/s +Step 29390/44000 | Loss: 2.9612 | LR: 1.28e-04 | 1321.9ms/step | 15.1 samples/s +Step 29400/44000 | Loss: 2.9562 | LR: 1.28e-04 | 1322.8ms/step | 15.1 samples/s +Step 29410/44000 | Loss: 2.9359 | LR: 1.28e-04 | 1322.5ms/step | 15.1 samples/s +Step 29420/44000 | Loss: 2.9180 | LR: 1.28e-04 | 1323.4ms/step | 15.1 samples/s +Step 29430/44000 | Loss: 2.9566 | LR: 1.27e-04 | 1324.0ms/step | 15.1 samples/s +Step 29440/44000 | Loss: 2.9179 | LR: 1.27e-04 | 1325.5ms/step | 15.1 samples/s +Step 29450/44000 | Loss: 2.9125 | LR: 1.27e-04 | 1322.9ms/step | 15.1 samples/s +Step 29460/44000 | Loss: 2.9194 | LR: 1.27e-04 | 1322.9ms/step | 15.1 samples/s +Step 29470/44000 | Loss: 2.9399 | LR: 1.27e-04 | 1320.9ms/step | 15.1 samples/s +Step 29480/44000 | Loss: 2.9224 | LR: 1.27e-04 | 1324.3ms/step | 15.1 samples/s +Step 29490/44000 | Loss: 2.9393 | LR: 1.26e-04 | 1323.9ms/step | 15.1 samples/s +Step 29500/44000 | Loss: 2.9431 | LR: 1.26e-04 | 1324.2ms/step | 15.1 samples/s +Validation at step 29500: Loss = 2.9397, Perplexity = 18.91 +Step 29510/44000 | Loss: 2.9181 | LR: 1.26e-04 | 3538.7ms/step | 5.7 samples/s +Step 29520/44000 | Loss: 2.9249 | LR: 1.26e-04 | 1321.8ms/step | 15.1 samples/s +Step 29530/44000 | Loss: 2.9160 | LR: 1.26e-04 | 1321.6ms/step | 15.1 samples/s +Step 29540/44000 | Loss: 2.9620 | LR: 1.26e-04 | 1322.8ms/step | 15.1 samples/s +Step 29550/44000 | Loss: 2.9368 | LR: 1.26e-04 | 1324.9ms/step | 15.1 samples/s +Step 29560/44000 | Loss: 2.9182 | LR: 1.25e-04 | 1322.7ms/step | 15.1 samples/s +Step 29570/44000 | Loss: 2.9410 | LR: 1.25e-04 | 1320.0ms/step | 15.2 samples/s +Step 29580/44000 | Loss: 2.9121 | LR: 1.25e-04 | 1319.4ms/step | 15.2 samples/s +Step 29590/44000 | Loss: 2.9423 | LR: 1.25e-04 | 1320.3ms/step | 15.1 samples/s +Step 29600/44000 | Loss: 2.9235 | LR: 1.25e-04 | 1323.1ms/step | 15.1 samples/s +Step 29610/44000 | Loss: 2.9280 | LR: 1.25e-04 | 1322.6ms/step | 15.1 samples/s +Step 29620/44000 | Loss: 2.9159 | LR: 1.24e-04 | 1323.9ms/step | 15.1 samples/s +Step 29630/44000 | Loss: 2.9183 | LR: 1.24e-04 | 1323.4ms/step | 15.1 samples/s +Step 29640/44000 | Loss: 2.9354 | LR: 1.24e-04 | 1321.8ms/step | 15.1 samples/s +Step 29650/44000 | Loss: 2.9550 | LR: 1.24e-04 | 1323.2ms/step | 15.1 samples/s +Step 29660/44000 | Loss: 2.9336 | LR: 1.24e-04 | 1323.2ms/step | 15.1 samples/s +Step 29670/44000 | Loss: 2.9357 | LR: 1.24e-04 | 1323.8ms/step | 15.1 samples/s +Step 29680/44000 | Loss: 2.9525 | LR: 1.23e-04 | 1325.2ms/step | 15.1 samples/s +Step 29690/44000 | Loss: 2.9273 | LR: 1.23e-04 | 1322.8ms/step | 15.1 samples/s +Step 29700/44000 | Loss: 2.9408 | LR: 1.23e-04 | 1322.3ms/step | 15.1 samples/s +Step 29710/44000 | Loss: 2.9442 | LR: 1.23e-04 | 1321.8ms/step | 15.1 samples/s +Step 29720/44000 | Loss: 2.9493 | LR: 1.23e-04 | 1326.3ms/step | 15.1 samples/s +Step 29730/44000 | Loss: 2.9361 | LR: 1.23e-04 | 1324.0ms/step | 15.1 samples/s +Step 29740/44000 | Loss: 2.9232 | LR: 1.23e-04 | 1323.5ms/step | 15.1 samples/s +Step 29750/44000 | Loss: 2.9492 | LR: 1.22e-04 | 1322.4ms/step | 15.1 samples/s +Validation at step 29750: Loss = 2.9385, Perplexity = 18.89 +Step 29760/44000 | Loss: 2.9324 | LR: 1.22e-04 | 3532.4ms/step | 5.7 samples/s +Step 29770/44000 | Loss: 2.8975 | LR: 1.22e-04 | 1326.4ms/step | 15.1 samples/s +Step 29780/44000 | Loss: 2.9758 | LR: 1.22e-04 | 1323.9ms/step | 15.1 samples/s +Step 29790/44000 | Loss: 2.9194 | LR: 1.22e-04 | 1322.5ms/step | 15.1 samples/s +Step 29800/44000 | Loss: 2.9104 | LR: 1.22e-04 | 1324.2ms/step | 15.1 samples/s +Step 29810/44000 | Loss: 2.9151 | LR: 1.21e-04 | 1323.7ms/step | 15.1 samples/s +Step 29820/44000 | Loss: 2.9240 | LR: 1.21e-04 | 1323.3ms/step | 15.1 samples/s +Step 29830/44000 | Loss: 2.9437 | LR: 1.21e-04 | 1325.5ms/step | 15.1 samples/s +Step 29840/44000 | Loss: 2.9108 | LR: 1.21e-04 | 1324.1ms/step | 15.1 samples/s +Step 29850/44000 | Loss: 2.9138 | LR: 1.21e-04 | 1323.9ms/step | 15.1 samples/s +Step 29860/44000 | Loss: 2.9430 | LR: 1.21e-04 | 1324.6ms/step | 15.1 samples/s +Step 29870/44000 | Loss: 2.9325 | LR: 1.21e-04 | 1325.0ms/step | 15.1 samples/s +Step 29880/44000 | Loss: 2.9358 | LR: 1.20e-04 | 1323.5ms/step | 15.1 samples/s +Step 29890/44000 | Loss: 2.8857 | LR: 1.20e-04 | 1323.1ms/step | 15.1 samples/s +Step 29900/44000 | Loss: 2.9053 | LR: 1.20e-04 | 1321.7ms/step | 15.1 samples/s +Step 29910/44000 | Loss: 2.9339 | LR: 1.20e-04 | 1322.5ms/step | 15.1 samples/s +Step 29920/44000 | Loss: 2.9196 | LR: 1.20e-04 | 1320.2ms/step | 15.1 samples/s +Step 29930/44000 | Loss: 2.9394 | LR: 1.20e-04 | 1321.9ms/step | 15.1 samples/s +Step 29940/44000 | Loss: 2.9107 | LR: 1.19e-04 | 1318.8ms/step | 15.2 samples/s +Step 29950/44000 | Loss: 2.9241 | LR: 1.19e-04 | 1323.7ms/step | 15.1 samples/s +Step 29960/44000 | Loss: 2.9127 | LR: 1.19e-04 | 1325.1ms/step | 15.1 samples/s +Step 29970/44000 | Loss: 2.9485 | LR: 1.19e-04 | 1323.3ms/step | 15.1 samples/s +Step 29980/44000 | Loss: 2.9199 | LR: 1.19e-04 | 1323.7ms/step | 15.1 samples/s +Step 29990/44000 | Loss: 2.9342 | LR: 1.19e-04 | 1321.9ms/step | 15.1 samples/s +Step 30000/44000 | Loss: 2.9013 | LR: 1.19e-04 | 1322.1ms/step | 15.1 samples/s +Validation at step 30000: Loss = 2.9376, Perplexity = 18.87 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-30000 +Step 30010/44000 | Loss: 2.9270 | LR: 1.18e-04 | 3856.0ms/step | 5.2 samples/s +Step 30020/44000 | Loss: 2.9232 | LR: 1.18e-04 | 1322.8ms/step | 15.1 samples/s +Step 30030/44000 | Loss: 2.9051 | LR: 1.18e-04 | 1321.2ms/step | 15.1 samples/s +Step 30040/44000 | Loss: 2.9354 | LR: 1.18e-04 | 1319.2ms/step | 15.2 samples/s +Step 30050/44000 | Loss: 2.9427 | LR: 1.18e-04 | 1324.6ms/step | 15.1 samples/s +Step 30060/44000 | Loss: 2.9266 | LR: 1.18e-04 | 1324.9ms/step | 15.1 samples/s +Step 30070/44000 | Loss: 2.9175 | LR: 1.17e-04 | 1323.1ms/step | 15.1 samples/s +Step 30080/44000 | Loss: 2.9090 | LR: 1.17e-04 | 1322.2ms/step | 15.1 samples/s +Step 30090/44000 | Loss: 2.9481 | LR: 1.17e-04 | 1323.6ms/step | 15.1 samples/s +Step 30100/44000 | Loss: 2.9071 | LR: 1.17e-04 | 1324.8ms/step | 15.1 samples/s +Step 30110/44000 | Loss: 2.9380 | LR: 1.17e-04 | 1322.9ms/step | 15.1 samples/s +Step 30120/44000 | Loss: 2.9349 | LR: 1.17e-04 | 1324.0ms/step | 15.1 samples/s +Step 30130/44000 | Loss: 2.9447 | LR: 1.17e-04 | 1320.9ms/step | 15.1 samples/s +Step 30140/44000 | Loss: 2.9215 | LR: 1.16e-04 | 1322.4ms/step | 15.1 samples/s +Step 30150/44000 | Loss: 2.9329 | LR: 1.16e-04 | 1323.0ms/step | 15.1 samples/s +Step 30160/44000 | Loss: 2.9378 | LR: 1.16e-04 | 1322.1ms/step | 15.1 samples/s +Step 30170/44000 | Loss: 2.9417 | LR: 1.16e-04 | 1323.6ms/step | 15.1 samples/s +Step 30180/44000 | Loss: 2.9046 | LR: 1.16e-04 | 1324.8ms/step | 15.1 samples/s +Step 30190/44000 | Loss: 2.9387 | LR: 1.16e-04 | 1323.9ms/step | 15.1 samples/s +Step 30200/44000 | Loss: 2.9379 | LR: 1.15e-04 | 1320.4ms/step | 15.1 samples/s +Step 30210/44000 | Loss: 2.9404 | LR: 1.15e-04 | 1321.9ms/step | 15.1 samples/s +Step 30220/44000 | Loss: 2.9341 | LR: 1.15e-04 | 1322.6ms/step | 15.1 samples/s +Step 30230/44000 | Loss: 2.9259 | LR: 1.15e-04 | 1323.2ms/step | 15.1 samples/s +Step 30240/44000 | Loss: 2.9322 | LR: 1.15e-04 | 1320.2ms/step | 15.1 samples/s +Step 30250/44000 | Loss: 2.9432 | LR: 1.15e-04 | 1323.1ms/step | 15.1 samples/s +Validation at step 30250: Loss = 2.9371, Perplexity = 18.86 +Step 30260/44000 | Loss: 2.9290 | LR: 1.15e-04 | 3532.8ms/step | 5.7 samples/s +Step 30270/44000 | Loss: 2.9005 | LR: 1.14e-04 | 1322.7ms/step | 15.1 samples/s +Step 30280/44000 | Loss: 2.9192 | LR: 1.14e-04 | 1320.0ms/step | 15.2 samples/s +Step 30290/44000 | Loss: 2.9589 | LR: 1.14e-04 | 1318.7ms/step | 15.2 samples/s +Step 30300/44000 | Loss: 2.9363 | LR: 1.14e-04 | 1319.6ms/step | 15.2 samples/s +Step 30310/44000 | Loss: 2.9196 | LR: 1.14e-04 | 1324.6ms/step | 15.1 samples/s +Step 30320/44000 | Loss: 2.8929 | LR: 1.14e-04 | 1325.6ms/step | 15.1 samples/s +Step 30330/44000 | Loss: 2.9231 | LR: 1.13e-04 | 1321.6ms/step | 15.1 samples/s +Step 30340/44000 | Loss: 2.9140 | LR: 1.13e-04 | 1320.0ms/step | 15.2 samples/s +Step 30350/44000 | Loss: 2.8826 | LR: 1.13e-04 | 1321.6ms/step | 15.1 samples/s +Step 30360/44000 | Loss: 2.9267 | LR: 1.13e-04 | 1325.5ms/step | 15.1 samples/s +Step 30370/44000 | Loss: 2.9317 | LR: 1.13e-04 | 1323.1ms/step | 15.1 samples/s +Step 30380/44000 | Loss: 2.9214 | LR: 1.13e-04 | 1320.3ms/step | 15.1 samples/s +Step 30390/44000 | Loss: 2.9183 | LR: 1.13e-04 | 1320.2ms/step | 15.1 samples/s +Step 30400/44000 | Loss: 2.9339 | LR: 1.12e-04 | 1319.8ms/step | 15.2 samples/s +Step 30410/44000 | Loss: 2.9010 | LR: 1.12e-04 | 1324.7ms/step | 15.1 samples/s +Step 30420/44000 | Loss: 2.9567 | LR: 1.12e-04 | 1322.2ms/step | 15.1 samples/s +Step 30430/44000 | Loss: 2.9297 | LR: 1.12e-04 | 1320.9ms/step | 15.1 samples/s +Step 30440/44000 | Loss: 2.9091 | LR: 1.12e-04 | 1320.5ms/step | 15.1 samples/s +Step 30450/44000 | Loss: 2.9126 | LR: 1.12e-04 | 1320.8ms/step | 15.1 samples/s +Step 30460/44000 | Loss: 2.8861 | LR: 1.11e-04 | 1319.2ms/step | 15.2 samples/s +Step 30470/44000 | Loss: 2.9233 | LR: 1.11e-04 | 1321.7ms/step | 15.1 samples/s +Step 30480/44000 | Loss: 2.9175 | LR: 1.11e-04 | 1319.8ms/step | 15.2 samples/s +Step 30490/44000 | Loss: 2.9220 | LR: 1.11e-04 | 1322.5ms/step | 15.1 samples/s +Step 30500/44000 | Loss: 2.9273 | LR: 1.11e-04 | 1321.7ms/step | 15.1 samples/s +Validation at step 30500: Loss = 2.9359, Perplexity = 18.84 +Step 30510/44000 | Loss: 2.9424 | LR: 1.11e-04 | 3534.2ms/step | 5.7 samples/s +Step 30520/44000 | Loss: 2.9245 | LR: 1.11e-04 | 1323.4ms/step | 15.1 samples/s +Step 30530/44000 | Loss: 2.8991 | LR: 1.10e-04 | 1325.0ms/step | 15.1 samples/s +Step 30540/44000 | Loss: 2.9556 | LR: 1.10e-04 | 1325.1ms/step | 15.1 samples/s +Step 30550/44000 | Loss: 2.8827 | LR: 1.10e-04 | 1320.9ms/step | 15.1 samples/s +Step 30560/44000 | Loss: 2.9482 | LR: 1.10e-04 | 1322.9ms/step | 15.1 samples/s +Step 30570/44000 | Loss: 2.9419 | LR: 1.10e-04 | 1324.3ms/step | 15.1 samples/s +Step 30580/44000 | Loss: 2.9377 | LR: 1.10e-04 | 1323.3ms/step | 15.1 samples/s +Step 30590/44000 | Loss: 2.9060 | LR: 1.10e-04 | 1320.0ms/step | 15.2 samples/s +Step 30600/44000 | Loss: 2.9382 | LR: 1.09e-04 | 1319.0ms/step | 15.2 samples/s +Step 30610/44000 | Loss: 2.9340 | LR: 1.09e-04 | 1323.7ms/step | 15.1 samples/s +Step 30620/44000 | Loss: 2.9307 | LR: 1.09e-04 | 1323.2ms/step | 15.1 samples/s +Step 30630/44000 | Loss: 2.9348 | LR: 1.09e-04 | 1323.3ms/step | 15.1 samples/s +Step 30640/44000 | Loss: 2.9334 | LR: 1.09e-04 | 1323.5ms/step | 15.1 samples/s +Step 30650/44000 | Loss: 2.9127 | LR: 1.09e-04 | 1324.0ms/step | 15.1 samples/s +Step 30660/44000 | Loss: 2.9334 | LR: 1.08e-04 | 1323.4ms/step | 15.1 samples/s +Step 30670/44000 | Loss: 2.9522 | LR: 1.08e-04 | 1322.3ms/step | 15.1 samples/s +Step 30680/44000 | Loss: 2.9353 | LR: 1.08e-04 | 1321.9ms/step | 15.1 samples/s +Step 30690/44000 | Loss: 2.8978 | LR: 1.08e-04 | 1326.0ms/step | 15.1 samples/s +Step 30700/44000 | Loss: 2.9171 | LR: 1.08e-04 | 1321.9ms/step | 15.1 samples/s +Step 30710/44000 | Loss: 2.9371 | LR: 1.08e-04 | 1319.4ms/step | 15.2 samples/s +Step 30720/44000 | Loss: 2.9682 | LR: 1.08e-04 | 1321.6ms/step | 15.1 samples/s +Step 30730/44000 | Loss: 2.9226 | LR: 1.07e-04 | 1322.4ms/step | 15.1 samples/s +Step 30740/44000 | Loss: 2.9046 | LR: 1.07e-04 | 1324.0ms/step | 15.1 samples/s +Step 30750/44000 | Loss: 2.9336 | LR: 1.07e-04 | 1323.0ms/step | 15.1 samples/s +Validation at step 30750: Loss = 2.9351, Perplexity = 18.82 +Step 30760/44000 | Loss: 2.9549 | LR: 1.07e-04 | 3534.4ms/step | 5.7 samples/s +Step 30770/44000 | Loss: 2.9358 | LR: 1.07e-04 | 1322.5ms/step | 15.1 samples/s +Step 30780/44000 | Loss: 2.9310 | LR: 1.07e-04 | 1321.9ms/step | 15.1 samples/s +Step 30790/44000 | Loss: 2.9606 | LR: 1.07e-04 | 1324.0ms/step | 15.1 samples/s +Step 30800/44000 | Loss: 2.9083 | LR: 1.06e-04 | 1325.1ms/step | 15.1 samples/s +Step 30810/44000 | Loss: 2.9391 | LR: 1.06e-04 | 1321.6ms/step | 15.1 samples/s +Step 30820/44000 | Loss: 2.9372 | LR: 1.06e-04 | 1322.8ms/step | 15.1 samples/s +Step 30830/44000 | Loss: 2.9330 | LR: 1.06e-04 | 1322.1ms/step | 15.1 samples/s +Step 30840/44000 | Loss: 2.9020 | LR: 1.06e-04 | 1321.1ms/step | 15.1 samples/s +Step 30850/44000 | Loss: 2.9168 | LR: 1.06e-04 | 1321.7ms/step | 15.1 samples/s +Step 30860/44000 | Loss: 2.9211 | LR: 1.05e-04 | 1322.5ms/step | 15.1 samples/s +Step 30870/44000 | Loss: 2.8974 | LR: 1.05e-04 | 1323.3ms/step | 15.1 samples/s +Step 30880/44000 | Loss: 2.9115 | LR: 1.05e-04 | 1323.6ms/step | 15.1 samples/s +Step 30890/44000 | Loss: 2.9108 | LR: 1.05e-04 | 1323.3ms/step | 15.1 samples/s +Step 30900/44000 | Loss: 2.9228 | LR: 1.05e-04 | 1322.5ms/step | 15.1 samples/s +Step 30910/44000 | Loss: 2.9505 | LR: 1.05e-04 | 1324.3ms/step | 15.1 samples/s +Step 30920/44000 | Loss: 2.9320 | LR: 1.05e-04 | 1325.2ms/step | 15.1 samples/s +Step 30930/44000 | Loss: 2.9130 | LR: 1.04e-04 | 1323.4ms/step | 15.1 samples/s +Step 30940/44000 | Loss: 2.9479 | LR: 1.04e-04 | 1346.2ms/step | 14.9 samples/s +Step 30950/44000 | Loss: 2.9105 | LR: 1.04e-04 | 1333.9ms/step | 15.0 samples/s +Step 30960/44000 | Loss: 2.9308 | LR: 1.04e-04 | 1319.8ms/step | 15.2 samples/s +Step 30970/44000 | Loss: 2.9265 | LR: 1.04e-04 | 1322.1ms/step | 15.1 samples/s +Step 30980/44000 | Loss: 2.9550 | LR: 1.04e-04 | 1324.7ms/step | 15.1 samples/s +Step 30990/44000 | Loss: 2.9371 | LR: 1.04e-04 | 1322.1ms/step | 15.1 samples/s +Step 31000/44000 | Loss: 2.9193 | LR: 1.03e-04 | 1319.2ms/step | 15.2 samples/s +Validation at step 31000: Loss = 2.9344, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-31000 +Step 31010/44000 | Loss: 2.8985 | LR: 1.03e-04 | 3857.5ms/step | 5.2 samples/s +Step 31020/44000 | Loss: 2.9135 | LR: 1.03e-04 | 1318.6ms/step | 15.2 samples/s +Step 31030/44000 | Loss: 2.9136 | LR: 1.03e-04 | 1321.0ms/step | 15.1 samples/s +Step 31040/44000 | Loss: 2.8796 | LR: 1.03e-04 | 1320.2ms/step | 15.1 samples/s +Step 31050/44000 | Loss: 2.9205 | LR: 1.03e-04 | 1319.5ms/step | 15.2 samples/s +Step 31060/44000 | Loss: 2.9521 | LR: 1.03e-04 | 1318.9ms/step | 15.2 samples/s +Step 31070/44000 | Loss: 2.9313 | LR: 1.02e-04 | 1322.0ms/step | 15.1 samples/s +Step 31080/44000 | Loss: 2.9300 | LR: 1.02e-04 | 1323.8ms/step | 15.1 samples/s +Step 31090/44000 | Loss: 2.9472 | LR: 1.02e-04 | 1323.7ms/step | 15.1 samples/s +Step 31100/44000 | Loss: 2.9279 | LR: 1.02e-04 | 1323.1ms/step | 15.1 samples/s +Step 31110/44000 | Loss: 2.9076 | LR: 1.02e-04 | 1322.7ms/step | 15.1 samples/s +Step 31120/44000 | Loss: 2.9462 | LR: 1.02e-04 | 1323.6ms/step | 15.1 samples/s +Step 31130/44000 | Loss: 2.9040 | LR: 1.02e-04 | 1322.3ms/step | 15.1 samples/s +Step 31140/44000 | Loss: 2.9627 | LR: 1.01e-04 | 1321.6ms/step | 15.1 samples/s +Step 31150/44000 | Loss: 2.9319 | LR: 1.01e-04 | 1324.3ms/step | 15.1 samples/s +Step 31160/44000 | Loss: 2.9032 | LR: 1.01e-04 | 1324.3ms/step | 15.1 samples/s +Step 31170/44000 | Loss: 2.9196 | LR: 1.01e-04 | 1323.9ms/step | 15.1 samples/s +Step 31180/44000 | Loss: 2.9443 | LR: 1.01e-04 | 1322.2ms/step | 15.1 samples/s +Step 31190/44000 | Loss: 2.9306 | LR: 1.01e-04 | 1320.6ms/step | 15.1 samples/s +Step 31200/44000 | Loss: 2.9277 | LR: 1.00e-04 | 1324.0ms/step | 15.1 samples/s +Step 31210/44000 | Loss: 2.9184 | LR: 1.00e-04 | 1324.3ms/step | 15.1 samples/s +Step 31220/44000 | Loss: 2.9333 | LR: 1.00e-04 | 1324.0ms/step | 15.1 samples/s +Step 31230/44000 | Loss: 2.9391 | LR: 1.00e-04 | 1324.8ms/step | 15.1 samples/s +Step 31240/44000 | Loss: 2.9086 | LR: 9.99e-05 | 1324.3ms/step | 15.1 samples/s +Step 31250/44000 | Loss: 2.9029 | LR: 9.98e-05 | 1321.2ms/step | 15.1 samples/s +Validation at step 31250: Loss = 2.9338, Perplexity = 18.80 +Step 31260/44000 | Loss: 2.9494 | LR: 9.96e-05 | 3534.5ms/step | 5.7 samples/s +Step 31270/44000 | Loss: 2.9259 | LR: 9.95e-05 | 1322.6ms/step | 15.1 samples/s +Step 31280/44000 | Loss: 2.9188 | LR: 9.93e-05 | 1324.0ms/step | 15.1 samples/s +Step 31290/44000 | Loss: 2.9365 | LR: 9.92e-05 | 1324.1ms/step | 15.1 samples/s +Step 31300/44000 | Loss: 2.9410 | LR: 9.90e-05 | 1324.6ms/step | 15.1 samples/s +Step 31310/44000 | Loss: 2.9281 | LR: 9.89e-05 | 1323.3ms/step | 15.1 samples/s +Step 31320/44000 | Loss: 2.9199 | LR: 9.88e-05 | 1322.3ms/step | 15.1 samples/s +Step 31330/44000 | Loss: 2.9322 | LR: 9.86e-05 | 1320.9ms/step | 15.1 samples/s +Step 31340/44000 | Loss: 2.9354 | LR: 9.85e-05 | 1317.6ms/step | 15.2 samples/s +Step 31350/44000 | Loss: 2.9394 | LR: 9.83e-05 | 1319.9ms/step | 15.2 samples/s +Step 31360/44000 | Loss: 2.9469 | LR: 9.82e-05 | 1322.0ms/step | 15.1 samples/s +Step 31370/44000 | Loss: 2.9006 | LR: 9.80e-05 | 1322.8ms/step | 15.1 samples/s +Step 31380/44000 | Loss: 2.9144 | LR: 9.79e-05 | 1322.1ms/step | 15.1 samples/s +Step 31390/44000 | Loss: 2.9155 | LR: 9.77e-05 | 1319.6ms/step | 15.2 samples/s +Step 31400/44000 | Loss: 2.9224 | LR: 9.76e-05 | 1324.9ms/step | 15.1 samples/s +Step 31410/44000 | Loss: 2.9399 | LR: 9.75e-05 | 1325.0ms/step | 15.1 samples/s +Step 31420/44000 | Loss: 2.9180 | LR: 9.73e-05 | 1322.1ms/step | 15.1 samples/s +Step 31430/44000 | Loss: 2.9148 | LR: 9.72e-05 | 1324.6ms/step | 15.1 samples/s +Step 31440/44000 | Loss: 2.9275 | LR: 9.70e-05 | 1323.5ms/step | 15.1 samples/s +Step 31450/44000 | Loss: 2.9392 | LR: 9.69e-05 | 1323.5ms/step | 15.1 samples/s +Step 31460/44000 | Loss: 2.8835 | LR: 9.67e-05 | 1323.6ms/step | 15.1 samples/s +Step 31470/44000 | Loss: 2.8850 | LR: 9.66e-05 | 1323.6ms/step | 15.1 samples/s +Step 31480/44000 | Loss: 2.9253 | LR: 9.65e-05 | 1321.6ms/step | 15.1 samples/s +Step 31490/44000 | Loss: 2.9128 | LR: 9.63e-05 | 1322.3ms/step | 15.1 samples/s +Step 31500/44000 | Loss: 2.9289 | LR: 9.62e-05 | 1323.4ms/step | 15.1 samples/s +Validation at step 31500: Loss = 2.9333, Perplexity = 18.79 +Step 31510/44000 | Loss: 2.9084 | LR: 9.60e-05 | 3533.0ms/step | 5.7 samples/s +Step 31520/44000 | Loss: 2.9258 | LR: 9.59e-05 | 1322.6ms/step | 15.1 samples/s +Step 31530/44000 | Loss: 2.9065 | LR: 9.57e-05 | 1323.6ms/step | 15.1 samples/s +Step 31540/44000 | Loss: 2.9162 | LR: 9.56e-05 | 1322.8ms/step | 15.1 samples/s +Step 31550/44000 | Loss: 2.9376 | LR: 9.54e-05 | 1323.9ms/step | 15.1 samples/s +Step 31560/44000 | Loss: 2.9329 | LR: 9.53e-05 | 1324.6ms/step | 15.1 samples/s +Step 31570/44000 | Loss: 2.9185 | LR: 9.52e-05 | 1323.4ms/step | 15.1 samples/s +Step 31580/44000 | Loss: 2.9125 | LR: 9.50e-05 | 1323.4ms/step | 15.1 samples/s +Step 31590/44000 | Loss: 2.9170 | LR: 9.49e-05 | 1324.0ms/step | 15.1 samples/s +Step 31600/44000 | Loss: 2.9034 | LR: 9.47e-05 | 1323.2ms/step | 15.1 samples/s +Step 31610/44000 | Loss: 2.9115 | LR: 9.46e-05 | 1321.2ms/step | 15.1 samples/s +Step 31620/44000 | Loss: 2.9302 | LR: 9.45e-05 | 1318.7ms/step | 15.2 samples/s +Step 31630/44000 | Loss: 2.9390 | LR: 9.43e-05 | 1319.2ms/step | 15.2 samples/s +Step 31640/44000 | Loss: 2.9026 | LR: 9.42e-05 | 1320.2ms/step | 15.1 samples/s +Step 31650/44000 | Loss: 2.9389 | LR: 9.40e-05 | 1320.3ms/step | 15.1 samples/s +Step 31660/44000 | Loss: 2.9336 | LR: 9.39e-05 | 1321.1ms/step | 15.1 samples/s +Step 31670/44000 | Loss: 2.9543 | LR: 9.37e-05 | 1320.8ms/step | 15.1 samples/s +Step 31680/44000 | Loss: 2.9067 | LR: 9.36e-05 | 1319.8ms/step | 15.2 samples/s +Step 31690/44000 | Loss: 2.9221 | LR: 9.35e-05 | 1322.7ms/step | 15.1 samples/s +Step 31700/44000 | Loss: 2.9348 | LR: 9.33e-05 | 1323.9ms/step | 15.1 samples/s +Step 31710/44000 | Loss: 2.9111 | LR: 9.32e-05 | 1323.9ms/step | 15.1 samples/s +Step 31720/44000 | Loss: 2.9413 | LR: 9.30e-05 | 1323.2ms/step | 15.1 samples/s +Step 31730/44000 | Loss: 2.9375 | LR: 9.29e-05 | 1324.5ms/step | 15.1 samples/s +Step 31740/44000 | Loss: 2.9408 | LR: 9.28e-05 | 1320.0ms/step | 15.2 samples/s +Step 31750/44000 | Loss: 2.9158 | LR: 9.26e-05 | 1320.1ms/step | 15.1 samples/s +Validation at step 31750: Loss = 2.9326, Perplexity = 18.78 +Step 31760/44000 | Loss: 2.9145 | LR: 9.25e-05 | 3525.1ms/step | 5.7 samples/s +Step 31770/44000 | Loss: 2.9203 | LR: 9.23e-05 | 1318.1ms/step | 15.2 samples/s +Step 31780/44000 | Loss: 2.9207 | LR: 9.22e-05 | 1316.4ms/step | 15.2 samples/s +Step 31790/44000 | Loss: 2.8960 | LR: 9.20e-05 | 1316.6ms/step | 15.2 samples/s +Step 31800/44000 | Loss: 2.9323 | LR: 9.19e-05 | 1321.3ms/step | 15.1 samples/s +Step 31810/44000 | Loss: 2.9408 | LR: 9.18e-05 | 1322.6ms/step | 15.1 samples/s +Step 31820/44000 | Loss: 2.9221 | LR: 9.16e-05 | 1319.7ms/step | 15.2 samples/s +Step 31830/44000 | Loss: 2.9227 | LR: 9.15e-05 | 1320.1ms/step | 15.2 samples/s +Step 31840/44000 | Loss: 2.9426 | LR: 9.13e-05 | 1320.3ms/step | 15.1 samples/s +Step 31850/44000 | Loss: 2.9288 | LR: 9.12e-05 | 1321.8ms/step | 15.1 samples/s +Step 31860/44000 | Loss: 2.9552 | LR: 9.11e-05 | 1323.4ms/step | 15.1 samples/s +Step 31870/44000 | Loss: 2.9394 | LR: 9.09e-05 | 1323.3ms/step | 15.1 samples/s +Step 31880/44000 | Loss: 2.9138 | LR: 9.08e-05 | 1321.1ms/step | 15.1 samples/s +Step 31890/44000 | Loss: 2.9271 | LR: 9.06e-05 | 1319.8ms/step | 15.2 samples/s +Step 31900/44000 | Loss: 2.9043 | LR: 9.05e-05 | 1323.9ms/step | 15.1 samples/s +Step 31910/44000 | Loss: 2.9415 | LR: 9.04e-05 | 1322.8ms/step | 15.1 samples/s +Step 31920/44000 | Loss: 2.8973 | LR: 9.02e-05 | 1322.5ms/step | 15.1 samples/s +Step 31930/44000 | Loss: 2.9074 | LR: 9.01e-05 | 1323.2ms/step | 15.1 samples/s +Step 31940/44000 | Loss: 2.9397 | LR: 8.99e-05 | 1323.3ms/step | 15.1 samples/s +Step 31950/44000 | Loss: 2.9308 | LR: 8.98e-05 | 1320.4ms/step | 15.1 samples/s +Step 31960/44000 | Loss: 2.9083 | LR: 8.97e-05 | 1323.8ms/step | 15.1 samples/s +Step 31970/44000 | Loss: 2.9056 | LR: 8.95e-05 | 1324.3ms/step | 15.1 samples/s +Step 31980/44000 | Loss: 2.9321 | LR: 8.94e-05 | 1326.5ms/step | 15.1 samples/s +Step 31990/44000 | Loss: 2.9423 | LR: 8.92e-05 | 1325.2ms/step | 15.1 samples/s +Step 32000/44000 | Loss: 2.9145 | LR: 8.91e-05 | 1323.5ms/step | 15.1 samples/s +Validation at step 32000: Loss = 2.9320, Perplexity = 18.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-32000 +Step 32010/44000 | Loss: 2.9039 | LR: 8.90e-05 | 3855.8ms/step | 5.2 samples/s +Step 32020/44000 | Loss: 2.8929 | LR: 8.88e-05 | 1321.8ms/step | 15.1 samples/s +Step 32030/44000 | Loss: 2.9474 | LR: 8.87e-05 | 1323.7ms/step | 15.1 samples/s +Step 32040/44000 | Loss: 2.9299 | LR: 8.86e-05 | 1323.9ms/step | 15.1 samples/s +Step 32050/44000 | Loss: 2.9162 | LR: 8.84e-05 | 1324.2ms/step | 15.1 samples/s +Step 32060/44000 | Loss: 2.9370 | LR: 8.83e-05 | 1323.6ms/step | 15.1 samples/s +Step 32070/44000 | Loss: 2.9262 | LR: 8.81e-05 | 1322.8ms/step | 15.1 samples/s +Step 32080/44000 | Loss: 2.9072 | LR: 8.80e-05 | 1323.2ms/step | 15.1 samples/s +Step 32090/44000 | Loss: 2.9246 | LR: 8.79e-05 | 1324.3ms/step | 15.1 samples/s +Step 32100/44000 | Loss: 2.9366 | LR: 8.77e-05 | 1325.7ms/step | 15.1 samples/s +Step 32110/44000 | Loss: 2.9312 | LR: 8.76e-05 | 1324.7ms/step | 15.1 samples/s +Step 32120/44000 | Loss: 2.9244 | LR: 8.74e-05 | 1322.2ms/step | 15.1 samples/s +Step 32130/44000 | Loss: 2.9346 | LR: 8.73e-05 | 1324.2ms/step | 15.1 samples/s +Step 32140/44000 | Loss: 2.9209 | LR: 8.72e-05 | 1323.4ms/step | 15.1 samples/s +Step 32150/44000 | Loss: 2.9173 | LR: 8.70e-05 | 1323.3ms/step | 15.1 samples/s +Step 32160/44000 | Loss: 2.8986 | LR: 8.69e-05 | 1325.7ms/step | 15.1 samples/s +Step 32170/44000 | Loss: 2.9414 | LR: 8.68e-05 | 1323.8ms/step | 15.1 samples/s +Step 32180/44000 | Loss: 2.9327 | LR: 8.66e-05 | 1323.1ms/step | 15.1 samples/s +Step 32190/44000 | Loss: 2.9060 | LR: 8.65e-05 | 1324.4ms/step | 15.1 samples/s +Step 32200/44000 | Loss: 2.9387 | LR: 8.63e-05 | 1323.9ms/step | 15.1 samples/s +Step 32210/44000 | Loss: 2.9437 | LR: 8.62e-05 | 1324.7ms/step | 15.1 samples/s +Step 32220/44000 | Loss: 2.9270 | LR: 8.61e-05 | 1320.7ms/step | 15.1 samples/s +Step 32230/44000 | Loss: 2.9298 | LR: 8.59e-05 | 1320.2ms/step | 15.1 samples/s +Step 32240/44000 | Loss: 2.9528 | LR: 8.58e-05 | 1319.9ms/step | 15.2 samples/s +Step 32250/44000 | Loss: 2.9496 | LR: 8.57e-05 | 1324.5ms/step | 15.1 samples/s +Validation at step 32250: Loss = 2.9315, Perplexity = 18.76 +Step 32260/44000 | Loss: 2.9082 | LR: 8.55e-05 | 3535.2ms/step | 5.7 samples/s +Step 32270/44000 | Loss: 2.9326 | LR: 8.54e-05 | 1323.8ms/step | 15.1 samples/s +Step 32280/44000 | Loss: 2.9231 | LR: 8.53e-05 | 1323.6ms/step | 15.1 samples/s +Step 32290/44000 | Loss: 2.9302 | LR: 8.51e-05 | 1324.0ms/step | 15.1 samples/s +Step 32300/44000 | Loss: 2.9242 | LR: 8.50e-05 | 1322.7ms/step | 15.1 samples/s +Step 32310/44000 | Loss: 2.9162 | LR: 8.48e-05 | 1324.1ms/step | 15.1 samples/s +Step 32320/44000 | Loss: 2.9364 | LR: 8.47e-05 | 1324.8ms/step | 15.1 samples/s +Step 32330/44000 | Loss: 2.9311 | LR: 8.46e-05 | 1320.7ms/step | 15.1 samples/s +Step 32340/44000 | Loss: 2.9050 | LR: 8.44e-05 | 1322.0ms/step | 15.1 samples/s +Step 32350/44000 | Loss: 2.9265 | LR: 8.43e-05 | 1324.4ms/step | 15.1 samples/s +Step 32360/44000 | Loss: 2.8977 | LR: 8.42e-05 | 1322.0ms/step | 15.1 samples/s +Step 32370/44000 | Loss: 2.9211 | LR: 8.40e-05 | 1321.6ms/step | 15.1 samples/s +Step 32380/44000 | Loss: 2.9237 | LR: 8.39e-05 | 1321.7ms/step | 15.1 samples/s +Step 32390/44000 | Loss: 2.9293 | LR: 8.38e-05 | 1323.7ms/step | 15.1 samples/s +Step 32400/44000 | Loss: 2.9172 | LR: 8.36e-05 | 1321.8ms/step | 15.1 samples/s +Step 32410/44000 | Loss: 2.9068 | LR: 8.35e-05 | 1323.0ms/step | 15.1 samples/s +Step 32420/44000 | Loss: 2.9053 | LR: 8.33e-05 | 1325.2ms/step | 15.1 samples/s +Step 32430/44000 | Loss: 2.9547 | LR: 8.32e-05 | 1325.1ms/step | 15.1 samples/s +Step 32440/44000 | Loss: 2.9035 | LR: 8.31e-05 | 1323.9ms/step | 15.1 samples/s +Step 32450/44000 | Loss: 2.9345 | LR: 8.29e-05 | 1322.2ms/step | 15.1 samples/s +Step 32460/44000 | Loss: 2.9248 | LR: 8.28e-05 | 1321.9ms/step | 15.1 samples/s +Step 32470/44000 | Loss: 2.9260 | LR: 8.27e-05 | 1323.0ms/step | 15.1 samples/s +Step 32480/44000 | Loss: 2.9573 | LR: 8.25e-05 | 1325.4ms/step | 15.1 samples/s +Step 32490/44000 | Loss: 2.9710 | LR: 8.24e-05 | 1324.0ms/step | 15.1 samples/s +Step 32500/44000 | Loss: 2.9316 | LR: 8.23e-05 | 1322.8ms/step | 15.1 samples/s +Validation at step 32500: Loss = 2.9309, Perplexity = 18.75 +Step 32510/44000 | Loss: 2.9533 | LR: 8.21e-05 | 3532.2ms/step | 5.7 samples/s +Step 32520/44000 | Loss: 2.9285 | LR: 8.20e-05 | 1322.2ms/step | 15.1 samples/s +Step 32530/44000 | Loss: 2.9236 | LR: 8.19e-05 | 1323.6ms/step | 15.1 samples/s +Step 32540/44000 | Loss: 2.9128 | LR: 8.17e-05 | 1324.8ms/step | 15.1 samples/s +Step 32550/44000 | Loss: 2.9461 | LR: 8.16e-05 | 1324.8ms/step | 15.1 samples/s +Step 32560/44000 | Loss: 2.9189 | LR: 8.15e-05 | 1323.1ms/step | 15.1 samples/s +Step 32570/44000 | Loss: 2.9038 | LR: 8.13e-05 | 1324.4ms/step | 15.1 samples/s +Step 32580/44000 | Loss: 2.9224 | LR: 8.12e-05 | 1324.3ms/step | 15.1 samples/s +Step 32590/44000 | Loss: 2.9296 | LR: 8.11e-05 | 1324.2ms/step | 15.1 samples/s +Step 32600/44000 | Loss: 2.9106 | LR: 8.09e-05 | 1324.1ms/step | 15.1 samples/s +Step 32610/44000 | Loss: 2.9276 | LR: 8.08e-05 | 1324.0ms/step | 15.1 samples/s +Step 32620/44000 | Loss: 2.9073 | LR: 8.07e-05 | 1321.5ms/step | 15.1 samples/s +Step 32630/44000 | Loss: 2.9003 | LR: 8.05e-05 | 1322.8ms/step | 15.1 samples/s +Step 32640/44000 | Loss: 2.9180 | LR: 8.04e-05 | 1322.6ms/step | 15.1 samples/s +Step 32650/44000 | Loss: 2.8913 | LR: 8.03e-05 | 1322.9ms/step | 15.1 samples/s +Step 32660/44000 | Loss: 2.9066 | LR: 8.01e-05 | 1322.0ms/step | 15.1 samples/s +Step 32670/44000 | Loss: 2.9337 | LR: 8.00e-05 | 1322.1ms/step | 15.1 samples/s +Step 32680/44000 | Loss: 2.9257 | LR: 7.99e-05 | 1323.7ms/step | 15.1 samples/s +Step 32690/44000 | Loss: 2.9225 | LR: 7.97e-05 | 1323.3ms/step | 15.1 samples/s +Step 32700/44000 | Loss: 2.9081 | LR: 7.96e-05 | 1323.2ms/step | 15.1 samples/s +Step 32710/44000 | Loss: 2.9285 | LR: 7.95e-05 | 1322.8ms/step | 15.1 samples/s +Step 32720/44000 | Loss: 2.9076 | LR: 7.93e-05 | 1323.2ms/step | 15.1 samples/s +Step 32730/44000 | Loss: 2.9068 | LR: 7.92e-05 | 1322.7ms/step | 15.1 samples/s +Step 32740/44000 | Loss: 2.9149 | LR: 7.91e-05 | 1321.6ms/step | 15.1 samples/s +Step 32750/44000 | Loss: 2.9273 | LR: 7.89e-05 | 1322.5ms/step | 15.1 samples/s +Validation at step 32750: Loss = 2.9306, Perplexity = 18.74 +Step 32760/44000 | Loss: 2.9535 | LR: 7.88e-05 | 3531.1ms/step | 5.7 samples/s +Step 32770/44000 | Loss: 2.9029 | LR: 7.87e-05 | 1346.4ms/step | 14.9 samples/s +Step 32780/44000 | Loss: 2.9331 | LR: 7.85e-05 | 1339.8ms/step | 14.9 samples/s +Step 32790/44000 | Loss: 2.8873 | LR: 7.84e-05 | 1340.6ms/step | 14.9 samples/s +Step 32800/44000 | Loss: 2.9111 | LR: 7.83e-05 | 1337.1ms/step | 15.0 samples/s +Step 32810/44000 | Loss: 2.9103 | LR: 7.81e-05 | 1342.5ms/step | 14.9 samples/s +Step 32820/44000 | Loss: 2.8889 | LR: 7.80e-05 | 1339.6ms/step | 14.9 samples/s +Step 32830/44000 | Loss: 2.9287 | LR: 7.79e-05 | 1338.9ms/step | 14.9 samples/s +Step 32840/44000 | Loss: 2.9392 | LR: 7.77e-05 | 1337.0ms/step | 15.0 samples/s +Step 32850/44000 | Loss: 2.9267 | LR: 7.76e-05 | 1344.4ms/step | 14.9 samples/s +Step 32860/44000 | Loss: 2.8980 | LR: 7.75e-05 | 1343.9ms/step | 14.9 samples/s +Step 32870/44000 | Loss: 2.9344 | LR: 7.73e-05 | 1342.4ms/step | 14.9 samples/s +Step 32880/44000 | Loss: 2.8963 | LR: 7.72e-05 | 1339.8ms/step | 14.9 samples/s +Step 32890/44000 | Loss: 2.8973 | LR: 7.71e-05 | 1342.7ms/step | 14.9 samples/s +Step 32900/44000 | Loss: 2.9045 | LR: 7.70e-05 | 1340.4ms/step | 14.9 samples/s +Step 32910/44000 | Loss: 2.9092 | LR: 7.68e-05 | 1341.3ms/step | 14.9 samples/s +Step 32920/44000 | Loss: 2.9299 | LR: 7.67e-05 | 1338.9ms/step | 14.9 samples/s +Step 32930/44000 | Loss: 2.9218 | LR: 7.66e-05 | 1341.8ms/step | 14.9 samples/s +Step 32940/44000 | Loss: 2.9184 | LR: 7.64e-05 | 1341.4ms/step | 14.9 samples/s +Step 32950/44000 | Loss: 2.9384 | LR: 7.63e-05 | 1341.2ms/step | 14.9 samples/s +Step 32960/44000 | Loss: 2.9608 | LR: 7.62e-05 | 1341.8ms/step | 14.9 samples/s +Step 32970/44000 | Loss: 2.9488 | LR: 7.60e-05 | 1342.5ms/step | 14.9 samples/s +Step 32980/44000 | Loss: 2.9242 | LR: 7.59e-05 | 1341.6ms/step | 14.9 samples/s +Step 32990/44000 | Loss: 2.9492 | LR: 7.58e-05 | 1337.6ms/step | 15.0 samples/s +Step 33000/44000 | Loss: 2.8950 | LR: 7.56e-05 | 1336.2ms/step | 15.0 samples/s +Validation at step 33000: Loss = 2.9302, Perplexity = 18.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-33000 +Step 33010/44000 | Loss: 2.9316 | LR: 7.55e-05 | 3882.7ms/step | 5.2 samples/s +Step 33020/44000 | Loss: 2.9258 | LR: 7.54e-05 | 1337.2ms/step | 15.0 samples/s +Step 33030/44000 | Loss: 2.9105 | LR: 7.53e-05 | 1339.3ms/step | 14.9 samples/s +Step 33040/44000 | Loss: 2.9254 | LR: 7.51e-05 | 1340.1ms/step | 14.9 samples/s +Step 33050/44000 | Loss: 2.9013 | LR: 7.50e-05 | 1337.9ms/step | 14.9 samples/s +Step 33060/44000 | Loss: 2.9485 | LR: 7.49e-05 | 1340.2ms/step | 14.9 samples/s +Step 33070/44000 | Loss: 2.9200 | LR: 7.47e-05 | 1336.7ms/step | 15.0 samples/s +Step 33080/44000 | Loss: 2.9080 | LR: 7.46e-05 | 1337.5ms/step | 15.0 samples/s +Step 33090/44000 | Loss: 2.9593 | LR: 7.45e-05 | 1333.4ms/step | 15.0 samples/s +Step 33100/44000 | Loss: 2.8904 | LR: 7.44e-05 | 1334.5ms/step | 15.0 samples/s +Step 33110/44000 | Loss: 2.9595 | LR: 7.42e-05 | 1344.9ms/step | 14.9 samples/s +Step 33120/44000 | Loss: 2.9200 | LR: 7.41e-05 | 1340.1ms/step | 14.9 samples/s +Step 33130/44000 | Loss: 2.9296 | LR: 7.40e-05 | 1338.9ms/step | 14.9 samples/s +Step 33140/44000 | Loss: 2.9131 | LR: 7.38e-05 | 1340.3ms/step | 14.9 samples/s +Step 33150/44000 | Loss: 2.9226 | LR: 7.37e-05 | 1340.9ms/step | 14.9 samples/s +Step 33160/44000 | Loss: 2.9319 | LR: 7.36e-05 | 1341.0ms/step | 14.9 samples/s +Step 33170/44000 | Loss: 2.9400 | LR: 7.34e-05 | 1343.1ms/step | 14.9 samples/s +Step 33180/44000 | Loss: 2.8924 | LR: 7.33e-05 | 1340.6ms/step | 14.9 samples/s +Step 33190/44000 | Loss: 2.9244 | LR: 7.32e-05 | 1341.7ms/step | 14.9 samples/s +Step 33200/44000 | Loss: 2.9329 | LR: 7.31e-05 | 1339.6ms/step | 14.9 samples/s +Step 33210/44000 | Loss: 2.9338 | LR: 7.29e-05 | 1339.3ms/step | 14.9 samples/s +Step 33220/44000 | Loss: 2.8712 | LR: 7.28e-05 | 1337.9ms/step | 14.9 samples/s +Step 33230/44000 | Loss: 2.9037 | LR: 7.27e-05 | 1339.4ms/step | 14.9 samples/s +Step 33240/44000 | Loss: 2.9258 | LR: 7.26e-05 | 1337.3ms/step | 15.0 samples/s +Step 33250/44000 | Loss: 2.9240 | LR: 7.24e-05 | 1339.6ms/step | 14.9 samples/s +Validation at step 33250: Loss = 2.9297, Perplexity = 18.72 +Step 33260/44000 | Loss: 2.9173 | LR: 7.23e-05 | 3568.4ms/step | 5.6 samples/s +Step 33270/44000 | Loss: 2.9431 | LR: 7.22e-05 | 1341.9ms/step | 14.9 samples/s +Step 33280/44000 | Loss: 2.9528 | LR: 7.20e-05 | 1343.2ms/step | 14.9 samples/s +Step 33290/44000 | Loss: 2.9087 | LR: 7.19e-05 | 1342.5ms/step | 14.9 samples/s +Step 33300/44000 | Loss: 2.9151 | LR: 7.18e-05 | 1342.2ms/step | 14.9 samples/s +Step 33310/44000 | Loss: 2.9221 | LR: 7.17e-05 | 1339.6ms/step | 14.9 samples/s +Step 33320/44000 | Loss: 2.9417 | LR: 7.15e-05 | 1350.0ms/step | 14.8 samples/s +Step 33330/44000 | Loss: 2.9058 | LR: 7.14e-05 | 1341.8ms/step | 14.9 samples/s +Step 33340/44000 | Loss: 2.9391 | LR: 7.13e-05 | 1336.8ms/step | 15.0 samples/s +Step 33350/44000 | Loss: 2.8855 | LR: 7.11e-05 | 1341.0ms/step | 14.9 samples/s +Step 33360/44000 | Loss: 2.9293 | LR: 7.10e-05 | 1339.0ms/step | 14.9 samples/s +Step 33370/44000 | Loss: 2.9390 | LR: 7.09e-05 | 1337.0ms/step | 15.0 samples/s +Step 33380/44000 | Loss: 2.9188 | LR: 7.08e-05 | 1337.2ms/step | 15.0 samples/s +Step 33390/44000 | Loss: 2.9272 | LR: 7.06e-05 | 1336.1ms/step | 15.0 samples/s +Step 33400/44000 | Loss: 2.9269 | LR: 7.05e-05 | 1333.7ms/step | 15.0 samples/s +Step 33410/44000 | Loss: 2.9115 | LR: 7.04e-05 | 1335.4ms/step | 15.0 samples/s +Step 33420/44000 | Loss: 2.9489 | LR: 7.03e-05 | 1336.6ms/step | 15.0 samples/s +Step 33430/44000 | Loss: 2.9064 | LR: 7.01e-05 | 1333.5ms/step | 15.0 samples/s +Step 33440/44000 | Loss: 2.9252 | LR: 7.00e-05 | 1335.2ms/step | 15.0 samples/s +Step 33450/44000 | Loss: 2.9378 | LR: 6.99e-05 | 1334.1ms/step | 15.0 samples/s +Step 33460/44000 | Loss: 2.9105 | LR: 6.98e-05 | 1334.3ms/step | 15.0 samples/s +Step 33470/44000 | Loss: 2.9156 | LR: 6.96e-05 | 1334.0ms/step | 15.0 samples/s +Step 33480/44000 | Loss: 2.9302 | LR: 6.95e-05 | 1333.6ms/step | 15.0 samples/s +Step 33490/44000 | Loss: 2.9375 | LR: 6.94e-05 | 1341.5ms/step | 14.9 samples/s +Step 33500/44000 | Loss: 2.9251 | LR: 6.93e-05 | 1343.0ms/step | 14.9 samples/s +Validation at step 33500: Loss = 2.9293, Perplexity = 18.71 +Step 33510/44000 | Loss: 2.9026 | LR: 6.91e-05 | 3565.8ms/step | 5.6 samples/s +Step 33520/44000 | Loss: 2.9060 | LR: 6.90e-05 | 1336.5ms/step | 15.0 samples/s +Step 33530/44000 | Loss: 2.9090 | LR: 6.89e-05 | 1344.9ms/step | 14.9 samples/s +Step 33540/44000 | Loss: 2.9349 | LR: 6.88e-05 | 1345.2ms/step | 14.9 samples/s +Step 33550/44000 | Loss: 2.9204 | LR: 6.86e-05 | 1338.7ms/step | 14.9 samples/s +Step 33560/44000 | Loss: 2.9011 | LR: 6.85e-05 | 1339.3ms/step | 14.9 samples/s +Step 33570/44000 | Loss: 2.9307 | LR: 6.84e-05 | 1337.3ms/step | 15.0 samples/s +Step 33580/44000 | Loss: 2.9530 | LR: 6.83e-05 | 1340.4ms/step | 14.9 samples/s +Step 33590/44000 | Loss: 2.9113 | LR: 6.81e-05 | 1337.4ms/step | 15.0 samples/s +Step 33600/44000 | Loss: 2.9354 | LR: 6.80e-05 | 1339.3ms/step | 14.9 samples/s +Step 33610/44000 | Loss: 2.9334 | LR: 6.79e-05 | 1336.0ms/step | 15.0 samples/s +Step 33620/44000 | Loss: 2.9231 | LR: 6.78e-05 | 1338.1ms/step | 14.9 samples/s +Step 33630/44000 | Loss: 2.9411 | LR: 6.76e-05 | 1341.2ms/step | 14.9 samples/s +Step 33640/44000 | Loss: 2.9160 | LR: 6.75e-05 | 1339.6ms/step | 14.9 samples/s +Step 33650/44000 | Loss: 2.9240 | LR: 6.74e-05 | 1338.2ms/step | 14.9 samples/s +Step 33660/44000 | Loss: 2.9289 | LR: 6.73e-05 | 1337.9ms/step | 14.9 samples/s +Step 33670/44000 | Loss: 2.9228 | LR: 6.71e-05 | 1336.4ms/step | 15.0 samples/s +Step 33680/44000 | Loss: 2.9125 | LR: 6.70e-05 | 1338.0ms/step | 14.9 samples/s +Step 33690/44000 | Loss: 2.9128 | LR: 6.69e-05 | 1337.4ms/step | 15.0 samples/s +Step 33700/44000 | Loss: 2.9403 | LR: 6.68e-05 | 1335.4ms/step | 15.0 samples/s +Step 33710/44000 | Loss: 2.9064 | LR: 6.66e-05 | 1343.1ms/step | 14.9 samples/s +Step 33720/44000 | Loss: 2.9113 | LR: 6.65e-05 | 1343.3ms/step | 14.9 samples/s +Step 33730/44000 | Loss: 2.9297 | LR: 6.64e-05 | 1339.4ms/step | 14.9 samples/s +Step 33740/44000 | Loss: 2.9248 | LR: 6.63e-05 | 1336.1ms/step | 15.0 samples/s +Step 33750/44000 | Loss: 2.9218 | LR: 6.61e-05 | 1336.3ms/step | 15.0 samples/s +Validation at step 33750: Loss = 2.9289, Perplexity = 18.71 +Step 33760/44000 | Loss: 2.9177 | LR: 6.60e-05 | 3575.9ms/step | 5.6 samples/s +Step 33770/44000 | Loss: 2.8848 | LR: 6.59e-05 | 1341.7ms/step | 14.9 samples/s +Step 33780/44000 | Loss: 2.9317 | LR: 6.58e-05 | 1330.7ms/step | 15.0 samples/s +Step 33790/44000 | Loss: 2.9237 | LR: 6.57e-05 | 1333.5ms/step | 15.0 samples/s +Step 33800/44000 | Loss: 2.8823 | LR: 6.55e-05 | 1336.3ms/step | 15.0 samples/s +Step 33810/44000 | Loss: 2.9127 | LR: 6.54e-05 | 1334.2ms/step | 15.0 samples/s +Step 33820/44000 | Loss: 2.9219 | LR: 6.53e-05 | 1342.0ms/step | 14.9 samples/s +Step 33830/44000 | Loss: 2.9154 | LR: 6.52e-05 | 1336.6ms/step | 15.0 samples/s +Step 33840/44000 | Loss: 2.9312 | LR: 6.50e-05 | 1342.2ms/step | 14.9 samples/s +Step 33850/44000 | Loss: 2.8969 | LR: 6.49e-05 | 1343.3ms/step | 14.9 samples/s +Step 33860/44000 | Loss: 2.9344 | LR: 6.48e-05 | 1335.2ms/step | 15.0 samples/s +Step 33870/44000 | Loss: 2.9270 | LR: 6.47e-05 | 1336.9ms/step | 15.0 samples/s +Step 33880/44000 | Loss: 2.9068 | LR: 6.46e-05 | 1344.1ms/step | 14.9 samples/s +Step 33890/44000 | Loss: 2.9373 | LR: 6.44e-05 | 1333.1ms/step | 15.0 samples/s +Step 33900/44000 | Loss: 2.8790 | LR: 6.43e-05 | 1331.2ms/step | 15.0 samples/s +Step 33910/44000 | Loss: 2.9039 | LR: 6.42e-05 | 1332.4ms/step | 15.0 samples/s +Step 33920/44000 | Loss: 2.9203 | LR: 6.41e-05 | 1338.5ms/step | 14.9 samples/s +Step 33930/44000 | Loss: 2.9185 | LR: 6.40e-05 | 1336.2ms/step | 15.0 samples/s +Step 33940/44000 | Loss: 2.9328 | LR: 6.38e-05 | 1339.7ms/step | 14.9 samples/s +Step 33950/44000 | Loss: 2.9061 | LR: 6.37e-05 | 1332.8ms/step | 15.0 samples/s +Step 33960/44000 | Loss: 2.8902 | LR: 6.36e-05 | 1334.8ms/step | 15.0 samples/s +Step 33970/44000 | Loss: 2.9241 | LR: 6.35e-05 | 1329.2ms/step | 15.0 samples/s +Step 33980/44000 | Loss: 2.9178 | LR: 6.33e-05 | 1328.6ms/step | 15.1 samples/s +Step 33990/44000 | Loss: 2.9065 | LR: 6.32e-05 | 1327.1ms/step | 15.1 samples/s +Step 34000/44000 | Loss: 2.9000 | LR: 6.31e-05 | 1327.9ms/step | 15.1 samples/s +Validation at step 34000: Loss = 2.9286, Perplexity = 18.70 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-34000 +Step 34010/44000 | Loss: 2.9403 | LR: 6.30e-05 | 3871.2ms/step | 5.2 samples/s +Step 34020/44000 | Loss: 2.9153 | LR: 6.29e-05 | 1338.6ms/step | 14.9 samples/s +Step 34030/44000 | Loss: 2.9123 | LR: 6.27e-05 | 1337.0ms/step | 15.0 samples/s +Step 34040/44000 | Loss: 2.9191 | LR: 6.26e-05 | 1335.8ms/step | 15.0 samples/s +Step 34050/44000 | Loss: 2.9364 | LR: 6.25e-05 | 1338.1ms/step | 14.9 samples/s +Step 34060/44000 | Loss: 2.8998 | LR: 6.24e-05 | 1337.7ms/step | 15.0 samples/s +Step 34070/44000 | Loss: 2.9134 | LR: 6.23e-05 | 1340.9ms/step | 14.9 samples/s +Step 34080/44000 | Loss: 2.9194 | LR: 6.21e-05 | 1345.2ms/step | 14.9 samples/s +Step 34090/44000 | Loss: 2.8959 | LR: 6.20e-05 | 1337.3ms/step | 15.0 samples/s +Step 34100/44000 | Loss: 2.8982 | LR: 6.19e-05 | 1334.3ms/step | 15.0 samples/s +Step 34110/44000 | Loss: 2.9293 | LR: 6.18e-05 | 1331.7ms/step | 15.0 samples/s +Step 34120/44000 | Loss: 2.9346 | LR: 6.17e-05 | 1332.6ms/step | 15.0 samples/s +Step 34130/44000 | Loss: 2.9076 | LR: 6.15e-05 | 1341.8ms/step | 14.9 samples/s +Step 34140/44000 | Loss: 2.9208 | LR: 6.14e-05 | 1334.6ms/step | 15.0 samples/s +Step 34150/44000 | Loss: 2.9430 | LR: 6.13e-05 | 1337.5ms/step | 15.0 samples/s +Step 34160/44000 | Loss: 2.9344 | LR: 6.12e-05 | 1335.0ms/step | 15.0 samples/s +Step 34170/44000 | Loss: 2.9055 | LR: 6.11e-05 | 1332.0ms/step | 15.0 samples/s +Step 34180/44000 | Loss: 2.9098 | LR: 6.09e-05 | 1332.5ms/step | 15.0 samples/s +Step 34190/44000 | Loss: 2.8880 | LR: 6.08e-05 | 1328.0ms/step | 15.1 samples/s +Step 34200/44000 | Loss: 2.8803 | LR: 6.07e-05 | 1337.8ms/step | 14.9 samples/s +Step 34210/44000 | Loss: 2.8900 | LR: 6.06e-05 | 1330.8ms/step | 15.0 samples/s +Step 34220/44000 | Loss: 2.9222 | LR: 6.05e-05 | 1334.7ms/step | 15.0 samples/s +Step 34230/44000 | Loss: 2.9293 | LR: 6.04e-05 | 1333.0ms/step | 15.0 samples/s +Step 34240/44000 | Loss: 2.9359 | LR: 6.02e-05 | 1330.8ms/step | 15.0 samples/s +Step 34250/44000 | Loss: 2.9163 | LR: 6.01e-05 | 1334.4ms/step | 15.0 samples/s +Validation at step 34250: Loss = 2.9283, Perplexity = 18.70 +Step 34260/44000 | Loss: 2.9448 | LR: 6.00e-05 | 3558.3ms/step | 5.6 samples/s +Step 34270/44000 | Loss: 2.8877 | LR: 5.99e-05 | 1331.5ms/step | 15.0 samples/s +Step 34280/44000 | Loss: 2.9245 | LR: 5.98e-05 | 1329.7ms/step | 15.0 samples/s +Step 34290/44000 | Loss: 2.9363 | LR: 5.96e-05 | 1338.7ms/step | 14.9 samples/s +Step 34300/44000 | Loss: 2.9322 | LR: 5.95e-05 | 1332.4ms/step | 15.0 samples/s +Step 34310/44000 | Loss: 2.9268 | LR: 5.94e-05 | 1331.5ms/step | 15.0 samples/s +Step 34320/44000 | Loss: 2.9096 | LR: 5.93e-05 | 1332.7ms/step | 15.0 samples/s +Step 34330/44000 | Loss: 2.9620 | LR: 5.92e-05 | 1333.9ms/step | 15.0 samples/s +Step 34340/44000 | Loss: 2.9208 | LR: 5.91e-05 | 1333.4ms/step | 15.0 samples/s +Step 34350/44000 | Loss: 2.9144 | LR: 5.89e-05 | 1329.9ms/step | 15.0 samples/s +Step 34360/44000 | Loss: 2.9157 | LR: 5.88e-05 | 1330.8ms/step | 15.0 samples/s +Step 34370/44000 | Loss: 2.9321 | LR: 5.87e-05 | 1334.2ms/step | 15.0 samples/s +Step 34380/44000 | Loss: 2.9082 | LR: 5.86e-05 | 1328.6ms/step | 15.1 samples/s +Step 34390/44000 | Loss: 2.9353 | LR: 5.85e-05 | 1327.1ms/step | 15.1 samples/s +Step 34400/44000 | Loss: 2.9204 | LR: 5.84e-05 | 1332.9ms/step | 15.0 samples/s +Step 34410/44000 | Loss: 2.9003 | LR: 5.82e-05 | 1323.9ms/step | 15.1 samples/s +Step 34420/44000 | Loss: 2.8950 | LR: 5.81e-05 | 1324.5ms/step | 15.1 samples/s +Step 34430/44000 | Loss: 2.9377 | LR: 5.80e-05 | 1329.2ms/step | 15.0 samples/s +Step 34440/44000 | Loss: 2.9208 | LR: 5.79e-05 | 1328.6ms/step | 15.1 samples/s +Step 34450/44000 | Loss: 2.9012 | LR: 5.78e-05 | 1324.3ms/step | 15.1 samples/s +Step 34460/44000 | Loss: 2.9175 | LR: 5.77e-05 | 1324.5ms/step | 15.1 samples/s +Step 34470/44000 | Loss: 2.9132 | LR: 5.75e-05 | 1325.6ms/step | 15.1 samples/s +Step 34480/44000 | Loss: 2.9435 | LR: 5.74e-05 | 1326.3ms/step | 15.1 samples/s +Step 34490/44000 | Loss: 2.9206 | LR: 5.73e-05 | 1332.4ms/step | 15.0 samples/s +Step 34500/44000 | Loss: 2.9176 | LR: 5.72e-05 | 1319.8ms/step | 15.2 samples/s +Validation at step 34500: Loss = 2.9279, Perplexity = 18.69 +Step 34510/44000 | Loss: 2.8860 | LR: 5.71e-05 | 3551.7ms/step | 5.6 samples/s +Step 34520/44000 | Loss: 2.9174 | LR: 5.70e-05 | 1328.8ms/step | 15.1 samples/s +Step 34530/44000 | Loss: 2.9169 | LR: 5.69e-05 | 1329.3ms/step | 15.0 samples/s +Step 34540/44000 | Loss: 2.9103 | LR: 5.67e-05 | 1333.0ms/step | 15.0 samples/s +Step 34550/44000 | Loss: 2.8970 | LR: 5.66e-05 | 1326.6ms/step | 15.1 samples/s +Step 34560/44000 | Loss: 2.9455 | LR: 5.65e-05 | 1328.9ms/step | 15.0 samples/s +Step 34570/44000 | Loss: 2.9586 | LR: 5.64e-05 | 1328.0ms/step | 15.1 samples/s +Step 34580/44000 | Loss: 2.9094 | LR: 5.63e-05 | 1328.8ms/step | 15.1 samples/s +Step 34590/44000 | Loss: 2.9077 | LR: 5.62e-05 | 1324.3ms/step | 15.1 samples/s +Step 34600/44000 | Loss: 2.9260 | LR: 5.60e-05 | 1325.0ms/step | 15.1 samples/s +Step 34610/44000 | Loss: 2.9286 | LR: 5.59e-05 | 1327.1ms/step | 15.1 samples/s +Step 34620/44000 | Loss: 2.8990 | LR: 5.58e-05 | 1327.6ms/step | 15.1 samples/s +Step 34630/44000 | Loss: 2.9369 | LR: 5.57e-05 | 1324.8ms/step | 15.1 samples/s +Step 34640/44000 | Loss: 2.8820 | LR: 5.56e-05 | 1327.5ms/step | 15.1 samples/s +Step 34650/44000 | Loss: 2.9185 | LR: 5.55e-05 | 1329.3ms/step | 15.0 samples/s +Step 34660/44000 | Loss: 2.9145 | LR: 5.54e-05 | 1327.1ms/step | 15.1 samples/s +Step 34670/44000 | Loss: 2.9185 | LR: 5.52e-05 | 1326.3ms/step | 15.1 samples/s +Step 34680/44000 | Loss: 2.9089 | LR: 5.51e-05 | 1328.1ms/step | 15.1 samples/s +Step 34690/44000 | Loss: 2.9104 | LR: 5.50e-05 | 1323.0ms/step | 15.1 samples/s +Step 34700/44000 | Loss: 2.9067 | LR: 5.49e-05 | 1323.5ms/step | 15.1 samples/s +Step 34710/44000 | Loss: 2.8997 | LR: 5.48e-05 | 1327.4ms/step | 15.1 samples/s +Step 34720/44000 | Loss: 2.9226 | LR: 5.47e-05 | 1326.0ms/step | 15.1 samples/s +Step 34730/44000 | Loss: 2.9025 | LR: 5.46e-05 | 1326.1ms/step | 15.1 samples/s +Step 34740/44000 | Loss: 2.9554 | LR: 5.45e-05 | 1326.3ms/step | 15.1 samples/s +Step 34750/44000 | Loss: 2.9260 | LR: 5.43e-05 | 1330.0ms/step | 15.0 samples/s +Validation at step 34750: Loss = 2.9276, Perplexity = 18.68 +Step 34760/44000 | Loss: 2.8964 | LR: 5.42e-05 | 3548.0ms/step | 5.6 samples/s +Step 34770/44000 | Loss: 2.9513 | LR: 5.41e-05 | 1326.2ms/step | 15.1 samples/s +Step 34780/44000 | Loss: 2.9197 | LR: 5.40e-05 | 1323.2ms/step | 15.1 samples/s +Step 34790/44000 | Loss: 2.9299 | LR: 5.39e-05 | 1324.2ms/step | 15.1 samples/s +Step 34800/44000 | Loss: 2.9432 | LR: 5.38e-05 | 1328.2ms/step | 15.1 samples/s +Step 34810/44000 | Loss: 2.9432 | LR: 5.37e-05 | 1328.8ms/step | 15.1 samples/s +Step 34820/44000 | Loss: 2.9216 | LR: 5.36e-05 | 1325.6ms/step | 15.1 samples/s +Step 34830/44000 | Loss: 2.9487 | LR: 5.34e-05 | 1322.8ms/step | 15.1 samples/s +Step 34840/44000 | Loss: 2.9286 | LR: 5.33e-05 | 1326.0ms/step | 15.1 samples/s +Step 34850/44000 | Loss: 2.8842 | LR: 5.32e-05 | 1326.9ms/step | 15.1 samples/s +Step 34860/44000 | Loss: 2.9035 | LR: 5.31e-05 | 1327.2ms/step | 15.1 samples/s +Step 34870/44000 | Loss: 2.8978 | LR: 5.30e-05 | 1324.8ms/step | 15.1 samples/s +Step 34880/44000 | Loss: 2.9072 | LR: 5.29e-05 | 1319.1ms/step | 15.2 samples/s +Step 34890/44000 | Loss: 2.8929 | LR: 5.28e-05 | 1322.2ms/step | 15.1 samples/s +Step 34900/44000 | Loss: 2.9300 | LR: 5.27e-05 | 1327.5ms/step | 15.1 samples/s +Step 34910/44000 | Loss: 2.9222 | LR: 5.25e-05 | 1326.7ms/step | 15.1 samples/s +Step 34920/44000 | Loss: 2.9108 | LR: 5.24e-05 | 1327.2ms/step | 15.1 samples/s +Step 34930/44000 | Loss: 2.9183 | LR: 5.23e-05 | 1324.0ms/step | 15.1 samples/s +Step 34940/44000 | Loss: 2.9259 | LR: 5.22e-05 | 1325.6ms/step | 15.1 samples/s +Step 34950/44000 | Loss: 2.9005 | LR: 5.21e-05 | 1322.3ms/step | 15.1 samples/s +Step 34960/44000 | Loss: 2.9459 | LR: 5.20e-05 | 1322.8ms/step | 15.1 samples/s +Step 34970/44000 | Loss: 2.9244 | LR: 5.19e-05 | 1325.6ms/step | 15.1 samples/s +Step 34980/44000 | Loss: 2.9369 | LR: 5.18e-05 | 1327.4ms/step | 15.1 samples/s +Step 34990/44000 | Loss: 2.9182 | LR: 5.17e-05 | 1327.3ms/step | 15.1 samples/s +Step 35000/44000 | Loss: 2.9255 | LR: 5.15e-05 | 1324.3ms/step | 15.1 samples/s +Validation at step 35000: Loss = 2.9273, Perplexity = 18.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-35000 +Step 35010/44000 | Loss: 2.9252 | LR: 5.14e-05 | 3861.7ms/step | 5.2 samples/s +Step 35020/44000 | Loss: 2.9305 | LR: 5.13e-05 | 1331.8ms/step | 15.0 samples/s +Step 35030/44000 | Loss: 2.9035 | LR: 5.12e-05 | 1322.8ms/step | 15.1 samples/s +Step 35040/44000 | Loss: 2.9487 | LR: 5.11e-05 | 1325.4ms/step | 15.1 samples/s +Step 35050/44000 | Loss: 2.9074 | LR: 5.10e-05 | 1322.9ms/step | 15.1 samples/s +Step 35060/44000 | Loss: 2.8993 | LR: 5.09e-05 | 1324.5ms/step | 15.1 samples/s +Step 35070/44000 | Loss: 2.8954 | LR: 5.08e-05 | 1322.6ms/step | 15.1 samples/s +Step 35080/44000 | Loss: 2.9222 | LR: 5.07e-05 | 1326.0ms/step | 15.1 samples/s +Step 35090/44000 | Loss: 2.9141 | LR: 5.06e-05 | 1325.7ms/step | 15.1 samples/s +Step 35100/44000 | Loss: 2.9238 | LR: 5.04e-05 | 1323.3ms/step | 15.1 samples/s +Step 35110/44000 | Loss: 2.9177 | LR: 5.03e-05 | 1323.7ms/step | 15.1 samples/s +Step 35120/44000 | Loss: 2.9235 | LR: 5.02e-05 | 1327.0ms/step | 15.1 samples/s +Step 35130/44000 | Loss: 2.9180 | LR: 5.01e-05 | 1325.5ms/step | 15.1 samples/s +Step 35140/44000 | Loss: 2.9113 | LR: 5.00e-05 | 1325.1ms/step | 15.1 samples/s +Step 35150/44000 | Loss: 2.8882 | LR: 4.99e-05 | 1322.8ms/step | 15.1 samples/s +Step 35160/44000 | Loss: 2.9205 | LR: 4.98e-05 | 1326.6ms/step | 15.1 samples/s +Step 35170/44000 | Loss: 2.9232 | LR: 4.97e-05 | 1324.1ms/step | 15.1 samples/s +Step 35180/44000 | Loss: 2.9241 | LR: 4.96e-05 | 1323.5ms/step | 15.1 samples/s +Step 35190/44000 | Loss: 2.9138 | LR: 4.95e-05 | 1324.1ms/step | 15.1 samples/s +Step 35200/44000 | Loss: 2.8993 | LR: 4.94e-05 | 1327.3ms/step | 15.1 samples/s +Step 35210/44000 | Loss: 2.9394 | LR: 4.93e-05 | 1329.1ms/step | 15.0 samples/s +Step 35220/44000 | Loss: 2.9150 | LR: 4.91e-05 | 1324.9ms/step | 15.1 samples/s +Step 35230/44000 | Loss: 2.9099 | LR: 4.90e-05 | 1325.2ms/step | 15.1 samples/s +Step 35240/44000 | Loss: 2.9149 | LR: 4.89e-05 | 1324.2ms/step | 15.1 samples/s +Step 35250/44000 | Loss: 2.8977 | LR: 4.88e-05 | 1325.5ms/step | 15.1 samples/s +Validation at step 35250: Loss = 2.9271, Perplexity = 18.67 +Step 35260/44000 | Loss: 2.9408 | LR: 4.87e-05 | 3562.7ms/step | 5.6 samples/s +Step 35270/44000 | Loss: 2.8978 | LR: 4.86e-05 | 1333.0ms/step | 15.0 samples/s +Step 35280/44000 | Loss: 2.9280 | LR: 4.85e-05 | 1334.9ms/step | 15.0 samples/s +Step 35290/44000 | Loss: 2.9061 | LR: 4.84e-05 | 1330.1ms/step | 15.0 samples/s +Step 35300/44000 | Loss: 2.8927 | LR: 4.83e-05 | 1331.4ms/step | 15.0 samples/s +Step 35310/44000 | Loss: 2.8839 | LR: 4.82e-05 | 1339.6ms/step | 14.9 samples/s +Step 35320/44000 | Loss: 2.9162 | LR: 4.81e-05 | 1334.0ms/step | 15.0 samples/s +Step 35330/44000 | Loss: 2.9295 | LR: 4.80e-05 | 1332.2ms/step | 15.0 samples/s +Step 35340/44000 | Loss: 2.9042 | LR: 4.79e-05 | 1328.4ms/step | 15.1 samples/s +Step 35350/44000 | Loss: 2.9094 | LR: 4.77e-05 | 1325.9ms/step | 15.1 samples/s +Step 35360/44000 | Loss: 2.9283 | LR: 4.76e-05 | 1324.3ms/step | 15.1 samples/s +Step 35370/44000 | Loss: 2.9264 | LR: 4.75e-05 | 1324.5ms/step | 15.1 samples/s +Step 35380/44000 | Loss: 2.9227 | LR: 4.74e-05 | 1325.3ms/step | 15.1 samples/s +Step 35390/44000 | Loss: 2.9175 | LR: 4.73e-05 | 1324.9ms/step | 15.1 samples/s +Step 35400/44000 | Loss: 2.9199 | LR: 4.72e-05 | 1323.6ms/step | 15.1 samples/s +Step 35410/44000 | Loss: 2.8942 | LR: 4.71e-05 | 1323.9ms/step | 15.1 samples/s +Step 35420/44000 | Loss: 2.9590 | LR: 4.70e-05 | 1323.7ms/step | 15.1 samples/s +Step 35430/44000 | Loss: 2.9204 | LR: 4.69e-05 | 1325.4ms/step | 15.1 samples/s +Step 35440/44000 | Loss: 2.9265 | LR: 4.68e-05 | 1325.6ms/step | 15.1 samples/s +Step 35450/44000 | Loss: 2.9129 | LR: 4.67e-05 | 1326.3ms/step | 15.1 samples/s +Step 35460/44000 | Loss: 2.9112 | LR: 4.66e-05 | 1331.2ms/step | 15.0 samples/s +Step 35470/44000 | Loss: 2.9072 | LR: 4.65e-05 | 1332.5ms/step | 15.0 samples/s +Step 35480/44000 | Loss: 2.9172 | LR: 4.64e-05 | 1330.3ms/step | 15.0 samples/s +Step 35490/44000 | Loss: 2.9236 | LR: 4.63e-05 | 1331.8ms/step | 15.0 samples/s +Step 35500/44000 | Loss: 2.9390 | LR: 4.62e-05 | 1332.0ms/step | 15.0 samples/s +Validation at step 35500: Loss = 2.9269, Perplexity = 18.67 +Step 35510/44000 | Loss: 2.9179 | LR: 4.61e-05 | 3560.6ms/step | 5.6 samples/s +Step 35520/44000 | Loss: 2.9164 | LR: 4.59e-05 | 1330.0ms/step | 15.0 samples/s +Step 35530/44000 | Loss: 2.9551 | LR: 4.58e-05 | 1332.0ms/step | 15.0 samples/s +Step 35540/44000 | Loss: 2.9332 | LR: 4.57e-05 | 1330.4ms/step | 15.0 samples/s +Step 35550/44000 | Loss: 2.9461 | LR: 4.56e-05 | 1328.0ms/step | 15.1 samples/s +Step 35560/44000 | Loss: 2.9032 | LR: 4.55e-05 | 1334.3ms/step | 15.0 samples/s +Step 35570/44000 | Loss: 2.9251 | LR: 4.54e-05 | 1333.5ms/step | 15.0 samples/s +Step 35580/44000 | Loss: 2.9152 | LR: 4.53e-05 | 1329.2ms/step | 15.0 samples/s +Step 35590/44000 | Loss: 2.9376 | LR: 4.52e-05 | 1330.4ms/step | 15.0 samples/s +Step 35600/44000 | Loss: 2.9269 | LR: 4.51e-05 | 1333.9ms/step | 15.0 samples/s +Step 35610/44000 | Loss: 2.8871 | LR: 4.50e-05 | 1332.8ms/step | 15.0 samples/s +Step 35620/44000 | Loss: 2.9383 | LR: 4.49e-05 | 1333.8ms/step | 15.0 samples/s +Step 35630/44000 | Loss: 2.9394 | LR: 4.48e-05 | 1332.6ms/step | 15.0 samples/s +Step 35640/44000 | Loss: 2.9311 | LR: 4.47e-05 | 1332.4ms/step | 15.0 samples/s +Step 35650/44000 | Loss: 2.9148 | LR: 4.46e-05 | 1327.8ms/step | 15.1 samples/s +Step 35660/44000 | Loss: 2.9260 | LR: 4.45e-05 | 1330.4ms/step | 15.0 samples/s +Step 35670/44000 | Loss: 2.9131 | LR: 4.44e-05 | 1332.3ms/step | 15.0 samples/s +Step 35680/44000 | Loss: 2.9103 | LR: 4.43e-05 | 1331.5ms/step | 15.0 samples/s +Step 35690/44000 | Loss: 2.8997 | LR: 4.42e-05 | 1332.0ms/step | 15.0 samples/s +Step 35700/44000 | Loss: 2.9379 | LR: 4.41e-05 | 1333.8ms/step | 15.0 samples/s +Step 35710/44000 | Loss: 2.9058 | LR: 4.40e-05 | 1330.8ms/step | 15.0 samples/s +Step 35720/44000 | Loss: 2.9017 | LR: 4.39e-05 | 1335.3ms/step | 15.0 samples/s +Step 35730/44000 | Loss: 2.9277 | LR: 4.38e-05 | 1334.1ms/step | 15.0 samples/s +Step 35740/44000 | Loss: 2.8847 | LR: 4.37e-05 | 1332.8ms/step | 15.0 samples/s +Step 35750/44000 | Loss: 2.9171 | LR: 4.36e-05 | 1332.4ms/step | 15.0 samples/s +Validation at step 35750: Loss = 2.9267, Perplexity = 18.67 +Step 35760/44000 | Loss: 2.9232 | LR: 4.35e-05 | 3564.4ms/step | 5.6 samples/s +Step 35770/44000 | Loss: 2.9107 | LR: 4.34e-05 | 1330.7ms/step | 15.0 samples/s +Step 35780/44000 | Loss: 2.9224 | LR: 4.33e-05 | 1331.8ms/step | 15.0 samples/s +Step 35790/44000 | Loss: 2.9266 | LR: 4.32e-05 | 1331.9ms/step | 15.0 samples/s +Step 35800/44000 | Loss: 2.9155 | LR: 4.31e-05 | 1336.8ms/step | 15.0 samples/s +Step 35810/44000 | Loss: 2.9149 | LR: 4.29e-05 | 1332.9ms/step | 15.0 samples/s +Step 35820/44000 | Loss: 2.9081 | LR: 4.28e-05 | 1330.8ms/step | 15.0 samples/s +Step 35830/44000 | Loss: 2.9080 | LR: 4.27e-05 | 1335.4ms/step | 15.0 samples/s +Step 35840/44000 | Loss: 2.8937 | LR: 4.26e-05 | 1333.3ms/step | 15.0 samples/s +Step 35850/44000 | Loss: 2.9000 | LR: 4.25e-05 | 1331.2ms/step | 15.0 samples/s +Step 35860/44000 | Loss: 2.9122 | LR: 4.24e-05 | 1332.7ms/step | 15.0 samples/s +Step 35870/44000 | Loss: 2.9476 | LR: 4.23e-05 | 1333.7ms/step | 15.0 samples/s +Step 35880/44000 | Loss: 2.9164 | LR: 4.22e-05 | 1336.7ms/step | 15.0 samples/s +Step 35890/44000 | Loss: 2.9034 | LR: 4.21e-05 | 1335.3ms/step | 15.0 samples/s +Step 35900/44000 | Loss: 2.9030 | LR: 4.20e-05 | 1334.1ms/step | 15.0 samples/s +Step 35910/44000 | Loss: 2.9281 | LR: 4.19e-05 | 1332.3ms/step | 15.0 samples/s +Step 35920/44000 | Loss: 2.9276 | LR: 4.18e-05 | 1334.3ms/step | 15.0 samples/s +Step 35930/44000 | Loss: 2.9211 | LR: 4.17e-05 | 1335.1ms/step | 15.0 samples/s +Step 35940/44000 | Loss: 2.8941 | LR: 4.16e-05 | 1333.6ms/step | 15.0 samples/s +Step 35950/44000 | Loss: 2.9244 | LR: 4.15e-05 | 1327.8ms/step | 15.1 samples/s +Step 35960/44000 | Loss: 2.9271 | LR: 4.14e-05 | 1328.0ms/step | 15.1 samples/s +Step 35970/44000 | Loss: 2.9128 | LR: 4.13e-05 | 1330.3ms/step | 15.0 samples/s +Step 35980/44000 | Loss: 2.9259 | LR: 4.12e-05 | 1331.6ms/step | 15.0 samples/s +Step 35990/44000 | Loss: 2.8857 | LR: 4.11e-05 | 1335.3ms/step | 15.0 samples/s +Step 36000/44000 | Loss: 2.8962 | LR: 4.10e-05 | 1331.0ms/step | 15.0 samples/s +Validation at step 36000: Loss = 2.9265, Perplexity = 18.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-36000 +Step 36010/44000 | Loss: 2.9332 | LR: 4.09e-05 | 3872.7ms/step | 5.2 samples/s +Step 36020/44000 | Loss: 2.9167 | LR: 4.08e-05 | 1334.0ms/step | 15.0 samples/s +Step 36030/44000 | Loss: 2.8994 | LR: 4.07e-05 | 1330.4ms/step | 15.0 samples/s +Step 36040/44000 | Loss: 2.9100 | LR: 4.06e-05 | 1335.6ms/step | 15.0 samples/s +Step 36050/44000 | Loss: 2.9205 | LR: 4.05e-05 | 1331.2ms/step | 15.0 samples/s +Step 36060/44000 | Loss: 2.9164 | LR: 4.04e-05 | 1332.2ms/step | 15.0 samples/s +Step 36070/44000 | Loss: 2.9202 | LR: 4.03e-05 | 1331.2ms/step | 15.0 samples/s +Step 36080/44000 | Loss: 2.9047 | LR: 4.02e-05 | 1331.8ms/step | 15.0 samples/s +Step 36090/44000 | Loss: 2.9067 | LR: 4.01e-05 | 1331.8ms/step | 15.0 samples/s +Step 36100/44000 | Loss: 2.9173 | LR: 4.00e-05 | 1334.4ms/step | 15.0 samples/s +Step 36110/44000 | Loss: 2.9418 | LR: 3.99e-05 | 1332.4ms/step | 15.0 samples/s +Step 36120/44000 | Loss: 2.9369 | LR: 3.98e-05 | 1331.1ms/step | 15.0 samples/s +Step 36130/44000 | Loss: 2.9162 | LR: 3.97e-05 | 1329.0ms/step | 15.0 samples/s +Step 36140/44000 | Loss: 2.9279 | LR: 3.97e-05 | 1333.7ms/step | 15.0 samples/s +Step 36150/44000 | Loss: 2.9519 | LR: 3.96e-05 | 1332.5ms/step | 15.0 samples/s +Step 36160/44000 | Loss: 2.9409 | LR: 3.95e-05 | 1334.5ms/step | 15.0 samples/s +Step 36170/44000 | Loss: 2.9134 | LR: 3.94e-05 | 1335.2ms/step | 15.0 samples/s +Step 36180/44000 | Loss: 2.9556 | LR: 3.93e-05 | 1329.4ms/step | 15.0 samples/s +Step 36190/44000 | Loss: 2.9186 | LR: 3.92e-05 | 1337.0ms/step | 15.0 samples/s +Step 36200/44000 | Loss: 2.9374 | LR: 3.91e-05 | 1335.3ms/step | 15.0 samples/s +Step 36210/44000 | Loss: 2.9189 | LR: 3.90e-05 | 1337.6ms/step | 15.0 samples/s +Step 36220/44000 | Loss: 2.8984 | LR: 3.89e-05 | 1333.9ms/step | 15.0 samples/s +Step 36230/44000 | Loss: 2.9065 | LR: 3.88e-05 | 1332.0ms/step | 15.0 samples/s +Step 36240/44000 | Loss: 2.9191 | LR: 3.87e-05 | 1332.0ms/step | 15.0 samples/s +Step 36250/44000 | Loss: 2.9574 | LR: 3.86e-05 | 1330.4ms/step | 15.0 samples/s +Validation at step 36250: Loss = 2.9264, Perplexity = 18.66 +Step 36260/44000 | Loss: 2.9385 | LR: 3.85e-05 | 3550.0ms/step | 5.6 samples/s +Step 36270/44000 | Loss: 2.9041 | LR: 3.84e-05 | 1333.4ms/step | 15.0 samples/s +Step 36280/44000 | Loss: 2.9052 | LR: 3.83e-05 | 1330.4ms/step | 15.0 samples/s +Step 36290/44000 | Loss: 2.9236 | LR: 3.82e-05 | 1332.9ms/step | 15.0 samples/s +Step 36300/44000 | Loss: 2.9359 | LR: 3.81e-05 | 1335.0ms/step | 15.0 samples/s +Step 36310/44000 | Loss: 2.9143 | LR: 3.80e-05 | 1333.3ms/step | 15.0 samples/s +Step 36320/44000 | Loss: 2.9090 | LR: 3.79e-05 | 1332.1ms/step | 15.0 samples/s +Step 36330/44000 | Loss: 2.9449 | LR: 3.78e-05 | 1333.0ms/step | 15.0 samples/s +Step 36340/44000 | Loss: 2.9232 | LR: 3.77e-05 | 1331.6ms/step | 15.0 samples/s +Step 36350/44000 | Loss: 2.9168 | LR: 3.76e-05 | 1334.4ms/step | 15.0 samples/s +Step 36360/44000 | Loss: 2.9141 | LR: 3.75e-05 | 1336.1ms/step | 15.0 samples/s +Step 36370/44000 | Loss: 2.9611 | LR: 3.74e-05 | 1335.8ms/step | 15.0 samples/s +Step 36380/44000 | Loss: 2.9278 | LR: 3.73e-05 | 1329.4ms/step | 15.0 samples/s +Step 36390/44000 | Loss: 2.9021 | LR: 3.72e-05 | 1333.1ms/step | 15.0 samples/s +Step 36400/44000 | Loss: 2.9253 | LR: 3.71e-05 | 1332.2ms/step | 15.0 samples/s +Step 36410/44000 | Loss: 2.8922 | LR: 3.70e-05 | 1333.3ms/step | 15.0 samples/s +Step 36420/44000 | Loss: 2.9133 | LR: 3.69e-05 | 1332.7ms/step | 15.0 samples/s +Step 36430/44000 | Loss: 2.8685 | LR: 3.69e-05 | 1333.9ms/step | 15.0 samples/s +Step 36440/44000 | Loss: 2.9067 | LR: 3.68e-05 | 1333.3ms/step | 15.0 samples/s +Step 36450/44000 | Loss: 2.9078 | LR: 3.67e-05 | 1329.6ms/step | 15.0 samples/s +Step 36460/44000 | Loss: 2.8876 | LR: 3.66e-05 | 1331.8ms/step | 15.0 samples/s +Step 36470/44000 | Loss: 2.9224 | LR: 3.65e-05 | 1334.5ms/step | 15.0 samples/s +Step 36480/44000 | Loss: 2.9429 | LR: 3.64e-05 | 1335.3ms/step | 15.0 samples/s +Step 36490/44000 | Loss: 2.9015 | LR: 3.63e-05 | 1331.8ms/step | 15.0 samples/s +Step 36500/44000 | Loss: 2.9313 | LR: 3.62e-05 | 1330.3ms/step | 15.0 samples/s +Validation at step 36500: Loss = 2.9262, Perplexity = 18.66 +Step 36510/44000 | Loss: 2.9237 | LR: 3.61e-05 | 3556.7ms/step | 5.6 samples/s +Step 36520/44000 | Loss: 2.9201 | LR: 3.60e-05 | 1331.4ms/step | 15.0 samples/s +Step 36530/44000 | Loss: 2.9374 | LR: 3.59e-05 | 1331.5ms/step | 15.0 samples/s +Step 36540/44000 | Loss: 2.9417 | LR: 3.58e-05 | 1330.0ms/step | 15.0 samples/s +Step 36550/44000 | Loss: 2.9395 | LR: 3.57e-05 | 1330.4ms/step | 15.0 samples/s +Step 36560/44000 | Loss: 2.9076 | LR: 3.56e-05 | 1334.9ms/step | 15.0 samples/s +Step 36570/44000 | Loss: 2.9114 | LR: 3.55e-05 | 1331.0ms/step | 15.0 samples/s +Step 36580/44000 | Loss: 2.8957 | LR: 3.54e-05 | 1333.2ms/step | 15.0 samples/s +Step 36590/44000 | Loss: 2.9363 | LR: 3.53e-05 | 1327.9ms/step | 15.1 samples/s +Step 36600/44000 | Loss: 2.8894 | LR: 3.53e-05 | 1334.0ms/step | 15.0 samples/s +Step 36610/44000 | Loss: 2.8967 | LR: 3.52e-05 | 1333.9ms/step | 15.0 samples/s +Step 36620/44000 | Loss: 2.9331 | LR: 3.51e-05 | 1330.0ms/step | 15.0 samples/s +Step 36630/44000 | Loss: 2.8939 | LR: 3.50e-05 | 1330.3ms/step | 15.0 samples/s +Step 36640/44000 | Loss: 2.8891 | LR: 3.49e-05 | 1335.3ms/step | 15.0 samples/s +Step 36650/44000 | Loss: 2.9089 | LR: 3.48e-05 | 1336.0ms/step | 15.0 samples/s +Step 36660/44000 | Loss: 2.8896 | LR: 3.47e-05 | 1331.8ms/step | 15.0 samples/s +Step 36670/44000 | Loss: 2.8899 | LR: 3.46e-05 | 1332.0ms/step | 15.0 samples/s +Step 36680/44000 | Loss: 2.9420 | LR: 3.45e-05 | 1329.3ms/step | 15.0 samples/s +Step 36690/44000 | Loss: 2.9380 | LR: 3.44e-05 | 1329.3ms/step | 15.0 samples/s +Step 36700/44000 | Loss: 2.9253 | LR: 3.43e-05 | 1333.0ms/step | 15.0 samples/s +Step 36710/44000 | Loss: 2.9407 | LR: 3.42e-05 | 1328.2ms/step | 15.1 samples/s +Step 36720/44000 | Loss: 2.9176 | LR: 3.41e-05 | 1328.6ms/step | 15.1 samples/s +Step 36730/44000 | Loss: 2.9164 | LR: 3.41e-05 | 1330.2ms/step | 15.0 samples/s +Step 36740/44000 | Loss: 2.9147 | LR: 3.40e-05 | 1330.0ms/step | 15.0 samples/s +Step 36750/44000 | Loss: 2.9264 | LR: 3.39e-05 | 1331.0ms/step | 15.0 samples/s +Validation at step 36750: Loss = 2.9261, Perplexity = 18.65 +Step 36760/44000 | Loss: 2.8977 | LR: 3.38e-05 | 3562.2ms/step | 5.6 samples/s +Step 36770/44000 | Loss: 2.9443 | LR: 3.37e-05 | 1330.1ms/step | 15.0 samples/s +Step 36780/44000 | Loss: 2.9267 | LR: 3.36e-05 | 1329.0ms/step | 15.0 samples/s +Step 36790/44000 | Loss: 2.9429 | LR: 3.35e-05 | 1332.2ms/step | 15.0 samples/s +Step 36800/44000 | Loss: 2.9073 | LR: 3.34e-05 | 1333.9ms/step | 15.0 samples/s +Step 36810/44000 | Loss: 2.8962 | LR: 3.33e-05 | 1332.0ms/step | 15.0 samples/s +Step 36820/44000 | Loss: 2.9370 | LR: 3.32e-05 | 1335.6ms/step | 15.0 samples/s +Step 36830/44000 | Loss: 2.9204 | LR: 3.31e-05 | 1335.9ms/step | 15.0 samples/s +Step 36840/44000 | Loss: 2.9147 | LR: 3.31e-05 | 1329.6ms/step | 15.0 samples/s +Step 36850/44000 | Loss: 2.9309 | LR: 3.30e-05 | 1331.5ms/step | 15.0 samples/s +Step 36860/44000 | Loss: 2.9179 | LR: 3.29e-05 | 1329.8ms/step | 15.0 samples/s +Step 36870/44000 | Loss: 2.9302 | LR: 3.28e-05 | 1333.8ms/step | 15.0 samples/s +Step 36880/44000 | Loss: 2.9318 | LR: 3.27e-05 | 1330.4ms/step | 15.0 samples/s +Step 36890/44000 | Loss: 2.9094 | LR: 3.26e-05 | 1332.2ms/step | 15.0 samples/s +Step 36900/44000 | Loss: 2.8996 | LR: 3.25e-05 | 1331.6ms/step | 15.0 samples/s +Step 36910/44000 | Loss: 2.9355 | LR: 3.24e-05 | 1331.9ms/step | 15.0 samples/s +Step 36920/44000 | Loss: 2.9287 | LR: 3.23e-05 | 1332.4ms/step | 15.0 samples/s +Step 36930/44000 | Loss: 2.9046 | LR: 3.22e-05 | 1333.1ms/step | 15.0 samples/s +Step 36940/44000 | Loss: 2.9029 | LR: 3.22e-05 | 1331.2ms/step | 15.0 samples/s +Step 36950/44000 | Loss: 2.8824 | LR: 3.21e-05 | 1331.2ms/step | 15.0 samples/s +Step 36960/44000 | Loss: 2.9155 | LR: 3.20e-05 | 1332.2ms/step | 15.0 samples/s +Step 36970/44000 | Loss: 2.9105 | LR: 3.19e-05 | 1333.7ms/step | 15.0 samples/s +Step 36980/44000 | Loss: 2.8911 | LR: 3.18e-05 | 1332.2ms/step | 15.0 samples/s +Step 36990/44000 | Loss: 2.9224 | LR: 3.17e-05 | 1329.7ms/step | 15.0 samples/s +Step 37000/44000 | Loss: 2.9471 | LR: 3.16e-05 | 1331.6ms/step | 15.0 samples/s +Validation at step 37000: Loss = 2.9260, Perplexity = 18.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-37000 +Step 37010/44000 | Loss: 2.9196 | LR: 3.15e-05 | 3858.8ms/step | 5.2 samples/s +Step 37020/44000 | Loss: 2.9084 | LR: 3.15e-05 | 1331.4ms/step | 15.0 samples/s +Step 37030/44000 | Loss: 2.9002 | LR: 3.14e-05 | 1334.5ms/step | 15.0 samples/s +Step 37040/44000 | Loss: 2.9156 | LR: 3.13e-05 | 1332.2ms/step | 15.0 samples/s +Step 37050/44000 | Loss: 2.9144 | LR: 3.12e-05 | 1333.7ms/step | 15.0 samples/s +Step 37060/44000 | Loss: 2.9376 | LR: 3.11e-05 | 1333.0ms/step | 15.0 samples/s +Step 37070/44000 | Loss: 2.9138 | LR: 3.10e-05 | 1330.3ms/step | 15.0 samples/s +Step 37080/44000 | Loss: 2.9209 | LR: 3.09e-05 | 1327.5ms/step | 15.1 samples/s +Step 37090/44000 | Loss: 2.9268 | LR: 3.08e-05 | 1327.0ms/step | 15.1 samples/s +Step 37100/44000 | Loss: 2.9228 | LR: 3.07e-05 | 1330.0ms/step | 15.0 samples/s +Step 37110/44000 | Loss: 2.8971 | LR: 3.07e-05 | 1331.2ms/step | 15.0 samples/s +Step 37120/44000 | Loss: 2.9221 | LR: 3.06e-05 | 1335.4ms/step | 15.0 samples/s +Step 37130/44000 | Loss: 2.9359 | LR: 3.05e-05 | 1331.1ms/step | 15.0 samples/s +Step 37140/44000 | Loss: 2.9049 | LR: 3.04e-05 | 1331.6ms/step | 15.0 samples/s +Step 37150/44000 | Loss: 2.8908 | LR: 3.03e-05 | 1333.2ms/step | 15.0 samples/s +Step 37160/44000 | Loss: 2.9206 | LR: 3.02e-05 | 1329.1ms/step | 15.0 samples/s +Step 37170/44000 | Loss: 2.9159 | LR: 3.01e-05 | 1330.7ms/step | 15.0 samples/s +Step 37180/44000 | Loss: 2.8926 | LR: 3.01e-05 | 1332.1ms/step | 15.0 samples/s +Step 37190/44000 | Loss: 2.9223 | LR: 3.00e-05 | 1331.1ms/step | 15.0 samples/s +Step 37200/44000 | Loss: 2.9323 | LR: 2.99e-05 | 1332.3ms/step | 15.0 samples/s +Step 37210/44000 | Loss: 2.9081 | LR: 2.98e-05 | 1330.0ms/step | 15.0 samples/s +Step 37220/44000 | Loss: 2.9066 | LR: 2.97e-05 | 1325.9ms/step | 15.1 samples/s +Step 37230/44000 | Loss: 2.9287 | LR: 2.96e-05 | 1331.4ms/step | 15.0 samples/s +Step 37240/44000 | Loss: 2.9401 | LR: 2.95e-05 | 1332.2ms/step | 15.0 samples/s +Step 37250/44000 | Loss: 2.9373 | LR: 2.95e-05 | 1333.5ms/step | 15.0 samples/s +Validation at step 37250: Loss = 2.9259, Perplexity = 18.65 +Step 37260/44000 | Loss: 2.9095 | LR: 2.94e-05 | 3547.9ms/step | 5.6 samples/s +Step 37270/44000 | Loss: 2.9419 | LR: 2.93e-05 | 1328.6ms/step | 15.1 samples/s +Step 37280/44000 | Loss: 2.9185 | LR: 2.92e-05 | 1327.9ms/step | 15.1 samples/s +Step 37290/44000 | Loss: 2.9267 | LR: 2.91e-05 | 1326.6ms/step | 15.1 samples/s +Step 37300/44000 | Loss: 2.9348 | LR: 2.90e-05 | 1325.5ms/step | 15.1 samples/s +Step 37310/44000 | Loss: 2.9056 | LR: 2.89e-05 | 1326.2ms/step | 15.1 samples/s +Step 37320/44000 | Loss: 2.9632 | LR: 2.89e-05 | 1326.1ms/step | 15.1 samples/s +Step 37330/44000 | Loss: 2.8971 | LR: 2.88e-05 | 1327.0ms/step | 15.1 samples/s +Step 37340/44000 | Loss: 2.9166 | LR: 2.87e-05 | 1330.1ms/step | 15.0 samples/s +Step 37350/44000 | Loss: 2.9230 | LR: 2.86e-05 | 1331.0ms/step | 15.0 samples/s +Step 37360/44000 | Loss: 2.9502 | LR: 2.85e-05 | 1331.7ms/step | 15.0 samples/s +Step 37370/44000 | Loss: 2.9100 | LR: 2.84e-05 | 1331.1ms/step | 15.0 samples/s +Step 37380/44000 | Loss: 2.9211 | LR: 2.84e-05 | 1330.6ms/step | 15.0 samples/s +Step 37390/44000 | Loss: 2.9164 | LR: 2.83e-05 | 1333.5ms/step | 15.0 samples/s +Step 37400/44000 | Loss: 2.9348 | LR: 2.82e-05 | 1332.0ms/step | 15.0 samples/s +Step 37410/44000 | Loss: 2.8937 | LR: 2.81e-05 | 1331.8ms/step | 15.0 samples/s +Step 37420/44000 | Loss: 2.9201 | LR: 2.80e-05 | 1329.7ms/step | 15.0 samples/s +Step 37430/44000 | Loss: 2.9249 | LR: 2.79e-05 | 1331.2ms/step | 15.0 samples/s +Step 37440/44000 | Loss: 2.9006 | LR: 2.78e-05 | 1335.5ms/step | 15.0 samples/s +Step 37450/44000 | Loss: 2.9423 | LR: 2.78e-05 | 1333.2ms/step | 15.0 samples/s +Step 37460/44000 | Loss: 2.9094 | LR: 2.77e-05 | 1336.0ms/step | 15.0 samples/s +Step 37470/44000 | Loss: 2.8969 | LR: 2.76e-05 | 1330.5ms/step | 15.0 samples/s +Step 37480/44000 | Loss: 2.9367 | LR: 2.75e-05 | 1334.6ms/step | 15.0 samples/s +Step 37490/44000 | Loss: 2.9166 | LR: 2.74e-05 | 1331.1ms/step | 15.0 samples/s +Step 37500/44000 | Loss: 2.8914 | LR: 2.74e-05 | 1327.9ms/step | 15.1 samples/s +Validation at step 37500: Loss = 2.9258, Perplexity = 18.65 +Step 37510/44000 | Loss: 2.9262 | LR: 2.73e-05 | 3555.1ms/step | 5.6 samples/s +Step 37520/44000 | Loss: 2.9428 | LR: 2.72e-05 | 1328.6ms/step | 15.1 samples/s +Step 37530/44000 | Loss: 2.9240 | LR: 2.71e-05 | 1329.5ms/step | 15.0 samples/s +Step 37540/44000 | Loss: 2.9442 | LR: 2.70e-05 | 1328.8ms/step | 15.1 samples/s +Step 37550/44000 | Loss: 2.9093 | LR: 2.69e-05 | 1334.0ms/step | 15.0 samples/s +Step 37560/44000 | Loss: 2.9168 | LR: 2.69e-05 | 1334.9ms/step | 15.0 samples/s +Step 37570/44000 | Loss: 2.9265 | LR: 2.68e-05 | 1327.2ms/step | 15.1 samples/s +Step 37580/44000 | Loss: 2.9114 | LR: 2.67e-05 | 1328.1ms/step | 15.1 samples/s +Step 37590/44000 | Loss: 2.9238 | LR: 2.66e-05 | 1332.2ms/step | 15.0 samples/s +Step 37600/44000 | Loss: 2.9100 | LR: 2.65e-05 | 1331.8ms/step | 15.0 samples/s +Step 37610/44000 | Loss: 2.9112 | LR: 2.65e-05 | 1329.2ms/step | 15.0 samples/s +Step 37620/44000 | Loss: 2.9216 | LR: 2.64e-05 | 1327.8ms/step | 15.1 samples/s +Step 37630/44000 | Loss: 2.9232 | LR: 2.63e-05 | 1330.5ms/step | 15.0 samples/s +Step 37640/44000 | Loss: 2.9271 | LR: 2.62e-05 | 1329.4ms/step | 15.0 samples/s +Step 37650/44000 | Loss: 2.9292 | LR: 2.61e-05 | 1331.9ms/step | 15.0 samples/s +Step 37660/44000 | Loss: 2.9368 | LR: 2.60e-05 | 1330.5ms/step | 15.0 samples/s +Step 37670/44000 | Loss: 2.8890 | LR: 2.60e-05 | 1328.4ms/step | 15.1 samples/s +Step 37680/44000 | Loss: 2.9337 | LR: 2.59e-05 | 1333.8ms/step | 15.0 samples/s +Step 37690/44000 | Loss: 2.9304 | LR: 2.58e-05 | 1334.5ms/step | 15.0 samples/s +Step 37700/44000 | Loss: 2.9038 | LR: 2.57e-05 | 1331.4ms/step | 15.0 samples/s +Step 37710/44000 | Loss: 2.8910 | LR: 2.56e-05 | 1330.1ms/step | 15.0 samples/s +Step 37720/44000 | Loss: 2.9123 | LR: 2.56e-05 | 1330.3ms/step | 15.0 samples/s +Step 37730/44000 | Loss: 2.9109 | LR: 2.55e-05 | 1325.8ms/step | 15.1 samples/s +Step 37740/44000 | Loss: 2.9443 | LR: 2.54e-05 | 1327.0ms/step | 15.1 samples/s +Step 37750/44000 | Loss: 2.9067 | LR: 2.53e-05 | 1327.9ms/step | 15.1 samples/s +Validation at step 37750: Loss = 2.9257, Perplexity = 18.65 +Step 37760/44000 | Loss: 2.9148 | LR: 2.52e-05 | 3562.2ms/step | 5.6 samples/s +Step 37770/44000 | Loss: 2.9258 | LR: 2.52e-05 | 1329.0ms/step | 15.0 samples/s +Step 37780/44000 | Loss: 2.8873 | LR: 2.51e-05 | 1328.8ms/step | 15.1 samples/s +Step 37790/44000 | Loss: 2.9151 | LR: 2.50e-05 | 1331.1ms/step | 15.0 samples/s +Step 37800/44000 | Loss: 2.9160 | LR: 2.49e-05 | 1329.1ms/step | 15.0 samples/s +Step 37810/44000 | Loss: 2.9057 | LR: 2.48e-05 | 1329.2ms/step | 15.0 samples/s +Step 37820/44000 | Loss: 2.9213 | LR: 2.48e-05 | 1329.6ms/step | 15.0 samples/s +Step 37830/44000 | Loss: 2.9389 | LR: 2.47e-05 | 1334.3ms/step | 15.0 samples/s +Step 37840/44000 | Loss: 2.8802 | LR: 2.46e-05 | 1334.3ms/step | 15.0 samples/s +Step 37850/44000 | Loss: 2.9239 | LR: 2.45e-05 | 1331.5ms/step | 15.0 samples/s +Step 37860/44000 | Loss: 2.8930 | LR: 2.45e-05 | 1329.0ms/step | 15.0 samples/s +Step 37870/44000 | Loss: 2.9086 | LR: 2.44e-05 | 1330.9ms/step | 15.0 samples/s +Step 37880/44000 | Loss: 2.8845 | LR: 2.43e-05 | 1329.4ms/step | 15.0 samples/s +Step 37890/44000 | Loss: 2.9040 | LR: 2.42e-05 | 1328.2ms/step | 15.1 samples/s +Step 37900/44000 | Loss: 2.9361 | LR: 2.41e-05 | 1332.5ms/step | 15.0 samples/s +Step 37910/44000 | Loss: 2.9132 | LR: 2.41e-05 | 1332.3ms/step | 15.0 samples/s +Step 37920/44000 | Loss: 2.9134 | LR: 2.40e-05 | 1331.6ms/step | 15.0 samples/s +Step 37930/44000 | Loss: 2.9248 | LR: 2.39e-05 | 1331.2ms/step | 15.0 samples/s +Step 37940/44000 | Loss: 2.9178 | LR: 2.38e-05 | 1330.2ms/step | 15.0 samples/s +Step 37950/44000 | Loss: 2.8938 | LR: 2.38e-05 | 1332.6ms/step | 15.0 samples/s +Step 37960/44000 | Loss: 2.9497 | LR: 2.37e-05 | 1334.7ms/step | 15.0 samples/s +Step 37970/44000 | Loss: 2.8857 | LR: 2.36e-05 | 1327.7ms/step | 15.1 samples/s +Step 37980/44000 | Loss: 2.8892 | LR: 2.35e-05 | 1333.8ms/step | 15.0 samples/s +Step 37990/44000 | Loss: 2.8964 | LR: 2.34e-05 | 1331.2ms/step | 15.0 samples/s +Step 38000/44000 | Loss: 2.9199 | LR: 2.34e-05 | 1332.8ms/step | 15.0 samples/s +Validation at step 38000: Loss = 2.9256, Perplexity = 18.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-38000 +Step 38010/44000 | Loss: 2.9323 | LR: 2.33e-05 | 3861.6ms/step | 5.2 samples/s +Step 38020/44000 | Loss: 2.9398 | LR: 2.32e-05 | 1331.8ms/step | 15.0 samples/s +Step 38030/44000 | Loss: 2.8930 | LR: 2.31e-05 | 1333.6ms/step | 15.0 samples/s +Step 38040/44000 | Loss: 2.9151 | LR: 2.31e-05 | 1330.5ms/step | 15.0 samples/s +Step 38050/44000 | Loss: 2.9229 | LR: 2.30e-05 | 1332.5ms/step | 15.0 samples/s +Step 38060/44000 | Loss: 2.9373 | LR: 2.29e-05 | 1337.3ms/step | 15.0 samples/s +Step 38070/44000 | Loss: 2.9148 | LR: 2.28e-05 | 1329.1ms/step | 15.0 samples/s +Step 38080/44000 | Loss: 2.8793 | LR: 2.28e-05 | 1331.6ms/step | 15.0 samples/s +Step 38090/44000 | Loss: 2.8931 | LR: 2.27e-05 | 1335.3ms/step | 15.0 samples/s +Step 38100/44000 | Loss: 2.9358 | LR: 2.26e-05 | 1331.1ms/step | 15.0 samples/s +Step 38110/44000 | Loss: 2.8930 | LR: 2.25e-05 | 1332.7ms/step | 15.0 samples/s +Step 38120/44000 | Loss: 2.9178 | LR: 2.25e-05 | 1330.6ms/step | 15.0 samples/s +Step 38130/44000 | Loss: 2.9064 | LR: 2.24e-05 | 1336.0ms/step | 15.0 samples/s +Step 38140/44000 | Loss: 2.8914 | LR: 2.23e-05 | 1332.9ms/step | 15.0 samples/s +Step 38150/44000 | Loss: 2.9211 | LR: 2.22e-05 | 1331.2ms/step | 15.0 samples/s +Step 38160/44000 | Loss: 2.9246 | LR: 2.22e-05 | 1331.2ms/step | 15.0 samples/s +Step 38170/44000 | Loss: 2.9144 | LR: 2.21e-05 | 1326.5ms/step | 15.1 samples/s +Step 38180/44000 | Loss: 2.9359 | LR: 2.20e-05 | 1327.7ms/step | 15.1 samples/s +Step 38190/44000 | Loss: 2.9419 | LR: 2.19e-05 | 1325.3ms/step | 15.1 samples/s +Step 38200/44000 | Loss: 2.8905 | LR: 2.19e-05 | 1325.7ms/step | 15.1 samples/s +Step 38210/44000 | Loss: 2.9017 | LR: 2.18e-05 | 1326.6ms/step | 15.1 samples/s +Step 38220/44000 | Loss: 2.9387 | LR: 2.17e-05 | 1328.6ms/step | 15.1 samples/s +Step 38230/44000 | Loss: 2.9371 | LR: 2.16e-05 | 1328.8ms/step | 15.1 samples/s +Step 38240/44000 | Loss: 2.9481 | LR: 2.16e-05 | 1322.8ms/step | 15.1 samples/s +Step 38250/44000 | Loss: 2.9090 | LR: 2.15e-05 | 1324.9ms/step | 15.1 samples/s +Validation at step 38250: Loss = 2.9256, Perplexity = 18.64 +Step 38260/44000 | Loss: 2.9247 | LR: 2.14e-05 | 3562.3ms/step | 5.6 samples/s +Step 38270/44000 | Loss: 2.9593 | LR: 2.13e-05 | 1332.4ms/step | 15.0 samples/s +Step 38280/44000 | Loss: 2.9310 | LR: 2.13e-05 | 1331.6ms/step | 15.0 samples/s +Step 38290/44000 | Loss: 2.9154 | LR: 2.12e-05 | 1333.9ms/step | 15.0 samples/s +Step 38300/44000 | Loss: 2.9430 | LR: 2.11e-05 | 1333.6ms/step | 15.0 samples/s +Step 38310/44000 | Loss: 2.9460 | LR: 2.11e-05 | 1332.8ms/step | 15.0 samples/s +Step 38320/44000 | Loss: 2.8842 | LR: 2.10e-05 | 1333.3ms/step | 15.0 samples/s +Step 38330/44000 | Loss: 2.9024 | LR: 2.09e-05 | 1331.5ms/step | 15.0 samples/s +Step 38340/44000 | Loss: 2.9285 | LR: 2.08e-05 | 1333.2ms/step | 15.0 samples/s +Step 38350/44000 | Loss: 2.9094 | LR: 2.08e-05 | 1332.1ms/step | 15.0 samples/s +Step 38360/44000 | Loss: 2.9051 | LR: 2.07e-05 | 1331.1ms/step | 15.0 samples/s +Step 38370/44000 | Loss: 2.9058 | LR: 2.06e-05 | 1332.3ms/step | 15.0 samples/s +Step 38380/44000 | Loss: 2.9009 | LR: 2.05e-05 | 1332.2ms/step | 15.0 samples/s +Step 38390/44000 | Loss: 2.9050 | LR: 2.05e-05 | 1330.8ms/step | 15.0 samples/s +Step 38400/44000 | Loss: 2.8840 | LR: 2.04e-05 | 1326.6ms/step | 15.1 samples/s +Step 38410/44000 | Loss: 2.9255 | LR: 2.03e-05 | 1331.2ms/step | 15.0 samples/s +Step 38420/44000 | Loss: 2.8813 | LR: 2.03e-05 | 1331.6ms/step | 15.0 samples/s +Step 38430/44000 | Loss: 2.8966 | LR: 2.02e-05 | 1326.6ms/step | 15.1 samples/s +Step 38440/44000 | Loss: 2.9259 | LR: 2.01e-05 | 1333.2ms/step | 15.0 samples/s +Step 38450/44000 | Loss: 2.9199 | LR: 2.00e-05 | 1331.5ms/step | 15.0 samples/s +Step 38460/44000 | Loss: 2.9488 | LR: 2.00e-05 | 1330.5ms/step | 15.0 samples/s +Step 38470/44000 | Loss: 2.9262 | LR: 1.99e-05 | 1336.2ms/step | 15.0 samples/s +Step 38480/44000 | Loss: 2.9123 | LR: 1.98e-05 | 1334.5ms/step | 15.0 samples/s +Step 38490/44000 | Loss: 2.9104 | LR: 1.98e-05 | 1335.1ms/step | 15.0 samples/s +Step 38500/44000 | Loss: 2.9195 | LR: 1.97e-05 | 1331.1ms/step | 15.0 samples/s +Validation at step 38500: Loss = 2.9255, Perplexity = 18.64 +Step 38510/44000 | Loss: 2.9042 | LR: 1.96e-05 | 3564.2ms/step | 5.6 samples/s +Step 38520/44000 | Loss: 2.9217 | LR: 1.95e-05 | 1334.2ms/step | 15.0 samples/s +Step 38530/44000 | Loss: 2.9061 | LR: 1.95e-05 | 1331.5ms/step | 15.0 samples/s +Step 38540/44000 | Loss: 2.9302 | LR: 1.94e-05 | 1332.4ms/step | 15.0 samples/s +Step 38550/44000 | Loss: 2.9405 | LR: 1.93e-05 | 1332.8ms/step | 15.0 samples/s +Step 38560/44000 | Loss: 2.9152 | LR: 1.93e-05 | 1325.5ms/step | 15.1 samples/s +Step 38570/44000 | Loss: 2.9732 | LR: 1.92e-05 | 1325.1ms/step | 15.1 samples/s +Step 38580/44000 | Loss: 2.9076 | LR: 1.91e-05 | 1329.2ms/step | 15.0 samples/s +Step 38590/44000 | Loss: 2.8993 | LR: 1.91e-05 | 1327.6ms/step | 15.1 samples/s +Step 38600/44000 | Loss: 2.9202 | LR: 1.90e-05 | 1332.3ms/step | 15.0 samples/s +Step 38610/44000 | Loss: 2.9079 | LR: 1.89e-05 | 1331.5ms/step | 15.0 samples/s +Step 38620/44000 | Loss: 2.9200 | LR: 1.88e-05 | 1331.7ms/step | 15.0 samples/s +Step 38630/44000 | Loss: 2.9022 | LR: 1.88e-05 | 1330.5ms/step | 15.0 samples/s +Step 38640/44000 | Loss: 2.9112 | LR: 1.87e-05 | 1333.4ms/step | 15.0 samples/s +Step 38650/44000 | Loss: 2.9230 | LR: 1.86e-05 | 1331.8ms/step | 15.0 samples/s +Step 38660/44000 | Loss: 2.8774 | LR: 1.86e-05 | 1331.3ms/step | 15.0 samples/s +Step 38670/44000 | Loss: 2.9364 | LR: 1.85e-05 | 1332.5ms/step | 15.0 samples/s +Step 38680/44000 | Loss: 2.9118 | LR: 1.84e-05 | 1330.6ms/step | 15.0 samples/s +Step 38690/44000 | Loss: 2.9078 | LR: 1.84e-05 | 1334.4ms/step | 15.0 samples/s +Step 38700/44000 | Loss: 2.9096 | LR: 1.83e-05 | 1332.8ms/step | 15.0 samples/s +Step 38710/44000 | Loss: 2.9315 | LR: 1.82e-05 | 1330.8ms/step | 15.0 samples/s +Step 38720/44000 | Loss: 2.8870 | LR: 1.82e-05 | 1330.9ms/step | 15.0 samples/s +Step 38730/44000 | Loss: 2.9338 | LR: 1.81e-05 | 1337.7ms/step | 15.0 samples/s +Step 38740/44000 | Loss: 2.9275 | LR: 1.80e-05 | 1334.0ms/step | 15.0 samples/s +Step 38750/44000 | Loss: 2.9198 | LR: 1.80e-05 | 1326.8ms/step | 15.1 samples/s +Validation at step 38750: Loss = 2.9255, Perplexity = 18.64 +Step 38760/44000 | Loss: 2.9174 | LR: 1.79e-05 | 3551.5ms/step | 5.6 samples/s +Step 38770/44000 | Loss: 2.9123 | LR: 1.78e-05 | 1331.4ms/step | 15.0 samples/s +Step 38780/44000 | Loss: 2.9087 | LR: 1.78e-05 | 1332.6ms/step | 15.0 samples/s +Step 38790/44000 | Loss: 2.9274 | LR: 1.77e-05 | 1332.4ms/step | 15.0 samples/s +Step 38800/44000 | Loss: 2.9121 | LR: 1.76e-05 | 1329.7ms/step | 15.0 samples/s +Step 38810/44000 | Loss: 2.8975 | LR: 1.76e-05 | 1332.5ms/step | 15.0 samples/s +Step 38820/44000 | Loss: 2.9046 | LR: 1.75e-05 | 1330.8ms/step | 15.0 samples/s +Step 38830/44000 | Loss: 2.9288 | LR: 1.74e-05 | 1326.3ms/step | 15.1 samples/s +Step 38840/44000 | Loss: 2.8946 | LR: 1.74e-05 | 1331.0ms/step | 15.0 samples/s +Step 38850/44000 | Loss: 2.9470 | LR: 1.73e-05 | 1334.7ms/step | 15.0 samples/s +Step 38860/44000 | Loss: 2.9508 | LR: 1.72e-05 | 1332.1ms/step | 15.0 samples/s +Step 38870/44000 | Loss: 2.9375 | LR: 1.72e-05 | 1333.7ms/step | 15.0 samples/s +Step 38880/44000 | Loss: 2.9229 | LR: 1.71e-05 | 1332.2ms/step | 15.0 samples/s +Step 38890/44000 | Loss: 2.9195 | LR: 1.70e-05 | 1333.7ms/step | 15.0 samples/s +Step 38900/44000 | Loss: 2.9705 | LR: 1.70e-05 | 1331.4ms/step | 15.0 samples/s +Step 38910/44000 | Loss: 2.9006 | LR: 1.69e-05 | 1332.1ms/step | 15.0 samples/s +Step 38920/44000 | Loss: 2.9048 | LR: 1.68e-05 | 1329.1ms/step | 15.0 samples/s +Step 38930/44000 | Loss: 2.9104 | LR: 1.68e-05 | 1330.3ms/step | 15.0 samples/s +Step 38940/44000 | Loss: 2.9241 | LR: 1.67e-05 | 1329.2ms/step | 15.0 samples/s +Step 38950/44000 | Loss: 2.8981 | LR: 1.66e-05 | 1330.8ms/step | 15.0 samples/s +Step 38960/44000 | Loss: 2.9152 | LR: 1.66e-05 | 1331.8ms/step | 15.0 samples/s +Step 38970/44000 | Loss: 2.8775 | LR: 1.65e-05 | 1332.0ms/step | 15.0 samples/s +Step 38980/44000 | Loss: 2.9221 | LR: 1.64e-05 | 1331.0ms/step | 15.0 samples/s +Step 38990/44000 | Loss: 2.9239 | LR: 1.64e-05 | 1333.7ms/step | 15.0 samples/s +Step 39000/44000 | Loss: 2.9154 | LR: 1.63e-05 | 1330.7ms/step | 15.0 samples/s +Validation at step 39000: Loss = 2.9254, Perplexity = 18.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-39000 +Step 39010/44000 | Loss: 2.9023 | LR: 1.62e-05 | 3856.5ms/step | 5.2 samples/s +Step 39020/44000 | Loss: 2.9132 | LR: 1.62e-05 | 1326.8ms/step | 15.1 samples/s +Step 39030/44000 | Loss: 2.9029 | LR: 1.61e-05 | 1330.5ms/step | 15.0 samples/s +Step 39040/44000 | Loss: 2.9118 | LR: 1.61e-05 | 1330.4ms/step | 15.0 samples/s +Step 39050/44000 | Loss: 2.9377 | LR: 1.60e-05 | 1329.4ms/step | 15.0 samples/s +Step 39060/44000 | Loss: 2.9276 | LR: 1.59e-05 | 1332.7ms/step | 15.0 samples/s +Step 39070/44000 | Loss: 2.9219 | LR: 1.59e-05 | 1325.6ms/step | 15.1 samples/s +Step 39080/44000 | Loss: 2.9043 | LR: 1.58e-05 | 1327.5ms/step | 15.1 samples/s +Step 39090/44000 | Loss: 2.9059 | LR: 1.57e-05 | 1329.3ms/step | 15.0 samples/s +Step 39100/44000 | Loss: 2.8859 | LR: 1.57e-05 | 1331.9ms/step | 15.0 samples/s +Step 39110/44000 | Loss: 2.9282 | LR: 1.56e-05 | 1334.4ms/step | 15.0 samples/s +Step 39120/44000 | Loss: 2.8946 | LR: 1.55e-05 | 1328.5ms/step | 15.1 samples/s +Step 39130/44000 | Loss: 2.9100 | LR: 1.55e-05 | 1334.7ms/step | 15.0 samples/s +Step 39140/44000 | Loss: 2.9156 | LR: 1.54e-05 | 1331.8ms/step | 15.0 samples/s +Step 39150/44000 | Loss: 2.8848 | LR: 1.54e-05 | 1332.1ms/step | 15.0 samples/s +Step 39160/44000 | Loss: 2.9631 | LR: 1.53e-05 | 1331.3ms/step | 15.0 samples/s +Step 39170/44000 | Loss: 2.9134 | LR: 1.52e-05 | 1335.5ms/step | 15.0 samples/s +Step 39180/44000 | Loss: 2.9268 | LR: 1.52e-05 | 1331.1ms/step | 15.0 samples/s +Step 39190/44000 | Loss: 2.9289 | LR: 1.51e-05 | 1327.8ms/step | 15.1 samples/s +Step 39200/44000 | Loss: 2.8974 | LR: 1.50e-05 | 1327.7ms/step | 15.1 samples/s +Step 39210/44000 | Loss: 2.8954 | LR: 1.50e-05 | 1329.8ms/step | 15.0 samples/s +Step 39220/44000 | Loss: 2.9215 | LR: 1.49e-05 | 1330.3ms/step | 15.0 samples/s +Step 39230/44000 | Loss: 2.9103 | LR: 1.49e-05 | 1328.6ms/step | 15.1 samples/s +Step 39240/44000 | Loss: 2.9127 | LR: 1.48e-05 | 1328.4ms/step | 15.1 samples/s +Step 39250/44000 | Loss: 2.9537 | LR: 1.47e-05 | 1327.3ms/step | 15.1 samples/s +Validation at step 39250: Loss = 2.9254, Perplexity = 18.64 +Step 39260/44000 | Loss: 2.9413 | LR: 1.47e-05 | 3546.2ms/step | 5.6 samples/s +Step 39270/44000 | Loss: 2.9184 | LR: 1.46e-05 | 1329.8ms/step | 15.0 samples/s +Step 39280/44000 | Loss: 2.9392 | LR: 1.46e-05 | 1331.8ms/step | 15.0 samples/s +Step 39290/44000 | Loss: 2.8957 | LR: 1.45e-05 | 1331.4ms/step | 15.0 samples/s +Step 39300/44000 | Loss: 2.9042 | LR: 1.44e-05 | 1338.2ms/step | 14.9 samples/s +Step 39310/44000 | Loss: 2.9156 | LR: 1.44e-05 | 1331.7ms/step | 15.0 samples/s +Step 39320/44000 | Loss: 2.9269 | LR: 1.43e-05 | 1331.9ms/step | 15.0 samples/s +Step 39330/44000 | Loss: 2.9401 | LR: 1.42e-05 | 1331.4ms/step | 15.0 samples/s +Step 39340/44000 | Loss: 2.9281 | LR: 1.42e-05 | 1326.1ms/step | 15.1 samples/s +Step 39350/44000 | Loss: 2.9124 | LR: 1.41e-05 | 1330.7ms/step | 15.0 samples/s +Step 39360/44000 | Loss: 2.9203 | LR: 1.41e-05 | 1331.8ms/step | 15.0 samples/s +Step 39370/44000 | Loss: 2.9501 | LR: 1.40e-05 | 1326.4ms/step | 15.1 samples/s +Step 39380/44000 | Loss: 2.9467 | LR: 1.39e-05 | 1326.5ms/step | 15.1 samples/s +Step 39390/44000 | Loss: 2.9021 | LR: 1.39e-05 | 1329.6ms/step | 15.0 samples/s +Step 39400/44000 | Loss: 2.9356 | LR: 1.38e-05 | 1332.7ms/step | 15.0 samples/s +Step 39410/44000 | Loss: 2.9555 | LR: 1.38e-05 | 1333.7ms/step | 15.0 samples/s +Step 39420/44000 | Loss: 2.9556 | LR: 1.37e-05 | 1332.8ms/step | 15.0 samples/s +Step 39430/44000 | Loss: 2.9134 | LR: 1.36e-05 | 1331.3ms/step | 15.0 samples/s +Step 39440/44000 | Loss: 2.9134 | LR: 1.36e-05 | 1330.3ms/step | 15.0 samples/s +Step 39450/44000 | Loss: 2.9261 | LR: 1.35e-05 | 1332.0ms/step | 15.0 samples/s +Step 39460/44000 | Loss: 2.9113 | LR: 1.35e-05 | 1329.9ms/step | 15.0 samples/s +Step 39470/44000 | Loss: 2.9107 | LR: 1.34e-05 | 1332.3ms/step | 15.0 samples/s +Step 39480/44000 | Loss: 2.9204 | LR: 1.34e-05 | 1332.2ms/step | 15.0 samples/s +Step 39490/44000 | Loss: 2.9207 | LR: 1.33e-05 | 1333.6ms/step | 15.0 samples/s +Step 39500/44000 | Loss: 2.9611 | LR: 1.32e-05 | 1337.3ms/step | 15.0 samples/s +Validation at step 39500: Loss = 2.9254, Perplexity = 18.64 +Step 39510/44000 | Loss: 2.9027 | LR: 1.32e-05 | 3701.8ms/step | 5.4 samples/s +Step 39520/44000 | Loss: 2.9047 | LR: 1.31e-05 | 1329.0ms/step | 15.0 samples/s +Step 39530/44000 | Loss: 2.9019 | LR: 1.31e-05 | 1326.5ms/step | 15.1 samples/s +Step 39540/44000 | Loss: 2.9310 | LR: 1.30e-05 | 1325.8ms/step | 15.1 samples/s +Step 39550/44000 | Loss: 2.9051 | LR: 1.29e-05 | 1328.0ms/step | 15.1 samples/s +Step 39560/44000 | Loss: 2.8944 | LR: 1.29e-05 | 1326.6ms/step | 15.1 samples/s +Step 39570/44000 | Loss: 2.9279 | LR: 1.28e-05 | 1327.7ms/step | 15.1 samples/s +Step 39580/44000 | Loss: 2.9190 | LR: 1.28e-05 | 1326.6ms/step | 15.1 samples/s +Step 39590/44000 | Loss: 2.9269 | LR: 1.27e-05 | 1326.7ms/step | 15.1 samples/s +Step 39600/44000 | Loss: 2.9423 | LR: 1.27e-05 | 1323.1ms/step | 15.1 samples/s +Step 39610/44000 | Loss: 2.8931 | LR: 1.26e-05 | 1325.8ms/step | 15.1 samples/s +Step 39620/44000 | Loss: 2.9038 | LR: 1.25e-05 | 1325.1ms/step | 15.1 samples/s +Step 39630/44000 | Loss: 2.9302 | LR: 1.25e-05 | 1331.1ms/step | 15.0 samples/s +Step 39640/44000 | Loss: 2.9133 | LR: 1.24e-05 | 1327.6ms/step | 15.1 samples/s +Step 39650/44000 | Loss: 2.9164 | LR: 1.24e-05 | 1324.4ms/step | 15.1 samples/s +Step 39660/44000 | Loss: 2.9265 | LR: 1.23e-05 | 1325.8ms/step | 15.1 samples/s +Step 39670/44000 | Loss: 2.8998 | LR: 1.23e-05 | 1331.2ms/step | 15.0 samples/s +Step 39680/44000 | Loss: 2.9276 | LR: 1.22e-05 | 1330.8ms/step | 15.0 samples/s +Step 39690/44000 | Loss: 2.9061 | LR: 1.22e-05 | 1328.4ms/step | 15.1 samples/s +Step 39700/44000 | Loss: 2.9487 | LR: 1.21e-05 | 1321.5ms/step | 15.1 samples/s +Step 39710/44000 | Loss: 2.9094 | LR: 1.20e-05 | 1327.0ms/step | 15.1 samples/s +Step 39720/44000 | Loss: 2.8969 | LR: 1.20e-05 | 1325.0ms/step | 15.1 samples/s +Step 39730/44000 | Loss: 2.9114 | LR: 1.19e-05 | 1331.3ms/step | 15.0 samples/s +Step 39740/44000 | Loss: 2.9093 | LR: 1.19e-05 | 1327.8ms/step | 15.1 samples/s +Step 39750/44000 | Loss: 2.9135 | LR: 1.18e-05 | 1324.4ms/step | 15.1 samples/s +Validation at step 39750: Loss = 2.9254, Perplexity = 18.64 +Step 39760/44000 | Loss: 2.8906 | LR: 1.18e-05 | 3553.4ms/step | 5.6 samples/s +Step 39770/44000 | Loss: 2.9198 | LR: 1.17e-05 | 1328.5ms/step | 15.1 samples/s +Step 39780/44000 | Loss: 2.8890 | LR: 1.17e-05 | 1329.1ms/step | 15.0 samples/s +Step 39790/44000 | Loss: 2.8887 | LR: 1.16e-05 | 1329.9ms/step | 15.0 samples/s +Step 39800/44000 | Loss: 2.9234 | LR: 1.15e-05 | 1332.2ms/step | 15.0 samples/s +Step 39810/44000 | Loss: 2.9208 | LR: 1.15e-05 | 1325.5ms/step | 15.1 samples/s +Step 39820/44000 | Loss: 2.9239 | LR: 1.14e-05 | 1327.6ms/step | 15.1 samples/s +Step 39830/44000 | Loss: 2.9186 | LR: 1.14e-05 | 1326.8ms/step | 15.1 samples/s +Step 39840/44000 | Loss: 2.9078 | LR: 1.13e-05 | 1326.3ms/step | 15.1 samples/s +Step 39850/44000 | Loss: 2.9162 | LR: 1.13e-05 | 1326.8ms/step | 15.1 samples/s +Step 39860/44000 | Loss: 2.9260 | LR: 1.12e-05 | 1324.9ms/step | 15.1 samples/s +Step 39870/44000 | Loss: 2.9149 | LR: 1.12e-05 | 1327.9ms/step | 15.1 samples/s +Step 39880/44000 | Loss: 2.9096 | LR: 1.11e-05 | 1321.9ms/step | 15.1 samples/s +Step 39890/44000 | Loss: 2.9150 | LR: 1.11e-05 | 1323.6ms/step | 15.1 samples/s +Step 39900/44000 | Loss: 2.9260 | LR: 1.10e-05 | 1327.0ms/step | 15.1 samples/s +Step 39910/44000 | Loss: 2.9231 | LR: 1.10e-05 | 1328.0ms/step | 15.1 samples/s +Step 39920/44000 | Loss: 2.9177 | LR: 1.09e-05 | 1327.2ms/step | 15.1 samples/s +Step 39930/44000 | Loss: 2.9222 | LR: 1.08e-05 | 1324.9ms/step | 15.1 samples/s +Step 39940/44000 | Loss: 2.9210 | LR: 1.08e-05 | 1325.8ms/step | 15.1 samples/s +Step 39950/44000 | Loss: 2.9230 | LR: 1.07e-05 | 1326.4ms/step | 15.1 samples/s +Step 39960/44000 | Loss: 2.9018 | LR: 1.07e-05 | 1328.9ms/step | 15.0 samples/s +Step 39970/44000 | Loss: 2.9252 | LR: 1.06e-05 | 1323.4ms/step | 15.1 samples/s +Step 39980/44000 | Loss: 2.9219 | LR: 1.06e-05 | 1325.1ms/step | 15.1 samples/s +Step 39990/44000 | Loss: 2.9355 | LR: 1.05e-05 | 1327.0ms/step | 15.1 samples/s +Step 40000/44000 | Loss: 2.9027 | LR: 1.05e-05 | 1330.9ms/step | 15.0 samples/s +Validation at step 40000: Loss = 2.9253, Perplexity = 18.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-40000 +Step 40010/44000 | Loss: 2.9054 | LR: 1.04e-05 | 3830.5ms/step | 5.2 samples/s +Step 40020/44000 | Loss: 2.9207 | LR: 1.04e-05 | 1333.3ms/step | 15.0 samples/s +Step 40030/44000 | Loss: 2.9095 | LR: 1.03e-05 | 1330.9ms/step | 15.0 samples/s +Step 40040/44000 | Loss: 2.8998 | LR: 1.03e-05 | 1332.9ms/step | 15.0 samples/s +Step 40050/44000 | Loss: 2.9243 | LR: 1.02e-05 | 1335.6ms/step | 15.0 samples/s +Step 40060/44000 | Loss: 2.8922 | LR: 1.02e-05 | 1334.0ms/step | 15.0 samples/s +Step 40070/44000 | Loss: 2.9095 | LR: 1.01e-05 | 1329.9ms/step | 15.0 samples/s +Step 40080/44000 | Loss: 2.9016 | LR: 1.01e-05 | 1329.3ms/step | 15.0 samples/s +Step 40090/44000 | Loss: 2.8973 | LR: 1.00e-05 | 1331.2ms/step | 15.0 samples/s +Step 40100/44000 | Loss: 2.9230 | LR: 9.96e-06 | 1331.6ms/step | 15.0 samples/s +Step 40110/44000 | Loss: 2.9210 | LR: 9.91e-06 | 1331.3ms/step | 15.0 samples/s +Step 40120/44000 | Loss: 2.9059 | LR: 9.86e-06 | 1329.9ms/step | 15.0 samples/s +Step 40130/44000 | Loss: 2.9229 | LR: 9.81e-06 | 1332.1ms/step | 15.0 samples/s +Step 40140/44000 | Loss: 2.9185 | LR: 9.76e-06 | 1331.0ms/step | 15.0 samples/s +Step 40150/44000 | Loss: 2.9221 | LR: 9.71e-06 | 1332.5ms/step | 15.0 samples/s +Step 40160/44000 | Loss: 2.9215 | LR: 9.66e-06 | 1331.9ms/step | 15.0 samples/s +Step 40170/44000 | Loss: 2.9092 | LR: 9.61e-06 | 1333.0ms/step | 15.0 samples/s +Step 40180/44000 | Loss: 2.9199 | LR: 9.56e-06 | 1331.3ms/step | 15.0 samples/s +Step 40190/44000 | Loss: 2.8994 | LR: 9.51e-06 | 1332.0ms/step | 15.0 samples/s +Step 40200/44000 | Loss: 2.9251 | LR: 9.46e-06 | 1330.8ms/step | 15.0 samples/s +Step 40210/44000 | Loss: 2.9470 | LR: 9.41e-06 | 1329.5ms/step | 15.0 samples/s +Step 40220/44000 | Loss: 2.9629 | LR: 9.36e-06 | 1328.9ms/step | 15.1 samples/s +Step 40230/44000 | Loss: 2.9170 | LR: 9.32e-06 | 1331.9ms/step | 15.0 samples/s +Step 40240/44000 | Loss: 2.8973 | LR: 9.27e-06 | 1328.5ms/step | 15.1 samples/s +Step 40250/44000 | Loss: 2.9563 | LR: 9.22e-06 | 1333.7ms/step | 15.0 samples/s +Validation at step 40250: Loss = 2.9253, Perplexity = 18.64 +Step 40260/44000 | Loss: 2.9425 | LR: 9.17e-06 | 3554.1ms/step | 5.6 samples/s +Step 40270/44000 | Loss: 2.9513 | LR: 9.12e-06 | 1332.7ms/step | 15.0 samples/s +Step 40280/44000 | Loss: 2.9304 | LR: 9.07e-06 | 1335.5ms/step | 15.0 samples/s +Step 40290/44000 | Loss: 2.9347 | LR: 9.02e-06 | 1336.1ms/step | 15.0 samples/s +Step 40300/44000 | Loss: 2.9167 | LR: 8.97e-06 | 1333.7ms/step | 15.0 samples/s +Step 40310/44000 | Loss: 2.9094 | LR: 8.93e-06 | 1331.5ms/step | 15.0 samples/s +Step 40320/44000 | Loss: 2.9064 | LR: 8.88e-06 | 1334.0ms/step | 15.0 samples/s +Step 40330/44000 | Loss: 2.9285 | LR: 8.83e-06 | 1332.7ms/step | 15.0 samples/s +Step 40340/44000 | Loss: 2.9376 | LR: 8.78e-06 | 1329.5ms/step | 15.0 samples/s +Step 40350/44000 | Loss: 2.9455 | LR: 8.74e-06 | 1330.1ms/step | 15.0 samples/s +Step 40360/44000 | Loss: 2.8895 | LR: 8.69e-06 | 1332.2ms/step | 15.0 samples/s +Step 40370/44000 | Loss: 2.8895 | LR: 8.64e-06 | 1331.2ms/step | 15.0 samples/s +Step 40380/44000 | Loss: 2.9325 | LR: 8.59e-06 | 1330.6ms/step | 15.0 samples/s +Step 40390/44000 | Loss: 2.9027 | LR: 8.55e-06 | 1333.8ms/step | 15.0 samples/s +Step 40400/44000 | Loss: 2.9322 | LR: 8.50e-06 | 1329.6ms/step | 15.0 samples/s +Step 40410/44000 | Loss: 2.9152 | LR: 8.45e-06 | 1330.4ms/step | 15.0 samples/s +Step 40420/44000 | Loss: 2.9295 | LR: 8.41e-06 | 1329.9ms/step | 15.0 samples/s +Step 40430/44000 | Loss: 2.9319 | LR: 8.36e-06 | 1328.5ms/step | 15.1 samples/s +Step 40440/44000 | Loss: 2.9128 | LR: 8.31e-06 | 1332.4ms/step | 15.0 samples/s +Step 40450/44000 | Loss: 2.9189 | LR: 8.27e-06 | 1328.8ms/step | 15.1 samples/s +Step 40460/44000 | Loss: 2.8998 | LR: 8.22e-06 | 1325.8ms/step | 15.1 samples/s +Step 40470/44000 | Loss: 2.9127 | LR: 8.17e-06 | 1333.7ms/step | 15.0 samples/s +Step 40480/44000 | Loss: 2.9185 | LR: 8.13e-06 | 1335.0ms/step | 15.0 samples/s +Step 40490/44000 | Loss: 2.9181 | LR: 8.08e-06 | 1333.3ms/step | 15.0 samples/s +Step 40500/44000 | Loss: 2.9367 | LR: 8.04e-06 | 1332.0ms/step | 15.0 samples/s +Validation at step 40500: Loss = 2.9253, Perplexity = 18.64 +Step 40510/44000 | Loss: 2.9339 | LR: 7.99e-06 | 3557.3ms/step | 5.6 samples/s +Step 40520/44000 | Loss: 2.9044 | LR: 7.94e-06 | 1332.9ms/step | 15.0 samples/s +Step 40530/44000 | Loss: 2.9057 | LR: 7.90e-06 | 1334.9ms/step | 15.0 samples/s +Step 40540/44000 | Loss: 2.9370 | LR: 7.85e-06 | 1330.6ms/step | 15.0 samples/s +Step 40550/44000 | Loss: 2.8930 | LR: 7.81e-06 | 1336.7ms/step | 15.0 samples/s +Step 40560/44000 | Loss: 2.9164 | LR: 7.76e-06 | 1333.4ms/step | 15.0 samples/s +Step 40570/44000 | Loss: 2.8824 | LR: 7.72e-06 | 1331.2ms/step | 15.0 samples/s +Step 40580/44000 | Loss: 2.9156 | LR: 7.67e-06 | 1333.8ms/step | 15.0 samples/s +Step 40590/44000 | Loss: 2.9181 | LR: 7.63e-06 | 1330.9ms/step | 15.0 samples/s +Step 40600/44000 | Loss: 2.9404 | LR: 7.59e-06 | 1334.0ms/step | 15.0 samples/s +Step 40610/44000 | Loss: 2.9314 | LR: 7.54e-06 | 1331.7ms/step | 15.0 samples/s +Step 40620/44000 | Loss: 2.9213 | LR: 7.50e-06 | 1333.3ms/step | 15.0 samples/s +Step 40630/44000 | Loss: 2.9310 | LR: 7.45e-06 | 1330.7ms/step | 15.0 samples/s +Step 40640/44000 | Loss: 2.9292 | LR: 7.41e-06 | 1330.6ms/step | 15.0 samples/s +Step 40650/44000 | Loss: 2.9250 | LR: 7.37e-06 | 1336.2ms/step | 15.0 samples/s +Step 40660/44000 | Loss: 2.9249 | LR: 7.32e-06 | 1333.6ms/step | 15.0 samples/s +Step 40670/44000 | Loss: 2.8958 | LR: 7.28e-06 | 1332.0ms/step | 15.0 samples/s +Step 40680/44000 | Loss: 2.9165 | LR: 7.23e-06 | 1337.0ms/step | 15.0 samples/s +Step 40690/44000 | Loss: 2.9175 | LR: 7.19e-06 | 1329.5ms/step | 15.0 samples/s +Step 40700/44000 | Loss: 2.8995 | LR: 7.15e-06 | 1332.4ms/step | 15.0 samples/s +Step 40710/44000 | Loss: 2.8732 | LR: 7.10e-06 | 1330.3ms/step | 15.0 samples/s +Step 40720/44000 | Loss: 2.9019 | LR: 7.06e-06 | 1328.6ms/step | 15.1 samples/s +Step 40730/44000 | Loss: 2.9017 | LR: 7.02e-06 | 1333.3ms/step | 15.0 samples/s +Step 40740/44000 | Loss: 2.9423 | LR: 6.98e-06 | 1330.1ms/step | 15.0 samples/s +Step 40750/44000 | Loss: 2.9238 | LR: 6.93e-06 | 1333.4ms/step | 15.0 samples/s +Validation at step 40750: Loss = 2.9253, Perplexity = 18.64 +Step 40760/44000 | Loss: 2.9184 | LR: 6.89e-06 | 3555.1ms/step | 5.6 samples/s +Step 40770/44000 | Loss: 2.9739 | LR: 6.85e-06 | 1333.7ms/step | 15.0 samples/s +Step 40780/44000 | Loss: 2.9211 | LR: 6.81e-06 | 1331.9ms/step | 15.0 samples/s +Step 40790/44000 | Loss: 2.9259 | LR: 6.77e-06 | 1329.2ms/step | 15.0 samples/s +Step 40800/44000 | Loss: 2.8975 | LR: 6.72e-06 | 1329.5ms/step | 15.0 samples/s +Step 40810/44000 | Loss: 2.9027 | LR: 6.68e-06 | 1325.9ms/step | 15.1 samples/s +Step 40820/44000 | Loss: 2.9167 | LR: 6.64e-06 | 1330.8ms/step | 15.0 samples/s +Step 40830/44000 | Loss: 2.9413 | LR: 6.60e-06 | 1332.3ms/step | 15.0 samples/s +Step 40840/44000 | Loss: 2.9229 | LR: 6.56e-06 | 1335.8ms/step | 15.0 samples/s +Step 40850/44000 | Loss: 2.9124 | LR: 6.52e-06 | 1335.9ms/step | 15.0 samples/s +Step 40860/44000 | Loss: 2.8811 | LR: 6.47e-06 | 1327.1ms/step | 15.1 samples/s +Step 40870/44000 | Loss: 2.9118 | LR: 6.43e-06 | 1332.8ms/step | 15.0 samples/s +Step 40880/44000 | Loss: 2.8949 | LR: 6.39e-06 | 1333.8ms/step | 15.0 samples/s +Step 40890/44000 | Loss: 2.8996 | LR: 6.35e-06 | 1333.7ms/step | 15.0 samples/s +Step 40900/44000 | Loss: 2.9171 | LR: 6.31e-06 | 1329.7ms/step | 15.0 samples/s +Step 40910/44000 | Loss: 2.9116 | LR: 6.27e-06 | 1336.7ms/step | 15.0 samples/s +Step 40920/44000 | Loss: 2.9311 | LR: 6.23e-06 | 1333.4ms/step | 15.0 samples/s +Step 40930/44000 | Loss: 2.9489 | LR: 6.19e-06 | 1332.6ms/step | 15.0 samples/s +Step 40940/44000 | Loss: 2.8833 | LR: 6.15e-06 | 1329.8ms/step | 15.0 samples/s +Step 40950/44000 | Loss: 2.9221 | LR: 6.11e-06 | 1330.8ms/step | 15.0 samples/s +Step 40960/44000 | Loss: 2.9299 | LR: 6.07e-06 | 1332.7ms/step | 15.0 samples/s +Step 40970/44000 | Loss: 2.9356 | LR: 6.03e-06 | 1336.2ms/step | 15.0 samples/s +Step 40980/44000 | Loss: 2.9098 | LR: 5.99e-06 | 1331.2ms/step | 15.0 samples/s +Step 40990/44000 | Loss: 2.9265 | LR: 5.95e-06 | 1331.1ms/step | 15.0 samples/s +Step 41000/44000 | Loss: 2.8989 | LR: 5.91e-06 | 1335.0ms/step | 15.0 samples/s +Validation at step 41000: Loss = 2.9253, Perplexity = 18.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-41000 +Step 41010/44000 | Loss: 2.9211 | LR: 5.87e-06 | 3859.3ms/step | 5.2 samples/s +Step 41020/44000 | Loss: 2.9275 | LR: 5.83e-06 | 1329.4ms/step | 15.0 samples/s +Step 41030/44000 | Loss: 2.8952 | LR: 5.80e-06 | 1327.7ms/step | 15.1 samples/s +Step 41040/44000 | Loss: 2.9715 | LR: 5.76e-06 | 1331.6ms/step | 15.0 samples/s +Step 41050/44000 | Loss: 2.9122 | LR: 5.72e-06 | 1331.0ms/step | 15.0 samples/s +Step 41060/44000 | Loss: 2.9056 | LR: 5.68e-06 | 1330.9ms/step | 15.0 samples/s +Step 41070/44000 | Loss: 2.9323 | LR: 5.64e-06 | 1331.9ms/step | 15.0 samples/s +Step 41080/44000 | Loss: 2.9033 | LR: 5.60e-06 | 1331.9ms/step | 15.0 samples/s +Step 41090/44000 | Loss: 2.9061 | LR: 5.56e-06 | 1332.2ms/step | 15.0 samples/s +Step 41100/44000 | Loss: 2.8996 | LR: 5.53e-06 | 1332.1ms/step | 15.0 samples/s +Step 41110/44000 | Loss: 2.9304 | LR: 5.49e-06 | 1337.7ms/step | 15.0 samples/s +Step 41120/44000 | Loss: 2.9084 | LR: 5.45e-06 | 1330.8ms/step | 15.0 samples/s +Step 41130/44000 | Loss: 2.9330 | LR: 5.41e-06 | 1330.5ms/step | 15.0 samples/s +Step 41140/44000 | Loss: 2.9582 | LR: 5.38e-06 | 1324.1ms/step | 15.1 samples/s +Step 41150/44000 | Loss: 2.8909 | LR: 5.34e-06 | 1325.7ms/step | 15.1 samples/s +Step 41160/44000 | Loss: 2.9103 | LR: 5.30e-06 | 1326.6ms/step | 15.1 samples/s +Step 41170/44000 | Loss: 2.9027 | LR: 5.26e-06 | 1325.7ms/step | 15.1 samples/s +Step 41180/44000 | Loss: 2.9326 | LR: 5.23e-06 | 1325.2ms/step | 15.1 samples/s +Step 41190/44000 | Loss: 2.9130 | LR: 5.19e-06 | 1322.1ms/step | 15.1 samples/s +Step 41200/44000 | Loss: 2.8933 | LR: 5.15e-06 | 1322.2ms/step | 15.1 samples/s +Step 41210/44000 | Loss: 2.9043 | LR: 5.12e-06 | 1326.4ms/step | 15.1 samples/s +Step 41220/44000 | Loss: 2.9337 | LR: 5.08e-06 | 1327.8ms/step | 15.1 samples/s +Step 41230/44000 | Loss: 2.9142 | LR: 5.04e-06 | 1331.4ms/step | 15.0 samples/s +Step 41240/44000 | Loss: 2.9325 | LR: 5.01e-06 | 1331.7ms/step | 15.0 samples/s +Step 41250/44000 | Loss: 2.8915 | LR: 4.97e-06 | 1331.7ms/step | 15.0 samples/s +Validation at step 41250: Loss = 2.9253, Perplexity = 18.64 +Step 41260/44000 | Loss: 2.9234 | LR: 4.94e-06 | 3560.2ms/step | 5.6 samples/s +Step 41270/44000 | Loss: 2.8918 | LR: 4.90e-06 | 1335.7ms/step | 15.0 samples/s +Step 41280/44000 | Loss: 2.9246 | LR: 4.86e-06 | 1332.3ms/step | 15.0 samples/s +Step 41290/44000 | Loss: 2.8904 | LR: 4.83e-06 | 1333.4ms/step | 15.0 samples/s +Step 41300/44000 | Loss: 2.9467 | LR: 4.79e-06 | 1334.0ms/step | 15.0 samples/s +Step 41310/44000 | Loss: 2.9260 | LR: 4.76e-06 | 1329.3ms/step | 15.0 samples/s +Step 41320/44000 | Loss: 2.9329 | LR: 4.72e-06 | 1332.5ms/step | 15.0 samples/s +Step 41330/44000 | Loss: 2.9197 | LR: 4.69e-06 | 1334.0ms/step | 15.0 samples/s +Step 41340/44000 | Loss: 2.8948 | LR: 4.65e-06 | 1328.8ms/step | 15.1 samples/s +Step 41350/44000 | Loss: 2.9130 | LR: 4.62e-06 | 1331.3ms/step | 15.0 samples/s +Step 41360/44000 | Loss: 2.9080 | LR: 4.58e-06 | 1333.5ms/step | 15.0 samples/s +Step 41370/44000 | Loss: 2.9033 | LR: 4.55e-06 | 1332.8ms/step | 15.0 samples/s +Step 41380/44000 | Loss: 2.9511 | LR: 4.51e-06 | 1342.2ms/step | 14.9 samples/s +Step 41390/44000 | Loss: 2.9215 | LR: 4.48e-06 | 1348.0ms/step | 14.8 samples/s +Step 41400/44000 | Loss: 2.9260 | LR: 4.45e-06 | 1348.3ms/step | 14.8 samples/s +Step 41410/44000 | Loss: 2.9295 | LR: 4.41e-06 | 1340.7ms/step | 14.9 samples/s +Step 41420/44000 | Loss: 2.9363 | LR: 4.38e-06 | 1338.4ms/step | 14.9 samples/s +Step 41430/44000 | Loss: 2.9024 | LR: 4.34e-06 | 1340.1ms/step | 14.9 samples/s +Step 41440/44000 | Loss: 2.9084 | LR: 4.31e-06 | 1333.8ms/step | 15.0 samples/s +Step 41450/44000 | Loss: 2.9284 | LR: 4.28e-06 | 1337.3ms/step | 15.0 samples/s +Step 41460/44000 | Loss: 2.9083 | LR: 4.24e-06 | 1337.5ms/step | 15.0 samples/s +Step 41470/44000 | Loss: 2.9220 | LR: 4.21e-06 | 1337.5ms/step | 15.0 samples/s +Step 41480/44000 | Loss: 2.9215 | LR: 4.18e-06 | 1338.8ms/step | 14.9 samples/s +Step 41490/44000 | Loss: 2.9189 | LR: 4.14e-06 | 1339.6ms/step | 14.9 samples/s +Step 41500/44000 | Loss: 2.9329 | LR: 4.11e-06 | 1335.7ms/step | 15.0 samples/s +Validation at step 41500: Loss = 2.9253, Perplexity = 18.64 +Step 41510/44000 | Loss: 2.9289 | LR: 4.08e-06 | 3559.8ms/step | 5.6 samples/s +Step 41520/44000 | Loss: 2.8901 | LR: 4.05e-06 | 1339.5ms/step | 14.9 samples/s +Step 41530/44000 | Loss: 2.9110 | LR: 4.01e-06 | 1335.9ms/step | 15.0 samples/s +Step 41540/44000 | Loss: 2.9209 | LR: 3.98e-06 | 1337.1ms/step | 15.0 samples/s +Step 41550/44000 | Loss: 2.8793 | LR: 3.95e-06 | 1337.3ms/step | 15.0 samples/s +Step 41560/44000 | Loss: 2.9235 | LR: 3.92e-06 | 1335.2ms/step | 15.0 samples/s +Step 41570/44000 | Loss: 2.9219 | LR: 3.88e-06 | 1334.8ms/step | 15.0 samples/s +Step 41580/44000 | Loss: 2.9223 | LR: 3.85e-06 | 1334.2ms/step | 15.0 samples/s +Step 41590/44000 | Loss: 2.9186 | LR: 3.82e-06 | 1340.0ms/step | 14.9 samples/s +Step 41600/44000 | Loss: 2.9119 | LR: 3.79e-06 | 1340.2ms/step | 14.9 samples/s +Step 41610/44000 | Loss: 2.9233 | LR: 3.76e-06 | 1337.8ms/step | 14.9 samples/s +Step 41620/44000 | Loss: 2.9097 | LR: 3.73e-06 | 1336.4ms/step | 15.0 samples/s +Step 41630/44000 | Loss: 2.9021 | LR: 3.70e-06 | 1335.2ms/step | 15.0 samples/s +Step 41640/44000 | Loss: 2.8884 | LR: 3.66e-06 | 1342.4ms/step | 14.9 samples/s +Step 41650/44000 | Loss: 2.8967 | LR: 3.63e-06 | 1340.8ms/step | 14.9 samples/s +Step 41660/44000 | Loss: 2.9216 | LR: 3.60e-06 | 1341.6ms/step | 14.9 samples/s +Step 41670/44000 | Loss: 2.9094 | LR: 3.57e-06 | 1337.5ms/step | 15.0 samples/s +Step 41680/44000 | Loss: 2.9357 | LR: 3.54e-06 | 1334.5ms/step | 15.0 samples/s +Step 41690/44000 | Loss: 2.9001 | LR: 3.51e-06 | 1337.3ms/step | 15.0 samples/s +Step 41700/44000 | Loss: 2.8963 | LR: 3.48e-06 | 1340.1ms/step | 14.9 samples/s +Step 41710/44000 | Loss: 2.9153 | LR: 3.45e-06 | 1337.8ms/step | 15.0 samples/s +Step 41720/44000 | Loss: 2.9124 | LR: 3.42e-06 | 1338.1ms/step | 14.9 samples/s +Step 41730/44000 | Loss: 2.9127 | LR: 3.39e-06 | 1330.9ms/step | 15.0 samples/s +Step 41740/44000 | Loss: 2.9345 | LR: 3.36e-06 | 1334.2ms/step | 15.0 samples/s +Step 41750/44000 | Loss: 2.8904 | LR: 3.33e-06 | 1335.6ms/step | 15.0 samples/s +Validation at step 41750: Loss = 2.9253, Perplexity = 18.64 +Step 41760/44000 | Loss: 2.9462 | LR: 3.30e-06 | 3549.5ms/step | 5.6 samples/s +Step 41770/44000 | Loss: 2.8829 | LR: 3.27e-06 | 1333.0ms/step | 15.0 samples/s +Step 41780/44000 | Loss: 2.9055 | LR: 3.24e-06 | 1338.2ms/step | 14.9 samples/s +Step 41790/44000 | Loss: 2.9079 | LR: 3.21e-06 | 1334.9ms/step | 15.0 samples/s +Step 41800/44000 | Loss: 2.9296 | LR: 3.19e-06 | 1337.1ms/step | 15.0 samples/s +Step 41810/44000 | Loss: 2.9200 | LR: 3.16e-06 | 1333.9ms/step | 15.0 samples/s +Step 41820/44000 | Loss: 2.9316 | LR: 3.13e-06 | 1334.6ms/step | 15.0 samples/s +Step 41830/44000 | Loss: 2.9103 | LR: 3.10e-06 | 1335.7ms/step | 15.0 samples/s +Step 41840/44000 | Loss: 2.9309 | LR: 3.07e-06 | 1331.6ms/step | 15.0 samples/s +Step 41850/44000 | Loss: 2.9252 | LR: 3.04e-06 | 1331.3ms/step | 15.0 samples/s +Step 41860/44000 | Loss: 2.9271 | LR: 3.01e-06 | 1330.0ms/step | 15.0 samples/s +Step 41870/44000 | Loss: 2.8996 | LR: 2.99e-06 | 1330.0ms/step | 15.0 samples/s +Step 41880/44000 | Loss: 2.8980 | LR: 2.96e-06 | 1329.6ms/step | 15.0 samples/s +Step 41890/44000 | Loss: 2.9324 | LR: 2.93e-06 | 1331.4ms/step | 15.0 samples/s +Step 41900/44000 | Loss: 2.9257 | LR: 2.90e-06 | 1330.6ms/step | 15.0 samples/s +Step 41910/44000 | Loss: 2.8986 | LR: 2.88e-06 | 1330.3ms/step | 15.0 samples/s +Step 41920/44000 | Loss: 2.8901 | LR: 2.85e-06 | 1330.5ms/step | 15.0 samples/s +Step 41930/44000 | Loss: 2.9485 | LR: 2.82e-06 | 1330.9ms/step | 15.0 samples/s +Step 41940/44000 | Loss: 2.9051 | LR: 2.79e-06 | 1335.3ms/step | 15.0 samples/s +Step 41950/44000 | Loss: 2.8958 | LR: 2.77e-06 | 1333.0ms/step | 15.0 samples/s +Step 41960/44000 | Loss: 2.8953 | LR: 2.74e-06 | 1334.6ms/step | 15.0 samples/s +Step 41970/44000 | Loss: 2.9180 | LR: 2.71e-06 | 1332.8ms/step | 15.0 samples/s +Step 41980/44000 | Loss: 2.9123 | LR: 2.69e-06 | 1334.6ms/step | 15.0 samples/s +Step 41990/44000 | Loss: 2.9491 | LR: 2.66e-06 | 1335.4ms/step | 15.0 samples/s +Step 42000/44000 | Loss: 2.9231 | LR: 2.63e-06 | 1337.2ms/step | 15.0 samples/s +Validation at step 42000: Loss = 2.9253, Perplexity = 18.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-42000 +Step 42010/44000 | Loss: 2.9247 | LR: 2.61e-06 | 3876.7ms/step | 5.2 samples/s +Step 42020/44000 | Loss: 2.9075 | LR: 2.58e-06 | 1333.3ms/step | 15.0 samples/s +Step 42030/44000 | Loss: 2.9258 | LR: 2.56e-06 | 1331.6ms/step | 15.0 samples/s +Step 42040/44000 | Loss: 2.9290 | LR: 2.53e-06 | 1332.5ms/step | 15.0 samples/s +Step 42050/44000 | Loss: 2.9323 | LR: 2.50e-06 | 1333.6ms/step | 15.0 samples/s +Step 42060/44000 | Loss: 2.9087 | LR: 2.48e-06 | 1332.3ms/step | 15.0 samples/s +Step 42070/44000 | Loss: 2.9037 | LR: 2.45e-06 | 1331.7ms/step | 15.0 samples/s +Step 42080/44000 | Loss: 2.9230 | LR: 2.43e-06 | 1332.1ms/step | 15.0 samples/s +Step 42090/44000 | Loss: 2.9407 | LR: 2.40e-06 | 1330.5ms/step | 15.0 samples/s +Step 42100/44000 | Loss: 2.8792 | LR: 2.38e-06 | 1328.8ms/step | 15.1 samples/s +Step 42110/44000 | Loss: 2.9203 | LR: 2.35e-06 | 1328.1ms/step | 15.1 samples/s +Step 42120/44000 | Loss: 2.8862 | LR: 2.33e-06 | 1330.7ms/step | 15.0 samples/s +Step 42130/44000 | Loss: 2.9462 | LR: 2.30e-06 | 1339.0ms/step | 14.9 samples/s +Step 42140/44000 | Loss: 2.8822 | LR: 2.28e-06 | 1337.7ms/step | 15.0 samples/s +Step 42150/44000 | Loss: 2.8952 | LR: 2.25e-06 | 1337.9ms/step | 14.9 samples/s +Step 42160/44000 | Loss: 2.9206 | LR: 2.23e-06 | 1339.4ms/step | 14.9 samples/s +Step 42170/44000 | Loss: 2.8922 | LR: 2.21e-06 | 1335.5ms/step | 15.0 samples/s +Step 42180/44000 | Loss: 2.9013 | LR: 2.18e-06 | 1338.1ms/step | 14.9 samples/s +Step 42190/44000 | Loss: 2.9341 | LR: 2.16e-06 | 1337.9ms/step | 14.9 samples/s +Step 42200/44000 | Loss: 2.9021 | LR: 2.13e-06 | 1339.2ms/step | 14.9 samples/s +Step 42210/44000 | Loss: 2.9356 | LR: 2.11e-06 | 1340.2ms/step | 14.9 samples/s +Step 42220/44000 | Loss: 2.9127 | LR: 2.09e-06 | 1334.4ms/step | 15.0 samples/s +Step 42230/44000 | Loss: 2.8906 | LR: 2.06e-06 | 1332.0ms/step | 15.0 samples/s +Step 42240/44000 | Loss: 2.9310 | LR: 2.04e-06 | 1325.3ms/step | 15.1 samples/s +Step 42250/44000 | Loss: 2.9093 | LR: 2.02e-06 | 1334.9ms/step | 15.0 samples/s +Validation at step 42250: Loss = 2.9253, Perplexity = 18.64 +Step 42260/44000 | Loss: 2.9212 | LR: 1.99e-06 | 3546.4ms/step | 5.6 samples/s +Step 42270/44000 | Loss: 2.9373 | LR: 1.97e-06 | 1332.0ms/step | 15.0 samples/s +Step 42280/44000 | Loss: 2.9338 | LR: 1.95e-06 | 1332.1ms/step | 15.0 samples/s +Step 42290/44000 | Loss: 2.8851 | LR: 1.93e-06 | 1331.0ms/step | 15.0 samples/s +Step 42300/44000 | Loss: 2.9222 | LR: 1.90e-06 | 1329.0ms/step | 15.0 samples/s +Step 42310/44000 | Loss: 2.9132 | LR: 1.88e-06 | 1328.1ms/step | 15.1 samples/s +Step 42320/44000 | Loss: 2.9331 | LR: 1.86e-06 | 1325.6ms/step | 15.1 samples/s +Step 42330/44000 | Loss: 2.9415 | LR: 1.84e-06 | 1327.1ms/step | 15.1 samples/s +Step 42340/44000 | Loss: 2.9147 | LR: 1.82e-06 | 1335.4ms/step | 15.0 samples/s +Step 42350/44000 | Loss: 2.9122 | LR: 1.79e-06 | 1331.9ms/step | 15.0 samples/s +Step 42360/44000 | Loss: 2.9114 | LR: 1.77e-06 | 1327.8ms/step | 15.1 samples/s +Step 42370/44000 | Loss: 2.9227 | LR: 1.75e-06 | 1323.9ms/step | 15.1 samples/s +Step 42380/44000 | Loss: 2.9376 | LR: 1.73e-06 | 1325.4ms/step | 15.1 samples/s +Step 42390/44000 | Loss: 2.9071 | LR: 1.71e-06 | 1330.3ms/step | 15.0 samples/s +Step 42400/44000 | Loss: 2.9145 | LR: 1.69e-06 | 1335.3ms/step | 15.0 samples/s +Step 42410/44000 | Loss: 2.8940 | LR: 1.67e-06 | 1327.0ms/step | 15.1 samples/s +Step 42420/44000 | Loss: 2.9210 | LR: 1.64e-06 | 1327.6ms/step | 15.1 samples/s +Step 42430/44000 | Loss: 2.9373 | LR: 1.62e-06 | 1326.6ms/step | 15.1 samples/s +Step 42440/44000 | Loss: 2.8936 | LR: 1.60e-06 | 1327.3ms/step | 15.1 samples/s +Step 42450/44000 | Loss: 2.9421 | LR: 1.58e-06 | 1326.7ms/step | 15.1 samples/s +Step 42460/44000 | Loss: 2.9218 | LR: 1.56e-06 | 1326.9ms/step | 15.1 samples/s +Step 42470/44000 | Loss: 2.9066 | LR: 1.54e-06 | 1330.7ms/step | 15.0 samples/s +Step 42480/44000 | Loss: 2.9334 | LR: 1.52e-06 | 1336.4ms/step | 15.0 samples/s +Step 42490/44000 | Loss: 2.8989 | LR: 1.50e-06 | 1335.3ms/step | 15.0 samples/s +Step 42500/44000 | Loss: 2.9236 | LR: 1.48e-06 | 1332.9ms/step | 15.0 samples/s +Validation at step 42500: Loss = 2.9253, Perplexity = 18.64 +Step 42510/44000 | Loss: 2.9057 | LR: 1.46e-06 | 3539.6ms/step | 5.7 samples/s +Step 42520/44000 | Loss: 2.9173 | LR: 1.44e-06 | 1334.1ms/step | 15.0 samples/s +Step 42530/44000 | Loss: 2.8867 | LR: 1.42e-06 | 1332.8ms/step | 15.0 samples/s +Step 42540/44000 | Loss: 2.9612 | LR: 1.40e-06 | 1330.0ms/step | 15.0 samples/s +Step 42550/44000 | Loss: 2.9117 | LR: 1.39e-06 | 1332.2ms/step | 15.0 samples/s +Step 42560/44000 | Loss: 2.9307 | LR: 1.37e-06 | 1332.1ms/step | 15.0 samples/s +Step 42570/44000 | Loss: 2.9250 | LR: 1.35e-06 | 1330.3ms/step | 15.0 samples/s +Step 42580/44000 | Loss: 2.9162 | LR: 1.33e-06 | 1326.9ms/step | 15.1 samples/s +Step 42590/44000 | Loss: 2.9241 | LR: 1.31e-06 | 1328.7ms/step | 15.1 samples/s +Step 42600/44000 | Loss: 2.9040 | LR: 1.29e-06 | 1328.1ms/step | 15.1 samples/s +Step 42610/44000 | Loss: 2.9075 | LR: 1.27e-06 | 1332.4ms/step | 15.0 samples/s +Step 42620/44000 | Loss: 2.9450 | LR: 1.25e-06 | 1328.2ms/step | 15.1 samples/s +Step 42630/44000 | Loss: 2.8892 | LR: 1.24e-06 | 1329.0ms/step | 15.0 samples/s +Step 42640/44000 | Loss: 2.9301 | LR: 1.22e-06 | 1333.2ms/step | 15.0 samples/s +Step 42650/44000 | Loss: 2.9276 | LR: 1.20e-06 | 1334.3ms/step | 15.0 samples/s +Step 42660/44000 | Loss: 2.9218 | LR: 1.18e-06 | 1330.2ms/step | 15.0 samples/s +Step 42670/44000 | Loss: 2.9085 | LR: 1.17e-06 | 1324.8ms/step | 15.1 samples/s +Step 42680/44000 | Loss: 2.9152 | LR: 1.15e-06 | 1324.8ms/step | 15.1 samples/s +Step 42690/44000 | Loss: 2.9446 | LR: 1.13e-06 | 1325.6ms/step | 15.1 samples/s +Step 42700/44000 | Loss: 2.8951 | LR: 1.11e-06 | 1323.9ms/step | 15.1 samples/s +Step 42710/44000 | Loss: 2.9148 | LR: 1.10e-06 | 1321.8ms/step | 15.1 samples/s +Step 42720/44000 | Loss: 2.8914 | LR: 1.08e-06 | 1321.7ms/step | 15.1 samples/s +Step 42730/44000 | Loss: 2.9032 | LR: 1.06e-06 | 1319.6ms/step | 15.2 samples/s +Step 42740/44000 | Loss: 2.9367 | LR: 1.05e-06 | 1321.8ms/step | 15.1 samples/s +Step 42750/44000 | Loss: 2.8832 | LR: 1.03e-06 | 1320.7ms/step | 15.1 samples/s +Validation at step 42750: Loss = 2.9253, Perplexity = 18.64 +Step 42760/44000 | Loss: 2.9342 | LR: 1.01e-06 | 3526.8ms/step | 5.7 samples/s +Step 42770/44000 | Loss: 2.8976 | LR: 9.97e-07 | 1320.9ms/step | 15.1 samples/s +Step 42780/44000 | Loss: 2.9030 | LR: 9.81e-07 | 1322.5ms/step | 15.1 samples/s +Step 42790/44000 | Loss: 2.9059 | LR: 9.65e-07 | 1319.4ms/step | 15.2 samples/s +Step 42800/44000 | Loss: 2.9348 | LR: 9.49e-07 | 1321.7ms/step | 15.1 samples/s +Step 42810/44000 | Loss: 2.9245 | LR: 9.33e-07 | 1320.5ms/step | 15.1 samples/s +Step 42820/44000 | Loss: 2.9254 | LR: 9.18e-07 | 1321.8ms/step | 15.1 samples/s +Step 42830/44000 | Loss: 2.9282 | LR: 9.02e-07 | 1320.6ms/step | 15.1 samples/s +Step 42840/44000 | Loss: 2.9156 | LR: 8.87e-07 | 1320.9ms/step | 15.1 samples/s +Step 42850/44000 | Loss: 2.9033 | LR: 8.72e-07 | 1323.1ms/step | 15.1 samples/s +Step 42860/44000 | Loss: 2.9368 | LR: 8.57e-07 | 1321.4ms/step | 15.1 samples/s +Step 42870/44000 | Loss: 2.9213 | LR: 8.42e-07 | 1317.3ms/step | 15.2 samples/s +Step 42880/44000 | Loss: 2.9042 | LR: 8.27e-07 | 1318.7ms/step | 15.2 samples/s +Step 42890/44000 | Loss: 2.9008 | LR: 8.12e-07 | 1320.0ms/step | 15.2 samples/s +Step 42900/44000 | Loss: 2.9089 | LR: 7.98e-07 | 1320.8ms/step | 15.1 samples/s +Step 42910/44000 | Loss: 2.9062 | LR: 7.83e-07 | 1321.1ms/step | 15.1 samples/s +Step 42920/44000 | Loss: 2.9401 | LR: 7.69e-07 | 1319.0ms/step | 15.2 samples/s +Step 42930/44000 | Loss: 2.9197 | LR: 7.55e-07 | 1322.6ms/step | 15.1 samples/s +Step 42940/44000 | Loss: 2.8821 | LR: 7.41e-07 | 1321.4ms/step | 15.1 samples/s +Step 42950/44000 | Loss: 2.9374 | LR: 7.27e-07 | 1320.1ms/step | 15.2 samples/s +Step 42960/44000 | Loss: 2.9351 | LR: 7.13e-07 | 1319.5ms/step | 15.2 samples/s +Step 42970/44000 | Loss: 2.9374 | LR: 6.99e-07 | 1319.0ms/step | 15.2 samples/s +Step 42980/44000 | Loss: 2.9210 | LR: 6.86e-07 | 1320.7ms/step | 15.1 samples/s +Step 42990/44000 | Loss: 2.9204 | LR: 6.72e-07 | 1323.7ms/step | 15.1 samples/s +Step 43000/44000 | Loss: 2.9424 | LR: 6.59e-07 | 1323.0ms/step | 15.1 samples/s +Validation at step 43000: Loss = 2.9253, Perplexity = 18.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-43000 +Step 43010/44000 | Loss: 2.9270 | LR: 6.46e-07 | 3845.5ms/step | 5.2 samples/s +Step 43020/44000 | Loss: 2.9059 | LR: 6.33e-07 | 1325.1ms/step | 15.1 samples/s +Step 43030/44000 | Loss: 2.9047 | LR: 6.20e-07 | 1325.2ms/step | 15.1 samples/s +Step 43040/44000 | Loss: 2.9220 | LR: 6.08e-07 | 1325.7ms/step | 15.1 samples/s +Step 43050/44000 | Loss: 2.8931 | LR: 5.95e-07 | 1325.1ms/step | 15.1 samples/s +Step 43060/44000 | Loss: 2.9290 | LR: 5.83e-07 | 1325.2ms/step | 15.1 samples/s +Step 43070/44000 | Loss: 2.8932 | LR: 5.70e-07 | 1324.4ms/step | 15.1 samples/s +Step 43080/44000 | Loss: 2.9142 | LR: 5.58e-07 | 1326.2ms/step | 15.1 samples/s +Step 43090/44000 | Loss: 2.9149 | LR: 5.46e-07 | 1326.4ms/step | 15.1 samples/s +Step 43100/44000 | Loss: 2.9118 | LR: 5.34e-07 | 1326.2ms/step | 15.1 samples/s +Step 43110/44000 | Loss: 2.9410 | LR: 5.22e-07 | 1325.9ms/step | 15.1 samples/s +Step 43120/44000 | Loss: 2.9166 | LR: 5.11e-07 | 1325.0ms/step | 15.1 samples/s +Step 43130/44000 | Loss: 2.9390 | LR: 4.99e-07 | 1324.2ms/step | 15.1 samples/s +Step 43140/44000 | Loss: 2.9230 | LR: 4.88e-07 | 1323.3ms/step | 15.1 samples/s +Step 43150/44000 | Loss: 2.9265 | LR: 4.76e-07 | 1324.0ms/step | 15.1 samples/s +Step 43160/44000 | Loss: 2.9111 | LR: 4.65e-07 | 1324.4ms/step | 15.1 samples/s +Step 43170/44000 | Loss: 2.9294 | LR: 4.54e-07 | 1326.9ms/step | 15.1 samples/s +Step 43180/44000 | Loss: 2.8915 | LR: 4.43e-07 | 1324.8ms/step | 15.1 samples/s +Step 43190/44000 | Loss: 2.9093 | LR: 4.33e-07 | 1324.4ms/step | 15.1 samples/s +Step 43200/44000 | Loss: 2.9393 | LR: 4.22e-07 | 1326.8ms/step | 15.1 samples/s +Step 43210/44000 | Loss: 2.9198 | LR: 4.12e-07 | 1327.3ms/step | 15.1 samples/s +Step 43220/44000 | Loss: 2.9268 | LR: 4.01e-07 | 1326.8ms/step | 15.1 samples/s +Step 43230/44000 | Loss: 2.8799 | LR: 3.91e-07 | 1324.2ms/step | 15.1 samples/s +Step 43240/44000 | Loss: 2.9356 | LR: 3.81e-07 | 1326.0ms/step | 15.1 samples/s +Step 43250/44000 | Loss: 2.9433 | LR: 3.71e-07 | 1325.2ms/step | 15.1 samples/s +Validation at step 43250: Loss = 2.9253, Perplexity = 18.64 +Step 43260/44000 | Loss: 2.8830 | LR: 3.61e-07 | 3540.4ms/step | 5.6 samples/s +Step 43270/44000 | Loss: 2.8935 | LR: 3.51e-07 | 1326.2ms/step | 15.1 samples/s +Step 43280/44000 | Loss: 2.8987 | LR: 3.42e-07 | 1325.4ms/step | 15.1 samples/s +Step 43290/44000 | Loss: 2.9376 | LR: 3.32e-07 | 1326.8ms/step | 15.1 samples/s +Step 43300/44000 | Loss: 2.9518 | LR: 3.23e-07 | 1325.4ms/step | 15.1 samples/s +Step 43310/44000 | Loss: 2.9423 | LR: 3.14e-07 | 1328.3ms/step | 15.1 samples/s +Step 43320/44000 | Loss: 2.9031 | LR: 3.05e-07 | 1326.2ms/step | 15.1 samples/s +Step 43330/44000 | Loss: 2.9043 | LR: 2.96e-07 | 1326.2ms/step | 15.1 samples/s +Step 43340/44000 | Loss: 2.9105 | LR: 2.87e-07 | 1325.0ms/step | 15.1 samples/s +Step 43350/44000 | Loss: 2.9265 | LR: 2.79e-07 | 1322.9ms/step | 15.1 samples/s +Step 43360/44000 | Loss: 2.8970 | LR: 2.70e-07 | 1324.2ms/step | 15.1 samples/s +Step 43370/44000 | Loss: 2.9087 | LR: 2.62e-07 | 1325.7ms/step | 15.1 samples/s +Step 43380/44000 | Loss: 2.9399 | LR: 2.53e-07 | 1326.7ms/step | 15.1 samples/s +Step 43390/44000 | Loss: 2.9034 | LR: 2.45e-07 | 1323.0ms/step | 15.1 samples/s +Step 43400/44000 | Loss: 2.9341 | LR: 2.37e-07 | 1325.0ms/step | 15.1 samples/s +Step 43410/44000 | Loss: 2.8924 | LR: 2.30e-07 | 1327.7ms/step | 15.1 samples/s +Step 43420/44000 | Loss: 2.9034 | LR: 2.22e-07 | 1329.9ms/step | 15.0 samples/s +Step 43430/44000 | Loss: 2.9142 | LR: 2.14e-07 | 1325.3ms/step | 15.1 samples/s +Step 43440/44000 | Loss: 2.8858 | LR: 2.07e-07 | 1325.4ms/step | 15.1 samples/s +Step 43450/44000 | Loss: 2.9165 | LR: 1.99e-07 | 1326.7ms/step | 15.1 samples/s +Step 43460/44000 | Loss: 2.8895 | LR: 1.92e-07 | 1325.6ms/step | 15.1 samples/s +Step 43470/44000 | Loss: 2.9238 | LR: 1.85e-07 | 1321.9ms/step | 15.1 samples/s +Step 43480/44000 | Loss: 2.9211 | LR: 1.78e-07 | 1322.3ms/step | 15.1 samples/s +Step 43490/44000 | Loss: 2.9029 | LR: 1.72e-07 | 1326.3ms/step | 15.1 samples/s +Step 43500/44000 | Loss: 2.9477 | LR: 1.65e-07 | 1326.0ms/step | 15.1 samples/s +Validation at step 43500: Loss = 2.9253, Perplexity = 18.64 +Step 43510/44000 | Loss: 2.9219 | LR: 1.58e-07 | 3538.2ms/step | 5.7 samples/s +Step 43520/44000 | Loss: 2.9434 | LR: 1.52e-07 | 1325.9ms/step | 15.1 samples/s +Step 43530/44000 | Loss: 2.9044 | LR: 1.46e-07 | 1328.6ms/step | 15.1 samples/s +Step 43540/44000 | Loss: 2.9418 | LR: 1.40e-07 | 1327.5ms/step | 15.1 samples/s +Step 43550/44000 | Loss: 2.9020 | LR: 1.34e-07 | 1328.6ms/step | 15.1 samples/s +Step 43560/44000 | Loss: 2.9021 | LR: 1.28e-07 | 1326.6ms/step | 15.1 samples/s +Step 43570/44000 | Loss: 2.9076 | LR: 1.22e-07 | 1324.5ms/step | 15.1 samples/s +Step 43580/44000 | Loss: 2.9389 | LR: 1.16e-07 | 1323.3ms/step | 15.1 samples/s +Step 43590/44000 | Loss: 2.9368 | LR: 1.11e-07 | 1324.8ms/step | 15.1 samples/s +Step 43600/44000 | Loss: 2.9135 | LR: 1.06e-07 | 1325.1ms/step | 15.1 samples/s +Step 43610/44000 | Loss: 2.9187 | LR: 1.00e-07 | 1324.4ms/step | 15.1 samples/s +Step 43620/44000 | Loss: 2.9192 | LR: 9.52e-08 | 1326.6ms/step | 15.1 samples/s +Step 43630/44000 | Loss: 2.9246 | LR: 9.03e-08 | 1325.4ms/step | 15.1 samples/s +Step 43640/44000 | Loss: 2.9006 | LR: 8.55e-08 | 1324.7ms/step | 15.1 samples/s +Step 43650/44000 | Loss: 2.9073 | LR: 8.08e-08 | 1324.2ms/step | 15.1 samples/s +Step 43660/44000 | Loss: 2.9280 | LR: 7.62e-08 | 1325.3ms/step | 15.1 samples/s +Step 43670/44000 | Loss: 2.9326 | LR: 7.18e-08 | 1324.5ms/step | 15.1 samples/s +Step 43680/44000 | Loss: 2.9285 | LR: 6.75e-08 | 1325.7ms/step | 15.1 samples/s +Step 43690/44000 | Loss: 2.9280 | LR: 6.34e-08 | 1323.8ms/step | 15.1 samples/s +Step 43700/44000 | Loss: 2.9035 | LR: 5.94e-08 | 1326.9ms/step | 15.1 samples/s +Step 43710/44000 | Loss: 2.9369 | LR: 5.55e-08 | 1326.1ms/step | 15.1 samples/s +Step 43720/44000 | Loss: 2.8596 | LR: 5.17e-08 | 1326.7ms/step | 15.1 samples/s +Step 43730/44000 | Loss: 2.9372 | LR: 4.81e-08 | 1326.0ms/step | 15.1 samples/s +Step 43740/44000 | Loss: 2.9115 | LR: 4.46e-08 | 1326.2ms/step | 15.1 samples/s +Step 43750/44000 | Loss: 2.8984 | LR: 4.12e-08 | 1324.6ms/step | 15.1 samples/s +Validation at step 43750: Loss = 2.9253, Perplexity = 18.64 +Step 43760/44000 | Loss: 2.8937 | LR: 3.80e-08 | 3538.2ms/step | 5.7 samples/s +Step 43770/44000 | Loss: 2.9167 | LR: 3.49e-08 | 1326.0ms/step | 15.1 samples/s +Step 43780/44000 | Loss: 2.9057 | LR: 3.19e-08 | 1324.9ms/step | 15.1 samples/s +Step 43790/44000 | Loss: 2.9700 | LR: 2.91e-08 | 1325.3ms/step | 15.1 samples/s +Step 43800/44000 | Loss: 2.9154 | LR: 2.64e-08 | 1325.7ms/step | 15.1 samples/s +Step 43810/44000 | Loss: 2.9269 | LR: 2.38e-08 | 1324.7ms/step | 15.1 samples/s +Step 43820/44000 | Loss: 2.9265 | LR: 2.14e-08 | 1323.0ms/step | 15.1 samples/s +Step 43830/44000 | Loss: 2.9251 | LR: 1.91e-08 | 1324.7ms/step | 15.1 samples/s +Step 43840/44000 | Loss: 2.9072 | LR: 1.69e-08 | 1325.9ms/step | 15.1 samples/s +Step 43850/44000 | Loss: 2.9170 | LR: 1.48e-08 | 1326.3ms/step | 15.1 samples/s +Step 43860/44000 | Loss: 2.9188 | LR: 1.29e-08 | 1327.2ms/step | 15.1 samples/s +Step 43870/44000 | Loss: 2.9313 | LR: 1.11e-08 | 1324.4ms/step | 15.1 samples/s +Step 43880/44000 | Loss: 2.8865 | LR: 9.50e-09 | 1323.5ms/step | 15.1 samples/s +Step 43890/44000 | Loss: 2.9152 | LR: 7.98e-09 | 1324.2ms/step | 15.1 samples/s +Step 43900/44000 | Loss: 2.9140 | LR: 6.60e-09 | 1321.4ms/step | 15.1 samples/s +Step 43910/44000 | Loss: 2.9248 | LR: 5.34e-09 | 1323.9ms/step | 15.1 samples/s +Step 43920/44000 | Loss: 2.8949 | LR: 4.22e-09 | 1324.5ms/step | 15.1 samples/s +Step 43930/44000 | Loss: 2.9165 | LR: 3.23e-09 | 1324.9ms/step | 15.1 samples/s +Step 43940/44000 | Loss: 2.9263 | LR: 2.37e-09 | 1325.3ms/step | 15.1 samples/s +Step 43950/44000 | Loss: 2.9453 | LR: 1.65e-09 | 1326.3ms/step | 15.1 samples/s +Step 43960/44000 | Loss: 2.9518 | LR: 1.06e-09 | 1326.1ms/step | 15.1 samples/s +Step 43970/44000 | Loss: 2.9049 | LR: 5.94e-10 | 1326.6ms/step | 15.1 samples/s +Step 43980/44000 | Loss: 2.9164 | LR: 2.64e-10 | 1325.0ms/step | 15.1 samples/s +Step 43990/44000 | Loss: 2.9197 | LR: 6.60e-11 | 1325.5ms/step | 15.1 samples/s +Step 44000/44000 | Loss: 2.9007 | LR: 0.00e+00 | 1324.0ms/step | 15.1 samples/s +Validation at step 44000: Loss = 2.9253, Perplexity = 18.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading history steps 4399-4399, summary, console lines 1917-1919 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss ██▇▆▆▆▅▅▅▅▄▄▄▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▇▅▅▅▄▄▃▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁█▁▁▁▁▁▇▁▁▁▇▁▇▁▁▁▇▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ███████████████████████████████████▁████ +wandb: speed/steps_per_sec ████████████████████████████████▁███████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.92529 +wandb: eval/perplexity 18.63964 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 1324.00978 +wandb: speed/samples_per_sec 15.10563 +wandb: speed/steps_per_sec 0.75528 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 resume 365m static 8k nrb35 seed44 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/na2e5dc1 +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_043357-na2e5dc1/logs +Training complete! diff --git a/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_resampled_nrb35_seed42.log b/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_resampled_nrb35_seed42.log new file mode 100644 index 0000000000000000000000000000000000000000..907c7c93a6675af98e9a3abf1443f4d4bd817b6e --- /dev/null +++ b/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_resampled_nrb35_seed42.log @@ -0,0 +1,2955 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-19000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 resume 365m resampled 8k nrb35 seed42", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "resampled", + "nrb35", + "seed42", + "resume", + "h1004", + "step19000" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-19000 +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-19000... +Preserving checkpoint LR state for exact resume: current=[0.0003106762328525677], schedule_base=[0.0005] +Resumed from checkpoint at step 19000 (epoch 0, micro_step 190000) +Resuming training from step 19000 +Resuming wandb run n56uphgo +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run n56uphgo +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042436-n56uphgo +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run P1 resume 365m resampled 8k nrb35 seed42 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/n56uphgo +Starting training... +Validation at step 19000: Loss = 3.0254, Perplexity = 20.60 +wandb: WARNING Tried to log to step 19000 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 19001: sparse_stochastic (configured sparse attention) +Step 19010/44000 | Loss: 2.9795 | LR: 3.11e-04 | 37888.4ms/step | 0.5 samples/s +wandb: WARNING Tried to log to step 19010 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19020/44000 | Loss: 3.0176 | LR: 3.10e-04 | 1335.9ms/step | 15.0 samples/s +wandb: WARNING Tried to log to step 19020 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19030/44000 | Loss: 2.9970 | LR: 3.10e-04 | 1343.0ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19030 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19040/44000 | Loss: 3.0084 | LR: 3.10e-04 | 1344.0ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19040 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19050/44000 | Loss: 3.0254 | LR: 3.10e-04 | 1357.3ms/step | 14.7 samples/s +wandb: WARNING Tried to log to step 19050 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19060/44000 | Loss: 3.0378 | LR: 3.10e-04 | 1342.1ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19060 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19070/44000 | Loss: 3.0122 | LR: 3.09e-04 | 1343.5ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19070 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19080/44000 | Loss: 2.9816 | LR: 3.09e-04 | 1343.9ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19080 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19090/44000 | Loss: 3.0036 | LR: 3.09e-04 | 1341.5ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19090 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19100/44000 | Loss: 2.9866 | LR: 3.09e-04 | 1343.7ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19100 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19110/44000 | Loss: 3.0184 | LR: 3.09e-04 | 1341.0ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19110 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19120/44000 | Loss: 3.0103 | LR: 3.09e-04 | 1340.0ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19120 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19130/44000 | Loss: 3.0241 | LR: 3.08e-04 | 1341.1ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19130 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19140/44000 | Loss: 2.9901 | LR: 3.08e-04 | 1342.2ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19140 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19150/44000 | Loss: 3.0037 | LR: 3.08e-04 | 1344.1ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19150 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19160/44000 | Loss: 3.0052 | LR: 3.08e-04 | 1344.8ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19160 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19170/44000 | Loss: 2.9655 | LR: 3.08e-04 | 1344.4ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19170 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19180/44000 | Loss: 3.0276 | LR: 3.08e-04 | 1341.5ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19180 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19190/44000 | Loss: 2.9799 | LR: 3.07e-04 | 1340.7ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19190 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19200/44000 | Loss: 2.9863 | LR: 3.07e-04 | 1342.1ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19200 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19210/44000 | Loss: 3.0127 | LR: 3.07e-04 | 1344.2ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19210 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19220/44000 | Loss: 2.9988 | LR: 3.07e-04 | 1341.1ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19220 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19230/44000 | Loss: 3.0626 | LR: 3.07e-04 | 1340.0ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19230 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19240/44000 | Loss: 3.0143 | LR: 3.06e-04 | 1340.8ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19240 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19250/44000 | Loss: 2.9982 | LR: 3.06e-04 | 1340.6ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19250 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 19250: Loss = 3.0238, Perplexity = 20.57 +Step 19260/44000 | Loss: 3.0085 | LR: 3.06e-04 | 3613.3ms/step | 5.5 samples/s +wandb: WARNING Tried to log to step 19260 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19270/44000 | Loss: 2.9949 | LR: 3.06e-04 | 1346.9ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19270 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19280/44000 | Loss: 2.9824 | LR: 3.06e-04 | 1349.0ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19280 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19290/44000 | Loss: 3.0049 | LR: 3.06e-04 | 1348.5ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19290 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19300/44000 | Loss: 2.9946 | LR: 3.05e-04 | 1349.9ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19300 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19310/44000 | Loss: 3.0196 | LR: 3.05e-04 | 1349.8ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19310 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19320/44000 | Loss: 3.0176 | LR: 3.05e-04 | 1349.8ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19320 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19330/44000 | Loss: 3.0297 | LR: 3.05e-04 | 1349.5ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19330 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19340/44000 | Loss: 3.0229 | LR: 3.05e-04 | 1348.6ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19340 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19350/44000 | Loss: 3.0092 | LR: 3.04e-04 | 1347.6ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19350 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19360/44000 | Loss: 3.0221 | LR: 3.04e-04 | 1347.1ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19360 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19370/44000 | Loss: 3.0156 | LR: 3.04e-04 | 1346.6ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19370 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19380/44000 | Loss: 2.9797 | LR: 3.04e-04 | 1344.1ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19380 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19390/44000 | Loss: 2.9868 | LR: 3.04e-04 | 1344.4ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19390 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19400/44000 | Loss: 3.0088 | LR: 3.04e-04 | 1347.5ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19400 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19410/44000 | Loss: 2.9982 | LR: 3.03e-04 | 1345.6ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19410 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19420/44000 | Loss: 3.0143 | LR: 3.03e-04 | 1348.9ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19420 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19430/44000 | Loss: 2.9756 | LR: 3.03e-04 | 1347.7ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19430 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19440/44000 | Loss: 3.0272 | LR: 3.03e-04 | 1347.7ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19440 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19450/44000 | Loss: 2.9991 | LR: 3.03e-04 | 1349.5ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19450 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19460/44000 | Loss: 3.0135 | LR: 3.03e-04 | 1345.7ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19460 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19470/44000 | Loss: 3.0215 | LR: 3.02e-04 | 1346.8ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19470 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19480/44000 | Loss: 2.9777 | LR: 3.02e-04 | 1344.6ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19480 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19490/44000 | Loss: 3.0272 | LR: 3.02e-04 | 1346.7ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19490 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19500/44000 | Loss: 2.9871 | LR: 3.02e-04 | 1349.0ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19500 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 19500: Loss = 3.0200, Perplexity = 20.49 +Step 19510/44000 | Loss: 3.0115 | LR: 3.02e-04 | 3606.5ms/step | 5.5 samples/s +wandb: WARNING Tried to log to step 19510 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19520/44000 | Loss: 2.9945 | LR: 3.01e-04 | 1347.0ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19520 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19530/44000 | Loss: 2.9731 | LR: 3.01e-04 | 1347.5ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19530 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19540/44000 | Loss: 3.0037 | LR: 3.01e-04 | 1349.2ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19540 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19550/44000 | Loss: 2.9922 | LR: 3.01e-04 | 1347.3ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19550 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19560/44000 | Loss: 2.9963 | LR: 3.01e-04 | 1344.0ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19560 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19570/44000 | Loss: 3.0122 | LR: 3.01e-04 | 1344.8ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19570 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19580/44000 | Loss: 3.0224 | LR: 3.00e-04 | 1343.4ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19580 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19590/44000 | Loss: 3.0081 | LR: 3.00e-04 | 1344.2ms/step | 14.9 samples/s +wandb: WARNING Tried to log to step 19590 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19600/44000 | Loss: 2.9819 | LR: 3.00e-04 | 1354.7ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19600 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19610/44000 | Loss: 3.0039 | LR: 3.00e-04 | 1378.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 19610 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19620/44000 | Loss: 3.0010 | LR: 3.00e-04 | 1371.8ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19620 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19630/44000 | Loss: 3.0269 | LR: 3.00e-04 | 1367.4ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19630 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19640/44000 | Loss: 3.0203 | LR: 2.99e-04 | 1369.1ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19640 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19650/44000 | Loss: 2.9883 | LR: 2.99e-04 | 1355.4ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19650 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19660/44000 | Loss: 3.0035 | LR: 2.99e-04 | 1367.3ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19660 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19670/44000 | Loss: 3.0126 | LR: 2.99e-04 | 1361.1ms/step | 14.7 samples/s +wandb: WARNING Tried to log to step 19670 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19680/44000 | Loss: 3.0276 | LR: 2.99e-04 | 1387.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 19680 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19690/44000 | Loss: 2.9870 | LR: 2.98e-04 | 1372.5ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19690 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19700/44000 | Loss: 3.0206 | LR: 2.98e-04 | 1370.9ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19700 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19710/44000 | Loss: 3.0141 | LR: 2.98e-04 | 1360.3ms/step | 14.7 samples/s +wandb: WARNING Tried to log to step 19710 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19720/44000 | Loss: 3.0017 | LR: 2.98e-04 | 1362.9ms/step | 14.7 samples/s +wandb: WARNING Tried to log to step 19720 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19730/44000 | Loss: 3.0277 | LR: 2.98e-04 | 1367.3ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19730 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19740/44000 | Loss: 3.0025 | LR: 2.98e-04 | 1366.3ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19740 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19750/44000 | Loss: 2.9838 | LR: 2.97e-04 | 1354.7ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19750 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 19750: Loss = 3.0181, Perplexity = 20.45 +Step 19760/44000 | Loss: 3.0176 | LR: 2.97e-04 | 3631.8ms/step | 5.5 samples/s +wandb: WARNING Tried to log to step 19760 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19770/44000 | Loss: 3.0237 | LR: 2.97e-04 | 1356.4ms/step | 14.7 samples/s +wandb: WARNING Tried to log to step 19770 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19780/44000 | Loss: 3.0202 | LR: 2.97e-04 | 1356.9ms/step | 14.7 samples/s +wandb: WARNING Tried to log to step 19780 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19790/44000 | Loss: 2.9995 | LR: 2.97e-04 | 1349.6ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19790 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19800/44000 | Loss: 3.0278 | LR: 2.96e-04 | 1350.1ms/step | 14.8 samples/s +wandb: WARNING Tried to log to step 19800 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19810/44000 | Loss: 2.9871 | LR: 2.96e-04 | 1390.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 19810 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19820/44000 | Loss: 2.9774 | LR: 2.96e-04 | 1400.8ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 19820 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19830/44000 | Loss: 3.0267 | LR: 2.96e-04 | 1387.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 19830 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19840/44000 | Loss: 3.0002 | LR: 2.96e-04 | 1366.9ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19840 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19850/44000 | Loss: 2.9956 | LR: 2.96e-04 | 1366.6ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19850 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19860/44000 | Loss: 3.0278 | LR: 2.95e-04 | 1369.7ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19860 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19870/44000 | Loss: 3.0230 | LR: 2.95e-04 | 1371.3ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19870 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19880/44000 | Loss: 3.0081 | LR: 2.95e-04 | 1365.9ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19880 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19890/44000 | Loss: 3.0110 | LR: 2.95e-04 | 1369.0ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19890 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19900/44000 | Loss: 2.9945 | LR: 2.95e-04 | 1370.2ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19900 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19910/44000 | Loss: 3.0051 | LR: 2.95e-04 | 1369.0ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19910 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19920/44000 | Loss: 3.0005 | LR: 2.94e-04 | 1372.5ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19920 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19930/44000 | Loss: 3.0115 | LR: 2.94e-04 | 1390.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 19930 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19940/44000 | Loss: 2.9736 | LR: 2.94e-04 | 1372.6ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19940 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19950/44000 | Loss: 2.9949 | LR: 2.94e-04 | 1371.8ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19950 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19960/44000 | Loss: 3.0340 | LR: 2.94e-04 | 1371.0ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 19960 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19970/44000 | Loss: 3.0234 | LR: 2.93e-04 | 1376.6ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 19970 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19980/44000 | Loss: 3.0193 | LR: 2.93e-04 | 1364.4ms/step | 14.7 samples/s +wandb: WARNING Tried to log to step 19980 that is less than the current step 19981. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 19990/44000 | Loss: 3.0069 | LR: 2.93e-04 | 1387.4ms/step | 14.4 samples/s +Step 20000/44000 | Loss: 3.0118 | LR: 2.93e-04 | 1370.2ms/step | 14.6 samples/s +Validation at step 20000: Loss = 3.0148, Perplexity = 20.39 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-20000 +Step 20010/44000 | Loss: 2.9882 | LR: 2.93e-04 | 3926.7ms/step | 5.1 samples/s +Step 20020/44000 | Loss: 3.0116 | LR: 2.93e-04 | 1392.4ms/step | 14.4 samples/s +Step 20030/44000 | Loss: 2.9849 | LR: 2.92e-04 | 1368.8ms/step | 14.6 samples/s +Step 20040/44000 | Loss: 2.9905 | LR: 2.92e-04 | 1370.9ms/step | 14.6 samples/s +Step 20050/44000 | Loss: 3.0049 | LR: 2.92e-04 | 1364.5ms/step | 14.7 samples/s +Step 20060/44000 | Loss: 2.9754 | LR: 2.92e-04 | 1363.9ms/step | 14.7 samples/s +Step 20070/44000 | Loss: 3.0004 | LR: 2.92e-04 | 1360.0ms/step | 14.7 samples/s +Step 20080/44000 | Loss: 2.9830 | LR: 2.91e-04 | 1353.4ms/step | 14.8 samples/s +Step 20090/44000 | Loss: 2.9915 | LR: 2.91e-04 | 1349.2ms/step | 14.8 samples/s +Step 20100/44000 | Loss: 3.0081 | LR: 2.91e-04 | 1349.9ms/step | 14.8 samples/s +Step 20110/44000 | Loss: 3.0004 | LR: 2.91e-04 | 1393.4ms/step | 14.4 samples/s +Step 20120/44000 | Loss: 3.0096 | LR: 2.91e-04 | 1379.9ms/step | 14.5 samples/s +Step 20130/44000 | Loss: 3.0062 | LR: 2.91e-04 | 1369.1ms/step | 14.6 samples/s +Step 20140/44000 | Loss: 3.0076 | LR: 2.90e-04 | 1375.6ms/step | 14.5 samples/s +Step 20150/44000 | Loss: 2.9942 | LR: 2.90e-04 | 1388.2ms/step | 14.4 samples/s +Step 20160/44000 | Loss: 3.0095 | LR: 2.90e-04 | 1369.6ms/step | 14.6 samples/s +Step 20170/44000 | Loss: 2.9642 | LR: 2.90e-04 | 1366.1ms/step | 14.6 samples/s +Step 20180/44000 | Loss: 2.9867 | LR: 2.90e-04 | 1376.3ms/step | 14.5 samples/s +Step 20190/44000 | Loss: 3.0163 | LR: 2.90e-04 | 1389.8ms/step | 14.4 samples/s +Step 20200/44000 | Loss: 2.9766 | LR: 2.89e-04 | 1383.9ms/step | 14.5 samples/s +Step 20210/44000 | Loss: 2.9668 | LR: 2.89e-04 | 1384.7ms/step | 14.4 samples/s +Step 20220/44000 | Loss: 3.0167 | LR: 2.89e-04 | 1385.5ms/step | 14.4 samples/s +Step 20230/44000 | Loss: 3.0044 | LR: 2.89e-04 | 1382.1ms/step | 14.5 samples/s +Step 20240/44000 | Loss: 3.0056 | LR: 2.89e-04 | 1382.9ms/step | 14.5 samples/s +Step 20250/44000 | Loss: 2.9897 | LR: 2.88e-04 | 1385.1ms/step | 14.4 samples/s +Validation at step 20250: Loss = 3.0124, Perplexity = 20.34 +Step 20260/44000 | Loss: 2.9589 | LR: 2.88e-04 | 3704.0ms/step | 5.4 samples/s +Step 20270/44000 | Loss: 3.0084 | LR: 2.88e-04 | 1381.1ms/step | 14.5 samples/s +Step 20280/44000 | Loss: 3.0050 | LR: 2.88e-04 | 1381.4ms/step | 14.5 samples/s +Step 20290/44000 | Loss: 2.9978 | LR: 2.88e-04 | 1378.3ms/step | 14.5 samples/s +Step 20300/44000 | Loss: 3.0206 | LR: 2.88e-04 | 1377.3ms/step | 14.5 samples/s +Step 20310/44000 | Loss: 2.9872 | LR: 2.87e-04 | 1374.7ms/step | 14.5 samples/s +Step 20320/44000 | Loss: 2.9947 | LR: 2.87e-04 | 1373.9ms/step | 14.6 samples/s +Step 20330/44000 | Loss: 2.9849 | LR: 2.87e-04 | 1374.6ms/step | 14.5 samples/s +Step 20340/44000 | Loss: 2.9735 | LR: 2.87e-04 | 1376.1ms/step | 14.5 samples/s +Step 20350/44000 | Loss: 3.0018 | LR: 2.87e-04 | 1376.0ms/step | 14.5 samples/s +Step 20360/44000 | Loss: 3.0159 | LR: 2.86e-04 | 1376.8ms/step | 14.5 samples/s +Step 20370/44000 | Loss: 2.9684 | LR: 2.86e-04 | 1373.9ms/step | 14.6 samples/s +Step 20380/44000 | Loss: 2.9777 | LR: 2.86e-04 | 1373.3ms/step | 14.6 samples/s +Step 20390/44000 | Loss: 2.9968 | LR: 2.86e-04 | 1370.3ms/step | 14.6 samples/s +Step 20400/44000 | Loss: 3.0132 | LR: 2.86e-04 | 1373.4ms/step | 14.6 samples/s +Step 20410/44000 | Loss: 2.9843 | LR: 2.86e-04 | 1370.2ms/step | 14.6 samples/s +Step 20420/44000 | Loss: 3.0102 | LR: 2.85e-04 | 1370.0ms/step | 14.6 samples/s +Step 20430/44000 | Loss: 2.9741 | LR: 2.85e-04 | 1373.8ms/step | 14.6 samples/s +Step 20440/44000 | Loss: 3.0044 | LR: 2.85e-04 | 1374.8ms/step | 14.5 samples/s +Step 20450/44000 | Loss: 2.9942 | LR: 2.85e-04 | 1372.8ms/step | 14.6 samples/s +Step 20460/44000 | Loss: 2.9991 | LR: 2.85e-04 | 1373.3ms/step | 14.6 samples/s +Step 20470/44000 | Loss: 3.0155 | LR: 2.84e-04 | 1372.8ms/step | 14.6 samples/s +Step 20480/44000 | Loss: 2.9931 | LR: 2.84e-04 | 1372.7ms/step | 14.6 samples/s +Step 20490/44000 | Loss: 2.9926 | LR: 2.84e-04 | 1372.7ms/step | 14.6 samples/s +Step 20500/44000 | Loss: 2.9748 | LR: 2.84e-04 | 1371.1ms/step | 14.6 samples/s +Validation at step 20500: Loss = 3.0099, Perplexity = 20.28 +Step 20510/44000 | Loss: 3.0215 | LR: 2.84e-04 | 3681.5ms/step | 5.4 samples/s +Step 20520/44000 | Loss: 2.9757 | LR: 2.84e-04 | 1370.4ms/step | 14.6 samples/s +Step 20530/44000 | Loss: 2.9676 | LR: 2.83e-04 | 1372.4ms/step | 14.6 samples/s +Step 20540/44000 | Loss: 2.9964 | LR: 2.83e-04 | 1371.6ms/step | 14.6 samples/s +Step 20550/44000 | Loss: 3.0056 | LR: 2.83e-04 | 1370.3ms/step | 14.6 samples/s +Step 20560/44000 | Loss: 3.0099 | LR: 2.83e-04 | 1370.3ms/step | 14.6 samples/s +Step 20570/44000 | Loss: 3.0050 | LR: 2.83e-04 | 1370.7ms/step | 14.6 samples/s +Step 20580/44000 | Loss: 2.9772 | LR: 2.83e-04 | 1370.3ms/step | 14.6 samples/s +Step 20590/44000 | Loss: 3.0160 | LR: 2.82e-04 | 1371.5ms/step | 14.6 samples/s +Step 20600/44000 | Loss: 2.9926 | LR: 2.82e-04 | 1369.9ms/step | 14.6 samples/s +Step 20610/44000 | Loss: 3.0007 | LR: 2.82e-04 | 1370.8ms/step | 14.6 samples/s +Step 20620/44000 | Loss: 2.9928 | LR: 2.82e-04 | 1369.2ms/step | 14.6 samples/s +Step 20630/44000 | Loss: 3.0031 | LR: 2.82e-04 | 1370.3ms/step | 14.6 samples/s +Step 20640/44000 | Loss: 2.9617 | LR: 2.81e-04 | 1366.0ms/step | 14.6 samples/s +Step 20650/44000 | Loss: 3.0103 | LR: 2.81e-04 | 1365.3ms/step | 14.6 samples/s +Step 20660/44000 | Loss: 2.9623 | LR: 2.81e-04 | 1368.8ms/step | 14.6 samples/s +Step 20670/44000 | Loss: 3.0096 | LR: 2.81e-04 | 1365.7ms/step | 14.6 samples/s +Step 20680/44000 | Loss: 3.0008 | LR: 2.81e-04 | 1371.7ms/step | 14.6 samples/s +Step 20690/44000 | Loss: 2.9887 | LR: 2.81e-04 | 1369.7ms/step | 14.6 samples/s +Step 20700/44000 | Loss: 2.9997 | LR: 2.80e-04 | 1370.1ms/step | 14.6 samples/s +Step 20710/44000 | Loss: 3.0058 | LR: 2.80e-04 | 1369.2ms/step | 14.6 samples/s +Step 20720/44000 | Loss: 3.0051 | LR: 2.80e-04 | 1371.2ms/step | 14.6 samples/s +Step 20730/44000 | Loss: 3.0189 | LR: 2.80e-04 | 1371.9ms/step | 14.6 samples/s +Step 20740/44000 | Loss: 3.0021 | LR: 2.80e-04 | 1370.5ms/step | 14.6 samples/s +Step 20750/44000 | Loss: 2.9967 | LR: 2.79e-04 | 1368.9ms/step | 14.6 samples/s +Validation at step 20750: Loss = 3.0068, Perplexity = 20.22 +Step 20760/44000 | Loss: 3.0041 | LR: 2.79e-04 | 3673.3ms/step | 5.4 samples/s +Step 20770/44000 | Loss: 2.9876 | LR: 2.79e-04 | 1371.0ms/step | 14.6 samples/s +Step 20780/44000 | Loss: 2.9637 | LR: 2.79e-04 | 1372.9ms/step | 14.6 samples/s +Step 20790/44000 | Loss: 2.9833 | LR: 2.79e-04 | 1370.3ms/step | 14.6 samples/s +Step 20800/44000 | Loss: 2.9844 | LR: 2.79e-04 | 1369.3ms/step | 14.6 samples/s +Step 20810/44000 | Loss: 3.0025 | LR: 2.78e-04 | 1370.2ms/step | 14.6 samples/s +Step 20820/44000 | Loss: 2.9983 | LR: 2.78e-04 | 1371.2ms/step | 14.6 samples/s +Step 20830/44000 | Loss: 3.0031 | LR: 2.78e-04 | 1372.4ms/step | 14.6 samples/s +Step 20840/44000 | Loss: 2.9898 | LR: 2.78e-04 | 1372.7ms/step | 14.6 samples/s +Step 20850/44000 | Loss: 2.9897 | LR: 2.78e-04 | 1366.8ms/step | 14.6 samples/s +Step 20860/44000 | Loss: 3.0081 | LR: 2.77e-04 | 1366.2ms/step | 14.6 samples/s +Step 20870/44000 | Loss: 3.0137 | LR: 2.77e-04 | 1366.7ms/step | 14.6 samples/s +Step 20880/44000 | Loss: 2.9862 | LR: 2.77e-04 | 1368.9ms/step | 14.6 samples/s +Step 20890/44000 | Loss: 2.9717 | LR: 2.77e-04 | 1365.9ms/step | 14.6 samples/s +Step 20900/44000 | Loss: 2.9726 | LR: 2.77e-04 | 1367.8ms/step | 14.6 samples/s +Step 20910/44000 | Loss: 3.0050 | LR: 2.77e-04 | 1364.6ms/step | 14.7 samples/s +Step 20920/44000 | Loss: 2.9666 | LR: 2.76e-04 | 1363.1ms/step | 14.7 samples/s +Step 20930/44000 | Loss: 2.9758 | LR: 2.76e-04 | 1366.7ms/step | 14.6 samples/s +Step 20940/44000 | Loss: 3.0011 | LR: 2.76e-04 | 1365.4ms/step | 14.6 samples/s +Step 20950/44000 | Loss: 2.9645 | LR: 2.76e-04 | 1364.0ms/step | 14.7 samples/s +Step 20960/44000 | Loss: 2.9806 | LR: 2.76e-04 | 1364.8ms/step | 14.7 samples/s +Step 20970/44000 | Loss: 2.9639 | LR: 2.75e-04 | 1364.6ms/step | 14.7 samples/s +Step 20980/44000 | Loss: 2.9697 | LR: 2.75e-04 | 1364.6ms/step | 14.7 samples/s +Step 20990/44000 | Loss: 2.9759 | LR: 2.75e-04 | 1363.7ms/step | 14.7 samples/s +Step 21000/44000 | Loss: 2.9785 | LR: 2.75e-04 | 1363.9ms/step | 14.7 samples/s +Validation at step 21000: Loss = 3.0060, Perplexity = 20.21 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-21000 +Step 21010/44000 | Loss: 2.9824 | LR: 2.75e-04 | 3945.8ms/step | 5.1 samples/s +Step 21020/44000 | Loss: 2.9843 | LR: 2.75e-04 | 1366.6ms/step | 14.6 samples/s +Step 21030/44000 | Loss: 3.0220 | LR: 2.74e-04 | 1370.4ms/step | 14.6 samples/s +Step 21040/44000 | Loss: 2.9646 | LR: 2.74e-04 | 1368.1ms/step | 14.6 samples/s +Step 21050/44000 | Loss: 2.9844 | LR: 2.74e-04 | 1358.8ms/step | 14.7 samples/s +Step 21060/44000 | Loss: 2.9685 | LR: 2.74e-04 | 1361.2ms/step | 14.7 samples/s +Step 21070/44000 | Loss: 3.0115 | LR: 2.74e-04 | 1362.9ms/step | 14.7 samples/s +Step 21080/44000 | Loss: 2.9545 | LR: 2.73e-04 | 1357.1ms/step | 14.7 samples/s +Step 21090/44000 | Loss: 3.0164 | LR: 2.73e-04 | 1355.3ms/step | 14.8 samples/s +Step 21100/44000 | Loss: 2.9982 | LR: 2.73e-04 | 1360.0ms/step | 14.7 samples/s +Step 21110/44000 | Loss: 2.9886 | LR: 2.73e-04 | 1361.1ms/step | 14.7 samples/s +Step 21120/44000 | Loss: 2.9996 | LR: 2.73e-04 | 1362.3ms/step | 14.7 samples/s +Step 21130/44000 | Loss: 3.0226 | LR: 2.73e-04 | 1363.0ms/step | 14.7 samples/s +Step 21140/44000 | Loss: 2.9834 | LR: 2.72e-04 | 1362.4ms/step | 14.7 samples/s +Step 21150/44000 | Loss: 2.9926 | LR: 2.72e-04 | 1360.2ms/step | 14.7 samples/s +Step 21160/44000 | Loss: 2.9718 | LR: 2.72e-04 | 1361.2ms/step | 14.7 samples/s +Step 21170/44000 | Loss: 2.9892 | LR: 2.72e-04 | 1362.6ms/step | 14.7 samples/s +Step 21180/44000 | Loss: 2.9960 | LR: 2.72e-04 | 1366.3ms/step | 14.6 samples/s +Step 21190/44000 | Loss: 2.9360 | LR: 2.71e-04 | 1361.6ms/step | 14.7 samples/s +Step 21200/44000 | Loss: 3.0076 | LR: 2.71e-04 | 1363.5ms/step | 14.7 samples/s +Step 21210/44000 | Loss: 2.9899 | LR: 2.71e-04 | 1362.4ms/step | 14.7 samples/s +Step 21220/44000 | Loss: 2.9793 | LR: 2.71e-04 | 1362.7ms/step | 14.7 samples/s +Step 21230/44000 | Loss: 2.9919 | LR: 2.71e-04 | 1362.3ms/step | 14.7 samples/s +Step 21240/44000 | Loss: 2.9817 | LR: 2.71e-04 | 1359.6ms/step | 14.7 samples/s +Step 21250/44000 | Loss: 2.9757 | LR: 2.70e-04 | 1360.8ms/step | 14.7 samples/s +Validation at step 21250: Loss = 3.0020, Perplexity = 20.12 +Step 21260/44000 | Loss: 3.0173 | LR: 2.70e-04 | 3646.3ms/step | 5.5 samples/s +Step 21270/44000 | Loss: 2.9955 | LR: 2.70e-04 | 1360.3ms/step | 14.7 samples/s +Step 21280/44000 | Loss: 2.9854 | LR: 2.70e-04 | 1360.7ms/step | 14.7 samples/s +Step 21290/44000 | Loss: 2.9807 | LR: 2.70e-04 | 1359.7ms/step | 14.7 samples/s +Step 21300/44000 | Loss: 2.9893 | LR: 2.70e-04 | 1362.3ms/step | 14.7 samples/s +Step 21310/44000 | Loss: 2.9907 | LR: 2.69e-04 | 1365.6ms/step | 14.6 samples/s +Step 21320/44000 | Loss: 3.0042 | LR: 2.69e-04 | 1362.9ms/step | 14.7 samples/s +Step 21330/44000 | Loss: 2.9852 | LR: 2.69e-04 | 1361.1ms/step | 14.7 samples/s +Step 21340/44000 | Loss: 2.9811 | LR: 2.69e-04 | 1359.5ms/step | 14.7 samples/s +Step 21350/44000 | Loss: 3.0150 | LR: 2.69e-04 | 1362.2ms/step | 14.7 samples/s +Step 21360/44000 | Loss: 2.9856 | LR: 2.68e-04 | 1363.8ms/step | 14.7 samples/s +Step 21370/44000 | Loss: 2.9863 | LR: 2.68e-04 | 1363.3ms/step | 14.7 samples/s +Step 21380/44000 | Loss: 2.9763 | LR: 2.68e-04 | 1363.7ms/step | 14.7 samples/s +Step 21390/44000 | Loss: 3.0063 | LR: 2.68e-04 | 1363.1ms/step | 14.7 samples/s +Step 21400/44000 | Loss: 2.9675 | LR: 2.68e-04 | 1365.0ms/step | 14.7 samples/s +Step 21410/44000 | Loss: 3.0082 | LR: 2.68e-04 | 1363.8ms/step | 14.7 samples/s +Step 21420/44000 | Loss: 2.9573 | LR: 2.67e-04 | 1360.1ms/step | 14.7 samples/s +Step 21430/44000 | Loss: 2.9735 | LR: 2.67e-04 | 1358.8ms/step | 14.7 samples/s +Step 21440/44000 | Loss: 2.9645 | LR: 2.67e-04 | 1357.1ms/step | 14.7 samples/s +Step 21450/44000 | Loss: 2.9820 | LR: 2.67e-04 | 1360.3ms/step | 14.7 samples/s +Step 21460/44000 | Loss: 2.9897 | LR: 2.67e-04 | 1358.6ms/step | 14.7 samples/s +Step 21470/44000 | Loss: 2.9931 | LR: 2.66e-04 | 1360.3ms/step | 14.7 samples/s +Step 21480/44000 | Loss: 2.9751 | LR: 2.66e-04 | 1360.4ms/step | 14.7 samples/s +Step 21490/44000 | Loss: 2.9891 | LR: 2.66e-04 | 1356.6ms/step | 14.7 samples/s +Step 21500/44000 | Loss: 2.9896 | LR: 2.66e-04 | 1359.5ms/step | 14.7 samples/s +Validation at step 21500: Loss = 2.9996, Perplexity = 20.08 +Step 21510/44000 | Loss: 2.9925 | LR: 2.66e-04 | 3655.0ms/step | 5.5 samples/s +Step 21520/44000 | Loss: 2.9806 | LR: 2.66e-04 | 1361.7ms/step | 14.7 samples/s +Step 21530/44000 | Loss: 2.9750 | LR: 2.65e-04 | 1365.0ms/step | 14.7 samples/s +Step 21540/44000 | Loss: 2.9859 | LR: 2.65e-04 | 1365.1ms/step | 14.7 samples/s +Step 21550/44000 | Loss: 2.9889 | LR: 2.65e-04 | 1366.0ms/step | 14.6 samples/s +Step 21560/44000 | Loss: 2.9913 | LR: 2.65e-04 | 1363.3ms/step | 14.7 samples/s +Step 21570/44000 | Loss: 2.9689 | LR: 2.65e-04 | 1361.1ms/step | 14.7 samples/s +Step 21580/44000 | Loss: 2.9628 | LR: 2.64e-04 | 1360.1ms/step | 14.7 samples/s +Step 21590/44000 | Loss: 2.9794 | LR: 2.64e-04 | 1362.9ms/step | 14.7 samples/s +Step 21600/44000 | Loss: 2.9803 | LR: 2.64e-04 | 1365.4ms/step | 14.6 samples/s +Step 21610/44000 | Loss: 2.9761 | LR: 2.64e-04 | 1360.4ms/step | 14.7 samples/s +Step 21620/44000 | Loss: 2.9968 | LR: 2.64e-04 | 1360.2ms/step | 14.7 samples/s +Step 21630/44000 | Loss: 2.9672 | LR: 2.64e-04 | 1360.6ms/step | 14.7 samples/s +Step 21640/44000 | Loss: 2.9599 | LR: 2.63e-04 | 1359.9ms/step | 14.7 samples/s +Step 21650/44000 | Loss: 2.9867 | LR: 2.63e-04 | 1362.2ms/step | 14.7 samples/s +Step 21660/44000 | Loss: 3.0222 | LR: 2.63e-04 | 1362.1ms/step | 14.7 samples/s +Step 21670/44000 | Loss: 2.9674 | LR: 2.63e-04 | 1359.4ms/step | 14.7 samples/s +Step 21680/44000 | Loss: 2.9903 | LR: 2.63e-04 | 1363.4ms/step | 14.7 samples/s +Step 21690/44000 | Loss: 3.0177 | LR: 2.62e-04 | 1362.3ms/step | 14.7 samples/s +Step 21700/44000 | Loss: 2.9991 | LR: 2.62e-04 | 1362.2ms/step | 14.7 samples/s +Step 21710/44000 | Loss: 2.9829 | LR: 2.62e-04 | 1361.2ms/step | 14.7 samples/s +Step 21720/44000 | Loss: 2.9552 | LR: 2.62e-04 | 1360.9ms/step | 14.7 samples/s +Step 21730/44000 | Loss: 2.9810 | LR: 2.62e-04 | 1359.2ms/step | 14.7 samples/s +Step 21740/44000 | Loss: 2.9704 | LR: 2.62e-04 | 1360.2ms/step | 14.7 samples/s +Step 21750/44000 | Loss: 2.9726 | LR: 2.61e-04 | 1358.2ms/step | 14.7 samples/s +Validation at step 21750: Loss = 2.9965, Perplexity = 20.01 +Step 21760/44000 | Loss: 2.9958 | LR: 2.61e-04 | 3643.2ms/step | 5.5 samples/s +Step 21770/44000 | Loss: 2.9790 | LR: 2.61e-04 | 1358.0ms/step | 14.7 samples/s +Step 21780/44000 | Loss: 2.9806 | LR: 2.61e-04 | 1359.9ms/step | 14.7 samples/s +Step 21790/44000 | Loss: 2.9467 | LR: 2.61e-04 | 1359.8ms/step | 14.7 samples/s +Step 21800/44000 | Loss: 2.9711 | LR: 2.60e-04 | 1358.7ms/step | 14.7 samples/s +Step 21810/44000 | Loss: 2.9687 | LR: 2.60e-04 | 1360.8ms/step | 14.7 samples/s +Step 21820/44000 | Loss: 2.9890 | LR: 2.60e-04 | 1361.4ms/step | 14.7 samples/s +Step 21830/44000 | Loss: 2.9597 | LR: 2.60e-04 | 1360.5ms/step | 14.7 samples/s +Step 21840/44000 | Loss: 2.9841 | LR: 2.60e-04 | 1363.3ms/step | 14.7 samples/s +Step 21850/44000 | Loss: 2.9788 | LR: 2.60e-04 | 1357.0ms/step | 14.7 samples/s +Step 21860/44000 | Loss: 2.9833 | LR: 2.59e-04 | 1356.6ms/step | 14.7 samples/s +Step 21870/44000 | Loss: 2.9961 | LR: 2.59e-04 | 1357.7ms/step | 14.7 samples/s +Step 21880/44000 | Loss: 2.9651 | LR: 2.59e-04 | 1358.1ms/step | 14.7 samples/s +Step 21890/44000 | Loss: 2.9673 | LR: 2.59e-04 | 1358.2ms/step | 14.7 samples/s +Step 21900/44000 | Loss: 2.9747 | LR: 2.59e-04 | 1354.9ms/step | 14.8 samples/s +Step 21910/44000 | Loss: 2.9759 | LR: 2.58e-04 | 1355.2ms/step | 14.8 samples/s +Step 21920/44000 | Loss: 2.9823 | LR: 2.58e-04 | 1356.5ms/step | 14.7 samples/s +Step 21930/44000 | Loss: 3.0084 | LR: 2.58e-04 | 1357.5ms/step | 14.7 samples/s +Step 21940/44000 | Loss: 2.9504 | LR: 2.58e-04 | 1362.0ms/step | 14.7 samples/s +Step 21950/44000 | Loss: 2.9815 | LR: 2.58e-04 | 1359.7ms/step | 14.7 samples/s +Step 21960/44000 | Loss: 2.9704 | LR: 2.58e-04 | 1359.1ms/step | 14.7 samples/s +Step 21970/44000 | Loss: 2.9670 | LR: 2.57e-04 | 1353.6ms/step | 14.8 samples/s +Step 21980/44000 | Loss: 2.9861 | LR: 2.57e-04 | 1356.4ms/step | 14.7 samples/s +Step 21990/44000 | Loss: 2.9901 | LR: 2.57e-04 | 1358.1ms/step | 14.7 samples/s +Step 22000/44000 | Loss: 2.9792 | LR: 2.57e-04 | 1359.0ms/step | 14.7 samples/s +Validation at step 22000: Loss = 2.9941, Perplexity = 19.97 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-22000 +Step 22010/44000 | Loss: 2.9807 | LR: 2.57e-04 | 3916.6ms/step | 5.1 samples/s +Step 22020/44000 | Loss: 2.9922 | LR: 2.56e-04 | 1357.9ms/step | 14.7 samples/s +Step 22030/44000 | Loss: 2.9884 | LR: 2.56e-04 | 1354.8ms/step | 14.8 samples/s +Step 22040/44000 | Loss: 3.0029 | LR: 2.56e-04 | 1352.5ms/step | 14.8 samples/s +Step 22050/44000 | Loss: 2.9733 | LR: 2.56e-04 | 1351.0ms/step | 14.8 samples/s +Step 22060/44000 | Loss: 2.9732 | LR: 2.56e-04 | 1351.3ms/step | 14.8 samples/s +Step 22070/44000 | Loss: 2.9784 | LR: 2.56e-04 | 1353.5ms/step | 14.8 samples/s +Step 22080/44000 | Loss: 2.9691 | LR: 2.55e-04 | 1354.6ms/step | 14.8 samples/s +Step 22090/44000 | Loss: 2.9956 | LR: 2.55e-04 | 1351.6ms/step | 14.8 samples/s +Step 22100/44000 | Loss: 2.9897 | LR: 2.55e-04 | 1350.4ms/step | 14.8 samples/s +Step 22110/44000 | Loss: 2.9750 | LR: 2.55e-04 | 1351.0ms/step | 14.8 samples/s +Step 22120/44000 | Loss: 2.9571 | LR: 2.55e-04 | 1352.1ms/step | 14.8 samples/s +Step 22130/44000 | Loss: 2.9664 | LR: 2.54e-04 | 1351.7ms/step | 14.8 samples/s +Step 22140/44000 | Loss: 2.9774 | LR: 2.54e-04 | 1355.4ms/step | 14.8 samples/s +Step 22150/44000 | Loss: 2.9555 | LR: 2.54e-04 | 1356.0ms/step | 14.7 samples/s +Step 22160/44000 | Loss: 2.9768 | LR: 2.54e-04 | 1356.8ms/step | 14.7 samples/s +Step 22170/44000 | Loss: 2.9964 | LR: 2.54e-04 | 1354.6ms/step | 14.8 samples/s +Step 22180/44000 | Loss: 2.9792 | LR: 2.54e-04 | 1352.5ms/step | 14.8 samples/s +Step 22190/44000 | Loss: 2.9775 | LR: 2.53e-04 | 1354.9ms/step | 14.8 samples/s +Step 22200/44000 | Loss: 2.9748 | LR: 2.53e-04 | 1353.7ms/step | 14.8 samples/s +Step 22210/44000 | Loss: 2.9792 | LR: 2.53e-04 | 1348.8ms/step | 14.8 samples/s +Step 22220/44000 | Loss: 2.9764 | LR: 2.53e-04 | 1355.6ms/step | 14.8 samples/s +Step 22230/44000 | Loss: 2.9920 | LR: 2.53e-04 | 1353.7ms/step | 14.8 samples/s +Step 22240/44000 | Loss: 3.0049 | LR: 2.52e-04 | 1353.2ms/step | 14.8 samples/s +Step 22250/44000 | Loss: 2.9925 | LR: 2.52e-04 | 1355.2ms/step | 14.8 samples/s +Validation at step 22250: Loss = 2.9924, Perplexity = 19.93 +Step 22260/44000 | Loss: 2.9600 | LR: 2.52e-04 | 3642.1ms/step | 5.5 samples/s +Step 22270/44000 | Loss: 2.9947 | LR: 2.52e-04 | 1359.4ms/step | 14.7 samples/s +Step 22280/44000 | Loss: 2.9736 | LR: 2.52e-04 | 1356.7ms/step | 14.7 samples/s +Step 22290/44000 | Loss: 2.9731 | LR: 2.52e-04 | 1357.4ms/step | 14.7 samples/s +Step 22300/44000 | Loss: 3.0090 | LR: 2.51e-04 | 1355.4ms/step | 14.8 samples/s +Step 22310/44000 | Loss: 2.9701 | LR: 2.51e-04 | 1356.4ms/step | 14.7 samples/s +Step 22320/44000 | Loss: 2.9554 | LR: 2.51e-04 | 1352.5ms/step | 14.8 samples/s +Step 22330/44000 | Loss: 2.9556 | LR: 2.51e-04 | 1353.5ms/step | 14.8 samples/s +Step 22340/44000 | Loss: 3.0055 | LR: 2.51e-04 | 1354.5ms/step | 14.8 samples/s +Step 22350/44000 | Loss: 2.9690 | LR: 2.50e-04 | 1354.9ms/step | 14.8 samples/s +Step 22360/44000 | Loss: 2.9686 | LR: 2.50e-04 | 1353.1ms/step | 14.8 samples/s +Step 22370/44000 | Loss: 2.9737 | LR: 2.50e-04 | 1356.9ms/step | 14.7 samples/s +Step 22380/44000 | Loss: 2.9771 | LR: 2.50e-04 | 1357.4ms/step | 14.7 samples/s +Step 22390/44000 | Loss: 2.9632 | LR: 2.50e-04 | 1357.4ms/step | 14.7 samples/s +Step 22400/44000 | Loss: 2.9488 | LR: 2.50e-04 | 1355.2ms/step | 14.8 samples/s +Step 22410/44000 | Loss: 2.9666 | LR: 2.49e-04 | 1353.4ms/step | 14.8 samples/s +Step 22420/44000 | Loss: 2.9691 | LR: 2.49e-04 | 1354.6ms/step | 14.8 samples/s +Step 22430/44000 | Loss: 2.9840 | LR: 2.49e-04 | 1356.2ms/step | 14.7 samples/s +Step 22440/44000 | Loss: 2.9711 | LR: 2.49e-04 | 1349.1ms/step | 14.8 samples/s +Step 22450/44000 | Loss: 2.9899 | LR: 2.49e-04 | 1354.6ms/step | 14.8 samples/s +Step 22460/44000 | Loss: 2.9682 | LR: 2.48e-04 | 1354.6ms/step | 14.8 samples/s +Step 22470/44000 | Loss: 2.9421 | LR: 2.48e-04 | 1354.4ms/step | 14.8 samples/s +Step 22480/44000 | Loss: 2.9568 | LR: 2.48e-04 | 1352.2ms/step | 14.8 samples/s +Step 22490/44000 | Loss: 2.9634 | LR: 2.48e-04 | 1360.0ms/step | 14.7 samples/s +Step 22500/44000 | Loss: 2.9480 | LR: 2.48e-04 | 1375.8ms/step | 14.5 samples/s +Validation at step 22500: Loss = 2.9895, Perplexity = 19.88 +Step 22510/44000 | Loss: 2.9577 | LR: 2.48e-04 | 3673.9ms/step | 5.4 samples/s +Step 22520/44000 | Loss: 3.0034 | LR: 2.47e-04 | 1364.4ms/step | 14.7 samples/s +Step 22530/44000 | Loss: 2.9414 | LR: 2.47e-04 | 1366.0ms/step | 14.6 samples/s +Step 22540/44000 | Loss: 2.9767 | LR: 2.47e-04 | 1364.4ms/step | 14.7 samples/s +Step 22550/44000 | Loss: 2.9421 | LR: 2.47e-04 | 1366.3ms/step | 14.6 samples/s +Step 22560/44000 | Loss: 2.9714 | LR: 2.47e-04 | 1364.4ms/step | 14.7 samples/s +Step 22570/44000 | Loss: 2.9462 | LR: 2.46e-04 | 1362.6ms/step | 14.7 samples/s +Step 22580/44000 | Loss: 2.9854 | LR: 2.46e-04 | 1365.1ms/step | 14.7 samples/s +Step 22590/44000 | Loss: 2.9797 | LR: 2.46e-04 | 1360.1ms/step | 14.7 samples/s +Step 22600/44000 | Loss: 2.9525 | LR: 2.46e-04 | 1356.1ms/step | 14.7 samples/s +Step 22610/44000 | Loss: 2.9699 | LR: 2.46e-04 | 1360.4ms/step | 14.7 samples/s +Step 22620/44000 | Loss: 2.9867 | LR: 2.46e-04 | 1361.1ms/step | 14.7 samples/s +Step 22630/44000 | Loss: 2.9668 | LR: 2.45e-04 | 1361.6ms/step | 14.7 samples/s +Step 22640/44000 | Loss: 2.9803 | LR: 2.45e-04 | 1361.2ms/step | 14.7 samples/s +Step 22650/44000 | Loss: 2.9687 | LR: 2.45e-04 | 1358.7ms/step | 14.7 samples/s +Step 22660/44000 | Loss: 3.0022 | LR: 2.45e-04 | 1353.9ms/step | 14.8 samples/s +Step 22670/44000 | Loss: 2.9772 | LR: 2.45e-04 | 1356.8ms/step | 14.7 samples/s +Step 22680/44000 | Loss: 2.9547 | LR: 2.44e-04 | 1361.8ms/step | 14.7 samples/s +Step 22690/44000 | Loss: 2.9850 | LR: 2.44e-04 | 1362.2ms/step | 14.7 samples/s +Step 22700/44000 | Loss: 2.9871 | LR: 2.44e-04 | 1361.7ms/step | 14.7 samples/s +Step 22710/44000 | Loss: 2.9442 | LR: 2.44e-04 | 1360.4ms/step | 14.7 samples/s +Step 22720/44000 | Loss: 2.9722 | LR: 2.44e-04 | 1361.7ms/step | 14.7 samples/s +Step 22730/44000 | Loss: 2.9890 | LR: 2.44e-04 | 1363.8ms/step | 14.7 samples/s +Step 22740/44000 | Loss: 2.9742 | LR: 2.43e-04 | 1360.9ms/step | 14.7 samples/s +Step 22750/44000 | Loss: 2.9494 | LR: 2.43e-04 | 1361.8ms/step | 14.7 samples/s +Validation at step 22750: Loss = 2.9869, Perplexity = 19.82 +Step 22760/44000 | Loss: 2.9847 | LR: 2.43e-04 | 3638.7ms/step | 5.5 samples/s +Step 22770/44000 | Loss: 2.9673 | LR: 2.43e-04 | 1359.1ms/step | 14.7 samples/s +Step 22780/44000 | Loss: 2.9778 | LR: 2.43e-04 | 1356.7ms/step | 14.7 samples/s +Step 22790/44000 | Loss: 2.9911 | LR: 2.42e-04 | 1354.8ms/step | 14.8 samples/s +Step 22800/44000 | Loss: 2.9929 | LR: 2.42e-04 | 1357.2ms/step | 14.7 samples/s +Step 22810/44000 | Loss: 2.9589 | LR: 2.42e-04 | 1357.9ms/step | 14.7 samples/s +Step 22820/44000 | Loss: 2.9714 | LR: 2.42e-04 | 1355.9ms/step | 14.8 samples/s +Step 22830/44000 | Loss: 2.9987 | LR: 2.42e-04 | 1354.8ms/step | 14.8 samples/s +Step 22840/44000 | Loss: 2.9827 | LR: 2.42e-04 | 1358.0ms/step | 14.7 samples/s +Step 22850/44000 | Loss: 2.9538 | LR: 2.41e-04 | 1354.6ms/step | 14.8 samples/s +Step 22860/44000 | Loss: 2.9938 | LR: 2.41e-04 | 1348.0ms/step | 14.8 samples/s +Step 22870/44000 | Loss: 2.9654 | LR: 2.41e-04 | 1353.4ms/step | 14.8 samples/s +Step 22880/44000 | Loss: 2.9667 | LR: 2.41e-04 | 1352.2ms/step | 14.8 samples/s +Step 22890/44000 | Loss: 2.9552 | LR: 2.41e-04 | 1353.7ms/step | 14.8 samples/s +Step 22900/44000 | Loss: 3.0001 | LR: 2.40e-04 | 1351.3ms/step | 14.8 samples/s +Step 22910/44000 | Loss: 2.9849 | LR: 2.40e-04 | 1352.0ms/step | 14.8 samples/s +Step 22920/44000 | Loss: 2.9734 | LR: 2.40e-04 | 1352.3ms/step | 14.8 samples/s +Step 22930/44000 | Loss: 2.9547 | LR: 2.40e-04 | 1346.0ms/step | 14.9 samples/s +Step 22940/44000 | Loss: 2.9579 | LR: 2.40e-04 | 1352.4ms/step | 14.8 samples/s +Step 22950/44000 | Loss: 2.9659 | LR: 2.40e-04 | 1355.7ms/step | 14.8 samples/s +Step 22960/44000 | Loss: 3.0048 | LR: 2.39e-04 | 1351.9ms/step | 14.8 samples/s +Step 22970/44000 | Loss: 2.9550 | LR: 2.39e-04 | 1350.1ms/step | 14.8 samples/s +Step 22980/44000 | Loss: 2.9698 | LR: 2.39e-04 | 1352.9ms/step | 14.8 samples/s +Step 22990/44000 | Loss: 2.9594 | LR: 2.39e-04 | 1355.9ms/step | 14.8 samples/s +Step 23000/44000 | Loss: 2.9546 | LR: 2.39e-04 | 1355.3ms/step | 14.8 samples/s +Validation at step 23000: Loss = 2.9846, Perplexity = 19.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-23000 +Step 23010/44000 | Loss: 2.9592 | LR: 2.38e-04 | 3920.1ms/step | 5.1 samples/s +Step 23020/44000 | Loss: 2.9707 | LR: 2.38e-04 | 1351.0ms/step | 14.8 samples/s +Step 23030/44000 | Loss: 2.9741 | LR: 2.38e-04 | 1353.1ms/step | 14.8 samples/s +Step 23040/44000 | Loss: 2.9881 | LR: 2.38e-04 | 1351.9ms/step | 14.8 samples/s +Step 23050/44000 | Loss: 3.0094 | LR: 2.38e-04 | 1353.8ms/step | 14.8 samples/s +Step 23060/44000 | Loss: 2.9722 | LR: 2.38e-04 | 1357.1ms/step | 14.7 samples/s +Step 23070/44000 | Loss: 2.9674 | LR: 2.37e-04 | 1354.1ms/step | 14.8 samples/s +Step 23080/44000 | Loss: 2.9705 | LR: 2.37e-04 | 1357.1ms/step | 14.7 samples/s +Step 23090/44000 | Loss: 2.9807 | LR: 2.37e-04 | 1352.0ms/step | 14.8 samples/s +Step 23100/44000 | Loss: 2.9678 | LR: 2.37e-04 | 1350.7ms/step | 14.8 samples/s +Step 23110/44000 | Loss: 2.9548 | LR: 2.37e-04 | 1354.2ms/step | 14.8 samples/s +Step 23120/44000 | Loss: 2.9632 | LR: 2.36e-04 | 1355.2ms/step | 14.8 samples/s +Step 23130/44000 | Loss: 2.9632 | LR: 2.36e-04 | 1352.5ms/step | 14.8 samples/s +Step 23140/44000 | Loss: 2.9448 | LR: 2.36e-04 | 1349.3ms/step | 14.8 samples/s +Step 23150/44000 | Loss: 2.9512 | LR: 2.36e-04 | 1350.8ms/step | 14.8 samples/s +Step 23160/44000 | Loss: 2.9755 | LR: 2.36e-04 | 1352.6ms/step | 14.8 samples/s +Step 23170/44000 | Loss: 2.9607 | LR: 2.36e-04 | 1357.7ms/step | 14.7 samples/s +Step 23180/44000 | Loss: 2.9762 | LR: 2.35e-04 | 1358.1ms/step | 14.7 samples/s +Step 23190/44000 | Loss: 2.9708 | LR: 2.35e-04 | 1357.2ms/step | 14.7 samples/s +Step 23200/44000 | Loss: 2.9687 | LR: 2.35e-04 | 1355.2ms/step | 14.8 samples/s +Step 23210/44000 | Loss: 2.9542 | LR: 2.35e-04 | 1350.9ms/step | 14.8 samples/s +Step 23220/44000 | Loss: 2.9740 | LR: 2.35e-04 | 1349.3ms/step | 14.8 samples/s +Step 23230/44000 | Loss: 2.9506 | LR: 2.34e-04 | 1349.0ms/step | 14.8 samples/s +Step 23240/44000 | Loss: 2.9725 | LR: 2.34e-04 | 1352.9ms/step | 14.8 samples/s +Step 23250/44000 | Loss: 2.9741 | LR: 2.34e-04 | 1350.2ms/step | 14.8 samples/s +Validation at step 23250: Loss = 2.9832, Perplexity = 19.75 +Step 23260/44000 | Loss: 2.9456 | LR: 2.34e-04 | 3623.8ms/step | 5.5 samples/s +Step 23270/44000 | Loss: 2.9678 | LR: 2.34e-04 | 1354.7ms/step | 14.8 samples/s +Step 23280/44000 | Loss: 2.9820 | LR: 2.34e-04 | 1350.0ms/step | 14.8 samples/s +Step 23290/44000 | Loss: 2.9618 | LR: 2.33e-04 | 1349.9ms/step | 14.8 samples/s +Step 23300/44000 | Loss: 2.9533 | LR: 2.33e-04 | 1352.7ms/step | 14.8 samples/s +Step 23310/44000 | Loss: 2.9336 | LR: 2.33e-04 | 1350.5ms/step | 14.8 samples/s +Step 23320/44000 | Loss: 2.9597 | LR: 2.33e-04 | 1351.6ms/step | 14.8 samples/s +Step 23330/44000 | Loss: 2.9504 | LR: 2.33e-04 | 1348.9ms/step | 14.8 samples/s +Step 23340/44000 | Loss: 2.9772 | LR: 2.32e-04 | 1351.5ms/step | 14.8 samples/s +Step 23350/44000 | Loss: 2.9608 | LR: 2.32e-04 | 1353.8ms/step | 14.8 samples/s +Step 23360/44000 | Loss: 3.0042 | LR: 2.32e-04 | 1353.9ms/step | 14.8 samples/s +Step 23370/44000 | Loss: 2.9549 | LR: 2.32e-04 | 1353.6ms/step | 14.8 samples/s +Step 23380/44000 | Loss: 2.9493 | LR: 2.32e-04 | 1354.5ms/step | 14.8 samples/s +Step 23390/44000 | Loss: 2.9361 | LR: 2.32e-04 | 1352.2ms/step | 14.8 samples/s +Step 23400/44000 | Loss: 2.9729 | LR: 2.31e-04 | 1354.5ms/step | 14.8 samples/s +Step 23410/44000 | Loss: 2.9630 | LR: 2.31e-04 | 1355.0ms/step | 14.8 samples/s +Step 23420/44000 | Loss: 2.9835 | LR: 2.31e-04 | 1355.6ms/step | 14.8 samples/s +Step 23430/44000 | Loss: 2.9524 | LR: 2.31e-04 | 1356.0ms/step | 14.7 samples/s +Step 23440/44000 | Loss: 2.9519 | LR: 2.31e-04 | 1354.8ms/step | 14.8 samples/s +Step 23450/44000 | Loss: 2.9700 | LR: 2.30e-04 | 1355.4ms/step | 14.8 samples/s +Step 23460/44000 | Loss: 2.9539 | LR: 2.30e-04 | 1356.5ms/step | 14.7 samples/s +Step 23470/44000 | Loss: 2.9756 | LR: 2.30e-04 | 1355.2ms/step | 14.8 samples/s +Step 23480/44000 | Loss: 2.9521 | LR: 2.30e-04 | 1352.8ms/step | 14.8 samples/s +Step 23490/44000 | Loss: 2.9682 | LR: 2.30e-04 | 1352.5ms/step | 14.8 samples/s +Step 23500/44000 | Loss: 2.9525 | LR: 2.30e-04 | 1356.8ms/step | 14.7 samples/s +Validation at step 23500: Loss = 2.9805, Perplexity = 19.70 +Step 23510/44000 | Loss: 2.9694 | LR: 2.29e-04 | 3637.9ms/step | 5.5 samples/s +Step 23520/44000 | Loss: 2.9595 | LR: 2.29e-04 | 1360.1ms/step | 14.7 samples/s +Step 23530/44000 | Loss: 2.9594 | LR: 2.29e-04 | 1361.1ms/step | 14.7 samples/s +Step 23540/44000 | Loss: 2.9424 | LR: 2.29e-04 | 1361.0ms/step | 14.7 samples/s +Step 23550/44000 | Loss: 2.9413 | LR: 2.29e-04 | 1358.7ms/step | 14.7 samples/s +Step 23560/44000 | Loss: 2.9600 | LR: 2.29e-04 | 1358.0ms/step | 14.7 samples/s +Step 23570/44000 | Loss: 2.9863 | LR: 2.28e-04 | 1356.0ms/step | 14.7 samples/s +Step 23580/44000 | Loss: 2.9366 | LR: 2.28e-04 | 1352.4ms/step | 14.8 samples/s +Step 23590/44000 | Loss: 2.9784 | LR: 2.28e-04 | 1350.1ms/step | 14.8 samples/s +Step 23600/44000 | Loss: 2.9919 | LR: 2.28e-04 | 1354.3ms/step | 14.8 samples/s +Step 23610/44000 | Loss: 2.9531 | LR: 2.28e-04 | 1353.7ms/step | 14.8 samples/s +Step 23620/44000 | Loss: 2.9591 | LR: 2.27e-04 | 1354.9ms/step | 14.8 samples/s +Step 23630/44000 | Loss: 2.9625 | LR: 2.27e-04 | 1354.9ms/step | 14.8 samples/s +Step 23640/44000 | Loss: 2.9783 | LR: 2.27e-04 | 1360.3ms/step | 14.7 samples/s +Step 23650/44000 | Loss: 2.9879 | LR: 2.27e-04 | 1359.6ms/step | 14.7 samples/s +Step 23660/44000 | Loss: 2.9644 | LR: 2.27e-04 | 1352.2ms/step | 14.8 samples/s +Step 23670/44000 | Loss: 2.9744 | LR: 2.27e-04 | 1350.5ms/step | 14.8 samples/s +Step 23680/44000 | Loss: 2.9721 | LR: 2.26e-04 | 1353.5ms/step | 14.8 samples/s +Step 23690/44000 | Loss: 2.9754 | LR: 2.26e-04 | 1356.9ms/step | 14.7 samples/s +Step 23700/44000 | Loss: 2.9595 | LR: 2.26e-04 | 1351.8ms/step | 14.8 samples/s +Step 23710/44000 | Loss: 2.9436 | LR: 2.26e-04 | 1351.5ms/step | 14.8 samples/s +Step 23720/44000 | Loss: 2.9489 | LR: 2.26e-04 | 1358.3ms/step | 14.7 samples/s +Step 23730/44000 | Loss: 2.9678 | LR: 2.25e-04 | 1354.6ms/step | 14.8 samples/s +Step 23740/44000 | Loss: 2.9660 | LR: 2.25e-04 | 1353.0ms/step | 14.8 samples/s +Step 23750/44000 | Loss: 2.9502 | LR: 2.25e-04 | 1352.9ms/step | 14.8 samples/s +Validation at step 23750: Loss = 2.9780, Perplexity = 19.65 +Step 23760/44000 | Loss: 2.9761 | LR: 2.25e-04 | 3628.2ms/step | 5.5 samples/s +Step 23770/44000 | Loss: 2.9513 | LR: 2.25e-04 | 1359.7ms/step | 14.7 samples/s +Step 23780/44000 | Loss: 2.9479 | LR: 2.25e-04 | 1359.5ms/step | 14.7 samples/s +Step 23790/44000 | Loss: 2.9435 | LR: 2.24e-04 | 1361.3ms/step | 14.7 samples/s +Step 23800/44000 | Loss: 2.9671 | LR: 2.24e-04 | 1359.9ms/step | 14.7 samples/s +Step 23810/44000 | Loss: 2.9430 | LR: 2.24e-04 | 1356.8ms/step | 14.7 samples/s +Step 23820/44000 | Loss: 2.9870 | LR: 2.24e-04 | 1351.2ms/step | 14.8 samples/s +Step 23830/44000 | Loss: 2.9552 | LR: 2.24e-04 | 1350.7ms/step | 14.8 samples/s +Step 23840/44000 | Loss: 2.9680 | LR: 2.23e-04 | 1353.4ms/step | 14.8 samples/s +Step 23850/44000 | Loss: 2.9716 | LR: 2.23e-04 | 1348.6ms/step | 14.8 samples/s +Step 23860/44000 | Loss: 2.9598 | LR: 2.23e-04 | 1348.6ms/step | 14.8 samples/s +Step 23870/44000 | Loss: 2.9734 | LR: 2.23e-04 | 1355.0ms/step | 14.8 samples/s +Step 23880/44000 | Loss: 2.9573 | LR: 2.23e-04 | 1353.4ms/step | 14.8 samples/s +Step 23890/44000 | Loss: 2.9562 | LR: 2.23e-04 | 1349.4ms/step | 14.8 samples/s +Step 23900/44000 | Loss: 2.9542 | LR: 2.22e-04 | 1351.2ms/step | 14.8 samples/s +Step 23910/44000 | Loss: 2.9495 | LR: 2.22e-04 | 1353.1ms/step | 14.8 samples/s +Step 23920/44000 | Loss: 2.9931 | LR: 2.22e-04 | 1352.0ms/step | 14.8 samples/s +Step 23930/44000 | Loss: 2.9669 | LR: 2.22e-04 | 1353.1ms/step | 14.8 samples/s +Step 23940/44000 | Loss: 2.9784 | LR: 2.22e-04 | 1352.0ms/step | 14.8 samples/s +Step 23950/44000 | Loss: 2.9728 | LR: 2.21e-04 | 1356.2ms/step | 14.7 samples/s +Step 23960/44000 | Loss: 2.9955 | LR: 2.21e-04 | 1352.9ms/step | 14.8 samples/s +Step 23970/44000 | Loss: 2.9751 | LR: 2.21e-04 | 1350.8ms/step | 14.8 samples/s +Step 23980/44000 | Loss: 2.9554 | LR: 2.21e-04 | 1353.5ms/step | 14.8 samples/s +Step 23990/44000 | Loss: 2.9330 | LR: 2.21e-04 | 1350.2ms/step | 14.8 samples/s +Step 24000/44000 | Loss: 2.9541 | LR: 2.21e-04 | 1348.5ms/step | 14.8 samples/s +Validation at step 24000: Loss = 2.9764, Perplexity = 19.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-24000 +Step 24010/44000 | Loss: 2.9659 | LR: 2.20e-04 | 3902.2ms/step | 5.1 samples/s +Step 24020/44000 | Loss: 2.9608 | LR: 2.20e-04 | 1351.8ms/step | 14.8 samples/s +Step 24030/44000 | Loss: 2.9837 | LR: 2.20e-04 | 1354.5ms/step | 14.8 samples/s +Step 24040/44000 | Loss: 2.9965 | LR: 2.20e-04 | 1351.1ms/step | 14.8 samples/s +Step 24050/44000 | Loss: 2.9861 | LR: 2.20e-04 | 1351.7ms/step | 14.8 samples/s +Step 24060/44000 | Loss: 2.9712 | LR: 2.19e-04 | 1350.7ms/step | 14.8 samples/s +Step 24070/44000 | Loss: 2.9865 | LR: 2.19e-04 | 1350.5ms/step | 14.8 samples/s +Step 24080/44000 | Loss: 2.9461 | LR: 2.19e-04 | 1354.4ms/step | 14.8 samples/s +Step 24090/44000 | Loss: 2.9459 | LR: 2.19e-04 | 1354.2ms/step | 14.8 samples/s +Step 24100/44000 | Loss: 2.9638 | LR: 2.19e-04 | 1354.3ms/step | 14.8 samples/s +Step 24110/44000 | Loss: 2.9745 | LR: 2.19e-04 | 1354.2ms/step | 14.8 samples/s +Step 24120/44000 | Loss: 2.9288 | LR: 2.18e-04 | 1352.9ms/step | 14.8 samples/s +Step 24130/44000 | Loss: 2.9763 | LR: 2.18e-04 | 1350.9ms/step | 14.8 samples/s +Step 24140/44000 | Loss: 2.9537 | LR: 2.18e-04 | 1353.6ms/step | 14.8 samples/s +Step 24150/44000 | Loss: 2.9801 | LR: 2.18e-04 | 1352.6ms/step | 14.8 samples/s +Step 24160/44000 | Loss: 2.9901 | LR: 2.18e-04 | 1352.0ms/step | 14.8 samples/s +Step 24170/44000 | Loss: 2.9455 | LR: 2.17e-04 | 1350.5ms/step | 14.8 samples/s +Step 24180/44000 | Loss: 2.9541 | LR: 2.17e-04 | 1351.0ms/step | 14.8 samples/s +Step 24190/44000 | Loss: 2.9756 | LR: 2.17e-04 | 1351.4ms/step | 14.8 samples/s +Step 24200/44000 | Loss: 2.9481 | LR: 2.17e-04 | 1353.0ms/step | 14.8 samples/s +Step 24210/44000 | Loss: 2.9410 | LR: 2.17e-04 | 1351.6ms/step | 14.8 samples/s +Step 24220/44000 | Loss: 2.9594 | LR: 2.17e-04 | 1353.0ms/step | 14.8 samples/s +Step 24230/44000 | Loss: 2.9819 | LR: 2.16e-04 | 1352.0ms/step | 14.8 samples/s +Step 24240/44000 | Loss: 2.9423 | LR: 2.16e-04 | 1350.4ms/step | 14.8 samples/s +Step 24250/44000 | Loss: 2.9675 | LR: 2.16e-04 | 1351.4ms/step | 14.8 samples/s +Validation at step 24250: Loss = 2.9746, Perplexity = 19.58 +Step 24260/44000 | Loss: 2.9379 | LR: 2.16e-04 | 3619.9ms/step | 5.5 samples/s +Step 24270/44000 | Loss: 2.9584 | LR: 2.16e-04 | 1348.7ms/step | 14.8 samples/s +Step 24280/44000 | Loss: 2.9826 | LR: 2.16e-04 | 1345.5ms/step | 14.9 samples/s +Step 24290/44000 | Loss: 2.9654 | LR: 2.15e-04 | 1343.2ms/step | 14.9 samples/s +Step 24300/44000 | Loss: 2.9487 | LR: 2.15e-04 | 1350.3ms/step | 14.8 samples/s +Step 24310/44000 | Loss: 2.9623 | LR: 2.15e-04 | 1350.5ms/step | 14.8 samples/s +Step 24320/44000 | Loss: 2.9612 | LR: 2.15e-04 | 1348.5ms/step | 14.8 samples/s +Step 24330/44000 | Loss: 2.9798 | LR: 2.15e-04 | 1349.3ms/step | 14.8 samples/s +Step 24340/44000 | Loss: 2.9437 | LR: 2.14e-04 | 1348.8ms/step | 14.8 samples/s +Step 24350/44000 | Loss: 2.9351 | LR: 2.14e-04 | 1348.9ms/step | 14.8 samples/s +Step 24360/44000 | Loss: 2.9614 | LR: 2.14e-04 | 1348.3ms/step | 14.8 samples/s +Step 24370/44000 | Loss: 2.9292 | LR: 2.14e-04 | 1348.4ms/step | 14.8 samples/s +Step 24380/44000 | Loss: 2.9233 | LR: 2.14e-04 | 1348.0ms/step | 14.8 samples/s +Step 24390/44000 | Loss: 2.9313 | LR: 2.14e-04 | 1347.8ms/step | 14.8 samples/s +Step 24400/44000 | Loss: 2.9621 | LR: 2.13e-04 | 1351.3ms/step | 14.8 samples/s +Step 24410/44000 | Loss: 2.9605 | LR: 2.13e-04 | 1351.7ms/step | 14.8 samples/s +Step 24420/44000 | Loss: 2.9737 | LR: 2.13e-04 | 1351.4ms/step | 14.8 samples/s +Step 24430/44000 | Loss: 2.9584 | LR: 2.13e-04 | 1348.6ms/step | 14.8 samples/s +Step 24440/44000 | Loss: 2.9655 | LR: 2.13e-04 | 1346.9ms/step | 14.8 samples/s +Step 24450/44000 | Loss: 2.9318 | LR: 2.12e-04 | 1347.5ms/step | 14.8 samples/s +Step 24460/44000 | Loss: 2.9380 | LR: 2.12e-04 | 1350.2ms/step | 14.8 samples/s +Step 24470/44000 | Loss: 2.9643 | LR: 2.12e-04 | 1352.9ms/step | 14.8 samples/s +Step 24480/44000 | Loss: 2.9598 | LR: 2.12e-04 | 1351.6ms/step | 14.8 samples/s +Step 24490/44000 | Loss: 2.9562 | LR: 2.12e-04 | 1351.1ms/step | 14.8 samples/s +Step 24500/44000 | Loss: 2.9707 | LR: 2.12e-04 | 1348.9ms/step | 14.8 samples/s +Validation at step 24500: Loss = 2.9726, Perplexity = 19.54 +Step 24510/44000 | Loss: 2.9604 | LR: 2.11e-04 | 3618.2ms/step | 5.5 samples/s +Step 24520/44000 | Loss: 2.9289 | LR: 2.11e-04 | 1351.4ms/step | 14.8 samples/s +Step 24530/44000 | Loss: 2.9550 | LR: 2.11e-04 | 1350.6ms/step | 14.8 samples/s +Step 24540/44000 | Loss: 2.9752 | LR: 2.11e-04 | 1350.3ms/step | 14.8 samples/s +Step 24550/44000 | Loss: 2.9880 | LR: 2.11e-04 | 1352.6ms/step | 14.8 samples/s +Step 24560/44000 | Loss: 2.9535 | LR: 2.10e-04 | 1354.2ms/step | 14.8 samples/s +Step 24570/44000 | Loss: 3.0226 | LR: 2.10e-04 | 1352.6ms/step | 14.8 samples/s +Step 24580/44000 | Loss: 2.9369 | LR: 2.10e-04 | 1351.2ms/step | 14.8 samples/s +Step 24590/44000 | Loss: 2.9853 | LR: 2.10e-04 | 1354.0ms/step | 14.8 samples/s +Step 24600/44000 | Loss: 2.9557 | LR: 2.10e-04 | 1354.9ms/step | 14.8 samples/s +Step 24610/44000 | Loss: 2.9723 | LR: 2.10e-04 | 1351.8ms/step | 14.8 samples/s +Step 24620/44000 | Loss: 2.9666 | LR: 2.09e-04 | 1353.6ms/step | 14.8 samples/s +Step 24630/44000 | Loss: 2.9864 | LR: 2.09e-04 | 1353.2ms/step | 14.8 samples/s +Step 24640/44000 | Loss: 2.9576 | LR: 2.09e-04 | 1354.0ms/step | 14.8 samples/s +Step 24650/44000 | Loss: 2.9582 | LR: 2.09e-04 | 1356.1ms/step | 14.7 samples/s +Step 24660/44000 | Loss: 2.9499 | LR: 2.09e-04 | 1354.5ms/step | 14.8 samples/s +Step 24670/44000 | Loss: 2.9529 | LR: 2.09e-04 | 1352.4ms/step | 14.8 samples/s +Step 24680/44000 | Loss: 2.9478 | LR: 2.08e-04 | 1351.8ms/step | 14.8 samples/s +Step 24690/44000 | Loss: 2.9544 | LR: 2.08e-04 | 1352.2ms/step | 14.8 samples/s +Step 24700/44000 | Loss: 2.9498 | LR: 2.08e-04 | 1353.4ms/step | 14.8 samples/s +Step 24710/44000 | Loss: 2.9515 | LR: 2.08e-04 | 1355.0ms/step | 14.8 samples/s +Step 24720/44000 | Loss: 2.9395 | LR: 2.08e-04 | 1354.0ms/step | 14.8 samples/s +Step 24730/44000 | Loss: 2.9519 | LR: 2.07e-04 | 1352.5ms/step | 14.8 samples/s +Step 24740/44000 | Loss: 2.9988 | LR: 2.07e-04 | 1349.2ms/step | 14.8 samples/s +Step 24750/44000 | Loss: 2.9343 | LR: 2.07e-04 | 1350.1ms/step | 14.8 samples/s +Validation at step 24750: Loss = 2.9710, Perplexity = 19.51 +Step 24760/44000 | Loss: 2.9553 | LR: 2.07e-04 | 3624.4ms/step | 5.5 samples/s +Step 24770/44000 | Loss: 2.9566 | LR: 2.07e-04 | 1351.0ms/step | 14.8 samples/s +Step 24780/44000 | Loss: 2.9797 | LR: 2.07e-04 | 1348.4ms/step | 14.8 samples/s +Step 24790/44000 | Loss: 2.9355 | LR: 2.06e-04 | 1349.5ms/step | 14.8 samples/s +Step 24800/44000 | Loss: 2.9402 | LR: 2.06e-04 | 1347.7ms/step | 14.8 samples/s +Step 24810/44000 | Loss: 2.9712 | LR: 2.06e-04 | 1350.1ms/step | 14.8 samples/s +Step 24820/44000 | Loss: 2.9548 | LR: 2.06e-04 | 1349.0ms/step | 14.8 samples/s +Step 24830/44000 | Loss: 2.9297 | LR: 2.06e-04 | 1349.1ms/step | 14.8 samples/s +Step 24840/44000 | Loss: 2.9473 | LR: 2.05e-04 | 1353.2ms/step | 14.8 samples/s +Step 24850/44000 | Loss: 2.9508 | LR: 2.05e-04 | 1349.7ms/step | 14.8 samples/s +Step 24860/44000 | Loss: 2.9516 | LR: 2.05e-04 | 1348.5ms/step | 14.8 samples/s +Step 24870/44000 | Loss: 2.9408 | LR: 2.05e-04 | 1351.1ms/step | 14.8 samples/s +Step 24880/44000 | Loss: 2.9674 | LR: 2.05e-04 | 1350.5ms/step | 14.8 samples/s +Step 24890/44000 | Loss: 2.9910 | LR: 2.05e-04 | 1351.0ms/step | 14.8 samples/s +Step 24900/44000 | Loss: 2.9470 | LR: 2.04e-04 | 1349.6ms/step | 14.8 samples/s +Step 24910/44000 | Loss: 2.9561 | LR: 2.04e-04 | 1349.0ms/step | 14.8 samples/s +Step 24920/44000 | Loss: 2.9625 | LR: 2.04e-04 | 1348.9ms/step | 14.8 samples/s +Step 24930/44000 | Loss: 2.9569 | LR: 2.04e-04 | 1350.9ms/step | 14.8 samples/s +Step 24940/44000 | Loss: 2.9671 | LR: 2.04e-04 | 1351.3ms/step | 14.8 samples/s +Step 24950/44000 | Loss: 2.9430 | LR: 2.04e-04 | 1351.6ms/step | 14.8 samples/s +Step 24960/44000 | Loss: 2.9920 | LR: 2.03e-04 | 1346.5ms/step | 14.9 samples/s +Step 24970/44000 | Loss: 2.9435 | LR: 2.03e-04 | 1346.9ms/step | 14.8 samples/s +Step 24980/44000 | Loss: 2.9768 | LR: 2.03e-04 | 1350.8ms/step | 14.8 samples/s +Step 24990/44000 | Loss: 2.9286 | LR: 2.03e-04 | 1353.6ms/step | 14.8 samples/s +Step 25000/44000 | Loss: 2.9729 | LR: 2.03e-04 | 1353.5ms/step | 14.8 samples/s +Validation at step 25000: Loss = 2.9680, Perplexity = 19.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-25000 +Step 25010/44000 | Loss: 2.9335 | LR: 2.02e-04 | 3916.7ms/step | 5.1 samples/s +Step 25020/44000 | Loss: 2.9638 | LR: 2.02e-04 | 1349.6ms/step | 14.8 samples/s +Step 25030/44000 | Loss: 2.9450 | LR: 2.02e-04 | 1350.6ms/step | 14.8 samples/s +Step 25040/44000 | Loss: 2.9595 | LR: 2.02e-04 | 1348.8ms/step | 14.8 samples/s +Step 25050/44000 | Loss: 2.9503 | LR: 2.02e-04 | 1351.5ms/step | 14.8 samples/s +Step 25060/44000 | Loss: 2.9471 | LR: 2.02e-04 | 1352.1ms/step | 14.8 samples/s +Step 25070/44000 | Loss: 2.9708 | LR: 2.01e-04 | 1348.8ms/step | 14.8 samples/s +Step 25080/44000 | Loss: 2.9338 | LR: 2.01e-04 | 1354.4ms/step | 14.8 samples/s +Step 25090/44000 | Loss: 2.9724 | LR: 2.01e-04 | 1353.4ms/step | 14.8 samples/s +Step 25100/44000 | Loss: 2.9322 | LR: 2.01e-04 | 1352.0ms/step | 14.8 samples/s +Step 25110/44000 | Loss: 2.9580 | LR: 2.01e-04 | 1350.9ms/step | 14.8 samples/s +Step 25120/44000 | Loss: 2.9322 | LR: 2.00e-04 | 1351.9ms/step | 14.8 samples/s +Step 25130/44000 | Loss: 2.9853 | LR: 2.00e-04 | 1350.6ms/step | 14.8 samples/s +Step 25140/44000 | Loss: 2.9945 | LR: 2.00e-04 | 1352.5ms/step | 14.8 samples/s +Step 25150/44000 | Loss: 2.9184 | LR: 2.00e-04 | 1347.0ms/step | 14.8 samples/s +Step 25160/44000 | Loss: 2.9311 | LR: 2.00e-04 | 1346.1ms/step | 14.9 samples/s +Step 25170/44000 | Loss: 2.9667 | LR: 2.00e-04 | 1350.0ms/step | 14.8 samples/s +Step 25180/44000 | Loss: 2.9066 | LR: 1.99e-04 | 1353.8ms/step | 14.8 samples/s +Step 25190/44000 | Loss: 2.9495 | LR: 1.99e-04 | 1351.8ms/step | 14.8 samples/s +Step 25200/44000 | Loss: 2.9705 | LR: 1.99e-04 | 1353.6ms/step | 14.8 samples/s +Step 25210/44000 | Loss: 2.9660 | LR: 1.99e-04 | 1350.1ms/step | 14.8 samples/s +Step 25220/44000 | Loss: 2.9343 | LR: 1.99e-04 | 1348.7ms/step | 14.8 samples/s +Step 25230/44000 | Loss: 2.9548 | LR: 1.99e-04 | 1346.7ms/step | 14.9 samples/s +Step 25240/44000 | Loss: 2.9369 | LR: 1.98e-04 | 1350.2ms/step | 14.8 samples/s +Step 25250/44000 | Loss: 2.9691 | LR: 1.98e-04 | 1352.8ms/step | 14.8 samples/s +Validation at step 25250: Loss = 2.9668, Perplexity = 19.43 +Step 25260/44000 | Loss: 2.9296 | LR: 1.98e-04 | 3611.4ms/step | 5.5 samples/s +Step 25270/44000 | Loss: 2.9609 | LR: 1.98e-04 | 1349.5ms/step | 14.8 samples/s +Step 25280/44000 | Loss: 2.9273 | LR: 1.98e-04 | 1348.5ms/step | 14.8 samples/s +Step 25290/44000 | Loss: 2.9490 | LR: 1.97e-04 | 1349.0ms/step | 14.8 samples/s +Step 25300/44000 | Loss: 2.9528 | LR: 1.97e-04 | 1350.7ms/step | 14.8 samples/s +Step 25310/44000 | Loss: 2.9035 | LR: 1.97e-04 | 1350.1ms/step | 14.8 samples/s +Step 25320/44000 | Loss: 2.9442 | LR: 1.97e-04 | 1348.8ms/step | 14.8 samples/s +Step 25330/44000 | Loss: 2.9553 | LR: 1.97e-04 | 1353.9ms/step | 14.8 samples/s +Step 25340/44000 | Loss: 2.9477 | LR: 1.97e-04 | 1351.6ms/step | 14.8 samples/s +Step 25350/44000 | Loss: 2.9386 | LR: 1.96e-04 | 1351.1ms/step | 14.8 samples/s +Step 25360/44000 | Loss: 2.9872 | LR: 1.96e-04 | 1352.9ms/step | 14.8 samples/s +Step 25370/44000 | Loss: 2.9339 | LR: 1.96e-04 | 1351.3ms/step | 14.8 samples/s +Step 25380/44000 | Loss: 2.9746 | LR: 1.96e-04 | 1351.4ms/step | 14.8 samples/s +Step 25390/44000 | Loss: 2.9462 | LR: 1.96e-04 | 1351.4ms/step | 14.8 samples/s +Step 25400/44000 | Loss: 2.9602 | LR: 1.96e-04 | 1354.2ms/step | 14.8 samples/s +Step 25410/44000 | Loss: 2.9898 | LR: 1.95e-04 | 1352.1ms/step | 14.8 samples/s +Step 25420/44000 | Loss: 2.9732 | LR: 1.95e-04 | 1353.2ms/step | 14.8 samples/s +Step 25430/44000 | Loss: 2.9518 | LR: 1.95e-04 | 1353.9ms/step | 14.8 samples/s +Step 25440/44000 | Loss: 2.9523 | LR: 1.95e-04 | 1351.6ms/step | 14.8 samples/s +Step 25450/44000 | Loss: 2.9883 | LR: 1.95e-04 | 1353.1ms/step | 14.8 samples/s +Step 25460/44000 | Loss: 2.9608 | LR: 1.94e-04 | 1349.9ms/step | 14.8 samples/s +Step 25470/44000 | Loss: 2.9400 | LR: 1.94e-04 | 1346.1ms/step | 14.9 samples/s +Step 25480/44000 | Loss: 2.9544 | LR: 1.94e-04 | 1348.4ms/step | 14.8 samples/s +Step 25490/44000 | Loss: 2.9548 | LR: 1.94e-04 | 1353.8ms/step | 14.8 samples/s +Step 25500/44000 | Loss: 2.9458 | LR: 1.94e-04 | 1354.0ms/step | 14.8 samples/s +Validation at step 25500: Loss = 2.9652, Perplexity = 19.40 +Step 25510/44000 | Loss: 2.9347 | LR: 1.94e-04 | 3618.5ms/step | 5.5 samples/s +Step 25520/44000 | Loss: 2.9324 | LR: 1.93e-04 | 1349.4ms/step | 14.8 samples/s +Step 25530/44000 | Loss: 2.9352 | LR: 1.93e-04 | 1350.3ms/step | 14.8 samples/s +Step 25540/44000 | Loss: 2.9430 | LR: 1.93e-04 | 1351.3ms/step | 14.8 samples/s +Step 25550/44000 | Loss: 2.9557 | LR: 1.93e-04 | 1351.2ms/step | 14.8 samples/s +Step 25560/44000 | Loss: 2.9616 | LR: 1.93e-04 | 1349.0ms/step | 14.8 samples/s +Step 25570/44000 | Loss: 2.9447 | LR: 1.92e-04 | 1356.7ms/step | 14.7 samples/s +Step 25580/44000 | Loss: 2.9770 | LR: 1.92e-04 | 1358.0ms/step | 14.7 samples/s +Step 25590/44000 | Loss: 2.9543 | LR: 1.92e-04 | 1353.3ms/step | 14.8 samples/s +Step 25600/44000 | Loss: 2.9231 | LR: 1.92e-04 | 1351.3ms/step | 14.8 samples/s +Step 25610/44000 | Loss: 2.9387 | LR: 1.92e-04 | 1350.9ms/step | 14.8 samples/s +Step 25620/44000 | Loss: 2.9269 | LR: 1.92e-04 | 1352.7ms/step | 14.8 samples/s +Step 25630/44000 | Loss: 2.9567 | LR: 1.91e-04 | 1355.1ms/step | 14.8 samples/s +Step 25640/44000 | Loss: 2.9411 | LR: 1.91e-04 | 1353.7ms/step | 14.8 samples/s +Step 25650/44000 | Loss: 2.9467 | LR: 1.91e-04 | 1353.0ms/step | 14.8 samples/s +Step 25660/44000 | Loss: 2.9671 | LR: 1.91e-04 | 1352.0ms/step | 14.8 samples/s +Step 25670/44000 | Loss: 2.9583 | LR: 1.91e-04 | 1354.6ms/step | 14.8 samples/s +Step 25680/44000 | Loss: 2.9251 | LR: 1.91e-04 | 1357.5ms/step | 14.7 samples/s +Step 25690/44000 | Loss: 2.9448 | LR: 1.90e-04 | 1359.7ms/step | 14.7 samples/s +Step 25700/44000 | Loss: 2.9263 | LR: 1.90e-04 | 1357.3ms/step | 14.7 samples/s +Step 25710/44000 | Loss: 2.9433 | LR: 1.90e-04 | 1352.8ms/step | 14.8 samples/s +Step 25720/44000 | Loss: 2.9358 | LR: 1.90e-04 | 1354.3ms/step | 14.8 samples/s +Step 25730/44000 | Loss: 2.9293 | LR: 1.90e-04 | 1354.5ms/step | 14.8 samples/s +Step 25740/44000 | Loss: 2.9427 | LR: 1.89e-04 | 1354.1ms/step | 14.8 samples/s +Step 25750/44000 | Loss: 2.9440 | LR: 1.89e-04 | 1355.7ms/step | 14.8 samples/s +Validation at step 25750: Loss = 2.9636, Perplexity = 19.37 +Step 25760/44000 | Loss: 2.9430 | LR: 1.89e-04 | 3620.2ms/step | 5.5 samples/s +Step 25770/44000 | Loss: 2.9659 | LR: 1.89e-04 | 1344.1ms/step | 14.9 samples/s +Step 25780/44000 | Loss: 2.9513 | LR: 1.89e-04 | 1346.8ms/step | 14.9 samples/s +Step 25790/44000 | Loss: 2.9871 | LR: 1.89e-04 | 1348.0ms/step | 14.8 samples/s +Step 25800/44000 | Loss: 2.9089 | LR: 1.88e-04 | 1348.3ms/step | 14.8 samples/s +Step 25810/44000 | Loss: 2.9438 | LR: 1.88e-04 | 1352.2ms/step | 14.8 samples/s +Step 25820/44000 | Loss: 2.9453 | LR: 1.88e-04 | 1351.2ms/step | 14.8 samples/s +Step 25830/44000 | Loss: 2.9477 | LR: 1.88e-04 | 1348.6ms/step | 14.8 samples/s +Step 25840/44000 | Loss: 2.9599 | LR: 1.88e-04 | 1348.7ms/step | 14.8 samples/s +Step 25850/44000 | Loss: 2.9234 | LR: 1.88e-04 | 1348.5ms/step | 14.8 samples/s +Step 25860/44000 | Loss: 2.9210 | LR: 1.87e-04 | 1350.9ms/step | 14.8 samples/s +Step 25870/44000 | Loss: 2.9241 | LR: 1.87e-04 | 1353.8ms/step | 14.8 samples/s +Step 25880/44000 | Loss: 2.9673 | LR: 1.87e-04 | 1352.8ms/step | 14.8 samples/s +Step 25890/44000 | Loss: 2.9470 | LR: 1.87e-04 | 1350.5ms/step | 14.8 samples/s +Step 25900/44000 | Loss: 2.9511 | LR: 1.87e-04 | 1347.9ms/step | 14.8 samples/s +Step 25910/44000 | Loss: 2.9348 | LR: 1.87e-04 | 1347.8ms/step | 14.8 samples/s +Step 25920/44000 | Loss: 2.9322 | LR: 1.86e-04 | 1345.2ms/step | 14.9 samples/s +Step 25930/44000 | Loss: 2.9638 | LR: 1.86e-04 | 1345.9ms/step | 14.9 samples/s +Step 25940/44000 | Loss: 2.9236 | LR: 1.86e-04 | 1351.4ms/step | 14.8 samples/s +Step 25950/44000 | Loss: 2.9245 | LR: 1.86e-04 | 1350.3ms/step | 14.8 samples/s +Step 25960/44000 | Loss: 2.9410 | LR: 1.86e-04 | 1348.6ms/step | 14.8 samples/s +Step 25970/44000 | Loss: 2.9201 | LR: 1.85e-04 | 1351.1ms/step | 14.8 samples/s +Step 25980/44000 | Loss: 2.9618 | LR: 1.85e-04 | 1350.5ms/step | 14.8 samples/s +Step 25990/44000 | Loss: 2.9554 | LR: 1.85e-04 | 1348.9ms/step | 14.8 samples/s +Step 26000/44000 | Loss: 2.9341 | LR: 1.85e-04 | 1346.3ms/step | 14.9 samples/s +Validation at step 26000: Loss = 2.9620, Perplexity = 19.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-26000 +Step 26010/44000 | Loss: 2.9500 | LR: 1.85e-04 | 3922.9ms/step | 5.1 samples/s +Step 26020/44000 | Loss: 2.9485 | LR: 1.85e-04 | 1348.3ms/step | 14.8 samples/s +Step 26030/44000 | Loss: 2.9353 | LR: 1.84e-04 | 1347.8ms/step | 14.8 samples/s +Step 26040/44000 | Loss: 2.9216 | LR: 1.84e-04 | 1345.5ms/step | 14.9 samples/s +Step 26050/44000 | Loss: 2.9593 | LR: 1.84e-04 | 1348.9ms/step | 14.8 samples/s +Step 26060/44000 | Loss: 2.9089 | LR: 1.84e-04 | 1348.6ms/step | 14.8 samples/s +Step 26070/44000 | Loss: 2.9374 | LR: 1.84e-04 | 1351.4ms/step | 14.8 samples/s +Step 26080/44000 | Loss: 2.9235 | LR: 1.84e-04 | 1349.4ms/step | 14.8 samples/s +Step 26090/44000 | Loss: 2.9378 | LR: 1.83e-04 | 1350.0ms/step | 14.8 samples/s +Step 26100/44000 | Loss: 2.9399 | LR: 1.83e-04 | 1348.5ms/step | 14.8 samples/s +Step 26110/44000 | Loss: 2.9567 | LR: 1.83e-04 | 1344.4ms/step | 14.9 samples/s +Step 26120/44000 | Loss: 2.9364 | LR: 1.83e-04 | 1345.1ms/step | 14.9 samples/s +Step 26130/44000 | Loss: 2.9579 | LR: 1.83e-04 | 1344.2ms/step | 14.9 samples/s +Step 26140/44000 | Loss: 2.9618 | LR: 1.82e-04 | 1346.5ms/step | 14.9 samples/s +Step 26150/44000 | Loss: 2.9535 | LR: 1.82e-04 | 1350.9ms/step | 14.8 samples/s +Step 26160/44000 | Loss: 2.9378 | LR: 1.82e-04 | 1347.6ms/step | 14.8 samples/s +Step 26170/44000 | Loss: 2.9412 | LR: 1.82e-04 | 1348.5ms/step | 14.8 samples/s +Step 26180/44000 | Loss: 2.9546 | LR: 1.82e-04 | 1348.1ms/step | 14.8 samples/s +Step 26190/44000 | Loss: 2.9270 | LR: 1.82e-04 | 1347.2ms/step | 14.8 samples/s +Step 26200/44000 | Loss: 2.9084 | LR: 1.81e-04 | 1344.8ms/step | 14.9 samples/s +Step 26210/44000 | Loss: 2.9724 | LR: 1.81e-04 | 1346.8ms/step | 14.9 samples/s +Step 26220/44000 | Loss: 2.9449 | LR: 1.81e-04 | 1346.8ms/step | 14.9 samples/s +Step 26230/44000 | Loss: 2.9552 | LR: 1.81e-04 | 1346.6ms/step | 14.9 samples/s +Step 26240/44000 | Loss: 2.9459 | LR: 1.81e-04 | 1347.9ms/step | 14.8 samples/s +Step 26250/44000 | Loss: 2.9553 | LR: 1.81e-04 | 1350.1ms/step | 14.8 samples/s +Validation at step 26250: Loss = 2.9607, Perplexity = 19.31 +Step 26260/44000 | Loss: 2.9386 | LR: 1.80e-04 | 3613.5ms/step | 5.5 samples/s +Step 26270/44000 | Loss: 2.9357 | LR: 1.80e-04 | 1346.9ms/step | 14.8 samples/s +Step 26280/44000 | Loss: 2.9554 | LR: 1.80e-04 | 1345.8ms/step | 14.9 samples/s +Step 26290/44000 | Loss: 2.9687 | LR: 1.80e-04 | 1344.6ms/step | 14.9 samples/s +Step 26300/44000 | Loss: 2.9495 | LR: 1.80e-04 | 1345.3ms/step | 14.9 samples/s +Step 26310/44000 | Loss: 2.9350 | LR: 1.80e-04 | 1344.6ms/step | 14.9 samples/s +Step 26320/44000 | Loss: 2.9532 | LR: 1.79e-04 | 1345.3ms/step | 14.9 samples/s +Step 26330/44000 | Loss: 2.9857 | LR: 1.79e-04 | 1346.0ms/step | 14.9 samples/s +Step 26340/44000 | Loss: 2.9407 | LR: 1.79e-04 | 1347.6ms/step | 14.8 samples/s +Step 26350/44000 | Loss: 2.9559 | LR: 1.79e-04 | 1347.9ms/step | 14.8 samples/s +Step 26360/44000 | Loss: 2.9323 | LR: 1.79e-04 | 1347.8ms/step | 14.8 samples/s +Step 26370/44000 | Loss: 2.9577 | LR: 1.78e-04 | 1348.2ms/step | 14.8 samples/s +Step 26380/44000 | Loss: 2.9622 | LR: 1.78e-04 | 1348.4ms/step | 14.8 samples/s +Step 26390/44000 | Loss: 2.9605 | LR: 1.78e-04 | 1348.7ms/step | 14.8 samples/s +Step 26400/44000 | Loss: 2.9625 | LR: 1.78e-04 | 1348.4ms/step | 14.8 samples/s +Step 26410/44000 | Loss: 2.9249 | LR: 1.78e-04 | 1348.3ms/step | 14.8 samples/s +Step 26420/44000 | Loss: 2.9206 | LR: 1.78e-04 | 1350.2ms/step | 14.8 samples/s +Step 26430/44000 | Loss: 2.9617 | LR: 1.77e-04 | 1349.6ms/step | 14.8 samples/s +Step 26440/44000 | Loss: 2.9192 | LR: 1.77e-04 | 1348.4ms/step | 14.8 samples/s +Step 26450/44000 | Loss: 2.9446 | LR: 1.77e-04 | 1350.0ms/step | 14.8 samples/s +Step 26460/44000 | Loss: 2.9478 | LR: 1.77e-04 | 1350.3ms/step | 14.8 samples/s +Step 26470/44000 | Loss: 2.9393 | LR: 1.77e-04 | 1348.3ms/step | 14.8 samples/s +Step 26480/44000 | Loss: 2.9371 | LR: 1.77e-04 | 1351.0ms/step | 14.8 samples/s +Step 26490/44000 | Loss: 2.9740 | LR: 1.76e-04 | 1349.6ms/step | 14.8 samples/s +Step 26500/44000 | Loss: 2.9568 | LR: 1.76e-04 | 1349.7ms/step | 14.8 samples/s +Validation at step 26500: Loss = 2.9592, Perplexity = 19.28 +Step 26510/44000 | Loss: 2.9680 | LR: 1.76e-04 | 3607.7ms/step | 5.5 samples/s +Step 26520/44000 | Loss: 2.9433 | LR: 1.76e-04 | 1345.0ms/step | 14.9 samples/s +Step 26530/44000 | Loss: 2.9466 | LR: 1.76e-04 | 1345.1ms/step | 14.9 samples/s +Step 26540/44000 | Loss: 2.9549 | LR: 1.76e-04 | 1349.3ms/step | 14.8 samples/s +Step 26550/44000 | Loss: 2.9597 | LR: 1.75e-04 | 1351.0ms/step | 14.8 samples/s +Step 26560/44000 | Loss: 2.9916 | LR: 1.75e-04 | 1352.2ms/step | 14.8 samples/s +Step 26570/44000 | Loss: 2.9179 | LR: 1.75e-04 | 1352.0ms/step | 14.8 samples/s +Step 26580/44000 | Loss: 2.9701 | LR: 1.75e-04 | 1348.6ms/step | 14.8 samples/s +Step 26590/44000 | Loss: 2.9732 | LR: 1.75e-04 | 1349.4ms/step | 14.8 samples/s +Step 26600/44000 | Loss: 2.9441 | LR: 1.74e-04 | 1348.4ms/step | 14.8 samples/s +Step 26610/44000 | Loss: 2.9523 | LR: 1.74e-04 | 1348.6ms/step | 14.8 samples/s +Step 26620/44000 | Loss: 2.9306 | LR: 1.74e-04 | 1348.4ms/step | 14.8 samples/s +Step 26630/44000 | Loss: 2.9379 | LR: 1.74e-04 | 1347.1ms/step | 14.8 samples/s +Step 26640/44000 | Loss: 2.9337 | LR: 1.74e-04 | 1348.1ms/step | 14.8 samples/s +Step 26650/44000 | Loss: 2.9653 | LR: 1.74e-04 | 1347.7ms/step | 14.8 samples/s +Step 26660/44000 | Loss: 2.9534 | LR: 1.73e-04 | 1345.9ms/step | 14.9 samples/s +Step 26670/44000 | Loss: 2.9407 | LR: 1.73e-04 | 1346.4ms/step | 14.9 samples/s +Step 26680/44000 | Loss: 2.9555 | LR: 1.73e-04 | 1345.7ms/step | 14.9 samples/s +Step 26690/44000 | Loss: 2.9630 | LR: 1.73e-04 | 1346.8ms/step | 14.8 samples/s +Step 26700/44000 | Loss: 2.9591 | LR: 1.73e-04 | 1348.2ms/step | 14.8 samples/s +Step 26710/44000 | Loss: 2.9288 | LR: 1.73e-04 | 1346.2ms/step | 14.9 samples/s +Step 26720/44000 | Loss: 2.9371 | LR: 1.72e-04 | 1346.6ms/step | 14.9 samples/s +Step 26730/44000 | Loss: 2.9248 | LR: 1.72e-04 | 1351.5ms/step | 14.8 samples/s +Step 26740/44000 | Loss: 2.9481 | LR: 1.72e-04 | 1347.2ms/step | 14.8 samples/s +Step 26750/44000 | Loss: 2.9857 | LR: 1.72e-04 | 1346.0ms/step | 14.9 samples/s +Validation at step 26750: Loss = 2.9580, Perplexity = 19.26 +Step 26760/44000 | Loss: 2.9376 | LR: 1.72e-04 | 3623.1ms/step | 5.5 samples/s +Step 26770/44000 | Loss: 2.9142 | LR: 1.72e-04 | 1353.4ms/step | 14.8 samples/s +Step 26780/44000 | Loss: 2.9273 | LR: 1.71e-04 | 1352.7ms/step | 14.8 samples/s +Step 26790/44000 | Loss: 2.9402 | LR: 1.71e-04 | 1352.9ms/step | 14.8 samples/s +Step 26800/44000 | Loss: 2.9498 | LR: 1.71e-04 | 1351.2ms/step | 14.8 samples/s +Step 26810/44000 | Loss: 2.9510 | LR: 1.71e-04 | 1343.8ms/step | 14.9 samples/s +Step 26820/44000 | Loss: 2.9670 | LR: 1.71e-04 | 1347.0ms/step | 14.8 samples/s +Step 26830/44000 | Loss: 2.9442 | LR: 1.71e-04 | 1347.7ms/step | 14.8 samples/s +Step 26840/44000 | Loss: 2.9647 | LR: 1.70e-04 | 1347.4ms/step | 14.8 samples/s +Step 26850/44000 | Loss: 2.9429 | LR: 1.70e-04 | 1346.1ms/step | 14.9 samples/s +Step 26860/44000 | Loss: 2.9264 | LR: 1.70e-04 | 1344.4ms/step | 14.9 samples/s +Step 26870/44000 | Loss: 2.9539 | LR: 1.70e-04 | 1345.7ms/step | 14.9 samples/s +Step 26880/44000 | Loss: 2.9178 | LR: 1.70e-04 | 1344.6ms/step | 14.9 samples/s +Step 26890/44000 | Loss: 2.9658 | LR: 1.69e-04 | 1347.6ms/step | 14.8 samples/s +Step 26900/44000 | Loss: 2.9141 | LR: 1.69e-04 | 1343.4ms/step | 14.9 samples/s +Step 26910/44000 | Loss: 2.9595 | LR: 1.69e-04 | 1342.9ms/step | 14.9 samples/s +Step 26920/44000 | Loss: 2.9317 | LR: 1.69e-04 | 1344.9ms/step | 14.9 samples/s +Step 26930/44000 | Loss: 2.9334 | LR: 1.69e-04 | 1346.2ms/step | 14.9 samples/s +Step 26940/44000 | Loss: 2.9465 | LR: 1.69e-04 | 1350.8ms/step | 14.8 samples/s +Step 26950/44000 | Loss: 2.9502 | LR: 1.68e-04 | 1350.9ms/step | 14.8 samples/s +Step 26960/44000 | Loss: 2.9335 | LR: 1.68e-04 | 1350.1ms/step | 14.8 samples/s +Step 26970/44000 | Loss: 2.9352 | LR: 1.68e-04 | 1350.0ms/step | 14.8 samples/s +Step 26980/44000 | Loss: 2.9685 | LR: 1.68e-04 | 1347.7ms/step | 14.8 samples/s +Step 26990/44000 | Loss: 2.9306 | LR: 1.68e-04 | 1348.0ms/step | 14.8 samples/s +Step 27000/44000 | Loss: 2.9438 | LR: 1.68e-04 | 1342.1ms/step | 14.9 samples/s +Validation at step 27000: Loss = 2.9561, Perplexity = 19.22 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-27000 +Step 27010/44000 | Loss: 2.9589 | LR: 1.67e-04 | 3910.5ms/step | 5.1 samples/s +Step 27020/44000 | Loss: 2.9410 | LR: 1.67e-04 | 1342.2ms/step | 14.9 samples/s +Step 27030/44000 | Loss: 2.9584 | LR: 1.67e-04 | 1343.2ms/step | 14.9 samples/s +Step 27040/44000 | Loss: 2.9467 | LR: 1.67e-04 | 1344.1ms/step | 14.9 samples/s +Step 27050/44000 | Loss: 2.9563 | LR: 1.67e-04 | 1346.5ms/step | 14.9 samples/s +Step 27060/44000 | Loss: 2.9146 | LR: 1.67e-04 | 1347.5ms/step | 14.8 samples/s +Step 27070/44000 | Loss: 2.9508 | LR: 1.66e-04 | 1347.2ms/step | 14.8 samples/s +Step 27080/44000 | Loss: 2.9524 | LR: 1.66e-04 | 1345.6ms/step | 14.9 samples/s +Step 27090/44000 | Loss: 2.9381 | LR: 1.66e-04 | 1343.8ms/step | 14.9 samples/s +Step 27100/44000 | Loss: 2.9257 | LR: 1.66e-04 | 1344.4ms/step | 14.9 samples/s +Step 27110/44000 | Loss: 2.9204 | LR: 1.66e-04 | 1347.3ms/step | 14.8 samples/s +Step 27120/44000 | Loss: 2.9276 | LR: 1.66e-04 | 1346.6ms/step | 14.9 samples/s +Step 27130/44000 | Loss: 2.9210 | LR: 1.65e-04 | 1346.3ms/step | 14.9 samples/s +Step 27140/44000 | Loss: 2.9341 | LR: 1.65e-04 | 1346.3ms/step | 14.9 samples/s +Step 27150/44000 | Loss: 2.9529 | LR: 1.65e-04 | 1347.2ms/step | 14.8 samples/s +Step 27160/44000 | Loss: 2.9183 | LR: 1.65e-04 | 1345.4ms/step | 14.9 samples/s +Step 27170/44000 | Loss: 2.9467 | LR: 1.65e-04 | 1342.2ms/step | 14.9 samples/s +Step 27180/44000 | Loss: 2.9534 | LR: 1.65e-04 | 1345.4ms/step | 14.9 samples/s +Step 27190/44000 | Loss: 2.9513 | LR: 1.64e-04 | 1342.9ms/step | 14.9 samples/s +Step 27200/44000 | Loss: 2.9561 | LR: 1.64e-04 | 1345.2ms/step | 14.9 samples/s +Step 27210/44000 | Loss: 2.9419 | LR: 1.64e-04 | 1346.3ms/step | 14.9 samples/s +Step 27220/44000 | Loss: 2.9239 | LR: 1.64e-04 | 1345.7ms/step | 14.9 samples/s +Step 27230/44000 | Loss: 2.9174 | LR: 1.64e-04 | 1345.7ms/step | 14.9 samples/s +Step 27240/44000 | Loss: 2.9503 | LR: 1.63e-04 | 1345.1ms/step | 14.9 samples/s +Step 27250/44000 | Loss: 2.9197 | LR: 1.63e-04 | 1344.3ms/step | 14.9 samples/s +Validation at step 27250: Loss = 2.9549, Perplexity = 19.20 +Step 27260/44000 | Loss: 2.9595 | LR: 1.63e-04 | 3606.6ms/step | 5.5 samples/s +Step 27270/44000 | Loss: 2.9702 | LR: 1.63e-04 | 1346.8ms/step | 14.9 samples/s +Step 27280/44000 | Loss: 2.9603 | LR: 1.63e-04 | 1345.5ms/step | 14.9 samples/s +Step 27290/44000 | Loss: 2.9297 | LR: 1.63e-04 | 1347.8ms/step | 14.8 samples/s +Step 27300/44000 | Loss: 2.9190 | LR: 1.62e-04 | 1347.7ms/step | 14.8 samples/s +Step 27310/44000 | Loss: 2.9485 | LR: 1.62e-04 | 1348.9ms/step | 14.8 samples/s +Step 27320/44000 | Loss: 2.9561 | LR: 1.62e-04 | 1349.3ms/step | 14.8 samples/s +Step 27330/44000 | Loss: 2.9591 | LR: 1.62e-04 | 1344.5ms/step | 14.9 samples/s +Step 27340/44000 | Loss: 2.9156 | LR: 1.62e-04 | 1343.0ms/step | 14.9 samples/s +Step 27350/44000 | Loss: 2.9143 | LR: 1.62e-04 | 1342.5ms/step | 14.9 samples/s +Step 27360/44000 | Loss: 2.9203 | LR: 1.61e-04 | 1343.4ms/step | 14.9 samples/s +Step 27370/44000 | Loss: 2.9278 | LR: 1.61e-04 | 1344.1ms/step | 14.9 samples/s +Step 27380/44000 | Loss: 2.9721 | LR: 1.61e-04 | 1346.6ms/step | 14.9 samples/s +Step 27390/44000 | Loss: 2.9293 | LR: 1.61e-04 | 1344.0ms/step | 14.9 samples/s +Step 27400/44000 | Loss: 2.9581 | LR: 1.61e-04 | 1343.3ms/step | 14.9 samples/s +Step 27410/44000 | Loss: 2.9393 | LR: 1.61e-04 | 1343.1ms/step | 14.9 samples/s +Step 27420/44000 | Loss: 2.9323 | LR: 1.60e-04 | 1346.0ms/step | 14.9 samples/s +Step 27430/44000 | Loss: 2.9554 | LR: 1.60e-04 | 1350.4ms/step | 14.8 samples/s +Step 27440/44000 | Loss: 2.9454 | LR: 1.60e-04 | 1347.6ms/step | 14.8 samples/s +Step 27450/44000 | Loss: 2.9130 | LR: 1.60e-04 | 1347.7ms/step | 14.8 samples/s +Step 27460/44000 | Loss: 2.9334 | LR: 1.60e-04 | 1347.9ms/step | 14.8 samples/s +Step 27470/44000 | Loss: 2.9644 | LR: 1.60e-04 | 1345.1ms/step | 14.9 samples/s +Step 27480/44000 | Loss: 2.9359 | LR: 1.59e-04 | 1346.3ms/step | 14.9 samples/s +Step 27490/44000 | Loss: 2.9550 | LR: 1.59e-04 | 1347.0ms/step | 14.8 samples/s +Step 27500/44000 | Loss: 2.9446 | LR: 1.59e-04 | 1346.4ms/step | 14.9 samples/s +Validation at step 27500: Loss = 2.9538, Perplexity = 19.18 +Step 27510/44000 | Loss: 2.9783 | LR: 1.59e-04 | 3619.3ms/step | 5.5 samples/s +Step 27520/44000 | Loss: 2.9366 | LR: 1.59e-04 | 1351.8ms/step | 14.8 samples/s +Step 27530/44000 | Loss: 2.9502 | LR: 1.59e-04 | 1350.5ms/step | 14.8 samples/s +Step 27540/44000 | Loss: 2.9069 | LR: 1.58e-04 | 1351.0ms/step | 14.8 samples/s +Step 27550/44000 | Loss: 2.9069 | LR: 1.58e-04 | 1353.2ms/step | 14.8 samples/s +Step 27560/44000 | Loss: 2.9232 | LR: 1.58e-04 | 1350.0ms/step | 14.8 samples/s +Step 27570/44000 | Loss: 2.9136 | LR: 1.58e-04 | 1351.2ms/step | 14.8 samples/s +Step 27580/44000 | Loss: 2.9658 | LR: 1.58e-04 | 1345.4ms/step | 14.9 samples/s +Step 27590/44000 | Loss: 2.9310 | LR: 1.58e-04 | 1349.6ms/step | 14.8 samples/s +Step 27600/44000 | Loss: 2.9455 | LR: 1.57e-04 | 1351.7ms/step | 14.8 samples/s +Step 27610/44000 | Loss: 2.9139 | LR: 1.57e-04 | 1351.3ms/step | 14.8 samples/s +Step 27620/44000 | Loss: 2.9454 | LR: 1.57e-04 | 1348.1ms/step | 14.8 samples/s +Step 27630/44000 | Loss: 2.9140 | LR: 1.57e-04 | 1350.7ms/step | 14.8 samples/s +Step 27640/44000 | Loss: 2.9743 | LR: 1.57e-04 | 1348.5ms/step | 14.8 samples/s +Step 27650/44000 | Loss: 2.9939 | LR: 1.57e-04 | 1348.3ms/step | 14.8 samples/s +Step 27660/44000 | Loss: 2.9433 | LR: 1.56e-04 | 1349.1ms/step | 14.8 samples/s +Step 27670/44000 | Loss: 2.9326 | LR: 1.56e-04 | 1346.5ms/step | 14.9 samples/s +Step 27680/44000 | Loss: 2.9488 | LR: 1.56e-04 | 1345.3ms/step | 14.9 samples/s +Step 27690/44000 | Loss: 2.9372 | LR: 1.56e-04 | 1342.6ms/step | 14.9 samples/s +Step 27700/44000 | Loss: 2.9281 | LR: 1.56e-04 | 1345.0ms/step | 14.9 samples/s +Step 27710/44000 | Loss: 2.9486 | LR: 1.56e-04 | 1346.1ms/step | 14.9 samples/s +Step 27720/44000 | Loss: 2.9333 | LR: 1.55e-04 | 1346.5ms/step | 14.9 samples/s +Step 27730/44000 | Loss: 2.9351 | LR: 1.55e-04 | 1345.8ms/step | 14.9 samples/s +Step 27740/44000 | Loss: 2.9328 | LR: 1.55e-04 | 1344.8ms/step | 14.9 samples/s +Step 27750/44000 | Loss: 2.9332 | LR: 1.55e-04 | 1342.5ms/step | 14.9 samples/s +Validation at step 27750: Loss = 2.9528, Perplexity = 19.16 +Step 27760/44000 | Loss: 2.9375 | LR: 1.55e-04 | 3604.4ms/step | 5.5 samples/s +Step 27770/44000 | Loss: 2.9335 | LR: 1.55e-04 | 1343.5ms/step | 14.9 samples/s +Step 27780/44000 | Loss: 2.9327 | LR: 1.54e-04 | 1342.9ms/step | 14.9 samples/s +Step 27790/44000 | Loss: 2.8954 | LR: 1.54e-04 | 1349.1ms/step | 14.8 samples/s +Step 27800/44000 | Loss: 2.9289 | LR: 1.54e-04 | 1348.9ms/step | 14.8 samples/s +Step 27810/44000 | Loss: 2.9004 | LR: 1.54e-04 | 1346.4ms/step | 14.9 samples/s +Step 27820/44000 | Loss: 2.9228 | LR: 1.54e-04 | 1344.3ms/step | 14.9 samples/s +Step 27830/44000 | Loss: 2.9402 | LR: 1.54e-04 | 1343.6ms/step | 14.9 samples/s +Step 27840/44000 | Loss: 2.9548 | LR: 1.53e-04 | 1344.0ms/step | 14.9 samples/s +Step 27850/44000 | Loss: 2.9213 | LR: 1.53e-04 | 1345.9ms/step | 14.9 samples/s +Step 27860/44000 | Loss: 2.9298 | LR: 1.53e-04 | 1347.9ms/step | 14.8 samples/s +Step 27870/44000 | Loss: 2.9616 | LR: 1.53e-04 | 1346.0ms/step | 14.9 samples/s +Step 27880/44000 | Loss: 2.9337 | LR: 1.53e-04 | 1345.9ms/step | 14.9 samples/s +Step 27890/44000 | Loss: 2.9114 | LR: 1.53e-04 | 1348.3ms/step | 14.8 samples/s +Step 27900/44000 | Loss: 2.9174 | LR: 1.52e-04 | 1349.7ms/step | 14.8 samples/s +Step 27910/44000 | Loss: 2.9303 | LR: 1.52e-04 | 1348.1ms/step | 14.8 samples/s +Step 27920/44000 | Loss: 2.9686 | LR: 1.52e-04 | 1348.4ms/step | 14.8 samples/s +Step 27930/44000 | Loss: 2.9453 | LR: 1.52e-04 | 1343.7ms/step | 14.9 samples/s +Step 27940/44000 | Loss: 2.9382 | LR: 1.52e-04 | 1343.6ms/step | 14.9 samples/s +Step 27950/44000 | Loss: 2.9573 | LR: 1.52e-04 | 1345.7ms/step | 14.9 samples/s +Step 27960/44000 | Loss: 2.9389 | LR: 1.51e-04 | 1347.6ms/step | 14.8 samples/s +Step 27970/44000 | Loss: 2.9330 | LR: 1.51e-04 | 1346.7ms/step | 14.9 samples/s +Step 27980/44000 | Loss: 2.9394 | LR: 1.51e-04 | 1346.6ms/step | 14.9 samples/s +Step 27990/44000 | Loss: 2.9312 | LR: 1.51e-04 | 1345.9ms/step | 14.9 samples/s +Step 28000/44000 | Loss: 2.9347 | LR: 1.51e-04 | 1345.9ms/step | 14.9 samples/s +Validation at step 28000: Loss = 2.9514, Perplexity = 19.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-28000 +Step 28010/44000 | Loss: 2.9353 | LR: 1.51e-04 | 3907.7ms/step | 5.1 samples/s +Step 28020/44000 | Loss: 2.9714 | LR: 1.50e-04 | 1339.8ms/step | 14.9 samples/s +Step 28030/44000 | Loss: 2.9285 | LR: 1.50e-04 | 1340.6ms/step | 14.9 samples/s +Step 28040/44000 | Loss: 2.9479 | LR: 1.50e-04 | 1340.5ms/step | 14.9 samples/s +Step 28050/44000 | Loss: 2.9525 | LR: 1.50e-04 | 1340.9ms/step | 14.9 samples/s +Step 28060/44000 | Loss: 2.9598 | LR: 1.50e-04 | 1340.5ms/step | 14.9 samples/s +Step 28070/44000 | Loss: 2.9314 | LR: 1.50e-04 | 1342.0ms/step | 14.9 samples/s +Step 28080/44000 | Loss: 2.9148 | LR: 1.49e-04 | 1343.9ms/step | 14.9 samples/s +Step 28090/44000 | Loss: 2.9401 | LR: 1.49e-04 | 1343.9ms/step | 14.9 samples/s +Step 28100/44000 | Loss: 2.9344 | LR: 1.49e-04 | 1344.5ms/step | 14.9 samples/s +Step 28110/44000 | Loss: 2.9138 | LR: 1.49e-04 | 1343.8ms/step | 14.9 samples/s +Step 28120/44000 | Loss: 2.9415 | LR: 1.49e-04 | 1343.6ms/step | 14.9 samples/s +Step 28130/44000 | Loss: 2.9423 | LR: 1.49e-04 | 1342.5ms/step | 14.9 samples/s +Step 28140/44000 | Loss: 2.9369 | LR: 1.48e-04 | 1343.2ms/step | 14.9 samples/s +Step 28150/44000 | Loss: 2.9297 | LR: 1.48e-04 | 1342.0ms/step | 14.9 samples/s +Step 28160/44000 | Loss: 2.9434 | LR: 1.48e-04 | 1341.3ms/step | 14.9 samples/s +Step 28170/44000 | Loss: 2.9189 | LR: 1.48e-04 | 1340.9ms/step | 14.9 samples/s +Step 28180/44000 | Loss: 2.9567 | LR: 1.48e-04 | 1339.8ms/step | 14.9 samples/s +Step 28190/44000 | Loss: 2.9486 | LR: 1.48e-04 | 1342.4ms/step | 14.9 samples/s +Step 28200/44000 | Loss: 2.9560 | LR: 1.47e-04 | 1341.9ms/step | 14.9 samples/s +Step 28210/44000 | Loss: 2.9046 | LR: 1.47e-04 | 1341.0ms/step | 14.9 samples/s +Step 28220/44000 | Loss: 2.9222 | LR: 1.47e-04 | 1343.0ms/step | 14.9 samples/s +Step 28230/44000 | Loss: 2.9429 | LR: 1.47e-04 | 1341.0ms/step | 14.9 samples/s +Step 28240/44000 | Loss: 2.9041 | LR: 1.47e-04 | 1341.4ms/step | 14.9 samples/s +Step 28250/44000 | Loss: 2.9177 | LR: 1.47e-04 | 1344.0ms/step | 14.9 samples/s +Validation at step 28250: Loss = 2.9504, Perplexity = 19.11 +Step 28260/44000 | Loss: 2.9380 | LR: 1.46e-04 | 3594.1ms/step | 5.6 samples/s +Step 28270/44000 | Loss: 2.9216 | LR: 1.46e-04 | 1344.8ms/step | 14.9 samples/s +Step 28280/44000 | Loss: 2.9655 | LR: 1.46e-04 | 1346.3ms/step | 14.9 samples/s +Step 28290/44000 | Loss: 2.8924 | LR: 1.46e-04 | 1344.1ms/step | 14.9 samples/s +Step 28300/44000 | Loss: 2.9337 | LR: 1.46e-04 | 1342.9ms/step | 14.9 samples/s +Step 28310/44000 | Loss: 2.9086 | LR: 1.46e-04 | 1344.8ms/step | 14.9 samples/s +Step 28320/44000 | Loss: 2.9728 | LR: 1.45e-04 | 1344.9ms/step | 14.9 samples/s +Step 28330/44000 | Loss: 2.9334 | LR: 1.45e-04 | 1348.8ms/step | 14.8 samples/s +Step 28340/44000 | Loss: 2.9466 | LR: 1.45e-04 | 1347.5ms/step | 14.8 samples/s +Step 28350/44000 | Loss: 2.9234 | LR: 1.45e-04 | 1344.3ms/step | 14.9 samples/s +Step 28360/44000 | Loss: 2.9317 | LR: 1.45e-04 | 1342.2ms/step | 14.9 samples/s +Step 28370/44000 | Loss: 2.9096 | LR: 1.45e-04 | 1340.5ms/step | 14.9 samples/s +Step 28380/44000 | Loss: 2.9176 | LR: 1.44e-04 | 1345.8ms/step | 14.9 samples/s +Step 28390/44000 | Loss: 2.9312 | LR: 1.44e-04 | 1347.0ms/step | 14.8 samples/s +Step 28400/44000 | Loss: 2.9323 | LR: 1.44e-04 | 1346.0ms/step | 14.9 samples/s +Step 28410/44000 | Loss: 2.9576 | LR: 1.44e-04 | 1345.9ms/step | 14.9 samples/s +Step 28420/44000 | Loss: 2.9771 | LR: 1.44e-04 | 1344.7ms/step | 14.9 samples/s +Step 28430/44000 | Loss: 2.9029 | LR: 1.44e-04 | 1344.7ms/step | 14.9 samples/s +Step 28440/44000 | Loss: 2.9444 | LR: 1.43e-04 | 1343.0ms/step | 14.9 samples/s +Step 28450/44000 | Loss: 2.9394 | LR: 1.43e-04 | 1343.5ms/step | 14.9 samples/s +Step 28460/44000 | Loss: 2.9481 | LR: 1.43e-04 | 1345.8ms/step | 14.9 samples/s +Step 28470/44000 | Loss: 2.9458 | LR: 1.43e-04 | 1344.6ms/step | 14.9 samples/s +Step 28480/44000 | Loss: 2.9264 | LR: 1.43e-04 | 1346.0ms/step | 14.9 samples/s +Step 28490/44000 | Loss: 2.9330 | LR: 1.43e-04 | 1349.0ms/step | 14.8 samples/s +Step 28500/44000 | Loss: 2.9332 | LR: 1.42e-04 | 1349.1ms/step | 14.8 samples/s +Validation at step 28500: Loss = 2.9490, Perplexity = 19.09 +Step 28510/44000 | Loss: 2.9598 | LR: 1.42e-04 | 3605.3ms/step | 5.5 samples/s +Step 28520/44000 | Loss: 2.9377 | LR: 1.42e-04 | 1344.5ms/step | 14.9 samples/s +Step 28530/44000 | Loss: 2.9271 | LR: 1.42e-04 | 1345.9ms/step | 14.9 samples/s +Step 28540/44000 | Loss: 2.9534 | LR: 1.42e-04 | 1346.3ms/step | 14.9 samples/s +Step 28550/44000 | Loss: 2.9699 | LR: 1.42e-04 | 1347.2ms/step | 14.8 samples/s +Step 28560/44000 | Loss: 2.9201 | LR: 1.41e-04 | 1344.1ms/step | 14.9 samples/s +Step 28570/44000 | Loss: 2.9126 | LR: 1.41e-04 | 1346.0ms/step | 14.9 samples/s +Step 28580/44000 | Loss: 2.9442 | LR: 1.41e-04 | 1347.7ms/step | 14.8 samples/s +Step 28590/44000 | Loss: 2.9618 | LR: 1.41e-04 | 1346.2ms/step | 14.9 samples/s +Step 28600/44000 | Loss: 2.9604 | LR: 1.41e-04 | 1344.0ms/step | 14.9 samples/s +Step 28610/44000 | Loss: 2.9257 | LR: 1.41e-04 | 1343.9ms/step | 14.9 samples/s +Step 28620/44000 | Loss: 2.9103 | LR: 1.40e-04 | 1344.9ms/step | 14.9 samples/s +Step 28630/44000 | Loss: 2.9203 | LR: 1.40e-04 | 1337.9ms/step | 14.9 samples/s +Step 28640/44000 | Loss: 2.9041 | LR: 1.40e-04 | 1340.5ms/step | 14.9 samples/s +Step 28650/44000 | Loss: 2.9275 | LR: 1.40e-04 | 1344.6ms/step | 14.9 samples/s +Step 28660/44000 | Loss: 2.9237 | LR: 1.40e-04 | 1342.9ms/step | 14.9 samples/s +Step 28670/44000 | Loss: 2.8940 | LR: 1.40e-04 | 1341.9ms/step | 14.9 samples/s +Step 28680/44000 | Loss: 2.9434 | LR: 1.39e-04 | 1342.1ms/step | 14.9 samples/s +Step 28690/44000 | Loss: 2.9389 | LR: 1.39e-04 | 1340.9ms/step | 14.9 samples/s +Step 28700/44000 | Loss: 2.9415 | LR: 1.39e-04 | 1344.5ms/step | 14.9 samples/s +Step 28710/44000 | Loss: 2.9467 | LR: 1.39e-04 | 1344.2ms/step | 14.9 samples/s +Step 28720/44000 | Loss: 2.9228 | LR: 1.39e-04 | 1341.2ms/step | 14.9 samples/s +Step 28730/44000 | Loss: 2.9236 | LR: 1.39e-04 | 1343.0ms/step | 14.9 samples/s +Step 28740/44000 | Loss: 2.9481 | LR: 1.38e-04 | 1343.5ms/step | 14.9 samples/s +Step 28750/44000 | Loss: 2.9117 | LR: 1.38e-04 | 1345.5ms/step | 14.9 samples/s +Validation at step 28750: Loss = 2.9482, Perplexity = 19.07 +Step 28760/44000 | Loss: 2.9395 | LR: 1.38e-04 | 3605.9ms/step | 5.5 samples/s +Step 28770/44000 | Loss: 2.9070 | LR: 1.38e-04 | 1343.5ms/step | 14.9 samples/s +Step 28780/44000 | Loss: 2.9413 | LR: 1.38e-04 | 1342.3ms/step | 14.9 samples/s +Step 28790/44000 | Loss: 2.9497 | LR: 1.38e-04 | 1344.9ms/step | 14.9 samples/s +Step 28800/44000 | Loss: 2.9518 | LR: 1.38e-04 | 1345.1ms/step | 14.9 samples/s +Step 28810/44000 | Loss: 2.9374 | LR: 1.37e-04 | 1342.8ms/step | 14.9 samples/s +Step 28820/44000 | Loss: 2.9110 | LR: 1.37e-04 | 1341.6ms/step | 14.9 samples/s +Step 28830/44000 | Loss: 2.9455 | LR: 1.37e-04 | 1341.5ms/step | 14.9 samples/s +Step 28840/44000 | Loss: 2.9387 | LR: 1.37e-04 | 1348.3ms/step | 14.8 samples/s +Step 28850/44000 | Loss: 2.9152 | LR: 1.37e-04 | 1346.2ms/step | 14.9 samples/s +Step 28860/44000 | Loss: 2.9482 | LR: 1.37e-04 | 1347.5ms/step | 14.8 samples/s +Step 28870/44000 | Loss: 2.9522 | LR: 1.36e-04 | 1347.5ms/step | 14.8 samples/s +Step 28880/44000 | Loss: 2.9256 | LR: 1.36e-04 | 1345.0ms/step | 14.9 samples/s +Step 28890/44000 | Loss: 2.9378 | LR: 1.36e-04 | 1348.5ms/step | 14.8 samples/s +Step 28900/44000 | Loss: 2.9311 | LR: 1.36e-04 | 1343.1ms/step | 14.9 samples/s +Step 28910/44000 | Loss: 2.9292 | LR: 1.36e-04 | 1344.3ms/step | 14.9 samples/s +Step 28920/44000 | Loss: 2.9434 | LR: 1.36e-04 | 1346.5ms/step | 14.9 samples/s +Step 28930/44000 | Loss: 2.9335 | LR: 1.35e-04 | 1347.7ms/step | 14.8 samples/s +Step 28940/44000 | Loss: 2.9285 | LR: 1.35e-04 | 1346.1ms/step | 14.9 samples/s +Step 28950/44000 | Loss: 2.9715 | LR: 1.35e-04 | 1347.2ms/step | 14.8 samples/s +Step 28960/44000 | Loss: 2.9317 | LR: 1.35e-04 | 1347.2ms/step | 14.8 samples/s +Step 28970/44000 | Loss: 2.9551 | LR: 1.35e-04 | 1348.1ms/step | 14.8 samples/s +Step 28980/44000 | Loss: 2.9392 | LR: 1.35e-04 | 1352.8ms/step | 14.8 samples/s +Step 28990/44000 | Loss: 2.9291 | LR: 1.34e-04 | 1352.7ms/step | 14.8 samples/s +Step 29000/44000 | Loss: 2.9273 | LR: 1.34e-04 | 1346.2ms/step | 14.9 samples/s +Validation at step 29000: Loss = 2.9471, Perplexity = 19.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-29000 +Step 29010/44000 | Loss: 2.9203 | LR: 1.34e-04 | 3908.8ms/step | 5.1 samples/s +Step 29020/44000 | Loss: 2.9333 | LR: 1.34e-04 | 1344.8ms/step | 14.9 samples/s +Step 29030/44000 | Loss: 2.9211 | LR: 1.34e-04 | 1348.9ms/step | 14.8 samples/s +Step 29040/44000 | Loss: 2.9078 | LR: 1.34e-04 | 1349.7ms/step | 14.8 samples/s +Step 29050/44000 | Loss: 2.9141 | LR: 1.33e-04 | 1344.8ms/step | 14.9 samples/s +Step 29060/44000 | Loss: 2.9422 | LR: 1.33e-04 | 1340.9ms/step | 14.9 samples/s +Step 29070/44000 | Loss: 2.9368 | LR: 1.33e-04 | 1340.1ms/step | 14.9 samples/s +Step 29080/44000 | Loss: 2.9061 | LR: 1.33e-04 | 1340.2ms/step | 14.9 samples/s +Step 29090/44000 | Loss: 2.9394 | LR: 1.33e-04 | 1340.3ms/step | 14.9 samples/s +Step 29100/44000 | Loss: 2.9506 | LR: 1.33e-04 | 1340.2ms/step | 14.9 samples/s +Step 29110/44000 | Loss: 2.9477 | LR: 1.33e-04 | 1344.1ms/step | 14.9 samples/s +Step 29120/44000 | Loss: 2.9142 | LR: 1.32e-04 | 1346.3ms/step | 14.9 samples/s +Step 29130/44000 | Loss: 2.9423 | LR: 1.32e-04 | 1344.0ms/step | 14.9 samples/s +Step 29140/44000 | Loss: 2.9373 | LR: 1.32e-04 | 1345.4ms/step | 14.9 samples/s +Step 29150/44000 | Loss: 2.9527 | LR: 1.32e-04 | 1347.6ms/step | 14.8 samples/s +Step 29160/44000 | Loss: 2.9311 | LR: 1.32e-04 | 1347.3ms/step | 14.8 samples/s +Step 29170/44000 | Loss: 2.9063 | LR: 1.32e-04 | 1346.3ms/step | 14.9 samples/s +Step 29180/44000 | Loss: 2.9076 | LR: 1.31e-04 | 1345.1ms/step | 14.9 samples/s +Step 29190/44000 | Loss: 2.9244 | LR: 1.31e-04 | 1346.9ms/step | 14.8 samples/s +Step 29200/44000 | Loss: 2.9304 | LR: 1.31e-04 | 1346.0ms/step | 14.9 samples/s +Step 29210/44000 | Loss: 2.8848 | LR: 1.31e-04 | 1346.2ms/step | 14.9 samples/s +Step 29220/44000 | Loss: 2.9260 | LR: 1.31e-04 | 1346.5ms/step | 14.9 samples/s +Step 29230/44000 | Loss: 2.9179 | LR: 1.31e-04 | 1343.2ms/step | 14.9 samples/s +Step 29240/44000 | Loss: 2.9435 | LR: 1.30e-04 | 1344.3ms/step | 14.9 samples/s +Step 29250/44000 | Loss: 2.9248 | LR: 1.30e-04 | 1340.9ms/step | 14.9 samples/s +Validation at step 29250: Loss = 2.9461, Perplexity = 19.03 +Step 29260/44000 | Loss: 2.9126 | LR: 1.30e-04 | 3609.3ms/step | 5.5 samples/s +Step 29270/44000 | Loss: 2.9306 | LR: 1.30e-04 | 1342.2ms/step | 14.9 samples/s +Step 29280/44000 | Loss: 2.9199 | LR: 1.30e-04 | 1342.0ms/step | 14.9 samples/s +Step 29290/44000 | Loss: 2.9363 | LR: 1.30e-04 | 1342.2ms/step | 14.9 samples/s +Step 29300/44000 | Loss: 2.9418 | LR: 1.29e-04 | 1348.3ms/step | 14.8 samples/s +Step 29310/44000 | Loss: 2.9463 | LR: 1.29e-04 | 1348.8ms/step | 14.8 samples/s +Step 29320/44000 | Loss: 2.9413 | LR: 1.29e-04 | 1344.5ms/step | 14.9 samples/s +Step 29330/44000 | Loss: 2.9465 | LR: 1.29e-04 | 1343.9ms/step | 14.9 samples/s +Step 29340/44000 | Loss: 2.9038 | LR: 1.29e-04 | 1343.4ms/step | 14.9 samples/s +Step 29350/44000 | Loss: 2.9450 | LR: 1.29e-04 | 1343.7ms/step | 14.9 samples/s +Step 29360/44000 | Loss: 2.9502 | LR: 1.29e-04 | 1342.7ms/step | 14.9 samples/s +Step 29370/44000 | Loss: 2.9214 | LR: 1.28e-04 | 1344.8ms/step | 14.9 samples/s +Step 29380/44000 | Loss: 2.9288 | LR: 1.28e-04 | 1345.9ms/step | 14.9 samples/s +Step 29390/44000 | Loss: 2.9457 | LR: 1.28e-04 | 1343.0ms/step | 14.9 samples/s +Step 29400/44000 | Loss: 2.9315 | LR: 1.28e-04 | 1345.3ms/step | 14.9 samples/s +Step 29410/44000 | Loss: 2.9209 | LR: 1.28e-04 | 1346.0ms/step | 14.9 samples/s +Step 29420/44000 | Loss: 2.9621 | LR: 1.28e-04 | 1343.8ms/step | 14.9 samples/s +Step 29430/44000 | Loss: 2.9355 | LR: 1.27e-04 | 1344.2ms/step | 14.9 samples/s +Step 29440/44000 | Loss: 2.9363 | LR: 1.27e-04 | 1344.5ms/step | 14.9 samples/s +Step 29450/44000 | Loss: 2.9550 | LR: 1.27e-04 | 1342.3ms/step | 14.9 samples/s +Step 29460/44000 | Loss: 2.9422 | LR: 1.27e-04 | 1343.1ms/step | 14.9 samples/s +Step 29470/44000 | Loss: 2.9354 | LR: 1.27e-04 | 1344.8ms/step | 14.9 samples/s +Step 29480/44000 | Loss: 2.9348 | LR: 1.27e-04 | 1343.6ms/step | 14.9 samples/s +Step 29490/44000 | Loss: 2.9236 | LR: 1.26e-04 | 1344.8ms/step | 14.9 samples/s +Step 29500/44000 | Loss: 2.9188 | LR: 1.26e-04 | 1350.6ms/step | 14.8 samples/s +Validation at step 29500: Loss = 2.9451, Perplexity = 19.01 +Step 29510/44000 | Loss: 2.9244 | LR: 1.26e-04 | 3618.8ms/step | 5.5 samples/s +Step 29520/44000 | Loss: 2.9370 | LR: 1.26e-04 | 1345.9ms/step | 14.9 samples/s +Step 29530/44000 | Loss: 2.9169 | LR: 1.26e-04 | 1345.6ms/step | 14.9 samples/s +Step 29540/44000 | Loss: 2.9133 | LR: 1.26e-04 | 1344.9ms/step | 14.9 samples/s +Step 29550/44000 | Loss: 2.9204 | LR: 1.26e-04 | 1343.2ms/step | 14.9 samples/s +Step 29560/44000 | Loss: 2.9193 | LR: 1.25e-04 | 1343.7ms/step | 14.9 samples/s +Step 29570/44000 | Loss: 2.9277 | LR: 1.25e-04 | 1344.0ms/step | 14.9 samples/s +Step 29580/44000 | Loss: 2.9187 | LR: 1.25e-04 | 1342.8ms/step | 14.9 samples/s +Step 29590/44000 | Loss: 2.9484 | LR: 1.25e-04 | 1342.7ms/step | 14.9 samples/s +Step 29600/44000 | Loss: 2.9377 | LR: 1.25e-04 | 1343.5ms/step | 14.9 samples/s +Step 29610/44000 | Loss: 2.9297 | LR: 1.25e-04 | 1344.5ms/step | 14.9 samples/s +Step 29620/44000 | Loss: 2.9135 | LR: 1.24e-04 | 1346.2ms/step | 14.9 samples/s +Step 29630/44000 | Loss: 2.9537 | LR: 1.24e-04 | 1341.8ms/step | 14.9 samples/s +Step 29640/44000 | Loss: 2.9246 | LR: 1.24e-04 | 1339.4ms/step | 14.9 samples/s +Step 29650/44000 | Loss: 2.9171 | LR: 1.24e-04 | 1339.5ms/step | 14.9 samples/s +Step 29660/44000 | Loss: 2.9720 | LR: 1.24e-04 | 1341.6ms/step | 14.9 samples/s +Step 29670/44000 | Loss: 2.9150 | LR: 1.24e-04 | 1344.1ms/step | 14.9 samples/s +Step 29680/44000 | Loss: 2.9272 | LR: 1.23e-04 | 1343.9ms/step | 14.9 samples/s +Step 29690/44000 | Loss: 2.9358 | LR: 1.23e-04 | 1342.2ms/step | 14.9 samples/s +Step 29700/44000 | Loss: 2.9379 | LR: 1.23e-04 | 1347.2ms/step | 14.8 samples/s +Step 29710/44000 | Loss: 2.9481 | LR: 1.23e-04 | 1340.3ms/step | 14.9 samples/s +Step 29720/44000 | Loss: 2.8874 | LR: 1.23e-04 | 1342.0ms/step | 14.9 samples/s +Step 29730/44000 | Loss: 2.9395 | LR: 1.23e-04 | 1340.7ms/step | 14.9 samples/s +Step 29740/44000 | Loss: 2.9037 | LR: 1.23e-04 | 1343.9ms/step | 14.9 samples/s +Step 29750/44000 | Loss: 2.9210 | LR: 1.22e-04 | 1343.8ms/step | 14.9 samples/s +Validation at step 29750: Loss = 2.9441, Perplexity = 18.99 +Step 29760/44000 | Loss: 2.9488 | LR: 1.22e-04 | 3601.9ms/step | 5.6 samples/s +Step 29770/44000 | Loss: 2.9217 | LR: 1.22e-04 | 1343.7ms/step | 14.9 samples/s +Step 29780/44000 | Loss: 2.9314 | LR: 1.22e-04 | 1340.9ms/step | 14.9 samples/s +Step 29790/44000 | Loss: 2.8987 | LR: 1.22e-04 | 1341.3ms/step | 14.9 samples/s +Step 29800/44000 | Loss: 2.9076 | LR: 1.22e-04 | 1342.1ms/step | 14.9 samples/s +Step 29810/44000 | Loss: 2.9150 | LR: 1.21e-04 | 1343.7ms/step | 14.9 samples/s +Step 29820/44000 | Loss: 2.9590 | LR: 1.21e-04 | 1343.9ms/step | 14.9 samples/s +Step 29830/44000 | Loss: 2.9470 | LR: 1.21e-04 | 1348.2ms/step | 14.8 samples/s +Step 29840/44000 | Loss: 2.8951 | LR: 1.21e-04 | 1344.9ms/step | 14.9 samples/s +Step 29850/44000 | Loss: 2.9194 | LR: 1.21e-04 | 1345.7ms/step | 14.9 samples/s +Step 29860/44000 | Loss: 2.9341 | LR: 1.21e-04 | 1344.0ms/step | 14.9 samples/s +Step 29870/44000 | Loss: 2.9044 | LR: 1.21e-04 | 1345.9ms/step | 14.9 samples/s +Step 29880/44000 | Loss: 2.9222 | LR: 1.20e-04 | 1345.7ms/step | 14.9 samples/s +Step 29890/44000 | Loss: 2.9269 | LR: 1.20e-04 | 1344.1ms/step | 14.9 samples/s +Step 29900/44000 | Loss: 2.9663 | LR: 1.20e-04 | 1344.0ms/step | 14.9 samples/s +Step 29910/44000 | Loss: 2.9235 | LR: 1.20e-04 | 1344.3ms/step | 14.9 samples/s +Step 29920/44000 | Loss: 2.9133 | LR: 1.20e-04 | 1344.8ms/step | 14.9 samples/s +Step 29930/44000 | Loss: 2.9199 | LR: 1.20e-04 | 1344.8ms/step | 14.9 samples/s +Step 29940/44000 | Loss: 2.9158 | LR: 1.19e-04 | 1343.8ms/step | 14.9 samples/s +Step 29950/44000 | Loss: 2.9256 | LR: 1.19e-04 | 1345.6ms/step | 14.9 samples/s +Step 29960/44000 | Loss: 2.9080 | LR: 1.19e-04 | 1346.3ms/step | 14.9 samples/s +Step 29970/44000 | Loss: 2.9197 | LR: 1.19e-04 | 1343.8ms/step | 14.9 samples/s +Step 29980/44000 | Loss: 2.9178 | LR: 1.19e-04 | 1344.9ms/step | 14.9 samples/s +Step 29990/44000 | Loss: 2.9334 | LR: 1.19e-04 | 1345.6ms/step | 14.9 samples/s +Step 30000/44000 | Loss: 2.9102 | LR: 1.19e-04 | 1344.1ms/step | 14.9 samples/s +Validation at step 30000: Loss = 2.9431, Perplexity = 18.97 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-30000 +Step 30010/44000 | Loss: 2.9391 | LR: 1.18e-04 | 3904.0ms/step | 5.1 samples/s +Step 30020/44000 | Loss: 2.9487 | LR: 1.18e-04 | 1345.0ms/step | 14.9 samples/s +Step 30030/44000 | Loss: 2.9179 | LR: 1.18e-04 | 1345.6ms/step | 14.9 samples/s +Step 30040/44000 | Loss: 2.9358 | LR: 1.18e-04 | 1343.4ms/step | 14.9 samples/s +Step 30050/44000 | Loss: 2.9101 | LR: 1.18e-04 | 1343.6ms/step | 14.9 samples/s +Step 30060/44000 | Loss: 2.9339 | LR: 1.18e-04 | 1346.0ms/step | 14.9 samples/s +Step 30070/44000 | Loss: 2.8879 | LR: 1.17e-04 | 1343.5ms/step | 14.9 samples/s +Step 30080/44000 | Loss: 2.9614 | LR: 1.17e-04 | 1340.8ms/step | 14.9 samples/s +Step 30090/44000 | Loss: 2.9377 | LR: 1.17e-04 | 1347.6ms/step | 14.8 samples/s +Step 30100/44000 | Loss: 2.9371 | LR: 1.17e-04 | 1345.0ms/step | 14.9 samples/s +Step 30110/44000 | Loss: 2.9230 | LR: 1.17e-04 | 1344.1ms/step | 14.9 samples/s +Step 30120/44000 | Loss: 2.9132 | LR: 1.17e-04 | 1345.2ms/step | 14.9 samples/s +Step 30130/44000 | Loss: 2.9400 | LR: 1.17e-04 | 1343.3ms/step | 14.9 samples/s +Step 30140/44000 | Loss: 2.9320 | LR: 1.16e-04 | 1343.7ms/step | 14.9 samples/s +Step 30150/44000 | Loss: 2.9144 | LR: 1.16e-04 | 1346.8ms/step | 14.8 samples/s +Step 30160/44000 | Loss: 2.9003 | LR: 1.16e-04 | 1348.5ms/step | 14.8 samples/s +Step 30170/44000 | Loss: 2.9033 | LR: 1.16e-04 | 1347.7ms/step | 14.8 samples/s +Step 30180/44000 | Loss: 2.9383 | LR: 1.16e-04 | 1348.3ms/step | 14.8 samples/s +Step 30190/44000 | Loss: 2.9146 | LR: 1.16e-04 | 1350.3ms/step | 14.8 samples/s +Step 30200/44000 | Loss: 2.9222 | LR: 1.15e-04 | 1351.1ms/step | 14.8 samples/s +Step 30210/44000 | Loss: 2.9241 | LR: 1.15e-04 | 1347.9ms/step | 14.8 samples/s +Step 30220/44000 | Loss: 2.8997 | LR: 1.15e-04 | 1346.2ms/step | 14.9 samples/s +Step 30230/44000 | Loss: 2.9460 | LR: 1.15e-04 | 1348.8ms/step | 14.8 samples/s +Step 30240/44000 | Loss: 2.9278 | LR: 1.15e-04 | 1349.1ms/step | 14.8 samples/s +Step 30250/44000 | Loss: 2.9148 | LR: 1.15e-04 | 1349.8ms/step | 14.8 samples/s +Validation at step 30250: Loss = 2.9423, Perplexity = 18.96 +Step 30260/44000 | Loss: 2.9187 | LR: 1.15e-04 | 3608.2ms/step | 5.5 samples/s +Step 30270/44000 | Loss: 2.9073 | LR: 1.14e-04 | 1344.1ms/step | 14.9 samples/s +Step 30280/44000 | Loss: 2.9512 | LR: 1.14e-04 | 1345.2ms/step | 14.9 samples/s +Step 30290/44000 | Loss: 2.9088 | LR: 1.14e-04 | 1348.4ms/step | 14.8 samples/s +Step 30300/44000 | Loss: 2.8920 | LR: 1.14e-04 | 1347.5ms/step | 14.8 samples/s +Step 30310/44000 | Loss: 2.9017 | LR: 1.14e-04 | 1345.3ms/step | 14.9 samples/s +Step 30320/44000 | Loss: 2.9016 | LR: 1.14e-04 | 1348.2ms/step | 14.8 samples/s +Step 30330/44000 | Loss: 2.9203 | LR: 1.13e-04 | 1342.4ms/step | 14.9 samples/s +Step 30340/44000 | Loss: 2.9254 | LR: 1.13e-04 | 1344.8ms/step | 14.9 samples/s +Step 30350/44000 | Loss: 2.8889 | LR: 1.13e-04 | 1345.0ms/step | 14.9 samples/s +Step 30360/44000 | Loss: 2.9264 | LR: 1.13e-04 | 1341.5ms/step | 14.9 samples/s +Step 30370/44000 | Loss: 2.9386 | LR: 1.13e-04 | 1341.5ms/step | 14.9 samples/s +Step 30380/44000 | Loss: 2.9475 | LR: 1.13e-04 | 1346.3ms/step | 14.9 samples/s +Step 30390/44000 | Loss: 2.9186 | LR: 1.13e-04 | 1345.4ms/step | 14.9 samples/s +Step 30400/44000 | Loss: 2.9209 | LR: 1.12e-04 | 1348.5ms/step | 14.8 samples/s +Step 30410/44000 | Loss: 2.9287 | LR: 1.12e-04 | 1352.4ms/step | 14.8 samples/s +Step 30420/44000 | Loss: 2.9236 | LR: 1.12e-04 | 1351.6ms/step | 14.8 samples/s +Step 30430/44000 | Loss: 2.9256 | LR: 1.12e-04 | 1345.8ms/step | 14.9 samples/s +Step 30440/44000 | Loss: 2.9261 | LR: 1.12e-04 | 1344.4ms/step | 14.9 samples/s +Step 30450/44000 | Loss: 2.8976 | LR: 1.12e-04 | 1344.1ms/step | 14.9 samples/s +Step 30460/44000 | Loss: 2.9272 | LR: 1.11e-04 | 1342.6ms/step | 14.9 samples/s +Step 30470/44000 | Loss: 2.9380 | LR: 1.11e-04 | 1343.1ms/step | 14.9 samples/s +Step 30480/44000 | Loss: 2.9279 | LR: 1.11e-04 | 1343.1ms/step | 14.9 samples/s +Step 30490/44000 | Loss: 2.9207 | LR: 1.11e-04 | 1344.2ms/step | 14.9 samples/s +Step 30500/44000 | Loss: 2.9153 | LR: 1.11e-04 | 1343.4ms/step | 14.9 samples/s +Validation at step 30500: Loss = 2.9416, Perplexity = 18.95 +Step 30510/44000 | Loss: 2.9156 | LR: 1.11e-04 | 3603.9ms/step | 5.5 samples/s +Step 30520/44000 | Loss: 2.9221 | LR: 1.11e-04 | 1345.8ms/step | 14.9 samples/s +Step 30530/44000 | Loss: 2.9614 | LR: 1.10e-04 | 1345.6ms/step | 14.9 samples/s +Step 30540/44000 | Loss: 2.9587 | LR: 1.10e-04 | 1342.1ms/step | 14.9 samples/s +Step 30550/44000 | Loss: 2.9311 | LR: 1.10e-04 | 1344.7ms/step | 14.9 samples/s +Step 30560/44000 | Loss: 2.9297 | LR: 1.10e-04 | 1345.3ms/step | 14.9 samples/s +Step 30570/44000 | Loss: 2.9433 | LR: 1.10e-04 | 1347.3ms/step | 14.8 samples/s +Step 30580/44000 | Loss: 2.9371 | LR: 1.10e-04 | 1347.7ms/step | 14.8 samples/s +Step 30590/44000 | Loss: 2.9268 | LR: 1.10e-04 | 1343.2ms/step | 14.9 samples/s +Step 30600/44000 | Loss: 2.9215 | LR: 1.09e-04 | 1342.5ms/step | 14.9 samples/s +Step 30610/44000 | Loss: 2.9238 | LR: 1.09e-04 | 1344.1ms/step | 14.9 samples/s +Step 30620/44000 | Loss: 2.9337 | LR: 1.09e-04 | 1344.5ms/step | 14.9 samples/s +Step 30630/44000 | Loss: 2.9093 | LR: 1.09e-04 | 1345.9ms/step | 14.9 samples/s +Step 30640/44000 | Loss: 2.9363 | LR: 1.09e-04 | 1344.7ms/step | 14.9 samples/s +Step 30650/44000 | Loss: 2.9459 | LR: 1.09e-04 | 1345.4ms/step | 14.9 samples/s +Step 30660/44000 | Loss: 2.9611 | LR: 1.08e-04 | 1345.2ms/step | 14.9 samples/s +Step 30670/44000 | Loss: 2.9496 | LR: 1.08e-04 | 1347.5ms/step | 14.8 samples/s +Step 30680/44000 | Loss: 2.9311 | LR: 1.08e-04 | 1347.4ms/step | 14.8 samples/s +Step 30690/44000 | Loss: 2.9164 | LR: 1.08e-04 | 1344.9ms/step | 14.9 samples/s +Step 30700/44000 | Loss: 2.9382 | LR: 1.08e-04 | 1343.7ms/step | 14.9 samples/s +Step 30710/44000 | Loss: 2.9189 | LR: 1.08e-04 | 1342.6ms/step | 14.9 samples/s +Step 30720/44000 | Loss: 2.9364 | LR: 1.08e-04 | 1345.4ms/step | 14.9 samples/s +Step 30730/44000 | Loss: 2.9205 | LR: 1.07e-04 | 1347.3ms/step | 14.8 samples/s +Step 30740/44000 | Loss: 2.9390 | LR: 1.07e-04 | 1343.4ms/step | 14.9 samples/s +Step 30750/44000 | Loss: 2.9228 | LR: 1.07e-04 | 1345.2ms/step | 14.9 samples/s +Validation at step 30750: Loss = 2.9406, Perplexity = 18.93 +Step 30760/44000 | Loss: 2.9274 | LR: 1.07e-04 | 3616.1ms/step | 5.5 samples/s +Step 30770/44000 | Loss: 2.9675 | LR: 1.07e-04 | 1351.4ms/step | 14.8 samples/s +Step 30780/44000 | Loss: 2.9187 | LR: 1.07e-04 | 1352.5ms/step | 14.8 samples/s +Step 30790/44000 | Loss: 2.9242 | LR: 1.07e-04 | 1352.0ms/step | 14.8 samples/s +Step 30800/44000 | Loss: 2.9321 | LR: 1.06e-04 | 1344.7ms/step | 14.9 samples/s +Step 30810/44000 | Loss: 2.9006 | LR: 1.06e-04 | 1342.2ms/step | 14.9 samples/s +Step 30820/44000 | Loss: 2.9314 | LR: 1.06e-04 | 1345.7ms/step | 14.9 samples/s +Step 30830/44000 | Loss: 2.9702 | LR: 1.06e-04 | 1345.5ms/step | 14.9 samples/s +Step 30840/44000 | Loss: 2.9409 | LR: 1.06e-04 | 1345.0ms/step | 14.9 samples/s +Step 30850/44000 | Loss: 2.9131 | LR: 1.06e-04 | 1345.1ms/step | 14.9 samples/s +Step 30860/44000 | Loss: 2.9431 | LR: 1.05e-04 | 1340.8ms/step | 14.9 samples/s +Step 30870/44000 | Loss: 2.9020 | LR: 1.05e-04 | 1343.2ms/step | 14.9 samples/s +Step 30880/44000 | Loss: 2.9195 | LR: 1.05e-04 | 1344.8ms/step | 14.9 samples/s +Step 30890/44000 | Loss: 2.8724 | LR: 1.05e-04 | 1345.4ms/step | 14.9 samples/s +Step 30900/44000 | Loss: 2.9024 | LR: 1.05e-04 | 1346.3ms/step | 14.9 samples/s +Step 30910/44000 | Loss: 2.9143 | LR: 1.05e-04 | 1344.6ms/step | 14.9 samples/s +Step 30920/44000 | Loss: 2.9310 | LR: 1.05e-04 | 1345.7ms/step | 14.9 samples/s +Step 30930/44000 | Loss: 2.9154 | LR: 1.04e-04 | 1346.5ms/step | 14.9 samples/s +Step 30940/44000 | Loss: 2.9025 | LR: 1.04e-04 | 1351.0ms/step | 14.8 samples/s +Step 30950/44000 | Loss: 2.9267 | LR: 1.04e-04 | 1348.0ms/step | 14.8 samples/s +Step 30960/44000 | Loss: 2.9192 | LR: 1.04e-04 | 1342.4ms/step | 14.9 samples/s +Step 30970/44000 | Loss: 2.9588 | LR: 1.04e-04 | 1343.3ms/step | 14.9 samples/s +Step 30980/44000 | Loss: 2.9062 | LR: 1.04e-04 | 1343.2ms/step | 14.9 samples/s +Step 30990/44000 | Loss: 2.9110 | LR: 1.04e-04 | 1345.6ms/step | 14.9 samples/s +Step 31000/44000 | Loss: 2.8983 | LR: 1.03e-04 | 1345.4ms/step | 14.9 samples/s +Validation at step 31000: Loss = 2.9398, Perplexity = 18.91 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-31000 +Step 31010/44000 | Loss: 2.9087 | LR: 1.03e-04 | 3898.6ms/step | 5.1 samples/s +Step 31020/44000 | Loss: 2.9387 | LR: 1.03e-04 | 1341.3ms/step | 14.9 samples/s +Step 31030/44000 | Loss: 2.8911 | LR: 1.03e-04 | 1347.2ms/step | 14.8 samples/s +Step 31040/44000 | Loss: 2.9210 | LR: 1.03e-04 | 1348.1ms/step | 14.8 samples/s +Step 31050/44000 | Loss: 2.9321 | LR: 1.03e-04 | 1345.5ms/step | 14.9 samples/s +Step 31060/44000 | Loss: 2.9394 | LR: 1.03e-04 | 1345.2ms/step | 14.9 samples/s +Step 31070/44000 | Loss: 2.9256 | LR: 1.02e-04 | 1344.5ms/step | 14.9 samples/s +Step 31080/44000 | Loss: 2.9185 | LR: 1.02e-04 | 1345.6ms/step | 14.9 samples/s +Step 31090/44000 | Loss: 2.9147 | LR: 1.02e-04 | 1346.0ms/step | 14.9 samples/s +Step 31100/44000 | Loss: 2.9070 | LR: 1.02e-04 | 1345.0ms/step | 14.9 samples/s +Step 31110/44000 | Loss: 2.9090 | LR: 1.02e-04 | 1342.5ms/step | 14.9 samples/s +Step 31120/44000 | Loss: 2.9271 | LR: 1.02e-04 | 1344.5ms/step | 14.9 samples/s +Step 31130/44000 | Loss: 2.9579 | LR: 1.02e-04 | 1347.8ms/step | 14.8 samples/s +Step 31140/44000 | Loss: 2.9111 | LR: 1.01e-04 | 1344.3ms/step | 14.9 samples/s +Step 31150/44000 | Loss: 2.9558 | LR: 1.01e-04 | 1343.5ms/step | 14.9 samples/s +Step 31160/44000 | Loss: 2.9274 | LR: 1.01e-04 | 1343.8ms/step | 14.9 samples/s +Step 31170/44000 | Loss: 2.9214 | LR: 1.01e-04 | 1343.0ms/step | 14.9 samples/s +Step 31180/44000 | Loss: 2.9301 | LR: 1.01e-04 | 1343.1ms/step | 14.9 samples/s +Step 31190/44000 | Loss: 2.9002 | LR: 1.01e-04 | 1348.4ms/step | 14.8 samples/s +Step 31200/44000 | Loss: 2.9128 | LR: 1.00e-04 | 1348.3ms/step | 14.8 samples/s +Step 31210/44000 | Loss: 2.9333 | LR: 1.00e-04 | 1349.1ms/step | 14.8 samples/s +Step 31220/44000 | Loss: 2.9205 | LR: 1.00e-04 | 1349.3ms/step | 14.8 samples/s +Step 31230/44000 | Loss: 2.9536 | LR: 1.00e-04 | 1348.6ms/step | 14.8 samples/s +Step 31240/44000 | Loss: 2.9222 | LR: 9.99e-05 | 1349.8ms/step | 14.8 samples/s +Step 31250/44000 | Loss: 2.9052 | LR: 9.98e-05 | 1348.4ms/step | 14.8 samples/s +Validation at step 31250: Loss = 2.9390, Perplexity = 18.90 +Step 31260/44000 | Loss: 2.9363 | LR: 9.96e-05 | 3608.2ms/step | 5.5 samples/s +Step 31270/44000 | Loss: 2.9503 | LR: 9.95e-05 | 1347.4ms/step | 14.8 samples/s +Step 31280/44000 | Loss: 2.9094 | LR: 9.93e-05 | 1349.3ms/step | 14.8 samples/s +Step 31290/44000 | Loss: 2.9367 | LR: 9.92e-05 | 1351.0ms/step | 14.8 samples/s +Step 31300/44000 | Loss: 2.9354 | LR: 9.90e-05 | 1347.5ms/step | 14.8 samples/s +Step 31310/44000 | Loss: 2.9239 | LR: 9.89e-05 | 1347.8ms/step | 14.8 samples/s +Step 31320/44000 | Loss: 2.9202 | LR: 9.88e-05 | 1347.3ms/step | 14.8 samples/s +Step 31330/44000 | Loss: 2.9490 | LR: 9.86e-05 | 1348.3ms/step | 14.8 samples/s +Step 31340/44000 | Loss: 2.9151 | LR: 9.85e-05 | 1347.0ms/step | 14.8 samples/s +Step 31350/44000 | Loss: 2.8947 | LR: 9.83e-05 | 1348.6ms/step | 14.8 samples/s +Step 31360/44000 | Loss: 2.9053 | LR: 9.82e-05 | 1348.8ms/step | 14.8 samples/s +Step 31370/44000 | Loss: 2.9367 | LR: 9.80e-05 | 1346.7ms/step | 14.9 samples/s +Step 31380/44000 | Loss: 2.9498 | LR: 9.79e-05 | 1346.3ms/step | 14.9 samples/s +Step 31390/44000 | Loss: 2.8893 | LR: 9.77e-05 | 1345.4ms/step | 14.9 samples/s +Step 31400/44000 | Loss: 2.9225 | LR: 9.76e-05 | 1345.8ms/step | 14.9 samples/s +Step 31410/44000 | Loss: 2.8933 | LR: 9.75e-05 | 1344.4ms/step | 14.9 samples/s +Step 31420/44000 | Loss: 2.9139 | LR: 9.73e-05 | 1345.8ms/step | 14.9 samples/s +Step 31430/44000 | Loss: 2.9344 | LR: 9.72e-05 | 1344.4ms/step | 14.9 samples/s +Step 31440/44000 | Loss: 2.9198 | LR: 9.70e-05 | 1342.0ms/step | 14.9 samples/s +Step 31450/44000 | Loss: 2.9305 | LR: 9.69e-05 | 1342.8ms/step | 14.9 samples/s +Step 31460/44000 | Loss: 2.9097 | LR: 9.67e-05 | 1341.4ms/step | 14.9 samples/s +Step 31470/44000 | Loss: 2.9382 | LR: 9.66e-05 | 1341.8ms/step | 14.9 samples/s +Step 31480/44000 | Loss: 2.9023 | LR: 9.65e-05 | 1343.5ms/step | 14.9 samples/s +Step 31490/44000 | Loss: 2.9118 | LR: 9.63e-05 | 1344.6ms/step | 14.9 samples/s +Step 31500/44000 | Loss: 2.9242 | LR: 9.62e-05 | 1342.5ms/step | 14.9 samples/s +Validation at step 31500: Loss = 2.9387, Perplexity = 18.89 +Step 31510/44000 | Loss: 2.9050 | LR: 9.60e-05 | 3604.3ms/step | 5.5 samples/s +Step 31520/44000 | Loss: 2.9165 | LR: 9.59e-05 | 1344.2ms/step | 14.9 samples/s +Step 31530/44000 | Loss: 2.9285 | LR: 9.57e-05 | 1346.2ms/step | 14.9 samples/s +Step 31540/44000 | Loss: 2.9104 | LR: 9.56e-05 | 1347.2ms/step | 14.8 samples/s +Step 31550/44000 | Loss: 2.9352 | LR: 9.54e-05 | 1346.2ms/step | 14.9 samples/s +Step 31560/44000 | Loss: 2.9205 | LR: 9.53e-05 | 1347.4ms/step | 14.8 samples/s +Step 31570/44000 | Loss: 2.9111 | LR: 9.52e-05 | 1345.1ms/step | 14.9 samples/s +Step 31580/44000 | Loss: 2.9349 | LR: 9.50e-05 | 1345.6ms/step | 14.9 samples/s +Step 31590/44000 | Loss: 2.9374 | LR: 9.49e-05 | 1344.8ms/step | 14.9 samples/s +Step 31600/44000 | Loss: 2.9187 | LR: 9.47e-05 | 1345.0ms/step | 14.9 samples/s +Step 31610/44000 | Loss: 2.9079 | LR: 9.46e-05 | 1344.0ms/step | 14.9 samples/s +Step 31620/44000 | Loss: 2.9314 | LR: 9.45e-05 | 1343.7ms/step | 14.9 samples/s +Step 31630/44000 | Loss: 2.9149 | LR: 9.43e-05 | 1345.0ms/step | 14.9 samples/s +Step 31640/44000 | Loss: 2.8894 | LR: 9.42e-05 | 1338.0ms/step | 14.9 samples/s +Step 31650/44000 | Loss: 2.8934 | LR: 9.40e-05 | 1340.4ms/step | 14.9 samples/s +Step 31660/44000 | Loss: 2.9082 | LR: 9.39e-05 | 1345.4ms/step | 14.9 samples/s +Step 31670/44000 | Loss: 2.9037 | LR: 9.37e-05 | 1343.1ms/step | 14.9 samples/s +Step 31680/44000 | Loss: 2.9160 | LR: 9.36e-05 | 1341.2ms/step | 14.9 samples/s +Step 31690/44000 | Loss: 2.9246 | LR: 9.35e-05 | 1341.1ms/step | 14.9 samples/s +Step 31700/44000 | Loss: 2.9395 | LR: 9.33e-05 | 1341.8ms/step | 14.9 samples/s +Step 31710/44000 | Loss: 2.9173 | LR: 9.32e-05 | 1346.5ms/step | 14.9 samples/s +Step 31720/44000 | Loss: 2.9150 | LR: 9.30e-05 | 1343.3ms/step | 14.9 samples/s +Step 31730/44000 | Loss: 2.8884 | LR: 9.29e-05 | 1343.3ms/step | 14.9 samples/s +Step 31740/44000 | Loss: 2.9231 | LR: 9.28e-05 | 1343.6ms/step | 14.9 samples/s +Step 31750/44000 | Loss: 2.9317 | LR: 9.26e-05 | 1342.8ms/step | 14.9 samples/s +Validation at step 31750: Loss = 2.9380, Perplexity = 18.88 +Step 31760/44000 | Loss: 2.9065 | LR: 9.25e-05 | 3600.7ms/step | 5.6 samples/s +Step 31770/44000 | Loss: 2.9327 | LR: 9.23e-05 | 1340.7ms/step | 14.9 samples/s +Step 31780/44000 | Loss: 2.9226 | LR: 9.22e-05 | 1337.6ms/step | 15.0 samples/s +Step 31790/44000 | Loss: 2.9249 | LR: 9.20e-05 | 1340.2ms/step | 14.9 samples/s +Step 31800/44000 | Loss: 2.8952 | LR: 9.19e-05 | 1339.7ms/step | 14.9 samples/s +Step 31810/44000 | Loss: 2.8945 | LR: 9.18e-05 | 1341.8ms/step | 14.9 samples/s +Step 31820/44000 | Loss: 2.9122 | LR: 9.16e-05 | 1341.5ms/step | 14.9 samples/s +Step 31830/44000 | Loss: 2.9179 | LR: 9.15e-05 | 1339.5ms/step | 14.9 samples/s +Step 31840/44000 | Loss: 2.9295 | LR: 9.13e-05 | 1338.9ms/step | 14.9 samples/s +Step 31850/44000 | Loss: 2.9332 | LR: 9.12e-05 | 1341.4ms/step | 14.9 samples/s +Step 31860/44000 | Loss: 2.9397 | LR: 9.11e-05 | 1340.8ms/step | 14.9 samples/s +Step 31870/44000 | Loss: 2.9171 | LR: 9.09e-05 | 1338.7ms/step | 14.9 samples/s +Step 31880/44000 | Loss: 2.9514 | LR: 9.08e-05 | 1342.7ms/step | 14.9 samples/s +Step 31890/44000 | Loss: 2.8923 | LR: 9.06e-05 | 1343.5ms/step | 14.9 samples/s +Step 31900/44000 | Loss: 2.9113 | LR: 9.05e-05 | 1344.7ms/step | 14.9 samples/s +Step 31910/44000 | Loss: 2.9447 | LR: 9.04e-05 | 1345.8ms/step | 14.9 samples/s +Step 31920/44000 | Loss: 2.8703 | LR: 9.02e-05 | 1347.3ms/step | 14.8 samples/s +Step 31930/44000 | Loss: 2.9274 | LR: 9.01e-05 | 1341.8ms/step | 14.9 samples/s +Step 31940/44000 | Loss: 2.9347 | LR: 8.99e-05 | 1342.5ms/step | 14.9 samples/s +Step 31950/44000 | Loss: 2.9247 | LR: 8.98e-05 | 1343.5ms/step | 14.9 samples/s +Step 31960/44000 | Loss: 2.9386 | LR: 8.97e-05 | 1345.2ms/step | 14.9 samples/s +Step 31970/44000 | Loss: 2.9073 | LR: 8.95e-05 | 1345.7ms/step | 14.9 samples/s +Step 31980/44000 | Loss: 2.9189 | LR: 8.94e-05 | 1343.5ms/step | 14.9 samples/s +Step 31990/44000 | Loss: 2.8858 | LR: 8.92e-05 | 1344.6ms/step | 14.9 samples/s +Step 32000/44000 | Loss: 2.9087 | LR: 8.91e-05 | 1341.8ms/step | 14.9 samples/s +Validation at step 32000: Loss = 2.9372, Perplexity = 18.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-32000 +Step 32010/44000 | Loss: 2.9296 | LR: 8.90e-05 | 3905.4ms/step | 5.1 samples/s +Step 32020/44000 | Loss: 2.9191 | LR: 8.88e-05 | 1344.9ms/step | 14.9 samples/s +Step 32030/44000 | Loss: 2.9278 | LR: 8.87e-05 | 1341.8ms/step | 14.9 samples/s +Step 32040/44000 | Loss: 2.9301 | LR: 8.86e-05 | 1342.5ms/step | 14.9 samples/s +Step 32050/44000 | Loss: 2.9045 | LR: 8.84e-05 | 1344.2ms/step | 14.9 samples/s +Step 32060/44000 | Loss: 2.8901 | LR: 8.83e-05 | 1345.4ms/step | 14.9 samples/s +Step 32070/44000 | Loss: 2.9146 | LR: 8.81e-05 | 1345.5ms/step | 14.9 samples/s +Step 32080/44000 | Loss: 2.8893 | LR: 8.80e-05 | 1346.9ms/step | 14.8 samples/s +Step 32090/44000 | Loss: 2.9236 | LR: 8.79e-05 | 1347.1ms/step | 14.8 samples/s +Step 32100/44000 | Loss: 2.9253 | LR: 8.77e-05 | 1346.7ms/step | 14.9 samples/s +Step 32110/44000 | Loss: 2.9070 | LR: 8.76e-05 | 1350.0ms/step | 14.8 samples/s +Step 32120/44000 | Loss: 2.9127 | LR: 8.74e-05 | 1346.5ms/step | 14.9 samples/s +Step 32130/44000 | Loss: 2.9068 | LR: 8.73e-05 | 1343.3ms/step | 14.9 samples/s +Step 32140/44000 | Loss: 2.9052 | LR: 8.72e-05 | 1344.4ms/step | 14.9 samples/s +Step 32150/44000 | Loss: 2.9218 | LR: 8.70e-05 | 1344.4ms/step | 14.9 samples/s +Step 32160/44000 | Loss: 2.8927 | LR: 8.69e-05 | 1343.7ms/step | 14.9 samples/s +Step 32170/44000 | Loss: 2.9159 | LR: 8.68e-05 | 1343.6ms/step | 14.9 samples/s +Step 32180/44000 | Loss: 2.9488 | LR: 8.66e-05 | 1342.0ms/step | 14.9 samples/s +Step 32190/44000 | Loss: 2.9210 | LR: 8.65e-05 | 1343.8ms/step | 14.9 samples/s +Step 32200/44000 | Loss: 2.9342 | LR: 8.63e-05 | 1347.0ms/step | 14.8 samples/s +Step 32210/44000 | Loss: 2.9392 | LR: 8.62e-05 | 1346.1ms/step | 14.9 samples/s +Step 32220/44000 | Loss: 2.8953 | LR: 8.61e-05 | 1346.9ms/step | 14.8 samples/s +Step 32230/44000 | Loss: 2.9235 | LR: 8.59e-05 | 1346.2ms/step | 14.9 samples/s +Step 32240/44000 | Loss: 2.9124 | LR: 8.58e-05 | 1344.5ms/step | 14.9 samples/s +Step 32250/44000 | Loss: 2.9328 | LR: 8.57e-05 | 1344.0ms/step | 14.9 samples/s +Validation at step 32250: Loss = 2.9366, Perplexity = 18.85 +Step 32260/44000 | Loss: 2.9448 | LR: 8.55e-05 | 3612.9ms/step | 5.5 samples/s +Step 32270/44000 | Loss: 2.9197 | LR: 8.54e-05 | 1346.8ms/step | 14.9 samples/s +Step 32280/44000 | Loss: 2.9406 | LR: 8.53e-05 | 1342.9ms/step | 14.9 samples/s +Step 32290/44000 | Loss: 2.9087 | LR: 8.51e-05 | 1345.0ms/step | 14.9 samples/s +Step 32300/44000 | Loss: 2.9044 | LR: 8.50e-05 | 1349.1ms/step | 14.8 samples/s +Step 32310/44000 | Loss: 2.9278 | LR: 8.48e-05 | 1347.2ms/step | 14.8 samples/s +Step 32320/44000 | Loss: 2.9468 | LR: 8.47e-05 | 1349.5ms/step | 14.8 samples/s +Step 32330/44000 | Loss: 2.9165 | LR: 8.46e-05 | 1346.9ms/step | 14.8 samples/s +Step 32340/44000 | Loss: 2.9124 | LR: 8.44e-05 | 1342.8ms/step | 14.9 samples/s +Step 32350/44000 | Loss: 2.9359 | LR: 8.43e-05 | 1348.3ms/step | 14.8 samples/s +Step 32360/44000 | Loss: 2.9150 | LR: 8.42e-05 | 1348.8ms/step | 14.8 samples/s +Step 32370/44000 | Loss: 2.9135 | LR: 8.40e-05 | 1346.7ms/step | 14.9 samples/s +Step 32380/44000 | Loss: 2.9133 | LR: 8.39e-05 | 1348.6ms/step | 14.8 samples/s +Step 32390/44000 | Loss: 2.9150 | LR: 8.38e-05 | 1351.0ms/step | 14.8 samples/s +Step 32400/44000 | Loss: 2.9228 | LR: 8.36e-05 | 1346.5ms/step | 14.9 samples/s +Step 32410/44000 | Loss: 2.9172 | LR: 8.35e-05 | 1349.6ms/step | 14.8 samples/s +Step 32420/44000 | Loss: 2.9146 | LR: 8.33e-05 | 1347.3ms/step | 14.8 samples/s +Step 32430/44000 | Loss: 2.9625 | LR: 8.32e-05 | 1343.9ms/step | 14.9 samples/s +Step 32440/44000 | Loss: 2.9290 | LR: 8.31e-05 | 1345.6ms/step | 14.9 samples/s +Step 32450/44000 | Loss: 2.9214 | LR: 8.29e-05 | 1345.0ms/step | 14.9 samples/s +Step 32460/44000 | Loss: 2.9423 | LR: 8.28e-05 | 1342.3ms/step | 14.9 samples/s +Step 32470/44000 | Loss: 2.8982 | LR: 8.27e-05 | 1343.2ms/step | 14.9 samples/s +Step 32480/44000 | Loss: 2.9312 | LR: 8.25e-05 | 1345.0ms/step | 14.9 samples/s +Step 32490/44000 | Loss: 2.9085 | LR: 8.24e-05 | 1345.7ms/step | 14.9 samples/s +Step 32500/44000 | Loss: 2.9257 | LR: 8.23e-05 | 1346.7ms/step | 14.9 samples/s +Validation at step 32500: Loss = 2.9363, Perplexity = 18.85 +Step 32510/44000 | Loss: 2.8929 | LR: 8.21e-05 | 3602.8ms/step | 5.6 samples/s +Step 32520/44000 | Loss: 2.9189 | LR: 8.20e-05 | 1346.7ms/step | 14.9 samples/s +Step 32530/44000 | Loss: 2.9496 | LR: 8.19e-05 | 1346.1ms/step | 14.9 samples/s +Step 32540/44000 | Loss: 2.9037 | LR: 8.17e-05 | 1340.2ms/step | 14.9 samples/s +Step 32550/44000 | Loss: 2.8752 | LR: 8.16e-05 | 1342.1ms/step | 14.9 samples/s +Step 32560/44000 | Loss: 2.9400 | LR: 8.15e-05 | 1341.5ms/step | 14.9 samples/s +Step 32570/44000 | Loss: 2.9219 | LR: 8.13e-05 | 1343.9ms/step | 14.9 samples/s +Step 32580/44000 | Loss: 2.9277 | LR: 8.12e-05 | 1347.8ms/step | 14.8 samples/s +Step 32590/44000 | Loss: 2.9523 | LR: 8.11e-05 | 1347.8ms/step | 14.8 samples/s +Step 32600/44000 | Loss: 2.9252 | LR: 8.09e-05 | 1349.3ms/step | 14.8 samples/s +Step 32610/44000 | Loss: 2.9287 | LR: 8.08e-05 | 1345.8ms/step | 14.9 samples/s +Step 32620/44000 | Loss: 2.9369 | LR: 8.07e-05 | 1345.4ms/step | 14.9 samples/s +Step 32630/44000 | Loss: 2.8970 | LR: 8.05e-05 | 1344.6ms/step | 14.9 samples/s +Step 32640/44000 | Loss: 2.9577 | LR: 8.04e-05 | 1342.0ms/step | 14.9 samples/s +Step 32650/44000 | Loss: 2.9351 | LR: 8.03e-05 | 1343.5ms/step | 14.9 samples/s +Step 32660/44000 | Loss: 2.9239 | LR: 8.01e-05 | 1347.1ms/step | 14.8 samples/s +Step 32670/44000 | Loss: 2.9603 | LR: 8.00e-05 | 1352.3ms/step | 14.8 samples/s +Step 32680/44000 | Loss: 2.9157 | LR: 7.99e-05 | 1347.9ms/step | 14.8 samples/s +Step 32690/44000 | Loss: 2.9284 | LR: 7.97e-05 | 1346.2ms/step | 14.9 samples/s +Step 32700/44000 | Loss: 2.9079 | LR: 7.96e-05 | 1345.9ms/step | 14.9 samples/s +Step 32710/44000 | Loss: 2.9349 | LR: 7.95e-05 | 1345.8ms/step | 14.9 samples/s +Step 32720/44000 | Loss: 2.9354 | LR: 7.93e-05 | 1347.2ms/step | 14.8 samples/s +Step 32730/44000 | Loss: 2.9127 | LR: 7.92e-05 | 1346.1ms/step | 14.9 samples/s +Step 32740/44000 | Loss: 2.9021 | LR: 7.91e-05 | 1348.0ms/step | 14.8 samples/s +Step 32750/44000 | Loss: 2.8954 | LR: 7.89e-05 | 1349.6ms/step | 14.8 samples/s +Validation at step 32750: Loss = 2.9360, Perplexity = 18.84 +Step 32760/44000 | Loss: 2.9297 | LR: 7.88e-05 | 3605.9ms/step | 5.5 samples/s +Step 32770/44000 | Loss: 2.9242 | LR: 7.87e-05 | 1349.6ms/step | 14.8 samples/s +Step 32780/44000 | Loss: 2.9488 | LR: 7.85e-05 | 1341.8ms/step | 14.9 samples/s +Step 32790/44000 | Loss: 2.9578 | LR: 7.84e-05 | 1342.7ms/step | 14.9 samples/s +Step 32800/44000 | Loss: 2.9121 | LR: 7.83e-05 | 1342.3ms/step | 14.9 samples/s +Step 32810/44000 | Loss: 2.9195 | LR: 7.81e-05 | 1346.4ms/step | 14.9 samples/s +Step 32820/44000 | Loss: 2.9393 | LR: 7.80e-05 | 1345.1ms/step | 14.9 samples/s +Step 32830/44000 | Loss: 2.9363 | LR: 7.79e-05 | 1342.4ms/step | 14.9 samples/s +Step 32840/44000 | Loss: 2.8872 | LR: 7.77e-05 | 1341.6ms/step | 14.9 samples/s +Step 32850/44000 | Loss: 2.9175 | LR: 7.76e-05 | 1342.4ms/step | 14.9 samples/s +Step 32860/44000 | Loss: 2.9161 | LR: 7.75e-05 | 1342.8ms/step | 14.9 samples/s +Step 32870/44000 | Loss: 2.9330 | LR: 7.73e-05 | 1342.5ms/step | 14.9 samples/s +Step 32880/44000 | Loss: 2.8931 | LR: 7.72e-05 | 1343.5ms/step | 14.9 samples/s +Step 32890/44000 | Loss: 2.9298 | LR: 7.71e-05 | 1342.6ms/step | 14.9 samples/s +Step 32900/44000 | Loss: 2.9259 | LR: 7.70e-05 | 1339.7ms/step | 14.9 samples/s +Step 32910/44000 | Loss: 2.9161 | LR: 7.68e-05 | 1342.6ms/step | 14.9 samples/s +Step 32920/44000 | Loss: 2.9403 | LR: 7.67e-05 | 1344.1ms/step | 14.9 samples/s +Step 32930/44000 | Loss: 2.9134 | LR: 7.66e-05 | 1345.6ms/step | 14.9 samples/s +Step 32940/44000 | Loss: 2.8984 | LR: 7.64e-05 | 1345.9ms/step | 14.9 samples/s +Step 32950/44000 | Loss: 2.9056 | LR: 7.63e-05 | 1345.7ms/step | 14.9 samples/s +Step 32960/44000 | Loss: 2.9140 | LR: 7.62e-05 | 1345.7ms/step | 14.9 samples/s +Step 32970/44000 | Loss: 2.9180 | LR: 7.60e-05 | 1344.6ms/step | 14.9 samples/s +Step 32980/44000 | Loss: 2.9205 | LR: 7.59e-05 | 1341.0ms/step | 14.9 samples/s +Step 32990/44000 | Loss: 2.9239 | LR: 7.58e-05 | 1342.3ms/step | 14.9 samples/s +Step 33000/44000 | Loss: 2.9448 | LR: 7.56e-05 | 1341.3ms/step | 14.9 samples/s +Validation at step 33000: Loss = 2.9355, Perplexity = 18.83 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-33000 +Step 33010/44000 | Loss: 2.9321 | LR: 7.55e-05 | 3915.9ms/step | 5.1 samples/s +Step 33020/44000 | Loss: 2.8970 | LR: 7.54e-05 | 1348.3ms/step | 14.8 samples/s +Step 33030/44000 | Loss: 2.9404 | LR: 7.53e-05 | 1344.6ms/step | 14.9 samples/s +Step 33040/44000 | Loss: 2.9248 | LR: 7.51e-05 | 1345.1ms/step | 14.9 samples/s +Step 33050/44000 | Loss: 2.9336 | LR: 7.50e-05 | 1346.8ms/step | 14.8 samples/s +Step 33060/44000 | Loss: 2.9028 | LR: 7.49e-05 | 1350.2ms/step | 14.8 samples/s +Step 33070/44000 | Loss: 2.9343 | LR: 7.47e-05 | 1347.6ms/step | 14.8 samples/s +Step 33080/44000 | Loss: 2.9335 | LR: 7.46e-05 | 1355.1ms/step | 14.8 samples/s +Step 33090/44000 | Loss: 2.9230 | LR: 7.45e-05 | 1354.7ms/step | 14.8 samples/s +Step 33100/44000 | Loss: 2.8963 | LR: 7.44e-05 | 1349.7ms/step | 14.8 samples/s +Step 33110/44000 | Loss: 2.8738 | LR: 7.42e-05 | 1346.8ms/step | 14.8 samples/s +Step 33120/44000 | Loss: 2.9070 | LR: 7.41e-05 | 1345.7ms/step | 14.9 samples/s +Step 33130/44000 | Loss: 2.9241 | LR: 7.40e-05 | 1349.0ms/step | 14.8 samples/s +Step 33140/44000 | Loss: 2.9414 | LR: 7.38e-05 | 1347.8ms/step | 14.8 samples/s +Step 33150/44000 | Loss: 2.9332 | LR: 7.37e-05 | 1346.2ms/step | 14.9 samples/s +Step 33160/44000 | Loss: 2.9088 | LR: 7.36e-05 | 1342.6ms/step | 14.9 samples/s +Step 33170/44000 | Loss: 2.9151 | LR: 7.34e-05 | 1342.8ms/step | 14.9 samples/s +Step 33180/44000 | Loss: 2.9395 | LR: 7.33e-05 | 1346.6ms/step | 14.9 samples/s +Step 33190/44000 | Loss: 2.9387 | LR: 7.32e-05 | 1345.7ms/step | 14.9 samples/s +Step 33200/44000 | Loss: 2.9281 | LR: 7.31e-05 | 1343.6ms/step | 14.9 samples/s +Step 33210/44000 | Loss: 2.9159 | LR: 7.29e-05 | 1345.2ms/step | 14.9 samples/s +Step 33220/44000 | Loss: 2.9327 | LR: 7.28e-05 | 1352.3ms/step | 14.8 samples/s +Step 33230/44000 | Loss: 2.9153 | LR: 7.27e-05 | 1345.3ms/step | 14.9 samples/s +Step 33240/44000 | Loss: 2.9096 | LR: 7.26e-05 | 1343.8ms/step | 14.9 samples/s +Step 33250/44000 | Loss: 2.9196 | LR: 7.24e-05 | 1342.0ms/step | 14.9 samples/s +Validation at step 33250: Loss = 2.9351, Perplexity = 18.82 +Step 33260/44000 | Loss: 2.9329 | LR: 7.23e-05 | 3610.9ms/step | 5.5 samples/s +Step 33270/44000 | Loss: 2.9127 | LR: 7.22e-05 | 1348.5ms/step | 14.8 samples/s +Step 33280/44000 | Loss: 2.9309 | LR: 7.20e-05 | 1347.8ms/step | 14.8 samples/s +Step 33290/44000 | Loss: 2.8805 | LR: 7.19e-05 | 1348.5ms/step | 14.8 samples/s +Step 33300/44000 | Loss: 2.9225 | LR: 7.18e-05 | 1345.9ms/step | 14.9 samples/s +Step 33310/44000 | Loss: 2.9378 | LR: 7.17e-05 | 1346.8ms/step | 14.8 samples/s +Step 33320/44000 | Loss: 2.9191 | LR: 7.15e-05 | 1348.1ms/step | 14.8 samples/s +Step 33330/44000 | Loss: 2.9097 | LR: 7.14e-05 | 1348.3ms/step | 14.8 samples/s +Step 33340/44000 | Loss: 2.9114 | LR: 7.13e-05 | 1347.4ms/step | 14.8 samples/s +Step 33350/44000 | Loss: 2.9211 | LR: 7.11e-05 | 1346.3ms/step | 14.9 samples/s +Step 33360/44000 | Loss: 2.9322 | LR: 7.10e-05 | 1348.3ms/step | 14.8 samples/s +Step 33370/44000 | Loss: 2.8923 | LR: 7.09e-05 | 1347.5ms/step | 14.8 samples/s +Step 33380/44000 | Loss: 2.9089 | LR: 7.08e-05 | 1344.0ms/step | 14.9 samples/s +Step 33390/44000 | Loss: 2.9393 | LR: 7.06e-05 | 1340.2ms/step | 14.9 samples/s +Step 33400/44000 | Loss: 2.8869 | LR: 7.05e-05 | 1344.3ms/step | 14.9 samples/s +Step 33410/44000 | Loss: 2.9337 | LR: 7.04e-05 | 1347.0ms/step | 14.8 samples/s +Step 33420/44000 | Loss: 2.9127 | LR: 7.03e-05 | 1347.7ms/step | 14.8 samples/s +Step 33430/44000 | Loss: 2.9093 | LR: 7.01e-05 | 1346.9ms/step | 14.8 samples/s +Step 33440/44000 | Loss: 2.9376 | LR: 7.00e-05 | 1345.9ms/step | 14.9 samples/s +Step 33450/44000 | Loss: 2.9324 | LR: 6.99e-05 | 1341.6ms/step | 14.9 samples/s +Step 33460/44000 | Loss: 2.9253 | LR: 6.98e-05 | 1344.3ms/step | 14.9 samples/s +Step 33470/44000 | Loss: 2.9163 | LR: 6.96e-05 | 1343.5ms/step | 14.9 samples/s +Step 33480/44000 | Loss: 2.9025 | LR: 6.95e-05 | 1347.6ms/step | 14.8 samples/s +Step 33490/44000 | Loss: 2.9349 | LR: 6.94e-05 | 1347.7ms/step | 14.8 samples/s +Step 33500/44000 | Loss: 2.8924 | LR: 6.93e-05 | 1347.4ms/step | 14.8 samples/s +Validation at step 33500: Loss = 2.9347, Perplexity = 18.82 +Step 33510/44000 | Loss: 2.8986 | LR: 6.91e-05 | 3609.9ms/step | 5.5 samples/s +Step 33520/44000 | Loss: 2.9370 | LR: 6.90e-05 | 1341.0ms/step | 14.9 samples/s +Step 33530/44000 | Loss: 2.9002 | LR: 6.89e-05 | 1336.9ms/step | 15.0 samples/s +Step 33540/44000 | Loss: 2.9076 | LR: 6.88e-05 | 1340.3ms/step | 14.9 samples/s +Step 33550/44000 | Loss: 2.9152 | LR: 6.86e-05 | 1346.3ms/step | 14.9 samples/s +Step 33560/44000 | Loss: 2.8957 | LR: 6.85e-05 | 1344.9ms/step | 14.9 samples/s +Step 33570/44000 | Loss: 2.8905 | LR: 6.84e-05 | 1341.9ms/step | 14.9 samples/s +Step 33580/44000 | Loss: 2.9357 | LR: 6.83e-05 | 1339.4ms/step | 14.9 samples/s +Step 33590/44000 | Loss: 2.9342 | LR: 6.81e-05 | 1338.7ms/step | 14.9 samples/s +Step 33600/44000 | Loss: 2.9399 | LR: 6.80e-05 | 1341.6ms/step | 14.9 samples/s +Step 33610/44000 | Loss: 2.8945 | LR: 6.79e-05 | 1341.1ms/step | 14.9 samples/s +Step 33620/44000 | Loss: 2.9108 | LR: 6.78e-05 | 1342.8ms/step | 14.9 samples/s +Step 33630/44000 | Loss: 2.8992 | LR: 6.76e-05 | 1342.0ms/step | 14.9 samples/s +Step 33640/44000 | Loss: 2.9131 | LR: 6.75e-05 | 1345.3ms/step | 14.9 samples/s +Step 33650/44000 | Loss: 2.9207 | LR: 6.74e-05 | 1342.6ms/step | 14.9 samples/s +Step 33660/44000 | Loss: 2.9248 | LR: 6.73e-05 | 1342.6ms/step | 14.9 samples/s +Step 33670/44000 | Loss: 2.9340 | LR: 6.71e-05 | 1339.2ms/step | 14.9 samples/s +Step 33680/44000 | Loss: 2.9194 | LR: 6.70e-05 | 1339.7ms/step | 14.9 samples/s +Step 33690/44000 | Loss: 2.9149 | LR: 6.69e-05 | 1346.6ms/step | 14.9 samples/s +Step 33700/44000 | Loss: 2.9311 | LR: 6.68e-05 | 1346.4ms/step | 14.9 samples/s +Step 33710/44000 | Loss: 2.9474 | LR: 6.66e-05 | 1345.8ms/step | 14.9 samples/s +Step 33720/44000 | Loss: 2.9253 | LR: 6.65e-05 | 1344.8ms/step | 14.9 samples/s +Step 33730/44000 | Loss: 2.9074 | LR: 6.64e-05 | 1342.2ms/step | 14.9 samples/s +Step 33740/44000 | Loss: 2.9223 | LR: 6.63e-05 | 1343.6ms/step | 14.9 samples/s +Step 33750/44000 | Loss: 2.9274 | LR: 6.61e-05 | 1341.7ms/step | 14.9 samples/s +Validation at step 33750: Loss = 2.9344, Perplexity = 18.81 +Step 33760/44000 | Loss: 2.8917 | LR: 6.60e-05 | 3600.4ms/step | 5.6 samples/s +Step 33770/44000 | Loss: 2.9417 | LR: 6.59e-05 | 1345.3ms/step | 14.9 samples/s +Step 33780/44000 | Loss: 2.9413 | LR: 6.58e-05 | 1343.2ms/step | 14.9 samples/s +Step 33790/44000 | Loss: 2.8969 | LR: 6.57e-05 | 1345.6ms/step | 14.9 samples/s +Step 33800/44000 | Loss: 2.9289 | LR: 6.55e-05 | 1347.0ms/step | 14.8 samples/s +Step 33810/44000 | Loss: 2.9469 | LR: 6.54e-05 | 1344.9ms/step | 14.9 samples/s +Step 33820/44000 | Loss: 2.9190 | LR: 6.53e-05 | 1346.5ms/step | 14.9 samples/s +Step 33830/44000 | Loss: 2.9395 | LR: 6.52e-05 | 1345.4ms/step | 14.9 samples/s +Step 33840/44000 | Loss: 2.9215 | LR: 6.50e-05 | 1345.4ms/step | 14.9 samples/s +Step 33850/44000 | Loss: 2.9309 | LR: 6.49e-05 | 1345.3ms/step | 14.9 samples/s +Step 33860/44000 | Loss: 2.9143 | LR: 6.48e-05 | 1346.6ms/step | 14.9 samples/s +Step 33870/44000 | Loss: 2.9243 | LR: 6.47e-05 | 1344.0ms/step | 14.9 samples/s +Step 33880/44000 | Loss: 2.9550 | LR: 6.46e-05 | 1342.8ms/step | 14.9 samples/s +Step 33890/44000 | Loss: 2.9118 | LR: 6.44e-05 | 1344.4ms/step | 14.9 samples/s +Step 33900/44000 | Loss: 2.9371 | LR: 6.43e-05 | 1345.5ms/step | 14.9 samples/s +Step 33910/44000 | Loss: 2.9264 | LR: 6.42e-05 | 1346.0ms/step | 14.9 samples/s +Step 33920/44000 | Loss: 2.9194 | LR: 6.41e-05 | 1344.4ms/step | 14.9 samples/s +Step 33930/44000 | Loss: 2.8794 | LR: 6.40e-05 | 1342.2ms/step | 14.9 samples/s +Step 33940/44000 | Loss: 2.9161 | LR: 6.38e-05 | 1342.5ms/step | 14.9 samples/s +Step 33950/44000 | Loss: 2.9110 | LR: 6.37e-05 | 1342.3ms/step | 14.9 samples/s +Step 33960/44000 | Loss: 2.9010 | LR: 6.36e-05 | 1344.4ms/step | 14.9 samples/s +Step 33970/44000 | Loss: 2.9093 | LR: 6.35e-05 | 1349.7ms/step | 14.8 samples/s +Step 33980/44000 | Loss: 2.9420 | LR: 6.33e-05 | 1352.8ms/step | 14.8 samples/s +Step 33990/44000 | Loss: 2.9351 | LR: 6.32e-05 | 1348.3ms/step | 14.8 samples/s +Step 34000/44000 | Loss: 2.9386 | LR: 6.31e-05 | 1346.3ms/step | 14.9 samples/s +Validation at step 34000: Loss = 2.9338, Perplexity = 18.80 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-34000 +Step 34010/44000 | Loss: 2.9175 | LR: 6.30e-05 | 3900.5ms/step | 5.1 samples/s +Step 34020/44000 | Loss: 2.9110 | LR: 6.29e-05 | 1345.5ms/step | 14.9 samples/s +Step 34030/44000 | Loss: 2.9596 | LR: 6.27e-05 | 1344.2ms/step | 14.9 samples/s +Step 34040/44000 | Loss: 2.9111 | LR: 6.26e-05 | 1345.8ms/step | 14.9 samples/s +Step 34050/44000 | Loss: 2.9116 | LR: 6.25e-05 | 1345.1ms/step | 14.9 samples/s +Step 34060/44000 | Loss: 2.9187 | LR: 6.24e-05 | 1344.9ms/step | 14.9 samples/s +Step 34070/44000 | Loss: 2.9417 | LR: 6.23e-05 | 1345.3ms/step | 14.9 samples/s +Step 34080/44000 | Loss: 2.9240 | LR: 6.21e-05 | 1348.0ms/step | 14.8 samples/s +Step 34090/44000 | Loss: 2.9061 | LR: 6.20e-05 | 1345.8ms/step | 14.9 samples/s +Step 34100/44000 | Loss: 2.9217 | LR: 6.19e-05 | 1345.1ms/step | 14.9 samples/s +Step 34110/44000 | Loss: 2.8906 | LR: 6.18e-05 | 1347.6ms/step | 14.8 samples/s +Step 34120/44000 | Loss: 2.9141 | LR: 6.17e-05 | 1349.4ms/step | 14.8 samples/s +Step 34130/44000 | Loss: 2.9169 | LR: 6.15e-05 | 1348.4ms/step | 14.8 samples/s +Step 34140/44000 | Loss: 2.9286 | LR: 6.14e-05 | 1349.2ms/step | 14.8 samples/s +Step 34150/44000 | Loss: 2.9229 | LR: 6.13e-05 | 1351.9ms/step | 14.8 samples/s +Step 34160/44000 | Loss: 2.9506 | LR: 6.12e-05 | 1342.9ms/step | 14.9 samples/s +Step 34170/44000 | Loss: 2.9278 | LR: 6.11e-05 | 1343.5ms/step | 14.9 samples/s +Step 34180/44000 | Loss: 2.9059 | LR: 6.09e-05 | 1346.6ms/step | 14.9 samples/s +Step 34190/44000 | Loss: 2.9271 | LR: 6.08e-05 | 1348.0ms/step | 14.8 samples/s +Step 34200/44000 | Loss: 2.9463 | LR: 6.07e-05 | 1347.9ms/step | 14.8 samples/s +Step 34210/44000 | Loss: 2.9270 | LR: 6.06e-05 | 1346.2ms/step | 14.9 samples/s +Step 34220/44000 | Loss: 2.9187 | LR: 6.05e-05 | 1345.8ms/step | 14.9 samples/s +Step 34230/44000 | Loss: 2.9345 | LR: 6.04e-05 | 1346.3ms/step | 14.9 samples/s +Step 34240/44000 | Loss: 2.9522 | LR: 6.02e-05 | 1344.7ms/step | 14.9 samples/s +Step 34250/44000 | Loss: 2.9302 | LR: 6.01e-05 | 1346.4ms/step | 14.9 samples/s +Validation at step 34250: Loss = 2.9338, Perplexity = 18.80 +Step 34260/44000 | Loss: 2.9266 | LR: 6.00e-05 | 3603.9ms/step | 5.5 samples/s +Step 34270/44000 | Loss: 2.9030 | LR: 5.99e-05 | 1343.1ms/step | 14.9 samples/s +Step 34280/44000 | Loss: 2.9438 | LR: 5.98e-05 | 1342.2ms/step | 14.9 samples/s +Step 34290/44000 | Loss: 2.9042 | LR: 5.96e-05 | 1343.0ms/step | 14.9 samples/s +Step 34300/44000 | Loss: 2.9034 | LR: 5.95e-05 | 1343.9ms/step | 14.9 samples/s +Step 34310/44000 | Loss: 2.9193 | LR: 5.94e-05 | 1344.3ms/step | 14.9 samples/s +Step 34320/44000 | Loss: 2.9667 | LR: 5.93e-05 | 1345.6ms/step | 14.9 samples/s +Step 34330/44000 | Loss: 2.8892 | LR: 5.92e-05 | 1346.0ms/step | 14.9 samples/s +Step 34340/44000 | Loss: 2.8914 | LR: 5.91e-05 | 1343.6ms/step | 14.9 samples/s +Step 34350/44000 | Loss: 2.9120 | LR: 5.89e-05 | 1345.3ms/step | 14.9 samples/s +Step 34360/44000 | Loss: 2.9235 | LR: 5.88e-05 | 1341.8ms/step | 14.9 samples/s +Step 34370/44000 | Loss: 2.9119 | LR: 5.87e-05 | 1347.3ms/step | 14.8 samples/s +Step 34380/44000 | Loss: 2.9109 | LR: 5.86e-05 | 1347.2ms/step | 14.8 samples/s +Step 34390/44000 | Loss: 2.9251 | LR: 5.85e-05 | 1349.6ms/step | 14.8 samples/s +Step 34400/44000 | Loss: 2.9235 | LR: 5.84e-05 | 1345.4ms/step | 14.9 samples/s +Step 34410/44000 | Loss: 2.9393 | LR: 5.82e-05 | 1342.9ms/step | 14.9 samples/s +Step 34420/44000 | Loss: 2.9202 | LR: 5.81e-05 | 1346.8ms/step | 14.9 samples/s +Step 34430/44000 | Loss: 2.9155 | LR: 5.80e-05 | 1345.0ms/step | 14.9 samples/s +Step 34440/44000 | Loss: 2.9275 | LR: 5.79e-05 | 1346.0ms/step | 14.9 samples/s +Step 34450/44000 | Loss: 2.9083 | LR: 5.78e-05 | 1344.8ms/step | 14.9 samples/s +Step 34460/44000 | Loss: 2.9361 | LR: 5.77e-05 | 1346.0ms/step | 14.9 samples/s +Step 34470/44000 | Loss: 2.9187 | LR: 5.75e-05 | 1346.6ms/step | 14.9 samples/s +Step 34480/44000 | Loss: 2.9251 | LR: 5.74e-05 | 1346.0ms/step | 14.9 samples/s +Step 34490/44000 | Loss: 2.8883 | LR: 5.73e-05 | 1344.5ms/step | 14.9 samples/s +Step 34500/44000 | Loss: 2.9358 | LR: 5.72e-05 | 1344.9ms/step | 14.9 samples/s +Validation at step 34500: Loss = 2.9332, Perplexity = 18.79 +Step 34510/44000 | Loss: 2.9188 | LR: 5.71e-05 | 3598.2ms/step | 5.6 samples/s +Step 34520/44000 | Loss: 2.9120 | LR: 5.70e-05 | 1342.6ms/step | 14.9 samples/s +Step 34530/44000 | Loss: 2.9033 | LR: 5.69e-05 | 1343.0ms/step | 14.9 samples/s +Step 34540/44000 | Loss: 2.9225 | LR: 5.67e-05 | 1339.0ms/step | 14.9 samples/s +Step 34550/44000 | Loss: 2.9407 | LR: 5.66e-05 | 1338.2ms/step | 14.9 samples/s +Step 34560/44000 | Loss: 2.8897 | LR: 5.65e-05 | 1343.3ms/step | 14.9 samples/s +Step 34570/44000 | Loss: 2.9058 | LR: 5.64e-05 | 1343.4ms/step | 14.9 samples/s +Step 34580/44000 | Loss: 2.8993 | LR: 5.63e-05 | 1343.1ms/step | 14.9 samples/s +Step 34590/44000 | Loss: 2.9116 | LR: 5.62e-05 | 1341.7ms/step | 14.9 samples/s +Step 34600/44000 | Loss: 2.9237 | LR: 5.60e-05 | 1341.9ms/step | 14.9 samples/s +Step 34610/44000 | Loss: 2.9249 | LR: 5.59e-05 | 1341.1ms/step | 14.9 samples/s +Step 34620/44000 | Loss: 2.9196 | LR: 5.58e-05 | 1339.4ms/step | 14.9 samples/s +Step 34630/44000 | Loss: 2.9073 | LR: 5.57e-05 | 1344.0ms/step | 14.9 samples/s +Step 34640/44000 | Loss: 2.9155 | LR: 5.56e-05 | 1342.1ms/step | 14.9 samples/s +Step 34650/44000 | Loss: 2.9299 | LR: 5.55e-05 | 1341.6ms/step | 14.9 samples/s +Step 34660/44000 | Loss: 2.9365 | LR: 5.54e-05 | 1341.8ms/step | 14.9 samples/s +Step 34670/44000 | Loss: 2.8879 | LR: 5.52e-05 | 1341.6ms/step | 14.9 samples/s +Step 34680/44000 | Loss: 2.9189 | LR: 5.51e-05 | 1342.2ms/step | 14.9 samples/s +Step 34690/44000 | Loss: 2.9041 | LR: 5.50e-05 | 1346.0ms/step | 14.9 samples/s +Step 34700/44000 | Loss: 2.8949 | LR: 5.49e-05 | 1344.2ms/step | 14.9 samples/s +Step 34710/44000 | Loss: 2.9320 | LR: 5.48e-05 | 1345.7ms/step | 14.9 samples/s +Step 34720/44000 | Loss: 2.9060 | LR: 5.47e-05 | 1344.4ms/step | 14.9 samples/s +Step 34730/44000 | Loss: 2.9319 | LR: 5.46e-05 | 1344.6ms/step | 14.9 samples/s +Step 34740/44000 | Loss: 2.9067 | LR: 5.45e-05 | 1345.5ms/step | 14.9 samples/s +Step 34750/44000 | Loss: 2.9143 | LR: 5.43e-05 | 1345.6ms/step | 14.9 samples/s +Validation at step 34750: Loss = 2.9332, Perplexity = 18.79 +Step 34760/44000 | Loss: 2.9020 | LR: 5.42e-05 | 3601.1ms/step | 5.6 samples/s +Step 34770/44000 | Loss: 2.9073 | LR: 5.41e-05 | 1347.2ms/step | 14.8 samples/s +Step 34780/44000 | Loss: 2.9465 | LR: 5.40e-05 | 1346.1ms/step | 14.9 samples/s +Step 34790/44000 | Loss: 2.9423 | LR: 5.39e-05 | 1343.6ms/step | 14.9 samples/s +Step 34800/44000 | Loss: 2.9209 | LR: 5.38e-05 | 1345.0ms/step | 14.9 samples/s +Step 34810/44000 | Loss: 2.9387 | LR: 5.37e-05 | 1343.6ms/step | 14.9 samples/s +Step 34820/44000 | Loss: 2.9176 | LR: 5.36e-05 | 1345.8ms/step | 14.9 samples/s +Step 34830/44000 | Loss: 2.9177 | LR: 5.34e-05 | 1343.5ms/step | 14.9 samples/s +Step 34840/44000 | Loss: 2.9265 | LR: 5.33e-05 | 1340.6ms/step | 14.9 samples/s +Step 34850/44000 | Loss: 2.9364 | LR: 5.32e-05 | 1339.8ms/step | 14.9 samples/s +Step 34860/44000 | Loss: 2.9271 | LR: 5.31e-05 | 1338.4ms/step | 14.9 samples/s +Step 34870/44000 | Loss: 2.9066 | LR: 5.30e-05 | 1344.2ms/step | 14.9 samples/s +Step 34880/44000 | Loss: 2.9142 | LR: 5.29e-05 | 1341.4ms/step | 14.9 samples/s +Step 34890/44000 | Loss: 2.9101 | LR: 5.28e-05 | 1344.9ms/step | 14.9 samples/s +Step 34900/44000 | Loss: 2.9279 | LR: 5.27e-05 | 1345.1ms/step | 14.9 samples/s +Step 34910/44000 | Loss: 2.9251 | LR: 5.25e-05 | 1346.1ms/step | 14.9 samples/s +Step 34920/44000 | Loss: 2.9100 | LR: 5.24e-05 | 1344.9ms/step | 14.9 samples/s +Step 34930/44000 | Loss: 2.8991 | LR: 5.23e-05 | 1345.5ms/step | 14.9 samples/s +Step 34940/44000 | Loss: 2.8731 | LR: 5.22e-05 | 1343.8ms/step | 14.9 samples/s +Step 34950/44000 | Loss: 2.9304 | LR: 5.21e-05 | 1345.7ms/step | 14.9 samples/s +Step 34960/44000 | Loss: 2.9219 | LR: 5.20e-05 | 1344.1ms/step | 14.9 samples/s +Step 34970/44000 | Loss: 2.9016 | LR: 5.19e-05 | 1340.1ms/step | 14.9 samples/s +Step 34980/44000 | Loss: 2.9258 | LR: 5.18e-05 | 1342.9ms/step | 14.9 samples/s +Step 34990/44000 | Loss: 2.9277 | LR: 5.17e-05 | 1344.9ms/step | 14.9 samples/s +Step 35000/44000 | Loss: 2.8965 | LR: 5.15e-05 | 1344.6ms/step | 14.9 samples/s +Validation at step 35000: Loss = 2.9328, Perplexity = 18.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-35000 +Step 35010/44000 | Loss: 2.9201 | LR: 5.14e-05 | 3896.0ms/step | 5.1 samples/s +Step 35020/44000 | Loss: 2.8889 | LR: 5.13e-05 | 1344.9ms/step | 14.9 samples/s +Step 35030/44000 | Loss: 2.8978 | LR: 5.12e-05 | 1347.0ms/step | 14.8 samples/s +Step 35040/44000 | Loss: 2.9088 | LR: 5.11e-05 | 1346.4ms/step | 14.9 samples/s +Step 35050/44000 | Loss: 2.9003 | LR: 5.10e-05 | 1345.7ms/step | 14.9 samples/s +Step 35060/44000 | Loss: 2.9098 | LR: 5.09e-05 | 1342.1ms/step | 14.9 samples/s +Step 35070/44000 | Loss: 2.8782 | LR: 5.08e-05 | 1345.1ms/step | 14.9 samples/s +Step 35080/44000 | Loss: 2.9149 | LR: 5.07e-05 | 1346.8ms/step | 14.8 samples/s +Step 35090/44000 | Loss: 2.9014 | LR: 5.06e-05 | 1347.0ms/step | 14.8 samples/s +Step 35100/44000 | Loss: 2.9443 | LR: 5.04e-05 | 1347.3ms/step | 14.8 samples/s +Step 35110/44000 | Loss: 2.9076 | LR: 5.03e-05 | 1346.0ms/step | 14.9 samples/s +Step 35120/44000 | Loss: 2.9223 | LR: 5.02e-05 | 1346.2ms/step | 14.9 samples/s +Step 35130/44000 | Loss: 2.9212 | LR: 5.01e-05 | 1345.4ms/step | 14.9 samples/s +Step 35140/44000 | Loss: 2.8878 | LR: 5.00e-05 | 1346.2ms/step | 14.9 samples/s +Step 35150/44000 | Loss: 2.9286 | LR: 4.99e-05 | 1344.7ms/step | 14.9 samples/s +Step 35160/44000 | Loss: 2.9508 | LR: 4.98e-05 | 1344.4ms/step | 14.9 samples/s +Step 35170/44000 | Loss: 2.9272 | LR: 4.97e-05 | 1345.4ms/step | 14.9 samples/s +Step 35180/44000 | Loss: 2.9336 | LR: 4.96e-05 | 1346.0ms/step | 14.9 samples/s +Step 35190/44000 | Loss: 2.9033 | LR: 4.95e-05 | 1345.8ms/step | 14.9 samples/s +Step 35200/44000 | Loss: 2.9338 | LR: 4.94e-05 | 1346.7ms/step | 14.9 samples/s +Step 35210/44000 | Loss: 2.9293 | LR: 4.93e-05 | 1345.0ms/step | 14.9 samples/s +Step 35220/44000 | Loss: 2.9080 | LR: 4.91e-05 | 1345.9ms/step | 14.9 samples/s +Step 35230/44000 | Loss: 2.9148 | LR: 4.90e-05 | 1350.3ms/step | 14.8 samples/s +Step 35240/44000 | Loss: 2.9050 | LR: 4.89e-05 | 1347.4ms/step | 14.8 samples/s +Step 35250/44000 | Loss: 2.9515 | LR: 4.88e-05 | 1347.0ms/step | 14.8 samples/s +Validation at step 35250: Loss = 2.9326, Perplexity = 18.78 +Step 35260/44000 | Loss: 2.9070 | LR: 4.87e-05 | 3608.4ms/step | 5.5 samples/s +Step 35270/44000 | Loss: 2.9387 | LR: 4.86e-05 | 1346.6ms/step | 14.9 samples/s +Step 35280/44000 | Loss: 2.8953 | LR: 4.85e-05 | 1345.1ms/step | 14.9 samples/s +Step 35290/44000 | Loss: 2.9039 | LR: 4.84e-05 | 1346.3ms/step | 14.9 samples/s +Step 35300/44000 | Loss: 2.9055 | LR: 4.83e-05 | 1343.6ms/step | 14.9 samples/s +Step 35310/44000 | Loss: 2.9258 | LR: 4.82e-05 | 1339.9ms/step | 14.9 samples/s +Step 35320/44000 | Loss: 2.8887 | LR: 4.81e-05 | 1343.5ms/step | 14.9 samples/s +Step 35330/44000 | Loss: 2.9232 | LR: 4.80e-05 | 1346.9ms/step | 14.8 samples/s +Step 35340/44000 | Loss: 2.9188 | LR: 4.79e-05 | 1350.3ms/step | 14.8 samples/s +Step 35350/44000 | Loss: 2.9447 | LR: 4.77e-05 | 1351.7ms/step | 14.8 samples/s +Step 35360/44000 | Loss: 2.9014 | LR: 4.76e-05 | 1348.6ms/step | 14.8 samples/s +Step 35370/44000 | Loss: 2.9114 | LR: 4.75e-05 | 1344.9ms/step | 14.9 samples/s +Step 35380/44000 | Loss: 2.9003 | LR: 4.74e-05 | 1343.8ms/step | 14.9 samples/s +Step 35390/44000 | Loss: 2.9061 | LR: 4.73e-05 | 1346.3ms/step | 14.9 samples/s +Step 35400/44000 | Loss: 2.8942 | LR: 4.72e-05 | 1347.9ms/step | 14.8 samples/s +Step 35410/44000 | Loss: 2.9283 | LR: 4.71e-05 | 1347.0ms/step | 14.8 samples/s +Step 35420/44000 | Loss: 2.9007 | LR: 4.70e-05 | 1344.1ms/step | 14.9 samples/s +Step 35430/44000 | Loss: 2.9582 | LR: 4.69e-05 | 1342.2ms/step | 14.9 samples/s +Step 35440/44000 | Loss: 2.9373 | LR: 4.68e-05 | 1343.5ms/step | 14.9 samples/s +Step 35450/44000 | Loss: 2.9210 | LR: 4.67e-05 | 1349.4ms/step | 14.8 samples/s +Step 35460/44000 | Loss: 2.9397 | LR: 4.66e-05 | 1348.9ms/step | 14.8 samples/s +Step 35470/44000 | Loss: 2.9120 | LR: 4.65e-05 | 1343.9ms/step | 14.9 samples/s +Step 35480/44000 | Loss: 2.9202 | LR: 4.64e-05 | 1344.7ms/step | 14.9 samples/s +Step 35490/44000 | Loss: 2.9264 | LR: 4.63e-05 | 1345.5ms/step | 14.9 samples/s +Step 35500/44000 | Loss: 2.8997 | LR: 4.62e-05 | 1345.7ms/step | 14.9 samples/s +Validation at step 35500: Loss = 2.9323, Perplexity = 18.77 +Step 35510/44000 | Loss: 2.9107 | LR: 4.61e-05 | 3608.6ms/step | 5.5 samples/s +Step 35520/44000 | Loss: 2.9204 | LR: 4.59e-05 | 1345.9ms/step | 14.9 samples/s +Step 35530/44000 | Loss: 2.9199 | LR: 4.58e-05 | 1346.3ms/step | 14.9 samples/s +Step 35540/44000 | Loss: 2.9374 | LR: 4.57e-05 | 1339.0ms/step | 14.9 samples/s +Step 35550/44000 | Loss: 2.9389 | LR: 4.56e-05 | 1344.7ms/step | 14.9 samples/s +Step 35560/44000 | Loss: 2.9309 | LR: 4.55e-05 | 1346.3ms/step | 14.9 samples/s +Step 35570/44000 | Loss: 2.9391 | LR: 4.54e-05 | 1345.5ms/step | 14.9 samples/s +Step 35580/44000 | Loss: 2.9223 | LR: 4.53e-05 | 1346.7ms/step | 14.9 samples/s +Step 35590/44000 | Loss: 2.9318 | LR: 4.52e-05 | 1344.4ms/step | 14.9 samples/s +Step 35600/44000 | Loss: 2.8968 | LR: 4.51e-05 | 1345.6ms/step | 14.9 samples/s +Step 35610/44000 | Loss: 2.9349 | LR: 4.50e-05 | 1347.4ms/step | 14.8 samples/s +Step 35620/44000 | Loss: 2.9115 | LR: 4.49e-05 | 1344.5ms/step | 14.9 samples/s +Step 35630/44000 | Loss: 2.9225 | LR: 4.48e-05 | 1346.1ms/step | 14.9 samples/s +Step 35640/44000 | Loss: 2.9004 | LR: 4.47e-05 | 1346.1ms/step | 14.9 samples/s +Step 35650/44000 | Loss: 2.9079 | LR: 4.46e-05 | 1346.1ms/step | 14.9 samples/s +Step 35660/44000 | Loss: 2.9230 | LR: 4.45e-05 | 1346.9ms/step | 14.8 samples/s +Step 35670/44000 | Loss: 2.9115 | LR: 4.44e-05 | 1347.5ms/step | 14.8 samples/s +Step 35680/44000 | Loss: 2.9172 | LR: 4.43e-05 | 1345.5ms/step | 14.9 samples/s +Step 35690/44000 | Loss: 2.9276 | LR: 4.42e-05 | 1347.3ms/step | 14.8 samples/s +Step 35700/44000 | Loss: 2.9031 | LR: 4.41e-05 | 1349.5ms/step | 14.8 samples/s +Step 35710/44000 | Loss: 2.9196 | LR: 4.40e-05 | 1348.1ms/step | 14.8 samples/s +Step 35720/44000 | Loss: 2.9024 | LR: 4.39e-05 | 1346.2ms/step | 14.9 samples/s +Step 35730/44000 | Loss: 2.8920 | LR: 4.38e-05 | 1345.9ms/step | 14.9 samples/s +Step 35740/44000 | Loss: 2.9243 | LR: 4.37e-05 | 1344.8ms/step | 14.9 samples/s +Step 35750/44000 | Loss: 2.9036 | LR: 4.36e-05 | 1345.0ms/step | 14.9 samples/s +Validation at step 35750: Loss = 2.9323, Perplexity = 18.77 +Step 35760/44000 | Loss: 2.9343 | LR: 4.35e-05 | 3608.0ms/step | 5.5 samples/s +Step 35770/44000 | Loss: 2.9098 | LR: 4.34e-05 | 1341.9ms/step | 14.9 samples/s +Step 35780/44000 | Loss: 2.8994 | LR: 4.33e-05 | 1344.7ms/step | 14.9 samples/s +Step 35790/44000 | Loss: 2.9237 | LR: 4.32e-05 | 1346.2ms/step | 14.9 samples/s +Step 35800/44000 | Loss: 2.9336 | LR: 4.31e-05 | 1347.0ms/step | 14.8 samples/s +Step 35810/44000 | Loss: 2.9304 | LR: 4.29e-05 | 1347.6ms/step | 14.8 samples/s +Step 35820/44000 | Loss: 2.9323 | LR: 4.28e-05 | 1348.0ms/step | 14.8 samples/s +Step 35830/44000 | Loss: 2.9501 | LR: 4.27e-05 | 1346.3ms/step | 14.9 samples/s +Step 35840/44000 | Loss: 2.9095 | LR: 4.26e-05 | 1340.7ms/step | 14.9 samples/s +Step 35850/44000 | Loss: 2.9188 | LR: 4.25e-05 | 1339.7ms/step | 14.9 samples/s +Step 35860/44000 | Loss: 2.9421 | LR: 4.24e-05 | 1348.0ms/step | 14.8 samples/s +Step 35870/44000 | Loss: 2.9201 | LR: 4.23e-05 | 1347.2ms/step | 14.8 samples/s +Step 35880/44000 | Loss: 2.9345 | LR: 4.22e-05 | 1347.8ms/step | 14.8 samples/s +Step 35890/44000 | Loss: 2.9010 | LR: 4.21e-05 | 1345.8ms/step | 14.9 samples/s +Step 35900/44000 | Loss: 2.9222 | LR: 4.20e-05 | 1346.6ms/step | 14.9 samples/s +Step 35910/44000 | Loss: 2.9053 | LR: 4.19e-05 | 1344.4ms/step | 14.9 samples/s +Step 35920/44000 | Loss: 2.9190 | LR: 4.18e-05 | 1344.8ms/step | 14.9 samples/s +Step 35930/44000 | Loss: 2.9079 | LR: 4.17e-05 | 1340.9ms/step | 14.9 samples/s +Step 35940/44000 | Loss: 2.9080 | LR: 4.16e-05 | 1344.3ms/step | 14.9 samples/s +Step 35950/44000 | Loss: 2.8938 | LR: 4.15e-05 | 1345.7ms/step | 14.9 samples/s +Step 35960/44000 | Loss: 2.8989 | LR: 4.14e-05 | 1347.2ms/step | 14.8 samples/s +Step 35970/44000 | Loss: 2.8905 | LR: 4.13e-05 | 1346.3ms/step | 14.9 samples/s +Step 35980/44000 | Loss: 2.9434 | LR: 4.12e-05 | 1343.4ms/step | 14.9 samples/s +Step 35990/44000 | Loss: 2.9150 | LR: 4.11e-05 | 1348.9ms/step | 14.8 samples/s +Step 36000/44000 | Loss: 2.9153 | LR: 4.10e-05 | 1345.1ms/step | 14.9 samples/s +Validation at step 36000: Loss = 2.9319, Perplexity = 18.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-36000 +Step 36010/44000 | Loss: 2.9308 | LR: 4.09e-05 | 3907.5ms/step | 5.1 samples/s +Step 36020/44000 | Loss: 2.9089 | LR: 4.08e-05 | 1345.2ms/step | 14.9 samples/s +Step 36030/44000 | Loss: 2.8977 | LR: 4.07e-05 | 1346.1ms/step | 14.9 samples/s +Step 36040/44000 | Loss: 2.9036 | LR: 4.06e-05 | 1339.9ms/step | 14.9 samples/s +Step 36050/44000 | Loss: 2.9010 | LR: 4.05e-05 | 1341.3ms/step | 14.9 samples/s +Step 36060/44000 | Loss: 2.9290 | LR: 4.04e-05 | 1346.8ms/step | 14.9 samples/s +Step 36070/44000 | Loss: 2.9191 | LR: 4.03e-05 | 1342.9ms/step | 14.9 samples/s +Step 36080/44000 | Loss: 2.9344 | LR: 4.02e-05 | 1340.1ms/step | 14.9 samples/s +Step 36090/44000 | Loss: 2.9414 | LR: 4.01e-05 | 1339.3ms/step | 14.9 samples/s +Step 36100/44000 | Loss: 2.9199 | LR: 4.00e-05 | 1338.7ms/step | 14.9 samples/s +Step 36110/44000 | Loss: 2.9359 | LR: 3.99e-05 | 1339.7ms/step | 14.9 samples/s +Step 36120/44000 | Loss: 2.9221 | LR: 3.98e-05 | 1338.9ms/step | 14.9 samples/s +Step 36130/44000 | Loss: 2.9249 | LR: 3.97e-05 | 1342.3ms/step | 14.9 samples/s +Step 36140/44000 | Loss: 2.9168 | LR: 3.97e-05 | 1341.8ms/step | 14.9 samples/s +Step 36150/44000 | Loss: 2.9262 | LR: 3.96e-05 | 1344.8ms/step | 14.9 samples/s +Step 36160/44000 | Loss: 2.9186 | LR: 3.95e-05 | 1346.9ms/step | 14.8 samples/s +Step 36170/44000 | Loss: 2.8773 | LR: 3.94e-05 | 1348.7ms/step | 14.8 samples/s +Step 36180/44000 | Loss: 2.9187 | LR: 3.93e-05 | 1347.7ms/step | 14.8 samples/s +Step 36190/44000 | Loss: 2.9145 | LR: 3.92e-05 | 1349.1ms/step | 14.8 samples/s +Step 36200/44000 | Loss: 2.9076 | LR: 3.91e-05 | 1349.2ms/step | 14.8 samples/s +Step 36210/44000 | Loss: 2.9171 | LR: 3.90e-05 | 1346.1ms/step | 14.9 samples/s +Step 36220/44000 | Loss: 2.9110 | LR: 3.89e-05 | 1348.5ms/step | 14.8 samples/s +Step 36230/44000 | Loss: 2.9128 | LR: 3.88e-05 | 1348.3ms/step | 14.8 samples/s +Step 36240/44000 | Loss: 2.9221 | LR: 3.87e-05 | 1340.9ms/step | 14.9 samples/s +Step 36250/44000 | Loss: 2.9372 | LR: 3.86e-05 | 1345.7ms/step | 14.9 samples/s +Validation at step 36250: Loss = 2.9315, Perplexity = 18.75 +Step 36260/44000 | Loss: 2.9287 | LR: 3.85e-05 | 3606.4ms/step | 5.5 samples/s +Step 36270/44000 | Loss: 2.9399 | LR: 3.84e-05 | 1350.0ms/step | 14.8 samples/s +Step 36280/44000 | Loss: 2.9096 | LR: 3.83e-05 | 1350.3ms/step | 14.8 samples/s +Step 36290/44000 | Loss: 2.9264 | LR: 3.82e-05 | 1348.8ms/step | 14.8 samples/s +Step 36300/44000 | Loss: 2.9100 | LR: 3.81e-05 | 1344.6ms/step | 14.9 samples/s +Step 36310/44000 | Loss: 2.9110 | LR: 3.80e-05 | 1344.7ms/step | 14.9 samples/s +Step 36320/44000 | Loss: 2.8868 | LR: 3.79e-05 | 1349.4ms/step | 14.8 samples/s +Step 36330/44000 | Loss: 2.9053 | LR: 3.78e-05 | 1348.1ms/step | 14.8 samples/s +Step 36340/44000 | Loss: 2.9240 | LR: 3.77e-05 | 1347.9ms/step | 14.8 samples/s +Step 36350/44000 | Loss: 2.8938 | LR: 3.76e-05 | 1347.6ms/step | 14.8 samples/s +Step 36360/44000 | Loss: 2.9011 | LR: 3.75e-05 | 1346.8ms/step | 14.8 samples/s +Step 36370/44000 | Loss: 2.9108 | LR: 3.74e-05 | 1348.8ms/step | 14.8 samples/s +Step 36380/44000 | Loss: 2.9111 | LR: 3.73e-05 | 1345.6ms/step | 14.9 samples/s +Step 36390/44000 | Loss: 2.9331 | LR: 3.72e-05 | 1346.3ms/step | 14.9 samples/s +Step 36400/44000 | Loss: 2.9104 | LR: 3.71e-05 | 1346.4ms/step | 14.9 samples/s +Step 36410/44000 | Loss: 2.9203 | LR: 3.70e-05 | 1344.9ms/step | 14.9 samples/s +Step 36420/44000 | Loss: 2.8880 | LR: 3.69e-05 | 1345.5ms/step | 14.9 samples/s +Step 36430/44000 | Loss: 2.8966 | LR: 3.69e-05 | 1346.5ms/step | 14.9 samples/s +Step 36440/44000 | Loss: 2.9062 | LR: 3.68e-05 | 1345.7ms/step | 14.9 samples/s +Step 36450/44000 | Loss: 2.9461 | LR: 3.67e-05 | 1344.5ms/step | 14.9 samples/s +Step 36460/44000 | Loss: 2.9214 | LR: 3.66e-05 | 1345.9ms/step | 14.9 samples/s +Step 36470/44000 | Loss: 2.8962 | LR: 3.65e-05 | 1346.0ms/step | 14.9 samples/s +Step 36480/44000 | Loss: 2.9202 | LR: 3.64e-05 | 1344.5ms/step | 14.9 samples/s +Step 36490/44000 | Loss: 2.9414 | LR: 3.63e-05 | 1344.1ms/step | 14.9 samples/s +Step 36500/44000 | Loss: 2.8905 | LR: 3.62e-05 | 1345.0ms/step | 14.9 samples/s +Validation at step 36500: Loss = 2.9319, Perplexity = 18.76 +Step 36510/44000 | Loss: 2.9302 | LR: 3.61e-05 | 3604.8ms/step | 5.5 samples/s +Step 36520/44000 | Loss: 2.9043 | LR: 3.60e-05 | 1345.6ms/step | 14.9 samples/s +Step 36530/44000 | Loss: 2.9120 | LR: 3.59e-05 | 1346.9ms/step | 14.8 samples/s +Step 36540/44000 | Loss: 2.8997 | LR: 3.58e-05 | 1345.2ms/step | 14.9 samples/s +Step 36550/44000 | Loss: 2.9075 | LR: 3.57e-05 | 1344.6ms/step | 14.9 samples/s +Step 36560/44000 | Loss: 2.9292 | LR: 3.56e-05 | 1344.4ms/step | 14.9 samples/s +Step 36570/44000 | Loss: 2.8894 | LR: 3.55e-05 | 1344.5ms/step | 14.9 samples/s +Step 36580/44000 | Loss: 2.9070 | LR: 3.54e-05 | 1344.1ms/step | 14.9 samples/s +Step 36590/44000 | Loss: 2.9165 | LR: 3.53e-05 | 1343.2ms/step | 14.9 samples/s +Step 36600/44000 | Loss: 2.8901 | LR: 3.53e-05 | 1344.4ms/step | 14.9 samples/s +Step 36610/44000 | Loss: 2.9116 | LR: 3.52e-05 | 1349.1ms/step | 14.8 samples/s +Step 36620/44000 | Loss: 2.9016 | LR: 3.51e-05 | 1346.0ms/step | 14.9 samples/s +Step 36630/44000 | Loss: 2.9158 | LR: 3.50e-05 | 1343.4ms/step | 14.9 samples/s +Step 36640/44000 | Loss: 2.9148 | LR: 3.49e-05 | 1342.8ms/step | 14.9 samples/s +Step 36650/44000 | Loss: 2.9121 | LR: 3.48e-05 | 1343.9ms/step | 14.9 samples/s +Step 36660/44000 | Loss: 2.9038 | LR: 3.47e-05 | 1343.6ms/step | 14.9 samples/s +Step 36670/44000 | Loss: 2.9437 | LR: 3.46e-05 | 1344.5ms/step | 14.9 samples/s +Step 36680/44000 | Loss: 2.9298 | LR: 3.45e-05 | 1346.8ms/step | 14.8 samples/s +Step 36690/44000 | Loss: 2.9075 | LR: 3.44e-05 | 1347.3ms/step | 14.8 samples/s +Step 36700/44000 | Loss: 2.9078 | LR: 3.43e-05 | 1346.1ms/step | 14.9 samples/s +Step 36710/44000 | Loss: 2.9008 | LR: 3.42e-05 | 1344.2ms/step | 14.9 samples/s +Step 36720/44000 | Loss: 2.9136 | LR: 3.41e-05 | 1343.4ms/step | 14.9 samples/s +Step 36730/44000 | Loss: 2.9048 | LR: 3.41e-05 | 1343.2ms/step | 14.9 samples/s +Step 36740/44000 | Loss: 2.9292 | LR: 3.40e-05 | 1342.2ms/step | 14.9 samples/s +Step 36750/44000 | Loss: 2.8975 | LR: 3.39e-05 | 1340.9ms/step | 14.9 samples/s +Validation at step 36750: Loss = 2.9316, Perplexity = 18.76 +Step 36760/44000 | Loss: 2.9050 | LR: 3.38e-05 | 3591.0ms/step | 5.6 samples/s +Step 36770/44000 | Loss: 2.9174 | LR: 3.37e-05 | 1348.0ms/step | 14.8 samples/s +Step 36780/44000 | Loss: 2.9260 | LR: 3.36e-05 | 1345.2ms/step | 14.9 samples/s +Step 36790/44000 | Loss: 2.9241 | LR: 3.35e-05 | 1344.2ms/step | 14.9 samples/s +Step 36800/44000 | Loss: 2.9149 | LR: 3.34e-05 | 1341.1ms/step | 14.9 samples/s +Step 36810/44000 | Loss: 2.9103 | LR: 3.33e-05 | 1340.9ms/step | 14.9 samples/s +Step 36820/44000 | Loss: 2.9074 | LR: 3.32e-05 | 1342.4ms/step | 14.9 samples/s +Step 36830/44000 | Loss: 2.9114 | LR: 3.31e-05 | 1342.1ms/step | 14.9 samples/s +Step 36840/44000 | Loss: 2.9162 | LR: 3.31e-05 | 1344.1ms/step | 14.9 samples/s +Step 36850/44000 | Loss: 2.9170 | LR: 3.30e-05 | 1343.4ms/step | 14.9 samples/s +Step 36860/44000 | Loss: 2.8965 | LR: 3.29e-05 | 1342.0ms/step | 14.9 samples/s +Step 36870/44000 | Loss: 2.9178 | LR: 3.28e-05 | 1341.3ms/step | 14.9 samples/s +Step 36880/44000 | Loss: 2.8914 | LR: 3.27e-05 | 1343.4ms/step | 14.9 samples/s +Step 36890/44000 | Loss: 2.8886 | LR: 3.26e-05 | 1343.6ms/step | 14.9 samples/s +Step 36900/44000 | Loss: 2.8923 | LR: 3.25e-05 | 1342.6ms/step | 14.9 samples/s +Step 36910/44000 | Loss: 2.9081 | LR: 3.24e-05 | 1342.9ms/step | 14.9 samples/s +Step 36920/44000 | Loss: 2.9081 | LR: 3.23e-05 | 1344.4ms/step | 14.9 samples/s +Step 36930/44000 | Loss: 2.8858 | LR: 3.22e-05 | 1339.6ms/step | 14.9 samples/s +Step 36940/44000 | Loss: 2.9251 | LR: 3.22e-05 | 1339.7ms/step | 14.9 samples/s +Step 36950/44000 | Loss: 2.9214 | LR: 3.21e-05 | 1340.3ms/step | 14.9 samples/s +Step 36960/44000 | Loss: 2.9048 | LR: 3.20e-05 | 1340.1ms/step | 14.9 samples/s +Step 36970/44000 | Loss: 2.8861 | LR: 3.19e-05 | 1342.3ms/step | 14.9 samples/s +Step 36980/44000 | Loss: 2.9080 | LR: 3.18e-05 | 1344.4ms/step | 14.9 samples/s +Step 36990/44000 | Loss: 2.9192 | LR: 3.17e-05 | 1346.7ms/step | 14.9 samples/s +Step 37000/44000 | Loss: 2.9292 | LR: 3.16e-05 | 1343.0ms/step | 14.9 samples/s +Validation at step 37000: Loss = 2.9314, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-37000 +Step 37010/44000 | Loss: 2.9265 | LR: 3.15e-05 | 4049.8ms/step | 4.9 samples/s +Step 37020/44000 | Loss: 2.9643 | LR: 3.15e-05 | 1390.3ms/step | 14.4 samples/s +Step 37030/44000 | Loss: 2.9300 | LR: 3.14e-05 | 1355.8ms/step | 14.8 samples/s +Step 37040/44000 | Loss: 2.9289 | LR: 3.13e-05 | 1353.7ms/step | 14.8 samples/s +Step 37050/44000 | Loss: 2.9200 | LR: 3.12e-05 | 1352.0ms/step | 14.8 samples/s +Step 37060/44000 | Loss: 2.8882 | LR: 3.11e-05 | 1356.1ms/step | 14.7 samples/s +Step 37070/44000 | Loss: 2.9177 | LR: 3.10e-05 | 1363.3ms/step | 14.7 samples/s +Step 37080/44000 | Loss: 2.9498 | LR: 3.09e-05 | 1381.0ms/step | 14.5 samples/s +Step 37090/44000 | Loss: 2.9275 | LR: 3.08e-05 | 1378.8ms/step | 14.5 samples/s +Step 37100/44000 | Loss: 2.9104 | LR: 3.07e-05 | 1368.8ms/step | 14.6 samples/s +Step 37110/44000 | Loss: 2.9061 | LR: 3.07e-05 | 1368.7ms/step | 14.6 samples/s +Step 37120/44000 | Loss: 2.9277 | LR: 3.06e-05 | 1366.4ms/step | 14.6 samples/s +Step 37130/44000 | Loss: 2.9188 | LR: 3.05e-05 | 1366.7ms/step | 14.6 samples/s +Step 37140/44000 | Loss: 2.9118 | LR: 3.04e-05 | 1366.5ms/step | 14.6 samples/s +Step 37150/44000 | Loss: 2.8741 | LR: 3.03e-05 | 1370.8ms/step | 14.6 samples/s +Step 37160/44000 | Loss: 2.9238 | LR: 3.02e-05 | 1387.4ms/step | 14.4 samples/s +Step 37170/44000 | Loss: 2.9562 | LR: 3.01e-05 | 1378.3ms/step | 14.5 samples/s +Step 37180/44000 | Loss: 2.9087 | LR: 3.01e-05 | 1370.7ms/step | 14.6 samples/s +Step 37190/44000 | Loss: 2.9125 | LR: 3.00e-05 | 1364.4ms/step | 14.7 samples/s +Step 37200/44000 | Loss: 2.9174 | LR: 2.99e-05 | 1361.2ms/step | 14.7 samples/s +Step 37210/44000 | Loss: 2.9308 | LR: 2.98e-05 | 1367.8ms/step | 14.6 samples/s +Step 37220/44000 | Loss: 2.9129 | LR: 2.97e-05 | 1397.4ms/step | 14.3 samples/s +Step 37230/44000 | Loss: 2.9401 | LR: 2.96e-05 | 1391.4ms/step | 14.4 samples/s +Step 37240/44000 | Loss: 2.9167 | LR: 2.95e-05 | 1378.1ms/step | 14.5 samples/s +Step 37250/44000 | Loss: 2.9084 | LR: 2.95e-05 | 1374.6ms/step | 14.5 samples/s +Validation at step 37250: Loss = 2.9313, Perplexity = 18.75 +Step 37260/44000 | Loss: 2.9224 | LR: 2.94e-05 | 3735.5ms/step | 5.4 samples/s +Step 37270/44000 | Loss: 2.9336 | LR: 2.93e-05 | 1364.3ms/step | 14.7 samples/s +Step 37280/44000 | Loss: 2.9133 | LR: 2.92e-05 | 1359.8ms/step | 14.7 samples/s +Step 37290/44000 | Loss: 2.9230 | LR: 2.91e-05 | 1358.2ms/step | 14.7 samples/s +Step 37300/44000 | Loss: 2.9100 | LR: 2.90e-05 | 1358.7ms/step | 14.7 samples/s +Step 37310/44000 | Loss: 2.9128 | LR: 2.89e-05 | 1355.2ms/step | 14.8 samples/s +Step 37320/44000 | Loss: 2.9175 | LR: 2.89e-05 | 1352.1ms/step | 14.8 samples/s +Step 37330/44000 | Loss: 2.9406 | LR: 2.88e-05 | 1371.4ms/step | 14.6 samples/s +Step 37340/44000 | Loss: 2.9130 | LR: 2.87e-05 | 1375.4ms/step | 14.5 samples/s +Step 37350/44000 | Loss: 2.9324 | LR: 2.86e-05 | 1358.0ms/step | 14.7 samples/s +Step 37360/44000 | Loss: 2.9169 | LR: 2.85e-05 | 1352.4ms/step | 14.8 samples/s +Step 37370/44000 | Loss: 2.9198 | LR: 2.84e-05 | 1368.2ms/step | 14.6 samples/s +Step 37380/44000 | Loss: 2.9027 | LR: 2.84e-05 | 1373.6ms/step | 14.6 samples/s +Step 37390/44000 | Loss: 2.8986 | LR: 2.83e-05 | 1373.0ms/step | 14.6 samples/s +Step 37400/44000 | Loss: 2.9040 | LR: 2.82e-05 | 1373.3ms/step | 14.6 samples/s +Step 37410/44000 | Loss: 2.9001 | LR: 2.81e-05 | 1372.1ms/step | 14.6 samples/s +Step 37420/44000 | Loss: 2.9279 | LR: 2.80e-05 | 1370.4ms/step | 14.6 samples/s +Step 37430/44000 | Loss: 2.8900 | LR: 2.79e-05 | 1367.6ms/step | 14.6 samples/s +Step 37440/44000 | Loss: 2.9034 | LR: 2.78e-05 | 1372.4ms/step | 14.6 samples/s +Step 37450/44000 | Loss: 2.9161 | LR: 2.78e-05 | 1369.4ms/step | 14.6 samples/s +Step 37460/44000 | Loss: 2.9315 | LR: 2.77e-05 | 1371.6ms/step | 14.6 samples/s +Step 37470/44000 | Loss: 2.9135 | LR: 2.76e-05 | 1369.4ms/step | 14.6 samples/s +Step 37480/44000 | Loss: 2.9259 | LR: 2.75e-05 | 1369.8ms/step | 14.6 samples/s +Step 37490/44000 | Loss: 2.9093 | LR: 2.74e-05 | 1366.9ms/step | 14.6 samples/s +Step 37500/44000 | Loss: 2.9152 | LR: 2.74e-05 | 1369.0ms/step | 14.6 samples/s +Validation at step 37500: Loss = 2.9314, Perplexity = 18.75 +Step 37510/44000 | Loss: 2.9362 | LR: 2.73e-05 | 3679.4ms/step | 5.4 samples/s +Step 37520/44000 | Loss: 2.8906 | LR: 2.72e-05 | 1364.1ms/step | 14.7 samples/s +Step 37530/44000 | Loss: 2.9552 | LR: 2.71e-05 | 1366.2ms/step | 14.6 samples/s +Step 37540/44000 | Loss: 2.9239 | LR: 2.70e-05 | 1363.3ms/step | 14.7 samples/s +Step 37550/44000 | Loss: 2.9158 | LR: 2.69e-05 | 1362.1ms/step | 14.7 samples/s +Step 37560/44000 | Loss: 2.9337 | LR: 2.69e-05 | 1361.4ms/step | 14.7 samples/s +Step 37570/44000 | Loss: 2.9134 | LR: 2.68e-05 | 1362.4ms/step | 14.7 samples/s +Step 37580/44000 | Loss: 2.9068 | LR: 2.67e-05 | 1362.2ms/step | 14.7 samples/s +Step 37590/44000 | Loss: 2.9077 | LR: 2.66e-05 | 1360.6ms/step | 14.7 samples/s +Step 37600/44000 | Loss: 2.9120 | LR: 2.65e-05 | 1363.0ms/step | 14.7 samples/s +Step 37610/44000 | Loss: 2.9033 | LR: 2.65e-05 | 1361.0ms/step | 14.7 samples/s +Step 37620/44000 | Loss: 2.9265 | LR: 2.64e-05 | 1355.5ms/step | 14.8 samples/s +Step 37630/44000 | Loss: 2.8812 | LR: 2.63e-05 | 1355.2ms/step | 14.8 samples/s +Step 37640/44000 | Loss: 2.9304 | LR: 2.62e-05 | 1356.8ms/step | 14.7 samples/s +Step 37650/44000 | Loss: 2.9005 | LR: 2.61e-05 | 1358.5ms/step | 14.7 samples/s +Step 37660/44000 | Loss: 2.9404 | LR: 2.60e-05 | 1358.0ms/step | 14.7 samples/s +Step 37670/44000 | Loss: 2.9206 | LR: 2.60e-05 | 1356.7ms/step | 14.7 samples/s +Step 37680/44000 | Loss: 2.9313 | LR: 2.59e-05 | 1357.0ms/step | 14.7 samples/s +Step 37690/44000 | Loss: 2.9091 | LR: 2.58e-05 | 1360.7ms/step | 14.7 samples/s +Step 37700/44000 | Loss: 2.9045 | LR: 2.57e-05 | 1358.6ms/step | 14.7 samples/s +Step 37710/44000 | Loss: 2.8973 | LR: 2.56e-05 | 1358.7ms/step | 14.7 samples/s +Step 37720/44000 | Loss: 2.9319 | LR: 2.56e-05 | 1357.2ms/step | 14.7 samples/s +Step 37730/44000 | Loss: 2.9035 | LR: 2.55e-05 | 1356.5ms/step | 14.7 samples/s +Step 37740/44000 | Loss: 2.9154 | LR: 2.54e-05 | 1358.5ms/step | 14.7 samples/s +Step 37750/44000 | Loss: 2.8988 | LR: 2.53e-05 | 1355.2ms/step | 14.8 samples/s +Validation at step 37750: Loss = 2.9311, Perplexity = 18.75 +Step 37760/44000 | Loss: 2.9203 | LR: 2.52e-05 | 3651.4ms/step | 5.5 samples/s +Step 37770/44000 | Loss: 2.8961 | LR: 2.52e-05 | 1355.2ms/step | 14.8 samples/s +Step 37780/44000 | Loss: 2.9007 | LR: 2.51e-05 | 1352.5ms/step | 14.8 samples/s +Step 37790/44000 | Loss: 2.9354 | LR: 2.50e-05 | 1349.9ms/step | 14.8 samples/s +Step 37800/44000 | Loss: 2.9044 | LR: 2.49e-05 | 1351.1ms/step | 14.8 samples/s +Step 37810/44000 | Loss: 2.8811 | LR: 2.48e-05 | 1355.2ms/step | 14.8 samples/s +Step 37820/44000 | Loss: 2.9389 | LR: 2.48e-05 | 1356.6ms/step | 14.7 samples/s +Step 37830/44000 | Loss: 2.9382 | LR: 2.47e-05 | 1354.6ms/step | 14.8 samples/s +Step 37840/44000 | Loss: 2.9212 | LR: 2.46e-05 | 1352.2ms/step | 14.8 samples/s +Step 37850/44000 | Loss: 2.9024 | LR: 2.45e-05 | 1351.8ms/step | 14.8 samples/s +Step 37860/44000 | Loss: 2.8824 | LR: 2.45e-05 | 1355.5ms/step | 14.8 samples/s +Step 37870/44000 | Loss: 2.9068 | LR: 2.44e-05 | 1355.0ms/step | 14.8 samples/s +Step 37880/44000 | Loss: 2.9036 | LR: 2.43e-05 | 1357.8ms/step | 14.7 samples/s +Step 37890/44000 | Loss: 2.9257 | LR: 2.42e-05 | 1352.6ms/step | 14.8 samples/s +Step 37900/44000 | Loss: 2.9316 | LR: 2.41e-05 | 1354.4ms/step | 14.8 samples/s +Step 37910/44000 | Loss: 2.9136 | LR: 2.41e-05 | 1353.8ms/step | 14.8 samples/s +Step 37920/44000 | Loss: 2.9331 | LR: 2.40e-05 | 1354.2ms/step | 14.8 samples/s +Step 37930/44000 | Loss: 2.9107 | LR: 2.39e-05 | 1350.4ms/step | 14.8 samples/s +Step 37940/44000 | Loss: 2.9222 | LR: 2.38e-05 | 1352.5ms/step | 14.8 samples/s +Step 37950/44000 | Loss: 2.9377 | LR: 2.38e-05 | 1353.9ms/step | 14.8 samples/s +Step 37960/44000 | Loss: 2.9549 | LR: 2.37e-05 | 1354.6ms/step | 14.8 samples/s +Step 37970/44000 | Loss: 2.9121 | LR: 2.36e-05 | 1357.8ms/step | 14.7 samples/s +Step 37980/44000 | Loss: 2.9443 | LR: 2.35e-05 | 1355.5ms/step | 14.8 samples/s +Step 37990/44000 | Loss: 2.9124 | LR: 2.34e-05 | 1355.0ms/step | 14.8 samples/s +Step 38000/44000 | Loss: 2.9160 | LR: 2.34e-05 | 1356.0ms/step | 14.7 samples/s +Validation at step 38000: Loss = 2.9311, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-38000 +Step 38010/44000 | Loss: 2.8677 | LR: 2.33e-05 | 3930.8ms/step | 5.1 samples/s +Step 38020/44000 | Loss: 2.9012 | LR: 2.32e-05 | 1351.6ms/step | 14.8 samples/s +Step 38030/44000 | Loss: 2.8691 | LR: 2.31e-05 | 1353.1ms/step | 14.8 samples/s +Step 38040/44000 | Loss: 2.9385 | LR: 2.31e-05 | 1352.2ms/step | 14.8 samples/s +Step 38050/44000 | Loss: 2.9226 | LR: 2.30e-05 | 1354.4ms/step | 14.8 samples/s +Step 38060/44000 | Loss: 2.9274 | LR: 2.29e-05 | 1352.6ms/step | 14.8 samples/s +Step 38070/44000 | Loss: 2.9006 | LR: 2.28e-05 | 1353.9ms/step | 14.8 samples/s +Step 38080/44000 | Loss: 2.9144 | LR: 2.28e-05 | 1351.5ms/step | 14.8 samples/s +Step 38090/44000 | Loss: 2.9267 | LR: 2.27e-05 | 1350.2ms/step | 14.8 samples/s +Step 38100/44000 | Loss: 2.9095 | LR: 2.26e-05 | 1354.7ms/step | 14.8 samples/s +Step 38110/44000 | Loss: 2.9214 | LR: 2.25e-05 | 1353.3ms/step | 14.8 samples/s +Step 38120/44000 | Loss: 2.9314 | LR: 2.25e-05 | 1355.2ms/step | 14.8 samples/s +Step 38130/44000 | Loss: 2.9083 | LR: 2.24e-05 | 1354.7ms/step | 14.8 samples/s +Step 38140/44000 | Loss: 2.9335 | LR: 2.23e-05 | 1351.4ms/step | 14.8 samples/s +Step 38150/44000 | Loss: 2.9513 | LR: 2.22e-05 | 1348.6ms/step | 14.8 samples/s +Step 38160/44000 | Loss: 2.9564 | LR: 2.22e-05 | 1346.7ms/step | 14.9 samples/s +Step 38170/44000 | Loss: 2.9196 | LR: 2.21e-05 | 1347.8ms/step | 14.8 samples/s +Step 38180/44000 | Loss: 2.9477 | LR: 2.20e-05 | 1352.5ms/step | 14.8 samples/s +Step 38190/44000 | Loss: 2.8909 | LR: 2.19e-05 | 1353.8ms/step | 14.8 samples/s +Step 38200/44000 | Loss: 2.9202 | LR: 2.19e-05 | 1354.0ms/step | 14.8 samples/s +Step 38210/44000 | Loss: 2.9182 | LR: 2.18e-05 | 1351.6ms/step | 14.8 samples/s +Step 38220/44000 | Loss: 2.9378 | LR: 2.17e-05 | 1352.2ms/step | 14.8 samples/s +Step 38230/44000 | Loss: 2.9211 | LR: 2.16e-05 | 1349.6ms/step | 14.8 samples/s +Step 38240/44000 | Loss: 2.9243 | LR: 2.16e-05 | 1349.7ms/step | 14.8 samples/s +Step 38250/44000 | Loss: 2.9255 | LR: 2.15e-05 | 1350.5ms/step | 14.8 samples/s +Validation at step 38250: Loss = 2.9311, Perplexity = 18.75 +Step 38260/44000 | Loss: 2.8974 | LR: 2.14e-05 | 3631.7ms/step | 5.5 samples/s +Step 38270/44000 | Loss: 2.9279 | LR: 2.13e-05 | 1352.2ms/step | 14.8 samples/s +Step 38280/44000 | Loss: 2.8867 | LR: 2.13e-05 | 1351.0ms/step | 14.8 samples/s +Step 38290/44000 | Loss: 2.8944 | LR: 2.12e-05 | 1346.1ms/step | 14.9 samples/s +Step 38300/44000 | Loss: 2.9286 | LR: 2.11e-05 | 1348.5ms/step | 14.8 samples/s +Step 38310/44000 | Loss: 2.8995 | LR: 2.11e-05 | 1351.7ms/step | 14.8 samples/s +Step 38320/44000 | Loss: 2.9162 | LR: 2.10e-05 | 1348.8ms/step | 14.8 samples/s +Step 38330/44000 | Loss: 2.9346 | LR: 2.09e-05 | 1353.4ms/step | 14.8 samples/s +Step 38340/44000 | Loss: 2.8842 | LR: 2.08e-05 | 1352.6ms/step | 14.8 samples/s +Step 38350/44000 | Loss: 2.9136 | LR: 2.08e-05 | 1347.7ms/step | 14.8 samples/s +Step 38360/44000 | Loss: 2.9198 | LR: 2.07e-05 | 1344.7ms/step | 14.9 samples/s +Step 38370/44000 | Loss: 2.9274 | LR: 2.06e-05 | 1340.7ms/step | 14.9 samples/s +Step 38380/44000 | Loss: 2.9329 | LR: 2.05e-05 | 1344.9ms/step | 14.9 samples/s +Step 38390/44000 | Loss: 2.9099 | LR: 2.05e-05 | 1345.2ms/step | 14.9 samples/s +Step 38400/44000 | Loss: 2.9307 | LR: 2.04e-05 | 1344.4ms/step | 14.9 samples/s +Step 38410/44000 | Loss: 2.9143 | LR: 2.03e-05 | 1346.3ms/step | 14.9 samples/s +Step 38420/44000 | Loss: 2.9238 | LR: 2.03e-05 | 1343.1ms/step | 14.9 samples/s +Step 38430/44000 | Loss: 2.9283 | LR: 2.02e-05 | 1347.0ms/step | 14.8 samples/s +Step 38440/44000 | Loss: 2.8823 | LR: 2.01e-05 | 1345.9ms/step | 14.9 samples/s +Step 38450/44000 | Loss: 2.9173 | LR: 2.00e-05 | 1346.0ms/step | 14.9 samples/s +Step 38460/44000 | Loss: 2.9176 | LR: 2.00e-05 | 1345.4ms/step | 14.9 samples/s +Step 38470/44000 | Loss: 2.9128 | LR: 1.99e-05 | 1350.1ms/step | 14.8 samples/s +Step 38480/44000 | Loss: 2.9357 | LR: 1.98e-05 | 1348.7ms/step | 14.8 samples/s +Step 38490/44000 | Loss: 2.9238 | LR: 1.98e-05 | 1350.9ms/step | 14.8 samples/s +Step 38500/44000 | Loss: 2.9243 | LR: 1.97e-05 | 1347.0ms/step | 14.8 samples/s +Validation at step 38500: Loss = 2.9312, Perplexity = 18.75 +Step 38510/44000 | Loss: 2.9165 | LR: 1.96e-05 | 3618.9ms/step | 5.5 samples/s +Step 38520/44000 | Loss: 2.8934 | LR: 1.95e-05 | 1345.5ms/step | 14.9 samples/s +Step 38530/44000 | Loss: 2.9162 | LR: 1.95e-05 | 1347.5ms/step | 14.8 samples/s +Step 38540/44000 | Loss: 2.9263 | LR: 1.94e-05 | 1342.2ms/step | 14.9 samples/s +Step 38550/44000 | Loss: 2.9361 | LR: 1.93e-05 | 1342.0ms/step | 14.9 samples/s +Step 38560/44000 | Loss: 2.9190 | LR: 1.93e-05 | 1344.1ms/step | 14.9 samples/s +Step 38570/44000 | Loss: 2.9112 | LR: 1.92e-05 | 1342.8ms/step | 14.9 samples/s +Step 38580/44000 | Loss: 2.8925 | LR: 1.91e-05 | 1348.5ms/step | 14.8 samples/s +Step 38590/44000 | Loss: 2.9222 | LR: 1.91e-05 | 1344.8ms/step | 14.9 samples/s +Step 38600/44000 | Loss: 2.8985 | LR: 1.90e-05 | 1350.2ms/step | 14.8 samples/s +Step 38610/44000 | Loss: 2.9051 | LR: 1.89e-05 | 1348.6ms/step | 14.8 samples/s +Step 38620/44000 | Loss: 2.9245 | LR: 1.88e-05 | 1349.5ms/step | 14.8 samples/s +Step 38630/44000 | Loss: 2.8997 | LR: 1.88e-05 | 1347.2ms/step | 14.8 samples/s +Step 38640/44000 | Loss: 2.9400 | LR: 1.87e-05 | 1352.1ms/step | 14.8 samples/s +Step 38650/44000 | Loss: 2.8952 | LR: 1.86e-05 | 1348.7ms/step | 14.8 samples/s +Step 38660/44000 | Loss: 2.9041 | LR: 1.86e-05 | 1347.7ms/step | 14.8 samples/s +Step 38670/44000 | Loss: 2.9202 | LR: 1.85e-05 | 1344.1ms/step | 14.9 samples/s +Step 38680/44000 | Loss: 2.9421 | LR: 1.84e-05 | 1342.3ms/step | 14.9 samples/s +Step 38690/44000 | Loss: 2.9179 | LR: 1.84e-05 | 1344.5ms/step | 14.9 samples/s +Step 38700/44000 | Loss: 2.9360 | LR: 1.83e-05 | 1345.3ms/step | 14.9 samples/s +Step 38710/44000 | Loss: 2.9204 | LR: 1.82e-05 | 1346.1ms/step | 14.9 samples/s +Step 38720/44000 | Loss: 2.8933 | LR: 1.82e-05 | 1348.6ms/step | 14.8 samples/s +Step 38730/44000 | Loss: 2.9091 | LR: 1.81e-05 | 1349.2ms/step | 14.8 samples/s +Step 38740/44000 | Loss: 2.9312 | LR: 1.80e-05 | 1345.6ms/step | 14.9 samples/s +Step 38750/44000 | Loss: 2.9133 | LR: 1.80e-05 | 1344.0ms/step | 14.9 samples/s +Validation at step 38750: Loss = 2.9312, Perplexity = 18.75 +Step 38760/44000 | Loss: 2.9234 | LR: 1.79e-05 | 3608.3ms/step | 5.5 samples/s +Step 38770/44000 | Loss: 2.9115 | LR: 1.78e-05 | 1346.1ms/step | 14.9 samples/s +Step 38780/44000 | Loss: 2.9016 | LR: 1.78e-05 | 1342.4ms/step | 14.9 samples/s +Step 38790/44000 | Loss: 2.9353 | LR: 1.77e-05 | 1341.5ms/step | 14.9 samples/s +Step 38800/44000 | Loss: 2.9011 | LR: 1.76e-05 | 1344.1ms/step | 14.9 samples/s +Step 38810/44000 | Loss: 2.8850 | LR: 1.76e-05 | 1343.4ms/step | 14.9 samples/s +Step 38820/44000 | Loss: 2.9220 | LR: 1.75e-05 | 1344.5ms/step | 14.9 samples/s +Step 38830/44000 | Loss: 2.9229 | LR: 1.74e-05 | 1346.9ms/step | 14.8 samples/s +Step 38840/44000 | Loss: 2.8861 | LR: 1.74e-05 | 1347.1ms/step | 14.8 samples/s +Step 38850/44000 | Loss: 2.9236 | LR: 1.73e-05 | 1346.7ms/step | 14.9 samples/s +Step 38860/44000 | Loss: 2.9101 | LR: 1.72e-05 | 1347.3ms/step | 14.8 samples/s +Step 38870/44000 | Loss: 2.9292 | LR: 1.72e-05 | 1347.9ms/step | 14.8 samples/s +Step 38880/44000 | Loss: 2.9287 | LR: 1.71e-05 | 1344.1ms/step | 14.9 samples/s +Step 38890/44000 | Loss: 2.9159 | LR: 1.70e-05 | 1344.7ms/step | 14.9 samples/s +Step 38900/44000 | Loss: 2.9129 | LR: 1.70e-05 | 1346.9ms/step | 14.8 samples/s +Step 38910/44000 | Loss: 2.9353 | LR: 1.69e-05 | 1346.8ms/step | 14.8 samples/s +Step 38920/44000 | Loss: 2.9165 | LR: 1.68e-05 | 1347.1ms/step | 14.8 samples/s +Step 38930/44000 | Loss: 2.8941 | LR: 1.68e-05 | 1346.3ms/step | 14.9 samples/s +Step 38940/44000 | Loss: 2.9034 | LR: 1.67e-05 | 1345.7ms/step | 14.9 samples/s +Step 38950/44000 | Loss: 2.8899 | LR: 1.66e-05 | 1340.3ms/step | 14.9 samples/s +Step 38960/44000 | Loss: 2.9067 | LR: 1.66e-05 | 1342.4ms/step | 14.9 samples/s +Step 38970/44000 | Loss: 2.9223 | LR: 1.65e-05 | 1341.1ms/step | 14.9 samples/s +Step 38980/44000 | Loss: 2.8960 | LR: 1.64e-05 | 1341.1ms/step | 14.9 samples/s +Step 38990/44000 | Loss: 2.9270 | LR: 1.64e-05 | 1344.1ms/step | 14.9 samples/s +Step 39000/44000 | Loss: 2.9151 | LR: 1.63e-05 | 1348.7ms/step | 14.8 samples/s +Validation at step 39000: Loss = 2.9311, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-39000 +Step 39010/44000 | Loss: 2.9518 | LR: 1.62e-05 | 3907.9ms/step | 5.1 samples/s +Step 39020/44000 | Loss: 2.9054 | LR: 1.62e-05 | 1343.3ms/step | 14.9 samples/s +Step 39030/44000 | Loss: 2.9420 | LR: 1.61e-05 | 1344.2ms/step | 14.9 samples/s +Step 39040/44000 | Loss: 2.8971 | LR: 1.61e-05 | 1344.5ms/step | 14.9 samples/s +Step 39050/44000 | Loss: 2.9048 | LR: 1.60e-05 | 1345.2ms/step | 14.9 samples/s +Step 39060/44000 | Loss: 2.9164 | LR: 1.59e-05 | 1344.4ms/step | 14.9 samples/s +Step 39070/44000 | Loss: 2.8951 | LR: 1.59e-05 | 1345.7ms/step | 14.9 samples/s +Step 39080/44000 | Loss: 2.9241 | LR: 1.58e-05 | 1343.1ms/step | 14.9 samples/s +Step 39090/44000 | Loss: 2.8849 | LR: 1.57e-05 | 1347.8ms/step | 14.8 samples/s +Step 39100/44000 | Loss: 2.8901 | LR: 1.57e-05 | 1345.0ms/step | 14.9 samples/s +Step 39110/44000 | Loss: 2.9286 | LR: 1.56e-05 | 1343.6ms/step | 14.9 samples/s +Step 39120/44000 | Loss: 2.9196 | LR: 1.55e-05 | 1342.8ms/step | 14.9 samples/s +Step 39130/44000 | Loss: 2.8887 | LR: 1.55e-05 | 1347.6ms/step | 14.8 samples/s +Step 39140/44000 | Loss: 2.9082 | LR: 1.54e-05 | 1347.4ms/step | 14.8 samples/s +Step 39150/44000 | Loss: 2.9212 | LR: 1.54e-05 | 1348.9ms/step | 14.8 samples/s +Step 39160/44000 | Loss: 2.9481 | LR: 1.53e-05 | 1343.3ms/step | 14.9 samples/s +Step 39170/44000 | Loss: 2.9247 | LR: 1.52e-05 | 1344.0ms/step | 14.9 samples/s +Step 39180/44000 | Loss: 2.9288 | LR: 1.52e-05 | 1344.0ms/step | 14.9 samples/s +Step 39190/44000 | Loss: 2.8700 | LR: 1.51e-05 | 1343.1ms/step | 14.9 samples/s +Step 39200/44000 | Loss: 2.8946 | LR: 1.50e-05 | 1344.2ms/step | 14.9 samples/s +Step 39210/44000 | Loss: 2.8816 | LR: 1.50e-05 | 1342.0ms/step | 14.9 samples/s +Step 39220/44000 | Loss: 2.9205 | LR: 1.49e-05 | 1341.2ms/step | 14.9 samples/s +Step 39230/44000 | Loss: 2.9064 | LR: 1.49e-05 | 1344.7ms/step | 14.9 samples/s +Step 39240/44000 | Loss: 2.9086 | LR: 1.48e-05 | 1346.0ms/step | 14.9 samples/s +Step 39250/44000 | Loss: 2.9068 | LR: 1.47e-05 | 1340.9ms/step | 14.9 samples/s +Validation at step 39250: Loss = 2.9310, Perplexity = 18.75 +Step 39260/44000 | Loss: 2.9054 | LR: 1.47e-05 | 3594.4ms/step | 5.6 samples/s +Step 39270/44000 | Loss: 2.8979 | LR: 1.46e-05 | 1343.3ms/step | 14.9 samples/s +Step 39280/44000 | Loss: 2.9104 | LR: 1.46e-05 | 1344.7ms/step | 14.9 samples/s +Step 39290/44000 | Loss: 2.9095 | LR: 1.45e-05 | 1344.9ms/step | 14.9 samples/s +Step 39300/44000 | Loss: 2.9089 | LR: 1.44e-05 | 1346.0ms/step | 14.9 samples/s +Step 39310/44000 | Loss: 2.8999 | LR: 1.44e-05 | 1344.8ms/step | 14.9 samples/s +Step 39320/44000 | Loss: 2.8785 | LR: 1.43e-05 | 1345.8ms/step | 14.9 samples/s +Step 39330/44000 | Loss: 2.9206 | LR: 1.42e-05 | 1345.5ms/step | 14.9 samples/s +Step 39340/44000 | Loss: 2.9193 | LR: 1.42e-05 | 1344.0ms/step | 14.9 samples/s +Step 39350/44000 | Loss: 2.9112 | LR: 1.41e-05 | 1343.1ms/step | 14.9 samples/s +Step 39360/44000 | Loss: 2.9503 | LR: 1.41e-05 | 1345.0ms/step | 14.9 samples/s +Step 39370/44000 | Loss: 2.8932 | LR: 1.40e-05 | 1346.6ms/step | 14.9 samples/s +Step 39380/44000 | Loss: 2.9329 | LR: 1.39e-05 | 1345.4ms/step | 14.9 samples/s +Step 39390/44000 | Loss: 2.9170 | LR: 1.39e-05 | 1345.1ms/step | 14.9 samples/s +Step 39400/44000 | Loss: 2.9049 | LR: 1.38e-05 | 1348.2ms/step | 14.8 samples/s +Step 39410/44000 | Loss: 2.9142 | LR: 1.38e-05 | 1340.3ms/step | 14.9 samples/s +Step 39420/44000 | Loss: 2.9387 | LR: 1.37e-05 | 1342.4ms/step | 14.9 samples/s +Step 39430/44000 | Loss: 2.9340 | LR: 1.36e-05 | 1344.2ms/step | 14.9 samples/s +Step 39440/44000 | Loss: 2.9013 | LR: 1.36e-05 | 1348.2ms/step | 14.8 samples/s +Step 39450/44000 | Loss: 2.9345 | LR: 1.35e-05 | 1347.4ms/step | 14.8 samples/s +Step 39460/44000 | Loss: 2.8968 | LR: 1.35e-05 | 1345.6ms/step | 14.9 samples/s +Step 39470/44000 | Loss: 2.9023 | LR: 1.34e-05 | 1344.7ms/step | 14.9 samples/s +Step 39480/44000 | Loss: 2.9254 | LR: 1.34e-05 | 1345.9ms/step | 14.9 samples/s +Step 39490/44000 | Loss: 2.9270 | LR: 1.33e-05 | 1345.3ms/step | 14.9 samples/s +Step 39500/44000 | Loss: 2.9091 | LR: 1.32e-05 | 1345.8ms/step | 14.9 samples/s +Validation at step 39500: Loss = 2.9310, Perplexity = 18.75 +Step 39510/44000 | Loss: 2.8922 | LR: 1.32e-05 | 3607.4ms/step | 5.5 samples/s +Step 39520/44000 | Loss: 2.8928 | LR: 1.31e-05 | 1347.5ms/step | 14.8 samples/s +Step 39530/44000 | Loss: 2.9132 | LR: 1.31e-05 | 1346.2ms/step | 14.9 samples/s +Step 39540/44000 | Loss: 2.9165 | LR: 1.30e-05 | 1344.6ms/step | 14.9 samples/s +Step 39550/44000 | Loss: 2.9094 | LR: 1.29e-05 | 1347.5ms/step | 14.8 samples/s +Step 39560/44000 | Loss: 2.9233 | LR: 1.29e-05 | 1344.6ms/step | 14.9 samples/s +Step 39570/44000 | Loss: 2.8689 | LR: 1.28e-05 | 1344.2ms/step | 14.9 samples/s +Step 39580/44000 | Loss: 2.9108 | LR: 1.28e-05 | 1343.2ms/step | 14.9 samples/s +Step 39590/44000 | Loss: 2.8986 | LR: 1.27e-05 | 1341.9ms/step | 14.9 samples/s +Step 39600/44000 | Loss: 2.9062 | LR: 1.27e-05 | 1344.1ms/step | 14.9 samples/s +Step 39610/44000 | Loss: 2.9094 | LR: 1.26e-05 | 1345.6ms/step | 14.9 samples/s +Step 39620/44000 | Loss: 2.9158 | LR: 1.25e-05 | 1346.8ms/step | 14.9 samples/s +Step 39630/44000 | Loss: 2.9180 | LR: 1.25e-05 | 1344.6ms/step | 14.9 samples/s +Step 39640/44000 | Loss: 2.9198 | LR: 1.24e-05 | 1346.8ms/step | 14.8 samples/s +Step 39650/44000 | Loss: 2.9182 | LR: 1.24e-05 | 1346.3ms/step | 14.9 samples/s +Step 39660/44000 | Loss: 2.9063 | LR: 1.23e-05 | 1343.9ms/step | 14.9 samples/s +Step 39670/44000 | Loss: 2.8965 | LR: 1.23e-05 | 1344.1ms/step | 14.9 samples/s +Step 39680/44000 | Loss: 2.9294 | LR: 1.22e-05 | 1344.9ms/step | 14.9 samples/s +Step 39690/44000 | Loss: 2.9283 | LR: 1.22e-05 | 1344.6ms/step | 14.9 samples/s +Step 39700/44000 | Loss: 2.9190 | LR: 1.21e-05 | 1341.8ms/step | 14.9 samples/s +Step 39710/44000 | Loss: 2.9566 | LR: 1.20e-05 | 1341.5ms/step | 14.9 samples/s +Step 39720/44000 | Loss: 2.9250 | LR: 1.20e-05 | 1340.7ms/step | 14.9 samples/s +Step 39730/44000 | Loss: 2.9216 | LR: 1.19e-05 | 1342.6ms/step | 14.9 samples/s +Step 39740/44000 | Loss: 2.9267 | LR: 1.19e-05 | 1342.3ms/step | 14.9 samples/s +Step 39750/44000 | Loss: 2.9424 | LR: 1.18e-05 | 1342.4ms/step | 14.9 samples/s +Validation at step 39750: Loss = 2.9310, Perplexity = 18.75 +Step 39760/44000 | Loss: 2.9462 | LR: 1.18e-05 | 3587.7ms/step | 5.6 samples/s +Step 39770/44000 | Loss: 2.9094 | LR: 1.17e-05 | 1342.1ms/step | 14.9 samples/s +Step 39780/44000 | Loss: 2.9123 | LR: 1.17e-05 | 1340.6ms/step | 14.9 samples/s +Step 39790/44000 | Loss: 2.8981 | LR: 1.16e-05 | 1341.3ms/step | 14.9 samples/s +Step 39800/44000 | Loss: 2.9145 | LR: 1.15e-05 | 1343.3ms/step | 14.9 samples/s +Step 39810/44000 | Loss: 2.9002 | LR: 1.15e-05 | 1345.3ms/step | 14.9 samples/s +Step 39820/44000 | Loss: 2.9101 | LR: 1.14e-05 | 1341.2ms/step | 14.9 samples/s +Step 39830/44000 | Loss: 2.9200 | LR: 1.14e-05 | 1342.1ms/step | 14.9 samples/s +Step 39840/44000 | Loss: 2.9347 | LR: 1.13e-05 | 1343.7ms/step | 14.9 samples/s +Step 39850/44000 | Loss: 2.9260 | LR: 1.13e-05 | 1343.6ms/step | 14.9 samples/s +Step 39860/44000 | Loss: 2.9169 | LR: 1.12e-05 | 1344.0ms/step | 14.9 samples/s +Step 39870/44000 | Loss: 2.9101 | LR: 1.12e-05 | 1342.5ms/step | 14.9 samples/s +Step 39880/44000 | Loss: 2.8879 | LR: 1.11e-05 | 1342.4ms/step | 14.9 samples/s +Step 39890/44000 | Loss: 2.9031 | LR: 1.11e-05 | 1340.2ms/step | 14.9 samples/s +Step 39900/44000 | Loss: 2.9176 | LR: 1.10e-05 | 1340.5ms/step | 14.9 samples/s +Step 39910/44000 | Loss: 2.8855 | LR: 1.10e-05 | 1341.5ms/step | 14.9 samples/s +Step 39920/44000 | Loss: 2.9427 | LR: 1.09e-05 | 1345.3ms/step | 14.9 samples/s +Step 39930/44000 | Loss: 2.9155 | LR: 1.08e-05 | 1350.9ms/step | 14.8 samples/s +Step 39940/44000 | Loss: 2.9381 | LR: 1.08e-05 | 1346.8ms/step | 14.8 samples/s +Step 39950/44000 | Loss: 2.8997 | LR: 1.07e-05 | 1343.0ms/step | 14.9 samples/s +Step 39960/44000 | Loss: 2.9187 | LR: 1.07e-05 | 1343.4ms/step | 14.9 samples/s +Step 39970/44000 | Loss: 2.9143 | LR: 1.06e-05 | 1342.3ms/step | 14.9 samples/s +Step 39980/44000 | Loss: 2.9032 | LR: 1.06e-05 | 1343.1ms/step | 14.9 samples/s +Step 39990/44000 | Loss: 2.9274 | LR: 1.05e-05 | 1341.6ms/step | 14.9 samples/s +Step 40000/44000 | Loss: 2.8984 | LR: 1.05e-05 | 1343.2ms/step | 14.9 samples/s +Validation at step 40000: Loss = 2.9310, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-40000 +Step 40010/44000 | Loss: 2.9161 | LR: 1.04e-05 | 3889.2ms/step | 5.1 samples/s +Step 40020/44000 | Loss: 2.9179 | LR: 1.04e-05 | 1345.7ms/step | 14.9 samples/s +Step 40030/44000 | Loss: 2.9279 | LR: 1.03e-05 | 1344.5ms/step | 14.9 samples/s +Step 40040/44000 | Loss: 2.9078 | LR: 1.03e-05 | 1342.8ms/step | 14.9 samples/s +Step 40050/44000 | Loss: 2.8897 | LR: 1.02e-05 | 1341.0ms/step | 14.9 samples/s +Step 40060/44000 | Loss: 2.8990 | LR: 1.02e-05 | 1343.0ms/step | 14.9 samples/s +Step 40070/44000 | Loss: 2.9220 | LR: 1.01e-05 | 1340.9ms/step | 14.9 samples/s +Step 40080/44000 | Loss: 2.9258 | LR: 1.01e-05 | 1348.6ms/step | 14.8 samples/s +Step 40090/44000 | Loss: 2.9060 | LR: 1.00e-05 | 1347.9ms/step | 14.8 samples/s +Step 40100/44000 | Loss: 2.9144 | LR: 9.96e-06 | 1345.6ms/step | 14.9 samples/s +Step 40110/44000 | Loss: 2.9142 | LR: 9.91e-06 | 1344.1ms/step | 14.9 samples/s +Step 40120/44000 | Loss: 2.9286 | LR: 9.86e-06 | 1344.8ms/step | 14.9 samples/s +Step 40130/44000 | Loss: 2.9221 | LR: 9.81e-06 | 1345.9ms/step | 14.9 samples/s +Step 40140/44000 | Loss: 2.9272 | LR: 9.76e-06 | 1345.9ms/step | 14.9 samples/s +Step 40150/44000 | Loss: 2.8862 | LR: 9.71e-06 | 1347.3ms/step | 14.8 samples/s +Step 40160/44000 | Loss: 2.8933 | LR: 9.66e-06 | 1350.3ms/step | 14.8 samples/s +Step 40170/44000 | Loss: 2.8985 | LR: 9.61e-06 | 1349.5ms/step | 14.8 samples/s +Step 40180/44000 | Loss: 2.9380 | LR: 9.56e-06 | 1351.7ms/step | 14.8 samples/s +Step 40190/44000 | Loss: 2.9149 | LR: 9.51e-06 | 1348.8ms/step | 14.8 samples/s +Step 40200/44000 | Loss: 2.9459 | LR: 9.46e-06 | 1347.8ms/step | 14.8 samples/s +Step 40210/44000 | Loss: 2.9120 | LR: 9.41e-06 | 1349.1ms/step | 14.8 samples/s +Step 40220/44000 | Loss: 2.9317 | LR: 9.36e-06 | 1346.4ms/step | 14.9 samples/s +Step 40230/44000 | Loss: 2.9042 | LR: 9.32e-06 | 1343.9ms/step | 14.9 samples/s +Step 40240/44000 | Loss: 2.9137 | LR: 9.27e-06 | 1346.4ms/step | 14.9 samples/s +Step 40250/44000 | Loss: 2.8768 | LR: 9.22e-06 | 1349.1ms/step | 14.8 samples/s +Validation at step 40250: Loss = 2.9306, Perplexity = 18.74 +Step 40260/44000 | Loss: 2.9048 | LR: 9.17e-06 | 3604.7ms/step | 5.5 samples/s +Step 40270/44000 | Loss: 2.9227 | LR: 9.12e-06 | 1342.1ms/step | 14.9 samples/s +Step 40280/44000 | Loss: 2.9185 | LR: 9.07e-06 | 1344.9ms/step | 14.9 samples/s +Step 40290/44000 | Loss: 2.8944 | LR: 9.02e-06 | 1345.8ms/step | 14.9 samples/s +Step 40300/44000 | Loss: 2.9214 | LR: 8.97e-06 | 1344.7ms/step | 14.9 samples/s +Step 40310/44000 | Loss: 2.9094 | LR: 8.93e-06 | 1347.9ms/step | 14.8 samples/s +Step 40320/44000 | Loss: 2.9381 | LR: 8.88e-06 | 1346.7ms/step | 14.9 samples/s +Step 40330/44000 | Loss: 2.9158 | LR: 8.83e-06 | 1349.0ms/step | 14.8 samples/s +Step 40340/44000 | Loss: 2.9151 | LR: 8.78e-06 | 1349.4ms/step | 14.8 samples/s +Step 40350/44000 | Loss: 2.9055 | LR: 8.74e-06 | 1349.3ms/step | 14.8 samples/s +Step 40360/44000 | Loss: 2.9194 | LR: 8.69e-06 | 1347.4ms/step | 14.8 samples/s +Step 40370/44000 | Loss: 2.8972 | LR: 8.64e-06 | 1350.0ms/step | 14.8 samples/s +Step 40380/44000 | Loss: 2.9167 | LR: 8.59e-06 | 1349.2ms/step | 14.8 samples/s +Step 40390/44000 | Loss: 2.9337 | LR: 8.55e-06 | 1348.1ms/step | 14.8 samples/s +Step 40400/44000 | Loss: 2.9007 | LR: 8.50e-06 | 1348.3ms/step | 14.8 samples/s +Step 40410/44000 | Loss: 2.8859 | LR: 8.45e-06 | 1348.0ms/step | 14.8 samples/s +Step 40420/44000 | Loss: 2.9096 | LR: 8.41e-06 | 1345.5ms/step | 14.9 samples/s +Step 40430/44000 | Loss: 2.9195 | LR: 8.36e-06 | 1345.8ms/step | 14.9 samples/s +Step 40440/44000 | Loss: 2.8960 | LR: 8.31e-06 | 1345.4ms/step | 14.9 samples/s +Step 40450/44000 | Loss: 2.9087 | LR: 8.27e-06 | 1344.8ms/step | 14.9 samples/s +Step 40460/44000 | Loss: 2.8761 | LR: 8.22e-06 | 1349.2ms/step | 14.8 samples/s +Step 40470/44000 | Loss: 2.9323 | LR: 8.17e-06 | 1347.6ms/step | 14.8 samples/s +Step 40480/44000 | Loss: 2.9203 | LR: 8.13e-06 | 1346.6ms/step | 14.9 samples/s +Step 40490/44000 | Loss: 2.9198 | LR: 8.08e-06 | 1345.8ms/step | 14.9 samples/s +Step 40500/44000 | Loss: 2.9145 | LR: 8.04e-06 | 1346.7ms/step | 14.9 samples/s +Validation at step 40500: Loss = 2.9310, Perplexity = 18.75 +Step 40510/44000 | Loss: 2.9019 | LR: 7.99e-06 | 3611.7ms/step | 5.5 samples/s +Step 40520/44000 | Loss: 2.9485 | LR: 7.94e-06 | 1346.3ms/step | 14.9 samples/s +Step 40530/44000 | Loss: 2.9369 | LR: 7.90e-06 | 1349.4ms/step | 14.8 samples/s +Step 40540/44000 | Loss: 2.9301 | LR: 7.85e-06 | 1347.3ms/step | 14.8 samples/s +Step 40550/44000 | Loss: 2.8878 | LR: 7.81e-06 | 1345.7ms/step | 14.9 samples/s +Step 40560/44000 | Loss: 2.8876 | LR: 7.76e-06 | 1347.0ms/step | 14.8 samples/s +Step 40570/44000 | Loss: 2.9217 | LR: 7.72e-06 | 1344.8ms/step | 14.9 samples/s +Step 40580/44000 | Loss: 2.9137 | LR: 7.67e-06 | 1343.8ms/step | 14.9 samples/s +Step 40590/44000 | Loss: 2.8904 | LR: 7.63e-06 | 1343.8ms/step | 14.9 samples/s +Step 40600/44000 | Loss: 2.9187 | LR: 7.59e-06 | 1347.5ms/step | 14.8 samples/s +Step 40610/44000 | Loss: 2.9472 | LR: 7.54e-06 | 1349.6ms/step | 14.8 samples/s +Step 40620/44000 | Loss: 2.9143 | LR: 7.50e-06 | 1350.5ms/step | 14.8 samples/s +Step 40630/44000 | Loss: 2.9291 | LR: 7.45e-06 | 1346.4ms/step | 14.9 samples/s +Step 40640/44000 | Loss: 2.9351 | LR: 7.41e-06 | 1346.6ms/step | 14.9 samples/s +Step 40650/44000 | Loss: 2.9337 | LR: 7.37e-06 | 1347.6ms/step | 14.8 samples/s +Step 40660/44000 | Loss: 2.9227 | LR: 7.32e-06 | 1347.2ms/step | 14.8 samples/s +Step 40670/44000 | Loss: 2.9312 | LR: 7.28e-06 | 1344.4ms/step | 14.9 samples/s +Step 40680/44000 | Loss: 2.9166 | LR: 7.23e-06 | 1342.0ms/step | 14.9 samples/s +Step 40690/44000 | Loss: 2.9275 | LR: 7.19e-06 | 1339.9ms/step | 14.9 samples/s +Step 40700/44000 | Loss: 2.9055 | LR: 7.15e-06 | 1338.8ms/step | 14.9 samples/s +Step 40710/44000 | Loss: 2.9202 | LR: 7.10e-06 | 1343.8ms/step | 14.9 samples/s +Step 40720/44000 | Loss: 2.9281 | LR: 7.06e-06 | 1345.9ms/step | 14.9 samples/s +Step 40730/44000 | Loss: 2.9354 | LR: 7.02e-06 | 1344.2ms/step | 14.9 samples/s +Step 40740/44000 | Loss: 2.9072 | LR: 6.98e-06 | 1348.1ms/step | 14.8 samples/s +Step 40750/44000 | Loss: 2.9039 | LR: 6.93e-06 | 1344.6ms/step | 14.9 samples/s +Validation at step 40750: Loss = 2.9308, Perplexity = 18.74 +Step 40760/44000 | Loss: 2.9316 | LR: 6.89e-06 | 3611.6ms/step | 5.5 samples/s +Step 40770/44000 | Loss: 2.9134 | LR: 6.85e-06 | 1345.0ms/step | 14.9 samples/s +Step 40780/44000 | Loss: 2.9317 | LR: 6.81e-06 | 1348.8ms/step | 14.8 samples/s +Step 40790/44000 | Loss: 2.9351 | LR: 6.77e-06 | 1352.0ms/step | 14.8 samples/s +Step 40800/44000 | Loss: 2.9379 | LR: 6.72e-06 | 1346.1ms/step | 14.9 samples/s +Step 40810/44000 | Loss: 2.9179 | LR: 6.68e-06 | 1343.7ms/step | 14.9 samples/s +Step 40820/44000 | Loss: 2.8962 | LR: 6.64e-06 | 1344.6ms/step | 14.9 samples/s +Step 40830/44000 | Loss: 2.8916 | LR: 6.60e-06 | 1348.8ms/step | 14.8 samples/s +Step 40840/44000 | Loss: 2.9360 | LR: 6.56e-06 | 1347.5ms/step | 14.8 samples/s +Step 40850/44000 | Loss: 2.9226 | LR: 6.52e-06 | 1346.9ms/step | 14.8 samples/s +Step 40860/44000 | Loss: 2.9355 | LR: 6.47e-06 | 1348.2ms/step | 14.8 samples/s +Step 40870/44000 | Loss: 2.9388 | LR: 6.43e-06 | 1347.4ms/step | 14.8 samples/s +Step 40880/44000 | Loss: 2.9092 | LR: 6.39e-06 | 1344.9ms/step | 14.9 samples/s +Step 40890/44000 | Loss: 2.9274 | LR: 6.35e-06 | 1347.8ms/step | 14.8 samples/s +Step 40900/44000 | Loss: 2.9034 | LR: 6.31e-06 | 1346.9ms/step | 14.8 samples/s +Step 40910/44000 | Loss: 2.9720 | LR: 6.27e-06 | 1349.6ms/step | 14.8 samples/s +Step 40920/44000 | Loss: 2.9215 | LR: 6.23e-06 | 1348.8ms/step | 14.8 samples/s +Step 40930/44000 | Loss: 2.9296 | LR: 6.19e-06 | 1349.8ms/step | 14.8 samples/s +Step 40940/44000 | Loss: 2.9316 | LR: 6.15e-06 | 1350.1ms/step | 14.8 samples/s +Step 40950/44000 | Loss: 2.8941 | LR: 6.11e-06 | 1349.1ms/step | 14.8 samples/s +Step 40960/44000 | Loss: 2.8812 | LR: 6.07e-06 | 1348.3ms/step | 14.8 samples/s +Step 40970/44000 | Loss: 2.9104 | LR: 6.03e-06 | 1347.4ms/step | 14.8 samples/s +Step 40980/44000 | Loss: 2.8992 | LR: 5.99e-06 | 1345.8ms/step | 14.9 samples/s +Step 40990/44000 | Loss: 2.9199 | LR: 5.95e-06 | 1351.3ms/step | 14.8 samples/s +Step 41000/44000 | Loss: 2.9238 | LR: 5.91e-06 | 1350.1ms/step | 14.8 samples/s +Validation at step 41000: Loss = 2.9309, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-41000 +Step 41010/44000 | Loss: 2.9328 | LR: 5.87e-06 | 3940.3ms/step | 5.1 samples/s +Step 41020/44000 | Loss: 2.9395 | LR: 5.83e-06 | 1345.6ms/step | 14.9 samples/s +Step 41030/44000 | Loss: 2.9149 | LR: 5.80e-06 | 1344.2ms/step | 14.9 samples/s +Step 41040/44000 | Loss: 2.9145 | LR: 5.76e-06 | 1348.0ms/step | 14.8 samples/s +Step 41050/44000 | Loss: 2.9123 | LR: 5.72e-06 | 1349.7ms/step | 14.8 samples/s +Step 41060/44000 | Loss: 2.8828 | LR: 5.68e-06 | 1345.5ms/step | 14.9 samples/s +Step 41070/44000 | Loss: 2.9102 | LR: 5.64e-06 | 1345.2ms/step | 14.9 samples/s +Step 41080/44000 | Loss: 2.9425 | LR: 5.60e-06 | 1342.0ms/step | 14.9 samples/s +Step 41090/44000 | Loss: 2.9598 | LR: 5.56e-06 | 1342.5ms/step | 14.9 samples/s +Step 41100/44000 | Loss: 2.9367 | LR: 5.53e-06 | 1342.0ms/step | 14.9 samples/s +Step 41110/44000 | Loss: 2.9600 | LR: 5.49e-06 | 1343.9ms/step | 14.9 samples/s +Step 41120/44000 | Loss: 2.9458 | LR: 5.45e-06 | 1343.7ms/step | 14.9 samples/s +Step 41130/44000 | Loss: 2.9166 | LR: 5.41e-06 | 1345.6ms/step | 14.9 samples/s +Step 41140/44000 | Loss: 2.8949 | LR: 5.38e-06 | 1347.9ms/step | 14.8 samples/s +Step 41150/44000 | Loss: 2.9368 | LR: 5.34e-06 | 1346.7ms/step | 14.9 samples/s +Step 41160/44000 | Loss: 2.9092 | LR: 5.30e-06 | 1343.8ms/step | 14.9 samples/s +Step 41170/44000 | Loss: 2.9204 | LR: 5.26e-06 | 1346.2ms/step | 14.9 samples/s +Step 41180/44000 | Loss: 2.8966 | LR: 5.23e-06 | 1346.4ms/step | 14.9 samples/s +Step 41190/44000 | Loss: 2.8786 | LR: 5.19e-06 | 1346.3ms/step | 14.9 samples/s +Step 41200/44000 | Loss: 2.9205 | LR: 5.15e-06 | 1345.1ms/step | 14.9 samples/s +Step 41210/44000 | Loss: 2.9289 | LR: 5.12e-06 | 1345.6ms/step | 14.9 samples/s +Step 41220/44000 | Loss: 2.9361 | LR: 5.08e-06 | 1345.9ms/step | 14.9 samples/s +Step 41230/44000 | Loss: 2.9103 | LR: 5.04e-06 | 1343.6ms/step | 14.9 samples/s +Step 41240/44000 | Loss: 2.8979 | LR: 5.01e-06 | 1341.9ms/step | 14.9 samples/s +Step 41250/44000 | Loss: 2.9162 | LR: 4.97e-06 | 1342.2ms/step | 14.9 samples/s +Validation at step 41250: Loss = 2.9309, Perplexity = 18.75 +Step 41260/44000 | Loss: 2.9419 | LR: 4.94e-06 | 3593.4ms/step | 5.6 samples/s +Step 41270/44000 | Loss: 2.9210 | LR: 4.90e-06 | 1344.4ms/step | 14.9 samples/s +Step 41280/44000 | Loss: 2.9178 | LR: 4.86e-06 | 1345.0ms/step | 14.9 samples/s +Step 41290/44000 | Loss: 2.9085 | LR: 4.83e-06 | 1347.1ms/step | 14.8 samples/s +Step 41300/44000 | Loss: 2.9129 | LR: 4.79e-06 | 1347.4ms/step | 14.8 samples/s +Step 41310/44000 | Loss: 2.8995 | LR: 4.76e-06 | 1345.3ms/step | 14.9 samples/s +Step 41320/44000 | Loss: 2.9248 | LR: 4.72e-06 | 1344.1ms/step | 14.9 samples/s +Step 41330/44000 | Loss: 2.9090 | LR: 4.69e-06 | 1343.4ms/step | 14.9 samples/s +Step 41340/44000 | Loss: 2.9322 | LR: 4.65e-06 | 1340.7ms/step | 14.9 samples/s +Step 41350/44000 | Loss: 2.8964 | LR: 4.62e-06 | 1345.7ms/step | 14.9 samples/s +Step 41360/44000 | Loss: 2.9450 | LR: 4.58e-06 | 1348.9ms/step | 14.8 samples/s +Step 41370/44000 | Loss: 2.9267 | LR: 4.55e-06 | 1345.5ms/step | 14.9 samples/s +Step 41380/44000 | Loss: 2.9200 | LR: 4.51e-06 | 1348.8ms/step | 14.8 samples/s +Step 41390/44000 | Loss: 2.8998 | LR: 4.48e-06 | 1343.4ms/step | 14.9 samples/s +Step 41400/44000 | Loss: 2.8872 | LR: 4.45e-06 | 1342.0ms/step | 14.9 samples/s +Step 41410/44000 | Loss: 2.8928 | LR: 4.41e-06 | 1342.0ms/step | 14.9 samples/s +Step 41420/44000 | Loss: 2.8858 | LR: 4.38e-06 | 1343.8ms/step | 14.9 samples/s +Step 41430/44000 | Loss: 2.9106 | LR: 4.34e-06 | 1344.0ms/step | 14.9 samples/s +Step 41440/44000 | Loss: 2.9508 | LR: 4.31e-06 | 1344.5ms/step | 14.9 samples/s +Step 41450/44000 | Loss: 2.9160 | LR: 4.28e-06 | 1346.0ms/step | 14.9 samples/s +Step 41460/44000 | Loss: 2.9173 | LR: 4.24e-06 | 1345.3ms/step | 14.9 samples/s +Step 41470/44000 | Loss: 2.9038 | LR: 4.21e-06 | 1348.5ms/step | 14.8 samples/s +Step 41480/44000 | Loss: 2.9331 | LR: 4.18e-06 | 1347.1ms/step | 14.8 samples/s +Step 41490/44000 | Loss: 2.9014 | LR: 4.14e-06 | 1346.3ms/step | 14.9 samples/s +Step 41500/44000 | Loss: 2.9170 | LR: 4.11e-06 | 1346.0ms/step | 14.9 samples/s +Validation at step 41500: Loss = 2.9308, Perplexity = 18.74 +Step 41510/44000 | Loss: 2.9130 | LR: 4.08e-06 | 3607.9ms/step | 5.5 samples/s +Step 41520/44000 | Loss: 2.9137 | LR: 4.05e-06 | 1342.3ms/step | 14.9 samples/s +Step 41530/44000 | Loss: 2.9194 | LR: 4.01e-06 | 1341.7ms/step | 14.9 samples/s +Step 41540/44000 | Loss: 2.9420 | LR: 3.98e-06 | 1343.4ms/step | 14.9 samples/s +Step 41550/44000 | Loss: 2.9148 | LR: 3.95e-06 | 1345.8ms/step | 14.9 samples/s +Step 41560/44000 | Loss: 2.9274 | LR: 3.92e-06 | 1346.5ms/step | 14.9 samples/s +Step 41570/44000 | Loss: 2.9450 | LR: 3.88e-06 | 1345.5ms/step | 14.9 samples/s +Step 41580/44000 | Loss: 2.9003 | LR: 3.85e-06 | 1343.1ms/step | 14.9 samples/s +Step 41590/44000 | Loss: 2.9142 | LR: 3.82e-06 | 1343.6ms/step | 14.9 samples/s +Step 41600/44000 | Loss: 2.9274 | LR: 3.79e-06 | 1346.0ms/step | 14.9 samples/s +Step 41610/44000 | Loss: 2.9051 | LR: 3.76e-06 | 1348.7ms/step | 14.8 samples/s +Step 41620/44000 | Loss: 2.8808 | LR: 3.73e-06 | 1349.9ms/step | 14.8 samples/s +Step 41630/44000 | Loss: 2.9386 | LR: 3.70e-06 | 1347.3ms/step | 14.8 samples/s +Step 41640/44000 | Loss: 2.9387 | LR: 3.66e-06 | 1344.5ms/step | 14.9 samples/s +Step 41650/44000 | Loss: 2.9203 | LR: 3.63e-06 | 1346.9ms/step | 14.8 samples/s +Step 41660/44000 | Loss: 2.9410 | LR: 3.60e-06 | 1347.2ms/step | 14.8 samples/s +Step 41670/44000 | Loss: 2.9135 | LR: 3.57e-06 | 1349.3ms/step | 14.8 samples/s +Step 41680/44000 | Loss: 2.8866 | LR: 3.54e-06 | 1348.6ms/step | 14.8 samples/s +Step 41690/44000 | Loss: 2.8997 | LR: 3.51e-06 | 1349.4ms/step | 14.8 samples/s +Step 41700/44000 | Loss: 2.9349 | LR: 3.48e-06 | 1350.6ms/step | 14.8 samples/s +Step 41710/44000 | Loss: 2.9074 | LR: 3.45e-06 | 1346.7ms/step | 14.9 samples/s +Step 41720/44000 | Loss: 2.9113 | LR: 3.42e-06 | 1348.7ms/step | 14.8 samples/s +Step 41730/44000 | Loss: 2.9229 | LR: 3.39e-06 | 1348.3ms/step | 14.8 samples/s +Step 41740/44000 | Loss: 2.9341 | LR: 3.36e-06 | 1347.8ms/step | 14.8 samples/s +Step 41750/44000 | Loss: 2.9088 | LR: 3.33e-06 | 1347.1ms/step | 14.8 samples/s +Validation at step 41750: Loss = 2.9306, Perplexity = 18.74 +Step 41760/44000 | Loss: 2.9424 | LR: 3.30e-06 | 3606.1ms/step | 5.5 samples/s +Step 41770/44000 | Loss: 2.9154 | LR: 3.27e-06 | 1349.2ms/step | 14.8 samples/s +Step 41780/44000 | Loss: 2.9090 | LR: 3.24e-06 | 1351.2ms/step | 14.8 samples/s +Step 41790/44000 | Loss: 2.9512 | LR: 3.21e-06 | 1348.7ms/step | 14.8 samples/s +Step 41800/44000 | Loss: 2.9625 | LR: 3.19e-06 | 1346.0ms/step | 14.9 samples/s +Step 41810/44000 | Loss: 2.9266 | LR: 3.16e-06 | 1344.7ms/step | 14.9 samples/s +Step 41820/44000 | Loss: 2.9026 | LR: 3.13e-06 | 1346.6ms/step | 14.9 samples/s +Step 41830/44000 | Loss: 2.9032 | LR: 3.10e-06 | 1346.2ms/step | 14.9 samples/s +Step 41840/44000 | Loss: 2.8853 | LR: 3.07e-06 | 1349.4ms/step | 14.8 samples/s +Step 41850/44000 | Loss: 2.8922 | LR: 3.04e-06 | 1350.7ms/step | 14.8 samples/s +Step 41860/44000 | Loss: 2.9388 | LR: 3.01e-06 | 1350.3ms/step | 14.8 samples/s +Step 41870/44000 | Loss: 2.9489 | LR: 2.99e-06 | 1348.9ms/step | 14.8 samples/s +Step 41880/44000 | Loss: 2.9277 | LR: 2.96e-06 | 1348.7ms/step | 14.8 samples/s +Step 41890/44000 | Loss: 2.9102 | LR: 2.93e-06 | 1346.8ms/step | 14.8 samples/s +Step 41900/44000 | Loss: 2.9120 | LR: 2.90e-06 | 1347.7ms/step | 14.8 samples/s +Step 41910/44000 | Loss: 2.9054 | LR: 2.88e-06 | 1347.7ms/step | 14.8 samples/s +Step 41920/44000 | Loss: 2.9572 | LR: 2.85e-06 | 1349.8ms/step | 14.8 samples/s +Step 41930/44000 | Loss: 2.9091 | LR: 2.82e-06 | 1346.6ms/step | 14.9 samples/s +Step 41940/44000 | Loss: 2.9182 | LR: 2.79e-06 | 1344.0ms/step | 14.9 samples/s +Step 41950/44000 | Loss: 2.8794 | LR: 2.77e-06 | 1345.6ms/step | 14.9 samples/s +Step 41960/44000 | Loss: 2.9271 | LR: 2.74e-06 | 1346.4ms/step | 14.9 samples/s +Step 41970/44000 | Loss: 2.9213 | LR: 2.71e-06 | 1352.0ms/step | 14.8 samples/s +Step 41980/44000 | Loss: 2.9147 | LR: 2.69e-06 | 1349.3ms/step | 14.8 samples/s +Step 41990/44000 | Loss: 2.9186 | LR: 2.66e-06 | 1346.1ms/step | 14.9 samples/s +Step 42000/44000 | Loss: 2.9152 | LR: 2.63e-06 | 1346.1ms/step | 14.9 samples/s +Validation at step 42000: Loss = 2.9307, Perplexity = 18.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-42000 +Step 42010/44000 | Loss: 2.9258 | LR: 2.61e-06 | 3915.4ms/step | 5.1 samples/s +Step 42020/44000 | Loss: 2.9346 | LR: 2.58e-06 | 1347.9ms/step | 14.8 samples/s +Step 42030/44000 | Loss: 2.9402 | LR: 2.56e-06 | 1346.6ms/step | 14.9 samples/s +Step 42040/44000 | Loss: 2.9378 | LR: 2.53e-06 | 1343.9ms/step | 14.9 samples/s +Step 42050/44000 | Loss: 2.8967 | LR: 2.50e-06 | 1345.5ms/step | 14.9 samples/s +Step 42060/44000 | Loss: 2.9321 | LR: 2.48e-06 | 1345.9ms/step | 14.9 samples/s +Step 42070/44000 | Loss: 2.9334 | LR: 2.45e-06 | 1346.6ms/step | 14.9 samples/s +Step 42080/44000 | Loss: 2.9131 | LR: 2.43e-06 | 1345.5ms/step | 14.9 samples/s +Step 42090/44000 | Loss: 2.8984 | LR: 2.40e-06 | 1347.4ms/step | 14.8 samples/s +Step 42100/44000 | Loss: 2.9213 | LR: 2.38e-06 | 1347.1ms/step | 14.8 samples/s +Step 42110/44000 | Loss: 2.9333 | LR: 2.35e-06 | 1344.8ms/step | 14.9 samples/s +Step 42120/44000 | Loss: 2.9185 | LR: 2.33e-06 | 1341.7ms/step | 14.9 samples/s +Step 42130/44000 | Loss: 2.9317 | LR: 2.30e-06 | 1344.8ms/step | 14.9 samples/s +Step 42140/44000 | Loss: 2.9290 | LR: 2.28e-06 | 1346.9ms/step | 14.8 samples/s +Step 42150/44000 | Loss: 2.9484 | LR: 2.25e-06 | 1343.9ms/step | 14.9 samples/s +Step 42160/44000 | Loss: 2.9148 | LR: 2.23e-06 | 1341.3ms/step | 14.9 samples/s +Step 42170/44000 | Loss: 2.9015 | LR: 2.21e-06 | 1344.3ms/step | 14.9 samples/s +Step 42180/44000 | Loss: 2.9312 | LR: 2.18e-06 | 1344.6ms/step | 14.9 samples/s +Step 42190/44000 | Loss: 2.9137 | LR: 2.16e-06 | 1343.5ms/step | 14.9 samples/s +Step 42200/44000 | Loss: 2.8992 | LR: 2.13e-06 | 1343.9ms/step | 14.9 samples/s +Step 42210/44000 | Loss: 2.9450 | LR: 2.11e-06 | 1346.8ms/step | 14.9 samples/s +Step 42220/44000 | Loss: 2.8847 | LR: 2.09e-06 | 1347.5ms/step | 14.8 samples/s +Step 42230/44000 | Loss: 2.9388 | LR: 2.06e-06 | 1348.8ms/step | 14.8 samples/s +Step 42240/44000 | Loss: 2.9166 | LR: 2.04e-06 | 1344.2ms/step | 14.9 samples/s +Step 42250/44000 | Loss: 2.9317 | LR: 2.02e-06 | 1345.9ms/step | 14.9 samples/s +Validation at step 42250: Loss = 2.9311, Perplexity = 18.75 +Step 42260/44000 | Loss: 2.9227 | LR: 1.99e-06 | 3613.3ms/step | 5.5 samples/s +Step 42270/44000 | Loss: 2.9169 | LR: 1.97e-06 | 1347.8ms/step | 14.8 samples/s +Step 42280/44000 | Loss: 2.9191 | LR: 1.95e-06 | 1344.3ms/step | 14.9 samples/s +Step 42290/44000 | Loss: 2.8863 | LR: 1.93e-06 | 1345.8ms/step | 14.9 samples/s +Step 42300/44000 | Loss: 2.9141 | LR: 1.90e-06 | 1346.0ms/step | 14.9 samples/s +Step 42310/44000 | Loss: 2.8887 | LR: 1.88e-06 | 1344.4ms/step | 14.9 samples/s +Step 42320/44000 | Loss: 2.9345 | LR: 1.86e-06 | 1344.8ms/step | 14.9 samples/s +Step 42330/44000 | Loss: 2.9008 | LR: 1.84e-06 | 1343.7ms/step | 14.9 samples/s +Step 42340/44000 | Loss: 2.9455 | LR: 1.82e-06 | 1344.5ms/step | 14.9 samples/s +Step 42350/44000 | Loss: 2.8755 | LR: 1.79e-06 | 1344.1ms/step | 14.9 samples/s +Step 42360/44000 | Loss: 2.9182 | LR: 1.77e-06 | 1345.2ms/step | 14.9 samples/s +Step 42370/44000 | Loss: 2.9176 | LR: 1.75e-06 | 1347.1ms/step | 14.8 samples/s +Step 42380/44000 | Loss: 2.9511 | LR: 1.73e-06 | 1342.5ms/step | 14.9 samples/s +Step 42390/44000 | Loss: 2.8829 | LR: 1.71e-06 | 1338.9ms/step | 14.9 samples/s +Step 42400/44000 | Loss: 2.9105 | LR: 1.69e-06 | 1341.8ms/step | 14.9 samples/s +Step 42410/44000 | Loss: 2.9151 | LR: 1.67e-06 | 1346.2ms/step | 14.9 samples/s +Step 42420/44000 | Loss: 2.9424 | LR: 1.64e-06 | 1342.9ms/step | 14.9 samples/s +Step 42430/44000 | Loss: 2.9452 | LR: 1.62e-06 | 1343.6ms/step | 14.9 samples/s +Step 42440/44000 | Loss: 2.9351 | LR: 1.60e-06 | 1342.8ms/step | 14.9 samples/s +Step 42450/44000 | Loss: 2.9050 | LR: 1.58e-06 | 1347.5ms/step | 14.8 samples/s +Step 42460/44000 | Loss: 2.9162 | LR: 1.56e-06 | 1345.1ms/step | 14.9 samples/s +Step 42470/44000 | Loss: 2.9499 | LR: 1.54e-06 | 1342.8ms/step | 14.9 samples/s +Step 42480/44000 | Loss: 2.9168 | LR: 1.52e-06 | 1343.2ms/step | 14.9 samples/s +Step 42490/44000 | Loss: 2.9245 | LR: 1.50e-06 | 1344.0ms/step | 14.9 samples/s +Step 42500/44000 | Loss: 2.8942 | LR: 1.48e-06 | 1345.0ms/step | 14.9 samples/s +Validation at step 42500: Loss = 2.9309, Perplexity = 18.75 +Step 42510/44000 | Loss: 2.9359 | LR: 1.46e-06 | 3607.5ms/step | 5.5 samples/s +Step 42520/44000 | Loss: 2.8946 | LR: 1.44e-06 | 1343.8ms/step | 14.9 samples/s +Step 42530/44000 | Loss: 2.9232 | LR: 1.42e-06 | 1345.0ms/step | 14.9 samples/s +Step 42540/44000 | Loss: 2.9107 | LR: 1.40e-06 | 1344.7ms/step | 14.9 samples/s +Step 42550/44000 | Loss: 2.9280 | LR: 1.39e-06 | 1345.9ms/step | 14.9 samples/s +Step 42560/44000 | Loss: 2.8994 | LR: 1.37e-06 | 1345.0ms/step | 14.9 samples/s +Step 42570/44000 | Loss: 2.9183 | LR: 1.35e-06 | 1344.5ms/step | 14.9 samples/s +Step 42580/44000 | Loss: 2.9175 | LR: 1.33e-06 | 1348.1ms/step | 14.8 samples/s +Step 42590/44000 | Loss: 2.9030 | LR: 1.31e-06 | 1346.5ms/step | 14.9 samples/s +Step 42600/44000 | Loss: 2.8974 | LR: 1.29e-06 | 1346.4ms/step | 14.9 samples/s +Step 42610/44000 | Loss: 2.9244 | LR: 1.27e-06 | 1345.8ms/step | 14.9 samples/s +Step 42620/44000 | Loss: 2.9328 | LR: 1.25e-06 | 1345.7ms/step | 14.9 samples/s +Step 42630/44000 | Loss: 2.9021 | LR: 1.24e-06 | 1345.2ms/step | 14.9 samples/s +Step 42640/44000 | Loss: 2.9106 | LR: 1.22e-06 | 1345.8ms/step | 14.9 samples/s +Step 42650/44000 | Loss: 2.9325 | LR: 1.20e-06 | 1345.4ms/step | 14.9 samples/s +Step 42660/44000 | Loss: 2.9029 | LR: 1.18e-06 | 1345.2ms/step | 14.9 samples/s +Step 42670/44000 | Loss: 2.9018 | LR: 1.17e-06 | 1344.1ms/step | 14.9 samples/s +Step 42680/44000 | Loss: 2.8952 | LR: 1.15e-06 | 1341.7ms/step | 14.9 samples/s +Step 42690/44000 | Loss: 2.8881 | LR: 1.13e-06 | 1340.4ms/step | 14.9 samples/s +Step 42700/44000 | Loss: 2.8924 | LR: 1.11e-06 | 1343.8ms/step | 14.9 samples/s +Step 42710/44000 | Loss: 2.9297 | LR: 1.10e-06 | 1345.1ms/step | 14.9 samples/s +Step 42720/44000 | Loss: 2.9230 | LR: 1.08e-06 | 1344.7ms/step | 14.9 samples/s +Step 42730/44000 | Loss: 2.9151 | LR: 1.06e-06 | 1345.7ms/step | 14.9 samples/s +Step 42740/44000 | Loss: 2.8909 | LR: 1.05e-06 | 1346.7ms/step | 14.9 samples/s +Step 42750/44000 | Loss: 2.8990 | LR: 1.03e-06 | 1350.4ms/step | 14.8 samples/s +Validation at step 42750: Loss = 2.9309, Perplexity = 18.74 +Step 42760/44000 | Loss: 2.9149 | LR: 1.01e-06 | 3615.8ms/step | 5.5 samples/s +Step 42770/44000 | Loss: 2.9296 | LR: 9.97e-07 | 1356.7ms/step | 14.7 samples/s +Step 42780/44000 | Loss: 2.9367 | LR: 9.81e-07 | 1350.4ms/step | 14.8 samples/s +Step 42790/44000 | Loss: 2.9085 | LR: 9.65e-07 | 1349.2ms/step | 14.8 samples/s +Step 42800/44000 | Loss: 2.9021 | LR: 9.49e-07 | 1348.3ms/step | 14.8 samples/s +Step 42810/44000 | Loss: 2.8967 | LR: 9.33e-07 | 1348.3ms/step | 14.8 samples/s +Step 42820/44000 | Loss: 2.9534 | LR: 9.18e-07 | 1348.5ms/step | 14.8 samples/s +Step 42830/44000 | Loss: 2.9158 | LR: 9.02e-07 | 1348.0ms/step | 14.8 samples/s +Step 42840/44000 | Loss: 2.9261 | LR: 8.87e-07 | 1347.8ms/step | 14.8 samples/s +Step 42850/44000 | Loss: 2.9252 | LR: 8.72e-07 | 1347.6ms/step | 14.8 samples/s +Step 42860/44000 | Loss: 2.9354 | LR: 8.57e-07 | 1349.2ms/step | 14.8 samples/s +Step 42870/44000 | Loss: 2.9311 | LR: 8.42e-07 | 1348.9ms/step | 14.8 samples/s +Step 42880/44000 | Loss: 2.9189 | LR: 8.27e-07 | 1349.7ms/step | 14.8 samples/s +Step 42890/44000 | Loss: 2.8716 | LR: 8.12e-07 | 1346.6ms/step | 14.9 samples/s +Step 42900/44000 | Loss: 2.9285 | LR: 7.98e-07 | 1348.9ms/step | 14.8 samples/s +Step 42910/44000 | Loss: 2.9206 | LR: 7.83e-07 | 1350.6ms/step | 14.8 samples/s +Step 42920/44000 | Loss: 2.9017 | LR: 7.69e-07 | 1349.2ms/step | 14.8 samples/s +Step 42930/44000 | Loss: 2.8978 | LR: 7.55e-07 | 1348.8ms/step | 14.8 samples/s +Step 42940/44000 | Loss: 2.9087 | LR: 7.41e-07 | 1348.0ms/step | 14.8 samples/s +Step 42950/44000 | Loss: 2.9240 | LR: 7.27e-07 | 1349.5ms/step | 14.8 samples/s +Step 42960/44000 | Loss: 2.9159 | LR: 7.13e-07 | 1348.6ms/step | 14.8 samples/s +Step 42970/44000 | Loss: 2.9089 | LR: 6.99e-07 | 1347.4ms/step | 14.8 samples/s +Step 42980/44000 | Loss: 2.9215 | LR: 6.86e-07 | 1347.7ms/step | 14.8 samples/s +Step 42990/44000 | Loss: 2.9161 | LR: 6.72e-07 | 1347.9ms/step | 14.8 samples/s +Step 43000/44000 | Loss: 2.9089 | LR: 6.59e-07 | 1348.1ms/step | 14.8 samples/s +Validation at step 43000: Loss = 2.9305, Perplexity = 18.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-43000 +Step 43010/44000 | Loss: 2.9137 | LR: 6.46e-07 | 3966.7ms/step | 5.0 samples/s +Step 43020/44000 | Loss: 2.9012 | LR: 6.33e-07 | 1345.0ms/step | 14.9 samples/s +Step 43030/44000 | Loss: 2.9209 | LR: 6.20e-07 | 1346.2ms/step | 14.9 samples/s +Step 43040/44000 | Loss: 2.8840 | LR: 6.08e-07 | 1348.4ms/step | 14.8 samples/s +Step 43050/44000 | Loss: 2.9115 | LR: 5.95e-07 | 1350.2ms/step | 14.8 samples/s +Step 43060/44000 | Loss: 2.9203 | LR: 5.83e-07 | 1348.3ms/step | 14.8 samples/s +Step 43070/44000 | Loss: 2.9359 | LR: 5.70e-07 | 1343.6ms/step | 14.9 samples/s +Step 43080/44000 | Loss: 2.9099 | LR: 5.58e-07 | 1347.7ms/step | 14.8 samples/s +Step 43090/44000 | Loss: 2.8909 | LR: 5.46e-07 | 1353.8ms/step | 14.8 samples/s +Step 43100/44000 | Loss: 2.9055 | LR: 5.34e-07 | 1344.9ms/step | 14.9 samples/s +Step 43110/44000 | Loss: 2.9179 | LR: 5.22e-07 | 1349.3ms/step | 14.8 samples/s +Step 43120/44000 | Loss: 2.9309 | LR: 5.11e-07 | 1346.4ms/step | 14.9 samples/s +Step 43130/44000 | Loss: 2.9337 | LR: 4.99e-07 | 1349.8ms/step | 14.8 samples/s +Step 43140/44000 | Loss: 2.9205 | LR: 4.88e-07 | 1346.9ms/step | 14.8 samples/s +Step 43150/44000 | Loss: 2.9032 | LR: 4.76e-07 | 1347.2ms/step | 14.8 samples/s +Step 43160/44000 | Loss: 2.9015 | LR: 4.65e-07 | 1346.4ms/step | 14.9 samples/s +Step 43170/44000 | Loss: 2.8891 | LR: 4.54e-07 | 1342.6ms/step | 14.9 samples/s +Step 43180/44000 | Loss: 2.9006 | LR: 4.43e-07 | 1342.7ms/step | 14.9 samples/s +Step 43190/44000 | Loss: 2.8964 | LR: 4.33e-07 | 1342.1ms/step | 14.9 samples/s +Step 43200/44000 | Loss: 2.8858 | LR: 4.22e-07 | 1345.0ms/step | 14.9 samples/s +Step 43210/44000 | Loss: 2.9209 | LR: 4.12e-07 | 1344.9ms/step | 14.9 samples/s +Step 43220/44000 | Loss: 2.9090 | LR: 4.01e-07 | 1342.2ms/step | 14.9 samples/s +Step 43230/44000 | Loss: 2.9311 | LR: 3.91e-07 | 1341.7ms/step | 14.9 samples/s +Step 43240/44000 | Loss: 2.9262 | LR: 3.81e-07 | 1343.1ms/step | 14.9 samples/s +Step 43250/44000 | Loss: 2.9296 | LR: 3.71e-07 | 1342.0ms/step | 14.9 samples/s +Validation at step 43250: Loss = 2.9309, Perplexity = 18.74 +Step 43260/44000 | Loss: 2.9235 | LR: 3.61e-07 | 3598.9ms/step | 5.6 samples/s +Step 43270/44000 | Loss: 2.9113 | LR: 3.51e-07 | 1343.9ms/step | 14.9 samples/s +Step 43280/44000 | Loss: 2.9170 | LR: 3.42e-07 | 1344.2ms/step | 14.9 samples/s +Step 43290/44000 | Loss: 2.9015 | LR: 3.32e-07 | 1342.1ms/step | 14.9 samples/s +Step 43300/44000 | Loss: 2.9341 | LR: 3.23e-07 | 1341.4ms/step | 14.9 samples/s +Step 43310/44000 | Loss: 2.9372 | LR: 3.14e-07 | 1343.1ms/step | 14.9 samples/s +Step 43320/44000 | Loss: 2.9422 | LR: 3.05e-07 | 1340.5ms/step | 14.9 samples/s +Step 43330/44000 | Loss: 2.9041 | LR: 2.96e-07 | 1342.8ms/step | 14.9 samples/s +Step 43340/44000 | Loss: 2.9249 | LR: 2.87e-07 | 1344.5ms/step | 14.9 samples/s +Step 43350/44000 | Loss: 2.8999 | LR: 2.79e-07 | 1344.6ms/step | 14.9 samples/s +Step 43360/44000 | Loss: 2.9210 | LR: 2.70e-07 | 1346.0ms/step | 14.9 samples/s +Step 43370/44000 | Loss: 2.9267 | LR: 2.62e-07 | 1344.9ms/step | 14.9 samples/s +Step 43380/44000 | Loss: 2.9198 | LR: 2.53e-07 | 1342.0ms/step | 14.9 samples/s +Step 43390/44000 | Loss: 2.8946 | LR: 2.45e-07 | 1344.7ms/step | 14.9 samples/s +Step 43400/44000 | Loss: 2.9112 | LR: 2.37e-07 | 1343.2ms/step | 14.9 samples/s +Step 43410/44000 | Loss: 2.9102 | LR: 2.30e-07 | 1341.0ms/step | 14.9 samples/s +Step 43420/44000 | Loss: 2.8887 | LR: 2.22e-07 | 1340.3ms/step | 14.9 samples/s +Step 43430/44000 | Loss: 2.9034 | LR: 2.14e-07 | 1339.8ms/step | 14.9 samples/s +Step 43440/44000 | Loss: 2.9506 | LR: 2.07e-07 | 1342.2ms/step | 14.9 samples/s +Step 43450/44000 | Loss: 2.9312 | LR: 1.99e-07 | 1343.5ms/step | 14.9 samples/s +Step 43460/44000 | Loss: 2.9236 | LR: 1.92e-07 | 1344.0ms/step | 14.9 samples/s +Step 43470/44000 | Loss: 2.9241 | LR: 1.85e-07 | 1343.7ms/step | 14.9 samples/s +Step 43480/44000 | Loss: 2.8914 | LR: 1.78e-07 | 1343.9ms/step | 14.9 samples/s +Step 43490/44000 | Loss: 2.9100 | LR: 1.72e-07 | 1344.0ms/step | 14.9 samples/s +Step 43500/44000 | Loss: 2.9499 | LR: 1.65e-07 | 1343.0ms/step | 14.9 samples/s +Validation at step 43500: Loss = 2.9307, Perplexity = 18.74 +Step 43510/44000 | Loss: 2.9157 | LR: 1.58e-07 | 3606.6ms/step | 5.5 samples/s +Step 43520/44000 | Loss: 2.9355 | LR: 1.52e-07 | 1343.3ms/step | 14.9 samples/s +Step 43530/44000 | Loss: 2.9002 | LR: 1.46e-07 | 1342.0ms/step | 14.9 samples/s +Step 43540/44000 | Loss: 2.8872 | LR: 1.40e-07 | 1344.3ms/step | 14.9 samples/s +Step 43550/44000 | Loss: 2.9102 | LR: 1.34e-07 | 1346.7ms/step | 14.9 samples/s +Step 43560/44000 | Loss: 2.9110 | LR: 1.28e-07 | 1346.6ms/step | 14.9 samples/s +Step 43570/44000 | Loss: 2.9088 | LR: 1.22e-07 | 1344.0ms/step | 14.9 samples/s +Step 43580/44000 | Loss: 2.9343 | LR: 1.16e-07 | 1341.3ms/step | 14.9 samples/s +Step 43590/44000 | Loss: 2.9103 | LR: 1.11e-07 | 1342.8ms/step | 14.9 samples/s +Step 43600/44000 | Loss: 2.9223 | LR: 1.06e-07 | 1346.6ms/step | 14.9 samples/s +Step 43610/44000 | Loss: 2.8685 | LR: 1.00e-07 | 1346.8ms/step | 14.8 samples/s +Step 43620/44000 | Loss: 2.9060 | LR: 9.52e-08 | 1344.8ms/step | 14.9 samples/s +Step 43630/44000 | Loss: 2.8988 | LR: 9.03e-08 | 1345.5ms/step | 14.9 samples/s +Step 43640/44000 | Loss: 2.9152 | LR: 8.55e-08 | 1346.0ms/step | 14.9 samples/s +Step 43650/44000 | Loss: 2.9183 | LR: 8.08e-08 | 1346.6ms/step | 14.9 samples/s +Step 43660/44000 | Loss: 2.9201 | LR: 7.62e-08 | 1346.7ms/step | 14.9 samples/s +Step 43670/44000 | Loss: 2.9265 | LR: 7.18e-08 | 1345.9ms/step | 14.9 samples/s +Step 43680/44000 | Loss: 2.9334 | LR: 6.75e-08 | 1348.0ms/step | 14.8 samples/s +Step 43690/44000 | Loss: 2.8942 | LR: 6.34e-08 | 1347.3ms/step | 14.8 samples/s +Step 43700/44000 | Loss: 2.9154 | LR: 5.94e-08 | 1349.6ms/step | 14.8 samples/s +Step 43710/44000 | Loss: 2.9345 | LR: 5.55e-08 | 1348.0ms/step | 14.8 samples/s +Step 43720/44000 | Loss: 2.9022 | LR: 5.17e-08 | 1348.5ms/step | 14.8 samples/s +Step 43730/44000 | Loss: 2.9391 | LR: 4.81e-08 | 1346.4ms/step | 14.9 samples/s +Step 43740/44000 | Loss: 2.9473 | LR: 4.46e-08 | 1348.0ms/step | 14.8 samples/s +Step 43750/44000 | Loss: 2.9182 | LR: 4.12e-08 | 1348.4ms/step | 14.8 samples/s +Validation at step 43750: Loss = 2.9309, Perplexity = 18.74 +Step 43760/44000 | Loss: 2.9366 | LR: 3.80e-08 | 3612.6ms/step | 5.5 samples/s +Step 43770/44000 | Loss: 2.9346 | LR: 3.49e-08 | 1349.9ms/step | 14.8 samples/s +Step 43780/44000 | Loss: 2.9157 | LR: 3.19e-08 | 1351.6ms/step | 14.8 samples/s +Step 43790/44000 | Loss: 2.9045 | LR: 2.91e-08 | 1350.2ms/step | 14.8 samples/s +Step 43800/44000 | Loss: 2.9069 | LR: 2.64e-08 | 1348.4ms/step | 14.8 samples/s +Step 43810/44000 | Loss: 2.9151 | LR: 2.38e-08 | 1348.6ms/step | 14.8 samples/s +Step 43820/44000 | Loss: 2.9068 | LR: 2.14e-08 | 1349.0ms/step | 14.8 samples/s +Step 43830/44000 | Loss: 2.9293 | LR: 1.91e-08 | 1351.3ms/step | 14.8 samples/s +Step 43840/44000 | Loss: 2.9158 | LR: 1.69e-08 | 1349.2ms/step | 14.8 samples/s +Step 43850/44000 | Loss: 2.9079 | LR: 1.48e-08 | 1349.6ms/step | 14.8 samples/s +Step 43860/44000 | Loss: 2.9189 | LR: 1.29e-08 | 1349.5ms/step | 14.8 samples/s +Step 43870/44000 | Loss: 2.9346 | LR: 1.11e-08 | 1346.8ms/step | 14.9 samples/s +Step 43880/44000 | Loss: 2.9510 | LR: 9.50e-09 | 1348.8ms/step | 14.8 samples/s +Step 43890/44000 | Loss: 2.9069 | LR: 7.98e-09 | 1350.2ms/step | 14.8 samples/s +Step 43900/44000 | Loss: 2.9160 | LR: 6.60e-09 | 1349.1ms/step | 14.8 samples/s +Step 43910/44000 | Loss: 2.9194 | LR: 5.34e-09 | 1349.5ms/step | 14.8 samples/s +Step 43920/44000 | Loss: 2.9030 | LR: 4.22e-09 | 1349.6ms/step | 14.8 samples/s +Step 43930/44000 | Loss: 2.8963 | LR: 3.23e-09 | 1349.9ms/step | 14.8 samples/s +Step 43940/44000 | Loss: 2.9047 | LR: 2.37e-09 | 1352.2ms/step | 14.8 samples/s +Step 43950/44000 | Loss: 2.9177 | LR: 1.65e-09 | 1351.0ms/step | 14.8 samples/s +Step 43960/44000 | Loss: 2.8987 | LR: 1.06e-09 | 1348.7ms/step | 14.8 samples/s +Step 43970/44000 | Loss: 2.9262 | LR: 5.94e-10 | 1350.2ms/step | 14.8 samples/s +Step 43980/44000 | Loss: 2.9300 | LR: 2.64e-10 | 1347.8ms/step | 14.8 samples/s +Step 43990/44000 | Loss: 2.9208 | LR: 6.60e-11 | 1347.9ms/step | 14.8 samples/s +Step 44000/44000 | Loss: 2.9625 | LR: 0.00e+00 | 1349.8ms/step | 14.8 samples/s +Validation at step 44000: Loss = 2.9311, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading output.log; uploading wandb-summary.json +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss ██▇▇▆▆▅▄▄▄▃▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▇▇▆▅▅▅▅▄▄▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▃▆▅▅▅▄▃▄▃▃▃▄▂▂▂▂▁▂▂▃▁▂▂▂▂▂▃▂▁▂█▅▅▃▃▂▁▂▂▂ +wandb: speed/samples_per_sec ▁▁▂▃▃▅▅▆▆▆▆▆▆▅▅▇▇▇▆▇▇▇▇▇▆▇▆▇▇█▁▅█▆▇▇▇▆▆▇ +wandb: speed/steps_per_sec ██▁█████████▁█████▁█████████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.93106 +wandb: eval/perplexity 18.74751 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 1349.83764 +wandb: speed/samples_per_sec 14.8166 +wandb: speed/steps_per_sec 0.74083 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 resume 365m resampled 8k nrb35 seed42 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/n56uphgo +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_042436-n56uphgo/logs +Training complete! diff --git a/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_resampled_nrb35_seed43.log b/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_resampled_nrb35_seed43.log new file mode 100644 index 0000000000000000000000000000000000000000..be6b974b5bca47ece0f737808047c8b6c687ef22 --- /dev/null +++ b/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_resampled_nrb35_seed43.log @@ -0,0 +1,2760 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-20000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 43, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 resume 365m resampled 8k nrb35 seed43", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "resampled", + "nrb35", + "seed43", + "resume", + "h1004", + "step20000" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-20000 +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-20000... +Preserving checkpoint LR state for exact resume: current=[0.00029291518939613315], schedule_base=[0.0005] +Resumed from checkpoint at step 20000 (epoch 0, micro_step 200000) +Resuming training from step 20000 +Resuming wandb run ryta2atg +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run ryta2atg +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042436-ryta2atg +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run P1 resume 365m resampled 8k nrb35 seed43 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ryta2atg +Starting training... +Validation at step 20000: Loss = 2.9920, Perplexity = 19.93 +wandb: WARNING Tried to log to step 20000 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 20001: sparse_stochastic (configured sparse attention) +Step 20010/44000 | Loss: 3.0014 | LR: 2.93e-04 | 37631.5ms/step | 0.5 samples/s +wandb: WARNING Tried to log to step 20010 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20020/44000 | Loss: 2.9980 | LR: 2.93e-04 | 1330.2ms/step | 15.0 samples/s +wandb: WARNING Tried to log to step 20020 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20030/44000 | Loss: 2.9734 | LR: 2.92e-04 | 1329.1ms/step | 15.0 samples/s +wandb: WARNING Tried to log to step 20030 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20040/44000 | Loss: 2.9946 | LR: 2.92e-04 | 1328.0ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 20040 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20050/44000 | Loss: 3.0091 | LR: 2.92e-04 | 1330.2ms/step | 15.0 samples/s +wandb: WARNING Tried to log to step 20050 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20060/44000 | Loss: 2.9848 | LR: 2.92e-04 | 1328.4ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 20060 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20070/44000 | Loss: 3.0437 | LR: 2.92e-04 | 1328.2ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 20070 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20080/44000 | Loss: 3.0042 | LR: 2.91e-04 | 1329.1ms/step | 15.0 samples/s +wandb: WARNING Tried to log to step 20080 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20090/44000 | Loss: 3.0005 | LR: 2.91e-04 | 1328.5ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 20090 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20100/44000 | Loss: 2.9965 | LR: 2.91e-04 | 1323.1ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 20100 that is less than the current step 20101. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20110/44000 | Loss: 2.9987 | LR: 2.91e-04 | 1329.9ms/step | 15.0 samples/s +Step 20120/44000 | Loss: 3.0177 | LR: 2.91e-04 | 1328.3ms/step | 15.1 samples/s +Step 20130/44000 | Loss: 2.9914 | LR: 2.91e-04 | 1328.0ms/step | 15.1 samples/s +Step 20140/44000 | Loss: 2.9956 | LR: 2.90e-04 | 1326.8ms/step | 15.1 samples/s +Step 20150/44000 | Loss: 3.0154 | LR: 2.90e-04 | 1325.5ms/step | 15.1 samples/s +Step 20160/44000 | Loss: 2.9885 | LR: 2.90e-04 | 1322.5ms/step | 15.1 samples/s +Step 20170/44000 | Loss: 3.0047 | LR: 2.90e-04 | 1321.1ms/step | 15.1 samples/s +Step 20180/44000 | Loss: 3.0024 | LR: 2.90e-04 | 1322.7ms/step | 15.1 samples/s +Step 20190/44000 | Loss: 3.0271 | LR: 2.90e-04 | 1320.0ms/step | 15.2 samples/s +Step 20200/44000 | Loss: 2.9941 | LR: 2.89e-04 | 1320.5ms/step | 15.1 samples/s +Step 20210/44000 | Loss: 2.9846 | LR: 2.89e-04 | 1321.3ms/step | 15.1 samples/s +Step 20220/44000 | Loss: 2.9987 | LR: 2.89e-04 | 1322.6ms/step | 15.1 samples/s +Step 20230/44000 | Loss: 3.0079 | LR: 2.89e-04 | 1325.0ms/step | 15.1 samples/s +Step 20240/44000 | Loss: 2.9764 | LR: 2.89e-04 | 1318.2ms/step | 15.2 samples/s +Step 20250/44000 | Loss: 2.9856 | LR: 2.88e-04 | 1322.7ms/step | 15.1 samples/s +Validation at step 20250: Loss = 2.9896, Perplexity = 19.88 +Step 20260/44000 | Loss: 2.9903 | LR: 2.88e-04 | 3554.5ms/step | 5.6 samples/s +Step 20270/44000 | Loss: 2.9914 | LR: 2.88e-04 | 1323.9ms/step | 15.1 samples/s +Step 20280/44000 | Loss: 3.0197 | LR: 2.88e-04 | 1323.3ms/step | 15.1 samples/s +Step 20290/44000 | Loss: 2.9889 | LR: 2.88e-04 | 1326.2ms/step | 15.1 samples/s +Step 20300/44000 | Loss: 3.0022 | LR: 2.88e-04 | 1327.8ms/step | 15.1 samples/s +Step 20310/44000 | Loss: 2.9915 | LR: 2.87e-04 | 1329.5ms/step | 15.0 samples/s +Step 20320/44000 | Loss: 3.0270 | LR: 2.87e-04 | 1327.8ms/step | 15.1 samples/s +Step 20330/44000 | Loss: 2.9962 | LR: 2.87e-04 | 1326.1ms/step | 15.1 samples/s +Step 20340/44000 | Loss: 2.9791 | LR: 2.87e-04 | 1329.0ms/step | 15.0 samples/s +Step 20350/44000 | Loss: 2.9795 | LR: 2.87e-04 | 1329.3ms/step | 15.0 samples/s +Step 20360/44000 | Loss: 2.9783 | LR: 2.86e-04 | 1328.5ms/step | 15.1 samples/s +Step 20370/44000 | Loss: 3.0057 | LR: 2.86e-04 | 1332.5ms/step | 15.0 samples/s +Step 20380/44000 | Loss: 3.0078 | LR: 2.86e-04 | 1324.2ms/step | 15.1 samples/s +Step 20390/44000 | Loss: 3.0164 | LR: 2.86e-04 | 1329.2ms/step | 15.0 samples/s +Step 20400/44000 | Loss: 3.0309 | LR: 2.86e-04 | 1329.5ms/step | 15.0 samples/s +Step 20410/44000 | Loss: 2.9997 | LR: 2.86e-04 | 1326.6ms/step | 15.1 samples/s +Step 20420/44000 | Loss: 3.0030 | LR: 2.85e-04 | 1324.6ms/step | 15.1 samples/s +Step 20430/44000 | Loss: 2.9894 | LR: 2.85e-04 | 1325.8ms/step | 15.1 samples/s +Step 20440/44000 | Loss: 3.0125 | LR: 2.85e-04 | 1327.5ms/step | 15.1 samples/s +Step 20450/44000 | Loss: 3.0122 | LR: 2.85e-04 | 1327.2ms/step | 15.1 samples/s +Step 20460/44000 | Loss: 2.9909 | LR: 2.85e-04 | 1323.6ms/step | 15.1 samples/s +Step 20470/44000 | Loss: 2.9482 | LR: 2.84e-04 | 1322.0ms/step | 15.1 samples/s +Step 20480/44000 | Loss: 2.9809 | LR: 2.84e-04 | 1327.1ms/step | 15.1 samples/s +Step 20490/44000 | Loss: 2.9861 | LR: 2.84e-04 | 1327.5ms/step | 15.1 samples/s +Step 20500/44000 | Loss: 2.9938 | LR: 2.84e-04 | 1330.8ms/step | 15.0 samples/s +Validation at step 20500: Loss = 2.9863, Perplexity = 19.81 +Step 20510/44000 | Loss: 3.0101 | LR: 2.84e-04 | 4097.8ms/step | 4.9 samples/s +Step 20520/44000 | Loss: 3.0178 | LR: 2.84e-04 | 1332.6ms/step | 15.0 samples/s +Step 20530/44000 | Loss: 2.9923 | LR: 2.83e-04 | 1330.2ms/step | 15.0 samples/s +Step 20540/44000 | Loss: 2.9645 | LR: 2.83e-04 | 1327.4ms/step | 15.1 samples/s +Step 20550/44000 | Loss: 3.0155 | LR: 2.83e-04 | 1325.7ms/step | 15.1 samples/s +Step 20560/44000 | Loss: 3.0085 | LR: 2.83e-04 | 1327.0ms/step | 15.1 samples/s +Step 20570/44000 | Loss: 2.9728 | LR: 2.83e-04 | 1327.4ms/step | 15.1 samples/s +Step 20580/44000 | Loss: 2.9958 | LR: 2.83e-04 | 1326.4ms/step | 15.1 samples/s +Step 20590/44000 | Loss: 3.0070 | LR: 2.82e-04 | 1325.1ms/step | 15.1 samples/s +Step 20600/44000 | Loss: 2.9771 | LR: 2.82e-04 | 1327.5ms/step | 15.1 samples/s +Step 20610/44000 | Loss: 3.0361 | LR: 2.82e-04 | 1328.2ms/step | 15.1 samples/s +Step 20620/44000 | Loss: 2.9658 | LR: 2.82e-04 | 1326.1ms/step | 15.1 samples/s +Step 20630/44000 | Loss: 2.9894 | LR: 2.82e-04 | 1331.7ms/step | 15.0 samples/s +Step 20640/44000 | Loss: 3.0107 | LR: 2.81e-04 | 1326.0ms/step | 15.1 samples/s +Step 20650/44000 | Loss: 2.9680 | LR: 2.81e-04 | 1325.5ms/step | 15.1 samples/s +Step 20660/44000 | Loss: 3.0125 | LR: 2.81e-04 | 1324.1ms/step | 15.1 samples/s +Step 20670/44000 | Loss: 2.9948 | LR: 2.81e-04 | 1325.4ms/step | 15.1 samples/s +Step 20680/44000 | Loss: 3.0119 | LR: 2.81e-04 | 1326.7ms/step | 15.1 samples/s +Step 20690/44000 | Loss: 2.9891 | LR: 2.81e-04 | 1329.1ms/step | 15.0 samples/s +Step 20700/44000 | Loss: 2.9871 | LR: 2.80e-04 | 1327.3ms/step | 15.1 samples/s +Step 20710/44000 | Loss: 2.9874 | LR: 2.80e-04 | 1327.8ms/step | 15.1 samples/s +Step 20720/44000 | Loss: 2.9921 | LR: 2.80e-04 | 1326.6ms/step | 15.1 samples/s +Step 20730/44000 | Loss: 2.9719 | LR: 2.80e-04 | 1326.8ms/step | 15.1 samples/s +Step 20740/44000 | Loss: 2.9675 | LR: 2.80e-04 | 1327.1ms/step | 15.1 samples/s +Step 20750/44000 | Loss: 3.0108 | LR: 2.79e-04 | 1327.7ms/step | 15.1 samples/s +Validation at step 20750: Loss = 2.9844, Perplexity = 19.77 +Step 20760/44000 | Loss: 3.0082 | LR: 2.79e-04 | 3549.2ms/step | 5.6 samples/s +Step 20770/44000 | Loss: 3.0088 | LR: 2.79e-04 | 1331.4ms/step | 15.0 samples/s +Step 20780/44000 | Loss: 3.0367 | LR: 2.79e-04 | 1330.8ms/step | 15.0 samples/s +Step 20790/44000 | Loss: 3.0178 | LR: 2.79e-04 | 1329.9ms/step | 15.0 samples/s +Step 20800/44000 | Loss: 2.9964 | LR: 2.79e-04 | 1330.4ms/step | 15.0 samples/s +Step 20810/44000 | Loss: 2.9716 | LR: 2.78e-04 | 1330.4ms/step | 15.0 samples/s +Step 20820/44000 | Loss: 3.0058 | LR: 2.78e-04 | 1332.3ms/step | 15.0 samples/s +Step 20830/44000 | Loss: 2.9693 | LR: 2.78e-04 | 1331.6ms/step | 15.0 samples/s +Step 20840/44000 | Loss: 3.0028 | LR: 2.78e-04 | 1332.0ms/step | 15.0 samples/s +Step 20850/44000 | Loss: 2.9824 | LR: 2.78e-04 | 1329.7ms/step | 15.0 samples/s +Step 20860/44000 | Loss: 2.9946 | LR: 2.77e-04 | 1335.0ms/step | 15.0 samples/s +Step 20870/44000 | Loss: 3.0229 | LR: 2.77e-04 | 1335.4ms/step | 15.0 samples/s +Step 20880/44000 | Loss: 2.9756 | LR: 2.77e-04 | 1334.4ms/step | 15.0 samples/s +Step 20890/44000 | Loss: 2.9666 | LR: 2.77e-04 | 1331.8ms/step | 15.0 samples/s +Step 20900/44000 | Loss: 2.9792 | LR: 2.77e-04 | 1332.1ms/step | 15.0 samples/s +Step 20910/44000 | Loss: 2.9805 | LR: 2.77e-04 | 1331.6ms/step | 15.0 samples/s +Step 20920/44000 | Loss: 3.0329 | LR: 2.76e-04 | 1332.0ms/step | 15.0 samples/s +Step 20930/44000 | Loss: 2.9924 | LR: 2.76e-04 | 1330.6ms/step | 15.0 samples/s +Step 20940/44000 | Loss: 2.9988 | LR: 2.76e-04 | 1331.1ms/step | 15.0 samples/s +Step 20950/44000 | Loss: 2.9880 | LR: 2.76e-04 | 1329.4ms/step | 15.0 samples/s +Step 20960/44000 | Loss: 2.9964 | LR: 2.76e-04 | 1330.6ms/step | 15.0 samples/s +Step 20970/44000 | Loss: 2.9931 | LR: 2.75e-04 | 1329.3ms/step | 15.0 samples/s +Step 20980/44000 | Loss: 3.0102 | LR: 2.75e-04 | 1332.5ms/step | 15.0 samples/s +Step 20990/44000 | Loss: 3.0051 | LR: 2.75e-04 | 1334.8ms/step | 15.0 samples/s +Step 21000/44000 | Loss: 2.9845 | LR: 2.75e-04 | 1331.3ms/step | 15.0 samples/s +Validation at step 21000: Loss = 2.9812, Perplexity = 19.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-21000 +Step 21010/44000 | Loss: 2.9882 | LR: 2.75e-04 | 3842.4ms/step | 5.2 samples/s +Step 21020/44000 | Loss: 3.0209 | LR: 2.75e-04 | 1323.1ms/step | 15.1 samples/s +Step 21030/44000 | Loss: 2.9701 | LR: 2.74e-04 | 1323.3ms/step | 15.1 samples/s +Step 21040/44000 | Loss: 2.9694 | LR: 2.74e-04 | 1324.6ms/step | 15.1 samples/s +Step 21050/44000 | Loss: 3.0101 | LR: 2.74e-04 | 1323.8ms/step | 15.1 samples/s +Step 21060/44000 | Loss: 3.0234 | LR: 2.74e-04 | 1324.3ms/step | 15.1 samples/s +Step 21070/44000 | Loss: 2.9764 | LR: 2.74e-04 | 1323.0ms/step | 15.1 samples/s +Step 21080/44000 | Loss: 3.0050 | LR: 2.73e-04 | 1321.8ms/step | 15.1 samples/s +Step 21090/44000 | Loss: 2.9809 | LR: 2.73e-04 | 1322.2ms/step | 15.1 samples/s +Step 21100/44000 | Loss: 2.9998 | LR: 2.73e-04 | 1325.0ms/step | 15.1 samples/s +Step 21110/44000 | Loss: 2.9581 | LR: 2.73e-04 | 1322.8ms/step | 15.1 samples/s +Step 21120/44000 | Loss: 3.0150 | LR: 2.73e-04 | 1319.8ms/step | 15.2 samples/s +Step 21130/44000 | Loss: 2.9987 | LR: 2.73e-04 | 1317.0ms/step | 15.2 samples/s +Step 21140/44000 | Loss: 3.0122 | LR: 2.72e-04 | 1316.9ms/step | 15.2 samples/s +Step 21150/44000 | Loss: 3.0223 | LR: 2.72e-04 | 1318.4ms/step | 15.2 samples/s +Step 21160/44000 | Loss: 2.9859 | LR: 2.72e-04 | 1321.6ms/step | 15.1 samples/s +Step 21170/44000 | Loss: 3.0052 | LR: 2.72e-04 | 1320.7ms/step | 15.1 samples/s +Step 21180/44000 | Loss: 2.9971 | LR: 2.72e-04 | 1322.7ms/step | 15.1 samples/s +Step 21190/44000 | Loss: 2.9932 | LR: 2.71e-04 | 1321.4ms/step | 15.1 samples/s +Step 21200/44000 | Loss: 2.9841 | LR: 2.71e-04 | 1324.1ms/step | 15.1 samples/s +Step 21210/44000 | Loss: 2.9658 | LR: 2.71e-04 | 1322.1ms/step | 15.1 samples/s +Step 21220/44000 | Loss: 2.9741 | LR: 2.71e-04 | 1323.5ms/step | 15.1 samples/s +Step 21230/44000 | Loss: 2.9790 | LR: 2.71e-04 | 1324.3ms/step | 15.1 samples/s +Step 21240/44000 | Loss: 2.9912 | LR: 2.71e-04 | 1324.4ms/step | 15.1 samples/s +Step 21250/44000 | Loss: 2.9877 | LR: 2.70e-04 | 1323.0ms/step | 15.1 samples/s +Validation at step 21250: Loss = 2.9790, Perplexity = 19.67 +Step 21260/44000 | Loss: 3.0037 | LR: 2.70e-04 | 3537.5ms/step | 5.7 samples/s +Step 21270/44000 | Loss: 2.9810 | LR: 2.70e-04 | 1321.7ms/step | 15.1 samples/s +Step 21280/44000 | Loss: 2.9632 | LR: 2.70e-04 | 1322.4ms/step | 15.1 samples/s +Step 21290/44000 | Loss: 2.9783 | LR: 2.70e-04 | 1323.4ms/step | 15.1 samples/s +Step 21300/44000 | Loss: 2.9586 | LR: 2.70e-04 | 1322.3ms/step | 15.1 samples/s +Step 21310/44000 | Loss: 2.9942 | LR: 2.69e-04 | 1319.5ms/step | 15.2 samples/s +Step 21320/44000 | Loss: 2.9923 | LR: 2.69e-04 | 1317.0ms/step | 15.2 samples/s +Step 21330/44000 | Loss: 2.9770 | LR: 2.69e-04 | 1317.3ms/step | 15.2 samples/s +Step 21340/44000 | Loss: 2.9795 | LR: 2.69e-04 | 1317.4ms/step | 15.2 samples/s +Step 21350/44000 | Loss: 2.9895 | LR: 2.69e-04 | 1319.1ms/step | 15.2 samples/s +Step 21360/44000 | Loss: 2.9784 | LR: 2.68e-04 | 1319.2ms/step | 15.2 samples/s +Step 21370/44000 | Loss: 3.0020 | LR: 2.68e-04 | 1320.0ms/step | 15.2 samples/s +Step 21380/44000 | Loss: 2.9946 | LR: 2.68e-04 | 1320.6ms/step | 15.1 samples/s +Step 21390/44000 | Loss: 2.9473 | LR: 2.68e-04 | 1323.7ms/step | 15.1 samples/s +Step 21400/44000 | Loss: 2.9824 | LR: 2.68e-04 | 1321.7ms/step | 15.1 samples/s +Step 21410/44000 | Loss: 2.9894 | LR: 2.68e-04 | 1318.2ms/step | 15.2 samples/s +Step 21420/44000 | Loss: 2.9989 | LR: 2.67e-04 | 1321.1ms/step | 15.1 samples/s +Step 21430/44000 | Loss: 2.9922 | LR: 2.67e-04 | 1320.3ms/step | 15.1 samples/s +Step 21440/44000 | Loss: 3.0188 | LR: 2.67e-04 | 1314.1ms/step | 15.2 samples/s +Step 21450/44000 | Loss: 2.9550 | LR: 2.67e-04 | 1313.6ms/step | 15.2 samples/s +Step 21460/44000 | Loss: 2.9846 | LR: 2.67e-04 | 1319.7ms/step | 15.2 samples/s +Step 21470/44000 | Loss: 2.9987 | LR: 2.66e-04 | 1317.6ms/step | 15.2 samples/s +Step 21480/44000 | Loss: 2.9888 | LR: 2.66e-04 | 1321.5ms/step | 15.1 samples/s +Step 21490/44000 | Loss: 2.9692 | LR: 2.66e-04 | 1322.2ms/step | 15.1 samples/s +Step 21500/44000 | Loss: 2.9638 | LR: 2.66e-04 | 1321.7ms/step | 15.1 samples/s +Validation at step 21500: Loss = 2.9758, Perplexity = 19.61 +Step 21510/44000 | Loss: 2.9576 | LR: 2.66e-04 | 3532.0ms/step | 5.7 samples/s +Step 21520/44000 | Loss: 2.9648 | LR: 2.66e-04 | 1320.1ms/step | 15.2 samples/s +Step 21530/44000 | Loss: 2.9478 | LR: 2.65e-04 | 1320.9ms/step | 15.1 samples/s +Step 21540/44000 | Loss: 2.9718 | LR: 2.65e-04 | 1320.0ms/step | 15.2 samples/s +Step 21550/44000 | Loss: 2.9905 | LR: 2.65e-04 | 1318.4ms/step | 15.2 samples/s +Step 21560/44000 | Loss: 3.0105 | LR: 2.65e-04 | 1316.8ms/step | 15.2 samples/s +Step 21570/44000 | Loss: 3.0110 | LR: 2.65e-04 | 1318.6ms/step | 15.2 samples/s +Step 21580/44000 | Loss: 3.0027 | LR: 2.64e-04 | 1321.1ms/step | 15.1 samples/s +Step 21590/44000 | Loss: 2.9662 | LR: 2.64e-04 | 1321.7ms/step | 15.1 samples/s +Step 21600/44000 | Loss: 2.9699 | LR: 2.64e-04 | 1320.3ms/step | 15.1 samples/s +Step 21610/44000 | Loss: 2.9894 | LR: 2.64e-04 | 1313.7ms/step | 15.2 samples/s +Step 21620/44000 | Loss: 2.9975 | LR: 2.64e-04 | 1311.2ms/step | 15.3 samples/s +Step 21630/44000 | Loss: 3.0070 | LR: 2.64e-04 | 1312.8ms/step | 15.2 samples/s +Step 21640/44000 | Loss: 2.9662 | LR: 2.63e-04 | 1315.1ms/step | 15.2 samples/s +Step 21650/44000 | Loss: 2.9892 | LR: 2.63e-04 | 1320.1ms/step | 15.2 samples/s +Step 21660/44000 | Loss: 2.9585 | LR: 2.63e-04 | 1317.5ms/step | 15.2 samples/s +Step 21670/44000 | Loss: 3.0003 | LR: 2.63e-04 | 1316.5ms/step | 15.2 samples/s +Step 21680/44000 | Loss: 2.9623 | LR: 2.63e-04 | 1319.4ms/step | 15.2 samples/s +Step 21690/44000 | Loss: 2.9521 | LR: 2.62e-04 | 1315.9ms/step | 15.2 samples/s +Step 21700/44000 | Loss: 2.9818 | LR: 2.62e-04 | 1316.5ms/step | 15.2 samples/s +Step 21710/44000 | Loss: 2.9499 | LR: 2.62e-04 | 1316.6ms/step | 15.2 samples/s +Step 21720/44000 | Loss: 2.9890 | LR: 2.62e-04 | 1316.3ms/step | 15.2 samples/s +Step 21730/44000 | Loss: 2.9825 | LR: 2.62e-04 | 1316.6ms/step | 15.2 samples/s +Step 21740/44000 | Loss: 2.9709 | LR: 2.62e-04 | 1320.5ms/step | 15.1 samples/s +Step 21750/44000 | Loss: 2.9780 | LR: 2.61e-04 | 1318.8ms/step | 15.2 samples/s +Validation at step 21750: Loss = 2.9735, Perplexity = 19.56 +Step 21760/44000 | Loss: 2.9693 | LR: 2.61e-04 | 3517.9ms/step | 5.7 samples/s +Step 21770/44000 | Loss: 2.9823 | LR: 2.61e-04 | 1311.9ms/step | 15.2 samples/s +Step 21780/44000 | Loss: 2.9663 | LR: 2.61e-04 | 1312.7ms/step | 15.2 samples/s +Step 21790/44000 | Loss: 2.9672 | LR: 2.61e-04 | 1315.2ms/step | 15.2 samples/s +Step 21800/44000 | Loss: 2.9869 | LR: 2.60e-04 | 1317.3ms/step | 15.2 samples/s +Step 21810/44000 | Loss: 2.9835 | LR: 2.60e-04 | 1318.8ms/step | 15.2 samples/s +Step 21820/44000 | Loss: 2.9973 | LR: 2.60e-04 | 1316.7ms/step | 15.2 samples/s +Step 21830/44000 | Loss: 2.9923 | LR: 2.60e-04 | 1316.7ms/step | 15.2 samples/s +Step 21840/44000 | Loss: 2.9870 | LR: 2.60e-04 | 1313.9ms/step | 15.2 samples/s +Step 21850/44000 | Loss: 2.9701 | LR: 2.60e-04 | 1314.6ms/step | 15.2 samples/s +Step 21860/44000 | Loss: 3.0008 | LR: 2.59e-04 | 1314.6ms/step | 15.2 samples/s +Step 21870/44000 | Loss: 2.9848 | LR: 2.59e-04 | 1313.5ms/step | 15.2 samples/s +Step 21880/44000 | Loss: 3.0157 | LR: 2.59e-04 | 1312.5ms/step | 15.2 samples/s +Step 21890/44000 | Loss: 3.0007 | LR: 2.59e-04 | 1313.0ms/step | 15.2 samples/s +Step 21900/44000 | Loss: 2.9717 | LR: 2.59e-04 | 1314.8ms/step | 15.2 samples/s +Step 21910/44000 | Loss: 2.9720 | LR: 2.58e-04 | 1316.5ms/step | 15.2 samples/s +Step 21920/44000 | Loss: 2.9517 | LR: 2.58e-04 | 1316.9ms/step | 15.2 samples/s +Step 21930/44000 | Loss: 2.9892 | LR: 2.58e-04 | 1313.5ms/step | 15.2 samples/s +Step 21940/44000 | Loss: 2.9955 | LR: 2.58e-04 | 1313.9ms/step | 15.2 samples/s +Step 21950/44000 | Loss: 3.0045 | LR: 2.58e-04 | 1315.0ms/step | 15.2 samples/s +Step 21960/44000 | Loss: 2.9787 | LR: 2.58e-04 | 1315.4ms/step | 15.2 samples/s +Step 21970/44000 | Loss: 3.0102 | LR: 2.57e-04 | 1314.9ms/step | 15.2 samples/s +Step 21980/44000 | Loss: 2.9472 | LR: 2.57e-04 | 1318.6ms/step | 15.2 samples/s +Step 21990/44000 | Loss: 2.9840 | LR: 2.57e-04 | 1316.9ms/step | 15.2 samples/s +Step 22000/44000 | Loss: 2.9692 | LR: 2.57e-04 | 1316.2ms/step | 15.2 samples/s +Validation at step 22000: Loss = 2.9711, Perplexity = 19.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-22000 +Step 22010/44000 | Loss: 2.9739 | LR: 2.57e-04 | 3826.6ms/step | 5.2 samples/s +Step 22020/44000 | Loss: 2.9838 | LR: 2.56e-04 | 1321.4ms/step | 15.1 samples/s +Step 22030/44000 | Loss: 3.0002 | LR: 2.56e-04 | 1320.6ms/step | 15.1 samples/s +Step 22040/44000 | Loss: 2.9927 | LR: 2.56e-04 | 1321.1ms/step | 15.1 samples/s +Step 22050/44000 | Loss: 2.9874 | LR: 2.56e-04 | 1315.5ms/step | 15.2 samples/s +Step 22060/44000 | Loss: 2.9938 | LR: 2.56e-04 | 1315.2ms/step | 15.2 samples/s +Step 22070/44000 | Loss: 2.9835 | LR: 2.56e-04 | 1316.9ms/step | 15.2 samples/s +Step 22080/44000 | Loss: 2.9829 | LR: 2.55e-04 | 1315.6ms/step | 15.2 samples/s +Step 22090/44000 | Loss: 2.9935 | LR: 2.55e-04 | 1314.8ms/step | 15.2 samples/s +Step 22100/44000 | Loss: 3.0045 | LR: 2.55e-04 | 1313.7ms/step | 15.2 samples/s +Step 22110/44000 | Loss: 2.9731 | LR: 2.55e-04 | 1315.3ms/step | 15.2 samples/s +Step 22120/44000 | Loss: 2.9642 | LR: 2.55e-04 | 1320.1ms/step | 15.1 samples/s +Step 22130/44000 | Loss: 3.0139 | LR: 2.54e-04 | 1316.0ms/step | 15.2 samples/s +Step 22140/44000 | Loss: 2.9678 | LR: 2.54e-04 | 1317.5ms/step | 15.2 samples/s +Step 22150/44000 | Loss: 2.9658 | LR: 2.54e-04 | 1314.4ms/step | 15.2 samples/s +Step 22160/44000 | Loss: 3.0003 | LR: 2.54e-04 | 1317.6ms/step | 15.2 samples/s +Step 22170/44000 | Loss: 2.9586 | LR: 2.54e-04 | 1319.3ms/step | 15.2 samples/s +Step 22180/44000 | Loss: 2.9983 | LR: 2.54e-04 | 1316.3ms/step | 15.2 samples/s +Step 22190/44000 | Loss: 2.9959 | LR: 2.53e-04 | 1314.4ms/step | 15.2 samples/s +Step 22200/44000 | Loss: 2.9725 | LR: 2.53e-04 | 1312.2ms/step | 15.2 samples/s +Step 22210/44000 | Loss: 2.9551 | LR: 2.53e-04 | 1311.0ms/step | 15.3 samples/s +Step 22220/44000 | Loss: 2.9778 | LR: 2.53e-04 | 1311.6ms/step | 15.2 samples/s +Step 22230/44000 | Loss: 2.9833 | LR: 2.53e-04 | 1312.3ms/step | 15.2 samples/s +Step 22240/44000 | Loss: 2.9820 | LR: 2.52e-04 | 1313.4ms/step | 15.2 samples/s +Step 22250/44000 | Loss: 2.9694 | LR: 2.52e-04 | 1311.8ms/step | 15.2 samples/s +Validation at step 22250: Loss = 2.9692, Perplexity = 19.48 +Step 22260/44000 | Loss: 2.9641 | LR: 2.52e-04 | 3517.1ms/step | 5.7 samples/s +Step 22270/44000 | Loss: 2.9828 | LR: 2.52e-04 | 1312.7ms/step | 15.2 samples/s +Step 22280/44000 | Loss: 2.9499 | LR: 2.52e-04 | 1312.5ms/step | 15.2 samples/s +Step 22290/44000 | Loss: 2.9962 | LR: 2.52e-04 | 1311.0ms/step | 15.3 samples/s +Step 22300/44000 | Loss: 2.9788 | LR: 2.51e-04 | 1314.8ms/step | 15.2 samples/s +Step 22310/44000 | Loss: 2.9889 | LR: 2.51e-04 | 1314.4ms/step | 15.2 samples/s +Step 22320/44000 | Loss: 2.9786 | LR: 2.51e-04 | 1315.8ms/step | 15.2 samples/s +Step 22330/44000 | Loss: 2.9743 | LR: 2.51e-04 | 1313.9ms/step | 15.2 samples/s +Step 22340/44000 | Loss: 2.9766 | LR: 2.51e-04 | 1313.0ms/step | 15.2 samples/s +Step 22350/44000 | Loss: 2.9976 | LR: 2.50e-04 | 1314.4ms/step | 15.2 samples/s +Step 22360/44000 | Loss: 2.9667 | LR: 2.50e-04 | 1314.1ms/step | 15.2 samples/s +Step 22370/44000 | Loss: 2.9646 | LR: 2.50e-04 | 1317.0ms/step | 15.2 samples/s +Step 22380/44000 | Loss: 2.9897 | LR: 2.50e-04 | 1316.1ms/step | 15.2 samples/s +Step 22390/44000 | Loss: 2.9945 | LR: 2.50e-04 | 1316.8ms/step | 15.2 samples/s +Step 22400/44000 | Loss: 2.9673 | LR: 2.50e-04 | 1315.7ms/step | 15.2 samples/s +Step 22410/44000 | Loss: 2.9940 | LR: 2.49e-04 | 1317.8ms/step | 15.2 samples/s +Step 22420/44000 | Loss: 2.9625 | LR: 2.49e-04 | 1317.4ms/step | 15.2 samples/s +Step 22430/44000 | Loss: 2.9967 | LR: 2.49e-04 | 1319.3ms/step | 15.2 samples/s +Step 22440/44000 | Loss: 3.0191 | LR: 2.49e-04 | 1322.6ms/step | 15.1 samples/s +Step 22450/44000 | Loss: 2.9899 | LR: 2.49e-04 | 1323.0ms/step | 15.1 samples/s +Step 22460/44000 | Loss: 2.9706 | LR: 2.48e-04 | 1316.5ms/step | 15.2 samples/s +Step 22470/44000 | Loss: 2.9702 | LR: 2.48e-04 | 1321.1ms/step | 15.1 samples/s +Step 22480/44000 | Loss: 2.9580 | LR: 2.48e-04 | 1311.9ms/step | 15.2 samples/s +Step 22490/44000 | Loss: 2.9874 | LR: 2.48e-04 | 1314.0ms/step | 15.2 samples/s +Step 22500/44000 | Loss: 2.9622 | LR: 2.48e-04 | 1319.6ms/step | 15.2 samples/s +Validation at step 22500: Loss = 2.9664, Perplexity = 19.42 +Step 22510/44000 | Loss: 2.9828 | LR: 2.48e-04 | 3526.3ms/step | 5.7 samples/s +Step 22520/44000 | Loss: 3.0101 | LR: 2.47e-04 | 1321.0ms/step | 15.1 samples/s +Step 22530/44000 | Loss: 2.9869 | LR: 2.47e-04 | 1321.9ms/step | 15.1 samples/s +Step 22540/44000 | Loss: 2.9672 | LR: 2.47e-04 | 1320.8ms/step | 15.1 samples/s +Step 22550/44000 | Loss: 3.0070 | LR: 2.47e-04 | 1321.1ms/step | 15.1 samples/s +Step 22560/44000 | Loss: 2.9750 | LR: 2.47e-04 | 1319.1ms/step | 15.2 samples/s +Step 22570/44000 | Loss: 2.9483 | LR: 2.46e-04 | 1320.1ms/step | 15.2 samples/s +Step 22580/44000 | Loss: 2.9867 | LR: 2.46e-04 | 1319.8ms/step | 15.2 samples/s +Step 22590/44000 | Loss: 2.9674 | LR: 2.46e-04 | 1318.7ms/step | 15.2 samples/s +Step 22600/44000 | Loss: 2.9846 | LR: 2.46e-04 | 1311.6ms/step | 15.2 samples/s +Step 22610/44000 | Loss: 2.9912 | LR: 2.46e-04 | 1311.9ms/step | 15.2 samples/s +Step 22620/44000 | Loss: 2.9958 | LR: 2.46e-04 | 1313.6ms/step | 15.2 samples/s +Step 22630/44000 | Loss: 2.9905 | LR: 2.45e-04 | 1313.8ms/step | 15.2 samples/s +Step 22640/44000 | Loss: 2.9708 | LR: 2.45e-04 | 1316.1ms/step | 15.2 samples/s +Step 22650/44000 | Loss: 2.9689 | LR: 2.45e-04 | 1318.7ms/step | 15.2 samples/s +Step 22660/44000 | Loss: 2.9845 | LR: 2.45e-04 | 1320.9ms/step | 15.1 samples/s +Step 22670/44000 | Loss: 2.9363 | LR: 2.45e-04 | 1322.6ms/step | 15.1 samples/s +Step 22680/44000 | Loss: 2.9693 | LR: 2.44e-04 | 1317.9ms/step | 15.2 samples/s +Step 22690/44000 | Loss: 2.9457 | LR: 2.44e-04 | 1315.3ms/step | 15.2 samples/s +Step 22700/44000 | Loss: 2.9699 | LR: 2.44e-04 | 1316.2ms/step | 15.2 samples/s +Step 22710/44000 | Loss: 2.9488 | LR: 2.44e-04 | 1315.0ms/step | 15.2 samples/s +Step 22720/44000 | Loss: 2.9685 | LR: 2.44e-04 | 1317.1ms/step | 15.2 samples/s +Step 22730/44000 | Loss: 2.9709 | LR: 2.44e-04 | 1318.0ms/step | 15.2 samples/s +Step 22740/44000 | Loss: 2.9746 | LR: 2.43e-04 | 1318.7ms/step | 15.2 samples/s +Step 22750/44000 | Loss: 2.9792 | LR: 2.43e-04 | 1321.0ms/step | 15.1 samples/s +Validation at step 22750: Loss = 2.9646, Perplexity = 19.39 +Step 22760/44000 | Loss: 2.9583 | LR: 2.43e-04 | 3527.0ms/step | 5.7 samples/s +Step 22770/44000 | Loss: 2.9680 | LR: 2.43e-04 | 1311.2ms/step | 15.3 samples/s +Step 22780/44000 | Loss: 2.9548 | LR: 2.43e-04 | 1313.0ms/step | 15.2 samples/s +Step 22790/44000 | Loss: 2.9820 | LR: 2.42e-04 | 1318.8ms/step | 15.2 samples/s +Step 22800/44000 | Loss: 2.9796 | LR: 2.42e-04 | 1319.9ms/step | 15.2 samples/s +Step 22810/44000 | Loss: 2.9669 | LR: 2.42e-04 | 1317.5ms/step | 15.2 samples/s +Step 22820/44000 | Loss: 2.9597 | LR: 2.42e-04 | 1312.2ms/step | 15.2 samples/s +Step 22830/44000 | Loss: 2.9797 | LR: 2.42e-04 | 1310.4ms/step | 15.3 samples/s +Step 22840/44000 | Loss: 2.9899 | LR: 2.42e-04 | 1312.5ms/step | 15.2 samples/s +Step 22850/44000 | Loss: 3.0017 | LR: 2.41e-04 | 1314.4ms/step | 15.2 samples/s +Step 22860/44000 | Loss: 2.9862 | LR: 2.41e-04 | 1314.1ms/step | 15.2 samples/s +Step 22870/44000 | Loss: 2.9727 | LR: 2.41e-04 | 1312.0ms/step | 15.2 samples/s +Step 22880/44000 | Loss: 2.9724 | LR: 2.41e-04 | 1314.0ms/step | 15.2 samples/s +Step 22890/44000 | Loss: 2.9743 | LR: 2.41e-04 | 1313.9ms/step | 15.2 samples/s +Step 22900/44000 | Loss: 2.9890 | LR: 2.40e-04 | 1313.6ms/step | 15.2 samples/s +Step 22910/44000 | Loss: 2.9923 | LR: 2.40e-04 | 1313.0ms/step | 15.2 samples/s +Step 22920/44000 | Loss: 2.9597 | LR: 2.40e-04 | 1315.5ms/step | 15.2 samples/s +Step 22930/44000 | Loss: 2.9600 | LR: 2.40e-04 | 1314.0ms/step | 15.2 samples/s +Step 22940/44000 | Loss: 2.9440 | LR: 2.40e-04 | 1314.3ms/step | 15.2 samples/s +Step 22950/44000 | Loss: 2.9963 | LR: 2.40e-04 | 1311.4ms/step | 15.3 samples/s +Step 22960/44000 | Loss: 2.9804 | LR: 2.39e-04 | 1310.4ms/step | 15.3 samples/s +Step 22970/44000 | Loss: 2.9793 | LR: 2.39e-04 | 1312.2ms/step | 15.2 samples/s +Step 22980/44000 | Loss: 2.9852 | LR: 2.39e-04 | 1313.1ms/step | 15.2 samples/s +Step 22990/44000 | Loss: 2.9876 | LR: 2.39e-04 | 1320.4ms/step | 15.1 samples/s +Step 23000/44000 | Loss: 2.9640 | LR: 2.39e-04 | 1317.1ms/step | 15.2 samples/s +Validation at step 23000: Loss = 2.9620, Perplexity = 19.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-23000 +Step 23010/44000 | Loss: 2.9829 | LR: 2.38e-04 | 3829.2ms/step | 5.2 samples/s +Step 23020/44000 | Loss: 2.9860 | LR: 2.38e-04 | 1316.9ms/step | 15.2 samples/s +Step 23030/44000 | Loss: 2.9712 | LR: 2.38e-04 | 1312.3ms/step | 15.2 samples/s +Step 23040/44000 | Loss: 3.0016 | LR: 2.38e-04 | 1313.5ms/step | 15.2 samples/s +Step 23050/44000 | Loss: 2.9556 | LR: 2.38e-04 | 1317.0ms/step | 15.2 samples/s +Step 23060/44000 | Loss: 2.9967 | LR: 2.38e-04 | 1315.9ms/step | 15.2 samples/s +Step 23070/44000 | Loss: 2.9688 | LR: 2.37e-04 | 1315.5ms/step | 15.2 samples/s +Step 23080/44000 | Loss: 2.9936 | LR: 2.37e-04 | 1317.3ms/step | 15.2 samples/s +Step 23090/44000 | Loss: 2.9702 | LR: 2.37e-04 | 1316.9ms/step | 15.2 samples/s +Step 23100/44000 | Loss: 2.9919 | LR: 2.37e-04 | 1318.4ms/step | 15.2 samples/s +Step 23110/44000 | Loss: 2.9736 | LR: 2.37e-04 | 1320.1ms/step | 15.2 samples/s +Step 23120/44000 | Loss: 3.0104 | LR: 2.36e-04 | 1319.9ms/step | 15.2 samples/s +Step 23130/44000 | Loss: 3.0186 | LR: 2.36e-04 | 1319.6ms/step | 15.2 samples/s +Step 23140/44000 | Loss: 2.9176 | LR: 2.36e-04 | 1319.6ms/step | 15.2 samples/s +Step 23150/44000 | Loss: 2.9506 | LR: 2.36e-04 | 1317.9ms/step | 15.2 samples/s +Step 23160/44000 | Loss: 2.9873 | LR: 2.36e-04 | 1317.6ms/step | 15.2 samples/s +Step 23170/44000 | Loss: 2.9756 | LR: 2.36e-04 | 1318.2ms/step | 15.2 samples/s +Step 23180/44000 | Loss: 2.9721 | LR: 2.35e-04 | 1318.9ms/step | 15.2 samples/s +Step 23190/44000 | Loss: 2.9686 | LR: 2.35e-04 | 1317.5ms/step | 15.2 samples/s +Step 23200/44000 | Loss: 2.9342 | LR: 2.35e-04 | 1319.9ms/step | 15.2 samples/s +Step 23210/44000 | Loss: 3.0072 | LR: 2.35e-04 | 1318.2ms/step | 15.2 samples/s +Step 23220/44000 | Loss: 2.9718 | LR: 2.35e-04 | 1317.7ms/step | 15.2 samples/s +Step 23230/44000 | Loss: 2.9783 | LR: 2.34e-04 | 1315.7ms/step | 15.2 samples/s +Step 23240/44000 | Loss: 2.9929 | LR: 2.34e-04 | 1320.5ms/step | 15.1 samples/s +Step 23250/44000 | Loss: 2.9460 | LR: 2.34e-04 | 1319.8ms/step | 15.2 samples/s +Validation at step 23250: Loss = 2.9598, Perplexity = 19.29 +Step 23260/44000 | Loss: 2.9660 | LR: 2.34e-04 | 3526.9ms/step | 5.7 samples/s +Step 23270/44000 | Loss: 2.9696 | LR: 2.34e-04 | 1319.4ms/step | 15.2 samples/s +Step 23280/44000 | Loss: 2.9674 | LR: 2.34e-04 | 1320.4ms/step | 15.1 samples/s +Step 23290/44000 | Loss: 2.9497 | LR: 2.33e-04 | 1320.0ms/step | 15.2 samples/s +Step 23300/44000 | Loss: 2.9826 | LR: 2.33e-04 | 1320.6ms/step | 15.1 samples/s +Step 23310/44000 | Loss: 2.9767 | LR: 2.33e-04 | 1317.5ms/step | 15.2 samples/s +Step 23320/44000 | Loss: 2.9843 | LR: 2.33e-04 | 1317.8ms/step | 15.2 samples/s +Step 23330/44000 | Loss: 2.9581 | LR: 2.33e-04 | 1317.7ms/step | 15.2 samples/s +Step 23340/44000 | Loss: 2.9559 | LR: 2.32e-04 | 1318.4ms/step | 15.2 samples/s +Step 23350/44000 | Loss: 2.9354 | LR: 2.32e-04 | 1316.8ms/step | 15.2 samples/s +Step 23360/44000 | Loss: 2.9712 | LR: 2.32e-04 | 1318.8ms/step | 15.2 samples/s +Step 23370/44000 | Loss: 2.9932 | LR: 2.32e-04 | 1319.4ms/step | 15.2 samples/s +Step 23380/44000 | Loss: 2.9779 | LR: 2.32e-04 | 1320.1ms/step | 15.2 samples/s +Step 23390/44000 | Loss: 3.0029 | LR: 2.32e-04 | 1319.0ms/step | 15.2 samples/s +Step 23400/44000 | Loss: 2.9687 | LR: 2.31e-04 | 1318.2ms/step | 15.2 samples/s +Step 23410/44000 | Loss: 2.9615 | LR: 2.31e-04 | 1319.1ms/step | 15.2 samples/s +Step 23420/44000 | Loss: 3.0096 | LR: 2.31e-04 | 1316.3ms/step | 15.2 samples/s +Step 23430/44000 | Loss: 2.9898 | LR: 2.31e-04 | 1316.1ms/step | 15.2 samples/s +Step 23440/44000 | Loss: 2.9382 | LR: 2.31e-04 | 1316.3ms/step | 15.2 samples/s +Step 23450/44000 | Loss: 2.9612 | LR: 2.30e-04 | 1316.5ms/step | 15.2 samples/s +Step 23460/44000 | Loss: 2.9693 | LR: 2.30e-04 | 1317.4ms/step | 15.2 samples/s +Step 23470/44000 | Loss: 2.9756 | LR: 2.30e-04 | 1314.7ms/step | 15.2 samples/s +Step 23480/44000 | Loss: 2.9662 | LR: 2.30e-04 | 1317.2ms/step | 15.2 samples/s +Step 23490/44000 | Loss: 2.9710 | LR: 2.30e-04 | 1318.9ms/step | 15.2 samples/s +Step 23500/44000 | Loss: 2.9779 | LR: 2.30e-04 | 1317.7ms/step | 15.2 samples/s +Validation at step 23500: Loss = 2.9573, Perplexity = 19.25 +Step 23510/44000 | Loss: 2.9809 | LR: 2.29e-04 | 3525.6ms/step | 5.7 samples/s +Step 23520/44000 | Loss: 2.9677 | LR: 2.29e-04 | 1320.7ms/step | 15.1 samples/s +Step 23530/44000 | Loss: 2.9376 | LR: 2.29e-04 | 1319.2ms/step | 15.2 samples/s +Step 23540/44000 | Loss: 2.9261 | LR: 2.29e-04 | 1318.6ms/step | 15.2 samples/s +Step 23550/44000 | Loss: 2.9590 | LR: 2.29e-04 | 1321.3ms/step | 15.1 samples/s +Step 23560/44000 | Loss: 2.9546 | LR: 2.29e-04 | 1320.0ms/step | 15.2 samples/s +Step 23570/44000 | Loss: 2.9598 | LR: 2.28e-04 | 1321.1ms/step | 15.1 samples/s +Step 23580/44000 | Loss: 2.9495 | LR: 2.28e-04 | 1322.1ms/step | 15.1 samples/s +Step 23590/44000 | Loss: 2.9892 | LR: 2.28e-04 | 1320.6ms/step | 15.1 samples/s +Step 23600/44000 | Loss: 2.9147 | LR: 2.28e-04 | 1319.7ms/step | 15.2 samples/s +Step 23610/44000 | Loss: 2.9626 | LR: 2.28e-04 | 1318.2ms/step | 15.2 samples/s +Step 23620/44000 | Loss: 2.9706 | LR: 2.27e-04 | 1317.2ms/step | 15.2 samples/s +Step 23630/44000 | Loss: 2.9299 | LR: 2.27e-04 | 1319.5ms/step | 15.2 samples/s +Step 23640/44000 | Loss: 2.9631 | LR: 2.27e-04 | 1320.0ms/step | 15.2 samples/s +Step 23650/44000 | Loss: 2.9894 | LR: 2.27e-04 | 1325.0ms/step | 15.1 samples/s +Step 23660/44000 | Loss: 2.9210 | LR: 2.27e-04 | 1320.2ms/step | 15.1 samples/s +Step 23670/44000 | Loss: 2.9526 | LR: 2.27e-04 | 1321.6ms/step | 15.1 samples/s +Step 23680/44000 | Loss: 2.9675 | LR: 2.26e-04 | 1320.1ms/step | 15.2 samples/s +Step 23690/44000 | Loss: 2.9592 | LR: 2.26e-04 | 1321.1ms/step | 15.1 samples/s +Step 23700/44000 | Loss: 2.9719 | LR: 2.26e-04 | 1323.2ms/step | 15.1 samples/s +Step 23710/44000 | Loss: 2.9583 | LR: 2.26e-04 | 1320.5ms/step | 15.1 samples/s +Step 23720/44000 | Loss: 2.9429 | LR: 2.26e-04 | 1320.1ms/step | 15.2 samples/s +Step 23730/44000 | Loss: 3.0037 | LR: 2.25e-04 | 1320.6ms/step | 15.1 samples/s +Step 23740/44000 | Loss: 2.9762 | LR: 2.25e-04 | 1321.1ms/step | 15.1 samples/s +Step 23750/44000 | Loss: 2.9968 | LR: 2.25e-04 | 1320.2ms/step | 15.1 samples/s +Validation at step 23750: Loss = 2.9553, Perplexity = 19.21 +Step 23760/44000 | Loss: 2.9590 | LR: 2.25e-04 | 3528.4ms/step | 5.7 samples/s +Step 23770/44000 | Loss: 2.9971 | LR: 2.25e-04 | 1319.7ms/step | 15.2 samples/s +Step 23780/44000 | Loss: 2.9791 | LR: 2.25e-04 | 1321.0ms/step | 15.1 samples/s +Step 23790/44000 | Loss: 2.9639 | LR: 2.24e-04 | 1319.6ms/step | 15.2 samples/s +Step 23800/44000 | Loss: 2.9637 | LR: 2.24e-04 | 1316.9ms/step | 15.2 samples/s +Step 23810/44000 | Loss: 2.9666 | LR: 2.24e-04 | 1320.1ms/step | 15.2 samples/s +Step 23820/44000 | Loss: 2.9752 | LR: 2.24e-04 | 1320.9ms/step | 15.1 samples/s +Step 23830/44000 | Loss: 2.9694 | LR: 2.24e-04 | 1321.0ms/step | 15.1 samples/s +Step 23840/44000 | Loss: 2.9752 | LR: 2.23e-04 | 1321.5ms/step | 15.1 samples/s +Step 23850/44000 | Loss: 2.9790 | LR: 2.23e-04 | 1320.8ms/step | 15.1 samples/s +Step 23860/44000 | Loss: 2.9603 | LR: 2.23e-04 | 1320.4ms/step | 15.1 samples/s +Step 23870/44000 | Loss: 2.9434 | LR: 2.23e-04 | 1322.3ms/step | 15.1 samples/s +Step 23880/44000 | Loss: 2.9869 | LR: 2.23e-04 | 1319.2ms/step | 15.2 samples/s +Step 23890/44000 | Loss: 2.9998 | LR: 2.23e-04 | 1318.8ms/step | 15.2 samples/s +Step 23900/44000 | Loss: 2.9619 | LR: 2.22e-04 | 1317.3ms/step | 15.2 samples/s +Step 23910/44000 | Loss: 2.9608 | LR: 2.22e-04 | 1318.6ms/step | 15.2 samples/s +Step 23920/44000 | Loss: 2.9866 | LR: 2.22e-04 | 1316.1ms/step | 15.2 samples/s +Step 23930/44000 | Loss: 2.9597 | LR: 2.22e-04 | 1315.3ms/step | 15.2 samples/s +Step 23940/44000 | Loss: 2.9502 | LR: 2.22e-04 | 1320.7ms/step | 15.1 samples/s +Step 23950/44000 | Loss: 2.9365 | LR: 2.21e-04 | 1321.0ms/step | 15.1 samples/s +Step 23960/44000 | Loss: 2.9608 | LR: 2.21e-04 | 1323.2ms/step | 15.1 samples/s +Step 23970/44000 | Loss: 2.9606 | LR: 2.21e-04 | 1322.5ms/step | 15.1 samples/s +Step 23980/44000 | Loss: 2.9560 | LR: 2.21e-04 | 1315.1ms/step | 15.2 samples/s +Step 23990/44000 | Loss: 2.9732 | LR: 2.21e-04 | 1314.3ms/step | 15.2 samples/s +Step 24000/44000 | Loss: 2.9826 | LR: 2.21e-04 | 1313.7ms/step | 15.2 samples/s +Validation at step 24000: Loss = 2.9536, Perplexity = 19.17 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-24000 +Step 24010/44000 | Loss: 2.9640 | LR: 2.20e-04 | 3829.8ms/step | 5.2 samples/s +Step 24020/44000 | Loss: 2.9425 | LR: 2.20e-04 | 1320.3ms/step | 15.1 samples/s +Step 24030/44000 | Loss: 2.9673 | LR: 2.20e-04 | 1315.3ms/step | 15.2 samples/s +Step 24040/44000 | Loss: 2.9396 | LR: 2.20e-04 | 1314.7ms/step | 15.2 samples/s +Step 24050/44000 | Loss: 2.9929 | LR: 2.20e-04 | 1316.9ms/step | 15.2 samples/s +Step 24060/44000 | Loss: 2.9310 | LR: 2.19e-04 | 1320.1ms/step | 15.2 samples/s +Step 24070/44000 | Loss: 2.9464 | LR: 2.19e-04 | 1317.5ms/step | 15.2 samples/s +Step 24080/44000 | Loss: 2.9612 | LR: 2.19e-04 | 1315.7ms/step | 15.2 samples/s +Step 24090/44000 | Loss: 2.9747 | LR: 2.19e-04 | 1317.5ms/step | 15.2 samples/s +Step 24100/44000 | Loss: 2.9450 | LR: 2.19e-04 | 1318.4ms/step | 15.2 samples/s +Step 24110/44000 | Loss: 2.9966 | LR: 2.19e-04 | 1308.6ms/step | 15.3 samples/s +Step 24120/44000 | Loss: 3.0036 | LR: 2.18e-04 | 1309.4ms/step | 15.3 samples/s +Step 24130/44000 | Loss: 2.9898 | LR: 2.18e-04 | 1319.6ms/step | 15.2 samples/s +Step 24140/44000 | Loss: 2.9627 | LR: 2.18e-04 | 1316.3ms/step | 15.2 samples/s +Step 24150/44000 | Loss: 2.9567 | LR: 2.18e-04 | 1315.1ms/step | 15.2 samples/s +Step 24160/44000 | Loss: 2.9734 | LR: 2.18e-04 | 1314.8ms/step | 15.2 samples/s +Step 24170/44000 | Loss: 2.9674 | LR: 2.17e-04 | 1316.1ms/step | 15.2 samples/s +Step 24180/44000 | Loss: 2.9471 | LR: 2.17e-04 | 1316.6ms/step | 15.2 samples/s +Step 24190/44000 | Loss: 2.9712 | LR: 2.17e-04 | 1312.6ms/step | 15.2 samples/s +Step 24200/44000 | Loss: 2.9695 | LR: 2.17e-04 | 1318.6ms/step | 15.2 samples/s +Step 24210/44000 | Loss: 2.9494 | LR: 2.17e-04 | 1319.6ms/step | 15.2 samples/s +Step 24220/44000 | Loss: 2.9516 | LR: 2.17e-04 | 1318.4ms/step | 15.2 samples/s +Step 24230/44000 | Loss: 2.9742 | LR: 2.16e-04 | 1317.4ms/step | 15.2 samples/s +Step 24240/44000 | Loss: 2.9501 | LR: 2.16e-04 | 1314.9ms/step | 15.2 samples/s +Step 24250/44000 | Loss: 2.9878 | LR: 2.16e-04 | 1316.3ms/step | 15.2 samples/s +Validation at step 24250: Loss = 2.9515, Perplexity = 19.13 +Step 24260/44000 | Loss: 2.9468 | LR: 2.16e-04 | 3521.0ms/step | 5.7 samples/s +Step 24270/44000 | Loss: 2.9534 | LR: 2.16e-04 | 1318.3ms/step | 15.2 samples/s +Step 24280/44000 | Loss: 2.9639 | LR: 2.16e-04 | 1318.7ms/step | 15.2 samples/s +Step 24290/44000 | Loss: 2.9376 | LR: 2.15e-04 | 1315.0ms/step | 15.2 samples/s +Step 24300/44000 | Loss: 2.9582 | LR: 2.15e-04 | 1312.0ms/step | 15.2 samples/s +Step 24310/44000 | Loss: 2.9493 | LR: 2.15e-04 | 1313.0ms/step | 15.2 samples/s +Step 24320/44000 | Loss: 2.9586 | LR: 2.15e-04 | 1317.1ms/step | 15.2 samples/s +Step 24330/44000 | Loss: 2.9533 | LR: 2.15e-04 | 1315.6ms/step | 15.2 samples/s +Step 24340/44000 | Loss: 2.9548 | LR: 2.14e-04 | 1316.2ms/step | 15.2 samples/s +Step 24350/44000 | Loss: 2.9587 | LR: 2.14e-04 | 1320.8ms/step | 15.1 samples/s +Step 24360/44000 | Loss: 2.9538 | LR: 2.14e-04 | 1319.4ms/step | 15.2 samples/s +Step 24370/44000 | Loss: 2.9730 | LR: 2.14e-04 | 1317.9ms/step | 15.2 samples/s +Step 24380/44000 | Loss: 2.9679 | LR: 2.14e-04 | 1317.6ms/step | 15.2 samples/s +Step 24390/44000 | Loss: 2.9582 | LR: 2.14e-04 | 1317.3ms/step | 15.2 samples/s +Step 24400/44000 | Loss: 2.9668 | LR: 2.13e-04 | 1317.6ms/step | 15.2 samples/s +Step 24410/44000 | Loss: 2.9757 | LR: 2.13e-04 | 1319.9ms/step | 15.2 samples/s +Step 24420/44000 | Loss: 2.9682 | LR: 2.13e-04 | 1318.1ms/step | 15.2 samples/s +Step 24430/44000 | Loss: 2.9572 | LR: 2.13e-04 | 1319.2ms/step | 15.2 samples/s +Step 24440/44000 | Loss: 2.9740 | LR: 2.13e-04 | 1318.6ms/step | 15.2 samples/s +Step 24450/44000 | Loss: 2.9754 | LR: 2.12e-04 | 1318.2ms/step | 15.2 samples/s +Step 24460/44000 | Loss: 2.9253 | LR: 2.12e-04 | 1318.2ms/step | 15.2 samples/s +Step 24470/44000 | Loss: 2.9782 | LR: 2.12e-04 | 1319.3ms/step | 15.2 samples/s +Step 24480/44000 | Loss: 2.9575 | LR: 2.12e-04 | 1321.0ms/step | 15.1 samples/s +Step 24490/44000 | Loss: 2.9539 | LR: 2.12e-04 | 1316.0ms/step | 15.2 samples/s +Step 24500/44000 | Loss: 2.9304 | LR: 2.12e-04 | 1315.7ms/step | 15.2 samples/s +Validation at step 24500: Loss = 2.9493, Perplexity = 19.09 +Step 24510/44000 | Loss: 2.9568 | LR: 2.11e-04 | 3525.3ms/step | 5.7 samples/s +Step 24520/44000 | Loss: 2.9553 | LR: 2.11e-04 | 1317.2ms/step | 15.2 samples/s +Step 24530/44000 | Loss: 2.9642 | LR: 2.11e-04 | 1320.5ms/step | 15.1 samples/s +Step 24540/44000 | Loss: 2.9490 | LR: 2.11e-04 | 1320.8ms/step | 15.1 samples/s +Step 24550/44000 | Loss: 2.9377 | LR: 2.11e-04 | 1318.3ms/step | 15.2 samples/s +Step 24560/44000 | Loss: 2.9699 | LR: 2.10e-04 | 1315.6ms/step | 15.2 samples/s +Step 24570/44000 | Loss: 2.9750 | LR: 2.10e-04 | 1315.4ms/step | 15.2 samples/s +Step 24580/44000 | Loss: 2.9607 | LR: 2.10e-04 | 1316.8ms/step | 15.2 samples/s +Step 24590/44000 | Loss: 2.9555 | LR: 2.10e-04 | 1320.3ms/step | 15.1 samples/s +Step 24600/44000 | Loss: 2.9497 | LR: 2.10e-04 | 1318.7ms/step | 15.2 samples/s +Step 24610/44000 | Loss: 2.9702 | LR: 2.10e-04 | 1313.9ms/step | 15.2 samples/s +Step 24620/44000 | Loss: 2.9585 | LR: 2.09e-04 | 1313.6ms/step | 15.2 samples/s +Step 24630/44000 | Loss: 2.9690 | LR: 2.09e-04 | 1311.1ms/step | 15.3 samples/s +Step 24640/44000 | Loss: 2.9486 | LR: 2.09e-04 | 1311.7ms/step | 15.2 samples/s +Step 24650/44000 | Loss: 2.9629 | LR: 2.09e-04 | 1313.1ms/step | 15.2 samples/s +Step 24660/44000 | Loss: 2.9790 | LR: 2.09e-04 | 1311.7ms/step | 15.2 samples/s +Step 24670/44000 | Loss: 2.9811 | LR: 2.09e-04 | 1315.1ms/step | 15.2 samples/s +Step 24680/44000 | Loss: 2.9547 | LR: 2.08e-04 | 1311.8ms/step | 15.2 samples/s +Step 24690/44000 | Loss: 2.9607 | LR: 2.08e-04 | 1309.5ms/step | 15.3 samples/s +Step 24700/44000 | Loss: 2.9353 | LR: 2.08e-04 | 1309.9ms/step | 15.3 samples/s +Step 24710/44000 | Loss: 2.9714 | LR: 2.08e-04 | 1310.9ms/step | 15.3 samples/s +Step 24720/44000 | Loss: 2.9344 | LR: 2.08e-04 | 1312.4ms/step | 15.2 samples/s +Step 24730/44000 | Loss: 2.9551 | LR: 2.07e-04 | 1319.2ms/step | 15.2 samples/s +Step 24740/44000 | Loss: 2.9425 | LR: 2.07e-04 | 1320.5ms/step | 15.1 samples/s +Step 24750/44000 | Loss: 2.9591 | LR: 2.07e-04 | 1321.8ms/step | 15.1 samples/s +Validation at step 24750: Loss = 2.9472, Perplexity = 19.05 +Step 24760/44000 | Loss: 2.9564 | LR: 2.07e-04 | 3525.9ms/step | 5.7 samples/s +Step 24770/44000 | Loss: 2.9432 | LR: 2.07e-04 | 1316.5ms/step | 15.2 samples/s +Step 24780/44000 | Loss: 2.9481 | LR: 2.07e-04 | 1318.2ms/step | 15.2 samples/s +Step 24790/44000 | Loss: 3.0226 | LR: 2.06e-04 | 1317.8ms/step | 15.2 samples/s +Step 24800/44000 | Loss: 2.9634 | LR: 2.06e-04 | 1318.4ms/step | 15.2 samples/s +Step 24810/44000 | Loss: 2.9624 | LR: 2.06e-04 | 1316.8ms/step | 15.2 samples/s +Step 24820/44000 | Loss: 2.9708 | LR: 2.06e-04 | 1314.1ms/step | 15.2 samples/s +Step 24830/44000 | Loss: 2.9608 | LR: 2.06e-04 | 1316.0ms/step | 15.2 samples/s +Step 24840/44000 | Loss: 2.9366 | LR: 2.05e-04 | 1313.5ms/step | 15.2 samples/s +Step 24850/44000 | Loss: 2.9573 | LR: 2.05e-04 | 1316.3ms/step | 15.2 samples/s +Step 24860/44000 | Loss: 2.9505 | LR: 2.05e-04 | 1314.0ms/step | 15.2 samples/s +Step 24870/44000 | Loss: 2.9597 | LR: 2.05e-04 | 1317.5ms/step | 15.2 samples/s +Step 24880/44000 | Loss: 2.9689 | LR: 2.05e-04 | 1318.1ms/step | 15.2 samples/s +Step 24890/44000 | Loss: 2.9389 | LR: 2.05e-04 | 1316.6ms/step | 15.2 samples/s +Step 24900/44000 | Loss: 2.9398 | LR: 2.04e-04 | 1315.8ms/step | 15.2 samples/s +Step 24910/44000 | Loss: 2.9821 | LR: 2.04e-04 | 1313.4ms/step | 15.2 samples/s +Step 24920/44000 | Loss: 3.0008 | LR: 2.04e-04 | 1313.1ms/step | 15.2 samples/s +Step 24930/44000 | Loss: 2.9725 | LR: 2.04e-04 | 1313.9ms/step | 15.2 samples/s +Step 24940/44000 | Loss: 2.9535 | LR: 2.04e-04 | 1316.4ms/step | 15.2 samples/s +Step 24950/44000 | Loss: 2.9493 | LR: 2.04e-04 | 1318.1ms/step | 15.2 samples/s +Step 24960/44000 | Loss: 2.9852 | LR: 2.03e-04 | 1312.7ms/step | 15.2 samples/s +Step 24970/44000 | Loss: 2.9717 | LR: 2.03e-04 | 1315.5ms/step | 15.2 samples/s +Step 24980/44000 | Loss: 2.9527 | LR: 2.03e-04 | 1316.2ms/step | 15.2 samples/s +Step 24990/44000 | Loss: 2.9488 | LR: 2.03e-04 | 1318.0ms/step | 15.2 samples/s +Step 25000/44000 | Loss: 2.9781 | LR: 2.03e-04 | 1319.9ms/step | 15.2 samples/s +Validation at step 25000: Loss = 2.9455, Perplexity = 19.02 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-25000 +Step 25010/44000 | Loss: 2.9692 | LR: 2.02e-04 | 3832.4ms/step | 5.2 samples/s +Step 25020/44000 | Loss: 2.9634 | LR: 2.02e-04 | 1318.1ms/step | 15.2 samples/s +Step 25030/44000 | Loss: 2.9734 | LR: 2.02e-04 | 1318.0ms/step | 15.2 samples/s +Step 25040/44000 | Loss: 2.9502 | LR: 2.02e-04 | 1318.4ms/step | 15.2 samples/s +Step 25050/44000 | Loss: 2.9548 | LR: 2.02e-04 | 1314.9ms/step | 15.2 samples/s +Step 25060/44000 | Loss: 2.9348 | LR: 2.02e-04 | 1313.4ms/step | 15.2 samples/s +Step 25070/44000 | Loss: 2.9405 | LR: 2.01e-04 | 1317.8ms/step | 15.2 samples/s +Step 25080/44000 | Loss: 2.9462 | LR: 2.01e-04 | 1317.8ms/step | 15.2 samples/s +Step 25090/44000 | Loss: 2.9912 | LR: 2.01e-04 | 1318.1ms/step | 15.2 samples/s +Step 25100/44000 | Loss: 2.9557 | LR: 2.01e-04 | 1317.5ms/step | 15.2 samples/s +Step 25110/44000 | Loss: 2.9508 | LR: 2.01e-04 | 1319.4ms/step | 15.2 samples/s +Step 25120/44000 | Loss: 2.9375 | LR: 2.00e-04 | 1318.5ms/step | 15.2 samples/s +Step 25130/44000 | Loss: 2.9565 | LR: 2.00e-04 | 1317.1ms/step | 15.2 samples/s +Step 25140/44000 | Loss: 2.9813 | LR: 2.00e-04 | 1318.3ms/step | 15.2 samples/s +Step 25150/44000 | Loss: 2.9839 | LR: 2.00e-04 | 1318.3ms/step | 15.2 samples/s +Step 25160/44000 | Loss: 2.9628 | LR: 2.00e-04 | 1319.5ms/step | 15.2 samples/s +Step 25170/44000 | Loss: 2.9480 | LR: 2.00e-04 | 1317.6ms/step | 15.2 samples/s +Step 25180/44000 | Loss: 2.9396 | LR: 1.99e-04 | 1317.2ms/step | 15.2 samples/s +Step 25190/44000 | Loss: 2.9824 | LR: 1.99e-04 | 1317.5ms/step | 15.2 samples/s +Step 25200/44000 | Loss: 2.9603 | LR: 1.99e-04 | 1318.4ms/step | 15.2 samples/s +Step 25210/44000 | Loss: 2.9507 | LR: 1.99e-04 | 1317.4ms/step | 15.2 samples/s +Step 25220/44000 | Loss: 2.9559 | LR: 1.99e-04 | 1316.0ms/step | 15.2 samples/s +Step 25230/44000 | Loss: 2.9586 | LR: 1.99e-04 | 1313.2ms/step | 15.2 samples/s +Step 25240/44000 | Loss: 2.9623 | LR: 1.98e-04 | 1314.7ms/step | 15.2 samples/s +Step 25250/44000 | Loss: 2.9788 | LR: 1.98e-04 | 1317.9ms/step | 15.2 samples/s +Validation at step 25250: Loss = 2.9434, Perplexity = 18.98 +Step 25260/44000 | Loss: 2.9448 | LR: 1.98e-04 | 3523.7ms/step | 5.7 samples/s +Step 25270/44000 | Loss: 2.9560 | LR: 1.98e-04 | 1314.8ms/step | 15.2 samples/s +Step 25280/44000 | Loss: 2.9466 | LR: 1.98e-04 | 1315.5ms/step | 15.2 samples/s +Step 25290/44000 | Loss: 2.9364 | LR: 1.97e-04 | 1320.0ms/step | 15.2 samples/s +Step 25300/44000 | Loss: 2.9715 | LR: 1.97e-04 | 1318.1ms/step | 15.2 samples/s +Step 25310/44000 | Loss: 2.9455 | LR: 1.97e-04 | 1317.9ms/step | 15.2 samples/s +Step 25320/44000 | Loss: 2.9359 | LR: 1.97e-04 | 1317.5ms/step | 15.2 samples/s +Step 25330/44000 | Loss: 2.9399 | LR: 1.97e-04 | 1318.2ms/step | 15.2 samples/s +Step 25340/44000 | Loss: 2.9230 | LR: 1.97e-04 | 1315.6ms/step | 15.2 samples/s +Step 25350/44000 | Loss: 2.9118 | LR: 1.96e-04 | 1318.3ms/step | 15.2 samples/s +Step 25360/44000 | Loss: 3.0083 | LR: 1.96e-04 | 1318.2ms/step | 15.2 samples/s +Step 25370/44000 | Loss: 2.9352 | LR: 1.96e-04 | 1318.4ms/step | 15.2 samples/s +Step 25380/44000 | Loss: 2.9507 | LR: 1.96e-04 | 1319.0ms/step | 15.2 samples/s +Step 25390/44000 | Loss: 2.9267 | LR: 1.96e-04 | 1321.6ms/step | 15.1 samples/s +Step 25400/44000 | Loss: 2.9936 | LR: 1.96e-04 | 1321.9ms/step | 15.1 samples/s +Step 25410/44000 | Loss: 2.9596 | LR: 1.95e-04 | 1320.1ms/step | 15.2 samples/s +Step 25420/44000 | Loss: 2.9377 | LR: 1.95e-04 | 1319.9ms/step | 15.2 samples/s +Step 25430/44000 | Loss: 2.9500 | LR: 1.95e-04 | 1317.5ms/step | 15.2 samples/s +Step 25440/44000 | Loss: 2.9339 | LR: 1.95e-04 | 1319.5ms/step | 15.2 samples/s +Step 25450/44000 | Loss: 2.9470 | LR: 1.95e-04 | 1320.0ms/step | 15.2 samples/s +Step 25460/44000 | Loss: 2.9353 | LR: 1.94e-04 | 1318.9ms/step | 15.2 samples/s +Step 25470/44000 | Loss: 2.9377 | LR: 1.94e-04 | 1320.4ms/step | 15.1 samples/s +Step 25480/44000 | Loss: 2.9329 | LR: 1.94e-04 | 1320.4ms/step | 15.1 samples/s +Step 25490/44000 | Loss: 2.9197 | LR: 1.94e-04 | 1315.1ms/step | 15.2 samples/s +Step 25500/44000 | Loss: 2.9547 | LR: 1.94e-04 | 1318.2ms/step | 15.2 samples/s +Validation at step 25500: Loss = 2.9421, Perplexity = 18.96 +Step 25510/44000 | Loss: 2.9447 | LR: 1.94e-04 | 3524.1ms/step | 5.7 samples/s +Step 25520/44000 | Loss: 2.9666 | LR: 1.93e-04 | 1316.6ms/step | 15.2 samples/s +Step 25530/44000 | Loss: 2.9487 | LR: 1.93e-04 | 1315.2ms/step | 15.2 samples/s +Step 25540/44000 | Loss: 2.9348 | LR: 1.93e-04 | 1316.7ms/step | 15.2 samples/s +Step 25550/44000 | Loss: 2.9497 | LR: 1.93e-04 | 1318.3ms/step | 15.2 samples/s +Step 25560/44000 | Loss: 2.9548 | LR: 1.93e-04 | 1318.1ms/step | 15.2 samples/s +Step 25570/44000 | Loss: 2.9616 | LR: 1.92e-04 | 1317.2ms/step | 15.2 samples/s +Step 25580/44000 | Loss: 2.9344 | LR: 1.92e-04 | 1315.6ms/step | 15.2 samples/s +Step 25590/44000 | Loss: 2.9208 | LR: 1.92e-04 | 1312.4ms/step | 15.2 samples/s +Step 25600/44000 | Loss: 2.9546 | LR: 1.92e-04 | 1312.6ms/step | 15.2 samples/s +Step 25610/44000 | Loss: 2.9517 | LR: 1.92e-04 | 1315.0ms/step | 15.2 samples/s +Step 25620/44000 | Loss: 2.9426 | LR: 1.92e-04 | 1316.0ms/step | 15.2 samples/s +Step 25630/44000 | Loss: 2.9478 | LR: 1.91e-04 | 1320.1ms/step | 15.2 samples/s +Step 25640/44000 | Loss: 2.9455 | LR: 1.91e-04 | 1317.1ms/step | 15.2 samples/s +Step 25650/44000 | Loss: 2.9664 | LR: 1.91e-04 | 1318.2ms/step | 15.2 samples/s +Step 25660/44000 | Loss: 2.9212 | LR: 1.91e-04 | 1318.6ms/step | 15.2 samples/s +Step 25670/44000 | Loss: 2.9346 | LR: 1.91e-04 | 1318.3ms/step | 15.2 samples/s +Step 25680/44000 | Loss: 2.9487 | LR: 1.91e-04 | 1317.5ms/step | 15.2 samples/s +Step 25690/44000 | Loss: 2.9525 | LR: 1.90e-04 | 1315.7ms/step | 15.2 samples/s +Step 25700/44000 | Loss: 2.9190 | LR: 1.90e-04 | 1316.5ms/step | 15.2 samples/s +Step 25710/44000 | Loss: 2.9784 | LR: 1.90e-04 | 1319.5ms/step | 15.2 samples/s +Step 25720/44000 | Loss: 2.9335 | LR: 1.90e-04 | 1320.8ms/step | 15.1 samples/s +Step 25730/44000 | Loss: 2.9519 | LR: 1.90e-04 | 1322.0ms/step | 15.1 samples/s +Step 25740/44000 | Loss: 2.9467 | LR: 1.89e-04 | 1319.6ms/step | 15.2 samples/s +Step 25750/44000 | Loss: 2.9391 | LR: 1.89e-04 | 1319.2ms/step | 15.2 samples/s +Validation at step 25750: Loss = 2.9402, Perplexity = 18.92 +Step 25760/44000 | Loss: 2.9511 | LR: 1.89e-04 | 3529.7ms/step | 5.7 samples/s +Step 25770/44000 | Loss: 2.9513 | LR: 1.89e-04 | 1318.1ms/step | 15.2 samples/s +Step 25780/44000 | Loss: 2.9510 | LR: 1.89e-04 | 1319.5ms/step | 15.2 samples/s +Step 25790/44000 | Loss: 2.9501 | LR: 1.89e-04 | 1320.5ms/step | 15.1 samples/s +Step 25800/44000 | Loss: 2.9461 | LR: 1.88e-04 | 1320.0ms/step | 15.2 samples/s +Step 25810/44000 | Loss: 2.9729 | LR: 1.88e-04 | 1320.5ms/step | 15.1 samples/s +Step 25820/44000 | Loss: 2.9417 | LR: 1.88e-04 | 1319.2ms/step | 15.2 samples/s +Step 25830/44000 | Loss: 2.9303 | LR: 1.88e-04 | 1320.4ms/step | 15.1 samples/s +Step 25840/44000 | Loss: 2.9870 | LR: 1.88e-04 | 1320.1ms/step | 15.2 samples/s +Step 25850/44000 | Loss: 2.9593 | LR: 1.88e-04 | 1324.9ms/step | 15.1 samples/s +Step 25860/44000 | Loss: 2.9396 | LR: 1.87e-04 | 1320.6ms/step | 15.1 samples/s +Step 25870/44000 | Loss: 2.9315 | LR: 1.87e-04 | 1319.4ms/step | 15.2 samples/s +Step 25880/44000 | Loss: 2.9680 | LR: 1.87e-04 | 1322.0ms/step | 15.1 samples/s +Step 25890/44000 | Loss: 2.9425 | LR: 1.87e-04 | 1323.8ms/step | 15.1 samples/s +Step 25900/44000 | Loss: 2.9827 | LR: 1.87e-04 | 1320.5ms/step | 15.1 samples/s +Step 25910/44000 | Loss: 2.9527 | LR: 1.87e-04 | 1319.4ms/step | 15.2 samples/s +Step 25920/44000 | Loss: 2.9670 | LR: 1.86e-04 | 1322.9ms/step | 15.1 samples/s +Step 25930/44000 | Loss: 2.9155 | LR: 1.86e-04 | 1319.4ms/step | 15.2 samples/s +Step 25940/44000 | Loss: 2.9284 | LR: 1.86e-04 | 1321.7ms/step | 15.1 samples/s +Step 25950/44000 | Loss: 2.9648 | LR: 1.86e-04 | 1319.6ms/step | 15.2 samples/s +Step 25960/44000 | Loss: 2.9620 | LR: 1.86e-04 | 1320.0ms/step | 15.2 samples/s +Step 25970/44000 | Loss: 2.9384 | LR: 1.85e-04 | 1318.2ms/step | 15.2 samples/s +Step 25980/44000 | Loss: 2.9191 | LR: 1.85e-04 | 1314.1ms/step | 15.2 samples/s +Step 25990/44000 | Loss: 2.9379 | LR: 1.85e-04 | 1314.2ms/step | 15.2 samples/s +Step 26000/44000 | Loss: 2.9577 | LR: 1.85e-04 | 1314.6ms/step | 15.2 samples/s +Validation at step 26000: Loss = 2.9382, Perplexity = 18.88 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-26000 +Step 26010/44000 | Loss: 2.9503 | LR: 1.85e-04 | 3813.4ms/step | 5.2 samples/s +Step 26020/44000 | Loss: 2.9579 | LR: 1.85e-04 | 1312.0ms/step | 15.2 samples/s +Step 26030/44000 | Loss: 2.9857 | LR: 1.84e-04 | 1312.2ms/step | 15.2 samples/s +Step 26040/44000 | Loss: 2.9526 | LR: 1.84e-04 | 1314.7ms/step | 15.2 samples/s +Step 26050/44000 | Loss: 2.9463 | LR: 1.84e-04 | 1321.0ms/step | 15.1 samples/s +Step 26060/44000 | Loss: 2.9814 | LR: 1.84e-04 | 1319.3ms/step | 15.2 samples/s +Step 26070/44000 | Loss: 2.9390 | LR: 1.84e-04 | 1315.9ms/step | 15.2 samples/s +Step 26080/44000 | Loss: 2.9275 | LR: 1.84e-04 | 1316.6ms/step | 15.2 samples/s +Step 26090/44000 | Loss: 2.9216 | LR: 1.83e-04 | 1316.9ms/step | 15.2 samples/s +Step 26100/44000 | Loss: 2.9417 | LR: 1.83e-04 | 1318.6ms/step | 15.2 samples/s +Step 26110/44000 | Loss: 2.9077 | LR: 1.83e-04 | 1316.8ms/step | 15.2 samples/s +Step 26120/44000 | Loss: 2.9353 | LR: 1.83e-04 | 1316.4ms/step | 15.2 samples/s +Step 26130/44000 | Loss: 2.9248 | LR: 1.83e-04 | 1315.7ms/step | 15.2 samples/s +Step 26140/44000 | Loss: 2.9473 | LR: 1.82e-04 | 1313.0ms/step | 15.2 samples/s +Step 26150/44000 | Loss: 2.9313 | LR: 1.82e-04 | 1317.0ms/step | 15.2 samples/s +Step 26160/44000 | Loss: 2.9430 | LR: 1.82e-04 | 1318.8ms/step | 15.2 samples/s +Step 26170/44000 | Loss: 2.9478 | LR: 1.82e-04 | 1319.1ms/step | 15.2 samples/s +Step 26180/44000 | Loss: 2.9479 | LR: 1.82e-04 | 1317.9ms/step | 15.2 samples/s +Step 26190/44000 | Loss: 2.9751 | LR: 1.82e-04 | 1321.3ms/step | 15.1 samples/s +Step 26200/44000 | Loss: 2.9322 | LR: 1.81e-04 | 1319.6ms/step | 15.2 samples/s +Step 26210/44000 | Loss: 2.9856 | LR: 1.81e-04 | 1321.1ms/step | 15.1 samples/s +Step 26220/44000 | Loss: 2.9732 | LR: 1.81e-04 | 1319.8ms/step | 15.2 samples/s +Step 26230/44000 | Loss: 2.9756 | LR: 1.81e-04 | 1317.1ms/step | 15.2 samples/s +Step 26240/44000 | Loss: 2.9218 | LR: 1.81e-04 | 1318.8ms/step | 15.2 samples/s +Step 26250/44000 | Loss: 2.9728 | LR: 1.81e-04 | 1318.6ms/step | 15.2 samples/s +Validation at step 26250: Loss = 2.9372, Perplexity = 18.86 +Step 26260/44000 | Loss: 2.9298 | LR: 1.80e-04 | 3529.0ms/step | 5.7 samples/s +Step 26270/44000 | Loss: 2.9598 | LR: 1.80e-04 | 1318.7ms/step | 15.2 samples/s +Step 26280/44000 | Loss: 2.9475 | LR: 1.80e-04 | 1316.9ms/step | 15.2 samples/s +Step 26290/44000 | Loss: 2.9396 | LR: 1.80e-04 | 1315.6ms/step | 15.2 samples/s +Step 26300/44000 | Loss: 2.9496 | LR: 1.80e-04 | 1316.7ms/step | 15.2 samples/s +Step 26310/44000 | Loss: 2.9706 | LR: 1.80e-04 | 1317.2ms/step | 15.2 samples/s +Step 26320/44000 | Loss: 2.9610 | LR: 1.79e-04 | 1318.6ms/step | 15.2 samples/s +Step 26330/44000 | Loss: 2.9231 | LR: 1.79e-04 | 1318.1ms/step | 15.2 samples/s +Step 26340/44000 | Loss: 2.9691 | LR: 1.79e-04 | 1317.1ms/step | 15.2 samples/s +Step 26350/44000 | Loss: 2.9199 | LR: 1.79e-04 | 1317.4ms/step | 15.2 samples/s +Step 26360/44000 | Loss: 2.9408 | LR: 1.79e-04 | 1317.8ms/step | 15.2 samples/s +Step 26370/44000 | Loss: 2.9778 | LR: 1.78e-04 | 1316.1ms/step | 15.2 samples/s +Step 26380/44000 | Loss: 2.9725 | LR: 1.78e-04 | 1319.2ms/step | 15.2 samples/s +Step 26390/44000 | Loss: 2.9534 | LR: 1.78e-04 | 1319.5ms/step | 15.2 samples/s +Step 26400/44000 | Loss: 2.9380 | LR: 1.78e-04 | 1317.4ms/step | 15.2 samples/s +Step 26410/44000 | Loss: 2.9434 | LR: 1.78e-04 | 1316.0ms/step | 15.2 samples/s +Step 26420/44000 | Loss: 2.9709 | LR: 1.78e-04 | 1313.7ms/step | 15.2 samples/s +Step 26430/44000 | Loss: 2.9336 | LR: 1.77e-04 | 1317.4ms/step | 15.2 samples/s +Step 26440/44000 | Loss: 2.9624 | LR: 1.77e-04 | 1317.7ms/step | 15.2 samples/s +Step 26450/44000 | Loss: 2.9363 | LR: 1.77e-04 | 1316.8ms/step | 15.2 samples/s +Step 26460/44000 | Loss: 2.9333 | LR: 1.77e-04 | 1317.8ms/step | 15.2 samples/s +Step 26470/44000 | Loss: 2.9679 | LR: 1.77e-04 | 1317.5ms/step | 15.2 samples/s +Step 26480/44000 | Loss: 2.9418 | LR: 1.77e-04 | 1318.1ms/step | 15.2 samples/s +Step 26490/44000 | Loss: 2.9366 | LR: 1.76e-04 | 1320.6ms/step | 15.1 samples/s +Step 26500/44000 | Loss: 2.9443 | LR: 1.76e-04 | 1319.2ms/step | 15.2 samples/s +Validation at step 26500: Loss = 2.9357, Perplexity = 18.83 +Step 26510/44000 | Loss: 2.9547 | LR: 1.76e-04 | 3525.2ms/step | 5.7 samples/s +Step 26520/44000 | Loss: 2.9177 | LR: 1.76e-04 | 1321.1ms/step | 15.1 samples/s +Step 26530/44000 | Loss: 2.9431 | LR: 1.76e-04 | 1320.4ms/step | 15.1 samples/s +Step 26540/44000 | Loss: 2.9267 | LR: 1.76e-04 | 1318.7ms/step | 15.2 samples/s +Step 26550/44000 | Loss: 2.9202 | LR: 1.75e-04 | 1318.5ms/step | 15.2 samples/s +Step 26560/44000 | Loss: 2.9499 | LR: 1.75e-04 | 1319.2ms/step | 15.2 samples/s +Step 26570/44000 | Loss: 2.9161 | LR: 1.75e-04 | 1323.4ms/step | 15.1 samples/s +Step 26580/44000 | Loss: 2.9314 | LR: 1.75e-04 | 1321.3ms/step | 15.1 samples/s +Step 26590/44000 | Loss: 2.9637 | LR: 1.75e-04 | 1316.9ms/step | 15.2 samples/s +Step 26600/44000 | Loss: 2.9286 | LR: 1.74e-04 | 1315.9ms/step | 15.2 samples/s +Step 26610/44000 | Loss: 2.9512 | LR: 1.74e-04 | 1318.7ms/step | 15.2 samples/s +Step 26620/44000 | Loss: 2.9427 | LR: 1.74e-04 | 1319.6ms/step | 15.2 samples/s +Step 26630/44000 | Loss: 2.9337 | LR: 1.74e-04 | 1318.9ms/step | 15.2 samples/s +Step 26640/44000 | Loss: 2.9382 | LR: 1.74e-04 | 1319.7ms/step | 15.2 samples/s +Step 26650/44000 | Loss: 2.9381 | LR: 1.74e-04 | 1319.6ms/step | 15.2 samples/s +Step 26660/44000 | Loss: 2.9267 | LR: 1.73e-04 | 1318.2ms/step | 15.2 samples/s +Step 26670/44000 | Loss: 2.9692 | LR: 1.73e-04 | 1317.9ms/step | 15.2 samples/s +Step 26680/44000 | Loss: 2.9256 | LR: 1.73e-04 | 1317.2ms/step | 15.2 samples/s +Step 26690/44000 | Loss: 2.9585 | LR: 1.73e-04 | 1316.9ms/step | 15.2 samples/s +Step 26700/44000 | Loss: 2.9343 | LR: 1.73e-04 | 1319.3ms/step | 15.2 samples/s +Step 26710/44000 | Loss: 2.9334 | LR: 1.73e-04 | 1319.1ms/step | 15.2 samples/s +Step 26720/44000 | Loss: 2.9159 | LR: 1.72e-04 | 1315.9ms/step | 15.2 samples/s +Step 26730/44000 | Loss: 2.9434 | LR: 1.72e-04 | 1316.6ms/step | 15.2 samples/s +Step 26740/44000 | Loss: 2.9504 | LR: 1.72e-04 | 1315.9ms/step | 15.2 samples/s +Step 26750/44000 | Loss: 2.9357 | LR: 1.72e-04 | 1316.8ms/step | 15.2 samples/s +Validation at step 26750: Loss = 2.9344, Perplexity = 18.81 +Step 26760/44000 | Loss: 2.9193 | LR: 1.72e-04 | 3531.1ms/step | 5.7 samples/s +Step 26770/44000 | Loss: 2.9532 | LR: 1.72e-04 | 1319.7ms/step | 15.2 samples/s +Step 26780/44000 | Loss: 2.9522 | LR: 1.71e-04 | 1318.9ms/step | 15.2 samples/s +Step 26790/44000 | Loss: 2.9306 | LR: 1.71e-04 | 1318.6ms/step | 15.2 samples/s +Step 26800/44000 | Loss: 2.9863 | LR: 1.71e-04 | 1318.7ms/step | 15.2 samples/s +Step 26810/44000 | Loss: 2.9530 | LR: 1.71e-04 | 1318.3ms/step | 15.2 samples/s +Step 26820/44000 | Loss: 2.9134 | LR: 1.71e-04 | 1319.0ms/step | 15.2 samples/s +Step 26830/44000 | Loss: 2.9374 | LR: 1.71e-04 | 1318.2ms/step | 15.2 samples/s +Step 26840/44000 | Loss: 2.9379 | LR: 1.70e-04 | 1322.2ms/step | 15.1 samples/s +Step 26850/44000 | Loss: 2.9462 | LR: 1.70e-04 | 1321.1ms/step | 15.1 samples/s +Step 26860/44000 | Loss: 2.9615 | LR: 1.70e-04 | 1319.7ms/step | 15.2 samples/s +Step 26870/44000 | Loss: 2.9353 | LR: 1.70e-04 | 1321.3ms/step | 15.1 samples/s +Step 26880/44000 | Loss: 2.9571 | LR: 1.70e-04 | 1321.7ms/step | 15.1 samples/s +Step 26890/44000 | Loss: 2.9757 | LR: 1.69e-04 | 1322.0ms/step | 15.1 samples/s +Step 26900/44000 | Loss: 2.9178 | LR: 1.69e-04 | 1323.2ms/step | 15.1 samples/s +Step 26910/44000 | Loss: 2.9275 | LR: 1.69e-04 | 1322.0ms/step | 15.1 samples/s +Step 26920/44000 | Loss: 2.9354 | LR: 1.69e-04 | 1321.0ms/step | 15.1 samples/s +Step 26930/44000 | Loss: 2.9649 | LR: 1.69e-04 | 1319.9ms/step | 15.2 samples/s +Step 26940/44000 | Loss: 2.9496 | LR: 1.69e-04 | 1320.2ms/step | 15.1 samples/s +Step 26950/44000 | Loss: 2.9653 | LR: 1.68e-04 | 1321.8ms/step | 15.1 samples/s +Step 26960/44000 | Loss: 2.9194 | LR: 1.68e-04 | 1318.0ms/step | 15.2 samples/s +Step 26970/44000 | Loss: 2.9376 | LR: 1.68e-04 | 1318.7ms/step | 15.2 samples/s +Step 26980/44000 | Loss: 2.9428 | LR: 1.68e-04 | 1317.4ms/step | 15.2 samples/s +Step 26990/44000 | Loss: 2.9399 | LR: 1.68e-04 | 1317.5ms/step | 15.2 samples/s +Step 27000/44000 | Loss: 2.9446 | LR: 1.68e-04 | 1315.1ms/step | 15.2 samples/s +Validation at step 27000: Loss = 2.9330, Perplexity = 18.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-27000 +Step 27010/44000 | Loss: 2.9141 | LR: 1.67e-04 | 3813.2ms/step | 5.2 samples/s +Step 27020/44000 | Loss: 2.9362 | LR: 1.67e-04 | 1318.3ms/step | 15.2 samples/s +Step 27030/44000 | Loss: 2.9367 | LR: 1.67e-04 | 1314.0ms/step | 15.2 samples/s +Step 27040/44000 | Loss: 2.9506 | LR: 1.67e-04 | 1315.9ms/step | 15.2 samples/s +Step 27050/44000 | Loss: 2.9615 | LR: 1.67e-04 | 1316.6ms/step | 15.2 samples/s +Step 27060/44000 | Loss: 2.9573 | LR: 1.67e-04 | 1321.6ms/step | 15.1 samples/s +Step 27070/44000 | Loss: 2.9695 | LR: 1.66e-04 | 1317.6ms/step | 15.2 samples/s +Step 27080/44000 | Loss: 2.9399 | LR: 1.66e-04 | 1316.3ms/step | 15.2 samples/s +Step 27090/44000 | Loss: 2.9531 | LR: 1.66e-04 | 1317.2ms/step | 15.2 samples/s +Step 27100/44000 | Loss: 2.9459 | LR: 1.66e-04 | 1318.1ms/step | 15.2 samples/s +Step 27110/44000 | Loss: 2.9428 | LR: 1.66e-04 | 1315.9ms/step | 15.2 samples/s +Step 27120/44000 | Loss: 2.9468 | LR: 1.66e-04 | 1316.6ms/step | 15.2 samples/s +Step 27130/44000 | Loss: 2.9727 | LR: 1.65e-04 | 1318.0ms/step | 15.2 samples/s +Step 27140/44000 | Loss: 2.9490 | LR: 1.65e-04 | 1317.5ms/step | 15.2 samples/s +Step 27150/44000 | Loss: 2.9347 | LR: 1.65e-04 | 1316.7ms/step | 15.2 samples/s +Step 27160/44000 | Loss: 2.9389 | LR: 1.65e-04 | 1316.8ms/step | 15.2 samples/s +Step 27170/44000 | Loss: 2.9423 | LR: 1.65e-04 | 1315.6ms/step | 15.2 samples/s +Step 27180/44000 | Loss: 2.9434 | LR: 1.65e-04 | 1316.9ms/step | 15.2 samples/s +Step 27190/44000 | Loss: 2.9474 | LR: 1.64e-04 | 1316.5ms/step | 15.2 samples/s +Step 27200/44000 | Loss: 2.9639 | LR: 1.64e-04 | 1316.1ms/step | 15.2 samples/s +Step 27210/44000 | Loss: 2.9157 | LR: 1.64e-04 | 1317.3ms/step | 15.2 samples/s +Step 27220/44000 | Loss: 2.9435 | LR: 1.64e-04 | 1319.9ms/step | 15.2 samples/s +Step 27230/44000 | Loss: 2.9300 | LR: 1.64e-04 | 1320.5ms/step | 15.1 samples/s +Step 27240/44000 | Loss: 2.9633 | LR: 1.63e-04 | 1314.0ms/step | 15.2 samples/s +Step 27250/44000 | Loss: 2.9531 | LR: 1.63e-04 | 1315.5ms/step | 15.2 samples/s +Validation at step 27250: Loss = 2.9318, Perplexity = 18.76 +Step 27260/44000 | Loss: 2.9447 | LR: 1.63e-04 | 3528.0ms/step | 5.7 samples/s +Step 27270/44000 | Loss: 2.9542 | LR: 1.63e-04 | 1321.7ms/step | 15.1 samples/s +Step 27280/44000 | Loss: 2.9165 | LR: 1.63e-04 | 1316.0ms/step | 15.2 samples/s +Step 27290/44000 | Loss: 2.9455 | LR: 1.63e-04 | 1317.8ms/step | 15.2 samples/s +Step 27300/44000 | Loss: 2.9269 | LR: 1.62e-04 | 1320.1ms/step | 15.2 samples/s +Step 27310/44000 | Loss: 2.9379 | LR: 1.62e-04 | 1319.5ms/step | 15.2 samples/s +Step 27320/44000 | Loss: 2.9147 | LR: 1.62e-04 | 1321.2ms/step | 15.1 samples/s +Step 27330/44000 | Loss: 2.9424 | LR: 1.62e-04 | 1321.3ms/step | 15.1 samples/s +Step 27340/44000 | Loss: 2.9485 | LR: 1.62e-04 | 1319.9ms/step | 15.2 samples/s +Step 27350/44000 | Loss: 2.9163 | LR: 1.62e-04 | 1319.1ms/step | 15.2 samples/s +Step 27360/44000 | Loss: 2.9409 | LR: 1.61e-04 | 1318.2ms/step | 15.2 samples/s +Step 27370/44000 | Loss: 2.9483 | LR: 1.61e-04 | 1321.6ms/step | 15.1 samples/s +Step 27380/44000 | Loss: 2.9151 | LR: 1.61e-04 | 1320.1ms/step | 15.2 samples/s +Step 27390/44000 | Loss: 2.9417 | LR: 1.61e-04 | 1319.7ms/step | 15.2 samples/s +Step 27400/44000 | Loss: 2.9598 | LR: 1.61e-04 | 1317.7ms/step | 15.2 samples/s +Step 27410/44000 | Loss: 2.9618 | LR: 1.61e-04 | 1320.4ms/step | 15.1 samples/s +Step 27420/44000 | Loss: 2.9437 | LR: 1.60e-04 | 1321.1ms/step | 15.1 samples/s +Step 27430/44000 | Loss: 2.9567 | LR: 1.60e-04 | 1320.2ms/step | 15.1 samples/s +Step 27440/44000 | Loss: 2.9331 | LR: 1.60e-04 | 1318.5ms/step | 15.2 samples/s +Step 27450/44000 | Loss: 2.9615 | LR: 1.60e-04 | 1322.3ms/step | 15.1 samples/s +Step 27460/44000 | Loss: 2.9567 | LR: 1.60e-04 | 1320.5ms/step | 15.1 samples/s +Step 27470/44000 | Loss: 2.9391 | LR: 1.60e-04 | 1321.6ms/step | 15.1 samples/s +Step 27480/44000 | Loss: 2.9218 | LR: 1.59e-04 | 1317.2ms/step | 15.2 samples/s +Step 27490/44000 | Loss: 2.9528 | LR: 1.59e-04 | 1315.2ms/step | 15.2 samples/s +Step 27500/44000 | Loss: 2.9529 | LR: 1.59e-04 | 1315.5ms/step | 15.2 samples/s +Validation at step 27500: Loss = 2.9302, Perplexity = 18.73 +Step 27510/44000 | Loss: 2.9440 | LR: 1.59e-04 | 3528.1ms/step | 5.7 samples/s +Step 27520/44000 | Loss: 2.9557 | LR: 1.59e-04 | 1319.6ms/step | 15.2 samples/s +Step 27530/44000 | Loss: 2.9217 | LR: 1.59e-04 | 1319.5ms/step | 15.2 samples/s +Step 27540/44000 | Loss: 2.9349 | LR: 1.58e-04 | 1319.9ms/step | 15.2 samples/s +Step 27550/44000 | Loss: 2.9681 | LR: 1.58e-04 | 1324.1ms/step | 15.1 samples/s +Step 27560/44000 | Loss: 2.9214 | LR: 1.58e-04 | 1318.4ms/step | 15.2 samples/s +Step 27570/44000 | Loss: 2.9258 | LR: 1.58e-04 | 1319.2ms/step | 15.2 samples/s +Step 27580/44000 | Loss: 2.9484 | LR: 1.58e-04 | 1318.8ms/step | 15.2 samples/s +Step 27590/44000 | Loss: 2.9503 | LR: 1.58e-04 | 1319.7ms/step | 15.2 samples/s +Step 27600/44000 | Loss: 2.9588 | LR: 1.57e-04 | 1320.1ms/step | 15.2 samples/s +Step 27610/44000 | Loss: 2.9200 | LR: 1.57e-04 | 1319.5ms/step | 15.2 samples/s +Step 27620/44000 | Loss: 2.9159 | LR: 1.57e-04 | 1320.2ms/step | 15.1 samples/s +Step 27630/44000 | Loss: 2.9079 | LR: 1.57e-04 | 1320.3ms/step | 15.1 samples/s +Step 27640/44000 | Loss: 2.9118 | LR: 1.57e-04 | 1321.1ms/step | 15.1 samples/s +Step 27650/44000 | Loss: 2.9366 | LR: 1.57e-04 | 1321.8ms/step | 15.1 samples/s +Step 27660/44000 | Loss: 2.9300 | LR: 1.56e-04 | 1318.8ms/step | 15.2 samples/s +Step 27670/44000 | Loss: 2.9400 | LR: 1.56e-04 | 1317.8ms/step | 15.2 samples/s +Step 27680/44000 | Loss: 2.9442 | LR: 1.56e-04 | 1320.1ms/step | 15.2 samples/s +Step 27690/44000 | Loss: 2.9172 | LR: 1.56e-04 | 1321.7ms/step | 15.1 samples/s +Step 27700/44000 | Loss: 2.9476 | LR: 1.56e-04 | 1321.3ms/step | 15.1 samples/s +Step 27710/44000 | Loss: 2.9388 | LR: 1.56e-04 | 1321.3ms/step | 15.1 samples/s +Step 27720/44000 | Loss: 2.8917 | LR: 1.55e-04 | 1318.2ms/step | 15.2 samples/s +Step 27730/44000 | Loss: 2.9318 | LR: 1.55e-04 | 1321.8ms/step | 15.1 samples/s +Step 27740/44000 | Loss: 2.9082 | LR: 1.55e-04 | 1320.6ms/step | 15.1 samples/s +Step 27750/44000 | Loss: 2.9679 | LR: 1.55e-04 | 1320.4ms/step | 15.1 samples/s +Validation at step 27750: Loss = 2.9291, Perplexity = 18.71 +Step 27760/44000 | Loss: 2.9519 | LR: 1.55e-04 | 3529.4ms/step | 5.7 samples/s +Step 27770/44000 | Loss: 2.9405 | LR: 1.55e-04 | 1317.2ms/step | 15.2 samples/s +Step 27780/44000 | Loss: 2.9548 | LR: 1.54e-04 | 1316.8ms/step | 15.2 samples/s +Step 27790/44000 | Loss: 2.9212 | LR: 1.54e-04 | 1315.9ms/step | 15.2 samples/s +Step 27800/44000 | Loss: 2.9584 | LR: 1.54e-04 | 1315.9ms/step | 15.2 samples/s +Step 27810/44000 | Loss: 2.9439 | LR: 1.54e-04 | 1316.7ms/step | 15.2 samples/s +Step 27820/44000 | Loss: 2.9128 | LR: 1.54e-04 | 1318.6ms/step | 15.2 samples/s +Step 27830/44000 | Loss: 2.9440 | LR: 1.54e-04 | 1318.0ms/step | 15.2 samples/s +Step 27840/44000 | Loss: 2.9630 | LR: 1.53e-04 | 1315.7ms/step | 15.2 samples/s +Step 27850/44000 | Loss: 2.9243 | LR: 1.53e-04 | 1315.9ms/step | 15.2 samples/s +Step 27860/44000 | Loss: 2.9443 | LR: 1.53e-04 | 1319.2ms/step | 15.2 samples/s +Step 27870/44000 | Loss: 2.9495 | LR: 1.53e-04 | 1318.2ms/step | 15.2 samples/s +Step 27880/44000 | Loss: 2.9642 | LR: 1.53e-04 | 1317.4ms/step | 15.2 samples/s +Step 27890/44000 | Loss: 2.9381 | LR: 1.53e-04 | 1315.3ms/step | 15.2 samples/s +Step 27900/44000 | Loss: 2.9430 | LR: 1.52e-04 | 1316.3ms/step | 15.2 samples/s +Step 27910/44000 | Loss: 2.9131 | LR: 1.52e-04 | 1319.8ms/step | 15.2 samples/s +Step 27920/44000 | Loss: 2.9491 | LR: 1.52e-04 | 1320.3ms/step | 15.1 samples/s +Step 27930/44000 | Loss: 2.9709 | LR: 1.52e-04 | 1316.9ms/step | 15.2 samples/s +Step 27940/44000 | Loss: 2.9188 | LR: 1.52e-04 | 1319.0ms/step | 15.2 samples/s +Step 27950/44000 | Loss: 2.9499 | LR: 1.52e-04 | 1321.0ms/step | 15.1 samples/s +Step 27960/44000 | Loss: 2.9320 | LR: 1.51e-04 | 1321.1ms/step | 15.1 samples/s +Step 27970/44000 | Loss: 2.9246 | LR: 1.51e-04 | 1321.3ms/step | 15.1 samples/s +Step 27980/44000 | Loss: 2.9599 | LR: 1.51e-04 | 1321.2ms/step | 15.1 samples/s +Step 27990/44000 | Loss: 2.9802 | LR: 1.51e-04 | 1320.3ms/step | 15.1 samples/s +Step 28000/44000 | Loss: 2.9486 | LR: 1.51e-04 | 1319.6ms/step | 15.2 samples/s +Validation at step 28000: Loss = 2.9279, Perplexity = 18.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-28000 +Step 28010/44000 | Loss: 2.9586 | LR: 1.51e-04 | 3820.7ms/step | 5.2 samples/s +Step 28020/44000 | Loss: 2.9544 | LR: 1.50e-04 | 1314.3ms/step | 15.2 samples/s +Step 28030/44000 | Loss: 2.9128 | LR: 1.50e-04 | 1315.9ms/step | 15.2 samples/s +Step 28040/44000 | Loss: 2.9373 | LR: 1.50e-04 | 1319.7ms/step | 15.2 samples/s +Step 28050/44000 | Loss: 2.9335 | LR: 1.50e-04 | 1321.0ms/step | 15.1 samples/s +Step 28060/44000 | Loss: 2.9599 | LR: 1.50e-04 | 1319.5ms/step | 15.2 samples/s +Step 28070/44000 | Loss: 2.9697 | LR: 1.50e-04 | 1317.1ms/step | 15.2 samples/s +Step 28080/44000 | Loss: 2.9196 | LR: 1.49e-04 | 1323.3ms/step | 15.1 samples/s +Step 28090/44000 | Loss: 2.9466 | LR: 1.49e-04 | 1320.6ms/step | 15.1 samples/s +Step 28100/44000 | Loss: 2.9533 | LR: 1.49e-04 | 1318.2ms/step | 15.2 samples/s +Step 28110/44000 | Loss: 2.9342 | LR: 1.49e-04 | 1319.1ms/step | 15.2 samples/s +Step 28120/44000 | Loss: 2.9193 | LR: 1.49e-04 | 1322.0ms/step | 15.1 samples/s +Step 28130/44000 | Loss: 2.9092 | LR: 1.49e-04 | 1321.6ms/step | 15.1 samples/s +Step 28140/44000 | Loss: 2.9274 | LR: 1.48e-04 | 1321.5ms/step | 15.1 samples/s +Step 28150/44000 | Loss: 2.9241 | LR: 1.48e-04 | 1317.2ms/step | 15.2 samples/s +Step 28160/44000 | Loss: 2.9381 | LR: 1.48e-04 | 1315.5ms/step | 15.2 samples/s +Step 28170/44000 | Loss: 2.9259 | LR: 1.48e-04 | 1320.9ms/step | 15.1 samples/s +Step 28180/44000 | Loss: 2.9433 | LR: 1.48e-04 | 1321.1ms/step | 15.1 samples/s +Step 28190/44000 | Loss: 2.9397 | LR: 1.48e-04 | 1319.2ms/step | 15.2 samples/s +Step 28200/44000 | Loss: 2.9213 | LR: 1.47e-04 | 1317.2ms/step | 15.2 samples/s +Step 28210/44000 | Loss: 2.9573 | LR: 1.47e-04 | 1319.8ms/step | 15.2 samples/s +Step 28220/44000 | Loss: 2.9850 | LR: 1.47e-04 | 1323.0ms/step | 15.1 samples/s +Step 28230/44000 | Loss: 2.9235 | LR: 1.47e-04 | 1319.9ms/step | 15.2 samples/s +Step 28240/44000 | Loss: 2.9337 | LR: 1.47e-04 | 1320.5ms/step | 15.1 samples/s +Step 28250/44000 | Loss: 2.9441 | LR: 1.47e-04 | 1319.5ms/step | 15.2 samples/s +Validation at step 28250: Loss = 2.9270, Perplexity = 18.67 +Step 28260/44000 | Loss: 2.9471 | LR: 1.46e-04 | 3527.4ms/step | 5.7 samples/s +Step 28270/44000 | Loss: 2.9470 | LR: 1.46e-04 | 1318.3ms/step | 15.2 samples/s +Step 28280/44000 | Loss: 2.9034 | LR: 1.46e-04 | 1319.0ms/step | 15.2 samples/s +Step 28290/44000 | Loss: 2.9147 | LR: 1.46e-04 | 1319.1ms/step | 15.2 samples/s +Step 28300/44000 | Loss: 2.9312 | LR: 1.46e-04 | 1320.6ms/step | 15.1 samples/s +Step 28310/44000 | Loss: 2.9409 | LR: 1.46e-04 | 1319.5ms/step | 15.2 samples/s +Step 28320/44000 | Loss: 2.9301 | LR: 1.45e-04 | 1316.0ms/step | 15.2 samples/s +Step 28330/44000 | Loss: 2.9506 | LR: 1.45e-04 | 1318.9ms/step | 15.2 samples/s +Step 28340/44000 | Loss: 2.9489 | LR: 1.45e-04 | 1322.6ms/step | 15.1 samples/s +Step 28350/44000 | Loss: 2.9131 | LR: 1.45e-04 | 1319.8ms/step | 15.2 samples/s +Step 28360/44000 | Loss: 2.9381 | LR: 1.45e-04 | 1319.5ms/step | 15.2 samples/s +Step 28370/44000 | Loss: 2.9128 | LR: 1.45e-04 | 1319.1ms/step | 15.2 samples/s +Step 28380/44000 | Loss: 2.9056 | LR: 1.44e-04 | 1319.7ms/step | 15.2 samples/s +Step 28390/44000 | Loss: 2.9215 | LR: 1.44e-04 | 1318.0ms/step | 15.2 samples/s +Step 28400/44000 | Loss: 2.9361 | LR: 1.44e-04 | 1316.0ms/step | 15.2 samples/s +Step 28410/44000 | Loss: 2.9452 | LR: 1.44e-04 | 1315.2ms/step | 15.2 samples/s +Step 28420/44000 | Loss: 2.9236 | LR: 1.44e-04 | 1315.7ms/step | 15.2 samples/s +Step 28430/44000 | Loss: 2.9169 | LR: 1.44e-04 | 1317.7ms/step | 15.2 samples/s +Step 28440/44000 | Loss: 2.9543 | LR: 1.43e-04 | 1315.8ms/step | 15.2 samples/s +Step 28450/44000 | Loss: 2.9408 | LR: 1.43e-04 | 1317.0ms/step | 15.2 samples/s +Step 28460/44000 | Loss: 2.9317 | LR: 1.43e-04 | 1320.5ms/step | 15.1 samples/s +Step 28470/44000 | Loss: 2.9289 | LR: 1.43e-04 | 1320.2ms/step | 15.1 samples/s +Step 28480/44000 | Loss: 2.9707 | LR: 1.43e-04 | 1323.4ms/step | 15.1 samples/s +Step 28490/44000 | Loss: 2.9279 | LR: 1.43e-04 | 1318.6ms/step | 15.2 samples/s +Step 28500/44000 | Loss: 2.9442 | LR: 1.42e-04 | 1316.9ms/step | 15.2 samples/s +Validation at step 28500: Loss = 2.9255, Perplexity = 18.64 +Step 28510/44000 | Loss: 2.9242 | LR: 1.42e-04 | 3525.8ms/step | 5.7 samples/s +Step 28520/44000 | Loss: 2.9440 | LR: 1.42e-04 | 1318.5ms/step | 15.2 samples/s +Step 28530/44000 | Loss: 2.9129 | LR: 1.42e-04 | 1319.9ms/step | 15.2 samples/s +Step 28540/44000 | Loss: 2.9484 | LR: 1.42e-04 | 1322.5ms/step | 15.1 samples/s +Step 28550/44000 | Loss: 2.9312 | LR: 1.42e-04 | 1319.7ms/step | 15.2 samples/s +Step 28560/44000 | Loss: 2.9455 | LR: 1.41e-04 | 1321.9ms/step | 15.1 samples/s +Step 28570/44000 | Loss: 2.9069 | LR: 1.41e-04 | 1323.0ms/step | 15.1 samples/s +Step 28580/44000 | Loss: 2.9526 | LR: 1.41e-04 | 1320.4ms/step | 15.1 samples/s +Step 28590/44000 | Loss: 2.9446 | LR: 1.41e-04 | 1319.9ms/step | 15.2 samples/s +Step 28600/44000 | Loss: 2.9452 | LR: 1.41e-04 | 1319.8ms/step | 15.2 samples/s +Step 28610/44000 | Loss: 2.9349 | LR: 1.41e-04 | 1321.0ms/step | 15.1 samples/s +Step 28620/44000 | Loss: 2.9006 | LR: 1.40e-04 | 1320.4ms/step | 15.1 samples/s +Step 28630/44000 | Loss: 2.9109 | LR: 1.40e-04 | 1320.2ms/step | 15.1 samples/s +Step 28640/44000 | Loss: 2.9247 | LR: 1.40e-04 | 1323.2ms/step | 15.1 samples/s +Step 28650/44000 | Loss: 2.9487 | LR: 1.40e-04 | 1323.0ms/step | 15.1 samples/s +Step 28660/44000 | Loss: 2.9187 | LR: 1.40e-04 | 1322.9ms/step | 15.1 samples/s +Step 28670/44000 | Loss: 2.9379 | LR: 1.40e-04 | 1321.4ms/step | 15.1 samples/s +Step 28680/44000 | Loss: 2.9406 | LR: 1.39e-04 | 1320.8ms/step | 15.1 samples/s +Step 28690/44000 | Loss: 2.9199 | LR: 1.39e-04 | 1320.8ms/step | 15.1 samples/s +Step 28700/44000 | Loss: 2.9545 | LR: 1.39e-04 | 1317.9ms/step | 15.2 samples/s +Step 28710/44000 | Loss: 2.9425 | LR: 1.39e-04 | 1318.3ms/step | 15.2 samples/s +Step 28720/44000 | Loss: 2.8965 | LR: 1.39e-04 | 1322.1ms/step | 15.1 samples/s +Step 28730/44000 | Loss: 2.9158 | LR: 1.39e-04 | 1323.2ms/step | 15.1 samples/s +Step 28740/44000 | Loss: 2.9281 | LR: 1.38e-04 | 1320.7ms/step | 15.1 samples/s +Step 28750/44000 | Loss: 2.9126 | LR: 1.38e-04 | 1319.7ms/step | 15.2 samples/s +Validation at step 28750: Loss = 2.9244, Perplexity = 18.62 +Step 28760/44000 | Loss: 2.9367 | LR: 1.38e-04 | 3525.0ms/step | 5.7 samples/s +Step 28770/44000 | Loss: 2.9234 | LR: 1.38e-04 | 1318.2ms/step | 15.2 samples/s +Step 28780/44000 | Loss: 2.9184 | LR: 1.38e-04 | 1318.3ms/step | 15.2 samples/s +Step 28790/44000 | Loss: 2.9413 | LR: 1.38e-04 | 1316.6ms/step | 15.2 samples/s +Step 28800/44000 | Loss: 2.9470 | LR: 1.38e-04 | 1317.3ms/step | 15.2 samples/s +Step 28810/44000 | Loss: 2.9384 | LR: 1.37e-04 | 1317.7ms/step | 15.2 samples/s +Step 28820/44000 | Loss: 2.9374 | LR: 1.37e-04 | 1321.2ms/step | 15.1 samples/s +Step 28830/44000 | Loss: 2.9316 | LR: 1.37e-04 | 1324.5ms/step | 15.1 samples/s +Step 28840/44000 | Loss: 2.9395 | LR: 1.37e-04 | 1321.4ms/step | 15.1 samples/s +Step 28850/44000 | Loss: 2.9155 | LR: 1.37e-04 | 1320.9ms/step | 15.1 samples/s +Step 28860/44000 | Loss: 2.9298 | LR: 1.37e-04 | 1321.1ms/step | 15.1 samples/s +Step 28870/44000 | Loss: 2.9725 | LR: 1.36e-04 | 1321.1ms/step | 15.1 samples/s +Step 28880/44000 | Loss: 2.8998 | LR: 1.36e-04 | 1319.8ms/step | 15.2 samples/s +Step 28890/44000 | Loss: 2.9241 | LR: 1.36e-04 | 1319.8ms/step | 15.2 samples/s +Step 28900/44000 | Loss: 2.9521 | LR: 1.36e-04 | 1322.8ms/step | 15.1 samples/s +Step 28910/44000 | Loss: 2.9223 | LR: 1.36e-04 | 1320.5ms/step | 15.1 samples/s +Step 28920/44000 | Loss: 2.9402 | LR: 1.36e-04 | 1320.1ms/step | 15.2 samples/s +Step 28930/44000 | Loss: 2.9624 | LR: 1.35e-04 | 1319.5ms/step | 15.2 samples/s +Step 28940/44000 | Loss: 2.9196 | LR: 1.35e-04 | 1319.5ms/step | 15.2 samples/s +Step 28950/44000 | Loss: 2.9292 | LR: 1.35e-04 | 1318.0ms/step | 15.2 samples/s +Step 28960/44000 | Loss: 2.9171 | LR: 1.35e-04 | 1318.8ms/step | 15.2 samples/s +Step 28970/44000 | Loss: 2.9623 | LR: 1.35e-04 | 1319.6ms/step | 15.2 samples/s +Step 28980/44000 | Loss: 2.9154 | LR: 1.35e-04 | 1319.4ms/step | 15.2 samples/s +Step 28990/44000 | Loss: 2.9334 | LR: 1.34e-04 | 1320.0ms/step | 15.2 samples/s +Step 29000/44000 | Loss: 2.9197 | LR: 1.34e-04 | 1317.8ms/step | 15.2 samples/s +Validation at step 29000: Loss = 2.9236, Perplexity = 18.61 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-29000 +Step 29010/44000 | Loss: 2.9444 | LR: 1.34e-04 | 3815.7ms/step | 5.2 samples/s +Step 29020/44000 | Loss: 2.9092 | LR: 1.34e-04 | 1319.1ms/step | 15.2 samples/s +Step 29030/44000 | Loss: 2.9246 | LR: 1.34e-04 | 1315.0ms/step | 15.2 samples/s +Step 29040/44000 | Loss: 2.9190 | LR: 1.34e-04 | 1312.8ms/step | 15.2 samples/s +Step 29050/44000 | Loss: 2.9067 | LR: 1.33e-04 | 1312.1ms/step | 15.2 samples/s +Step 29060/44000 | Loss: 2.9362 | LR: 1.33e-04 | 1313.6ms/step | 15.2 samples/s +Step 29070/44000 | Loss: 2.9019 | LR: 1.33e-04 | 1314.0ms/step | 15.2 samples/s +Step 29080/44000 | Loss: 2.9574 | LR: 1.33e-04 | 1312.3ms/step | 15.2 samples/s +Step 29090/44000 | Loss: 2.9362 | LR: 1.33e-04 | 1309.7ms/step | 15.3 samples/s +Step 29100/44000 | Loss: 2.9508 | LR: 1.33e-04 | 1310.1ms/step | 15.3 samples/s +Step 29110/44000 | Loss: 2.9393 | LR: 1.33e-04 | 1307.8ms/step | 15.3 samples/s +Step 29120/44000 | Loss: 2.9336 | LR: 1.32e-04 | 1310.9ms/step | 15.3 samples/s +Step 29130/44000 | Loss: 2.9045 | LR: 1.32e-04 | 1308.9ms/step | 15.3 samples/s +Step 29140/44000 | Loss: 2.9144 | LR: 1.32e-04 | 1314.2ms/step | 15.2 samples/s +Step 29150/44000 | Loss: 2.9147 | LR: 1.32e-04 | 1313.1ms/step | 15.2 samples/s +Step 29160/44000 | Loss: 2.9418 | LR: 1.32e-04 | 1312.4ms/step | 15.2 samples/s +Step 29170/44000 | Loss: 2.9476 | LR: 1.32e-04 | 1311.6ms/step | 15.2 samples/s +Step 29180/44000 | Loss: 2.9358 | LR: 1.31e-04 | 1313.4ms/step | 15.2 samples/s +Step 29190/44000 | Loss: 2.9385 | LR: 1.31e-04 | 1312.6ms/step | 15.2 samples/s +Step 29200/44000 | Loss: 2.9144 | LR: 1.31e-04 | 1313.0ms/step | 15.2 samples/s +Step 29210/44000 | Loss: 2.9118 | LR: 1.31e-04 | 1313.2ms/step | 15.2 samples/s +Step 29220/44000 | Loss: 2.9558 | LR: 1.31e-04 | 1313.7ms/step | 15.2 samples/s +Step 29230/44000 | Loss: 2.9325 | LR: 1.31e-04 | 1314.9ms/step | 15.2 samples/s +Step 29240/44000 | Loss: 2.9307 | LR: 1.30e-04 | 1313.6ms/step | 15.2 samples/s +Step 29250/44000 | Loss: 2.9436 | LR: 1.30e-04 | 1313.4ms/step | 15.2 samples/s +Validation at step 29250: Loss = 2.9224, Perplexity = 18.59 +Step 29260/44000 | Loss: 2.9340 | LR: 1.30e-04 | 3515.4ms/step | 5.7 samples/s +Step 29270/44000 | Loss: 2.9406 | LR: 1.30e-04 | 1318.1ms/step | 15.2 samples/s +Step 29280/44000 | Loss: 2.9144 | LR: 1.30e-04 | 1321.0ms/step | 15.1 samples/s +Step 29290/44000 | Loss: 2.9507 | LR: 1.30e-04 | 1322.9ms/step | 15.1 samples/s +Step 29300/44000 | Loss: 2.9424 | LR: 1.29e-04 | 1322.8ms/step | 15.1 samples/s +Step 29310/44000 | Loss: 2.9369 | LR: 1.29e-04 | 1320.0ms/step | 15.2 samples/s +Step 29320/44000 | Loss: 2.9126 | LR: 1.29e-04 | 1320.6ms/step | 15.1 samples/s +Step 29330/44000 | Loss: 2.9302 | LR: 1.29e-04 | 1322.1ms/step | 15.1 samples/s +Step 29340/44000 | Loss: 2.9011 | LR: 1.29e-04 | 1318.2ms/step | 15.2 samples/s +Step 29350/44000 | Loss: 2.9260 | LR: 1.29e-04 | 1321.0ms/step | 15.1 samples/s +Step 29360/44000 | Loss: 2.9233 | LR: 1.29e-04 | 1318.9ms/step | 15.2 samples/s +Step 29370/44000 | Loss: 2.9531 | LR: 1.28e-04 | 1318.7ms/step | 15.2 samples/s +Step 29380/44000 | Loss: 2.9158 | LR: 1.28e-04 | 1318.2ms/step | 15.2 samples/s +Step 29390/44000 | Loss: 2.9629 | LR: 1.28e-04 | 1318.0ms/step | 15.2 samples/s +Step 29400/44000 | Loss: 2.9238 | LR: 1.28e-04 | 1320.5ms/step | 15.1 samples/s +Step 29410/44000 | Loss: 2.9306 | LR: 1.28e-04 | 1318.9ms/step | 15.2 samples/s +Step 29420/44000 | Loss: 2.9683 | LR: 1.28e-04 | 1321.2ms/step | 15.1 samples/s +Step 29430/44000 | Loss: 2.9221 | LR: 1.27e-04 | 1318.9ms/step | 15.2 samples/s +Step 29440/44000 | Loss: 2.9380 | LR: 1.27e-04 | 1317.7ms/step | 15.2 samples/s +Step 29450/44000 | Loss: 2.9369 | LR: 1.27e-04 | 1319.0ms/step | 15.2 samples/s +Step 29460/44000 | Loss: 2.9482 | LR: 1.27e-04 | 1320.0ms/step | 15.2 samples/s +Step 29470/44000 | Loss: 2.9046 | LR: 1.27e-04 | 1320.1ms/step | 15.1 samples/s +Step 29480/44000 | Loss: 2.9608 | LR: 1.27e-04 | 1321.1ms/step | 15.1 samples/s +Step 29490/44000 | Loss: 2.9496 | LR: 1.26e-04 | 1321.7ms/step | 15.1 samples/s +Step 29500/44000 | Loss: 2.9411 | LR: 1.26e-04 | 1322.8ms/step | 15.1 samples/s +Validation at step 29500: Loss = 2.9214, Perplexity = 18.57 +Step 29510/44000 | Loss: 2.9392 | LR: 1.26e-04 | 3531.4ms/step | 5.7 samples/s +Step 29520/44000 | Loss: 2.9557 | LR: 1.26e-04 | 1318.6ms/step | 15.2 samples/s +Step 29530/44000 | Loss: 2.9229 | LR: 1.26e-04 | 1318.2ms/step | 15.2 samples/s +Step 29540/44000 | Loss: 2.9293 | LR: 1.26e-04 | 1318.6ms/step | 15.2 samples/s +Step 29550/44000 | Loss: 2.9055 | LR: 1.26e-04 | 1320.6ms/step | 15.1 samples/s +Step 29560/44000 | Loss: 2.9331 | LR: 1.25e-04 | 1321.0ms/step | 15.1 samples/s +Step 29570/44000 | Loss: 2.9170 | LR: 1.25e-04 | 1318.4ms/step | 15.2 samples/s +Step 29580/44000 | Loss: 2.9364 | LR: 1.25e-04 | 1318.9ms/step | 15.2 samples/s +Step 29590/44000 | Loss: 2.9225 | LR: 1.25e-04 | 1318.2ms/step | 15.2 samples/s +Step 29600/44000 | Loss: 2.9497 | LR: 1.25e-04 | 1316.9ms/step | 15.2 samples/s +Step 29610/44000 | Loss: 2.9499 | LR: 1.25e-04 | 1320.8ms/step | 15.1 samples/s +Step 29620/44000 | Loss: 2.9387 | LR: 1.24e-04 | 1321.2ms/step | 15.1 samples/s +Step 29630/44000 | Loss: 2.9413 | LR: 1.24e-04 | 1318.9ms/step | 15.2 samples/s +Step 29640/44000 | Loss: 2.9133 | LR: 1.24e-04 | 1318.8ms/step | 15.2 samples/s +Step 29650/44000 | Loss: 2.9139 | LR: 1.24e-04 | 1319.5ms/step | 15.2 samples/s +Step 29660/44000 | Loss: 2.9418 | LR: 1.24e-04 | 1319.3ms/step | 15.2 samples/s +Step 29670/44000 | Loss: 2.9257 | LR: 1.24e-04 | 1323.1ms/step | 15.1 samples/s +Step 29680/44000 | Loss: 2.9392 | LR: 1.23e-04 | 1319.8ms/step | 15.2 samples/s +Step 29690/44000 | Loss: 2.9455 | LR: 1.23e-04 | 1318.3ms/step | 15.2 samples/s +Step 29700/44000 | Loss: 2.9254 | LR: 1.23e-04 | 1319.3ms/step | 15.2 samples/s +Step 29710/44000 | Loss: 2.9250 | LR: 1.23e-04 | 1319.2ms/step | 15.2 samples/s +Step 29720/44000 | Loss: 2.9529 | LR: 1.23e-04 | 1319.5ms/step | 15.2 samples/s +Step 29730/44000 | Loss: 2.9348 | LR: 1.23e-04 | 1320.5ms/step | 15.1 samples/s +Step 29740/44000 | Loss: 2.9307 | LR: 1.23e-04 | 1319.9ms/step | 15.2 samples/s +Step 29750/44000 | Loss: 2.9322 | LR: 1.22e-04 | 1321.1ms/step | 15.1 samples/s +Validation at step 29750: Loss = 2.9206, Perplexity = 18.55 +Step 29760/44000 | Loss: 2.9470 | LR: 1.22e-04 | 3532.0ms/step | 5.7 samples/s +Step 29770/44000 | Loss: 2.9495 | LR: 1.22e-04 | 1321.9ms/step | 15.1 samples/s +Step 29780/44000 | Loss: 2.8938 | LR: 1.22e-04 | 1316.3ms/step | 15.2 samples/s +Step 29790/44000 | Loss: 2.9114 | LR: 1.22e-04 | 1316.9ms/step | 15.2 samples/s +Step 29800/44000 | Loss: 2.9256 | LR: 1.22e-04 | 1317.1ms/step | 15.2 samples/s +Step 29810/44000 | Loss: 2.9409 | LR: 1.21e-04 | 1321.7ms/step | 15.1 samples/s +Step 29820/44000 | Loss: 2.9325 | LR: 1.21e-04 | 1318.9ms/step | 15.2 samples/s +Step 29830/44000 | Loss: 2.9123 | LR: 1.21e-04 | 1317.4ms/step | 15.2 samples/s +Step 29840/44000 | Loss: 2.9315 | LR: 1.21e-04 | 1320.4ms/step | 15.1 samples/s +Step 29850/44000 | Loss: 2.9369 | LR: 1.21e-04 | 1315.8ms/step | 15.2 samples/s +Step 29860/44000 | Loss: 2.9374 | LR: 1.21e-04 | 1318.5ms/step | 15.2 samples/s +Step 29870/44000 | Loss: 2.9366 | LR: 1.21e-04 | 1318.0ms/step | 15.2 samples/s +Step 29880/44000 | Loss: 2.9259 | LR: 1.20e-04 | 1318.9ms/step | 15.2 samples/s +Step 29890/44000 | Loss: 2.9076 | LR: 1.20e-04 | 1319.4ms/step | 15.2 samples/s +Step 29900/44000 | Loss: 2.9370 | LR: 1.20e-04 | 1319.2ms/step | 15.2 samples/s +Step 29910/44000 | Loss: 2.9248 | LR: 1.20e-04 | 1316.5ms/step | 15.2 samples/s +Step 29920/44000 | Loss: 2.9497 | LR: 1.20e-04 | 1319.0ms/step | 15.2 samples/s +Step 29930/44000 | Loss: 2.9643 | LR: 1.20e-04 | 1318.9ms/step | 15.2 samples/s +Step 29940/44000 | Loss: 2.9409 | LR: 1.19e-04 | 1319.5ms/step | 15.2 samples/s +Step 29950/44000 | Loss: 2.9386 | LR: 1.19e-04 | 1318.7ms/step | 15.2 samples/s +Step 29960/44000 | Loss: 2.9602 | LR: 1.19e-04 | 1318.4ms/step | 15.2 samples/s +Step 29970/44000 | Loss: 2.9235 | LR: 1.19e-04 | 1318.1ms/step | 15.2 samples/s +Step 29980/44000 | Loss: 2.9369 | LR: 1.19e-04 | 1317.2ms/step | 15.2 samples/s +Step 29990/44000 | Loss: 2.9515 | LR: 1.19e-04 | 1315.3ms/step | 15.2 samples/s +Step 30000/44000 | Loss: 2.9243 | LR: 1.19e-04 | 1316.9ms/step | 15.2 samples/s +Validation at step 30000: Loss = 2.9194, Perplexity = 18.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-30000 +Step 30010/44000 | Loss: 2.9097 | LR: 1.18e-04 | 3815.8ms/step | 5.2 samples/s +Step 30020/44000 | Loss: 2.9186 | LR: 1.18e-04 | 1319.2ms/step | 15.2 samples/s +Step 30030/44000 | Loss: 2.9156 | LR: 1.18e-04 | 1318.7ms/step | 15.2 samples/s +Step 30040/44000 | Loss: 2.9284 | LR: 1.18e-04 | 1320.8ms/step | 15.1 samples/s +Step 30050/44000 | Loss: 2.9390 | LR: 1.18e-04 | 1320.3ms/step | 15.1 samples/s +Step 30060/44000 | Loss: 2.9459 | LR: 1.18e-04 | 1318.7ms/step | 15.2 samples/s +Step 30070/44000 | Loss: 2.8977 | LR: 1.17e-04 | 1320.9ms/step | 15.1 samples/s +Step 30080/44000 | Loss: 2.9347 | LR: 1.17e-04 | 1316.1ms/step | 15.2 samples/s +Step 30090/44000 | Loss: 2.9403 | LR: 1.17e-04 | 1319.8ms/step | 15.2 samples/s +Step 30100/44000 | Loss: 2.9081 | LR: 1.17e-04 | 1317.7ms/step | 15.2 samples/s +Step 30110/44000 | Loss: 2.9312 | LR: 1.17e-04 | 1318.2ms/step | 15.2 samples/s +Step 30120/44000 | Loss: 2.9115 | LR: 1.17e-04 | 1321.0ms/step | 15.1 samples/s +Step 30130/44000 | Loss: 2.9120 | LR: 1.17e-04 | 1319.2ms/step | 15.2 samples/s +Step 30140/44000 | Loss: 2.9275 | LR: 1.16e-04 | 1319.0ms/step | 15.2 samples/s +Step 30150/44000 | Loss: 2.9324 | LR: 1.16e-04 | 1319.6ms/step | 15.2 samples/s +Step 30160/44000 | Loss: 2.9262 | LR: 1.16e-04 | 1317.8ms/step | 15.2 samples/s +Step 30170/44000 | Loss: 2.9532 | LR: 1.16e-04 | 1316.5ms/step | 15.2 samples/s +Step 30180/44000 | Loss: 2.9324 | LR: 1.16e-04 | 1318.7ms/step | 15.2 samples/s +Step 30190/44000 | Loss: 2.9414 | LR: 1.16e-04 | 1315.9ms/step | 15.2 samples/s +Step 30200/44000 | Loss: 2.9235 | LR: 1.15e-04 | 1317.7ms/step | 15.2 samples/s +Step 30210/44000 | Loss: 2.9337 | LR: 1.15e-04 | 1316.8ms/step | 15.2 samples/s +Step 30220/44000 | Loss: 2.9131 | LR: 1.15e-04 | 1321.4ms/step | 15.1 samples/s +Step 30230/44000 | Loss: 2.9405 | LR: 1.15e-04 | 1321.1ms/step | 15.1 samples/s +Step 30240/44000 | Loss: 2.9239 | LR: 1.15e-04 | 1322.5ms/step | 15.1 samples/s +Step 30250/44000 | Loss: 2.9132 | LR: 1.15e-04 | 1322.7ms/step | 15.1 samples/s +Validation at step 30250: Loss = 2.9187, Perplexity = 18.52 +Step 30260/44000 | Loss: 2.9424 | LR: 1.15e-04 | 3535.7ms/step | 5.7 samples/s +Step 30270/44000 | Loss: 2.9439 | LR: 1.14e-04 | 1322.0ms/step | 15.1 samples/s +Step 30280/44000 | Loss: 2.9335 | LR: 1.14e-04 | 1320.5ms/step | 15.1 samples/s +Step 30290/44000 | Loss: 2.9126 | LR: 1.14e-04 | 1323.1ms/step | 15.1 samples/s +Step 30300/44000 | Loss: 2.9184 | LR: 1.14e-04 | 1317.9ms/step | 15.2 samples/s +Step 30310/44000 | Loss: 2.8740 | LR: 1.14e-04 | 1317.9ms/step | 15.2 samples/s +Step 30320/44000 | Loss: 2.9045 | LR: 1.14e-04 | 1315.5ms/step | 15.2 samples/s +Step 30330/44000 | Loss: 2.9339 | LR: 1.13e-04 | 1315.8ms/step | 15.2 samples/s +Step 30340/44000 | Loss: 2.9242 | LR: 1.13e-04 | 1315.2ms/step | 15.2 samples/s +Step 30350/44000 | Loss: 2.9405 | LR: 1.13e-04 | 1317.3ms/step | 15.2 samples/s +Step 30360/44000 | Loss: 2.9393 | LR: 1.13e-04 | 1316.0ms/step | 15.2 samples/s +Step 30370/44000 | Loss: 2.9218 | LR: 1.13e-04 | 1318.4ms/step | 15.2 samples/s +Step 30380/44000 | Loss: 2.9217 | LR: 1.13e-04 | 1320.7ms/step | 15.1 samples/s +Step 30390/44000 | Loss: 2.9278 | LR: 1.13e-04 | 1319.9ms/step | 15.2 samples/s +Step 30400/44000 | Loss: 2.9286 | LR: 1.12e-04 | 1319.7ms/step | 15.2 samples/s +Step 30410/44000 | Loss: 2.9380 | LR: 1.12e-04 | 1318.7ms/step | 15.2 samples/s +Step 30420/44000 | Loss: 2.9472 | LR: 1.12e-04 | 1320.8ms/step | 15.1 samples/s +Step 30430/44000 | Loss: 2.9175 | LR: 1.12e-04 | 1322.0ms/step | 15.1 samples/s +Step 30440/44000 | Loss: 2.9524 | LR: 1.12e-04 | 1322.3ms/step | 15.1 samples/s +Step 30450/44000 | Loss: 2.9270 | LR: 1.12e-04 | 1321.1ms/step | 15.1 samples/s +Step 30460/44000 | Loss: 2.9091 | LR: 1.11e-04 | 1320.0ms/step | 15.2 samples/s +Step 30470/44000 | Loss: 2.9424 | LR: 1.11e-04 | 1319.9ms/step | 15.2 samples/s +Step 30480/44000 | Loss: 2.9288 | LR: 1.11e-04 | 1324.1ms/step | 15.1 samples/s +Step 30490/44000 | Loss: 2.9260 | LR: 1.11e-04 | 1321.8ms/step | 15.1 samples/s +Step 30500/44000 | Loss: 2.9230 | LR: 1.11e-04 | 1321.9ms/step | 15.1 samples/s +Validation at step 30500: Loss = 2.9174, Perplexity = 18.49 +Step 30510/44000 | Loss: 2.8991 | LR: 1.11e-04 | 3529.1ms/step | 5.7 samples/s +Step 30520/44000 | Loss: 2.9285 | LR: 1.11e-04 | 1317.6ms/step | 15.2 samples/s +Step 30530/44000 | Loss: 2.9397 | LR: 1.10e-04 | 1314.9ms/step | 15.2 samples/s +Step 30540/44000 | Loss: 2.9442 | LR: 1.10e-04 | 1315.3ms/step | 15.2 samples/s +Step 30550/44000 | Loss: 2.9327 | LR: 1.10e-04 | 1316.3ms/step | 15.2 samples/s +Step 30560/44000 | Loss: 2.9610 | LR: 1.10e-04 | 1317.6ms/step | 15.2 samples/s +Step 30570/44000 | Loss: 2.9333 | LR: 1.10e-04 | 1320.6ms/step | 15.1 samples/s +Step 30580/44000 | Loss: 2.9594 | LR: 1.10e-04 | 1319.5ms/step | 15.2 samples/s +Step 30590/44000 | Loss: 2.9435 | LR: 1.10e-04 | 1317.8ms/step | 15.2 samples/s +Step 30600/44000 | Loss: 2.9050 | LR: 1.09e-04 | 1317.0ms/step | 15.2 samples/s +Step 30610/44000 | Loss: 2.9654 | LR: 1.09e-04 | 1320.9ms/step | 15.1 samples/s +Step 30620/44000 | Loss: 2.9123 | LR: 1.09e-04 | 1317.3ms/step | 15.2 samples/s +Step 30630/44000 | Loss: 2.9032 | LR: 1.09e-04 | 1320.9ms/step | 15.1 samples/s +Step 30640/44000 | Loss: 2.9277 | LR: 1.09e-04 | 1320.0ms/step | 15.2 samples/s +Step 30650/44000 | Loss: 2.9194 | LR: 1.09e-04 | 1319.0ms/step | 15.2 samples/s +Step 30660/44000 | Loss: 2.9279 | LR: 1.08e-04 | 1321.7ms/step | 15.1 samples/s +Step 30670/44000 | Loss: 2.9251 | LR: 1.08e-04 | 1319.7ms/step | 15.2 samples/s +Step 30680/44000 | Loss: 2.9283 | LR: 1.08e-04 | 1320.6ms/step | 15.1 samples/s +Step 30690/44000 | Loss: 2.9396 | LR: 1.08e-04 | 1319.3ms/step | 15.2 samples/s +Step 30700/44000 | Loss: 2.9289 | LR: 1.08e-04 | 1321.2ms/step | 15.1 samples/s +Step 30710/44000 | Loss: 2.9452 | LR: 1.08e-04 | 1318.1ms/step | 15.2 samples/s +Step 30720/44000 | Loss: 2.9472 | LR: 1.08e-04 | 1318.6ms/step | 15.2 samples/s +Step 30730/44000 | Loss: 2.9365 | LR: 1.07e-04 | 1318.2ms/step | 15.2 samples/s +Step 30740/44000 | Loss: 2.9083 | LR: 1.07e-04 | 1322.2ms/step | 15.1 samples/s +Step 30750/44000 | Loss: 2.9380 | LR: 1.07e-04 | 1319.8ms/step | 15.2 samples/s +Validation at step 30750: Loss = 2.9169, Perplexity = 18.48 +Step 30760/44000 | Loss: 2.9344 | LR: 1.07e-04 | 3534.2ms/step | 5.7 samples/s +Step 30770/44000 | Loss: 2.9176 | LR: 1.07e-04 | 1319.7ms/step | 15.2 samples/s +Step 30780/44000 | Loss: 2.9338 | LR: 1.07e-04 | 1323.6ms/step | 15.1 samples/s +Step 30790/44000 | Loss: 2.9318 | LR: 1.07e-04 | 1319.3ms/step | 15.2 samples/s +Step 30800/44000 | Loss: 2.9203 | LR: 1.06e-04 | 1319.2ms/step | 15.2 samples/s +Step 30810/44000 | Loss: 2.9184 | LR: 1.06e-04 | 1320.5ms/step | 15.1 samples/s +Step 30820/44000 | Loss: 2.9427 | LR: 1.06e-04 | 1318.3ms/step | 15.2 samples/s +Step 30830/44000 | Loss: 2.9197 | LR: 1.06e-04 | 1319.3ms/step | 15.2 samples/s +Step 30840/44000 | Loss: 2.9018 | LR: 1.06e-04 | 1321.5ms/step | 15.1 samples/s +Step 30850/44000 | Loss: 2.9341 | LR: 1.06e-04 | 1320.3ms/step | 15.1 samples/s +Step 30860/44000 | Loss: 2.9351 | LR: 1.05e-04 | 1318.7ms/step | 15.2 samples/s +Step 30870/44000 | Loss: 2.9394 | LR: 1.05e-04 | 1319.3ms/step | 15.2 samples/s +Step 30880/44000 | Loss: 2.9282 | LR: 1.05e-04 | 1320.2ms/step | 15.1 samples/s +Step 30890/44000 | Loss: 2.9510 | LR: 1.05e-04 | 1323.9ms/step | 15.1 samples/s +Step 30900/44000 | Loss: 2.9336 | LR: 1.05e-04 | 1321.8ms/step | 15.1 samples/s +Step 30910/44000 | Loss: 2.9084 | LR: 1.05e-04 | 1320.6ms/step | 15.1 samples/s +Step 30920/44000 | Loss: 2.9383 | LR: 1.05e-04 | 1314.5ms/step | 15.2 samples/s +Step 30930/44000 | Loss: 2.8998 | LR: 1.04e-04 | 1318.7ms/step | 15.2 samples/s +Step 30940/44000 | Loss: 2.9166 | LR: 1.04e-04 | 1318.4ms/step | 15.2 samples/s +Step 30950/44000 | Loss: 2.9063 | LR: 1.04e-04 | 1318.9ms/step | 15.2 samples/s +Step 30960/44000 | Loss: 2.9135 | LR: 1.04e-04 | 1316.3ms/step | 15.2 samples/s +Step 30970/44000 | Loss: 2.9338 | LR: 1.04e-04 | 1315.7ms/step | 15.2 samples/s +Step 30980/44000 | Loss: 2.9442 | LR: 1.04e-04 | 1315.0ms/step | 15.2 samples/s +Step 30990/44000 | Loss: 2.9350 | LR: 1.04e-04 | 1316.8ms/step | 15.2 samples/s +Step 31000/44000 | Loss: 2.9236 | LR: 1.03e-04 | 1317.2ms/step | 15.2 samples/s +Validation at step 31000: Loss = 2.9161, Perplexity = 18.47 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-31000 +Step 31010/44000 | Loss: 2.9207 | LR: 1.03e-04 | 3816.9ms/step | 5.2 samples/s +Step 31020/44000 | Loss: 2.9375 | LR: 1.03e-04 | 1318.4ms/step | 15.2 samples/s +Step 31030/44000 | Loss: 2.9093 | LR: 1.03e-04 | 1319.0ms/step | 15.2 samples/s +Step 31040/44000 | Loss: 2.9388 | LR: 1.03e-04 | 1315.1ms/step | 15.2 samples/s +Step 31050/44000 | Loss: 2.8923 | LR: 1.03e-04 | 1321.4ms/step | 15.1 samples/s +Step 31060/44000 | Loss: 2.9199 | LR: 1.03e-04 | 1319.0ms/step | 15.2 samples/s +Step 31070/44000 | Loss: 2.9106 | LR: 1.02e-04 | 1318.5ms/step | 15.2 samples/s +Step 31080/44000 | Loss: 2.9327 | LR: 1.02e-04 | 1319.0ms/step | 15.2 samples/s +Step 31090/44000 | Loss: 2.9283 | LR: 1.02e-04 | 1320.7ms/step | 15.1 samples/s +Step 31100/44000 | Loss: 2.9515 | LR: 1.02e-04 | 1320.6ms/step | 15.1 samples/s +Step 31110/44000 | Loss: 2.9150 | LR: 1.02e-04 | 1320.2ms/step | 15.1 samples/s +Step 31120/44000 | Loss: 2.9412 | LR: 1.02e-04 | 1319.3ms/step | 15.2 samples/s +Step 31130/44000 | Loss: 2.9412 | LR: 1.02e-04 | 1319.0ms/step | 15.2 samples/s +Step 31140/44000 | Loss: 2.9220 | LR: 1.01e-04 | 1318.5ms/step | 15.2 samples/s +Step 31150/44000 | Loss: 2.8982 | LR: 1.01e-04 | 1320.7ms/step | 15.1 samples/s +Step 31160/44000 | Loss: 2.9060 | LR: 1.01e-04 | 1319.3ms/step | 15.2 samples/s +Step 31170/44000 | Loss: 2.9375 | LR: 1.01e-04 | 1314.9ms/step | 15.2 samples/s +Step 31180/44000 | Loss: 2.9333 | LR: 1.01e-04 | 1314.7ms/step | 15.2 samples/s +Step 31190/44000 | Loss: 2.9535 | LR: 1.01e-04 | 1317.9ms/step | 15.2 samples/s +Step 31200/44000 | Loss: 2.9345 | LR: 1.00e-04 | 1320.1ms/step | 15.2 samples/s +Step 31210/44000 | Loss: 2.9107 | LR: 1.00e-04 | 1317.6ms/step | 15.2 samples/s +Step 31220/44000 | Loss: 2.9672 | LR: 1.00e-04 | 1318.4ms/step | 15.2 samples/s +Step 31230/44000 | Loss: 2.9741 | LR: 1.00e-04 | 1318.7ms/step | 15.2 samples/s +Step 31240/44000 | Loss: 2.9116 | LR: 9.99e-05 | 1319.3ms/step | 15.2 samples/s +Step 31250/44000 | Loss: 2.9321 | LR: 9.98e-05 | 1320.0ms/step | 15.2 samples/s +Validation at step 31250: Loss = 2.9156, Perplexity = 18.46 +Step 31260/44000 | Loss: 2.9263 | LR: 9.96e-05 | 3526.1ms/step | 5.7 samples/s +Step 31270/44000 | Loss: 2.9131 | LR: 9.95e-05 | 1315.4ms/step | 15.2 samples/s +Step 31280/44000 | Loss: 2.9216 | LR: 9.93e-05 | 1315.7ms/step | 15.2 samples/s +Step 31290/44000 | Loss: 2.9342 | LR: 9.92e-05 | 1315.8ms/step | 15.2 samples/s +Step 31300/44000 | Loss: 2.9340 | LR: 9.90e-05 | 1316.9ms/step | 15.2 samples/s +Step 31310/44000 | Loss: 2.9266 | LR: 9.89e-05 | 1315.9ms/step | 15.2 samples/s +Step 31320/44000 | Loss: 2.9329 | LR: 9.88e-05 | 1316.8ms/step | 15.2 samples/s +Step 31330/44000 | Loss: 2.9357 | LR: 9.86e-05 | 1320.0ms/step | 15.2 samples/s +Step 31340/44000 | Loss: 2.9212 | LR: 9.85e-05 | 1320.3ms/step | 15.1 samples/s +Step 31350/44000 | Loss: 2.9264 | LR: 9.83e-05 | 1320.1ms/step | 15.2 samples/s +Step 31360/44000 | Loss: 2.9234 | LR: 9.82e-05 | 1320.0ms/step | 15.2 samples/s +Step 31370/44000 | Loss: 2.9206 | LR: 9.80e-05 | 1323.5ms/step | 15.1 samples/s +Step 31380/44000 | Loss: 2.9289 | LR: 9.79e-05 | 1318.4ms/step | 15.2 samples/s +Step 31390/44000 | Loss: 2.9164 | LR: 9.77e-05 | 1320.0ms/step | 15.2 samples/s +Step 31400/44000 | Loss: 2.9242 | LR: 9.76e-05 | 1319.9ms/step | 15.2 samples/s +Step 31410/44000 | Loss: 2.9059 | LR: 9.75e-05 | 1319.1ms/step | 15.2 samples/s +Step 31420/44000 | Loss: 2.9259 | LR: 9.73e-05 | 1318.5ms/step | 15.2 samples/s +Step 31430/44000 | Loss: 2.9086 | LR: 9.72e-05 | 1321.7ms/step | 15.1 samples/s +Step 31440/44000 | Loss: 2.9095 | LR: 9.70e-05 | 1320.2ms/step | 15.1 samples/s +Step 31450/44000 | Loss: 2.9331 | LR: 9.69e-05 | 1320.7ms/step | 15.1 samples/s +Step 31460/44000 | Loss: 2.9142 | LR: 9.67e-05 | 1323.9ms/step | 15.1 samples/s +Step 31470/44000 | Loss: 2.9419 | LR: 9.66e-05 | 1321.4ms/step | 15.1 samples/s +Step 31480/44000 | Loss: 2.9295 | LR: 9.65e-05 | 1323.4ms/step | 15.1 samples/s +Step 31490/44000 | Loss: 2.9400 | LR: 9.63e-05 | 1324.1ms/step | 15.1 samples/s +Step 31500/44000 | Loss: 2.9083 | LR: 9.62e-05 | 1319.7ms/step | 15.2 samples/s +Validation at step 31500: Loss = 2.9149, Perplexity = 18.45 +Step 31510/44000 | Loss: 2.9217 | LR: 9.60e-05 | 3530.6ms/step | 5.7 samples/s +Step 31520/44000 | Loss: 2.9236 | LR: 9.59e-05 | 1322.6ms/step | 15.1 samples/s +Step 31530/44000 | Loss: 2.9290 | LR: 9.57e-05 | 1321.7ms/step | 15.1 samples/s +Step 31540/44000 | Loss: 2.9224 | LR: 9.56e-05 | 1319.3ms/step | 15.2 samples/s +Step 31550/44000 | Loss: 2.9200 | LR: 9.54e-05 | 1320.5ms/step | 15.1 samples/s +Step 31560/44000 | Loss: 2.9422 | LR: 9.53e-05 | 1320.4ms/step | 15.1 samples/s +Step 31570/44000 | Loss: 2.9078 | LR: 9.52e-05 | 1320.0ms/step | 15.2 samples/s +Step 31580/44000 | Loss: 2.9370 | LR: 9.50e-05 | 1321.0ms/step | 15.1 samples/s +Step 31590/44000 | Loss: 2.9346 | LR: 9.49e-05 | 1316.8ms/step | 15.2 samples/s +Step 31600/44000 | Loss: 2.9032 | LR: 9.47e-05 | 1314.9ms/step | 15.2 samples/s +Step 31610/44000 | Loss: 2.9439 | LR: 9.46e-05 | 1320.7ms/step | 15.1 samples/s +Step 31620/44000 | Loss: 2.9191 | LR: 9.45e-05 | 1323.3ms/step | 15.1 samples/s +Step 31630/44000 | Loss: 2.9194 | LR: 9.43e-05 | 1318.0ms/step | 15.2 samples/s +Step 31640/44000 | Loss: 2.9314 | LR: 9.42e-05 | 1316.4ms/step | 15.2 samples/s +Step 31650/44000 | Loss: 2.9281 | LR: 9.40e-05 | 1319.9ms/step | 15.2 samples/s +Step 31660/44000 | Loss: 2.9346 | LR: 9.39e-05 | 1316.1ms/step | 15.2 samples/s +Step 31670/44000 | Loss: 2.9187 | LR: 9.37e-05 | 1318.1ms/step | 15.2 samples/s +Step 31680/44000 | Loss: 2.8920 | LR: 9.36e-05 | 1317.9ms/step | 15.2 samples/s +Step 31690/44000 | Loss: 2.9145 | LR: 9.35e-05 | 1315.6ms/step | 15.2 samples/s +Step 31700/44000 | Loss: 2.9192 | LR: 9.33e-05 | 1317.4ms/step | 15.2 samples/s +Step 31710/44000 | Loss: 2.9236 | LR: 9.32e-05 | 1312.9ms/step | 15.2 samples/s +Step 31720/44000 | Loss: 2.9227 | LR: 9.30e-05 | 1320.6ms/step | 15.1 samples/s +Step 31730/44000 | Loss: 2.9293 | LR: 9.29e-05 | 1316.7ms/step | 15.2 samples/s +Step 31740/44000 | Loss: 2.9119 | LR: 9.28e-05 | 1316.2ms/step | 15.2 samples/s +Step 31750/44000 | Loss: 2.9283 | LR: 9.26e-05 | 1319.4ms/step | 15.2 samples/s +Validation at step 31750: Loss = 2.9142, Perplexity = 18.43 +Step 31760/44000 | Loss: 2.9401 | LR: 9.25e-05 | 3527.0ms/step | 5.7 samples/s +Step 31770/44000 | Loss: 2.9158 | LR: 9.23e-05 | 1320.1ms/step | 15.2 samples/s +Step 31780/44000 | Loss: 2.9146 | LR: 9.22e-05 | 1318.5ms/step | 15.2 samples/s +Step 31790/44000 | Loss: 2.9180 | LR: 9.20e-05 | 1319.3ms/step | 15.2 samples/s +Step 31800/44000 | Loss: 2.9007 | LR: 9.19e-05 | 1319.1ms/step | 15.2 samples/s +Step 31810/44000 | Loss: 2.9156 | LR: 9.18e-05 | 1319.9ms/step | 15.2 samples/s +Step 31820/44000 | Loss: 2.9184 | LR: 9.16e-05 | 1318.7ms/step | 15.2 samples/s +Step 31830/44000 | Loss: 2.9055 | LR: 9.15e-05 | 1319.1ms/step | 15.2 samples/s +Step 31840/44000 | Loss: 2.9098 | LR: 9.13e-05 | 1317.9ms/step | 15.2 samples/s +Step 31850/44000 | Loss: 2.9260 | LR: 9.12e-05 | 1319.7ms/step | 15.2 samples/s +Step 31860/44000 | Loss: 2.9104 | LR: 9.11e-05 | 1318.1ms/step | 15.2 samples/s +Step 31870/44000 | Loss: 2.9086 | LR: 9.09e-05 | 1324.2ms/step | 15.1 samples/s +Step 31880/44000 | Loss: 2.8930 | LR: 9.08e-05 | 1321.2ms/step | 15.1 samples/s +Step 31890/44000 | Loss: 2.9204 | LR: 9.06e-05 | 1313.5ms/step | 15.2 samples/s +Step 31900/44000 | Loss: 2.9222 | LR: 9.05e-05 | 1315.0ms/step | 15.2 samples/s +Step 31910/44000 | Loss: 2.9321 | LR: 9.04e-05 | 1315.3ms/step | 15.2 samples/s +Step 31920/44000 | Loss: 2.9121 | LR: 9.02e-05 | 1321.9ms/step | 15.1 samples/s +Step 31930/44000 | Loss: 2.9191 | LR: 9.01e-05 | 1320.0ms/step | 15.2 samples/s +Step 31940/44000 | Loss: 2.9498 | LR: 8.99e-05 | 1316.4ms/step | 15.2 samples/s +Step 31950/44000 | Loss: 2.9301 | LR: 8.98e-05 | 1316.5ms/step | 15.2 samples/s +Step 31960/44000 | Loss: 2.8799 | LR: 8.97e-05 | 1317.1ms/step | 15.2 samples/s +Step 31970/44000 | Loss: 2.9457 | LR: 8.95e-05 | 1317.2ms/step | 15.2 samples/s +Step 31980/44000 | Loss: 2.9216 | LR: 8.94e-05 | 1314.7ms/step | 15.2 samples/s +Step 31990/44000 | Loss: 2.9476 | LR: 8.92e-05 | 1315.4ms/step | 15.2 samples/s +Step 32000/44000 | Loss: 2.9484 | LR: 8.91e-05 | 1314.1ms/step | 15.2 samples/s +Validation at step 32000: Loss = 2.9134, Perplexity = 18.42 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-32000 +Step 32010/44000 | Loss: 2.9444 | LR: 8.90e-05 | 3843.8ms/step | 5.2 samples/s +Step 32020/44000 | Loss: 2.8924 | LR: 8.88e-05 | 1317.9ms/step | 15.2 samples/s +Step 32030/44000 | Loss: 2.9431 | LR: 8.87e-05 | 1318.4ms/step | 15.2 samples/s +Step 32040/44000 | Loss: 2.9233 | LR: 8.86e-05 | 1319.8ms/step | 15.2 samples/s +Step 32050/44000 | Loss: 2.9311 | LR: 8.84e-05 | 1318.4ms/step | 15.2 samples/s +Step 32060/44000 | Loss: 2.9556 | LR: 8.83e-05 | 1321.4ms/step | 15.1 samples/s +Step 32070/44000 | Loss: 2.9397 | LR: 8.81e-05 | 1320.6ms/step | 15.1 samples/s +Step 32080/44000 | Loss: 2.9205 | LR: 8.80e-05 | 1321.1ms/step | 15.1 samples/s +Step 32090/44000 | Loss: 2.9405 | LR: 8.79e-05 | 1320.4ms/step | 15.1 samples/s +Step 32100/44000 | Loss: 2.9314 | LR: 8.77e-05 | 1319.7ms/step | 15.2 samples/s +Step 32110/44000 | Loss: 2.9297 | LR: 8.76e-05 | 1319.3ms/step | 15.2 samples/s +Step 32120/44000 | Loss: 2.9271 | LR: 8.74e-05 | 1320.9ms/step | 15.1 samples/s +Step 32130/44000 | Loss: 2.9050 | LR: 8.73e-05 | 1319.7ms/step | 15.2 samples/s +Step 32140/44000 | Loss: 2.9422 | LR: 8.72e-05 | 1319.8ms/step | 15.2 samples/s +Step 32150/44000 | Loss: 2.9375 | LR: 8.70e-05 | 1321.0ms/step | 15.1 samples/s +Step 32160/44000 | Loss: 2.9093 | LR: 8.69e-05 | 1315.8ms/step | 15.2 samples/s +Step 32170/44000 | Loss: 2.9224 | LR: 8.68e-05 | 1316.5ms/step | 15.2 samples/s +Step 32180/44000 | Loss: 2.9237 | LR: 8.66e-05 | 1319.3ms/step | 15.2 samples/s +Step 32190/44000 | Loss: 2.9691 | LR: 8.65e-05 | 1319.0ms/step | 15.2 samples/s +Step 32200/44000 | Loss: 2.9491 | LR: 8.63e-05 | 1321.2ms/step | 15.1 samples/s +Step 32210/44000 | Loss: 2.9264 | LR: 8.62e-05 | 1316.2ms/step | 15.2 samples/s +Step 32220/44000 | Loss: 2.9068 | LR: 8.61e-05 | 1317.9ms/step | 15.2 samples/s +Step 32230/44000 | Loss: 2.9453 | LR: 8.59e-05 | 1316.5ms/step | 15.2 samples/s +Step 32240/44000 | Loss: 2.9182 | LR: 8.58e-05 | 1318.4ms/step | 15.2 samples/s +Step 32250/44000 | Loss: 2.9359 | LR: 8.57e-05 | 1319.7ms/step | 15.2 samples/s +Validation at step 32250: Loss = 2.9132, Perplexity = 18.42 +Step 32260/44000 | Loss: 2.9355 | LR: 8.55e-05 | 3524.0ms/step | 5.7 samples/s +Step 32270/44000 | Loss: 2.9315 | LR: 8.54e-05 | 1317.2ms/step | 15.2 samples/s +Step 32280/44000 | Loss: 2.9102 | LR: 8.53e-05 | 1318.7ms/step | 15.2 samples/s +Step 32290/44000 | Loss: 2.9383 | LR: 8.51e-05 | 1320.4ms/step | 15.1 samples/s +Step 32300/44000 | Loss: 2.9309 | LR: 8.50e-05 | 1318.6ms/step | 15.2 samples/s +Step 32310/44000 | Loss: 2.9231 | LR: 8.48e-05 | 1320.3ms/step | 15.1 samples/s +Step 32320/44000 | Loss: 2.8971 | LR: 8.47e-05 | 1314.9ms/step | 15.2 samples/s +Step 32330/44000 | Loss: 2.9473 | LR: 8.46e-05 | 1318.9ms/step | 15.2 samples/s +Step 32340/44000 | Loss: 2.9124 | LR: 8.44e-05 | 1320.2ms/step | 15.1 samples/s +Step 32350/44000 | Loss: 2.9728 | LR: 8.43e-05 | 1319.5ms/step | 15.2 samples/s +Step 32360/44000 | Loss: 2.9260 | LR: 8.42e-05 | 1319.9ms/step | 15.2 samples/s +Step 32370/44000 | Loss: 2.9175 | LR: 8.40e-05 | 1317.3ms/step | 15.2 samples/s +Step 32380/44000 | Loss: 2.9215 | LR: 8.39e-05 | 1317.7ms/step | 15.2 samples/s +Step 32390/44000 | Loss: 2.9212 | LR: 8.38e-05 | 1320.9ms/step | 15.1 samples/s +Step 32400/44000 | Loss: 2.9098 | LR: 8.36e-05 | 1321.4ms/step | 15.1 samples/s +Step 32410/44000 | Loss: 2.9063 | LR: 8.35e-05 | 1319.8ms/step | 15.2 samples/s +Step 32420/44000 | Loss: 2.9243 | LR: 8.33e-05 | 1321.4ms/step | 15.1 samples/s +Step 32430/44000 | Loss: 2.9349 | LR: 8.32e-05 | 1321.9ms/step | 15.1 samples/s +Step 32440/44000 | Loss: 2.9338 | LR: 8.31e-05 | 1321.0ms/step | 15.1 samples/s +Step 32450/44000 | Loss: 2.9086 | LR: 8.29e-05 | 1322.3ms/step | 15.1 samples/s +Step 32460/44000 | Loss: 2.9039 | LR: 8.28e-05 | 1321.1ms/step | 15.1 samples/s +Step 32470/44000 | Loss: 2.9326 | LR: 8.27e-05 | 1318.9ms/step | 15.2 samples/s +Step 32480/44000 | Loss: 2.9151 | LR: 8.25e-05 | 1323.1ms/step | 15.1 samples/s +Step 32490/44000 | Loss: 2.9169 | LR: 8.24e-05 | 1322.7ms/step | 15.1 samples/s +Step 32500/44000 | Loss: 2.9348 | LR: 8.23e-05 | 1319.7ms/step | 15.2 samples/s +Validation at step 32500: Loss = 2.9125, Perplexity = 18.40 +Step 32510/44000 | Loss: 2.9402 | LR: 8.21e-05 | 3532.1ms/step | 5.7 samples/s +Step 32520/44000 | Loss: 2.9159 | LR: 8.20e-05 | 1321.9ms/step | 15.1 samples/s +Step 32530/44000 | Loss: 2.9409 | LR: 8.19e-05 | 1322.4ms/step | 15.1 samples/s +Step 32540/44000 | Loss: 2.9478 | LR: 8.17e-05 | 1323.8ms/step | 15.1 samples/s +Step 32550/44000 | Loss: 2.9510 | LR: 8.16e-05 | 1323.1ms/step | 15.1 samples/s +Step 32560/44000 | Loss: 2.9406 | LR: 8.15e-05 | 1321.1ms/step | 15.1 samples/s +Step 32570/44000 | Loss: 2.9148 | LR: 8.13e-05 | 1319.5ms/step | 15.2 samples/s +Step 32580/44000 | Loss: 2.9163 | LR: 8.12e-05 | 1321.2ms/step | 15.1 samples/s +Step 32590/44000 | Loss: 2.9536 | LR: 8.11e-05 | 1322.2ms/step | 15.1 samples/s +Step 32600/44000 | Loss: 2.9371 | LR: 8.09e-05 | 1322.1ms/step | 15.1 samples/s +Step 32610/44000 | Loss: 2.9511 | LR: 8.08e-05 | 1318.0ms/step | 15.2 samples/s +Step 32620/44000 | Loss: 2.9104 | LR: 8.07e-05 | 1318.3ms/step | 15.2 samples/s +Step 32630/44000 | Loss: 2.9385 | LR: 8.05e-05 | 1321.5ms/step | 15.1 samples/s +Step 32640/44000 | Loss: 2.9431 | LR: 8.04e-05 | 1323.3ms/step | 15.1 samples/s +Step 32650/44000 | Loss: 2.9162 | LR: 8.03e-05 | 1321.8ms/step | 15.1 samples/s +Step 32660/44000 | Loss: 2.8970 | LR: 8.01e-05 | 1317.8ms/step | 15.2 samples/s +Step 32670/44000 | Loss: 2.9217 | LR: 8.00e-05 | 1317.8ms/step | 15.2 samples/s +Step 32680/44000 | Loss: 2.9147 | LR: 7.99e-05 | 1320.1ms/step | 15.2 samples/s +Step 32690/44000 | Loss: 2.8961 | LR: 7.97e-05 | 1319.1ms/step | 15.2 samples/s +Step 32700/44000 | Loss: 2.9093 | LR: 7.96e-05 | 1320.9ms/step | 15.1 samples/s +Step 32710/44000 | Loss: 2.9475 | LR: 7.95e-05 | 1323.5ms/step | 15.1 samples/s +Step 32720/44000 | Loss: 2.9334 | LR: 7.93e-05 | 1322.7ms/step | 15.1 samples/s +Step 32730/44000 | Loss: 2.9068 | LR: 7.92e-05 | 1322.1ms/step | 15.1 samples/s +Step 32740/44000 | Loss: 2.9157 | LR: 7.91e-05 | 1321.6ms/step | 15.1 samples/s +Step 32750/44000 | Loss: 2.9172 | LR: 7.89e-05 | 1320.3ms/step | 15.1 samples/s +Validation at step 32750: Loss = 2.9121, Perplexity = 18.40 +Step 32760/44000 | Loss: 2.9135 | LR: 7.88e-05 | 3523.5ms/step | 5.7 samples/s +Step 32770/44000 | Loss: 2.9090 | LR: 7.87e-05 | 1318.5ms/step | 15.2 samples/s +Step 32780/44000 | Loss: 2.8972 | LR: 7.85e-05 | 1319.9ms/step | 15.2 samples/s +Step 32790/44000 | Loss: 2.9045 | LR: 7.84e-05 | 1320.0ms/step | 15.2 samples/s +Step 32800/44000 | Loss: 2.9231 | LR: 7.83e-05 | 1320.4ms/step | 15.1 samples/s +Step 32810/44000 | Loss: 2.9371 | LR: 7.81e-05 | 1318.4ms/step | 15.2 samples/s +Step 32820/44000 | Loss: 2.9506 | LR: 7.80e-05 | 1317.6ms/step | 15.2 samples/s +Step 32830/44000 | Loss: 2.9186 | LR: 7.79e-05 | 1319.8ms/step | 15.2 samples/s +Step 32840/44000 | Loss: 2.9437 | LR: 7.77e-05 | 1319.9ms/step | 15.2 samples/s +Step 32850/44000 | Loss: 2.9225 | LR: 7.76e-05 | 1323.7ms/step | 15.1 samples/s +Step 32860/44000 | Loss: 2.9018 | LR: 7.75e-05 | 1326.5ms/step | 15.1 samples/s +Step 32870/44000 | Loss: 2.8689 | LR: 7.73e-05 | 1319.7ms/step | 15.2 samples/s +Step 32880/44000 | Loss: 2.9072 | LR: 7.72e-05 | 1317.9ms/step | 15.2 samples/s +Step 32890/44000 | Loss: 2.9354 | LR: 7.71e-05 | 1320.2ms/step | 15.1 samples/s +Step 32900/44000 | Loss: 2.9455 | LR: 7.70e-05 | 1322.3ms/step | 15.1 samples/s +Step 32910/44000 | Loss: 2.9293 | LR: 7.68e-05 | 1320.3ms/step | 15.1 samples/s +Step 32920/44000 | Loss: 2.9381 | LR: 7.67e-05 | 1319.9ms/step | 15.2 samples/s +Step 32930/44000 | Loss: 2.9185 | LR: 7.66e-05 | 1316.3ms/step | 15.2 samples/s +Step 32940/44000 | Loss: 2.9221 | LR: 7.64e-05 | 1317.3ms/step | 15.2 samples/s +Step 32950/44000 | Loss: 2.9261 | LR: 7.63e-05 | 1317.9ms/step | 15.2 samples/s +Step 32960/44000 | Loss: 2.9129 | LR: 7.62e-05 | 1317.4ms/step | 15.2 samples/s +Step 32970/44000 | Loss: 2.9289 | LR: 7.60e-05 | 1319.3ms/step | 15.2 samples/s +Step 32980/44000 | Loss: 2.9272 | LR: 7.59e-05 | 1318.6ms/step | 15.2 samples/s +Step 32990/44000 | Loss: 2.9008 | LR: 7.58e-05 | 1317.6ms/step | 15.2 samples/s +Step 33000/44000 | Loss: 2.9083 | LR: 7.56e-05 | 1318.3ms/step | 15.2 samples/s +Validation at step 33000: Loss = 2.9115, Perplexity = 18.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-33000 +Step 33010/44000 | Loss: 2.9510 | LR: 7.55e-05 | 3834.7ms/step | 5.2 samples/s +Step 33020/44000 | Loss: 2.9475 | LR: 7.54e-05 | 1319.4ms/step | 15.2 samples/s +Step 33030/44000 | Loss: 2.9484 | LR: 7.53e-05 | 1317.1ms/step | 15.2 samples/s +Step 33040/44000 | Loss: 2.9054 | LR: 7.51e-05 | 1317.4ms/step | 15.2 samples/s +Step 33050/44000 | Loss: 2.9131 | LR: 7.50e-05 | 1317.7ms/step | 15.2 samples/s +Step 33060/44000 | Loss: 2.9280 | LR: 7.49e-05 | 1318.0ms/step | 15.2 samples/s +Step 33070/44000 | Loss: 2.9496 | LR: 7.47e-05 | 1317.2ms/step | 15.2 samples/s +Step 33080/44000 | Loss: 2.9117 | LR: 7.46e-05 | 1314.5ms/step | 15.2 samples/s +Step 33090/44000 | Loss: 2.9260 | LR: 7.45e-05 | 1315.0ms/step | 15.2 samples/s +Step 33100/44000 | Loss: 2.8933 | LR: 7.44e-05 | 1317.3ms/step | 15.2 samples/s +Step 33110/44000 | Loss: 2.8783 | LR: 7.42e-05 | 1320.2ms/step | 15.1 samples/s +Step 33120/44000 | Loss: 2.9345 | LR: 7.41e-05 | 1318.8ms/step | 15.2 samples/s +Step 33130/44000 | Loss: 2.9329 | LR: 7.40e-05 | 1318.4ms/step | 15.2 samples/s +Step 33140/44000 | Loss: 2.8657 | LR: 7.38e-05 | 1315.2ms/step | 15.2 samples/s +Step 33150/44000 | Loss: 2.9453 | LR: 7.37e-05 | 1317.9ms/step | 15.2 samples/s +Step 33160/44000 | Loss: 2.9036 | LR: 7.36e-05 | 1320.4ms/step | 15.1 samples/s +Step 33170/44000 | Loss: 2.9347 | LR: 7.34e-05 | 1320.3ms/step | 15.1 samples/s +Step 33180/44000 | Loss: 2.9087 | LR: 7.33e-05 | 1317.6ms/step | 15.2 samples/s +Step 33190/44000 | Loss: 2.9327 | LR: 7.32e-05 | 1317.7ms/step | 15.2 samples/s +Step 33200/44000 | Loss: 2.9423 | LR: 7.31e-05 | 1316.8ms/step | 15.2 samples/s +Step 33210/44000 | Loss: 2.8959 | LR: 7.29e-05 | 1318.7ms/step | 15.2 samples/s +Step 33220/44000 | Loss: 2.8828 | LR: 7.28e-05 | 1322.4ms/step | 15.1 samples/s +Step 33230/44000 | Loss: 2.9345 | LR: 7.27e-05 | 1322.0ms/step | 15.1 samples/s +Step 33240/44000 | Loss: 2.9086 | LR: 7.26e-05 | 1322.0ms/step | 15.1 samples/s +Step 33250/44000 | Loss: 2.9213 | LR: 7.24e-05 | 1320.9ms/step | 15.1 samples/s +Validation at step 33250: Loss = 2.9113, Perplexity = 18.38 +Step 33260/44000 | Loss: 2.8766 | LR: 7.23e-05 | 3529.7ms/step | 5.7 samples/s +Step 33270/44000 | Loss: 2.8876 | LR: 7.22e-05 | 1319.2ms/step | 15.2 samples/s +Step 33280/44000 | Loss: 2.9201 | LR: 7.20e-05 | 1322.7ms/step | 15.1 samples/s +Step 33290/44000 | Loss: 2.9145 | LR: 7.19e-05 | 1321.3ms/step | 15.1 samples/s +Step 33300/44000 | Loss: 2.9220 | LR: 7.18e-05 | 1319.7ms/step | 15.2 samples/s +Step 33310/44000 | Loss: 2.9026 | LR: 7.17e-05 | 1322.1ms/step | 15.1 samples/s +Step 33320/44000 | Loss: 2.9179 | LR: 7.15e-05 | 1321.5ms/step | 15.1 samples/s +Step 33330/44000 | Loss: 2.9296 | LR: 7.14e-05 | 1321.9ms/step | 15.1 samples/s +Step 33340/44000 | Loss: 2.9155 | LR: 7.13e-05 | 1318.9ms/step | 15.2 samples/s +Step 33350/44000 | Loss: 2.9073 | LR: 7.11e-05 | 1321.0ms/step | 15.1 samples/s +Step 33360/44000 | Loss: 2.9543 | LR: 7.10e-05 | 1321.0ms/step | 15.1 samples/s +Step 33370/44000 | Loss: 2.9263 | LR: 7.09e-05 | 1321.9ms/step | 15.1 samples/s +Step 33380/44000 | Loss: 2.9162 | LR: 7.08e-05 | 1319.6ms/step | 15.2 samples/s +Step 33390/44000 | Loss: 2.9348 | LR: 7.06e-05 | 1319.4ms/step | 15.2 samples/s +Step 33400/44000 | Loss: 2.8976 | LR: 7.05e-05 | 1319.6ms/step | 15.2 samples/s +Step 33410/44000 | Loss: 2.9137 | LR: 7.04e-05 | 1319.4ms/step | 15.2 samples/s +Step 33420/44000 | Loss: 2.9328 | LR: 7.03e-05 | 1321.0ms/step | 15.1 samples/s +Step 33430/44000 | Loss: 2.9517 | LR: 7.01e-05 | 1318.0ms/step | 15.2 samples/s +Step 33440/44000 | Loss: 2.9502 | LR: 7.00e-05 | 1315.4ms/step | 15.2 samples/s +Step 33450/44000 | Loss: 2.9220 | LR: 6.99e-05 | 1321.4ms/step | 15.1 samples/s +Step 33460/44000 | Loss: 2.8996 | LR: 6.98e-05 | 1317.9ms/step | 15.2 samples/s +Step 33470/44000 | Loss: 2.9043 | LR: 6.96e-05 | 1313.3ms/step | 15.2 samples/s +Step 33480/44000 | Loss: 2.8891 | LR: 6.95e-05 | 1315.3ms/step | 15.2 samples/s +Step 33490/44000 | Loss: 2.9169 | LR: 6.94e-05 | 1313.1ms/step | 15.2 samples/s +Step 33500/44000 | Loss: 2.9368 | LR: 6.93e-05 | 1314.1ms/step | 15.2 samples/s +Validation at step 33500: Loss = 2.9105, Perplexity = 18.37 +Step 33510/44000 | Loss: 2.9092 | LR: 6.91e-05 | 3515.7ms/step | 5.7 samples/s +Step 33520/44000 | Loss: 2.9199 | LR: 6.90e-05 | 1313.6ms/step | 15.2 samples/s +Step 33530/44000 | Loss: 2.9185 | LR: 6.89e-05 | 1316.5ms/step | 15.2 samples/s +Step 33540/44000 | Loss: 2.8985 | LR: 6.88e-05 | 1315.3ms/step | 15.2 samples/s +Step 33550/44000 | Loss: 2.9279 | LR: 6.86e-05 | 1316.7ms/step | 15.2 samples/s +Step 33560/44000 | Loss: 2.9301 | LR: 6.85e-05 | 1320.3ms/step | 15.1 samples/s +Step 33570/44000 | Loss: 2.9192 | LR: 6.84e-05 | 1321.6ms/step | 15.1 samples/s +Step 33580/44000 | Loss: 2.9315 | LR: 6.83e-05 | 1319.9ms/step | 15.2 samples/s +Step 33590/44000 | Loss: 2.8990 | LR: 6.81e-05 | 1318.6ms/step | 15.2 samples/s +Step 33600/44000 | Loss: 2.9327 | LR: 6.80e-05 | 1319.9ms/step | 15.2 samples/s +Step 33610/44000 | Loss: 2.9178 | LR: 6.79e-05 | 1316.2ms/step | 15.2 samples/s +Step 33620/44000 | Loss: 2.9148 | LR: 6.78e-05 | 1313.5ms/step | 15.2 samples/s +Step 33630/44000 | Loss: 2.9268 | LR: 6.76e-05 | 1314.7ms/step | 15.2 samples/s +Step 33640/44000 | Loss: 2.9147 | LR: 6.75e-05 | 1316.2ms/step | 15.2 samples/s +Step 33650/44000 | Loss: 2.9092 | LR: 6.74e-05 | 1316.5ms/step | 15.2 samples/s +Step 33660/44000 | Loss: 2.9410 | LR: 6.73e-05 | 1314.8ms/step | 15.2 samples/s +Step 33670/44000 | Loss: 2.9201 | LR: 6.71e-05 | 1312.4ms/step | 15.2 samples/s +Step 33680/44000 | Loss: 2.9179 | LR: 6.70e-05 | 1312.4ms/step | 15.2 samples/s +Step 33690/44000 | Loss: 2.9226 | LR: 6.69e-05 | 1312.7ms/step | 15.2 samples/s +Step 33700/44000 | Loss: 2.9257 | LR: 6.68e-05 | 1316.3ms/step | 15.2 samples/s +Step 33710/44000 | Loss: 2.9316 | LR: 6.66e-05 | 1322.1ms/step | 15.1 samples/s +Step 33720/44000 | Loss: 2.9228 | LR: 6.65e-05 | 1315.8ms/step | 15.2 samples/s +Step 33730/44000 | Loss: 2.9321 | LR: 6.64e-05 | 1314.3ms/step | 15.2 samples/s +Step 33740/44000 | Loss: 2.9353 | LR: 6.63e-05 | 1314.0ms/step | 15.2 samples/s +Step 33750/44000 | Loss: 2.9215 | LR: 6.61e-05 | 1314.4ms/step | 15.2 samples/s +Validation at step 33750: Loss = 2.9103, Perplexity = 18.36 +Step 33760/44000 | Loss: 2.9190 | LR: 6.60e-05 | 3520.9ms/step | 5.7 samples/s +Step 33770/44000 | Loss: 2.8990 | LR: 6.59e-05 | 1321.9ms/step | 15.1 samples/s +Step 33780/44000 | Loss: 2.8991 | LR: 6.58e-05 | 1321.3ms/step | 15.1 samples/s +Step 33790/44000 | Loss: 2.9298 | LR: 6.57e-05 | 1319.2ms/step | 15.2 samples/s +Step 33800/44000 | Loss: 2.9268 | LR: 6.55e-05 | 1322.1ms/step | 15.1 samples/s +Step 33810/44000 | Loss: 2.9253 | LR: 6.54e-05 | 1320.4ms/step | 15.1 samples/s +Step 33820/44000 | Loss: 2.9054 | LR: 6.53e-05 | 1320.0ms/step | 15.2 samples/s +Step 33830/44000 | Loss: 2.9065 | LR: 6.52e-05 | 1315.1ms/step | 15.2 samples/s +Step 33840/44000 | Loss: 2.8874 | LR: 6.50e-05 | 1318.6ms/step | 15.2 samples/s +Step 33850/44000 | Loss: 2.9031 | LR: 6.49e-05 | 1316.3ms/step | 15.2 samples/s +Step 33860/44000 | Loss: 2.9001 | LR: 6.48e-05 | 1313.3ms/step | 15.2 samples/s +Step 33870/44000 | Loss: 2.9219 | LR: 6.47e-05 | 1317.4ms/step | 15.2 samples/s +Step 33880/44000 | Loss: 2.9113 | LR: 6.46e-05 | 1319.5ms/step | 15.2 samples/s +Step 33890/44000 | Loss: 2.9356 | LR: 6.44e-05 | 1320.5ms/step | 15.1 samples/s +Step 33900/44000 | Loss: 2.9094 | LR: 6.43e-05 | 1318.3ms/step | 15.2 samples/s +Step 33910/44000 | Loss: 2.9316 | LR: 6.42e-05 | 1319.5ms/step | 15.2 samples/s +Step 33920/44000 | Loss: 2.8832 | LR: 6.41e-05 | 1319.3ms/step | 15.2 samples/s +Step 33930/44000 | Loss: 2.9068 | LR: 6.40e-05 | 1319.0ms/step | 15.2 samples/s +Step 33940/44000 | Loss: 2.9432 | LR: 6.38e-05 | 1311.0ms/step | 15.3 samples/s +Step 33950/44000 | Loss: 2.9313 | LR: 6.37e-05 | 1314.1ms/step | 15.2 samples/s +Step 33960/44000 | Loss: 2.9005 | LR: 6.36e-05 | 1313.8ms/step | 15.2 samples/s +Step 33970/44000 | Loss: 2.9206 | LR: 6.35e-05 | 1313.0ms/step | 15.2 samples/s +Step 33980/44000 | Loss: 2.9576 | LR: 6.33e-05 | 1315.7ms/step | 15.2 samples/s +Step 33990/44000 | Loss: 2.9524 | LR: 6.32e-05 | 1315.3ms/step | 15.2 samples/s +Step 34000/44000 | Loss: 2.9152 | LR: 6.31e-05 | 1315.4ms/step | 15.2 samples/s +Validation at step 34000: Loss = 2.9098, Perplexity = 18.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-34000 +Step 34010/44000 | Loss: 2.9344 | LR: 6.30e-05 | 3829.4ms/step | 5.2 samples/s +Step 34020/44000 | Loss: 2.8812 | LR: 6.29e-05 | 1315.4ms/step | 15.2 samples/s +Step 34030/44000 | Loss: 2.9235 | LR: 6.27e-05 | 1314.6ms/step | 15.2 samples/s +Step 34040/44000 | Loss: 2.9553 | LR: 6.26e-05 | 1316.3ms/step | 15.2 samples/s +Step 34050/44000 | Loss: 2.9286 | LR: 6.25e-05 | 1317.5ms/step | 15.2 samples/s +Step 34060/44000 | Loss: 2.9426 | LR: 6.24e-05 | 1317.2ms/step | 15.2 samples/s +Step 34070/44000 | Loss: 2.9151 | LR: 6.23e-05 | 1319.0ms/step | 15.2 samples/s +Step 34080/44000 | Loss: 2.9019 | LR: 6.21e-05 | 1320.4ms/step | 15.1 samples/s +Step 34090/44000 | Loss: 2.9700 | LR: 6.20e-05 | 1316.9ms/step | 15.2 samples/s +Step 34100/44000 | Loss: 2.9224 | LR: 6.19e-05 | 1320.7ms/step | 15.1 samples/s +Step 34110/44000 | Loss: 2.8920 | LR: 6.18e-05 | 1320.7ms/step | 15.1 samples/s +Step 34120/44000 | Loss: 2.9062 | LR: 6.17e-05 | 1318.8ms/step | 15.2 samples/s +Step 34130/44000 | Loss: 2.8906 | LR: 6.15e-05 | 1310.3ms/step | 15.3 samples/s +Step 34140/44000 | Loss: 2.9122 | LR: 6.14e-05 | 1317.0ms/step | 15.2 samples/s +Step 34150/44000 | Loss: 2.9382 | LR: 6.13e-05 | 1314.4ms/step | 15.2 samples/s +Step 34160/44000 | Loss: 2.9398 | LR: 6.12e-05 | 1315.4ms/step | 15.2 samples/s +Step 34170/44000 | Loss: 2.9328 | LR: 6.11e-05 | 1318.2ms/step | 15.2 samples/s +Step 34180/44000 | Loss: 2.9004 | LR: 6.09e-05 | 1317.2ms/step | 15.2 samples/s +Step 34190/44000 | Loss: 2.9282 | LR: 6.08e-05 | 1319.6ms/step | 15.2 samples/s +Step 34200/44000 | Loss: 2.9263 | LR: 6.07e-05 | 1318.7ms/step | 15.2 samples/s +Step 34210/44000 | Loss: 2.9322 | LR: 6.06e-05 | 1316.8ms/step | 15.2 samples/s +Step 34220/44000 | Loss: 2.9200 | LR: 6.05e-05 | 1315.1ms/step | 15.2 samples/s +Step 34230/44000 | Loss: 2.9214 | LR: 6.04e-05 | 1316.9ms/step | 15.2 samples/s +Step 34240/44000 | Loss: 2.9254 | LR: 6.02e-05 | 1315.8ms/step | 15.2 samples/s +Step 34250/44000 | Loss: 2.9114 | LR: 6.01e-05 | 1318.3ms/step | 15.2 samples/s +Validation at step 34250: Loss = 2.9095, Perplexity = 18.35 +Step 34260/44000 | Loss: 2.9421 | LR: 6.00e-05 | 3520.8ms/step | 5.7 samples/s +Step 34270/44000 | Loss: 2.9167 | LR: 5.99e-05 | 1315.3ms/step | 15.2 samples/s +Step 34280/44000 | Loss: 2.9431 | LR: 5.98e-05 | 1316.5ms/step | 15.2 samples/s +Step 34290/44000 | Loss: 2.9188 | LR: 5.96e-05 | 1315.2ms/step | 15.2 samples/s +Step 34300/44000 | Loss: 2.9472 | LR: 5.95e-05 | 1315.5ms/step | 15.2 samples/s +Step 34310/44000 | Loss: 2.9225 | LR: 5.94e-05 | 1318.3ms/step | 15.2 samples/s +Step 34320/44000 | Loss: 2.9200 | LR: 5.93e-05 | 1315.9ms/step | 15.2 samples/s +Step 34330/44000 | Loss: 2.9201 | LR: 5.92e-05 | 1316.6ms/step | 15.2 samples/s +Step 34340/44000 | Loss: 2.9281 | LR: 5.91e-05 | 1316.9ms/step | 15.2 samples/s +Step 34350/44000 | Loss: 2.9231 | LR: 5.89e-05 | 1323.8ms/step | 15.1 samples/s +Step 34360/44000 | Loss: 2.9445 | LR: 5.88e-05 | 1319.6ms/step | 15.2 samples/s +Step 34370/44000 | Loss: 2.9231 | LR: 5.87e-05 | 1319.1ms/step | 15.2 samples/s +Step 34380/44000 | Loss: 2.9316 | LR: 5.86e-05 | 1319.0ms/step | 15.2 samples/s +Step 34390/44000 | Loss: 2.9271 | LR: 5.85e-05 | 1320.4ms/step | 15.1 samples/s +Step 34400/44000 | Loss: 2.8783 | LR: 5.84e-05 | 1319.4ms/step | 15.2 samples/s +Step 34410/44000 | Loss: 2.9102 | LR: 5.82e-05 | 1320.2ms/step | 15.1 samples/s +Step 34420/44000 | Loss: 2.9108 | LR: 5.81e-05 | 1313.5ms/step | 15.2 samples/s +Step 34430/44000 | Loss: 2.9255 | LR: 5.80e-05 | 1316.1ms/step | 15.2 samples/s +Step 34440/44000 | Loss: 2.9391 | LR: 5.79e-05 | 1314.4ms/step | 15.2 samples/s +Step 34450/44000 | Loss: 2.8952 | LR: 5.78e-05 | 1313.1ms/step | 15.2 samples/s +Step 34460/44000 | Loss: 2.9197 | LR: 5.77e-05 | 1311.4ms/step | 15.3 samples/s +Step 34470/44000 | Loss: 2.9453 | LR: 5.75e-05 | 1312.6ms/step | 15.2 samples/s +Step 34480/44000 | Loss: 2.9377 | LR: 5.74e-05 | 1312.1ms/step | 15.2 samples/s +Step 34490/44000 | Loss: 2.9246 | LR: 5.73e-05 | 1313.8ms/step | 15.2 samples/s +Step 34500/44000 | Loss: 2.9247 | LR: 5.72e-05 | 1311.8ms/step | 15.2 samples/s +Validation at step 34500: Loss = 2.9092, Perplexity = 18.34 +Step 34510/44000 | Loss: 2.9559 | LR: 5.71e-05 | 3516.4ms/step | 5.7 samples/s +Step 34520/44000 | Loss: 2.8967 | LR: 5.70e-05 | 1315.0ms/step | 15.2 samples/s +Step 34530/44000 | Loss: 2.9089 | LR: 5.69e-05 | 1309.6ms/step | 15.3 samples/s +Step 34540/44000 | Loss: 2.9398 | LR: 5.67e-05 | 1311.9ms/step | 15.2 samples/s +Step 34550/44000 | Loss: 2.8981 | LR: 5.66e-05 | 1311.4ms/step | 15.3 samples/s +Step 34560/44000 | Loss: 2.8871 | LR: 5.65e-05 | 1313.6ms/step | 15.2 samples/s +Step 34570/44000 | Loss: 2.9132 | LR: 5.64e-05 | 1310.9ms/step | 15.3 samples/s +Step 34580/44000 | Loss: 2.9239 | LR: 5.63e-05 | 1311.5ms/step | 15.2 samples/s +Step 34590/44000 | Loss: 2.9094 | LR: 5.62e-05 | 1309.9ms/step | 15.3 samples/s +Step 34600/44000 | Loss: 2.9054 | LR: 5.60e-05 | 1311.1ms/step | 15.3 samples/s +Step 34610/44000 | Loss: 2.9050 | LR: 5.59e-05 | 1314.5ms/step | 15.2 samples/s +Step 34620/44000 | Loss: 2.9263 | LR: 5.58e-05 | 1313.8ms/step | 15.2 samples/s +Step 34630/44000 | Loss: 2.9385 | LR: 5.57e-05 | 1312.7ms/step | 15.2 samples/s +Step 34640/44000 | Loss: 2.9257 | LR: 5.56e-05 | 1315.0ms/step | 15.2 samples/s +Step 34650/44000 | Loss: 2.9365 | LR: 5.55e-05 | 1312.2ms/step | 15.2 samples/s +Step 34660/44000 | Loss: 2.9101 | LR: 5.54e-05 | 1311.9ms/step | 15.2 samples/s +Step 34670/44000 | Loss: 2.9264 | LR: 5.52e-05 | 1313.0ms/step | 15.2 samples/s +Step 34680/44000 | Loss: 2.9654 | LR: 5.51e-05 | 1314.3ms/step | 15.2 samples/s +Step 34690/44000 | Loss: 2.9209 | LR: 5.50e-05 | 1313.8ms/step | 15.2 samples/s +Step 34700/44000 | Loss: 2.9097 | LR: 5.49e-05 | 1314.3ms/step | 15.2 samples/s +Step 34710/44000 | Loss: 2.9177 | LR: 5.48e-05 | 1312.7ms/step | 15.2 samples/s +Step 34720/44000 | Loss: 2.9239 | LR: 5.47e-05 | 1316.5ms/step | 15.2 samples/s +Step 34730/44000 | Loss: 2.9292 | LR: 5.46e-05 | 1318.8ms/step | 15.2 samples/s +Step 34740/44000 | Loss: 2.8900 | LR: 5.45e-05 | 1315.6ms/step | 15.2 samples/s +Step 34750/44000 | Loss: 2.9105 | LR: 5.43e-05 | 1312.9ms/step | 15.2 samples/s +Validation at step 34750: Loss = 2.9090, Perplexity = 18.34 +Step 34760/44000 | Loss: 2.9114 | LR: 5.42e-05 | 3518.7ms/step | 5.7 samples/s +Step 34770/44000 | Loss: 2.9186 | LR: 5.41e-05 | 1314.8ms/step | 15.2 samples/s +Step 34780/44000 | Loss: 2.8906 | LR: 5.40e-05 | 1315.9ms/step | 15.2 samples/s +Step 34790/44000 | Loss: 2.8986 | LR: 5.39e-05 | 1317.9ms/step | 15.2 samples/s +Step 34800/44000 | Loss: 2.8926 | LR: 5.38e-05 | 1312.6ms/step | 15.2 samples/s +Step 34810/44000 | Loss: 2.9242 | LR: 5.37e-05 | 1316.2ms/step | 15.2 samples/s +Step 34820/44000 | Loss: 2.9551 | LR: 5.36e-05 | 1315.7ms/step | 15.2 samples/s +Step 34830/44000 | Loss: 2.9179 | LR: 5.34e-05 | 1310.7ms/step | 15.3 samples/s +Step 34840/44000 | Loss: 2.8956 | LR: 5.33e-05 | 1312.2ms/step | 15.2 samples/s +Step 34850/44000 | Loss: 2.8947 | LR: 5.32e-05 | 1313.1ms/step | 15.2 samples/s +Step 34860/44000 | Loss: 2.9270 | LR: 5.31e-05 | 1313.2ms/step | 15.2 samples/s +Step 34870/44000 | Loss: 2.9205 | LR: 5.30e-05 | 1314.6ms/step | 15.2 samples/s +Step 34880/44000 | Loss: 2.9142 | LR: 5.29e-05 | 1316.7ms/step | 15.2 samples/s +Step 34890/44000 | Loss: 2.9264 | LR: 5.28e-05 | 1320.3ms/step | 15.1 samples/s +Step 34900/44000 | Loss: 2.9337 | LR: 5.27e-05 | 1316.5ms/step | 15.2 samples/s +Step 34910/44000 | Loss: 2.8951 | LR: 5.25e-05 | 1317.1ms/step | 15.2 samples/s +Step 34920/44000 | Loss: 2.9727 | LR: 5.24e-05 | 1313.2ms/step | 15.2 samples/s +Step 34930/44000 | Loss: 2.8896 | LR: 5.23e-05 | 1312.9ms/step | 15.2 samples/s +Step 34940/44000 | Loss: 2.9155 | LR: 5.22e-05 | 1313.8ms/step | 15.2 samples/s +Step 34950/44000 | Loss: 2.9034 | LR: 5.21e-05 | 1314.4ms/step | 15.2 samples/s +Step 34960/44000 | Loss: 2.9188 | LR: 5.20e-05 | 1315.9ms/step | 15.2 samples/s +Step 34970/44000 | Loss: 2.8861 | LR: 5.19e-05 | 1314.1ms/step | 15.2 samples/s +Step 34980/44000 | Loss: 2.9068 | LR: 5.18e-05 | 1315.0ms/step | 15.2 samples/s +Step 34990/44000 | Loss: 2.9475 | LR: 5.17e-05 | 1315.7ms/step | 15.2 samples/s +Step 35000/44000 | Loss: 2.9379 | LR: 5.15e-05 | 1313.6ms/step | 15.2 samples/s +Validation at step 35000: Loss = 2.9088, Perplexity = 18.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-35000 +Step 35010/44000 | Loss: 2.9194 | LR: 5.14e-05 | 3849.2ms/step | 5.2 samples/s +Step 35020/44000 | Loss: 2.9294 | LR: 5.13e-05 | 1318.3ms/step | 15.2 samples/s +Step 35030/44000 | Loss: 2.9142 | LR: 5.12e-05 | 1318.7ms/step | 15.2 samples/s +Step 35040/44000 | Loss: 2.9186 | LR: 5.11e-05 | 1322.7ms/step | 15.1 samples/s +Step 35050/44000 | Loss: 2.9147 | LR: 5.10e-05 | 1318.2ms/step | 15.2 samples/s +Step 35060/44000 | Loss: 2.9230 | LR: 5.09e-05 | 1316.9ms/step | 15.2 samples/s +Step 35070/44000 | Loss: 2.9198 | LR: 5.08e-05 | 1317.6ms/step | 15.2 samples/s +Step 35080/44000 | Loss: 2.9109 | LR: 5.07e-05 | 1316.1ms/step | 15.2 samples/s +Step 35090/44000 | Loss: 2.9070 | LR: 5.06e-05 | 1317.5ms/step | 15.2 samples/s +Step 35100/44000 | Loss: 2.8959 | LR: 5.04e-05 | 1317.6ms/step | 15.2 samples/s +Step 35110/44000 | Loss: 2.9280 | LR: 5.03e-05 | 1314.9ms/step | 15.2 samples/s +Step 35120/44000 | Loss: 2.9183 | LR: 5.02e-05 | 1312.6ms/step | 15.2 samples/s +Step 35130/44000 | Loss: 2.9157 | LR: 5.01e-05 | 1319.1ms/step | 15.2 samples/s +Step 35140/44000 | Loss: 2.9130 | LR: 5.00e-05 | 1313.7ms/step | 15.2 samples/s +Step 35150/44000 | Loss: 2.9235 | LR: 4.99e-05 | 1315.9ms/step | 15.2 samples/s +Step 35160/44000 | Loss: 2.9283 | LR: 4.98e-05 | 1313.5ms/step | 15.2 samples/s +Step 35170/44000 | Loss: 2.9151 | LR: 4.97e-05 | 1313.1ms/step | 15.2 samples/s +Step 35180/44000 | Loss: 2.9122 | LR: 4.96e-05 | 1311.9ms/step | 15.2 samples/s +Step 35190/44000 | Loss: 2.9234 | LR: 4.95e-05 | 1313.4ms/step | 15.2 samples/s +Step 35200/44000 | Loss: 2.8973 | LR: 4.94e-05 | 1313.0ms/step | 15.2 samples/s +Step 35210/44000 | Loss: 2.9004 | LR: 4.93e-05 | 1319.7ms/step | 15.2 samples/s +Step 35220/44000 | Loss: 2.8889 | LR: 4.91e-05 | 1316.4ms/step | 15.2 samples/s +Step 35230/44000 | Loss: 2.9025 | LR: 4.90e-05 | 1316.3ms/step | 15.2 samples/s +Step 35240/44000 | Loss: 2.9263 | LR: 4.89e-05 | 1314.0ms/step | 15.2 samples/s +Step 35250/44000 | Loss: 2.8977 | LR: 4.88e-05 | 1311.9ms/step | 15.2 samples/s +Validation at step 35250: Loss = 2.9084, Perplexity = 18.33 +Step 35260/44000 | Loss: 2.9012 | LR: 4.87e-05 | 3515.8ms/step | 5.7 samples/s +Step 35270/44000 | Loss: 2.9357 | LR: 4.86e-05 | 1313.9ms/step | 15.2 samples/s +Step 35280/44000 | Loss: 2.9406 | LR: 4.85e-05 | 1317.5ms/step | 15.2 samples/s +Step 35290/44000 | Loss: 2.9158 | LR: 4.84e-05 | 1317.2ms/step | 15.2 samples/s +Step 35300/44000 | Loss: 2.9139 | LR: 4.83e-05 | 1319.4ms/step | 15.2 samples/s +Step 35310/44000 | Loss: 2.9121 | LR: 4.82e-05 | 1318.1ms/step | 15.2 samples/s +Step 35320/44000 | Loss: 2.8855 | LR: 4.81e-05 | 1318.6ms/step | 15.2 samples/s +Step 35330/44000 | Loss: 2.9050 | LR: 4.80e-05 | 1313.8ms/step | 15.2 samples/s +Step 35340/44000 | Loss: 2.8956 | LR: 4.79e-05 | 1310.8ms/step | 15.3 samples/s +Step 35350/44000 | Loss: 2.9322 | LR: 4.77e-05 | 1311.8ms/step | 15.2 samples/s +Step 35360/44000 | Loss: 2.9113 | LR: 4.76e-05 | 1315.0ms/step | 15.2 samples/s +Step 35370/44000 | Loss: 2.8941 | LR: 4.75e-05 | 1318.9ms/step | 15.2 samples/s +Step 35380/44000 | Loss: 2.8946 | LR: 4.74e-05 | 1319.6ms/step | 15.2 samples/s +Step 35390/44000 | Loss: 2.9296 | LR: 4.73e-05 | 1312.1ms/step | 15.2 samples/s +Step 35400/44000 | Loss: 2.9019 | LR: 4.72e-05 | 1312.1ms/step | 15.2 samples/s +Step 35410/44000 | Loss: 2.9281 | LR: 4.71e-05 | 1311.9ms/step | 15.2 samples/s +Step 35420/44000 | Loss: 2.9467 | LR: 4.70e-05 | 1312.8ms/step | 15.2 samples/s +Step 35430/44000 | Loss: 2.9309 | LR: 4.69e-05 | 1310.8ms/step | 15.3 samples/s +Step 35440/44000 | Loss: 2.9303 | LR: 4.68e-05 | 1311.0ms/step | 15.3 samples/s +Step 35450/44000 | Loss: 2.9270 | LR: 4.67e-05 | 1312.2ms/step | 15.2 samples/s +Step 35460/44000 | Loss: 2.9368 | LR: 4.66e-05 | 1314.1ms/step | 15.2 samples/s +Step 35470/44000 | Loss: 2.9171 | LR: 4.65e-05 | 1317.5ms/step | 15.2 samples/s +Step 35480/44000 | Loss: 2.9323 | LR: 4.64e-05 | 1320.0ms/step | 15.2 samples/s +Step 35490/44000 | Loss: 2.9167 | LR: 4.63e-05 | 1316.5ms/step | 15.2 samples/s +Step 35500/44000 | Loss: 2.9357 | LR: 4.62e-05 | 1310.9ms/step | 15.3 samples/s +Validation at step 35500: Loss = 2.9082, Perplexity = 18.32 +Step 35510/44000 | Loss: 2.9238 | LR: 4.61e-05 | 3517.6ms/step | 5.7 samples/s +Step 35520/44000 | Loss: 2.9175 | LR: 4.59e-05 | 1319.0ms/step | 15.2 samples/s +Step 35530/44000 | Loss: 2.9306 | LR: 4.58e-05 | 1317.2ms/step | 15.2 samples/s +Step 35540/44000 | Loss: 2.9434 | LR: 4.57e-05 | 1316.5ms/step | 15.2 samples/s +Step 35550/44000 | Loss: 2.9297 | LR: 4.56e-05 | 1316.0ms/step | 15.2 samples/s +Step 35560/44000 | Loss: 2.9252 | LR: 4.55e-05 | 1316.1ms/step | 15.2 samples/s +Step 35570/44000 | Loss: 2.9576 | LR: 4.54e-05 | 1318.4ms/step | 15.2 samples/s +Step 35580/44000 | Loss: 2.9019 | LR: 4.53e-05 | 1315.5ms/step | 15.2 samples/s +Step 35590/44000 | Loss: 2.9390 | LR: 4.52e-05 | 1314.9ms/step | 15.2 samples/s +Step 35600/44000 | Loss: 2.9457 | LR: 4.51e-05 | 1315.4ms/step | 15.2 samples/s +Step 35610/44000 | Loss: 2.8961 | LR: 4.50e-05 | 1312.3ms/step | 15.2 samples/s +Step 35620/44000 | Loss: 2.9411 | LR: 4.49e-05 | 1314.9ms/step | 15.2 samples/s +Step 35630/44000 | Loss: 2.9149 | LR: 4.48e-05 | 1313.7ms/step | 15.2 samples/s +Step 35640/44000 | Loss: 2.8702 | LR: 4.47e-05 | 1314.4ms/step | 15.2 samples/s +Step 35650/44000 | Loss: 2.9191 | LR: 4.46e-05 | 1319.1ms/step | 15.2 samples/s +Step 35660/44000 | Loss: 2.9464 | LR: 4.45e-05 | 1315.3ms/step | 15.2 samples/s +Step 35670/44000 | Loss: 2.9242 | LR: 4.44e-05 | 1319.4ms/step | 15.2 samples/s +Step 35680/44000 | Loss: 2.9066 | LR: 4.43e-05 | 1317.9ms/step | 15.2 samples/s +Step 35690/44000 | Loss: 2.9176 | LR: 4.42e-05 | 1320.2ms/step | 15.1 samples/s +Step 35700/44000 | Loss: 2.9015 | LR: 4.41e-05 | 1317.2ms/step | 15.2 samples/s +Step 35710/44000 | Loss: 2.9216 | LR: 4.40e-05 | 1317.9ms/step | 15.2 samples/s +Step 35720/44000 | Loss: 2.8928 | LR: 4.39e-05 | 1317.3ms/step | 15.2 samples/s +Step 35730/44000 | Loss: 2.9423 | LR: 4.38e-05 | 1316.0ms/step | 15.2 samples/s +Step 35740/44000 | Loss: 2.8865 | LR: 4.37e-05 | 1314.1ms/step | 15.2 samples/s +Step 35750/44000 | Loss: 2.9061 | LR: 4.36e-05 | 1316.9ms/step | 15.2 samples/s +Validation at step 35750: Loss = 2.9081, Perplexity = 18.32 +Step 35760/44000 | Loss: 2.9351 | LR: 4.35e-05 | 3517.7ms/step | 5.7 samples/s +Step 35770/44000 | Loss: 2.9232 | LR: 4.34e-05 | 1316.0ms/step | 15.2 samples/s +Step 35780/44000 | Loss: 2.9071 | LR: 4.33e-05 | 1313.8ms/step | 15.2 samples/s +Step 35790/44000 | Loss: 2.9176 | LR: 4.32e-05 | 1312.1ms/step | 15.2 samples/s +Step 35800/44000 | Loss: 2.9471 | LR: 4.31e-05 | 1309.2ms/step | 15.3 samples/s +Step 35810/44000 | Loss: 2.9292 | LR: 4.29e-05 | 1310.4ms/step | 15.3 samples/s +Step 35820/44000 | Loss: 2.9185 | LR: 4.28e-05 | 1313.1ms/step | 15.2 samples/s +Step 35830/44000 | Loss: 2.9161 | LR: 4.27e-05 | 1313.3ms/step | 15.2 samples/s +Step 35840/44000 | Loss: 2.9269 | LR: 4.26e-05 | 1312.2ms/step | 15.2 samples/s +Step 35850/44000 | Loss: 2.9382 | LR: 4.25e-05 | 1317.0ms/step | 15.2 samples/s +Step 35860/44000 | Loss: 2.9268 | LR: 4.24e-05 | 1314.9ms/step | 15.2 samples/s +Step 35870/44000 | Loss: 2.9050 | LR: 4.23e-05 | 1315.2ms/step | 15.2 samples/s +Step 35880/44000 | Loss: 2.9253 | LR: 4.22e-05 | 1311.6ms/step | 15.2 samples/s +Step 35890/44000 | Loss: 2.9081 | LR: 4.21e-05 | 1314.4ms/step | 15.2 samples/s +Step 35900/44000 | Loss: 2.8789 | LR: 4.20e-05 | 1320.4ms/step | 15.1 samples/s +Step 35910/44000 | Loss: 2.9129 | LR: 4.19e-05 | 1318.7ms/step | 15.2 samples/s +Step 35920/44000 | Loss: 2.8961 | LR: 4.18e-05 | 1319.5ms/step | 15.2 samples/s +Step 35930/44000 | Loss: 2.9244 | LR: 4.17e-05 | 1320.2ms/step | 15.1 samples/s +Step 35940/44000 | Loss: 2.9397 | LR: 4.16e-05 | 1317.2ms/step | 15.2 samples/s +Step 35950/44000 | Loss: 2.9307 | LR: 4.15e-05 | 1321.8ms/step | 15.1 samples/s +Step 35960/44000 | Loss: 2.9301 | LR: 4.14e-05 | 1319.6ms/step | 15.2 samples/s +Step 35970/44000 | Loss: 2.9229 | LR: 4.13e-05 | 1317.6ms/step | 15.2 samples/s +Step 35980/44000 | Loss: 2.8808 | LR: 4.12e-05 | 1320.0ms/step | 15.2 samples/s +Step 35990/44000 | Loss: 2.9122 | LR: 4.11e-05 | 1320.9ms/step | 15.1 samples/s +Step 36000/44000 | Loss: 2.9187 | LR: 4.10e-05 | 1320.9ms/step | 15.1 samples/s +Validation at step 36000: Loss = 2.9081, Perplexity = 18.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-36000 +Step 36010/44000 | Loss: 2.9207 | LR: 4.09e-05 | 3832.0ms/step | 5.2 samples/s +Step 36020/44000 | Loss: 2.9244 | LR: 4.08e-05 | 1317.7ms/step | 15.2 samples/s +Step 36030/44000 | Loss: 2.9177 | LR: 4.07e-05 | 1321.4ms/step | 15.1 samples/s +Step 36040/44000 | Loss: 2.9187 | LR: 4.06e-05 | 1317.5ms/step | 15.2 samples/s +Step 36050/44000 | Loss: 2.9155 | LR: 4.05e-05 | 1319.4ms/step | 15.2 samples/s +Step 36060/44000 | Loss: 2.9161 | LR: 4.04e-05 | 1313.3ms/step | 15.2 samples/s +Step 36070/44000 | Loss: 2.8823 | LR: 4.03e-05 | 1312.5ms/step | 15.2 samples/s +Step 36080/44000 | Loss: 2.9256 | LR: 4.02e-05 | 1310.2ms/step | 15.3 samples/s +Step 36090/44000 | Loss: 2.9094 | LR: 4.01e-05 | 1313.2ms/step | 15.2 samples/s +Step 36100/44000 | Loss: 2.9115 | LR: 4.00e-05 | 1317.6ms/step | 15.2 samples/s +Step 36110/44000 | Loss: 2.9444 | LR: 3.99e-05 | 1317.9ms/step | 15.2 samples/s +Step 36120/44000 | Loss: 2.8869 | LR: 3.98e-05 | 1318.7ms/step | 15.2 samples/s +Step 36130/44000 | Loss: 2.9014 | LR: 3.97e-05 | 1316.9ms/step | 15.2 samples/s +Step 36140/44000 | Loss: 2.9242 | LR: 3.97e-05 | 1318.8ms/step | 15.2 samples/s +Step 36150/44000 | Loss: 2.8987 | LR: 3.96e-05 | 1312.8ms/step | 15.2 samples/s +Step 36160/44000 | Loss: 2.9318 | LR: 3.95e-05 | 1314.7ms/step | 15.2 samples/s +Step 36170/44000 | Loss: 2.9529 | LR: 3.94e-05 | 1318.6ms/step | 15.2 samples/s +Step 36180/44000 | Loss: 2.9400 | LR: 3.93e-05 | 1318.2ms/step | 15.2 samples/s +Step 36190/44000 | Loss: 2.9081 | LR: 3.92e-05 | 1315.6ms/step | 15.2 samples/s +Step 36200/44000 | Loss: 2.9241 | LR: 3.91e-05 | 1311.4ms/step | 15.3 samples/s +Step 36210/44000 | Loss: 2.9299 | LR: 3.90e-05 | 1312.8ms/step | 15.2 samples/s +Step 36220/44000 | Loss: 2.9288 | LR: 3.89e-05 | 1314.9ms/step | 15.2 samples/s +Step 36230/44000 | Loss: 2.8911 | LR: 3.88e-05 | 1315.4ms/step | 15.2 samples/s +Step 36240/44000 | Loss: 2.9202 | LR: 3.87e-05 | 1311.7ms/step | 15.2 samples/s +Step 36250/44000 | Loss: 2.9366 | LR: 3.86e-05 | 1311.8ms/step | 15.2 samples/s +Validation at step 36250: Loss = 2.9079, Perplexity = 18.32 +Step 36260/44000 | Loss: 2.8848 | LR: 3.85e-05 | 3520.9ms/step | 5.7 samples/s +Step 36270/44000 | Loss: 2.8853 | LR: 3.84e-05 | 1319.0ms/step | 15.2 samples/s +Step 36280/44000 | Loss: 2.8979 | LR: 3.83e-05 | 1318.2ms/step | 15.2 samples/s +Step 36290/44000 | Loss: 2.9144 | LR: 3.82e-05 | 1312.8ms/step | 15.2 samples/s +Step 36300/44000 | Loss: 2.9137 | LR: 3.81e-05 | 1308.2ms/step | 15.3 samples/s +Step 36310/44000 | Loss: 2.8932 | LR: 3.80e-05 | 1309.8ms/step | 15.3 samples/s +Step 36320/44000 | Loss: 2.8947 | LR: 3.79e-05 | 1312.2ms/step | 15.2 samples/s +Step 36330/44000 | Loss: 2.8802 | LR: 3.78e-05 | 1314.6ms/step | 15.2 samples/s +Step 36340/44000 | Loss: 2.9106 | LR: 3.77e-05 | 1319.8ms/step | 15.2 samples/s +Step 36350/44000 | Loss: 2.9366 | LR: 3.76e-05 | 1319.1ms/step | 15.2 samples/s +Step 36360/44000 | Loss: 2.9288 | LR: 3.75e-05 | 1320.3ms/step | 15.1 samples/s +Step 36370/44000 | Loss: 2.9331 | LR: 3.74e-05 | 1314.0ms/step | 15.2 samples/s +Step 36380/44000 | Loss: 2.9097 | LR: 3.73e-05 | 1315.8ms/step | 15.2 samples/s +Step 36390/44000 | Loss: 2.9209 | LR: 3.72e-05 | 1316.3ms/step | 15.2 samples/s +Step 36400/44000 | Loss: 2.9369 | LR: 3.71e-05 | 1316.2ms/step | 15.2 samples/s +Step 36410/44000 | Loss: 2.9270 | LR: 3.70e-05 | 1313.6ms/step | 15.2 samples/s +Step 36420/44000 | Loss: 2.9368 | LR: 3.69e-05 | 1314.0ms/step | 15.2 samples/s +Step 36430/44000 | Loss: 2.9301 | LR: 3.69e-05 | 1315.4ms/step | 15.2 samples/s +Step 36440/44000 | Loss: 2.9212 | LR: 3.68e-05 | 1314.7ms/step | 15.2 samples/s +Step 36450/44000 | Loss: 2.8920 | LR: 3.67e-05 | 1315.3ms/step | 15.2 samples/s +Step 36460/44000 | Loss: 2.8882 | LR: 3.66e-05 | 1317.8ms/step | 15.2 samples/s +Step 36470/44000 | Loss: 2.9161 | LR: 3.65e-05 | 1319.6ms/step | 15.2 samples/s +Step 36480/44000 | Loss: 2.9314 | LR: 3.64e-05 | 1322.3ms/step | 15.1 samples/s +Step 36490/44000 | Loss: 2.9124 | LR: 3.63e-05 | 1320.4ms/step | 15.1 samples/s +Step 36500/44000 | Loss: 2.9236 | LR: 3.62e-05 | 1316.2ms/step | 15.2 samples/s +Validation at step 36500: Loss = 2.9077, Perplexity = 18.32 +Step 36510/44000 | Loss: 2.9359 | LR: 3.61e-05 | 3517.9ms/step | 5.7 samples/s +Step 36520/44000 | Loss: 2.9684 | LR: 3.60e-05 | 1313.1ms/step | 15.2 samples/s +Step 36530/44000 | Loss: 2.8891 | LR: 3.59e-05 | 1313.6ms/step | 15.2 samples/s +Step 36540/44000 | Loss: 2.8967 | LR: 3.58e-05 | 1313.9ms/step | 15.2 samples/s +Step 36550/44000 | Loss: 2.9328 | LR: 3.57e-05 | 1315.8ms/step | 15.2 samples/s +Step 36560/44000 | Loss: 2.8889 | LR: 3.56e-05 | 1312.6ms/step | 15.2 samples/s +Step 36570/44000 | Loss: 2.9318 | LR: 3.55e-05 | 1314.7ms/step | 15.2 samples/s +Step 36580/44000 | Loss: 2.9202 | LR: 3.54e-05 | 1314.4ms/step | 15.2 samples/s +Step 36590/44000 | Loss: 2.9200 | LR: 3.53e-05 | 1312.5ms/step | 15.2 samples/s +Step 36600/44000 | Loss: 2.9168 | LR: 3.53e-05 | 1311.0ms/step | 15.3 samples/s +Step 36610/44000 | Loss: 2.9457 | LR: 3.52e-05 | 1313.6ms/step | 15.2 samples/s +Step 36620/44000 | Loss: 2.9494 | LR: 3.51e-05 | 1315.9ms/step | 15.2 samples/s +Step 36630/44000 | Loss: 2.8813 | LR: 3.50e-05 | 1317.4ms/step | 15.2 samples/s +Step 36640/44000 | Loss: 2.9306 | LR: 3.49e-05 | 1315.8ms/step | 15.2 samples/s +Step 36650/44000 | Loss: 2.9098 | LR: 3.48e-05 | 1318.4ms/step | 15.2 samples/s +Step 36660/44000 | Loss: 2.9055 | LR: 3.47e-05 | 1315.6ms/step | 15.2 samples/s +Step 36670/44000 | Loss: 2.9246 | LR: 3.46e-05 | 1314.4ms/step | 15.2 samples/s +Step 36680/44000 | Loss: 2.9044 | LR: 3.45e-05 | 1314.7ms/step | 15.2 samples/s +Step 36690/44000 | Loss: 2.9217 | LR: 3.44e-05 | 1317.2ms/step | 15.2 samples/s +Step 36700/44000 | Loss: 2.9525 | LR: 3.43e-05 | 1316.0ms/step | 15.2 samples/s +Step 36710/44000 | Loss: 2.9308 | LR: 3.42e-05 | 1314.2ms/step | 15.2 samples/s +Step 36720/44000 | Loss: 2.9163 | LR: 3.41e-05 | 1313.5ms/step | 15.2 samples/s +Step 36730/44000 | Loss: 2.8973 | LR: 3.41e-05 | 1313.3ms/step | 15.2 samples/s +Step 36740/44000 | Loss: 2.9428 | LR: 3.40e-05 | 1311.4ms/step | 15.3 samples/s +Step 36750/44000 | Loss: 2.9212 | LR: 3.39e-05 | 1317.5ms/step | 15.2 samples/s +Validation at step 36750: Loss = 2.9075, Perplexity = 18.31 +Step 36760/44000 | Loss: 2.9502 | LR: 3.38e-05 | 3511.8ms/step | 5.7 samples/s +Step 36770/44000 | Loss: 2.9253 | LR: 3.37e-05 | 1310.4ms/step | 15.3 samples/s +Step 36780/44000 | Loss: 2.9366 | LR: 3.36e-05 | 1311.0ms/step | 15.3 samples/s +Step 36790/44000 | Loss: 2.8977 | LR: 3.35e-05 | 1309.9ms/step | 15.3 samples/s +Step 36800/44000 | Loss: 2.8959 | LR: 3.34e-05 | 1314.1ms/step | 15.2 samples/s +Step 36810/44000 | Loss: 2.9040 | LR: 3.33e-05 | 1311.1ms/step | 15.3 samples/s +Step 36820/44000 | Loss: 2.9215 | LR: 3.32e-05 | 1314.2ms/step | 15.2 samples/s +Step 36830/44000 | Loss: 2.9199 | LR: 3.31e-05 | 1315.3ms/step | 15.2 samples/s +Step 36840/44000 | Loss: 2.8932 | LR: 3.31e-05 | 1319.3ms/step | 15.2 samples/s +Step 36850/44000 | Loss: 2.9220 | LR: 3.30e-05 | 1313.3ms/step | 15.2 samples/s +Step 36860/44000 | Loss: 2.9253 | LR: 3.29e-05 | 1309.6ms/step | 15.3 samples/s +Step 36870/44000 | Loss: 2.9007 | LR: 3.28e-05 | 1308.2ms/step | 15.3 samples/s +Step 36880/44000 | Loss: 2.8917 | LR: 3.27e-05 | 1309.7ms/step | 15.3 samples/s +Step 36890/44000 | Loss: 2.9449 | LR: 3.26e-05 | 1309.2ms/step | 15.3 samples/s +Step 36900/44000 | Loss: 2.9116 | LR: 3.25e-05 | 1311.4ms/step | 15.3 samples/s +Step 36910/44000 | Loss: 2.8969 | LR: 3.24e-05 | 1309.5ms/step | 15.3 samples/s +Step 36920/44000 | Loss: 2.8987 | LR: 3.23e-05 | 1314.1ms/step | 15.2 samples/s +Step 36930/44000 | Loss: 2.9300 | LR: 3.22e-05 | 1315.8ms/step | 15.2 samples/s +Step 36940/44000 | Loss: 2.9003 | LR: 3.22e-05 | 1314.6ms/step | 15.2 samples/s +Step 36950/44000 | Loss: 2.9150 | LR: 3.21e-05 | 1318.9ms/step | 15.2 samples/s +Step 36960/44000 | Loss: 2.9207 | LR: 3.20e-05 | 1317.4ms/step | 15.2 samples/s +Step 36970/44000 | Loss: 2.9134 | LR: 3.19e-05 | 1314.4ms/step | 15.2 samples/s +Step 36980/44000 | Loss: 2.9313 | LR: 3.18e-05 | 1314.7ms/step | 15.2 samples/s +Step 36990/44000 | Loss: 2.9107 | LR: 3.17e-05 | 1316.3ms/step | 15.2 samples/s +Step 37000/44000 | Loss: 2.9026 | LR: 3.16e-05 | 1313.1ms/step | 15.2 samples/s +Validation at step 37000: Loss = 2.9078, Perplexity = 18.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-37000 +Step 37010/44000 | Loss: 2.9282 | LR: 3.15e-05 | 3837.6ms/step | 5.2 samples/s +Step 37020/44000 | Loss: 2.9042 | LR: 3.15e-05 | 1320.1ms/step | 15.1 samples/s +Step 37030/44000 | Loss: 2.9473 | LR: 3.14e-05 | 1324.5ms/step | 15.1 samples/s +Step 37040/44000 | Loss: 2.9023 | LR: 3.13e-05 | 1320.7ms/step | 15.1 samples/s +Step 37050/44000 | Loss: 2.9048 | LR: 3.12e-05 | 1321.9ms/step | 15.1 samples/s +Step 37060/44000 | Loss: 2.9066 | LR: 3.11e-05 | 1320.6ms/step | 15.1 samples/s +Step 37070/44000 | Loss: 2.9335 | LR: 3.10e-05 | 1321.9ms/step | 15.1 samples/s +Step 37080/44000 | Loss: 2.9239 | LR: 3.09e-05 | 1319.1ms/step | 15.2 samples/s +Step 37090/44000 | Loss: 2.9403 | LR: 3.08e-05 | 1319.0ms/step | 15.2 samples/s +Step 37100/44000 | Loss: 2.9045 | LR: 3.07e-05 | 1318.5ms/step | 15.2 samples/s +Step 37110/44000 | Loss: 2.8945 | LR: 3.07e-05 | 1320.8ms/step | 15.1 samples/s +Step 37120/44000 | Loss: 2.8761 | LR: 3.06e-05 | 1320.7ms/step | 15.1 samples/s +Step 37130/44000 | Loss: 2.9176 | LR: 3.05e-05 | 1320.2ms/step | 15.1 samples/s +Step 37140/44000 | Loss: 2.9163 | LR: 3.04e-05 | 1317.1ms/step | 15.2 samples/s +Step 37150/44000 | Loss: 2.9149 | LR: 3.03e-05 | 1315.3ms/step | 15.2 samples/s +Step 37160/44000 | Loss: 2.9100 | LR: 3.02e-05 | 1317.1ms/step | 15.2 samples/s +Step 37170/44000 | Loss: 2.9075 | LR: 3.01e-05 | 1317.6ms/step | 15.2 samples/s +Step 37180/44000 | Loss: 2.9287 | LR: 3.01e-05 | 1314.8ms/step | 15.2 samples/s +Step 37190/44000 | Loss: 2.8902 | LR: 3.00e-05 | 1314.0ms/step | 15.2 samples/s +Step 37200/44000 | Loss: 2.9370 | LR: 2.99e-05 | 1314.5ms/step | 15.2 samples/s +Step 37210/44000 | Loss: 2.9200 | LR: 2.98e-05 | 1320.8ms/step | 15.1 samples/s +Step 37220/44000 | Loss: 2.9062 | LR: 2.97e-05 | 1319.5ms/step | 15.2 samples/s +Step 37230/44000 | Loss: 2.9286 | LR: 2.96e-05 | 1323.2ms/step | 15.1 samples/s +Step 37240/44000 | Loss: 2.9235 | LR: 2.95e-05 | 1316.5ms/step | 15.2 samples/s +Step 37250/44000 | Loss: 2.9047 | LR: 2.95e-05 | 1315.6ms/step | 15.2 samples/s +Validation at step 37250: Loss = 2.9074, Perplexity = 18.31 +Step 37260/44000 | Loss: 2.8836 | LR: 2.94e-05 | 3522.6ms/step | 5.7 samples/s +Step 37270/44000 | Loss: 2.9322 | LR: 2.93e-05 | 1320.2ms/step | 15.1 samples/s +Step 37280/44000 | Loss: 2.9241 | LR: 2.92e-05 | 1321.3ms/step | 15.1 samples/s +Step 37290/44000 | Loss: 2.9297 | LR: 2.91e-05 | 1318.1ms/step | 15.2 samples/s +Step 37300/44000 | Loss: 2.9151 | LR: 2.90e-05 | 1319.5ms/step | 15.2 samples/s +Step 37310/44000 | Loss: 2.8916 | LR: 2.89e-05 | 1316.5ms/step | 15.2 samples/s +Step 37320/44000 | Loss: 2.9242 | LR: 2.89e-05 | 1316.6ms/step | 15.2 samples/s +Step 37330/44000 | Loss: 2.9227 | LR: 2.88e-05 | 1323.4ms/step | 15.1 samples/s +Step 37340/44000 | Loss: 2.9337 | LR: 2.87e-05 | 1318.4ms/step | 15.2 samples/s +Step 37350/44000 | Loss: 2.9167 | LR: 2.86e-05 | 1319.3ms/step | 15.2 samples/s +Step 37360/44000 | Loss: 2.9147 | LR: 2.85e-05 | 1319.2ms/step | 15.2 samples/s +Step 37370/44000 | Loss: 2.9308 | LR: 2.84e-05 | 1319.0ms/step | 15.2 samples/s +Step 37380/44000 | Loss: 2.8980 | LR: 2.84e-05 | 1317.7ms/step | 15.2 samples/s +Step 37390/44000 | Loss: 2.9096 | LR: 2.83e-05 | 1316.0ms/step | 15.2 samples/s +Step 37400/44000 | Loss: 2.8816 | LR: 2.82e-05 | 1320.3ms/step | 15.1 samples/s +Step 37410/44000 | Loss: 2.8760 | LR: 2.81e-05 | 1319.5ms/step | 15.2 samples/s +Step 37420/44000 | Loss: 2.9402 | LR: 2.80e-05 | 1318.1ms/step | 15.2 samples/s +Step 37430/44000 | Loss: 2.8733 | LR: 2.79e-05 | 1322.2ms/step | 15.1 samples/s +Step 37440/44000 | Loss: 2.9348 | LR: 2.78e-05 | 1317.9ms/step | 15.2 samples/s +Step 37450/44000 | Loss: 2.9241 | LR: 2.78e-05 | 1318.9ms/step | 15.2 samples/s +Step 37460/44000 | Loss: 2.9214 | LR: 2.77e-05 | 1315.6ms/step | 15.2 samples/s +Step 37470/44000 | Loss: 2.9208 | LR: 2.76e-05 | 1316.1ms/step | 15.2 samples/s +Step 37480/44000 | Loss: 2.9095 | LR: 2.75e-05 | 1315.2ms/step | 15.2 samples/s +Step 37490/44000 | Loss: 2.9019 | LR: 2.74e-05 | 1317.3ms/step | 15.2 samples/s +Step 37500/44000 | Loss: 2.9354 | LR: 2.74e-05 | 1319.2ms/step | 15.2 samples/s +Validation at step 37500: Loss = 2.9073, Perplexity = 18.31 +Step 37510/44000 | Loss: 2.9259 | LR: 2.73e-05 | 3524.1ms/step | 5.7 samples/s +Step 37520/44000 | Loss: 2.9130 | LR: 2.72e-05 | 1318.5ms/step | 15.2 samples/s +Step 37530/44000 | Loss: 2.9232 | LR: 2.71e-05 | 1317.6ms/step | 15.2 samples/s +Step 37540/44000 | Loss: 2.9155 | LR: 2.70e-05 | 1318.9ms/step | 15.2 samples/s +Step 37550/44000 | Loss: 2.9202 | LR: 2.69e-05 | 1324.1ms/step | 15.1 samples/s +Step 37560/44000 | Loss: 2.9149 | LR: 2.69e-05 | 1327.6ms/step | 15.1 samples/s +Step 37570/44000 | Loss: 2.9343 | LR: 2.68e-05 | 1324.4ms/step | 15.1 samples/s +Step 37580/44000 | Loss: 2.8855 | LR: 2.67e-05 | 1325.1ms/step | 15.1 samples/s +Step 37590/44000 | Loss: 2.9224 | LR: 2.66e-05 | 1324.1ms/step | 15.1 samples/s +Step 37600/44000 | Loss: 2.9305 | LR: 2.65e-05 | 1326.4ms/step | 15.1 samples/s +Step 37610/44000 | Loss: 2.9011 | LR: 2.65e-05 | 1322.4ms/step | 15.1 samples/s +Step 37620/44000 | Loss: 2.8992 | LR: 2.64e-05 | 1322.2ms/step | 15.1 samples/s +Step 37630/44000 | Loss: 2.8930 | LR: 2.63e-05 | 1321.1ms/step | 15.1 samples/s +Step 37640/44000 | Loss: 2.9132 | LR: 2.62e-05 | 1320.0ms/step | 15.2 samples/s +Step 37650/44000 | Loss: 2.9210 | LR: 2.61e-05 | 1316.8ms/step | 15.2 samples/s +Step 37660/44000 | Loss: 2.9264 | LR: 2.60e-05 | 1317.3ms/step | 15.2 samples/s +Step 37670/44000 | Loss: 2.9406 | LR: 2.60e-05 | 1321.8ms/step | 15.1 samples/s +Step 37680/44000 | Loss: 2.9265 | LR: 2.59e-05 | 1320.6ms/step | 15.1 samples/s +Step 37690/44000 | Loss: 2.9210 | LR: 2.58e-05 | 1323.6ms/step | 15.1 samples/s +Step 37700/44000 | Loss: 2.9105 | LR: 2.57e-05 | 1319.9ms/step | 15.2 samples/s +Step 37710/44000 | Loss: 2.9089 | LR: 2.56e-05 | 1317.4ms/step | 15.2 samples/s +Step 37720/44000 | Loss: 2.9170 | LR: 2.56e-05 | 1319.8ms/step | 15.2 samples/s +Step 37730/44000 | Loss: 2.9012 | LR: 2.55e-05 | 1321.0ms/step | 15.1 samples/s +Step 37740/44000 | Loss: 2.9034 | LR: 2.54e-05 | 1321.9ms/step | 15.1 samples/s +Step 37750/44000 | Loss: 2.9166 | LR: 2.53e-05 | 1322.3ms/step | 15.1 samples/s +Validation at step 37750: Loss = 2.9072, Perplexity = 18.30 +Step 37760/44000 | Loss: 2.9261 | LR: 2.52e-05 | 3534.9ms/step | 5.7 samples/s +Step 37770/44000 | Loss: 2.9237 | LR: 2.52e-05 | 1319.3ms/step | 15.2 samples/s +Step 37780/44000 | Loss: 2.9097 | LR: 2.51e-05 | 1322.7ms/step | 15.1 samples/s +Step 37790/44000 | Loss: 2.9173 | LR: 2.50e-05 | 1319.4ms/step | 15.2 samples/s +Step 37800/44000 | Loss: 2.9224 | LR: 2.49e-05 | 1320.1ms/step | 15.2 samples/s +Step 37810/44000 | Loss: 2.8778 | LR: 2.48e-05 | 1319.4ms/step | 15.2 samples/s +Step 37820/44000 | Loss: 2.9132 | LR: 2.48e-05 | 1320.1ms/step | 15.2 samples/s +Step 37830/44000 | Loss: 2.9141 | LR: 2.47e-05 | 1319.4ms/step | 15.2 samples/s +Step 37840/44000 | Loss: 2.9275 | LR: 2.46e-05 | 1314.7ms/step | 15.2 samples/s +Step 37850/44000 | Loss: 2.9294 | LR: 2.45e-05 | 1317.4ms/step | 15.2 samples/s +Step 37860/44000 | Loss: 2.9175 | LR: 2.45e-05 | 1316.9ms/step | 15.2 samples/s +Step 37870/44000 | Loss: 2.9153 | LR: 2.44e-05 | 1323.4ms/step | 15.1 samples/s +Step 37880/44000 | Loss: 2.8979 | LR: 2.43e-05 | 1320.0ms/step | 15.2 samples/s +Step 37890/44000 | Loss: 2.8978 | LR: 2.42e-05 | 1314.5ms/step | 15.2 samples/s +Step 37900/44000 | Loss: 2.9410 | LR: 2.41e-05 | 1314.3ms/step | 15.2 samples/s +Step 37910/44000 | Loss: 2.9153 | LR: 2.41e-05 | 1314.2ms/step | 15.2 samples/s +Step 37920/44000 | Loss: 2.9028 | LR: 2.40e-05 | 1316.0ms/step | 15.2 samples/s +Step 37930/44000 | Loss: 2.9071 | LR: 2.39e-05 | 1318.7ms/step | 15.2 samples/s +Step 37940/44000 | Loss: 2.9041 | LR: 2.38e-05 | 1318.7ms/step | 15.2 samples/s +Step 37950/44000 | Loss: 2.9340 | LR: 2.38e-05 | 1318.8ms/step | 15.2 samples/s +Step 37960/44000 | Loss: 2.9016 | LR: 2.37e-05 | 1320.1ms/step | 15.2 samples/s +Step 37970/44000 | Loss: 2.8960 | LR: 2.36e-05 | 1318.1ms/step | 15.2 samples/s +Step 37980/44000 | Loss: 2.9412 | LR: 2.35e-05 | 1315.8ms/step | 15.2 samples/s +Step 37990/44000 | Loss: 2.9173 | LR: 2.34e-05 | 1317.8ms/step | 15.2 samples/s +Step 38000/44000 | Loss: 2.9172 | LR: 2.34e-05 | 1319.1ms/step | 15.2 samples/s +Validation at step 38000: Loss = 2.9073, Perplexity = 18.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-38000 +Step 38010/44000 | Loss: 2.9060 | LR: 2.33e-05 | 3841.4ms/step | 5.2 samples/s +Step 38020/44000 | Loss: 2.8957 | LR: 2.32e-05 | 1313.8ms/step | 15.2 samples/s +Step 38030/44000 | Loss: 2.8900 | LR: 2.31e-05 | 1315.3ms/step | 15.2 samples/s +Step 38040/44000 | Loss: 2.9097 | LR: 2.31e-05 | 1317.1ms/step | 15.2 samples/s +Step 38050/44000 | Loss: 2.9353 | LR: 2.30e-05 | 1318.7ms/step | 15.2 samples/s +Step 38060/44000 | Loss: 2.8999 | LR: 2.29e-05 | 1319.7ms/step | 15.2 samples/s +Step 38070/44000 | Loss: 2.9026 | LR: 2.28e-05 | 1317.8ms/step | 15.2 samples/s +Step 38080/44000 | Loss: 2.9261 | LR: 2.28e-05 | 1319.7ms/step | 15.2 samples/s +Step 38090/44000 | Loss: 2.9297 | LR: 2.27e-05 | 1317.5ms/step | 15.2 samples/s +Step 38100/44000 | Loss: 2.9335 | LR: 2.26e-05 | 1315.3ms/step | 15.2 samples/s +Step 38110/44000 | Loss: 2.9119 | LR: 2.25e-05 | 1316.3ms/step | 15.2 samples/s +Step 38120/44000 | Loss: 2.8995 | LR: 2.25e-05 | 1314.4ms/step | 15.2 samples/s +Step 38130/44000 | Loss: 2.9434 | LR: 2.24e-05 | 1317.5ms/step | 15.2 samples/s +Step 38140/44000 | Loss: 2.9293 | LR: 2.23e-05 | 1318.6ms/step | 15.2 samples/s +Step 38150/44000 | Loss: 2.8997 | LR: 2.22e-05 | 1316.8ms/step | 15.2 samples/s +Step 38160/44000 | Loss: 2.9289 | LR: 2.22e-05 | 1315.6ms/step | 15.2 samples/s +Step 38170/44000 | Loss: 2.9178 | LR: 2.21e-05 | 1316.0ms/step | 15.2 samples/s +Step 38180/44000 | Loss: 2.9075 | LR: 2.20e-05 | 1315.9ms/step | 15.2 samples/s +Step 38190/44000 | Loss: 2.8986 | LR: 2.19e-05 | 1316.0ms/step | 15.2 samples/s +Step 38200/44000 | Loss: 2.9260 | LR: 2.19e-05 | 1314.4ms/step | 15.2 samples/s +Step 38210/44000 | Loss: 2.9401 | LR: 2.18e-05 | 1314.2ms/step | 15.2 samples/s +Step 38220/44000 | Loss: 2.9017 | LR: 2.17e-05 | 1313.5ms/step | 15.2 samples/s +Step 38230/44000 | Loss: 2.8906 | LR: 2.16e-05 | 1313.9ms/step | 15.2 samples/s +Step 38240/44000 | Loss: 2.9048 | LR: 2.16e-05 | 1317.4ms/step | 15.2 samples/s +Step 38250/44000 | Loss: 2.9264 | LR: 2.15e-05 | 1316.2ms/step | 15.2 samples/s +Validation at step 38250: Loss = 2.9068, Perplexity = 18.30 +Step 38260/44000 | Loss: 2.9201 | LR: 2.14e-05 | 3521.2ms/step | 5.7 samples/s +Step 38270/44000 | Loss: 2.9407 | LR: 2.13e-05 | 1318.2ms/step | 15.2 samples/s +Step 38280/44000 | Loss: 2.9221 | LR: 2.13e-05 | 1317.9ms/step | 15.2 samples/s +Step 38290/44000 | Loss: 2.9251 | LR: 2.12e-05 | 1317.9ms/step | 15.2 samples/s +Step 38300/44000 | Loss: 2.9480 | LR: 2.11e-05 | 1317.9ms/step | 15.2 samples/s +Step 38310/44000 | Loss: 2.9089 | LR: 2.11e-05 | 1319.6ms/step | 15.2 samples/s +Step 38320/44000 | Loss: 2.9076 | LR: 2.10e-05 | 1321.0ms/step | 15.1 samples/s +Step 38330/44000 | Loss: 2.9417 | LR: 2.09e-05 | 1317.2ms/step | 15.2 samples/s +Step 38340/44000 | Loss: 2.9060 | LR: 2.08e-05 | 1316.6ms/step | 15.2 samples/s +Step 38350/44000 | Loss: 2.9204 | LR: 2.08e-05 | 1317.4ms/step | 15.2 samples/s +Step 38360/44000 | Loss: 2.9137 | LR: 2.07e-05 | 1317.6ms/step | 15.2 samples/s +Step 38370/44000 | Loss: 2.9571 | LR: 2.06e-05 | 1319.6ms/step | 15.2 samples/s +Step 38380/44000 | Loss: 2.8971 | LR: 2.05e-05 | 1322.1ms/step | 15.1 samples/s +Step 38390/44000 | Loss: 2.9011 | LR: 2.05e-05 | 1317.7ms/step | 15.2 samples/s +Step 38400/44000 | Loss: 2.9414 | LR: 2.04e-05 | 1318.7ms/step | 15.2 samples/s +Step 38410/44000 | Loss: 2.9247 | LR: 2.03e-05 | 1319.1ms/step | 15.2 samples/s +Step 38420/44000 | Loss: 2.8979 | LR: 2.03e-05 | 1317.9ms/step | 15.2 samples/s +Step 38430/44000 | Loss: 2.9451 | LR: 2.02e-05 | 1320.0ms/step | 15.2 samples/s +Step 38440/44000 | Loss: 2.9285 | LR: 2.01e-05 | 1319.6ms/step | 15.2 samples/s +Step 38450/44000 | Loss: 2.9480 | LR: 2.00e-05 | 1320.9ms/step | 15.1 samples/s +Step 38460/44000 | Loss: 2.9192 | LR: 2.00e-05 | 1319.0ms/step | 15.2 samples/s +Step 38470/44000 | Loss: 2.9033 | LR: 1.99e-05 | 1318.0ms/step | 15.2 samples/s +Step 38480/44000 | Loss: 2.9471 | LR: 1.98e-05 | 1320.4ms/step | 15.1 samples/s +Step 38490/44000 | Loss: 2.9149 | LR: 1.98e-05 | 1318.1ms/step | 15.2 samples/s +Step 38500/44000 | Loss: 2.8863 | LR: 1.97e-05 | 1317.7ms/step | 15.2 samples/s +Validation at step 38500: Loss = 2.9072, Perplexity = 18.31 +Step 38510/44000 | Loss: 2.9188 | LR: 1.96e-05 | 3525.1ms/step | 5.7 samples/s +Step 38520/44000 | Loss: 2.8980 | LR: 1.95e-05 | 1319.4ms/step | 15.2 samples/s +Step 38530/44000 | Loss: 2.9205 | LR: 1.95e-05 | 1313.2ms/step | 15.2 samples/s +Step 38540/44000 | Loss: 2.9497 | LR: 1.94e-05 | 1315.2ms/step | 15.2 samples/s +Step 38550/44000 | Loss: 2.8979 | LR: 1.93e-05 | 1314.0ms/step | 15.2 samples/s +Step 38560/44000 | Loss: 2.9414 | LR: 1.93e-05 | 1320.3ms/step | 15.1 samples/s +Step 38570/44000 | Loss: 2.9190 | LR: 1.92e-05 | 1321.8ms/step | 15.1 samples/s +Step 38580/44000 | Loss: 2.9307 | LR: 1.91e-05 | 1319.4ms/step | 15.2 samples/s +Step 38590/44000 | Loss: 2.9283 | LR: 1.91e-05 | 1319.1ms/step | 15.2 samples/s +Step 38600/44000 | Loss: 2.9024 | LR: 1.90e-05 | 1316.3ms/step | 15.2 samples/s +Step 38610/44000 | Loss: 2.9280 | LR: 1.89e-05 | 1313.6ms/step | 15.2 samples/s +Step 38620/44000 | Loss: 2.9343 | LR: 1.88e-05 | 1312.7ms/step | 15.2 samples/s +Step 38630/44000 | Loss: 2.9291 | LR: 1.88e-05 | 1317.4ms/step | 15.2 samples/s +Step 38640/44000 | Loss: 2.9285 | LR: 1.87e-05 | 1318.7ms/step | 15.2 samples/s +Step 38650/44000 | Loss: 2.9042 | LR: 1.86e-05 | 1318.7ms/step | 15.2 samples/s +Step 38660/44000 | Loss: 2.9149 | LR: 1.86e-05 | 1316.3ms/step | 15.2 samples/s +Step 38670/44000 | Loss: 2.9203 | LR: 1.85e-05 | 1314.5ms/step | 15.2 samples/s +Step 38680/44000 | Loss: 2.9527 | LR: 1.84e-05 | 1313.7ms/step | 15.2 samples/s +Step 38690/44000 | Loss: 2.9075 | LR: 1.84e-05 | 1312.3ms/step | 15.2 samples/s +Step 38700/44000 | Loss: 2.9176 | LR: 1.83e-05 | 1312.8ms/step | 15.2 samples/s +Step 38710/44000 | Loss: 2.9080 | LR: 1.82e-05 | 1311.8ms/step | 15.2 samples/s +Step 38720/44000 | Loss: 2.9134 | LR: 1.82e-05 | 1314.0ms/step | 15.2 samples/s +Step 38730/44000 | Loss: 2.8838 | LR: 1.81e-05 | 1316.4ms/step | 15.2 samples/s +Step 38740/44000 | Loss: 2.9040 | LR: 1.80e-05 | 1314.4ms/step | 15.2 samples/s +Step 38750/44000 | Loss: 2.9242 | LR: 1.80e-05 | 1314.3ms/step | 15.2 samples/s +Validation at step 38750: Loss = 2.9068, Perplexity = 18.30 +Step 38760/44000 | Loss: 2.9168 | LR: 1.79e-05 | 3515.3ms/step | 5.7 samples/s +Step 38770/44000 | Loss: 2.8703 | LR: 1.78e-05 | 1315.5ms/step | 15.2 samples/s +Step 38780/44000 | Loss: 2.9476 | LR: 1.78e-05 | 1313.1ms/step | 15.2 samples/s +Step 38790/44000 | Loss: 2.9104 | LR: 1.77e-05 | 1312.3ms/step | 15.2 samples/s +Step 38800/44000 | Loss: 2.9236 | LR: 1.76e-05 | 1318.0ms/step | 15.2 samples/s +Step 38810/44000 | Loss: 2.9287 | LR: 1.76e-05 | 1318.0ms/step | 15.2 samples/s +Step 38820/44000 | Loss: 2.9118 | LR: 1.75e-05 | 1316.2ms/step | 15.2 samples/s +Step 38830/44000 | Loss: 2.9213 | LR: 1.74e-05 | 1319.8ms/step | 15.2 samples/s +Step 38840/44000 | Loss: 2.8997 | LR: 1.74e-05 | 1315.5ms/step | 15.2 samples/s +Step 38850/44000 | Loss: 2.9327 | LR: 1.73e-05 | 1315.9ms/step | 15.2 samples/s +Step 38860/44000 | Loss: 2.9641 | LR: 1.72e-05 | 1316.2ms/step | 15.2 samples/s +Step 38870/44000 | Loss: 2.9199 | LR: 1.72e-05 | 1316.3ms/step | 15.2 samples/s +Step 38880/44000 | Loss: 2.9089 | LR: 1.71e-05 | 1320.7ms/step | 15.1 samples/s +Step 38890/44000 | Loss: 2.9075 | LR: 1.70e-05 | 1311.7ms/step | 15.2 samples/s +Step 38900/44000 | Loss: 2.9051 | LR: 1.70e-05 | 1315.5ms/step | 15.2 samples/s +Step 38910/44000 | Loss: 2.9258 | LR: 1.69e-05 | 1314.9ms/step | 15.2 samples/s +Step 38920/44000 | Loss: 2.9319 | LR: 1.68e-05 | 1312.3ms/step | 15.2 samples/s +Step 38930/44000 | Loss: 2.9083 | LR: 1.68e-05 | 1314.6ms/step | 15.2 samples/s +Step 38940/44000 | Loss: 2.9308 | LR: 1.67e-05 | 1313.6ms/step | 15.2 samples/s +Step 38950/44000 | Loss: 2.9260 | LR: 1.66e-05 | 1314.0ms/step | 15.2 samples/s +Step 38960/44000 | Loss: 2.9121 | LR: 1.66e-05 | 1312.9ms/step | 15.2 samples/s +Step 38970/44000 | Loss: 2.9172 | LR: 1.65e-05 | 1317.1ms/step | 15.2 samples/s +Step 38980/44000 | Loss: 2.9097 | LR: 1.64e-05 | 1314.5ms/step | 15.2 samples/s +Step 38990/44000 | Loss: 2.9207 | LR: 1.64e-05 | 1313.7ms/step | 15.2 samples/s +Step 39000/44000 | Loss: 2.9122 | LR: 1.63e-05 | 1313.2ms/step | 15.2 samples/s +Validation at step 39000: Loss = 2.9068, Perplexity = 18.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-39000 +Step 39010/44000 | Loss: 2.9261 | LR: 1.62e-05 | 3830.6ms/step | 5.2 samples/s +Step 39020/44000 | Loss: 2.9115 | LR: 1.62e-05 | 1321.9ms/step | 15.1 samples/s +Step 39030/44000 | Loss: 2.8963 | LR: 1.61e-05 | 1320.7ms/step | 15.1 samples/s +Step 39040/44000 | Loss: 2.9178 | LR: 1.61e-05 | 1319.5ms/step | 15.2 samples/s +Step 39050/44000 | Loss: 2.9304 | LR: 1.60e-05 | 1315.7ms/step | 15.2 samples/s +Step 39060/44000 | Loss: 2.9099 | LR: 1.59e-05 | 1317.3ms/step | 15.2 samples/s +Step 39070/44000 | Loss: 2.9089 | LR: 1.59e-05 | 1318.9ms/step | 15.2 samples/s +Step 39080/44000 | Loss: 2.9248 | LR: 1.58e-05 | 1318.0ms/step | 15.2 samples/s +Step 39090/44000 | Loss: 2.9395 | LR: 1.57e-05 | 1315.3ms/step | 15.2 samples/s +Step 39100/44000 | Loss: 2.9405 | LR: 1.57e-05 | 1318.9ms/step | 15.2 samples/s +Step 39110/44000 | Loss: 2.9029 | LR: 1.56e-05 | 1314.1ms/step | 15.2 samples/s +Step 39120/44000 | Loss: 2.9081 | LR: 1.55e-05 | 1314.0ms/step | 15.2 samples/s +Step 39130/44000 | Loss: 2.9504 | LR: 1.55e-05 | 1311.8ms/step | 15.2 samples/s +Step 39140/44000 | Loss: 2.8967 | LR: 1.54e-05 | 1311.7ms/step | 15.2 samples/s +Step 39150/44000 | Loss: 2.9033 | LR: 1.54e-05 | 1314.9ms/step | 15.2 samples/s +Step 39160/44000 | Loss: 2.8780 | LR: 1.53e-05 | 1313.6ms/step | 15.2 samples/s +Step 39170/44000 | Loss: 2.9212 | LR: 1.52e-05 | 1314.2ms/step | 15.2 samples/s +Step 39180/44000 | Loss: 2.9148 | LR: 1.52e-05 | 1311.7ms/step | 15.2 samples/s +Step 39190/44000 | Loss: 2.9344 | LR: 1.51e-05 | 1316.3ms/step | 15.2 samples/s +Step 39200/44000 | Loss: 2.9297 | LR: 1.50e-05 | 1318.0ms/step | 15.2 samples/s +Step 39210/44000 | Loss: 2.9131 | LR: 1.50e-05 | 1318.3ms/step | 15.2 samples/s +Step 39220/44000 | Loss: 2.9073 | LR: 1.49e-05 | 1316.7ms/step | 15.2 samples/s +Step 39230/44000 | Loss: 2.8943 | LR: 1.49e-05 | 1316.9ms/step | 15.2 samples/s +Step 39240/44000 | Loss: 2.9253 | LR: 1.48e-05 | 1313.4ms/step | 15.2 samples/s +Step 39250/44000 | Loss: 2.9354 | LR: 1.47e-05 | 1314.8ms/step | 15.2 samples/s +Validation at step 39250: Loss = 2.9069, Perplexity = 18.30 +Step 39260/44000 | Loss: 2.9121 | LR: 1.47e-05 | 3515.1ms/step | 5.7 samples/s +Step 39270/44000 | Loss: 2.9176 | LR: 1.46e-05 | 1311.8ms/step | 15.2 samples/s +Step 39280/44000 | Loss: 2.9062 | LR: 1.46e-05 | 1312.7ms/step | 15.2 samples/s +Step 39290/44000 | Loss: 2.9351 | LR: 1.45e-05 | 1313.3ms/step | 15.2 samples/s +Step 39300/44000 | Loss: 2.9251 | LR: 1.44e-05 | 1315.4ms/step | 15.2 samples/s +Step 39310/44000 | Loss: 2.9186 | LR: 1.44e-05 | 1316.7ms/step | 15.2 samples/s +Step 39320/44000 | Loss: 2.9052 | LR: 1.43e-05 | 1313.6ms/step | 15.2 samples/s +Step 39330/44000 | Loss: 2.9128 | LR: 1.42e-05 | 1309.9ms/step | 15.3 samples/s +Step 39340/44000 | Loss: 2.9193 | LR: 1.42e-05 | 1311.3ms/step | 15.3 samples/s +Step 39350/44000 | Loss: 2.8938 | LR: 1.41e-05 | 1318.0ms/step | 15.2 samples/s +Step 39360/44000 | Loss: 2.8916 | LR: 1.41e-05 | 1319.8ms/step | 15.2 samples/s +Step 39370/44000 | Loss: 2.9128 | LR: 1.40e-05 | 1319.0ms/step | 15.2 samples/s +Step 39380/44000 | Loss: 2.9328 | LR: 1.39e-05 | 1314.1ms/step | 15.2 samples/s +Step 39390/44000 | Loss: 2.9209 | LR: 1.39e-05 | 1312.4ms/step | 15.2 samples/s +Step 39400/44000 | Loss: 2.9262 | LR: 1.38e-05 | 1315.6ms/step | 15.2 samples/s +Step 39410/44000 | Loss: 2.9312 | LR: 1.38e-05 | 1316.0ms/step | 15.2 samples/s +Step 39420/44000 | Loss: 2.9202 | LR: 1.37e-05 | 1315.0ms/step | 15.2 samples/s +Step 39430/44000 | Loss: 2.8982 | LR: 1.36e-05 | 1313.0ms/step | 15.2 samples/s +Step 39440/44000 | Loss: 2.8989 | LR: 1.36e-05 | 1314.6ms/step | 15.2 samples/s +Step 39450/44000 | Loss: 2.9551 | LR: 1.35e-05 | 1314.5ms/step | 15.2 samples/s +Step 39460/44000 | Loss: 2.9077 | LR: 1.35e-05 | 1315.2ms/step | 15.2 samples/s +Step 39470/44000 | Loss: 2.9272 | LR: 1.34e-05 | 1315.8ms/step | 15.2 samples/s +Step 39480/44000 | Loss: 2.9361 | LR: 1.34e-05 | 1317.6ms/step | 15.2 samples/s +Step 39490/44000 | Loss: 2.8955 | LR: 1.33e-05 | 1317.4ms/step | 15.2 samples/s +Step 39500/44000 | Loss: 2.9214 | LR: 1.32e-05 | 1315.8ms/step | 15.2 samples/s +Validation at step 39500: Loss = 2.9069, Perplexity = 18.30 +Step 39510/44000 | Loss: 2.8857 | LR: 1.32e-05 | 3531.6ms/step | 5.7 samples/s +Step 39520/44000 | Loss: 2.9237 | LR: 1.31e-05 | 1320.1ms/step | 15.2 samples/s +Step 39530/44000 | Loss: 2.9105 | LR: 1.31e-05 | 1321.5ms/step | 15.1 samples/s +Step 39540/44000 | Loss: 2.9499 | LR: 1.30e-05 | 1320.3ms/step | 15.1 samples/s +Step 39550/44000 | Loss: 2.9371 | LR: 1.29e-05 | 1321.0ms/step | 15.1 samples/s +Step 39560/44000 | Loss: 2.9211 | LR: 1.29e-05 | 1319.4ms/step | 15.2 samples/s +Step 39570/44000 | Loss: 2.9365 | LR: 1.28e-05 | 1319.5ms/step | 15.2 samples/s +Step 39580/44000 | Loss: 2.9354 | LR: 1.28e-05 | 1319.8ms/step | 15.2 samples/s +Step 39590/44000 | Loss: 2.9105 | LR: 1.27e-05 | 1322.2ms/step | 15.1 samples/s +Step 39600/44000 | Loss: 2.9119 | LR: 1.27e-05 | 1320.2ms/step | 15.1 samples/s +Step 39610/44000 | Loss: 2.9121 | LR: 1.26e-05 | 1318.8ms/step | 15.2 samples/s +Step 39620/44000 | Loss: 2.8922 | LR: 1.25e-05 | 1321.1ms/step | 15.1 samples/s +Step 39630/44000 | Loss: 2.9274 | LR: 1.25e-05 | 1320.5ms/step | 15.1 samples/s +Step 39640/44000 | Loss: 2.9120 | LR: 1.24e-05 | 1321.9ms/step | 15.1 samples/s +Step 39650/44000 | Loss: 2.9053 | LR: 1.24e-05 | 1317.3ms/step | 15.2 samples/s +Step 39660/44000 | Loss: 2.9310 | LR: 1.23e-05 | 1318.1ms/step | 15.2 samples/s +Step 39670/44000 | Loss: 2.9210 | LR: 1.23e-05 | 1316.0ms/step | 15.2 samples/s +Step 39680/44000 | Loss: 2.9158 | LR: 1.22e-05 | 1318.0ms/step | 15.2 samples/s +Step 39690/44000 | Loss: 2.8989 | LR: 1.22e-05 | 1319.1ms/step | 15.2 samples/s +Step 39700/44000 | Loss: 2.9016 | LR: 1.21e-05 | 1318.8ms/step | 15.2 samples/s +Step 39710/44000 | Loss: 2.9035 | LR: 1.20e-05 | 1319.8ms/step | 15.2 samples/s +Step 39720/44000 | Loss: 2.9345 | LR: 1.20e-05 | 1320.8ms/step | 15.1 samples/s +Step 39730/44000 | Loss: 2.9051 | LR: 1.19e-05 | 1319.5ms/step | 15.2 samples/s +Step 39740/44000 | Loss: 2.9081 | LR: 1.19e-05 | 1322.1ms/step | 15.1 samples/s +Step 39750/44000 | Loss: 2.8954 | LR: 1.18e-05 | 1319.7ms/step | 15.2 samples/s +Validation at step 39750: Loss = 2.9068, Perplexity = 18.30 +Step 39760/44000 | Loss: 2.9006 | LR: 1.18e-05 | 3525.9ms/step | 5.7 samples/s +Step 39770/44000 | Loss: 2.9250 | LR: 1.17e-05 | 1319.1ms/step | 15.2 samples/s +Step 39780/44000 | Loss: 2.9184 | LR: 1.17e-05 | 1316.0ms/step | 15.2 samples/s +Step 39790/44000 | Loss: 2.9021 | LR: 1.16e-05 | 1317.4ms/step | 15.2 samples/s +Step 39800/44000 | Loss: 2.9157 | LR: 1.15e-05 | 1319.0ms/step | 15.2 samples/s +Step 39810/44000 | Loss: 2.9244 | LR: 1.15e-05 | 1316.6ms/step | 15.2 samples/s +Step 39820/44000 | Loss: 2.9222 | LR: 1.14e-05 | 1319.8ms/step | 15.2 samples/s +Step 39830/44000 | Loss: 2.9050 | LR: 1.14e-05 | 1318.4ms/step | 15.2 samples/s +Step 39840/44000 | Loss: 2.9093 | LR: 1.13e-05 | 1321.6ms/step | 15.1 samples/s +Step 39850/44000 | Loss: 2.9077 | LR: 1.13e-05 | 1319.2ms/step | 15.2 samples/s +Step 39860/44000 | Loss: 2.9509 | LR: 1.12e-05 | 1319.5ms/step | 15.2 samples/s +Step 39870/44000 | Loss: 2.9175 | LR: 1.12e-05 | 1318.9ms/step | 15.2 samples/s +Step 39880/44000 | Loss: 2.8850 | LR: 1.11e-05 | 1320.4ms/step | 15.1 samples/s +Step 39890/44000 | Loss: 2.8995 | LR: 1.11e-05 | 1320.1ms/step | 15.2 samples/s +Step 39900/44000 | Loss: 2.9299 | LR: 1.10e-05 | 1319.6ms/step | 15.2 samples/s +Step 39910/44000 | Loss: 2.9216 | LR: 1.10e-05 | 1321.5ms/step | 15.1 samples/s +Step 39920/44000 | Loss: 2.9225 | LR: 1.09e-05 | 1323.7ms/step | 15.1 samples/s +Step 39930/44000 | Loss: 2.9083 | LR: 1.08e-05 | 1319.3ms/step | 15.2 samples/s +Step 39940/44000 | Loss: 2.9246 | LR: 1.08e-05 | 1318.8ms/step | 15.2 samples/s +Step 39950/44000 | Loss: 2.9200 | LR: 1.07e-05 | 1319.6ms/step | 15.2 samples/s +Step 39960/44000 | Loss: 2.9166 | LR: 1.07e-05 | 1321.3ms/step | 15.1 samples/s +Step 39970/44000 | Loss: 2.9234 | LR: 1.06e-05 | 1318.2ms/step | 15.2 samples/s +Step 39980/44000 | Loss: 2.8965 | LR: 1.06e-05 | 1318.8ms/step | 15.2 samples/s +Step 39990/44000 | Loss: 2.9250 | LR: 1.05e-05 | 1319.8ms/step | 15.2 samples/s +Step 40000/44000 | Loss: 2.9146 | LR: 1.05e-05 | 1319.0ms/step | 15.2 samples/s +Validation at step 40000: Loss = 2.9070, Perplexity = 18.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-40000 +Step 40010/44000 | Loss: 2.8967 | LR: 1.04e-05 | 3851.3ms/step | 5.2 samples/s +Step 40020/44000 | Loss: 2.9202 | LR: 1.04e-05 | 1315.9ms/step | 15.2 samples/s +Step 40030/44000 | Loss: 2.9281 | LR: 1.03e-05 | 1317.5ms/step | 15.2 samples/s +Step 40040/44000 | Loss: 2.9257 | LR: 1.03e-05 | 1318.8ms/step | 15.2 samples/s +Step 40050/44000 | Loss: 2.9216 | LR: 1.02e-05 | 1317.4ms/step | 15.2 samples/s +Step 40060/44000 | Loss: 2.9332 | LR: 1.02e-05 | 1315.9ms/step | 15.2 samples/s +Step 40070/44000 | Loss: 2.9362 | LR: 1.01e-05 | 1315.0ms/step | 15.2 samples/s +Step 40080/44000 | Loss: 2.8936 | LR: 1.01e-05 | 1313.4ms/step | 15.2 samples/s +Step 40090/44000 | Loss: 2.9354 | LR: 1.00e-05 | 1317.0ms/step | 15.2 samples/s +Step 40100/44000 | Loss: 2.9247 | LR: 9.96e-06 | 1314.5ms/step | 15.2 samples/s +Step 40110/44000 | Loss: 2.9333 | LR: 9.91e-06 | 1313.5ms/step | 15.2 samples/s +Step 40120/44000 | Loss: 2.9543 | LR: 9.86e-06 | 1323.2ms/step | 15.1 samples/s +Step 40130/44000 | Loss: 2.9431 | LR: 9.81e-06 | 1319.8ms/step | 15.2 samples/s +Step 40140/44000 | Loss: 2.9020 | LR: 9.76e-06 | 1318.6ms/step | 15.2 samples/s +Step 40150/44000 | Loss: 2.9199 | LR: 9.71e-06 | 1316.7ms/step | 15.2 samples/s +Step 40160/44000 | Loss: 2.9242 | LR: 9.66e-06 | 1319.3ms/step | 15.2 samples/s +Step 40170/44000 | Loss: 2.8972 | LR: 9.61e-06 | 1316.2ms/step | 15.2 samples/s +Step 40180/44000 | Loss: 2.9158 | LR: 9.56e-06 | 1316.8ms/step | 15.2 samples/s +Step 40190/44000 | Loss: 2.9035 | LR: 9.51e-06 | 1319.7ms/step | 15.2 samples/s +Step 40200/44000 | Loss: 2.9181 | LR: 9.46e-06 | 1318.1ms/step | 15.2 samples/s +Step 40210/44000 | Loss: 2.9042 | LR: 9.41e-06 | 1319.0ms/step | 15.2 samples/s +Step 40220/44000 | Loss: 2.8941 | LR: 9.36e-06 | 1315.4ms/step | 15.2 samples/s +Step 40230/44000 | Loss: 2.9161 | LR: 9.32e-06 | 1312.8ms/step | 15.2 samples/s +Step 40240/44000 | Loss: 2.9199 | LR: 9.27e-06 | 1317.8ms/step | 15.2 samples/s +Step 40250/44000 | Loss: 2.9439 | LR: 9.22e-06 | 1319.4ms/step | 15.2 samples/s +Validation at step 40250: Loss = 2.9069, Perplexity = 18.30 +Step 40260/44000 | Loss: 2.9243 | LR: 9.17e-06 | 3524.9ms/step | 5.7 samples/s +Step 40270/44000 | Loss: 2.8969 | LR: 9.12e-06 | 1320.3ms/step | 15.1 samples/s +Step 40280/44000 | Loss: 2.9025 | LR: 9.07e-06 | 1319.9ms/step | 15.2 samples/s +Step 40290/44000 | Loss: 2.8984 | LR: 9.02e-06 | 1316.3ms/step | 15.2 samples/s +Step 40300/44000 | Loss: 2.9261 | LR: 8.97e-06 | 1317.3ms/step | 15.2 samples/s +Step 40310/44000 | Loss: 2.9327 | LR: 8.93e-06 | 1320.9ms/step | 15.1 samples/s +Step 40320/44000 | Loss: 2.9129 | LR: 8.88e-06 | 1318.7ms/step | 15.2 samples/s +Step 40330/44000 | Loss: 2.9412 | LR: 8.83e-06 | 1318.0ms/step | 15.2 samples/s +Step 40340/44000 | Loss: 2.9509 | LR: 8.78e-06 | 1316.7ms/step | 15.2 samples/s +Step 40350/44000 | Loss: 2.9228 | LR: 8.74e-06 | 1315.4ms/step | 15.2 samples/s +Step 40360/44000 | Loss: 2.8953 | LR: 8.69e-06 | 1315.0ms/step | 15.2 samples/s +Step 40370/44000 | Loss: 2.9019 | LR: 8.64e-06 | 1315.2ms/step | 15.2 samples/s +Step 40380/44000 | Loss: 2.9207 | LR: 8.59e-06 | 1316.7ms/step | 15.2 samples/s +Step 40390/44000 | Loss: 2.8879 | LR: 8.55e-06 | 1315.6ms/step | 15.2 samples/s +Step 40400/44000 | Loss: 2.9568 | LR: 8.50e-06 | 1319.3ms/step | 15.2 samples/s +Step 40410/44000 | Loss: 2.9192 | LR: 8.45e-06 | 1319.1ms/step | 15.2 samples/s +Step 40420/44000 | Loss: 2.9071 | LR: 8.41e-06 | 1319.1ms/step | 15.2 samples/s +Step 40430/44000 | Loss: 2.9001 | LR: 8.36e-06 | 1319.7ms/step | 15.2 samples/s +Step 40440/44000 | Loss: 2.9130 | LR: 8.31e-06 | 1318.8ms/step | 15.2 samples/s +Step 40450/44000 | Loss: 2.9155 | LR: 8.27e-06 | 1320.7ms/step | 15.1 samples/s +Step 40460/44000 | Loss: 2.9447 | LR: 8.22e-06 | 1320.0ms/step | 15.2 samples/s +Step 40470/44000 | Loss: 2.9214 | LR: 8.17e-06 | 1319.5ms/step | 15.2 samples/s +Step 40480/44000 | Loss: 2.9134 | LR: 8.13e-06 | 1319.6ms/step | 15.2 samples/s +Step 40490/44000 | Loss: 2.9329 | LR: 8.08e-06 | 1320.8ms/step | 15.1 samples/s +Step 40500/44000 | Loss: 2.9172 | LR: 8.04e-06 | 1319.3ms/step | 15.2 samples/s +Validation at step 40500: Loss = 2.9068, Perplexity = 18.30 +Step 40510/44000 | Loss: 2.8704 | LR: 7.99e-06 | 3524.7ms/step | 5.7 samples/s +Step 40520/44000 | Loss: 2.9199 | LR: 7.94e-06 | 1318.4ms/step | 15.2 samples/s +Step 40530/44000 | Loss: 2.9203 | LR: 7.90e-06 | 1318.4ms/step | 15.2 samples/s +Step 40540/44000 | Loss: 2.9017 | LR: 7.85e-06 | 1319.5ms/step | 15.2 samples/s +Step 40550/44000 | Loss: 2.9310 | LR: 7.81e-06 | 1322.7ms/step | 15.1 samples/s +Step 40560/44000 | Loss: 2.9303 | LR: 7.76e-06 | 1320.8ms/step | 15.1 samples/s +Step 40570/44000 | Loss: 2.9063 | LR: 7.72e-06 | 1319.1ms/step | 15.2 samples/s +Step 40580/44000 | Loss: 2.9109 | LR: 7.67e-06 | 1320.3ms/step | 15.1 samples/s +Step 40590/44000 | Loss: 2.9093 | LR: 7.63e-06 | 1320.1ms/step | 15.1 samples/s +Step 40600/44000 | Loss: 2.9076 | LR: 7.59e-06 | 1317.4ms/step | 15.2 samples/s +Step 40610/44000 | Loss: 2.9046 | LR: 7.54e-06 | 1317.4ms/step | 15.2 samples/s +Step 40620/44000 | Loss: 2.9045 | LR: 7.50e-06 | 1314.6ms/step | 15.2 samples/s +Step 40630/44000 | Loss: 2.9215 | LR: 7.45e-06 | 1318.6ms/step | 15.2 samples/s +Step 40640/44000 | Loss: 2.8947 | LR: 7.41e-06 | 1322.5ms/step | 15.1 samples/s +Step 40650/44000 | Loss: 2.9136 | LR: 7.37e-06 | 1319.7ms/step | 15.2 samples/s +Step 40660/44000 | Loss: 2.9560 | LR: 7.32e-06 | 1318.7ms/step | 15.2 samples/s +Step 40670/44000 | Loss: 2.9525 | LR: 7.28e-06 | 1319.9ms/step | 15.2 samples/s +Step 40680/44000 | Loss: 2.8997 | LR: 7.23e-06 | 1318.1ms/step | 15.2 samples/s +Step 40690/44000 | Loss: 2.8776 | LR: 7.19e-06 | 1321.6ms/step | 15.1 samples/s +Step 40700/44000 | Loss: 2.9237 | LR: 7.15e-06 | 1321.7ms/step | 15.1 samples/s +Step 40710/44000 | Loss: 2.8985 | LR: 7.10e-06 | 1319.8ms/step | 15.2 samples/s +Step 40720/44000 | Loss: 2.9141 | LR: 7.06e-06 | 1321.2ms/step | 15.1 samples/s +Step 40730/44000 | Loss: 2.9201 | LR: 7.02e-06 | 1319.3ms/step | 15.2 samples/s +Step 40740/44000 | Loss: 2.9214 | LR: 6.98e-06 | 1316.7ms/step | 15.2 samples/s +Step 40750/44000 | Loss: 2.9151 | LR: 6.93e-06 | 1318.4ms/step | 15.2 samples/s +Validation at step 40750: Loss = 2.9067, Perplexity = 18.30 +Step 40760/44000 | Loss: 2.9461 | LR: 6.89e-06 | 3527.1ms/step | 5.7 samples/s +Step 40770/44000 | Loss: 2.9198 | LR: 6.85e-06 | 1318.2ms/step | 15.2 samples/s +Step 40780/44000 | Loss: 2.9053 | LR: 6.81e-06 | 1319.3ms/step | 15.2 samples/s +Step 40790/44000 | Loss: 2.9020 | LR: 6.77e-06 | 1320.4ms/step | 15.1 samples/s +Step 40800/44000 | Loss: 2.9033 | LR: 6.72e-06 | 1318.0ms/step | 15.2 samples/s +Step 40810/44000 | Loss: 2.9129 | LR: 6.68e-06 | 1317.4ms/step | 15.2 samples/s +Step 40820/44000 | Loss: 2.9225 | LR: 6.64e-06 | 1318.4ms/step | 15.2 samples/s +Step 40830/44000 | Loss: 2.9164 | LR: 6.60e-06 | 1320.9ms/step | 15.1 samples/s +Step 40840/44000 | Loss: 2.9352 | LR: 6.56e-06 | 1319.0ms/step | 15.2 samples/s +Step 40850/44000 | Loss: 2.9271 | LR: 6.52e-06 | 1318.6ms/step | 15.2 samples/s +Step 40860/44000 | Loss: 2.9137 | LR: 6.47e-06 | 1320.1ms/step | 15.1 samples/s +Step 40870/44000 | Loss: 2.9149 | LR: 6.43e-06 | 1321.1ms/step | 15.1 samples/s +Step 40880/44000 | Loss: 2.8619 | LR: 6.39e-06 | 1321.2ms/step | 15.1 samples/s +Step 40890/44000 | Loss: 2.8923 | LR: 6.35e-06 | 1323.6ms/step | 15.1 samples/s +Step 40900/44000 | Loss: 2.9043 | LR: 6.31e-06 | 1319.8ms/step | 15.2 samples/s +Step 40910/44000 | Loss: 2.9258 | LR: 6.27e-06 | 1318.7ms/step | 15.2 samples/s +Step 40920/44000 | Loss: 2.9233 | LR: 6.23e-06 | 1318.2ms/step | 15.2 samples/s +Step 40930/44000 | Loss: 2.9497 | LR: 6.19e-06 | 1317.9ms/step | 15.2 samples/s +Step 40940/44000 | Loss: 2.8788 | LR: 6.15e-06 | 1317.6ms/step | 15.2 samples/s +Step 40950/44000 | Loss: 2.9015 | LR: 6.11e-06 | 1317.3ms/step | 15.2 samples/s +Step 40960/44000 | Loss: 2.8932 | LR: 6.07e-06 | 1320.8ms/step | 15.1 samples/s +Step 40970/44000 | Loss: 2.9171 | LR: 6.03e-06 | 1317.5ms/step | 15.2 samples/s +Step 40980/44000 | Loss: 2.9120 | LR: 5.99e-06 | 1315.1ms/step | 15.2 samples/s +Step 40990/44000 | Loss: 2.9081 | LR: 5.95e-06 | 1320.0ms/step | 15.2 samples/s +Step 41000/44000 | Loss: 2.8897 | LR: 5.91e-06 | 1320.6ms/step | 15.1 samples/s +Validation at step 41000: Loss = 2.9068, Perplexity = 18.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-41000 +Step 41010/44000 | Loss: 2.9051 | LR: 5.87e-06 | 3847.1ms/step | 5.2 samples/s +Step 41020/44000 | Loss: 2.9047 | LR: 5.83e-06 | 1319.4ms/step | 15.2 samples/s +Step 41030/44000 | Loss: 2.9217 | LR: 5.80e-06 | 1320.3ms/step | 15.1 samples/s +Step 41040/44000 | Loss: 2.9331 | LR: 5.76e-06 | 1319.3ms/step | 15.2 samples/s +Step 41050/44000 | Loss: 2.9250 | LR: 5.72e-06 | 1318.6ms/step | 15.2 samples/s +Step 41060/44000 | Loss: 2.8952 | LR: 5.68e-06 | 1319.3ms/step | 15.2 samples/s +Step 41070/44000 | Loss: 2.8984 | LR: 5.64e-06 | 1317.0ms/step | 15.2 samples/s +Step 41080/44000 | Loss: 2.9243 | LR: 5.60e-06 | 1317.7ms/step | 15.2 samples/s +Step 41090/44000 | Loss: 2.9426 | LR: 5.56e-06 | 1319.2ms/step | 15.2 samples/s +Step 41100/44000 | Loss: 2.9079 | LR: 5.53e-06 | 1318.1ms/step | 15.2 samples/s +Step 41110/44000 | Loss: 2.8966 | LR: 5.49e-06 | 1319.1ms/step | 15.2 samples/s +Step 41120/44000 | Loss: 2.9230 | LR: 5.45e-06 | 1317.9ms/step | 15.2 samples/s +Step 41130/44000 | Loss: 2.9171 | LR: 5.41e-06 | 1315.8ms/step | 15.2 samples/s +Step 41140/44000 | Loss: 2.8943 | LR: 5.38e-06 | 1318.2ms/step | 15.2 samples/s +Step 41150/44000 | Loss: 2.9224 | LR: 5.34e-06 | 1319.2ms/step | 15.2 samples/s +Step 41160/44000 | Loss: 2.9091 | LR: 5.30e-06 | 1315.1ms/step | 15.2 samples/s +Step 41170/44000 | Loss: 2.9008 | LR: 5.26e-06 | 1315.5ms/step | 15.2 samples/s +Step 41180/44000 | Loss: 2.9056 | LR: 5.23e-06 | 1314.5ms/step | 15.2 samples/s +Step 41190/44000 | Loss: 2.9366 | LR: 5.19e-06 | 1319.8ms/step | 15.2 samples/s +Step 41200/44000 | Loss: 2.9027 | LR: 5.15e-06 | 1321.0ms/step | 15.1 samples/s +Step 41210/44000 | Loss: 2.9153 | LR: 5.12e-06 | 1322.7ms/step | 15.1 samples/s +Step 41220/44000 | Loss: 2.9075 | LR: 5.08e-06 | 1319.9ms/step | 15.2 samples/s +Step 41230/44000 | Loss: 2.9046 | LR: 5.04e-06 | 1319.8ms/step | 15.2 samples/s +Step 41240/44000 | Loss: 2.9260 | LR: 5.01e-06 | 1319.6ms/step | 15.2 samples/s +Step 41250/44000 | Loss: 2.8985 | LR: 4.97e-06 | 1319.6ms/step | 15.2 samples/s +Validation at step 41250: Loss = 2.9070, Perplexity = 18.30 +Step 41260/44000 | Loss: 2.9363 | LR: 4.94e-06 | 3518.5ms/step | 5.7 samples/s +Step 41270/44000 | Loss: 2.8781 | LR: 4.90e-06 | 1316.5ms/step | 15.2 samples/s +Step 41280/44000 | Loss: 2.9358 | LR: 4.86e-06 | 1317.9ms/step | 15.2 samples/s +Step 41290/44000 | Loss: 2.9316 | LR: 4.83e-06 | 1317.5ms/step | 15.2 samples/s +Step 41300/44000 | Loss: 2.9289 | LR: 4.79e-06 | 1320.1ms/step | 15.2 samples/s +Step 41310/44000 | Loss: 2.9143 | LR: 4.76e-06 | 1321.0ms/step | 15.1 samples/s +Step 41320/44000 | Loss: 2.9432 | LR: 4.72e-06 | 1322.1ms/step | 15.1 samples/s +Step 41330/44000 | Loss: 2.8895 | LR: 4.69e-06 | 1321.3ms/step | 15.1 samples/s +Step 41340/44000 | Loss: 2.9386 | LR: 4.65e-06 | 1318.7ms/step | 15.2 samples/s +Step 41350/44000 | Loss: 2.9428 | LR: 4.62e-06 | 1316.9ms/step | 15.2 samples/s +Step 41360/44000 | Loss: 2.8926 | LR: 4.58e-06 | 1318.8ms/step | 15.2 samples/s +Step 41370/44000 | Loss: 2.9267 | LR: 4.55e-06 | 1317.1ms/step | 15.2 samples/s +Step 41380/44000 | Loss: 2.9132 | LR: 4.51e-06 | 1317.3ms/step | 15.2 samples/s +Step 41390/44000 | Loss: 2.9116 | LR: 4.48e-06 | 1315.2ms/step | 15.2 samples/s +Step 41400/44000 | Loss: 2.9221 | LR: 4.45e-06 | 1317.3ms/step | 15.2 samples/s +Step 41410/44000 | Loss: 2.9566 | LR: 4.41e-06 | 1317.3ms/step | 15.2 samples/s +Step 41420/44000 | Loss: 2.9288 | LR: 4.38e-06 | 1320.0ms/step | 15.2 samples/s +Step 41430/44000 | Loss: 2.8969 | LR: 4.34e-06 | 1319.8ms/step | 15.2 samples/s +Step 41440/44000 | Loss: 2.9048 | LR: 4.31e-06 | 1320.8ms/step | 15.1 samples/s +Step 41450/44000 | Loss: 2.9117 | LR: 4.28e-06 | 1319.0ms/step | 15.2 samples/s +Step 41460/44000 | Loss: 2.9094 | LR: 4.24e-06 | 1321.4ms/step | 15.1 samples/s +Step 41470/44000 | Loss: 2.9443 | LR: 4.21e-06 | 1315.3ms/step | 15.2 samples/s +Step 41480/44000 | Loss: 2.9220 | LR: 4.18e-06 | 1314.8ms/step | 15.2 samples/s +Step 41490/44000 | Loss: 2.9299 | LR: 4.14e-06 | 1314.1ms/step | 15.2 samples/s +Step 41500/44000 | Loss: 2.8924 | LR: 4.11e-06 | 1315.0ms/step | 15.2 samples/s +Validation at step 41500: Loss = 2.9067, Perplexity = 18.30 +Step 41510/44000 | Loss: 2.9311 | LR: 4.08e-06 | 3520.5ms/step | 5.7 samples/s +Step 41520/44000 | Loss: 2.9170 | LR: 4.05e-06 | 1319.1ms/step | 15.2 samples/s +Step 41530/44000 | Loss: 2.9337 | LR: 4.01e-06 | 1315.0ms/step | 15.2 samples/s +Step 41540/44000 | Loss: 2.9288 | LR: 3.98e-06 | 1313.1ms/step | 15.2 samples/s +Step 41550/44000 | Loss: 2.8982 | LR: 3.95e-06 | 1315.1ms/step | 15.2 samples/s +Step 41560/44000 | Loss: 2.9226 | LR: 3.92e-06 | 1318.5ms/step | 15.2 samples/s +Step 41570/44000 | Loss: 2.9049 | LR: 3.88e-06 | 1320.3ms/step | 15.1 samples/s +Step 41580/44000 | Loss: 2.8867 | LR: 3.85e-06 | 1318.8ms/step | 15.2 samples/s +Step 41590/44000 | Loss: 2.9276 | LR: 3.82e-06 | 1321.1ms/step | 15.1 samples/s +Step 41600/44000 | Loss: 2.8777 | LR: 3.79e-06 | 1317.6ms/step | 15.2 samples/s +Step 41610/44000 | Loss: 2.9008 | LR: 3.76e-06 | 1317.1ms/step | 15.2 samples/s +Step 41620/44000 | Loss: 2.9167 | LR: 3.73e-06 | 1318.1ms/step | 15.2 samples/s +Step 41630/44000 | Loss: 2.8880 | LR: 3.70e-06 | 1319.8ms/step | 15.2 samples/s +Step 41640/44000 | Loss: 2.9289 | LR: 3.66e-06 | 1319.1ms/step | 15.2 samples/s +Step 41650/44000 | Loss: 2.9122 | LR: 3.63e-06 | 1319.4ms/step | 15.2 samples/s +Step 41660/44000 | Loss: 2.9090 | LR: 3.60e-06 | 1317.7ms/step | 15.2 samples/s +Step 41670/44000 | Loss: 2.9393 | LR: 3.57e-06 | 1317.0ms/step | 15.2 samples/s +Step 41680/44000 | Loss: 2.9160 | LR: 3.54e-06 | 1315.9ms/step | 15.2 samples/s +Step 41690/44000 | Loss: 2.9031 | LR: 3.51e-06 | 1314.7ms/step | 15.2 samples/s +Step 41700/44000 | Loss: 2.8907 | LR: 3.48e-06 | 1319.1ms/step | 15.2 samples/s +Step 41710/44000 | Loss: 2.8998 | LR: 3.45e-06 | 1316.1ms/step | 15.2 samples/s +Step 41720/44000 | Loss: 2.9415 | LR: 3.42e-06 | 1318.1ms/step | 15.2 samples/s +Step 41730/44000 | Loss: 2.9377 | LR: 3.39e-06 | 1319.5ms/step | 15.2 samples/s +Step 41740/44000 | Loss: 2.9609 | LR: 3.36e-06 | 1317.6ms/step | 15.2 samples/s +Step 41750/44000 | Loss: 2.9033 | LR: 3.33e-06 | 1316.9ms/step | 15.2 samples/s +Validation at step 41750: Loss = 2.9068, Perplexity = 18.30 +Step 41760/44000 | Loss: 2.9150 | LR: 3.30e-06 | 3525.0ms/step | 5.7 samples/s +Step 41770/44000 | Loss: 2.8866 | LR: 3.27e-06 | 1314.2ms/step | 15.2 samples/s +Step 41780/44000 | Loss: 2.9053 | LR: 3.24e-06 | 1312.7ms/step | 15.2 samples/s +Step 41790/44000 | Loss: 2.9144 | LR: 3.21e-06 | 1315.8ms/step | 15.2 samples/s +Step 41800/44000 | Loss: 2.9631 | LR: 3.19e-06 | 1315.3ms/step | 15.2 samples/s +Step 41810/44000 | Loss: 2.9264 | LR: 3.16e-06 | 1316.6ms/step | 15.2 samples/s +Step 41820/44000 | Loss: 2.9478 | LR: 3.13e-06 | 1319.3ms/step | 15.2 samples/s +Step 41830/44000 | Loss: 2.8890 | LR: 3.10e-06 | 1319.2ms/step | 15.2 samples/s +Step 41840/44000 | Loss: 2.9159 | LR: 3.07e-06 | 1321.1ms/step | 15.1 samples/s +Step 41850/44000 | Loss: 2.9361 | LR: 3.04e-06 | 1322.6ms/step | 15.1 samples/s +Step 41860/44000 | Loss: 2.9041 | LR: 3.01e-06 | 1319.6ms/step | 15.2 samples/s +Step 41870/44000 | Loss: 2.9034 | LR: 2.99e-06 | 1316.5ms/step | 15.2 samples/s +Step 41880/44000 | Loss: 2.8783 | LR: 2.96e-06 | 1315.0ms/step | 15.2 samples/s +Step 41890/44000 | Loss: 2.9282 | LR: 2.93e-06 | 1314.7ms/step | 15.2 samples/s +Step 41900/44000 | Loss: 2.9072 | LR: 2.90e-06 | 1316.0ms/step | 15.2 samples/s +Step 41910/44000 | Loss: 2.9029 | LR: 2.88e-06 | 1317.6ms/step | 15.2 samples/s +Step 41920/44000 | Loss: 2.9178 | LR: 2.85e-06 | 1322.0ms/step | 15.1 samples/s +Step 41930/44000 | Loss: 2.8975 | LR: 2.82e-06 | 1319.1ms/step | 15.2 samples/s +Step 41940/44000 | Loss: 2.9393 | LR: 2.79e-06 | 1319.1ms/step | 15.2 samples/s +Step 41950/44000 | Loss: 2.9092 | LR: 2.77e-06 | 1320.1ms/step | 15.1 samples/s +Step 41960/44000 | Loss: 2.8909 | LR: 2.74e-06 | 1321.4ms/step | 15.1 samples/s +Step 41970/44000 | Loss: 2.9142 | LR: 2.71e-06 | 1316.6ms/step | 15.2 samples/s +Step 41980/44000 | Loss: 2.9263 | LR: 2.69e-06 | 1317.5ms/step | 15.2 samples/s +Step 41990/44000 | Loss: 2.9472 | LR: 2.66e-06 | 1319.5ms/step | 15.2 samples/s +Step 42000/44000 | Loss: 2.8863 | LR: 2.63e-06 | 1318.4ms/step | 15.2 samples/s +Validation at step 42000: Loss = 2.9068, Perplexity = 18.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-42000 +Step 42010/44000 | Loss: 2.9265 | LR: 2.61e-06 | 3830.7ms/step | 5.2 samples/s +Step 42020/44000 | Loss: 2.9217 | LR: 2.58e-06 | 1316.9ms/step | 15.2 samples/s +Step 42030/44000 | Loss: 2.9069 | LR: 2.56e-06 | 1316.9ms/step | 15.2 samples/s +Step 42040/44000 | Loss: 2.9102 | LR: 2.53e-06 | 1317.9ms/step | 15.2 samples/s +Step 42050/44000 | Loss: 2.9110 | LR: 2.50e-06 | 1315.6ms/step | 15.2 samples/s +Step 42060/44000 | Loss: 2.9179 | LR: 2.48e-06 | 1317.0ms/step | 15.2 samples/s +Step 42070/44000 | Loss: 2.9383 | LR: 2.45e-06 | 1319.6ms/step | 15.2 samples/s +Step 42080/44000 | Loss: 2.8980 | LR: 2.43e-06 | 1316.4ms/step | 15.2 samples/s +Step 42090/44000 | Loss: 2.9126 | LR: 2.40e-06 | 1318.5ms/step | 15.2 samples/s +Step 42100/44000 | Loss: 2.8886 | LR: 2.38e-06 | 1317.1ms/step | 15.2 samples/s +Step 42110/44000 | Loss: 2.9367 | LR: 2.35e-06 | 1314.6ms/step | 15.2 samples/s +Step 42120/44000 | Loss: 2.9186 | LR: 2.33e-06 | 1312.9ms/step | 15.2 samples/s +Step 42130/44000 | Loss: 2.9223 | LR: 2.30e-06 | 1311.7ms/step | 15.2 samples/s +Step 42140/44000 | Loss: 2.9223 | LR: 2.28e-06 | 1313.9ms/step | 15.2 samples/s +Step 42150/44000 | Loss: 2.8840 | LR: 2.25e-06 | 1310.0ms/step | 15.3 samples/s +Step 42160/44000 | Loss: 2.8906 | LR: 2.23e-06 | 1312.5ms/step | 15.2 samples/s +Step 42170/44000 | Loss: 2.8942 | LR: 2.21e-06 | 1310.8ms/step | 15.3 samples/s +Step 42180/44000 | Loss: 2.9331 | LR: 2.18e-06 | 1308.4ms/step | 15.3 samples/s +Step 42190/44000 | Loss: 2.9030 | LR: 2.16e-06 | 1308.2ms/step | 15.3 samples/s +Step 42200/44000 | Loss: 2.9475 | LR: 2.13e-06 | 1317.7ms/step | 15.2 samples/s +Step 42210/44000 | Loss: 2.9471 | LR: 2.11e-06 | 1316.9ms/step | 15.2 samples/s +Step 42220/44000 | Loss: 2.9120 | LR: 2.09e-06 | 1315.9ms/step | 15.2 samples/s +Step 42230/44000 | Loss: 2.9253 | LR: 2.06e-06 | 1316.3ms/step | 15.2 samples/s +Step 42240/44000 | Loss: 2.9233 | LR: 2.04e-06 | 1318.5ms/step | 15.2 samples/s +Step 42250/44000 | Loss: 2.9313 | LR: 2.02e-06 | 1318.6ms/step | 15.2 samples/s +Validation at step 42250: Loss = 2.9069, Perplexity = 18.30 +Step 42260/44000 | Loss: 2.8904 | LR: 1.99e-06 | 3522.3ms/step | 5.7 samples/s +Step 42270/44000 | Loss: 2.9230 | LR: 1.97e-06 | 1317.0ms/step | 15.2 samples/s +Step 42280/44000 | Loss: 2.9094 | LR: 1.95e-06 | 1314.7ms/step | 15.2 samples/s +Step 42290/44000 | Loss: 2.9264 | LR: 1.93e-06 | 1313.2ms/step | 15.2 samples/s +Step 42300/44000 | Loss: 2.9115 | LR: 1.90e-06 | 1314.5ms/step | 15.2 samples/s +Step 42310/44000 | Loss: 2.8881 | LR: 1.88e-06 | 1313.9ms/step | 15.2 samples/s +Step 42320/44000 | Loss: 2.9083 | LR: 1.86e-06 | 1319.2ms/step | 15.2 samples/s +Step 42330/44000 | Loss: 2.9149 | LR: 1.84e-06 | 1316.0ms/step | 15.2 samples/s +Step 42340/44000 | Loss: 2.9266 | LR: 1.82e-06 | 1318.6ms/step | 15.2 samples/s +Step 42350/44000 | Loss: 2.9338 | LR: 1.79e-06 | 1321.5ms/step | 15.1 samples/s +Step 42360/44000 | Loss: 2.9153 | LR: 1.77e-06 | 1318.3ms/step | 15.2 samples/s +Step 42370/44000 | Loss: 2.8998 | LR: 1.75e-06 | 1314.1ms/step | 15.2 samples/s +Step 42380/44000 | Loss: 2.9110 | LR: 1.73e-06 | 1314.1ms/step | 15.2 samples/s +Step 42390/44000 | Loss: 2.9400 | LR: 1.71e-06 | 1313.6ms/step | 15.2 samples/s +Step 42400/44000 | Loss: 2.9258 | LR: 1.69e-06 | 1316.0ms/step | 15.2 samples/s +Step 42410/44000 | Loss: 2.9280 | LR: 1.67e-06 | 1320.1ms/step | 15.1 samples/s +Step 42420/44000 | Loss: 2.8916 | LR: 1.64e-06 | 1319.0ms/step | 15.2 samples/s +Step 42430/44000 | Loss: 2.8971 | LR: 1.62e-06 | 1318.0ms/step | 15.2 samples/s +Step 42440/44000 | Loss: 2.9158 | LR: 1.60e-06 | 1310.8ms/step | 15.3 samples/s +Step 42450/44000 | Loss: 2.9059 | LR: 1.58e-06 | 1317.0ms/step | 15.2 samples/s +Step 42460/44000 | Loss: 2.9259 | LR: 1.56e-06 | 1317.3ms/step | 15.2 samples/s +Step 42470/44000 | Loss: 2.9101 | LR: 1.54e-06 | 1318.1ms/step | 15.2 samples/s +Step 42480/44000 | Loss: 2.9319 | LR: 1.52e-06 | 1314.6ms/step | 15.2 samples/s +Step 42490/44000 | Loss: 2.9423 | LR: 1.50e-06 | 1313.2ms/step | 15.2 samples/s +Step 42500/44000 | Loss: 2.8995 | LR: 1.48e-06 | 1310.2ms/step | 15.3 samples/s +Validation at step 42500: Loss = 2.9066, Perplexity = 18.29 +Step 42510/44000 | Loss: 2.9284 | LR: 1.46e-06 | 3517.6ms/step | 5.7 samples/s +Step 42520/44000 | Loss: 2.9010 | LR: 1.44e-06 | 1315.3ms/step | 15.2 samples/s +Step 42530/44000 | Loss: 2.9156 | LR: 1.42e-06 | 1314.1ms/step | 15.2 samples/s +Step 42540/44000 | Loss: 2.9527 | LR: 1.40e-06 | 1317.0ms/step | 15.2 samples/s +Step 42550/44000 | Loss: 2.9313 | LR: 1.39e-06 | 1316.2ms/step | 15.2 samples/s +Step 42560/44000 | Loss: 2.8848 | LR: 1.37e-06 | 1318.3ms/step | 15.2 samples/s +Step 42570/44000 | Loss: 2.9405 | LR: 1.35e-06 | 1321.0ms/step | 15.1 samples/s +Step 42580/44000 | Loss: 2.9139 | LR: 1.33e-06 | 1321.9ms/step | 15.1 samples/s +Step 42590/44000 | Loss: 2.8983 | LR: 1.31e-06 | 1321.6ms/step | 15.1 samples/s +Step 42600/44000 | Loss: 2.9186 | LR: 1.29e-06 | 1319.0ms/step | 15.2 samples/s +Step 42610/44000 | Loss: 2.8942 | LR: 1.27e-06 | 1319.3ms/step | 15.2 samples/s +Step 42620/44000 | Loss: 2.9182 | LR: 1.25e-06 | 1316.6ms/step | 15.2 samples/s +Step 42630/44000 | Loss: 2.9345 | LR: 1.24e-06 | 1317.4ms/step | 15.2 samples/s +Step 42640/44000 | Loss: 2.9187 | LR: 1.22e-06 | 1316.6ms/step | 15.2 samples/s +Step 42650/44000 | Loss: 2.9168 | LR: 1.20e-06 | 1313.4ms/step | 15.2 samples/s +Step 42660/44000 | Loss: 2.9444 | LR: 1.18e-06 | 1311.9ms/step | 15.2 samples/s +Step 42670/44000 | Loss: 2.9087 | LR: 1.17e-06 | 1311.0ms/step | 15.3 samples/s +Step 42680/44000 | Loss: 2.9244 | LR: 1.15e-06 | 1311.4ms/step | 15.3 samples/s +Step 42690/44000 | Loss: 2.9202 | LR: 1.13e-06 | 1317.7ms/step | 15.2 samples/s +Step 42700/44000 | Loss: 2.9102 | LR: 1.11e-06 | 1319.7ms/step | 15.2 samples/s +Step 42710/44000 | Loss: 2.8922 | LR: 1.10e-06 | 1320.4ms/step | 15.1 samples/s +Step 42720/44000 | Loss: 2.9270 | LR: 1.08e-06 | 1320.1ms/step | 15.2 samples/s +Step 42730/44000 | Loss: 2.9378 | LR: 1.06e-06 | 1317.6ms/step | 15.2 samples/s +Step 42740/44000 | Loss: 2.8930 | LR: 1.05e-06 | 1319.0ms/step | 15.2 samples/s +Step 42750/44000 | Loss: 2.9469 | LR: 1.03e-06 | 1313.9ms/step | 15.2 samples/s +Validation at step 42750: Loss = 2.9066, Perplexity = 18.29 +Step 42760/44000 | Loss: 2.9267 | LR: 1.01e-06 | 3510.2ms/step | 5.7 samples/s +Step 42770/44000 | Loss: 2.9211 | LR: 9.97e-07 | 1309.0ms/step | 15.3 samples/s +Step 42780/44000 | Loss: 2.9222 | LR: 9.81e-07 | 1310.4ms/step | 15.3 samples/s +Step 42790/44000 | Loss: 2.9238 | LR: 9.65e-07 | 1312.9ms/step | 15.2 samples/s +Step 42800/44000 | Loss: 2.9146 | LR: 9.49e-07 | 1312.0ms/step | 15.2 samples/s +Step 42810/44000 | Loss: 2.9191 | LR: 9.33e-07 | 1309.4ms/step | 15.3 samples/s +Step 42820/44000 | Loss: 2.9185 | LR: 9.18e-07 | 1309.9ms/step | 15.3 samples/s +Step 42830/44000 | Loss: 2.9015 | LR: 9.02e-07 | 1310.4ms/step | 15.3 samples/s +Step 42840/44000 | Loss: 2.9475 | LR: 8.87e-07 | 1312.0ms/step | 15.2 samples/s +Step 42850/44000 | Loss: 2.8960 | LR: 8.72e-07 | 1312.0ms/step | 15.2 samples/s +Step 42860/44000 | Loss: 2.9375 | LR: 8.57e-07 | 1313.1ms/step | 15.2 samples/s +Step 42870/44000 | Loss: 2.9113 | LR: 8.42e-07 | 1319.7ms/step | 15.2 samples/s +Step 42880/44000 | Loss: 2.9260 | LR: 8.27e-07 | 1313.4ms/step | 15.2 samples/s +Step 42890/44000 | Loss: 2.9114 | LR: 8.12e-07 | 1312.5ms/step | 15.2 samples/s +Step 42900/44000 | Loss: 2.8871 | LR: 7.98e-07 | 1312.7ms/step | 15.2 samples/s +Step 42910/44000 | Loss: 2.9309 | LR: 7.83e-07 | 1309.4ms/step | 15.3 samples/s +Step 42920/44000 | Loss: 2.8805 | LR: 7.69e-07 | 1311.3ms/step | 15.3 samples/s +Step 42930/44000 | Loss: 2.9124 | LR: 7.55e-07 | 1311.0ms/step | 15.3 samples/s +Step 42940/44000 | Loss: 2.9211 | LR: 7.41e-07 | 1312.1ms/step | 15.2 samples/s +Step 42950/44000 | Loss: 2.9362 | LR: 7.27e-07 | 1312.7ms/step | 15.2 samples/s +Step 42960/44000 | Loss: 2.9015 | LR: 7.13e-07 | 1312.9ms/step | 15.2 samples/s +Step 42970/44000 | Loss: 2.9016 | LR: 6.99e-07 | 1310.9ms/step | 15.3 samples/s +Step 42980/44000 | Loss: 2.9160 | LR: 6.86e-07 | 1309.8ms/step | 15.3 samples/s +Step 42990/44000 | Loss: 2.8827 | LR: 6.72e-07 | 1308.9ms/step | 15.3 samples/s +Step 43000/44000 | Loss: 2.9208 | LR: 6.59e-07 | 1311.7ms/step | 15.2 samples/s +Validation at step 43000: Loss = 2.9068, Perplexity = 18.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-43000 +Step 43010/44000 | Loss: 2.9463 | LR: 6.46e-07 | 3835.9ms/step | 5.2 samples/s +Step 43020/44000 | Loss: 2.9313 | LR: 6.33e-07 | 1319.4ms/step | 15.2 samples/s +Step 43030/44000 | Loss: 2.9315 | LR: 6.20e-07 | 1319.9ms/step | 15.2 samples/s +Step 43040/44000 | Loss: 2.9201 | LR: 6.08e-07 | 1317.9ms/step | 15.2 samples/s +Step 43050/44000 | Loss: 2.9341 | LR: 5.95e-07 | 1319.2ms/step | 15.2 samples/s +Step 43060/44000 | Loss: 2.9336 | LR: 5.83e-07 | 1318.4ms/step | 15.2 samples/s +Step 43070/44000 | Loss: 2.9313 | LR: 5.70e-07 | 1319.3ms/step | 15.2 samples/s +Step 43080/44000 | Loss: 2.8978 | LR: 5.58e-07 | 1322.0ms/step | 15.1 samples/s +Step 43090/44000 | Loss: 2.8935 | LR: 5.46e-07 | 1320.6ms/step | 15.1 samples/s +Step 43100/44000 | Loss: 2.9112 | LR: 5.34e-07 | 1321.8ms/step | 15.1 samples/s +Step 43110/44000 | Loss: 2.9435 | LR: 5.22e-07 | 1320.3ms/step | 15.1 samples/s +Step 43120/44000 | Loss: 2.9165 | LR: 5.11e-07 | 1319.6ms/step | 15.2 samples/s +Step 43130/44000 | Loss: 2.9288 | LR: 4.99e-07 | 1321.4ms/step | 15.1 samples/s +Step 43140/44000 | Loss: 2.9492 | LR: 4.88e-07 | 1321.0ms/step | 15.1 samples/s +Step 43150/44000 | Loss: 2.8995 | LR: 4.76e-07 | 1322.0ms/step | 15.1 samples/s +Step 43160/44000 | Loss: 2.9294 | LR: 4.65e-07 | 1319.6ms/step | 15.2 samples/s +Step 43170/44000 | Loss: 2.8610 | LR: 4.54e-07 | 1318.9ms/step | 15.2 samples/s +Step 43180/44000 | Loss: 2.9023 | LR: 4.43e-07 | 1319.2ms/step | 15.2 samples/s +Step 43190/44000 | Loss: 2.9326 | LR: 4.33e-07 | 1319.2ms/step | 15.2 samples/s +Step 43200/44000 | Loss: 2.9338 | LR: 4.22e-07 | 1319.4ms/step | 15.2 samples/s +Step 43210/44000 | Loss: 2.9159 | LR: 4.12e-07 | 1318.3ms/step | 15.2 samples/s +Step 43220/44000 | Loss: 2.9046 | LR: 4.01e-07 | 1318.0ms/step | 15.2 samples/s +Step 43230/44000 | Loss: 2.9295 | LR: 3.91e-07 | 1318.0ms/step | 15.2 samples/s +Step 43240/44000 | Loss: 2.9142 | LR: 3.81e-07 | 1318.9ms/step | 15.2 samples/s +Step 43250/44000 | Loss: 2.9140 | LR: 3.71e-07 | 1314.3ms/step | 15.2 samples/s +Validation at step 43250: Loss = 2.9066, Perplexity = 18.29 +Step 43260/44000 | Loss: 2.8972 | LR: 3.61e-07 | 3512.8ms/step | 5.7 samples/s +Step 43270/44000 | Loss: 2.9333 | LR: 3.51e-07 | 1310.1ms/step | 15.3 samples/s +Step 43280/44000 | Loss: 2.9241 | LR: 3.42e-07 | 1310.8ms/step | 15.3 samples/s +Step 43290/44000 | Loss: 2.9262 | LR: 3.32e-07 | 1308.0ms/step | 15.3 samples/s +Step 43300/44000 | Loss: 2.9127 | LR: 3.23e-07 | 1310.3ms/step | 15.3 samples/s +Step 43310/44000 | Loss: 2.9122 | LR: 3.14e-07 | 1312.9ms/step | 15.2 samples/s +Step 43320/44000 | Loss: 2.9014 | LR: 3.05e-07 | 1314.6ms/step | 15.2 samples/s +Step 43330/44000 | Loss: 2.8962 | LR: 2.96e-07 | 1314.7ms/step | 15.2 samples/s +Step 43340/44000 | Loss: 2.9161 | LR: 2.87e-07 | 1313.3ms/step | 15.2 samples/s +Step 43350/44000 | Loss: 2.9174 | LR: 2.79e-07 | 1320.3ms/step | 15.1 samples/s +Step 43360/44000 | Loss: 2.9178 | LR: 2.70e-07 | 1319.5ms/step | 15.2 samples/s +Step 43370/44000 | Loss: 2.9249 | LR: 2.62e-07 | 1314.1ms/step | 15.2 samples/s +Step 43380/44000 | Loss: 2.9224 | LR: 2.53e-07 | 1313.0ms/step | 15.2 samples/s +Step 43390/44000 | Loss: 2.9198 | LR: 2.45e-07 | 1314.1ms/step | 15.2 samples/s +Step 43400/44000 | Loss: 2.9141 | LR: 2.37e-07 | 1313.9ms/step | 15.2 samples/s +Step 43410/44000 | Loss: 2.9114 | LR: 2.30e-07 | 1314.6ms/step | 15.2 samples/s +Step 43420/44000 | Loss: 2.9403 | LR: 2.22e-07 | 1315.4ms/step | 15.2 samples/s +Step 43430/44000 | Loss: 2.9200 | LR: 2.14e-07 | 1312.7ms/step | 15.2 samples/s +Step 43440/44000 | Loss: 2.9067 | LR: 2.07e-07 | 1311.5ms/step | 15.2 samples/s +Step 43450/44000 | Loss: 2.9107 | LR: 1.99e-07 | 1311.3ms/step | 15.3 samples/s +Step 43460/44000 | Loss: 2.8975 | LR: 1.92e-07 | 1312.7ms/step | 15.2 samples/s +Step 43470/44000 | Loss: 2.9128 | LR: 1.85e-07 | 1315.2ms/step | 15.2 samples/s +Step 43480/44000 | Loss: 2.9068 | LR: 1.78e-07 | 1312.1ms/step | 15.2 samples/s +Step 43490/44000 | Loss: 2.9301 | LR: 1.72e-07 | 1313.1ms/step | 15.2 samples/s +Step 43500/44000 | Loss: 2.9108 | LR: 1.65e-07 | 1314.3ms/step | 15.2 samples/s +Validation at step 43500: Loss = 2.9069, Perplexity = 18.30 +Step 43510/44000 | Loss: 2.9027 | LR: 1.58e-07 | 3517.7ms/step | 5.7 samples/s +Step 43520/44000 | Loss: 2.8936 | LR: 1.52e-07 | 1311.3ms/step | 15.3 samples/s +Step 43530/44000 | Loss: 2.9146 | LR: 1.46e-07 | 1312.0ms/step | 15.2 samples/s +Step 43540/44000 | Loss: 2.9060 | LR: 1.40e-07 | 1308.7ms/step | 15.3 samples/s +Step 43550/44000 | Loss: 2.9014 | LR: 1.34e-07 | 1309.8ms/step | 15.3 samples/s +Step 43560/44000 | Loss: 2.8741 | LR: 1.28e-07 | 1310.0ms/step | 15.3 samples/s +Step 43570/44000 | Loss: 2.9230 | LR: 1.22e-07 | 1312.1ms/step | 15.2 samples/s +Step 43580/44000 | Loss: 2.9420 | LR: 1.16e-07 | 1312.6ms/step | 15.2 samples/s +Step 43590/44000 | Loss: 2.9184 | LR: 1.11e-07 | 1314.5ms/step | 15.2 samples/s +Step 43600/44000 | Loss: 2.9314 | LR: 1.06e-07 | 1315.3ms/step | 15.2 samples/s +Step 43610/44000 | Loss: 2.9180 | LR: 1.00e-07 | 1313.3ms/step | 15.2 samples/s +Step 43620/44000 | Loss: 2.9107 | LR: 9.52e-08 | 1314.1ms/step | 15.2 samples/s +Step 43630/44000 | Loss: 2.9608 | LR: 9.03e-08 | 1313.3ms/step | 15.2 samples/s +Step 43640/44000 | Loss: 2.9195 | LR: 8.55e-08 | 1314.3ms/step | 15.2 samples/s +Step 43650/44000 | Loss: 2.9271 | LR: 8.08e-08 | 1315.1ms/step | 15.2 samples/s +Step 43660/44000 | Loss: 2.9066 | LR: 7.62e-08 | 1317.0ms/step | 15.2 samples/s +Step 43670/44000 | Loss: 2.9302 | LR: 7.18e-08 | 1314.5ms/step | 15.2 samples/s +Step 43680/44000 | Loss: 2.9112 | LR: 6.75e-08 | 1316.3ms/step | 15.2 samples/s +Step 43690/44000 | Loss: 2.9380 | LR: 6.34e-08 | 1314.5ms/step | 15.2 samples/s +Step 43700/44000 | Loss: 2.9059 | LR: 5.94e-08 | 1309.6ms/step | 15.3 samples/s +Step 43710/44000 | Loss: 2.9302 | LR: 5.55e-08 | 1311.6ms/step | 15.2 samples/s +Step 43720/44000 | Loss: 2.9456 | LR: 5.17e-08 | 1309.6ms/step | 15.3 samples/s +Step 43730/44000 | Loss: 2.8996 | LR: 4.81e-08 | 1309.4ms/step | 15.3 samples/s +Step 43740/44000 | Loss: 2.9074 | LR: 4.46e-08 | 1310.4ms/step | 15.3 samples/s +Step 43750/44000 | Loss: 2.9343 | LR: 4.12e-08 | 1316.7ms/step | 15.2 samples/s +Validation at step 43750: Loss = 2.9069, Perplexity = 18.30 +Step 43760/44000 | Loss: 2.9240 | LR: 3.80e-08 | 3510.2ms/step | 5.7 samples/s +Step 43770/44000 | Loss: 2.9143 | LR: 3.49e-08 | 1314.3ms/step | 15.2 samples/s +Step 43780/44000 | Loss: 2.9012 | LR: 3.19e-08 | 1313.3ms/step | 15.2 samples/s +Step 43790/44000 | Loss: 2.8890 | LR: 2.91e-08 | 1310.1ms/step | 15.3 samples/s +Step 43800/44000 | Loss: 2.9161 | LR: 2.64e-08 | 1309.1ms/step | 15.3 samples/s +Step 43810/44000 | Loss: 2.9079 | LR: 2.38e-08 | 1311.1ms/step | 15.3 samples/s +Step 43820/44000 | Loss: 2.9119 | LR: 2.14e-08 | 1310.9ms/step | 15.3 samples/s +Step 43830/44000 | Loss: 2.9286 | LR: 1.91e-08 | 1314.2ms/step | 15.2 samples/s +Step 43840/44000 | Loss: 2.9254 | LR: 1.69e-08 | 1315.5ms/step | 15.2 samples/s +Step 43850/44000 | Loss: 2.9334 | LR: 1.48e-08 | 1312.7ms/step | 15.2 samples/s +Step 43860/44000 | Loss: 2.9010 | LR: 1.29e-08 | 1313.3ms/step | 15.2 samples/s +Step 43870/44000 | Loss: 2.9000 | LR: 1.11e-08 | 1310.5ms/step | 15.3 samples/s +Step 43880/44000 | Loss: 2.8907 | LR: 9.50e-09 | 1307.5ms/step | 15.3 samples/s +Step 43890/44000 | Loss: 2.9526 | LR: 7.98e-09 | 1310.6ms/step | 15.3 samples/s +Step 43900/44000 | Loss: 2.8775 | LR: 6.60e-09 | 1316.3ms/step | 15.2 samples/s +Step 43910/44000 | Loss: 2.9185 | LR: 5.34e-09 | 1314.7ms/step | 15.2 samples/s +Step 43920/44000 | Loss: 2.9220 | LR: 4.22e-09 | 1312.7ms/step | 15.2 samples/s +Step 43930/44000 | Loss: 2.9037 | LR: 3.23e-09 | 1310.5ms/step | 15.3 samples/s +Step 43940/44000 | Loss: 2.9391 | LR: 2.37e-09 | 1312.8ms/step | 15.2 samples/s +Step 43950/44000 | Loss: 2.9020 | LR: 1.65e-09 | 1313.0ms/step | 15.2 samples/s +Step 43960/44000 | Loss: 2.9365 | LR: 1.06e-09 | 1315.8ms/step | 15.2 samples/s +Step 43970/44000 | Loss: 2.8784 | LR: 5.94e-10 | 1316.3ms/step | 15.2 samples/s +Step 43980/44000 | Loss: 2.9182 | LR: 2.64e-10 | 1312.6ms/step | 15.2 samples/s +Step 43990/44000 | Loss: 2.9405 | LR: 6.60e-11 | 1311.0ms/step | 15.3 samples/s +Step 44000/44000 | Loss: 2.9203 | LR: 0.00e+00 | 1312.9ms/step | 15.2 samples/s +Validation at step 44000: Loss = 2.9070, Perplexity = 18.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▇▆▅▅▄▄▄▄▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity ███▇▆▅▅▅▅▄▄▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step █▄▂▂▂▄▆▆▅▃▄▃▇▅▃▅▄▆▄▆▅▆▄▄▄▁▆▄▃▆▅▄▅▆▅▄▂▄▂▂ +wandb: speed/samples_per_sec ▂▁▂▅▆▄▅▅▄▆▇▅▆▆▅▅▄▄▅▄▄▅█▃▆▇▅▇▆▅▇▆▄▅▆▇▅▆▇▇ +wandb: speed/steps_per_sec ██████████████▁▁████████████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.90698 +wandb: eval/perplexity 18.30144 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 1312.88652 +wandb: speed/samples_per_sec 15.23361 +wandb: speed/steps_per_sec 0.76168 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 resume 365m resampled 8k nrb35 seed43 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ryta2atg +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_042436-ryta2atg/logs +Training complete! diff --git a/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_static_nrb35_seed42.log b/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_static_nrb35_seed42.log new file mode 100644 index 0000000000000000000000000000000000000000..07dc6db22ccdfcd0844b2a1413971548cdcdfaf7 --- /dev/null +++ b/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_static_nrb35_seed42.log @@ -0,0 +1,2757 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-20000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 resume 365m static 8k nrb35 seed42", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "static", + "nrb35", + "seed42", + "resume", + "h1004", + "step20000" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1", + "sparse_mask_seed": 1, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-20000 +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-20000... +Preserving checkpoint LR state for exact resume: current=[0.00029291518939613315], schedule_base=[0.0005] +Resumed from checkpoint at step 20000 (epoch 0, micro_step 200000) +Resuming training from step 20000 +Resuming wandb run bsc91ke5 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042228-bsc91ke5 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run P1 resume 365m static 8k nrb35 seed42 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/bsc91ke5 +Starting training... +Validation at step 20000: Loss = 3.0139, Perplexity = 20.37 +wandb: WARNING Tried to log to step 20000 that is less than the current step 20071. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 20001: sparse_stochastic (configured sparse attention) +Step 20010/44000 | Loss: 2.9866 | LR: 2.93e-04 | 50292.9ms/step | 0.4 samples/s +wandb: WARNING Tried to log to step 20010 that is less than the current step 20071. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20020/44000 | Loss: 3.0107 | LR: 2.93e-04 | 1292.2ms/step | 15.5 samples/s +wandb: WARNING Tried to log to step 20020 that is less than the current step 20071. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20030/44000 | Loss: 2.9833 | LR: 2.92e-04 | 1298.3ms/step | 15.4 samples/s +wandb: WARNING Tried to log to step 20030 that is less than the current step 20071. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20040/44000 | Loss: 2.9860 | LR: 2.92e-04 | 1299.8ms/step | 15.4 samples/s +wandb: WARNING Tried to log to step 20040 that is less than the current step 20071. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20050/44000 | Loss: 3.0036 | LR: 2.92e-04 | 1296.7ms/step | 15.4 samples/s +wandb: WARNING Tried to log to step 20050 that is less than the current step 20071. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20060/44000 | Loss: 2.9726 | LR: 2.92e-04 | 1298.9ms/step | 15.4 samples/s +wandb: WARNING Tried to log to step 20060 that is less than the current step 20071. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20070/44000 | Loss: 2.9971 | LR: 2.92e-04 | 1294.6ms/step | 15.4 samples/s +wandb: WARNING Tried to log to step 20070 that is less than the current step 20071. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20080/44000 | Loss: 2.9810 | LR: 2.91e-04 | 1295.8ms/step | 15.4 samples/s +Step 20090/44000 | Loss: 2.9875 | LR: 2.91e-04 | 1297.3ms/step | 15.4 samples/s +Step 20100/44000 | Loss: 3.0052 | LR: 2.91e-04 | 1301.2ms/step | 15.4 samples/s +Step 20110/44000 | Loss: 2.9981 | LR: 2.91e-04 | 1300.6ms/step | 15.4 samples/s +Step 20120/44000 | Loss: 3.0077 | LR: 2.91e-04 | 1302.3ms/step | 15.4 samples/s +Step 20130/44000 | Loss: 3.0031 | LR: 2.91e-04 | 1299.1ms/step | 15.4 samples/s +Step 20140/44000 | Loss: 3.0066 | LR: 2.90e-04 | 1299.9ms/step | 15.4 samples/s +Step 20150/44000 | Loss: 2.9926 | LR: 2.90e-04 | 1300.1ms/step | 15.4 samples/s +Step 20160/44000 | Loss: 3.0069 | LR: 2.90e-04 | 1300.0ms/step | 15.4 samples/s +Step 20170/44000 | Loss: 2.9622 | LR: 2.90e-04 | 1300.6ms/step | 15.4 samples/s +Step 20180/44000 | Loss: 2.9857 | LR: 2.90e-04 | 1297.9ms/step | 15.4 samples/s +Step 20190/44000 | Loss: 3.0126 | LR: 2.90e-04 | 1296.2ms/step | 15.4 samples/s +Step 20200/44000 | Loss: 2.9729 | LR: 2.89e-04 | 1295.8ms/step | 15.4 samples/s +Step 20210/44000 | Loss: 2.9640 | LR: 2.89e-04 | 1296.7ms/step | 15.4 samples/s +Step 20220/44000 | Loss: 3.0143 | LR: 2.89e-04 | 1295.1ms/step | 15.4 samples/s +Step 20230/44000 | Loss: 3.0000 | LR: 2.89e-04 | 1299.6ms/step | 15.4 samples/s +Step 20240/44000 | Loss: 3.0023 | LR: 2.89e-04 | 1300.0ms/step | 15.4 samples/s +Step 20250/44000 | Loss: 2.9861 | LR: 2.88e-04 | 1300.0ms/step | 15.4 samples/s +Validation at step 20250: Loss = 3.0112, Perplexity = 20.31 +Step 20260/44000 | Loss: 2.9548 | LR: 2.88e-04 | 3490.6ms/step | 5.7 samples/s +Step 20270/44000 | Loss: 3.0067 | LR: 2.88e-04 | 1300.2ms/step | 15.4 samples/s +Step 20280/44000 | Loss: 3.0028 | LR: 2.88e-04 | 1299.9ms/step | 15.4 samples/s +Step 20290/44000 | Loss: 2.9945 | LR: 2.88e-04 | 1301.0ms/step | 15.4 samples/s +Step 20300/44000 | Loss: 3.0186 | LR: 2.88e-04 | 1302.1ms/step | 15.4 samples/s +Step 20310/44000 | Loss: 2.9857 | LR: 2.87e-04 | 1300.6ms/step | 15.4 samples/s +Step 20320/44000 | Loss: 2.9935 | LR: 2.87e-04 | 1302.4ms/step | 15.4 samples/s +Step 20330/44000 | Loss: 2.9838 | LR: 2.87e-04 | 1303.7ms/step | 15.3 samples/s +Step 20340/44000 | Loss: 2.9730 | LR: 2.87e-04 | 1305.7ms/step | 15.3 samples/s +Step 20350/44000 | Loss: 2.9995 | LR: 2.87e-04 | 1304.6ms/step | 15.3 samples/s +Step 20360/44000 | Loss: 3.0138 | LR: 2.86e-04 | 1301.6ms/step | 15.4 samples/s +Step 20370/44000 | Loss: 2.9656 | LR: 2.86e-04 | 1301.6ms/step | 15.4 samples/s +Step 20380/44000 | Loss: 2.9735 | LR: 2.86e-04 | 1300.8ms/step | 15.4 samples/s +Step 20390/44000 | Loss: 2.9940 | LR: 2.86e-04 | 1297.7ms/step | 15.4 samples/s +Step 20400/44000 | Loss: 3.0114 | LR: 2.86e-04 | 1301.7ms/step | 15.4 samples/s +Step 20410/44000 | Loss: 2.9822 | LR: 2.86e-04 | 1304.8ms/step | 15.3 samples/s +Step 20420/44000 | Loss: 3.0072 | LR: 2.85e-04 | 1301.9ms/step | 15.4 samples/s +Step 20430/44000 | Loss: 2.9717 | LR: 2.85e-04 | 1307.0ms/step | 15.3 samples/s +Step 20440/44000 | Loss: 3.0015 | LR: 2.85e-04 | 1305.3ms/step | 15.3 samples/s +Step 20450/44000 | Loss: 2.9920 | LR: 2.85e-04 | 1304.5ms/step | 15.3 samples/s +Step 20460/44000 | Loss: 2.9940 | LR: 2.85e-04 | 1303.4ms/step | 15.3 samples/s +Step 20470/44000 | Loss: 3.0138 | LR: 2.84e-04 | 1304.4ms/step | 15.3 samples/s +Step 20480/44000 | Loss: 2.9892 | LR: 2.84e-04 | 1304.4ms/step | 15.3 samples/s +Step 20490/44000 | Loss: 2.9906 | LR: 2.84e-04 | 1303.7ms/step | 15.3 samples/s +Step 20500/44000 | Loss: 2.9714 | LR: 2.84e-04 | 1299.9ms/step | 15.4 samples/s +Validation at step 20500: Loss = 3.0082, Perplexity = 20.25 +Step 20510/44000 | Loss: 3.0173 | LR: 2.84e-04 | 3485.8ms/step | 5.7 samples/s +Step 20520/44000 | Loss: 2.9734 | LR: 2.84e-04 | 1299.6ms/step | 15.4 samples/s +Step 20530/44000 | Loss: 2.9660 | LR: 2.83e-04 | 1299.4ms/step | 15.4 samples/s +Step 20540/44000 | Loss: 2.9953 | LR: 2.83e-04 | 1301.2ms/step | 15.4 samples/s +Step 20550/44000 | Loss: 3.0024 | LR: 2.83e-04 | 1300.0ms/step | 15.4 samples/s +Step 20560/44000 | Loss: 3.0073 | LR: 2.83e-04 | 1298.6ms/step | 15.4 samples/s +Step 20570/44000 | Loss: 3.0047 | LR: 2.83e-04 | 1298.2ms/step | 15.4 samples/s +Step 20580/44000 | Loss: 2.9731 | LR: 2.83e-04 | 1299.0ms/step | 15.4 samples/s +Step 20590/44000 | Loss: 3.0140 | LR: 2.82e-04 | 1299.7ms/step | 15.4 samples/s +Step 20600/44000 | Loss: 2.9912 | LR: 2.82e-04 | 1298.1ms/step | 15.4 samples/s +Step 20610/44000 | Loss: 2.9986 | LR: 2.82e-04 | 1297.2ms/step | 15.4 samples/s +Step 20620/44000 | Loss: 2.9923 | LR: 2.82e-04 | 1298.7ms/step | 15.4 samples/s +Step 20630/44000 | Loss: 3.0006 | LR: 2.82e-04 | 1298.3ms/step | 15.4 samples/s +Step 20640/44000 | Loss: 2.9595 | LR: 2.81e-04 | 1298.5ms/step | 15.4 samples/s +Step 20650/44000 | Loss: 3.0067 | LR: 2.81e-04 | 1299.3ms/step | 15.4 samples/s +Step 20660/44000 | Loss: 2.9606 | LR: 2.81e-04 | 1299.0ms/step | 15.4 samples/s +Step 20670/44000 | Loss: 3.0064 | LR: 2.81e-04 | 1298.7ms/step | 15.4 samples/s +Step 20680/44000 | Loss: 2.9958 | LR: 2.81e-04 | 1300.3ms/step | 15.4 samples/s +Step 20690/44000 | Loss: 2.9872 | LR: 2.81e-04 | 1298.9ms/step | 15.4 samples/s +Step 20700/44000 | Loss: 2.9974 | LR: 2.80e-04 | 1298.9ms/step | 15.4 samples/s +Step 20710/44000 | Loss: 3.0013 | LR: 2.80e-04 | 1299.2ms/step | 15.4 samples/s +Step 20720/44000 | Loss: 3.0034 | LR: 2.80e-04 | 1301.1ms/step | 15.4 samples/s +Step 20730/44000 | Loss: 3.0167 | LR: 2.80e-04 | 1301.5ms/step | 15.4 samples/s +Step 20740/44000 | Loss: 3.0001 | LR: 2.80e-04 | 1302.7ms/step | 15.4 samples/s +Step 20750/44000 | Loss: 2.9943 | LR: 2.79e-04 | 1300.8ms/step | 15.4 samples/s +Validation at step 20750: Loss = 3.0054, Perplexity = 20.19 +Step 20760/44000 | Loss: 2.9992 | LR: 2.79e-04 | 3483.9ms/step | 5.7 samples/s +Step 20770/44000 | Loss: 2.9843 | LR: 2.79e-04 | 1302.0ms/step | 15.4 samples/s +Step 20780/44000 | Loss: 2.9603 | LR: 2.79e-04 | 1305.2ms/step | 15.3 samples/s +Step 20790/44000 | Loss: 2.9802 | LR: 2.79e-04 | 1303.5ms/step | 15.3 samples/s +Step 20800/44000 | Loss: 2.9820 | LR: 2.79e-04 | 1302.9ms/step | 15.3 samples/s +Step 20810/44000 | Loss: 3.0011 | LR: 2.78e-04 | 1302.6ms/step | 15.4 samples/s +Step 20820/44000 | Loss: 2.9935 | LR: 2.78e-04 | 1302.2ms/step | 15.4 samples/s +Step 20830/44000 | Loss: 3.0011 | LR: 2.78e-04 | 1305.0ms/step | 15.3 samples/s +Step 20840/44000 | Loss: 2.9878 | LR: 2.78e-04 | 1300.2ms/step | 15.4 samples/s +Step 20850/44000 | Loss: 2.9873 | LR: 2.78e-04 | 1298.1ms/step | 15.4 samples/s +Step 20860/44000 | Loss: 3.0067 | LR: 2.77e-04 | 1300.6ms/step | 15.4 samples/s +Step 20870/44000 | Loss: 3.0134 | LR: 2.77e-04 | 1304.3ms/step | 15.3 samples/s +Step 20880/44000 | Loss: 2.9842 | LR: 2.77e-04 | 1301.6ms/step | 15.4 samples/s +Step 20890/44000 | Loss: 2.9693 | LR: 2.77e-04 | 1301.5ms/step | 15.4 samples/s +Step 20900/44000 | Loss: 2.9727 | LR: 2.77e-04 | 1302.0ms/step | 15.4 samples/s +Step 20910/44000 | Loss: 3.0028 | LR: 2.77e-04 | 1300.3ms/step | 15.4 samples/s +Step 20920/44000 | Loss: 2.9640 | LR: 2.76e-04 | 1300.8ms/step | 15.4 samples/s +Step 20930/44000 | Loss: 2.9729 | LR: 2.76e-04 | 1300.0ms/step | 15.4 samples/s +Step 20940/44000 | Loss: 2.9992 | LR: 2.76e-04 | 1300.5ms/step | 15.4 samples/s +Step 20950/44000 | Loss: 2.9624 | LR: 2.76e-04 | 1303.1ms/step | 15.3 samples/s +Step 20960/44000 | Loss: 2.9782 | LR: 2.76e-04 | 1301.7ms/step | 15.4 samples/s +Step 20970/44000 | Loss: 2.9613 | LR: 2.75e-04 | 1301.3ms/step | 15.4 samples/s +Step 20980/44000 | Loss: 2.9667 | LR: 2.75e-04 | 1301.6ms/step | 15.4 samples/s +Step 20990/44000 | Loss: 2.9727 | LR: 2.75e-04 | 1301.8ms/step | 15.4 samples/s +Step 21000/44000 | Loss: 2.9751 | LR: 2.75e-04 | 1300.3ms/step | 15.4 samples/s +Validation at step 21000: Loss = 3.0043, Perplexity = 20.17 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-21000 +Step 21010/44000 | Loss: 2.9794 | LR: 2.75e-04 | 3801.9ms/step | 5.3 samples/s +Step 21020/44000 | Loss: 2.9829 | LR: 2.75e-04 | 1305.9ms/step | 15.3 samples/s +Step 21030/44000 | Loss: 3.0195 | LR: 2.74e-04 | 1300.2ms/step | 15.4 samples/s +Step 21040/44000 | Loss: 2.9645 | LR: 2.74e-04 | 1302.6ms/step | 15.4 samples/s +Step 21050/44000 | Loss: 2.9802 | LR: 2.74e-04 | 1303.3ms/step | 15.3 samples/s +Step 21060/44000 | Loss: 2.9653 | LR: 2.74e-04 | 1304.9ms/step | 15.3 samples/s +Step 21070/44000 | Loss: 3.0096 | LR: 2.74e-04 | 1302.1ms/step | 15.4 samples/s +Step 21080/44000 | Loss: 2.9527 | LR: 2.73e-04 | 1302.9ms/step | 15.4 samples/s +Step 21090/44000 | Loss: 3.0141 | LR: 2.73e-04 | 1304.2ms/step | 15.3 samples/s +Step 21100/44000 | Loss: 2.9947 | LR: 2.73e-04 | 1304.0ms/step | 15.3 samples/s +Step 21110/44000 | Loss: 2.9841 | LR: 2.73e-04 | 1303.9ms/step | 15.3 samples/s +Step 21120/44000 | Loss: 2.9960 | LR: 2.73e-04 | 1303.0ms/step | 15.3 samples/s +Step 21130/44000 | Loss: 3.0203 | LR: 2.73e-04 | 1305.8ms/step | 15.3 samples/s +Step 21140/44000 | Loss: 2.9804 | LR: 2.72e-04 | 1304.6ms/step | 15.3 samples/s +Step 21150/44000 | Loss: 2.9894 | LR: 2.72e-04 | 1302.0ms/step | 15.4 samples/s +Step 21160/44000 | Loss: 2.9685 | LR: 2.72e-04 | 1300.2ms/step | 15.4 samples/s +Step 21170/44000 | Loss: 2.9864 | LR: 2.72e-04 | 1300.2ms/step | 15.4 samples/s +Step 21180/44000 | Loss: 2.9925 | LR: 2.72e-04 | 1300.9ms/step | 15.4 samples/s +Step 21190/44000 | Loss: 2.9340 | LR: 2.71e-04 | 1303.6ms/step | 15.3 samples/s +Step 21200/44000 | Loss: 3.0042 | LR: 2.71e-04 | 1302.6ms/step | 15.4 samples/s +Step 21210/44000 | Loss: 2.9867 | LR: 2.71e-04 | 1303.2ms/step | 15.3 samples/s +Step 21220/44000 | Loss: 2.9766 | LR: 2.71e-04 | 1302.6ms/step | 15.4 samples/s +Step 21230/44000 | Loss: 2.9916 | LR: 2.71e-04 | 1303.5ms/step | 15.3 samples/s +Step 21240/44000 | Loss: 2.9793 | LR: 2.71e-04 | 1302.8ms/step | 15.4 samples/s +Step 21250/44000 | Loss: 2.9719 | LR: 2.70e-04 | 1299.9ms/step | 15.4 samples/s +Validation at step 21250: Loss = 3.0003, Perplexity = 20.09 +Step 21260/44000 | Loss: 3.0151 | LR: 2.70e-04 | 3480.7ms/step | 5.7 samples/s +Step 21270/44000 | Loss: 2.9931 | LR: 2.70e-04 | 1302.1ms/step | 15.4 samples/s +Step 21280/44000 | Loss: 2.9826 | LR: 2.70e-04 | 1301.6ms/step | 15.4 samples/s +Step 21290/44000 | Loss: 2.9782 | LR: 2.70e-04 | 1303.1ms/step | 15.3 samples/s +Step 21300/44000 | Loss: 2.9876 | LR: 2.70e-04 | 1303.6ms/step | 15.3 samples/s +Step 21310/44000 | Loss: 2.9885 | LR: 2.69e-04 | 1303.3ms/step | 15.3 samples/s +Step 21320/44000 | Loss: 2.9999 | LR: 2.69e-04 | 1303.5ms/step | 15.3 samples/s +Step 21330/44000 | Loss: 2.9844 | LR: 2.69e-04 | 1302.9ms/step | 15.4 samples/s +Step 21340/44000 | Loss: 2.9778 | LR: 2.69e-04 | 1300.3ms/step | 15.4 samples/s +Step 21350/44000 | Loss: 3.0128 | LR: 2.69e-04 | 1302.9ms/step | 15.3 samples/s +Step 21360/44000 | Loss: 2.9829 | LR: 2.68e-04 | 1304.8ms/step | 15.3 samples/s +Step 21370/44000 | Loss: 2.9845 | LR: 2.68e-04 | 1303.3ms/step | 15.3 samples/s +Step 21380/44000 | Loss: 2.9743 | LR: 2.68e-04 | 1304.4ms/step | 15.3 samples/s +Step 21390/44000 | Loss: 3.0036 | LR: 2.68e-04 | 1304.0ms/step | 15.3 samples/s +Step 21400/44000 | Loss: 2.9657 | LR: 2.68e-04 | 1303.0ms/step | 15.3 samples/s +Step 21410/44000 | Loss: 3.0053 | LR: 2.68e-04 | 1303.7ms/step | 15.3 samples/s +Step 21420/44000 | Loss: 2.9543 | LR: 2.67e-04 | 1300.2ms/step | 15.4 samples/s +Step 21430/44000 | Loss: 2.9712 | LR: 2.67e-04 | 1301.1ms/step | 15.4 samples/s +Step 21440/44000 | Loss: 2.9617 | LR: 2.67e-04 | 1301.7ms/step | 15.4 samples/s +Step 21450/44000 | Loss: 2.9796 | LR: 2.67e-04 | 1300.6ms/step | 15.4 samples/s +Step 21460/44000 | Loss: 2.9883 | LR: 2.67e-04 | 1302.3ms/step | 15.4 samples/s +Step 21470/44000 | Loss: 2.9922 | LR: 2.66e-04 | 1302.9ms/step | 15.4 samples/s +Step 21480/44000 | Loss: 2.9730 | LR: 2.66e-04 | 1301.2ms/step | 15.4 samples/s +Step 21490/44000 | Loss: 2.9874 | LR: 2.66e-04 | 1299.1ms/step | 15.4 samples/s +Step 21500/44000 | Loss: 2.9874 | LR: 2.66e-04 | 1304.1ms/step | 15.3 samples/s +Validation at step 21500: Loss = 2.9981, Perplexity = 20.05 +Step 21510/44000 | Loss: 2.9904 | LR: 2.66e-04 | 3492.2ms/step | 5.7 samples/s +Step 21520/44000 | Loss: 2.9784 | LR: 2.66e-04 | 1305.1ms/step | 15.3 samples/s +Step 21530/44000 | Loss: 2.9728 | LR: 2.65e-04 | 1302.4ms/step | 15.4 samples/s +Step 21540/44000 | Loss: 2.9850 | LR: 2.65e-04 | 1301.1ms/step | 15.4 samples/s +Step 21550/44000 | Loss: 2.9857 | LR: 2.65e-04 | 1299.7ms/step | 15.4 samples/s +Step 21560/44000 | Loss: 2.9891 | LR: 2.65e-04 | 1305.7ms/step | 15.3 samples/s +Step 21570/44000 | Loss: 2.9653 | LR: 2.65e-04 | 1302.8ms/step | 15.4 samples/s +Step 21580/44000 | Loss: 2.9607 | LR: 2.64e-04 | 1301.3ms/step | 15.4 samples/s +Step 21590/44000 | Loss: 2.9778 | LR: 2.64e-04 | 1302.5ms/step | 15.4 samples/s +Step 21600/44000 | Loss: 2.9776 | LR: 2.64e-04 | 1303.7ms/step | 15.3 samples/s +Step 21610/44000 | Loss: 2.9737 | LR: 2.64e-04 | 1302.2ms/step | 15.4 samples/s +Step 21620/44000 | Loss: 2.9948 | LR: 2.64e-04 | 1298.7ms/step | 15.4 samples/s +Step 21630/44000 | Loss: 2.9661 | LR: 2.64e-04 | 1299.1ms/step | 15.4 samples/s +Step 21640/44000 | Loss: 2.9574 | LR: 2.63e-04 | 1300.1ms/step | 15.4 samples/s +Step 21650/44000 | Loss: 2.9845 | LR: 2.63e-04 | 1299.8ms/step | 15.4 samples/s +Step 21660/44000 | Loss: 3.0196 | LR: 2.63e-04 | 1303.1ms/step | 15.3 samples/s +Step 21670/44000 | Loss: 2.9654 | LR: 2.63e-04 | 1301.2ms/step | 15.4 samples/s +Step 21680/44000 | Loss: 2.9876 | LR: 2.63e-04 | 1301.6ms/step | 15.4 samples/s +Step 21690/44000 | Loss: 3.0161 | LR: 2.62e-04 | 1303.7ms/step | 15.3 samples/s +Step 21700/44000 | Loss: 2.9972 | LR: 2.62e-04 | 1301.7ms/step | 15.4 samples/s +Step 21710/44000 | Loss: 2.9818 | LR: 2.62e-04 | 1300.8ms/step | 15.4 samples/s +Step 21720/44000 | Loss: 2.9529 | LR: 2.62e-04 | 1300.0ms/step | 15.4 samples/s +Step 21730/44000 | Loss: 2.9794 | LR: 2.62e-04 | 1300.9ms/step | 15.4 samples/s +Step 21740/44000 | Loss: 2.9686 | LR: 2.62e-04 | 1300.2ms/step | 15.4 samples/s +Step 21750/44000 | Loss: 2.9702 | LR: 2.61e-04 | 1300.8ms/step | 15.4 samples/s +Validation at step 21750: Loss = 2.9954, Perplexity = 19.99 +Step 21760/44000 | Loss: 2.9957 | LR: 2.61e-04 | 3484.0ms/step | 5.7 samples/s +Step 21770/44000 | Loss: 2.9778 | LR: 2.61e-04 | 1301.0ms/step | 15.4 samples/s +Step 21780/44000 | Loss: 2.9780 | LR: 2.61e-04 | 1300.0ms/step | 15.4 samples/s +Step 21790/44000 | Loss: 2.9434 | LR: 2.61e-04 | 1300.0ms/step | 15.4 samples/s +Step 21800/44000 | Loss: 2.9697 | LR: 2.60e-04 | 1300.1ms/step | 15.4 samples/s +Step 21810/44000 | Loss: 2.9671 | LR: 2.60e-04 | 1300.9ms/step | 15.4 samples/s +Step 21820/44000 | Loss: 2.9872 | LR: 2.60e-04 | 1300.1ms/step | 15.4 samples/s +Step 21830/44000 | Loss: 2.9579 | LR: 2.60e-04 | 1301.0ms/step | 15.4 samples/s +Step 21840/44000 | Loss: 2.9814 | LR: 2.60e-04 | 1301.6ms/step | 15.4 samples/s +Step 21850/44000 | Loss: 2.9764 | LR: 2.60e-04 | 1303.3ms/step | 15.3 samples/s +Step 21860/44000 | Loss: 2.9820 | LR: 2.59e-04 | 1303.3ms/step | 15.3 samples/s +Step 21870/44000 | Loss: 2.9930 | LR: 2.59e-04 | 1302.2ms/step | 15.4 samples/s +Step 21880/44000 | Loss: 2.9629 | LR: 2.59e-04 | 1302.9ms/step | 15.4 samples/s +Step 21890/44000 | Loss: 2.9654 | LR: 2.59e-04 | 1303.1ms/step | 15.3 samples/s +Step 21900/44000 | Loss: 2.9729 | LR: 2.59e-04 | 1303.0ms/step | 15.3 samples/s +Step 21910/44000 | Loss: 2.9742 | LR: 2.58e-04 | 1302.2ms/step | 15.4 samples/s +Step 21920/44000 | Loss: 2.9796 | LR: 2.58e-04 | 1306.1ms/step | 15.3 samples/s +Step 21930/44000 | Loss: 3.0057 | LR: 2.58e-04 | 1306.1ms/step | 15.3 samples/s +Step 21940/44000 | Loss: 2.9503 | LR: 2.58e-04 | 1301.2ms/step | 15.4 samples/s +Step 21950/44000 | Loss: 2.9803 | LR: 2.58e-04 | 1299.7ms/step | 15.4 samples/s +Step 21960/44000 | Loss: 2.9694 | LR: 2.58e-04 | 1299.8ms/step | 15.4 samples/s +Step 21970/44000 | Loss: 2.9633 | LR: 2.57e-04 | 1301.1ms/step | 15.4 samples/s +Step 21980/44000 | Loss: 2.9842 | LR: 2.57e-04 | 1301.5ms/step | 15.4 samples/s +Step 21990/44000 | Loss: 2.9878 | LR: 2.57e-04 | 1303.5ms/step | 15.3 samples/s +Step 22000/44000 | Loss: 2.9758 | LR: 2.57e-04 | 1303.9ms/step | 15.3 samples/s +Validation at step 22000: Loss = 2.9931, Perplexity = 19.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-22000 +Step 22010/44000 | Loss: 2.9774 | LR: 2.57e-04 | 3807.7ms/step | 5.3 samples/s +Step 22020/44000 | Loss: 2.9896 | LR: 2.56e-04 | 1306.1ms/step | 15.3 samples/s +Step 22030/44000 | Loss: 2.9858 | LR: 2.56e-04 | 1304.5ms/step | 15.3 samples/s +Step 22040/44000 | Loss: 3.0011 | LR: 2.56e-04 | 1303.8ms/step | 15.3 samples/s +Step 22050/44000 | Loss: 2.9711 | LR: 2.56e-04 | 1306.1ms/step | 15.3 samples/s +Step 22060/44000 | Loss: 2.9713 | LR: 2.56e-04 | 1303.7ms/step | 15.3 samples/s +Step 22070/44000 | Loss: 2.9771 | LR: 2.56e-04 | 1305.2ms/step | 15.3 samples/s +Step 22080/44000 | Loss: 2.9680 | LR: 2.55e-04 | 1304.0ms/step | 15.3 samples/s +Step 22090/44000 | Loss: 2.9934 | LR: 2.55e-04 | 1303.0ms/step | 15.3 samples/s +Step 22100/44000 | Loss: 2.9869 | LR: 2.55e-04 | 1308.9ms/step | 15.3 samples/s +Step 22110/44000 | Loss: 2.9732 | LR: 2.55e-04 | 1305.3ms/step | 15.3 samples/s +Step 22120/44000 | Loss: 2.9529 | LR: 2.55e-04 | 1308.7ms/step | 15.3 samples/s +Step 22130/44000 | Loss: 2.9641 | LR: 2.54e-04 | 1306.8ms/step | 15.3 samples/s +Step 22140/44000 | Loss: 2.9757 | LR: 2.54e-04 | 1310.9ms/step | 15.3 samples/s +Step 22150/44000 | Loss: 2.9539 | LR: 2.54e-04 | 1306.9ms/step | 15.3 samples/s +Step 22160/44000 | Loss: 2.9765 | LR: 2.54e-04 | 1308.6ms/step | 15.3 samples/s +Step 22170/44000 | Loss: 2.9939 | LR: 2.54e-04 | 1306.4ms/step | 15.3 samples/s +Step 22180/44000 | Loss: 2.9767 | LR: 2.54e-04 | 1306.1ms/step | 15.3 samples/s +Step 22190/44000 | Loss: 2.9743 | LR: 2.53e-04 | 1302.5ms/step | 15.4 samples/s +Step 22200/44000 | Loss: 2.9731 | LR: 2.53e-04 | 1300.8ms/step | 15.4 samples/s +Step 22210/44000 | Loss: 2.9773 | LR: 2.53e-04 | 1303.2ms/step | 15.3 samples/s +Step 22220/44000 | Loss: 2.9743 | LR: 2.53e-04 | 1303.8ms/step | 15.3 samples/s +Step 22230/44000 | Loss: 2.9914 | LR: 2.53e-04 | 1304.8ms/step | 15.3 samples/s +Step 22240/44000 | Loss: 3.0036 | LR: 2.52e-04 | 1303.2ms/step | 15.3 samples/s +Step 22250/44000 | Loss: 2.9905 | LR: 2.52e-04 | 1304.9ms/step | 15.3 samples/s +Validation at step 22250: Loss = 2.9907, Perplexity = 19.90 +Step 22260/44000 | Loss: 2.9570 | LR: 2.52e-04 | 3486.8ms/step | 5.7 samples/s +Step 22270/44000 | Loss: 2.9900 | LR: 2.52e-04 | 1307.7ms/step | 15.3 samples/s +Step 22280/44000 | Loss: 2.9713 | LR: 2.52e-04 | 1307.6ms/step | 15.3 samples/s +Step 22290/44000 | Loss: 2.9711 | LR: 2.52e-04 | 1305.9ms/step | 15.3 samples/s +Step 22300/44000 | Loss: 3.0074 | LR: 2.51e-04 | 1305.5ms/step | 15.3 samples/s +Step 22310/44000 | Loss: 2.9682 | LR: 2.51e-04 | 1308.0ms/step | 15.3 samples/s +Step 22320/44000 | Loss: 2.9524 | LR: 2.51e-04 | 1306.4ms/step | 15.3 samples/s +Step 22330/44000 | Loss: 2.9549 | LR: 2.51e-04 | 1309.1ms/step | 15.3 samples/s +Step 22340/44000 | Loss: 3.0029 | LR: 2.51e-04 | 1306.1ms/step | 15.3 samples/s +Step 22350/44000 | Loss: 2.9664 | LR: 2.50e-04 | 1303.4ms/step | 15.3 samples/s +Step 22360/44000 | Loss: 2.9664 | LR: 2.50e-04 | 1303.8ms/step | 15.3 samples/s +Step 22370/44000 | Loss: 2.9714 | LR: 2.50e-04 | 1303.6ms/step | 15.3 samples/s +Step 22380/44000 | Loss: 2.9737 | LR: 2.50e-04 | 1305.3ms/step | 15.3 samples/s +Step 22390/44000 | Loss: 2.9622 | LR: 2.50e-04 | 1302.8ms/step | 15.4 samples/s +Step 22400/44000 | Loss: 2.9469 | LR: 2.50e-04 | 1303.0ms/step | 15.3 samples/s +Step 22410/44000 | Loss: 2.9651 | LR: 2.49e-04 | 1305.0ms/step | 15.3 samples/s +Step 22420/44000 | Loss: 2.9669 | LR: 2.49e-04 | 1307.0ms/step | 15.3 samples/s +Step 22430/44000 | Loss: 2.9825 | LR: 2.49e-04 | 1305.9ms/step | 15.3 samples/s +Step 22440/44000 | Loss: 2.9706 | LR: 2.49e-04 | 1304.0ms/step | 15.3 samples/s +Step 22450/44000 | Loss: 2.9869 | LR: 2.49e-04 | 1302.9ms/step | 15.4 samples/s +Step 22460/44000 | Loss: 2.9643 | LR: 2.48e-04 | 1305.1ms/step | 15.3 samples/s +Step 22470/44000 | Loss: 2.9411 | LR: 2.48e-04 | 1305.6ms/step | 15.3 samples/s +Step 22480/44000 | Loss: 2.9541 | LR: 2.48e-04 | 1305.3ms/step | 15.3 samples/s +Step 22490/44000 | Loss: 2.9612 | LR: 2.48e-04 | 1304.8ms/step | 15.3 samples/s +Step 22500/44000 | Loss: 2.9457 | LR: 2.48e-04 | 1306.3ms/step | 15.3 samples/s +Validation at step 22500: Loss = 2.9877, Perplexity = 19.84 +Step 22510/44000 | Loss: 2.9562 | LR: 2.48e-04 | 3491.1ms/step | 5.7 samples/s +Step 22520/44000 | Loss: 3.0015 | LR: 2.47e-04 | 1304.5ms/step | 15.3 samples/s +Step 22530/44000 | Loss: 2.9404 | LR: 2.47e-04 | 1303.4ms/step | 15.3 samples/s +Step 22540/44000 | Loss: 2.9736 | LR: 2.47e-04 | 1304.9ms/step | 15.3 samples/s +Step 22550/44000 | Loss: 2.9413 | LR: 2.47e-04 | 1303.4ms/step | 15.3 samples/s +Step 22560/44000 | Loss: 2.9693 | LR: 2.47e-04 | 1307.6ms/step | 15.3 samples/s +Step 22570/44000 | Loss: 2.9446 | LR: 2.46e-04 | 1305.1ms/step | 15.3 samples/s +Step 22580/44000 | Loss: 2.9835 | LR: 2.46e-04 | 1303.2ms/step | 15.3 samples/s +Step 22590/44000 | Loss: 2.9783 | LR: 2.46e-04 | 1305.5ms/step | 15.3 samples/s +Step 22600/44000 | Loss: 2.9513 | LR: 2.46e-04 | 1302.5ms/step | 15.4 samples/s +Step 22610/44000 | Loss: 2.9675 | LR: 2.46e-04 | 1304.0ms/step | 15.3 samples/s +Step 22620/44000 | Loss: 2.9842 | LR: 2.46e-04 | 1304.1ms/step | 15.3 samples/s +Step 22630/44000 | Loss: 2.9648 | LR: 2.45e-04 | 1307.1ms/step | 15.3 samples/s +Step 22640/44000 | Loss: 2.9775 | LR: 2.45e-04 | 1307.3ms/step | 15.3 samples/s +Step 22650/44000 | Loss: 2.9662 | LR: 2.45e-04 | 1305.1ms/step | 15.3 samples/s +Step 22660/44000 | Loss: 3.0002 | LR: 2.45e-04 | 1303.9ms/step | 15.3 samples/s +Step 22670/44000 | Loss: 2.9752 | LR: 2.45e-04 | 1306.0ms/step | 15.3 samples/s +Step 22680/44000 | Loss: 2.9524 | LR: 2.44e-04 | 1307.1ms/step | 15.3 samples/s +Step 22690/44000 | Loss: 2.9820 | LR: 2.44e-04 | 1304.6ms/step | 15.3 samples/s +Step 22700/44000 | Loss: 2.9857 | LR: 2.44e-04 | 1304.4ms/step | 15.3 samples/s +Step 22710/44000 | Loss: 2.9440 | LR: 2.44e-04 | 1306.8ms/step | 15.3 samples/s +Step 22720/44000 | Loss: 2.9697 | LR: 2.44e-04 | 1306.1ms/step | 15.3 samples/s +Step 22730/44000 | Loss: 2.9856 | LR: 2.44e-04 | 1307.1ms/step | 15.3 samples/s +Step 22740/44000 | Loss: 2.9723 | LR: 2.43e-04 | 1306.8ms/step | 15.3 samples/s +Step 22750/44000 | Loss: 2.9489 | LR: 2.43e-04 | 1307.1ms/step | 15.3 samples/s +Validation at step 22750: Loss = 2.9858, Perplexity = 19.80 +Step 22760/44000 | Loss: 2.9820 | LR: 2.43e-04 | 3484.8ms/step | 5.7 samples/s +Step 22770/44000 | Loss: 2.9657 | LR: 2.43e-04 | 1304.0ms/step | 15.3 samples/s +Step 22780/44000 | Loss: 2.9775 | LR: 2.43e-04 | 1304.7ms/step | 15.3 samples/s +Step 22790/44000 | Loss: 2.9899 | LR: 2.42e-04 | 1306.5ms/step | 15.3 samples/s +Step 22800/44000 | Loss: 2.9908 | LR: 2.42e-04 | 1305.1ms/step | 15.3 samples/s +Step 22810/44000 | Loss: 2.9575 | LR: 2.42e-04 | 1305.9ms/step | 15.3 samples/s +Step 22820/44000 | Loss: 2.9707 | LR: 2.42e-04 | 1307.2ms/step | 15.3 samples/s +Step 22830/44000 | Loss: 2.9968 | LR: 2.42e-04 | 1302.5ms/step | 15.4 samples/s +Step 22840/44000 | Loss: 2.9801 | LR: 2.42e-04 | 1307.0ms/step | 15.3 samples/s +Step 22850/44000 | Loss: 2.9516 | LR: 2.41e-04 | 1306.2ms/step | 15.3 samples/s +Step 22860/44000 | Loss: 2.9911 | LR: 2.41e-04 | 1308.2ms/step | 15.3 samples/s +Step 22870/44000 | Loss: 2.9622 | LR: 2.41e-04 | 1304.1ms/step | 15.3 samples/s +Step 22880/44000 | Loss: 2.9643 | LR: 2.41e-04 | 1305.2ms/step | 15.3 samples/s +Step 22890/44000 | Loss: 2.9537 | LR: 2.41e-04 | 1304.4ms/step | 15.3 samples/s +Step 22900/44000 | Loss: 2.9980 | LR: 2.40e-04 | 1302.4ms/step | 15.4 samples/s +Step 22910/44000 | Loss: 2.9828 | LR: 2.40e-04 | 1304.7ms/step | 15.3 samples/s +Step 22920/44000 | Loss: 2.9718 | LR: 2.40e-04 | 1305.1ms/step | 15.3 samples/s +Step 22930/44000 | Loss: 2.9540 | LR: 2.40e-04 | 1303.3ms/step | 15.3 samples/s +Step 22940/44000 | Loss: 2.9554 | LR: 2.40e-04 | 1306.3ms/step | 15.3 samples/s +Step 22950/44000 | Loss: 2.9629 | LR: 2.40e-04 | 1304.1ms/step | 15.3 samples/s +Step 22960/44000 | Loss: 3.0035 | LR: 2.39e-04 | 1305.9ms/step | 15.3 samples/s +Step 22970/44000 | Loss: 2.9523 | LR: 2.39e-04 | 1305.9ms/step | 15.3 samples/s +Step 22980/44000 | Loss: 2.9667 | LR: 2.39e-04 | 1307.4ms/step | 15.3 samples/s +Step 22990/44000 | Loss: 2.9570 | LR: 2.39e-04 | 1306.7ms/step | 15.3 samples/s +Step 23000/44000 | Loss: 2.9522 | LR: 2.39e-04 | 1309.4ms/step | 15.3 samples/s +Validation at step 23000: Loss = 2.9834, Perplexity = 19.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-23000 +Step 23010/44000 | Loss: 2.9568 | LR: 2.38e-04 | 3803.6ms/step | 5.3 samples/s +Step 23020/44000 | Loss: 2.9678 | LR: 2.38e-04 | 1302.9ms/step | 15.3 samples/s +Step 23030/44000 | Loss: 2.9722 | LR: 2.38e-04 | 1302.1ms/step | 15.4 samples/s +Step 23040/44000 | Loss: 2.9867 | LR: 2.38e-04 | 1303.1ms/step | 15.3 samples/s +Step 23050/44000 | Loss: 3.0068 | LR: 2.38e-04 | 1302.0ms/step | 15.4 samples/s +Step 23060/44000 | Loss: 2.9696 | LR: 2.38e-04 | 1302.9ms/step | 15.3 samples/s +Step 23070/44000 | Loss: 2.9648 | LR: 2.37e-04 | 1305.8ms/step | 15.3 samples/s +Step 23080/44000 | Loss: 2.9687 | LR: 2.37e-04 | 1301.9ms/step | 15.4 samples/s +Step 23090/44000 | Loss: 2.9794 | LR: 2.37e-04 | 1302.3ms/step | 15.4 samples/s +Step 23100/44000 | Loss: 2.9674 | LR: 2.37e-04 | 1303.2ms/step | 15.3 samples/s +Step 23110/44000 | Loss: 2.9534 | LR: 2.37e-04 | 1302.7ms/step | 15.4 samples/s +Step 23120/44000 | Loss: 2.9612 | LR: 2.36e-04 | 1303.2ms/step | 15.3 samples/s +Step 23130/44000 | Loss: 2.9607 | LR: 2.36e-04 | 1303.9ms/step | 15.3 samples/s +Step 23140/44000 | Loss: 2.9423 | LR: 2.36e-04 | 1302.8ms/step | 15.4 samples/s +Step 23150/44000 | Loss: 2.9481 | LR: 2.36e-04 | 1303.1ms/step | 15.3 samples/s +Step 23160/44000 | Loss: 2.9738 | LR: 2.36e-04 | 1303.5ms/step | 15.3 samples/s +Step 23170/44000 | Loss: 2.9583 | LR: 2.36e-04 | 1302.6ms/step | 15.4 samples/s +Step 23180/44000 | Loss: 2.9739 | LR: 2.35e-04 | 1301.8ms/step | 15.4 samples/s +Step 23190/44000 | Loss: 2.9682 | LR: 2.35e-04 | 1303.4ms/step | 15.3 samples/s +Step 23200/44000 | Loss: 2.9681 | LR: 2.35e-04 | 1303.2ms/step | 15.3 samples/s +Step 23210/44000 | Loss: 2.9525 | LR: 2.35e-04 | 1304.3ms/step | 15.3 samples/s +Step 23220/44000 | Loss: 2.9718 | LR: 2.35e-04 | 1304.0ms/step | 15.3 samples/s +Step 23230/44000 | Loss: 2.9473 | LR: 2.34e-04 | 1301.3ms/step | 15.4 samples/s +Step 23240/44000 | Loss: 2.9706 | LR: 2.34e-04 | 1303.9ms/step | 15.3 samples/s +Step 23250/44000 | Loss: 2.9722 | LR: 2.34e-04 | 1302.9ms/step | 15.4 samples/s +Validation at step 23250: Loss = 2.9812, Perplexity = 19.71 +Step 23260/44000 | Loss: 2.9446 | LR: 2.34e-04 | 3485.1ms/step | 5.7 samples/s +Step 23270/44000 | Loss: 2.9655 | LR: 2.34e-04 | 1304.0ms/step | 15.3 samples/s +Step 23280/44000 | Loss: 2.9804 | LR: 2.34e-04 | 1302.7ms/step | 15.4 samples/s +Step 23290/44000 | Loss: 2.9588 | LR: 2.33e-04 | 1307.2ms/step | 15.3 samples/s +Step 23300/44000 | Loss: 2.9517 | LR: 2.33e-04 | 1304.8ms/step | 15.3 samples/s +Step 23310/44000 | Loss: 2.9310 | LR: 2.33e-04 | 1304.0ms/step | 15.3 samples/s +Step 23320/44000 | Loss: 2.9576 | LR: 2.33e-04 | 1304.0ms/step | 15.3 samples/s +Step 23330/44000 | Loss: 2.9485 | LR: 2.33e-04 | 1302.5ms/step | 15.4 samples/s +Step 23340/44000 | Loss: 2.9757 | LR: 2.32e-04 | 1303.2ms/step | 15.3 samples/s +Step 23350/44000 | Loss: 2.9594 | LR: 2.32e-04 | 1304.0ms/step | 15.3 samples/s +Step 23360/44000 | Loss: 3.0031 | LR: 2.32e-04 | 1302.5ms/step | 15.4 samples/s +Step 23370/44000 | Loss: 2.9534 | LR: 2.32e-04 | 1303.6ms/step | 15.3 samples/s +Step 23380/44000 | Loss: 2.9469 | LR: 2.32e-04 | 1301.8ms/step | 15.4 samples/s +Step 23390/44000 | Loss: 2.9332 | LR: 2.32e-04 | 1303.3ms/step | 15.3 samples/s +Step 23400/44000 | Loss: 2.9717 | LR: 2.31e-04 | 1304.0ms/step | 15.3 samples/s +Step 23410/44000 | Loss: 2.9600 | LR: 2.31e-04 | 1302.0ms/step | 15.4 samples/s +Step 23420/44000 | Loss: 2.9807 | LR: 2.31e-04 | 1303.4ms/step | 15.3 samples/s +Step 23430/44000 | Loss: 2.9503 | LR: 2.31e-04 | 1304.7ms/step | 15.3 samples/s +Step 23440/44000 | Loss: 2.9497 | LR: 2.31e-04 | 1305.9ms/step | 15.3 samples/s +Step 23450/44000 | Loss: 2.9688 | LR: 2.30e-04 | 1305.9ms/step | 15.3 samples/s +Step 23460/44000 | Loss: 2.9527 | LR: 2.30e-04 | 1308.2ms/step | 15.3 samples/s +Step 23470/44000 | Loss: 2.9729 | LR: 2.30e-04 | 1304.6ms/step | 15.3 samples/s +Step 23480/44000 | Loss: 2.9509 | LR: 2.30e-04 | 1303.6ms/step | 15.3 samples/s +Step 23490/44000 | Loss: 2.9665 | LR: 2.30e-04 | 1302.6ms/step | 15.4 samples/s +Step 23500/44000 | Loss: 2.9510 | LR: 2.30e-04 | 1304.4ms/step | 15.3 samples/s +Validation at step 23500: Loss = 2.9792, Perplexity = 19.67 +Step 23510/44000 | Loss: 2.9669 | LR: 2.29e-04 | 3486.7ms/step | 5.7 samples/s +Step 23520/44000 | Loss: 2.9577 | LR: 2.29e-04 | 1306.7ms/step | 15.3 samples/s +Step 23530/44000 | Loss: 2.9567 | LR: 2.29e-04 | 1307.4ms/step | 15.3 samples/s +Step 23540/44000 | Loss: 2.9408 | LR: 2.29e-04 | 1304.8ms/step | 15.3 samples/s +Step 23550/44000 | Loss: 2.9403 | LR: 2.29e-04 | 1308.7ms/step | 15.3 samples/s +Step 23560/44000 | Loss: 2.9588 | LR: 2.29e-04 | 1303.0ms/step | 15.3 samples/s +Step 23570/44000 | Loss: 2.9837 | LR: 2.28e-04 | 1304.0ms/step | 15.3 samples/s +Step 23580/44000 | Loss: 2.9343 | LR: 2.28e-04 | 1303.3ms/step | 15.3 samples/s +Step 23590/44000 | Loss: 2.9764 | LR: 2.28e-04 | 1301.6ms/step | 15.4 samples/s +Step 23600/44000 | Loss: 2.9905 | LR: 2.28e-04 | 1303.4ms/step | 15.3 samples/s +Step 23610/44000 | Loss: 2.9509 | LR: 2.28e-04 | 1299.7ms/step | 15.4 samples/s +Step 23620/44000 | Loss: 2.9569 | LR: 2.27e-04 | 1303.0ms/step | 15.3 samples/s +Step 23630/44000 | Loss: 2.9602 | LR: 2.27e-04 | 1305.2ms/step | 15.3 samples/s +Step 23640/44000 | Loss: 2.9757 | LR: 2.27e-04 | 1304.0ms/step | 15.3 samples/s +Step 23650/44000 | Loss: 2.9848 | LR: 2.27e-04 | 1303.9ms/step | 15.3 samples/s +Step 23660/44000 | Loss: 2.9623 | LR: 2.27e-04 | 1305.2ms/step | 15.3 samples/s +Step 23670/44000 | Loss: 2.9719 | LR: 2.27e-04 | 1306.1ms/step | 15.3 samples/s +Step 23680/44000 | Loss: 2.9693 | LR: 2.26e-04 | 1307.8ms/step | 15.3 samples/s +Step 23690/44000 | Loss: 2.9747 | LR: 2.26e-04 | 1305.8ms/step | 15.3 samples/s +Step 23700/44000 | Loss: 2.9568 | LR: 2.26e-04 | 1305.3ms/step | 15.3 samples/s +Step 23710/44000 | Loss: 2.9405 | LR: 2.26e-04 | 1306.6ms/step | 15.3 samples/s +Step 23720/44000 | Loss: 2.9466 | LR: 2.26e-04 | 1307.0ms/step | 15.3 samples/s +Step 23730/44000 | Loss: 2.9653 | LR: 2.25e-04 | 1307.5ms/step | 15.3 samples/s +Step 23740/44000 | Loss: 2.9636 | LR: 2.25e-04 | 1305.2ms/step | 15.3 samples/s +Step 23750/44000 | Loss: 2.9476 | LR: 2.25e-04 | 1307.0ms/step | 15.3 samples/s +Validation at step 23750: Loss = 2.9766, Perplexity = 19.62 +Step 23760/44000 | Loss: 2.9739 | LR: 2.25e-04 | 3492.5ms/step | 5.7 samples/s +Step 23770/44000 | Loss: 2.9481 | LR: 2.25e-04 | 1306.7ms/step | 15.3 samples/s +Step 23780/44000 | Loss: 2.9457 | LR: 2.25e-04 | 1306.8ms/step | 15.3 samples/s +Step 23790/44000 | Loss: 2.9420 | LR: 2.24e-04 | 1306.5ms/step | 15.3 samples/s +Step 23800/44000 | Loss: 2.9662 | LR: 2.24e-04 | 1307.2ms/step | 15.3 samples/s +Step 23810/44000 | Loss: 2.9410 | LR: 2.24e-04 | 1304.8ms/step | 15.3 samples/s +Step 23820/44000 | Loss: 2.9853 | LR: 2.24e-04 | 1305.6ms/step | 15.3 samples/s +Step 23830/44000 | Loss: 2.9524 | LR: 2.24e-04 | 1304.5ms/step | 15.3 samples/s +Step 23840/44000 | Loss: 2.9656 | LR: 2.23e-04 | 1305.6ms/step | 15.3 samples/s +Step 23850/44000 | Loss: 2.9700 | LR: 2.23e-04 | 1304.4ms/step | 15.3 samples/s +Step 23860/44000 | Loss: 2.9586 | LR: 2.23e-04 | 1303.8ms/step | 15.3 samples/s +Step 23870/44000 | Loss: 2.9707 | LR: 2.23e-04 | 1304.2ms/step | 15.3 samples/s +Step 23880/44000 | Loss: 2.9544 | LR: 2.23e-04 | 1306.9ms/step | 15.3 samples/s +Step 23890/44000 | Loss: 2.9536 | LR: 2.23e-04 | 1304.7ms/step | 15.3 samples/s +Step 23900/44000 | Loss: 2.9519 | LR: 2.22e-04 | 1305.4ms/step | 15.3 samples/s +Step 23910/44000 | Loss: 2.9485 | LR: 2.22e-04 | 1307.1ms/step | 15.3 samples/s +Step 23920/44000 | Loss: 2.9905 | LR: 2.22e-04 | 1305.5ms/step | 15.3 samples/s +Step 23930/44000 | Loss: 2.9659 | LR: 2.22e-04 | 1304.4ms/step | 15.3 samples/s +Step 23940/44000 | Loss: 2.9778 | LR: 2.22e-04 | 1303.3ms/step | 15.3 samples/s +Step 23950/44000 | Loss: 2.9706 | LR: 2.21e-04 | 1303.5ms/step | 15.3 samples/s +Step 23960/44000 | Loss: 2.9939 | LR: 2.21e-04 | 1309.2ms/step | 15.3 samples/s +Step 23970/44000 | Loss: 2.9716 | LR: 2.21e-04 | 1306.9ms/step | 15.3 samples/s +Step 23980/44000 | Loss: 2.9544 | LR: 2.21e-04 | 1309.2ms/step | 15.3 samples/s +Step 23990/44000 | Loss: 2.9313 | LR: 2.21e-04 | 1307.1ms/step | 15.3 samples/s +Step 24000/44000 | Loss: 2.9523 | LR: 2.21e-04 | 1305.7ms/step | 15.3 samples/s +Validation at step 24000: Loss = 2.9747, Perplexity = 19.58 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-24000 +Step 24010/44000 | Loss: 2.9632 | LR: 2.20e-04 | 3797.7ms/step | 5.3 samples/s +Step 24020/44000 | Loss: 2.9588 | LR: 2.20e-04 | 1303.0ms/step | 15.3 samples/s +Step 24030/44000 | Loss: 2.9806 | LR: 2.20e-04 | 1302.1ms/step | 15.4 samples/s +Step 24040/44000 | Loss: 2.9934 | LR: 2.20e-04 | 1302.5ms/step | 15.4 samples/s +Step 24050/44000 | Loss: 2.9853 | LR: 2.20e-04 | 1303.4ms/step | 15.3 samples/s +Step 24060/44000 | Loss: 2.9679 | LR: 2.19e-04 | 1303.1ms/step | 15.3 samples/s +Step 24070/44000 | Loss: 2.9848 | LR: 2.19e-04 | 1303.0ms/step | 15.3 samples/s +Step 24080/44000 | Loss: 2.9439 | LR: 2.19e-04 | 1302.8ms/step | 15.4 samples/s +Step 24090/44000 | Loss: 2.9444 | LR: 2.19e-04 | 1302.6ms/step | 15.4 samples/s +Step 24100/44000 | Loss: 2.9626 | LR: 2.19e-04 | 1300.4ms/step | 15.4 samples/s +Step 24110/44000 | Loss: 2.9719 | LR: 2.19e-04 | 1299.8ms/step | 15.4 samples/s +Step 24120/44000 | Loss: 2.9263 | LR: 2.18e-04 | 1301.1ms/step | 15.4 samples/s +Step 24130/44000 | Loss: 2.9749 | LR: 2.18e-04 | 1300.0ms/step | 15.4 samples/s +Step 24140/44000 | Loss: 2.9502 | LR: 2.18e-04 | 1299.8ms/step | 15.4 samples/s +Step 24150/44000 | Loss: 2.9774 | LR: 2.18e-04 | 1303.2ms/step | 15.3 samples/s +Step 24160/44000 | Loss: 2.9870 | LR: 2.18e-04 | 1303.6ms/step | 15.3 samples/s +Step 24170/44000 | Loss: 2.9441 | LR: 2.17e-04 | 1302.6ms/step | 15.4 samples/s +Step 24180/44000 | Loss: 2.9524 | LR: 2.17e-04 | 1302.6ms/step | 15.4 samples/s +Step 24190/44000 | Loss: 2.9738 | LR: 2.17e-04 | 1302.1ms/step | 15.4 samples/s +Step 24200/44000 | Loss: 2.9472 | LR: 2.17e-04 | 1300.1ms/step | 15.4 samples/s +Step 24210/44000 | Loss: 2.9387 | LR: 2.17e-04 | 1302.2ms/step | 15.4 samples/s +Step 24220/44000 | Loss: 2.9569 | LR: 2.17e-04 | 1302.3ms/step | 15.4 samples/s +Step 24230/44000 | Loss: 2.9810 | LR: 2.16e-04 | 1303.4ms/step | 15.3 samples/s +Step 24240/44000 | Loss: 2.9402 | LR: 2.16e-04 | 1302.6ms/step | 15.4 samples/s +Step 24250/44000 | Loss: 2.9659 | LR: 2.16e-04 | 1302.2ms/step | 15.4 samples/s +Validation at step 24250: Loss = 2.9735, Perplexity = 19.56 +Step 24260/44000 | Loss: 2.9360 | LR: 2.16e-04 | 3488.0ms/step | 5.7 samples/s +Step 24270/44000 | Loss: 2.9555 | LR: 2.16e-04 | 1302.9ms/step | 15.4 samples/s +Step 24280/44000 | Loss: 2.9810 | LR: 2.16e-04 | 1302.4ms/step | 15.4 samples/s +Step 24290/44000 | Loss: 2.9635 | LR: 2.15e-04 | 1299.8ms/step | 15.4 samples/s +Step 24300/44000 | Loss: 2.9474 | LR: 2.15e-04 | 1304.1ms/step | 15.3 samples/s +Step 24310/44000 | Loss: 2.9599 | LR: 2.15e-04 | 1300.3ms/step | 15.4 samples/s +Step 24320/44000 | Loss: 2.9594 | LR: 2.15e-04 | 1303.7ms/step | 15.3 samples/s +Step 24330/44000 | Loss: 2.9788 | LR: 2.15e-04 | 1307.9ms/step | 15.3 samples/s +Step 24340/44000 | Loss: 2.9416 | LR: 2.14e-04 | 1303.8ms/step | 15.3 samples/s +Step 24350/44000 | Loss: 2.9339 | LR: 2.14e-04 | 1300.0ms/step | 15.4 samples/s +Step 24360/44000 | Loss: 2.9583 | LR: 2.14e-04 | 1300.7ms/step | 15.4 samples/s +Step 24370/44000 | Loss: 2.9282 | LR: 2.14e-04 | 1299.8ms/step | 15.4 samples/s +Step 24380/44000 | Loss: 2.9214 | LR: 2.14e-04 | 1300.8ms/step | 15.4 samples/s +Step 24390/44000 | Loss: 2.9299 | LR: 2.14e-04 | 1303.7ms/step | 15.3 samples/s +Step 24400/44000 | Loss: 2.9598 | LR: 2.13e-04 | 1302.4ms/step | 15.4 samples/s +Step 24410/44000 | Loss: 2.9578 | LR: 2.13e-04 | 1303.8ms/step | 15.3 samples/s +Step 24420/44000 | Loss: 2.9722 | LR: 2.13e-04 | 1302.9ms/step | 15.4 samples/s +Step 24430/44000 | Loss: 2.9581 | LR: 2.13e-04 | 1301.9ms/step | 15.4 samples/s +Step 24440/44000 | Loss: 2.9635 | LR: 2.13e-04 | 1300.7ms/step | 15.4 samples/s +Step 24450/44000 | Loss: 2.9307 | LR: 2.12e-04 | 1303.6ms/step | 15.3 samples/s +Step 24460/44000 | Loss: 2.9367 | LR: 2.12e-04 | 1301.8ms/step | 15.4 samples/s +Step 24470/44000 | Loss: 2.9626 | LR: 2.12e-04 | 1303.9ms/step | 15.3 samples/s +Step 24480/44000 | Loss: 2.9581 | LR: 2.12e-04 | 1302.9ms/step | 15.4 samples/s +Step 24490/44000 | Loss: 2.9537 | LR: 2.12e-04 | 1302.0ms/step | 15.4 samples/s +Step 24500/44000 | Loss: 2.9694 | LR: 2.12e-04 | 1302.2ms/step | 15.4 samples/s +Validation at step 24500: Loss = 2.9712, Perplexity = 19.51 +Step 24510/44000 | Loss: 2.9586 | LR: 2.11e-04 | 3484.9ms/step | 5.7 samples/s +Step 24520/44000 | Loss: 2.9268 | LR: 2.11e-04 | 1300.6ms/step | 15.4 samples/s +Step 24530/44000 | Loss: 2.9521 | LR: 2.11e-04 | 1300.8ms/step | 15.4 samples/s +Step 24540/44000 | Loss: 2.9733 | LR: 2.11e-04 | 1296.5ms/step | 15.4 samples/s +Step 24550/44000 | Loss: 2.9850 | LR: 2.11e-04 | 1298.7ms/step | 15.4 samples/s +Step 24560/44000 | Loss: 2.9509 | LR: 2.10e-04 | 1296.7ms/step | 15.4 samples/s +Step 24570/44000 | Loss: 3.0188 | LR: 2.10e-04 | 1296.8ms/step | 15.4 samples/s +Step 24580/44000 | Loss: 2.9348 | LR: 2.10e-04 | 1297.5ms/step | 15.4 samples/s +Step 24590/44000 | Loss: 2.9824 | LR: 2.10e-04 | 1298.1ms/step | 15.4 samples/s +Step 24600/44000 | Loss: 2.9536 | LR: 2.10e-04 | 1299.1ms/step | 15.4 samples/s +Step 24610/44000 | Loss: 2.9690 | LR: 2.10e-04 | 1297.0ms/step | 15.4 samples/s +Step 24620/44000 | Loss: 2.9647 | LR: 2.09e-04 | 1299.3ms/step | 15.4 samples/s +Step 24630/44000 | Loss: 2.9842 | LR: 2.09e-04 | 1299.1ms/step | 15.4 samples/s +Step 24640/44000 | Loss: 2.9555 | LR: 2.09e-04 | 1302.4ms/step | 15.4 samples/s +Step 24650/44000 | Loss: 2.9569 | LR: 2.09e-04 | 1301.1ms/step | 15.4 samples/s +Step 24660/44000 | Loss: 2.9471 | LR: 2.09e-04 | 1305.2ms/step | 15.3 samples/s +Step 24670/44000 | Loss: 2.9502 | LR: 2.09e-04 | 1302.5ms/step | 15.4 samples/s +Step 24680/44000 | Loss: 2.9461 | LR: 2.08e-04 | 1301.1ms/step | 15.4 samples/s +Step 24690/44000 | Loss: 2.9521 | LR: 2.08e-04 | 1301.1ms/step | 15.4 samples/s +Step 24700/44000 | Loss: 2.9472 | LR: 2.08e-04 | 1300.6ms/step | 15.4 samples/s +Step 24710/44000 | Loss: 2.9505 | LR: 2.08e-04 | 1301.0ms/step | 15.4 samples/s +Step 24720/44000 | Loss: 2.9369 | LR: 2.08e-04 | 1302.1ms/step | 15.4 samples/s +Step 24730/44000 | Loss: 2.9497 | LR: 2.07e-04 | 1302.7ms/step | 15.4 samples/s +Step 24740/44000 | Loss: 2.9964 | LR: 2.07e-04 | 1303.0ms/step | 15.3 samples/s +Step 24750/44000 | Loss: 2.9322 | LR: 2.07e-04 | 1301.3ms/step | 15.4 samples/s +Validation at step 24750: Loss = 2.9695, Perplexity = 19.48 +Step 24760/44000 | Loss: 2.9538 | LR: 2.07e-04 | 3484.4ms/step | 5.7 samples/s +Step 24770/44000 | Loss: 2.9546 | LR: 2.07e-04 | 1305.6ms/step | 15.3 samples/s +Step 24780/44000 | Loss: 2.9781 | LR: 2.07e-04 | 1301.8ms/step | 15.4 samples/s +Step 24790/44000 | Loss: 2.9354 | LR: 2.06e-04 | 1304.9ms/step | 15.3 samples/s +Step 24800/44000 | Loss: 2.9388 | LR: 2.06e-04 | 1306.0ms/step | 15.3 samples/s +Step 24810/44000 | Loss: 2.9704 | LR: 2.06e-04 | 1303.1ms/step | 15.3 samples/s +Step 24820/44000 | Loss: 2.9535 | LR: 2.06e-04 | 1304.2ms/step | 15.3 samples/s +Step 24830/44000 | Loss: 2.9281 | LR: 2.06e-04 | 1304.0ms/step | 15.3 samples/s +Step 24840/44000 | Loss: 2.9453 | LR: 2.05e-04 | 1302.4ms/step | 15.4 samples/s +Step 24850/44000 | Loss: 2.9485 | LR: 2.05e-04 | 1302.6ms/step | 15.4 samples/s +Step 24860/44000 | Loss: 2.9502 | LR: 2.05e-04 | 1302.7ms/step | 15.4 samples/s +Step 24870/44000 | Loss: 2.9396 | LR: 2.05e-04 | 1301.4ms/step | 15.4 samples/s +Step 24880/44000 | Loss: 2.9660 | LR: 2.05e-04 | 1302.8ms/step | 15.4 samples/s +Step 24890/44000 | Loss: 2.9901 | LR: 2.05e-04 | 1302.2ms/step | 15.4 samples/s +Step 24900/44000 | Loss: 2.9455 | LR: 2.04e-04 | 1303.2ms/step | 15.3 samples/s +Step 24910/44000 | Loss: 2.9549 | LR: 2.04e-04 | 1300.8ms/step | 15.4 samples/s +Step 24920/44000 | Loss: 2.9611 | LR: 2.04e-04 | 1303.9ms/step | 15.3 samples/s +Step 24930/44000 | Loss: 2.9547 | LR: 2.04e-04 | 1301.3ms/step | 15.4 samples/s +Step 24940/44000 | Loss: 2.9659 | LR: 2.04e-04 | 1301.0ms/step | 15.4 samples/s +Step 24950/44000 | Loss: 2.9415 | LR: 2.04e-04 | 1304.2ms/step | 15.3 samples/s +Step 24960/44000 | Loss: 2.9907 | LR: 2.03e-04 | 1304.8ms/step | 15.3 samples/s +Step 24970/44000 | Loss: 2.9404 | LR: 2.03e-04 | 1300.7ms/step | 15.4 samples/s +Step 24980/44000 | Loss: 2.9757 | LR: 2.03e-04 | 1300.0ms/step | 15.4 samples/s +Step 24990/44000 | Loss: 2.9269 | LR: 2.03e-04 | 1300.1ms/step | 15.4 samples/s +Step 25000/44000 | Loss: 2.9704 | LR: 2.03e-04 | 1299.8ms/step | 15.4 samples/s +Validation at step 25000: Loss = 2.9671, Perplexity = 19.43 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-25000 +Step 25010/44000 | Loss: 2.9311 | LR: 2.02e-04 | 3785.1ms/step | 5.3 samples/s +Step 25020/44000 | Loss: 2.9628 | LR: 2.02e-04 | 1300.8ms/step | 15.4 samples/s +Step 25030/44000 | Loss: 2.9421 | LR: 2.02e-04 | 1302.3ms/step | 15.4 samples/s +Step 25040/44000 | Loss: 2.9581 | LR: 2.02e-04 | 1302.9ms/step | 15.4 samples/s +Step 25050/44000 | Loss: 2.9493 | LR: 2.02e-04 | 1303.8ms/step | 15.3 samples/s +Step 25060/44000 | Loss: 2.9462 | LR: 2.02e-04 | 1303.3ms/step | 15.3 samples/s +Step 25070/44000 | Loss: 2.9695 | LR: 2.01e-04 | 1304.1ms/step | 15.3 samples/s +Step 25080/44000 | Loss: 2.9321 | LR: 2.01e-04 | 1301.4ms/step | 15.4 samples/s +Step 25090/44000 | Loss: 2.9710 | LR: 2.01e-04 | 1302.5ms/step | 15.4 samples/s +Step 25100/44000 | Loss: 2.9303 | LR: 2.01e-04 | 1303.5ms/step | 15.3 samples/s +Step 25110/44000 | Loss: 2.9554 | LR: 2.01e-04 | 1304.6ms/step | 15.3 samples/s +Step 25120/44000 | Loss: 2.9302 | LR: 2.00e-04 | 1305.6ms/step | 15.3 samples/s +Step 25130/44000 | Loss: 2.9833 | LR: 2.00e-04 | 1303.2ms/step | 15.3 samples/s +Step 25140/44000 | Loss: 2.9924 | LR: 2.00e-04 | 1302.7ms/step | 15.4 samples/s +Step 25150/44000 | Loss: 2.9164 | LR: 2.00e-04 | 1304.3ms/step | 15.3 samples/s +Step 25160/44000 | Loss: 2.9298 | LR: 2.00e-04 | 1304.7ms/step | 15.3 samples/s +Step 25170/44000 | Loss: 2.9652 | LR: 2.00e-04 | 1300.1ms/step | 15.4 samples/s +Step 25180/44000 | Loss: 2.9038 | LR: 1.99e-04 | 1299.8ms/step | 15.4 samples/s +Step 25190/44000 | Loss: 2.9473 | LR: 1.99e-04 | 1303.6ms/step | 15.3 samples/s +Step 25200/44000 | Loss: 2.9682 | LR: 1.99e-04 | 1303.5ms/step | 15.3 samples/s +Step 25210/44000 | Loss: 2.9642 | LR: 1.99e-04 | 1301.5ms/step | 15.4 samples/s +Step 25220/44000 | Loss: 2.9327 | LR: 1.99e-04 | 1302.1ms/step | 15.4 samples/s +Step 25230/44000 | Loss: 2.9533 | LR: 1.99e-04 | 1303.5ms/step | 15.3 samples/s +Step 25240/44000 | Loss: 2.9350 | LR: 1.98e-04 | 1303.1ms/step | 15.3 samples/s +Step 25250/44000 | Loss: 2.9675 | LR: 1.98e-04 | 1303.2ms/step | 15.3 samples/s +Validation at step 25250: Loss = 2.9654, Perplexity = 19.40 +Step 25260/44000 | Loss: 2.9267 | LR: 1.98e-04 | 3486.3ms/step | 5.7 samples/s +Step 25270/44000 | Loss: 2.9587 | LR: 1.98e-04 | 1303.6ms/step | 15.3 samples/s +Step 25280/44000 | Loss: 2.9260 | LR: 1.98e-04 | 1302.9ms/step | 15.4 samples/s +Step 25290/44000 | Loss: 2.9462 | LR: 1.97e-04 | 1302.0ms/step | 15.4 samples/s +Step 25300/44000 | Loss: 2.9506 | LR: 1.97e-04 | 1306.6ms/step | 15.3 samples/s +Step 25310/44000 | Loss: 2.9017 | LR: 1.97e-04 | 1305.9ms/step | 15.3 samples/s +Step 25320/44000 | Loss: 2.9418 | LR: 1.97e-04 | 1301.1ms/step | 15.4 samples/s +Step 25330/44000 | Loss: 2.9531 | LR: 1.97e-04 | 1299.3ms/step | 15.4 samples/s +Step 25340/44000 | Loss: 2.9449 | LR: 1.97e-04 | 1302.4ms/step | 15.4 samples/s +Step 25350/44000 | Loss: 2.9367 | LR: 1.96e-04 | 1303.8ms/step | 15.3 samples/s +Step 25360/44000 | Loss: 2.9844 | LR: 1.96e-04 | 1303.9ms/step | 15.3 samples/s +Step 25370/44000 | Loss: 2.9322 | LR: 1.96e-04 | 1300.8ms/step | 15.4 samples/s +Step 25380/44000 | Loss: 2.9736 | LR: 1.96e-04 | 1302.5ms/step | 15.4 samples/s +Step 25390/44000 | Loss: 2.9441 | LR: 1.96e-04 | 1304.7ms/step | 15.3 samples/s +Step 25400/44000 | Loss: 2.9580 | LR: 1.96e-04 | 1305.6ms/step | 15.3 samples/s +Step 25410/44000 | Loss: 2.9874 | LR: 1.95e-04 | 1305.0ms/step | 15.3 samples/s +Step 25420/44000 | Loss: 2.9728 | LR: 1.95e-04 | 1305.3ms/step | 15.3 samples/s +Step 25430/44000 | Loss: 2.9485 | LR: 1.95e-04 | 1302.8ms/step | 15.4 samples/s +Step 25440/44000 | Loss: 2.9497 | LR: 1.95e-04 | 1302.2ms/step | 15.4 samples/s +Step 25450/44000 | Loss: 2.9865 | LR: 1.95e-04 | 1302.1ms/step | 15.4 samples/s +Step 25460/44000 | Loss: 2.9580 | LR: 1.94e-04 | 1304.0ms/step | 15.3 samples/s +Step 25470/44000 | Loss: 2.9390 | LR: 1.94e-04 | 1298.4ms/step | 15.4 samples/s +Step 25480/44000 | Loss: 2.9536 | LR: 1.94e-04 | 1300.0ms/step | 15.4 samples/s +Step 25490/44000 | Loss: 2.9528 | LR: 1.94e-04 | 1299.4ms/step | 15.4 samples/s +Step 25500/44000 | Loss: 2.9425 | LR: 1.94e-04 | 1298.7ms/step | 15.4 samples/s +Validation at step 25500: Loss = 2.9634, Perplexity = 19.36 +Step 25510/44000 | Loss: 2.9334 | LR: 1.94e-04 | 3483.9ms/step | 5.7 samples/s +Step 25520/44000 | Loss: 2.9310 | LR: 1.93e-04 | 1303.2ms/step | 15.3 samples/s +Step 25530/44000 | Loss: 2.9337 | LR: 1.93e-04 | 1301.7ms/step | 15.4 samples/s +Step 25540/44000 | Loss: 2.9412 | LR: 1.93e-04 | 1302.3ms/step | 15.4 samples/s +Step 25550/44000 | Loss: 2.9530 | LR: 1.93e-04 | 1303.2ms/step | 15.3 samples/s +Step 25560/44000 | Loss: 2.9594 | LR: 1.93e-04 | 1302.2ms/step | 15.4 samples/s +Step 25570/44000 | Loss: 2.9434 | LR: 1.92e-04 | 1304.7ms/step | 15.3 samples/s +Step 25580/44000 | Loss: 2.9756 | LR: 1.92e-04 | 1300.9ms/step | 15.4 samples/s +Step 25590/44000 | Loss: 2.9533 | LR: 1.92e-04 | 1301.0ms/step | 15.4 samples/s +Step 25600/44000 | Loss: 2.9220 | LR: 1.92e-04 | 1300.9ms/step | 15.4 samples/s +Step 25610/44000 | Loss: 2.9359 | LR: 1.92e-04 | 1301.2ms/step | 15.4 samples/s +Step 25620/44000 | Loss: 2.9244 | LR: 1.92e-04 | 1301.5ms/step | 15.4 samples/s +Step 25630/44000 | Loss: 2.9557 | LR: 1.91e-04 | 1299.4ms/step | 15.4 samples/s +Step 25640/44000 | Loss: 2.9391 | LR: 1.91e-04 | 1302.7ms/step | 15.4 samples/s +Step 25650/44000 | Loss: 2.9447 | LR: 1.91e-04 | 1303.3ms/step | 15.3 samples/s +Step 25660/44000 | Loss: 2.9652 | LR: 1.91e-04 | 1303.8ms/step | 15.3 samples/s +Step 25670/44000 | Loss: 2.9558 | LR: 1.91e-04 | 1302.0ms/step | 15.4 samples/s +Step 25680/44000 | Loss: 2.9228 | LR: 1.91e-04 | 1301.0ms/step | 15.4 samples/s +Step 25690/44000 | Loss: 2.9433 | LR: 1.90e-04 | 1301.7ms/step | 15.4 samples/s +Step 25700/44000 | Loss: 2.9254 | LR: 1.90e-04 | 1303.6ms/step | 15.3 samples/s +Step 25710/44000 | Loss: 2.9405 | LR: 1.90e-04 | 1304.7ms/step | 15.3 samples/s +Step 25720/44000 | Loss: 2.9352 | LR: 1.90e-04 | 1305.0ms/step | 15.3 samples/s +Step 25730/44000 | Loss: 2.9272 | LR: 1.90e-04 | 1302.2ms/step | 15.4 samples/s +Step 25740/44000 | Loss: 2.9420 | LR: 1.89e-04 | 1304.7ms/step | 15.3 samples/s +Step 25750/44000 | Loss: 2.9419 | LR: 1.89e-04 | 1304.1ms/step | 15.3 samples/s +Validation at step 25750: Loss = 2.9622, Perplexity = 19.34 +Step 25760/44000 | Loss: 2.9420 | LR: 1.89e-04 | 3487.0ms/step | 5.7 samples/s +Step 25770/44000 | Loss: 2.9643 | LR: 1.89e-04 | 1303.0ms/step | 15.3 samples/s +Step 25780/44000 | Loss: 2.9492 | LR: 1.89e-04 | 1303.9ms/step | 15.3 samples/s +Step 25790/44000 | Loss: 2.9852 | LR: 1.89e-04 | 1304.0ms/step | 15.3 samples/s +Step 25800/44000 | Loss: 2.9085 | LR: 1.88e-04 | 1304.0ms/step | 15.3 samples/s +Step 25810/44000 | Loss: 2.9412 | LR: 1.88e-04 | 1302.7ms/step | 15.4 samples/s +Step 25820/44000 | Loss: 2.9448 | LR: 1.88e-04 | 1303.4ms/step | 15.3 samples/s +Step 25830/44000 | Loss: 2.9475 | LR: 1.88e-04 | 1303.6ms/step | 15.3 samples/s +Step 25840/44000 | Loss: 2.9571 | LR: 1.88e-04 | 1303.3ms/step | 15.3 samples/s +Step 25850/44000 | Loss: 2.9202 | LR: 1.88e-04 | 1304.0ms/step | 15.3 samples/s +Step 25860/44000 | Loss: 2.9187 | LR: 1.87e-04 | 1303.4ms/step | 15.3 samples/s +Step 25870/44000 | Loss: 2.9216 | LR: 1.87e-04 | 1308.5ms/step | 15.3 samples/s +Step 25880/44000 | Loss: 2.9656 | LR: 1.87e-04 | 1303.1ms/step | 15.3 samples/s +Step 25890/44000 | Loss: 2.9455 | LR: 1.87e-04 | 1306.2ms/step | 15.3 samples/s +Step 25900/44000 | Loss: 2.9485 | LR: 1.87e-04 | 1305.6ms/step | 15.3 samples/s +Step 25910/44000 | Loss: 2.9323 | LR: 1.87e-04 | 1304.0ms/step | 15.3 samples/s +Step 25920/44000 | Loss: 2.9300 | LR: 1.86e-04 | 1305.5ms/step | 15.3 samples/s +Step 25930/44000 | Loss: 2.9608 | LR: 1.86e-04 | 1302.6ms/step | 15.4 samples/s +Step 25940/44000 | Loss: 2.9215 | LR: 1.86e-04 | 1303.0ms/step | 15.3 samples/s +Step 25950/44000 | Loss: 2.9220 | LR: 1.86e-04 | 1302.6ms/step | 15.4 samples/s +Step 25960/44000 | Loss: 2.9406 | LR: 1.86e-04 | 1304.6ms/step | 15.3 samples/s +Step 25970/44000 | Loss: 2.9184 | LR: 1.85e-04 | 1308.7ms/step | 15.3 samples/s +Step 25980/44000 | Loss: 2.9620 | LR: 1.85e-04 | 1304.1ms/step | 15.3 samples/s +Step 25990/44000 | Loss: 2.9533 | LR: 1.85e-04 | 1303.9ms/step | 15.3 samples/s +Step 26000/44000 | Loss: 2.9322 | LR: 1.85e-04 | 1304.6ms/step | 15.3 samples/s +Validation at step 26000: Loss = 2.9605, Perplexity = 19.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-26000 +Step 26010/44000 | Loss: 2.9474 | LR: 1.85e-04 | 3802.4ms/step | 5.3 samples/s +Step 26020/44000 | Loss: 2.9464 | LR: 1.85e-04 | 1304.0ms/step | 15.3 samples/s +Step 26030/44000 | Loss: 2.9337 | LR: 1.84e-04 | 1303.9ms/step | 15.3 samples/s +Step 26040/44000 | Loss: 2.9204 | LR: 1.84e-04 | 1300.8ms/step | 15.4 samples/s +Step 26050/44000 | Loss: 2.9567 | LR: 1.84e-04 | 1300.0ms/step | 15.4 samples/s +Step 26060/44000 | Loss: 2.9061 | LR: 1.84e-04 | 1300.1ms/step | 15.4 samples/s +Step 26070/44000 | Loss: 2.9363 | LR: 1.84e-04 | 1301.1ms/step | 15.4 samples/s +Step 26080/44000 | Loss: 2.9216 | LR: 1.84e-04 | 1300.4ms/step | 15.4 samples/s +Step 26090/44000 | Loss: 2.9363 | LR: 1.83e-04 | 1301.5ms/step | 15.4 samples/s +Step 26100/44000 | Loss: 2.9381 | LR: 1.83e-04 | 1300.9ms/step | 15.4 samples/s +Step 26110/44000 | Loss: 2.9550 | LR: 1.83e-04 | 1299.8ms/step | 15.4 samples/s +Step 26120/44000 | Loss: 2.9349 | LR: 1.83e-04 | 1300.2ms/step | 15.4 samples/s +Step 26130/44000 | Loss: 2.9561 | LR: 1.83e-04 | 1299.8ms/step | 15.4 samples/s +Step 26140/44000 | Loss: 2.9598 | LR: 1.82e-04 | 1302.9ms/step | 15.4 samples/s +Step 26150/44000 | Loss: 2.9512 | LR: 1.82e-04 | 1300.3ms/step | 15.4 samples/s +Step 26160/44000 | Loss: 2.9358 | LR: 1.82e-04 | 1303.0ms/step | 15.3 samples/s +Step 26170/44000 | Loss: 2.9382 | LR: 1.82e-04 | 1303.2ms/step | 15.3 samples/s +Step 26180/44000 | Loss: 2.9523 | LR: 1.82e-04 | 1305.9ms/step | 15.3 samples/s +Step 26190/44000 | Loss: 2.9245 | LR: 1.82e-04 | 1306.9ms/step | 15.3 samples/s +Step 26200/44000 | Loss: 2.9073 | LR: 1.81e-04 | 1304.8ms/step | 15.3 samples/s +Step 26210/44000 | Loss: 2.9706 | LR: 1.81e-04 | 1303.0ms/step | 15.3 samples/s +Step 26220/44000 | Loss: 2.9437 | LR: 1.81e-04 | 1304.7ms/step | 15.3 samples/s +Step 26230/44000 | Loss: 2.9540 | LR: 1.81e-04 | 1313.0ms/step | 15.2 samples/s +Step 26240/44000 | Loss: 2.9440 | LR: 1.81e-04 | 1311.8ms/step | 15.2 samples/s +Step 26250/44000 | Loss: 2.9528 | LR: 1.81e-04 | 1308.6ms/step | 15.3 samples/s +Validation at step 26250: Loss = 2.9592, Perplexity = 19.28 +Step 26260/44000 | Loss: 2.9362 | LR: 1.80e-04 | 3489.0ms/step | 5.7 samples/s +Step 26270/44000 | Loss: 2.9327 | LR: 1.80e-04 | 1304.4ms/step | 15.3 samples/s +Step 26280/44000 | Loss: 2.9539 | LR: 1.80e-04 | 1309.8ms/step | 15.3 samples/s +Step 26290/44000 | Loss: 2.9670 | LR: 1.80e-04 | 1311.8ms/step | 15.2 samples/s +Step 26300/44000 | Loss: 2.9476 | LR: 1.80e-04 | 1304.0ms/step | 15.3 samples/s +Step 26310/44000 | Loss: 2.9337 | LR: 1.80e-04 | 1303.8ms/step | 15.3 samples/s +Step 26320/44000 | Loss: 2.9515 | LR: 1.79e-04 | 1303.7ms/step | 15.3 samples/s +Step 26330/44000 | Loss: 2.9830 | LR: 1.79e-04 | 1303.9ms/step | 15.3 samples/s +Step 26340/44000 | Loss: 2.9389 | LR: 1.79e-04 | 1305.6ms/step | 15.3 samples/s +Step 26350/44000 | Loss: 2.9549 | LR: 1.79e-04 | 1304.8ms/step | 15.3 samples/s +Step 26360/44000 | Loss: 2.9307 | LR: 1.79e-04 | 1303.3ms/step | 15.3 samples/s +Step 26370/44000 | Loss: 2.9559 | LR: 1.78e-04 | 1304.1ms/step | 15.3 samples/s +Step 26380/44000 | Loss: 2.9595 | LR: 1.78e-04 | 1301.4ms/step | 15.4 samples/s +Step 26390/44000 | Loss: 2.9589 | LR: 1.78e-04 | 1301.0ms/step | 15.4 samples/s +Step 26400/44000 | Loss: 2.9589 | LR: 1.78e-04 | 1299.9ms/step | 15.4 samples/s +Step 26410/44000 | Loss: 2.9229 | LR: 1.78e-04 | 1299.1ms/step | 15.4 samples/s +Step 26420/44000 | Loss: 2.9184 | LR: 1.78e-04 | 1299.7ms/step | 15.4 samples/s +Step 26430/44000 | Loss: 2.9598 | LR: 1.77e-04 | 1300.2ms/step | 15.4 samples/s +Step 26440/44000 | Loss: 2.9186 | LR: 1.77e-04 | 1300.1ms/step | 15.4 samples/s +Step 26450/44000 | Loss: 2.9432 | LR: 1.77e-04 | 1300.8ms/step | 15.4 samples/s +Step 26460/44000 | Loss: 2.9460 | LR: 1.77e-04 | 1299.1ms/step | 15.4 samples/s +Step 26470/44000 | Loss: 2.9379 | LR: 1.77e-04 | 1301.9ms/step | 15.4 samples/s +Step 26480/44000 | Loss: 2.9354 | LR: 1.77e-04 | 1305.4ms/step | 15.3 samples/s +Step 26490/44000 | Loss: 2.9707 | LR: 1.76e-04 | 1306.5ms/step | 15.3 samples/s +Step 26500/44000 | Loss: 2.9535 | LR: 1.76e-04 | 1304.5ms/step | 15.3 samples/s +Validation at step 26500: Loss = 2.9577, Perplexity = 19.25 +Step 26510/44000 | Loss: 2.9661 | LR: 1.76e-04 | 3483.7ms/step | 5.7 samples/s +Step 26520/44000 | Loss: 2.9416 | LR: 1.76e-04 | 1302.6ms/step | 15.4 samples/s +Step 26530/44000 | Loss: 2.9451 | LR: 1.76e-04 | 1303.7ms/step | 15.3 samples/s +Step 26540/44000 | Loss: 2.9531 | LR: 1.76e-04 | 1302.4ms/step | 15.4 samples/s +Step 26550/44000 | Loss: 2.9585 | LR: 1.75e-04 | 1300.9ms/step | 15.4 samples/s +Step 26560/44000 | Loss: 2.9878 | LR: 1.75e-04 | 1300.1ms/step | 15.4 samples/s +Step 26570/44000 | Loss: 2.9171 | LR: 1.75e-04 | 1301.0ms/step | 15.4 samples/s +Step 26580/44000 | Loss: 2.9679 | LR: 1.75e-04 | 1299.9ms/step | 15.4 samples/s +Step 26590/44000 | Loss: 2.9707 | LR: 1.75e-04 | 1300.5ms/step | 15.4 samples/s +Step 26600/44000 | Loss: 2.9420 | LR: 1.74e-04 | 1302.5ms/step | 15.4 samples/s +Step 26610/44000 | Loss: 2.9506 | LR: 1.74e-04 | 1304.0ms/step | 15.3 samples/s +Step 26620/44000 | Loss: 2.9283 | LR: 1.74e-04 | 1303.4ms/step | 15.3 samples/s +Step 26630/44000 | Loss: 2.9353 | LR: 1.74e-04 | 1301.0ms/step | 15.4 samples/s +Step 26640/44000 | Loss: 2.9315 | LR: 1.74e-04 | 1302.7ms/step | 15.4 samples/s +Step 26650/44000 | Loss: 2.9624 | LR: 1.74e-04 | 1303.2ms/step | 15.3 samples/s +Step 26660/44000 | Loss: 2.9511 | LR: 1.73e-04 | 1305.6ms/step | 15.3 samples/s +Step 26670/44000 | Loss: 2.9389 | LR: 1.73e-04 | 1305.3ms/step | 15.3 samples/s +Step 26680/44000 | Loss: 2.9529 | LR: 1.73e-04 | 1303.8ms/step | 15.3 samples/s +Step 26690/44000 | Loss: 2.9619 | LR: 1.73e-04 | 1305.4ms/step | 15.3 samples/s +Step 26700/44000 | Loss: 2.9571 | LR: 1.73e-04 | 1303.2ms/step | 15.3 samples/s +Step 26710/44000 | Loss: 2.9266 | LR: 1.73e-04 | 1307.8ms/step | 15.3 samples/s +Step 26720/44000 | Loss: 2.9351 | LR: 1.72e-04 | 1301.1ms/step | 15.4 samples/s +Step 26730/44000 | Loss: 2.9218 | LR: 1.72e-04 | 1303.7ms/step | 15.3 samples/s +Step 26740/44000 | Loss: 2.9462 | LR: 1.72e-04 | 1304.1ms/step | 15.3 samples/s +Step 26750/44000 | Loss: 2.9822 | LR: 1.72e-04 | 1303.5ms/step | 15.3 samples/s +Validation at step 26750: Loss = 2.9564, Perplexity = 19.23 +Step 26760/44000 | Loss: 2.9360 | LR: 1.72e-04 | 3484.7ms/step | 5.7 samples/s +Step 26770/44000 | Loss: 2.9125 | LR: 1.72e-04 | 1301.6ms/step | 15.4 samples/s +Step 26780/44000 | Loss: 2.9258 | LR: 1.71e-04 | 1304.0ms/step | 15.3 samples/s +Step 26790/44000 | Loss: 2.9380 | LR: 1.71e-04 | 1305.1ms/step | 15.3 samples/s +Step 26800/44000 | Loss: 2.9478 | LR: 1.71e-04 | 1310.3ms/step | 15.3 samples/s +Step 26810/44000 | Loss: 2.9482 | LR: 1.71e-04 | 1309.4ms/step | 15.3 samples/s +Step 26820/44000 | Loss: 2.9661 | LR: 1.71e-04 | 1309.4ms/step | 15.3 samples/s +Step 26830/44000 | Loss: 2.9427 | LR: 1.71e-04 | 1304.4ms/step | 15.3 samples/s +Step 26840/44000 | Loss: 2.9628 | LR: 1.70e-04 | 1303.4ms/step | 15.3 samples/s +Step 26850/44000 | Loss: 2.9400 | LR: 1.70e-04 | 1300.3ms/step | 15.4 samples/s +Step 26860/44000 | Loss: 2.9254 | LR: 1.70e-04 | 1303.3ms/step | 15.3 samples/s +Step 26870/44000 | Loss: 2.9520 | LR: 1.70e-04 | 1299.3ms/step | 15.4 samples/s +Step 26880/44000 | Loss: 2.9174 | LR: 1.70e-04 | 1305.5ms/step | 15.3 samples/s +Step 26890/44000 | Loss: 2.9636 | LR: 1.69e-04 | 1305.6ms/step | 15.3 samples/s +Step 26900/44000 | Loss: 2.9117 | LR: 1.69e-04 | 1306.1ms/step | 15.3 samples/s +Step 26910/44000 | Loss: 2.9574 | LR: 1.69e-04 | 1304.4ms/step | 15.3 samples/s +Step 26920/44000 | Loss: 2.9298 | LR: 1.69e-04 | 1304.2ms/step | 15.3 samples/s +Step 26930/44000 | Loss: 2.9312 | LR: 1.69e-04 | 1304.8ms/step | 15.3 samples/s +Step 26940/44000 | Loss: 2.9444 | LR: 1.69e-04 | 1304.5ms/step | 15.3 samples/s +Step 26950/44000 | Loss: 2.9488 | LR: 1.68e-04 | 1305.2ms/step | 15.3 samples/s +Step 26960/44000 | Loss: 2.9325 | LR: 1.68e-04 | 1302.9ms/step | 15.4 samples/s +Step 26970/44000 | Loss: 2.9343 | LR: 1.68e-04 | 1301.9ms/step | 15.4 samples/s +Step 26980/44000 | Loss: 2.9659 | LR: 1.68e-04 | 1299.9ms/step | 15.4 samples/s +Step 26990/44000 | Loss: 2.9280 | LR: 1.68e-04 | 1300.0ms/step | 15.4 samples/s +Step 27000/44000 | Loss: 2.9424 | LR: 1.68e-04 | 1301.1ms/step | 15.4 samples/s +Validation at step 27000: Loss = 2.9550, Perplexity = 19.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-27000 +Step 27010/44000 | Loss: 2.9571 | LR: 1.67e-04 | 3786.5ms/step | 5.3 samples/s +Step 27020/44000 | Loss: 2.9392 | LR: 1.67e-04 | 1305.9ms/step | 15.3 samples/s +Step 27030/44000 | Loss: 2.9567 | LR: 1.67e-04 | 1305.3ms/step | 15.3 samples/s +Step 27040/44000 | Loss: 2.9442 | LR: 1.67e-04 | 1302.3ms/step | 15.4 samples/s +Step 27050/44000 | Loss: 2.9544 | LR: 1.67e-04 | 1301.9ms/step | 15.4 samples/s +Step 27060/44000 | Loss: 2.9130 | LR: 1.67e-04 | 1301.8ms/step | 15.4 samples/s +Step 27070/44000 | Loss: 2.9493 | LR: 1.66e-04 | 1301.2ms/step | 15.4 samples/s +Step 27080/44000 | Loss: 2.9516 | LR: 1.66e-04 | 1301.6ms/step | 15.4 samples/s +Step 27090/44000 | Loss: 2.9360 | LR: 1.66e-04 | 1302.5ms/step | 15.4 samples/s +Step 27100/44000 | Loss: 2.9236 | LR: 1.66e-04 | 1302.4ms/step | 15.4 samples/s +Step 27110/44000 | Loss: 2.9180 | LR: 1.66e-04 | 1302.4ms/step | 15.4 samples/s +Step 27120/44000 | Loss: 2.9256 | LR: 1.66e-04 | 1304.1ms/step | 15.3 samples/s +Step 27130/44000 | Loss: 2.9182 | LR: 1.65e-04 | 1298.9ms/step | 15.4 samples/s +Step 27140/44000 | Loss: 2.9327 | LR: 1.65e-04 | 1301.9ms/step | 15.4 samples/s +Step 27150/44000 | Loss: 2.9504 | LR: 1.65e-04 | 1301.2ms/step | 15.4 samples/s +Step 27160/44000 | Loss: 2.9177 | LR: 1.65e-04 | 1303.2ms/step | 15.3 samples/s +Step 27170/44000 | Loss: 2.9451 | LR: 1.65e-04 | 1303.1ms/step | 15.3 samples/s +Step 27180/44000 | Loss: 2.9513 | LR: 1.65e-04 | 1303.0ms/step | 15.3 samples/s +Step 27190/44000 | Loss: 2.9498 | LR: 1.64e-04 | 1304.4ms/step | 15.3 samples/s +Step 27200/44000 | Loss: 2.9551 | LR: 1.64e-04 | 1303.7ms/step | 15.3 samples/s +Step 27210/44000 | Loss: 2.9383 | LR: 1.64e-04 | 1302.5ms/step | 15.4 samples/s +Step 27220/44000 | Loss: 2.9212 | LR: 1.64e-04 | 1302.7ms/step | 15.4 samples/s +Step 27230/44000 | Loss: 2.9150 | LR: 1.64e-04 | 1302.6ms/step | 15.4 samples/s +Step 27240/44000 | Loss: 2.9470 | LR: 1.63e-04 | 1304.4ms/step | 15.3 samples/s +Step 27250/44000 | Loss: 2.9177 | LR: 1.63e-04 | 1302.5ms/step | 15.4 samples/s +Validation at step 27250: Loss = 2.9537, Perplexity = 19.18 +Step 27260/44000 | Loss: 2.9574 | LR: 1.63e-04 | 3485.5ms/step | 5.7 samples/s +Step 27270/44000 | Loss: 2.9682 | LR: 1.63e-04 | 1300.1ms/step | 15.4 samples/s +Step 27280/44000 | Loss: 2.9590 | LR: 1.63e-04 | 1303.5ms/step | 15.3 samples/s +Step 27290/44000 | Loss: 2.9283 | LR: 1.63e-04 | 1302.1ms/step | 15.4 samples/s +Step 27300/44000 | Loss: 2.9163 | LR: 1.62e-04 | 1302.4ms/step | 15.4 samples/s +Step 27310/44000 | Loss: 2.9483 | LR: 1.62e-04 | 1302.3ms/step | 15.4 samples/s +Step 27320/44000 | Loss: 2.9545 | LR: 1.62e-04 | 1301.1ms/step | 15.4 samples/s +Step 27330/44000 | Loss: 2.9574 | LR: 1.62e-04 | 1301.8ms/step | 15.4 samples/s +Step 27340/44000 | Loss: 2.9142 | LR: 1.62e-04 | 1301.4ms/step | 15.4 samples/s +Step 27350/44000 | Loss: 2.9126 | LR: 1.62e-04 | 1302.5ms/step | 15.4 samples/s +Step 27360/44000 | Loss: 2.9175 | LR: 1.61e-04 | 1301.1ms/step | 15.4 samples/s +Step 27370/44000 | Loss: 2.9265 | LR: 1.61e-04 | 1299.6ms/step | 15.4 samples/s +Step 27380/44000 | Loss: 2.9689 | LR: 1.61e-04 | 1302.6ms/step | 15.4 samples/s +Step 27390/44000 | Loss: 2.9279 | LR: 1.61e-04 | 1303.5ms/step | 15.3 samples/s +Step 27400/44000 | Loss: 2.9554 | LR: 1.61e-04 | 1302.1ms/step | 15.4 samples/s +Step 27410/44000 | Loss: 2.9391 | LR: 1.61e-04 | 1305.0ms/step | 15.3 samples/s +Step 27420/44000 | Loss: 2.9310 | LR: 1.60e-04 | 1304.1ms/step | 15.3 samples/s +Step 27430/44000 | Loss: 2.9555 | LR: 1.60e-04 | 1305.1ms/step | 15.3 samples/s +Step 27440/44000 | Loss: 2.9443 | LR: 1.60e-04 | 1305.2ms/step | 15.3 samples/s +Step 27450/44000 | Loss: 2.9116 | LR: 1.60e-04 | 1308.7ms/step | 15.3 samples/s +Step 27460/44000 | Loss: 2.9323 | LR: 1.60e-04 | 1306.5ms/step | 15.3 samples/s +Step 27470/44000 | Loss: 2.9637 | LR: 1.60e-04 | 1307.4ms/step | 15.3 samples/s +Step 27480/44000 | Loss: 2.9340 | LR: 1.59e-04 | 1307.1ms/step | 15.3 samples/s +Step 27490/44000 | Loss: 2.9519 | LR: 1.59e-04 | 1306.8ms/step | 15.3 samples/s +Step 27500/44000 | Loss: 2.9423 | LR: 1.59e-04 | 1305.8ms/step | 15.3 samples/s +Validation at step 27500: Loss = 2.9524, Perplexity = 19.15 +Step 27510/44000 | Loss: 2.9760 | LR: 1.59e-04 | 3487.0ms/step | 5.7 samples/s +Step 27520/44000 | Loss: 2.9348 | LR: 1.59e-04 | 1300.3ms/step | 15.4 samples/s +Step 27530/44000 | Loss: 2.9482 | LR: 1.59e-04 | 1302.7ms/step | 15.4 samples/s +Step 27540/44000 | Loss: 2.9069 | LR: 1.58e-04 | 1303.2ms/step | 15.3 samples/s +Step 27550/44000 | Loss: 2.9045 | LR: 1.58e-04 | 1300.9ms/step | 15.4 samples/s +Step 27560/44000 | Loss: 2.9216 | LR: 1.58e-04 | 1303.3ms/step | 15.3 samples/s +Step 27570/44000 | Loss: 2.9117 | LR: 1.58e-04 | 1302.6ms/step | 15.4 samples/s +Step 27580/44000 | Loss: 2.9638 | LR: 1.58e-04 | 1305.1ms/step | 15.3 samples/s +Step 27590/44000 | Loss: 2.9293 | LR: 1.58e-04 | 1300.6ms/step | 15.4 samples/s +Step 27600/44000 | Loss: 2.9436 | LR: 1.57e-04 | 1298.6ms/step | 15.4 samples/s +Step 27610/44000 | Loss: 2.9123 | LR: 1.57e-04 | 1300.5ms/step | 15.4 samples/s +Step 27620/44000 | Loss: 2.9430 | LR: 1.57e-04 | 1300.3ms/step | 15.4 samples/s +Step 27630/44000 | Loss: 2.9123 | LR: 1.57e-04 | 1300.0ms/step | 15.4 samples/s +Step 27640/44000 | Loss: 2.9708 | LR: 1.57e-04 | 1299.9ms/step | 15.4 samples/s +Step 27650/44000 | Loss: 2.9920 | LR: 1.57e-04 | 1300.1ms/step | 15.4 samples/s +Step 27660/44000 | Loss: 2.9410 | LR: 1.56e-04 | 1299.9ms/step | 15.4 samples/s +Step 27670/44000 | Loss: 2.9304 | LR: 1.56e-04 | 1300.1ms/step | 15.4 samples/s +Step 27680/44000 | Loss: 2.9473 | LR: 1.56e-04 | 1299.9ms/step | 15.4 samples/s +Step 27690/44000 | Loss: 2.9366 | LR: 1.56e-04 | 1301.5ms/step | 15.4 samples/s +Step 27700/44000 | Loss: 2.9255 | LR: 1.56e-04 | 1300.6ms/step | 15.4 samples/s +Step 27710/44000 | Loss: 2.9465 | LR: 1.56e-04 | 1299.7ms/step | 15.4 samples/s +Step 27720/44000 | Loss: 2.9296 | LR: 1.55e-04 | 1299.0ms/step | 15.4 samples/s +Step 27730/44000 | Loss: 2.9324 | LR: 1.55e-04 | 1300.5ms/step | 15.4 samples/s +Step 27740/44000 | Loss: 2.9304 | LR: 1.55e-04 | 1299.3ms/step | 15.4 samples/s +Step 27750/44000 | Loss: 2.9317 | LR: 1.55e-04 | 1299.4ms/step | 15.4 samples/s +Validation at step 27750: Loss = 2.9512, Perplexity = 19.13 +Step 27760/44000 | Loss: 2.9357 | LR: 1.55e-04 | 3478.8ms/step | 5.7 samples/s +Step 27770/44000 | Loss: 2.9320 | LR: 1.55e-04 | 1300.5ms/step | 15.4 samples/s +Step 27780/44000 | Loss: 2.9299 | LR: 1.54e-04 | 1302.6ms/step | 15.4 samples/s +Step 27790/44000 | Loss: 2.8926 | LR: 1.54e-04 | 1300.2ms/step | 15.4 samples/s +Step 27800/44000 | Loss: 2.9268 | LR: 1.54e-04 | 1300.6ms/step | 15.4 samples/s +Step 27810/44000 | Loss: 2.8981 | LR: 1.54e-04 | 1298.5ms/step | 15.4 samples/s +Step 27820/44000 | Loss: 2.9223 | LR: 1.54e-04 | 1298.7ms/step | 15.4 samples/s +Step 27830/44000 | Loss: 2.9385 | LR: 1.54e-04 | 1298.6ms/step | 15.4 samples/s +Step 27840/44000 | Loss: 2.9517 | LR: 1.53e-04 | 1298.3ms/step | 15.4 samples/s +Step 27850/44000 | Loss: 2.9185 | LR: 1.53e-04 | 1298.8ms/step | 15.4 samples/s +Step 27860/44000 | Loss: 2.9273 | LR: 1.53e-04 | 1299.8ms/step | 15.4 samples/s +Step 27870/44000 | Loss: 2.9585 | LR: 1.53e-04 | 1297.8ms/step | 15.4 samples/s +Step 27880/44000 | Loss: 2.9319 | LR: 1.53e-04 | 1299.6ms/step | 15.4 samples/s +Step 27890/44000 | Loss: 2.9096 | LR: 1.53e-04 | 1299.9ms/step | 15.4 samples/s +Step 27900/44000 | Loss: 2.9151 | LR: 1.52e-04 | 1300.0ms/step | 15.4 samples/s +Step 27910/44000 | Loss: 2.9291 | LR: 1.52e-04 | 1300.0ms/step | 15.4 samples/s +Step 27920/44000 | Loss: 2.9670 | LR: 1.52e-04 | 1300.1ms/step | 15.4 samples/s +Step 27930/44000 | Loss: 2.9432 | LR: 1.52e-04 | 1299.9ms/step | 15.4 samples/s +Step 27940/44000 | Loss: 2.9364 | LR: 1.52e-04 | 1299.9ms/step | 15.4 samples/s +Step 27950/44000 | Loss: 2.9539 | LR: 1.52e-04 | 1301.4ms/step | 15.4 samples/s +Step 27960/44000 | Loss: 2.9371 | LR: 1.51e-04 | 1301.3ms/step | 15.4 samples/s +Step 27970/44000 | Loss: 2.9302 | LR: 1.51e-04 | 1301.9ms/step | 15.4 samples/s +Step 27980/44000 | Loss: 2.9375 | LR: 1.51e-04 | 1300.2ms/step | 15.4 samples/s +Step 27990/44000 | Loss: 2.9292 | LR: 1.51e-04 | 1301.6ms/step | 15.4 samples/s +Step 28000/44000 | Loss: 2.9328 | LR: 1.51e-04 | 1303.3ms/step | 15.3 samples/s +Validation at step 28000: Loss = 2.9498, Perplexity = 19.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-28000 +Step 28010/44000 | Loss: 2.9324 | LR: 1.51e-04 | 3785.4ms/step | 5.3 samples/s +Step 28020/44000 | Loss: 2.9701 | LR: 1.50e-04 | 1302.8ms/step | 15.4 samples/s +Step 28030/44000 | Loss: 2.9264 | LR: 1.50e-04 | 1302.3ms/step | 15.4 samples/s +Step 28040/44000 | Loss: 2.9465 | LR: 1.50e-04 | 1301.9ms/step | 15.4 samples/s +Step 28050/44000 | Loss: 2.9504 | LR: 1.50e-04 | 1303.4ms/step | 15.3 samples/s +Step 28060/44000 | Loss: 2.9579 | LR: 1.50e-04 | 1302.4ms/step | 15.4 samples/s +Step 28070/44000 | Loss: 2.9287 | LR: 1.50e-04 | 1301.3ms/step | 15.4 samples/s +Step 28080/44000 | Loss: 2.9110 | LR: 1.49e-04 | 1305.7ms/step | 15.3 samples/s +Step 28090/44000 | Loss: 2.9392 | LR: 1.49e-04 | 1302.1ms/step | 15.4 samples/s +Step 28100/44000 | Loss: 2.9331 | LR: 1.49e-04 | 1303.3ms/step | 15.3 samples/s +Step 28110/44000 | Loss: 2.9117 | LR: 1.49e-04 | 1304.0ms/step | 15.3 samples/s +Step 28120/44000 | Loss: 2.9395 | LR: 1.49e-04 | 1303.1ms/step | 15.3 samples/s +Step 28130/44000 | Loss: 2.9414 | LR: 1.49e-04 | 1303.4ms/step | 15.3 samples/s +Step 28140/44000 | Loss: 2.9358 | LR: 1.48e-04 | 1301.0ms/step | 15.4 samples/s +Step 28150/44000 | Loss: 2.9274 | LR: 1.48e-04 | 1302.8ms/step | 15.4 samples/s +Step 28160/44000 | Loss: 2.9427 | LR: 1.48e-04 | 1303.5ms/step | 15.3 samples/s +Step 28170/44000 | Loss: 2.9172 | LR: 1.48e-04 | 1304.7ms/step | 15.3 samples/s +Step 28180/44000 | Loss: 2.9554 | LR: 1.48e-04 | 1302.5ms/step | 15.4 samples/s +Step 28190/44000 | Loss: 2.9467 | LR: 1.48e-04 | 1303.4ms/step | 15.3 samples/s +Step 28200/44000 | Loss: 2.9538 | LR: 1.47e-04 | 1300.8ms/step | 15.4 samples/s +Step 28210/44000 | Loss: 2.9019 | LR: 1.47e-04 | 1301.5ms/step | 15.4 samples/s +Step 28220/44000 | Loss: 2.9204 | LR: 1.47e-04 | 1303.7ms/step | 15.3 samples/s +Step 28230/44000 | Loss: 2.9401 | LR: 1.47e-04 | 1304.8ms/step | 15.3 samples/s +Step 28240/44000 | Loss: 2.9030 | LR: 1.47e-04 | 1303.6ms/step | 15.3 samples/s +Step 28250/44000 | Loss: 2.9150 | LR: 1.47e-04 | 1303.5ms/step | 15.3 samples/s +Validation at step 28250: Loss = 2.9491, Perplexity = 19.09 +Step 28260/44000 | Loss: 2.9344 | LR: 1.46e-04 | 3487.9ms/step | 5.7 samples/s +Step 28270/44000 | Loss: 2.9198 | LR: 1.46e-04 | 1306.3ms/step | 15.3 samples/s +Step 28280/44000 | Loss: 2.9618 | LR: 1.46e-04 | 1306.6ms/step | 15.3 samples/s +Step 28290/44000 | Loss: 2.8904 | LR: 1.46e-04 | 1305.3ms/step | 15.3 samples/s +Step 28300/44000 | Loss: 2.9333 | LR: 1.46e-04 | 1303.5ms/step | 15.3 samples/s +Step 28310/44000 | Loss: 2.9074 | LR: 1.46e-04 | 1303.3ms/step | 15.3 samples/s +Step 28320/44000 | Loss: 2.9708 | LR: 1.45e-04 | 1303.9ms/step | 15.3 samples/s +Step 28330/44000 | Loss: 2.9323 | LR: 1.45e-04 | 1303.2ms/step | 15.3 samples/s +Step 28340/44000 | Loss: 2.9434 | LR: 1.45e-04 | 1304.8ms/step | 15.3 samples/s +Step 28350/44000 | Loss: 2.9202 | LR: 1.45e-04 | 1304.3ms/step | 15.3 samples/s +Step 28360/44000 | Loss: 2.9308 | LR: 1.45e-04 | 1304.0ms/step | 15.3 samples/s +Step 28370/44000 | Loss: 2.9074 | LR: 1.45e-04 | 1301.1ms/step | 15.4 samples/s +Step 28380/44000 | Loss: 2.9157 | LR: 1.44e-04 | 1300.9ms/step | 15.4 samples/s +Step 28390/44000 | Loss: 2.9301 | LR: 1.44e-04 | 1301.8ms/step | 15.4 samples/s +Step 28400/44000 | Loss: 2.9289 | LR: 1.44e-04 | 1303.0ms/step | 15.3 samples/s +Step 28410/44000 | Loss: 2.9564 | LR: 1.44e-04 | 1304.6ms/step | 15.3 samples/s +Step 28420/44000 | Loss: 2.9755 | LR: 1.44e-04 | 1304.7ms/step | 15.3 samples/s +Step 28430/44000 | Loss: 2.9014 | LR: 1.44e-04 | 1304.7ms/step | 15.3 samples/s +Step 28440/44000 | Loss: 2.9407 | LR: 1.43e-04 | 1304.8ms/step | 15.3 samples/s +Step 28450/44000 | Loss: 2.9388 | LR: 1.43e-04 | 1302.9ms/step | 15.4 samples/s +Step 28460/44000 | Loss: 2.9452 | LR: 1.43e-04 | 1300.1ms/step | 15.4 samples/s +Step 28470/44000 | Loss: 2.9428 | LR: 1.43e-04 | 1299.8ms/step | 15.4 samples/s +Step 28480/44000 | Loss: 2.9238 | LR: 1.43e-04 | 1300.9ms/step | 15.4 samples/s +Step 28490/44000 | Loss: 2.9322 | LR: 1.43e-04 | 1300.0ms/step | 15.4 samples/s +Step 28500/44000 | Loss: 2.9322 | LR: 1.42e-04 | 1302.3ms/step | 15.4 samples/s +Validation at step 28500: Loss = 2.9475, Perplexity = 19.06 +Step 28510/44000 | Loss: 2.9573 | LR: 1.42e-04 | 3481.6ms/step | 5.7 samples/s +Step 28520/44000 | Loss: 2.9363 | LR: 1.42e-04 | 1302.1ms/step | 15.4 samples/s +Step 28530/44000 | Loss: 2.9257 | LR: 1.42e-04 | 1302.3ms/step | 15.4 samples/s +Step 28540/44000 | Loss: 2.9513 | LR: 1.42e-04 | 1300.9ms/step | 15.4 samples/s +Step 28550/44000 | Loss: 2.9681 | LR: 1.42e-04 | 1303.4ms/step | 15.3 samples/s +Step 28560/44000 | Loss: 2.9180 | LR: 1.41e-04 | 1304.4ms/step | 15.3 samples/s +Step 28570/44000 | Loss: 2.9106 | LR: 1.41e-04 | 1302.8ms/step | 15.4 samples/s +Step 28580/44000 | Loss: 2.9428 | LR: 1.41e-04 | 1304.6ms/step | 15.3 samples/s +Step 28590/44000 | Loss: 2.9606 | LR: 1.41e-04 | 1303.2ms/step | 15.3 samples/s +Step 28600/44000 | Loss: 2.9586 | LR: 1.41e-04 | 1303.6ms/step | 15.3 samples/s +Step 28610/44000 | Loss: 2.9237 | LR: 1.41e-04 | 1304.1ms/step | 15.3 samples/s +Step 28620/44000 | Loss: 2.9090 | LR: 1.40e-04 | 1301.8ms/step | 15.4 samples/s +Step 28630/44000 | Loss: 2.9191 | LR: 1.40e-04 | 1302.3ms/step | 15.4 samples/s +Step 28640/44000 | Loss: 2.9014 | LR: 1.40e-04 | 1304.0ms/step | 15.3 samples/s +Step 28650/44000 | Loss: 2.9256 | LR: 1.40e-04 | 1306.8ms/step | 15.3 samples/s +Step 28660/44000 | Loss: 2.9214 | LR: 1.40e-04 | 1307.1ms/step | 15.3 samples/s +Step 28670/44000 | Loss: 2.8910 | LR: 1.40e-04 | 1303.8ms/step | 15.3 samples/s +Step 28680/44000 | Loss: 2.9402 | LR: 1.39e-04 | 1302.8ms/step | 15.4 samples/s +Step 28690/44000 | Loss: 2.9375 | LR: 1.39e-04 | 1302.4ms/step | 15.4 samples/s +Step 28700/44000 | Loss: 2.9398 | LR: 1.39e-04 | 1306.3ms/step | 15.3 samples/s +Step 28710/44000 | Loss: 2.9451 | LR: 1.39e-04 | 1307.0ms/step | 15.3 samples/s +Step 28720/44000 | Loss: 2.9198 | LR: 1.39e-04 | 1304.4ms/step | 15.3 samples/s +Step 28730/44000 | Loss: 2.9214 | LR: 1.39e-04 | 1304.3ms/step | 15.3 samples/s +Step 28740/44000 | Loss: 2.9461 | LR: 1.38e-04 | 1305.8ms/step | 15.3 samples/s +Step 28750/44000 | Loss: 2.9097 | LR: 1.38e-04 | 1305.2ms/step | 15.3 samples/s +Validation at step 28750: Loss = 2.9466, Perplexity = 19.04 +Step 28760/44000 | Loss: 2.9363 | LR: 1.38e-04 | 3485.1ms/step | 5.7 samples/s +Step 28770/44000 | Loss: 2.9037 | LR: 1.38e-04 | 1305.5ms/step | 15.3 samples/s +Step 28780/44000 | Loss: 2.9386 | LR: 1.38e-04 | 1305.5ms/step | 15.3 samples/s +Step 28790/44000 | Loss: 2.9478 | LR: 1.38e-04 | 1305.8ms/step | 15.3 samples/s +Step 28800/44000 | Loss: 2.9490 | LR: 1.38e-04 | 1306.2ms/step | 15.3 samples/s +Step 28810/44000 | Loss: 2.9347 | LR: 1.37e-04 | 1307.6ms/step | 15.3 samples/s +Step 28820/44000 | Loss: 2.9101 | LR: 1.37e-04 | 1304.4ms/step | 15.3 samples/s +Step 28830/44000 | Loss: 2.9419 | LR: 1.37e-04 | 1303.9ms/step | 15.3 samples/s +Step 28840/44000 | Loss: 2.9362 | LR: 1.37e-04 | 1305.1ms/step | 15.3 samples/s +Step 28850/44000 | Loss: 2.9137 | LR: 1.37e-04 | 1305.2ms/step | 15.3 samples/s +Step 28860/44000 | Loss: 2.9460 | LR: 1.37e-04 | 1304.4ms/step | 15.3 samples/s +Step 28870/44000 | Loss: 2.9505 | LR: 1.36e-04 | 1304.1ms/step | 15.3 samples/s +Step 28880/44000 | Loss: 2.9243 | LR: 1.36e-04 | 1305.1ms/step | 15.3 samples/s +Step 28890/44000 | Loss: 2.9357 | LR: 1.36e-04 | 1302.1ms/step | 15.4 samples/s +Step 28900/44000 | Loss: 2.9291 | LR: 1.36e-04 | 1304.4ms/step | 15.3 samples/s +Step 28910/44000 | Loss: 2.9266 | LR: 1.36e-04 | 1303.0ms/step | 15.3 samples/s +Step 28920/44000 | Loss: 2.9401 | LR: 1.36e-04 | 1304.4ms/step | 15.3 samples/s +Step 28930/44000 | Loss: 2.9319 | LR: 1.35e-04 | 1307.8ms/step | 15.3 samples/s +Step 28940/44000 | Loss: 2.9256 | LR: 1.35e-04 | 1307.1ms/step | 15.3 samples/s +Step 28950/44000 | Loss: 2.9700 | LR: 1.35e-04 | 1308.1ms/step | 15.3 samples/s +Step 28960/44000 | Loss: 2.9303 | LR: 1.35e-04 | 1307.0ms/step | 15.3 samples/s +Step 28970/44000 | Loss: 2.9524 | LR: 1.35e-04 | 1307.0ms/step | 15.3 samples/s +Step 28980/44000 | Loss: 2.9365 | LR: 1.35e-04 | 1309.2ms/step | 15.3 samples/s +Step 28990/44000 | Loss: 2.9266 | LR: 1.34e-04 | 1309.0ms/step | 15.3 samples/s +Step 29000/44000 | Loss: 2.9250 | LR: 1.34e-04 | 1304.8ms/step | 15.3 samples/s +Validation at step 29000: Loss = 2.9455, Perplexity = 19.02 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-29000 +Step 29010/44000 | Loss: 2.9188 | LR: 1.34e-04 | 3789.0ms/step | 5.3 samples/s +Step 29020/44000 | Loss: 2.9315 | LR: 1.34e-04 | 1303.9ms/step | 15.3 samples/s +Step 29030/44000 | Loss: 2.9177 | LR: 1.34e-04 | 1305.0ms/step | 15.3 samples/s +Step 29040/44000 | Loss: 2.9051 | LR: 1.34e-04 | 1303.7ms/step | 15.3 samples/s +Step 29050/44000 | Loss: 2.9117 | LR: 1.33e-04 | 1305.2ms/step | 15.3 samples/s +Step 29060/44000 | Loss: 2.9409 | LR: 1.33e-04 | 1303.4ms/step | 15.3 samples/s +Step 29070/44000 | Loss: 2.9351 | LR: 1.33e-04 | 1303.7ms/step | 15.3 samples/s +Step 29080/44000 | Loss: 2.9037 | LR: 1.33e-04 | 1304.2ms/step | 15.3 samples/s +Step 29090/44000 | Loss: 2.9377 | LR: 1.33e-04 | 1303.1ms/step | 15.3 samples/s +Step 29100/44000 | Loss: 2.9487 | LR: 1.33e-04 | 1303.0ms/step | 15.3 samples/s +Step 29110/44000 | Loss: 2.9468 | LR: 1.33e-04 | 1303.4ms/step | 15.3 samples/s +Step 29120/44000 | Loss: 2.9130 | LR: 1.32e-04 | 1300.8ms/step | 15.4 samples/s +Step 29130/44000 | Loss: 2.9400 | LR: 1.32e-04 | 1301.8ms/step | 15.4 samples/s +Step 29140/44000 | Loss: 2.9355 | LR: 1.32e-04 | 1302.4ms/step | 15.4 samples/s +Step 29150/44000 | Loss: 2.9508 | LR: 1.32e-04 | 1303.1ms/step | 15.3 samples/s +Step 29160/44000 | Loss: 2.9286 | LR: 1.32e-04 | 1303.0ms/step | 15.3 samples/s +Step 29170/44000 | Loss: 2.9058 | LR: 1.32e-04 | 1301.7ms/step | 15.4 samples/s +Step 29180/44000 | Loss: 2.9056 | LR: 1.31e-04 | 1301.2ms/step | 15.4 samples/s +Step 29190/44000 | Loss: 2.9226 | LR: 1.31e-04 | 1302.1ms/step | 15.4 samples/s +Step 29200/44000 | Loss: 2.9281 | LR: 1.31e-04 | 1301.2ms/step | 15.4 samples/s +Step 29210/44000 | Loss: 2.8829 | LR: 1.31e-04 | 1303.6ms/step | 15.3 samples/s +Step 29220/44000 | Loss: 2.9240 | LR: 1.31e-04 | 1305.5ms/step | 15.3 samples/s +Step 29230/44000 | Loss: 2.9146 | LR: 1.31e-04 | 1305.9ms/step | 15.3 samples/s +Step 29240/44000 | Loss: 2.9408 | LR: 1.30e-04 | 1304.6ms/step | 15.3 samples/s +Step 29250/44000 | Loss: 2.9232 | LR: 1.30e-04 | 1303.3ms/step | 15.3 samples/s +Validation at step 29250: Loss = 2.9445, Perplexity = 19.00 +Step 29260/44000 | Loss: 2.9105 | LR: 1.30e-04 | 3484.8ms/step | 5.7 samples/s +Step 29270/44000 | Loss: 2.9288 | LR: 1.30e-04 | 1300.9ms/step | 15.4 samples/s +Step 29280/44000 | Loss: 2.9185 | LR: 1.30e-04 | 1302.1ms/step | 15.4 samples/s +Step 29290/44000 | Loss: 2.9352 | LR: 1.30e-04 | 1300.0ms/step | 15.4 samples/s +Step 29300/44000 | Loss: 2.9389 | LR: 1.29e-04 | 1302.8ms/step | 15.4 samples/s +Step 29310/44000 | Loss: 2.9435 | LR: 1.29e-04 | 1304.3ms/step | 15.3 samples/s +Step 29320/44000 | Loss: 2.9385 | LR: 1.29e-04 | 1301.9ms/step | 15.4 samples/s +Step 29330/44000 | Loss: 2.9441 | LR: 1.29e-04 | 1301.1ms/step | 15.4 samples/s +Step 29340/44000 | Loss: 2.9022 | LR: 1.29e-04 | 1302.6ms/step | 15.4 samples/s +Step 29350/44000 | Loss: 2.9439 | LR: 1.29e-04 | 1302.6ms/step | 15.4 samples/s +Step 29360/44000 | Loss: 2.9479 | LR: 1.29e-04 | 1305.4ms/step | 15.3 samples/s +Step 29370/44000 | Loss: 2.9191 | LR: 1.28e-04 | 1304.8ms/step | 15.3 samples/s +Step 29380/44000 | Loss: 2.9279 | LR: 1.28e-04 | 1304.5ms/step | 15.3 samples/s +Step 29390/44000 | Loss: 2.9449 | LR: 1.28e-04 | 1302.7ms/step | 15.4 samples/s +Step 29400/44000 | Loss: 2.9311 | LR: 1.28e-04 | 1302.3ms/step | 15.4 samples/s +Step 29410/44000 | Loss: 2.9180 | LR: 1.28e-04 | 1301.8ms/step | 15.4 samples/s +Step 29420/44000 | Loss: 2.9605 | LR: 1.28e-04 | 1303.8ms/step | 15.3 samples/s +Step 29430/44000 | Loss: 2.9335 | LR: 1.27e-04 | 1302.2ms/step | 15.4 samples/s +Step 29440/44000 | Loss: 2.9339 | LR: 1.27e-04 | 1304.3ms/step | 15.3 samples/s +Step 29450/44000 | Loss: 2.9520 | LR: 1.27e-04 | 1302.6ms/step | 15.4 samples/s +Step 29460/44000 | Loss: 2.9394 | LR: 1.27e-04 | 1305.3ms/step | 15.3 samples/s +Step 29470/44000 | Loss: 2.9331 | LR: 1.27e-04 | 1306.9ms/step | 15.3 samples/s +Step 29480/44000 | Loss: 2.9320 | LR: 1.27e-04 | 1304.3ms/step | 15.3 samples/s +Step 29490/44000 | Loss: 2.9200 | LR: 1.26e-04 | 1304.4ms/step | 15.3 samples/s +Step 29500/44000 | Loss: 2.9165 | LR: 1.26e-04 | 1303.2ms/step | 15.3 samples/s +Validation at step 29500: Loss = 2.9437, Perplexity = 18.99 +Step 29510/44000 | Loss: 2.9219 | LR: 1.26e-04 | 3487.0ms/step | 5.7 samples/s +Step 29520/44000 | Loss: 2.9358 | LR: 1.26e-04 | 1303.7ms/step | 15.3 samples/s +Step 29530/44000 | Loss: 2.9144 | LR: 1.26e-04 | 1306.6ms/step | 15.3 samples/s +Step 29540/44000 | Loss: 2.9125 | LR: 1.26e-04 | 1302.4ms/step | 15.4 samples/s +Step 29550/44000 | Loss: 2.9188 | LR: 1.26e-04 | 1303.0ms/step | 15.3 samples/s +Step 29560/44000 | Loss: 2.9176 | LR: 1.25e-04 | 1306.7ms/step | 15.3 samples/s +Step 29570/44000 | Loss: 2.9257 | LR: 1.25e-04 | 1309.7ms/step | 15.3 samples/s +Step 29580/44000 | Loss: 2.9171 | LR: 1.25e-04 | 1307.6ms/step | 15.3 samples/s +Step 29590/44000 | Loss: 2.9458 | LR: 1.25e-04 | 1305.0ms/step | 15.3 samples/s +Step 29600/44000 | Loss: 2.9373 | LR: 1.25e-04 | 1303.2ms/step | 15.3 samples/s +Step 29610/44000 | Loss: 2.9280 | LR: 1.25e-04 | 1303.5ms/step | 15.3 samples/s +Step 29620/44000 | Loss: 2.9105 | LR: 1.24e-04 | 1303.9ms/step | 15.3 samples/s +Step 29630/44000 | Loss: 2.9512 | LR: 1.24e-04 | 1302.8ms/step | 15.4 samples/s +Step 29640/44000 | Loss: 2.9224 | LR: 1.24e-04 | 1300.6ms/step | 15.4 samples/s +Step 29650/44000 | Loss: 2.9150 | LR: 1.24e-04 | 1299.7ms/step | 15.4 samples/s +Step 29660/44000 | Loss: 2.9690 | LR: 1.24e-04 | 1301.2ms/step | 15.4 samples/s +Step 29670/44000 | Loss: 2.9135 | LR: 1.24e-04 | 1299.1ms/step | 15.4 samples/s +Step 29680/44000 | Loss: 2.9253 | LR: 1.23e-04 | 1299.9ms/step | 15.4 samples/s +Step 29690/44000 | Loss: 2.9342 | LR: 1.23e-04 | 1300.2ms/step | 15.4 samples/s +Step 29700/44000 | Loss: 2.9361 | LR: 1.23e-04 | 1299.9ms/step | 15.4 samples/s +Step 29710/44000 | Loss: 2.9463 | LR: 1.23e-04 | 1302.7ms/step | 15.4 samples/s +Step 29720/44000 | Loss: 2.8861 | LR: 1.23e-04 | 1302.2ms/step | 15.4 samples/s +Step 29730/44000 | Loss: 2.9374 | LR: 1.23e-04 | 1301.0ms/step | 15.4 samples/s +Step 29740/44000 | Loss: 2.9019 | LR: 1.23e-04 | 1304.2ms/step | 15.3 samples/s +Step 29750/44000 | Loss: 2.9198 | LR: 1.22e-04 | 1304.9ms/step | 15.3 samples/s +Validation at step 29750: Loss = 2.9426, Perplexity = 18.97 +Step 29760/44000 | Loss: 2.9476 | LR: 1.22e-04 | 3484.0ms/step | 5.7 samples/s +Step 29770/44000 | Loss: 2.9195 | LR: 1.22e-04 | 1301.6ms/step | 15.4 samples/s +Step 29780/44000 | Loss: 2.9283 | LR: 1.22e-04 | 1302.7ms/step | 15.4 samples/s +Step 29790/44000 | Loss: 2.8965 | LR: 1.22e-04 | 1305.8ms/step | 15.3 samples/s +Step 29800/44000 | Loss: 2.9060 | LR: 1.22e-04 | 1302.7ms/step | 15.4 samples/s +Step 29810/44000 | Loss: 2.9119 | LR: 1.21e-04 | 1301.9ms/step | 15.4 samples/s +Step 29820/44000 | Loss: 2.9566 | LR: 1.21e-04 | 1301.6ms/step | 15.4 samples/s +Step 29830/44000 | Loss: 2.9442 | LR: 1.21e-04 | 1304.2ms/step | 15.3 samples/s +Step 29840/44000 | Loss: 2.8941 | LR: 1.21e-04 | 1302.8ms/step | 15.4 samples/s +Step 29850/44000 | Loss: 2.9178 | LR: 1.21e-04 | 1300.1ms/step | 15.4 samples/s +Step 29860/44000 | Loss: 2.9322 | LR: 1.21e-04 | 1300.1ms/step | 15.4 samples/s +Step 29870/44000 | Loss: 2.9015 | LR: 1.21e-04 | 1301.7ms/step | 15.4 samples/s +Step 29880/44000 | Loss: 2.9204 | LR: 1.20e-04 | 1306.2ms/step | 15.3 samples/s +Step 29890/44000 | Loss: 2.9238 | LR: 1.20e-04 | 1305.1ms/step | 15.3 samples/s +Step 29900/44000 | Loss: 2.9642 | LR: 1.20e-04 | 1304.2ms/step | 15.3 samples/s +Step 29910/44000 | Loss: 2.9218 | LR: 1.20e-04 | 1308.3ms/step | 15.3 samples/s +Step 29920/44000 | Loss: 2.9111 | LR: 1.20e-04 | 1308.8ms/step | 15.3 samples/s +Step 29930/44000 | Loss: 2.9183 | LR: 1.20e-04 | 1304.2ms/step | 15.3 samples/s +Step 29940/44000 | Loss: 2.9135 | LR: 1.19e-04 | 1301.2ms/step | 15.4 samples/s +Step 29950/44000 | Loss: 2.9237 | LR: 1.19e-04 | 1305.2ms/step | 15.3 samples/s +Step 29960/44000 | Loss: 2.9054 | LR: 1.19e-04 | 1303.1ms/step | 15.3 samples/s +Step 29970/44000 | Loss: 2.9187 | LR: 1.19e-04 | 1302.1ms/step | 15.4 samples/s +Step 29980/44000 | Loss: 2.9152 | LR: 1.19e-04 | 1301.2ms/step | 15.4 samples/s +Step 29990/44000 | Loss: 2.9306 | LR: 1.19e-04 | 1301.8ms/step | 15.4 samples/s +Step 30000/44000 | Loss: 2.9079 | LR: 1.19e-04 | 1300.0ms/step | 15.4 samples/s +Validation at step 30000: Loss = 2.9417, Perplexity = 18.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-30000 +Step 30010/44000 | Loss: 2.9365 | LR: 1.18e-04 | 3799.9ms/step | 5.3 samples/s +Step 30020/44000 | Loss: 2.9473 | LR: 1.18e-04 | 1302.4ms/step | 15.4 samples/s +Step 30030/44000 | Loss: 2.9166 | LR: 1.18e-04 | 1302.5ms/step | 15.4 samples/s +Step 30040/44000 | Loss: 2.9339 | LR: 1.18e-04 | 1302.0ms/step | 15.4 samples/s +Step 30050/44000 | Loss: 2.9085 | LR: 1.18e-04 | 1302.8ms/step | 15.4 samples/s +Step 30060/44000 | Loss: 2.9320 | LR: 1.18e-04 | 1300.0ms/step | 15.4 samples/s +Step 30070/44000 | Loss: 2.8869 | LR: 1.17e-04 | 1300.1ms/step | 15.4 samples/s +Step 30080/44000 | Loss: 2.9584 | LR: 1.17e-04 | 1299.9ms/step | 15.4 samples/s +Step 30090/44000 | Loss: 2.9366 | LR: 1.17e-04 | 1299.8ms/step | 15.4 samples/s +Step 30100/44000 | Loss: 2.9355 | LR: 1.17e-04 | 1300.1ms/step | 15.4 samples/s +Step 30110/44000 | Loss: 2.9209 | LR: 1.17e-04 | 1300.3ms/step | 15.4 samples/s +Step 30120/44000 | Loss: 2.9107 | LR: 1.17e-04 | 1301.6ms/step | 15.4 samples/s +Step 30130/44000 | Loss: 2.9390 | LR: 1.17e-04 | 1309.3ms/step | 15.3 samples/s +Step 30140/44000 | Loss: 2.9292 | LR: 1.16e-04 | 1309.8ms/step | 15.3 samples/s +Step 30150/44000 | Loss: 2.9126 | LR: 1.16e-04 | 1305.3ms/step | 15.3 samples/s +Step 30160/44000 | Loss: 2.8976 | LR: 1.16e-04 | 1302.7ms/step | 15.4 samples/s +Step 30170/44000 | Loss: 2.9001 | LR: 1.16e-04 | 1305.3ms/step | 15.3 samples/s +Step 30180/44000 | Loss: 2.9367 | LR: 1.16e-04 | 1304.1ms/step | 15.3 samples/s +Step 30190/44000 | Loss: 2.9133 | LR: 1.16e-04 | 1304.4ms/step | 15.3 samples/s +Step 30200/44000 | Loss: 2.9220 | LR: 1.15e-04 | 1301.8ms/step | 15.4 samples/s +Step 30210/44000 | Loss: 2.9221 | LR: 1.15e-04 | 1303.7ms/step | 15.3 samples/s +Step 30220/44000 | Loss: 2.8987 | LR: 1.15e-04 | 1304.3ms/step | 15.3 samples/s +Step 30230/44000 | Loss: 2.9441 | LR: 1.15e-04 | 1307.4ms/step | 15.3 samples/s +Step 30240/44000 | Loss: 2.9246 | LR: 1.15e-04 | 1306.1ms/step | 15.3 samples/s +Step 30250/44000 | Loss: 2.9129 | LR: 1.15e-04 | 1305.1ms/step | 15.3 samples/s +Validation at step 30250: Loss = 2.9406, Perplexity = 18.93 +Step 30260/44000 | Loss: 2.9165 | LR: 1.15e-04 | 3481.9ms/step | 5.7 samples/s +Step 30270/44000 | Loss: 2.9063 | LR: 1.14e-04 | 1301.9ms/step | 15.4 samples/s +Step 30280/44000 | Loss: 2.9499 | LR: 1.14e-04 | 1299.9ms/step | 15.4 samples/s +Step 30290/44000 | Loss: 2.9069 | LR: 1.14e-04 | 1301.1ms/step | 15.4 samples/s +Step 30300/44000 | Loss: 2.8911 | LR: 1.14e-04 | 1303.0ms/step | 15.3 samples/s +Step 30310/44000 | Loss: 2.8995 | LR: 1.14e-04 | 1303.5ms/step | 15.3 samples/s +Step 30320/44000 | Loss: 2.9009 | LR: 1.14e-04 | 1303.7ms/step | 15.3 samples/s +Step 30330/44000 | Loss: 2.9172 | LR: 1.13e-04 | 1303.7ms/step | 15.3 samples/s +Step 30340/44000 | Loss: 2.9238 | LR: 1.13e-04 | 1304.5ms/step | 15.3 samples/s +Step 30350/44000 | Loss: 2.8888 | LR: 1.13e-04 | 1306.9ms/step | 15.3 samples/s +Step 30360/44000 | Loss: 2.9247 | LR: 1.13e-04 | 1309.0ms/step | 15.3 samples/s +Step 30370/44000 | Loss: 2.9363 | LR: 1.13e-04 | 1315.3ms/step | 15.2 samples/s +Step 30380/44000 | Loss: 2.9454 | LR: 1.13e-04 | 1312.5ms/step | 15.2 samples/s +Step 30390/44000 | Loss: 2.9177 | LR: 1.13e-04 | 1310.6ms/step | 15.3 samples/s +Step 30400/44000 | Loss: 2.9185 | LR: 1.12e-04 | 1312.4ms/step | 15.2 samples/s +Step 30410/44000 | Loss: 2.9262 | LR: 1.12e-04 | 1312.8ms/step | 15.2 samples/s +Step 30420/44000 | Loss: 2.9211 | LR: 1.12e-04 | 1313.5ms/step | 15.2 samples/s +Step 30430/44000 | Loss: 2.9222 | LR: 1.12e-04 | 1313.3ms/step | 15.2 samples/s +Step 30440/44000 | Loss: 2.9250 | LR: 1.12e-04 | 1316.4ms/step | 15.2 samples/s +Step 30450/44000 | Loss: 2.8956 | LR: 1.12e-04 | 1315.0ms/step | 15.2 samples/s +Step 30460/44000 | Loss: 2.9256 | LR: 1.11e-04 | 1309.8ms/step | 15.3 samples/s +Step 30470/44000 | Loss: 2.9370 | LR: 1.11e-04 | 1311.2ms/step | 15.3 samples/s +Step 30480/44000 | Loss: 2.9263 | LR: 1.11e-04 | 1308.4ms/step | 15.3 samples/s +Step 30490/44000 | Loss: 2.9188 | LR: 1.11e-04 | 1308.0ms/step | 15.3 samples/s +Step 30500/44000 | Loss: 2.9144 | LR: 1.11e-04 | 1307.3ms/step | 15.3 samples/s +Validation at step 30500: Loss = 2.9399, Perplexity = 18.91 +Step 30510/44000 | Loss: 2.9127 | LR: 1.11e-04 | 3493.7ms/step | 5.7 samples/s +Step 30520/44000 | Loss: 2.9187 | LR: 1.11e-04 | 1309.7ms/step | 15.3 samples/s +Step 30530/44000 | Loss: 2.9590 | LR: 1.10e-04 | 1308.4ms/step | 15.3 samples/s +Step 30540/44000 | Loss: 2.9565 | LR: 1.10e-04 | 1308.0ms/step | 15.3 samples/s +Step 30550/44000 | Loss: 2.9263 | LR: 1.10e-04 | 1308.6ms/step | 15.3 samples/s +Step 30560/44000 | Loss: 2.9276 | LR: 1.10e-04 | 1308.0ms/step | 15.3 samples/s +Step 30570/44000 | Loss: 2.9395 | LR: 1.10e-04 | 1308.4ms/step | 15.3 samples/s +Step 30580/44000 | Loss: 2.9352 | LR: 1.10e-04 | 1308.9ms/step | 15.3 samples/s +Step 30590/44000 | Loss: 2.9236 | LR: 1.10e-04 | 1308.1ms/step | 15.3 samples/s +Step 30600/44000 | Loss: 2.9188 | LR: 1.09e-04 | 1309.4ms/step | 15.3 samples/s +Step 30610/44000 | Loss: 2.9227 | LR: 1.09e-04 | 1309.6ms/step | 15.3 samples/s +Step 30620/44000 | Loss: 2.9318 | LR: 1.09e-04 | 1308.6ms/step | 15.3 samples/s +Step 30630/44000 | Loss: 2.9078 | LR: 1.09e-04 | 1308.4ms/step | 15.3 samples/s +Step 30640/44000 | Loss: 2.9344 | LR: 1.09e-04 | 1309.0ms/step | 15.3 samples/s +Step 30650/44000 | Loss: 2.9436 | LR: 1.09e-04 | 1309.4ms/step | 15.3 samples/s +Step 30660/44000 | Loss: 2.9593 | LR: 1.08e-04 | 1310.4ms/step | 15.3 samples/s +Step 30670/44000 | Loss: 2.9470 | LR: 1.08e-04 | 1310.3ms/step | 15.3 samples/s +Step 30680/44000 | Loss: 2.9301 | LR: 1.08e-04 | 1309.5ms/step | 15.3 samples/s +Step 30690/44000 | Loss: 2.9147 | LR: 1.08e-04 | 1308.4ms/step | 15.3 samples/s +Step 30700/44000 | Loss: 2.9370 | LR: 1.08e-04 | 1313.6ms/step | 15.2 samples/s +Step 30710/44000 | Loss: 2.9186 | LR: 1.08e-04 | 1309.8ms/step | 15.3 samples/s +Step 30720/44000 | Loss: 2.9352 | LR: 1.08e-04 | 1311.3ms/step | 15.3 samples/s +Step 30730/44000 | Loss: 2.9185 | LR: 1.07e-04 | 1306.6ms/step | 15.3 samples/s +Step 30740/44000 | Loss: 2.9370 | LR: 1.07e-04 | 1306.6ms/step | 15.3 samples/s +Step 30750/44000 | Loss: 2.9210 | LR: 1.07e-04 | 1306.8ms/step | 15.3 samples/s +Validation at step 30750: Loss = 2.9391, Perplexity = 18.90 +Step 30760/44000 | Loss: 2.9254 | LR: 1.07e-04 | 3483.7ms/step | 5.7 samples/s +Step 30770/44000 | Loss: 2.9663 | LR: 1.07e-04 | 1306.4ms/step | 15.3 samples/s +Step 30780/44000 | Loss: 2.9165 | LR: 1.07e-04 | 1308.7ms/step | 15.3 samples/s +Step 30790/44000 | Loss: 2.9217 | LR: 1.07e-04 | 1308.4ms/step | 15.3 samples/s +Step 30800/44000 | Loss: 2.9308 | LR: 1.06e-04 | 1309.9ms/step | 15.3 samples/s +Step 30810/44000 | Loss: 2.9002 | LR: 1.06e-04 | 1312.7ms/step | 15.2 samples/s +Step 30820/44000 | Loss: 2.9294 | LR: 1.06e-04 | 1309.9ms/step | 15.3 samples/s +Step 30830/44000 | Loss: 2.9686 | LR: 1.06e-04 | 1308.0ms/step | 15.3 samples/s +Step 30840/44000 | Loss: 2.9385 | LR: 1.06e-04 | 1308.0ms/step | 15.3 samples/s +Step 30850/44000 | Loss: 2.9118 | LR: 1.06e-04 | 1309.5ms/step | 15.3 samples/s +Step 30860/44000 | Loss: 2.9411 | LR: 1.05e-04 | 1309.4ms/step | 15.3 samples/s +Step 30870/44000 | Loss: 2.9001 | LR: 1.05e-04 | 1308.6ms/step | 15.3 samples/s +Step 30880/44000 | Loss: 2.9172 | LR: 1.05e-04 | 1307.7ms/step | 15.3 samples/s +Step 30890/44000 | Loss: 2.8711 | LR: 1.05e-04 | 1307.6ms/step | 15.3 samples/s +Step 30900/44000 | Loss: 2.8998 | LR: 1.05e-04 | 1308.4ms/step | 15.3 samples/s +Step 30910/44000 | Loss: 2.9128 | LR: 1.05e-04 | 1310.8ms/step | 15.3 samples/s +Step 30920/44000 | Loss: 2.9288 | LR: 1.05e-04 | 1311.1ms/step | 15.3 samples/s +Step 30930/44000 | Loss: 2.9143 | LR: 1.04e-04 | 1310.2ms/step | 15.3 samples/s +Step 30940/44000 | Loss: 2.9013 | LR: 1.04e-04 | 1308.4ms/step | 15.3 samples/s +Step 30950/44000 | Loss: 2.9239 | LR: 1.04e-04 | 1309.7ms/step | 15.3 samples/s +Step 30960/44000 | Loss: 2.9174 | LR: 1.04e-04 | 1311.4ms/step | 15.3 samples/s +Step 30970/44000 | Loss: 2.9580 | LR: 1.04e-04 | 1311.4ms/step | 15.3 samples/s +Step 30980/44000 | Loss: 2.9043 | LR: 1.04e-04 | 1309.9ms/step | 15.3 samples/s +Step 30990/44000 | Loss: 2.9087 | LR: 1.04e-04 | 1310.3ms/step | 15.3 samples/s +Step 31000/44000 | Loss: 2.8965 | LR: 1.03e-04 | 1313.2ms/step | 15.2 samples/s +Validation at step 31000: Loss = 2.9383, Perplexity = 18.88 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-31000 +Step 31010/44000 | Loss: 2.9060 | LR: 1.03e-04 | 3820.5ms/step | 5.2 samples/s +Step 31020/44000 | Loss: 2.9362 | LR: 1.03e-04 | 1309.0ms/step | 15.3 samples/s +Step 31030/44000 | Loss: 2.8880 | LR: 1.03e-04 | 1309.8ms/step | 15.3 samples/s +Step 31040/44000 | Loss: 2.9180 | LR: 1.03e-04 | 1308.2ms/step | 15.3 samples/s +Step 31050/44000 | Loss: 2.9305 | LR: 1.03e-04 | 1309.6ms/step | 15.3 samples/s +Step 31060/44000 | Loss: 2.9364 | LR: 1.03e-04 | 1310.3ms/step | 15.3 samples/s +Step 31070/44000 | Loss: 2.9250 | LR: 1.02e-04 | 1309.9ms/step | 15.3 samples/s +Step 31080/44000 | Loss: 2.9160 | LR: 1.02e-04 | 1308.4ms/step | 15.3 samples/s +Step 31090/44000 | Loss: 2.9135 | LR: 1.02e-04 | 1308.0ms/step | 15.3 samples/s +Step 31100/44000 | Loss: 2.9053 | LR: 1.02e-04 | 1307.2ms/step | 15.3 samples/s +Step 31110/44000 | Loss: 2.9081 | LR: 1.02e-04 | 1305.7ms/step | 15.3 samples/s +Step 31120/44000 | Loss: 2.9255 | LR: 1.02e-04 | 1306.9ms/step | 15.3 samples/s +Step 31130/44000 | Loss: 2.9563 | LR: 1.02e-04 | 1307.3ms/step | 15.3 samples/s +Step 31140/44000 | Loss: 2.9086 | LR: 1.01e-04 | 1306.9ms/step | 15.3 samples/s +Step 31150/44000 | Loss: 2.9544 | LR: 1.01e-04 | 1309.6ms/step | 15.3 samples/s +Step 31160/44000 | Loss: 2.9260 | LR: 1.01e-04 | 1311.2ms/step | 15.3 samples/s +Step 31170/44000 | Loss: 2.9199 | LR: 1.01e-04 | 1309.6ms/step | 15.3 samples/s +Step 31180/44000 | Loss: 2.9277 | LR: 1.01e-04 | 1309.4ms/step | 15.3 samples/s +Step 31190/44000 | Loss: 2.8989 | LR: 1.01e-04 | 1309.9ms/step | 15.3 samples/s +Step 31200/44000 | Loss: 2.9128 | LR: 1.00e-04 | 1309.6ms/step | 15.3 samples/s +Step 31210/44000 | Loss: 2.9315 | LR: 1.00e-04 | 1310.2ms/step | 15.3 samples/s +Step 31220/44000 | Loss: 2.9179 | LR: 1.00e-04 | 1310.0ms/step | 15.3 samples/s +Step 31230/44000 | Loss: 2.9532 | LR: 1.00e-04 | 1309.2ms/step | 15.3 samples/s +Step 31240/44000 | Loss: 2.9195 | LR: 9.99e-05 | 1307.4ms/step | 15.3 samples/s +Step 31250/44000 | Loss: 2.9036 | LR: 9.98e-05 | 1303.2ms/step | 15.3 samples/s +Validation at step 31250: Loss = 2.9379, Perplexity = 18.88 +Step 31260/44000 | Loss: 2.9343 | LR: 9.96e-05 | 3491.5ms/step | 5.7 samples/s +Step 31270/44000 | Loss: 2.9487 | LR: 9.95e-05 | 1308.3ms/step | 15.3 samples/s +Step 31280/44000 | Loss: 2.9092 | LR: 9.93e-05 | 1309.3ms/step | 15.3 samples/s +Step 31290/44000 | Loss: 2.9364 | LR: 9.92e-05 | 1308.8ms/step | 15.3 samples/s +Step 31300/44000 | Loss: 2.9341 | LR: 9.90e-05 | 1308.5ms/step | 15.3 samples/s +Step 31310/44000 | Loss: 2.9226 | LR: 9.89e-05 | 1312.7ms/step | 15.2 samples/s +Step 31320/44000 | Loss: 2.9175 | LR: 9.88e-05 | 1312.7ms/step | 15.2 samples/s +Step 31330/44000 | Loss: 2.9465 | LR: 9.86e-05 | 1309.6ms/step | 15.3 samples/s +Step 31340/44000 | Loss: 2.9131 | LR: 9.85e-05 | 1309.2ms/step | 15.3 samples/s +Step 31350/44000 | Loss: 2.8929 | LR: 9.83e-05 | 1311.2ms/step | 15.3 samples/s +Step 31360/44000 | Loss: 2.9018 | LR: 9.82e-05 | 1310.2ms/step | 15.3 samples/s +Step 31370/44000 | Loss: 2.9352 | LR: 9.80e-05 | 1311.1ms/step | 15.3 samples/s +Step 31380/44000 | Loss: 2.9481 | LR: 9.79e-05 | 1308.1ms/step | 15.3 samples/s +Step 31390/44000 | Loss: 2.8869 | LR: 9.77e-05 | 1309.2ms/step | 15.3 samples/s +Step 31400/44000 | Loss: 2.9203 | LR: 9.76e-05 | 1308.1ms/step | 15.3 samples/s +Step 31410/44000 | Loss: 2.8923 | LR: 9.75e-05 | 1310.7ms/step | 15.3 samples/s +Step 31420/44000 | Loss: 2.9120 | LR: 9.73e-05 | 1317.3ms/step | 15.2 samples/s +Step 31430/44000 | Loss: 2.9328 | LR: 9.72e-05 | 1318.2ms/step | 15.2 samples/s +Step 31440/44000 | Loss: 2.9173 | LR: 9.70e-05 | 1316.0ms/step | 15.2 samples/s +Step 31450/44000 | Loss: 2.9295 | LR: 9.69e-05 | 1317.0ms/step | 15.2 samples/s +Step 31460/44000 | Loss: 2.9078 | LR: 9.67e-05 | 1311.7ms/step | 15.2 samples/s +Step 31470/44000 | Loss: 2.9348 | LR: 9.66e-05 | 1307.8ms/step | 15.3 samples/s +Step 31480/44000 | Loss: 2.9006 | LR: 9.65e-05 | 1307.2ms/step | 15.3 samples/s +Step 31490/44000 | Loss: 2.9087 | LR: 9.63e-05 | 1309.2ms/step | 15.3 samples/s +Step 31500/44000 | Loss: 2.9213 | LR: 9.62e-05 | 1310.5ms/step | 15.3 samples/s +Validation at step 31500: Loss = 2.9370, Perplexity = 18.86 +Step 31510/44000 | Loss: 2.9027 | LR: 9.60e-05 | 3492.3ms/step | 5.7 samples/s +Step 31520/44000 | Loss: 2.9142 | LR: 9.59e-05 | 1309.6ms/step | 15.3 samples/s +Step 31530/44000 | Loss: 2.9268 | LR: 9.57e-05 | 1309.7ms/step | 15.3 samples/s +Step 31540/44000 | Loss: 2.9080 | LR: 9.56e-05 | 1308.6ms/step | 15.3 samples/s +Step 31550/44000 | Loss: 2.9341 | LR: 9.54e-05 | 1308.3ms/step | 15.3 samples/s +Step 31560/44000 | Loss: 2.9187 | LR: 9.53e-05 | 1309.9ms/step | 15.3 samples/s +Step 31570/44000 | Loss: 2.9092 | LR: 9.52e-05 | 1313.1ms/step | 15.2 samples/s +Step 31580/44000 | Loss: 2.9333 | LR: 9.50e-05 | 1310.1ms/step | 15.3 samples/s +Step 31590/44000 | Loss: 2.9344 | LR: 9.49e-05 | 1311.5ms/step | 15.3 samples/s +Step 31600/44000 | Loss: 2.9168 | LR: 9.47e-05 | 1311.4ms/step | 15.3 samples/s +Step 31610/44000 | Loss: 2.9061 | LR: 9.46e-05 | 1312.7ms/step | 15.2 samples/s +Step 31620/44000 | Loss: 2.9294 | LR: 9.45e-05 | 1312.1ms/step | 15.2 samples/s +Step 31630/44000 | Loss: 2.9130 | LR: 9.43e-05 | 1310.0ms/step | 15.3 samples/s +Step 31640/44000 | Loss: 2.8869 | LR: 9.42e-05 | 1310.9ms/step | 15.3 samples/s +Step 31650/44000 | Loss: 2.8918 | LR: 9.40e-05 | 1311.5ms/step | 15.3 samples/s +Step 31660/44000 | Loss: 2.9078 | LR: 9.39e-05 | 1311.2ms/step | 15.3 samples/s +Step 31670/44000 | Loss: 2.9013 | LR: 9.37e-05 | 1315.4ms/step | 15.2 samples/s +Step 31680/44000 | Loss: 2.9144 | LR: 9.36e-05 | 1311.9ms/step | 15.2 samples/s +Step 31690/44000 | Loss: 2.9230 | LR: 9.35e-05 | 1311.3ms/step | 15.3 samples/s +Step 31700/44000 | Loss: 2.9386 | LR: 9.33e-05 | 1310.1ms/step | 15.3 samples/s +Step 31710/44000 | Loss: 2.9143 | LR: 9.32e-05 | 1309.7ms/step | 15.3 samples/s +Step 31720/44000 | Loss: 2.9151 | LR: 9.30e-05 | 1311.5ms/step | 15.2 samples/s +Step 31730/44000 | Loss: 2.8859 | LR: 9.29e-05 | 1313.5ms/step | 15.2 samples/s +Step 31740/44000 | Loss: 2.9212 | LR: 9.28e-05 | 1311.4ms/step | 15.3 samples/s +Step 31750/44000 | Loss: 2.9290 | LR: 9.26e-05 | 1310.6ms/step | 15.3 samples/s +Validation at step 31750: Loss = 2.9364, Perplexity = 18.85 +Step 31760/44000 | Loss: 2.9048 | LR: 9.25e-05 | 3495.7ms/step | 5.7 samples/s +Step 31770/44000 | Loss: 2.9288 | LR: 9.23e-05 | 1310.3ms/step | 15.3 samples/s +Step 31780/44000 | Loss: 2.9195 | LR: 9.22e-05 | 1312.1ms/step | 15.2 samples/s +Step 31790/44000 | Loss: 2.9231 | LR: 9.20e-05 | 1309.4ms/step | 15.3 samples/s +Step 31800/44000 | Loss: 2.8934 | LR: 9.19e-05 | 1306.1ms/step | 15.3 samples/s +Step 31810/44000 | Loss: 2.8924 | LR: 9.18e-05 | 1305.3ms/step | 15.3 samples/s +Step 31820/44000 | Loss: 2.9106 | LR: 9.16e-05 | 1307.7ms/step | 15.3 samples/s +Step 31830/44000 | Loss: 2.9160 | LR: 9.15e-05 | 1306.2ms/step | 15.3 samples/s +Step 31840/44000 | Loss: 2.9285 | LR: 9.13e-05 | 1309.7ms/step | 15.3 samples/s +Step 31850/44000 | Loss: 2.9320 | LR: 9.12e-05 | 1306.8ms/step | 15.3 samples/s +Step 31860/44000 | Loss: 2.9377 | LR: 9.11e-05 | 1307.2ms/step | 15.3 samples/s +Step 31870/44000 | Loss: 2.9147 | LR: 9.09e-05 | 1305.9ms/step | 15.3 samples/s +Step 31880/44000 | Loss: 2.9500 | LR: 9.08e-05 | 1304.0ms/step | 15.3 samples/s +Step 31890/44000 | Loss: 2.8894 | LR: 9.06e-05 | 1307.7ms/step | 15.3 samples/s +Step 31900/44000 | Loss: 2.9098 | LR: 9.05e-05 | 1310.2ms/step | 15.3 samples/s +Step 31910/44000 | Loss: 2.9413 | LR: 9.04e-05 | 1307.9ms/step | 15.3 samples/s +Step 31920/44000 | Loss: 2.8673 | LR: 9.02e-05 | 1307.1ms/step | 15.3 samples/s +Step 31930/44000 | Loss: 2.9251 | LR: 9.01e-05 | 1309.6ms/step | 15.3 samples/s +Step 31940/44000 | Loss: 2.9328 | LR: 8.99e-05 | 1311.7ms/step | 15.2 samples/s +Step 31950/44000 | Loss: 2.9219 | LR: 8.98e-05 | 1305.6ms/step | 15.3 samples/s +Step 31960/44000 | Loss: 2.9367 | LR: 8.97e-05 | 1305.0ms/step | 15.3 samples/s +Step 31970/44000 | Loss: 2.9054 | LR: 8.95e-05 | 1306.7ms/step | 15.3 samples/s +Step 31980/44000 | Loss: 2.9167 | LR: 8.94e-05 | 1309.5ms/step | 15.3 samples/s +Step 31990/44000 | Loss: 2.8835 | LR: 8.92e-05 | 1309.5ms/step | 15.3 samples/s +Step 32000/44000 | Loss: 2.9067 | LR: 8.91e-05 | 1309.5ms/step | 15.3 samples/s +Validation at step 32000: Loss = 2.9358, Perplexity = 18.84 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-32000 +Step 32010/44000 | Loss: 2.9288 | LR: 8.90e-05 | 3812.5ms/step | 5.2 samples/s +Step 32020/44000 | Loss: 2.9169 | LR: 8.88e-05 | 1307.6ms/step | 15.3 samples/s +Step 32030/44000 | Loss: 2.9248 | LR: 8.87e-05 | 1307.0ms/step | 15.3 samples/s +Step 32040/44000 | Loss: 2.9278 | LR: 8.86e-05 | 1305.9ms/step | 15.3 samples/s +Step 32050/44000 | Loss: 2.9023 | LR: 8.84e-05 | 1309.9ms/step | 15.3 samples/s +Step 32060/44000 | Loss: 2.8883 | LR: 8.83e-05 | 1309.9ms/step | 15.3 samples/s +Step 32070/44000 | Loss: 2.9124 | LR: 8.81e-05 | 1309.0ms/step | 15.3 samples/s +Step 32080/44000 | Loss: 2.8881 | LR: 8.80e-05 | 1308.9ms/step | 15.3 samples/s +Step 32090/44000 | Loss: 2.9222 | LR: 8.79e-05 | 1308.3ms/step | 15.3 samples/s +Step 32100/44000 | Loss: 2.9222 | LR: 8.77e-05 | 1307.9ms/step | 15.3 samples/s +Step 32110/44000 | Loss: 2.9060 | LR: 8.76e-05 | 1307.9ms/step | 15.3 samples/s +Step 32120/44000 | Loss: 2.9094 | LR: 8.74e-05 | 1307.4ms/step | 15.3 samples/s +Step 32130/44000 | Loss: 2.9047 | LR: 8.73e-05 | 1307.9ms/step | 15.3 samples/s +Step 32140/44000 | Loss: 2.9026 | LR: 8.72e-05 | 1308.7ms/step | 15.3 samples/s +Step 32150/44000 | Loss: 2.9196 | LR: 8.70e-05 | 1307.4ms/step | 15.3 samples/s +Step 32160/44000 | Loss: 2.8893 | LR: 8.69e-05 | 1309.7ms/step | 15.3 samples/s +Step 32170/44000 | Loss: 2.9153 | LR: 8.68e-05 | 1305.6ms/step | 15.3 samples/s +Step 32180/44000 | Loss: 2.9459 | LR: 8.66e-05 | 1307.2ms/step | 15.3 samples/s +Step 32190/44000 | Loss: 2.9185 | LR: 8.65e-05 | 1309.1ms/step | 15.3 samples/s +Step 32200/44000 | Loss: 2.9325 | LR: 8.63e-05 | 1308.0ms/step | 15.3 samples/s +Step 32210/44000 | Loss: 2.9374 | LR: 8.62e-05 | 1309.2ms/step | 15.3 samples/s +Step 32220/44000 | Loss: 2.8930 | LR: 8.61e-05 | 1308.7ms/step | 15.3 samples/s +Step 32230/44000 | Loss: 2.9217 | LR: 8.59e-05 | 1308.7ms/step | 15.3 samples/s +Step 32240/44000 | Loss: 2.9098 | LR: 8.58e-05 | 1308.0ms/step | 15.3 samples/s +Step 32250/44000 | Loss: 2.9298 | LR: 8.57e-05 | 1308.9ms/step | 15.3 samples/s +Validation at step 32250: Loss = 2.9353, Perplexity = 18.83 +Step 32260/44000 | Loss: 2.9428 | LR: 8.55e-05 | 3489.6ms/step | 5.7 samples/s +Step 32270/44000 | Loss: 2.9171 | LR: 8.54e-05 | 1310.6ms/step | 15.3 samples/s +Step 32280/44000 | Loss: 2.9387 | LR: 8.53e-05 | 1309.9ms/step | 15.3 samples/s +Step 32290/44000 | Loss: 2.9061 | LR: 8.51e-05 | 1306.9ms/step | 15.3 samples/s +Step 32300/44000 | Loss: 2.9032 | LR: 8.50e-05 | 1309.9ms/step | 15.3 samples/s +Step 32310/44000 | Loss: 2.9255 | LR: 8.48e-05 | 1308.6ms/step | 15.3 samples/s +Step 32320/44000 | Loss: 2.9445 | LR: 8.47e-05 | 1308.8ms/step | 15.3 samples/s +Step 32330/44000 | Loss: 2.9130 | LR: 8.46e-05 | 1306.5ms/step | 15.3 samples/s +Step 32340/44000 | Loss: 2.9108 | LR: 8.44e-05 | 1308.1ms/step | 15.3 samples/s +Step 32350/44000 | Loss: 2.9330 | LR: 8.43e-05 | 1307.6ms/step | 15.3 samples/s +Step 32360/44000 | Loss: 2.9140 | LR: 8.42e-05 | 1309.1ms/step | 15.3 samples/s +Step 32370/44000 | Loss: 2.9117 | LR: 8.40e-05 | 1311.0ms/step | 15.3 samples/s +Step 32380/44000 | Loss: 2.9117 | LR: 8.39e-05 | 1308.7ms/step | 15.3 samples/s +Step 32390/44000 | Loss: 2.9132 | LR: 8.38e-05 | 1307.8ms/step | 15.3 samples/s +Step 32400/44000 | Loss: 2.9207 | LR: 8.36e-05 | 1304.5ms/step | 15.3 samples/s +Step 32410/44000 | Loss: 2.9141 | LR: 8.35e-05 | 1304.7ms/step | 15.3 samples/s +Step 32420/44000 | Loss: 2.9129 | LR: 8.33e-05 | 1303.7ms/step | 15.3 samples/s +Step 32430/44000 | Loss: 2.9608 | LR: 8.32e-05 | 1303.1ms/step | 15.3 samples/s +Step 32440/44000 | Loss: 2.9263 | LR: 8.31e-05 | 1308.2ms/step | 15.3 samples/s +Step 32450/44000 | Loss: 2.9199 | LR: 8.29e-05 | 1310.2ms/step | 15.3 samples/s +Step 32460/44000 | Loss: 2.9402 | LR: 8.28e-05 | 1311.9ms/step | 15.2 samples/s +Step 32470/44000 | Loss: 2.8961 | LR: 8.27e-05 | 1309.9ms/step | 15.3 samples/s +Step 32480/44000 | Loss: 2.9289 | LR: 8.25e-05 | 1311.2ms/step | 15.3 samples/s +Step 32490/44000 | Loss: 2.9063 | LR: 8.24e-05 | 1313.4ms/step | 15.2 samples/s +Step 32500/44000 | Loss: 2.9246 | LR: 8.23e-05 | 1313.5ms/step | 15.2 samples/s +Validation at step 32500: Loss = 2.9348, Perplexity = 18.82 +Step 32510/44000 | Loss: 2.8924 | LR: 8.21e-05 | 3492.1ms/step | 5.7 samples/s +Step 32520/44000 | Loss: 2.9171 | LR: 8.20e-05 | 1309.8ms/step | 15.3 samples/s +Step 32530/44000 | Loss: 2.9468 | LR: 8.19e-05 | 1308.8ms/step | 15.3 samples/s +Step 32540/44000 | Loss: 2.9031 | LR: 8.17e-05 | 1310.8ms/step | 15.3 samples/s +Step 32550/44000 | Loss: 2.8718 | LR: 8.16e-05 | 1312.5ms/step | 15.2 samples/s +Step 32560/44000 | Loss: 2.9376 | LR: 8.15e-05 | 1309.9ms/step | 15.3 samples/s +Step 32570/44000 | Loss: 2.9199 | LR: 8.13e-05 | 1307.4ms/step | 15.3 samples/s +Step 32580/44000 | Loss: 2.9249 | LR: 8.12e-05 | 1309.1ms/step | 15.3 samples/s +Step 32590/44000 | Loss: 2.9509 | LR: 8.11e-05 | 1307.0ms/step | 15.3 samples/s +Step 32600/44000 | Loss: 2.9236 | LR: 8.09e-05 | 1309.6ms/step | 15.3 samples/s +Step 32610/44000 | Loss: 2.9274 | LR: 8.08e-05 | 1309.0ms/step | 15.3 samples/s +Step 32620/44000 | Loss: 2.9355 | LR: 8.07e-05 | 1310.1ms/step | 15.3 samples/s +Step 32630/44000 | Loss: 2.8951 | LR: 8.05e-05 | 1307.9ms/step | 15.3 samples/s +Step 32640/44000 | Loss: 2.9557 | LR: 8.04e-05 | 1308.5ms/step | 15.3 samples/s +Step 32650/44000 | Loss: 2.9335 | LR: 8.03e-05 | 1308.9ms/step | 15.3 samples/s +Step 32660/44000 | Loss: 2.9204 | LR: 8.01e-05 | 1309.5ms/step | 15.3 samples/s +Step 32670/44000 | Loss: 2.9574 | LR: 8.00e-05 | 1310.2ms/step | 15.3 samples/s +Step 32680/44000 | Loss: 2.9143 | LR: 7.99e-05 | 1310.3ms/step | 15.3 samples/s +Step 32690/44000 | Loss: 2.9260 | LR: 7.97e-05 | 1309.7ms/step | 15.3 samples/s +Step 32700/44000 | Loss: 2.9065 | LR: 7.96e-05 | 1310.0ms/step | 15.3 samples/s +Step 32710/44000 | Loss: 2.9341 | LR: 7.95e-05 | 1308.8ms/step | 15.3 samples/s +Step 32720/44000 | Loss: 2.9329 | LR: 7.93e-05 | 1310.0ms/step | 15.3 samples/s +Step 32730/44000 | Loss: 2.9110 | LR: 7.92e-05 | 1309.0ms/step | 15.3 samples/s +Step 32740/44000 | Loss: 2.9006 | LR: 7.91e-05 | 1310.0ms/step | 15.3 samples/s +Step 32750/44000 | Loss: 2.8929 | LR: 7.89e-05 | 1311.9ms/step | 15.2 samples/s +Validation at step 32750: Loss = 2.9343, Perplexity = 18.81 +Step 32760/44000 | Loss: 2.9275 | LR: 7.88e-05 | 3495.7ms/step | 5.7 samples/s +Step 32770/44000 | Loss: 2.9219 | LR: 7.87e-05 | 1314.2ms/step | 15.2 samples/s +Step 32780/44000 | Loss: 2.9480 | LR: 7.85e-05 | 1310.5ms/step | 15.3 samples/s +Step 32790/44000 | Loss: 2.9548 | LR: 7.84e-05 | 1310.9ms/step | 15.3 samples/s +Step 32800/44000 | Loss: 2.9085 | LR: 7.83e-05 | 1309.2ms/step | 15.3 samples/s +Step 32810/44000 | Loss: 2.9192 | LR: 7.81e-05 | 1310.3ms/step | 15.3 samples/s +Step 32820/44000 | Loss: 2.9374 | LR: 7.80e-05 | 1310.1ms/step | 15.3 samples/s +Step 32830/44000 | Loss: 2.9333 | LR: 7.79e-05 | 1309.8ms/step | 15.3 samples/s +Step 32840/44000 | Loss: 2.8855 | LR: 7.77e-05 | 1311.6ms/step | 15.2 samples/s +Step 32850/44000 | Loss: 2.9146 | LR: 7.76e-05 | 1311.7ms/step | 15.2 samples/s +Step 32860/44000 | Loss: 2.9139 | LR: 7.75e-05 | 1310.1ms/step | 15.3 samples/s +Step 32870/44000 | Loss: 2.9317 | LR: 7.73e-05 | 1310.3ms/step | 15.3 samples/s +Step 32880/44000 | Loss: 2.8906 | LR: 7.72e-05 | 1311.5ms/step | 15.2 samples/s +Step 32890/44000 | Loss: 2.9277 | LR: 7.71e-05 | 1312.9ms/step | 15.2 samples/s +Step 32900/44000 | Loss: 2.9242 | LR: 7.70e-05 | 1311.1ms/step | 15.3 samples/s +Step 32910/44000 | Loss: 2.9139 | LR: 7.68e-05 | 1311.3ms/step | 15.3 samples/s +Step 32920/44000 | Loss: 2.9387 | LR: 7.67e-05 | 1308.0ms/step | 15.3 samples/s +Step 32930/44000 | Loss: 2.9107 | LR: 7.66e-05 | 1311.0ms/step | 15.3 samples/s +Step 32940/44000 | Loss: 2.8962 | LR: 7.64e-05 | 1310.8ms/step | 15.3 samples/s +Step 32950/44000 | Loss: 2.9054 | LR: 7.63e-05 | 1313.3ms/step | 15.2 samples/s +Step 32960/44000 | Loss: 2.9114 | LR: 7.62e-05 | 1310.1ms/step | 15.3 samples/s +Step 32970/44000 | Loss: 2.9160 | LR: 7.60e-05 | 1309.1ms/step | 15.3 samples/s +Step 32980/44000 | Loss: 2.9199 | LR: 7.59e-05 | 1308.9ms/step | 15.3 samples/s +Step 32990/44000 | Loss: 2.9225 | LR: 7.58e-05 | 1307.9ms/step | 15.3 samples/s +Step 33000/44000 | Loss: 2.9436 | LR: 7.56e-05 | 1308.5ms/step | 15.3 samples/s +Validation at step 33000: Loss = 2.9339, Perplexity = 18.80 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-33000 +Step 33010/44000 | Loss: 2.9305 | LR: 7.55e-05 | 3815.3ms/step | 5.2 samples/s +Step 33020/44000 | Loss: 2.8946 | LR: 7.54e-05 | 1309.4ms/step | 15.3 samples/s +Step 33030/44000 | Loss: 2.9372 | LR: 7.53e-05 | 1303.9ms/step | 15.3 samples/s +Step 33040/44000 | Loss: 2.9224 | LR: 7.51e-05 | 1303.2ms/step | 15.3 samples/s +Step 33050/44000 | Loss: 2.9325 | LR: 7.50e-05 | 1308.7ms/step | 15.3 samples/s +Step 33060/44000 | Loss: 2.9012 | LR: 7.49e-05 | 1309.1ms/step | 15.3 samples/s +Step 33070/44000 | Loss: 2.9331 | LR: 7.47e-05 | 1309.1ms/step | 15.3 samples/s +Step 33080/44000 | Loss: 2.9329 | LR: 7.46e-05 | 1306.3ms/step | 15.3 samples/s +Step 33090/44000 | Loss: 2.9224 | LR: 7.45e-05 | 1307.2ms/step | 15.3 samples/s +Step 33100/44000 | Loss: 2.8946 | LR: 7.44e-05 | 1309.3ms/step | 15.3 samples/s +Step 33110/44000 | Loss: 2.8716 | LR: 7.42e-05 | 1309.3ms/step | 15.3 samples/s +Step 33120/44000 | Loss: 2.9045 | LR: 7.41e-05 | 1309.0ms/step | 15.3 samples/s +Step 33130/44000 | Loss: 2.9213 | LR: 7.40e-05 | 1307.7ms/step | 15.3 samples/s +Step 33140/44000 | Loss: 2.9378 | LR: 7.38e-05 | 1307.9ms/step | 15.3 samples/s +Step 33150/44000 | Loss: 2.9315 | LR: 7.37e-05 | 1306.8ms/step | 15.3 samples/s +Step 33160/44000 | Loss: 2.9078 | LR: 7.36e-05 | 1305.1ms/step | 15.3 samples/s +Step 33170/44000 | Loss: 2.9132 | LR: 7.34e-05 | 1305.9ms/step | 15.3 samples/s +Step 33180/44000 | Loss: 2.9366 | LR: 7.33e-05 | 1307.6ms/step | 15.3 samples/s +Step 33190/44000 | Loss: 2.9360 | LR: 7.32e-05 | 1305.6ms/step | 15.3 samples/s +Step 33200/44000 | Loss: 2.9256 | LR: 7.31e-05 | 1306.0ms/step | 15.3 samples/s +Step 33210/44000 | Loss: 2.9136 | LR: 7.29e-05 | 1305.2ms/step | 15.3 samples/s +Step 33220/44000 | Loss: 2.9316 | LR: 7.28e-05 | 1305.7ms/step | 15.3 samples/s +Step 33230/44000 | Loss: 2.9143 | LR: 7.27e-05 | 1305.5ms/step | 15.3 samples/s +Step 33240/44000 | Loss: 2.9095 | LR: 7.26e-05 | 1307.5ms/step | 15.3 samples/s +Step 33250/44000 | Loss: 2.9173 | LR: 7.24e-05 | 1307.2ms/step | 15.3 samples/s +Validation at step 33250: Loss = 2.9335, Perplexity = 18.79 +Step 33260/44000 | Loss: 2.9300 | LR: 7.23e-05 | 3491.2ms/step | 5.7 samples/s +Step 33270/44000 | Loss: 2.9116 | LR: 7.22e-05 | 1310.2ms/step | 15.3 samples/s +Step 33280/44000 | Loss: 2.9291 | LR: 7.20e-05 | 1307.7ms/step | 15.3 samples/s +Step 33290/44000 | Loss: 2.8779 | LR: 7.19e-05 | 1307.4ms/step | 15.3 samples/s +Step 33300/44000 | Loss: 2.9195 | LR: 7.18e-05 | 1307.2ms/step | 15.3 samples/s +Step 33310/44000 | Loss: 2.9350 | LR: 7.17e-05 | 1310.0ms/step | 15.3 samples/s +Step 33320/44000 | Loss: 2.9169 | LR: 7.15e-05 | 1308.1ms/step | 15.3 samples/s +Step 33330/44000 | Loss: 2.9081 | LR: 7.14e-05 | 1310.9ms/step | 15.3 samples/s +Step 33340/44000 | Loss: 2.9087 | LR: 7.13e-05 | 1313.7ms/step | 15.2 samples/s +Step 33350/44000 | Loss: 2.9193 | LR: 7.11e-05 | 1312.9ms/step | 15.2 samples/s +Step 33360/44000 | Loss: 2.9316 | LR: 7.10e-05 | 1309.2ms/step | 15.3 samples/s +Step 33370/44000 | Loss: 2.8908 | LR: 7.09e-05 | 1309.6ms/step | 15.3 samples/s +Step 33380/44000 | Loss: 2.9083 | LR: 7.08e-05 | 1306.8ms/step | 15.3 samples/s +Step 33390/44000 | Loss: 2.9378 | LR: 7.06e-05 | 1304.0ms/step | 15.3 samples/s +Step 33400/44000 | Loss: 2.8845 | LR: 7.05e-05 | 1307.1ms/step | 15.3 samples/s +Step 33410/44000 | Loss: 2.9323 | LR: 7.04e-05 | 1309.0ms/step | 15.3 samples/s +Step 33420/44000 | Loss: 2.9111 | LR: 7.03e-05 | 1306.2ms/step | 15.3 samples/s +Step 33430/44000 | Loss: 2.9073 | LR: 7.01e-05 | 1307.8ms/step | 15.3 samples/s +Step 33440/44000 | Loss: 2.9354 | LR: 7.00e-05 | 1310.5ms/step | 15.3 samples/s +Step 33450/44000 | Loss: 2.9299 | LR: 6.99e-05 | 1309.5ms/step | 15.3 samples/s +Step 33460/44000 | Loss: 2.9234 | LR: 6.98e-05 | 1309.3ms/step | 15.3 samples/s +Step 33470/44000 | Loss: 2.9151 | LR: 6.96e-05 | 1307.7ms/step | 15.3 samples/s +Step 33480/44000 | Loss: 2.9009 | LR: 6.95e-05 | 1308.0ms/step | 15.3 samples/s +Step 33490/44000 | Loss: 2.9314 | LR: 6.94e-05 | 1310.4ms/step | 15.3 samples/s +Step 33500/44000 | Loss: 2.8906 | LR: 6.93e-05 | 1314.3ms/step | 15.2 samples/s +Validation at step 33500: Loss = 2.9331, Perplexity = 18.79 +Step 33510/44000 | Loss: 2.8966 | LR: 6.91e-05 | 3606.7ms/step | 5.5 samples/s +Step 33520/44000 | Loss: 2.9353 | LR: 6.90e-05 | 1306.1ms/step | 15.3 samples/s +Step 33530/44000 | Loss: 2.8977 | LR: 6.89e-05 | 1306.7ms/step | 15.3 samples/s +Step 33540/44000 | Loss: 2.9055 | LR: 6.88e-05 | 1304.8ms/step | 15.3 samples/s +Step 33550/44000 | Loss: 2.9137 | LR: 6.86e-05 | 1302.1ms/step | 15.4 samples/s +Step 33560/44000 | Loss: 2.8941 | LR: 6.85e-05 | 1307.6ms/step | 15.3 samples/s +Step 33570/44000 | Loss: 2.8899 | LR: 6.84e-05 | 1308.1ms/step | 15.3 samples/s +Step 33580/44000 | Loss: 2.9342 | LR: 6.83e-05 | 1308.9ms/step | 15.3 samples/s +Step 33590/44000 | Loss: 2.9319 | LR: 6.81e-05 | 1307.7ms/step | 15.3 samples/s +Step 33600/44000 | Loss: 2.9394 | LR: 6.80e-05 | 1306.1ms/step | 15.3 samples/s +Step 33610/44000 | Loss: 2.8919 | LR: 6.79e-05 | 1308.3ms/step | 15.3 samples/s +Step 33620/44000 | Loss: 2.9079 | LR: 6.78e-05 | 1309.4ms/step | 15.3 samples/s +Step 33630/44000 | Loss: 2.8968 | LR: 6.76e-05 | 1309.2ms/step | 15.3 samples/s +Step 33640/44000 | Loss: 2.9117 | LR: 6.75e-05 | 1311.1ms/step | 15.3 samples/s +Step 33650/44000 | Loss: 2.9175 | LR: 6.74e-05 | 1307.2ms/step | 15.3 samples/s +Step 33660/44000 | Loss: 2.9225 | LR: 6.73e-05 | 1308.1ms/step | 15.3 samples/s +Step 33670/44000 | Loss: 2.9319 | LR: 6.71e-05 | 1307.6ms/step | 15.3 samples/s +Step 33680/44000 | Loss: 2.9171 | LR: 6.70e-05 | 1308.3ms/step | 15.3 samples/s +Step 33690/44000 | Loss: 2.9130 | LR: 6.69e-05 | 1305.9ms/step | 15.3 samples/s +Step 33700/44000 | Loss: 2.9287 | LR: 6.68e-05 | 1307.0ms/step | 15.3 samples/s +Step 33710/44000 | Loss: 2.9445 | LR: 6.66e-05 | 1305.7ms/step | 15.3 samples/s +Step 33720/44000 | Loss: 2.9232 | LR: 6.65e-05 | 1307.6ms/step | 15.3 samples/s +Step 33730/44000 | Loss: 2.9050 | LR: 6.64e-05 | 1305.6ms/step | 15.3 samples/s +Step 33740/44000 | Loss: 2.9199 | LR: 6.63e-05 | 1306.7ms/step | 15.3 samples/s +Step 33750/44000 | Loss: 2.9261 | LR: 6.61e-05 | 1305.9ms/step | 15.3 samples/s +Validation at step 33750: Loss = 2.9327, Perplexity = 18.78 +Step 33760/44000 | Loss: 2.8892 | LR: 6.60e-05 | 3486.2ms/step | 5.7 samples/s +Step 33770/44000 | Loss: 2.9406 | LR: 6.59e-05 | 1307.7ms/step | 15.3 samples/s +Step 33780/44000 | Loss: 2.9401 | LR: 6.58e-05 | 1307.6ms/step | 15.3 samples/s +Step 33790/44000 | Loss: 2.8946 | LR: 6.57e-05 | 1305.2ms/step | 15.3 samples/s +Step 33800/44000 | Loss: 2.9283 | LR: 6.55e-05 | 1307.0ms/step | 15.3 samples/s +Step 33810/44000 | Loss: 2.9443 | LR: 6.54e-05 | 1308.1ms/step | 15.3 samples/s +Step 33820/44000 | Loss: 2.9177 | LR: 6.53e-05 | 1304.9ms/step | 15.3 samples/s +Step 33830/44000 | Loss: 2.9377 | LR: 6.52e-05 | 1305.6ms/step | 15.3 samples/s +Step 33840/44000 | Loss: 2.9198 | LR: 6.50e-05 | 1307.1ms/step | 15.3 samples/s +Step 33850/44000 | Loss: 2.9288 | LR: 6.49e-05 | 1306.4ms/step | 15.3 samples/s +Step 33860/44000 | Loss: 2.9128 | LR: 6.48e-05 | 1307.3ms/step | 15.3 samples/s +Step 33870/44000 | Loss: 2.9217 | LR: 6.47e-05 | 1307.4ms/step | 15.3 samples/s +Step 33880/44000 | Loss: 2.9528 | LR: 6.46e-05 | 1306.3ms/step | 15.3 samples/s +Step 33890/44000 | Loss: 2.9097 | LR: 6.44e-05 | 1307.4ms/step | 15.3 samples/s +Step 33900/44000 | Loss: 2.9354 | LR: 6.43e-05 | 1305.9ms/step | 15.3 samples/s +Step 33910/44000 | Loss: 2.9242 | LR: 6.42e-05 | 1305.7ms/step | 15.3 samples/s +Step 33920/44000 | Loss: 2.9164 | LR: 6.41e-05 | 1308.5ms/step | 15.3 samples/s +Step 33930/44000 | Loss: 2.8765 | LR: 6.40e-05 | 1313.8ms/step | 15.2 samples/s +Step 33940/44000 | Loss: 2.9128 | LR: 6.38e-05 | 1313.1ms/step | 15.2 samples/s +Step 33950/44000 | Loss: 2.9091 | LR: 6.37e-05 | 1311.4ms/step | 15.3 samples/s +Step 33960/44000 | Loss: 2.8983 | LR: 6.36e-05 | 1307.8ms/step | 15.3 samples/s +Step 33970/44000 | Loss: 2.9079 | LR: 6.35e-05 | 1307.2ms/step | 15.3 samples/s +Step 33980/44000 | Loss: 2.9395 | LR: 6.33e-05 | 1308.1ms/step | 15.3 samples/s +Step 33990/44000 | Loss: 2.9330 | LR: 6.32e-05 | 1303.6ms/step | 15.3 samples/s +Step 34000/44000 | Loss: 2.9366 | LR: 6.31e-05 | 1304.3ms/step | 15.3 samples/s +Validation at step 34000: Loss = 2.9324, Perplexity = 18.77 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-34000 +Step 34010/44000 | Loss: 2.9146 | LR: 6.30e-05 | 3811.8ms/step | 5.2 samples/s +Step 34020/44000 | Loss: 2.9097 | LR: 6.29e-05 | 1308.1ms/step | 15.3 samples/s +Step 34030/44000 | Loss: 2.9573 | LR: 6.27e-05 | 1307.8ms/step | 15.3 samples/s +Step 34040/44000 | Loss: 2.9095 | LR: 6.26e-05 | 1308.1ms/step | 15.3 samples/s +Step 34050/44000 | Loss: 2.9105 | LR: 6.25e-05 | 1308.2ms/step | 15.3 samples/s +Step 34060/44000 | Loss: 2.9170 | LR: 6.24e-05 | 1309.3ms/step | 15.3 samples/s +Step 34070/44000 | Loss: 2.9388 | LR: 6.23e-05 | 1309.3ms/step | 15.3 samples/s +Step 34080/44000 | Loss: 2.9219 | LR: 6.21e-05 | 1311.5ms/step | 15.2 samples/s +Step 34090/44000 | Loss: 2.9035 | LR: 6.20e-05 | 1311.5ms/step | 15.2 samples/s +Step 34100/44000 | Loss: 2.9198 | LR: 6.19e-05 | 1308.1ms/step | 15.3 samples/s +Step 34110/44000 | Loss: 2.8889 | LR: 6.18e-05 | 1307.6ms/step | 15.3 samples/s +Step 34120/44000 | Loss: 2.9132 | LR: 6.17e-05 | 1306.5ms/step | 15.3 samples/s +Step 34130/44000 | Loss: 2.9161 | LR: 6.15e-05 | 1304.4ms/step | 15.3 samples/s +Step 34140/44000 | Loss: 2.9262 | LR: 6.14e-05 | 1306.0ms/step | 15.3 samples/s +Step 34150/44000 | Loss: 2.9201 | LR: 6.13e-05 | 1307.0ms/step | 15.3 samples/s +Step 34160/44000 | Loss: 2.9494 | LR: 6.12e-05 | 1308.6ms/step | 15.3 samples/s +Step 34170/44000 | Loss: 2.9260 | LR: 6.11e-05 | 1309.9ms/step | 15.3 samples/s +Step 34180/44000 | Loss: 2.9034 | LR: 6.09e-05 | 1308.7ms/step | 15.3 samples/s +Step 34190/44000 | Loss: 2.9248 | LR: 6.08e-05 | 1309.4ms/step | 15.3 samples/s +Step 34200/44000 | Loss: 2.9436 | LR: 6.07e-05 | 1309.8ms/step | 15.3 samples/s +Step 34210/44000 | Loss: 2.9251 | LR: 6.06e-05 | 1310.2ms/step | 15.3 samples/s +Step 34220/44000 | Loss: 2.9158 | LR: 6.05e-05 | 1308.8ms/step | 15.3 samples/s +Step 34230/44000 | Loss: 2.9327 | LR: 6.04e-05 | 1308.8ms/step | 15.3 samples/s +Step 34240/44000 | Loss: 2.9508 | LR: 6.02e-05 | 1310.0ms/step | 15.3 samples/s +Step 34250/44000 | Loss: 2.9284 | LR: 6.01e-05 | 1310.7ms/step | 15.3 samples/s +Validation at step 34250: Loss = 2.9321, Perplexity = 18.77 +Step 34260/44000 | Loss: 2.9246 | LR: 6.00e-05 | 3495.7ms/step | 5.7 samples/s +Step 34270/44000 | Loss: 2.8996 | LR: 5.99e-05 | 1310.9ms/step | 15.3 samples/s +Step 34280/44000 | Loss: 2.9426 | LR: 5.98e-05 | 1309.8ms/step | 15.3 samples/s +Step 34290/44000 | Loss: 2.9016 | LR: 5.96e-05 | 1307.7ms/step | 15.3 samples/s +Step 34300/44000 | Loss: 2.9007 | LR: 5.95e-05 | 1308.0ms/step | 15.3 samples/s +Step 34310/44000 | Loss: 2.9170 | LR: 5.94e-05 | 1309.2ms/step | 15.3 samples/s +Step 34320/44000 | Loss: 2.9646 | LR: 5.93e-05 | 1308.3ms/step | 15.3 samples/s +Step 34330/44000 | Loss: 2.8868 | LR: 5.92e-05 | 1307.0ms/step | 15.3 samples/s +Step 34340/44000 | Loss: 2.8899 | LR: 5.91e-05 | 1309.5ms/step | 15.3 samples/s +Step 34350/44000 | Loss: 2.9106 | LR: 5.89e-05 | 1306.9ms/step | 15.3 samples/s +Step 34360/44000 | Loss: 2.9229 | LR: 5.88e-05 | 1306.3ms/step | 15.3 samples/s +Step 34370/44000 | Loss: 2.9101 | LR: 5.87e-05 | 1305.5ms/step | 15.3 samples/s +Step 34380/44000 | Loss: 2.9090 | LR: 5.86e-05 | 1307.5ms/step | 15.3 samples/s +Step 34390/44000 | Loss: 2.9228 | LR: 5.85e-05 | 1306.1ms/step | 15.3 samples/s +Step 34400/44000 | Loss: 2.9205 | LR: 5.84e-05 | 1306.4ms/step | 15.3 samples/s +Step 34410/44000 | Loss: 2.9380 | LR: 5.82e-05 | 1309.2ms/step | 15.3 samples/s +Step 34420/44000 | Loss: 2.9179 | LR: 5.81e-05 | 1309.1ms/step | 15.3 samples/s +Step 34430/44000 | Loss: 2.9136 | LR: 5.80e-05 | 1307.7ms/step | 15.3 samples/s +Step 34440/44000 | Loss: 2.9259 | LR: 5.79e-05 | 1305.3ms/step | 15.3 samples/s +Step 34450/44000 | Loss: 2.9064 | LR: 5.78e-05 | 1306.7ms/step | 15.3 samples/s +Step 34460/44000 | Loss: 2.9349 | LR: 5.77e-05 | 1308.8ms/step | 15.3 samples/s +Step 34470/44000 | Loss: 2.9168 | LR: 5.75e-05 | 1307.1ms/step | 15.3 samples/s +Step 34480/44000 | Loss: 2.9228 | LR: 5.74e-05 | 1306.9ms/step | 15.3 samples/s +Step 34490/44000 | Loss: 2.8869 | LR: 5.73e-05 | 1309.3ms/step | 15.3 samples/s +Step 34500/44000 | Loss: 2.9339 | LR: 5.72e-05 | 1310.0ms/step | 15.3 samples/s +Validation at step 34500: Loss = 2.9317, Perplexity = 18.76 +Step 34510/44000 | Loss: 2.9164 | LR: 5.71e-05 | 3494.6ms/step | 5.7 samples/s +Step 34520/44000 | Loss: 2.9089 | LR: 5.70e-05 | 1309.3ms/step | 15.3 samples/s +Step 34530/44000 | Loss: 2.9010 | LR: 5.69e-05 | 1307.9ms/step | 15.3 samples/s +Step 34540/44000 | Loss: 2.9201 | LR: 5.67e-05 | 1318.6ms/step | 15.2 samples/s +Step 34550/44000 | Loss: 2.9382 | LR: 5.66e-05 | 1312.1ms/step | 15.2 samples/s +Step 34560/44000 | Loss: 2.8874 | LR: 5.65e-05 | 1343.8ms/step | 14.9 samples/s +Step 34570/44000 | Loss: 2.9035 | LR: 5.64e-05 | 1323.3ms/step | 15.1 samples/s +Step 34580/44000 | Loss: 2.8972 | LR: 5.63e-05 | 1310.3ms/step | 15.3 samples/s +Step 34590/44000 | Loss: 2.9086 | LR: 5.62e-05 | 1310.3ms/step | 15.3 samples/s +Step 34600/44000 | Loss: 2.9217 | LR: 5.60e-05 | 1310.2ms/step | 15.3 samples/s +Step 34610/44000 | Loss: 2.9225 | LR: 5.59e-05 | 1311.9ms/step | 15.2 samples/s +Step 34620/44000 | Loss: 2.9178 | LR: 5.58e-05 | 1310.0ms/step | 15.3 samples/s +Step 34630/44000 | Loss: 2.9051 | LR: 5.57e-05 | 1308.8ms/step | 15.3 samples/s +Step 34640/44000 | Loss: 2.9136 | LR: 5.56e-05 | 1309.9ms/step | 15.3 samples/s +Step 34650/44000 | Loss: 2.9272 | LR: 5.55e-05 | 1310.4ms/step | 15.3 samples/s +Step 34660/44000 | Loss: 2.9340 | LR: 5.54e-05 | 1309.6ms/step | 15.3 samples/s +Step 34670/44000 | Loss: 2.8860 | LR: 5.52e-05 | 1310.0ms/step | 15.3 samples/s +Step 34680/44000 | Loss: 2.9173 | LR: 5.51e-05 | 1309.9ms/step | 15.3 samples/s +Step 34690/44000 | Loss: 2.9006 | LR: 5.50e-05 | 1314.1ms/step | 15.2 samples/s +Step 34700/44000 | Loss: 2.8925 | LR: 5.49e-05 | 1312.3ms/step | 15.2 samples/s +Step 34710/44000 | Loss: 2.9303 | LR: 5.48e-05 | 1310.4ms/step | 15.3 samples/s +Step 34720/44000 | Loss: 2.9040 | LR: 5.47e-05 | 1308.8ms/step | 15.3 samples/s +Step 34730/44000 | Loss: 2.9294 | LR: 5.46e-05 | 1315.0ms/step | 15.2 samples/s +Step 34740/44000 | Loss: 2.9044 | LR: 5.45e-05 | 1316.3ms/step | 15.2 samples/s +Step 34750/44000 | Loss: 2.9121 | LR: 5.43e-05 | 1318.2ms/step | 15.2 samples/s +Validation at step 34750: Loss = 2.9314, Perplexity = 18.75 +Step 34760/44000 | Loss: 2.8991 | LR: 5.42e-05 | 3490.3ms/step | 5.7 samples/s +Step 34770/44000 | Loss: 2.9045 | LR: 5.41e-05 | 1307.3ms/step | 15.3 samples/s +Step 34780/44000 | Loss: 2.9452 | LR: 5.40e-05 | 1309.5ms/step | 15.3 samples/s +Step 34790/44000 | Loss: 2.9405 | LR: 5.39e-05 | 1310.1ms/step | 15.3 samples/s +Step 34800/44000 | Loss: 2.9187 | LR: 5.38e-05 | 1309.2ms/step | 15.3 samples/s +Step 34810/44000 | Loss: 2.9364 | LR: 5.37e-05 | 1308.3ms/step | 15.3 samples/s +Step 34820/44000 | Loss: 2.9162 | LR: 5.36e-05 | 1309.6ms/step | 15.3 samples/s +Step 34830/44000 | Loss: 2.9156 | LR: 5.34e-05 | 1309.3ms/step | 15.3 samples/s +Step 34840/44000 | Loss: 2.9249 | LR: 5.33e-05 | 1309.1ms/step | 15.3 samples/s +Step 34850/44000 | Loss: 2.9348 | LR: 5.32e-05 | 1308.7ms/step | 15.3 samples/s +Step 34860/44000 | Loss: 2.9246 | LR: 5.31e-05 | 1309.9ms/step | 15.3 samples/s +Step 34870/44000 | Loss: 2.9047 | LR: 5.30e-05 | 1310.6ms/step | 15.3 samples/s +Step 34880/44000 | Loss: 2.9130 | LR: 5.29e-05 | 1309.8ms/step | 15.3 samples/s +Step 34890/44000 | Loss: 2.9086 | LR: 5.28e-05 | 1309.6ms/step | 15.3 samples/s +Step 34900/44000 | Loss: 2.9253 | LR: 5.27e-05 | 1308.7ms/step | 15.3 samples/s +Step 34910/44000 | Loss: 2.9230 | LR: 5.25e-05 | 1312.4ms/step | 15.2 samples/s +Step 34920/44000 | Loss: 2.9078 | LR: 5.24e-05 | 1309.7ms/step | 15.3 samples/s +Step 34930/44000 | Loss: 2.8973 | LR: 5.23e-05 | 1310.8ms/step | 15.3 samples/s +Step 34940/44000 | Loss: 2.8728 | LR: 5.22e-05 | 1311.0ms/step | 15.3 samples/s +Step 34950/44000 | Loss: 2.9286 | LR: 5.21e-05 | 1312.1ms/step | 15.2 samples/s +Step 34960/44000 | Loss: 2.9202 | LR: 5.20e-05 | 1309.6ms/step | 15.3 samples/s +Step 34970/44000 | Loss: 2.8998 | LR: 5.19e-05 | 1309.7ms/step | 15.3 samples/s +Step 34980/44000 | Loss: 2.9240 | LR: 5.18e-05 | 1309.7ms/step | 15.3 samples/s +Step 34990/44000 | Loss: 2.9254 | LR: 5.17e-05 | 1310.1ms/step | 15.3 samples/s +Step 35000/44000 | Loss: 2.8946 | LR: 5.15e-05 | 1310.4ms/step | 15.3 samples/s +Validation at step 35000: Loss = 2.9311, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-35000 +Step 35010/44000 | Loss: 2.9184 | LR: 5.14e-05 | 3803.6ms/step | 5.3 samples/s +Step 35020/44000 | Loss: 2.8871 | LR: 5.13e-05 | 1304.9ms/step | 15.3 samples/s +Step 35030/44000 | Loss: 2.8960 | LR: 5.12e-05 | 1305.7ms/step | 15.3 samples/s +Step 35040/44000 | Loss: 2.9067 | LR: 5.11e-05 | 1307.2ms/step | 15.3 samples/s +Step 35050/44000 | Loss: 2.8988 | LR: 5.10e-05 | 1306.4ms/step | 15.3 samples/s +Step 35060/44000 | Loss: 2.9075 | LR: 5.09e-05 | 1309.3ms/step | 15.3 samples/s +Step 35070/44000 | Loss: 2.8756 | LR: 5.08e-05 | 1308.0ms/step | 15.3 samples/s +Step 35080/44000 | Loss: 2.9126 | LR: 5.07e-05 | 1309.6ms/step | 15.3 samples/s +Step 35090/44000 | Loss: 2.8990 | LR: 5.06e-05 | 1307.1ms/step | 15.3 samples/s +Step 35100/44000 | Loss: 2.9421 | LR: 5.04e-05 | 1306.7ms/step | 15.3 samples/s +Step 35110/44000 | Loss: 2.9051 | LR: 5.03e-05 | 1307.4ms/step | 15.3 samples/s +Step 35120/44000 | Loss: 2.9193 | LR: 5.02e-05 | 1306.9ms/step | 15.3 samples/s +Step 35130/44000 | Loss: 2.9192 | LR: 5.01e-05 | 1309.9ms/step | 15.3 samples/s +Step 35140/44000 | Loss: 2.8861 | LR: 5.00e-05 | 1311.1ms/step | 15.3 samples/s +Step 35150/44000 | Loss: 2.9269 | LR: 4.99e-05 | 1311.6ms/step | 15.2 samples/s +Step 35160/44000 | Loss: 2.9485 | LR: 4.98e-05 | 1308.5ms/step | 15.3 samples/s +Step 35170/44000 | Loss: 2.9247 | LR: 4.97e-05 | 1308.5ms/step | 15.3 samples/s +Step 35180/44000 | Loss: 2.9325 | LR: 4.96e-05 | 1311.7ms/step | 15.2 samples/s +Step 35190/44000 | Loss: 2.9012 | LR: 4.95e-05 | 1306.6ms/step | 15.3 samples/s +Step 35200/44000 | Loss: 2.9313 | LR: 4.94e-05 | 1308.7ms/step | 15.3 samples/s +Step 35210/44000 | Loss: 2.9274 | LR: 4.93e-05 | 1309.0ms/step | 15.3 samples/s +Step 35220/44000 | Loss: 2.9054 | LR: 4.91e-05 | 1309.2ms/step | 15.3 samples/s +Step 35230/44000 | Loss: 2.9138 | LR: 4.90e-05 | 1307.8ms/step | 15.3 samples/s +Step 35240/44000 | Loss: 2.9029 | LR: 4.89e-05 | 1306.5ms/step | 15.3 samples/s +Step 35250/44000 | Loss: 2.9482 | LR: 4.88e-05 | 1305.8ms/step | 15.3 samples/s +Validation at step 35250: Loss = 2.9310, Perplexity = 18.75 +Step 35260/44000 | Loss: 2.9049 | LR: 4.87e-05 | 3485.9ms/step | 5.7 samples/s +Step 35270/44000 | Loss: 2.9372 | LR: 4.86e-05 | 1308.4ms/step | 15.3 samples/s +Step 35280/44000 | Loss: 2.8925 | LR: 4.85e-05 | 1306.1ms/step | 15.3 samples/s +Step 35290/44000 | Loss: 2.9017 | LR: 4.84e-05 | 1308.1ms/step | 15.3 samples/s +Step 35300/44000 | Loss: 2.9042 | LR: 4.83e-05 | 1309.6ms/step | 15.3 samples/s +Step 35310/44000 | Loss: 2.9234 | LR: 4.82e-05 | 1309.9ms/step | 15.3 samples/s +Step 35320/44000 | Loss: 2.8866 | LR: 4.81e-05 | 1307.8ms/step | 15.3 samples/s +Step 35330/44000 | Loss: 2.9208 | LR: 4.80e-05 | 1308.1ms/step | 15.3 samples/s +Step 35340/44000 | Loss: 2.9166 | LR: 4.79e-05 | 1308.7ms/step | 15.3 samples/s +Step 35350/44000 | Loss: 2.9424 | LR: 4.77e-05 | 1307.9ms/step | 15.3 samples/s +Step 35360/44000 | Loss: 2.8981 | LR: 4.76e-05 | 1308.5ms/step | 15.3 samples/s +Step 35370/44000 | Loss: 2.9109 | LR: 4.75e-05 | 1306.6ms/step | 15.3 samples/s +Step 35380/44000 | Loss: 2.8988 | LR: 4.74e-05 | 1307.3ms/step | 15.3 samples/s +Step 35390/44000 | Loss: 2.9034 | LR: 4.73e-05 | 1310.8ms/step | 15.3 samples/s +Step 35400/44000 | Loss: 2.8939 | LR: 4.72e-05 | 1308.5ms/step | 15.3 samples/s +Step 35410/44000 | Loss: 2.9261 | LR: 4.71e-05 | 1306.3ms/step | 15.3 samples/s +Step 35420/44000 | Loss: 2.8990 | LR: 4.70e-05 | 1308.3ms/step | 15.3 samples/s +Step 35430/44000 | Loss: 2.9559 | LR: 4.69e-05 | 1309.1ms/step | 15.3 samples/s +Step 35440/44000 | Loss: 2.9351 | LR: 4.68e-05 | 1308.6ms/step | 15.3 samples/s +Step 35450/44000 | Loss: 2.9183 | LR: 4.67e-05 | 1305.5ms/step | 15.3 samples/s +Step 35460/44000 | Loss: 2.9389 | LR: 4.66e-05 | 1304.5ms/step | 15.3 samples/s +Step 35470/44000 | Loss: 2.9093 | LR: 4.65e-05 | 1307.7ms/step | 15.3 samples/s +Step 35480/44000 | Loss: 2.9191 | LR: 4.64e-05 | 1309.5ms/step | 15.3 samples/s +Step 35490/44000 | Loss: 2.9246 | LR: 4.63e-05 | 1306.1ms/step | 15.3 samples/s +Step 35500/44000 | Loss: 2.8974 | LR: 4.62e-05 | 1306.8ms/step | 15.3 samples/s +Validation at step 35500: Loss = 2.9307, Perplexity = 18.74 +Step 35510/44000 | Loss: 2.9096 | LR: 4.61e-05 | 3492.6ms/step | 5.7 samples/s +Step 35520/44000 | Loss: 2.9178 | LR: 4.59e-05 | 1312.6ms/step | 15.2 samples/s +Step 35530/44000 | Loss: 2.9169 | LR: 4.58e-05 | 1308.2ms/step | 15.3 samples/s +Step 35540/44000 | Loss: 2.9352 | LR: 4.57e-05 | 1310.6ms/step | 15.3 samples/s +Step 35550/44000 | Loss: 2.9373 | LR: 4.56e-05 | 1315.8ms/step | 15.2 samples/s +Step 35560/44000 | Loss: 2.9283 | LR: 4.55e-05 | 1314.6ms/step | 15.2 samples/s +Step 35570/44000 | Loss: 2.9363 | LR: 4.54e-05 | 1309.8ms/step | 15.3 samples/s +Step 35580/44000 | Loss: 2.9199 | LR: 4.53e-05 | 1313.1ms/step | 15.2 samples/s +Step 35590/44000 | Loss: 2.9289 | LR: 4.52e-05 | 1312.1ms/step | 15.2 samples/s +Step 35600/44000 | Loss: 2.8944 | LR: 4.51e-05 | 1309.5ms/step | 15.3 samples/s +Step 35610/44000 | Loss: 2.9318 | LR: 4.50e-05 | 1308.7ms/step | 15.3 samples/s +Step 35620/44000 | Loss: 2.9104 | LR: 4.49e-05 | 1311.4ms/step | 15.3 samples/s +Step 35630/44000 | Loss: 2.9206 | LR: 4.48e-05 | 1310.1ms/step | 15.3 samples/s +Step 35640/44000 | Loss: 2.8985 | LR: 4.47e-05 | 1313.9ms/step | 15.2 samples/s +Step 35650/44000 | Loss: 2.9052 | LR: 4.46e-05 | 1315.7ms/step | 15.2 samples/s +Step 35660/44000 | Loss: 2.9204 | LR: 4.45e-05 | 1310.7ms/step | 15.3 samples/s +Step 35670/44000 | Loss: 2.9093 | LR: 4.44e-05 | 1312.7ms/step | 15.2 samples/s +Step 35680/44000 | Loss: 2.9159 | LR: 4.43e-05 | 1310.9ms/step | 15.3 samples/s +Step 35690/44000 | Loss: 2.9267 | LR: 4.42e-05 | 1311.0ms/step | 15.3 samples/s +Step 35700/44000 | Loss: 2.9030 | LR: 4.41e-05 | 1314.9ms/step | 15.2 samples/s +Step 35710/44000 | Loss: 2.9166 | LR: 4.40e-05 | 1313.7ms/step | 15.2 samples/s +Step 35720/44000 | Loss: 2.8996 | LR: 4.39e-05 | 1312.7ms/step | 15.2 samples/s +Step 35730/44000 | Loss: 2.8901 | LR: 4.38e-05 | 1313.2ms/step | 15.2 samples/s +Step 35740/44000 | Loss: 2.9231 | LR: 4.37e-05 | 1310.5ms/step | 15.3 samples/s +Step 35750/44000 | Loss: 2.9022 | LR: 4.36e-05 | 1310.0ms/step | 15.3 samples/s +Validation at step 35750: Loss = 2.9305, Perplexity = 18.74 +Step 35760/44000 | Loss: 2.9329 | LR: 4.35e-05 | 3494.2ms/step | 5.7 samples/s +Step 35770/44000 | Loss: 2.9077 | LR: 4.34e-05 | 1308.0ms/step | 15.3 samples/s +Step 35780/44000 | Loss: 2.8976 | LR: 4.33e-05 | 1308.5ms/step | 15.3 samples/s +Step 35790/44000 | Loss: 2.9208 | LR: 4.32e-05 | 1309.1ms/step | 15.3 samples/s +Step 35800/44000 | Loss: 2.9319 | LR: 4.31e-05 | 1308.9ms/step | 15.3 samples/s +Step 35810/44000 | Loss: 2.9280 | LR: 4.29e-05 | 1310.3ms/step | 15.3 samples/s +Step 35820/44000 | Loss: 2.9294 | LR: 4.28e-05 | 1310.0ms/step | 15.3 samples/s +Step 35830/44000 | Loss: 2.9479 | LR: 4.27e-05 | 1309.0ms/step | 15.3 samples/s +Step 35840/44000 | Loss: 2.9075 | LR: 4.26e-05 | 1308.8ms/step | 15.3 samples/s +Step 35850/44000 | Loss: 2.9176 | LR: 4.25e-05 | 1310.3ms/step | 15.3 samples/s +Step 35860/44000 | Loss: 2.9415 | LR: 4.24e-05 | 1308.3ms/step | 15.3 samples/s +Step 35870/44000 | Loss: 2.9184 | LR: 4.23e-05 | 1307.0ms/step | 15.3 samples/s +Step 35880/44000 | Loss: 2.9325 | LR: 4.22e-05 | 1304.7ms/step | 15.3 samples/s +Step 35890/44000 | Loss: 2.8994 | LR: 4.21e-05 | 1308.6ms/step | 15.3 samples/s +Step 35900/44000 | Loss: 2.9201 | LR: 4.20e-05 | 1307.5ms/step | 15.3 samples/s +Step 35910/44000 | Loss: 2.9048 | LR: 4.19e-05 | 1312.9ms/step | 15.2 samples/s +Step 35920/44000 | Loss: 2.9171 | LR: 4.18e-05 | 1307.6ms/step | 15.3 samples/s +Step 35930/44000 | Loss: 2.9053 | LR: 4.17e-05 | 1305.9ms/step | 15.3 samples/s +Step 35940/44000 | Loss: 2.9058 | LR: 4.16e-05 | 1308.3ms/step | 15.3 samples/s +Step 35950/44000 | Loss: 2.8921 | LR: 4.15e-05 | 1310.2ms/step | 15.3 samples/s +Step 35960/44000 | Loss: 2.8967 | LR: 4.14e-05 | 1312.1ms/step | 15.2 samples/s +Step 35970/44000 | Loss: 2.8888 | LR: 4.13e-05 | 1310.4ms/step | 15.3 samples/s +Step 35980/44000 | Loss: 2.9412 | LR: 4.12e-05 | 1313.2ms/step | 15.2 samples/s +Step 35990/44000 | Loss: 2.9135 | LR: 4.11e-05 | 1310.3ms/step | 15.3 samples/s +Step 36000/44000 | Loss: 2.9129 | LR: 4.10e-05 | 1311.6ms/step | 15.2 samples/s +Validation at step 36000: Loss = 2.9304, Perplexity = 18.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-36000 +Step 36010/44000 | Loss: 2.9295 | LR: 4.09e-05 | 3791.8ms/step | 5.3 samples/s +Step 36020/44000 | Loss: 2.9064 | LR: 4.08e-05 | 1308.2ms/step | 15.3 samples/s +Step 36030/44000 | Loss: 2.8952 | LR: 4.07e-05 | 1306.7ms/step | 15.3 samples/s +Step 36040/44000 | Loss: 2.9014 | LR: 4.06e-05 | 1306.3ms/step | 15.3 samples/s +Step 36050/44000 | Loss: 2.8976 | LR: 4.05e-05 | 1306.5ms/step | 15.3 samples/s +Step 36060/44000 | Loss: 2.9273 | LR: 4.04e-05 | 1305.5ms/step | 15.3 samples/s +Step 36070/44000 | Loss: 2.9182 | LR: 4.03e-05 | 1305.6ms/step | 15.3 samples/s +Step 36080/44000 | Loss: 2.9333 | LR: 4.02e-05 | 1306.1ms/step | 15.3 samples/s +Step 36090/44000 | Loss: 2.9390 | LR: 4.01e-05 | 1305.9ms/step | 15.3 samples/s +Step 36100/44000 | Loss: 2.9178 | LR: 4.00e-05 | 1306.3ms/step | 15.3 samples/s +Step 36110/44000 | Loss: 2.9342 | LR: 3.99e-05 | 1309.4ms/step | 15.3 samples/s +Step 36120/44000 | Loss: 2.9192 | LR: 3.98e-05 | 1309.4ms/step | 15.3 samples/s +Step 36130/44000 | Loss: 2.9221 | LR: 3.97e-05 | 1308.0ms/step | 15.3 samples/s +Step 36140/44000 | Loss: 2.9141 | LR: 3.97e-05 | 1305.4ms/step | 15.3 samples/s +Step 36150/44000 | Loss: 2.9234 | LR: 3.96e-05 | 1309.7ms/step | 15.3 samples/s +Step 36160/44000 | Loss: 2.9170 | LR: 3.95e-05 | 1308.1ms/step | 15.3 samples/s +Step 36170/44000 | Loss: 2.8757 | LR: 3.94e-05 | 1310.3ms/step | 15.3 samples/s +Step 36180/44000 | Loss: 2.9158 | LR: 3.93e-05 | 1309.4ms/step | 15.3 samples/s +Step 36190/44000 | Loss: 2.9117 | LR: 3.92e-05 | 1306.5ms/step | 15.3 samples/s +Step 36200/44000 | Loss: 2.9048 | LR: 3.91e-05 | 1304.8ms/step | 15.3 samples/s +Step 36210/44000 | Loss: 2.9149 | LR: 3.90e-05 | 1304.7ms/step | 15.3 samples/s +Step 36220/44000 | Loss: 2.9099 | LR: 3.89e-05 | 1304.1ms/step | 15.3 samples/s +Step 36230/44000 | Loss: 2.9109 | LR: 3.88e-05 | 1303.0ms/step | 15.3 samples/s +Step 36240/44000 | Loss: 2.9199 | LR: 3.87e-05 | 1307.8ms/step | 15.3 samples/s +Step 36250/44000 | Loss: 2.9349 | LR: 3.86e-05 | 1311.2ms/step | 15.3 samples/s +Validation at step 36250: Loss = 2.9302, Perplexity = 18.73 +Step 36260/44000 | Loss: 2.9264 | LR: 3.85e-05 | 3485.8ms/step | 5.7 samples/s +Step 36270/44000 | Loss: 2.9373 | LR: 3.84e-05 | 1305.7ms/step | 15.3 samples/s +Step 36280/44000 | Loss: 2.9086 | LR: 3.83e-05 | 1305.6ms/step | 15.3 samples/s +Step 36290/44000 | Loss: 2.9240 | LR: 3.82e-05 | 1307.5ms/step | 15.3 samples/s +Step 36300/44000 | Loss: 2.9074 | LR: 3.81e-05 | 1309.2ms/step | 15.3 samples/s +Step 36310/44000 | Loss: 2.9099 | LR: 3.80e-05 | 1311.3ms/step | 15.3 samples/s +Step 36320/44000 | Loss: 2.8852 | LR: 3.79e-05 | 1309.5ms/step | 15.3 samples/s +Step 36330/44000 | Loss: 2.9035 | LR: 3.78e-05 | 1307.3ms/step | 15.3 samples/s +Step 36340/44000 | Loss: 2.9211 | LR: 3.77e-05 | 1308.2ms/step | 15.3 samples/s +Step 36350/44000 | Loss: 2.8918 | LR: 3.76e-05 | 1308.9ms/step | 15.3 samples/s +Step 36360/44000 | Loss: 2.8988 | LR: 3.75e-05 | 1308.3ms/step | 15.3 samples/s +Step 36370/44000 | Loss: 2.9080 | LR: 3.74e-05 | 1309.7ms/step | 15.3 samples/s +Step 36380/44000 | Loss: 2.9087 | LR: 3.73e-05 | 1310.8ms/step | 15.3 samples/s +Step 36390/44000 | Loss: 2.9305 | LR: 3.72e-05 | 1311.0ms/step | 15.3 samples/s +Step 36400/44000 | Loss: 2.9086 | LR: 3.71e-05 | 1308.8ms/step | 15.3 samples/s +Step 36410/44000 | Loss: 2.9189 | LR: 3.70e-05 | 1307.2ms/step | 15.3 samples/s +Step 36420/44000 | Loss: 2.8862 | LR: 3.69e-05 | 1307.2ms/step | 15.3 samples/s +Step 36430/44000 | Loss: 2.8930 | LR: 3.69e-05 | 1306.6ms/step | 15.3 samples/s +Step 36440/44000 | Loss: 2.9037 | LR: 3.68e-05 | 1309.1ms/step | 15.3 samples/s +Step 36450/44000 | Loss: 2.9447 | LR: 3.67e-05 | 1307.9ms/step | 15.3 samples/s +Step 36460/44000 | Loss: 2.9190 | LR: 3.66e-05 | 1309.1ms/step | 15.3 samples/s +Step 36470/44000 | Loss: 2.8935 | LR: 3.65e-05 | 1310.5ms/step | 15.3 samples/s +Step 36480/44000 | Loss: 2.9179 | LR: 3.64e-05 | 1312.6ms/step | 15.2 samples/s +Step 36490/44000 | Loss: 2.9399 | LR: 3.63e-05 | 1312.8ms/step | 15.2 samples/s +Step 36500/44000 | Loss: 2.8882 | LR: 3.62e-05 | 1311.8ms/step | 15.2 samples/s +Validation at step 36500: Loss = 2.9301, Perplexity = 18.73 +Step 36510/44000 | Loss: 2.9283 | LR: 3.61e-05 | 3488.4ms/step | 5.7 samples/s +Step 36520/44000 | Loss: 2.9021 | LR: 3.60e-05 | 1308.3ms/step | 15.3 samples/s +Step 36530/44000 | Loss: 2.9091 | LR: 3.59e-05 | 1311.0ms/step | 15.3 samples/s +Step 36540/44000 | Loss: 2.8973 | LR: 3.58e-05 | 1309.5ms/step | 15.3 samples/s +Step 36550/44000 | Loss: 2.9054 | LR: 3.57e-05 | 1310.5ms/step | 15.3 samples/s +Step 36560/44000 | Loss: 2.9272 | LR: 3.56e-05 | 1314.1ms/step | 15.2 samples/s +Step 36570/44000 | Loss: 2.8861 | LR: 3.55e-05 | 1312.0ms/step | 15.2 samples/s +Step 36580/44000 | Loss: 2.9047 | LR: 3.54e-05 | 1309.6ms/step | 15.3 samples/s +Step 36590/44000 | Loss: 2.9140 | LR: 3.53e-05 | 1311.1ms/step | 15.3 samples/s +Step 36600/44000 | Loss: 2.8866 | LR: 3.53e-05 | 1311.7ms/step | 15.2 samples/s +Step 36610/44000 | Loss: 2.9110 | LR: 3.52e-05 | 1308.6ms/step | 15.3 samples/s +Step 36620/44000 | Loss: 2.8996 | LR: 3.51e-05 | 1308.5ms/step | 15.3 samples/s +Step 36630/44000 | Loss: 2.9133 | LR: 3.50e-05 | 1307.5ms/step | 15.3 samples/s +Step 36640/44000 | Loss: 2.9131 | LR: 3.49e-05 | 1308.3ms/step | 15.3 samples/s +Step 36650/44000 | Loss: 2.9100 | LR: 3.48e-05 | 1308.7ms/step | 15.3 samples/s +Step 36660/44000 | Loss: 2.9023 | LR: 3.47e-05 | 1309.2ms/step | 15.3 samples/s +Step 36670/44000 | Loss: 2.9416 | LR: 3.46e-05 | 1309.7ms/step | 15.3 samples/s +Step 36680/44000 | Loss: 2.9274 | LR: 3.45e-05 | 1311.7ms/step | 15.2 samples/s +Step 36690/44000 | Loss: 2.9053 | LR: 3.44e-05 | 1309.9ms/step | 15.3 samples/s +Step 36700/44000 | Loss: 2.9068 | LR: 3.43e-05 | 1307.3ms/step | 15.3 samples/s +Step 36710/44000 | Loss: 2.8981 | LR: 3.42e-05 | 1308.7ms/step | 15.3 samples/s +Step 36720/44000 | Loss: 2.9128 | LR: 3.41e-05 | 1310.2ms/step | 15.3 samples/s +Step 36730/44000 | Loss: 2.9026 | LR: 3.41e-05 | 1308.7ms/step | 15.3 samples/s +Step 36740/44000 | Loss: 2.9265 | LR: 3.40e-05 | 1308.0ms/step | 15.3 samples/s +Step 36750/44000 | Loss: 2.8965 | LR: 3.39e-05 | 1308.0ms/step | 15.3 samples/s +Validation at step 36750: Loss = 2.9299, Perplexity = 18.73 +Step 36760/44000 | Loss: 2.9029 | LR: 3.38e-05 | 3493.4ms/step | 5.7 samples/s +Step 36770/44000 | Loss: 2.9146 | LR: 3.37e-05 | 1310.2ms/step | 15.3 samples/s +Step 36780/44000 | Loss: 2.9245 | LR: 3.36e-05 | 1311.8ms/step | 15.2 samples/s +Step 36790/44000 | Loss: 2.9219 | LR: 3.35e-05 | 1317.2ms/step | 15.2 samples/s +Step 36800/44000 | Loss: 2.9128 | LR: 3.34e-05 | 1316.5ms/step | 15.2 samples/s +Step 36810/44000 | Loss: 2.9085 | LR: 3.33e-05 | 1312.3ms/step | 15.2 samples/s +Step 36820/44000 | Loss: 2.9060 | LR: 3.32e-05 | 1309.7ms/step | 15.3 samples/s +Step 36830/44000 | Loss: 2.9082 | LR: 3.31e-05 | 1309.9ms/step | 15.3 samples/s +Step 36840/44000 | Loss: 2.9140 | LR: 3.31e-05 | 1310.4ms/step | 15.3 samples/s +Step 36850/44000 | Loss: 2.9146 | LR: 3.30e-05 | 1307.9ms/step | 15.3 samples/s +Step 36860/44000 | Loss: 2.8948 | LR: 3.29e-05 | 1308.7ms/step | 15.3 samples/s +Step 36870/44000 | Loss: 2.9156 | LR: 3.28e-05 | 1312.2ms/step | 15.2 samples/s +Step 36880/44000 | Loss: 2.8900 | LR: 3.27e-05 | 1311.8ms/step | 15.2 samples/s +Step 36890/44000 | Loss: 2.8860 | LR: 3.26e-05 | 1311.3ms/step | 15.3 samples/s +Step 36900/44000 | Loss: 2.8901 | LR: 3.25e-05 | 1309.1ms/step | 15.3 samples/s +Step 36910/44000 | Loss: 2.9065 | LR: 3.24e-05 | 1309.4ms/step | 15.3 samples/s +Step 36920/44000 | Loss: 2.9054 | LR: 3.23e-05 | 1311.0ms/step | 15.3 samples/s +Step 36930/44000 | Loss: 2.8849 | LR: 3.22e-05 | 1310.6ms/step | 15.3 samples/s +Step 36940/44000 | Loss: 2.9228 | LR: 3.22e-05 | 1308.2ms/step | 15.3 samples/s +Step 36950/44000 | Loss: 2.9191 | LR: 3.21e-05 | 1309.8ms/step | 15.3 samples/s +Step 36960/44000 | Loss: 2.9023 | LR: 3.20e-05 | 1310.7ms/step | 15.3 samples/s +Step 36970/44000 | Loss: 2.8830 | LR: 3.19e-05 | 1309.4ms/step | 15.3 samples/s +Step 36980/44000 | Loss: 2.9056 | LR: 3.18e-05 | 1307.4ms/step | 15.3 samples/s +Step 36990/44000 | Loss: 2.9180 | LR: 3.17e-05 | 1308.5ms/step | 15.3 samples/s +Step 37000/44000 | Loss: 2.9273 | LR: 3.16e-05 | 1308.1ms/step | 15.3 samples/s +Validation at step 37000: Loss = 2.9298, Perplexity = 18.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-37000 +Step 37010/44000 | Loss: 2.9235 | LR: 3.15e-05 | 3798.2ms/step | 5.3 samples/s +Step 37020/44000 | Loss: 2.9618 | LR: 3.15e-05 | 1309.3ms/step | 15.3 samples/s +Step 37030/44000 | Loss: 2.9281 | LR: 3.14e-05 | 1307.3ms/step | 15.3 samples/s +Step 37040/44000 | Loss: 2.9272 | LR: 3.13e-05 | 1309.4ms/step | 15.3 samples/s +Step 37050/44000 | Loss: 2.9193 | LR: 3.12e-05 | 1309.1ms/step | 15.3 samples/s +Step 37060/44000 | Loss: 2.8860 | LR: 3.11e-05 | 1309.7ms/step | 15.3 samples/s +Step 37070/44000 | Loss: 2.9163 | LR: 3.10e-05 | 1310.1ms/step | 15.3 samples/s +Step 37080/44000 | Loss: 2.9471 | LR: 3.09e-05 | 1311.0ms/step | 15.3 samples/s +Step 37090/44000 | Loss: 2.9252 | LR: 3.08e-05 | 1309.9ms/step | 15.3 samples/s +Step 37100/44000 | Loss: 2.9087 | LR: 3.07e-05 | 1307.4ms/step | 15.3 samples/s +Step 37110/44000 | Loss: 2.9035 | LR: 3.07e-05 | 1306.9ms/step | 15.3 samples/s +Step 37120/44000 | Loss: 2.9272 | LR: 3.06e-05 | 1308.9ms/step | 15.3 samples/s +Step 37130/44000 | Loss: 2.9164 | LR: 3.05e-05 | 1307.7ms/step | 15.3 samples/s +Step 37140/44000 | Loss: 2.9106 | LR: 3.04e-05 | 1309.1ms/step | 15.3 samples/s +Step 37150/44000 | Loss: 2.8723 | LR: 3.03e-05 | 1309.4ms/step | 15.3 samples/s +Step 37160/44000 | Loss: 2.9229 | LR: 3.02e-05 | 1311.9ms/step | 15.2 samples/s +Step 37170/44000 | Loss: 2.9539 | LR: 3.01e-05 | 1308.4ms/step | 15.3 samples/s +Step 37180/44000 | Loss: 2.9065 | LR: 3.01e-05 | 1311.7ms/step | 15.2 samples/s +Step 37190/44000 | Loss: 2.9097 | LR: 3.00e-05 | 1309.6ms/step | 15.3 samples/s +Step 37200/44000 | Loss: 2.9158 | LR: 2.99e-05 | 1308.8ms/step | 15.3 samples/s +Step 37210/44000 | Loss: 2.9300 | LR: 2.98e-05 | 1307.8ms/step | 15.3 samples/s +Step 37220/44000 | Loss: 2.9101 | LR: 2.97e-05 | 1308.6ms/step | 15.3 samples/s +Step 37230/44000 | Loss: 2.9383 | LR: 2.96e-05 | 1308.8ms/step | 15.3 samples/s +Step 37240/44000 | Loss: 2.9137 | LR: 2.95e-05 | 1308.8ms/step | 15.3 samples/s +Step 37250/44000 | Loss: 2.9064 | LR: 2.95e-05 | 1308.2ms/step | 15.3 samples/s +Validation at step 37250: Loss = 2.9297, Perplexity = 18.72 +Step 37260/44000 | Loss: 2.9207 | LR: 2.94e-05 | 3487.9ms/step | 5.7 samples/s +Step 37270/44000 | Loss: 2.9323 | LR: 2.93e-05 | 1309.4ms/step | 15.3 samples/s +Step 37280/44000 | Loss: 2.9108 | LR: 2.92e-05 | 1312.7ms/step | 15.2 samples/s +Step 37290/44000 | Loss: 2.9220 | LR: 2.91e-05 | 1309.6ms/step | 15.3 samples/s +Step 37300/44000 | Loss: 2.9075 | LR: 2.90e-05 | 1307.1ms/step | 15.3 samples/s +Step 37310/44000 | Loss: 2.9104 | LR: 2.89e-05 | 1310.5ms/step | 15.3 samples/s +Step 37320/44000 | Loss: 2.9147 | LR: 2.89e-05 | 1309.3ms/step | 15.3 samples/s +Step 37330/44000 | Loss: 2.9381 | LR: 2.88e-05 | 1309.3ms/step | 15.3 samples/s +Step 37340/44000 | Loss: 2.9106 | LR: 2.87e-05 | 1308.9ms/step | 15.3 samples/s +Step 37350/44000 | Loss: 2.9296 | LR: 2.86e-05 | 1307.9ms/step | 15.3 samples/s +Step 37360/44000 | Loss: 2.9156 | LR: 2.85e-05 | 1306.7ms/step | 15.3 samples/s +Step 37370/44000 | Loss: 2.9166 | LR: 2.84e-05 | 1307.6ms/step | 15.3 samples/s +Step 37380/44000 | Loss: 2.8999 | LR: 2.84e-05 | 1307.9ms/step | 15.3 samples/s +Step 37390/44000 | Loss: 2.8970 | LR: 2.83e-05 | 1307.7ms/step | 15.3 samples/s +Step 37400/44000 | Loss: 2.9024 | LR: 2.82e-05 | 1308.8ms/step | 15.3 samples/s +Step 37410/44000 | Loss: 2.8977 | LR: 2.81e-05 | 1308.1ms/step | 15.3 samples/s +Step 37420/44000 | Loss: 2.9258 | LR: 2.80e-05 | 1308.2ms/step | 15.3 samples/s +Step 37430/44000 | Loss: 2.8874 | LR: 2.79e-05 | 1308.6ms/step | 15.3 samples/s +Step 37440/44000 | Loss: 2.9013 | LR: 2.78e-05 | 1308.8ms/step | 15.3 samples/s +Step 37450/44000 | Loss: 2.9142 | LR: 2.78e-05 | 1309.6ms/step | 15.3 samples/s +Step 37460/44000 | Loss: 2.9291 | LR: 2.77e-05 | 1310.2ms/step | 15.3 samples/s +Step 37470/44000 | Loss: 2.9127 | LR: 2.76e-05 | 1313.1ms/step | 15.2 samples/s +Step 37480/44000 | Loss: 2.9248 | LR: 2.75e-05 | 1309.7ms/step | 15.3 samples/s +Step 37490/44000 | Loss: 2.9066 | LR: 2.74e-05 | 1309.9ms/step | 15.3 samples/s +Step 37500/44000 | Loss: 2.9136 | LR: 2.74e-05 | 1308.5ms/step | 15.3 samples/s +Validation at step 37500: Loss = 2.9297, Perplexity = 18.72 +Step 37510/44000 | Loss: 2.9324 | LR: 2.73e-05 | 3490.3ms/step | 5.7 samples/s +Step 37520/44000 | Loss: 2.8877 | LR: 2.72e-05 | 1308.0ms/step | 15.3 samples/s +Step 37530/44000 | Loss: 2.9531 | LR: 2.71e-05 | 1310.5ms/step | 15.3 samples/s +Step 37540/44000 | Loss: 2.9222 | LR: 2.70e-05 | 1309.4ms/step | 15.3 samples/s +Step 37550/44000 | Loss: 2.9133 | LR: 2.69e-05 | 1308.7ms/step | 15.3 samples/s +Step 37560/44000 | Loss: 2.9313 | LR: 2.69e-05 | 1309.3ms/step | 15.3 samples/s +Step 37570/44000 | Loss: 2.9112 | LR: 2.68e-05 | 1310.8ms/step | 15.3 samples/s +Step 37580/44000 | Loss: 2.9046 | LR: 2.67e-05 | 1311.9ms/step | 15.2 samples/s +Step 37590/44000 | Loss: 2.9059 | LR: 2.66e-05 | 1313.8ms/step | 15.2 samples/s +Step 37600/44000 | Loss: 2.9104 | LR: 2.65e-05 | 1309.4ms/step | 15.3 samples/s +Step 37610/44000 | Loss: 2.9014 | LR: 2.65e-05 | 1309.2ms/step | 15.3 samples/s +Step 37620/44000 | Loss: 2.9239 | LR: 2.64e-05 | 1308.1ms/step | 15.3 samples/s +Step 37630/44000 | Loss: 2.8796 | LR: 2.63e-05 | 1308.8ms/step | 15.3 samples/s +Step 37640/44000 | Loss: 2.9274 | LR: 2.62e-05 | 1310.3ms/step | 15.3 samples/s +Step 37650/44000 | Loss: 2.8984 | LR: 2.61e-05 | 1310.9ms/step | 15.3 samples/s +Step 37660/44000 | Loss: 2.9385 | LR: 2.60e-05 | 1309.8ms/step | 15.3 samples/s +Step 37670/44000 | Loss: 2.9191 | LR: 2.60e-05 | 1309.2ms/step | 15.3 samples/s +Step 37680/44000 | Loss: 2.9299 | LR: 2.59e-05 | 1307.7ms/step | 15.3 samples/s +Step 37690/44000 | Loss: 2.9068 | LR: 2.58e-05 | 1308.9ms/step | 15.3 samples/s +Step 37700/44000 | Loss: 2.9028 | LR: 2.57e-05 | 1307.5ms/step | 15.3 samples/s +Step 37710/44000 | Loss: 2.8954 | LR: 2.56e-05 | 1309.6ms/step | 15.3 samples/s +Step 37720/44000 | Loss: 2.9299 | LR: 2.56e-05 | 1306.9ms/step | 15.3 samples/s +Step 37730/44000 | Loss: 2.9007 | LR: 2.55e-05 | 1309.1ms/step | 15.3 samples/s +Step 37740/44000 | Loss: 2.9135 | LR: 2.54e-05 | 1310.4ms/step | 15.3 samples/s +Step 37750/44000 | Loss: 2.8970 | LR: 2.53e-05 | 1310.2ms/step | 15.3 samples/s +Validation at step 37750: Loss = 2.9296, Perplexity = 18.72 +Step 37760/44000 | Loss: 2.9182 | LR: 2.52e-05 | 3496.6ms/step | 5.7 samples/s +Step 37770/44000 | Loss: 2.8940 | LR: 2.52e-05 | 1310.0ms/step | 15.3 samples/s +Step 37780/44000 | Loss: 2.8982 | LR: 2.51e-05 | 1310.7ms/step | 15.3 samples/s +Step 37790/44000 | Loss: 2.9332 | LR: 2.50e-05 | 1309.0ms/step | 15.3 samples/s +Step 37800/44000 | Loss: 2.9026 | LR: 2.49e-05 | 1311.4ms/step | 15.3 samples/s +Step 37810/44000 | Loss: 2.8787 | LR: 2.48e-05 | 1310.5ms/step | 15.3 samples/s +Step 37820/44000 | Loss: 2.9375 | LR: 2.48e-05 | 1310.3ms/step | 15.3 samples/s +Step 37830/44000 | Loss: 2.9357 | LR: 2.47e-05 | 1309.5ms/step | 15.3 samples/s +Step 37840/44000 | Loss: 2.9192 | LR: 2.46e-05 | 1309.1ms/step | 15.3 samples/s +Step 37850/44000 | Loss: 2.9004 | LR: 2.45e-05 | 1311.4ms/step | 15.3 samples/s +Step 37860/44000 | Loss: 2.8789 | LR: 2.45e-05 | 1310.4ms/step | 15.3 samples/s +Step 37870/44000 | Loss: 2.9055 | LR: 2.44e-05 | 1311.2ms/step | 15.3 samples/s +Step 37880/44000 | Loss: 2.9001 | LR: 2.43e-05 | 1310.6ms/step | 15.3 samples/s +Step 37890/44000 | Loss: 2.9245 | LR: 2.42e-05 | 1311.2ms/step | 15.3 samples/s +Step 37900/44000 | Loss: 2.9291 | LR: 2.41e-05 | 1311.1ms/step | 15.3 samples/s +Step 37910/44000 | Loss: 2.9125 | LR: 2.41e-05 | 1310.3ms/step | 15.3 samples/s +Step 37920/44000 | Loss: 2.9314 | LR: 2.40e-05 | 1309.8ms/step | 15.3 samples/s +Step 37930/44000 | Loss: 2.9097 | LR: 2.39e-05 | 1309.8ms/step | 15.3 samples/s +Step 37940/44000 | Loss: 2.9194 | LR: 2.38e-05 | 1310.2ms/step | 15.3 samples/s +Step 37950/44000 | Loss: 2.9346 | LR: 2.38e-05 | 1309.8ms/step | 15.3 samples/s +Step 37960/44000 | Loss: 2.9518 | LR: 2.37e-05 | 1308.9ms/step | 15.3 samples/s +Step 37970/44000 | Loss: 2.9092 | LR: 2.36e-05 | 1311.3ms/step | 15.3 samples/s +Step 37980/44000 | Loss: 2.9409 | LR: 2.35e-05 | 1307.8ms/step | 15.3 samples/s +Step 37990/44000 | Loss: 2.9103 | LR: 2.34e-05 | 1308.0ms/step | 15.3 samples/s +Step 38000/44000 | Loss: 2.9138 | LR: 2.34e-05 | 1310.1ms/step | 15.3 samples/s +Validation at step 38000: Loss = 2.9295, Perplexity = 18.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-38000 +Step 38010/44000 | Loss: 2.8661 | LR: 2.33e-05 | 3798.6ms/step | 5.3 samples/s +Step 38020/44000 | Loss: 2.8987 | LR: 2.32e-05 | 1310.7ms/step | 15.3 samples/s +Step 38030/44000 | Loss: 2.8670 | LR: 2.31e-05 | 1309.0ms/step | 15.3 samples/s +Step 38040/44000 | Loss: 2.9368 | LR: 2.31e-05 | 1309.2ms/step | 15.3 samples/s +Step 38050/44000 | Loss: 2.9210 | LR: 2.30e-05 | 1309.2ms/step | 15.3 samples/s +Step 38060/44000 | Loss: 2.9262 | LR: 2.29e-05 | 1313.9ms/step | 15.2 samples/s +Step 38070/44000 | Loss: 2.8980 | LR: 2.28e-05 | 1308.9ms/step | 15.3 samples/s +Step 38080/44000 | Loss: 2.9114 | LR: 2.28e-05 | 1308.6ms/step | 15.3 samples/s +Step 38090/44000 | Loss: 2.9239 | LR: 2.27e-05 | 1306.7ms/step | 15.3 samples/s +Step 38100/44000 | Loss: 2.9063 | LR: 2.26e-05 | 1308.2ms/step | 15.3 samples/s +Step 38110/44000 | Loss: 2.9198 | LR: 2.25e-05 | 1309.9ms/step | 15.3 samples/s +Step 38120/44000 | Loss: 2.9302 | LR: 2.25e-05 | 1310.0ms/step | 15.3 samples/s +Step 38130/44000 | Loss: 2.9062 | LR: 2.24e-05 | 1309.4ms/step | 15.3 samples/s +Step 38140/44000 | Loss: 2.9310 | LR: 2.23e-05 | 1310.1ms/step | 15.3 samples/s +Step 38150/44000 | Loss: 2.9507 | LR: 2.22e-05 | 1310.7ms/step | 15.3 samples/s +Step 38160/44000 | Loss: 2.9551 | LR: 2.22e-05 | 1311.6ms/step | 15.2 samples/s +Step 38170/44000 | Loss: 2.9178 | LR: 2.21e-05 | 1313.0ms/step | 15.2 samples/s +Step 38180/44000 | Loss: 2.9456 | LR: 2.20e-05 | 1307.8ms/step | 15.3 samples/s +Step 38190/44000 | Loss: 2.8892 | LR: 2.19e-05 | 1308.0ms/step | 15.3 samples/s +Step 38200/44000 | Loss: 2.9172 | LR: 2.19e-05 | 1316.3ms/step | 15.2 samples/s +Step 38210/44000 | Loss: 2.9167 | LR: 2.18e-05 | 1316.0ms/step | 15.2 samples/s +Step 38220/44000 | Loss: 2.9349 | LR: 2.17e-05 | 1315.4ms/step | 15.2 samples/s +Step 38230/44000 | Loss: 2.9197 | LR: 2.16e-05 | 1310.7ms/step | 15.3 samples/s +Step 38240/44000 | Loss: 2.9220 | LR: 2.16e-05 | 1314.3ms/step | 15.2 samples/s +Step 38250/44000 | Loss: 2.9225 | LR: 2.15e-05 | 1309.0ms/step | 15.3 samples/s +Validation at step 38250: Loss = 2.9295, Perplexity = 18.72 +Step 38260/44000 | Loss: 2.8967 | LR: 2.14e-05 | 3509.9ms/step | 5.7 samples/s +Step 38270/44000 | Loss: 2.9259 | LR: 2.13e-05 | 1317.5ms/step | 15.2 samples/s +Step 38280/44000 | Loss: 2.8849 | LR: 2.13e-05 | 1318.7ms/step | 15.2 samples/s +Step 38290/44000 | Loss: 2.8917 | LR: 2.12e-05 | 1319.2ms/step | 15.2 samples/s +Step 38300/44000 | Loss: 2.9266 | LR: 2.11e-05 | 1320.3ms/step | 15.1 samples/s +Step 38310/44000 | Loss: 2.8963 | LR: 2.11e-05 | 1310.4ms/step | 15.3 samples/s +Step 38320/44000 | Loss: 2.9148 | LR: 2.10e-05 | 1309.5ms/step | 15.3 samples/s +Step 38330/44000 | Loss: 2.9327 | LR: 2.09e-05 | 1312.5ms/step | 15.2 samples/s +Step 38340/44000 | Loss: 2.8821 | LR: 2.08e-05 | 1309.9ms/step | 15.3 samples/s +Step 38350/44000 | Loss: 2.9122 | LR: 2.08e-05 | 1310.6ms/step | 15.3 samples/s +Step 38360/44000 | Loss: 2.9189 | LR: 2.07e-05 | 1310.1ms/step | 15.3 samples/s +Step 38370/44000 | Loss: 2.9253 | LR: 2.06e-05 | 1308.1ms/step | 15.3 samples/s +Step 38380/44000 | Loss: 2.9314 | LR: 2.05e-05 | 1307.6ms/step | 15.3 samples/s +Step 38390/44000 | Loss: 2.9094 | LR: 2.05e-05 | 1312.0ms/step | 15.2 samples/s +Step 38400/44000 | Loss: 2.9284 | LR: 2.04e-05 | 1312.5ms/step | 15.2 samples/s +Step 38410/44000 | Loss: 2.9122 | LR: 2.03e-05 | 1309.0ms/step | 15.3 samples/s +Step 38420/44000 | Loss: 2.9220 | LR: 2.03e-05 | 1309.1ms/step | 15.3 samples/s +Step 38430/44000 | Loss: 2.9272 | LR: 2.02e-05 | 1307.7ms/step | 15.3 samples/s +Step 38440/44000 | Loss: 2.8796 | LR: 2.01e-05 | 1312.8ms/step | 15.2 samples/s +Step 38450/44000 | Loss: 2.9155 | LR: 2.00e-05 | 1307.9ms/step | 15.3 samples/s +Step 38460/44000 | Loss: 2.9157 | LR: 2.00e-05 | 1310.1ms/step | 15.3 samples/s +Step 38470/44000 | Loss: 2.9110 | LR: 1.99e-05 | 1309.4ms/step | 15.3 samples/s +Step 38480/44000 | Loss: 2.9345 | LR: 1.98e-05 | 1306.3ms/step | 15.3 samples/s +Step 38490/44000 | Loss: 2.9217 | LR: 1.98e-05 | 1310.2ms/step | 15.3 samples/s +Step 38500/44000 | Loss: 2.9228 | LR: 1.97e-05 | 1311.6ms/step | 15.2 samples/s +Validation at step 38500: Loss = 2.9294, Perplexity = 18.72 +Step 38510/44000 | Loss: 2.9143 | LR: 1.96e-05 | 3487.7ms/step | 5.7 samples/s +Step 38520/44000 | Loss: 2.8918 | LR: 1.95e-05 | 1308.1ms/step | 15.3 samples/s +Step 38530/44000 | Loss: 2.9141 | LR: 1.95e-05 | 1308.9ms/step | 15.3 samples/s +Step 38540/44000 | Loss: 2.9244 | LR: 1.94e-05 | 1305.4ms/step | 15.3 samples/s +Step 38550/44000 | Loss: 2.9333 | LR: 1.93e-05 | 1308.8ms/step | 15.3 samples/s +Step 38560/44000 | Loss: 2.9175 | LR: 1.93e-05 | 1312.4ms/step | 15.2 samples/s +Step 38570/44000 | Loss: 2.9105 | LR: 1.92e-05 | 1311.5ms/step | 15.3 samples/s +Step 38580/44000 | Loss: 2.8920 | LR: 1.91e-05 | 1311.4ms/step | 15.3 samples/s +Step 38590/44000 | Loss: 2.9202 | LR: 1.91e-05 | 1310.8ms/step | 15.3 samples/s +Step 38600/44000 | Loss: 2.8977 | LR: 1.90e-05 | 1308.8ms/step | 15.3 samples/s +Step 38610/44000 | Loss: 2.9017 | LR: 1.89e-05 | 1311.2ms/step | 15.3 samples/s +Step 38620/44000 | Loss: 2.9220 | LR: 1.88e-05 | 1312.6ms/step | 15.2 samples/s +Step 38630/44000 | Loss: 2.8988 | LR: 1.88e-05 | 1312.2ms/step | 15.2 samples/s +Step 38640/44000 | Loss: 2.9384 | LR: 1.87e-05 | 1308.0ms/step | 15.3 samples/s +Step 38650/44000 | Loss: 2.8927 | LR: 1.86e-05 | 1310.4ms/step | 15.3 samples/s +Step 38660/44000 | Loss: 2.9018 | LR: 1.86e-05 | 1314.7ms/step | 15.2 samples/s +Step 38670/44000 | Loss: 2.9191 | LR: 1.85e-05 | 1310.1ms/step | 15.3 samples/s +Step 38680/44000 | Loss: 2.9414 | LR: 1.84e-05 | 1307.8ms/step | 15.3 samples/s +Step 38690/44000 | Loss: 2.9162 | LR: 1.84e-05 | 1309.7ms/step | 15.3 samples/s +Step 38700/44000 | Loss: 2.9337 | LR: 1.83e-05 | 1310.3ms/step | 15.3 samples/s +Step 38710/44000 | Loss: 2.9193 | LR: 1.82e-05 | 1309.9ms/step | 15.3 samples/s +Step 38720/44000 | Loss: 2.8924 | LR: 1.82e-05 | 1308.8ms/step | 15.3 samples/s +Step 38730/44000 | Loss: 2.9067 | LR: 1.81e-05 | 1307.2ms/step | 15.3 samples/s +Step 38740/44000 | Loss: 2.9292 | LR: 1.80e-05 | 1307.9ms/step | 15.3 samples/s +Step 38750/44000 | Loss: 2.9109 | LR: 1.80e-05 | 1307.4ms/step | 15.3 samples/s +Validation at step 38750: Loss = 2.9293, Perplexity = 18.72 +Step 38760/44000 | Loss: 2.9217 | LR: 1.79e-05 | 3489.3ms/step | 5.7 samples/s +Step 38770/44000 | Loss: 2.9101 | LR: 1.78e-05 | 1308.3ms/step | 15.3 samples/s +Step 38780/44000 | Loss: 2.9000 | LR: 1.78e-05 | 1310.4ms/step | 15.3 samples/s +Step 38790/44000 | Loss: 2.9333 | LR: 1.77e-05 | 1308.5ms/step | 15.3 samples/s +Step 38800/44000 | Loss: 2.8991 | LR: 1.76e-05 | 1309.2ms/step | 15.3 samples/s +Step 38810/44000 | Loss: 2.8831 | LR: 1.76e-05 | 1308.6ms/step | 15.3 samples/s +Step 38820/44000 | Loss: 2.9187 | LR: 1.75e-05 | 1307.5ms/step | 15.3 samples/s +Step 38830/44000 | Loss: 2.9206 | LR: 1.74e-05 | 1308.7ms/step | 15.3 samples/s +Step 38840/44000 | Loss: 2.8838 | LR: 1.74e-05 | 1307.2ms/step | 15.3 samples/s +Step 38850/44000 | Loss: 2.9218 | LR: 1.73e-05 | 1308.9ms/step | 15.3 samples/s +Step 38860/44000 | Loss: 2.9089 | LR: 1.72e-05 | 1308.7ms/step | 15.3 samples/s +Step 38870/44000 | Loss: 2.9263 | LR: 1.72e-05 | 1309.6ms/step | 15.3 samples/s +Step 38880/44000 | Loss: 2.9277 | LR: 1.71e-05 | 1309.7ms/step | 15.3 samples/s +Step 38890/44000 | Loss: 2.9133 | LR: 1.70e-05 | 1310.9ms/step | 15.3 samples/s +Step 38900/44000 | Loss: 2.9119 | LR: 1.70e-05 | 1310.1ms/step | 15.3 samples/s +Step 38910/44000 | Loss: 2.9329 | LR: 1.69e-05 | 1315.5ms/step | 15.2 samples/s +Step 38920/44000 | Loss: 2.9152 | LR: 1.68e-05 | 1312.4ms/step | 15.2 samples/s +Step 38930/44000 | Loss: 2.8948 | LR: 1.68e-05 | 1309.9ms/step | 15.3 samples/s +Step 38940/44000 | Loss: 2.9013 | LR: 1.67e-05 | 1310.2ms/step | 15.3 samples/s +Step 38950/44000 | Loss: 2.8895 | LR: 1.66e-05 | 1308.9ms/step | 15.3 samples/s +Step 38960/44000 | Loss: 2.9034 | LR: 1.66e-05 | 1309.0ms/step | 15.3 samples/s +Step 38970/44000 | Loss: 2.9202 | LR: 1.65e-05 | 1311.4ms/step | 15.3 samples/s +Step 38980/44000 | Loss: 2.8952 | LR: 1.64e-05 | 1314.3ms/step | 15.2 samples/s +Step 38990/44000 | Loss: 2.9255 | LR: 1.64e-05 | 1314.2ms/step | 15.2 samples/s +Step 39000/44000 | Loss: 2.9121 | LR: 1.63e-05 | 1309.2ms/step | 15.3 samples/s +Validation at step 39000: Loss = 2.9293, Perplexity = 18.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-39000 +Step 39010/44000 | Loss: 2.9500 | LR: 1.62e-05 | 3788.1ms/step | 5.3 samples/s +Step 39020/44000 | Loss: 2.9023 | LR: 1.62e-05 | 1310.3ms/step | 15.3 samples/s +Step 39030/44000 | Loss: 2.9399 | LR: 1.61e-05 | 1305.5ms/step | 15.3 samples/s +Step 39040/44000 | Loss: 2.8951 | LR: 1.61e-05 | 1305.2ms/step | 15.3 samples/s +Step 39050/44000 | Loss: 2.9018 | LR: 1.60e-05 | 1306.9ms/step | 15.3 samples/s +Step 39060/44000 | Loss: 2.9144 | LR: 1.59e-05 | 1310.6ms/step | 15.3 samples/s +Step 39070/44000 | Loss: 2.8923 | LR: 1.59e-05 | 1303.4ms/step | 15.3 samples/s +Step 39080/44000 | Loss: 2.9215 | LR: 1.58e-05 | 1303.2ms/step | 15.3 samples/s +Step 39090/44000 | Loss: 2.8829 | LR: 1.57e-05 | 1307.9ms/step | 15.3 samples/s +Step 39100/44000 | Loss: 2.8875 | LR: 1.57e-05 | 1306.0ms/step | 15.3 samples/s +Step 39110/44000 | Loss: 2.9270 | LR: 1.56e-05 | 1303.4ms/step | 15.3 samples/s +Step 39120/44000 | Loss: 2.9174 | LR: 1.55e-05 | 1302.6ms/step | 15.4 samples/s +Step 39130/44000 | Loss: 2.8883 | LR: 1.55e-05 | 1305.8ms/step | 15.3 samples/s +Step 39140/44000 | Loss: 2.9062 | LR: 1.54e-05 | 1304.1ms/step | 15.3 samples/s +Step 39150/44000 | Loss: 2.9197 | LR: 1.54e-05 | 1304.9ms/step | 15.3 samples/s +Step 39160/44000 | Loss: 2.9458 | LR: 1.53e-05 | 1304.0ms/step | 15.3 samples/s +Step 39170/44000 | Loss: 2.9236 | LR: 1.52e-05 | 1303.9ms/step | 15.3 samples/s +Step 39180/44000 | Loss: 2.9254 | LR: 1.52e-05 | 1305.7ms/step | 15.3 samples/s +Step 39190/44000 | Loss: 2.8683 | LR: 1.51e-05 | 1307.8ms/step | 15.3 samples/s +Step 39200/44000 | Loss: 2.8921 | LR: 1.50e-05 | 1304.7ms/step | 15.3 samples/s +Step 39210/44000 | Loss: 2.8798 | LR: 1.50e-05 | 1305.9ms/step | 15.3 samples/s +Step 39220/44000 | Loss: 2.9182 | LR: 1.49e-05 | 1309.1ms/step | 15.3 samples/s +Step 39230/44000 | Loss: 2.9040 | LR: 1.49e-05 | 1307.9ms/step | 15.3 samples/s +Step 39240/44000 | Loss: 2.9066 | LR: 1.48e-05 | 1307.7ms/step | 15.3 samples/s +Step 39250/44000 | Loss: 2.9062 | LR: 1.47e-05 | 1306.1ms/step | 15.3 samples/s +Validation at step 39250: Loss = 2.9293, Perplexity = 18.71 +Step 39260/44000 | Loss: 2.9036 | LR: 1.47e-05 | 3485.0ms/step | 5.7 samples/s +Step 39270/44000 | Loss: 2.8953 | LR: 1.46e-05 | 1305.0ms/step | 15.3 samples/s +Step 39280/44000 | Loss: 2.9090 | LR: 1.46e-05 | 1304.4ms/step | 15.3 samples/s +Step 39290/44000 | Loss: 2.9073 | LR: 1.45e-05 | 1303.4ms/step | 15.3 samples/s +Step 39300/44000 | Loss: 2.9066 | LR: 1.44e-05 | 1305.5ms/step | 15.3 samples/s +Step 39310/44000 | Loss: 2.8990 | LR: 1.44e-05 | 1305.8ms/step | 15.3 samples/s +Step 39320/44000 | Loss: 2.8758 | LR: 1.43e-05 | 1301.9ms/step | 15.4 samples/s +Step 39330/44000 | Loss: 2.9190 | LR: 1.42e-05 | 1301.8ms/step | 15.4 samples/s +Step 39340/44000 | Loss: 2.9164 | LR: 1.42e-05 | 1300.9ms/step | 15.4 samples/s +Step 39350/44000 | Loss: 2.9084 | LR: 1.41e-05 | 1301.2ms/step | 15.4 samples/s +Step 39360/44000 | Loss: 2.9466 | LR: 1.41e-05 | 1301.9ms/step | 15.4 samples/s +Step 39370/44000 | Loss: 2.8907 | LR: 1.40e-05 | 1305.1ms/step | 15.3 samples/s +Step 39380/44000 | Loss: 2.9311 | LR: 1.39e-05 | 1302.3ms/step | 15.4 samples/s +Step 39390/44000 | Loss: 2.9150 | LR: 1.39e-05 | 1302.6ms/step | 15.4 samples/s +Step 39400/44000 | Loss: 2.9033 | LR: 1.38e-05 | 1301.9ms/step | 15.4 samples/s +Step 39410/44000 | Loss: 2.9126 | LR: 1.38e-05 | 1300.8ms/step | 15.4 samples/s +Step 39420/44000 | Loss: 2.9350 | LR: 1.37e-05 | 1297.2ms/step | 15.4 samples/s +Step 39430/44000 | Loss: 2.9319 | LR: 1.36e-05 | 1297.2ms/step | 15.4 samples/s +Step 39440/44000 | Loss: 2.8995 | LR: 1.36e-05 | 1298.9ms/step | 15.4 samples/s +Step 39450/44000 | Loss: 2.9324 | LR: 1.35e-05 | 1302.8ms/step | 15.4 samples/s +Step 39460/44000 | Loss: 2.8957 | LR: 1.35e-05 | 1301.5ms/step | 15.4 samples/s +Step 39470/44000 | Loss: 2.9010 | LR: 1.34e-05 | 1302.3ms/step | 15.4 samples/s +Step 39480/44000 | Loss: 2.9226 | LR: 1.34e-05 | 1301.3ms/step | 15.4 samples/s +Step 39490/44000 | Loss: 2.9244 | LR: 1.33e-05 | 1301.0ms/step | 15.4 samples/s +Step 39500/44000 | Loss: 2.9065 | LR: 1.32e-05 | 1302.9ms/step | 15.4 samples/s +Validation at step 39500: Loss = 2.9293, Perplexity = 18.71 +Step 39510/44000 | Loss: 2.8912 | LR: 1.32e-05 | 3487.7ms/step | 5.7 samples/s +Step 39520/44000 | Loss: 2.8911 | LR: 1.31e-05 | 1302.9ms/step | 15.4 samples/s +Step 39530/44000 | Loss: 2.9112 | LR: 1.31e-05 | 1300.0ms/step | 15.4 samples/s +Step 39540/44000 | Loss: 2.9138 | LR: 1.30e-05 | 1297.2ms/step | 15.4 samples/s +Step 39550/44000 | Loss: 2.9079 | LR: 1.29e-05 | 1301.3ms/step | 15.4 samples/s +Step 39560/44000 | Loss: 2.9214 | LR: 1.29e-05 | 1306.7ms/step | 15.3 samples/s +Step 39570/44000 | Loss: 2.8664 | LR: 1.28e-05 | 1305.9ms/step | 15.3 samples/s +Step 39580/44000 | Loss: 2.9085 | LR: 1.28e-05 | 1304.0ms/step | 15.3 samples/s +Step 39590/44000 | Loss: 2.8959 | LR: 1.27e-05 | 1303.1ms/step | 15.3 samples/s +Step 39600/44000 | Loss: 2.9052 | LR: 1.27e-05 | 1304.1ms/step | 15.3 samples/s +Step 39610/44000 | Loss: 2.9080 | LR: 1.26e-05 | 1304.8ms/step | 15.3 samples/s +Step 39620/44000 | Loss: 2.9153 | LR: 1.25e-05 | 1302.0ms/step | 15.4 samples/s +Step 39630/44000 | Loss: 2.9156 | LR: 1.25e-05 | 1301.2ms/step | 15.4 samples/s +Step 39640/44000 | Loss: 2.9173 | LR: 1.24e-05 | 1303.9ms/step | 15.3 samples/s +Step 39650/44000 | Loss: 2.9157 | LR: 1.24e-05 | 1300.7ms/step | 15.4 samples/s +Step 39660/44000 | Loss: 2.9038 | LR: 1.23e-05 | 1303.8ms/step | 15.3 samples/s +Step 39670/44000 | Loss: 2.8941 | LR: 1.23e-05 | 1304.1ms/step | 15.3 samples/s +Step 39680/44000 | Loss: 2.9279 | LR: 1.22e-05 | 1305.1ms/step | 15.3 samples/s +Step 39690/44000 | Loss: 2.9267 | LR: 1.22e-05 | 1307.4ms/step | 15.3 samples/s +Step 39700/44000 | Loss: 2.9174 | LR: 1.21e-05 | 1307.7ms/step | 15.3 samples/s +Step 39710/44000 | Loss: 2.9544 | LR: 1.20e-05 | 1308.8ms/step | 15.3 samples/s +Step 39720/44000 | Loss: 2.9235 | LR: 1.20e-05 | 1311.1ms/step | 15.3 samples/s +Step 39730/44000 | Loss: 2.9194 | LR: 1.19e-05 | 1308.9ms/step | 15.3 samples/s +Step 39740/44000 | Loss: 2.9252 | LR: 1.19e-05 | 1304.3ms/step | 15.3 samples/s +Step 39750/44000 | Loss: 2.9397 | LR: 1.18e-05 | 1306.2ms/step | 15.3 samples/s +Validation at step 39750: Loss = 2.9292, Perplexity = 18.71 +Step 39760/44000 | Loss: 2.9436 | LR: 1.18e-05 | 3487.6ms/step | 5.7 samples/s +Step 39770/44000 | Loss: 2.9074 | LR: 1.17e-05 | 1310.1ms/step | 15.3 samples/s +Step 39780/44000 | Loss: 2.9096 | LR: 1.17e-05 | 1303.6ms/step | 15.3 samples/s +Step 39790/44000 | Loss: 2.8964 | LR: 1.16e-05 | 1303.4ms/step | 15.3 samples/s +Step 39800/44000 | Loss: 2.9128 | LR: 1.15e-05 | 1303.8ms/step | 15.3 samples/s +Step 39810/44000 | Loss: 2.8976 | LR: 1.15e-05 | 1299.6ms/step | 15.4 samples/s +Step 39820/44000 | Loss: 2.9078 | LR: 1.14e-05 | 1300.1ms/step | 15.4 samples/s +Step 39830/44000 | Loss: 2.9179 | LR: 1.14e-05 | 1300.1ms/step | 15.4 samples/s +Step 39840/44000 | Loss: 2.9332 | LR: 1.13e-05 | 1300.8ms/step | 15.4 samples/s +Step 39850/44000 | Loss: 2.9249 | LR: 1.13e-05 | 1302.1ms/step | 15.4 samples/s +Step 39860/44000 | Loss: 2.9165 | LR: 1.12e-05 | 1300.8ms/step | 15.4 samples/s +Step 39870/44000 | Loss: 2.9078 | LR: 1.12e-05 | 1301.2ms/step | 15.4 samples/s +Step 39880/44000 | Loss: 2.8851 | LR: 1.11e-05 | 1301.8ms/step | 15.4 samples/s +Step 39890/44000 | Loss: 2.9014 | LR: 1.11e-05 | 1302.7ms/step | 15.4 samples/s +Step 39900/44000 | Loss: 2.9161 | LR: 1.10e-05 | 1303.7ms/step | 15.3 samples/s +Step 39910/44000 | Loss: 2.8839 | LR: 1.10e-05 | 1300.9ms/step | 15.4 samples/s +Step 39920/44000 | Loss: 2.9401 | LR: 1.09e-05 | 1304.0ms/step | 15.3 samples/s +Step 39930/44000 | Loss: 2.9137 | LR: 1.08e-05 | 1307.0ms/step | 15.3 samples/s +Step 39940/44000 | Loss: 2.9353 | LR: 1.08e-05 | 1304.6ms/step | 15.3 samples/s +Step 39950/44000 | Loss: 2.8971 | LR: 1.07e-05 | 1303.1ms/step | 15.3 samples/s +Step 39960/44000 | Loss: 2.9169 | LR: 1.07e-05 | 1303.4ms/step | 15.3 samples/s +Step 39970/44000 | Loss: 2.9126 | LR: 1.06e-05 | 1304.0ms/step | 15.3 samples/s +Step 39980/44000 | Loss: 2.9022 | LR: 1.06e-05 | 1303.8ms/step | 15.3 samples/s +Step 39990/44000 | Loss: 2.9263 | LR: 1.05e-05 | 1306.2ms/step | 15.3 samples/s +Step 40000/44000 | Loss: 2.8964 | LR: 1.05e-05 | 1303.8ms/step | 15.3 samples/s +Validation at step 40000: Loss = 2.9292, Perplexity = 18.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-40000 +Step 40010/44000 | Loss: 2.9150 | LR: 1.04e-05 | 3799.7ms/step | 5.3 samples/s +Step 40020/44000 | Loss: 2.9159 | LR: 1.04e-05 | 1299.1ms/step | 15.4 samples/s +Step 40030/44000 | Loss: 2.9264 | LR: 1.03e-05 | 1299.0ms/step | 15.4 samples/s +Step 40040/44000 | Loss: 2.9072 | LR: 1.03e-05 | 1300.2ms/step | 15.4 samples/s +Step 40050/44000 | Loss: 2.8890 | LR: 1.02e-05 | 1302.1ms/step | 15.4 samples/s +Step 40060/44000 | Loss: 2.8985 | LR: 1.02e-05 | 1304.5ms/step | 15.3 samples/s +Step 40070/44000 | Loss: 2.9199 | LR: 1.01e-05 | 1302.2ms/step | 15.4 samples/s +Step 40080/44000 | Loss: 2.9239 | LR: 1.01e-05 | 1298.8ms/step | 15.4 samples/s +Step 40090/44000 | Loss: 2.9043 | LR: 1.00e-05 | 1300.3ms/step | 15.4 samples/s +Step 40100/44000 | Loss: 2.9119 | LR: 9.96e-06 | 1300.2ms/step | 15.4 samples/s +Step 40110/44000 | Loss: 2.9129 | LR: 9.91e-06 | 1300.7ms/step | 15.4 samples/s +Step 40120/44000 | Loss: 2.9262 | LR: 9.86e-06 | 1299.1ms/step | 15.4 samples/s +Step 40130/44000 | Loss: 2.9199 | LR: 9.81e-06 | 1299.9ms/step | 15.4 samples/s +Step 40140/44000 | Loss: 2.9253 | LR: 9.76e-06 | 1299.7ms/step | 15.4 samples/s +Step 40150/44000 | Loss: 2.8854 | LR: 9.71e-06 | 1295.5ms/step | 15.4 samples/s +Step 40160/44000 | Loss: 2.8929 | LR: 9.66e-06 | 1299.1ms/step | 15.4 samples/s +Step 40170/44000 | Loss: 2.8977 | LR: 9.61e-06 | 1301.8ms/step | 15.4 samples/s +Step 40180/44000 | Loss: 2.9369 | LR: 9.56e-06 | 1299.4ms/step | 15.4 samples/s +Step 40190/44000 | Loss: 2.9128 | LR: 9.51e-06 | 1298.0ms/step | 15.4 samples/s +Step 40200/44000 | Loss: 2.9455 | LR: 9.46e-06 | 1300.7ms/step | 15.4 samples/s +Step 40210/44000 | Loss: 2.9100 | LR: 9.41e-06 | 1300.8ms/step | 15.4 samples/s +Step 40220/44000 | Loss: 2.9288 | LR: 9.36e-06 | 1296.2ms/step | 15.4 samples/s +Step 40230/44000 | Loss: 2.9027 | LR: 9.32e-06 | 1298.7ms/step | 15.4 samples/s +Step 40240/44000 | Loss: 2.9124 | LR: 9.27e-06 | 1300.0ms/step | 15.4 samples/s +Step 40250/44000 | Loss: 2.8746 | LR: 9.22e-06 | 1297.4ms/step | 15.4 samples/s +Validation at step 40250: Loss = 2.9292, Perplexity = 18.71 +Step 40260/44000 | Loss: 2.9033 | LR: 9.17e-06 | 3484.8ms/step | 5.7 samples/s +Step 40270/44000 | Loss: 2.9214 | LR: 9.12e-06 | 1305.8ms/step | 15.3 samples/s +Step 40280/44000 | Loss: 2.9172 | LR: 9.07e-06 | 1305.2ms/step | 15.3 samples/s +Step 40290/44000 | Loss: 2.8915 | LR: 9.02e-06 | 1304.4ms/step | 15.3 samples/s +Step 40300/44000 | Loss: 2.9190 | LR: 8.97e-06 | 1301.3ms/step | 15.4 samples/s +Step 40310/44000 | Loss: 2.9074 | LR: 8.93e-06 | 1301.1ms/step | 15.4 samples/s +Step 40320/44000 | Loss: 2.9346 | LR: 8.88e-06 | 1300.8ms/step | 15.4 samples/s +Step 40330/44000 | Loss: 2.9156 | LR: 8.83e-06 | 1300.9ms/step | 15.4 samples/s +Step 40340/44000 | Loss: 2.9131 | LR: 8.78e-06 | 1300.3ms/step | 15.4 samples/s +Step 40350/44000 | Loss: 2.9028 | LR: 8.74e-06 | 1298.7ms/step | 15.4 samples/s +Step 40360/44000 | Loss: 2.9174 | LR: 8.69e-06 | 1300.6ms/step | 15.4 samples/s +Step 40370/44000 | Loss: 2.8949 | LR: 8.64e-06 | 1299.3ms/step | 15.4 samples/s +Step 40380/44000 | Loss: 2.9140 | LR: 8.59e-06 | 1301.7ms/step | 15.4 samples/s +Step 40390/44000 | Loss: 2.9312 | LR: 8.55e-06 | 1302.7ms/step | 15.4 samples/s +Step 40400/44000 | Loss: 2.8986 | LR: 8.50e-06 | 1304.1ms/step | 15.3 samples/s +Step 40410/44000 | Loss: 2.8851 | LR: 8.45e-06 | 1304.9ms/step | 15.3 samples/s +Step 40420/44000 | Loss: 2.9080 | LR: 8.41e-06 | 1302.1ms/step | 15.4 samples/s +Step 40430/44000 | Loss: 2.9175 | LR: 8.36e-06 | 1301.6ms/step | 15.4 samples/s +Step 40440/44000 | Loss: 2.8960 | LR: 8.31e-06 | 1302.3ms/step | 15.4 samples/s +Step 40450/44000 | Loss: 2.9066 | LR: 8.27e-06 | 1302.3ms/step | 15.4 samples/s +Step 40460/44000 | Loss: 2.8752 | LR: 8.22e-06 | 1302.7ms/step | 15.4 samples/s +Step 40470/44000 | Loss: 2.9298 | LR: 8.17e-06 | 1302.9ms/step | 15.4 samples/s +Step 40480/44000 | Loss: 2.9172 | LR: 8.13e-06 | 1303.5ms/step | 15.3 samples/s +Step 40490/44000 | Loss: 2.9186 | LR: 8.08e-06 | 1305.4ms/step | 15.3 samples/s +Step 40500/44000 | Loss: 2.9125 | LR: 8.04e-06 | 1302.4ms/step | 15.4 samples/s +Validation at step 40500: Loss = 2.9292, Perplexity = 18.71 +Step 40510/44000 | Loss: 2.9004 | LR: 7.99e-06 | 3485.7ms/step | 5.7 samples/s +Step 40520/44000 | Loss: 2.9475 | LR: 7.94e-06 | 1304.0ms/step | 15.3 samples/s +Step 40530/44000 | Loss: 2.9354 | LR: 7.90e-06 | 1304.5ms/step | 15.3 samples/s +Step 40540/44000 | Loss: 2.9281 | LR: 7.85e-06 | 1304.1ms/step | 15.3 samples/s +Step 40550/44000 | Loss: 2.8863 | LR: 7.81e-06 | 1306.9ms/step | 15.3 samples/s +Step 40560/44000 | Loss: 2.8858 | LR: 7.76e-06 | 1304.3ms/step | 15.3 samples/s +Step 40570/44000 | Loss: 2.9196 | LR: 7.72e-06 | 1302.9ms/step | 15.3 samples/s +Step 40580/44000 | Loss: 2.9116 | LR: 7.67e-06 | 1304.1ms/step | 15.3 samples/s +Step 40590/44000 | Loss: 2.8889 | LR: 7.63e-06 | 1304.2ms/step | 15.3 samples/s +Step 40600/44000 | Loss: 2.9158 | LR: 7.59e-06 | 1308.9ms/step | 15.3 samples/s +Step 40610/44000 | Loss: 2.9472 | LR: 7.54e-06 | 1306.9ms/step | 15.3 samples/s +Step 40620/44000 | Loss: 2.9128 | LR: 7.50e-06 | 1301.0ms/step | 15.4 samples/s +Step 40630/44000 | Loss: 2.9268 | LR: 7.45e-06 | 1301.9ms/step | 15.4 samples/s +Step 40640/44000 | Loss: 2.9327 | LR: 7.41e-06 | 1305.0ms/step | 15.3 samples/s +Step 40650/44000 | Loss: 2.9306 | LR: 7.37e-06 | 1303.1ms/step | 15.3 samples/s +Step 40660/44000 | Loss: 2.9220 | LR: 7.32e-06 | 1309.9ms/step | 15.3 samples/s +Step 40670/44000 | Loss: 2.9297 | LR: 7.28e-06 | 1303.9ms/step | 15.3 samples/s +Step 40680/44000 | Loss: 2.9157 | LR: 7.23e-06 | 1304.3ms/step | 15.3 samples/s +Step 40690/44000 | Loss: 2.9251 | LR: 7.19e-06 | 1304.6ms/step | 15.3 samples/s +Step 40700/44000 | Loss: 2.9032 | LR: 7.15e-06 | 1302.8ms/step | 15.4 samples/s +Step 40710/44000 | Loss: 2.9189 | LR: 7.10e-06 | 1303.5ms/step | 15.3 samples/s +Step 40720/44000 | Loss: 2.9262 | LR: 7.06e-06 | 1303.0ms/step | 15.3 samples/s +Step 40730/44000 | Loss: 2.9342 | LR: 7.02e-06 | 1303.1ms/step | 15.3 samples/s +Step 40740/44000 | Loss: 2.9058 | LR: 6.98e-06 | 1304.9ms/step | 15.3 samples/s +Step 40750/44000 | Loss: 2.9026 | LR: 6.93e-06 | 1303.4ms/step | 15.3 samples/s +Validation at step 40750: Loss = 2.9292, Perplexity = 18.71 +Step 40760/44000 | Loss: 2.9301 | LR: 6.89e-06 | 3486.8ms/step | 5.7 samples/s +Step 40770/44000 | Loss: 2.9110 | LR: 6.85e-06 | 1304.5ms/step | 15.3 samples/s +Step 40780/44000 | Loss: 2.9301 | LR: 6.81e-06 | 1303.1ms/step | 15.3 samples/s +Step 40790/44000 | Loss: 2.9328 | LR: 6.77e-06 | 1304.2ms/step | 15.3 samples/s +Step 40800/44000 | Loss: 2.9364 | LR: 6.72e-06 | 1302.7ms/step | 15.4 samples/s +Step 40810/44000 | Loss: 2.9157 | LR: 6.68e-06 | 1302.2ms/step | 15.4 samples/s +Step 40820/44000 | Loss: 2.8941 | LR: 6.64e-06 | 1303.3ms/step | 15.3 samples/s +Step 40830/44000 | Loss: 2.8899 | LR: 6.60e-06 | 1303.6ms/step | 15.3 samples/s +Step 40840/44000 | Loss: 2.9338 | LR: 6.56e-06 | 1304.0ms/step | 15.3 samples/s +Step 40850/44000 | Loss: 2.9214 | LR: 6.52e-06 | 1304.7ms/step | 15.3 samples/s +Step 40860/44000 | Loss: 2.9329 | LR: 6.47e-06 | 1302.3ms/step | 15.4 samples/s +Step 40870/44000 | Loss: 2.9364 | LR: 6.43e-06 | 1304.2ms/step | 15.3 samples/s +Step 40880/44000 | Loss: 2.9079 | LR: 6.39e-06 | 1302.3ms/step | 15.4 samples/s +Step 40890/44000 | Loss: 2.9253 | LR: 6.35e-06 | 1302.4ms/step | 15.4 samples/s +Step 40900/44000 | Loss: 2.9010 | LR: 6.31e-06 | 1305.3ms/step | 15.3 samples/s +Step 40910/44000 | Loss: 2.9704 | LR: 6.27e-06 | 1305.1ms/step | 15.3 samples/s +Step 40920/44000 | Loss: 2.9188 | LR: 6.23e-06 | 1304.9ms/step | 15.3 samples/s +Step 40930/44000 | Loss: 2.9279 | LR: 6.19e-06 | 1304.2ms/step | 15.3 samples/s +Step 40940/44000 | Loss: 2.9301 | LR: 6.15e-06 | 1302.7ms/step | 15.4 samples/s +Step 40950/44000 | Loss: 2.8912 | LR: 6.11e-06 | 1302.9ms/step | 15.4 samples/s +Step 40960/44000 | Loss: 2.8781 | LR: 6.07e-06 | 1303.8ms/step | 15.3 samples/s +Step 40970/44000 | Loss: 2.9078 | LR: 6.03e-06 | 1304.0ms/step | 15.3 samples/s +Step 40980/44000 | Loss: 2.8970 | LR: 5.99e-06 | 1304.9ms/step | 15.3 samples/s +Step 40990/44000 | Loss: 2.9192 | LR: 5.95e-06 | 1303.2ms/step | 15.3 samples/s +Step 41000/44000 | Loss: 2.9227 | LR: 5.91e-06 | 1303.9ms/step | 15.3 samples/s +Validation at step 41000: Loss = 2.9292, Perplexity = 18.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-41000 +Step 41010/44000 | Loss: 2.9306 | LR: 5.87e-06 | 3783.9ms/step | 5.3 samples/s +Step 41020/44000 | Loss: 2.9370 | LR: 5.83e-06 | 1301.0ms/step | 15.4 samples/s +Step 41030/44000 | Loss: 2.9129 | LR: 5.80e-06 | 1304.0ms/step | 15.3 samples/s +Step 41040/44000 | Loss: 2.9130 | LR: 5.76e-06 | 1300.7ms/step | 15.4 samples/s +Step 41050/44000 | Loss: 2.9103 | LR: 5.72e-06 | 1300.4ms/step | 15.4 samples/s +Step 41060/44000 | Loss: 2.8805 | LR: 5.68e-06 | 1303.0ms/step | 15.3 samples/s +Step 41070/44000 | Loss: 2.9083 | LR: 5.64e-06 | 1301.5ms/step | 15.4 samples/s +Step 41080/44000 | Loss: 2.9410 | LR: 5.60e-06 | 1301.8ms/step | 15.4 samples/s +Step 41090/44000 | Loss: 2.9564 | LR: 5.56e-06 | 1304.5ms/step | 15.3 samples/s +Step 41100/44000 | Loss: 2.9356 | LR: 5.53e-06 | 1301.2ms/step | 15.4 samples/s +Step 41110/44000 | Loss: 2.9566 | LR: 5.49e-06 | 1302.5ms/step | 15.4 samples/s +Step 41120/44000 | Loss: 2.9459 | LR: 5.45e-06 | 1303.2ms/step | 15.3 samples/s +Step 41130/44000 | Loss: 2.9136 | LR: 5.41e-06 | 1303.2ms/step | 15.3 samples/s +Step 41140/44000 | Loss: 2.8921 | LR: 5.38e-06 | 1304.8ms/step | 15.3 samples/s +Step 41150/44000 | Loss: 2.9349 | LR: 5.34e-06 | 1303.9ms/step | 15.3 samples/s +Step 41160/44000 | Loss: 2.9074 | LR: 5.30e-06 | 1301.2ms/step | 15.4 samples/s +Step 41170/44000 | Loss: 2.9179 | LR: 5.26e-06 | 1299.8ms/step | 15.4 samples/s +Step 41180/44000 | Loss: 2.8952 | LR: 5.23e-06 | 1299.9ms/step | 15.4 samples/s +Step 41190/44000 | Loss: 2.8774 | LR: 5.19e-06 | 1299.9ms/step | 15.4 samples/s +Step 41200/44000 | Loss: 2.9175 | LR: 5.15e-06 | 1300.5ms/step | 15.4 samples/s +Step 41210/44000 | Loss: 2.9268 | LR: 5.12e-06 | 1303.0ms/step | 15.3 samples/s +Step 41220/44000 | Loss: 2.9341 | LR: 5.08e-06 | 1306.1ms/step | 15.3 samples/s +Step 41230/44000 | Loss: 2.9080 | LR: 5.04e-06 | 1307.8ms/step | 15.3 samples/s +Step 41240/44000 | Loss: 2.8962 | LR: 5.01e-06 | 1307.3ms/step | 15.3 samples/s +Step 41250/44000 | Loss: 2.9140 | LR: 4.97e-06 | 1301.2ms/step | 15.4 samples/s +Validation at step 41250: Loss = 2.9292, Perplexity = 18.71 +Step 41260/44000 | Loss: 2.9391 | LR: 4.94e-06 | 3492.4ms/step | 5.7 samples/s +Step 41270/44000 | Loss: 2.9191 | LR: 4.90e-06 | 1306.2ms/step | 15.3 samples/s +Step 41280/44000 | Loss: 2.9153 | LR: 4.86e-06 | 1303.8ms/step | 15.3 samples/s +Step 41290/44000 | Loss: 2.9059 | LR: 4.83e-06 | 1305.1ms/step | 15.3 samples/s +Step 41300/44000 | Loss: 2.9113 | LR: 4.79e-06 | 1303.3ms/step | 15.3 samples/s +Step 41310/44000 | Loss: 2.8962 | LR: 4.76e-06 | 1300.7ms/step | 15.4 samples/s +Step 41320/44000 | Loss: 2.9212 | LR: 4.72e-06 | 1303.9ms/step | 15.3 samples/s +Step 41330/44000 | Loss: 2.9074 | LR: 4.69e-06 | 1303.2ms/step | 15.3 samples/s +Step 41340/44000 | Loss: 2.9300 | LR: 4.65e-06 | 1306.2ms/step | 15.3 samples/s +Step 41350/44000 | Loss: 2.8937 | LR: 4.62e-06 | 1304.7ms/step | 15.3 samples/s +Step 41360/44000 | Loss: 2.9431 | LR: 4.58e-06 | 1302.6ms/step | 15.4 samples/s +Step 41370/44000 | Loss: 2.9244 | LR: 4.55e-06 | 1302.9ms/step | 15.4 samples/s +Step 41380/44000 | Loss: 2.9194 | LR: 4.51e-06 | 1302.9ms/step | 15.4 samples/s +Step 41390/44000 | Loss: 2.8978 | LR: 4.48e-06 | 1303.0ms/step | 15.3 samples/s +Step 41400/44000 | Loss: 2.8846 | LR: 4.45e-06 | 1306.8ms/step | 15.3 samples/s +Step 41410/44000 | Loss: 2.8913 | LR: 4.41e-06 | 1305.5ms/step | 15.3 samples/s +Step 41420/44000 | Loss: 2.8824 | LR: 4.38e-06 | 1303.1ms/step | 15.3 samples/s +Step 41430/44000 | Loss: 2.9088 | LR: 4.34e-06 | 1303.3ms/step | 15.3 samples/s +Step 41440/44000 | Loss: 2.9492 | LR: 4.31e-06 | 1300.6ms/step | 15.4 samples/s +Step 41450/44000 | Loss: 2.9148 | LR: 4.28e-06 | 1304.2ms/step | 15.3 samples/s +Step 41460/44000 | Loss: 2.9147 | LR: 4.24e-06 | 1308.9ms/step | 15.3 samples/s +Step 41470/44000 | Loss: 2.9010 | LR: 4.21e-06 | 1307.9ms/step | 15.3 samples/s +Step 41480/44000 | Loss: 2.9311 | LR: 4.18e-06 | 1301.0ms/step | 15.4 samples/s +Step 41490/44000 | Loss: 2.9004 | LR: 4.14e-06 | 1303.1ms/step | 15.3 samples/s +Step 41500/44000 | Loss: 2.9159 | LR: 4.11e-06 | 1306.1ms/step | 15.3 samples/s +Validation at step 41500: Loss = 2.9292, Perplexity = 18.71 +Step 41510/44000 | Loss: 2.9113 | LR: 4.08e-06 | 3484.9ms/step | 5.7 samples/s +Step 41520/44000 | Loss: 2.9127 | LR: 4.05e-06 | 1305.3ms/step | 15.3 samples/s +Step 41530/44000 | Loss: 2.9171 | LR: 4.01e-06 | 1308.0ms/step | 15.3 samples/s +Step 41540/44000 | Loss: 2.9395 | LR: 3.98e-06 | 1305.3ms/step | 15.3 samples/s +Step 41550/44000 | Loss: 2.9129 | LR: 3.95e-06 | 1306.4ms/step | 15.3 samples/s +Step 41560/44000 | Loss: 2.9255 | LR: 3.92e-06 | 1308.5ms/step | 15.3 samples/s +Step 41570/44000 | Loss: 2.9439 | LR: 3.88e-06 | 1307.7ms/step | 15.3 samples/s +Step 41580/44000 | Loss: 2.8995 | LR: 3.85e-06 | 1307.6ms/step | 15.3 samples/s +Step 41590/44000 | Loss: 2.9108 | LR: 3.82e-06 | 1305.7ms/step | 15.3 samples/s +Step 41600/44000 | Loss: 2.9259 | LR: 3.79e-06 | 1305.1ms/step | 15.3 samples/s +Step 41610/44000 | Loss: 2.9040 | LR: 3.76e-06 | 1304.8ms/step | 15.3 samples/s +Step 41620/44000 | Loss: 2.8788 | LR: 3.73e-06 | 1304.4ms/step | 15.3 samples/s +Step 41630/44000 | Loss: 2.9375 | LR: 3.70e-06 | 1301.8ms/step | 15.4 samples/s +Step 41640/44000 | Loss: 2.9362 | LR: 3.66e-06 | 1305.2ms/step | 15.3 samples/s +Step 41650/44000 | Loss: 2.9188 | LR: 3.63e-06 | 1310.2ms/step | 15.3 samples/s +Step 41660/44000 | Loss: 2.9394 | LR: 3.60e-06 | 1307.4ms/step | 15.3 samples/s +Step 41670/44000 | Loss: 2.9118 | LR: 3.57e-06 | 1305.4ms/step | 15.3 samples/s +Step 41680/44000 | Loss: 2.8842 | LR: 3.54e-06 | 1304.0ms/step | 15.3 samples/s +Step 41690/44000 | Loss: 2.8979 | LR: 3.51e-06 | 1305.0ms/step | 15.3 samples/s +Step 41700/44000 | Loss: 2.9330 | LR: 3.48e-06 | 1305.5ms/step | 15.3 samples/s +Step 41710/44000 | Loss: 2.9046 | LR: 3.45e-06 | 1306.4ms/step | 15.3 samples/s +Step 41720/44000 | Loss: 2.9090 | LR: 3.42e-06 | 1306.1ms/step | 15.3 samples/s +Step 41730/44000 | Loss: 2.9222 | LR: 3.39e-06 | 1306.4ms/step | 15.3 samples/s +Step 41740/44000 | Loss: 2.9329 | LR: 3.36e-06 | 1307.1ms/step | 15.3 samples/s +Step 41750/44000 | Loss: 2.9077 | LR: 3.33e-06 | 1304.8ms/step | 15.3 samples/s +Validation at step 41750: Loss = 2.9292, Perplexity = 18.71 +Step 41760/44000 | Loss: 2.9392 | LR: 3.30e-06 | 3481.1ms/step | 5.7 samples/s +Step 41770/44000 | Loss: 2.9132 | LR: 3.27e-06 | 1305.6ms/step | 15.3 samples/s +Step 41780/44000 | Loss: 2.9057 | LR: 3.24e-06 | 1305.9ms/step | 15.3 samples/s +Step 41790/44000 | Loss: 2.9489 | LR: 3.21e-06 | 1304.5ms/step | 15.3 samples/s +Step 41800/44000 | Loss: 2.9612 | LR: 3.19e-06 | 1305.1ms/step | 15.3 samples/s +Step 41810/44000 | Loss: 2.9252 | LR: 3.16e-06 | 1304.6ms/step | 15.3 samples/s +Step 41820/44000 | Loss: 2.8995 | LR: 3.13e-06 | 1304.9ms/step | 15.3 samples/s +Step 41830/44000 | Loss: 2.9021 | LR: 3.10e-06 | 1304.4ms/step | 15.3 samples/s +Step 41840/44000 | Loss: 2.8839 | LR: 3.07e-06 | 1305.4ms/step | 15.3 samples/s +Step 41850/44000 | Loss: 2.8897 | LR: 3.04e-06 | 1303.6ms/step | 15.3 samples/s +Step 41860/44000 | Loss: 2.9372 | LR: 3.01e-06 | 1305.9ms/step | 15.3 samples/s +Step 41870/44000 | Loss: 2.9470 | LR: 2.99e-06 | 1305.4ms/step | 15.3 samples/s +Step 41880/44000 | Loss: 2.9248 | LR: 2.96e-06 | 1304.9ms/step | 15.3 samples/s +Step 41890/44000 | Loss: 2.9086 | LR: 2.93e-06 | 1305.9ms/step | 15.3 samples/s +Step 41900/44000 | Loss: 2.9087 | LR: 2.90e-06 | 1305.7ms/step | 15.3 samples/s +Step 41910/44000 | Loss: 2.9036 | LR: 2.88e-06 | 1302.3ms/step | 15.4 samples/s +Step 41920/44000 | Loss: 2.9555 | LR: 2.85e-06 | 1301.2ms/step | 15.4 samples/s +Step 41930/44000 | Loss: 2.9061 | LR: 2.82e-06 | 1301.8ms/step | 15.4 samples/s +Step 41940/44000 | Loss: 2.9156 | LR: 2.79e-06 | 1305.0ms/step | 15.3 samples/s +Step 41950/44000 | Loss: 2.8766 | LR: 2.77e-06 | 1306.2ms/step | 15.3 samples/s +Step 41960/44000 | Loss: 2.9248 | LR: 2.74e-06 | 1304.7ms/step | 15.3 samples/s +Step 41970/44000 | Loss: 2.9183 | LR: 2.71e-06 | 1305.0ms/step | 15.3 samples/s +Step 41980/44000 | Loss: 2.9128 | LR: 2.69e-06 | 1307.0ms/step | 15.3 samples/s +Step 41990/44000 | Loss: 2.9164 | LR: 2.66e-06 | 1304.4ms/step | 15.3 samples/s +Step 42000/44000 | Loss: 2.9131 | LR: 2.63e-06 | 1301.6ms/step | 15.4 samples/s +Validation at step 42000: Loss = 2.9292, Perplexity = 18.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-42000 +Step 42010/44000 | Loss: 2.9239 | LR: 2.61e-06 | 3814.1ms/step | 5.2 samples/s +Step 42020/44000 | Loss: 2.9315 | LR: 2.58e-06 | 1304.8ms/step | 15.3 samples/s +Step 42030/44000 | Loss: 2.9385 | LR: 2.56e-06 | 1306.3ms/step | 15.3 samples/s +Step 42040/44000 | Loss: 2.9360 | LR: 2.53e-06 | 1305.5ms/step | 15.3 samples/s +Step 42050/44000 | Loss: 2.8930 | LR: 2.50e-06 | 1305.5ms/step | 15.3 samples/s +Step 42060/44000 | Loss: 2.9306 | LR: 2.48e-06 | 1307.7ms/step | 15.3 samples/s +Step 42070/44000 | Loss: 2.9313 | LR: 2.45e-06 | 1306.9ms/step | 15.3 samples/s +Step 42080/44000 | Loss: 2.9110 | LR: 2.43e-06 | 1306.1ms/step | 15.3 samples/s +Step 42090/44000 | Loss: 2.8969 | LR: 2.40e-06 | 1304.0ms/step | 15.3 samples/s +Step 42100/44000 | Loss: 2.9184 | LR: 2.38e-06 | 1303.3ms/step | 15.3 samples/s +Step 42110/44000 | Loss: 2.9312 | LR: 2.35e-06 | 1309.0ms/step | 15.3 samples/s +Step 42120/44000 | Loss: 2.9167 | LR: 2.33e-06 | 1309.7ms/step | 15.3 samples/s +Step 42130/44000 | Loss: 2.9298 | LR: 2.30e-06 | 1310.1ms/step | 15.3 samples/s +Step 42140/44000 | Loss: 2.9263 | LR: 2.28e-06 | 1307.3ms/step | 15.3 samples/s +Step 42150/44000 | Loss: 2.9458 | LR: 2.25e-06 | 1308.4ms/step | 15.3 samples/s +Step 42160/44000 | Loss: 2.9135 | LR: 2.23e-06 | 1308.1ms/step | 15.3 samples/s +Step 42170/44000 | Loss: 2.8996 | LR: 2.21e-06 | 1307.1ms/step | 15.3 samples/s +Step 42180/44000 | Loss: 2.9289 | LR: 2.18e-06 | 1305.4ms/step | 15.3 samples/s +Step 42190/44000 | Loss: 2.9116 | LR: 2.16e-06 | 1305.8ms/step | 15.3 samples/s +Step 42200/44000 | Loss: 2.8975 | LR: 2.13e-06 | 1305.3ms/step | 15.3 samples/s +Step 42210/44000 | Loss: 2.9434 | LR: 2.11e-06 | 1307.5ms/step | 15.3 samples/s +Step 42220/44000 | Loss: 2.8830 | LR: 2.09e-06 | 1308.1ms/step | 15.3 samples/s +Step 42230/44000 | Loss: 2.9372 | LR: 2.06e-06 | 1308.0ms/step | 15.3 samples/s +Step 42240/44000 | Loss: 2.9139 | LR: 2.04e-06 | 1311.3ms/step | 15.3 samples/s +Step 42250/44000 | Loss: 2.9292 | LR: 2.02e-06 | 1312.1ms/step | 15.2 samples/s +Validation at step 42250: Loss = 2.9292, Perplexity = 18.71 +Step 42260/44000 | Loss: 2.9199 | LR: 1.99e-06 | 3488.4ms/step | 5.7 samples/s +Step 42270/44000 | Loss: 2.9144 | LR: 1.97e-06 | 1305.6ms/step | 15.3 samples/s +Step 42280/44000 | Loss: 2.9173 | LR: 1.95e-06 | 1307.3ms/step | 15.3 samples/s +Step 42290/44000 | Loss: 2.8846 | LR: 1.93e-06 | 1303.1ms/step | 15.3 samples/s +Step 42300/44000 | Loss: 2.9130 | LR: 1.90e-06 | 1307.3ms/step | 15.3 samples/s +Step 42310/44000 | Loss: 2.8874 | LR: 1.88e-06 | 1303.7ms/step | 15.3 samples/s +Step 42320/44000 | Loss: 2.9329 | LR: 1.86e-06 | 1305.8ms/step | 15.3 samples/s +Step 42330/44000 | Loss: 2.8989 | LR: 1.84e-06 | 1305.4ms/step | 15.3 samples/s +Step 42340/44000 | Loss: 2.9427 | LR: 1.82e-06 | 1307.6ms/step | 15.3 samples/s +Step 42350/44000 | Loss: 2.8734 | LR: 1.79e-06 | 1306.2ms/step | 15.3 samples/s +Step 42360/44000 | Loss: 2.9162 | LR: 1.77e-06 | 1309.0ms/step | 15.3 samples/s +Step 42370/44000 | Loss: 2.9163 | LR: 1.75e-06 | 1305.3ms/step | 15.3 samples/s +Step 42380/44000 | Loss: 2.9502 | LR: 1.73e-06 | 1303.6ms/step | 15.3 samples/s +Step 42390/44000 | Loss: 2.8809 | LR: 1.71e-06 | 1303.0ms/step | 15.3 samples/s +Step 42400/44000 | Loss: 2.9081 | LR: 1.69e-06 | 1304.1ms/step | 15.3 samples/s +Step 42410/44000 | Loss: 2.9129 | LR: 1.67e-06 | 1302.8ms/step | 15.4 samples/s +Step 42420/44000 | Loss: 2.9409 | LR: 1.64e-06 | 1302.4ms/step | 15.4 samples/s +Step 42430/44000 | Loss: 2.9426 | LR: 1.62e-06 | 1303.1ms/step | 15.3 samples/s +Step 42440/44000 | Loss: 2.9337 | LR: 1.60e-06 | 1302.3ms/step | 15.4 samples/s +Step 42450/44000 | Loss: 2.9043 | LR: 1.58e-06 | 1302.7ms/step | 15.4 samples/s +Step 42460/44000 | Loss: 2.9131 | LR: 1.56e-06 | 1305.9ms/step | 15.3 samples/s +Step 42470/44000 | Loss: 2.9471 | LR: 1.54e-06 | 1305.7ms/step | 15.3 samples/s +Step 42480/44000 | Loss: 2.9161 | LR: 1.52e-06 | 1305.9ms/step | 15.3 samples/s +Step 42490/44000 | Loss: 2.9217 | LR: 1.50e-06 | 1306.9ms/step | 15.3 samples/s +Step 42500/44000 | Loss: 2.8909 | LR: 1.48e-06 | 1303.3ms/step | 15.3 samples/s +Validation at step 42500: Loss = 2.9292, Perplexity = 18.71 +Step 42510/44000 | Loss: 2.9330 | LR: 1.46e-06 | 3487.1ms/step | 5.7 samples/s +Step 42520/44000 | Loss: 2.8934 | LR: 1.44e-06 | 1306.7ms/step | 15.3 samples/s +Step 42530/44000 | Loss: 2.9211 | LR: 1.42e-06 | 1305.1ms/step | 15.3 samples/s +Step 42540/44000 | Loss: 2.9086 | LR: 1.40e-06 | 1306.0ms/step | 15.3 samples/s +Step 42550/44000 | Loss: 2.9246 | LR: 1.39e-06 | 1306.6ms/step | 15.3 samples/s +Step 42560/44000 | Loss: 2.8976 | LR: 1.37e-06 | 1304.9ms/step | 15.3 samples/s +Step 42570/44000 | Loss: 2.9162 | LR: 1.35e-06 | 1307.4ms/step | 15.3 samples/s +Step 42580/44000 | Loss: 2.9151 | LR: 1.33e-06 | 1305.9ms/step | 15.3 samples/s +Step 42590/44000 | Loss: 2.9006 | LR: 1.31e-06 | 1304.1ms/step | 15.3 samples/s +Step 42600/44000 | Loss: 2.8950 | LR: 1.29e-06 | 1301.3ms/step | 15.4 samples/s +Step 42610/44000 | Loss: 2.9229 | LR: 1.27e-06 | 1302.7ms/step | 15.4 samples/s +Step 42620/44000 | Loss: 2.9308 | LR: 1.25e-06 | 1300.8ms/step | 15.4 samples/s +Step 42630/44000 | Loss: 2.8996 | LR: 1.24e-06 | 1302.0ms/step | 15.4 samples/s +Step 42640/44000 | Loss: 2.9097 | LR: 1.22e-06 | 1303.2ms/step | 15.3 samples/s +Step 42650/44000 | Loss: 2.9307 | LR: 1.20e-06 | 1306.2ms/step | 15.3 samples/s +Step 42660/44000 | Loss: 2.9013 | LR: 1.18e-06 | 1307.7ms/step | 15.3 samples/s +Step 42670/44000 | Loss: 2.9004 | LR: 1.17e-06 | 1309.2ms/step | 15.3 samples/s +Step 42680/44000 | Loss: 2.8926 | LR: 1.15e-06 | 1310.5ms/step | 15.3 samples/s +Step 42690/44000 | Loss: 2.8861 | LR: 1.13e-06 | 1308.4ms/step | 15.3 samples/s +Step 42700/44000 | Loss: 2.8907 | LR: 1.11e-06 | 1306.8ms/step | 15.3 samples/s +Step 42710/44000 | Loss: 2.9266 | LR: 1.10e-06 | 1306.0ms/step | 15.3 samples/s +Step 42720/44000 | Loss: 2.9205 | LR: 1.08e-06 | 1307.5ms/step | 15.3 samples/s +Step 42730/44000 | Loss: 2.9136 | LR: 1.06e-06 | 1306.4ms/step | 15.3 samples/s +Step 42740/44000 | Loss: 2.8883 | LR: 1.05e-06 | 1306.1ms/step | 15.3 samples/s +Step 42750/44000 | Loss: 2.8969 | LR: 1.03e-06 | 1306.9ms/step | 15.3 samples/s +Validation at step 42750: Loss = 2.9292, Perplexity = 18.71 +Step 42760/44000 | Loss: 2.9140 | LR: 1.01e-06 | 3493.3ms/step | 5.7 samples/s +Step 42770/44000 | Loss: 2.9285 | LR: 9.97e-07 | 1313.5ms/step | 15.2 samples/s +Step 42780/44000 | Loss: 2.9351 | LR: 9.81e-07 | 1305.8ms/step | 15.3 samples/s +Step 42790/44000 | Loss: 2.9069 | LR: 9.65e-07 | 1306.6ms/step | 15.3 samples/s +Step 42800/44000 | Loss: 2.8993 | LR: 9.49e-07 | 1304.7ms/step | 15.3 samples/s +Step 42810/44000 | Loss: 2.8947 | LR: 9.33e-07 | 1303.0ms/step | 15.3 samples/s +Step 42820/44000 | Loss: 2.9513 | LR: 9.18e-07 | 1303.9ms/step | 15.3 samples/s +Step 42830/44000 | Loss: 2.9151 | LR: 9.02e-07 | 1303.3ms/step | 15.3 samples/s +Step 42840/44000 | Loss: 2.9243 | LR: 8.87e-07 | 1301.8ms/step | 15.4 samples/s +Step 42850/44000 | Loss: 2.9239 | LR: 8.72e-07 | 1304.0ms/step | 15.3 samples/s +Step 42860/44000 | Loss: 2.9324 | LR: 8.57e-07 | 1302.9ms/step | 15.3 samples/s +Step 42870/44000 | Loss: 2.9291 | LR: 8.42e-07 | 1303.9ms/step | 15.3 samples/s +Step 42880/44000 | Loss: 2.9165 | LR: 8.27e-07 | 1304.2ms/step | 15.3 samples/s +Step 42890/44000 | Loss: 2.8691 | LR: 8.12e-07 | 1303.9ms/step | 15.3 samples/s +Step 42900/44000 | Loss: 2.9270 | LR: 7.98e-07 | 1302.4ms/step | 15.4 samples/s +Step 42910/44000 | Loss: 2.9193 | LR: 7.83e-07 | 1303.7ms/step | 15.3 samples/s +Step 42920/44000 | Loss: 2.9003 | LR: 7.69e-07 | 1303.4ms/step | 15.3 samples/s +Step 42930/44000 | Loss: 2.8964 | LR: 7.55e-07 | 1302.3ms/step | 15.4 samples/s +Step 42940/44000 | Loss: 2.9070 | LR: 7.41e-07 | 1301.6ms/step | 15.4 samples/s +Step 42950/44000 | Loss: 2.9217 | LR: 7.27e-07 | 1299.8ms/step | 15.4 samples/s +Step 42960/44000 | Loss: 2.9131 | LR: 7.13e-07 | 1302.0ms/step | 15.4 samples/s +Step 42970/44000 | Loss: 2.9071 | LR: 6.99e-07 | 1305.9ms/step | 15.3 samples/s +Step 42980/44000 | Loss: 2.9194 | LR: 6.86e-07 | 1307.8ms/step | 15.3 samples/s +Step 42990/44000 | Loss: 2.9140 | LR: 6.72e-07 | 1304.2ms/step | 15.3 samples/s +Step 43000/44000 | Loss: 2.9059 | LR: 6.59e-07 | 1304.7ms/step | 15.3 samples/s +Validation at step 43000: Loss = 2.9292, Perplexity = 18.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-43000 +Step 43010/44000 | Loss: 2.9121 | LR: 6.46e-07 | 3810.1ms/step | 5.2 samples/s +Step 43020/44000 | Loss: 2.8987 | LR: 6.33e-07 | 1302.2ms/step | 15.4 samples/s +Step 43030/44000 | Loss: 2.9180 | LR: 6.20e-07 | 1301.5ms/step | 15.4 samples/s +Step 43040/44000 | Loss: 2.8828 | LR: 6.08e-07 | 1300.0ms/step | 15.4 samples/s +Step 43050/44000 | Loss: 2.9101 | LR: 5.95e-07 | 1300.8ms/step | 15.4 samples/s +Step 43060/44000 | Loss: 2.9186 | LR: 5.83e-07 | 1299.8ms/step | 15.4 samples/s +Step 43070/44000 | Loss: 2.9337 | LR: 5.70e-07 | 1300.9ms/step | 15.4 samples/s +Step 43080/44000 | Loss: 2.9078 | LR: 5.58e-07 | 1302.3ms/step | 15.4 samples/s +Step 43090/44000 | Loss: 2.8890 | LR: 5.46e-07 | 1299.4ms/step | 15.4 samples/s +Step 43100/44000 | Loss: 2.9041 | LR: 5.34e-07 | 1301.8ms/step | 15.4 samples/s +Step 43110/44000 | Loss: 2.9155 | LR: 5.22e-07 | 1306.2ms/step | 15.3 samples/s +Step 43120/44000 | Loss: 2.9284 | LR: 5.11e-07 | 1304.9ms/step | 15.3 samples/s +Step 43130/44000 | Loss: 2.9326 | LR: 4.99e-07 | 1303.7ms/step | 15.3 samples/s +Step 43140/44000 | Loss: 2.9174 | LR: 4.88e-07 | 1299.4ms/step | 15.4 samples/s +Step 43150/44000 | Loss: 2.9021 | LR: 4.76e-07 | 1298.3ms/step | 15.4 samples/s +Step 43160/44000 | Loss: 2.8997 | LR: 4.65e-07 | 1298.5ms/step | 15.4 samples/s +Step 43170/44000 | Loss: 2.8869 | LR: 4.54e-07 | 1299.2ms/step | 15.4 samples/s +Step 43180/44000 | Loss: 2.8985 | LR: 4.43e-07 | 1303.0ms/step | 15.3 samples/s +Step 43190/44000 | Loss: 2.8946 | LR: 4.33e-07 | 1297.5ms/step | 15.4 samples/s +Step 43200/44000 | Loss: 2.8838 | LR: 4.22e-07 | 1297.0ms/step | 15.4 samples/s +Step 43210/44000 | Loss: 2.9188 | LR: 4.12e-07 | 1296.5ms/step | 15.4 samples/s +Step 43220/44000 | Loss: 2.9059 | LR: 4.01e-07 | 1296.3ms/step | 15.4 samples/s +Step 43230/44000 | Loss: 2.9284 | LR: 3.91e-07 | 1300.6ms/step | 15.4 samples/s +Step 43240/44000 | Loss: 2.9248 | LR: 3.81e-07 | 1302.9ms/step | 15.4 samples/s +Step 43250/44000 | Loss: 2.9284 | LR: 3.71e-07 | 1301.1ms/step | 15.4 samples/s +Validation at step 43250: Loss = 2.9292, Perplexity = 18.71 +Step 43260/44000 | Loss: 2.9203 | LR: 3.61e-07 | 3478.1ms/step | 5.8 samples/s +Step 43270/44000 | Loss: 2.9085 | LR: 3.51e-07 | 1303.8ms/step | 15.3 samples/s +Step 43280/44000 | Loss: 2.9153 | LR: 3.42e-07 | 1303.5ms/step | 15.3 samples/s +Step 43290/44000 | Loss: 2.8999 | LR: 3.32e-07 | 1301.6ms/step | 15.4 samples/s +Step 43300/44000 | Loss: 2.9320 | LR: 3.23e-07 | 1303.1ms/step | 15.3 samples/s +Step 43310/44000 | Loss: 2.9352 | LR: 3.14e-07 | 1304.6ms/step | 15.3 samples/s +Step 43320/44000 | Loss: 2.9404 | LR: 3.05e-07 | 1306.3ms/step | 15.3 samples/s +Step 43330/44000 | Loss: 2.9018 | LR: 2.96e-07 | 1306.0ms/step | 15.3 samples/s +Step 43340/44000 | Loss: 2.9233 | LR: 2.87e-07 | 1306.6ms/step | 15.3 samples/s +Step 43350/44000 | Loss: 2.8992 | LR: 2.79e-07 | 1306.0ms/step | 15.3 samples/s +Step 43360/44000 | Loss: 2.9195 | LR: 2.70e-07 | 1304.4ms/step | 15.3 samples/s +Step 43370/44000 | Loss: 2.9241 | LR: 2.62e-07 | 1303.2ms/step | 15.3 samples/s +Step 43380/44000 | Loss: 2.9170 | LR: 2.53e-07 | 1303.8ms/step | 15.3 samples/s +Step 43390/44000 | Loss: 2.8931 | LR: 2.45e-07 | 1303.6ms/step | 15.3 samples/s +Step 43400/44000 | Loss: 2.9099 | LR: 2.37e-07 | 1300.1ms/step | 15.4 samples/s +Step 43410/44000 | Loss: 2.9070 | LR: 2.30e-07 | 1300.0ms/step | 15.4 samples/s +Step 43420/44000 | Loss: 2.8866 | LR: 2.22e-07 | 1303.3ms/step | 15.3 samples/s +Step 43430/44000 | Loss: 2.9018 | LR: 2.14e-07 | 1302.9ms/step | 15.4 samples/s +Step 43440/44000 | Loss: 2.9483 | LR: 2.07e-07 | 1302.2ms/step | 15.4 samples/s +Step 43450/44000 | Loss: 2.9289 | LR: 1.99e-07 | 1303.2ms/step | 15.3 samples/s +Step 43460/44000 | Loss: 2.9204 | LR: 1.92e-07 | 1302.9ms/step | 15.3 samples/s +Step 43470/44000 | Loss: 2.9220 | LR: 1.85e-07 | 1303.7ms/step | 15.3 samples/s +Step 43480/44000 | Loss: 2.8895 | LR: 1.78e-07 | 1305.2ms/step | 15.3 samples/s +Step 43490/44000 | Loss: 2.9064 | LR: 1.72e-07 | 1305.0ms/step | 15.3 samples/s +Step 43500/44000 | Loss: 2.9483 | LR: 1.65e-07 | 1305.7ms/step | 15.3 samples/s +Validation at step 43500: Loss = 2.9292, Perplexity = 18.71 +Step 43510/44000 | Loss: 2.9143 | LR: 1.58e-07 | 3484.0ms/step | 5.7 samples/s +Step 43520/44000 | Loss: 2.9356 | LR: 1.52e-07 | 1306.0ms/step | 15.3 samples/s +Step 43530/44000 | Loss: 2.8970 | LR: 1.46e-07 | 1303.7ms/step | 15.3 samples/s +Step 43540/44000 | Loss: 2.8847 | LR: 1.40e-07 | 1303.6ms/step | 15.3 samples/s +Step 43550/44000 | Loss: 2.9071 | LR: 1.34e-07 | 1304.7ms/step | 15.3 samples/s +Step 43560/44000 | Loss: 2.9089 | LR: 1.28e-07 | 1303.1ms/step | 15.3 samples/s +Step 43570/44000 | Loss: 2.9062 | LR: 1.22e-07 | 1300.6ms/step | 15.4 samples/s +Step 43580/44000 | Loss: 2.9321 | LR: 1.16e-07 | 1299.9ms/step | 15.4 samples/s +Step 43590/44000 | Loss: 2.9075 | LR: 1.11e-07 | 1300.1ms/step | 15.4 samples/s +Step 43600/44000 | Loss: 2.9202 | LR: 1.06e-07 | 1304.3ms/step | 15.3 samples/s +Step 43610/44000 | Loss: 2.8666 | LR: 1.00e-07 | 1304.8ms/step | 15.3 samples/s +Step 43620/44000 | Loss: 2.9047 | LR: 9.52e-08 | 1305.0ms/step | 15.3 samples/s +Step 43630/44000 | Loss: 2.8965 | LR: 9.03e-08 | 1306.0ms/step | 15.3 samples/s +Step 43640/44000 | Loss: 2.9139 | LR: 8.55e-08 | 1305.0ms/step | 15.3 samples/s +Step 43650/44000 | Loss: 2.9167 | LR: 8.08e-08 | 1305.1ms/step | 15.3 samples/s +Step 43660/44000 | Loss: 2.9175 | LR: 7.62e-08 | 1304.9ms/step | 15.3 samples/s +Step 43670/44000 | Loss: 2.9234 | LR: 7.18e-08 | 1304.8ms/step | 15.3 samples/s +Step 43680/44000 | Loss: 2.9309 | LR: 6.75e-08 | 1306.1ms/step | 15.3 samples/s +Step 43690/44000 | Loss: 2.8931 | LR: 6.34e-08 | 1305.8ms/step | 15.3 samples/s +Step 43700/44000 | Loss: 2.9133 | LR: 5.94e-08 | 1306.4ms/step | 15.3 samples/s +Step 43710/44000 | Loss: 2.9321 | LR: 5.55e-08 | 1309.0ms/step | 15.3 samples/s +Step 43720/44000 | Loss: 2.8999 | LR: 5.17e-08 | 1305.3ms/step | 15.3 samples/s +Step 43730/44000 | Loss: 2.9373 | LR: 4.81e-08 | 1303.8ms/step | 15.3 samples/s +Step 43740/44000 | Loss: 2.9449 | LR: 4.46e-08 | 1305.8ms/step | 15.3 samples/s +Step 43750/44000 | Loss: 2.9165 | LR: 4.12e-08 | 1305.9ms/step | 15.3 samples/s +Validation at step 43750: Loss = 2.9292, Perplexity = 18.71 +Step 43760/44000 | Loss: 2.9350 | LR: 3.80e-08 | 3490.0ms/step | 5.7 samples/s +Step 43770/44000 | Loss: 2.9329 | LR: 3.49e-08 | 1304.8ms/step | 15.3 samples/s +Step 43780/44000 | Loss: 2.9137 | LR: 3.19e-08 | 1304.3ms/step | 15.3 samples/s +Step 43790/44000 | Loss: 2.9022 | LR: 2.91e-08 | 1304.5ms/step | 15.3 samples/s +Step 43800/44000 | Loss: 2.9058 | LR: 2.64e-08 | 1306.3ms/step | 15.3 samples/s +Step 43810/44000 | Loss: 2.9128 | LR: 2.38e-08 | 1305.8ms/step | 15.3 samples/s +Step 43820/44000 | Loss: 2.9058 | LR: 2.14e-08 | 1300.1ms/step | 15.4 samples/s +Step 43830/44000 | Loss: 2.9290 | LR: 1.91e-08 | 1300.3ms/step | 15.4 samples/s +Step 43840/44000 | Loss: 2.9145 | LR: 1.69e-08 | 1300.6ms/step | 15.4 samples/s +Step 43850/44000 | Loss: 2.9056 | LR: 1.48e-08 | 1302.0ms/step | 15.4 samples/s +Step 43860/44000 | Loss: 2.9169 | LR: 1.29e-08 | 1303.0ms/step | 15.3 samples/s +Step 43870/44000 | Loss: 2.9328 | LR: 1.11e-08 | 1303.4ms/step | 15.3 samples/s +Step 43880/44000 | Loss: 2.9483 | LR: 9.50e-09 | 1304.7ms/step | 15.3 samples/s +Step 43890/44000 | Loss: 2.9044 | LR: 7.98e-09 | 1303.7ms/step | 15.3 samples/s +Step 43900/44000 | Loss: 2.9136 | LR: 6.60e-09 | 1300.9ms/step | 15.4 samples/s +Step 43910/44000 | Loss: 2.9176 | LR: 5.34e-09 | 1300.3ms/step | 15.4 samples/s +Step 43920/44000 | Loss: 2.9011 | LR: 4.22e-09 | 1303.1ms/step | 15.3 samples/s +Step 43930/44000 | Loss: 2.8947 | LR: 3.23e-09 | 1301.8ms/step | 15.4 samples/s +Step 43940/44000 | Loss: 2.9031 | LR: 2.37e-09 | 1300.2ms/step | 15.4 samples/s +Step 43950/44000 | Loss: 2.9163 | LR: 1.65e-09 | 1299.6ms/step | 15.4 samples/s +Step 43960/44000 | Loss: 2.8970 | LR: 1.06e-09 | 1300.2ms/step | 15.4 samples/s +Step 43970/44000 | Loss: 2.9250 | LR: 5.94e-10 | 1302.1ms/step | 15.4 samples/s +Step 43980/44000 | Loss: 2.9284 | LR: 2.64e-10 | 1303.3ms/step | 15.3 samples/s +Step 43990/44000 | Loss: 2.9200 | LR: 6.60e-11 | 1305.9ms/step | 15.3 samples/s +Step 44000/44000 | Loss: 2.9605 | LR: 0.00e+00 | 1304.7ms/step | 15.3 samples/s +Validation at step 44000: Loss = 2.9292, Perplexity = 18.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json +wandb: uploading history steps 4399-4399, summary, console lines 2529-2531 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▆▆▆▅▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity ██▇▇▆▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▇▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ███████▁████████████████████████████████ +wandb: speed/steps_per_sec █████████████████████████████▁██████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.92918 +wandb: eval/perplexity 18.71229 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 1304.67098 +wandb: speed/samples_per_sec 15.32954 +wandb: speed/steps_per_sec 0.76648 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 resume 365m static 8k nrb35 seed42 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/bsc91ke5 +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_042228-bsc91ke5/logs +Training complete! diff --git a/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_static_nrb35_seed43.log b/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_static_nrb35_seed43.log new file mode 100644 index 0000000000000000000000000000000000000000..44c94e3ef85980898d69c861639c29d440ed56d7 --- /dev/null +++ b/epfl_final/train_logs/p1_resume_h1004_20260822/l8k365m_static_nrb35_seed43.log @@ -0,0 +1,2760 @@ +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-20000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 43, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 resume 365m static 8k nrb35 seed43", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "static", + "nrb35", + "seed43", + "resume", + "h1004", + "step20000" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1", + "sparse_mask_seed": 1, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-20000 +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-20000... +Preserving checkpoint LR state for exact resume: current=[0.00029291518939613315], schedule_base=[0.0005] +Resumed from checkpoint at step 20000 (epoch 0, micro_step 200000) +Resuming training from step 20000 +Resuming wandb run ino9n2jx +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run ino9n2jx +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042228-ino9n2jx +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run P1 resume 365m static 8k nrb35 seed43 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ino9n2jx +Starting training... +Validation at step 20000: Loss = 2.9897, Perplexity = 19.88 +wandb: WARNING Tried to log to step 20000 that is less than the current step 20081. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 20001: sparse_stochastic (configured sparse attention) +Step 20010/44000 | Loss: 2.9986 | LR: 2.93e-04 | 50195.2ms/step | 0.4 samples/s +wandb: WARNING Tried to log to step 20010 that is less than the current step 20081. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20020/44000 | Loss: 2.9961 | LR: 2.93e-04 | 1315.7ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 20020 that is less than the current step 20081. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20030/44000 | Loss: 2.9699 | LR: 2.92e-04 | 1319.7ms/step | 15.2 samples/s +wandb: WARNING Tried to log to step 20030 that is less than the current step 20081. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20040/44000 | Loss: 2.9922 | LR: 2.92e-04 | 1321.2ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 20040 that is less than the current step 20081. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20050/44000 | Loss: 3.0070 | LR: 2.92e-04 | 1324.0ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 20050 that is less than the current step 20081. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20060/44000 | Loss: 2.9821 | LR: 2.92e-04 | 1325.2ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 20060 that is less than the current step 20081. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20070/44000 | Loss: 3.0399 | LR: 2.92e-04 | 1323.1ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 20070 that is less than the current step 20081. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20080/44000 | Loss: 2.9997 | LR: 2.91e-04 | 1323.4ms/step | 15.1 samples/s +wandb: WARNING Tried to log to step 20080 that is less than the current step 20081. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 20090/44000 | Loss: 2.9958 | LR: 2.91e-04 | 1326.2ms/step | 15.1 samples/s +Step 20100/44000 | Loss: 2.9930 | LR: 2.91e-04 | 1325.6ms/step | 15.1 samples/s +Step 20110/44000 | Loss: 2.9954 | LR: 2.91e-04 | 1325.4ms/step | 15.1 samples/s +Step 20120/44000 | Loss: 3.0133 | LR: 2.91e-04 | 1326.3ms/step | 15.1 samples/s +Step 20130/44000 | Loss: 2.9888 | LR: 2.91e-04 | 1324.7ms/step | 15.1 samples/s +Step 20140/44000 | Loss: 2.9933 | LR: 2.90e-04 | 1325.2ms/step | 15.1 samples/s +Step 20150/44000 | Loss: 3.0131 | LR: 2.90e-04 | 1326.5ms/step | 15.1 samples/s +Step 20160/44000 | Loss: 2.9853 | LR: 2.90e-04 | 1326.7ms/step | 15.1 samples/s +Step 20170/44000 | Loss: 3.0012 | LR: 2.90e-04 | 1328.8ms/step | 15.1 samples/s +Step 20180/44000 | Loss: 3.0001 | LR: 2.90e-04 | 1330.9ms/step | 15.0 samples/s +Step 20190/44000 | Loss: 3.0243 | LR: 2.90e-04 | 1329.3ms/step | 15.0 samples/s +Step 20200/44000 | Loss: 2.9911 | LR: 2.89e-04 | 1330.7ms/step | 15.0 samples/s +Step 20210/44000 | Loss: 2.9814 | LR: 2.89e-04 | 1330.7ms/step | 15.0 samples/s +Step 20220/44000 | Loss: 2.9959 | LR: 2.89e-04 | 1325.8ms/step | 15.1 samples/s +Step 20230/44000 | Loss: 3.0051 | LR: 2.89e-04 | 1324.6ms/step | 15.1 samples/s +Step 20240/44000 | Loss: 2.9735 | LR: 2.89e-04 | 1325.9ms/step | 15.1 samples/s +Step 20250/44000 | Loss: 2.9814 | LR: 2.88e-04 | 1323.9ms/step | 15.1 samples/s +Validation at step 20250: Loss = 2.9876, Perplexity = 19.84 +Step 20260/44000 | Loss: 2.9857 | LR: 2.88e-04 | 3558.6ms/step | 5.6 samples/s +Step 20270/44000 | Loss: 2.9893 | LR: 2.88e-04 | 1331.7ms/step | 15.0 samples/s +Step 20280/44000 | Loss: 3.0163 | LR: 2.88e-04 | 1330.9ms/step | 15.0 samples/s +Step 20290/44000 | Loss: 2.9859 | LR: 2.88e-04 | 1333.5ms/step | 15.0 samples/s +Step 20300/44000 | Loss: 3.0004 | LR: 2.88e-04 | 1335.2ms/step | 15.0 samples/s +Step 20310/44000 | Loss: 2.9891 | LR: 2.87e-04 | 1330.7ms/step | 15.0 samples/s +Step 20320/44000 | Loss: 3.0236 | LR: 2.87e-04 | 1330.8ms/step | 15.0 samples/s +Step 20330/44000 | Loss: 2.9931 | LR: 2.87e-04 | 1332.2ms/step | 15.0 samples/s +Step 20340/44000 | Loss: 2.9754 | LR: 2.87e-04 | 1332.4ms/step | 15.0 samples/s +Step 20350/44000 | Loss: 2.9769 | LR: 2.87e-04 | 1331.5ms/step | 15.0 samples/s +Step 20360/44000 | Loss: 2.9747 | LR: 2.86e-04 | 1328.6ms/step | 15.1 samples/s +Step 20370/44000 | Loss: 3.0016 | LR: 2.86e-04 | 1330.3ms/step | 15.0 samples/s +Step 20380/44000 | Loss: 3.0054 | LR: 2.86e-04 | 1329.9ms/step | 15.0 samples/s +Step 20390/44000 | Loss: 3.0157 | LR: 2.86e-04 | 1331.3ms/step | 15.0 samples/s +Step 20400/44000 | Loss: 3.0284 | LR: 2.86e-04 | 1332.1ms/step | 15.0 samples/s +Step 20410/44000 | Loss: 2.9974 | LR: 2.86e-04 | 1334.1ms/step | 15.0 samples/s +Step 20420/44000 | Loss: 3.0001 | LR: 2.85e-04 | 1333.3ms/step | 15.0 samples/s +Step 20430/44000 | Loss: 2.9862 | LR: 2.85e-04 | 1331.8ms/step | 15.0 samples/s +Step 20440/44000 | Loss: 3.0084 | LR: 2.85e-04 | 1330.0ms/step | 15.0 samples/s +Step 20450/44000 | Loss: 3.0092 | LR: 2.85e-04 | 1330.9ms/step | 15.0 samples/s +Step 20460/44000 | Loss: 2.9875 | LR: 2.85e-04 | 1333.7ms/step | 15.0 samples/s +Step 20470/44000 | Loss: 2.9468 | LR: 2.84e-04 | 1332.7ms/step | 15.0 samples/s +Step 20480/44000 | Loss: 2.9786 | LR: 2.84e-04 | 1329.2ms/step | 15.0 samples/s +Step 20490/44000 | Loss: 2.9838 | LR: 2.84e-04 | 1328.7ms/step | 15.1 samples/s +Step 20500/44000 | Loss: 2.9911 | LR: 2.84e-04 | 1327.8ms/step | 15.1 samples/s +Validation at step 20500: Loss = 2.9840, Perplexity = 19.77 +Step 20510/44000 | Loss: 3.0086 | LR: 2.84e-04 | 3560.2ms/step | 5.6 samples/s +Step 20520/44000 | Loss: 3.0146 | LR: 2.84e-04 | 1330.5ms/step | 15.0 samples/s +Step 20530/44000 | Loss: 2.9891 | LR: 2.83e-04 | 1328.8ms/step | 15.1 samples/s +Step 20540/44000 | Loss: 2.9620 | LR: 2.83e-04 | 1330.6ms/step | 15.0 samples/s +Step 20550/44000 | Loss: 3.0127 | LR: 2.83e-04 | 1331.5ms/step | 15.0 samples/s +Step 20560/44000 | Loss: 3.0055 | LR: 2.83e-04 | 1331.5ms/step | 15.0 samples/s +Step 20570/44000 | Loss: 2.9711 | LR: 2.83e-04 | 1329.1ms/step | 15.0 samples/s +Step 20580/44000 | Loss: 2.9933 | LR: 2.83e-04 | 1330.1ms/step | 15.0 samples/s +Step 20590/44000 | Loss: 3.0045 | LR: 2.82e-04 | 1333.4ms/step | 15.0 samples/s +Step 20600/44000 | Loss: 2.9745 | LR: 2.82e-04 | 1331.8ms/step | 15.0 samples/s +Step 20610/44000 | Loss: 3.0337 | LR: 2.82e-04 | 1336.5ms/step | 15.0 samples/s +Step 20620/44000 | Loss: 2.9625 | LR: 2.82e-04 | 1334.4ms/step | 15.0 samples/s +Step 20630/44000 | Loss: 2.9862 | LR: 2.82e-04 | 1333.5ms/step | 15.0 samples/s +Step 20640/44000 | Loss: 3.0084 | LR: 2.81e-04 | 1334.8ms/step | 15.0 samples/s +Step 20650/44000 | Loss: 2.9654 | LR: 2.81e-04 | 1334.7ms/step | 15.0 samples/s +Step 20660/44000 | Loss: 3.0104 | LR: 2.81e-04 | 1336.3ms/step | 15.0 samples/s +Step 20670/44000 | Loss: 2.9919 | LR: 2.81e-04 | 1334.8ms/step | 15.0 samples/s +Step 20680/44000 | Loss: 3.0089 | LR: 2.81e-04 | 1337.0ms/step | 15.0 samples/s +Step 20690/44000 | Loss: 2.9859 | LR: 2.81e-04 | 1338.3ms/step | 14.9 samples/s +Step 20700/44000 | Loss: 2.9836 | LR: 2.80e-04 | 1335.2ms/step | 15.0 samples/s +Step 20710/44000 | Loss: 2.9840 | LR: 2.80e-04 | 1333.4ms/step | 15.0 samples/s +Step 20720/44000 | Loss: 2.9892 | LR: 2.80e-04 | 1332.0ms/step | 15.0 samples/s +Step 20730/44000 | Loss: 2.9686 | LR: 2.80e-04 | 1333.9ms/step | 15.0 samples/s +Step 20740/44000 | Loss: 2.9654 | LR: 2.80e-04 | 1333.1ms/step | 15.0 samples/s +Step 20750/44000 | Loss: 3.0087 | LR: 2.79e-04 | 1334.4ms/step | 15.0 samples/s +Validation at step 20750: Loss = 2.9817, Perplexity = 19.72 +Step 20760/44000 | Loss: 3.0043 | LR: 2.79e-04 | 3574.4ms/step | 5.6 samples/s +Step 20770/44000 | Loss: 3.0064 | LR: 2.79e-04 | 1336.3ms/step | 15.0 samples/s +Step 20780/44000 | Loss: 3.0341 | LR: 2.79e-04 | 1338.5ms/step | 14.9 samples/s +Step 20790/44000 | Loss: 3.0156 | LR: 2.79e-04 | 1336.6ms/step | 15.0 samples/s +Step 20800/44000 | Loss: 2.9946 | LR: 2.79e-04 | 1336.8ms/step | 15.0 samples/s +Step 20810/44000 | Loss: 2.9695 | LR: 2.78e-04 | 1336.7ms/step | 15.0 samples/s +Step 20820/44000 | Loss: 3.0036 | LR: 2.78e-04 | 1335.0ms/step | 15.0 samples/s +Step 20830/44000 | Loss: 2.9671 | LR: 2.78e-04 | 1334.9ms/step | 15.0 samples/s +Step 20840/44000 | Loss: 2.9991 | LR: 2.78e-04 | 1338.2ms/step | 14.9 samples/s +Step 20850/44000 | Loss: 2.9785 | LR: 2.78e-04 | 1337.0ms/step | 15.0 samples/s +Step 20860/44000 | Loss: 2.9935 | LR: 2.77e-04 | 1337.0ms/step | 15.0 samples/s +Step 20870/44000 | Loss: 3.0206 | LR: 2.77e-04 | 1337.3ms/step | 15.0 samples/s +Step 20880/44000 | Loss: 2.9721 | LR: 2.77e-04 | 1335.0ms/step | 15.0 samples/s +Step 20890/44000 | Loss: 2.9645 | LR: 2.77e-04 | 1333.8ms/step | 15.0 samples/s +Step 20900/44000 | Loss: 2.9756 | LR: 2.77e-04 | 1334.7ms/step | 15.0 samples/s +Step 20910/44000 | Loss: 2.9771 | LR: 2.77e-04 | 1334.2ms/step | 15.0 samples/s +Step 20920/44000 | Loss: 3.0288 | LR: 2.76e-04 | 1336.0ms/step | 15.0 samples/s +Step 20930/44000 | Loss: 2.9884 | LR: 2.76e-04 | 1334.9ms/step | 15.0 samples/s +Step 20940/44000 | Loss: 2.9953 | LR: 2.76e-04 | 1333.9ms/step | 15.0 samples/s +Step 20950/44000 | Loss: 2.9835 | LR: 2.76e-04 | 1334.1ms/step | 15.0 samples/s +Step 20960/44000 | Loss: 2.9939 | LR: 2.76e-04 | 1334.1ms/step | 15.0 samples/s +Step 20970/44000 | Loss: 2.9894 | LR: 2.75e-04 | 1335.1ms/step | 15.0 samples/s +Step 20980/44000 | Loss: 3.0074 | LR: 2.75e-04 | 1336.7ms/step | 15.0 samples/s +Step 20990/44000 | Loss: 3.0025 | LR: 2.75e-04 | 1339.0ms/step | 14.9 samples/s +Step 21000/44000 | Loss: 2.9827 | LR: 2.75e-04 | 1336.3ms/step | 15.0 samples/s +Validation at step 21000: Loss = 2.9786, Perplexity = 19.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-21000 +Step 21010/44000 | Loss: 2.9855 | LR: 2.75e-04 | 3871.8ms/step | 5.2 samples/s +Step 21020/44000 | Loss: 3.0188 | LR: 2.75e-04 | 1325.3ms/step | 15.1 samples/s +Step 21030/44000 | Loss: 2.9672 | LR: 2.74e-04 | 1330.0ms/step | 15.0 samples/s +Step 21040/44000 | Loss: 2.9668 | LR: 2.74e-04 | 1331.0ms/step | 15.0 samples/s +Step 21050/44000 | Loss: 3.0083 | LR: 2.74e-04 | 1328.4ms/step | 15.1 samples/s +Step 21060/44000 | Loss: 3.0208 | LR: 2.74e-04 | 1331.2ms/step | 15.0 samples/s +Step 21070/44000 | Loss: 2.9723 | LR: 2.74e-04 | 1331.1ms/step | 15.0 samples/s +Step 21080/44000 | Loss: 3.0019 | LR: 2.73e-04 | 1329.9ms/step | 15.0 samples/s +Step 21090/44000 | Loss: 2.9782 | LR: 2.73e-04 | 1330.1ms/step | 15.0 samples/s +Step 21100/44000 | Loss: 2.9968 | LR: 2.73e-04 | 1332.0ms/step | 15.0 samples/s +Step 21110/44000 | Loss: 2.9555 | LR: 2.73e-04 | 1331.1ms/step | 15.0 samples/s +Step 21120/44000 | Loss: 3.0118 | LR: 2.73e-04 | 1330.9ms/step | 15.0 samples/s +Step 21130/44000 | Loss: 2.9953 | LR: 2.73e-04 | 1333.5ms/step | 15.0 samples/s +Step 21140/44000 | Loss: 3.0097 | LR: 2.72e-04 | 1332.7ms/step | 15.0 samples/s +Step 21150/44000 | Loss: 3.0195 | LR: 2.72e-04 | 1331.6ms/step | 15.0 samples/s +Step 21160/44000 | Loss: 2.9847 | LR: 2.72e-04 | 1333.8ms/step | 15.0 samples/s +Step 21170/44000 | Loss: 3.0025 | LR: 2.72e-04 | 1332.8ms/step | 15.0 samples/s +Step 21180/44000 | Loss: 2.9951 | LR: 2.72e-04 | 1331.6ms/step | 15.0 samples/s +Step 21190/44000 | Loss: 2.9903 | LR: 2.71e-04 | 1328.2ms/step | 15.1 samples/s +Step 21200/44000 | Loss: 2.9814 | LR: 2.71e-04 | 1331.6ms/step | 15.0 samples/s +Step 21210/44000 | Loss: 2.9629 | LR: 2.71e-04 | 1332.0ms/step | 15.0 samples/s +Step 21220/44000 | Loss: 2.9708 | LR: 2.71e-04 | 1331.6ms/step | 15.0 samples/s +Step 21230/44000 | Loss: 2.9764 | LR: 2.71e-04 | 1332.2ms/step | 15.0 samples/s +Step 21240/44000 | Loss: 2.9890 | LR: 2.71e-04 | 1332.9ms/step | 15.0 samples/s +Step 21250/44000 | Loss: 2.9842 | LR: 2.70e-04 | 1330.4ms/step | 15.0 samples/s +Validation at step 21250: Loss = 2.9764, Perplexity = 19.62 +Step 21260/44000 | Loss: 2.9997 | LR: 2.70e-04 | 3552.1ms/step | 5.6 samples/s +Step 21270/44000 | Loss: 2.9773 | LR: 2.70e-04 | 1330.5ms/step | 15.0 samples/s +Step 21280/44000 | Loss: 2.9613 | LR: 2.70e-04 | 1329.2ms/step | 15.0 samples/s +Step 21290/44000 | Loss: 2.9751 | LR: 2.70e-04 | 1330.3ms/step | 15.0 samples/s +Step 21300/44000 | Loss: 2.9538 | LR: 2.70e-04 | 1332.0ms/step | 15.0 samples/s +Step 21310/44000 | Loss: 2.9925 | LR: 2.69e-04 | 1335.9ms/step | 15.0 samples/s +Step 21320/44000 | Loss: 2.9884 | LR: 2.69e-04 | 1332.9ms/step | 15.0 samples/s +Step 21330/44000 | Loss: 2.9742 | LR: 2.69e-04 | 1331.2ms/step | 15.0 samples/s +Step 21340/44000 | Loss: 2.9771 | LR: 2.69e-04 | 1330.9ms/step | 15.0 samples/s +Step 21350/44000 | Loss: 2.9873 | LR: 2.69e-04 | 1327.4ms/step | 15.1 samples/s +Step 21360/44000 | Loss: 2.9758 | LR: 2.68e-04 | 1329.1ms/step | 15.0 samples/s +Step 21370/44000 | Loss: 3.0001 | LR: 2.68e-04 | 1327.8ms/step | 15.1 samples/s +Step 21380/44000 | Loss: 2.9909 | LR: 2.68e-04 | 1329.8ms/step | 15.0 samples/s +Step 21390/44000 | Loss: 2.9451 | LR: 2.68e-04 | 1330.5ms/step | 15.0 samples/s +Step 21400/44000 | Loss: 2.9794 | LR: 2.68e-04 | 1331.2ms/step | 15.0 samples/s +Step 21410/44000 | Loss: 2.9852 | LR: 2.68e-04 | 1333.1ms/step | 15.0 samples/s +Step 21420/44000 | Loss: 2.9965 | LR: 2.67e-04 | 1332.8ms/step | 15.0 samples/s +Step 21430/44000 | Loss: 2.9896 | LR: 2.67e-04 | 1335.4ms/step | 15.0 samples/s +Step 21440/44000 | Loss: 3.0153 | LR: 2.67e-04 | 1334.3ms/step | 15.0 samples/s +Step 21450/44000 | Loss: 2.9524 | LR: 2.67e-04 | 1336.4ms/step | 15.0 samples/s +Step 21460/44000 | Loss: 2.9815 | LR: 2.67e-04 | 1331.9ms/step | 15.0 samples/s +Step 21470/44000 | Loss: 2.9966 | LR: 2.66e-04 | 1332.5ms/step | 15.0 samples/s +Step 21480/44000 | Loss: 2.9871 | LR: 2.66e-04 | 1335.5ms/step | 15.0 samples/s +Step 21490/44000 | Loss: 2.9667 | LR: 2.66e-04 | 1330.1ms/step | 15.0 samples/s +Step 21500/44000 | Loss: 2.9601 | LR: 2.66e-04 | 1331.6ms/step | 15.0 samples/s +Validation at step 21500: Loss = 2.9737, Perplexity = 19.56 +Step 21510/44000 | Loss: 2.9550 | LR: 2.66e-04 | 3560.9ms/step | 5.6 samples/s +Step 21520/44000 | Loss: 2.9617 | LR: 2.66e-04 | 1333.9ms/step | 15.0 samples/s +Step 21530/44000 | Loss: 2.9454 | LR: 2.65e-04 | 1333.7ms/step | 15.0 samples/s +Step 21540/44000 | Loss: 2.9715 | LR: 2.65e-04 | 1331.0ms/step | 15.0 samples/s +Step 21550/44000 | Loss: 2.9890 | LR: 2.65e-04 | 1330.1ms/step | 15.0 samples/s +Step 21560/44000 | Loss: 3.0076 | LR: 2.65e-04 | 1331.0ms/step | 15.0 samples/s +Step 21570/44000 | Loss: 3.0082 | LR: 2.65e-04 | 1331.1ms/step | 15.0 samples/s +Step 21580/44000 | Loss: 3.0000 | LR: 2.64e-04 | 1332.4ms/step | 15.0 samples/s +Step 21590/44000 | Loss: 2.9629 | LR: 2.64e-04 | 1331.8ms/step | 15.0 samples/s +Step 21600/44000 | Loss: 2.9663 | LR: 2.64e-04 | 1331.9ms/step | 15.0 samples/s +Step 21610/44000 | Loss: 2.9850 | LR: 2.64e-04 | 1332.6ms/step | 15.0 samples/s +Step 21620/44000 | Loss: 2.9937 | LR: 2.64e-04 | 1331.6ms/step | 15.0 samples/s +Step 21630/44000 | Loss: 3.0043 | LR: 2.64e-04 | 1335.1ms/step | 15.0 samples/s +Step 21640/44000 | Loss: 2.9633 | LR: 2.63e-04 | 1337.4ms/step | 15.0 samples/s +Step 21650/44000 | Loss: 2.9863 | LR: 2.63e-04 | 1334.3ms/step | 15.0 samples/s +Step 21660/44000 | Loss: 2.9560 | LR: 2.63e-04 | 1335.9ms/step | 15.0 samples/s +Step 21670/44000 | Loss: 2.9959 | LR: 2.63e-04 | 1333.7ms/step | 15.0 samples/s +Step 21680/44000 | Loss: 2.9607 | LR: 2.63e-04 | 1332.2ms/step | 15.0 samples/s +Step 21690/44000 | Loss: 2.9489 | LR: 2.62e-04 | 1334.4ms/step | 15.0 samples/s +Step 21700/44000 | Loss: 2.9782 | LR: 2.62e-04 | 1335.1ms/step | 15.0 samples/s +Step 21710/44000 | Loss: 2.9472 | LR: 2.62e-04 | 1335.4ms/step | 15.0 samples/s +Step 21720/44000 | Loss: 2.9865 | LR: 2.62e-04 | 1335.0ms/step | 15.0 samples/s +Step 21730/44000 | Loss: 2.9807 | LR: 2.62e-04 | 1335.1ms/step | 15.0 samples/s +Step 21740/44000 | Loss: 2.9681 | LR: 2.62e-04 | 1333.7ms/step | 15.0 samples/s +Step 21750/44000 | Loss: 2.9759 | LR: 2.61e-04 | 1337.6ms/step | 15.0 samples/s +Validation at step 21750: Loss = 2.9718, Perplexity = 19.53 +Step 21760/44000 | Loss: 2.9665 | LR: 2.61e-04 | 3567.0ms/step | 5.6 samples/s +Step 21770/44000 | Loss: 2.9790 | LR: 2.61e-04 | 1331.2ms/step | 15.0 samples/s +Step 21780/44000 | Loss: 2.9633 | LR: 2.61e-04 | 1334.7ms/step | 15.0 samples/s +Step 21790/44000 | Loss: 2.9635 | LR: 2.61e-04 | 1337.0ms/step | 15.0 samples/s +Step 21800/44000 | Loss: 2.9847 | LR: 2.60e-04 | 1334.8ms/step | 15.0 samples/s +Step 21810/44000 | Loss: 2.9809 | LR: 2.60e-04 | 1336.1ms/step | 15.0 samples/s +Step 21820/44000 | Loss: 2.9939 | LR: 2.60e-04 | 1335.4ms/step | 15.0 samples/s +Step 21830/44000 | Loss: 2.9897 | LR: 2.60e-04 | 1335.6ms/step | 15.0 samples/s +Step 21840/44000 | Loss: 2.9848 | LR: 2.60e-04 | 1336.6ms/step | 15.0 samples/s +Step 21850/44000 | Loss: 2.9675 | LR: 2.60e-04 | 1334.0ms/step | 15.0 samples/s +Step 21860/44000 | Loss: 2.9994 | LR: 2.59e-04 | 1334.2ms/step | 15.0 samples/s +Step 21870/44000 | Loss: 2.9836 | LR: 2.59e-04 | 1333.5ms/step | 15.0 samples/s +Step 21880/44000 | Loss: 3.0124 | LR: 2.59e-04 | 1331.3ms/step | 15.0 samples/s +Step 21890/44000 | Loss: 2.9981 | LR: 2.59e-04 | 1333.7ms/step | 15.0 samples/s +Step 21900/44000 | Loss: 2.9685 | LR: 2.59e-04 | 1332.1ms/step | 15.0 samples/s +Step 21910/44000 | Loss: 2.9689 | LR: 2.58e-04 | 1333.9ms/step | 15.0 samples/s +Step 21920/44000 | Loss: 2.9486 | LR: 2.58e-04 | 1331.7ms/step | 15.0 samples/s +Step 21930/44000 | Loss: 2.9853 | LR: 2.58e-04 | 1332.5ms/step | 15.0 samples/s +Step 21940/44000 | Loss: 2.9925 | LR: 2.58e-04 | 1335.9ms/step | 15.0 samples/s +Step 21950/44000 | Loss: 3.0019 | LR: 2.58e-04 | 1336.5ms/step | 15.0 samples/s +Step 21960/44000 | Loss: 2.9764 | LR: 2.58e-04 | 1337.8ms/step | 14.9 samples/s +Step 21970/44000 | Loss: 3.0070 | LR: 2.57e-04 | 1336.1ms/step | 15.0 samples/s +Step 21980/44000 | Loss: 2.9440 | LR: 2.57e-04 | 1334.5ms/step | 15.0 samples/s +Step 21990/44000 | Loss: 2.9816 | LR: 2.57e-04 | 1337.9ms/step | 14.9 samples/s +Step 22000/44000 | Loss: 2.9663 | LR: 2.57e-04 | 1335.0ms/step | 15.0 samples/s +Validation at step 22000: Loss = 2.9691, Perplexity = 19.47 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-22000 +Step 22010/44000 | Loss: 2.9713 | LR: 2.57e-04 | 3861.2ms/step | 5.2 samples/s +Step 22020/44000 | Loss: 2.9807 | LR: 2.56e-04 | 1326.3ms/step | 15.1 samples/s +Step 22030/44000 | Loss: 2.9984 | LR: 2.56e-04 | 1355.3ms/step | 14.8 samples/s +Step 22040/44000 | Loss: 2.9901 | LR: 2.56e-04 | 1350.1ms/step | 14.8 samples/s +Step 22050/44000 | Loss: 2.9847 | LR: 2.56e-04 | 1345.7ms/step | 14.9 samples/s +Step 22060/44000 | Loss: 2.9912 | LR: 2.56e-04 | 1335.4ms/step | 15.0 samples/s +Step 22070/44000 | Loss: 2.9819 | LR: 2.56e-04 | 1331.8ms/step | 15.0 samples/s +Step 22080/44000 | Loss: 2.9792 | LR: 2.55e-04 | 1330.0ms/step | 15.0 samples/s +Step 22090/44000 | Loss: 2.9908 | LR: 2.55e-04 | 1332.5ms/step | 15.0 samples/s +Step 22100/44000 | Loss: 3.0018 | LR: 2.55e-04 | 1335.4ms/step | 15.0 samples/s +Step 22110/44000 | Loss: 2.9701 | LR: 2.55e-04 | 1331.2ms/step | 15.0 samples/s +Step 22120/44000 | Loss: 2.9616 | LR: 2.55e-04 | 1332.2ms/step | 15.0 samples/s +Step 22130/44000 | Loss: 3.0103 | LR: 2.54e-04 | 1332.8ms/step | 15.0 samples/s +Step 22140/44000 | Loss: 2.9645 | LR: 2.54e-04 | 1328.3ms/step | 15.1 samples/s +Step 22150/44000 | Loss: 2.9623 | LR: 2.54e-04 | 1327.3ms/step | 15.1 samples/s +Step 22160/44000 | Loss: 2.9982 | LR: 2.54e-04 | 1329.9ms/step | 15.0 samples/s +Step 22170/44000 | Loss: 2.9554 | LR: 2.54e-04 | 1331.3ms/step | 15.0 samples/s +Step 22180/44000 | Loss: 2.9951 | LR: 2.54e-04 | 1333.0ms/step | 15.0 samples/s +Step 22190/44000 | Loss: 2.9932 | LR: 2.53e-04 | 1335.3ms/step | 15.0 samples/s +Step 22200/44000 | Loss: 2.9699 | LR: 2.53e-04 | 1333.4ms/step | 15.0 samples/s +Step 22210/44000 | Loss: 2.9509 | LR: 2.53e-04 | 1332.2ms/step | 15.0 samples/s +Step 22220/44000 | Loss: 2.9746 | LR: 2.53e-04 | 1332.2ms/step | 15.0 samples/s +Step 22230/44000 | Loss: 2.9799 | LR: 2.53e-04 | 1333.6ms/step | 15.0 samples/s +Step 22240/44000 | Loss: 2.9791 | LR: 2.52e-04 | 1335.3ms/step | 15.0 samples/s +Step 22250/44000 | Loss: 2.9667 | LR: 2.52e-04 | 1332.9ms/step | 15.0 samples/s +Validation at step 22250: Loss = 2.9668, Perplexity = 19.43 +Step 22260/44000 | Loss: 2.9614 | LR: 2.52e-04 | 3561.8ms/step | 5.6 samples/s +Step 22270/44000 | Loss: 2.9800 | LR: 2.52e-04 | 1332.2ms/step | 15.0 samples/s +Step 22280/44000 | Loss: 2.9479 | LR: 2.52e-04 | 1329.3ms/step | 15.0 samples/s +Step 22290/44000 | Loss: 2.9930 | LR: 2.52e-04 | 1329.6ms/step | 15.0 samples/s +Step 22300/44000 | Loss: 2.9759 | LR: 2.51e-04 | 1330.5ms/step | 15.0 samples/s +Step 22310/44000 | Loss: 2.9863 | LR: 2.51e-04 | 1331.5ms/step | 15.0 samples/s +Step 22320/44000 | Loss: 2.9759 | LR: 2.51e-04 | 1330.1ms/step | 15.0 samples/s +Step 22330/44000 | Loss: 2.9712 | LR: 2.51e-04 | 1330.6ms/step | 15.0 samples/s +Step 22340/44000 | Loss: 2.9733 | LR: 2.51e-04 | 1332.9ms/step | 15.0 samples/s +Step 22350/44000 | Loss: 2.9952 | LR: 2.50e-04 | 1335.6ms/step | 15.0 samples/s +Step 22360/44000 | Loss: 2.9646 | LR: 2.50e-04 | 1336.2ms/step | 15.0 samples/s +Step 22370/44000 | Loss: 2.9620 | LR: 2.50e-04 | 1334.9ms/step | 15.0 samples/s +Step 22380/44000 | Loss: 2.9868 | LR: 2.50e-04 | 1333.3ms/step | 15.0 samples/s +Step 22390/44000 | Loss: 2.9913 | LR: 2.50e-04 | 1331.6ms/step | 15.0 samples/s +Step 22400/44000 | Loss: 2.9637 | LR: 2.50e-04 | 1330.1ms/step | 15.0 samples/s +Step 22410/44000 | Loss: 2.9910 | LR: 2.49e-04 | 1331.8ms/step | 15.0 samples/s +Step 22420/44000 | Loss: 2.9590 | LR: 2.49e-04 | 1334.6ms/step | 15.0 samples/s +Step 22430/44000 | Loss: 2.9936 | LR: 2.49e-04 | 1330.2ms/step | 15.0 samples/s +Step 22440/44000 | Loss: 3.0166 | LR: 2.49e-04 | 1330.3ms/step | 15.0 samples/s +Step 22450/44000 | Loss: 2.9868 | LR: 2.49e-04 | 1331.4ms/step | 15.0 samples/s +Step 22460/44000 | Loss: 2.9675 | LR: 2.48e-04 | 1334.0ms/step | 15.0 samples/s +Step 22470/44000 | Loss: 2.9676 | LR: 2.48e-04 | 1331.5ms/step | 15.0 samples/s +Step 22480/44000 | Loss: 2.9557 | LR: 2.48e-04 | 1327.9ms/step | 15.1 samples/s +Step 22490/44000 | Loss: 2.9853 | LR: 2.48e-04 | 1324.0ms/step | 15.1 samples/s +Step 22500/44000 | Loss: 2.9596 | LR: 2.48e-04 | 1322.3ms/step | 15.1 samples/s +Validation at step 22500: Loss = 2.9640, Perplexity = 19.38 +Step 22510/44000 | Loss: 2.9795 | LR: 2.48e-04 | 3542.6ms/step | 5.6 samples/s +Step 22520/44000 | Loss: 3.0082 | LR: 2.47e-04 | 1329.4ms/step | 15.0 samples/s +Step 22530/44000 | Loss: 2.9844 | LR: 2.47e-04 | 1332.1ms/step | 15.0 samples/s +Step 22540/44000 | Loss: 2.9651 | LR: 2.47e-04 | 1328.7ms/step | 15.1 samples/s +Step 22550/44000 | Loss: 3.0035 | LR: 2.47e-04 | 1325.9ms/step | 15.1 samples/s +Step 22560/44000 | Loss: 2.9719 | LR: 2.47e-04 | 1330.0ms/step | 15.0 samples/s +Step 22570/44000 | Loss: 2.9454 | LR: 2.46e-04 | 1330.4ms/step | 15.0 samples/s +Step 22580/44000 | Loss: 2.9838 | LR: 2.46e-04 | 1332.4ms/step | 15.0 samples/s +Step 22590/44000 | Loss: 2.9645 | LR: 2.46e-04 | 1331.3ms/step | 15.0 samples/s +Step 22600/44000 | Loss: 2.9813 | LR: 2.46e-04 | 1330.4ms/step | 15.0 samples/s +Step 22610/44000 | Loss: 2.9875 | LR: 2.46e-04 | 1333.5ms/step | 15.0 samples/s +Step 22620/44000 | Loss: 2.9933 | LR: 2.46e-04 | 1334.4ms/step | 15.0 samples/s +Step 22630/44000 | Loss: 2.9878 | LR: 2.45e-04 | 1331.3ms/step | 15.0 samples/s +Step 22640/44000 | Loss: 2.9694 | LR: 2.45e-04 | 1330.9ms/step | 15.0 samples/s +Step 22650/44000 | Loss: 2.9663 | LR: 2.45e-04 | 1328.7ms/step | 15.1 samples/s +Step 22660/44000 | Loss: 2.9811 | LR: 2.45e-04 | 1332.5ms/step | 15.0 samples/s +Step 22670/44000 | Loss: 2.9332 | LR: 2.45e-04 | 1331.0ms/step | 15.0 samples/s +Step 22680/44000 | Loss: 2.9660 | LR: 2.44e-04 | 1329.8ms/step | 15.0 samples/s +Step 22690/44000 | Loss: 2.9435 | LR: 2.44e-04 | 1331.8ms/step | 15.0 samples/s +Step 22700/44000 | Loss: 2.9680 | LR: 2.44e-04 | 1329.2ms/step | 15.0 samples/s +Step 22710/44000 | Loss: 2.9463 | LR: 2.44e-04 | 1327.3ms/step | 15.1 samples/s +Step 22720/44000 | Loss: 2.9663 | LR: 2.44e-04 | 1327.2ms/step | 15.1 samples/s +Step 22730/44000 | Loss: 2.9684 | LR: 2.44e-04 | 1325.7ms/step | 15.1 samples/s +Step 22740/44000 | Loss: 2.9715 | LR: 2.43e-04 | 1330.2ms/step | 15.0 samples/s +Step 22750/44000 | Loss: 2.9766 | LR: 2.43e-04 | 1329.3ms/step | 15.0 samples/s +Validation at step 22750: Loss = 2.9623, Perplexity = 19.34 +Step 22760/44000 | Loss: 2.9552 | LR: 2.43e-04 | 3557.3ms/step | 5.6 samples/s +Step 22770/44000 | Loss: 2.9649 | LR: 2.43e-04 | 1329.5ms/step | 15.0 samples/s +Step 22780/44000 | Loss: 2.9524 | LR: 2.43e-04 | 1327.8ms/step | 15.1 samples/s +Step 22790/44000 | Loss: 2.9795 | LR: 2.42e-04 | 1329.3ms/step | 15.0 samples/s +Step 22800/44000 | Loss: 2.9782 | LR: 2.42e-04 | 1329.7ms/step | 15.0 samples/s +Step 22810/44000 | Loss: 2.9643 | LR: 2.42e-04 | 1330.6ms/step | 15.0 samples/s +Step 22820/44000 | Loss: 2.9568 | LR: 2.42e-04 | 1330.4ms/step | 15.0 samples/s +Step 22830/44000 | Loss: 2.9775 | LR: 2.42e-04 | 1331.5ms/step | 15.0 samples/s +Step 22840/44000 | Loss: 2.9875 | LR: 2.42e-04 | 1332.7ms/step | 15.0 samples/s +Step 22850/44000 | Loss: 2.9994 | LR: 2.41e-04 | 1331.8ms/step | 15.0 samples/s +Step 22860/44000 | Loss: 2.9838 | LR: 2.41e-04 | 1333.1ms/step | 15.0 samples/s +Step 22870/44000 | Loss: 2.9704 | LR: 2.41e-04 | 1332.6ms/step | 15.0 samples/s +Step 22880/44000 | Loss: 2.9696 | LR: 2.41e-04 | 1333.2ms/step | 15.0 samples/s +Step 22890/44000 | Loss: 2.9716 | LR: 2.41e-04 | 1332.5ms/step | 15.0 samples/s +Step 22900/44000 | Loss: 2.9864 | LR: 2.40e-04 | 1335.0ms/step | 15.0 samples/s +Step 22910/44000 | Loss: 2.9899 | LR: 2.40e-04 | 1331.4ms/step | 15.0 samples/s +Step 22920/44000 | Loss: 2.9574 | LR: 2.40e-04 | 1332.1ms/step | 15.0 samples/s +Step 22930/44000 | Loss: 2.9570 | LR: 2.40e-04 | 1332.5ms/step | 15.0 samples/s +Step 22940/44000 | Loss: 2.9418 | LR: 2.40e-04 | 1331.2ms/step | 15.0 samples/s +Step 22950/44000 | Loss: 2.9942 | LR: 2.40e-04 | 1332.8ms/step | 15.0 samples/s +Step 22960/44000 | Loss: 2.9767 | LR: 2.39e-04 | 1332.2ms/step | 15.0 samples/s +Step 22970/44000 | Loss: 2.9770 | LR: 2.39e-04 | 1329.9ms/step | 15.0 samples/s +Step 22980/44000 | Loss: 2.9837 | LR: 2.39e-04 | 1323.3ms/step | 15.1 samples/s +Step 22990/44000 | Loss: 2.9855 | LR: 2.39e-04 | 1322.2ms/step | 15.1 samples/s +Step 23000/44000 | Loss: 2.9621 | LR: 2.39e-04 | 1325.5ms/step | 15.1 samples/s +Validation at step 23000: Loss = 2.9596, Perplexity = 19.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-23000 +Step 23010/44000 | Loss: 2.9805 | LR: 2.38e-04 | 3840.7ms/step | 5.2 samples/s +Step 23020/44000 | Loss: 2.9829 | LR: 2.38e-04 | 1323.7ms/step | 15.1 samples/s +Step 23030/44000 | Loss: 2.9678 | LR: 2.38e-04 | 1324.3ms/step | 15.1 samples/s +Step 23040/44000 | Loss: 2.9994 | LR: 2.38e-04 | 1324.7ms/step | 15.1 samples/s +Step 23050/44000 | Loss: 2.9537 | LR: 2.38e-04 | 1327.7ms/step | 15.1 samples/s +Step 23060/44000 | Loss: 2.9935 | LR: 2.38e-04 | 1325.7ms/step | 15.1 samples/s +Step 23070/44000 | Loss: 2.9665 | LR: 2.37e-04 | 1323.2ms/step | 15.1 samples/s +Step 23080/44000 | Loss: 2.9902 | LR: 2.37e-04 | 1326.2ms/step | 15.1 samples/s +Step 23090/44000 | Loss: 2.9678 | LR: 2.37e-04 | 1328.3ms/step | 15.1 samples/s +Step 23100/44000 | Loss: 2.9886 | LR: 2.37e-04 | 1328.4ms/step | 15.1 samples/s +Step 23110/44000 | Loss: 2.9714 | LR: 2.37e-04 | 1328.3ms/step | 15.1 samples/s +Step 23120/44000 | Loss: 3.0081 | LR: 2.36e-04 | 1324.5ms/step | 15.1 samples/s +Step 23130/44000 | Loss: 3.0150 | LR: 2.36e-04 | 1323.5ms/step | 15.1 samples/s +Step 23140/44000 | Loss: 2.9154 | LR: 2.36e-04 | 1322.8ms/step | 15.1 samples/s +Step 23150/44000 | Loss: 2.9479 | LR: 2.36e-04 | 1330.1ms/step | 15.0 samples/s +Step 23160/44000 | Loss: 2.9843 | LR: 2.36e-04 | 1326.6ms/step | 15.1 samples/s +Step 23170/44000 | Loss: 2.9727 | LR: 2.36e-04 | 1325.5ms/step | 15.1 samples/s +Step 23180/44000 | Loss: 2.9684 | LR: 2.35e-04 | 1328.0ms/step | 15.1 samples/s +Step 23190/44000 | Loss: 2.9660 | LR: 2.35e-04 | 1330.1ms/step | 15.0 samples/s +Step 23200/44000 | Loss: 2.9322 | LR: 2.35e-04 | 1328.8ms/step | 15.1 samples/s +Step 23210/44000 | Loss: 3.0052 | LR: 2.35e-04 | 1327.5ms/step | 15.1 samples/s +Step 23220/44000 | Loss: 2.9684 | LR: 2.35e-04 | 1329.9ms/step | 15.0 samples/s +Step 23230/44000 | Loss: 2.9759 | LR: 2.34e-04 | 1327.6ms/step | 15.1 samples/s +Step 23240/44000 | Loss: 2.9910 | LR: 2.34e-04 | 1328.6ms/step | 15.1 samples/s +Step 23250/44000 | Loss: 2.9443 | LR: 2.34e-04 | 1325.5ms/step | 15.1 samples/s +Validation at step 23250: Loss = 2.9577, Perplexity = 19.25 +Step 23260/44000 | Loss: 2.9629 | LR: 2.34e-04 | 3541.6ms/step | 5.6 samples/s +Step 23270/44000 | Loss: 2.9669 | LR: 2.34e-04 | 1325.4ms/step | 15.1 samples/s +Step 23280/44000 | Loss: 2.9658 | LR: 2.34e-04 | 1323.4ms/step | 15.1 samples/s +Step 23290/44000 | Loss: 2.9484 | LR: 2.33e-04 | 1323.3ms/step | 15.1 samples/s +Step 23300/44000 | Loss: 2.9806 | LR: 2.33e-04 | 1321.1ms/step | 15.1 samples/s +Step 23310/44000 | Loss: 2.9743 | LR: 2.33e-04 | 1326.9ms/step | 15.1 samples/s +Step 23320/44000 | Loss: 2.9811 | LR: 2.33e-04 | 1328.5ms/step | 15.1 samples/s +Step 23330/44000 | Loss: 2.9546 | LR: 2.33e-04 | 1329.1ms/step | 15.0 samples/s +Step 23340/44000 | Loss: 2.9541 | LR: 2.32e-04 | 1325.8ms/step | 15.1 samples/s +Step 23350/44000 | Loss: 2.9336 | LR: 2.32e-04 | 1328.7ms/step | 15.1 samples/s +Step 23360/44000 | Loss: 2.9684 | LR: 2.32e-04 | 1329.7ms/step | 15.0 samples/s +Step 23370/44000 | Loss: 2.9919 | LR: 2.32e-04 | 1327.7ms/step | 15.1 samples/s +Step 23380/44000 | Loss: 2.9753 | LR: 2.32e-04 | 1331.8ms/step | 15.0 samples/s +Step 23390/44000 | Loss: 3.0004 | LR: 2.32e-04 | 1328.8ms/step | 15.1 samples/s +Step 23400/44000 | Loss: 2.9655 | LR: 2.31e-04 | 1325.4ms/step | 15.1 samples/s +Step 23410/44000 | Loss: 2.9585 | LR: 2.31e-04 | 1324.1ms/step | 15.1 samples/s +Step 23420/44000 | Loss: 3.0072 | LR: 2.31e-04 | 1326.8ms/step | 15.1 samples/s +Step 23430/44000 | Loss: 2.9872 | LR: 2.31e-04 | 1326.8ms/step | 15.1 samples/s +Step 23440/44000 | Loss: 2.9352 | LR: 2.31e-04 | 1328.1ms/step | 15.1 samples/s +Step 23450/44000 | Loss: 2.9574 | LR: 2.30e-04 | 1329.3ms/step | 15.0 samples/s +Step 23460/44000 | Loss: 2.9670 | LR: 2.30e-04 | 1332.6ms/step | 15.0 samples/s +Step 23470/44000 | Loss: 2.9733 | LR: 2.30e-04 | 1330.3ms/step | 15.0 samples/s +Step 23480/44000 | Loss: 2.9623 | LR: 2.30e-04 | 1330.6ms/step | 15.0 samples/s +Step 23490/44000 | Loss: 2.9679 | LR: 2.30e-04 | 1327.1ms/step | 15.1 samples/s +Step 23500/44000 | Loss: 2.9769 | LR: 2.30e-04 | 1329.8ms/step | 15.0 samples/s +Validation at step 23500: Loss = 2.9548, Perplexity = 19.20 +Step 23510/44000 | Loss: 2.9779 | LR: 2.29e-04 | 3547.6ms/step | 5.6 samples/s +Step 23520/44000 | Loss: 2.9652 | LR: 2.29e-04 | 1327.8ms/step | 15.1 samples/s +Step 23530/44000 | Loss: 2.9356 | LR: 2.29e-04 | 1329.8ms/step | 15.0 samples/s +Step 23540/44000 | Loss: 2.9233 | LR: 2.29e-04 | 1330.5ms/step | 15.0 samples/s +Step 23550/44000 | Loss: 2.9558 | LR: 2.29e-04 | 1333.1ms/step | 15.0 samples/s +Step 23560/44000 | Loss: 2.9511 | LR: 2.29e-04 | 1333.7ms/step | 15.0 samples/s +Step 23570/44000 | Loss: 2.9571 | LR: 2.28e-04 | 1331.9ms/step | 15.0 samples/s +Step 23580/44000 | Loss: 2.9476 | LR: 2.28e-04 | 1330.6ms/step | 15.0 samples/s +Step 23590/44000 | Loss: 2.9863 | LR: 2.28e-04 | 1329.3ms/step | 15.0 samples/s +Step 23600/44000 | Loss: 2.9131 | LR: 2.28e-04 | 1330.1ms/step | 15.0 samples/s +Step 23610/44000 | Loss: 2.9591 | LR: 2.28e-04 | 1331.0ms/step | 15.0 samples/s +Step 23620/44000 | Loss: 2.9663 | LR: 2.27e-04 | 1332.7ms/step | 15.0 samples/s +Step 23630/44000 | Loss: 2.9273 | LR: 2.27e-04 | 1335.5ms/step | 15.0 samples/s +Step 23640/44000 | Loss: 2.9592 | LR: 2.27e-04 | 1335.4ms/step | 15.0 samples/s +Step 23650/44000 | Loss: 2.9867 | LR: 2.27e-04 | 1333.5ms/step | 15.0 samples/s +Step 23660/44000 | Loss: 2.9188 | LR: 2.27e-04 | 1332.3ms/step | 15.0 samples/s +Step 23670/44000 | Loss: 2.9491 | LR: 2.27e-04 | 1335.5ms/step | 15.0 samples/s +Step 23680/44000 | Loss: 2.9640 | LR: 2.26e-04 | 1335.8ms/step | 15.0 samples/s +Step 23690/44000 | Loss: 2.9558 | LR: 2.26e-04 | 1333.0ms/step | 15.0 samples/s +Step 23700/44000 | Loss: 2.9686 | LR: 2.26e-04 | 1331.7ms/step | 15.0 samples/s +Step 23710/44000 | Loss: 2.9554 | LR: 2.26e-04 | 1335.7ms/step | 15.0 samples/s +Step 23720/44000 | Loss: 2.9400 | LR: 2.26e-04 | 1335.2ms/step | 15.0 samples/s +Step 23730/44000 | Loss: 3.0008 | LR: 2.25e-04 | 1334.9ms/step | 15.0 samples/s +Step 23740/44000 | Loss: 2.9733 | LR: 2.25e-04 | 1334.8ms/step | 15.0 samples/s +Step 23750/44000 | Loss: 2.9942 | LR: 2.25e-04 | 1331.3ms/step | 15.0 samples/s +Validation at step 23750: Loss = 2.9534, Perplexity = 19.17 +Step 23760/44000 | Loss: 2.9568 | LR: 2.25e-04 | 3561.5ms/step | 5.6 samples/s +Step 23770/44000 | Loss: 2.9942 | LR: 2.25e-04 | 1334.0ms/step | 15.0 samples/s +Step 23780/44000 | Loss: 2.9762 | LR: 2.25e-04 | 1335.0ms/step | 15.0 samples/s +Step 23790/44000 | Loss: 2.9609 | LR: 2.24e-04 | 1332.9ms/step | 15.0 samples/s +Step 23800/44000 | Loss: 2.9608 | LR: 2.24e-04 | 1331.1ms/step | 15.0 samples/s +Step 23810/44000 | Loss: 2.9638 | LR: 2.24e-04 | 1331.0ms/step | 15.0 samples/s +Step 23820/44000 | Loss: 2.9714 | LR: 2.24e-04 | 1331.6ms/step | 15.0 samples/s +Step 23830/44000 | Loss: 2.9669 | LR: 2.24e-04 | 1331.6ms/step | 15.0 samples/s +Step 23840/44000 | Loss: 2.9730 | LR: 2.23e-04 | 1332.8ms/step | 15.0 samples/s +Step 23850/44000 | Loss: 2.9768 | LR: 2.23e-04 | 1330.8ms/step | 15.0 samples/s +Step 23860/44000 | Loss: 2.9578 | LR: 2.23e-04 | 1327.9ms/step | 15.1 samples/s +Step 23870/44000 | Loss: 2.9396 | LR: 2.23e-04 | 1328.1ms/step | 15.1 samples/s +Step 23880/44000 | Loss: 2.9836 | LR: 2.23e-04 | 1330.1ms/step | 15.0 samples/s +Step 23890/44000 | Loss: 2.9965 | LR: 2.23e-04 | 1330.4ms/step | 15.0 samples/s +Step 23900/44000 | Loss: 2.9585 | LR: 2.22e-04 | 1331.1ms/step | 15.0 samples/s +Step 23910/44000 | Loss: 2.9578 | LR: 2.22e-04 | 1332.7ms/step | 15.0 samples/s +Step 23920/44000 | Loss: 2.9841 | LR: 2.22e-04 | 1330.6ms/step | 15.0 samples/s +Step 23930/44000 | Loss: 2.9573 | LR: 2.22e-04 | 1329.2ms/step | 15.0 samples/s +Step 23940/44000 | Loss: 2.9486 | LR: 2.22e-04 | 1332.2ms/step | 15.0 samples/s +Step 23950/44000 | Loss: 2.9343 | LR: 2.21e-04 | 1329.9ms/step | 15.0 samples/s +Step 23960/44000 | Loss: 2.9576 | LR: 2.21e-04 | 1325.0ms/step | 15.1 samples/s +Step 23970/44000 | Loss: 2.9572 | LR: 2.21e-04 | 1323.9ms/step | 15.1 samples/s +Step 23980/44000 | Loss: 2.9540 | LR: 2.21e-04 | 1324.1ms/step | 15.1 samples/s +Step 23990/44000 | Loss: 2.9697 | LR: 2.21e-04 | 1328.2ms/step | 15.1 samples/s +Step 24000/44000 | Loss: 2.9800 | LR: 2.21e-04 | 1327.9ms/step | 15.1 samples/s +Validation at step 24000: Loss = 2.9513, Perplexity = 19.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-24000 +Step 24010/44000 | Loss: 2.9618 | LR: 2.20e-04 | 3838.6ms/step | 5.2 samples/s +Step 24020/44000 | Loss: 2.9398 | LR: 2.20e-04 | 1322.7ms/step | 15.1 samples/s +Step 24030/44000 | Loss: 2.9655 | LR: 2.20e-04 | 1327.6ms/step | 15.1 samples/s +Step 24040/44000 | Loss: 2.9376 | LR: 2.20e-04 | 1321.9ms/step | 15.1 samples/s +Step 24050/44000 | Loss: 2.9892 | LR: 2.20e-04 | 1318.7ms/step | 15.2 samples/s +Step 24060/44000 | Loss: 2.9292 | LR: 2.19e-04 | 1320.9ms/step | 15.1 samples/s +Step 24070/44000 | Loss: 2.9436 | LR: 2.19e-04 | 1325.8ms/step | 15.1 samples/s +Step 24080/44000 | Loss: 2.9580 | LR: 2.19e-04 | 1331.5ms/step | 15.0 samples/s +Step 24090/44000 | Loss: 2.9720 | LR: 2.19e-04 | 1330.7ms/step | 15.0 samples/s +Step 24100/44000 | Loss: 2.9434 | LR: 2.19e-04 | 1329.7ms/step | 15.0 samples/s +Step 24110/44000 | Loss: 2.9941 | LR: 2.19e-04 | 1326.1ms/step | 15.1 samples/s +Step 24120/44000 | Loss: 3.0008 | LR: 2.18e-04 | 1324.9ms/step | 15.1 samples/s +Step 24130/44000 | Loss: 2.9871 | LR: 2.18e-04 | 1323.9ms/step | 15.1 samples/s +Step 24140/44000 | Loss: 2.9604 | LR: 2.18e-04 | 1328.5ms/step | 15.1 samples/s +Step 24150/44000 | Loss: 2.9542 | LR: 2.18e-04 | 1330.0ms/step | 15.0 samples/s +Step 24160/44000 | Loss: 2.9721 | LR: 2.18e-04 | 1330.6ms/step | 15.0 samples/s +Step 24170/44000 | Loss: 2.9661 | LR: 2.17e-04 | 1328.5ms/step | 15.1 samples/s +Step 24180/44000 | Loss: 2.9456 | LR: 2.17e-04 | 1326.7ms/step | 15.1 samples/s +Step 24190/44000 | Loss: 2.9688 | LR: 2.17e-04 | 1326.3ms/step | 15.1 samples/s +Step 24200/44000 | Loss: 2.9663 | LR: 2.17e-04 | 1325.1ms/step | 15.1 samples/s +Step 24210/44000 | Loss: 2.9472 | LR: 2.17e-04 | 1328.5ms/step | 15.1 samples/s +Step 24220/44000 | Loss: 2.9480 | LR: 2.17e-04 | 1328.8ms/step | 15.1 samples/s +Step 24230/44000 | Loss: 2.9721 | LR: 2.16e-04 | 1326.6ms/step | 15.1 samples/s +Step 24240/44000 | Loss: 2.9473 | LR: 2.16e-04 | 1321.8ms/step | 15.1 samples/s +Step 24250/44000 | Loss: 2.9860 | LR: 2.16e-04 | 1324.2ms/step | 15.1 samples/s +Validation at step 24250: Loss = 2.9492, Perplexity = 19.09 +Step 24260/44000 | Loss: 2.9438 | LR: 2.16e-04 | 3554.0ms/step | 5.6 samples/s +Step 24270/44000 | Loss: 2.9505 | LR: 2.16e-04 | 1331.7ms/step | 15.0 samples/s +Step 24280/44000 | Loss: 2.9608 | LR: 2.16e-04 | 1328.6ms/step | 15.1 samples/s +Step 24290/44000 | Loss: 2.9351 | LR: 2.15e-04 | 1328.4ms/step | 15.1 samples/s +Step 24300/44000 | Loss: 2.9558 | LR: 2.15e-04 | 1326.4ms/step | 15.1 samples/s +Step 24310/44000 | Loss: 2.9466 | LR: 2.15e-04 | 1326.1ms/step | 15.1 samples/s +Step 24320/44000 | Loss: 2.9557 | LR: 2.15e-04 | 1327.4ms/step | 15.1 samples/s +Step 24330/44000 | Loss: 2.9499 | LR: 2.15e-04 | 1326.8ms/step | 15.1 samples/s +Step 24340/44000 | Loss: 2.9519 | LR: 2.14e-04 | 1327.3ms/step | 15.1 samples/s +Step 24350/44000 | Loss: 2.9560 | LR: 2.14e-04 | 1328.6ms/step | 15.1 samples/s +Step 24360/44000 | Loss: 2.9502 | LR: 2.14e-04 | 1326.3ms/step | 15.1 samples/s +Step 24370/44000 | Loss: 2.9706 | LR: 2.14e-04 | 1328.6ms/step | 15.1 samples/s +Step 24380/44000 | Loss: 2.9658 | LR: 2.14e-04 | 1328.7ms/step | 15.1 samples/s +Step 24390/44000 | Loss: 2.9558 | LR: 2.14e-04 | 1327.5ms/step | 15.1 samples/s +Step 24400/44000 | Loss: 2.9632 | LR: 2.13e-04 | 1331.3ms/step | 15.0 samples/s +Step 24410/44000 | Loss: 2.9734 | LR: 2.13e-04 | 1330.4ms/step | 15.0 samples/s +Step 24420/44000 | Loss: 2.9673 | LR: 2.13e-04 | 1326.4ms/step | 15.1 samples/s +Step 24430/44000 | Loss: 2.9551 | LR: 2.13e-04 | 1328.8ms/step | 15.1 samples/s +Step 24440/44000 | Loss: 2.9707 | LR: 2.13e-04 | 1325.1ms/step | 15.1 samples/s +Step 24450/44000 | Loss: 2.9727 | LR: 2.12e-04 | 1330.5ms/step | 15.0 samples/s +Step 24460/44000 | Loss: 2.9213 | LR: 2.12e-04 | 1329.1ms/step | 15.0 samples/s +Step 24470/44000 | Loss: 2.9763 | LR: 2.12e-04 | 1326.6ms/step | 15.1 samples/s +Step 24480/44000 | Loss: 2.9559 | LR: 2.12e-04 | 1325.1ms/step | 15.1 samples/s +Step 24490/44000 | Loss: 2.9516 | LR: 2.12e-04 | 1329.1ms/step | 15.0 samples/s +Step 24500/44000 | Loss: 2.9281 | LR: 2.12e-04 | 1329.8ms/step | 15.0 samples/s +Validation at step 24500: Loss = 2.9473, Perplexity = 19.05 +Step 24510/44000 | Loss: 2.9549 | LR: 2.11e-04 | 3553.1ms/step | 5.6 samples/s +Step 24520/44000 | Loss: 2.9536 | LR: 2.11e-04 | 1332.0ms/step | 15.0 samples/s +Step 24530/44000 | Loss: 2.9613 | LR: 2.11e-04 | 1332.9ms/step | 15.0 samples/s +Step 24540/44000 | Loss: 2.9469 | LR: 2.11e-04 | 1332.6ms/step | 15.0 samples/s +Step 24550/44000 | Loss: 2.9350 | LR: 2.11e-04 | 1328.5ms/step | 15.1 samples/s +Step 24560/44000 | Loss: 2.9683 | LR: 2.10e-04 | 1326.8ms/step | 15.1 samples/s +Step 24570/44000 | Loss: 2.9721 | LR: 2.10e-04 | 1326.0ms/step | 15.1 samples/s +Step 24580/44000 | Loss: 2.9582 | LR: 2.10e-04 | 1327.7ms/step | 15.1 samples/s +Step 24590/44000 | Loss: 2.9527 | LR: 2.10e-04 | 1332.7ms/step | 15.0 samples/s +Step 24600/44000 | Loss: 2.9475 | LR: 2.10e-04 | 1334.3ms/step | 15.0 samples/s +Step 24610/44000 | Loss: 2.9670 | LR: 2.10e-04 | 1334.0ms/step | 15.0 samples/s +Step 24620/44000 | Loss: 2.9556 | LR: 2.09e-04 | 1333.5ms/step | 15.0 samples/s +Step 24630/44000 | Loss: 2.9661 | LR: 2.09e-04 | 1333.9ms/step | 15.0 samples/s +Step 24640/44000 | Loss: 2.9463 | LR: 2.09e-04 | 1337.1ms/step | 15.0 samples/s +Step 24650/44000 | Loss: 2.9611 | LR: 2.09e-04 | 1333.0ms/step | 15.0 samples/s +Step 24660/44000 | Loss: 2.9766 | LR: 2.09e-04 | 1330.7ms/step | 15.0 samples/s +Step 24670/44000 | Loss: 2.9783 | LR: 2.09e-04 | 1329.7ms/step | 15.0 samples/s +Step 24680/44000 | Loss: 2.9516 | LR: 2.08e-04 | 1331.6ms/step | 15.0 samples/s +Step 24690/44000 | Loss: 2.9569 | LR: 2.08e-04 | 1330.5ms/step | 15.0 samples/s +Step 24700/44000 | Loss: 2.9327 | LR: 2.08e-04 | 1325.7ms/step | 15.1 samples/s +Step 24710/44000 | Loss: 2.9693 | LR: 2.08e-04 | 1326.9ms/step | 15.1 samples/s +Step 24720/44000 | Loss: 2.9310 | LR: 2.08e-04 | 1330.4ms/step | 15.0 samples/s +Step 24730/44000 | Loss: 2.9526 | LR: 2.07e-04 | 1331.2ms/step | 15.0 samples/s +Step 24740/44000 | Loss: 2.9392 | LR: 2.07e-04 | 1332.1ms/step | 15.0 samples/s +Step 24750/44000 | Loss: 2.9558 | LR: 2.07e-04 | 1332.8ms/step | 15.0 samples/s +Validation at step 24750: Loss = 2.9450, Perplexity = 19.01 +Step 24760/44000 | Loss: 2.9532 | LR: 2.07e-04 | 3560.5ms/step | 5.6 samples/s +Step 24770/44000 | Loss: 2.9403 | LR: 2.07e-04 | 1328.8ms/step | 15.1 samples/s +Step 24780/44000 | Loss: 2.9450 | LR: 2.07e-04 | 1331.1ms/step | 15.0 samples/s +Step 24790/44000 | Loss: 3.0171 | LR: 2.06e-04 | 1334.1ms/step | 15.0 samples/s +Step 24800/44000 | Loss: 2.9604 | LR: 2.06e-04 | 1330.4ms/step | 15.0 samples/s +Step 24810/44000 | Loss: 2.9598 | LR: 2.06e-04 | 1333.0ms/step | 15.0 samples/s +Step 24820/44000 | Loss: 2.9688 | LR: 2.06e-04 | 1335.4ms/step | 15.0 samples/s +Step 24830/44000 | Loss: 2.9603 | LR: 2.06e-04 | 1328.3ms/step | 15.1 samples/s +Step 24840/44000 | Loss: 2.9345 | LR: 2.05e-04 | 1329.9ms/step | 15.0 samples/s +Step 24850/44000 | Loss: 2.9550 | LR: 2.05e-04 | 1328.6ms/step | 15.1 samples/s +Step 24860/44000 | Loss: 2.9479 | LR: 2.05e-04 | 1328.2ms/step | 15.1 samples/s +Step 24870/44000 | Loss: 2.9570 | LR: 2.05e-04 | 1330.3ms/step | 15.0 samples/s +Step 24880/44000 | Loss: 2.9676 | LR: 2.05e-04 | 1332.1ms/step | 15.0 samples/s +Step 24890/44000 | Loss: 2.9360 | LR: 2.05e-04 | 1331.1ms/step | 15.0 samples/s +Step 24900/44000 | Loss: 2.9380 | LR: 2.04e-04 | 1332.3ms/step | 15.0 samples/s +Step 24910/44000 | Loss: 2.9796 | LR: 2.04e-04 | 1335.4ms/step | 15.0 samples/s +Step 24920/44000 | Loss: 2.9983 | LR: 2.04e-04 | 1332.7ms/step | 15.0 samples/s +Step 24930/44000 | Loss: 2.9699 | LR: 2.04e-04 | 1331.0ms/step | 15.0 samples/s +Step 24940/44000 | Loss: 2.9504 | LR: 2.04e-04 | 1326.7ms/step | 15.1 samples/s +Step 24950/44000 | Loss: 2.9471 | LR: 2.04e-04 | 1327.4ms/step | 15.1 samples/s +Step 24960/44000 | Loss: 2.9823 | LR: 2.03e-04 | 1331.4ms/step | 15.0 samples/s +Step 24970/44000 | Loss: 2.9696 | LR: 2.03e-04 | 1328.5ms/step | 15.1 samples/s +Step 24980/44000 | Loss: 2.9503 | LR: 2.03e-04 | 1328.7ms/step | 15.1 samples/s +Step 24990/44000 | Loss: 2.9461 | LR: 2.03e-04 | 1329.0ms/step | 15.0 samples/s +Step 25000/44000 | Loss: 2.9757 | LR: 2.03e-04 | 1325.6ms/step | 15.1 samples/s +Validation at step 25000: Loss = 2.9433, Perplexity = 18.98 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-25000 +Step 25010/44000 | Loss: 2.9672 | LR: 2.02e-04 | 3829.3ms/step | 5.2 samples/s +Step 25020/44000 | Loss: 2.9614 | LR: 2.02e-04 | 1325.0ms/step | 15.1 samples/s +Step 25030/44000 | Loss: 2.9708 | LR: 2.02e-04 | 1325.4ms/step | 15.1 samples/s +Step 25040/44000 | Loss: 2.9480 | LR: 2.02e-04 | 1329.6ms/step | 15.0 samples/s +Step 25050/44000 | Loss: 2.9525 | LR: 2.02e-04 | 1328.2ms/step | 15.1 samples/s +Step 25060/44000 | Loss: 2.9305 | LR: 2.02e-04 | 1327.9ms/step | 15.1 samples/s +Step 25070/44000 | Loss: 2.9384 | LR: 2.01e-04 | 1327.2ms/step | 15.1 samples/s +Step 25080/44000 | Loss: 2.9438 | LR: 2.01e-04 | 1325.3ms/step | 15.1 samples/s +Step 25090/44000 | Loss: 2.9887 | LR: 2.01e-04 | 1324.5ms/step | 15.1 samples/s +Step 25100/44000 | Loss: 2.9531 | LR: 2.01e-04 | 1326.9ms/step | 15.1 samples/s +Step 25110/44000 | Loss: 2.9477 | LR: 2.01e-04 | 1331.0ms/step | 15.0 samples/s +Step 25120/44000 | Loss: 2.9348 | LR: 2.00e-04 | 1328.3ms/step | 15.1 samples/s +Step 25130/44000 | Loss: 2.9525 | LR: 2.00e-04 | 1328.3ms/step | 15.1 samples/s +Step 25140/44000 | Loss: 2.9793 | LR: 2.00e-04 | 1327.3ms/step | 15.1 samples/s +Step 25150/44000 | Loss: 2.9813 | LR: 2.00e-04 | 1329.6ms/step | 15.0 samples/s +Step 25160/44000 | Loss: 2.9594 | LR: 2.00e-04 | 1327.7ms/step | 15.1 samples/s +Step 25170/44000 | Loss: 2.9455 | LR: 2.00e-04 | 1325.4ms/step | 15.1 samples/s +Step 25180/44000 | Loss: 2.9373 | LR: 1.99e-04 | 1322.9ms/step | 15.1 samples/s +Step 25190/44000 | Loss: 2.9803 | LR: 1.99e-04 | 1326.0ms/step | 15.1 samples/s +Step 25200/44000 | Loss: 2.9578 | LR: 1.99e-04 | 1324.5ms/step | 15.1 samples/s +Step 25210/44000 | Loss: 2.9484 | LR: 1.99e-04 | 1325.5ms/step | 15.1 samples/s +Step 25220/44000 | Loss: 2.9529 | LR: 1.99e-04 | 1326.6ms/step | 15.1 samples/s +Step 25230/44000 | Loss: 2.9575 | LR: 1.99e-04 | 1326.3ms/step | 15.1 samples/s +Step 25240/44000 | Loss: 2.9596 | LR: 1.98e-04 | 1327.3ms/step | 15.1 samples/s +Step 25250/44000 | Loss: 2.9764 | LR: 1.98e-04 | 1326.2ms/step | 15.1 samples/s +Validation at step 25250: Loss = 2.9414, Perplexity = 18.94 +Step 25260/44000 | Loss: 2.9421 | LR: 1.98e-04 | 3554.8ms/step | 5.6 samples/s +Step 25270/44000 | Loss: 2.9533 | LR: 1.98e-04 | 1326.1ms/step | 15.1 samples/s +Step 25280/44000 | Loss: 2.9444 | LR: 1.98e-04 | 1327.8ms/step | 15.1 samples/s +Step 25290/44000 | Loss: 2.9335 | LR: 1.97e-04 | 1326.0ms/step | 15.1 samples/s +Step 25300/44000 | Loss: 2.9692 | LR: 1.97e-04 | 1327.2ms/step | 15.1 samples/s +Step 25310/44000 | Loss: 2.9432 | LR: 1.97e-04 | 1328.6ms/step | 15.1 samples/s +Step 25320/44000 | Loss: 2.9332 | LR: 1.97e-04 | 1328.0ms/step | 15.1 samples/s +Step 25330/44000 | Loss: 2.9371 | LR: 1.97e-04 | 1328.4ms/step | 15.1 samples/s +Step 25340/44000 | Loss: 2.9205 | LR: 1.97e-04 | 1329.6ms/step | 15.0 samples/s +Step 25350/44000 | Loss: 2.9094 | LR: 1.96e-04 | 1327.7ms/step | 15.1 samples/s +Step 25360/44000 | Loss: 3.0043 | LR: 1.96e-04 | 1327.8ms/step | 15.1 samples/s +Step 25370/44000 | Loss: 2.9339 | LR: 1.96e-04 | 1326.4ms/step | 15.1 samples/s +Step 25380/44000 | Loss: 2.9474 | LR: 1.96e-04 | 1329.3ms/step | 15.0 samples/s +Step 25390/44000 | Loss: 2.9237 | LR: 1.96e-04 | 1329.6ms/step | 15.0 samples/s +Step 25400/44000 | Loss: 2.9904 | LR: 1.96e-04 | 1328.8ms/step | 15.1 samples/s +Step 25410/44000 | Loss: 2.9564 | LR: 1.95e-04 | 1327.1ms/step | 15.1 samples/s +Step 25420/44000 | Loss: 2.9348 | LR: 1.95e-04 | 1327.9ms/step | 15.1 samples/s +Step 25430/44000 | Loss: 2.9469 | LR: 1.95e-04 | 1325.9ms/step | 15.1 samples/s +Step 25440/44000 | Loss: 2.9304 | LR: 1.95e-04 | 1327.6ms/step | 15.1 samples/s +Step 25450/44000 | Loss: 2.9443 | LR: 1.95e-04 | 1325.8ms/step | 15.1 samples/s +Step 25460/44000 | Loss: 2.9329 | LR: 1.94e-04 | 1324.4ms/step | 15.1 samples/s +Step 25470/44000 | Loss: 2.9353 | LR: 1.94e-04 | 1326.1ms/step | 15.1 samples/s +Step 25480/44000 | Loss: 2.9301 | LR: 1.94e-04 | 1326.0ms/step | 15.1 samples/s +Step 25490/44000 | Loss: 2.9161 | LR: 1.94e-04 | 1323.6ms/step | 15.1 samples/s +Step 25500/44000 | Loss: 2.9518 | LR: 1.94e-04 | 1325.1ms/step | 15.1 samples/s +Validation at step 25500: Loss = 2.9399, Perplexity = 18.91 +Step 25510/44000 | Loss: 2.9444 | LR: 1.94e-04 | 3546.9ms/step | 5.6 samples/s +Step 25520/44000 | Loss: 2.9643 | LR: 1.93e-04 | 1330.8ms/step | 15.0 samples/s +Step 25530/44000 | Loss: 2.9468 | LR: 1.93e-04 | 1328.8ms/step | 15.1 samples/s +Step 25540/44000 | Loss: 2.9327 | LR: 1.93e-04 | 1326.1ms/step | 15.1 samples/s +Step 25550/44000 | Loss: 2.9471 | LR: 1.93e-04 | 1326.4ms/step | 15.1 samples/s +Step 25560/44000 | Loss: 2.9532 | LR: 1.93e-04 | 1322.3ms/step | 15.1 samples/s +Step 25570/44000 | Loss: 2.9589 | LR: 1.92e-04 | 1325.7ms/step | 15.1 samples/s +Step 25580/44000 | Loss: 2.9325 | LR: 1.92e-04 | 1329.1ms/step | 15.0 samples/s +Step 25590/44000 | Loss: 2.9180 | LR: 1.92e-04 | 1330.4ms/step | 15.0 samples/s +Step 25600/44000 | Loss: 2.9524 | LR: 1.92e-04 | 1327.7ms/step | 15.1 samples/s +Step 25610/44000 | Loss: 2.9496 | LR: 1.92e-04 | 1328.7ms/step | 15.1 samples/s +Step 25620/44000 | Loss: 2.9404 | LR: 1.92e-04 | 1330.7ms/step | 15.0 samples/s +Step 25630/44000 | Loss: 2.9453 | LR: 1.91e-04 | 1334.0ms/step | 15.0 samples/s +Step 25640/44000 | Loss: 2.9417 | LR: 1.91e-04 | 1330.9ms/step | 15.0 samples/s +Step 25650/44000 | Loss: 2.9643 | LR: 1.91e-04 | 1332.6ms/step | 15.0 samples/s +Step 25660/44000 | Loss: 2.9182 | LR: 1.91e-04 | 1330.1ms/step | 15.0 samples/s +Step 25670/44000 | Loss: 2.9314 | LR: 1.91e-04 | 1329.4ms/step | 15.0 samples/s +Step 25680/44000 | Loss: 2.9460 | LR: 1.91e-04 | 1330.0ms/step | 15.0 samples/s +Step 25690/44000 | Loss: 2.9494 | LR: 1.90e-04 | 1332.6ms/step | 15.0 samples/s +Step 25700/44000 | Loss: 2.9168 | LR: 1.90e-04 | 1333.3ms/step | 15.0 samples/s +Step 25710/44000 | Loss: 2.9761 | LR: 1.90e-04 | 1329.2ms/step | 15.0 samples/s +Step 25720/44000 | Loss: 2.9317 | LR: 1.90e-04 | 1328.7ms/step | 15.1 samples/s +Step 25730/44000 | Loss: 2.9491 | LR: 1.90e-04 | 1330.2ms/step | 15.0 samples/s +Step 25740/44000 | Loss: 2.9452 | LR: 1.89e-04 | 1327.6ms/step | 15.1 samples/s +Step 25750/44000 | Loss: 2.9377 | LR: 1.89e-04 | 1330.6ms/step | 15.0 samples/s +Validation at step 25750: Loss = 2.9381, Perplexity = 18.88 +Step 25760/44000 | Loss: 2.9478 | LR: 1.89e-04 | 3543.6ms/step | 5.6 samples/s +Step 25770/44000 | Loss: 2.9483 | LR: 1.89e-04 | 1329.2ms/step | 15.0 samples/s +Step 25780/44000 | Loss: 2.9477 | LR: 1.89e-04 | 1328.8ms/step | 15.1 samples/s +Step 25790/44000 | Loss: 2.9474 | LR: 1.89e-04 | 1330.8ms/step | 15.0 samples/s +Step 25800/44000 | Loss: 2.9441 | LR: 1.88e-04 | 1329.2ms/step | 15.0 samples/s +Step 25810/44000 | Loss: 2.9720 | LR: 1.88e-04 | 1329.6ms/step | 15.0 samples/s +Step 25820/44000 | Loss: 2.9395 | LR: 1.88e-04 | 1331.8ms/step | 15.0 samples/s +Step 25830/44000 | Loss: 2.9280 | LR: 1.88e-04 | 1332.2ms/step | 15.0 samples/s +Step 25840/44000 | Loss: 2.9848 | LR: 1.88e-04 | 1332.9ms/step | 15.0 samples/s +Step 25850/44000 | Loss: 2.9571 | LR: 1.88e-04 | 1330.8ms/step | 15.0 samples/s +Step 25860/44000 | Loss: 2.9381 | LR: 1.87e-04 | 1326.9ms/step | 15.1 samples/s +Step 25870/44000 | Loss: 2.9287 | LR: 1.87e-04 | 1328.5ms/step | 15.1 samples/s +Step 25880/44000 | Loss: 2.9661 | LR: 1.87e-04 | 1327.9ms/step | 15.1 samples/s +Step 25890/44000 | Loss: 2.9381 | LR: 1.87e-04 | 1330.0ms/step | 15.0 samples/s +Step 25900/44000 | Loss: 2.9801 | LR: 1.87e-04 | 1328.7ms/step | 15.1 samples/s +Step 25910/44000 | Loss: 2.9515 | LR: 1.87e-04 | 1329.6ms/step | 15.0 samples/s +Step 25920/44000 | Loss: 2.9647 | LR: 1.86e-04 | 1329.5ms/step | 15.0 samples/s +Step 25930/44000 | Loss: 2.9123 | LR: 1.86e-04 | 1326.8ms/step | 15.1 samples/s +Step 25940/44000 | Loss: 2.9243 | LR: 1.86e-04 | 1325.0ms/step | 15.1 samples/s +Step 25950/44000 | Loss: 2.9618 | LR: 1.86e-04 | 1324.7ms/step | 15.1 samples/s +Step 25960/44000 | Loss: 2.9608 | LR: 1.86e-04 | 1327.3ms/step | 15.1 samples/s +Step 25970/44000 | Loss: 2.9353 | LR: 1.85e-04 | 1327.1ms/step | 15.1 samples/s +Step 25980/44000 | Loss: 2.9179 | LR: 1.85e-04 | 1326.8ms/step | 15.1 samples/s +Step 25990/44000 | Loss: 2.9345 | LR: 1.85e-04 | 1327.0ms/step | 15.1 samples/s +Step 26000/44000 | Loss: 2.9562 | LR: 1.85e-04 | 1327.7ms/step | 15.1 samples/s +Validation at step 26000: Loss = 2.9366, Perplexity = 18.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-26000 +Step 26010/44000 | Loss: 2.9483 | LR: 1.85e-04 | 3839.0ms/step | 5.2 samples/s +Step 26020/44000 | Loss: 2.9553 | LR: 1.85e-04 | 1323.8ms/step | 15.1 samples/s +Step 26030/44000 | Loss: 2.9841 | LR: 1.84e-04 | 1321.9ms/step | 15.1 samples/s +Step 26040/44000 | Loss: 2.9500 | LR: 1.84e-04 | 1323.4ms/step | 15.1 samples/s +Step 26050/44000 | Loss: 2.9433 | LR: 1.84e-04 | 1323.9ms/step | 15.1 samples/s +Step 26060/44000 | Loss: 2.9793 | LR: 1.84e-04 | 1326.9ms/step | 15.1 samples/s +Step 26070/44000 | Loss: 2.9373 | LR: 1.84e-04 | 1321.9ms/step | 15.1 samples/s +Step 26080/44000 | Loss: 2.9243 | LR: 1.84e-04 | 1320.5ms/step | 15.1 samples/s +Step 26090/44000 | Loss: 2.9188 | LR: 1.83e-04 | 1321.5ms/step | 15.1 samples/s +Step 26100/44000 | Loss: 2.9394 | LR: 1.83e-04 | 1322.1ms/step | 15.1 samples/s +Step 26110/44000 | Loss: 2.9059 | LR: 1.83e-04 | 1323.8ms/step | 15.1 samples/s +Step 26120/44000 | Loss: 2.9325 | LR: 1.83e-04 | 1328.3ms/step | 15.1 samples/s +Step 26130/44000 | Loss: 2.9238 | LR: 1.83e-04 | 1329.1ms/step | 15.0 samples/s +Step 26140/44000 | Loss: 2.9455 | LR: 1.82e-04 | 1329.6ms/step | 15.0 samples/s +Step 26150/44000 | Loss: 2.9298 | LR: 1.82e-04 | 1332.5ms/step | 15.0 samples/s +Step 26160/44000 | Loss: 2.9404 | LR: 1.82e-04 | 1333.0ms/step | 15.0 samples/s +Step 26170/44000 | Loss: 2.9464 | LR: 1.82e-04 | 1331.1ms/step | 15.0 samples/s +Step 26180/44000 | Loss: 2.9454 | LR: 1.82e-04 | 1330.0ms/step | 15.0 samples/s +Step 26190/44000 | Loss: 2.9732 | LR: 1.82e-04 | 1328.3ms/step | 15.1 samples/s +Step 26200/44000 | Loss: 2.9298 | LR: 1.81e-04 | 1328.2ms/step | 15.1 samples/s +Step 26210/44000 | Loss: 2.9832 | LR: 1.81e-04 | 1324.6ms/step | 15.1 samples/s +Step 26220/44000 | Loss: 2.9719 | LR: 1.81e-04 | 1323.6ms/step | 15.1 samples/s +Step 26230/44000 | Loss: 2.9736 | LR: 1.81e-04 | 1325.4ms/step | 15.1 samples/s +Step 26240/44000 | Loss: 2.9194 | LR: 1.81e-04 | 1324.6ms/step | 15.1 samples/s +Step 26250/44000 | Loss: 2.9709 | LR: 1.81e-04 | 1326.9ms/step | 15.1 samples/s +Validation at step 26250: Loss = 2.9354, Perplexity = 18.83 +Step 26260/44000 | Loss: 2.9277 | LR: 1.80e-04 | 3546.1ms/step | 5.6 samples/s +Step 26270/44000 | Loss: 2.9581 | LR: 1.80e-04 | 1328.2ms/step | 15.1 samples/s +Step 26280/44000 | Loss: 2.9443 | LR: 1.80e-04 | 1326.7ms/step | 15.1 samples/s +Step 26290/44000 | Loss: 2.9370 | LR: 1.80e-04 | 1323.9ms/step | 15.1 samples/s +Step 26300/44000 | Loss: 2.9475 | LR: 1.80e-04 | 1324.2ms/step | 15.1 samples/s +Step 26310/44000 | Loss: 2.9684 | LR: 1.80e-04 | 1328.0ms/step | 15.1 samples/s +Step 26320/44000 | Loss: 2.9576 | LR: 1.79e-04 | 1329.5ms/step | 15.0 samples/s +Step 26330/44000 | Loss: 2.9209 | LR: 1.79e-04 | 1328.2ms/step | 15.1 samples/s +Step 26340/44000 | Loss: 2.9663 | LR: 1.79e-04 | 1328.2ms/step | 15.1 samples/s +Step 26350/44000 | Loss: 2.9173 | LR: 1.79e-04 | 1328.9ms/step | 15.1 samples/s +Step 26360/44000 | Loss: 2.9373 | LR: 1.79e-04 | 1330.2ms/step | 15.0 samples/s +Step 26370/44000 | Loss: 2.9748 | LR: 1.78e-04 | 1329.0ms/step | 15.0 samples/s +Step 26380/44000 | Loss: 2.9688 | LR: 1.78e-04 | 1329.1ms/step | 15.0 samples/s +Step 26390/44000 | Loss: 2.9508 | LR: 1.78e-04 | 1323.6ms/step | 15.1 samples/s +Step 26400/44000 | Loss: 2.9353 | LR: 1.78e-04 | 1323.7ms/step | 15.1 samples/s +Step 26410/44000 | Loss: 2.9413 | LR: 1.78e-04 | 1329.0ms/step | 15.0 samples/s +Step 26420/44000 | Loss: 2.9689 | LR: 1.78e-04 | 1328.2ms/step | 15.1 samples/s +Step 26430/44000 | Loss: 2.9301 | LR: 1.77e-04 | 1328.3ms/step | 15.1 samples/s +Step 26440/44000 | Loss: 2.9605 | LR: 1.77e-04 | 1326.1ms/step | 15.1 samples/s +Step 26450/44000 | Loss: 2.9341 | LR: 1.77e-04 | 1326.7ms/step | 15.1 samples/s +Step 26460/44000 | Loss: 2.9304 | LR: 1.77e-04 | 1326.2ms/step | 15.1 samples/s +Step 26470/44000 | Loss: 2.9642 | LR: 1.77e-04 | 1328.1ms/step | 15.1 samples/s +Step 26480/44000 | Loss: 2.9403 | LR: 1.77e-04 | 1328.9ms/step | 15.1 samples/s +Step 26490/44000 | Loss: 2.9340 | LR: 1.76e-04 | 1328.2ms/step | 15.1 samples/s +Step 26500/44000 | Loss: 2.9431 | LR: 1.76e-04 | 1329.9ms/step | 15.0 samples/s +Validation at step 26500: Loss = 2.9339, Perplexity = 18.80 +Step 26510/44000 | Loss: 2.9522 | LR: 1.76e-04 | 3544.0ms/step | 5.6 samples/s +Step 26520/44000 | Loss: 2.9155 | LR: 1.76e-04 | 1327.9ms/step | 15.1 samples/s +Step 26530/44000 | Loss: 2.9417 | LR: 1.76e-04 | 1330.8ms/step | 15.0 samples/s +Step 26540/44000 | Loss: 2.9237 | LR: 1.76e-04 | 1330.7ms/step | 15.0 samples/s +Step 26550/44000 | Loss: 2.9181 | LR: 1.75e-04 | 1330.6ms/step | 15.0 samples/s +Step 26560/44000 | Loss: 2.9482 | LR: 1.75e-04 | 1331.5ms/step | 15.0 samples/s +Step 26570/44000 | Loss: 2.9138 | LR: 1.75e-04 | 1329.2ms/step | 15.0 samples/s +Step 26580/44000 | Loss: 2.9273 | LR: 1.75e-04 | 1328.6ms/step | 15.1 samples/s +Step 26590/44000 | Loss: 2.9609 | LR: 1.75e-04 | 1329.9ms/step | 15.0 samples/s +Step 26600/44000 | Loss: 2.9265 | LR: 1.74e-04 | 1327.7ms/step | 15.1 samples/s +Step 26610/44000 | Loss: 2.9477 | LR: 1.74e-04 | 1332.5ms/step | 15.0 samples/s +Step 26620/44000 | Loss: 2.9407 | LR: 1.74e-04 | 1330.7ms/step | 15.0 samples/s +Step 26630/44000 | Loss: 2.9314 | LR: 1.74e-04 | 1331.5ms/step | 15.0 samples/s +Step 26640/44000 | Loss: 2.9348 | LR: 1.74e-04 | 1333.2ms/step | 15.0 samples/s +Step 26650/44000 | Loss: 2.9359 | LR: 1.74e-04 | 1330.4ms/step | 15.0 samples/s +Step 26660/44000 | Loss: 2.9246 | LR: 1.73e-04 | 1326.3ms/step | 15.1 samples/s +Step 26670/44000 | Loss: 2.9661 | LR: 1.73e-04 | 1325.4ms/step | 15.1 samples/s +Step 26680/44000 | Loss: 2.9233 | LR: 1.73e-04 | 1331.1ms/step | 15.0 samples/s +Step 26690/44000 | Loss: 2.9565 | LR: 1.73e-04 | 1329.9ms/step | 15.0 samples/s +Step 26700/44000 | Loss: 2.9330 | LR: 1.73e-04 | 1332.1ms/step | 15.0 samples/s +Step 26710/44000 | Loss: 2.9312 | LR: 1.73e-04 | 1333.3ms/step | 15.0 samples/s +Step 26720/44000 | Loss: 2.9131 | LR: 1.72e-04 | 1331.9ms/step | 15.0 samples/s +Step 26730/44000 | Loss: 2.9414 | LR: 1.72e-04 | 1331.7ms/step | 15.0 samples/s +Step 26740/44000 | Loss: 2.9484 | LR: 1.72e-04 | 1333.9ms/step | 15.0 samples/s +Step 26750/44000 | Loss: 2.9327 | LR: 1.72e-04 | 1332.5ms/step | 15.0 samples/s +Validation at step 26750: Loss = 2.9322, Perplexity = 18.77 +Step 26760/44000 | Loss: 2.9167 | LR: 1.72e-04 | 3553.5ms/step | 5.6 samples/s +Step 26770/44000 | Loss: 2.9493 | LR: 1.72e-04 | 1331.6ms/step | 15.0 samples/s +Step 26780/44000 | Loss: 2.9499 | LR: 1.71e-04 | 1330.0ms/step | 15.0 samples/s +Step 26790/44000 | Loss: 2.9294 | LR: 1.71e-04 | 1327.5ms/step | 15.1 samples/s +Step 26800/44000 | Loss: 2.9838 | LR: 1.71e-04 | 1328.9ms/step | 15.0 samples/s +Step 26810/44000 | Loss: 2.9508 | LR: 1.71e-04 | 1328.9ms/step | 15.1 samples/s +Step 26820/44000 | Loss: 2.9108 | LR: 1.71e-04 | 1334.9ms/step | 15.0 samples/s +Step 26830/44000 | Loss: 2.9356 | LR: 1.71e-04 | 1332.5ms/step | 15.0 samples/s +Step 26840/44000 | Loss: 2.9350 | LR: 1.70e-04 | 1329.8ms/step | 15.0 samples/s +Step 26850/44000 | Loss: 2.9439 | LR: 1.70e-04 | 1334.0ms/step | 15.0 samples/s +Step 26860/44000 | Loss: 2.9592 | LR: 1.70e-04 | 1358.4ms/step | 14.7 samples/s +Step 26870/44000 | Loss: 2.9330 | LR: 1.70e-04 | 1347.3ms/step | 14.8 samples/s +Step 26880/44000 | Loss: 2.9539 | LR: 1.70e-04 | 1344.8ms/step | 14.9 samples/s +Step 26890/44000 | Loss: 2.9738 | LR: 1.69e-04 | 1343.6ms/step | 14.9 samples/s +Step 26900/44000 | Loss: 2.9153 | LR: 1.69e-04 | 1343.1ms/step | 14.9 samples/s +Step 26910/44000 | Loss: 2.9258 | LR: 1.69e-04 | 1342.1ms/step | 14.9 samples/s +Step 26920/44000 | Loss: 2.9330 | LR: 1.69e-04 | 1339.6ms/step | 14.9 samples/s +Step 26930/44000 | Loss: 2.9630 | LR: 1.69e-04 | 1341.4ms/step | 14.9 samples/s +Step 26940/44000 | Loss: 2.9476 | LR: 1.69e-04 | 1342.3ms/step | 14.9 samples/s +Step 26950/44000 | Loss: 2.9626 | LR: 1.68e-04 | 1340.2ms/step | 14.9 samples/s +Step 26960/44000 | Loss: 2.9178 | LR: 1.68e-04 | 1340.4ms/step | 14.9 samples/s +Step 26970/44000 | Loss: 2.9358 | LR: 1.68e-04 | 1342.4ms/step | 14.9 samples/s +Step 26980/44000 | Loss: 2.9399 | LR: 1.68e-04 | 1339.4ms/step | 14.9 samples/s +Step 26990/44000 | Loss: 2.9383 | LR: 1.68e-04 | 1333.8ms/step | 15.0 samples/s +Step 27000/44000 | Loss: 2.9423 | LR: 1.68e-04 | 1335.2ms/step | 15.0 samples/s +Validation at step 27000: Loss = 2.9309, Perplexity = 18.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-27000 +Step 27010/44000 | Loss: 2.9111 | LR: 1.67e-04 | 3891.1ms/step | 5.1 samples/s +Step 27020/44000 | Loss: 2.9339 | LR: 1.67e-04 | 1339.9ms/step | 14.9 samples/s +Step 27030/44000 | Loss: 2.9336 | LR: 1.67e-04 | 1337.2ms/step | 15.0 samples/s +Step 27040/44000 | Loss: 2.9480 | LR: 1.67e-04 | 1337.4ms/step | 15.0 samples/s +Step 27050/44000 | Loss: 2.9581 | LR: 1.67e-04 | 1338.0ms/step | 14.9 samples/s +Step 27060/44000 | Loss: 2.9553 | LR: 1.67e-04 | 1336.7ms/step | 15.0 samples/s +Step 27070/44000 | Loss: 2.9670 | LR: 1.66e-04 | 1329.0ms/step | 15.0 samples/s +Step 27080/44000 | Loss: 2.9368 | LR: 1.66e-04 | 1329.6ms/step | 15.0 samples/s +Step 27090/44000 | Loss: 2.9512 | LR: 1.66e-04 | 1335.0ms/step | 15.0 samples/s +Step 27100/44000 | Loss: 2.9440 | LR: 1.66e-04 | 1342.6ms/step | 14.9 samples/s +Step 27110/44000 | Loss: 2.9395 | LR: 1.66e-04 | 1343.6ms/step | 14.9 samples/s +Step 27120/44000 | Loss: 2.9434 | LR: 1.66e-04 | 1341.7ms/step | 14.9 samples/s +Step 27130/44000 | Loss: 2.9699 | LR: 1.65e-04 | 1337.9ms/step | 14.9 samples/s +Step 27140/44000 | Loss: 2.9462 | LR: 1.65e-04 | 1335.9ms/step | 15.0 samples/s +Step 27150/44000 | Loss: 2.9323 | LR: 1.65e-04 | 1337.3ms/step | 15.0 samples/s +Step 27160/44000 | Loss: 2.9346 | LR: 1.65e-04 | 1340.2ms/step | 14.9 samples/s +Step 27170/44000 | Loss: 2.9398 | LR: 1.65e-04 | 1340.6ms/step | 14.9 samples/s +Step 27180/44000 | Loss: 2.9414 | LR: 1.65e-04 | 1335.5ms/step | 15.0 samples/s +Step 27190/44000 | Loss: 2.9447 | LR: 1.64e-04 | 1338.8ms/step | 14.9 samples/s +Step 27200/44000 | Loss: 2.9604 | LR: 1.64e-04 | 1340.9ms/step | 14.9 samples/s +Step 27210/44000 | Loss: 2.9133 | LR: 1.64e-04 | 1336.6ms/step | 15.0 samples/s +Step 27220/44000 | Loss: 2.9406 | LR: 1.64e-04 | 1339.5ms/step | 14.9 samples/s +Step 27230/44000 | Loss: 2.9275 | LR: 1.64e-04 | 1340.2ms/step | 14.9 samples/s +Step 27240/44000 | Loss: 2.9609 | LR: 1.63e-04 | 1338.9ms/step | 14.9 samples/s +Step 27250/44000 | Loss: 2.9507 | LR: 1.63e-04 | 1339.9ms/step | 14.9 samples/s +Validation at step 27250: Loss = 2.9298, Perplexity = 18.72 +Step 27260/44000 | Loss: 2.9415 | LR: 1.63e-04 | 3571.3ms/step | 5.6 samples/s +Step 27270/44000 | Loss: 2.9520 | LR: 1.63e-04 | 1339.9ms/step | 14.9 samples/s +Step 27280/44000 | Loss: 2.9139 | LR: 1.63e-04 | 1337.4ms/step | 15.0 samples/s +Step 27290/44000 | Loss: 2.9433 | LR: 1.63e-04 | 1341.0ms/step | 14.9 samples/s +Step 27300/44000 | Loss: 2.9240 | LR: 1.62e-04 | 1337.9ms/step | 14.9 samples/s +Step 27310/44000 | Loss: 2.9355 | LR: 1.62e-04 | 1339.6ms/step | 14.9 samples/s +Step 27320/44000 | Loss: 2.9139 | LR: 1.62e-04 | 1341.3ms/step | 14.9 samples/s +Step 27330/44000 | Loss: 2.9402 | LR: 1.62e-04 | 1344.5ms/step | 14.9 samples/s +Step 27340/44000 | Loss: 2.9459 | LR: 1.62e-04 | 1344.8ms/step | 14.9 samples/s +Step 27350/44000 | Loss: 2.9132 | LR: 1.62e-04 | 1341.6ms/step | 14.9 samples/s +Step 27360/44000 | Loss: 2.9393 | LR: 1.61e-04 | 1344.0ms/step | 14.9 samples/s +Step 27370/44000 | Loss: 2.9466 | LR: 1.61e-04 | 1343.4ms/step | 14.9 samples/s +Step 27380/44000 | Loss: 2.9123 | LR: 1.61e-04 | 1345.2ms/step | 14.9 samples/s +Step 27390/44000 | Loss: 2.9390 | LR: 1.61e-04 | 1346.3ms/step | 14.9 samples/s +Step 27400/44000 | Loss: 2.9572 | LR: 1.61e-04 | 1344.3ms/step | 14.9 samples/s +Step 27410/44000 | Loss: 2.9592 | LR: 1.61e-04 | 1343.8ms/step | 14.9 samples/s +Step 27420/44000 | Loss: 2.9416 | LR: 1.60e-04 | 1346.3ms/step | 14.9 samples/s +Step 27430/44000 | Loss: 2.9537 | LR: 1.60e-04 | 1343.5ms/step | 14.9 samples/s +Step 27440/44000 | Loss: 2.9310 | LR: 1.60e-04 | 1345.6ms/step | 14.9 samples/s +Step 27450/44000 | Loss: 2.9588 | LR: 1.60e-04 | 1343.4ms/step | 14.9 samples/s +Step 27460/44000 | Loss: 2.9548 | LR: 1.60e-04 | 1343.1ms/step | 14.9 samples/s +Step 27470/44000 | Loss: 2.9370 | LR: 1.60e-04 | 1344.8ms/step | 14.9 samples/s +Step 27480/44000 | Loss: 2.9192 | LR: 1.59e-04 | 1344.8ms/step | 14.9 samples/s +Step 27490/44000 | Loss: 2.9502 | LR: 1.59e-04 | 1348.7ms/step | 14.8 samples/s +Step 27500/44000 | Loss: 2.9498 | LR: 1.59e-04 | 1343.6ms/step | 14.9 samples/s +Validation at step 27500: Loss = 2.9283, Perplexity = 18.70 +Step 27510/44000 | Loss: 2.9420 | LR: 1.59e-04 | 3582.7ms/step | 5.6 samples/s +Step 27520/44000 | Loss: 2.9526 | LR: 1.59e-04 | 1343.4ms/step | 14.9 samples/s +Step 27530/44000 | Loss: 2.9203 | LR: 1.59e-04 | 1344.3ms/step | 14.9 samples/s +Step 27540/44000 | Loss: 2.9337 | LR: 1.58e-04 | 1348.9ms/step | 14.8 samples/s +Step 27550/44000 | Loss: 2.9665 | LR: 1.58e-04 | 1350.0ms/step | 14.8 samples/s +Step 27560/44000 | Loss: 2.9184 | LR: 1.58e-04 | 1350.4ms/step | 14.8 samples/s +Step 27570/44000 | Loss: 2.9231 | LR: 1.58e-04 | 1347.2ms/step | 14.8 samples/s +Step 27580/44000 | Loss: 2.9462 | LR: 1.58e-04 | 1347.3ms/step | 14.8 samples/s +Step 27590/44000 | Loss: 2.9468 | LR: 1.58e-04 | 1347.1ms/step | 14.8 samples/s +Step 27600/44000 | Loss: 2.9567 | LR: 1.57e-04 | 1347.6ms/step | 14.8 samples/s +Step 27610/44000 | Loss: 2.9172 | LR: 1.57e-04 | 1350.8ms/step | 14.8 samples/s +Step 27620/44000 | Loss: 2.9150 | LR: 1.57e-04 | 1352.0ms/step | 14.8 samples/s +Step 27630/44000 | Loss: 2.9051 | LR: 1.57e-04 | 1349.9ms/step | 14.8 samples/s +Step 27640/44000 | Loss: 2.9095 | LR: 1.57e-04 | 1350.2ms/step | 14.8 samples/s +Step 27650/44000 | Loss: 2.9345 | LR: 1.57e-04 | 1345.5ms/step | 14.9 samples/s +Step 27660/44000 | Loss: 2.9275 | LR: 1.56e-04 | 1347.0ms/step | 14.8 samples/s +Step 27670/44000 | Loss: 2.9381 | LR: 1.56e-04 | 1349.7ms/step | 14.8 samples/s +Step 27680/44000 | Loss: 2.9419 | LR: 1.56e-04 | 1350.2ms/step | 14.8 samples/s +Step 27690/44000 | Loss: 2.9160 | LR: 1.56e-04 | 1347.0ms/step | 14.8 samples/s +Step 27700/44000 | Loss: 2.9455 | LR: 1.56e-04 | 1345.0ms/step | 14.9 samples/s +Step 27710/44000 | Loss: 2.9364 | LR: 1.56e-04 | 1346.3ms/step | 14.9 samples/s +Step 27720/44000 | Loss: 2.8889 | LR: 1.55e-04 | 1349.4ms/step | 14.8 samples/s +Step 27730/44000 | Loss: 2.9297 | LR: 1.55e-04 | 1348.7ms/step | 14.8 samples/s +Step 27740/44000 | Loss: 2.9061 | LR: 1.55e-04 | 1349.5ms/step | 14.8 samples/s +Step 27750/44000 | Loss: 2.9647 | LR: 1.55e-04 | 1349.2ms/step | 14.8 samples/s +Validation at step 27750: Loss = 2.9271, Perplexity = 18.67 +Step 27760/44000 | Loss: 2.9498 | LR: 1.55e-04 | 3588.0ms/step | 5.6 samples/s +Step 27770/44000 | Loss: 2.9389 | LR: 1.55e-04 | 1345.8ms/step | 14.9 samples/s +Step 27780/44000 | Loss: 2.9514 | LR: 1.54e-04 | 1346.0ms/step | 14.9 samples/s +Step 27790/44000 | Loss: 2.9200 | LR: 1.54e-04 | 1345.1ms/step | 14.9 samples/s +Step 27800/44000 | Loss: 2.9557 | LR: 1.54e-04 | 1345.1ms/step | 14.9 samples/s +Step 27810/44000 | Loss: 2.9430 | LR: 1.54e-04 | 1345.5ms/step | 14.9 samples/s +Step 27820/44000 | Loss: 2.9104 | LR: 1.54e-04 | 1347.1ms/step | 14.8 samples/s +Step 27830/44000 | Loss: 2.9413 | LR: 1.54e-04 | 1348.6ms/step | 14.8 samples/s +Step 27840/44000 | Loss: 2.9609 | LR: 1.53e-04 | 1359.4ms/step | 14.7 samples/s +Step 27850/44000 | Loss: 2.9218 | LR: 1.53e-04 | 1357.4ms/step | 14.7 samples/s +Step 27860/44000 | Loss: 2.9418 | LR: 1.53e-04 | 1350.8ms/step | 14.8 samples/s +Step 27870/44000 | Loss: 2.9459 | LR: 1.53e-04 | 1346.7ms/step | 14.9 samples/s +Step 27880/44000 | Loss: 2.9622 | LR: 1.53e-04 | 1342.2ms/step | 14.9 samples/s +Step 27890/44000 | Loss: 2.9354 | LR: 1.53e-04 | 1342.7ms/step | 14.9 samples/s +Step 27900/44000 | Loss: 2.9404 | LR: 1.52e-04 | 1343.0ms/step | 14.9 samples/s +Step 27910/44000 | Loss: 2.9111 | LR: 1.52e-04 | 1344.6ms/step | 14.9 samples/s +Step 27920/44000 | Loss: 2.9460 | LR: 1.52e-04 | 1345.8ms/step | 14.9 samples/s +Step 27930/44000 | Loss: 2.9686 | LR: 1.52e-04 | 1343.3ms/step | 14.9 samples/s +Step 27940/44000 | Loss: 2.9171 | LR: 1.52e-04 | 1345.9ms/step | 14.9 samples/s +Step 27950/44000 | Loss: 2.9476 | LR: 1.52e-04 | 1347.3ms/step | 14.8 samples/s +Step 27960/44000 | Loss: 2.9294 | LR: 1.51e-04 | 1348.0ms/step | 14.8 samples/s +Step 27970/44000 | Loss: 2.9205 | LR: 1.51e-04 | 1339.8ms/step | 14.9 samples/s +Step 27980/44000 | Loss: 2.9600 | LR: 1.51e-04 | 1342.0ms/step | 14.9 samples/s +Step 27990/44000 | Loss: 2.9789 | LR: 1.51e-04 | 1341.2ms/step | 14.9 samples/s +Step 28000/44000 | Loss: 2.9470 | LR: 1.51e-04 | 1346.8ms/step | 14.8 samples/s +Validation at step 28000: Loss = 2.9258, Perplexity = 18.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-28000 +Step 28010/44000 | Loss: 2.9565 | LR: 1.51e-04 | 3891.3ms/step | 5.1 samples/s +Step 28020/44000 | Loss: 2.9520 | LR: 1.50e-04 | 1339.2ms/step | 14.9 samples/s +Step 28030/44000 | Loss: 2.9099 | LR: 1.50e-04 | 1339.8ms/step | 14.9 samples/s +Step 28040/44000 | Loss: 2.9343 | LR: 1.50e-04 | 1341.8ms/step | 14.9 samples/s +Step 28050/44000 | Loss: 2.9309 | LR: 1.50e-04 | 1342.3ms/step | 14.9 samples/s +Step 28060/44000 | Loss: 2.9583 | LR: 1.50e-04 | 1342.7ms/step | 14.9 samples/s +Step 28070/44000 | Loss: 2.9668 | LR: 1.50e-04 | 1342.9ms/step | 14.9 samples/s +Step 28080/44000 | Loss: 2.9169 | LR: 1.49e-04 | 1345.0ms/step | 14.9 samples/s +Step 28090/44000 | Loss: 2.9444 | LR: 1.49e-04 | 1344.9ms/step | 14.9 samples/s +Step 28100/44000 | Loss: 2.9513 | LR: 1.49e-04 | 1345.1ms/step | 14.9 samples/s +Step 28110/44000 | Loss: 2.9312 | LR: 1.49e-04 | 1341.9ms/step | 14.9 samples/s +Step 28120/44000 | Loss: 2.9169 | LR: 1.49e-04 | 1340.6ms/step | 14.9 samples/s +Step 28130/44000 | Loss: 2.9057 | LR: 1.49e-04 | 1342.8ms/step | 14.9 samples/s +Step 28140/44000 | Loss: 2.9242 | LR: 1.48e-04 | 1344.0ms/step | 14.9 samples/s +Step 28150/44000 | Loss: 2.9208 | LR: 1.48e-04 | 1343.6ms/step | 14.9 samples/s +Step 28160/44000 | Loss: 2.9362 | LR: 1.48e-04 | 1344.8ms/step | 14.9 samples/s +Step 28170/44000 | Loss: 2.9240 | LR: 1.48e-04 | 1344.7ms/step | 14.9 samples/s +Step 28180/44000 | Loss: 2.9410 | LR: 1.48e-04 | 1347.1ms/step | 14.8 samples/s +Step 28190/44000 | Loss: 2.9377 | LR: 1.48e-04 | 1343.5ms/step | 14.9 samples/s +Step 28200/44000 | Loss: 2.9191 | LR: 1.47e-04 | 1340.5ms/step | 14.9 samples/s +Step 28210/44000 | Loss: 2.9549 | LR: 1.47e-04 | 1340.3ms/step | 14.9 samples/s +Step 28220/44000 | Loss: 2.9820 | LR: 1.47e-04 | 1342.3ms/step | 14.9 samples/s +Step 28230/44000 | Loss: 2.9213 | LR: 1.47e-04 | 1343.2ms/step | 14.9 samples/s +Step 28240/44000 | Loss: 2.9324 | LR: 1.47e-04 | 1342.8ms/step | 14.9 samples/s +Step 28250/44000 | Loss: 2.9420 | LR: 1.47e-04 | 1344.5ms/step | 14.9 samples/s +Validation at step 28250: Loss = 2.9249, Perplexity = 18.63 +Step 28260/44000 | Loss: 2.9446 | LR: 1.46e-04 | 3580.0ms/step | 5.6 samples/s +Step 28270/44000 | Loss: 2.9448 | LR: 1.46e-04 | 1342.4ms/step | 14.9 samples/s +Step 28280/44000 | Loss: 2.9011 | LR: 1.46e-04 | 1341.8ms/step | 14.9 samples/s +Step 28290/44000 | Loss: 2.9112 | LR: 1.46e-04 | 1345.2ms/step | 14.9 samples/s +Step 28300/44000 | Loss: 2.9293 | LR: 1.46e-04 | 1339.3ms/step | 14.9 samples/s +Step 28310/44000 | Loss: 2.9383 | LR: 1.46e-04 | 1340.4ms/step | 14.9 samples/s +Step 28320/44000 | Loss: 2.9273 | LR: 1.45e-04 | 1338.4ms/step | 14.9 samples/s +Step 28330/44000 | Loss: 2.9475 | LR: 1.45e-04 | 1338.3ms/step | 14.9 samples/s +Step 28340/44000 | Loss: 2.9455 | LR: 1.45e-04 | 1339.7ms/step | 14.9 samples/s +Step 28350/44000 | Loss: 2.9121 | LR: 1.45e-04 | 1340.7ms/step | 14.9 samples/s +Step 28360/44000 | Loss: 2.9363 | LR: 1.45e-04 | 1339.4ms/step | 14.9 samples/s +Step 28370/44000 | Loss: 2.9114 | LR: 1.45e-04 | 1344.5ms/step | 14.9 samples/s +Step 28380/44000 | Loss: 2.9031 | LR: 1.44e-04 | 1344.9ms/step | 14.9 samples/s +Step 28390/44000 | Loss: 2.9190 | LR: 1.44e-04 | 1344.7ms/step | 14.9 samples/s +Step 28400/44000 | Loss: 2.9343 | LR: 1.44e-04 | 1345.2ms/step | 14.9 samples/s +Step 28410/44000 | Loss: 2.9433 | LR: 1.44e-04 | 1346.2ms/step | 14.9 samples/s +Step 28420/44000 | Loss: 2.9206 | LR: 1.44e-04 | 1347.4ms/step | 14.8 samples/s +Step 28430/44000 | Loss: 2.9143 | LR: 1.44e-04 | 1336.8ms/step | 15.0 samples/s +Step 28440/44000 | Loss: 2.9516 | LR: 1.43e-04 | 1331.3ms/step | 15.0 samples/s +Step 28450/44000 | Loss: 2.9380 | LR: 1.43e-04 | 1339.2ms/step | 14.9 samples/s +Step 28460/44000 | Loss: 2.9314 | LR: 1.43e-04 | 1340.2ms/step | 14.9 samples/s +Step 28470/44000 | Loss: 2.9267 | LR: 1.43e-04 | 1339.6ms/step | 14.9 samples/s +Step 28480/44000 | Loss: 2.9680 | LR: 1.43e-04 | 1343.6ms/step | 14.9 samples/s +Step 28490/44000 | Loss: 2.9246 | LR: 1.43e-04 | 1346.1ms/step | 14.9 samples/s +Step 28500/44000 | Loss: 2.9419 | LR: 1.42e-04 | 1345.2ms/step | 14.9 samples/s +Validation at step 28500: Loss = 2.9235, Perplexity = 18.61 +Step 28510/44000 | Loss: 2.9218 | LR: 1.42e-04 | 3597.7ms/step | 5.6 samples/s +Step 28520/44000 | Loss: 2.9420 | LR: 1.42e-04 | 1343.8ms/step | 14.9 samples/s +Step 28530/44000 | Loss: 2.9098 | LR: 1.42e-04 | 1346.1ms/step | 14.9 samples/s +Step 28540/44000 | Loss: 2.9463 | LR: 1.42e-04 | 1345.3ms/step | 14.9 samples/s +Step 28550/44000 | Loss: 2.9289 | LR: 1.42e-04 | 1346.0ms/step | 14.9 samples/s +Step 28560/44000 | Loss: 2.9441 | LR: 1.41e-04 | 1346.2ms/step | 14.9 samples/s +Step 28570/44000 | Loss: 2.9054 | LR: 1.41e-04 | 1345.4ms/step | 14.9 samples/s +Step 28580/44000 | Loss: 2.9500 | LR: 1.41e-04 | 1349.2ms/step | 14.8 samples/s +Step 28590/44000 | Loss: 2.9416 | LR: 1.41e-04 | 1345.2ms/step | 14.9 samples/s +Step 28600/44000 | Loss: 2.9425 | LR: 1.41e-04 | 1345.3ms/step | 14.9 samples/s +Step 28610/44000 | Loss: 2.9324 | LR: 1.41e-04 | 1345.0ms/step | 14.9 samples/s +Step 28620/44000 | Loss: 2.8981 | LR: 1.40e-04 | 1346.8ms/step | 14.9 samples/s +Step 28630/44000 | Loss: 2.9085 | LR: 1.40e-04 | 1345.6ms/step | 14.9 samples/s +Step 28640/44000 | Loss: 2.9212 | LR: 1.40e-04 | 1346.0ms/step | 14.9 samples/s +Step 28650/44000 | Loss: 2.9465 | LR: 1.40e-04 | 1346.0ms/step | 14.9 samples/s +Step 28660/44000 | Loss: 2.9166 | LR: 1.40e-04 | 1344.9ms/step | 14.9 samples/s +Step 28670/44000 | Loss: 2.9347 | LR: 1.40e-04 | 1344.8ms/step | 14.9 samples/s +Step 28680/44000 | Loss: 2.9380 | LR: 1.39e-04 | 1345.1ms/step | 14.9 samples/s +Step 28690/44000 | Loss: 2.9176 | LR: 1.39e-04 | 1343.1ms/step | 14.9 samples/s +Step 28700/44000 | Loss: 2.9510 | LR: 1.39e-04 | 1341.3ms/step | 14.9 samples/s +Step 28710/44000 | Loss: 2.9402 | LR: 1.39e-04 | 1341.3ms/step | 14.9 samples/s +Step 28720/44000 | Loss: 2.8948 | LR: 1.39e-04 | 1342.7ms/step | 14.9 samples/s +Step 28730/44000 | Loss: 2.9127 | LR: 1.39e-04 | 1345.0ms/step | 14.9 samples/s +Step 28740/44000 | Loss: 2.9255 | LR: 1.38e-04 | 1350.4ms/step | 14.8 samples/s +Step 28750/44000 | Loss: 2.9100 | LR: 1.38e-04 | 1353.5ms/step | 14.8 samples/s +Validation at step 28750: Loss = 2.9224, Perplexity = 18.59 +Step 28760/44000 | Loss: 2.9340 | LR: 1.38e-04 | 3589.7ms/step | 5.6 samples/s +Step 28770/44000 | Loss: 2.9206 | LR: 1.38e-04 | 1347.3ms/step | 14.8 samples/s +Step 28780/44000 | Loss: 2.9166 | LR: 1.38e-04 | 1345.5ms/step | 14.9 samples/s +Step 28790/44000 | Loss: 2.9381 | LR: 1.38e-04 | 1348.4ms/step | 14.8 samples/s +Step 28800/44000 | Loss: 2.9445 | LR: 1.38e-04 | 1342.8ms/step | 14.9 samples/s +Step 28810/44000 | Loss: 2.9362 | LR: 1.37e-04 | 1344.9ms/step | 14.9 samples/s +Step 28820/44000 | Loss: 2.9354 | LR: 1.37e-04 | 1343.0ms/step | 14.9 samples/s +Step 28830/44000 | Loss: 2.9295 | LR: 1.37e-04 | 1343.9ms/step | 14.9 samples/s +Step 28840/44000 | Loss: 2.9373 | LR: 1.37e-04 | 1345.2ms/step | 14.9 samples/s +Step 28850/44000 | Loss: 2.9122 | LR: 1.37e-04 | 1341.2ms/step | 14.9 samples/s +Step 28860/44000 | Loss: 2.9268 | LR: 1.37e-04 | 1341.5ms/step | 14.9 samples/s +Step 28870/44000 | Loss: 2.9695 | LR: 1.36e-04 | 1342.5ms/step | 14.9 samples/s +Step 28880/44000 | Loss: 2.8971 | LR: 1.36e-04 | 1343.1ms/step | 14.9 samples/s +Step 28890/44000 | Loss: 2.9211 | LR: 1.36e-04 | 1342.9ms/step | 14.9 samples/s +Step 28900/44000 | Loss: 2.9487 | LR: 1.36e-04 | 1340.7ms/step | 14.9 samples/s +Step 28910/44000 | Loss: 2.9197 | LR: 1.36e-04 | 1341.0ms/step | 14.9 samples/s +Step 28920/44000 | Loss: 2.9382 | LR: 1.36e-04 | 1338.5ms/step | 14.9 samples/s +Step 28930/44000 | Loss: 2.9604 | LR: 1.35e-04 | 1340.8ms/step | 14.9 samples/s +Step 28940/44000 | Loss: 2.9173 | LR: 1.35e-04 | 1339.8ms/step | 14.9 samples/s +Step 28950/44000 | Loss: 2.9267 | LR: 1.35e-04 | 1339.5ms/step | 14.9 samples/s +Step 28960/44000 | Loss: 2.9148 | LR: 1.35e-04 | 1340.0ms/step | 14.9 samples/s +Step 28970/44000 | Loss: 2.9599 | LR: 1.35e-04 | 1340.0ms/step | 14.9 samples/s +Step 28980/44000 | Loss: 2.9122 | LR: 1.35e-04 | 1343.6ms/step | 14.9 samples/s +Step 28990/44000 | Loss: 2.9303 | LR: 1.34e-04 | 1344.8ms/step | 14.9 samples/s +Step 29000/44000 | Loss: 2.9179 | LR: 1.34e-04 | 1346.0ms/step | 14.9 samples/s +Validation at step 29000: Loss = 2.9215, Perplexity = 18.57 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-29000 +Step 29010/44000 | Loss: 2.9417 | LR: 1.34e-04 | 3875.8ms/step | 5.2 samples/s +Step 29020/44000 | Loss: 2.9066 | LR: 1.34e-04 | 1338.6ms/step | 14.9 samples/s +Step 29030/44000 | Loss: 2.9227 | LR: 1.34e-04 | 1339.2ms/step | 14.9 samples/s +Step 29040/44000 | Loss: 2.9166 | LR: 1.34e-04 | 1342.8ms/step | 14.9 samples/s +Step 29050/44000 | Loss: 2.9047 | LR: 1.33e-04 | 1342.4ms/step | 14.9 samples/s +Step 29060/44000 | Loss: 2.9326 | LR: 1.33e-04 | 1342.3ms/step | 14.9 samples/s +Step 29070/44000 | Loss: 2.8986 | LR: 1.33e-04 | 1345.5ms/step | 14.9 samples/s +Step 29080/44000 | Loss: 2.9559 | LR: 1.33e-04 | 1340.4ms/step | 14.9 samples/s +Step 29090/44000 | Loss: 2.9334 | LR: 1.33e-04 | 1340.5ms/step | 14.9 samples/s +Step 29100/44000 | Loss: 2.9481 | LR: 1.33e-04 | 1344.8ms/step | 14.9 samples/s +Step 29110/44000 | Loss: 2.9374 | LR: 1.33e-04 | 1348.0ms/step | 14.8 samples/s +Step 29120/44000 | Loss: 2.9309 | LR: 1.32e-04 | 1343.9ms/step | 14.9 samples/s +Step 29130/44000 | Loss: 2.9026 | LR: 1.32e-04 | 1342.8ms/step | 14.9 samples/s +Step 29140/44000 | Loss: 2.9122 | LR: 1.32e-04 | 1343.4ms/step | 14.9 samples/s +Step 29150/44000 | Loss: 2.9124 | LR: 1.32e-04 | 1343.6ms/step | 14.9 samples/s +Step 29160/44000 | Loss: 2.9401 | LR: 1.32e-04 | 1346.7ms/step | 14.9 samples/s +Step 29170/44000 | Loss: 2.9462 | LR: 1.32e-04 | 1343.0ms/step | 14.9 samples/s +Step 29180/44000 | Loss: 2.9331 | LR: 1.31e-04 | 1340.5ms/step | 14.9 samples/s +Step 29190/44000 | Loss: 2.9361 | LR: 1.31e-04 | 1337.0ms/step | 15.0 samples/s +Step 29200/44000 | Loss: 2.9123 | LR: 1.31e-04 | 1337.4ms/step | 15.0 samples/s +Step 29210/44000 | Loss: 2.9091 | LR: 1.31e-04 | 1340.2ms/step | 14.9 samples/s +Step 29220/44000 | Loss: 2.9530 | LR: 1.31e-04 | 1339.5ms/step | 14.9 samples/s +Step 29230/44000 | Loss: 2.9300 | LR: 1.31e-04 | 1340.1ms/step | 14.9 samples/s +Step 29240/44000 | Loss: 2.9286 | LR: 1.30e-04 | 1340.5ms/step | 14.9 samples/s +Step 29250/44000 | Loss: 2.9415 | LR: 1.30e-04 | 1338.4ms/step | 14.9 samples/s +Validation at step 29250: Loss = 2.9203, Perplexity = 18.55 +Step 29260/44000 | Loss: 2.9314 | LR: 1.30e-04 | 3575.5ms/step | 5.6 samples/s +Step 29270/44000 | Loss: 2.9389 | LR: 1.30e-04 | 1342.8ms/step | 14.9 samples/s +Step 29280/44000 | Loss: 2.9122 | LR: 1.30e-04 | 1343.3ms/step | 14.9 samples/s +Step 29290/44000 | Loss: 2.9487 | LR: 1.30e-04 | 1343.2ms/step | 14.9 samples/s +Step 29300/44000 | Loss: 2.9404 | LR: 1.29e-04 | 1341.2ms/step | 14.9 samples/s +Step 29310/44000 | Loss: 2.9351 | LR: 1.29e-04 | 1342.1ms/step | 14.9 samples/s +Step 29320/44000 | Loss: 2.9108 | LR: 1.29e-04 | 1343.0ms/step | 14.9 samples/s +Step 29330/44000 | Loss: 2.9285 | LR: 1.29e-04 | 1345.8ms/step | 14.9 samples/s +Step 29340/44000 | Loss: 2.8994 | LR: 1.29e-04 | 1348.6ms/step | 14.8 samples/s +Step 29350/44000 | Loss: 2.9244 | LR: 1.29e-04 | 1346.1ms/step | 14.9 samples/s +Step 29360/44000 | Loss: 2.9208 | LR: 1.29e-04 | 1347.4ms/step | 14.8 samples/s +Step 29370/44000 | Loss: 2.9499 | LR: 1.28e-04 | 1346.5ms/step | 14.9 samples/s +Step 29380/44000 | Loss: 2.9128 | LR: 1.28e-04 | 1342.6ms/step | 14.9 samples/s +Step 29390/44000 | Loss: 2.9595 | LR: 1.28e-04 | 1340.7ms/step | 14.9 samples/s +Step 29400/44000 | Loss: 2.9218 | LR: 1.28e-04 | 1340.4ms/step | 14.9 samples/s +Step 29410/44000 | Loss: 2.9280 | LR: 1.28e-04 | 1339.8ms/step | 14.9 samples/s +Step 29420/44000 | Loss: 2.9657 | LR: 1.28e-04 | 1336.7ms/step | 15.0 samples/s +Step 29430/44000 | Loss: 2.9180 | LR: 1.27e-04 | 1336.5ms/step | 15.0 samples/s +Step 29440/44000 | Loss: 2.9365 | LR: 1.27e-04 | 1337.0ms/step | 15.0 samples/s +Step 29450/44000 | Loss: 2.9348 | LR: 1.27e-04 | 1337.3ms/step | 15.0 samples/s +Step 29460/44000 | Loss: 2.9475 | LR: 1.27e-04 | 1335.8ms/step | 15.0 samples/s +Step 29470/44000 | Loss: 2.9025 | LR: 1.27e-04 | 1335.4ms/step | 15.0 samples/s +Step 29480/44000 | Loss: 2.9585 | LR: 1.27e-04 | 1337.8ms/step | 14.9 samples/s +Step 29490/44000 | Loss: 2.9465 | LR: 1.26e-04 | 1337.8ms/step | 15.0 samples/s +Step 29500/44000 | Loss: 2.9380 | LR: 1.26e-04 | 1336.7ms/step | 15.0 samples/s +Validation at step 29500: Loss = 2.9195, Perplexity = 18.53 +Step 29510/44000 | Loss: 2.9373 | LR: 1.26e-04 | 3577.0ms/step | 5.6 samples/s +Step 29520/44000 | Loss: 2.9543 | LR: 1.26e-04 | 1339.8ms/step | 14.9 samples/s +Step 29530/44000 | Loss: 2.9200 | LR: 1.26e-04 | 1340.2ms/step | 14.9 samples/s +Step 29540/44000 | Loss: 2.9273 | LR: 1.26e-04 | 1340.9ms/step | 14.9 samples/s +Step 29550/44000 | Loss: 2.9034 | LR: 1.26e-04 | 1340.9ms/step | 14.9 samples/s +Step 29560/44000 | Loss: 2.9304 | LR: 1.25e-04 | 1341.0ms/step | 14.9 samples/s +Step 29570/44000 | Loss: 2.9141 | LR: 1.25e-04 | 1339.8ms/step | 14.9 samples/s +Step 29580/44000 | Loss: 2.9348 | LR: 1.25e-04 | 1336.3ms/step | 15.0 samples/s +Step 29590/44000 | Loss: 2.9209 | LR: 1.25e-04 | 1336.5ms/step | 15.0 samples/s +Step 29600/44000 | Loss: 2.9473 | LR: 1.25e-04 | 1339.9ms/step | 14.9 samples/s +Step 29610/44000 | Loss: 2.9478 | LR: 1.25e-04 | 1345.3ms/step | 14.9 samples/s +Step 29620/44000 | Loss: 2.9368 | LR: 1.24e-04 | 1339.1ms/step | 14.9 samples/s +Step 29630/44000 | Loss: 2.9377 | LR: 1.24e-04 | 1338.8ms/step | 14.9 samples/s +Step 29640/44000 | Loss: 2.9108 | LR: 1.24e-04 | 1339.6ms/step | 14.9 samples/s +Step 29650/44000 | Loss: 2.9120 | LR: 1.24e-04 | 1340.0ms/step | 14.9 samples/s +Step 29660/44000 | Loss: 2.9384 | LR: 1.24e-04 | 1341.5ms/step | 14.9 samples/s +Step 29670/44000 | Loss: 2.9223 | LR: 1.24e-04 | 1340.9ms/step | 14.9 samples/s +Step 29680/44000 | Loss: 2.9373 | LR: 1.23e-04 | 1336.6ms/step | 15.0 samples/s +Step 29690/44000 | Loss: 2.9434 | LR: 1.23e-04 | 1334.0ms/step | 15.0 samples/s +Step 29700/44000 | Loss: 2.9243 | LR: 1.23e-04 | 1340.7ms/step | 14.9 samples/s +Step 29710/44000 | Loss: 2.9245 | LR: 1.23e-04 | 1341.1ms/step | 14.9 samples/s +Step 29720/44000 | Loss: 2.9499 | LR: 1.23e-04 | 1339.4ms/step | 14.9 samples/s +Step 29730/44000 | Loss: 2.9328 | LR: 1.23e-04 | 1341.4ms/step | 14.9 samples/s +Step 29740/44000 | Loss: 2.9292 | LR: 1.23e-04 | 1342.0ms/step | 14.9 samples/s +Step 29750/44000 | Loss: 2.9295 | LR: 1.22e-04 | 1343.4ms/step | 14.9 samples/s +Validation at step 29750: Loss = 2.9185, Perplexity = 18.51 +Step 29760/44000 | Loss: 2.9457 | LR: 1.22e-04 | 3563.7ms/step | 5.6 samples/s +Step 29770/44000 | Loss: 2.9472 | LR: 1.22e-04 | 1333.6ms/step | 15.0 samples/s +Step 29780/44000 | Loss: 2.8914 | LR: 1.22e-04 | 1340.6ms/step | 14.9 samples/s +Step 29790/44000 | Loss: 2.9080 | LR: 1.22e-04 | 1340.2ms/step | 14.9 samples/s +Step 29800/44000 | Loss: 2.9228 | LR: 1.22e-04 | 1334.3ms/step | 15.0 samples/s +Step 29810/44000 | Loss: 2.9386 | LR: 1.21e-04 | 1333.0ms/step | 15.0 samples/s +Step 29820/44000 | Loss: 2.9297 | LR: 1.21e-04 | 1333.5ms/step | 15.0 samples/s +Step 29830/44000 | Loss: 2.9110 | LR: 1.21e-04 | 1334.9ms/step | 15.0 samples/s +Step 29840/44000 | Loss: 2.9294 | LR: 1.21e-04 | 1335.9ms/step | 15.0 samples/s +Step 29850/44000 | Loss: 2.9356 | LR: 1.21e-04 | 1335.9ms/step | 15.0 samples/s +Step 29860/44000 | Loss: 2.9343 | LR: 1.21e-04 | 1340.0ms/step | 14.9 samples/s +Step 29870/44000 | Loss: 2.9344 | LR: 1.21e-04 | 1336.2ms/step | 15.0 samples/s +Step 29880/44000 | Loss: 2.9228 | LR: 1.20e-04 | 1332.8ms/step | 15.0 samples/s +Step 29890/44000 | Loss: 2.9046 | LR: 1.20e-04 | 1337.5ms/step | 15.0 samples/s +Step 29900/44000 | Loss: 2.9355 | LR: 1.20e-04 | 1341.5ms/step | 14.9 samples/s +Step 29910/44000 | Loss: 2.9222 | LR: 1.20e-04 | 1341.0ms/step | 14.9 samples/s +Step 29920/44000 | Loss: 2.9466 | LR: 1.20e-04 | 1338.6ms/step | 14.9 samples/s +Step 29930/44000 | Loss: 2.9609 | LR: 1.20e-04 | 1337.1ms/step | 15.0 samples/s +Step 29940/44000 | Loss: 2.9384 | LR: 1.19e-04 | 1336.0ms/step | 15.0 samples/s +Step 29950/44000 | Loss: 2.9352 | LR: 1.19e-04 | 1335.8ms/step | 15.0 samples/s +Step 29960/44000 | Loss: 2.9578 | LR: 1.19e-04 | 1339.5ms/step | 14.9 samples/s +Step 29970/44000 | Loss: 2.9216 | LR: 1.19e-04 | 1338.6ms/step | 14.9 samples/s +Step 29980/44000 | Loss: 2.9343 | LR: 1.19e-04 | 1341.7ms/step | 14.9 samples/s +Step 29990/44000 | Loss: 2.9495 | LR: 1.19e-04 | 1338.0ms/step | 14.9 samples/s +Step 30000/44000 | Loss: 2.9209 | LR: 1.19e-04 | 1336.5ms/step | 15.0 samples/s +Validation at step 30000: Loss = 2.9175, Perplexity = 18.50 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-30000 +Step 30010/44000 | Loss: 2.9067 | LR: 1.18e-04 | 3894.0ms/step | 5.1 samples/s +Step 30020/44000 | Loss: 2.9155 | LR: 1.18e-04 | 1337.1ms/step | 15.0 samples/s +Step 30030/44000 | Loss: 2.9131 | LR: 1.18e-04 | 1337.9ms/step | 14.9 samples/s +Step 30040/44000 | Loss: 2.9267 | LR: 1.18e-04 | 1339.2ms/step | 14.9 samples/s +Step 30050/44000 | Loss: 2.9362 | LR: 1.18e-04 | 1336.9ms/step | 15.0 samples/s +Step 30060/44000 | Loss: 2.9428 | LR: 1.18e-04 | 1336.0ms/step | 15.0 samples/s +Step 30070/44000 | Loss: 2.8957 | LR: 1.17e-04 | 1338.1ms/step | 14.9 samples/s +Step 30080/44000 | Loss: 2.9330 | LR: 1.17e-04 | 1338.3ms/step | 14.9 samples/s +Step 30090/44000 | Loss: 2.9375 | LR: 1.17e-04 | 1335.6ms/step | 15.0 samples/s +Step 30100/44000 | Loss: 2.9051 | LR: 1.17e-04 | 1336.6ms/step | 15.0 samples/s +Step 30110/44000 | Loss: 2.9283 | LR: 1.17e-04 | 1338.6ms/step | 14.9 samples/s +Step 30120/44000 | Loss: 2.9092 | LR: 1.17e-04 | 1341.9ms/step | 14.9 samples/s +Step 30130/44000 | Loss: 2.9086 | LR: 1.17e-04 | 1341.7ms/step | 14.9 samples/s +Step 30140/44000 | Loss: 2.9266 | LR: 1.16e-04 | 1338.8ms/step | 14.9 samples/s +Step 30150/44000 | Loss: 2.9303 | LR: 1.16e-04 | 1336.7ms/step | 15.0 samples/s +Step 30160/44000 | Loss: 2.9232 | LR: 1.16e-04 | 1340.0ms/step | 14.9 samples/s +Step 30170/44000 | Loss: 2.9505 | LR: 1.16e-04 | 1341.4ms/step | 14.9 samples/s +Step 30180/44000 | Loss: 2.9307 | LR: 1.16e-04 | 1339.4ms/step | 14.9 samples/s +Step 30190/44000 | Loss: 2.9383 | LR: 1.16e-04 | 1339.8ms/step | 14.9 samples/s +Step 30200/44000 | Loss: 2.9210 | LR: 1.15e-04 | 1338.8ms/step | 14.9 samples/s +Step 30210/44000 | Loss: 2.9310 | LR: 1.15e-04 | 1340.0ms/step | 14.9 samples/s +Step 30220/44000 | Loss: 2.9107 | LR: 1.15e-04 | 1339.9ms/step | 14.9 samples/s +Step 30230/44000 | Loss: 2.9380 | LR: 1.15e-04 | 1337.2ms/step | 15.0 samples/s +Step 30240/44000 | Loss: 2.9217 | LR: 1.15e-04 | 1337.2ms/step | 15.0 samples/s +Step 30250/44000 | Loss: 2.9119 | LR: 1.15e-04 | 1338.0ms/step | 14.9 samples/s +Validation at step 30250: Loss = 2.9166, Perplexity = 18.48 +Step 30260/44000 | Loss: 2.9404 | LR: 1.15e-04 | 3573.9ms/step | 5.6 samples/s +Step 30270/44000 | Loss: 2.9416 | LR: 1.14e-04 | 1341.0ms/step | 14.9 samples/s +Step 30280/44000 | Loss: 2.9305 | LR: 1.14e-04 | 1342.8ms/step | 14.9 samples/s +Step 30290/44000 | Loss: 2.9106 | LR: 1.14e-04 | 1339.0ms/step | 14.9 samples/s +Step 30300/44000 | Loss: 2.9157 | LR: 1.14e-04 | 1338.2ms/step | 14.9 samples/s +Step 30310/44000 | Loss: 2.8714 | LR: 1.14e-04 | 1340.8ms/step | 14.9 samples/s +Step 30320/44000 | Loss: 2.9016 | LR: 1.14e-04 | 1340.6ms/step | 14.9 samples/s +Step 30330/44000 | Loss: 2.9328 | LR: 1.13e-04 | 1340.5ms/step | 14.9 samples/s +Step 30340/44000 | Loss: 2.9217 | LR: 1.13e-04 | 1341.1ms/step | 14.9 samples/s +Step 30350/44000 | Loss: 2.9377 | LR: 1.13e-04 | 1339.7ms/step | 14.9 samples/s +Step 30360/44000 | Loss: 2.9373 | LR: 1.13e-04 | 1342.6ms/step | 14.9 samples/s +Step 30370/44000 | Loss: 2.9191 | LR: 1.13e-04 | 1339.8ms/step | 14.9 samples/s +Step 30380/44000 | Loss: 2.9197 | LR: 1.13e-04 | 1340.7ms/step | 14.9 samples/s +Step 30390/44000 | Loss: 2.9249 | LR: 1.13e-04 | 1340.8ms/step | 14.9 samples/s +Step 30400/44000 | Loss: 2.9261 | LR: 1.12e-04 | 1340.5ms/step | 14.9 samples/s +Step 30410/44000 | Loss: 2.9346 | LR: 1.12e-04 | 1341.7ms/step | 14.9 samples/s +Step 30420/44000 | Loss: 2.9438 | LR: 1.12e-04 | 1341.5ms/step | 14.9 samples/s +Step 30430/44000 | Loss: 2.9136 | LR: 1.12e-04 | 1341.7ms/step | 14.9 samples/s +Step 30440/44000 | Loss: 2.9498 | LR: 1.12e-04 | 1337.6ms/step | 15.0 samples/s +Step 30450/44000 | Loss: 2.9243 | LR: 1.12e-04 | 1342.3ms/step | 14.9 samples/s +Step 30460/44000 | Loss: 2.9066 | LR: 1.11e-04 | 1343.3ms/step | 14.9 samples/s +Step 30470/44000 | Loss: 2.9394 | LR: 1.11e-04 | 1343.6ms/step | 14.9 samples/s +Step 30480/44000 | Loss: 2.9260 | LR: 1.11e-04 | 1340.8ms/step | 14.9 samples/s +Step 30490/44000 | Loss: 2.9241 | LR: 1.11e-04 | 1333.2ms/step | 15.0 samples/s +Step 30500/44000 | Loss: 2.9192 | LR: 1.11e-04 | 1332.9ms/step | 15.0 samples/s +Validation at step 30500: Loss = 2.9157, Perplexity = 18.46 +Step 30510/44000 | Loss: 2.8974 | LR: 1.11e-04 | 3559.5ms/step | 5.6 samples/s +Step 30520/44000 | Loss: 2.9255 | LR: 1.11e-04 | 1335.6ms/step | 15.0 samples/s +Step 30530/44000 | Loss: 2.9379 | LR: 1.10e-04 | 1334.6ms/step | 15.0 samples/s +Step 30540/44000 | Loss: 2.9412 | LR: 1.10e-04 | 1335.0ms/step | 15.0 samples/s +Step 30550/44000 | Loss: 2.9301 | LR: 1.10e-04 | 1338.8ms/step | 14.9 samples/s +Step 30560/44000 | Loss: 2.9586 | LR: 1.10e-04 | 1347.1ms/step | 14.8 samples/s +Step 30570/44000 | Loss: 2.9309 | LR: 1.10e-04 | 1340.1ms/step | 14.9 samples/s +Step 30580/44000 | Loss: 2.9567 | LR: 1.10e-04 | 1337.8ms/step | 14.9 samples/s +Step 30590/44000 | Loss: 2.9412 | LR: 1.10e-04 | 1334.0ms/step | 15.0 samples/s +Step 30600/44000 | Loss: 2.9031 | LR: 1.09e-04 | 1333.3ms/step | 15.0 samples/s +Step 30610/44000 | Loss: 2.9632 | LR: 1.09e-04 | 1335.7ms/step | 15.0 samples/s +Step 30620/44000 | Loss: 2.9092 | LR: 1.09e-04 | 1338.2ms/step | 14.9 samples/s +Step 30630/44000 | Loss: 2.9016 | LR: 1.09e-04 | 1338.8ms/step | 14.9 samples/s +Step 30640/44000 | Loss: 2.9248 | LR: 1.09e-04 | 1338.2ms/step | 14.9 samples/s +Step 30650/44000 | Loss: 2.9164 | LR: 1.09e-04 | 1339.0ms/step | 14.9 samples/s +Step 30660/44000 | Loss: 2.9245 | LR: 1.08e-04 | 1342.2ms/step | 14.9 samples/s +Step 30670/44000 | Loss: 2.9230 | LR: 1.08e-04 | 1342.5ms/step | 14.9 samples/s +Step 30680/44000 | Loss: 2.9266 | LR: 1.08e-04 | 1339.8ms/step | 14.9 samples/s +Step 30690/44000 | Loss: 2.9372 | LR: 1.08e-04 | 1337.5ms/step | 15.0 samples/s +Step 30700/44000 | Loss: 2.9270 | LR: 1.08e-04 | 1340.0ms/step | 14.9 samples/s +Step 30710/44000 | Loss: 2.9430 | LR: 1.08e-04 | 1339.1ms/step | 14.9 samples/s +Step 30720/44000 | Loss: 2.9454 | LR: 1.08e-04 | 1339.4ms/step | 14.9 samples/s +Step 30730/44000 | Loss: 2.9335 | LR: 1.07e-04 | 1338.7ms/step | 14.9 samples/s +Step 30740/44000 | Loss: 2.9056 | LR: 1.07e-04 | 1338.9ms/step | 14.9 samples/s +Step 30750/44000 | Loss: 2.9365 | LR: 1.07e-04 | 1339.1ms/step | 14.9 samples/s +Validation at step 30750: Loss = 2.9149, Perplexity = 18.45 +Step 30760/44000 | Loss: 2.9315 | LR: 1.07e-04 | 3574.5ms/step | 5.6 samples/s +Step 30770/44000 | Loss: 2.9160 | LR: 1.07e-04 | 1338.8ms/step | 14.9 samples/s +Step 30780/44000 | Loss: 2.9310 | LR: 1.07e-04 | 1337.4ms/step | 15.0 samples/s +Step 30790/44000 | Loss: 2.9288 | LR: 1.07e-04 | 1336.6ms/step | 15.0 samples/s +Step 30800/44000 | Loss: 2.9191 | LR: 1.06e-04 | 1337.5ms/step | 15.0 samples/s +Step 30810/44000 | Loss: 2.9155 | LR: 1.06e-04 | 1337.3ms/step | 15.0 samples/s +Step 30820/44000 | Loss: 2.9392 | LR: 1.06e-04 | 1336.6ms/step | 15.0 samples/s +Step 30830/44000 | Loss: 2.9181 | LR: 1.06e-04 | 1335.6ms/step | 15.0 samples/s +Step 30840/44000 | Loss: 2.8997 | LR: 1.06e-04 | 1337.5ms/step | 15.0 samples/s +Step 30850/44000 | Loss: 2.9317 | LR: 1.06e-04 | 1338.7ms/step | 14.9 samples/s +Step 30860/44000 | Loss: 2.9321 | LR: 1.05e-04 | 1335.8ms/step | 15.0 samples/s +Step 30870/44000 | Loss: 2.9378 | LR: 1.05e-04 | 1334.3ms/step | 15.0 samples/s +Step 30880/44000 | Loss: 2.9254 | LR: 1.05e-04 | 1332.5ms/step | 15.0 samples/s +Step 30890/44000 | Loss: 2.9494 | LR: 1.05e-04 | 1334.4ms/step | 15.0 samples/s +Step 30900/44000 | Loss: 2.9317 | LR: 1.05e-04 | 1336.8ms/step | 15.0 samples/s +Step 30910/44000 | Loss: 2.9060 | LR: 1.05e-04 | 1335.5ms/step | 15.0 samples/s +Step 30920/44000 | Loss: 2.9367 | LR: 1.05e-04 | 1335.4ms/step | 15.0 samples/s +Step 30930/44000 | Loss: 2.8974 | LR: 1.04e-04 | 1340.0ms/step | 14.9 samples/s +Step 30940/44000 | Loss: 2.9141 | LR: 1.04e-04 | 1341.7ms/step | 14.9 samples/s +Step 30950/44000 | Loss: 2.9033 | LR: 1.04e-04 | 1342.7ms/step | 14.9 samples/s +Step 30960/44000 | Loss: 2.9115 | LR: 1.04e-04 | 1342.1ms/step | 14.9 samples/s +Step 30970/44000 | Loss: 2.9318 | LR: 1.04e-04 | 1342.1ms/step | 14.9 samples/s +Step 30980/44000 | Loss: 2.9416 | LR: 1.04e-04 | 1341.7ms/step | 14.9 samples/s +Step 30990/44000 | Loss: 2.9326 | LR: 1.04e-04 | 1340.2ms/step | 14.9 samples/s +Step 31000/44000 | Loss: 2.9218 | LR: 1.03e-04 | 1339.9ms/step | 14.9 samples/s +Validation at step 31000: Loss = 2.9142, Perplexity = 18.43 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-31000 +Step 31010/44000 | Loss: 2.9196 | LR: 1.03e-04 | 3896.2ms/step | 5.1 samples/s +Step 31020/44000 | Loss: 2.9345 | LR: 1.03e-04 | 1333.6ms/step | 15.0 samples/s +Step 31030/44000 | Loss: 2.9077 | LR: 1.03e-04 | 1338.2ms/step | 14.9 samples/s +Step 31040/44000 | Loss: 2.9369 | LR: 1.03e-04 | 1342.4ms/step | 14.9 samples/s +Step 31050/44000 | Loss: 2.8911 | LR: 1.03e-04 | 1342.0ms/step | 14.9 samples/s +Step 31060/44000 | Loss: 2.9186 | LR: 1.03e-04 | 1341.7ms/step | 14.9 samples/s +Step 31070/44000 | Loss: 2.9079 | LR: 1.02e-04 | 1341.4ms/step | 14.9 samples/s +Step 31080/44000 | Loss: 2.9303 | LR: 1.02e-04 | 1342.7ms/step | 14.9 samples/s +Step 31090/44000 | Loss: 2.9258 | LR: 1.02e-04 | 1343.6ms/step | 14.9 samples/s +Step 31100/44000 | Loss: 2.9494 | LR: 1.02e-04 | 1344.3ms/step | 14.9 samples/s +Step 31110/44000 | Loss: 2.9131 | LR: 1.02e-04 | 1344.7ms/step | 14.9 samples/s +Step 31120/44000 | Loss: 2.9398 | LR: 1.02e-04 | 1342.7ms/step | 14.9 samples/s +Step 31130/44000 | Loss: 2.9381 | LR: 1.02e-04 | 1342.4ms/step | 14.9 samples/s +Step 31140/44000 | Loss: 2.9191 | LR: 1.01e-04 | 1343.1ms/step | 14.9 samples/s +Step 31150/44000 | Loss: 2.8965 | LR: 1.01e-04 | 1342.0ms/step | 14.9 samples/s +Step 31160/44000 | Loss: 2.9037 | LR: 1.01e-04 | 1339.8ms/step | 14.9 samples/s +Step 31170/44000 | Loss: 2.9350 | LR: 1.01e-04 | 1342.5ms/step | 14.9 samples/s +Step 31180/44000 | Loss: 2.9309 | LR: 1.01e-04 | 1340.5ms/step | 14.9 samples/s +Step 31190/44000 | Loss: 2.9514 | LR: 1.01e-04 | 1341.3ms/step | 14.9 samples/s +Step 31200/44000 | Loss: 2.9322 | LR: 1.00e-04 | 1343.5ms/step | 14.9 samples/s +Step 31210/44000 | Loss: 2.9086 | LR: 1.00e-04 | 1341.8ms/step | 14.9 samples/s +Step 31220/44000 | Loss: 2.9640 | LR: 1.00e-04 | 1342.4ms/step | 14.9 samples/s +Step 31230/44000 | Loss: 2.9720 | LR: 1.00e-04 | 1341.3ms/step | 14.9 samples/s +Step 31240/44000 | Loss: 2.9098 | LR: 9.99e-05 | 1343.1ms/step | 14.9 samples/s +Step 31250/44000 | Loss: 2.9297 | LR: 9.98e-05 | 1343.7ms/step | 14.9 samples/s +Validation at step 31250: Loss = 2.9135, Perplexity = 18.42 +Step 31260/44000 | Loss: 2.9245 | LR: 9.96e-05 | 3576.5ms/step | 5.6 samples/s +Step 31270/44000 | Loss: 2.9109 | LR: 9.95e-05 | 1340.5ms/step | 14.9 samples/s +Step 31280/44000 | Loss: 2.9203 | LR: 9.93e-05 | 1337.9ms/step | 14.9 samples/s +Step 31290/44000 | Loss: 2.9316 | LR: 9.92e-05 | 1339.6ms/step | 14.9 samples/s +Step 31300/44000 | Loss: 2.9322 | LR: 9.90e-05 | 1338.9ms/step | 14.9 samples/s +Step 31310/44000 | Loss: 2.9243 | LR: 9.89e-05 | 1341.2ms/step | 14.9 samples/s +Step 31320/44000 | Loss: 2.9309 | LR: 9.88e-05 | 1341.9ms/step | 14.9 samples/s +Step 31330/44000 | Loss: 2.9330 | LR: 9.86e-05 | 1341.0ms/step | 14.9 samples/s +Step 31340/44000 | Loss: 2.9184 | LR: 9.85e-05 | 1342.3ms/step | 14.9 samples/s +Step 31350/44000 | Loss: 2.9240 | LR: 9.83e-05 | 1341.7ms/step | 14.9 samples/s +Step 31360/44000 | Loss: 2.9216 | LR: 9.82e-05 | 1343.7ms/step | 14.9 samples/s +Step 31370/44000 | Loss: 2.9180 | LR: 9.80e-05 | 1343.6ms/step | 14.9 samples/s +Step 31380/44000 | Loss: 2.9269 | LR: 9.79e-05 | 1342.3ms/step | 14.9 samples/s +Step 31390/44000 | Loss: 2.9135 | LR: 9.77e-05 | 1343.8ms/step | 14.9 samples/s +Step 31400/44000 | Loss: 2.9217 | LR: 9.76e-05 | 1344.7ms/step | 14.9 samples/s +Step 31410/44000 | Loss: 2.9032 | LR: 9.75e-05 | 1345.4ms/step | 14.9 samples/s +Step 31420/44000 | Loss: 2.9247 | LR: 9.73e-05 | 1342.6ms/step | 14.9 samples/s +Step 31430/44000 | Loss: 2.9059 | LR: 9.72e-05 | 1345.0ms/step | 14.9 samples/s +Step 31440/44000 | Loss: 2.9073 | LR: 9.70e-05 | 1347.5ms/step | 14.8 samples/s +Step 31450/44000 | Loss: 2.9305 | LR: 9.69e-05 | 1347.2ms/step | 14.8 samples/s +Step 31460/44000 | Loss: 2.9120 | LR: 9.67e-05 | 1343.6ms/step | 14.9 samples/s +Step 31470/44000 | Loss: 2.9389 | LR: 9.66e-05 | 1344.3ms/step | 14.9 samples/s +Step 31480/44000 | Loss: 2.9267 | LR: 9.65e-05 | 1341.8ms/step | 14.9 samples/s +Step 31490/44000 | Loss: 2.9378 | LR: 9.63e-05 | 1339.3ms/step | 14.9 samples/s +Step 31500/44000 | Loss: 2.9054 | LR: 9.62e-05 | 1340.8ms/step | 14.9 samples/s +Validation at step 31500: Loss = 2.9128, Perplexity = 18.41 +Step 31510/44000 | Loss: 2.9204 | LR: 9.60e-05 | 3578.8ms/step | 5.6 samples/s +Step 31520/44000 | Loss: 2.9221 | LR: 9.59e-05 | 1333.2ms/step | 15.0 samples/s +Step 31530/44000 | Loss: 2.9268 | LR: 9.57e-05 | 1338.6ms/step | 14.9 samples/s +Step 31540/44000 | Loss: 2.9199 | LR: 9.56e-05 | 1342.7ms/step | 14.9 samples/s +Step 31550/44000 | Loss: 2.9180 | LR: 9.54e-05 | 1339.4ms/step | 14.9 samples/s +Step 31560/44000 | Loss: 2.9411 | LR: 9.53e-05 | 1343.2ms/step | 14.9 samples/s +Step 31570/44000 | Loss: 2.9059 | LR: 9.52e-05 | 1344.2ms/step | 14.9 samples/s +Step 31580/44000 | Loss: 2.9343 | LR: 9.50e-05 | 1342.4ms/step | 14.9 samples/s +Step 31590/44000 | Loss: 2.9320 | LR: 9.49e-05 | 1341.4ms/step | 14.9 samples/s +Step 31600/44000 | Loss: 2.9014 | LR: 9.47e-05 | 1343.0ms/step | 14.9 samples/s +Step 31610/44000 | Loss: 2.9409 | LR: 9.46e-05 | 1340.8ms/step | 14.9 samples/s +Step 31620/44000 | Loss: 2.9175 | LR: 9.45e-05 | 1339.3ms/step | 14.9 samples/s +Step 31630/44000 | Loss: 2.9183 | LR: 9.43e-05 | 1338.7ms/step | 14.9 samples/s +Step 31640/44000 | Loss: 2.9294 | LR: 9.42e-05 | 1334.5ms/step | 15.0 samples/s +Step 31650/44000 | Loss: 2.9258 | LR: 9.40e-05 | 1333.8ms/step | 15.0 samples/s +Step 31660/44000 | Loss: 2.9339 | LR: 9.39e-05 | 1333.7ms/step | 15.0 samples/s +Step 31670/44000 | Loss: 2.9173 | LR: 9.37e-05 | 1340.5ms/step | 14.9 samples/s +Step 31680/44000 | Loss: 2.8886 | LR: 9.36e-05 | 1343.5ms/step | 14.9 samples/s +Step 31690/44000 | Loss: 2.9113 | LR: 9.35e-05 | 1341.5ms/step | 14.9 samples/s +Step 31700/44000 | Loss: 2.9179 | LR: 9.33e-05 | 1338.2ms/step | 14.9 samples/s +Step 31710/44000 | Loss: 2.9210 | LR: 9.32e-05 | 1337.0ms/step | 15.0 samples/s +Step 31720/44000 | Loss: 2.9194 | LR: 9.30e-05 | 1337.5ms/step | 15.0 samples/s +Step 31730/44000 | Loss: 2.9267 | LR: 9.29e-05 | 1335.9ms/step | 15.0 samples/s +Step 31740/44000 | Loss: 2.9094 | LR: 9.28e-05 | 1340.1ms/step | 14.9 samples/s +Step 31750/44000 | Loss: 2.9256 | LR: 9.26e-05 | 1340.7ms/step | 14.9 samples/s +Validation at step 31750: Loss = 2.9123, Perplexity = 18.40 +Step 31760/44000 | Loss: 2.9374 | LR: 9.25e-05 | 3565.1ms/step | 5.6 samples/s +Step 31770/44000 | Loss: 2.9133 | LR: 9.23e-05 | 1333.9ms/step | 15.0 samples/s +Step 31780/44000 | Loss: 2.9122 | LR: 9.22e-05 | 1333.8ms/step | 15.0 samples/s +Step 31790/44000 | Loss: 2.9170 | LR: 9.20e-05 | 1334.7ms/step | 15.0 samples/s +Step 31800/44000 | Loss: 2.8996 | LR: 9.19e-05 | 1339.2ms/step | 14.9 samples/s +Step 31810/44000 | Loss: 2.9134 | LR: 9.18e-05 | 1344.7ms/step | 14.9 samples/s +Step 31820/44000 | Loss: 2.9161 | LR: 9.16e-05 | 1343.1ms/step | 14.9 samples/s +Step 31830/44000 | Loss: 2.9023 | LR: 9.15e-05 | 1344.1ms/step | 14.9 samples/s +Step 31840/44000 | Loss: 2.9075 | LR: 9.13e-05 | 1340.9ms/step | 14.9 samples/s +Step 31850/44000 | Loss: 2.9228 | LR: 9.12e-05 | 1339.7ms/step | 14.9 samples/s +Step 31860/44000 | Loss: 2.9081 | LR: 9.11e-05 | 1337.6ms/step | 15.0 samples/s +Step 31870/44000 | Loss: 2.9068 | LR: 9.09e-05 | 1340.7ms/step | 14.9 samples/s +Step 31880/44000 | Loss: 2.8908 | LR: 9.08e-05 | 1339.6ms/step | 14.9 samples/s +Step 31890/44000 | Loss: 2.9174 | LR: 9.06e-05 | 1336.8ms/step | 15.0 samples/s +Step 31900/44000 | Loss: 2.9194 | LR: 9.05e-05 | 1333.8ms/step | 15.0 samples/s +Step 31910/44000 | Loss: 2.9293 | LR: 9.04e-05 | 1333.4ms/step | 15.0 samples/s +Step 31920/44000 | Loss: 2.9090 | LR: 9.02e-05 | 1334.3ms/step | 15.0 samples/s +Step 31930/44000 | Loss: 2.9168 | LR: 9.01e-05 | 1334.7ms/step | 15.0 samples/s +Step 31940/44000 | Loss: 2.9473 | LR: 8.99e-05 | 1338.4ms/step | 14.9 samples/s +Step 31950/44000 | Loss: 2.9277 | LR: 8.98e-05 | 1337.3ms/step | 15.0 samples/s +Step 31960/44000 | Loss: 2.8769 | LR: 8.97e-05 | 1340.2ms/step | 14.9 samples/s +Step 31970/44000 | Loss: 2.9434 | LR: 8.95e-05 | 1341.0ms/step | 14.9 samples/s +Step 31980/44000 | Loss: 2.9192 | LR: 8.94e-05 | 1339.9ms/step | 14.9 samples/s +Step 31990/44000 | Loss: 2.9448 | LR: 8.92e-05 | 1338.2ms/step | 14.9 samples/s +Step 32000/44000 | Loss: 2.9453 | LR: 8.91e-05 | 1338.2ms/step | 14.9 samples/s +Validation at step 32000: Loss = 2.9116, Perplexity = 18.39 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-32000 +Step 32010/44000 | Loss: 2.9429 | LR: 8.90e-05 | 3894.8ms/step | 5.1 samples/s +Step 32020/44000 | Loss: 2.8906 | LR: 8.88e-05 | 1335.5ms/step | 15.0 samples/s +Step 32030/44000 | Loss: 2.9416 | LR: 8.87e-05 | 1338.0ms/step | 14.9 samples/s +Step 32040/44000 | Loss: 2.9209 | LR: 8.86e-05 | 1339.4ms/step | 14.9 samples/s +Step 32050/44000 | Loss: 2.9274 | LR: 8.84e-05 | 1340.2ms/step | 14.9 samples/s +Step 32060/44000 | Loss: 2.9532 | LR: 8.83e-05 | 1334.3ms/step | 15.0 samples/s +Step 32070/44000 | Loss: 2.9376 | LR: 8.81e-05 | 1334.9ms/step | 15.0 samples/s +Step 32080/44000 | Loss: 2.9186 | LR: 8.80e-05 | 1338.7ms/step | 14.9 samples/s +Step 32090/44000 | Loss: 2.9390 | LR: 8.79e-05 | 1336.9ms/step | 15.0 samples/s +Step 32100/44000 | Loss: 2.9295 | LR: 8.77e-05 | 1333.8ms/step | 15.0 samples/s +Step 32110/44000 | Loss: 2.9275 | LR: 8.76e-05 | 1336.1ms/step | 15.0 samples/s +Step 32120/44000 | Loss: 2.9248 | LR: 8.74e-05 | 1336.7ms/step | 15.0 samples/s +Step 32130/44000 | Loss: 2.9027 | LR: 8.73e-05 | 1337.5ms/step | 15.0 samples/s +Step 32140/44000 | Loss: 2.9393 | LR: 8.72e-05 | 1335.7ms/step | 15.0 samples/s +Step 32150/44000 | Loss: 2.9367 | LR: 8.70e-05 | 1338.1ms/step | 14.9 samples/s +Step 32160/44000 | Loss: 2.9077 | LR: 8.69e-05 | 1342.0ms/step | 14.9 samples/s +Step 32170/44000 | Loss: 2.9203 | LR: 8.68e-05 | 1339.5ms/step | 14.9 samples/s +Step 32180/44000 | Loss: 2.9219 | LR: 8.66e-05 | 1339.5ms/step | 14.9 samples/s +Step 32190/44000 | Loss: 2.9664 | LR: 8.65e-05 | 1341.1ms/step | 14.9 samples/s +Step 32200/44000 | Loss: 2.9463 | LR: 8.63e-05 | 1343.0ms/step | 14.9 samples/s +Step 32210/44000 | Loss: 2.9244 | LR: 8.62e-05 | 1346.7ms/step | 14.9 samples/s +Step 32220/44000 | Loss: 2.9042 | LR: 8.61e-05 | 1343.0ms/step | 14.9 samples/s +Step 32230/44000 | Loss: 2.9438 | LR: 8.59e-05 | 1339.3ms/step | 14.9 samples/s +Step 32240/44000 | Loss: 2.9167 | LR: 8.58e-05 | 1342.2ms/step | 14.9 samples/s +Step 32250/44000 | Loss: 2.9347 | LR: 8.57e-05 | 1339.5ms/step | 14.9 samples/s +Validation at step 32250: Loss = 2.9112, Perplexity = 18.38 +Step 32260/44000 | Loss: 2.9324 | LR: 8.55e-05 | 3574.8ms/step | 5.6 samples/s +Step 32270/44000 | Loss: 2.9300 | LR: 8.54e-05 | 1336.4ms/step | 15.0 samples/s +Step 32280/44000 | Loss: 2.9075 | LR: 8.53e-05 | 1338.8ms/step | 14.9 samples/s +Step 32290/44000 | Loss: 2.9362 | LR: 8.51e-05 | 1341.8ms/step | 14.9 samples/s +Step 32300/44000 | Loss: 2.9293 | LR: 8.50e-05 | 1338.5ms/step | 14.9 samples/s +Step 32310/44000 | Loss: 2.9209 | LR: 8.48e-05 | 1342.3ms/step | 14.9 samples/s +Step 32320/44000 | Loss: 2.8950 | LR: 8.47e-05 | 1343.6ms/step | 14.9 samples/s +Step 32330/44000 | Loss: 2.9451 | LR: 8.46e-05 | 1349.2ms/step | 14.8 samples/s +Step 32340/44000 | Loss: 2.9093 | LR: 8.44e-05 | 1353.3ms/step | 14.8 samples/s +Step 32350/44000 | Loss: 2.9704 | LR: 8.43e-05 | 1352.8ms/step | 14.8 samples/s +Step 32360/44000 | Loss: 2.9226 | LR: 8.42e-05 | 1347.9ms/step | 14.8 samples/s +Step 32370/44000 | Loss: 2.9156 | LR: 8.40e-05 | 1345.1ms/step | 14.9 samples/s +Step 32380/44000 | Loss: 2.9185 | LR: 8.39e-05 | 1342.9ms/step | 14.9 samples/s +Step 32390/44000 | Loss: 2.9202 | LR: 8.38e-05 | 1343.4ms/step | 14.9 samples/s +Step 32400/44000 | Loss: 2.9077 | LR: 8.36e-05 | 1344.7ms/step | 14.9 samples/s +Step 32410/44000 | Loss: 2.9038 | LR: 8.35e-05 | 1342.0ms/step | 14.9 samples/s +Step 32420/44000 | Loss: 2.9214 | LR: 8.33e-05 | 1343.0ms/step | 14.9 samples/s +Step 32430/44000 | Loss: 2.9322 | LR: 8.32e-05 | 1343.1ms/step | 14.9 samples/s +Step 32440/44000 | Loss: 2.9314 | LR: 8.31e-05 | 1343.2ms/step | 14.9 samples/s +Step 32450/44000 | Loss: 2.9068 | LR: 8.29e-05 | 1343.7ms/step | 14.9 samples/s +Step 32460/44000 | Loss: 2.9012 | LR: 8.28e-05 | 1341.7ms/step | 14.9 samples/s +Step 32470/44000 | Loss: 2.9297 | LR: 8.27e-05 | 1342.9ms/step | 14.9 samples/s +Step 32480/44000 | Loss: 2.9122 | LR: 8.25e-05 | 1344.9ms/step | 14.9 samples/s +Step 32490/44000 | Loss: 2.9155 | LR: 8.24e-05 | 1342.8ms/step | 14.9 samples/s +Step 32500/44000 | Loss: 2.9313 | LR: 8.23e-05 | 1342.6ms/step | 14.9 samples/s +Validation at step 32500: Loss = 2.9106, Perplexity = 18.37 +Step 32510/44000 | Loss: 2.9380 | LR: 8.21e-05 | 3584.3ms/step | 5.6 samples/s +Step 32520/44000 | Loss: 2.9136 | LR: 8.20e-05 | 1344.3ms/step | 14.9 samples/s +Step 32530/44000 | Loss: 2.9384 | LR: 8.19e-05 | 1345.4ms/step | 14.9 samples/s +Step 32540/44000 | Loss: 2.9459 | LR: 8.17e-05 | 1343.2ms/step | 14.9 samples/s +Step 32550/44000 | Loss: 2.9494 | LR: 8.16e-05 | 1341.7ms/step | 14.9 samples/s +Step 32560/44000 | Loss: 2.9386 | LR: 8.15e-05 | 1340.1ms/step | 14.9 samples/s +Step 32570/44000 | Loss: 2.9130 | LR: 8.13e-05 | 1341.7ms/step | 14.9 samples/s +Step 32580/44000 | Loss: 2.9133 | LR: 8.12e-05 | 1343.8ms/step | 14.9 samples/s +Step 32590/44000 | Loss: 2.9514 | LR: 8.11e-05 | 1342.3ms/step | 14.9 samples/s +Step 32600/44000 | Loss: 2.9342 | LR: 8.09e-05 | 1340.2ms/step | 14.9 samples/s +Step 32610/44000 | Loss: 2.9483 | LR: 8.08e-05 | 1337.3ms/step | 15.0 samples/s +Step 32620/44000 | Loss: 2.9084 | LR: 8.07e-05 | 1333.3ms/step | 15.0 samples/s +Step 32630/44000 | Loss: 2.9348 | LR: 8.05e-05 | 1333.7ms/step | 15.0 samples/s +Step 32640/44000 | Loss: 2.9402 | LR: 8.04e-05 | 1336.9ms/step | 15.0 samples/s +Step 32650/44000 | Loss: 2.9138 | LR: 8.03e-05 | 1336.9ms/step | 15.0 samples/s +Step 32660/44000 | Loss: 2.8954 | LR: 8.01e-05 | 1340.0ms/step | 14.9 samples/s +Step 32670/44000 | Loss: 2.9199 | LR: 8.00e-05 | 1344.8ms/step | 14.9 samples/s +Step 32680/44000 | Loss: 2.9134 | LR: 7.99e-05 | 1345.9ms/step | 14.9 samples/s +Step 32690/44000 | Loss: 2.8950 | LR: 7.97e-05 | 1341.7ms/step | 14.9 samples/s +Step 32700/44000 | Loss: 2.9063 | LR: 7.96e-05 | 1340.5ms/step | 14.9 samples/s +Step 32710/44000 | Loss: 2.9443 | LR: 7.95e-05 | 1340.8ms/step | 14.9 samples/s +Step 32720/44000 | Loss: 2.9316 | LR: 7.93e-05 | 1341.1ms/step | 14.9 samples/s +Step 32730/44000 | Loss: 2.9048 | LR: 7.92e-05 | 1342.0ms/step | 14.9 samples/s +Step 32740/44000 | Loss: 2.9135 | LR: 7.91e-05 | 1340.8ms/step | 14.9 samples/s +Step 32750/44000 | Loss: 2.9145 | LR: 7.89e-05 | 1341.5ms/step | 14.9 samples/s +Validation at step 32750: Loss = 2.9101, Perplexity = 18.36 +Step 32760/44000 | Loss: 2.9122 | LR: 7.88e-05 | 3571.2ms/step | 5.6 samples/s +Step 32770/44000 | Loss: 2.9061 | LR: 7.87e-05 | 1335.3ms/step | 15.0 samples/s +Step 32780/44000 | Loss: 2.8955 | LR: 7.85e-05 | 1336.7ms/step | 15.0 samples/s +Step 32790/44000 | Loss: 2.9012 | LR: 7.84e-05 | 1336.0ms/step | 15.0 samples/s +Step 32800/44000 | Loss: 2.9200 | LR: 7.83e-05 | 1338.0ms/step | 14.9 samples/s +Step 32810/44000 | Loss: 2.9347 | LR: 7.81e-05 | 1338.2ms/step | 14.9 samples/s +Step 32820/44000 | Loss: 2.9482 | LR: 7.80e-05 | 1337.2ms/step | 15.0 samples/s +Step 32830/44000 | Loss: 2.9168 | LR: 7.79e-05 | 1334.6ms/step | 15.0 samples/s +Step 32840/44000 | Loss: 2.9412 | LR: 7.77e-05 | 1336.2ms/step | 15.0 samples/s +Step 32850/44000 | Loss: 2.9194 | LR: 7.76e-05 | 1333.3ms/step | 15.0 samples/s +Step 32860/44000 | Loss: 2.8998 | LR: 7.75e-05 | 1333.8ms/step | 15.0 samples/s +Step 32870/44000 | Loss: 2.8674 | LR: 7.73e-05 | 1335.5ms/step | 15.0 samples/s +Step 32880/44000 | Loss: 2.9050 | LR: 7.72e-05 | 1332.5ms/step | 15.0 samples/s +Step 32890/44000 | Loss: 2.9328 | LR: 7.71e-05 | 1332.4ms/step | 15.0 samples/s +Step 32900/44000 | Loss: 2.9437 | LR: 7.70e-05 | 1332.2ms/step | 15.0 samples/s +Step 32910/44000 | Loss: 2.9276 | LR: 7.68e-05 | 1332.1ms/step | 15.0 samples/s +Step 32920/44000 | Loss: 2.9364 | LR: 7.67e-05 | 1333.1ms/step | 15.0 samples/s +Step 32930/44000 | Loss: 2.9154 | LR: 7.66e-05 | 1333.4ms/step | 15.0 samples/s +Step 32940/44000 | Loss: 2.9205 | LR: 7.64e-05 | 1334.6ms/step | 15.0 samples/s +Step 32950/44000 | Loss: 2.9236 | LR: 7.63e-05 | 1333.4ms/step | 15.0 samples/s +Step 32960/44000 | Loss: 2.9104 | LR: 7.62e-05 | 1335.0ms/step | 15.0 samples/s +Step 32970/44000 | Loss: 2.9263 | LR: 7.60e-05 | 1339.0ms/step | 14.9 samples/s +Step 32980/44000 | Loss: 2.9257 | LR: 7.59e-05 | 1339.4ms/step | 14.9 samples/s +Step 32990/44000 | Loss: 2.8986 | LR: 7.58e-05 | 1338.6ms/step | 14.9 samples/s +Step 33000/44000 | Loss: 2.9063 | LR: 7.56e-05 | 1342.7ms/step | 14.9 samples/s +Validation at step 33000: Loss = 2.9096, Perplexity = 18.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-33000 +Step 33010/44000 | Loss: 2.9489 | LR: 7.55e-05 | 3883.9ms/step | 5.1 samples/s +Step 33020/44000 | Loss: 2.9445 | LR: 7.54e-05 | 1335.3ms/step | 15.0 samples/s +Step 33030/44000 | Loss: 2.9453 | LR: 7.53e-05 | 1335.8ms/step | 15.0 samples/s +Step 33040/44000 | Loss: 2.9032 | LR: 7.51e-05 | 1338.1ms/step | 14.9 samples/s +Step 33050/44000 | Loss: 2.9108 | LR: 7.50e-05 | 1339.4ms/step | 14.9 samples/s +Step 33060/44000 | Loss: 2.9250 | LR: 7.49e-05 | 1338.5ms/step | 14.9 samples/s +Step 33070/44000 | Loss: 2.9481 | LR: 7.47e-05 | 1338.6ms/step | 14.9 samples/s +Step 33080/44000 | Loss: 2.9092 | LR: 7.46e-05 | 1339.9ms/step | 14.9 samples/s +Step 33090/44000 | Loss: 2.9237 | LR: 7.45e-05 | 1341.2ms/step | 14.9 samples/s +Step 33100/44000 | Loss: 2.8901 | LR: 7.44e-05 | 1337.8ms/step | 14.9 samples/s +Step 33110/44000 | Loss: 2.8766 | LR: 7.42e-05 | 1337.1ms/step | 15.0 samples/s +Step 33120/44000 | Loss: 2.9331 | LR: 7.41e-05 | 1337.9ms/step | 14.9 samples/s +Step 33130/44000 | Loss: 2.9309 | LR: 7.40e-05 | 1340.7ms/step | 14.9 samples/s +Step 33140/44000 | Loss: 2.8636 | LR: 7.38e-05 | 1345.0ms/step | 14.9 samples/s +Step 33150/44000 | Loss: 2.9425 | LR: 7.37e-05 | 1346.2ms/step | 14.9 samples/s +Step 33160/44000 | Loss: 2.9012 | LR: 7.36e-05 | 1345.0ms/step | 14.9 samples/s +Step 33170/44000 | Loss: 2.9330 | LR: 7.34e-05 | 1345.7ms/step | 14.9 samples/s +Step 33180/44000 | Loss: 2.9054 | LR: 7.33e-05 | 1343.2ms/step | 14.9 samples/s +Step 33190/44000 | Loss: 2.9300 | LR: 7.32e-05 | 1341.6ms/step | 14.9 samples/s +Step 33200/44000 | Loss: 2.9401 | LR: 7.31e-05 | 1340.4ms/step | 14.9 samples/s +Step 33210/44000 | Loss: 2.8957 | LR: 7.29e-05 | 1340.0ms/step | 14.9 samples/s +Step 33220/44000 | Loss: 2.8801 | LR: 7.28e-05 | 1340.2ms/step | 14.9 samples/s +Step 33230/44000 | Loss: 2.9322 | LR: 7.27e-05 | 1338.9ms/step | 14.9 samples/s +Step 33240/44000 | Loss: 2.9056 | LR: 7.26e-05 | 1340.1ms/step | 14.9 samples/s +Step 33250/44000 | Loss: 2.9195 | LR: 7.24e-05 | 1338.8ms/step | 14.9 samples/s +Validation at step 33250: Loss = 2.9091, Perplexity = 18.34 +Step 33260/44000 | Loss: 2.8741 | LR: 7.23e-05 | 3578.6ms/step | 5.6 samples/s +Step 33270/44000 | Loss: 2.8852 | LR: 7.22e-05 | 1345.6ms/step | 14.9 samples/s +Step 33280/44000 | Loss: 2.9175 | LR: 7.20e-05 | 1343.7ms/step | 14.9 samples/s +Step 33290/44000 | Loss: 2.9135 | LR: 7.19e-05 | 1343.2ms/step | 14.9 samples/s +Step 33300/44000 | Loss: 2.9191 | LR: 7.18e-05 | 1342.6ms/step | 14.9 samples/s +Step 33310/44000 | Loss: 2.9004 | LR: 7.17e-05 | 1345.1ms/step | 14.9 samples/s +Step 33320/44000 | Loss: 2.9149 | LR: 7.15e-05 | 1344.7ms/step | 14.9 samples/s +Step 33330/44000 | Loss: 2.9275 | LR: 7.14e-05 | 1343.6ms/step | 14.9 samples/s +Step 33340/44000 | Loss: 2.9128 | LR: 7.13e-05 | 1343.2ms/step | 14.9 samples/s +Step 33350/44000 | Loss: 2.9046 | LR: 7.11e-05 | 1343.4ms/step | 14.9 samples/s +Step 33360/44000 | Loss: 2.9526 | LR: 7.10e-05 | 1346.2ms/step | 14.9 samples/s +Step 33370/44000 | Loss: 2.9243 | LR: 7.09e-05 | 1343.3ms/step | 14.9 samples/s +Step 33380/44000 | Loss: 2.9141 | LR: 7.08e-05 | 1350.6ms/step | 14.8 samples/s +Step 33390/44000 | Loss: 2.9321 | LR: 7.06e-05 | 1349.9ms/step | 14.8 samples/s +Step 33400/44000 | Loss: 2.8957 | LR: 7.05e-05 | 1350.9ms/step | 14.8 samples/s +Step 33410/44000 | Loss: 2.9122 | LR: 7.04e-05 | 1351.2ms/step | 14.8 samples/s +Step 33420/44000 | Loss: 2.9299 | LR: 7.03e-05 | 1346.7ms/step | 14.9 samples/s +Step 33430/44000 | Loss: 2.9492 | LR: 7.01e-05 | 1350.1ms/step | 14.8 samples/s +Step 33440/44000 | Loss: 2.9475 | LR: 7.00e-05 | 1350.3ms/step | 14.8 samples/s +Step 33450/44000 | Loss: 2.9199 | LR: 6.99e-05 | 1350.0ms/step | 14.8 samples/s +Step 33460/44000 | Loss: 2.8965 | LR: 6.98e-05 | 1348.5ms/step | 14.8 samples/s +Step 33470/44000 | Loss: 2.9030 | LR: 6.96e-05 | 1344.8ms/step | 14.9 samples/s +Step 33480/44000 | Loss: 2.8867 | LR: 6.95e-05 | 1345.6ms/step | 14.9 samples/s +Step 33490/44000 | Loss: 2.9143 | LR: 6.94e-05 | 1347.7ms/step | 14.8 samples/s +Step 33500/44000 | Loss: 2.9330 | LR: 6.93e-05 | 1346.8ms/step | 14.8 samples/s +Validation at step 33500: Loss = 2.9088, Perplexity = 18.33 +Step 33510/44000 | Loss: 2.9063 | LR: 6.91e-05 | 3704.9ms/step | 5.4 samples/s +Step 33520/44000 | Loss: 2.9170 | LR: 6.90e-05 | 1346.7ms/step | 14.9 samples/s +Step 33530/44000 | Loss: 2.9161 | LR: 6.89e-05 | 1344.0ms/step | 14.9 samples/s +Step 33540/44000 | Loss: 2.8965 | LR: 6.88e-05 | 1344.2ms/step | 14.9 samples/s +Step 33550/44000 | Loss: 2.9248 | LR: 6.86e-05 | 1349.5ms/step | 14.8 samples/s +Step 33560/44000 | Loss: 2.9264 | LR: 6.85e-05 | 1347.2ms/step | 14.8 samples/s +Step 33570/44000 | Loss: 2.9153 | LR: 6.84e-05 | 1346.1ms/step | 14.9 samples/s +Step 33580/44000 | Loss: 2.9297 | LR: 6.83e-05 | 1342.5ms/step | 14.9 samples/s +Step 33590/44000 | Loss: 2.8965 | LR: 6.81e-05 | 1342.3ms/step | 14.9 samples/s +Step 33600/44000 | Loss: 2.9301 | LR: 6.80e-05 | 1340.8ms/step | 14.9 samples/s +Step 33610/44000 | Loss: 2.9161 | LR: 6.79e-05 | 1337.0ms/step | 15.0 samples/s +Step 33620/44000 | Loss: 2.9135 | LR: 6.78e-05 | 1335.9ms/step | 15.0 samples/s +Step 33630/44000 | Loss: 2.9248 | LR: 6.76e-05 | 1333.8ms/step | 15.0 samples/s +Step 33640/44000 | Loss: 2.9121 | LR: 6.75e-05 | 1334.9ms/step | 15.0 samples/s +Step 33650/44000 | Loss: 2.9062 | LR: 6.74e-05 | 1337.9ms/step | 14.9 samples/s +Step 33660/44000 | Loss: 2.9394 | LR: 6.73e-05 | 1338.6ms/step | 14.9 samples/s +Step 33670/44000 | Loss: 2.9179 | LR: 6.71e-05 | 1338.9ms/step | 14.9 samples/s +Step 33680/44000 | Loss: 2.9164 | LR: 6.70e-05 | 1339.8ms/step | 14.9 samples/s +Step 33690/44000 | Loss: 2.9197 | LR: 6.69e-05 | 1340.8ms/step | 14.9 samples/s +Step 33700/44000 | Loss: 2.9228 | LR: 6.68e-05 | 1343.2ms/step | 14.9 samples/s +Step 33710/44000 | Loss: 2.9292 | LR: 6.66e-05 | 1342.1ms/step | 14.9 samples/s +Step 33720/44000 | Loss: 2.9196 | LR: 6.65e-05 | 1340.0ms/step | 14.9 samples/s +Step 33730/44000 | Loss: 2.9300 | LR: 6.64e-05 | 1340.1ms/step | 14.9 samples/s +Step 33740/44000 | Loss: 2.9335 | LR: 6.63e-05 | 1338.5ms/step | 14.9 samples/s +Step 33750/44000 | Loss: 2.9195 | LR: 6.61e-05 | 1339.3ms/step | 14.9 samples/s +Validation at step 33750: Loss = 2.9084, Perplexity = 18.33 +Step 33760/44000 | Loss: 2.9169 | LR: 6.60e-05 | 3577.5ms/step | 5.6 samples/s +Step 33770/44000 | Loss: 2.8977 | LR: 6.59e-05 | 1339.6ms/step | 14.9 samples/s +Step 33780/44000 | Loss: 2.8976 | LR: 6.58e-05 | 1336.5ms/step | 15.0 samples/s +Step 33790/44000 | Loss: 2.9276 | LR: 6.57e-05 | 1335.2ms/step | 15.0 samples/s +Step 33800/44000 | Loss: 2.9250 | LR: 6.55e-05 | 1331.6ms/step | 15.0 samples/s +Step 33810/44000 | Loss: 2.9243 | LR: 6.54e-05 | 1334.1ms/step | 15.0 samples/s +Step 33820/44000 | Loss: 2.9032 | LR: 6.53e-05 | 1335.6ms/step | 15.0 samples/s +Step 33830/44000 | Loss: 2.9045 | LR: 6.52e-05 | 1336.3ms/step | 15.0 samples/s +Step 33840/44000 | Loss: 2.8852 | LR: 6.50e-05 | 1340.2ms/step | 14.9 samples/s +Step 33850/44000 | Loss: 2.9006 | LR: 6.49e-05 | 1340.4ms/step | 14.9 samples/s +Step 33860/44000 | Loss: 2.8981 | LR: 6.48e-05 | 1340.3ms/step | 14.9 samples/s +Step 33870/44000 | Loss: 2.9192 | LR: 6.47e-05 | 1341.3ms/step | 14.9 samples/s +Step 33880/44000 | Loss: 2.9095 | LR: 6.46e-05 | 1340.6ms/step | 14.9 samples/s +Step 33890/44000 | Loss: 2.9330 | LR: 6.44e-05 | 1339.6ms/step | 14.9 samples/s +Step 33900/44000 | Loss: 2.9076 | LR: 6.43e-05 | 1338.5ms/step | 14.9 samples/s +Step 33910/44000 | Loss: 2.9285 | LR: 6.42e-05 | 1341.1ms/step | 14.9 samples/s +Step 33920/44000 | Loss: 2.8800 | LR: 6.41e-05 | 1343.6ms/step | 14.9 samples/s +Step 33930/44000 | Loss: 2.9052 | LR: 6.40e-05 | 1340.7ms/step | 14.9 samples/s +Step 33940/44000 | Loss: 2.9407 | LR: 6.38e-05 | 1341.1ms/step | 14.9 samples/s +Step 33950/44000 | Loss: 2.9300 | LR: 6.37e-05 | 1342.9ms/step | 14.9 samples/s +Step 33960/44000 | Loss: 2.8986 | LR: 6.36e-05 | 1345.9ms/step | 14.9 samples/s +Step 33970/44000 | Loss: 2.9181 | LR: 6.35e-05 | 1346.8ms/step | 14.9 samples/s +Step 33980/44000 | Loss: 2.9557 | LR: 6.33e-05 | 1346.0ms/step | 14.9 samples/s +Step 33990/44000 | Loss: 2.9493 | LR: 6.32e-05 | 1342.9ms/step | 14.9 samples/s +Step 34000/44000 | Loss: 2.9125 | LR: 6.31e-05 | 1341.7ms/step | 14.9 samples/s +Validation at step 34000: Loss = 2.9080, Perplexity = 18.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-34000 +Step 34010/44000 | Loss: 2.9321 | LR: 6.30e-05 | 3884.0ms/step | 5.1 samples/s +Step 34020/44000 | Loss: 2.8792 | LR: 6.29e-05 | 1333.6ms/step | 15.0 samples/s +Step 34030/44000 | Loss: 2.9212 | LR: 6.27e-05 | 1334.4ms/step | 15.0 samples/s +Step 34040/44000 | Loss: 2.9524 | LR: 6.26e-05 | 1338.2ms/step | 14.9 samples/s +Step 34050/44000 | Loss: 2.9280 | LR: 6.25e-05 | 1335.6ms/step | 15.0 samples/s +Step 34060/44000 | Loss: 2.9395 | LR: 6.24e-05 | 1333.7ms/step | 15.0 samples/s +Step 34070/44000 | Loss: 2.9119 | LR: 6.23e-05 | 1335.2ms/step | 15.0 samples/s +Step 34080/44000 | Loss: 2.9003 | LR: 6.21e-05 | 1333.5ms/step | 15.0 samples/s +Step 34090/44000 | Loss: 2.9670 | LR: 6.20e-05 | 1335.3ms/step | 15.0 samples/s +Step 34100/44000 | Loss: 2.9205 | LR: 6.19e-05 | 1335.7ms/step | 15.0 samples/s +Step 34110/44000 | Loss: 2.8901 | LR: 6.18e-05 | 1337.1ms/step | 15.0 samples/s +Step 34120/44000 | Loss: 2.9046 | LR: 6.17e-05 | 1338.6ms/step | 14.9 samples/s +Step 34130/44000 | Loss: 2.8891 | LR: 6.15e-05 | 1338.4ms/step | 14.9 samples/s +Step 34140/44000 | Loss: 2.9098 | LR: 6.14e-05 | 1337.2ms/step | 15.0 samples/s +Step 34150/44000 | Loss: 2.9346 | LR: 6.13e-05 | 1336.0ms/step | 15.0 samples/s +Step 34160/44000 | Loss: 2.9379 | LR: 6.12e-05 | 1338.0ms/step | 14.9 samples/s +Step 34170/44000 | Loss: 2.9321 | LR: 6.11e-05 | 1340.8ms/step | 14.9 samples/s +Step 34180/44000 | Loss: 2.8983 | LR: 6.09e-05 | 1338.4ms/step | 14.9 samples/s +Step 34190/44000 | Loss: 2.9264 | LR: 6.08e-05 | 1338.9ms/step | 14.9 samples/s +Step 34200/44000 | Loss: 2.9248 | LR: 6.07e-05 | 1339.0ms/step | 14.9 samples/s +Step 34210/44000 | Loss: 2.9299 | LR: 6.06e-05 | 1340.9ms/step | 14.9 samples/s +Step 34220/44000 | Loss: 2.9188 | LR: 6.05e-05 | 1350.0ms/step | 14.8 samples/s +Step 34230/44000 | Loss: 2.9195 | LR: 6.04e-05 | 1346.7ms/step | 14.9 samples/s +Step 34240/44000 | Loss: 2.9223 | LR: 6.02e-05 | 1343.6ms/step | 14.9 samples/s +Step 34250/44000 | Loss: 2.9097 | LR: 6.01e-05 | 1344.1ms/step | 14.9 samples/s +Validation at step 34250: Loss = 2.9077, Perplexity = 18.32 +Step 34260/44000 | Loss: 2.9396 | LR: 6.00e-05 | 3585.1ms/step | 5.6 samples/s +Step 34270/44000 | Loss: 2.9145 | LR: 5.99e-05 | 1343.7ms/step | 14.9 samples/s +Step 34280/44000 | Loss: 2.9395 | LR: 5.98e-05 | 1343.2ms/step | 14.9 samples/s +Step 34290/44000 | Loss: 2.9171 | LR: 5.96e-05 | 1344.8ms/step | 14.9 samples/s +Step 34300/44000 | Loss: 2.9445 | LR: 5.95e-05 | 1344.7ms/step | 14.9 samples/s +Step 34310/44000 | Loss: 2.9201 | LR: 5.94e-05 | 1341.7ms/step | 14.9 samples/s +Step 34320/44000 | Loss: 2.9166 | LR: 5.93e-05 | 1339.0ms/step | 14.9 samples/s +Step 34330/44000 | Loss: 2.9174 | LR: 5.92e-05 | 1340.8ms/step | 14.9 samples/s +Step 34340/44000 | Loss: 2.9266 | LR: 5.91e-05 | 1341.4ms/step | 14.9 samples/s +Step 34350/44000 | Loss: 2.9216 | LR: 5.89e-05 | 1342.7ms/step | 14.9 samples/s +Step 34360/44000 | Loss: 2.9416 | LR: 5.88e-05 | 1341.2ms/step | 14.9 samples/s +Step 34370/44000 | Loss: 2.9205 | LR: 5.87e-05 | 1341.4ms/step | 14.9 samples/s +Step 34380/44000 | Loss: 2.9299 | LR: 5.86e-05 | 1341.5ms/step | 14.9 samples/s +Step 34390/44000 | Loss: 2.9243 | LR: 5.85e-05 | 1342.6ms/step | 14.9 samples/s +Step 34400/44000 | Loss: 2.8753 | LR: 5.84e-05 | 1341.7ms/step | 14.9 samples/s +Step 34410/44000 | Loss: 2.9088 | LR: 5.82e-05 | 1341.7ms/step | 14.9 samples/s +Step 34420/44000 | Loss: 2.9080 | LR: 5.81e-05 | 1340.5ms/step | 14.9 samples/s +Step 34430/44000 | Loss: 2.9231 | LR: 5.80e-05 | 1339.8ms/step | 14.9 samples/s +Step 34440/44000 | Loss: 2.9372 | LR: 5.79e-05 | 1341.6ms/step | 14.9 samples/s +Step 34450/44000 | Loss: 2.8934 | LR: 5.78e-05 | 1340.7ms/step | 14.9 samples/s +Step 34460/44000 | Loss: 2.9173 | LR: 5.77e-05 | 1343.0ms/step | 14.9 samples/s +Step 34470/44000 | Loss: 2.9438 | LR: 5.75e-05 | 1344.3ms/step | 14.9 samples/s +Step 34480/44000 | Loss: 2.9355 | LR: 5.74e-05 | 1343.8ms/step | 14.9 samples/s +Step 34490/44000 | Loss: 2.9217 | LR: 5.73e-05 | 1342.8ms/step | 14.9 samples/s +Step 34500/44000 | Loss: 2.9220 | LR: 5.72e-05 | 1340.6ms/step | 14.9 samples/s +Validation at step 34500: Loss = 2.9074, Perplexity = 18.31 +Step 34510/44000 | Loss: 2.9536 | LR: 5.71e-05 | 3589.6ms/step | 5.6 samples/s +Step 34520/44000 | Loss: 2.8938 | LR: 5.70e-05 | 1348.8ms/step | 14.8 samples/s +Step 34530/44000 | Loss: 2.9065 | LR: 5.69e-05 | 1349.6ms/step | 14.8 samples/s +Step 34540/44000 | Loss: 2.9374 | LR: 5.67e-05 | 1348.3ms/step | 14.8 samples/s +Step 34550/44000 | Loss: 2.8968 | LR: 5.66e-05 | 1343.1ms/step | 14.9 samples/s +Step 34560/44000 | Loss: 2.8852 | LR: 5.65e-05 | 1347.8ms/step | 14.8 samples/s +Step 34570/44000 | Loss: 2.9116 | LR: 5.64e-05 | 1344.1ms/step | 14.9 samples/s +Step 34580/44000 | Loss: 2.9222 | LR: 5.63e-05 | 1345.2ms/step | 14.9 samples/s +Step 34590/44000 | Loss: 2.9057 | LR: 5.62e-05 | 1345.9ms/step | 14.9 samples/s +Step 34600/44000 | Loss: 2.9025 | LR: 5.60e-05 | 1346.2ms/step | 14.9 samples/s +Step 34610/44000 | Loss: 2.9031 | LR: 5.59e-05 | 1351.1ms/step | 14.8 samples/s +Step 34620/44000 | Loss: 2.9240 | LR: 5.58e-05 | 1342.6ms/step | 14.9 samples/s +Step 34630/44000 | Loss: 2.9364 | LR: 5.57e-05 | 1341.5ms/step | 14.9 samples/s +Step 34640/44000 | Loss: 2.9223 | LR: 5.56e-05 | 1339.3ms/step | 14.9 samples/s +Step 34650/44000 | Loss: 2.9342 | LR: 5.55e-05 | 1339.8ms/step | 14.9 samples/s +Step 34660/44000 | Loss: 2.9077 | LR: 5.54e-05 | 1337.6ms/step | 15.0 samples/s +Step 34670/44000 | Loss: 2.9243 | LR: 5.52e-05 | 1338.1ms/step | 14.9 samples/s +Step 34680/44000 | Loss: 2.9624 | LR: 5.51e-05 | 1339.1ms/step | 14.9 samples/s +Step 34690/44000 | Loss: 2.9178 | LR: 5.50e-05 | 1334.5ms/step | 15.0 samples/s +Step 34700/44000 | Loss: 2.9077 | LR: 5.49e-05 | 1335.0ms/step | 15.0 samples/s +Step 34710/44000 | Loss: 2.9154 | LR: 5.48e-05 | 1337.1ms/step | 15.0 samples/s +Step 34720/44000 | Loss: 2.9223 | LR: 5.47e-05 | 1336.9ms/step | 15.0 samples/s +Step 34730/44000 | Loss: 2.9270 | LR: 5.46e-05 | 1337.1ms/step | 15.0 samples/s +Step 34740/44000 | Loss: 2.8876 | LR: 5.45e-05 | 1338.7ms/step | 14.9 samples/s +Step 34750/44000 | Loss: 2.9074 | LR: 5.43e-05 | 1341.0ms/step | 14.9 samples/s +Validation at step 34750: Loss = 2.9071, Perplexity = 18.30 +Step 34760/44000 | Loss: 2.9089 | LR: 5.42e-05 | 3573.4ms/step | 5.6 samples/s +Step 34770/44000 | Loss: 2.9167 | LR: 5.41e-05 | 1338.9ms/step | 14.9 samples/s +Step 34780/44000 | Loss: 2.8880 | LR: 5.40e-05 | 1339.1ms/step | 14.9 samples/s +Step 34790/44000 | Loss: 2.8963 | LR: 5.39e-05 | 1338.7ms/step | 14.9 samples/s +Step 34800/44000 | Loss: 2.8899 | LR: 5.38e-05 | 1339.3ms/step | 14.9 samples/s +Step 34810/44000 | Loss: 2.9217 | LR: 5.37e-05 | 1339.4ms/step | 14.9 samples/s +Step 34820/44000 | Loss: 2.9540 | LR: 5.36e-05 | 1337.6ms/step | 15.0 samples/s +Step 34830/44000 | Loss: 2.9165 | LR: 5.34e-05 | 1337.0ms/step | 15.0 samples/s +Step 34840/44000 | Loss: 2.8936 | LR: 5.33e-05 | 1333.9ms/step | 15.0 samples/s +Step 34850/44000 | Loss: 2.8926 | LR: 5.32e-05 | 1336.2ms/step | 15.0 samples/s +Step 34860/44000 | Loss: 2.9242 | LR: 5.31e-05 | 1340.3ms/step | 14.9 samples/s +Step 34870/44000 | Loss: 2.9177 | LR: 5.30e-05 | 1338.7ms/step | 14.9 samples/s +Step 34880/44000 | Loss: 2.9112 | LR: 5.29e-05 | 1339.9ms/step | 14.9 samples/s +Step 34890/44000 | Loss: 2.9241 | LR: 5.28e-05 | 1337.2ms/step | 15.0 samples/s +Step 34900/44000 | Loss: 2.9315 | LR: 5.27e-05 | 1337.7ms/step | 15.0 samples/s +Step 34910/44000 | Loss: 2.8937 | LR: 5.25e-05 | 1337.7ms/step | 15.0 samples/s +Step 34920/44000 | Loss: 2.9703 | LR: 5.24e-05 | 1336.5ms/step | 15.0 samples/s +Step 34930/44000 | Loss: 2.8867 | LR: 5.23e-05 | 1334.6ms/step | 15.0 samples/s +Step 34940/44000 | Loss: 2.9122 | LR: 5.22e-05 | 1336.4ms/step | 15.0 samples/s +Step 34950/44000 | Loss: 2.9017 | LR: 5.21e-05 | 1336.5ms/step | 15.0 samples/s +Step 34960/44000 | Loss: 2.9173 | LR: 5.20e-05 | 1336.3ms/step | 15.0 samples/s +Step 34970/44000 | Loss: 2.8838 | LR: 5.19e-05 | 1337.6ms/step | 15.0 samples/s +Step 34980/44000 | Loss: 2.9038 | LR: 5.18e-05 | 1339.2ms/step | 14.9 samples/s +Step 34990/44000 | Loss: 2.9449 | LR: 5.17e-05 | 1339.8ms/step | 14.9 samples/s +Step 35000/44000 | Loss: 2.9361 | LR: 5.15e-05 | 1339.6ms/step | 14.9 samples/s +Validation at step 35000: Loss = 2.9069, Perplexity = 18.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-35000 +Step 35010/44000 | Loss: 2.9175 | LR: 5.14e-05 | 3877.0ms/step | 5.2 samples/s +Step 35020/44000 | Loss: 2.9274 | LR: 5.13e-05 | 1332.8ms/step | 15.0 samples/s +Step 35030/44000 | Loss: 2.9116 | LR: 5.12e-05 | 1334.0ms/step | 15.0 samples/s +Step 35040/44000 | Loss: 2.9166 | LR: 5.11e-05 | 1332.8ms/step | 15.0 samples/s +Step 35050/44000 | Loss: 2.9122 | LR: 5.10e-05 | 1336.9ms/step | 15.0 samples/s +Step 35060/44000 | Loss: 2.9199 | LR: 5.09e-05 | 1336.8ms/step | 15.0 samples/s +Step 35070/44000 | Loss: 2.9172 | LR: 5.08e-05 | 1336.6ms/step | 15.0 samples/s +Step 35080/44000 | Loss: 2.9088 | LR: 5.07e-05 | 1336.6ms/step | 15.0 samples/s +Step 35090/44000 | Loss: 2.9038 | LR: 5.06e-05 | 1334.8ms/step | 15.0 samples/s +Step 35100/44000 | Loss: 2.8938 | LR: 5.04e-05 | 1334.6ms/step | 15.0 samples/s +Step 35110/44000 | Loss: 2.9256 | LR: 5.03e-05 | 1334.8ms/step | 15.0 samples/s +Step 35120/44000 | Loss: 2.9160 | LR: 5.02e-05 | 1333.3ms/step | 15.0 samples/s +Step 35130/44000 | Loss: 2.9135 | LR: 5.01e-05 | 1338.1ms/step | 14.9 samples/s +Step 35140/44000 | Loss: 2.9112 | LR: 5.00e-05 | 1337.9ms/step | 14.9 samples/s +Step 35150/44000 | Loss: 2.9213 | LR: 4.99e-05 | 1339.4ms/step | 14.9 samples/s +Step 35160/44000 | Loss: 2.9261 | LR: 4.98e-05 | 1340.6ms/step | 14.9 samples/s +Step 35170/44000 | Loss: 2.9121 | LR: 4.97e-05 | 1339.9ms/step | 14.9 samples/s +Step 35180/44000 | Loss: 2.9099 | LR: 4.96e-05 | 1334.6ms/step | 15.0 samples/s +Step 35190/44000 | Loss: 2.9217 | LR: 4.95e-05 | 1336.1ms/step | 15.0 samples/s +Step 35200/44000 | Loss: 2.8946 | LR: 4.94e-05 | 1337.4ms/step | 15.0 samples/s +Step 35210/44000 | Loss: 2.8979 | LR: 4.93e-05 | 1338.0ms/step | 14.9 samples/s +Step 35220/44000 | Loss: 2.8863 | LR: 4.91e-05 | 1342.2ms/step | 14.9 samples/s +Step 35230/44000 | Loss: 2.9004 | LR: 4.90e-05 | 1339.9ms/step | 14.9 samples/s +Step 35240/44000 | Loss: 2.9236 | LR: 4.89e-05 | 1341.0ms/step | 14.9 samples/s +Step 35250/44000 | Loss: 2.8965 | LR: 4.88e-05 | 1341.1ms/step | 14.9 samples/s +Validation at step 35250: Loss = 2.9067, Perplexity = 18.30 +Step 35260/44000 | Loss: 2.8998 | LR: 4.87e-05 | 3579.1ms/step | 5.6 samples/s +Step 35270/44000 | Loss: 2.9340 | LR: 4.86e-05 | 1340.8ms/step | 14.9 samples/s +Step 35280/44000 | Loss: 2.9383 | LR: 4.85e-05 | 1343.1ms/step | 14.9 samples/s +Step 35290/44000 | Loss: 2.9129 | LR: 4.84e-05 | 1343.9ms/step | 14.9 samples/s +Step 35300/44000 | Loss: 2.9114 | LR: 4.83e-05 | 1344.1ms/step | 14.9 samples/s +Step 35310/44000 | Loss: 2.9105 | LR: 4.82e-05 | 1345.8ms/step | 14.9 samples/s +Step 35320/44000 | Loss: 2.8831 | LR: 4.81e-05 | 1347.3ms/step | 14.8 samples/s +Step 35330/44000 | Loss: 2.9020 | LR: 4.80e-05 | 1345.0ms/step | 14.9 samples/s +Step 35340/44000 | Loss: 2.8940 | LR: 4.79e-05 | 1345.7ms/step | 14.9 samples/s +Step 35350/44000 | Loss: 2.9314 | LR: 4.77e-05 | 1344.9ms/step | 14.9 samples/s +Step 35360/44000 | Loss: 2.9092 | LR: 4.76e-05 | 1342.8ms/step | 14.9 samples/s +Step 35370/44000 | Loss: 2.8916 | LR: 4.75e-05 | 1346.1ms/step | 14.9 samples/s +Step 35380/44000 | Loss: 2.8911 | LR: 4.74e-05 | 1343.0ms/step | 14.9 samples/s +Step 35390/44000 | Loss: 2.9272 | LR: 4.73e-05 | 1344.3ms/step | 14.9 samples/s +Step 35400/44000 | Loss: 2.8992 | LR: 4.72e-05 | 1344.6ms/step | 14.9 samples/s +Step 35410/44000 | Loss: 2.9260 | LR: 4.71e-05 | 1342.9ms/step | 14.9 samples/s +Step 35420/44000 | Loss: 2.9451 | LR: 4.70e-05 | 1344.8ms/step | 14.9 samples/s +Step 35430/44000 | Loss: 2.9282 | LR: 4.69e-05 | 1344.5ms/step | 14.9 samples/s +Step 35440/44000 | Loss: 2.9284 | LR: 4.68e-05 | 1344.8ms/step | 14.9 samples/s +Step 35450/44000 | Loss: 2.9253 | LR: 4.67e-05 | 1345.1ms/step | 14.9 samples/s +Step 35460/44000 | Loss: 2.9337 | LR: 4.66e-05 | 1342.4ms/step | 14.9 samples/s +Step 35470/44000 | Loss: 2.9153 | LR: 4.65e-05 | 1345.7ms/step | 14.9 samples/s +Step 35480/44000 | Loss: 2.9308 | LR: 4.64e-05 | 1344.3ms/step | 14.9 samples/s +Step 35490/44000 | Loss: 2.9144 | LR: 4.63e-05 | 1344.0ms/step | 14.9 samples/s +Step 35500/44000 | Loss: 2.9345 | LR: 4.62e-05 | 1344.3ms/step | 14.9 samples/s +Validation at step 35500: Loss = 2.9064, Perplexity = 18.29 +Step 35510/44000 | Loss: 2.9212 | LR: 4.61e-05 | 3582.9ms/step | 5.6 samples/s +Step 35520/44000 | Loss: 2.9146 | LR: 4.59e-05 | 1344.4ms/step | 14.9 samples/s +Step 35530/44000 | Loss: 2.9278 | LR: 4.58e-05 | 1345.4ms/step | 14.9 samples/s +Step 35540/44000 | Loss: 2.9403 | LR: 4.57e-05 | 1343.5ms/step | 14.9 samples/s +Step 35550/44000 | Loss: 2.9271 | LR: 4.56e-05 | 1342.3ms/step | 14.9 samples/s +Step 35560/44000 | Loss: 2.9228 | LR: 4.55e-05 | 1345.2ms/step | 14.9 samples/s +Step 35570/44000 | Loss: 2.9554 | LR: 4.54e-05 | 1348.9ms/step | 14.8 samples/s +Step 35580/44000 | Loss: 2.8999 | LR: 4.53e-05 | 1346.0ms/step | 14.9 samples/s +Step 35590/44000 | Loss: 2.9379 | LR: 4.52e-05 | 1346.0ms/step | 14.9 samples/s +Step 35600/44000 | Loss: 2.9438 | LR: 4.51e-05 | 1344.3ms/step | 14.9 samples/s +Step 35610/44000 | Loss: 2.8943 | LR: 4.50e-05 | 1345.7ms/step | 14.9 samples/s +Step 35620/44000 | Loss: 2.9379 | LR: 4.49e-05 | 1345.8ms/step | 14.9 samples/s +Step 35630/44000 | Loss: 2.9123 | LR: 4.48e-05 | 1345.3ms/step | 14.9 samples/s +Step 35640/44000 | Loss: 2.8686 | LR: 4.47e-05 | 1343.3ms/step | 14.9 samples/s +Step 35650/44000 | Loss: 2.9160 | LR: 4.46e-05 | 1341.0ms/step | 14.9 samples/s +Step 35660/44000 | Loss: 2.9450 | LR: 4.45e-05 | 1340.8ms/step | 14.9 samples/s +Step 35670/44000 | Loss: 2.9215 | LR: 4.44e-05 | 1339.1ms/step | 14.9 samples/s +Step 35680/44000 | Loss: 2.9041 | LR: 4.43e-05 | 1336.9ms/step | 15.0 samples/s +Step 35690/44000 | Loss: 2.9162 | LR: 4.42e-05 | 1335.2ms/step | 15.0 samples/s +Step 35700/44000 | Loss: 2.8994 | LR: 4.41e-05 | 1333.4ms/step | 15.0 samples/s +Step 35710/44000 | Loss: 2.9187 | LR: 4.40e-05 | 1334.3ms/step | 15.0 samples/s +Step 35720/44000 | Loss: 2.8911 | LR: 4.39e-05 | 1336.4ms/step | 15.0 samples/s +Step 35730/44000 | Loss: 2.9401 | LR: 4.38e-05 | 1336.4ms/step | 15.0 samples/s +Step 35740/44000 | Loss: 2.8836 | LR: 4.37e-05 | 1334.7ms/step | 15.0 samples/s +Step 35750/44000 | Loss: 2.9035 | LR: 4.36e-05 | 1339.6ms/step | 14.9 samples/s +Validation at step 35750: Loss = 2.9062, Perplexity = 18.29 +Step 35760/44000 | Loss: 2.9330 | LR: 4.35e-05 | 3561.8ms/step | 5.6 samples/s +Step 35770/44000 | Loss: 2.9216 | LR: 4.34e-05 | 1332.7ms/step | 15.0 samples/s +Step 35780/44000 | Loss: 2.9044 | LR: 4.33e-05 | 1334.6ms/step | 15.0 samples/s +Step 35790/44000 | Loss: 2.9145 | LR: 4.32e-05 | 1340.7ms/step | 14.9 samples/s +Step 35800/44000 | Loss: 2.9447 | LR: 4.31e-05 | 1339.0ms/step | 14.9 samples/s +Step 35810/44000 | Loss: 2.9282 | LR: 4.29e-05 | 1337.0ms/step | 15.0 samples/s +Step 35820/44000 | Loss: 2.9157 | LR: 4.28e-05 | 1333.9ms/step | 15.0 samples/s +Step 35830/44000 | Loss: 2.9137 | LR: 4.27e-05 | 1331.8ms/step | 15.0 samples/s +Step 35840/44000 | Loss: 2.9245 | LR: 4.26e-05 | 1332.1ms/step | 15.0 samples/s +Step 35850/44000 | Loss: 2.9358 | LR: 4.25e-05 | 1336.1ms/step | 15.0 samples/s +Step 35860/44000 | Loss: 2.9239 | LR: 4.24e-05 | 1339.0ms/step | 14.9 samples/s +Step 35870/44000 | Loss: 2.9027 | LR: 4.23e-05 | 1338.3ms/step | 14.9 samples/s +Step 35880/44000 | Loss: 2.9237 | LR: 4.22e-05 | 1338.7ms/step | 14.9 samples/s +Step 35890/44000 | Loss: 2.9056 | LR: 4.21e-05 | 1338.1ms/step | 14.9 samples/s +Step 35900/44000 | Loss: 2.8769 | LR: 4.20e-05 | 1334.6ms/step | 15.0 samples/s +Step 35910/44000 | Loss: 2.9098 | LR: 4.19e-05 | 1332.5ms/step | 15.0 samples/s +Step 35920/44000 | Loss: 2.8936 | LR: 4.18e-05 | 1331.9ms/step | 15.0 samples/s +Step 35930/44000 | Loss: 2.9228 | LR: 4.17e-05 | 1335.9ms/step | 15.0 samples/s +Step 35940/44000 | Loss: 2.9371 | LR: 4.16e-05 | 1334.8ms/step | 15.0 samples/s +Step 35950/44000 | Loss: 2.9278 | LR: 4.15e-05 | 1336.1ms/step | 15.0 samples/s +Step 35960/44000 | Loss: 2.9277 | LR: 4.14e-05 | 1339.0ms/step | 14.9 samples/s +Step 35970/44000 | Loss: 2.9199 | LR: 4.13e-05 | 1335.7ms/step | 15.0 samples/s +Step 35980/44000 | Loss: 2.8802 | LR: 4.12e-05 | 1333.9ms/step | 15.0 samples/s +Step 35990/44000 | Loss: 2.9102 | LR: 4.11e-05 | 1336.5ms/step | 15.0 samples/s +Step 36000/44000 | Loss: 2.9163 | LR: 4.10e-05 | 1338.8ms/step | 14.9 samples/s +Validation at step 36000: Loss = 2.9061, Perplexity = 18.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-36000 +Step 36010/44000 | Loss: 2.9188 | LR: 4.09e-05 | 3893.5ms/step | 5.1 samples/s +Step 36020/44000 | Loss: 2.9212 | LR: 4.08e-05 | 1337.8ms/step | 14.9 samples/s +Step 36030/44000 | Loss: 2.9152 | LR: 4.07e-05 | 1337.7ms/step | 15.0 samples/s +Step 36040/44000 | Loss: 2.9166 | LR: 4.06e-05 | 1337.6ms/step | 15.0 samples/s +Step 36050/44000 | Loss: 2.9145 | LR: 4.05e-05 | 1335.2ms/step | 15.0 samples/s +Step 36060/44000 | Loss: 2.9140 | LR: 4.04e-05 | 1335.5ms/step | 15.0 samples/s +Step 36070/44000 | Loss: 2.8814 | LR: 4.03e-05 | 1336.1ms/step | 15.0 samples/s +Step 36080/44000 | Loss: 2.9226 | LR: 4.02e-05 | 1339.5ms/step | 14.9 samples/s +Step 36090/44000 | Loss: 2.9071 | LR: 4.01e-05 | 1339.2ms/step | 14.9 samples/s +Step 36100/44000 | Loss: 2.9093 | LR: 4.00e-05 | 1340.3ms/step | 14.9 samples/s +Step 36110/44000 | Loss: 2.9413 | LR: 3.99e-05 | 1338.0ms/step | 14.9 samples/s +Step 36120/44000 | Loss: 2.8855 | LR: 3.98e-05 | 1338.7ms/step | 14.9 samples/s +Step 36130/44000 | Loss: 2.8984 | LR: 3.97e-05 | 1338.4ms/step | 14.9 samples/s +Step 36140/44000 | Loss: 2.9219 | LR: 3.97e-05 | 1338.2ms/step | 14.9 samples/s +Step 36150/44000 | Loss: 2.8969 | LR: 3.96e-05 | 1337.7ms/step | 15.0 samples/s +Step 36160/44000 | Loss: 2.9296 | LR: 3.95e-05 | 1338.7ms/step | 14.9 samples/s +Step 36170/44000 | Loss: 2.9511 | LR: 3.94e-05 | 1338.7ms/step | 14.9 samples/s +Step 36180/44000 | Loss: 2.9378 | LR: 3.93e-05 | 1340.2ms/step | 14.9 samples/s +Step 36190/44000 | Loss: 2.9063 | LR: 3.92e-05 | 1340.6ms/step | 14.9 samples/s +Step 36200/44000 | Loss: 2.9222 | LR: 3.91e-05 | 1342.9ms/step | 14.9 samples/s +Step 36210/44000 | Loss: 2.9271 | LR: 3.90e-05 | 1344.8ms/step | 14.9 samples/s +Step 36220/44000 | Loss: 2.9268 | LR: 3.89e-05 | 1345.0ms/step | 14.9 samples/s +Step 36230/44000 | Loss: 2.8886 | LR: 3.88e-05 | 1343.4ms/step | 14.9 samples/s +Step 36240/44000 | Loss: 2.9182 | LR: 3.87e-05 | 1341.4ms/step | 14.9 samples/s +Step 36250/44000 | Loss: 2.9331 | LR: 3.86e-05 | 1344.0ms/step | 14.9 samples/s +Validation at step 36250: Loss = 2.9059, Perplexity = 18.28 +Step 36260/44000 | Loss: 2.8819 | LR: 3.85e-05 | 3564.7ms/step | 5.6 samples/s +Step 36270/44000 | Loss: 2.8828 | LR: 3.84e-05 | 1335.8ms/step | 15.0 samples/s +Step 36280/44000 | Loss: 2.8951 | LR: 3.83e-05 | 1334.2ms/step | 15.0 samples/s +Step 36290/44000 | Loss: 2.9113 | LR: 3.82e-05 | 1339.2ms/step | 14.9 samples/s +Step 36300/44000 | Loss: 2.9124 | LR: 3.81e-05 | 1340.4ms/step | 14.9 samples/s +Step 36310/44000 | Loss: 2.8903 | LR: 3.80e-05 | 1340.5ms/step | 14.9 samples/s +Step 36320/44000 | Loss: 2.8913 | LR: 3.79e-05 | 1340.1ms/step | 14.9 samples/s +Step 36330/44000 | Loss: 2.8775 | LR: 3.78e-05 | 1342.2ms/step | 14.9 samples/s +Step 36340/44000 | Loss: 2.9087 | LR: 3.77e-05 | 1347.9ms/step | 14.8 samples/s +Step 36350/44000 | Loss: 2.9349 | LR: 3.76e-05 | 1343.9ms/step | 14.9 samples/s +Step 36360/44000 | Loss: 2.9268 | LR: 3.75e-05 | 1342.2ms/step | 14.9 samples/s +Step 36370/44000 | Loss: 2.9313 | LR: 3.74e-05 | 1342.6ms/step | 14.9 samples/s +Step 36380/44000 | Loss: 2.9074 | LR: 3.73e-05 | 1342.9ms/step | 14.9 samples/s +Step 36390/44000 | Loss: 2.9184 | LR: 3.72e-05 | 1341.7ms/step | 14.9 samples/s +Step 36400/44000 | Loss: 2.9347 | LR: 3.71e-05 | 1342.4ms/step | 14.9 samples/s +Step 36410/44000 | Loss: 2.9245 | LR: 3.70e-05 | 1342.2ms/step | 14.9 samples/s +Step 36420/44000 | Loss: 2.9348 | LR: 3.69e-05 | 1344.7ms/step | 14.9 samples/s +Step 36430/44000 | Loss: 2.9276 | LR: 3.69e-05 | 1343.1ms/step | 14.9 samples/s +Step 36440/44000 | Loss: 2.9188 | LR: 3.68e-05 | 1343.7ms/step | 14.9 samples/s +Step 36450/44000 | Loss: 2.8897 | LR: 3.67e-05 | 1342.5ms/step | 14.9 samples/s +Step 36460/44000 | Loss: 2.8868 | LR: 3.66e-05 | 1345.1ms/step | 14.9 samples/s +Step 36470/44000 | Loss: 2.9138 | LR: 3.65e-05 | 1344.1ms/step | 14.9 samples/s +Step 36480/44000 | Loss: 2.9295 | LR: 3.64e-05 | 1344.1ms/step | 14.9 samples/s +Step 36490/44000 | Loss: 2.9095 | LR: 3.63e-05 | 1341.0ms/step | 14.9 samples/s +Step 36500/44000 | Loss: 2.9213 | LR: 3.62e-05 | 1340.3ms/step | 14.9 samples/s +Validation at step 36500: Loss = 2.9058, Perplexity = 18.28 +Step 36510/44000 | Loss: 2.9329 | LR: 3.61e-05 | 3584.4ms/step | 5.6 samples/s +Step 36520/44000 | Loss: 2.9671 | LR: 3.60e-05 | 1345.0ms/step | 14.9 samples/s +Step 36530/44000 | Loss: 2.8872 | LR: 3.59e-05 | 1343.5ms/step | 14.9 samples/s +Step 36540/44000 | Loss: 2.8938 | LR: 3.58e-05 | 1342.3ms/step | 14.9 samples/s +Step 36550/44000 | Loss: 2.9299 | LR: 3.57e-05 | 1341.5ms/step | 14.9 samples/s +Step 36560/44000 | Loss: 2.8863 | LR: 3.56e-05 | 1343.9ms/step | 14.9 samples/s +Step 36570/44000 | Loss: 2.9290 | LR: 3.55e-05 | 1342.8ms/step | 14.9 samples/s +Step 36580/44000 | Loss: 2.9179 | LR: 3.54e-05 | 1342.2ms/step | 14.9 samples/s +Step 36590/44000 | Loss: 2.9177 | LR: 3.53e-05 | 1339.8ms/step | 14.9 samples/s +Step 36600/44000 | Loss: 2.9159 | LR: 3.53e-05 | 1341.2ms/step | 14.9 samples/s +Step 36610/44000 | Loss: 2.9445 | LR: 3.52e-05 | 1341.7ms/step | 14.9 samples/s +Step 36620/44000 | Loss: 2.9476 | LR: 3.51e-05 | 1343.4ms/step | 14.9 samples/s +Step 36630/44000 | Loss: 2.8788 | LR: 3.50e-05 | 1341.1ms/step | 14.9 samples/s +Step 36640/44000 | Loss: 2.9287 | LR: 3.49e-05 | 1338.2ms/step | 14.9 samples/s +Step 36650/44000 | Loss: 2.9081 | LR: 3.48e-05 | 1337.9ms/step | 14.9 samples/s +Step 36660/44000 | Loss: 2.9039 | LR: 3.47e-05 | 1340.2ms/step | 14.9 samples/s +Step 36670/44000 | Loss: 2.9228 | LR: 3.46e-05 | 1339.6ms/step | 14.9 samples/s +Step 36680/44000 | Loss: 2.9025 | LR: 3.45e-05 | 1338.4ms/step | 14.9 samples/s +Step 36690/44000 | Loss: 2.9197 | LR: 3.44e-05 | 1336.6ms/step | 15.0 samples/s +Step 36700/44000 | Loss: 2.9503 | LR: 3.43e-05 | 1341.0ms/step | 14.9 samples/s +Step 36710/44000 | Loss: 2.9286 | LR: 3.42e-05 | 1342.3ms/step | 14.9 samples/s +Step 36720/44000 | Loss: 2.9152 | LR: 3.41e-05 | 1340.5ms/step | 14.9 samples/s +Step 36730/44000 | Loss: 2.8940 | LR: 3.41e-05 | 1337.6ms/step | 15.0 samples/s +Step 36740/44000 | Loss: 2.9412 | LR: 3.40e-05 | 1336.7ms/step | 15.0 samples/s +Step 36750/44000 | Loss: 2.9191 | LR: 3.39e-05 | 1340.2ms/step | 14.9 samples/s +Validation at step 36750: Loss = 2.9057, Perplexity = 18.28 +Step 36760/44000 | Loss: 2.9478 | LR: 3.38e-05 | 3575.8ms/step | 5.6 samples/s +Step 36770/44000 | Loss: 2.9229 | LR: 3.37e-05 | 1337.9ms/step | 14.9 samples/s +Step 36780/44000 | Loss: 2.9339 | LR: 3.36e-05 | 1338.1ms/step | 14.9 samples/s +Step 36790/44000 | Loss: 2.8956 | LR: 3.35e-05 | 1342.1ms/step | 14.9 samples/s +Step 36800/44000 | Loss: 2.8930 | LR: 3.34e-05 | 1341.3ms/step | 14.9 samples/s +Step 36810/44000 | Loss: 2.9011 | LR: 3.33e-05 | 1338.9ms/step | 14.9 samples/s +Step 36820/44000 | Loss: 2.9191 | LR: 3.32e-05 | 1338.4ms/step | 14.9 samples/s +Step 36830/44000 | Loss: 2.9173 | LR: 3.31e-05 | 1336.8ms/step | 15.0 samples/s +Step 36840/44000 | Loss: 2.8907 | LR: 3.31e-05 | 1338.1ms/step | 14.9 samples/s +Step 36850/44000 | Loss: 2.9198 | LR: 3.30e-05 | 1338.4ms/step | 14.9 samples/s +Step 36860/44000 | Loss: 2.9230 | LR: 3.29e-05 | 1340.9ms/step | 14.9 samples/s +Step 36870/44000 | Loss: 2.8983 | LR: 3.28e-05 | 1340.0ms/step | 14.9 samples/s +Step 36880/44000 | Loss: 2.8898 | LR: 3.27e-05 | 1338.6ms/step | 14.9 samples/s +Step 36890/44000 | Loss: 2.9433 | LR: 3.26e-05 | 1337.1ms/step | 15.0 samples/s +Step 36900/44000 | Loss: 2.9089 | LR: 3.25e-05 | 1334.4ms/step | 15.0 samples/s +Step 36910/44000 | Loss: 2.8952 | LR: 3.24e-05 | 1333.2ms/step | 15.0 samples/s +Step 36920/44000 | Loss: 2.8970 | LR: 3.23e-05 | 1335.3ms/step | 15.0 samples/s +Step 36930/44000 | Loss: 2.9270 | LR: 3.22e-05 | 1335.5ms/step | 15.0 samples/s +Step 36940/44000 | Loss: 2.8979 | LR: 3.22e-05 | 1336.3ms/step | 15.0 samples/s +Step 36950/44000 | Loss: 2.9135 | LR: 3.21e-05 | 1338.0ms/step | 14.9 samples/s +Step 36960/44000 | Loss: 2.9188 | LR: 3.20e-05 | 1338.6ms/step | 14.9 samples/s +Step 36970/44000 | Loss: 2.9115 | LR: 3.19e-05 | 1334.0ms/step | 15.0 samples/s +Step 36980/44000 | Loss: 2.9288 | LR: 3.18e-05 | 1335.9ms/step | 15.0 samples/s +Step 36990/44000 | Loss: 2.9082 | LR: 3.17e-05 | 1340.3ms/step | 14.9 samples/s +Step 37000/44000 | Loss: 2.9002 | LR: 3.16e-05 | 1338.7ms/step | 14.9 samples/s +Validation at step 37000: Loss = 2.9055, Perplexity = 18.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-37000 +Step 37010/44000 | Loss: 2.9272 | LR: 3.15e-05 | 3886.9ms/step | 5.1 samples/s +Step 37020/44000 | Loss: 2.9020 | LR: 3.15e-05 | 1334.3ms/step | 15.0 samples/s +Step 37030/44000 | Loss: 2.9443 | LR: 3.14e-05 | 1336.0ms/step | 15.0 samples/s +Step 37040/44000 | Loss: 2.8999 | LR: 3.13e-05 | 1337.7ms/step | 15.0 samples/s +Step 37050/44000 | Loss: 2.9033 | LR: 3.12e-05 | 1339.9ms/step | 14.9 samples/s +Step 37060/44000 | Loss: 2.9043 | LR: 3.11e-05 | 1340.4ms/step | 14.9 samples/s +Step 37070/44000 | Loss: 2.9323 | LR: 3.10e-05 | 1339.7ms/step | 14.9 samples/s +Step 37080/44000 | Loss: 2.9210 | LR: 3.09e-05 | 1339.4ms/step | 14.9 samples/s +Step 37090/44000 | Loss: 2.9381 | LR: 3.08e-05 | 1334.0ms/step | 15.0 samples/s +Step 37100/44000 | Loss: 2.9017 | LR: 3.07e-05 | 1336.6ms/step | 15.0 samples/s +Step 37110/44000 | Loss: 2.8923 | LR: 3.07e-05 | 1338.7ms/step | 14.9 samples/s +Step 37120/44000 | Loss: 2.8749 | LR: 3.06e-05 | 1337.8ms/step | 14.9 samples/s +Step 37130/44000 | Loss: 2.9150 | LR: 3.05e-05 | 1336.7ms/step | 15.0 samples/s +Step 37140/44000 | Loss: 2.9150 | LR: 3.04e-05 | 1338.6ms/step | 14.9 samples/s +Step 37150/44000 | Loss: 2.9132 | LR: 3.03e-05 | 1337.7ms/step | 15.0 samples/s +Step 37160/44000 | Loss: 2.9085 | LR: 3.02e-05 | 1335.2ms/step | 15.0 samples/s +Step 37170/44000 | Loss: 2.9048 | LR: 3.01e-05 | 1334.8ms/step | 15.0 samples/s +Step 37180/44000 | Loss: 2.9276 | LR: 3.01e-05 | 1333.1ms/step | 15.0 samples/s +Step 37190/44000 | Loss: 2.8876 | LR: 3.00e-05 | 1334.1ms/step | 15.0 samples/s +Step 37200/44000 | Loss: 2.9353 | LR: 2.99e-05 | 1337.0ms/step | 15.0 samples/s +Step 37210/44000 | Loss: 2.9172 | LR: 2.98e-05 | 1337.3ms/step | 15.0 samples/s +Step 37220/44000 | Loss: 2.9034 | LR: 2.97e-05 | 1338.6ms/step | 14.9 samples/s +Step 37230/44000 | Loss: 2.9266 | LR: 2.96e-05 | 1341.0ms/step | 14.9 samples/s +Step 37240/44000 | Loss: 2.9211 | LR: 2.95e-05 | 1342.2ms/step | 14.9 samples/s +Step 37250/44000 | Loss: 2.9029 | LR: 2.95e-05 | 1341.9ms/step | 14.9 samples/s +Validation at step 37250: Loss = 2.9054, Perplexity = 18.27 +Step 37260/44000 | Loss: 2.8813 | LR: 2.94e-05 | 3581.3ms/step | 5.6 samples/s +Step 37270/44000 | Loss: 2.9293 | LR: 2.93e-05 | 1338.5ms/step | 14.9 samples/s +Step 37280/44000 | Loss: 2.9229 | LR: 2.92e-05 | 1338.3ms/step | 14.9 samples/s +Step 37290/44000 | Loss: 2.9266 | LR: 2.91e-05 | 1340.2ms/step | 14.9 samples/s +Step 37300/44000 | Loss: 2.9130 | LR: 2.90e-05 | 1339.7ms/step | 14.9 samples/s +Step 37310/44000 | Loss: 2.8898 | LR: 2.89e-05 | 1339.0ms/step | 14.9 samples/s +Step 37320/44000 | Loss: 2.9227 | LR: 2.89e-05 | 1340.2ms/step | 14.9 samples/s +Step 37330/44000 | Loss: 2.9209 | LR: 2.88e-05 | 1341.3ms/step | 14.9 samples/s +Step 37340/44000 | Loss: 2.9315 | LR: 2.87e-05 | 1339.4ms/step | 14.9 samples/s +Step 37350/44000 | Loss: 2.9150 | LR: 2.86e-05 | 1342.4ms/step | 14.9 samples/s +Step 37360/44000 | Loss: 2.9126 | LR: 2.85e-05 | 1342.1ms/step | 14.9 samples/s +Step 37370/44000 | Loss: 2.9286 | LR: 2.84e-05 | 1341.9ms/step | 14.9 samples/s +Step 37380/44000 | Loss: 2.8951 | LR: 2.84e-05 | 1341.6ms/step | 14.9 samples/s +Step 37390/44000 | Loss: 2.9070 | LR: 2.83e-05 | 1342.3ms/step | 14.9 samples/s +Step 37400/44000 | Loss: 2.8781 | LR: 2.82e-05 | 1342.0ms/step | 14.9 samples/s +Step 37410/44000 | Loss: 2.8746 | LR: 2.81e-05 | 1342.1ms/step | 14.9 samples/s +Step 37420/44000 | Loss: 2.9382 | LR: 2.80e-05 | 1340.8ms/step | 14.9 samples/s +Step 37430/44000 | Loss: 2.8713 | LR: 2.79e-05 | 1341.4ms/step | 14.9 samples/s +Step 37440/44000 | Loss: 2.9323 | LR: 2.78e-05 | 1343.5ms/step | 14.9 samples/s +Step 37450/44000 | Loss: 2.9222 | LR: 2.78e-05 | 1344.6ms/step | 14.9 samples/s +Step 37460/44000 | Loss: 2.9194 | LR: 2.77e-05 | 1343.9ms/step | 14.9 samples/s +Step 37470/44000 | Loss: 2.9184 | LR: 2.76e-05 | 1343.7ms/step | 14.9 samples/s +Step 37480/44000 | Loss: 2.9087 | LR: 2.75e-05 | 1343.5ms/step | 14.9 samples/s +Step 37490/44000 | Loss: 2.8990 | LR: 2.74e-05 | 1345.7ms/step | 14.9 samples/s +Step 37500/44000 | Loss: 2.9341 | LR: 2.74e-05 | 1345.1ms/step | 14.9 samples/s +Validation at step 37500: Loss = 2.9053, Perplexity = 18.27 +Step 37510/44000 | Loss: 2.9236 | LR: 2.73e-05 | 3587.4ms/step | 5.6 samples/s +Step 37520/44000 | Loss: 2.9108 | LR: 2.72e-05 | 1345.1ms/step | 14.9 samples/s +Step 37530/44000 | Loss: 2.9215 | LR: 2.71e-05 | 1343.8ms/step | 14.9 samples/s +Step 37540/44000 | Loss: 2.9126 | LR: 2.70e-05 | 1343.4ms/step | 14.9 samples/s +Step 37550/44000 | Loss: 2.9171 | LR: 2.69e-05 | 1345.1ms/step | 14.9 samples/s +Step 37560/44000 | Loss: 2.9129 | LR: 2.69e-05 | 1340.4ms/step | 14.9 samples/s +Step 37570/44000 | Loss: 2.9312 | LR: 2.68e-05 | 1341.1ms/step | 14.9 samples/s +Step 37580/44000 | Loss: 2.8828 | LR: 2.67e-05 | 1339.9ms/step | 14.9 samples/s +Step 37590/44000 | Loss: 2.9200 | LR: 2.66e-05 | 1340.0ms/step | 14.9 samples/s +Step 37600/44000 | Loss: 2.9288 | LR: 2.65e-05 | 1338.3ms/step | 14.9 samples/s +Step 37610/44000 | Loss: 2.8988 | LR: 2.65e-05 | 1341.3ms/step | 14.9 samples/s +Step 37620/44000 | Loss: 2.8967 | LR: 2.64e-05 | 1341.4ms/step | 14.9 samples/s +Step 37630/44000 | Loss: 2.8901 | LR: 2.63e-05 | 1340.2ms/step | 14.9 samples/s +Step 37640/44000 | Loss: 2.9112 | LR: 2.62e-05 | 1341.4ms/step | 14.9 samples/s +Step 37650/44000 | Loss: 2.9184 | LR: 2.61e-05 | 1343.5ms/step | 14.9 samples/s +Step 37660/44000 | Loss: 2.9239 | LR: 2.60e-05 | 1338.8ms/step | 14.9 samples/s +Step 37670/44000 | Loss: 2.9389 | LR: 2.60e-05 | 1338.9ms/step | 14.9 samples/s +Step 37680/44000 | Loss: 2.9244 | LR: 2.59e-05 | 1336.3ms/step | 15.0 samples/s +Step 37690/44000 | Loss: 2.9185 | LR: 2.58e-05 | 1333.4ms/step | 15.0 samples/s +Step 37700/44000 | Loss: 2.9086 | LR: 2.57e-05 | 1335.1ms/step | 15.0 samples/s +Step 37710/44000 | Loss: 2.9061 | LR: 2.56e-05 | 1339.2ms/step | 14.9 samples/s +Step 37720/44000 | Loss: 2.9148 | LR: 2.56e-05 | 1341.9ms/step | 14.9 samples/s +Step 37730/44000 | Loss: 2.8995 | LR: 2.55e-05 | 1343.2ms/step | 14.9 samples/s +Step 37740/44000 | Loss: 2.9018 | LR: 2.54e-05 | 1342.4ms/step | 14.9 samples/s +Step 37750/44000 | Loss: 2.9133 | LR: 2.53e-05 | 1339.7ms/step | 14.9 samples/s +Validation at step 37750: Loss = 2.9052, Perplexity = 18.27 +Step 37760/44000 | Loss: 2.9245 | LR: 2.52e-05 | 3576.2ms/step | 5.6 samples/s +Step 37770/44000 | Loss: 2.9214 | LR: 2.52e-05 | 1340.4ms/step | 14.9 samples/s +Step 37780/44000 | Loss: 2.9071 | LR: 2.51e-05 | 1339.2ms/step | 14.9 samples/s +Step 37790/44000 | Loss: 2.9154 | LR: 2.50e-05 | 1343.7ms/step | 14.9 samples/s +Step 37800/44000 | Loss: 2.9199 | LR: 2.49e-05 | 1340.3ms/step | 14.9 samples/s +Step 37810/44000 | Loss: 2.8744 | LR: 2.48e-05 | 1334.9ms/step | 15.0 samples/s +Step 37820/44000 | Loss: 2.9110 | LR: 2.48e-05 | 1332.9ms/step | 15.0 samples/s +Step 37830/44000 | Loss: 2.9119 | LR: 2.47e-05 | 1330.7ms/step | 15.0 samples/s +Step 37840/44000 | Loss: 2.9257 | LR: 2.46e-05 | 1333.6ms/step | 15.0 samples/s +Step 37850/44000 | Loss: 2.9283 | LR: 2.45e-05 | 1336.8ms/step | 15.0 samples/s +Step 37860/44000 | Loss: 2.9150 | LR: 2.45e-05 | 1334.9ms/step | 15.0 samples/s +Step 37870/44000 | Loss: 2.9137 | LR: 2.44e-05 | 1337.9ms/step | 14.9 samples/s +Step 37880/44000 | Loss: 2.8952 | LR: 2.43e-05 | 1337.3ms/step | 15.0 samples/s +Step 37890/44000 | Loss: 2.8962 | LR: 2.42e-05 | 1339.5ms/step | 14.9 samples/s +Step 37900/44000 | Loss: 2.9384 | LR: 2.41e-05 | 1335.1ms/step | 15.0 samples/s +Step 37910/44000 | Loss: 2.9123 | LR: 2.41e-05 | 1334.9ms/step | 15.0 samples/s +Step 37920/44000 | Loss: 2.9002 | LR: 2.40e-05 | 1334.7ms/step | 15.0 samples/s +Step 37930/44000 | Loss: 2.9040 | LR: 2.39e-05 | 1333.4ms/step | 15.0 samples/s +Step 37940/44000 | Loss: 2.9022 | LR: 2.38e-05 | 1333.6ms/step | 15.0 samples/s +Step 37950/44000 | Loss: 2.9305 | LR: 2.38e-05 | 1334.5ms/step | 15.0 samples/s +Step 37960/44000 | Loss: 2.9001 | LR: 2.37e-05 | 1335.5ms/step | 15.0 samples/s +Step 37970/44000 | Loss: 2.8940 | LR: 2.36e-05 | 1337.6ms/step | 15.0 samples/s +Step 37980/44000 | Loss: 2.9389 | LR: 2.35e-05 | 1335.6ms/step | 15.0 samples/s +Step 37990/44000 | Loss: 2.9150 | LR: 2.34e-05 | 1334.3ms/step | 15.0 samples/s +Step 38000/44000 | Loss: 2.9147 | LR: 2.34e-05 | 1334.0ms/step | 15.0 samples/s +Validation at step 38000: Loss = 2.9052, Perplexity = 18.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-38000 +Step 38010/44000 | Loss: 2.9040 | LR: 2.33e-05 | 3857.6ms/step | 5.2 samples/s +Step 38020/44000 | Loss: 2.8933 | LR: 2.32e-05 | 1331.2ms/step | 15.0 samples/s +Step 38030/44000 | Loss: 2.8878 | LR: 2.31e-05 | 1333.7ms/step | 15.0 samples/s +Step 38040/44000 | Loss: 2.9072 | LR: 2.31e-05 | 1333.2ms/step | 15.0 samples/s +Step 38050/44000 | Loss: 2.9320 | LR: 2.30e-05 | 1333.2ms/step | 15.0 samples/s +Step 38060/44000 | Loss: 2.8986 | LR: 2.29e-05 | 1334.5ms/step | 15.0 samples/s +Step 38070/44000 | Loss: 2.9000 | LR: 2.28e-05 | 1333.1ms/step | 15.0 samples/s +Step 38080/44000 | Loss: 2.9234 | LR: 2.28e-05 | 1332.4ms/step | 15.0 samples/s +Step 38090/44000 | Loss: 2.9273 | LR: 2.27e-05 | 1334.0ms/step | 15.0 samples/s +Step 38100/44000 | Loss: 2.9310 | LR: 2.26e-05 | 1333.4ms/step | 15.0 samples/s +Step 38110/44000 | Loss: 2.9099 | LR: 2.25e-05 | 1331.9ms/step | 15.0 samples/s +Step 38120/44000 | Loss: 2.8970 | LR: 2.25e-05 | 1334.5ms/step | 15.0 samples/s +Step 38130/44000 | Loss: 2.9403 | LR: 2.24e-05 | 1333.9ms/step | 15.0 samples/s +Step 38140/44000 | Loss: 2.9275 | LR: 2.23e-05 | 1339.6ms/step | 14.9 samples/s +Step 38150/44000 | Loss: 2.8976 | LR: 2.22e-05 | 1339.7ms/step | 14.9 samples/s +Step 38160/44000 | Loss: 2.9273 | LR: 2.22e-05 | 1338.3ms/step | 14.9 samples/s +Step 38170/44000 | Loss: 2.9164 | LR: 2.21e-05 | 1339.7ms/step | 14.9 samples/s +Step 38180/44000 | Loss: 2.9054 | LR: 2.20e-05 | 1340.5ms/step | 14.9 samples/s +Step 38190/44000 | Loss: 2.8958 | LR: 2.19e-05 | 1339.9ms/step | 14.9 samples/s +Step 38200/44000 | Loss: 2.9243 | LR: 2.19e-05 | 1339.3ms/step | 14.9 samples/s +Step 38210/44000 | Loss: 2.9385 | LR: 2.18e-05 | 1342.5ms/step | 14.9 samples/s +Step 38220/44000 | Loss: 2.9007 | LR: 2.17e-05 | 1339.5ms/step | 14.9 samples/s +Step 38230/44000 | Loss: 2.8881 | LR: 2.16e-05 | 1340.7ms/step | 14.9 samples/s +Step 38240/44000 | Loss: 2.9009 | LR: 2.16e-05 | 1340.4ms/step | 14.9 samples/s +Step 38250/44000 | Loss: 2.9239 | LR: 2.15e-05 | 1346.8ms/step | 14.9 samples/s +Validation at step 38250: Loss = 2.9051, Perplexity = 18.27 +Step 38260/44000 | Loss: 2.9182 | LR: 2.14e-05 | 3567.9ms/step | 5.6 samples/s +Step 38270/44000 | Loss: 2.9377 | LR: 2.13e-05 | 1340.8ms/step | 14.9 samples/s +Step 38280/44000 | Loss: 2.9196 | LR: 2.13e-05 | 1340.3ms/step | 14.9 samples/s +Step 38290/44000 | Loss: 2.9233 | LR: 2.12e-05 | 1342.9ms/step | 14.9 samples/s +Step 38300/44000 | Loss: 2.9463 | LR: 2.11e-05 | 1340.3ms/step | 14.9 samples/s +Step 38310/44000 | Loss: 2.9068 | LR: 2.11e-05 | 1340.5ms/step | 14.9 samples/s +Step 38320/44000 | Loss: 2.9052 | LR: 2.10e-05 | 1337.6ms/step | 15.0 samples/s +Step 38330/44000 | Loss: 2.9406 | LR: 2.09e-05 | 1337.4ms/step | 15.0 samples/s +Step 38340/44000 | Loss: 2.9033 | LR: 2.08e-05 | 1338.0ms/step | 14.9 samples/s +Step 38350/44000 | Loss: 2.9170 | LR: 2.08e-05 | 1338.5ms/step | 14.9 samples/s +Step 38360/44000 | Loss: 2.9112 | LR: 2.07e-05 | 1340.5ms/step | 14.9 samples/s +Step 38370/44000 | Loss: 2.9543 | LR: 2.06e-05 | 1339.8ms/step | 14.9 samples/s +Step 38380/44000 | Loss: 2.8944 | LR: 2.05e-05 | 1339.6ms/step | 14.9 samples/s +Step 38390/44000 | Loss: 2.8997 | LR: 2.05e-05 | 1335.7ms/step | 15.0 samples/s +Step 38400/44000 | Loss: 2.9389 | LR: 2.04e-05 | 1336.1ms/step | 15.0 samples/s +Step 38410/44000 | Loss: 2.9232 | LR: 2.03e-05 | 1336.4ms/step | 15.0 samples/s +Step 38420/44000 | Loss: 2.8960 | LR: 2.03e-05 | 1340.7ms/step | 14.9 samples/s +Step 38430/44000 | Loss: 2.9440 | LR: 2.02e-05 | 1343.1ms/step | 14.9 samples/s +Step 38440/44000 | Loss: 2.9269 | LR: 2.01e-05 | 1340.0ms/step | 14.9 samples/s +Step 38450/44000 | Loss: 2.9452 | LR: 2.00e-05 | 1342.8ms/step | 14.9 samples/s +Step 38460/44000 | Loss: 2.9173 | LR: 2.00e-05 | 1342.7ms/step | 14.9 samples/s +Step 38470/44000 | Loss: 2.9006 | LR: 1.99e-05 | 1343.6ms/step | 14.9 samples/s +Step 38480/44000 | Loss: 2.9447 | LR: 1.98e-05 | 1343.3ms/step | 14.9 samples/s +Step 38490/44000 | Loss: 2.9125 | LR: 1.98e-05 | 1343.4ms/step | 14.9 samples/s +Step 38500/44000 | Loss: 2.8828 | LR: 1.97e-05 | 1342.7ms/step | 14.9 samples/s +Validation at step 38500: Loss = 2.9051, Perplexity = 18.27 +Step 38510/44000 | Loss: 2.9167 | LR: 1.96e-05 | 3574.5ms/step | 5.6 samples/s +Step 38520/44000 | Loss: 2.8952 | LR: 1.95e-05 | 1342.0ms/step | 14.9 samples/s +Step 38530/44000 | Loss: 2.9187 | LR: 1.95e-05 | 1342.2ms/step | 14.9 samples/s +Step 38540/44000 | Loss: 2.9470 | LR: 1.94e-05 | 1341.6ms/step | 14.9 samples/s +Step 38550/44000 | Loss: 2.8955 | LR: 1.93e-05 | 1341.8ms/step | 14.9 samples/s +Step 38560/44000 | Loss: 2.9385 | LR: 1.93e-05 | 1343.7ms/step | 14.9 samples/s +Step 38570/44000 | Loss: 2.9162 | LR: 1.92e-05 | 1342.1ms/step | 14.9 samples/s +Step 38580/44000 | Loss: 2.9295 | LR: 1.91e-05 | 1345.3ms/step | 14.9 samples/s +Step 38590/44000 | Loss: 2.9258 | LR: 1.91e-05 | 1345.3ms/step | 14.9 samples/s +Step 38600/44000 | Loss: 2.8997 | LR: 1.90e-05 | 1344.2ms/step | 14.9 samples/s +Step 38610/44000 | Loss: 2.9258 | LR: 1.89e-05 | 1346.0ms/step | 14.9 samples/s +Step 38620/44000 | Loss: 2.9324 | LR: 1.88e-05 | 1346.4ms/step | 14.9 samples/s +Step 38630/44000 | Loss: 2.9269 | LR: 1.88e-05 | 1344.8ms/step | 14.9 samples/s +Step 38640/44000 | Loss: 2.9266 | LR: 1.87e-05 | 1345.2ms/step | 14.9 samples/s +Step 38650/44000 | Loss: 2.9010 | LR: 1.86e-05 | 1344.1ms/step | 14.9 samples/s +Step 38660/44000 | Loss: 2.9126 | LR: 1.86e-05 | 1343.7ms/step | 14.9 samples/s +Step 38670/44000 | Loss: 2.9184 | LR: 1.85e-05 | 1342.8ms/step | 14.9 samples/s +Step 38680/44000 | Loss: 2.9500 | LR: 1.84e-05 | 1340.9ms/step | 14.9 samples/s +Step 38690/44000 | Loss: 2.9054 | LR: 1.84e-05 | 1341.1ms/step | 14.9 samples/s +Step 38700/44000 | Loss: 2.9159 | LR: 1.83e-05 | 1343.4ms/step | 14.9 samples/s +Step 38710/44000 | Loss: 2.9056 | LR: 1.82e-05 | 1342.1ms/step | 14.9 samples/s +Step 38720/44000 | Loss: 2.9117 | LR: 1.82e-05 | 1343.2ms/step | 14.9 samples/s +Step 38730/44000 | Loss: 2.8804 | LR: 1.81e-05 | 1342.1ms/step | 14.9 samples/s +Step 38740/44000 | Loss: 2.9024 | LR: 1.80e-05 | 1340.1ms/step | 14.9 samples/s +Step 38750/44000 | Loss: 2.9220 | LR: 1.80e-05 | 1338.3ms/step | 14.9 samples/s +Validation at step 38750: Loss = 2.9050, Perplexity = 18.27 +Step 38760/44000 | Loss: 2.9144 | LR: 1.79e-05 | 3570.7ms/step | 5.6 samples/s +Step 38770/44000 | Loss: 2.8684 | LR: 1.78e-05 | 1339.3ms/step | 14.9 samples/s +Step 38780/44000 | Loss: 2.9458 | LR: 1.78e-05 | 1338.3ms/step | 14.9 samples/s +Step 38790/44000 | Loss: 2.9096 | LR: 1.77e-05 | 1341.9ms/step | 14.9 samples/s +Step 38800/44000 | Loss: 2.9209 | LR: 1.76e-05 | 1344.0ms/step | 14.9 samples/s +Step 38810/44000 | Loss: 2.9261 | LR: 1.76e-05 | 1338.1ms/step | 14.9 samples/s +Step 38820/44000 | Loss: 2.9094 | LR: 1.75e-05 | 1336.7ms/step | 15.0 samples/s +Step 38830/44000 | Loss: 2.9202 | LR: 1.74e-05 | 1338.4ms/step | 14.9 samples/s +Step 38840/44000 | Loss: 2.8970 | LR: 1.74e-05 | 1338.8ms/step | 14.9 samples/s +Step 38850/44000 | Loss: 2.9310 | LR: 1.73e-05 | 1340.1ms/step | 14.9 samples/s +Step 38860/44000 | Loss: 2.9617 | LR: 1.72e-05 | 1342.1ms/step | 14.9 samples/s +Step 38870/44000 | Loss: 2.9185 | LR: 1.72e-05 | 1344.7ms/step | 14.9 samples/s +Step 38880/44000 | Loss: 2.9061 | LR: 1.71e-05 | 1342.3ms/step | 14.9 samples/s +Step 38890/44000 | Loss: 2.9051 | LR: 1.70e-05 | 1341.0ms/step | 14.9 samples/s +Step 38900/44000 | Loss: 2.9030 | LR: 1.70e-05 | 1341.7ms/step | 14.9 samples/s +Step 38910/44000 | Loss: 2.9235 | LR: 1.69e-05 | 1344.1ms/step | 14.9 samples/s +Step 38920/44000 | Loss: 2.9286 | LR: 1.68e-05 | 1343.2ms/step | 14.9 samples/s +Step 38930/44000 | Loss: 2.9061 | LR: 1.68e-05 | 1339.9ms/step | 14.9 samples/s +Step 38940/44000 | Loss: 2.9267 | LR: 1.67e-05 | 1334.8ms/step | 15.0 samples/s +Step 38950/44000 | Loss: 2.9242 | LR: 1.66e-05 | 1337.2ms/step | 15.0 samples/s +Step 38960/44000 | Loss: 2.9089 | LR: 1.66e-05 | 1339.2ms/step | 14.9 samples/s +Step 38970/44000 | Loss: 2.9149 | LR: 1.65e-05 | 1338.9ms/step | 14.9 samples/s +Step 38980/44000 | Loss: 2.9077 | LR: 1.64e-05 | 1338.5ms/step | 14.9 samples/s +Step 38990/44000 | Loss: 2.9179 | LR: 1.64e-05 | 1337.5ms/step | 15.0 samples/s +Step 39000/44000 | Loss: 2.9095 | LR: 1.63e-05 | 1340.8ms/step | 14.9 samples/s +Validation at step 39000: Loss = 2.9050, Perplexity = 18.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-39000 +Step 39010/44000 | Loss: 2.9245 | LR: 1.62e-05 | 3889.8ms/step | 5.1 samples/s +Step 39020/44000 | Loss: 2.9088 | LR: 1.62e-05 | 1340.8ms/step | 14.9 samples/s +Step 39030/44000 | Loss: 2.8945 | LR: 1.61e-05 | 1337.8ms/step | 14.9 samples/s +Step 39040/44000 | Loss: 2.9157 | LR: 1.61e-05 | 1334.4ms/step | 15.0 samples/s +Step 39050/44000 | Loss: 2.9274 | LR: 1.60e-05 | 1340.9ms/step | 14.9 samples/s +Step 39060/44000 | Loss: 2.9079 | LR: 1.59e-05 | 1344.3ms/step | 14.9 samples/s +Step 39070/44000 | Loss: 2.9062 | LR: 1.59e-05 | 1340.6ms/step | 14.9 samples/s +Step 39080/44000 | Loss: 2.9221 | LR: 1.58e-05 | 1339.3ms/step | 14.9 samples/s +Step 39090/44000 | Loss: 2.9357 | LR: 1.57e-05 | 1338.5ms/step | 14.9 samples/s +Step 39100/44000 | Loss: 2.9376 | LR: 1.57e-05 | 1337.0ms/step | 15.0 samples/s +Step 39110/44000 | Loss: 2.9005 | LR: 1.56e-05 | 1339.6ms/step | 14.9 samples/s +Step 39120/44000 | Loss: 2.9052 | LR: 1.55e-05 | 1339.5ms/step | 14.9 samples/s +Step 39130/44000 | Loss: 2.9478 | LR: 1.55e-05 | 1339.8ms/step | 14.9 samples/s +Step 39140/44000 | Loss: 2.8950 | LR: 1.54e-05 | 1341.0ms/step | 14.9 samples/s +Step 39150/44000 | Loss: 2.9005 | LR: 1.54e-05 | 1342.1ms/step | 14.9 samples/s +Step 39160/44000 | Loss: 2.8765 | LR: 1.53e-05 | 1342.3ms/step | 14.9 samples/s +Step 39170/44000 | Loss: 2.9181 | LR: 1.52e-05 | 1343.4ms/step | 14.9 samples/s +Step 39180/44000 | Loss: 2.9128 | LR: 1.52e-05 | 1343.4ms/step | 14.9 samples/s +Step 39190/44000 | Loss: 2.9326 | LR: 1.51e-05 | 1343.8ms/step | 14.9 samples/s +Step 39200/44000 | Loss: 2.9279 | LR: 1.50e-05 | 1337.9ms/step | 14.9 samples/s +Step 39210/44000 | Loss: 2.9114 | LR: 1.50e-05 | 1335.0ms/step | 15.0 samples/s +Step 39220/44000 | Loss: 2.9056 | LR: 1.49e-05 | 1335.8ms/step | 15.0 samples/s +Step 39230/44000 | Loss: 2.8923 | LR: 1.49e-05 | 1334.8ms/step | 15.0 samples/s +Step 39240/44000 | Loss: 2.9227 | LR: 1.48e-05 | 1336.0ms/step | 15.0 samples/s +Step 39250/44000 | Loss: 2.9331 | LR: 1.47e-05 | 1338.4ms/step | 14.9 samples/s +Validation at step 39250: Loss = 2.9050, Perplexity = 18.26 +Step 39260/44000 | Loss: 2.9098 | LR: 1.47e-05 | 3575.1ms/step | 5.6 samples/s +Step 39270/44000 | Loss: 2.9150 | LR: 1.46e-05 | 1341.4ms/step | 14.9 samples/s +Step 39280/44000 | Loss: 2.9045 | LR: 1.46e-05 | 1341.5ms/step | 14.9 samples/s +Step 39290/44000 | Loss: 2.9333 | LR: 1.45e-05 | 1337.7ms/step | 15.0 samples/s +Step 39300/44000 | Loss: 2.9239 | LR: 1.44e-05 | 1338.1ms/step | 14.9 samples/s +Step 39310/44000 | Loss: 2.9167 | LR: 1.44e-05 | 1338.5ms/step | 14.9 samples/s +Step 39320/44000 | Loss: 2.9034 | LR: 1.43e-05 | 1341.0ms/step | 14.9 samples/s +Step 39330/44000 | Loss: 2.9101 | LR: 1.42e-05 | 1344.5ms/step | 14.9 samples/s +Step 39340/44000 | Loss: 2.9170 | LR: 1.42e-05 | 1344.9ms/step | 14.9 samples/s +Step 39350/44000 | Loss: 2.8924 | LR: 1.41e-05 | 1341.8ms/step | 14.9 samples/s +Step 39360/44000 | Loss: 2.8887 | LR: 1.41e-05 | 1340.7ms/step | 14.9 samples/s +Step 39370/44000 | Loss: 2.9099 | LR: 1.40e-05 | 1338.3ms/step | 14.9 samples/s +Step 39380/44000 | Loss: 2.9313 | LR: 1.39e-05 | 1343.5ms/step | 14.9 samples/s +Step 39390/44000 | Loss: 2.9174 | LR: 1.39e-05 | 1350.5ms/step | 14.8 samples/s +Step 39400/44000 | Loss: 2.9242 | LR: 1.38e-05 | 1350.8ms/step | 14.8 samples/s +Step 39410/44000 | Loss: 2.9287 | LR: 1.38e-05 | 1343.8ms/step | 14.9 samples/s +Step 39420/44000 | Loss: 2.9185 | LR: 1.37e-05 | 1343.6ms/step | 14.9 samples/s +Step 39430/44000 | Loss: 2.8972 | LR: 1.36e-05 | 1340.8ms/step | 14.9 samples/s +Step 39440/44000 | Loss: 2.8970 | LR: 1.36e-05 | 1341.8ms/step | 14.9 samples/s +Step 39450/44000 | Loss: 2.9525 | LR: 1.35e-05 | 1340.7ms/step | 14.9 samples/s +Step 39460/44000 | Loss: 2.9059 | LR: 1.35e-05 | 1343.3ms/step | 14.9 samples/s +Step 39470/44000 | Loss: 2.9240 | LR: 1.34e-05 | 1348.3ms/step | 14.8 samples/s +Step 39480/44000 | Loss: 2.9340 | LR: 1.34e-05 | 1345.0ms/step | 14.9 samples/s +Step 39490/44000 | Loss: 2.8924 | LR: 1.33e-05 | 1345.4ms/step | 14.9 samples/s +Step 39500/44000 | Loss: 2.9202 | LR: 1.32e-05 | 1341.8ms/step | 14.9 samples/s +Validation at step 39500: Loss = 2.9049, Perplexity = 18.26 +Step 39510/44000 | Loss: 2.8827 | LR: 1.32e-05 | 3582.7ms/step | 5.6 samples/s +Step 39520/44000 | Loss: 2.9216 | LR: 1.31e-05 | 1341.7ms/step | 14.9 samples/s +Step 39530/44000 | Loss: 2.9073 | LR: 1.31e-05 | 1340.9ms/step | 14.9 samples/s +Step 39540/44000 | Loss: 2.9481 | LR: 1.30e-05 | 1340.0ms/step | 14.9 samples/s +Step 39550/44000 | Loss: 2.9342 | LR: 1.29e-05 | 1344.3ms/step | 14.9 samples/s +Step 39560/44000 | Loss: 2.9191 | LR: 1.29e-05 | 1339.6ms/step | 14.9 samples/s +Step 39570/44000 | Loss: 2.9332 | LR: 1.28e-05 | 1334.7ms/step | 15.0 samples/s +Step 39580/44000 | Loss: 2.9334 | LR: 1.28e-05 | 1339.4ms/step | 14.9 samples/s +Step 39590/44000 | Loss: 2.9089 | LR: 1.27e-05 | 1341.2ms/step | 14.9 samples/s +Step 39600/44000 | Loss: 2.9098 | LR: 1.27e-05 | 1342.2ms/step | 14.9 samples/s +Step 39610/44000 | Loss: 2.9096 | LR: 1.26e-05 | 1341.7ms/step | 14.9 samples/s +Step 39620/44000 | Loss: 2.8903 | LR: 1.25e-05 | 1340.6ms/step | 14.9 samples/s +Step 39630/44000 | Loss: 2.9259 | LR: 1.25e-05 | 1339.6ms/step | 14.9 samples/s +Step 39640/44000 | Loss: 2.9097 | LR: 1.24e-05 | 1339.5ms/step | 14.9 samples/s +Step 39650/44000 | Loss: 2.9038 | LR: 1.24e-05 | 1344.1ms/step | 14.9 samples/s +Step 39660/44000 | Loss: 2.9288 | LR: 1.23e-05 | 1342.0ms/step | 14.9 samples/s +Step 39670/44000 | Loss: 2.9188 | LR: 1.23e-05 | 1341.5ms/step | 14.9 samples/s +Step 39680/44000 | Loss: 2.9133 | LR: 1.22e-05 | 1341.1ms/step | 14.9 samples/s +Step 39690/44000 | Loss: 2.8972 | LR: 1.22e-05 | 1339.6ms/step | 14.9 samples/s +Step 39700/44000 | Loss: 2.9004 | LR: 1.21e-05 | 1335.9ms/step | 15.0 samples/s +Step 39710/44000 | Loss: 2.9010 | LR: 1.20e-05 | 1337.6ms/step | 15.0 samples/s +Step 39720/44000 | Loss: 2.9327 | LR: 1.20e-05 | 1338.7ms/step | 14.9 samples/s +Step 39730/44000 | Loss: 2.9027 | LR: 1.19e-05 | 1337.0ms/step | 15.0 samples/s +Step 39740/44000 | Loss: 2.9060 | LR: 1.19e-05 | 1340.6ms/step | 14.9 samples/s +Step 39750/44000 | Loss: 2.8945 | LR: 1.18e-05 | 1341.7ms/step | 14.9 samples/s +Validation at step 39750: Loss = 2.9049, Perplexity = 18.26 +Step 39760/44000 | Loss: 2.8983 | LR: 1.18e-05 | 3564.4ms/step | 5.6 samples/s +Step 39770/44000 | Loss: 2.9214 | LR: 1.17e-05 | 1336.7ms/step | 15.0 samples/s +Step 39780/44000 | Loss: 2.9165 | LR: 1.17e-05 | 1339.3ms/step | 14.9 samples/s +Step 39790/44000 | Loss: 2.8985 | LR: 1.16e-05 | 1338.5ms/step | 14.9 samples/s +Step 39800/44000 | Loss: 2.9134 | LR: 1.15e-05 | 1341.7ms/step | 14.9 samples/s +Step 39810/44000 | Loss: 2.9214 | LR: 1.15e-05 | 1340.7ms/step | 14.9 samples/s +Step 39820/44000 | Loss: 2.9202 | LR: 1.14e-05 | 1341.3ms/step | 14.9 samples/s +Step 39830/44000 | Loss: 2.9030 | LR: 1.14e-05 | 1341.1ms/step | 14.9 samples/s +Step 39840/44000 | Loss: 2.9057 | LR: 1.13e-05 | 1337.9ms/step | 14.9 samples/s +Step 39850/44000 | Loss: 2.9060 | LR: 1.13e-05 | 1337.9ms/step | 14.9 samples/s +Step 39860/44000 | Loss: 2.9482 | LR: 1.12e-05 | 1339.5ms/step | 14.9 samples/s +Step 39870/44000 | Loss: 2.9150 | LR: 1.12e-05 | 1337.8ms/step | 15.0 samples/s +Step 39880/44000 | Loss: 2.8822 | LR: 1.11e-05 | 1332.8ms/step | 15.0 samples/s +Step 39890/44000 | Loss: 2.8962 | LR: 1.11e-05 | 1336.6ms/step | 15.0 samples/s +Step 39900/44000 | Loss: 2.9275 | LR: 1.10e-05 | 1334.0ms/step | 15.0 samples/s +Step 39910/44000 | Loss: 2.9184 | LR: 1.10e-05 | 1333.3ms/step | 15.0 samples/s +Step 39920/44000 | Loss: 2.9192 | LR: 1.09e-05 | 1337.4ms/step | 15.0 samples/s +Step 39930/44000 | Loss: 2.9060 | LR: 1.08e-05 | 1339.7ms/step | 14.9 samples/s +Step 39940/44000 | Loss: 2.9223 | LR: 1.08e-05 | 1339.7ms/step | 14.9 samples/s +Step 39950/44000 | Loss: 2.9178 | LR: 1.07e-05 | 1338.4ms/step | 14.9 samples/s +Step 39960/44000 | Loss: 2.9147 | LR: 1.07e-05 | 1335.5ms/step | 15.0 samples/s +Step 39970/44000 | Loss: 2.9215 | LR: 1.06e-05 | 1336.8ms/step | 15.0 samples/s +Step 39980/44000 | Loss: 2.8934 | LR: 1.06e-05 | 1337.8ms/step | 14.9 samples/s +Step 39990/44000 | Loss: 2.9234 | LR: 1.05e-05 | 1342.1ms/step | 14.9 samples/s +Step 40000/44000 | Loss: 2.9120 | LR: 1.05e-05 | 1340.6ms/step | 14.9 samples/s +Validation at step 40000: Loss = 2.9049, Perplexity = 18.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-40000 +Step 40010/44000 | Loss: 2.8941 | LR: 1.04e-05 | 3906.7ms/step | 5.1 samples/s +Step 40020/44000 | Loss: 2.9179 | LR: 1.04e-05 | 1338.1ms/step | 14.9 samples/s +Step 40030/44000 | Loss: 2.9262 | LR: 1.03e-05 | 1338.4ms/step | 14.9 samples/s +Step 40040/44000 | Loss: 2.9229 | LR: 1.03e-05 | 1343.5ms/step | 14.9 samples/s +Step 40050/44000 | Loss: 2.9209 | LR: 1.02e-05 | 1344.0ms/step | 14.9 samples/s +Step 40060/44000 | Loss: 2.9314 | LR: 1.02e-05 | 1343.2ms/step | 14.9 samples/s +Step 40070/44000 | Loss: 2.9340 | LR: 1.01e-05 | 1341.1ms/step | 14.9 samples/s +Step 40080/44000 | Loss: 2.8910 | LR: 1.01e-05 | 1339.8ms/step | 14.9 samples/s +Step 40090/44000 | Loss: 2.9322 | LR: 1.00e-05 | 1337.1ms/step | 15.0 samples/s +Step 40100/44000 | Loss: 2.9227 | LR: 9.96e-06 | 1339.5ms/step | 14.9 samples/s +Step 40110/44000 | Loss: 2.9312 | LR: 9.91e-06 | 1340.5ms/step | 14.9 samples/s +Step 40120/44000 | Loss: 2.9527 | LR: 9.86e-06 | 1340.3ms/step | 14.9 samples/s +Step 40130/44000 | Loss: 2.9412 | LR: 9.81e-06 | 1340.6ms/step | 14.9 samples/s +Step 40140/44000 | Loss: 2.8992 | LR: 9.76e-06 | 1341.2ms/step | 14.9 samples/s +Step 40150/44000 | Loss: 2.9180 | LR: 9.71e-06 | 1339.0ms/step | 14.9 samples/s +Step 40160/44000 | Loss: 2.9226 | LR: 9.66e-06 | 1340.5ms/step | 14.9 samples/s +Step 40170/44000 | Loss: 2.8961 | LR: 9.61e-06 | 1341.9ms/step | 14.9 samples/s +Step 40180/44000 | Loss: 2.9135 | LR: 9.56e-06 | 1343.1ms/step | 14.9 samples/s +Step 40190/44000 | Loss: 2.9019 | LR: 9.51e-06 | 1339.5ms/step | 14.9 samples/s +Step 40200/44000 | Loss: 2.9166 | LR: 9.46e-06 | 1339.2ms/step | 14.9 samples/s +Step 40210/44000 | Loss: 2.9011 | LR: 9.41e-06 | 1339.7ms/step | 14.9 samples/s +Step 40220/44000 | Loss: 2.8918 | LR: 9.36e-06 | 1338.1ms/step | 14.9 samples/s +Step 40230/44000 | Loss: 2.9137 | LR: 9.32e-06 | 1335.2ms/step | 15.0 samples/s +Step 40240/44000 | Loss: 2.9183 | LR: 9.27e-06 | 1336.5ms/step | 15.0 samples/s +Step 40250/44000 | Loss: 2.9413 | LR: 9.22e-06 | 1337.0ms/step | 15.0 samples/s +Validation at step 40250: Loss = 2.9049, Perplexity = 18.26 +Step 40260/44000 | Loss: 2.9218 | LR: 9.17e-06 | 3575.9ms/step | 5.6 samples/s +Step 40270/44000 | Loss: 2.8953 | LR: 9.12e-06 | 1342.7ms/step | 14.9 samples/s +Step 40280/44000 | Loss: 2.8998 | LR: 9.07e-06 | 1343.3ms/step | 14.9 samples/s +Step 40290/44000 | Loss: 2.8954 | LR: 9.02e-06 | 1342.8ms/step | 14.9 samples/s +Step 40300/44000 | Loss: 2.9232 | LR: 8.97e-06 | 1343.8ms/step | 14.9 samples/s +Step 40310/44000 | Loss: 2.9310 | LR: 8.93e-06 | 1345.7ms/step | 14.9 samples/s +Step 40320/44000 | Loss: 2.9109 | LR: 8.88e-06 | 1338.3ms/step | 14.9 samples/s +Step 40330/44000 | Loss: 2.9379 | LR: 8.83e-06 | 1338.9ms/step | 14.9 samples/s +Step 40340/44000 | Loss: 2.9481 | LR: 8.78e-06 | 1341.6ms/step | 14.9 samples/s +Step 40350/44000 | Loss: 2.9213 | LR: 8.74e-06 | 1341.9ms/step | 14.9 samples/s +Step 40360/44000 | Loss: 2.8930 | LR: 8.69e-06 | 1343.6ms/step | 14.9 samples/s +Step 40370/44000 | Loss: 2.8994 | LR: 8.64e-06 | 1343.4ms/step | 14.9 samples/s +Step 40380/44000 | Loss: 2.9189 | LR: 8.59e-06 | 1340.8ms/step | 14.9 samples/s +Step 40390/44000 | Loss: 2.8850 | LR: 8.55e-06 | 1338.8ms/step | 14.9 samples/s +Step 40400/44000 | Loss: 2.9553 | LR: 8.50e-06 | 1338.0ms/step | 14.9 samples/s +Step 40410/44000 | Loss: 2.9169 | LR: 8.45e-06 | 1340.3ms/step | 14.9 samples/s +Step 40420/44000 | Loss: 2.9039 | LR: 8.41e-06 | 1343.1ms/step | 14.9 samples/s +Step 40430/44000 | Loss: 2.8978 | LR: 8.36e-06 | 1342.7ms/step | 14.9 samples/s +Step 40440/44000 | Loss: 2.9112 | LR: 8.31e-06 | 1342.0ms/step | 14.9 samples/s +Step 40450/44000 | Loss: 2.9143 | LR: 8.27e-06 | 1340.7ms/step | 14.9 samples/s +Step 40460/44000 | Loss: 2.9421 | LR: 8.22e-06 | 1338.5ms/step | 14.9 samples/s +Step 40470/44000 | Loss: 2.9197 | LR: 8.17e-06 | 1339.0ms/step | 14.9 samples/s +Step 40480/44000 | Loss: 2.9112 | LR: 8.13e-06 | 1339.2ms/step | 14.9 samples/s +Step 40490/44000 | Loss: 2.9308 | LR: 8.08e-06 | 1339.9ms/step | 14.9 samples/s +Step 40500/44000 | Loss: 2.9162 | LR: 8.04e-06 | 1341.0ms/step | 14.9 samples/s +Validation at step 40500: Loss = 2.9049, Perplexity = 18.26 +Step 40510/44000 | Loss: 2.8674 | LR: 7.99e-06 | 3574.8ms/step | 5.6 samples/s +Step 40520/44000 | Loss: 2.9186 | LR: 7.94e-06 | 1340.3ms/step | 14.9 samples/s +Step 40530/44000 | Loss: 2.9177 | LR: 7.90e-06 | 1339.4ms/step | 14.9 samples/s +Step 40540/44000 | Loss: 2.8984 | LR: 7.85e-06 | 1340.6ms/step | 14.9 samples/s +Step 40550/44000 | Loss: 2.9289 | LR: 7.81e-06 | 1340.7ms/step | 14.9 samples/s +Step 40560/44000 | Loss: 2.9282 | LR: 7.76e-06 | 1341.5ms/step | 14.9 samples/s +Step 40570/44000 | Loss: 2.9046 | LR: 7.72e-06 | 1340.6ms/step | 14.9 samples/s +Step 40580/44000 | Loss: 2.9081 | LR: 7.67e-06 | 1341.5ms/step | 14.9 samples/s +Step 40590/44000 | Loss: 2.9071 | LR: 7.63e-06 | 1340.3ms/step | 14.9 samples/s +Step 40600/44000 | Loss: 2.9054 | LR: 7.59e-06 | 1339.8ms/step | 14.9 samples/s +Step 40610/44000 | Loss: 2.9019 | LR: 7.54e-06 | 1341.0ms/step | 14.9 samples/s +Step 40620/44000 | Loss: 2.9018 | LR: 7.50e-06 | 1337.1ms/step | 15.0 samples/s +Step 40630/44000 | Loss: 2.9196 | LR: 7.45e-06 | 1336.1ms/step | 15.0 samples/s +Step 40640/44000 | Loss: 2.8930 | LR: 7.41e-06 | 1337.4ms/step | 15.0 samples/s +Step 40650/44000 | Loss: 2.9119 | LR: 7.37e-06 | 1340.1ms/step | 14.9 samples/s +Step 40660/44000 | Loss: 2.9536 | LR: 7.32e-06 | 1341.0ms/step | 14.9 samples/s +Step 40670/44000 | Loss: 2.9508 | LR: 7.28e-06 | 1342.2ms/step | 14.9 samples/s +Step 40680/44000 | Loss: 2.8964 | LR: 7.23e-06 | 1339.9ms/step | 14.9 samples/s +Step 40690/44000 | Loss: 2.8749 | LR: 7.19e-06 | 1338.8ms/step | 14.9 samples/s +Step 40700/44000 | Loss: 2.9211 | LR: 7.15e-06 | 1340.7ms/step | 14.9 samples/s +Step 40710/44000 | Loss: 2.8966 | LR: 7.10e-06 | 1342.3ms/step | 14.9 samples/s +Step 40720/44000 | Loss: 2.9114 | LR: 7.06e-06 | 1341.8ms/step | 14.9 samples/s +Step 40730/44000 | Loss: 2.9189 | LR: 7.02e-06 | 1340.7ms/step | 14.9 samples/s +Step 40740/44000 | Loss: 2.9189 | LR: 6.98e-06 | 1340.2ms/step | 14.9 samples/s +Step 40750/44000 | Loss: 2.9135 | LR: 6.93e-06 | 1340.3ms/step | 14.9 samples/s +Validation at step 40750: Loss = 2.9049, Perplexity = 18.26 +Step 40760/44000 | Loss: 2.9435 | LR: 6.89e-06 | 3572.6ms/step | 5.6 samples/s +Step 40770/44000 | Loss: 2.9177 | LR: 6.85e-06 | 1341.2ms/step | 14.9 samples/s +Step 40780/44000 | Loss: 2.9033 | LR: 6.81e-06 | 1341.0ms/step | 14.9 samples/s +Step 40790/44000 | Loss: 2.8989 | LR: 6.77e-06 | 1340.4ms/step | 14.9 samples/s +Step 40800/44000 | Loss: 2.8998 | LR: 6.72e-06 | 1338.5ms/step | 14.9 samples/s +Step 40810/44000 | Loss: 2.9095 | LR: 6.68e-06 | 1341.7ms/step | 14.9 samples/s +Step 40820/44000 | Loss: 2.9202 | LR: 6.64e-06 | 1341.9ms/step | 14.9 samples/s +Step 40830/44000 | Loss: 2.9140 | LR: 6.60e-06 | 1341.2ms/step | 14.9 samples/s +Step 40840/44000 | Loss: 2.9329 | LR: 6.56e-06 | 1340.1ms/step | 14.9 samples/s +Step 40850/44000 | Loss: 2.9236 | LR: 6.52e-06 | 1340.1ms/step | 14.9 samples/s +Step 40860/44000 | Loss: 2.9118 | LR: 6.47e-06 | 1341.1ms/step | 14.9 samples/s +Step 40870/44000 | Loss: 2.9139 | LR: 6.43e-06 | 1345.1ms/step | 14.9 samples/s +Step 40880/44000 | Loss: 2.8598 | LR: 6.39e-06 | 1343.1ms/step | 14.9 samples/s +Step 40890/44000 | Loss: 2.8907 | LR: 6.35e-06 | 1344.2ms/step | 14.9 samples/s +Step 40900/44000 | Loss: 2.9014 | LR: 6.31e-06 | 1342.7ms/step | 14.9 samples/s +Step 40910/44000 | Loss: 2.9235 | LR: 6.27e-06 | 1340.8ms/step | 14.9 samples/s +Step 40920/44000 | Loss: 2.9212 | LR: 6.23e-06 | 1337.8ms/step | 14.9 samples/s +Step 40930/44000 | Loss: 2.9475 | LR: 6.19e-06 | 1333.1ms/step | 15.0 samples/s +Step 40940/44000 | Loss: 2.8770 | LR: 6.15e-06 | 1332.2ms/step | 15.0 samples/s +Step 40950/44000 | Loss: 2.8995 | LR: 6.11e-06 | 1333.3ms/step | 15.0 samples/s +Step 40960/44000 | Loss: 2.8906 | LR: 6.07e-06 | 1333.5ms/step | 15.0 samples/s +Step 40970/44000 | Loss: 2.9146 | LR: 6.03e-06 | 1332.4ms/step | 15.0 samples/s +Step 40980/44000 | Loss: 2.9097 | LR: 5.99e-06 | 1333.2ms/step | 15.0 samples/s +Step 40990/44000 | Loss: 2.9058 | LR: 5.95e-06 | 1335.0ms/step | 15.0 samples/s +Step 41000/44000 | Loss: 2.8865 | LR: 5.91e-06 | 1337.4ms/step | 15.0 samples/s +Validation at step 41000: Loss = 2.9049, Perplexity = 18.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-41000 +Step 41010/44000 | Loss: 2.9035 | LR: 5.87e-06 | 3883.2ms/step | 5.2 samples/s +Step 41020/44000 | Loss: 2.9032 | LR: 5.83e-06 | 1340.4ms/step | 14.9 samples/s +Step 41030/44000 | Loss: 2.9189 | LR: 5.80e-06 | 1343.5ms/step | 14.9 samples/s +Step 41040/44000 | Loss: 2.9303 | LR: 5.76e-06 | 1342.8ms/step | 14.9 samples/s +Step 41050/44000 | Loss: 2.9231 | LR: 5.72e-06 | 1341.4ms/step | 14.9 samples/s +Step 41060/44000 | Loss: 2.8926 | LR: 5.68e-06 | 1342.2ms/step | 14.9 samples/s +Step 41070/44000 | Loss: 2.8960 | LR: 5.64e-06 | 1342.3ms/step | 14.9 samples/s +Step 41080/44000 | Loss: 2.9224 | LR: 5.60e-06 | 1341.5ms/step | 14.9 samples/s +Step 41090/44000 | Loss: 2.9395 | LR: 5.56e-06 | 1338.3ms/step | 14.9 samples/s +Step 41100/44000 | Loss: 2.9068 | LR: 5.53e-06 | 1335.2ms/step | 15.0 samples/s +Step 41110/44000 | Loss: 2.8936 | LR: 5.49e-06 | 1336.4ms/step | 15.0 samples/s +Step 41120/44000 | Loss: 2.9208 | LR: 5.45e-06 | 1341.4ms/step | 14.9 samples/s +Step 41130/44000 | Loss: 2.9144 | LR: 5.41e-06 | 1341.8ms/step | 14.9 samples/s +Step 41140/44000 | Loss: 2.8920 | LR: 5.38e-06 | 1343.2ms/step | 14.9 samples/s +Step 41150/44000 | Loss: 2.9197 | LR: 5.34e-06 | 1343.2ms/step | 14.9 samples/s +Step 41160/44000 | Loss: 2.9067 | LR: 5.30e-06 | 1342.6ms/step | 14.9 samples/s +Step 41170/44000 | Loss: 2.8982 | LR: 5.26e-06 | 1342.3ms/step | 14.9 samples/s +Step 41180/44000 | Loss: 2.9032 | LR: 5.23e-06 | 1338.8ms/step | 14.9 samples/s +Step 41190/44000 | Loss: 2.9339 | LR: 5.19e-06 | 1336.5ms/step | 15.0 samples/s +Step 41200/44000 | Loss: 2.8998 | LR: 5.15e-06 | 1336.3ms/step | 15.0 samples/s +Step 41210/44000 | Loss: 2.9140 | LR: 5.12e-06 | 1340.0ms/step | 14.9 samples/s +Step 41220/44000 | Loss: 2.9056 | LR: 5.08e-06 | 1344.4ms/step | 14.9 samples/s +Step 41230/44000 | Loss: 2.9024 | LR: 5.04e-06 | 1347.5ms/step | 14.8 samples/s +Step 41240/44000 | Loss: 2.9234 | LR: 5.01e-06 | 1343.8ms/step | 14.9 samples/s +Step 41250/44000 | Loss: 2.8962 | LR: 4.97e-06 | 1340.9ms/step | 14.9 samples/s +Validation at step 41250: Loss = 2.9049, Perplexity = 18.26 +Step 41260/44000 | Loss: 2.9343 | LR: 4.94e-06 | 3567.8ms/step | 5.6 samples/s +Step 41270/44000 | Loss: 2.8761 | LR: 4.90e-06 | 1344.5ms/step | 14.9 samples/s +Step 41280/44000 | Loss: 2.9339 | LR: 4.86e-06 | 1344.2ms/step | 14.9 samples/s +Step 41290/44000 | Loss: 2.9295 | LR: 4.83e-06 | 1343.8ms/step | 14.9 samples/s +Step 41300/44000 | Loss: 2.9262 | LR: 4.79e-06 | 1343.3ms/step | 14.9 samples/s +Step 41310/44000 | Loss: 2.9133 | LR: 4.76e-06 | 1340.8ms/step | 14.9 samples/s +Step 41320/44000 | Loss: 2.9410 | LR: 4.72e-06 | 1334.4ms/step | 15.0 samples/s +Step 41330/44000 | Loss: 2.8875 | LR: 4.69e-06 | 1333.9ms/step | 15.0 samples/s +Step 41340/44000 | Loss: 2.9363 | LR: 4.65e-06 | 1337.2ms/step | 15.0 samples/s +Step 41350/44000 | Loss: 2.9414 | LR: 4.62e-06 | 1337.2ms/step | 15.0 samples/s +Step 41360/44000 | Loss: 2.8899 | LR: 4.58e-06 | 1335.8ms/step | 15.0 samples/s +Step 41370/44000 | Loss: 2.9243 | LR: 4.55e-06 | 1335.7ms/step | 15.0 samples/s +Step 41380/44000 | Loss: 2.9107 | LR: 4.51e-06 | 1338.2ms/step | 14.9 samples/s +Step 41390/44000 | Loss: 2.9083 | LR: 4.48e-06 | 1340.9ms/step | 14.9 samples/s +Step 41400/44000 | Loss: 2.9199 | LR: 4.45e-06 | 1347.6ms/step | 14.8 samples/s +Step 41410/44000 | Loss: 2.9544 | LR: 4.41e-06 | 1344.7ms/step | 14.9 samples/s +Step 41420/44000 | Loss: 2.9264 | LR: 4.38e-06 | 1345.9ms/step | 14.9 samples/s +Step 41430/44000 | Loss: 2.8947 | LR: 4.34e-06 | 1343.4ms/step | 14.9 samples/s +Step 41440/44000 | Loss: 2.9024 | LR: 4.31e-06 | 1340.3ms/step | 14.9 samples/s +Step 41450/44000 | Loss: 2.9102 | LR: 4.28e-06 | 1335.1ms/step | 15.0 samples/s +Step 41460/44000 | Loss: 2.9087 | LR: 4.24e-06 | 1337.5ms/step | 15.0 samples/s +Step 41470/44000 | Loss: 2.9422 | LR: 4.21e-06 | 1339.7ms/step | 14.9 samples/s +Step 41480/44000 | Loss: 2.9193 | LR: 4.18e-06 | 1339.0ms/step | 14.9 samples/s +Step 41490/44000 | Loss: 2.9258 | LR: 4.14e-06 | 1334.1ms/step | 15.0 samples/s +Step 41500/44000 | Loss: 2.8904 | LR: 4.11e-06 | 1333.7ms/step | 15.0 samples/s +Validation at step 41500: Loss = 2.9048, Perplexity = 18.26 +Step 41510/44000 | Loss: 2.9282 | LR: 4.08e-06 | 3563.4ms/step | 5.6 samples/s +Step 41520/44000 | Loss: 2.9145 | LR: 4.05e-06 | 1340.1ms/step | 14.9 samples/s +Step 41530/44000 | Loss: 2.9320 | LR: 4.01e-06 | 1339.5ms/step | 14.9 samples/s +Step 41540/44000 | Loss: 2.9266 | LR: 3.98e-06 | 1339.3ms/step | 14.9 samples/s +Step 41550/44000 | Loss: 2.8960 | LR: 3.95e-06 | 1339.2ms/step | 14.9 samples/s +Step 41560/44000 | Loss: 2.9190 | LR: 3.92e-06 | 1337.5ms/step | 15.0 samples/s +Step 41570/44000 | Loss: 2.9026 | LR: 3.88e-06 | 1338.4ms/step | 14.9 samples/s +Step 41580/44000 | Loss: 2.8834 | LR: 3.85e-06 | 1340.7ms/step | 14.9 samples/s +Step 41590/44000 | Loss: 2.9249 | LR: 3.82e-06 | 1339.5ms/step | 14.9 samples/s +Step 41600/44000 | Loss: 2.8754 | LR: 3.79e-06 | 1343.2ms/step | 14.9 samples/s +Step 41610/44000 | Loss: 2.8981 | LR: 3.76e-06 | 1346.9ms/step | 14.8 samples/s +Step 41620/44000 | Loss: 2.9146 | LR: 3.73e-06 | 1339.3ms/step | 14.9 samples/s +Step 41630/44000 | Loss: 2.8844 | LR: 3.70e-06 | 1339.7ms/step | 14.9 samples/s +Step 41640/44000 | Loss: 2.9265 | LR: 3.66e-06 | 1338.1ms/step | 14.9 samples/s +Step 41650/44000 | Loss: 2.9096 | LR: 3.63e-06 | 1338.7ms/step | 14.9 samples/s +Step 41660/44000 | Loss: 2.9070 | LR: 3.60e-06 | 1340.3ms/step | 14.9 samples/s +Step 41670/44000 | Loss: 2.9375 | LR: 3.57e-06 | 1335.6ms/step | 15.0 samples/s +Step 41680/44000 | Loss: 2.9145 | LR: 3.54e-06 | 1338.8ms/step | 14.9 samples/s +Step 41690/44000 | Loss: 2.9006 | LR: 3.51e-06 | 1336.8ms/step | 15.0 samples/s +Step 41700/44000 | Loss: 2.8883 | LR: 3.48e-06 | 1335.9ms/step | 15.0 samples/s +Step 41710/44000 | Loss: 2.8985 | LR: 3.45e-06 | 1335.5ms/step | 15.0 samples/s +Step 41720/44000 | Loss: 2.9390 | LR: 3.42e-06 | 1337.0ms/step | 15.0 samples/s +Step 41730/44000 | Loss: 2.9344 | LR: 3.39e-06 | 1339.1ms/step | 14.9 samples/s +Step 41740/44000 | Loss: 2.9581 | LR: 3.36e-06 | 1338.1ms/step | 14.9 samples/s +Step 41750/44000 | Loss: 2.9007 | LR: 3.33e-06 | 1337.7ms/step | 15.0 samples/s +Validation at step 41750: Loss = 2.9049, Perplexity = 18.26 +Step 41760/44000 | Loss: 2.9134 | LR: 3.30e-06 | 3564.3ms/step | 5.6 samples/s +Step 41770/44000 | Loss: 2.8842 | LR: 3.27e-06 | 1331.8ms/step | 15.0 samples/s +Step 41780/44000 | Loss: 2.9034 | LR: 3.24e-06 | 1335.2ms/step | 15.0 samples/s +Step 41790/44000 | Loss: 2.9137 | LR: 3.21e-06 | 1336.7ms/step | 15.0 samples/s +Step 41800/44000 | Loss: 2.9616 | LR: 3.19e-06 | 1338.7ms/step | 14.9 samples/s +Step 41810/44000 | Loss: 2.9246 | LR: 3.16e-06 | 1336.2ms/step | 15.0 samples/s +Step 41820/44000 | Loss: 2.9449 | LR: 3.13e-06 | 1341.2ms/step | 14.9 samples/s +Step 41830/44000 | Loss: 2.8869 | LR: 3.10e-06 | 1339.7ms/step | 14.9 samples/s +Step 41840/44000 | Loss: 2.9146 | LR: 3.07e-06 | 1340.4ms/step | 14.9 samples/s +Step 41850/44000 | Loss: 2.9336 | LR: 3.04e-06 | 1338.0ms/step | 14.9 samples/s +Step 41860/44000 | Loss: 2.9014 | LR: 3.01e-06 | 1337.4ms/step | 15.0 samples/s +Step 41870/44000 | Loss: 2.9010 | LR: 2.99e-06 | 1341.1ms/step | 14.9 samples/s +Step 41880/44000 | Loss: 2.8766 | LR: 2.96e-06 | 1340.1ms/step | 14.9 samples/s +Step 41890/44000 | Loss: 2.9265 | LR: 2.93e-06 | 1337.1ms/step | 15.0 samples/s +Step 41900/44000 | Loss: 2.9049 | LR: 2.90e-06 | 1336.0ms/step | 15.0 samples/s +Step 41910/44000 | Loss: 2.9019 | LR: 2.88e-06 | 1331.9ms/step | 15.0 samples/s +Step 41920/44000 | Loss: 2.9163 | LR: 2.85e-06 | 1331.7ms/step | 15.0 samples/s +Step 41930/44000 | Loss: 2.8960 | LR: 2.82e-06 | 1333.1ms/step | 15.0 samples/s +Step 41940/44000 | Loss: 2.9369 | LR: 2.79e-06 | 1335.7ms/step | 15.0 samples/s +Step 41950/44000 | Loss: 2.9068 | LR: 2.77e-06 | 1338.1ms/step | 14.9 samples/s +Step 41960/44000 | Loss: 2.8893 | LR: 2.74e-06 | 1338.3ms/step | 14.9 samples/s +Step 41970/44000 | Loss: 2.9116 | LR: 2.71e-06 | 1338.6ms/step | 14.9 samples/s +Step 41980/44000 | Loss: 2.9232 | LR: 2.69e-06 | 1342.0ms/step | 14.9 samples/s +Step 41990/44000 | Loss: 2.9456 | LR: 2.66e-06 | 1341.5ms/step | 14.9 samples/s +Step 42000/44000 | Loss: 2.8832 | LR: 2.63e-06 | 1340.8ms/step | 14.9 samples/s +Validation at step 42000: Loss = 2.9049, Perplexity = 18.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-42000 +Step 42010/44000 | Loss: 2.9243 | LR: 2.61e-06 | 3901.2ms/step | 5.1 samples/s +Step 42020/44000 | Loss: 2.9189 | LR: 2.58e-06 | 1335.7ms/step | 15.0 samples/s +Step 42030/44000 | Loss: 2.9048 | LR: 2.56e-06 | 1341.6ms/step | 14.9 samples/s +Step 42040/44000 | Loss: 2.9078 | LR: 2.53e-06 | 1340.5ms/step | 14.9 samples/s +Step 42050/44000 | Loss: 2.9086 | LR: 2.50e-06 | 1340.6ms/step | 14.9 samples/s +Step 42060/44000 | Loss: 2.9160 | LR: 2.48e-06 | 1341.5ms/step | 14.9 samples/s +Step 42070/44000 | Loss: 2.9365 | LR: 2.45e-06 | 1341.7ms/step | 14.9 samples/s +Step 42080/44000 | Loss: 2.8959 | LR: 2.43e-06 | 1342.3ms/step | 14.9 samples/s +Step 42090/44000 | Loss: 2.9097 | LR: 2.40e-06 | 1338.6ms/step | 14.9 samples/s +Step 42100/44000 | Loss: 2.8865 | LR: 2.38e-06 | 1342.5ms/step | 14.9 samples/s +Step 42110/44000 | Loss: 2.9331 | LR: 2.35e-06 | 1342.0ms/step | 14.9 samples/s +Step 42120/44000 | Loss: 2.9151 | LR: 2.33e-06 | 1343.0ms/step | 14.9 samples/s +Step 42130/44000 | Loss: 2.9191 | LR: 2.30e-06 | 1343.1ms/step | 14.9 samples/s +Step 42140/44000 | Loss: 2.9211 | LR: 2.28e-06 | 1342.2ms/step | 14.9 samples/s +Step 42150/44000 | Loss: 2.8805 | LR: 2.25e-06 | 1345.1ms/step | 14.9 samples/s +Step 42160/44000 | Loss: 2.8869 | LR: 2.23e-06 | 1344.5ms/step | 14.9 samples/s +Step 42170/44000 | Loss: 2.8921 | LR: 2.21e-06 | 1343.9ms/step | 14.9 samples/s +Step 42180/44000 | Loss: 2.9316 | LR: 2.18e-06 | 1343.9ms/step | 14.9 samples/s +Step 42190/44000 | Loss: 2.9010 | LR: 2.16e-06 | 1350.6ms/step | 14.8 samples/s +Step 42200/44000 | Loss: 2.9445 | LR: 2.13e-06 | 1344.2ms/step | 14.9 samples/s +Step 42210/44000 | Loss: 2.9439 | LR: 2.11e-06 | 1339.8ms/step | 14.9 samples/s +Step 42220/44000 | Loss: 2.9104 | LR: 2.09e-06 | 1339.2ms/step | 14.9 samples/s +Step 42230/44000 | Loss: 2.9231 | LR: 2.06e-06 | 1337.9ms/step | 14.9 samples/s +Step 42240/44000 | Loss: 2.9216 | LR: 2.04e-06 | 1335.2ms/step | 15.0 samples/s +Step 42250/44000 | Loss: 2.9286 | LR: 2.02e-06 | 1333.6ms/step | 15.0 samples/s +Validation at step 42250: Loss = 2.9048, Perplexity = 18.26 +Step 42260/44000 | Loss: 2.8874 | LR: 1.99e-06 | 3555.4ms/step | 5.6 samples/s +Step 42270/44000 | Loss: 2.9209 | LR: 1.97e-06 | 1333.6ms/step | 15.0 samples/s +Step 42280/44000 | Loss: 2.9069 | LR: 1.95e-06 | 1331.2ms/step | 15.0 samples/s +Step 42290/44000 | Loss: 2.9236 | LR: 1.93e-06 | 1329.2ms/step | 15.0 samples/s +Step 42300/44000 | Loss: 2.9094 | LR: 1.90e-06 | 1326.7ms/step | 15.1 samples/s +Step 42310/44000 | Loss: 2.8855 | LR: 1.88e-06 | 1331.6ms/step | 15.0 samples/s +Step 42320/44000 | Loss: 2.9058 | LR: 1.86e-06 | 1330.4ms/step | 15.0 samples/s +Step 42330/44000 | Loss: 2.9124 | LR: 1.84e-06 | 1331.8ms/step | 15.0 samples/s +Step 42340/44000 | Loss: 2.9240 | LR: 1.82e-06 | 1334.3ms/step | 15.0 samples/s +Step 42350/44000 | Loss: 2.9322 | LR: 1.79e-06 | 1337.3ms/step | 15.0 samples/s +Step 42360/44000 | Loss: 2.9133 | LR: 1.77e-06 | 1334.7ms/step | 15.0 samples/s +Step 42370/44000 | Loss: 2.8968 | LR: 1.75e-06 | 1332.4ms/step | 15.0 samples/s +Step 42380/44000 | Loss: 2.9085 | LR: 1.73e-06 | 1328.8ms/step | 15.1 samples/s +Step 42390/44000 | Loss: 2.9374 | LR: 1.71e-06 | 1327.8ms/step | 15.1 samples/s +Step 42400/44000 | Loss: 2.9232 | LR: 1.69e-06 | 1328.6ms/step | 15.1 samples/s +Step 42410/44000 | Loss: 2.9261 | LR: 1.67e-06 | 1329.9ms/step | 15.0 samples/s +Step 42420/44000 | Loss: 2.8896 | LR: 1.64e-06 | 1332.8ms/step | 15.0 samples/s +Step 42430/44000 | Loss: 2.8946 | LR: 1.62e-06 | 1333.0ms/step | 15.0 samples/s +Step 42440/44000 | Loss: 2.9145 | LR: 1.60e-06 | 1332.3ms/step | 15.0 samples/s +Step 42450/44000 | Loss: 2.9032 | LR: 1.58e-06 | 1333.2ms/step | 15.0 samples/s +Step 42460/44000 | Loss: 2.9240 | LR: 1.56e-06 | 1334.7ms/step | 15.0 samples/s +Step 42470/44000 | Loss: 2.9070 | LR: 1.54e-06 | 1334.4ms/step | 15.0 samples/s +Step 42480/44000 | Loss: 2.9290 | LR: 1.52e-06 | 1334.5ms/step | 15.0 samples/s +Step 42490/44000 | Loss: 2.9408 | LR: 1.50e-06 | 1330.1ms/step | 15.0 samples/s +Step 42500/44000 | Loss: 2.8968 | LR: 1.48e-06 | 1328.7ms/step | 15.1 samples/s +Validation at step 42500: Loss = 2.9048, Perplexity = 18.26 +Step 42510/44000 | Loss: 2.9273 | LR: 1.46e-06 | 3558.7ms/step | 5.6 samples/s +Step 42520/44000 | Loss: 2.8987 | LR: 1.44e-06 | 1335.2ms/step | 15.0 samples/s +Step 42530/44000 | Loss: 2.9128 | LR: 1.42e-06 | 1335.4ms/step | 15.0 samples/s +Step 42540/44000 | Loss: 2.9506 | LR: 1.40e-06 | 1331.3ms/step | 15.0 samples/s +Step 42550/44000 | Loss: 2.9287 | LR: 1.39e-06 | 1329.8ms/step | 15.0 samples/s +Step 42560/44000 | Loss: 2.8821 | LR: 1.37e-06 | 1324.8ms/step | 15.1 samples/s +Step 42570/44000 | Loss: 2.9387 | LR: 1.35e-06 | 1330.5ms/step | 15.0 samples/s +Step 42580/44000 | Loss: 2.9122 | LR: 1.33e-06 | 1331.5ms/step | 15.0 samples/s +Step 42590/44000 | Loss: 2.8957 | LR: 1.31e-06 | 1332.2ms/step | 15.0 samples/s +Step 42600/44000 | Loss: 2.9169 | LR: 1.29e-06 | 1333.2ms/step | 15.0 samples/s +Step 42610/44000 | Loss: 2.8926 | LR: 1.27e-06 | 1332.1ms/step | 15.0 samples/s +Step 42620/44000 | Loss: 2.9155 | LR: 1.25e-06 | 1332.1ms/step | 15.0 samples/s +Step 42630/44000 | Loss: 2.9319 | LR: 1.24e-06 | 1332.4ms/step | 15.0 samples/s +Step 42640/44000 | Loss: 2.9166 | LR: 1.22e-06 | 1332.7ms/step | 15.0 samples/s +Step 42650/44000 | Loss: 2.9144 | LR: 1.20e-06 | 1331.4ms/step | 15.0 samples/s +Step 42660/44000 | Loss: 2.9415 | LR: 1.18e-06 | 1334.1ms/step | 15.0 samples/s +Step 42670/44000 | Loss: 2.9067 | LR: 1.17e-06 | 1331.9ms/step | 15.0 samples/s +Step 42680/44000 | Loss: 2.9219 | LR: 1.15e-06 | 1327.5ms/step | 15.1 samples/s +Step 42690/44000 | Loss: 2.9183 | LR: 1.13e-06 | 1327.0ms/step | 15.1 samples/s +Step 42700/44000 | Loss: 2.9076 | LR: 1.11e-06 | 1330.6ms/step | 15.0 samples/s +Step 42710/44000 | Loss: 2.8911 | LR: 1.10e-06 | 1331.5ms/step | 15.0 samples/s +Step 42720/44000 | Loss: 2.9235 | LR: 1.08e-06 | 1327.9ms/step | 15.1 samples/s +Step 42730/44000 | Loss: 2.9354 | LR: 1.06e-06 | 1328.9ms/step | 15.1 samples/s +Step 42740/44000 | Loss: 2.8913 | LR: 1.05e-06 | 1332.6ms/step | 15.0 samples/s +Step 42750/44000 | Loss: 2.9445 | LR: 1.03e-06 | 1332.5ms/step | 15.0 samples/s +Validation at step 42750: Loss = 2.9048, Perplexity = 18.26 +Step 42760/44000 | Loss: 2.9243 | LR: 1.01e-06 | 3557.0ms/step | 5.6 samples/s +Step 42770/44000 | Loss: 2.9192 | LR: 9.97e-07 | 1333.7ms/step | 15.0 samples/s +Step 42780/44000 | Loss: 2.9197 | LR: 9.81e-07 | 1330.8ms/step | 15.0 samples/s +Step 42790/44000 | Loss: 2.9205 | LR: 9.65e-07 | 1331.9ms/step | 15.0 samples/s +Step 42800/44000 | Loss: 2.9117 | LR: 9.49e-07 | 1329.2ms/step | 15.0 samples/s +Step 42810/44000 | Loss: 2.9178 | LR: 9.33e-07 | 1330.1ms/step | 15.0 samples/s +Step 42820/44000 | Loss: 2.9166 | LR: 9.18e-07 | 1328.4ms/step | 15.1 samples/s +Step 42830/44000 | Loss: 2.8991 | LR: 9.02e-07 | 1329.4ms/step | 15.0 samples/s +Step 42840/44000 | Loss: 2.9451 | LR: 8.87e-07 | 1340.9ms/step | 14.9 samples/s +Step 42850/44000 | Loss: 2.8943 | LR: 8.72e-07 | 1360.7ms/step | 14.7 samples/s +Step 42860/44000 | Loss: 2.9342 | LR: 8.57e-07 | 1349.1ms/step | 14.8 samples/s +Step 42870/44000 | Loss: 2.9092 | LR: 8.42e-07 | 1345.8ms/step | 14.9 samples/s +Step 42880/44000 | Loss: 2.9240 | LR: 8.27e-07 | 1336.4ms/step | 15.0 samples/s +Step 42890/44000 | Loss: 2.9089 | LR: 8.12e-07 | 1325.8ms/step | 15.1 samples/s +Step 42900/44000 | Loss: 2.8844 | LR: 7.98e-07 | 1328.4ms/step | 15.1 samples/s +Step 42910/44000 | Loss: 2.9294 | LR: 7.83e-07 | 1330.5ms/step | 15.0 samples/s +Step 42920/44000 | Loss: 2.8792 | LR: 7.69e-07 | 1332.4ms/step | 15.0 samples/s +Step 42930/44000 | Loss: 2.9105 | LR: 7.55e-07 | 1334.5ms/step | 15.0 samples/s +Step 42940/44000 | Loss: 2.9189 | LR: 7.41e-07 | 1332.2ms/step | 15.0 samples/s +Step 42950/44000 | Loss: 2.9337 | LR: 7.27e-07 | 1329.7ms/step | 15.0 samples/s +Step 42960/44000 | Loss: 2.8999 | LR: 7.13e-07 | 1332.7ms/step | 15.0 samples/s +Step 42970/44000 | Loss: 2.8999 | LR: 6.99e-07 | 1333.9ms/step | 15.0 samples/s +Step 42980/44000 | Loss: 2.9144 | LR: 6.86e-07 | 1332.1ms/step | 15.0 samples/s +Step 42990/44000 | Loss: 2.8801 | LR: 6.72e-07 | 1331.8ms/step | 15.0 samples/s +Step 43000/44000 | Loss: 2.9184 | LR: 6.59e-07 | 1332.6ms/step | 15.0 samples/s +Validation at step 43000: Loss = 2.9048, Perplexity = 18.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-43000 +Step 43010/44000 | Loss: 2.9442 | LR: 6.46e-07 | 3874.4ms/step | 5.2 samples/s +Step 43020/44000 | Loss: 2.9285 | LR: 6.33e-07 | 1333.4ms/step | 15.0 samples/s +Step 43030/44000 | Loss: 2.9287 | LR: 6.20e-07 | 1332.4ms/step | 15.0 samples/s +Step 43040/44000 | Loss: 2.9168 | LR: 6.08e-07 | 1331.1ms/step | 15.0 samples/s +Step 43050/44000 | Loss: 2.9314 | LR: 5.95e-07 | 1331.5ms/step | 15.0 samples/s +Step 43060/44000 | Loss: 2.9318 | LR: 5.83e-07 | 1333.1ms/step | 15.0 samples/s +Step 43070/44000 | Loss: 2.9296 | LR: 5.70e-07 | 1333.1ms/step | 15.0 samples/s +Step 43080/44000 | Loss: 2.8956 | LR: 5.58e-07 | 1334.2ms/step | 15.0 samples/s +Step 43090/44000 | Loss: 2.8914 | LR: 5.46e-07 | 1333.2ms/step | 15.0 samples/s +Step 43100/44000 | Loss: 2.9075 | LR: 5.34e-07 | 1335.0ms/step | 15.0 samples/s +Step 43110/44000 | Loss: 2.9407 | LR: 5.22e-07 | 1331.8ms/step | 15.0 samples/s +Step 43120/44000 | Loss: 2.9144 | LR: 5.11e-07 | 1330.9ms/step | 15.0 samples/s +Step 43130/44000 | Loss: 2.9268 | LR: 4.99e-07 | 1330.7ms/step | 15.0 samples/s +Step 43140/44000 | Loss: 2.9480 | LR: 4.88e-07 | 1331.0ms/step | 15.0 samples/s +Step 43150/44000 | Loss: 2.8971 | LR: 4.76e-07 | 1333.0ms/step | 15.0 samples/s +Step 43160/44000 | Loss: 2.9268 | LR: 4.65e-07 | 1333.5ms/step | 15.0 samples/s +Step 43170/44000 | Loss: 2.8599 | LR: 4.54e-07 | 1337.0ms/step | 15.0 samples/s +Step 43180/44000 | Loss: 2.8994 | LR: 4.43e-07 | 1334.3ms/step | 15.0 samples/s +Step 43190/44000 | Loss: 2.9308 | LR: 4.33e-07 | 1336.2ms/step | 15.0 samples/s +Step 43200/44000 | Loss: 2.9319 | LR: 4.22e-07 | 1334.3ms/step | 15.0 samples/s +Step 43210/44000 | Loss: 2.9139 | LR: 4.12e-07 | 1334.4ms/step | 15.0 samples/s +Step 43220/44000 | Loss: 2.9018 | LR: 4.01e-07 | 1335.3ms/step | 15.0 samples/s +Step 43230/44000 | Loss: 2.9276 | LR: 3.91e-07 | 1332.3ms/step | 15.0 samples/s +Step 43240/44000 | Loss: 2.9121 | LR: 3.81e-07 | 1332.8ms/step | 15.0 samples/s +Step 43250/44000 | Loss: 2.9108 | LR: 3.71e-07 | 1337.5ms/step | 15.0 samples/s +Validation at step 43250: Loss = 2.9048, Perplexity = 18.26 +Step 43260/44000 | Loss: 2.8945 | LR: 3.61e-07 | 3561.0ms/step | 5.6 samples/s +Step 43270/44000 | Loss: 2.9316 | LR: 3.51e-07 | 1332.4ms/step | 15.0 samples/s +Step 43280/44000 | Loss: 2.9212 | LR: 3.42e-07 | 1333.8ms/step | 15.0 samples/s +Step 43290/44000 | Loss: 2.9245 | LR: 3.32e-07 | 1333.9ms/step | 15.0 samples/s +Step 43300/44000 | Loss: 2.9101 | LR: 3.23e-07 | 1333.5ms/step | 15.0 samples/s +Step 43310/44000 | Loss: 2.9099 | LR: 3.14e-07 | 1333.5ms/step | 15.0 samples/s +Step 43320/44000 | Loss: 2.9003 | LR: 3.05e-07 | 1338.0ms/step | 14.9 samples/s +Step 43330/44000 | Loss: 2.8944 | LR: 2.96e-07 | 1340.7ms/step | 14.9 samples/s +Step 43340/44000 | Loss: 2.9145 | LR: 2.87e-07 | 1332.0ms/step | 15.0 samples/s +Step 43350/44000 | Loss: 2.9150 | LR: 2.79e-07 | 1328.6ms/step | 15.1 samples/s +Step 43360/44000 | Loss: 2.9158 | LR: 2.70e-07 | 1329.5ms/step | 15.0 samples/s +Step 43370/44000 | Loss: 2.9227 | LR: 2.62e-07 | 1333.9ms/step | 15.0 samples/s +Step 43380/44000 | Loss: 2.9195 | LR: 2.53e-07 | 1329.0ms/step | 15.0 samples/s +Step 43390/44000 | Loss: 2.9180 | LR: 2.45e-07 | 1328.3ms/step | 15.1 samples/s +Step 43400/44000 | Loss: 2.9123 | LR: 2.37e-07 | 1329.1ms/step | 15.0 samples/s +Step 43410/44000 | Loss: 2.9091 | LR: 2.30e-07 | 1326.9ms/step | 15.1 samples/s +Step 43420/44000 | Loss: 2.9379 | LR: 2.22e-07 | 1330.2ms/step | 15.0 samples/s +Step 43430/44000 | Loss: 2.9174 | LR: 2.14e-07 | 1333.3ms/step | 15.0 samples/s +Step 43440/44000 | Loss: 2.9045 | LR: 2.07e-07 | 1333.7ms/step | 15.0 samples/s +Step 43450/44000 | Loss: 2.9093 | LR: 1.99e-07 | 1334.4ms/step | 15.0 samples/s +Step 43460/44000 | Loss: 2.8967 | LR: 1.92e-07 | 1333.5ms/step | 15.0 samples/s +Step 43470/44000 | Loss: 2.9106 | LR: 1.85e-07 | 1342.9ms/step | 14.9 samples/s +Step 43480/44000 | Loss: 2.9058 | LR: 1.78e-07 | 1342.3ms/step | 14.9 samples/s +Step 43490/44000 | Loss: 2.9273 | LR: 1.72e-07 | 1344.1ms/step | 14.9 samples/s +Step 43500/44000 | Loss: 2.9077 | LR: 1.65e-07 | 1344.2ms/step | 14.9 samples/s +Validation at step 43500: Loss = 2.9048, Perplexity = 18.26 +Step 43510/44000 | Loss: 2.9002 | LR: 1.58e-07 | 3591.9ms/step | 5.6 samples/s +Step 43520/44000 | Loss: 2.8926 | LR: 1.52e-07 | 1344.7ms/step | 14.9 samples/s +Step 43530/44000 | Loss: 2.9114 | LR: 1.46e-07 | 1340.9ms/step | 14.9 samples/s +Step 43540/44000 | Loss: 2.9043 | LR: 1.40e-07 | 1339.4ms/step | 14.9 samples/s +Step 43550/44000 | Loss: 2.8982 | LR: 1.34e-07 | 1337.6ms/step | 15.0 samples/s +Step 43560/44000 | Loss: 2.8719 | LR: 1.28e-07 | 1336.4ms/step | 15.0 samples/s +Step 43570/44000 | Loss: 2.9201 | LR: 1.22e-07 | 1336.3ms/step | 15.0 samples/s +Step 43580/44000 | Loss: 2.9401 | LR: 1.16e-07 | 1337.7ms/step | 15.0 samples/s +Step 43590/44000 | Loss: 2.9165 | LR: 1.11e-07 | 1340.4ms/step | 14.9 samples/s +Step 43600/44000 | Loss: 2.9292 | LR: 1.06e-07 | 1341.1ms/step | 14.9 samples/s +Step 43610/44000 | Loss: 2.9157 | LR: 1.00e-07 | 1337.2ms/step | 15.0 samples/s +Step 43620/44000 | Loss: 2.9094 | LR: 9.52e-08 | 1331.8ms/step | 15.0 samples/s +Step 43630/44000 | Loss: 2.9591 | LR: 9.03e-08 | 1333.9ms/step | 15.0 samples/s +Step 43640/44000 | Loss: 2.9182 | LR: 8.55e-08 | 1335.6ms/step | 15.0 samples/s +Step 43650/44000 | Loss: 2.9243 | LR: 8.08e-08 | 1337.7ms/step | 15.0 samples/s +Step 43660/44000 | Loss: 2.9044 | LR: 7.62e-08 | 1336.3ms/step | 15.0 samples/s +Step 43670/44000 | Loss: 2.9272 | LR: 7.18e-08 | 1334.6ms/step | 15.0 samples/s +Step 43680/44000 | Loss: 2.9092 | LR: 6.75e-08 | 1335.0ms/step | 15.0 samples/s +Step 43690/44000 | Loss: 2.9363 | LR: 6.34e-08 | 1334.3ms/step | 15.0 samples/s +Step 43700/44000 | Loss: 2.9039 | LR: 5.94e-08 | 1334.5ms/step | 15.0 samples/s +Step 43710/44000 | Loss: 2.9279 | LR: 5.55e-08 | 1331.6ms/step | 15.0 samples/s +Step 43720/44000 | Loss: 2.9433 | LR: 5.17e-08 | 1329.8ms/step | 15.0 samples/s +Step 43730/44000 | Loss: 2.8970 | LR: 4.81e-08 | 1326.6ms/step | 15.1 samples/s +Step 43740/44000 | Loss: 2.9040 | LR: 4.46e-08 | 1330.9ms/step | 15.0 samples/s +Step 43750/44000 | Loss: 2.9314 | LR: 4.12e-08 | 1332.5ms/step | 15.0 samples/s +Validation at step 43750: Loss = 2.9048, Perplexity = 18.26 +Step 43760/44000 | Loss: 2.9215 | LR: 3.80e-08 | 3558.5ms/step | 5.6 samples/s +Step 43770/44000 | Loss: 2.9134 | LR: 3.49e-08 | 1334.0ms/step | 15.0 samples/s +Step 43780/44000 | Loss: 2.8981 | LR: 3.19e-08 | 1335.5ms/step | 15.0 samples/s +Step 43790/44000 | Loss: 2.8873 | LR: 2.91e-08 | 1331.4ms/step | 15.0 samples/s +Step 43800/44000 | Loss: 2.9132 | LR: 2.64e-08 | 1332.3ms/step | 15.0 samples/s +Step 43810/44000 | Loss: 2.9061 | LR: 2.38e-08 | 1332.7ms/step | 15.0 samples/s +Step 43820/44000 | Loss: 2.9099 | LR: 2.14e-08 | 1332.9ms/step | 15.0 samples/s +Step 43830/44000 | Loss: 2.9267 | LR: 1.91e-08 | 1327.4ms/step | 15.1 samples/s +Step 43840/44000 | Loss: 2.9243 | LR: 1.69e-08 | 1328.5ms/step | 15.1 samples/s +Step 43850/44000 | Loss: 2.9310 | LR: 1.48e-08 | 1330.0ms/step | 15.0 samples/s +Step 43860/44000 | Loss: 2.8977 | LR: 1.29e-08 | 1329.7ms/step | 15.0 samples/s +Step 43870/44000 | Loss: 2.8973 | LR: 1.11e-08 | 1330.4ms/step | 15.0 samples/s +Step 43880/44000 | Loss: 2.8885 | LR: 9.50e-09 | 1329.7ms/step | 15.0 samples/s +Step 43890/44000 | Loss: 2.9510 | LR: 7.98e-09 | 1325.3ms/step | 15.1 samples/s +Step 43900/44000 | Loss: 2.8748 | LR: 6.60e-09 | 1325.6ms/step | 15.1 samples/s +Step 43910/44000 | Loss: 2.9169 | LR: 5.34e-09 | 1325.8ms/step | 15.1 samples/s +Step 43920/44000 | Loss: 2.9201 | LR: 4.22e-09 | 1328.7ms/step | 15.1 samples/s +Step 43930/44000 | Loss: 2.9016 | LR: 3.23e-09 | 1329.8ms/step | 15.0 samples/s +Step 43940/44000 | Loss: 2.9356 | LR: 2.37e-09 | 1330.1ms/step | 15.0 samples/s +Step 43950/44000 | Loss: 2.8997 | LR: 1.65e-09 | 1330.7ms/step | 15.0 samples/s +Step 43960/44000 | Loss: 2.9336 | LR: 1.06e-09 | 1330.7ms/step | 15.0 samples/s +Step 43970/44000 | Loss: 2.8766 | LR: 5.94e-10 | 1332.1ms/step | 15.0 samples/s +Step 43980/44000 | Loss: 2.9158 | LR: 2.64e-10 | 1331.7ms/step | 15.0 samples/s +Step 43990/44000 | Loss: 2.9380 | LR: 6.60e-11 | 1332.0ms/step | 15.0 samples/s +Step 44000/44000 | Loss: 2.9183 | LR: 0.00e+00 | 1331.4ms/step | 15.0 samples/s +Validation at step 44000: Loss = 2.9048, Perplexity = 18.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading output.log +wandb: uploading history steps 4399-4399, summary, console lines 2530-2532 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss ██▇▇▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity ███▇▇▇▆▆▆▅▅▅▅▄▄▃▃▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▁ +wandb: speed/samples_per_sec ▅▄▆██▆▇▆▇▅▂▃▃▅▄▂▄▄▂▃▂▄▃▂▁▄▁▂▃▃▃▃▂▅▃▂▅▆▅▆ +wandb: speed/steps_per_sec ▆▆▆▆█▆▆▆▇▆▆▄▃▂▂▄▄▅▄▃▂▃▅▅▃▄▄▄▃▃▄▁▄▄▄▃▇▅▆▇ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.90483 +wandb: eval/perplexity 18.26221 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 1331.37 +wandb: speed/samples_per_sec 15.02212 +wandb: speed/steps_per_sec 0.75111 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 resume 365m static 8k nrb35 seed43 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ino9n2jx +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_042228-ino9n2jx/logs +Training complete! diff --git a/epfl_final/train_logs/p1_resume_h1004_20260822/supervisor.log b/epfl_final/train_logs/p1_resume_h1004_20260822/supervisor.log new file mode 100644 index 0000000000000000000000000000000000000000..a94e78a81fa746526fe1151d130a74eab1088259 --- /dev/null +++ b/epfl_final/train_logs/p1_resume_h1004_20260822/supervisor.log @@ -0,0 +1,9 @@ +[supervisor] launching gpu=0 name=l8k365m_static_nrb35_seed42 resume 2026-08-22T04:21:20Z +[supervisor] pid=245 +[supervisor] launching gpu=1 name=l8k365m_resampled_nrb35_seed42 resume 2026-08-22T04:21:20Z +[supervisor] pid=247 +[supervisor] launching gpu=2 name=l8k365m_static_nrb35_seed43 resume 2026-08-22T04:21:20Z +[supervisor] pid=249 +[supervisor] launching gpu=3 name=l8k365m_resampled_nrb35_seed43 resume 2026-08-22T04:21:20Z +[supervisor] pid=251 +[supervisor] all four resumes launched 2026-08-22T04:21:20Z diff --git a/epfl_final/train_logs/p1_scheduled_a100_seed47_20260822/scheduler.log b/epfl_final/train_logs/p1_scheduled_a100_seed47_20260822/scheduler.log new file mode 100644 index 0000000000000000000000000000000000000000..d2be0df9256e66f5e3982e157a429e213120daf4 --- /dev/null +++ b/epfl_final/train_logs/p1_scheduled_a100_seed47_20260822/scheduler.log @@ -0,0 +1,49 @@ +[scheduler] waiting for static seed47 to finish 2026-08-22T12:46:16Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:46:16Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:47:16Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:48:16Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:49:16Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:50:16Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:51:17Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:52:17Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:53:17Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:54:17Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:55:17Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:56:17Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:57:17Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:58:17Z +[scheduler] active=0 busy_gpus=1 2026-08-22T12:59:17Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:00:17Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:01:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:02:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:03:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:04:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:05:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:06:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:07:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:08:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:09:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:10:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:11:18Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:12:19Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:13:19Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:14:19Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:15:19Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:16:19Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:17:19Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:18:19Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:19:19Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:20:19Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:21:20Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:22:20Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:23:20Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:24:20Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:25:20Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:26:20Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:27:20Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:28:20Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:29:20Z +[scheduler] active=0 busy_gpus=1 2026-08-22T13:30:20Z +[scheduler] active=0 busy_gpus=0 2026-08-22T13:31:21Z +[scheduler] condition met; launching resampled seed47 on all five GPUs 2026-08-22T13:31:21Z +[scheduler] training exited rc=0 2026-08-22T22:30:28Z diff --git a/epfl_final/train_logs/p1_scheduled_h1002_20260822/l8k365m_resampled_nrb35_seed46.log b/epfl_final/train_logs/p1_scheduled_h1002_20260822/l8k365m_resampled_nrb35_seed46.log new file mode 100644 index 0000000000000000000000000000000000000000..8db75ef643659917dee341a75c60b887fe28d3ef --- /dev/null +++ b/epfl_final/train_logs/p1_scheduled_h1002_20260822/l8k365m_resampled_nrb35_seed46.log @@ -0,0 +1,2438 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 46, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 DDP2 365m resampled 8k nrb35 seed46", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "resampled", + "nrb35", + "seed46", + "ddp2", + "h1002", + "scheduled" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_115228-8xh3oikj +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 DDP2 365m resampled 8k nrb35 seed46 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8xh3oikj +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0614 | LR: 6.67e-06 | 7035.4ms/step | 2.8 samples/s +Step 20/44000 | Loss: 11.9519 | LR: 1.33e-05 | 686.7ms/step | 29.1 samples/s +Step 30/44000 | Loss: 11.6407 | LR: 2.00e-05 | 688.0ms/step | 29.1 samples/s +Step 40/44000 | Loss: 11.1095 | LR: 2.67e-05 | 686.6ms/step | 29.1 samples/s +Step 50/44000 | Loss: 10.7690 | LR: 3.33e-05 | 683.4ms/step | 29.3 samples/s +Step 60/44000 | Loss: 10.5287 | LR: 4.00e-05 | 685.8ms/step | 29.2 samples/s +Step 70/44000 | Loss: 10.3000 | LR: 4.67e-05 | 686.4ms/step | 29.1 samples/s +Step 80/44000 | Loss: 9.9697 | LR: 5.33e-05 | 685.3ms/step | 29.2 samples/s +Step 90/44000 | Loss: 9.5827 | LR: 6.00e-05 | 685.4ms/step | 29.2 samples/s +Step 100/44000 | Loss: 9.1102 | LR: 6.67e-05 | 686.5ms/step | 29.1 samples/s +Step 110/44000 | Loss: 8.6303 | LR: 7.33e-05 | 689.5ms/step | 29.0 samples/s +Step 120/44000 | Loss: 8.1522 | LR: 8.00e-05 | 685.8ms/step | 29.2 samples/s +Step 130/44000 | Loss: 7.7697 | LR: 8.67e-05 | 686.3ms/step | 29.1 samples/s +Step 140/44000 | Loss: 7.5001 | LR: 9.33e-05 | 685.7ms/step | 29.2 samples/s +Step 150/44000 | Loss: 7.3297 | LR: 1.00e-04 | 687.0ms/step | 29.1 samples/s +Step 160/44000 | Loss: 7.2521 | LR: 1.07e-04 | 689.4ms/step | 29.0 samples/s +Step 170/44000 | Loss: 7.1036 | LR: 1.13e-04 | 688.2ms/step | 29.1 samples/s +Step 180/44000 | Loss: 7.0159 | LR: 1.20e-04 | 688.8ms/step | 29.0 samples/s +Step 190/44000 | Loss: 6.9471 | LR: 1.27e-04 | 692.6ms/step | 28.9 samples/s +Step 200/44000 | Loss: 6.8499 | LR: 1.33e-04 | 689.1ms/step | 29.0 samples/s +Step 210/44000 | Loss: 6.7850 | LR: 1.40e-04 | 689.5ms/step | 29.0 samples/s +Step 220/44000 | Loss: 6.6765 | LR: 1.47e-04 | 688.5ms/step | 29.1 samples/s +Step 230/44000 | Loss: 6.6380 | LR: 1.53e-04 | 690.3ms/step | 29.0 samples/s +Step 240/44000 | Loss: 6.5394 | LR: 1.60e-04 | 690.6ms/step | 29.0 samples/s +Step 250/44000 | Loss: 6.4712 | LR: 1.67e-04 | 689.8ms/step | 29.0 samples/s +Validation at step 250: Loss = 6.4397, Perplexity = 626.20 +Step 260/44000 | Loss: 6.4094 | LR: 1.73e-04 | 4655.9ms/step | 4.3 samples/s +Step 270/44000 | Loss: 6.3511 | LR: 1.80e-04 | 690.4ms/step | 29.0 samples/s +Step 280/44000 | Loss: 6.3288 | LR: 1.87e-04 | 689.6ms/step | 29.0 samples/s +Step 290/44000 | Loss: 6.2482 | LR: 1.93e-04 | 691.8ms/step | 28.9 samples/s +Step 300/44000 | Loss: 6.2188 | LR: 2.00e-04 | 691.3ms/step | 28.9 samples/s +Step 310/44000 | Loss: 6.1746 | LR: 2.07e-04 | 691.1ms/step | 28.9 samples/s +Step 320/44000 | Loss: 6.1306 | LR: 2.13e-04 | 693.8ms/step | 28.8 samples/s +Step 330/44000 | Loss: 6.0658 | LR: 2.20e-04 | 693.6ms/step | 28.8 samples/s +Step 340/44000 | Loss: 6.0717 | LR: 2.27e-04 | 690.6ms/step | 29.0 samples/s +Step 350/44000 | Loss: 5.9771 | LR: 2.33e-04 | 691.8ms/step | 28.9 samples/s +Step 360/44000 | Loss: 5.9580 | LR: 2.40e-04 | 692.2ms/step | 28.9 samples/s +Step 370/44000 | Loss: 5.9571 | LR: 2.47e-04 | 692.9ms/step | 28.9 samples/s +Step 380/44000 | Loss: 5.8864 | LR: 2.53e-04 | 691.7ms/step | 28.9 samples/s +Step 390/44000 | Loss: 5.8665 | LR: 2.60e-04 | 692.7ms/step | 28.9 samples/s +Step 400/44000 | Loss: 5.8006 | LR: 2.67e-04 | 692.3ms/step | 28.9 samples/s +Step 410/44000 | Loss: 5.8164 | LR: 2.73e-04 | 692.3ms/step | 28.9 samples/s +Step 420/44000 | Loss: 5.7779 | LR: 2.80e-04 | 689.2ms/step | 29.0 samples/s +Step 430/44000 | Loss: 5.7081 | LR: 2.87e-04 | 689.7ms/step | 29.0 samples/s +Step 440/44000 | Loss: 5.6971 | LR: 2.93e-04 | 689.9ms/step | 29.0 samples/s +Step 450/44000 | Loss: 5.6153 | LR: 3.00e-04 | 690.1ms/step | 29.0 samples/s +Step 460/44000 | Loss: 5.6056 | LR: 3.07e-04 | 690.5ms/step | 29.0 samples/s +Step 470/44000 | Loss: 5.6102 | LR: 3.13e-04 | 690.2ms/step | 29.0 samples/s +Step 480/44000 | Loss: 5.5148 | LR: 3.20e-04 | 689.7ms/step | 29.0 samples/s +Step 490/44000 | Loss: 5.4657 | LR: 3.27e-04 | 689.0ms/step | 29.0 samples/s +Step 500/44000 | Loss: 5.4442 | LR: 3.33e-04 | 689.6ms/step | 29.0 samples/s +Validation at step 500: Loss = 5.4685, Perplexity = 237.12 +Step 510/44000 | Loss: 5.4669 | LR: 3.40e-04 | 1262.4ms/step | 15.8 samples/s +Step 520/44000 | Loss: 5.4257 | LR: 3.47e-04 | 691.4ms/step | 28.9 samples/s +Step 530/44000 | Loss: 5.3790 | LR: 3.53e-04 | 690.5ms/step | 29.0 samples/s +Step 540/44000 | Loss: 5.3338 | LR: 3.60e-04 | 691.3ms/step | 28.9 samples/s +Step 550/44000 | Loss: 5.3401 | LR: 3.67e-04 | 691.9ms/step | 28.9 samples/s +Step 560/44000 | Loss: 5.2820 | LR: 3.73e-04 | 690.5ms/step | 29.0 samples/s +Step 570/44000 | Loss: 5.2755 | LR: 3.80e-04 | 691.4ms/step | 28.9 samples/s +Step 580/44000 | Loss: 5.2696 | LR: 3.87e-04 | 690.7ms/step | 29.0 samples/s +Step 590/44000 | Loss: 5.2328 | LR: 3.93e-04 | 692.6ms/step | 28.9 samples/s +Step 600/44000 | Loss: 5.1882 | LR: 4.00e-04 | 695.2ms/step | 28.8 samples/s +Step 610/44000 | Loss: 5.1426 | LR: 4.07e-04 | 690.9ms/step | 28.9 samples/s +Step 620/44000 | Loss: 5.1168 | LR: 4.13e-04 | 691.9ms/step | 28.9 samples/s +Step 630/44000 | Loss: 5.0931 | LR: 4.20e-04 | 691.2ms/step | 28.9 samples/s +Step 640/44000 | Loss: 5.0436 | LR: 4.27e-04 | 693.3ms/step | 28.8 samples/s +Step 650/44000 | Loss: 5.0631 | LR: 4.33e-04 | 692.3ms/step | 28.9 samples/s +Step 660/44000 | Loss: 5.0189 | LR: 4.40e-04 | 692.7ms/step | 28.9 samples/s +Step 670/44000 | Loss: 4.9423 | LR: 4.47e-04 | 696.1ms/step | 28.7 samples/s +Step 680/44000 | Loss: 4.9567 | LR: 4.53e-04 | 691.2ms/step | 28.9 samples/s +Step 690/44000 | Loss: 4.9354 | LR: 4.60e-04 | 692.2ms/step | 28.9 samples/s +Step 700/44000 | Loss: 4.9375 | LR: 4.67e-04 | 691.2ms/step | 28.9 samples/s +Step 710/44000 | Loss: 4.8511 | LR: 4.73e-04 | 690.5ms/step | 29.0 samples/s +Step 720/44000 | Loss: 4.8131 | LR: 4.80e-04 | 691.7ms/step | 28.9 samples/s +Step 730/44000 | Loss: 4.8396 | LR: 4.87e-04 | 692.1ms/step | 28.9 samples/s +Step 740/44000 | Loss: 4.8451 | LR: 4.93e-04 | 692.0ms/step | 28.9 samples/s +Step 750/44000 | Loss: 4.8465 | LR: 5.00e-04 | 692.1ms/step | 28.9 samples/s +Validation at step 750: Loss = 4.7995, Perplexity = 121.45 +Step 760/44000 | Loss: 4.7645 | LR: 5.00e-04 | 1258.8ms/step | 15.9 samples/s +Step 770/44000 | Loss: 4.6930 | LR: 5.00e-04 | 691.4ms/step | 28.9 samples/s +Step 780/44000 | Loss: 4.6710 | LR: 5.00e-04 | 692.1ms/step | 28.9 samples/s +Step 790/44000 | Loss: 4.6953 | LR: 5.00e-04 | 692.6ms/step | 28.9 samples/s +Step 800/44000 | Loss: 4.6186 | LR: 5.00e-04 | 695.0ms/step | 28.8 samples/s +Step 810/44000 | Loss: 4.6240 | LR: 5.00e-04 | 693.4ms/step | 28.8 samples/s +Step 820/44000 | Loss: 4.5711 | LR: 5.00e-04 | 694.7ms/step | 28.8 samples/s +Step 830/44000 | Loss: 4.5907 | LR: 5.00e-04 | 693.1ms/step | 28.9 samples/s +Step 840/44000 | Loss: 4.5451 | LR: 5.00e-04 | 692.9ms/step | 28.9 samples/s +Step 850/44000 | Loss: 4.5108 | LR: 5.00e-04 | 692.5ms/step | 28.9 samples/s +Step 860/44000 | Loss: 4.4763 | LR: 5.00e-04 | 692.3ms/step | 28.9 samples/s +Step 870/44000 | Loss: 4.4876 | LR: 5.00e-04 | 691.8ms/step | 28.9 samples/s +Step 880/44000 | Loss: 4.5146 | LR: 5.00e-04 | 692.2ms/step | 28.9 samples/s +Step 890/44000 | Loss: 4.4421 | LR: 5.00e-04 | 692.6ms/step | 28.9 samples/s +Step 900/44000 | Loss: 4.4196 | LR: 5.00e-04 | 692.1ms/step | 28.9 samples/s +Step 910/44000 | Loss: 4.4359 | LR: 5.00e-04 | 692.8ms/step | 28.9 samples/s +Step 920/44000 | Loss: 4.4164 | LR: 5.00e-04 | 692.3ms/step | 28.9 samples/s +Step 930/44000 | Loss: 4.4449 | LR: 5.00e-04 | 691.6ms/step | 28.9 samples/s +Step 940/44000 | Loss: 4.4139 | LR: 5.00e-04 | 692.5ms/step | 28.9 samples/s +Step 950/44000 | Loss: 4.3868 | LR: 5.00e-04 | 692.0ms/step | 28.9 samples/s +Step 960/44000 | Loss: 4.3470 | LR: 5.00e-04 | 691.6ms/step | 28.9 samples/s +Step 970/44000 | Loss: 4.3532 | LR: 5.00e-04 | 693.1ms/step | 28.9 samples/s +Step 980/44000 | Loss: 4.3351 | LR: 5.00e-04 | 691.0ms/step | 28.9 samples/s +Step 990/44000 | Loss: 4.3128 | LR: 5.00e-04 | 690.6ms/step | 29.0 samples/s +Step 1000/44000 | Loss: 4.2638 | LR: 5.00e-04 | 690.7ms/step | 29.0 samples/s +Validation at step 1000: Loss = 4.2960, Perplexity = 73.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-1000 +Step 1010/44000 | Loss: 4.2822 | LR: 5.00e-04 | 1625.7ms/step | 12.3 samples/s +Step 1020/44000 | Loss: 4.2927 | LR: 5.00e-04 | 693.5ms/step | 28.8 samples/s +Step 1030/44000 | Loss: 4.2742 | LR: 5.00e-04 | 691.8ms/step | 28.9 samples/s +Step 1040/44000 | Loss: 4.2575 | LR: 5.00e-04 | 692.0ms/step | 28.9 samples/s +Step 1050/44000 | Loss: 4.2373 | LR: 5.00e-04 | 692.3ms/step | 28.9 samples/s +Step 1060/44000 | Loss: 4.2173 | LR: 5.00e-04 | 690.3ms/step | 29.0 samples/s +Step 1070/44000 | Loss: 4.1916 | LR: 5.00e-04 | 692.9ms/step | 28.9 samples/s +Step 1080/44000 | Loss: 4.2278 | LR: 5.00e-04 | 692.0ms/step | 28.9 samples/s +Step 1090/44000 | Loss: 4.2060 | LR: 5.00e-04 | 692.3ms/step | 28.9 samples/s +Step 1100/44000 | Loss: 4.1965 | LR: 5.00e-04 | 691.2ms/step | 28.9 samples/s +Step 1110/44000 | Loss: 4.1662 | LR: 5.00e-04 | 691.2ms/step | 28.9 samples/s +Step 1120/44000 | Loss: 4.1685 | LR: 5.00e-04 | 692.3ms/step | 28.9 samples/s +Step 1130/44000 | Loss: 4.1287 | LR: 5.00e-04 | 690.9ms/step | 28.9 samples/s +Step 1140/44000 | Loss: 4.1551 | LR: 5.00e-04 | 691.2ms/step | 28.9 samples/s +Step 1150/44000 | Loss: 4.1660 | LR: 5.00e-04 | 692.0ms/step | 28.9 samples/s +Step 1160/44000 | Loss: 4.1385 | LR: 5.00e-04 | 691.8ms/step | 28.9 samples/s +Step 1170/44000 | Loss: 4.1150 | LR: 5.00e-04 | 690.9ms/step | 28.9 samples/s +Step 1180/44000 | Loss: 4.0618 | LR: 5.00e-04 | 692.7ms/step | 28.9 samples/s +Step 1190/44000 | Loss: 4.0781 | LR: 5.00e-04 | 691.3ms/step | 28.9 samples/s +Step 1200/44000 | Loss: 4.0702 | LR: 5.00e-04 | 691.8ms/step | 28.9 samples/s +Step 1210/44000 | Loss: 4.1246 | LR: 5.00e-04 | 690.7ms/step | 29.0 samples/s +Step 1220/44000 | Loss: 4.1181 | LR: 5.00e-04 | 692.1ms/step | 28.9 samples/s +Step 1230/44000 | Loss: 4.0344 | LR: 5.00e-04 | 694.8ms/step | 28.8 samples/s +Step 1240/44000 | Loss: 4.0826 | LR: 5.00e-04 | 692.6ms/step | 28.9 samples/s +Step 1250/44000 | Loss: 4.0635 | LR: 5.00e-04 | 691.3ms/step | 28.9 samples/s +Validation at step 1250: Loss = 4.0704, Perplexity = 58.58 +Step 1260/44000 | Loss: 4.0741 | LR: 5.00e-04 | 1261.2ms/step | 15.9 samples/s +Step 1270/44000 | Loss: 4.0440 | LR: 5.00e-04 | 691.8ms/step | 28.9 samples/s +Step 1280/44000 | Loss: 4.0426 | LR: 5.00e-04 | 690.5ms/step | 29.0 samples/s +Step 1290/44000 | Loss: 4.0466 | LR: 5.00e-04 | 692.5ms/step | 28.9 samples/s +Step 1300/44000 | Loss: 4.0080 | LR: 5.00e-04 | 691.9ms/step | 28.9 samples/s +Step 1310/44000 | Loss: 4.0010 | LR: 5.00e-04 | 691.6ms/step | 28.9 samples/s +Step 1320/44000 | Loss: 4.0024 | LR: 5.00e-04 | 692.4ms/step | 28.9 samples/s +Step 1330/44000 | Loss: 4.0181 | LR: 5.00e-04 | 690.0ms/step | 29.0 samples/s +Step 1340/44000 | Loss: 3.9773 | LR: 5.00e-04 | 691.7ms/step | 28.9 samples/s +Step 1350/44000 | Loss: 3.9600 | LR: 5.00e-04 | 692.1ms/step | 28.9 samples/s +Step 1360/44000 | Loss: 3.9424 | LR: 5.00e-04 | 693.2ms/step | 28.9 samples/s +Step 1370/44000 | Loss: 3.9855 | LR: 5.00e-04 | 692.0ms/step | 28.9 samples/s +Step 1380/44000 | Loss: 3.9556 | LR: 5.00e-04 | 692.3ms/step | 28.9 samples/s +Step 1390/44000 | Loss: 3.9887 | LR: 5.00e-04 | 693.4ms/step | 28.8 samples/s +Step 1400/44000 | Loss: 3.9509 | LR: 5.00e-04 | 692.7ms/step | 28.9 samples/s +Step 1410/44000 | Loss: 3.9580 | LR: 5.00e-04 | 691.0ms/step | 28.9 samples/s +Step 1420/44000 | Loss: 3.9550 | LR: 5.00e-04 | 691.3ms/step | 28.9 samples/s +Step 1430/44000 | Loss: 3.9119 | LR: 5.00e-04 | 691.5ms/step | 28.9 samples/s +Step 1440/44000 | Loss: 3.9666 | LR: 5.00e-04 | 693.9ms/step | 28.8 samples/s +Step 1450/44000 | Loss: 3.9775 | LR: 5.00e-04 | 691.7ms/step | 28.9 samples/s +Step 1460/44000 | Loss: 3.9181 | LR: 5.00e-04 | 690.5ms/step | 29.0 samples/s +Step 1470/44000 | Loss: 3.9159 | LR: 5.00e-04 | 692.7ms/step | 28.9 samples/s +Step 1480/44000 | Loss: 3.8797 | LR: 5.00e-04 | 690.9ms/step | 28.9 samples/s +Step 1490/44000 | Loss: 3.9445 | LR: 5.00e-04 | 691.1ms/step | 28.9 samples/s +Step 1500/44000 | Loss: 3.8755 | LR: 5.00e-04 | 692.9ms/step | 28.9 samples/s +Validation at step 1500: Loss = 3.9109, Perplexity = 49.95 +Step 1510/44000 | Loss: 3.9071 | LR: 5.00e-04 | 1261.0ms/step | 15.9 samples/s +Step 1520/44000 | Loss: 3.9034 | LR: 5.00e-04 | 690.9ms/step | 28.9 samples/s +Step 1530/44000 | Loss: 3.8526 | LR: 5.00e-04 | 690.8ms/step | 28.9 samples/s +Step 1540/44000 | Loss: 3.8682 | LR: 5.00e-04 | 689.4ms/step | 29.0 samples/s +Step 1550/44000 | Loss: 3.8881 | LR: 5.00e-04 | 691.2ms/step | 28.9 samples/s +Step 1560/44000 | Loss: 3.9180 | LR: 5.00e-04 | 693.1ms/step | 28.9 samples/s +Step 1570/44000 | Loss: 3.9047 | LR: 5.00e-04 | 697.5ms/step | 28.7 samples/s +Step 1580/44000 | Loss: 3.8798 | LR: 5.00e-04 | 692.7ms/step | 28.9 samples/s +Step 1590/44000 | Loss: 3.8135 | LR: 5.00e-04 | 691.6ms/step | 28.9 samples/s +Step 1600/44000 | Loss: 3.8457 | LR: 5.00e-04 | 690.8ms/step | 29.0 samples/s +Step 1610/44000 | Loss: 3.8355 | LR: 5.00e-04 | 692.3ms/step | 28.9 samples/s +Step 1620/44000 | Loss: 3.8409 | LR: 5.00e-04 | 691.6ms/step | 28.9 samples/s +Step 1630/44000 | Loss: 3.8276 | LR: 4.99e-04 | 690.8ms/step | 28.9 samples/s +Step 1640/44000 | Loss: 3.8406 | LR: 4.99e-04 | 692.7ms/step | 28.9 samples/s +Step 1650/44000 | Loss: 3.8362 | LR: 4.99e-04 | 691.4ms/step | 28.9 samples/s +Step 1660/44000 | Loss: 3.8157 | LR: 4.99e-04 | 691.6ms/step | 28.9 samples/s +Step 1670/44000 | Loss: 3.8256 | LR: 4.99e-04 | 690.4ms/step | 29.0 samples/s +Step 1680/44000 | Loss: 3.8504 | LR: 4.99e-04 | 690.5ms/step | 29.0 samples/s +Step 1690/44000 | Loss: 3.8657 | LR: 4.99e-04 | 691.4ms/step | 28.9 samples/s +Step 1700/44000 | Loss: 3.8083 | LR: 4.99e-04 | 690.2ms/step | 29.0 samples/s +Step 1710/44000 | Loss: 3.7907 | LR: 4.99e-04 | 690.6ms/step | 29.0 samples/s +Step 1720/44000 | Loss: 3.8139 | LR: 4.99e-04 | 690.3ms/step | 29.0 samples/s +Step 1730/44000 | Loss: 3.8325 | LR: 4.99e-04 | 691.6ms/step | 28.9 samples/s +Step 1740/44000 | Loss: 3.7696 | LR: 4.99e-04 | 690.5ms/step | 29.0 samples/s +Step 1750/44000 | Loss: 3.7673 | LR: 4.99e-04 | 691.3ms/step | 28.9 samples/s +Validation at step 1750: Loss = 3.8022, Perplexity = 44.80 +Step 1760/44000 | Loss: 3.7646 | LR: 4.99e-04 | 1260.9ms/step | 15.9 samples/s +Step 1770/44000 | Loss: 3.7843 | LR: 4.99e-04 | 694.2ms/step | 28.8 samples/s +Step 1780/44000 | Loss: 3.7544 | LR: 4.99e-04 | 692.6ms/step | 28.9 samples/s +Step 1790/44000 | Loss: 3.7831 | LR: 4.99e-04 | 691.9ms/step | 28.9 samples/s +Step 1800/44000 | Loss: 3.7336 | LR: 4.99e-04 | 692.4ms/step | 28.9 samples/s +Step 1810/44000 | Loss: 3.7505 | LR: 4.99e-04 | 692.7ms/step | 28.9 samples/s +Step 1820/44000 | Loss: 3.7939 | LR: 4.99e-04 | 692.5ms/step | 28.9 samples/s +Step 1830/44000 | Loss: 3.7539 | LR: 4.99e-04 | 692.3ms/step | 28.9 samples/s +Step 1840/44000 | Loss: 3.7645 | LR: 4.99e-04 | 692.7ms/step | 28.9 samples/s +Step 1850/44000 | Loss: 3.7463 | LR: 4.99e-04 | 692.7ms/step | 28.9 samples/s +Step 1860/44000 | Loss: 3.7260 | LR: 4.99e-04 | 694.2ms/step | 28.8 samples/s +Step 1870/44000 | Loss: 3.7539 | LR: 4.99e-04 | 691.6ms/step | 28.9 samples/s +Step 1880/44000 | Loss: 3.7728 | LR: 4.99e-04 | 693.8ms/step | 28.8 samples/s +Step 1890/44000 | Loss: 3.7383 | LR: 4.99e-04 | 691.7ms/step | 28.9 samples/s +Step 1900/44000 | Loss: 3.7570 | LR: 4.99e-04 | 693.2ms/step | 28.9 samples/s +Step 1910/44000 | Loss: 3.7388 | LR: 4.99e-04 | 699.7ms/step | 28.6 samples/s +Step 1920/44000 | Loss: 3.7604 | LR: 4.99e-04 | 692.5ms/step | 28.9 samples/s +Step 1930/44000 | Loss: 3.7109 | LR: 4.99e-04 | 692.5ms/step | 28.9 samples/s +Step 1940/44000 | Loss: 3.7028 | LR: 4.99e-04 | 694.9ms/step | 28.8 samples/s +Step 1950/44000 | Loss: 3.7275 | LR: 4.99e-04 | 693.7ms/step | 28.8 samples/s +Step 1960/44000 | Loss: 3.7107 | LR: 4.99e-04 | 693.5ms/step | 28.8 samples/s +Step 1970/44000 | Loss: 3.6807 | LR: 4.99e-04 | 695.1ms/step | 28.8 samples/s +Step 1980/44000 | Loss: 3.6855 | LR: 4.99e-04 | 696.2ms/step | 28.7 samples/s +Step 1990/44000 | Loss: 3.7659 | LR: 4.99e-04 | 695.4ms/step | 28.8 samples/s +Step 2000/44000 | Loss: 3.7398 | LR: 4.99e-04 | 695.2ms/step | 28.8 samples/s +Validation at step 2000: Loss = 3.7286, Perplexity = 41.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-2000 +Step 2010/44000 | Loss: 3.7508 | LR: 4.99e-04 | 1601.0ms/step | 12.5 samples/s +Step 2020/44000 | Loss: 3.6660 | LR: 4.99e-04 | 694.9ms/step | 28.8 samples/s +Step 2030/44000 | Loss: 3.7121 | LR: 4.99e-04 | 692.7ms/step | 28.9 samples/s +Step 2040/44000 | Loss: 3.7181 | LR: 4.99e-04 | 693.7ms/step | 28.8 samples/s +Step 2050/44000 | Loss: 3.6963 | LR: 4.99e-04 | 692.0ms/step | 28.9 samples/s +Step 2060/44000 | Loss: 3.7268 | LR: 4.99e-04 | 693.4ms/step | 28.8 samples/s +Step 2070/44000 | Loss: 3.6525 | LR: 4.99e-04 | 692.1ms/step | 28.9 samples/s +Step 2080/44000 | Loss: 3.6990 | LR: 4.99e-04 | 695.4ms/step | 28.8 samples/s +Step 2090/44000 | Loss: 3.6550 | LR: 4.99e-04 | 696.2ms/step | 28.7 samples/s +Step 2100/44000 | Loss: 3.7393 | LR: 4.99e-04 | 691.8ms/step | 28.9 samples/s +Step 2110/44000 | Loss: 3.6916 | LR: 4.99e-04 | 690.4ms/step | 29.0 samples/s +Step 2120/44000 | Loss: 3.6727 | LR: 4.99e-04 | 692.0ms/step | 28.9 samples/s +Step 2130/44000 | Loss: 3.6961 | LR: 4.99e-04 | 695.0ms/step | 28.8 samples/s +Step 2140/44000 | Loss: 3.7305 | LR: 4.99e-04 | 693.6ms/step | 28.8 samples/s +Step 2150/44000 | Loss: 3.6877 | LR: 4.99e-04 | 693.2ms/step | 28.9 samples/s +Step 2160/44000 | Loss: 3.6730 | LR: 4.99e-04 | 695.0ms/step | 28.8 samples/s +Step 2170/44000 | Loss: 3.6970 | LR: 4.99e-04 | 693.6ms/step | 28.8 samples/s +Step 2180/44000 | Loss: 3.6158 | LR: 4.99e-04 | 694.3ms/step | 28.8 samples/s +Step 2190/44000 | Loss: 3.6885 | LR: 4.99e-04 | 694.6ms/step | 28.8 samples/s +Step 2200/44000 | Loss: 3.6257 | LR: 4.99e-04 | 693.3ms/step | 28.8 samples/s +Step 2210/44000 | Loss: 3.6706 | LR: 4.99e-04 | 693.0ms/step | 28.9 samples/s +Step 2220/44000 | Loss: 3.6684 | LR: 4.99e-04 | 694.0ms/step | 28.8 samples/s +Step 2230/44000 | Loss: 3.6560 | LR: 4.99e-04 | 694.6ms/step | 28.8 samples/s +Step 2240/44000 | Loss: 3.6465 | LR: 4.99e-04 | 693.4ms/step | 28.8 samples/s +Step 2250/44000 | Loss: 3.7183 | LR: 4.99e-04 | 693.1ms/step | 28.9 samples/s +Validation at step 2250: Loss = 3.6613, Perplexity = 38.91 +Step 2260/44000 | Loss: 3.6919 | LR: 4.98e-04 | 1261.2ms/step | 15.9 samples/s +Step 2270/44000 | Loss: 3.6037 | LR: 4.98e-04 | 695.7ms/step | 28.7 samples/s +Step 2280/44000 | Loss: 3.6551 | LR: 4.98e-04 | 695.9ms/step | 28.7 samples/s +Step 2290/44000 | Loss: 3.5965 | LR: 4.98e-04 | 692.4ms/step | 28.9 samples/s +Step 2300/44000 | Loss: 3.6105 | LR: 4.98e-04 | 696.9ms/step | 28.7 samples/s +Step 2310/44000 | Loss: 3.6599 | LR: 4.98e-04 | 693.2ms/step | 28.9 samples/s +Step 2320/44000 | Loss: 3.6739 | LR: 4.98e-04 | 693.4ms/step | 28.8 samples/s +Step 2330/44000 | Loss: 3.6874 | LR: 4.98e-04 | 694.1ms/step | 28.8 samples/s +Step 2340/44000 | Loss: 3.5981 | LR: 4.98e-04 | 693.5ms/step | 28.8 samples/s +Step 2350/44000 | Loss: 3.6362 | LR: 4.98e-04 | 693.6ms/step | 28.8 samples/s +Step 2360/44000 | Loss: 3.6278 | LR: 4.98e-04 | 695.6ms/step | 28.8 samples/s +Step 2370/44000 | Loss: 3.6327 | LR: 4.98e-04 | 695.5ms/step | 28.8 samples/s +Step 2380/44000 | Loss: 3.6185 | LR: 4.98e-04 | 693.3ms/step | 28.8 samples/s +Step 2390/44000 | Loss: 3.6355 | LR: 4.98e-04 | 693.1ms/step | 28.9 samples/s +Step 2400/44000 | Loss: 3.6074 | LR: 4.98e-04 | 693.2ms/step | 28.9 samples/s +Step 2410/44000 | Loss: 3.6047 | LR: 4.98e-04 | 693.5ms/step | 28.8 samples/s +Step 2420/44000 | Loss: 3.5737 | LR: 4.98e-04 | 693.2ms/step | 28.9 samples/s +Step 2430/44000 | Loss: 3.6191 | LR: 4.98e-04 | 691.8ms/step | 28.9 samples/s +Step 2440/44000 | Loss: 3.6152 | LR: 4.98e-04 | 692.9ms/step | 28.9 samples/s +Step 2450/44000 | Loss: 3.6067 | LR: 4.98e-04 | 693.1ms/step | 28.9 samples/s +Step 2460/44000 | Loss: 3.6162 | LR: 4.98e-04 | 693.4ms/step | 28.8 samples/s +Step 2470/44000 | Loss: 3.5773 | LR: 4.98e-04 | 693.0ms/step | 28.9 samples/s +Step 2480/44000 | Loss: 3.5432 | LR: 4.98e-04 | 693.1ms/step | 28.9 samples/s +Step 2490/44000 | Loss: 3.6092 | LR: 4.98e-04 | 693.1ms/step | 28.9 samples/s +Step 2500/44000 | Loss: 3.5971 | LR: 4.98e-04 | 692.3ms/step | 28.9 samples/s +Validation at step 2500: Loss = 3.6059, Perplexity = 36.81 +Step 2510/44000 | Loss: 3.5643 | LR: 4.98e-04 | 1261.9ms/step | 15.8 samples/s +Step 2520/44000 | Loss: 3.5801 | LR: 4.98e-04 | 692.2ms/step | 28.9 samples/s +Step 2530/44000 | Loss: 3.5895 | LR: 4.98e-04 | 692.4ms/step | 28.9 samples/s +Step 2540/44000 | Loss: 3.5287 | LR: 4.98e-04 | 692.6ms/step | 28.9 samples/s +Step 2550/44000 | Loss: 3.6018 | LR: 4.98e-04 | 692.0ms/step | 28.9 samples/s +Step 2560/44000 | Loss: 3.6336 | LR: 4.98e-04 | 691.1ms/step | 28.9 samples/s +Step 2570/44000 | Loss: 3.6078 | LR: 4.98e-04 | 693.3ms/step | 28.8 samples/s +Step 2580/44000 | Loss: 3.5668 | LR: 4.98e-04 | 693.8ms/step | 28.8 samples/s +Step 2590/44000 | Loss: 3.5830 | LR: 4.98e-04 | 691.7ms/step | 28.9 samples/s +Step 2600/44000 | Loss: 3.5876 | LR: 4.98e-04 | 692.5ms/step | 28.9 samples/s +Step 2610/44000 | Loss: 3.5725 | LR: 4.98e-04 | 693.3ms/step | 28.8 samples/s +Step 2620/44000 | Loss: 3.5846 | LR: 4.98e-04 | 692.2ms/step | 28.9 samples/s +Step 2630/44000 | Loss: 3.5292 | LR: 4.98e-04 | 692.0ms/step | 28.9 samples/s +Step 2640/44000 | Loss: 3.5363 | LR: 4.98e-04 | 692.8ms/step | 28.9 samples/s +Step 2650/44000 | Loss: 3.5880 | LR: 4.98e-04 | 692.3ms/step | 28.9 samples/s +Step 2660/44000 | Loss: 3.5384 | LR: 4.98e-04 | 694.9ms/step | 28.8 samples/s +Step 2670/44000 | Loss: 3.6146 | LR: 4.98e-04 | 692.2ms/step | 28.9 samples/s +Step 2680/44000 | Loss: 3.5810 | LR: 4.98e-04 | 692.1ms/step | 28.9 samples/s +Step 2690/44000 | Loss: 3.5684 | LR: 4.98e-04 | 691.6ms/step | 28.9 samples/s +Step 2700/44000 | Loss: 3.5389 | LR: 4.97e-04 | 692.1ms/step | 28.9 samples/s +Step 2710/44000 | Loss: 3.5657 | LR: 4.97e-04 | 694.0ms/step | 28.8 samples/s +Step 2720/44000 | Loss: 3.5771 | LR: 4.97e-04 | 693.8ms/step | 28.8 samples/s +Step 2730/44000 | Loss: 3.5746 | LR: 4.97e-04 | 693.1ms/step | 28.9 samples/s +Step 2740/44000 | Loss: 3.5672 | LR: 4.97e-04 | 692.4ms/step | 28.9 samples/s +Step 2750/44000 | Loss: 3.5756 | LR: 4.97e-04 | 692.0ms/step | 28.9 samples/s +Validation at step 2750: Loss = 3.5631, Perplexity = 35.27 +Step 2760/44000 | Loss: 3.5691 | LR: 4.97e-04 | 1262.4ms/step | 15.8 samples/s +Step 2770/44000 | Loss: 3.5454 | LR: 4.97e-04 | 693.3ms/step | 28.8 samples/s +Step 2780/44000 | Loss: 3.5469 | LR: 4.97e-04 | 692.2ms/step | 28.9 samples/s +Step 2790/44000 | Loss: 3.5402 | LR: 4.97e-04 | 692.0ms/step | 28.9 samples/s +Step 2800/44000 | Loss: 3.5059 | LR: 4.97e-04 | 692.7ms/step | 28.9 samples/s +Step 2810/44000 | Loss: 3.5724 | LR: 4.97e-04 | 694.3ms/step | 28.8 samples/s +Step 2820/44000 | Loss: 3.5400 | LR: 4.97e-04 | 694.2ms/step | 28.8 samples/s +Step 2830/44000 | Loss: 3.5699 | LR: 4.97e-04 | 693.0ms/step | 28.9 samples/s +Step 2840/44000 | Loss: 3.5223 | LR: 4.97e-04 | 694.0ms/step | 28.8 samples/s +Step 2850/44000 | Loss: 3.5378 | LR: 4.97e-04 | 693.3ms/step | 28.8 samples/s +Step 2860/44000 | Loss: 3.5645 | LR: 4.97e-04 | 694.1ms/step | 28.8 samples/s +Step 2870/44000 | Loss: 3.5063 | LR: 4.97e-04 | 694.3ms/step | 28.8 samples/s +Step 2880/44000 | Loss: 3.4897 | LR: 4.97e-04 | 693.0ms/step | 28.9 samples/s +Step 2890/44000 | Loss: 3.5258 | LR: 4.97e-04 | 693.2ms/step | 28.9 samples/s +Step 2900/44000 | Loss: 3.5601 | LR: 4.97e-04 | 692.9ms/step | 28.9 samples/s +Step 2910/44000 | Loss: 3.5253 | LR: 4.97e-04 | 692.9ms/step | 28.9 samples/s +Step 2920/44000 | Loss: 3.5013 | LR: 4.97e-04 | 693.0ms/step | 28.9 samples/s +Step 2930/44000 | Loss: 3.5052 | LR: 4.97e-04 | 692.6ms/step | 28.9 samples/s +Step 2940/44000 | Loss: 3.5335 | LR: 4.97e-04 | 703.8ms/step | 28.4 samples/s +Step 2950/44000 | Loss: 3.4757 | LR: 4.97e-04 | 712.0ms/step | 28.1 samples/s +Step 2960/44000 | Loss: 3.5201 | LR: 4.97e-04 | 712.6ms/step | 28.1 samples/s +Step 2970/44000 | Loss: 3.5139 | LR: 4.97e-04 | 711.8ms/step | 28.1 samples/s +Step 2980/44000 | Loss: 3.5002 | LR: 4.97e-04 | 710.4ms/step | 28.2 samples/s +Step 2990/44000 | Loss: 3.5289 | LR: 4.97e-04 | 710.1ms/step | 28.2 samples/s +Step 3000/44000 | Loss: 3.5286 | LR: 4.97e-04 | 710.3ms/step | 28.2 samples/s +Validation at step 3000: Loss = 3.5248, Perplexity = 33.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-3000 +Step 3010/44000 | Loss: 3.5340 | LR: 4.97e-04 | 1620.2ms/step | 12.3 samples/s +Step 3020/44000 | Loss: 3.5230 | LR: 4.97e-04 | 697.4ms/step | 28.7 samples/s +Step 3030/44000 | Loss: 3.4686 | LR: 4.97e-04 | 704.5ms/step | 28.4 samples/s +Step 3040/44000 | Loss: 3.5426 | LR: 4.97e-04 | 714.2ms/step | 28.0 samples/s +Step 3050/44000 | Loss: 3.4709 | LR: 4.97e-04 | 716.1ms/step | 27.9 samples/s +Step 3060/44000 | Loss: 3.5352 | LR: 4.96e-04 | 709.5ms/step | 28.2 samples/s +Step 3070/44000 | Loss: 3.5174 | LR: 4.96e-04 | 709.0ms/step | 28.2 samples/s +Step 3080/44000 | Loss: 3.5230 | LR: 4.96e-04 | 710.1ms/step | 28.2 samples/s +Step 3090/44000 | Loss: 3.5171 | LR: 4.96e-04 | 710.5ms/step | 28.2 samples/s +Step 3100/44000 | Loss: 3.5050 | LR: 4.96e-04 | 711.0ms/step | 28.1 samples/s +Step 3110/44000 | Loss: 3.5097 | LR: 4.96e-04 | 708.2ms/step | 28.2 samples/s +Step 3120/44000 | Loss: 3.5151 | LR: 4.96e-04 | 707.7ms/step | 28.3 samples/s +Step 3130/44000 | Loss: 3.4947 | LR: 4.96e-04 | 703.5ms/step | 28.4 samples/s +Step 3140/44000 | Loss: 3.4995 | LR: 4.96e-04 | 699.1ms/step | 28.6 samples/s +Step 3150/44000 | Loss: 3.4898 | LR: 4.96e-04 | 699.0ms/step | 28.6 samples/s +Step 3160/44000 | Loss: 3.5189 | LR: 4.96e-04 | 700.9ms/step | 28.5 samples/s +Step 3170/44000 | Loss: 3.5048 | LR: 4.96e-04 | 699.3ms/step | 28.6 samples/s +Step 3180/44000 | Loss: 3.4361 | LR: 4.96e-04 | 698.1ms/step | 28.6 samples/s +Step 3190/44000 | Loss: 3.5202 | LR: 4.96e-04 | 700.5ms/step | 28.6 samples/s +Step 3200/44000 | Loss: 3.4599 | LR: 4.96e-04 | 701.3ms/step | 28.5 samples/s +Step 3210/44000 | Loss: 3.4623 | LR: 4.96e-04 | 700.0ms/step | 28.6 samples/s +Step 3220/44000 | Loss: 3.5084 | LR: 4.96e-04 | 700.0ms/step | 28.6 samples/s +Step 3230/44000 | Loss: 3.4675 | LR: 4.96e-04 | 700.0ms/step | 28.6 samples/s +Step 3240/44000 | Loss: 3.4952 | LR: 4.96e-04 | 701.0ms/step | 28.5 samples/s +Step 3250/44000 | Loss: 3.4906 | LR: 4.96e-04 | 698.5ms/step | 28.6 samples/s +Validation at step 3250: Loss = 3.4910, Perplexity = 32.82 +Step 3260/44000 | Loss: 3.4367 | LR: 4.96e-04 | 1293.1ms/step | 15.5 samples/s +Step 3270/44000 | Loss: 3.5045 | LR: 4.96e-04 | 709.6ms/step | 28.2 samples/s +Step 3280/44000 | Loss: 3.4695 | LR: 4.96e-04 | 713.9ms/step | 28.0 samples/s +Step 3290/44000 | Loss: 3.4513 | LR: 4.96e-04 | 710.9ms/step | 28.1 samples/s +Step 3300/44000 | Loss: 3.4711 | LR: 4.96e-04 | 709.1ms/step | 28.2 samples/s +Step 3310/44000 | Loss: 3.4574 | LR: 4.96e-04 | 709.1ms/step | 28.2 samples/s +Step 3320/44000 | Loss: 3.5165 | LR: 4.96e-04 | 711.2ms/step | 28.1 samples/s +Step 3330/44000 | Loss: 3.4619 | LR: 4.96e-04 | 709.8ms/step | 28.2 samples/s +Step 3340/44000 | Loss: 3.4378 | LR: 4.96e-04 | 708.0ms/step | 28.2 samples/s +Step 3350/44000 | Loss: 3.4909 | LR: 4.96e-04 | 706.2ms/step | 28.3 samples/s +Step 3360/44000 | Loss: 3.5068 | LR: 4.96e-04 | 705.9ms/step | 28.3 samples/s +Step 3370/44000 | Loss: 3.4363 | LR: 4.95e-04 | 706.2ms/step | 28.3 samples/s +Step 3380/44000 | Loss: 3.4852 | LR: 4.95e-04 | 703.8ms/step | 28.4 samples/s +Step 3390/44000 | Loss: 3.4542 | LR: 4.95e-04 | 712.0ms/step | 28.1 samples/s +Step 3400/44000 | Loss: 3.4498 | LR: 4.95e-04 | 700.0ms/step | 28.6 samples/s +Step 3410/44000 | Loss: 3.4528 | LR: 4.95e-04 | 703.9ms/step | 28.4 samples/s +Step 3420/44000 | Loss: 3.4699 | LR: 4.95e-04 | 701.9ms/step | 28.5 samples/s +Step 3430/44000 | Loss: 3.4680 | LR: 4.95e-04 | 702.1ms/step | 28.5 samples/s +Step 3440/44000 | Loss: 3.4300 | LR: 4.95e-04 | 703.7ms/step | 28.4 samples/s +Step 3450/44000 | Loss: 3.4156 | LR: 4.95e-04 | 701.3ms/step | 28.5 samples/s +Step 3460/44000 | Loss: 3.4307 | LR: 4.95e-04 | 701.0ms/step | 28.5 samples/s +Step 3470/44000 | Loss: 3.4618 | LR: 4.95e-04 | 705.2ms/step | 28.4 samples/s +Step 3480/44000 | Loss: 3.4557 | LR: 4.95e-04 | 704.5ms/step | 28.4 samples/s +Step 3490/44000 | Loss: 3.4417 | LR: 4.95e-04 | 704.8ms/step | 28.4 samples/s +Step 3500/44000 | Loss: 3.4412 | LR: 4.95e-04 | 707.2ms/step | 28.3 samples/s +Validation at step 3500: Loss = 3.4643, Perplexity = 31.96 +Step 3510/44000 | Loss: 3.4644 | LR: 4.95e-04 | 1299.8ms/step | 15.4 samples/s +Step 3520/44000 | Loss: 3.4410 | LR: 4.95e-04 | 704.2ms/step | 28.4 samples/s +Step 3530/44000 | Loss: 3.4406 | LR: 4.95e-04 | 704.3ms/step | 28.4 samples/s +Step 3540/44000 | Loss: 3.4862 | LR: 4.95e-04 | 705.5ms/step | 28.3 samples/s +Step 3550/44000 | Loss: 3.4500 | LR: 4.95e-04 | 705.2ms/step | 28.4 samples/s +Step 3560/44000 | Loss: 3.4049 | LR: 4.95e-04 | 704.1ms/step | 28.4 samples/s +Step 3570/44000 | Loss: 3.3861 | LR: 4.95e-04 | 704.4ms/step | 28.4 samples/s +Step 3580/44000 | Loss: 3.4592 | LR: 4.95e-04 | 703.4ms/step | 28.4 samples/s +Step 3590/44000 | Loss: 3.4396 | LR: 4.95e-04 | 706.9ms/step | 28.3 samples/s +Step 3600/44000 | Loss: 3.4740 | LR: 4.95e-04 | 704.1ms/step | 28.4 samples/s +Step 3610/44000 | Loss: 3.4283 | LR: 4.95e-04 | 702.3ms/step | 28.5 samples/s +Step 3620/44000 | Loss: 3.4051 | LR: 4.95e-04 | 704.0ms/step | 28.4 samples/s +Step 3630/44000 | Loss: 3.4531 | LR: 4.95e-04 | 703.6ms/step | 28.4 samples/s +Step 3640/44000 | Loss: 3.4331 | LR: 4.95e-04 | 700.0ms/step | 28.6 samples/s +Step 3650/44000 | Loss: 3.4435 | LR: 4.94e-04 | 704.0ms/step | 28.4 samples/s +Step 3660/44000 | Loss: 3.4228 | LR: 4.94e-04 | 704.2ms/step | 28.4 samples/s +Step 3670/44000 | Loss: 3.4008 | LR: 4.94e-04 | 703.8ms/step | 28.4 samples/s +Step 3680/44000 | Loss: 3.4273 | LR: 4.94e-04 | 705.3ms/step | 28.4 samples/s +Step 3690/44000 | Loss: 3.4384 | LR: 4.94e-04 | 702.8ms/step | 28.5 samples/s +Step 3700/44000 | Loss: 3.4430 | LR: 4.94e-04 | 704.4ms/step | 28.4 samples/s +Step 3710/44000 | Loss: 3.4495 | LR: 4.94e-04 | 703.7ms/step | 28.4 samples/s +Step 3720/44000 | Loss: 3.4424 | LR: 4.94e-04 | 702.9ms/step | 28.5 samples/s +Step 3730/44000 | Loss: 3.4119 | LR: 4.94e-04 | 702.9ms/step | 28.5 samples/s +Step 3740/44000 | Loss: 3.4245 | LR: 4.94e-04 | 702.7ms/step | 28.5 samples/s +Step 3750/44000 | Loss: 3.3941 | LR: 4.94e-04 | 703.6ms/step | 28.4 samples/s +Validation at step 3750: Loss = 3.4389, Perplexity = 31.15 +Step 3760/44000 | Loss: 3.4403 | LR: 4.94e-04 | 1293.8ms/step | 15.5 samples/s +Step 3770/44000 | Loss: 3.4310 | LR: 4.94e-04 | 703.9ms/step | 28.4 samples/s +Step 3780/44000 | Loss: 3.4619 | LR: 4.94e-04 | 702.1ms/step | 28.5 samples/s +Step 3790/44000 | Loss: 3.4229 | LR: 4.94e-04 | 700.0ms/step | 28.6 samples/s +Step 3800/44000 | Loss: 3.3992 | LR: 4.94e-04 | 702.0ms/step | 28.5 samples/s +Step 3810/44000 | Loss: 3.4424 | LR: 4.94e-04 | 699.9ms/step | 28.6 samples/s +Step 3820/44000 | Loss: 3.3878 | LR: 4.94e-04 | 702.7ms/step | 28.5 samples/s +Step 3830/44000 | Loss: 3.4490 | LR: 4.94e-04 | 701.3ms/step | 28.5 samples/s +Step 3840/44000 | Loss: 3.4114 | LR: 4.94e-04 | 702.7ms/step | 28.5 samples/s +Step 3850/44000 | Loss: 3.4392 | LR: 4.94e-04 | 701.3ms/step | 28.5 samples/s +Step 3860/44000 | Loss: 3.4282 | LR: 4.94e-04 | 700.0ms/step | 28.6 samples/s +Step 3870/44000 | Loss: 3.4036 | LR: 4.94e-04 | 700.1ms/step | 28.6 samples/s +Step 3880/44000 | Loss: 3.4005 | LR: 4.94e-04 | 700.0ms/step | 28.6 samples/s +Step 3890/44000 | Loss: 3.4313 | LR: 4.94e-04 | 700.0ms/step | 28.6 samples/s +Step 3900/44000 | Loss: 3.4335 | LR: 4.93e-04 | 700.0ms/step | 28.6 samples/s +Step 3910/44000 | Loss: 3.3735 | LR: 4.93e-04 | 699.9ms/step | 28.6 samples/s +Step 3920/44000 | Loss: 3.4188 | LR: 4.93e-04 | 700.0ms/step | 28.6 samples/s +Step 3930/44000 | Loss: 3.3446 | LR: 4.93e-04 | 700.0ms/step | 28.6 samples/s +Step 3940/44000 | Loss: 3.3673 | LR: 4.93e-04 | 700.0ms/step | 28.6 samples/s +Step 3950/44000 | Loss: 3.4175 | LR: 4.93e-04 | 699.8ms/step | 28.6 samples/s +Step 3960/44000 | Loss: 3.4211 | LR: 4.93e-04 | 700.2ms/step | 28.6 samples/s +Step 3970/44000 | Loss: 3.4085 | LR: 4.93e-04 | 700.4ms/step | 28.6 samples/s +Step 3980/44000 | Loss: 3.4048 | LR: 4.93e-04 | 699.7ms/step | 28.6 samples/s +Step 3990/44000 | Loss: 3.4309 | LR: 4.93e-04 | 699.9ms/step | 28.6 samples/s +Step 4000/44000 | Loss: 3.3990 | LR: 4.93e-04 | 700.0ms/step | 28.6 samples/s +Validation at step 4000: Loss = 3.4165, Perplexity = 30.46 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-4000 +Step 4010/44000 | Loss: 3.4302 | LR: 4.93e-04 | 1638.1ms/step | 12.2 samples/s +Step 4020/44000 | Loss: 3.4175 | LR: 4.93e-04 | 700.0ms/step | 28.6 samples/s +Step 4030/44000 | Loss: 3.4445 | LR: 4.93e-04 | 701.9ms/step | 28.5 samples/s +Step 4040/44000 | Loss: 3.4375 | LR: 4.93e-04 | 700.0ms/step | 28.6 samples/s +Step 4050/44000 | Loss: 3.3789 | LR: 4.93e-04 | 700.4ms/step | 28.6 samples/s +Step 4060/44000 | Loss: 3.4047 | LR: 4.93e-04 | 699.7ms/step | 28.6 samples/s +Step 4070/44000 | Loss: 3.3625 | LR: 4.93e-04 | 701.1ms/step | 28.5 samples/s +Step 4080/44000 | Loss: 3.3783 | LR: 4.93e-04 | 702.7ms/step | 28.5 samples/s +Step 4090/44000 | Loss: 3.3834 | LR: 4.93e-04 | 700.8ms/step | 28.5 samples/s +Step 4100/44000 | Loss: 3.4051 | LR: 4.93e-04 | 699.2ms/step | 28.6 samples/s +Step 4110/44000 | Loss: 3.4176 | LR: 4.93e-04 | 701.2ms/step | 28.5 samples/s +Step 4120/44000 | Loss: 3.3690 | LR: 4.93e-04 | 700.8ms/step | 28.5 samples/s +Step 4130/44000 | Loss: 3.3801 | LR: 4.93e-04 | 699.9ms/step | 28.6 samples/s +Step 4140/44000 | Loss: 3.4070 | LR: 4.92e-04 | 700.3ms/step | 28.6 samples/s +Step 4150/44000 | Loss: 3.4145 | LR: 4.92e-04 | 699.7ms/step | 28.6 samples/s +Step 4160/44000 | Loss: 3.3873 | LR: 4.92e-04 | 700.2ms/step | 28.6 samples/s +Step 4170/44000 | Loss: 3.3954 | LR: 4.92e-04 | 699.8ms/step | 28.6 samples/s +Step 4180/44000 | Loss: 3.3927 | LR: 4.92e-04 | 700.1ms/step | 28.6 samples/s +Step 4190/44000 | Loss: 3.4017 | LR: 4.92e-04 | 699.9ms/step | 28.6 samples/s +Step 4200/44000 | Loss: 3.3814 | LR: 4.92e-04 | 699.8ms/step | 28.6 samples/s +Step 4210/44000 | Loss: 3.4026 | LR: 4.92e-04 | 700.3ms/step | 28.6 samples/s +Step 4220/44000 | Loss: 3.3372 | LR: 4.92e-04 | 699.9ms/step | 28.6 samples/s +Step 4230/44000 | Loss: 3.3490 | LR: 4.92e-04 | 700.1ms/step | 28.6 samples/s +Step 4240/44000 | Loss: 3.3857 | LR: 4.92e-04 | 700.9ms/step | 28.5 samples/s +Step 4250/44000 | Loss: 3.3843 | LR: 4.92e-04 | 700.9ms/step | 28.5 samples/s +Validation at step 4250: Loss = 3.3965, Perplexity = 29.86 +Step 4260/44000 | Loss: 3.3918 | LR: 4.92e-04 | 1286.0ms/step | 15.6 samples/s +Step 4270/44000 | Loss: 3.3912 | LR: 4.92e-04 | 700.0ms/step | 28.6 samples/s +Step 4280/44000 | Loss: 3.3931 | LR: 4.92e-04 | 700.0ms/step | 28.6 samples/s +Step 4290/44000 | Loss: 3.3914 | LR: 4.92e-04 | 699.9ms/step | 28.6 samples/s +Step 4300/44000 | Loss: 3.4290 | LR: 4.92e-04 | 699.9ms/step | 28.6 samples/s +Step 4310/44000 | Loss: 3.3386 | LR: 4.92e-04 | 700.2ms/step | 28.6 samples/s +Step 4320/44000 | Loss: 3.4173 | LR: 4.92e-04 | 700.0ms/step | 28.6 samples/s +Step 4330/44000 | Loss: 3.4121 | LR: 4.92e-04 | 699.8ms/step | 28.6 samples/s +Step 4340/44000 | Loss: 3.3502 | LR: 4.92e-04 | 700.1ms/step | 28.6 samples/s +Step 4350/44000 | Loss: 3.3983 | LR: 4.92e-04 | 701.2ms/step | 28.5 samples/s +Step 4360/44000 | Loss: 3.3756 | LR: 4.91e-04 | 700.8ms/step | 28.5 samples/s +Step 4370/44000 | Loss: 3.3929 | LR: 4.91e-04 | 699.8ms/step | 28.6 samples/s +Step 4380/44000 | Loss: 3.3636 | LR: 4.91e-04 | 699.4ms/step | 28.6 samples/s +Step 4390/44000 | Loss: 3.4470 | LR: 4.91e-04 | 698.4ms/step | 28.6 samples/s +Step 4400/44000 | Loss: 3.3738 | LR: 4.91e-04 | 698.8ms/step | 28.6 samples/s +Step 4410/44000 | Loss: 3.3344 | LR: 4.91e-04 | 697.2ms/step | 28.7 samples/s +Step 4420/44000 | Loss: 3.3529 | LR: 4.91e-04 | 700.5ms/step | 28.6 samples/s +Step 4430/44000 | Loss: 3.3850 | LR: 4.91e-04 | 699.4ms/step | 28.6 samples/s +Step 4440/44000 | Loss: 3.3493 | LR: 4.91e-04 | 698.3ms/step | 28.6 samples/s +Step 4450/44000 | Loss: 3.4026 | LR: 4.91e-04 | 698.7ms/step | 28.6 samples/s +Step 4460/44000 | Loss: 3.3555 | LR: 4.91e-04 | 699.3ms/step | 28.6 samples/s +Step 4470/44000 | Loss: 3.3526 | LR: 4.91e-04 | 698.4ms/step | 28.6 samples/s +Step 4480/44000 | Loss: 3.3921 | LR: 4.91e-04 | 696.6ms/step | 28.7 samples/s +Step 4490/44000 | Loss: 3.4075 | LR: 4.91e-04 | 700.5ms/step | 28.5 samples/s +Step 4500/44000 | Loss: 3.3814 | LR: 4.91e-04 | 698.2ms/step | 28.6 samples/s +Validation at step 4500: Loss = 3.3770, Perplexity = 29.28 +Step 4510/44000 | Loss: 3.3769 | LR: 4.91e-04 | 1282.6ms/step | 15.6 samples/s +Step 4520/44000 | Loss: 3.4082 | LR: 4.91e-04 | 699.8ms/step | 28.6 samples/s +Step 4530/44000 | Loss: 3.3694 | LR: 4.91e-04 | 699.9ms/step | 28.6 samples/s +Step 4540/44000 | Loss: 3.3588 | LR: 4.91e-04 | 699.7ms/step | 28.6 samples/s +Step 4550/44000 | Loss: 3.3934 | LR: 4.91e-04 | 699.1ms/step | 28.6 samples/s +Step 4560/44000 | Loss: 3.3101 | LR: 4.90e-04 | 697.6ms/step | 28.7 samples/s +Step 4570/44000 | Loss: 3.3769 | LR: 4.90e-04 | 699.4ms/step | 28.6 samples/s +Step 4580/44000 | Loss: 3.3440 | LR: 4.90e-04 | 697.5ms/step | 28.7 samples/s +Step 4590/44000 | Loss: 3.3661 | LR: 4.90e-04 | 698.4ms/step | 28.6 samples/s +Step 4600/44000 | Loss: 3.3660 | LR: 4.90e-04 | 697.8ms/step | 28.7 samples/s +Step 4610/44000 | Loss: 3.3055 | LR: 4.90e-04 | 695.8ms/step | 28.7 samples/s +Step 4620/44000 | Loss: 3.3977 | LR: 4.90e-04 | 696.5ms/step | 28.7 samples/s +Step 4630/44000 | Loss: 3.3602 | LR: 4.90e-04 | 697.2ms/step | 28.7 samples/s +Step 4640/44000 | Loss: 3.3809 | LR: 4.90e-04 | 696.8ms/step | 28.7 samples/s +Step 4650/44000 | Loss: 3.3944 | LR: 4.90e-04 | 697.1ms/step | 28.7 samples/s +Step 4660/44000 | Loss: 3.3746 | LR: 4.90e-04 | 695.4ms/step | 28.8 samples/s +Step 4670/44000 | Loss: 3.3725 | LR: 4.90e-04 | 697.0ms/step | 28.7 samples/s +Step 4680/44000 | Loss: 3.4105 | LR: 4.90e-04 | 696.0ms/step | 28.7 samples/s +Step 4690/44000 | Loss: 3.3452 | LR: 4.90e-04 | 694.5ms/step | 28.8 samples/s +Step 4700/44000 | Loss: 3.3430 | LR: 4.90e-04 | 695.5ms/step | 28.8 samples/s +Step 4710/44000 | Loss: 3.3688 | LR: 4.90e-04 | 695.7ms/step | 28.7 samples/s +Step 4720/44000 | Loss: 3.3238 | LR: 4.90e-04 | 695.4ms/step | 28.8 samples/s +Step 4730/44000 | Loss: 3.3899 | LR: 4.90e-04 | 696.3ms/step | 28.7 samples/s +Step 4740/44000 | Loss: 3.3458 | LR: 4.90e-04 | 697.0ms/step | 28.7 samples/s +Step 4750/44000 | Loss: 3.3398 | LR: 4.90e-04 | 697.8ms/step | 28.7 samples/s +Validation at step 4750: Loss = 3.3634, Perplexity = 28.89 +Step 4760/44000 | Loss: 3.3586 | LR: 4.89e-04 | 1284.2ms/step | 15.6 samples/s +Step 4770/44000 | Loss: 3.3574 | LR: 4.89e-04 | 697.2ms/step | 28.7 samples/s +Step 4780/44000 | Loss: 3.3664 | LR: 4.89e-04 | 696.8ms/step | 28.7 samples/s +Step 4790/44000 | Loss: 3.2801 | LR: 4.89e-04 | 698.9ms/step | 28.6 samples/s +Step 4800/44000 | Loss: 3.3672 | LR: 4.89e-04 | 699.0ms/step | 28.6 samples/s +Step 4810/44000 | Loss: 3.3262 | LR: 4.89e-04 | 696.9ms/step | 28.7 samples/s +Step 4820/44000 | Loss: 3.3291 | LR: 4.89e-04 | 697.0ms/step | 28.7 samples/s +Step 4830/44000 | Loss: 3.3598 | LR: 4.89e-04 | 696.8ms/step | 28.7 samples/s +Step 4840/44000 | Loss: 3.3397 | LR: 4.89e-04 | 696.3ms/step | 28.7 samples/s +Step 4850/44000 | Loss: 3.3358 | LR: 4.89e-04 | 696.8ms/step | 28.7 samples/s +Step 4860/44000 | Loss: 3.3347 | LR: 4.89e-04 | 698.0ms/step | 28.7 samples/s +Step 4870/44000 | Loss: 3.3202 | LR: 4.89e-04 | 698.1ms/step | 28.6 samples/s +Step 4880/44000 | Loss: 3.3371 | LR: 4.89e-04 | 697.3ms/step | 28.7 samples/s +Step 4890/44000 | Loss: 3.3181 | LR: 4.89e-04 | 696.1ms/step | 28.7 samples/s +Step 4900/44000 | Loss: 3.3225 | LR: 4.89e-04 | 696.4ms/step | 28.7 samples/s +Step 4910/44000 | Loss: 3.2976 | LR: 4.89e-04 | 698.2ms/step | 28.6 samples/s +Step 4920/44000 | Loss: 3.3514 | LR: 4.89e-04 | 696.9ms/step | 28.7 samples/s +Step 4930/44000 | Loss: 3.3312 | LR: 4.89e-04 | 695.7ms/step | 28.7 samples/s +Step 4940/44000 | Loss: 3.3430 | LR: 4.89e-04 | 696.5ms/step | 28.7 samples/s +Step 4950/44000 | Loss: 3.3052 | LR: 4.88e-04 | 698.4ms/step | 28.6 samples/s +Step 4960/44000 | Loss: 3.3652 | LR: 4.88e-04 | 695.9ms/step | 28.7 samples/s +Step 4970/44000 | Loss: 3.3197 | LR: 4.88e-04 | 696.7ms/step | 28.7 samples/s +Step 4980/44000 | Loss: 3.3789 | LR: 4.88e-04 | 695.7ms/step | 28.7 samples/s +Step 4990/44000 | Loss: 3.3260 | LR: 4.88e-04 | 697.0ms/step | 28.7 samples/s +Step 5000/44000 | Loss: 3.3539 | LR: 4.88e-04 | 697.3ms/step | 28.7 samples/s +Validation at step 5000: Loss = 3.3502, Perplexity = 28.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-5000 +Step 5010/44000 | Loss: 3.3371 | LR: 4.88e-04 | 1574.0ms/step | 12.7 samples/s +Step 5020/44000 | Loss: 3.3464 | LR: 4.88e-04 | 697.1ms/step | 28.7 samples/s +Step 5030/44000 | Loss: 3.3196 | LR: 4.88e-04 | 695.1ms/step | 28.8 samples/s +Step 5040/44000 | Loss: 3.2857 | LR: 4.88e-04 | 696.9ms/step | 28.7 samples/s +Step 5050/44000 | Loss: 3.3181 | LR: 4.88e-04 | 698.6ms/step | 28.6 samples/s +Step 5060/44000 | Loss: 3.3028 | LR: 4.88e-04 | 697.7ms/step | 28.7 samples/s +Step 5070/44000 | Loss: 3.3388 | LR: 4.88e-04 | 696.2ms/step | 28.7 samples/s +Step 5080/44000 | Loss: 3.3218 | LR: 4.88e-04 | 696.4ms/step | 28.7 samples/s +Step 5090/44000 | Loss: 3.3035 | LR: 4.88e-04 | 698.2ms/step | 28.6 samples/s +Step 5100/44000 | Loss: 3.2883 | LR: 4.88e-04 | 697.7ms/step | 28.7 samples/s +Step 5110/44000 | Loss: 3.3079 | LR: 4.88e-04 | 696.4ms/step | 28.7 samples/s +Step 5120/44000 | Loss: 3.3084 | LR: 4.88e-04 | 697.1ms/step | 28.7 samples/s +Step 5130/44000 | Loss: 3.3258 | LR: 4.87e-04 | 695.9ms/step | 28.7 samples/s +Step 5140/44000 | Loss: 3.3033 | LR: 4.87e-04 | 695.3ms/step | 28.8 samples/s +Step 5150/44000 | Loss: 3.2807 | LR: 4.87e-04 | 693.6ms/step | 28.8 samples/s +Step 5160/44000 | Loss: 3.3214 | LR: 4.87e-04 | 696.7ms/step | 28.7 samples/s +Step 5170/44000 | Loss: 3.2952 | LR: 4.87e-04 | 695.3ms/step | 28.8 samples/s +Step 5180/44000 | Loss: 3.3170 | LR: 4.87e-04 | 695.2ms/step | 28.8 samples/s +Step 5190/44000 | Loss: 3.3486 | LR: 4.87e-04 | 698.1ms/step | 28.6 samples/s +Step 5200/44000 | Loss: 3.2937 | LR: 4.87e-04 | 699.3ms/step | 28.6 samples/s +Step 5210/44000 | Loss: 3.3051 | LR: 4.87e-04 | 698.4ms/step | 28.6 samples/s +Step 5220/44000 | Loss: 3.2645 | LR: 4.87e-04 | 696.4ms/step | 28.7 samples/s +Step 5230/44000 | Loss: 3.3136 | LR: 4.87e-04 | 695.9ms/step | 28.7 samples/s +Step 5240/44000 | Loss: 3.3074 | LR: 4.87e-04 | 696.6ms/step | 28.7 samples/s +Step 5250/44000 | Loss: 3.3227 | LR: 4.87e-04 | 694.8ms/step | 28.8 samples/s +Validation at step 5250: Loss = 3.3304, Perplexity = 27.95 +Step 5260/44000 | Loss: 3.3274 | LR: 4.87e-04 | 1275.0ms/step | 15.7 samples/s +Step 5270/44000 | Loss: 3.3585 | LR: 4.87e-04 | 697.5ms/step | 28.7 samples/s +Step 5280/44000 | Loss: 3.2706 | LR: 4.87e-04 | 697.4ms/step | 28.7 samples/s +Step 5290/44000 | Loss: 3.3321 | LR: 4.87e-04 | 696.8ms/step | 28.7 samples/s +Step 5300/44000 | Loss: 3.3181 | LR: 4.86e-04 | 697.2ms/step | 28.7 samples/s +Step 5310/44000 | Loss: 3.2983 | LR: 4.86e-04 | 698.2ms/step | 28.6 samples/s +Step 5320/44000 | Loss: 3.3103 | LR: 4.86e-04 | 696.1ms/step | 28.7 samples/s +Step 5330/44000 | Loss: 3.3136 | LR: 4.86e-04 | 696.9ms/step | 28.7 samples/s +Step 5340/44000 | Loss: 3.3031 | LR: 4.86e-04 | 697.1ms/step | 28.7 samples/s +Step 5350/44000 | Loss: 3.2956 | LR: 4.86e-04 | 696.9ms/step | 28.7 samples/s +Step 5360/44000 | Loss: 3.3204 | LR: 4.86e-04 | 698.0ms/step | 28.7 samples/s +Step 5370/44000 | Loss: 3.2873 | LR: 4.86e-04 | 698.2ms/step | 28.6 samples/s +Step 5380/44000 | Loss: 3.2977 | LR: 4.86e-04 | 697.7ms/step | 28.7 samples/s +Step 5390/44000 | Loss: 3.3038 | LR: 4.86e-04 | 696.6ms/step | 28.7 samples/s +Step 5400/44000 | Loss: 3.2878 | LR: 4.86e-04 | 696.7ms/step | 28.7 samples/s +Step 5410/44000 | Loss: 3.3020 | LR: 4.86e-04 | 697.1ms/step | 28.7 samples/s +Step 5420/44000 | Loss: 3.3397 | LR: 4.86e-04 | 697.7ms/step | 28.7 samples/s +Step 5430/44000 | Loss: 3.2769 | LR: 4.86e-04 | 697.2ms/step | 28.7 samples/s +Step 5440/44000 | Loss: 3.3192 | LR: 4.86e-04 | 695.6ms/step | 28.8 samples/s +Step 5450/44000 | Loss: 3.2930 | LR: 4.86e-04 | 694.9ms/step | 28.8 samples/s +Step 5460/44000 | Loss: 3.3474 | LR: 4.86e-04 | 694.8ms/step | 28.8 samples/s +Step 5470/44000 | Loss: 3.3363 | LR: 4.85e-04 | 695.3ms/step | 28.8 samples/s +Step 5480/44000 | Loss: 3.3139 | LR: 4.85e-04 | 695.4ms/step | 28.8 samples/s +Step 5490/44000 | Loss: 3.3263 | LR: 4.85e-04 | 695.0ms/step | 28.8 samples/s +Step 5500/44000 | Loss: 3.2924 | LR: 4.85e-04 | 694.6ms/step | 28.8 samples/s +Validation at step 5500: Loss = 3.3159, Perplexity = 27.55 +Step 5510/44000 | Loss: 3.3334 | LR: 4.85e-04 | 1270.3ms/step | 15.7 samples/s +Step 5520/44000 | Loss: 3.3148 | LR: 4.85e-04 | 693.9ms/step | 28.8 samples/s +Step 5530/44000 | Loss: 3.3302 | LR: 4.85e-04 | 693.4ms/step | 28.8 samples/s +Step 5540/44000 | Loss: 3.2996 | LR: 4.85e-04 | 693.8ms/step | 28.8 samples/s +Step 5550/44000 | Loss: 3.3293 | LR: 4.85e-04 | 695.5ms/step | 28.8 samples/s +Step 5560/44000 | Loss: 3.2843 | LR: 4.85e-04 | 694.4ms/step | 28.8 samples/s +Step 5570/44000 | Loss: 3.3009 | LR: 4.85e-04 | 695.6ms/step | 28.8 samples/s +Step 5580/44000 | Loss: 3.2980 | LR: 4.85e-04 | 694.6ms/step | 28.8 samples/s +Step 5590/44000 | Loss: 3.3330 | LR: 4.85e-04 | 694.4ms/step | 28.8 samples/s +Step 5600/44000 | Loss: 3.3361 | LR: 4.85e-04 | 693.2ms/step | 28.9 samples/s +Step 5610/44000 | Loss: 3.3276 | LR: 4.85e-04 | 694.6ms/step | 28.8 samples/s +Step 5620/44000 | Loss: 3.3100 | LR: 4.85e-04 | 694.3ms/step | 28.8 samples/s +Step 5630/44000 | Loss: 3.2906 | LR: 4.84e-04 | 693.8ms/step | 28.8 samples/s +Step 5640/44000 | Loss: 3.2704 | LR: 4.84e-04 | 692.9ms/step | 28.9 samples/s +Step 5650/44000 | Loss: 3.2696 | LR: 4.84e-04 | 693.7ms/step | 28.8 samples/s +Step 5660/44000 | Loss: 3.2938 | LR: 4.84e-04 | 693.5ms/step | 28.8 samples/s +Step 5670/44000 | Loss: 3.2712 | LR: 4.84e-04 | 695.8ms/step | 28.7 samples/s +Step 5680/44000 | Loss: 3.3508 | LR: 4.84e-04 | 695.6ms/step | 28.8 samples/s +Step 5690/44000 | Loss: 3.3271 | LR: 4.84e-04 | 692.8ms/step | 28.9 samples/s +Step 5700/44000 | Loss: 3.2730 | LR: 4.84e-04 | 694.7ms/step | 28.8 samples/s +Step 5710/44000 | Loss: 3.3194 | LR: 4.84e-04 | 694.0ms/step | 28.8 samples/s +Step 5720/44000 | Loss: 3.2915 | LR: 4.84e-04 | 694.3ms/step | 28.8 samples/s +Step 5730/44000 | Loss: 3.2868 | LR: 4.84e-04 | 695.0ms/step | 28.8 samples/s +Step 5740/44000 | Loss: 3.2702 | LR: 4.84e-04 | 692.7ms/step | 28.9 samples/s +Step 5750/44000 | Loss: 3.2539 | LR: 4.84e-04 | 692.4ms/step | 28.9 samples/s +Validation at step 5750: Loss = 3.3015, Perplexity = 27.15 +Step 5760/44000 | Loss: 3.2834 | LR: 4.84e-04 | 1268.2ms/step | 15.8 samples/s +Step 5770/44000 | Loss: 3.3036 | LR: 4.84e-04 | 693.9ms/step | 28.8 samples/s +Step 5780/44000 | Loss: 3.2996 | LR: 4.83e-04 | 693.2ms/step | 28.9 samples/s +Step 5790/44000 | Loss: 3.2854 | LR: 4.83e-04 | 694.1ms/step | 28.8 samples/s +Step 5800/44000 | Loss: 3.2893 | LR: 4.83e-04 | 693.2ms/step | 28.9 samples/s +Step 5810/44000 | Loss: 3.2849 | LR: 4.83e-04 | 692.4ms/step | 28.9 samples/s +Step 5820/44000 | Loss: 3.2872 | LR: 4.83e-04 | 694.9ms/step | 28.8 samples/s +Step 5830/44000 | Loss: 3.2888 | LR: 4.83e-04 | 694.0ms/step | 28.8 samples/s +Step 5840/44000 | Loss: 3.2760 | LR: 4.83e-04 | 694.5ms/step | 28.8 samples/s +Step 5850/44000 | Loss: 3.2381 | LR: 4.83e-04 | 695.2ms/step | 28.8 samples/s +Step 5860/44000 | Loss: 3.3224 | LR: 4.83e-04 | 695.1ms/step | 28.8 samples/s +Step 5870/44000 | Loss: 3.2407 | LR: 4.83e-04 | 693.6ms/step | 28.8 samples/s +Step 5880/44000 | Loss: 3.2840 | LR: 4.83e-04 | 693.1ms/step | 28.9 samples/s +Step 5890/44000 | Loss: 3.3236 | LR: 4.83e-04 | 692.2ms/step | 28.9 samples/s +Step 5900/44000 | Loss: 3.2522 | LR: 4.83e-04 | 692.4ms/step | 28.9 samples/s +Step 5910/44000 | Loss: 3.2683 | LR: 4.83e-04 | 693.1ms/step | 28.9 samples/s +Step 5920/44000 | Loss: 3.3089 | LR: 4.83e-04 | 692.9ms/step | 28.9 samples/s +Step 5930/44000 | Loss: 3.2723 | LR: 4.83e-04 | 694.5ms/step | 28.8 samples/s +Step 5940/44000 | Loss: 3.2558 | LR: 4.82e-04 | 692.5ms/step | 28.9 samples/s +Step 5950/44000 | Loss: 3.2701 | LR: 4.82e-04 | 693.6ms/step | 28.8 samples/s +Step 5960/44000 | Loss: 3.2839 | LR: 4.82e-04 | 692.3ms/step | 28.9 samples/s +Step 5970/44000 | Loss: 3.2239 | LR: 4.82e-04 | 693.1ms/step | 28.9 samples/s +Step 5980/44000 | Loss: 3.2940 | LR: 4.82e-04 | 694.0ms/step | 28.8 samples/s +Step 5990/44000 | Loss: 3.2792 | LR: 4.82e-04 | 692.6ms/step | 28.9 samples/s +Step 6000/44000 | Loss: 3.3041 | LR: 4.82e-04 | 693.4ms/step | 28.8 samples/s +Validation at step 6000: Loss = 3.2901, Perplexity = 26.84 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-6000 +Step 6010/44000 | Loss: 3.2738 | LR: 4.82e-04 | 1632.0ms/step | 12.3 samples/s +Step 6020/44000 | Loss: 3.2856 | LR: 4.82e-04 | 695.8ms/step | 28.7 samples/s +Step 6030/44000 | Loss: 3.3238 | LR: 4.82e-04 | 695.4ms/step | 28.8 samples/s +Step 6040/44000 | Loss: 3.2672 | LR: 4.82e-04 | 695.2ms/step | 28.8 samples/s +Step 6050/44000 | Loss: 3.2850 | LR: 4.82e-04 | 697.1ms/step | 28.7 samples/s +Step 6060/44000 | Loss: 3.2714 | LR: 4.82e-04 | 696.8ms/step | 28.7 samples/s +Step 6070/44000 | Loss: 3.2552 | LR: 4.82e-04 | 695.0ms/step | 28.8 samples/s +Step 6080/44000 | Loss: 3.2643 | LR: 4.81e-04 | 695.4ms/step | 28.8 samples/s +Step 6090/44000 | Loss: 3.2492 | LR: 4.81e-04 | 694.9ms/step | 28.8 samples/s +Step 6100/44000 | Loss: 3.2307 | LR: 4.81e-04 | 696.9ms/step | 28.7 samples/s +Step 6110/44000 | Loss: 3.2682 | LR: 4.81e-04 | 698.0ms/step | 28.7 samples/s +Step 6120/44000 | Loss: 3.2914 | LR: 4.81e-04 | 695.8ms/step | 28.7 samples/s +Step 6130/44000 | Loss: 3.2430 | LR: 4.81e-04 | 695.1ms/step | 28.8 samples/s +Step 6140/44000 | Loss: 3.2757 | LR: 4.81e-04 | 694.9ms/step | 28.8 samples/s +Step 6150/44000 | Loss: 3.2434 | LR: 4.81e-04 | 697.3ms/step | 28.7 samples/s +Step 6160/44000 | Loss: 3.2954 | LR: 4.81e-04 | 695.7ms/step | 28.7 samples/s +Step 6170/44000 | Loss: 3.2595 | LR: 4.81e-04 | 695.8ms/step | 28.7 samples/s +Step 6180/44000 | Loss: 3.3087 | LR: 4.81e-04 | 694.5ms/step | 28.8 samples/s +Step 6190/44000 | Loss: 3.2467 | LR: 4.81e-04 | 695.2ms/step | 28.8 samples/s +Step 6200/44000 | Loss: 3.2587 | LR: 4.81e-04 | 694.7ms/step | 28.8 samples/s +Step 6210/44000 | Loss: 3.2714 | LR: 4.81e-04 | 696.6ms/step | 28.7 samples/s +Step 6220/44000 | Loss: 3.2238 | LR: 4.81e-04 | 695.4ms/step | 28.8 samples/s +Step 6230/44000 | Loss: 3.2895 | LR: 4.80e-04 | 695.9ms/step | 28.7 samples/s +Step 6240/44000 | Loss: 3.2498 | LR: 4.80e-04 | 695.9ms/step | 28.7 samples/s +Step 6250/44000 | Loss: 3.2496 | LR: 4.80e-04 | 695.1ms/step | 28.8 samples/s +Validation at step 6250: Loss = 3.2786, Perplexity = 26.54 +Step 6260/44000 | Loss: 3.2642 | LR: 4.80e-04 | 1273.1ms/step | 15.7 samples/s +Step 6270/44000 | Loss: 3.2765 | LR: 4.80e-04 | 695.6ms/step | 28.8 samples/s +Step 6280/44000 | Loss: 3.2209 | LR: 4.80e-04 | 696.0ms/step | 28.7 samples/s +Step 6290/44000 | Loss: 3.2684 | LR: 4.80e-04 | 695.0ms/step | 28.8 samples/s +Step 6300/44000 | Loss: 3.2912 | LR: 4.80e-04 | 696.6ms/step | 28.7 samples/s +Step 6310/44000 | Loss: 3.3259 | LR: 4.80e-04 | 697.5ms/step | 28.7 samples/s +Step 6320/44000 | Loss: 3.2634 | LR: 4.80e-04 | 694.9ms/step | 28.8 samples/s +Step 6330/44000 | Loss: 3.2372 | LR: 4.80e-04 | 696.5ms/step | 28.7 samples/s +Step 6340/44000 | Loss: 3.2765 | LR: 4.80e-04 | 698.9ms/step | 28.6 samples/s +Step 6350/44000 | Loss: 3.2432 | LR: 4.80e-04 | 696.4ms/step | 28.7 samples/s +Step 6360/44000 | Loss: 3.2201 | LR: 4.80e-04 | 695.4ms/step | 28.8 samples/s +Step 6370/44000 | Loss: 3.2353 | LR: 4.79e-04 | 697.0ms/step | 28.7 samples/s +Step 6380/44000 | Loss: 3.2668 | LR: 4.79e-04 | 694.5ms/step | 28.8 samples/s +Step 6390/44000 | Loss: 3.2602 | LR: 4.79e-04 | 694.2ms/step | 28.8 samples/s +Step 6400/44000 | Loss: 3.2772 | LR: 4.79e-04 | 693.6ms/step | 28.8 samples/s +Step 6410/44000 | Loss: 3.2225 | LR: 4.79e-04 | 694.0ms/step | 28.8 samples/s +Step 6420/44000 | Loss: 3.2974 | LR: 4.79e-04 | 695.8ms/step | 28.7 samples/s +Step 6430/44000 | Loss: 3.2464 | LR: 4.79e-04 | 694.1ms/step | 28.8 samples/s +Step 6440/44000 | Loss: 3.2850 | LR: 4.79e-04 | 692.7ms/step | 28.9 samples/s +Step 6450/44000 | Loss: 3.2491 | LR: 4.79e-04 | 693.4ms/step | 28.8 samples/s +Step 6460/44000 | Loss: 3.2562 | LR: 4.79e-04 | 693.0ms/step | 28.9 samples/s +Step 6470/44000 | Loss: 3.2390 | LR: 4.79e-04 | 693.1ms/step | 28.9 samples/s +Step 6480/44000 | Loss: 3.2154 | LR: 4.79e-04 | 694.0ms/step | 28.8 samples/s +Step 6490/44000 | Loss: 3.2582 | LR: 4.79e-04 | 693.0ms/step | 28.9 samples/s +Step 6500/44000 | Loss: 3.2515 | LR: 4.79e-04 | 694.2ms/step | 28.8 samples/s +Validation at step 6500: Loss = 3.2673, Perplexity = 26.24 +Step 6510/44000 | Loss: 3.2300 | LR: 4.78e-04 | 1269.5ms/step | 15.8 samples/s +Step 6520/44000 | Loss: 3.2892 | LR: 4.78e-04 | 695.5ms/step | 28.8 samples/s +Step 6530/44000 | Loss: 3.2823 | LR: 4.78e-04 | 694.2ms/step | 28.8 samples/s +Step 6540/44000 | Loss: 3.2420 | LR: 4.78e-04 | 694.2ms/step | 28.8 samples/s +Step 6550/44000 | Loss: 3.2678 | LR: 4.78e-04 | 693.9ms/step | 28.8 samples/s +Step 6560/44000 | Loss: 3.2133 | LR: 4.78e-04 | 694.1ms/step | 28.8 samples/s +Step 6570/44000 | Loss: 3.2311 | LR: 4.78e-04 | 695.1ms/step | 28.8 samples/s +Step 6580/44000 | Loss: 3.2683 | LR: 4.78e-04 | 694.7ms/step | 28.8 samples/s +Step 6590/44000 | Loss: 3.2571 | LR: 4.78e-04 | 694.5ms/step | 28.8 samples/s +Step 6600/44000 | Loss: 3.2366 | LR: 4.78e-04 | 694.2ms/step | 28.8 samples/s +Step 6610/44000 | Loss: 3.2489 | LR: 4.78e-04 | 695.3ms/step | 28.8 samples/s +Step 6620/44000 | Loss: 3.2758 | LR: 4.78e-04 | 694.8ms/step | 28.8 samples/s +Step 6630/44000 | Loss: 3.2162 | LR: 4.78e-04 | 696.3ms/step | 28.7 samples/s +Step 6640/44000 | Loss: 3.2386 | LR: 4.77e-04 | 694.3ms/step | 28.8 samples/s +Step 6650/44000 | Loss: 3.2633 | LR: 4.77e-04 | 693.5ms/step | 28.8 samples/s +Step 6660/44000 | Loss: 3.2780 | LR: 4.77e-04 | 694.0ms/step | 28.8 samples/s +Step 6670/44000 | Loss: 3.2354 | LR: 4.77e-04 | 692.6ms/step | 28.9 samples/s +Step 6680/44000 | Loss: 3.2564 | LR: 4.77e-04 | 693.6ms/step | 28.8 samples/s +Step 6690/44000 | Loss: 3.2548 | LR: 4.77e-04 | 693.8ms/step | 28.8 samples/s +Step 6700/44000 | Loss: 3.2682 | LR: 4.77e-04 | 692.5ms/step | 28.9 samples/s +Step 6710/44000 | Loss: 3.2766 | LR: 4.77e-04 | 693.4ms/step | 28.8 samples/s +Step 6720/44000 | Loss: 3.2591 | LR: 4.77e-04 | 693.5ms/step | 28.8 samples/s +Step 6730/44000 | Loss: 3.2647 | LR: 4.77e-04 | 693.8ms/step | 28.8 samples/s +Step 6740/44000 | Loss: 3.2811 | LR: 4.77e-04 | 694.0ms/step | 28.8 samples/s +Step 6750/44000 | Loss: 3.2698 | LR: 4.77e-04 | 693.5ms/step | 28.8 samples/s +Validation at step 6750: Loss = 3.2591, Perplexity = 26.02 +Step 6760/44000 | Loss: 3.2475 | LR: 4.77e-04 | 1266.3ms/step | 15.8 samples/s +Step 6770/44000 | Loss: 3.2447 | LR: 4.76e-04 | 694.5ms/step | 28.8 samples/s +Step 6780/44000 | Loss: 3.2701 | LR: 4.76e-04 | 696.1ms/step | 28.7 samples/s +Step 6790/44000 | Loss: 3.2387 | LR: 4.76e-04 | 695.1ms/step | 28.8 samples/s +Step 6800/44000 | Loss: 3.2412 | LR: 4.76e-04 | 693.5ms/step | 28.8 samples/s +Step 6810/44000 | Loss: 3.2695 | LR: 4.76e-04 | 696.4ms/step | 28.7 samples/s +Step 6820/44000 | Loss: 3.2474 | LR: 4.76e-04 | 694.1ms/step | 28.8 samples/s +Step 6830/44000 | Loss: 3.2533 | LR: 4.76e-04 | 693.9ms/step | 28.8 samples/s +Step 6840/44000 | Loss: 3.2885 | LR: 4.76e-04 | 694.7ms/step | 28.8 samples/s +Step 6850/44000 | Loss: 3.2565 | LR: 4.76e-04 | 693.7ms/step | 28.8 samples/s +Step 6860/44000 | Loss: 3.2336 | LR: 4.76e-04 | 695.5ms/step | 28.8 samples/s +Step 6870/44000 | Loss: 3.2525 | LR: 4.76e-04 | 695.9ms/step | 28.7 samples/s +Step 6880/44000 | Loss: 3.2440 | LR: 4.76e-04 | 694.8ms/step | 28.8 samples/s +Step 6890/44000 | Loss: 3.2688 | LR: 4.76e-04 | 695.1ms/step | 28.8 samples/s +Step 6900/44000 | Loss: 3.2527 | LR: 4.75e-04 | 694.4ms/step | 28.8 samples/s +Step 6910/44000 | Loss: 3.2330 | LR: 4.75e-04 | 694.7ms/step | 28.8 samples/s +Step 6920/44000 | Loss: 3.2388 | LR: 4.75e-04 | 694.4ms/step | 28.8 samples/s +Step 6930/44000 | Loss: 3.2121 | LR: 4.75e-04 | 694.6ms/step | 28.8 samples/s +Step 6940/44000 | Loss: 3.2360 | LR: 4.75e-04 | 693.4ms/step | 28.8 samples/s +Step 6950/44000 | Loss: 3.2674 | LR: 4.75e-04 | 693.4ms/step | 28.8 samples/s +Step 6960/44000 | Loss: 3.2180 | LR: 4.75e-04 | 694.7ms/step | 28.8 samples/s +Step 6970/44000 | Loss: 3.2198 | LR: 4.75e-04 | 694.6ms/step | 28.8 samples/s +Step 6980/44000 | Loss: 3.2268 | LR: 4.75e-04 | 695.4ms/step | 28.8 samples/s +Step 6990/44000 | Loss: 3.2178 | LR: 4.75e-04 | 693.9ms/step | 28.8 samples/s +Step 7000/44000 | Loss: 3.2454 | LR: 4.75e-04 | 695.4ms/step | 28.8 samples/s +Validation at step 7000: Loss = 3.2492, Perplexity = 25.77 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-7000 +Step 7010/44000 | Loss: 3.2149 | LR: 4.75e-04 | 1617.8ms/step | 12.4 samples/s +Step 7020/44000 | Loss: 3.2559 | LR: 4.75e-04 | 694.2ms/step | 28.8 samples/s +Step 7030/44000 | Loss: 3.2157 | LR: 4.74e-04 | 694.9ms/step | 28.8 samples/s +Step 7040/44000 | Loss: 3.2598 | LR: 4.74e-04 | 695.5ms/step | 28.8 samples/s +Step 7050/44000 | Loss: 3.2314 | LR: 4.74e-04 | 693.6ms/step | 28.8 samples/s +Step 7060/44000 | Loss: 3.2597 | LR: 4.74e-04 | 694.8ms/step | 28.8 samples/s +Step 7070/44000 | Loss: 3.2447 | LR: 4.74e-04 | 693.7ms/step | 28.8 samples/s +Step 7080/44000 | Loss: 3.2396 | LR: 4.74e-04 | 694.2ms/step | 28.8 samples/s +Step 7090/44000 | Loss: 3.1830 | LR: 4.74e-04 | 693.5ms/step | 28.8 samples/s +Step 7100/44000 | Loss: 3.2503 | LR: 4.74e-04 | 693.1ms/step | 28.9 samples/s +Step 7110/44000 | Loss: 3.2245 | LR: 4.74e-04 | 693.5ms/step | 28.8 samples/s +Step 7120/44000 | Loss: 3.2369 | LR: 4.74e-04 | 694.0ms/step | 28.8 samples/s +Step 7130/44000 | Loss: 3.2499 | LR: 4.74e-04 | 693.1ms/step | 28.9 samples/s +Step 7140/44000 | Loss: 3.2428 | LR: 4.74e-04 | 692.9ms/step | 28.9 samples/s +Step 7150/44000 | Loss: 3.2660 | LR: 4.73e-04 | 692.3ms/step | 28.9 samples/s +Step 7160/44000 | Loss: 3.2336 | LR: 4.73e-04 | 692.2ms/step | 28.9 samples/s +Step 7170/44000 | Loss: 3.2184 | LR: 4.73e-04 | 692.4ms/step | 28.9 samples/s +Step 7180/44000 | Loss: 3.2572 | LR: 4.73e-04 | 693.6ms/step | 28.8 samples/s +Step 7190/44000 | Loss: 3.2267 | LR: 4.73e-04 | 694.9ms/step | 28.8 samples/s +Step 7200/44000 | Loss: 3.2260 | LR: 4.73e-04 | 694.2ms/step | 28.8 samples/s +Step 7210/44000 | Loss: 3.2040 | LR: 4.73e-04 | 694.3ms/step | 28.8 samples/s +Step 7220/44000 | Loss: 3.1987 | LR: 4.73e-04 | 694.6ms/step | 28.8 samples/s +Step 7230/44000 | Loss: 3.2250 | LR: 4.73e-04 | 694.4ms/step | 28.8 samples/s +Step 7240/44000 | Loss: 3.1798 | LR: 4.73e-04 | 693.8ms/step | 28.8 samples/s +Step 7250/44000 | Loss: 3.2232 | LR: 4.73e-04 | 693.1ms/step | 28.9 samples/s +Validation at step 7250: Loss = 3.2398, Perplexity = 25.53 +Step 7260/44000 | Loss: 3.2635 | LR: 4.73e-04 | 1271.4ms/step | 15.7 samples/s +Step 7270/44000 | Loss: 3.2453 | LR: 4.72e-04 | 695.0ms/step | 28.8 samples/s +Step 7280/44000 | Loss: 3.2055 | LR: 4.72e-04 | 694.9ms/step | 28.8 samples/s +Step 7290/44000 | Loss: 3.2649 | LR: 4.72e-04 | 694.2ms/step | 28.8 samples/s +Step 7300/44000 | Loss: 3.1933 | LR: 4.72e-04 | 694.1ms/step | 28.8 samples/s +Step 7310/44000 | Loss: 3.2294 | LR: 4.72e-04 | 694.4ms/step | 28.8 samples/s +Step 7320/44000 | Loss: 3.2010 | LR: 4.72e-04 | 694.3ms/step | 28.8 samples/s +Step 7330/44000 | Loss: 3.2134 | LR: 4.72e-04 | 695.7ms/step | 28.7 samples/s +Step 7340/44000 | Loss: 3.2519 | LR: 4.72e-04 | 695.5ms/step | 28.8 samples/s +Step 7350/44000 | Loss: 3.1809 | LR: 4.72e-04 | 695.5ms/step | 28.8 samples/s +Step 7360/44000 | Loss: 3.2030 | LR: 4.72e-04 | 694.4ms/step | 28.8 samples/s +Step 7370/44000 | Loss: 3.2047 | LR: 4.72e-04 | 691.4ms/step | 28.9 samples/s +Step 7380/44000 | Loss: 3.2061 | LR: 4.72e-04 | 694.9ms/step | 28.8 samples/s +Step 7390/44000 | Loss: 3.2215 | LR: 4.71e-04 | 693.3ms/step | 28.8 samples/s +Step 7400/44000 | Loss: 3.2153 | LR: 4.71e-04 | 695.0ms/step | 28.8 samples/s +Step 7410/44000 | Loss: 3.2311 | LR: 4.71e-04 | 693.2ms/step | 28.8 samples/s +Step 7420/44000 | Loss: 3.1921 | LR: 4.71e-04 | 695.0ms/step | 28.8 samples/s +Step 7430/44000 | Loss: 3.2220 | LR: 4.71e-04 | 694.0ms/step | 28.8 samples/s +Step 7440/44000 | Loss: 3.2531 | LR: 4.71e-04 | 692.6ms/step | 28.9 samples/s +Step 7450/44000 | Loss: 3.2242 | LR: 4.71e-04 | 694.3ms/step | 28.8 samples/s +Step 7460/44000 | Loss: 3.2185 | LR: 4.71e-04 | 694.5ms/step | 28.8 samples/s +Step 7470/44000 | Loss: 3.2194 | LR: 4.71e-04 | 693.9ms/step | 28.8 samples/s +Step 7480/44000 | Loss: 3.1760 | LR: 4.71e-04 | 693.5ms/step | 28.8 samples/s +Step 7490/44000 | Loss: 3.1956 | LR: 4.71e-04 | 692.7ms/step | 28.9 samples/s +Step 7500/44000 | Loss: 3.2500 | LR: 4.71e-04 | 694.6ms/step | 28.8 samples/s +Validation at step 7500: Loss = 3.2331, Perplexity = 25.36 +Step 7510/44000 | Loss: 3.2379 | LR: 4.70e-04 | 1265.7ms/step | 15.8 samples/s +Step 7520/44000 | Loss: 3.2177 | LR: 4.70e-04 | 693.6ms/step | 28.8 samples/s +Step 7530/44000 | Loss: 3.2199 | LR: 4.70e-04 | 694.0ms/step | 28.8 samples/s +Step 7540/44000 | Loss: 3.1991 | LR: 4.70e-04 | 693.6ms/step | 28.8 samples/s +Step 7550/44000 | Loss: 3.1816 | LR: 4.70e-04 | 693.1ms/step | 28.9 samples/s +Step 7560/44000 | Loss: 3.2182 | LR: 4.70e-04 | 692.9ms/step | 28.9 samples/s +Step 7570/44000 | Loss: 3.2181 | LR: 4.70e-04 | 692.5ms/step | 28.9 samples/s +Step 7580/44000 | Loss: 3.2644 | LR: 4.70e-04 | 694.8ms/step | 28.8 samples/s +Step 7590/44000 | Loss: 3.1408 | LR: 4.70e-04 | 693.0ms/step | 28.9 samples/s +Step 7600/44000 | Loss: 3.2077 | LR: 4.70e-04 | 693.2ms/step | 28.9 samples/s +Step 7610/44000 | Loss: 3.2164 | LR: 4.70e-04 | 693.3ms/step | 28.8 samples/s +Step 7620/44000 | Loss: 3.2201 | LR: 4.70e-04 | 694.1ms/step | 28.8 samples/s +Step 7630/44000 | Loss: 3.2213 | LR: 4.69e-04 | 694.2ms/step | 28.8 samples/s +Step 7640/44000 | Loss: 3.2077 | LR: 4.69e-04 | 693.7ms/step | 28.8 samples/s +Step 7650/44000 | Loss: 3.2618 | LR: 4.69e-04 | 694.3ms/step | 28.8 samples/s +Step 7660/44000 | Loss: 3.2065 | LR: 4.69e-04 | 695.6ms/step | 28.8 samples/s +Step 7670/44000 | Loss: 3.2141 | LR: 4.69e-04 | 693.2ms/step | 28.9 samples/s +Step 7680/44000 | Loss: 3.2046 | LR: 4.69e-04 | 696.7ms/step | 28.7 samples/s +Step 7690/44000 | Loss: 3.2172 | LR: 4.69e-04 | 695.3ms/step | 28.8 samples/s +Step 7700/44000 | Loss: 3.1903 | LR: 4.69e-04 | 695.0ms/step | 28.8 samples/s +Step 7710/44000 | Loss: 3.2269 | LR: 4.69e-04 | 694.6ms/step | 28.8 samples/s +Step 7720/44000 | Loss: 3.2170 | LR: 4.69e-04 | 694.2ms/step | 28.8 samples/s +Step 7730/44000 | Loss: 3.2205 | LR: 4.69e-04 | 695.4ms/step | 28.8 samples/s +Step 7740/44000 | Loss: 3.2258 | LR: 4.68e-04 | 696.8ms/step | 28.7 samples/s +Step 7750/44000 | Loss: 3.2481 | LR: 4.68e-04 | 694.3ms/step | 28.8 samples/s +Validation at step 7750: Loss = 3.2226, Perplexity = 25.09 +Step 7760/44000 | Loss: 3.2330 | LR: 4.68e-04 | 1267.4ms/step | 15.8 samples/s +Step 7770/44000 | Loss: 3.2109 | LR: 4.68e-04 | 695.8ms/step | 28.7 samples/s +Step 7780/44000 | Loss: 3.2739 | LR: 4.68e-04 | 695.7ms/step | 28.7 samples/s +Step 7790/44000 | Loss: 3.2390 | LR: 4.68e-04 | 695.2ms/step | 28.8 samples/s +Step 7800/44000 | Loss: 3.2272 | LR: 4.68e-04 | 693.8ms/step | 28.8 samples/s +Step 7810/44000 | Loss: 3.2193 | LR: 4.68e-04 | 693.9ms/step | 28.8 samples/s +Step 7820/44000 | Loss: 3.1758 | LR: 4.68e-04 | 697.1ms/step | 28.7 samples/s +Step 7830/44000 | Loss: 3.2157 | LR: 4.68e-04 | 693.9ms/step | 28.8 samples/s +Step 7840/44000 | Loss: 3.2094 | LR: 4.68e-04 | 694.4ms/step | 28.8 samples/s +Step 7850/44000 | Loss: 3.1639 | LR: 4.67e-04 | 695.9ms/step | 28.7 samples/s +Step 7860/44000 | Loss: 3.2264 | LR: 4.67e-04 | 694.5ms/step | 28.8 samples/s +Step 7870/44000 | Loss: 3.1491 | LR: 4.67e-04 | 696.6ms/step | 28.7 samples/s +Step 7880/44000 | Loss: 3.2056 | LR: 4.67e-04 | 695.4ms/step | 28.8 samples/s +Step 7890/44000 | Loss: 3.1986 | LR: 4.67e-04 | 695.5ms/step | 28.8 samples/s +Step 7900/44000 | Loss: 3.2386 | LR: 4.67e-04 | 694.6ms/step | 28.8 samples/s +Step 7910/44000 | Loss: 3.1615 | LR: 4.67e-04 | 695.5ms/step | 28.8 samples/s +Step 7920/44000 | Loss: 3.2107 | LR: 4.67e-04 | 695.4ms/step | 28.8 samples/s +Step 7930/44000 | Loss: 3.1544 | LR: 4.67e-04 | 694.8ms/step | 28.8 samples/s +Step 7940/44000 | Loss: 3.1842 | LR: 4.67e-04 | 696.2ms/step | 28.7 samples/s +Step 7950/44000 | Loss: 3.2192 | LR: 4.67e-04 | 694.2ms/step | 28.8 samples/s +Step 7960/44000 | Loss: 3.2392 | LR: 4.66e-04 | 694.6ms/step | 28.8 samples/s +Step 7970/44000 | Loss: 3.1762 | LR: 4.66e-04 | 694.5ms/step | 28.8 samples/s +Step 7980/44000 | Loss: 3.1642 | LR: 4.66e-04 | 693.4ms/step | 28.8 samples/s +Step 7990/44000 | Loss: 3.1795 | LR: 4.66e-04 | 693.0ms/step | 28.9 samples/s +Step 8000/44000 | Loss: 3.1965 | LR: 4.66e-04 | 694.4ms/step | 28.8 samples/s +Validation at step 8000: Loss = 3.2150, Perplexity = 24.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-8000 +Step 8010/44000 | Loss: 3.1707 | LR: 4.66e-04 | 1581.6ms/step | 12.6 samples/s +Step 8020/44000 | Loss: 3.1871 | LR: 4.66e-04 | 693.5ms/step | 28.8 samples/s +Step 8030/44000 | Loss: 3.2050 | LR: 4.66e-04 | 694.6ms/step | 28.8 samples/s +Step 8040/44000 | Loss: 3.1609 | LR: 4.66e-04 | 695.0ms/step | 28.8 samples/s +Step 8050/44000 | Loss: 3.2062 | LR: 4.66e-04 | 693.7ms/step | 28.8 samples/s +Step 8060/44000 | Loss: 3.1950 | LR: 4.66e-04 | 693.4ms/step | 28.8 samples/s +Step 8070/44000 | Loss: 3.1411 | LR: 4.65e-04 | 694.2ms/step | 28.8 samples/s +Step 8080/44000 | Loss: 3.2420 | LR: 4.65e-04 | 695.3ms/step | 28.8 samples/s +Step 8090/44000 | Loss: 3.2169 | LR: 4.65e-04 | 694.6ms/step | 28.8 samples/s +Step 8100/44000 | Loss: 3.1842 | LR: 4.65e-04 | 694.5ms/step | 28.8 samples/s +Step 8110/44000 | Loss: 3.2059 | LR: 4.65e-04 | 695.3ms/step | 28.8 samples/s +Step 8120/44000 | Loss: 3.1842 | LR: 4.65e-04 | 694.7ms/step | 28.8 samples/s +Step 8130/44000 | Loss: 3.1921 | LR: 4.65e-04 | 693.7ms/step | 28.8 samples/s +Step 8140/44000 | Loss: 3.1875 | LR: 4.65e-04 | 692.4ms/step | 28.9 samples/s +Step 8150/44000 | Loss: 3.2248 | LR: 4.65e-04 | 692.3ms/step | 28.9 samples/s +Step 8160/44000 | Loss: 3.1885 | LR: 4.65e-04 | 692.5ms/step | 28.9 samples/s +Step 8170/44000 | Loss: 3.2402 | LR: 4.65e-04 | 691.7ms/step | 28.9 samples/s +Step 8180/44000 | Loss: 3.2073 | LR: 4.64e-04 | 692.2ms/step | 28.9 samples/s +Step 8190/44000 | Loss: 3.1790 | LR: 4.64e-04 | 691.9ms/step | 28.9 samples/s +Step 8200/44000 | Loss: 3.2268 | LR: 4.64e-04 | 691.3ms/step | 28.9 samples/s +Step 8210/44000 | Loss: 3.1974 | LR: 4.64e-04 | 690.6ms/step | 29.0 samples/s +Step 8220/44000 | Loss: 3.1966 | LR: 4.64e-04 | 692.1ms/step | 28.9 samples/s +Step 8230/44000 | Loss: 3.2204 | LR: 4.64e-04 | 692.9ms/step | 28.9 samples/s +Step 8240/44000 | Loss: 3.2280 | LR: 4.64e-04 | 695.2ms/step | 28.8 samples/s +Step 8250/44000 | Loss: 3.1454 | LR: 4.64e-04 | 693.6ms/step | 28.8 samples/s +Validation at step 8250: Loss = 3.2063, Perplexity = 24.69 +Step 8260/44000 | Loss: 3.1959 | LR: 4.64e-04 | 1266.4ms/step | 15.8 samples/s +Step 8270/44000 | Loss: 3.1912 | LR: 4.64e-04 | 694.0ms/step | 28.8 samples/s +Step 8280/44000 | Loss: 3.1988 | LR: 4.64e-04 | 694.0ms/step | 28.8 samples/s +Step 8290/44000 | Loss: 3.1871 | LR: 4.63e-04 | 693.7ms/step | 28.8 samples/s +Step 8300/44000 | Loss: 3.1547 | LR: 4.63e-04 | 693.5ms/step | 28.8 samples/s +Step 8310/44000 | Loss: 3.2126 | LR: 4.63e-04 | 693.4ms/step | 28.8 samples/s +Step 8320/44000 | Loss: 3.1590 | LR: 4.63e-04 | 693.0ms/step | 28.9 samples/s +Step 8330/44000 | Loss: 3.1706 | LR: 4.63e-04 | 693.2ms/step | 28.9 samples/s +Step 8340/44000 | Loss: 3.1804 | LR: 4.63e-04 | 693.6ms/step | 28.8 samples/s +Step 8350/44000 | Loss: 3.2089 | LR: 4.63e-04 | 694.3ms/step | 28.8 samples/s +Step 8360/44000 | Loss: 3.2062 | LR: 4.63e-04 | 694.3ms/step | 28.8 samples/s +Step 8370/44000 | Loss: 3.2119 | LR: 4.63e-04 | 694.9ms/step | 28.8 samples/s +Step 8380/44000 | Loss: 3.1939 | LR: 4.63e-04 | 693.9ms/step | 28.8 samples/s +Step 8390/44000 | Loss: 3.1731 | LR: 4.62e-04 | 694.2ms/step | 28.8 samples/s +Step 8400/44000 | Loss: 3.2081 | LR: 4.62e-04 | 694.0ms/step | 28.8 samples/s +Step 8410/44000 | Loss: 3.1875 | LR: 4.62e-04 | 694.2ms/step | 28.8 samples/s +Step 8420/44000 | Loss: 3.1892 | LR: 4.62e-04 | 694.0ms/step | 28.8 samples/s +Step 8430/44000 | Loss: 3.2300 | LR: 4.62e-04 | 693.3ms/step | 28.8 samples/s +Step 8440/44000 | Loss: 3.2164 | LR: 4.62e-04 | 694.3ms/step | 28.8 samples/s +Step 8450/44000 | Loss: 3.1894 | LR: 4.62e-04 | 693.8ms/step | 28.8 samples/s +Step 8460/44000 | Loss: 3.1767 | LR: 4.62e-04 | 693.1ms/step | 28.9 samples/s +Step 8470/44000 | Loss: 3.2156 | LR: 4.62e-04 | 693.6ms/step | 28.8 samples/s +Step 8480/44000 | Loss: 3.1863 | LR: 4.62e-04 | 694.0ms/step | 28.8 samples/s +Step 8490/44000 | Loss: 3.1823 | LR: 4.62e-04 | 696.1ms/step | 28.7 samples/s +Step 8500/44000 | Loss: 3.1876 | LR: 4.61e-04 | 695.0ms/step | 28.8 samples/s +Validation at step 8500: Loss = 3.1992, Perplexity = 24.51 +Step 8510/44000 | Loss: 3.2157 | LR: 4.61e-04 | 1277.5ms/step | 15.7 samples/s +Step 8520/44000 | Loss: 3.2010 | LR: 4.61e-04 | 696.6ms/step | 28.7 samples/s +Step 8530/44000 | Loss: 3.1606 | LR: 4.61e-04 | 697.1ms/step | 28.7 samples/s +Step 8540/44000 | Loss: 3.1537 | LR: 4.61e-04 | 695.9ms/step | 28.7 samples/s +Step 8550/44000 | Loss: 3.1856 | LR: 4.61e-04 | 696.7ms/step | 28.7 samples/s +Step 8560/44000 | Loss: 3.1683 | LR: 4.61e-04 | 695.9ms/step | 28.7 samples/s +Step 8570/44000 | Loss: 3.1798 | LR: 4.61e-04 | 695.2ms/step | 28.8 samples/s +Step 8580/44000 | Loss: 3.2011 | LR: 4.61e-04 | 694.6ms/step | 28.8 samples/s +Step 8590/44000 | Loss: 3.2095 | LR: 4.61e-04 | 694.0ms/step | 28.8 samples/s +Step 8600/44000 | Loss: 3.1938 | LR: 4.60e-04 | 694.2ms/step | 28.8 samples/s +Step 8610/44000 | Loss: 3.2307 | LR: 4.60e-04 | 694.6ms/step | 28.8 samples/s +Step 8620/44000 | Loss: 3.2017 | LR: 4.60e-04 | 693.3ms/step | 28.8 samples/s +Step 8630/44000 | Loss: 3.2080 | LR: 4.60e-04 | 694.4ms/step | 28.8 samples/s +Step 8640/44000 | Loss: 3.1616 | LR: 4.60e-04 | 694.1ms/step | 28.8 samples/s +Step 8650/44000 | Loss: 3.2518 | LR: 4.60e-04 | 694.7ms/step | 28.8 samples/s +Step 8660/44000 | Loss: 3.1880 | LR: 4.60e-04 | 694.2ms/step | 28.8 samples/s +Step 8670/44000 | Loss: 3.2181 | LR: 4.60e-04 | 695.6ms/step | 28.8 samples/s +Step 8680/44000 | Loss: 3.1458 | LR: 4.60e-04 | 695.3ms/step | 28.8 samples/s +Step 8690/44000 | Loss: 3.1625 | LR: 4.60e-04 | 693.2ms/step | 28.9 samples/s +Step 8700/44000 | Loss: 3.2053 | LR: 4.59e-04 | 695.1ms/step | 28.8 samples/s +Step 8710/44000 | Loss: 3.1926 | LR: 4.59e-04 | 694.8ms/step | 28.8 samples/s +Step 8720/44000 | Loss: 3.1683 | LR: 4.59e-04 | 693.0ms/step | 28.9 samples/s +Step 8730/44000 | Loss: 3.1878 | LR: 4.59e-04 | 694.0ms/step | 28.8 samples/s +Step 8740/44000 | Loss: 3.2058 | LR: 4.59e-04 | 695.9ms/step | 28.7 samples/s +Step 8750/44000 | Loss: 3.2264 | LR: 4.59e-04 | 697.6ms/step | 28.7 samples/s +Validation at step 8750: Loss = 3.1914, Perplexity = 24.32 +Step 8760/44000 | Loss: 3.1667 | LR: 4.59e-04 | 1274.8ms/step | 15.7 samples/s +Step 8770/44000 | Loss: 3.1310 | LR: 4.59e-04 | 695.9ms/step | 28.7 samples/s +Step 8780/44000 | Loss: 3.2045 | LR: 4.59e-04 | 696.3ms/step | 28.7 samples/s +Step 8790/44000 | Loss: 3.1941 | LR: 4.59e-04 | 696.5ms/step | 28.7 samples/s +Step 8800/44000 | Loss: 3.1718 | LR: 4.58e-04 | 696.0ms/step | 28.7 samples/s +Step 8810/44000 | Loss: 3.1902 | LR: 4.58e-04 | 696.0ms/step | 28.7 samples/s +Step 8820/44000 | Loss: 3.1785 | LR: 4.58e-04 | 695.5ms/step | 28.8 samples/s +Step 8830/44000 | Loss: 3.1769 | LR: 4.58e-04 | 708.4ms/step | 28.2 samples/s +Step 8840/44000 | Loss: 3.2001 | LR: 4.58e-04 | 698.2ms/step | 28.6 samples/s +Step 8850/44000 | Loss: 3.1543 | LR: 4.58e-04 | 694.7ms/step | 28.8 samples/s +Step 8860/44000 | Loss: 3.1876 | LR: 4.58e-04 | 695.8ms/step | 28.7 samples/s +Step 8870/44000 | Loss: 3.1645 | LR: 4.58e-04 | 694.3ms/step | 28.8 samples/s +Step 8880/44000 | Loss: 3.2561 | LR: 4.58e-04 | 698.1ms/step | 28.6 samples/s +Step 8890/44000 | Loss: 3.1237 | LR: 4.58e-04 | 699.5ms/step | 28.6 samples/s +Step 8900/44000 | Loss: 3.1976 | LR: 4.57e-04 | 700.3ms/step | 28.6 samples/s +Step 8910/44000 | Loss: 3.1700 | LR: 4.57e-04 | 699.1ms/step | 28.6 samples/s +Step 8920/44000 | Loss: 3.1839 | LR: 4.57e-04 | 698.0ms/step | 28.7 samples/s +Step 8930/44000 | Loss: 3.1960 | LR: 4.57e-04 | 696.1ms/step | 28.7 samples/s +Step 8940/44000 | Loss: 3.2031 | LR: 4.57e-04 | 697.7ms/step | 28.7 samples/s +Step 8950/44000 | Loss: 3.2015 | LR: 4.57e-04 | 696.7ms/step | 28.7 samples/s +Step 8960/44000 | Loss: 3.1719 | LR: 4.57e-04 | 696.4ms/step | 28.7 samples/s +Step 8970/44000 | Loss: 3.1713 | LR: 4.57e-04 | 696.1ms/step | 28.7 samples/s +Step 8980/44000 | Loss: 3.2284 | LR: 4.57e-04 | 695.7ms/step | 28.7 samples/s +Step 8990/44000 | Loss: 3.1871 | LR: 4.57e-04 | 695.5ms/step | 28.8 samples/s +Step 9000/44000 | Loss: 3.1144 | LR: 4.56e-04 | 695.7ms/step | 28.7 samples/s +Validation at step 9000: Loss = 3.1857, Perplexity = 24.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-9000 +Step 9010/44000 | Loss: 3.1106 | LR: 4.56e-04 | 1574.7ms/step | 12.7 samples/s +Step 9020/44000 | Loss: 3.1898 | LR: 4.56e-04 | 698.1ms/step | 28.6 samples/s +Step 9030/44000 | Loss: 3.1637 | LR: 4.56e-04 | 698.0ms/step | 28.7 samples/s +Step 9040/44000 | Loss: 3.2227 | LR: 4.56e-04 | 699.2ms/step | 28.6 samples/s +Step 9050/44000 | Loss: 3.1298 | LR: 4.56e-04 | 698.1ms/step | 28.7 samples/s +Step 9060/44000 | Loss: 3.1341 | LR: 4.56e-04 | 699.0ms/step | 28.6 samples/s +Step 9070/44000 | Loss: 3.1895 | LR: 4.56e-04 | 697.9ms/step | 28.7 samples/s +Step 9080/44000 | Loss: 3.1638 | LR: 4.56e-04 | 697.8ms/step | 28.7 samples/s +Step 9090/44000 | Loss: 3.1949 | LR: 4.56e-04 | 697.5ms/step | 28.7 samples/s +Step 9100/44000 | Loss: 3.1416 | LR: 4.55e-04 | 695.6ms/step | 28.8 samples/s +Step 9110/44000 | Loss: 3.1870 | LR: 4.55e-04 | 696.0ms/step | 28.7 samples/s +Step 9120/44000 | Loss: 3.1640 | LR: 4.55e-04 | 695.8ms/step | 28.7 samples/s +Step 9130/44000 | Loss: 3.1710 | LR: 4.55e-04 | 696.4ms/step | 28.7 samples/s +Step 9140/44000 | Loss: 3.1900 | LR: 4.55e-04 | 695.8ms/step | 28.7 samples/s +Step 9150/44000 | Loss: 3.1578 | LR: 4.55e-04 | 696.2ms/step | 28.7 samples/s +Step 9160/44000 | Loss: 3.1943 | LR: 4.55e-04 | 696.5ms/step | 28.7 samples/s +Step 9170/44000 | Loss: 3.2140 | LR: 4.55e-04 | 696.5ms/step | 28.7 samples/s +Step 9180/44000 | Loss: 3.1541 | LR: 4.55e-04 | 695.7ms/step | 28.7 samples/s +Step 9190/44000 | Loss: 3.1882 | LR: 4.54e-04 | 696.2ms/step | 28.7 samples/s +Step 9200/44000 | Loss: 3.1754 | LR: 4.54e-04 | 695.2ms/step | 28.8 samples/s +Step 9210/44000 | Loss: 3.1997 | LR: 4.54e-04 | 697.7ms/step | 28.7 samples/s +Step 9220/44000 | Loss: 3.1789 | LR: 4.54e-04 | 697.5ms/step | 28.7 samples/s +Step 9230/44000 | Loss: 3.1360 | LR: 4.54e-04 | 698.5ms/step | 28.6 samples/s +Step 9240/44000 | Loss: 3.1454 | LR: 4.54e-04 | 696.2ms/step | 28.7 samples/s +Step 9250/44000 | Loss: 3.1980 | LR: 4.54e-04 | 696.4ms/step | 28.7 samples/s +Validation at step 9250: Loss = 3.1801, Perplexity = 24.05 +Step 9260/44000 | Loss: 3.1701 | LR: 4.54e-04 | 1272.0ms/step | 15.7 samples/s +Step 9270/44000 | Loss: 3.1251 | LR: 4.54e-04 | 695.9ms/step | 28.7 samples/s +Step 9280/44000 | Loss: 3.1760 | LR: 4.54e-04 | 696.4ms/step | 28.7 samples/s +Step 9290/44000 | Loss: 3.1874 | LR: 4.53e-04 | 698.1ms/step | 28.7 samples/s +Step 9300/44000 | Loss: 3.1786 | LR: 4.53e-04 | 697.1ms/step | 28.7 samples/s +Step 9310/44000 | Loss: 3.1689 | LR: 4.53e-04 | 697.0ms/step | 28.7 samples/s +Step 9320/44000 | Loss: 3.1704 | LR: 4.53e-04 | 697.1ms/step | 28.7 samples/s +Step 9330/44000 | Loss: 3.1570 | LR: 4.53e-04 | 696.3ms/step | 28.7 samples/s +Step 9340/44000 | Loss: 3.1198 | LR: 4.53e-04 | 697.2ms/step | 28.7 samples/s +Step 9350/44000 | Loss: 3.1775 | LR: 4.53e-04 | 697.0ms/step | 28.7 samples/s +Step 9360/44000 | Loss: 3.1583 | LR: 4.53e-04 | 696.2ms/step | 28.7 samples/s +Step 9370/44000 | Loss: 3.1948 | LR: 4.53e-04 | 695.6ms/step | 28.8 samples/s +Step 9380/44000 | Loss: 3.1797 | LR: 4.52e-04 | 696.0ms/step | 28.7 samples/s +Step 9390/44000 | Loss: 3.1928 | LR: 4.52e-04 | 696.0ms/step | 28.7 samples/s +Step 9400/44000 | Loss: 3.1149 | LR: 4.52e-04 | 696.7ms/step | 28.7 samples/s +Step 9410/44000 | Loss: 3.1811 | LR: 4.52e-04 | 697.4ms/step | 28.7 samples/s +Step 9420/44000 | Loss: 3.1566 | LR: 4.52e-04 | 695.8ms/step | 28.7 samples/s +Step 9430/44000 | Loss: 3.1930 | LR: 4.52e-04 | 694.6ms/step | 28.8 samples/s +Step 9440/44000 | Loss: 3.1987 | LR: 4.52e-04 | 695.5ms/step | 28.8 samples/s +Step 9450/44000 | Loss: 3.1360 | LR: 4.52e-04 | 697.3ms/step | 28.7 samples/s +Step 9460/44000 | Loss: 3.1566 | LR: 4.52e-04 | 694.8ms/step | 28.8 samples/s +Step 9470/44000 | Loss: 3.1492 | LR: 4.52e-04 | 696.4ms/step | 28.7 samples/s +Step 9480/44000 | Loss: 3.1528 | LR: 4.51e-04 | 694.5ms/step | 28.8 samples/s +Step 9490/44000 | Loss: 3.1794 | LR: 4.51e-04 | 695.6ms/step | 28.8 samples/s +Step 9500/44000 | Loss: 3.1355 | LR: 4.51e-04 | 695.6ms/step | 28.8 samples/s +Validation at step 9500: Loss = 3.1743, Perplexity = 23.91 +Step 9510/44000 | Loss: 3.1416 | LR: 4.51e-04 | 1268.3ms/step | 15.8 samples/s +Step 9520/44000 | Loss: 3.1488 | LR: 4.51e-04 | 699.5ms/step | 28.6 samples/s +Step 9530/44000 | Loss: 3.1642 | LR: 4.51e-04 | 697.6ms/step | 28.7 samples/s +Step 9540/44000 | Loss: 3.1342 | LR: 4.51e-04 | 697.0ms/step | 28.7 samples/s +Step 9550/44000 | Loss: 3.1893 | LR: 4.51e-04 | 695.3ms/step | 28.8 samples/s +Step 9560/44000 | Loss: 3.1815 | LR: 4.51e-04 | 695.9ms/step | 28.7 samples/s +Step 9570/44000 | Loss: 3.1704 | LR: 4.50e-04 | 696.4ms/step | 28.7 samples/s +Step 9580/44000 | Loss: 3.1969 | LR: 4.50e-04 | 696.3ms/step | 28.7 samples/s +Step 9590/44000 | Loss: 3.1402 | LR: 4.50e-04 | 696.4ms/step | 28.7 samples/s +Step 9600/44000 | Loss: 3.1698 | LR: 4.50e-04 | 696.0ms/step | 28.7 samples/s +Step 9610/44000 | Loss: 3.1522 | LR: 4.50e-04 | 696.1ms/step | 28.7 samples/s +Step 9620/44000 | Loss: 3.1431 | LR: 4.50e-04 | 695.8ms/step | 28.7 samples/s +Step 9630/44000 | Loss: 3.1498 | LR: 4.50e-04 | 696.1ms/step | 28.7 samples/s +Step 9640/44000 | Loss: 3.1446 | LR: 4.50e-04 | 695.8ms/step | 28.7 samples/s +Step 9650/44000 | Loss: 3.1273 | LR: 4.50e-04 | 698.3ms/step | 28.6 samples/s +Step 9660/44000 | Loss: 3.1957 | LR: 4.49e-04 | 697.1ms/step | 28.7 samples/s +Step 9670/44000 | Loss: 3.1512 | LR: 4.49e-04 | 696.7ms/step | 28.7 samples/s +Step 9680/44000 | Loss: 3.1077 | LR: 4.49e-04 | 697.9ms/step | 28.7 samples/s +Step 9690/44000 | Loss: 3.1805 | LR: 4.49e-04 | 698.2ms/step | 28.6 samples/s +Step 9700/44000 | Loss: 3.1373 | LR: 4.49e-04 | 697.8ms/step | 28.7 samples/s +Step 9710/44000 | Loss: 3.1536 | LR: 4.49e-04 | 695.5ms/step | 28.8 samples/s +Step 9720/44000 | Loss: 3.1545 | LR: 4.49e-04 | 695.3ms/step | 28.8 samples/s +Step 9730/44000 | Loss: 3.1584 | LR: 4.49e-04 | 695.2ms/step | 28.8 samples/s +Step 9740/44000 | Loss: 3.1815 | LR: 4.49e-04 | 695.2ms/step | 28.8 samples/s +Step 9750/44000 | Loss: 3.1318 | LR: 4.48e-04 | 694.5ms/step | 28.8 samples/s +Validation at step 9750: Loss = 3.1676, Perplexity = 23.75 +Step 9760/44000 | Loss: 3.1671 | LR: 4.48e-04 | 1269.4ms/step | 15.8 samples/s +Step 9770/44000 | Loss: 3.1462 | LR: 4.48e-04 | 696.5ms/step | 28.7 samples/s +Step 9780/44000 | Loss: 3.1302 | LR: 4.48e-04 | 696.4ms/step | 28.7 samples/s +Step 9790/44000 | Loss: 3.1672 | LR: 4.48e-04 | 696.2ms/step | 28.7 samples/s +Step 9800/44000 | Loss: 3.1398 | LR: 4.48e-04 | 696.9ms/step | 28.7 samples/s +Step 9810/44000 | Loss: 3.1586 | LR: 4.48e-04 | 697.4ms/step | 28.7 samples/s +Step 9820/44000 | Loss: 3.1414 | LR: 4.48e-04 | 696.2ms/step | 28.7 samples/s +Step 9830/44000 | Loss: 3.1256 | LR: 4.48e-04 | 696.4ms/step | 28.7 samples/s +Step 9840/44000 | Loss: 3.1592 | LR: 4.47e-04 | 696.2ms/step | 28.7 samples/s +Step 9850/44000 | Loss: 3.1781 | LR: 4.47e-04 | 697.6ms/step | 28.7 samples/s +Step 9860/44000 | Loss: 3.1589 | LR: 4.47e-04 | 696.8ms/step | 28.7 samples/s +Step 9870/44000 | Loss: 3.1497 | LR: 4.47e-04 | 697.4ms/step | 28.7 samples/s +Step 9880/44000 | Loss: 3.1349 | LR: 4.47e-04 | 698.6ms/step | 28.6 samples/s +Step 9890/44000 | Loss: 3.1569 | LR: 4.47e-04 | 697.9ms/step | 28.7 samples/s +Step 9900/44000 | Loss: 3.1651 | LR: 4.47e-04 | 694.3ms/step | 28.8 samples/s +Step 9910/44000 | Loss: 3.1427 | LR: 4.47e-04 | 694.9ms/step | 28.8 samples/s +Step 9920/44000 | Loss: 3.1892 | LR: 4.47e-04 | 694.9ms/step | 28.8 samples/s +Step 9930/44000 | Loss: 3.1117 | LR: 4.46e-04 | 695.2ms/step | 28.8 samples/s +Step 9940/44000 | Loss: 3.1357 | LR: 4.46e-04 | 694.4ms/step | 28.8 samples/s +Step 9950/44000 | Loss: 3.1401 | LR: 4.46e-04 | 695.4ms/step | 28.8 samples/s +Step 9960/44000 | Loss: 3.1319 | LR: 4.46e-04 | 693.8ms/step | 28.8 samples/s +Step 9970/44000 | Loss: 3.1811 | LR: 4.46e-04 | 693.8ms/step | 28.8 samples/s +Step 9980/44000 | Loss: 3.1545 | LR: 4.46e-04 | 693.5ms/step | 28.8 samples/s +Step 9990/44000 | Loss: 3.1728 | LR: 4.46e-04 | 695.9ms/step | 28.7 samples/s +Step 10000/44000 | Loss: 3.1078 | LR: 4.46e-04 | 695.5ms/step | 28.8 samples/s +Validation at step 10000: Loss = 3.1607, Perplexity = 23.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-10000 +Step 10010/44000 | Loss: 3.1570 | LR: 4.46e-04 | 1610.0ms/step | 12.4 samples/s +Step 10020/44000 | Loss: 3.1584 | LR: 4.45e-04 | 694.3ms/step | 28.8 samples/s +Step 10030/44000 | Loss: 3.1425 | LR: 4.45e-04 | 695.7ms/step | 28.7 samples/s +Step 10040/44000 | Loss: 3.1758 | LR: 4.45e-04 | 695.0ms/step | 28.8 samples/s +Step 10050/44000 | Loss: 3.1319 | LR: 4.45e-04 | 695.1ms/step | 28.8 samples/s +Step 10060/44000 | Loss: 3.1132 | LR: 4.45e-04 | 695.0ms/step | 28.8 samples/s +Step 10070/44000 | Loss: 3.1493 | LR: 4.45e-04 | 695.5ms/step | 28.8 samples/s +Step 10080/44000 | Loss: 3.1777 | LR: 4.45e-04 | 695.5ms/step | 28.8 samples/s +Step 10090/44000 | Loss: 3.1433 | LR: 4.45e-04 | 694.8ms/step | 28.8 samples/s +Step 10100/44000 | Loss: 3.1497 | LR: 4.45e-04 | 695.1ms/step | 28.8 samples/s +Step 10110/44000 | Loss: 3.1574 | LR: 4.44e-04 | 694.6ms/step | 28.8 samples/s +Step 10120/44000 | Loss: 3.1503 | LR: 4.44e-04 | 692.7ms/step | 28.9 samples/s +Step 10130/44000 | Loss: 3.1373 | LR: 4.44e-04 | 693.4ms/step | 28.8 samples/s +Step 10140/44000 | Loss: 3.1275 | LR: 4.44e-04 | 694.2ms/step | 28.8 samples/s +Step 10150/44000 | Loss: 3.1592 | LR: 4.44e-04 | 694.0ms/step | 28.8 samples/s +Step 10160/44000 | Loss: 3.1098 | LR: 4.44e-04 | 694.9ms/step | 28.8 samples/s +Step 10170/44000 | Loss: 3.1400 | LR: 4.44e-04 | 692.8ms/step | 28.9 samples/s +Step 10180/44000 | Loss: 3.1409 | LR: 4.44e-04 | 693.9ms/step | 28.8 samples/s +Step 10190/44000 | Loss: 3.1207 | LR: 4.43e-04 | 693.8ms/step | 28.8 samples/s +Step 10200/44000 | Loss: 3.1400 | LR: 4.43e-04 | 693.4ms/step | 28.8 samples/s +Step 10210/44000 | Loss: 3.0931 | LR: 4.43e-04 | 693.2ms/step | 28.9 samples/s +Step 10220/44000 | Loss: 3.2055 | LR: 4.43e-04 | 693.3ms/step | 28.8 samples/s +Step 10230/44000 | Loss: 3.1424 | LR: 4.43e-04 | 693.5ms/step | 28.8 samples/s +Step 10240/44000 | Loss: 3.1849 | LR: 4.43e-04 | 693.0ms/step | 28.9 samples/s +Step 10250/44000 | Loss: 3.1201 | LR: 4.43e-04 | 693.5ms/step | 28.8 samples/s +Validation at step 10250: Loss = 3.1559, Perplexity = 23.48 +Step 10260/44000 | Loss: 3.1412 | LR: 4.43e-04 | 1269.7ms/step | 15.8 samples/s +Step 10270/44000 | Loss: 3.1387 | LR: 4.43e-04 | 695.4ms/step | 28.8 samples/s +Step 10280/44000 | Loss: 3.1409 | LR: 4.42e-04 | 695.7ms/step | 28.7 samples/s +Step 10290/44000 | Loss: 3.1284 | LR: 4.42e-04 | 697.0ms/step | 28.7 samples/s +Step 10300/44000 | Loss: 3.0881 | LR: 4.42e-04 | 696.7ms/step | 28.7 samples/s +Step 10310/44000 | Loss: 3.1411 | LR: 4.42e-04 | 696.8ms/step | 28.7 samples/s +Step 10320/44000 | Loss: 3.1164 | LR: 4.42e-04 | 696.3ms/step | 28.7 samples/s +Step 10330/44000 | Loss: 3.1136 | LR: 4.42e-04 | 696.4ms/step | 28.7 samples/s +Step 10340/44000 | Loss: 3.1833 | LR: 4.42e-04 | 697.0ms/step | 28.7 samples/s +Step 10350/44000 | Loss: 3.1361 | LR: 4.42e-04 | 694.1ms/step | 28.8 samples/s +Step 10360/44000 | Loss: 3.1737 | LR: 4.42e-04 | 694.9ms/step | 28.8 samples/s +Step 10370/44000 | Loss: 3.1318 | LR: 4.41e-04 | 695.2ms/step | 28.8 samples/s +Step 10380/44000 | Loss: 3.1472 | LR: 4.41e-04 | 693.1ms/step | 28.9 samples/s +Step 10390/44000 | Loss: 3.1246 | LR: 4.41e-04 | 695.1ms/step | 28.8 samples/s +Step 10400/44000 | Loss: 3.1521 | LR: 4.41e-04 | 696.6ms/step | 28.7 samples/s +Step 10410/44000 | Loss: 3.1517 | LR: 4.41e-04 | 693.8ms/step | 28.8 samples/s +Step 10420/44000 | Loss: 3.1471 | LR: 4.41e-04 | 692.4ms/step | 28.9 samples/s +Step 10430/44000 | Loss: 3.1379 | LR: 4.41e-04 | 692.4ms/step | 28.9 samples/s +Step 10440/44000 | Loss: 3.1568 | LR: 4.41e-04 | 693.9ms/step | 28.8 samples/s +Step 10450/44000 | Loss: 3.1077 | LR: 4.40e-04 | 694.7ms/step | 28.8 samples/s +Step 10460/44000 | Loss: 3.1258 | LR: 4.40e-04 | 696.0ms/step | 28.7 samples/s +Step 10470/44000 | Loss: 3.1148 | LR: 4.40e-04 | 696.1ms/step | 28.7 samples/s +Step 10480/44000 | Loss: 3.1178 | LR: 4.40e-04 | 695.1ms/step | 28.8 samples/s +Step 10490/44000 | Loss: 3.1655 | LR: 4.40e-04 | 698.4ms/step | 28.6 samples/s +Step 10500/44000 | Loss: 3.1021 | LR: 4.40e-04 | 695.8ms/step | 28.7 samples/s +Validation at step 10500: Loss = 3.1503, Perplexity = 23.34 +Step 10510/44000 | Loss: 3.1365 | LR: 4.40e-04 | 1269.6ms/step | 15.8 samples/s +Step 10520/44000 | Loss: 3.0953 | LR: 4.40e-04 | 694.7ms/step | 28.8 samples/s +Step 10530/44000 | Loss: 3.1391 | LR: 4.40e-04 | 694.6ms/step | 28.8 samples/s +Step 10540/44000 | Loss: 3.1714 | LR: 4.39e-04 | 693.5ms/step | 28.8 samples/s +Step 10550/44000 | Loss: 3.1742 | LR: 4.39e-04 | 695.4ms/step | 28.8 samples/s +Step 10560/44000 | Loss: 3.1091 | LR: 4.39e-04 | 695.7ms/step | 28.7 samples/s +Step 10570/44000 | Loss: 3.1308 | LR: 4.39e-04 | 695.6ms/step | 28.8 samples/s +Step 10580/44000 | Loss: 3.1477 | LR: 4.39e-04 | 696.2ms/step | 28.7 samples/s +Step 10590/44000 | Loss: 3.1724 | LR: 4.39e-04 | 696.2ms/step | 28.7 samples/s +Step 10600/44000 | Loss: 3.1610 | LR: 4.39e-04 | 695.8ms/step | 28.7 samples/s +Step 10610/44000 | Loss: 3.1317 | LR: 4.39e-04 | 696.1ms/step | 28.7 samples/s +Step 10620/44000 | Loss: 3.1238 | LR: 4.38e-04 | 696.6ms/step | 28.7 samples/s +Step 10630/44000 | Loss: 3.1319 | LR: 4.38e-04 | 695.6ms/step | 28.8 samples/s +Step 10640/44000 | Loss: 3.1541 | LR: 4.38e-04 | 696.3ms/step | 28.7 samples/s +Step 10650/44000 | Loss: 3.1470 | LR: 4.38e-04 | 696.3ms/step | 28.7 samples/s +Step 10660/44000 | Loss: 3.1479 | LR: 4.38e-04 | 695.7ms/step | 28.7 samples/s +Step 10670/44000 | Loss: 3.1704 | LR: 4.38e-04 | 695.2ms/step | 28.8 samples/s +Step 10680/44000 | Loss: 3.1339 | LR: 4.38e-04 | 697.0ms/step | 28.7 samples/s +Step 10690/44000 | Loss: 3.1793 | LR: 4.38e-04 | 695.7ms/step | 28.7 samples/s +Step 10700/44000 | Loss: 3.1229 | LR: 4.37e-04 | 697.3ms/step | 28.7 samples/s +Step 10710/44000 | Loss: 3.1287 | LR: 4.37e-04 | 695.2ms/step | 28.8 samples/s +Step 10720/44000 | Loss: 3.1348 | LR: 4.37e-04 | 695.2ms/step | 28.8 samples/s +Step 10730/44000 | Loss: 3.1132 | LR: 4.37e-04 | 693.6ms/step | 28.8 samples/s +Step 10740/44000 | Loss: 3.1165 | LR: 4.37e-04 | 696.0ms/step | 28.7 samples/s +Step 10750/44000 | Loss: 3.1541 | LR: 4.37e-04 | 696.4ms/step | 28.7 samples/s +Validation at step 10750: Loss = 3.1467, Perplexity = 23.26 +Step 10760/44000 | Loss: 3.1252 | LR: 4.37e-04 | 1266.1ms/step | 15.8 samples/s +Step 10770/44000 | Loss: 3.1009 | LR: 4.37e-04 | 693.2ms/step | 28.8 samples/s +Step 10780/44000 | Loss: 3.0957 | LR: 4.37e-04 | 695.6ms/step | 28.8 samples/s +Step 10790/44000 | Loss: 3.1397 | LR: 4.36e-04 | 697.0ms/step | 28.7 samples/s +Step 10800/44000 | Loss: 3.1001 | LR: 4.36e-04 | 696.2ms/step | 28.7 samples/s +Step 10810/44000 | Loss: 3.1882 | LR: 4.36e-04 | 696.3ms/step | 28.7 samples/s +Step 10820/44000 | Loss: 3.1107 | LR: 4.36e-04 | 696.3ms/step | 28.7 samples/s +Step 10830/44000 | Loss: 3.1432 | LR: 4.36e-04 | 703.8ms/step | 28.4 samples/s +Step 10840/44000 | Loss: 3.0917 | LR: 4.36e-04 | 706.4ms/step | 28.3 samples/s +Step 10850/44000 | Loss: 3.1484 | LR: 4.36e-04 | 703.5ms/step | 28.4 samples/s +Step 10860/44000 | Loss: 3.1202 | LR: 4.36e-04 | 703.1ms/step | 28.4 samples/s +Step 10870/44000 | Loss: 3.1483 | LR: 4.35e-04 | 698.7ms/step | 28.6 samples/s +Step 10880/44000 | Loss: 3.0990 | LR: 4.35e-04 | 693.5ms/step | 28.8 samples/s +Step 10890/44000 | Loss: 3.1403 | LR: 4.35e-04 | 693.9ms/step | 28.8 samples/s +Step 10900/44000 | Loss: 3.2067 | LR: 4.35e-04 | 694.2ms/step | 28.8 samples/s +Step 10910/44000 | Loss: 3.0915 | LR: 4.35e-04 | 695.1ms/step | 28.8 samples/s +Step 10920/44000 | Loss: 3.1363 | LR: 4.35e-04 | 694.9ms/step | 28.8 samples/s +Step 10930/44000 | Loss: 3.1152 | LR: 4.35e-04 | 694.6ms/step | 28.8 samples/s +Step 10940/44000 | Loss: 3.1345 | LR: 4.35e-04 | 698.1ms/step | 28.7 samples/s +Step 10950/44000 | Loss: 3.1300 | LR: 4.34e-04 | 702.9ms/step | 28.5 samples/s +Step 10960/44000 | Loss: 3.1157 | LR: 4.34e-04 | 703.5ms/step | 28.4 samples/s +Step 10970/44000 | Loss: 3.1452 | LR: 4.34e-04 | 705.5ms/step | 28.3 samples/s +Step 10980/44000 | Loss: 3.1445 | LR: 4.34e-04 | 709.9ms/step | 28.2 samples/s +Step 10990/44000 | Loss: 3.1056 | LR: 4.34e-04 | 700.5ms/step | 28.5 samples/s +Step 11000/44000 | Loss: 3.1067 | LR: 4.34e-04 | 700.0ms/step | 28.6 samples/s +Validation at step 11000: Loss = 3.1400, Perplexity = 23.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-11000 +Step 11010/44000 | Loss: 3.1173 | LR: 4.34e-04 | 1585.3ms/step | 12.6 samples/s +Step 11020/44000 | Loss: 3.1480 | LR: 4.34e-04 | 700.7ms/step | 28.5 samples/s +Step 11030/44000 | Loss: 3.1360 | LR: 4.33e-04 | 699.9ms/step | 28.6 samples/s +Step 11040/44000 | Loss: 3.1727 | LR: 4.33e-04 | 700.1ms/step | 28.6 samples/s +Step 11050/44000 | Loss: 3.1282 | LR: 4.33e-04 | 699.7ms/step | 28.6 samples/s +Step 11060/44000 | Loss: 3.0990 | LR: 4.33e-04 | 700.1ms/step | 28.6 samples/s +Step 11070/44000 | Loss: 3.1217 | LR: 4.33e-04 | 699.9ms/step | 28.6 samples/s +Step 11080/44000 | Loss: 3.1358 | LR: 4.33e-04 | 700.3ms/step | 28.6 samples/s +Step 11090/44000 | Loss: 3.1703 | LR: 4.33e-04 | 699.8ms/step | 28.6 samples/s +Step 11100/44000 | Loss: 3.1183 | LR: 4.33e-04 | 700.0ms/step | 28.6 samples/s +Step 11110/44000 | Loss: 3.1828 | LR: 4.32e-04 | 699.9ms/step | 28.6 samples/s +Step 11120/44000 | Loss: 3.0747 | LR: 4.32e-04 | 700.2ms/step | 28.6 samples/s +Step 11130/44000 | Loss: 3.1321 | LR: 4.32e-04 | 700.0ms/step | 28.6 samples/s +Step 11140/44000 | Loss: 3.1129 | LR: 4.32e-04 | 699.9ms/step | 28.6 samples/s +Step 11150/44000 | Loss: 3.1459 | LR: 4.32e-04 | 700.0ms/step | 28.6 samples/s +Step 11160/44000 | Loss: 3.1292 | LR: 4.32e-04 | 699.6ms/step | 28.6 samples/s +Step 11170/44000 | Loss: 3.1535 | LR: 4.32e-04 | 700.4ms/step | 28.6 samples/s +Step 11180/44000 | Loss: 3.1550 | LR: 4.32e-04 | 700.1ms/step | 28.6 samples/s +Step 11190/44000 | Loss: 3.1082 | LR: 4.31e-04 | 699.7ms/step | 28.6 samples/s +Step 11200/44000 | Loss: 3.1553 | LR: 4.31e-04 | 700.3ms/step | 28.6 samples/s +Step 11210/44000 | Loss: 3.1126 | LR: 4.31e-04 | 701.9ms/step | 28.5 samples/s +Step 11220/44000 | Loss: 3.1224 | LR: 4.31e-04 | 708.6ms/step | 28.2 samples/s +Step 11230/44000 | Loss: 3.1478 | LR: 4.31e-04 | 705.5ms/step | 28.3 samples/s +Step 11240/44000 | Loss: 3.1458 | LR: 4.31e-04 | 704.5ms/step | 28.4 samples/s +Step 11250/44000 | Loss: 3.1175 | LR: 4.31e-04 | 708.0ms/step | 28.2 samples/s +Validation at step 11250: Loss = 3.1344, Perplexity = 22.98 +Step 11260/44000 | Loss: 3.0679 | LR: 4.31e-04 | 1297.8ms/step | 15.4 samples/s +Step 11270/44000 | Loss: 3.1273 | LR: 4.30e-04 | 704.2ms/step | 28.4 samples/s +Step 11280/44000 | Loss: 3.0968 | LR: 4.30e-04 | 703.5ms/step | 28.4 samples/s +Step 11290/44000 | Loss: 3.1223 | LR: 4.30e-04 | 704.0ms/step | 28.4 samples/s +Step 11300/44000 | Loss: 3.1253 | LR: 4.30e-04 | 704.5ms/step | 28.4 samples/s +Step 11310/44000 | Loss: 3.1289 | LR: 4.30e-04 | 702.0ms/step | 28.5 samples/s +Step 11320/44000 | Loss: 3.1132 | LR: 4.30e-04 | 701.5ms/step | 28.5 samples/s +Step 11330/44000 | Loss: 3.1218 | LR: 4.30e-04 | 700.3ms/step | 28.6 samples/s +Step 11340/44000 | Loss: 3.1464 | LR: 4.30e-04 | 700.1ms/step | 28.6 samples/s +Step 11350/44000 | Loss: 3.1055 | LR: 4.29e-04 | 701.4ms/step | 28.5 samples/s +Step 11360/44000 | Loss: 3.1433 | LR: 4.29e-04 | 700.1ms/step | 28.6 samples/s +Step 11370/44000 | Loss: 3.1398 | LR: 4.29e-04 | 702.1ms/step | 28.5 samples/s +Step 11380/44000 | Loss: 3.1543 | LR: 4.29e-04 | 699.9ms/step | 28.6 samples/s +Step 11390/44000 | Loss: 3.1194 | LR: 4.29e-04 | 699.8ms/step | 28.6 samples/s +Step 11400/44000 | Loss: 3.0954 | LR: 4.29e-04 | 700.0ms/step | 28.6 samples/s +Step 11410/44000 | Loss: 3.1133 | LR: 4.29e-04 | 699.9ms/step | 28.6 samples/s +Step 11420/44000 | Loss: 3.1431 | LR: 4.29e-04 | 698.4ms/step | 28.6 samples/s +Step 11430/44000 | Loss: 3.1077 | LR: 4.28e-04 | 699.9ms/step | 28.6 samples/s +Step 11440/44000 | Loss: 3.1409 | LR: 4.28e-04 | 702.0ms/step | 28.5 samples/s +Step 11450/44000 | Loss: 3.1508 | LR: 4.28e-04 | 700.0ms/step | 28.6 samples/s +Step 11460/44000 | Loss: 3.1311 | LR: 4.28e-04 | 699.9ms/step | 28.6 samples/s +Step 11470/44000 | Loss: 3.1245 | LR: 4.28e-04 | 699.4ms/step | 28.6 samples/s +Step 11480/44000 | Loss: 3.1428 | LR: 4.28e-04 | 697.3ms/step | 28.7 samples/s +Step 11490/44000 | Loss: 3.1225 | LR: 4.28e-04 | 696.7ms/step | 28.7 samples/s +Step 11500/44000 | Loss: 3.0850 | LR: 4.28e-04 | 698.2ms/step | 28.6 samples/s +Validation at step 11500: Loss = 3.1305, Perplexity = 22.88 +Step 11510/44000 | Loss: 3.0771 | LR: 4.27e-04 | 1277.1ms/step | 15.7 samples/s +Step 11520/44000 | Loss: 3.1145 | LR: 4.27e-04 | 697.6ms/step | 28.7 samples/s +Step 11530/44000 | Loss: 3.0802 | LR: 4.27e-04 | 697.4ms/step | 28.7 samples/s +Step 11540/44000 | Loss: 3.0671 | LR: 4.27e-04 | 697.5ms/step | 28.7 samples/s +Step 11550/44000 | Loss: 3.1293 | LR: 4.27e-04 | 698.2ms/step | 28.6 samples/s +Step 11560/44000 | Loss: 3.1124 | LR: 4.27e-04 | 698.7ms/step | 28.6 samples/s +Step 11570/44000 | Loss: 3.1250 | LR: 4.27e-04 | 700.1ms/step | 28.6 samples/s +Step 11580/44000 | Loss: 3.1489 | LR: 4.27e-04 | 699.9ms/step | 28.6 samples/s +Step 11590/44000 | Loss: 3.0771 | LR: 4.26e-04 | 699.6ms/step | 28.6 samples/s +Step 11600/44000 | Loss: 3.1124 | LR: 4.26e-04 | 701.0ms/step | 28.5 samples/s +Step 11610/44000 | Loss: 3.1175 | LR: 4.26e-04 | 709.2ms/step | 28.2 samples/s +Step 11620/44000 | Loss: 3.1168 | LR: 4.26e-04 | 702.1ms/step | 28.5 samples/s +Step 11630/44000 | Loss: 3.1387 | LR: 4.26e-04 | 702.0ms/step | 28.5 samples/s +Step 11640/44000 | Loss: 3.0508 | LR: 4.26e-04 | 699.7ms/step | 28.6 samples/s +Step 11650/44000 | Loss: 3.1000 | LR: 4.26e-04 | 700.1ms/step | 28.6 samples/s +Step 11660/44000 | Loss: 3.1435 | LR: 4.26e-04 | 700.0ms/step | 28.6 samples/s +Step 11670/44000 | Loss: 3.1160 | LR: 4.25e-04 | 701.5ms/step | 28.5 samples/s +Step 11680/44000 | Loss: 3.0723 | LR: 4.25e-04 | 700.5ms/step | 28.6 samples/s +Step 11690/44000 | Loss: 3.1056 | LR: 4.25e-04 | 699.6ms/step | 28.6 samples/s +Step 11700/44000 | Loss: 3.0950 | LR: 4.25e-04 | 698.8ms/step | 28.6 samples/s +Step 11710/44000 | Loss: 3.1205 | LR: 4.25e-04 | 699.5ms/step | 28.6 samples/s +Step 11720/44000 | Loss: 3.1083 | LR: 4.25e-04 | 700.0ms/step | 28.6 samples/s +Step 11730/44000 | Loss: 3.1016 | LR: 4.25e-04 | 699.9ms/step | 28.6 samples/s +Step 11740/44000 | Loss: 3.1308 | LR: 4.24e-04 | 700.1ms/step | 28.6 samples/s +Step 11750/44000 | Loss: 3.0820 | LR: 4.24e-04 | 699.6ms/step | 28.6 samples/s +Validation at step 11750: Loss = 3.1269, Perplexity = 22.80 +Step 11760/44000 | Loss: 3.1014 | LR: 4.24e-04 | 1290.4ms/step | 15.5 samples/s +Step 11770/44000 | Loss: 3.1559 | LR: 4.24e-04 | 699.7ms/step | 28.6 samples/s +Step 11780/44000 | Loss: 3.1364 | LR: 4.24e-04 | 698.0ms/step | 28.7 samples/s +Step 11790/44000 | Loss: 3.0980 | LR: 4.24e-04 | 697.8ms/step | 28.7 samples/s +Step 11800/44000 | Loss: 3.1582 | LR: 4.24e-04 | 697.2ms/step | 28.7 samples/s +Step 11810/44000 | Loss: 3.1197 | LR: 4.24e-04 | 698.1ms/step | 28.6 samples/s +Step 11820/44000 | Loss: 3.1133 | LR: 4.23e-04 | 698.1ms/step | 28.6 samples/s +Step 11830/44000 | Loss: 3.1260 | LR: 4.23e-04 | 698.2ms/step | 28.6 samples/s +Step 11840/44000 | Loss: 3.1285 | LR: 4.23e-04 | 698.4ms/step | 28.6 samples/s +Step 11850/44000 | Loss: 3.0854 | LR: 4.23e-04 | 697.3ms/step | 28.7 samples/s +Step 11860/44000 | Loss: 3.1183 | LR: 4.23e-04 | 697.7ms/step | 28.7 samples/s +Step 11870/44000 | Loss: 3.1304 | LR: 4.23e-04 | 699.3ms/step | 28.6 samples/s +Step 11880/44000 | Loss: 3.1253 | LR: 4.23e-04 | 699.1ms/step | 28.6 samples/s +Step 11890/44000 | Loss: 3.1419 | LR: 4.23e-04 | 697.0ms/step | 28.7 samples/s +Step 11900/44000 | Loss: 3.1225 | LR: 4.22e-04 | 698.3ms/step | 28.6 samples/s +Step 11910/44000 | Loss: 3.1593 | LR: 4.22e-04 | 697.7ms/step | 28.7 samples/s +Step 11920/44000 | Loss: 3.1624 | LR: 4.22e-04 | 697.7ms/step | 28.7 samples/s +Step 11930/44000 | Loss: 3.1339 | LR: 4.22e-04 | 697.8ms/step | 28.7 samples/s +Step 11940/44000 | Loss: 3.1227 | LR: 4.22e-04 | 695.9ms/step | 28.7 samples/s +Step 11950/44000 | Loss: 3.1334 | LR: 4.22e-04 | 696.6ms/step | 28.7 samples/s +Step 11960/44000 | Loss: 3.1044 | LR: 4.22e-04 | 697.2ms/step | 28.7 samples/s +Step 11970/44000 | Loss: 3.1152 | LR: 4.21e-04 | 696.6ms/step | 28.7 samples/s +Step 11980/44000 | Loss: 3.1309 | LR: 4.21e-04 | 696.6ms/step | 28.7 samples/s +Step 11990/44000 | Loss: 3.0839 | LR: 4.21e-04 | 696.9ms/step | 28.7 samples/s +Step 12000/44000 | Loss: 3.1532 | LR: 4.21e-04 | 698.8ms/step | 28.6 samples/s +Validation at step 12000: Loss = 3.1229, Perplexity = 22.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-12000 +Step 12010/44000 | Loss: 3.1140 | LR: 4.21e-04 | 1575.1ms/step | 12.7 samples/s +Step 12020/44000 | Loss: 3.1096 | LR: 4.21e-04 | 700.2ms/step | 28.6 samples/s +Step 12030/44000 | Loss: 3.1128 | LR: 4.21e-04 | 698.5ms/step | 28.6 samples/s +Step 12040/44000 | Loss: 3.0946 | LR: 4.21e-04 | 696.8ms/step | 28.7 samples/s +Step 12050/44000 | Loss: 3.1103 | LR: 4.20e-04 | 697.5ms/step | 28.7 samples/s +Step 12060/44000 | Loss: 3.0818 | LR: 4.20e-04 | 697.4ms/step | 28.7 samples/s +Step 12070/44000 | Loss: 3.0689 | LR: 4.20e-04 | 696.9ms/step | 28.7 samples/s +Step 12080/44000 | Loss: 3.1050 | LR: 4.20e-04 | 698.7ms/step | 28.6 samples/s +Step 12090/44000 | Loss: 3.1379 | LR: 4.20e-04 | 697.7ms/step | 28.7 samples/s +Step 12100/44000 | Loss: 3.1209 | LR: 4.20e-04 | 698.7ms/step | 28.6 samples/s +Step 12110/44000 | Loss: 3.1283 | LR: 4.20e-04 | 698.3ms/step | 28.6 samples/s +Step 12120/44000 | Loss: 3.1377 | LR: 4.19e-04 | 697.8ms/step | 28.7 samples/s +Step 12130/44000 | Loss: 3.0889 | LR: 4.19e-04 | 697.4ms/step | 28.7 samples/s +Step 12140/44000 | Loss: 3.1107 | LR: 4.19e-04 | 696.5ms/step | 28.7 samples/s +Step 12150/44000 | Loss: 3.0483 | LR: 4.19e-04 | 697.4ms/step | 28.7 samples/s +Step 12160/44000 | Loss: 3.0689 | LR: 4.19e-04 | 696.9ms/step | 28.7 samples/s +Step 12170/44000 | Loss: 3.1152 | LR: 4.19e-04 | 697.9ms/step | 28.7 samples/s +Step 12180/44000 | Loss: 3.1008 | LR: 4.19e-04 | 697.5ms/step | 28.7 samples/s +Step 12190/44000 | Loss: 3.0840 | LR: 4.19e-04 | 696.5ms/step | 28.7 samples/s +Step 12200/44000 | Loss: 3.0712 | LR: 4.18e-04 | 697.2ms/step | 28.7 samples/s +Step 12210/44000 | Loss: 3.0621 | LR: 4.18e-04 | 696.9ms/step | 28.7 samples/s +Step 12220/44000 | Loss: 3.0876 | LR: 4.18e-04 | 697.1ms/step | 28.7 samples/s +Step 12230/44000 | Loss: 3.1267 | LR: 4.18e-04 | 697.2ms/step | 28.7 samples/s +Step 12240/44000 | Loss: 3.1169 | LR: 4.18e-04 | 696.8ms/step | 28.7 samples/s +Step 12250/44000 | Loss: 3.0509 | LR: 4.18e-04 | 696.4ms/step | 28.7 samples/s +Validation at step 12250: Loss = 3.1178, Perplexity = 22.60 +Step 12260/44000 | Loss: 3.1537 | LR: 4.18e-04 | 1270.2ms/step | 15.7 samples/s +Step 12270/44000 | Loss: 3.0981 | LR: 4.17e-04 | 699.3ms/step | 28.6 samples/s +Step 12280/44000 | Loss: 3.0905 | LR: 4.17e-04 | 700.8ms/step | 28.5 samples/s +Step 12290/44000 | Loss: 3.0436 | LR: 4.17e-04 | 699.8ms/step | 28.6 samples/s +Step 12300/44000 | Loss: 3.1170 | LR: 4.17e-04 | 698.4ms/step | 28.6 samples/s +Step 12310/44000 | Loss: 3.0843 | LR: 4.17e-04 | 697.7ms/step | 28.7 samples/s +Step 12320/44000 | Loss: 3.0917 | LR: 4.17e-04 | 698.4ms/step | 28.6 samples/s +Step 12330/44000 | Loss: 3.0921 | LR: 4.17e-04 | 698.0ms/step | 28.7 samples/s +Step 12340/44000 | Loss: 3.0965 | LR: 4.17e-04 | 699.3ms/step | 28.6 samples/s +Step 12350/44000 | Loss: 3.0910 | LR: 4.16e-04 | 699.8ms/step | 28.6 samples/s +Step 12360/44000 | Loss: 3.0709 | LR: 4.16e-04 | 700.7ms/step | 28.5 samples/s +Step 12370/44000 | Loss: 3.1138 | LR: 4.16e-04 | 699.9ms/step | 28.6 samples/s +Step 12380/44000 | Loss: 3.0776 | LR: 4.16e-04 | 698.0ms/step | 28.7 samples/s +Step 12390/44000 | Loss: 3.0664 | LR: 4.16e-04 | 696.1ms/step | 28.7 samples/s +Step 12400/44000 | Loss: 3.0579 | LR: 4.16e-04 | 696.7ms/step | 28.7 samples/s +Step 12410/44000 | Loss: 3.1215 | LR: 4.16e-04 | 695.2ms/step | 28.8 samples/s +Step 12420/44000 | Loss: 3.1082 | LR: 4.15e-04 | 696.2ms/step | 28.7 samples/s +Step 12430/44000 | Loss: 3.1262 | LR: 4.15e-04 | 696.0ms/step | 28.7 samples/s +Step 12440/44000 | Loss: 3.0958 | LR: 4.15e-04 | 697.8ms/step | 28.7 samples/s +Step 12450/44000 | Loss: 3.1027 | LR: 4.15e-04 | 698.4ms/step | 28.6 samples/s +Step 12460/44000 | Loss: 3.0981 | LR: 4.15e-04 | 697.6ms/step | 28.7 samples/s +Step 12470/44000 | Loss: 3.0848 | LR: 4.15e-04 | 697.4ms/step | 28.7 samples/s +Step 12480/44000 | Loss: 3.0513 | LR: 4.15e-04 | 698.9ms/step | 28.6 samples/s +Step 12490/44000 | Loss: 3.0961 | LR: 4.14e-04 | 698.7ms/step | 28.6 samples/s +Step 12500/44000 | Loss: 3.1157 | LR: 4.14e-04 | 699.1ms/step | 28.6 samples/s +Validation at step 12500: Loss = 3.1120, Perplexity = 22.47 +Step 12510/44000 | Loss: 3.0808 | LR: 4.14e-04 | 1272.2ms/step | 15.7 samples/s +Step 12520/44000 | Loss: 3.1413 | LR: 4.14e-04 | 698.1ms/step | 28.6 samples/s +Step 12530/44000 | Loss: 3.0950 | LR: 4.14e-04 | 697.6ms/step | 28.7 samples/s +Step 12540/44000 | Loss: 3.0804 | LR: 4.14e-04 | 696.6ms/step | 28.7 samples/s +Step 12550/44000 | Loss: 3.1044 | LR: 4.14e-04 | 698.4ms/step | 28.6 samples/s +Step 12560/44000 | Loss: 3.1090 | LR: 4.14e-04 | 696.4ms/step | 28.7 samples/s +Step 12570/44000 | Loss: 3.0995 | LR: 4.13e-04 | 696.8ms/step | 28.7 samples/s +Step 12580/44000 | Loss: 3.1215 | LR: 4.13e-04 | 696.2ms/step | 28.7 samples/s +Step 12590/44000 | Loss: 3.1203 | LR: 4.13e-04 | 697.1ms/step | 28.7 samples/s +Step 12600/44000 | Loss: 3.0850 | LR: 4.13e-04 | 697.3ms/step | 28.7 samples/s +Step 12610/44000 | Loss: 3.1060 | LR: 4.13e-04 | 696.3ms/step | 28.7 samples/s +Step 12620/44000 | Loss: 3.0695 | LR: 4.13e-04 | 695.2ms/step | 28.8 samples/s +Step 12630/44000 | Loss: 3.0870 | LR: 4.13e-04 | 695.7ms/step | 28.7 samples/s +Step 12640/44000 | Loss: 3.1342 | LR: 4.12e-04 | 696.2ms/step | 28.7 samples/s +Step 12650/44000 | Loss: 3.0904 | LR: 4.12e-04 | 695.9ms/step | 28.7 samples/s +Step 12660/44000 | Loss: 3.0642 | LR: 4.12e-04 | 697.9ms/step | 28.7 samples/s +Step 12670/44000 | Loss: 3.0723 | LR: 4.12e-04 | 697.7ms/step | 28.7 samples/s +Step 12680/44000 | Loss: 3.1230 | LR: 4.12e-04 | 696.1ms/step | 28.7 samples/s +Step 12690/44000 | Loss: 3.0862 | LR: 4.12e-04 | 696.2ms/step | 28.7 samples/s +Step 12700/44000 | Loss: 3.1015 | LR: 4.12e-04 | 699.4ms/step | 28.6 samples/s +Step 12710/44000 | Loss: 3.1424 | LR: 4.11e-04 | 696.5ms/step | 28.7 samples/s +Step 12720/44000 | Loss: 3.0863 | LR: 4.11e-04 | 696.3ms/step | 28.7 samples/s +Step 12730/44000 | Loss: 3.1396 | LR: 4.11e-04 | 698.5ms/step | 28.6 samples/s +Step 12740/44000 | Loss: 3.1177 | LR: 4.11e-04 | 693.0ms/step | 28.9 samples/s +Step 12750/44000 | Loss: 3.0529 | LR: 4.11e-04 | 694.2ms/step | 28.8 samples/s +Validation at step 12750: Loss = 3.1079, Perplexity = 22.37 +Step 12760/44000 | Loss: 3.1128 | LR: 4.11e-04 | 1269.0ms/step | 15.8 samples/s +Step 12770/44000 | Loss: 3.0938 | LR: 4.11e-04 | 694.5ms/step | 28.8 samples/s +Step 12780/44000 | Loss: 3.0889 | LR: 4.10e-04 | 694.8ms/step | 28.8 samples/s +Step 12790/44000 | Loss: 3.1008 | LR: 4.10e-04 | 697.6ms/step | 28.7 samples/s +Step 12800/44000 | Loss: 3.1050 | LR: 4.10e-04 | 696.8ms/step | 28.7 samples/s +Step 12810/44000 | Loss: 3.0766 | LR: 4.10e-04 | 696.5ms/step | 28.7 samples/s +Step 12820/44000 | Loss: 3.0725 | LR: 4.10e-04 | 696.3ms/step | 28.7 samples/s +Step 12830/44000 | Loss: 3.0745 | LR: 4.10e-04 | 696.1ms/step | 28.7 samples/s +Step 12840/44000 | Loss: 3.1177 | LR: 4.10e-04 | 697.8ms/step | 28.7 samples/s +Step 12850/44000 | Loss: 3.0904 | LR: 4.09e-04 | 698.6ms/step | 28.6 samples/s +Step 12860/44000 | Loss: 3.1428 | LR: 4.09e-04 | 696.4ms/step | 28.7 samples/s +Step 12870/44000 | Loss: 3.0651 | LR: 4.09e-04 | 696.6ms/step | 28.7 samples/s +Step 12880/44000 | Loss: 3.0933 | LR: 4.09e-04 | 697.2ms/step | 28.7 samples/s +Step 12890/44000 | Loss: 3.0739 | LR: 4.09e-04 | 700.1ms/step | 28.6 samples/s +Step 12900/44000 | Loss: 3.1517 | LR: 4.09e-04 | 706.0ms/step | 28.3 samples/s +Step 12910/44000 | Loss: 3.0580 | LR: 4.09e-04 | 697.4ms/step | 28.7 samples/s +Step 12920/44000 | Loss: 3.0745 | LR: 4.09e-04 | 695.8ms/step | 28.7 samples/s +Step 12930/44000 | Loss: 3.1091 | LR: 4.08e-04 | 694.6ms/step | 28.8 samples/s +Step 12940/44000 | Loss: 3.0811 | LR: 4.08e-04 | 696.0ms/step | 28.7 samples/s +Step 12950/44000 | Loss: 3.0646 | LR: 4.08e-04 | 696.1ms/step | 28.7 samples/s +Step 12960/44000 | Loss: 3.0629 | LR: 4.08e-04 | 695.8ms/step | 28.7 samples/s +Step 12970/44000 | Loss: 3.0773 | LR: 4.08e-04 | 695.0ms/step | 28.8 samples/s +Step 12980/44000 | Loss: 3.0967 | LR: 4.08e-04 | 694.3ms/step | 28.8 samples/s +Step 12990/44000 | Loss: 3.0945 | LR: 4.08e-04 | 695.2ms/step | 28.8 samples/s +Step 13000/44000 | Loss: 3.1169 | LR: 4.07e-04 | 695.2ms/step | 28.8 samples/s +Validation at step 13000: Loss = 3.1048, Perplexity = 22.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-13000 +Step 13010/44000 | Loss: 3.0850 | LR: 4.07e-04 | 1614.4ms/step | 12.4 samples/s +Step 13020/44000 | Loss: 3.0548 | LR: 4.07e-04 | 697.1ms/step | 28.7 samples/s +Step 13030/44000 | Loss: 3.0637 | LR: 4.07e-04 | 695.0ms/step | 28.8 samples/s +Step 13040/44000 | Loss: 3.0838 | LR: 4.07e-04 | 700.3ms/step | 28.6 samples/s +Step 13050/44000 | Loss: 3.0772 | LR: 4.07e-04 | 705.7ms/step | 28.3 samples/s +Step 13060/44000 | Loss: 3.0970 | LR: 4.07e-04 | 699.6ms/step | 28.6 samples/s +Step 13070/44000 | Loss: 3.0876 | LR: 4.06e-04 | 700.1ms/step | 28.6 samples/s +Step 13080/44000 | Loss: 3.1144 | LR: 4.06e-04 | 699.5ms/step | 28.6 samples/s +Step 13090/44000 | Loss: 3.0893 | LR: 4.06e-04 | 700.8ms/step | 28.5 samples/s +Step 13100/44000 | Loss: 3.0862 | LR: 4.06e-04 | 699.8ms/step | 28.6 samples/s +Step 13110/44000 | Loss: 3.0752 | LR: 4.06e-04 | 702.2ms/step | 28.5 samples/s +Step 13120/44000 | Loss: 3.0605 | LR: 4.06e-04 | 699.8ms/step | 28.6 samples/s +Step 13130/44000 | Loss: 3.1283 | LR: 4.06e-04 | 699.9ms/step | 28.6 samples/s +Step 13140/44000 | Loss: 3.0934 | LR: 4.05e-04 | 699.7ms/step | 28.6 samples/s +Step 13150/44000 | Loss: 3.1075 | LR: 4.05e-04 | 698.5ms/step | 28.6 samples/s +Step 13160/44000 | Loss: 3.0715 | LR: 4.05e-04 | 702.0ms/step | 28.5 samples/s +Step 13170/44000 | Loss: 3.0813 | LR: 4.05e-04 | 699.7ms/step | 28.6 samples/s +Step 13180/44000 | Loss: 3.0479 | LR: 4.05e-04 | 699.4ms/step | 28.6 samples/s +Step 13190/44000 | Loss: 3.0802 | LR: 4.05e-04 | 698.1ms/step | 28.7 samples/s +Step 13200/44000 | Loss: 3.1191 | LR: 4.05e-04 | 696.5ms/step | 28.7 samples/s +Step 13210/44000 | Loss: 3.1048 | LR: 4.04e-04 | 697.3ms/step | 28.7 samples/s +Step 13220/44000 | Loss: 3.1057 | LR: 4.04e-04 | 696.9ms/step | 28.7 samples/s +Step 13230/44000 | Loss: 3.1033 | LR: 4.04e-04 | 697.9ms/step | 28.7 samples/s +Step 13240/44000 | Loss: 3.0985 | LR: 4.04e-04 | 700.2ms/step | 28.6 samples/s +Step 13250/44000 | Loss: 3.0649 | LR: 4.04e-04 | 698.5ms/step | 28.6 samples/s +Validation at step 13250: Loss = 3.1016, Perplexity = 22.23 +Step 13260/44000 | Loss: 3.1011 | LR: 4.04e-04 | 1283.0ms/step | 15.6 samples/s +Step 13270/44000 | Loss: 3.1184 | LR: 4.04e-04 | 696.9ms/step | 28.7 samples/s +Step 13280/44000 | Loss: 3.1127 | LR: 4.03e-04 | 697.9ms/step | 28.7 samples/s +Step 13290/44000 | Loss: 3.1124 | LR: 4.03e-04 | 697.5ms/step | 28.7 samples/s +Step 13300/44000 | Loss: 3.0599 | LR: 4.03e-04 | 697.3ms/step | 28.7 samples/s +Step 13310/44000 | Loss: 3.0881 | LR: 4.03e-04 | 697.0ms/step | 28.7 samples/s +Step 13320/44000 | Loss: 3.0669 | LR: 4.03e-04 | 697.8ms/step | 28.7 samples/s +Step 13330/44000 | Loss: 3.0951 | LR: 4.03e-04 | 697.5ms/step | 28.7 samples/s +Step 13340/44000 | Loss: 3.1008 | LR: 4.03e-04 | 696.1ms/step | 28.7 samples/s +Step 13350/44000 | Loss: 3.0986 | LR: 4.02e-04 | 697.9ms/step | 28.7 samples/s +Step 13360/44000 | Loss: 3.1241 | LR: 4.02e-04 | 700.0ms/step | 28.6 samples/s +Step 13370/44000 | Loss: 3.0647 | LR: 4.02e-04 | 698.2ms/step | 28.6 samples/s +Step 13380/44000 | Loss: 3.0427 | LR: 4.02e-04 | 698.4ms/step | 28.6 samples/s +Step 13390/44000 | Loss: 3.0908 | LR: 4.02e-04 | 695.9ms/step | 28.7 samples/s +Step 13400/44000 | Loss: 3.0741 | LR: 4.02e-04 | 696.1ms/step | 28.7 samples/s +Step 13410/44000 | Loss: 3.0729 | LR: 4.02e-04 | 700.1ms/step | 28.6 samples/s +Step 13420/44000 | Loss: 3.0898 | LR: 4.01e-04 | 699.3ms/step | 28.6 samples/s +Step 13430/44000 | Loss: 3.0742 | LR: 4.01e-04 | 697.5ms/step | 28.7 samples/s +Step 13440/44000 | Loss: 3.1204 | LR: 4.01e-04 | 695.5ms/step | 28.8 samples/s +Step 13450/44000 | Loss: 3.0809 | LR: 4.01e-04 | 697.2ms/step | 28.7 samples/s +Step 13460/44000 | Loss: 3.0977 | LR: 4.01e-04 | 695.9ms/step | 28.7 samples/s +Step 13470/44000 | Loss: 3.0813 | LR: 4.01e-04 | 697.1ms/step | 28.7 samples/s +Step 13480/44000 | Loss: 3.0897 | LR: 4.01e-04 | 696.9ms/step | 28.7 samples/s +Step 13490/44000 | Loss: 3.1063 | LR: 4.00e-04 | 698.7ms/step | 28.6 samples/s +Step 13500/44000 | Loss: 3.0693 | LR: 4.00e-04 | 700.0ms/step | 28.6 samples/s +Validation at step 13500: Loss = 3.0964, Perplexity = 22.12 +Step 13510/44000 | Loss: 3.1275 | LR: 4.00e-04 | 1279.6ms/step | 15.6 samples/s +Step 13520/44000 | Loss: 3.0838 | LR: 4.00e-04 | 696.2ms/step | 28.7 samples/s +Step 13530/44000 | Loss: 3.1118 | LR: 4.00e-04 | 695.4ms/step | 28.8 samples/s +Step 13540/44000 | Loss: 3.0773 | LR: 4.00e-04 | 696.3ms/step | 28.7 samples/s +Step 13550/44000 | Loss: 3.0659 | LR: 4.00e-04 | 697.1ms/step | 28.7 samples/s +Step 13560/44000 | Loss: 3.1021 | LR: 3.99e-04 | 696.3ms/step | 28.7 samples/s +Step 13570/44000 | Loss: 3.0527 | LR: 3.99e-04 | 695.0ms/step | 28.8 samples/s +Step 13580/44000 | Loss: 3.1144 | LR: 3.99e-04 | 695.6ms/step | 28.8 samples/s +Step 13590/44000 | Loss: 3.0322 | LR: 3.99e-04 | 695.3ms/step | 28.8 samples/s +Step 13600/44000 | Loss: 3.0765 | LR: 3.99e-04 | 696.5ms/step | 28.7 samples/s +Step 13610/44000 | Loss: 3.0371 | LR: 3.99e-04 | 695.4ms/step | 28.8 samples/s +Step 13620/44000 | Loss: 3.0707 | LR: 3.98e-04 | 695.8ms/step | 28.7 samples/s +Step 13630/44000 | Loss: 3.0411 | LR: 3.98e-04 | 696.2ms/step | 28.7 samples/s +Step 13640/44000 | Loss: 3.0901 | LR: 3.98e-04 | 696.7ms/step | 28.7 samples/s +Step 13650/44000 | Loss: 3.0763 | LR: 3.98e-04 | 697.6ms/step | 28.7 samples/s +Step 13660/44000 | Loss: 3.0662 | LR: 3.98e-04 | 696.9ms/step | 28.7 samples/s +Step 13670/44000 | Loss: 3.0642 | LR: 3.98e-04 | 705.8ms/step | 28.3 samples/s +Step 13680/44000 | Loss: 3.0876 | LR: 3.98e-04 | 705.9ms/step | 28.3 samples/s +Step 13690/44000 | Loss: 3.0970 | LR: 3.97e-04 | 699.9ms/step | 28.6 samples/s +Step 13700/44000 | Loss: 3.0902 | LR: 3.97e-04 | 700.0ms/step | 28.6 samples/s +Step 13710/44000 | Loss: 3.1073 | LR: 3.97e-04 | 700.0ms/step | 28.6 samples/s +Step 13720/44000 | Loss: 3.0526 | LR: 3.97e-04 | 700.0ms/step | 28.6 samples/s +Step 13730/44000 | Loss: 3.1138 | LR: 3.97e-04 | 700.1ms/step | 28.6 samples/s +Step 13740/44000 | Loss: 3.0505 | LR: 3.97e-04 | 702.0ms/step | 28.5 samples/s +Step 13750/44000 | Loss: 3.1239 | LR: 3.97e-04 | 697.0ms/step | 28.7 samples/s +Validation at step 13750: Loss = 3.0928, Perplexity = 22.04 +Step 13760/44000 | Loss: 3.1038 | LR: 3.96e-04 | 1274.2ms/step | 15.7 samples/s +Step 13770/44000 | Loss: 3.0774 | LR: 3.96e-04 | 696.3ms/step | 28.7 samples/s +Step 13780/44000 | Loss: 3.0633 | LR: 3.96e-04 | 698.0ms/step | 28.7 samples/s +Step 13790/44000 | Loss: 3.0572 | LR: 3.96e-04 | 698.2ms/step | 28.6 samples/s +Step 13800/44000 | Loss: 3.1071 | LR: 3.96e-04 | 697.7ms/step | 28.7 samples/s +Step 13810/44000 | Loss: 3.0797 | LR: 3.96e-04 | 697.7ms/step | 28.7 samples/s +Step 13820/44000 | Loss: 3.1096 | LR: 3.96e-04 | 695.9ms/step | 28.7 samples/s +Step 13830/44000 | Loss: 3.0636 | LR: 3.95e-04 | 696.8ms/step | 28.7 samples/s +Step 13840/44000 | Loss: 3.0582 | LR: 3.95e-04 | 695.4ms/step | 28.8 samples/s +Step 13850/44000 | Loss: 3.0704 | LR: 3.95e-04 | 695.4ms/step | 28.8 samples/s +Step 13860/44000 | Loss: 3.0873 | LR: 3.95e-04 | 694.9ms/step | 28.8 samples/s +Step 13870/44000 | Loss: 3.0192 | LR: 3.95e-04 | 697.6ms/step | 28.7 samples/s +Step 13880/44000 | Loss: 3.0919 | LR: 3.95e-04 | 696.7ms/step | 28.7 samples/s +Step 13890/44000 | Loss: 3.0279 | LR: 3.95e-04 | 696.5ms/step | 28.7 samples/s +Step 13900/44000 | Loss: 3.0711 | LR: 3.94e-04 | 697.1ms/step | 28.7 samples/s +Step 13910/44000 | Loss: 3.0436 | LR: 3.94e-04 | 698.1ms/step | 28.6 samples/s +Step 13920/44000 | Loss: 3.0802 | LR: 3.94e-04 | 697.5ms/step | 28.7 samples/s +Step 13930/44000 | Loss: 3.0889 | LR: 3.94e-04 | 696.7ms/step | 28.7 samples/s +Step 13940/44000 | Loss: 3.0448 | LR: 3.94e-04 | 697.3ms/step | 28.7 samples/s +Step 13950/44000 | Loss: 3.0683 | LR: 3.94e-04 | 696.2ms/step | 28.7 samples/s +Step 13960/44000 | Loss: 3.0663 | LR: 3.93e-04 | 696.4ms/step | 28.7 samples/s +Step 13970/44000 | Loss: 3.1006 | LR: 3.93e-04 | 695.5ms/step | 28.8 samples/s +Step 13980/44000 | Loss: 3.0783 | LR: 3.93e-04 | 696.2ms/step | 28.7 samples/s +Step 13990/44000 | Loss: 3.0795 | LR: 3.93e-04 | 695.9ms/step | 28.7 samples/s +Step 14000/44000 | Loss: 3.0680 | LR: 3.93e-04 | 695.6ms/step | 28.8 samples/s +Validation at step 14000: Loss = 3.0893, Perplexity = 21.96 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-14000 +Step 14010/44000 | Loss: 3.1220 | LR: 3.93e-04 | 1583.9ms/step | 12.6 samples/s +Step 14020/44000 | Loss: 3.0312 | LR: 3.93e-04 | 695.8ms/step | 28.7 samples/s +Step 14030/44000 | Loss: 3.0836 | LR: 3.92e-04 | 695.6ms/step | 28.8 samples/s +Step 14040/44000 | Loss: 3.0807 | LR: 3.92e-04 | 698.2ms/step | 28.6 samples/s +Step 14050/44000 | Loss: 3.0834 | LR: 3.92e-04 | 700.5ms/step | 28.6 samples/s +Step 14060/44000 | Loss: 3.0715 | LR: 3.92e-04 | 708.6ms/step | 28.2 samples/s +Step 14070/44000 | Loss: 3.0718 | LR: 3.92e-04 | 707.4ms/step | 28.3 samples/s +Step 14080/44000 | Loss: 3.0356 | LR: 3.92e-04 | 707.7ms/step | 28.3 samples/s +Step 14090/44000 | Loss: 3.0415 | LR: 3.92e-04 | 707.5ms/step | 28.3 samples/s +Step 14100/44000 | Loss: 3.0751 | LR: 3.91e-04 | 706.3ms/step | 28.3 samples/s +Step 14110/44000 | Loss: 3.0664 | LR: 3.91e-04 | 707.5ms/step | 28.3 samples/s +Step 14120/44000 | Loss: 3.1320 | LR: 3.91e-04 | 706.3ms/step | 28.3 samples/s +Step 14130/44000 | Loss: 3.0911 | LR: 3.91e-04 | 704.3ms/step | 28.4 samples/s +Step 14140/44000 | Loss: 3.0524 | LR: 3.91e-04 | 704.6ms/step | 28.4 samples/s +Step 14150/44000 | Loss: 3.0917 | LR: 3.91e-04 | 706.2ms/step | 28.3 samples/s +Step 14160/44000 | Loss: 3.0480 | LR: 3.90e-04 | 701.4ms/step | 28.5 samples/s +Step 14170/44000 | Loss: 3.0542 | LR: 3.90e-04 | 706.4ms/step | 28.3 samples/s +Step 14180/44000 | Loss: 3.0554 | LR: 3.90e-04 | 702.5ms/step | 28.5 samples/s +Step 14190/44000 | Loss: 3.0771 | LR: 3.90e-04 | 706.0ms/step | 28.3 samples/s +Step 14200/44000 | Loss: 3.0571 | LR: 3.90e-04 | 706.8ms/step | 28.3 samples/s +Step 14210/44000 | Loss: 3.0675 | LR: 3.90e-04 | 704.4ms/step | 28.4 samples/s +Step 14220/44000 | Loss: 3.0566 | LR: 3.90e-04 | 704.1ms/step | 28.4 samples/s +Step 14230/44000 | Loss: 3.0759 | LR: 3.89e-04 | 703.1ms/step | 28.4 samples/s +Step 14240/44000 | Loss: 3.0691 | LR: 3.89e-04 | 700.6ms/step | 28.5 samples/s +Step 14250/44000 | Loss: 3.0697 | LR: 3.89e-04 | 695.4ms/step | 28.8 samples/s +Validation at step 14250: Loss = 3.0860, Perplexity = 21.89 +Step 14260/44000 | Loss: 3.0589 | LR: 3.89e-04 | 1280.4ms/step | 15.6 samples/s +Step 14270/44000 | Loss: 3.0762 | LR: 3.89e-04 | 701.6ms/step | 28.5 samples/s +Step 14280/44000 | Loss: 3.0725 | LR: 3.89e-04 | 702.0ms/step | 28.5 samples/s +Step 14290/44000 | Loss: 3.0573 | LR: 3.89e-04 | 699.8ms/step | 28.6 samples/s +Step 14300/44000 | Loss: 3.0751 | LR: 3.88e-04 | 700.2ms/step | 28.6 samples/s +Step 14310/44000 | Loss: 3.0718 | LR: 3.88e-04 | 700.0ms/step | 28.6 samples/s +Step 14320/44000 | Loss: 3.0675 | LR: 3.88e-04 | 699.9ms/step | 28.6 samples/s +Step 14330/44000 | Loss: 3.0884 | LR: 3.88e-04 | 700.1ms/step | 28.6 samples/s +Step 14340/44000 | Loss: 3.1088 | LR: 3.88e-04 | 700.3ms/step | 28.6 samples/s +Step 14350/44000 | Loss: 3.0802 | LR: 3.88e-04 | 699.5ms/step | 28.6 samples/s +Step 14360/44000 | Loss: 3.0518 | LR: 3.87e-04 | 700.0ms/step | 28.6 samples/s +Step 14370/44000 | Loss: 3.0492 | LR: 3.87e-04 | 699.9ms/step | 28.6 samples/s +Step 14380/44000 | Loss: 3.0723 | LR: 3.87e-04 | 700.1ms/step | 28.6 samples/s +Step 14390/44000 | Loss: 3.0837 | LR: 3.87e-04 | 699.9ms/step | 28.6 samples/s +Step 14400/44000 | Loss: 3.0923 | LR: 3.87e-04 | 700.0ms/step | 28.6 samples/s +Step 14410/44000 | Loss: 3.0556 | LR: 3.87e-04 | 702.0ms/step | 28.5 samples/s +Step 14420/44000 | Loss: 3.0859 | LR: 3.87e-04 | 700.6ms/step | 28.5 samples/s +Step 14430/44000 | Loss: 3.0940 | LR: 3.86e-04 | 698.6ms/step | 28.6 samples/s +Step 14440/44000 | Loss: 3.0290 | LR: 3.86e-04 | 698.6ms/step | 28.6 samples/s +Step 14450/44000 | Loss: 3.0531 | LR: 3.86e-04 | 696.9ms/step | 28.7 samples/s +Step 14460/44000 | Loss: 3.0770 | LR: 3.86e-04 | 699.5ms/step | 28.6 samples/s +Step 14470/44000 | Loss: 3.0947 | LR: 3.86e-04 | 698.6ms/step | 28.6 samples/s +Step 14480/44000 | Loss: 3.0965 | LR: 3.86e-04 | 699.2ms/step | 28.6 samples/s +Step 14490/44000 | Loss: 3.0560 | LR: 3.85e-04 | 700.2ms/step | 28.6 samples/s +Step 14500/44000 | Loss: 3.1157 | LR: 3.85e-04 | 699.5ms/step | 28.6 samples/s +Validation at step 14500: Loss = 3.0820, Perplexity = 21.80 +Step 14510/44000 | Loss: 3.0790 | LR: 3.85e-04 | 1292.5ms/step | 15.5 samples/s +Step 14520/44000 | Loss: 3.0323 | LR: 3.85e-04 | 699.8ms/step | 28.6 samples/s +Step 14530/44000 | Loss: 3.1072 | LR: 3.85e-04 | 697.7ms/step | 28.7 samples/s +Step 14540/44000 | Loss: 3.0333 | LR: 3.85e-04 | 697.3ms/step | 28.7 samples/s +Step 14550/44000 | Loss: 3.0348 | LR: 3.85e-04 | 697.2ms/step | 28.7 samples/s +Step 14560/44000 | Loss: 3.0764 | LR: 3.84e-04 | 697.0ms/step | 28.7 samples/s +Step 14570/44000 | Loss: 3.0934 | LR: 3.84e-04 | 698.3ms/step | 28.6 samples/s +Step 14580/44000 | Loss: 3.0954 | LR: 3.84e-04 | 696.9ms/step | 28.7 samples/s +Step 14590/44000 | Loss: 3.0666 | LR: 3.84e-04 | 699.5ms/step | 28.6 samples/s +Step 14600/44000 | Loss: 3.0739 | LR: 3.84e-04 | 697.8ms/step | 28.7 samples/s +Step 14610/44000 | Loss: 3.1088 | LR: 3.84e-04 | 696.3ms/step | 28.7 samples/s +Step 14620/44000 | Loss: 3.0833 | LR: 3.83e-04 | 695.9ms/step | 28.7 samples/s +Step 14630/44000 | Loss: 3.0599 | LR: 3.83e-04 | 696.2ms/step | 28.7 samples/s +Step 14640/44000 | Loss: 3.0942 | LR: 3.83e-04 | 696.0ms/step | 28.7 samples/s +Step 14650/44000 | Loss: 3.0726 | LR: 3.83e-04 | 697.9ms/step | 28.7 samples/s +Step 14660/44000 | Loss: 3.0738 | LR: 3.83e-04 | 695.9ms/step | 28.7 samples/s +Step 14670/44000 | Loss: 3.0455 | LR: 3.83e-04 | 698.4ms/step | 28.6 samples/s +Step 14680/44000 | Loss: 3.0814 | LR: 3.83e-04 | 696.8ms/step | 28.7 samples/s +Step 14690/44000 | Loss: 3.0645 | LR: 3.82e-04 | 698.1ms/step | 28.7 samples/s +Step 14700/44000 | Loss: 3.0561 | LR: 3.82e-04 | 697.2ms/step | 28.7 samples/s +Step 14710/44000 | Loss: 3.0403 | LR: 3.82e-04 | 696.9ms/step | 28.7 samples/s +Step 14720/44000 | Loss: 3.0994 | LR: 3.82e-04 | 696.7ms/step | 28.7 samples/s +Step 14730/44000 | Loss: 3.0668 | LR: 3.82e-04 | 697.5ms/step | 28.7 samples/s +Step 14740/44000 | Loss: 3.0973 | LR: 3.82e-04 | 695.2ms/step | 28.8 samples/s +Step 14750/44000 | Loss: 3.0568 | LR: 3.81e-04 | 694.9ms/step | 28.8 samples/s +Validation at step 14750: Loss = 3.0795, Perplexity = 21.75 +Step 14760/44000 | Loss: 3.0839 | LR: 3.81e-04 | 1277.6ms/step | 15.7 samples/s +Step 14770/44000 | Loss: 3.0657 | LR: 3.81e-04 | 696.5ms/step | 28.7 samples/s +Step 14780/44000 | Loss: 3.0832 | LR: 3.81e-04 | 696.2ms/step | 28.7 samples/s +Step 14790/44000 | Loss: 3.0515 | LR: 3.81e-04 | 695.8ms/step | 28.7 samples/s +Step 14800/44000 | Loss: 3.0620 | LR: 3.81e-04 | 696.9ms/step | 28.7 samples/s +Step 14810/44000 | Loss: 3.0218 | LR: 3.81e-04 | 696.2ms/step | 28.7 samples/s +Step 14820/44000 | Loss: 3.0748 | LR: 3.80e-04 | 696.7ms/step | 28.7 samples/s +Step 14830/44000 | Loss: 3.0425 | LR: 3.80e-04 | 696.4ms/step | 28.7 samples/s +Step 14840/44000 | Loss: 3.0445 | LR: 3.80e-04 | 696.0ms/step | 28.7 samples/s +Step 14850/44000 | Loss: 3.0217 | LR: 3.80e-04 | 695.8ms/step | 28.7 samples/s +Step 14860/44000 | Loss: 3.0923 | LR: 3.80e-04 | 696.8ms/step | 28.7 samples/s +Step 14870/44000 | Loss: 3.0445 | LR: 3.80e-04 | 695.6ms/step | 28.8 samples/s +Step 14880/44000 | Loss: 3.0413 | LR: 3.79e-04 | 695.9ms/step | 28.7 samples/s +Step 14890/44000 | Loss: 3.0815 | LR: 3.79e-04 | 696.6ms/step | 28.7 samples/s +Step 14900/44000 | Loss: 3.0793 | LR: 3.79e-04 | 697.9ms/step | 28.7 samples/s +Step 14910/44000 | Loss: 3.0758 | LR: 3.79e-04 | 697.9ms/step | 28.7 samples/s +Step 14920/44000 | Loss: 3.0513 | LR: 3.79e-04 | 697.3ms/step | 28.7 samples/s +Step 14930/44000 | Loss: 3.0245 | LR: 3.79e-04 | 696.3ms/step | 28.7 samples/s +Step 14940/44000 | Loss: 3.0772 | LR: 3.79e-04 | 694.8ms/step | 28.8 samples/s +Step 14950/44000 | Loss: 3.0288 | LR: 3.78e-04 | 696.8ms/step | 28.7 samples/s +Step 14960/44000 | Loss: 3.1057 | LR: 3.78e-04 | 697.1ms/step | 28.7 samples/s +Step 14970/44000 | Loss: 3.0837 | LR: 3.78e-04 | 695.2ms/step | 28.8 samples/s +Step 14980/44000 | Loss: 3.0472 | LR: 3.78e-04 | 695.6ms/step | 28.8 samples/s +Step 14990/44000 | Loss: 3.0899 | LR: 3.78e-04 | 694.5ms/step | 28.8 samples/s +Step 15000/44000 | Loss: 3.0714 | LR: 3.78e-04 | 696.5ms/step | 28.7 samples/s +Validation at step 15000: Loss = 3.0757, Perplexity = 21.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-15000 +Step 15010/44000 | Loss: 3.0820 | LR: 3.77e-04 | 1615.0ms/step | 12.4 samples/s +Step 15020/44000 | Loss: 3.0390 | LR: 3.77e-04 | 700.2ms/step | 28.6 samples/s +Step 15030/44000 | Loss: 3.0411 | LR: 3.77e-04 | 696.3ms/step | 28.7 samples/s +Step 15040/44000 | Loss: 3.0645 | LR: 3.77e-04 | 695.2ms/step | 28.8 samples/s +Step 15050/44000 | Loss: 3.0618 | LR: 3.77e-04 | 695.8ms/step | 28.7 samples/s +Step 15060/44000 | Loss: 3.0724 | LR: 3.77e-04 | 695.2ms/step | 28.8 samples/s +Step 15070/44000 | Loss: 3.0921 | LR: 3.77e-04 | 694.9ms/step | 28.8 samples/s +Step 15080/44000 | Loss: 3.0588 | LR: 3.76e-04 | 693.7ms/step | 28.8 samples/s +Step 15090/44000 | Loss: 3.0654 | LR: 3.76e-04 | 694.8ms/step | 28.8 samples/s +Step 15100/44000 | Loss: 3.0593 | LR: 3.76e-04 | 693.9ms/step | 28.8 samples/s +Step 15110/44000 | Loss: 3.0712 | LR: 3.76e-04 | 695.4ms/step | 28.8 samples/s +Step 15120/44000 | Loss: 3.1075 | LR: 3.76e-04 | 694.0ms/step | 28.8 samples/s +Step 15130/44000 | Loss: 3.0589 | LR: 3.76e-04 | 695.2ms/step | 28.8 samples/s +Step 15140/44000 | Loss: 3.0400 | LR: 3.75e-04 | 695.6ms/step | 28.8 samples/s +Step 15150/44000 | Loss: 3.0943 | LR: 3.75e-04 | 694.6ms/step | 28.8 samples/s +Step 15160/44000 | Loss: 3.0486 | LR: 3.75e-04 | 695.7ms/step | 28.7 samples/s +Step 15170/44000 | Loss: 3.0674 | LR: 3.75e-04 | 696.0ms/step | 28.7 samples/s +Step 15180/44000 | Loss: 3.1094 | LR: 3.75e-04 | 697.3ms/step | 28.7 samples/s +Step 15190/44000 | Loss: 3.0604 | LR: 3.75e-04 | 694.7ms/step | 28.8 samples/s +Step 15200/44000 | Loss: 2.9870 | LR: 3.74e-04 | 695.6ms/step | 28.8 samples/s +Step 15210/44000 | Loss: 3.0251 | LR: 3.74e-04 | 694.8ms/step | 28.8 samples/s +Step 15220/44000 | Loss: 3.0936 | LR: 3.74e-04 | 695.2ms/step | 28.8 samples/s +Step 15230/44000 | Loss: 3.0604 | LR: 3.74e-04 | 694.9ms/step | 28.8 samples/s +Step 15240/44000 | Loss: 3.0705 | LR: 3.74e-04 | 692.6ms/step | 28.9 samples/s +Step 15250/44000 | Loss: 3.0580 | LR: 3.74e-04 | 694.2ms/step | 28.8 samples/s +Validation at step 15250: Loss = 3.0717, Perplexity = 21.58 +Step 15260/44000 | Loss: 3.0479 | LR: 3.74e-04 | 1272.2ms/step | 15.7 samples/s +Step 15270/44000 | Loss: 3.0545 | LR: 3.73e-04 | 693.7ms/step | 28.8 samples/s +Step 15280/44000 | Loss: 3.0426 | LR: 3.73e-04 | 693.8ms/step | 28.8 samples/s +Step 15290/44000 | Loss: 3.0586 | LR: 3.73e-04 | 696.3ms/step | 28.7 samples/s +Step 15300/44000 | Loss: 3.0693 | LR: 3.73e-04 | 693.1ms/step | 28.9 samples/s +Step 15310/44000 | Loss: 3.0517 | LR: 3.73e-04 | 694.5ms/step | 28.8 samples/s +Step 15320/44000 | Loss: 3.0900 | LR: 3.73e-04 | 696.4ms/step | 28.7 samples/s +Step 15330/44000 | Loss: 3.0680 | LR: 3.72e-04 | 693.8ms/step | 28.8 samples/s +Step 15340/44000 | Loss: 3.0414 | LR: 3.72e-04 | 694.9ms/step | 28.8 samples/s +Step 15350/44000 | Loss: 3.1013 | LR: 3.72e-04 | 695.8ms/step | 28.7 samples/s +Step 15360/44000 | Loss: 3.0382 | LR: 3.72e-04 | 693.7ms/step | 28.8 samples/s +Step 15370/44000 | Loss: 3.0502 | LR: 3.72e-04 | 692.8ms/step | 28.9 samples/s +Step 15380/44000 | Loss: 3.0810 | LR: 3.72e-04 | 693.3ms/step | 28.8 samples/s +Step 15390/44000 | Loss: 3.0607 | LR: 3.71e-04 | 694.4ms/step | 28.8 samples/s +Step 15400/44000 | Loss: 3.0427 | LR: 3.71e-04 | 693.1ms/step | 28.9 samples/s +Step 15410/44000 | Loss: 3.0531 | LR: 3.71e-04 | 693.5ms/step | 28.8 samples/s +Step 15420/44000 | Loss: 3.0818 | LR: 3.71e-04 | 693.4ms/step | 28.8 samples/s +Step 15430/44000 | Loss: 3.0662 | LR: 3.71e-04 | 693.3ms/step | 28.8 samples/s +Step 15440/44000 | Loss: 3.0612 | LR: 3.71e-04 | 694.8ms/step | 28.8 samples/s +Step 15450/44000 | Loss: 3.0604 | LR: 3.71e-04 | 694.2ms/step | 28.8 samples/s +Step 15460/44000 | Loss: 3.0054 | LR: 3.70e-04 | 696.3ms/step | 28.7 samples/s +Step 15470/44000 | Loss: 3.0426 | LR: 3.70e-04 | 693.5ms/step | 28.8 samples/s +Step 15480/44000 | Loss: 3.1065 | LR: 3.70e-04 | 696.2ms/step | 28.7 samples/s +Step 15490/44000 | Loss: 3.0498 | LR: 3.70e-04 | 694.5ms/step | 28.8 samples/s +Step 15500/44000 | Loss: 3.0787 | LR: 3.70e-04 | 693.8ms/step | 28.8 samples/s +Validation at step 15500: Loss = 3.0692, Perplexity = 21.52 +Step 15510/44000 | Loss: 3.0397 | LR: 3.70e-04 | 1269.1ms/step | 15.8 samples/s +Step 15520/44000 | Loss: 3.0417 | LR: 3.69e-04 | 695.4ms/step | 28.8 samples/s +Step 15530/44000 | Loss: 3.0841 | LR: 3.69e-04 | 693.4ms/step | 28.8 samples/s +Step 15540/44000 | Loss: 3.0257 | LR: 3.69e-04 | 695.0ms/step | 28.8 samples/s +Step 15550/44000 | Loss: 3.0386 | LR: 3.69e-04 | 692.3ms/step | 28.9 samples/s +Step 15560/44000 | Loss: 3.0613 | LR: 3.69e-04 | 694.3ms/step | 28.8 samples/s +Step 15570/44000 | Loss: 3.0687 | LR: 3.69e-04 | 693.8ms/step | 28.8 samples/s +Step 15580/44000 | Loss: 3.0409 | LR: 3.68e-04 | 693.6ms/step | 28.8 samples/s +Step 15590/44000 | Loss: 3.0967 | LR: 3.68e-04 | 694.6ms/step | 28.8 samples/s +Step 15600/44000 | Loss: 3.0373 | LR: 3.68e-04 | 695.8ms/step | 28.7 samples/s +Step 15610/44000 | Loss: 3.0552 | LR: 3.68e-04 | 693.3ms/step | 28.8 samples/s +Step 15620/44000 | Loss: 3.0397 | LR: 3.68e-04 | 693.7ms/step | 28.8 samples/s +Step 15630/44000 | Loss: 3.0245 | LR: 3.68e-04 | 693.0ms/step | 28.9 samples/s +Step 15640/44000 | Loss: 3.0362 | LR: 3.67e-04 | 693.2ms/step | 28.8 samples/s +Step 15650/44000 | Loss: 3.0322 | LR: 3.67e-04 | 694.3ms/step | 28.8 samples/s +Step 15660/44000 | Loss: 3.0712 | LR: 3.67e-04 | 692.3ms/step | 28.9 samples/s +Step 15670/44000 | Loss: 3.0798 | LR: 3.67e-04 | 695.1ms/step | 28.8 samples/s +Step 15680/44000 | Loss: 3.0526 | LR: 3.67e-04 | 693.0ms/step | 28.9 samples/s +Step 15690/44000 | Loss: 3.1075 | LR: 3.67e-04 | 692.3ms/step | 28.9 samples/s +Step 15700/44000 | Loss: 3.0326 | LR: 3.67e-04 | 693.8ms/step | 28.8 samples/s +Step 15710/44000 | Loss: 3.0778 | LR: 3.66e-04 | 693.3ms/step | 28.8 samples/s +Step 15720/44000 | Loss: 3.0367 | LR: 3.66e-04 | 693.6ms/step | 28.8 samples/s +Step 15730/44000 | Loss: 3.0588 | LR: 3.66e-04 | 692.2ms/step | 28.9 samples/s +Step 15740/44000 | Loss: 3.0197 | LR: 3.66e-04 | 691.5ms/step | 28.9 samples/s +Step 15750/44000 | Loss: 3.0133 | LR: 3.66e-04 | 692.3ms/step | 28.9 samples/s +Validation at step 15750: Loss = 3.0668, Perplexity = 21.47 +Step 15760/44000 | Loss: 3.0900 | LR: 3.66e-04 | 1265.0ms/step | 15.8 samples/s +Step 15770/44000 | Loss: 3.0424 | LR: 3.65e-04 | 693.0ms/step | 28.9 samples/s +Step 15780/44000 | Loss: 3.0821 | LR: 3.65e-04 | 691.8ms/step | 28.9 samples/s +Step 15790/44000 | Loss: 3.0206 | LR: 3.65e-04 | 692.3ms/step | 28.9 samples/s +Step 15800/44000 | Loss: 3.0496 | LR: 3.65e-04 | 691.4ms/step | 28.9 samples/s +Step 15810/44000 | Loss: 2.9902 | LR: 3.65e-04 | 691.2ms/step | 28.9 samples/s +Step 15820/44000 | Loss: 3.0371 | LR: 3.65e-04 | 692.1ms/step | 28.9 samples/s +Step 15830/44000 | Loss: 3.0244 | LR: 3.64e-04 | 692.5ms/step | 28.9 samples/s +Step 15840/44000 | Loss: 3.0156 | LR: 3.64e-04 | 694.2ms/step | 28.8 samples/s +Step 15850/44000 | Loss: 3.0846 | LR: 3.64e-04 | 693.6ms/step | 28.8 samples/s +Step 15860/44000 | Loss: 3.0481 | LR: 3.64e-04 | 691.4ms/step | 28.9 samples/s +Step 15870/44000 | Loss: 3.0545 | LR: 3.64e-04 | 692.5ms/step | 28.9 samples/s +Step 15880/44000 | Loss: 3.0975 | LR: 3.64e-04 | 691.6ms/step | 28.9 samples/s +Step 15890/44000 | Loss: 3.0698 | LR: 3.63e-04 | 693.4ms/step | 28.8 samples/s +Step 15900/44000 | Loss: 3.0829 | LR: 3.63e-04 | 692.0ms/step | 28.9 samples/s +Step 15910/44000 | Loss: 3.0878 | LR: 3.63e-04 | 691.8ms/step | 28.9 samples/s +Step 15920/44000 | Loss: 3.0573 | LR: 3.63e-04 | 692.0ms/step | 28.9 samples/s +Step 15930/44000 | Loss: 3.0908 | LR: 3.63e-04 | 691.9ms/step | 28.9 samples/s +Step 15940/44000 | Loss: 3.0634 | LR: 3.63e-04 | 692.0ms/step | 28.9 samples/s +Step 15950/44000 | Loss: 3.0100 | LR: 3.62e-04 | 693.0ms/step | 28.9 samples/s +Step 15960/44000 | Loss: 3.0546 | LR: 3.62e-04 | 694.3ms/step | 28.8 samples/s +Step 15970/44000 | Loss: 3.0364 | LR: 3.62e-04 | 693.8ms/step | 28.8 samples/s +Step 15980/44000 | Loss: 3.0596 | LR: 3.62e-04 | 694.1ms/step | 28.8 samples/s +Step 15990/44000 | Loss: 3.0652 | LR: 3.62e-04 | 691.8ms/step | 28.9 samples/s +Step 16000/44000 | Loss: 3.0951 | LR: 3.62e-04 | 692.8ms/step | 28.9 samples/s +Validation at step 16000: Loss = 3.0607, Perplexity = 21.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-16000 +Step 16010/44000 | Loss: 3.0645 | LR: 3.62e-04 | 1585.8ms/step | 12.6 samples/s +Step 16020/44000 | Loss: 3.0445 | LR: 3.61e-04 | 692.4ms/step | 28.9 samples/s +Step 16030/44000 | Loss: 3.0242 | LR: 3.61e-04 | 693.0ms/step | 28.9 samples/s +Step 16040/44000 | Loss: 3.0652 | LR: 3.61e-04 | 691.6ms/step | 28.9 samples/s +Step 16050/44000 | Loss: 2.9678 | LR: 3.61e-04 | 693.3ms/step | 28.8 samples/s +Step 16060/44000 | Loss: 3.0830 | LR: 3.61e-04 | 692.6ms/step | 28.9 samples/s +Step 16070/44000 | Loss: 3.0450 | LR: 3.61e-04 | 692.4ms/step | 28.9 samples/s +Step 16080/44000 | Loss: 3.0649 | LR: 3.60e-04 | 692.6ms/step | 28.9 samples/s +Step 16090/44000 | Loss: 3.0926 | LR: 3.60e-04 | 692.7ms/step | 28.9 samples/s +Step 16100/44000 | Loss: 3.0624 | LR: 3.60e-04 | 694.0ms/step | 28.8 samples/s +Step 16110/44000 | Loss: 3.0560 | LR: 3.60e-04 | 692.5ms/step | 28.9 samples/s +Step 16120/44000 | Loss: 3.0192 | LR: 3.60e-04 | 691.3ms/step | 28.9 samples/s +Step 16130/44000 | Loss: 3.0849 | LR: 3.60e-04 | 692.0ms/step | 28.9 samples/s +Step 16140/44000 | Loss: 2.9956 | LR: 3.59e-04 | 692.6ms/step | 28.9 samples/s +Step 16150/44000 | Loss: 3.0622 | LR: 3.59e-04 | 693.4ms/step | 28.8 samples/s +Step 16160/44000 | Loss: 3.0421 | LR: 3.59e-04 | 693.3ms/step | 28.8 samples/s +Step 16170/44000 | Loss: 3.0630 | LR: 3.59e-04 | 693.1ms/step | 28.9 samples/s +Step 16180/44000 | Loss: 3.0536 | LR: 3.59e-04 | 692.7ms/step | 28.9 samples/s +Step 16190/44000 | Loss: 3.0427 | LR: 3.59e-04 | 691.7ms/step | 28.9 samples/s +Step 16200/44000 | Loss: 3.0574 | LR: 3.58e-04 | 691.9ms/step | 28.9 samples/s +Step 16210/44000 | Loss: 3.0536 | LR: 3.58e-04 | 693.1ms/step | 28.9 samples/s +Step 16220/44000 | Loss: 3.0177 | LR: 3.58e-04 | 693.7ms/step | 28.8 samples/s +Step 16230/44000 | Loss: 3.0131 | LR: 3.58e-04 | 692.1ms/step | 28.9 samples/s +Step 16240/44000 | Loss: 3.0034 | LR: 3.58e-04 | 692.5ms/step | 28.9 samples/s +Step 16250/44000 | Loss: 3.0491 | LR: 3.58e-04 | 692.6ms/step | 28.9 samples/s +Validation at step 16250: Loss = 3.0572, Perplexity = 21.27 +Step 16260/44000 | Loss: 3.0553 | LR: 3.57e-04 | 1265.2ms/step | 15.8 samples/s +Step 16270/44000 | Loss: 3.0398 | LR: 3.57e-04 | 692.0ms/step | 28.9 samples/s +Step 16280/44000 | Loss: 3.0611 | LR: 3.57e-04 | 693.1ms/step | 28.9 samples/s +Step 16290/44000 | Loss: 3.0679 | LR: 3.57e-04 | 692.0ms/step | 28.9 samples/s +Step 16300/44000 | Loss: 3.0434 | LR: 3.57e-04 | 691.8ms/step | 28.9 samples/s +Step 16310/44000 | Loss: 3.0422 | LR: 3.57e-04 | 691.3ms/step | 28.9 samples/s +Step 16320/44000 | Loss: 3.0708 | LR: 3.56e-04 | 693.4ms/step | 28.8 samples/s +Step 16330/44000 | Loss: 3.0477 | LR: 3.56e-04 | 690.6ms/step | 29.0 samples/s +Step 16340/44000 | Loss: 3.0498 | LR: 3.56e-04 | 691.7ms/step | 28.9 samples/s +Step 16350/44000 | Loss: 3.0615 | LR: 3.56e-04 | 693.7ms/step | 28.8 samples/s +Step 16360/44000 | Loss: 3.1411 | LR: 3.56e-04 | 691.2ms/step | 28.9 samples/s +Step 16370/44000 | Loss: 3.0329 | LR: 3.56e-04 | 694.0ms/step | 28.8 samples/s +Step 16380/44000 | Loss: 3.0320 | LR: 3.55e-04 | 692.4ms/step | 28.9 samples/s +Step 16390/44000 | Loss: 3.0262 | LR: 3.55e-04 | 694.3ms/step | 28.8 samples/s +Step 16400/44000 | Loss: 3.0543 | LR: 3.55e-04 | 692.2ms/step | 28.9 samples/s +Step 16410/44000 | Loss: 3.0462 | LR: 3.55e-04 | 693.4ms/step | 28.8 samples/s +Step 16420/44000 | Loss: 3.0017 | LR: 3.55e-04 | 692.6ms/step | 28.9 samples/s +Step 16430/44000 | Loss: 3.0342 | LR: 3.55e-04 | 695.0ms/step | 28.8 samples/s +Step 16440/44000 | Loss: 3.0514 | LR: 3.54e-04 | 693.7ms/step | 28.8 samples/s +Step 16450/44000 | Loss: 3.0624 | LR: 3.54e-04 | 692.5ms/step | 28.9 samples/s +Step 16460/44000 | Loss: 3.0570 | LR: 3.54e-04 | 693.2ms/step | 28.9 samples/s +Step 16470/44000 | Loss: 3.0337 | LR: 3.54e-04 | 694.3ms/step | 28.8 samples/s +Step 16480/44000 | Loss: 3.0431 | LR: 3.54e-04 | 693.2ms/step | 28.9 samples/s +Step 16490/44000 | Loss: 3.0555 | LR: 3.54e-04 | 692.7ms/step | 28.9 samples/s +Step 16500/44000 | Loss: 3.0288 | LR: 3.53e-04 | 691.8ms/step | 28.9 samples/s +Validation at step 16500: Loss = 3.0553, Perplexity = 21.23 +Step 16510/44000 | Loss: 3.0323 | LR: 3.53e-04 | 1270.9ms/step | 15.7 samples/s +Step 16520/44000 | Loss: 3.0293 | LR: 3.53e-04 | 693.6ms/step | 28.8 samples/s +Step 16530/44000 | Loss: 3.0704 | LR: 3.53e-04 | 696.3ms/step | 28.7 samples/s +Step 16540/44000 | Loss: 3.0045 | LR: 3.53e-04 | 693.1ms/step | 28.9 samples/s +Step 16550/44000 | Loss: 3.0467 | LR: 3.53e-04 | 694.2ms/step | 28.8 samples/s +Step 16560/44000 | Loss: 3.0117 | LR: 3.52e-04 | 693.9ms/step | 28.8 samples/s +Step 16570/44000 | Loss: 3.0581 | LR: 3.52e-04 | 693.0ms/step | 28.9 samples/s +Step 16580/44000 | Loss: 3.0406 | LR: 3.52e-04 | 695.4ms/step | 28.8 samples/s +Step 16590/44000 | Loss: 3.0861 | LR: 3.52e-04 | 694.1ms/step | 28.8 samples/s +Step 16600/44000 | Loss: 3.0409 | LR: 3.52e-04 | 694.2ms/step | 28.8 samples/s +Step 16610/44000 | Loss: 3.0480 | LR: 3.52e-04 | 693.5ms/step | 28.8 samples/s +Step 16620/44000 | Loss: 3.0264 | LR: 3.51e-04 | 693.4ms/step | 28.8 samples/s +Step 16630/44000 | Loss: 3.0423 | LR: 3.51e-04 | 693.6ms/step | 28.8 samples/s +Step 16640/44000 | Loss: 3.0465 | LR: 3.51e-04 | 693.0ms/step | 28.9 samples/s +Step 16650/44000 | Loss: 3.0256 | LR: 3.51e-04 | 692.8ms/step | 28.9 samples/s +Step 16660/44000 | Loss: 3.0729 | LR: 3.51e-04 | 692.9ms/step | 28.9 samples/s +Step 16670/44000 | Loss: 3.0696 | LR: 3.51e-04 | 693.2ms/step | 28.8 samples/s +Step 16680/44000 | Loss: 3.0388 | LR: 3.50e-04 | 694.5ms/step | 28.8 samples/s +Step 16690/44000 | Loss: 3.0574 | LR: 3.50e-04 | 695.2ms/step | 28.8 samples/s +Step 16700/44000 | Loss: 3.0159 | LR: 3.50e-04 | 694.1ms/step | 28.8 samples/s +Step 16710/44000 | Loss: 3.0323 | LR: 3.50e-04 | 695.3ms/step | 28.8 samples/s +Step 16720/44000 | Loss: 3.0610 | LR: 3.50e-04 | 692.5ms/step | 28.9 samples/s +Step 16730/44000 | Loss: 3.0550 | LR: 3.50e-04 | 692.7ms/step | 28.9 samples/s +Step 16740/44000 | Loss: 3.0822 | LR: 3.49e-04 | 692.4ms/step | 28.9 samples/s +Step 16750/44000 | Loss: 3.0378 | LR: 3.49e-04 | 691.8ms/step | 28.9 samples/s +Validation at step 16750: Loss = 3.0523, Perplexity = 21.16 +Step 16760/44000 | Loss: 3.0301 | LR: 3.49e-04 | 1266.0ms/step | 15.8 samples/s +Step 16770/44000 | Loss: 3.0427 | LR: 3.49e-04 | 693.9ms/step | 28.8 samples/s +Step 16780/44000 | Loss: 3.0509 | LR: 3.49e-04 | 693.5ms/step | 28.8 samples/s +Step 16790/44000 | Loss: 3.0583 | LR: 3.49e-04 | 693.3ms/step | 28.8 samples/s +Step 16800/44000 | Loss: 3.0602 | LR: 3.48e-04 | 693.1ms/step | 28.9 samples/s +Step 16810/44000 | Loss: 3.0480 | LR: 3.48e-04 | 694.0ms/step | 28.8 samples/s +Step 16820/44000 | Loss: 3.0276 | LR: 3.48e-04 | 693.4ms/step | 28.8 samples/s +Step 16830/44000 | Loss: 3.0479 | LR: 3.48e-04 | 695.1ms/step | 28.8 samples/s +Step 16840/44000 | Loss: 3.0352 | LR: 3.48e-04 | 695.1ms/step | 28.8 samples/s +Step 16850/44000 | Loss: 3.0368 | LR: 3.48e-04 | 694.2ms/step | 28.8 samples/s +Step 16860/44000 | Loss: 3.0444 | LR: 3.47e-04 | 693.9ms/step | 28.8 samples/s +Step 16870/44000 | Loss: 3.0611 | LR: 3.47e-04 | 694.2ms/step | 28.8 samples/s +Step 16880/44000 | Loss: 3.0208 | LR: 3.47e-04 | 693.4ms/step | 28.8 samples/s +Step 16890/44000 | Loss: 3.0264 | LR: 3.47e-04 | 693.0ms/step | 28.9 samples/s +Step 16900/44000 | Loss: 3.0287 | LR: 3.47e-04 | 692.9ms/step | 28.9 samples/s +Step 16910/44000 | Loss: 3.0355 | LR: 3.47e-04 | 693.9ms/step | 28.8 samples/s +Step 16920/44000 | Loss: 3.0595 | LR: 3.46e-04 | 692.4ms/step | 28.9 samples/s +Step 16930/44000 | Loss: 3.0217 | LR: 3.46e-04 | 692.2ms/step | 28.9 samples/s +Step 16940/44000 | Loss: 3.0212 | LR: 3.46e-04 | 692.8ms/step | 28.9 samples/s +Step 16950/44000 | Loss: 3.0438 | LR: 3.46e-04 | 692.8ms/step | 28.9 samples/s +Step 16960/44000 | Loss: 3.0217 | LR: 3.46e-04 | 693.7ms/step | 28.8 samples/s +Step 16970/44000 | Loss: 3.0213 | LR: 3.46e-04 | 692.5ms/step | 28.9 samples/s +Step 16980/44000 | Loss: 3.0702 | LR: 3.45e-04 | 693.8ms/step | 28.8 samples/s +Step 16990/44000 | Loss: 3.0088 | LR: 3.45e-04 | 692.4ms/step | 28.9 samples/s +Step 17000/44000 | Loss: 3.0290 | LR: 3.45e-04 | 693.1ms/step | 28.9 samples/s +Validation at step 17000: Loss = 3.0482, Perplexity = 21.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-17000 +Step 17010/44000 | Loss: 3.0219 | LR: 3.45e-04 | 1631.2ms/step | 12.3 samples/s +Step 17020/44000 | Loss: 3.0850 | LR: 3.45e-04 | 693.7ms/step | 28.8 samples/s +Step 17030/44000 | Loss: 2.9867 | LR: 3.45e-04 | 694.4ms/step | 28.8 samples/s +Step 17040/44000 | Loss: 3.0049 | LR: 3.44e-04 | 693.9ms/step | 28.8 samples/s +Step 17050/44000 | Loss: 3.0149 | LR: 3.44e-04 | 695.4ms/step | 28.8 samples/s +Step 17060/44000 | Loss: 3.0279 | LR: 3.44e-04 | 695.1ms/step | 28.8 samples/s +Step 17070/44000 | Loss: 3.0118 | LR: 3.44e-04 | 693.4ms/step | 28.8 samples/s +Step 17080/44000 | Loss: 3.0693 | LR: 3.44e-04 | 693.3ms/step | 28.8 samples/s +Step 17090/44000 | Loss: 3.0423 | LR: 3.44e-04 | 694.0ms/step | 28.8 samples/s +Step 17100/44000 | Loss: 3.0341 | LR: 3.43e-04 | 695.0ms/step | 28.8 samples/s +Step 17110/44000 | Loss: 2.9708 | LR: 3.43e-04 | 693.6ms/step | 28.8 samples/s +Step 17120/44000 | Loss: 3.0678 | LR: 3.43e-04 | 692.9ms/step | 28.9 samples/s +Step 17130/44000 | Loss: 3.0424 | LR: 3.43e-04 | 693.3ms/step | 28.8 samples/s +Step 17140/44000 | Loss: 3.0364 | LR: 3.43e-04 | 693.6ms/step | 28.8 samples/s +Step 17150/44000 | Loss: 3.0157 | LR: 3.43e-04 | 692.4ms/step | 28.9 samples/s +Step 17160/44000 | Loss: 3.0448 | LR: 3.42e-04 | 694.9ms/step | 28.8 samples/s +Step 17170/44000 | Loss: 2.9945 | LR: 3.42e-04 | 692.6ms/step | 28.9 samples/s +Step 17180/44000 | Loss: 3.0200 | LR: 3.42e-04 | 693.0ms/step | 28.9 samples/s +Step 17190/44000 | Loss: 3.0079 | LR: 3.42e-04 | 693.0ms/step | 28.9 samples/s +Step 17200/44000 | Loss: 3.0084 | LR: 3.42e-04 | 693.2ms/step | 28.8 samples/s +Step 17210/44000 | Loss: 3.0310 | LR: 3.42e-04 | 694.3ms/step | 28.8 samples/s +Step 17220/44000 | Loss: 3.0691 | LR: 3.41e-04 | 692.7ms/step | 28.9 samples/s +Step 17230/44000 | Loss: 3.0280 | LR: 3.41e-04 | 694.5ms/step | 28.8 samples/s +Step 17240/44000 | Loss: 2.9800 | LR: 3.41e-04 | 693.5ms/step | 28.8 samples/s +Step 17250/44000 | Loss: 3.0104 | LR: 3.41e-04 | 693.0ms/step | 28.9 samples/s +Validation at step 17250: Loss = 3.0456, Perplexity = 21.02 +Step 17260/44000 | Loss: 3.0477 | LR: 3.41e-04 | 1265.2ms/step | 15.8 samples/s +Step 17270/44000 | Loss: 3.1103 | LR: 3.41e-04 | 693.2ms/step | 28.8 samples/s +Step 17280/44000 | Loss: 3.0355 | LR: 3.40e-04 | 693.3ms/step | 28.8 samples/s +Step 17290/44000 | Loss: 2.9829 | LR: 3.40e-04 | 695.1ms/step | 28.8 samples/s +Step 17300/44000 | Loss: 3.0571 | LR: 3.40e-04 | 695.8ms/step | 28.7 samples/s +Step 17310/44000 | Loss: 3.0427 | LR: 3.40e-04 | 693.9ms/step | 28.8 samples/s +Step 17320/44000 | Loss: 3.0468 | LR: 3.40e-04 | 694.8ms/step | 28.8 samples/s +Step 17330/44000 | Loss: 3.0134 | LR: 3.40e-04 | 693.1ms/step | 28.9 samples/s +Step 17340/44000 | Loss: 3.0521 | LR: 3.39e-04 | 693.2ms/step | 28.9 samples/s +Step 17350/44000 | Loss: 3.0088 | LR: 3.39e-04 | 694.1ms/step | 28.8 samples/s +Step 17360/44000 | Loss: 3.0823 | LR: 3.39e-04 | 692.3ms/step | 28.9 samples/s +Step 17370/44000 | Loss: 3.0105 | LR: 3.39e-04 | 693.7ms/step | 28.8 samples/s +Step 17380/44000 | Loss: 2.9955 | LR: 3.39e-04 | 692.2ms/step | 28.9 samples/s +Step 17390/44000 | Loss: 3.0567 | LR: 3.39e-04 | 693.8ms/step | 28.8 samples/s +Step 17400/44000 | Loss: 3.0720 | LR: 3.38e-04 | 694.0ms/step | 28.8 samples/s +Step 17410/44000 | Loss: 2.9737 | LR: 3.38e-04 | 695.0ms/step | 28.8 samples/s +Step 17420/44000 | Loss: 3.0350 | LR: 3.38e-04 | 693.8ms/step | 28.8 samples/s +Step 17430/44000 | Loss: 3.0132 | LR: 3.38e-04 | 694.5ms/step | 28.8 samples/s +Step 17440/44000 | Loss: 3.0697 | LR: 3.38e-04 | 694.8ms/step | 28.8 samples/s +Step 17450/44000 | Loss: 3.0384 | LR: 3.38e-04 | 694.5ms/step | 28.8 samples/s +Step 17460/44000 | Loss: 3.0467 | LR: 3.37e-04 | 694.7ms/step | 28.8 samples/s +Step 17470/44000 | Loss: 3.0120 | LR: 3.37e-04 | 694.4ms/step | 28.8 samples/s +Step 17480/44000 | Loss: 3.0901 | LR: 3.37e-04 | 695.9ms/step | 28.7 samples/s +Step 17490/44000 | Loss: 3.0406 | LR: 3.37e-04 | 693.2ms/step | 28.9 samples/s +Step 17500/44000 | Loss: 3.0112 | LR: 3.37e-04 | 693.4ms/step | 28.8 samples/s +Validation at step 17500: Loss = 3.0426, Perplexity = 20.96 +Step 17510/44000 | Loss: 3.0055 | LR: 3.37e-04 | 1263.3ms/step | 15.8 samples/s +Step 17520/44000 | Loss: 3.0671 | LR: 3.36e-04 | 694.3ms/step | 28.8 samples/s +Step 17530/44000 | Loss: 3.0095 | LR: 3.36e-04 | 694.8ms/step | 28.8 samples/s +Step 17540/44000 | Loss: 3.0410 | LR: 3.36e-04 | 693.3ms/step | 28.8 samples/s +Step 17550/44000 | Loss: 3.0393 | LR: 3.36e-04 | 694.3ms/step | 28.8 samples/s +Step 17560/44000 | Loss: 3.0070 | LR: 3.36e-04 | 694.7ms/step | 28.8 samples/s +Step 17570/44000 | Loss: 3.0528 | LR: 3.35e-04 | 694.3ms/step | 28.8 samples/s +Step 17580/44000 | Loss: 3.0345 | LR: 3.35e-04 | 696.7ms/step | 28.7 samples/s +Step 17590/44000 | Loss: 2.9736 | LR: 3.35e-04 | 695.5ms/step | 28.8 samples/s +Step 17600/44000 | Loss: 3.0473 | LR: 3.35e-04 | 695.3ms/step | 28.8 samples/s +Step 17610/44000 | Loss: 3.0160 | LR: 3.35e-04 | 696.5ms/step | 28.7 samples/s +Step 17620/44000 | Loss: 3.0400 | LR: 3.35e-04 | 695.2ms/step | 28.8 samples/s +Step 17630/44000 | Loss: 3.0165 | LR: 3.34e-04 | 694.8ms/step | 28.8 samples/s +Step 17640/44000 | Loss: 2.9904 | LR: 3.34e-04 | 695.9ms/step | 28.7 samples/s +Step 17650/44000 | Loss: 3.0569 | LR: 3.34e-04 | 694.1ms/step | 28.8 samples/s +Step 17660/44000 | Loss: 2.9856 | LR: 3.34e-04 | 697.0ms/step | 28.7 samples/s +Step 17670/44000 | Loss: 3.0592 | LR: 3.34e-04 | 695.1ms/step | 28.8 samples/s +Step 17680/44000 | Loss: 3.0329 | LR: 3.34e-04 | 695.7ms/step | 28.7 samples/s +Step 17690/44000 | Loss: 3.0508 | LR: 3.33e-04 | 694.6ms/step | 28.8 samples/s +Step 17700/44000 | Loss: 3.0043 | LR: 3.33e-04 | 696.7ms/step | 28.7 samples/s +Step 17710/44000 | Loss: 2.9730 | LR: 3.33e-04 | 694.1ms/step | 28.8 samples/s +Step 17720/44000 | Loss: 3.0085 | LR: 3.33e-04 | 694.5ms/step | 28.8 samples/s +Step 17730/44000 | Loss: 3.0339 | LR: 3.33e-04 | 694.5ms/step | 28.8 samples/s +Step 17740/44000 | Loss: 3.0030 | LR: 3.33e-04 | 695.3ms/step | 28.8 samples/s +Step 17750/44000 | Loss: 2.9958 | LR: 3.32e-04 | 696.4ms/step | 28.7 samples/s +Validation at step 17750: Loss = 3.0401, Perplexity = 20.91 +Step 17760/44000 | Loss: 3.0330 | LR: 3.32e-04 | 1265.6ms/step | 15.8 samples/s +Step 17770/44000 | Loss: 3.0166 | LR: 3.32e-04 | 694.7ms/step | 28.8 samples/s +Step 17780/44000 | Loss: 3.0518 | LR: 3.32e-04 | 696.5ms/step | 28.7 samples/s +Step 17790/44000 | Loss: 3.0689 | LR: 3.32e-04 | 694.9ms/step | 28.8 samples/s +Step 17800/44000 | Loss: 3.0321 | LR: 3.32e-04 | 696.3ms/step | 28.7 samples/s +Step 17810/44000 | Loss: 3.0045 | LR: 3.31e-04 | 695.5ms/step | 28.8 samples/s +Step 17820/44000 | Loss: 3.0244 | LR: 3.31e-04 | 694.8ms/step | 28.8 samples/s +Step 17830/44000 | Loss: 3.0634 | LR: 3.31e-04 | 695.1ms/step | 28.8 samples/s +Step 17840/44000 | Loss: 3.0136 | LR: 3.31e-04 | 695.9ms/step | 28.7 samples/s +Step 17850/44000 | Loss: 3.0514 | LR: 3.31e-04 | 695.9ms/step | 28.7 samples/s +Step 17860/44000 | Loss: 3.0375 | LR: 3.31e-04 | 693.4ms/step | 28.8 samples/s +Step 17870/44000 | Loss: 3.0389 | LR: 3.30e-04 | 694.3ms/step | 28.8 samples/s +Step 17880/44000 | Loss: 3.0477 | LR: 3.30e-04 | 694.2ms/step | 28.8 samples/s +Step 17890/44000 | Loss: 3.0015 | LR: 3.30e-04 | 693.9ms/step | 28.8 samples/s +Step 17900/44000 | Loss: 3.0549 | LR: 3.30e-04 | 693.3ms/step | 28.8 samples/s +Step 17910/44000 | Loss: 3.0254 | LR: 3.30e-04 | 693.8ms/step | 28.8 samples/s +Step 17920/44000 | Loss: 3.0469 | LR: 3.29e-04 | 698.2ms/step | 28.6 samples/s +Step 17930/44000 | Loss: 2.9718 | LR: 3.29e-04 | 693.2ms/step | 28.8 samples/s +Step 17940/44000 | Loss: 3.0303 | LR: 3.29e-04 | 692.9ms/step | 28.9 samples/s +Step 17950/44000 | Loss: 3.0565 | LR: 3.29e-04 | 693.9ms/step | 28.8 samples/s +Step 17960/44000 | Loss: 2.9976 | LR: 3.29e-04 | 692.6ms/step | 28.9 samples/s +Step 17970/44000 | Loss: 3.0162 | LR: 3.29e-04 | 693.2ms/step | 28.9 samples/s +Step 17980/44000 | Loss: 2.9756 | LR: 3.28e-04 | 695.0ms/step | 28.8 samples/s +Step 17990/44000 | Loss: 3.0363 | LR: 3.28e-04 | 696.3ms/step | 28.7 samples/s +Step 18000/44000 | Loss: 3.0257 | LR: 3.28e-04 | 694.9ms/step | 28.8 samples/s +Validation at step 18000: Loss = 3.0374, Perplexity = 20.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-18000 +Step 18010/44000 | Loss: 2.9848 | LR: 3.28e-04 | 1608.9ms/step | 12.4 samples/s +Step 18020/44000 | Loss: 3.0159 | LR: 3.28e-04 | 692.7ms/step | 28.9 samples/s +Step 18030/44000 | Loss: 3.0500 | LR: 3.28e-04 | 692.7ms/step | 28.9 samples/s +Step 18040/44000 | Loss: 3.0153 | LR: 3.27e-04 | 694.7ms/step | 28.8 samples/s +Step 18050/44000 | Loss: 3.0286 | LR: 3.27e-04 | 694.4ms/step | 28.8 samples/s +Step 18060/44000 | Loss: 3.0481 | LR: 3.27e-04 | 694.4ms/step | 28.8 samples/s +Step 18070/44000 | Loss: 3.0492 | LR: 3.27e-04 | 693.6ms/step | 28.8 samples/s +Step 18080/44000 | Loss: 3.0398 | LR: 3.27e-04 | 694.8ms/step | 28.8 samples/s +Step 18090/44000 | Loss: 3.0387 | LR: 3.27e-04 | 693.7ms/step | 28.8 samples/s +Step 18100/44000 | Loss: 3.0057 | LR: 3.26e-04 | 693.9ms/step | 28.8 samples/s +Step 18110/44000 | Loss: 3.0237 | LR: 3.26e-04 | 696.0ms/step | 28.7 samples/s +Step 18120/44000 | Loss: 3.0490 | LR: 3.26e-04 | 693.5ms/step | 28.8 samples/s +Step 18130/44000 | Loss: 3.0448 | LR: 3.26e-04 | 693.8ms/step | 28.8 samples/s +Step 18140/44000 | Loss: 3.0581 | LR: 3.26e-04 | 693.6ms/step | 28.8 samples/s +Step 18150/44000 | Loss: 2.9903 | LR: 3.26e-04 | 693.2ms/step | 28.9 samples/s +Step 18160/44000 | Loss: 3.0367 | LR: 3.25e-04 | 694.5ms/step | 28.8 samples/s +Step 18170/44000 | Loss: 2.9452 | LR: 3.25e-04 | 696.4ms/step | 28.7 samples/s +Step 18180/44000 | Loss: 3.0421 | LR: 3.25e-04 | 694.6ms/step | 28.8 samples/s +Step 18190/44000 | Loss: 3.0032 | LR: 3.25e-04 | 696.5ms/step | 28.7 samples/s +Step 18200/44000 | Loss: 2.9907 | LR: 3.25e-04 | 695.4ms/step | 28.8 samples/s +Step 18210/44000 | Loss: 3.0405 | LR: 3.24e-04 | 694.8ms/step | 28.8 samples/s +Step 18220/44000 | Loss: 3.0189 | LR: 3.24e-04 | 695.2ms/step | 28.8 samples/s +Step 18230/44000 | Loss: 3.0371 | LR: 3.24e-04 | 694.2ms/step | 28.8 samples/s +Step 18240/44000 | Loss: 3.0060 | LR: 3.24e-04 | 694.7ms/step | 28.8 samples/s +Step 18250/44000 | Loss: 3.0349 | LR: 3.24e-04 | 695.7ms/step | 28.7 samples/s +Validation at step 18250: Loss = 3.0338, Perplexity = 20.78 +Step 18260/44000 | Loss: 3.0301 | LR: 3.24e-04 | 1266.3ms/step | 15.8 samples/s +Step 18270/44000 | Loss: 2.9991 | LR: 3.23e-04 | 694.9ms/step | 28.8 samples/s +Step 18280/44000 | Loss: 3.0137 | LR: 3.23e-04 | 695.0ms/step | 28.8 samples/s +Step 18290/44000 | Loss: 3.0306 | LR: 3.23e-04 | 694.6ms/step | 28.8 samples/s +Step 18300/44000 | Loss: 3.0038 | LR: 3.23e-04 | 697.0ms/step | 28.7 samples/s +Step 18310/44000 | Loss: 2.9992 | LR: 3.23e-04 | 694.1ms/step | 28.8 samples/s +Step 18320/44000 | Loss: 3.0445 | LR: 3.23e-04 | 695.6ms/step | 28.8 samples/s +Step 18330/44000 | Loss: 2.9683 | LR: 3.22e-04 | 696.1ms/step | 28.7 samples/s +Step 18340/44000 | Loss: 3.0169 | LR: 3.22e-04 | 694.2ms/step | 28.8 samples/s +Step 18350/44000 | Loss: 3.0664 | LR: 3.22e-04 | 693.4ms/step | 28.8 samples/s +Step 18360/44000 | Loss: 3.0115 | LR: 3.22e-04 | 694.7ms/step | 28.8 samples/s +Step 18370/44000 | Loss: 3.0118 | LR: 3.22e-04 | 695.0ms/step | 28.8 samples/s +Step 18380/44000 | Loss: 2.9554 | LR: 3.22e-04 | 695.1ms/step | 28.8 samples/s +Step 18390/44000 | Loss: 3.0228 | LR: 3.21e-04 | 695.2ms/step | 28.8 samples/s +Step 18400/44000 | Loss: 3.0779 | LR: 3.21e-04 | 693.8ms/step | 28.8 samples/s +Step 18410/44000 | Loss: 3.0382 | LR: 3.21e-04 | 694.4ms/step | 28.8 samples/s +Step 18420/44000 | Loss: 3.0146 | LR: 3.21e-04 | 694.1ms/step | 28.8 samples/s +Step 18430/44000 | Loss: 3.0138 | LR: 3.21e-04 | 694.1ms/step | 28.8 samples/s +Step 18440/44000 | Loss: 3.0085 | LR: 3.20e-04 | 694.1ms/step | 28.8 samples/s +Step 18450/44000 | Loss: 2.9539 | LR: 3.20e-04 | 693.3ms/step | 28.8 samples/s +Step 18460/44000 | Loss: 3.0286 | LR: 3.20e-04 | 694.8ms/step | 28.8 samples/s +Step 18470/44000 | Loss: 3.0322 | LR: 3.20e-04 | 695.4ms/step | 28.8 samples/s +Step 18480/44000 | Loss: 3.0174 | LR: 3.20e-04 | 694.2ms/step | 28.8 samples/s +Step 18490/44000 | Loss: 3.0309 | LR: 3.20e-04 | 694.8ms/step | 28.8 samples/s +Step 18500/44000 | Loss: 3.0014 | LR: 3.19e-04 | 695.7ms/step | 28.7 samples/s +Validation at step 18500: Loss = 3.0323, Perplexity = 20.74 +Step 18510/44000 | Loss: 3.0550 | LR: 3.19e-04 | 1265.0ms/step | 15.8 samples/s +Step 18520/44000 | Loss: 3.0197 | LR: 3.19e-04 | 693.1ms/step | 28.9 samples/s +Step 18530/44000 | Loss: 3.0460 | LR: 3.19e-04 | 693.2ms/step | 28.9 samples/s +Step 18540/44000 | Loss: 3.0079 | LR: 3.19e-04 | 695.0ms/step | 28.8 samples/s +Step 18550/44000 | Loss: 2.9994 | LR: 3.19e-04 | 694.7ms/step | 28.8 samples/s +Step 18560/44000 | Loss: 3.0511 | LR: 3.18e-04 | 695.2ms/step | 28.8 samples/s +Step 18570/44000 | Loss: 3.0269 | LR: 3.18e-04 | 693.2ms/step | 28.9 samples/s +Step 18580/44000 | Loss: 3.0486 | LR: 3.18e-04 | 694.8ms/step | 28.8 samples/s +Step 18590/44000 | Loss: 3.0131 | LR: 3.18e-04 | 694.4ms/step | 28.8 samples/s +Step 18600/44000 | Loss: 2.9881 | LR: 3.18e-04 | 693.1ms/step | 28.9 samples/s +Step 18610/44000 | Loss: 3.0203 | LR: 3.18e-04 | 694.7ms/step | 28.8 samples/s +Step 18620/44000 | Loss: 3.0123 | LR: 3.17e-04 | 694.2ms/step | 28.8 samples/s +Step 18630/44000 | Loss: 2.9974 | LR: 3.17e-04 | 694.4ms/step | 28.8 samples/s +Step 18640/44000 | Loss: 3.0540 | LR: 3.17e-04 | 693.9ms/step | 28.8 samples/s +Step 18650/44000 | Loss: 2.9917 | LR: 3.17e-04 | 694.0ms/step | 28.8 samples/s +Step 18660/44000 | Loss: 3.0033 | LR: 3.17e-04 | 692.4ms/step | 28.9 samples/s +Step 18670/44000 | Loss: 2.9943 | LR: 3.16e-04 | 695.6ms/step | 28.8 samples/s +Step 18680/44000 | Loss: 3.0255 | LR: 3.16e-04 | 694.5ms/step | 28.8 samples/s +Step 18690/44000 | Loss: 3.0090 | LR: 3.16e-04 | 694.6ms/step | 28.8 samples/s +Step 18700/44000 | Loss: 3.0185 | LR: 3.16e-04 | 698.1ms/step | 28.6 samples/s +Step 18710/44000 | Loss: 3.0149 | LR: 3.16e-04 | 694.1ms/step | 28.8 samples/s +Step 18720/44000 | Loss: 3.0449 | LR: 3.16e-04 | 694.5ms/step | 28.8 samples/s +Step 18730/44000 | Loss: 3.0482 | LR: 3.15e-04 | 693.6ms/step | 28.8 samples/s +Step 18740/44000 | Loss: 2.9921 | LR: 3.15e-04 | 697.4ms/step | 28.7 samples/s +Step 18750/44000 | Loss: 2.9874 | LR: 3.15e-04 | 694.4ms/step | 28.8 samples/s +Validation at step 18750: Loss = 3.0282, Perplexity = 20.66 +Step 18760/44000 | Loss: 3.0194 | LR: 3.15e-04 | 1267.0ms/step | 15.8 samples/s +Step 18770/44000 | Loss: 3.0104 | LR: 3.15e-04 | 695.2ms/step | 28.8 samples/s +Step 18780/44000 | Loss: 3.0242 | LR: 3.15e-04 | 695.9ms/step | 28.7 samples/s +Step 18790/44000 | Loss: 2.9807 | LR: 3.14e-04 | 694.1ms/step | 28.8 samples/s +Step 18800/44000 | Loss: 3.0124 | LR: 3.14e-04 | 694.3ms/step | 28.8 samples/s +Step 18810/44000 | Loss: 3.0105 | LR: 3.14e-04 | 693.9ms/step | 28.8 samples/s +Step 18820/44000 | Loss: 3.0083 | LR: 3.14e-04 | 695.5ms/step | 28.8 samples/s +Step 18830/44000 | Loss: 2.9862 | LR: 3.14e-04 | 694.7ms/step | 28.8 samples/s +Step 18840/44000 | Loss: 3.0248 | LR: 3.13e-04 | 694.3ms/step | 28.8 samples/s +Step 18850/44000 | Loss: 3.0054 | LR: 3.13e-04 | 694.7ms/step | 28.8 samples/s +Step 18860/44000 | Loss: 2.9766 | LR: 3.13e-04 | 695.5ms/step | 28.8 samples/s +Step 18870/44000 | Loss: 2.9894 | LR: 3.13e-04 | 695.8ms/step | 28.7 samples/s +Step 18880/44000 | Loss: 3.0006 | LR: 3.13e-04 | 695.5ms/step | 28.8 samples/s +Step 18890/44000 | Loss: 2.9973 | LR: 3.13e-04 | 695.7ms/step | 28.7 samples/s +Step 18900/44000 | Loss: 2.9931 | LR: 3.12e-04 | 695.6ms/step | 28.8 samples/s +Step 18910/44000 | Loss: 3.0390 | LR: 3.12e-04 | 695.3ms/step | 28.8 samples/s +Step 18920/44000 | Loss: 3.0565 | LR: 3.12e-04 | 694.9ms/step | 28.8 samples/s +Step 18930/44000 | Loss: 3.0208 | LR: 3.12e-04 | 694.8ms/step | 28.8 samples/s +Step 18940/44000 | Loss: 2.9808 | LR: 3.12e-04 | 694.2ms/step | 28.8 samples/s +Step 18950/44000 | Loss: 2.9847 | LR: 3.12e-04 | 693.4ms/step | 28.8 samples/s +Step 18960/44000 | Loss: 2.9931 | LR: 3.11e-04 | 693.2ms/step | 28.9 samples/s +Step 18970/44000 | Loss: 2.9736 | LR: 3.11e-04 | 694.3ms/step | 28.8 samples/s +Step 18980/44000 | Loss: 3.0570 | LR: 3.11e-04 | 693.6ms/step | 28.8 samples/s +Step 18990/44000 | Loss: 3.0012 | LR: 3.11e-04 | 694.3ms/step | 28.8 samples/s +Step 19000/44000 | Loss: 3.0153 | LR: 3.11e-04 | 694.6ms/step | 28.8 samples/s +Validation at step 19000: Loss = 3.0285, Perplexity = 20.67 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-19000 +Step 19010/44000 | Loss: 3.0220 | LR: 3.11e-04 | 1621.0ms/step | 12.3 samples/s +Step 19020/44000 | Loss: 2.9967 | LR: 3.10e-04 | 697.4ms/step | 28.7 samples/s +Step 19030/44000 | Loss: 3.0049 | LR: 3.10e-04 | 696.6ms/step | 28.7 samples/s +Step 19040/44000 | Loss: 3.0142 | LR: 3.10e-04 | 695.2ms/step | 28.8 samples/s +Step 19050/44000 | Loss: 2.9651 | LR: 3.10e-04 | 695.1ms/step | 28.8 samples/s +Step 19060/44000 | Loss: 3.0459 | LR: 3.10e-04 | 694.5ms/step | 28.8 samples/s +Step 19070/44000 | Loss: 3.0105 | LR: 3.09e-04 | 695.3ms/step | 28.8 samples/s +Step 19080/44000 | Loss: 3.0273 | LR: 3.09e-04 | 695.5ms/step | 28.8 samples/s +Step 19090/44000 | Loss: 3.0407 | LR: 3.09e-04 | 695.1ms/step | 28.8 samples/s +Step 19100/44000 | Loss: 2.9930 | LR: 3.09e-04 | 696.1ms/step | 28.7 samples/s +Step 19110/44000 | Loss: 3.0154 | LR: 3.09e-04 | 695.0ms/step | 28.8 samples/s +Step 19120/44000 | Loss: 3.0356 | LR: 3.09e-04 | 695.8ms/step | 28.7 samples/s +Step 19130/44000 | Loss: 3.0028 | LR: 3.08e-04 | 696.0ms/step | 28.7 samples/s +Step 19140/44000 | Loss: 3.0038 | LR: 3.08e-04 | 695.9ms/step | 28.7 samples/s +Step 19150/44000 | Loss: 2.9828 | LR: 3.08e-04 | 692.9ms/step | 28.9 samples/s +Step 19160/44000 | Loss: 2.9922 | LR: 3.08e-04 | 693.1ms/step | 28.9 samples/s +Step 19170/44000 | Loss: 2.9972 | LR: 3.08e-04 | 694.5ms/step | 28.8 samples/s +Step 19180/44000 | Loss: 3.0220 | LR: 3.08e-04 | 693.9ms/step | 28.8 samples/s +Step 19190/44000 | Loss: 3.0370 | LR: 3.07e-04 | 693.2ms/step | 28.9 samples/s +Step 19200/44000 | Loss: 3.0450 | LR: 3.07e-04 | 694.1ms/step | 28.8 samples/s +Step 19210/44000 | Loss: 3.0171 | LR: 3.07e-04 | 693.5ms/step | 28.8 samples/s +Step 19220/44000 | Loss: 3.0213 | LR: 3.07e-04 | 693.2ms/step | 28.9 samples/s +Step 19230/44000 | Loss: 3.0241 | LR: 3.07e-04 | 692.1ms/step | 28.9 samples/s +Step 19240/44000 | Loss: 3.0192 | LR: 3.06e-04 | 692.5ms/step | 28.9 samples/s +Step 19250/44000 | Loss: 3.0498 | LR: 3.06e-04 | 693.3ms/step | 28.8 samples/s +Validation at step 19250: Loss = 3.0229, Perplexity = 20.55 +Step 19260/44000 | Loss: 3.0303 | LR: 3.06e-04 | 1261.8ms/step | 15.8 samples/s +Step 19270/44000 | Loss: 3.0152 | LR: 3.06e-04 | 693.3ms/step | 28.8 samples/s +Step 19280/44000 | Loss: 2.9725 | LR: 3.06e-04 | 693.4ms/step | 28.8 samples/s +Step 19290/44000 | Loss: 2.9810 | LR: 3.06e-04 | 693.7ms/step | 28.8 samples/s +Step 19300/44000 | Loss: 2.9990 | LR: 3.05e-04 | 693.9ms/step | 28.8 samples/s +Step 19310/44000 | Loss: 3.0374 | LR: 3.05e-04 | 693.5ms/step | 28.8 samples/s +Step 19320/44000 | Loss: 3.0837 | LR: 3.05e-04 | 694.2ms/step | 28.8 samples/s +Step 19330/44000 | Loss: 3.0067 | LR: 3.05e-04 | 693.4ms/step | 28.8 samples/s +Step 19340/44000 | Loss: 3.0218 | LR: 3.05e-04 | 693.3ms/step | 28.8 samples/s +Step 19350/44000 | Loss: 2.9811 | LR: 3.04e-04 | 691.7ms/step | 28.9 samples/s +Step 19360/44000 | Loss: 3.0050 | LR: 3.04e-04 | 694.0ms/step | 28.8 samples/s +Step 19370/44000 | Loss: 2.9840 | LR: 3.04e-04 | 692.4ms/step | 28.9 samples/s +Step 19380/44000 | Loss: 2.9753 | LR: 3.04e-04 | 694.0ms/step | 28.8 samples/s +Step 19390/44000 | Loss: 3.0492 | LR: 3.04e-04 | 693.2ms/step | 28.9 samples/s +Step 19400/44000 | Loss: 3.0056 | LR: 3.04e-04 | 693.8ms/step | 28.8 samples/s +Step 19410/44000 | Loss: 3.0095 | LR: 3.03e-04 | 693.1ms/step | 28.9 samples/s +Step 19420/44000 | Loss: 2.9930 | LR: 3.03e-04 | 692.3ms/step | 28.9 samples/s +Step 19430/44000 | Loss: 3.0393 | LR: 3.03e-04 | 693.7ms/step | 28.8 samples/s +Step 19440/44000 | Loss: 3.0246 | LR: 3.03e-04 | 692.9ms/step | 28.9 samples/s +Step 19450/44000 | Loss: 2.9874 | LR: 3.03e-04 | 693.8ms/step | 28.8 samples/s +Step 19460/44000 | Loss: 3.0362 | LR: 3.03e-04 | 692.7ms/step | 28.9 samples/s +Step 19470/44000 | Loss: 2.9464 | LR: 3.02e-04 | 693.2ms/step | 28.9 samples/s +Step 19480/44000 | Loss: 3.0273 | LR: 3.02e-04 | 692.9ms/step | 28.9 samples/s +Step 19490/44000 | Loss: 3.0321 | LR: 3.02e-04 | 695.3ms/step | 28.8 samples/s +Step 19500/44000 | Loss: 3.0014 | LR: 3.02e-04 | 694.4ms/step | 28.8 samples/s +Validation at step 19500: Loss = 3.0200, Perplexity = 20.49 +Step 19510/44000 | Loss: 3.0542 | LR: 3.02e-04 | 1265.9ms/step | 15.8 samples/s +Step 19520/44000 | Loss: 3.0437 | LR: 3.01e-04 | 694.9ms/step | 28.8 samples/s +Step 19530/44000 | Loss: 2.9671 | LR: 3.01e-04 | 694.2ms/step | 28.8 samples/s +Step 19540/44000 | Loss: 3.0350 | LR: 3.01e-04 | 692.0ms/step | 28.9 samples/s +Step 19550/44000 | Loss: 3.0363 | LR: 3.01e-04 | 692.7ms/step | 28.9 samples/s +Step 19560/44000 | Loss: 3.0284 | LR: 3.01e-04 | 692.8ms/step | 28.9 samples/s +Step 19570/44000 | Loss: 2.9910 | LR: 3.01e-04 | 692.5ms/step | 28.9 samples/s +Step 19580/44000 | Loss: 3.0116 | LR: 3.00e-04 | 693.7ms/step | 28.8 samples/s +Step 19590/44000 | Loss: 3.0109 | LR: 3.00e-04 | 693.8ms/step | 28.8 samples/s +Step 19600/44000 | Loss: 3.0376 | LR: 3.00e-04 | 694.3ms/step | 28.8 samples/s +Step 19610/44000 | Loss: 3.0065 | LR: 3.00e-04 | 695.4ms/step | 28.8 samples/s +Step 19620/44000 | Loss: 3.0224 | LR: 3.00e-04 | 693.5ms/step | 28.8 samples/s +Step 19630/44000 | Loss: 2.9785 | LR: 3.00e-04 | 692.9ms/step | 28.9 samples/s +Step 19640/44000 | Loss: 3.0137 | LR: 2.99e-04 | 691.1ms/step | 28.9 samples/s +Step 19650/44000 | Loss: 3.0223 | LR: 2.99e-04 | 692.5ms/step | 28.9 samples/s +Step 19660/44000 | Loss: 3.0028 | LR: 2.99e-04 | 692.0ms/step | 28.9 samples/s +Step 19670/44000 | Loss: 3.0567 | LR: 2.99e-04 | 691.3ms/step | 28.9 samples/s +Step 19680/44000 | Loss: 3.0110 | LR: 2.99e-04 | 692.7ms/step | 28.9 samples/s +Step 19690/44000 | Loss: 3.0048 | LR: 2.98e-04 | 691.6ms/step | 28.9 samples/s +Step 19700/44000 | Loss: 2.9864 | LR: 2.98e-04 | 691.6ms/step | 28.9 samples/s +Step 19710/44000 | Loss: 3.0253 | LR: 2.98e-04 | 692.5ms/step | 28.9 samples/s +Step 19720/44000 | Loss: 3.0251 | LR: 2.98e-04 | 691.5ms/step | 28.9 samples/s +Step 19730/44000 | Loss: 2.9987 | LR: 2.98e-04 | 691.8ms/step | 28.9 samples/s +Step 19740/44000 | Loss: 2.9685 | LR: 2.98e-04 | 692.4ms/step | 28.9 samples/s +Step 19750/44000 | Loss: 3.0028 | LR: 2.97e-04 | 691.7ms/step | 28.9 samples/s +Validation at step 19750: Loss = 3.0171, Perplexity = 20.43 +Step 19760/44000 | Loss: 2.9865 | LR: 2.97e-04 | 1269.1ms/step | 15.8 samples/s +Step 19770/44000 | Loss: 3.0115 | LR: 2.97e-04 | 692.6ms/step | 28.9 samples/s +Step 19780/44000 | Loss: 3.0336 | LR: 2.97e-04 | 692.7ms/step | 28.9 samples/s +Step 19790/44000 | Loss: 2.9978 | LR: 2.97e-04 | 693.9ms/step | 28.8 samples/s +Step 19800/44000 | Loss: 2.9900 | LR: 2.96e-04 | 692.7ms/step | 28.9 samples/s +Step 19810/44000 | Loss: 3.0101 | LR: 2.96e-04 | 693.1ms/step | 28.9 samples/s +Step 19820/44000 | Loss: 3.0254 | LR: 2.96e-04 | 692.9ms/step | 28.9 samples/s +Step 19830/44000 | Loss: 3.0227 | LR: 2.96e-04 | 693.9ms/step | 28.8 samples/s +Step 19840/44000 | Loss: 2.9428 | LR: 2.96e-04 | 694.6ms/step | 28.8 samples/s +Step 19850/44000 | Loss: 3.0032 | LR: 2.96e-04 | 692.8ms/step | 28.9 samples/s +Step 19860/44000 | Loss: 3.0153 | LR: 2.95e-04 | 692.9ms/step | 28.9 samples/s +Step 19870/44000 | Loss: 2.9698 | LR: 2.95e-04 | 692.4ms/step | 28.9 samples/s +Step 19880/44000 | Loss: 3.0306 | LR: 2.95e-04 | 691.3ms/step | 28.9 samples/s +Step 19890/44000 | Loss: 3.0058 | LR: 2.95e-04 | 691.9ms/step | 28.9 samples/s +Step 19900/44000 | Loss: 2.9810 | LR: 2.95e-04 | 693.1ms/step | 28.9 samples/s +Step 19910/44000 | Loss: 3.0156 | LR: 2.95e-04 | 692.2ms/step | 28.9 samples/s +Step 19920/44000 | Loss: 2.9931 | LR: 2.94e-04 | 695.5ms/step | 28.8 samples/s +Step 19930/44000 | Loss: 3.0358 | LR: 2.94e-04 | 693.3ms/step | 28.8 samples/s +Step 19940/44000 | Loss: 2.9690 | LR: 2.94e-04 | 693.6ms/step | 28.8 samples/s +Step 19950/44000 | Loss: 3.0373 | LR: 2.94e-04 | 694.7ms/step | 28.8 samples/s +Step 19960/44000 | Loss: 2.9642 | LR: 2.94e-04 | 694.0ms/step | 28.8 samples/s +Step 19970/44000 | Loss: 3.0225 | LR: 2.93e-04 | 692.5ms/step | 28.9 samples/s +Step 19980/44000 | Loss: 3.0442 | LR: 2.93e-04 | 693.2ms/step | 28.9 samples/s +Step 19990/44000 | Loss: 2.9883 | LR: 2.93e-04 | 693.2ms/step | 28.9 samples/s +Step 20000/44000 | Loss: 3.0353 | LR: 2.93e-04 | 692.7ms/step | 28.9 samples/s +Validation at step 20000: Loss = 3.0140, Perplexity = 20.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-20000 +Step 20010/44000 | Loss: 2.9439 | LR: 2.93e-04 | 1633.2ms/step | 12.2 samples/s +Step 20020/44000 | Loss: 3.0252 | LR: 2.93e-04 | 697.4ms/step | 28.7 samples/s +Step 20030/44000 | Loss: 3.0099 | LR: 2.92e-04 | 696.2ms/step | 28.7 samples/s +Step 20040/44000 | Loss: 3.0158 | LR: 2.92e-04 | 696.4ms/step | 28.7 samples/s +Step 20050/44000 | Loss: 3.0145 | LR: 2.92e-04 | 697.0ms/step | 28.7 samples/s +Step 20060/44000 | Loss: 3.0010 | LR: 2.92e-04 | 695.9ms/step | 28.7 samples/s +Step 20070/44000 | Loss: 3.0168 | LR: 2.92e-04 | 696.2ms/step | 28.7 samples/s +Step 20080/44000 | Loss: 2.9639 | LR: 2.91e-04 | 698.4ms/step | 28.6 samples/s +Step 20090/44000 | Loss: 2.9898 | LR: 2.91e-04 | 695.9ms/step | 28.7 samples/s +Step 20100/44000 | Loss: 3.0053 | LR: 2.91e-04 | 696.1ms/step | 28.7 samples/s +Step 20110/44000 | Loss: 3.0313 | LR: 2.91e-04 | 695.0ms/step | 28.8 samples/s +Step 20120/44000 | Loss: 2.9808 | LR: 2.91e-04 | 695.5ms/step | 28.8 samples/s +Step 20130/44000 | Loss: 3.0164 | LR: 2.91e-04 | 695.5ms/step | 28.8 samples/s +Step 20140/44000 | Loss: 2.9924 | LR: 2.90e-04 | 694.8ms/step | 28.8 samples/s +Step 20150/44000 | Loss: 2.9505 | LR: 2.90e-04 | 696.5ms/step | 28.7 samples/s +Step 20160/44000 | Loss: 3.0074 | LR: 2.90e-04 | 695.3ms/step | 28.8 samples/s +Step 20170/44000 | Loss: 3.0041 | LR: 2.90e-04 | 696.5ms/step | 28.7 samples/s +Step 20180/44000 | Loss: 3.0067 | LR: 2.90e-04 | 694.1ms/step | 28.8 samples/s +Step 20190/44000 | Loss: 3.0292 | LR: 2.90e-04 | 696.4ms/step | 28.7 samples/s +Step 20200/44000 | Loss: 3.0258 | LR: 2.89e-04 | 694.7ms/step | 28.8 samples/s +Step 20210/44000 | Loss: 3.0160 | LR: 2.89e-04 | 697.1ms/step | 28.7 samples/s +Step 20220/44000 | Loss: 3.0531 | LR: 2.89e-04 | 693.9ms/step | 28.8 samples/s +Step 20230/44000 | Loss: 3.0711 | LR: 2.89e-04 | 695.0ms/step | 28.8 samples/s +Step 20240/44000 | Loss: 2.9847 | LR: 2.89e-04 | 695.4ms/step | 28.8 samples/s +Step 20250/44000 | Loss: 2.9364 | LR: 2.88e-04 | 695.3ms/step | 28.8 samples/s +Validation at step 20250: Loss = 3.0130, Perplexity = 20.35 +Step 20260/44000 | Loss: 3.0036 | LR: 2.88e-04 | 1266.6ms/step | 15.8 samples/s +Step 20270/44000 | Loss: 2.9640 | LR: 2.88e-04 | 696.5ms/step | 28.7 samples/s +Step 20280/44000 | Loss: 3.0173 | LR: 2.88e-04 | 696.0ms/step | 28.7 samples/s +Step 20290/44000 | Loss: 2.9966 | LR: 2.88e-04 | 695.0ms/step | 28.8 samples/s +Step 20300/44000 | Loss: 3.0117 | LR: 2.88e-04 | 695.6ms/step | 28.8 samples/s +Step 20310/44000 | Loss: 3.0305 | LR: 2.87e-04 | 698.4ms/step | 28.6 samples/s +Step 20320/44000 | Loss: 2.9986 | LR: 2.87e-04 | 695.1ms/step | 28.8 samples/s +Step 20330/44000 | Loss: 2.9783 | LR: 2.87e-04 | 693.4ms/step | 28.8 samples/s +Step 20340/44000 | Loss: 3.0162 | LR: 2.87e-04 | 695.3ms/step | 28.8 samples/s +Step 20350/44000 | Loss: 3.0259 | LR: 2.87e-04 | 695.7ms/step | 28.7 samples/s +Step 20360/44000 | Loss: 3.0000 | LR: 2.86e-04 | 695.4ms/step | 28.8 samples/s +Step 20370/44000 | Loss: 3.0117 | LR: 2.86e-04 | 693.9ms/step | 28.8 samples/s +Step 20380/44000 | Loss: 2.9796 | LR: 2.86e-04 | 693.3ms/step | 28.8 samples/s +Step 20390/44000 | Loss: 2.9947 | LR: 2.86e-04 | 693.1ms/step | 28.9 samples/s +Step 20400/44000 | Loss: 2.9801 | LR: 2.86e-04 | 692.3ms/step | 28.9 samples/s +Step 20410/44000 | Loss: 2.9735 | LR: 2.86e-04 | 692.3ms/step | 28.9 samples/s +Step 20420/44000 | Loss: 3.0115 | LR: 2.85e-04 | 692.2ms/step | 28.9 samples/s +Step 20430/44000 | Loss: 2.9714 | LR: 2.85e-04 | 693.3ms/step | 28.8 samples/s +Step 20440/44000 | Loss: 3.0069 | LR: 2.85e-04 | 692.9ms/step | 28.9 samples/s +Step 20450/44000 | Loss: 3.0250 | LR: 2.85e-04 | 692.8ms/step | 28.9 samples/s +Step 20460/44000 | Loss: 3.0184 | LR: 2.85e-04 | 693.2ms/step | 28.9 samples/s +Step 20470/44000 | Loss: 3.0514 | LR: 2.84e-04 | 693.2ms/step | 28.9 samples/s +Step 20480/44000 | Loss: 2.9475 | LR: 2.84e-04 | 694.2ms/step | 28.8 samples/s +Step 20490/44000 | Loss: 2.9993 | LR: 2.84e-04 | 693.1ms/step | 28.9 samples/s +Step 20500/44000 | Loss: 2.9555 | LR: 2.84e-04 | 694.0ms/step | 28.8 samples/s +Validation at step 20500: Loss = 3.0089, Perplexity = 20.27 +Step 20510/44000 | Loss: 2.9693 | LR: 2.84e-04 | 1261.5ms/step | 15.9 samples/s +Step 20520/44000 | Loss: 2.9655 | LR: 2.84e-04 | 694.0ms/step | 28.8 samples/s +Step 20530/44000 | Loss: 2.9870 | LR: 2.83e-04 | 692.5ms/step | 28.9 samples/s +Step 20540/44000 | Loss: 2.9940 | LR: 2.83e-04 | 693.8ms/step | 28.8 samples/s +Step 20550/44000 | Loss: 3.0117 | LR: 2.83e-04 | 694.1ms/step | 28.8 samples/s +Step 20560/44000 | Loss: 2.9927 | LR: 2.83e-04 | 698.7ms/step | 28.6 samples/s +Step 20570/44000 | Loss: 3.0318 | LR: 2.83e-04 | 692.8ms/step | 28.9 samples/s +Step 20580/44000 | Loss: 2.9873 | LR: 2.83e-04 | 692.6ms/step | 28.9 samples/s +Step 20590/44000 | Loss: 3.0309 | LR: 2.82e-04 | 694.6ms/step | 28.8 samples/s +Step 20600/44000 | Loss: 3.0195 | LR: 2.82e-04 | 695.6ms/step | 28.8 samples/s +Step 20610/44000 | Loss: 2.9812 | LR: 2.82e-04 | 694.6ms/step | 28.8 samples/s +Step 20620/44000 | Loss: 3.0150 | LR: 2.82e-04 | 695.7ms/step | 28.7 samples/s +Step 20630/44000 | Loss: 3.0066 | LR: 2.82e-04 | 694.2ms/step | 28.8 samples/s +Step 20640/44000 | Loss: 3.0034 | LR: 2.81e-04 | 694.4ms/step | 28.8 samples/s +Step 20650/44000 | Loss: 2.9860 | LR: 2.81e-04 | 694.4ms/step | 28.8 samples/s +Step 20660/44000 | Loss: 2.9740 | LR: 2.81e-04 | 697.3ms/step | 28.7 samples/s +Step 20670/44000 | Loss: 2.9764 | LR: 2.81e-04 | 694.8ms/step | 28.8 samples/s +Step 20680/44000 | Loss: 3.0260 | LR: 2.81e-04 | 694.8ms/step | 28.8 samples/s +Step 20690/44000 | Loss: 2.9846 | LR: 2.81e-04 | 694.1ms/step | 28.8 samples/s +Step 20700/44000 | Loss: 3.0486 | LR: 2.80e-04 | 695.4ms/step | 28.8 samples/s +Step 20710/44000 | Loss: 3.0040 | LR: 2.80e-04 | 695.5ms/step | 28.8 samples/s +Step 20720/44000 | Loss: 3.0052 | LR: 2.80e-04 | 696.0ms/step | 28.7 samples/s +Step 20730/44000 | Loss: 2.9782 | LR: 2.80e-04 | 695.0ms/step | 28.8 samples/s +Step 20740/44000 | Loss: 2.9937 | LR: 2.80e-04 | 696.3ms/step | 28.7 samples/s +Step 20750/44000 | Loss: 3.0213 | LR: 2.79e-04 | 694.1ms/step | 28.8 samples/s +Validation at step 20750: Loss = 3.0064, Perplexity = 20.21 +Step 20760/44000 | Loss: 2.9892 | LR: 2.79e-04 | 1264.7ms/step | 15.8 samples/s +Step 20770/44000 | Loss: 2.9768 | LR: 2.79e-04 | 697.0ms/step | 28.7 samples/s +Step 20780/44000 | Loss: 3.0013 | LR: 2.79e-04 | 695.5ms/step | 28.8 samples/s +Step 20790/44000 | Loss: 2.9746 | LR: 2.79e-04 | 694.7ms/step | 28.8 samples/s +Step 20800/44000 | Loss: 2.9986 | LR: 2.79e-04 | 694.9ms/step | 28.8 samples/s +Step 20810/44000 | Loss: 2.9849 | LR: 2.78e-04 | 695.7ms/step | 28.7 samples/s +Step 20820/44000 | Loss: 2.9485 | LR: 2.78e-04 | 695.7ms/step | 28.7 samples/s +Step 20830/44000 | Loss: 2.9554 | LR: 2.78e-04 | 696.1ms/step | 28.7 samples/s +Step 20840/44000 | Loss: 3.0410 | LR: 2.78e-04 | 696.1ms/step | 28.7 samples/s +Step 20850/44000 | Loss: 2.9649 | LR: 2.78e-04 | 695.4ms/step | 28.8 samples/s +Step 20860/44000 | Loss: 2.9611 | LR: 2.77e-04 | 694.8ms/step | 28.8 samples/s +Step 20870/44000 | Loss: 3.0054 | LR: 2.77e-04 | 694.4ms/step | 28.8 samples/s +Step 20880/44000 | Loss: 3.0144 | LR: 2.77e-04 | 696.1ms/step | 28.7 samples/s +Step 20890/44000 | Loss: 2.9865 | LR: 2.77e-04 | 694.6ms/step | 28.8 samples/s +Step 20900/44000 | Loss: 2.9605 | LR: 2.77e-04 | 693.9ms/step | 28.8 samples/s +Step 20910/44000 | Loss: 2.9746 | LR: 2.77e-04 | 694.9ms/step | 28.8 samples/s +Step 20920/44000 | Loss: 2.9884 | LR: 2.76e-04 | 696.0ms/step | 28.7 samples/s +Step 20930/44000 | Loss: 2.9731 | LR: 2.76e-04 | 693.5ms/step | 28.8 samples/s +Step 20940/44000 | Loss: 2.9542 | LR: 2.76e-04 | 698.5ms/step | 28.6 samples/s +Step 20950/44000 | Loss: 2.9995 | LR: 2.76e-04 | 695.0ms/step | 28.8 samples/s +Step 20960/44000 | Loss: 2.9712 | LR: 2.76e-04 | 693.6ms/step | 28.8 samples/s +Step 20970/44000 | Loss: 2.9618 | LR: 2.75e-04 | 694.1ms/step | 28.8 samples/s +Step 20980/44000 | Loss: 3.0000 | LR: 2.75e-04 | 696.0ms/step | 28.7 samples/s +Step 20990/44000 | Loss: 3.0430 | LR: 2.75e-04 | 692.7ms/step | 28.9 samples/s +Step 21000/44000 | Loss: 2.9800 | LR: 2.75e-04 | 693.3ms/step | 28.8 samples/s +Validation at step 21000: Loss = 3.0037, Perplexity = 20.16 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-21000 +Step 21010/44000 | Loss: 3.0153 | LR: 2.75e-04 | 1572.8ms/step | 12.7 samples/s +Step 21020/44000 | Loss: 2.9755 | LR: 2.75e-04 | 694.7ms/step | 28.8 samples/s +Step 21030/44000 | Loss: 2.9880 | LR: 2.74e-04 | 694.6ms/step | 28.8 samples/s +Step 21040/44000 | Loss: 2.9869 | LR: 2.74e-04 | 694.7ms/step | 28.8 samples/s +Step 21050/44000 | Loss: 2.9946 | LR: 2.74e-04 | 695.0ms/step | 28.8 samples/s +Step 21060/44000 | Loss: 2.9805 | LR: 2.74e-04 | 695.3ms/step | 28.8 samples/s +Step 21070/44000 | Loss: 2.9641 | LR: 2.74e-04 | 693.5ms/step | 28.8 samples/s +Step 21080/44000 | Loss: 2.9639 | LR: 2.73e-04 | 695.5ms/step | 28.8 samples/s +Step 21090/44000 | Loss: 2.9440 | LR: 2.73e-04 | 695.1ms/step | 28.8 samples/s +Step 21100/44000 | Loss: 3.0217 | LR: 2.73e-04 | 697.0ms/step | 28.7 samples/s +Step 21110/44000 | Loss: 2.9944 | LR: 2.73e-04 | 693.7ms/step | 28.8 samples/s +Step 21120/44000 | Loss: 2.9990 | LR: 2.73e-04 | 695.1ms/step | 28.8 samples/s +Step 21130/44000 | Loss: 2.9722 | LR: 2.73e-04 | 695.4ms/step | 28.8 samples/s +Step 21140/44000 | Loss: 2.9998 | LR: 2.72e-04 | 694.5ms/step | 28.8 samples/s +Step 21150/44000 | Loss: 2.9543 | LR: 2.72e-04 | 693.9ms/step | 28.8 samples/s +Step 21160/44000 | Loss: 2.9575 | LR: 2.72e-04 | 693.9ms/step | 28.8 samples/s +Step 21170/44000 | Loss: 3.0348 | LR: 2.72e-04 | 697.4ms/step | 28.7 samples/s +Step 21180/44000 | Loss: 2.9848 | LR: 2.72e-04 | 699.6ms/step | 28.6 samples/s +Step 21190/44000 | Loss: 3.0109 | LR: 2.71e-04 | 694.4ms/step | 28.8 samples/s +Step 21200/44000 | Loss: 2.9953 | LR: 2.71e-04 | 695.1ms/step | 28.8 samples/s +Step 21210/44000 | Loss: 2.9642 | LR: 2.71e-04 | 694.5ms/step | 28.8 samples/s +Step 21220/44000 | Loss: 3.0058 | LR: 2.71e-04 | 694.1ms/step | 28.8 samples/s +Step 21230/44000 | Loss: 3.0136 | LR: 2.71e-04 | 696.9ms/step | 28.7 samples/s +Step 21240/44000 | Loss: 3.0070 | LR: 2.71e-04 | 694.0ms/step | 28.8 samples/s +Step 21250/44000 | Loss: 3.0233 | LR: 2.70e-04 | 693.9ms/step | 28.8 samples/s +Validation at step 21250: Loss = 3.0015, Perplexity = 20.12 +Step 21260/44000 | Loss: 2.9586 | LR: 2.70e-04 | 1265.1ms/step | 15.8 samples/s +Step 21270/44000 | Loss: 3.0441 | LR: 2.70e-04 | 695.5ms/step | 28.8 samples/s +Step 21280/44000 | Loss: 2.9948 | LR: 2.70e-04 | 700.2ms/step | 28.6 samples/s +Step 21290/44000 | Loss: 3.0214 | LR: 2.70e-04 | 700.8ms/step | 28.5 samples/s +Step 21300/44000 | Loss: 2.9619 | LR: 2.70e-04 | 694.1ms/step | 28.8 samples/s +Step 21310/44000 | Loss: 3.0030 | LR: 2.69e-04 | 693.8ms/step | 28.8 samples/s +Step 21320/44000 | Loss: 2.9609 | LR: 2.69e-04 | 693.8ms/step | 28.8 samples/s +Step 21330/44000 | Loss: 2.9795 | LR: 2.69e-04 | 695.1ms/step | 28.8 samples/s +Step 21340/44000 | Loss: 3.0086 | LR: 2.69e-04 | 693.6ms/step | 28.8 samples/s +Step 21350/44000 | Loss: 2.9824 | LR: 2.69e-04 | 694.1ms/step | 28.8 samples/s +Step 21360/44000 | Loss: 2.9865 | LR: 2.68e-04 | 694.4ms/step | 28.8 samples/s +Step 21370/44000 | Loss: 2.9890 | LR: 2.68e-04 | 695.9ms/step | 28.7 samples/s +Step 21380/44000 | Loss: 2.9836 | LR: 2.68e-04 | 695.8ms/step | 28.7 samples/s diff --git a/epfl_final/train_logs/p1_scheduled_h1002_20260822/relaunch.log b/epfl_final/train_logs/p1_scheduled_h1002_20260822/relaunch.log new file mode 100644 index 0000000000000000000000000000000000000000..87662959ad3b7f11001e6fea2e747ec98f717d6f --- /dev/null +++ b/epfl_final/train_logs/p1_scheduled_h1002_20260822/relaunch.log @@ -0,0 +1,2690 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-21000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-21000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 46, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 DDP2 365m resampled 8k nrb35 seed46", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "resampled", + "nrb35", + "seed46", + "ddp2", + "h1002", + "scheduled" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-21000 +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-21000... +Preserving checkpoint LR state for exact resume: current=[0.00027492781308970804], schedule_base=[0.0005] +Resumed from checkpoint at step 21000 (epoch 0, micro_step 105000) +Resuming training from step 21000 +Resuming wandb run 8xh3oikj +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 8xh3oikj +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_161845-8xh3oikj +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run P1 DDP2 365m resampled 8k nrb35 seed46 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8xh3oikj +Starting training... +Validation at step 21000: Loss = 3.0042, Perplexity = 20.17 +wandb: WARNING Tried to log to step 21000 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 21001: sparse_stochastic (configured sparse attention) +Step 21010/44000 | Loss: 3.0156 | LR: 2.75e-04 | 30141.9ms/step | 0.7 samples/s +Step 21020/44000 | Loss: 2.9752 | LR: 2.75e-04 | 692.9ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21010 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21020 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21030/44000 | Loss: 2.9876 | LR: 2.74e-04 | 692.1ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21030 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21040/44000 | Loss: 2.9865 | LR: 2.74e-04 | 689.8ms/step | 29.0 samples/s +Step 21050/44000 | Loss: 2.9940 | LR: 2.74e-04 | 691.0ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21040 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21050 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21060/44000 | Loss: 2.9800 | LR: 2.74e-04 | 693.0ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21060 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21070/44000 | Loss: 2.9643 | LR: 2.74e-04 | 691.3ms/step | 28.9 samples/s +Step 21080/44000 | Loss: 2.9634 | LR: 2.73e-04 | 692.6ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21070 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21080 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21090/44000 | Loss: 2.9432 | LR: 2.73e-04 | 694.3ms/step | 28.8 samples/s +wandb: WARNING Tried to log to step 21090 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21100/44000 | Loss: 3.0220 | LR: 2.73e-04 | 691.9ms/step | 28.9 samples/s +Step 21110/44000 | Loss: 2.9949 | LR: 2.73e-04 | 691.3ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21100 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21110 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21120/44000 | Loss: 2.9989 | LR: 2.73e-04 | 694.1ms/step | 28.8 samples/s +wandb: WARNING Tried to log to step 21120 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21130/44000 | Loss: 2.9720 | LR: 2.73e-04 | 692.2ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21130 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21140/44000 | Loss: 2.9999 | LR: 2.72e-04 | 693.0ms/step | 28.9 samples/s +Step 21150/44000 | Loss: 2.9542 | LR: 2.72e-04 | 693.1ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21140 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21150 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21160/44000 | Loss: 2.9584 | LR: 2.72e-04 | 692.7ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21160 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21170/44000 | Loss: 3.0351 | LR: 2.72e-04 | 691.2ms/step | 28.9 samples/s +Step 21180/44000 | Loss: 2.9852 | LR: 2.72e-04 | 693.9ms/step | 28.8 samples/s +wandb: WARNING Tried to log to step 21170 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21180 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21190/44000 | Loss: 3.0103 | LR: 2.71e-04 | 695.6ms/step | 28.8 samples/s +wandb: WARNING Tried to log to step 21190 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21200/44000 | Loss: 2.9946 | LR: 2.71e-04 | 692.1ms/step | 28.9 samples/s +Step 21210/44000 | Loss: 2.9644 | LR: 2.71e-04 | 694.3ms/step | 28.8 samples/s +wandb: WARNING Tried to log to step 21200 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21210 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21220/44000 | Loss: 3.0062 | LR: 2.71e-04 | 694.7ms/step | 28.8 samples/s +wandb: WARNING Tried to log to step 21220 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21230/44000 | Loss: 3.0143 | LR: 2.71e-04 | 693.0ms/step | 28.9 samples/s +Step 21240/44000 | Loss: 3.0073 | LR: 2.71e-04 | 691.3ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21230 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21240 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21250/44000 | Loss: 3.0227 | LR: 2.70e-04 | 692.9ms/step | 28.9 samples/s +wandb: WARNING Tried to log to step 21250 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 21250: Loss = 3.0017, Perplexity = 20.12 +Step 21260/44000 | Loss: 2.9590 | LR: 2.70e-04 | 1267.6ms/step | 15.8 samples/s +wandb: WARNING Tried to log to step 21260 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21270/44000 | Loss: 3.0435 | LR: 2.70e-04 | 695.9ms/step | 28.7 samples/s +wandb: WARNING Tried to log to step 21270 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21280/44000 | Loss: 2.9942 | LR: 2.70e-04 | 695.5ms/step | 28.8 samples/s +wandb: WARNING Tried to log to step 21280 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21290/44000 | Loss: 3.0211 | LR: 2.70e-04 | 695.5ms/step | 28.8 samples/s +Step 21300/44000 | Loss: 2.9620 | LR: 2.70e-04 | 698.6ms/step | 28.6 samples/s +wandb: WARNING Tried to log to step 21290 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21300 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21310/44000 | Loss: 3.0041 | LR: 2.69e-04 | 696.9ms/step | 28.7 samples/s +wandb: WARNING Tried to log to step 21310 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21320/44000 | Loss: 2.9607 | LR: 2.69e-04 | 696.1ms/step | 28.7 samples/s +Step 21330/44000 | Loss: 2.9791 | LR: 2.69e-04 | 695.0ms/step | 28.8 samples/s +wandb: WARNING Tried to log to step 21320 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21330 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21340/44000 | Loss: 3.0085 | LR: 2.69e-04 | 698.1ms/step | 28.6 samples/s +wandb: WARNING Tried to log to step 21340 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21350/44000 | Loss: 2.9818 | LR: 2.69e-04 | 694.4ms/step | 28.8 samples/s +Step 21360/44000 | Loss: 2.9856 | LR: 2.68e-04 | 696.7ms/step | 28.7 samples/s +wandb: WARNING Tried to log to step 21350 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 21360 that is less than the current step 21361. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 21370/44000 | Loss: 2.9898 | LR: 2.68e-04 | 694.5ms/step | 28.8 samples/s +Step 21380/44000 | Loss: 2.9836 | LR: 2.68e-04 | 695.6ms/step | 28.8 samples/s +Step 21390/44000 | Loss: 2.9697 | LR: 2.68e-04 | 695.1ms/step | 28.8 samples/s +Step 21400/44000 | Loss: 2.9679 | LR: 2.68e-04 | 695.9ms/step | 28.7 samples/s +Step 21410/44000 | Loss: 2.9756 | LR: 2.68e-04 | 696.3ms/step | 28.7 samples/s +Step 21420/44000 | Loss: 3.0023 | LR: 2.67e-04 | 695.3ms/step | 28.8 samples/s +Step 21430/44000 | Loss: 2.9883 | LR: 2.67e-04 | 695.5ms/step | 28.8 samples/s +Step 21440/44000 | Loss: 2.9839 | LR: 2.67e-04 | 697.2ms/step | 28.7 samples/s +Step 21450/44000 | Loss: 3.0103 | LR: 2.67e-04 | 693.9ms/step | 28.8 samples/s +Step 21460/44000 | Loss: 2.9538 | LR: 2.67e-04 | 695.5ms/step | 28.8 samples/s +Step 21470/44000 | Loss: 3.0031 | LR: 2.66e-04 | 696.3ms/step | 28.7 samples/s +Step 21480/44000 | Loss: 2.9852 | LR: 2.66e-04 | 695.3ms/step | 28.8 samples/s +Step 21490/44000 | Loss: 2.9650 | LR: 2.66e-04 | 694.8ms/step | 28.8 samples/s +Step 21500/44000 | Loss: 3.0278 | LR: 2.66e-04 | 697.8ms/step | 28.7 samples/s +Validation at step 21500: Loss = 3.0006, Perplexity = 20.10 +Step 21510/44000 | Loss: 2.9910 | LR: 2.66e-04 | 1267.9ms/step | 15.8 samples/s +Step 21520/44000 | Loss: 2.9934 | LR: 2.66e-04 | 698.0ms/step | 28.7 samples/s +Step 21530/44000 | Loss: 2.9512 | LR: 2.65e-04 | 695.1ms/step | 28.8 samples/s +Step 21540/44000 | Loss: 2.9367 | LR: 2.65e-04 | 695.2ms/step | 28.8 samples/s +Step 21550/44000 | Loss: 2.9801 | LR: 2.65e-04 | 695.3ms/step | 28.8 samples/s +Step 21560/44000 | Loss: 2.9811 | LR: 2.65e-04 | 696.9ms/step | 28.7 samples/s +Step 21570/44000 | Loss: 2.9872 | LR: 2.65e-04 | 696.2ms/step | 28.7 samples/s +Step 21580/44000 | Loss: 2.9801 | LR: 2.64e-04 | 695.3ms/step | 28.8 samples/s +Step 21590/44000 | Loss: 3.0237 | LR: 2.64e-04 | 699.5ms/step | 28.6 samples/s +Step 21600/44000 | Loss: 2.9892 | LR: 2.64e-04 | 695.7ms/step | 28.7 samples/s +Step 21610/44000 | Loss: 2.9593 | LR: 2.64e-04 | 695.0ms/step | 28.8 samples/s +Step 21620/44000 | Loss: 2.9864 | LR: 2.64e-04 | 696.4ms/step | 28.7 samples/s +Step 21630/44000 | Loss: 2.9883 | LR: 2.64e-04 | 696.8ms/step | 28.7 samples/s +Step 21640/44000 | Loss: 2.9813 | LR: 2.63e-04 | 697.5ms/step | 28.7 samples/s +Step 21650/44000 | Loss: 2.9553 | LR: 2.63e-04 | 695.8ms/step | 28.7 samples/s +Step 21660/44000 | Loss: 2.9732 | LR: 2.63e-04 | 695.9ms/step | 28.7 samples/s +Step 21670/44000 | Loss: 3.0260 | LR: 2.63e-04 | 695.5ms/step | 28.8 samples/s +Step 21680/44000 | Loss: 2.9406 | LR: 2.63e-04 | 694.9ms/step | 28.8 samples/s +Step 21690/44000 | Loss: 3.0211 | LR: 2.62e-04 | 698.3ms/step | 28.6 samples/s +Step 21700/44000 | Loss: 3.0425 | LR: 2.62e-04 | 698.5ms/step | 28.6 samples/s +Step 21710/44000 | Loss: 3.0156 | LR: 2.62e-04 | 695.7ms/step | 28.7 samples/s +Step 21720/44000 | Loss: 2.9763 | LR: 2.62e-04 | 697.2ms/step | 28.7 samples/s +Step 21730/44000 | Loss: 2.9763 | LR: 2.62e-04 | 694.7ms/step | 28.8 samples/s +Step 21740/44000 | Loss: 3.0037 | LR: 2.62e-04 | 696.4ms/step | 28.7 samples/s +Step 21750/44000 | Loss: 3.0188 | LR: 2.61e-04 | 696.3ms/step | 28.7 samples/s +Validation at step 21750: Loss = 2.9970, Perplexity = 20.02 +Step 21760/44000 | Loss: 2.9967 | LR: 2.61e-04 | 1272.9ms/step | 15.7 samples/s +Step 21770/44000 | Loss: 3.0133 | LR: 2.61e-04 | 697.3ms/step | 28.7 samples/s +Step 21780/44000 | Loss: 2.9882 | LR: 2.61e-04 | 698.2ms/step | 28.6 samples/s +Step 21790/44000 | Loss: 2.9793 | LR: 2.61e-04 | 698.1ms/step | 28.7 samples/s +Step 21800/44000 | Loss: 2.9286 | LR: 2.60e-04 | 696.5ms/step | 28.7 samples/s +Step 21810/44000 | Loss: 2.9776 | LR: 2.60e-04 | 697.3ms/step | 28.7 samples/s +Step 21820/44000 | Loss: 2.9676 | LR: 2.60e-04 | 697.2ms/step | 28.7 samples/s +Step 21830/44000 | Loss: 2.9845 | LR: 2.60e-04 | 696.8ms/step | 28.7 samples/s +Step 21840/44000 | Loss: 2.9542 | LR: 2.60e-04 | 699.7ms/step | 28.6 samples/s +Step 21850/44000 | Loss: 2.9616 | LR: 2.60e-04 | 696.9ms/step | 28.7 samples/s +Step 21860/44000 | Loss: 2.9853 | LR: 2.59e-04 | 696.0ms/step | 28.7 samples/s +Step 21870/44000 | Loss: 3.0107 | LR: 2.59e-04 | 695.6ms/step | 28.8 samples/s +Step 21880/44000 | Loss: 2.9711 | LR: 2.59e-04 | 696.7ms/step | 28.7 samples/s +Step 21890/44000 | Loss: 2.9593 | LR: 2.59e-04 | 697.9ms/step | 28.7 samples/s +Step 21900/44000 | Loss: 3.0041 | LR: 2.59e-04 | 696.5ms/step | 28.7 samples/s +Step 21910/44000 | Loss: 2.9833 | LR: 2.58e-04 | 696.7ms/step | 28.7 samples/s +Step 21920/44000 | Loss: 3.0305 | LR: 2.58e-04 | 696.2ms/step | 28.7 samples/s +Step 21930/44000 | Loss: 2.9584 | LR: 2.58e-04 | 696.5ms/step | 28.7 samples/s +Step 21940/44000 | Loss: 3.0373 | LR: 2.58e-04 | 697.2ms/step | 28.7 samples/s +Step 21950/44000 | Loss: 2.9788 | LR: 2.58e-04 | 698.0ms/step | 28.7 samples/s +Step 21960/44000 | Loss: 2.9849 | LR: 2.58e-04 | 695.8ms/step | 28.7 samples/s +Step 21970/44000 | Loss: 2.9974 | LR: 2.57e-04 | 698.9ms/step | 28.6 samples/s +Step 21980/44000 | Loss: 2.9886 | LR: 2.57e-04 | 698.3ms/step | 28.6 samples/s +Step 21990/44000 | Loss: 2.9652 | LR: 2.57e-04 | 696.2ms/step | 28.7 samples/s +Step 22000/44000 | Loss: 2.9961 | LR: 2.57e-04 | 695.9ms/step | 28.7 samples/s +Validation at step 22000: Loss = 2.9957, Perplexity = 20.00 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-22000 +Step 22010/44000 | Loss: 2.9809 | LR: 2.57e-04 | 1573.6ms/step | 12.7 samples/s +Step 22020/44000 | Loss: 3.0021 | LR: 2.56e-04 | 695.1ms/step | 28.8 samples/s +Step 22030/44000 | Loss: 2.9889 | LR: 2.56e-04 | 696.0ms/step | 28.7 samples/s +Step 22040/44000 | Loss: 2.9714 | LR: 2.56e-04 | 695.4ms/step | 28.8 samples/s +Step 22050/44000 | Loss: 2.9898 | LR: 2.56e-04 | 695.0ms/step | 28.8 samples/s +Step 22060/44000 | Loss: 2.9951 | LR: 2.56e-04 | 695.4ms/step | 28.8 samples/s +Step 22070/44000 | Loss: 2.9905 | LR: 2.56e-04 | 695.7ms/step | 28.7 samples/s +Step 22080/44000 | Loss: 2.9866 | LR: 2.55e-04 | 695.4ms/step | 28.8 samples/s +Step 22090/44000 | Loss: 3.0163 | LR: 2.55e-04 | 696.3ms/step | 28.7 samples/s +Step 22100/44000 | Loss: 2.9650 | LR: 2.55e-04 | 694.9ms/step | 28.8 samples/s +Step 22110/44000 | Loss: 2.9874 | LR: 2.55e-04 | 695.7ms/step | 28.7 samples/s +Step 22120/44000 | Loss: 3.0149 | LR: 2.55e-04 | 697.6ms/step | 28.7 samples/s +Step 22130/44000 | Loss: 2.9608 | LR: 2.54e-04 | 696.9ms/step | 28.7 samples/s +Step 22140/44000 | Loss: 3.0132 | LR: 2.54e-04 | 695.4ms/step | 28.8 samples/s +Step 22150/44000 | Loss: 2.9802 | LR: 2.54e-04 | 696.0ms/step | 28.7 samples/s +Step 22160/44000 | Loss: 2.9465 | LR: 2.54e-04 | 698.9ms/step | 28.6 samples/s +Step 22170/44000 | Loss: 2.9713 | LR: 2.54e-04 | 695.2ms/step | 28.8 samples/s +Step 22180/44000 | Loss: 2.9826 | LR: 2.54e-04 | 694.6ms/step | 28.8 samples/s +Step 22190/44000 | Loss: 3.0171 | LR: 2.53e-04 | 697.7ms/step | 28.7 samples/s +Step 22200/44000 | Loss: 2.9626 | LR: 2.53e-04 | 694.2ms/step | 28.8 samples/s +Step 22210/44000 | Loss: 2.9886 | LR: 2.53e-04 | 694.7ms/step | 28.8 samples/s +Step 22220/44000 | Loss: 2.9569 | LR: 2.53e-04 | 695.2ms/step | 28.8 samples/s +Step 22230/44000 | Loss: 2.9710 | LR: 2.53e-04 | 694.5ms/step | 28.8 samples/s +Step 22240/44000 | Loss: 2.9732 | LR: 2.52e-04 | 698.0ms/step | 28.7 samples/s +Step 22250/44000 | Loss: 2.9958 | LR: 2.52e-04 | 696.0ms/step | 28.7 samples/s +Validation at step 22250: Loss = 2.9926, Perplexity = 19.94 +Step 22260/44000 | Loss: 2.9895 | LR: 2.52e-04 | 1267.0ms/step | 15.8 samples/s +Step 22270/44000 | Loss: 3.0191 | LR: 2.52e-04 | 697.7ms/step | 28.7 samples/s +Step 22280/44000 | Loss: 3.0306 | LR: 2.52e-04 | 696.8ms/step | 28.7 samples/s +Step 22290/44000 | Loss: 3.0013 | LR: 2.52e-04 | 695.3ms/step | 28.8 samples/s +Step 22300/44000 | Loss: 2.9887 | LR: 2.51e-04 | 700.1ms/step | 28.6 samples/s +Step 22310/44000 | Loss: 2.9650 | LR: 2.51e-04 | 698.3ms/step | 28.6 samples/s +Step 22320/44000 | Loss: 2.9834 | LR: 2.51e-04 | 695.2ms/step | 28.8 samples/s +Step 22330/44000 | Loss: 2.9492 | LR: 2.51e-04 | 694.3ms/step | 28.8 samples/s +Step 22340/44000 | Loss: 2.9277 | LR: 2.51e-04 | 694.6ms/step | 28.8 samples/s +Step 22350/44000 | Loss: 3.0030 | LR: 2.50e-04 | 695.3ms/step | 28.8 samples/s +Step 22360/44000 | Loss: 2.9733 | LR: 2.50e-04 | 695.5ms/step | 28.8 samples/s +Step 22370/44000 | Loss: 2.9963 | LR: 2.50e-04 | 695.8ms/step | 28.7 samples/s +Step 22380/44000 | Loss: 2.9752 | LR: 2.50e-04 | 696.2ms/step | 28.7 samples/s +Step 22390/44000 | Loss: 2.9697 | LR: 2.50e-04 | 695.9ms/step | 28.7 samples/s +Step 22400/44000 | Loss: 2.9852 | LR: 2.50e-04 | 695.2ms/step | 28.8 samples/s +Step 22410/44000 | Loss: 2.9896 | LR: 2.49e-04 | 698.0ms/step | 28.7 samples/s +Step 22420/44000 | Loss: 2.9894 | LR: 2.49e-04 | 694.4ms/step | 28.8 samples/s +Step 22430/44000 | Loss: 3.0235 | LR: 2.49e-04 | 695.0ms/step | 28.8 samples/s +Step 22440/44000 | Loss: 3.0078 | LR: 2.49e-04 | 695.1ms/step | 28.8 samples/s +Step 22450/44000 | Loss: 2.9963 | LR: 2.49e-04 | 695.0ms/step | 28.8 samples/s +Step 22460/44000 | Loss: 2.9571 | LR: 2.48e-04 | 696.0ms/step | 28.7 samples/s +Step 22470/44000 | Loss: 2.9912 | LR: 2.48e-04 | 695.3ms/step | 28.8 samples/s +Step 22480/44000 | Loss: 2.9853 | LR: 2.48e-04 | 696.8ms/step | 28.7 samples/s +Step 22490/44000 | Loss: 3.0203 | LR: 2.48e-04 | 695.4ms/step | 28.8 samples/s +Step 22500/44000 | Loss: 2.9764 | LR: 2.48e-04 | 695.0ms/step | 28.8 samples/s +Validation at step 22500: Loss = 2.9897, Perplexity = 19.88 +Step 22510/44000 | Loss: 2.9808 | LR: 2.48e-04 | 1262.3ms/step | 15.8 samples/s +Step 22520/44000 | Loss: 2.9637 | LR: 2.47e-04 | 694.7ms/step | 28.8 samples/s +Step 22530/44000 | Loss: 2.9760 | LR: 2.47e-04 | 695.0ms/step | 28.8 samples/s +Step 22540/44000 | Loss: 3.0282 | LR: 2.47e-04 | 695.4ms/step | 28.8 samples/s +Step 22550/44000 | Loss: 3.0010 | LR: 2.47e-04 | 693.9ms/step | 28.8 samples/s +Step 22560/44000 | Loss: 2.9612 | LR: 2.47e-04 | 695.7ms/step | 28.7 samples/s +Step 22570/44000 | Loss: 2.9695 | LR: 2.46e-04 | 696.1ms/step | 28.7 samples/s +Step 22580/44000 | Loss: 2.9696 | LR: 2.46e-04 | 695.1ms/step | 28.8 samples/s +Step 22590/44000 | Loss: 2.9470 | LR: 2.46e-04 | 696.4ms/step | 28.7 samples/s +Step 22600/44000 | Loss: 2.9702 | LR: 2.46e-04 | 695.1ms/step | 28.8 samples/s +Step 22610/44000 | Loss: 2.9731 | LR: 2.46e-04 | 695.4ms/step | 28.8 samples/s +Step 22620/44000 | Loss: 2.9524 | LR: 2.46e-04 | 695.0ms/step | 28.8 samples/s +Step 22630/44000 | Loss: 2.9370 | LR: 2.45e-04 | 698.8ms/step | 28.6 samples/s +Step 22640/44000 | Loss: 2.9603 | LR: 2.45e-04 | 694.5ms/step | 28.8 samples/s +Step 22650/44000 | Loss: 2.9507 | LR: 2.45e-04 | 694.8ms/step | 28.8 samples/s +Step 22660/44000 | Loss: 2.9534 | LR: 2.45e-04 | 698.3ms/step | 28.6 samples/s +Step 22670/44000 | Loss: 2.9891 | LR: 2.45e-04 | 697.4ms/step | 28.7 samples/s +Step 22680/44000 | Loss: 2.8803 | LR: 2.44e-04 | 694.5ms/step | 28.8 samples/s +Step 22690/44000 | Loss: 3.0171 | LR: 2.44e-04 | 695.5ms/step | 28.8 samples/s +Step 22700/44000 | Loss: 2.9369 | LR: 2.44e-04 | 695.6ms/step | 28.8 samples/s +Step 22710/44000 | Loss: 2.9632 | LR: 2.44e-04 | 697.0ms/step | 28.7 samples/s +Step 22720/44000 | Loss: 3.0002 | LR: 2.44e-04 | 696.1ms/step | 28.7 samples/s +Step 22730/44000 | Loss: 2.9293 | LR: 2.44e-04 | 696.8ms/step | 28.7 samples/s +Step 22740/44000 | Loss: 3.0179 | LR: 2.43e-04 | 694.8ms/step | 28.8 samples/s +Step 22750/44000 | Loss: 3.0292 | LR: 2.43e-04 | 695.5ms/step | 28.8 samples/s +Validation at step 22750: Loss = 2.9870, Perplexity = 19.83 +Step 22760/44000 | Loss: 2.9988 | LR: 2.43e-04 | 1264.2ms/step | 15.8 samples/s +Step 22770/44000 | Loss: 2.9921 | LR: 2.43e-04 | 694.7ms/step | 28.8 samples/s +Step 22780/44000 | Loss: 2.9638 | LR: 2.43e-04 | 695.5ms/step | 28.8 samples/s +Step 22790/44000 | Loss: 2.9771 | LR: 2.42e-04 | 693.4ms/step | 28.8 samples/s +Step 22800/44000 | Loss: 2.9741 | LR: 2.42e-04 | 692.6ms/step | 28.9 samples/s +Step 22810/44000 | Loss: 2.9626 | LR: 2.42e-04 | 693.5ms/step | 28.8 samples/s +Step 22820/44000 | Loss: 2.9756 | LR: 2.42e-04 | 693.1ms/step | 28.9 samples/s +Step 22830/44000 | Loss: 2.9671 | LR: 2.42e-04 | 692.4ms/step | 28.9 samples/s +Step 22840/44000 | Loss: 2.9938 | LR: 2.42e-04 | 694.0ms/step | 28.8 samples/s +Step 22850/44000 | Loss: 3.0024 | LR: 2.41e-04 | 693.6ms/step | 28.8 samples/s +Step 22860/44000 | Loss: 2.9463 | LR: 2.41e-04 | 697.0ms/step | 28.7 samples/s +Step 22870/44000 | Loss: 2.9596 | LR: 2.41e-04 | 695.0ms/step | 28.8 samples/s +Step 22880/44000 | Loss: 2.9717 | LR: 2.41e-04 | 694.7ms/step | 28.8 samples/s +Step 22890/44000 | Loss: 3.0316 | LR: 2.41e-04 | 693.3ms/step | 28.8 samples/s +Step 22900/44000 | Loss: 2.9489 | LR: 2.40e-04 | 694.5ms/step | 28.8 samples/s +Step 22910/44000 | Loss: 2.9921 | LR: 2.40e-04 | 695.6ms/step | 28.8 samples/s +Step 22920/44000 | Loss: 2.9843 | LR: 2.40e-04 | 693.6ms/step | 28.8 samples/s +Step 22930/44000 | Loss: 2.9853 | LR: 2.40e-04 | 694.9ms/step | 28.8 samples/s +Step 22940/44000 | Loss: 2.9192 | LR: 2.40e-04 | 695.9ms/step | 28.7 samples/s +Step 22950/44000 | Loss: 2.9953 | LR: 2.40e-04 | 694.8ms/step | 28.8 samples/s +Step 22960/44000 | Loss: 2.9524 | LR: 2.39e-04 | 694.6ms/step | 28.8 samples/s +Step 22970/44000 | Loss: 2.9634 | LR: 2.39e-04 | 695.4ms/step | 28.8 samples/s +Step 22980/44000 | Loss: 2.9431 | LR: 2.39e-04 | 694.4ms/step | 28.8 samples/s +Step 22990/44000 | Loss: 2.9989 | LR: 2.39e-04 | 695.0ms/step | 28.8 samples/s +Step 23000/44000 | Loss: 2.9838 | LR: 2.39e-04 | 694.9ms/step | 28.8 samples/s +Validation at step 23000: Loss = 2.9849, Perplexity = 19.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-23000 +Step 23010/44000 | Loss: 2.9828 | LR: 2.38e-04 | 1569.1ms/step | 12.7 samples/s +Step 23020/44000 | Loss: 3.0515 | LR: 2.38e-04 | 693.4ms/step | 28.8 samples/s +Step 23030/44000 | Loss: 2.9762 | LR: 2.38e-04 | 695.9ms/step | 28.7 samples/s +Step 23040/44000 | Loss: 2.9805 | LR: 2.38e-04 | 694.4ms/step | 28.8 samples/s +Step 23050/44000 | Loss: 2.9861 | LR: 2.38e-04 | 693.1ms/step | 28.9 samples/s +Step 23060/44000 | Loss: 2.9474 | LR: 2.38e-04 | 694.7ms/step | 28.8 samples/s +Step 23070/44000 | Loss: 2.9631 | LR: 2.37e-04 | 698.2ms/step | 28.6 samples/s +Step 23080/44000 | Loss: 3.0402 | LR: 2.37e-04 | 693.6ms/step | 28.8 samples/s +Step 23090/44000 | Loss: 2.9590 | LR: 2.37e-04 | 695.5ms/step | 28.8 samples/s +Step 23100/44000 | Loss: 2.9324 | LR: 2.37e-04 | 692.9ms/step | 28.9 samples/s +Step 23110/44000 | Loss: 3.0051 | LR: 2.37e-04 | 694.3ms/step | 28.8 samples/s +Step 23120/44000 | Loss: 2.9884 | LR: 2.36e-04 | 694.8ms/step | 28.8 samples/s +Step 23130/44000 | Loss: 2.9726 | LR: 2.36e-04 | 694.8ms/step | 28.8 samples/s +Step 23140/44000 | Loss: 2.9551 | LR: 2.36e-04 | 692.9ms/step | 28.9 samples/s +Step 23150/44000 | Loss: 3.0146 | LR: 2.36e-04 | 693.7ms/step | 28.8 samples/s +Step 23160/44000 | Loss: 2.9433 | LR: 2.36e-04 | 694.3ms/step | 28.8 samples/s +Step 23170/44000 | Loss: 2.9707 | LR: 2.36e-04 | 695.2ms/step | 28.8 samples/s +Step 23180/44000 | Loss: 3.0040 | LR: 2.35e-04 | 693.3ms/step | 28.8 samples/s +Step 23190/44000 | Loss: 2.9282 | LR: 2.35e-04 | 694.6ms/step | 28.8 samples/s +Step 23200/44000 | Loss: 2.9313 | LR: 2.35e-04 | 693.1ms/step | 28.9 samples/s +Step 23210/44000 | Loss: 3.0037 | LR: 2.35e-04 | 694.2ms/step | 28.8 samples/s +Step 23220/44000 | Loss: 2.9660 | LR: 2.35e-04 | 693.9ms/step | 28.8 samples/s +Step 23230/44000 | Loss: 2.9829 | LR: 2.34e-04 | 694.1ms/step | 28.8 samples/s +Step 23240/44000 | Loss: 2.9902 | LR: 2.34e-04 | 695.7ms/step | 28.7 samples/s +Step 23250/44000 | Loss: 2.9996 | LR: 2.34e-04 | 693.9ms/step | 28.8 samples/s +Validation at step 23250: Loss = 2.9821, Perplexity = 19.73 +Step 23260/44000 | Loss: 2.9658 | LR: 2.34e-04 | 1262.7ms/step | 15.8 samples/s +Step 23270/44000 | Loss: 2.9538 | LR: 2.34e-04 | 693.3ms/step | 28.8 samples/s +Step 23280/44000 | Loss: 2.9799 | LR: 2.34e-04 | 695.2ms/step | 28.8 samples/s +Step 23290/44000 | Loss: 2.9489 | LR: 2.33e-04 | 695.1ms/step | 28.8 samples/s +Step 23300/44000 | Loss: 3.0100 | LR: 2.33e-04 | 693.8ms/step | 28.8 samples/s +Step 23310/44000 | Loss: 2.9972 | LR: 2.33e-04 | 693.3ms/step | 28.8 samples/s +Step 23320/44000 | Loss: 2.9635 | LR: 2.33e-04 | 692.1ms/step | 28.9 samples/s +Step 23330/44000 | Loss: 2.9496 | LR: 2.33e-04 | 693.7ms/step | 28.8 samples/s +Step 23340/44000 | Loss: 2.9660 | LR: 2.32e-04 | 692.6ms/step | 28.9 samples/s +Step 23350/44000 | Loss: 2.9715 | LR: 2.32e-04 | 692.9ms/step | 28.9 samples/s +Step 23360/44000 | Loss: 3.0135 | LR: 2.32e-04 | 692.8ms/step | 28.9 samples/s +Step 23370/44000 | Loss: 2.9678 | LR: 2.32e-04 | 692.1ms/step | 28.9 samples/s +Step 23380/44000 | Loss: 2.9669 | LR: 2.32e-04 | 693.5ms/step | 28.8 samples/s +Step 23390/44000 | Loss: 2.9199 | LR: 2.32e-04 | 691.9ms/step | 28.9 samples/s +Step 23400/44000 | Loss: 2.9608 | LR: 2.31e-04 | 694.8ms/step | 28.8 samples/s +Step 23410/44000 | Loss: 2.9707 | LR: 2.31e-04 | 694.6ms/step | 28.8 samples/s +Step 23420/44000 | Loss: 2.9297 | LR: 2.31e-04 | 694.4ms/step | 28.8 samples/s +Step 23430/44000 | Loss: 2.9464 | LR: 2.31e-04 | 692.7ms/step | 28.9 samples/s +Step 23440/44000 | Loss: 2.9369 | LR: 2.31e-04 | 692.8ms/step | 28.9 samples/s +Step 23450/44000 | Loss: 2.9810 | LR: 2.30e-04 | 691.4ms/step | 28.9 samples/s +Step 23460/44000 | Loss: 2.9873 | LR: 2.30e-04 | 692.0ms/step | 28.9 samples/s +Step 23470/44000 | Loss: 2.9446 | LR: 2.30e-04 | 692.9ms/step | 28.9 samples/s +Step 23480/44000 | Loss: 2.9931 | LR: 2.30e-04 | 693.6ms/step | 28.8 samples/s +Step 23490/44000 | Loss: 2.9648 | LR: 2.30e-04 | 692.0ms/step | 28.9 samples/s +Step 23500/44000 | Loss: 2.9807 | LR: 2.30e-04 | 693.1ms/step | 28.9 samples/s +Validation at step 23500: Loss = 2.9808, Perplexity = 19.70 +Step 23510/44000 | Loss: 2.9439 | LR: 2.29e-04 | 1260.5ms/step | 15.9 samples/s +Step 23520/44000 | Loss: 2.9795 | LR: 2.29e-04 | 693.8ms/step | 28.8 samples/s +Step 23530/44000 | Loss: 2.9563 | LR: 2.29e-04 | 694.2ms/step | 28.8 samples/s +Step 23540/44000 | Loss: 2.9621 | LR: 2.29e-04 | 693.1ms/step | 28.9 samples/s +Step 23550/44000 | Loss: 2.9621 | LR: 2.29e-04 | 692.5ms/step | 28.9 samples/s +Step 23560/44000 | Loss: 2.9544 | LR: 2.29e-04 | 692.6ms/step | 28.9 samples/s +Step 23570/44000 | Loss: 3.0294 | LR: 2.28e-04 | 693.9ms/step | 28.8 samples/s +Step 23580/44000 | Loss: 2.9716 | LR: 2.28e-04 | 692.6ms/step | 28.9 samples/s +Step 23590/44000 | Loss: 2.9881 | LR: 2.28e-04 | 694.6ms/step | 28.8 samples/s +Step 23600/44000 | Loss: 2.9700 | LR: 2.28e-04 | 694.0ms/step | 28.8 samples/s +Step 23610/44000 | Loss: 2.9297 | LR: 2.28e-04 | 693.3ms/step | 28.8 samples/s +Step 23620/44000 | Loss: 2.9301 | LR: 2.27e-04 | 692.9ms/step | 28.9 samples/s +Step 23630/44000 | Loss: 2.9232 | LR: 2.27e-04 | 694.2ms/step | 28.8 samples/s +Step 23640/44000 | Loss: 2.9598 | LR: 2.27e-04 | 695.9ms/step | 28.7 samples/s +Step 23650/44000 | Loss: 2.9747 | LR: 2.27e-04 | 693.8ms/step | 28.8 samples/s +Step 23660/44000 | Loss: 2.9833 | LR: 2.27e-04 | 693.3ms/step | 28.8 samples/s +Step 23670/44000 | Loss: 2.9451 | LR: 2.27e-04 | 693.7ms/step | 28.8 samples/s +Step 23680/44000 | Loss: 2.9626 | LR: 2.26e-04 | 693.1ms/step | 28.9 samples/s +Step 23690/44000 | Loss: 2.9703 | LR: 2.26e-04 | 693.7ms/step | 28.8 samples/s +Step 23700/44000 | Loss: 2.9720 | LR: 2.26e-04 | 695.1ms/step | 28.8 samples/s +Step 23710/44000 | Loss: 2.9665 | LR: 2.26e-04 | 693.2ms/step | 28.9 samples/s +Step 23720/44000 | Loss: 2.9522 | LR: 2.26e-04 | 692.4ms/step | 28.9 samples/s +Step 23730/44000 | Loss: 2.9891 | LR: 2.25e-04 | 693.3ms/step | 28.8 samples/s +Step 23740/44000 | Loss: 2.9649 | LR: 2.25e-04 | 693.9ms/step | 28.8 samples/s +Step 23750/44000 | Loss: 2.9776 | LR: 2.25e-04 | 694.4ms/step | 28.8 samples/s +Validation at step 23750: Loss = 2.9785, Perplexity = 19.66 +Step 23760/44000 | Loss: 2.9867 | LR: 2.25e-04 | 1264.1ms/step | 15.8 samples/s +Step 23770/44000 | Loss: 2.9580 | LR: 2.25e-04 | 693.4ms/step | 28.8 samples/s +Step 23780/44000 | Loss: 2.9592 | LR: 2.25e-04 | 694.3ms/step | 28.8 samples/s +Step 23790/44000 | Loss: 2.9729 | LR: 2.24e-04 | 695.8ms/step | 28.7 samples/s +Step 23800/44000 | Loss: 2.9676 | LR: 2.24e-04 | 693.1ms/step | 28.9 samples/s +Step 23810/44000 | Loss: 2.9678 | LR: 2.24e-04 | 692.6ms/step | 28.9 samples/s +Step 23820/44000 | Loss: 2.9276 | LR: 2.24e-04 | 694.1ms/step | 28.8 samples/s +Step 23830/44000 | Loss: 2.9606 | LR: 2.24e-04 | 694.8ms/step | 28.8 samples/s +Step 23840/44000 | Loss: 2.9509 | LR: 2.23e-04 | 694.2ms/step | 28.8 samples/s +Step 23850/44000 | Loss: 2.9508 | LR: 2.23e-04 | 694.8ms/step | 28.8 samples/s +Step 23860/44000 | Loss: 2.9826 | LR: 2.23e-04 | 694.0ms/step | 28.8 samples/s +Step 23870/44000 | Loss: 2.9429 | LR: 2.23e-04 | 694.1ms/step | 28.8 samples/s +Step 23880/44000 | Loss: 2.9573 | LR: 2.23e-04 | 696.2ms/step | 28.7 samples/s +Step 23890/44000 | Loss: 2.9917 | LR: 2.23e-04 | 695.9ms/step | 28.7 samples/s +Step 23900/44000 | Loss: 2.9616 | LR: 2.22e-04 | 696.1ms/step | 28.7 samples/s +Step 23910/44000 | Loss: 2.9440 | LR: 2.22e-04 | 692.9ms/step | 28.9 samples/s +Step 23920/44000 | Loss: 2.9863 | LR: 2.22e-04 | 693.5ms/step | 28.8 samples/s +Step 23930/44000 | Loss: 2.9585 | LR: 2.22e-04 | 692.5ms/step | 28.9 samples/s +Step 23940/44000 | Loss: 2.9301 | LR: 2.22e-04 | 693.0ms/step | 28.9 samples/s +Step 23950/44000 | Loss: 2.9673 | LR: 2.21e-04 | 692.9ms/step | 28.9 samples/s +Step 23960/44000 | Loss: 2.9930 | LR: 2.21e-04 | 693.3ms/step | 28.8 samples/s +Step 23970/44000 | Loss: 2.9661 | LR: 2.21e-04 | 692.5ms/step | 28.9 samples/s +Step 23980/44000 | Loss: 2.9673 | LR: 2.21e-04 | 693.3ms/step | 28.8 samples/s +Step 23990/44000 | Loss: 2.9465 | LR: 2.21e-04 | 694.2ms/step | 28.8 samples/s +Step 24000/44000 | Loss: 2.9436 | LR: 2.21e-04 | 694.3ms/step | 28.8 samples/s +Validation at step 24000: Loss = 2.9756, Perplexity = 19.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-24000 +Step 24010/44000 | Loss: 3.0011 | LR: 2.20e-04 | 1580.9ms/step | 12.7 samples/s +Step 24020/44000 | Loss: 2.9600 | LR: 2.20e-04 | 694.7ms/step | 28.8 samples/s +Step 24030/44000 | Loss: 2.9890 | LR: 2.20e-04 | 696.2ms/step | 28.7 samples/s +Step 24040/44000 | Loss: 2.9757 | LR: 2.20e-04 | 697.5ms/step | 28.7 samples/s +Step 24050/44000 | Loss: 2.9318 | LR: 2.20e-04 | 695.2ms/step | 28.8 samples/s +Step 24060/44000 | Loss: 2.9583 | LR: 2.19e-04 | 694.7ms/step | 28.8 samples/s +Step 24070/44000 | Loss: 2.9760 | LR: 2.19e-04 | 696.2ms/step | 28.7 samples/s +Step 24080/44000 | Loss: 2.9308 | LR: 2.19e-04 | 694.8ms/step | 28.8 samples/s +Step 24090/44000 | Loss: 2.9703 | LR: 2.19e-04 | 694.6ms/step | 28.8 samples/s +Step 24100/44000 | Loss: 2.9826 | LR: 2.19e-04 | 698.3ms/step | 28.6 samples/s +Step 24110/44000 | Loss: 2.9367 | LR: 2.19e-04 | 693.6ms/step | 28.8 samples/s +Step 24120/44000 | Loss: 2.9220 | LR: 2.18e-04 | 693.5ms/step | 28.8 samples/s +Step 24130/44000 | Loss: 2.9528 | LR: 2.18e-04 | 694.2ms/step | 28.8 samples/s +Step 24140/44000 | Loss: 2.9671 | LR: 2.18e-04 | 695.4ms/step | 28.8 samples/s +Step 24150/44000 | Loss: 2.9303 | LR: 2.18e-04 | 694.9ms/step | 28.8 samples/s +Step 24160/44000 | Loss: 2.9834 | LR: 2.18e-04 | 696.8ms/step | 28.7 samples/s +Step 24170/44000 | Loss: 2.9744 | LR: 2.17e-04 | 696.2ms/step | 28.7 samples/s +Step 24180/44000 | Loss: 2.9354 | LR: 2.17e-04 | 693.9ms/step | 28.8 samples/s +Step 24190/44000 | Loss: 2.9699 | LR: 2.17e-04 | 694.0ms/step | 28.8 samples/s +Step 24200/44000 | Loss: 2.9490 | LR: 2.17e-04 | 694.0ms/step | 28.8 samples/s +Step 24210/44000 | Loss: 2.9474 | LR: 2.17e-04 | 694.3ms/step | 28.8 samples/s +Step 24220/44000 | Loss: 2.9531 | LR: 2.17e-04 | 693.5ms/step | 28.8 samples/s +Step 24230/44000 | Loss: 2.9817 | LR: 2.16e-04 | 694.3ms/step | 28.8 samples/s +Step 24240/44000 | Loss: 2.8868 | LR: 2.16e-04 | 693.2ms/step | 28.8 samples/s +Step 24250/44000 | Loss: 2.9422 | LR: 2.16e-04 | 693.6ms/step | 28.8 samples/s +Validation at step 24250: Loss = 2.9734, Perplexity = 19.56 +Step 24260/44000 | Loss: 2.9494 | LR: 2.16e-04 | 1263.9ms/step | 15.8 samples/s +Step 24270/44000 | Loss: 2.9076 | LR: 2.16e-04 | 694.1ms/step | 28.8 samples/s +Step 24280/44000 | Loss: 2.9511 | LR: 2.16e-04 | 694.5ms/step | 28.8 samples/s +Step 24290/44000 | Loss: 2.9632 | LR: 2.15e-04 | 694.8ms/step | 28.8 samples/s +Step 24300/44000 | Loss: 2.9694 | LR: 2.15e-04 | 692.9ms/step | 28.9 samples/s +Step 24310/44000 | Loss: 2.9822 | LR: 2.15e-04 | 692.8ms/step | 28.9 samples/s +Step 24320/44000 | Loss: 2.9535 | LR: 2.15e-04 | 692.4ms/step | 28.9 samples/s +Step 24330/44000 | Loss: 2.9341 | LR: 2.15e-04 | 694.1ms/step | 28.8 samples/s +Step 24340/44000 | Loss: 2.9447 | LR: 2.14e-04 | 693.7ms/step | 28.8 samples/s +Step 24350/44000 | Loss: 2.9639 | LR: 2.14e-04 | 694.2ms/step | 28.8 samples/s +Step 24360/44000 | Loss: 2.9786 | LR: 2.14e-04 | 696.1ms/step | 28.7 samples/s +Step 24370/44000 | Loss: 2.9612 | LR: 2.14e-04 | 694.1ms/step | 28.8 samples/s +Step 24380/44000 | Loss: 2.9242 | LR: 2.14e-04 | 695.1ms/step | 28.8 samples/s +Step 24390/44000 | Loss: 2.9499 | LR: 2.14e-04 | 695.1ms/step | 28.8 samples/s +Step 24400/44000 | Loss: 2.9214 | LR: 2.13e-04 | 694.5ms/step | 28.8 samples/s +Step 24410/44000 | Loss: 2.9496 | LR: 2.13e-04 | 692.7ms/step | 28.9 samples/s +Step 24420/44000 | Loss: 2.9716 | LR: 2.13e-04 | 693.6ms/step | 28.8 samples/s +Step 24430/44000 | Loss: 2.9323 | LR: 2.13e-04 | 694.5ms/step | 28.8 samples/s +Step 24440/44000 | Loss: 2.9360 | LR: 2.13e-04 | 693.8ms/step | 28.8 samples/s +Step 24450/44000 | Loss: 2.9831 | LR: 2.12e-04 | 695.1ms/step | 28.8 samples/s +Step 24460/44000 | Loss: 2.9646 | LR: 2.12e-04 | 697.0ms/step | 28.7 samples/s +Step 24470/44000 | Loss: 2.9965 | LR: 2.12e-04 | 696.2ms/step | 28.7 samples/s +Step 24480/44000 | Loss: 2.8992 | LR: 2.12e-04 | 694.6ms/step | 28.8 samples/s +Step 24490/44000 | Loss: 2.9837 | LR: 2.12e-04 | 695.9ms/step | 28.7 samples/s +Step 24500/44000 | Loss: 2.9500 | LR: 2.12e-04 | 695.5ms/step | 28.8 samples/s +Validation at step 24500: Loss = 2.9729, Perplexity = 19.55 +Step 24510/44000 | Loss: 2.9446 | LR: 2.11e-04 | 1262.5ms/step | 15.8 samples/s +Step 24520/44000 | Loss: 2.9583 | LR: 2.11e-04 | 696.0ms/step | 28.7 samples/s +Step 24530/44000 | Loss: 2.9694 | LR: 2.11e-04 | 695.0ms/step | 28.8 samples/s +Step 24540/44000 | Loss: 2.9600 | LR: 2.11e-04 | 696.7ms/step | 28.7 samples/s +Step 24550/44000 | Loss: 2.9528 | LR: 2.11e-04 | 696.1ms/step | 28.7 samples/s +Step 24560/44000 | Loss: 2.9289 | LR: 2.10e-04 | 693.2ms/step | 28.9 samples/s +Step 24570/44000 | Loss: 2.9621 | LR: 2.10e-04 | 694.9ms/step | 28.8 samples/s +Step 24580/44000 | Loss: 3.0156 | LR: 2.10e-04 | 693.4ms/step | 28.8 samples/s +Step 24590/44000 | Loss: 2.9287 | LR: 2.10e-04 | 694.0ms/step | 28.8 samples/s +Step 24600/44000 | Loss: 2.9383 | LR: 2.10e-04 | 694.8ms/step | 28.8 samples/s +Step 24610/44000 | Loss: 2.9731 | LR: 2.10e-04 | 694.1ms/step | 28.8 samples/s +Step 24620/44000 | Loss: 2.9667 | LR: 2.09e-04 | 696.7ms/step | 28.7 samples/s +Step 24630/44000 | Loss: 2.9560 | LR: 2.09e-04 | 694.7ms/step | 28.8 samples/s +Step 24640/44000 | Loss: 2.9849 | LR: 2.09e-04 | 696.3ms/step | 28.7 samples/s +Step 24650/44000 | Loss: 2.9860 | LR: 2.09e-04 | 695.8ms/step | 28.7 samples/s +Step 24660/44000 | Loss: 2.9431 | LR: 2.09e-04 | 693.8ms/step | 28.8 samples/s +Step 24670/44000 | Loss: 2.9219 | LR: 2.09e-04 | 696.2ms/step | 28.7 samples/s +Step 24680/44000 | Loss: 2.9740 | LR: 2.08e-04 | 695.1ms/step | 28.8 samples/s +Step 24690/44000 | Loss: 2.9462 | LR: 2.08e-04 | 694.7ms/step | 28.8 samples/s +Step 24700/44000 | Loss: 2.9777 | LR: 2.08e-04 | 696.8ms/step | 28.7 samples/s +Step 24710/44000 | Loss: 2.9459 | LR: 2.08e-04 | 694.6ms/step | 28.8 samples/s +Step 24720/44000 | Loss: 2.9630 | LR: 2.08e-04 | 693.2ms/step | 28.9 samples/s +Step 24730/44000 | Loss: 2.9231 | LR: 2.07e-04 | 695.3ms/step | 28.8 samples/s +Step 24740/44000 | Loss: 2.9480 | LR: 2.07e-04 | 695.5ms/step | 28.8 samples/s +Step 24750/44000 | Loss: 2.9619 | LR: 2.07e-04 | 693.9ms/step | 28.8 samples/s +Validation at step 24750: Loss = 2.9702, Perplexity = 19.50 +Step 24760/44000 | Loss: 2.9041 | LR: 2.07e-04 | 1264.1ms/step | 15.8 samples/s +Step 24770/44000 | Loss: 2.9252 | LR: 2.07e-04 | 695.0ms/step | 28.8 samples/s +Step 24780/44000 | Loss: 2.9828 | LR: 2.07e-04 | 694.0ms/step | 28.8 samples/s +Step 24790/44000 | Loss: 2.9368 | LR: 2.06e-04 | 697.5ms/step | 28.7 samples/s +Step 24800/44000 | Loss: 2.9464 | LR: 2.06e-04 | 699.0ms/step | 28.6 samples/s +Step 24810/44000 | Loss: 2.9696 | LR: 2.06e-04 | 694.2ms/step | 28.8 samples/s +Step 24820/44000 | Loss: 2.9866 | LR: 2.06e-04 | 696.5ms/step | 28.7 samples/s +Step 24830/44000 | Loss: 2.9917 | LR: 2.06e-04 | 697.5ms/step | 28.7 samples/s +Step 24840/44000 | Loss: 2.9683 | LR: 2.05e-04 | 695.1ms/step | 28.8 samples/s +Step 24850/44000 | Loss: 2.9671 | LR: 2.05e-04 | 696.3ms/step | 28.7 samples/s +Step 24860/44000 | Loss: 2.9472 | LR: 2.05e-04 | 693.6ms/step | 28.8 samples/s +Step 24870/44000 | Loss: 2.9674 | LR: 2.05e-04 | 698.1ms/step | 28.6 samples/s +Step 24880/44000 | Loss: 2.9866 | LR: 2.05e-04 | 699.3ms/step | 28.6 samples/s +Step 24890/44000 | Loss: 2.9851 | LR: 2.05e-04 | 694.6ms/step | 28.8 samples/s +Step 24900/44000 | Loss: 2.9754 | LR: 2.04e-04 | 696.7ms/step | 28.7 samples/s +Step 24910/44000 | Loss: 2.9809 | LR: 2.04e-04 | 695.2ms/step | 28.8 samples/s +Step 24920/44000 | Loss: 2.9749 | LR: 2.04e-04 | 695.0ms/step | 28.8 samples/s +Step 24930/44000 | Loss: 2.9295 | LR: 2.04e-04 | 696.7ms/step | 28.7 samples/s +Step 24940/44000 | Loss: 2.9899 | LR: 2.04e-04 | 698.1ms/step | 28.6 samples/s +Step 24950/44000 | Loss: 2.9476 | LR: 2.04e-04 | 694.4ms/step | 28.8 samples/s +Step 24960/44000 | Loss: 2.9538 | LR: 2.03e-04 | 699.1ms/step | 28.6 samples/s +Step 24970/44000 | Loss: 2.9495 | LR: 2.03e-04 | 709.2ms/step | 28.2 samples/s +Step 24980/44000 | Loss: 2.9958 | LR: 2.03e-04 | 698.4ms/step | 28.6 samples/s +Step 24990/44000 | Loss: 2.9869 | LR: 2.03e-04 | 697.9ms/step | 28.7 samples/s +Step 25000/44000 | Loss: 3.0003 | LR: 2.03e-04 | 696.8ms/step | 28.7 samples/s +Validation at step 25000: Loss = 2.9680, Perplexity = 19.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-25000 +Step 25010/44000 | Loss: 2.9710 | LR: 2.02e-04 | 1577.6ms/step | 12.7 samples/s +Step 25020/44000 | Loss: 2.9506 | LR: 2.02e-04 | 698.9ms/step | 28.6 samples/s +Step 25030/44000 | Loss: 2.9459 | LR: 2.02e-04 | 694.3ms/step | 28.8 samples/s +Step 25040/44000 | Loss: 2.9249 | LR: 2.02e-04 | 695.8ms/step | 28.7 samples/s +Step 25050/44000 | Loss: 2.9521 | LR: 2.02e-04 | 695.9ms/step | 28.7 samples/s +Step 25060/44000 | Loss: 2.9648 | LR: 2.02e-04 | 696.6ms/step | 28.7 samples/s +Step 25070/44000 | Loss: 2.9555 | LR: 2.01e-04 | 694.6ms/step | 28.8 samples/s +Step 25080/44000 | Loss: 2.9603 | LR: 2.01e-04 | 694.5ms/step | 28.8 samples/s +Step 25090/44000 | Loss: 2.9706 | LR: 2.01e-04 | 694.8ms/step | 28.8 samples/s +Step 25100/44000 | Loss: 2.9637 | LR: 2.01e-04 | 693.1ms/step | 28.9 samples/s +Step 25110/44000 | Loss: 2.9305 | LR: 2.01e-04 | 695.9ms/step | 28.7 samples/s +Step 25120/44000 | Loss: 2.9458 | LR: 2.00e-04 | 696.5ms/step | 28.7 samples/s +Step 25130/44000 | Loss: 2.9630 | LR: 2.00e-04 | 693.6ms/step | 28.8 samples/s +Step 25140/44000 | Loss: 2.9429 | LR: 2.00e-04 | 696.0ms/step | 28.7 samples/s +Step 25150/44000 | Loss: 2.9927 | LR: 2.00e-04 | 696.8ms/step | 28.7 samples/s +Step 25160/44000 | Loss: 2.9880 | LR: 2.00e-04 | 695.7ms/step | 28.7 samples/s +Step 25170/44000 | Loss: 2.9759 | LR: 2.00e-04 | 693.6ms/step | 28.8 samples/s +Step 25180/44000 | Loss: 2.9873 | LR: 1.99e-04 | 695.9ms/step | 28.7 samples/s +Step 25190/44000 | Loss: 2.9460 | LR: 1.99e-04 | 695.0ms/step | 28.8 samples/s +Step 25200/44000 | Loss: 2.9898 | LR: 1.99e-04 | 694.1ms/step | 28.8 samples/s +Step 25210/44000 | Loss: 2.9313 | LR: 1.99e-04 | 696.2ms/step | 28.7 samples/s +Step 25220/44000 | Loss: 2.9678 | LR: 1.99e-04 | 693.8ms/step | 28.8 samples/s +Step 25230/44000 | Loss: 2.9677 | LR: 1.99e-04 | 693.2ms/step | 28.8 samples/s +Step 25240/44000 | Loss: 2.9384 | LR: 1.98e-04 | 694.6ms/step | 28.8 samples/s +Step 25250/44000 | Loss: 2.9356 | LR: 1.98e-04 | 694.3ms/step | 28.8 samples/s +Validation at step 25250: Loss = 2.9661, Perplexity = 19.42 +Step 25260/44000 | Loss: 2.9790 | LR: 1.98e-04 | 1267.7ms/step | 15.8 samples/s +Step 25270/44000 | Loss: 2.9422 | LR: 1.98e-04 | 698.9ms/step | 28.6 samples/s +Step 25280/44000 | Loss: 2.9661 | LR: 1.98e-04 | 695.7ms/step | 28.7 samples/s +Step 25290/44000 | Loss: 2.9924 | LR: 1.97e-04 | 695.6ms/step | 28.8 samples/s +Step 25300/44000 | Loss: 2.9693 | LR: 1.97e-04 | 696.3ms/step | 28.7 samples/s +Step 25310/44000 | Loss: 2.9461 | LR: 1.97e-04 | 693.9ms/step | 28.8 samples/s +Step 25320/44000 | Loss: 2.9600 | LR: 1.97e-04 | 694.4ms/step | 28.8 samples/s +Step 25330/44000 | Loss: 2.9497 | LR: 1.97e-04 | 696.1ms/step | 28.7 samples/s +Step 25340/44000 | Loss: 2.9770 | LR: 1.97e-04 | 694.1ms/step | 28.8 samples/s +Step 25350/44000 | Loss: 2.9838 | LR: 1.96e-04 | 695.5ms/step | 28.8 samples/s +Step 25360/44000 | Loss: 2.9916 | LR: 1.96e-04 | 693.3ms/step | 28.8 samples/s +Step 25370/44000 | Loss: 2.9372 | LR: 1.96e-04 | 697.5ms/step | 28.7 samples/s +Step 25380/44000 | Loss: 2.9151 | LR: 1.96e-04 | 692.8ms/step | 28.9 samples/s +Step 25390/44000 | Loss: 3.0042 | LR: 1.96e-04 | 695.8ms/step | 28.7 samples/s +Step 25400/44000 | Loss: 2.8953 | LR: 1.96e-04 | 695.7ms/step | 28.7 samples/s +Step 25410/44000 | Loss: 2.9530 | LR: 1.95e-04 | 693.7ms/step | 28.8 samples/s +Step 25420/44000 | Loss: 2.9909 | LR: 1.95e-04 | 693.5ms/step | 28.8 samples/s +Step 25430/44000 | Loss: 2.9394 | LR: 1.95e-04 | 693.2ms/step | 28.8 samples/s +Step 25440/44000 | Loss: 2.9452 | LR: 1.95e-04 | 693.6ms/step | 28.8 samples/s +Step 25450/44000 | Loss: 2.9399 | LR: 1.95e-04 | 694.2ms/step | 28.8 samples/s +Step 25460/44000 | Loss: 2.9605 | LR: 1.94e-04 | 706.6ms/step | 28.3 samples/s +Step 25470/44000 | Loss: 2.9466 | LR: 1.94e-04 | 694.6ms/step | 28.8 samples/s +Step 25480/44000 | Loss: 2.9025 | LR: 1.94e-04 | 693.4ms/step | 28.8 samples/s +Step 25490/44000 | Loss: 2.9046 | LR: 1.94e-04 | 694.5ms/step | 28.8 samples/s +Step 25500/44000 | Loss: 2.9644 | LR: 1.94e-04 | 695.9ms/step | 28.7 samples/s +Validation at step 25500: Loss = 2.9644, Perplexity = 19.38 +Step 25510/44000 | Loss: 2.9063 | LR: 1.94e-04 | 1267.0ms/step | 15.8 samples/s +Step 25520/44000 | Loss: 2.9626 | LR: 1.93e-04 | 694.9ms/step | 28.8 samples/s +Step 25530/44000 | Loss: 2.9229 | LR: 1.93e-04 | 695.3ms/step | 28.8 samples/s +Step 25540/44000 | Loss: 2.9540 | LR: 1.93e-04 | 693.7ms/step | 28.8 samples/s +Step 25550/44000 | Loss: 2.9332 | LR: 1.93e-04 | 694.2ms/step | 28.8 samples/s +Step 25560/44000 | Loss: 2.9530 | LR: 1.93e-04 | 693.2ms/step | 28.9 samples/s +Step 25570/44000 | Loss: 2.9153 | LR: 1.92e-04 | 692.8ms/step | 28.9 samples/s +Step 25580/44000 | Loss: 2.9622 | LR: 1.92e-04 | 694.9ms/step | 28.8 samples/s +Step 25590/44000 | Loss: 2.9560 | LR: 1.92e-04 | 694.9ms/step | 28.8 samples/s +Step 25600/44000 | Loss: 2.9408 | LR: 1.92e-04 | 693.6ms/step | 28.8 samples/s +Step 25610/44000 | Loss: 2.9317 | LR: 1.92e-04 | 694.7ms/step | 28.8 samples/s +Step 25620/44000 | Loss: 2.9516 | LR: 1.92e-04 | 694.2ms/step | 28.8 samples/s +Step 25630/44000 | Loss: 2.9842 | LR: 1.91e-04 | 694.1ms/step | 28.8 samples/s +Step 25640/44000 | Loss: 2.9700 | LR: 1.91e-04 | 693.5ms/step | 28.8 samples/s +Step 25650/44000 | Loss: 2.9381 | LR: 1.91e-04 | 694.6ms/step | 28.8 samples/s +Step 25660/44000 | Loss: 2.9412 | LR: 1.91e-04 | 692.6ms/step | 28.9 samples/s +Step 25670/44000 | Loss: 2.9426 | LR: 1.91e-04 | 692.0ms/step | 28.9 samples/s +Step 25680/44000 | Loss: 2.9162 | LR: 1.91e-04 | 693.1ms/step | 28.9 samples/s +Step 25690/44000 | Loss: 2.8851 | LR: 1.90e-04 | 694.0ms/step | 28.8 samples/s +Step 25700/44000 | Loss: 2.9948 | LR: 1.90e-04 | 692.6ms/step | 28.9 samples/s +Step 25710/44000 | Loss: 2.9589 | LR: 1.90e-04 | 693.2ms/step | 28.9 samples/s +Step 25720/44000 | Loss: 2.9546 | LR: 1.90e-04 | 693.0ms/step | 28.9 samples/s +Step 25730/44000 | Loss: 2.9672 | LR: 1.90e-04 | 691.2ms/step | 28.9 samples/s +Step 25740/44000 | Loss: 3.0030 | LR: 1.89e-04 | 694.7ms/step | 28.8 samples/s +Step 25750/44000 | Loss: 2.9712 | LR: 1.89e-04 | 693.8ms/step | 28.8 samples/s +Validation at step 25750: Loss = 2.9635, Perplexity = 19.37 +Step 25760/44000 | Loss: 2.9438 | LR: 1.89e-04 | 1261.8ms/step | 15.9 samples/s +Step 25770/44000 | Loss: 2.9457 | LR: 1.89e-04 | 695.9ms/step | 28.7 samples/s +Step 25780/44000 | Loss: 2.9734 | LR: 1.89e-04 | 695.4ms/step | 28.8 samples/s +Step 25790/44000 | Loss: 2.9423 | LR: 1.89e-04 | 693.5ms/step | 28.8 samples/s +Step 25800/44000 | Loss: 2.9345 | LR: 1.88e-04 | 695.6ms/step | 28.8 samples/s +Step 25810/44000 | Loss: 2.9675 | LR: 1.88e-04 | 694.3ms/step | 28.8 samples/s +Step 25820/44000 | Loss: 2.9609 | LR: 1.88e-04 | 693.9ms/step | 28.8 samples/s +Step 25830/44000 | Loss: 2.9439 | LR: 1.88e-04 | 693.2ms/step | 28.9 samples/s +Step 25840/44000 | Loss: 2.9537 | LR: 1.88e-04 | 694.9ms/step | 28.8 samples/s +Step 25850/44000 | Loss: 2.9264 | LR: 1.88e-04 | 695.5ms/step | 28.8 samples/s +Step 25860/44000 | Loss: 2.9309 | LR: 1.87e-04 | 695.8ms/step | 28.7 samples/s +Step 25870/44000 | Loss: 2.9511 | LR: 1.87e-04 | 694.6ms/step | 28.8 samples/s +Step 25880/44000 | Loss: 2.9514 | LR: 1.87e-04 | 693.8ms/step | 28.8 samples/s +Step 25890/44000 | Loss: 2.9375 | LR: 1.87e-04 | 694.6ms/step | 28.8 samples/s +Step 25900/44000 | Loss: 2.9034 | LR: 1.87e-04 | 697.5ms/step | 28.7 samples/s +Step 25910/44000 | Loss: 2.9761 | LR: 1.87e-04 | 694.6ms/step | 28.8 samples/s +Step 25920/44000 | Loss: 2.9971 | LR: 1.86e-04 | 694.5ms/step | 28.8 samples/s +Step 25930/44000 | Loss: 2.9426 | LR: 1.86e-04 | 694.9ms/step | 28.8 samples/s +Step 25940/44000 | Loss: 2.9972 | LR: 1.86e-04 | 695.0ms/step | 28.8 samples/s +Step 25950/44000 | Loss: 2.8967 | LR: 1.86e-04 | 694.2ms/step | 28.8 samples/s +Step 25960/44000 | Loss: 2.9252 | LR: 1.86e-04 | 694.8ms/step | 28.8 samples/s +Step 25970/44000 | Loss: 2.9054 | LR: 1.85e-04 | 694.7ms/step | 28.8 samples/s +Step 25980/44000 | Loss: 2.9860 | LR: 1.85e-04 | 693.8ms/step | 28.8 samples/s +Step 25990/44000 | Loss: 2.9412 | LR: 1.85e-04 | 695.3ms/step | 28.8 samples/s +Step 26000/44000 | Loss: 2.9175 | LR: 1.85e-04 | 695.5ms/step | 28.8 samples/s +Validation at step 26000: Loss = 2.9612, Perplexity = 19.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-26000 +Step 26010/44000 | Loss: 2.9807 | LR: 1.85e-04 | 1568.8ms/step | 12.7 samples/s +Step 26020/44000 | Loss: 2.9454 | LR: 1.85e-04 | 693.7ms/step | 28.8 samples/s +Step 26030/44000 | Loss: 2.9528 | LR: 1.84e-04 | 694.1ms/step | 28.8 samples/s +Step 26040/44000 | Loss: 2.9537 | LR: 1.84e-04 | 693.1ms/step | 28.9 samples/s +Step 26050/44000 | Loss: 2.9484 | LR: 1.84e-04 | 695.0ms/step | 28.8 samples/s +Step 26060/44000 | Loss: 2.9380 | LR: 1.84e-04 | 693.7ms/step | 28.8 samples/s +Step 26070/44000 | Loss: 2.9688 | LR: 1.84e-04 | 694.0ms/step | 28.8 samples/s +Step 26080/44000 | Loss: 2.9277 | LR: 1.84e-04 | 693.9ms/step | 28.8 samples/s +Step 26090/44000 | Loss: 2.9830 | LR: 1.83e-04 | 693.7ms/step | 28.8 samples/s +Step 26100/44000 | Loss: 2.9635 | LR: 1.83e-04 | 692.9ms/step | 28.9 samples/s +Step 26110/44000 | Loss: 2.9471 | LR: 1.83e-04 | 694.0ms/step | 28.8 samples/s +Step 26120/44000 | Loss: 2.9093 | LR: 1.83e-04 | 696.0ms/step | 28.7 samples/s +Step 26130/44000 | Loss: 2.9184 | LR: 1.83e-04 | 695.3ms/step | 28.8 samples/s +Step 26140/44000 | Loss: 2.9149 | LR: 1.82e-04 | 694.2ms/step | 28.8 samples/s +Step 26150/44000 | Loss: 2.9913 | LR: 1.82e-04 | 695.9ms/step | 28.7 samples/s +Step 26160/44000 | Loss: 2.9515 | LR: 1.82e-04 | 694.2ms/step | 28.8 samples/s +Step 26170/44000 | Loss: 2.9502 | LR: 1.82e-04 | 696.3ms/step | 28.7 samples/s +Step 26180/44000 | Loss: 2.9845 | LR: 1.82e-04 | 695.5ms/step | 28.8 samples/s +Step 26190/44000 | Loss: 2.9735 | LR: 1.82e-04 | 696.9ms/step | 28.7 samples/s +Step 26200/44000 | Loss: 2.9726 | LR: 1.81e-04 | 694.8ms/step | 28.8 samples/s +Step 26210/44000 | Loss: 2.9515 | LR: 1.81e-04 | 694.4ms/step | 28.8 samples/s +Step 26220/44000 | Loss: 2.9701 | LR: 1.81e-04 | 693.7ms/step | 28.8 samples/s +Step 26230/44000 | Loss: 2.8925 | LR: 1.81e-04 | 693.9ms/step | 28.8 samples/s +Step 26240/44000 | Loss: 2.9099 | LR: 1.81e-04 | 695.1ms/step | 28.8 samples/s +Step 26250/44000 | Loss: 2.9433 | LR: 1.81e-04 | 694.5ms/step | 28.8 samples/s +Validation at step 26250: Loss = 2.9608, Perplexity = 19.31 +Step 26260/44000 | Loss: 2.9168 | LR: 1.80e-04 | 1263.2ms/step | 15.8 samples/s +Step 26270/44000 | Loss: 2.9168 | LR: 1.80e-04 | 694.8ms/step | 28.8 samples/s +Step 26280/44000 | Loss: 2.9609 | LR: 1.80e-04 | 693.6ms/step | 28.8 samples/s +Step 26290/44000 | Loss: 2.9429 | LR: 1.80e-04 | 694.2ms/step | 28.8 samples/s +Step 26300/44000 | Loss: 2.9211 | LR: 1.80e-04 | 695.3ms/step | 28.8 samples/s +Step 26310/44000 | Loss: 2.9223 | LR: 1.80e-04 | 694.7ms/step | 28.8 samples/s +Step 26320/44000 | Loss: 2.9258 | LR: 1.79e-04 | 694.1ms/step | 28.8 samples/s +Step 26330/44000 | Loss: 2.9541 | LR: 1.79e-04 | 696.1ms/step | 28.7 samples/s +Step 26340/44000 | Loss: 2.9303 | LR: 1.79e-04 | 692.9ms/step | 28.9 samples/s +Step 26350/44000 | Loss: 2.9061 | LR: 1.79e-04 | 695.0ms/step | 28.8 samples/s +Step 26360/44000 | Loss: 2.9553 | LR: 1.79e-04 | 693.8ms/step | 28.8 samples/s +Step 26370/44000 | Loss: 2.9686 | LR: 1.78e-04 | 695.7ms/step | 28.7 samples/s +Step 26380/44000 | Loss: 2.9416 | LR: 1.78e-04 | 694.9ms/step | 28.8 samples/s +Step 26390/44000 | Loss: 2.9487 | LR: 1.78e-04 | 693.2ms/step | 28.9 samples/s +Step 26400/44000 | Loss: 2.9301 | LR: 1.78e-04 | 694.8ms/step | 28.8 samples/s +Step 26410/44000 | Loss: 2.9171 | LR: 1.78e-04 | 692.5ms/step | 28.9 samples/s +Step 26420/44000 | Loss: 2.9647 | LR: 1.78e-04 | 693.1ms/step | 28.9 samples/s +Step 26430/44000 | Loss: 2.9427 | LR: 1.77e-04 | 693.1ms/step | 28.9 samples/s +Step 26440/44000 | Loss: 2.9781 | LR: 1.77e-04 | 694.0ms/step | 28.8 samples/s +Step 26450/44000 | Loss: 2.9754 | LR: 1.77e-04 | 693.3ms/step | 28.8 samples/s +Step 26460/44000 | Loss: 2.9327 | LR: 1.77e-04 | 693.5ms/step | 28.8 samples/s +Step 26470/44000 | Loss: 2.9679 | LR: 1.77e-04 | 694.1ms/step | 28.8 samples/s +Step 26480/44000 | Loss: 2.9586 | LR: 1.77e-04 | 694.2ms/step | 28.8 samples/s +Step 26490/44000 | Loss: 2.9708 | LR: 1.76e-04 | 694.5ms/step | 28.8 samples/s +Step 26500/44000 | Loss: 2.9173 | LR: 1.76e-04 | 694.0ms/step | 28.8 samples/s +Validation at step 26500: Loss = 2.9590, Perplexity = 19.28 +Step 26510/44000 | Loss: 3.0015 | LR: 1.76e-04 | 1262.2ms/step | 15.8 samples/s +Step 26520/44000 | Loss: 2.9365 | LR: 1.76e-04 | 692.5ms/step | 28.9 samples/s +Step 26530/44000 | Loss: 2.9692 | LR: 1.76e-04 | 694.0ms/step | 28.8 samples/s +Step 26540/44000 | Loss: 2.9813 | LR: 1.76e-04 | 693.4ms/step | 28.8 samples/s +Step 26550/44000 | Loss: 2.9274 | LR: 1.75e-04 | 694.1ms/step | 28.8 samples/s +Step 26560/44000 | Loss: 2.9556 | LR: 1.75e-04 | 696.6ms/step | 28.7 samples/s +Step 26570/44000 | Loss: 2.9385 | LR: 1.75e-04 | 693.9ms/step | 28.8 samples/s +Step 26580/44000 | Loss: 2.9406 | LR: 1.75e-04 | 693.0ms/step | 28.9 samples/s +Step 26590/44000 | Loss: 2.9275 | LR: 1.75e-04 | 693.7ms/step | 28.8 samples/s +Step 26600/44000 | Loss: 2.9336 | LR: 1.74e-04 | 692.8ms/step | 28.9 samples/s +Step 26610/44000 | Loss: 2.9681 | LR: 1.74e-04 | 693.9ms/step | 28.8 samples/s +Step 26620/44000 | Loss: 2.9653 | LR: 1.74e-04 | 694.1ms/step | 28.8 samples/s +Step 26630/44000 | Loss: 2.9824 | LR: 1.74e-04 | 694.4ms/step | 28.8 samples/s +Step 26640/44000 | Loss: 3.0000 | LR: 1.74e-04 | 694.5ms/step | 28.8 samples/s +Step 26650/44000 | Loss: 2.9343 | LR: 1.74e-04 | 695.0ms/step | 28.8 samples/s +Step 26660/44000 | Loss: 2.9330 | LR: 1.73e-04 | 693.3ms/step | 28.8 samples/s +Step 26670/44000 | Loss: 2.9114 | LR: 1.73e-04 | 694.6ms/step | 28.8 samples/s +Step 26680/44000 | Loss: 2.9975 | LR: 1.73e-04 | 695.5ms/step | 28.8 samples/s +Step 26690/44000 | Loss: 2.9419 | LR: 1.73e-04 | 694.9ms/step | 28.8 samples/s +Step 26700/44000 | Loss: 2.9870 | LR: 1.73e-04 | 694.9ms/step | 28.8 samples/s +Step 26710/44000 | Loss: 2.8914 | LR: 1.73e-04 | 695.7ms/step | 28.7 samples/s +Step 26720/44000 | Loss: 2.9308 | LR: 1.72e-04 | 695.9ms/step | 28.7 samples/s +Step 26730/44000 | Loss: 2.9431 | LR: 1.72e-04 | 694.1ms/step | 28.8 samples/s +Step 26740/44000 | Loss: 2.9551 | LR: 1.72e-04 | 695.5ms/step | 28.8 samples/s +Step 26750/44000 | Loss: 2.9076 | LR: 1.72e-04 | 699.2ms/step | 28.6 samples/s +Validation at step 26750: Loss = 2.9580, Perplexity = 19.26 +Step 26760/44000 | Loss: 2.9282 | LR: 1.72e-04 | 1262.8ms/step | 15.8 samples/s +Step 26770/44000 | Loss: 2.9026 | LR: 1.72e-04 | 695.4ms/step | 28.8 samples/s +Step 26780/44000 | Loss: 2.9127 | LR: 1.71e-04 | 695.1ms/step | 28.8 samples/s +Step 26790/44000 | Loss: 2.9993 | LR: 1.71e-04 | 694.7ms/step | 28.8 samples/s +Step 26800/44000 | Loss: 2.9152 | LR: 1.71e-04 | 695.5ms/step | 28.8 samples/s +Step 26810/44000 | Loss: 2.9164 | LR: 1.71e-04 | 695.8ms/step | 28.7 samples/s +Step 26820/44000 | Loss: 2.9151 | LR: 1.71e-04 | 696.4ms/step | 28.7 samples/s +Step 26830/44000 | Loss: 2.9637 | LR: 1.71e-04 | 694.8ms/step | 28.8 samples/s +Step 26840/44000 | Loss: 2.9789 | LR: 1.70e-04 | 696.3ms/step | 28.7 samples/s +Step 26850/44000 | Loss: 2.9734 | LR: 1.70e-04 | 696.6ms/step | 28.7 samples/s +Step 26860/44000 | Loss: 2.9293 | LR: 1.70e-04 | 696.0ms/step | 28.7 samples/s +Step 26870/44000 | Loss: 2.9083 | LR: 1.70e-04 | 697.4ms/step | 28.7 samples/s +Step 26880/44000 | Loss: 3.0071 | LR: 1.70e-04 | 698.5ms/step | 28.6 samples/s +Step 26890/44000 | Loss: 2.9342 | LR: 1.69e-04 | 694.5ms/step | 28.8 samples/s +Step 26900/44000 | Loss: 2.9672 | LR: 1.69e-04 | 694.8ms/step | 28.8 samples/s +Step 26910/44000 | Loss: 2.9720 | LR: 1.69e-04 | 694.7ms/step | 28.8 samples/s +Step 26920/44000 | Loss: 2.9854 | LR: 1.69e-04 | 692.9ms/step | 28.9 samples/s +Step 26930/44000 | Loss: 2.9287 | LR: 1.69e-04 | 696.6ms/step | 28.7 samples/s +Step 26940/44000 | Loss: 2.9259 | LR: 1.69e-04 | 693.9ms/step | 28.8 samples/s +Step 26950/44000 | Loss: 2.9331 | LR: 1.68e-04 | 698.8ms/step | 28.6 samples/s +Step 26960/44000 | Loss: 2.9370 | LR: 1.68e-04 | 692.7ms/step | 28.9 samples/s +Step 26970/44000 | Loss: 2.9425 | LR: 1.68e-04 | 693.4ms/step | 28.8 samples/s +Step 26980/44000 | Loss: 2.9603 | LR: 1.68e-04 | 694.2ms/step | 28.8 samples/s +Step 26990/44000 | Loss: 2.9384 | LR: 1.68e-04 | 699.7ms/step | 28.6 samples/s +Step 27000/44000 | Loss: 2.9456 | LR: 1.68e-04 | 695.5ms/step | 28.8 samples/s +Validation at step 27000: Loss = 2.9558, Perplexity = 19.22 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-27000 +Step 27010/44000 | Loss: 2.9288 | LR: 1.67e-04 | 1575.5ms/step | 12.7 samples/s +Step 27020/44000 | Loss: 2.9157 | LR: 1.67e-04 | 696.8ms/step | 28.7 samples/s +Step 27030/44000 | Loss: 2.9287 | LR: 1.67e-04 | 697.0ms/step | 28.7 samples/s +Step 27040/44000 | Loss: 2.9403 | LR: 1.67e-04 | 694.1ms/step | 28.8 samples/s +Step 27050/44000 | Loss: 2.9498 | LR: 1.67e-04 | 694.2ms/step | 28.8 samples/s +Step 27060/44000 | Loss: 2.9626 | LR: 1.67e-04 | 695.5ms/step | 28.8 samples/s +Step 27070/44000 | Loss: 2.9368 | LR: 1.66e-04 | 694.4ms/step | 28.8 samples/s +Step 27080/44000 | Loss: 2.9436 | LR: 1.66e-04 | 696.6ms/step | 28.7 samples/s +Step 27090/44000 | Loss: 2.9432 | LR: 1.66e-04 | 694.8ms/step | 28.8 samples/s +Step 27100/44000 | Loss: 2.9022 | LR: 1.66e-04 | 694.1ms/step | 28.8 samples/s +Step 27110/44000 | Loss: 2.9531 | LR: 1.66e-04 | 694.1ms/step | 28.8 samples/s +Step 27120/44000 | Loss: 2.9231 | LR: 1.66e-04 | 694.8ms/step | 28.8 samples/s +Step 27130/44000 | Loss: 2.9085 | LR: 1.65e-04 | 695.4ms/step | 28.8 samples/s +Step 27140/44000 | Loss: 2.9090 | LR: 1.65e-04 | 693.9ms/step | 28.8 samples/s +Step 27150/44000 | Loss: 2.9561 | LR: 1.65e-04 | 694.4ms/step | 28.8 samples/s +Step 27160/44000 | Loss: 2.9289 | LR: 1.65e-04 | 695.2ms/step | 28.8 samples/s +Step 27170/44000 | Loss: 2.9728 | LR: 1.65e-04 | 692.9ms/step | 28.9 samples/s +Step 27180/44000 | Loss: 2.9309 | LR: 1.65e-04 | 694.7ms/step | 28.8 samples/s +Step 27190/44000 | Loss: 2.9510 | LR: 1.64e-04 | 693.9ms/step | 28.8 samples/s +Step 27200/44000 | Loss: 2.9479 | LR: 1.64e-04 | 693.9ms/step | 28.8 samples/s +Step 27210/44000 | Loss: 2.9252 | LR: 1.64e-04 | 694.0ms/step | 28.8 samples/s +Step 27220/44000 | Loss: 2.9404 | LR: 1.64e-04 | 694.6ms/step | 28.8 samples/s +Step 27230/44000 | Loss: 2.9393 | LR: 1.64e-04 | 696.3ms/step | 28.7 samples/s +Step 27240/44000 | Loss: 2.9472 | LR: 1.63e-04 | 693.7ms/step | 28.8 samples/s +Step 27250/44000 | Loss: 2.9670 | LR: 1.63e-04 | 693.7ms/step | 28.8 samples/s +Validation at step 27250: Loss = 2.9544, Perplexity = 19.19 +Step 27260/44000 | Loss: 2.9432 | LR: 1.63e-04 | 1259.5ms/step | 15.9 samples/s +Step 27270/44000 | Loss: 2.9473 | LR: 1.63e-04 | 693.1ms/step | 28.9 samples/s +Step 27280/44000 | Loss: 2.9348 | LR: 1.63e-04 | 696.3ms/step | 28.7 samples/s +Step 27290/44000 | Loss: 2.9581 | LR: 1.63e-04 | 694.5ms/step | 28.8 samples/s +Step 27300/44000 | Loss: 2.9507 | LR: 1.62e-04 | 693.8ms/step | 28.8 samples/s +Step 27310/44000 | Loss: 2.9155 | LR: 1.62e-04 | 694.3ms/step | 28.8 samples/s +Step 27320/44000 | Loss: 2.9481 | LR: 1.62e-04 | 703.1ms/step | 28.4 samples/s +Step 27330/44000 | Loss: 2.9299 | LR: 1.62e-04 | 695.3ms/step | 28.8 samples/s +Step 27340/44000 | Loss: 2.9282 | LR: 1.62e-04 | 693.0ms/step | 28.9 samples/s +Step 27350/44000 | Loss: 2.9528 | LR: 1.62e-04 | 694.1ms/step | 28.8 samples/s +Step 27360/44000 | Loss: 2.9204 | LR: 1.61e-04 | 693.9ms/step | 28.8 samples/s +Step 27370/44000 | Loss: 3.0052 | LR: 1.61e-04 | 697.6ms/step | 28.7 samples/s +Step 27380/44000 | Loss: 2.9715 | LR: 1.61e-04 | 694.3ms/step | 28.8 samples/s +Step 27390/44000 | Loss: 2.9957 | LR: 1.61e-04 | 693.9ms/step | 28.8 samples/s +Step 27400/44000 | Loss: 2.9057 | LR: 1.61e-04 | 694.1ms/step | 28.8 samples/s +Step 27410/44000 | Loss: 2.8881 | LR: 1.61e-04 | 696.2ms/step | 28.7 samples/s +Step 27420/44000 | Loss: 2.9282 | LR: 1.60e-04 | 694.3ms/step | 28.8 samples/s +Step 27430/44000 | Loss: 2.9751 | LR: 1.60e-04 | 695.3ms/step | 28.8 samples/s +Step 27440/44000 | Loss: 2.9170 | LR: 1.60e-04 | 694.6ms/step | 28.8 samples/s +Step 27450/44000 | Loss: 2.9559 | LR: 1.60e-04 | 697.8ms/step | 28.7 samples/s +Step 27460/44000 | Loss: 2.9688 | LR: 1.60e-04 | 694.8ms/step | 28.8 samples/s +Step 27470/44000 | Loss: 2.9210 | LR: 1.60e-04 | 696.1ms/step | 28.7 samples/s +Step 27480/44000 | Loss: 2.9708 | LR: 1.59e-04 | 695.7ms/step | 28.7 samples/s +Step 27490/44000 | Loss: 2.9484 | LR: 1.59e-04 | 693.5ms/step | 28.8 samples/s +Step 27500/44000 | Loss: 2.9521 | LR: 1.59e-04 | 694.5ms/step | 28.8 samples/s +Validation at step 27500: Loss = 2.9536, Perplexity = 19.17 +Step 27510/44000 | Loss: 2.9461 | LR: 1.59e-04 | 1267.5ms/step | 15.8 samples/s +Step 27520/44000 | Loss: 2.9222 | LR: 1.59e-04 | 694.5ms/step | 28.8 samples/s +Step 27530/44000 | Loss: 2.9269 | LR: 1.59e-04 | 694.9ms/step | 28.8 samples/s +Step 27540/44000 | Loss: 2.9542 | LR: 1.58e-04 | 695.1ms/step | 28.8 samples/s +Step 27550/44000 | Loss: 2.9545 | LR: 1.58e-04 | 695.4ms/step | 28.8 samples/s +Step 27560/44000 | Loss: 2.9465 | LR: 1.58e-04 | 695.3ms/step | 28.8 samples/s +Step 27570/44000 | Loss: 2.9570 | LR: 1.58e-04 | 695.0ms/step | 28.8 samples/s +Step 27580/44000 | Loss: 2.9650 | LR: 1.58e-04 | 696.9ms/step | 28.7 samples/s +Step 27590/44000 | Loss: 2.9533 | LR: 1.58e-04 | 694.5ms/step | 28.8 samples/s +Step 27600/44000 | Loss: 2.9723 | LR: 1.57e-04 | 695.6ms/step | 28.8 samples/s +Step 27610/44000 | Loss: 2.8886 | LR: 1.57e-04 | 695.2ms/step | 28.8 samples/s +Step 27620/44000 | Loss: 2.9584 | LR: 1.57e-04 | 694.5ms/step | 28.8 samples/s +Step 27630/44000 | Loss: 3.0024 | LR: 1.57e-04 | 695.6ms/step | 28.8 samples/s +Step 27640/44000 | Loss: 2.9250 | LR: 1.57e-04 | 695.2ms/step | 28.8 samples/s +Step 27650/44000 | Loss: 2.9237 | LR: 1.57e-04 | 695.4ms/step | 28.8 samples/s +Step 27660/44000 | Loss: 2.9827 | LR: 1.56e-04 | 695.6ms/step | 28.8 samples/s +Step 27670/44000 | Loss: 2.9345 | LR: 1.56e-04 | 694.7ms/step | 28.8 samples/s +Step 27680/44000 | Loss: 2.9386 | LR: 1.56e-04 | 696.3ms/step | 28.7 samples/s +Step 27690/44000 | Loss: 2.9815 | LR: 1.56e-04 | 695.0ms/step | 28.8 samples/s +Step 27700/44000 | Loss: 2.9602 | LR: 1.56e-04 | 696.0ms/step | 28.7 samples/s +Step 27710/44000 | Loss: 2.9419 | LR: 1.56e-04 | 696.9ms/step | 28.7 samples/s +Step 27720/44000 | Loss: 2.9403 | LR: 1.55e-04 | 695.1ms/step | 28.8 samples/s +Step 27730/44000 | Loss: 2.8614 | LR: 1.55e-04 | 694.8ms/step | 28.8 samples/s +Step 27740/44000 | Loss: 2.9624 | LR: 1.55e-04 | 692.5ms/step | 28.9 samples/s +Step 27750/44000 | Loss: 2.9275 | LR: 1.55e-04 | 695.1ms/step | 28.8 samples/s +Validation at step 27750: Loss = 2.9520, Perplexity = 19.14 +Step 27760/44000 | Loss: 2.9260 | LR: 1.55e-04 | 1267.0ms/step | 15.8 samples/s +Step 27770/44000 | Loss: 2.9074 | LR: 1.55e-04 | 696.0ms/step | 28.7 samples/s +Step 27780/44000 | Loss: 2.9017 | LR: 1.54e-04 | 695.3ms/step | 28.8 samples/s +Step 27790/44000 | Loss: 2.9099 | LR: 1.54e-04 | 695.8ms/step | 28.7 samples/s +Step 27800/44000 | Loss: 2.9772 | LR: 1.54e-04 | 695.1ms/step | 28.8 samples/s +Step 27810/44000 | Loss: 2.9343 | LR: 1.54e-04 | 694.0ms/step | 28.8 samples/s +Step 27820/44000 | Loss: 2.9408 | LR: 1.54e-04 | 694.0ms/step | 28.8 samples/s +Step 27830/44000 | Loss: 2.9535 | LR: 1.54e-04 | 693.6ms/step | 28.8 samples/s +Step 27840/44000 | Loss: 2.9719 | LR: 1.53e-04 | 692.8ms/step | 28.9 samples/s +Step 27850/44000 | Loss: 2.9243 | LR: 1.53e-04 | 693.4ms/step | 28.8 samples/s +Step 27860/44000 | Loss: 2.9159 | LR: 1.53e-04 | 694.8ms/step | 28.8 samples/s +Step 27870/44000 | Loss: 2.9710 | LR: 1.53e-04 | 693.4ms/step | 28.8 samples/s +Step 27880/44000 | Loss: 2.9368 | LR: 1.53e-04 | 694.9ms/step | 28.8 samples/s +Step 27890/44000 | Loss: 2.9717 | LR: 1.53e-04 | 696.6ms/step | 28.7 samples/s +Step 27900/44000 | Loss: 2.9281 | LR: 1.52e-04 | 693.6ms/step | 28.8 samples/s +Step 27910/44000 | Loss: 2.9308 | LR: 1.52e-04 | 693.5ms/step | 28.8 samples/s +Step 27920/44000 | Loss: 2.9730 | LR: 1.52e-04 | 698.1ms/step | 28.6 samples/s +Step 27930/44000 | Loss: 2.9282 | LR: 1.52e-04 | 696.4ms/step | 28.7 samples/s +Step 27940/44000 | Loss: 2.9321 | LR: 1.52e-04 | 696.0ms/step | 28.7 samples/s +Step 27950/44000 | Loss: 2.9637 | LR: 1.52e-04 | 696.3ms/step | 28.7 samples/s +Step 27960/44000 | Loss: 2.9465 | LR: 1.51e-04 | 695.9ms/step | 28.7 samples/s +Step 27970/44000 | Loss: 2.9507 | LR: 1.51e-04 | 695.6ms/step | 28.8 samples/s +Step 27980/44000 | Loss: 2.9591 | LR: 1.51e-04 | 694.4ms/step | 28.8 samples/s +Step 27990/44000 | Loss: 2.9507 | LR: 1.51e-04 | 697.3ms/step | 28.7 samples/s +Step 28000/44000 | Loss: 2.9078 | LR: 1.51e-04 | 696.2ms/step | 28.7 samples/s +Validation at step 28000: Loss = 2.9515, Perplexity = 19.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-28000 +Step 28010/44000 | Loss: 2.8860 | LR: 1.51e-04 | 1592.9ms/step | 12.6 samples/s +Step 28020/44000 | Loss: 2.9575 | LR: 1.50e-04 | 693.9ms/step | 28.8 samples/s +Step 28030/44000 | Loss: 2.8899 | LR: 1.50e-04 | 693.5ms/step | 28.8 samples/s +Step 28040/44000 | Loss: 2.9291 | LR: 1.50e-04 | 695.7ms/step | 28.7 samples/s +Step 28050/44000 | Loss: 2.9381 | LR: 1.50e-04 | 695.5ms/step | 28.8 samples/s +Step 28060/44000 | Loss: 2.9094 | LR: 1.50e-04 | 695.8ms/step | 28.7 samples/s +Step 28070/44000 | Loss: 2.9810 | LR: 1.50e-04 | 694.4ms/step | 28.8 samples/s +Step 28080/44000 | Loss: 2.9492 | LR: 1.49e-04 | 693.9ms/step | 28.8 samples/s +Step 28090/44000 | Loss: 2.9378 | LR: 1.49e-04 | 694.2ms/step | 28.8 samples/s +Step 28100/44000 | Loss: 2.9555 | LR: 1.49e-04 | 695.4ms/step | 28.8 samples/s +Step 28110/44000 | Loss: 2.9796 | LR: 1.49e-04 | 693.4ms/step | 28.8 samples/s +Step 28120/44000 | Loss: 2.9627 | LR: 1.49e-04 | 695.1ms/step | 28.8 samples/s +Step 28130/44000 | Loss: 2.8949 | LR: 1.49e-04 | 696.0ms/step | 28.7 samples/s +Step 28140/44000 | Loss: 2.9437 | LR: 1.48e-04 | 696.9ms/step | 28.7 samples/s +Step 28150/44000 | Loss: 2.9495 | LR: 1.48e-04 | 696.6ms/step | 28.7 samples/s +Step 28160/44000 | Loss: 2.9377 | LR: 1.48e-04 | 697.1ms/step | 28.7 samples/s +Step 28170/44000 | Loss: 2.9550 | LR: 1.48e-04 | 694.6ms/step | 28.8 samples/s +Step 28180/44000 | Loss: 2.9291 | LR: 1.48e-04 | 695.0ms/step | 28.8 samples/s +Step 28190/44000 | Loss: 2.9600 | LR: 1.48e-04 | 693.6ms/step | 28.8 samples/s +Step 28200/44000 | Loss: 2.9866 | LR: 1.47e-04 | 695.7ms/step | 28.7 samples/s +Step 28210/44000 | Loss: 2.9326 | LR: 1.47e-04 | 693.3ms/step | 28.8 samples/s +Step 28220/44000 | Loss: 2.9219 | LR: 1.47e-04 | 695.2ms/step | 28.8 samples/s +Step 28230/44000 | Loss: 2.9662 | LR: 1.47e-04 | 693.5ms/step | 28.8 samples/s +Step 28240/44000 | Loss: 2.9162 | LR: 1.47e-04 | 696.0ms/step | 28.7 samples/s +Step 28250/44000 | Loss: 2.9479 | LR: 1.47e-04 | 693.3ms/step | 28.8 samples/s +Validation at step 28250: Loss = 2.9501, Perplexity = 19.11 +Step 28260/44000 | Loss: 2.9713 | LR: 1.46e-04 | 1264.8ms/step | 15.8 samples/s +Step 28270/44000 | Loss: 2.9220 | LR: 1.46e-04 | 694.0ms/step | 28.8 samples/s +Step 28280/44000 | Loss: 2.9665 | LR: 1.46e-04 | 694.6ms/step | 28.8 samples/s +Step 28290/44000 | Loss: 2.9346 | LR: 1.46e-04 | 693.7ms/step | 28.8 samples/s +Step 28300/44000 | Loss: 2.9060 | LR: 1.46e-04 | 693.2ms/step | 28.8 samples/s +Step 28310/44000 | Loss: 2.9442 | LR: 1.46e-04 | 697.9ms/step | 28.7 samples/s +Step 28320/44000 | Loss: 2.9652 | LR: 1.45e-04 | 695.8ms/step | 28.7 samples/s +Step 28330/44000 | Loss: 2.9700 | LR: 1.45e-04 | 696.7ms/step | 28.7 samples/s +Step 28340/44000 | Loss: 2.9231 | LR: 1.45e-04 | 693.8ms/step | 28.8 samples/s +Step 28350/44000 | Loss: 2.9026 | LR: 1.45e-04 | 694.1ms/step | 28.8 samples/s +Step 28360/44000 | Loss: 2.9742 | LR: 1.45e-04 | 695.2ms/step | 28.8 samples/s +Step 28370/44000 | Loss: 2.9250 | LR: 1.45e-04 | 692.9ms/step | 28.9 samples/s +Step 28380/44000 | Loss: 2.9732 | LR: 1.44e-04 | 693.4ms/step | 28.8 samples/s +Step 28390/44000 | Loss: 2.9281 | LR: 1.44e-04 | 697.8ms/step | 28.7 samples/s +Step 28400/44000 | Loss: 2.9069 | LR: 1.44e-04 | 695.9ms/step | 28.7 samples/s +Step 28410/44000 | Loss: 2.9631 | LR: 1.44e-04 | 696.7ms/step | 28.7 samples/s +Step 28420/44000 | Loss: 2.9704 | LR: 1.44e-04 | 696.6ms/step | 28.7 samples/s +Step 28430/44000 | Loss: 2.9261 | LR: 1.44e-04 | 693.6ms/step | 28.8 samples/s +Step 28440/44000 | Loss: 2.9370 | LR: 1.43e-04 | 693.6ms/step | 28.8 samples/s +Step 28450/44000 | Loss: 2.9225 | LR: 1.43e-04 | 693.0ms/step | 28.9 samples/s +Step 28460/44000 | Loss: 2.9865 | LR: 1.43e-04 | 692.8ms/step | 28.9 samples/s +Step 28470/44000 | Loss: 2.9277 | LR: 1.43e-04 | 693.0ms/step | 28.9 samples/s +Step 28480/44000 | Loss: 2.9369 | LR: 1.43e-04 | 693.6ms/step | 28.8 samples/s +Step 28490/44000 | Loss: 2.9052 | LR: 1.43e-04 | 693.5ms/step | 28.8 samples/s +Step 28500/44000 | Loss: 2.9414 | LR: 1.42e-04 | 694.0ms/step | 28.8 samples/s +Validation at step 28500: Loss = 2.9488, Perplexity = 19.08 +Step 28510/44000 | Loss: 2.9323 | LR: 1.42e-04 | 1262.3ms/step | 15.8 samples/s +Step 28520/44000 | Loss: 2.9231 | LR: 1.42e-04 | 693.3ms/step | 28.8 samples/s +Step 28530/44000 | Loss: 2.9342 | LR: 1.42e-04 | 692.0ms/step | 28.9 samples/s +Step 28540/44000 | Loss: 2.9606 | LR: 1.42e-04 | 693.3ms/step | 28.8 samples/s +Step 28550/44000 | Loss: 2.9427 | LR: 1.42e-04 | 694.2ms/step | 28.8 samples/s +Step 28560/44000 | Loss: 2.9348 | LR: 1.41e-04 | 693.4ms/step | 28.8 samples/s +Step 28570/44000 | Loss: 2.9254 | LR: 1.41e-04 | 694.8ms/step | 28.8 samples/s +Step 28580/44000 | Loss: 2.9126 | LR: 1.41e-04 | 695.6ms/step | 28.8 samples/s +Step 28590/44000 | Loss: 2.9308 | LR: 1.41e-04 | 693.4ms/step | 28.8 samples/s +Step 28600/44000 | Loss: 2.9350 | LR: 1.41e-04 | 693.8ms/step | 28.8 samples/s +Step 28610/44000 | Loss: 2.9359 | LR: 1.41e-04 | 693.8ms/step | 28.8 samples/s +Step 28620/44000 | Loss: 2.9175 | LR: 1.40e-04 | 695.2ms/step | 28.8 samples/s +Step 28630/44000 | Loss: 2.9188 | LR: 1.40e-04 | 694.4ms/step | 28.8 samples/s +Step 28640/44000 | Loss: 2.9452 | LR: 1.40e-04 | 693.9ms/step | 28.8 samples/s +Step 28650/44000 | Loss: 2.9548 | LR: 1.40e-04 | 695.0ms/step | 28.8 samples/s +Step 28660/44000 | Loss: 2.9528 | LR: 1.40e-04 | 693.6ms/step | 28.8 samples/s +Step 28670/44000 | Loss: 2.9267 | LR: 1.40e-04 | 693.8ms/step | 28.8 samples/s +Step 28680/44000 | Loss: 2.9125 | LR: 1.39e-04 | 693.9ms/step | 28.8 samples/s +Step 28690/44000 | Loss: 2.9525 | LR: 1.39e-04 | 696.7ms/step | 28.7 samples/s +Step 28700/44000 | Loss: 2.9146 | LR: 1.39e-04 | 693.6ms/step | 28.8 samples/s +Step 28710/44000 | Loss: 2.8950 | LR: 1.39e-04 | 695.2ms/step | 28.8 samples/s +Step 28720/44000 | Loss: 2.9365 | LR: 1.39e-04 | 692.9ms/step | 28.9 samples/s +Step 28730/44000 | Loss: 2.8825 | LR: 1.39e-04 | 691.6ms/step | 28.9 samples/s +Step 28740/44000 | Loss: 2.9521 | LR: 1.38e-04 | 693.6ms/step | 28.8 samples/s +Step 28750/44000 | Loss: 2.9120 | LR: 1.38e-04 | 693.9ms/step | 28.8 samples/s +Validation at step 28750: Loss = 2.9474, Perplexity = 19.06 +Step 28760/44000 | Loss: 2.9427 | LR: 1.38e-04 | 1261.3ms/step | 15.9 samples/s +Step 28770/44000 | Loss: 2.9236 | LR: 1.38e-04 | 692.7ms/step | 28.9 samples/s +Step 28780/44000 | Loss: 2.9026 | LR: 1.38e-04 | 692.9ms/step | 28.9 samples/s +Step 28790/44000 | Loss: 2.9641 | LR: 1.38e-04 | 692.9ms/step | 28.9 samples/s +Step 28800/44000 | Loss: 2.9571 | LR: 1.38e-04 | 694.0ms/step | 28.8 samples/s +Step 28810/44000 | Loss: 2.9142 | LR: 1.37e-04 | 693.2ms/step | 28.9 samples/s +Step 28820/44000 | Loss: 2.9078 | LR: 1.37e-04 | 693.4ms/step | 28.8 samples/s +Step 28830/44000 | Loss: 2.8967 | LR: 1.37e-04 | 694.7ms/step | 28.8 samples/s +Step 28840/44000 | Loss: 2.9098 | LR: 1.37e-04 | 693.7ms/step | 28.8 samples/s +Step 28850/44000 | Loss: 2.9411 | LR: 1.37e-04 | 692.0ms/step | 28.9 samples/s +Step 28860/44000 | Loss: 2.9160 | LR: 1.37e-04 | 695.4ms/step | 28.8 samples/s +Step 28870/44000 | Loss: 2.8852 | LR: 1.36e-04 | 693.6ms/step | 28.8 samples/s +Step 28880/44000 | Loss: 2.9634 | LR: 1.36e-04 | 694.0ms/step | 28.8 samples/s +Step 28890/44000 | Loss: 2.9922 | LR: 1.36e-04 | 694.0ms/step | 28.8 samples/s +Step 28900/44000 | Loss: 2.9470 | LR: 1.36e-04 | 693.6ms/step | 28.8 samples/s +Step 28910/44000 | Loss: 2.9088 | LR: 1.36e-04 | 694.8ms/step | 28.8 samples/s +Step 28920/44000 | Loss: 2.9006 | LR: 1.36e-04 | 694.3ms/step | 28.8 samples/s +Step 28930/44000 | Loss: 2.9393 | LR: 1.35e-04 | 694.5ms/step | 28.8 samples/s +Step 28940/44000 | Loss: 2.9667 | LR: 1.35e-04 | 694.4ms/step | 28.8 samples/s +Step 28950/44000 | Loss: 2.9259 | LR: 1.35e-04 | 692.8ms/step | 28.9 samples/s +Step 28960/44000 | Loss: 2.9265 | LR: 1.35e-04 | 693.6ms/step | 28.8 samples/s +Step 28970/44000 | Loss: 2.9327 | LR: 1.35e-04 | 693.6ms/step | 28.8 samples/s +Step 28980/44000 | Loss: 2.9199 | LR: 1.35e-04 | 694.0ms/step | 28.8 samples/s +Step 28990/44000 | Loss: 2.9324 | LR: 1.34e-04 | 693.8ms/step | 28.8 samples/s +Step 29000/44000 | Loss: 2.9286 | LR: 1.34e-04 | 695.1ms/step | 28.8 samples/s +Validation at step 29000: Loss = 2.9466, Perplexity = 19.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-29000 +Step 29010/44000 | Loss: 2.9380 | LR: 1.34e-04 | 1572.1ms/step | 12.7 samples/s +Step 29020/44000 | Loss: 2.9621 | LR: 1.34e-04 | 696.7ms/step | 28.7 samples/s +Step 29030/44000 | Loss: 2.9331 | LR: 1.34e-04 | 693.7ms/step | 28.8 samples/s +Step 29040/44000 | Loss: 2.9109 | LR: 1.34e-04 | 696.7ms/step | 28.7 samples/s +Step 29050/44000 | Loss: 2.9488 | LR: 1.33e-04 | 696.1ms/step | 28.7 samples/s +Step 29060/44000 | Loss: 2.9592 | LR: 1.33e-04 | 692.8ms/step | 28.9 samples/s +Step 29070/44000 | Loss: 2.9064 | LR: 1.33e-04 | 693.7ms/step | 28.8 samples/s +Step 29080/44000 | Loss: 2.9454 | LR: 1.33e-04 | 694.3ms/step | 28.8 samples/s +Step 29090/44000 | Loss: 2.9134 | LR: 1.33e-04 | 694.6ms/step | 28.8 samples/s +Step 29100/44000 | Loss: 2.9672 | LR: 1.33e-04 | 694.4ms/step | 28.8 samples/s +Step 29110/44000 | Loss: 2.8776 | LR: 1.33e-04 | 694.9ms/step | 28.8 samples/s +Step 29120/44000 | Loss: 2.9371 | LR: 1.32e-04 | 694.1ms/step | 28.8 samples/s +Step 29130/44000 | Loss: 2.9201 | LR: 1.32e-04 | 695.8ms/step | 28.7 samples/s +Step 29140/44000 | Loss: 2.9216 | LR: 1.32e-04 | 694.9ms/step | 28.8 samples/s +Step 29150/44000 | Loss: 2.9436 | LR: 1.32e-04 | 696.3ms/step | 28.7 samples/s +Step 29160/44000 | Loss: 2.9376 | LR: 1.32e-04 | 695.1ms/step | 28.8 samples/s +Step 29170/44000 | Loss: 2.9434 | LR: 1.32e-04 | 694.4ms/step | 28.8 samples/s +Step 29180/44000 | Loss: 2.8950 | LR: 1.31e-04 | 694.7ms/step | 28.8 samples/s +Step 29190/44000 | Loss: 2.9100 | LR: 1.31e-04 | 695.5ms/step | 28.8 samples/s +Step 29200/44000 | Loss: 2.8838 | LR: 1.31e-04 | 694.8ms/step | 28.8 samples/s +Step 29210/44000 | Loss: 2.9500 | LR: 1.31e-04 | 694.3ms/step | 28.8 samples/s +Step 29220/44000 | Loss: 2.9005 | LR: 1.31e-04 | 694.3ms/step | 28.8 samples/s +Step 29230/44000 | Loss: 2.9083 | LR: 1.31e-04 | 694.0ms/step | 28.8 samples/s +Step 29240/44000 | Loss: 2.9455 | LR: 1.30e-04 | 693.3ms/step | 28.8 samples/s +Step 29250/44000 | Loss: 2.9237 | LR: 1.30e-04 | 694.1ms/step | 28.8 samples/s +Validation at step 29250: Loss = 2.9459, Perplexity = 19.03 +Step 29260/44000 | Loss: 2.9388 | LR: 1.30e-04 | 1260.9ms/step | 15.9 samples/s +Step 29270/44000 | Loss: 2.9412 | LR: 1.30e-04 | 693.6ms/step | 28.8 samples/s +Step 29280/44000 | Loss: 2.9636 | LR: 1.30e-04 | 693.3ms/step | 28.8 samples/s +Step 29290/44000 | Loss: 2.9413 | LR: 1.30e-04 | 692.8ms/step | 28.9 samples/s +Step 29300/44000 | Loss: 2.9266 | LR: 1.29e-04 | 694.3ms/step | 28.8 samples/s +Step 29310/44000 | Loss: 2.9802 | LR: 1.29e-04 | 693.9ms/step | 28.8 samples/s +Step 29320/44000 | Loss: 2.9022 | LR: 1.29e-04 | 694.4ms/step | 28.8 samples/s +Step 29330/44000 | Loss: 2.9257 | LR: 1.29e-04 | 696.0ms/step | 28.7 samples/s +Step 29340/44000 | Loss: 2.9131 | LR: 1.29e-04 | 693.7ms/step | 28.8 samples/s +Step 29350/44000 | Loss: 2.9711 | LR: 1.29e-04 | 696.2ms/step | 28.7 samples/s +Step 29360/44000 | Loss: 2.9266 | LR: 1.29e-04 | 693.4ms/step | 28.8 samples/s +Step 29370/44000 | Loss: 2.9643 | LR: 1.28e-04 | 696.0ms/step | 28.7 samples/s +Step 29380/44000 | Loss: 2.9338 | LR: 1.28e-04 | 693.1ms/step | 28.9 samples/s +Step 29390/44000 | Loss: 2.9403 | LR: 1.28e-04 | 694.5ms/step | 28.8 samples/s +Step 29400/44000 | Loss: 2.8847 | LR: 1.28e-04 | 693.9ms/step | 28.8 samples/s +Step 29410/44000 | Loss: 2.9146 | LR: 1.28e-04 | 694.6ms/step | 28.8 samples/s +Step 29420/44000 | Loss: 2.9478 | LR: 1.28e-04 | 692.4ms/step | 28.9 samples/s +Step 29430/44000 | Loss: 2.9344 | LR: 1.27e-04 | 692.8ms/step | 28.9 samples/s +Step 29440/44000 | Loss: 2.9049 | LR: 1.27e-04 | 694.7ms/step | 28.8 samples/s +Step 29450/44000 | Loss: 2.9052 | LR: 1.27e-04 | 693.2ms/step | 28.9 samples/s +Step 29460/44000 | Loss: 2.9865 | LR: 1.27e-04 | 691.1ms/step | 28.9 samples/s +Step 29470/44000 | Loss: 2.9408 | LR: 1.27e-04 | 692.7ms/step | 28.9 samples/s +Step 29480/44000 | Loss: 2.9212 | LR: 1.27e-04 | 692.5ms/step | 28.9 samples/s +Step 29490/44000 | Loss: 2.9365 | LR: 1.26e-04 | 692.3ms/step | 28.9 samples/s +Step 29500/44000 | Loss: 2.9511 | LR: 1.26e-04 | 693.0ms/step | 28.9 samples/s +Validation at step 29500: Loss = 2.9448, Perplexity = 19.01 +Step 29510/44000 | Loss: 2.9420 | LR: 1.26e-04 | 1260.7ms/step | 15.9 samples/s +Step 29520/44000 | Loss: 2.9760 | LR: 1.26e-04 | 694.2ms/step | 28.8 samples/s +Step 29530/44000 | Loss: 2.9194 | LR: 1.26e-04 | 693.9ms/step | 28.8 samples/s +Step 29540/44000 | Loss: 2.9093 | LR: 1.26e-04 | 693.2ms/step | 28.8 samples/s +Step 29550/44000 | Loss: 2.9319 | LR: 1.26e-04 | 693.9ms/step | 28.8 samples/s +Step 29560/44000 | Loss: 2.9220 | LR: 1.25e-04 | 693.1ms/step | 28.9 samples/s +Step 29570/44000 | Loss: 2.9544 | LR: 1.25e-04 | 692.5ms/step | 28.9 samples/s +Step 29580/44000 | Loss: 2.9018 | LR: 1.25e-04 | 692.6ms/step | 28.9 samples/s +Step 29590/44000 | Loss: 2.9167 | LR: 1.25e-04 | 693.2ms/step | 28.9 samples/s +Step 29600/44000 | Loss: 2.9099 | LR: 1.25e-04 | 692.9ms/step | 28.9 samples/s +Step 29610/44000 | Loss: 2.9424 | LR: 1.25e-04 | 693.0ms/step | 28.9 samples/s +Step 29620/44000 | Loss: 2.9549 | LR: 1.24e-04 | 694.7ms/step | 28.8 samples/s +Step 29630/44000 | Loss: 2.9554 | LR: 1.24e-04 | 693.9ms/step | 28.8 samples/s +Step 29640/44000 | Loss: 2.9771 | LR: 1.24e-04 | 693.0ms/step | 28.9 samples/s +Step 29650/44000 | Loss: 2.9495 | LR: 1.24e-04 | 694.1ms/step | 28.8 samples/s +Step 29660/44000 | Loss: 2.9207 | LR: 1.24e-04 | 695.4ms/step | 28.8 samples/s +Step 29670/44000 | Loss: 2.9132 | LR: 1.24e-04 | 693.0ms/step | 28.9 samples/s +Step 29680/44000 | Loss: 2.9539 | LR: 1.23e-04 | 694.3ms/step | 28.8 samples/s +Step 29690/44000 | Loss: 2.9272 | LR: 1.23e-04 | 695.7ms/step | 28.7 samples/s +Step 29700/44000 | Loss: 2.9133 | LR: 1.23e-04 | 693.0ms/step | 28.9 samples/s +Step 29710/44000 | Loss: 2.9413 | LR: 1.23e-04 | 695.0ms/step | 28.8 samples/s +Step 29720/44000 | Loss: 2.9184 | LR: 1.23e-04 | 694.5ms/step | 28.8 samples/s +Step 29730/44000 | Loss: 2.9403 | LR: 1.23e-04 | 694.4ms/step | 28.8 samples/s +Step 29740/44000 | Loss: 2.9238 | LR: 1.23e-04 | 694.0ms/step | 28.8 samples/s +Step 29750/44000 | Loss: 2.9395 | LR: 1.22e-04 | 691.8ms/step | 28.9 samples/s +Validation at step 29750: Loss = 2.9439, Perplexity = 18.99 +Step 29760/44000 | Loss: 2.8980 | LR: 1.22e-04 | 1268.4ms/step | 15.8 samples/s +Step 29770/44000 | Loss: 2.8968 | LR: 1.22e-04 | 693.3ms/step | 28.8 samples/s +Step 29780/44000 | Loss: 2.9430 | LR: 1.22e-04 | 693.5ms/step | 28.8 samples/s +Step 29790/44000 | Loss: 2.9021 | LR: 1.22e-04 | 692.5ms/step | 28.9 samples/s +Step 29800/44000 | Loss: 2.9311 | LR: 1.22e-04 | 693.6ms/step | 28.8 samples/s +Step 29810/44000 | Loss: 2.9534 | LR: 1.21e-04 | 693.5ms/step | 28.8 samples/s +Step 29820/44000 | Loss: 2.9197 | LR: 1.21e-04 | 694.1ms/step | 28.8 samples/s +Step 29830/44000 | Loss: 2.9407 | LR: 1.21e-04 | 693.9ms/step | 28.8 samples/s +Step 29840/44000 | Loss: 2.9087 | LR: 1.21e-04 | 693.6ms/step | 28.8 samples/s +Step 29850/44000 | Loss: 2.9151 | LR: 1.21e-04 | 692.9ms/step | 28.9 samples/s +Step 29860/44000 | Loss: 2.9440 | LR: 1.21e-04 | 692.6ms/step | 28.9 samples/s +Step 29870/44000 | Loss: 2.9229 | LR: 1.21e-04 | 693.9ms/step | 28.8 samples/s +Step 29880/44000 | Loss: 2.8781 | LR: 1.20e-04 | 692.3ms/step | 28.9 samples/s +Step 29890/44000 | Loss: 2.8951 | LR: 1.20e-04 | 693.5ms/step | 28.8 samples/s +Step 29900/44000 | Loss: 2.8894 | LR: 1.20e-04 | 693.1ms/step | 28.9 samples/s +Step 29910/44000 | Loss: 2.9401 | LR: 1.20e-04 | 696.3ms/step | 28.7 samples/s +Step 29920/44000 | Loss: 2.9307 | LR: 1.20e-04 | 693.5ms/step | 28.8 samples/s +Step 29930/44000 | Loss: 2.9165 | LR: 1.20e-04 | 694.0ms/step | 28.8 samples/s +Step 29940/44000 | Loss: 2.9328 | LR: 1.19e-04 | 695.6ms/step | 28.8 samples/s +Step 29950/44000 | Loss: 2.9130 | LR: 1.19e-04 | 693.8ms/step | 28.8 samples/s +Step 29960/44000 | Loss: 2.9153 | LR: 1.19e-04 | 694.1ms/step | 28.8 samples/s +Step 29970/44000 | Loss: 2.9646 | LR: 1.19e-04 | 693.7ms/step | 28.8 samples/s +Step 29980/44000 | Loss: 2.9434 | LR: 1.19e-04 | 693.0ms/step | 28.9 samples/s +Step 29990/44000 | Loss: 2.9293 | LR: 1.19e-04 | 693.4ms/step | 28.8 samples/s +Step 30000/44000 | Loss: 2.9029 | LR: 1.19e-04 | 695.8ms/step | 28.7 samples/s +Validation at step 30000: Loss = 2.9428, Perplexity = 18.97 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-30000 +Step 30010/44000 | Loss: 2.8872 | LR: 1.18e-04 | 1592.0ms/step | 12.6 samples/s +Step 30020/44000 | Loss: 2.8951 | LR: 1.18e-04 | 693.1ms/step | 28.9 samples/s +Step 30030/44000 | Loss: 2.9312 | LR: 1.18e-04 | 695.3ms/step | 28.8 samples/s +Step 30040/44000 | Loss: 2.9236 | LR: 1.18e-04 | 694.5ms/step | 28.8 samples/s +Step 30050/44000 | Loss: 2.9775 | LR: 1.18e-04 | 693.6ms/step | 28.8 samples/s +Step 30060/44000 | Loss: 2.9353 | LR: 1.18e-04 | 695.3ms/step | 28.8 samples/s +Step 30070/44000 | Loss: 2.9216 | LR: 1.17e-04 | 695.6ms/step | 28.8 samples/s +Step 30080/44000 | Loss: 2.8899 | LR: 1.17e-04 | 695.1ms/step | 28.8 samples/s +Step 30090/44000 | Loss: 2.9405 | LR: 1.17e-04 | 695.7ms/step | 28.7 samples/s +Step 30100/44000 | Loss: 2.9586 | LR: 1.17e-04 | 694.2ms/step | 28.8 samples/s +Step 30110/44000 | Loss: 2.9585 | LR: 1.17e-04 | 695.7ms/step | 28.7 samples/s +Step 30120/44000 | Loss: 2.9017 | LR: 1.17e-04 | 694.7ms/step | 28.8 samples/s +Step 30130/44000 | Loss: 2.9555 | LR: 1.17e-04 | 695.5ms/step | 28.8 samples/s +Step 30140/44000 | Loss: 2.8934 | LR: 1.16e-04 | 694.7ms/step | 28.8 samples/s +Step 30150/44000 | Loss: 2.8944 | LR: 1.16e-04 | 695.6ms/step | 28.8 samples/s +Step 30160/44000 | Loss: 2.9428 | LR: 1.16e-04 | 695.2ms/step | 28.8 samples/s +Step 30170/44000 | Loss: 2.9312 | LR: 1.16e-04 | 694.8ms/step | 28.8 samples/s +Step 30180/44000 | Loss: 2.9420 | LR: 1.16e-04 | 714.5ms/step | 28.0 samples/s +Step 30190/44000 | Loss: 2.9303 | LR: 1.16e-04 | 717.1ms/step | 27.9 samples/s +Step 30200/44000 | Loss: 2.9604 | LR: 1.15e-04 | 716.5ms/step | 27.9 samples/s +Step 30210/44000 | Loss: 2.9520 | LR: 1.15e-04 | 717.7ms/step | 27.9 samples/s +Step 30220/44000 | Loss: 2.9575 | LR: 1.15e-04 | 717.2ms/step | 27.9 samples/s +Step 30230/44000 | Loss: 2.8797 | LR: 1.15e-04 | 716.1ms/step | 27.9 samples/s +Step 30240/44000 | Loss: 2.9281 | LR: 1.15e-04 | 715.0ms/step | 28.0 samples/s +Step 30250/44000 | Loss: 2.9184 | LR: 1.15e-04 | 714.9ms/step | 28.0 samples/s +Validation at step 30250: Loss = 2.9418, Perplexity = 18.95 +Step 30260/44000 | Loss: 2.9171 | LR: 1.15e-04 | 1280.5ms/step | 15.6 samples/s +Step 30270/44000 | Loss: 2.9484 | LR: 1.14e-04 | 697.9ms/step | 28.7 samples/s +Step 30280/44000 | Loss: 2.9316 | LR: 1.14e-04 | 712.6ms/step | 28.1 samples/s +Step 30290/44000 | Loss: 2.9332 | LR: 1.14e-04 | 717.8ms/step | 27.9 samples/s +Step 30300/44000 | Loss: 2.9647 | LR: 1.14e-04 | 700.5ms/step | 28.6 samples/s +Step 30310/44000 | Loss: 2.9062 | LR: 1.14e-04 | 695.9ms/step | 28.7 samples/s +Step 30320/44000 | Loss: 2.9361 | LR: 1.14e-04 | 695.9ms/step | 28.7 samples/s +Step 30330/44000 | Loss: 2.9109 | LR: 1.13e-04 | 695.5ms/step | 28.8 samples/s +Step 30340/44000 | Loss: 2.9032 | LR: 1.13e-04 | 695.0ms/step | 28.8 samples/s +Step 30350/44000 | Loss: 2.9558 | LR: 1.13e-04 | 695.9ms/step | 28.7 samples/s +Step 30360/44000 | Loss: 2.9259 | LR: 1.13e-04 | 697.3ms/step | 28.7 samples/s +Step 30370/44000 | Loss: 2.9323 | LR: 1.13e-04 | 698.9ms/step | 28.6 samples/s +Step 30380/44000 | Loss: 2.9272 | LR: 1.13e-04 | 698.0ms/step | 28.7 samples/s +Step 30390/44000 | Loss: 2.9210 | LR: 1.13e-04 | 695.6ms/step | 28.8 samples/s +Step 30400/44000 | Loss: 2.8874 | LR: 1.12e-04 | 695.6ms/step | 28.8 samples/s +Step 30410/44000 | Loss: 2.9243 | LR: 1.12e-04 | 696.1ms/step | 28.7 samples/s +Step 30420/44000 | Loss: 2.9294 | LR: 1.12e-04 | 697.6ms/step | 28.7 samples/s +Step 30430/44000 | Loss: 2.9260 | LR: 1.12e-04 | 697.0ms/step | 28.7 samples/s +Step 30440/44000 | Loss: 2.9136 | LR: 1.12e-04 | 697.0ms/step | 28.7 samples/s +Step 30450/44000 | Loss: 2.9066 | LR: 1.12e-04 | 696.6ms/step | 28.7 samples/s +Step 30460/44000 | Loss: 2.9273 | LR: 1.11e-04 | 694.9ms/step | 28.8 samples/s +Step 30470/44000 | Loss: 2.8801 | LR: 1.11e-04 | 694.6ms/step | 28.8 samples/s +Step 30480/44000 | Loss: 2.8944 | LR: 1.11e-04 | 696.1ms/step | 28.7 samples/s +Step 30490/44000 | Loss: 2.9302 | LR: 1.11e-04 | 694.1ms/step | 28.8 samples/s +Step 30500/44000 | Loss: 2.9056 | LR: 1.11e-04 | 695.7ms/step | 28.7 samples/s +Validation at step 30500: Loss = 2.9408, Perplexity = 18.93 +Step 30510/44000 | Loss: 2.9608 | LR: 1.11e-04 | 1268.9ms/step | 15.8 samples/s +Step 30520/44000 | Loss: 2.9492 | LR: 1.11e-04 | 693.2ms/step | 28.9 samples/s +Step 30530/44000 | Loss: 2.9110 | LR: 1.10e-04 | 695.8ms/step | 28.7 samples/s +Step 30540/44000 | Loss: 2.8956 | LR: 1.10e-04 | 694.8ms/step | 28.8 samples/s +Step 30550/44000 | Loss: 2.8943 | LR: 1.10e-04 | 694.4ms/step | 28.8 samples/s +Step 30560/44000 | Loss: 2.9222 | LR: 1.10e-04 | 693.9ms/step | 28.8 samples/s +Step 30570/44000 | Loss: 2.9357 | LR: 1.10e-04 | 694.2ms/step | 28.8 samples/s +Step 30580/44000 | Loss: 2.9371 | LR: 1.10e-04 | 694.0ms/step | 28.8 samples/s +Step 30590/44000 | Loss: 2.9346 | LR: 1.10e-04 | 693.6ms/step | 28.8 samples/s +Step 30600/44000 | Loss: 2.9443 | LR: 1.09e-04 | 694.4ms/step | 28.8 samples/s +Step 30610/44000 | Loss: 2.9406 | LR: 1.09e-04 | 693.8ms/step | 28.8 samples/s +Step 30620/44000 | Loss: 2.9412 | LR: 1.09e-04 | 694.6ms/step | 28.8 samples/s +Step 30630/44000 | Loss: 2.9570 | LR: 1.09e-04 | 693.6ms/step | 28.8 samples/s +Step 30640/44000 | Loss: 2.9139 | LR: 1.09e-04 | 695.0ms/step | 28.8 samples/s +Step 30650/44000 | Loss: 2.8856 | LR: 1.09e-04 | 693.8ms/step | 28.8 samples/s +Step 30660/44000 | Loss: 2.9130 | LR: 1.08e-04 | 694.5ms/step | 28.8 samples/s +Step 30670/44000 | Loss: 2.9661 | LR: 1.08e-04 | 695.3ms/step | 28.8 samples/s +Step 30680/44000 | Loss: 2.9615 | LR: 1.08e-04 | 695.4ms/step | 28.8 samples/s +Step 30690/44000 | Loss: 2.9032 | LR: 1.08e-04 | 694.3ms/step | 28.8 samples/s +Step 30700/44000 | Loss: 2.9120 | LR: 1.08e-04 | 695.8ms/step | 28.7 samples/s +Step 30710/44000 | Loss: 2.9412 | LR: 1.08e-04 | 694.5ms/step | 28.8 samples/s +Step 30720/44000 | Loss: 2.9342 | LR: 1.08e-04 | 694.7ms/step | 28.8 samples/s +Step 30730/44000 | Loss: 2.9623 | LR: 1.07e-04 | 694.4ms/step | 28.8 samples/s +Step 30740/44000 | Loss: 2.9300 | LR: 1.07e-04 | 693.7ms/step | 28.8 samples/s +Step 30750/44000 | Loss: 2.9269 | LR: 1.07e-04 | 692.8ms/step | 28.9 samples/s +Validation at step 30750: Loss = 2.9403, Perplexity = 18.92 +Step 30760/44000 | Loss: 2.9149 | LR: 1.07e-04 | 1261.2ms/step | 15.9 samples/s +Step 30770/44000 | Loss: 2.9390 | LR: 1.07e-04 | 693.4ms/step | 28.8 samples/s +Step 30780/44000 | Loss: 2.9656 | LR: 1.07e-04 | 694.0ms/step | 28.8 samples/s +Step 30790/44000 | Loss: 2.9156 | LR: 1.07e-04 | 693.9ms/step | 28.8 samples/s +Step 30800/44000 | Loss: 2.9487 | LR: 1.06e-04 | 695.0ms/step | 28.8 samples/s +Step 30810/44000 | Loss: 2.9002 | LR: 1.06e-04 | 693.5ms/step | 28.8 samples/s +Step 30820/44000 | Loss: 2.8847 | LR: 1.06e-04 | 693.6ms/step | 28.8 samples/s +Step 30830/44000 | Loss: 2.9058 | LR: 1.06e-04 | 694.2ms/step | 28.8 samples/s +Step 30840/44000 | Loss: 2.9073 | LR: 1.06e-04 | 694.1ms/step | 28.8 samples/s +Step 30850/44000 | Loss: 2.9658 | LR: 1.06e-04 | 693.7ms/step | 28.8 samples/s +Step 30860/44000 | Loss: 2.8969 | LR: 1.05e-04 | 694.5ms/step | 28.8 samples/s +Step 30870/44000 | Loss: 2.9088 | LR: 1.05e-04 | 695.2ms/step | 28.8 samples/s +Step 30880/44000 | Loss: 2.9395 | LR: 1.05e-04 | 693.4ms/step | 28.8 samples/s +Step 30890/44000 | Loss: 2.8942 | LR: 1.05e-04 | 693.4ms/step | 28.8 samples/s +Step 30900/44000 | Loss: 2.8986 | LR: 1.05e-04 | 694.7ms/step | 28.8 samples/s +Step 30910/44000 | Loss: 2.9454 | LR: 1.05e-04 | 693.8ms/step | 28.8 samples/s +Step 30920/44000 | Loss: 2.9407 | LR: 1.05e-04 | 693.0ms/step | 28.9 samples/s +Step 30930/44000 | Loss: 2.9298 | LR: 1.04e-04 | 693.5ms/step | 28.8 samples/s +Step 30940/44000 | Loss: 2.8985 | LR: 1.04e-04 | 695.0ms/step | 28.8 samples/s +Step 30950/44000 | Loss: 2.9674 | LR: 1.04e-04 | 694.4ms/step | 28.8 samples/s +Step 30960/44000 | Loss: 2.9481 | LR: 1.04e-04 | 694.0ms/step | 28.8 samples/s +Step 30970/44000 | Loss: 2.9708 | LR: 1.04e-04 | 695.0ms/step | 28.8 samples/s +Step 30980/44000 | Loss: 2.9562 | LR: 1.04e-04 | 692.3ms/step | 28.9 samples/s +Step 30990/44000 | Loss: 2.9415 | LR: 1.04e-04 | 694.7ms/step | 28.8 samples/s +Step 31000/44000 | Loss: 2.9148 | LR: 1.03e-04 | 695.4ms/step | 28.8 samples/s +Validation at step 31000: Loss = 2.9399, Perplexity = 18.91 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-31000 +Step 31010/44000 | Loss: 2.9288 | LR: 1.03e-04 | 1575.2ms/step | 12.7 samples/s +Step 31020/44000 | Loss: 2.9181 | LR: 1.03e-04 | 698.9ms/step | 28.6 samples/s +Step 31030/44000 | Loss: 2.9062 | LR: 1.03e-04 | 693.8ms/step | 28.8 samples/s +Step 31040/44000 | Loss: 2.8930 | LR: 1.03e-04 | 696.2ms/step | 28.7 samples/s +Step 31050/44000 | Loss: 2.9145 | LR: 1.03e-04 | 694.6ms/step | 28.8 samples/s +Step 31060/44000 | Loss: 2.9758 | LR: 1.03e-04 | 694.9ms/step | 28.8 samples/s +Step 31070/44000 | Loss: 2.9505 | LR: 1.02e-04 | 695.1ms/step | 28.8 samples/s +Step 31080/44000 | Loss: 2.9366 | LR: 1.02e-04 | 695.2ms/step | 28.8 samples/s +Step 31090/44000 | Loss: 2.8783 | LR: 1.02e-04 | 694.2ms/step | 28.8 samples/s +Step 31100/44000 | Loss: 2.9500 | LR: 1.02e-04 | 694.6ms/step | 28.8 samples/s +Step 31110/44000 | Loss: 2.9560 | LR: 1.02e-04 | 697.9ms/step | 28.7 samples/s +Step 31120/44000 | Loss: 2.9381 | LR: 1.02e-04 | 696.1ms/step | 28.7 samples/s +Step 31130/44000 | Loss: 2.9439 | LR: 1.02e-04 | 695.2ms/step | 28.8 samples/s +Step 31140/44000 | Loss: 2.9223 | LR: 1.01e-04 | 695.4ms/step | 28.8 samples/s +Step 31150/44000 | Loss: 2.9394 | LR: 1.01e-04 | 696.4ms/step | 28.7 samples/s +Step 31160/44000 | Loss: 2.9376 | LR: 1.01e-04 | 694.5ms/step | 28.8 samples/s +Step 31170/44000 | Loss: 2.9925 | LR: 1.01e-04 | 694.2ms/step | 28.8 samples/s +Step 31180/44000 | Loss: 2.9430 | LR: 1.01e-04 | 695.4ms/step | 28.8 samples/s +Step 31190/44000 | Loss: 2.9304 | LR: 1.01e-04 | 694.7ms/step | 28.8 samples/s +Step 31200/44000 | Loss: 2.9315 | LR: 1.00e-04 | 695.4ms/step | 28.8 samples/s +Step 31210/44000 | Loss: 2.9287 | LR: 1.00e-04 | 696.8ms/step | 28.7 samples/s +Step 31220/44000 | Loss: 2.9437 | LR: 1.00e-04 | 696.0ms/step | 28.7 samples/s +Step 31230/44000 | Loss: 2.9002 | LR: 1.00e-04 | 693.2ms/step | 28.9 samples/s +Step 31240/44000 | Loss: 2.9207 | LR: 9.99e-05 | 695.1ms/step | 28.8 samples/s +Step 31250/44000 | Loss: 2.9023 | LR: 9.98e-05 | 696.2ms/step | 28.7 samples/s +Validation at step 31250: Loss = 2.9387, Perplexity = 18.89 +Step 31260/44000 | Loss: 2.9217 | LR: 9.96e-05 | 1262.9ms/step | 15.8 samples/s +Step 31270/44000 | Loss: 2.9562 | LR: 9.95e-05 | 696.0ms/step | 28.7 samples/s +Step 31280/44000 | Loss: 2.8925 | LR: 9.93e-05 | 695.5ms/step | 28.8 samples/s +Step 31290/44000 | Loss: 2.9145 | LR: 9.92e-05 | 696.2ms/step | 28.7 samples/s +Step 31300/44000 | Loss: 2.9375 | LR: 9.90e-05 | 695.5ms/step | 28.8 samples/s +Step 31310/44000 | Loss: 2.9223 | LR: 9.89e-05 | 694.4ms/step | 28.8 samples/s +Step 31320/44000 | Loss: 2.8989 | LR: 9.88e-05 | 692.4ms/step | 28.9 samples/s +Step 31330/44000 | Loss: 2.9403 | LR: 9.86e-05 | 693.7ms/step | 28.8 samples/s +Step 31340/44000 | Loss: 2.9222 | LR: 9.85e-05 | 696.6ms/step | 28.7 samples/s +Step 31350/44000 | Loss: 2.9098 | LR: 9.83e-05 | 693.9ms/step | 28.8 samples/s +Step 31360/44000 | Loss: 2.9108 | LR: 9.82e-05 | 695.0ms/step | 28.8 samples/s +Step 31370/44000 | Loss: 2.9425 | LR: 9.80e-05 | 695.6ms/step | 28.8 samples/s +Step 31380/44000 | Loss: 2.9556 | LR: 9.79e-05 | 695.8ms/step | 28.7 samples/s +Step 31390/44000 | Loss: 2.9528 | LR: 9.77e-05 | 694.0ms/step | 28.8 samples/s +Step 31400/44000 | Loss: 2.9529 | LR: 9.76e-05 | 696.1ms/step | 28.7 samples/s +Step 31410/44000 | Loss: 2.9397 | LR: 9.75e-05 | 693.3ms/step | 28.8 samples/s +Step 31420/44000 | Loss: 2.8768 | LR: 9.73e-05 | 696.6ms/step | 28.7 samples/s +Step 31430/44000 | Loss: 2.9749 | LR: 9.72e-05 | 694.2ms/step | 28.8 samples/s +Step 31440/44000 | Loss: 2.9243 | LR: 9.70e-05 | 693.3ms/step | 28.8 samples/s +Step 31450/44000 | Loss: 2.9235 | LR: 9.69e-05 | 695.7ms/step | 28.7 samples/s +Step 31460/44000 | Loss: 2.9436 | LR: 9.67e-05 | 694.1ms/step | 28.8 samples/s +Step 31470/44000 | Loss: 2.9213 | LR: 9.66e-05 | 696.7ms/step | 28.7 samples/s +Step 31480/44000 | Loss: 2.9196 | LR: 9.65e-05 | 694.3ms/step | 28.8 samples/s +Step 31490/44000 | Loss: 2.9293 | LR: 9.63e-05 | 694.6ms/step | 28.8 samples/s +Step 31500/44000 | Loss: 2.9210 | LR: 9.62e-05 | 696.1ms/step | 28.7 samples/s +Validation at step 31500: Loss = 2.9382, Perplexity = 18.88 +Step 31510/44000 | Loss: 2.8854 | LR: 9.60e-05 | 1265.2ms/step | 15.8 samples/s +Step 31520/44000 | Loss: 2.9209 | LR: 9.59e-05 | 694.7ms/step | 28.8 samples/s +Step 31530/44000 | Loss: 2.9218 | LR: 9.57e-05 | 695.1ms/step | 28.8 samples/s +Step 31540/44000 | Loss: 2.8840 | LR: 9.56e-05 | 693.0ms/step | 28.9 samples/s +Step 31550/44000 | Loss: 2.9272 | LR: 9.54e-05 | 694.9ms/step | 28.8 samples/s +Step 31560/44000 | Loss: 2.9395 | LR: 9.53e-05 | 693.3ms/step | 28.8 samples/s +Step 31570/44000 | Loss: 2.8980 | LR: 9.52e-05 | 695.0ms/step | 28.8 samples/s +Step 31580/44000 | Loss: 2.9700 | LR: 9.50e-05 | 694.1ms/step | 28.8 samples/s +Step 31590/44000 | Loss: 2.9483 | LR: 9.49e-05 | 694.7ms/step | 28.8 samples/s +Step 31600/44000 | Loss: 2.9339 | LR: 9.47e-05 | 695.3ms/step | 28.8 samples/s +Step 31610/44000 | Loss: 2.9201 | LR: 9.46e-05 | 694.7ms/step | 28.8 samples/s +Step 31620/44000 | Loss: 2.9336 | LR: 9.45e-05 | 694.2ms/step | 28.8 samples/s +Step 31630/44000 | Loss: 2.9549 | LR: 9.43e-05 | 695.0ms/step | 28.8 samples/s +Step 31640/44000 | Loss: 2.9266 | LR: 9.42e-05 | 696.9ms/step | 28.7 samples/s +Step 31650/44000 | Loss: 2.9268 | LR: 9.40e-05 | 692.3ms/step | 28.9 samples/s +Step 31660/44000 | Loss: 2.9010 | LR: 9.39e-05 | 695.0ms/step | 28.8 samples/s +Step 31670/44000 | Loss: 2.9317 | LR: 9.37e-05 | 693.6ms/step | 28.8 samples/s +Step 31680/44000 | Loss: 2.9357 | LR: 9.36e-05 | 694.3ms/step | 28.8 samples/s +Step 31690/44000 | Loss: 2.8874 | LR: 9.35e-05 | 693.4ms/step | 28.8 samples/s +Step 31700/44000 | Loss: 2.9187 | LR: 9.33e-05 | 693.1ms/step | 28.9 samples/s +Step 31710/44000 | Loss: 2.9789 | LR: 9.32e-05 | 694.1ms/step | 28.8 samples/s +Step 31720/44000 | Loss: 2.9236 | LR: 9.30e-05 | 693.1ms/step | 28.9 samples/s +Step 31730/44000 | Loss: 2.9177 | LR: 9.29e-05 | 692.8ms/step | 28.9 samples/s +Step 31740/44000 | Loss: 2.9484 | LR: 9.28e-05 | 692.5ms/step | 28.9 samples/s +Step 31750/44000 | Loss: 2.9012 | LR: 9.26e-05 | 693.4ms/step | 28.8 samples/s +Validation at step 31750: Loss = 2.9376, Perplexity = 18.87 +Step 31760/44000 | Loss: 2.9695 | LR: 9.25e-05 | 1264.8ms/step | 15.8 samples/s +Step 31770/44000 | Loss: 2.9467 | LR: 9.23e-05 | 694.6ms/step | 28.8 samples/s +Step 31780/44000 | Loss: 2.8832 | LR: 9.22e-05 | 695.4ms/step | 28.8 samples/s +Step 31790/44000 | Loss: 2.9121 | LR: 9.20e-05 | 695.4ms/step | 28.8 samples/s +Step 31800/44000 | Loss: 2.8864 | LR: 9.19e-05 | 694.1ms/step | 28.8 samples/s +Step 31810/44000 | Loss: 2.9197 | LR: 9.18e-05 | 693.6ms/step | 28.8 samples/s +Step 31820/44000 | Loss: 2.9528 | LR: 9.16e-05 | 696.3ms/step | 28.7 samples/s +Step 31830/44000 | Loss: 2.9326 | LR: 9.15e-05 | 692.0ms/step | 28.9 samples/s +Step 31840/44000 | Loss: 2.9175 | LR: 9.13e-05 | 692.6ms/step | 28.9 samples/s +Step 31850/44000 | Loss: 2.9247 | LR: 9.12e-05 | 694.4ms/step | 28.8 samples/s +Step 31860/44000 | Loss: 2.9396 | LR: 9.11e-05 | 694.8ms/step | 28.8 samples/s +Step 31870/44000 | Loss: 2.9413 | LR: 9.09e-05 | 693.0ms/step | 28.9 samples/s +Step 31880/44000 | Loss: 2.9135 | LR: 9.08e-05 | 694.0ms/step | 28.8 samples/s +Step 31890/44000 | Loss: 2.9630 | LR: 9.06e-05 | 693.6ms/step | 28.8 samples/s +Step 31900/44000 | Loss: 2.9271 | LR: 9.05e-05 | 693.4ms/step | 28.8 samples/s +Step 31910/44000 | Loss: 2.9071 | LR: 9.04e-05 | 693.0ms/step | 28.9 samples/s +Step 31920/44000 | Loss: 2.9334 | LR: 9.02e-05 | 693.5ms/step | 28.8 samples/s +Step 31930/44000 | Loss: 2.9269 | LR: 9.01e-05 | 692.4ms/step | 28.9 samples/s +Step 31940/44000 | Loss: 2.9290 | LR: 8.99e-05 | 693.5ms/step | 28.8 samples/s +Step 31950/44000 | Loss: 2.9250 | LR: 8.98e-05 | 692.4ms/step | 28.9 samples/s +Step 31960/44000 | Loss: 2.9100 | LR: 8.97e-05 | 694.2ms/step | 28.8 samples/s +Step 31970/44000 | Loss: 2.9790 | LR: 8.95e-05 | 692.1ms/step | 28.9 samples/s +Step 31980/44000 | Loss: 2.9637 | LR: 8.94e-05 | 693.9ms/step | 28.8 samples/s +Step 31990/44000 | Loss: 2.9110 | LR: 8.92e-05 | 694.6ms/step | 28.8 samples/s +Step 32000/44000 | Loss: 2.8998 | LR: 8.91e-05 | 692.9ms/step | 28.9 samples/s +Validation at step 32000: Loss = 2.9374, Perplexity = 18.87 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-32000 +Step 32010/44000 | Loss: 2.9209 | LR: 8.90e-05 | 1577.1ms/step | 12.7 samples/s +Step 32020/44000 | Loss: 2.9910 | LR: 8.88e-05 | 693.2ms/step | 28.9 samples/s +Step 32030/44000 | Loss: 2.9268 | LR: 8.87e-05 | 693.8ms/step | 28.8 samples/s +Step 32040/44000 | Loss: 2.9349 | LR: 8.86e-05 | 693.4ms/step | 28.8 samples/s +Step 32050/44000 | Loss: 2.8930 | LR: 8.84e-05 | 693.2ms/step | 28.9 samples/s +Step 32060/44000 | Loss: 2.9212 | LR: 8.83e-05 | 694.0ms/step | 28.8 samples/s +Step 32070/44000 | Loss: 2.8935 | LR: 8.81e-05 | 695.7ms/step | 28.7 samples/s +Step 32080/44000 | Loss: 2.9464 | LR: 8.80e-05 | 694.0ms/step | 28.8 samples/s +Step 32090/44000 | Loss: 2.9489 | LR: 8.79e-05 | 693.2ms/step | 28.9 samples/s +Step 32100/44000 | Loss: 2.9032 | LR: 8.77e-05 | 695.5ms/step | 28.8 samples/s +Step 32110/44000 | Loss: 2.9279 | LR: 8.76e-05 | 693.4ms/step | 28.8 samples/s +Step 32120/44000 | Loss: 2.9371 | LR: 8.74e-05 | 693.8ms/step | 28.8 samples/s +Step 32130/44000 | Loss: 2.9289 | LR: 8.73e-05 | 694.4ms/step | 28.8 samples/s +Step 32140/44000 | Loss: 2.9418 | LR: 8.72e-05 | 694.3ms/step | 28.8 samples/s +Step 32150/44000 | Loss: 2.8937 | LR: 8.70e-05 | 693.9ms/step | 28.8 samples/s +Step 32160/44000 | Loss: 2.9474 | LR: 8.69e-05 | 693.5ms/step | 28.8 samples/s +Step 32170/44000 | Loss: 2.9086 | LR: 8.68e-05 | 695.4ms/step | 28.8 samples/s +Step 32180/44000 | Loss: 2.9171 | LR: 8.66e-05 | 692.7ms/step | 28.9 samples/s +Step 32190/44000 | Loss: 2.9082 | LR: 8.65e-05 | 694.1ms/step | 28.8 samples/s +Step 32200/44000 | Loss: 2.8974 | LR: 8.63e-05 | 693.4ms/step | 28.8 samples/s +Step 32210/44000 | Loss: 2.9050 | LR: 8.62e-05 | 694.8ms/step | 28.8 samples/s +Step 32220/44000 | Loss: 2.9093 | LR: 8.61e-05 | 694.0ms/step | 28.8 samples/s +Step 32230/44000 | Loss: 2.9560 | LR: 8.59e-05 | 693.5ms/step | 28.8 samples/s +Step 32240/44000 | Loss: 2.9410 | LR: 8.58e-05 | 694.0ms/step | 28.8 samples/s +Step 32250/44000 | Loss: 2.9726 | LR: 8.57e-05 | 695.8ms/step | 28.7 samples/s +Validation at step 32250: Loss = 2.9366, Perplexity = 18.85 +Step 32260/44000 | Loss: 2.9286 | LR: 8.55e-05 | 1262.4ms/step | 15.8 samples/s +Step 32270/44000 | Loss: 2.9855 | LR: 8.54e-05 | 693.3ms/step | 28.8 samples/s +Step 32280/44000 | Loss: 2.9217 | LR: 8.53e-05 | 693.2ms/step | 28.9 samples/s +Step 32290/44000 | Loss: 2.9768 | LR: 8.51e-05 | 693.7ms/step | 28.8 samples/s +Step 32300/44000 | Loss: 2.9394 | LR: 8.50e-05 | 693.8ms/step | 28.8 samples/s +Step 32310/44000 | Loss: 2.8996 | LR: 8.48e-05 | 692.6ms/step | 28.9 samples/s +Step 32320/44000 | Loss: 2.8888 | LR: 8.47e-05 | 693.3ms/step | 28.8 samples/s +Step 32330/44000 | Loss: 2.9128 | LR: 8.46e-05 | 695.9ms/step | 28.7 samples/s +Step 32340/44000 | Loss: 2.9142 | LR: 8.44e-05 | 693.0ms/step | 28.9 samples/s +Step 32350/44000 | Loss: 2.9171 | LR: 8.43e-05 | 693.5ms/step | 28.8 samples/s +Step 32360/44000 | Loss: 2.9677 | LR: 8.42e-05 | 696.2ms/step | 28.7 samples/s +Step 32370/44000 | Loss: 2.9228 | LR: 8.40e-05 | 694.3ms/step | 28.8 samples/s +Step 32380/44000 | Loss: 2.9346 | LR: 8.39e-05 | 693.7ms/step | 28.8 samples/s +Step 32390/44000 | Loss: 2.9330 | LR: 8.38e-05 | 695.2ms/step | 28.8 samples/s +Step 32400/44000 | Loss: 2.9071 | LR: 8.36e-05 | 693.9ms/step | 28.8 samples/s +Step 32410/44000 | Loss: 2.9394 | LR: 8.35e-05 | 693.7ms/step | 28.8 samples/s +Step 32420/44000 | Loss: 2.9127 | LR: 8.33e-05 | 693.6ms/step | 28.8 samples/s +Step 32430/44000 | Loss: 2.9523 | LR: 8.32e-05 | 693.9ms/step | 28.8 samples/s +Step 32440/44000 | Loss: 2.9398 | LR: 8.31e-05 | 694.9ms/step | 28.8 samples/s +Step 32450/44000 | Loss: 2.9103 | LR: 8.29e-05 | 694.5ms/step | 28.8 samples/s +Step 32460/44000 | Loss: 2.9176 | LR: 8.28e-05 | 694.7ms/step | 28.8 samples/s +Step 32470/44000 | Loss: 2.9146 | LR: 8.27e-05 | 695.2ms/step | 28.8 samples/s +Step 32480/44000 | Loss: 2.9431 | LR: 8.25e-05 | 693.8ms/step | 28.8 samples/s +Step 32490/44000 | Loss: 2.9543 | LR: 8.24e-05 | 694.5ms/step | 28.8 samples/s +Step 32500/44000 | Loss: 2.8833 | LR: 8.23e-05 | 695.2ms/step | 28.8 samples/s +Validation at step 32500: Loss = 2.9359, Perplexity = 18.84 +Step 32510/44000 | Loss: 2.9773 | LR: 8.21e-05 | 1263.4ms/step | 15.8 samples/s +Step 32520/44000 | Loss: 2.9158 | LR: 8.20e-05 | 695.5ms/step | 28.8 samples/s +Step 32530/44000 | Loss: 2.9068 | LR: 8.19e-05 | 693.7ms/step | 28.8 samples/s +Step 32540/44000 | Loss: 2.9378 | LR: 8.17e-05 | 693.6ms/step | 28.8 samples/s +Step 32550/44000 | Loss: 2.9195 | LR: 8.16e-05 | 694.3ms/step | 28.8 samples/s +Step 32560/44000 | Loss: 2.9382 | LR: 8.15e-05 | 694.0ms/step | 28.8 samples/s +Step 32570/44000 | Loss: 2.9505 | LR: 8.13e-05 | 693.6ms/step | 28.8 samples/s +Step 32580/44000 | Loss: 2.8794 | LR: 8.12e-05 | 693.4ms/step | 28.8 samples/s +Step 32590/44000 | Loss: 2.9224 | LR: 8.11e-05 | 694.5ms/step | 28.8 samples/s +Step 32600/44000 | Loss: 2.9621 | LR: 8.09e-05 | 694.9ms/step | 28.8 samples/s +Step 32610/44000 | Loss: 2.9346 | LR: 8.08e-05 | 696.5ms/step | 28.7 samples/s +Step 32620/44000 | Loss: 2.9483 | LR: 8.07e-05 | 696.0ms/step | 28.7 samples/s +Step 32630/44000 | Loss: 2.9469 | LR: 8.05e-05 | 697.2ms/step | 28.7 samples/s +Step 32640/44000 | Loss: 2.9085 | LR: 8.04e-05 | 694.3ms/step | 28.8 samples/s +Step 32650/44000 | Loss: 2.9593 | LR: 8.03e-05 | 694.9ms/step | 28.8 samples/s +Step 32660/44000 | Loss: 2.9359 | LR: 8.01e-05 | 694.3ms/step | 28.8 samples/s +Step 32670/44000 | Loss: 2.9090 | LR: 8.00e-05 | 693.5ms/step | 28.8 samples/s +Step 32680/44000 | Loss: 2.9542 | LR: 7.99e-05 | 694.7ms/step | 28.8 samples/s +Step 32690/44000 | Loss: 2.9030 | LR: 7.97e-05 | 694.6ms/step | 28.8 samples/s +Step 32700/44000 | Loss: 2.9035 | LR: 7.96e-05 | 693.9ms/step | 28.8 samples/s +Step 32710/44000 | Loss: 2.9493 | LR: 7.95e-05 | 695.8ms/step | 28.7 samples/s +Step 32720/44000 | Loss: 2.9044 | LR: 7.93e-05 | 695.2ms/step | 28.8 samples/s +Step 32730/44000 | Loss: 2.9094 | LR: 7.92e-05 | 692.9ms/step | 28.9 samples/s +Step 32740/44000 | Loss: 2.9001 | LR: 7.91e-05 | 692.9ms/step | 28.9 samples/s +Step 32750/44000 | Loss: 2.8885 | LR: 7.89e-05 | 694.4ms/step | 28.8 samples/s +Validation at step 32750: Loss = 2.9358, Perplexity = 18.84 +Step 32760/44000 | Loss: 2.9113 | LR: 7.88e-05 | 1265.4ms/step | 15.8 samples/s +Step 32770/44000 | Loss: 2.9103 | LR: 7.87e-05 | 695.4ms/step | 28.8 samples/s +Step 32780/44000 | Loss: 2.9434 | LR: 7.85e-05 | 691.7ms/step | 28.9 samples/s +Step 32790/44000 | Loss: 2.9361 | LR: 7.84e-05 | 693.2ms/step | 28.9 samples/s +Step 32800/44000 | Loss: 2.9198 | LR: 7.83e-05 | 694.9ms/step | 28.8 samples/s +Step 32810/44000 | Loss: 2.9234 | LR: 7.81e-05 | 694.4ms/step | 28.8 samples/s +Step 32820/44000 | Loss: 2.9053 | LR: 7.80e-05 | 695.4ms/step | 28.8 samples/s +Step 32830/44000 | Loss: 2.9709 | LR: 7.79e-05 | 693.4ms/step | 28.8 samples/s +Step 32840/44000 | Loss: 2.8998 | LR: 7.77e-05 | 693.3ms/step | 28.8 samples/s +Step 32850/44000 | Loss: 2.9821 | LR: 7.76e-05 | 693.1ms/step | 28.9 samples/s +Step 32860/44000 | Loss: 2.9050 | LR: 7.75e-05 | 692.1ms/step | 28.9 samples/s +Step 32870/44000 | Loss: 2.9052 | LR: 7.73e-05 | 694.0ms/step | 28.8 samples/s +Step 32880/44000 | Loss: 2.9496 | LR: 7.72e-05 | 694.2ms/step | 28.8 samples/s +Step 32890/44000 | Loss: 2.8754 | LR: 7.71e-05 | 693.5ms/step | 28.8 samples/s +Step 32900/44000 | Loss: 2.9062 | LR: 7.70e-05 | 693.6ms/step | 28.8 samples/s +Step 32910/44000 | Loss: 2.9371 | LR: 7.68e-05 | 693.5ms/step | 28.8 samples/s +Step 32920/44000 | Loss: 2.9285 | LR: 7.67e-05 | 694.3ms/step | 28.8 samples/s +Step 32930/44000 | Loss: 2.9129 | LR: 7.66e-05 | 693.9ms/step | 28.8 samples/s +Step 32940/44000 | Loss: 2.9459 | LR: 7.64e-05 | 695.4ms/step | 28.8 samples/s +Step 32950/44000 | Loss: 2.9118 | LR: 7.63e-05 | 692.9ms/step | 28.9 samples/s +Step 32960/44000 | Loss: 2.9178 | LR: 7.62e-05 | 694.3ms/step | 28.8 samples/s +Step 32970/44000 | Loss: 2.9706 | LR: 7.60e-05 | 694.6ms/step | 28.8 samples/s +Step 32980/44000 | Loss: 2.9120 | LR: 7.59e-05 | 696.9ms/step | 28.7 samples/s +Step 32990/44000 | Loss: 2.9226 | LR: 7.58e-05 | 693.7ms/step | 28.8 samples/s +Step 33000/44000 | Loss: 2.9295 | LR: 7.56e-05 | 696.3ms/step | 28.7 samples/s +Validation at step 33000: Loss = 2.9351, Perplexity = 18.82 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-33000 +Step 33010/44000 | Loss: 2.9233 | LR: 7.55e-05 | 1566.8ms/step | 12.8 samples/s +Step 33020/44000 | Loss: 2.9379 | LR: 7.54e-05 | 692.8ms/step | 28.9 samples/s +Step 33030/44000 | Loss: 2.9443 | LR: 7.53e-05 | 693.3ms/step | 28.8 samples/s +Step 33040/44000 | Loss: 2.9185 | LR: 7.51e-05 | 692.9ms/step | 28.9 samples/s +Step 33050/44000 | Loss: 2.9093 | LR: 7.50e-05 | 691.8ms/step | 28.9 samples/s +Step 33060/44000 | Loss: 2.9191 | LR: 7.49e-05 | 695.6ms/step | 28.8 samples/s +Step 33070/44000 | Loss: 2.9549 | LR: 7.47e-05 | 693.5ms/step | 28.8 samples/s +Step 33080/44000 | Loss: 2.9243 | LR: 7.46e-05 | 692.7ms/step | 28.9 samples/s +Step 33090/44000 | Loss: 2.9286 | LR: 7.45e-05 | 693.6ms/step | 28.8 samples/s +Step 33100/44000 | Loss: 2.9543 | LR: 7.44e-05 | 693.7ms/step | 28.8 samples/s +Step 33110/44000 | Loss: 2.9249 | LR: 7.42e-05 | 693.1ms/step | 28.9 samples/s +Step 33120/44000 | Loss: 2.9558 | LR: 7.41e-05 | 694.5ms/step | 28.8 samples/s +Step 33130/44000 | Loss: 2.9351 | LR: 7.40e-05 | 696.1ms/step | 28.7 samples/s +Step 33140/44000 | Loss: 2.8671 | LR: 7.38e-05 | 693.6ms/step | 28.8 samples/s +Step 33150/44000 | Loss: 2.8997 | LR: 7.37e-05 | 694.9ms/step | 28.8 samples/s +Step 33160/44000 | Loss: 2.9021 | LR: 7.36e-05 | 693.7ms/step | 28.8 samples/s +Step 33170/44000 | Loss: 2.8888 | LR: 7.34e-05 | 692.6ms/step | 28.9 samples/s +Step 33180/44000 | Loss: 2.9272 | LR: 7.33e-05 | 693.2ms/step | 28.9 samples/s +Step 33190/44000 | Loss: 2.9303 | LR: 7.32e-05 | 695.3ms/step | 28.8 samples/s +Step 33200/44000 | Loss: 2.9152 | LR: 7.31e-05 | 694.8ms/step | 28.8 samples/s +Step 33210/44000 | Loss: 2.9347 | LR: 7.29e-05 | 693.8ms/step | 28.8 samples/s +Step 33220/44000 | Loss: 2.9490 | LR: 7.28e-05 | 693.4ms/step | 28.8 samples/s +Step 33230/44000 | Loss: 2.9249 | LR: 7.27e-05 | 694.2ms/step | 28.8 samples/s +Step 33240/44000 | Loss: 2.9329 | LR: 7.26e-05 | 693.5ms/step | 28.8 samples/s +Step 33250/44000 | Loss: 2.9104 | LR: 7.24e-05 | 693.6ms/step | 28.8 samples/s +Validation at step 33250: Loss = 2.9349, Perplexity = 18.82 +Step 33260/44000 | Loss: 2.9677 | LR: 7.23e-05 | 1261.3ms/step | 15.9 samples/s +Step 33270/44000 | Loss: 2.9461 | LR: 7.22e-05 | 692.9ms/step | 28.9 samples/s +Step 33280/44000 | Loss: 2.9462 | LR: 7.20e-05 | 694.3ms/step | 28.8 samples/s +Step 33290/44000 | Loss: 2.9300 | LR: 7.19e-05 | 693.4ms/step | 28.8 samples/s +Step 33300/44000 | Loss: 2.8939 | LR: 7.18e-05 | 693.5ms/step | 28.8 samples/s +Step 33310/44000 | Loss: 2.9528 | LR: 7.17e-05 | 692.9ms/step | 28.9 samples/s +Step 33320/44000 | Loss: 2.9167 | LR: 7.15e-05 | 696.0ms/step | 28.7 samples/s +Step 33330/44000 | Loss: 2.9258 | LR: 7.14e-05 | 693.8ms/step | 28.8 samples/s +Step 33340/44000 | Loss: 2.9077 | LR: 7.13e-05 | 693.5ms/step | 28.8 samples/s +Step 33350/44000 | Loss: 2.9525 | LR: 7.11e-05 | 693.1ms/step | 28.9 samples/s +Step 33360/44000 | Loss: 2.9221 | LR: 7.10e-05 | 694.0ms/step | 28.8 samples/s +Step 33370/44000 | Loss: 2.9181 | LR: 7.09e-05 | 693.0ms/step | 28.9 samples/s +Step 33380/44000 | Loss: 2.9022 | LR: 7.08e-05 | 693.4ms/step | 28.8 samples/s +Step 33390/44000 | Loss: 2.9094 | LR: 7.06e-05 | 692.2ms/step | 28.9 samples/s +Step 33400/44000 | Loss: 2.9677 | LR: 7.05e-05 | 692.4ms/step | 28.9 samples/s +Step 33410/44000 | Loss: 2.9476 | LR: 7.04e-05 | 692.0ms/step | 28.9 samples/s +Step 33420/44000 | Loss: 2.9033 | LR: 7.03e-05 | 694.0ms/step | 28.8 samples/s +Step 33430/44000 | Loss: 2.9622 | LR: 7.01e-05 | 692.4ms/step | 28.9 samples/s +Step 33440/44000 | Loss: 2.8910 | LR: 7.00e-05 | 691.7ms/step | 28.9 samples/s +Step 33450/44000 | Loss: 2.9300 | LR: 6.99e-05 | 693.7ms/step | 28.8 samples/s +Step 33460/44000 | Loss: 2.9385 | LR: 6.98e-05 | 693.4ms/step | 28.8 samples/s +Step 33470/44000 | Loss: 2.9114 | LR: 6.96e-05 | 692.0ms/step | 28.9 samples/s +Step 33480/44000 | Loss: 2.9476 | LR: 6.95e-05 | 695.3ms/step | 28.8 samples/s +Step 33490/44000 | Loss: 2.9221 | LR: 6.94e-05 | 693.6ms/step | 28.8 samples/s +Step 33500/44000 | Loss: 2.9132 | LR: 6.93e-05 | 695.1ms/step | 28.8 samples/s +Validation at step 33500: Loss = 2.9340, Perplexity = 18.80 +Step 33510/44000 | Loss: 2.9027 | LR: 6.91e-05 | 1260.7ms/step | 15.9 samples/s +Step 33520/44000 | Loss: 2.9274 | LR: 6.90e-05 | 692.4ms/step | 28.9 samples/s +Step 33530/44000 | Loss: 2.9407 | LR: 6.89e-05 | 693.4ms/step | 28.8 samples/s +Step 33540/44000 | Loss: 2.9209 | LR: 6.88e-05 | 694.0ms/step | 28.8 samples/s +Step 33550/44000 | Loss: 2.9276 | LR: 6.86e-05 | 693.1ms/step | 28.9 samples/s +Step 33560/44000 | Loss: 2.8769 | LR: 6.85e-05 | 693.8ms/step | 28.8 samples/s +Step 33570/44000 | Loss: 2.9481 | LR: 6.84e-05 | 693.8ms/step | 28.8 samples/s +Step 33580/44000 | Loss: 2.9529 | LR: 6.83e-05 | 693.6ms/step | 28.8 samples/s +Step 33590/44000 | Loss: 2.9431 | LR: 6.81e-05 | 693.3ms/step | 28.8 samples/s +Step 33600/44000 | Loss: 2.9246 | LR: 6.80e-05 | 692.6ms/step | 28.9 samples/s +Step 33610/44000 | Loss: 2.9261 | LR: 6.79e-05 | 694.6ms/step | 28.8 samples/s +Step 33620/44000 | Loss: 2.9193 | LR: 6.78e-05 | 693.8ms/step | 28.8 samples/s +Step 33630/44000 | Loss: 2.9765 | LR: 6.76e-05 | 694.4ms/step | 28.8 samples/s +Step 33640/44000 | Loss: 2.9229 | LR: 6.75e-05 | 694.2ms/step | 28.8 samples/s +Step 33650/44000 | Loss: 2.9098 | LR: 6.74e-05 | 692.8ms/step | 28.9 samples/s +Step 33660/44000 | Loss: 2.9222 | LR: 6.73e-05 | 692.0ms/step | 28.9 samples/s +Step 33670/44000 | Loss: 2.9203 | LR: 6.71e-05 | 692.8ms/step | 28.9 samples/s +Step 33680/44000 | Loss: 2.9059 | LR: 6.70e-05 | 692.6ms/step | 28.9 samples/s +Step 33690/44000 | Loss: 2.8775 | LR: 6.69e-05 | 693.7ms/step | 28.8 samples/s +Step 33700/44000 | Loss: 2.9268 | LR: 6.68e-05 | 693.8ms/step | 28.8 samples/s +Step 33710/44000 | Loss: 2.9408 | LR: 6.66e-05 | 692.4ms/step | 28.9 samples/s +Step 33720/44000 | Loss: 2.9030 | LR: 6.65e-05 | 693.3ms/step | 28.8 samples/s +Step 33730/44000 | Loss: 2.9096 | LR: 6.64e-05 | 692.7ms/step | 28.9 samples/s +Step 33740/44000 | Loss: 2.8926 | LR: 6.63e-05 | 692.9ms/step | 28.9 samples/s +Step 33750/44000 | Loss: 2.9068 | LR: 6.61e-05 | 693.1ms/step | 28.9 samples/s +Validation at step 33750: Loss = 2.9345, Perplexity = 18.81 +Step 33760/44000 | Loss: 2.8877 | LR: 6.60e-05 | 1259.9ms/step | 15.9 samples/s +Step 33770/44000 | Loss: 2.9179 | LR: 6.59e-05 | 693.7ms/step | 28.8 samples/s +Step 33780/44000 | Loss: 2.9470 | LR: 6.58e-05 | 694.5ms/step | 28.8 samples/s +Step 33790/44000 | Loss: 2.9187 | LR: 6.57e-05 | 693.7ms/step | 28.8 samples/s +Step 33800/44000 | Loss: 2.9299 | LR: 6.55e-05 | 694.6ms/step | 28.8 samples/s +Step 33810/44000 | Loss: 2.9063 | LR: 6.54e-05 | 693.4ms/step | 28.8 samples/s +Step 33820/44000 | Loss: 2.9141 | LR: 6.53e-05 | 692.6ms/step | 28.9 samples/s +Step 33830/44000 | Loss: 2.9445 | LR: 6.52e-05 | 693.1ms/step | 28.9 samples/s +Step 33840/44000 | Loss: 2.9009 | LR: 6.50e-05 | 692.6ms/step | 28.9 samples/s +Step 33850/44000 | Loss: 2.8891 | LR: 6.49e-05 | 695.0ms/step | 28.8 samples/s +Step 33860/44000 | Loss: 2.8438 | LR: 6.48e-05 | 694.0ms/step | 28.8 samples/s +Step 33870/44000 | Loss: 2.8954 | LR: 6.47e-05 | 692.4ms/step | 28.9 samples/s +Step 33880/44000 | Loss: 2.9331 | LR: 6.46e-05 | 691.4ms/step | 28.9 samples/s +Step 33890/44000 | Loss: 2.9152 | LR: 6.44e-05 | 693.8ms/step | 28.8 samples/s +Step 33900/44000 | Loss: 2.9127 | LR: 6.43e-05 | 692.3ms/step | 28.9 samples/s +Step 33910/44000 | Loss: 2.9303 | LR: 6.42e-05 | 693.5ms/step | 28.8 samples/s +Step 33920/44000 | Loss: 2.8588 | LR: 6.41e-05 | 693.6ms/step | 28.8 samples/s +Step 33930/44000 | Loss: 2.9179 | LR: 6.40e-05 | 693.9ms/step | 28.8 samples/s +Step 33940/44000 | Loss: 2.9797 | LR: 6.38e-05 | 693.3ms/step | 28.8 samples/s +Step 33950/44000 | Loss: 2.9145 | LR: 6.37e-05 | 693.4ms/step | 28.8 samples/s +Step 33960/44000 | Loss: 2.9111 | LR: 6.36e-05 | 692.1ms/step | 28.9 samples/s +Step 33970/44000 | Loss: 2.9216 | LR: 6.35e-05 | 692.3ms/step | 28.9 samples/s +Step 33980/44000 | Loss: 2.9292 | LR: 6.33e-05 | 690.7ms/step | 29.0 samples/s +Step 33990/44000 | Loss: 2.9466 | LR: 6.32e-05 | 692.4ms/step | 28.9 samples/s +Step 34000/44000 | Loss: 2.8783 | LR: 6.31e-05 | 692.2ms/step | 28.9 samples/s +Validation at step 34000: Loss = 2.9335, Perplexity = 18.79 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-34000 +Step 34010/44000 | Loss: 2.8775 | LR: 6.30e-05 | 1570.5ms/step | 12.7 samples/s +Step 34020/44000 | Loss: 2.9155 | LR: 6.29e-05 | 691.9ms/step | 28.9 samples/s +Step 34030/44000 | Loss: 2.9192 | LR: 6.27e-05 | 692.7ms/step | 28.9 samples/s +Step 34040/44000 | Loss: 2.8874 | LR: 6.26e-05 | 693.3ms/step | 28.8 samples/s +Step 34050/44000 | Loss: 2.9275 | LR: 6.25e-05 | 692.4ms/step | 28.9 samples/s +Step 34060/44000 | Loss: 2.9220 | LR: 6.24e-05 | 693.1ms/step | 28.9 samples/s +Step 34070/44000 | Loss: 2.9219 | LR: 6.23e-05 | 693.0ms/step | 28.9 samples/s +Step 34080/44000 | Loss: 2.9521 | LR: 6.21e-05 | 693.3ms/step | 28.8 samples/s +Step 34090/44000 | Loss: 2.9289 | LR: 6.20e-05 | 692.5ms/step | 28.9 samples/s +Step 34100/44000 | Loss: 2.9178 | LR: 6.19e-05 | 693.5ms/step | 28.8 samples/s +Step 34110/44000 | Loss: 2.8911 | LR: 6.18e-05 | 692.9ms/step | 28.9 samples/s +Step 34120/44000 | Loss: 2.9481 | LR: 6.17e-05 | 693.5ms/step | 28.8 samples/s +Step 34130/44000 | Loss: 2.9071 | LR: 6.15e-05 | 693.2ms/step | 28.9 samples/s +Step 34140/44000 | Loss: 2.9052 | LR: 6.14e-05 | 692.9ms/step | 28.9 samples/s +Step 34150/44000 | Loss: 2.9136 | LR: 6.13e-05 | 693.6ms/step | 28.8 samples/s +Step 34160/44000 | Loss: 2.9346 | LR: 6.12e-05 | 693.9ms/step | 28.8 samples/s +Step 34170/44000 | Loss: 2.9374 | LR: 6.11e-05 | 693.1ms/step | 28.9 samples/s +Step 34180/44000 | Loss: 2.9784 | LR: 6.09e-05 | 692.5ms/step | 28.9 samples/s +Step 34190/44000 | Loss: 2.8715 | LR: 6.08e-05 | 694.4ms/step | 28.8 samples/s +Step 34200/44000 | Loss: 2.9099 | LR: 6.07e-05 | 692.2ms/step | 28.9 samples/s +Step 34210/44000 | Loss: 2.9295 | LR: 6.06e-05 | 692.0ms/step | 28.9 samples/s +Step 34220/44000 | Loss: 2.8797 | LR: 6.05e-05 | 692.8ms/step | 28.9 samples/s +Step 34230/44000 | Loss: 2.9285 | LR: 6.04e-05 | 693.2ms/step | 28.9 samples/s +Step 34240/44000 | Loss: 2.9107 | LR: 6.02e-05 | 694.0ms/step | 28.8 samples/s +Step 34250/44000 | Loss: 2.9482 | LR: 6.01e-05 | 695.7ms/step | 28.7 samples/s +Validation at step 34250: Loss = 2.9334, Perplexity = 18.79 +Step 34260/44000 | Loss: 2.8909 | LR: 6.00e-05 | 1260.2ms/step | 15.9 samples/s +Step 34270/44000 | Loss: 2.8974 | LR: 5.99e-05 | 692.8ms/step | 28.9 samples/s +Step 34280/44000 | Loss: 2.8833 | LR: 5.98e-05 | 692.7ms/step | 28.9 samples/s +Step 34290/44000 | Loss: 2.9075 | LR: 5.96e-05 | 691.6ms/step | 28.9 samples/s +Step 34300/44000 | Loss: 2.9176 | LR: 5.95e-05 | 692.3ms/step | 28.9 samples/s +Step 34310/44000 | Loss: 2.9142 | LR: 5.94e-05 | 692.3ms/step | 28.9 samples/s +Step 34320/44000 | Loss: 2.8817 | LR: 5.93e-05 | 692.7ms/step | 28.9 samples/s +Step 34330/44000 | Loss: 2.9596 | LR: 5.92e-05 | 692.1ms/step | 28.9 samples/s +Step 34340/44000 | Loss: 2.8943 | LR: 5.91e-05 | 691.8ms/step | 28.9 samples/s +Step 34350/44000 | Loss: 2.9079 | LR: 5.89e-05 | 692.4ms/step | 28.9 samples/s +Step 34360/44000 | Loss: 2.9455 | LR: 5.88e-05 | 693.2ms/step | 28.9 samples/s +Step 34370/44000 | Loss: 2.9479 | LR: 5.87e-05 | 692.4ms/step | 28.9 samples/s +Step 34380/44000 | Loss: 2.9700 | LR: 5.86e-05 | 692.1ms/step | 28.9 samples/s +Step 34390/44000 | Loss: 2.9068 | LR: 5.85e-05 | 692.7ms/step | 28.9 samples/s +Step 34400/44000 | Loss: 2.9084 | LR: 5.84e-05 | 692.8ms/step | 28.9 samples/s +Step 34410/44000 | Loss: 2.9398 | LR: 5.82e-05 | 693.6ms/step | 28.8 samples/s +Step 34420/44000 | Loss: 2.9686 | LR: 5.81e-05 | 693.4ms/step | 28.8 samples/s +Step 34430/44000 | Loss: 2.9132 | LR: 5.80e-05 | 692.5ms/step | 28.9 samples/s +Step 34440/44000 | Loss: 2.9002 | LR: 5.79e-05 | 692.6ms/step | 28.9 samples/s +Step 34450/44000 | Loss: 2.9440 | LR: 5.78e-05 | 693.2ms/step | 28.9 samples/s +Step 34460/44000 | Loss: 2.8932 | LR: 5.77e-05 | 691.6ms/step | 28.9 samples/s +Step 34470/44000 | Loss: 2.9189 | LR: 5.75e-05 | 691.7ms/step | 28.9 samples/s +Step 34480/44000 | Loss: 2.9138 | LR: 5.74e-05 | 692.9ms/step | 28.9 samples/s +Step 34490/44000 | Loss: 2.9530 | LR: 5.73e-05 | 692.4ms/step | 28.9 samples/s +Step 34500/44000 | Loss: 2.9523 | LR: 5.72e-05 | 693.0ms/step | 28.9 samples/s +Validation at step 34500: Loss = 2.9328, Perplexity = 18.78 +Step 34510/44000 | Loss: 2.9153 | LR: 5.71e-05 | 1259.5ms/step | 15.9 samples/s +Step 34520/44000 | Loss: 2.9611 | LR: 5.70e-05 | 692.9ms/step | 28.9 samples/s +Step 34530/44000 | Loss: 2.9070 | LR: 5.69e-05 | 695.9ms/step | 28.7 samples/s +Step 34540/44000 | Loss: 2.9559 | LR: 5.67e-05 | 693.0ms/step | 28.9 samples/s +Step 34550/44000 | Loss: 2.9608 | LR: 5.66e-05 | 691.7ms/step | 28.9 samples/s +Step 34560/44000 | Loss: 2.8973 | LR: 5.65e-05 | 693.0ms/step | 28.9 samples/s +Step 34570/44000 | Loss: 2.9496 | LR: 5.64e-05 | 692.2ms/step | 28.9 samples/s +Step 34580/44000 | Loss: 2.9169 | LR: 5.63e-05 | 692.2ms/step | 28.9 samples/s +Step 34590/44000 | Loss: 2.9480 | LR: 5.62e-05 | 691.9ms/step | 28.9 samples/s +Step 34600/44000 | Loss: 2.9369 | LR: 5.60e-05 | 691.9ms/step | 28.9 samples/s +Step 34610/44000 | Loss: 2.9120 | LR: 5.59e-05 | 692.9ms/step | 28.9 samples/s +Step 34620/44000 | Loss: 2.9108 | LR: 5.58e-05 | 691.7ms/step | 28.9 samples/s +Step 34630/44000 | Loss: 2.8809 | LR: 5.57e-05 | 691.4ms/step | 28.9 samples/s +Step 34640/44000 | Loss: 2.8910 | LR: 5.56e-05 | 693.4ms/step | 28.8 samples/s +Step 34650/44000 | Loss: 2.9138 | LR: 5.55e-05 | 692.2ms/step | 28.9 samples/s +Step 34660/44000 | Loss: 2.9696 | LR: 5.54e-05 | 692.3ms/step | 28.9 samples/s +Step 34670/44000 | Loss: 2.9056 | LR: 5.52e-05 | 693.9ms/step | 28.8 samples/s +Step 34680/44000 | Loss: 2.9103 | LR: 5.51e-05 | 691.2ms/step | 28.9 samples/s +Step 34690/44000 | Loss: 2.9486 | LR: 5.50e-05 | 694.0ms/step | 28.8 samples/s +Step 34700/44000 | Loss: 2.9382 | LR: 5.49e-05 | 692.6ms/step | 28.9 samples/s +Step 34710/44000 | Loss: 2.9557 | LR: 5.48e-05 | 693.1ms/step | 28.9 samples/s +Step 34720/44000 | Loss: 2.9189 | LR: 5.47e-05 | 692.4ms/step | 28.9 samples/s +Step 34730/44000 | Loss: 2.9271 | LR: 5.46e-05 | 693.6ms/step | 28.8 samples/s +Step 34740/44000 | Loss: 2.9446 | LR: 5.45e-05 | 692.1ms/step | 28.9 samples/s +Step 34750/44000 | Loss: 2.8943 | LR: 5.43e-05 | 692.4ms/step | 28.9 samples/s +Validation at step 34750: Loss = 2.9328, Perplexity = 18.78 +Step 34760/44000 | Loss: 2.8829 | LR: 5.42e-05 | 1260.9ms/step | 15.9 samples/s +Step 34770/44000 | Loss: 2.9495 | LR: 5.41e-05 | 693.9ms/step | 28.8 samples/s +Step 34780/44000 | Loss: 2.9282 | LR: 5.40e-05 | 693.9ms/step | 28.8 samples/s +Step 34790/44000 | Loss: 2.9119 | LR: 5.39e-05 | 692.9ms/step | 28.9 samples/s +Step 34800/44000 | Loss: 2.9212 | LR: 5.38e-05 | 692.4ms/step | 28.9 samples/s +Step 34810/44000 | Loss: 2.9583 | LR: 5.37e-05 | 692.2ms/step | 28.9 samples/s +Step 34820/44000 | Loss: 2.9055 | LR: 5.36e-05 | 693.3ms/step | 28.8 samples/s +Step 34830/44000 | Loss: 2.8813 | LR: 5.34e-05 | 692.2ms/step | 28.9 samples/s +Step 34840/44000 | Loss: 2.8905 | LR: 5.33e-05 | 692.0ms/step | 28.9 samples/s +Step 34850/44000 | Loss: 2.8885 | LR: 5.32e-05 | 694.6ms/step | 28.8 samples/s +Step 34860/44000 | Loss: 2.8677 | LR: 5.31e-05 | 695.0ms/step | 28.8 samples/s +Step 34870/44000 | Loss: 2.9393 | LR: 5.30e-05 | 693.1ms/step | 28.9 samples/s +Step 34880/44000 | Loss: 2.9568 | LR: 5.29e-05 | 693.9ms/step | 28.8 samples/s +Step 34890/44000 | Loss: 2.9292 | LR: 5.28e-05 | 692.0ms/step | 28.9 samples/s +Step 34900/44000 | Loss: 2.9753 | LR: 5.27e-05 | 693.9ms/step | 28.8 samples/s +Step 34910/44000 | Loss: 2.8635 | LR: 5.25e-05 | 693.3ms/step | 28.8 samples/s +Step 34920/44000 | Loss: 2.9253 | LR: 5.24e-05 | 692.3ms/step | 28.9 samples/s +Step 34930/44000 | Loss: 2.9046 | LR: 5.23e-05 | 694.5ms/step | 28.8 samples/s +Step 34940/44000 | Loss: 2.9543 | LR: 5.22e-05 | 694.1ms/step | 28.8 samples/s +Step 34950/44000 | Loss: 2.8915 | LR: 5.21e-05 | 693.8ms/step | 28.8 samples/s +Step 34960/44000 | Loss: 2.8952 | LR: 5.20e-05 | 691.4ms/step | 28.9 samples/s +Step 34970/44000 | Loss: 2.9421 | LR: 5.19e-05 | 691.0ms/step | 28.9 samples/s +Step 34980/44000 | Loss: 2.9001 | LR: 5.18e-05 | 692.1ms/step | 28.9 samples/s +Step 34990/44000 | Loss: 2.9384 | LR: 5.17e-05 | 691.9ms/step | 28.9 samples/s +Step 35000/44000 | Loss: 2.8934 | LR: 5.15e-05 | 692.7ms/step | 28.9 samples/s +Validation at step 35000: Loss = 2.9327, Perplexity = 18.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-35000 +Step 35010/44000 | Loss: 2.9286 | LR: 5.14e-05 | 1571.9ms/step | 12.7 samples/s +Step 35020/44000 | Loss: 2.9531 | LR: 5.13e-05 | 693.4ms/step | 28.8 samples/s +Step 35030/44000 | Loss: 2.8851 | LR: 5.12e-05 | 693.5ms/step | 28.8 samples/s +Step 35040/44000 | Loss: 2.9059 | LR: 5.11e-05 | 695.3ms/step | 28.8 samples/s +Step 35050/44000 | Loss: 2.9394 | LR: 5.10e-05 | 694.4ms/step | 28.8 samples/s +Step 35060/44000 | Loss: 2.9340 | LR: 5.09e-05 | 693.4ms/step | 28.8 samples/s +Step 35070/44000 | Loss: 2.9308 | LR: 5.08e-05 | 692.9ms/step | 28.9 samples/s +Step 35080/44000 | Loss: 2.9142 | LR: 5.07e-05 | 692.2ms/step | 28.9 samples/s +Step 35090/44000 | Loss: 2.9111 | LR: 5.06e-05 | 692.5ms/step | 28.9 samples/s +Step 35100/44000 | Loss: 2.8933 | LR: 5.04e-05 | 694.8ms/step | 28.8 samples/s +Step 35110/44000 | Loss: 2.9408 | LR: 5.03e-05 | 694.2ms/step | 28.8 samples/s +Step 35120/44000 | Loss: 2.9320 | LR: 5.02e-05 | 694.4ms/step | 28.8 samples/s +Step 35130/44000 | Loss: 2.9587 | LR: 5.01e-05 | 696.9ms/step | 28.7 samples/s +Step 35140/44000 | Loss: 2.8923 | LR: 5.00e-05 | 691.6ms/step | 28.9 samples/s +Step 35150/44000 | Loss: 2.8980 | LR: 4.99e-05 | 692.4ms/step | 28.9 samples/s +Step 35160/44000 | Loss: 2.9312 | LR: 4.98e-05 | 691.2ms/step | 28.9 samples/s +Step 35170/44000 | Loss: 2.8951 | LR: 4.97e-05 | 693.3ms/step | 28.8 samples/s +Step 35180/44000 | Loss: 2.9240 | LR: 4.96e-05 | 692.9ms/step | 28.9 samples/s +Step 35190/44000 | Loss: 2.9125 | LR: 4.95e-05 | 694.4ms/step | 28.8 samples/s +Step 35200/44000 | Loss: 2.8733 | LR: 4.94e-05 | 693.0ms/step | 28.9 samples/s +Step 35210/44000 | Loss: 2.8942 | LR: 4.93e-05 | 693.1ms/step | 28.9 samples/s +Step 35220/44000 | Loss: 2.9563 | LR: 4.91e-05 | 691.8ms/step | 28.9 samples/s +Step 35230/44000 | Loss: 2.9457 | LR: 4.90e-05 | 693.4ms/step | 28.8 samples/s +Step 35240/44000 | Loss: 2.9328 | LR: 4.89e-05 | 692.6ms/step | 28.9 samples/s +Step 35250/44000 | Loss: 2.9351 | LR: 4.88e-05 | 692.7ms/step | 28.9 samples/s +Validation at step 35250: Loss = 2.9323, Perplexity = 18.77 +Step 35260/44000 | Loss: 2.9343 | LR: 4.87e-05 | 1261.8ms/step | 15.9 samples/s +Step 35270/44000 | Loss: 2.9018 | LR: 4.86e-05 | 694.3ms/step | 28.8 samples/s +Step 35280/44000 | Loss: 2.9081 | LR: 4.85e-05 | 693.4ms/step | 28.8 samples/s +Step 35290/44000 | Loss: 2.8944 | LR: 4.84e-05 | 693.2ms/step | 28.9 samples/s +Step 35300/44000 | Loss: 2.9334 | LR: 4.83e-05 | 696.2ms/step | 28.7 samples/s +Step 35310/44000 | Loss: 2.8960 | LR: 4.82e-05 | 693.3ms/step | 28.8 samples/s +Step 35320/44000 | Loss: 2.8966 | LR: 4.81e-05 | 693.5ms/step | 28.8 samples/s +Step 35330/44000 | Loss: 2.9317 | LR: 4.80e-05 | 694.5ms/step | 28.8 samples/s +Step 35340/44000 | Loss: 2.9240 | LR: 4.79e-05 | 692.8ms/step | 28.9 samples/s +Step 35350/44000 | Loss: 2.9261 | LR: 4.77e-05 | 694.3ms/step | 28.8 samples/s +Step 35360/44000 | Loss: 2.9269 | LR: 4.76e-05 | 693.0ms/step | 28.9 samples/s +Step 35370/44000 | Loss: 2.9208 | LR: 4.75e-05 | 694.4ms/step | 28.8 samples/s +Step 35380/44000 | Loss: 2.8913 | LR: 4.74e-05 | 692.5ms/step | 28.9 samples/s +Step 35390/44000 | Loss: 2.9191 | LR: 4.73e-05 | 693.4ms/step | 28.8 samples/s +Step 35400/44000 | Loss: 2.8884 | LR: 4.72e-05 | 693.3ms/step | 28.8 samples/s +Step 35410/44000 | Loss: 2.9021 | LR: 4.71e-05 | 694.1ms/step | 28.8 samples/s +Step 35420/44000 | Loss: 2.9169 | LR: 4.70e-05 | 693.0ms/step | 28.9 samples/s +Step 35430/44000 | Loss: 2.9174 | LR: 4.69e-05 | 693.5ms/step | 28.8 samples/s +Step 35440/44000 | Loss: 2.9513 | LR: 4.68e-05 | 692.8ms/step | 28.9 samples/s +Step 35450/44000 | Loss: 2.9662 | LR: 4.67e-05 | 693.1ms/step | 28.9 samples/s +Step 35460/44000 | Loss: 2.8986 | LR: 4.66e-05 | 694.4ms/step | 28.8 samples/s +Step 35470/44000 | Loss: 2.9165 | LR: 4.65e-05 | 693.8ms/step | 28.8 samples/s +Step 35480/44000 | Loss: 2.9437 | LR: 4.64e-05 | 693.5ms/step | 28.8 samples/s +Step 35490/44000 | Loss: 2.9616 | LR: 4.63e-05 | 693.7ms/step | 28.8 samples/s +Step 35500/44000 | Loss: 2.9089 | LR: 4.62e-05 | 694.2ms/step | 28.8 samples/s +Validation at step 35500: Loss = 2.9317, Perplexity = 18.76 +Step 35510/44000 | Loss: 2.9370 | LR: 4.61e-05 | 1260.9ms/step | 15.9 samples/s +Step 35520/44000 | Loss: 2.9508 | LR: 4.59e-05 | 692.0ms/step | 28.9 samples/s +Step 35530/44000 | Loss: 2.9646 | LR: 4.58e-05 | 695.6ms/step | 28.8 samples/s +Step 35540/44000 | Loss: 2.9294 | LR: 4.57e-05 | 691.8ms/step | 28.9 samples/s +Step 35550/44000 | Loss: 2.9037 | LR: 4.56e-05 | 693.1ms/step | 28.9 samples/s +Step 35560/44000 | Loss: 2.9507 | LR: 4.55e-05 | 694.6ms/step | 28.8 samples/s +Step 35570/44000 | Loss: 2.9205 | LR: 4.54e-05 | 694.3ms/step | 28.8 samples/s +Step 35580/44000 | Loss: 2.9007 | LR: 4.53e-05 | 692.2ms/step | 28.9 samples/s +Step 35590/44000 | Loss: 2.8687 | LR: 4.52e-05 | 693.3ms/step | 28.8 samples/s +Step 35600/44000 | Loss: 2.9030 | LR: 4.51e-05 | 693.4ms/step | 28.8 samples/s +Step 35610/44000 | Loss: 2.9523 | LR: 4.50e-05 | 694.0ms/step | 28.8 samples/s +Step 35620/44000 | Loss: 2.9014 | LR: 4.49e-05 | 693.6ms/step | 28.8 samples/s +Step 35630/44000 | Loss: 2.9544 | LR: 4.48e-05 | 695.6ms/step | 28.8 samples/s +Step 35640/44000 | Loss: 2.9179 | LR: 4.47e-05 | 694.1ms/step | 28.8 samples/s +Step 35650/44000 | Loss: 2.9416 | LR: 4.46e-05 | 692.4ms/step | 28.9 samples/s +Step 35660/44000 | Loss: 2.9178 | LR: 4.45e-05 | 692.7ms/step | 28.9 samples/s +Step 35670/44000 | Loss: 2.9382 | LR: 4.44e-05 | 693.8ms/step | 28.8 samples/s +Step 35680/44000 | Loss: 2.9315 | LR: 4.43e-05 | 693.0ms/step | 28.9 samples/s +Step 35690/44000 | Loss: 2.9138 | LR: 4.42e-05 | 695.6ms/step | 28.8 samples/s +Step 35700/44000 | Loss: 2.8870 | LR: 4.41e-05 | 694.3ms/step | 28.8 samples/s +Step 35710/44000 | Loss: 2.9255 | LR: 4.40e-05 | 694.0ms/step | 28.8 samples/s +Step 35720/44000 | Loss: 2.9277 | LR: 4.39e-05 | 694.0ms/step | 28.8 samples/s +Step 35730/44000 | Loss: 2.8915 | LR: 4.38e-05 | 691.9ms/step | 28.9 samples/s +Step 35740/44000 | Loss: 2.8868 | LR: 4.37e-05 | 691.7ms/step | 28.9 samples/s +Step 35750/44000 | Loss: 2.9173 | LR: 4.36e-05 | 692.7ms/step | 28.9 samples/s +Validation at step 35750: Loss = 2.9315, Perplexity = 18.76 +Step 35760/44000 | Loss: 2.9352 | LR: 4.35e-05 | 1261.2ms/step | 15.9 samples/s +Step 35770/44000 | Loss: 2.9477 | LR: 4.34e-05 | 694.3ms/step | 28.8 samples/s +Step 35780/44000 | Loss: 2.9126 | LR: 4.33e-05 | 694.6ms/step | 28.8 samples/s +Step 35790/44000 | Loss: 2.9263 | LR: 4.32e-05 | 695.2ms/step | 28.8 samples/s +Step 35800/44000 | Loss: 2.9559 | LR: 4.31e-05 | 694.0ms/step | 28.8 samples/s +Step 35810/44000 | Loss: 2.9054 | LR: 4.29e-05 | 693.5ms/step | 28.8 samples/s +Step 35820/44000 | Loss: 2.9251 | LR: 4.28e-05 | 693.9ms/step | 28.8 samples/s +Step 35830/44000 | Loss: 2.8948 | LR: 4.27e-05 | 694.1ms/step | 28.8 samples/s +Step 35840/44000 | Loss: 2.9363 | LR: 4.26e-05 | 691.5ms/step | 28.9 samples/s +Step 35850/44000 | Loss: 2.9333 | LR: 4.25e-05 | 692.2ms/step | 28.9 samples/s +Step 35860/44000 | Loss: 2.9069 | LR: 4.24e-05 | 692.7ms/step | 28.9 samples/s +Step 35870/44000 | Loss: 2.9753 | LR: 4.23e-05 | 692.7ms/step | 28.9 samples/s +Step 35880/44000 | Loss: 2.8860 | LR: 4.22e-05 | 693.3ms/step | 28.8 samples/s +Step 35890/44000 | Loss: 2.9160 | LR: 4.21e-05 | 694.5ms/step | 28.8 samples/s +Step 35900/44000 | Loss: 2.9142 | LR: 4.20e-05 | 692.6ms/step | 28.9 samples/s +Step 35910/44000 | Loss: 2.9248 | LR: 4.19e-05 | 693.9ms/step | 28.8 samples/s +Step 35920/44000 | Loss: 2.9381 | LR: 4.18e-05 | 694.6ms/step | 28.8 samples/s +Step 35930/44000 | Loss: 2.9182 | LR: 4.17e-05 | 693.3ms/step | 28.8 samples/s +Step 35940/44000 | Loss: 2.9093 | LR: 4.16e-05 | 695.1ms/step | 28.8 samples/s +Step 35950/44000 | Loss: 2.9080 | LR: 4.15e-05 | 693.5ms/step | 28.8 samples/s +Step 35960/44000 | Loss: 2.9383 | LR: 4.14e-05 | 695.7ms/step | 28.7 samples/s +Step 35970/44000 | Loss: 2.9003 | LR: 4.13e-05 | 693.8ms/step | 28.8 samples/s +Step 35980/44000 | Loss: 2.9372 | LR: 4.12e-05 | 693.3ms/step | 28.8 samples/s +Step 35990/44000 | Loss: 2.9218 | LR: 4.11e-05 | 692.4ms/step | 28.9 samples/s +Step 36000/44000 | Loss: 2.9323 | LR: 4.10e-05 | 692.4ms/step | 28.9 samples/s +Validation at step 36000: Loss = 2.9317, Perplexity = 18.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-36000 +Step 36010/44000 | Loss: 2.9250 | LR: 4.09e-05 | 1574.5ms/step | 12.7 samples/s +Step 36020/44000 | Loss: 2.9504 | LR: 4.08e-05 | 692.1ms/step | 28.9 samples/s +Step 36030/44000 | Loss: 2.9306 | LR: 4.07e-05 | 691.5ms/step | 28.9 samples/s +Step 36040/44000 | Loss: 2.8799 | LR: 4.06e-05 | 694.2ms/step | 28.8 samples/s +Step 36050/44000 | Loss: 2.9341 | LR: 4.05e-05 | 694.1ms/step | 28.8 samples/s +Step 36060/44000 | Loss: 2.9431 | LR: 4.04e-05 | 694.6ms/step | 28.8 samples/s +Step 36070/44000 | Loss: 2.9246 | LR: 4.03e-05 | 693.4ms/step | 28.8 samples/s +Step 36080/44000 | Loss: 2.8805 | LR: 4.02e-05 | 692.6ms/step | 28.9 samples/s +Step 36090/44000 | Loss: 2.9328 | LR: 4.01e-05 | 692.9ms/step | 28.9 samples/s +Step 36100/44000 | Loss: 2.9176 | LR: 4.00e-05 | 692.6ms/step | 28.9 samples/s +Step 36110/44000 | Loss: 2.9105 | LR: 3.99e-05 | 692.9ms/step | 28.9 samples/s +Step 36120/44000 | Loss: 2.8886 | LR: 3.98e-05 | 691.8ms/step | 28.9 samples/s +Step 36130/44000 | Loss: 2.9249 | LR: 3.97e-05 | 692.8ms/step | 28.9 samples/s +Step 36140/44000 | Loss: 2.9075 | LR: 3.97e-05 | 693.2ms/step | 28.9 samples/s +Step 36150/44000 | Loss: 2.9022 | LR: 3.96e-05 | 693.7ms/step | 28.8 samples/s +Step 36160/44000 | Loss: 2.9377 | LR: 3.95e-05 | 691.8ms/step | 28.9 samples/s +Step 36170/44000 | Loss: 2.9142 | LR: 3.94e-05 | 694.5ms/step | 28.8 samples/s +Step 36180/44000 | Loss: 2.9124 | LR: 3.93e-05 | 696.3ms/step | 28.7 samples/s +Step 36190/44000 | Loss: 2.9529 | LR: 3.92e-05 | 695.8ms/step | 28.7 samples/s +Step 36200/44000 | Loss: 2.9352 | LR: 3.91e-05 | 694.9ms/step | 28.8 samples/s +Step 36210/44000 | Loss: 2.8806 | LR: 3.90e-05 | 694.2ms/step | 28.8 samples/s +Step 36220/44000 | Loss: 2.8949 | LR: 3.89e-05 | 696.5ms/step | 28.7 samples/s +Step 36230/44000 | Loss: 2.9413 | LR: 3.88e-05 | 695.5ms/step | 28.8 samples/s +Step 36240/44000 | Loss: 2.9325 | LR: 3.87e-05 | 691.0ms/step | 28.9 samples/s +Step 36250/44000 | Loss: 2.9058 | LR: 3.86e-05 | 692.8ms/step | 28.9 samples/s +Validation at step 36250: Loss = 2.9319, Perplexity = 18.76 +Step 36260/44000 | Loss: 2.8996 | LR: 3.85e-05 | 1259.1ms/step | 15.9 samples/s +Step 36270/44000 | Loss: 2.9077 | LR: 3.84e-05 | 691.1ms/step | 28.9 samples/s +Step 36280/44000 | Loss: 2.9513 | LR: 3.83e-05 | 693.7ms/step | 28.8 samples/s +Step 36290/44000 | Loss: 2.8938 | LR: 3.82e-05 | 691.9ms/step | 28.9 samples/s +Step 36300/44000 | Loss: 2.9544 | LR: 3.81e-05 | 691.8ms/step | 28.9 samples/s +Step 36310/44000 | Loss: 2.9246 | LR: 3.80e-05 | 691.8ms/step | 28.9 samples/s +Step 36320/44000 | Loss: 2.9580 | LR: 3.79e-05 | 691.5ms/step | 28.9 samples/s +Step 36330/44000 | Loss: 2.9413 | LR: 3.78e-05 | 693.0ms/step | 28.9 samples/s +Step 36340/44000 | Loss: 2.9031 | LR: 3.77e-05 | 691.8ms/step | 28.9 samples/s +Step 36350/44000 | Loss: 2.9103 | LR: 3.76e-05 | 691.2ms/step | 28.9 samples/s +Step 36360/44000 | Loss: 2.9188 | LR: 3.75e-05 | 690.6ms/step | 29.0 samples/s +Step 36370/44000 | Loss: 2.8731 | LR: 3.74e-05 | 693.6ms/step | 28.8 samples/s +Step 36380/44000 | Loss: 2.9222 | LR: 3.73e-05 | 694.3ms/step | 28.8 samples/s +Step 36390/44000 | Loss: 2.9101 | LR: 3.72e-05 | 691.6ms/step | 28.9 samples/s +Step 36400/44000 | Loss: 2.9207 | LR: 3.71e-05 | 691.8ms/step | 28.9 samples/s +Step 36410/44000 | Loss: 2.9563 | LR: 3.70e-05 | 691.6ms/step | 28.9 samples/s +Step 36420/44000 | Loss: 2.9183 | LR: 3.69e-05 | 692.0ms/step | 28.9 samples/s +Step 36430/44000 | Loss: 2.9245 | LR: 3.69e-05 | 692.3ms/step | 28.9 samples/s +Step 36440/44000 | Loss: 2.9456 | LR: 3.68e-05 | 694.4ms/step | 28.8 samples/s +Step 36450/44000 | Loss: 2.9107 | LR: 3.67e-05 | 693.0ms/step | 28.9 samples/s +Step 36460/44000 | Loss: 2.9401 | LR: 3.66e-05 | 694.6ms/step | 28.8 samples/s +Step 36470/44000 | Loss: 2.8878 | LR: 3.65e-05 | 694.1ms/step | 28.8 samples/s +Step 36480/44000 | Loss: 2.9540 | LR: 3.64e-05 | 692.4ms/step | 28.9 samples/s +Step 36490/44000 | Loss: 2.9510 | LR: 3.63e-05 | 694.6ms/step | 28.8 samples/s +Step 36500/44000 | Loss: 2.8782 | LR: 3.62e-05 | 692.8ms/step | 28.9 samples/s +Validation at step 36500: Loss = 2.9309, Perplexity = 18.74 +Step 36510/44000 | Loss: 2.8921 | LR: 3.61e-05 | 1259.7ms/step | 15.9 samples/s +Step 36520/44000 | Loss: 2.9009 | LR: 3.60e-05 | 693.4ms/step | 28.8 samples/s +Step 36530/44000 | Loss: 2.9037 | LR: 3.59e-05 | 695.3ms/step | 28.8 samples/s +Step 36540/44000 | Loss: 2.9291 | LR: 3.58e-05 | 693.3ms/step | 28.8 samples/s +Step 36550/44000 | Loss: 2.8772 | LR: 3.57e-05 | 692.9ms/step | 28.9 samples/s +Step 36560/44000 | Loss: 2.9041 | LR: 3.56e-05 | 693.8ms/step | 28.8 samples/s +Step 36570/44000 | Loss: 2.8936 | LR: 3.55e-05 | 694.9ms/step | 28.8 samples/s +Step 36580/44000 | Loss: 2.8987 | LR: 3.54e-05 | 694.5ms/step | 28.8 samples/s +Step 36590/44000 | Loss: 2.8988 | LR: 3.53e-05 | 695.5ms/step | 28.8 samples/s +Step 36600/44000 | Loss: 2.9394 | LR: 3.53e-05 | 693.7ms/step | 28.8 samples/s +Step 36610/44000 | Loss: 2.9087 | LR: 3.52e-05 | 694.6ms/step | 28.8 samples/s +Step 36620/44000 | Loss: 2.9415 | LR: 3.51e-05 | 692.7ms/step | 28.9 samples/s +Step 36630/44000 | Loss: 2.9231 | LR: 3.50e-05 | 693.8ms/step | 28.8 samples/s +Step 36640/44000 | Loss: 2.9478 | LR: 3.49e-05 | 695.1ms/step | 28.8 samples/s +Step 36650/44000 | Loss: 2.9244 | LR: 3.48e-05 | 693.3ms/step | 28.8 samples/s +Step 36660/44000 | Loss: 2.9013 | LR: 3.47e-05 | 695.3ms/step | 28.8 samples/s +Step 36670/44000 | Loss: 2.8991 | LR: 3.46e-05 | 694.0ms/step | 28.8 samples/s +Step 36680/44000 | Loss: 2.9184 | LR: 3.45e-05 | 693.4ms/step | 28.8 samples/s +Step 36690/44000 | Loss: 2.9226 | LR: 3.44e-05 | 694.1ms/step | 28.8 samples/s +Step 36700/44000 | Loss: 2.9090 | LR: 3.43e-05 | 692.9ms/step | 28.9 samples/s +Step 36710/44000 | Loss: 2.9030 | LR: 3.42e-05 | 693.6ms/step | 28.8 samples/s +Step 36720/44000 | Loss: 2.9472 | LR: 3.41e-05 | 691.9ms/step | 28.9 samples/s +Step 36730/44000 | Loss: 2.9572 | LR: 3.41e-05 | 693.4ms/step | 28.8 samples/s +Step 36740/44000 | Loss: 2.8848 | LR: 3.40e-05 | 694.2ms/step | 28.8 samples/s +Step 36750/44000 | Loss: 2.9472 | LR: 3.39e-05 | 697.3ms/step | 28.7 samples/s +Validation at step 36750: Loss = 2.9308, Perplexity = 18.74 +Step 36760/44000 | Loss: 2.9027 | LR: 3.38e-05 | 1259.0ms/step | 15.9 samples/s +Step 36770/44000 | Loss: 2.8868 | LR: 3.37e-05 | 692.3ms/step | 28.9 samples/s +Step 36780/44000 | Loss: 2.9033 | LR: 3.36e-05 | 694.7ms/step | 28.8 samples/s +Step 36790/44000 | Loss: 2.9645 | LR: 3.35e-05 | 694.1ms/step | 28.8 samples/s +Step 36800/44000 | Loss: 2.8965 | LR: 3.34e-05 | 693.3ms/step | 28.8 samples/s +Step 36810/44000 | Loss: 2.9368 | LR: 3.33e-05 | 692.3ms/step | 28.9 samples/s +Step 36820/44000 | Loss: 2.9219 | LR: 3.32e-05 | 693.9ms/step | 28.8 samples/s +Step 36830/44000 | Loss: 2.8972 | LR: 3.31e-05 | 693.5ms/step | 28.8 samples/s +Step 36840/44000 | Loss: 2.9974 | LR: 3.31e-05 | 692.5ms/step | 28.9 samples/s +Step 36850/44000 | Loss: 2.9024 | LR: 3.30e-05 | 692.4ms/step | 28.9 samples/s +Step 36860/44000 | Loss: 2.8946 | LR: 3.29e-05 | 692.1ms/step | 28.9 samples/s +Step 36870/44000 | Loss: 2.9403 | LR: 3.28e-05 | 694.1ms/step | 28.8 samples/s +Step 36880/44000 | Loss: 2.9524 | LR: 3.27e-05 | 693.6ms/step | 28.8 samples/s +Step 36890/44000 | Loss: 2.9308 | LR: 3.26e-05 | 693.6ms/step | 28.8 samples/s +Step 36900/44000 | Loss: 2.9330 | LR: 3.25e-05 | 692.8ms/step | 28.9 samples/s +Step 36910/44000 | Loss: 2.9039 | LR: 3.24e-05 | 693.3ms/step | 28.8 samples/s +Step 36920/44000 | Loss: 2.9110 | LR: 3.23e-05 | 693.3ms/step | 28.8 samples/s +Step 36930/44000 | Loss: 2.9182 | LR: 3.22e-05 | 692.8ms/step | 28.9 samples/s +Step 36940/44000 | Loss: 2.9089 | LR: 3.22e-05 | 692.0ms/step | 28.9 samples/s +Step 36950/44000 | Loss: 2.9250 | LR: 3.21e-05 | 693.8ms/step | 28.8 samples/s +Step 36960/44000 | Loss: 2.9399 | LR: 3.20e-05 | 692.3ms/step | 28.9 samples/s +Step 36970/44000 | Loss: 2.8515 | LR: 3.19e-05 | 693.1ms/step | 28.9 samples/s +Step 36980/44000 | Loss: 2.9517 | LR: 3.18e-05 | 692.3ms/step | 28.9 samples/s +Step 36990/44000 | Loss: 2.9302 | LR: 3.17e-05 | 692.6ms/step | 28.9 samples/s +Step 37000/44000 | Loss: 2.9407 | LR: 3.16e-05 | 693.2ms/step | 28.9 samples/s +Validation at step 37000: Loss = 2.9310, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-37000 +Step 37010/44000 | Loss: 2.9042 | LR: 3.15e-05 | 1568.8ms/step | 12.7 samples/s +Step 37020/44000 | Loss: 2.8787 | LR: 3.15e-05 | 693.1ms/step | 28.9 samples/s +Step 37030/44000 | Loss: 2.9187 | LR: 3.14e-05 | 692.8ms/step | 28.9 samples/s +Step 37040/44000 | Loss: 2.8766 | LR: 3.13e-05 | 694.6ms/step | 28.8 samples/s +Step 37050/44000 | Loss: 2.9303 | LR: 3.12e-05 | 694.0ms/step | 28.8 samples/s +Step 37060/44000 | Loss: 2.9169 | LR: 3.11e-05 | 694.9ms/step | 28.8 samples/s +Step 37070/44000 | Loss: 2.9015 | LR: 3.10e-05 | 694.6ms/step | 28.8 samples/s +Step 37080/44000 | Loss: 2.9010 | LR: 3.09e-05 | 692.6ms/step | 28.9 samples/s +Step 37090/44000 | Loss: 2.9510 | LR: 3.08e-05 | 694.0ms/step | 28.8 samples/s +Step 37100/44000 | Loss: 2.9488 | LR: 3.07e-05 | 695.1ms/step | 28.8 samples/s +Step 37110/44000 | Loss: 2.9500 | LR: 3.07e-05 | 692.5ms/step | 28.9 samples/s +Step 37120/44000 | Loss: 2.9310 | LR: 3.06e-05 | 693.4ms/step | 28.8 samples/s +Step 37130/44000 | Loss: 2.9022 | LR: 3.05e-05 | 693.7ms/step | 28.8 samples/s +Step 37140/44000 | Loss: 2.9052 | LR: 3.04e-05 | 695.8ms/step | 28.7 samples/s +Step 37150/44000 | Loss: 2.8848 | LR: 3.03e-05 | 694.0ms/step | 28.8 samples/s +Step 37160/44000 | Loss: 2.9080 | LR: 3.02e-05 | 693.9ms/step | 28.8 samples/s +Step 37170/44000 | Loss: 2.9182 | LR: 3.01e-05 | 692.9ms/step | 28.9 samples/s +Step 37180/44000 | Loss: 2.8762 | LR: 3.01e-05 | 692.4ms/step | 28.9 samples/s +Step 37190/44000 | Loss: 2.9314 | LR: 3.00e-05 | 692.7ms/step | 28.9 samples/s +Step 37200/44000 | Loss: 2.8934 | LR: 2.99e-05 | 692.4ms/step | 28.9 samples/s +Step 37210/44000 | Loss: 2.9118 | LR: 2.98e-05 | 694.0ms/step | 28.8 samples/s +Step 37220/44000 | Loss: 2.9075 | LR: 2.97e-05 | 694.6ms/step | 28.8 samples/s +Step 37230/44000 | Loss: 2.9167 | LR: 2.96e-05 | 694.2ms/step | 28.8 samples/s +Step 37240/44000 | Loss: 2.8745 | LR: 2.95e-05 | 694.3ms/step | 28.8 samples/s +Step 37250/44000 | Loss: 2.8938 | LR: 2.95e-05 | 693.7ms/step | 28.8 samples/s +Validation at step 37250: Loss = 2.9313, Perplexity = 18.75 +Step 37260/44000 | Loss: 2.9243 | LR: 2.94e-05 | 1261.6ms/step | 15.9 samples/s +Step 37270/44000 | Loss: 2.9473 | LR: 2.93e-05 | 696.2ms/step | 28.7 samples/s +Step 37280/44000 | Loss: 2.9035 | LR: 2.92e-05 | 693.6ms/step | 28.8 samples/s +Step 37290/44000 | Loss: 2.9369 | LR: 2.91e-05 | 694.1ms/step | 28.8 samples/s +Step 37300/44000 | Loss: 2.9068 | LR: 2.90e-05 | 698.7ms/step | 28.6 samples/s +Step 37310/44000 | Loss: 2.9092 | LR: 2.89e-05 | 697.3ms/step | 28.7 samples/s +Step 37320/44000 | Loss: 2.9450 | LR: 2.89e-05 | 694.9ms/step | 28.8 samples/s +Step 37330/44000 | Loss: 2.9494 | LR: 2.88e-05 | 693.4ms/step | 28.8 samples/s +Step 37340/44000 | Loss: 2.9133 | LR: 2.87e-05 | 693.8ms/step | 28.8 samples/s +Step 37350/44000 | Loss: 2.9039 | LR: 2.86e-05 | 695.3ms/step | 28.8 samples/s +Step 37360/44000 | Loss: 2.8943 | LR: 2.85e-05 | 693.2ms/step | 28.9 samples/s +Step 37370/44000 | Loss: 2.9469 | LR: 2.84e-05 | 694.3ms/step | 28.8 samples/s +Step 37380/44000 | Loss: 2.9432 | LR: 2.84e-05 | 693.0ms/step | 28.9 samples/s +Step 37390/44000 | Loss: 2.9302 | LR: 2.83e-05 | 691.5ms/step | 28.9 samples/s +Step 37400/44000 | Loss: 2.9127 | LR: 2.82e-05 | 696.3ms/step | 28.7 samples/s +Step 37410/44000 | Loss: 2.9513 | LR: 2.81e-05 | 692.0ms/step | 28.9 samples/s +Step 37420/44000 | Loss: 2.9211 | LR: 2.80e-05 | 691.9ms/step | 28.9 samples/s +Step 37430/44000 | Loss: 2.9600 | LR: 2.79e-05 | 695.6ms/step | 28.8 samples/s +Step 37440/44000 | Loss: 2.9260 | LR: 2.78e-05 | 694.3ms/step | 28.8 samples/s +Step 37450/44000 | Loss: 2.9100 | LR: 2.78e-05 | 694.7ms/step | 28.8 samples/s +Step 37460/44000 | Loss: 2.9252 | LR: 2.77e-05 | 694.1ms/step | 28.8 samples/s +Step 37470/44000 | Loss: 2.9115 | LR: 2.76e-05 | 693.8ms/step | 28.8 samples/s +Step 37480/44000 | Loss: 2.9287 | LR: 2.75e-05 | 697.5ms/step | 28.7 samples/s +Step 37490/44000 | Loss: 2.8555 | LR: 2.74e-05 | 693.5ms/step | 28.8 samples/s +Step 37500/44000 | Loss: 2.8858 | LR: 2.74e-05 | 692.7ms/step | 28.9 samples/s +Validation at step 37500: Loss = 2.9307, Perplexity = 18.74 +Step 37510/44000 | Loss: 2.9089 | LR: 2.73e-05 | 1262.1ms/step | 15.8 samples/s +Step 37520/44000 | Loss: 2.9534 | LR: 2.72e-05 | 694.9ms/step | 28.8 samples/s +Step 37530/44000 | Loss: 2.9196 | LR: 2.71e-05 | 693.6ms/step | 28.8 samples/s +Step 37540/44000 | Loss: 2.9011 | LR: 2.70e-05 | 693.6ms/step | 28.8 samples/s +Step 37550/44000 | Loss: 2.9192 | LR: 2.69e-05 | 695.2ms/step | 28.8 samples/s +Step 37560/44000 | Loss: 2.9197 | LR: 2.69e-05 | 691.6ms/step | 28.9 samples/s +Step 37570/44000 | Loss: 2.8875 | LR: 2.68e-05 | 694.6ms/step | 28.8 samples/s +Step 37580/44000 | Loss: 2.9293 | LR: 2.67e-05 | 693.4ms/step | 28.8 samples/s +Step 37590/44000 | Loss: 2.9054 | LR: 2.66e-05 | 692.9ms/step | 28.9 samples/s +Step 37600/44000 | Loss: 2.9417 | LR: 2.65e-05 | 691.7ms/step | 28.9 samples/s +Step 37610/44000 | Loss: 2.9148 | LR: 2.65e-05 | 693.7ms/step | 28.8 samples/s +Step 37620/44000 | Loss: 2.9300 | LR: 2.64e-05 | 692.5ms/step | 28.9 samples/s +Step 37630/44000 | Loss: 2.9395 | LR: 2.63e-05 | 692.9ms/step | 28.9 samples/s +Step 37640/44000 | Loss: 2.9395 | LR: 2.62e-05 | 693.2ms/step | 28.9 samples/s +Step 37650/44000 | Loss: 2.9282 | LR: 2.61e-05 | 690.7ms/step | 29.0 samples/s +Step 37660/44000 | Loss: 2.9461 | LR: 2.60e-05 | 693.5ms/step | 28.8 samples/s +Step 37670/44000 | Loss: 2.9263 | LR: 2.60e-05 | 693.1ms/step | 28.9 samples/s +Step 37680/44000 | Loss: 2.9384 | LR: 2.59e-05 | 692.7ms/step | 28.9 samples/s +Step 37690/44000 | Loss: 2.8964 | LR: 2.58e-05 | 691.5ms/step | 28.9 samples/s +Step 37700/44000 | Loss: 2.9036 | LR: 2.57e-05 | 694.5ms/step | 28.8 samples/s +Step 37710/44000 | Loss: 2.9406 | LR: 2.56e-05 | 693.0ms/step | 28.9 samples/s +Step 37720/44000 | Loss: 2.9223 | LR: 2.56e-05 | 692.9ms/step | 28.9 samples/s +Step 37730/44000 | Loss: 2.9345 | LR: 2.55e-05 | 694.8ms/step | 28.8 samples/s +Step 37740/44000 | Loss: 2.8637 | LR: 2.54e-05 | 693.4ms/step | 28.8 samples/s +Step 37750/44000 | Loss: 2.9478 | LR: 2.53e-05 | 693.7ms/step | 28.8 samples/s +Validation at step 37750: Loss = 2.9311, Perplexity = 18.75 +Step 37760/44000 | Loss: 2.8932 | LR: 2.52e-05 | 1260.1ms/step | 15.9 samples/s +Step 37770/44000 | Loss: 2.9395 | LR: 2.52e-05 | 693.9ms/step | 28.8 samples/s +Step 37780/44000 | Loss: 2.9475 | LR: 2.51e-05 | 692.9ms/step | 28.9 samples/s +Step 37790/44000 | Loss: 2.9529 | LR: 2.50e-05 | 693.1ms/step | 28.9 samples/s +Step 37800/44000 | Loss: 2.9246 | LR: 2.49e-05 | 692.3ms/step | 28.9 samples/s +Step 37810/44000 | Loss: 2.9085 | LR: 2.48e-05 | 692.6ms/step | 28.9 samples/s +Step 37820/44000 | Loss: 2.9322 | LR: 2.48e-05 | 691.8ms/step | 28.9 samples/s +Step 37830/44000 | Loss: 2.9275 | LR: 2.47e-05 | 692.5ms/step | 28.9 samples/s +Step 37840/44000 | Loss: 2.9116 | LR: 2.46e-05 | 692.9ms/step | 28.9 samples/s +Step 37850/44000 | Loss: 2.9027 | LR: 2.45e-05 | 692.9ms/step | 28.9 samples/s +Step 37860/44000 | Loss: 2.9524 | LR: 2.45e-05 | 693.1ms/step | 28.9 samples/s +Step 37870/44000 | Loss: 2.9204 | LR: 2.44e-05 | 692.8ms/step | 28.9 samples/s +Step 37880/44000 | Loss: 2.9409 | LR: 2.43e-05 | 693.2ms/step | 28.9 samples/s +Step 37890/44000 | Loss: 2.9108 | LR: 2.42e-05 | 693.9ms/step | 28.8 samples/s +Step 37900/44000 | Loss: 2.9540 | LR: 2.41e-05 | 693.6ms/step | 28.8 samples/s +Step 37910/44000 | Loss: 2.9103 | LR: 2.41e-05 | 694.8ms/step | 28.8 samples/s +Step 37920/44000 | Loss: 2.9638 | LR: 2.40e-05 | 693.4ms/step | 28.8 samples/s +Step 37930/44000 | Loss: 2.9487 | LR: 2.39e-05 | 693.1ms/step | 28.9 samples/s +Step 37940/44000 | Loss: 2.9072 | LR: 2.38e-05 | 693.3ms/step | 28.8 samples/s +Step 37950/44000 | Loss: 2.8808 | LR: 2.38e-05 | 692.7ms/step | 28.9 samples/s +Step 37960/44000 | Loss: 2.8912 | LR: 2.37e-05 | 693.3ms/step | 28.8 samples/s +Step 37970/44000 | Loss: 2.9766 | LR: 2.36e-05 | 694.2ms/step | 28.8 samples/s +Step 37980/44000 | Loss: 2.9000 | LR: 2.35e-05 | 692.3ms/step | 28.9 samples/s +Step 37990/44000 | Loss: 2.9070 | LR: 2.34e-05 | 690.8ms/step | 29.0 samples/s +Step 38000/44000 | Loss: 2.8973 | LR: 2.34e-05 | 693.1ms/step | 28.9 samples/s +Validation at step 38000: Loss = 2.9309, Perplexity = 18.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-38000 +Step 38010/44000 | Loss: 2.9140 | LR: 2.33e-05 | 1567.9ms/step | 12.8 samples/s +Step 38020/44000 | Loss: 2.8791 | LR: 2.32e-05 | 693.5ms/step | 28.8 samples/s +Step 38030/44000 | Loss: 2.8972 | LR: 2.31e-05 | 692.5ms/step | 28.9 samples/s +Step 38040/44000 | Loss: 2.9281 | LR: 2.31e-05 | 694.0ms/step | 28.8 samples/s +Step 38050/44000 | Loss: 2.8996 | LR: 2.30e-05 | 693.0ms/step | 28.9 samples/s +Step 38060/44000 | Loss: 2.9183 | LR: 2.29e-05 | 691.9ms/step | 28.9 samples/s +Step 38070/44000 | Loss: 2.9507 | LR: 2.28e-05 | 694.7ms/step | 28.8 samples/s +Step 38080/44000 | Loss: 2.9603 | LR: 2.28e-05 | 693.6ms/step | 28.8 samples/s +Step 38090/44000 | Loss: 2.8712 | LR: 2.27e-05 | 693.7ms/step | 28.8 samples/s +Step 38100/44000 | Loss: 2.9513 | LR: 2.26e-05 | 692.9ms/step | 28.9 samples/s +Step 38110/44000 | Loss: 2.8829 | LR: 2.25e-05 | 693.2ms/step | 28.9 samples/s +Step 38120/44000 | Loss: 2.8991 | LR: 2.25e-05 | 692.4ms/step | 28.9 samples/s +Step 38130/44000 | Loss: 2.9633 | LR: 2.24e-05 | 694.8ms/step | 28.8 samples/s +Step 38140/44000 | Loss: 2.9169 | LR: 2.23e-05 | 692.4ms/step | 28.9 samples/s +Step 38150/44000 | Loss: 2.9016 | LR: 2.22e-05 | 693.9ms/step | 28.8 samples/s +Step 38160/44000 | Loss: 2.8775 | LR: 2.22e-05 | 692.7ms/step | 28.9 samples/s +Step 38170/44000 | Loss: 2.9028 | LR: 2.21e-05 | 693.4ms/step | 28.8 samples/s +Step 38180/44000 | Loss: 2.9394 | LR: 2.20e-05 | 691.3ms/step | 28.9 samples/s +Step 38190/44000 | Loss: 2.9051 | LR: 2.19e-05 | 694.2ms/step | 28.8 samples/s +Step 38200/44000 | Loss: 2.8698 | LR: 2.19e-05 | 692.1ms/step | 28.9 samples/s +Step 38210/44000 | Loss: 2.9243 | LR: 2.18e-05 | 691.9ms/step | 28.9 samples/s +Step 38220/44000 | Loss: 2.8898 | LR: 2.17e-05 | 694.9ms/step | 28.8 samples/s +Step 38230/44000 | Loss: 2.8922 | LR: 2.16e-05 | 693.0ms/step | 28.9 samples/s +Step 38240/44000 | Loss: 2.9394 | LR: 2.16e-05 | 694.0ms/step | 28.8 samples/s +Step 38250/44000 | Loss: 2.9164 | LR: 2.15e-05 | 692.9ms/step | 28.9 samples/s +Validation at step 38250: Loss = 2.9301, Perplexity = 18.73 +Step 38260/44000 | Loss: 2.8757 | LR: 2.14e-05 | 1258.2ms/step | 15.9 samples/s +Step 38270/44000 | Loss: 2.8821 | LR: 2.13e-05 | 692.6ms/step | 28.9 samples/s +Step 38280/44000 | Loss: 2.8860 | LR: 2.13e-05 | 693.8ms/step | 28.8 samples/s +Step 38290/44000 | Loss: 2.9003 | LR: 2.12e-05 | 694.3ms/step | 28.8 samples/s +Step 38300/44000 | Loss: 2.8940 | LR: 2.11e-05 | 693.6ms/step | 28.8 samples/s +Step 38310/44000 | Loss: 2.8724 | LR: 2.11e-05 | 693.3ms/step | 28.8 samples/s +Step 38320/44000 | Loss: 2.9299 | LR: 2.10e-05 | 690.3ms/step | 29.0 samples/s +Step 38330/44000 | Loss: 2.9268 | LR: 2.09e-05 | 691.2ms/step | 28.9 samples/s +Step 38340/44000 | Loss: 2.9529 | LR: 2.08e-05 | 690.0ms/step | 29.0 samples/s +Step 38350/44000 | Loss: 2.8909 | LR: 2.08e-05 | 691.4ms/step | 28.9 samples/s +Step 38360/44000 | Loss: 2.8917 | LR: 2.07e-05 | 690.9ms/step | 28.9 samples/s +Step 38370/44000 | Loss: 2.9269 | LR: 2.06e-05 | 690.1ms/step | 29.0 samples/s +Step 38380/44000 | Loss: 2.9076 | LR: 2.05e-05 | 692.7ms/step | 28.9 samples/s +Step 38390/44000 | Loss: 2.9074 | LR: 2.05e-05 | 691.8ms/step | 28.9 samples/s +Step 38400/44000 | Loss: 2.9640 | LR: 2.04e-05 | 691.6ms/step | 28.9 samples/s +Step 38410/44000 | Loss: 2.9633 | LR: 2.03e-05 | 692.2ms/step | 28.9 samples/s +Step 38420/44000 | Loss: 2.9257 | LR: 2.03e-05 | 691.5ms/step | 28.9 samples/s +Step 38430/44000 | Loss: 2.9298 | LR: 2.02e-05 | 692.6ms/step | 28.9 samples/s +Step 38440/44000 | Loss: 2.9555 | LR: 2.01e-05 | 690.3ms/step | 29.0 samples/s +Step 38450/44000 | Loss: 2.9157 | LR: 2.00e-05 | 691.5ms/step | 28.9 samples/s +Step 38460/44000 | Loss: 2.9022 | LR: 2.00e-05 | 692.8ms/step | 28.9 samples/s +Step 38470/44000 | Loss: 2.9238 | LR: 1.99e-05 | 692.4ms/step | 28.9 samples/s +Step 38480/44000 | Loss: 2.9268 | LR: 1.98e-05 | 692.5ms/step | 28.9 samples/s +Step 38490/44000 | Loss: 2.9402 | LR: 1.98e-05 | 691.9ms/step | 28.9 samples/s +Step 38500/44000 | Loss: 2.9369 | LR: 1.97e-05 | 693.8ms/step | 28.8 samples/s +Validation at step 38500: Loss = 2.9304, Perplexity = 18.74 +Step 38510/44000 | Loss: 2.9667 | LR: 1.96e-05 | 1260.8ms/step | 15.9 samples/s +Step 38520/44000 | Loss: 2.9252 | LR: 1.95e-05 | 695.4ms/step | 28.8 samples/s +Step 38530/44000 | Loss: 2.9531 | LR: 1.95e-05 | 693.7ms/step | 28.8 samples/s +Step 38540/44000 | Loss: 2.9164 | LR: 1.94e-05 | 692.7ms/step | 28.9 samples/s +Step 38550/44000 | Loss: 2.9180 | LR: 1.93e-05 | 693.5ms/step | 28.8 samples/s +Step 38560/44000 | Loss: 2.9263 | LR: 1.93e-05 | 693.1ms/step | 28.9 samples/s +Step 38570/44000 | Loss: 2.8881 | LR: 1.92e-05 | 692.6ms/step | 28.9 samples/s +Step 38580/44000 | Loss: 2.9107 | LR: 1.91e-05 | 693.2ms/step | 28.9 samples/s +Step 38590/44000 | Loss: 2.9232 | LR: 1.91e-05 | 691.7ms/step | 28.9 samples/s +Step 38600/44000 | Loss: 2.9179 | LR: 1.90e-05 | 693.8ms/step | 28.8 samples/s +Step 38610/44000 | Loss: 2.9382 | LR: 1.89e-05 | 694.0ms/step | 28.8 samples/s +Step 38620/44000 | Loss: 2.8915 | LR: 1.88e-05 | 695.3ms/step | 28.8 samples/s +Step 38630/44000 | Loss: 2.8960 | LR: 1.88e-05 | 693.5ms/step | 28.8 samples/s +Step 38640/44000 | Loss: 2.9575 | LR: 1.87e-05 | 693.6ms/step | 28.8 samples/s +Step 38650/44000 | Loss: 2.9416 | LR: 1.86e-05 | 694.1ms/step | 28.8 samples/s +Step 38660/44000 | Loss: 2.9093 | LR: 1.86e-05 | 695.0ms/step | 28.8 samples/s +Step 38670/44000 | Loss: 2.8758 | LR: 1.85e-05 | 692.5ms/step | 28.9 samples/s +Step 38680/44000 | Loss: 2.9024 | LR: 1.84e-05 | 693.0ms/step | 28.9 samples/s +Step 38690/44000 | Loss: 2.8875 | LR: 1.84e-05 | 695.4ms/step | 28.8 samples/s +Step 38700/44000 | Loss: 2.9065 | LR: 1.83e-05 | 694.0ms/step | 28.8 samples/s +Step 38710/44000 | Loss: 2.9215 | LR: 1.82e-05 | 692.2ms/step | 28.9 samples/s +Step 38720/44000 | Loss: 2.8964 | LR: 1.82e-05 | 694.3ms/step | 28.8 samples/s +Step 38730/44000 | Loss: 2.9514 | LR: 1.81e-05 | 694.3ms/step | 28.8 samples/s +Step 38740/44000 | Loss: 2.9166 | LR: 1.80e-05 | 694.5ms/step | 28.8 samples/s +Step 38750/44000 | Loss: 2.8907 | LR: 1.80e-05 | 695.0ms/step | 28.8 samples/s +Validation at step 38750: Loss = 2.9307, Perplexity = 18.74 +Step 38760/44000 | Loss: 2.9294 | LR: 1.79e-05 | 1263.5ms/step | 15.8 samples/s +Step 38770/44000 | Loss: 2.8985 | LR: 1.78e-05 | 694.6ms/step | 28.8 samples/s +Step 38780/44000 | Loss: 2.9577 | LR: 1.78e-05 | 690.6ms/step | 29.0 samples/s +Step 38790/44000 | Loss: 2.9217 | LR: 1.77e-05 | 690.5ms/step | 29.0 samples/s +Step 38800/44000 | Loss: 2.9555 | LR: 1.76e-05 | 690.8ms/step | 29.0 samples/s +Step 38810/44000 | Loss: 2.9123 | LR: 1.76e-05 | 690.1ms/step | 29.0 samples/s +Step 38820/44000 | Loss: 2.8944 | LR: 1.75e-05 | 691.4ms/step | 28.9 samples/s +Step 38830/44000 | Loss: 2.9140 | LR: 1.74e-05 | 691.2ms/step | 28.9 samples/s +Step 38840/44000 | Loss: 2.8832 | LR: 1.74e-05 | 691.2ms/step | 28.9 samples/s +Step 38850/44000 | Loss: 2.9133 | LR: 1.73e-05 | 690.9ms/step | 28.9 samples/s +Step 38860/44000 | Loss: 2.9063 | LR: 1.72e-05 | 690.8ms/step | 29.0 samples/s +Step 38870/44000 | Loss: 2.9149 | LR: 1.72e-05 | 692.5ms/step | 28.9 samples/s +Step 38880/44000 | Loss: 2.8649 | LR: 1.71e-05 | 692.0ms/step | 28.9 samples/s +Step 38890/44000 | Loss: 2.8883 | LR: 1.70e-05 | 694.2ms/step | 28.8 samples/s +Step 38900/44000 | Loss: 2.9167 | LR: 1.70e-05 | 689.9ms/step | 29.0 samples/s +Step 38910/44000 | Loss: 2.9182 | LR: 1.69e-05 | 692.5ms/step | 28.9 samples/s +Step 38920/44000 | Loss: 2.8705 | LR: 1.68e-05 | 694.2ms/step | 28.8 samples/s +Step 38930/44000 | Loss: 2.8876 | LR: 1.68e-05 | 691.4ms/step | 28.9 samples/s +Step 38940/44000 | Loss: 2.9380 | LR: 1.67e-05 | 692.4ms/step | 28.9 samples/s +Step 38950/44000 | Loss: 2.9305 | LR: 1.66e-05 | 691.0ms/step | 28.9 samples/s +Step 38960/44000 | Loss: 2.9205 | LR: 1.66e-05 | 690.8ms/step | 29.0 samples/s +Step 38970/44000 | Loss: 2.9389 | LR: 1.65e-05 | 692.5ms/step | 28.9 samples/s +Step 38980/44000 | Loss: 2.8895 | LR: 1.64e-05 | 692.6ms/step | 28.9 samples/s +Step 38990/44000 | Loss: 2.9444 | LR: 1.64e-05 | 691.5ms/step | 28.9 samples/s +Step 39000/44000 | Loss: 2.9258 | LR: 1.63e-05 | 694.0ms/step | 28.8 samples/s +Validation at step 39000: Loss = 2.9304, Perplexity = 18.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-39000 +Step 39010/44000 | Loss: 2.9717 | LR: 1.62e-05 | 1574.7ms/step | 12.7 samples/s +Step 39020/44000 | Loss: 2.9543 | LR: 1.62e-05 | 694.4ms/step | 28.8 samples/s +Step 39030/44000 | Loss: 2.8528 | LR: 1.61e-05 | 691.1ms/step | 28.9 samples/s +Step 39040/44000 | Loss: 2.9545 | LR: 1.61e-05 | 692.0ms/step | 28.9 samples/s +Step 39050/44000 | Loss: 2.8755 | LR: 1.60e-05 | 693.5ms/step | 28.8 samples/s +Step 39060/44000 | Loss: 2.9333 | LR: 1.59e-05 | 692.7ms/step | 28.9 samples/s +Step 39070/44000 | Loss: 2.9276 | LR: 1.59e-05 | 694.2ms/step | 28.8 samples/s +Step 39080/44000 | Loss: 2.9558 | LR: 1.58e-05 | 691.7ms/step | 28.9 samples/s +Step 39090/44000 | Loss: 2.9533 | LR: 1.57e-05 | 692.2ms/step | 28.9 samples/s +Step 39100/44000 | Loss: 2.9056 | LR: 1.57e-05 | 693.0ms/step | 28.9 samples/s +Step 39110/44000 | Loss: 2.9191 | LR: 1.56e-05 | 692.8ms/step | 28.9 samples/s +Step 39120/44000 | Loss: 2.9651 | LR: 1.55e-05 | 691.9ms/step | 28.9 samples/s +Step 39130/44000 | Loss: 2.9335 | LR: 1.55e-05 | 694.6ms/step | 28.8 samples/s +Step 39140/44000 | Loss: 2.9482 | LR: 1.54e-05 | 695.2ms/step | 28.8 samples/s +Step 39150/44000 | Loss: 2.9545 | LR: 1.54e-05 | 692.5ms/step | 28.9 samples/s +Step 39160/44000 | Loss: 2.9302 | LR: 1.53e-05 | 692.1ms/step | 28.9 samples/s +Step 39170/44000 | Loss: 2.9108 | LR: 1.52e-05 | 693.4ms/step | 28.8 samples/s +Step 39180/44000 | Loss: 2.9210 | LR: 1.52e-05 | 696.3ms/step | 28.7 samples/s +Step 39190/44000 | Loss: 2.9163 | LR: 1.51e-05 | 692.3ms/step | 28.9 samples/s +Step 39200/44000 | Loss: 2.9278 | LR: 1.50e-05 | 693.0ms/step | 28.9 samples/s +Step 39210/44000 | Loss: 2.9486 | LR: 1.50e-05 | 694.5ms/step | 28.8 samples/s +Step 39220/44000 | Loss: 2.9127 | LR: 1.49e-05 | 694.1ms/step | 28.8 samples/s +Step 39230/44000 | Loss: 2.9182 | LR: 1.49e-05 | 692.4ms/step | 28.9 samples/s +Step 39240/44000 | Loss: 2.8982 | LR: 1.48e-05 | 693.7ms/step | 28.8 samples/s +Step 39250/44000 | Loss: 2.9518 | LR: 1.47e-05 | 691.9ms/step | 28.9 samples/s +Validation at step 39250: Loss = 2.9304, Perplexity = 18.73 +Step 39260/44000 | Loss: 2.9372 | LR: 1.47e-05 | 1257.6ms/step | 15.9 samples/s +Step 39270/44000 | Loss: 2.9468 | LR: 1.46e-05 | 694.7ms/step | 28.8 samples/s +Step 39280/44000 | Loss: 2.9932 | LR: 1.46e-05 | 691.8ms/step | 28.9 samples/s +Step 39290/44000 | Loss: 2.9462 | LR: 1.45e-05 | 692.2ms/step | 28.9 samples/s +Step 39300/44000 | Loss: 2.9207 | LR: 1.44e-05 | 690.8ms/step | 29.0 samples/s +Step 39310/44000 | Loss: 2.9401 | LR: 1.44e-05 | 691.8ms/step | 28.9 samples/s +Step 39320/44000 | Loss: 2.8929 | LR: 1.43e-05 | 691.8ms/step | 28.9 samples/s +Step 39330/44000 | Loss: 2.9060 | LR: 1.42e-05 | 692.4ms/step | 28.9 samples/s +Step 39340/44000 | Loss: 2.8872 | LR: 1.42e-05 | 694.1ms/step | 28.8 samples/s +Step 39350/44000 | Loss: 2.9236 | LR: 1.41e-05 | 691.7ms/step | 28.9 samples/s +Step 39360/44000 | Loss: 2.8857 | LR: 1.41e-05 | 692.4ms/step | 28.9 samples/s +Step 39370/44000 | Loss: 2.9271 | LR: 1.40e-05 | 691.4ms/step | 28.9 samples/s +Step 39380/44000 | Loss: 2.9159 | LR: 1.39e-05 | 692.1ms/step | 28.9 samples/s +Step 39390/44000 | Loss: 2.9458 | LR: 1.39e-05 | 691.4ms/step | 28.9 samples/s +Step 39400/44000 | Loss: 2.9305 | LR: 1.38e-05 | 691.0ms/step | 28.9 samples/s +Step 39410/44000 | Loss: 2.9387 | LR: 1.38e-05 | 691.4ms/step | 28.9 samples/s +Step 39420/44000 | Loss: 2.9104 | LR: 1.37e-05 | 692.7ms/step | 28.9 samples/s +Step 39430/44000 | Loss: 2.9104 | LR: 1.36e-05 | 693.2ms/step | 28.9 samples/s +Step 39440/44000 | Loss: 2.9239 | LR: 1.36e-05 | 692.2ms/step | 28.9 samples/s +Step 39450/44000 | Loss: 2.9414 | LR: 1.35e-05 | 692.0ms/step | 28.9 samples/s +Step 39460/44000 | Loss: 2.9343 | LR: 1.35e-05 | 693.8ms/step | 28.8 samples/s +Step 39470/44000 | Loss: 2.9061 | LR: 1.34e-05 | 694.2ms/step | 28.8 samples/s +Step 39480/44000 | Loss: 2.9505 | LR: 1.34e-05 | 694.0ms/step | 28.8 samples/s +Step 39490/44000 | Loss: 2.9291 | LR: 1.33e-05 | 693.6ms/step | 28.8 samples/s +Step 39500/44000 | Loss: 2.9060 | LR: 1.32e-05 | 691.6ms/step | 28.9 samples/s +Validation at step 39500: Loss = 2.9305, Perplexity = 18.74 +Step 39510/44000 | Loss: 2.8732 | LR: 1.32e-05 | 1259.4ms/step | 15.9 samples/s +Step 39520/44000 | Loss: 2.9247 | LR: 1.31e-05 | 693.5ms/step | 28.8 samples/s +Step 39530/44000 | Loss: 2.8997 | LR: 1.31e-05 | 693.2ms/step | 28.9 samples/s +Step 39540/44000 | Loss: 2.9318 | LR: 1.30e-05 | 693.9ms/step | 28.8 samples/s +Step 39550/44000 | Loss: 2.9367 | LR: 1.29e-05 | 692.9ms/step | 28.9 samples/s +Step 39560/44000 | Loss: 2.8967 | LR: 1.29e-05 | 698.0ms/step | 28.7 samples/s +Step 39570/44000 | Loss: 2.9326 | LR: 1.28e-05 | 694.9ms/step | 28.8 samples/s +Step 39580/44000 | Loss: 2.8737 | LR: 1.28e-05 | 694.7ms/step | 28.8 samples/s +Step 39590/44000 | Loss: 2.9172 | LR: 1.27e-05 | 693.9ms/step | 28.8 samples/s +Step 39600/44000 | Loss: 2.9258 | LR: 1.27e-05 | 694.4ms/step | 28.8 samples/s +Step 39610/44000 | Loss: 2.9123 | LR: 1.26e-05 | 694.0ms/step | 28.8 samples/s +Step 39620/44000 | Loss: 2.8783 | LR: 1.25e-05 | 693.9ms/step | 28.8 samples/s +Step 39630/44000 | Loss: 2.9165 | LR: 1.25e-05 | 694.7ms/step | 28.8 samples/s +Step 39640/44000 | Loss: 2.9217 | LR: 1.24e-05 | 692.7ms/step | 28.9 samples/s +Step 39650/44000 | Loss: 2.9098 | LR: 1.24e-05 | 692.4ms/step | 28.9 samples/s +Step 39660/44000 | Loss: 2.9059 | LR: 1.23e-05 | 692.0ms/step | 28.9 samples/s +Step 39670/44000 | Loss: 2.8785 | LR: 1.23e-05 | 693.3ms/step | 28.8 samples/s +Step 39680/44000 | Loss: 2.9542 | LR: 1.22e-05 | 694.3ms/step | 28.8 samples/s +Step 39690/44000 | Loss: 2.9273 | LR: 1.22e-05 | 692.4ms/step | 28.9 samples/s +Step 39700/44000 | Loss: 2.9196 | LR: 1.21e-05 | 691.6ms/step | 28.9 samples/s +Step 39710/44000 | Loss: 2.9021 | LR: 1.20e-05 | 691.1ms/step | 28.9 samples/s +Step 39720/44000 | Loss: 2.9031 | LR: 1.20e-05 | 689.5ms/step | 29.0 samples/s +Step 39730/44000 | Loss: 2.9328 | LR: 1.19e-05 | 691.5ms/step | 28.9 samples/s +Step 39740/44000 | Loss: 2.8767 | LR: 1.19e-05 | 691.1ms/step | 28.9 samples/s +Step 39750/44000 | Loss: 2.9191 | LR: 1.18e-05 | 692.5ms/step | 28.9 samples/s +Validation at step 39750: Loss = 2.9299, Perplexity = 18.73 +Step 39760/44000 | Loss: 2.9229 | LR: 1.18e-05 | 1264.7ms/step | 15.8 samples/s +Step 39770/44000 | Loss: 2.8925 | LR: 1.17e-05 | 691.7ms/step | 28.9 samples/s +Step 39780/44000 | Loss: 2.9271 | LR: 1.17e-05 | 694.2ms/step | 28.8 samples/s +Step 39790/44000 | Loss: 2.9278 | LR: 1.16e-05 | 693.0ms/step | 28.9 samples/s +Step 39800/44000 | Loss: 2.9342 | LR: 1.15e-05 | 691.5ms/step | 28.9 samples/s +Step 39810/44000 | Loss: 2.8961 | LR: 1.15e-05 | 690.4ms/step | 29.0 samples/s +Step 39820/44000 | Loss: 2.9084 | LR: 1.14e-05 | 691.9ms/step | 28.9 samples/s +Step 39830/44000 | Loss: 2.9378 | LR: 1.14e-05 | 696.1ms/step | 28.7 samples/s +Step 39840/44000 | Loss: 2.9552 | LR: 1.13e-05 | 692.4ms/step | 28.9 samples/s +Step 39850/44000 | Loss: 2.8883 | LR: 1.13e-05 | 691.1ms/step | 28.9 samples/s +Step 39860/44000 | Loss: 2.9204 | LR: 1.12e-05 | 691.1ms/step | 28.9 samples/s +Step 39870/44000 | Loss: 2.9025 | LR: 1.12e-05 | 693.0ms/step | 28.9 samples/s +Step 39880/44000 | Loss: 2.8862 | LR: 1.11e-05 | 692.6ms/step | 28.9 samples/s +Step 39890/44000 | Loss: 2.9166 | LR: 1.11e-05 | 691.1ms/step | 28.9 samples/s +Step 39900/44000 | Loss: 2.9354 | LR: 1.10e-05 | 693.3ms/step | 28.8 samples/s +Step 39910/44000 | Loss: 2.9200 | LR: 1.10e-05 | 693.3ms/step | 28.8 samples/s +Step 39920/44000 | Loss: 2.9097 | LR: 1.09e-05 | 692.4ms/step | 28.9 samples/s +Step 39930/44000 | Loss: 2.9176 | LR: 1.08e-05 | 693.2ms/step | 28.9 samples/s +Step 39940/44000 | Loss: 2.9074 | LR: 1.08e-05 | 692.5ms/step | 28.9 samples/s +Step 39950/44000 | Loss: 2.9212 | LR: 1.07e-05 | 692.6ms/step | 28.9 samples/s +Step 39960/44000 | Loss: 2.9172 | LR: 1.07e-05 | 692.8ms/step | 28.9 samples/s +Step 39970/44000 | Loss: 2.9187 | LR: 1.06e-05 | 694.4ms/step | 28.8 samples/s +Step 39980/44000 | Loss: 2.8904 | LR: 1.06e-05 | 691.2ms/step | 28.9 samples/s +Step 39990/44000 | Loss: 2.9195 | LR: 1.05e-05 | 692.5ms/step | 28.9 samples/s +Step 40000/44000 | Loss: 2.9327 | LR: 1.05e-05 | 691.8ms/step | 28.9 samples/s +Validation at step 40000: Loss = 2.9302, Perplexity = 18.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-40000 +Step 40010/44000 | Loss: 2.8970 | LR: 1.04e-05 | 1561.1ms/step | 12.8 samples/s +Step 40020/44000 | Loss: 2.8893 | LR: 1.04e-05 | 693.0ms/step | 28.9 samples/s +Step 40030/44000 | Loss: 2.9213 | LR: 1.03e-05 | 691.9ms/step | 28.9 samples/s +Step 40040/44000 | Loss: 2.9192 | LR: 1.03e-05 | 691.7ms/step | 28.9 samples/s +Step 40050/44000 | Loss: 2.8885 | LR: 1.02e-05 | 692.4ms/step | 28.9 samples/s +Step 40060/44000 | Loss: 2.9032 | LR: 1.02e-05 | 690.9ms/step | 28.9 samples/s +Step 40070/44000 | Loss: 2.9247 | LR: 1.01e-05 | 692.4ms/step | 28.9 samples/s +Step 40080/44000 | Loss: 2.9053 | LR: 1.01e-05 | 692.8ms/step | 28.9 samples/s +Step 40090/44000 | Loss: 2.9070 | LR: 1.00e-05 | 693.4ms/step | 28.8 samples/s +Step 40100/44000 | Loss: 2.9515 | LR: 9.96e-06 | 691.6ms/step | 28.9 samples/s +Step 40110/44000 | Loss: 2.9558 | LR: 9.91e-06 | 692.4ms/step | 28.9 samples/s +Step 40120/44000 | Loss: 2.9075 | LR: 9.86e-06 | 693.2ms/step | 28.9 samples/s +Step 40130/44000 | Loss: 2.8634 | LR: 9.81e-06 | 692.3ms/step | 28.9 samples/s +Step 40140/44000 | Loss: 2.9060 | LR: 9.76e-06 | 690.8ms/step | 29.0 samples/s +Step 40150/44000 | Loss: 2.9496 | LR: 9.71e-06 | 689.8ms/step | 29.0 samples/s +Step 40160/44000 | Loss: 2.9350 | LR: 9.66e-06 | 690.0ms/step | 29.0 samples/s +Step 40170/44000 | Loss: 2.8969 | LR: 9.61e-06 | 692.4ms/step | 28.9 samples/s +Step 40180/44000 | Loss: 2.9511 | LR: 9.56e-06 | 691.1ms/step | 28.9 samples/s +Step 40190/44000 | Loss: 2.8890 | LR: 9.51e-06 | 691.3ms/step | 28.9 samples/s +Step 40200/44000 | Loss: 2.9043 | LR: 9.46e-06 | 690.8ms/step | 29.0 samples/s +Step 40210/44000 | Loss: 2.9234 | LR: 9.41e-06 | 691.8ms/step | 28.9 samples/s +Step 40220/44000 | Loss: 2.9289 | LR: 9.36e-06 | 690.7ms/step | 29.0 samples/s +Step 40230/44000 | Loss: 2.9200 | LR: 9.32e-06 | 692.0ms/step | 28.9 samples/s +Step 40240/44000 | Loss: 2.8848 | LR: 9.27e-06 | 690.2ms/step | 29.0 samples/s +Step 40250/44000 | Loss: 2.9387 | LR: 9.22e-06 | 690.7ms/step | 29.0 samples/s +Validation at step 40250: Loss = 2.9304, Perplexity = 18.73 +Step 40260/44000 | Loss: 2.9115 | LR: 9.17e-06 | 1262.0ms/step | 15.8 samples/s +Step 40270/44000 | Loss: 2.9187 | LR: 9.12e-06 | 694.0ms/step | 28.8 samples/s +Step 40280/44000 | Loss: 2.8961 | LR: 9.07e-06 | 693.8ms/step | 28.8 samples/s +Step 40290/44000 | Loss: 2.9379 | LR: 9.02e-06 | 691.3ms/step | 28.9 samples/s +Step 40300/44000 | Loss: 2.9465 | LR: 8.97e-06 | 693.4ms/step | 28.8 samples/s +Step 40310/44000 | Loss: 2.9263 | LR: 8.93e-06 | 692.6ms/step | 28.9 samples/s +Step 40320/44000 | Loss: 2.9085 | LR: 8.88e-06 | 693.3ms/step | 28.8 samples/s +Step 40330/44000 | Loss: 2.9391 | LR: 8.83e-06 | 693.1ms/step | 28.9 samples/s +Step 40340/44000 | Loss: 2.9109 | LR: 8.78e-06 | 694.0ms/step | 28.8 samples/s +Step 40350/44000 | Loss: 2.9710 | LR: 8.74e-06 | 693.7ms/step | 28.8 samples/s +Step 40360/44000 | Loss: 2.8866 | LR: 8.69e-06 | 694.1ms/step | 28.8 samples/s +Step 40370/44000 | Loss: 2.9182 | LR: 8.64e-06 | 693.0ms/step | 28.9 samples/s +Step 40380/44000 | Loss: 2.8971 | LR: 8.59e-06 | 693.8ms/step | 28.8 samples/s +Step 40390/44000 | Loss: 2.9002 | LR: 8.55e-06 | 694.0ms/step | 28.8 samples/s +Step 40400/44000 | Loss: 2.9291 | LR: 8.50e-06 | 692.9ms/step | 28.9 samples/s +Step 40410/44000 | Loss: 2.9015 | LR: 8.45e-06 | 694.3ms/step | 28.8 samples/s +Step 40420/44000 | Loss: 2.9460 | LR: 8.41e-06 | 692.1ms/step | 28.9 samples/s +Step 40430/44000 | Loss: 2.8739 | LR: 8.36e-06 | 694.3ms/step | 28.8 samples/s +Step 40440/44000 | Loss: 2.9339 | LR: 8.31e-06 | 694.1ms/step | 28.8 samples/s +Step 40450/44000 | Loss: 2.8949 | LR: 8.27e-06 | 693.3ms/step | 28.8 samples/s +Step 40460/44000 | Loss: 2.9470 | LR: 8.22e-06 | 692.8ms/step | 28.9 samples/s +Step 40470/44000 | Loss: 2.9073 | LR: 8.17e-06 | 692.9ms/step | 28.9 samples/s +Step 40480/44000 | Loss: 2.9329 | LR: 8.13e-06 | 693.1ms/step | 28.9 samples/s +Step 40490/44000 | Loss: 2.9034 | LR: 8.08e-06 | 694.4ms/step | 28.8 samples/s +Step 40500/44000 | Loss: 2.9368 | LR: 8.04e-06 | 693.9ms/step | 28.8 samples/s +Validation at step 40500: Loss = 2.9304, Perplexity = 18.74 +Step 40510/44000 | Loss: 2.9027 | LR: 7.99e-06 | 1263.2ms/step | 15.8 samples/s +Step 40520/44000 | Loss: 2.9156 | LR: 7.94e-06 | 694.8ms/step | 28.8 samples/s +Step 40530/44000 | Loss: 2.8873 | LR: 7.90e-06 | 693.3ms/step | 28.8 samples/s +Step 40540/44000 | Loss: 2.9197 | LR: 7.85e-06 | 693.4ms/step | 28.8 samples/s +Step 40550/44000 | Loss: 2.8898 | LR: 7.81e-06 | 694.8ms/step | 28.8 samples/s +Step 40560/44000 | Loss: 2.9203 | LR: 7.76e-06 | 692.7ms/step | 28.9 samples/s +Step 40570/44000 | Loss: 2.9188 | LR: 7.72e-06 | 693.2ms/step | 28.9 samples/s +Step 40580/44000 | Loss: 2.9301 | LR: 7.67e-06 | 694.6ms/step | 28.8 samples/s +Step 40590/44000 | Loss: 2.9252 | LR: 7.63e-06 | 693.3ms/step | 28.8 samples/s +Step 40600/44000 | Loss: 2.9213 | LR: 7.59e-06 | 692.6ms/step | 28.9 samples/s +Step 40610/44000 | Loss: 2.9577 | LR: 7.54e-06 | 693.6ms/step | 28.8 samples/s +Step 40620/44000 | Loss: 2.9208 | LR: 7.50e-06 | 694.5ms/step | 28.8 samples/s +Step 40630/44000 | Loss: 2.9059 | LR: 7.45e-06 | 692.7ms/step | 28.9 samples/s +Step 40640/44000 | Loss: 2.8937 | LR: 7.41e-06 | 693.2ms/step | 28.9 samples/s +Step 40650/44000 | Loss: 2.8806 | LR: 7.37e-06 | 694.4ms/step | 28.8 samples/s +Step 40660/44000 | Loss: 2.9200 | LR: 7.32e-06 | 694.3ms/step | 28.8 samples/s +Step 40670/44000 | Loss: 2.8787 | LR: 7.28e-06 | 694.9ms/step | 28.8 samples/s +Step 40680/44000 | Loss: 2.9301 | LR: 7.23e-06 | 693.0ms/step | 28.9 samples/s +Step 40690/44000 | Loss: 2.9207 | LR: 7.19e-06 | 693.1ms/step | 28.9 samples/s +Step 40700/44000 | Loss: 2.8751 | LR: 7.15e-06 | 695.5ms/step | 28.8 samples/s +Step 40710/44000 | Loss: 2.9441 | LR: 7.10e-06 | 693.3ms/step | 28.8 samples/s +Step 40720/44000 | Loss: 2.9117 | LR: 7.06e-06 | 694.0ms/step | 28.8 samples/s +Step 40730/44000 | Loss: 2.9288 | LR: 7.02e-06 | 693.3ms/step | 28.8 samples/s +Step 40740/44000 | Loss: 2.9362 | LR: 6.98e-06 | 694.0ms/step | 28.8 samples/s +Step 40750/44000 | Loss: 2.9165 | LR: 6.93e-06 | 693.9ms/step | 28.8 samples/s +Validation at step 40750: Loss = 2.9302, Perplexity = 18.73 +Step 40760/44000 | Loss: 2.8764 | LR: 6.89e-06 | 1262.2ms/step | 15.8 samples/s +Step 40770/44000 | Loss: 2.9368 | LR: 6.85e-06 | 692.8ms/step | 28.9 samples/s +Step 40780/44000 | Loss: 2.9063 | LR: 6.81e-06 | 695.2ms/step | 28.8 samples/s +Step 40790/44000 | Loss: 2.9301 | LR: 6.77e-06 | 692.5ms/step | 28.9 samples/s +Step 40800/44000 | Loss: 2.9434 | LR: 6.72e-06 | 693.0ms/step | 28.9 samples/s +Step 40810/44000 | Loss: 2.9509 | LR: 6.68e-06 | 694.4ms/step | 28.8 samples/s +Step 40820/44000 | Loss: 2.9337 | LR: 6.64e-06 | 694.0ms/step | 28.8 samples/s +Step 40830/44000 | Loss: 2.8930 | LR: 6.60e-06 | 693.1ms/step | 28.9 samples/s +Step 40840/44000 | Loss: 2.9090 | LR: 6.56e-06 | 694.7ms/step | 28.8 samples/s +Step 40850/44000 | Loss: 2.9344 | LR: 6.52e-06 | 693.2ms/step | 28.9 samples/s +Step 40860/44000 | Loss: 2.9113 | LR: 6.47e-06 | 692.6ms/step | 28.9 samples/s +Step 40870/44000 | Loss: 2.9284 | LR: 6.43e-06 | 694.8ms/step | 28.8 samples/s +Step 40880/44000 | Loss: 2.8963 | LR: 6.39e-06 | 693.1ms/step | 28.9 samples/s +Step 40890/44000 | Loss: 2.9399 | LR: 6.35e-06 | 696.5ms/step | 28.7 samples/s +Step 40900/44000 | Loss: 2.9117 | LR: 6.31e-06 | 693.7ms/step | 28.8 samples/s +Step 40910/44000 | Loss: 2.9184 | LR: 6.27e-06 | 694.0ms/step | 28.8 samples/s +Step 40920/44000 | Loss: 2.8959 | LR: 6.23e-06 | 691.9ms/step | 28.9 samples/s +Step 40930/44000 | Loss: 2.9912 | LR: 6.19e-06 | 692.0ms/step | 28.9 samples/s +Step 40940/44000 | Loss: 2.9514 | LR: 6.15e-06 | 693.7ms/step | 28.8 samples/s +Step 40950/44000 | Loss: 2.9242 | LR: 6.11e-06 | 693.1ms/step | 28.9 samples/s +Step 40960/44000 | Loss: 2.9308 | LR: 6.07e-06 | 693.8ms/step | 28.8 samples/s +Step 40970/44000 | Loss: 2.9307 | LR: 6.03e-06 | 693.7ms/step | 28.8 samples/s +Step 40980/44000 | Loss: 2.9402 | LR: 5.99e-06 | 693.1ms/step | 28.9 samples/s +Step 40990/44000 | Loss: 2.8868 | LR: 5.95e-06 | 692.3ms/step | 28.9 samples/s +Step 41000/44000 | Loss: 2.9362 | LR: 5.91e-06 | 697.2ms/step | 28.7 samples/s +Validation at step 41000: Loss = 2.9304, Perplexity = 18.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-41000 +Step 41010/44000 | Loss: 2.9124 | LR: 5.87e-06 | 1574.7ms/step | 12.7 samples/s +Step 41020/44000 | Loss: 2.9523 | LR: 5.83e-06 | 691.7ms/step | 28.9 samples/s +Step 41030/44000 | Loss: 2.9315 | LR: 5.80e-06 | 691.6ms/step | 28.9 samples/s +Step 41040/44000 | Loss: 2.9149 | LR: 5.76e-06 | 695.3ms/step | 28.8 samples/s +Step 41050/44000 | Loss: 2.8997 | LR: 5.72e-06 | 692.8ms/step | 28.9 samples/s +Step 41060/44000 | Loss: 2.8590 | LR: 5.68e-06 | 694.2ms/step | 28.8 samples/s +Step 41070/44000 | Loss: 2.8940 | LR: 5.64e-06 | 694.6ms/step | 28.8 samples/s +Step 41080/44000 | Loss: 2.9239 | LR: 5.60e-06 | 693.6ms/step | 28.8 samples/s +Step 41090/44000 | Loss: 2.9002 | LR: 5.56e-06 | 692.9ms/step | 28.9 samples/s +Step 41100/44000 | Loss: 2.9444 | LR: 5.53e-06 | 692.5ms/step | 28.9 samples/s +Step 41110/44000 | Loss: 2.8898 | LR: 5.49e-06 | 693.4ms/step | 28.8 samples/s +Step 41120/44000 | Loss: 2.8512 | LR: 5.45e-06 | 693.2ms/step | 28.9 samples/s +Step 41130/44000 | Loss: 2.8792 | LR: 5.41e-06 | 693.8ms/step | 28.8 samples/s +Step 41140/44000 | Loss: 2.9144 | LR: 5.38e-06 | 693.1ms/step | 28.9 samples/s +Step 41150/44000 | Loss: 2.9087 | LR: 5.34e-06 | 691.1ms/step | 28.9 samples/s +Step 41160/44000 | Loss: 2.9663 | LR: 5.30e-06 | 691.9ms/step | 28.9 samples/s +Step 41170/44000 | Loss: 2.9114 | LR: 5.26e-06 | 694.7ms/step | 28.8 samples/s +Step 41180/44000 | Loss: 2.8849 | LR: 5.23e-06 | 692.0ms/step | 28.9 samples/s +Step 41190/44000 | Loss: 2.8737 | LR: 5.19e-06 | 694.0ms/step | 28.8 samples/s +Step 41200/44000 | Loss: 2.9400 | LR: 5.15e-06 | 693.4ms/step | 28.8 samples/s +Step 41210/44000 | Loss: 2.8558 | LR: 5.12e-06 | 692.9ms/step | 28.9 samples/s +Step 41220/44000 | Loss: 2.8917 | LR: 5.08e-06 | 692.3ms/step | 28.9 samples/s +Step 41230/44000 | Loss: 2.9416 | LR: 5.04e-06 | 693.1ms/step | 28.9 samples/s +Step 41240/44000 | Loss: 2.9068 | LR: 5.01e-06 | 693.0ms/step | 28.9 samples/s +Step 41250/44000 | Loss: 2.9602 | LR: 4.97e-06 | 693.5ms/step | 28.8 samples/s +Validation at step 41250: Loss = 2.9305, Perplexity = 18.74 +Step 41260/44000 | Loss: 2.8909 | LR: 4.94e-06 | 1258.9ms/step | 15.9 samples/s +Step 41270/44000 | Loss: 2.9592 | LR: 4.90e-06 | 692.3ms/step | 28.9 samples/s +Step 41280/44000 | Loss: 2.8949 | LR: 4.86e-06 | 693.4ms/step | 28.8 samples/s +Step 41290/44000 | Loss: 2.9132 | LR: 4.83e-06 | 691.4ms/step | 28.9 samples/s +Step 41300/44000 | Loss: 2.9696 | LR: 4.79e-06 | 692.9ms/step | 28.9 samples/s +Step 41310/44000 | Loss: 2.8845 | LR: 4.76e-06 | 692.1ms/step | 28.9 samples/s +Step 41320/44000 | Loss: 2.8994 | LR: 4.72e-06 | 693.1ms/step | 28.9 samples/s +Step 41330/44000 | Loss: 2.9125 | LR: 4.69e-06 | 693.7ms/step | 28.8 samples/s +Step 41340/44000 | Loss: 2.9480 | LR: 4.65e-06 | 694.2ms/step | 28.8 samples/s +Step 41350/44000 | Loss: 2.9362 | LR: 4.62e-06 | 694.0ms/step | 28.8 samples/s +Step 41360/44000 | Loss: 2.9022 | LR: 4.58e-06 | 693.8ms/step | 28.8 samples/s +Step 41370/44000 | Loss: 2.9275 | LR: 4.55e-06 | 693.3ms/step | 28.8 samples/s +Step 41380/44000 | Loss: 2.9254 | LR: 4.51e-06 | 692.8ms/step | 28.9 samples/s +Step 41390/44000 | Loss: 2.9192 | LR: 4.48e-06 | 693.9ms/step | 28.8 samples/s +Step 41400/44000 | Loss: 2.9232 | LR: 4.45e-06 | 692.3ms/step | 28.9 samples/s +Step 41410/44000 | Loss: 2.9078 | LR: 4.41e-06 | 691.7ms/step | 28.9 samples/s +Step 41420/44000 | Loss: 2.9453 | LR: 4.38e-06 | 692.2ms/step | 28.9 samples/s +Step 41430/44000 | Loss: 2.9212 | LR: 4.34e-06 | 692.7ms/step | 28.9 samples/s +Step 41440/44000 | Loss: 2.8686 | LR: 4.31e-06 | 692.8ms/step | 28.9 samples/s +Step 41450/44000 | Loss: 2.8889 | LR: 4.28e-06 | 692.9ms/step | 28.9 samples/s +Step 41460/44000 | Loss: 2.9006 | LR: 4.24e-06 | 694.0ms/step | 28.8 samples/s +Step 41470/44000 | Loss: 2.9071 | LR: 4.21e-06 | 695.2ms/step | 28.8 samples/s +Step 41480/44000 | Loss: 2.8983 | LR: 4.18e-06 | 692.7ms/step | 28.9 samples/s +Step 41490/44000 | Loss: 2.8988 | LR: 4.14e-06 | 691.5ms/step | 28.9 samples/s +Step 41500/44000 | Loss: 2.8934 | LR: 4.11e-06 | 692.2ms/step | 28.9 samples/s +Validation at step 41500: Loss = 2.9305, Perplexity = 18.74 +Step 41510/44000 | Loss: 2.9353 | LR: 4.08e-06 | 1261.0ms/step | 15.9 samples/s +Step 41520/44000 | Loss: 2.9416 | LR: 4.05e-06 | 693.6ms/step | 28.8 samples/s +Step 41530/44000 | Loss: 2.9039 | LR: 4.01e-06 | 694.2ms/step | 28.8 samples/s +Step 41540/44000 | Loss: 2.9318 | LR: 3.98e-06 | 693.7ms/step | 28.8 samples/s +Step 41550/44000 | Loss: 2.9086 | LR: 3.95e-06 | 693.4ms/step | 28.8 samples/s +Step 41560/44000 | Loss: 2.8942 | LR: 3.92e-06 | 693.8ms/step | 28.8 samples/s +Step 41570/44000 | Loss: 2.9355 | LR: 3.88e-06 | 693.5ms/step | 28.8 samples/s +Step 41580/44000 | Loss: 2.9763 | LR: 3.85e-06 | 694.3ms/step | 28.8 samples/s +Step 41590/44000 | Loss: 2.8721 | LR: 3.82e-06 | 694.7ms/step | 28.8 samples/s +Step 41600/44000 | Loss: 2.9001 | LR: 3.79e-06 | 693.4ms/step | 28.8 samples/s +Step 41610/44000 | Loss: 2.9131 | LR: 3.76e-06 | 693.2ms/step | 28.9 samples/s +Step 41620/44000 | Loss: 2.8869 | LR: 3.73e-06 | 693.7ms/step | 28.8 samples/s +Step 41630/44000 | Loss: 2.9041 | LR: 3.70e-06 | 692.6ms/step | 28.9 samples/s +Step 41640/44000 | Loss: 2.8988 | LR: 3.66e-06 | 691.5ms/step | 28.9 samples/s +Step 41650/44000 | Loss: 2.8517 | LR: 3.63e-06 | 693.1ms/step | 28.9 samples/s +Step 41660/44000 | Loss: 2.8926 | LR: 3.60e-06 | 692.9ms/step | 28.9 samples/s +Step 41670/44000 | Loss: 2.9453 | LR: 3.57e-06 | 693.1ms/step | 28.9 samples/s +Step 41680/44000 | Loss: 2.9250 | LR: 3.54e-06 | 692.4ms/step | 28.9 samples/s +Step 41690/44000 | Loss: 2.9407 | LR: 3.51e-06 | 692.7ms/step | 28.9 samples/s +Step 41700/44000 | Loss: 2.9086 | LR: 3.48e-06 | 691.8ms/step | 28.9 samples/s +Step 41710/44000 | Loss: 2.9496 | LR: 3.45e-06 | 692.6ms/step | 28.9 samples/s +Step 41720/44000 | Loss: 2.9174 | LR: 3.42e-06 | 693.3ms/step | 28.8 samples/s +Step 41730/44000 | Loss: 2.9726 | LR: 3.39e-06 | 694.3ms/step | 28.8 samples/s +Step 41740/44000 | Loss: 2.9212 | LR: 3.36e-06 | 692.8ms/step | 28.9 samples/s +Step 41750/44000 | Loss: 2.8720 | LR: 3.33e-06 | 693.0ms/step | 28.9 samples/s +Validation at step 41750: Loss = 2.9307, Perplexity = 18.74 +Step 41760/44000 | Loss: 2.8964 | LR: 3.30e-06 | 1261.2ms/step | 15.9 samples/s +Step 41770/44000 | Loss: 2.9315 | LR: 3.27e-06 | 692.1ms/step | 28.9 samples/s +Step 41780/44000 | Loss: 2.8906 | LR: 3.24e-06 | 692.6ms/step | 28.9 samples/s +Step 41790/44000 | Loss: 2.9317 | LR: 3.21e-06 | 692.0ms/step | 28.9 samples/s +Step 41800/44000 | Loss: 2.8829 | LR: 3.19e-06 | 692.8ms/step | 28.9 samples/s +Step 41810/44000 | Loss: 2.9525 | LR: 3.16e-06 | 692.4ms/step | 28.9 samples/s +Step 41820/44000 | Loss: 2.9222 | LR: 3.13e-06 | 692.5ms/step | 28.9 samples/s +Step 41830/44000 | Loss: 2.8991 | LR: 3.10e-06 | 691.8ms/step | 28.9 samples/s +Step 41840/44000 | Loss: 2.9268 | LR: 3.07e-06 | 692.9ms/step | 28.9 samples/s +Step 41850/44000 | Loss: 2.9069 | LR: 3.04e-06 | 693.9ms/step | 28.8 samples/s +Step 41860/44000 | Loss: 2.9105 | LR: 3.01e-06 | 692.3ms/step | 28.9 samples/s +Step 41870/44000 | Loss: 2.9101 | LR: 2.99e-06 | 691.8ms/step | 28.9 samples/s +Step 41880/44000 | Loss: 2.8983 | LR: 2.96e-06 | 693.9ms/step | 28.8 samples/s +Step 41890/44000 | Loss: 2.9173 | LR: 2.93e-06 | 693.2ms/step | 28.9 samples/s +Step 41900/44000 | Loss: 2.9383 | LR: 2.90e-06 | 692.7ms/step | 28.9 samples/s +Step 41910/44000 | Loss: 2.9375 | LR: 2.88e-06 | 693.9ms/step | 28.8 samples/s +Step 41920/44000 | Loss: 2.9017 | LR: 2.85e-06 | 691.4ms/step | 28.9 samples/s +Step 41930/44000 | Loss: 2.9085 | LR: 2.82e-06 | 694.6ms/step | 28.8 samples/s +Step 41940/44000 | Loss: 2.9300 | LR: 2.79e-06 | 692.4ms/step | 28.9 samples/s +Step 41950/44000 | Loss: 2.9544 | LR: 2.77e-06 | 692.4ms/step | 28.9 samples/s +Step 41960/44000 | Loss: 2.9204 | LR: 2.74e-06 | 693.1ms/step | 28.9 samples/s +Step 41970/44000 | Loss: 2.9120 | LR: 2.71e-06 | 691.5ms/step | 28.9 samples/s +Step 41980/44000 | Loss: 2.9228 | LR: 2.69e-06 | 691.1ms/step | 28.9 samples/s +Step 41990/44000 | Loss: 2.9238 | LR: 2.66e-06 | 694.9ms/step | 28.8 samples/s +Step 42000/44000 | Loss: 2.8846 | LR: 2.63e-06 | 693.3ms/step | 28.8 samples/s +Validation at step 42000: Loss = 2.9311, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-42000 +Step 42010/44000 | Loss: 2.9150 | LR: 2.61e-06 | 1573.0ms/step | 12.7 samples/s +Step 42020/44000 | Loss: 2.9212 | LR: 2.58e-06 | 692.4ms/step | 28.9 samples/s +Step 42030/44000 | Loss: 2.9403 | LR: 2.56e-06 | 695.1ms/step | 28.8 samples/s +Step 42040/44000 | Loss: 2.9609 | LR: 2.53e-06 | 694.4ms/step | 28.8 samples/s +Step 42050/44000 | Loss: 2.8980 | LR: 2.50e-06 | 693.9ms/step | 28.8 samples/s +Step 42060/44000 | Loss: 2.9384 | LR: 2.48e-06 | 694.5ms/step | 28.8 samples/s +Step 42070/44000 | Loss: 2.9870 | LR: 2.45e-06 | 694.7ms/step | 28.8 samples/s +Step 42080/44000 | Loss: 2.9284 | LR: 2.43e-06 | 693.6ms/step | 28.8 samples/s +Step 42090/44000 | Loss: 2.8780 | LR: 2.40e-06 | 693.0ms/step | 28.9 samples/s +Step 42100/44000 | Loss: 2.9113 | LR: 2.38e-06 | 693.9ms/step | 28.8 samples/s +Step 42110/44000 | Loss: 2.9401 | LR: 2.35e-06 | 693.6ms/step | 28.8 samples/s +Step 42120/44000 | Loss: 2.8931 | LR: 2.33e-06 | 694.7ms/step | 28.8 samples/s +Step 42130/44000 | Loss: 2.9750 | LR: 2.30e-06 | 693.3ms/step | 28.8 samples/s +Step 42140/44000 | Loss: 2.9244 | LR: 2.28e-06 | 691.9ms/step | 28.9 samples/s +Step 42150/44000 | Loss: 2.8897 | LR: 2.25e-06 | 693.6ms/step | 28.8 samples/s +Step 42160/44000 | Loss: 2.9412 | LR: 2.23e-06 | 692.7ms/step | 28.9 samples/s +Step 42170/44000 | Loss: 2.9252 | LR: 2.21e-06 | 694.3ms/step | 28.8 samples/s +Step 42180/44000 | Loss: 2.9183 | LR: 2.18e-06 | 692.4ms/step | 28.9 samples/s +Step 42190/44000 | Loss: 2.8858 | LR: 2.16e-06 | 694.9ms/step | 28.8 samples/s +Step 42200/44000 | Loss: 2.9299 | LR: 2.13e-06 | 693.9ms/step | 28.8 samples/s +Step 42210/44000 | Loss: 2.9217 | LR: 2.11e-06 | 691.2ms/step | 28.9 samples/s +Step 42220/44000 | Loss: 2.9071 | LR: 2.09e-06 | 692.4ms/step | 28.9 samples/s +Step 42230/44000 | Loss: 2.9125 | LR: 2.06e-06 | 692.3ms/step | 28.9 samples/s +Step 42240/44000 | Loss: 2.9072 | LR: 2.04e-06 | 694.6ms/step | 28.8 samples/s +Step 42250/44000 | Loss: 2.9313 | LR: 2.02e-06 | 691.2ms/step | 28.9 samples/s +Validation at step 42250: Loss = 2.9301, Perplexity = 18.73 +Step 42260/44000 | Loss: 2.9095 | LR: 1.99e-06 | 1260.2ms/step | 15.9 samples/s +Step 42270/44000 | Loss: 2.9516 | LR: 1.97e-06 | 693.4ms/step | 28.8 samples/s +Step 42280/44000 | Loss: 2.9032 | LR: 1.95e-06 | 692.6ms/step | 28.9 samples/s +Step 42290/44000 | Loss: 2.9212 | LR: 1.93e-06 | 691.3ms/step | 28.9 samples/s +Step 42300/44000 | Loss: 2.9262 | LR: 1.90e-06 | 693.4ms/step | 28.8 samples/s +Step 42310/44000 | Loss: 2.9450 | LR: 1.88e-06 | 692.1ms/step | 28.9 samples/s +Step 42320/44000 | Loss: 2.8977 | LR: 1.86e-06 | 693.6ms/step | 28.8 samples/s +Step 42330/44000 | Loss: 2.9656 | LR: 1.84e-06 | 693.6ms/step | 28.8 samples/s +Step 42340/44000 | Loss: 2.9106 | LR: 1.82e-06 | 692.7ms/step | 28.9 samples/s +Step 42350/44000 | Loss: 2.9029 | LR: 1.79e-06 | 694.5ms/step | 28.8 samples/s +Step 42360/44000 | Loss: 2.9063 | LR: 1.77e-06 | 694.6ms/step | 28.8 samples/s +Step 42370/44000 | Loss: 2.9305 | LR: 1.75e-06 | 693.0ms/step | 28.9 samples/s +Step 42380/44000 | Loss: 2.9024 | LR: 1.73e-06 | 693.6ms/step | 28.8 samples/s +Step 42390/44000 | Loss: 2.9427 | LR: 1.71e-06 | 695.6ms/step | 28.8 samples/s +Step 42400/44000 | Loss: 2.8894 | LR: 1.69e-06 | 693.6ms/step | 28.8 samples/s +Step 42410/44000 | Loss: 2.9124 | LR: 1.67e-06 | 692.9ms/step | 28.9 samples/s +Step 42420/44000 | Loss: 2.9273 | LR: 1.64e-06 | 692.5ms/step | 28.9 samples/s +Step 42430/44000 | Loss: 2.9491 | LR: 1.62e-06 | 694.6ms/step | 28.8 samples/s +Step 42440/44000 | Loss: 2.9058 | LR: 1.60e-06 | 692.4ms/step | 28.9 samples/s +Step 42450/44000 | Loss: 2.9143 | LR: 1.58e-06 | 694.7ms/step | 28.8 samples/s +Step 42460/44000 | Loss: 2.9275 | LR: 1.56e-06 | 694.9ms/step | 28.8 samples/s +Step 42470/44000 | Loss: 2.9458 | LR: 1.54e-06 | 692.9ms/step | 28.9 samples/s +Step 42480/44000 | Loss: 2.9243 | LR: 1.52e-06 | 692.0ms/step | 28.9 samples/s +Step 42490/44000 | Loss: 2.9221 | LR: 1.50e-06 | 692.9ms/step | 28.9 samples/s +Step 42500/44000 | Loss: 2.8735 | LR: 1.48e-06 | 692.7ms/step | 28.9 samples/s +Validation at step 42500: Loss = 2.9304, Perplexity = 18.73 +Step 42510/44000 | Loss: 2.9055 | LR: 1.46e-06 | 1260.4ms/step | 15.9 samples/s +Step 42520/44000 | Loss: 2.8934 | LR: 1.44e-06 | 692.6ms/step | 28.9 samples/s +Step 42530/44000 | Loss: 2.9148 | LR: 1.42e-06 | 692.3ms/step | 28.9 samples/s +Step 42540/44000 | Loss: 2.8991 | LR: 1.40e-06 | 692.7ms/step | 28.9 samples/s +Step 42550/44000 | Loss: 2.8867 | LR: 1.39e-06 | 692.5ms/step | 28.9 samples/s +Step 42560/44000 | Loss: 2.9221 | LR: 1.37e-06 | 693.0ms/step | 28.9 samples/s +Step 42570/44000 | Loss: 2.8791 | LR: 1.35e-06 | 694.7ms/step | 28.8 samples/s +Step 42580/44000 | Loss: 2.9302 | LR: 1.33e-06 | 692.3ms/step | 28.9 samples/s +Step 42590/44000 | Loss: 2.9248 | LR: 1.31e-06 | 692.1ms/step | 28.9 samples/s +Step 42600/44000 | Loss: 2.9179 | LR: 1.29e-06 | 692.3ms/step | 28.9 samples/s +Step 42610/44000 | Loss: 2.9277 | LR: 1.27e-06 | 694.2ms/step | 28.8 samples/s +Step 42620/44000 | Loss: 2.9456 | LR: 1.25e-06 | 692.6ms/step | 28.9 samples/s +Step 42630/44000 | Loss: 2.9106 | LR: 1.24e-06 | 695.5ms/step | 28.8 samples/s +Step 42640/44000 | Loss: 2.9250 | LR: 1.22e-06 | 693.0ms/step | 28.9 samples/s +Step 42650/44000 | Loss: 2.8815 | LR: 1.20e-06 | 693.6ms/step | 28.8 samples/s +Step 42660/44000 | Loss: 2.9524 | LR: 1.18e-06 | 694.5ms/step | 28.8 samples/s +Step 42670/44000 | Loss: 2.9909 | LR: 1.17e-06 | 693.4ms/step | 28.8 samples/s +Step 42680/44000 | Loss: 2.9327 | LR: 1.15e-06 | 693.2ms/step | 28.9 samples/s +Step 42690/44000 | Loss: 2.9193 | LR: 1.13e-06 | 692.3ms/step | 28.9 samples/s +Step 42700/44000 | Loss: 2.8682 | LR: 1.11e-06 | 694.2ms/step | 28.8 samples/s +Step 42710/44000 | Loss: 2.9285 | LR: 1.10e-06 | 692.3ms/step | 28.9 samples/s +Step 42720/44000 | Loss: 2.9339 | LR: 1.08e-06 | 693.9ms/step | 28.8 samples/s +Step 42730/44000 | Loss: 2.9258 | LR: 1.06e-06 | 691.8ms/step | 28.9 samples/s +Step 42740/44000 | Loss: 2.9372 | LR: 1.05e-06 | 692.0ms/step | 28.9 samples/s +Step 42750/44000 | Loss: 2.9067 | LR: 1.03e-06 | 692.9ms/step | 28.9 samples/s +Validation at step 42750: Loss = 2.9305, Perplexity = 18.74 +Step 42760/44000 | Loss: 2.9141 | LR: 1.01e-06 | 1262.0ms/step | 15.8 samples/s +Step 42770/44000 | Loss: 2.9670 | LR: 9.97e-07 | 693.7ms/step | 28.8 samples/s +Step 42780/44000 | Loss: 2.9292 | LR: 9.81e-07 | 692.6ms/step | 28.9 samples/s +Step 42790/44000 | Loss: 2.8978 | LR: 9.65e-07 | 692.9ms/step | 28.9 samples/s +Step 42800/44000 | Loss: 2.8996 | LR: 9.49e-07 | 695.9ms/step | 28.7 samples/s +Step 42810/44000 | Loss: 2.9096 | LR: 9.33e-07 | 694.9ms/step | 28.8 samples/s +Step 42820/44000 | Loss: 2.9000 | LR: 9.18e-07 | 692.8ms/step | 28.9 samples/s +Step 42830/44000 | Loss: 2.9113 | LR: 9.02e-07 | 692.7ms/step | 28.9 samples/s +Step 42840/44000 | Loss: 2.9315 | LR: 8.87e-07 | 692.8ms/step | 28.9 samples/s +Step 42850/44000 | Loss: 2.8893 | LR: 8.72e-07 | 695.1ms/step | 28.8 samples/s +Step 42860/44000 | Loss: 2.8998 | LR: 8.57e-07 | 692.0ms/step | 28.9 samples/s +Step 42870/44000 | Loss: 2.9552 | LR: 8.42e-07 | 693.2ms/step | 28.9 samples/s +Step 42880/44000 | Loss: 2.9012 | LR: 8.27e-07 | 695.3ms/step | 28.8 samples/s +Step 42890/44000 | Loss: 2.9635 | LR: 8.12e-07 | 693.6ms/step | 28.8 samples/s +Step 42900/44000 | Loss: 2.8915 | LR: 7.98e-07 | 692.4ms/step | 28.9 samples/s +Step 42910/44000 | Loss: 2.9181 | LR: 7.83e-07 | 697.6ms/step | 28.7 samples/s +Step 42920/44000 | Loss: 2.9019 | LR: 7.69e-07 | 693.8ms/step | 28.8 samples/s +Step 42930/44000 | Loss: 2.9041 | LR: 7.55e-07 | 693.5ms/step | 28.8 samples/s +Step 42940/44000 | Loss: 2.9555 | LR: 7.41e-07 | 693.3ms/step | 28.8 samples/s +Step 42950/44000 | Loss: 2.8997 | LR: 7.27e-07 | 691.7ms/step | 28.9 samples/s +Step 42960/44000 | Loss: 2.9050 | LR: 7.13e-07 | 693.9ms/step | 28.8 samples/s +Step 42970/44000 | Loss: 2.9017 | LR: 6.99e-07 | 693.1ms/step | 28.9 samples/s +Step 42980/44000 | Loss: 2.9389 | LR: 6.86e-07 | 695.2ms/step | 28.8 samples/s +Step 42990/44000 | Loss: 2.8915 | LR: 6.72e-07 | 694.3ms/step | 28.8 samples/s +Step 43000/44000 | Loss: 2.9339 | LR: 6.59e-07 | 694.5ms/step | 28.8 samples/s +Validation at step 43000: Loss = 2.9305, Perplexity = 18.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-43000 +Step 43010/44000 | Loss: 2.9049 | LR: 6.46e-07 | 1564.0ms/step | 12.8 samples/s +Step 43020/44000 | Loss: 2.9315 | LR: 6.33e-07 | 695.7ms/step | 28.7 samples/s +Step 43030/44000 | Loss: 2.9148 | LR: 6.20e-07 | 694.2ms/step | 28.8 samples/s +Step 43040/44000 | Loss: 2.9546 | LR: 6.08e-07 | 695.2ms/step | 28.8 samples/s +Step 43050/44000 | Loss: 2.9178 | LR: 5.95e-07 | 693.3ms/step | 28.8 samples/s +Step 43060/44000 | Loss: 2.9037 | LR: 5.83e-07 | 692.2ms/step | 28.9 samples/s +Step 43070/44000 | Loss: 2.9182 | LR: 5.70e-07 | 692.0ms/step | 28.9 samples/s +Step 43080/44000 | Loss: 2.9520 | LR: 5.58e-07 | 692.3ms/step | 28.9 samples/s +Step 43090/44000 | Loss: 2.8786 | LR: 5.46e-07 | 693.2ms/step | 28.9 samples/s +Step 43100/44000 | Loss: 2.8624 | LR: 5.34e-07 | 692.8ms/step | 28.9 samples/s +Step 43110/44000 | Loss: 2.9145 | LR: 5.22e-07 | 692.2ms/step | 28.9 samples/s +Step 43120/44000 | Loss: 2.9305 | LR: 5.11e-07 | 693.0ms/step | 28.9 samples/s +Step 43130/44000 | Loss: 2.9559 | LR: 4.99e-07 | 693.8ms/step | 28.8 samples/s +Step 43140/44000 | Loss: 2.9207 | LR: 4.88e-07 | 692.7ms/step | 28.9 samples/s +Step 43150/44000 | Loss: 2.9621 | LR: 4.76e-07 | 693.0ms/step | 28.9 samples/s +Step 43160/44000 | Loss: 2.9092 | LR: 4.65e-07 | 694.0ms/step | 28.8 samples/s +Step 43170/44000 | Loss: 2.9234 | LR: 4.54e-07 | 692.7ms/step | 28.9 samples/s +Step 43180/44000 | Loss: 2.9161 | LR: 4.43e-07 | 692.7ms/step | 28.9 samples/s +Step 43190/44000 | Loss: 2.8993 | LR: 4.33e-07 | 692.5ms/step | 28.9 samples/s +Step 43200/44000 | Loss: 2.9743 | LR: 4.22e-07 | 694.1ms/step | 28.8 samples/s +Step 43210/44000 | Loss: 2.9452 | LR: 4.12e-07 | 695.3ms/step | 28.8 samples/s +Step 43220/44000 | Loss: 2.9164 | LR: 4.01e-07 | 693.7ms/step | 28.8 samples/s +Step 43230/44000 | Loss: 2.9255 | LR: 3.91e-07 | 695.2ms/step | 28.8 samples/s +Step 43240/44000 | Loss: 2.9644 | LR: 3.81e-07 | 693.0ms/step | 28.9 samples/s +Step 43250/44000 | Loss: 2.9464 | LR: 3.71e-07 | 692.7ms/step | 28.9 samples/s +Validation at step 43250: Loss = 2.9306, Perplexity = 18.74 +Step 43260/44000 | Loss: 2.8807 | LR: 3.61e-07 | 1261.1ms/step | 15.9 samples/s +Step 43270/44000 | Loss: 2.9416 | LR: 3.51e-07 | 694.1ms/step | 28.8 samples/s +Step 43280/44000 | Loss: 2.9340 | LR: 3.42e-07 | 697.0ms/step | 28.7 samples/s +Step 43290/44000 | Loss: 2.9641 | LR: 3.32e-07 | 693.8ms/step | 28.8 samples/s +Step 43300/44000 | Loss: 2.9001 | LR: 3.23e-07 | 693.5ms/step | 28.8 samples/s +Step 43310/44000 | Loss: 2.9418 | LR: 3.14e-07 | 694.3ms/step | 28.8 samples/s +Step 43320/44000 | Loss: 2.9064 | LR: 3.05e-07 | 694.4ms/step | 28.8 samples/s +Step 43330/44000 | Loss: 2.8778 | LR: 2.96e-07 | 696.5ms/step | 28.7 samples/s +Step 43340/44000 | Loss: 2.9135 | LR: 2.87e-07 | 692.9ms/step | 28.9 samples/s +Step 43350/44000 | Loss: 2.9397 | LR: 2.79e-07 | 691.4ms/step | 28.9 samples/s +Step 43360/44000 | Loss: 2.9169 | LR: 2.70e-07 | 690.9ms/step | 28.9 samples/s +Step 43370/44000 | Loss: 2.9190 | LR: 2.62e-07 | 692.7ms/step | 28.9 samples/s +Step 43380/44000 | Loss: 2.9302 | LR: 2.53e-07 | 693.5ms/step | 28.8 samples/s +Step 43390/44000 | Loss: 2.9238 | LR: 2.45e-07 | 693.2ms/step | 28.9 samples/s +Step 43400/44000 | Loss: 2.8974 | LR: 2.37e-07 | 695.6ms/step | 28.8 samples/s +Step 43410/44000 | Loss: 2.9377 | LR: 2.30e-07 | 694.0ms/step | 28.8 samples/s +Step 43420/44000 | Loss: 2.9151 | LR: 2.22e-07 | 694.2ms/step | 28.8 samples/s +Step 43430/44000 | Loss: 2.9180 | LR: 2.14e-07 | 693.3ms/step | 28.8 samples/s +Step 43440/44000 | Loss: 2.9235 | LR: 2.07e-07 | 693.3ms/step | 28.8 samples/s +Step 43450/44000 | Loss: 2.9294 | LR: 1.99e-07 | 694.8ms/step | 28.8 samples/s +Step 43460/44000 | Loss: 2.9090 | LR: 1.92e-07 | 693.6ms/step | 28.8 samples/s +Step 43470/44000 | Loss: 2.9275 | LR: 1.85e-07 | 693.1ms/step | 28.9 samples/s +Step 43480/44000 | Loss: 2.8904 | LR: 1.78e-07 | 692.5ms/step | 28.9 samples/s +Step 43490/44000 | Loss: 2.9159 | LR: 1.72e-07 | 693.1ms/step | 28.9 samples/s +Step 43500/44000 | Loss: 2.8986 | LR: 1.65e-07 | 695.7ms/step | 28.7 samples/s +Validation at step 43500: Loss = 2.9304, Perplexity = 18.73 +Step 43510/44000 | Loss: 2.9174 | LR: 1.58e-07 | 1262.4ms/step | 15.8 samples/s +Step 43520/44000 | Loss: 2.9115 | LR: 1.52e-07 | 693.9ms/step | 28.8 samples/s +Step 43530/44000 | Loss: 2.9184 | LR: 1.46e-07 | 693.9ms/step | 28.8 samples/s +Step 43540/44000 | Loss: 2.9109 | LR: 1.40e-07 | 694.0ms/step | 28.8 samples/s +Step 43550/44000 | Loss: 2.9159 | LR: 1.34e-07 | 693.6ms/step | 28.8 samples/s +Step 43560/44000 | Loss: 2.8923 | LR: 1.28e-07 | 693.2ms/step | 28.9 samples/s +Step 43570/44000 | Loss: 2.8960 | LR: 1.22e-07 | 696.1ms/step | 28.7 samples/s +Step 43580/44000 | Loss: 2.9265 | LR: 1.16e-07 | 692.6ms/step | 28.9 samples/s +Step 43590/44000 | Loss: 2.8556 | LR: 1.11e-07 | 693.6ms/step | 28.8 samples/s +Step 43600/44000 | Loss: 2.8862 | LR: 1.06e-07 | 692.7ms/step | 28.9 samples/s +Step 43610/44000 | Loss: 2.8708 | LR: 1.00e-07 | 692.6ms/step | 28.9 samples/s +Step 43620/44000 | Loss: 2.9046 | LR: 9.52e-08 | 692.4ms/step | 28.9 samples/s +Step 43630/44000 | Loss: 2.9284 | LR: 9.03e-08 | 692.6ms/step | 28.9 samples/s +Step 43640/44000 | Loss: 2.8985 | LR: 8.55e-08 | 694.1ms/step | 28.8 samples/s +Step 43650/44000 | Loss: 2.9418 | LR: 8.08e-08 | 693.6ms/step | 28.8 samples/s +Step 43660/44000 | Loss: 2.9227 | LR: 7.62e-08 | 692.4ms/step | 28.9 samples/s +Step 43670/44000 | Loss: 2.8942 | LR: 7.18e-08 | 692.4ms/step | 28.9 samples/s +Step 43680/44000 | Loss: 2.9552 | LR: 6.75e-08 | 694.4ms/step | 28.8 samples/s +Step 43690/44000 | Loss: 2.9236 | LR: 6.34e-08 | 693.9ms/step | 28.8 samples/s +Step 43700/44000 | Loss: 2.9483 | LR: 5.94e-08 | 693.3ms/step | 28.8 samples/s +Step 43710/44000 | Loss: 2.8989 | LR: 5.55e-08 | 692.0ms/step | 28.9 samples/s +Step 43720/44000 | Loss: 2.8991 | LR: 5.17e-08 | 692.4ms/step | 28.9 samples/s +Step 43730/44000 | Loss: 2.9567 | LR: 4.81e-08 | 692.1ms/step | 28.9 samples/s +Step 43740/44000 | Loss: 2.9632 | LR: 4.46e-08 | 693.4ms/step | 28.8 samples/s +Step 43750/44000 | Loss: 2.8574 | LR: 4.12e-08 | 693.6ms/step | 28.8 samples/s +Validation at step 43750: Loss = 2.9299, Perplexity = 18.72 +Step 43760/44000 | Loss: 2.9197 | LR: 3.80e-08 | 1262.5ms/step | 15.8 samples/s +Step 43770/44000 | Loss: 2.8981 | LR: 3.49e-08 | 692.9ms/step | 28.9 samples/s +Step 43780/44000 | Loss: 2.9229 | LR: 3.19e-08 | 691.6ms/step | 28.9 samples/s +Step 43790/44000 | Loss: 2.8993 | LR: 2.91e-08 | 693.1ms/step | 28.9 samples/s +Step 43800/44000 | Loss: 2.9162 | LR: 2.64e-08 | 693.5ms/step | 28.8 samples/s +Step 43810/44000 | Loss: 2.9013 | LR: 2.38e-08 | 694.2ms/step | 28.8 samples/s +Step 43820/44000 | Loss: 2.8982 | LR: 2.14e-08 | 693.4ms/step | 28.8 samples/s +Step 43830/44000 | Loss: 2.8904 | LR: 1.91e-08 | 691.2ms/step | 28.9 samples/s +Step 43840/44000 | Loss: 2.8888 | LR: 1.69e-08 | 691.3ms/step | 28.9 samples/s +Step 43850/44000 | Loss: 2.8839 | LR: 1.48e-08 | 692.2ms/step | 28.9 samples/s +Step 43860/44000 | Loss: 2.9365 | LR: 1.29e-08 | 691.9ms/step | 28.9 samples/s +Step 43870/44000 | Loss: 2.9290 | LR: 1.11e-08 | 692.1ms/step | 28.9 samples/s +Step 43880/44000 | Loss: 2.9329 | LR: 9.50e-09 | 693.0ms/step | 28.9 samples/s +Step 43890/44000 | Loss: 2.9146 | LR: 7.98e-09 | 694.7ms/step | 28.8 samples/s +Step 43900/44000 | Loss: 2.9368 | LR: 6.60e-09 | 692.9ms/step | 28.9 samples/s +Step 43910/44000 | Loss: 2.9303 | LR: 5.34e-09 | 693.2ms/step | 28.9 samples/s +Step 43920/44000 | Loss: 2.9421 | LR: 4.22e-09 | 691.9ms/step | 28.9 samples/s +Step 43930/44000 | Loss: 2.8926 | LR: 3.23e-09 | 693.3ms/step | 28.8 samples/s +Step 43940/44000 | Loss: 2.8700 | LR: 2.37e-09 | 693.5ms/step | 28.8 samples/s +Step 43950/44000 | Loss: 2.9292 | LR: 1.65e-09 | 694.8ms/step | 28.8 samples/s +Step 43960/44000 | Loss: 2.8754 | LR: 1.06e-09 | 693.7ms/step | 28.8 samples/s +Step 43970/44000 | Loss: 2.9370 | LR: 5.94e-10 | 692.6ms/step | 28.9 samples/s +Step 43980/44000 | Loss: 2.8967 | LR: 2.64e-10 | 694.6ms/step | 28.8 samples/s +Step 43990/44000 | Loss: 2.9229 | LR: 6.60e-11 | 693.3ms/step | 28.8 samples/s +Step 44000/44000 | Loss: 2.8590 | LR: 0.00e+00 | 691.0ms/step | 28.9 samples/s +Validation at step 44000: Loss = 2.9298, Perplexity = 18.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-44000 +wandb: updating run metadata +wandb: uploading wandb-summary.json; uploading config.yaml; uploading output.log +wandb: uploading history steps 4399-4399, summary, console lines 2453-2455 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▇▇▇▆▆▅▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▇▇▆▆▅▅▄▄▄▃▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ▃▂▁▃▃▄▂▄▄▄▅▃▄▅▄▅▆▅▅▅▆▅▅▅▄▄▅▇▆█▇▆▄▅▅▇▂▆▆▅ +wandb: speed/steps_per_sec ████████████████████████████▂██████▁████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.92984 +wandb: eval/perplexity 18.72461 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 691.04841 +wandb: speed/samples_per_sec 28.94153 +wandb: speed/steps_per_sec 1.44708 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 DDP2 365m resampled 8k nrb35 seed46 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8xh3oikj +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_161845-8xh3oikj/logs +Training complete! diff --git a/epfl_final/train_logs/p1_scheduled_h1002_20260822/scheduler.log b/epfl_final/train_logs/p1_scheduled_h1002_20260822/scheduler.log new file mode 100644 index 0000000000000000000000000000000000000000..a0ad21688c7aa4e9838ae43fc8204df94668b75d --- /dev/null +++ b/epfl_final/train_logs/p1_scheduled_h1002_20260822/scheduler.log @@ -0,0 +1,432 @@ +[scheduler] waiting for current H1002 runs to finish 2026-08-22T04:41:01Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:41:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:42:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:43:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:44:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:45:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:46:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:47:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:48:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:49:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:50:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:51:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:52:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:53:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:54:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:55:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:56:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:57:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:58:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T04:59:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:00:02Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:01:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:02:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:03:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:04:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:05:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:06:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:07:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:08:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:09:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:10:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:11:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:12:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:13:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:14:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:15:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:16:03Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:17:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:18:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:19:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:20:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:21:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:22:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:23:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:24:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:25:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:26:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:27:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:28:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:29:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:30:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:31:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:32:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:33:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:34:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:35:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:36:04Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:37:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:38:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:39:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:40:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:41:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:42:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:43:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:44:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:45:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:46:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:47:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:48:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:49:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:50:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:51:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:52:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:53:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:54:05Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:55:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:56:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:57:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:58:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T05:59:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:00:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:01:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:02:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:03:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:04:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:05:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:06:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:07:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:08:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:09:06Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:10:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:11:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:12:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:13:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:14:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:15:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:16:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:17:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:18:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:19:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:20:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:21:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:22:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:23:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:24:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:25:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:26:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:27:07Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:28:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:29:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:30:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:31:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:32:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:33:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:34:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:35:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:36:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:37:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:38:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:39:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:40:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:41:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:42:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:43:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:44:08Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:45:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:46:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:47:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:48:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:49:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:50:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:51:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:52:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:53:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:54:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:55:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:56:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:57:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:58:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T06:59:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:00:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:01:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:02:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:03:09Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:04:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:05:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:06:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:07:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:08:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:09:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:10:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:11:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:12:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:13:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:14:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:15:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:16:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:17:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:18:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:19:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:20:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:21:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:22:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:23:10Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:24:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:25:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:26:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:27:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:28:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:29:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:30:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:31:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:32:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:33:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:34:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:35:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:36:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:37:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:38:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:39:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:40:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:41:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:42:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:43:11Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:44:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:45:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:46:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:47:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:48:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:49:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:50:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:51:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:52:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:53:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:54:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:55:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:56:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:57:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:58:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T07:59:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:00:12Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:01:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:02:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:03:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:04:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:05:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:06:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:07:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:08:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:09:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:10:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:11:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:12:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:13:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:14:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:15:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:16:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:17:13Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:18:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:19:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:20:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:21:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:22:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:23:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:24:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:25:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:26:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:27:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:28:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:29:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:30:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:31:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:32:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:33:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:34:14Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:35:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:36:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:37:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:38:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:39:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:40:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:41:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:42:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:43:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:44:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:45:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:46:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:47:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:48:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:49:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:50:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:51:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:52:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:53:15Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:54:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:55:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:56:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:57:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:58:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T08:59:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:00:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:01:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:02:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:03:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:04:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:05:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:06:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:07:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:08:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:09:16Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:10:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:11:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:12:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:13:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:14:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:15:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:16:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:17:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:18:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:19:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:20:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:21:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:22:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:23:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:24:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:25:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:26:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:27:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:28:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:29:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:30:17Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:31:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:32:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:33:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:34:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:35:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:36:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:37:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:38:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:39:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:40:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:41:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:42:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:43:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:44:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:45:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:46:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:47:18Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:48:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:49:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:50:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:51:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:52:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:53:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:54:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:55:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:56:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:57:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:58:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T09:59:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:00:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:01:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:02:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:03:19Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:04:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:05:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:06:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:07:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:08:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:09:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:10:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:11:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:12:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:13:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:14:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:15:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:16:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:17:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:18:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:19:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:20:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:21:20Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:22:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:23:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:24:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:25:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:26:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:27:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:28:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:29:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:30:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:31:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:32:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:33:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:34:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:35:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:36:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:37:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:38:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:39:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:40:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:41:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:42:21Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:43:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:44:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:45:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:46:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:47:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:48:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:49:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:50:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:51:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:52:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:53:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:54:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:55:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:56:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:57:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:58:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T10:59:22Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:00:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:01:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:02:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:03:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:04:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:05:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:06:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:07:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:08:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:09:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:10:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:11:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:12:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:13:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:14:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:15:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:16:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:17:23Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:18:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:19:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:20:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:21:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:22:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:23:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:24:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:25:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:26:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:27:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:28:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:29:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:30:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:31:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:32:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:33:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:34:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:35:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:36:24Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:37:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:38:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:39:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:40:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:41:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:42:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:43:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:44:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:45:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:46:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:47:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:48:25Z +[scheduler] active=35 busy_gpus=1 2026-08-22T11:49:25Z +[scheduler] active=0 busy_gpus=0 2026-08-22T11:50:25Z +[scheduler] condition met; launching resampled seed46 on both GPUs 2026-08-22T11:50:25Z diff --git a/epfl_final/train_logs/p1_scheduled_h1004_20260822/l8k365m_resampled_nrb35_seed45.log b/epfl_final/train_logs/p1_scheduled_h1004_20260822/l8k365m_resampled_nrb35_seed45.log new file mode 100644 index 0000000000000000000000000000000000000000..4a7d7a8269f15627c8169eb141f01949c96fc16f --- /dev/null +++ b/epfl_final/train_logs/p1_scheduled_h1004_20260822/l8k365m_resampled_nrb35_seed45.log @@ -0,0 +1,1604 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 45, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 DDP4 365m resampled 8k nrb35 seed45", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "resampled", + "nrb35", + "seed45", + "ddp4", + "h1004", + "scheduled" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 8192] -- -- True +├─Qwen3Model: 1-1 -- [1, 8192, 768] -- True +│ └─Embedding: 2-1 [1, 8192] [1, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [1, 8192, 768] [1, 8192, 768] 768 True +├─Linear: 1-2 [1, 8192, 768] [1, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 475.65 +================================================================================================================================================================ +Input size (MB): 0.07 +Forward/backward pass size (MB): 15598.62 +Params size (MB): 951.29 +Estimated Total Size (MB): 16549.97 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 1 +World size (GPUs): 4 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_143443-9nppzrru +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run P1 DDP4 365m resampled 8k nrb35 seed45 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/9nppzrru +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0710 | LR: 6.67e-06 | 6354.8ms/step | 3.1 samples/s +Step 20/44000 | Loss: 11.9805 | LR: 1.33e-05 | 400.5ms/step | 49.9 samples/s +Step 30/44000 | Loss: 11.6747 | LR: 2.00e-05 | 401.1ms/step | 49.9 samples/s +Step 40/44000 | Loss: 11.1453 | LR: 2.67e-05 | 401.9ms/step | 49.8 samples/s +Step 50/44000 | Loss: 10.8314 | LR: 3.33e-05 | 400.5ms/step | 49.9 samples/s +Step 60/44000 | Loss: 10.6068 | LR: 4.00e-05 | 398.3ms/step | 50.2 samples/s +Step 70/44000 | Loss: 10.3505 | LR: 4.67e-05 | 396.0ms/step | 50.5 samples/s +Step 80/44000 | Loss: 10.0361 | LR: 5.33e-05 | 395.2ms/step | 50.6 samples/s +Step 90/44000 | Loss: 9.6634 | LR: 6.00e-05 | 396.0ms/step | 50.5 samples/s +Step 100/44000 | Loss: 9.1964 | LR: 6.67e-05 | 394.9ms/step | 50.6 samples/s +Step 110/44000 | Loss: 8.6348 | LR: 7.33e-05 | 395.5ms/step | 50.6 samples/s +Step 120/44000 | Loss: 8.1610 | LR: 8.00e-05 | 397.5ms/step | 50.3 samples/s +Step 130/44000 | Loss: 7.7494 | LR: 8.67e-05 | 398.2ms/step | 50.2 samples/s +Step 140/44000 | Loss: 7.5241 | LR: 9.33e-05 | 396.4ms/step | 50.5 samples/s +Step 150/44000 | Loss: 7.3958 | LR: 1.00e-04 | 397.2ms/step | 50.4 samples/s +Step 160/44000 | Loss: 7.2202 | LR: 1.07e-04 | 398.5ms/step | 50.2 samples/s +Step 170/44000 | Loss: 7.1075 | LR: 1.13e-04 | 397.4ms/step | 50.3 samples/s +Step 180/44000 | Loss: 6.9946 | LR: 1.20e-04 | 398.3ms/step | 50.2 samples/s +Step 190/44000 | Loss: 6.9041 | LR: 1.27e-04 | 400.6ms/step | 49.9 samples/s +Step 200/44000 | Loss: 6.8149 | LR: 1.33e-04 | 399.6ms/step | 50.0 samples/s +Step 210/44000 | Loss: 6.8088 | LR: 1.40e-04 | 400.5ms/step | 49.9 samples/s +Step 220/44000 | Loss: 6.6739 | LR: 1.47e-04 | 404.6ms/step | 49.4 samples/s +Step 230/44000 | Loss: 6.5946 | LR: 1.53e-04 | 399.2ms/step | 50.1 samples/s +Step 240/44000 | Loss: 6.5667 | LR: 1.60e-04 | 398.5ms/step | 50.2 samples/s +Step 250/44000 | Loss: 6.4902 | LR: 1.67e-04 | 397.8ms/step | 50.3 samples/s +Validation at step 250: Loss = 6.4339, Perplexity = 622.63 +Step 260/44000 | Loss: 6.4013 | LR: 1.73e-04 | 3937.9ms/step | 5.1 samples/s +Step 270/44000 | Loss: 6.3490 | LR: 1.80e-04 | 404.7ms/step | 49.4 samples/s +Step 280/44000 | Loss: 6.2944 | LR: 1.87e-04 | 404.9ms/step | 49.4 samples/s +Step 290/44000 | Loss: 6.3066 | LR: 1.93e-04 | 404.3ms/step | 49.5 samples/s +Step 300/44000 | Loss: 6.2124 | LR: 2.00e-04 | 405.1ms/step | 49.4 samples/s +Step 310/44000 | Loss: 6.1957 | LR: 2.07e-04 | 405.5ms/step | 49.3 samples/s +Step 320/44000 | Loss: 6.1285 | LR: 2.13e-04 | 405.4ms/step | 49.3 samples/s +Step 330/44000 | Loss: 6.1118 | LR: 2.20e-04 | 405.6ms/step | 49.3 samples/s +Step 340/44000 | Loss: 6.0422 | LR: 2.27e-04 | 405.2ms/step | 49.4 samples/s +Step 350/44000 | Loss: 6.0044 | LR: 2.33e-04 | 405.5ms/step | 49.3 samples/s +Step 360/44000 | Loss: 5.9528 | LR: 2.40e-04 | 403.5ms/step | 49.6 samples/s +Step 370/44000 | Loss: 5.9436 | LR: 2.47e-04 | 403.8ms/step | 49.5 samples/s +Step 380/44000 | Loss: 5.9609 | LR: 2.53e-04 | 407.1ms/step | 49.1 samples/s +Step 390/44000 | Loss: 5.9482 | LR: 2.60e-04 | 403.5ms/step | 49.6 samples/s +Step 400/44000 | Loss: 5.7951 | LR: 2.67e-04 | 404.0ms/step | 49.5 samples/s +Step 410/44000 | Loss: 5.7916 | LR: 2.73e-04 | 406.1ms/step | 49.3 samples/s +Step 420/44000 | Loss: 5.7557 | LR: 2.80e-04 | 405.2ms/step | 49.4 samples/s +Step 430/44000 | Loss: 5.7174 | LR: 2.87e-04 | 404.8ms/step | 49.4 samples/s +Step 440/44000 | Loss: 5.7217 | LR: 2.93e-04 | 402.9ms/step | 49.6 samples/s +Step 450/44000 | Loss: 5.7310 | LR: 3.00e-04 | 403.6ms/step | 49.6 samples/s +Step 460/44000 | Loss: 5.6892 | LR: 3.07e-04 | 403.4ms/step | 49.6 samples/s +Step 470/44000 | Loss: 5.6032 | LR: 3.13e-04 | 405.6ms/step | 49.3 samples/s +Step 480/44000 | Loss: 5.5677 | LR: 3.20e-04 | 409.7ms/step | 48.8 samples/s +Step 490/44000 | Loss: 5.5813 | LR: 3.27e-04 | 405.6ms/step | 49.3 samples/s +Step 500/44000 | Loss: 5.5175 | LR: 3.33e-04 | 405.9ms/step | 49.3 samples/s +Validation at step 500: Loss = 5.4656, Perplexity = 236.42 +Step 510/44000 | Loss: 5.4970 | LR: 3.40e-04 | 577.7ms/step | 34.6 samples/s +Step 520/44000 | Loss: 5.3788 | LR: 3.47e-04 | 404.5ms/step | 49.4 samples/s +Step 530/44000 | Loss: 5.4058 | LR: 3.53e-04 | 405.5ms/step | 49.3 samples/s +Step 540/44000 | Loss: 5.3656 | LR: 3.60e-04 | 404.9ms/step | 49.4 samples/s +Step 550/44000 | Loss: 5.3135 | LR: 3.67e-04 | 404.7ms/step | 49.4 samples/s +Step 560/44000 | Loss: 5.3204 | LR: 3.73e-04 | 404.9ms/step | 49.4 samples/s +Step 570/44000 | Loss: 5.2736 | LR: 3.80e-04 | 405.1ms/step | 49.4 samples/s +Step 580/44000 | Loss: 5.2254 | LR: 3.87e-04 | 406.0ms/step | 49.3 samples/s +Step 590/44000 | Loss: 5.1967 | LR: 3.93e-04 | 404.4ms/step | 49.5 samples/s +Step 600/44000 | Loss: 5.1723 | LR: 4.00e-04 | 403.0ms/step | 49.6 samples/s +Step 610/44000 | Loss: 5.1194 | LR: 4.07e-04 | 402.5ms/step | 49.7 samples/s +Step 620/44000 | Loss: 5.0704 | LR: 4.13e-04 | 403.5ms/step | 49.6 samples/s +Step 630/44000 | Loss: 5.1049 | LR: 4.20e-04 | 402.0ms/step | 49.7 samples/s +Step 640/44000 | Loss: 5.0635 | LR: 4.27e-04 | 402.0ms/step | 49.7 samples/s +Step 650/44000 | Loss: 5.0709 | LR: 4.33e-04 | 402.7ms/step | 49.7 samples/s +Step 660/44000 | Loss: 4.9676 | LR: 4.40e-04 | 402.5ms/step | 49.7 samples/s +Step 670/44000 | Loss: 5.0166 | LR: 4.47e-04 | 403.0ms/step | 49.6 samples/s +Step 680/44000 | Loss: 4.9635 | LR: 4.53e-04 | 402.6ms/step | 49.7 samples/s +Step 690/44000 | Loss: 4.8329 | LR: 4.60e-04 | 403.1ms/step | 49.6 samples/s +Step 700/44000 | Loss: 4.9265 | LR: 4.67e-04 | 402.7ms/step | 49.7 samples/s +Step 710/44000 | Loss: 4.8803 | LR: 4.73e-04 | 402.4ms/step | 49.7 samples/s +Step 720/44000 | Loss: 4.8434 | LR: 4.80e-04 | 403.1ms/step | 49.6 samples/s +Step 730/44000 | Loss: 4.7903 | LR: 4.87e-04 | 402.1ms/step | 49.7 samples/s +Step 740/44000 | Loss: 4.7689 | LR: 4.93e-04 | 401.8ms/step | 49.8 samples/s +Step 750/44000 | Loss: 4.7792 | LR: 5.00e-04 | 401.4ms/step | 49.8 samples/s +Validation at step 750: Loss = 4.7434, Perplexity = 114.82 +Step 760/44000 | Loss: 4.7202 | LR: 5.00e-04 | 572.2ms/step | 35.0 samples/s +Step 770/44000 | Loss: 4.6773 | LR: 5.00e-04 | 403.6ms/step | 49.6 samples/s +Step 780/44000 | Loss: 4.7015 | LR: 5.00e-04 | 405.1ms/step | 49.4 samples/s +Step 790/44000 | Loss: 4.6473 | LR: 5.00e-04 | 403.9ms/step | 49.5 samples/s +Step 800/44000 | Loss: 4.6661 | LR: 5.00e-04 | 404.1ms/step | 49.5 samples/s +Step 810/44000 | Loss: 4.5851 | LR: 5.00e-04 | 404.1ms/step | 49.5 samples/s +Step 820/44000 | Loss: 4.6023 | LR: 5.00e-04 | 404.6ms/step | 49.4 samples/s +Step 830/44000 | Loss: 4.6083 | LR: 5.00e-04 | 405.1ms/step | 49.4 samples/s +Step 840/44000 | Loss: 4.5306 | LR: 5.00e-04 | 404.2ms/step | 49.5 samples/s +Step 850/44000 | Loss: 4.5263 | LR: 5.00e-04 | 404.4ms/step | 49.5 samples/s +Step 860/44000 | Loss: 4.5439 | LR: 5.00e-04 | 404.9ms/step | 49.4 samples/s +Step 870/44000 | Loss: 4.5115 | LR: 5.00e-04 | 405.0ms/step | 49.4 samples/s +Step 880/44000 | Loss: 4.4664 | LR: 5.00e-04 | 404.3ms/step | 49.5 samples/s +Step 890/44000 | Loss: 4.4191 | LR: 5.00e-04 | 405.0ms/step | 49.4 samples/s +Step 900/44000 | Loss: 4.4583 | LR: 5.00e-04 | 404.5ms/step | 49.4 samples/s +Step 910/44000 | Loss: 4.4267 | LR: 5.00e-04 | 404.6ms/step | 49.4 samples/s +Step 920/44000 | Loss: 4.4193 | LR: 5.00e-04 | 403.7ms/step | 49.5 samples/s +Step 930/44000 | Loss: 4.4190 | LR: 5.00e-04 | 404.1ms/step | 49.5 samples/s +Step 940/44000 | Loss: 4.3881 | LR: 5.00e-04 | 404.4ms/step | 49.5 samples/s +Step 950/44000 | Loss: 4.3709 | LR: 5.00e-04 | 403.3ms/step | 49.6 samples/s +Step 960/44000 | Loss: 4.3115 | LR: 5.00e-04 | 404.3ms/step | 49.5 samples/s +Step 970/44000 | Loss: 4.2979 | LR: 5.00e-04 | 403.6ms/step | 49.6 samples/s +Step 980/44000 | Loss: 4.3107 | LR: 5.00e-04 | 404.1ms/step | 49.5 samples/s +Step 990/44000 | Loss: 4.3545 | LR: 5.00e-04 | 404.8ms/step | 49.4 samples/s +Step 1000/44000 | Loss: 4.2597 | LR: 5.00e-04 | 403.3ms/step | 49.6 samples/s +Validation at step 1000: Loss = 4.2848, Perplexity = 72.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-1000 +Step 1010/44000 | Loss: 4.2343 | LR: 5.00e-04 | 939.9ms/step | 21.3 samples/s +Step 1020/44000 | Loss: 4.2910 | LR: 5.00e-04 | 402.5ms/step | 49.7 samples/s +Step 1030/44000 | Loss: 4.3569 | LR: 5.00e-04 | 402.5ms/step | 49.7 samples/s +Step 1040/44000 | Loss: 4.2606 | LR: 5.00e-04 | 402.8ms/step | 49.7 samples/s +Step 1050/44000 | Loss: 4.2402 | LR: 5.00e-04 | 402.0ms/step | 49.7 samples/s +Step 1060/44000 | Loss: 4.2433 | LR: 5.00e-04 | 403.6ms/step | 49.6 samples/s +Step 1070/44000 | Loss: 4.1836 | LR: 5.00e-04 | 403.3ms/step | 49.6 samples/s +Step 1080/44000 | Loss: 4.2356 | LR: 5.00e-04 | 402.6ms/step | 49.7 samples/s +Step 1090/44000 | Loss: 4.1922 | LR: 5.00e-04 | 402.2ms/step | 49.7 samples/s +Step 1100/44000 | Loss: 4.1905 | LR: 5.00e-04 | 403.8ms/step | 49.5 samples/s +Step 1110/44000 | Loss: 4.2120 | LR: 5.00e-04 | 403.8ms/step | 49.5 samples/s +Step 1120/44000 | Loss: 4.1832 | LR: 5.00e-04 | 404.7ms/step | 49.4 samples/s +Step 1130/44000 | Loss: 4.1989 | LR: 5.00e-04 | 404.6ms/step | 49.4 samples/s +Step 1140/44000 | Loss: 4.1781 | LR: 5.00e-04 | 404.2ms/step | 49.5 samples/s +Step 1150/44000 | Loss: 4.1194 | LR: 5.00e-04 | 403.9ms/step | 49.5 samples/s +Step 1160/44000 | Loss: 4.1582 | LR: 5.00e-04 | 403.2ms/step | 49.6 samples/s +Step 1170/44000 | Loss: 4.1188 | LR: 5.00e-04 | 403.7ms/step | 49.5 samples/s +Step 1180/44000 | Loss: 4.1245 | LR: 5.00e-04 | 403.4ms/step | 49.6 samples/s +Step 1190/44000 | Loss: 4.1661 | LR: 5.00e-04 | 404.0ms/step | 49.5 samples/s +Step 1200/44000 | Loss: 4.1084 | LR: 5.00e-04 | 404.4ms/step | 49.5 samples/s +Step 1210/44000 | Loss: 4.1023 | LR: 5.00e-04 | 404.3ms/step | 49.5 samples/s +Step 1220/44000 | Loss: 4.0285 | LR: 5.00e-04 | 404.5ms/step | 49.4 samples/s +Step 1230/44000 | Loss: 4.0305 | LR: 5.00e-04 | 403.8ms/step | 49.5 samples/s +Step 1240/44000 | Loss: 4.0849 | LR: 5.00e-04 | 402.5ms/step | 49.7 samples/s +Step 1250/44000 | Loss: 4.0387 | LR: 5.00e-04 | 404.5ms/step | 49.4 samples/s +Validation at step 1250: Loss = 4.0454, Perplexity = 57.13 +Step 1260/44000 | Loss: 4.0748 | LR: 5.00e-04 | 571.6ms/step | 35.0 samples/s +Step 1270/44000 | Loss: 4.0587 | LR: 5.00e-04 | 404.5ms/step | 49.4 samples/s +Step 1280/44000 | Loss: 3.9964 | LR: 5.00e-04 | 404.0ms/step | 49.5 samples/s +Step 1290/44000 | Loss: 4.0025 | LR: 5.00e-04 | 403.8ms/step | 49.5 samples/s +Step 1300/44000 | Loss: 3.9710 | LR: 5.00e-04 | 403.3ms/step | 49.6 samples/s +Step 1310/44000 | Loss: 4.0158 | LR: 5.00e-04 | 403.7ms/step | 49.5 samples/s +Step 1320/44000 | Loss: 4.0167 | LR: 5.00e-04 | 404.2ms/step | 49.5 samples/s +Step 1330/44000 | Loss: 4.0005 | LR: 5.00e-04 | 404.0ms/step | 49.5 samples/s +Step 1340/44000 | Loss: 3.9302 | LR: 5.00e-04 | 404.4ms/step | 49.5 samples/s +Step 1350/44000 | Loss: 4.0107 | LR: 5.00e-04 | 404.1ms/step | 49.5 samples/s +Step 1360/44000 | Loss: 3.9637 | LR: 5.00e-04 | 402.7ms/step | 49.7 samples/s +Step 1370/44000 | Loss: 3.9374 | LR: 5.00e-04 | 404.1ms/step | 49.5 samples/s +Step 1380/44000 | Loss: 4.0080 | LR: 5.00e-04 | 403.9ms/step | 49.5 samples/s +Step 1390/44000 | Loss: 4.0484 | LR: 5.00e-04 | 403.7ms/step | 49.5 samples/s +Step 1400/44000 | Loss: 4.0173 | LR: 5.00e-04 | 403.3ms/step | 49.6 samples/s +Step 1410/44000 | Loss: 3.9962 | LR: 5.00e-04 | 404.1ms/step | 49.5 samples/s +Step 1420/44000 | Loss: 3.9473 | LR: 5.00e-04 | 404.4ms/step | 49.5 samples/s +Step 1430/44000 | Loss: 3.8401 | LR: 5.00e-04 | 404.9ms/step | 49.4 samples/s +Step 1440/44000 | Loss: 4.0185 | LR: 5.00e-04 | 404.4ms/step | 49.5 samples/s +Step 1450/44000 | Loss: 3.9872 | LR: 5.00e-04 | 404.0ms/step | 49.5 samples/s +Step 1460/44000 | Loss: 3.8946 | LR: 5.00e-04 | 404.1ms/step | 49.5 samples/s +Step 1470/44000 | Loss: 3.9897 | LR: 5.00e-04 | 404.6ms/step | 49.4 samples/s +Step 1480/44000 | Loss: 3.8670 | LR: 5.00e-04 | 404.5ms/step | 49.4 samples/s +Step 1490/44000 | Loss: 3.8718 | LR: 5.00e-04 | 404.8ms/step | 49.4 samples/s +Step 1500/44000 | Loss: 3.9307 | LR: 5.00e-04 | 405.4ms/step | 49.3 samples/s +Validation at step 1500: Loss = 3.8984, Perplexity = 49.32 +Step 1510/44000 | Loss: 3.9172 | LR: 5.00e-04 | 573.0ms/step | 34.9 samples/s +Step 1520/44000 | Loss: 3.8866 | LR: 5.00e-04 | 404.2ms/step | 49.5 samples/s +Step 1530/44000 | Loss: 3.9005 | LR: 5.00e-04 | 405.0ms/step | 49.4 samples/s +Step 1540/44000 | Loss: 3.9186 | LR: 5.00e-04 | 404.9ms/step | 49.4 samples/s +Step 1550/44000 | Loss: 3.9119 | LR: 5.00e-04 | 404.4ms/step | 49.5 samples/s +Step 1560/44000 | Loss: 3.8676 | LR: 5.00e-04 | 404.2ms/step | 49.5 samples/s +Step 1570/44000 | Loss: 3.8712 | LR: 5.00e-04 | 404.4ms/step | 49.5 samples/s +Step 1580/44000 | Loss: 3.8563 | LR: 5.00e-04 | 404.8ms/step | 49.4 samples/s +Step 1590/44000 | Loss: 3.8646 | LR: 5.00e-04 | 404.1ms/step | 49.5 samples/s +Step 1600/44000 | Loss: 3.8825 | LR: 5.00e-04 | 405.1ms/step | 49.4 samples/s +Step 1610/44000 | Loss: 3.8365 | LR: 5.00e-04 | 404.9ms/step | 49.4 samples/s +Step 1620/44000 | Loss: 3.8684 | LR: 5.00e-04 | 405.1ms/step | 49.4 samples/s +Step 1630/44000 | Loss: 3.7862 | LR: 4.99e-04 | 406.5ms/step | 49.2 samples/s +Step 1640/44000 | Loss: 3.8756 | LR: 4.99e-04 | 403.9ms/step | 49.5 samples/s +Step 1650/44000 | Loss: 3.7885 | LR: 4.99e-04 | 404.0ms/step | 49.5 samples/s +Step 1660/44000 | Loss: 3.8816 | LR: 4.99e-04 | 403.6ms/step | 49.5 samples/s +Step 1670/44000 | Loss: 3.8757 | LR: 4.99e-04 | 403.3ms/step | 49.6 samples/s +Step 1680/44000 | Loss: 3.8584 | LR: 4.99e-04 | 404.2ms/step | 49.5 samples/s +Step 1690/44000 | Loss: 3.8096 | LR: 4.99e-04 | 403.3ms/step | 49.6 samples/s +Step 1700/44000 | Loss: 3.8114 | LR: 4.99e-04 | 404.2ms/step | 49.5 samples/s +Step 1710/44000 | Loss: 3.8522 | LR: 4.99e-04 | 403.8ms/step | 49.5 samples/s +Step 1720/44000 | Loss: 3.8310 | LR: 4.99e-04 | 403.9ms/step | 49.5 samples/s +Step 1730/44000 | Loss: 3.8188 | LR: 4.99e-04 | 404.1ms/step | 49.5 samples/s +Step 1740/44000 | Loss: 3.7490 | LR: 4.99e-04 | 403.7ms/step | 49.5 samples/s +Step 1750/44000 | Loss: 3.7638 | LR: 4.99e-04 | 404.1ms/step | 49.5 samples/s +Validation at step 1750: Loss = 3.7886, Perplexity = 44.19 +Step 1760/44000 | Loss: 3.7832 | LR: 4.99e-04 | 572.4ms/step | 34.9 samples/s +Step 1770/44000 | Loss: 3.8018 | LR: 4.99e-04 | 404.0ms/step | 49.5 samples/s +Step 1780/44000 | Loss: 3.7902 | LR: 4.99e-04 | 403.7ms/step | 49.5 samples/s +Step 1790/44000 | Loss: 3.8482 | LR: 4.99e-04 | 404.0ms/step | 49.5 samples/s +Step 1800/44000 | Loss: 3.7319 | LR: 4.99e-04 | 403.4ms/step | 49.6 samples/s +Step 1810/44000 | Loss: 3.7703 | LR: 4.99e-04 | 403.9ms/step | 49.5 samples/s +Step 1820/44000 | Loss: 3.7499 | LR: 4.99e-04 | 406.5ms/step | 49.2 samples/s +Step 1830/44000 | Loss: 3.7969 | LR: 4.99e-04 | 404.5ms/step | 49.4 samples/s +Step 1840/44000 | Loss: 3.7834 | LR: 4.99e-04 | 403.4ms/step | 49.6 samples/s +Step 1850/44000 | Loss: 3.7463 | LR: 4.99e-04 | 403.3ms/step | 49.6 samples/s +Step 1860/44000 | Loss: 3.7504 | LR: 4.99e-04 | 403.7ms/step | 49.5 samples/s +Step 1870/44000 | Loss: 3.7469 | LR: 4.99e-04 | 402.6ms/step | 49.7 samples/s +Step 1880/44000 | Loss: 3.7423 | LR: 4.99e-04 | 400.7ms/step | 49.9 samples/s +Step 1890/44000 | Loss: 3.7011 | LR: 4.99e-04 | 400.1ms/step | 50.0 samples/s +Step 1900/44000 | Loss: 3.7761 | LR: 4.99e-04 | 400.6ms/step | 49.9 samples/s +Step 1910/44000 | Loss: 3.7208 | LR: 4.99e-04 | 401.1ms/step | 49.9 samples/s +Step 1920/44000 | Loss: 3.7022 | LR: 4.99e-04 | 400.7ms/step | 49.9 samples/s +Step 1930/44000 | Loss: 3.7384 | LR: 4.99e-04 | 401.4ms/step | 49.8 samples/s +Step 1940/44000 | Loss: 3.7219 | LR: 4.99e-04 | 400.4ms/step | 50.0 samples/s +Step 1950/44000 | Loss: 3.7307 | LR: 4.99e-04 | 401.3ms/step | 49.8 samples/s +Step 1960/44000 | Loss: 3.7341 | LR: 4.99e-04 | 402.8ms/step | 49.7 samples/s +Step 1970/44000 | Loss: 3.7875 | LR: 4.99e-04 | 403.0ms/step | 49.6 samples/s +Step 1980/44000 | Loss: 3.7018 | LR: 4.99e-04 | 403.0ms/step | 49.6 samples/s +Step 1990/44000 | Loss: 3.7182 | LR: 4.99e-04 | 403.5ms/step | 49.6 samples/s +Step 2000/44000 | Loss: 3.6812 | LR: 4.99e-04 | 403.4ms/step | 49.6 samples/s +Validation at step 2000: Loss = 3.7073, Perplexity = 40.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-2000 +Step 2010/44000 | Loss: 3.7391 | LR: 4.99e-04 | 903.7ms/step | 22.1 samples/s +Step 2020/44000 | Loss: 3.7457 | LR: 4.99e-04 | 401.2ms/step | 49.9 samples/s +Step 2030/44000 | Loss: 3.7011 | LR: 4.99e-04 | 401.4ms/step | 49.8 samples/s +Step 2040/44000 | Loss: 3.7481 | LR: 4.99e-04 | 401.3ms/step | 49.8 samples/s +Step 2050/44000 | Loss: 3.7152 | LR: 4.99e-04 | 401.2ms/step | 49.9 samples/s +Step 2060/44000 | Loss: 3.7317 | LR: 4.99e-04 | 400.4ms/step | 50.0 samples/s +Step 2070/44000 | Loss: 3.6431 | LR: 4.99e-04 | 399.7ms/step | 50.0 samples/s +Step 2080/44000 | Loss: 3.6824 | LR: 4.99e-04 | 400.2ms/step | 50.0 samples/s +Step 2090/44000 | Loss: 3.6782 | LR: 4.99e-04 | 399.8ms/step | 50.0 samples/s +Step 2100/44000 | Loss: 3.6926 | LR: 4.99e-04 | 401.7ms/step | 49.8 samples/s +Step 2110/44000 | Loss: 3.7210 | LR: 4.99e-04 | 400.7ms/step | 49.9 samples/s +Step 2120/44000 | Loss: 3.6805 | LR: 4.99e-04 | 402.6ms/step | 49.7 samples/s +Step 2130/44000 | Loss: 3.6481 | LR: 4.99e-04 | 402.6ms/step | 49.7 samples/s +Step 2140/44000 | Loss: 3.7280 | LR: 4.99e-04 | 402.6ms/step | 49.7 samples/s +Step 2150/44000 | Loss: 3.6590 | LR: 4.99e-04 | 403.0ms/step | 49.6 samples/s +Step 2160/44000 | Loss: 3.7120 | LR: 4.99e-04 | 403.0ms/step | 49.6 samples/s +Step 2170/44000 | Loss: 3.7093 | LR: 4.99e-04 | 403.2ms/step | 49.6 samples/s +Step 2180/44000 | Loss: 3.6733 | LR: 4.99e-04 | 402.8ms/step | 49.6 samples/s +Step 2190/44000 | Loss: 3.6184 | LR: 4.99e-04 | 403.9ms/step | 49.5 samples/s +Step 2200/44000 | Loss: 3.6867 | LR: 4.99e-04 | 402.3ms/step | 49.7 samples/s +Step 2210/44000 | Loss: 3.6366 | LR: 4.99e-04 | 403.9ms/step | 49.5 samples/s +Step 2220/44000 | Loss: 3.6731 | LR: 4.99e-04 | 402.3ms/step | 49.7 samples/s +Step 2230/44000 | Loss: 3.6703 | LR: 4.99e-04 | 404.5ms/step | 49.4 samples/s +Step 2240/44000 | Loss: 3.6087 | LR: 4.99e-04 | 402.3ms/step | 49.7 samples/s +Step 2250/44000 | Loss: 3.7262 | LR: 4.99e-04 | 401.4ms/step | 49.8 samples/s +Validation at step 2250: Loss = 3.6456, Perplexity = 38.30 +Step 2260/44000 | Loss: 3.6073 | LR: 4.98e-04 | 571.4ms/step | 35.0 samples/s +Step 2270/44000 | Loss: 3.6777 | LR: 4.98e-04 | 402.7ms/step | 49.7 samples/s +Step 2280/44000 | Loss: 3.6624 | LR: 4.98e-04 | 403.4ms/step | 49.6 samples/s +Step 2290/44000 | Loss: 3.5779 | LR: 4.98e-04 | 404.0ms/step | 49.5 samples/s +Step 2300/44000 | Loss: 3.5962 | LR: 4.98e-04 | 402.6ms/step | 49.7 samples/s +Step 2310/44000 | Loss: 3.6356 | LR: 4.98e-04 | 402.8ms/step | 49.7 samples/s +Step 2320/44000 | Loss: 3.5951 | LR: 4.98e-04 | 403.0ms/step | 49.6 samples/s +Step 2330/44000 | Loss: 3.6184 | LR: 4.98e-04 | 403.3ms/step | 49.6 samples/s +Step 2340/44000 | Loss: 3.6320 | LR: 4.98e-04 | 403.0ms/step | 49.6 samples/s +Step 2350/44000 | Loss: 3.6077 | LR: 4.98e-04 | 402.1ms/step | 49.7 samples/s +Step 2360/44000 | Loss: 3.6493 | LR: 4.98e-04 | 402.5ms/step | 49.7 samples/s +Step 2370/44000 | Loss: 3.6023 | LR: 4.98e-04 | 402.6ms/step | 49.7 samples/s +Step 2380/44000 | Loss: 3.6565 | LR: 4.98e-04 | 403.4ms/step | 49.6 samples/s +Step 2390/44000 | Loss: 3.6218 | LR: 4.98e-04 | 403.1ms/step | 49.6 samples/s +Step 2400/44000 | Loss: 3.6254 | LR: 4.98e-04 | 403.2ms/step | 49.6 samples/s +Step 2410/44000 | Loss: 3.5720 | LR: 4.98e-04 | 403.1ms/step | 49.6 samples/s +Step 2420/44000 | Loss: 3.5786 | LR: 4.98e-04 | 403.3ms/step | 49.6 samples/s +Step 2430/44000 | Loss: 3.6699 | LR: 4.98e-04 | 403.5ms/step | 49.6 samples/s +Step 2440/44000 | Loss: 3.5327 | LR: 4.98e-04 | 403.2ms/step | 49.6 samples/s +Step 2450/44000 | Loss: 3.5936 | LR: 4.98e-04 | 402.8ms/step | 49.6 samples/s +Step 2460/44000 | Loss: 3.6259 | LR: 4.98e-04 | 404.9ms/step | 49.4 samples/s +Step 2470/44000 | Loss: 3.5571 | LR: 4.98e-04 | 404.0ms/step | 49.5 samples/s +Step 2480/44000 | Loss: 3.5722 | LR: 4.98e-04 | 402.5ms/step | 49.7 samples/s +Step 2490/44000 | Loss: 3.5625 | LR: 4.98e-04 | 403.2ms/step | 49.6 samples/s +Step 2500/44000 | Loss: 3.6102 | LR: 4.98e-04 | 403.8ms/step | 49.5 samples/s +Validation at step 2500: Loss = 3.5935, Perplexity = 36.36 +Step 2510/44000 | Loss: 3.6049 | LR: 4.98e-04 | 573.3ms/step | 34.9 samples/s +Step 2520/44000 | Loss: 3.5701 | LR: 4.98e-04 | 403.4ms/step | 49.6 samples/s +Step 2530/44000 | Loss: 3.5375 | LR: 4.98e-04 | 403.8ms/step | 49.5 samples/s +Step 2540/44000 | Loss: 3.6446 | LR: 4.98e-04 | 403.6ms/step | 49.5 samples/s +Step 2550/44000 | Loss: 3.5712 | LR: 4.98e-04 | 403.2ms/step | 49.6 samples/s +Step 2560/44000 | Loss: 3.5838 | LR: 4.98e-04 | 402.7ms/step | 49.7 samples/s +Step 2570/44000 | Loss: 3.5616 | LR: 4.98e-04 | 403.1ms/step | 49.6 samples/s +Step 2580/44000 | Loss: 3.5990 | LR: 4.98e-04 | 403.4ms/step | 49.6 samples/s +Step 2590/44000 | Loss: 3.5846 | LR: 4.98e-04 | 404.3ms/step | 49.5 samples/s +Step 2600/44000 | Loss: 3.5869 | LR: 4.98e-04 | 403.3ms/step | 49.6 samples/s +Step 2610/44000 | Loss: 3.5471 | LR: 4.98e-04 | 403.5ms/step | 49.6 samples/s +Step 2620/44000 | Loss: 3.5624 | LR: 4.98e-04 | 403.0ms/step | 49.6 samples/s +Step 2630/44000 | Loss: 3.6598 | LR: 4.98e-04 | 402.8ms/step | 49.7 samples/s +Step 2640/44000 | Loss: 3.5877 | LR: 4.98e-04 | 402.7ms/step | 49.7 samples/s +Step 2650/44000 | Loss: 3.5699 | LR: 4.98e-04 | 403.7ms/step | 49.5 samples/s +Step 2660/44000 | Loss: 3.5492 | LR: 4.98e-04 | 404.4ms/step | 49.5 samples/s +Step 2670/44000 | Loss: 3.5841 | LR: 4.98e-04 | 403.5ms/step | 49.6 samples/s +Step 2680/44000 | Loss: 3.6024 | LR: 4.98e-04 | 402.7ms/step | 49.7 samples/s +Step 2690/44000 | Loss: 3.5802 | LR: 4.98e-04 | 403.0ms/step | 49.6 samples/s +Step 2700/44000 | Loss: 3.5550 | LR: 4.97e-04 | 403.8ms/step | 49.5 samples/s +Step 2710/44000 | Loss: 3.5012 | LR: 4.97e-04 | 405.1ms/step | 49.4 samples/s +Step 2720/44000 | Loss: 3.5285 | LR: 4.97e-04 | 404.4ms/step | 49.5 samples/s +Step 2730/44000 | Loss: 3.6039 | LR: 4.97e-04 | 405.4ms/step | 49.3 samples/s +Step 2740/44000 | Loss: 3.5409 | LR: 4.97e-04 | 402.8ms/step | 49.7 samples/s +Step 2750/44000 | Loss: 3.5687 | LR: 4.97e-04 | 403.1ms/step | 49.6 samples/s +Validation at step 2750: Loss = 3.5486, Perplexity = 34.77 +Step 2760/44000 | Loss: 3.5729 | LR: 4.97e-04 | 574.6ms/step | 34.8 samples/s +Step 2770/44000 | Loss: 3.5195 | LR: 4.97e-04 | 403.9ms/step | 49.5 samples/s +Step 2780/44000 | Loss: 3.5354 | LR: 4.97e-04 | 404.0ms/step | 49.5 samples/s +Step 2790/44000 | Loss: 3.5201 | LR: 4.97e-04 | 421.4ms/step | 47.5 samples/s +Step 2800/44000 | Loss: 3.4920 | LR: 4.97e-04 | 422.1ms/step | 47.4 samples/s +Step 2810/44000 | Loss: 3.5938 | LR: 4.97e-04 | 423.6ms/step | 47.2 samples/s +Step 2820/44000 | Loss: 3.5057 | LR: 4.97e-04 | 413.0ms/step | 48.4 samples/s +Step 2830/44000 | Loss: 3.5568 | LR: 4.97e-04 | 414.1ms/step | 48.3 samples/s +Step 2840/44000 | Loss: 3.4971 | LR: 4.97e-04 | 415.5ms/step | 48.1 samples/s +Step 2850/44000 | Loss: 3.5120 | LR: 4.97e-04 | 407.5ms/step | 49.1 samples/s +Step 2860/44000 | Loss: 3.5138 | LR: 4.97e-04 | 406.7ms/step | 49.2 samples/s +Step 2870/44000 | Loss: 3.5239 | LR: 4.97e-04 | 412.4ms/step | 48.5 samples/s +Step 2880/44000 | Loss: 3.5204 | LR: 4.97e-04 | 415.6ms/step | 48.1 samples/s +Step 2890/44000 | Loss: 3.5098 | LR: 4.97e-04 | 415.7ms/step | 48.1 samples/s +Step 2900/44000 | Loss: 3.5188 | LR: 4.97e-04 | 415.3ms/step | 48.2 samples/s +Step 2910/44000 | Loss: 3.5592 | LR: 4.97e-04 | 414.9ms/step | 48.2 samples/s +Step 2920/44000 | Loss: 3.5118 | LR: 4.97e-04 | 414.4ms/step | 48.3 samples/s +Step 2930/44000 | Loss: 3.5228 | LR: 4.97e-04 | 414.0ms/step | 48.3 samples/s +Step 2940/44000 | Loss: 3.4667 | LR: 4.97e-04 | 415.9ms/step | 48.1 samples/s +Step 2950/44000 | Loss: 3.5205 | LR: 4.97e-04 | 407.3ms/step | 49.1 samples/s +Step 2960/44000 | Loss: 3.5447 | LR: 4.97e-04 | 406.4ms/step | 49.2 samples/s +Step 2970/44000 | Loss: 3.4986 | LR: 4.97e-04 | 406.2ms/step | 49.2 samples/s +Step 2980/44000 | Loss: 3.5133 | LR: 4.97e-04 | 407.8ms/step | 49.0 samples/s +Step 2990/44000 | Loss: 3.5809 | LR: 4.97e-04 | 409.2ms/step | 48.9 samples/s +Step 3000/44000 | Loss: 3.5040 | LR: 4.97e-04 | 413.3ms/step | 48.4 samples/s +Validation at step 3000: Loss = 3.5100, Perplexity = 33.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-3000 +Step 3010/44000 | Loss: 3.5381 | LR: 4.97e-04 | 921.4ms/step | 21.7 samples/s +Step 3020/44000 | Loss: 3.4782 | LR: 4.97e-04 | 405.9ms/step | 49.3 samples/s +Step 3030/44000 | Loss: 3.4888 | LR: 4.97e-04 | 406.6ms/step | 49.2 samples/s +Step 3040/44000 | Loss: 3.4878 | LR: 4.97e-04 | 406.1ms/step | 49.3 samples/s +Step 3050/44000 | Loss: 3.4712 | LR: 4.97e-04 | 406.5ms/step | 49.2 samples/s +Step 3060/44000 | Loss: 3.4680 | LR: 4.96e-04 | 407.6ms/step | 49.1 samples/s +Step 3070/44000 | Loss: 3.5104 | LR: 4.96e-04 | 406.7ms/step | 49.2 samples/s +Step 3080/44000 | Loss: 3.4494 | LR: 4.96e-04 | 404.7ms/step | 49.4 samples/s +Step 3090/44000 | Loss: 3.5569 | LR: 4.96e-04 | 405.9ms/step | 49.3 samples/s +Step 3100/44000 | Loss: 3.5211 | LR: 4.96e-04 | 405.6ms/step | 49.3 samples/s +Step 3110/44000 | Loss: 3.4753 | LR: 4.96e-04 | 404.8ms/step | 49.4 samples/s +Step 3120/44000 | Loss: 3.4882 | LR: 4.96e-04 | 406.2ms/step | 49.2 samples/s +Step 3130/44000 | Loss: 3.5280 | LR: 4.96e-04 | 405.5ms/step | 49.3 samples/s +Step 3140/44000 | Loss: 3.5422 | LR: 4.96e-04 | 405.2ms/step | 49.4 samples/s +Step 3150/44000 | Loss: 3.4689 | LR: 4.96e-04 | 403.8ms/step | 49.5 samples/s +Step 3160/44000 | Loss: 3.5386 | LR: 4.96e-04 | 403.3ms/step | 49.6 samples/s +Step 3170/44000 | Loss: 3.4370 | LR: 4.96e-04 | 403.6ms/step | 49.6 samples/s +Step 3180/44000 | Loss: 3.4823 | LR: 4.96e-04 | 402.6ms/step | 49.7 samples/s +Step 3190/44000 | Loss: 3.4823 | LR: 4.96e-04 | 402.5ms/step | 49.7 samples/s +Step 3200/44000 | Loss: 3.4894 | LR: 4.96e-04 | 402.1ms/step | 49.7 samples/s +Step 3210/44000 | Loss: 3.4740 | LR: 4.96e-04 | 402.4ms/step | 49.7 samples/s +Step 3220/44000 | Loss: 3.5377 | LR: 4.96e-04 | 402.1ms/step | 49.7 samples/s +Step 3230/44000 | Loss: 3.5016 | LR: 4.96e-04 | 401.3ms/step | 49.8 samples/s +Step 3240/44000 | Loss: 3.5160 | LR: 4.96e-04 | 402.7ms/step | 49.7 samples/s +Step 3250/44000 | Loss: 3.4444 | LR: 4.96e-04 | 402.3ms/step | 49.7 samples/s +Validation at step 3250: Loss = 3.4824, Perplexity = 32.54 +Step 3260/44000 | Loss: 3.4903 | LR: 4.96e-04 | 573.0ms/step | 34.9 samples/s +Step 3270/44000 | Loss: 3.4433 | LR: 4.96e-04 | 401.5ms/step | 49.8 samples/s +Step 3280/44000 | Loss: 3.5025 | LR: 4.96e-04 | 400.2ms/step | 50.0 samples/s +Step 3290/44000 | Loss: 3.4507 | LR: 4.96e-04 | 400.1ms/step | 50.0 samples/s +Step 3300/44000 | Loss: 3.5182 | LR: 4.96e-04 | 399.7ms/step | 50.0 samples/s +Step 3310/44000 | Loss: 3.5296 | LR: 4.96e-04 | 402.5ms/step | 49.7 samples/s +Step 3320/44000 | Loss: 3.4394 | LR: 4.96e-04 | 403.7ms/step | 49.5 samples/s +Step 3330/44000 | Loss: 3.4185 | LR: 4.96e-04 | 403.4ms/step | 49.6 samples/s +Step 3340/44000 | Loss: 3.4875 | LR: 4.96e-04 | 402.9ms/step | 49.6 samples/s +Step 3350/44000 | Loss: 3.5266 | LR: 4.96e-04 | 403.5ms/step | 49.6 samples/s +Step 3360/44000 | Loss: 3.4487 | LR: 4.96e-04 | 402.7ms/step | 49.7 samples/s +Step 3370/44000 | Loss: 3.4709 | LR: 4.95e-04 | 403.8ms/step | 49.5 samples/s +Step 3380/44000 | Loss: 3.4301 | LR: 4.95e-04 | 403.6ms/step | 49.6 samples/s +Step 3390/44000 | Loss: 3.4620 | LR: 4.95e-04 | 403.5ms/step | 49.6 samples/s +Step 3400/44000 | Loss: 3.4870 | LR: 4.95e-04 | 403.3ms/step | 49.6 samples/s +Step 3410/44000 | Loss: 3.4668 | LR: 4.95e-04 | 403.0ms/step | 49.6 samples/s +Step 3420/44000 | Loss: 3.4778 | LR: 4.95e-04 | 403.7ms/step | 49.5 samples/s +Step 3430/44000 | Loss: 3.4810 | LR: 4.95e-04 | 408.7ms/step | 48.9 samples/s +Step 3440/44000 | Loss: 3.4747 | LR: 4.95e-04 | 402.0ms/step | 49.7 samples/s +Step 3450/44000 | Loss: 3.5310 | LR: 4.95e-04 | 399.7ms/step | 50.0 samples/s +Step 3460/44000 | Loss: 3.4579 | LR: 4.95e-04 | 401.6ms/step | 49.8 samples/s +Step 3470/44000 | Loss: 3.4675 | LR: 4.95e-04 | 403.1ms/step | 49.6 samples/s +Step 3480/44000 | Loss: 3.5053 | LR: 4.95e-04 | 403.0ms/step | 49.6 samples/s +Step 3490/44000 | Loss: 3.4232 | LR: 4.95e-04 | 402.3ms/step | 49.7 samples/s +Step 3500/44000 | Loss: 3.4047 | LR: 4.95e-04 | 402.7ms/step | 49.7 samples/s +Validation at step 3500: Loss = 3.4489, Perplexity = 31.47 +Step 3510/44000 | Loss: 3.3928 | LR: 4.95e-04 | 571.4ms/step | 35.0 samples/s +Step 3520/44000 | Loss: 3.4724 | LR: 4.95e-04 | 403.5ms/step | 49.6 samples/s +Step 3530/44000 | Loss: 3.4251 | LR: 4.95e-04 | 402.7ms/step | 49.7 samples/s +Step 3540/44000 | Loss: 3.4006 | LR: 4.95e-04 | 403.8ms/step | 49.5 samples/s +Step 3550/44000 | Loss: 3.4671 | LR: 4.95e-04 | 402.8ms/step | 49.6 samples/s +Step 3560/44000 | Loss: 3.4644 | LR: 4.95e-04 | 402.8ms/step | 49.7 samples/s +Step 3570/44000 | Loss: 3.3605 | LR: 4.95e-04 | 402.2ms/step | 49.7 samples/s +Step 3580/44000 | Loss: 3.4181 | LR: 4.95e-04 | 402.4ms/step | 49.7 samples/s +Step 3590/44000 | Loss: 3.4385 | LR: 4.95e-04 | 401.7ms/step | 49.8 samples/s +Step 3600/44000 | Loss: 3.4636 | LR: 4.95e-04 | 402.4ms/step | 49.7 samples/s +Step 3610/44000 | Loss: 3.4740 | LR: 4.95e-04 | 402.2ms/step | 49.7 samples/s +Step 3620/44000 | Loss: 3.4613 | LR: 4.95e-04 | 402.4ms/step | 49.7 samples/s +Step 3630/44000 | Loss: 3.4295 | LR: 4.95e-04 | 403.0ms/step | 49.6 samples/s +Step 3640/44000 | Loss: 3.4360 | LR: 4.95e-04 | 400.3ms/step | 50.0 samples/s +Step 3650/44000 | Loss: 3.4306 | LR: 4.94e-04 | 400.0ms/step | 50.0 samples/s +Step 3660/44000 | Loss: 3.4670 | LR: 4.94e-04 | 402.0ms/step | 49.7 samples/s +Step 3670/44000 | Loss: 3.5072 | LR: 4.94e-04 | 403.2ms/step | 49.6 samples/s +Step 3680/44000 | Loss: 3.3734 | LR: 4.94e-04 | 404.0ms/step | 49.5 samples/s +Step 3690/44000 | Loss: 3.4255 | LR: 4.94e-04 | 402.1ms/step | 49.7 samples/s +Step 3700/44000 | Loss: 3.4802 | LR: 4.94e-04 | 402.4ms/step | 49.7 samples/s +Step 3710/44000 | Loss: 3.4184 | LR: 4.94e-04 | 402.3ms/step | 49.7 samples/s +Step 3720/44000 | Loss: 3.4831 | LR: 4.94e-04 | 399.8ms/step | 50.0 samples/s +Step 3730/44000 | Loss: 3.4393 | LR: 4.94e-04 | 400.1ms/step | 50.0 samples/s +Step 3740/44000 | Loss: 3.4542 | LR: 4.94e-04 | 399.9ms/step | 50.0 samples/s +Step 3750/44000 | Loss: 3.4932 | LR: 4.94e-04 | 400.4ms/step | 49.9 samples/s +Validation at step 3750: Loss = 3.4242, Perplexity = 30.70 +Step 3760/44000 | Loss: 3.4059 | LR: 4.94e-04 | 573.0ms/step | 34.9 samples/s +Step 3770/44000 | Loss: 3.4631 | LR: 4.94e-04 | 402.9ms/step | 49.6 samples/s +Step 3780/44000 | Loss: 3.3918 | LR: 4.94e-04 | 404.6ms/step | 49.4 samples/s +Step 3790/44000 | Loss: 3.4025 | LR: 4.94e-04 | 403.1ms/step | 49.6 samples/s +Step 3800/44000 | Loss: 3.4107 | LR: 4.94e-04 | 402.8ms/step | 49.7 samples/s +Step 3810/44000 | Loss: 3.4229 | LR: 4.94e-04 | 400.7ms/step | 49.9 samples/s +Step 3820/44000 | Loss: 3.4178 | LR: 4.94e-04 | 400.2ms/step | 50.0 samples/s +Step 3830/44000 | Loss: 3.4840 | LR: 4.94e-04 | 400.4ms/step | 49.9 samples/s +Step 3840/44000 | Loss: 3.4394 | LR: 4.94e-04 | 401.8ms/step | 49.8 samples/s +Step 3850/44000 | Loss: 3.4320 | LR: 4.94e-04 | 399.6ms/step | 50.1 samples/s +Step 3860/44000 | Loss: 3.4742 | LR: 4.94e-04 | 403.8ms/step | 49.5 samples/s +Step 3870/44000 | Loss: 3.4614 | LR: 4.94e-04 | 402.3ms/step | 49.7 samples/s +Step 3880/44000 | Loss: 3.3504 | LR: 4.94e-04 | 403.7ms/step | 49.5 samples/s +Step 3890/44000 | Loss: 3.3812 | LR: 4.94e-04 | 403.5ms/step | 49.6 samples/s +Step 3900/44000 | Loss: 3.4193 | LR: 4.93e-04 | 402.0ms/step | 49.7 samples/s +Step 3910/44000 | Loss: 3.4576 | LR: 4.93e-04 | 400.6ms/step | 49.9 samples/s +Step 3920/44000 | Loss: 3.4052 | LR: 4.93e-04 | 400.3ms/step | 50.0 samples/s +Step 3930/44000 | Loss: 3.4685 | LR: 4.93e-04 | 403.5ms/step | 49.6 samples/s +Step 3940/44000 | Loss: 3.3874 | LR: 4.93e-04 | 400.5ms/step | 49.9 samples/s +Step 3950/44000 | Loss: 3.4032 | LR: 4.93e-04 | 399.8ms/step | 50.0 samples/s +Step 3960/44000 | Loss: 3.4383 | LR: 4.93e-04 | 401.8ms/step | 49.8 samples/s +Step 3970/44000 | Loss: 3.4005 | LR: 4.93e-04 | 402.1ms/step | 49.7 samples/s +Step 3980/44000 | Loss: 3.3891 | LR: 4.93e-04 | 399.8ms/step | 50.0 samples/s +Step 3990/44000 | Loss: 3.4866 | LR: 4.93e-04 | 400.0ms/step | 50.0 samples/s +Step 4000/44000 | Loss: 3.3911 | LR: 4.93e-04 | 400.0ms/step | 50.0 samples/s +Validation at step 4000: Loss = 3.3978, Perplexity = 29.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-4000 +Step 4010/44000 | Loss: 3.3683 | LR: 4.93e-04 | 886.8ms/step | 22.6 samples/s +Step 4020/44000 | Loss: 3.4432 | LR: 4.93e-04 | 401.3ms/step | 49.8 samples/s +Step 4030/44000 | Loss: 3.4854 | LR: 4.93e-04 | 401.5ms/step | 49.8 samples/s +Step 4040/44000 | Loss: 3.4640 | LR: 4.93e-04 | 401.1ms/step | 49.9 samples/s +Step 4050/44000 | Loss: 3.4202 | LR: 4.93e-04 | 401.5ms/step | 49.8 samples/s +Step 4060/44000 | Loss: 3.4462 | LR: 4.93e-04 | 401.0ms/step | 49.9 samples/s +Step 4070/44000 | Loss: 3.4364 | LR: 4.93e-04 | 401.4ms/step | 49.8 samples/s +Step 4080/44000 | Loss: 3.4054 | LR: 4.93e-04 | 400.3ms/step | 50.0 samples/s +Step 4090/44000 | Loss: 3.4126 | LR: 4.93e-04 | 401.4ms/step | 49.8 samples/s +Step 4100/44000 | Loss: 3.3957 | LR: 4.93e-04 | 399.7ms/step | 50.0 samples/s +Step 4110/44000 | Loss: 3.4314 | LR: 4.93e-04 | 400.0ms/step | 50.0 samples/s +Step 4120/44000 | Loss: 3.4442 | LR: 4.93e-04 | 400.1ms/step | 50.0 samples/s +Step 4130/44000 | Loss: 3.4108 | LR: 4.93e-04 | 400.2ms/step | 50.0 samples/s +Step 4140/44000 | Loss: 3.3763 | LR: 4.92e-04 | 399.9ms/step | 50.0 samples/s +Step 4150/44000 | Loss: 3.4047 | LR: 4.92e-04 | 400.0ms/step | 50.0 samples/s +Step 4160/44000 | Loss: 3.4463 | LR: 4.92e-04 | 400.6ms/step | 49.9 samples/s +Step 4170/44000 | Loss: 3.3773 | LR: 4.92e-04 | 399.5ms/step | 50.1 samples/s +Step 4180/44000 | Loss: 3.3719 | LR: 4.92e-04 | 399.7ms/step | 50.0 samples/s +Step 4190/44000 | Loss: 3.4006 | LR: 4.92e-04 | 401.8ms/step | 49.8 samples/s +Step 4200/44000 | Loss: 3.3973 | LR: 4.92e-04 | 402.2ms/step | 49.7 samples/s +Step 4210/44000 | Loss: 3.3378 | LR: 4.92e-04 | 402.2ms/step | 49.7 samples/s +Step 4220/44000 | Loss: 3.3653 | LR: 4.92e-04 | 400.4ms/step | 49.9 samples/s +Step 4230/44000 | Loss: 3.4088 | LR: 4.92e-04 | 399.8ms/step | 50.0 samples/s +Step 4240/44000 | Loss: 3.4050 | LR: 4.92e-04 | 399.9ms/step | 50.0 samples/s +Step 4250/44000 | Loss: 3.3607 | LR: 4.92e-04 | 400.3ms/step | 50.0 samples/s +Validation at step 4250: Loss = 3.3803, Perplexity = 29.38 +Step 4260/44000 | Loss: 3.4155 | LR: 4.92e-04 | 569.6ms/step | 35.1 samples/s +Step 4270/44000 | Loss: 3.4102 | LR: 4.92e-04 | 401.5ms/step | 49.8 samples/s +Step 4280/44000 | Loss: 3.4300 | LR: 4.92e-04 | 401.9ms/step | 49.8 samples/s +Step 4290/44000 | Loss: 3.3865 | LR: 4.92e-04 | 402.6ms/step | 49.7 samples/s +Step 4300/44000 | Loss: 3.3278 | LR: 4.92e-04 | 402.5ms/step | 49.7 samples/s +Step 4310/44000 | Loss: 3.3360 | LR: 4.92e-04 | 402.3ms/step | 49.7 samples/s +Step 4320/44000 | Loss: 3.3829 | LR: 4.92e-04 | 400.7ms/step | 49.9 samples/s +Step 4330/44000 | Loss: 3.3018 | LR: 4.92e-04 | 403.0ms/step | 49.6 samples/s +Step 4340/44000 | Loss: 3.4128 | LR: 4.92e-04 | 404.0ms/step | 49.5 samples/s +Step 4350/44000 | Loss: 3.4124 | LR: 4.92e-04 | 404.8ms/step | 49.4 samples/s +Step 4360/44000 | Loss: 3.3595 | LR: 4.91e-04 | 405.3ms/step | 49.3 samples/s +Step 4370/44000 | Loss: 3.3831 | LR: 4.91e-04 | 406.2ms/step | 49.2 samples/s +Step 4380/44000 | Loss: 3.4075 | LR: 4.91e-04 | 405.2ms/step | 49.4 samples/s +Step 4390/44000 | Loss: 3.4485 | LR: 4.91e-04 | 405.3ms/step | 49.3 samples/s +Step 4400/44000 | Loss: 3.3965 | LR: 4.91e-04 | 404.8ms/step | 49.4 samples/s +Step 4410/44000 | Loss: 3.3758 | LR: 4.91e-04 | 402.6ms/step | 49.7 samples/s +Step 4420/44000 | Loss: 3.3775 | LR: 4.91e-04 | 402.8ms/step | 49.7 samples/s +Step 4430/44000 | Loss: 3.4295 | LR: 4.91e-04 | 404.5ms/step | 49.4 samples/s +Step 4440/44000 | Loss: 3.3380 | LR: 4.91e-04 | 404.4ms/step | 49.5 samples/s +Step 4450/44000 | Loss: 3.3427 | LR: 4.91e-04 | 403.9ms/step | 49.5 samples/s +Step 4460/44000 | Loss: 3.3849 | LR: 4.91e-04 | 404.7ms/step | 49.4 samples/s +Step 4470/44000 | Loss: 3.3500 | LR: 4.91e-04 | 404.3ms/step | 49.5 samples/s +Step 4480/44000 | Loss: 3.3161 | LR: 4.91e-04 | 404.4ms/step | 49.5 samples/s +Step 4490/44000 | Loss: 3.3088 | LR: 4.91e-04 | 404.9ms/step | 49.4 samples/s +Step 4500/44000 | Loss: 3.3690 | LR: 4.91e-04 | 404.2ms/step | 49.5 samples/s +Validation at step 4500: Loss = 3.3614, Perplexity = 28.83 +Step 4510/44000 | Loss: 3.3881 | LR: 4.91e-04 | 571.3ms/step | 35.0 samples/s +Step 4520/44000 | Loss: 3.3841 | LR: 4.91e-04 | 400.7ms/step | 49.9 samples/s +Step 4530/44000 | Loss: 3.3933 | LR: 4.91e-04 | 402.9ms/step | 49.6 samples/s +Step 4540/44000 | Loss: 3.3553 | LR: 4.91e-04 | 401.6ms/step | 49.8 samples/s +Step 4550/44000 | Loss: 3.3786 | LR: 4.91e-04 | 402.1ms/step | 49.7 samples/s +Step 4560/44000 | Loss: 3.3977 | LR: 4.90e-04 | 402.5ms/step | 49.7 samples/s +Step 4570/44000 | Loss: 3.4023 | LR: 4.90e-04 | 402.0ms/step | 49.7 samples/s +Step 4580/44000 | Loss: 3.3517 | LR: 4.90e-04 | 402.6ms/step | 49.7 samples/s +Step 4590/44000 | Loss: 3.3507 | LR: 4.90e-04 | 402.2ms/step | 49.7 samples/s +Step 4600/44000 | Loss: 3.3610 | LR: 4.90e-04 | 403.3ms/step | 49.6 samples/s +Step 4610/44000 | Loss: 3.3767 | LR: 4.90e-04 | 402.1ms/step | 49.7 samples/s +Step 4620/44000 | Loss: 3.3752 | LR: 4.90e-04 | 402.4ms/step | 49.7 samples/s +Step 4630/44000 | Loss: 3.3708 | LR: 4.90e-04 | 402.7ms/step | 49.7 samples/s +Step 4640/44000 | Loss: 3.3776 | LR: 4.90e-04 | 402.1ms/step | 49.7 samples/s +Step 4650/44000 | Loss: 3.3387 | LR: 4.90e-04 | 402.1ms/step | 49.7 samples/s +Step 4660/44000 | Loss: 3.3164 | LR: 4.90e-04 | 403.3ms/step | 49.6 samples/s +Step 4670/44000 | Loss: 3.3761 | LR: 4.90e-04 | 402.0ms/step | 49.8 samples/s +Step 4680/44000 | Loss: 3.3579 | LR: 4.90e-04 | 402.5ms/step | 49.7 samples/s +Step 4690/44000 | Loss: 3.3970 | LR: 4.90e-04 | 401.9ms/step | 49.8 samples/s +Step 4700/44000 | Loss: 3.4225 | LR: 4.90e-04 | 402.3ms/step | 49.7 samples/s +Step 4710/44000 | Loss: 3.3754 | LR: 4.90e-04 | 401.1ms/step | 49.9 samples/s +Step 4720/44000 | Loss: 3.2979 | LR: 4.90e-04 | 402.3ms/step | 49.7 samples/s +Step 4730/44000 | Loss: 3.4189 | LR: 4.90e-04 | 401.3ms/step | 49.8 samples/s +Step 4740/44000 | Loss: 3.3893 | LR: 4.90e-04 | 402.0ms/step | 49.7 samples/s +Step 4750/44000 | Loss: 3.3714 | LR: 4.90e-04 | 401.6ms/step | 49.8 samples/s +Validation at step 4750: Loss = 3.3437, Perplexity = 28.32 +Step 4760/44000 | Loss: 3.4141 | LR: 4.89e-04 | 569.9ms/step | 35.1 samples/s +Step 4770/44000 | Loss: 3.3717 | LR: 4.89e-04 | 402.9ms/step | 49.6 samples/s +Step 4780/44000 | Loss: 3.3963 | LR: 4.89e-04 | 402.8ms/step | 49.7 samples/s +Step 4790/44000 | Loss: 3.3959 | LR: 4.89e-04 | 401.4ms/step | 49.8 samples/s +Step 4800/44000 | Loss: 3.4129 | LR: 4.89e-04 | 401.7ms/step | 49.8 samples/s +Step 4810/44000 | Loss: 3.3397 | LR: 4.89e-04 | 401.6ms/step | 49.8 samples/s +Step 4820/44000 | Loss: 3.4205 | LR: 4.89e-04 | 401.2ms/step | 49.9 samples/s +Step 4830/44000 | Loss: 3.3177 | LR: 4.89e-04 | 402.7ms/step | 49.7 samples/s +Step 4840/44000 | Loss: 3.4046 | LR: 4.89e-04 | 404.1ms/step | 49.5 samples/s +Step 4850/44000 | Loss: 3.3172 | LR: 4.89e-04 | 401.7ms/step | 49.8 samples/s +Step 4860/44000 | Loss: 3.3471 | LR: 4.89e-04 | 402.1ms/step | 49.7 samples/s +Step 4870/44000 | Loss: 3.3367 | LR: 4.89e-04 | 400.2ms/step | 50.0 samples/s +Step 4880/44000 | Loss: 3.3555 | LR: 4.89e-04 | 402.0ms/step | 49.8 samples/s +Step 4890/44000 | Loss: 3.3321 | LR: 4.89e-04 | 403.5ms/step | 49.6 samples/s +Step 4900/44000 | Loss: 3.3465 | LR: 4.89e-04 | 404.6ms/step | 49.4 samples/s +Step 4910/44000 | Loss: 3.3588 | LR: 4.89e-04 | 403.4ms/step | 49.6 samples/s +Step 4920/44000 | Loss: 3.3936 | LR: 4.89e-04 | 401.3ms/step | 49.8 samples/s +Step 4930/44000 | Loss: 3.2983 | LR: 4.89e-04 | 405.8ms/step | 49.3 samples/s +Step 4940/44000 | Loss: 3.3109 | LR: 4.89e-04 | 403.9ms/step | 49.5 samples/s +Step 4950/44000 | Loss: 3.3354 | LR: 4.88e-04 | 404.9ms/step | 49.4 samples/s +Step 4960/44000 | Loss: 3.3539 | LR: 4.88e-04 | 403.6ms/step | 49.6 samples/s +Step 4970/44000 | Loss: 3.3668 | LR: 4.88e-04 | 402.9ms/step | 49.6 samples/s +Step 4980/44000 | Loss: 3.3406 | LR: 4.88e-04 | 399.9ms/step | 50.0 samples/s +Step 4990/44000 | Loss: 3.3685 | LR: 4.88e-04 | 399.9ms/step | 50.0 samples/s +Step 5000/44000 | Loss: 3.2971 | LR: 4.88e-04 | 400.1ms/step | 50.0 samples/s +Validation at step 5000: Loss = 3.3284, Perplexity = 27.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-5000 +Step 5010/44000 | Loss: 3.3185 | LR: 4.88e-04 | 863.8ms/step | 23.2 samples/s +Step 5020/44000 | Loss: 3.3159 | LR: 4.88e-04 | 400.3ms/step | 50.0 samples/s +Step 5030/44000 | Loss: 3.2978 | LR: 4.88e-04 | 400.0ms/step | 50.0 samples/s +Step 5040/44000 | Loss: 3.3962 | LR: 4.88e-04 | 400.1ms/step | 50.0 samples/s +Step 5050/44000 | Loss: 3.3235 | LR: 4.88e-04 | 402.0ms/step | 49.8 samples/s +Step 5060/44000 | Loss: 3.3039 | LR: 4.88e-04 | 403.1ms/step | 49.6 samples/s +Step 5070/44000 | Loss: 3.2869 | LR: 4.88e-04 | 402.1ms/step | 49.7 samples/s +Step 5080/44000 | Loss: 3.3274 | LR: 4.88e-04 | 402.8ms/step | 49.7 samples/s +Step 5090/44000 | Loss: 3.3651 | LR: 4.88e-04 | 401.0ms/step | 49.9 samples/s +Step 5100/44000 | Loss: 3.2900 | LR: 4.88e-04 | 401.7ms/step | 49.8 samples/s +Step 5110/44000 | Loss: 3.2848 | LR: 4.88e-04 | 401.9ms/step | 49.8 samples/s +Step 5120/44000 | Loss: 3.3482 | LR: 4.88e-04 | 401.6ms/step | 49.8 samples/s +Step 5130/44000 | Loss: 3.3370 | LR: 4.87e-04 | 401.9ms/step | 49.8 samples/s +Step 5140/44000 | Loss: 3.3523 | LR: 4.87e-04 | 402.0ms/step | 49.8 samples/s +Step 5150/44000 | Loss: 3.3704 | LR: 4.87e-04 | 400.7ms/step | 49.9 samples/s +Step 5160/44000 | Loss: 3.3460 | LR: 4.87e-04 | 401.3ms/step | 49.8 samples/s +Step 5170/44000 | Loss: 3.3444 | LR: 4.87e-04 | 401.3ms/step | 49.8 samples/s +Step 5180/44000 | Loss: 3.2899 | LR: 4.87e-04 | 400.5ms/step | 49.9 samples/s +Step 5190/44000 | Loss: 3.3813 | LR: 4.87e-04 | 400.0ms/step | 50.0 samples/s +Step 5200/44000 | Loss: 3.3434 | LR: 4.87e-04 | 402.0ms/step | 49.7 samples/s +Step 5210/44000 | Loss: 3.3706 | LR: 4.87e-04 | 401.4ms/step | 49.8 samples/s +Step 5220/44000 | Loss: 3.3502 | LR: 4.87e-04 | 400.7ms/step | 49.9 samples/s +Step 5230/44000 | Loss: 3.3234 | LR: 4.87e-04 | 400.1ms/step | 50.0 samples/s +Step 5240/44000 | Loss: 3.3308 | LR: 4.87e-04 | 401.0ms/step | 49.9 samples/s +Step 5250/44000 | Loss: 3.3120 | LR: 4.87e-04 | 401.2ms/step | 49.9 samples/s +Validation at step 5250: Loss = 3.3181, Perplexity = 27.61 +Step 5260/44000 | Loss: 3.3141 | LR: 4.87e-04 | 568.1ms/step | 35.2 samples/s +Step 5270/44000 | Loss: 3.3422 | LR: 4.87e-04 | 399.6ms/step | 50.1 samples/s +Step 5280/44000 | Loss: 3.2974 | LR: 4.87e-04 | 402.1ms/step | 49.7 samples/s +Step 5290/44000 | Loss: 3.3244 | LR: 4.87e-04 | 400.5ms/step | 49.9 samples/s +Step 5300/44000 | Loss: 3.3659 | LR: 4.86e-04 | 401.8ms/step | 49.8 samples/s +Step 5310/44000 | Loss: 3.3024 | LR: 4.86e-04 | 400.7ms/step | 49.9 samples/s +Step 5320/44000 | Loss: 3.3904 | LR: 4.86e-04 | 401.7ms/step | 49.8 samples/s +Step 5330/44000 | Loss: 3.2933 | LR: 4.86e-04 | 401.6ms/step | 49.8 samples/s +Step 5340/44000 | Loss: 3.3024 | LR: 4.86e-04 | 401.7ms/step | 49.8 samples/s +Step 5350/44000 | Loss: 3.3009 | LR: 4.86e-04 | 401.8ms/step | 49.8 samples/s +Step 5360/44000 | Loss: 3.2184 | LR: 4.86e-04 | 400.3ms/step | 50.0 samples/s +Step 5370/44000 | Loss: 3.3159 | LR: 4.86e-04 | 400.6ms/step | 49.9 samples/s +Step 5380/44000 | Loss: 3.2842 | LR: 4.86e-04 | 401.3ms/step | 49.8 samples/s +Step 5390/44000 | Loss: 3.2892 | LR: 4.86e-04 | 401.1ms/step | 49.9 samples/s +Step 5400/44000 | Loss: 3.3059 | LR: 4.86e-04 | 401.0ms/step | 49.9 samples/s +Step 5410/44000 | Loss: 3.3327 | LR: 4.86e-04 | 400.6ms/step | 49.9 samples/s +Step 5420/44000 | Loss: 3.3498 | LR: 4.86e-04 | 399.9ms/step | 50.0 samples/s +Step 5430/44000 | Loss: 3.3080 | LR: 4.86e-04 | 399.9ms/step | 50.0 samples/s +Step 5440/44000 | Loss: 3.2497 | LR: 4.86e-04 | 399.6ms/step | 50.1 samples/s +Step 5450/44000 | Loss: 3.3254 | LR: 4.86e-04 | 400.4ms/step | 49.9 samples/s +Step 5460/44000 | Loss: 3.2858 | LR: 4.86e-04 | 399.8ms/step | 50.0 samples/s +Step 5470/44000 | Loss: 3.2754 | LR: 4.85e-04 | 399.8ms/step | 50.0 samples/s +Step 5480/44000 | Loss: 3.2906 | LR: 4.85e-04 | 400.1ms/step | 50.0 samples/s +Step 5490/44000 | Loss: 3.3602 | LR: 4.85e-04 | 400.0ms/step | 50.0 samples/s +Step 5500/44000 | Loss: 3.3006 | LR: 4.85e-04 | 400.0ms/step | 50.0 samples/s +Validation at step 5500: Loss = 3.2998, Perplexity = 27.11 +Step 5510/44000 | Loss: 3.2526 | LR: 4.85e-04 | 567.8ms/step | 35.2 samples/s +Step 5520/44000 | Loss: 3.2872 | LR: 4.85e-04 | 400.2ms/step | 50.0 samples/s +Step 5530/44000 | Loss: 3.2938 | LR: 4.85e-04 | 402.0ms/step | 49.8 samples/s +Step 5540/44000 | Loss: 3.3038 | LR: 4.85e-04 | 399.9ms/step | 50.0 samples/s +Step 5550/44000 | Loss: 3.3244 | LR: 4.85e-04 | 402.1ms/step | 49.7 samples/s +Step 5560/44000 | Loss: 3.2354 | LR: 4.85e-04 | 399.9ms/step | 50.0 samples/s +Step 5570/44000 | Loss: 3.2878 | LR: 4.85e-04 | 400.8ms/step | 49.9 samples/s +Step 5580/44000 | Loss: 3.2777 | LR: 4.85e-04 | 401.4ms/step | 49.8 samples/s +Step 5590/44000 | Loss: 3.2812 | LR: 4.85e-04 | 400.2ms/step | 50.0 samples/s +Step 5600/44000 | Loss: 3.3112 | LR: 4.85e-04 | 400.6ms/step | 49.9 samples/s +Step 5610/44000 | Loss: 3.3123 | LR: 4.85e-04 | 401.4ms/step | 49.8 samples/s +Step 5620/44000 | Loss: 3.3801 | LR: 4.85e-04 | 399.9ms/step | 50.0 samples/s +Step 5630/44000 | Loss: 3.3204 | LR: 4.84e-04 | 401.5ms/step | 49.8 samples/s +Step 5640/44000 | Loss: 3.3382 | LR: 4.84e-04 | 401.0ms/step | 49.9 samples/s +Step 5650/44000 | Loss: 3.3095 | LR: 4.84e-04 | 400.9ms/step | 49.9 samples/s +Step 5660/44000 | Loss: 3.2796 | LR: 4.84e-04 | 400.5ms/step | 49.9 samples/s +Step 5670/44000 | Loss: 3.2720 | LR: 4.84e-04 | 400.2ms/step | 50.0 samples/s +Step 5680/44000 | Loss: 3.2778 | LR: 4.84e-04 | 401.5ms/step | 49.8 samples/s +Step 5690/44000 | Loss: 3.3170 | LR: 4.84e-04 | 400.7ms/step | 49.9 samples/s +Step 5700/44000 | Loss: 3.2691 | LR: 4.84e-04 | 401.5ms/step | 49.8 samples/s +Step 5710/44000 | Loss: 3.2514 | LR: 4.84e-04 | 403.4ms/step | 49.6 samples/s +Step 5720/44000 | Loss: 3.2877 | LR: 4.84e-04 | 402.7ms/step | 49.7 samples/s +Step 5730/44000 | Loss: 3.3597 | LR: 4.84e-04 | 404.6ms/step | 49.4 samples/s +Step 5740/44000 | Loss: 3.3089 | LR: 4.84e-04 | 403.0ms/step | 49.6 samples/s +Step 5750/44000 | Loss: 3.2892 | LR: 4.84e-04 | 402.9ms/step | 49.6 samples/s +Validation at step 5750: Loss = 3.2891, Perplexity = 26.82 +Step 5760/44000 | Loss: 3.2442 | LR: 4.84e-04 | 573.3ms/step | 34.9 samples/s +Step 5770/44000 | Loss: 3.2860 | LR: 4.84e-04 | 402.8ms/step | 49.6 samples/s +Step 5780/44000 | Loss: 3.2636 | LR: 4.83e-04 | 402.1ms/step | 49.7 samples/s +Step 5790/44000 | Loss: 3.3441 | LR: 4.83e-04 | 402.2ms/step | 49.7 samples/s +Step 5800/44000 | Loss: 3.3127 | LR: 4.83e-04 | 401.3ms/step | 49.8 samples/s +Step 5810/44000 | Loss: 3.3465 | LR: 4.83e-04 | 402.2ms/step | 49.7 samples/s +Step 5820/44000 | Loss: 3.2580 | LR: 4.83e-04 | 401.2ms/step | 49.8 samples/s +Step 5830/44000 | Loss: 3.2737 | LR: 4.83e-04 | 400.6ms/step | 49.9 samples/s +Step 5840/44000 | Loss: 3.2800 | LR: 4.83e-04 | 401.1ms/step | 49.9 samples/s +Step 5850/44000 | Loss: 3.2983 | LR: 4.83e-04 | 400.2ms/step | 50.0 samples/s +Step 5860/44000 | Loss: 3.3021 | LR: 4.83e-04 | 401.5ms/step | 49.8 samples/s +Step 5870/44000 | Loss: 3.2969 | LR: 4.83e-04 | 401.2ms/step | 49.9 samples/s +Step 5880/44000 | Loss: 3.2761 | LR: 4.83e-04 | 401.3ms/step | 49.8 samples/s +Step 5890/44000 | Loss: 3.2555 | LR: 4.83e-04 | 400.2ms/step | 50.0 samples/s +Step 5900/44000 | Loss: 3.3380 | LR: 4.83e-04 | 400.1ms/step | 50.0 samples/s +Step 5910/44000 | Loss: 3.3054 | LR: 4.83e-04 | 400.1ms/step | 50.0 samples/s +Step 5920/44000 | Loss: 3.3140 | LR: 4.83e-04 | 402.0ms/step | 49.8 samples/s +Step 5930/44000 | Loss: 3.3157 | LR: 4.83e-04 | 401.0ms/step | 49.9 samples/s +Step 5940/44000 | Loss: 3.2606 | LR: 4.82e-04 | 398.8ms/step | 50.1 samples/s +Step 5950/44000 | Loss: 3.3343 | LR: 4.82e-04 | 402.0ms/step | 49.8 samples/s +Step 5960/44000 | Loss: 3.3424 | LR: 4.82e-04 | 400.3ms/step | 50.0 samples/s +Step 5970/44000 | Loss: 3.3349 | LR: 4.82e-04 | 401.3ms/step | 49.8 samples/s +Step 5980/44000 | Loss: 3.3221 | LR: 4.82e-04 | 400.2ms/step | 50.0 samples/s +Step 5990/44000 | Loss: 3.3193 | LR: 4.82e-04 | 400.7ms/step | 49.9 samples/s +Step 6000/44000 | Loss: 3.2964 | LR: 4.82e-04 | 401.3ms/step | 49.8 samples/s +Validation at step 6000: Loss = 3.2737, Perplexity = 26.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-6000 +Step 6010/44000 | Loss: 3.2860 | LR: 4.82e-04 | 925.6ms/step | 21.6 samples/s +Step 6020/44000 | Loss: 3.3137 | LR: 4.82e-04 | 400.0ms/step | 50.0 samples/s +Step 6030/44000 | Loss: 3.2456 | LR: 4.82e-04 | 399.8ms/step | 50.0 samples/s +Step 6040/44000 | Loss: 3.2379 | LR: 4.82e-04 | 400.3ms/step | 50.0 samples/s +Step 6050/44000 | Loss: 3.3111 | LR: 4.82e-04 | 399.9ms/step | 50.0 samples/s +Step 6060/44000 | Loss: 3.2660 | LR: 4.82e-04 | 400.1ms/step | 50.0 samples/s +Step 6070/44000 | Loss: 3.2725 | LR: 4.82e-04 | 400.0ms/step | 50.0 samples/s +Step 6080/44000 | Loss: 3.2460 | LR: 4.81e-04 | 400.0ms/step | 50.0 samples/s +Step 6090/44000 | Loss: 3.2457 | LR: 4.81e-04 | 399.3ms/step | 50.1 samples/s +Step 6100/44000 | Loss: 3.2165 | LR: 4.81e-04 | 399.4ms/step | 50.1 samples/s +Step 6110/44000 | Loss: 3.3285 | LR: 4.81e-04 | 401.6ms/step | 49.8 samples/s +Step 6120/44000 | Loss: 3.2460 | LR: 4.81e-04 | 401.3ms/step | 49.8 samples/s +Step 6130/44000 | Loss: 3.2705 | LR: 4.81e-04 | 401.4ms/step | 49.8 samples/s +Step 6140/44000 | Loss: 3.2530 | LR: 4.81e-04 | 401.8ms/step | 49.8 samples/s +Step 6150/44000 | Loss: 3.2755 | LR: 4.81e-04 | 399.9ms/step | 50.0 samples/s +Step 6160/44000 | Loss: 3.2658 | LR: 4.81e-04 | 400.5ms/step | 49.9 samples/s +Step 6170/44000 | Loss: 3.3334 | LR: 4.81e-04 | 401.1ms/step | 49.9 samples/s +Step 6180/44000 | Loss: 3.2505 | LR: 4.81e-04 | 400.2ms/step | 50.0 samples/s +Step 6190/44000 | Loss: 3.2484 | LR: 4.81e-04 | 400.3ms/step | 50.0 samples/s +Step 6200/44000 | Loss: 3.2567 | LR: 4.81e-04 | 401.7ms/step | 49.8 samples/s +Step 6210/44000 | Loss: 3.2375 | LR: 4.81e-04 | 399.9ms/step | 50.0 samples/s +Step 6220/44000 | Loss: 3.3099 | LR: 4.81e-04 | 400.7ms/step | 49.9 samples/s +Step 6230/44000 | Loss: 3.2473 | LR: 4.80e-04 | 400.2ms/step | 50.0 samples/s +Step 6240/44000 | Loss: 3.2153 | LR: 4.80e-04 | 401.1ms/step | 49.9 samples/s +Step 6250/44000 | Loss: 3.2836 | LR: 4.80e-04 | 402.3ms/step | 49.7 samples/s +Validation at step 6250: Loss = 3.2629, Perplexity = 26.13 +Step 6260/44000 | Loss: 3.2608 | LR: 4.80e-04 | 567.7ms/step | 35.2 samples/s +Step 6270/44000 | Loss: 3.2331 | LR: 4.80e-04 | 400.0ms/step | 50.0 samples/s +Step 6280/44000 | Loss: 3.2955 | LR: 4.80e-04 | 400.0ms/step | 50.0 samples/s +Step 6290/44000 | Loss: 3.2793 | LR: 4.80e-04 | 400.0ms/step | 50.0 samples/s +Step 6300/44000 | Loss: 3.2871 | LR: 4.80e-04 | 399.9ms/step | 50.0 samples/s +Step 6310/44000 | Loss: 3.2327 | LR: 4.80e-04 | 400.1ms/step | 50.0 samples/s +Step 6320/44000 | Loss: 3.2875 | LR: 4.80e-04 | 400.0ms/step | 50.0 samples/s +Step 6330/44000 | Loss: 3.2515 | LR: 4.80e-04 | 400.1ms/step | 50.0 samples/s +Step 6340/44000 | Loss: 3.2520 | LR: 4.80e-04 | 399.9ms/step | 50.0 samples/s +Step 6350/44000 | Loss: 3.3280 | LR: 4.80e-04 | 400.1ms/step | 50.0 samples/s +Step 6360/44000 | Loss: 3.2737 | LR: 4.80e-04 | 399.6ms/step | 50.1 samples/s +Step 6370/44000 | Loss: 3.2503 | LR: 4.79e-04 | 400.4ms/step | 50.0 samples/s +Step 6380/44000 | Loss: 3.2578 | LR: 4.79e-04 | 399.4ms/step | 50.1 samples/s +Step 6390/44000 | Loss: 3.2915 | LR: 4.79e-04 | 399.3ms/step | 50.1 samples/s +Step 6400/44000 | Loss: 3.2265 | LR: 4.79e-04 | 399.6ms/step | 50.1 samples/s +Step 6410/44000 | Loss: 3.2479 | LR: 4.79e-04 | 401.5ms/step | 49.8 samples/s +Step 6420/44000 | Loss: 3.2926 | LR: 4.79e-04 | 400.2ms/step | 50.0 samples/s +Step 6430/44000 | Loss: 3.3236 | LR: 4.79e-04 | 400.0ms/step | 50.0 samples/s +Step 6440/44000 | Loss: 3.3200 | LR: 4.79e-04 | 400.0ms/step | 50.0 samples/s +Step 6450/44000 | Loss: 3.2351 | LR: 4.79e-04 | 400.1ms/step | 50.0 samples/s +Step 6460/44000 | Loss: 3.2510 | LR: 4.79e-04 | 399.9ms/step | 50.0 samples/s +Step 6470/44000 | Loss: 3.2529 | LR: 4.79e-04 | 400.1ms/step | 50.0 samples/s +Step 6480/44000 | Loss: 3.2913 | LR: 4.79e-04 | 400.5ms/step | 49.9 samples/s +Step 6490/44000 | Loss: 3.2556 | LR: 4.79e-04 | 399.5ms/step | 50.1 samples/s +Step 6500/44000 | Loss: 3.2688 | LR: 4.79e-04 | 400.3ms/step | 50.0 samples/s +Validation at step 6500: Loss = 3.2510, Perplexity = 25.82 +Step 6510/44000 | Loss: 3.2252 | LR: 4.78e-04 | 565.8ms/step | 35.3 samples/s +Step 6520/44000 | Loss: 3.2739 | LR: 4.78e-04 | 399.9ms/step | 50.0 samples/s +Step 6530/44000 | Loss: 3.2961 | LR: 4.78e-04 | 400.0ms/step | 50.0 samples/s +Step 6540/44000 | Loss: 3.2423 | LR: 4.78e-04 | 400.0ms/step | 50.0 samples/s +Step 6550/44000 | Loss: 3.3215 | LR: 4.78e-04 | 400.1ms/step | 50.0 samples/s +Step 6560/44000 | Loss: 3.1921 | LR: 4.78e-04 | 399.8ms/step | 50.0 samples/s +Step 6570/44000 | Loss: 3.2278 | LR: 4.78e-04 | 400.0ms/step | 50.0 samples/s +Step 6580/44000 | Loss: 3.2581 | LR: 4.78e-04 | 399.9ms/step | 50.0 samples/s +Step 6590/44000 | Loss: 3.2869 | LR: 4.78e-04 | 399.9ms/step | 50.0 samples/s +Step 6600/44000 | Loss: 3.2516 | LR: 4.78e-04 | 400.3ms/step | 50.0 samples/s +Step 6610/44000 | Loss: 3.2372 | LR: 4.78e-04 | 399.8ms/step | 50.0 samples/s +Step 6620/44000 | Loss: 3.2956 | LR: 4.78e-04 | 400.1ms/step | 50.0 samples/s +Step 6630/44000 | Loss: 3.2919 | LR: 4.78e-04 | 400.1ms/step | 50.0 samples/s +Step 6640/44000 | Loss: 3.2307 | LR: 4.77e-04 | 400.0ms/step | 50.0 samples/s +Step 6650/44000 | Loss: 3.1492 | LR: 4.77e-04 | 400.1ms/step | 50.0 samples/s +Step 6660/44000 | Loss: 3.2948 | LR: 4.77e-04 | 400.1ms/step | 50.0 samples/s +Step 6670/44000 | Loss: 3.2436 | LR: 4.77e-04 | 399.9ms/step | 50.0 samples/s +Step 6680/44000 | Loss: 3.2790 | LR: 4.77e-04 | 399.6ms/step | 50.1 samples/s +Step 6690/44000 | Loss: 3.2750 | LR: 4.77e-04 | 398.2ms/step | 50.2 samples/s +Step 6700/44000 | Loss: 3.2354 | LR: 4.77e-04 | 401.1ms/step | 49.9 samples/s +Step 6710/44000 | Loss: 3.2617 | LR: 4.77e-04 | 401.2ms/step | 49.8 samples/s +Step 6720/44000 | Loss: 3.2479 | LR: 4.77e-04 | 400.3ms/step | 50.0 samples/s +Step 6730/44000 | Loss: 3.2527 | LR: 4.77e-04 | 400.2ms/step | 50.0 samples/s +Step 6740/44000 | Loss: 3.2301 | LR: 4.77e-04 | 400.8ms/step | 49.9 samples/s +Step 6750/44000 | Loss: 3.2061 | LR: 4.77e-04 | 401.6ms/step | 49.8 samples/s +Validation at step 6750: Loss = 3.2431, Perplexity = 25.61 +Step 6760/44000 | Loss: 3.2992 | LR: 4.77e-04 | 568.4ms/step | 35.2 samples/s +Step 6770/44000 | Loss: 3.1886 | LR: 4.76e-04 | 401.7ms/step | 49.8 samples/s +Step 6780/44000 | Loss: 3.2457 | LR: 4.76e-04 | 401.1ms/step | 49.9 samples/s +Step 6790/44000 | Loss: 3.1740 | LR: 4.76e-04 | 400.3ms/step | 50.0 samples/s +Step 6800/44000 | Loss: 3.2443 | LR: 4.76e-04 | 399.4ms/step | 50.1 samples/s +Step 6810/44000 | Loss: 3.2475 | LR: 4.76e-04 | 399.7ms/step | 50.0 samples/s +Step 6820/44000 | Loss: 3.1969 | LR: 4.76e-04 | 400.4ms/step | 49.9 samples/s +Step 6830/44000 | Loss: 3.2592 | LR: 4.76e-04 | 400.1ms/step | 50.0 samples/s +Step 6840/44000 | Loss: 3.2960 | LR: 4.76e-04 | 399.8ms/step | 50.0 samples/s +Step 6850/44000 | Loss: 3.1773 | LR: 4.76e-04 | 402.2ms/step | 49.7 samples/s +Step 6860/44000 | Loss: 3.2137 | LR: 4.76e-04 | 400.1ms/step | 50.0 samples/s +Step 6870/44000 | Loss: 3.2652 | LR: 4.76e-04 | 399.9ms/step | 50.0 samples/s +Step 6880/44000 | Loss: 3.2897 | LR: 4.76e-04 | 399.0ms/step | 50.1 samples/s +Step 6890/44000 | Loss: 3.2722 | LR: 4.76e-04 | 399.0ms/step | 50.1 samples/s +Step 6900/44000 | Loss: 3.2965 | LR: 4.75e-04 | 398.3ms/step | 50.2 samples/s +Step 6910/44000 | Loss: 3.2356 | LR: 4.75e-04 | 399.2ms/step | 50.1 samples/s +Step 6920/44000 | Loss: 3.2547 | LR: 4.75e-04 | 399.8ms/step | 50.0 samples/s +Step 6930/44000 | Loss: 3.2523 | LR: 4.75e-04 | 398.3ms/step | 50.2 samples/s +Step 6940/44000 | Loss: 3.2487 | LR: 4.75e-04 | 398.5ms/step | 50.2 samples/s +Step 6950/44000 | Loss: 3.1834 | LR: 4.75e-04 | 399.9ms/step | 50.0 samples/s +Step 6960/44000 | Loss: 3.2080 | LR: 4.75e-04 | 400.6ms/step | 49.9 samples/s +Step 6970/44000 | Loss: 3.2202 | LR: 4.75e-04 | 399.2ms/step | 50.1 samples/s +Step 6980/44000 | Loss: 3.2764 | LR: 4.75e-04 | 401.2ms/step | 49.9 samples/s +Step 6990/44000 | Loss: 3.2588 | LR: 4.75e-04 | 401.1ms/step | 49.9 samples/s +Step 7000/44000 | Loss: 3.2272 | LR: 4.75e-04 | 400.3ms/step | 50.0 samples/s +Validation at step 7000: Loss = 3.2335, Perplexity = 25.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-7000 +Step 7010/44000 | Loss: 3.3239 | LR: 4.75e-04 | 919.5ms/step | 21.8 samples/s +Step 7020/44000 | Loss: 3.2367 | LR: 4.75e-04 | 399.8ms/step | 50.0 samples/s +Step 7030/44000 | Loss: 3.2962 | LR: 4.74e-04 | 400.1ms/step | 50.0 samples/s +Step 7040/44000 | Loss: 3.3104 | LR: 4.74e-04 | 400.4ms/step | 49.9 samples/s +Step 7050/44000 | Loss: 3.2305 | LR: 4.74e-04 | 399.8ms/step | 50.0 samples/s +Step 7060/44000 | Loss: 3.2352 | LR: 4.74e-04 | 400.0ms/step | 50.0 samples/s +Step 7070/44000 | Loss: 3.2578 | LR: 4.74e-04 | 399.4ms/step | 50.1 samples/s +Step 7080/44000 | Loss: 3.2685 | LR: 4.74e-04 | 400.3ms/step | 50.0 samples/s +Step 7090/44000 | Loss: 3.1920 | LR: 4.74e-04 | 399.9ms/step | 50.0 samples/s +Step 7100/44000 | Loss: 3.2242 | LR: 4.74e-04 | 400.3ms/step | 50.0 samples/s +Step 7110/44000 | Loss: 3.2136 | LR: 4.74e-04 | 400.0ms/step | 50.0 samples/s +Step 7120/44000 | Loss: 3.2247 | LR: 4.74e-04 | 399.9ms/step | 50.0 samples/s +Step 7130/44000 | Loss: 3.2143 | LR: 4.74e-04 | 400.0ms/step | 50.0 samples/s +Step 7140/44000 | Loss: 3.2249 | LR: 4.74e-04 | 400.2ms/step | 50.0 samples/s +Step 7150/44000 | Loss: 3.1710 | LR: 4.73e-04 | 400.0ms/step | 50.0 samples/s +Step 7160/44000 | Loss: 3.2747 | LR: 4.73e-04 | 400.3ms/step | 50.0 samples/s +Step 7170/44000 | Loss: 3.3053 | LR: 4.73e-04 | 399.9ms/step | 50.0 samples/s +Step 7180/44000 | Loss: 3.2337 | LR: 4.73e-04 | 399.9ms/step | 50.0 samples/s +Step 7190/44000 | Loss: 3.2119 | LR: 4.73e-04 | 400.0ms/step | 50.0 samples/s +Step 7200/44000 | Loss: 3.2380 | LR: 4.73e-04 | 401.2ms/step | 49.9 samples/s +Step 7210/44000 | Loss: 3.1365 | LR: 4.73e-04 | 401.4ms/step | 49.8 samples/s +Step 7220/44000 | Loss: 3.2550 | LR: 4.73e-04 | 401.4ms/step | 49.8 samples/s +Step 7230/44000 | Loss: 3.2957 | LR: 4.73e-04 | 401.3ms/step | 49.8 samples/s +Step 7240/44000 | Loss: 3.2333 | LR: 4.73e-04 | 401.8ms/step | 49.8 samples/s +Step 7250/44000 | Loss: 3.2461 | LR: 4.73e-04 | 401.8ms/step | 49.8 samples/s +Validation at step 7250: Loss = 3.2235, Perplexity = 25.12 +Step 7260/44000 | Loss: 3.2025 | LR: 4.73e-04 | 568.2ms/step | 35.2 samples/s +Step 7270/44000 | Loss: 3.1691 | LR: 4.72e-04 | 401.0ms/step | 49.9 samples/s +Step 7280/44000 | Loss: 3.2726 | LR: 4.72e-04 | 401.9ms/step | 49.8 samples/s +Step 7290/44000 | Loss: 3.2082 | LR: 4.72e-04 | 401.3ms/step | 49.8 samples/s +Step 7300/44000 | Loss: 3.2528 | LR: 4.72e-04 | 401.3ms/step | 49.8 samples/s +Step 7310/44000 | Loss: 3.1969 | LR: 4.72e-04 | 401.6ms/step | 49.8 samples/s +Step 7320/44000 | Loss: 3.2414 | LR: 4.72e-04 | 401.9ms/step | 49.8 samples/s +Step 7330/44000 | Loss: 3.2332 | LR: 4.72e-04 | 400.8ms/step | 49.9 samples/s +Step 7340/44000 | Loss: 3.1943 | LR: 4.72e-04 | 402.0ms/step | 49.7 samples/s +Step 7350/44000 | Loss: 3.2304 | LR: 4.72e-04 | 401.8ms/step | 49.8 samples/s +Step 7360/44000 | Loss: 3.1652 | LR: 4.72e-04 | 403.0ms/step | 49.6 samples/s +Step 7370/44000 | Loss: 3.2179 | LR: 4.72e-04 | 401.6ms/step | 49.8 samples/s +Step 7380/44000 | Loss: 3.2603 | LR: 4.72e-04 | 403.4ms/step | 49.6 samples/s +Step 7390/44000 | Loss: 3.1736 | LR: 4.71e-04 | 401.6ms/step | 49.8 samples/s +Step 7400/44000 | Loss: 3.2335 | LR: 4.71e-04 | 402.2ms/step | 49.7 samples/s +Step 7410/44000 | Loss: 3.1781 | LR: 4.71e-04 | 400.1ms/step | 50.0 samples/s +Step 7420/44000 | Loss: 3.2878 | LR: 4.71e-04 | 401.3ms/step | 49.8 samples/s +Step 7430/44000 | Loss: 3.1918 | LR: 4.71e-04 | 402.2ms/step | 49.7 samples/s +Step 7440/44000 | Loss: 3.2566 | LR: 4.71e-04 | 399.3ms/step | 50.1 samples/s +Step 7450/44000 | Loss: 3.1200 | LR: 4.71e-04 | 400.4ms/step | 49.9 samples/s +Step 7460/44000 | Loss: 3.2512 | LR: 4.71e-04 | 400.3ms/step | 50.0 samples/s +Step 7470/44000 | Loss: 3.1744 | LR: 4.71e-04 | 399.8ms/step | 50.0 samples/s +Step 7480/44000 | Loss: 3.2436 | LR: 4.71e-04 | 400.3ms/step | 50.0 samples/s +Step 7490/44000 | Loss: 3.2362 | LR: 4.71e-04 | 399.9ms/step | 50.0 samples/s +Step 7500/44000 | Loss: 3.2213 | LR: 4.71e-04 | 400.6ms/step | 49.9 samples/s +Validation at step 7500: Loss = 3.2187, Perplexity = 25.00 +Step 7510/44000 | Loss: 3.2213 | LR: 4.70e-04 | 568.7ms/step | 35.2 samples/s +Step 7520/44000 | Loss: 3.2500 | LR: 4.70e-04 | 400.7ms/step | 49.9 samples/s +Step 7530/44000 | Loss: 3.1206 | LR: 4.70e-04 | 399.9ms/step | 50.0 samples/s +Step 7540/44000 | Loss: 3.2359 | LR: 4.70e-04 | 400.5ms/step | 49.9 samples/s +Step 7550/44000 | Loss: 3.2671 | LR: 4.70e-04 | 401.7ms/step | 49.8 samples/s +Step 7560/44000 | Loss: 3.2429 | LR: 4.70e-04 | 401.1ms/step | 49.9 samples/s +Step 7570/44000 | Loss: 3.1663 | LR: 4.70e-04 | 401.1ms/step | 49.9 samples/s +Step 7580/44000 | Loss: 3.2128 | LR: 4.70e-04 | 400.1ms/step | 50.0 samples/s +Step 7590/44000 | Loss: 3.2313 | LR: 4.70e-04 | 399.5ms/step | 50.1 samples/s +Step 7600/44000 | Loss: 3.2464 | LR: 4.70e-04 | 400.1ms/step | 50.0 samples/s +Step 7610/44000 | Loss: 3.2609 | LR: 4.70e-04 | 400.3ms/step | 50.0 samples/s +Step 7620/44000 | Loss: 3.2137 | LR: 4.70e-04 | 401.6ms/step | 49.8 samples/s +Step 7630/44000 | Loss: 3.1920 | LR: 4.69e-04 | 400.8ms/step | 49.9 samples/s +Step 7640/44000 | Loss: 3.2166 | LR: 4.69e-04 | 401.2ms/step | 49.8 samples/s +Step 7650/44000 | Loss: 3.2171 | LR: 4.69e-04 | 400.0ms/step | 50.0 samples/s +Step 7660/44000 | Loss: 3.2536 | LR: 4.69e-04 | 400.8ms/step | 49.9 samples/s +Step 7670/44000 | Loss: 3.2568 | LR: 4.69e-04 | 402.0ms/step | 49.8 samples/s +Step 7680/44000 | Loss: 3.1989 | LR: 4.69e-04 | 400.6ms/step | 49.9 samples/s +Step 7690/44000 | Loss: 3.3043 | LR: 4.69e-04 | 401.2ms/step | 49.9 samples/s +Step 7700/44000 | Loss: 3.2156 | LR: 4.69e-04 | 399.5ms/step | 50.1 samples/s +Step 7710/44000 | Loss: 3.1290 | LR: 4.69e-04 | 401.3ms/step | 49.8 samples/s +Step 7720/44000 | Loss: 3.1595 | LR: 4.69e-04 | 401.4ms/step | 49.8 samples/s +Step 7730/44000 | Loss: 3.2531 | LR: 4.69e-04 | 401.5ms/step | 49.8 samples/s +Step 7740/44000 | Loss: 3.2515 | LR: 4.68e-04 | 402.1ms/step | 49.7 samples/s +Step 7750/44000 | Loss: 3.2264 | LR: 4.68e-04 | 401.9ms/step | 49.8 samples/s +Validation at step 7750: Loss = 3.2078, Perplexity = 24.72 +Step 7760/44000 | Loss: 3.1740 | LR: 4.68e-04 | 567.2ms/step | 35.3 samples/s +Step 7770/44000 | Loss: 3.2169 | LR: 4.68e-04 | 400.6ms/step | 49.9 samples/s +Step 7780/44000 | Loss: 3.1998 | LR: 4.68e-04 | 399.4ms/step | 50.1 samples/s +Step 7790/44000 | Loss: 3.1580 | LR: 4.68e-04 | 398.7ms/step | 50.2 samples/s +Step 7800/44000 | Loss: 3.1680 | LR: 4.68e-04 | 400.0ms/step | 50.0 samples/s +Step 7810/44000 | Loss: 3.1999 | LR: 4.68e-04 | 400.2ms/step | 50.0 samples/s +Step 7820/44000 | Loss: 3.2315 | LR: 4.68e-04 | 399.9ms/step | 50.0 samples/s +Step 7830/44000 | Loss: 3.1886 | LR: 4.68e-04 | 399.9ms/step | 50.0 samples/s +Step 7840/44000 | Loss: 3.1569 | LR: 4.68e-04 | 399.7ms/step | 50.0 samples/s +Step 7850/44000 | Loss: 3.1991 | LR: 4.67e-04 | 400.1ms/step | 50.0 samples/s +Step 7860/44000 | Loss: 3.1825 | LR: 4.67e-04 | 400.1ms/step | 50.0 samples/s +Step 7870/44000 | Loss: 3.1542 | LR: 4.67e-04 | 400.8ms/step | 49.9 samples/s +Step 7880/44000 | Loss: 3.2277 | LR: 4.67e-04 | 401.3ms/step | 49.8 samples/s +Step 7890/44000 | Loss: 3.2286 | LR: 4.67e-04 | 400.9ms/step | 49.9 samples/s +Step 7900/44000 | Loss: 3.2201 | LR: 4.67e-04 | 401.1ms/step | 49.9 samples/s +Step 7910/44000 | Loss: 3.2408 | LR: 4.67e-04 | 400.7ms/step | 49.9 samples/s +Step 7920/44000 | Loss: 3.2296 | LR: 4.67e-04 | 402.3ms/step | 49.7 samples/s +Step 7930/44000 | Loss: 3.2267 | LR: 4.67e-04 | 401.1ms/step | 49.9 samples/s +Step 7940/44000 | Loss: 3.1834 | LR: 4.67e-04 | 402.0ms/step | 49.8 samples/s +Step 7950/44000 | Loss: 3.2256 | LR: 4.67e-04 | 401.2ms/step | 49.9 samples/s +Step 7960/44000 | Loss: 3.2106 | LR: 4.66e-04 | 401.2ms/step | 49.9 samples/s +Step 7970/44000 | Loss: 3.2030 | LR: 4.66e-04 | 401.4ms/step | 49.8 samples/s +Step 7980/44000 | Loss: 3.2110 | LR: 4.66e-04 | 401.7ms/step | 49.8 samples/s +Step 7990/44000 | Loss: 3.1915 | LR: 4.66e-04 | 401.3ms/step | 49.8 samples/s +Step 8000/44000 | Loss: 3.2157 | LR: 4.66e-04 | 401.5ms/step | 49.8 samples/s +Validation at step 8000: Loss = 3.2008, Perplexity = 24.55 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-8000 +Step 8010/44000 | Loss: 3.2051 | LR: 4.66e-04 | 886.5ms/step | 22.6 samples/s +Step 8020/44000 | Loss: 3.2458 | LR: 4.66e-04 | 401.8ms/step | 49.8 samples/s +Step 8030/44000 | Loss: 3.1963 | LR: 4.66e-04 | 401.7ms/step | 49.8 samples/s +Step 8040/44000 | Loss: 3.2174 | LR: 4.66e-04 | 401.5ms/step | 49.8 samples/s +Step 8050/44000 | Loss: 3.1310 | LR: 4.66e-04 | 400.8ms/step | 49.9 samples/s +Step 8060/44000 | Loss: 3.0946 | LR: 4.66e-04 | 400.7ms/step | 49.9 samples/s +Step 8070/44000 | Loss: 3.2179 | LR: 4.65e-04 | 398.7ms/step | 50.2 samples/s +Step 8080/44000 | Loss: 3.2397 | LR: 4.65e-04 | 398.1ms/step | 50.2 samples/s +Step 8090/44000 | Loss: 3.1492 | LR: 4.65e-04 | 398.6ms/step | 50.2 samples/s +Step 8100/44000 | Loss: 3.1764 | LR: 4.65e-04 | 398.8ms/step | 50.1 samples/s +Step 8110/44000 | Loss: 3.2497 | LR: 4.65e-04 | 400.1ms/step | 50.0 samples/s +Step 8120/44000 | Loss: 3.1507 | LR: 4.65e-04 | 398.2ms/step | 50.2 samples/s +Step 8130/44000 | Loss: 3.2236 | LR: 4.65e-04 | 399.3ms/step | 50.1 samples/s +Step 8140/44000 | Loss: 3.2018 | LR: 4.65e-04 | 398.4ms/step | 50.2 samples/s +Step 8150/44000 | Loss: 3.1857 | LR: 4.65e-04 | 400.0ms/step | 50.0 samples/s +Step 8160/44000 | Loss: 3.2109 | LR: 4.65e-04 | 398.5ms/step | 50.2 samples/s +Step 8170/44000 | Loss: 3.1607 | LR: 4.65e-04 | 399.4ms/step | 50.1 samples/s +Step 8180/44000 | Loss: 3.1588 | LR: 4.64e-04 | 398.5ms/step | 50.2 samples/s +Step 8190/44000 | Loss: 3.2478 | LR: 4.64e-04 | 399.4ms/step | 50.1 samples/s +Step 8200/44000 | Loss: 3.1721 | LR: 4.64e-04 | 400.2ms/step | 50.0 samples/s +Step 8210/44000 | Loss: 3.2315 | LR: 4.64e-04 | 399.8ms/step | 50.0 samples/s +Step 8220/44000 | Loss: 3.2020 | LR: 4.64e-04 | 400.7ms/step | 49.9 samples/s +Step 8230/44000 | Loss: 3.1763 | LR: 4.64e-04 | 401.8ms/step | 49.8 samples/s +Step 8240/44000 | Loss: 3.1952 | LR: 4.64e-04 | 401.8ms/step | 49.8 samples/s +Step 8250/44000 | Loss: 3.1988 | LR: 4.64e-04 | 401.3ms/step | 49.8 samples/s +Validation at step 8250: Loss = 3.1928, Perplexity = 24.36 +Step 8260/44000 | Loss: 3.2004 | LR: 4.64e-04 | 566.3ms/step | 35.3 samples/s +Step 8270/44000 | Loss: 3.2441 | LR: 4.64e-04 | 400.2ms/step | 50.0 samples/s +Step 8280/44000 | Loss: 3.2057 | LR: 4.64e-04 | 402.5ms/step | 49.7 samples/s +Step 8290/44000 | Loss: 3.1469 | LR: 4.63e-04 | 401.8ms/step | 49.8 samples/s +Step 8300/44000 | Loss: 3.2281 | LR: 4.63e-04 | 401.4ms/step | 49.8 samples/s +Step 8310/44000 | Loss: 3.2134 | LR: 4.63e-04 | 401.0ms/step | 49.9 samples/s +Step 8320/44000 | Loss: 3.1973 | LR: 4.63e-04 | 401.1ms/step | 49.9 samples/s +Step 8330/44000 | Loss: 3.2426 | LR: 4.63e-04 | 400.4ms/step | 49.9 samples/s +Step 8340/44000 | Loss: 3.1604 | LR: 4.63e-04 | 400.2ms/step | 50.0 samples/s +Step 8350/44000 | Loss: 3.2175 | LR: 4.63e-04 | 401.1ms/step | 49.9 samples/s +Step 8360/44000 | Loss: 3.2106 | LR: 4.63e-04 | 400.8ms/step | 49.9 samples/s +Step 8370/44000 | Loss: 3.2684 | LR: 4.63e-04 | 400.4ms/step | 49.9 samples/s +Step 8380/44000 | Loss: 3.2378 | LR: 4.63e-04 | 401.0ms/step | 49.9 samples/s +Step 8390/44000 | Loss: 3.1491 | LR: 4.62e-04 | 400.3ms/step | 50.0 samples/s +Step 8400/44000 | Loss: 3.1384 | LR: 4.62e-04 | 400.8ms/step | 49.9 samples/s +Step 8410/44000 | Loss: 3.2046 | LR: 4.62e-04 | 401.8ms/step | 49.8 samples/s +Step 8420/44000 | Loss: 3.1932 | LR: 4.62e-04 | 402.5ms/step | 49.7 samples/s +Step 8430/44000 | Loss: 3.1827 | LR: 4.62e-04 | 401.9ms/step | 49.8 samples/s +Step 8440/44000 | Loss: 3.1658 | LR: 4.62e-04 | 401.9ms/step | 49.8 samples/s +Step 8450/44000 | Loss: 3.2368 | LR: 4.62e-04 | 401.7ms/step | 49.8 samples/s +Step 8460/44000 | Loss: 3.2599 | LR: 4.62e-04 | 402.3ms/step | 49.7 samples/s +Step 8470/44000 | Loss: 3.1841 | LR: 4.62e-04 | 402.1ms/step | 49.7 samples/s +Step 8480/44000 | Loss: 3.2119 | LR: 4.62e-04 | 402.2ms/step | 49.7 samples/s +Step 8490/44000 | Loss: 3.1554 | LR: 4.62e-04 | 402.4ms/step | 49.7 samples/s +Step 8500/44000 | Loss: 3.1780 | LR: 4.61e-04 | 401.6ms/step | 49.8 samples/s +Validation at step 8500: Loss = 3.1864, Perplexity = 24.20 +Step 8510/44000 | Loss: 3.1846 | LR: 4.61e-04 | 567.8ms/step | 35.2 samples/s +Step 8520/44000 | Loss: 3.2312 | LR: 4.61e-04 | 402.2ms/step | 49.7 samples/s +Step 8530/44000 | Loss: 3.1897 | LR: 4.61e-04 | 401.4ms/step | 49.8 samples/s +Step 8540/44000 | Loss: 3.1646 | LR: 4.61e-04 | 401.8ms/step | 49.8 samples/s +Step 8550/44000 | Loss: 3.1535 | LR: 4.61e-04 | 401.8ms/step | 49.8 samples/s +Step 8560/44000 | Loss: 3.2028 | LR: 4.61e-04 | 401.0ms/step | 49.9 samples/s +Step 8570/44000 | Loss: 3.1559 | LR: 4.61e-04 | 402.9ms/step | 49.6 samples/s +Step 8580/44000 | Loss: 3.1472 | LR: 4.61e-04 | 401.1ms/step | 49.9 samples/s +Step 8590/44000 | Loss: 3.1948 | LR: 4.61e-04 | 402.6ms/step | 49.7 samples/s +Step 8600/44000 | Loss: 3.1806 | LR: 4.60e-04 | 402.7ms/step | 49.7 samples/s +Step 8610/44000 | Loss: 3.2315 | LR: 4.60e-04 | 402.2ms/step | 49.7 samples/s +Step 8620/44000 | Loss: 3.1543 | LR: 4.60e-04 | 401.8ms/step | 49.8 samples/s +Step 8630/44000 | Loss: 3.2575 | LR: 4.60e-04 | 402.3ms/step | 49.7 samples/s +Step 8640/44000 | Loss: 3.1818 | LR: 4.60e-04 | 402.3ms/step | 49.7 samples/s +Step 8650/44000 | Loss: 3.1578 | LR: 4.60e-04 | 402.1ms/step | 49.7 samples/s +Step 8660/44000 | Loss: 3.2858 | LR: 4.60e-04 | 403.5ms/step | 49.6 samples/s +Step 8670/44000 | Loss: 3.2250 | LR: 4.60e-04 | 404.4ms/step | 49.4 samples/s +Step 8680/44000 | Loss: 3.1791 | LR: 4.60e-04 | 404.4ms/step | 49.5 samples/s +Step 8690/44000 | Loss: 3.1588 | LR: 4.60e-04 | 403.4ms/step | 49.6 samples/s +Step 8700/44000 | Loss: 3.2249 | LR: 4.59e-04 | 403.5ms/step | 49.6 samples/s +Step 8710/44000 | Loss: 3.1615 | LR: 4.59e-04 | 404.1ms/step | 49.5 samples/s +Step 8720/44000 | Loss: 3.1938 | LR: 4.59e-04 | 404.7ms/step | 49.4 samples/s +Step 8730/44000 | Loss: 3.2156 | LR: 4.59e-04 | 402.8ms/step | 49.7 samples/s +Step 8740/44000 | Loss: 3.2524 | LR: 4.59e-04 | 402.9ms/step | 49.6 samples/s +Step 8750/44000 | Loss: 3.1515 | LR: 4.59e-04 | 403.8ms/step | 49.5 samples/s +Validation at step 8750: Loss = 3.1799, Perplexity = 24.04 +Step 8760/44000 | Loss: 3.1392 | LR: 4.59e-04 | 571.8ms/step | 35.0 samples/s +Step 8770/44000 | Loss: 3.2298 | LR: 4.59e-04 | 405.7ms/step | 49.3 samples/s +Step 8780/44000 | Loss: 3.1651 | LR: 4.59e-04 | 403.9ms/step | 49.5 samples/s +Step 8790/44000 | Loss: 3.1443 | LR: 4.59e-04 | 404.4ms/step | 49.5 samples/s +Step 8800/44000 | Loss: 3.1596 | LR: 4.58e-04 | 403.4ms/step | 49.6 samples/s +Step 8810/44000 | Loss: 3.2062 | LR: 4.58e-04 | 404.8ms/step | 49.4 samples/s +Step 8820/44000 | Loss: 3.2154 | LR: 4.58e-04 | 403.4ms/step | 49.6 samples/s +Step 8830/44000 | Loss: 3.1527 | LR: 4.58e-04 | 403.7ms/step | 49.5 samples/s +Step 8840/44000 | Loss: 3.2052 | LR: 4.58e-04 | 404.4ms/step | 49.5 samples/s +Step 8850/44000 | Loss: 3.1845 | LR: 4.58e-04 | 403.4ms/step | 49.6 samples/s +Step 8860/44000 | Loss: 3.2368 | LR: 4.58e-04 | 403.7ms/step | 49.5 samples/s +Step 8870/44000 | Loss: 3.1967 | LR: 4.58e-04 | 404.8ms/step | 49.4 samples/s +Step 8880/44000 | Loss: 3.1461 | LR: 4.58e-04 | 404.0ms/step | 49.5 samples/s +Step 8890/44000 | Loss: 3.1707 | LR: 4.58e-04 | 403.9ms/step | 49.5 samples/s +Step 8900/44000 | Loss: 3.1780 | LR: 4.57e-04 | 400.6ms/step | 49.9 samples/s +Step 8910/44000 | Loss: 3.1342 | LR: 4.57e-04 | 402.9ms/step | 49.6 samples/s +Step 8920/44000 | Loss: 3.1741 | LR: 4.57e-04 | 404.7ms/step | 49.4 samples/s +Step 8930/44000 | Loss: 3.1660 | LR: 4.57e-04 | 404.3ms/step | 49.5 samples/s +Step 8940/44000 | Loss: 3.1230 | LR: 4.57e-04 | 401.4ms/step | 49.8 samples/s +Step 8950/44000 | Loss: 3.2337 | LR: 4.57e-04 | 401.4ms/step | 49.8 samples/s +Step 8960/44000 | Loss: 3.1792 | LR: 4.57e-04 | 402.7ms/step | 49.7 samples/s +Step 8970/44000 | Loss: 3.1633 | LR: 4.57e-04 | 404.2ms/step | 49.5 samples/s +Step 8980/44000 | Loss: 3.2091 | LR: 4.57e-04 | 404.1ms/step | 49.5 samples/s +Step 8990/44000 | Loss: 3.1837 | LR: 4.57e-04 | 403.5ms/step | 49.6 samples/s +Step 9000/44000 | Loss: 3.1670 | LR: 4.56e-04 | 399.6ms/step | 50.0 samples/s +Validation at step 9000: Loss = 3.1700, Perplexity = 23.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-9000 +Step 9010/44000 | Loss: 3.1426 | LR: 4.56e-04 | 890.5ms/step | 22.5 samples/s +Step 9020/44000 | Loss: 3.1959 | LR: 4.56e-04 | 399.7ms/step | 50.0 samples/s +Step 9030/44000 | Loss: 3.1653 | LR: 4.56e-04 | 400.8ms/step | 49.9 samples/s +Step 9040/44000 | Loss: 3.1823 | LR: 4.56e-04 | 400.6ms/step | 49.9 samples/s +Step 9050/44000 | Loss: 3.1168 | LR: 4.56e-04 | 400.8ms/step | 49.9 samples/s +Step 9060/44000 | Loss: 3.1602 | LR: 4.56e-04 | 400.5ms/step | 49.9 samples/s +Step 9070/44000 | Loss: 3.1795 | LR: 4.56e-04 | 401.9ms/step | 49.8 samples/s +Step 9080/44000 | Loss: 3.1659 | LR: 4.56e-04 | 400.4ms/step | 49.9 samples/s +Step 9090/44000 | Loss: 3.2370 | LR: 4.56e-04 | 399.7ms/step | 50.0 samples/s +Step 9100/44000 | Loss: 3.1935 | LR: 4.55e-04 | 401.6ms/step | 49.8 samples/s +Step 9110/44000 | Loss: 3.2283 | LR: 4.55e-04 | 400.8ms/step | 49.9 samples/s +Step 9120/44000 | Loss: 3.1883 | LR: 4.55e-04 | 401.6ms/step | 49.8 samples/s +Step 9130/44000 | Loss: 3.1824 | LR: 4.55e-04 | 401.5ms/step | 49.8 samples/s +Step 9140/44000 | Loss: 3.1290 | LR: 4.55e-04 | 400.9ms/step | 49.9 samples/s +Step 9150/44000 | Loss: 3.1408 | LR: 4.55e-04 | 402.1ms/step | 49.7 samples/s +Step 9160/44000 | Loss: 3.1846 | LR: 4.55e-04 | 401.2ms/step | 49.8 samples/s +Step 9170/44000 | Loss: 3.1298 | LR: 4.55e-04 | 401.2ms/step | 49.8 samples/s +Step 9180/44000 | Loss: 3.2014 | LR: 4.55e-04 | 402.5ms/step | 49.7 samples/s +Step 9190/44000 | Loss: 3.1719 | LR: 4.54e-04 | 402.1ms/step | 49.7 samples/s +Step 9200/44000 | Loss: 3.2160 | LR: 4.54e-04 | 401.9ms/step | 49.8 samples/s +Step 9210/44000 | Loss: 3.1404 | LR: 4.54e-04 | 400.1ms/step | 50.0 samples/s +Step 9220/44000 | Loss: 3.1769 | LR: 4.54e-04 | 401.1ms/step | 49.9 samples/s +Step 9230/44000 | Loss: 3.1067 | LR: 4.54e-04 | 401.0ms/step | 49.9 samples/s +Step 9240/44000 | Loss: 3.2563 | LR: 4.54e-04 | 400.8ms/step | 49.9 samples/s +Step 9250/44000 | Loss: 3.1914 | LR: 4.54e-04 | 401.6ms/step | 49.8 samples/s +Validation at step 9250: Loss = 3.1646, Perplexity = 23.68 +Step 9260/44000 | Loss: 3.0910 | LR: 4.54e-04 | 568.7ms/step | 35.2 samples/s +Step 9270/44000 | Loss: 3.1740 | LR: 4.54e-04 | 403.2ms/step | 49.6 samples/s +Step 9280/44000 | Loss: 3.2080 | LR: 4.54e-04 | 401.5ms/step | 49.8 samples/s +Step 9290/44000 | Loss: 3.2016 | LR: 4.53e-04 | 401.3ms/step | 49.8 samples/s +Step 9300/44000 | Loss: 3.1437 | LR: 4.53e-04 | 401.9ms/step | 49.8 samples/s +Step 9310/44000 | Loss: 3.1890 | LR: 4.53e-04 | 401.4ms/step | 49.8 samples/s +Step 9320/44000 | Loss: 3.2618 | LR: 4.53e-04 | 401.7ms/step | 49.8 samples/s +Step 9330/44000 | Loss: 3.1101 | LR: 4.53e-04 | 402.2ms/step | 49.7 samples/s +Step 9340/44000 | Loss: 3.2035 | LR: 4.53e-04 | 401.8ms/step | 49.8 samples/s +Step 9350/44000 | Loss: 3.2060 | LR: 4.53e-04 | 401.7ms/step | 49.8 samples/s +Step 9360/44000 | Loss: 3.2011 | LR: 4.53e-04 | 401.7ms/step | 49.8 samples/s +Step 9370/44000 | Loss: 3.1456 | LR: 4.53e-04 | 400.7ms/step | 49.9 samples/s +Step 9380/44000 | Loss: 3.1540 | LR: 4.52e-04 | 401.8ms/step | 49.8 samples/s +Step 9390/44000 | Loss: 3.2053 | LR: 4.52e-04 | 400.9ms/step | 49.9 samples/s +Step 9400/44000 | Loss: 3.1393 | LR: 4.52e-04 | 399.7ms/step | 50.0 samples/s +Step 9410/44000 | Loss: 3.1561 | LR: 4.52e-04 | 399.4ms/step | 50.1 samples/s +Step 9420/44000 | Loss: 3.2366 | LR: 4.52e-04 | 400.1ms/step | 50.0 samples/s +Step 9430/44000 | Loss: 3.1414 | LR: 4.52e-04 | 399.9ms/step | 50.0 samples/s +Step 9440/44000 | Loss: 3.1180 | LR: 4.52e-04 | 399.8ms/step | 50.0 samples/s +Step 9450/44000 | Loss: 3.1624 | LR: 4.52e-04 | 400.1ms/step | 50.0 samples/s +Step 9460/44000 | Loss: 3.1549 | LR: 4.52e-04 | 400.6ms/step | 49.9 samples/s +Step 9470/44000 | Loss: 3.1625 | LR: 4.52e-04 | 401.9ms/step | 49.8 samples/s +Step 9480/44000 | Loss: 3.1532 | LR: 4.51e-04 | 400.0ms/step | 50.0 samples/s +Step 9490/44000 | Loss: 3.2046 | LR: 4.51e-04 | 399.6ms/step | 50.0 samples/s +Step 9500/44000 | Loss: 3.1775 | LR: 4.51e-04 | 401.9ms/step | 49.8 samples/s +Validation at step 9500: Loss = 3.1575, Perplexity = 23.51 +Step 9510/44000 | Loss: 3.2049 | LR: 4.51e-04 | 568.0ms/step | 35.2 samples/s +Step 9520/44000 | Loss: 3.1921 | LR: 4.51e-04 | 401.8ms/step | 49.8 samples/s +Step 9530/44000 | Loss: 3.1214 | LR: 4.51e-04 | 401.0ms/step | 49.9 samples/s +Step 9540/44000 | Loss: 3.2181 | LR: 4.51e-04 | 402.1ms/step | 49.7 samples/s +Step 9550/44000 | Loss: 3.1741 | LR: 4.51e-04 | 401.7ms/step | 49.8 samples/s +Step 9560/44000 | Loss: 3.2054 | LR: 4.51e-04 | 401.1ms/step | 49.9 samples/s +Step 9570/44000 | Loss: 3.1404 | LR: 4.50e-04 | 401.3ms/step | 49.8 samples/s +Step 9580/44000 | Loss: 3.1528 | LR: 4.50e-04 | 401.8ms/step | 49.8 samples/s +Step 9590/44000 | Loss: 3.1482 | LR: 4.50e-04 | 401.1ms/step | 49.9 samples/s +Step 9600/44000 | Loss: 3.1739 | LR: 4.50e-04 | 401.3ms/step | 49.8 samples/s +Step 9610/44000 | Loss: 3.1249 | LR: 4.50e-04 | 400.7ms/step | 49.9 samples/s +Step 9620/44000 | Loss: 3.1935 | LR: 4.50e-04 | 399.5ms/step | 50.1 samples/s +Step 9630/44000 | Loss: 3.1870 | LR: 4.50e-04 | 400.0ms/step | 50.0 samples/s +Step 9640/44000 | Loss: 3.1670 | LR: 4.50e-04 | 400.6ms/step | 49.9 samples/s +Step 9650/44000 | Loss: 3.1481 | LR: 4.50e-04 | 400.9ms/step | 49.9 samples/s +Step 9660/44000 | Loss: 3.1699 | LR: 4.49e-04 | 401.2ms/step | 49.9 samples/s +Step 9670/44000 | Loss: 3.1479 | LR: 4.49e-04 | 399.7ms/step | 50.0 samples/s +Step 9680/44000 | Loss: 3.1739 | LR: 4.49e-04 | 399.7ms/step | 50.0 samples/s +Step 9690/44000 | Loss: 3.1754 | LR: 4.49e-04 | 400.8ms/step | 49.9 samples/s +Step 9700/44000 | Loss: 3.2364 | LR: 4.49e-04 | 401.8ms/step | 49.8 samples/s +Step 9710/44000 | Loss: 3.1415 | LR: 4.49e-04 | 398.2ms/step | 50.2 samples/s +Step 9720/44000 | Loss: 3.1691 | LR: 4.49e-04 | 399.5ms/step | 50.1 samples/s +Step 9730/44000 | Loss: 3.1587 | LR: 4.49e-04 | 399.8ms/step | 50.0 samples/s +Step 9740/44000 | Loss: 3.1579 | LR: 4.49e-04 | 400.2ms/step | 50.0 samples/s +Step 9750/44000 | Loss: 3.1896 | LR: 4.48e-04 | 399.7ms/step | 50.0 samples/s +Validation at step 9750: Loss = 3.1502, Perplexity = 23.34 +Step 9760/44000 | Loss: 3.2146 | LR: 4.48e-04 | 564.5ms/step | 35.4 samples/s +Step 9770/44000 | Loss: 3.1224 | LR: 4.48e-04 | 399.9ms/step | 50.0 samples/s +Step 9780/44000 | Loss: 3.1433 | LR: 4.48e-04 | 400.2ms/step | 50.0 samples/s +Step 9790/44000 | Loss: 3.1308 | LR: 4.48e-04 | 399.9ms/step | 50.0 samples/s +Step 9800/44000 | Loss: 3.2024 | LR: 4.48e-04 | 400.0ms/step | 50.0 samples/s +Step 9810/44000 | Loss: 3.2282 | LR: 4.48e-04 | 399.9ms/step | 50.0 samples/s +Step 9820/44000 | Loss: 3.1509 | LR: 4.48e-04 | 400.0ms/step | 50.0 samples/s +Step 9830/44000 | Loss: 3.1614 | LR: 4.48e-04 | 400.1ms/step | 50.0 samples/s +Step 9840/44000 | Loss: 3.1716 | LR: 4.47e-04 | 399.9ms/step | 50.0 samples/s +Step 9850/44000 | Loss: 3.1708 | LR: 4.47e-04 | 400.1ms/step | 50.0 samples/s +Step 9860/44000 | Loss: 3.1644 | LR: 4.47e-04 | 400.6ms/step | 49.9 samples/s +Step 9870/44000 | Loss: 3.1320 | LR: 4.47e-04 | 401.3ms/step | 49.8 samples/s +Step 9880/44000 | Loss: 3.1056 | LR: 4.47e-04 | 399.9ms/step | 50.0 samples/s +Step 9890/44000 | Loss: 3.1211 | LR: 4.47e-04 | 402.2ms/step | 49.7 samples/s +Step 9900/44000 | Loss: 3.1168 | LR: 4.47e-04 | 402.3ms/step | 49.7 samples/s +Step 9910/44000 | Loss: 3.1590 | LR: 4.47e-04 | 402.3ms/step | 49.7 samples/s +Step 9920/44000 | Loss: 3.2066 | LR: 4.47e-04 | 401.3ms/step | 49.8 samples/s +Step 9930/44000 | Loss: 3.1716 | LR: 4.46e-04 | 400.0ms/step | 50.0 samples/s +Step 9940/44000 | Loss: 3.1794 | LR: 4.46e-04 | 400.0ms/step | 50.0 samples/s +Step 9950/44000 | Loss: 3.1487 | LR: 4.46e-04 | 400.2ms/step | 50.0 samples/s +Step 9960/44000 | Loss: 3.1461 | LR: 4.46e-04 | 400.3ms/step | 50.0 samples/s +Step 9970/44000 | Loss: 3.1194 | LR: 4.46e-04 | 400.9ms/step | 49.9 samples/s +Step 9980/44000 | Loss: 3.2028 | LR: 4.46e-04 | 400.6ms/step | 49.9 samples/s +Step 9990/44000 | Loss: 3.1075 | LR: 4.46e-04 | 399.8ms/step | 50.0 samples/s +Step 10000/44000 | Loss: 3.1512 | LR: 4.46e-04 | 399.1ms/step | 50.1 samples/s +Validation at step 10000: Loss = 3.1459, Perplexity = 23.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-10000 +Step 10010/44000 | Loss: 3.1946 | LR: 4.46e-04 | 868.9ms/step | 23.0 samples/s +Step 10020/44000 | Loss: 3.1354 | LR: 4.45e-04 | 400.1ms/step | 50.0 samples/s +Step 10030/44000 | Loss: 3.1196 | LR: 4.45e-04 | 399.8ms/step | 50.0 samples/s +Step 10040/44000 | Loss: 3.1727 | LR: 4.45e-04 | 400.6ms/step | 49.9 samples/s +Step 10050/44000 | Loss: 3.1468 | LR: 4.45e-04 | 398.5ms/step | 50.2 samples/s +Step 10060/44000 | Loss: 3.1180 | LR: 4.45e-04 | 398.3ms/step | 50.2 samples/s +Step 10070/44000 | Loss: 3.1233 | LR: 4.45e-04 | 398.9ms/step | 50.1 samples/s +Step 10080/44000 | Loss: 3.2344 | LR: 4.45e-04 | 399.6ms/step | 50.1 samples/s +Step 10090/44000 | Loss: 3.2368 | LR: 4.45e-04 | 398.9ms/step | 50.1 samples/s +Step 10100/44000 | Loss: 3.0848 | LR: 4.45e-04 | 401.6ms/step | 49.8 samples/s +Step 10110/44000 | Loss: 3.1266 | LR: 4.44e-04 | 400.5ms/step | 49.9 samples/s +Step 10120/44000 | Loss: 3.1914 | LR: 4.44e-04 | 399.5ms/step | 50.1 samples/s +Step 10130/44000 | Loss: 3.1321 | LR: 4.44e-04 | 400.6ms/step | 49.9 samples/s +Step 10140/44000 | Loss: 3.1206 | LR: 4.44e-04 | 399.6ms/step | 50.1 samples/s +Step 10150/44000 | Loss: 3.1126 | LR: 4.44e-04 | 400.0ms/step | 50.0 samples/s +Step 10160/44000 | Loss: 3.1270 | LR: 4.44e-04 | 399.3ms/step | 50.1 samples/s +Step 10170/44000 | Loss: 3.1315 | LR: 4.44e-04 | 398.0ms/step | 50.3 samples/s +Step 10180/44000 | Loss: 3.1242 | LR: 4.44e-04 | 398.2ms/step | 50.2 samples/s +Step 10190/44000 | Loss: 3.1475 | LR: 4.43e-04 | 399.0ms/step | 50.1 samples/s +Step 10200/44000 | Loss: 3.1077 | LR: 4.43e-04 | 400.0ms/step | 50.0 samples/s +Step 10210/44000 | Loss: 3.0830 | LR: 4.43e-04 | 401.4ms/step | 49.8 samples/s +Step 10220/44000 | Loss: 3.2239 | LR: 4.43e-04 | 400.2ms/step | 50.0 samples/s +Step 10230/44000 | Loss: 3.1749 | LR: 4.43e-04 | 401.1ms/step | 49.9 samples/s +Step 10240/44000 | Loss: 3.1450 | LR: 4.43e-04 | 401.5ms/step | 49.8 samples/s +Step 10250/44000 | Loss: 3.2024 | LR: 4.43e-04 | 401.3ms/step | 49.8 samples/s +Validation at step 10250: Loss = 3.1406, Perplexity = 23.12 +Step 10260/44000 | Loss: 3.1092 | LR: 4.43e-04 | 568.6ms/step | 35.2 samples/s +Step 10270/44000 | Loss: 3.1856 | LR: 4.43e-04 | 401.2ms/step | 49.9 samples/s +Step 10280/44000 | Loss: 3.1529 | LR: 4.42e-04 | 400.6ms/step | 49.9 samples/s +Step 10290/44000 | Loss: 3.1872 | LR: 4.42e-04 | 400.8ms/step | 49.9 samples/s +Step 10300/44000 | Loss: 3.1950 | LR: 4.42e-04 | 400.9ms/step | 49.9 samples/s +Step 10310/44000 | Loss: 3.1541 | LR: 4.42e-04 | 399.9ms/step | 50.0 samples/s +Step 10320/44000 | Loss: 3.1518 | LR: 4.42e-04 | 400.1ms/step | 50.0 samples/s +Step 10330/44000 | Loss: 3.1912 | LR: 4.42e-04 | 400.0ms/step | 50.0 samples/s +Step 10340/44000 | Loss: 3.1706 | LR: 4.42e-04 | 400.3ms/step | 50.0 samples/s +Step 10350/44000 | Loss: 3.1692 | LR: 4.42e-04 | 399.6ms/step | 50.1 samples/s +Step 10360/44000 | Loss: 3.1417 | LR: 4.42e-04 | 400.6ms/step | 49.9 samples/s +Step 10370/44000 | Loss: 3.1540 | LR: 4.41e-04 | 400.4ms/step | 50.0 samples/s +Step 10380/44000 | Loss: 3.1345 | LR: 4.41e-04 | 401.4ms/step | 49.8 samples/s +Step 10390/44000 | Loss: 3.1610 | LR: 4.41e-04 | 400.5ms/step | 49.9 samples/s +Step 10400/44000 | Loss: 3.1572 | LR: 4.41e-04 | 401.0ms/step | 49.9 samples/s +Step 10410/44000 | Loss: 3.1436 | LR: 4.41e-04 | 400.8ms/step | 49.9 samples/s +Step 10420/44000 | Loss: 3.1453 | LR: 4.41e-04 | 401.5ms/step | 49.8 samples/s +Step 10430/44000 | Loss: 3.1406 | LR: 4.41e-04 | 400.5ms/step | 49.9 samples/s +Step 10440/44000 | Loss: 3.1279 | LR: 4.41e-04 | 400.0ms/step | 50.0 samples/s +Step 10450/44000 | Loss: 3.1037 | LR: 4.40e-04 | 401.7ms/step | 49.8 samples/s +Step 10460/44000 | Loss: 3.1545 | LR: 4.40e-04 | 400.1ms/step | 50.0 samples/s +Step 10470/44000 | Loss: 3.1186 | LR: 4.40e-04 | 400.5ms/step | 49.9 samples/s +Step 10480/44000 | Loss: 3.1009 | LR: 4.40e-04 | 401.2ms/step | 49.8 samples/s +Step 10490/44000 | Loss: 3.2060 | LR: 4.40e-04 | 401.0ms/step | 49.9 samples/s +Step 10500/44000 | Loss: 3.1486 | LR: 4.40e-04 | 400.8ms/step | 49.9 samples/s +Validation at step 10500: Loss = 3.1379, Perplexity = 23.05 +Step 10510/44000 | Loss: 3.1927 | LR: 4.40e-04 | 568.4ms/step | 35.2 samples/s +Step 10520/44000 | Loss: 3.2097 | LR: 4.40e-04 | 401.6ms/step | 49.8 samples/s +Step 10530/44000 | Loss: 3.1389 | LR: 4.40e-04 | 401.2ms/step | 49.9 samples/s +Step 10540/44000 | Loss: 3.1304 | LR: 4.39e-04 | 401.6ms/step | 49.8 samples/s +Step 10550/44000 | Loss: 3.1601 | LR: 4.39e-04 | 401.3ms/step | 49.8 samples/s +Step 10560/44000 | Loss: 3.1054 | LR: 4.39e-04 | 400.5ms/step | 49.9 samples/s +Step 10570/44000 | Loss: 3.1395 | LR: 4.39e-04 | 401.6ms/step | 49.8 samples/s +Step 10580/44000 | Loss: 3.0709 | LR: 4.39e-04 | 399.8ms/step | 50.0 samples/s +Step 10590/44000 | Loss: 3.1850 | LR: 4.39e-04 | 400.0ms/step | 50.0 samples/s +Step 10600/44000 | Loss: 3.1381 | LR: 4.39e-04 | 400.0ms/step | 50.0 samples/s +Step 10610/44000 | Loss: 3.1292 | LR: 4.39e-04 | 400.2ms/step | 50.0 samples/s +Step 10620/44000 | Loss: 3.1249 | LR: 4.38e-04 | 400.1ms/step | 50.0 samples/s +Step 10630/44000 | Loss: 3.1093 | LR: 4.38e-04 | 400.6ms/step | 49.9 samples/s +Step 10640/44000 | Loss: 3.1849 | LR: 4.38e-04 | 401.4ms/step | 49.8 samples/s +Step 10650/44000 | Loss: 3.2035 | LR: 4.38e-04 | 400.3ms/step | 50.0 samples/s +Step 10660/44000 | Loss: 3.1378 | LR: 4.38e-04 | 400.8ms/step | 49.9 samples/s +Step 10670/44000 | Loss: 3.1279 | LR: 4.38e-04 | 401.6ms/step | 49.8 samples/s +Step 10680/44000 | Loss: 3.1805 | LR: 4.38e-04 | 402.0ms/step | 49.7 samples/s +Step 10690/44000 | Loss: 3.1306 | LR: 4.38e-04 | 401.8ms/step | 49.8 samples/s +Step 10700/44000 | Loss: 3.1496 | LR: 4.37e-04 | 401.8ms/step | 49.8 samples/s +Step 10710/44000 | Loss: 3.1842 | LR: 4.37e-04 | 401.5ms/step | 49.8 samples/s +Step 10720/44000 | Loss: 3.1733 | LR: 4.37e-04 | 401.1ms/step | 49.9 samples/s +Step 10730/44000 | Loss: 3.1471 | LR: 4.37e-04 | 401.1ms/step | 49.9 samples/s +Step 10740/44000 | Loss: 3.1774 | LR: 4.37e-04 | 400.6ms/step | 49.9 samples/s +Step 10750/44000 | Loss: 3.1630 | LR: 4.37e-04 | 401.5ms/step | 49.8 samples/s +Validation at step 10750: Loss = 3.1300, Perplexity = 22.87 +Step 10760/44000 | Loss: 3.1726 | LR: 4.37e-04 | 569.8ms/step | 35.1 samples/s +Step 10770/44000 | Loss: 3.1404 | LR: 4.37e-04 | 400.5ms/step | 49.9 samples/s +Step 10780/44000 | Loss: 3.1157 | LR: 4.37e-04 | 401.6ms/step | 49.8 samples/s +Step 10790/44000 | Loss: 3.1142 | LR: 4.36e-04 | 401.8ms/step | 49.8 samples/s +Step 10800/44000 | Loss: 3.0844 | LR: 4.36e-04 | 401.0ms/step | 49.9 samples/s +Step 10810/44000 | Loss: 3.1633 | LR: 4.36e-04 | 402.1ms/step | 49.7 samples/s +Step 10820/44000 | Loss: 3.1187 | LR: 4.36e-04 | 401.2ms/step | 49.9 samples/s +Step 10830/44000 | Loss: 3.1140 | LR: 4.36e-04 | 402.1ms/step | 49.7 samples/s +Step 10840/44000 | Loss: 3.0839 | LR: 4.36e-04 | 401.6ms/step | 49.8 samples/s +Step 10850/44000 | Loss: 3.1712 | LR: 4.36e-04 | 401.2ms/step | 49.9 samples/s +Step 10860/44000 | Loss: 3.1314 | LR: 4.36e-04 | 401.4ms/step | 49.8 samples/s +Step 10870/44000 | Loss: 3.0939 | LR: 4.35e-04 | 400.5ms/step | 49.9 samples/s +Step 10880/44000 | Loss: 3.1247 | LR: 4.35e-04 | 401.8ms/step | 49.8 samples/s +Step 10890/44000 | Loss: 3.1264 | LR: 4.35e-04 | 401.3ms/step | 49.8 samples/s +Step 10900/44000 | Loss: 3.1312 | LR: 4.35e-04 | 400.6ms/step | 49.9 samples/s +Step 10910/44000 | Loss: 3.0765 | LR: 4.35e-04 | 401.3ms/step | 49.8 samples/s +Step 10920/44000 | Loss: 3.1013 | LR: 4.35e-04 | 401.9ms/step | 49.8 samples/s +Step 10930/44000 | Loss: 3.1301 | LR: 4.35e-04 | 402.4ms/step | 49.7 samples/s +Step 10940/44000 | Loss: 3.1153 | LR: 4.35e-04 | 401.9ms/step | 49.8 samples/s +Step 10950/44000 | Loss: 3.0689 | LR: 4.34e-04 | 401.7ms/step | 49.8 samples/s +Step 10960/44000 | Loss: 3.1323 | LR: 4.34e-04 | 401.9ms/step | 49.8 samples/s +Step 10970/44000 | Loss: 3.1263 | LR: 4.34e-04 | 402.0ms/step | 49.8 samples/s +Step 10980/44000 | Loss: 3.0736 | LR: 4.34e-04 | 401.7ms/step | 49.8 samples/s +Step 10990/44000 | Loss: 3.1586 | LR: 4.34e-04 | 400.3ms/step | 50.0 samples/s +Step 11000/44000 | Loss: 3.1656 | LR: 4.34e-04 | 399.9ms/step | 50.0 samples/s +Validation at step 11000: Loss = 3.1252, Perplexity = 22.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-11000 +Step 11010/44000 | Loss: 3.1156 | LR: 4.34e-04 | 872.0ms/step | 22.9 samples/s +Step 11020/44000 | Loss: 3.1392 | LR: 4.34e-04 | 400.1ms/step | 50.0 samples/s +Step 11030/44000 | Loss: 3.0817 | LR: 4.33e-04 | 400.1ms/step | 50.0 samples/s +Step 11040/44000 | Loss: 3.1409 | LR: 4.33e-04 | 400.0ms/step | 50.0 samples/s +Step 11050/44000 | Loss: 3.1186 | LR: 4.33e-04 | 400.1ms/step | 50.0 samples/s +Step 11060/44000 | Loss: 3.0737 | LR: 4.33e-04 | 400.1ms/step | 50.0 samples/s +Step 11070/44000 | Loss: 3.1499 | LR: 4.33e-04 | 400.3ms/step | 50.0 samples/s +Step 11080/44000 | Loss: 3.1059 | LR: 4.33e-04 | 401.4ms/step | 49.8 samples/s +Step 11090/44000 | Loss: 3.1159 | LR: 4.33e-04 | 400.1ms/step | 50.0 samples/s +Step 11100/44000 | Loss: 3.1455 | LR: 4.33e-04 | 399.2ms/step | 50.1 samples/s +Step 11110/44000 | Loss: 3.1314 | LR: 4.32e-04 | 398.0ms/step | 50.3 samples/s +Step 11120/44000 | Loss: 3.0862 | LR: 4.32e-04 | 399.8ms/step | 50.0 samples/s +Step 11130/44000 | Loss: 3.1167 | LR: 4.32e-04 | 398.9ms/step | 50.1 samples/s +Step 11140/44000 | Loss: 3.1561 | LR: 4.32e-04 | 399.4ms/step | 50.1 samples/s +Step 11150/44000 | Loss: 3.1652 | LR: 4.32e-04 | 398.9ms/step | 50.1 samples/s +Step 11160/44000 | Loss: 3.1878 | LR: 4.32e-04 | 399.9ms/step | 50.0 samples/s +Step 11170/44000 | Loss: 3.1893 | LR: 4.32e-04 | 400.6ms/step | 49.9 samples/s +Step 11180/44000 | Loss: 3.1244 | LR: 4.32e-04 | 400.2ms/step | 50.0 samples/s +Step 11190/44000 | Loss: 3.1837 | LR: 4.31e-04 | 400.3ms/step | 50.0 samples/s +Step 11200/44000 | Loss: 3.1422 | LR: 4.31e-04 | 400.9ms/step | 49.9 samples/s +Step 11210/44000 | Loss: 3.1813 | LR: 4.31e-04 | 400.0ms/step | 50.0 samples/s +Step 11220/44000 | Loss: 3.1526 | LR: 4.31e-04 | 399.6ms/step | 50.0 samples/s +Step 11230/44000 | Loss: 3.1350 | LR: 4.31e-04 | 400.1ms/step | 50.0 samples/s +Step 11240/44000 | Loss: 3.1411 | LR: 4.31e-04 | 399.5ms/step | 50.1 samples/s +Step 11250/44000 | Loss: 3.1109 | LR: 4.31e-04 | 398.7ms/step | 50.2 samples/s +Validation at step 11250: Loss = 3.1201, Perplexity = 22.65 +Step 11260/44000 | Loss: 3.2066 | LR: 4.31e-04 | 565.9ms/step | 35.3 samples/s +Step 11270/44000 | Loss: 3.1924 | LR: 4.30e-04 | 400.6ms/step | 49.9 samples/s +Step 11280/44000 | Loss: 3.1187 | LR: 4.30e-04 | 400.4ms/step | 49.9 samples/s +Step 11290/44000 | Loss: 3.1815 | LR: 4.30e-04 | 401.4ms/step | 49.8 samples/s +Step 11300/44000 | Loss: 3.1426 | LR: 4.30e-04 | 400.1ms/step | 50.0 samples/s +Step 11310/44000 | Loss: 3.1151 | LR: 4.30e-04 | 400.8ms/step | 49.9 samples/s +Step 11320/44000 | Loss: 3.1340 | LR: 4.30e-04 | 400.8ms/step | 49.9 samples/s +Step 11330/44000 | Loss: 3.1189 | LR: 4.30e-04 | 400.3ms/step | 50.0 samples/s +Step 11340/44000 | Loss: 3.1736 | LR: 4.30e-04 | 401.4ms/step | 49.8 samples/s +Step 11350/44000 | Loss: 3.1860 | LR: 4.29e-04 | 400.5ms/step | 49.9 samples/s +Step 11360/44000 | Loss: 3.0940 | LR: 4.29e-04 | 399.7ms/step | 50.0 samples/s +Step 11370/44000 | Loss: 3.1728 | LR: 4.29e-04 | 400.9ms/step | 49.9 samples/s +Step 11380/44000 | Loss: 3.2057 | LR: 4.29e-04 | 401.2ms/step | 49.9 samples/s +Step 11390/44000 | Loss: 3.1167 | LR: 4.29e-04 | 400.5ms/step | 49.9 samples/s +Step 11400/44000 | Loss: 3.1387 | LR: 4.29e-04 | 402.6ms/step | 49.7 samples/s +Step 11410/44000 | Loss: 3.0991 | LR: 4.29e-04 | 400.8ms/step | 49.9 samples/s +Step 11420/44000 | Loss: 3.0828 | LR: 4.29e-04 | 401.1ms/step | 49.9 samples/s +Step 11430/44000 | Loss: 3.1686 | LR: 4.28e-04 | 401.3ms/step | 49.8 samples/s +Step 11440/44000 | Loss: 3.1232 | LR: 4.28e-04 | 402.6ms/step | 49.7 samples/s +Step 11450/44000 | Loss: 3.0724 | LR: 4.28e-04 | 401.2ms/step | 49.9 samples/s +Step 11460/44000 | Loss: 3.1659 | LR: 4.28e-04 | 400.1ms/step | 50.0 samples/s +Step 11470/44000 | Loss: 3.1114 | LR: 4.28e-04 | 400.0ms/step | 50.0 samples/s +Step 11480/44000 | Loss: 3.1600 | LR: 4.28e-04 | 399.8ms/step | 50.0 samples/s +Step 11490/44000 | Loss: 3.1972 | LR: 4.28e-04 | 400.0ms/step | 50.0 samples/s +Step 11500/44000 | Loss: 3.1628 | LR: 4.28e-04 | 400.3ms/step | 50.0 samples/s +Validation at step 11500: Loss = 3.1158, Perplexity = 22.55 +Step 11510/44000 | Loss: 3.1064 | LR: 4.27e-04 | 566.3ms/step | 35.3 samples/s +Step 11520/44000 | Loss: 3.1529 | LR: 4.27e-04 | 400.8ms/step | 49.9 samples/s +Step 11530/44000 | Loss: 3.1386 | LR: 4.27e-04 | 399.5ms/step | 50.1 samples/s +Step 11540/44000 | Loss: 3.0770 | LR: 4.27e-04 | 399.2ms/step | 50.1 samples/s +Step 11550/44000 | Loss: 3.1105 | LR: 4.27e-04 | 398.1ms/step | 50.2 samples/s +Step 11560/44000 | Loss: 3.1122 | LR: 4.27e-04 | 399.0ms/step | 50.1 samples/s +Step 11570/44000 | Loss: 3.1510 | LR: 4.27e-04 | 398.6ms/step | 50.2 samples/s +Step 11580/44000 | Loss: 3.1269 | LR: 4.27e-04 | 400.4ms/step | 49.9 samples/s +Step 11590/44000 | Loss: 3.1042 | LR: 4.26e-04 | 400.3ms/step | 50.0 samples/s +Step 11600/44000 | Loss: 3.1036 | LR: 4.26e-04 | 399.7ms/step | 50.0 samples/s +Step 11610/44000 | Loss: 3.1432 | LR: 4.26e-04 | 400.1ms/step | 50.0 samples/s +Step 11620/44000 | Loss: 3.1196 | LR: 4.26e-04 | 400.4ms/step | 49.9 samples/s +Step 11630/44000 | Loss: 3.0763 | LR: 4.26e-04 | 402.8ms/step | 49.7 samples/s +Step 11640/44000 | Loss: 3.0990 | LR: 4.26e-04 | 403.4ms/step | 49.6 samples/s +Step 11650/44000 | Loss: 3.1018 | LR: 4.26e-04 | 404.1ms/step | 49.5 samples/s +Step 11660/44000 | Loss: 3.0719 | LR: 4.26e-04 | 401.6ms/step | 49.8 samples/s +Step 11670/44000 | Loss: 3.1491 | LR: 4.25e-04 | 401.3ms/step | 49.8 samples/s +Step 11680/44000 | Loss: 3.1740 | LR: 4.25e-04 | 401.2ms/step | 49.9 samples/s +Step 11690/44000 | Loss: 3.0986 | LR: 4.25e-04 | 401.9ms/step | 49.8 samples/s +Step 11700/44000 | Loss: 3.1547 | LR: 4.25e-04 | 401.5ms/step | 49.8 samples/s +Step 11710/44000 | Loss: 3.0746 | LR: 4.25e-04 | 401.5ms/step | 49.8 samples/s +Step 11720/44000 | Loss: 3.1109 | LR: 4.25e-04 | 401.5ms/step | 49.8 samples/s +Step 11730/44000 | Loss: 3.1387 | LR: 4.25e-04 | 400.7ms/step | 49.9 samples/s +Step 11740/44000 | Loss: 3.1132 | LR: 4.24e-04 | 401.6ms/step | 49.8 samples/s +Step 11750/44000 | Loss: 3.0688 | LR: 4.24e-04 | 400.8ms/step | 49.9 samples/s +Validation at step 11750: Loss = 3.1135, Perplexity = 22.50 +Step 11760/44000 | Loss: 3.0490 | LR: 4.24e-04 | 568.2ms/step | 35.2 samples/s +Step 11770/44000 | Loss: 3.1342 | LR: 4.24e-04 | 401.4ms/step | 49.8 samples/s +Step 11780/44000 | Loss: 3.1122 | LR: 4.24e-04 | 400.6ms/step | 49.9 samples/s +Step 11790/44000 | Loss: 3.0648 | LR: 4.24e-04 | 401.4ms/step | 49.8 samples/s +Step 11800/44000 | Loss: 3.0833 | LR: 4.24e-04 | 400.0ms/step | 50.0 samples/s +Step 11810/44000 | Loss: 3.1555 | LR: 4.24e-04 | 399.9ms/step | 50.0 samples/s +Step 11820/44000 | Loss: 3.1530 | LR: 4.23e-04 | 400.0ms/step | 50.0 samples/s +Step 11830/44000 | Loss: 3.0672 | LR: 4.23e-04 | 400.2ms/step | 50.0 samples/s +Step 11840/44000 | Loss: 3.1313 | LR: 4.23e-04 | 400.2ms/step | 50.0 samples/s +Step 11850/44000 | Loss: 3.1286 | LR: 4.23e-04 | 400.0ms/step | 50.0 samples/s +Step 11860/44000 | Loss: 3.1811 | LR: 4.23e-04 | 399.9ms/step | 50.0 samples/s +Step 11870/44000 | Loss: 3.1119 | LR: 4.23e-04 | 400.6ms/step | 49.9 samples/s +Step 11880/44000 | Loss: 3.1102 | LR: 4.23e-04 | 401.3ms/step | 49.8 samples/s +Step 11890/44000 | Loss: 3.0900 | LR: 4.23e-04 | 401.2ms/step | 49.8 samples/s +Step 11900/44000 | Loss: 3.1156 | LR: 4.22e-04 | 400.9ms/step | 49.9 samples/s +Step 11910/44000 | Loss: 3.1183 | LR: 4.22e-04 | 401.4ms/step | 49.8 samples/s +Step 11920/44000 | Loss: 3.0928 | LR: 4.22e-04 | 400.2ms/step | 50.0 samples/s +Step 11930/44000 | Loss: 3.1333 | LR: 4.22e-04 | 400.5ms/step | 49.9 samples/s +Step 11940/44000 | Loss: 3.1473 | LR: 4.22e-04 | 401.4ms/step | 49.8 samples/s +Step 11950/44000 | Loss: 3.1524 | LR: 4.22e-04 | 400.2ms/step | 50.0 samples/s +Step 11960/44000 | Loss: 3.1262 | LR: 4.22e-04 | 401.8ms/step | 49.8 samples/s +Step 11970/44000 | Loss: 3.1238 | LR: 4.21e-04 | 400.1ms/step | 50.0 samples/s +Step 11980/44000 | Loss: 3.1324 | LR: 4.21e-04 | 400.1ms/step | 50.0 samples/s +Step 11990/44000 | Loss: 3.1105 | LR: 4.21e-04 | 399.9ms/step | 50.0 samples/s +Step 12000/44000 | Loss: 3.1258 | LR: 4.21e-04 | 399.9ms/step | 50.0 samples/s +Validation at step 12000: Loss = 3.1053, Perplexity = 22.32 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-12000 +Step 12010/44000 | Loss: 3.0736 | LR: 4.21e-04 | 910.1ms/step | 22.0 samples/s +Step 12020/44000 | Loss: 3.0934 | LR: 4.21e-04 | 399.9ms/step | 50.0 samples/s +Step 12030/44000 | Loss: 3.1222 | LR: 4.21e-04 | 400.2ms/step | 50.0 samples/s +Step 12040/44000 | Loss: 3.0956 | LR: 4.21e-04 | 400.0ms/step | 50.0 samples/s +Step 12050/44000 | Loss: 3.1096 | LR: 4.20e-04 | 399.9ms/step | 50.0 samples/s +Step 12060/44000 | Loss: 3.0944 | LR: 4.20e-04 | 399.7ms/step | 50.0 samples/s +Step 12070/44000 | Loss: 3.1210 | LR: 4.20e-04 | 400.1ms/step | 50.0 samples/s +Step 12080/44000 | Loss: 3.1134 | LR: 4.20e-04 | 398.1ms/step | 50.2 samples/s +Step 12090/44000 | Loss: 3.1427 | LR: 4.20e-04 | 398.7ms/step | 50.2 samples/s +Step 12100/44000 | Loss: 3.1582 | LR: 4.20e-04 | 398.5ms/step | 50.2 samples/s +Step 12110/44000 | Loss: 3.0807 | LR: 4.20e-04 | 398.4ms/step | 50.2 samples/s +Step 12120/44000 | Loss: 3.1042 | LR: 4.19e-04 | 398.6ms/step | 50.2 samples/s +Step 12130/44000 | Loss: 3.1001 | LR: 4.19e-04 | 399.3ms/step | 50.1 samples/s +Step 12140/44000 | Loss: 3.1181 | LR: 4.19e-04 | 398.5ms/step | 50.2 samples/s +Step 12150/44000 | Loss: 3.1055 | LR: 4.19e-04 | 399.8ms/step | 50.0 samples/s +Step 12160/44000 | Loss: 3.1349 | LR: 4.19e-04 | 399.8ms/step | 50.0 samples/s +Step 12170/44000 | Loss: 3.0752 | LR: 4.19e-04 | 400.3ms/step | 50.0 samples/s +Step 12180/44000 | Loss: 3.1162 | LR: 4.19e-04 | 400.0ms/step | 50.0 samples/s +Step 12190/44000 | Loss: 3.1040 | LR: 4.19e-04 | 399.9ms/step | 50.0 samples/s +Step 12200/44000 | Loss: 3.0999 | LR: 4.18e-04 | 400.6ms/step | 49.9 samples/s +Step 12210/44000 | Loss: 3.1001 | LR: 4.18e-04 | 399.5ms/step | 50.1 samples/s +Step 12220/44000 | Loss: 3.1658 | LR: 4.18e-04 | 399.6ms/step | 50.0 samples/s +Step 12230/44000 | Loss: 3.0749 | LR: 4.18e-04 | 398.2ms/step | 50.2 samples/s +Step 12240/44000 | Loss: 3.1212 | LR: 4.18e-04 | 399.6ms/step | 50.0 samples/s +Step 12250/44000 | Loss: 3.0725 | LR: 4.18e-04 | 399.8ms/step | 50.0 samples/s +Validation at step 12250: Loss = 3.1008, Perplexity = 22.22 +Step 12260/44000 | Loss: 3.0818 | LR: 4.18e-04 | 566.5ms/step | 35.3 samples/s +Step 12270/44000 | Loss: 3.1366 | LR: 4.17e-04 | 400.4ms/step | 50.0 samples/s +Step 12280/44000 | Loss: 3.0542 | LR: 4.17e-04 | 400.3ms/step | 50.0 samples/s +Step 12290/44000 | Loss: 3.1116 | LR: 4.17e-04 | 401.7ms/step | 49.8 samples/s +Step 12300/44000 | Loss: 3.1035 | LR: 4.17e-04 | 400.5ms/step | 49.9 samples/s +Step 12310/44000 | Loss: 3.0484 | LR: 4.17e-04 | 400.4ms/step | 50.0 samples/s +Step 12320/44000 | Loss: 3.1032 | LR: 4.17e-04 | 401.0ms/step | 49.9 samples/s +Step 12330/44000 | Loss: 3.0423 | LR: 4.17e-04 | 400.0ms/step | 50.0 samples/s +Step 12340/44000 | Loss: 3.0183 | LR: 4.17e-04 | 400.2ms/step | 50.0 samples/s +Step 12350/44000 | Loss: 3.0900 | LR: 4.16e-04 | 400.1ms/step | 50.0 samples/s +Step 12360/44000 | Loss: 3.0741 | LR: 4.16e-04 | 399.7ms/step | 50.0 samples/s +Step 12370/44000 | Loss: 3.1397 | LR: 4.16e-04 | 400.1ms/step | 50.0 samples/s +Step 12380/44000 | Loss: 3.0971 | LR: 4.16e-04 | 401.0ms/step | 49.9 samples/s +Step 12390/44000 | Loss: 3.1453 | LR: 4.16e-04 | 401.0ms/step | 49.9 samples/s +Step 12400/44000 | Loss: 3.0790 | LR: 4.16e-04 | 400.3ms/step | 50.0 samples/s +Step 12410/44000 | Loss: 3.1344 | LR: 4.16e-04 | 400.1ms/step | 50.0 samples/s +Step 12420/44000 | Loss: 3.0975 | LR: 4.15e-04 | 399.8ms/step | 50.0 samples/s +Step 12430/44000 | Loss: 3.0809 | LR: 4.15e-04 | 400.0ms/step | 50.0 samples/s +Step 12440/44000 | Loss: 3.1820 | LR: 4.15e-04 | 401.7ms/step | 49.8 samples/s +Step 12450/44000 | Loss: 3.0776 | LR: 4.15e-04 | 398.3ms/step | 50.2 samples/s +Step 12460/44000 | Loss: 3.0054 | LR: 4.15e-04 | 399.5ms/step | 50.1 samples/s +Step 12470/44000 | Loss: 3.0729 | LR: 4.15e-04 | 398.8ms/step | 50.1 samples/s +Step 12480/44000 | Loss: 3.0667 | LR: 4.15e-04 | 399.4ms/step | 50.1 samples/s +Step 12490/44000 | Loss: 3.1817 | LR: 4.14e-04 | 400.0ms/step | 50.0 samples/s +Step 12500/44000 | Loss: 3.1259 | LR: 4.14e-04 | 399.9ms/step | 50.0 samples/s +Validation at step 12500: Loss = 3.0988, Perplexity = 22.17 +Step 12510/44000 | Loss: 3.0779 | LR: 4.14e-04 | 566.2ms/step | 35.3 samples/s +Step 12520/44000 | Loss: 3.0726 | LR: 4.14e-04 | 399.9ms/step | 50.0 samples/s +Step 12530/44000 | Loss: 3.1248 | LR: 4.14e-04 | 400.0ms/step | 50.0 samples/s +Step 12540/44000 | Loss: 3.1125 | LR: 4.14e-04 | 399.9ms/step | 50.0 samples/s +Step 12550/44000 | Loss: 3.1285 | LR: 4.14e-04 | 398.2ms/step | 50.2 samples/s +Step 12560/44000 | Loss: 3.1399 | LR: 4.14e-04 | 399.0ms/step | 50.1 samples/s +Step 12570/44000 | Loss: 3.0421 | LR: 4.13e-04 | 398.6ms/step | 50.2 samples/s +Step 12580/44000 | Loss: 3.1392 | LR: 4.13e-04 | 397.7ms/step | 50.3 samples/s +Step 12590/44000 | Loss: 3.1037 | LR: 4.13e-04 | 398.5ms/step | 50.2 samples/s +Step 12600/44000 | Loss: 3.0802 | LR: 4.13e-04 | 399.0ms/step | 50.1 samples/s +Step 12610/44000 | Loss: 3.1211 | LR: 4.13e-04 | 397.7ms/step | 50.3 samples/s +Step 12620/44000 | Loss: 3.0730 | LR: 4.13e-04 | 397.6ms/step | 50.3 samples/s +Step 12630/44000 | Loss: 3.1400 | LR: 4.13e-04 | 399.8ms/step | 50.0 samples/s +Step 12640/44000 | Loss: 3.1098 | LR: 4.12e-04 | 400.2ms/step | 50.0 samples/s +Step 12650/44000 | Loss: 3.1263 | LR: 4.12e-04 | 399.9ms/step | 50.0 samples/s +Step 12660/44000 | Loss: 3.1222 | LR: 4.12e-04 | 400.6ms/step | 49.9 samples/s +Step 12670/44000 | Loss: 3.1132 | LR: 4.12e-04 | 401.5ms/step | 49.8 samples/s +Step 12680/44000 | Loss: 3.1055 | LR: 4.12e-04 | 399.9ms/step | 50.0 samples/s +Step 12690/44000 | Loss: 3.1179 | LR: 4.12e-04 | 400.8ms/step | 49.9 samples/s +Step 12700/44000 | Loss: 3.0717 | LR: 4.12e-04 | 400.3ms/step | 50.0 samples/s +Step 12710/44000 | Loss: 3.0775 | LR: 4.11e-04 | 401.3ms/step | 49.8 samples/s +Step 12720/44000 | Loss: 3.0751 | LR: 4.11e-04 | 399.6ms/step | 50.0 samples/s +Step 12730/44000 | Loss: 3.1128 | LR: 4.11e-04 | 400.3ms/step | 50.0 samples/s +Step 12740/44000 | Loss: 3.0832 | LR: 4.11e-04 | 399.8ms/step | 50.0 samples/s +Step 12750/44000 | Loss: 3.1094 | LR: 4.11e-04 | 399.9ms/step | 50.0 samples/s +Validation at step 12750: Loss = 3.0941, Perplexity = 22.07 +Step 12760/44000 | Loss: 3.0291 | LR: 4.11e-04 | 565.9ms/step | 35.3 samples/s +Step 12770/44000 | Loss: 3.0456 | LR: 4.11e-04 | 400.1ms/step | 50.0 samples/s +Step 12780/44000 | Loss: 3.1789 | LR: 4.10e-04 | 400.4ms/step | 50.0 samples/s +Step 12790/44000 | Loss: 3.1012 | LR: 4.10e-04 | 399.5ms/step | 50.1 samples/s +Step 12800/44000 | Loss: 3.0946 | LR: 4.10e-04 | 400.2ms/step | 50.0 samples/s +Step 12810/44000 | Loss: 3.1430 | LR: 4.10e-04 | 399.9ms/step | 50.0 samples/s +Step 12820/44000 | Loss: 3.0854 | LR: 4.10e-04 | 399.9ms/step | 50.0 samples/s +Step 12830/44000 | Loss: 3.0689 | LR: 4.10e-04 | 401.9ms/step | 49.8 samples/s +Step 12840/44000 | Loss: 3.0400 | LR: 4.10e-04 | 400.0ms/step | 50.0 samples/s +Step 12850/44000 | Loss: 3.0994 | LR: 4.09e-04 | 400.0ms/step | 50.0 samples/s +Step 12860/44000 | Loss: 3.0938 | LR: 4.09e-04 | 400.7ms/step | 49.9 samples/s +Step 12870/44000 | Loss: 3.0963 | LR: 4.09e-04 | 401.3ms/step | 49.8 samples/s +Step 12880/44000 | Loss: 3.1170 | LR: 4.09e-04 | 400.8ms/step | 49.9 samples/s +Step 12890/44000 | Loss: 3.0632 | LR: 4.09e-04 | 401.4ms/step | 49.8 samples/s +Step 12900/44000 | Loss: 3.0973 | LR: 4.09e-04 | 400.1ms/step | 50.0 samples/s +Step 12910/44000 | Loss: 3.0469 | LR: 4.09e-04 | 401.0ms/step | 49.9 samples/s +Step 12920/44000 | Loss: 3.1265 | LR: 4.09e-04 | 400.6ms/step | 49.9 samples/s +Step 12930/44000 | Loss: 3.1117 | LR: 4.08e-04 | 400.1ms/step | 50.0 samples/s +Step 12940/44000 | Loss: 3.0898 | LR: 4.08e-04 | 400.3ms/step | 50.0 samples/s +Step 12950/44000 | Loss: 3.0754 | LR: 4.08e-04 | 400.7ms/step | 49.9 samples/s +Step 12960/44000 | Loss: 3.1326 | LR: 4.08e-04 | 401.3ms/step | 49.8 samples/s +Step 12970/44000 | Loss: 3.0773 | LR: 4.08e-04 | 400.0ms/step | 50.0 samples/s +Step 12980/44000 | Loss: 3.0541 | LR: 4.08e-04 | 401.3ms/step | 49.8 samples/s +Step 12990/44000 | Loss: 3.0561 | LR: 4.08e-04 | 401.3ms/step | 49.8 samples/s +Step 13000/44000 | Loss: 3.0975 | LR: 4.07e-04 | 401.3ms/step | 49.8 samples/s +Validation at step 13000: Loss = 3.0900, Perplexity = 21.98 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-13000 +Step 13010/44000 | Loss: 3.0730 | LR: 4.07e-04 | 888.2ms/step | 22.5 samples/s +Step 13020/44000 | Loss: 3.0640 | LR: 4.07e-04 | 402.5ms/step | 49.7 samples/s +Step 13030/44000 | Loss: 3.0722 | LR: 4.07e-04 | 402.2ms/step | 49.7 samples/s +Step 13040/44000 | Loss: 3.0705 | LR: 4.07e-04 | 400.8ms/step | 49.9 samples/s +Step 13050/44000 | Loss: 3.0951 | LR: 4.07e-04 | 401.6ms/step | 49.8 samples/s +Step 13060/44000 | Loss: 3.0741 | LR: 4.07e-04 | 400.5ms/step | 49.9 samples/s +Step 13070/44000 | Loss: 3.1126 | LR: 4.06e-04 | 401.5ms/step | 49.8 samples/s +Step 13080/44000 | Loss: 3.0682 | LR: 4.06e-04 | 402.0ms/step | 49.8 samples/s +Step 13090/44000 | Loss: 3.0625 | LR: 4.06e-04 | 402.4ms/step | 49.7 samples/s +Step 13100/44000 | Loss: 3.0780 | LR: 4.06e-04 | 400.8ms/step | 49.9 samples/s +Step 13110/44000 | Loss: 3.0503 | LR: 4.06e-04 | 399.4ms/step | 50.1 samples/s +Step 13120/44000 | Loss: 3.1059 | LR: 4.06e-04 | 399.8ms/step | 50.0 samples/s +Step 13130/44000 | Loss: 3.0532 | LR: 4.06e-04 | 400.0ms/step | 50.0 samples/s +Step 13140/44000 | Loss: 3.0950 | LR: 4.05e-04 | 400.4ms/step | 50.0 samples/s +Step 13150/44000 | Loss: 3.0864 | LR: 4.05e-04 | 400.1ms/step | 50.0 samples/s +Step 13160/44000 | Loss: 3.0596 | LR: 4.05e-04 | 400.4ms/step | 49.9 samples/s +Step 13170/44000 | Loss: 3.0226 | LR: 4.05e-04 | 401.3ms/step | 49.8 samples/s +Step 13180/44000 | Loss: 3.0742 | LR: 4.05e-04 | 402.1ms/step | 49.7 samples/s +Step 13190/44000 | Loss: 3.0793 | LR: 4.05e-04 | 401.6ms/step | 49.8 samples/s +Step 13200/44000 | Loss: 3.0553 | LR: 4.05e-04 | 400.7ms/step | 49.9 samples/s +Step 13210/44000 | Loss: 3.0931 | LR: 4.04e-04 | 401.6ms/step | 49.8 samples/s +Step 13220/44000 | Loss: 3.1104 | LR: 4.04e-04 | 401.7ms/step | 49.8 samples/s +Step 13230/44000 | Loss: 3.0550 | LR: 4.04e-04 | 401.4ms/step | 49.8 samples/s +Step 13240/44000 | Loss: 3.0624 | LR: 4.04e-04 | 401.1ms/step | 49.9 samples/s +Step 13250/44000 | Loss: 3.0719 | LR: 4.04e-04 | 401.7ms/step | 49.8 samples/s +Validation at step 13250: Loss = 3.0870, Perplexity = 21.91 +Step 13260/44000 | Loss: 3.0427 | LR: 4.04e-04 | 566.0ms/step | 35.3 samples/s +Step 13270/44000 | Loss: 3.0999 | LR: 4.04e-04 | 400.0ms/step | 50.0 samples/s +Step 13280/44000 | Loss: 3.0730 | LR: 4.03e-04 | 399.9ms/step | 50.0 samples/s +Step 13290/44000 | Loss: 3.0240 | LR: 4.03e-04 | 400.4ms/step | 49.9 samples/s +Step 13300/44000 | Loss: 3.1394 | LR: 4.03e-04 | 399.8ms/step | 50.0 samples/s +Step 13310/44000 | Loss: 3.0392 | LR: 4.03e-04 | 400.0ms/step | 50.0 samples/s +Step 13320/44000 | Loss: 3.1011 | LR: 4.03e-04 | 399.9ms/step | 50.0 samples/s +Step 13330/44000 | Loss: 3.0665 | LR: 4.03e-04 | 400.0ms/step | 50.0 samples/s +Step 13340/44000 | Loss: 3.1241 | LR: 4.03e-04 | 400.7ms/step | 49.9 samples/s +Step 13350/44000 | Loss: 3.0762 | LR: 4.02e-04 | 399.2ms/step | 50.1 samples/s +Step 13360/44000 | Loss: 3.1151 | LR: 4.02e-04 | 399.9ms/step | 50.0 samples/s +Step 13370/44000 | Loss: 3.1493 | LR: 4.02e-04 | 400.2ms/step | 50.0 samples/s +Step 13380/44000 | Loss: 3.1071 | LR: 4.02e-04 | 400.1ms/step | 50.0 samples/s diff --git a/epfl_final/train_logs/p1_scheduled_h1004_20260822/relaunch.log b/epfl_final/train_logs/p1_scheduled_h1004_20260822/relaunch.log new file mode 100644 index 0000000000000000000000000000000000000000..8c94825865a315e374bf8aea010b03b0f356b4e0 --- /dev/null +++ b/epfl_final/train_logs/p1_scheduled_h1004_20260822/relaunch.log @@ -0,0 +1,3534 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-13000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-13000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 45, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "P1 DDP4 365m resampled 8k nrb35 seed45", + "wandb_tags": [ + "p1", + "structured-control", + "qwen3", + "365m", + "8k", + "resampled", + "nrb35", + "seed45", + "ddp4", + "h1004", + "scheduled" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 35, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-13000 +Checkpoint activations: FalseCheckpoint activations: FalseCheckpoint activations: FalseCheckpoint activations: False + + + + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 35 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 8192] -- -- True +├─Qwen3Model: 1-1 -- [1, 8192, 768] -- True +│ └─Embedding: 2-1 [1, 8192] [1, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [1, 8192, 768] [1, 8192, 768] 768 True +├─Linear: 1-2 [1, 8192, 768] [1, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 475.65 +================================================================================================================================================================ +Input size (MB): 0.07 +Forward/backward pass size (MB): 15598.62 +Params size (MB): 951.29 +Estimated Total Size (MB): 16549.97 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... + +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) + + +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 1 +World size (GPUs): 4 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-13000... +Preserving checkpoint LR state for exact resume: current=[0.0004073888929652048], schedule_base=[0.0005] +Resumed from checkpoint at step 13000 (epoch 0, micro_step 65000) +Resuming training from step 13000 +Resuming wandb run 9nppzrru +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_161907-9nppzrru +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run P1 DDP4 365m resampled 8k nrb35 seed45 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/9nppzrru +Starting training... +Validation at step 13000: Loss = 3.0903, Perplexity = 21.98 +wandb: WARNING Tried to log to step 13000 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 13001: sparse_stochastic (configured sparse attention) +Step 13010/44000 | Loss: 3.0725 | LR: 4.07e-04 | 19959.2ms/step | 1.0 samples/s +wandb: WARNING Tried to log to step 13010 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13020/44000 | Loss: 3.0626 | LR: 4.07e-04 | 419.9ms/step | 47.6 samples/s +Step 13030/44000 | Loss: 3.0736 | LR: 4.07e-04 | 425.5ms/step | 47.0 samples/s +wandb: WARNING Tried to log to step 13020 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13030 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13040/44000 | Loss: 3.0707 | LR: 4.07e-04 | 420.3ms/step | 47.6 samples/s +Step 13050/44000 | Loss: 3.0940 | LR: 4.07e-04 | 421.1ms/step | 47.5 samples/s +Step 13060/44000 | Loss: 3.0739 | LR: 4.07e-04 | 420.5ms/step | 47.6 samples/s +wandb: WARNING Tried to log to step 13040 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13050 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13060 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13070/44000 | Loss: 3.1124 | LR: 4.06e-04 | 421.6ms/step | 47.4 samples/s +Step 13080/44000 | Loss: 3.0686 | LR: 4.06e-04 | 423.8ms/step | 47.2 samples/s +wandb: WARNING Tried to log to step 13070 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13080 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13090/44000 | Loss: 3.0635 | LR: 4.06e-04 | 423.0ms/step | 47.3 samples/s +Step 13100/44000 | Loss: 3.0792 | LR: 4.06e-04 | 422.8ms/step | 47.3 samples/s +wandb: WARNING Tried to log to step 13090 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13100 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13110/44000 | Loss: 3.0476 | LR: 4.06e-04 | 426.2ms/step | 46.9 samples/s +Step 13120/44000 | Loss: 3.1070 | LR: 4.06e-04 | 427.4ms/step | 46.8 samples/s +Step 13130/44000 | Loss: 3.0536 | LR: 4.06e-04 | 423.0ms/step | 47.3 samples/s +wandb: WARNING Tried to log to step 13110 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13120 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13130 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13140/44000 | Loss: 3.0952 | LR: 4.05e-04 | 420.8ms/step | 47.5 samples/s +Step 13150/44000 | Loss: 3.0857 | LR: 4.05e-04 | 418.9ms/step | 47.7 samples/s +wandb: WARNING Tried to log to step 13140 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13150 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13160/44000 | Loss: 3.0598 | LR: 4.05e-04 | 419.7ms/step | 47.7 samples/s +Step 13170/44000 | Loss: 3.0226 | LR: 4.05e-04 | 419.5ms/step | 47.7 samples/s +wandb: WARNING Tried to log to step 13160 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13170 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13180/44000 | Loss: 3.0747 | LR: 4.05e-04 | 420.9ms/step | 47.5 samples/s +Step 13190/44000 | Loss: 3.0809 | LR: 4.05e-04 | 419.2ms/step | 47.7 samples/s +Step 13200/44000 | Loss: 3.0559 | LR: 4.05e-04 | 417.9ms/step | 47.9 samples/s +wandb: WARNING Tried to log to step 13180 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13190 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13200 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13210/44000 | Loss: 3.0930 | LR: 4.04e-04 | 418.7ms/step | 47.8 samples/s +Step 13220/44000 | Loss: 3.1097 | LR: 4.04e-04 | 419.8ms/step | 47.6 samples/s +wandb: WARNING Tried to log to step 13210 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13220 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13230/44000 | Loss: 3.0560 | LR: 4.04e-04 | 418.4ms/step | 47.8 samples/s +Step 13240/44000 | Loss: 3.0613 | LR: 4.04e-04 | 422.3ms/step | 47.4 samples/s +Step 13250/44000 | Loss: 3.0725 | LR: 4.04e-04 | 421.8ms/step | 47.4 samples/s +wandb: WARNING Tried to log to step 13230 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13240 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13250 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Validation at step 13250: Loss = 3.0873, Perplexity = 21.92 +Step 13260/44000 | Loss: 3.0437 | LR: 4.04e-04 | 601.7ms/step | 33.2 samples/s +wandb: WARNING Tried to log to step 13260 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13270/44000 | Loss: 3.1001 | LR: 4.04e-04 | 420.3ms/step | 47.6 samples/s +Step 13280/44000 | Loss: 3.0719 | LR: 4.03e-04 | 419.4ms/step | 47.7 samples/s +Step 13290/44000 | Loss: 3.0250 | LR: 4.03e-04 | 419.3ms/step | 47.7 samples/s +wandb: WARNING Tried to log to step 13270 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13280 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13290 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13300/44000 | Loss: 3.1383 | LR: 4.03e-04 | 421.1ms/step | 47.5 samples/s +Step 13310/44000 | Loss: 3.0386 | LR: 4.03e-04 | 421.8ms/step | 47.4 samples/s +wandb: WARNING Tried to log to step 13300 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13310 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13320/44000 | Loss: 3.1014 | LR: 4.03e-04 | 418.1ms/step | 47.8 samples/s +Step 13330/44000 | Loss: 3.0670 | LR: 4.03e-04 | 429.3ms/step | 46.6 samples/s +Step 13340/44000 | Loss: 3.1238 | LR: 4.03e-04 | 429.2ms/step | 46.6 samples/s +wandb: WARNING Tried to log to step 13320 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13330 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 13340 that is less than the current step 13341. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13350/44000 | Loss: 3.0775 | LR: 4.02e-04 | 431.8ms/step | 46.3 samples/s +Step 13360/44000 | Loss: 3.1151 | LR: 4.02e-04 | 426.9ms/step | 46.8 samples/s +Step 13370/44000 | Loss: 3.1491 | LR: 4.02e-04 | 424.5ms/step | 47.1 samples/s +Step 13380/44000 | Loss: 3.1061 | LR: 4.02e-04 | 424.5ms/step | 47.1 samples/s +Step 13390/44000 | Loss: 3.0798 | LR: 4.02e-04 | 425.1ms/step | 47.1 samples/s +Step 13400/44000 | Loss: 3.1154 | LR: 4.02e-04 | 425.3ms/step | 47.0 samples/s +Step 13410/44000 | Loss: 3.0953 | LR: 4.02e-04 | 425.0ms/step | 47.1 samples/s +Step 13420/44000 | Loss: 3.0612 | LR: 4.01e-04 | 426.4ms/step | 46.9 samples/s +Step 13430/44000 | Loss: 3.1598 | LR: 4.01e-04 | 423.8ms/step | 47.2 samples/s +Step 13440/44000 | Loss: 3.0708 | LR: 4.01e-04 | 422.7ms/step | 47.3 samples/s +Step 13450/44000 | Loss: 3.0927 | LR: 4.01e-04 | 423.7ms/step | 47.2 samples/s +Step 13460/44000 | Loss: 3.0784 | LR: 4.01e-04 | 422.9ms/step | 47.3 samples/s +Step 13470/44000 | Loss: 3.0895 | LR: 4.01e-04 | 422.8ms/step | 47.3 samples/s +Step 13480/44000 | Loss: 3.0838 | LR: 4.01e-04 | 422.8ms/step | 47.3 samples/s +Step 13490/44000 | Loss: 3.0684 | LR: 4.00e-04 | 423.8ms/step | 47.2 samples/s +Step 13500/44000 | Loss: 3.0401 | LR: 4.00e-04 | 424.2ms/step | 47.2 samples/s +Validation at step 13500: Loss = 3.0837, Perplexity = 21.84 +Step 13510/44000 | Loss: 3.0998 | LR: 4.00e-04 | 606.7ms/step | 33.0 samples/s +Step 13520/44000 | Loss: 3.1058 | LR: 4.00e-04 | 422.7ms/step | 47.3 samples/s +Step 13530/44000 | Loss: 3.1082 | LR: 4.00e-04 | 422.6ms/step | 47.3 samples/s +Step 13540/44000 | Loss: 3.0481 | LR: 4.00e-04 | 421.2ms/step | 47.5 samples/s +Step 13550/44000 | Loss: 3.1096 | LR: 4.00e-04 | 422.3ms/step | 47.4 samples/s +Step 13560/44000 | Loss: 3.0563 | LR: 3.99e-04 | 422.2ms/step | 47.4 samples/s +Step 13570/44000 | Loss: 3.1224 | LR: 3.99e-04 | 422.2ms/step | 47.4 samples/s +Step 13580/44000 | Loss: 3.0534 | LR: 3.99e-04 | 421.4ms/step | 47.5 samples/s +Step 13590/44000 | Loss: 3.0895 | LR: 3.99e-04 | 421.5ms/step | 47.5 samples/s +Step 13600/44000 | Loss: 3.0944 | LR: 3.99e-04 | 423.4ms/step | 47.2 samples/s +Step 13610/44000 | Loss: 3.0983 | LR: 3.99e-04 | 421.0ms/step | 47.5 samples/s +Step 13620/44000 | Loss: 3.1080 | LR: 3.98e-04 | 420.4ms/step | 47.6 samples/s +Step 13630/44000 | Loss: 3.0783 | LR: 3.98e-04 | 419.8ms/step | 47.6 samples/s +Step 13640/44000 | Loss: 3.0737 | LR: 3.98e-04 | 418.6ms/step | 47.8 samples/s +Step 13650/44000 | Loss: 3.0972 | LR: 3.98e-04 | 419.1ms/step | 47.7 samples/s +Step 13660/44000 | Loss: 3.0766 | LR: 3.98e-04 | 417.7ms/step | 47.9 samples/s +Step 13670/44000 | Loss: 3.1102 | LR: 3.98e-04 | 420.5ms/step | 47.6 samples/s +Step 13680/44000 | Loss: 3.0683 | LR: 3.98e-04 | 421.5ms/step | 47.4 samples/s +Step 13690/44000 | Loss: 3.0437 | LR: 3.97e-04 | 421.1ms/step | 47.5 samples/s +Step 13700/44000 | Loss: 3.1530 | LR: 3.97e-04 | 421.6ms/step | 47.4 samples/s +Step 13710/44000 | Loss: 3.0272 | LR: 3.97e-04 | 421.0ms/step | 47.5 samples/s +Step 13720/44000 | Loss: 3.0678 | LR: 3.97e-04 | 421.4ms/step | 47.5 samples/s +Step 13730/44000 | Loss: 3.0881 | LR: 3.97e-04 | 420.1ms/step | 47.6 samples/s +Step 13740/44000 | Loss: 3.0682 | LR: 3.97e-04 | 420.2ms/step | 47.6 samples/s +Step 13750/44000 | Loss: 3.1206 | LR: 3.97e-04 | 420.1ms/step | 47.6 samples/s +Validation at step 13750: Loss = 3.0797, Perplexity = 21.75 +Step 13760/44000 | Loss: 3.0922 | LR: 3.96e-04 | 598.0ms/step | 33.4 samples/s +Step 13770/44000 | Loss: 3.0854 | LR: 3.96e-04 | 421.2ms/step | 47.5 samples/s +Step 13780/44000 | Loss: 3.0888 | LR: 3.96e-04 | 419.4ms/step | 47.7 samples/s +Step 13790/44000 | Loss: 3.0648 | LR: 3.96e-04 | 420.3ms/step | 47.6 samples/s +Step 13800/44000 | Loss: 3.0663 | LR: 3.96e-04 | 421.3ms/step | 47.5 samples/s +Step 13810/44000 | Loss: 3.0824 | LR: 3.96e-04 | 421.0ms/step | 47.5 samples/s +Step 13820/44000 | Loss: 3.0756 | LR: 3.96e-04 | 420.2ms/step | 47.6 samples/s +Step 13830/44000 | Loss: 3.0123 | LR: 3.95e-04 | 420.3ms/step | 47.6 samples/s +Step 13840/44000 | Loss: 3.0628 | LR: 3.95e-04 | 419.3ms/step | 47.7 samples/s +Step 13850/44000 | Loss: 3.1062 | LR: 3.95e-04 | 420.6ms/step | 47.5 samples/s +Step 13860/44000 | Loss: 3.0847 | LR: 3.95e-04 | 420.1ms/step | 47.6 samples/s +Step 13870/44000 | Loss: 3.1510 | LR: 3.95e-04 | 419.9ms/step | 47.6 samples/s +Step 13880/44000 | Loss: 3.1074 | LR: 3.95e-04 | 420.1ms/step | 47.6 samples/s +Step 13890/44000 | Loss: 3.0943 | LR: 3.95e-04 | 419.3ms/step | 47.7 samples/s +Step 13900/44000 | Loss: 3.1242 | LR: 3.94e-04 | 418.6ms/step | 47.8 samples/s +Step 13910/44000 | Loss: 3.0367 | LR: 3.94e-04 | 419.0ms/step | 47.7 samples/s +Step 13920/44000 | Loss: 3.0736 | LR: 3.94e-04 | 419.5ms/step | 47.7 samples/s +Step 13930/44000 | Loss: 3.0695 | LR: 3.94e-04 | 419.3ms/step | 47.7 samples/s +Step 13940/44000 | Loss: 3.1289 | LR: 3.94e-04 | 417.8ms/step | 47.9 samples/s +Step 13950/44000 | Loss: 3.0534 | LR: 3.94e-04 | 417.3ms/step | 47.9 samples/s +Step 13960/44000 | Loss: 3.1480 | LR: 3.93e-04 | 417.2ms/step | 47.9 samples/s +Step 13970/44000 | Loss: 3.0734 | LR: 3.93e-04 | 417.2ms/step | 47.9 samples/s +Step 13980/44000 | Loss: 3.0770 | LR: 3.93e-04 | 417.3ms/step | 47.9 samples/s +Step 13990/44000 | Loss: 3.0844 | LR: 3.93e-04 | 417.5ms/step | 47.9 samples/s +Step 14000/44000 | Loss: 3.0463 | LR: 3.93e-04 | 417.9ms/step | 47.9 samples/s +Validation at step 14000: Loss = 3.0765, Perplexity = 21.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-14000 +Step 14010/44000 | Loss: 3.0844 | LR: 3.93e-04 | 917.4ms/step | 21.8 samples/s +Step 14020/44000 | Loss: 3.0220 | LR: 3.93e-04 | 417.0ms/step | 48.0 samples/s +Step 14030/44000 | Loss: 3.0877 | LR: 3.92e-04 | 416.9ms/step | 48.0 samples/s +Step 14040/44000 | Loss: 3.0448 | LR: 3.92e-04 | 416.1ms/step | 48.1 samples/s +Step 14050/44000 | Loss: 2.9705 | LR: 3.92e-04 | 417.9ms/step | 47.9 samples/s +Step 14060/44000 | Loss: 3.1163 | LR: 3.92e-04 | 417.6ms/step | 47.9 samples/s +Step 14070/44000 | Loss: 3.1038 | LR: 3.92e-04 | 417.0ms/step | 48.0 samples/s +Step 14080/44000 | Loss: 3.0551 | LR: 3.92e-04 | 417.7ms/step | 47.9 samples/s +Step 14090/44000 | Loss: 3.0982 | LR: 3.92e-04 | 417.5ms/step | 47.9 samples/s +Step 14100/44000 | Loss: 3.0895 | LR: 3.91e-04 | 417.3ms/step | 47.9 samples/s +Step 14110/44000 | Loss: 3.1433 | LR: 3.91e-04 | 417.0ms/step | 48.0 samples/s +Step 14120/44000 | Loss: 3.0497 | LR: 3.91e-04 | 417.1ms/step | 48.0 samples/s +Step 14130/44000 | Loss: 3.0792 | LR: 3.91e-04 | 418.2ms/step | 47.8 samples/s +Step 14140/44000 | Loss: 3.0351 | LR: 3.91e-04 | 417.3ms/step | 47.9 samples/s +Step 14150/44000 | Loss: 3.0772 | LR: 3.91e-04 | 416.9ms/step | 48.0 samples/s +Step 14160/44000 | Loss: 3.0904 | LR: 3.90e-04 | 415.9ms/step | 48.1 samples/s +Step 14170/44000 | Loss: 3.0777 | LR: 3.90e-04 | 416.9ms/step | 48.0 samples/s +Step 14180/44000 | Loss: 3.0804 | LR: 3.90e-04 | 417.0ms/step | 48.0 samples/s +Step 14190/44000 | Loss: 3.0305 | LR: 3.90e-04 | 416.0ms/step | 48.1 samples/s +Step 14200/44000 | Loss: 3.1171 | LR: 3.90e-04 | 417.4ms/step | 47.9 samples/s +Step 14210/44000 | Loss: 3.1247 | LR: 3.90e-04 | 418.5ms/step | 47.8 samples/s +Step 14220/44000 | Loss: 3.0460 | LR: 3.90e-04 | 416.8ms/step | 48.0 samples/s +Step 14230/44000 | Loss: 3.0948 | LR: 3.89e-04 | 414.9ms/step | 48.2 samples/s +Step 14240/44000 | Loss: 3.0409 | LR: 3.89e-04 | 417.2ms/step | 47.9 samples/s +Step 14250/44000 | Loss: 3.0333 | LR: 3.89e-04 | 416.6ms/step | 48.0 samples/s +Validation at step 14250: Loss = 3.0750, Perplexity = 21.65 +Step 14260/44000 | Loss: 3.0684 | LR: 3.89e-04 | 592.8ms/step | 33.7 samples/s +Step 14270/44000 | Loss: 3.0035 | LR: 3.89e-04 | 416.7ms/step | 48.0 samples/s +Step 14280/44000 | Loss: 3.0334 | LR: 3.89e-04 | 415.7ms/step | 48.1 samples/s +Step 14290/44000 | Loss: 3.0866 | LR: 3.89e-04 | 417.4ms/step | 47.9 samples/s +Step 14300/44000 | Loss: 3.0492 | LR: 3.88e-04 | 416.0ms/step | 48.1 samples/s +Step 14310/44000 | Loss: 3.0951 | LR: 3.88e-04 | 418.0ms/step | 47.9 samples/s +Step 14320/44000 | Loss: 3.0898 | LR: 3.88e-04 | 416.3ms/step | 48.0 samples/s +Step 14330/44000 | Loss: 3.0636 | LR: 3.88e-04 | 417.7ms/step | 47.9 samples/s +Step 14340/44000 | Loss: 3.0321 | LR: 3.88e-04 | 416.5ms/step | 48.0 samples/s +Step 14350/44000 | Loss: 3.0387 | LR: 3.88e-04 | 417.3ms/step | 47.9 samples/s +Step 14360/44000 | Loss: 3.0505 | LR: 3.87e-04 | 416.6ms/step | 48.0 samples/s +Step 14370/44000 | Loss: 3.0960 | LR: 3.87e-04 | 416.5ms/step | 48.0 samples/s +Step 14380/44000 | Loss: 3.0222 | LR: 3.87e-04 | 417.2ms/step | 47.9 samples/s +Step 14390/44000 | Loss: 3.0287 | LR: 3.87e-04 | 418.3ms/step | 47.8 samples/s +Step 14400/44000 | Loss: 3.1118 | LR: 3.87e-04 | 414.3ms/step | 48.3 samples/s +Step 14410/44000 | Loss: 3.0761 | LR: 3.87e-04 | 414.7ms/step | 48.2 samples/s +Step 14420/44000 | Loss: 3.0668 | LR: 3.87e-04 | 412.8ms/step | 48.5 samples/s +Step 14430/44000 | Loss: 3.0366 | LR: 3.86e-04 | 412.9ms/step | 48.4 samples/s +Step 14440/44000 | Loss: 3.0629 | LR: 3.86e-04 | 414.6ms/step | 48.2 samples/s +Step 14450/44000 | Loss: 3.1117 | LR: 3.86e-04 | 413.2ms/step | 48.4 samples/s +Step 14460/44000 | Loss: 3.1148 | LR: 3.86e-04 | 413.9ms/step | 48.3 samples/s +Step 14470/44000 | Loss: 3.0770 | LR: 3.86e-04 | 412.5ms/step | 48.5 samples/s +Step 14480/44000 | Loss: 3.0871 | LR: 3.86e-04 | 413.5ms/step | 48.4 samples/s +Step 14490/44000 | Loss: 3.0910 | LR: 3.85e-04 | 412.5ms/step | 48.5 samples/s +Step 14500/44000 | Loss: 3.1005 | LR: 3.85e-04 | 412.1ms/step | 48.5 samples/s +Validation at step 14500: Loss = 3.0706, Perplexity = 21.55 +Step 14510/44000 | Loss: 3.0937 | LR: 3.85e-04 | 590.0ms/step | 33.9 samples/s +Step 14520/44000 | Loss: 3.1643 | LR: 3.85e-04 | 413.4ms/step | 48.4 samples/s +Step 14530/44000 | Loss: 3.0594 | LR: 3.85e-04 | 412.7ms/step | 48.5 samples/s +Step 14540/44000 | Loss: 2.9849 | LR: 3.85e-04 | 412.4ms/step | 48.5 samples/s +Step 14550/44000 | Loss: 3.0168 | LR: 3.85e-04 | 412.0ms/step | 48.5 samples/s +Step 14560/44000 | Loss: 3.1019 | LR: 3.84e-04 | 413.3ms/step | 48.4 samples/s +Step 14570/44000 | Loss: 3.0645 | LR: 3.84e-04 | 412.3ms/step | 48.5 samples/s +Step 14580/44000 | Loss: 3.0985 | LR: 3.84e-04 | 412.8ms/step | 48.4 samples/s +Step 14590/44000 | Loss: 3.0511 | LR: 3.84e-04 | 412.3ms/step | 48.5 samples/s +Step 14600/44000 | Loss: 3.1117 | LR: 3.84e-04 | 412.1ms/step | 48.5 samples/s +Step 14610/44000 | Loss: 3.1059 | LR: 3.84e-04 | 412.2ms/step | 48.5 samples/s +Step 14620/44000 | Loss: 3.0257 | LR: 3.83e-04 | 413.7ms/step | 48.3 samples/s +Step 14630/44000 | Loss: 3.0241 | LR: 3.83e-04 | 412.1ms/step | 48.5 samples/s +Step 14640/44000 | Loss: 3.0587 | LR: 3.83e-04 | 412.9ms/step | 48.4 samples/s +Step 14650/44000 | Loss: 3.0775 | LR: 3.83e-04 | 414.4ms/step | 48.3 samples/s +Step 14660/44000 | Loss: 3.0613 | LR: 3.83e-04 | 414.0ms/step | 48.3 samples/s +Step 14670/44000 | Loss: 3.1241 | LR: 3.83e-04 | 415.0ms/step | 48.2 samples/s +Step 14680/44000 | Loss: 3.0519 | LR: 3.83e-04 | 414.5ms/step | 48.3 samples/s +Step 14690/44000 | Loss: 3.0983 | LR: 3.82e-04 | 414.1ms/step | 48.3 samples/s +Step 14700/44000 | Loss: 3.0343 | LR: 3.82e-04 | 415.8ms/step | 48.1 samples/s +Step 14710/44000 | Loss: 3.0604 | LR: 3.82e-04 | 414.1ms/step | 48.3 samples/s +Step 14720/44000 | Loss: 3.1016 | LR: 3.82e-04 | 413.5ms/step | 48.4 samples/s +Step 14730/44000 | Loss: 3.0996 | LR: 3.82e-04 | 413.8ms/step | 48.3 samples/s +Step 14740/44000 | Loss: 3.0748 | LR: 3.82e-04 | 413.8ms/step | 48.3 samples/s +Step 14750/44000 | Loss: 3.0442 | LR: 3.81e-04 | 412.9ms/step | 48.4 samples/s +Validation at step 14750: Loss = 3.0657, Perplexity = 21.45 +Step 14760/44000 | Loss: 3.0658 | LR: 3.81e-04 | 587.5ms/step | 34.0 samples/s +Step 14770/44000 | Loss: 3.0324 | LR: 3.81e-04 | 413.7ms/step | 48.3 samples/s +Step 14780/44000 | Loss: 3.0746 | LR: 3.81e-04 | 411.7ms/step | 48.6 samples/s +Step 14790/44000 | Loss: 3.0517 | LR: 3.81e-04 | 411.3ms/step | 48.6 samples/s +Step 14800/44000 | Loss: 3.0402 | LR: 3.81e-04 | 410.5ms/step | 48.7 samples/s +Step 14810/44000 | Loss: 3.1190 | LR: 3.81e-04 | 413.9ms/step | 48.3 samples/s +Step 14820/44000 | Loss: 3.0578 | LR: 3.80e-04 | 410.3ms/step | 48.7 samples/s +Step 14830/44000 | Loss: 3.0679 | LR: 3.80e-04 | 409.8ms/step | 48.8 samples/s +Step 14840/44000 | Loss: 3.1319 | LR: 3.80e-04 | 411.8ms/step | 48.6 samples/s +Step 14850/44000 | Loss: 3.0088 | LR: 3.80e-04 | 412.3ms/step | 48.5 samples/s +Step 14860/44000 | Loss: 3.0517 | LR: 3.80e-04 | 415.0ms/step | 48.2 samples/s +Step 14870/44000 | Loss: 3.0342 | LR: 3.80e-04 | 416.5ms/step | 48.0 samples/s +Step 14880/44000 | Loss: 3.0939 | LR: 3.79e-04 | 415.4ms/step | 48.1 samples/s +Step 14890/44000 | Loss: 3.0178 | LR: 3.79e-04 | 413.4ms/step | 48.4 samples/s +Step 14900/44000 | Loss: 3.0369 | LR: 3.79e-04 | 412.0ms/step | 48.5 samples/s +Step 14910/44000 | Loss: 3.0591 | LR: 3.79e-04 | 414.0ms/step | 48.3 samples/s +Step 14920/44000 | Loss: 3.1261 | LR: 3.79e-04 | 410.0ms/step | 48.8 samples/s +Step 14930/44000 | Loss: 3.0176 | LR: 3.79e-04 | 410.3ms/step | 48.7 samples/s +Step 14940/44000 | Loss: 3.1127 | LR: 3.79e-04 | 410.7ms/step | 48.7 samples/s +Step 14950/44000 | Loss: 3.0891 | LR: 3.78e-04 | 410.0ms/step | 48.8 samples/s +Step 14960/44000 | Loss: 3.0892 | LR: 3.78e-04 | 411.4ms/step | 48.6 samples/s +Step 14970/44000 | Loss: 3.0993 | LR: 3.78e-04 | 409.8ms/step | 48.8 samples/s +Step 14980/44000 | Loss: 3.0173 | LR: 3.78e-04 | 410.9ms/step | 48.7 samples/s +Step 14990/44000 | Loss: 3.0701 | LR: 3.78e-04 | 412.4ms/step | 48.5 samples/s +Step 15000/44000 | Loss: 3.0899 | LR: 3.78e-04 | 410.4ms/step | 48.7 samples/s +Validation at step 15000: Loss = 3.0626, Perplexity = 21.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-15000 +Step 15010/44000 | Loss: 3.0803 | LR: 3.77e-04 | 890.9ms/step | 22.5 samples/s +Step 15020/44000 | Loss: 3.0767 | LR: 3.77e-04 | 412.4ms/step | 48.5 samples/s +Step 15030/44000 | Loss: 3.0592 | LR: 3.77e-04 | 411.3ms/step | 48.6 samples/s +Step 15040/44000 | Loss: 3.1086 | LR: 3.77e-04 | 412.7ms/step | 48.5 samples/s +Step 15050/44000 | Loss: 3.0110 | LR: 3.77e-04 | 411.9ms/step | 48.6 samples/s +Step 15060/44000 | Loss: 3.0873 | LR: 3.77e-04 | 412.4ms/step | 48.5 samples/s +Step 15070/44000 | Loss: 3.0527 | LR: 3.77e-04 | 409.9ms/step | 48.8 samples/s +Step 15080/44000 | Loss: 3.0443 | LR: 3.76e-04 | 412.5ms/step | 48.5 samples/s +Step 15090/44000 | Loss: 3.1188 | LR: 3.76e-04 | 410.0ms/step | 48.8 samples/s +Step 15100/44000 | Loss: 3.1085 | LR: 3.76e-04 | 412.3ms/step | 48.5 samples/s +Step 15110/44000 | Loss: 3.0540 | LR: 3.76e-04 | 409.8ms/step | 48.8 samples/s +Step 15120/44000 | Loss: 3.0767 | LR: 3.76e-04 | 409.8ms/step | 48.8 samples/s +Step 15130/44000 | Loss: 3.1079 | LR: 3.76e-04 | 410.1ms/step | 48.8 samples/s +Step 15140/44000 | Loss: 3.0099 | LR: 3.75e-04 | 411.2ms/step | 48.6 samples/s +Step 15150/44000 | Loss: 3.0458 | LR: 3.75e-04 | 409.8ms/step | 48.8 samples/s +Step 15160/44000 | Loss: 3.0920 | LR: 3.75e-04 | 409.8ms/step | 48.8 samples/s +Step 15170/44000 | Loss: 3.0408 | LR: 3.75e-04 | 411.2ms/step | 48.6 samples/s +Step 15180/44000 | Loss: 3.0343 | LR: 3.75e-04 | 409.8ms/step | 48.8 samples/s +Step 15190/44000 | Loss: 3.1238 | LR: 3.75e-04 | 410.8ms/step | 48.7 samples/s +Step 15200/44000 | Loss: 3.1095 | LR: 3.74e-04 | 409.2ms/step | 48.9 samples/s +Step 15210/44000 | Loss: 3.0725 | LR: 3.74e-04 | 410.3ms/step | 48.7 samples/s +Step 15220/44000 | Loss: 3.1173 | LR: 3.74e-04 | 412.1ms/step | 48.5 samples/s +Step 15230/44000 | Loss: 3.0728 | LR: 3.74e-04 | 411.3ms/step | 48.6 samples/s +Step 15240/44000 | Loss: 3.1217 | LR: 3.74e-04 | 411.6ms/step | 48.6 samples/s +Step 15250/44000 | Loss: 3.0313 | LR: 3.74e-04 | 411.2ms/step | 48.6 samples/s +Validation at step 15250: Loss = 3.0594, Perplexity = 21.32 +Step 15260/44000 | Loss: 3.0391 | LR: 3.74e-04 | 583.7ms/step | 34.3 samples/s +Step 15270/44000 | Loss: 3.0917 | LR: 3.73e-04 | 411.0ms/step | 48.7 samples/s +Step 15280/44000 | Loss: 3.1471 | LR: 3.73e-04 | 411.2ms/step | 48.6 samples/s +Step 15290/44000 | Loss: 3.0530 | LR: 3.73e-04 | 409.6ms/step | 48.8 samples/s +Step 15300/44000 | Loss: 3.0940 | LR: 3.73e-04 | 408.8ms/step | 48.9 samples/s +Step 15310/44000 | Loss: 3.0202 | LR: 3.73e-04 | 409.2ms/step | 48.9 samples/s +Step 15320/44000 | Loss: 3.0687 | LR: 3.73e-04 | 408.8ms/step | 48.9 samples/s +Step 15330/44000 | Loss: 3.0800 | LR: 3.72e-04 | 408.5ms/step | 49.0 samples/s +Step 15340/44000 | Loss: 3.0790 | LR: 3.72e-04 | 408.9ms/step | 48.9 samples/s +Step 15350/44000 | Loss: 2.9939 | LR: 3.72e-04 | 410.6ms/step | 48.7 samples/s +Step 15360/44000 | Loss: 3.0249 | LR: 3.72e-04 | 409.4ms/step | 48.9 samples/s +Step 15370/44000 | Loss: 3.0248 | LR: 3.72e-04 | 409.4ms/step | 48.8 samples/s +Step 15380/44000 | Loss: 3.0263 | LR: 3.72e-04 | 410.8ms/step | 48.7 samples/s +Step 15390/44000 | Loss: 3.0566 | LR: 3.71e-04 | 410.2ms/step | 48.8 samples/s +Step 15400/44000 | Loss: 3.1252 | LR: 3.71e-04 | 410.3ms/step | 48.7 samples/s +Step 15410/44000 | Loss: 2.9950 | LR: 3.71e-04 | 410.4ms/step | 48.7 samples/s +Step 15420/44000 | Loss: 3.0151 | LR: 3.71e-04 | 411.2ms/step | 48.6 samples/s +Step 15430/44000 | Loss: 3.1095 | LR: 3.71e-04 | 411.9ms/step | 48.6 samples/s +Step 15440/44000 | Loss: 2.9965 | LR: 3.71e-04 | 410.7ms/step | 48.7 samples/s +Step 15450/44000 | Loss: 3.0131 | LR: 3.71e-04 | 410.4ms/step | 48.7 samples/s +Step 15460/44000 | Loss: 3.1192 | LR: 3.70e-04 | 411.4ms/step | 48.6 samples/s +Step 15470/44000 | Loss: 3.1173 | LR: 3.70e-04 | 410.5ms/step | 48.7 samples/s +Step 15480/44000 | Loss: 3.0562 | LR: 3.70e-04 | 410.7ms/step | 48.7 samples/s +Step 15490/44000 | Loss: 3.0795 | LR: 3.70e-04 | 410.2ms/step | 48.8 samples/s +Step 15500/44000 | Loss: 3.0622 | LR: 3.70e-04 | 410.3ms/step | 48.7 samples/s +Validation at step 15500: Loss = 3.0562, Perplexity = 21.25 +Step 15510/44000 | Loss: 3.0416 | LR: 3.70e-04 | 584.3ms/step | 34.2 samples/s +Step 15520/44000 | Loss: 3.0522 | LR: 3.69e-04 | 410.2ms/step | 48.8 samples/s +Step 15530/44000 | Loss: 3.0416 | LR: 3.69e-04 | 410.2ms/step | 48.8 samples/s +Step 15540/44000 | Loss: 3.0695 | LR: 3.69e-04 | 408.7ms/step | 48.9 samples/s +Step 15550/44000 | Loss: 3.0642 | LR: 3.69e-04 | 410.4ms/step | 48.7 samples/s +Step 15560/44000 | Loss: 3.0896 | LR: 3.69e-04 | 411.6ms/step | 48.6 samples/s +Step 15570/44000 | Loss: 3.0262 | LR: 3.69e-04 | 409.3ms/step | 48.9 samples/s +Step 15580/44000 | Loss: 3.0189 | LR: 3.68e-04 | 409.8ms/step | 48.8 samples/s +Step 15590/44000 | Loss: 3.0778 | LR: 3.68e-04 | 409.2ms/step | 48.9 samples/s +Step 15600/44000 | Loss: 3.0541 | LR: 3.68e-04 | 409.6ms/step | 48.8 samples/s +Step 15610/44000 | Loss: 3.0255 | LR: 3.68e-04 | 409.4ms/step | 48.9 samples/s +Step 15620/44000 | Loss: 3.0465 | LR: 3.68e-04 | 408.9ms/step | 48.9 samples/s +Step 15630/44000 | Loss: 3.0545 | LR: 3.68e-04 | 409.8ms/step | 48.8 samples/s +Step 15640/44000 | Loss: 3.0947 | LR: 3.67e-04 | 409.9ms/step | 48.8 samples/s +Step 15650/44000 | Loss: 3.0213 | LR: 3.67e-04 | 411.6ms/step | 48.6 samples/s +Step 15660/44000 | Loss: 2.9838 | LR: 3.67e-04 | 410.3ms/step | 48.7 samples/s +Step 15670/44000 | Loss: 3.0629 | LR: 3.67e-04 | 410.5ms/step | 48.7 samples/s +Step 15680/44000 | Loss: 3.0478 | LR: 3.67e-04 | 409.8ms/step | 48.8 samples/s +Step 15690/44000 | Loss: 3.0811 | LR: 3.67e-04 | 410.3ms/step | 48.7 samples/s +Step 15700/44000 | Loss: 3.0526 | LR: 3.67e-04 | 409.7ms/step | 48.8 samples/s +Step 15710/44000 | Loss: 3.1096 | LR: 3.66e-04 | 409.7ms/step | 48.8 samples/s +Step 15720/44000 | Loss: 3.0901 | LR: 3.66e-04 | 409.4ms/step | 48.9 samples/s +Step 15730/44000 | Loss: 3.0600 | LR: 3.66e-04 | 408.9ms/step | 48.9 samples/s +Step 15740/44000 | Loss: 3.0908 | LR: 3.66e-04 | 409.1ms/step | 48.9 samples/s +Step 15750/44000 | Loss: 3.0192 | LR: 3.66e-04 | 409.1ms/step | 48.9 samples/s +Validation at step 15750: Loss = 3.0521, Perplexity = 21.16 +Step 15760/44000 | Loss: 3.0814 | LR: 3.66e-04 | 579.9ms/step | 34.5 samples/s +Step 15770/44000 | Loss: 3.0734 | LR: 3.65e-04 | 406.8ms/step | 49.2 samples/s +Step 15780/44000 | Loss: 3.0920 | LR: 3.65e-04 | 407.0ms/step | 49.1 samples/s +Step 15790/44000 | Loss: 2.9961 | LR: 3.65e-04 | 408.0ms/step | 49.0 samples/s +Step 15800/44000 | Loss: 3.0698 | LR: 3.65e-04 | 409.2ms/step | 48.9 samples/s +Step 15810/44000 | Loss: 2.9992 | LR: 3.65e-04 | 409.5ms/step | 48.8 samples/s +Step 15820/44000 | Loss: 3.0159 | LR: 3.65e-04 | 409.6ms/step | 48.8 samples/s +Step 15830/44000 | Loss: 2.9919 | LR: 3.64e-04 | 409.6ms/step | 48.8 samples/s +Step 15840/44000 | Loss: 3.0398 | LR: 3.64e-04 | 409.4ms/step | 48.9 samples/s +Step 15850/44000 | Loss: 3.0406 | LR: 3.64e-04 | 410.6ms/step | 48.7 samples/s +Step 15860/44000 | Loss: 3.0486 | LR: 3.64e-04 | 411.3ms/step | 48.6 samples/s +Step 15870/44000 | Loss: 3.0586 | LR: 3.64e-04 | 409.9ms/step | 48.8 samples/s +Step 15880/44000 | Loss: 3.0312 | LR: 3.64e-04 | 409.5ms/step | 48.8 samples/s +Step 15890/44000 | Loss: 3.0256 | LR: 3.63e-04 | 409.9ms/step | 48.8 samples/s +Step 15900/44000 | Loss: 3.0024 | LR: 3.63e-04 | 409.7ms/step | 48.8 samples/s +Step 15910/44000 | Loss: 3.0588 | LR: 3.63e-04 | 410.4ms/step | 48.7 samples/s +Step 15920/44000 | Loss: 3.0008 | LR: 3.63e-04 | 409.4ms/step | 48.9 samples/s +Step 15930/44000 | Loss: 3.0262 | LR: 3.63e-04 | 409.1ms/step | 48.9 samples/s +Step 15940/44000 | Loss: 3.0365 | LR: 3.63e-04 | 409.8ms/step | 48.8 samples/s +Step 15950/44000 | Loss: 3.0221 | LR: 3.62e-04 | 408.7ms/step | 48.9 samples/s +Step 15960/44000 | Loss: 3.0643 | LR: 3.62e-04 | 408.7ms/step | 48.9 samples/s +Step 15970/44000 | Loss: 3.0697 | LR: 3.62e-04 | 407.9ms/step | 49.0 samples/s +Step 15980/44000 | Loss: 3.0627 | LR: 3.62e-04 | 408.8ms/step | 48.9 samples/s +Step 15990/44000 | Loss: 3.0862 | LR: 3.62e-04 | 408.3ms/step | 49.0 samples/s +Step 16000/44000 | Loss: 3.0280 | LR: 3.62e-04 | 408.9ms/step | 48.9 samples/s +Validation at step 16000: Loss = 3.0502, Perplexity = 21.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-16000 +Step 16010/44000 | Loss: 3.0308 | LR: 3.62e-04 | 887.8ms/step | 22.5 samples/s +Step 16020/44000 | Loss: 3.0460 | LR: 3.61e-04 | 408.5ms/step | 49.0 samples/s +Step 16030/44000 | Loss: 3.0056 | LR: 3.61e-04 | 408.6ms/step | 48.9 samples/s +Step 16040/44000 | Loss: 3.0212 | LR: 3.61e-04 | 408.9ms/step | 48.9 samples/s +Step 16050/44000 | Loss: 3.0767 | LR: 3.61e-04 | 409.0ms/step | 48.9 samples/s +Step 16060/44000 | Loss: 3.0346 | LR: 3.61e-04 | 409.9ms/step | 48.8 samples/s +Step 16070/44000 | Loss: 3.1031 | LR: 3.61e-04 | 409.9ms/step | 48.8 samples/s +Step 16080/44000 | Loss: 3.1131 | LR: 3.60e-04 | 409.3ms/step | 48.9 samples/s +Step 16090/44000 | Loss: 2.9976 | LR: 3.60e-04 | 409.4ms/step | 48.9 samples/s +Step 16100/44000 | Loss: 3.1242 | LR: 3.60e-04 | 408.8ms/step | 48.9 samples/s +Step 16110/44000 | Loss: 3.0456 | LR: 3.60e-04 | 408.6ms/step | 48.9 samples/s +Step 16120/44000 | Loss: 3.0314 | LR: 3.60e-04 | 409.9ms/step | 48.8 samples/s +Step 16130/44000 | Loss: 3.0099 | LR: 3.60e-04 | 409.5ms/step | 48.8 samples/s +Step 16140/44000 | Loss: 3.0436 | LR: 3.59e-04 | 409.5ms/step | 48.8 samples/s +Step 16150/44000 | Loss: 3.0073 | LR: 3.59e-04 | 409.6ms/step | 48.8 samples/s +Step 16160/44000 | Loss: 3.0444 | LR: 3.59e-04 | 409.6ms/step | 48.8 samples/s +Step 16170/44000 | Loss: 3.0336 | LR: 3.59e-04 | 409.6ms/step | 48.8 samples/s +Step 16180/44000 | Loss: 3.0470 | LR: 3.59e-04 | 410.5ms/step | 48.7 samples/s +Step 16190/44000 | Loss: 3.1055 | LR: 3.59e-04 | 409.0ms/step | 48.9 samples/s +Step 16200/44000 | Loss: 3.0425 | LR: 3.58e-04 | 409.5ms/step | 48.8 samples/s +Step 16210/44000 | Loss: 3.1137 | LR: 3.58e-04 | 409.7ms/step | 48.8 samples/s +Step 16220/44000 | Loss: 3.0161 | LR: 3.58e-04 | 409.7ms/step | 48.8 samples/s +Step 16230/44000 | Loss: 3.0550 | LR: 3.58e-04 | 409.1ms/step | 48.9 samples/s +Step 16240/44000 | Loss: 3.0830 | LR: 3.58e-04 | 410.3ms/step | 48.7 samples/s +Step 16250/44000 | Loss: 3.0409 | LR: 3.58e-04 | 409.7ms/step | 48.8 samples/s +Validation at step 16250: Loss = 3.0467, Perplexity = 21.05 +Step 16260/44000 | Loss: 3.0826 | LR: 3.57e-04 | 580.8ms/step | 34.4 samples/s +Step 16270/44000 | Loss: 3.0963 | LR: 3.57e-04 | 409.2ms/step | 48.9 samples/s +Step 16280/44000 | Loss: 3.0622 | LR: 3.57e-04 | 407.0ms/step | 49.1 samples/s +Step 16290/44000 | Loss: 3.0842 | LR: 3.57e-04 | 408.9ms/step | 48.9 samples/s +Step 16300/44000 | Loss: 3.0450 | LR: 3.57e-04 | 409.3ms/step | 48.9 samples/s +Step 16310/44000 | Loss: 3.0789 | LR: 3.57e-04 | 409.5ms/step | 48.8 samples/s +Step 16320/44000 | Loss: 3.0720 | LR: 3.56e-04 | 408.4ms/step | 49.0 samples/s +Step 16330/44000 | Loss: 2.9390 | LR: 3.56e-04 | 409.8ms/step | 48.8 samples/s +Step 16340/44000 | Loss: 3.0550 | LR: 3.56e-04 | 409.0ms/step | 48.9 samples/s +Step 16350/44000 | Loss: 3.0758 | LR: 3.56e-04 | 409.0ms/step | 48.9 samples/s +Step 16360/44000 | Loss: 3.1187 | LR: 3.56e-04 | 409.3ms/step | 48.9 samples/s +Step 16370/44000 | Loss: 3.0718 | LR: 3.56e-04 | 409.5ms/step | 48.8 samples/s +Step 16380/44000 | Loss: 3.0235 | LR: 3.55e-04 | 407.7ms/step | 49.1 samples/s +Step 16390/44000 | Loss: 2.9920 | LR: 3.55e-04 | 409.7ms/step | 48.8 samples/s +Step 16400/44000 | Loss: 3.1021 | LR: 3.55e-04 | 409.6ms/step | 48.8 samples/s +Step 16410/44000 | Loss: 3.0783 | LR: 3.55e-04 | 409.1ms/step | 48.9 samples/s +Step 16420/44000 | Loss: 3.0475 | LR: 3.55e-04 | 409.1ms/step | 48.9 samples/s +Step 16430/44000 | Loss: 3.0374 | LR: 3.55e-04 | 409.1ms/step | 48.9 samples/s +Step 16440/44000 | Loss: 3.0330 | LR: 3.54e-04 | 408.9ms/step | 48.9 samples/s +Step 16450/44000 | Loss: 3.0958 | LR: 3.54e-04 | 410.1ms/step | 48.8 samples/s +Step 16460/44000 | Loss: 3.0777 | LR: 3.54e-04 | 408.6ms/step | 48.9 samples/s +Step 16470/44000 | Loss: 3.0104 | LR: 3.54e-04 | 406.6ms/step | 49.2 samples/s +Step 16480/44000 | Loss: 3.1020 | LR: 3.54e-04 | 407.6ms/step | 49.1 samples/s +Step 16490/44000 | Loss: 3.0400 | LR: 3.54e-04 | 406.1ms/step | 49.2 samples/s +Step 16500/44000 | Loss: 3.0146 | LR: 3.53e-04 | 406.3ms/step | 49.2 samples/s +Validation at step 16500: Loss = 3.0422, Perplexity = 20.95 +Step 16510/44000 | Loss: 3.0424 | LR: 3.53e-04 | 580.3ms/step | 34.5 samples/s +Step 16520/44000 | Loss: 3.0889 | LR: 3.53e-04 | 406.6ms/step | 49.2 samples/s +Step 16530/44000 | Loss: 3.0630 | LR: 3.53e-04 | 406.1ms/step | 49.2 samples/s +Step 16540/44000 | Loss: 3.0282 | LR: 3.53e-04 | 406.2ms/step | 49.2 samples/s +Step 16550/44000 | Loss: 3.0421 | LR: 3.53e-04 | 408.1ms/step | 49.0 samples/s +Step 16560/44000 | Loss: 3.0778 | LR: 3.52e-04 | 405.9ms/step | 49.3 samples/s +Step 16570/44000 | Loss: 3.0267 | LR: 3.52e-04 | 405.7ms/step | 49.3 samples/s +Step 16580/44000 | Loss: 3.0327 | LR: 3.52e-04 | 405.7ms/step | 49.3 samples/s +Step 16590/44000 | Loss: 3.0060 | LR: 3.52e-04 | 405.9ms/step | 49.3 samples/s +Step 16600/44000 | Loss: 3.0235 | LR: 3.52e-04 | 406.4ms/step | 49.2 samples/s +Step 16610/44000 | Loss: 3.1319 | LR: 3.52e-04 | 405.1ms/step | 49.4 samples/s +Step 16620/44000 | Loss: 3.0748 | LR: 3.51e-04 | 405.4ms/step | 49.3 samples/s +Step 16630/44000 | Loss: 2.9577 | LR: 3.51e-04 | 405.7ms/step | 49.3 samples/s +Step 16640/44000 | Loss: 3.0695 | LR: 3.51e-04 | 405.2ms/step | 49.4 samples/s +Step 16650/44000 | Loss: 3.0373 | LR: 3.51e-04 | 405.8ms/step | 49.3 samples/s +Step 16660/44000 | Loss: 3.0258 | LR: 3.51e-04 | 406.0ms/step | 49.3 samples/s +Step 16670/44000 | Loss: 3.0836 | LR: 3.51e-04 | 405.4ms/step | 49.3 samples/s +Step 16680/44000 | Loss: 3.0242 | LR: 3.50e-04 | 407.4ms/step | 49.1 samples/s +Step 16690/44000 | Loss: 3.0302 | LR: 3.50e-04 | 407.5ms/step | 49.1 samples/s +Step 16700/44000 | Loss: 3.1120 | LR: 3.50e-04 | 408.0ms/step | 49.0 samples/s +Step 16710/44000 | Loss: 2.9896 | LR: 3.50e-04 | 407.4ms/step | 49.1 samples/s +Step 16720/44000 | Loss: 3.0417 | LR: 3.50e-04 | 407.2ms/step | 49.1 samples/s +Step 16730/44000 | Loss: 3.0434 | LR: 3.50e-04 | 407.6ms/step | 49.1 samples/s +Step 16740/44000 | Loss: 3.0237 | LR: 3.49e-04 | 406.6ms/step | 49.2 samples/s +Step 16750/44000 | Loss: 3.0309 | LR: 3.49e-04 | 407.1ms/step | 49.1 samples/s +Validation at step 16750: Loss = 3.0402, Perplexity = 20.91 +Step 16760/44000 | Loss: 3.0438 | LR: 3.49e-04 | 578.0ms/step | 34.6 samples/s +Step 16770/44000 | Loss: 3.2317 | LR: 3.49e-04 | 407.7ms/step | 49.1 samples/s +Step 16780/44000 | Loss: 3.0450 | LR: 3.49e-04 | 407.5ms/step | 49.1 samples/s +Step 16790/44000 | Loss: 3.0213 | LR: 3.49e-04 | 407.5ms/step | 49.1 samples/s +Step 16800/44000 | Loss: 3.0674 | LR: 3.48e-04 | 407.8ms/step | 49.0 samples/s +Step 16810/44000 | Loss: 3.0277 | LR: 3.48e-04 | 407.3ms/step | 49.1 samples/s +Step 16820/44000 | Loss: 3.0673 | LR: 3.48e-04 | 408.2ms/step | 49.0 samples/s +Step 16830/44000 | Loss: 3.0050 | LR: 3.48e-04 | 407.5ms/step | 49.1 samples/s +Step 16840/44000 | Loss: 3.0499 | LR: 3.48e-04 | 407.5ms/step | 49.1 samples/s +Step 16850/44000 | Loss: 3.0783 | LR: 3.48e-04 | 407.4ms/step | 49.1 samples/s +Step 16860/44000 | Loss: 3.0038 | LR: 3.47e-04 | 408.2ms/step | 49.0 samples/s +Step 16870/44000 | Loss: 3.0418 | LR: 3.47e-04 | 409.3ms/step | 48.9 samples/s +Step 16880/44000 | Loss: 3.1159 | LR: 3.47e-04 | 407.6ms/step | 49.1 samples/s +Step 16890/44000 | Loss: 2.9823 | LR: 3.47e-04 | 408.0ms/step | 49.0 samples/s +Step 16900/44000 | Loss: 3.0279 | LR: 3.47e-04 | 408.3ms/step | 49.0 samples/s +Step 16910/44000 | Loss: 3.0317 | LR: 3.47e-04 | 408.1ms/step | 49.0 samples/s +Step 16920/44000 | Loss: 3.0381 | LR: 3.46e-04 | 407.4ms/step | 49.1 samples/s +Step 16930/44000 | Loss: 3.0218 | LR: 3.46e-04 | 407.1ms/step | 49.1 samples/s +Step 16940/44000 | Loss: 3.0635 | LR: 3.46e-04 | 407.3ms/step | 49.1 samples/s +Step 16950/44000 | Loss: 3.0038 | LR: 3.46e-04 | 406.7ms/step | 49.2 samples/s +Step 16960/44000 | Loss: 3.0399 | LR: 3.46e-04 | 404.8ms/step | 49.4 samples/s +Step 16970/44000 | Loss: 3.0321 | LR: 3.46e-04 | 405.3ms/step | 49.3 samples/s +Step 16980/44000 | Loss: 3.0806 | LR: 3.45e-04 | 404.3ms/step | 49.5 samples/s +Step 16990/44000 | Loss: 3.0597 | LR: 3.45e-04 | 406.2ms/step | 49.2 samples/s +Step 17000/44000 | Loss: 3.0307 | LR: 3.45e-04 | 407.9ms/step | 49.0 samples/s +Validation at step 17000: Loss = 3.0363, Perplexity = 20.83 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-17000 +Step 17010/44000 | Loss: 3.0814 | LR: 3.45e-04 | 884.4ms/step | 22.6 samples/s +Step 17020/44000 | Loss: 2.9996 | LR: 3.45e-04 | 404.2ms/step | 49.5 samples/s +Step 17030/44000 | Loss: 3.0380 | LR: 3.45e-04 | 405.5ms/step | 49.3 samples/s +Step 17040/44000 | Loss: 2.9723 | LR: 3.44e-04 | 405.2ms/step | 49.4 samples/s +Step 17050/44000 | Loss: 3.0358 | LR: 3.44e-04 | 404.7ms/step | 49.4 samples/s +Step 17060/44000 | Loss: 3.0265 | LR: 3.44e-04 | 404.3ms/step | 49.5 samples/s +Step 17070/44000 | Loss: 3.0270 | LR: 3.44e-04 | 404.4ms/step | 49.5 samples/s +Step 17080/44000 | Loss: 3.0826 | LR: 3.44e-04 | 405.8ms/step | 49.3 samples/s +Step 17090/44000 | Loss: 2.9865 | LR: 3.44e-04 | 406.1ms/step | 49.2 samples/s +Step 17100/44000 | Loss: 3.0143 | LR: 3.43e-04 | 406.0ms/step | 49.3 samples/s +Step 17110/44000 | Loss: 3.0442 | LR: 3.43e-04 | 403.9ms/step | 49.5 samples/s +Step 17120/44000 | Loss: 3.0805 | LR: 3.43e-04 | 403.9ms/step | 49.5 samples/s +Step 17130/44000 | Loss: 2.9611 | LR: 3.43e-04 | 404.5ms/step | 49.4 samples/s +Step 17140/44000 | Loss: 3.0486 | LR: 3.43e-04 | 405.9ms/step | 49.3 samples/s +Step 17150/44000 | Loss: 3.0231 | LR: 3.43e-04 | 407.0ms/step | 49.1 samples/s +Step 17160/44000 | Loss: 3.0385 | LR: 3.42e-04 | 407.6ms/step | 49.1 samples/s +Step 17170/44000 | Loss: 3.0328 | LR: 3.42e-04 | 407.0ms/step | 49.1 samples/s +Step 17180/44000 | Loss: 3.0846 | LR: 3.42e-04 | 406.7ms/step | 49.2 samples/s +Step 17190/44000 | Loss: 3.0197 | LR: 3.42e-04 | 407.4ms/step | 49.1 samples/s +Step 17200/44000 | Loss: 3.0066 | LR: 3.42e-04 | 407.2ms/step | 49.1 samples/s +Step 17210/44000 | Loss: 3.0468 | LR: 3.42e-04 | 406.5ms/step | 49.2 samples/s +Step 17220/44000 | Loss: 3.0234 | LR: 3.41e-04 | 408.2ms/step | 49.0 samples/s +Step 17230/44000 | Loss: 3.0345 | LR: 3.41e-04 | 406.6ms/step | 49.2 samples/s +Step 17240/44000 | Loss: 3.0164 | LR: 3.41e-04 | 408.9ms/step | 48.9 samples/s +Step 17250/44000 | Loss: 3.0281 | LR: 3.41e-04 | 408.7ms/step | 48.9 samples/s +Validation at step 17250: Loss = 3.0353, Perplexity = 20.81 +Step 17260/44000 | Loss: 3.0793 | LR: 3.41e-04 | 585.0ms/step | 34.2 samples/s +Step 17270/44000 | Loss: 3.0456 | LR: 3.41e-04 | 409.8ms/step | 48.8 samples/s +Step 17280/44000 | Loss: 3.0540 | LR: 3.40e-04 | 409.2ms/step | 48.9 samples/s +Step 17290/44000 | Loss: 2.9917 | LR: 3.40e-04 | 410.6ms/step | 48.7 samples/s +Step 17300/44000 | Loss: 3.1256 | LR: 3.40e-04 | 409.3ms/step | 48.9 samples/s +Step 17310/44000 | Loss: 3.0814 | LR: 3.40e-04 | 410.2ms/step | 48.8 samples/s +Step 17320/44000 | Loss: 3.0518 | LR: 3.40e-04 | 407.2ms/step | 49.1 samples/s +Step 17330/44000 | Loss: 2.9830 | LR: 3.40e-04 | 406.0ms/step | 49.3 samples/s +Step 17340/44000 | Loss: 3.0394 | LR: 3.39e-04 | 406.9ms/step | 49.1 samples/s +Step 17350/44000 | Loss: 3.0965 | LR: 3.39e-04 | 406.4ms/step | 49.2 samples/s +Step 17360/44000 | Loss: 3.0293 | LR: 3.39e-04 | 405.3ms/step | 49.4 samples/s +Step 17370/44000 | Loss: 2.9960 | LR: 3.39e-04 | 408.6ms/step | 48.9 samples/s +Step 17380/44000 | Loss: 3.0440 | LR: 3.39e-04 | 409.4ms/step | 48.9 samples/s +Step 17390/44000 | Loss: 2.9915 | LR: 3.39e-04 | 408.9ms/step | 48.9 samples/s +Step 17400/44000 | Loss: 3.0149 | LR: 3.38e-04 | 408.5ms/step | 49.0 samples/s +Step 17410/44000 | Loss: 2.9866 | LR: 3.38e-04 | 410.5ms/step | 48.7 samples/s +Step 17420/44000 | Loss: 3.0270 | LR: 3.38e-04 | 409.1ms/step | 48.9 samples/s +Step 17430/44000 | Loss: 3.0478 | LR: 3.38e-04 | 408.8ms/step | 48.9 samples/s +Step 17440/44000 | Loss: 2.9776 | LR: 3.38e-04 | 406.7ms/step | 49.2 samples/s +Step 17450/44000 | Loss: 3.0031 | LR: 3.38e-04 | 407.9ms/step | 49.0 samples/s +Step 17460/44000 | Loss: 2.9860 | LR: 3.37e-04 | 409.4ms/step | 48.9 samples/s +Step 17470/44000 | Loss: 3.0244 | LR: 3.37e-04 | 406.6ms/step | 49.2 samples/s +Step 17480/44000 | Loss: 3.0545 | LR: 3.37e-04 | 405.6ms/step | 49.3 samples/s +Step 17490/44000 | Loss: 3.0204 | LR: 3.37e-04 | 405.4ms/step | 49.3 samples/s +Step 17500/44000 | Loss: 3.0194 | LR: 3.37e-04 | 404.9ms/step | 49.4 samples/s +Validation at step 17500: Loss = 3.0317, Perplexity = 20.73 +Step 17510/44000 | Loss: 3.0718 | LR: 3.37e-04 | 574.2ms/step | 34.8 samples/s +Step 17520/44000 | Loss: 3.0484 | LR: 3.36e-04 | 404.9ms/step | 49.4 samples/s +Step 17530/44000 | Loss: 3.0598 | LR: 3.36e-04 | 408.3ms/step | 49.0 samples/s +Step 17540/44000 | Loss: 3.0503 | LR: 3.36e-04 | 404.4ms/step | 49.5 samples/s +Step 17550/44000 | Loss: 2.9504 | LR: 3.36e-04 | 404.1ms/step | 49.5 samples/s +Step 17560/44000 | Loss: 3.0179 | LR: 3.36e-04 | 405.6ms/step | 49.3 samples/s +Step 17570/44000 | Loss: 3.0330 | LR: 3.35e-04 | 406.6ms/step | 49.2 samples/s +Step 17580/44000 | Loss: 3.0229 | LR: 3.35e-04 | 406.1ms/step | 49.2 samples/s +Step 17590/44000 | Loss: 3.0364 | LR: 3.35e-04 | 406.1ms/step | 49.3 samples/s +Step 17600/44000 | Loss: 3.0756 | LR: 3.35e-04 | 406.8ms/step | 49.2 samples/s +Step 17610/44000 | Loss: 2.9841 | LR: 3.35e-04 | 406.8ms/step | 49.2 samples/s +Step 17620/44000 | Loss: 2.9921 | LR: 3.35e-04 | 407.3ms/step | 49.1 samples/s +Step 17630/44000 | Loss: 3.0257 | LR: 3.34e-04 | 406.3ms/step | 49.2 samples/s +Step 17640/44000 | Loss: 3.0684 | LR: 3.34e-04 | 406.4ms/step | 49.2 samples/s +Step 17650/44000 | Loss: 3.0443 | LR: 3.34e-04 | 406.8ms/step | 49.2 samples/s +Step 17660/44000 | Loss: 3.0526 | LR: 3.34e-04 | 406.3ms/step | 49.2 samples/s +Step 17670/44000 | Loss: 3.0129 | LR: 3.34e-04 | 406.5ms/step | 49.2 samples/s +Step 17680/44000 | Loss: 3.0573 | LR: 3.34e-04 | 407.0ms/step | 49.1 samples/s +Step 17690/44000 | Loss: 3.0408 | LR: 3.33e-04 | 406.6ms/step | 49.2 samples/s +Step 17700/44000 | Loss: 3.0420 | LR: 3.33e-04 | 406.7ms/step | 49.2 samples/s +Step 17710/44000 | Loss: 3.0583 | LR: 3.33e-04 | 406.6ms/step | 49.2 samples/s +Step 17720/44000 | Loss: 3.0173 | LR: 3.33e-04 | 406.9ms/step | 49.2 samples/s +Step 17730/44000 | Loss: 2.9730 | LR: 3.33e-04 | 406.0ms/step | 49.3 samples/s +Step 17740/44000 | Loss: 3.0834 | LR: 3.33e-04 | 406.4ms/step | 49.2 samples/s +Step 17750/44000 | Loss: 3.0393 | LR: 3.32e-04 | 406.1ms/step | 49.2 samples/s +Validation at step 17750: Loss = 3.0293, Perplexity = 20.68 +Step 17760/44000 | Loss: 3.0923 | LR: 3.32e-04 | 574.8ms/step | 34.8 samples/s +Step 17770/44000 | Loss: 3.0665 | LR: 3.32e-04 | 407.3ms/step | 49.1 samples/s +Step 17780/44000 | Loss: 3.0178 | LR: 3.32e-04 | 407.4ms/step | 49.1 samples/s +Step 17790/44000 | Loss: 3.0095 | LR: 3.32e-04 | 406.9ms/step | 49.2 samples/s +Step 17800/44000 | Loss: 2.9954 | LR: 3.32e-04 | 407.3ms/step | 49.1 samples/s +Step 17810/44000 | Loss: 3.0627 | LR: 3.31e-04 | 407.3ms/step | 49.1 samples/s +Step 17820/44000 | Loss: 3.0259 | LR: 3.31e-04 | 408.4ms/step | 49.0 samples/s +Step 17830/44000 | Loss: 3.0059 | LR: 3.31e-04 | 407.2ms/step | 49.1 samples/s +Step 17840/44000 | Loss: 2.9790 | LR: 3.31e-04 | 407.2ms/step | 49.1 samples/s +Step 17850/44000 | Loss: 3.0481 | LR: 3.31e-04 | 406.8ms/step | 49.2 samples/s +Step 17860/44000 | Loss: 3.0017 | LR: 3.31e-04 | 407.3ms/step | 49.1 samples/s +Step 17870/44000 | Loss: 3.0058 | LR: 3.30e-04 | 407.0ms/step | 49.1 samples/s +Step 17880/44000 | Loss: 3.0508 | LR: 3.30e-04 | 407.1ms/step | 49.1 samples/s +Step 17890/44000 | Loss: 3.0498 | LR: 3.30e-04 | 406.4ms/step | 49.2 samples/s +Step 17900/44000 | Loss: 3.0533 | LR: 3.30e-04 | 406.5ms/step | 49.2 samples/s +Step 17910/44000 | Loss: 3.0041 | LR: 3.30e-04 | 406.2ms/step | 49.2 samples/s +Step 17920/44000 | Loss: 3.0297 | LR: 3.29e-04 | 406.5ms/step | 49.2 samples/s +Step 17930/44000 | Loss: 3.0461 | LR: 3.29e-04 | 406.3ms/step | 49.2 samples/s +Step 17940/44000 | Loss: 2.9872 | LR: 3.29e-04 | 406.7ms/step | 49.2 samples/s +Step 17950/44000 | Loss: 2.9789 | LR: 3.29e-04 | 406.0ms/step | 49.3 samples/s +Step 17960/44000 | Loss: 3.0064 | LR: 3.29e-04 | 405.7ms/step | 49.3 samples/s +Step 17970/44000 | Loss: 3.0818 | LR: 3.29e-04 | 407.1ms/step | 49.1 samples/s +Step 17980/44000 | Loss: 2.9954 | LR: 3.28e-04 | 405.8ms/step | 49.3 samples/s +Step 17990/44000 | Loss: 3.0407 | LR: 3.28e-04 | 406.5ms/step | 49.2 samples/s +Step 18000/44000 | Loss: 3.0154 | LR: 3.28e-04 | 406.4ms/step | 49.2 samples/s +Validation at step 18000: Loss = 3.0253, Perplexity = 20.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-18000 +Step 18010/44000 | Loss: 2.9752 | LR: 3.28e-04 | 871.8ms/step | 22.9 samples/s +Step 18020/44000 | Loss: 3.0377 | LR: 3.28e-04 | 403.7ms/step | 49.5 samples/s +Step 18030/44000 | Loss: 2.9759 | LR: 3.28e-04 | 403.5ms/step | 49.6 samples/s +Step 18040/44000 | Loss: 3.0580 | LR: 3.27e-04 | 404.1ms/step | 49.5 samples/s +Step 18050/44000 | Loss: 2.9878 | LR: 3.27e-04 | 402.6ms/step | 49.7 samples/s +Step 18060/44000 | Loss: 2.9949 | LR: 3.27e-04 | 403.7ms/step | 49.5 samples/s +Step 18070/44000 | Loss: 3.0471 | LR: 3.27e-04 | 403.4ms/step | 49.6 samples/s +Step 18080/44000 | Loss: 2.9899 | LR: 3.27e-04 | 404.2ms/step | 49.5 samples/s +Step 18090/44000 | Loss: 2.9812 | LR: 3.27e-04 | 404.8ms/step | 49.4 samples/s +Step 18100/44000 | Loss: 3.0713 | LR: 3.26e-04 | 406.1ms/step | 49.3 samples/s +Step 18110/44000 | Loss: 3.0214 | LR: 3.26e-04 | 407.2ms/step | 49.1 samples/s +Step 18120/44000 | Loss: 3.0064 | LR: 3.26e-04 | 405.6ms/step | 49.3 samples/s +Step 18130/44000 | Loss: 3.0459 | LR: 3.26e-04 | 405.5ms/step | 49.3 samples/s +Step 18140/44000 | Loss: 3.0797 | LR: 3.26e-04 | 403.8ms/step | 49.5 samples/s +Step 18150/44000 | Loss: 3.0679 | LR: 3.26e-04 | 403.3ms/step | 49.6 samples/s +Step 18160/44000 | Loss: 2.9968 | LR: 3.25e-04 | 404.0ms/step | 49.5 samples/s +Step 18170/44000 | Loss: 3.0418 | LR: 3.25e-04 | 404.4ms/step | 49.5 samples/s +Step 18180/44000 | Loss: 3.0532 | LR: 3.25e-04 | 404.1ms/step | 49.5 samples/s +Step 18190/44000 | Loss: 2.9268 | LR: 3.25e-04 | 404.1ms/step | 49.5 samples/s +Step 18200/44000 | Loss: 3.0284 | LR: 3.25e-04 | 403.8ms/step | 49.5 samples/s +Step 18210/44000 | Loss: 3.1214 | LR: 3.24e-04 | 404.5ms/step | 49.4 samples/s +Step 18220/44000 | Loss: 3.0109 | LR: 3.24e-04 | 403.4ms/step | 49.6 samples/s +Step 18230/44000 | Loss: 3.0148 | LR: 3.24e-04 | 404.1ms/step | 49.5 samples/s +Step 18240/44000 | Loss: 3.0206 | LR: 3.24e-04 | 403.5ms/step | 49.6 samples/s +Step 18250/44000 | Loss: 3.0035 | LR: 3.24e-04 | 404.3ms/step | 49.5 samples/s +Validation at step 18250: Loss = 3.0226, Perplexity = 20.54 +Step 18260/44000 | Loss: 3.0379 | LR: 3.24e-04 | 576.5ms/step | 34.7 samples/s +Step 18270/44000 | Loss: 3.0560 | LR: 3.23e-04 | 405.9ms/step | 49.3 samples/s +Step 18280/44000 | Loss: 3.0145 | LR: 3.23e-04 | 406.5ms/step | 49.2 samples/s +Step 18290/44000 | Loss: 3.0204 | LR: 3.23e-04 | 405.3ms/step | 49.3 samples/s +Step 18300/44000 | Loss: 3.0437 | LR: 3.23e-04 | 405.6ms/step | 49.3 samples/s +Step 18310/44000 | Loss: 3.0584 | LR: 3.23e-04 | 407.1ms/step | 49.1 samples/s +Step 18320/44000 | Loss: 2.9724 | LR: 3.23e-04 | 405.4ms/step | 49.3 samples/s +Step 18330/44000 | Loss: 2.9867 | LR: 3.22e-04 | 406.2ms/step | 49.2 samples/s +Step 18340/44000 | Loss: 2.9897 | LR: 3.22e-04 | 405.2ms/step | 49.4 samples/s +Step 18350/44000 | Loss: 3.0287 | LR: 3.22e-04 | 405.7ms/step | 49.3 samples/s +Step 18360/44000 | Loss: 3.0549 | LR: 3.22e-04 | 406.6ms/step | 49.2 samples/s +Step 18370/44000 | Loss: 2.9928 | LR: 3.22e-04 | 406.8ms/step | 49.2 samples/s +Step 18380/44000 | Loss: 3.0713 | LR: 3.22e-04 | 407.1ms/step | 49.1 samples/s +Step 18390/44000 | Loss: 3.0756 | LR: 3.21e-04 | 406.9ms/step | 49.1 samples/s +Step 18400/44000 | Loss: 2.9879 | LR: 3.21e-04 | 408.0ms/step | 49.0 samples/s +Step 18410/44000 | Loss: 3.0013 | LR: 3.21e-04 | 407.4ms/step | 49.1 samples/s +Step 18420/44000 | Loss: 3.0336 | LR: 3.21e-04 | 406.4ms/step | 49.2 samples/s +Step 18430/44000 | Loss: 3.0473 | LR: 3.21e-04 | 406.6ms/step | 49.2 samples/s +Step 18440/44000 | Loss: 2.9871 | LR: 3.20e-04 | 405.5ms/step | 49.3 samples/s +Step 18450/44000 | Loss: 3.0263 | LR: 3.20e-04 | 406.9ms/step | 49.1 samples/s +Step 18460/44000 | Loss: 3.0207 | LR: 3.20e-04 | 405.5ms/step | 49.3 samples/s +Step 18470/44000 | Loss: 3.0239 | LR: 3.20e-04 | 406.1ms/step | 49.2 samples/s +Step 18480/44000 | Loss: 3.0231 | LR: 3.20e-04 | 406.5ms/step | 49.2 samples/s +Step 18490/44000 | Loss: 2.9426 | LR: 3.20e-04 | 405.7ms/step | 49.3 samples/s +Step 18500/44000 | Loss: 2.9981 | LR: 3.19e-04 | 405.4ms/step | 49.3 samples/s +Validation at step 18500: Loss = 3.0190, Perplexity = 20.47 +Step 18510/44000 | Loss: 2.9714 | LR: 3.19e-04 | 575.5ms/step | 34.8 samples/s +Step 18520/44000 | Loss: 3.0358 | LR: 3.19e-04 | 406.0ms/step | 49.3 samples/s +Step 18530/44000 | Loss: 3.0517 | LR: 3.19e-04 | 405.5ms/step | 49.3 samples/s +Step 18540/44000 | Loss: 3.0831 | LR: 3.19e-04 | 405.0ms/step | 49.4 samples/s +Step 18550/44000 | Loss: 2.9890 | LR: 3.19e-04 | 404.1ms/step | 49.5 samples/s +Step 18560/44000 | Loss: 3.0385 | LR: 3.18e-04 | 403.0ms/step | 49.6 samples/s +Step 18570/44000 | Loss: 3.0411 | LR: 3.18e-04 | 404.0ms/step | 49.5 samples/s +Step 18580/44000 | Loss: 3.0310 | LR: 3.18e-04 | 402.8ms/step | 49.7 samples/s +Step 18590/44000 | Loss: 3.0179 | LR: 3.18e-04 | 403.5ms/step | 49.6 samples/s +Step 18600/44000 | Loss: 2.9830 | LR: 3.18e-04 | 405.3ms/step | 49.3 samples/s +Step 18610/44000 | Loss: 3.0450 | LR: 3.18e-04 | 404.8ms/step | 49.4 samples/s +Step 18620/44000 | Loss: 3.0372 | LR: 3.17e-04 | 403.9ms/step | 49.5 samples/s +Step 18630/44000 | Loss: 3.0290 | LR: 3.17e-04 | 403.6ms/step | 49.6 samples/s +Step 18640/44000 | Loss: 3.0271 | LR: 3.17e-04 | 404.8ms/step | 49.4 samples/s +Step 18650/44000 | Loss: 3.0648 | LR: 3.17e-04 | 405.2ms/step | 49.4 samples/s +Step 18660/44000 | Loss: 2.9733 | LR: 3.17e-04 | 404.5ms/step | 49.4 samples/s +Step 18670/44000 | Loss: 3.0717 | LR: 3.16e-04 | 404.9ms/step | 49.4 samples/s +Step 18680/44000 | Loss: 3.0078 | LR: 3.16e-04 | 404.5ms/step | 49.4 samples/s +Step 18690/44000 | Loss: 3.0814 | LR: 3.16e-04 | 404.5ms/step | 49.4 samples/s +Step 18700/44000 | Loss: 2.9753 | LR: 3.16e-04 | 405.5ms/step | 49.3 samples/s +Step 18710/44000 | Loss: 3.0658 | LR: 3.16e-04 | 404.5ms/step | 49.4 samples/s +Step 18720/44000 | Loss: 3.0140 | LR: 3.16e-04 | 404.7ms/step | 49.4 samples/s +Step 18730/44000 | Loss: 3.0709 | LR: 3.15e-04 | 405.2ms/step | 49.4 samples/s +Step 18740/44000 | Loss: 3.0335 | LR: 3.15e-04 | 405.5ms/step | 49.3 samples/s +Step 18750/44000 | Loss: 2.9210 | LR: 3.15e-04 | 405.7ms/step | 49.3 samples/s +Validation at step 18750: Loss = 3.0167, Perplexity = 20.42 +Step 18760/44000 | Loss: 3.0024 | LR: 3.15e-04 | 574.6ms/step | 34.8 samples/s +Step 18770/44000 | Loss: 3.0075 | LR: 3.15e-04 | 405.1ms/step | 49.4 samples/s +Step 18780/44000 | Loss: 2.9763 | LR: 3.15e-04 | 405.2ms/step | 49.4 samples/s +Step 18790/44000 | Loss: 3.0724 | LR: 3.14e-04 | 404.6ms/step | 49.4 samples/s +Step 18800/44000 | Loss: 3.0529 | LR: 3.14e-04 | 405.0ms/step | 49.4 samples/s +Step 18810/44000 | Loss: 3.0701 | LR: 3.14e-04 | 404.3ms/step | 49.5 samples/s +Step 18820/44000 | Loss: 3.0096 | LR: 3.14e-04 | 405.0ms/step | 49.4 samples/s +Step 18830/44000 | Loss: 3.0106 | LR: 3.14e-04 | 405.5ms/step | 49.3 samples/s +Step 18840/44000 | Loss: 3.0042 | LR: 3.13e-04 | 405.5ms/step | 49.3 samples/s +Step 18850/44000 | Loss: 3.0198 | LR: 3.13e-04 | 405.7ms/step | 49.3 samples/s +Step 18860/44000 | Loss: 3.0253 | LR: 3.13e-04 | 404.6ms/step | 49.4 samples/s +Step 18870/44000 | Loss: 3.0387 | LR: 3.13e-04 | 405.5ms/step | 49.3 samples/s +Step 18880/44000 | Loss: 3.0368 | LR: 3.13e-04 | 404.8ms/step | 49.4 samples/s +Step 18890/44000 | Loss: 3.0377 | LR: 3.13e-04 | 404.4ms/step | 49.5 samples/s +Step 18900/44000 | Loss: 2.9896 | LR: 3.12e-04 | 405.3ms/step | 49.3 samples/s +Step 18910/44000 | Loss: 2.9694 | LR: 3.12e-04 | 404.8ms/step | 49.4 samples/s +Step 18920/44000 | Loss: 3.0060 | LR: 3.12e-04 | 404.5ms/step | 49.4 samples/s +Step 18930/44000 | Loss: 3.0059 | LR: 3.12e-04 | 405.0ms/step | 49.4 samples/s +Step 18940/44000 | Loss: 3.0160 | LR: 3.12e-04 | 405.2ms/step | 49.4 samples/s +Step 18950/44000 | Loss: 2.9491 | LR: 3.12e-04 | 404.7ms/step | 49.4 samples/s +Step 18960/44000 | Loss: 2.9835 | LR: 3.11e-04 | 405.7ms/step | 49.3 samples/s +Step 18970/44000 | Loss: 2.9853 | LR: 3.11e-04 | 405.2ms/step | 49.4 samples/s +Step 18980/44000 | Loss: 3.0492 | LR: 3.11e-04 | 405.8ms/step | 49.3 samples/s +Step 18990/44000 | Loss: 2.9913 | LR: 3.11e-04 | 404.8ms/step | 49.4 samples/s +Step 19000/44000 | Loss: 2.9980 | LR: 3.11e-04 | 404.9ms/step | 49.4 samples/s +Validation at step 19000: Loss = 3.0130, Perplexity = 20.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-19000 +Step 19010/44000 | Loss: 3.0355 | LR: 3.11e-04 | 879.6ms/step | 22.7 samples/s +Step 19020/44000 | Loss: 3.0004 | LR: 3.10e-04 | 404.8ms/step | 49.4 samples/s +Step 19030/44000 | Loss: 3.0271 | LR: 3.10e-04 | 405.1ms/step | 49.4 samples/s +Step 19040/44000 | Loss: 3.0160 | LR: 3.10e-04 | 405.1ms/step | 49.4 samples/s +Step 19050/44000 | Loss: 3.0374 | LR: 3.10e-04 | 405.1ms/step | 49.4 samples/s +Step 19060/44000 | Loss: 2.9648 | LR: 3.10e-04 | 404.9ms/step | 49.4 samples/s +Step 19070/44000 | Loss: 3.0444 | LR: 3.09e-04 | 404.5ms/step | 49.4 samples/s +Step 19080/44000 | Loss: 2.9992 | LR: 3.09e-04 | 404.3ms/step | 49.5 samples/s +Step 19090/44000 | Loss: 3.0558 | LR: 3.09e-04 | 404.1ms/step | 49.5 samples/s +Step 19100/44000 | Loss: 2.9924 | LR: 3.09e-04 | 405.1ms/step | 49.4 samples/s +Step 19110/44000 | Loss: 3.0211 | LR: 3.09e-04 | 405.2ms/step | 49.4 samples/s +Step 19120/44000 | Loss: 3.0124 | LR: 3.09e-04 | 406.7ms/step | 49.2 samples/s +Step 19130/44000 | Loss: 3.0338 | LR: 3.08e-04 | 404.3ms/step | 49.5 samples/s +Step 19140/44000 | Loss: 3.0237 | LR: 3.08e-04 | 404.7ms/step | 49.4 samples/s +Step 19150/44000 | Loss: 3.0692 | LR: 3.08e-04 | 403.8ms/step | 49.5 samples/s +Step 19160/44000 | Loss: 2.9994 | LR: 3.08e-04 | 403.6ms/step | 49.6 samples/s +Step 19170/44000 | Loss: 2.9884 | LR: 3.08e-04 | 403.2ms/step | 49.6 samples/s +Step 19180/44000 | Loss: 3.0298 | LR: 3.08e-04 | 403.2ms/step | 49.6 samples/s +Step 19190/44000 | Loss: 3.0284 | LR: 3.07e-04 | 403.6ms/step | 49.6 samples/s +Step 19200/44000 | Loss: 3.0324 | LR: 3.07e-04 | 404.4ms/step | 49.5 samples/s +Step 19210/44000 | Loss: 3.0606 | LR: 3.07e-04 | 403.6ms/step | 49.6 samples/s +Step 19220/44000 | Loss: 3.0240 | LR: 3.07e-04 | 403.9ms/step | 49.5 samples/s +Step 19230/44000 | Loss: 3.0075 | LR: 3.07e-04 | 403.1ms/step | 49.6 samples/s +Step 19240/44000 | Loss: 3.0305 | LR: 3.06e-04 | 402.9ms/step | 49.6 samples/s +Step 19250/44000 | Loss: 2.9808 | LR: 3.06e-04 | 403.1ms/step | 49.6 samples/s +Validation at step 19250: Loss = 3.0111, Perplexity = 20.31 +Step 19260/44000 | Loss: 2.9945 | LR: 3.06e-04 | 574.1ms/step | 34.8 samples/s +Step 19270/44000 | Loss: 3.0360 | LR: 3.06e-04 | 403.3ms/step | 49.6 samples/s +Step 19280/44000 | Loss: 3.0402 | LR: 3.06e-04 | 405.3ms/step | 49.3 samples/s +Step 19290/44000 | Loss: 2.9872 | LR: 3.06e-04 | 404.6ms/step | 49.4 samples/s +Step 19300/44000 | Loss: 3.0257 | LR: 3.05e-04 | 405.2ms/step | 49.4 samples/s +Step 19310/44000 | Loss: 2.9704 | LR: 3.05e-04 | 404.2ms/step | 49.5 samples/s +Step 19320/44000 | Loss: 2.9766 | LR: 3.05e-04 | 405.3ms/step | 49.3 samples/s +Step 19330/44000 | Loss: 2.9811 | LR: 3.05e-04 | 406.0ms/step | 49.3 samples/s +Step 19340/44000 | Loss: 3.0355 | LR: 3.05e-04 | 408.1ms/step | 49.0 samples/s +Step 19350/44000 | Loss: 3.0504 | LR: 3.04e-04 | 407.6ms/step | 49.1 samples/s +Step 19360/44000 | Loss: 3.0922 | LR: 3.04e-04 | 406.5ms/step | 49.2 samples/s +Step 19370/44000 | Loss: 3.0293 | LR: 3.04e-04 | 406.2ms/step | 49.2 samples/s +Step 19380/44000 | Loss: 2.9927 | LR: 3.04e-04 | 406.6ms/step | 49.2 samples/s +Step 19390/44000 | Loss: 3.0047 | LR: 3.04e-04 | 406.4ms/step | 49.2 samples/s +Step 19400/44000 | Loss: 3.0331 | LR: 3.04e-04 | 406.1ms/step | 49.2 samples/s +Step 19410/44000 | Loss: 3.0474 | LR: 3.03e-04 | 406.1ms/step | 49.2 samples/s +Step 19420/44000 | Loss: 3.0231 | LR: 3.03e-04 | 405.7ms/step | 49.3 samples/s +Step 19430/44000 | Loss: 3.0231 | LR: 3.03e-04 | 406.1ms/step | 49.3 samples/s +Step 19440/44000 | Loss: 3.0405 | LR: 3.03e-04 | 405.8ms/step | 49.3 samples/s +Step 19450/44000 | Loss: 2.9918 | LR: 3.03e-04 | 405.4ms/step | 49.3 samples/s +Step 19460/44000 | Loss: 3.0246 | LR: 3.03e-04 | 405.6ms/step | 49.3 samples/s +Step 19470/44000 | Loss: 3.0162 | LR: 3.02e-04 | 404.7ms/step | 49.4 samples/s +Step 19480/44000 | Loss: 2.9824 | LR: 3.02e-04 | 404.0ms/step | 49.5 samples/s +Step 19490/44000 | Loss: 2.9688 | LR: 3.02e-04 | 405.4ms/step | 49.3 samples/s +Step 19500/44000 | Loss: 3.0055 | LR: 3.02e-04 | 404.8ms/step | 49.4 samples/s +Validation at step 19500: Loss = 3.0082, Perplexity = 20.25 +Step 19510/44000 | Loss: 3.0283 | LR: 3.02e-04 | 573.6ms/step | 34.9 samples/s +Step 19520/44000 | Loss: 2.9938 | LR: 3.01e-04 | 403.7ms/step | 49.5 samples/s +Step 19530/44000 | Loss: 3.0250 | LR: 3.01e-04 | 404.3ms/step | 49.5 samples/s +Step 19540/44000 | Loss: 3.0281 | LR: 3.01e-04 | 408.3ms/step | 49.0 samples/s +Step 19550/44000 | Loss: 3.0101 | LR: 3.01e-04 | 405.5ms/step | 49.3 samples/s +Step 19560/44000 | Loss: 3.0219 | LR: 3.01e-04 | 405.1ms/step | 49.4 samples/s +Step 19570/44000 | Loss: 3.0277 | LR: 3.01e-04 | 405.2ms/step | 49.4 samples/s +Step 19580/44000 | Loss: 3.0630 | LR: 3.00e-04 | 405.0ms/step | 49.4 samples/s +Step 19590/44000 | Loss: 3.0688 | LR: 3.00e-04 | 406.0ms/step | 49.3 samples/s +Step 19600/44000 | Loss: 3.0768 | LR: 3.00e-04 | 405.6ms/step | 49.3 samples/s +Step 19610/44000 | Loss: 3.0088 | LR: 3.00e-04 | 404.7ms/step | 49.4 samples/s +Step 19620/44000 | Loss: 3.0428 | LR: 3.00e-04 | 405.2ms/step | 49.4 samples/s +Step 19630/44000 | Loss: 3.0835 | LR: 3.00e-04 | 405.5ms/step | 49.3 samples/s +Step 19640/44000 | Loss: 2.9342 | LR: 2.99e-04 | 405.3ms/step | 49.3 samples/s +Step 19650/44000 | Loss: 2.9946 | LR: 2.99e-04 | 404.7ms/step | 49.4 samples/s +Step 19660/44000 | Loss: 3.0361 | LR: 2.99e-04 | 404.7ms/step | 49.4 samples/s +Step 19670/44000 | Loss: 3.0111 | LR: 2.99e-04 | 404.5ms/step | 49.4 samples/s +Step 19680/44000 | Loss: 2.9607 | LR: 2.99e-04 | 404.9ms/step | 49.4 samples/s +Step 19690/44000 | Loss: 2.9831 | LR: 2.98e-04 | 405.9ms/step | 49.3 samples/s +Step 19700/44000 | Loss: 3.0236 | LR: 2.98e-04 | 405.7ms/step | 49.3 samples/s +Step 19710/44000 | Loss: 3.0551 | LR: 2.98e-04 | 406.7ms/step | 49.2 samples/s +Step 19720/44000 | Loss: 2.9406 | LR: 2.98e-04 | 405.6ms/step | 49.3 samples/s +Step 19730/44000 | Loss: 2.9845 | LR: 2.98e-04 | 407.0ms/step | 49.1 samples/s +Step 19740/44000 | Loss: 3.0067 | LR: 2.98e-04 | 405.7ms/step | 49.3 samples/s +Step 19750/44000 | Loss: 3.0180 | LR: 2.97e-04 | 405.5ms/step | 49.3 samples/s +Validation at step 19750: Loss = 3.0064, Perplexity = 20.21 +Step 19760/44000 | Loss: 3.0537 | LR: 2.97e-04 | 573.9ms/step | 34.9 samples/s +Step 19770/44000 | Loss: 3.0106 | LR: 2.97e-04 | 407.6ms/step | 49.1 samples/s +Step 19780/44000 | Loss: 2.9939 | LR: 2.97e-04 | 402.9ms/step | 49.6 samples/s +Step 19790/44000 | Loss: 2.9675 | LR: 2.97e-04 | 404.1ms/step | 49.5 samples/s +Step 19800/44000 | Loss: 2.9560 | LR: 2.96e-04 | 403.2ms/step | 49.6 samples/s +Step 19810/44000 | Loss: 2.9916 | LR: 2.96e-04 | 402.3ms/step | 49.7 samples/s +Step 19820/44000 | Loss: 2.9589 | LR: 2.96e-04 | 402.3ms/step | 49.7 samples/s +Step 19830/44000 | Loss: 2.9972 | LR: 2.96e-04 | 402.1ms/step | 49.7 samples/s +Step 19840/44000 | Loss: 2.9950 | LR: 2.96e-04 | 402.0ms/step | 49.8 samples/s +Step 19850/44000 | Loss: 3.0329 | LR: 2.96e-04 | 401.8ms/step | 49.8 samples/s +Step 19860/44000 | Loss: 2.9816 | LR: 2.95e-04 | 402.1ms/step | 49.7 samples/s +Step 19870/44000 | Loss: 3.0158 | LR: 2.95e-04 | 402.5ms/step | 49.7 samples/s +Step 19880/44000 | Loss: 2.9830 | LR: 2.95e-04 | 402.2ms/step | 49.7 samples/s +Step 19890/44000 | Loss: 2.9931 | LR: 2.95e-04 | 401.4ms/step | 49.8 samples/s +Step 19900/44000 | Loss: 3.0361 | LR: 2.95e-04 | 402.6ms/step | 49.7 samples/s +Step 19910/44000 | Loss: 2.9866 | LR: 2.95e-04 | 403.0ms/step | 49.6 samples/s +Step 19920/44000 | Loss: 3.0315 | LR: 2.94e-04 | 401.9ms/step | 49.8 samples/s +Step 19930/44000 | Loss: 3.0002 | LR: 2.94e-04 | 402.0ms/step | 49.8 samples/s +Step 19940/44000 | Loss: 2.9251 | LR: 2.94e-04 | 402.1ms/step | 49.7 samples/s +Step 19950/44000 | Loss: 2.9635 | LR: 2.94e-04 | 402.0ms/step | 49.7 samples/s +Step 19960/44000 | Loss: 2.9874 | LR: 2.94e-04 | 402.2ms/step | 49.7 samples/s +Step 19970/44000 | Loss: 2.9548 | LR: 2.93e-04 | 402.3ms/step | 49.7 samples/s +Step 19980/44000 | Loss: 3.0061 | LR: 2.93e-04 | 402.9ms/step | 49.6 samples/s +Step 19990/44000 | Loss: 3.0350 | LR: 2.93e-04 | 402.5ms/step | 49.7 samples/s +Step 20000/44000 | Loss: 2.9373 | LR: 2.93e-04 | 402.5ms/step | 49.7 samples/s +Validation at step 20000: Loss = 3.0051, Perplexity = 20.19 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-20000 +Step 20010/44000 | Loss: 3.0348 | LR: 2.93e-04 | 877.9ms/step | 22.8 samples/s +Step 20020/44000 | Loss: 3.0284 | LR: 2.93e-04 | 401.8ms/step | 49.8 samples/s +Step 20030/44000 | Loss: 2.9830 | LR: 2.92e-04 | 402.4ms/step | 49.7 samples/s +Step 20040/44000 | Loss: 2.9941 | LR: 2.92e-04 | 402.6ms/step | 49.7 samples/s +Step 20050/44000 | Loss: 3.0060 | LR: 2.92e-04 | 403.2ms/step | 49.6 samples/s +Step 20060/44000 | Loss: 3.0786 | LR: 2.92e-04 | 402.9ms/step | 49.6 samples/s +Step 20070/44000 | Loss: 3.0139 | LR: 2.92e-04 | 402.9ms/step | 49.6 samples/s +Step 20080/44000 | Loss: 2.9796 | LR: 2.91e-04 | 402.5ms/step | 49.7 samples/s +Step 20090/44000 | Loss: 3.0346 | LR: 2.91e-04 | 403.0ms/step | 49.6 samples/s +Step 20100/44000 | Loss: 2.9880 | LR: 2.91e-04 | 402.6ms/step | 49.7 samples/s +Step 20110/44000 | Loss: 2.9772 | LR: 2.91e-04 | 402.3ms/step | 49.7 samples/s +Step 20120/44000 | Loss: 2.9794 | LR: 2.91e-04 | 403.0ms/step | 49.6 samples/s +Step 20130/44000 | Loss: 2.9303 | LR: 2.91e-04 | 401.9ms/step | 49.8 samples/s +Step 20140/44000 | Loss: 2.9734 | LR: 2.90e-04 | 402.7ms/step | 49.7 samples/s +Step 20150/44000 | Loss: 3.0500 | LR: 2.90e-04 | 402.2ms/step | 49.7 samples/s +Step 20160/44000 | Loss: 3.0014 | LR: 2.90e-04 | 402.1ms/step | 49.7 samples/s +Step 20170/44000 | Loss: 3.0330 | LR: 2.90e-04 | 403.9ms/step | 49.5 samples/s +Step 20180/44000 | Loss: 3.0009 | LR: 2.90e-04 | 405.0ms/step | 49.4 samples/s +Step 20190/44000 | Loss: 3.0189 | LR: 2.90e-04 | 404.9ms/step | 49.4 samples/s +Step 20200/44000 | Loss: 3.0062 | LR: 2.89e-04 | 404.3ms/step | 49.5 samples/s +Step 20210/44000 | Loss: 2.9640 | LR: 2.89e-04 | 404.6ms/step | 49.4 samples/s +Step 20220/44000 | Loss: 3.0315 | LR: 2.89e-04 | 404.6ms/step | 49.4 samples/s +Step 20230/44000 | Loss: 2.9974 | LR: 2.89e-04 | 404.6ms/step | 49.4 samples/s +Step 20240/44000 | Loss: 3.0215 | LR: 2.89e-04 | 403.8ms/step | 49.5 samples/s +Step 20250/44000 | Loss: 2.9366 | LR: 2.88e-04 | 401.3ms/step | 49.8 samples/s +Validation at step 20250: Loss = 3.0003, Perplexity = 20.09 +Step 20260/44000 | Loss: 2.9688 | LR: 2.88e-04 | 570.6ms/step | 35.1 samples/s +Step 20270/44000 | Loss: 3.0273 | LR: 2.88e-04 | 402.2ms/step | 49.7 samples/s +Step 20280/44000 | Loss: 3.0741 | LR: 2.88e-04 | 401.9ms/step | 49.8 samples/s +Step 20290/44000 | Loss: 3.0523 | LR: 2.88e-04 | 402.2ms/step | 49.7 samples/s +Step 20300/44000 | Loss: 2.9821 | LR: 2.88e-04 | 402.3ms/step | 49.7 samples/s +Step 20310/44000 | Loss: 3.0090 | LR: 2.87e-04 | 403.1ms/step | 49.6 samples/s +Step 20320/44000 | Loss: 2.9465 | LR: 2.87e-04 | 402.5ms/step | 49.7 samples/s +Step 20330/44000 | Loss: 3.0020 | LR: 2.87e-04 | 403.4ms/step | 49.6 samples/s +Step 20340/44000 | Loss: 2.9865 | LR: 2.87e-04 | 402.6ms/step | 49.7 samples/s +Step 20350/44000 | Loss: 2.9652 | LR: 2.87e-04 | 402.5ms/step | 49.7 samples/s +Step 20360/44000 | Loss: 2.9962 | LR: 2.86e-04 | 402.0ms/step | 49.7 samples/s +Step 20370/44000 | Loss: 3.0339 | LR: 2.86e-04 | 402.0ms/step | 49.8 samples/s +Step 20380/44000 | Loss: 3.1116 | LR: 2.86e-04 | 401.4ms/step | 49.8 samples/s +Step 20390/44000 | Loss: 2.9845 | LR: 2.86e-04 | 400.6ms/step | 49.9 samples/s +Step 20400/44000 | Loss: 2.9595 | LR: 2.86e-04 | 401.8ms/step | 49.8 samples/s +Step 20410/44000 | Loss: 3.0254 | LR: 2.86e-04 | 401.8ms/step | 49.8 samples/s +Step 20420/44000 | Loss: 2.9498 | LR: 2.85e-04 | 400.6ms/step | 49.9 samples/s +Step 20430/44000 | Loss: 3.0645 | LR: 2.85e-04 | 403.1ms/step | 49.6 samples/s +Step 20440/44000 | Loss: 2.9591 | LR: 2.85e-04 | 402.2ms/step | 49.7 samples/s +Step 20450/44000 | Loss: 3.0003 | LR: 2.85e-04 | 400.5ms/step | 49.9 samples/s +Step 20460/44000 | Loss: 2.9787 | LR: 2.85e-04 | 403.0ms/step | 49.6 samples/s +Step 20470/44000 | Loss: 3.0226 | LR: 2.84e-04 | 400.9ms/step | 49.9 samples/s +Step 20480/44000 | Loss: 3.0127 | LR: 2.84e-04 | 402.1ms/step | 49.7 samples/s +Step 20490/44000 | Loss: 3.0260 | LR: 2.84e-04 | 401.6ms/step | 49.8 samples/s +Step 20500/44000 | Loss: 2.9854 | LR: 2.84e-04 | 401.4ms/step | 49.8 samples/s +Validation at step 20500: Loss = 2.9968, Perplexity = 20.02 +Step 20510/44000 | Loss: 3.0220 | LR: 2.84e-04 | 569.2ms/step | 35.1 samples/s +Step 20520/44000 | Loss: 3.0370 | LR: 2.84e-04 | 401.8ms/step | 49.8 samples/s +Step 20530/44000 | Loss: 3.0313 | LR: 2.83e-04 | 402.6ms/step | 49.7 samples/s +Step 20540/44000 | Loss: 3.0027 | LR: 2.83e-04 | 401.5ms/step | 49.8 samples/s +Step 20550/44000 | Loss: 2.9602 | LR: 2.83e-04 | 402.4ms/step | 49.7 samples/s +Step 20560/44000 | Loss: 3.0457 | LR: 2.83e-04 | 401.9ms/step | 49.8 samples/s +Step 20570/44000 | Loss: 2.9608 | LR: 2.83e-04 | 401.4ms/step | 49.8 samples/s +Step 20580/44000 | Loss: 3.0070 | LR: 2.83e-04 | 401.8ms/step | 49.8 samples/s +Step 20590/44000 | Loss: 3.0040 | LR: 2.82e-04 | 400.2ms/step | 50.0 samples/s +Step 20600/44000 | Loss: 3.0730 | LR: 2.82e-04 | 402.4ms/step | 49.7 samples/s +Step 20610/44000 | Loss: 2.9681 | LR: 2.82e-04 | 400.5ms/step | 49.9 samples/s +Step 20620/44000 | Loss: 3.0104 | LR: 2.82e-04 | 401.4ms/step | 49.8 samples/s +Step 20630/44000 | Loss: 2.9782 | LR: 2.82e-04 | 400.4ms/step | 50.0 samples/s +Step 20640/44000 | Loss: 3.0094 | LR: 2.81e-04 | 399.7ms/step | 50.0 samples/s +Step 20650/44000 | Loss: 3.0106 | LR: 2.81e-04 | 401.4ms/step | 49.8 samples/s +Step 20660/44000 | Loss: 2.9924 | LR: 2.81e-04 | 400.2ms/step | 50.0 samples/s +Step 20670/44000 | Loss: 2.9986 | LR: 2.81e-04 | 400.1ms/step | 50.0 samples/s +Step 20680/44000 | Loss: 3.0391 | LR: 2.81e-04 | 400.2ms/step | 50.0 samples/s +Step 20690/44000 | Loss: 2.9935 | LR: 2.81e-04 | 399.9ms/step | 50.0 samples/s +Step 20700/44000 | Loss: 2.9850 | LR: 2.80e-04 | 399.9ms/step | 50.0 samples/s +Step 20710/44000 | Loss: 2.9533 | LR: 2.80e-04 | 400.8ms/step | 49.9 samples/s +Step 20720/44000 | Loss: 3.0022 | LR: 2.80e-04 | 401.6ms/step | 49.8 samples/s +Step 20730/44000 | Loss: 3.0099 | LR: 2.80e-04 | 401.9ms/step | 49.8 samples/s +Step 20740/44000 | Loss: 3.0457 | LR: 2.80e-04 | 402.1ms/step | 49.7 samples/s +Step 20750/44000 | Loss: 2.9929 | LR: 2.79e-04 | 402.0ms/step | 49.8 samples/s +Validation at step 20750: Loss = 2.9960, Perplexity = 20.00 +Step 20760/44000 | Loss: 2.9804 | LR: 2.79e-04 | 569.4ms/step | 35.1 samples/s +Step 20770/44000 | Loss: 2.9959 | LR: 2.79e-04 | 401.5ms/step | 49.8 samples/s +Step 20780/44000 | Loss: 2.9794 | LR: 2.79e-04 | 402.3ms/step | 49.7 samples/s +Step 20790/44000 | Loss: 3.0013 | LR: 2.79e-04 | 401.0ms/step | 49.9 samples/s +Step 20800/44000 | Loss: 3.0095 | LR: 2.79e-04 | 401.0ms/step | 49.9 samples/s +Step 20810/44000 | Loss: 3.0127 | LR: 2.78e-04 | 400.1ms/step | 50.0 samples/s +Step 20820/44000 | Loss: 2.9347 | LR: 2.78e-04 | 400.6ms/step | 49.9 samples/s +Step 20830/44000 | Loss: 2.9855 | LR: 2.78e-04 | 401.7ms/step | 49.8 samples/s +Step 20840/44000 | Loss: 2.9432 | LR: 2.78e-04 | 401.8ms/step | 49.8 samples/s +Step 20850/44000 | Loss: 2.9448 | LR: 2.78e-04 | 401.4ms/step | 49.8 samples/s +Step 20860/44000 | Loss: 2.9574 | LR: 2.77e-04 | 400.6ms/step | 49.9 samples/s +Step 20870/44000 | Loss: 3.0396 | LR: 2.77e-04 | 403.0ms/step | 49.6 samples/s +Step 20880/44000 | Loss: 3.0445 | LR: 2.77e-04 | 403.5ms/step | 49.6 samples/s +Step 20890/44000 | Loss: 2.9358 | LR: 2.77e-04 | 405.1ms/step | 49.4 samples/s +Step 20900/44000 | Loss: 3.0043 | LR: 2.77e-04 | 404.9ms/step | 49.4 samples/s +Step 20910/44000 | Loss: 2.9888 | LR: 2.77e-04 | 403.3ms/step | 49.6 samples/s +Step 20920/44000 | Loss: 2.9854 | LR: 2.76e-04 | 402.9ms/step | 49.6 samples/s +Step 20930/44000 | Loss: 2.9970 | LR: 2.76e-04 | 402.9ms/step | 49.6 samples/s +Step 20940/44000 | Loss: 2.9695 | LR: 2.76e-04 | 402.9ms/step | 49.6 samples/s +Step 20950/44000 | Loss: 2.9911 | LR: 2.76e-04 | 403.2ms/step | 49.6 samples/s +Step 20960/44000 | Loss: 3.0207 | LR: 2.76e-04 | 403.1ms/step | 49.6 samples/s +Step 20970/44000 | Loss: 2.9339 | LR: 2.75e-04 | 402.7ms/step | 49.7 samples/s +Step 20980/44000 | Loss: 3.0032 | LR: 2.75e-04 | 402.4ms/step | 49.7 samples/s +Step 20990/44000 | Loss: 2.9882 | LR: 2.75e-04 | 399.8ms/step | 50.0 samples/s +Step 21000/44000 | Loss: 2.9913 | LR: 2.75e-04 | 400.5ms/step | 49.9 samples/s +Validation at step 21000: Loss = 2.9926, Perplexity = 19.94 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-21000 +Step 21010/44000 | Loss: 2.9545 | LR: 2.75e-04 | 882.7ms/step | 22.7 samples/s +Step 21020/44000 | Loss: 3.0098 | LR: 2.75e-04 | 403.4ms/step | 49.6 samples/s +Step 21030/44000 | Loss: 3.0037 | LR: 2.74e-04 | 402.7ms/step | 49.7 samples/s +Step 21040/44000 | Loss: 2.9838 | LR: 2.74e-04 | 402.8ms/step | 49.7 samples/s +Step 21050/44000 | Loss: 2.9747 | LR: 2.74e-04 | 400.7ms/step | 49.9 samples/s +Step 21060/44000 | Loss: 3.0172 | LR: 2.74e-04 | 403.7ms/step | 49.5 samples/s +Step 21070/44000 | Loss: 2.9797 | LR: 2.74e-04 | 403.4ms/step | 49.6 samples/s +Step 21080/44000 | Loss: 2.9828 | LR: 2.73e-04 | 404.4ms/step | 49.5 samples/s +Step 21090/44000 | Loss: 2.9823 | LR: 2.73e-04 | 403.6ms/step | 49.6 samples/s +Step 21100/44000 | Loss: 2.9610 | LR: 2.73e-04 | 403.4ms/step | 49.6 samples/s +Step 21110/44000 | Loss: 2.9968 | LR: 2.73e-04 | 403.8ms/step | 49.5 samples/s +Step 21120/44000 | Loss: 2.9580 | LR: 2.73e-04 | 403.7ms/step | 49.5 samples/s +Step 21130/44000 | Loss: 2.9651 | LR: 2.73e-04 | 403.7ms/step | 49.5 samples/s +Step 21140/44000 | Loss: 2.9695 | LR: 2.72e-04 | 403.9ms/step | 49.5 samples/s +Step 21150/44000 | Loss: 3.0442 | LR: 2.72e-04 | 402.6ms/step | 49.7 samples/s +Step 21160/44000 | Loss: 2.9259 | LR: 2.72e-04 | 402.5ms/step | 49.7 samples/s +Step 21170/44000 | Loss: 2.9903 | LR: 2.72e-04 | 403.7ms/step | 49.5 samples/s +Step 21180/44000 | Loss: 3.0008 | LR: 2.72e-04 | 403.4ms/step | 49.6 samples/s +Step 21190/44000 | Loss: 2.9664 | LR: 2.71e-04 | 404.2ms/step | 49.5 samples/s +Step 21200/44000 | Loss: 2.9369 | LR: 2.71e-04 | 403.8ms/step | 49.5 samples/s +Step 21210/44000 | Loss: 2.9209 | LR: 2.71e-04 | 403.4ms/step | 49.6 samples/s +Step 21220/44000 | Loss: 2.9964 | LR: 2.71e-04 | 403.0ms/step | 49.6 samples/s +Step 21230/44000 | Loss: 2.9871 | LR: 2.71e-04 | 403.9ms/step | 49.5 samples/s +Step 21240/44000 | Loss: 2.9916 | LR: 2.71e-04 | 404.4ms/step | 49.5 samples/s +Step 21250/44000 | Loss: 3.0029 | LR: 2.70e-04 | 404.3ms/step | 49.5 samples/s +Validation at step 21250: Loss = 2.9889, Perplexity = 19.86 +Step 21260/44000 | Loss: 2.9387 | LR: 2.70e-04 | 573.3ms/step | 34.9 samples/s +Step 21270/44000 | Loss: 2.9931 | LR: 2.70e-04 | 404.1ms/step | 49.5 samples/s +Step 21280/44000 | Loss: 2.9651 | LR: 2.70e-04 | 403.3ms/step | 49.6 samples/s +Step 21290/44000 | Loss: 3.0888 | LR: 2.70e-04 | 404.0ms/step | 49.5 samples/s +Step 21300/44000 | Loss: 2.9795 | LR: 2.70e-04 | 403.4ms/step | 49.6 samples/s +Step 21310/44000 | Loss: 2.9425 | LR: 2.69e-04 | 403.9ms/step | 49.5 samples/s +Step 21320/44000 | Loss: 2.9886 | LR: 2.69e-04 | 403.0ms/step | 49.6 samples/s +Step 21330/44000 | Loss: 2.9779 | LR: 2.69e-04 | 403.7ms/step | 49.5 samples/s +Step 21340/44000 | Loss: 3.0119 | LR: 2.69e-04 | 404.9ms/step | 49.4 samples/s +Step 21350/44000 | Loss: 3.0265 | LR: 2.69e-04 | 405.4ms/step | 49.3 samples/s +Step 21360/44000 | Loss: 3.0000 | LR: 2.68e-04 | 403.8ms/step | 49.5 samples/s +Step 21370/44000 | Loss: 2.9910 | LR: 2.68e-04 | 404.1ms/step | 49.5 samples/s +Step 21380/44000 | Loss: 2.9606 | LR: 2.68e-04 | 404.8ms/step | 49.4 samples/s +Step 21390/44000 | Loss: 3.0331 | LR: 2.68e-04 | 404.2ms/step | 49.5 samples/s +Step 21400/44000 | Loss: 2.9999 | LR: 2.68e-04 | 404.3ms/step | 49.5 samples/s +Step 21410/44000 | Loss: 3.0222 | LR: 2.68e-04 | 404.1ms/step | 49.5 samples/s +Step 21420/44000 | Loss: 3.0425 | LR: 2.67e-04 | 404.4ms/step | 49.5 samples/s +Step 21430/44000 | Loss: 2.9808 | LR: 2.67e-04 | 404.2ms/step | 49.5 samples/s +Step 21440/44000 | Loss: 2.9977 | LR: 2.67e-04 | 404.5ms/step | 49.4 samples/s +Step 21450/44000 | Loss: 2.9814 | LR: 2.67e-04 | 403.6ms/step | 49.6 samples/s +Step 21460/44000 | Loss: 2.9558 | LR: 2.67e-04 | 402.2ms/step | 49.7 samples/s +Step 21470/44000 | Loss: 2.9991 | LR: 2.66e-04 | 402.2ms/step | 49.7 samples/s +Step 21480/44000 | Loss: 3.0673 | LR: 2.66e-04 | 401.2ms/step | 49.9 samples/s +Step 21490/44000 | Loss: 3.0272 | LR: 2.66e-04 | 402.3ms/step | 49.7 samples/s +Step 21500/44000 | Loss: 2.9933 | LR: 2.66e-04 | 403.6ms/step | 49.6 samples/s +Validation at step 21500: Loss = 2.9890, Perplexity = 19.87 +Step 21510/44000 | Loss: 3.0187 | LR: 2.66e-04 | 572.8ms/step | 34.9 samples/s +Step 21520/44000 | Loss: 2.9895 | LR: 2.66e-04 | 405.4ms/step | 49.3 samples/s +Step 21530/44000 | Loss: 2.9289 | LR: 2.65e-04 | 403.4ms/step | 49.6 samples/s +Step 21540/44000 | Loss: 2.9278 | LR: 2.65e-04 | 404.6ms/step | 49.4 samples/s +Step 21550/44000 | Loss: 2.9365 | LR: 2.65e-04 | 403.1ms/step | 49.6 samples/s +Step 21560/44000 | Loss: 2.8923 | LR: 2.65e-04 | 403.7ms/step | 49.5 samples/s +Step 21570/44000 | Loss: 2.9432 | LR: 2.65e-04 | 403.1ms/step | 49.6 samples/s +Step 21580/44000 | Loss: 2.9616 | LR: 2.64e-04 | 400.2ms/step | 50.0 samples/s +Step 21590/44000 | Loss: 2.9354 | LR: 2.64e-04 | 400.9ms/step | 49.9 samples/s +Step 21600/44000 | Loss: 3.0167 | LR: 2.64e-04 | 403.6ms/step | 49.6 samples/s +Step 21610/44000 | Loss: 2.9788 | LR: 2.64e-04 | 402.9ms/step | 49.6 samples/s +Step 21620/44000 | Loss: 2.9416 | LR: 2.64e-04 | 403.5ms/step | 49.6 samples/s +Step 21630/44000 | Loss: 3.0407 | LR: 2.64e-04 | 403.9ms/step | 49.5 samples/s +Step 21640/44000 | Loss: 2.9530 | LR: 2.63e-04 | 404.0ms/step | 49.5 samples/s +Step 21650/44000 | Loss: 3.0558 | LR: 2.63e-04 | 404.4ms/step | 49.5 samples/s +Step 21660/44000 | Loss: 3.0128 | LR: 2.63e-04 | 404.0ms/step | 49.5 samples/s +Step 21670/44000 | Loss: 2.9552 | LR: 2.63e-04 | 403.5ms/step | 49.6 samples/s +Step 21680/44000 | Loss: 2.9668 | LR: 2.63e-04 | 403.2ms/step | 49.6 samples/s +Step 21690/44000 | Loss: 2.9594 | LR: 2.62e-04 | 404.0ms/step | 49.5 samples/s +Step 21700/44000 | Loss: 3.0305 | LR: 2.62e-04 | 404.5ms/step | 49.4 samples/s +Step 21710/44000 | Loss: 3.0045 | LR: 2.62e-04 | 404.4ms/step | 49.5 samples/s +Step 21720/44000 | Loss: 2.9820 | LR: 2.62e-04 | 403.8ms/step | 49.5 samples/s +Step 21730/44000 | Loss: 3.0028 | LR: 2.62e-04 | 404.0ms/step | 49.5 samples/s +Step 21740/44000 | Loss: 3.0503 | LR: 2.62e-04 | 404.9ms/step | 49.4 samples/s +Step 21750/44000 | Loss: 3.0156 | LR: 2.61e-04 | 403.4ms/step | 49.6 samples/s +Validation at step 21750: Loss = 2.9857, Perplexity = 19.80 +Step 21760/44000 | Loss: 3.0011 | LR: 2.61e-04 | 572.0ms/step | 35.0 samples/s +Step 21770/44000 | Loss: 2.9499 | LR: 2.61e-04 | 403.5ms/step | 49.6 samples/s +Step 21780/44000 | Loss: 3.0452 | LR: 2.61e-04 | 404.4ms/step | 49.5 samples/s +Step 21790/44000 | Loss: 2.9915 | LR: 2.61e-04 | 403.6ms/step | 49.6 samples/s +Step 21800/44000 | Loss: 2.9532 | LR: 2.60e-04 | 404.4ms/step | 49.5 samples/s +Step 21810/44000 | Loss: 2.9099 | LR: 2.60e-04 | 405.4ms/step | 49.3 samples/s +Step 21820/44000 | Loss: 2.9992 | LR: 2.60e-04 | 405.2ms/step | 49.4 samples/s +Step 21830/44000 | Loss: 2.9485 | LR: 2.60e-04 | 404.5ms/step | 49.4 samples/s +Step 21840/44000 | Loss: 2.9725 | LR: 2.60e-04 | 404.8ms/step | 49.4 samples/s +Step 21850/44000 | Loss: 3.0193 | LR: 2.60e-04 | 404.0ms/step | 49.5 samples/s +Step 21860/44000 | Loss: 3.0078 | LR: 2.59e-04 | 404.0ms/step | 49.5 samples/s +Step 21870/44000 | Loss: 2.9745 | LR: 2.59e-04 | 404.5ms/step | 49.4 samples/s +Step 21880/44000 | Loss: 3.0548 | LR: 2.59e-04 | 403.9ms/step | 49.5 samples/s +Step 21890/44000 | Loss: 3.0302 | LR: 2.59e-04 | 404.1ms/step | 49.5 samples/s +Step 21900/44000 | Loss: 3.0007 | LR: 2.59e-04 | 403.7ms/step | 49.5 samples/s +Step 21910/44000 | Loss: 2.9281 | LR: 2.58e-04 | 405.0ms/step | 49.4 samples/s +Step 21920/44000 | Loss: 3.0132 | LR: 2.58e-04 | 404.5ms/step | 49.4 samples/s +Step 21930/44000 | Loss: 2.9661 | LR: 2.58e-04 | 405.6ms/step | 49.3 samples/s +Step 21940/44000 | Loss: 3.0112 | LR: 2.58e-04 | 403.4ms/step | 49.6 samples/s +Step 21950/44000 | Loss: 2.9470 | LR: 2.58e-04 | 402.4ms/step | 49.7 samples/s +Step 21960/44000 | Loss: 2.9810 | LR: 2.58e-04 | 403.3ms/step | 49.6 samples/s +Step 21970/44000 | Loss: 3.0374 | LR: 2.57e-04 | 406.0ms/step | 49.3 samples/s +Step 21980/44000 | Loss: 2.9991 | LR: 2.57e-04 | 404.6ms/step | 49.4 samples/s +Step 21990/44000 | Loss: 3.0378 | LR: 2.57e-04 | 405.1ms/step | 49.4 samples/s +Step 22000/44000 | Loss: 3.0091 | LR: 2.57e-04 | 404.8ms/step | 49.4 samples/s +Validation at step 22000: Loss = 2.9820, Perplexity = 19.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-22000 +Step 22010/44000 | Loss: 2.9687 | LR: 2.57e-04 | 885.2ms/step | 22.6 samples/s +Step 22020/44000 | Loss: 3.0507 | LR: 2.56e-04 | 403.7ms/step | 49.5 samples/s +Step 22030/44000 | Loss: 3.0137 | LR: 2.56e-04 | 404.3ms/step | 49.5 samples/s +Step 22040/44000 | Loss: 2.9944 | LR: 2.56e-04 | 403.9ms/step | 49.5 samples/s +Step 22050/44000 | Loss: 2.9866 | LR: 2.56e-04 | 404.0ms/step | 49.5 samples/s +Step 22060/44000 | Loss: 2.9689 | LR: 2.56e-04 | 404.6ms/step | 49.4 samples/s +Step 22070/44000 | Loss: 3.0286 | LR: 2.56e-04 | 404.6ms/step | 49.4 samples/s +Step 22080/44000 | Loss: 2.9628 | LR: 2.55e-04 | 404.6ms/step | 49.4 samples/s +Step 22090/44000 | Loss: 2.9540 | LR: 2.55e-04 | 403.9ms/step | 49.5 samples/s +Step 22100/44000 | Loss: 3.0368 | LR: 2.55e-04 | 403.9ms/step | 49.5 samples/s +Step 22110/44000 | Loss: 2.9804 | LR: 2.55e-04 | 403.4ms/step | 49.6 samples/s +Step 22120/44000 | Loss: 2.9523 | LR: 2.55e-04 | 404.5ms/step | 49.4 samples/s +Step 22130/44000 | Loss: 2.9200 | LR: 2.54e-04 | 405.1ms/step | 49.4 samples/s +Step 22140/44000 | Loss: 2.9837 | LR: 2.54e-04 | 404.8ms/step | 49.4 samples/s +Step 22150/44000 | Loss: 2.9876 | LR: 2.54e-04 | 404.4ms/step | 49.5 samples/s +Step 22160/44000 | Loss: 2.9399 | LR: 2.54e-04 | 404.2ms/step | 49.5 samples/s +Step 22170/44000 | Loss: 3.0231 | LR: 2.54e-04 | 404.7ms/step | 49.4 samples/s +Step 22180/44000 | Loss: 3.0027 | LR: 2.54e-04 | 404.6ms/step | 49.4 samples/s +Step 22190/44000 | Loss: 2.9684 | LR: 2.53e-04 | 403.9ms/step | 49.5 samples/s +Step 22200/44000 | Loss: 3.0190 | LR: 2.53e-04 | 404.1ms/step | 49.5 samples/s +Step 22210/44000 | Loss: 2.9801 | LR: 2.53e-04 | 405.1ms/step | 49.4 samples/s +Step 22220/44000 | Loss: 3.0077 | LR: 2.53e-04 | 405.7ms/step | 49.3 samples/s +Step 22230/44000 | Loss: 2.9906 | LR: 2.53e-04 | 404.3ms/step | 49.5 samples/s +Step 22240/44000 | Loss: 2.9655 | LR: 2.52e-04 | 404.3ms/step | 49.5 samples/s +Step 22250/44000 | Loss: 2.9643 | LR: 2.52e-04 | 405.0ms/step | 49.4 samples/s +Validation at step 22250: Loss = 2.9798, Perplexity = 19.68 +Step 22260/44000 | Loss: 2.9553 | LR: 2.52e-04 | 572.6ms/step | 34.9 samples/s +Step 22270/44000 | Loss: 2.9605 | LR: 2.52e-04 | 404.8ms/step | 49.4 samples/s +Step 22280/44000 | Loss: 3.0057 | LR: 2.52e-04 | 404.4ms/step | 49.5 samples/s +Step 22290/44000 | Loss: 2.9788 | LR: 2.52e-04 | 404.9ms/step | 49.4 samples/s +Step 22300/44000 | Loss: 3.0392 | LR: 2.51e-04 | 404.8ms/step | 49.4 samples/s +Step 22310/44000 | Loss: 3.0045 | LR: 2.51e-04 | 403.8ms/step | 49.5 samples/s +Step 22320/44000 | Loss: 2.9571 | LR: 2.51e-04 | 404.4ms/step | 49.5 samples/s +Step 22330/44000 | Loss: 3.0412 | LR: 2.51e-04 | 404.2ms/step | 49.5 samples/s +Step 22340/44000 | Loss: 3.0001 | LR: 2.51e-04 | 404.1ms/step | 49.5 samples/s +Step 22350/44000 | Loss: 2.9634 | LR: 2.50e-04 | 403.6ms/step | 49.6 samples/s +Step 22360/44000 | Loss: 3.0061 | LR: 2.50e-04 | 404.5ms/step | 49.4 samples/s +Step 22370/44000 | Loss: 2.9223 | LR: 2.50e-04 | 401.5ms/step | 49.8 samples/s +Step 22380/44000 | Loss: 3.0336 | LR: 2.50e-04 | 400.4ms/step | 50.0 samples/s +Step 22390/44000 | Loss: 2.9963 | LR: 2.50e-04 | 401.5ms/step | 49.8 samples/s +Step 22400/44000 | Loss: 2.9989 | LR: 2.50e-04 | 402.1ms/step | 49.7 samples/s +Step 22410/44000 | Loss: 2.9968 | LR: 2.49e-04 | 402.9ms/step | 49.6 samples/s +Step 22420/44000 | Loss: 2.9148 | LR: 2.49e-04 | 404.1ms/step | 49.5 samples/s +Step 22430/44000 | Loss: 2.9865 | LR: 2.49e-04 | 404.4ms/step | 49.5 samples/s +Step 22440/44000 | Loss: 3.0336 | LR: 2.49e-04 | 404.2ms/step | 49.5 samples/s +Step 22450/44000 | Loss: 3.0104 | LR: 2.49e-04 | 404.4ms/step | 49.5 samples/s +Step 22460/44000 | Loss: 2.9795 | LR: 2.48e-04 | 404.5ms/step | 49.4 samples/s +Step 22470/44000 | Loss: 2.9210 | LR: 2.48e-04 | 410.1ms/step | 48.8 samples/s +Step 22480/44000 | Loss: 2.9832 | LR: 2.48e-04 | 417.7ms/step | 47.9 samples/s +Step 22490/44000 | Loss: 2.9950 | LR: 2.48e-04 | 418.0ms/step | 47.8 samples/s +Step 22500/44000 | Loss: 2.9725 | LR: 2.48e-04 | 404.7ms/step | 49.4 samples/s +Validation at step 22500: Loss = 2.9777, Perplexity = 19.64 +Step 22510/44000 | Loss: 2.9475 | LR: 2.48e-04 | 572.7ms/step | 34.9 samples/s +Step 22520/44000 | Loss: 2.9397 | LR: 2.47e-04 | 404.9ms/step | 49.4 samples/s +Step 22530/44000 | Loss: 2.9863 | LR: 2.47e-04 | 404.9ms/step | 49.4 samples/s +Step 22540/44000 | Loss: 2.9657 | LR: 2.47e-04 | 402.3ms/step | 49.7 samples/s +Step 22550/44000 | Loss: 2.9893 | LR: 2.47e-04 | 402.3ms/step | 49.7 samples/s +Step 22560/44000 | Loss: 2.9448 | LR: 2.47e-04 | 402.1ms/step | 49.7 samples/s +Step 22570/44000 | Loss: 3.0258 | LR: 2.46e-04 | 402.4ms/step | 49.7 samples/s +Step 22580/44000 | Loss: 2.9736 | LR: 2.46e-04 | 402.3ms/step | 49.7 samples/s +Step 22590/44000 | Loss: 2.9403 | LR: 2.46e-04 | 402.1ms/step | 49.7 samples/s +Step 22600/44000 | Loss: 2.9733 | LR: 2.46e-04 | 402.4ms/step | 49.7 samples/s +Step 22610/44000 | Loss: 2.9820 | LR: 2.46e-04 | 401.6ms/step | 49.8 samples/s +Step 22620/44000 | Loss: 3.0087 | LR: 2.46e-04 | 404.3ms/step | 49.5 samples/s +Step 22630/44000 | Loss: 2.9794 | LR: 2.45e-04 | 404.1ms/step | 49.5 samples/s +Step 22640/44000 | Loss: 3.0179 | LR: 2.45e-04 | 405.1ms/step | 49.4 samples/s +Step 22650/44000 | Loss: 3.0086 | LR: 2.45e-04 | 406.2ms/step | 49.2 samples/s +Step 22660/44000 | Loss: 2.9068 | LR: 2.45e-04 | 404.9ms/step | 49.4 samples/s +Step 22670/44000 | Loss: 2.9803 | LR: 2.45e-04 | 404.7ms/step | 49.4 samples/s +Step 22680/44000 | Loss: 2.9012 | LR: 2.44e-04 | 405.1ms/step | 49.4 samples/s +Step 22690/44000 | Loss: 2.9444 | LR: 2.44e-04 | 404.9ms/step | 49.4 samples/s +Step 22700/44000 | Loss: 2.9272 | LR: 2.44e-04 | 404.1ms/step | 49.5 samples/s +Step 22710/44000 | Loss: 2.9802 | LR: 2.44e-04 | 404.6ms/step | 49.4 samples/s +Step 22720/44000 | Loss: 2.9243 | LR: 2.44e-04 | 405.8ms/step | 49.3 samples/s +Step 22730/44000 | Loss: 2.9895 | LR: 2.44e-04 | 405.2ms/step | 49.4 samples/s +Step 22740/44000 | Loss: 2.9739 | LR: 2.43e-04 | 404.2ms/step | 49.5 samples/s +Step 22750/44000 | Loss: 3.0283 | LR: 2.43e-04 | 405.7ms/step | 49.3 samples/s +Validation at step 22750: Loss = 2.9747, Perplexity = 19.58 +Step 22760/44000 | Loss: 2.9451 | LR: 2.43e-04 | 572.1ms/step | 35.0 samples/s +Step 22770/44000 | Loss: 2.9748 | LR: 2.43e-04 | 403.5ms/step | 49.6 samples/s +Step 22780/44000 | Loss: 2.9908 | LR: 2.43e-04 | 405.2ms/step | 49.4 samples/s +Step 22790/44000 | Loss: 3.0124 | LR: 2.42e-04 | 404.6ms/step | 49.4 samples/s +Step 22800/44000 | Loss: 2.9886 | LR: 2.42e-04 | 405.3ms/step | 49.3 samples/s +Step 22810/44000 | Loss: 3.0475 | LR: 2.42e-04 | 404.4ms/step | 49.5 samples/s +Step 22820/44000 | Loss: 2.9033 | LR: 2.42e-04 | 404.1ms/step | 49.5 samples/s +Step 22830/44000 | Loss: 3.0193 | LR: 2.42e-04 | 404.4ms/step | 49.5 samples/s +Step 22840/44000 | Loss: 2.9140 | LR: 2.42e-04 | 404.7ms/step | 49.4 samples/s +Step 22850/44000 | Loss: 2.9255 | LR: 2.41e-04 | 404.0ms/step | 49.5 samples/s +Step 22860/44000 | Loss: 2.9924 | LR: 2.41e-04 | 404.3ms/step | 49.5 samples/s +Step 22870/44000 | Loss: 2.9756 | LR: 2.41e-04 | 405.3ms/step | 49.3 samples/s +Step 22880/44000 | Loss: 2.9467 | LR: 2.41e-04 | 404.4ms/step | 49.5 samples/s +Step 22890/44000 | Loss: 2.9966 | LR: 2.41e-04 | 403.7ms/step | 49.5 samples/s +Step 22900/44000 | Loss: 2.9906 | LR: 2.40e-04 | 404.1ms/step | 49.5 samples/s +Step 22910/44000 | Loss: 2.9967 | LR: 2.40e-04 | 404.8ms/step | 49.4 samples/s +Step 22920/44000 | Loss: 2.9575 | LR: 2.40e-04 | 403.4ms/step | 49.6 samples/s +Step 22930/44000 | Loss: 2.9824 | LR: 2.40e-04 | 404.0ms/step | 49.5 samples/s +Step 22940/44000 | Loss: 2.9245 | LR: 2.40e-04 | 403.7ms/step | 49.5 samples/s +Step 22950/44000 | Loss: 2.9971 | LR: 2.40e-04 | 406.5ms/step | 49.2 samples/s +Step 22960/44000 | Loss: 2.9753 | LR: 2.39e-04 | 404.2ms/step | 49.5 samples/s +Step 22970/44000 | Loss: 2.9175 | LR: 2.39e-04 | 404.8ms/step | 49.4 samples/s +Step 22980/44000 | Loss: 2.9958 | LR: 2.39e-04 | 403.8ms/step | 49.5 samples/s +Step 22990/44000 | Loss: 2.9589 | LR: 2.39e-04 | 404.2ms/step | 49.5 samples/s +Step 23000/44000 | Loss: 2.9751 | LR: 2.39e-04 | 404.9ms/step | 49.4 samples/s +Validation at step 23000: Loss = 2.9735, Perplexity = 19.56 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-23000 +Step 23010/44000 | Loss: 2.9517 | LR: 2.38e-04 | 883.3ms/step | 22.6 samples/s +Step 23020/44000 | Loss: 2.9655 | LR: 2.38e-04 | 403.9ms/step | 49.5 samples/s +Step 23030/44000 | Loss: 2.9906 | LR: 2.38e-04 | 403.1ms/step | 49.6 samples/s +Step 23040/44000 | Loss: 2.9831 | LR: 2.38e-04 | 404.1ms/step | 49.5 samples/s +Step 23050/44000 | Loss: 3.0313 | LR: 2.38e-04 | 403.9ms/step | 49.5 samples/s +Step 23060/44000 | Loss: 2.9367 | LR: 2.38e-04 | 404.4ms/step | 49.5 samples/s +Step 23070/44000 | Loss: 2.9362 | LR: 2.37e-04 | 403.5ms/step | 49.6 samples/s +Step 23080/44000 | Loss: 3.0053 | LR: 2.37e-04 | 404.6ms/step | 49.4 samples/s +Step 23090/44000 | Loss: 2.9430 | LR: 2.37e-04 | 407.1ms/step | 49.1 samples/s +Step 23100/44000 | Loss: 2.9965 | LR: 2.37e-04 | 403.6ms/step | 49.6 samples/s +Step 23110/44000 | Loss: 2.9680 | LR: 2.37e-04 | 404.5ms/step | 49.4 samples/s +Step 23120/44000 | Loss: 2.9998 | LR: 2.36e-04 | 404.0ms/step | 49.5 samples/s +Step 23130/44000 | Loss: 3.0015 | LR: 2.36e-04 | 404.5ms/step | 49.4 samples/s +Step 23140/44000 | Loss: 3.0147 | LR: 2.36e-04 | 404.6ms/step | 49.4 samples/s +Step 23150/44000 | Loss: 2.9829 | LR: 2.36e-04 | 405.2ms/step | 49.4 samples/s +Step 23160/44000 | Loss: 3.0142 | LR: 2.36e-04 | 404.8ms/step | 49.4 samples/s +Step 23170/44000 | Loss: 3.0208 | LR: 2.36e-04 | 404.5ms/step | 49.4 samples/s +Step 23180/44000 | Loss: 2.9732 | LR: 2.35e-04 | 404.6ms/step | 49.4 samples/s +Step 23190/44000 | Loss: 2.9376 | LR: 2.35e-04 | 403.7ms/step | 49.5 samples/s +Step 23200/44000 | Loss: 2.9530 | LR: 2.35e-04 | 404.4ms/step | 49.5 samples/s +Step 23210/44000 | Loss: 2.9527 | LR: 2.35e-04 | 405.6ms/step | 49.3 samples/s +Step 23220/44000 | Loss: 3.0000 | LR: 2.35e-04 | 404.2ms/step | 49.5 samples/s +Step 23230/44000 | Loss: 2.9852 | LR: 2.34e-04 | 404.3ms/step | 49.5 samples/s +Step 23240/44000 | Loss: 2.9645 | LR: 2.34e-04 | 405.0ms/step | 49.4 samples/s +Step 23250/44000 | Loss: 2.9834 | LR: 2.34e-04 | 404.5ms/step | 49.4 samples/s +Validation at step 23250: Loss = 2.9716, Perplexity = 19.52 +Step 23260/44000 | Loss: 2.9848 | LR: 2.34e-04 | 570.9ms/step | 35.0 samples/s +Step 23270/44000 | Loss: 2.9476 | LR: 2.34e-04 | 404.8ms/step | 49.4 samples/s +Step 23280/44000 | Loss: 2.9506 | LR: 2.34e-04 | 403.6ms/step | 49.6 samples/s +Step 23290/44000 | Loss: 2.9759 | LR: 2.33e-04 | 405.4ms/step | 49.3 samples/s +Step 23300/44000 | Loss: 2.9609 | LR: 2.33e-04 | 405.2ms/step | 49.4 samples/s +Step 23310/44000 | Loss: 2.9827 | LR: 2.33e-04 | 405.0ms/step | 49.4 samples/s +Step 23320/44000 | Loss: 2.9226 | LR: 2.33e-04 | 404.4ms/step | 49.5 samples/s +Step 23330/44000 | Loss: 2.9682 | LR: 2.33e-04 | 404.8ms/step | 49.4 samples/s +Step 23340/44000 | Loss: 2.9297 | LR: 2.32e-04 | 405.0ms/step | 49.4 samples/s +Step 23350/44000 | Loss: 2.9969 | LR: 2.32e-04 | 403.5ms/step | 49.6 samples/s +Step 23360/44000 | Loss: 2.9986 | LR: 2.32e-04 | 405.4ms/step | 49.3 samples/s +Step 23370/44000 | Loss: 2.8926 | LR: 2.32e-04 | 403.5ms/step | 49.6 samples/s +Step 23380/44000 | Loss: 2.9058 | LR: 2.32e-04 | 403.8ms/step | 49.5 samples/s +Step 23390/44000 | Loss: 2.9439 | LR: 2.32e-04 | 404.1ms/step | 49.5 samples/s +Step 23400/44000 | Loss: 2.9793 | LR: 2.31e-04 | 404.9ms/step | 49.4 samples/s +Step 23410/44000 | Loss: 3.0906 | LR: 2.31e-04 | 404.5ms/step | 49.4 samples/s +Step 23420/44000 | Loss: 2.9771 | LR: 2.31e-04 | 405.2ms/step | 49.4 samples/s +Step 23430/44000 | Loss: 2.9845 | LR: 2.31e-04 | 404.0ms/step | 49.5 samples/s +Step 23440/44000 | Loss: 2.9610 | LR: 2.31e-04 | 403.3ms/step | 49.6 samples/s +Step 23450/44000 | Loss: 3.0369 | LR: 2.30e-04 | 404.4ms/step | 49.5 samples/s +Step 23460/44000 | Loss: 2.9682 | LR: 2.30e-04 | 405.1ms/step | 49.4 samples/s +Step 23470/44000 | Loss: 2.9279 | LR: 2.30e-04 | 404.7ms/step | 49.4 samples/s +Step 23480/44000 | Loss: 2.9146 | LR: 2.30e-04 | 405.5ms/step | 49.3 samples/s +Step 23490/44000 | Loss: 2.9161 | LR: 2.30e-04 | 404.3ms/step | 49.5 samples/s +Step 23500/44000 | Loss: 3.0026 | LR: 2.30e-04 | 405.6ms/step | 49.3 samples/s +Validation at step 23500: Loss = 2.9697, Perplexity = 19.49 +Step 23510/44000 | Loss: 2.9622 | LR: 2.29e-04 | 573.0ms/step | 34.9 samples/s +Step 23520/44000 | Loss: 2.9787 | LR: 2.29e-04 | 404.5ms/step | 49.4 samples/s +Step 23530/44000 | Loss: 2.9564 | LR: 2.29e-04 | 404.5ms/step | 49.4 samples/s +Step 23540/44000 | Loss: 2.9644 | LR: 2.29e-04 | 403.8ms/step | 49.5 samples/s +Step 23550/44000 | Loss: 3.0028 | LR: 2.29e-04 | 404.2ms/step | 49.5 samples/s +Step 23560/44000 | Loss: 3.0084 | LR: 2.29e-04 | 403.4ms/step | 49.6 samples/s +Step 23570/44000 | Loss: 2.9996 | LR: 2.28e-04 | 403.4ms/step | 49.6 samples/s +Step 23580/44000 | Loss: 3.0088 | LR: 2.28e-04 | 404.7ms/step | 49.4 samples/s +Step 23590/44000 | Loss: 2.9662 | LR: 2.28e-04 | 402.8ms/step | 49.6 samples/s +Step 23600/44000 | Loss: 2.9329 | LR: 2.28e-04 | 403.6ms/step | 49.5 samples/s +Step 23610/44000 | Loss: 2.9088 | LR: 2.28e-04 | 403.8ms/step | 49.5 samples/s +Step 23620/44000 | Loss: 2.9999 | LR: 2.27e-04 | 403.9ms/step | 49.5 samples/s +Step 23630/44000 | Loss: 2.9320 | LR: 2.27e-04 | 403.7ms/step | 49.5 samples/s +Step 23640/44000 | Loss: 2.9833 | LR: 2.27e-04 | 403.7ms/step | 49.5 samples/s +Step 23650/44000 | Loss: 2.9397 | LR: 2.27e-04 | 403.7ms/step | 49.5 samples/s +Step 23660/44000 | Loss: 2.9655 | LR: 2.27e-04 | 403.0ms/step | 49.6 samples/s +Step 23670/44000 | Loss: 2.9451 | LR: 2.27e-04 | 403.1ms/step | 49.6 samples/s +Step 23680/44000 | Loss: 2.9485 | LR: 2.26e-04 | 403.8ms/step | 49.5 samples/s +Step 23690/44000 | Loss: 2.9485 | LR: 2.26e-04 | 403.7ms/step | 49.5 samples/s +Step 23700/44000 | Loss: 3.0129 | LR: 2.26e-04 | 402.9ms/step | 49.6 samples/s +Step 23710/44000 | Loss: 2.9662 | LR: 2.26e-04 | 403.0ms/step | 49.6 samples/s +Step 23720/44000 | Loss: 3.0406 | LR: 2.26e-04 | 403.5ms/step | 49.6 samples/s +Step 23730/44000 | Loss: 2.9419 | LR: 2.25e-04 | 403.7ms/step | 49.5 samples/s +Step 23740/44000 | Loss: 2.9369 | LR: 2.25e-04 | 404.2ms/step | 49.5 samples/s +Step 23750/44000 | Loss: 2.9758 | LR: 2.25e-04 | 403.6ms/step | 49.6 samples/s +Validation at step 23750: Loss = 2.9662, Perplexity = 19.42 +Step 23760/44000 | Loss: 2.9761 | LR: 2.25e-04 | 570.9ms/step | 35.0 samples/s +Step 23770/44000 | Loss: 3.0336 | LR: 2.25e-04 | 404.3ms/step | 49.5 samples/s +Step 23780/44000 | Loss: 3.0294 | LR: 2.25e-04 | 404.1ms/step | 49.5 samples/s +Step 23790/44000 | Loss: 2.9804 | LR: 2.24e-04 | 404.4ms/step | 49.5 samples/s +Step 23800/44000 | Loss: 2.9562 | LR: 2.24e-04 | 404.6ms/step | 49.4 samples/s +Step 23810/44000 | Loss: 2.9328 | LR: 2.24e-04 | 403.8ms/step | 49.5 samples/s +Step 23820/44000 | Loss: 2.9856 | LR: 2.24e-04 | 403.3ms/step | 49.6 samples/s +Step 23830/44000 | Loss: 2.9489 | LR: 2.24e-04 | 401.7ms/step | 49.8 samples/s +Step 23840/44000 | Loss: 2.9672 | LR: 2.23e-04 | 401.0ms/step | 49.9 samples/s +Step 23850/44000 | Loss: 2.9768 | LR: 2.23e-04 | 404.5ms/step | 49.4 samples/s +Step 23860/44000 | Loss: 2.9717 | LR: 2.23e-04 | 403.9ms/step | 49.5 samples/s +Step 23870/44000 | Loss: 2.9770 | LR: 2.23e-04 | 404.4ms/step | 49.5 samples/s +Step 23880/44000 | Loss: 2.9408 | LR: 2.23e-04 | 403.5ms/step | 49.6 samples/s +Step 23890/44000 | Loss: 2.9556 | LR: 2.23e-04 | 404.0ms/step | 49.5 samples/s +Step 23900/44000 | Loss: 2.9552 | LR: 2.22e-04 | 403.4ms/step | 49.6 samples/s +Step 23910/44000 | Loss: 2.9982 | LR: 2.22e-04 | 401.0ms/step | 49.9 samples/s +Step 23920/44000 | Loss: 3.0355 | LR: 2.22e-04 | 400.2ms/step | 50.0 samples/s +Step 23930/44000 | Loss: 2.9589 | LR: 2.22e-04 | 400.9ms/step | 49.9 samples/s +Step 23940/44000 | Loss: 2.9363 | LR: 2.22e-04 | 400.2ms/step | 50.0 samples/s +Step 23950/44000 | Loss: 2.9437 | LR: 2.21e-04 | 400.1ms/step | 50.0 samples/s +Step 23960/44000 | Loss: 2.9415 | LR: 2.21e-04 | 399.8ms/step | 50.0 samples/s +Step 23970/44000 | Loss: 2.9168 | LR: 2.21e-04 | 399.9ms/step | 50.0 samples/s +Step 23980/44000 | Loss: 2.9146 | LR: 2.21e-04 | 400.2ms/step | 50.0 samples/s +Step 23990/44000 | Loss: 2.9498 | LR: 2.21e-04 | 401.4ms/step | 49.8 samples/s +Step 24000/44000 | Loss: 3.0444 | LR: 2.21e-04 | 403.0ms/step | 49.6 samples/s +Validation at step 24000: Loss = 2.9643, Perplexity = 19.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-24000 +Step 24010/44000 | Loss: 2.9838 | LR: 2.20e-04 | 869.5ms/step | 23.0 samples/s +Step 24020/44000 | Loss: 2.8937 | LR: 2.20e-04 | 404.0ms/step | 49.5 samples/s +Step 24030/44000 | Loss: 2.9605 | LR: 2.20e-04 | 404.2ms/step | 49.5 samples/s +Step 24040/44000 | Loss: 2.9408 | LR: 2.20e-04 | 404.3ms/step | 49.5 samples/s +Step 24050/44000 | Loss: 3.0000 | LR: 2.20e-04 | 404.2ms/step | 49.5 samples/s +Step 24060/44000 | Loss: 2.9093 | LR: 2.19e-04 | 404.7ms/step | 49.4 samples/s +Step 24070/44000 | Loss: 2.9139 | LR: 2.19e-04 | 404.0ms/step | 49.5 samples/s +Step 24080/44000 | Loss: 3.0010 | LR: 2.19e-04 | 404.9ms/step | 49.4 samples/s +Step 24090/44000 | Loss: 2.9507 | LR: 2.19e-04 | 404.8ms/step | 49.4 samples/s +Step 24100/44000 | Loss: 3.0131 | LR: 2.19e-04 | 404.8ms/step | 49.4 samples/s +Step 24110/44000 | Loss: 2.9274 | LR: 2.19e-04 | 406.4ms/step | 49.2 samples/s +Step 24120/44000 | Loss: 2.9505 | LR: 2.18e-04 | 405.6ms/step | 49.3 samples/s +Step 24130/44000 | Loss: 3.0033 | LR: 2.18e-04 | 404.4ms/step | 49.5 samples/s +Step 24140/44000 | Loss: 2.9402 | LR: 2.18e-04 | 404.1ms/step | 49.5 samples/s +Step 24150/44000 | Loss: 2.9315 | LR: 2.18e-04 | 404.6ms/step | 49.4 samples/s +Step 24160/44000 | Loss: 2.9096 | LR: 2.18e-04 | 403.3ms/step | 49.6 samples/s +Step 24170/44000 | Loss: 2.9905 | LR: 2.17e-04 | 404.3ms/step | 49.5 samples/s +Step 24180/44000 | Loss: 2.9573 | LR: 2.17e-04 | 404.4ms/step | 49.5 samples/s +Step 24190/44000 | Loss: 2.9995 | LR: 2.17e-04 | 403.9ms/step | 49.5 samples/s +Step 24200/44000 | Loss: 2.9214 | LR: 2.17e-04 | 403.2ms/step | 49.6 samples/s +Step 24210/44000 | Loss: 2.9247 | LR: 2.17e-04 | 404.3ms/step | 49.5 samples/s +Step 24220/44000 | Loss: 2.9656 | LR: 2.17e-04 | 403.1ms/step | 49.6 samples/s +Step 24230/44000 | Loss: 2.9933 | LR: 2.16e-04 | 403.4ms/step | 49.6 samples/s +Step 24240/44000 | Loss: 2.9821 | LR: 2.16e-04 | 403.9ms/step | 49.5 samples/s +Step 24250/44000 | Loss: 2.9740 | LR: 2.16e-04 | 403.9ms/step | 49.5 samples/s +Validation at step 24250: Loss = 2.9624, Perplexity = 19.34 +Step 24260/44000 | Loss: 3.0058 | LR: 2.16e-04 | 572.8ms/step | 34.9 samples/s +Step 24270/44000 | Loss: 2.9628 | LR: 2.16e-04 | 403.9ms/step | 49.5 samples/s +Step 24280/44000 | Loss: 3.0390 | LR: 2.16e-04 | 401.2ms/step | 49.9 samples/s +Step 24290/44000 | Loss: 2.9836 | LR: 2.15e-04 | 401.9ms/step | 49.8 samples/s +Step 24300/44000 | Loss: 2.9582 | LR: 2.15e-04 | 402.5ms/step | 49.7 samples/s +Step 24310/44000 | Loss: 2.8625 | LR: 2.15e-04 | 400.9ms/step | 49.9 samples/s +Step 24320/44000 | Loss: 2.9397 | LR: 2.15e-04 | 401.3ms/step | 49.8 samples/s +Step 24330/44000 | Loss: 2.9735 | LR: 2.15e-04 | 402.1ms/step | 49.7 samples/s +Step 24340/44000 | Loss: 2.9990 | LR: 2.14e-04 | 400.7ms/step | 49.9 samples/s +Step 24350/44000 | Loss: 2.9401 | LR: 2.14e-04 | 403.7ms/step | 49.5 samples/s +Step 24360/44000 | Loss: 2.9059 | LR: 2.14e-04 | 404.3ms/step | 49.5 samples/s +Step 24370/44000 | Loss: 3.0016 | LR: 2.14e-04 | 404.1ms/step | 49.5 samples/s +Step 24380/44000 | Loss: 2.9473 | LR: 2.14e-04 | 403.3ms/step | 49.6 samples/s +Step 24390/44000 | Loss: 3.0206 | LR: 2.14e-04 | 403.9ms/step | 49.5 samples/s +Step 24400/44000 | Loss: 2.9615 | LR: 2.13e-04 | 403.6ms/step | 49.6 samples/s +Step 24410/44000 | Loss: 2.9404 | LR: 2.13e-04 | 404.4ms/step | 49.5 samples/s +Step 24420/44000 | Loss: 2.9499 | LR: 2.13e-04 | 405.4ms/step | 49.3 samples/s +Step 24430/44000 | Loss: 2.9265 | LR: 2.13e-04 | 404.2ms/step | 49.5 samples/s +Step 24440/44000 | Loss: 2.9412 | LR: 2.13e-04 | 404.1ms/step | 49.5 samples/s +Step 24450/44000 | Loss: 2.9205 | LR: 2.12e-04 | 402.3ms/step | 49.7 samples/s +Step 24460/44000 | Loss: 2.9972 | LR: 2.12e-04 | 402.7ms/step | 49.7 samples/s +Step 24470/44000 | Loss: 2.9606 | LR: 2.12e-04 | 402.3ms/step | 49.7 samples/s +Step 24480/44000 | Loss: 2.9448 | LR: 2.12e-04 | 402.0ms/step | 49.8 samples/s +Step 24490/44000 | Loss: 2.8975 | LR: 2.12e-04 | 402.9ms/step | 49.6 samples/s +Step 24500/44000 | Loss: 2.9533 | LR: 2.12e-04 | 403.3ms/step | 49.6 samples/s +Validation at step 24500: Loss = 2.9613, Perplexity = 19.32 +Step 24510/44000 | Loss: 2.9731 | LR: 2.11e-04 | 569.0ms/step | 35.2 samples/s +Step 24520/44000 | Loss: 2.9451 | LR: 2.11e-04 | 403.1ms/step | 49.6 samples/s +Step 24530/44000 | Loss: 3.0102 | LR: 2.11e-04 | 402.4ms/step | 49.7 samples/s +Step 24540/44000 | Loss: 2.9710 | LR: 2.11e-04 | 402.7ms/step | 49.7 samples/s +Step 24550/44000 | Loss: 2.8929 | LR: 2.11e-04 | 402.4ms/step | 49.7 samples/s +Step 24560/44000 | Loss: 2.9025 | LR: 2.10e-04 | 402.3ms/step | 49.7 samples/s +Step 24570/44000 | Loss: 2.9065 | LR: 2.10e-04 | 407.0ms/step | 49.1 samples/s +Step 24580/44000 | Loss: 2.9422 | LR: 2.10e-04 | 402.5ms/step | 49.7 samples/s +Step 24590/44000 | Loss: 2.9578 | LR: 2.10e-04 | 402.8ms/step | 49.7 samples/s +Step 24600/44000 | Loss: 2.9769 | LR: 2.10e-04 | 401.5ms/step | 49.8 samples/s +Step 24610/44000 | Loss: 3.0032 | LR: 2.10e-04 | 401.1ms/step | 49.9 samples/s +Step 24620/44000 | Loss: 2.9736 | LR: 2.09e-04 | 400.7ms/step | 49.9 samples/s +Step 24630/44000 | Loss: 2.9271 | LR: 2.09e-04 | 402.4ms/step | 49.7 samples/s +Step 24640/44000 | Loss: 3.0270 | LR: 2.09e-04 | 402.5ms/step | 49.7 samples/s +Step 24650/44000 | Loss: 2.9696 | LR: 2.09e-04 | 401.7ms/step | 49.8 samples/s +Step 24660/44000 | Loss: 2.9418 | LR: 2.09e-04 | 402.8ms/step | 49.7 samples/s +Step 24670/44000 | Loss: 2.9658 | LR: 2.09e-04 | 402.5ms/step | 49.7 samples/s +Step 24680/44000 | Loss: 2.9292 | LR: 2.08e-04 | 402.8ms/step | 49.7 samples/s +Step 24690/44000 | Loss: 2.9794 | LR: 2.08e-04 | 402.5ms/step | 49.7 samples/s +Step 24700/44000 | Loss: 2.9398 | LR: 2.08e-04 | 402.5ms/step | 49.7 samples/s +Step 24710/44000 | Loss: 2.9517 | LR: 2.08e-04 | 403.0ms/step | 49.6 samples/s +Step 24720/44000 | Loss: 2.9645 | LR: 2.08e-04 | 405.5ms/step | 49.3 samples/s +Step 24730/44000 | Loss: 2.9676 | LR: 2.07e-04 | 406.6ms/step | 49.2 samples/s +Step 24740/44000 | Loss: 2.9884 | LR: 2.07e-04 | 406.7ms/step | 49.2 samples/s +Step 24750/44000 | Loss: 2.9475 | LR: 2.07e-04 | 404.6ms/step | 49.4 samples/s +Validation at step 24750: Loss = 2.9584, Perplexity = 19.27 +Step 24760/44000 | Loss: 2.9606 | LR: 2.07e-04 | 572.3ms/step | 34.9 samples/s +Step 24770/44000 | Loss: 2.9781 | LR: 2.07e-04 | 405.1ms/step | 49.4 samples/s +Step 24780/44000 | Loss: 2.9397 | LR: 2.07e-04 | 404.6ms/step | 49.4 samples/s +Step 24790/44000 | Loss: 2.9986 | LR: 2.06e-04 | 404.5ms/step | 49.4 samples/s +Step 24800/44000 | Loss: 2.9918 | LR: 2.06e-04 | 404.3ms/step | 49.5 samples/s +Step 24810/44000 | Loss: 2.9628 | LR: 2.06e-04 | 404.7ms/step | 49.4 samples/s +Step 24820/44000 | Loss: 2.9490 | LR: 2.06e-04 | 405.1ms/step | 49.4 samples/s +Step 24830/44000 | Loss: 2.9707 | LR: 2.06e-04 | 405.2ms/step | 49.4 samples/s +Step 24840/44000 | Loss: 2.9381 | LR: 2.05e-04 | 405.1ms/step | 49.4 samples/s +Step 24850/44000 | Loss: 3.0085 | LR: 2.05e-04 | 404.3ms/step | 49.5 samples/s +Step 24860/44000 | Loss: 2.9444 | LR: 2.05e-04 | 405.7ms/step | 49.3 samples/s +Step 24870/44000 | Loss: 2.9116 | LR: 2.05e-04 | 403.9ms/step | 49.5 samples/s +Step 24880/44000 | Loss: 2.9545 | LR: 2.05e-04 | 404.9ms/step | 49.4 samples/s +Step 24890/44000 | Loss: 2.9437 | LR: 2.05e-04 | 404.0ms/step | 49.5 samples/s +Step 24900/44000 | Loss: 2.9749 | LR: 2.04e-04 | 404.5ms/step | 49.4 samples/s +Step 24910/44000 | Loss: 2.9955 | LR: 2.04e-04 | 404.6ms/step | 49.4 samples/s +Step 24920/44000 | Loss: 2.9554 | LR: 2.04e-04 | 404.0ms/step | 49.5 samples/s +Step 24930/44000 | Loss: 2.9268 | LR: 2.04e-04 | 404.2ms/step | 49.5 samples/s +Step 24940/44000 | Loss: 2.9301 | LR: 2.04e-04 | 404.2ms/step | 49.5 samples/s +Step 24950/44000 | Loss: 2.9934 | LR: 2.04e-04 | 404.1ms/step | 49.5 samples/s +Step 24960/44000 | Loss: 3.0070 | LR: 2.03e-04 | 404.8ms/step | 49.4 samples/s +Step 24970/44000 | Loss: 3.0040 | LR: 2.03e-04 | 404.0ms/step | 49.5 samples/s +Step 24980/44000 | Loss: 2.9039 | LR: 2.03e-04 | 404.5ms/step | 49.4 samples/s +Step 24990/44000 | Loss: 2.9615 | LR: 2.03e-04 | 403.7ms/step | 49.5 samples/s +Step 25000/44000 | Loss: 2.9481 | LR: 2.03e-04 | 404.7ms/step | 49.4 samples/s +Validation at step 25000: Loss = 2.9563, Perplexity = 19.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-25000 +Step 25010/44000 | Loss: 2.9508 | LR: 2.02e-04 | 879.6ms/step | 22.7 samples/s +Step 25020/44000 | Loss: 2.9743 | LR: 2.02e-04 | 403.7ms/step | 49.5 samples/s +Step 25030/44000 | Loss: 2.9597 | LR: 2.02e-04 | 403.6ms/step | 49.6 samples/s +Step 25040/44000 | Loss: 2.9933 | LR: 2.02e-04 | 403.1ms/step | 49.6 samples/s +Step 25050/44000 | Loss: 2.9942 | LR: 2.02e-04 | 403.2ms/step | 49.6 samples/s +Step 25060/44000 | Loss: 2.9425 | LR: 2.02e-04 | 402.6ms/step | 49.7 samples/s +Step 25070/44000 | Loss: 2.9549 | LR: 2.01e-04 | 403.8ms/step | 49.5 samples/s +Step 25080/44000 | Loss: 2.9318 | LR: 2.01e-04 | 402.8ms/step | 49.7 samples/s +Step 25090/44000 | Loss: 2.9423 | LR: 2.01e-04 | 403.9ms/step | 49.5 samples/s +Step 25100/44000 | Loss: 2.9562 | LR: 2.01e-04 | 405.9ms/step | 49.3 samples/s +Step 25110/44000 | Loss: 2.9393 | LR: 2.01e-04 | 405.8ms/step | 49.3 samples/s +Step 25120/44000 | Loss: 2.8841 | LR: 2.00e-04 | 403.4ms/step | 49.6 samples/s +Step 25130/44000 | Loss: 2.9249 | LR: 2.00e-04 | 403.3ms/step | 49.6 samples/s +Step 25140/44000 | Loss: 2.9355 | LR: 2.00e-04 | 403.3ms/step | 49.6 samples/s +Step 25150/44000 | Loss: 2.9080 | LR: 2.00e-04 | 400.7ms/step | 49.9 samples/s +Step 25160/44000 | Loss: 2.9659 | LR: 2.00e-04 | 402.3ms/step | 49.7 samples/s +Step 25170/44000 | Loss: 2.9402 | LR: 2.00e-04 | 402.5ms/step | 49.7 samples/s +Step 25180/44000 | Loss: 2.8989 | LR: 1.99e-04 | 403.3ms/step | 49.6 samples/s +Step 25190/44000 | Loss: 2.9659 | LR: 1.99e-04 | 403.3ms/step | 49.6 samples/s +Step 25200/44000 | Loss: 2.9789 | LR: 1.99e-04 | 403.0ms/step | 49.6 samples/s +Step 25210/44000 | Loss: 2.9829 | LR: 1.99e-04 | 403.6ms/step | 49.6 samples/s +Step 25220/44000 | Loss: 2.9174 | LR: 1.99e-04 | 403.6ms/step | 49.5 samples/s +Step 25230/44000 | Loss: 2.9795 | LR: 1.99e-04 | 403.6ms/step | 49.5 samples/s +Step 25240/44000 | Loss: 3.0194 | LR: 1.98e-04 | 404.4ms/step | 49.5 samples/s +Step 25250/44000 | Loss: 2.9596 | LR: 1.98e-04 | 403.4ms/step | 49.6 samples/s +Validation at step 25250: Loss = 2.9551, Perplexity = 19.20 +Step 25260/44000 | Loss: 2.9198 | LR: 1.98e-04 | 569.2ms/step | 35.1 samples/s +Step 25270/44000 | Loss: 2.9704 | LR: 1.98e-04 | 404.0ms/step | 49.5 samples/s +Step 25280/44000 | Loss: 2.9677 | LR: 1.98e-04 | 403.3ms/step | 49.6 samples/s +Step 25290/44000 | Loss: 2.9414 | LR: 1.97e-04 | 404.1ms/step | 49.5 samples/s +Step 25300/44000 | Loss: 2.9585 | LR: 1.97e-04 | 402.6ms/step | 49.7 samples/s +Step 25310/44000 | Loss: 2.8663 | LR: 1.97e-04 | 400.9ms/step | 49.9 samples/s +Step 25320/44000 | Loss: 2.9886 | LR: 1.97e-04 | 401.1ms/step | 49.9 samples/s +Step 25330/44000 | Loss: 3.0165 | LR: 1.97e-04 | 399.9ms/step | 50.0 samples/s +Step 25340/44000 | Loss: 2.9520 | LR: 1.97e-04 | 400.0ms/step | 50.0 samples/s +Step 25350/44000 | Loss: 2.9480 | LR: 1.96e-04 | 399.9ms/step | 50.0 samples/s +Step 25360/44000 | Loss: 2.9300 | LR: 1.96e-04 | 400.1ms/step | 50.0 samples/s +Step 25370/44000 | Loss: 2.9755 | LR: 1.96e-04 | 400.0ms/step | 50.0 samples/s +Step 25380/44000 | Loss: 2.9086 | LR: 1.96e-04 | 400.1ms/step | 50.0 samples/s +Step 25390/44000 | Loss: 2.9399 | LR: 1.96e-04 | 399.8ms/step | 50.0 samples/s +Step 25400/44000 | Loss: 2.9744 | LR: 1.96e-04 | 400.1ms/step | 50.0 samples/s +Step 25410/44000 | Loss: 2.9983 | LR: 1.95e-04 | 400.0ms/step | 50.0 samples/s +Step 25420/44000 | Loss: 2.9885 | LR: 1.95e-04 | 400.0ms/step | 50.0 samples/s +Step 25430/44000 | Loss: 2.9760 | LR: 1.95e-04 | 400.5ms/step | 49.9 samples/s +Step 25440/44000 | Loss: 2.9126 | LR: 1.95e-04 | 401.6ms/step | 49.8 samples/s +Step 25450/44000 | Loss: 2.9012 | LR: 1.95e-04 | 400.4ms/step | 49.9 samples/s +Step 25460/44000 | Loss: 2.9384 | LR: 1.94e-04 | 401.4ms/step | 49.8 samples/s +Step 25470/44000 | Loss: 2.9595 | LR: 1.94e-04 | 403.3ms/step | 49.6 samples/s +Step 25480/44000 | Loss: 2.9572 | LR: 1.94e-04 | 403.8ms/step | 49.5 samples/s +Step 25490/44000 | Loss: 2.9825 | LR: 1.94e-04 | 404.1ms/step | 49.5 samples/s +Step 25500/44000 | Loss: 2.9625 | LR: 1.94e-04 | 403.4ms/step | 49.6 samples/s +Validation at step 25500: Loss = 2.9531, Perplexity = 19.17 +Step 25510/44000 | Loss: 2.9780 | LR: 1.94e-04 | 569.4ms/step | 35.1 samples/s +Step 25520/44000 | Loss: 2.9630 | LR: 1.93e-04 | 402.8ms/step | 49.6 samples/s +Step 25530/44000 | Loss: 2.9529 | LR: 1.93e-04 | 403.6ms/step | 49.6 samples/s +Step 25540/44000 | Loss: 2.9583 | LR: 1.93e-04 | 403.4ms/step | 49.6 samples/s +Step 25550/44000 | Loss: 2.9431 | LR: 1.93e-04 | 402.6ms/step | 49.7 samples/s +Step 25560/44000 | Loss: 2.9752 | LR: 1.93e-04 | 403.4ms/step | 49.6 samples/s +Step 25570/44000 | Loss: 2.9782 | LR: 1.92e-04 | 403.3ms/step | 49.6 samples/s +Step 25580/44000 | Loss: 2.9394 | LR: 1.92e-04 | 403.5ms/step | 49.6 samples/s +Step 25590/44000 | Loss: 2.8869 | LR: 1.92e-04 | 403.8ms/step | 49.5 samples/s +Step 25600/44000 | Loss: 2.9655 | LR: 1.92e-04 | 403.3ms/step | 49.6 samples/s +Step 25610/44000 | Loss: 2.9751 | LR: 1.92e-04 | 405.7ms/step | 49.3 samples/s +Step 25620/44000 | Loss: 2.9302 | LR: 1.92e-04 | 402.8ms/step | 49.7 samples/s +Step 25630/44000 | Loss: 2.9824 | LR: 1.91e-04 | 405.0ms/step | 49.4 samples/s +Step 25640/44000 | Loss: 2.9129 | LR: 1.91e-04 | 402.9ms/step | 49.6 samples/s +Step 25650/44000 | Loss: 2.9238 | LR: 1.91e-04 | 403.6ms/step | 49.6 samples/s +Step 25660/44000 | Loss: 2.8881 | LR: 1.91e-04 | 402.9ms/step | 49.6 samples/s +Step 25670/44000 | Loss: 2.9310 | LR: 1.91e-04 | 401.4ms/step | 49.8 samples/s +Step 25680/44000 | Loss: 2.9776 | LR: 1.91e-04 | 401.0ms/step | 49.9 samples/s +Step 25690/44000 | Loss: 2.9314 | LR: 1.90e-04 | 401.6ms/step | 49.8 samples/s +Step 25700/44000 | Loss: 2.9740 | LR: 1.90e-04 | 401.9ms/step | 49.8 samples/s +Step 25710/44000 | Loss: 2.9355 | LR: 1.90e-04 | 401.5ms/step | 49.8 samples/s +Step 25720/44000 | Loss: 2.9791 | LR: 1.90e-04 | 401.9ms/step | 49.8 samples/s +Step 25730/44000 | Loss: 2.9799 | LR: 1.90e-04 | 402.0ms/step | 49.8 samples/s +Step 25740/44000 | Loss: 2.9893 | LR: 1.89e-04 | 402.0ms/step | 49.8 samples/s +Step 25750/44000 | Loss: 2.9913 | LR: 1.89e-04 | 401.6ms/step | 49.8 samples/s +Validation at step 25750: Loss = 2.9518, Perplexity = 19.14 +Step 25760/44000 | Loss: 2.9213 | LR: 1.89e-04 | 568.1ms/step | 35.2 samples/s +Step 25770/44000 | Loss: 2.9552 | LR: 1.89e-04 | 404.4ms/step | 49.5 samples/s +Step 25780/44000 | Loss: 3.0163 | LR: 1.89e-04 | 402.3ms/step | 49.7 samples/s +Step 25790/44000 | Loss: 2.8999 | LR: 1.89e-04 | 402.5ms/step | 49.7 samples/s +Step 25800/44000 | Loss: 3.0237 | LR: 1.88e-04 | 402.9ms/step | 49.6 samples/s +Step 25810/44000 | Loss: 2.9905 | LR: 1.88e-04 | 402.4ms/step | 49.7 samples/s +Step 25820/44000 | Loss: 2.9099 | LR: 1.88e-04 | 402.1ms/step | 49.7 samples/s +Step 25830/44000 | Loss: 2.9657 | LR: 1.88e-04 | 402.1ms/step | 49.7 samples/s +Step 25840/44000 | Loss: 2.9464 | LR: 1.88e-04 | 402.0ms/step | 49.7 samples/s +Step 25850/44000 | Loss: 2.9446 | LR: 1.88e-04 | 402.1ms/step | 49.7 samples/s +Step 25860/44000 | Loss: 2.9627 | LR: 1.87e-04 | 401.7ms/step | 49.8 samples/s +Step 25870/44000 | Loss: 2.9425 | LR: 1.87e-04 | 402.1ms/step | 49.7 samples/s +Step 25880/44000 | Loss: 3.0279 | LR: 1.87e-04 | 402.5ms/step | 49.7 samples/s +Step 25890/44000 | Loss: 2.8902 | LR: 1.87e-04 | 402.7ms/step | 49.7 samples/s +Step 25900/44000 | Loss: 2.9240 | LR: 1.87e-04 | 400.9ms/step | 49.9 samples/s +Step 25910/44000 | Loss: 2.9958 | LR: 1.87e-04 | 402.9ms/step | 49.6 samples/s +Step 25920/44000 | Loss: 2.9948 | LR: 1.86e-04 | 402.8ms/step | 49.7 samples/s +Step 25930/44000 | Loss: 2.9586 | LR: 1.86e-04 | 402.8ms/step | 49.7 samples/s +Step 25940/44000 | Loss: 2.9754 | LR: 1.86e-04 | 402.6ms/step | 49.7 samples/s +Step 25950/44000 | Loss: 3.0030 | LR: 1.86e-04 | 402.6ms/step | 49.7 samples/s +Step 25960/44000 | Loss: 2.9537 | LR: 1.86e-04 | 403.4ms/step | 49.6 samples/s +Step 25970/44000 | Loss: 2.9782 | LR: 1.85e-04 | 403.6ms/step | 49.6 samples/s +Step 25980/44000 | Loss: 2.9518 | LR: 1.85e-04 | 403.4ms/step | 49.6 samples/s +Step 25990/44000 | Loss: 2.9799 | LR: 1.85e-04 | 404.2ms/step | 49.5 samples/s +Step 26000/44000 | Loss: 2.9634 | LR: 1.85e-04 | 403.8ms/step | 49.5 samples/s +Validation at step 26000: Loss = 2.9505, Perplexity = 19.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-26000 +Step 26010/44000 | Loss: 2.9349 | LR: 1.85e-04 | 880.2ms/step | 22.7 samples/s +Step 26020/44000 | Loss: 2.9776 | LR: 1.85e-04 | 404.4ms/step | 49.5 samples/s +Step 26030/44000 | Loss: 2.9766 | LR: 1.84e-04 | 402.3ms/step | 49.7 samples/s +Step 26040/44000 | Loss: 2.9842 | LR: 1.84e-04 | 403.3ms/step | 49.6 samples/s +Step 26050/44000 | Loss: 2.9426 | LR: 1.84e-04 | 401.8ms/step | 49.8 samples/s +Step 26060/44000 | Loss: 2.8731 | LR: 1.84e-04 | 402.1ms/step | 49.7 samples/s +Step 26070/44000 | Loss: 2.9640 | LR: 1.84e-04 | 403.0ms/step | 49.6 samples/s +Step 26080/44000 | Loss: 3.0051 | LR: 1.84e-04 | 403.1ms/step | 49.6 samples/s +Step 26090/44000 | Loss: 2.9527 | LR: 1.83e-04 | 404.5ms/step | 49.4 samples/s +Step 26100/44000 | Loss: 2.9798 | LR: 1.83e-04 | 403.3ms/step | 49.6 samples/s +Step 26110/44000 | Loss: 2.9901 | LR: 1.83e-04 | 404.0ms/step | 49.5 samples/s +Step 26120/44000 | Loss: 2.9383 | LR: 1.83e-04 | 403.5ms/step | 49.6 samples/s +Step 26130/44000 | Loss: 2.9876 | LR: 1.83e-04 | 403.8ms/step | 49.5 samples/s +Step 26140/44000 | Loss: 2.9243 | LR: 1.82e-04 | 403.8ms/step | 49.5 samples/s +Step 26150/44000 | Loss: 2.8511 | LR: 1.82e-04 | 404.3ms/step | 49.5 samples/s +Step 26160/44000 | Loss: 2.9018 | LR: 1.82e-04 | 404.0ms/step | 49.5 samples/s +Step 26170/44000 | Loss: 2.9230 | LR: 1.82e-04 | 404.5ms/step | 49.4 samples/s +Step 26180/44000 | Loss: 2.9533 | LR: 1.82e-04 | 403.3ms/step | 49.6 samples/s +Step 26190/44000 | Loss: 2.9193 | LR: 1.82e-04 | 403.7ms/step | 49.5 samples/s +Step 26200/44000 | Loss: 2.9561 | LR: 1.81e-04 | 402.7ms/step | 49.7 samples/s +Step 26210/44000 | Loss: 2.9729 | LR: 1.81e-04 | 402.7ms/step | 49.7 samples/s +Step 26220/44000 | Loss: 2.9463 | LR: 1.81e-04 | 401.8ms/step | 49.8 samples/s +Step 26230/44000 | Loss: 2.9412 | LR: 1.81e-04 | 401.1ms/step | 49.9 samples/s +Step 26240/44000 | Loss: 2.9890 | LR: 1.81e-04 | 402.1ms/step | 49.7 samples/s +Step 26250/44000 | Loss: 2.9366 | LR: 1.81e-04 | 401.8ms/step | 49.8 samples/s +Validation at step 26250: Loss = 2.9486, Perplexity = 19.08 +Step 26260/44000 | Loss: 2.9310 | LR: 1.80e-04 | 570.5ms/step | 35.1 samples/s +Step 26270/44000 | Loss: 2.9371 | LR: 1.80e-04 | 401.3ms/step | 49.8 samples/s +Step 26280/44000 | Loss: 2.9975 | LR: 1.80e-04 | 402.3ms/step | 49.7 samples/s +Step 26290/44000 | Loss: 2.9699 | LR: 1.80e-04 | 402.5ms/step | 49.7 samples/s +Step 26300/44000 | Loss: 2.9544 | LR: 1.80e-04 | 402.0ms/step | 49.8 samples/s +Step 26310/44000 | Loss: 2.9002 | LR: 1.80e-04 | 401.1ms/step | 49.9 samples/s +Step 26320/44000 | Loss: 2.9744 | LR: 1.79e-04 | 402.7ms/step | 49.7 samples/s +Step 26330/44000 | Loss: 3.0015 | LR: 1.79e-04 | 402.4ms/step | 49.7 samples/s +Step 26340/44000 | Loss: 2.9535 | LR: 1.79e-04 | 401.7ms/step | 49.8 samples/s +Step 26350/44000 | Loss: 2.9575 | LR: 1.79e-04 | 401.9ms/step | 49.8 samples/s +Step 26360/44000 | Loss: 2.9091 | LR: 1.79e-04 | 401.4ms/step | 49.8 samples/s +Step 26370/44000 | Loss: 2.9130 | LR: 1.78e-04 | 400.7ms/step | 49.9 samples/s +Step 26380/44000 | Loss: 2.9734 | LR: 1.78e-04 | 402.4ms/step | 49.7 samples/s +Step 26390/44000 | Loss: 2.9037 | LR: 1.78e-04 | 401.3ms/step | 49.8 samples/s +Step 26400/44000 | Loss: 2.8766 | LR: 1.78e-04 | 400.7ms/step | 49.9 samples/s +Step 26410/44000 | Loss: 2.9603 | LR: 1.78e-04 | 403.7ms/step | 49.5 samples/s +Step 26420/44000 | Loss: 2.9527 | LR: 1.78e-04 | 403.5ms/step | 49.6 samples/s +Step 26430/44000 | Loss: 2.9209 | LR: 1.77e-04 | 402.4ms/step | 49.7 samples/s +Step 26440/44000 | Loss: 2.9878 | LR: 1.77e-04 | 402.4ms/step | 49.7 samples/s +Step 26450/44000 | Loss: 2.9225 | LR: 1.77e-04 | 402.9ms/step | 49.6 samples/s +Step 26460/44000 | Loss: 2.9536 | LR: 1.77e-04 | 403.4ms/step | 49.6 samples/s +Step 26470/44000 | Loss: 2.9536 | LR: 1.77e-04 | 404.4ms/step | 49.5 samples/s +Step 26480/44000 | Loss: 2.9803 | LR: 1.77e-04 | 404.0ms/step | 49.5 samples/s +Step 26490/44000 | Loss: 3.0020 | LR: 1.76e-04 | 405.2ms/step | 49.4 samples/s +Step 26500/44000 | Loss: 2.9610 | LR: 1.76e-04 | 406.8ms/step | 49.2 samples/s +Validation at step 26500: Loss = 2.9483, Perplexity = 19.07 +Step 26510/44000 | Loss: 2.9929 | LR: 1.76e-04 | 575.4ms/step | 34.8 samples/s +Step 26520/44000 | Loss: 2.9435 | LR: 1.76e-04 | 406.2ms/step | 49.2 samples/s +Step 26530/44000 | Loss: 2.8615 | LR: 1.76e-04 | 403.9ms/step | 49.5 samples/s +Step 26540/44000 | Loss: 2.9112 | LR: 1.76e-04 | 403.0ms/step | 49.6 samples/s +Step 26550/44000 | Loss: 2.9983 | LR: 1.75e-04 | 402.4ms/step | 49.7 samples/s +Step 26560/44000 | Loss: 2.8984 | LR: 1.75e-04 | 403.0ms/step | 49.6 samples/s +Step 26570/44000 | Loss: 2.9301 | LR: 1.75e-04 | 403.0ms/step | 49.6 samples/s +Step 26580/44000 | Loss: 2.9519 | LR: 1.75e-04 | 404.0ms/step | 49.5 samples/s +Step 26590/44000 | Loss: 2.9691 | LR: 1.75e-04 | 402.3ms/step | 49.7 samples/s +Step 26600/44000 | Loss: 2.9298 | LR: 1.74e-04 | 401.9ms/step | 49.8 samples/s +Step 26610/44000 | Loss: 2.9761 | LR: 1.74e-04 | 400.9ms/step | 49.9 samples/s +Step 26620/44000 | Loss: 3.0038 | LR: 1.74e-04 | 402.8ms/step | 49.7 samples/s +Step 26630/44000 | Loss: 3.0101 | LR: 1.74e-04 | 403.7ms/step | 49.5 samples/s +Step 26640/44000 | Loss: 2.9347 | LR: 1.74e-04 | 403.1ms/step | 49.6 samples/s +Step 26650/44000 | Loss: 2.9180 | LR: 1.74e-04 | 403.8ms/step | 49.5 samples/s +Step 26660/44000 | Loss: 2.9931 | LR: 1.73e-04 | 403.6ms/step | 49.6 samples/s +Step 26670/44000 | Loss: 2.9236 | LR: 1.73e-04 | 403.0ms/step | 49.6 samples/s +Step 26680/44000 | Loss: 2.9512 | LR: 1.73e-04 | 403.1ms/step | 49.6 samples/s +Step 26690/44000 | Loss: 3.0090 | LR: 1.73e-04 | 402.4ms/step | 49.7 samples/s +Step 26700/44000 | Loss: 2.9034 | LR: 1.73e-04 | 403.6ms/step | 49.6 samples/s +Step 26710/44000 | Loss: 3.0183 | LR: 1.73e-04 | 404.1ms/step | 49.5 samples/s +Step 26720/44000 | Loss: 2.9353 | LR: 1.72e-04 | 403.5ms/step | 49.6 samples/s +Step 26730/44000 | Loss: 2.9191 | LR: 1.72e-04 | 404.3ms/step | 49.5 samples/s +Step 26740/44000 | Loss: 2.9672 | LR: 1.72e-04 | 403.3ms/step | 49.6 samples/s +Step 26750/44000 | Loss: 2.9454 | LR: 1.72e-04 | 403.6ms/step | 49.6 samples/s +Validation at step 26750: Loss = 2.9468, Perplexity = 19.04 +Step 26760/44000 | Loss: 2.9430 | LR: 1.72e-04 | 571.3ms/step | 35.0 samples/s +Step 26770/44000 | Loss: 2.8804 | LR: 1.72e-04 | 402.9ms/step | 49.6 samples/s +Step 26780/44000 | Loss: 2.9002 | LR: 1.71e-04 | 403.6ms/step | 49.6 samples/s +Step 26790/44000 | Loss: 2.8664 | LR: 1.71e-04 | 403.6ms/step | 49.6 samples/s +Step 26800/44000 | Loss: 2.9133 | LR: 1.71e-04 | 403.5ms/step | 49.6 samples/s +Step 26810/44000 | Loss: 3.0185 | LR: 1.71e-04 | 403.0ms/step | 49.6 samples/s +Step 26820/44000 | Loss: 2.9529 | LR: 1.71e-04 | 403.3ms/step | 49.6 samples/s +Step 26830/44000 | Loss: 2.9662 | LR: 1.71e-04 | 402.7ms/step | 49.7 samples/s +Step 26840/44000 | Loss: 2.9526 | LR: 1.70e-04 | 402.9ms/step | 49.6 samples/s +Step 26850/44000 | Loss: 2.9702 | LR: 1.70e-04 | 402.6ms/step | 49.7 samples/s +Step 26860/44000 | Loss: 2.9697 | LR: 1.70e-04 | 402.7ms/step | 49.7 samples/s +Step 26870/44000 | Loss: 2.9632 | LR: 1.70e-04 | 403.2ms/step | 49.6 samples/s +Step 26880/44000 | Loss: 2.9397 | LR: 1.70e-04 | 402.5ms/step | 49.7 samples/s +Step 26890/44000 | Loss: 2.9824 | LR: 1.69e-04 | 402.3ms/step | 49.7 samples/s +Step 26900/44000 | Loss: 2.9314 | LR: 1.69e-04 | 402.5ms/step | 49.7 samples/s +Step 26910/44000 | Loss: 3.0011 | LR: 1.69e-04 | 403.8ms/step | 49.5 samples/s +Step 26920/44000 | Loss: 2.9213 | LR: 1.69e-04 | 405.8ms/step | 49.3 samples/s +Step 26930/44000 | Loss: 2.9948 | LR: 1.69e-04 | 404.1ms/step | 49.5 samples/s +Step 26940/44000 | Loss: 2.9924 | LR: 1.69e-04 | 403.4ms/step | 49.6 samples/s +Step 26950/44000 | Loss: 2.9101 | LR: 1.68e-04 | 402.7ms/step | 49.7 samples/s +Step 26960/44000 | Loss: 2.9340 | LR: 1.68e-04 | 401.8ms/step | 49.8 samples/s +Step 26970/44000 | Loss: 2.9570 | LR: 1.68e-04 | 402.3ms/step | 49.7 samples/s +Step 26980/44000 | Loss: 2.9661 | LR: 1.68e-04 | 403.6ms/step | 49.6 samples/s +Step 26990/44000 | Loss: 2.9790 | LR: 1.68e-04 | 403.4ms/step | 49.6 samples/s +Step 27000/44000 | Loss: 2.9497 | LR: 1.68e-04 | 402.6ms/step | 49.7 samples/s +Validation at step 27000: Loss = 2.9446, Perplexity = 19.00 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-27000 +Step 27010/44000 | Loss: 2.9736 | LR: 1.67e-04 | 872.1ms/step | 22.9 samples/s +Step 27020/44000 | Loss: 2.9100 | LR: 1.67e-04 | 403.0ms/step | 49.6 samples/s +Step 27030/44000 | Loss: 2.9339 | LR: 1.67e-04 | 403.6ms/step | 49.6 samples/s +Step 27040/44000 | Loss: 2.9182 | LR: 1.67e-04 | 402.5ms/step | 49.7 samples/s +Step 27050/44000 | Loss: 2.9470 | LR: 1.67e-04 | 402.2ms/step | 49.7 samples/s +Step 27060/44000 | Loss: 2.9286 | LR: 1.67e-04 | 401.2ms/step | 49.8 samples/s +Step 27070/44000 | Loss: 2.9293 | LR: 1.66e-04 | 400.4ms/step | 49.9 samples/s +Step 27080/44000 | Loss: 2.9712 | LR: 1.66e-04 | 400.6ms/step | 49.9 samples/s +Step 27090/44000 | Loss: 2.9529 | LR: 1.66e-04 | 403.3ms/step | 49.6 samples/s +Step 27100/44000 | Loss: 2.9811 | LR: 1.66e-04 | 403.6ms/step | 49.6 samples/s +Step 27110/44000 | Loss: 2.9297 | LR: 1.66e-04 | 403.7ms/step | 49.5 samples/s +Step 27120/44000 | Loss: 2.9599 | LR: 1.66e-04 | 404.2ms/step | 49.5 samples/s +Step 27130/44000 | Loss: 2.9375 | LR: 1.65e-04 | 403.7ms/step | 49.5 samples/s +Step 27140/44000 | Loss: 2.9139 | LR: 1.65e-04 | 403.2ms/step | 49.6 samples/s +Step 27150/44000 | Loss: 2.9255 | LR: 1.65e-04 | 402.4ms/step | 49.7 samples/s +Step 27160/44000 | Loss: 2.9396 | LR: 1.65e-04 | 403.8ms/step | 49.5 samples/s +Step 27170/44000 | Loss: 3.0201 | LR: 1.65e-04 | 403.4ms/step | 49.6 samples/s +Step 27180/44000 | Loss: 2.9235 | LR: 1.65e-04 | 403.4ms/step | 49.6 samples/s +Step 27190/44000 | Loss: 2.9931 | LR: 1.64e-04 | 403.7ms/step | 49.5 samples/s +Step 27200/44000 | Loss: 2.9268 | LR: 1.64e-04 | 404.1ms/step | 49.5 samples/s +Step 27210/44000 | Loss: 2.9076 | LR: 1.64e-04 | 403.7ms/step | 49.5 samples/s +Step 27220/44000 | Loss: 2.9111 | LR: 1.64e-04 | 403.7ms/step | 49.5 samples/s +Step 27230/44000 | Loss: 2.9493 | LR: 1.64e-04 | 404.0ms/step | 49.5 samples/s +Step 27240/44000 | Loss: 2.9475 | LR: 1.63e-04 | 403.6ms/step | 49.6 samples/s +Step 27250/44000 | Loss: 2.9478 | LR: 1.63e-04 | 403.3ms/step | 49.6 samples/s +Validation at step 27250: Loss = 2.9432, Perplexity = 18.98 +Step 27260/44000 | Loss: 2.9153 | LR: 1.63e-04 | 571.4ms/step | 35.0 samples/s +Step 27270/44000 | Loss: 2.9662 | LR: 1.63e-04 | 403.2ms/step | 49.6 samples/s +Step 27280/44000 | Loss: 2.9603 | LR: 1.63e-04 | 402.8ms/step | 49.6 samples/s +Step 27290/44000 | Loss: 3.0257 | LR: 1.63e-04 | 402.8ms/step | 49.7 samples/s +Step 27300/44000 | Loss: 2.9364 | LR: 1.62e-04 | 400.7ms/step | 49.9 samples/s +Step 27310/44000 | Loss: 2.9472 | LR: 1.62e-04 | 400.2ms/step | 50.0 samples/s +Step 27320/44000 | Loss: 2.9855 | LR: 1.62e-04 | 400.0ms/step | 50.0 samples/s +Step 27330/44000 | Loss: 2.8699 | LR: 1.62e-04 | 400.0ms/step | 50.0 samples/s +Step 27340/44000 | Loss: 2.9350 | LR: 1.62e-04 | 399.9ms/step | 50.0 samples/s +Step 27350/44000 | Loss: 2.9400 | LR: 1.62e-04 | 401.8ms/step | 49.8 samples/s +Step 27360/44000 | Loss: 2.9692 | LR: 1.61e-04 | 400.3ms/step | 50.0 samples/s +Step 27370/44000 | Loss: 2.8981 | LR: 1.61e-04 | 400.6ms/step | 49.9 samples/s +Step 27380/44000 | Loss: 2.8976 | LR: 1.61e-04 | 401.2ms/step | 49.9 samples/s +Step 27390/44000 | Loss: 2.9514 | LR: 1.61e-04 | 400.4ms/step | 49.9 samples/s +Step 27400/44000 | Loss: 2.8901 | LR: 1.61e-04 | 401.1ms/step | 49.9 samples/s +Step 27410/44000 | Loss: 2.9919 | LR: 1.61e-04 | 400.6ms/step | 49.9 samples/s +Step 27420/44000 | Loss: 2.9145 | LR: 1.60e-04 | 400.0ms/step | 50.0 samples/s +Step 27430/44000 | Loss: 2.9814 | LR: 1.60e-04 | 400.0ms/step | 50.0 samples/s +Step 27440/44000 | Loss: 2.9621 | LR: 1.60e-04 | 400.0ms/step | 50.0 samples/s +Step 27450/44000 | Loss: 2.9629 | LR: 1.60e-04 | 400.1ms/step | 50.0 samples/s +Step 27460/44000 | Loss: 2.9017 | LR: 1.60e-04 | 400.0ms/step | 50.0 samples/s +Step 27470/44000 | Loss: 2.9589 | LR: 1.60e-04 | 400.2ms/step | 50.0 samples/s +Step 27480/44000 | Loss: 2.9414 | LR: 1.59e-04 | 401.1ms/step | 49.9 samples/s +Step 27490/44000 | Loss: 2.9776 | LR: 1.59e-04 | 401.1ms/step | 49.9 samples/s +Step 27500/44000 | Loss: 2.9021 | LR: 1.59e-04 | 402.5ms/step | 49.7 samples/s +Validation at step 27500: Loss = 2.9424, Perplexity = 18.96 +Step 27510/44000 | Loss: 2.9585 | LR: 1.59e-04 | 569.3ms/step | 35.1 samples/s +Step 27520/44000 | Loss: 2.9960 | LR: 1.59e-04 | 403.0ms/step | 49.6 samples/s +Step 27530/44000 | Loss: 2.9130 | LR: 1.59e-04 | 402.9ms/step | 49.6 samples/s +Step 27540/44000 | Loss: 2.8820 | LR: 1.58e-04 | 402.9ms/step | 49.6 samples/s +Step 27550/44000 | Loss: 2.8886 | LR: 1.58e-04 | 401.2ms/step | 49.8 samples/s +Step 27560/44000 | Loss: 2.9443 | LR: 1.58e-04 | 400.7ms/step | 49.9 samples/s +Step 27570/44000 | Loss: 2.9283 | LR: 1.58e-04 | 402.2ms/step | 49.7 samples/s +Step 27580/44000 | Loss: 2.9601 | LR: 1.58e-04 | 401.1ms/step | 49.9 samples/s +Step 27590/44000 | Loss: 2.9219 | LR: 1.58e-04 | 400.9ms/step | 49.9 samples/s +Step 27600/44000 | Loss: 2.9275 | LR: 1.57e-04 | 400.9ms/step | 49.9 samples/s +Step 27610/44000 | Loss: 2.9199 | LR: 1.57e-04 | 400.2ms/step | 50.0 samples/s +Step 27620/44000 | Loss: 2.9383 | LR: 1.57e-04 | 402.9ms/step | 49.6 samples/s +Step 27630/44000 | Loss: 2.9106 | LR: 1.57e-04 | 403.4ms/step | 49.6 samples/s +Step 27640/44000 | Loss: 2.8738 | LR: 1.57e-04 | 403.6ms/step | 49.5 samples/s +Step 27650/44000 | Loss: 3.0076 | LR: 1.57e-04 | 403.9ms/step | 49.5 samples/s +Step 27660/44000 | Loss: 2.9410 | LR: 1.56e-04 | 403.2ms/step | 49.6 samples/s +Step 27670/44000 | Loss: 2.9611 | LR: 1.56e-04 | 403.6ms/step | 49.6 samples/s +Step 27680/44000 | Loss: 2.8868 | LR: 1.56e-04 | 403.8ms/step | 49.5 samples/s +Step 27690/44000 | Loss: 2.9393 | LR: 1.56e-04 | 403.5ms/step | 49.6 samples/s +Step 27700/44000 | Loss: 2.9355 | LR: 1.56e-04 | 403.1ms/step | 49.6 samples/s +Step 27710/44000 | Loss: 2.8988 | LR: 1.56e-04 | 402.8ms/step | 49.7 samples/s +Step 27720/44000 | Loss: 2.9797 | LR: 1.55e-04 | 402.8ms/step | 49.6 samples/s +Step 27730/44000 | Loss: 2.9723 | LR: 1.55e-04 | 402.9ms/step | 49.6 samples/s +Step 27740/44000 | Loss: 2.9536 | LR: 1.55e-04 | 403.3ms/step | 49.6 samples/s +Step 27750/44000 | Loss: 2.9541 | LR: 1.55e-04 | 403.8ms/step | 49.5 samples/s +Validation at step 27750: Loss = 2.9403, Perplexity = 18.92 +Step 27760/44000 | Loss: 2.8869 | LR: 1.55e-04 | 567.8ms/step | 35.2 samples/s +Step 27770/44000 | Loss: 2.9372 | LR: 1.55e-04 | 402.9ms/step | 49.6 samples/s +Step 27780/44000 | Loss: 2.9561 | LR: 1.54e-04 | 402.4ms/step | 49.7 samples/s +Step 27790/44000 | Loss: 2.9213 | LR: 1.54e-04 | 401.8ms/step | 49.8 samples/s +Step 27800/44000 | Loss: 2.9392 | LR: 1.54e-04 | 401.7ms/step | 49.8 samples/s +Step 27810/44000 | Loss: 2.9320 | LR: 1.54e-04 | 400.8ms/step | 49.9 samples/s +Step 27820/44000 | Loss: 2.9096 | LR: 1.54e-04 | 401.1ms/step | 49.9 samples/s +Step 27830/44000 | Loss: 2.9552 | LR: 1.54e-04 | 400.5ms/step | 49.9 samples/s +Step 27840/44000 | Loss: 2.9641 | LR: 1.53e-04 | 401.0ms/step | 49.9 samples/s +Step 27850/44000 | Loss: 2.9681 | LR: 1.53e-04 | 402.4ms/step | 49.7 samples/s +Step 27860/44000 | Loss: 2.9622 | LR: 1.53e-04 | 401.2ms/step | 49.8 samples/s +Step 27870/44000 | Loss: 2.9407 | LR: 1.53e-04 | 401.4ms/step | 49.8 samples/s +Step 27880/44000 | Loss: 2.9304 | LR: 1.53e-04 | 400.2ms/step | 50.0 samples/s +Step 27890/44000 | Loss: 2.9610 | LR: 1.53e-04 | 403.1ms/step | 49.6 samples/s +Step 27900/44000 | Loss: 2.9049 | LR: 1.52e-04 | 403.3ms/step | 49.6 samples/s +Step 27910/44000 | Loss: 2.9803 | LR: 1.52e-04 | 402.6ms/step | 49.7 samples/s +Step 27920/44000 | Loss: 2.9315 | LR: 1.52e-04 | 401.8ms/step | 49.8 samples/s +Step 27930/44000 | Loss: 2.9285 | LR: 1.52e-04 | 403.3ms/step | 49.6 samples/s +Step 27940/44000 | Loss: 2.8599 | LR: 1.52e-04 | 404.0ms/step | 49.5 samples/s +Step 27950/44000 | Loss: 2.9288 | LR: 1.52e-04 | 402.1ms/step | 49.7 samples/s +Step 27960/44000 | Loss: 2.9453 | LR: 1.51e-04 | 403.0ms/step | 49.6 samples/s +Step 27970/44000 | Loss: 2.9185 | LR: 1.51e-04 | 403.2ms/step | 49.6 samples/s +Step 27980/44000 | Loss: 2.8426 | LR: 1.51e-04 | 404.0ms/step | 49.5 samples/s +Step 27990/44000 | Loss: 2.9126 | LR: 1.51e-04 | 403.4ms/step | 49.6 samples/s +Step 28000/44000 | Loss: 2.9230 | LR: 1.51e-04 | 404.2ms/step | 49.5 samples/s +Validation at step 28000: Loss = 2.9391, Perplexity = 18.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-28000 +Step 28010/44000 | Loss: 2.9181 | LR: 1.51e-04 | 879.7ms/step | 22.7 samples/s +Step 28020/44000 | Loss: 2.9881 | LR: 1.50e-04 | 403.3ms/step | 49.6 samples/s +Step 28030/44000 | Loss: 2.9292 | LR: 1.50e-04 | 404.0ms/step | 49.5 samples/s +Step 28040/44000 | Loss: 2.9582 | LR: 1.50e-04 | 402.8ms/step | 49.6 samples/s +Step 28050/44000 | Loss: 2.9542 | LR: 1.50e-04 | 402.6ms/step | 49.7 samples/s +Step 28060/44000 | Loss: 3.0043 | LR: 1.50e-04 | 403.1ms/step | 49.6 samples/s +Step 28070/44000 | Loss: 2.9256 | LR: 1.50e-04 | 403.3ms/step | 49.6 samples/s +Step 28080/44000 | Loss: 2.9616 | LR: 1.49e-04 | 404.0ms/step | 49.5 samples/s +Step 28090/44000 | Loss: 2.9134 | LR: 1.49e-04 | 405.4ms/step | 49.3 samples/s +Step 28100/44000 | Loss: 2.9069 | LR: 1.49e-04 | 403.7ms/step | 49.5 samples/s +Step 28110/44000 | Loss: 2.9497 | LR: 1.49e-04 | 404.0ms/step | 49.5 samples/s +Step 28120/44000 | Loss: 2.9138 | LR: 1.49e-04 | 404.7ms/step | 49.4 samples/s +Step 28130/44000 | Loss: 2.9109 | LR: 1.49e-04 | 406.3ms/step | 49.2 samples/s +Step 28140/44000 | Loss: 3.0087 | LR: 1.48e-04 | 406.1ms/step | 49.3 samples/s +Step 28150/44000 | Loss: 2.9947 | LR: 1.48e-04 | 405.7ms/step | 49.3 samples/s +Step 28160/44000 | Loss: 2.9528 | LR: 1.48e-04 | 404.9ms/step | 49.4 samples/s +Step 28170/44000 | Loss: 2.9663 | LR: 1.48e-04 | 404.8ms/step | 49.4 samples/s +Step 28180/44000 | Loss: 2.9891 | LR: 1.48e-04 | 405.8ms/step | 49.3 samples/s +Step 28190/44000 | Loss: 2.9350 | LR: 1.48e-04 | 406.5ms/step | 49.2 samples/s +Step 28200/44000 | Loss: 2.9256 | LR: 1.47e-04 | 406.0ms/step | 49.3 samples/s +Step 28210/44000 | Loss: 2.9466 | LR: 1.47e-04 | 403.4ms/step | 49.6 samples/s +Step 28220/44000 | Loss: 2.9259 | LR: 1.47e-04 | 403.1ms/step | 49.6 samples/s +Step 28230/44000 | Loss: 2.9542 | LR: 1.47e-04 | 405.7ms/step | 49.3 samples/s +Step 28240/44000 | Loss: 2.9339 | LR: 1.47e-04 | 405.8ms/step | 49.3 samples/s +Step 28250/44000 | Loss: 2.9603 | LR: 1.47e-04 | 407.0ms/step | 49.1 samples/s +Validation at step 28250: Loss = 2.9376, Perplexity = 18.87 +Step 28260/44000 | Loss: 2.9695 | LR: 1.46e-04 | 578.8ms/step | 34.6 samples/s +Step 28270/44000 | Loss: 2.9657 | LR: 1.46e-04 | 405.5ms/step | 49.3 samples/s +Step 28280/44000 | Loss: 2.9828 | LR: 1.46e-04 | 405.8ms/step | 49.3 samples/s +Step 28290/44000 | Loss: 2.9555 | LR: 1.46e-04 | 406.0ms/step | 49.3 samples/s +Step 28300/44000 | Loss: 2.9584 | LR: 1.46e-04 | 406.8ms/step | 49.2 samples/s +Step 28310/44000 | Loss: 2.9531 | LR: 1.46e-04 | 406.3ms/step | 49.2 samples/s +Step 28320/44000 | Loss: 2.8714 | LR: 1.45e-04 | 406.0ms/step | 49.3 samples/s +Step 28330/44000 | Loss: 2.9241 | LR: 1.45e-04 | 405.9ms/step | 49.3 samples/s +Step 28340/44000 | Loss: 2.9518 | LR: 1.45e-04 | 405.7ms/step | 49.3 samples/s +Step 28350/44000 | Loss: 2.9816 | LR: 1.45e-04 | 405.8ms/step | 49.3 samples/s +Step 28360/44000 | Loss: 2.9169 | LR: 1.45e-04 | 405.5ms/step | 49.3 samples/s +Step 28370/44000 | Loss: 2.9055 | LR: 1.45e-04 | 404.9ms/step | 49.4 samples/s +Step 28380/44000 | Loss: 2.8986 | LR: 1.44e-04 | 404.5ms/step | 49.4 samples/s +Step 28390/44000 | Loss: 2.9221 | LR: 1.44e-04 | 404.7ms/step | 49.4 samples/s +Step 28400/44000 | Loss: 2.8791 | LR: 1.44e-04 | 404.9ms/step | 49.4 samples/s +Step 28410/44000 | Loss: 2.9453 | LR: 1.44e-04 | 405.5ms/step | 49.3 samples/s +Step 28420/44000 | Loss: 2.8998 | LR: 1.44e-04 | 403.9ms/step | 49.5 samples/s +Step 28430/44000 | Loss: 2.9385 | LR: 1.44e-04 | 404.0ms/step | 49.5 samples/s +Step 28440/44000 | Loss: 2.9299 | LR: 1.43e-04 | 404.4ms/step | 49.5 samples/s +Step 28450/44000 | Loss: 2.9758 | LR: 1.43e-04 | 403.8ms/step | 49.5 samples/s +Step 28460/44000 | Loss: 2.9509 | LR: 1.43e-04 | 403.5ms/step | 49.6 samples/s +Step 28470/44000 | Loss: 2.9047 | LR: 1.43e-04 | 404.2ms/step | 49.5 samples/s +Step 28480/44000 | Loss: 2.9227 | LR: 1.43e-04 | 404.3ms/step | 49.5 samples/s +Step 28490/44000 | Loss: 2.9400 | LR: 1.43e-04 | 404.3ms/step | 49.5 samples/s +Step 28500/44000 | Loss: 2.8977 | LR: 1.42e-04 | 401.9ms/step | 49.8 samples/s +Validation at step 28500: Loss = 2.9380, Perplexity = 18.88 +Step 28510/44000 | Loss: 2.9742 | LR: 1.42e-04 | 571.5ms/step | 35.0 samples/s +Step 28520/44000 | Loss: 2.9364 | LR: 1.42e-04 | 405.8ms/step | 49.3 samples/s +Step 28530/44000 | Loss: 2.8996 | LR: 1.42e-04 | 405.2ms/step | 49.4 samples/s +Step 28540/44000 | Loss: 2.9392 | LR: 1.42e-04 | 404.8ms/step | 49.4 samples/s +Step 28550/44000 | Loss: 2.9479 | LR: 1.42e-04 | 404.5ms/step | 49.4 samples/s +Step 28560/44000 | Loss: 2.9622 | LR: 1.41e-04 | 404.7ms/step | 49.4 samples/s +Step 28570/44000 | Loss: 2.9358 | LR: 1.41e-04 | 406.0ms/step | 49.3 samples/s +Step 28580/44000 | Loss: 2.9146 | LR: 1.41e-04 | 406.1ms/step | 49.3 samples/s +Step 28590/44000 | Loss: 2.8658 | LR: 1.41e-04 | 405.0ms/step | 49.4 samples/s +Step 28600/44000 | Loss: 2.8978 | LR: 1.41e-04 | 405.3ms/step | 49.3 samples/s +Step 28610/44000 | Loss: 2.9410 | LR: 1.41e-04 | 405.5ms/step | 49.3 samples/s +Step 28620/44000 | Loss: 2.8867 | LR: 1.40e-04 | 405.2ms/step | 49.4 samples/s +Step 28630/44000 | Loss: 2.8992 | LR: 1.40e-04 | 405.2ms/step | 49.4 samples/s +Step 28640/44000 | Loss: 2.9730 | LR: 1.40e-04 | 405.0ms/step | 49.4 samples/s +Step 28650/44000 | Loss: 2.9298 | LR: 1.40e-04 | 405.0ms/step | 49.4 samples/s +Step 28660/44000 | Loss: 2.9308 | LR: 1.40e-04 | 404.5ms/step | 49.4 samples/s +Step 28670/44000 | Loss: 2.9914 | LR: 1.40e-04 | 404.1ms/step | 49.5 samples/s +Step 28680/44000 | Loss: 2.9722 | LR: 1.39e-04 | 403.7ms/step | 49.5 samples/s +Step 28690/44000 | Loss: 2.9847 | LR: 1.39e-04 | 404.3ms/step | 49.5 samples/s +Step 28700/44000 | Loss: 2.8924 | LR: 1.39e-04 | 404.3ms/step | 49.5 samples/s +Step 28710/44000 | Loss: 2.8735 | LR: 1.39e-04 | 404.5ms/step | 49.4 samples/s +Step 28720/44000 | Loss: 2.9423 | LR: 1.39e-04 | 404.0ms/step | 49.5 samples/s +Step 28730/44000 | Loss: 2.8713 | LR: 1.39e-04 | 404.2ms/step | 49.5 samples/s +Step 28740/44000 | Loss: 2.9243 | LR: 1.38e-04 | 401.7ms/step | 49.8 samples/s +Step 28750/44000 | Loss: 2.9030 | LR: 1.38e-04 | 401.4ms/step | 49.8 samples/s +Validation at step 28750: Loss = 2.9353, Perplexity = 18.83 +Step 28760/44000 | Loss: 2.9708 | LR: 1.38e-04 | 570.8ms/step | 35.0 samples/s +Step 28770/44000 | Loss: 2.9143 | LR: 1.38e-04 | 404.4ms/step | 49.5 samples/s +Step 28780/44000 | Loss: 3.0211 | LR: 1.38e-04 | 403.2ms/step | 49.6 samples/s +Step 28790/44000 | Loss: 2.9350 | LR: 1.38e-04 | 401.9ms/step | 49.8 samples/s +Step 28800/44000 | Loss: 2.9145 | LR: 1.38e-04 | 401.5ms/step | 49.8 samples/s +Step 28810/44000 | Loss: 2.9187 | LR: 1.37e-04 | 401.6ms/step | 49.8 samples/s +Step 28820/44000 | Loss: 2.9899 | LR: 1.37e-04 | 401.5ms/step | 49.8 samples/s +Step 28830/44000 | Loss: 2.9214 | LR: 1.37e-04 | 402.8ms/step | 49.6 samples/s +Step 28840/44000 | Loss: 2.9878 | LR: 1.37e-04 | 400.5ms/step | 49.9 samples/s +Step 28850/44000 | Loss: 2.9213 | LR: 1.37e-04 | 401.5ms/step | 49.8 samples/s +Step 28860/44000 | Loss: 2.9229 | LR: 1.37e-04 | 400.3ms/step | 50.0 samples/s +Step 28870/44000 | Loss: 2.9570 | LR: 1.36e-04 | 399.9ms/step | 50.0 samples/s +Step 28880/44000 | Loss: 2.9417 | LR: 1.36e-04 | 401.4ms/step | 49.8 samples/s +Step 28890/44000 | Loss: 2.9217 | LR: 1.36e-04 | 401.7ms/step | 49.8 samples/s +Step 28900/44000 | Loss: 2.9424 | LR: 1.36e-04 | 400.8ms/step | 49.9 samples/s +Step 28910/44000 | Loss: 2.9434 | LR: 1.36e-04 | 401.6ms/step | 49.8 samples/s +Step 28920/44000 | Loss: 2.9842 | LR: 1.36e-04 | 401.0ms/step | 49.9 samples/s +Step 28930/44000 | Loss: 2.8935 | LR: 1.35e-04 | 402.3ms/step | 49.7 samples/s +Step 28940/44000 | Loss: 2.9543 | LR: 1.35e-04 | 401.6ms/step | 49.8 samples/s +Step 28950/44000 | Loss: 2.9373 | LR: 1.35e-04 | 400.2ms/step | 50.0 samples/s +Step 28960/44000 | Loss: 2.9511 | LR: 1.35e-04 | 401.4ms/step | 49.8 samples/s +Step 28970/44000 | Loss: 2.9221 | LR: 1.35e-04 | 401.2ms/step | 49.9 samples/s +Step 28980/44000 | Loss: 2.8683 | LR: 1.35e-04 | 401.7ms/step | 49.8 samples/s +Step 28990/44000 | Loss: 2.9844 | LR: 1.34e-04 | 401.5ms/step | 49.8 samples/s +Step 29000/44000 | Loss: 2.9510 | LR: 1.34e-04 | 402.7ms/step | 49.7 samples/s +Validation at step 29000: Loss = 2.9350, Perplexity = 18.82 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-29000 +Step 29010/44000 | Loss: 2.9429 | LR: 1.34e-04 | 876.9ms/step | 22.8 samples/s +Step 29020/44000 | Loss: 2.9008 | LR: 1.34e-04 | 404.0ms/step | 49.5 samples/s +Step 29030/44000 | Loss: 2.8579 | LR: 1.34e-04 | 404.4ms/step | 49.5 samples/s +Step 29040/44000 | Loss: 2.9190 | LR: 1.34e-04 | 404.3ms/step | 49.5 samples/s +Step 29050/44000 | Loss: 2.9656 | LR: 1.33e-04 | 402.0ms/step | 49.7 samples/s +Step 29060/44000 | Loss: 2.9139 | LR: 1.33e-04 | 402.2ms/step | 49.7 samples/s +Step 29070/44000 | Loss: 2.9360 | LR: 1.33e-04 | 405.2ms/step | 49.4 samples/s +Step 29080/44000 | Loss: 2.9014 | LR: 1.33e-04 | 405.2ms/step | 49.4 samples/s +Step 29090/44000 | Loss: 2.9461 | LR: 1.33e-04 | 405.5ms/step | 49.3 samples/s +Step 29100/44000 | Loss: 2.9286 | LR: 1.33e-04 | 405.2ms/step | 49.4 samples/s +Step 29110/44000 | Loss: 2.9335 | LR: 1.33e-04 | 405.9ms/step | 49.3 samples/s +Step 29120/44000 | Loss: 2.9440 | LR: 1.32e-04 | 404.3ms/step | 49.5 samples/s +Step 29130/44000 | Loss: 2.9274 | LR: 1.32e-04 | 405.4ms/step | 49.3 samples/s +Step 29140/44000 | Loss: 2.9138 | LR: 1.32e-04 | 405.3ms/step | 49.3 samples/s +Step 29150/44000 | Loss: 2.9724 | LR: 1.32e-04 | 404.7ms/step | 49.4 samples/s +Step 29160/44000 | Loss: 2.9439 | LR: 1.32e-04 | 403.9ms/step | 49.5 samples/s +Step 29170/44000 | Loss: 2.8873 | LR: 1.32e-04 | 404.3ms/step | 49.5 samples/s +Step 29180/44000 | Loss: 2.9340 | LR: 1.31e-04 | 405.0ms/step | 49.4 samples/s +Step 29190/44000 | Loss: 2.9742 | LR: 1.31e-04 | 404.8ms/step | 49.4 samples/s +Step 29200/44000 | Loss: 2.9956 | LR: 1.31e-04 | 404.5ms/step | 49.4 samples/s +Step 29210/44000 | Loss: 2.9736 | LR: 1.31e-04 | 405.2ms/step | 49.4 samples/s +Step 29220/44000 | Loss: 2.9943 | LR: 1.31e-04 | 404.5ms/step | 49.4 samples/s +Step 29230/44000 | Loss: 2.9572 | LR: 1.31e-04 | 404.2ms/step | 49.5 samples/s +Step 29240/44000 | Loss: 2.9364 | LR: 1.30e-04 | 404.3ms/step | 49.5 samples/s +Step 29250/44000 | Loss: 2.9630 | LR: 1.30e-04 | 404.0ms/step | 49.5 samples/s +Validation at step 29250: Loss = 2.9339, Perplexity = 18.80 +Step 29260/44000 | Loss: 2.9672 | LR: 1.30e-04 | 572.7ms/step | 34.9 samples/s +Step 29270/44000 | Loss: 2.9777 | LR: 1.30e-04 | 403.4ms/step | 49.6 samples/s +Step 29280/44000 | Loss: 2.9657 | LR: 1.30e-04 | 404.1ms/step | 49.5 samples/s +Step 29290/44000 | Loss: 2.9053 | LR: 1.30e-04 | 405.5ms/step | 49.3 samples/s +Step 29300/44000 | Loss: 2.9442 | LR: 1.29e-04 | 404.0ms/step | 49.5 samples/s +Step 29310/44000 | Loss: 2.9862 | LR: 1.29e-04 | 404.6ms/step | 49.4 samples/s +Step 29320/44000 | Loss: 3.0043 | LR: 1.29e-04 | 404.2ms/step | 49.5 samples/s +Step 29330/44000 | Loss: 2.9051 | LR: 1.29e-04 | 404.8ms/step | 49.4 samples/s +Step 29340/44000 | Loss: 2.9522 | LR: 1.29e-04 | 403.0ms/step | 49.6 samples/s +Step 29350/44000 | Loss: 2.9143 | LR: 1.29e-04 | 403.9ms/step | 49.5 samples/s +Step 29360/44000 | Loss: 2.9259 | LR: 1.29e-04 | 404.0ms/step | 49.5 samples/s +Step 29370/44000 | Loss: 2.8868 | LR: 1.28e-04 | 404.4ms/step | 49.5 samples/s +Step 29380/44000 | Loss: 2.9721 | LR: 1.28e-04 | 404.1ms/step | 49.5 samples/s +Step 29390/44000 | Loss: 2.9823 | LR: 1.28e-04 | 404.8ms/step | 49.4 samples/s +Step 29400/44000 | Loss: 2.9280 | LR: 1.28e-04 | 404.1ms/step | 49.5 samples/s +Step 29410/44000 | Loss: 2.9518 | LR: 1.28e-04 | 404.2ms/step | 49.5 samples/s +Step 29420/44000 | Loss: 2.9397 | LR: 1.28e-04 | 403.0ms/step | 49.6 samples/s +Step 29430/44000 | Loss: 2.9600 | LR: 1.27e-04 | 404.6ms/step | 49.4 samples/s +Step 29440/44000 | Loss: 2.9531 | LR: 1.27e-04 | 404.3ms/step | 49.5 samples/s +Step 29450/44000 | Loss: 2.9840 | LR: 1.27e-04 | 404.5ms/step | 49.4 samples/s +Step 29460/44000 | Loss: 2.9241 | LR: 1.27e-04 | 404.5ms/step | 49.4 samples/s +Step 29470/44000 | Loss: 2.9658 | LR: 1.27e-04 | 402.6ms/step | 49.7 samples/s +Step 29480/44000 | Loss: 2.9812 | LR: 1.27e-04 | 417.4ms/step | 47.9 samples/s +Step 29490/44000 | Loss: 3.0021 | LR: 1.26e-04 | 420.5ms/step | 47.6 samples/s +Step 29500/44000 | Loss: 2.9763 | LR: 1.26e-04 | 421.0ms/step | 47.5 samples/s +Validation at step 29500: Loss = 2.9333, Perplexity = 18.79 +Step 29510/44000 | Loss: 2.9438 | LR: 1.26e-04 | 610.5ms/step | 32.8 samples/s +Step 29520/44000 | Loss: 2.9197 | LR: 1.26e-04 | 411.5ms/step | 48.6 samples/s +Step 29530/44000 | Loss: 2.9441 | LR: 1.26e-04 | 405.1ms/step | 49.4 samples/s +Step 29540/44000 | Loss: 2.9810 | LR: 1.26e-04 | 404.2ms/step | 49.5 samples/s +Step 29550/44000 | Loss: 2.8983 | LR: 1.26e-04 | 406.4ms/step | 49.2 samples/s +Step 29560/44000 | Loss: 2.9618 | LR: 1.25e-04 | 404.3ms/step | 49.5 samples/s +Step 29570/44000 | Loss: 2.9087 | LR: 1.25e-04 | 404.9ms/step | 49.4 samples/s +Step 29580/44000 | Loss: 2.9181 | LR: 1.25e-04 | 412.3ms/step | 48.5 samples/s +Step 29590/44000 | Loss: 2.9615 | LR: 1.25e-04 | 411.9ms/step | 48.6 samples/s +Step 29600/44000 | Loss: 2.9447 | LR: 1.25e-04 | 411.3ms/step | 48.6 samples/s +Step 29610/44000 | Loss: 2.9499 | LR: 1.25e-04 | 410.9ms/step | 48.7 samples/s +Step 29620/44000 | Loss: 2.9330 | LR: 1.24e-04 | 410.8ms/step | 48.7 samples/s +Step 29630/44000 | Loss: 2.8840 | LR: 1.24e-04 | 410.6ms/step | 48.7 samples/s +Step 29640/44000 | Loss: 2.9137 | LR: 1.24e-04 | 410.2ms/step | 48.8 samples/s +Step 29650/44000 | Loss: 2.9327 | LR: 1.24e-04 | 409.7ms/step | 48.8 samples/s +Step 29660/44000 | Loss: 2.9238 | LR: 1.24e-04 | 409.6ms/step | 48.8 samples/s +Step 29670/44000 | Loss: 2.9538 | LR: 1.24e-04 | 409.3ms/step | 48.9 samples/s +Step 29680/44000 | Loss: 2.9266 | LR: 1.23e-04 | 409.5ms/step | 48.8 samples/s +Step 29690/44000 | Loss: 2.9442 | LR: 1.23e-04 | 408.6ms/step | 48.9 samples/s +Step 29700/44000 | Loss: 2.9659 | LR: 1.23e-04 | 409.1ms/step | 48.9 samples/s +Step 29710/44000 | Loss: 2.9385 | LR: 1.23e-04 | 408.3ms/step | 49.0 samples/s +Step 29720/44000 | Loss: 2.8908 | LR: 1.23e-04 | 409.1ms/step | 48.9 samples/s +Step 29730/44000 | Loss: 2.9477 | LR: 1.23e-04 | 409.9ms/step | 48.8 samples/s +Step 29740/44000 | Loss: 2.9647 | LR: 1.23e-04 | 409.8ms/step | 48.8 samples/s +Step 29750/44000 | Loss: 2.9141 | LR: 1.22e-04 | 410.2ms/step | 48.8 samples/s +Validation at step 29750: Loss = 2.9318, Perplexity = 18.76 +Step 29760/44000 | Loss: 2.9533 | LR: 1.22e-04 | 588.6ms/step | 34.0 samples/s +Step 29770/44000 | Loss: 2.9438 | LR: 1.22e-04 | 409.4ms/step | 48.8 samples/s +Step 29780/44000 | Loss: 2.9067 | LR: 1.22e-04 | 408.7ms/step | 48.9 samples/s +Step 29790/44000 | Loss: 2.9198 | LR: 1.22e-04 | 408.9ms/step | 48.9 samples/s +Step 29800/44000 | Loss: 2.9039 | LR: 1.22e-04 | 409.1ms/step | 48.9 samples/s +Step 29810/44000 | Loss: 2.8940 | LR: 1.21e-04 | 408.7ms/step | 48.9 samples/s +Step 29820/44000 | Loss: 2.8847 | LR: 1.21e-04 | 408.4ms/step | 49.0 samples/s +Step 29830/44000 | Loss: 2.9388 | LR: 1.21e-04 | 408.4ms/step | 49.0 samples/s +Step 29840/44000 | Loss: 2.9473 | LR: 1.21e-04 | 408.3ms/step | 49.0 samples/s +Step 29850/44000 | Loss: 2.9023 | LR: 1.21e-04 | 407.9ms/step | 49.0 samples/s +Step 29860/44000 | Loss: 2.9224 | LR: 1.21e-04 | 408.0ms/step | 49.0 samples/s +Step 29870/44000 | Loss: 2.9658 | LR: 1.21e-04 | 407.9ms/step | 49.0 samples/s +Step 29880/44000 | Loss: 2.9185 | LR: 1.20e-04 | 407.2ms/step | 49.1 samples/s +Step 29890/44000 | Loss: 2.9613 | LR: 1.20e-04 | 407.8ms/step | 49.0 samples/s +Step 29900/44000 | Loss: 2.8813 | LR: 1.20e-04 | 407.2ms/step | 49.1 samples/s +Step 29910/44000 | Loss: 2.9964 | LR: 1.20e-04 | 407.3ms/step | 49.1 samples/s +Step 29920/44000 | Loss: 2.9010 | LR: 1.20e-04 | 407.5ms/step | 49.1 samples/s +Step 29930/44000 | Loss: 2.9193 | LR: 1.20e-04 | 407.3ms/step | 49.1 samples/s +Step 29940/44000 | Loss: 2.9261 | LR: 1.19e-04 | 407.9ms/step | 49.0 samples/s +Step 29950/44000 | Loss: 2.9294 | LR: 1.19e-04 | 407.5ms/step | 49.1 samples/s +Step 29960/44000 | Loss: 2.8856 | LR: 1.19e-04 | 407.7ms/step | 49.1 samples/s +Step 29970/44000 | Loss: 2.9393 | LR: 1.19e-04 | 407.8ms/step | 49.0 samples/s +Step 29980/44000 | Loss: 2.9196 | LR: 1.19e-04 | 407.2ms/step | 49.1 samples/s +Step 29990/44000 | Loss: 2.9242 | LR: 1.19e-04 | 409.2ms/step | 48.9 samples/s +Step 30000/44000 | Loss: 2.8489 | LR: 1.19e-04 | 407.4ms/step | 49.1 samples/s +Validation at step 30000: Loss = 2.9316, Perplexity = 18.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-30000 +Step 30010/44000 | Loss: 2.9332 | LR: 1.18e-04 | 880.3ms/step | 22.7 samples/s +Step 30020/44000 | Loss: 2.9667 | LR: 1.18e-04 | 407.2ms/step | 49.1 samples/s +Step 30030/44000 | Loss: 2.9045 | LR: 1.18e-04 | 408.6ms/step | 48.9 samples/s +Step 30040/44000 | Loss: 2.9346 | LR: 1.18e-04 | 408.6ms/step | 49.0 samples/s +Step 30050/44000 | Loss: 2.9253 | LR: 1.18e-04 | 407.6ms/step | 49.1 samples/s +Step 30060/44000 | Loss: 2.9952 | LR: 1.18e-04 | 406.8ms/step | 49.2 samples/s +Step 30070/44000 | Loss: 2.9633 | LR: 1.17e-04 | 406.3ms/step | 49.2 samples/s +Step 30080/44000 | Loss: 2.9385 | LR: 1.17e-04 | 407.4ms/step | 49.1 samples/s +Step 30090/44000 | Loss: 2.8979 | LR: 1.17e-04 | 407.0ms/step | 49.1 samples/s +Step 30100/44000 | Loss: 2.9371 | LR: 1.17e-04 | 407.6ms/step | 49.1 samples/s +Step 30110/44000 | Loss: 2.9081 | LR: 1.17e-04 | 407.0ms/step | 49.1 samples/s +Step 30120/44000 | Loss: 2.9016 | LR: 1.17e-04 | 407.7ms/step | 49.1 samples/s +Step 30130/44000 | Loss: 2.9408 | LR: 1.17e-04 | 407.9ms/step | 49.0 samples/s +Step 30140/44000 | Loss: 2.9141 | LR: 1.16e-04 | 406.9ms/step | 49.2 samples/s +Step 30150/44000 | Loss: 2.8707 | LR: 1.16e-04 | 407.3ms/step | 49.1 samples/s +Step 30160/44000 | Loss: 2.8783 | LR: 1.16e-04 | 406.8ms/step | 49.2 samples/s +Step 30170/44000 | Loss: 2.9303 | LR: 1.16e-04 | 408.4ms/step | 49.0 samples/s +Step 30180/44000 | Loss: 2.9542 | LR: 1.16e-04 | 408.1ms/step | 49.0 samples/s +Step 30190/44000 | Loss: 2.8988 | LR: 1.16e-04 | 407.4ms/step | 49.1 samples/s +Step 30200/44000 | Loss: 2.8931 | LR: 1.15e-04 | 406.1ms/step | 49.3 samples/s +Step 30210/44000 | Loss: 2.9122 | LR: 1.15e-04 | 405.7ms/step | 49.3 samples/s +Step 30220/44000 | Loss: 2.9434 | LR: 1.15e-04 | 406.4ms/step | 49.2 samples/s +Step 30230/44000 | Loss: 2.8944 | LR: 1.15e-04 | 406.7ms/step | 49.2 samples/s +Step 30240/44000 | Loss: 2.9255 | LR: 1.15e-04 | 406.6ms/step | 49.2 samples/s +Step 30250/44000 | Loss: 2.9716 | LR: 1.15e-04 | 406.5ms/step | 49.2 samples/s +Validation at step 30250: Loss = 2.9299, Perplexity = 18.73 +Step 30260/44000 | Loss: 2.9252 | LR: 1.15e-04 | 580.7ms/step | 34.4 samples/s +Step 30270/44000 | Loss: 2.9470 | LR: 1.14e-04 | 408.2ms/step | 49.0 samples/s +Step 30280/44000 | Loss: 2.9102 | LR: 1.14e-04 | 407.3ms/step | 49.1 samples/s +Step 30290/44000 | Loss: 2.9611 | LR: 1.14e-04 | 407.3ms/step | 49.1 samples/s +Step 30300/44000 | Loss: 2.9347 | LR: 1.14e-04 | 407.3ms/step | 49.1 samples/s +Step 30310/44000 | Loss: 2.9066 | LR: 1.14e-04 | 407.6ms/step | 49.1 samples/s +Step 30320/44000 | Loss: 2.9281 | LR: 1.14e-04 | 407.1ms/step | 49.1 samples/s +Step 30330/44000 | Loss: 2.9593 | LR: 1.13e-04 | 406.4ms/step | 49.2 samples/s +Step 30340/44000 | Loss: 2.9299 | LR: 1.13e-04 | 408.3ms/step | 49.0 samples/s +Step 30350/44000 | Loss: 2.9474 | LR: 1.13e-04 | 407.0ms/step | 49.1 samples/s +Step 30360/44000 | Loss: 2.9257 | LR: 1.13e-04 | 406.8ms/step | 49.2 samples/s +Step 30370/44000 | Loss: 2.9319 | LR: 1.13e-04 | 407.0ms/step | 49.1 samples/s +Step 30380/44000 | Loss: 2.9336 | LR: 1.13e-04 | 407.1ms/step | 49.1 samples/s +Step 30390/44000 | Loss: 3.0101 | LR: 1.13e-04 | 408.0ms/step | 49.0 samples/s +Step 30400/44000 | Loss: 2.9854 | LR: 1.12e-04 | 406.7ms/step | 49.2 samples/s +Step 30410/44000 | Loss: 2.9847 | LR: 1.12e-04 | 407.2ms/step | 49.1 samples/s +Step 30420/44000 | Loss: 2.8735 | LR: 1.12e-04 | 408.0ms/step | 49.0 samples/s +Step 30430/44000 | Loss: 2.9083 | LR: 1.12e-04 | 406.5ms/step | 49.2 samples/s +Step 30440/44000 | Loss: 2.9449 | LR: 1.12e-04 | 407.1ms/step | 49.1 samples/s +Step 30450/44000 | Loss: 2.9977 | LR: 1.12e-04 | 406.1ms/step | 49.3 samples/s +Step 30460/44000 | Loss: 2.9107 | LR: 1.11e-04 | 406.0ms/step | 49.3 samples/s +Step 30470/44000 | Loss: 2.8826 | LR: 1.11e-04 | 406.3ms/step | 49.2 samples/s +Step 30480/44000 | Loss: 2.8925 | LR: 1.11e-04 | 406.2ms/step | 49.2 samples/s +Step 30490/44000 | Loss: 2.9382 | LR: 1.11e-04 | 405.3ms/step | 49.3 samples/s +Step 30500/44000 | Loss: 2.9552 | LR: 1.11e-04 | 404.5ms/step | 49.4 samples/s +Validation at step 30500: Loss = 2.9297, Perplexity = 18.72 +Step 30510/44000 | Loss: 2.9690 | LR: 1.11e-04 | 579.6ms/step | 34.5 samples/s +Step 30520/44000 | Loss: 2.9002 | LR: 1.11e-04 | 404.0ms/step | 49.5 samples/s +Step 30530/44000 | Loss: 2.9339 | LR: 1.10e-04 | 404.6ms/step | 49.4 samples/s +Step 30540/44000 | Loss: 2.8911 | LR: 1.10e-04 | 403.9ms/step | 49.5 samples/s +Step 30550/44000 | Loss: 3.0056 | LR: 1.10e-04 | 404.3ms/step | 49.5 samples/s +Step 30560/44000 | Loss: 2.9355 | LR: 1.10e-04 | 404.6ms/step | 49.4 samples/s +Step 30570/44000 | Loss: 2.9729 | LR: 1.10e-04 | 403.8ms/step | 49.5 samples/s +Step 30580/44000 | Loss: 2.9209 | LR: 1.10e-04 | 405.3ms/step | 49.3 samples/s +Step 30590/44000 | Loss: 2.9475 | LR: 1.10e-04 | 403.7ms/step | 49.5 samples/s +Step 30600/44000 | Loss: 2.8929 | LR: 1.09e-04 | 404.0ms/step | 49.5 samples/s +Step 30610/44000 | Loss: 2.9304 | LR: 1.09e-04 | 405.6ms/step | 49.3 samples/s +Step 30620/44000 | Loss: 2.8910 | LR: 1.09e-04 | 404.1ms/step | 49.5 samples/s +Step 30630/44000 | Loss: 2.9080 | LR: 1.09e-04 | 405.5ms/step | 49.3 samples/s +Step 30640/44000 | Loss: 2.9246 | LR: 1.09e-04 | 406.3ms/step | 49.2 samples/s +Step 30650/44000 | Loss: 2.9691 | LR: 1.09e-04 | 405.8ms/step | 49.3 samples/s +Step 30660/44000 | Loss: 2.9323 | LR: 1.08e-04 | 404.7ms/step | 49.4 samples/s +Step 30670/44000 | Loss: 2.9410 | LR: 1.08e-04 | 405.1ms/step | 49.4 samples/s +Step 30680/44000 | Loss: 2.8975 | LR: 1.08e-04 | 405.7ms/step | 49.3 samples/s +Step 30690/44000 | Loss: 2.9131 | LR: 1.08e-04 | 406.0ms/step | 49.3 samples/s +Step 30700/44000 | Loss: 3.0044 | LR: 1.08e-04 | 405.6ms/step | 49.3 samples/s +Step 30710/44000 | Loss: 2.9198 | LR: 1.08e-04 | 405.9ms/step | 49.3 samples/s +Step 30720/44000 | Loss: 2.9350 | LR: 1.08e-04 | 404.7ms/step | 49.4 samples/s +Step 30730/44000 | Loss: 2.9158 | LR: 1.07e-04 | 405.1ms/step | 49.4 samples/s +Step 30740/44000 | Loss: 2.9497 | LR: 1.07e-04 | 405.4ms/step | 49.3 samples/s +Step 30750/44000 | Loss: 2.9371 | LR: 1.07e-04 | 405.0ms/step | 49.4 samples/s +Validation at step 30750: Loss = 2.9292, Perplexity = 18.71 +Step 30760/44000 | Loss: 2.9541 | LR: 1.07e-04 | 578.2ms/step | 34.6 samples/s +Step 30770/44000 | Loss: 2.9207 | LR: 1.07e-04 | 405.3ms/step | 49.3 samples/s +Step 30780/44000 | Loss: 2.9402 | LR: 1.07e-04 | 405.5ms/step | 49.3 samples/s +Step 30790/44000 | Loss: 2.9324 | LR: 1.07e-04 | 405.6ms/step | 49.3 samples/s +Step 30800/44000 | Loss: 2.9388 | LR: 1.06e-04 | 405.4ms/step | 49.3 samples/s +Step 30810/44000 | Loss: 2.9317 | LR: 1.06e-04 | 404.2ms/step | 49.5 samples/s +Step 30820/44000 | Loss: 2.9015 | LR: 1.06e-04 | 404.7ms/step | 49.4 samples/s +Step 30830/44000 | Loss: 2.8893 | LR: 1.06e-04 | 404.6ms/step | 49.4 samples/s +Step 30840/44000 | Loss: 2.9594 | LR: 1.06e-04 | 404.4ms/step | 49.5 samples/s +Step 30850/44000 | Loss: 2.9420 | LR: 1.06e-04 | 404.9ms/step | 49.4 samples/s +Step 30860/44000 | Loss: 2.9330 | LR: 1.05e-04 | 405.2ms/step | 49.4 samples/s +Step 30870/44000 | Loss: 2.8601 | LR: 1.05e-04 | 404.2ms/step | 49.5 samples/s +Step 30880/44000 | Loss: 2.9288 | LR: 1.05e-04 | 405.1ms/step | 49.4 samples/s +Step 30890/44000 | Loss: 2.9769 | LR: 1.05e-04 | 404.9ms/step | 49.4 samples/s +Step 30900/44000 | Loss: 2.9474 | LR: 1.05e-04 | 407.0ms/step | 49.1 samples/s +Step 30910/44000 | Loss: 2.9315 | LR: 1.05e-04 | 407.3ms/step | 49.1 samples/s +Step 30920/44000 | Loss: 2.9609 | LR: 1.05e-04 | 408.5ms/step | 49.0 samples/s +Step 30930/44000 | Loss: 2.9831 | LR: 1.04e-04 | 406.2ms/step | 49.2 samples/s +Step 30940/44000 | Loss: 2.9430 | LR: 1.04e-04 | 407.7ms/step | 49.1 samples/s +Step 30950/44000 | Loss: 2.9359 | LR: 1.04e-04 | 406.7ms/step | 49.2 samples/s +Step 30960/44000 | Loss: 2.9943 | LR: 1.04e-04 | 404.6ms/step | 49.4 samples/s +Step 30970/44000 | Loss: 2.9571 | LR: 1.04e-04 | 406.1ms/step | 49.3 samples/s +Step 30980/44000 | Loss: 2.9431 | LR: 1.04e-04 | 406.0ms/step | 49.3 samples/s +Step 30990/44000 | Loss: 2.9075 | LR: 1.04e-04 | 404.6ms/step | 49.4 samples/s +Step 31000/44000 | Loss: 2.9646 | LR: 1.03e-04 | 404.9ms/step | 49.4 samples/s +Validation at step 31000: Loss = 2.9279, Perplexity = 18.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-31000 +Step 31010/44000 | Loss: 2.8783 | LR: 1.03e-04 | 871.1ms/step | 23.0 samples/s +Step 31020/44000 | Loss: 2.9289 | LR: 1.03e-04 | 406.1ms/step | 49.3 samples/s +Step 31030/44000 | Loss: 2.9914 | LR: 1.03e-04 | 405.3ms/step | 49.3 samples/s +Step 31040/44000 | Loss: 3.0095 | LR: 1.03e-04 | 405.0ms/step | 49.4 samples/s +Step 31050/44000 | Loss: 2.8608 | LR: 1.03e-04 | 405.3ms/step | 49.3 samples/s +Step 31060/44000 | Loss: 2.9292 | LR: 1.03e-04 | 405.0ms/step | 49.4 samples/s +Step 31070/44000 | Loss: 2.8774 | LR: 1.02e-04 | 403.9ms/step | 49.5 samples/s +Step 31080/44000 | Loss: 2.8955 | LR: 1.02e-04 | 404.0ms/step | 49.5 samples/s +Step 31090/44000 | Loss: 2.9722 | LR: 1.02e-04 | 404.4ms/step | 49.5 samples/s +Step 31100/44000 | Loss: 2.9477 | LR: 1.02e-04 | 405.3ms/step | 49.4 samples/s +Step 31110/44000 | Loss: 2.9382 | LR: 1.02e-04 | 404.8ms/step | 49.4 samples/s +Step 31120/44000 | Loss: 2.9675 | LR: 1.02e-04 | 404.1ms/step | 49.5 samples/s +Step 31130/44000 | Loss: 2.9150 | LR: 1.02e-04 | 403.9ms/step | 49.5 samples/s +Step 31140/44000 | Loss: 2.8849 | LR: 1.01e-04 | 404.8ms/step | 49.4 samples/s +Step 31150/44000 | Loss: 2.9115 | LR: 1.01e-04 | 405.8ms/step | 49.3 samples/s +Step 31160/44000 | Loss: 2.8877 | LR: 1.01e-04 | 405.6ms/step | 49.3 samples/s +Step 31170/44000 | Loss: 2.9257 | LR: 1.01e-04 | 404.8ms/step | 49.4 samples/s +Step 31180/44000 | Loss: 2.8532 | LR: 1.01e-04 | 405.1ms/step | 49.4 samples/s +Step 31190/44000 | Loss: 2.9288 | LR: 1.01e-04 | 406.0ms/step | 49.3 samples/s +Step 31200/44000 | Loss: 2.9532 | LR: 1.00e-04 | 405.6ms/step | 49.3 samples/s +Step 31210/44000 | Loss: 2.9153 | LR: 1.00e-04 | 404.9ms/step | 49.4 samples/s +Step 31220/44000 | Loss: 2.8830 | LR: 1.00e-04 | 404.9ms/step | 49.4 samples/s +Step 31230/44000 | Loss: 2.9012 | LR: 1.00e-04 | 405.9ms/step | 49.3 samples/s +Step 31240/44000 | Loss: 2.9645 | LR: 9.99e-05 | 404.5ms/step | 49.4 samples/s +Step 31250/44000 | Loss: 2.8734 | LR: 9.98e-05 | 403.8ms/step | 49.5 samples/s +Validation at step 31250: Loss = 2.9277, Perplexity = 18.69 +Step 31260/44000 | Loss: 2.9364 | LR: 9.96e-05 | 574.2ms/step | 34.8 samples/s +Step 31270/44000 | Loss: 2.9309 | LR: 9.95e-05 | 404.4ms/step | 49.5 samples/s +Step 31280/44000 | Loss: 2.9227 | LR: 9.93e-05 | 404.2ms/step | 49.5 samples/s +Step 31290/44000 | Loss: 2.9242 | LR: 9.92e-05 | 404.2ms/step | 49.5 samples/s +Step 31300/44000 | Loss: 2.8945 | LR: 9.90e-05 | 404.5ms/step | 49.4 samples/s +Step 31310/44000 | Loss: 2.9011 | LR: 9.89e-05 | 405.1ms/step | 49.4 samples/s +Step 31320/44000 | Loss: 2.8933 | LR: 9.88e-05 | 404.4ms/step | 49.5 samples/s +Step 31330/44000 | Loss: 2.8815 | LR: 9.86e-05 | 404.5ms/step | 49.4 samples/s +Step 31340/44000 | Loss: 2.9510 | LR: 9.85e-05 | 403.7ms/step | 49.5 samples/s +Step 31350/44000 | Loss: 2.9187 | LR: 9.83e-05 | 403.6ms/step | 49.6 samples/s +Step 31360/44000 | Loss: 2.9090 | LR: 9.82e-05 | 405.2ms/step | 49.4 samples/s +Step 31370/44000 | Loss: 2.9489 | LR: 9.80e-05 | 403.9ms/step | 49.5 samples/s +Step 31380/44000 | Loss: 2.9338 | LR: 9.79e-05 | 404.6ms/step | 49.4 samples/s +Step 31390/44000 | Loss: 2.9223 | LR: 9.77e-05 | 404.0ms/step | 49.5 samples/s +Step 31400/44000 | Loss: 2.9308 | LR: 9.76e-05 | 404.4ms/step | 49.5 samples/s +Step 31410/44000 | Loss: 2.9370 | LR: 9.75e-05 | 403.8ms/step | 49.5 samples/s +Step 31420/44000 | Loss: 2.9648 | LR: 9.73e-05 | 403.1ms/step | 49.6 samples/s +Step 31430/44000 | Loss: 2.9301 | LR: 9.72e-05 | 404.4ms/step | 49.5 samples/s +Step 31440/44000 | Loss: 2.9375 | LR: 9.70e-05 | 404.3ms/step | 49.5 samples/s +Step 31450/44000 | Loss: 2.9294 | LR: 9.69e-05 | 405.0ms/step | 49.4 samples/s +Step 31460/44000 | Loss: 2.9442 | LR: 9.67e-05 | 403.3ms/step | 49.6 samples/s +Step 31470/44000 | Loss: 2.9536 | LR: 9.66e-05 | 403.7ms/step | 49.5 samples/s +Step 31480/44000 | Loss: 2.9385 | LR: 9.65e-05 | 403.6ms/step | 49.5 samples/s +Step 31490/44000 | Loss: 2.9165 | LR: 9.63e-05 | 403.9ms/step | 49.5 samples/s +Step 31500/44000 | Loss: 2.9104 | LR: 9.62e-05 | 404.6ms/step | 49.4 samples/s +Validation at step 31500: Loss = 2.9270, Perplexity = 18.67 +Step 31510/44000 | Loss: 2.9829 | LR: 9.60e-05 | 573.3ms/step | 34.9 samples/s +Step 31520/44000 | Loss: 2.9001 | LR: 9.59e-05 | 403.9ms/step | 49.5 samples/s +Step 31530/44000 | Loss: 2.9411 | LR: 9.57e-05 | 404.0ms/step | 49.5 samples/s +Step 31540/44000 | Loss: 2.9609 | LR: 9.56e-05 | 403.9ms/step | 49.5 samples/s +Step 31550/44000 | Loss: 3.0028 | LR: 9.54e-05 | 405.6ms/step | 49.3 samples/s +Step 31560/44000 | Loss: 2.9317 | LR: 9.53e-05 | 404.9ms/step | 49.4 samples/s +Step 31570/44000 | Loss: 2.9252 | LR: 9.52e-05 | 405.2ms/step | 49.4 samples/s +Step 31580/44000 | Loss: 2.9264 | LR: 9.50e-05 | 404.5ms/step | 49.4 samples/s +Step 31590/44000 | Loss: 2.8882 | LR: 9.49e-05 | 405.8ms/step | 49.3 samples/s +Step 31600/44000 | Loss: 2.9841 | LR: 9.47e-05 | 404.8ms/step | 49.4 samples/s +Step 31610/44000 | Loss: 2.9021 | LR: 9.46e-05 | 404.5ms/step | 49.4 samples/s +Step 31620/44000 | Loss: 2.9650 | LR: 9.45e-05 | 404.8ms/step | 49.4 samples/s +Step 31630/44000 | Loss: 2.9579 | LR: 9.43e-05 | 404.6ms/step | 49.4 samples/s +Step 31640/44000 | Loss: 2.9340 | LR: 9.42e-05 | 404.9ms/step | 49.4 samples/s +Step 31650/44000 | Loss: 2.9510 | LR: 9.40e-05 | 404.1ms/step | 49.5 samples/s +Step 31660/44000 | Loss: 2.9394 | LR: 9.39e-05 | 405.2ms/step | 49.4 samples/s +Step 31670/44000 | Loss: 2.9432 | LR: 9.37e-05 | 405.9ms/step | 49.3 samples/s +Step 31680/44000 | Loss: 2.9212 | LR: 9.36e-05 | 405.4ms/step | 49.3 samples/s +Step 31690/44000 | Loss: 2.9398 | LR: 9.35e-05 | 405.7ms/step | 49.3 samples/s +Step 31700/44000 | Loss: 2.9470 | LR: 9.33e-05 | 405.7ms/step | 49.3 samples/s +Step 31710/44000 | Loss: 2.8820 | LR: 9.32e-05 | 405.5ms/step | 49.3 samples/s +Step 31720/44000 | Loss: 2.9201 | LR: 9.30e-05 | 406.9ms/step | 49.2 samples/s +Step 31730/44000 | Loss: 2.9390 | LR: 9.29e-05 | 406.1ms/step | 49.3 samples/s +Step 31740/44000 | Loss: 2.9325 | LR: 9.28e-05 | 406.2ms/step | 49.2 samples/s +Step 31750/44000 | Loss: 2.8996 | LR: 9.26e-05 | 403.5ms/step | 49.6 samples/s +Validation at step 31750: Loss = 2.9267, Perplexity = 18.67 +Step 31760/44000 | Loss: 2.9312 | LR: 9.25e-05 | 574.8ms/step | 34.8 samples/s +Step 31770/44000 | Loss: 2.8382 | LR: 9.23e-05 | 405.3ms/step | 49.3 samples/s +Step 31780/44000 | Loss: 2.9131 | LR: 9.22e-05 | 405.6ms/step | 49.3 samples/s +Step 31790/44000 | Loss: 2.8999 | LR: 9.20e-05 | 404.6ms/step | 49.4 samples/s +Step 31800/44000 | Loss: 2.8908 | LR: 9.19e-05 | 403.7ms/step | 49.5 samples/s +Step 31810/44000 | Loss: 2.9505 | LR: 9.18e-05 | 405.0ms/step | 49.4 samples/s +Step 31820/44000 | Loss: 2.9763 | LR: 9.16e-05 | 404.7ms/step | 49.4 samples/s +Step 31830/44000 | Loss: 2.9256 | LR: 9.15e-05 | 404.2ms/step | 49.5 samples/s +Step 31840/44000 | Loss: 2.9383 | LR: 9.13e-05 | 403.9ms/step | 49.5 samples/s +Step 31850/44000 | Loss: 2.9840 | LR: 9.12e-05 | 403.6ms/step | 49.6 samples/s +Step 31860/44000 | Loss: 2.9720 | LR: 9.11e-05 | 404.3ms/step | 49.5 samples/s +Step 31870/44000 | Loss: 2.8783 | LR: 9.09e-05 | 403.7ms/step | 49.5 samples/s +Step 31880/44000 | Loss: 2.9769 | LR: 9.08e-05 | 404.8ms/step | 49.4 samples/s +Step 31890/44000 | Loss: 2.8380 | LR: 9.06e-05 | 403.9ms/step | 49.5 samples/s +Step 31900/44000 | Loss: 2.9218 | LR: 9.05e-05 | 404.0ms/step | 49.5 samples/s +Step 31910/44000 | Loss: 2.9521 | LR: 9.04e-05 | 404.4ms/step | 49.5 samples/s +Step 31920/44000 | Loss: 2.9194 | LR: 9.02e-05 | 404.5ms/step | 49.4 samples/s +Step 31930/44000 | Loss: 3.0045 | LR: 9.01e-05 | 404.6ms/step | 49.4 samples/s +Step 31940/44000 | Loss: 2.9252 | LR: 8.99e-05 | 405.9ms/step | 49.3 samples/s +Step 31950/44000 | Loss: 2.9050 | LR: 8.98e-05 | 404.7ms/step | 49.4 samples/s +Step 31960/44000 | Loss: 2.9311 | LR: 8.97e-05 | 405.0ms/step | 49.4 samples/s +Step 31970/44000 | Loss: 2.8966 | LR: 8.95e-05 | 405.8ms/step | 49.3 samples/s +Step 31980/44000 | Loss: 2.9254 | LR: 8.94e-05 | 405.0ms/step | 49.4 samples/s +Step 31990/44000 | Loss: 2.9268 | LR: 8.92e-05 | 404.7ms/step | 49.4 samples/s +Step 32000/44000 | Loss: 2.9233 | LR: 8.91e-05 | 404.2ms/step | 49.5 samples/s +Validation at step 32000: Loss = 2.9258, Perplexity = 18.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-32000 +Step 32010/44000 | Loss: 2.9126 | LR: 8.90e-05 | 891.2ms/step | 22.4 samples/s +Step 32020/44000 | Loss: 2.9601 | LR: 8.88e-05 | 404.9ms/step | 49.4 samples/s +Step 32030/44000 | Loss: 3.0090 | LR: 8.87e-05 | 403.8ms/step | 49.5 samples/s +Step 32040/44000 | Loss: 2.9158 | LR: 8.86e-05 | 403.5ms/step | 49.6 samples/s +Step 32050/44000 | Loss: 2.8475 | LR: 8.84e-05 | 403.1ms/step | 49.6 samples/s +Step 32060/44000 | Loss: 2.8858 | LR: 8.83e-05 | 403.2ms/step | 49.6 samples/s +Step 32070/44000 | Loss: 2.8849 | LR: 8.81e-05 | 403.8ms/step | 49.5 samples/s +Step 32080/44000 | Loss: 2.9229 | LR: 8.80e-05 | 403.8ms/step | 49.5 samples/s +Step 32090/44000 | Loss: 2.8993 | LR: 8.79e-05 | 403.1ms/step | 49.6 samples/s +Step 32100/44000 | Loss: 2.9583 | LR: 8.77e-05 | 404.6ms/step | 49.4 samples/s +Step 32110/44000 | Loss: 2.9346 | LR: 8.76e-05 | 403.8ms/step | 49.5 samples/s +Step 32120/44000 | Loss: 2.9724 | LR: 8.74e-05 | 404.1ms/step | 49.5 samples/s +Step 32130/44000 | Loss: 2.9651 | LR: 8.73e-05 | 404.3ms/step | 49.5 samples/s +Step 32140/44000 | Loss: 2.8662 | LR: 8.72e-05 | 403.8ms/step | 49.5 samples/s +Step 32150/44000 | Loss: 2.9531 | LR: 8.70e-05 | 403.8ms/step | 49.5 samples/s +Step 32160/44000 | Loss: 2.8994 | LR: 8.69e-05 | 403.2ms/step | 49.6 samples/s +Step 32170/44000 | Loss: 2.9150 | LR: 8.68e-05 | 405.0ms/step | 49.4 samples/s +Step 32180/44000 | Loss: 2.9339 | LR: 8.66e-05 | 404.4ms/step | 49.5 samples/s +Step 32190/44000 | Loss: 2.8990 | LR: 8.65e-05 | 402.9ms/step | 49.6 samples/s +Step 32200/44000 | Loss: 2.9543 | LR: 8.63e-05 | 404.6ms/step | 49.4 samples/s +Step 32210/44000 | Loss: 2.9533 | LR: 8.62e-05 | 404.0ms/step | 49.5 samples/s +Step 32220/44000 | Loss: 2.9418 | LR: 8.61e-05 | 404.0ms/step | 49.5 samples/s +Step 32230/44000 | Loss: 2.9022 | LR: 8.59e-05 | 404.4ms/step | 49.5 samples/s +Step 32240/44000 | Loss: 2.9396 | LR: 8.58e-05 | 403.8ms/step | 49.5 samples/s +Step 32250/44000 | Loss: 2.9301 | LR: 8.57e-05 | 404.5ms/step | 49.4 samples/s +Validation at step 32250: Loss = 2.9255, Perplexity = 18.64 +Step 32260/44000 | Loss: 2.9320 | LR: 8.55e-05 | 573.4ms/step | 34.9 samples/s +Step 32270/44000 | Loss: 3.0088 | LR: 8.54e-05 | 404.1ms/step | 49.5 samples/s +Step 32280/44000 | Loss: 2.9810 | LR: 8.53e-05 | 403.4ms/step | 49.6 samples/s +Step 32290/44000 | Loss: 2.9377 | LR: 8.51e-05 | 403.7ms/step | 49.5 samples/s +Step 32300/44000 | Loss: 2.9335 | LR: 8.50e-05 | 404.9ms/step | 49.4 samples/s +Step 32310/44000 | Loss: 2.9733 | LR: 8.48e-05 | 405.9ms/step | 49.3 samples/s +Step 32320/44000 | Loss: 2.9437 | LR: 8.47e-05 | 404.6ms/step | 49.4 samples/s +Step 32330/44000 | Loss: 2.9054 | LR: 8.46e-05 | 403.0ms/step | 49.6 samples/s +Step 32340/44000 | Loss: 2.9000 | LR: 8.44e-05 | 403.1ms/step | 49.6 samples/s +Step 32350/44000 | Loss: 2.9538 | LR: 8.43e-05 | 403.2ms/step | 49.6 samples/s +Step 32360/44000 | Loss: 2.9634 | LR: 8.42e-05 | 403.5ms/step | 49.6 samples/s +Step 32370/44000 | Loss: 2.8689 | LR: 8.40e-05 | 404.6ms/step | 49.4 samples/s +Step 32380/44000 | Loss: 2.9513 | LR: 8.39e-05 | 403.4ms/step | 49.6 samples/s +Step 32390/44000 | Loss: 2.9455 | LR: 8.38e-05 | 404.1ms/step | 49.5 samples/s +Step 32400/44000 | Loss: 2.9223 | LR: 8.36e-05 | 405.1ms/step | 49.4 samples/s +Step 32410/44000 | Loss: 2.8942 | LR: 8.35e-05 | 405.6ms/step | 49.3 samples/s +Step 32420/44000 | Loss: 2.9505 | LR: 8.33e-05 | 404.7ms/step | 49.4 samples/s +Step 32430/44000 | Loss: 2.8731 | LR: 8.32e-05 | 403.5ms/step | 49.6 samples/s +Step 32440/44000 | Loss: 2.9341 | LR: 8.31e-05 | 403.7ms/step | 49.5 samples/s +Step 32450/44000 | Loss: 2.9229 | LR: 8.29e-05 | 403.3ms/step | 49.6 samples/s +Step 32460/44000 | Loss: 2.9413 | LR: 8.28e-05 | 402.9ms/step | 49.6 samples/s +Step 32470/44000 | Loss: 2.9618 | LR: 8.27e-05 | 403.2ms/step | 49.6 samples/s +Step 32480/44000 | Loss: 2.9064 | LR: 8.25e-05 | 402.7ms/step | 49.7 samples/s +Step 32490/44000 | Loss: 2.9158 | LR: 8.24e-05 | 403.7ms/step | 49.5 samples/s +Step 32500/44000 | Loss: 2.9572 | LR: 8.23e-05 | 402.8ms/step | 49.7 samples/s +Validation at step 32500: Loss = 2.9257, Perplexity = 18.65 +Step 32510/44000 | Loss: 2.8968 | LR: 8.21e-05 | 574.9ms/step | 34.8 samples/s +Step 32520/44000 | Loss: 2.8868 | LR: 8.20e-05 | 404.1ms/step | 49.5 samples/s +Step 32530/44000 | Loss: 2.9274 | LR: 8.19e-05 | 404.3ms/step | 49.5 samples/s +Step 32540/44000 | Loss: 2.9429 | LR: 8.17e-05 | 404.3ms/step | 49.5 samples/s +Step 32550/44000 | Loss: 2.9766 | LR: 8.16e-05 | 403.9ms/step | 49.5 samples/s +Step 32560/44000 | Loss: 2.9200 | LR: 8.15e-05 | 404.7ms/step | 49.4 samples/s +Step 32570/44000 | Loss: 2.9649 | LR: 8.13e-05 | 404.2ms/step | 49.5 samples/s +Step 32580/44000 | Loss: 2.8866 | LR: 8.12e-05 | 404.6ms/step | 49.4 samples/s +Step 32590/44000 | Loss: 2.9145 | LR: 8.11e-05 | 404.5ms/step | 49.4 samples/s +Step 32600/44000 | Loss: 2.9062 | LR: 8.09e-05 | 403.9ms/step | 49.5 samples/s +Step 32610/44000 | Loss: 2.9277 | LR: 8.08e-05 | 404.8ms/step | 49.4 samples/s +Step 32620/44000 | Loss: 2.9274 | LR: 8.07e-05 | 404.7ms/step | 49.4 samples/s +Step 32630/44000 | Loss: 2.8881 | LR: 8.05e-05 | 404.8ms/step | 49.4 samples/s +Step 32640/44000 | Loss: 2.9779 | LR: 8.04e-05 | 404.1ms/step | 49.5 samples/s +Step 32650/44000 | Loss: 2.9539 | LR: 8.03e-05 | 403.7ms/step | 49.5 samples/s +Step 32660/44000 | Loss: 2.9481 | LR: 8.01e-05 | 403.7ms/step | 49.5 samples/s +Step 32670/44000 | Loss: 2.9168 | LR: 8.00e-05 | 403.7ms/step | 49.5 samples/s +Step 32680/44000 | Loss: 2.8998 | LR: 7.99e-05 | 403.2ms/step | 49.6 samples/s +Step 32690/44000 | Loss: 2.8660 | LR: 7.97e-05 | 404.7ms/step | 49.4 samples/s +Step 32700/44000 | Loss: 2.9568 | LR: 7.96e-05 | 403.4ms/step | 49.6 samples/s +Step 32710/44000 | Loss: 2.9668 | LR: 7.95e-05 | 403.4ms/step | 49.6 samples/s +Step 32720/44000 | Loss: 2.9504 | LR: 7.93e-05 | 403.0ms/step | 49.6 samples/s +Step 32730/44000 | Loss: 2.8996 | LR: 7.92e-05 | 400.8ms/step | 49.9 samples/s +Step 32740/44000 | Loss: 2.9401 | LR: 7.91e-05 | 401.3ms/step | 49.8 samples/s +Step 32750/44000 | Loss: 2.8862 | LR: 7.89e-05 | 403.3ms/step | 49.6 samples/s +Validation at step 32750: Loss = 2.9245, Perplexity = 18.62 +Step 32760/44000 | Loss: 2.9562 | LR: 7.88e-05 | 571.3ms/step | 35.0 samples/s +Step 32770/44000 | Loss: 2.9365 | LR: 7.87e-05 | 401.1ms/step | 49.9 samples/s +Step 32780/44000 | Loss: 2.9032 | LR: 7.85e-05 | 401.9ms/step | 49.8 samples/s +Step 32790/44000 | Loss: 2.8739 | LR: 7.84e-05 | 401.3ms/step | 49.8 samples/s +Step 32800/44000 | Loss: 2.8504 | LR: 7.83e-05 | 402.2ms/step | 49.7 samples/s +Step 32810/44000 | Loss: 2.8996 | LR: 7.81e-05 | 402.0ms/step | 49.8 samples/s +Step 32820/44000 | Loss: 2.8800 | LR: 7.80e-05 | 401.8ms/step | 49.8 samples/s +Step 32830/44000 | Loss: 2.9895 | LR: 7.79e-05 | 404.6ms/step | 49.4 samples/s +Step 32840/44000 | Loss: 2.9045 | LR: 7.77e-05 | 404.6ms/step | 49.4 samples/s +Step 32850/44000 | Loss: 2.8955 | LR: 7.76e-05 | 404.1ms/step | 49.5 samples/s +Step 32860/44000 | Loss: 2.8403 | LR: 7.75e-05 | 403.8ms/step | 49.5 samples/s +Step 32870/44000 | Loss: 2.9547 | LR: 7.73e-05 | 403.4ms/step | 49.6 samples/s +Step 32880/44000 | Loss: 2.9077 | LR: 7.72e-05 | 404.7ms/step | 49.4 samples/s +Step 32890/44000 | Loss: 2.8928 | LR: 7.71e-05 | 403.5ms/step | 49.6 samples/s +Step 32900/44000 | Loss: 2.8778 | LR: 7.70e-05 | 403.9ms/step | 49.5 samples/s +Step 32910/44000 | Loss: 2.9370 | LR: 7.68e-05 | 404.7ms/step | 49.4 samples/s +Step 32920/44000 | Loss: 2.8806 | LR: 7.67e-05 | 403.2ms/step | 49.6 samples/s +Step 32930/44000 | Loss: 2.9054 | LR: 7.66e-05 | 404.1ms/step | 49.5 samples/s +Step 32940/44000 | Loss: 2.9195 | LR: 7.64e-05 | 403.1ms/step | 49.6 samples/s +Step 32950/44000 | Loss: 2.8921 | LR: 7.63e-05 | 403.1ms/step | 49.6 samples/s +Step 32960/44000 | Loss: 2.9039 | LR: 7.62e-05 | 403.3ms/step | 49.6 samples/s +Step 32970/44000 | Loss: 2.9446 | LR: 7.60e-05 | 402.6ms/step | 49.7 samples/s +Step 32980/44000 | Loss: 2.9152 | LR: 7.59e-05 | 403.2ms/step | 49.6 samples/s +Step 32990/44000 | Loss: 2.8949 | LR: 7.58e-05 | 403.4ms/step | 49.6 samples/s +Step 33000/44000 | Loss: 2.9770 | LR: 7.56e-05 | 403.0ms/step | 49.6 samples/s +Validation at step 33000: Loss = 2.9246, Perplexity = 18.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-33000 +Step 33010/44000 | Loss: 2.9111 | LR: 7.55e-05 | 881.5ms/step | 22.7 samples/s +Step 33020/44000 | Loss: 2.8949 | LR: 7.54e-05 | 404.1ms/step | 49.5 samples/s +Step 33030/44000 | Loss: 2.9881 | LR: 7.53e-05 | 403.8ms/step | 49.5 samples/s +Step 33040/44000 | Loss: 2.8249 | LR: 7.51e-05 | 404.1ms/step | 49.5 samples/s +Step 33050/44000 | Loss: 2.8919 | LR: 7.50e-05 | 405.3ms/step | 49.4 samples/s +Step 33060/44000 | Loss: 2.9146 | LR: 7.49e-05 | 403.8ms/step | 49.5 samples/s +Step 33070/44000 | Loss: 2.8902 | LR: 7.47e-05 | 404.5ms/step | 49.4 samples/s +Step 33080/44000 | Loss: 2.9676 | LR: 7.46e-05 | 404.4ms/step | 49.5 samples/s +Step 33090/44000 | Loss: 2.9082 | LR: 7.45e-05 | 404.8ms/step | 49.4 samples/s +Step 33100/44000 | Loss: 2.9333 | LR: 7.44e-05 | 404.0ms/step | 49.5 samples/s +Step 33110/44000 | Loss: 2.9094 | LR: 7.42e-05 | 404.6ms/step | 49.4 samples/s +Step 33120/44000 | Loss: 2.9324 | LR: 7.41e-05 | 404.4ms/step | 49.5 samples/s +Step 33130/44000 | Loss: 2.8971 | LR: 7.40e-05 | 403.9ms/step | 49.5 samples/s +Step 33140/44000 | Loss: 2.9225 | LR: 7.38e-05 | 404.2ms/step | 49.5 samples/s +Step 33150/44000 | Loss: 2.9501 | LR: 7.37e-05 | 404.6ms/step | 49.4 samples/s +Step 33160/44000 | Loss: 2.8356 | LR: 7.36e-05 | 404.5ms/step | 49.4 samples/s +Step 33170/44000 | Loss: 2.9750 | LR: 7.34e-05 | 404.3ms/step | 49.5 samples/s +Step 33180/44000 | Loss: 2.8458 | LR: 7.33e-05 | 405.8ms/step | 49.3 samples/s +Step 33190/44000 | Loss: 2.9130 | LR: 7.32e-05 | 403.6ms/step | 49.6 samples/s +Step 33200/44000 | Loss: 2.9617 | LR: 7.31e-05 | 404.2ms/step | 49.5 samples/s +Step 33210/44000 | Loss: 2.9761 | LR: 7.29e-05 | 405.1ms/step | 49.4 samples/s +Step 33220/44000 | Loss: 2.9130 | LR: 7.28e-05 | 404.4ms/step | 49.5 samples/s +Step 33230/44000 | Loss: 2.9658 | LR: 7.27e-05 | 404.3ms/step | 49.5 samples/s +Step 33240/44000 | Loss: 2.9833 | LR: 7.26e-05 | 405.0ms/step | 49.4 samples/s +Step 33250/44000 | Loss: 2.9624 | LR: 7.24e-05 | 404.1ms/step | 49.5 samples/s +Validation at step 33250: Loss = 2.9239, Perplexity = 18.61 +Step 33260/44000 | Loss: 2.8501 | LR: 7.23e-05 | 573.5ms/step | 34.9 samples/s +Step 33270/44000 | Loss: 2.9199 | LR: 7.22e-05 | 404.8ms/step | 49.4 samples/s +Step 33280/44000 | Loss: 2.9166 | LR: 7.20e-05 | 405.2ms/step | 49.4 samples/s +Step 33290/44000 | Loss: 2.9261 | LR: 7.19e-05 | 404.9ms/step | 49.4 samples/s +Step 33300/44000 | Loss: 2.9219 | LR: 7.18e-05 | 404.4ms/step | 49.5 samples/s +Step 33310/44000 | Loss: 2.9117 | LR: 7.17e-05 | 404.5ms/step | 49.4 samples/s +Step 33320/44000 | Loss: 2.9184 | LR: 7.15e-05 | 402.9ms/step | 49.6 samples/s +Step 33330/44000 | Loss: 2.8742 | LR: 7.14e-05 | 404.3ms/step | 49.5 samples/s +Step 33340/44000 | Loss: 2.9170 | LR: 7.13e-05 | 404.3ms/step | 49.5 samples/s +Step 33350/44000 | Loss: 2.8650 | LR: 7.11e-05 | 404.4ms/step | 49.5 samples/s +Step 33360/44000 | Loss: 2.8507 | LR: 7.10e-05 | 404.3ms/step | 49.5 samples/s +Step 33370/44000 | Loss: 2.9302 | LR: 7.09e-05 | 404.3ms/step | 49.5 samples/s +Step 33380/44000 | Loss: 2.8895 | LR: 7.08e-05 | 405.4ms/step | 49.3 samples/s +Step 33390/44000 | Loss: 2.8580 | LR: 7.06e-05 | 403.8ms/step | 49.5 samples/s +Step 33400/44000 | Loss: 2.8689 | LR: 7.05e-05 | 404.2ms/step | 49.5 samples/s +Step 33410/44000 | Loss: 2.9677 | LR: 7.04e-05 | 405.6ms/step | 49.3 samples/s +Step 33420/44000 | Loss: 2.9134 | LR: 7.03e-05 | 405.1ms/step | 49.4 samples/s +Step 33430/44000 | Loss: 2.9097 | LR: 7.01e-05 | 404.2ms/step | 49.5 samples/s +Step 33440/44000 | Loss: 2.9539 | LR: 7.00e-05 | 404.2ms/step | 49.5 samples/s +Step 33450/44000 | Loss: 2.9340 | LR: 6.99e-05 | 403.7ms/step | 49.5 samples/s +Step 33460/44000 | Loss: 2.9429 | LR: 6.98e-05 | 404.5ms/step | 49.4 samples/s +Step 33470/44000 | Loss: 2.8664 | LR: 6.96e-05 | 403.9ms/step | 49.5 samples/s +Step 33480/44000 | Loss: 2.9141 | LR: 6.95e-05 | 404.1ms/step | 49.5 samples/s +Step 33490/44000 | Loss: 2.8860 | LR: 6.94e-05 | 404.8ms/step | 49.4 samples/s +Step 33500/44000 | Loss: 2.9297 | LR: 6.93e-05 | 403.7ms/step | 49.5 samples/s +Validation at step 33500: Loss = 2.9236, Perplexity = 18.61 +Step 33510/44000 | Loss: 2.8976 | LR: 6.91e-05 | 572.1ms/step | 35.0 samples/s +Step 33520/44000 | Loss: 2.9124 | LR: 6.90e-05 | 404.4ms/step | 49.5 samples/s +Step 33530/44000 | Loss: 2.9186 | LR: 6.89e-05 | 404.1ms/step | 49.5 samples/s +Step 33540/44000 | Loss: 2.9392 | LR: 6.88e-05 | 404.3ms/step | 49.5 samples/s +Step 33550/44000 | Loss: 2.9008 | LR: 6.86e-05 | 404.9ms/step | 49.4 samples/s +Step 33560/44000 | Loss: 2.9159 | LR: 6.85e-05 | 405.0ms/step | 49.4 samples/s +Step 33570/44000 | Loss: 2.9419 | LR: 6.84e-05 | 404.6ms/step | 49.4 samples/s +Step 33580/44000 | Loss: 2.8542 | LR: 6.83e-05 | 407.2ms/step | 49.1 samples/s +Step 33590/44000 | Loss: 2.9814 | LR: 6.81e-05 | 405.7ms/step | 49.3 samples/s +Step 33600/44000 | Loss: 2.9253 | LR: 6.80e-05 | 405.5ms/step | 49.3 samples/s +Step 33610/44000 | Loss: 2.9888 | LR: 6.79e-05 | 404.6ms/step | 49.4 samples/s +Step 33620/44000 | Loss: 2.9293 | LR: 6.78e-05 | 404.6ms/step | 49.4 samples/s +Step 33630/44000 | Loss: 2.9470 | LR: 6.76e-05 | 405.7ms/step | 49.3 samples/s +Step 33640/44000 | Loss: 2.8612 | LR: 6.75e-05 | 405.9ms/step | 49.3 samples/s +Step 33650/44000 | Loss: 2.9335 | LR: 6.74e-05 | 405.9ms/step | 49.3 samples/s +Step 33660/44000 | Loss: 2.8583 | LR: 6.73e-05 | 406.0ms/step | 49.3 samples/s +Step 33670/44000 | Loss: 2.9376 | LR: 6.71e-05 | 406.9ms/step | 49.2 samples/s +Step 33680/44000 | Loss: 2.8569 | LR: 6.70e-05 | 405.8ms/step | 49.3 samples/s +Step 33690/44000 | Loss: 2.9224 | LR: 6.69e-05 | 405.5ms/step | 49.3 samples/s +Step 33700/44000 | Loss: 2.8858 | LR: 6.68e-05 | 405.4ms/step | 49.3 samples/s +Step 33710/44000 | Loss: 2.9048 | LR: 6.66e-05 | 406.5ms/step | 49.2 samples/s +Step 33720/44000 | Loss: 2.9515 | LR: 6.65e-05 | 404.6ms/step | 49.4 samples/s +Step 33730/44000 | Loss: 2.8872 | LR: 6.64e-05 | 403.1ms/step | 49.6 samples/s +Step 33740/44000 | Loss: 2.8367 | LR: 6.63e-05 | 403.0ms/step | 49.6 samples/s +Step 33750/44000 | Loss: 2.9229 | LR: 6.61e-05 | 404.4ms/step | 49.5 samples/s +Validation at step 33750: Loss = 2.9229, Perplexity = 18.60 +Step 33760/44000 | Loss: 2.8948 | LR: 6.60e-05 | 571.6ms/step | 35.0 samples/s +Step 33770/44000 | Loss: 2.9199 | LR: 6.59e-05 | 405.1ms/step | 49.4 samples/s +Step 33780/44000 | Loss: 2.9186 | LR: 6.58e-05 | 404.6ms/step | 49.4 samples/s +Step 33790/44000 | Loss: 2.9455 | LR: 6.57e-05 | 405.7ms/step | 49.3 samples/s +Step 33800/44000 | Loss: 2.9194 | LR: 6.55e-05 | 405.6ms/step | 49.3 samples/s +Step 33810/44000 | Loss: 2.8954 | LR: 6.54e-05 | 405.7ms/step | 49.3 samples/s +Step 33820/44000 | Loss: 2.8441 | LR: 6.53e-05 | 405.0ms/step | 49.4 samples/s +Step 33830/44000 | Loss: 2.9000 | LR: 6.52e-05 | 404.7ms/step | 49.4 samples/s +Step 33840/44000 | Loss: 2.8908 | LR: 6.50e-05 | 405.2ms/step | 49.4 samples/s +Step 33850/44000 | Loss: 2.9369 | LR: 6.49e-05 | 404.6ms/step | 49.4 samples/s +Step 33860/44000 | Loss: 2.9035 | LR: 6.48e-05 | 404.7ms/step | 49.4 samples/s +Step 33870/44000 | Loss: 2.9522 | LR: 6.47e-05 | 404.9ms/step | 49.4 samples/s +Step 33880/44000 | Loss: 2.9279 | LR: 6.46e-05 | 404.3ms/step | 49.5 samples/s +Step 33890/44000 | Loss: 2.8953 | LR: 6.44e-05 | 404.2ms/step | 49.5 samples/s +Step 33900/44000 | Loss: 2.9454 | LR: 6.43e-05 | 404.3ms/step | 49.5 samples/s +Step 33910/44000 | Loss: 2.9102 | LR: 6.42e-05 | 404.1ms/step | 49.5 samples/s +Step 33920/44000 | Loss: 2.9504 | LR: 6.41e-05 | 404.2ms/step | 49.5 samples/s +Step 33930/44000 | Loss: 2.9503 | LR: 6.40e-05 | 405.3ms/step | 49.3 samples/s +Step 33940/44000 | Loss: 2.9216 | LR: 6.38e-05 | 404.4ms/step | 49.5 samples/s +Step 33950/44000 | Loss: 2.9116 | LR: 6.37e-05 | 403.9ms/step | 49.5 samples/s +Step 33960/44000 | Loss: 2.8696 | LR: 6.36e-05 | 403.7ms/step | 49.5 samples/s +Step 33970/44000 | Loss: 2.8829 | LR: 6.35e-05 | 404.6ms/step | 49.4 samples/s +Step 33980/44000 | Loss: 2.9318 | LR: 6.33e-05 | 404.0ms/step | 49.5 samples/s +Step 33990/44000 | Loss: 2.8587 | LR: 6.32e-05 | 404.6ms/step | 49.4 samples/s +Step 34000/44000 | Loss: 2.8914 | LR: 6.31e-05 | 404.2ms/step | 49.5 samples/s +Validation at step 34000: Loss = 2.9220, Perplexity = 18.58 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-34000 +Step 34010/44000 | Loss: 2.8918 | LR: 6.30e-05 | 878.5ms/step | 22.8 samples/s +Step 34020/44000 | Loss: 2.8671 | LR: 6.29e-05 | 403.9ms/step | 49.5 samples/s +Step 34030/44000 | Loss: 3.0338 | LR: 6.27e-05 | 403.7ms/step | 49.5 samples/s +Step 34040/44000 | Loss: 2.9178 | LR: 6.26e-05 | 404.3ms/step | 49.5 samples/s +Step 34050/44000 | Loss: 2.9004 | LR: 6.25e-05 | 403.9ms/step | 49.5 samples/s +Step 34060/44000 | Loss: 2.9856 | LR: 6.24e-05 | 403.0ms/step | 49.6 samples/s +Step 34070/44000 | Loss: 2.8938 | LR: 6.23e-05 | 404.6ms/step | 49.4 samples/s +Step 34080/44000 | Loss: 2.9710 | LR: 6.21e-05 | 404.6ms/step | 49.4 samples/s +Step 34090/44000 | Loss: 2.9271 | LR: 6.20e-05 | 405.2ms/step | 49.4 samples/s +Step 34100/44000 | Loss: 2.9823 | LR: 6.19e-05 | 404.5ms/step | 49.4 samples/s +Step 34110/44000 | Loss: 2.9589 | LR: 6.18e-05 | 404.3ms/step | 49.5 samples/s +Step 34120/44000 | Loss: 2.8973 | LR: 6.17e-05 | 404.8ms/step | 49.4 samples/s +Step 34130/44000 | Loss: 2.8848 | LR: 6.15e-05 | 404.1ms/step | 49.5 samples/s +Step 34140/44000 | Loss: 2.9296 | LR: 6.14e-05 | 404.5ms/step | 49.4 samples/s +Step 34150/44000 | Loss: 2.9374 | LR: 6.13e-05 | 405.6ms/step | 49.3 samples/s +Step 34160/44000 | Loss: 2.9789 | LR: 6.12e-05 | 405.5ms/step | 49.3 samples/s +Step 34170/44000 | Loss: 2.9559 | LR: 6.11e-05 | 404.4ms/step | 49.5 samples/s +Step 34180/44000 | Loss: 2.9199 | LR: 6.09e-05 | 404.1ms/step | 49.5 samples/s +Step 34190/44000 | Loss: 2.9977 | LR: 6.08e-05 | 404.8ms/step | 49.4 samples/s +Step 34200/44000 | Loss: 2.9338 | LR: 6.07e-05 | 404.7ms/step | 49.4 samples/s +Step 34210/44000 | Loss: 2.9197 | LR: 6.06e-05 | 405.5ms/step | 49.3 samples/s +Step 34220/44000 | Loss: 2.9287 | LR: 6.05e-05 | 405.3ms/step | 49.3 samples/s +Step 34230/44000 | Loss: 2.8811 | LR: 6.04e-05 | 406.4ms/step | 49.2 samples/s +Step 34240/44000 | Loss: 2.8877 | LR: 6.02e-05 | 404.2ms/step | 49.5 samples/s +Step 34250/44000 | Loss: 2.9512 | LR: 6.01e-05 | 404.5ms/step | 49.4 samples/s +Validation at step 34250: Loss = 2.9228, Perplexity = 18.59 +Step 34260/44000 | Loss: 2.9506 | LR: 6.00e-05 | 573.6ms/step | 34.9 samples/s +Step 34270/44000 | Loss: 2.9191 | LR: 5.99e-05 | 405.4ms/step | 49.3 samples/s +Step 34280/44000 | Loss: 2.9255 | LR: 5.98e-05 | 403.5ms/step | 49.6 samples/s +Step 34290/44000 | Loss: 2.9343 | LR: 5.96e-05 | 403.6ms/step | 49.5 samples/s +Step 34300/44000 | Loss: 2.9413 | LR: 5.95e-05 | 403.4ms/step | 49.6 samples/s +Step 34310/44000 | Loss: 2.9435 | LR: 5.94e-05 | 403.0ms/step | 49.6 samples/s +Step 34320/44000 | Loss: 2.9352 | LR: 5.93e-05 | 404.8ms/step | 49.4 samples/s +Step 34330/44000 | Loss: 2.9587 | LR: 5.92e-05 | 403.9ms/step | 49.5 samples/s +Step 34340/44000 | Loss: 2.9213 | LR: 5.91e-05 | 404.1ms/step | 49.5 samples/s +Step 34350/44000 | Loss: 2.8754 | LR: 5.89e-05 | 404.1ms/step | 49.5 samples/s +Step 34360/44000 | Loss: 2.9176 | LR: 5.88e-05 | 404.4ms/step | 49.5 samples/s +Step 34370/44000 | Loss: 2.9884 | LR: 5.87e-05 | 404.8ms/step | 49.4 samples/s +Step 34380/44000 | Loss: 2.9309 | LR: 5.86e-05 | 404.1ms/step | 49.5 samples/s +Step 34390/44000 | Loss: 2.9683 | LR: 5.85e-05 | 404.8ms/step | 49.4 samples/s +Step 34400/44000 | Loss: 2.8604 | LR: 5.84e-05 | 404.9ms/step | 49.4 samples/s +Step 34410/44000 | Loss: 2.9953 | LR: 5.82e-05 | 404.6ms/step | 49.4 samples/s +Step 34420/44000 | Loss: 2.8964 | LR: 5.81e-05 | 404.8ms/step | 49.4 samples/s +Step 34430/44000 | Loss: 2.9425 | LR: 5.80e-05 | 404.6ms/step | 49.4 samples/s +Step 34440/44000 | Loss: 2.9036 | LR: 5.79e-05 | 404.5ms/step | 49.4 samples/s +Step 34450/44000 | Loss: 2.9197 | LR: 5.78e-05 | 404.7ms/step | 49.4 samples/s +Step 34460/44000 | Loss: 2.9104 | LR: 5.77e-05 | 404.7ms/step | 49.4 samples/s +Step 34470/44000 | Loss: 2.9071 | LR: 5.75e-05 | 404.6ms/step | 49.4 samples/s +Step 34480/44000 | Loss: 2.9056 | LR: 5.74e-05 | 404.2ms/step | 49.5 samples/s +Step 34490/44000 | Loss: 2.8588 | LR: 5.73e-05 | 404.2ms/step | 49.5 samples/s +Step 34500/44000 | Loss: 2.8929 | LR: 5.72e-05 | 404.1ms/step | 49.5 samples/s +Validation at step 34500: Loss = 2.9213, Perplexity = 18.57 +Step 34510/44000 | Loss: 2.8318 | LR: 5.71e-05 | 574.1ms/step | 34.8 samples/s +Step 34520/44000 | Loss: 2.9000 | LR: 5.70e-05 | 403.8ms/step | 49.5 samples/s +Step 34530/44000 | Loss: 2.9463 | LR: 5.69e-05 | 403.8ms/step | 49.5 samples/s +Step 34540/44000 | Loss: 2.8900 | LR: 5.67e-05 | 404.3ms/step | 49.5 samples/s +Step 34550/44000 | Loss: 2.9047 | LR: 5.66e-05 | 404.1ms/step | 49.5 samples/s +Step 34560/44000 | Loss: 2.9040 | LR: 5.65e-05 | 404.3ms/step | 49.5 samples/s +Step 34570/44000 | Loss: 2.9791 | LR: 5.64e-05 | 404.6ms/step | 49.4 samples/s +Step 34580/44000 | Loss: 2.9409 | LR: 5.63e-05 | 404.2ms/step | 49.5 samples/s +Step 34590/44000 | Loss: 2.9645 | LR: 5.62e-05 | 404.2ms/step | 49.5 samples/s +Step 34600/44000 | Loss: 2.9458 | LR: 5.60e-05 | 404.0ms/step | 49.5 samples/s +Step 34610/44000 | Loss: 2.9009 | LR: 5.59e-05 | 404.3ms/step | 49.5 samples/s +Step 34620/44000 | Loss: 2.9623 | LR: 5.58e-05 | 404.4ms/step | 49.5 samples/s +Step 34630/44000 | Loss: 2.9460 | LR: 5.57e-05 | 404.1ms/step | 49.5 samples/s +Step 34640/44000 | Loss: 2.9720 | LR: 5.56e-05 | 404.0ms/step | 49.5 samples/s +Step 34650/44000 | Loss: 2.9618 | LR: 5.55e-05 | 403.9ms/step | 49.5 samples/s +Step 34660/44000 | Loss: 2.9369 | LR: 5.54e-05 | 404.7ms/step | 49.4 samples/s +Step 34670/44000 | Loss: 2.9560 | LR: 5.52e-05 | 404.3ms/step | 49.5 samples/s +Step 34680/44000 | Loss: 2.9404 | LR: 5.51e-05 | 404.3ms/step | 49.5 samples/s +Step 34690/44000 | Loss: 2.9335 | LR: 5.50e-05 | 405.1ms/step | 49.4 samples/s +Step 34700/44000 | Loss: 2.9177 | LR: 5.49e-05 | 404.2ms/step | 49.5 samples/s +Step 34710/44000 | Loss: 2.8626 | LR: 5.48e-05 | 404.4ms/step | 49.5 samples/s +Step 34720/44000 | Loss: 2.9551 | LR: 5.47e-05 | 404.7ms/step | 49.4 samples/s +Step 34730/44000 | Loss: 2.8609 | LR: 5.46e-05 | 405.2ms/step | 49.4 samples/s +Step 34740/44000 | Loss: 2.9534 | LR: 5.45e-05 | 406.4ms/step | 49.2 samples/s +Step 34750/44000 | Loss: 2.9554 | LR: 5.43e-05 | 403.7ms/step | 49.5 samples/s +Validation at step 34750: Loss = 2.9210, Perplexity = 18.56 +Step 34760/44000 | Loss: 2.8926 | LR: 5.42e-05 | 572.2ms/step | 35.0 samples/s +Step 34770/44000 | Loss: 2.9201 | LR: 5.41e-05 | 403.7ms/step | 49.5 samples/s +Step 34780/44000 | Loss: 2.8857 | LR: 5.40e-05 | 403.7ms/step | 49.5 samples/s +Step 34790/44000 | Loss: 2.8995 | LR: 5.39e-05 | 403.6ms/step | 49.6 samples/s +Step 34800/44000 | Loss: 2.9133 | LR: 5.38e-05 | 402.5ms/step | 49.7 samples/s +Step 34810/44000 | Loss: 2.9164 | LR: 5.37e-05 | 403.0ms/step | 49.6 samples/s +Step 34820/44000 | Loss: 2.9319 | LR: 5.36e-05 | 402.3ms/step | 49.7 samples/s +Step 34830/44000 | Loss: 2.9281 | LR: 5.34e-05 | 403.1ms/step | 49.6 samples/s +Step 34840/44000 | Loss: 2.8904 | LR: 5.33e-05 | 403.8ms/step | 49.5 samples/s +Step 34850/44000 | Loss: 2.9286 | LR: 5.32e-05 | 403.5ms/step | 49.6 samples/s +Step 34860/44000 | Loss: 2.9756 | LR: 5.31e-05 | 403.4ms/step | 49.6 samples/s +Step 34870/44000 | Loss: 2.9391 | LR: 5.30e-05 | 403.9ms/step | 49.5 samples/s +Step 34880/44000 | Loss: 2.8511 | LR: 5.29e-05 | 403.9ms/step | 49.5 samples/s +Step 34890/44000 | Loss: 2.9385 | LR: 5.28e-05 | 403.8ms/step | 49.5 samples/s +Step 34900/44000 | Loss: 2.9272 | LR: 5.27e-05 | 403.7ms/step | 49.5 samples/s +Step 34910/44000 | Loss: 2.8750 | LR: 5.25e-05 | 403.6ms/step | 49.6 samples/s +Step 34920/44000 | Loss: 2.8833 | LR: 5.24e-05 | 403.4ms/step | 49.6 samples/s +Step 34930/44000 | Loss: 2.9777 | LR: 5.23e-05 | 403.5ms/step | 49.6 samples/s +Step 34940/44000 | Loss: 2.9276 | LR: 5.22e-05 | 403.0ms/step | 49.6 samples/s +Step 34950/44000 | Loss: 2.9462 | LR: 5.21e-05 | 403.3ms/step | 49.6 samples/s +Step 34960/44000 | Loss: 2.9200 | LR: 5.20e-05 | 404.9ms/step | 49.4 samples/s +Step 34970/44000 | Loss: 2.9612 | LR: 5.19e-05 | 403.5ms/step | 49.6 samples/s +Step 34980/44000 | Loss: 2.9224 | LR: 5.18e-05 | 404.0ms/step | 49.5 samples/s +Step 34990/44000 | Loss: 2.9716 | LR: 5.17e-05 | 403.5ms/step | 49.6 samples/s +Step 35000/44000 | Loss: 2.9252 | LR: 5.15e-05 | 404.0ms/step | 49.5 samples/s +Validation at step 35000: Loss = 2.9208, Perplexity = 18.56 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-35000 +Step 35010/44000 | Loss: 2.8796 | LR: 5.14e-05 | 876.6ms/step | 22.8 samples/s +Step 35020/44000 | Loss: 2.9642 | LR: 5.13e-05 | 404.5ms/step | 49.4 samples/s +Step 35030/44000 | Loss: 2.8988 | LR: 5.12e-05 | 404.1ms/step | 49.5 samples/s +Step 35040/44000 | Loss: 2.9690 | LR: 5.11e-05 | 403.7ms/step | 49.5 samples/s +Step 35050/44000 | Loss: 2.9185 | LR: 5.10e-05 | 404.2ms/step | 49.5 samples/s +Step 35060/44000 | Loss: 2.9740 | LR: 5.09e-05 | 404.4ms/step | 49.5 samples/s +Step 35070/44000 | Loss: 2.9579 | LR: 5.08e-05 | 404.9ms/step | 49.4 samples/s +Step 35080/44000 | Loss: 2.9133 | LR: 5.07e-05 | 403.9ms/step | 49.5 samples/s +Step 35090/44000 | Loss: 2.9255 | LR: 5.06e-05 | 404.0ms/step | 49.5 samples/s +Step 35100/44000 | Loss: 2.9179 | LR: 5.04e-05 | 404.4ms/step | 49.5 samples/s +Step 35110/44000 | Loss: 2.9608 | LR: 5.03e-05 | 404.4ms/step | 49.5 samples/s +Step 35120/44000 | Loss: 2.8915 | LR: 5.02e-05 | 404.5ms/step | 49.4 samples/s +Step 35130/44000 | Loss: 2.9394 | LR: 5.01e-05 | 403.7ms/step | 49.5 samples/s +Step 35140/44000 | Loss: 3.0169 | LR: 5.00e-05 | 404.3ms/step | 49.5 samples/s +Step 35150/44000 | Loss: 2.9113 | LR: 4.99e-05 | 405.0ms/step | 49.4 samples/s +Step 35160/44000 | Loss: 2.9201 | LR: 4.98e-05 | 405.0ms/step | 49.4 samples/s +Step 35170/44000 | Loss: 3.0053 | LR: 4.97e-05 | 404.5ms/step | 49.4 samples/s +Step 35180/44000 | Loss: 2.9358 | LR: 4.96e-05 | 405.1ms/step | 49.4 samples/s +Step 35190/44000 | Loss: 2.9435 | LR: 4.95e-05 | 405.2ms/step | 49.4 samples/s +Step 35200/44000 | Loss: 2.9881 | LR: 4.94e-05 | 404.8ms/step | 49.4 samples/s +Step 35210/44000 | Loss: 2.8928 | LR: 4.93e-05 | 404.5ms/step | 49.4 samples/s +Step 35220/44000 | Loss: 2.8583 | LR: 4.91e-05 | 404.6ms/step | 49.4 samples/s +Step 35230/44000 | Loss: 2.9311 | LR: 4.90e-05 | 404.4ms/step | 49.5 samples/s +Step 35240/44000 | Loss: 2.9271 | LR: 4.89e-05 | 405.3ms/step | 49.3 samples/s +Step 35250/44000 | Loss: 2.9598 | LR: 4.88e-05 | 405.3ms/step | 49.3 samples/s +Validation at step 35250: Loss = 2.9207, Perplexity = 18.55 +Step 35260/44000 | Loss: 2.9300 | LR: 4.87e-05 | 573.6ms/step | 34.9 samples/s +Step 35270/44000 | Loss: 2.9159 | LR: 4.86e-05 | 404.5ms/step | 49.4 samples/s +Step 35280/44000 | Loss: 2.9347 | LR: 4.85e-05 | 405.1ms/step | 49.4 samples/s +Step 35290/44000 | Loss: 2.9447 | LR: 4.84e-05 | 405.1ms/step | 49.4 samples/s +Step 35300/44000 | Loss: 2.8950 | LR: 4.83e-05 | 405.6ms/step | 49.3 samples/s +Step 35310/44000 | Loss: 2.9004 | LR: 4.82e-05 | 404.9ms/step | 49.4 samples/s +Step 35320/44000 | Loss: 2.8927 | LR: 4.81e-05 | 404.0ms/step | 49.5 samples/s +Step 35330/44000 | Loss: 2.9559 | LR: 4.80e-05 | 405.2ms/step | 49.4 samples/s +Step 35340/44000 | Loss: 2.8820 | LR: 4.79e-05 | 404.4ms/step | 49.5 samples/s +Step 35350/44000 | Loss: 2.8548 | LR: 4.77e-05 | 404.3ms/step | 49.5 samples/s +Step 35360/44000 | Loss: 2.9116 | LR: 4.76e-05 | 406.1ms/step | 49.2 samples/s +Step 35370/44000 | Loss: 2.9271 | LR: 4.75e-05 | 406.4ms/step | 49.2 samples/s +Step 35380/44000 | Loss: 2.8765 | LR: 4.74e-05 | 405.5ms/step | 49.3 samples/s +Step 35390/44000 | Loss: 2.9821 | LR: 4.73e-05 | 405.3ms/step | 49.3 samples/s +Step 35400/44000 | Loss: 2.9929 | LR: 4.72e-05 | 403.1ms/step | 49.6 samples/s +Step 35410/44000 | Loss: 2.8748 | LR: 4.71e-05 | 405.3ms/step | 49.3 samples/s +Step 35420/44000 | Loss: 2.9548 | LR: 4.70e-05 | 404.6ms/step | 49.4 samples/s +Step 35430/44000 | Loss: 2.9637 | LR: 4.69e-05 | 405.0ms/step | 49.4 samples/s +Step 35440/44000 | Loss: 2.8979 | LR: 4.68e-05 | 405.0ms/step | 49.4 samples/s +Step 35450/44000 | Loss: 2.9361 | LR: 4.67e-05 | 403.2ms/step | 49.6 samples/s +Step 35460/44000 | Loss: 2.9318 | LR: 4.66e-05 | 403.3ms/step | 49.6 samples/s +Step 35470/44000 | Loss: 2.9339 | LR: 4.65e-05 | 402.9ms/step | 49.6 samples/s +Step 35480/44000 | Loss: 2.9388 | LR: 4.64e-05 | 403.2ms/step | 49.6 samples/s +Step 35490/44000 | Loss: 2.9909 | LR: 4.63e-05 | 402.7ms/step | 49.7 samples/s +Step 35500/44000 | Loss: 2.8590 | LR: 4.62e-05 | 402.8ms/step | 49.7 samples/s +Validation at step 35500: Loss = 2.9211, Perplexity = 18.56 +Step 35510/44000 | Loss: 2.8722 | LR: 4.61e-05 | 570.8ms/step | 35.0 samples/s +Step 35520/44000 | Loss: 2.9463 | LR: 4.59e-05 | 403.6ms/step | 49.6 samples/s +Step 35530/44000 | Loss: 2.9037 | LR: 4.58e-05 | 404.0ms/step | 49.5 samples/s +Step 35540/44000 | Loss: 3.0315 | LR: 4.57e-05 | 404.1ms/step | 49.5 samples/s +Step 35550/44000 | Loss: 2.9360 | LR: 4.56e-05 | 402.9ms/step | 49.6 samples/s +Step 35560/44000 | Loss: 2.9716 | LR: 4.55e-05 | 402.9ms/step | 49.6 samples/s +Step 35570/44000 | Loss: 2.9538 | LR: 4.54e-05 | 402.8ms/step | 49.7 samples/s +Step 35580/44000 | Loss: 2.9324 | LR: 4.53e-05 | 403.7ms/step | 49.5 samples/s +Step 35590/44000 | Loss: 2.9243 | LR: 4.52e-05 | 402.4ms/step | 49.7 samples/s +Step 35600/44000 | Loss: 2.9350 | LR: 4.51e-05 | 404.9ms/step | 49.4 samples/s +Step 35610/44000 | Loss: 2.8803 | LR: 4.50e-05 | 405.4ms/step | 49.3 samples/s +Step 35620/44000 | Loss: 2.8711 | LR: 4.49e-05 | 405.1ms/step | 49.4 samples/s +Step 35630/44000 | Loss: 2.9249 | LR: 4.48e-05 | 404.3ms/step | 49.5 samples/s +Step 35640/44000 | Loss: 2.9415 | LR: 4.47e-05 | 401.0ms/step | 49.9 samples/s +Step 35650/44000 | Loss: 2.9930 | LR: 4.46e-05 | 401.5ms/step | 49.8 samples/s +Step 35660/44000 | Loss: 2.9140 | LR: 4.45e-05 | 401.6ms/step | 49.8 samples/s +Step 35670/44000 | Loss: 2.9292 | LR: 4.44e-05 | 402.0ms/step | 49.8 samples/s +Step 35680/44000 | Loss: 2.9372 | LR: 4.43e-05 | 403.7ms/step | 49.5 samples/s +Step 35690/44000 | Loss: 2.8857 | LR: 4.42e-05 | 404.0ms/step | 49.5 samples/s +Step 35700/44000 | Loss: 2.9125 | LR: 4.41e-05 | 403.8ms/step | 49.5 samples/s +Step 35710/44000 | Loss: 2.9154 | LR: 4.40e-05 | 404.5ms/step | 49.4 samples/s +Step 35720/44000 | Loss: 2.9423 | LR: 4.39e-05 | 404.1ms/step | 49.5 samples/s +Step 35730/44000 | Loss: 2.9019 | LR: 4.38e-05 | 404.0ms/step | 49.5 samples/s +Step 35740/44000 | Loss: 2.9433 | LR: 4.37e-05 | 404.0ms/step | 49.5 samples/s +Step 35750/44000 | Loss: 2.9463 | LR: 4.36e-05 | 403.0ms/step | 49.6 samples/s +Validation at step 35750: Loss = 2.9205, Perplexity = 18.55 +Step 35760/44000 | Loss: 2.9183 | LR: 4.35e-05 | 572.4ms/step | 34.9 samples/s +Step 35770/44000 | Loss: 2.9279 | LR: 4.34e-05 | 402.9ms/step | 49.6 samples/s +Step 35780/44000 | Loss: 2.8892 | LR: 4.33e-05 | 403.6ms/step | 49.6 samples/s +Step 35790/44000 | Loss: 2.9307 | LR: 4.32e-05 | 402.8ms/step | 49.7 samples/s +Step 35800/44000 | Loss: 2.9244 | LR: 4.31e-05 | 403.8ms/step | 49.5 samples/s +Step 35810/44000 | Loss: 2.8817 | LR: 4.29e-05 | 403.5ms/step | 49.6 samples/s +Step 35820/44000 | Loss: 2.9770 | LR: 4.28e-05 | 403.3ms/step | 49.6 samples/s +Step 35830/44000 | Loss: 2.9088 | LR: 4.27e-05 | 404.0ms/step | 49.5 samples/s +Step 35840/44000 | Loss: 2.9306 | LR: 4.26e-05 | 404.3ms/step | 49.5 samples/s +Step 35850/44000 | Loss: 2.8947 | LR: 4.25e-05 | 404.7ms/step | 49.4 samples/s +Step 35860/44000 | Loss: 2.8653 | LR: 4.24e-05 | 404.5ms/step | 49.4 samples/s +Step 35870/44000 | Loss: 2.9241 | LR: 4.23e-05 | 405.3ms/step | 49.3 samples/s +Step 35880/44000 | Loss: 2.9061 | LR: 4.22e-05 | 404.3ms/step | 49.5 samples/s +Step 35890/44000 | Loss: 2.9194 | LR: 4.21e-05 | 404.5ms/step | 49.4 samples/s +Step 35900/44000 | Loss: 2.9048 | LR: 4.20e-05 | 404.7ms/step | 49.4 samples/s +Step 35910/44000 | Loss: 2.9669 | LR: 4.19e-05 | 404.0ms/step | 49.5 samples/s +Step 35920/44000 | Loss: 2.9288 | LR: 4.18e-05 | 404.1ms/step | 49.5 samples/s +Step 35930/44000 | Loss: 2.9223 | LR: 4.17e-05 | 403.9ms/step | 49.5 samples/s +Step 35940/44000 | Loss: 2.8655 | LR: 4.16e-05 | 404.6ms/step | 49.4 samples/s +Step 35950/44000 | Loss: 2.8970 | LR: 4.15e-05 | 404.3ms/step | 49.5 samples/s +Step 35960/44000 | Loss: 2.9233 | LR: 4.14e-05 | 404.0ms/step | 49.5 samples/s +Step 35970/44000 | Loss: 2.9205 | LR: 4.13e-05 | 404.3ms/step | 49.5 samples/s +Step 35980/44000 | Loss: 2.9578 | LR: 4.12e-05 | 403.9ms/step | 49.5 samples/s +Step 35990/44000 | Loss: 2.8582 | LR: 4.11e-05 | 405.0ms/step | 49.4 samples/s +Step 36000/44000 | Loss: 2.9747 | LR: 4.10e-05 | 403.9ms/step | 49.5 samples/s +Validation at step 36000: Loss = 2.9207, Perplexity = 18.55 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-36000 +Step 36010/44000 | Loss: 2.8432 | LR: 4.09e-05 | 873.6ms/step | 22.9 samples/s +Step 36020/44000 | Loss: 2.9878 | LR: 4.08e-05 | 403.8ms/step | 49.5 samples/s +Step 36030/44000 | Loss: 2.9456 | LR: 4.07e-05 | 404.4ms/step | 49.5 samples/s +Step 36040/44000 | Loss: 2.9892 | LR: 4.06e-05 | 404.4ms/step | 49.5 samples/s +Step 36050/44000 | Loss: 2.9367 | LR: 4.05e-05 | 403.7ms/step | 49.5 samples/s +Step 36060/44000 | Loss: 2.9571 | LR: 4.04e-05 | 404.0ms/step | 49.5 samples/s +Step 36070/44000 | Loss: 2.9509 | LR: 4.03e-05 | 404.3ms/step | 49.5 samples/s +Step 36080/44000 | Loss: 2.8739 | LR: 4.02e-05 | 404.3ms/step | 49.5 samples/s +Step 36090/44000 | Loss: 2.8967 | LR: 4.01e-05 | 404.0ms/step | 49.5 samples/s +Step 36100/44000 | Loss: 2.9348 | LR: 4.00e-05 | 403.8ms/step | 49.5 samples/s +Step 36110/44000 | Loss: 2.9331 | LR: 3.99e-05 | 404.4ms/step | 49.5 samples/s +Step 36120/44000 | Loss: 2.9217 | LR: 3.98e-05 | 404.5ms/step | 49.4 samples/s +Step 36130/44000 | Loss: 2.9363 | LR: 3.97e-05 | 403.4ms/step | 49.6 samples/s +Step 36140/44000 | Loss: 2.9193 | LR: 3.97e-05 | 404.5ms/step | 49.4 samples/s +Step 36150/44000 | Loss: 2.8976 | LR: 3.96e-05 | 405.0ms/step | 49.4 samples/s +Step 36160/44000 | Loss: 2.9479 | LR: 3.95e-05 | 404.3ms/step | 49.5 samples/s +Step 36170/44000 | Loss: 2.8715 | LR: 3.94e-05 | 403.5ms/step | 49.6 samples/s +Step 36180/44000 | Loss: 3.0054 | LR: 3.93e-05 | 404.5ms/step | 49.4 samples/s +Step 36190/44000 | Loss: 2.9164 | LR: 3.92e-05 | 404.6ms/step | 49.4 samples/s +Step 36200/44000 | Loss: 2.9607 | LR: 3.91e-05 | 405.0ms/step | 49.4 samples/s +Step 36210/44000 | Loss: 2.9339 | LR: 3.90e-05 | 404.7ms/step | 49.4 samples/s +Step 36220/44000 | Loss: 2.8858 | LR: 3.89e-05 | 404.9ms/step | 49.4 samples/s +Step 36230/44000 | Loss: 2.9240 | LR: 3.88e-05 | 404.4ms/step | 49.5 samples/s +Step 36240/44000 | Loss: 2.9102 | LR: 3.87e-05 | 404.5ms/step | 49.4 samples/s +Step 36250/44000 | Loss: 2.9479 | LR: 3.86e-05 | 404.4ms/step | 49.5 samples/s +Validation at step 36250: Loss = 2.9197, Perplexity = 18.54 +Step 36260/44000 | Loss: 2.8973 | LR: 3.85e-05 | 573.1ms/step | 34.9 samples/s +Step 36270/44000 | Loss: 2.9473 | LR: 3.84e-05 | 405.3ms/step | 49.3 samples/s +Step 36280/44000 | Loss: 2.9150 | LR: 3.83e-05 | 403.5ms/step | 49.6 samples/s +Step 36290/44000 | Loss: 2.8764 | LR: 3.82e-05 | 403.2ms/step | 49.6 samples/s +Step 36300/44000 | Loss: 2.9325 | LR: 3.81e-05 | 402.6ms/step | 49.7 samples/s +Step 36310/44000 | Loss: 2.9505 | LR: 3.80e-05 | 402.6ms/step | 49.7 samples/s +Step 36320/44000 | Loss: 2.9711 | LR: 3.79e-05 | 403.0ms/step | 49.6 samples/s +Step 36330/44000 | Loss: 2.8777 | LR: 3.78e-05 | 402.8ms/step | 49.7 samples/s +Step 36340/44000 | Loss: 2.8937 | LR: 3.77e-05 | 402.2ms/step | 49.7 samples/s +Step 36350/44000 | Loss: 2.9309 | LR: 3.76e-05 | 402.9ms/step | 49.6 samples/s +Step 36360/44000 | Loss: 2.9457 | LR: 3.75e-05 | 401.9ms/step | 49.8 samples/s +Step 36370/44000 | Loss: 2.8089 | LR: 3.74e-05 | 402.5ms/step | 49.7 samples/s +Step 36380/44000 | Loss: 2.9094 | LR: 3.73e-05 | 401.9ms/step | 49.8 samples/s +Step 36390/44000 | Loss: 2.9220 | LR: 3.72e-05 | 401.9ms/step | 49.8 samples/s +Step 36400/44000 | Loss: 2.8797 | LR: 3.71e-05 | 401.6ms/step | 49.8 samples/s +Step 36410/44000 | Loss: 2.8905 | LR: 3.70e-05 | 402.4ms/step | 49.7 samples/s +Step 36420/44000 | Loss: 2.9468 | LR: 3.69e-05 | 401.9ms/step | 49.8 samples/s +Step 36430/44000 | Loss: 2.8952 | LR: 3.69e-05 | 402.7ms/step | 49.7 samples/s +Step 36440/44000 | Loss: 2.8895 | LR: 3.68e-05 | 403.4ms/step | 49.6 samples/s +Step 36450/44000 | Loss: 2.9189 | LR: 3.67e-05 | 403.0ms/step | 49.6 samples/s +Step 36460/44000 | Loss: 2.9535 | LR: 3.66e-05 | 403.3ms/step | 49.6 samples/s +Step 36470/44000 | Loss: 2.9069 | LR: 3.65e-05 | 405.2ms/step | 49.4 samples/s +Step 36480/44000 | Loss: 2.9849 | LR: 3.64e-05 | 404.5ms/step | 49.4 samples/s +Step 36490/44000 | Loss: 2.9174 | LR: 3.63e-05 | 404.1ms/step | 49.5 samples/s +Step 36500/44000 | Loss: 2.9635 | LR: 3.62e-05 | 404.5ms/step | 49.4 samples/s +Validation at step 36500: Loss = 2.9207, Perplexity = 18.55 +Step 36510/44000 | Loss: 2.9305 | LR: 3.61e-05 | 571.9ms/step | 35.0 samples/s +Step 36520/44000 | Loss: 2.9337 | LR: 3.60e-05 | 404.6ms/step | 49.4 samples/s +Step 36530/44000 | Loss: 2.9702 | LR: 3.59e-05 | 403.6ms/step | 49.6 samples/s +Step 36540/44000 | Loss: 2.9532 | LR: 3.58e-05 | 403.5ms/step | 49.6 samples/s +Step 36550/44000 | Loss: 2.9266 | LR: 3.57e-05 | 403.7ms/step | 49.5 samples/s +Step 36560/44000 | Loss: 2.9119 | LR: 3.56e-05 | 403.9ms/step | 49.5 samples/s +Step 36570/44000 | Loss: 2.9540 | LR: 3.55e-05 | 404.8ms/step | 49.4 samples/s +Step 36580/44000 | Loss: 2.9334 | LR: 3.54e-05 | 402.4ms/step | 49.7 samples/s +Step 36590/44000 | Loss: 2.8970 | LR: 3.53e-05 | 402.3ms/step | 49.7 samples/s +Step 36600/44000 | Loss: 2.8615 | LR: 3.53e-05 | 401.7ms/step | 49.8 samples/s +Step 36610/44000 | Loss: 2.9162 | LR: 3.52e-05 | 402.4ms/step | 49.7 samples/s +Step 36620/44000 | Loss: 2.9384 | LR: 3.51e-05 | 403.0ms/step | 49.6 samples/s +Step 36630/44000 | Loss: 2.9232 | LR: 3.50e-05 | 403.8ms/step | 49.5 samples/s +Step 36640/44000 | Loss: 2.8879 | LR: 3.49e-05 | 403.3ms/step | 49.6 samples/s +Step 36650/44000 | Loss: 2.9176 | LR: 3.48e-05 | 404.0ms/step | 49.5 samples/s +Step 36660/44000 | Loss: 2.9079 | LR: 3.47e-05 | 403.1ms/step | 49.6 samples/s +Step 36670/44000 | Loss: 2.8747 | LR: 3.46e-05 | 403.6ms/step | 49.6 samples/s +Step 36680/44000 | Loss: 2.9097 | LR: 3.45e-05 | 402.9ms/step | 49.6 samples/s +Step 36690/44000 | Loss: 2.8801 | LR: 3.44e-05 | 403.0ms/step | 49.6 samples/s +Step 36700/44000 | Loss: 2.9270 | LR: 3.43e-05 | 403.3ms/step | 49.6 samples/s +Step 36710/44000 | Loss: 2.9345 | LR: 3.42e-05 | 402.8ms/step | 49.6 samples/s +Step 36720/44000 | Loss: 2.9092 | LR: 3.41e-05 | 404.1ms/step | 49.5 samples/s +Step 36730/44000 | Loss: 2.9704 | LR: 3.41e-05 | 403.6ms/step | 49.6 samples/s +Step 36740/44000 | Loss: 2.9380 | LR: 3.40e-05 | 403.8ms/step | 49.5 samples/s +Step 36750/44000 | Loss: 2.9735 | LR: 3.39e-05 | 402.8ms/step | 49.6 samples/s +Validation at step 36750: Loss = 2.9195, Perplexity = 18.53 +Step 36760/44000 | Loss: 2.8805 | LR: 3.38e-05 | 568.9ms/step | 35.2 samples/s +Step 36770/44000 | Loss: 2.9498 | LR: 3.37e-05 | 403.3ms/step | 49.6 samples/s +Step 36780/44000 | Loss: 2.8979 | LR: 3.36e-05 | 403.1ms/step | 49.6 samples/s +Step 36790/44000 | Loss: 2.9516 | LR: 3.35e-05 | 403.1ms/step | 49.6 samples/s +Step 36800/44000 | Loss: 2.9257 | LR: 3.34e-05 | 402.9ms/step | 49.6 samples/s +Step 36810/44000 | Loss: 2.9464 | LR: 3.33e-05 | 403.8ms/step | 49.5 samples/s +Step 36820/44000 | Loss: 2.8988 | LR: 3.32e-05 | 403.7ms/step | 49.5 samples/s +Step 36830/44000 | Loss: 2.9065 | LR: 3.31e-05 | 402.7ms/step | 49.7 samples/s +Step 36840/44000 | Loss: 2.9229 | LR: 3.31e-05 | 403.3ms/step | 49.6 samples/s +Step 36850/44000 | Loss: 2.9714 | LR: 3.30e-05 | 402.7ms/step | 49.7 samples/s +Step 36860/44000 | Loss: 2.9191 | LR: 3.29e-05 | 403.7ms/step | 49.5 samples/s +Step 36870/44000 | Loss: 2.8949 | LR: 3.28e-05 | 402.2ms/step | 49.7 samples/s +Step 36880/44000 | Loss: 2.9812 | LR: 3.27e-05 | 403.5ms/step | 49.6 samples/s +Step 36890/44000 | Loss: 2.9435 | LR: 3.26e-05 | 403.6ms/step | 49.5 samples/s +Step 36900/44000 | Loss: 2.8431 | LR: 3.25e-05 | 403.1ms/step | 49.6 samples/s +Step 36910/44000 | Loss: 2.9342 | LR: 3.24e-05 | 404.0ms/step | 49.5 samples/s +Step 36920/44000 | Loss: 2.9532 | LR: 3.23e-05 | 404.1ms/step | 49.5 samples/s +Step 36930/44000 | Loss: 2.9344 | LR: 3.22e-05 | 403.9ms/step | 49.5 samples/s +Step 36940/44000 | Loss: 2.9588 | LR: 3.22e-05 | 403.4ms/step | 49.6 samples/s +Step 36950/44000 | Loss: 2.9268 | LR: 3.21e-05 | 402.6ms/step | 49.7 samples/s +Step 36960/44000 | Loss: 2.9173 | LR: 3.20e-05 | 402.0ms/step | 49.8 samples/s +Step 36970/44000 | Loss: 2.9408 | LR: 3.19e-05 | 402.0ms/step | 49.8 samples/s +Step 36980/44000 | Loss: 2.9139 | LR: 3.18e-05 | 401.5ms/step | 49.8 samples/s +Step 36990/44000 | Loss: 2.8966 | LR: 3.17e-05 | 401.5ms/step | 49.8 samples/s +Step 37000/44000 | Loss: 2.9079 | LR: 3.16e-05 | 400.5ms/step | 49.9 samples/s +Validation at step 37000: Loss = 2.9197, Perplexity = 18.54 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-37000 +Step 37010/44000 | Loss: 2.9166 | LR: 3.15e-05 | 869.4ms/step | 23.0 samples/s +Step 37020/44000 | Loss: 2.9055 | LR: 3.15e-05 | 401.3ms/step | 49.8 samples/s +Step 37030/44000 | Loss: 2.9387 | LR: 3.14e-05 | 402.2ms/step | 49.7 samples/s +Step 37040/44000 | Loss: 2.8198 | LR: 3.13e-05 | 401.9ms/step | 49.8 samples/s +Step 37050/44000 | Loss: 2.9624 | LR: 3.12e-05 | 401.8ms/step | 49.8 samples/s +Step 37060/44000 | Loss: 2.9052 | LR: 3.11e-05 | 401.9ms/step | 49.8 samples/s +Step 37070/44000 | Loss: 2.9857 | LR: 3.10e-05 | 402.0ms/step | 49.7 samples/s +Step 37080/44000 | Loss: 2.9354 | LR: 3.09e-05 | 402.3ms/step | 49.7 samples/s +Step 37090/44000 | Loss: 2.9139 | LR: 3.08e-05 | 402.9ms/step | 49.6 samples/s +Step 37100/44000 | Loss: 2.9167 | LR: 3.07e-05 | 402.0ms/step | 49.8 samples/s +Step 37110/44000 | Loss: 2.9026 | LR: 3.07e-05 | 402.5ms/step | 49.7 samples/s +Step 37120/44000 | Loss: 2.8950 | LR: 3.06e-05 | 402.5ms/step | 49.7 samples/s +Step 37130/44000 | Loss: 2.9766 | LR: 3.05e-05 | 402.9ms/step | 49.6 samples/s +Step 37140/44000 | Loss: 2.8528 | LR: 3.04e-05 | 402.2ms/step | 49.7 samples/s +Step 37150/44000 | Loss: 2.9791 | LR: 3.03e-05 | 403.6ms/step | 49.6 samples/s +Step 37160/44000 | Loss: 2.9403 | LR: 3.02e-05 | 403.1ms/step | 49.6 samples/s +Step 37170/44000 | Loss: 2.9241 | LR: 3.01e-05 | 403.7ms/step | 49.5 samples/s +Step 37180/44000 | Loss: 2.9108 | LR: 3.01e-05 | 403.7ms/step | 49.5 samples/s +Step 37190/44000 | Loss: 3.0470 | LR: 3.00e-05 | 403.3ms/step | 49.6 samples/s +Step 37200/44000 | Loss: 2.9527 | LR: 2.99e-05 | 403.8ms/step | 49.5 samples/s +Step 37210/44000 | Loss: 2.9214 | LR: 2.98e-05 | 403.0ms/step | 49.6 samples/s +Step 37220/44000 | Loss: 2.9353 | LR: 2.97e-05 | 403.4ms/step | 49.6 samples/s +Step 37230/44000 | Loss: 2.8797 | LR: 2.96e-05 | 404.0ms/step | 49.5 samples/s +Step 37240/44000 | Loss: 2.8456 | LR: 2.95e-05 | 402.4ms/step | 49.7 samples/s +Step 37250/44000 | Loss: 2.8898 | LR: 2.95e-05 | 402.8ms/step | 49.6 samples/s +Validation at step 37250: Loss = 2.9201, Perplexity = 18.54 +Step 37260/44000 | Loss: 2.9020 | LR: 2.94e-05 | 571.4ms/step | 35.0 samples/s +Step 37270/44000 | Loss: 2.9439 | LR: 2.93e-05 | 403.4ms/step | 49.6 samples/s +Step 37280/44000 | Loss: 2.9389 | LR: 2.92e-05 | 403.7ms/step | 49.5 samples/s +Step 37290/44000 | Loss: 2.9547 | LR: 2.91e-05 | 402.9ms/step | 49.6 samples/s +Step 37300/44000 | Loss: 2.9387 | LR: 2.90e-05 | 403.4ms/step | 49.6 samples/s +Step 37310/44000 | Loss: 2.9380 | LR: 2.89e-05 | 403.0ms/step | 49.6 samples/s +Step 37320/44000 | Loss: 2.9333 | LR: 2.89e-05 | 403.3ms/step | 49.6 samples/s +Step 37330/44000 | Loss: 2.9154 | LR: 2.88e-05 | 403.4ms/step | 49.6 samples/s +Step 37340/44000 | Loss: 2.8475 | LR: 2.87e-05 | 403.0ms/step | 49.6 samples/s +Step 37350/44000 | Loss: 2.8770 | LR: 2.86e-05 | 402.3ms/step | 49.7 samples/s +Step 37360/44000 | Loss: 2.9327 | LR: 2.85e-05 | 401.9ms/step | 49.8 samples/s +Step 37370/44000 | Loss: 2.8929 | LR: 2.84e-05 | 402.7ms/step | 49.7 samples/s +Step 37380/44000 | Loss: 2.9118 | LR: 2.84e-05 | 401.7ms/step | 49.8 samples/s +Step 37390/44000 | Loss: 2.9201 | LR: 2.83e-05 | 403.0ms/step | 49.6 samples/s +Step 37400/44000 | Loss: 2.9405 | LR: 2.82e-05 | 402.3ms/step | 49.7 samples/s +Step 37410/44000 | Loss: 2.9427 | LR: 2.81e-05 | 402.1ms/step | 49.7 samples/s +Step 37420/44000 | Loss: 2.8528 | LR: 2.80e-05 | 403.1ms/step | 49.6 samples/s +Step 37430/44000 | Loss: 2.9590 | LR: 2.79e-05 | 402.5ms/step | 49.7 samples/s +Step 37440/44000 | Loss: 2.9459 | LR: 2.78e-05 | 402.0ms/step | 49.7 samples/s +Step 37450/44000 | Loss: 2.9404 | LR: 2.78e-05 | 402.8ms/step | 49.7 samples/s +Step 37460/44000 | Loss: 2.9045 | LR: 2.77e-05 | 402.4ms/step | 49.7 samples/s +Step 37470/44000 | Loss: 2.8466 | LR: 2.76e-05 | 402.3ms/step | 49.7 samples/s +Step 37480/44000 | Loss: 2.9913 | LR: 2.75e-05 | 402.9ms/step | 49.6 samples/s +Step 37490/44000 | Loss: 2.9766 | LR: 2.74e-05 | 402.5ms/step | 49.7 samples/s +Step 37500/44000 | Loss: 2.9791 | LR: 2.74e-05 | 402.6ms/step | 49.7 samples/s +Validation at step 37500: Loss = 2.9199, Perplexity = 18.54 +Step 37510/44000 | Loss: 2.9306 | LR: 2.73e-05 | 570.7ms/step | 35.0 samples/s +Step 37520/44000 | Loss: 2.9694 | LR: 2.72e-05 | 402.8ms/step | 49.7 samples/s +Step 37530/44000 | Loss: 3.0211 | LR: 2.71e-05 | 402.6ms/step | 49.7 samples/s +Step 37540/44000 | Loss: 2.8955 | LR: 2.70e-05 | 403.2ms/step | 49.6 samples/s +Step 37550/44000 | Loss: 2.9217 | LR: 2.69e-05 | 403.0ms/step | 49.6 samples/s +Step 37560/44000 | Loss: 2.9132 | LR: 2.69e-05 | 402.6ms/step | 49.7 samples/s +Step 37570/44000 | Loss: 2.9213 | LR: 2.68e-05 | 402.6ms/step | 49.7 samples/s +Step 37580/44000 | Loss: 2.9062 | LR: 2.67e-05 | 403.6ms/step | 49.6 samples/s +Step 37590/44000 | Loss: 2.9821 | LR: 2.66e-05 | 403.0ms/step | 49.6 samples/s +Step 37600/44000 | Loss: 2.9271 | LR: 2.65e-05 | 404.6ms/step | 49.4 samples/s +Step 37610/44000 | Loss: 2.8891 | LR: 2.65e-05 | 403.6ms/step | 49.6 samples/s +Step 37620/44000 | Loss: 2.9052 | LR: 2.64e-05 | 403.2ms/step | 49.6 samples/s +Step 37630/44000 | Loss: 2.8987 | LR: 2.63e-05 | 403.3ms/step | 49.6 samples/s +Step 37640/44000 | Loss: 2.8854 | LR: 2.62e-05 | 402.9ms/step | 49.6 samples/s +Step 37650/44000 | Loss: 2.9105 | LR: 2.61e-05 | 403.5ms/step | 49.6 samples/s +Step 37660/44000 | Loss: 2.8790 | LR: 2.60e-05 | 403.1ms/step | 49.6 samples/s +Step 37670/44000 | Loss: 2.9079 | LR: 2.60e-05 | 404.6ms/step | 49.4 samples/s +Step 37680/44000 | Loss: 2.9272 | LR: 2.59e-05 | 403.8ms/step | 49.5 samples/s +Step 37690/44000 | Loss: 2.9036 | LR: 2.58e-05 | 402.8ms/step | 49.7 samples/s +Step 37700/44000 | Loss: 2.8955 | LR: 2.57e-05 | 403.0ms/step | 49.6 samples/s +Step 37710/44000 | Loss: 2.9033 | LR: 2.56e-05 | 403.3ms/step | 49.6 samples/s +Step 37720/44000 | Loss: 2.9007 | LR: 2.56e-05 | 403.6ms/step | 49.6 samples/s +Step 37730/44000 | Loss: 2.8773 | LR: 2.55e-05 | 404.6ms/step | 49.4 samples/s +Step 37740/44000 | Loss: 2.9163 | LR: 2.54e-05 | 403.7ms/step | 49.5 samples/s +Step 37750/44000 | Loss: 2.9426 | LR: 2.53e-05 | 404.7ms/step | 49.4 samples/s +Validation at step 37750: Loss = 2.9198, Perplexity = 18.54 +Step 37760/44000 | Loss: 2.9319 | LR: 2.52e-05 | 576.0ms/step | 34.7 samples/s +Step 37770/44000 | Loss: 2.9355 | LR: 2.52e-05 | 404.3ms/step | 49.5 samples/s +Step 37780/44000 | Loss: 2.9332 | LR: 2.51e-05 | 404.5ms/step | 49.4 samples/s +Step 37790/44000 | Loss: 2.9580 | LR: 2.50e-05 | 403.3ms/step | 49.6 samples/s +Step 37800/44000 | Loss: 2.9260 | LR: 2.49e-05 | 402.7ms/step | 49.7 samples/s +Step 37810/44000 | Loss: 2.9604 | LR: 2.48e-05 | 403.8ms/step | 49.5 samples/s +Step 37820/44000 | Loss: 2.9328 | LR: 2.48e-05 | 403.4ms/step | 49.6 samples/s +Step 37830/44000 | Loss: 2.8551 | LR: 2.47e-05 | 403.1ms/step | 49.6 samples/s +Step 37840/44000 | Loss: 2.8983 | LR: 2.46e-05 | 403.5ms/step | 49.6 samples/s +Step 37850/44000 | Loss: 2.9198 | LR: 2.45e-05 | 402.4ms/step | 49.7 samples/s +Step 37860/44000 | Loss: 2.9790 | LR: 2.45e-05 | 403.5ms/step | 49.6 samples/s +Step 37870/44000 | Loss: 2.9447 | LR: 2.44e-05 | 402.9ms/step | 49.6 samples/s +Step 37880/44000 | Loss: 2.9093 | LR: 2.43e-05 | 401.5ms/step | 49.8 samples/s +Step 37890/44000 | Loss: 2.9437 | LR: 2.42e-05 | 402.5ms/step | 49.7 samples/s +Step 37900/44000 | Loss: 2.9437 | LR: 2.41e-05 | 401.6ms/step | 49.8 samples/s +Step 37910/44000 | Loss: 2.9059 | LR: 2.41e-05 | 401.5ms/step | 49.8 samples/s +Step 37920/44000 | Loss: 2.9116 | LR: 2.40e-05 | 401.3ms/step | 49.8 samples/s +Step 37930/44000 | Loss: 2.8935 | LR: 2.39e-05 | 402.5ms/step | 49.7 samples/s +Step 37940/44000 | Loss: 2.9572 | LR: 2.38e-05 | 404.1ms/step | 49.5 samples/s +Step 37950/44000 | Loss: 2.8708 | LR: 2.38e-05 | 404.0ms/step | 49.5 samples/s +Step 37960/44000 | Loss: 2.9597 | LR: 2.37e-05 | 403.2ms/step | 49.6 samples/s +Step 37970/44000 | Loss: 2.9292 | LR: 2.36e-05 | 403.4ms/step | 49.6 samples/s +Step 37980/44000 | Loss: 2.8679 | LR: 2.35e-05 | 404.2ms/step | 49.5 samples/s +Step 37990/44000 | Loss: 2.9062 | LR: 2.34e-05 | 404.1ms/step | 49.5 samples/s +Step 38000/44000 | Loss: 2.9547 | LR: 2.34e-05 | 405.2ms/step | 49.4 samples/s +Validation at step 38000: Loss = 2.9201, Perplexity = 18.54 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-38000 +Step 38010/44000 | Loss: 2.8969 | LR: 2.33e-05 | 874.2ms/step | 22.9 samples/s +Step 38020/44000 | Loss: 2.8942 | LR: 2.32e-05 | 402.6ms/step | 49.7 samples/s +Step 38030/44000 | Loss: 2.9084 | LR: 2.31e-05 | 403.2ms/step | 49.6 samples/s +Step 38040/44000 | Loss: 2.9261 | LR: 2.31e-05 | 402.6ms/step | 49.7 samples/s +Step 38050/44000 | Loss: 2.8736 | LR: 2.30e-05 | 402.8ms/step | 49.6 samples/s +Step 38060/44000 | Loss: 2.9202 | LR: 2.29e-05 | 402.6ms/step | 49.7 samples/s +Step 38070/44000 | Loss: 2.9098 | LR: 2.28e-05 | 402.6ms/step | 49.7 samples/s +Step 38080/44000 | Loss: 2.8602 | LR: 2.28e-05 | 403.6ms/step | 49.6 samples/s +Step 38090/44000 | Loss: 2.9463 | LR: 2.27e-05 | 404.2ms/step | 49.5 samples/s +Step 38100/44000 | Loss: 2.8938 | LR: 2.26e-05 | 402.7ms/step | 49.7 samples/s +Step 38110/44000 | Loss: 2.9467 | LR: 2.25e-05 | 404.2ms/step | 49.5 samples/s +Step 38120/44000 | Loss: 2.9689 | LR: 2.25e-05 | 403.5ms/step | 49.6 samples/s +Step 38130/44000 | Loss: 2.9303 | LR: 2.24e-05 | 404.7ms/step | 49.4 samples/s +Step 38140/44000 | Loss: 2.8977 | LR: 2.23e-05 | 403.5ms/step | 49.6 samples/s +Step 38150/44000 | Loss: 2.9648 | LR: 2.22e-05 | 404.3ms/step | 49.5 samples/s +Step 38160/44000 | Loss: 2.9293 | LR: 2.22e-05 | 403.4ms/step | 49.6 samples/s +Step 38170/44000 | Loss: 2.9199 | LR: 2.21e-05 | 403.4ms/step | 49.6 samples/s +Step 38180/44000 | Loss: 2.8846 | LR: 2.20e-05 | 404.4ms/step | 49.5 samples/s +Step 38190/44000 | Loss: 2.8758 | LR: 2.19e-05 | 403.9ms/step | 49.5 samples/s +Step 38200/44000 | Loss: 2.9112 | LR: 2.19e-05 | 403.0ms/step | 49.6 samples/s +Step 38210/44000 | Loss: 2.9111 | LR: 2.18e-05 | 403.1ms/step | 49.6 samples/s +Step 38220/44000 | Loss: 2.9422 | LR: 2.17e-05 | 403.0ms/step | 49.6 samples/s +Step 38230/44000 | Loss: 3.0037 | LR: 2.16e-05 | 402.9ms/step | 49.6 samples/s +Step 38240/44000 | Loss: 2.8357 | LR: 2.16e-05 | 402.6ms/step | 49.7 samples/s +Step 38250/44000 | Loss: 2.9040 | LR: 2.15e-05 | 403.3ms/step | 49.6 samples/s +Validation at step 38250: Loss = 2.9190, Perplexity = 18.52 +Step 38260/44000 | Loss: 2.9101 | LR: 2.14e-05 | 571.5ms/step | 35.0 samples/s +Step 38270/44000 | Loss: 2.9160 | LR: 2.13e-05 | 402.5ms/step | 49.7 samples/s +Step 38280/44000 | Loss: 2.9525 | LR: 2.13e-05 | 402.4ms/step | 49.7 samples/s +Step 38290/44000 | Loss: 2.9086 | LR: 2.12e-05 | 402.5ms/step | 49.7 samples/s +Step 38300/44000 | Loss: 2.8884 | LR: 2.11e-05 | 402.9ms/step | 49.6 samples/s +Step 38310/44000 | Loss: 2.9189 | LR: 2.11e-05 | 403.9ms/step | 49.5 samples/s +Step 38320/44000 | Loss: 2.8889 | LR: 2.10e-05 | 403.3ms/step | 49.6 samples/s +Step 38330/44000 | Loss: 2.9326 | LR: 2.09e-05 | 403.4ms/step | 49.6 samples/s +Step 38340/44000 | Loss: 2.9717 | LR: 2.08e-05 | 402.8ms/step | 49.7 samples/s +Step 38350/44000 | Loss: 2.9531 | LR: 2.08e-05 | 402.4ms/step | 49.7 samples/s +Step 38360/44000 | Loss: 2.8954 | LR: 2.07e-05 | 402.3ms/step | 49.7 samples/s +Step 38370/44000 | Loss: 2.9153 | LR: 2.06e-05 | 402.6ms/step | 49.7 samples/s +Step 38380/44000 | Loss: 2.9253 | LR: 2.05e-05 | 403.0ms/step | 49.6 samples/s +Step 38390/44000 | Loss: 2.9370 | LR: 2.05e-05 | 402.1ms/step | 49.7 samples/s +Step 38400/44000 | Loss: 2.9303 | LR: 2.04e-05 | 402.3ms/step | 49.7 samples/s +Step 38410/44000 | Loss: 2.9488 | LR: 2.03e-05 | 402.5ms/step | 49.7 samples/s +Step 38420/44000 | Loss: 2.8835 | LR: 2.03e-05 | 402.1ms/step | 49.7 samples/s +Step 38430/44000 | Loss: 2.9325 | LR: 2.02e-05 | 403.6ms/step | 49.6 samples/s +Step 38440/44000 | Loss: 2.8514 | LR: 2.01e-05 | 403.3ms/step | 49.6 samples/s +Step 38450/44000 | Loss: 2.9080 | LR: 2.00e-05 | 402.9ms/step | 49.6 samples/s +Step 38460/44000 | Loss: 2.9266 | LR: 2.00e-05 | 403.3ms/step | 49.6 samples/s +Step 38470/44000 | Loss: 2.9305 | LR: 1.99e-05 | 402.6ms/step | 49.7 samples/s +Step 38480/44000 | Loss: 2.9480 | LR: 1.98e-05 | 403.9ms/step | 49.5 samples/s +Step 38490/44000 | Loss: 2.9187 | LR: 1.98e-05 | 403.2ms/step | 49.6 samples/s +Step 38500/44000 | Loss: 2.8232 | LR: 1.97e-05 | 402.3ms/step | 49.7 samples/s +Validation at step 38500: Loss = 2.9190, Perplexity = 18.52 +Step 38510/44000 | Loss: 2.8973 | LR: 1.96e-05 | 572.0ms/step | 35.0 samples/s +Step 38520/44000 | Loss: 2.9048 | LR: 1.95e-05 | 399.9ms/step | 50.0 samples/s +Step 38530/44000 | Loss: 2.9708 | LR: 1.95e-05 | 401.1ms/step | 49.9 samples/s +Step 38540/44000 | Loss: 2.8195 | LR: 1.94e-05 | 400.8ms/step | 49.9 samples/s +Step 38550/44000 | Loss: 2.8795 | LR: 1.93e-05 | 400.6ms/step | 49.9 samples/s +Step 38560/44000 | Loss: 2.8954 | LR: 1.93e-05 | 400.0ms/step | 50.0 samples/s +Step 38570/44000 | Loss: 2.9528 | LR: 1.92e-05 | 401.2ms/step | 49.9 samples/s +Step 38580/44000 | Loss: 2.8878 | LR: 1.91e-05 | 400.5ms/step | 49.9 samples/s +Step 38590/44000 | Loss: 2.9237 | LR: 1.91e-05 | 400.2ms/step | 50.0 samples/s +Step 38600/44000 | Loss: 2.9087 | LR: 1.90e-05 | 401.9ms/step | 49.8 samples/s +Step 38610/44000 | Loss: 2.8594 | LR: 1.89e-05 | 402.5ms/step | 49.7 samples/s +Step 38620/44000 | Loss: 2.9332 | LR: 1.88e-05 | 402.3ms/step | 49.7 samples/s +Step 38630/44000 | Loss: 2.8973 | LR: 1.88e-05 | 403.7ms/step | 49.5 samples/s +Step 38640/44000 | Loss: 2.9350 | LR: 1.87e-05 | 403.0ms/step | 49.6 samples/s +Step 38650/44000 | Loss: 2.9410 | LR: 1.86e-05 | 402.8ms/step | 49.7 samples/s +Step 38660/44000 | Loss: 2.9410 | LR: 1.86e-05 | 403.4ms/step | 49.6 samples/s +Step 38670/44000 | Loss: 2.9399 | LR: 1.85e-05 | 402.3ms/step | 49.7 samples/s +Step 38680/44000 | Loss: 2.9265 | LR: 1.84e-05 | 404.1ms/step | 49.5 samples/s +Step 38690/44000 | Loss: 2.9280 | LR: 1.84e-05 | 403.6ms/step | 49.6 samples/s +Step 38700/44000 | Loss: 2.8978 | LR: 1.83e-05 | 403.5ms/step | 49.6 samples/s +Step 38710/44000 | Loss: 2.9111 | LR: 1.82e-05 | 403.8ms/step | 49.5 samples/s +Step 38720/44000 | Loss: 2.8664 | LR: 1.82e-05 | 403.4ms/step | 49.6 samples/s +Step 38730/44000 | Loss: 2.9279 | LR: 1.81e-05 | 403.4ms/step | 49.6 samples/s +Step 38740/44000 | Loss: 2.9114 | LR: 1.80e-05 | 403.1ms/step | 49.6 samples/s +Step 38750/44000 | Loss: 2.8928 | LR: 1.80e-05 | 403.4ms/step | 49.6 samples/s +Validation at step 38750: Loss = 2.9199, Perplexity = 18.54 +Step 38760/44000 | Loss: 2.9604 | LR: 1.79e-05 | 570.5ms/step | 35.1 samples/s +Step 38770/44000 | Loss: 2.9228 | LR: 1.78e-05 | 403.1ms/step | 49.6 samples/s +Step 38780/44000 | Loss: 2.9873 | LR: 1.78e-05 | 402.3ms/step | 49.7 samples/s +Step 38790/44000 | Loss: 2.9568 | LR: 1.77e-05 | 402.9ms/step | 49.6 samples/s +Step 38800/44000 | Loss: 2.9582 | LR: 1.76e-05 | 402.2ms/step | 49.7 samples/s +Step 38810/44000 | Loss: 2.9316 | LR: 1.76e-05 | 402.5ms/step | 49.7 samples/s +Step 38820/44000 | Loss: 2.9071 | LR: 1.75e-05 | 401.3ms/step | 49.8 samples/s +Step 38830/44000 | Loss: 2.8862 | LR: 1.74e-05 | 401.9ms/step | 49.8 samples/s +Step 38840/44000 | Loss: 2.9335 | LR: 1.74e-05 | 402.5ms/step | 49.7 samples/s +Step 38850/44000 | Loss: 2.9047 | LR: 1.73e-05 | 402.6ms/step | 49.7 samples/s +Step 38860/44000 | Loss: 2.9573 | LR: 1.72e-05 | 402.0ms/step | 49.8 samples/s +Step 38870/44000 | Loss: 2.9077 | LR: 1.72e-05 | 402.4ms/step | 49.7 samples/s +Step 38880/44000 | Loss: 2.9190 | LR: 1.71e-05 | 403.5ms/step | 49.6 samples/s +Step 38890/44000 | Loss: 2.9573 | LR: 1.70e-05 | 402.6ms/step | 49.7 samples/s +Step 38900/44000 | Loss: 2.9321 | LR: 1.70e-05 | 403.3ms/step | 49.6 samples/s +Step 38910/44000 | Loss: 2.8990 | LR: 1.69e-05 | 403.3ms/step | 49.6 samples/s +Step 38920/44000 | Loss: 2.9191 | LR: 1.68e-05 | 403.3ms/step | 49.6 samples/s +Step 38930/44000 | Loss: 2.9263 | LR: 1.68e-05 | 402.9ms/step | 49.6 samples/s +Step 38940/44000 | Loss: 2.8941 | LR: 1.67e-05 | 402.2ms/step | 49.7 samples/s +Step 38950/44000 | Loss: 2.9474 | LR: 1.66e-05 | 402.1ms/step | 49.7 samples/s +Step 38960/44000 | Loss: 2.9723 | LR: 1.66e-05 | 402.4ms/step | 49.7 samples/s +Step 38970/44000 | Loss: 2.8852 | LR: 1.65e-05 | 402.2ms/step | 49.7 samples/s +Step 38980/44000 | Loss: 2.8998 | LR: 1.64e-05 | 402.4ms/step | 49.7 samples/s +Step 38990/44000 | Loss: 2.9591 | LR: 1.64e-05 | 402.5ms/step | 49.7 samples/s +Step 39000/44000 | Loss: 2.9413 | LR: 1.63e-05 | 402.6ms/step | 49.7 samples/s +Validation at step 39000: Loss = 2.9195, Perplexity = 18.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-39000 +Step 39010/44000 | Loss: 2.8622 | LR: 1.62e-05 | 866.7ms/step | 23.1 samples/s +Step 39020/44000 | Loss: 2.9153 | LR: 1.62e-05 | 403.1ms/step | 49.6 samples/s +Step 39030/44000 | Loss: 2.9729 | LR: 1.61e-05 | 403.1ms/step | 49.6 samples/s +Step 39040/44000 | Loss: 2.8891 | LR: 1.61e-05 | 403.0ms/step | 49.6 samples/s +Step 39050/44000 | Loss: 2.9266 | LR: 1.60e-05 | 403.4ms/step | 49.6 samples/s +Step 39060/44000 | Loss: 2.9332 | LR: 1.59e-05 | 402.9ms/step | 49.6 samples/s +Step 39070/44000 | Loss: 2.9655 | LR: 1.59e-05 | 402.8ms/step | 49.7 samples/s +Step 39080/44000 | Loss: 2.9364 | LR: 1.58e-05 | 401.0ms/step | 49.9 samples/s +Step 39090/44000 | Loss: 2.8839 | LR: 1.57e-05 | 402.1ms/step | 49.7 samples/s +Step 39100/44000 | Loss: 2.9079 | LR: 1.57e-05 | 402.2ms/step | 49.7 samples/s +Step 39110/44000 | Loss: 2.9333 | LR: 1.56e-05 | 402.7ms/step | 49.7 samples/s +Step 39120/44000 | Loss: 2.9577 | LR: 1.55e-05 | 402.9ms/step | 49.6 samples/s +Step 39130/44000 | Loss: 2.9374 | LR: 1.55e-05 | 402.4ms/step | 49.7 samples/s +Step 39140/44000 | Loss: 2.8508 | LR: 1.54e-05 | 403.6ms/step | 49.6 samples/s +Step 39150/44000 | Loss: 2.9807 | LR: 1.54e-05 | 402.5ms/step | 49.7 samples/s +Step 39160/44000 | Loss: 2.9521 | LR: 1.53e-05 | 403.4ms/step | 49.6 samples/s +Step 39170/44000 | Loss: 2.9603 | LR: 1.52e-05 | 402.1ms/step | 49.7 samples/s +Step 39180/44000 | Loss: 2.9227 | LR: 1.52e-05 | 403.1ms/step | 49.6 samples/s +Step 39190/44000 | Loss: 2.9250 | LR: 1.51e-05 | 403.2ms/step | 49.6 samples/s +Step 39200/44000 | Loss: 2.8592 | LR: 1.50e-05 | 403.4ms/step | 49.6 samples/s +Step 39210/44000 | Loss: 2.8916 | LR: 1.50e-05 | 404.9ms/step | 49.4 samples/s +Step 39220/44000 | Loss: 2.9463 | LR: 1.49e-05 | 403.5ms/step | 49.6 samples/s +Step 39230/44000 | Loss: 2.9157 | LR: 1.49e-05 | 404.9ms/step | 49.4 samples/s +Step 39240/44000 | Loss: 2.9323 | LR: 1.48e-05 | 404.6ms/step | 49.4 samples/s +Step 39250/44000 | Loss: 2.9245 | LR: 1.47e-05 | 403.9ms/step | 49.5 samples/s +Validation at step 39250: Loss = 2.9179, Perplexity = 18.50 +Step 39260/44000 | Loss: 2.9095 | LR: 1.47e-05 | 571.0ms/step | 35.0 samples/s +Step 39270/44000 | Loss: 2.9102 | LR: 1.46e-05 | 403.7ms/step | 49.5 samples/s +Step 39280/44000 | Loss: 2.8427 | LR: 1.46e-05 | 403.6ms/step | 49.6 samples/s +Step 39290/44000 | Loss: 2.9031 | LR: 1.45e-05 | 403.6ms/step | 49.5 samples/s +Step 39300/44000 | Loss: 2.9382 | LR: 1.44e-05 | 403.0ms/step | 49.6 samples/s +Step 39310/44000 | Loss: 2.9316 | LR: 1.44e-05 | 403.1ms/step | 49.6 samples/s +Step 39320/44000 | Loss: 2.9464 | LR: 1.43e-05 | 403.4ms/step | 49.6 samples/s +Step 39330/44000 | Loss: 2.9029 | LR: 1.42e-05 | 403.2ms/step | 49.6 samples/s +Step 39340/44000 | Loss: 2.9259 | LR: 1.42e-05 | 402.3ms/step | 49.7 samples/s +Step 39350/44000 | Loss: 2.9191 | LR: 1.41e-05 | 403.2ms/step | 49.6 samples/s +Step 39360/44000 | Loss: 2.8968 | LR: 1.41e-05 | 403.0ms/step | 49.6 samples/s +Step 39370/44000 | Loss: 2.9004 | LR: 1.40e-05 | 402.9ms/step | 49.6 samples/s +Step 39380/44000 | Loss: 2.9466 | LR: 1.39e-05 | 402.0ms/step | 49.7 samples/s +Step 39390/44000 | Loss: 2.9362 | LR: 1.39e-05 | 402.4ms/step | 49.7 samples/s +Step 39400/44000 | Loss: 2.9111 | LR: 1.38e-05 | 403.1ms/step | 49.6 samples/s +Step 39410/44000 | Loss: 2.9041 | LR: 1.38e-05 | 402.8ms/step | 49.7 samples/s +Step 39420/44000 | Loss: 2.8963 | LR: 1.37e-05 | 404.0ms/step | 49.5 samples/s +Step 39430/44000 | Loss: 2.9469 | LR: 1.36e-05 | 403.4ms/step | 49.6 samples/s +Step 39440/44000 | Loss: 2.9287 | LR: 1.36e-05 | 403.6ms/step | 49.5 samples/s +Step 39450/44000 | Loss: 2.9761 | LR: 1.35e-05 | 403.0ms/step | 49.6 samples/s +Step 39460/44000 | Loss: 2.9170 | LR: 1.35e-05 | 404.0ms/step | 49.5 samples/s +Step 39470/44000 | Loss: 2.9269 | LR: 1.34e-05 | 402.8ms/step | 49.7 samples/s +Step 39480/44000 | Loss: 2.9073 | LR: 1.34e-05 | 404.0ms/step | 49.5 samples/s +Step 39490/44000 | Loss: 2.9117 | LR: 1.33e-05 | 402.9ms/step | 49.6 samples/s +Step 39500/44000 | Loss: 2.8666 | LR: 1.32e-05 | 402.8ms/step | 49.7 samples/s +Validation at step 39500: Loss = 2.9194, Perplexity = 18.53 +Step 39510/44000 | Loss: 2.9043 | LR: 1.32e-05 | 572.1ms/step | 35.0 samples/s +Step 39520/44000 | Loss: 2.8743 | LR: 1.31e-05 | 403.0ms/step | 49.6 samples/s +Step 39530/44000 | Loss: 2.9738 | LR: 1.31e-05 | 404.2ms/step | 49.5 samples/s +Step 39540/44000 | Loss: 2.9238 | LR: 1.30e-05 | 405.0ms/step | 49.4 samples/s +Step 39550/44000 | Loss: 2.8893 | LR: 1.29e-05 | 406.2ms/step | 49.2 samples/s +Step 39560/44000 | Loss: 2.8828 | LR: 1.29e-05 | 405.6ms/step | 49.3 samples/s +Step 39570/44000 | Loss: 2.9124 | LR: 1.28e-05 | 405.1ms/step | 49.4 samples/s +Step 39580/44000 | Loss: 2.9725 | LR: 1.28e-05 | 403.4ms/step | 49.6 samples/s +Step 39590/44000 | Loss: 2.8910 | LR: 1.27e-05 | 403.3ms/step | 49.6 samples/s +Step 39600/44000 | Loss: 2.9443 | LR: 1.27e-05 | 403.8ms/step | 49.5 samples/s +Step 39610/44000 | Loss: 2.8820 | LR: 1.26e-05 | 403.0ms/step | 49.6 samples/s +Step 39620/44000 | Loss: 2.9097 | LR: 1.25e-05 | 403.8ms/step | 49.5 samples/s +Step 39630/44000 | Loss: 2.8801 | LR: 1.25e-05 | 403.4ms/step | 49.6 samples/s +Step 39640/44000 | Loss: 2.9273 | LR: 1.24e-05 | 402.6ms/step | 49.7 samples/s +Step 39650/44000 | Loss: 2.9084 | LR: 1.24e-05 | 404.2ms/step | 49.5 samples/s +Step 39660/44000 | Loss: 2.9606 | LR: 1.23e-05 | 402.5ms/step | 49.7 samples/s +Step 39670/44000 | Loss: 2.8459 | LR: 1.23e-05 | 403.5ms/step | 49.6 samples/s +Step 39680/44000 | Loss: 2.9216 | LR: 1.22e-05 | 403.9ms/step | 49.5 samples/s +Step 39690/44000 | Loss: 2.9871 | LR: 1.22e-05 | 403.9ms/step | 49.5 samples/s +Step 39700/44000 | Loss: 2.9636 | LR: 1.21e-05 | 403.3ms/step | 49.6 samples/s +Step 39710/44000 | Loss: 2.8616 | LR: 1.20e-05 | 403.4ms/step | 49.6 samples/s +Step 39720/44000 | Loss: 2.8894 | LR: 1.20e-05 | 404.4ms/step | 49.5 samples/s +Step 39730/44000 | Loss: 2.9294 | LR: 1.19e-05 | 404.0ms/step | 49.5 samples/s +Step 39740/44000 | Loss: 2.9811 | LR: 1.19e-05 | 404.1ms/step | 49.5 samples/s +Step 39750/44000 | Loss: 2.8940 | LR: 1.18e-05 | 404.1ms/step | 49.5 samples/s +Validation at step 39750: Loss = 2.9196, Perplexity = 18.53 +Step 39760/44000 | Loss: 2.9295 | LR: 1.18e-05 | 570.0ms/step | 35.1 samples/s +Step 39770/44000 | Loss: 2.9157 | LR: 1.17e-05 | 404.7ms/step | 49.4 samples/s +Step 39780/44000 | Loss: 2.9147 | LR: 1.17e-05 | 404.0ms/step | 49.5 samples/s +Step 39790/44000 | Loss: 2.8320 | LR: 1.16e-05 | 404.9ms/step | 49.4 samples/s +Step 39800/44000 | Loss: 2.9266 | LR: 1.15e-05 | 403.7ms/step | 49.5 samples/s +Step 39810/44000 | Loss: 2.9258 | LR: 1.15e-05 | 402.9ms/step | 49.6 samples/s +Step 39820/44000 | Loss: 2.9118 | LR: 1.14e-05 | 403.7ms/step | 49.5 samples/s +Step 39830/44000 | Loss: 2.8787 | LR: 1.14e-05 | 403.8ms/step | 49.5 samples/s +Step 39840/44000 | Loss: 2.8767 | LR: 1.13e-05 | 404.0ms/step | 49.5 samples/s +Step 39850/44000 | Loss: 2.8824 | LR: 1.13e-05 | 404.0ms/step | 49.5 samples/s +Step 39860/44000 | Loss: 2.9703 | LR: 1.12e-05 | 403.1ms/step | 49.6 samples/s +Step 39870/44000 | Loss: 2.9191 | LR: 1.12e-05 | 403.5ms/step | 49.6 samples/s +Step 39880/44000 | Loss: 2.9655 | LR: 1.11e-05 | 402.1ms/step | 49.7 samples/s +Step 39890/44000 | Loss: 2.8948 | LR: 1.11e-05 | 402.1ms/step | 49.7 samples/s +Step 39900/44000 | Loss: 2.8641 | LR: 1.10e-05 | 402.9ms/step | 49.6 samples/s +Step 39910/44000 | Loss: 2.9475 | LR: 1.10e-05 | 402.6ms/step | 49.7 samples/s +Step 39920/44000 | Loss: 2.9308 | LR: 1.09e-05 | 402.0ms/step | 49.8 samples/s +Step 39930/44000 | Loss: 2.9600 | LR: 1.08e-05 | 402.4ms/step | 49.7 samples/s +Step 39940/44000 | Loss: 2.9321 | LR: 1.08e-05 | 403.2ms/step | 49.6 samples/s +Step 39950/44000 | Loss: 2.8842 | LR: 1.07e-05 | 403.1ms/step | 49.6 samples/s +Step 39960/44000 | Loss: 2.8981 | LR: 1.07e-05 | 403.5ms/step | 49.6 samples/s +Step 39970/44000 | Loss: 2.8860 | LR: 1.06e-05 | 402.8ms/step | 49.7 samples/s +Step 39980/44000 | Loss: 2.9262 | LR: 1.06e-05 | 402.8ms/step | 49.6 samples/s +Step 39990/44000 | Loss: 2.9034 | LR: 1.05e-05 | 403.3ms/step | 49.6 samples/s +Step 40000/44000 | Loss: 2.8367 | LR: 1.05e-05 | 402.8ms/step | 49.7 samples/s +Validation at step 40000: Loss = 2.9193, Perplexity = 18.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-40000 +Step 40010/44000 | Loss: 2.9673 | LR: 1.04e-05 | 872.0ms/step | 22.9 samples/s +Step 40020/44000 | Loss: 2.8651 | LR: 1.04e-05 | 401.8ms/step | 49.8 samples/s +Step 40030/44000 | Loss: 2.9265 | LR: 1.03e-05 | 403.5ms/step | 49.6 samples/s +Step 40040/44000 | Loss: 2.9030 | LR: 1.03e-05 | 403.2ms/step | 49.6 samples/s +Step 40050/44000 | Loss: 2.9831 | LR: 1.02e-05 | 402.3ms/step | 49.7 samples/s +Step 40060/44000 | Loss: 2.9605 | LR: 1.02e-05 | 402.9ms/step | 49.6 samples/s +Step 40070/44000 | Loss: 2.8933 | LR: 1.01e-05 | 402.9ms/step | 49.6 samples/s +Step 40080/44000 | Loss: 2.9912 | LR: 1.01e-05 | 403.8ms/step | 49.5 samples/s +Step 40090/44000 | Loss: 2.9687 | LR: 1.00e-05 | 404.4ms/step | 49.5 samples/s +Step 40100/44000 | Loss: 2.9300 | LR: 9.96e-06 | 403.1ms/step | 49.6 samples/s +Step 40110/44000 | Loss: 2.8490 | LR: 9.91e-06 | 405.1ms/step | 49.4 samples/s +Step 40120/44000 | Loss: 2.9898 | LR: 9.86e-06 | 403.6ms/step | 49.6 samples/s +Step 40130/44000 | Loss: 2.9478 | LR: 9.81e-06 | 404.8ms/step | 49.4 samples/s +Step 40140/44000 | Loss: 2.9538 | LR: 9.76e-06 | 403.8ms/step | 49.5 samples/s +Step 40150/44000 | Loss: 2.8977 | LR: 9.71e-06 | 403.3ms/step | 49.6 samples/s +Step 40160/44000 | Loss: 2.9217 | LR: 9.66e-06 | 403.5ms/step | 49.6 samples/s +Step 40170/44000 | Loss: 2.9639 | LR: 9.61e-06 | 402.2ms/step | 49.7 samples/s +Step 40180/44000 | Loss: 2.9047 | LR: 9.56e-06 | 402.4ms/step | 49.7 samples/s +Step 40190/44000 | Loss: 2.8961 | LR: 9.51e-06 | 403.3ms/step | 49.6 samples/s +Step 40200/44000 | Loss: 2.8928 | LR: 9.46e-06 | 402.9ms/step | 49.6 samples/s +Step 40210/44000 | Loss: 2.9660 | LR: 9.41e-06 | 402.6ms/step | 49.7 samples/s +Step 40220/44000 | Loss: 2.9235 | LR: 9.36e-06 | 402.4ms/step | 49.7 samples/s +Step 40230/44000 | Loss: 2.9432 | LR: 9.32e-06 | 402.6ms/step | 49.7 samples/s +Step 40240/44000 | Loss: 2.9618 | LR: 9.27e-06 | 403.7ms/step | 49.5 samples/s +Step 40250/44000 | Loss: 2.9144 | LR: 9.22e-06 | 403.2ms/step | 49.6 samples/s +Validation at step 40250: Loss = 2.9187, Perplexity = 18.52 +Step 40260/44000 | Loss: 2.9361 | LR: 9.17e-06 | 571.1ms/step | 35.0 samples/s +Step 40270/44000 | Loss: 2.9918 | LR: 9.12e-06 | 403.5ms/step | 49.6 samples/s +Step 40280/44000 | Loss: 2.9325 | LR: 9.07e-06 | 403.3ms/step | 49.6 samples/s +Step 40290/44000 | Loss: 2.9036 | LR: 9.02e-06 | 403.0ms/step | 49.6 samples/s +Step 40300/44000 | Loss: 2.8824 | LR: 8.97e-06 | 402.0ms/step | 49.8 samples/s +Step 40310/44000 | Loss: 2.9176 | LR: 8.93e-06 | 401.0ms/step | 49.9 samples/s +Step 40320/44000 | Loss: 2.9211 | LR: 8.88e-06 | 402.8ms/step | 49.7 samples/s +Step 40330/44000 | Loss: 2.8735 | LR: 8.83e-06 | 401.7ms/step | 49.8 samples/s +Step 40340/44000 | Loss: 2.9065 | LR: 8.78e-06 | 401.7ms/step | 49.8 samples/s +Step 40350/44000 | Loss: 2.8785 | LR: 8.74e-06 | 402.4ms/step | 49.7 samples/s +Step 40360/44000 | Loss: 2.8932 | LR: 8.69e-06 | 402.3ms/step | 49.7 samples/s +Step 40370/44000 | Loss: 2.8324 | LR: 8.64e-06 | 402.1ms/step | 49.7 samples/s +Step 40380/44000 | Loss: 2.8588 | LR: 8.59e-06 | 402.1ms/step | 49.7 samples/s +Step 40390/44000 | Loss: 2.8559 | LR: 8.55e-06 | 403.7ms/step | 49.5 samples/s +Step 40400/44000 | Loss: 2.9311 | LR: 8.50e-06 | 403.9ms/step | 49.5 samples/s +Step 40410/44000 | Loss: 2.8865 | LR: 8.45e-06 | 403.8ms/step | 49.5 samples/s +Step 40420/44000 | Loss: 2.9137 | LR: 8.41e-06 | 403.7ms/step | 49.5 samples/s +Step 40430/44000 | Loss: 2.9731 | LR: 8.36e-06 | 403.8ms/step | 49.5 samples/s +Step 40440/44000 | Loss: 2.9166 | LR: 8.31e-06 | 403.4ms/step | 49.6 samples/s +Step 40450/44000 | Loss: 2.9112 | LR: 8.27e-06 | 402.1ms/step | 49.7 samples/s +Step 40460/44000 | Loss: 2.8975 | LR: 8.22e-06 | 401.4ms/step | 49.8 samples/s +Step 40470/44000 | Loss: 2.9739 | LR: 8.17e-06 | 402.7ms/step | 49.7 samples/s +Step 40480/44000 | Loss: 2.9184 | LR: 8.13e-06 | 402.4ms/step | 49.7 samples/s +Step 40490/44000 | Loss: 2.9424 | LR: 8.08e-06 | 402.0ms/step | 49.8 samples/s +Step 40500/44000 | Loss: 2.9231 | LR: 8.04e-06 | 402.9ms/step | 49.6 samples/s +Validation at step 40500: Loss = 2.9190, Perplexity = 18.52 +Step 40510/44000 | Loss: 2.8889 | LR: 7.99e-06 | 569.1ms/step | 35.1 samples/s +Step 40520/44000 | Loss: 2.9245 | LR: 7.94e-06 | 403.7ms/step | 49.5 samples/s +Step 40530/44000 | Loss: 2.9338 | LR: 7.90e-06 | 404.6ms/step | 49.4 samples/s +Step 40540/44000 | Loss: 2.9393 | LR: 7.85e-06 | 403.6ms/step | 49.5 samples/s +Step 40550/44000 | Loss: 2.8996 | LR: 7.81e-06 | 403.7ms/step | 49.5 samples/s +Step 40560/44000 | Loss: 2.9240 | LR: 7.76e-06 | 402.9ms/step | 49.6 samples/s +Step 40570/44000 | Loss: 2.9067 | LR: 7.72e-06 | 403.8ms/step | 49.5 samples/s +Step 40580/44000 | Loss: 2.9375 | LR: 7.67e-06 | 403.7ms/step | 49.5 samples/s +Step 40590/44000 | Loss: 2.9506 | LR: 7.63e-06 | 403.3ms/step | 49.6 samples/s +Step 40600/44000 | Loss: 2.9461 | LR: 7.59e-06 | 404.0ms/step | 49.5 samples/s +Step 40610/44000 | Loss: 2.9033 | LR: 7.54e-06 | 403.2ms/step | 49.6 samples/s +Step 40620/44000 | Loss: 2.9310 | LR: 7.50e-06 | 404.1ms/step | 49.5 samples/s +Step 40630/44000 | Loss: 2.8781 | LR: 7.45e-06 | 403.2ms/step | 49.6 samples/s +Step 40640/44000 | Loss: 2.9159 | LR: 7.41e-06 | 403.4ms/step | 49.6 samples/s +Step 40650/44000 | Loss: 2.9365 | LR: 7.37e-06 | 403.5ms/step | 49.6 samples/s +Step 40660/44000 | Loss: 2.8948 | LR: 7.32e-06 | 403.7ms/step | 49.5 samples/s +Step 40670/44000 | Loss: 2.9042 | LR: 7.28e-06 | 404.6ms/step | 49.4 samples/s +Step 40680/44000 | Loss: 2.9076 | LR: 7.23e-06 | 403.4ms/step | 49.6 samples/s +Step 40690/44000 | Loss: 2.9407 | LR: 7.19e-06 | 403.9ms/step | 49.5 samples/s +Step 40700/44000 | Loss: 2.8772 | LR: 7.15e-06 | 404.1ms/step | 49.5 samples/s +Step 40710/44000 | Loss: 2.9069 | LR: 7.10e-06 | 404.0ms/step | 49.5 samples/s +Step 40720/44000 | Loss: 2.8474 | LR: 7.06e-06 | 403.6ms/step | 49.6 samples/s +Step 40730/44000 | Loss: 2.9311 | LR: 7.02e-06 | 403.9ms/step | 49.5 samples/s +Step 40740/44000 | Loss: 2.8985 | LR: 6.98e-06 | 403.3ms/step | 49.6 samples/s +Step 40750/44000 | Loss: 2.8970 | LR: 6.93e-06 | 403.6ms/step | 49.5 samples/s +Validation at step 40750: Loss = 2.9181, Perplexity = 18.51 +Step 40760/44000 | Loss: 2.9641 | LR: 6.89e-06 | 571.9ms/step | 35.0 samples/s +Step 40770/44000 | Loss: 2.9198 | LR: 6.85e-06 | 404.6ms/step | 49.4 samples/s +Step 40780/44000 | Loss: 2.9005 | LR: 6.81e-06 | 405.1ms/step | 49.4 samples/s +Step 40790/44000 | Loss: 2.9652 | LR: 6.77e-06 | 403.7ms/step | 49.5 samples/s +Step 40800/44000 | Loss: 2.9012 | LR: 6.72e-06 | 403.4ms/step | 49.6 samples/s +Step 40810/44000 | Loss: 2.9336 | LR: 6.68e-06 | 403.5ms/step | 49.6 samples/s +Step 40820/44000 | Loss: 2.8557 | LR: 6.64e-06 | 403.9ms/step | 49.5 samples/s +Step 40830/44000 | Loss: 2.8939 | LR: 6.60e-06 | 404.7ms/step | 49.4 samples/s +Step 40840/44000 | Loss: 2.9558 | LR: 6.56e-06 | 404.6ms/step | 49.4 samples/s +Step 40850/44000 | Loss: 2.9262 | LR: 6.52e-06 | 404.5ms/step | 49.4 samples/s +Step 40860/44000 | Loss: 2.8892 | LR: 6.47e-06 | 405.2ms/step | 49.4 samples/s +Step 40870/44000 | Loss: 2.9427 | LR: 6.43e-06 | 404.7ms/step | 49.4 samples/s +Step 40880/44000 | Loss: 2.9450 | LR: 6.39e-06 | 404.9ms/step | 49.4 samples/s +Step 40890/44000 | Loss: 2.8949 | LR: 6.35e-06 | 405.2ms/step | 49.4 samples/s +Step 40900/44000 | Loss: 2.8784 | LR: 6.31e-06 | 405.1ms/step | 49.4 samples/s +Step 40910/44000 | Loss: 2.9218 | LR: 6.27e-06 | 405.5ms/step | 49.3 samples/s +Step 40920/44000 | Loss: 2.8946 | LR: 6.23e-06 | 404.8ms/step | 49.4 samples/s +Step 40930/44000 | Loss: 3.0008 | LR: 6.19e-06 | 404.4ms/step | 49.5 samples/s +Step 40940/44000 | Loss: 2.8595 | LR: 6.15e-06 | 404.2ms/step | 49.5 samples/s +Step 40950/44000 | Loss: 2.9523 | LR: 6.11e-06 | 403.4ms/step | 49.6 samples/s +Step 40960/44000 | Loss: 2.9010 | LR: 6.07e-06 | 404.4ms/step | 49.5 samples/s +Step 40970/44000 | Loss: 2.9990 | LR: 6.03e-06 | 403.6ms/step | 49.5 samples/s +Step 40980/44000 | Loss: 2.9299 | LR: 5.99e-06 | 406.4ms/step | 49.2 samples/s +Step 40990/44000 | Loss: 2.9179 | LR: 5.95e-06 | 405.4ms/step | 49.3 samples/s +Step 41000/44000 | Loss: 2.8897 | LR: 5.91e-06 | 403.9ms/step | 49.5 samples/s +Validation at step 41000: Loss = 2.9190, Perplexity = 18.52 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-41000 +Step 41010/44000 | Loss: 2.9105 | LR: 5.87e-06 | 863.6ms/step | 23.2 samples/s +Step 41020/44000 | Loss: 2.9599 | LR: 5.83e-06 | 403.4ms/step | 49.6 samples/s +Step 41030/44000 | Loss: 2.8867 | LR: 5.80e-06 | 403.3ms/step | 49.6 samples/s +Step 41040/44000 | Loss: 2.9201 | LR: 5.76e-06 | 402.3ms/step | 49.7 samples/s +Step 41050/44000 | Loss: 2.9016 | LR: 5.72e-06 | 403.0ms/step | 49.6 samples/s +Step 41060/44000 | Loss: 2.9951 | LR: 5.68e-06 | 402.4ms/step | 49.7 samples/s +Step 41070/44000 | Loss: 2.8985 | LR: 5.64e-06 | 402.0ms/step | 49.7 samples/s +Step 41080/44000 | Loss: 2.9535 | LR: 5.60e-06 | 402.0ms/step | 49.7 samples/s +Step 41090/44000 | Loss: 2.8969 | LR: 5.56e-06 | 401.6ms/step | 49.8 samples/s +Step 41100/44000 | Loss: 2.9269 | LR: 5.53e-06 | 401.9ms/step | 49.8 samples/s +Step 41110/44000 | Loss: 2.9411 | LR: 5.49e-06 | 402.1ms/step | 49.7 samples/s +Step 41120/44000 | Loss: 2.8621 | LR: 5.45e-06 | 403.6ms/step | 49.6 samples/s +Step 41130/44000 | Loss: 2.9524 | LR: 5.41e-06 | 402.4ms/step | 49.7 samples/s +Step 41140/44000 | Loss: 2.8964 | LR: 5.38e-06 | 402.8ms/step | 49.6 samples/s +Step 41150/44000 | Loss: 2.8800 | LR: 5.34e-06 | 404.2ms/step | 49.5 samples/s +Step 41160/44000 | Loss: 2.9077 | LR: 5.30e-06 | 402.9ms/step | 49.6 samples/s +Step 41170/44000 | Loss: 2.9277 | LR: 5.26e-06 | 402.5ms/step | 49.7 samples/s +Step 41180/44000 | Loss: 2.8960 | LR: 5.23e-06 | 402.8ms/step | 49.7 samples/s +Step 41190/44000 | Loss: 2.9295 | LR: 5.19e-06 | 401.8ms/step | 49.8 samples/s +Step 41200/44000 | Loss: 2.8880 | LR: 5.15e-06 | 403.8ms/step | 49.5 samples/s +Step 41210/44000 | Loss: 2.9050 | LR: 5.12e-06 | 403.2ms/step | 49.6 samples/s +Step 41220/44000 | Loss: 2.9410 | LR: 5.08e-06 | 405.2ms/step | 49.4 samples/s +Step 41230/44000 | Loss: 2.9996 | LR: 5.04e-06 | 403.2ms/step | 49.6 samples/s +Step 41240/44000 | Loss: 2.8788 | LR: 5.01e-06 | 403.7ms/step | 49.5 samples/s +Step 41250/44000 | Loss: 2.9374 | LR: 4.97e-06 | 404.1ms/step | 49.5 samples/s +Validation at step 41250: Loss = 2.9197, Perplexity = 18.54 +Step 41260/44000 | Loss: 2.9162 | LR: 4.94e-06 | 570.4ms/step | 35.1 samples/s +Step 41270/44000 | Loss: 2.8756 | LR: 4.90e-06 | 401.7ms/step | 49.8 samples/s +Step 41280/44000 | Loss: 2.8514 | LR: 4.86e-06 | 405.0ms/step | 49.4 samples/s +Step 41290/44000 | Loss: 2.9536 | LR: 4.83e-06 | 404.7ms/step | 49.4 samples/s +Step 41300/44000 | Loss: 2.9241 | LR: 4.79e-06 | 404.0ms/step | 49.5 samples/s +Step 41310/44000 | Loss: 2.9369 | LR: 4.76e-06 | 404.2ms/step | 49.5 samples/s +Step 41320/44000 | Loss: 2.9059 | LR: 4.72e-06 | 404.2ms/step | 49.5 samples/s +Step 41330/44000 | Loss: 2.9695 | LR: 4.69e-06 | 403.6ms/step | 49.6 samples/s +Step 41340/44000 | Loss: 2.8570 | LR: 4.65e-06 | 404.9ms/step | 49.4 samples/s +Step 41350/44000 | Loss: 2.9740 | LR: 4.62e-06 | 404.7ms/step | 49.4 samples/s +Step 41360/44000 | Loss: 2.9202 | LR: 4.58e-06 | 404.2ms/step | 49.5 samples/s +Step 41370/44000 | Loss: 2.9703 | LR: 4.55e-06 | 403.8ms/step | 49.5 samples/s +Step 41380/44000 | Loss: 2.9193 | LR: 4.51e-06 | 404.0ms/step | 49.5 samples/s +Step 41390/44000 | Loss: 2.8979 | LR: 4.48e-06 | 404.7ms/step | 49.4 samples/s +Step 41400/44000 | Loss: 2.8965 | LR: 4.45e-06 | 404.0ms/step | 49.5 samples/s +Step 41410/44000 | Loss: 2.9425 | LR: 4.41e-06 | 404.9ms/step | 49.4 samples/s +Step 41420/44000 | Loss: 2.9115 | LR: 4.38e-06 | 404.4ms/step | 49.5 samples/s +Step 41430/44000 | Loss: 2.8796 | LR: 4.34e-06 | 403.3ms/step | 49.6 samples/s +Step 41440/44000 | Loss: 2.9073 | LR: 4.31e-06 | 403.7ms/step | 49.5 samples/s +Step 41450/44000 | Loss: 2.9031 | LR: 4.28e-06 | 403.5ms/step | 49.6 samples/s +Step 41460/44000 | Loss: 2.8804 | LR: 4.24e-06 | 403.2ms/step | 49.6 samples/s +Step 41470/44000 | Loss: 2.9166 | LR: 4.21e-06 | 403.6ms/step | 49.6 samples/s +Step 41480/44000 | Loss: 2.8899 | LR: 4.18e-06 | 403.7ms/step | 49.5 samples/s +Step 41490/44000 | Loss: 2.8397 | LR: 4.14e-06 | 403.1ms/step | 49.6 samples/s +Step 41500/44000 | Loss: 2.9895 | LR: 4.11e-06 | 402.7ms/step | 49.7 samples/s +Validation at step 41500: Loss = 2.9200, Perplexity = 18.54 +Step 41510/44000 | Loss: 2.8710 | LR: 4.08e-06 | 569.5ms/step | 35.1 samples/s +Step 41520/44000 | Loss: 2.9008 | LR: 4.05e-06 | 404.9ms/step | 49.4 samples/s +Step 41530/44000 | Loss: 2.8838 | LR: 4.01e-06 | 404.6ms/step | 49.4 samples/s +Step 41540/44000 | Loss: 2.9337 | LR: 3.98e-06 | 404.4ms/step | 49.5 samples/s +Step 41550/44000 | Loss: 2.9295 | LR: 3.95e-06 | 405.3ms/step | 49.3 samples/s +Step 41560/44000 | Loss: 2.9397 | LR: 3.92e-06 | 405.2ms/step | 49.4 samples/s +Step 41570/44000 | Loss: 2.9494 | LR: 3.88e-06 | 404.1ms/step | 49.5 samples/s +Step 41580/44000 | Loss: 2.8973 | LR: 3.85e-06 | 403.5ms/step | 49.6 samples/s +Step 41590/44000 | Loss: 2.9342 | LR: 3.82e-06 | 403.4ms/step | 49.6 samples/s +Step 41600/44000 | Loss: 2.9084 | LR: 3.79e-06 | 402.6ms/step | 49.7 samples/s +Step 41610/44000 | Loss: 2.9417 | LR: 3.76e-06 | 403.1ms/step | 49.6 samples/s +Step 41620/44000 | Loss: 2.8522 | LR: 3.73e-06 | 403.0ms/step | 49.6 samples/s +Step 41630/44000 | Loss: 2.8940 | LR: 3.70e-06 | 403.4ms/step | 49.6 samples/s +Step 41640/44000 | Loss: 2.9105 | LR: 3.66e-06 | 402.8ms/step | 49.7 samples/s +Step 41650/44000 | Loss: 2.9684 | LR: 3.63e-06 | 401.7ms/step | 49.8 samples/s +Step 41660/44000 | Loss: 2.9377 | LR: 3.60e-06 | 405.3ms/step | 49.3 samples/s +Step 41670/44000 | Loss: 2.9222 | LR: 3.57e-06 | 403.9ms/step | 49.5 samples/s +Step 41680/44000 | Loss: 2.9501 | LR: 3.54e-06 | 404.4ms/step | 49.5 samples/s +Step 41690/44000 | Loss: 2.9367 | LR: 3.51e-06 | 404.2ms/step | 49.5 samples/s +Step 41700/44000 | Loss: 2.9147 | LR: 3.48e-06 | 405.4ms/step | 49.3 samples/s +Step 41710/44000 | Loss: 2.9211 | LR: 3.45e-06 | 405.8ms/step | 49.3 samples/s +Step 41720/44000 | Loss: 2.9047 | LR: 3.42e-06 | 404.6ms/step | 49.4 samples/s +Step 41730/44000 | Loss: 2.9240 | LR: 3.39e-06 | 404.7ms/step | 49.4 samples/s +Step 41740/44000 | Loss: 2.9171 | LR: 3.36e-06 | 405.3ms/step | 49.3 samples/s +Step 41750/44000 | Loss: 2.9313 | LR: 3.33e-06 | 404.5ms/step | 49.4 samples/s +Validation at step 41750: Loss = 2.9183, Perplexity = 18.51 +Step 41760/44000 | Loss: 2.8915 | LR: 3.30e-06 | 570.5ms/step | 35.1 samples/s +Step 41770/44000 | Loss: 2.9169 | LR: 3.27e-06 | 403.6ms/step | 49.6 samples/s +Step 41780/44000 | Loss: 2.9090 | LR: 3.24e-06 | 403.5ms/step | 49.6 samples/s +Step 41790/44000 | Loss: 2.9464 | LR: 3.21e-06 | 402.6ms/step | 49.7 samples/s +Step 41800/44000 | Loss: 2.8995 | LR: 3.19e-06 | 402.8ms/step | 49.6 samples/s +Step 41810/44000 | Loss: 2.9052 | LR: 3.16e-06 | 403.3ms/step | 49.6 samples/s +Step 41820/44000 | Loss: 2.9058 | LR: 3.13e-06 | 402.4ms/step | 49.7 samples/s +Step 41830/44000 | Loss: 2.8513 | LR: 3.10e-06 | 404.1ms/step | 49.5 samples/s +Step 41840/44000 | Loss: 2.9003 | LR: 3.07e-06 | 403.2ms/step | 49.6 samples/s +Step 41850/44000 | Loss: 2.9012 | LR: 3.04e-06 | 403.8ms/step | 49.5 samples/s +Step 41860/44000 | Loss: 2.9269 | LR: 3.01e-06 | 402.9ms/step | 49.6 samples/s +Step 41870/44000 | Loss: 2.8724 | LR: 2.99e-06 | 403.9ms/step | 49.5 samples/s +Step 41880/44000 | Loss: 2.9051 | LR: 2.96e-06 | 404.3ms/step | 49.5 samples/s +Step 41890/44000 | Loss: 2.8970 | LR: 2.93e-06 | 402.8ms/step | 49.6 samples/s +Step 41900/44000 | Loss: 2.9019 | LR: 2.90e-06 | 402.3ms/step | 49.7 samples/s +Step 41910/44000 | Loss: 2.9379 | LR: 2.88e-06 | 402.8ms/step | 49.6 samples/s +Step 41920/44000 | Loss: 2.8952 | LR: 2.85e-06 | 403.9ms/step | 49.5 samples/s +Step 41930/44000 | Loss: 2.9726 | LR: 2.82e-06 | 404.8ms/step | 49.4 samples/s +Step 41940/44000 | Loss: 2.8892 | LR: 2.79e-06 | 404.0ms/step | 49.5 samples/s +Step 41950/44000 | Loss: 2.8643 | LR: 2.77e-06 | 404.4ms/step | 49.5 samples/s +Step 41960/44000 | Loss: 2.8833 | LR: 2.74e-06 | 404.1ms/step | 49.5 samples/s +Step 41970/44000 | Loss: 2.9340 | LR: 2.71e-06 | 403.9ms/step | 49.5 samples/s +Step 41980/44000 | Loss: 2.9556 | LR: 2.69e-06 | 403.9ms/step | 49.5 samples/s +Step 41990/44000 | Loss: 2.9427 | LR: 2.66e-06 | 404.2ms/step | 49.5 samples/s +Step 42000/44000 | Loss: 2.8743 | LR: 2.63e-06 | 405.8ms/step | 49.3 samples/s +Validation at step 42000: Loss = 2.9187, Perplexity = 18.52 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-42000 +Step 42010/44000 | Loss: 2.9124 | LR: 2.61e-06 | 940.4ms/step | 21.3 samples/s +Step 42020/44000 | Loss: 2.9065 | LR: 2.58e-06 | 404.1ms/step | 49.5 samples/s +Step 42030/44000 | Loss: 2.9633 | LR: 2.56e-06 | 402.7ms/step | 49.7 samples/s +Step 42040/44000 | Loss: 2.9231 | LR: 2.53e-06 | 403.0ms/step | 49.6 samples/s +Step 42050/44000 | Loss: 2.9480 | LR: 2.50e-06 | 404.0ms/step | 49.5 samples/s +Step 42060/44000 | Loss: 2.9260 | LR: 2.48e-06 | 401.5ms/step | 49.8 samples/s +Step 42070/44000 | Loss: 2.9303 | LR: 2.45e-06 | 402.6ms/step | 49.7 samples/s +Step 42080/44000 | Loss: 2.9452 | LR: 2.43e-06 | 404.4ms/step | 49.5 samples/s +Step 42090/44000 | Loss: 2.9725 | LR: 2.40e-06 | 409.0ms/step | 48.9 samples/s +Step 42100/44000 | Loss: 2.8605 | LR: 2.38e-06 | 408.9ms/step | 48.9 samples/s +Step 42110/44000 | Loss: 2.9377 | LR: 2.35e-06 | 404.2ms/step | 49.5 samples/s +Step 42120/44000 | Loss: 2.9156 | LR: 2.33e-06 | 403.1ms/step | 49.6 samples/s +Step 42130/44000 | Loss: 2.9092 | LR: 2.30e-06 | 402.5ms/step | 49.7 samples/s +Step 42140/44000 | Loss: 2.9346 | LR: 2.28e-06 | 404.1ms/step | 49.5 samples/s +Step 42150/44000 | Loss: 2.8707 | LR: 2.25e-06 | 401.9ms/step | 49.8 samples/s +Step 42160/44000 | Loss: 2.8943 | LR: 2.23e-06 | 401.6ms/step | 49.8 samples/s +Step 42170/44000 | Loss: 2.9251 | LR: 2.21e-06 | 402.1ms/step | 49.7 samples/s +Step 42180/44000 | Loss: 2.8663 | LR: 2.18e-06 | 402.2ms/step | 49.7 samples/s +Step 42190/44000 | Loss: 2.9177 | LR: 2.16e-06 | 403.3ms/step | 49.6 samples/s +Step 42200/44000 | Loss: 3.0075 | LR: 2.13e-06 | 401.6ms/step | 49.8 samples/s +Step 42210/44000 | Loss: 2.9416 | LR: 2.11e-06 | 402.3ms/step | 49.7 samples/s +Step 42220/44000 | Loss: 2.9553 | LR: 2.09e-06 | 402.2ms/step | 49.7 samples/s +Step 42230/44000 | Loss: 2.9962 | LR: 2.06e-06 | 402.6ms/step | 49.7 samples/s +Step 42240/44000 | Loss: 2.9986 | LR: 2.04e-06 | 402.6ms/step | 49.7 samples/s +Step 42250/44000 | Loss: 2.9242 | LR: 2.02e-06 | 403.6ms/step | 49.6 samples/s +Validation at step 42250: Loss = 2.9192, Perplexity = 18.53 +Step 42260/44000 | Loss: 2.9019 | LR: 1.99e-06 | 574.3ms/step | 34.8 samples/s +Step 42270/44000 | Loss: 2.8899 | LR: 1.97e-06 | 403.4ms/step | 49.6 samples/s +Step 42280/44000 | Loss: 2.9598 | LR: 1.95e-06 | 405.0ms/step | 49.4 samples/s +Step 42290/44000 | Loss: 2.9647 | LR: 1.93e-06 | 404.8ms/step | 49.4 samples/s +Step 42300/44000 | Loss: 2.9555 | LR: 1.90e-06 | 404.0ms/step | 49.5 samples/s +Step 42310/44000 | Loss: 2.8839 | LR: 1.88e-06 | 405.0ms/step | 49.4 samples/s +Step 42320/44000 | Loss: 2.9274 | LR: 1.86e-06 | 404.4ms/step | 49.5 samples/s +Step 42330/44000 | Loss: 2.8719 | LR: 1.84e-06 | 404.7ms/step | 49.4 samples/s +Step 42340/44000 | Loss: 2.8418 | LR: 1.82e-06 | 404.1ms/step | 49.5 samples/s +Step 42350/44000 | Loss: 3.0112 | LR: 1.79e-06 | 405.4ms/step | 49.3 samples/s +Step 42360/44000 | Loss: 2.9427 | LR: 1.77e-06 | 404.5ms/step | 49.4 samples/s +Step 42370/44000 | Loss: 2.9794 | LR: 1.75e-06 | 403.8ms/step | 49.5 samples/s +Step 42380/44000 | Loss: 2.9114 | LR: 1.73e-06 | 404.5ms/step | 49.4 samples/s +Step 42390/44000 | Loss: 2.9931 | LR: 1.71e-06 | 403.8ms/step | 49.5 samples/s +Step 42400/44000 | Loss: 2.9241 | LR: 1.69e-06 | 403.6ms/step | 49.6 samples/s +Step 42410/44000 | Loss: 2.9060 | LR: 1.67e-06 | 403.6ms/step | 49.6 samples/s +Step 42420/44000 | Loss: 2.8978 | LR: 1.64e-06 | 401.7ms/step | 49.8 samples/s +Step 42430/44000 | Loss: 2.9404 | LR: 1.62e-06 | 402.8ms/step | 49.7 samples/s +Step 42440/44000 | Loss: 2.8972 | LR: 1.60e-06 | 402.6ms/step | 49.7 samples/s +Step 42450/44000 | Loss: 2.9370 | LR: 1.58e-06 | 402.3ms/step | 49.7 samples/s +Step 42460/44000 | Loss: 2.8883 | LR: 1.56e-06 | 402.3ms/step | 49.7 samples/s +Step 42470/44000 | Loss: 2.9006 | LR: 1.54e-06 | 401.6ms/step | 49.8 samples/s +Step 42480/44000 | Loss: 2.8843 | LR: 1.52e-06 | 403.0ms/step | 49.6 samples/s +Step 42490/44000 | Loss: 2.9016 | LR: 1.50e-06 | 402.2ms/step | 49.7 samples/s +Step 42500/44000 | Loss: 2.9304 | LR: 1.48e-06 | 403.2ms/step | 49.6 samples/s +Validation at step 42500: Loss = 2.9187, Perplexity = 18.52 +Step 42510/44000 | Loss: 2.9081 | LR: 1.46e-06 | 573.2ms/step | 34.9 samples/s +Step 42520/44000 | Loss: 2.9260 | LR: 1.44e-06 | 404.1ms/step | 49.5 samples/s +Step 42530/44000 | Loss: 2.8941 | LR: 1.42e-06 | 401.8ms/step | 49.8 samples/s +Step 42540/44000 | Loss: 2.9069 | LR: 1.40e-06 | 401.4ms/step | 49.8 samples/s +Step 42550/44000 | Loss: 2.9311 | LR: 1.39e-06 | 402.5ms/step | 49.7 samples/s +Step 42560/44000 | Loss: 2.9800 | LR: 1.37e-06 | 403.3ms/step | 49.6 samples/s +Step 42570/44000 | Loss: 2.9300 | LR: 1.35e-06 | 403.7ms/step | 49.5 samples/s +Step 42580/44000 | Loss: 2.9643 | LR: 1.33e-06 | 404.0ms/step | 49.5 samples/s +Step 42590/44000 | Loss: 2.9175 | LR: 1.31e-06 | 403.6ms/step | 49.6 samples/s +Step 42600/44000 | Loss: 2.9816 | LR: 1.29e-06 | 403.5ms/step | 49.6 samples/s +Step 42610/44000 | Loss: 2.8423 | LR: 1.27e-06 | 403.4ms/step | 49.6 samples/s +Step 42620/44000 | Loss: 2.9411 | LR: 1.25e-06 | 403.1ms/step | 49.6 samples/s +Step 42630/44000 | Loss: 2.9287 | LR: 1.24e-06 | 402.9ms/step | 49.6 samples/s +Step 42640/44000 | Loss: 2.9316 | LR: 1.22e-06 | 402.8ms/step | 49.7 samples/s +Step 42650/44000 | Loss: 2.9349 | LR: 1.20e-06 | 403.0ms/step | 49.6 samples/s +Step 42660/44000 | Loss: 2.9821 | LR: 1.18e-06 | 401.4ms/step | 49.8 samples/s +Step 42670/44000 | Loss: 2.9138 | LR: 1.17e-06 | 401.7ms/step | 49.8 samples/s +Step 42680/44000 | Loss: 2.8983 | LR: 1.15e-06 | 401.4ms/step | 49.8 samples/s +Step 42690/44000 | Loss: 2.9495 | LR: 1.13e-06 | 402.0ms/step | 49.8 samples/s +Step 42700/44000 | Loss: 2.9745 | LR: 1.11e-06 | 400.6ms/step | 49.9 samples/s +Step 42710/44000 | Loss: 2.9156 | LR: 1.10e-06 | 400.1ms/step | 50.0 samples/s +Step 42720/44000 | Loss: 2.9278 | LR: 1.08e-06 | 400.8ms/step | 49.9 samples/s +Step 42730/44000 | Loss: 2.9060 | LR: 1.06e-06 | 401.1ms/step | 49.9 samples/s +Step 42740/44000 | Loss: 2.9285 | LR: 1.05e-06 | 399.9ms/step | 50.0 samples/s +Step 42750/44000 | Loss: 2.9229 | LR: 1.03e-06 | 401.7ms/step | 49.8 samples/s +Validation at step 42750: Loss = 2.9187, Perplexity = 18.52 +Step 42760/44000 | Loss: 2.9381 | LR: 1.01e-06 | 568.4ms/step | 35.2 samples/s +Step 42770/44000 | Loss: 2.9972 | LR: 9.97e-07 | 402.7ms/step | 49.7 samples/s +Step 42780/44000 | Loss: 2.9065 | LR: 9.81e-07 | 402.6ms/step | 49.7 samples/s +Step 42790/44000 | Loss: 2.9433 | LR: 9.65e-07 | 402.1ms/step | 49.7 samples/s +Step 42800/44000 | Loss: 2.8811 | LR: 9.49e-07 | 402.0ms/step | 49.8 samples/s +Step 42810/44000 | Loss: 2.9006 | LR: 9.33e-07 | 402.1ms/step | 49.7 samples/s +Step 42820/44000 | Loss: 2.9767 | LR: 9.18e-07 | 402.0ms/step | 49.7 samples/s +Step 42830/44000 | Loss: 2.8848 | LR: 9.02e-07 | 400.8ms/step | 49.9 samples/s +Step 42840/44000 | Loss: 2.9916 | LR: 8.87e-07 | 401.1ms/step | 49.9 samples/s +Step 42850/44000 | Loss: 2.8812 | LR: 8.72e-07 | 400.0ms/step | 50.0 samples/s +Step 42860/44000 | Loss: 2.9652 | LR: 8.57e-07 | 400.0ms/step | 50.0 samples/s +Step 42870/44000 | Loss: 2.9244 | LR: 8.42e-07 | 400.0ms/step | 50.0 samples/s +Step 42880/44000 | Loss: 2.9405 | LR: 8.27e-07 | 401.0ms/step | 49.9 samples/s +Step 42890/44000 | Loss: 2.9068 | LR: 8.12e-07 | 401.4ms/step | 49.8 samples/s +Step 42900/44000 | Loss: 2.8885 | LR: 7.98e-07 | 400.3ms/step | 50.0 samples/s +Step 42910/44000 | Loss: 2.9292 | LR: 7.83e-07 | 399.8ms/step | 50.0 samples/s +Step 42920/44000 | Loss: 2.8767 | LR: 7.69e-07 | 399.7ms/step | 50.0 samples/s +Step 42930/44000 | Loss: 2.8712 | LR: 7.55e-07 | 399.8ms/step | 50.0 samples/s +Step 42940/44000 | Loss: 2.9557 | LR: 7.41e-07 | 400.2ms/step | 50.0 samples/s +Step 42950/44000 | Loss: 2.9672 | LR: 7.27e-07 | 399.9ms/step | 50.0 samples/s +Step 42960/44000 | Loss: 2.9157 | LR: 7.13e-07 | 399.9ms/step | 50.0 samples/s +Step 42970/44000 | Loss: 2.9058 | LR: 6.99e-07 | 400.1ms/step | 50.0 samples/s +Step 42980/44000 | Loss: 2.9879 | LR: 6.86e-07 | 400.7ms/step | 49.9 samples/s +Step 42990/44000 | Loss: 2.9037 | LR: 6.72e-07 | 399.9ms/step | 50.0 samples/s +Step 43000/44000 | Loss: 2.9674 | LR: 6.59e-07 | 400.9ms/step | 49.9 samples/s +Validation at step 43000: Loss = 2.9189, Perplexity = 18.52 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-43000 +Step 43010/44000 | Loss: 2.9235 | LR: 6.46e-07 | 869.8ms/step | 23.0 samples/s +Step 43020/44000 | Loss: 2.9123 | LR: 6.33e-07 | 403.3ms/step | 49.6 samples/s +Step 43030/44000 | Loss: 2.9453 | LR: 6.20e-07 | 401.7ms/step | 49.8 samples/s +Step 43040/44000 | Loss: 2.9253 | LR: 6.08e-07 | 401.8ms/step | 49.8 samples/s +Step 43050/44000 | Loss: 2.9680 | LR: 5.95e-07 | 401.5ms/step | 49.8 samples/s +Step 43060/44000 | Loss: 2.9671 | LR: 5.83e-07 | 404.0ms/step | 49.5 samples/s +Step 43070/44000 | Loss: 2.8792 | LR: 5.70e-07 | 403.3ms/step | 49.6 samples/s +Step 43080/44000 | Loss: 2.9314 | LR: 5.58e-07 | 403.4ms/step | 49.6 samples/s +Step 43090/44000 | Loss: 2.9599 | LR: 5.46e-07 | 402.4ms/step | 49.7 samples/s +Step 43100/44000 | Loss: 2.9454 | LR: 5.34e-07 | 403.2ms/step | 49.6 samples/s +Step 43110/44000 | Loss: 2.8634 | LR: 5.22e-07 | 402.9ms/step | 49.6 samples/s +Step 43120/44000 | Loss: 2.8941 | LR: 5.11e-07 | 403.2ms/step | 49.6 samples/s +Step 43130/44000 | Loss: 2.8883 | LR: 4.99e-07 | 403.1ms/step | 49.6 samples/s +Step 43140/44000 | Loss: 2.9219 | LR: 4.88e-07 | 402.4ms/step | 49.7 samples/s +Step 43150/44000 | Loss: 2.9376 | LR: 4.76e-07 | 401.9ms/step | 49.8 samples/s +Step 43160/44000 | Loss: 2.9224 | LR: 4.65e-07 | 402.6ms/step | 49.7 samples/s +Step 43170/44000 | Loss: 2.9335 | LR: 4.54e-07 | 403.2ms/step | 49.6 samples/s +Step 43180/44000 | Loss: 2.9151 | LR: 4.43e-07 | 403.2ms/step | 49.6 samples/s +Step 43190/44000 | Loss: 2.9496 | LR: 4.33e-07 | 402.2ms/step | 49.7 samples/s +Step 43200/44000 | Loss: 2.8784 | LR: 4.22e-07 | 402.1ms/step | 49.7 samples/s +Step 43210/44000 | Loss: 2.8898 | LR: 4.12e-07 | 403.5ms/step | 49.6 samples/s +Step 43220/44000 | Loss: 2.9198 | LR: 4.01e-07 | 402.2ms/step | 49.7 samples/s +Step 43230/44000 | Loss: 2.9401 | LR: 3.91e-07 | 401.6ms/step | 49.8 samples/s +Step 43240/44000 | Loss: 2.9001 | LR: 3.81e-07 | 401.5ms/step | 49.8 samples/s +Step 43250/44000 | Loss: 2.9180 | LR: 3.71e-07 | 401.6ms/step | 49.8 samples/s +Validation at step 43250: Loss = 2.9187, Perplexity = 18.52 +Step 43260/44000 | Loss: 2.9106 | LR: 3.61e-07 | 569.5ms/step | 35.1 samples/s +Step 43270/44000 | Loss: 2.9069 | LR: 3.51e-07 | 401.9ms/step | 49.8 samples/s +Step 43280/44000 | Loss: 2.9483 | LR: 3.42e-07 | 401.7ms/step | 49.8 samples/s +Step 43290/44000 | Loss: 2.8931 | LR: 3.32e-07 | 402.2ms/step | 49.7 samples/s +Step 43300/44000 | Loss: 2.9411 | LR: 3.23e-07 | 402.8ms/step | 49.7 samples/s +Step 43310/44000 | Loss: 2.9252 | LR: 3.14e-07 | 401.8ms/step | 49.8 samples/s +Step 43320/44000 | Loss: 2.8930 | LR: 3.05e-07 | 403.1ms/step | 49.6 samples/s +Step 43330/44000 | Loss: 2.9298 | LR: 2.96e-07 | 401.9ms/step | 49.8 samples/s +Step 43340/44000 | Loss: 2.9586 | LR: 2.87e-07 | 402.2ms/step | 49.7 samples/s +Step 43350/44000 | Loss: 2.9126 | LR: 2.79e-07 | 402.7ms/step | 49.7 samples/s +Step 43360/44000 | Loss: 2.9228 | LR: 2.70e-07 | 402.9ms/step | 49.6 samples/s +Step 43370/44000 | Loss: 2.9509 | LR: 2.62e-07 | 402.3ms/step | 49.7 samples/s +Step 43380/44000 | Loss: 2.8958 | LR: 2.53e-07 | 404.6ms/step | 49.4 samples/s +Step 43390/44000 | Loss: 2.8937 | LR: 2.45e-07 | 403.2ms/step | 49.6 samples/s +Step 43400/44000 | Loss: 2.9761 | LR: 2.37e-07 | 403.6ms/step | 49.6 samples/s +Step 43410/44000 | Loss: 2.9347 | LR: 2.30e-07 | 403.6ms/step | 49.6 samples/s +Step 43420/44000 | Loss: 2.8793 | LR: 2.22e-07 | 403.1ms/step | 49.6 samples/s +Step 43430/44000 | Loss: 2.9205 | LR: 2.14e-07 | 403.1ms/step | 49.6 samples/s +Step 43440/44000 | Loss: 2.8576 | LR: 2.07e-07 | 403.2ms/step | 49.6 samples/s +Step 43450/44000 | Loss: 2.9186 | LR: 1.99e-07 | 403.3ms/step | 49.6 samples/s +Step 43460/44000 | Loss: 2.9506 | LR: 1.92e-07 | 403.2ms/step | 49.6 samples/s +Step 43470/44000 | Loss: 2.8977 | LR: 1.85e-07 | 403.2ms/step | 49.6 samples/s +Step 43480/44000 | Loss: 2.8999 | LR: 1.78e-07 | 401.7ms/step | 49.8 samples/s +Step 43490/44000 | Loss: 2.9354 | LR: 1.72e-07 | 401.5ms/step | 49.8 samples/s +Step 43500/44000 | Loss: 2.8148 | LR: 1.65e-07 | 402.3ms/step | 49.7 samples/s +Validation at step 43500: Loss = 2.9198, Perplexity = 18.54 +Step 43510/44000 | Loss: 2.9594 | LR: 1.58e-07 | 570.7ms/step | 35.0 samples/s +Step 43520/44000 | Loss: 2.8857 | LR: 1.52e-07 | 402.6ms/step | 49.7 samples/s +Step 43530/44000 | Loss: 2.9430 | LR: 1.46e-07 | 401.5ms/step | 49.8 samples/s +Step 43540/44000 | Loss: 2.9239 | LR: 1.40e-07 | 402.1ms/step | 49.7 samples/s +Step 43550/44000 | Loss: 2.8910 | LR: 1.34e-07 | 402.1ms/step | 49.7 samples/s +Step 43560/44000 | Loss: 2.9241 | LR: 1.28e-07 | 401.8ms/step | 49.8 samples/s +Step 43570/44000 | Loss: 2.9167 | LR: 1.22e-07 | 401.8ms/step | 49.8 samples/s +Step 43580/44000 | Loss: 2.9006 | LR: 1.16e-07 | 401.3ms/step | 49.8 samples/s +Step 43590/44000 | Loss: 2.9626 | LR: 1.11e-07 | 402.3ms/step | 49.7 samples/s +Step 43600/44000 | Loss: 2.9038 | LR: 1.06e-07 | 401.4ms/step | 49.8 samples/s +Step 43610/44000 | Loss: 2.9538 | LR: 1.00e-07 | 401.5ms/step | 49.8 samples/s +Step 43620/44000 | Loss: 2.8828 | LR: 9.52e-08 | 401.1ms/step | 49.9 samples/s +Step 43630/44000 | Loss: 2.9274 | LR: 9.03e-08 | 399.9ms/step | 50.0 samples/s +Step 43640/44000 | Loss: 2.8879 | LR: 8.55e-08 | 400.7ms/step | 49.9 samples/s +Step 43650/44000 | Loss: 2.9563 | LR: 8.08e-08 | 401.7ms/step | 49.8 samples/s +Step 43660/44000 | Loss: 2.9227 | LR: 7.62e-08 | 402.3ms/step | 49.7 samples/s +Step 43670/44000 | Loss: 2.9858 | LR: 7.18e-08 | 402.1ms/step | 49.7 samples/s +Step 43680/44000 | Loss: 2.9039 | LR: 6.75e-08 | 402.1ms/step | 49.7 samples/s +Step 43690/44000 | Loss: 2.9706 | LR: 6.34e-08 | 402.1ms/step | 49.7 samples/s +Step 43700/44000 | Loss: 2.8903 | LR: 5.94e-08 | 402.0ms/step | 49.8 samples/s +Step 43710/44000 | Loss: 2.9317 | LR: 5.55e-08 | 401.5ms/step | 49.8 samples/s +Step 43720/44000 | Loss: 2.9805 | LR: 5.17e-08 | 402.2ms/step | 49.7 samples/s +Step 43730/44000 | Loss: 2.9018 | LR: 4.81e-08 | 402.6ms/step | 49.7 samples/s +Step 43740/44000 | Loss: 2.9348 | LR: 4.46e-08 | 401.6ms/step | 49.8 samples/s +Step 43750/44000 | Loss: 2.8551 | LR: 4.12e-08 | 401.7ms/step | 49.8 samples/s +Validation at step 43750: Loss = 2.9190, Perplexity = 18.52 +Step 43760/44000 | Loss: 2.9046 | LR: 3.80e-08 | 569.5ms/step | 35.1 samples/s +Step 43770/44000 | Loss: 2.8771 | LR: 3.49e-08 | 402.8ms/step | 49.7 samples/s +Step 43780/44000 | Loss: 2.9488 | LR: 3.19e-08 | 402.5ms/step | 49.7 samples/s +Step 43790/44000 | Loss: 2.9725 | LR: 2.91e-08 | 402.8ms/step | 49.6 samples/s +Step 43800/44000 | Loss: 2.8834 | LR: 2.64e-08 | 401.8ms/step | 49.8 samples/s +Step 43810/44000 | Loss: 2.9887 | LR: 2.38e-08 | 400.1ms/step | 50.0 samples/s +Step 43820/44000 | Loss: 2.9364 | LR: 2.14e-08 | 401.0ms/step | 49.9 samples/s +Step 43830/44000 | Loss: 2.9244 | LR: 1.91e-08 | 401.6ms/step | 49.8 samples/s +Step 43840/44000 | Loss: 2.8794 | LR: 1.69e-08 | 399.8ms/step | 50.0 samples/s +Step 43850/44000 | Loss: 2.9304 | LR: 1.48e-08 | 399.9ms/step | 50.0 samples/s +Step 43860/44000 | Loss: 2.8562 | LR: 1.29e-08 | 399.7ms/step | 50.0 samples/s +Step 43870/44000 | Loss: 2.9404 | LR: 1.11e-08 | 400.7ms/step | 49.9 samples/s +Step 43880/44000 | Loss: 2.9495 | LR: 9.50e-09 | 400.2ms/step | 50.0 samples/s +Step 43890/44000 | Loss: 2.9737 | LR: 7.98e-09 | 401.7ms/step | 49.8 samples/s +Step 43900/44000 | Loss: 2.8871 | LR: 6.60e-09 | 401.5ms/step | 49.8 samples/s +Step 43910/44000 | Loss: 2.9093 | LR: 5.34e-09 | 402.2ms/step | 49.7 samples/s +Step 43920/44000 | Loss: 2.9382 | LR: 4.22e-09 | 401.0ms/step | 49.9 samples/s +Step 43930/44000 | Loss: 2.8885 | LR: 3.23e-09 | 401.3ms/step | 49.8 samples/s +Step 43940/44000 | Loss: 2.9135 | LR: 2.37e-09 | 402.3ms/step | 49.7 samples/s +Step 43950/44000 | Loss: 2.9752 | LR: 1.65e-09 | 401.4ms/step | 49.8 samples/s +Step 43960/44000 | Loss: 2.9840 | LR: 1.06e-09 | 402.5ms/step | 49.7 samples/s +Step 43970/44000 | Loss: 2.9355 | LR: 5.94e-10 | 402.5ms/step | 49.7 samples/s +Step 43980/44000 | Loss: 2.9146 | LR: 2.64e-10 | 402.3ms/step | 49.7 samples/s +Step 43990/44000 | Loss: 2.9443 | LR: 6.60e-11 | 401.9ms/step | 49.8 samples/s +Step 44000/44000 | Loss: 2.8273 | LR: 0.00e+00 | 402.0ms/step | 49.7 samples/s +Validation at step 44000: Loss = 2.9184, Perplexity = 18.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-44000 +wandb: updating run metadata +wandb: uploading wandb-summary.json; uploading config.yaml +wandb: uploading wandb-summary.json; uploading config.yaml; uploading output.log +wandb: uploading history steps 4397-4399, summary, console lines 3289-3293 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss ██▇▇▆▆▅▅▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity ██▇▆▆▆▅▅▄▄▄▄▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█ +wandb: speed/samples_per_sec ▇▇████████████████████▁█████████████████ +wandb: speed/steps_per_sec ▇▇▇▇▇███▁███████████▁▇█████████▁████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.91835 +wandb: eval/perplexity 18.51078 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 402.03183 +wandb: speed/samples_per_sec 49.74731 +wandb: speed/steps_per_sec 2.48737 +wandb: +4 ... +wandb: +wandb: 🚀 View run P1 DDP4 365m resampled 8k nrb35 seed45 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/9nppzrru +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260822_161907-9nppzrru/logs +Training complete! diff --git a/epfl_final/train_logs/p1_scheduled_h1004_20260822/scheduler.log b/epfl_final/train_logs/p1_scheduled_h1004_20260822/scheduler.log new file mode 100644 index 0000000000000000000000000000000000000000..8043c0ca843eb1d240a49fcb4fae344236a5e775 --- /dev/null +++ b/epfl_final/train_logs/p1_scheduled_h1004_20260822/scheduler.log @@ -0,0 +1,595 @@ +[scheduler] waiting for all four H1004 GPUs to become free 2026-08-22T04:40:03Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:40:03Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:41:03Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:42:03Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:43:03Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:44:03Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:45:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:46:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:47:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:48:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:49:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:50:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:51:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:52:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:53:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:54:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:55:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:56:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:57:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:58:04Z +[scheduler] active=68 busy_gpus=1 2026-08-22T04:59:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:00:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:01:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:02:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:03:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:04:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:05:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:06:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:07:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:08:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:09:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:10:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:11:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:12:05Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:13:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:14:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:15:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:16:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:17:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:18:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:19:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:20:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:21:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:22:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:23:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:24:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:25:06Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:26:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:27:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:28:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:29:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:30:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:31:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:32:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:33:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:34:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:35:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:36:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:37:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:38:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:39:07Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:40:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:41:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:42:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:43:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:44:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:45:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:46:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:47:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:48:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:49:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:50:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:51:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:52:08Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:53:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:54:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:55:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:56:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:57:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:58:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T05:59:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:00:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:01:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:02:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:03:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:04:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:05:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:06:09Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:07:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:08:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:09:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:10:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:11:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:12:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:13:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:14:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:15:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:16:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:17:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:18:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:19:10Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:20:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:21:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:22:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:23:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:24:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:25:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:26:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:27:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:28:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:29:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:30:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:31:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:32:11Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:33:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:34:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:35:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:36:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:37:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:38:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:39:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:40:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:41:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:42:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:43:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:44:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:45:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:46:12Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:47:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:48:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:49:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:50:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:51:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:52:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:53:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:54:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:55:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:56:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:57:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:58:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T06:59:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:00:13Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:01:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:02:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:03:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:04:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:05:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:06:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:07:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:08:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:09:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:10:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:11:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:12:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:13:14Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:14:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:15:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:16:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:17:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:18:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:19:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:20:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:21:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:22:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:23:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:24:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:25:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:26:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:27:15Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:28:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:29:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:30:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:31:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:32:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:33:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:34:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:35:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:36:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:37:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:38:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:39:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:40:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:41:16Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:42:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:43:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:44:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:45:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:46:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:47:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:48:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:49:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:50:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:51:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:52:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:53:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:54:17Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:55:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:56:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:57:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:58:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T07:59:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:00:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:01:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:02:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:03:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:04:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:05:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:06:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:07:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:08:18Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:09:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:10:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:11:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:12:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:13:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:14:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:15:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:16:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:17:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:18:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:19:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:20:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:21:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:22:19Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:23:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:24:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:25:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:26:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:27:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:28:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:29:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:30:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:31:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:32:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:33:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:34:20Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:35:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:36:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:37:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:38:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:39:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:40:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:41:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:42:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:43:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:44:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:45:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:46:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:47:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:48:21Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:49:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:50:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:51:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:52:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:53:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:54:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:55:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:56:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:57:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:58:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T08:59:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:00:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:01:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:02:22Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:03:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:04:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:05:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:06:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:07:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:08:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:09:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:10:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:11:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:12:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:13:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:14:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:15:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:16:23Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:17:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:18:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:19:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:20:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:21:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:22:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:23:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:24:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:25:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:26:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:27:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:28:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:29:24Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:30:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:31:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:32:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:33:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:34:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:35:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:36:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:37:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:38:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:39:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:40:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:41:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:42:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:43:25Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:44:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:45:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:46:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:47:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:48:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:49:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:50:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:51:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:52:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:53:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:54:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:55:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:56:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:57:26Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:58:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T09:59:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:00:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:01:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:02:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:03:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:04:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:05:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:06:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:07:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:08:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:09:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:10:27Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:11:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:12:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:13:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:14:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:15:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:16:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:17:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:18:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:19:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:20:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:21:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:22:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:23:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:24:28Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:25:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:26:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:27:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:28:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:29:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:30:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:31:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:32:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:33:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:34:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:35:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:36:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:37:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:38:29Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:39:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:40:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:41:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:42:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:43:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:44:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:45:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:46:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:47:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:48:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:49:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:50:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:51:30Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:52:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:53:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:54:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:55:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:56:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:57:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:58:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T10:59:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:00:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:01:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:02:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:03:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:04:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:05:31Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:06:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:07:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:08:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:09:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:10:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:11:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:12:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:13:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:14:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:15:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:16:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:17:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:18:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:19:32Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:20:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:21:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:22:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:23:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:24:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:25:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:26:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:27:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:28:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:29:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:30:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:31:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:32:33Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:33:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:34:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:35:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:36:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:37:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:38:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:39:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:40:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:41:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:42:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:43:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:44:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:45:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:46:34Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:47:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:48:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:49:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:50:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:51:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:52:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:53:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:54:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:55:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:56:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:57:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:58:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T11:59:35Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:00:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:01:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:02:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:03:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:04:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:05:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:06:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:07:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:08:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:09:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:10:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:11:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:12:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:13:36Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:14:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:15:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:16:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:17:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:18:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:19:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:20:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:21:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:22:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:23:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:24:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:25:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:26:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:27:37Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:28:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:29:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:30:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:31:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:32:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:33:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:34:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:35:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:36:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:37:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:38:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:39:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:40:38Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:41:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:42:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:43:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:44:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:45:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:46:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:47:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:48:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:49:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:50:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:51:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:52:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:53:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:54:39Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:55:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:56:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:57:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:58:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T12:59:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:00:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:01:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:02:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:03:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:04:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:05:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:06:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:07:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:08:40Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:09:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:10:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:11:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:12:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:13:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:14:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:15:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:16:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:17:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:18:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:19:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:20:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:21:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:22:41Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:23:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:24:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:25:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:26:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:27:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:28:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:29:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:30:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:31:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:32:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:33:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:34:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:35:42Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:36:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:37:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:38:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:39:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:40:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:41:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:42:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:43:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:44:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:45:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:46:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:47:43Z +[scheduler] active=68 busy_gpus=1 2026-08-22T13:48:43Z +[scheduler] active=51 busy_gpus=1 2026-08-22T13:49:43Z +[scheduler] active=51 busy_gpus=1 2026-08-22T13:50:44Z +[scheduler] active=51 busy_gpus=1 2026-08-22T13:51:44Z +[scheduler] active=51 busy_gpus=1 2026-08-22T13:52:44Z +[scheduler] active=51 busy_gpus=1 2026-08-22T13:53:44Z +[scheduler] active=35 busy_gpus=1 2026-08-22T13:54:44Z +[scheduler] active=34 busy_gpus=1 2026-08-22T13:55:44Z +[scheduler] active=34 busy_gpus=1 2026-08-22T13:56:44Z +[scheduler] active=34 busy_gpus=1 2026-08-22T13:57:44Z +[scheduler] active=34 busy_gpus=1 2026-08-22T13:58:44Z +[scheduler] active=34 busy_gpus=1 2026-08-22T13:59:44Z +[scheduler] active=34 busy_gpus=1 2026-08-22T14:00:44Z +[scheduler] active=34 busy_gpus=1 2026-08-22T14:01:44Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:02:44Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:03:44Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:04:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:05:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:06:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:07:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:08:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:09:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:10:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:11:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:12:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:13:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:14:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:15:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:16:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:17:45Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:18:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:19:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:20:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:21:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:22:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:23:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:24:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:25:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:26:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:27:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:28:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:29:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:30:46Z +[scheduler] active=17 busy_gpus=1 2026-08-22T14:31:46Z +[scheduler] active=0 busy_gpus=0 2026-08-22T14:32:47Z +[scheduler] condition met; launching resampled seed45 on all four GPUs 2026-08-22T14:32:47Z diff --git a/epfl_final/train_logs/ropectl_a1p0_a10040_ddp2/seed44.20260820T140639Z.log b/epfl_final/train_logs/ropectl_a1p0_a10040_ddp2/seed44.20260820T140639Z.log new file mode 100644 index 0000000000000000000000000000000000000000..9f32dbb5e45a00f3d52a017e2c05d29830f6849f --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_a10040_ddp2/seed44.20260820T140639Z.log @@ -0,0 +1,3038 @@ +W0820 14:06:49.821000 215 site-packages/torch/distributed/run.py:851] +W0820 14:06:49.821000 215 site-packages/torch/distributed/run.py:851] ***************************************** +W0820 14:06:49.821000 215 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0820 14:06:49.821000 215 site-packages/torch/distributed/run.py:851] ***************************************** +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-17000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-17000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 44, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed44", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control", + "ddp2-a100-40gb" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-17000 +Checkpoint activations: False +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-17000... +Preserving checkpoint LR state for exact resume: current=[0.0003451463250063146], schedule_base=[0.0005] +Resumed from checkpoint at step 17000 (epoch 0, micro_step 170000) +Resuming training from step 17000 +Resuming wandb run bajm6b15 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run bajm6b15 +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140814-bajm6b15 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed44 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/bajm6b15 +Starting training... +Validation at step 17000: Loss = 3.0154, Perplexity = 20.40 +wandb: WARNING Tried to log to step 17000 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 17001: dense (dense step interval) +Step 17010/44000 | Loss: 3.0132 | LR: 3.45e-04 | 47949.1ms/step | 0.4 samples/s +wandb: WARNING Tried to log to step 17010 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17020/44000 | Loss: 2.9933 | LR: 3.45e-04 | 2442.2ms/step | 8.2 samples/s +wandb: WARNING Tried to log to step 17020 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17030/44000 | Loss: 3.0145 | LR: 3.45e-04 | 2457.9ms/step | 8.1 samples/s +wandb: WARNING Tried to log to step 17030 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17040/44000 | Loss: 3.0099 | LR: 3.44e-04 | 2373.6ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17040 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17050/44000 | Loss: 3.0145 | LR: 3.44e-04 | 2329.5ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17050 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17060/44000 | Loss: 3.0033 | LR: 3.44e-04 | 2365.4ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17060 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17070/44000 | Loss: 3.0220 | LR: 3.44e-04 | 2334.7ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17070 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17080/44000 | Loss: 2.9430 | LR: 3.44e-04 | 2236.4ms/step | 8.9 samples/s +wandb: WARNING Tried to log to step 17080 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17090/44000 | Loss: 3.0100 | LR: 3.44e-04 | 2232.8ms/step | 9.0 samples/s +wandb: WARNING Tried to log to step 17090 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17100/44000 | Loss: 3.0354 | LR: 3.43e-04 | 2226.9ms/step | 9.0 samples/s +wandb: WARNING Tried to log to step 17100 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17110/44000 | Loss: 2.9991 | LR: 3.43e-04 | 2234.3ms/step | 9.0 samples/s +wandb: WARNING Tried to log to step 17110 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17120/44000 | Loss: 2.9437 | LR: 3.43e-04 | 2278.6ms/step | 8.8 samples/s +wandb: WARNING Tried to log to step 17120 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17130/44000 | Loss: 3.0106 | LR: 3.43e-04 | 2320.1ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17130 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17140/44000 | Loss: 2.9788 | LR: 3.43e-04 | 2282.0ms/step | 8.8 samples/s +wandb: WARNING Tried to log to step 17140 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17150/44000 | Loss: 3.0379 | LR: 3.43e-04 | 2242.8ms/step | 8.9 samples/s +wandb: WARNING Tried to log to step 17150 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17160/44000 | Loss: 2.9880 | LR: 3.42e-04 | 2262.7ms/step | 8.8 samples/s +wandb: WARNING Tried to log to step 17160 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17170/44000 | Loss: 3.0079 | LR: 3.42e-04 | 2254.5ms/step | 8.9 samples/s +wandb: WARNING Tried to log to step 17170 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17180/44000 | Loss: 2.9830 | LR: 3.42e-04 | 2267.0ms/step | 8.8 samples/s +wandb: WARNING Tried to log to step 17180 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17190/44000 | Loss: 3.0154 | LR: 3.42e-04 | 2307.3ms/step | 8.7 samples/s +wandb: WARNING Tried to log to step 17190 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17200/44000 | Loss: 3.0141 | LR: 3.42e-04 | 2285.5ms/step | 8.8 samples/s +wandb: WARNING Tried to log to step 17200 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17210/44000 | Loss: 3.0525 | LR: 3.42e-04 | 2357.7ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17210 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17220/44000 | Loss: 3.0044 | LR: 3.41e-04 | 2286.2ms/step | 8.7 samples/s +wandb: WARNING Tried to log to step 17220 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17230/44000 | Loss: 3.0443 | LR: 3.41e-04 | 2304.2ms/step | 8.7 samples/s +wandb: WARNING Tried to log to step 17230 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17240/44000 | Loss: 2.9949 | LR: 3.41e-04 | 2294.7ms/step | 8.7 samples/s +wandb: WARNING Tried to log to step 17240 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17250/44000 | Loss: 2.9877 | LR: 3.41e-04 | 2348.8ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17250 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17260/44000 | Loss: 3.0079 | LR: 3.41e-04 | 2286.2ms/step | 8.7 samples/s +wandb: WARNING Tried to log to step 17260 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17270/44000 | Loss: 3.0174 | LR: 3.41e-04 | 2315.9ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17270 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17280/44000 | Loss: 3.0221 | LR: 3.40e-04 | 2286.9ms/step | 8.7 samples/s +wandb: WARNING Tried to log to step 17280 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17290/44000 | Loss: 3.0018 | LR: 3.40e-04 | 2336.5ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17290 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17300/44000 | Loss: 2.9468 | LR: 3.40e-04 | 2460.7ms/step | 8.1 samples/s +wandb: WARNING Tried to log to step 17300 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17310/44000 | Loss: 3.0042 | LR: 3.40e-04 | 2402.3ms/step | 8.3 samples/s +wandb: WARNING Tried to log to step 17310 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17320/44000 | Loss: 3.0217 | LR: 3.40e-04 | 2443.3ms/step | 8.2 samples/s +wandb: WARNING Tried to log to step 17320 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17330/44000 | Loss: 3.0238 | LR: 3.40e-04 | 2449.0ms/step | 8.2 samples/s +wandb: WARNING Tried to log to step 17330 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17340/44000 | Loss: 3.0100 | LR: 3.39e-04 | 2388.7ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17340 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17350/44000 | Loss: 3.0131 | LR: 3.39e-04 | 2415.0ms/step | 8.3 samples/s +wandb: WARNING Tried to log to step 17350 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17360/44000 | Loss: 2.9869 | LR: 3.39e-04 | 2365.7ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17360 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17370/44000 | Loss: 3.0099 | LR: 3.39e-04 | 2382.2ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17370 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17380/44000 | Loss: 2.9890 | LR: 3.39e-04 | 2372.5ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17380 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17390/44000 | Loss: 3.0273 | LR: 3.39e-04 | 2437.3ms/step | 8.2 samples/s +wandb: WARNING Tried to log to step 17390 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17400/44000 | Loss: 3.0113 | LR: 3.38e-04 | 2433.2ms/step | 8.2 samples/s +wandb: WARNING Tried to log to step 17400 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17410/44000 | Loss: 2.9719 | LR: 3.38e-04 | 2396.5ms/step | 8.3 samples/s +wandb: WARNING Tried to log to step 17410 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17420/44000 | Loss: 2.9585 | LR: 3.38e-04 | 2396.4ms/step | 8.3 samples/s +wandb: WARNING Tried to log to step 17420 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17430/44000 | Loss: 2.9635 | LR: 3.38e-04 | 2430.5ms/step | 8.2 samples/s +wandb: WARNING Tried to log to step 17430 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17440/44000 | Loss: 3.0429 | LR: 3.38e-04 | 2374.5ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17440 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17450/44000 | Loss: 2.9795 | LR: 3.38e-04 | 2329.6ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17450 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17460/44000 | Loss: 2.9929 | LR: 3.37e-04 | 2366.1ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17460 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17470/44000 | Loss: 3.0356 | LR: 3.37e-04 | 2359.7ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17470 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17480/44000 | Loss: 2.9818 | LR: 3.37e-04 | 2373.6ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17480 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17490/44000 | Loss: 2.9860 | LR: 3.37e-04 | 2354.6ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17490 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17500/44000 | Loss: 3.0231 | LR: 3.37e-04 | 2387.4ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17500 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17510/44000 | Loss: 2.9789 | LR: 3.37e-04 | 2379.2ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17510 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17520/44000 | Loss: 3.0002 | LR: 3.36e-04 | 2363.0ms/step | 8.5 samples/s +Step 17530/44000 | Loss: 3.0097 | LR: 3.36e-04 | 2405.1ms/step | 8.3 samples/s +Step 17540/44000 | Loss: 2.9936 | LR: 3.36e-04 | 2368.4ms/step | 8.4 samples/s +Step 17550/44000 | Loss: 2.9838 | LR: 3.36e-04 | 2394.4ms/step | 8.4 samples/s +Step 17560/44000 | Loss: 2.9656 | LR: 3.36e-04 | 2425.8ms/step | 8.2 samples/s +Step 17570/44000 | Loss: 3.0001 | LR: 3.35e-04 | 2395.7ms/step | 8.3 samples/s +Step 17580/44000 | Loss: 3.0016 | LR: 3.35e-04 | 2332.3ms/step | 8.6 samples/s +Step 17590/44000 | Loss: 2.9861 | LR: 3.35e-04 | 2377.7ms/step | 8.4 samples/s +Step 17600/44000 | Loss: 3.0221 | LR: 3.35e-04 | 2356.2ms/step | 8.5 samples/s +Step 17610/44000 | Loss: 2.9889 | LR: 3.35e-04 | 2372.5ms/step | 8.4 samples/s +Step 17620/44000 | Loss: 3.0058 | LR: 3.35e-04 | 2433.7ms/step | 8.2 samples/s +Step 17630/44000 | Loss: 2.9957 | LR: 3.34e-04 | 2394.6ms/step | 8.4 samples/s +Step 17640/44000 | Loss: 2.9789 | LR: 3.34e-04 | 2386.2ms/step | 8.4 samples/s +Step 17650/44000 | Loss: 2.9793 | LR: 3.34e-04 | 2370.6ms/step | 8.4 samples/s +Step 17660/44000 | Loss: 3.0244 | LR: 3.34e-04 | 2360.9ms/step | 8.5 samples/s +Step 17670/44000 | Loss: 3.0268 | LR: 3.34e-04 | 2352.3ms/step | 8.5 samples/s +Step 17680/44000 | Loss: 2.9840 | LR: 3.34e-04 | 2309.8ms/step | 8.7 samples/s +Step 17690/44000 | Loss: 2.9711 | LR: 3.33e-04 | 2341.9ms/step | 8.5 samples/s +Step 17700/44000 | Loss: 2.9875 | LR: 3.33e-04 | 2393.8ms/step | 8.4 samples/s +Step 17710/44000 | Loss: 2.9712 | LR: 3.33e-04 | 2356.7ms/step | 8.5 samples/s +Step 17720/44000 | Loss: 2.9729 | LR: 3.33e-04 | 2373.1ms/step | 8.4 samples/s +Step 17730/44000 | Loss: 3.0134 | LR: 3.33e-04 | 2446.2ms/step | 8.2 samples/s +Step 17740/44000 | Loss: 3.0074 | LR: 3.33e-04 | 2493.5ms/step | 8.0 samples/s +Step 17750/44000 | Loss: 3.0057 | LR: 3.32e-04 | 2467.4ms/step | 8.1 samples/s +Step 17760/44000 | Loss: 2.9977 | LR: 3.32e-04 | 2477.7ms/step | 8.1 samples/s +Step 17770/44000 | Loss: 3.0045 | LR: 3.32e-04 | 2488.2ms/step | 8.0 samples/s +Step 17780/44000 | Loss: 3.0371 | LR: 3.32e-04 | 2388.3ms/step | 8.4 samples/s +Step 17790/44000 | Loss: 2.9664 | LR: 3.32e-04 | 2313.9ms/step | 8.6 samples/s +Step 17800/44000 | Loss: 3.0085 | LR: 3.32e-04 | 2328.7ms/step | 8.6 samples/s +Step 17810/44000 | Loss: 3.0283 | LR: 3.31e-04 | 2338.3ms/step | 8.6 samples/s +Step 17820/44000 | Loss: 2.9774 | LR: 3.31e-04 | 2368.8ms/step | 8.4 samples/s +Step 17830/44000 | Loss: 2.9978 | LR: 3.31e-04 | 2377.4ms/step | 8.4 samples/s +Step 17840/44000 | Loss: 3.0074 | LR: 3.31e-04 | 2383.2ms/step | 8.4 samples/s +Step 17850/44000 | Loss: 2.9917 | LR: 3.31e-04 | 2383.3ms/step | 8.4 samples/s +Step 17860/44000 | Loss: 2.9703 | LR: 3.31e-04 | 2366.6ms/step | 8.5 samples/s +Step 17870/44000 | Loss: 3.0014 | LR: 3.30e-04 | 2417.6ms/step | 8.3 samples/s +Step 17880/44000 | Loss: 2.9909 | LR: 3.30e-04 | 2394.6ms/step | 8.4 samples/s +Step 17890/44000 | Loss: 2.9845 | LR: 3.30e-04 | 2375.0ms/step | 8.4 samples/s +Step 17900/44000 | Loss: 2.9876 | LR: 3.30e-04 | 2398.0ms/step | 8.3 samples/s +Step 17910/44000 | Loss: 3.0224 | LR: 3.30e-04 | 2457.1ms/step | 8.1 samples/s +Step 17920/44000 | Loss: 3.0132 | LR: 3.29e-04 | 2383.5ms/step | 8.4 samples/s +Step 17930/44000 | Loss: 2.9996 | LR: 3.29e-04 | 2401.8ms/step | 8.3 samples/s +Step 17940/44000 | Loss: 3.0217 | LR: 3.29e-04 | 2383.5ms/step | 8.4 samples/s +Step 17950/44000 | Loss: 2.9685 | LR: 3.29e-04 | 2332.0ms/step | 8.6 samples/s +Step 17960/44000 | Loss: 3.0512 | LR: 3.29e-04 | 2384.4ms/step | 8.4 samples/s +Step 17970/44000 | Loss: 2.9853 | LR: 3.29e-04 | 2363.7ms/step | 8.5 samples/s +Step 17980/44000 | Loss: 3.0166 | LR: 3.28e-04 | 2369.4ms/step | 8.4 samples/s +Step 17990/44000 | Loss: 3.0295 | LR: 3.28e-04 | 2356.8ms/step | 8.5 samples/s +Step 18000/44000 | Loss: 3.0462 | LR: 3.28e-04 | 2383.1ms/step | 8.4 samples/s +Validation at step 18000: Loss = 3.0038, Perplexity = 20.16 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-18000 +Step 18010/44000 | Loss: 3.0193 | LR: 3.28e-04 | 4856.5ms/step | 4.1 samples/s +Step 18020/44000 | Loss: 3.0014 | LR: 3.28e-04 | 2160.1ms/step | 9.3 samples/s +Step 18030/44000 | Loss: 2.9693 | LR: 3.28e-04 | 2382.7ms/step | 8.4 samples/s +Step 18040/44000 | Loss: 3.0332 | LR: 3.27e-04 | 2379.5ms/step | 8.4 samples/s +Step 18050/44000 | Loss: 3.0053 | LR: 3.27e-04 | 2431.4ms/step | 8.2 samples/s +Step 18060/44000 | Loss: 2.9655 | LR: 3.27e-04 | 2381.5ms/step | 8.4 samples/s +Step 18070/44000 | Loss: 3.0158 | LR: 3.27e-04 | 2428.3ms/step | 8.2 samples/s +Step 18080/44000 | Loss: 3.0138 | LR: 3.27e-04 | 2443.3ms/step | 8.2 samples/s +Step 18090/44000 | Loss: 2.9786 | LR: 3.27e-04 | 2372.0ms/step | 8.4 samples/s +Step 18100/44000 | Loss: 3.0239 | LR: 3.26e-04 | 2399.6ms/step | 8.3 samples/s +Step 18110/44000 | Loss: 2.9654 | LR: 3.26e-04 | 2368.8ms/step | 8.4 samples/s +Step 18120/44000 | Loss: 3.0208 | LR: 3.26e-04 | 2320.5ms/step | 8.6 samples/s +Step 18130/44000 | Loss: 3.0214 | LR: 3.26e-04 | 2338.8ms/step | 8.6 samples/s +Step 18140/44000 | Loss: 3.0253 | LR: 3.26e-04 | 2325.1ms/step | 8.6 samples/s +Step 18150/44000 | Loss: 2.9846 | LR: 3.26e-04 | 2412.2ms/step | 8.3 samples/s +Step 18160/44000 | Loss: 3.0205 | LR: 3.25e-04 | 2399.4ms/step | 8.3 samples/s +Step 18170/44000 | Loss: 3.0231 | LR: 3.25e-04 | 2340.7ms/step | 8.5 samples/s +Step 18180/44000 | Loss: 3.0321 | LR: 3.25e-04 | 2343.4ms/step | 8.5 samples/s +Step 18190/44000 | Loss: 3.0283 | LR: 3.25e-04 | 2369.9ms/step | 8.4 samples/s +Step 18200/44000 | Loss: 3.0508 | LR: 3.25e-04 | 2422.8ms/step | 8.3 samples/s +Step 18210/44000 | Loss: 2.9973 | LR: 3.24e-04 | 2362.6ms/step | 8.5 samples/s +Step 18220/44000 | Loss: 2.9642 | LR: 3.24e-04 | 2362.4ms/step | 8.5 samples/s +Step 18230/44000 | Loss: 3.0085 | LR: 3.24e-04 | 2334.4ms/step | 8.6 samples/s +Step 18240/44000 | Loss: 2.9536 | LR: 3.24e-04 | 2384.2ms/step | 8.4 samples/s +Step 18250/44000 | Loss: 3.0123 | LR: 3.24e-04 | 2384.0ms/step | 8.4 samples/s +Step 18260/44000 | Loss: 2.9358 | LR: 3.24e-04 | 2412.5ms/step | 8.3 samples/s +Step 18270/44000 | Loss: 3.0002 | LR: 3.23e-04 | 2384.8ms/step | 8.4 samples/s +Step 18280/44000 | Loss: 3.0111 | LR: 3.23e-04 | 2394.1ms/step | 8.4 samples/s +Step 18290/44000 | Loss: 2.9839 | LR: 3.23e-04 | 2472.8ms/step | 8.1 samples/s +Step 18300/44000 | Loss: 2.9816 | LR: 3.23e-04 | 2609.6ms/step | 7.7 samples/s +Step 18310/44000 | Loss: 3.0112 | LR: 3.23e-04 | 2620.3ms/step | 7.6 samples/s +Step 18320/44000 | Loss: 2.9708 | LR: 3.23e-04 | 2641.2ms/step | 7.6 samples/s +Step 18330/44000 | Loss: 2.9854 | LR: 3.22e-04 | 2604.8ms/step | 7.7 samples/s +Step 18340/44000 | Loss: 3.0262 | LR: 3.22e-04 | 2508.9ms/step | 8.0 samples/s +Step 18350/44000 | Loss: 2.9915 | LR: 3.22e-04 | 2644.7ms/step | 7.6 samples/s +Step 18360/44000 | Loss: 2.9496 | LR: 3.22e-04 | 2537.0ms/step | 7.9 samples/s +Step 18370/44000 | Loss: 2.9677 | LR: 3.22e-04 | 2528.8ms/step | 7.9 samples/s +Step 18380/44000 | Loss: 3.0431 | LR: 3.22e-04 | 2468.0ms/step | 8.1 samples/s +Step 18390/44000 | Loss: 2.9657 | LR: 3.21e-04 | 2515.9ms/step | 7.9 samples/s +Step 18400/44000 | Loss: 2.9349 | LR: 3.21e-04 | 2546.7ms/step | 7.9 samples/s +Step 18410/44000 | Loss: 2.9846 | LR: 3.21e-04 | 2477.8ms/step | 8.1 samples/s +Step 18420/44000 | Loss: 3.0415 | LR: 3.21e-04 | 2509.3ms/step | 8.0 samples/s +Step 18430/44000 | Loss: 3.0291 | LR: 3.21e-04 | 2573.2ms/step | 7.8 samples/s +Step 18440/44000 | Loss: 3.0088 | LR: 3.20e-04 | 2688.6ms/step | 7.4 samples/s +Step 18450/44000 | Loss: 2.9628 | LR: 3.20e-04 | 2681.3ms/step | 7.5 samples/s +Step 18460/44000 | Loss: 2.9844 | LR: 3.20e-04 | 2605.9ms/step | 7.7 samples/s +Step 18470/44000 | Loss: 2.9659 | LR: 3.20e-04 | 2684.7ms/step | 7.4 samples/s +Step 18480/44000 | Loss: 3.0070 | LR: 3.20e-04 | 2571.0ms/step | 7.8 samples/s +Step 18490/44000 | Loss: 3.0280 | LR: 3.20e-04 | 2544.9ms/step | 7.9 samples/s +Step 18500/44000 | Loss: 3.0188 | LR: 3.19e-04 | 2491.3ms/step | 8.0 samples/s +Step 18510/44000 | Loss: 3.0218 | LR: 3.19e-04 | 2575.0ms/step | 7.8 samples/s +Step 18520/44000 | Loss: 2.9882 | LR: 3.19e-04 | 2521.5ms/step | 7.9 samples/s +Step 18530/44000 | Loss: 2.9764 | LR: 3.19e-04 | 2597.8ms/step | 7.7 samples/s +Step 18540/44000 | Loss: 2.9824 | LR: 3.19e-04 | 2626.4ms/step | 7.6 samples/s +Step 18550/44000 | Loss: 3.0132 | LR: 3.19e-04 | 2588.2ms/step | 7.7 samples/s +Step 18560/44000 | Loss: 3.0079 | LR: 3.18e-04 | 2587.5ms/step | 7.7 samples/s +Step 18570/44000 | Loss: 2.9618 | LR: 3.18e-04 | 2559.5ms/step | 7.8 samples/s +Step 18580/44000 | Loss: 2.9884 | LR: 3.18e-04 | 2547.7ms/step | 7.9 samples/s +Step 18590/44000 | Loss: 2.9777 | LR: 3.18e-04 | 2498.2ms/step | 8.0 samples/s +Step 18600/44000 | Loss: 3.0229 | LR: 3.18e-04 | 2556.1ms/step | 7.8 samples/s +Step 18610/44000 | Loss: 3.0187 | LR: 3.18e-04 | 2589.1ms/step | 7.7 samples/s +Step 18620/44000 | Loss: 2.9889 | LR: 3.17e-04 | 2697.3ms/step | 7.4 samples/s +Step 18630/44000 | Loss: 2.9728 | LR: 3.17e-04 | 2598.3ms/step | 7.7 samples/s +Step 18640/44000 | Loss: 2.9864 | LR: 3.17e-04 | 2538.5ms/step | 7.9 samples/s +Step 18650/44000 | Loss: 3.0030 | LR: 3.17e-04 | 2541.7ms/step | 7.9 samples/s +Step 18660/44000 | Loss: 2.9514 | LR: 3.17e-04 | 2508.2ms/step | 8.0 samples/s +Step 18670/44000 | Loss: 2.9767 | LR: 3.16e-04 | 2488.7ms/step | 8.0 samples/s +Step 18680/44000 | Loss: 3.0132 | LR: 3.16e-04 | 2533.1ms/step | 7.9 samples/s +Step 18690/44000 | Loss: 3.0008 | LR: 3.16e-04 | 2461.3ms/step | 8.1 samples/s +Step 18700/44000 | Loss: 2.9792 | LR: 3.16e-04 | 2498.1ms/step | 8.0 samples/s +Step 18710/44000 | Loss: 2.9755 | LR: 3.16e-04 | 2540.0ms/step | 7.9 samples/s +Step 18720/44000 | Loss: 2.9553 | LR: 3.16e-04 | 2601.9ms/step | 7.7 samples/s +Step 18730/44000 | Loss: 2.9990 | LR: 3.15e-04 | 2551.0ms/step | 7.8 samples/s +Step 18740/44000 | Loss: 2.9586 | LR: 3.15e-04 | 2557.7ms/step | 7.8 samples/s +Step 18750/44000 | Loss: 2.9810 | LR: 3.15e-04 | 2608.7ms/step | 7.7 samples/s +Step 18760/44000 | Loss: 2.9611 | LR: 3.15e-04 | 2496.8ms/step | 8.0 samples/s +Step 18770/44000 | Loss: 3.0173 | LR: 3.15e-04 | 2340.0ms/step | 8.5 samples/s +Step 18780/44000 | Loss: 2.9941 | LR: 3.15e-04 | 2331.5ms/step | 8.6 samples/s +Step 18790/44000 | Loss: 3.0056 | LR: 3.14e-04 | 2336.1ms/step | 8.6 samples/s +Step 18800/44000 | Loss: 2.9845 | LR: 3.14e-04 | 2519.7ms/step | 7.9 samples/s +Step 18810/44000 | Loss: 3.0484 | LR: 3.14e-04 | 2572.5ms/step | 7.8 samples/s +Step 18820/44000 | Loss: 2.9963 | LR: 3.14e-04 | 2572.3ms/step | 7.8 samples/s +Step 18830/44000 | Loss: 3.0189 | LR: 3.14e-04 | 2527.0ms/step | 7.9 samples/s +Step 18840/44000 | Loss: 3.0035 | LR: 3.13e-04 | 2488.0ms/step | 8.0 samples/s +Step 18850/44000 | Loss: 3.0194 | LR: 3.13e-04 | 2568.8ms/step | 7.8 samples/s +Step 18860/44000 | Loss: 2.9700 | LR: 3.13e-04 | 2607.1ms/step | 7.7 samples/s +Step 18870/44000 | Loss: 2.9786 | LR: 3.13e-04 | 2532.4ms/step | 7.9 samples/s +Step 18880/44000 | Loss: 3.0046 | LR: 3.13e-04 | 2538.8ms/step | 7.9 samples/s +Step 18890/44000 | Loss: 2.9795 | LR: 3.13e-04 | 2519.2ms/step | 7.9 samples/s +Step 18900/44000 | Loss: 2.9441 | LR: 3.12e-04 | 2573.9ms/step | 7.8 samples/s +Step 18910/44000 | Loss: 2.9920 | LR: 3.12e-04 | 2777.4ms/step | 7.2 samples/s +Step 18920/44000 | Loss: 2.9749 | LR: 3.12e-04 | 2767.9ms/step | 7.2 samples/s +Step 18930/44000 | Loss: 3.0128 | LR: 3.12e-04 | 2731.4ms/step | 7.3 samples/s +Step 18940/44000 | Loss: 2.9510 | LR: 3.12e-04 | 2599.6ms/step | 7.7 samples/s +Step 18950/44000 | Loss: 2.9609 | LR: 3.12e-04 | 2662.9ms/step | 7.5 samples/s +Step 18960/44000 | Loss: 2.9909 | LR: 3.11e-04 | 2608.1ms/step | 7.7 samples/s +Step 18970/44000 | Loss: 2.9824 | LR: 3.11e-04 | 2816.0ms/step | 7.1 samples/s +Step 18980/44000 | Loss: 3.0170 | LR: 3.11e-04 | 2674.7ms/step | 7.5 samples/s +Step 18990/44000 | Loss: 3.0136 | LR: 3.11e-04 | 2722.9ms/step | 7.3 samples/s +Step 19000/44000 | Loss: 2.9614 | LR: 3.11e-04 | 2680.1ms/step | 7.5 samples/s +Validation at step 19000: Loss = 2.9915, Perplexity = 19.92 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-19000 +Step 19010/44000 | Loss: 3.0122 | LR: 3.11e-04 | 5325.0ms/step | 3.8 samples/s +Step 19020/44000 | Loss: 2.9779 | LR: 3.10e-04 | 2526.0ms/step | 7.9 samples/s +Step 19030/44000 | Loss: 2.9472 | LR: 3.10e-04 | 2661.0ms/step | 7.5 samples/s +Step 19040/44000 | Loss: 2.9867 | LR: 3.10e-04 | 2581.0ms/step | 7.7 samples/s +Step 19050/44000 | Loss: 2.9608 | LR: 3.10e-04 | 2302.9ms/step | 8.7 samples/s +Step 19060/44000 | Loss: 2.9886 | LR: 3.10e-04 | 2550.4ms/step | 7.8 samples/s +Step 19070/44000 | Loss: 2.9563 | LR: 3.09e-04 | 2532.8ms/step | 7.9 samples/s +Step 19080/44000 | Loss: 2.9885 | LR: 3.09e-04 | 2588.9ms/step | 7.7 samples/s +Step 19090/44000 | Loss: 2.9416 | LR: 3.09e-04 | 2671.5ms/step | 7.5 samples/s +Step 19100/44000 | Loss: 2.9573 | LR: 3.09e-04 | 2589.7ms/step | 7.7 samples/s +Step 19110/44000 | Loss: 2.9395 | LR: 3.09e-04 | 2520.6ms/step | 7.9 samples/s +Step 19120/44000 | Loss: 2.9985 | LR: 3.09e-04 | 2499.5ms/step | 8.0 samples/s +Step 19130/44000 | Loss: 3.0212 | LR: 3.08e-04 | 2488.6ms/step | 8.0 samples/s +Step 19140/44000 | Loss: 2.9502 | LR: 3.08e-04 | 2519.9ms/step | 7.9 samples/s +Step 19150/44000 | Loss: 2.9427 | LR: 3.08e-04 | 2630.3ms/step | 7.6 samples/s +Step 19160/44000 | Loss: 2.9581 | LR: 3.08e-04 | 2643.8ms/step | 7.6 samples/s +Step 19170/44000 | Loss: 2.9454 | LR: 3.08e-04 | 2677.7ms/step | 7.5 samples/s +Step 19180/44000 | Loss: 2.9685 | LR: 3.08e-04 | 2601.5ms/step | 7.7 samples/s +Step 19190/44000 | Loss: 2.9839 | LR: 3.07e-04 | 2588.0ms/step | 7.7 samples/s +Step 19200/44000 | Loss: 2.9996 | LR: 3.07e-04 | 2598.7ms/step | 7.7 samples/s +Step 19210/44000 | Loss: 2.9573 | LR: 3.07e-04 | 2478.7ms/step | 8.1 samples/s +Step 19220/44000 | Loss: 2.9581 | LR: 3.07e-04 | 2615.1ms/step | 7.6 samples/s +Step 19230/44000 | Loss: 3.0003 | LR: 3.07e-04 | 2352.5ms/step | 8.5 samples/s +Step 19240/44000 | Loss: 2.9555 | LR: 3.06e-04 | 2363.4ms/step | 8.5 samples/s +Step 19250/44000 | Loss: 3.0094 | LR: 3.06e-04 | 2405.4ms/step | 8.3 samples/s +Step 19260/44000 | Loss: 3.0089 | LR: 3.06e-04 | 2532.8ms/step | 7.9 samples/s +Step 19270/44000 | Loss: 2.9866 | LR: 3.06e-04 | 2613.8ms/step | 7.7 samples/s +Step 19280/44000 | Loss: 2.9573 | LR: 3.06e-04 | 2662.1ms/step | 7.5 samples/s +Step 19290/44000 | Loss: 2.9558 | LR: 3.06e-04 | 2641.9ms/step | 7.6 samples/s +Step 19300/44000 | Loss: 3.0027 | LR: 3.05e-04 | 2629.0ms/step | 7.6 samples/s +Step 19310/44000 | Loss: 2.9616 | LR: 3.05e-04 | 2635.5ms/step | 7.6 samples/s +Step 19320/44000 | Loss: 2.9468 | LR: 3.05e-04 | 2544.6ms/step | 7.9 samples/s +Step 19330/44000 | Loss: 3.0071 | LR: 3.05e-04 | 2590.2ms/step | 7.7 samples/s +Step 19340/44000 | Loss: 2.9271 | LR: 3.05e-04 | 2606.0ms/step | 7.7 samples/s +Step 19350/44000 | Loss: 2.9928 | LR: 3.04e-04 | 2557.3ms/step | 7.8 samples/s +Step 19360/44000 | Loss: 2.9719 | LR: 3.04e-04 | 2565.1ms/step | 7.8 samples/s +Step 19370/44000 | Loss: 2.9417 | LR: 3.04e-04 | 2552.9ms/step | 7.8 samples/s +Step 19380/44000 | Loss: 2.9923 | LR: 3.04e-04 | 2584.2ms/step | 7.7 samples/s +Step 19390/44000 | Loss: 2.9904 | LR: 3.04e-04 | 2609.4ms/step | 7.7 samples/s +Step 19400/44000 | Loss: 2.9517 | LR: 3.04e-04 | 2586.0ms/step | 7.7 samples/s +Step 19410/44000 | Loss: 2.9590 | LR: 3.03e-04 | 2578.7ms/step | 7.8 samples/s +Step 19420/44000 | Loss: 2.9943 | LR: 3.03e-04 | 2537.2ms/step | 7.9 samples/s +Step 19430/44000 | Loss: 2.9909 | LR: 3.03e-04 | 2482.8ms/step | 8.1 samples/s +Step 19440/44000 | Loss: 3.0043 | LR: 3.03e-04 | 2671.2ms/step | 7.5 samples/s +Step 19450/44000 | Loss: 2.9750 | LR: 3.03e-04 | 2559.6ms/step | 7.8 samples/s +Step 19460/44000 | Loss: 3.0194 | LR: 3.03e-04 | 2632.4ms/step | 7.6 samples/s +Step 19470/44000 | Loss: 2.9873 | LR: 3.02e-04 | 2588.5ms/step | 7.7 samples/s +Step 19480/44000 | Loss: 2.9938 | LR: 3.02e-04 | 2563.6ms/step | 7.8 samples/s +Step 19490/44000 | Loss: 3.0041 | LR: 3.02e-04 | 2547.2ms/step | 7.9 samples/s +Step 19500/44000 | Loss: 2.9662 | LR: 3.02e-04 | 2584.0ms/step | 7.7 samples/s +Step 19510/44000 | Loss: 3.0492 | LR: 3.02e-04 | 2568.1ms/step | 7.8 samples/s +Step 19520/44000 | Loss: 2.9619 | LR: 3.01e-04 | 2590.0ms/step | 7.7 samples/s +Step 19530/44000 | Loss: 2.9580 | LR: 3.01e-04 | 2643.9ms/step | 7.6 samples/s +Step 19540/44000 | Loss: 2.9635 | LR: 3.01e-04 | 2622.6ms/step | 7.6 samples/s +Step 19550/44000 | Loss: 3.0232 | LR: 3.01e-04 | 2635.4ms/step | 7.6 samples/s +Step 19560/44000 | Loss: 3.0132 | LR: 3.01e-04 | 2673.0ms/step | 7.5 samples/s +Step 19570/44000 | Loss: 2.9480 | LR: 3.01e-04 | 2519.4ms/step | 7.9 samples/s +Step 19580/44000 | Loss: 2.9365 | LR: 3.00e-04 | 2590.4ms/step | 7.7 samples/s +Step 19590/44000 | Loss: 2.9881 | LR: 3.00e-04 | 2619.2ms/step | 7.6 samples/s +Step 19600/44000 | Loss: 2.9633 | LR: 3.00e-04 | 2530.8ms/step | 7.9 samples/s +Step 19610/44000 | Loss: 2.9348 | LR: 3.00e-04 | 2533.9ms/step | 7.9 samples/s +Step 19620/44000 | Loss: 3.0074 | LR: 3.00e-04 | 2588.6ms/step | 7.7 samples/s +Step 19630/44000 | Loss: 2.9621 | LR: 3.00e-04 | 2518.0ms/step | 7.9 samples/s +Step 19640/44000 | Loss: 2.9607 | LR: 2.99e-04 | 2692.3ms/step | 7.4 samples/s +Step 19650/44000 | Loss: 3.0013 | LR: 2.99e-04 | 2638.6ms/step | 7.6 samples/s +Step 19660/44000 | Loss: 2.9828 | LR: 2.99e-04 | 2707.3ms/step | 7.4 samples/s +Step 19670/44000 | Loss: 2.9682 | LR: 2.99e-04 | 2636.1ms/step | 7.6 samples/s +Step 19680/44000 | Loss: 3.0012 | LR: 2.99e-04 | 2634.9ms/step | 7.6 samples/s +Step 19690/44000 | Loss: 2.9356 | LR: 2.98e-04 | 2584.7ms/step | 7.7 samples/s +Step 19700/44000 | Loss: 2.9220 | LR: 2.98e-04 | 2312.2ms/step | 8.6 samples/s +Step 19710/44000 | Loss: 2.9700 | LR: 2.98e-04 | 2303.8ms/step | 8.7 samples/s +Step 19720/44000 | Loss: 2.9784 | LR: 2.98e-04 | 2380.4ms/step | 8.4 samples/s +Step 19730/44000 | Loss: 2.9971 | LR: 2.98e-04 | 2606.4ms/step | 7.7 samples/s +Step 19740/44000 | Loss: 2.9801 | LR: 2.98e-04 | 2576.8ms/step | 7.8 samples/s +Step 19750/44000 | Loss: 2.9682 | LR: 2.97e-04 | 2608.1ms/step | 7.7 samples/s +Step 19760/44000 | Loss: 3.0157 | LR: 2.97e-04 | 2605.5ms/step | 7.7 samples/s +Step 19770/44000 | Loss: 2.9630 | LR: 2.97e-04 | 2559.6ms/step | 7.8 samples/s +Step 19780/44000 | Loss: 2.9576 | LR: 2.97e-04 | 2614.7ms/step | 7.6 samples/s +Step 19790/44000 | Loss: 2.9345 | LR: 2.97e-04 | 2567.1ms/step | 7.8 samples/s +Step 19800/44000 | Loss: 2.9738 | LR: 2.96e-04 | 2590.7ms/step | 7.7 samples/s +Step 19810/44000 | Loss: 3.0180 | LR: 2.96e-04 | 2535.9ms/step | 7.9 samples/s +Step 19820/44000 | Loss: 2.9348 | LR: 2.96e-04 | 2682.9ms/step | 7.5 samples/s +Step 19830/44000 | Loss: 2.9816 | LR: 2.96e-04 | 2588.0ms/step | 7.7 samples/s +Step 19840/44000 | Loss: 2.9962 | LR: 2.96e-04 | 2625.8ms/step | 7.6 samples/s +Step 19850/44000 | Loss: 2.9497 | LR: 2.96e-04 | 2638.6ms/step | 7.6 samples/s +Step 19860/44000 | Loss: 2.9985 | LR: 2.95e-04 | 2522.5ms/step | 7.9 samples/s +Step 19870/44000 | Loss: 2.9649 | LR: 2.95e-04 | 2487.1ms/step | 8.0 samples/s +Step 19880/44000 | Loss: 2.9747 | LR: 2.95e-04 | 2556.9ms/step | 7.8 samples/s +Step 19890/44000 | Loss: 2.9997 | LR: 2.95e-04 | 2611.0ms/step | 7.7 samples/s +Step 19900/44000 | Loss: 2.9863 | LR: 2.95e-04 | 2602.1ms/step | 7.7 samples/s +Step 19910/44000 | Loss: 2.9483 | LR: 2.95e-04 | 2626.0ms/step | 7.6 samples/s +Step 19920/44000 | Loss: 2.9964 | LR: 2.94e-04 | 2595.7ms/step | 7.7 samples/s +Step 19930/44000 | Loss: 2.9863 | LR: 2.94e-04 | 2570.4ms/step | 7.8 samples/s +Step 19940/44000 | Loss: 2.9989 | LR: 2.94e-04 | 2568.5ms/step | 7.8 samples/s +Step 19950/44000 | Loss: 2.9840 | LR: 2.94e-04 | 2636.8ms/step | 7.6 samples/s +Step 19960/44000 | Loss: 2.9754 | LR: 2.94e-04 | 2546.6ms/step | 7.9 samples/s +Step 19970/44000 | Loss: 2.9429 | LR: 2.93e-04 | 2552.4ms/step | 7.8 samples/s +Step 19980/44000 | Loss: 2.9621 | LR: 2.93e-04 | 2528.0ms/step | 7.9 samples/s +Step 19990/44000 | Loss: 3.0133 | LR: 2.93e-04 | 2614.9ms/step | 7.6 samples/s +Step 20000/44000 | Loss: 2.9731 | LR: 2.93e-04 | 2578.9ms/step | 7.8 samples/s +Validation at step 20000: Loss = 2.9803, Perplexity = 19.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-20000 +Step 20010/44000 | Loss: 2.9683 | LR: 2.93e-04 | 5442.3ms/step | 3.7 samples/s +Step 20020/44000 | Loss: 2.9690 | LR: 2.93e-04 | 2686.1ms/step | 7.4 samples/s +Step 20030/44000 | Loss: 2.9804 | LR: 2.92e-04 | 2725.5ms/step | 7.3 samples/s +Step 20040/44000 | Loss: 2.9645 | LR: 2.92e-04 | 2647.8ms/step | 7.6 samples/s +Step 20050/44000 | Loss: 2.9574 | LR: 2.92e-04 | 2680.9ms/step | 7.5 samples/s +Step 20060/44000 | Loss: 2.9298 | LR: 2.92e-04 | 2671.1ms/step | 7.5 samples/s +Step 20070/44000 | Loss: 2.9926 | LR: 2.92e-04 | 2598.8ms/step | 7.7 samples/s +Step 20080/44000 | Loss: 2.9830 | LR: 2.91e-04 | 2303.1ms/step | 8.7 samples/s +Step 20090/44000 | Loss: 2.9230 | LR: 2.91e-04 | 2570.1ms/step | 7.8 samples/s +Step 20100/44000 | Loss: 2.9264 | LR: 2.91e-04 | 2597.9ms/step | 7.7 samples/s +Step 20110/44000 | Loss: 2.9319 | LR: 2.91e-04 | 2601.5ms/step | 7.7 samples/s +Step 20120/44000 | Loss: 2.9608 | LR: 2.91e-04 | 2549.4ms/step | 7.8 samples/s +Step 20130/44000 | Loss: 2.9753 | LR: 2.91e-04 | 2598.4ms/step | 7.7 samples/s +Step 20140/44000 | Loss: 2.9304 | LR: 2.90e-04 | 2638.3ms/step | 7.6 samples/s +Step 20150/44000 | Loss: 2.9895 | LR: 2.90e-04 | 2636.0ms/step | 7.6 samples/s +Step 20160/44000 | Loss: 2.9423 | LR: 2.90e-04 | 2510.3ms/step | 8.0 samples/s +Step 20170/44000 | Loss: 2.9283 | LR: 2.90e-04 | 2364.8ms/step | 8.5 samples/s +Step 20180/44000 | Loss: 2.9607 | LR: 2.90e-04 | 2382.9ms/step | 8.4 samples/s +Step 20190/44000 | Loss: 2.9822 | LR: 2.90e-04 | 2391.0ms/step | 8.4 samples/s +Step 20200/44000 | Loss: 2.8927 | LR: 2.89e-04 | 2603.5ms/step | 7.7 samples/s +Step 20210/44000 | Loss: 2.9612 | LR: 2.89e-04 | 2659.5ms/step | 7.5 samples/s +Step 20220/44000 | Loss: 2.9488 | LR: 2.89e-04 | 2616.1ms/step | 7.6 samples/s +Step 20230/44000 | Loss: 2.9458 | LR: 2.89e-04 | 2637.7ms/step | 7.6 samples/s +Step 20240/44000 | Loss: 3.0224 | LR: 2.89e-04 | 2694.3ms/step | 7.4 samples/s +Step 20250/44000 | Loss: 2.9644 | LR: 2.88e-04 | 2595.0ms/step | 7.7 samples/s +Step 20260/44000 | Loss: 2.9954 | LR: 2.88e-04 | 2603.6ms/step | 7.7 samples/s +Step 20270/44000 | Loss: 2.9576 | LR: 2.88e-04 | 2562.9ms/step | 7.8 samples/s +Step 20280/44000 | Loss: 2.9664 | LR: 2.88e-04 | 2482.2ms/step | 8.1 samples/s +Step 20290/44000 | Loss: 2.9895 | LR: 2.88e-04 | 2492.4ms/step | 8.0 samples/s +Step 20300/44000 | Loss: 2.9640 | LR: 2.88e-04 | 2517.9ms/step | 7.9 samples/s +Step 20310/44000 | Loss: 2.9811 | LR: 2.87e-04 | 2520.2ms/step | 7.9 samples/s +Step 20320/44000 | Loss: 2.9774 | LR: 2.87e-04 | 2507.4ms/step | 8.0 samples/s +Step 20330/44000 | Loss: 2.9419 | LR: 2.87e-04 | 2507.9ms/step | 8.0 samples/s +Step 20340/44000 | Loss: 2.9464 | LR: 2.87e-04 | 2562.6ms/step | 7.8 samples/s +Step 20350/44000 | Loss: 2.9516 | LR: 2.87e-04 | 2547.7ms/step | 7.9 samples/s +Step 20360/44000 | Loss: 2.9725 | LR: 2.86e-04 | 2512.8ms/step | 8.0 samples/s +Step 20370/44000 | Loss: 2.9786 | LR: 2.86e-04 | 2491.1ms/step | 8.0 samples/s +Step 20380/44000 | Loss: 2.9027 | LR: 2.86e-04 | 2474.4ms/step | 8.1 samples/s +Step 20390/44000 | Loss: 2.9653 | LR: 2.86e-04 | 2515.7ms/step | 8.0 samples/s +Step 20400/44000 | Loss: 3.0287 | LR: 2.86e-04 | 2498.5ms/step | 8.0 samples/s +Step 20410/44000 | Loss: 2.9965 | LR: 2.86e-04 | 2553.7ms/step | 7.8 samples/s +Step 20420/44000 | Loss: 2.9648 | LR: 2.85e-04 | 2506.0ms/step | 8.0 samples/s +Step 20430/44000 | Loss: 2.9299 | LR: 2.85e-04 | 2563.5ms/step | 7.8 samples/s +Step 20440/44000 | Loss: 2.9836 | LR: 2.85e-04 | 2448.3ms/step | 8.2 samples/s +Step 20450/44000 | Loss: 2.9716 | LR: 2.85e-04 | 2440.9ms/step | 8.2 samples/s +Step 20460/44000 | Loss: 2.9619 | LR: 2.85e-04 | 2459.3ms/step | 8.1 samples/s +Step 20470/44000 | Loss: 2.9912 | LR: 2.84e-04 | 2508.9ms/step | 8.0 samples/s +Step 20480/44000 | Loss: 2.9662 | LR: 2.84e-04 | 2531.9ms/step | 7.9 samples/s +Step 20490/44000 | Loss: 2.9335 | LR: 2.84e-04 | 2509.7ms/step | 8.0 samples/s +Step 20500/44000 | Loss: 2.9565 | LR: 2.84e-04 | 2594.7ms/step | 7.7 samples/s +Step 20510/44000 | Loss: 2.9713 | LR: 2.84e-04 | 2501.7ms/step | 8.0 samples/s +Step 20520/44000 | Loss: 2.9290 | LR: 2.84e-04 | 2476.9ms/step | 8.1 samples/s +Step 20530/44000 | Loss: 2.9784 | LR: 2.83e-04 | 2515.9ms/step | 7.9 samples/s +Step 20540/44000 | Loss: 2.9764 | LR: 2.83e-04 | 2541.3ms/step | 7.9 samples/s +Step 20550/44000 | Loss: 2.9687 | LR: 2.83e-04 | 2526.7ms/step | 7.9 samples/s +Step 20560/44000 | Loss: 2.9442 | LR: 2.83e-04 | 2468.5ms/step | 8.1 samples/s +Step 20570/44000 | Loss: 2.9646 | LR: 2.83e-04 | 2521.0ms/step | 7.9 samples/s +Step 20580/44000 | Loss: 2.9550 | LR: 2.83e-04 | 2564.8ms/step | 7.8 samples/s +Step 20590/44000 | Loss: 2.9598 | LR: 2.82e-04 | 2528.9ms/step | 7.9 samples/s +Step 20600/44000 | Loss: 2.9929 | LR: 2.82e-04 | 2530.5ms/step | 7.9 samples/s +Step 20610/44000 | Loss: 2.9811 | LR: 2.82e-04 | 2588.1ms/step | 7.7 samples/s +Step 20620/44000 | Loss: 2.9988 | LR: 2.82e-04 | 2634.9ms/step | 7.6 samples/s +Step 20630/44000 | Loss: 2.9532 | LR: 2.82e-04 | 2556.2ms/step | 7.8 samples/s +Step 20640/44000 | Loss: 2.9953 | LR: 2.81e-04 | 2552.5ms/step | 7.8 samples/s +Step 20650/44000 | Loss: 2.9960 | LR: 2.81e-04 | 2575.0ms/step | 7.8 samples/s +Step 20660/44000 | Loss: 2.9858 | LR: 2.81e-04 | 2434.7ms/step | 8.2 samples/s +Step 20670/44000 | Loss: 2.9589 | LR: 2.81e-04 | 2325.1ms/step | 8.6 samples/s +Step 20680/44000 | Loss: 2.9491 | LR: 2.81e-04 | 2372.3ms/step | 8.4 samples/s +Step 20690/44000 | Loss: 2.9792 | LR: 2.81e-04 | 2551.7ms/step | 7.8 samples/s +Step 20700/44000 | Loss: 2.9713 | LR: 2.80e-04 | 2621.5ms/step | 7.6 samples/s +Step 20710/44000 | Loss: 2.9628 | LR: 2.80e-04 | 2584.8ms/step | 7.7 samples/s +Step 20720/44000 | Loss: 2.9807 | LR: 2.80e-04 | 2577.2ms/step | 7.8 samples/s +Step 20730/44000 | Loss: 3.0081 | LR: 2.80e-04 | 2553.5ms/step | 7.8 samples/s +Step 20740/44000 | Loss: 2.9454 | LR: 2.80e-04 | 2567.7ms/step | 7.8 samples/s +Step 20750/44000 | Loss: 2.9293 | LR: 2.79e-04 | 2510.7ms/step | 8.0 samples/s +Step 20760/44000 | Loss: 2.9841 | LR: 2.79e-04 | 2498.0ms/step | 8.0 samples/s +Step 20770/44000 | Loss: 2.9393 | LR: 2.79e-04 | 2561.3ms/step | 7.8 samples/s +Step 20780/44000 | Loss: 3.0065 | LR: 2.79e-04 | 2594.7ms/step | 7.7 samples/s +Step 20790/44000 | Loss: 2.9470 | LR: 2.79e-04 | 2631.2ms/step | 7.6 samples/s +Step 20800/44000 | Loss: 2.9999 | LR: 2.79e-04 | 2601.3ms/step | 7.7 samples/s +Step 20810/44000 | Loss: 2.9620 | LR: 2.78e-04 | 2635.2ms/step | 7.6 samples/s +Step 20820/44000 | Loss: 2.9743 | LR: 2.78e-04 | 2601.0ms/step | 7.7 samples/s +Step 20830/44000 | Loss: 2.9583 | LR: 2.78e-04 | 2594.6ms/step | 7.7 samples/s +Step 20840/44000 | Loss: 2.9451 | LR: 2.78e-04 | 2586.5ms/step | 7.7 samples/s +Step 20850/44000 | Loss: 2.9318 | LR: 2.78e-04 | 2582.2ms/step | 7.7 samples/s +Step 20860/44000 | Loss: 2.9305 | LR: 2.77e-04 | 2581.5ms/step | 7.7 samples/s +Step 20870/44000 | Loss: 2.9954 | LR: 2.77e-04 | 2502.1ms/step | 8.0 samples/s +Step 20880/44000 | Loss: 2.9836 | LR: 2.77e-04 | 2588.7ms/step | 7.7 samples/s +Step 20890/44000 | Loss: 2.9363 | LR: 2.77e-04 | 2562.9ms/step | 7.8 samples/s +Step 20900/44000 | Loss: 2.9533 | LR: 2.77e-04 | 2594.8ms/step | 7.7 samples/s +Step 20910/44000 | Loss: 2.9569 | LR: 2.77e-04 | 2634.2ms/step | 7.6 samples/s +Step 20920/44000 | Loss: 2.9742 | LR: 2.76e-04 | 2578.5ms/step | 7.8 samples/s +Step 20930/44000 | Loss: 2.9938 | LR: 2.76e-04 | 2592.0ms/step | 7.7 samples/s +Step 20940/44000 | Loss: 2.9325 | LR: 2.76e-04 | 2554.1ms/step | 7.8 samples/s +Step 20950/44000 | Loss: 2.8880 | LR: 2.76e-04 | 2594.1ms/step | 7.7 samples/s +Step 20960/44000 | Loss: 2.9881 | LR: 2.76e-04 | 2609.8ms/step | 7.7 samples/s +Step 20970/44000 | Loss: 2.9817 | LR: 2.75e-04 | 2506.1ms/step | 8.0 samples/s +Step 20980/44000 | Loss: 2.9885 | LR: 2.75e-04 | 2577.0ms/step | 7.8 samples/s +Step 20990/44000 | Loss: 2.9351 | LR: 2.75e-04 | 2462.6ms/step | 8.1 samples/s +Step 21000/44000 | Loss: 2.9290 | LR: 2.75e-04 | 2325.4ms/step | 8.6 samples/s +Validation at step 21000: Loss = 2.9709, Perplexity = 19.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-21000 +Step 21010/44000 | Loss: 2.9830 | LR: 2.75e-04 | 4748.4ms/step | 4.2 samples/s +Step 21020/44000 | Loss: 2.9414 | LR: 2.75e-04 | 2281.5ms/step | 8.8 samples/s +Step 21030/44000 | Loss: 2.9608 | LR: 2.74e-04 | 2172.2ms/step | 9.2 samples/s +Step 21040/44000 | Loss: 2.9019 | LR: 2.74e-04 | 2171.6ms/step | 9.2 samples/s +Step 21050/44000 | Loss: 2.9849 | LR: 2.74e-04 | 2196.8ms/step | 9.1 samples/s +Step 21060/44000 | Loss: 2.9647 | LR: 2.74e-04 | 2158.7ms/step | 9.3 samples/s +Step 21070/44000 | Loss: 2.9312 | LR: 2.74e-04 | 2149.4ms/step | 9.3 samples/s +Step 21080/44000 | Loss: 2.9727 | LR: 2.73e-04 | 2136.4ms/step | 9.4 samples/s +Step 21090/44000 | Loss: 2.9762 | LR: 2.73e-04 | 2146.0ms/step | 9.3 samples/s +Step 21100/44000 | Loss: 2.9517 | LR: 2.73e-04 | 2165.4ms/step | 9.2 samples/s +Step 21110/44000 | Loss: 2.9276 | LR: 2.73e-04 | 2095.8ms/step | 9.5 samples/s +Step 21120/44000 | Loss: 2.9760 | LR: 2.73e-04 | 2039.4ms/step | 9.8 samples/s +Step 21130/44000 | Loss: 2.9791 | LR: 2.73e-04 | 2152.7ms/step | 9.3 samples/s +Step 21140/44000 | Loss: 2.9347 | LR: 2.72e-04 | 2150.6ms/step | 9.3 samples/s +Step 21150/44000 | Loss: 2.9568 | LR: 2.72e-04 | 2115.7ms/step | 9.5 samples/s +Step 21160/44000 | Loss: 2.9609 | LR: 2.72e-04 | 2178.6ms/step | 9.2 samples/s +Step 21170/44000 | Loss: 2.9506 | LR: 2.72e-04 | 2153.4ms/step | 9.3 samples/s +Step 21180/44000 | Loss: 2.9603 | LR: 2.72e-04 | 2180.9ms/step | 9.2 samples/s +Step 21190/44000 | Loss: 2.9532 | LR: 2.71e-04 | 2158.3ms/step | 9.3 samples/s +Step 21200/44000 | Loss: 3.0095 | LR: 2.71e-04 | 2162.6ms/step | 9.2 samples/s +Step 21210/44000 | Loss: 3.0133 | LR: 2.71e-04 | 2172.2ms/step | 9.2 samples/s +Step 21220/44000 | Loss: 2.9289 | LR: 2.71e-04 | 2152.9ms/step | 9.3 samples/s +Step 21230/44000 | Loss: 2.9294 | LR: 2.71e-04 | 2156.2ms/step | 9.3 samples/s +Step 21240/44000 | Loss: 2.9684 | LR: 2.71e-04 | 2172.7ms/step | 9.2 samples/s +Step 21250/44000 | Loss: 2.9715 | LR: 2.70e-04 | 2134.3ms/step | 9.4 samples/s +Step 21260/44000 | Loss: 2.9767 | LR: 2.70e-04 | 2152.3ms/step | 9.3 samples/s +Step 21270/44000 | Loss: 2.9805 | LR: 2.70e-04 | 2140.9ms/step | 9.3 samples/s +Step 21280/44000 | Loss: 2.9829 | LR: 2.70e-04 | 2152.8ms/step | 9.3 samples/s +Step 21290/44000 | Loss: 2.9498 | LR: 2.70e-04 | 2157.7ms/step | 9.3 samples/s +Step 21300/44000 | Loss: 2.9658 | LR: 2.70e-04 | 2121.5ms/step | 9.4 samples/s +Step 21310/44000 | Loss: 2.9614 | LR: 2.69e-04 | 2164.3ms/step | 9.2 samples/s +Step 21320/44000 | Loss: 2.9557 | LR: 2.69e-04 | 2147.9ms/step | 9.3 samples/s +Step 21330/44000 | Loss: 3.0207 | LR: 2.69e-04 | 2142.2ms/step | 9.3 samples/s +Step 21340/44000 | Loss: 2.9631 | LR: 2.69e-04 | 2150.8ms/step | 9.3 samples/s +Step 21350/44000 | Loss: 2.9656 | LR: 2.69e-04 | 2136.9ms/step | 9.4 samples/s +Step 21360/44000 | Loss: 2.9541 | LR: 2.68e-04 | 2159.9ms/step | 9.3 samples/s +Step 21370/44000 | Loss: 2.9043 | LR: 2.68e-04 | 2128.9ms/step | 9.4 samples/s +Step 21380/44000 | Loss: 2.9319 | LR: 2.68e-04 | 2159.5ms/step | 9.3 samples/s +Step 21390/44000 | Loss: 2.9527 | LR: 2.68e-04 | 2172.4ms/step | 9.2 samples/s +Step 21400/44000 | Loss: 2.8944 | LR: 2.68e-04 | 2160.1ms/step | 9.3 samples/s +Step 21410/44000 | Loss: 2.9517 | LR: 2.68e-04 | 2156.7ms/step | 9.3 samples/s +Step 21420/44000 | Loss: 2.9719 | LR: 2.67e-04 | 2137.3ms/step | 9.4 samples/s +Step 21430/44000 | Loss: 2.9340 | LR: 2.67e-04 | 2143.6ms/step | 9.3 samples/s +Step 21440/44000 | Loss: 2.9220 | LR: 2.67e-04 | 2144.7ms/step | 9.3 samples/s +Step 21450/44000 | Loss: 2.9828 | LR: 2.67e-04 | 2143.1ms/step | 9.3 samples/s +Step 21460/44000 | Loss: 2.9686 | LR: 2.67e-04 | 2144.7ms/step | 9.3 samples/s +Step 21470/44000 | Loss: 2.9254 | LR: 2.66e-04 | 2148.9ms/step | 9.3 samples/s +Step 21480/44000 | Loss: 2.9868 | LR: 2.66e-04 | 2154.9ms/step | 9.3 samples/s +Step 21490/44000 | Loss: 2.9564 | LR: 2.66e-04 | 2148.4ms/step | 9.3 samples/s +Step 21500/44000 | Loss: 3.0027 | LR: 2.66e-04 | 2150.0ms/step | 9.3 samples/s +Step 21510/44000 | Loss: 2.9615 | LR: 2.66e-04 | 2135.1ms/step | 9.4 samples/s +Step 21520/44000 | Loss: 2.9676 | LR: 2.66e-04 | 2152.3ms/step | 9.3 samples/s +Step 21530/44000 | Loss: 2.9421 | LR: 2.65e-04 | 2169.0ms/step | 9.2 samples/s +Step 21540/44000 | Loss: 2.9323 | LR: 2.65e-04 | 2119.3ms/step | 9.4 samples/s +Step 21550/44000 | Loss: 2.9531 | LR: 2.65e-04 | 2175.5ms/step | 9.2 samples/s +Step 21560/44000 | Loss: 2.9379 | LR: 2.65e-04 | 2147.2ms/step | 9.3 samples/s +Step 21570/44000 | Loss: 2.9576 | LR: 2.65e-04 | 2175.6ms/step | 9.2 samples/s +Step 21580/44000 | Loss: 2.9784 | LR: 2.64e-04 | 2109.8ms/step | 9.5 samples/s +Step 21590/44000 | Loss: 2.9333 | LR: 2.64e-04 | 2086.7ms/step | 9.6 samples/s +Step 21600/44000 | Loss: 2.9596 | LR: 2.64e-04 | 2123.9ms/step | 9.4 samples/s +Step 21610/44000 | Loss: 2.9443 | LR: 2.64e-04 | 2084.4ms/step | 9.6 samples/s +Step 21620/44000 | Loss: 2.9649 | LR: 2.64e-04 | 2093.1ms/step | 9.6 samples/s +Step 21630/44000 | Loss: 2.9216 | LR: 2.64e-04 | 2126.3ms/step | 9.4 samples/s +Step 21640/44000 | Loss: 2.9104 | LR: 2.63e-04 | 2091.8ms/step | 9.6 samples/s +Step 21650/44000 | Loss: 2.9783 | LR: 2.63e-04 | 2121.5ms/step | 9.4 samples/s +Step 21660/44000 | Loss: 2.9175 | LR: 2.63e-04 | 2101.0ms/step | 9.5 samples/s +Step 21670/44000 | Loss: 2.9769 | LR: 2.63e-04 | 2110.9ms/step | 9.5 samples/s +Step 21680/44000 | Loss: 2.9679 | LR: 2.63e-04 | 2110.9ms/step | 9.5 samples/s +Step 21690/44000 | Loss: 2.9495 | LR: 2.62e-04 | 2114.2ms/step | 9.5 samples/s +Step 21700/44000 | Loss: 2.9339 | LR: 2.62e-04 | 2108.7ms/step | 9.5 samples/s +Step 21710/44000 | Loss: 2.9577 | LR: 2.62e-04 | 2124.3ms/step | 9.4 samples/s +Step 21720/44000 | Loss: 2.9475 | LR: 2.62e-04 | 2095.4ms/step | 9.5 samples/s +Step 21730/44000 | Loss: 2.9306 | LR: 2.62e-04 | 2099.0ms/step | 9.5 samples/s +Step 21740/44000 | Loss: 2.9688 | LR: 2.62e-04 | 2124.2ms/step | 9.4 samples/s +Step 21750/44000 | Loss: 3.0183 | LR: 2.61e-04 | 2173.5ms/step | 9.2 samples/s +Step 21760/44000 | Loss: 2.9559 | LR: 2.61e-04 | 2153.0ms/step | 9.3 samples/s +Step 21770/44000 | Loss: 2.9566 | LR: 2.61e-04 | 2169.7ms/step | 9.2 samples/s +Step 21780/44000 | Loss: 2.9250 | LR: 2.61e-04 | 2157.5ms/step | 9.3 samples/s +Step 21790/44000 | Loss: 3.0031 | LR: 2.61e-04 | 2163.2ms/step | 9.2 samples/s +Step 21800/44000 | Loss: 2.9263 | LR: 2.60e-04 | 2188.8ms/step | 9.1 samples/s +Step 21810/44000 | Loss: 2.9234 | LR: 2.60e-04 | 2157.7ms/step | 9.3 samples/s +Step 21820/44000 | Loss: 2.9283 | LR: 2.60e-04 | 2171.2ms/step | 9.2 samples/s +Step 21830/44000 | Loss: 2.9823 | LR: 2.60e-04 | 2157.4ms/step | 9.3 samples/s +Step 21840/44000 | Loss: 2.9646 | LR: 2.60e-04 | 2184.1ms/step | 9.2 samples/s +Step 21850/44000 | Loss: 2.9451 | LR: 2.60e-04 | 2177.2ms/step | 9.2 samples/s +Step 21860/44000 | Loss: 2.9717 | LR: 2.59e-04 | 2163.7ms/step | 9.2 samples/s +Step 21870/44000 | Loss: 2.9712 | LR: 2.59e-04 | 2203.5ms/step | 9.1 samples/s +Step 21880/44000 | Loss: 2.9407 | LR: 2.59e-04 | 2188.5ms/step | 9.1 samples/s +Step 21890/44000 | Loss: 2.9573 | LR: 2.59e-04 | 2141.2ms/step | 9.3 samples/s +Step 21900/44000 | Loss: 2.9263 | LR: 2.59e-04 | 2164.0ms/step | 9.2 samples/s +Step 21910/44000 | Loss: 2.9808 | LR: 2.58e-04 | 2150.9ms/step | 9.3 samples/s +Step 21920/44000 | Loss: 2.9790 | LR: 2.58e-04 | 2152.6ms/step | 9.3 samples/s +Step 21930/44000 | Loss: 2.9766 | LR: 2.58e-04 | 2152.7ms/step | 9.3 samples/s +Step 21940/44000 | Loss: 2.9399 | LR: 2.58e-04 | 2151.2ms/step | 9.3 samples/s +Step 21950/44000 | Loss: 2.9520 | LR: 2.58e-04 | 2145.5ms/step | 9.3 samples/s +Step 21960/44000 | Loss: 2.9548 | LR: 2.58e-04 | 2161.4ms/step | 9.3 samples/s +Step 21970/44000 | Loss: 2.9763 | LR: 2.57e-04 | 2164.3ms/step | 9.2 samples/s +Step 21980/44000 | Loss: 2.9734 | LR: 2.57e-04 | 2147.6ms/step | 9.3 samples/s +Step 21990/44000 | Loss: 2.9536 | LR: 2.57e-04 | 2212.8ms/step | 9.0 samples/s +Step 22000/44000 | Loss: 2.9285 | LR: 2.57e-04 | 2184.5ms/step | 9.2 samples/s +Validation at step 22000: Loss = 2.9591, Perplexity = 19.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-22000 +Step 22010/44000 | Loss: 2.9392 | LR: 2.57e-04 | 4478.6ms/step | 4.5 samples/s +Step 22020/44000 | Loss: 2.9549 | LR: 2.56e-04 | 2147.3ms/step | 9.3 samples/s +Step 22030/44000 | Loss: 2.9555 | LR: 2.56e-04 | 2129.1ms/step | 9.4 samples/s +Step 22040/44000 | Loss: 2.9041 | LR: 2.56e-04 | 2142.6ms/step | 9.3 samples/s +Step 22050/44000 | Loss: 2.8819 | LR: 2.56e-04 | 2165.7ms/step | 9.2 samples/s +Step 22060/44000 | Loss: 2.9666 | LR: 2.56e-04 | 2147.5ms/step | 9.3 samples/s +Step 22070/44000 | Loss: 2.9523 | LR: 2.56e-04 | 2153.0ms/step | 9.3 samples/s +Step 22080/44000 | Loss: 2.9062 | LR: 2.55e-04 | 2149.9ms/step | 9.3 samples/s +Step 22090/44000 | Loss: 2.9985 | LR: 2.55e-04 | 2161.1ms/step | 9.3 samples/s +Step 22100/44000 | Loss: 2.9850 | LR: 2.55e-04 | 2135.5ms/step | 9.4 samples/s +Step 22110/44000 | Loss: 2.9070 | LR: 2.55e-04 | 2132.9ms/step | 9.4 samples/s +Step 22120/44000 | Loss: 2.9266 | LR: 2.55e-04 | 2053.4ms/step | 9.7 samples/s +Step 22130/44000 | Loss: 2.9567 | LR: 2.54e-04 | 2151.8ms/step | 9.3 samples/s +Step 22140/44000 | Loss: 2.9427 | LR: 2.54e-04 | 2155.0ms/step | 9.3 samples/s +Step 22150/44000 | Loss: 2.9802 | LR: 2.54e-04 | 2148.6ms/step | 9.3 samples/s +Step 22160/44000 | Loss: 3.0100 | LR: 2.54e-04 | 2171.0ms/step | 9.2 samples/s +Step 22170/44000 | Loss: 2.9404 | LR: 2.54e-04 | 2158.6ms/step | 9.3 samples/s +Step 22180/44000 | Loss: 2.9728 | LR: 2.54e-04 | 2156.3ms/step | 9.3 samples/s +Step 22190/44000 | Loss: 2.9739 | LR: 2.53e-04 | 2172.4ms/step | 9.2 samples/s +Step 22200/44000 | Loss: 2.8838 | LR: 2.53e-04 | 2163.8ms/step | 9.2 samples/s +Step 22210/44000 | Loss: 2.9932 | LR: 2.53e-04 | 2173.6ms/step | 9.2 samples/s +Step 22220/44000 | Loss: 2.8945 | LR: 2.53e-04 | 2155.5ms/step | 9.3 samples/s +Step 22230/44000 | Loss: 2.9478 | LR: 2.53e-04 | 2139.8ms/step | 9.3 samples/s +Step 22240/44000 | Loss: 2.9471 | LR: 2.52e-04 | 2171.6ms/step | 9.2 samples/s +Step 22250/44000 | Loss: 2.9391 | LR: 2.52e-04 | 2182.6ms/step | 9.2 samples/s +Step 22260/44000 | Loss: 2.9383 | LR: 2.52e-04 | 2161.2ms/step | 9.3 samples/s +Step 22270/44000 | Loss: 2.8954 | LR: 2.52e-04 | 2138.9ms/step | 9.4 samples/s +Step 22280/44000 | Loss: 2.9507 | LR: 2.52e-04 | 2157.0ms/step | 9.3 samples/s +Step 22290/44000 | Loss: 2.9643 | LR: 2.52e-04 | 2149.4ms/step | 9.3 samples/s +Step 22300/44000 | Loss: 2.9563 | LR: 2.51e-04 | 2156.2ms/step | 9.3 samples/s +Step 22310/44000 | Loss: 2.9135 | LR: 2.51e-04 | 2141.7ms/step | 9.3 samples/s +Step 22320/44000 | Loss: 2.8937 | LR: 2.51e-04 | 2172.5ms/step | 9.2 samples/s +Step 22330/44000 | Loss: 2.9550 | LR: 2.51e-04 | 2129.0ms/step | 9.4 samples/s +Step 22340/44000 | Loss: 2.9508 | LR: 2.51e-04 | 2116.7ms/step | 9.4 samples/s +Step 22350/44000 | Loss: 2.9493 | LR: 2.50e-04 | 2189.4ms/step | 9.1 samples/s +Step 22360/44000 | Loss: 2.9193 | LR: 2.50e-04 | 2147.2ms/step | 9.3 samples/s +Step 22370/44000 | Loss: 3.0008 | LR: 2.50e-04 | 2155.6ms/step | 9.3 samples/s +Step 22380/44000 | Loss: 2.9424 | LR: 2.50e-04 | 2128.6ms/step | 9.4 samples/s +Step 22390/44000 | Loss: 2.9716 | LR: 2.50e-04 | 2145.8ms/step | 9.3 samples/s +Step 22400/44000 | Loss: 2.9455 | LR: 2.50e-04 | 2132.4ms/step | 9.4 samples/s +Step 22410/44000 | Loss: 2.9672 | LR: 2.49e-04 | 2137.6ms/step | 9.4 samples/s +Step 22420/44000 | Loss: 2.9246 | LR: 2.49e-04 | 2153.2ms/step | 9.3 samples/s +Step 22430/44000 | Loss: 2.9141 | LR: 2.49e-04 | 2137.5ms/step | 9.4 samples/s +Step 22440/44000 | Loss: 2.9809 | LR: 2.49e-04 | 2133.1ms/step | 9.4 samples/s +Step 22450/44000 | Loss: 2.9413 | LR: 2.49e-04 | 2183.9ms/step | 9.2 samples/s +Step 22460/44000 | Loss: 2.9497 | LR: 2.48e-04 | 2180.0ms/step | 9.2 samples/s +Step 22470/44000 | Loss: 2.9631 | LR: 2.48e-04 | 2139.4ms/step | 9.3 samples/s +Step 22480/44000 | Loss: 2.9185 | LR: 2.48e-04 | 2144.2ms/step | 9.3 samples/s +Step 22490/44000 | Loss: 2.9220 | LR: 2.48e-04 | 2156.2ms/step | 9.3 samples/s +Step 22500/44000 | Loss: 2.9266 | LR: 2.48e-04 | 2140.0ms/step | 9.3 samples/s +Step 22510/44000 | Loss: 2.9514 | LR: 2.48e-04 | 2164.0ms/step | 9.2 samples/s +Step 22520/44000 | Loss: 2.9665 | LR: 2.47e-04 | 2146.3ms/step | 9.3 samples/s +Step 22530/44000 | Loss: 3.0431 | LR: 2.47e-04 | 2157.4ms/step | 9.3 samples/s +Step 22540/44000 | Loss: 2.9466 | LR: 2.47e-04 | 2158.5ms/step | 9.3 samples/s +Step 22550/44000 | Loss: 2.9628 | LR: 2.47e-04 | 2144.6ms/step | 9.3 samples/s +Step 22560/44000 | Loss: 2.9402 | LR: 2.47e-04 | 2152.4ms/step | 9.3 samples/s +Step 22570/44000 | Loss: 2.9854 | LR: 2.46e-04 | 2216.7ms/step | 9.0 samples/s +Step 22580/44000 | Loss: 2.9582 | LR: 2.46e-04 | 2187.7ms/step | 9.1 samples/s +Step 22590/44000 | Loss: 2.9382 | LR: 2.46e-04 | 2145.6ms/step | 9.3 samples/s +Step 22600/44000 | Loss: 2.9045 | LR: 2.46e-04 | 2173.9ms/step | 9.2 samples/s +Step 22610/44000 | Loss: 2.9582 | LR: 2.46e-04 | 2151.6ms/step | 9.3 samples/s +Step 22620/44000 | Loss: 2.9556 | LR: 2.46e-04 | 2142.1ms/step | 9.3 samples/s +Step 22630/44000 | Loss: 2.9088 | LR: 2.45e-04 | 2168.2ms/step | 9.2 samples/s +Step 22640/44000 | Loss: 2.9612 | LR: 2.45e-04 | 2187.1ms/step | 9.1 samples/s +Step 22650/44000 | Loss: 2.9682 | LR: 2.45e-04 | 2156.5ms/step | 9.3 samples/s +Step 22660/44000 | Loss: 2.9546 | LR: 2.45e-04 | 2175.3ms/step | 9.2 samples/s +Step 22670/44000 | Loss: 2.9342 | LR: 2.45e-04 | 2165.1ms/step | 9.2 samples/s +Step 22680/44000 | Loss: 2.9942 | LR: 2.44e-04 | 2159.8ms/step | 9.3 samples/s +Step 22690/44000 | Loss: 2.9020 | LR: 2.44e-04 | 2166.3ms/step | 9.2 samples/s +Step 22700/44000 | Loss: 2.9202 | LR: 2.44e-04 | 2160.2ms/step | 9.3 samples/s +Step 22710/44000 | Loss: 2.9329 | LR: 2.44e-04 | 2157.7ms/step | 9.3 samples/s +Step 22720/44000 | Loss: 2.9574 | LR: 2.44e-04 | 2152.3ms/step | 9.3 samples/s +Step 22730/44000 | Loss: 2.9100 | LR: 2.44e-04 | 2184.3ms/step | 9.2 samples/s +Step 22740/44000 | Loss: 2.9353 | LR: 2.43e-04 | 2156.6ms/step | 9.3 samples/s +Step 22750/44000 | Loss: 2.9038 | LR: 2.43e-04 | 2195.2ms/step | 9.1 samples/s +Step 22760/44000 | Loss: 2.9363 | LR: 2.43e-04 | 2169.6ms/step | 9.2 samples/s +Step 22770/44000 | Loss: 2.9267 | LR: 2.43e-04 | 2158.5ms/step | 9.3 samples/s +Step 22780/44000 | Loss: 2.9611 | LR: 2.43e-04 | 2163.2ms/step | 9.2 samples/s +Step 22790/44000 | Loss: 2.9512 | LR: 2.42e-04 | 2143.3ms/step | 9.3 samples/s +Step 22800/44000 | Loss: 2.9469 | LR: 2.42e-04 | 2160.1ms/step | 9.3 samples/s +Step 22810/44000 | Loss: 2.9714 | LR: 2.42e-04 | 2170.0ms/step | 9.2 samples/s +Step 22820/44000 | Loss: 2.9620 | LR: 2.42e-04 | 2137.6ms/step | 9.4 samples/s +Step 22830/44000 | Loss: 2.9232 | LR: 2.42e-04 | 2153.4ms/step | 9.3 samples/s +Step 22840/44000 | Loss: 2.9576 | LR: 2.42e-04 | 2182.0ms/step | 9.2 samples/s +Step 22850/44000 | Loss: 2.9293 | LR: 2.41e-04 | 2138.6ms/step | 9.4 samples/s +Step 22860/44000 | Loss: 2.9542 | LR: 2.41e-04 | 2127.6ms/step | 9.4 samples/s +Step 22870/44000 | Loss: 2.9389 | LR: 2.41e-04 | 2137.3ms/step | 9.4 samples/s +Step 22880/44000 | Loss: 2.9497 | LR: 2.41e-04 | 2133.5ms/step | 9.4 samples/s +Step 22890/44000 | Loss: 2.9315 | LR: 2.41e-04 | 2178.2ms/step | 9.2 samples/s +Step 22900/44000 | Loss: 2.9616 | LR: 2.40e-04 | 2141.6ms/step | 9.3 samples/s +Step 22910/44000 | Loss: 2.9291 | LR: 2.40e-04 | 2180.7ms/step | 9.2 samples/s +Step 22920/44000 | Loss: 2.9542 | LR: 2.40e-04 | 2149.2ms/step | 9.3 samples/s +Step 22930/44000 | Loss: 2.9048 | LR: 2.40e-04 | 2153.9ms/step | 9.3 samples/s +Step 22940/44000 | Loss: 2.9620 | LR: 2.40e-04 | 2151.0ms/step | 9.3 samples/s +Step 22950/44000 | Loss: 2.9153 | LR: 2.40e-04 | 2145.2ms/step | 9.3 samples/s +Step 22960/44000 | Loss: 2.9116 | LR: 2.39e-04 | 2167.6ms/step | 9.2 samples/s +Step 22970/44000 | Loss: 2.9622 | LR: 2.39e-04 | 2156.9ms/step | 9.3 samples/s +Step 22980/44000 | Loss: 2.9696 | LR: 2.39e-04 | 2139.7ms/step | 9.3 samples/s +Step 22990/44000 | Loss: 2.9271 | LR: 2.39e-04 | 2157.9ms/step | 9.3 samples/s +Step 23000/44000 | Loss: 2.9450 | LR: 2.39e-04 | 2168.1ms/step | 9.2 samples/s +Validation at step 23000: Loss = 2.9497, Perplexity = 19.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-23000 +Step 23010/44000 | Loss: 2.9275 | LR: 2.38e-04 | 4522.9ms/step | 4.4 samples/s +Step 23020/44000 | Loss: 2.9390 | LR: 2.38e-04 | 2139.7ms/step | 9.3 samples/s +Step 23030/44000 | Loss: 2.9572 | LR: 2.38e-04 | 2130.3ms/step | 9.4 samples/s +Step 23040/44000 | Loss: 2.9350 | LR: 2.38e-04 | 2163.8ms/step | 9.2 samples/s +Step 23050/44000 | Loss: 2.9180 | LR: 2.38e-04 | 2141.9ms/step | 9.3 samples/s +Step 23060/44000 | Loss: 2.9466 | LR: 2.38e-04 | 2130.2ms/step | 9.4 samples/s +Step 23070/44000 | Loss: 2.9110 | LR: 2.37e-04 | 2203.4ms/step | 9.1 samples/s +Step 23080/44000 | Loss: 2.9390 | LR: 2.37e-04 | 2177.6ms/step | 9.2 samples/s +Step 23090/44000 | Loss: 2.9239 | LR: 2.37e-04 | 2171.1ms/step | 9.2 samples/s +Step 23100/44000 | Loss: 2.9147 | LR: 2.37e-04 | 2145.9ms/step | 9.3 samples/s +Step 23110/44000 | Loss: 2.9276 | LR: 2.37e-04 | 2142.6ms/step | 9.3 samples/s +Step 23120/44000 | Loss: 3.0088 | LR: 2.36e-04 | 2056.2ms/step | 9.7 samples/s +Step 23130/44000 | Loss: 2.9484 | LR: 2.36e-04 | 2095.3ms/step | 9.5 samples/s +Step 23140/44000 | Loss: 2.9229 | LR: 2.36e-04 | 2168.3ms/step | 9.2 samples/s +Step 23150/44000 | Loss: 2.9469 | LR: 2.36e-04 | 2140.8ms/step | 9.3 samples/s +Step 23160/44000 | Loss: 2.9413 | LR: 2.36e-04 | 2130.9ms/step | 9.4 samples/s +Step 23170/44000 | Loss: 2.9387 | LR: 2.36e-04 | 2177.8ms/step | 9.2 samples/s +Step 23180/44000 | Loss: 2.9701 | LR: 2.35e-04 | 2175.7ms/step | 9.2 samples/s +Step 23190/44000 | Loss: 2.9859 | LR: 2.35e-04 | 2111.7ms/step | 9.5 samples/s +Step 23200/44000 | Loss: 2.9274 | LR: 2.35e-04 | 2187.5ms/step | 9.1 samples/s +Step 23210/44000 | Loss: 2.9414 | LR: 2.35e-04 | 2146.4ms/step | 9.3 samples/s +Step 23220/44000 | Loss: 2.9337 | LR: 2.35e-04 | 2201.9ms/step | 9.1 samples/s +Step 23230/44000 | Loss: 2.8961 | LR: 2.34e-04 | 2154.9ms/step | 9.3 samples/s +Step 23240/44000 | Loss: 2.9726 | LR: 2.34e-04 | 2142.3ms/step | 9.3 samples/s +Step 23250/44000 | Loss: 2.9226 | LR: 2.34e-04 | 2177.4ms/step | 9.2 samples/s +Step 23260/44000 | Loss: 3.0267 | LR: 2.34e-04 | 2190.9ms/step | 9.1 samples/s +Step 23270/44000 | Loss: 2.9006 | LR: 2.34e-04 | 2145.0ms/step | 9.3 samples/s +Step 23280/44000 | Loss: 2.9129 | LR: 2.34e-04 | 2160.2ms/step | 9.3 samples/s +Step 23290/44000 | Loss: 2.8867 | LR: 2.33e-04 | 2163.3ms/step | 9.2 samples/s +Step 23300/44000 | Loss: 2.9092 | LR: 2.33e-04 | 2217.2ms/step | 9.0 samples/s +Step 23310/44000 | Loss: 2.9415 | LR: 2.33e-04 | 2161.3ms/step | 9.3 samples/s +Step 23320/44000 | Loss: 2.9713 | LR: 2.33e-04 | 2164.9ms/step | 9.2 samples/s +Step 23330/44000 | Loss: 2.9605 | LR: 2.33e-04 | 2179.4ms/step | 9.2 samples/s +Step 23340/44000 | Loss: 2.9301 | LR: 2.32e-04 | 2207.1ms/step | 9.1 samples/s +Step 23350/44000 | Loss: 2.9116 | LR: 2.32e-04 | 2161.2ms/step | 9.3 samples/s +Step 23360/44000 | Loss: 2.9377 | LR: 2.32e-04 | 2156.1ms/step | 9.3 samples/s +Step 23370/44000 | Loss: 2.9575 | LR: 2.32e-04 | 2161.2ms/step | 9.3 samples/s +Step 23380/44000 | Loss: 2.9336 | LR: 2.32e-04 | 2181.8ms/step | 9.2 samples/s +Step 23390/44000 | Loss: 2.9618 | LR: 2.32e-04 | 2172.6ms/step | 9.2 samples/s +Step 23400/44000 | Loss: 2.9312 | LR: 2.31e-04 | 2165.8ms/step | 9.2 samples/s +Step 23410/44000 | Loss: 2.9351 | LR: 2.31e-04 | 2169.9ms/step | 9.2 samples/s +Step 23420/44000 | Loss: 2.9252 | LR: 2.31e-04 | 2197.3ms/step | 9.1 samples/s +Step 23430/44000 | Loss: 2.9410 | LR: 2.31e-04 | 2174.7ms/step | 9.2 samples/s +Step 23440/44000 | Loss: 2.8995 | LR: 2.31e-04 | 2147.6ms/step | 9.3 samples/s +Step 23450/44000 | Loss: 2.9564 | LR: 2.30e-04 | 2170.9ms/step | 9.2 samples/s +Step 23460/44000 | Loss: 2.9101 | LR: 2.30e-04 | 2203.0ms/step | 9.1 samples/s +Step 23470/44000 | Loss: 2.9197 | LR: 2.30e-04 | 2181.4ms/step | 9.2 samples/s +Step 23480/44000 | Loss: 2.9446 | LR: 2.30e-04 | 2143.7ms/step | 9.3 samples/s +Step 23490/44000 | Loss: 2.9241 | LR: 2.30e-04 | 2130.9ms/step | 9.4 samples/s +Step 23500/44000 | Loss: 2.8970 | LR: 2.30e-04 | 2171.1ms/step | 9.2 samples/s +Step 23510/44000 | Loss: 2.9389 | LR: 2.29e-04 | 2209.3ms/step | 9.1 samples/s +Step 23520/44000 | Loss: 2.9272 | LR: 2.29e-04 | 2191.1ms/step | 9.1 samples/s +Step 23530/44000 | Loss: 2.9452 | LR: 2.29e-04 | 2168.7ms/step | 9.2 samples/s +Step 23540/44000 | Loss: 2.9669 | LR: 2.29e-04 | 2169.0ms/step | 9.2 samples/s +Step 23550/44000 | Loss: 2.9256 | LR: 2.29e-04 | 2232.7ms/step | 9.0 samples/s +Step 23560/44000 | Loss: 2.9236 | LR: 2.29e-04 | 2132.1ms/step | 9.4 samples/s +Step 23570/44000 | Loss: 2.9131 | LR: 2.28e-04 | 2157.9ms/step | 9.3 samples/s +Step 23580/44000 | Loss: 2.9495 | LR: 2.28e-04 | 2159.8ms/step | 9.3 samples/s +Step 23590/44000 | Loss: 2.9505 | LR: 2.28e-04 | 2176.3ms/step | 9.2 samples/s +Step 23600/44000 | Loss: 2.9038 | LR: 2.28e-04 | 2216.4ms/step | 9.0 samples/s +Step 23610/44000 | Loss: 2.9398 | LR: 2.28e-04 | 2152.4ms/step | 9.3 samples/s +Step 23620/44000 | Loss: 2.9502 | LR: 2.27e-04 | 2154.8ms/step | 9.3 samples/s +Step 23630/44000 | Loss: 2.8860 | LR: 2.27e-04 | 2379.8ms/step | 8.4 samples/s +Step 23640/44000 | Loss: 2.9521 | LR: 2.27e-04 | 2205.8ms/step | 9.1 samples/s +Step 23650/44000 | Loss: 2.8904 | LR: 2.27e-04 | 2172.9ms/step | 9.2 samples/s +Step 23660/44000 | Loss: 2.9548 | LR: 2.27e-04 | 2160.8ms/step | 9.3 samples/s +Step 23670/44000 | Loss: 2.9419 | LR: 2.27e-04 | 2139.7ms/step | 9.3 samples/s +Step 23680/44000 | Loss: 2.9314 | LR: 2.26e-04 | 2148.6ms/step | 9.3 samples/s +Step 23690/44000 | Loss: 2.9499 | LR: 2.26e-04 | 2238.0ms/step | 8.9 samples/s +Step 23700/44000 | Loss: 2.9210 | LR: 2.26e-04 | 2334.4ms/step | 8.6 samples/s +Step 23710/44000 | Loss: 2.9180 | LR: 2.26e-04 | 2143.2ms/step | 9.3 samples/s +Step 23720/44000 | Loss: 2.9550 | LR: 2.26e-04 | 2146.3ms/step | 9.3 samples/s +Step 23730/44000 | Loss: 2.9634 | LR: 2.25e-04 | 2151.0ms/step | 9.3 samples/s +Step 23740/44000 | Loss: 2.9455 | LR: 2.25e-04 | 2172.4ms/step | 9.2 samples/s +Step 23750/44000 | Loss: 2.9461 | LR: 2.25e-04 | 2156.2ms/step | 9.3 samples/s +Step 23760/44000 | Loss: 2.9362 | LR: 2.25e-04 | 2194.1ms/step | 9.1 samples/s +Step 23770/44000 | Loss: 2.9502 | LR: 2.25e-04 | 2169.3ms/step | 9.2 samples/s +Step 23780/44000 | Loss: 2.8950 | LR: 2.25e-04 | 2142.7ms/step | 9.3 samples/s +Step 23790/44000 | Loss: 2.9127 | LR: 2.24e-04 | 2157.0ms/step | 9.3 samples/s +Step 23800/44000 | Loss: 2.9307 | LR: 2.24e-04 | 2142.6ms/step | 9.3 samples/s +Step 23810/44000 | Loss: 2.8996 | LR: 2.24e-04 | 2180.2ms/step | 9.2 samples/s +Step 23820/44000 | Loss: 2.9300 | LR: 2.24e-04 | 2190.7ms/step | 9.1 samples/s +Step 23830/44000 | Loss: 2.9308 | LR: 2.24e-04 | 2154.3ms/step | 9.3 samples/s +Step 23840/44000 | Loss: 2.8520 | LR: 2.23e-04 | 2151.1ms/step | 9.3 samples/s +Step 23850/44000 | Loss: 2.9414 | LR: 2.23e-04 | 2137.5ms/step | 9.4 samples/s +Step 23860/44000 | Loss: 2.9652 | LR: 2.23e-04 | 2130.5ms/step | 9.4 samples/s +Step 23870/44000 | Loss: 2.9311 | LR: 2.23e-04 | 2144.4ms/step | 9.3 samples/s +Step 23880/44000 | Loss: 2.9119 | LR: 2.23e-04 | 2147.7ms/step | 9.3 samples/s +Step 23890/44000 | Loss: 2.9108 | LR: 2.23e-04 | 2139.8ms/step | 9.3 samples/s +Step 23900/44000 | Loss: 2.9368 | LR: 2.22e-04 | 2158.9ms/step | 9.3 samples/s +Step 23910/44000 | Loss: 2.9403 | LR: 2.22e-04 | 2139.2ms/step | 9.3 samples/s +Step 23920/44000 | Loss: 2.9739 | LR: 2.22e-04 | 2128.2ms/step | 9.4 samples/s +Step 23930/44000 | Loss: 2.9793 | LR: 2.22e-04 | 2148.4ms/step | 9.3 samples/s +Step 23940/44000 | Loss: 2.9322 | LR: 2.22e-04 | 2139.1ms/step | 9.3 samples/s +Step 23950/44000 | Loss: 2.9587 | LR: 2.21e-04 | 2151.5ms/step | 9.3 samples/s +Step 23960/44000 | Loss: 2.9877 | LR: 2.21e-04 | 2117.1ms/step | 9.4 samples/s +Step 23970/44000 | Loss: 2.9640 | LR: 2.21e-04 | 2139.1ms/step | 9.3 samples/s +Step 23980/44000 | Loss: 2.9056 | LR: 2.21e-04 | 2157.2ms/step | 9.3 samples/s +Step 23990/44000 | Loss: 2.9446 | LR: 2.21e-04 | 2187.4ms/step | 9.1 samples/s +Step 24000/44000 | Loss: 2.9227 | LR: 2.21e-04 | 2166.0ms/step | 9.2 samples/s +Validation at step 24000: Loss = 2.9421, Perplexity = 18.96 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-24000 +Step 24010/44000 | Loss: 2.9562 | LR: 2.20e-04 | 4473.1ms/step | 4.5 samples/s +Step 24020/44000 | Loss: 2.9561 | LR: 2.20e-04 | 2149.4ms/step | 9.3 samples/s +Step 24030/44000 | Loss: 2.9363 | LR: 2.20e-04 | 2135.7ms/step | 9.4 samples/s +Step 24040/44000 | Loss: 2.9737 | LR: 2.20e-04 | 2143.9ms/step | 9.3 samples/s +Step 24050/44000 | Loss: 2.9081 | LR: 2.20e-04 | 2144.9ms/step | 9.3 samples/s +Step 24060/44000 | Loss: 2.9455 | LR: 2.19e-04 | 2150.3ms/step | 9.3 samples/s +Step 24070/44000 | Loss: 2.9186 | LR: 2.19e-04 | 2155.6ms/step | 9.3 samples/s +Step 24080/44000 | Loss: 2.9149 | LR: 2.19e-04 | 2161.2ms/step | 9.3 samples/s +Step 24090/44000 | Loss: 2.9444 | LR: 2.19e-04 | 2161.0ms/step | 9.3 samples/s +Step 24100/44000 | Loss: 2.9035 | LR: 2.19e-04 | 2146.3ms/step | 9.3 samples/s +Step 24110/44000 | Loss: 2.9679 | LR: 2.19e-04 | 2148.4ms/step | 9.3 samples/s +Step 24120/44000 | Loss: 2.9079 | LR: 2.18e-04 | 2125.7ms/step | 9.4 samples/s +Step 24130/44000 | Loss: 2.9752 | LR: 2.18e-04 | 2014.7ms/step | 9.9 samples/s +Step 24140/44000 | Loss: 2.9175 | LR: 2.18e-04 | 2134.8ms/step | 9.4 samples/s +Step 24150/44000 | Loss: 2.9420 | LR: 2.18e-04 | 2151.8ms/step | 9.3 samples/s +Step 24160/44000 | Loss: 2.9404 | LR: 2.18e-04 | 2154.4ms/step | 9.3 samples/s +Step 24170/44000 | Loss: 2.9044 | LR: 2.17e-04 | 2158.6ms/step | 9.3 samples/s +Step 24180/44000 | Loss: 2.9527 | LR: 2.17e-04 | 2143.7ms/step | 9.3 samples/s +Step 24190/44000 | Loss: 2.9904 | LR: 2.17e-04 | 2165.8ms/step | 9.2 samples/s +Step 24200/44000 | Loss: 2.9138 | LR: 2.17e-04 | 2158.9ms/step | 9.3 samples/s +Step 24210/44000 | Loss: 2.9766 | LR: 2.17e-04 | 2153.7ms/step | 9.3 samples/s +Step 24220/44000 | Loss: 2.9080 | LR: 2.17e-04 | 2128.7ms/step | 9.4 samples/s +Step 24230/44000 | Loss: 2.9032 | LR: 2.16e-04 | 2132.5ms/step | 9.4 samples/s +Step 24240/44000 | Loss: 2.9067 | LR: 2.16e-04 | 2169.5ms/step | 9.2 samples/s +Step 24250/44000 | Loss: 2.9335 | LR: 2.16e-04 | 2128.7ms/step | 9.4 samples/s +Step 24260/44000 | Loss: 2.9157 | LR: 2.16e-04 | 2149.3ms/step | 9.3 samples/s +Step 24270/44000 | Loss: 2.9528 | LR: 2.16e-04 | 2134.1ms/step | 9.4 samples/s +Step 24280/44000 | Loss: 2.9340 | LR: 2.16e-04 | 2152.6ms/step | 9.3 samples/s +Step 24290/44000 | Loss: 2.9241 | LR: 2.15e-04 | 2144.8ms/step | 9.3 samples/s +Step 24300/44000 | Loss: 2.9766 | LR: 2.15e-04 | 2148.9ms/step | 9.3 samples/s +Step 24310/44000 | Loss: 2.9298 | LR: 2.15e-04 | 2144.7ms/step | 9.3 samples/s +Step 24320/44000 | Loss: 2.8956 | LR: 2.15e-04 | 2147.2ms/step | 9.3 samples/s +Step 24330/44000 | Loss: 2.9312 | LR: 2.15e-04 | 2138.9ms/step | 9.4 samples/s +Step 24340/44000 | Loss: 2.9226 | LR: 2.14e-04 | 2147.7ms/step | 9.3 samples/s +Step 24350/44000 | Loss: 2.9023 | LR: 2.14e-04 | 2148.3ms/step | 9.3 samples/s +Step 24360/44000 | Loss: 2.9374 | LR: 2.14e-04 | 2156.9ms/step | 9.3 samples/s +Step 24370/44000 | Loss: 2.9204 | LR: 2.14e-04 | 2149.3ms/step | 9.3 samples/s +Step 24380/44000 | Loss: 2.9034 | LR: 2.14e-04 | 2143.7ms/step | 9.3 samples/s +Step 24390/44000 | Loss: 2.9135 | LR: 2.14e-04 | 2138.3ms/step | 9.4 samples/s +Step 24400/44000 | Loss: 2.9275 | LR: 2.13e-04 | 2145.7ms/step | 9.3 samples/s +Step 24410/44000 | Loss: 2.9606 | LR: 2.13e-04 | 2168.1ms/step | 9.2 samples/s +Step 24420/44000 | Loss: 2.9117 | LR: 2.13e-04 | 2153.9ms/step | 9.3 samples/s +Step 24430/44000 | Loss: 2.9524 | LR: 2.13e-04 | 2127.7ms/step | 9.4 samples/s +Step 24440/44000 | Loss: 2.9004 | LR: 2.13e-04 | 2159.7ms/step | 9.3 samples/s +Step 24450/44000 | Loss: 2.9326 | LR: 2.12e-04 | 2145.6ms/step | 9.3 samples/s +Step 24460/44000 | Loss: 2.9243 | LR: 2.12e-04 | 2153.8ms/step | 9.3 samples/s +Step 24470/44000 | Loss: 2.9543 | LR: 2.12e-04 | 2140.8ms/step | 9.3 samples/s +Step 24480/44000 | Loss: 2.9313 | LR: 2.12e-04 | 2138.2ms/step | 9.4 samples/s +Step 24490/44000 | Loss: 2.9415 | LR: 2.12e-04 | 2146.9ms/step | 9.3 samples/s +Step 24500/44000 | Loss: 2.9422 | LR: 2.12e-04 | 2134.6ms/step | 9.4 samples/s +Step 24510/44000 | Loss: 2.9473 | LR: 2.11e-04 | 2142.8ms/step | 9.3 samples/s +Step 24520/44000 | Loss: 2.9439 | LR: 2.11e-04 | 2144.4ms/step | 9.3 samples/s +Step 24530/44000 | Loss: 2.9040 | LR: 2.11e-04 | 2144.1ms/step | 9.3 samples/s +Step 24540/44000 | Loss: 2.9113 | LR: 2.11e-04 | 2164.9ms/step | 9.2 samples/s +Step 24550/44000 | Loss: 2.9554 | LR: 2.11e-04 | 2157.5ms/step | 9.3 samples/s +Step 24560/44000 | Loss: 2.9377 | LR: 2.10e-04 | 2147.0ms/step | 9.3 samples/s +Step 24570/44000 | Loss: 2.9233 | LR: 2.10e-04 | 2167.8ms/step | 9.2 samples/s +Step 24580/44000 | Loss: 2.9323 | LR: 2.10e-04 | 2148.9ms/step | 9.3 samples/s +Step 24590/44000 | Loss: 2.8727 | LR: 2.10e-04 | 2139.0ms/step | 9.4 samples/s +Step 24600/44000 | Loss: 2.9459 | LR: 2.10e-04 | 2139.4ms/step | 9.3 samples/s +Step 24610/44000 | Loss: 2.9350 | LR: 2.10e-04 | 2152.4ms/step | 9.3 samples/s +Step 24620/44000 | Loss: 2.9065 | LR: 2.09e-04 | 2160.1ms/step | 9.3 samples/s +Step 24630/44000 | Loss: 2.9149 | LR: 2.09e-04 | 2141.8ms/step | 9.3 samples/s +Step 24640/44000 | Loss: 2.9425 | LR: 2.09e-04 | 2156.8ms/step | 9.3 samples/s +Step 24650/44000 | Loss: 2.9076 | LR: 2.09e-04 | 2169.5ms/step | 9.2 samples/s +Step 24660/44000 | Loss: 2.9026 | LR: 2.09e-04 | 2180.7ms/step | 9.2 samples/s +Step 24670/44000 | Loss: 2.9294 | LR: 2.09e-04 | 2160.4ms/step | 9.3 samples/s +Step 24680/44000 | Loss: 2.8990 | LR: 2.08e-04 | 2137.0ms/step | 9.4 samples/s +Step 24690/44000 | Loss: 2.9410 | LR: 2.08e-04 | 2139.5ms/step | 9.3 samples/s +Step 24700/44000 | Loss: 2.9357 | LR: 2.08e-04 | 2118.0ms/step | 9.4 samples/s +Step 24710/44000 | Loss: 2.9194 | LR: 2.08e-04 | 2143.2ms/step | 9.3 samples/s +Step 24720/44000 | Loss: 2.9380 | LR: 2.08e-04 | 2138.7ms/step | 9.4 samples/s +Step 24730/44000 | Loss: 2.9254 | LR: 2.07e-04 | 2135.2ms/step | 9.4 samples/s +Step 24740/44000 | Loss: 2.8992 | LR: 2.07e-04 | 2136.8ms/step | 9.4 samples/s +Step 24750/44000 | Loss: 2.9140 | LR: 2.07e-04 | 2126.7ms/step | 9.4 samples/s +Step 24760/44000 | Loss: 2.9033 | LR: 2.07e-04 | 2120.1ms/step | 9.4 samples/s +Step 24770/44000 | Loss: 2.9222 | LR: 2.07e-04 | 2147.4ms/step | 9.3 samples/s +Step 24780/44000 | Loss: 2.9257 | LR: 2.07e-04 | 2146.9ms/step | 9.3 samples/s +Step 24790/44000 | Loss: 2.8898 | LR: 2.06e-04 | 2151.1ms/step | 9.3 samples/s +Step 24800/44000 | Loss: 2.9692 | LR: 2.06e-04 | 2149.9ms/step | 9.3 samples/s +Step 24810/44000 | Loss: 2.8787 | LR: 2.06e-04 | 2160.3ms/step | 9.3 samples/s +Step 24820/44000 | Loss: 2.8787 | LR: 2.06e-04 | 2159.6ms/step | 9.3 samples/s +Step 24830/44000 | Loss: 2.9517 | LR: 2.06e-04 | 2161.5ms/step | 9.3 samples/s +Step 24840/44000 | Loss: 2.9495 | LR: 2.05e-04 | 2140.4ms/step | 9.3 samples/s +Step 24850/44000 | Loss: 2.9068 | LR: 2.05e-04 | 2161.7ms/step | 9.3 samples/s +Step 24860/44000 | Loss: 2.9623 | LR: 2.05e-04 | 2156.3ms/step | 9.3 samples/s +Step 24870/44000 | Loss: 2.9239 | LR: 2.05e-04 | 2168.8ms/step | 9.2 samples/s +Step 24880/44000 | Loss: 2.9380 | LR: 2.05e-04 | 2153.9ms/step | 9.3 samples/s +Step 24890/44000 | Loss: 2.9600 | LR: 2.05e-04 | 2148.7ms/step | 9.3 samples/s +Step 24900/44000 | Loss: 2.9417 | LR: 2.04e-04 | 2150.0ms/step | 9.3 samples/s +Step 24910/44000 | Loss: 2.9122 | LR: 2.04e-04 | 2161.0ms/step | 9.3 samples/s +Step 24920/44000 | Loss: 2.9241 | LR: 2.04e-04 | 2138.6ms/step | 9.4 samples/s +Step 24930/44000 | Loss: 2.9601 | LR: 2.04e-04 | 2163.6ms/step | 9.2 samples/s +Step 24940/44000 | Loss: 2.9044 | LR: 2.04e-04 | 2145.3ms/step | 9.3 samples/s +Step 24950/44000 | Loss: 2.9021 | LR: 2.04e-04 | 2159.7ms/step | 9.3 samples/s +Step 24960/44000 | Loss: 2.9011 | LR: 2.03e-04 | 2170.4ms/step | 9.2 samples/s +Step 24970/44000 | Loss: 2.9382 | LR: 2.03e-04 | 2159.2ms/step | 9.3 samples/s +Step 24980/44000 | Loss: 2.9226 | LR: 2.03e-04 | 2151.8ms/step | 9.3 samples/s +Step 24990/44000 | Loss: 2.9037 | LR: 2.03e-04 | 2171.9ms/step | 9.2 samples/s +Step 25000/44000 | Loss: 2.9402 | LR: 2.03e-04 | 2135.3ms/step | 9.4 samples/s +Validation at step 25000: Loss = 2.9337, Perplexity = 18.80 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-25000 +Step 25010/44000 | Loss: 2.9440 | LR: 2.02e-04 | 4498.3ms/step | 4.4 samples/s +Step 25020/44000 | Loss: 2.9517 | LR: 2.02e-04 | 2164.3ms/step | 9.2 samples/s +Step 25030/44000 | Loss: 2.9453 | LR: 2.02e-04 | 2165.1ms/step | 9.2 samples/s +Step 25040/44000 | Loss: 2.8953 | LR: 2.02e-04 | 2164.0ms/step | 9.2 samples/s +Step 25050/44000 | Loss: 2.9294 | LR: 2.02e-04 | 2152.0ms/step | 9.3 samples/s +Step 25060/44000 | Loss: 2.9282 | LR: 2.02e-04 | 2154.2ms/step | 9.3 samples/s +Step 25070/44000 | Loss: 2.9522 | LR: 2.01e-04 | 2147.5ms/step | 9.3 samples/s +Step 25080/44000 | Loss: 2.9275 | LR: 2.01e-04 | 2141.5ms/step | 9.3 samples/s +Step 25090/44000 | Loss: 2.9199 | LR: 2.01e-04 | 2158.2ms/step | 9.3 samples/s +Step 25100/44000 | Loss: 2.9101 | LR: 2.01e-04 | 2174.4ms/step | 9.2 samples/s +Step 25110/44000 | Loss: 2.9066 | LR: 2.01e-04 | 2134.4ms/step | 9.4 samples/s +Step 25120/44000 | Loss: 2.9461 | LR: 2.00e-04 | 2140.0ms/step | 9.3 samples/s +Step 25130/44000 | Loss: 2.8971 | LR: 2.00e-04 | 2019.9ms/step | 9.9 samples/s +Step 25140/44000 | Loss: 2.9516 | LR: 2.00e-04 | 2171.5ms/step | 9.2 samples/s +Step 25150/44000 | Loss: 2.9089 | LR: 2.00e-04 | 2151.0ms/step | 9.3 samples/s +Step 25160/44000 | Loss: 2.9526 | LR: 2.00e-04 | 2177.7ms/step | 9.2 samples/s +Step 25170/44000 | Loss: 2.9234 | LR: 2.00e-04 | 2165.7ms/step | 9.2 samples/s +Step 25180/44000 | Loss: 2.9143 | LR: 1.99e-04 | 2156.5ms/step | 9.3 samples/s +Step 25190/44000 | Loss: 2.9403 | LR: 1.99e-04 | 2134.8ms/step | 9.4 samples/s +Step 25200/44000 | Loss: 2.9143 | LR: 1.99e-04 | 2146.7ms/step | 9.3 samples/s +Step 25210/44000 | Loss: 2.8849 | LR: 1.99e-04 | 2137.6ms/step | 9.4 samples/s +Step 25220/44000 | Loss: 2.9161 | LR: 1.99e-04 | 2150.9ms/step | 9.3 samples/s +Step 25230/44000 | Loss: 2.9284 | LR: 1.99e-04 | 2151.4ms/step | 9.3 samples/s +Step 25240/44000 | Loss: 2.9594 | LR: 1.98e-04 | 2150.4ms/step | 9.3 samples/s +Step 25250/44000 | Loss: 2.9450 | LR: 1.98e-04 | 2166.7ms/step | 9.2 samples/s +Step 25260/44000 | Loss: 2.8971 | LR: 1.98e-04 | 2194.0ms/step | 9.1 samples/s +Step 25270/44000 | Loss: 2.9425 | LR: 1.98e-04 | 2140.6ms/step | 9.3 samples/s +Step 25280/44000 | Loss: 2.9026 | LR: 1.98e-04 | 2175.5ms/step | 9.2 samples/s +Step 25290/44000 | Loss: 2.9425 | LR: 1.97e-04 | 2140.0ms/step | 9.3 samples/s +Step 25300/44000 | Loss: 2.9204 | LR: 1.97e-04 | 2132.9ms/step | 9.4 samples/s +Step 25310/44000 | Loss: 2.9263 | LR: 1.97e-04 | 2172.9ms/step | 9.2 samples/s +Step 25320/44000 | Loss: 2.9198 | LR: 1.97e-04 | 2140.6ms/step | 9.3 samples/s +Step 25330/44000 | Loss: 2.9168 | LR: 1.97e-04 | 2146.1ms/step | 9.3 samples/s +Step 25340/44000 | Loss: 2.9236 | LR: 1.97e-04 | 2175.5ms/step | 9.2 samples/s +Step 25350/44000 | Loss: 2.9565 | LR: 1.96e-04 | 2205.6ms/step | 9.1 samples/s +Step 25360/44000 | Loss: 2.9028 | LR: 1.96e-04 | 2150.1ms/step | 9.3 samples/s +Step 25370/44000 | Loss: 2.9057 | LR: 1.96e-04 | 2158.5ms/step | 9.3 samples/s +Step 25380/44000 | Loss: 2.8991 | LR: 1.96e-04 | 2142.6ms/step | 9.3 samples/s +Step 25390/44000 | Loss: 2.9620 | LR: 1.96e-04 | 2177.3ms/step | 9.2 samples/s +Step 25400/44000 | Loss: 2.9570 | LR: 1.96e-04 | 2208.3ms/step | 9.1 samples/s +Step 25410/44000 | Loss: 2.9109 | LR: 1.95e-04 | 2148.2ms/step | 9.3 samples/s +Step 25420/44000 | Loss: 2.9478 | LR: 1.95e-04 | 2163.7ms/step | 9.2 samples/s +Step 25430/44000 | Loss: 2.8934 | LR: 1.95e-04 | 2151.2ms/step | 9.3 samples/s +Step 25440/44000 | Loss: 2.9109 | LR: 1.95e-04 | 2168.7ms/step | 9.2 samples/s +Step 25450/44000 | Loss: 2.9087 | LR: 1.95e-04 | 2148.8ms/step | 9.3 samples/s +Step 25460/44000 | Loss: 2.9117 | LR: 1.94e-04 | 2195.0ms/step | 9.1 samples/s +Step 25470/44000 | Loss: 2.8965 | LR: 1.94e-04 | 2199.0ms/step | 9.1 samples/s +Step 25480/44000 | Loss: 2.9180 | LR: 1.94e-04 | 2138.3ms/step | 9.4 samples/s +Step 25490/44000 | Loss: 2.9262 | LR: 1.94e-04 | 2126.4ms/step | 9.4 samples/s +Step 25500/44000 | Loss: 2.8776 | LR: 1.94e-04 | 2161.9ms/step | 9.3 samples/s +Step 25510/44000 | Loss: 2.9343 | LR: 1.94e-04 | 2137.7ms/step | 9.4 samples/s +Step 25520/44000 | Loss: 2.9382 | LR: 1.93e-04 | 2170.3ms/step | 9.2 samples/s +Step 25530/44000 | Loss: 2.9312 | LR: 1.93e-04 | 2159.8ms/step | 9.3 samples/s +Step 25540/44000 | Loss: 2.9187 | LR: 1.93e-04 | 2153.1ms/step | 9.3 samples/s +Step 25550/44000 | Loss: 2.9139 | LR: 1.93e-04 | 2146.1ms/step | 9.3 samples/s +Step 25560/44000 | Loss: 2.8895 | LR: 1.93e-04 | 2154.4ms/step | 9.3 samples/s +Step 25570/44000 | Loss: 2.8941 | LR: 1.92e-04 | 2159.6ms/step | 9.3 samples/s +Step 25580/44000 | Loss: 2.9098 | LR: 1.92e-04 | 2143.1ms/step | 9.3 samples/s +Step 25590/44000 | Loss: 2.9269 | LR: 1.92e-04 | 2137.2ms/step | 9.4 samples/s +Step 25600/44000 | Loss: 2.9169 | LR: 1.92e-04 | 2175.0ms/step | 9.2 samples/s +Step 25610/44000 | Loss: 2.9072 | LR: 1.92e-04 | 2144.0ms/step | 9.3 samples/s +Step 25620/44000 | Loss: 2.9290 | LR: 1.92e-04 | 2153.8ms/step | 9.3 samples/s +Step 25630/44000 | Loss: 2.9219 | LR: 1.91e-04 | 2189.7ms/step | 9.1 samples/s +Step 25640/44000 | Loss: 2.9321 | LR: 1.91e-04 | 2277.9ms/step | 8.8 samples/s +Step 25650/44000 | Loss: 2.9039 | LR: 1.91e-04 | 2172.1ms/step | 9.2 samples/s +Step 25660/44000 | Loss: 2.9224 | LR: 1.91e-04 | 2170.8ms/step | 9.2 samples/s +Step 25670/44000 | Loss: 2.8884 | LR: 1.91e-04 | 2191.9ms/step | 9.1 samples/s +Step 25680/44000 | Loss: 2.9308 | LR: 1.91e-04 | 2157.9ms/step | 9.3 samples/s +Step 25690/44000 | Loss: 2.9074 | LR: 1.90e-04 | 2171.3ms/step | 9.2 samples/s +Step 25700/44000 | Loss: 2.9553 | LR: 1.90e-04 | 2132.9ms/step | 9.4 samples/s +Step 25710/44000 | Loss: 2.8917 | LR: 1.90e-04 | 2147.6ms/step | 9.3 samples/s +Step 25720/44000 | Loss: 2.9029 | LR: 1.90e-04 | 2179.5ms/step | 9.2 samples/s +Step 25730/44000 | Loss: 2.9120 | LR: 1.90e-04 | 2158.5ms/step | 9.3 samples/s +Step 25740/44000 | Loss: 2.9587 | LR: 1.89e-04 | 2137.6ms/step | 9.4 samples/s +Step 25750/44000 | Loss: 2.8510 | LR: 1.89e-04 | 2142.3ms/step | 9.3 samples/s +Step 25760/44000 | Loss: 2.9032 | LR: 1.89e-04 | 2155.9ms/step | 9.3 samples/s +Step 25770/44000 | Loss: 2.9481 | LR: 1.89e-04 | 2159.1ms/step | 9.3 samples/s +Step 25780/44000 | Loss: 2.9266 | LR: 1.89e-04 | 2162.3ms/step | 9.2 samples/s +Step 25790/44000 | Loss: 2.9180 | LR: 1.89e-04 | 2159.6ms/step | 9.3 samples/s +Step 25800/44000 | Loss: 2.9411 | LR: 1.88e-04 | 2146.0ms/step | 9.3 samples/s +Step 25810/44000 | Loss: 2.9288 | LR: 1.88e-04 | 2137.7ms/step | 9.4 samples/s +Step 25820/44000 | Loss: 2.9563 | LR: 1.88e-04 | 2135.2ms/step | 9.4 samples/s +Step 25830/44000 | Loss: 2.9187 | LR: 1.88e-04 | 2150.0ms/step | 9.3 samples/s +Step 25840/44000 | Loss: 2.9770 | LR: 1.88e-04 | 2149.2ms/step | 9.3 samples/s +Step 25850/44000 | Loss: 2.9438 | LR: 1.88e-04 | 2128.9ms/step | 9.4 samples/s +Step 25860/44000 | Loss: 2.9377 | LR: 1.87e-04 | 2155.4ms/step | 9.3 samples/s +Step 25870/44000 | Loss: 2.9329 | LR: 1.87e-04 | 2182.9ms/step | 9.2 samples/s +Step 25880/44000 | Loss: 2.9043 | LR: 1.87e-04 | 2160.9ms/step | 9.3 samples/s +Step 25890/44000 | Loss: 2.9498 | LR: 1.87e-04 | 2128.0ms/step | 9.4 samples/s +Step 25900/44000 | Loss: 2.9248 | LR: 1.87e-04 | 2158.5ms/step | 9.3 samples/s +Step 25910/44000 | Loss: 2.9083 | LR: 1.87e-04 | 2151.4ms/step | 9.3 samples/s +Step 25920/44000 | Loss: 2.9433 | LR: 1.86e-04 | 2155.0ms/step | 9.3 samples/s +Step 25930/44000 | Loss: 2.9175 | LR: 1.86e-04 | 2152.8ms/step | 9.3 samples/s +Step 25940/44000 | Loss: 2.9235 | LR: 1.86e-04 | 2154.2ms/step | 9.3 samples/s +Step 25950/44000 | Loss: 2.9708 | LR: 1.86e-04 | 2173.9ms/step | 9.2 samples/s +Step 25960/44000 | Loss: 2.9458 | LR: 1.86e-04 | 2175.3ms/step | 9.2 samples/s +Step 25970/44000 | Loss: 2.9189 | LR: 1.85e-04 | 2140.5ms/step | 9.3 samples/s +Step 25980/44000 | Loss: 2.8825 | LR: 1.85e-04 | 2146.3ms/step | 9.3 samples/s +Step 25990/44000 | Loss: 2.8973 | LR: 1.85e-04 | 2169.6ms/step | 9.2 samples/s +Step 26000/44000 | Loss: 2.9376 | LR: 1.85e-04 | 2166.0ms/step | 9.2 samples/s +Validation at step 26000: Loss = 2.9264, Perplexity = 18.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-26000 +Step 26010/44000 | Loss: 2.9393 | LR: 1.85e-04 | 4437.9ms/step | 4.5 samples/s +Step 26020/44000 | Loss: 2.8848 | LR: 1.85e-04 | 2200.6ms/step | 9.1 samples/s +Step 26030/44000 | Loss: 2.8608 | LR: 1.84e-04 | 2145.5ms/step | 9.3 samples/s +Step 26040/44000 | Loss: 2.9171 | LR: 1.84e-04 | 2148.2ms/step | 9.3 samples/s +Step 26050/44000 | Loss: 2.8976 | LR: 1.84e-04 | 2145.9ms/step | 9.3 samples/s +Step 26060/44000 | Loss: 2.9165 | LR: 1.84e-04 | 2159.8ms/step | 9.3 samples/s +Step 26070/44000 | Loss: 2.9699 | LR: 1.84e-04 | 2148.7ms/step | 9.3 samples/s +Step 26080/44000 | Loss: 2.9104 | LR: 1.84e-04 | 2130.6ms/step | 9.4 samples/s +Step 26090/44000 | Loss: 2.9177 | LR: 1.83e-04 | 2161.5ms/step | 9.3 samples/s +Step 26100/44000 | Loss: 2.9029 | LR: 1.83e-04 | 2142.0ms/step | 9.3 samples/s +Step 26110/44000 | Loss: 2.9446 | LR: 1.83e-04 | 2132.6ms/step | 9.4 samples/s +Step 26120/44000 | Loss: 2.9082 | LR: 1.83e-04 | 2143.4ms/step | 9.3 samples/s +Step 26130/44000 | Loss: 2.9446 | LR: 1.83e-04 | 2093.9ms/step | 9.6 samples/s +Step 26140/44000 | Loss: 2.9245 | LR: 1.82e-04 | 2080.1ms/step | 9.6 samples/s +Step 26150/44000 | Loss: 2.8790 | LR: 1.82e-04 | 2120.2ms/step | 9.4 samples/s +Step 26160/44000 | Loss: 2.9361 | LR: 1.82e-04 | 2192.0ms/step | 9.1 samples/s +Step 26170/44000 | Loss: 2.9002 | LR: 1.82e-04 | 2147.7ms/step | 9.3 samples/s +Step 26180/44000 | Loss: 2.9185 | LR: 1.82e-04 | 2154.9ms/step | 9.3 samples/s +Step 26190/44000 | Loss: 2.9664 | LR: 1.82e-04 | 2168.4ms/step | 9.2 samples/s +Step 26200/44000 | Loss: 2.8989 | LR: 1.81e-04 | 2165.7ms/step | 9.2 samples/s +Step 26210/44000 | Loss: 2.9508 | LR: 1.81e-04 | 2186.2ms/step | 9.1 samples/s +Step 26220/44000 | Loss: 2.9042 | LR: 1.81e-04 | 2164.0ms/step | 9.2 samples/s +Step 26230/44000 | Loss: 2.9025 | LR: 1.81e-04 | 2145.7ms/step | 9.3 samples/s +Step 26240/44000 | Loss: 2.9149 | LR: 1.81e-04 | 2160.3ms/step | 9.3 samples/s +Step 26250/44000 | Loss: 2.9258 | LR: 1.81e-04 | 2146.5ms/step | 9.3 samples/s +Step 26260/44000 | Loss: 2.9303 | LR: 1.80e-04 | 2167.7ms/step | 9.2 samples/s +Step 26270/44000 | Loss: 2.8901 | LR: 1.80e-04 | 2170.9ms/step | 9.2 samples/s +Step 26280/44000 | Loss: 2.8804 | LR: 1.80e-04 | 2137.1ms/step | 9.4 samples/s +Step 26290/44000 | Loss: 2.9000 | LR: 1.80e-04 | 2182.1ms/step | 9.2 samples/s +Step 26300/44000 | Loss: 2.9377 | LR: 1.80e-04 | 2170.6ms/step | 9.2 samples/s +Step 26310/44000 | Loss: 2.9024 | LR: 1.80e-04 | 2142.1ms/step | 9.3 samples/s +Step 26320/44000 | Loss: 2.8486 | LR: 1.79e-04 | 2132.4ms/step | 9.4 samples/s +Step 26330/44000 | Loss: 2.9051 | LR: 1.79e-04 | 2150.1ms/step | 9.3 samples/s +Step 26340/44000 | Loss: 2.9066 | LR: 1.79e-04 | 2135.7ms/step | 9.4 samples/s +Step 26350/44000 | Loss: 2.9101 | LR: 1.79e-04 | 2152.1ms/step | 9.3 samples/s +Step 26360/44000 | Loss: 2.9090 | LR: 1.79e-04 | 2147.5ms/step | 9.3 samples/s +Step 26370/44000 | Loss: 2.8829 | LR: 1.78e-04 | 2153.5ms/step | 9.3 samples/s +Step 26380/44000 | Loss: 2.9042 | LR: 1.78e-04 | 2169.7ms/step | 9.2 samples/s +Step 26390/44000 | Loss: 2.8752 | LR: 1.78e-04 | 2157.2ms/step | 9.3 samples/s +Step 26400/44000 | Loss: 2.8910 | LR: 1.78e-04 | 2166.2ms/step | 9.2 samples/s +Step 26410/44000 | Loss: 2.9240 | LR: 1.78e-04 | 2155.2ms/step | 9.3 samples/s +Step 26420/44000 | Loss: 2.9415 | LR: 1.78e-04 | 2154.3ms/step | 9.3 samples/s +Step 26430/44000 | Loss: 2.8872 | LR: 1.77e-04 | 2153.7ms/step | 9.3 samples/s +Step 26440/44000 | Loss: 2.8770 | LR: 1.77e-04 | 2155.5ms/step | 9.3 samples/s +Step 26450/44000 | Loss: 2.9139 | LR: 1.77e-04 | 2168.7ms/step | 9.2 samples/s +Step 26460/44000 | Loss: 2.8749 | LR: 1.77e-04 | 2150.3ms/step | 9.3 samples/s +Step 26470/44000 | Loss: 2.9625 | LR: 1.77e-04 | 2161.7ms/step | 9.3 samples/s +Step 26480/44000 | Loss: 2.9455 | LR: 1.77e-04 | 2146.4ms/step | 9.3 samples/s +Step 26490/44000 | Loss: 2.8836 | LR: 1.76e-04 | 2157.8ms/step | 9.3 samples/s +Step 26500/44000 | Loss: 2.9430 | LR: 1.76e-04 | 2142.4ms/step | 9.3 samples/s +Step 26510/44000 | Loss: 2.9493 | LR: 1.76e-04 | 2178.8ms/step | 9.2 samples/s +Step 26520/44000 | Loss: 2.8572 | LR: 1.76e-04 | 2173.3ms/step | 9.2 samples/s +Step 26530/44000 | Loss: 2.8717 | LR: 1.76e-04 | 2166.5ms/step | 9.2 samples/s +Step 26540/44000 | Loss: 2.9049 | LR: 1.76e-04 | 2152.7ms/step | 9.3 samples/s +Step 26550/44000 | Loss: 2.9511 | LR: 1.75e-04 | 2180.9ms/step | 9.2 samples/s +Step 26560/44000 | Loss: 2.9217 | LR: 1.75e-04 | 2170.8ms/step | 9.2 samples/s +Step 26570/44000 | Loss: 2.8717 | LR: 1.75e-04 | 2161.9ms/step | 9.3 samples/s +Step 26580/44000 | Loss: 2.9253 | LR: 1.75e-04 | 2166.8ms/step | 9.2 samples/s +Step 26590/44000 | Loss: 2.9195 | LR: 1.75e-04 | 2148.7ms/step | 9.3 samples/s +Step 26600/44000 | Loss: 2.9098 | LR: 1.74e-04 | 2149.3ms/step | 9.3 samples/s +Step 26610/44000 | Loss: 2.9179 | LR: 1.74e-04 | 2167.3ms/step | 9.2 samples/s +Step 26620/44000 | Loss: 2.9267 | LR: 1.74e-04 | 2130.2ms/step | 9.4 samples/s +Step 26630/44000 | Loss: 2.9365 | LR: 1.74e-04 | 2125.0ms/step | 9.4 samples/s +Step 26640/44000 | Loss: 2.9419 | LR: 1.74e-04 | 2159.0ms/step | 9.3 samples/s +Step 26650/44000 | Loss: 2.8970 | LR: 1.74e-04 | 2165.0ms/step | 9.2 samples/s +Step 26660/44000 | Loss: 2.9314 | LR: 1.73e-04 | 2161.0ms/step | 9.3 samples/s +Step 26670/44000 | Loss: 2.9131 | LR: 1.73e-04 | 2129.2ms/step | 9.4 samples/s +Step 26680/44000 | Loss: 2.9411 | LR: 1.73e-04 | 2158.0ms/step | 9.3 samples/s +Step 26690/44000 | Loss: 2.9377 | LR: 1.73e-04 | 2152.6ms/step | 9.3 samples/s +Step 26700/44000 | Loss: 2.9743 | LR: 1.73e-04 | 2157.4ms/step | 9.3 samples/s +Step 26710/44000 | Loss: 2.9131 | LR: 1.73e-04 | 2169.2ms/step | 9.2 samples/s +Step 26720/44000 | Loss: 2.9231 | LR: 1.72e-04 | 2130.9ms/step | 9.4 samples/s +Step 26730/44000 | Loss: 2.8984 | LR: 1.72e-04 | 2181.5ms/step | 9.2 samples/s +Step 26740/44000 | Loss: 2.9130 | LR: 1.72e-04 | 2160.0ms/step | 9.3 samples/s +Step 26750/44000 | Loss: 2.9267 | LR: 1.72e-04 | 2152.5ms/step | 9.3 samples/s +Step 26760/44000 | Loss: 2.9203 | LR: 1.72e-04 | 2137.9ms/step | 9.4 samples/s +Step 26770/44000 | Loss: 2.9394 | LR: 1.72e-04 | 2122.7ms/step | 9.4 samples/s +Step 26780/44000 | Loss: 2.9316 | LR: 1.71e-04 | 2143.0ms/step | 9.3 samples/s +Step 26790/44000 | Loss: 2.9236 | LR: 1.71e-04 | 2156.9ms/step | 9.3 samples/s +Step 26800/44000 | Loss: 2.9157 | LR: 1.71e-04 | 2154.3ms/step | 9.3 samples/s +Step 26810/44000 | Loss: 2.9097 | LR: 1.71e-04 | 2144.0ms/step | 9.3 samples/s +Step 26820/44000 | Loss: 2.9409 | LR: 1.71e-04 | 2138.9ms/step | 9.4 samples/s +Step 26830/44000 | Loss: 2.9209 | LR: 1.71e-04 | 2157.0ms/step | 9.3 samples/s +Step 26840/44000 | Loss: 2.8979 | LR: 1.70e-04 | 2153.9ms/step | 9.3 samples/s +Step 26850/44000 | Loss: 2.9250 | LR: 1.70e-04 | 2141.8ms/step | 9.3 samples/s +Step 26860/44000 | Loss: 2.9310 | LR: 1.70e-04 | 2160.0ms/step | 9.3 samples/s +Step 26870/44000 | Loss: 2.9060 | LR: 1.70e-04 | 2150.5ms/step | 9.3 samples/s +Step 26880/44000 | Loss: 2.9155 | LR: 1.70e-04 | 2142.7ms/step | 9.3 samples/s +Step 26890/44000 | Loss: 2.9394 | LR: 1.69e-04 | 2130.1ms/step | 9.4 samples/s +Step 26900/44000 | Loss: 2.9327 | LR: 1.69e-04 | 2139.9ms/step | 9.3 samples/s +Step 26910/44000 | Loss: 2.9375 | LR: 1.69e-04 | 2160.9ms/step | 9.3 samples/s +Step 26920/44000 | Loss: 2.8830 | LR: 1.69e-04 | 2156.7ms/step | 9.3 samples/s +Step 26930/44000 | Loss: 2.9079 | LR: 1.69e-04 | 2141.7ms/step | 9.3 samples/s +Step 26940/44000 | Loss: 2.8519 | LR: 1.69e-04 | 2149.0ms/step | 9.3 samples/s +Step 26950/44000 | Loss: 2.8706 | LR: 1.68e-04 | 2169.6ms/step | 9.2 samples/s +Step 26960/44000 | Loss: 2.9372 | LR: 1.68e-04 | 2149.9ms/step | 9.3 samples/s +Step 26970/44000 | Loss: 2.8761 | LR: 1.68e-04 | 2145.7ms/step | 9.3 samples/s +Step 26980/44000 | Loss: 2.8910 | LR: 1.68e-04 | 2175.4ms/step | 9.2 samples/s +Step 26990/44000 | Loss: 2.9086 | LR: 1.68e-04 | 2177.8ms/step | 9.2 samples/s +Step 27000/44000 | Loss: 2.9073 | LR: 1.68e-04 | 2155.3ms/step | 9.3 samples/s +Validation at step 27000: Loss = 2.9211, Perplexity = 18.56 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-27000 +Step 27010/44000 | Loss: 2.8894 | LR: 1.67e-04 | 4495.0ms/step | 4.4 samples/s +Step 27020/44000 | Loss: 2.9114 | LR: 1.67e-04 | 2133.4ms/step | 9.4 samples/s +Step 27030/44000 | Loss: 2.9176 | LR: 1.67e-04 | 2163.8ms/step | 9.2 samples/s +Step 27040/44000 | Loss: 2.9106 | LR: 1.67e-04 | 2168.5ms/step | 9.2 samples/s +Step 27050/44000 | Loss: 2.9197 | LR: 1.67e-04 | 2156.6ms/step | 9.3 samples/s +Step 27060/44000 | Loss: 2.9365 | LR: 1.67e-04 | 2162.9ms/step | 9.2 samples/s +Step 27070/44000 | Loss: 2.9392 | LR: 1.66e-04 | 2168.1ms/step | 9.2 samples/s +Step 27080/44000 | Loss: 2.9439 | LR: 1.66e-04 | 2139.5ms/step | 9.3 samples/s +Step 27090/44000 | Loss: 2.9103 | LR: 1.66e-04 | 2149.8ms/step | 9.3 samples/s +Step 27100/44000 | Loss: 2.9132 | LR: 1.66e-04 | 2135.9ms/step | 9.4 samples/s +Step 27110/44000 | Loss: 2.9068 | LR: 1.66e-04 | 2164.1ms/step | 9.2 samples/s +Step 27120/44000 | Loss: 2.8904 | LR: 1.66e-04 | 2164.2ms/step | 9.2 samples/s +Step 27130/44000 | Loss: 2.8819 | LR: 1.65e-04 | 2127.0ms/step | 9.4 samples/s +Step 27140/44000 | Loss: 2.9125 | LR: 1.65e-04 | 2012.6ms/step | 9.9 samples/s +Step 27150/44000 | Loss: 2.8902 | LR: 1.65e-04 | 2158.8ms/step | 9.3 samples/s +Step 27160/44000 | Loss: 2.9283 | LR: 1.65e-04 | 2154.7ms/step | 9.3 samples/s +Step 27170/44000 | Loss: 2.9436 | LR: 1.65e-04 | 2131.1ms/step | 9.4 samples/s +Step 27180/44000 | Loss: 2.8796 | LR: 1.65e-04 | 2162.4ms/step | 9.2 samples/s +Step 27190/44000 | Loss: 2.9552 | LR: 1.64e-04 | 2149.6ms/step | 9.3 samples/s +Step 27200/44000 | Loss: 2.9100 | LR: 1.64e-04 | 2164.4ms/step | 9.2 samples/s +Step 27210/44000 | Loss: 2.9134 | LR: 1.64e-04 | 2180.9ms/step | 9.2 samples/s +Step 27220/44000 | Loss: 2.9063 | LR: 1.64e-04 | 2194.7ms/step | 9.1 samples/s +Step 27230/44000 | Loss: 2.8778 | LR: 1.64e-04 | 2163.1ms/step | 9.2 samples/s +Step 27240/44000 | Loss: 2.9422 | LR: 1.63e-04 | 2132.8ms/step | 9.4 samples/s +Step 27250/44000 | Loss: 2.9172 | LR: 1.63e-04 | 2188.9ms/step | 9.1 samples/s +Step 27260/44000 | Loss: 2.9101 | LR: 1.63e-04 | 2154.0ms/step | 9.3 samples/s +Step 27270/44000 | Loss: 2.9170 | LR: 1.63e-04 | 2138.8ms/step | 9.4 samples/s +Step 27280/44000 | Loss: 2.8938 | LR: 1.63e-04 | 2145.2ms/step | 9.3 samples/s +Step 27290/44000 | Loss: 2.9210 | LR: 1.63e-04 | 2168.7ms/step | 9.2 samples/s +Step 27300/44000 | Loss: 2.8889 | LR: 1.62e-04 | 2180.5ms/step | 9.2 samples/s +Step 27310/44000 | Loss: 2.8799 | LR: 1.62e-04 | 2188.0ms/step | 9.1 samples/s +Step 27320/44000 | Loss: 2.8897 | LR: 1.62e-04 | 2192.4ms/step | 9.1 samples/s +Step 27330/44000 | Loss: 2.9107 | LR: 1.62e-04 | 2158.5ms/step | 9.3 samples/s +Step 27340/44000 | Loss: 2.8901 | LR: 1.62e-04 | 2138.7ms/step | 9.4 samples/s +Step 27350/44000 | Loss: 2.9009 | LR: 1.62e-04 | 2163.3ms/step | 9.2 samples/s +Step 27360/44000 | Loss: 2.9196 | LR: 1.61e-04 | 2166.5ms/step | 9.2 samples/s +Step 27370/44000 | Loss: 2.9078 | LR: 1.61e-04 | 2157.4ms/step | 9.3 samples/s +Step 27380/44000 | Loss: 2.8698 | LR: 1.61e-04 | 2158.4ms/step | 9.3 samples/s +Step 27390/44000 | Loss: 2.8790 | LR: 1.61e-04 | 2141.9ms/step | 9.3 samples/s +Step 27400/44000 | Loss: 2.9486 | LR: 1.61e-04 | 2165.6ms/step | 9.2 samples/s +Step 27410/44000 | Loss: 2.9341 | LR: 1.61e-04 | 2153.4ms/step | 9.3 samples/s +Step 27420/44000 | Loss: 2.9143 | LR: 1.60e-04 | 2138.7ms/step | 9.4 samples/s +Step 27430/44000 | Loss: 2.8887 | LR: 1.60e-04 | 2152.0ms/step | 9.3 samples/s +Step 27440/44000 | Loss: 2.8964 | LR: 1.60e-04 | 2144.2ms/step | 9.3 samples/s +Step 27450/44000 | Loss: 2.9437 | LR: 1.60e-04 | 2151.0ms/step | 9.3 samples/s +Step 27460/44000 | Loss: 2.8791 | LR: 1.60e-04 | 2152.3ms/step | 9.3 samples/s +Step 27470/44000 | Loss: 2.9418 | LR: 1.60e-04 | 2148.7ms/step | 9.3 samples/s +Step 27480/44000 | Loss: 2.9133 | LR: 1.59e-04 | 2168.7ms/step | 9.2 samples/s +Step 27490/44000 | Loss: 2.9247 | LR: 1.59e-04 | 2158.8ms/step | 9.3 samples/s +Step 27500/44000 | Loss: 2.9135 | LR: 1.59e-04 | 2164.1ms/step | 9.2 samples/s +Step 27510/44000 | Loss: 2.9134 | LR: 1.59e-04 | 2142.7ms/step | 9.3 samples/s +Step 27520/44000 | Loss: 2.9178 | LR: 1.59e-04 | 2176.8ms/step | 9.2 samples/s +Step 27530/44000 | Loss: 2.9350 | LR: 1.59e-04 | 2216.4ms/step | 9.0 samples/s +Step 27540/44000 | Loss: 2.9095 | LR: 1.58e-04 | 2198.3ms/step | 9.1 samples/s +Step 27550/44000 | Loss: 2.9136 | LR: 1.58e-04 | 2221.9ms/step | 9.0 samples/s +Step 27560/44000 | Loss: 2.9152 | LR: 1.58e-04 | 2197.2ms/step | 9.1 samples/s +Step 27570/44000 | Loss: 2.9042 | LR: 1.58e-04 | 2185.2ms/step | 9.2 samples/s +Step 27580/44000 | Loss: 2.9241 | LR: 1.58e-04 | 2219.2ms/step | 9.0 samples/s +Step 27590/44000 | Loss: 2.8955 | LR: 1.58e-04 | 2183.7ms/step | 9.2 samples/s +Step 27600/44000 | Loss: 2.9154 | LR: 1.57e-04 | 2191.7ms/step | 9.1 samples/s +Step 27610/44000 | Loss: 2.9360 | LR: 1.57e-04 | 2254.9ms/step | 8.9 samples/s +Step 27620/44000 | Loss: 2.9186 | LR: 1.57e-04 | 2280.5ms/step | 8.8 samples/s +Step 27630/44000 | Loss: 2.9390 | LR: 1.57e-04 | 2194.9ms/step | 9.1 samples/s +Step 27640/44000 | Loss: 2.9030 | LR: 1.57e-04 | 2210.6ms/step | 9.0 samples/s +Step 27650/44000 | Loss: 2.9208 | LR: 1.57e-04 | 2208.3ms/step | 9.1 samples/s +Step 27660/44000 | Loss: 2.9455 | LR: 1.56e-04 | 2213.6ms/step | 9.0 samples/s +Step 27670/44000 | Loss: 2.9427 | LR: 1.56e-04 | 2207.5ms/step | 9.1 samples/s +Step 27680/44000 | Loss: 2.8965 | LR: 1.56e-04 | 2231.8ms/step | 9.0 samples/s +Step 27690/44000 | Loss: 2.8848 | LR: 1.56e-04 | 2209.6ms/step | 9.1 samples/s +Step 27700/44000 | Loss: 2.9033 | LR: 1.56e-04 | 2240.8ms/step | 8.9 samples/s +Step 27710/44000 | Loss: 2.9392 | LR: 1.56e-04 | 2251.4ms/step | 8.9 samples/s +Step 27720/44000 | Loss: 2.9362 | LR: 1.55e-04 | 2262.6ms/step | 8.8 samples/s +Step 27730/44000 | Loss: 2.9414 | LR: 1.55e-04 | 2286.2ms/step | 8.7 samples/s +Step 27740/44000 | Loss: 2.8949 | LR: 1.55e-04 | 2313.8ms/step | 8.6 samples/s +Step 27750/44000 | Loss: 2.9359 | LR: 1.55e-04 | 2263.9ms/step | 8.8 samples/s +Step 27760/44000 | Loss: 2.8679 | LR: 1.55e-04 | 2271.1ms/step | 8.8 samples/s +Step 27770/44000 | Loss: 2.8841 | LR: 1.55e-04 | 2271.3ms/step | 8.8 samples/s +Step 27780/44000 | Loss: 2.9254 | LR: 1.54e-04 | 2244.0ms/step | 8.9 samples/s +Step 27790/44000 | Loss: 2.9084 | LR: 1.54e-04 | 2279.4ms/step | 8.8 samples/s +Step 27800/44000 | Loss: 2.8591 | LR: 1.54e-04 | 2270.4ms/step | 8.8 samples/s +Step 27810/44000 | Loss: 2.9097 | LR: 1.54e-04 | 2234.3ms/step | 9.0 samples/s +Step 27820/44000 | Loss: 2.9112 | LR: 1.54e-04 | 2214.1ms/step | 9.0 samples/s +Step 27830/44000 | Loss: 2.9104 | LR: 1.54e-04 | 2295.9ms/step | 8.7 samples/s +Step 27840/44000 | Loss: 2.9418 | LR: 1.53e-04 | 2254.5ms/step | 8.9 samples/s +Step 27850/44000 | Loss: 2.9076 | LR: 1.53e-04 | 2225.9ms/step | 9.0 samples/s +Step 27860/44000 | Loss: 2.9185 | LR: 1.53e-04 | 2251.3ms/step | 8.9 samples/s +Step 27870/44000 | Loss: 2.9094 | LR: 1.53e-04 | 2231.8ms/step | 9.0 samples/s +Step 27880/44000 | Loss: 2.8828 | LR: 1.53e-04 | 2222.1ms/step | 9.0 samples/s +Step 27890/44000 | Loss: 2.9119 | LR: 1.53e-04 | 2258.1ms/step | 8.9 samples/s +Step 27900/44000 | Loss: 2.8974 | LR: 1.52e-04 | 2289.7ms/step | 8.7 samples/s +Step 27910/44000 | Loss: 2.9582 | LR: 1.52e-04 | 2290.3ms/step | 8.7 samples/s +Step 27920/44000 | Loss: 2.8831 | LR: 1.52e-04 | 2284.8ms/step | 8.8 samples/s +Step 27930/44000 | Loss: 2.9135 | LR: 1.52e-04 | 2243.3ms/step | 8.9 samples/s +Step 27940/44000 | Loss: 2.9186 | LR: 1.52e-04 | 2235.7ms/step | 8.9 samples/s +Step 27950/44000 | Loss: 2.9357 | LR: 1.52e-04 | 2258.4ms/step | 8.9 samples/s +Step 27960/44000 | Loss: 2.8919 | LR: 1.51e-04 | 2231.6ms/step | 9.0 samples/s +Step 27970/44000 | Loss: 2.9471 | LR: 1.51e-04 | 2216.2ms/step | 9.0 samples/s +Step 27980/44000 | Loss: 2.8853 | LR: 1.51e-04 | 2258.2ms/step | 8.9 samples/s +Step 27990/44000 | Loss: 2.9009 | LR: 1.51e-04 | 2278.5ms/step | 8.8 samples/s +Step 28000/44000 | Loss: 2.8990 | LR: 1.51e-04 | 2218.4ms/step | 9.0 samples/s +Validation at step 28000: Loss = 2.9157, Perplexity = 18.46 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-28000 +Step 28010/44000 | Loss: 2.9150 | LR: 1.51e-04 | 4690.2ms/step | 4.3 samples/s +Step 28020/44000 | Loss: 2.9043 | LR: 1.50e-04 | 2167.4ms/step | 9.2 samples/s +Step 28030/44000 | Loss: 2.8885 | LR: 1.50e-04 | 2147.2ms/step | 9.3 samples/s +Step 28040/44000 | Loss: 2.9137 | LR: 1.50e-04 | 2151.8ms/step | 9.3 samples/s +Step 28050/44000 | Loss: 2.8848 | LR: 1.50e-04 | 2141.0ms/step | 9.3 samples/s +Step 28060/44000 | Loss: 2.9068 | LR: 1.50e-04 | 2156.4ms/step | 9.3 samples/s +Step 28070/44000 | Loss: 2.9016 | LR: 1.50e-04 | 2177.8ms/step | 9.2 samples/s +Step 28080/44000 | Loss: 2.8675 | LR: 1.49e-04 | 2137.5ms/step | 9.4 samples/s +Step 28090/44000 | Loss: 2.9235 | LR: 1.49e-04 | 2184.5ms/step | 9.2 samples/s +Step 28100/44000 | Loss: 2.9440 | LR: 1.49e-04 | 2164.5ms/step | 9.2 samples/s +Step 28110/44000 | Loss: 2.8515 | LR: 1.49e-04 | 2159.4ms/step | 9.3 samples/s +Step 28120/44000 | Loss: 2.9151 | LR: 1.49e-04 | 2166.9ms/step | 9.2 samples/s +Step 28130/44000 | Loss: 2.8803 | LR: 1.49e-04 | 2140.9ms/step | 9.3 samples/s +Step 28140/44000 | Loss: 2.9028 | LR: 1.48e-04 | 2054.8ms/step | 9.7 samples/s +Step 28150/44000 | Loss: 2.8965 | LR: 1.48e-04 | 2134.7ms/step | 9.4 samples/s +Step 28160/44000 | Loss: 2.9002 | LR: 1.48e-04 | 2132.7ms/step | 9.4 samples/s +Step 28170/44000 | Loss: 2.8988 | LR: 1.48e-04 | 2152.9ms/step | 9.3 samples/s +Step 28180/44000 | Loss: 2.9040 | LR: 1.48e-04 | 2154.1ms/step | 9.3 samples/s +Step 28190/44000 | Loss: 2.8829 | LR: 1.48e-04 | 2161.0ms/step | 9.3 samples/s +Step 28200/44000 | Loss: 2.9379 | LR: 1.47e-04 | 2143.4ms/step | 9.3 samples/s +Step 28210/44000 | Loss: 2.9431 | LR: 1.47e-04 | 2154.5ms/step | 9.3 samples/s +Step 28220/44000 | Loss: 2.8701 | LR: 1.47e-04 | 2129.3ms/step | 9.4 samples/s +Step 28230/44000 | Loss: 2.8997 | LR: 1.47e-04 | 2138.5ms/step | 9.4 samples/s +Step 28240/44000 | Loss: 2.9410 | LR: 1.47e-04 | 2143.0ms/step | 9.3 samples/s +Step 28250/44000 | Loss: 2.8776 | LR: 1.47e-04 | 2163.9ms/step | 9.2 samples/s +Step 28260/44000 | Loss: 2.9204 | LR: 1.46e-04 | 2174.1ms/step | 9.2 samples/s +Step 28270/44000 | Loss: 2.8959 | LR: 1.46e-04 | 2169.2ms/step | 9.2 samples/s +Step 28280/44000 | Loss: 2.9199 | LR: 1.46e-04 | 2122.4ms/step | 9.4 samples/s +Step 28290/44000 | Loss: 2.9168 | LR: 1.46e-04 | 2142.2ms/step | 9.3 samples/s +Step 28300/44000 | Loss: 2.9149 | LR: 1.46e-04 | 2138.5ms/step | 9.4 samples/s +Step 28310/44000 | Loss: 2.8949 | LR: 1.46e-04 | 2155.0ms/step | 9.3 samples/s +Step 28320/44000 | Loss: 2.9042 | LR: 1.45e-04 | 2161.0ms/step | 9.3 samples/s +Step 28330/44000 | Loss: 2.9353 | LR: 1.45e-04 | 2144.6ms/step | 9.3 samples/s +Step 28340/44000 | Loss: 2.9251 | LR: 1.45e-04 | 2152.7ms/step | 9.3 samples/s +Step 28350/44000 | Loss: 2.8813 | LR: 1.45e-04 | 2128.5ms/step | 9.4 samples/s +Step 28360/44000 | Loss: 2.9382 | LR: 1.45e-04 | 2163.4ms/step | 9.2 samples/s +Step 28370/44000 | Loss: 2.9224 | LR: 1.45e-04 | 2152.9ms/step | 9.3 samples/s +Step 28380/44000 | Loss: 2.9694 | LR: 1.44e-04 | 2140.3ms/step | 9.3 samples/s +Step 28390/44000 | Loss: 2.9496 | LR: 1.44e-04 | 2167.5ms/step | 9.2 samples/s +Step 28400/44000 | Loss: 2.9589 | LR: 1.44e-04 | 2121.8ms/step | 9.4 samples/s +Step 28410/44000 | Loss: 2.8820 | LR: 1.44e-04 | 2170.4ms/step | 9.2 samples/s +Step 28420/44000 | Loss: 2.9046 | LR: 1.44e-04 | 2169.4ms/step | 9.2 samples/s +Step 28430/44000 | Loss: 2.9115 | LR: 1.44e-04 | 2143.7ms/step | 9.3 samples/s +Step 28440/44000 | Loss: 2.9087 | LR: 1.43e-04 | 2133.4ms/step | 9.4 samples/s +Step 28450/44000 | Loss: 2.9356 | LR: 1.43e-04 | 2164.9ms/step | 9.2 samples/s +Step 28460/44000 | Loss: 2.8956 | LR: 1.43e-04 | 2153.1ms/step | 9.3 samples/s +Step 28470/44000 | Loss: 2.9529 | LR: 1.43e-04 | 2129.3ms/step | 9.4 samples/s +Step 28480/44000 | Loss: 2.9291 | LR: 1.43e-04 | 2147.6ms/step | 9.3 samples/s +Step 28490/44000 | Loss: 2.8975 | LR: 1.43e-04 | 2163.6ms/step | 9.2 samples/s +Step 28500/44000 | Loss: 2.8729 | LR: 1.42e-04 | 2150.7ms/step | 9.3 samples/s +Step 28510/44000 | Loss: 2.9171 | LR: 1.42e-04 | 2157.4ms/step | 9.3 samples/s +Step 28520/44000 | Loss: 2.8682 | LR: 1.42e-04 | 2158.6ms/step | 9.3 samples/s +Step 28530/44000 | Loss: 2.9285 | LR: 1.42e-04 | 2164.3ms/step | 9.2 samples/s +Step 28540/44000 | Loss: 2.8900 | LR: 1.42e-04 | 2155.9ms/step | 9.3 samples/s +Step 28550/44000 | Loss: 2.9545 | LR: 1.42e-04 | 2153.1ms/step | 9.3 samples/s +Step 28560/44000 | Loss: 2.9062 | LR: 1.41e-04 | 2121.1ms/step | 9.4 samples/s +Step 28570/44000 | Loss: 2.9490 | LR: 1.41e-04 | 2141.3ms/step | 9.3 samples/s +Step 28580/44000 | Loss: 2.9367 | LR: 1.41e-04 | 2148.1ms/step | 9.3 samples/s +Step 28590/44000 | Loss: 2.9039 | LR: 1.41e-04 | 2170.8ms/step | 9.2 samples/s +Step 28600/44000 | Loss: 2.9100 | LR: 1.41e-04 | 2188.4ms/step | 9.1 samples/s +Step 28610/44000 | Loss: 2.9222 | LR: 1.41e-04 | 2167.0ms/step | 9.2 samples/s +Step 28620/44000 | Loss: 2.9103 | LR: 1.40e-04 | 2148.0ms/step | 9.3 samples/s +Step 28630/44000 | Loss: 2.9126 | LR: 1.40e-04 | 2157.5ms/step | 9.3 samples/s +Step 28640/44000 | Loss: 2.8767 | LR: 1.40e-04 | 2135.5ms/step | 9.4 samples/s +Step 28650/44000 | Loss: 2.9208 | LR: 1.40e-04 | 2161.1ms/step | 9.3 samples/s +Step 28660/44000 | Loss: 2.8596 | LR: 1.40e-04 | 2157.0ms/step | 9.3 samples/s +Step 28670/44000 | Loss: 2.8902 | LR: 1.40e-04 | 2150.8ms/step | 9.3 samples/s +Step 28680/44000 | Loss: 2.9102 | LR: 1.39e-04 | 2153.2ms/step | 9.3 samples/s +Step 28690/44000 | Loss: 2.9080 | LR: 1.39e-04 | 2149.2ms/step | 9.3 samples/s +Step 28700/44000 | Loss: 2.9120 | LR: 1.39e-04 | 2164.4ms/step | 9.2 samples/s +Step 28710/44000 | Loss: 2.9426 | LR: 1.39e-04 | 2156.7ms/step | 9.3 samples/s +Step 28720/44000 | Loss: 2.8666 | LR: 1.39e-04 | 2148.4ms/step | 9.3 samples/s +Step 28730/44000 | Loss: 2.9510 | LR: 1.39e-04 | 2164.1ms/step | 9.2 samples/s +Step 28740/44000 | Loss: 2.9137 | LR: 1.38e-04 | 2161.6ms/step | 9.3 samples/s +Step 28750/44000 | Loss: 2.9277 | LR: 1.38e-04 | 2145.4ms/step | 9.3 samples/s +Step 28760/44000 | Loss: 2.8755 | LR: 1.38e-04 | 2132.7ms/step | 9.4 samples/s +Step 28770/44000 | Loss: 2.8975 | LR: 1.38e-04 | 2144.1ms/step | 9.3 samples/s +Step 28780/44000 | Loss: 2.8980 | LR: 1.38e-04 | 2174.7ms/step | 9.2 samples/s +Step 28790/44000 | Loss: 2.9330 | LR: 1.38e-04 | 2173.3ms/step | 9.2 samples/s +Step 28800/44000 | Loss: 2.9478 | LR: 1.38e-04 | 2167.9ms/step | 9.2 samples/s +Step 28810/44000 | Loss: 2.9054 | LR: 1.37e-04 | 2143.3ms/step | 9.3 samples/s +Step 28820/44000 | Loss: 2.9197 | LR: 1.37e-04 | 2145.9ms/step | 9.3 samples/s +Step 28830/44000 | Loss: 2.9252 | LR: 1.37e-04 | 2136.1ms/step | 9.4 samples/s +Step 28840/44000 | Loss: 2.9252 | LR: 1.37e-04 | 2154.8ms/step | 9.3 samples/s +Step 28850/44000 | Loss: 2.9167 | LR: 1.37e-04 | 2153.8ms/step | 9.3 samples/s +Step 28860/44000 | Loss: 2.8750 | LR: 1.37e-04 | 2157.2ms/step | 9.3 samples/s +Step 28870/44000 | Loss: 2.9094 | LR: 1.36e-04 | 2153.2ms/step | 9.3 samples/s +Step 28880/44000 | Loss: 2.9127 | LR: 1.36e-04 | 2146.0ms/step | 9.3 samples/s +Step 28890/44000 | Loss: 2.9095 | LR: 1.36e-04 | 2157.8ms/step | 9.3 samples/s +Step 28900/44000 | Loss: 2.9159 | LR: 1.36e-04 | 2149.9ms/step | 9.3 samples/s +Step 28910/44000 | Loss: 2.9344 | LR: 1.36e-04 | 2148.3ms/step | 9.3 samples/s +Step 28920/44000 | Loss: 2.9201 | LR: 1.36e-04 | 2160.9ms/step | 9.3 samples/s +Step 28930/44000 | Loss: 2.8736 | LR: 1.35e-04 | 2148.6ms/step | 9.3 samples/s +Step 28940/44000 | Loss: 2.9429 | LR: 1.35e-04 | 2150.9ms/step | 9.3 samples/s +Step 28950/44000 | Loss: 2.8656 | LR: 1.35e-04 | 2147.9ms/step | 9.3 samples/s +Step 28960/44000 | Loss: 2.8679 | LR: 1.35e-04 | 2277.5ms/step | 8.8 samples/s +Step 28970/44000 | Loss: 2.9105 | LR: 1.35e-04 | 2231.5ms/step | 9.0 samples/s +Step 28980/44000 | Loss: 2.9329 | LR: 1.35e-04 | 2280.7ms/step | 8.8 samples/s +Step 28990/44000 | Loss: 2.8809 | LR: 1.34e-04 | 2225.6ms/step | 9.0 samples/s +Step 29000/44000 | Loss: 2.9069 | LR: 1.34e-04 | 2299.8ms/step | 8.7 samples/s +Validation at step 29000: Loss = 2.9110, Perplexity = 18.37 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-29000 +Step 29010/44000 | Loss: 2.9085 | LR: 1.34e-04 | 4762.7ms/step | 4.2 samples/s +Step 29020/44000 | Loss: 2.9523 | LR: 1.34e-04 | 2331.6ms/step | 8.6 samples/s +Step 29030/44000 | Loss: 2.8898 | LR: 1.34e-04 | 2288.6ms/step | 8.7 samples/s +Step 29040/44000 | Loss: 2.9071 | LR: 1.34e-04 | 2229.5ms/step | 9.0 samples/s +Step 29050/44000 | Loss: 2.9163 | LR: 1.33e-04 | 2185.5ms/step | 9.2 samples/s +Step 29060/44000 | Loss: 2.9169 | LR: 1.33e-04 | 2335.1ms/step | 8.6 samples/s +Step 29070/44000 | Loss: 2.9113 | LR: 1.33e-04 | 2195.3ms/step | 9.1 samples/s +Step 29080/44000 | Loss: 2.8734 | LR: 1.33e-04 | 2292.1ms/step | 8.7 samples/s +Step 29090/44000 | Loss: 2.8650 | LR: 1.33e-04 | 2211.7ms/step | 9.0 samples/s +Step 29100/44000 | Loss: 2.9124 | LR: 1.33e-04 | 2279.3ms/step | 8.8 samples/s +Step 29110/44000 | Loss: 2.8794 | LR: 1.33e-04 | 2142.4ms/step | 9.3 samples/s +Step 29120/44000 | Loss: 2.8721 | LR: 1.32e-04 | 2164.6ms/step | 9.2 samples/s +Step 29130/44000 | Loss: 2.8994 | LR: 1.32e-04 | 2124.6ms/step | 9.4 samples/s +Step 29140/44000 | Loss: 2.9451 | LR: 1.32e-04 | 2023.1ms/step | 9.9 samples/s +Step 29150/44000 | Loss: 2.9115 | LR: 1.32e-04 | 2194.7ms/step | 9.1 samples/s +Step 29160/44000 | Loss: 2.8825 | LR: 1.32e-04 | 2155.7ms/step | 9.3 samples/s +Step 29170/44000 | Loss: 2.9039 | LR: 1.32e-04 | 2161.3ms/step | 9.3 samples/s +Step 29180/44000 | Loss: 2.9439 | LR: 1.31e-04 | 2152.5ms/step | 9.3 samples/s +Step 29190/44000 | Loss: 2.9194 | LR: 1.31e-04 | 2131.1ms/step | 9.4 samples/s +Step 29200/44000 | Loss: 2.9385 | LR: 1.31e-04 | 2160.3ms/step | 9.3 samples/s +Step 29210/44000 | Loss: 2.9620 | LR: 1.31e-04 | 2159.4ms/step | 9.3 samples/s +Step 29220/44000 | Loss: 2.8842 | LR: 1.31e-04 | 2175.1ms/step | 9.2 samples/s +Step 29230/44000 | Loss: 2.8751 | LR: 1.31e-04 | 2130.0ms/step | 9.4 samples/s +Step 29240/44000 | Loss: 2.9188 | LR: 1.30e-04 | 2146.6ms/step | 9.3 samples/s +Step 29250/44000 | Loss: 2.9022 | LR: 1.30e-04 | 2146.8ms/step | 9.3 samples/s +Step 29260/44000 | Loss: 2.8852 | LR: 1.30e-04 | 2151.5ms/step | 9.3 samples/s +Step 29270/44000 | Loss: 2.8928 | LR: 1.30e-04 | 2133.9ms/step | 9.4 samples/s +Step 29280/44000 | Loss: 2.9109 | LR: 1.30e-04 | 2152.3ms/step | 9.3 samples/s +Step 29290/44000 | Loss: 2.9310 | LR: 1.30e-04 | 2154.0ms/step | 9.3 samples/s +Step 29300/44000 | Loss: 2.9201 | LR: 1.29e-04 | 2143.5ms/step | 9.3 samples/s +Step 29310/44000 | Loss: 2.8638 | LR: 1.29e-04 | 2153.7ms/step | 9.3 samples/s +Step 29320/44000 | Loss: 2.9034 | LR: 1.29e-04 | 2163.0ms/step | 9.2 samples/s +Step 29330/44000 | Loss: 2.9048 | LR: 1.29e-04 | 2123.6ms/step | 9.4 samples/s +Step 29340/44000 | Loss: 2.9475 | LR: 1.29e-04 | 2156.1ms/step | 9.3 samples/s +Step 29350/44000 | Loss: 2.8781 | LR: 1.29e-04 | 2135.9ms/step | 9.4 samples/s +Step 29360/44000 | Loss: 2.9354 | LR: 1.29e-04 | 2172.2ms/step | 9.2 samples/s +Step 29370/44000 | Loss: 2.8707 | LR: 1.28e-04 | 2152.6ms/step | 9.3 samples/s +Step 29380/44000 | Loss: 2.8800 | LR: 1.28e-04 | 2146.3ms/step | 9.3 samples/s +Step 29390/44000 | Loss: 2.9186 | LR: 1.28e-04 | 2132.8ms/step | 9.4 samples/s +Step 29400/44000 | Loss: 2.9070 | LR: 1.28e-04 | 2156.6ms/step | 9.3 samples/s +Step 29410/44000 | Loss: 2.9216 | LR: 1.28e-04 | 2165.8ms/step | 9.2 samples/s +Step 29420/44000 | Loss: 2.8879 | LR: 1.28e-04 | 2149.2ms/step | 9.3 samples/s +Step 29430/44000 | Loss: 2.8407 | LR: 1.27e-04 | 2151.0ms/step | 9.3 samples/s +Step 29440/44000 | Loss: 2.9087 | LR: 1.27e-04 | 2140.5ms/step | 9.3 samples/s +Step 29450/44000 | Loss: 2.8778 | LR: 1.27e-04 | 2152.7ms/step | 9.3 samples/s +Step 29460/44000 | Loss: 2.8697 | LR: 1.27e-04 | 2128.4ms/step | 9.4 samples/s +Step 29470/44000 | Loss: 2.8731 | LR: 1.27e-04 | 2148.4ms/step | 9.3 samples/s +Step 29480/44000 | Loss: 2.8949 | LR: 1.27e-04 | 2141.8ms/step | 9.3 samples/s +Step 29490/44000 | Loss: 2.8727 | LR: 1.26e-04 | 2155.9ms/step | 9.3 samples/s +Step 29500/44000 | Loss: 2.8650 | LR: 1.26e-04 | 2155.7ms/step | 9.3 samples/s +Step 29510/44000 | Loss: 2.8600 | LR: 1.26e-04 | 2132.0ms/step | 9.4 samples/s +Step 29520/44000 | Loss: 2.9318 | LR: 1.26e-04 | 2139.5ms/step | 9.3 samples/s +Step 29530/44000 | Loss: 2.9185 | LR: 1.26e-04 | 2191.7ms/step | 9.1 samples/s +Step 29540/44000 | Loss: 2.9056 | LR: 1.26e-04 | 2315.8ms/step | 8.6 samples/s +Step 29550/44000 | Loss: 2.8959 | LR: 1.26e-04 | 2257.0ms/step | 8.9 samples/s +Step 29560/44000 | Loss: 2.8748 | LR: 1.25e-04 | 2244.1ms/step | 8.9 samples/s +Step 29570/44000 | Loss: 2.8780 | LR: 1.25e-04 | 2320.6ms/step | 8.6 samples/s +Step 29580/44000 | Loss: 2.8864 | LR: 1.25e-04 | 2285.4ms/step | 8.8 samples/s +Step 29590/44000 | Loss: 2.8744 | LR: 1.25e-04 | 2298.9ms/step | 8.7 samples/s +Step 29600/44000 | Loss: 2.8915 | LR: 1.25e-04 | 2215.8ms/step | 9.0 samples/s +Step 29610/44000 | Loss: 2.8421 | LR: 1.25e-04 | 2293.6ms/step | 8.7 samples/s +Step 29620/44000 | Loss: 2.9163 | LR: 1.24e-04 | 2285.9ms/step | 8.7 samples/s +Step 29630/44000 | Loss: 2.8921 | LR: 1.24e-04 | 2198.3ms/step | 9.1 samples/s +Step 29640/44000 | Loss: 2.8960 | LR: 1.24e-04 | 2331.6ms/step | 8.6 samples/s +Step 29650/44000 | Loss: 2.8928 | LR: 1.24e-04 | 2191.7ms/step | 9.1 samples/s +Step 29660/44000 | Loss: 2.9483 | LR: 1.24e-04 | 2323.5ms/step | 8.6 samples/s +Step 29670/44000 | Loss: 2.9193 | LR: 1.24e-04 | 2191.0ms/step | 9.1 samples/s +Step 29680/44000 | Loss: 2.9241 | LR: 1.23e-04 | 2305.8ms/step | 8.7 samples/s +Step 29690/44000 | Loss: 2.8916 | LR: 1.23e-04 | 2312.7ms/step | 8.6 samples/s +Step 29700/44000 | Loss: 2.8494 | LR: 1.23e-04 | 2202.3ms/step | 9.1 samples/s +Step 29710/44000 | Loss: 2.8974 | LR: 1.23e-04 | 2168.4ms/step | 9.2 samples/s +Step 29720/44000 | Loss: 2.9070 | LR: 1.23e-04 | 2140.6ms/step | 9.3 samples/s +Step 29730/44000 | Loss: 2.8987 | LR: 1.23e-04 | 2151.2ms/step | 9.3 samples/s +Step 29740/44000 | Loss: 2.9078 | LR: 1.23e-04 | 2131.5ms/step | 9.4 samples/s +Step 29750/44000 | Loss: 2.9194 | LR: 1.22e-04 | 2164.9ms/step | 9.2 samples/s +Step 29760/44000 | Loss: 2.8898 | LR: 1.22e-04 | 2132.7ms/step | 9.4 samples/s +Step 29770/44000 | Loss: 2.9329 | LR: 1.22e-04 | 2138.1ms/step | 9.4 samples/s +Step 29780/44000 | Loss: 2.8701 | LR: 1.22e-04 | 2180.2ms/step | 9.2 samples/s +Step 29790/44000 | Loss: 2.8818 | LR: 1.22e-04 | 2139.5ms/step | 9.3 samples/s +Step 29800/44000 | Loss: 2.9571 | LR: 1.22e-04 | 2129.8ms/step | 9.4 samples/s +Step 29810/44000 | Loss: 2.9064 | LR: 1.21e-04 | 2156.8ms/step | 9.3 samples/s +Step 29820/44000 | Loss: 2.8806 | LR: 1.21e-04 | 2165.6ms/step | 9.2 samples/s +Step 29830/44000 | Loss: 2.9118 | LR: 1.21e-04 | 2127.5ms/step | 9.4 samples/s +Step 29840/44000 | Loss: 2.9202 | LR: 1.21e-04 | 2101.9ms/step | 9.5 samples/s +Step 29850/44000 | Loss: 2.8579 | LR: 1.21e-04 | 2111.0ms/step | 9.5 samples/s +Step 29860/44000 | Loss: 2.9007 | LR: 1.21e-04 | 2089.9ms/step | 9.6 samples/s +Step 29870/44000 | Loss: 2.8880 | LR: 1.21e-04 | 2116.4ms/step | 9.5 samples/s +Step 29880/44000 | Loss: 2.8824 | LR: 1.20e-04 | 2090.1ms/step | 9.6 samples/s +Step 29890/44000 | Loss: 2.8895 | LR: 1.20e-04 | 2100.5ms/step | 9.5 samples/s +Step 29900/44000 | Loss: 2.9145 | LR: 1.20e-04 | 2091.8ms/step | 9.6 samples/s +Step 29910/44000 | Loss: 2.8750 | LR: 1.20e-04 | 2118.9ms/step | 9.4 samples/s +Step 29920/44000 | Loss: 2.9122 | LR: 1.20e-04 | 2116.5ms/step | 9.4 samples/s +Step 29930/44000 | Loss: 2.9140 | LR: 1.20e-04 | 2116.7ms/step | 9.4 samples/s +Step 29940/44000 | Loss: 2.8811 | LR: 1.19e-04 | 2110.6ms/step | 9.5 samples/s +Step 29950/44000 | Loss: 2.9161 | LR: 1.19e-04 | 2097.1ms/step | 9.5 samples/s +Step 29960/44000 | Loss: 2.8839 | LR: 1.19e-04 | 2101.9ms/step | 9.5 samples/s +Step 29970/44000 | Loss: 2.9187 | LR: 1.19e-04 | 2105.4ms/step | 9.5 samples/s +Step 29980/44000 | Loss: 2.9315 | LR: 1.19e-04 | 2101.2ms/step | 9.5 samples/s +Step 29990/44000 | Loss: 2.9222 | LR: 1.19e-04 | 2104.3ms/step | 9.5 samples/s +Step 30000/44000 | Loss: 2.8556 | LR: 1.19e-04 | 2101.9ms/step | 9.5 samples/s +Validation at step 30000: Loss = 2.9072, Perplexity = 18.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-30000 +Step 30010/44000 | Loss: 2.8886 | LR: 1.18e-04 | 4208.1ms/step | 4.8 samples/s +Step 30020/44000 | Loss: 2.8855 | LR: 1.18e-04 | 2111.0ms/step | 9.5 samples/s +Step 30030/44000 | Loss: 2.8688 | LR: 1.18e-04 | 2108.2ms/step | 9.5 samples/s +Step 30040/44000 | Loss: 2.8941 | LR: 1.18e-04 | 2115.9ms/step | 9.5 samples/s +Step 30050/44000 | Loss: 2.8916 | LR: 1.18e-04 | 2088.0ms/step | 9.6 samples/s +Step 30060/44000 | Loss: 2.8841 | LR: 1.18e-04 | 2124.6ms/step | 9.4 samples/s +Step 30070/44000 | Loss: 2.9059 | LR: 1.17e-04 | 2106.9ms/step | 9.5 samples/s +Step 30080/44000 | Loss: 2.8973 | LR: 1.17e-04 | 2095.3ms/step | 9.5 samples/s +Step 30090/44000 | Loss: 2.9109 | LR: 1.17e-04 | 2105.3ms/step | 9.5 samples/s +Step 30100/44000 | Loss: 2.9101 | LR: 1.17e-04 | 2110.1ms/step | 9.5 samples/s +Step 30110/44000 | Loss: 2.8902 | LR: 1.17e-04 | 2107.0ms/step | 9.5 samples/s +Step 30120/44000 | Loss: 2.8477 | LR: 1.17e-04 | 2102.0ms/step | 9.5 samples/s +Step 30130/44000 | Loss: 2.8918 | LR: 1.17e-04 | 2096.3ms/step | 9.5 samples/s +Step 30140/44000 | Loss: 2.9169 | LR: 1.16e-04 | 2022.7ms/step | 9.9 samples/s +Step 30150/44000 | Loss: 2.8573 | LR: 1.16e-04 | 2119.5ms/step | 9.4 samples/s +Step 30160/44000 | Loss: 2.9149 | LR: 1.16e-04 | 2102.5ms/step | 9.5 samples/s +Step 30170/44000 | Loss: 2.9005 | LR: 1.16e-04 | 2122.3ms/step | 9.4 samples/s +Step 30180/44000 | Loss: 2.9228 | LR: 1.16e-04 | 2098.4ms/step | 9.5 samples/s +Step 30190/44000 | Loss: 2.8499 | LR: 1.16e-04 | 2106.7ms/step | 9.5 samples/s +Step 30200/44000 | Loss: 2.8960 | LR: 1.15e-04 | 2106.3ms/step | 9.5 samples/s +Step 30210/44000 | Loss: 2.9114 | LR: 1.15e-04 | 2118.4ms/step | 9.4 samples/s +Step 30220/44000 | Loss: 2.9723 | LR: 1.15e-04 | 2090.5ms/step | 9.6 samples/s +Step 30230/44000 | Loss: 2.9105 | LR: 1.15e-04 | 2100.9ms/step | 9.5 samples/s +Step 30240/44000 | Loss: 2.9511 | LR: 1.15e-04 | 2099.1ms/step | 9.5 samples/s +Step 30250/44000 | Loss: 2.9154 | LR: 1.15e-04 | 2119.6ms/step | 9.4 samples/s +Step 30260/44000 | Loss: 2.9312 | LR: 1.15e-04 | 2096.5ms/step | 9.5 samples/s +Step 30270/44000 | Loss: 2.9237 | LR: 1.14e-04 | 2104.9ms/step | 9.5 samples/s +Step 30280/44000 | Loss: 2.8930 | LR: 1.14e-04 | 2107.4ms/step | 9.5 samples/s +Step 30290/44000 | Loss: 2.9094 | LR: 1.14e-04 | 2097.3ms/step | 9.5 samples/s +Step 30300/44000 | Loss: 2.9302 | LR: 1.14e-04 | 2101.6ms/step | 9.5 samples/s +Step 30310/44000 | Loss: 2.8941 | LR: 1.14e-04 | 2106.8ms/step | 9.5 samples/s +Step 30320/44000 | Loss: 2.8724 | LR: 1.14e-04 | 2104.6ms/step | 9.5 samples/s +Step 30330/44000 | Loss: 2.8638 | LR: 1.13e-04 | 2127.4ms/step | 9.4 samples/s +Step 30340/44000 | Loss: 2.9230 | LR: 1.13e-04 | 2079.5ms/step | 9.6 samples/s +Step 30350/44000 | Loss: 2.9319 | LR: 1.13e-04 | 2100.6ms/step | 9.5 samples/s +Step 30360/44000 | Loss: 2.9173 | LR: 1.13e-04 | 2114.3ms/step | 9.5 samples/s +Step 30370/44000 | Loss: 2.9623 | LR: 1.13e-04 | 2101.1ms/step | 9.5 samples/s +Step 30380/44000 | Loss: 2.9059 | LR: 1.13e-04 | 2120.8ms/step | 9.4 samples/s +Step 30390/44000 | Loss: 2.8754 | LR: 1.13e-04 | 2097.8ms/step | 9.5 samples/s +Step 30400/44000 | Loss: 2.9020 | LR: 1.12e-04 | 2106.9ms/step | 9.5 samples/s +Step 30410/44000 | Loss: 2.8806 | LR: 1.12e-04 | 2101.1ms/step | 9.5 samples/s +Step 30420/44000 | Loss: 2.8847 | LR: 1.12e-04 | 2084.9ms/step | 9.6 samples/s +Step 30430/44000 | Loss: 2.8845 | LR: 1.12e-04 | 2108.1ms/step | 9.5 samples/s +Step 30440/44000 | Loss: 2.8830 | LR: 1.12e-04 | 2101.0ms/step | 9.5 samples/s +Step 30450/44000 | Loss: 2.8950 | LR: 1.12e-04 | 2101.5ms/step | 9.5 samples/s +Step 30460/44000 | Loss: 2.8658 | LR: 1.11e-04 | 2101.5ms/step | 9.5 samples/s +Step 30470/44000 | Loss: 2.9179 | LR: 1.11e-04 | 2095.7ms/step | 9.5 samples/s +Step 30480/44000 | Loss: 2.8592 | LR: 1.11e-04 | 2096.1ms/step | 9.5 samples/s +Step 30490/44000 | Loss: 2.8843 | LR: 1.11e-04 | 2085.8ms/step | 9.6 samples/s +Step 30500/44000 | Loss: 2.8626 | LR: 1.11e-04 | 2117.6ms/step | 9.4 samples/s +Step 30510/44000 | Loss: 2.8897 | LR: 1.11e-04 | 2105.6ms/step | 9.5 samples/s +Step 30520/44000 | Loss: 2.9032 | LR: 1.11e-04 | 2104.8ms/step | 9.5 samples/s +Step 30530/44000 | Loss: 2.8855 | LR: 1.10e-04 | 2119.1ms/step | 9.4 samples/s +Step 30540/44000 | Loss: 2.8681 | LR: 1.10e-04 | 2100.8ms/step | 9.5 samples/s +Step 30550/44000 | Loss: 2.8880 | LR: 1.10e-04 | 2102.4ms/step | 9.5 samples/s +Step 30560/44000 | Loss: 2.8830 | LR: 1.10e-04 | 2089.4ms/step | 9.6 samples/s +Step 30570/44000 | Loss: 2.8995 | LR: 1.10e-04 | 2121.1ms/step | 9.4 samples/s +Step 30580/44000 | Loss: 2.8905 | LR: 1.10e-04 | 2095.9ms/step | 9.5 samples/s +Step 30590/44000 | Loss: 2.8978 | LR: 1.10e-04 | 2108.3ms/step | 9.5 samples/s +Step 30600/44000 | Loss: 2.9151 | LR: 1.09e-04 | 2103.8ms/step | 9.5 samples/s +Step 30610/44000 | Loss: 2.9348 | LR: 1.09e-04 | 2085.0ms/step | 9.6 samples/s +Step 30620/44000 | Loss: 2.9472 | LR: 1.09e-04 | 2105.4ms/step | 9.5 samples/s +Step 30630/44000 | Loss: 2.9065 | LR: 1.09e-04 | 2105.9ms/step | 9.5 samples/s +Step 30640/44000 | Loss: 2.9146 | LR: 1.09e-04 | 2111.1ms/step | 9.5 samples/s +Step 30650/44000 | Loss: 2.9042 | LR: 1.09e-04 | 2123.7ms/step | 9.4 samples/s +Step 30660/44000 | Loss: 2.8831 | LR: 1.08e-04 | 2111.6ms/step | 9.5 samples/s +Step 30670/44000 | Loss: 2.8547 | LR: 1.08e-04 | 2121.4ms/step | 9.4 samples/s +Step 30680/44000 | Loss: 2.9075 | LR: 1.08e-04 | 2106.9ms/step | 9.5 samples/s +Step 30690/44000 | Loss: 2.8953 | LR: 1.08e-04 | 2109.7ms/step | 9.5 samples/s +Step 30700/44000 | Loss: 2.8662 | LR: 1.08e-04 | 2104.6ms/step | 9.5 samples/s +Step 30710/44000 | Loss: 2.8920 | LR: 1.08e-04 | 2104.3ms/step | 9.5 samples/s +Step 30720/44000 | Loss: 2.8850 | LR: 1.08e-04 | 2110.1ms/step | 9.5 samples/s +Step 30730/44000 | Loss: 2.8882 | LR: 1.07e-04 | 2107.5ms/step | 9.5 samples/s +Step 30740/44000 | Loss: 2.9295 | LR: 1.07e-04 | 2099.7ms/step | 9.5 samples/s +Step 30750/44000 | Loss: 2.8520 | LR: 1.07e-04 | 2089.1ms/step | 9.6 samples/s +Step 30760/44000 | Loss: 2.9013 | LR: 1.07e-04 | 2096.5ms/step | 9.5 samples/s +Step 30770/44000 | Loss: 2.8893 | LR: 1.07e-04 | 2124.5ms/step | 9.4 samples/s +Step 30780/44000 | Loss: 2.9032 | LR: 1.07e-04 | 2102.5ms/step | 9.5 samples/s +Step 30790/44000 | Loss: 2.9126 | LR: 1.07e-04 | 2101.3ms/step | 9.5 samples/s +Step 30800/44000 | Loss: 2.8700 | LR: 1.06e-04 | 2082.3ms/step | 9.6 samples/s +Step 30810/44000 | Loss: 2.9337 | LR: 1.06e-04 | 2094.2ms/step | 9.6 samples/s +Step 30820/44000 | Loss: 2.8730 | LR: 1.06e-04 | 2099.5ms/step | 9.5 samples/s +Step 30830/44000 | Loss: 2.8978 | LR: 1.06e-04 | 2103.5ms/step | 9.5 samples/s +Step 30840/44000 | Loss: 2.8492 | LR: 1.06e-04 | 2139.2ms/step | 9.3 samples/s +Step 30850/44000 | Loss: 2.9293 | LR: 1.06e-04 | 2117.7ms/step | 9.4 samples/s +Step 30860/44000 | Loss: 2.9271 | LR: 1.05e-04 | 2105.0ms/step | 9.5 samples/s +Step 30870/44000 | Loss: 2.9199 | LR: 1.05e-04 | 2125.2ms/step | 9.4 samples/s +Step 30880/44000 | Loss: 2.9489 | LR: 1.05e-04 | 2117.3ms/step | 9.4 samples/s +Step 30890/44000 | Loss: 2.9105 | LR: 1.05e-04 | 2107.0ms/step | 9.5 samples/s +Step 30900/44000 | Loss: 2.9273 | LR: 1.05e-04 | 2110.6ms/step | 9.5 samples/s +Step 30910/44000 | Loss: 2.9275 | LR: 1.05e-04 | 2109.2ms/step | 9.5 samples/s +Step 30920/44000 | Loss: 2.9290 | LR: 1.05e-04 | 2101.3ms/step | 9.5 samples/s +Step 30930/44000 | Loss: 2.9244 | LR: 1.04e-04 | 2107.0ms/step | 9.5 samples/s +Step 30940/44000 | Loss: 2.8517 | LR: 1.04e-04 | 2099.4ms/step | 9.5 samples/s +Step 30950/44000 | Loss: 2.8934 | LR: 1.04e-04 | 2120.4ms/step | 9.4 samples/s +Step 30960/44000 | Loss: 2.8838 | LR: 1.04e-04 | 2096.5ms/step | 9.5 samples/s +Step 30970/44000 | Loss: 2.8879 | LR: 1.04e-04 | 2112.1ms/step | 9.5 samples/s +Step 30980/44000 | Loss: 2.8863 | LR: 1.04e-04 | 2091.1ms/step | 9.6 samples/s +Step 30990/44000 | Loss: 2.8988 | LR: 1.04e-04 | 2097.9ms/step | 9.5 samples/s +Step 31000/44000 | Loss: 2.9014 | LR: 1.03e-04 | 2125.6ms/step | 9.4 samples/s +Validation at step 31000: Loss = 2.9039, Perplexity = 18.25 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-31000 +Step 31010/44000 | Loss: 2.9085 | LR: 1.03e-04 | 4342.9ms/step | 4.6 samples/s +Step 31020/44000 | Loss: 2.9055 | LR: 1.03e-04 | 2105.1ms/step | 9.5 samples/s +Step 31030/44000 | Loss: 2.8716 | LR: 1.03e-04 | 2111.8ms/step | 9.5 samples/s +Step 31040/44000 | Loss: 2.9109 | LR: 1.03e-04 | 2090.6ms/step | 9.6 samples/s +Step 31050/44000 | Loss: 2.9078 | LR: 1.03e-04 | 2111.5ms/step | 9.5 samples/s +Step 31060/44000 | Loss: 2.8870 | LR: 1.03e-04 | 2095.3ms/step | 9.5 samples/s +Step 31070/44000 | Loss: 2.8835 | LR: 1.02e-04 | 2114.9ms/step | 9.5 samples/s +Step 31080/44000 | Loss: 2.9252 | LR: 1.02e-04 | 2236.3ms/step | 8.9 samples/s +Step 31090/44000 | Loss: 2.9265 | LR: 1.02e-04 | 2223.0ms/step | 9.0 samples/s +Step 31100/44000 | Loss: 2.8477 | LR: 1.02e-04 | 2254.2ms/step | 8.9 samples/s +Step 31110/44000 | Loss: 2.8892 | LR: 1.02e-04 | 2235.2ms/step | 8.9 samples/s +Step 31120/44000 | Loss: 2.8690 | LR: 1.02e-04 | 2220.2ms/step | 9.0 samples/s +Step 31130/44000 | Loss: 2.9091 | LR: 1.02e-04 | 2207.4ms/step | 9.1 samples/s +Step 31140/44000 | Loss: 2.8872 | LR: 1.01e-04 | 2112.9ms/step | 9.5 samples/s +Step 31150/44000 | Loss: 2.8897 | LR: 1.01e-04 | 2203.1ms/step | 9.1 samples/s +Step 31160/44000 | Loss: 2.8834 | LR: 1.01e-04 | 2315.6ms/step | 8.6 samples/s +Step 31170/44000 | Loss: 2.8999 | LR: 1.01e-04 | 2296.2ms/step | 8.7 samples/s +Step 31180/44000 | Loss: 2.8807 | LR: 1.01e-04 | 2246.9ms/step | 8.9 samples/s +Step 31190/44000 | Loss: 2.9025 | LR: 1.01e-04 | 2196.3ms/step | 9.1 samples/s +Step 31200/44000 | Loss: 2.9167 | LR: 1.00e-04 | 2214.0ms/step | 9.0 samples/s +Step 31210/44000 | Loss: 2.9088 | LR: 1.00e-04 | 2225.5ms/step | 9.0 samples/s +Step 31220/44000 | Loss: 2.8615 | LR: 1.00e-04 | 2243.2ms/step | 8.9 samples/s +Step 31230/44000 | Loss: 2.8754 | LR: 1.00e-04 | 2236.9ms/step | 8.9 samples/s +Step 31240/44000 | Loss: 2.9007 | LR: 9.99e-05 | 2262.7ms/step | 8.8 samples/s +Step 31250/44000 | Loss: 2.9487 | LR: 9.98e-05 | 2226.4ms/step | 9.0 samples/s +Step 31260/44000 | Loss: 2.8585 | LR: 9.96e-05 | 2197.4ms/step | 9.1 samples/s +Step 31270/44000 | Loss: 2.9506 | LR: 9.95e-05 | 2252.8ms/step | 8.9 samples/s +Step 31280/44000 | Loss: 2.8701 | LR: 9.93e-05 | 2226.0ms/step | 9.0 samples/s +Step 31290/44000 | Loss: 2.8513 | LR: 9.92e-05 | 2219.1ms/step | 9.0 samples/s +Step 31300/44000 | Loss: 2.9042 | LR: 9.90e-05 | 2215.3ms/step | 9.0 samples/s +Step 31310/44000 | Loss: 2.9169 | LR: 9.89e-05 | 2211.8ms/step | 9.0 samples/s +Step 31320/44000 | Loss: 2.8926 | LR: 9.88e-05 | 2225.2ms/step | 9.0 samples/s +Step 31330/44000 | Loss: 2.8804 | LR: 9.86e-05 | 2227.3ms/step | 9.0 samples/s +Step 31340/44000 | Loss: 2.8979 | LR: 9.85e-05 | 2237.8ms/step | 8.9 samples/s +Step 31350/44000 | Loss: 2.9106 | LR: 9.83e-05 | 2222.3ms/step | 9.0 samples/s +Step 31360/44000 | Loss: 2.8671 | LR: 9.82e-05 | 2237.5ms/step | 8.9 samples/s +Step 31370/44000 | Loss: 2.9113 | LR: 9.80e-05 | 2261.4ms/step | 8.8 samples/s +Step 31380/44000 | Loss: 2.9429 | LR: 9.79e-05 | 2253.3ms/step | 8.9 samples/s +Step 31390/44000 | Loss: 2.8992 | LR: 9.77e-05 | 2248.4ms/step | 8.9 samples/s +Step 31400/44000 | Loss: 2.8999 | LR: 9.76e-05 | 2260.0ms/step | 8.8 samples/s +Step 31410/44000 | Loss: 2.8670 | LR: 9.75e-05 | 2300.1ms/step | 8.7 samples/s +Step 31420/44000 | Loss: 2.9201 | LR: 9.73e-05 | 2320.6ms/step | 8.6 samples/s +Step 31430/44000 | Loss: 2.9143 | LR: 9.72e-05 | 2279.7ms/step | 8.8 samples/s +Step 31440/44000 | Loss: 2.9020 | LR: 9.70e-05 | 2269.7ms/step | 8.8 samples/s +Step 31450/44000 | Loss: 2.8926 | LR: 9.69e-05 | 2257.9ms/step | 8.9 samples/s +Step 31460/44000 | Loss: 2.8966 | LR: 9.67e-05 | 2280.4ms/step | 8.8 samples/s +Step 31470/44000 | Loss: 2.9462 | LR: 9.66e-05 | 2283.1ms/step | 8.8 samples/s +Step 31480/44000 | Loss: 2.8500 | LR: 9.65e-05 | 2279.1ms/step | 8.8 samples/s +Step 31490/44000 | Loss: 2.9149 | LR: 9.63e-05 | 2231.3ms/step | 9.0 samples/s +Step 31500/44000 | Loss: 2.8534 | LR: 9.62e-05 | 2123.8ms/step | 9.4 samples/s +Step 31510/44000 | Loss: 2.8999 | LR: 9.60e-05 | 2090.9ms/step | 9.6 samples/s +Step 31520/44000 | Loss: 2.8733 | LR: 9.59e-05 | 2105.7ms/step | 9.5 samples/s +Step 31530/44000 | Loss: 2.8745 | LR: 9.57e-05 | 2149.9ms/step | 9.3 samples/s +Step 31540/44000 | Loss: 2.8936 | LR: 9.56e-05 | 2148.2ms/step | 9.3 samples/s +Step 31550/44000 | Loss: 2.8754 | LR: 9.54e-05 | 2187.5ms/step | 9.1 samples/s +Step 31560/44000 | Loss: 2.8940 | LR: 9.53e-05 | 2130.6ms/step | 9.4 samples/s +Step 31570/44000 | Loss: 2.8700 | LR: 9.52e-05 | 2101.4ms/step | 9.5 samples/s +Step 31580/44000 | Loss: 2.9153 | LR: 9.50e-05 | 2110.0ms/step | 9.5 samples/s +Step 31590/44000 | Loss: 2.8499 | LR: 9.49e-05 | 2131.2ms/step | 9.4 samples/s +Step 31600/44000 | Loss: 2.8880 | LR: 9.47e-05 | 2129.1ms/step | 9.4 samples/s +Step 31610/44000 | Loss: 2.8814 | LR: 9.46e-05 | 2220.7ms/step | 9.0 samples/s +Step 31620/44000 | Loss: 2.9214 | LR: 9.45e-05 | 2229.9ms/step | 9.0 samples/s +Step 31630/44000 | Loss: 2.9035 | LR: 9.43e-05 | 2239.8ms/step | 8.9 samples/s +Step 31640/44000 | Loss: 2.9026 | LR: 9.42e-05 | 2108.9ms/step | 9.5 samples/s +Step 31650/44000 | Loss: 2.8465 | LR: 9.40e-05 | 2098.9ms/step | 9.5 samples/s +Step 31660/44000 | Loss: 2.8890 | LR: 9.39e-05 | 2127.0ms/step | 9.4 samples/s +Step 31670/44000 | Loss: 2.8468 | LR: 9.37e-05 | 2158.1ms/step | 9.3 samples/s +Step 31680/44000 | Loss: 2.8811 | LR: 9.36e-05 | 2243.4ms/step | 8.9 samples/s +Step 31690/44000 | Loss: 2.9109 | LR: 9.35e-05 | 2200.8ms/step | 9.1 samples/s +Step 31700/44000 | Loss: 2.8981 | LR: 9.33e-05 | 2158.6ms/step | 9.3 samples/s +Step 31710/44000 | Loss: 2.8881 | LR: 9.32e-05 | 2201.1ms/step | 9.1 samples/s +Step 31720/44000 | Loss: 2.9098 | LR: 9.30e-05 | 2221.8ms/step | 9.0 samples/s +Step 31730/44000 | Loss: 2.8935 | LR: 9.29e-05 | 2222.0ms/step | 9.0 samples/s +Step 31740/44000 | Loss: 2.8396 | LR: 9.28e-05 | 2201.7ms/step | 9.1 samples/s +Step 31750/44000 | Loss: 2.8807 | LR: 9.26e-05 | 2243.5ms/step | 8.9 samples/s +Step 31760/44000 | Loss: 2.9315 | LR: 9.25e-05 | 2200.0ms/step | 9.1 samples/s +Step 31770/44000 | Loss: 2.8821 | LR: 9.23e-05 | 2196.7ms/step | 9.1 samples/s +Step 31780/44000 | Loss: 2.9209 | LR: 9.22e-05 | 2244.2ms/step | 8.9 samples/s +Step 31790/44000 | Loss: 2.8939 | LR: 9.20e-05 | 2237.1ms/step | 8.9 samples/s +Step 31800/44000 | Loss: 2.8708 | LR: 9.19e-05 | 2156.1ms/step | 9.3 samples/s +Step 31810/44000 | Loss: 2.8861 | LR: 9.18e-05 | 2255.5ms/step | 8.9 samples/s +Step 31820/44000 | Loss: 2.9279 | LR: 9.16e-05 | 2190.4ms/step | 9.1 samples/s +Step 31830/44000 | Loss: 2.8871 | LR: 9.15e-05 | 2237.0ms/step | 8.9 samples/s +Step 31840/44000 | Loss: 2.8907 | LR: 9.13e-05 | 2176.1ms/step | 9.2 samples/s +Step 31850/44000 | Loss: 2.9524 | LR: 9.12e-05 | 2206.1ms/step | 9.1 samples/s +Step 31860/44000 | Loss: 2.8382 | LR: 9.11e-05 | 2251.3ms/step | 8.9 samples/s +Step 31870/44000 | Loss: 2.8566 | LR: 9.09e-05 | 2151.7ms/step | 9.3 samples/s +Step 31880/44000 | Loss: 2.9200 | LR: 9.08e-05 | 2221.4ms/step | 9.0 samples/s +Step 31890/44000 | Loss: 2.8360 | LR: 9.06e-05 | 2216.9ms/step | 9.0 samples/s +Step 31900/44000 | Loss: 2.8930 | LR: 9.05e-05 | 2261.6ms/step | 8.8 samples/s +Step 31910/44000 | Loss: 2.9072 | LR: 9.04e-05 | 2186.1ms/step | 9.1 samples/s +Step 31920/44000 | Loss: 2.8686 | LR: 9.02e-05 | 2190.8ms/step | 9.1 samples/s +Step 31930/44000 | Loss: 2.8747 | LR: 9.01e-05 | 2168.2ms/step | 9.2 samples/s +Step 31940/44000 | Loss: 2.8804 | LR: 8.99e-05 | 2190.5ms/step | 9.1 samples/s +Step 31950/44000 | Loss: 2.9481 | LR: 8.98e-05 | 2191.9ms/step | 9.1 samples/s +Step 31960/44000 | Loss: 2.9411 | LR: 8.97e-05 | 2168.5ms/step | 9.2 samples/s +Step 31970/44000 | Loss: 2.8625 | LR: 8.95e-05 | 2238.6ms/step | 8.9 samples/s +Step 31980/44000 | Loss: 2.8750 | LR: 8.94e-05 | 2200.5ms/step | 9.1 samples/s +Step 31990/44000 | Loss: 2.8747 | LR: 8.92e-05 | 2196.4ms/step | 9.1 samples/s +Step 32000/44000 | Loss: 2.8608 | LR: 8.91e-05 | 2201.0ms/step | 9.1 samples/s +Validation at step 32000: Loss = 2.9013, Perplexity = 18.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-32000 +Step 32010/44000 | Loss: 2.8663 | LR: 8.90e-05 | 4496.9ms/step | 4.4 samples/s +Step 32020/44000 | Loss: 2.8782 | LR: 8.88e-05 | 2173.7ms/step | 9.2 samples/s +Step 32030/44000 | Loss: 2.8603 | LR: 8.87e-05 | 2331.5ms/step | 8.6 samples/s +Step 32040/44000 | Loss: 2.9381 | LR: 8.86e-05 | 2434.6ms/step | 8.2 samples/s +Step 32050/44000 | Loss: 2.8604 | LR: 8.84e-05 | 2385.3ms/step | 8.4 samples/s +Step 32060/44000 | Loss: 2.9050 | LR: 8.83e-05 | 2429.1ms/step | 8.2 samples/s +Step 32070/44000 | Loss: 2.9189 | LR: 8.81e-05 | 2358.7ms/step | 8.5 samples/s +Step 32080/44000 | Loss: 2.8720 | LR: 8.80e-05 | 2416.1ms/step | 8.3 samples/s +Step 32090/44000 | Loss: 2.9036 | LR: 8.79e-05 | 2381.6ms/step | 8.4 samples/s +Step 32100/44000 | Loss: 2.8748 | LR: 8.77e-05 | 2364.8ms/step | 8.5 samples/s +Step 32110/44000 | Loss: 2.8972 | LR: 8.76e-05 | 2328.9ms/step | 8.6 samples/s +Step 32120/44000 | Loss: 2.9120 | LR: 8.74e-05 | 2207.2ms/step | 9.1 samples/s +Step 32130/44000 | Loss: 2.8839 | LR: 8.73e-05 | 2339.1ms/step | 8.6 samples/s +Step 32140/44000 | Loss: 2.9103 | LR: 8.72e-05 | 2421.4ms/step | 8.3 samples/s +Step 32150/44000 | Loss: 2.8967 | LR: 8.70e-05 | 2360.9ms/step | 8.5 samples/s +Step 32160/44000 | Loss: 2.8812 | LR: 8.69e-05 | 2371.7ms/step | 8.4 samples/s +Step 32170/44000 | Loss: 2.8797 | LR: 8.68e-05 | 2342.9ms/step | 8.5 samples/s +Step 32180/44000 | Loss: 2.9056 | LR: 8.66e-05 | 2337.4ms/step | 8.6 samples/s +Step 32190/44000 | Loss: 2.9137 | LR: 8.65e-05 | 2349.4ms/step | 8.5 samples/s +Step 32200/44000 | Loss: 2.8896 | LR: 8.63e-05 | 2395.2ms/step | 8.4 samples/s +Step 32210/44000 | Loss: 2.8945 | LR: 8.62e-05 | 2404.4ms/step | 8.3 samples/s +Step 32220/44000 | Loss: 2.8805 | LR: 8.61e-05 | 2417.8ms/step | 8.3 samples/s +Step 32230/44000 | Loss: 2.8067 | LR: 8.59e-05 | 2410.6ms/step | 8.3 samples/s +Step 32240/44000 | Loss: 2.8717 | LR: 8.58e-05 | 2408.6ms/step | 8.3 samples/s +Step 32250/44000 | Loss: 2.8671 | LR: 8.57e-05 | 2400.2ms/step | 8.3 samples/s +Step 32260/44000 | Loss: 2.8966 | LR: 8.55e-05 | 2349.7ms/step | 8.5 samples/s +Step 32270/44000 | Loss: 2.8720 | LR: 8.54e-05 | 2399.4ms/step | 8.3 samples/s +Step 32280/44000 | Loss: 2.8903 | LR: 8.53e-05 | 2353.0ms/step | 8.5 samples/s +Step 32290/44000 | Loss: 2.8753 | LR: 8.51e-05 | 2373.6ms/step | 8.4 samples/s +Step 32300/44000 | Loss: 2.8917 | LR: 8.50e-05 | 2434.5ms/step | 8.2 samples/s +Step 32310/44000 | Loss: 2.8908 | LR: 8.48e-05 | 2425.8ms/step | 8.2 samples/s +Step 32320/44000 | Loss: 2.8579 | LR: 8.47e-05 | 2385.8ms/step | 8.4 samples/s +Step 32330/44000 | Loss: 2.8633 | LR: 8.46e-05 | 2419.1ms/step | 8.3 samples/s +Step 32340/44000 | Loss: 2.8951 | LR: 8.44e-05 | 2391.7ms/step | 8.4 samples/s +Step 32350/44000 | Loss: 2.9298 | LR: 8.43e-05 | 2347.9ms/step | 8.5 samples/s +Step 32360/44000 | Loss: 2.9277 | LR: 8.42e-05 | 2358.1ms/step | 8.5 samples/s +Step 32370/44000 | Loss: 2.9209 | LR: 8.40e-05 | 2396.8ms/step | 8.3 samples/s +Step 32380/44000 | Loss: 2.8903 | LR: 8.39e-05 | 2385.4ms/step | 8.4 samples/s +Step 32390/44000 | Loss: 2.8882 | LR: 8.38e-05 | 2359.1ms/step | 8.5 samples/s +Step 32400/44000 | Loss: 2.9384 | LR: 8.36e-05 | 2348.5ms/step | 8.5 samples/s +Step 32410/44000 | Loss: 2.8931 | LR: 8.35e-05 | 2352.7ms/step | 8.5 samples/s +Step 32420/44000 | Loss: 2.8818 | LR: 8.33e-05 | 2376.5ms/step | 8.4 samples/s +Step 32430/44000 | Loss: 2.8888 | LR: 8.32e-05 | 2415.7ms/step | 8.3 samples/s +Step 32440/44000 | Loss: 2.9183 | LR: 8.31e-05 | 2346.7ms/step | 8.5 samples/s +Step 32450/44000 | Loss: 2.8859 | LR: 8.29e-05 | 2400.7ms/step | 8.3 samples/s +Step 32460/44000 | Loss: 2.8790 | LR: 8.28e-05 | 2401.1ms/step | 8.3 samples/s +Step 32470/44000 | Loss: 2.8738 | LR: 8.27e-05 | 2395.9ms/step | 8.3 samples/s +Step 32480/44000 | Loss: 2.8663 | LR: 8.25e-05 | 2367.0ms/step | 8.4 samples/s +Step 32490/44000 | Loss: 2.8706 | LR: 8.24e-05 | 2378.8ms/step | 8.4 samples/s +Step 32500/44000 | Loss: 2.8494 | LR: 8.23e-05 | 2373.3ms/step | 8.4 samples/s +Step 32510/44000 | Loss: 2.9090 | LR: 8.21e-05 | 2358.3ms/step | 8.5 samples/s +Step 32520/44000 | Loss: 2.9038 | LR: 8.20e-05 | 2384.2ms/step | 8.4 samples/s +Step 32530/44000 | Loss: 2.9514 | LR: 8.19e-05 | 2410.8ms/step | 8.3 samples/s +Step 32540/44000 | Loss: 2.8926 | LR: 8.17e-05 | 2391.7ms/step | 8.4 samples/s +Step 32550/44000 | Loss: 2.8984 | LR: 8.16e-05 | 2355.5ms/step | 8.5 samples/s +Step 32560/44000 | Loss: 2.9508 | LR: 8.15e-05 | 2377.7ms/step | 8.4 samples/s +Step 32570/44000 | Loss: 2.8825 | LR: 8.13e-05 | 2402.7ms/step | 8.3 samples/s +Step 32580/44000 | Loss: 2.8999 | LR: 8.12e-05 | 2314.0ms/step | 8.6 samples/s +Step 32590/44000 | Loss: 2.8813 | LR: 8.11e-05 | 2381.0ms/step | 8.4 samples/s +Step 32600/44000 | Loss: 2.8973 | LR: 8.09e-05 | 2358.7ms/step | 8.5 samples/s +Step 32610/44000 | Loss: 2.8703 | LR: 8.08e-05 | 2368.0ms/step | 8.4 samples/s +Step 32620/44000 | Loss: 2.9168 | LR: 8.07e-05 | 2344.2ms/step | 8.5 samples/s +Step 32630/44000 | Loss: 2.8787 | LR: 8.05e-05 | 2318.2ms/step | 8.6 samples/s +Step 32640/44000 | Loss: 2.8888 | LR: 8.04e-05 | 2375.8ms/step | 8.4 samples/s +Step 32650/44000 | Loss: 2.9123 | LR: 8.03e-05 | 2347.6ms/step | 8.5 samples/s +Step 32660/44000 | Loss: 2.8959 | LR: 8.01e-05 | 2288.8ms/step | 8.7 samples/s +Step 32670/44000 | Loss: 2.8910 | LR: 8.00e-05 | 2321.8ms/step | 8.6 samples/s +Step 32680/44000 | Loss: 2.9038 | LR: 7.99e-05 | 2368.9ms/step | 8.4 samples/s +Step 32690/44000 | Loss: 2.9150 | LR: 7.97e-05 | 2374.3ms/step | 8.4 samples/s +Step 32700/44000 | Loss: 2.8598 | LR: 7.96e-05 | 2357.8ms/step | 8.5 samples/s +Step 32710/44000 | Loss: 2.9428 | LR: 7.95e-05 | 2361.0ms/step | 8.5 samples/s +Step 32720/44000 | Loss: 2.9132 | LR: 7.93e-05 | 2391.3ms/step | 8.4 samples/s +Step 32730/44000 | Loss: 2.8377 | LR: 7.92e-05 | 2318.5ms/step | 8.6 samples/s +Step 32740/44000 | Loss: 2.9076 | LR: 7.91e-05 | 2345.5ms/step | 8.5 samples/s +Step 32750/44000 | Loss: 2.8855 | LR: 7.89e-05 | 2366.2ms/step | 8.5 samples/s +Step 32760/44000 | Loss: 2.8897 | LR: 7.88e-05 | 2344.2ms/step | 8.5 samples/s +Step 32770/44000 | Loss: 2.8759 | LR: 7.87e-05 | 2330.6ms/step | 8.6 samples/s +Step 32780/44000 | Loss: 2.8864 | LR: 7.85e-05 | 2345.0ms/step | 8.5 samples/s +Step 32790/44000 | Loss: 2.8995 | LR: 7.84e-05 | 2340.4ms/step | 8.5 samples/s +Step 32800/44000 | Loss: 2.8836 | LR: 7.83e-05 | 2357.0ms/step | 8.5 samples/s +Step 32810/44000 | Loss: 2.8885 | LR: 7.81e-05 | 2391.1ms/step | 8.4 samples/s +Step 32820/44000 | Loss: 2.8539 | LR: 7.80e-05 | 2379.5ms/step | 8.4 samples/s +Step 32830/44000 | Loss: 2.8839 | LR: 7.79e-05 | 2373.2ms/step | 8.4 samples/s +Step 32840/44000 | Loss: 2.8495 | LR: 7.77e-05 | 2369.0ms/step | 8.4 samples/s +Step 32850/44000 | Loss: 2.9226 | LR: 7.76e-05 | 2367.8ms/step | 8.4 samples/s +Step 32860/44000 | Loss: 2.8811 | LR: 7.75e-05 | 2385.2ms/step | 8.4 samples/s +Step 32870/44000 | Loss: 2.8726 | LR: 7.73e-05 | 2353.7ms/step | 8.5 samples/s +Step 32880/44000 | Loss: 2.8718 | LR: 7.72e-05 | 2373.5ms/step | 8.4 samples/s +Step 32890/44000 | Loss: 2.8943 | LR: 7.71e-05 | 2360.2ms/step | 8.5 samples/s +Step 32900/44000 | Loss: 2.9271 | LR: 7.70e-05 | 2383.3ms/step | 8.4 samples/s +Step 32910/44000 | Loss: 2.9014 | LR: 7.68e-05 | 2304.0ms/step | 8.7 samples/s +Step 32920/44000 | Loss: 2.9341 | LR: 7.67e-05 | 2398.7ms/step | 8.3 samples/s +Step 32930/44000 | Loss: 2.9142 | LR: 7.66e-05 | 2356.2ms/step | 8.5 samples/s +Step 32940/44000 | Loss: 2.9066 | LR: 7.64e-05 | 2360.7ms/step | 8.5 samples/s +Step 32950/44000 | Loss: 2.8928 | LR: 7.63e-05 | 2345.2ms/step | 8.5 samples/s +Step 32960/44000 | Loss: 2.8573 | LR: 7.62e-05 | 2382.9ms/step | 8.4 samples/s +Step 32970/44000 | Loss: 2.8991 | LR: 7.60e-05 | 2333.9ms/step | 8.6 samples/s +Step 32980/44000 | Loss: 2.8884 | LR: 7.59e-05 | 2341.3ms/step | 8.5 samples/s +Step 32990/44000 | Loss: 2.8791 | LR: 7.58e-05 | 2365.1ms/step | 8.5 samples/s +Step 33000/44000 | Loss: 2.8473 | LR: 7.56e-05 | 2412.6ms/step | 8.3 samples/s +Validation at step 33000: Loss = 2.8994, Perplexity = 18.16 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-33000 +Step 33010/44000 | Loss: 2.9088 | LR: 7.55e-05 | 4873.9ms/step | 4.1 samples/s +Step 33020/44000 | Loss: 2.8847 | LR: 7.54e-05 | 2322.8ms/step | 8.6 samples/s +Step 33030/44000 | Loss: 2.9133 | LR: 7.53e-05 | 2358.0ms/step | 8.5 samples/s +Step 33040/44000 | Loss: 2.8613 | LR: 7.51e-05 | 2331.6ms/step | 8.6 samples/s +Step 33050/44000 | Loss: 2.8841 | LR: 7.50e-05 | 2326.4ms/step | 8.6 samples/s +Step 33060/44000 | Loss: 2.8789 | LR: 7.49e-05 | 2344.4ms/step | 8.5 samples/s +Step 33070/44000 | Loss: 2.9017 | LR: 7.47e-05 | 2303.9ms/step | 8.7 samples/s +Step 33080/44000 | Loss: 2.9341 | LR: 7.46e-05 | 2395.2ms/step | 8.3 samples/s +Step 33090/44000 | Loss: 2.8932 | LR: 7.45e-05 | 2340.5ms/step | 8.5 samples/s +Step 33100/44000 | Loss: 2.8772 | LR: 7.44e-05 | 2216.2ms/step | 9.0 samples/s +Step 33110/44000 | Loss: 2.9393 | LR: 7.42e-05 | 2328.0ms/step | 8.6 samples/s +Step 33120/44000 | Loss: 2.9025 | LR: 7.41e-05 | 2368.8ms/step | 8.4 samples/s +Step 33130/44000 | Loss: 2.8613 | LR: 7.40e-05 | 2343.7ms/step | 8.5 samples/s +Step 33140/44000 | Loss: 2.8969 | LR: 7.38e-05 | 2372.6ms/step | 8.4 samples/s +Step 33150/44000 | Loss: 2.8944 | LR: 7.37e-05 | 2339.8ms/step | 8.5 samples/s +Step 33160/44000 | Loss: 2.9450 | LR: 7.36e-05 | 2357.6ms/step | 8.5 samples/s +Step 33170/44000 | Loss: 2.9129 | LR: 7.34e-05 | 2323.1ms/step | 8.6 samples/s +Step 33180/44000 | Loss: 2.9116 | LR: 7.33e-05 | 2400.5ms/step | 8.3 samples/s +Step 33190/44000 | Loss: 2.9100 | LR: 7.32e-05 | 2343.3ms/step | 8.5 samples/s +Step 33200/44000 | Loss: 2.9189 | LR: 7.31e-05 | 2431.8ms/step | 8.2 samples/s +Step 33210/44000 | Loss: 2.8910 | LR: 7.29e-05 | 2410.9ms/step | 8.3 samples/s +Step 33220/44000 | Loss: 2.8467 | LR: 7.28e-05 | 2395.6ms/step | 8.3 samples/s +Step 33230/44000 | Loss: 2.8747 | LR: 7.27e-05 | 2352.3ms/step | 8.5 samples/s +Step 33240/44000 | Loss: 2.8658 | LR: 7.26e-05 | 2401.1ms/step | 8.3 samples/s +Step 33250/44000 | Loss: 2.8825 | LR: 7.24e-05 | 2354.4ms/step | 8.5 samples/s +Step 33260/44000 | Loss: 2.8735 | LR: 7.23e-05 | 2402.3ms/step | 8.3 samples/s +Step 33270/44000 | Loss: 2.8792 | LR: 7.22e-05 | 2380.7ms/step | 8.4 samples/s +Step 33280/44000 | Loss: 2.8991 | LR: 7.20e-05 | 2388.5ms/step | 8.4 samples/s +Step 33290/44000 | Loss: 2.8694 | LR: 7.19e-05 | 2384.1ms/step | 8.4 samples/s +Step 33300/44000 | Loss: 2.8922 | LR: 7.18e-05 | 2336.8ms/step | 8.6 samples/s +Step 33310/44000 | Loss: 2.8400 | LR: 7.17e-05 | 2439.3ms/step | 8.2 samples/s +Step 33320/44000 | Loss: 2.8639 | LR: 7.15e-05 | 2357.4ms/step | 8.5 samples/s +Step 33330/44000 | Loss: 2.8722 | LR: 7.14e-05 | 2361.9ms/step | 8.5 samples/s +Step 33340/44000 | Loss: 2.8757 | LR: 7.13e-05 | 2422.3ms/step | 8.3 samples/s +Step 33350/44000 | Loss: 2.8727 | LR: 7.11e-05 | 2385.6ms/step | 8.4 samples/s +Step 33360/44000 | Loss: 2.8946 | LR: 7.10e-05 | 2374.9ms/step | 8.4 samples/s +Step 33370/44000 | Loss: 2.8969 | LR: 7.09e-05 | 2399.2ms/step | 8.3 samples/s +Step 33380/44000 | Loss: 2.9250 | LR: 7.08e-05 | 2397.7ms/step | 8.3 samples/s +Step 33390/44000 | Loss: 2.9053 | LR: 7.06e-05 | 2333.3ms/step | 8.6 samples/s +Step 33400/44000 | Loss: 2.9191 | LR: 7.05e-05 | 2311.6ms/step | 8.7 samples/s +Step 33410/44000 | Loss: 2.8899 | LR: 7.04e-05 | 2337.6ms/step | 8.6 samples/s +Step 33420/44000 | Loss: 2.8530 | LR: 7.03e-05 | 2353.2ms/step | 8.5 samples/s +Step 33430/44000 | Loss: 2.9160 | LR: 7.01e-05 | 2369.3ms/step | 8.4 samples/s +Step 33440/44000 | Loss: 2.9329 | LR: 7.00e-05 | 2411.8ms/step | 8.3 samples/s +Step 33450/44000 | Loss: 2.8855 | LR: 6.99e-05 | 2343.5ms/step | 8.5 samples/s +Step 33460/44000 | Loss: 2.9250 | LR: 6.98e-05 | 2361.9ms/step | 8.5 samples/s +Step 33470/44000 | Loss: 2.9426 | LR: 6.96e-05 | 2416.3ms/step | 8.3 samples/s +Step 33480/44000 | Loss: 2.9229 | LR: 6.95e-05 | 2372.1ms/step | 8.4 samples/s +Step 33490/44000 | Loss: 2.8601 | LR: 6.94e-05 | 2329.2ms/step | 8.6 samples/s +Step 33500/44000 | Loss: 2.8821 | LR: 6.93e-05 | 2360.0ms/step | 8.5 samples/s +Step 33510/44000 | Loss: 2.9024 | LR: 6.91e-05 | 2339.5ms/step | 8.5 samples/s +Step 33520/44000 | Loss: 2.8692 | LR: 6.90e-05 | 2390.0ms/step | 8.4 samples/s +Step 33530/44000 | Loss: 2.9045 | LR: 6.89e-05 | 2359.9ms/step | 8.5 samples/s +Step 33540/44000 | Loss: 2.9058 | LR: 6.88e-05 | 2374.4ms/step | 8.4 samples/s +Step 33550/44000 | Loss: 2.9086 | LR: 6.86e-05 | 2359.1ms/step | 8.5 samples/s +Step 33560/44000 | Loss: 2.8819 | LR: 6.85e-05 | 2413.0ms/step | 8.3 samples/s +Step 33570/44000 | Loss: 2.8889 | LR: 6.84e-05 | 2350.7ms/step | 8.5 samples/s +Step 33580/44000 | Loss: 2.9095 | LR: 6.83e-05 | 2341.6ms/step | 8.5 samples/s +Step 33590/44000 | Loss: 2.8616 | LR: 6.81e-05 | 2315.0ms/step | 8.6 samples/s +Step 33600/44000 | Loss: 2.8793 | LR: 6.80e-05 | 2355.9ms/step | 8.5 samples/s +Step 33610/44000 | Loss: 2.9031 | LR: 6.79e-05 | 2296.7ms/step | 8.7 samples/s +Step 33620/44000 | Loss: 2.9173 | LR: 6.78e-05 | 2390.9ms/step | 8.4 samples/s +Step 33630/44000 | Loss: 2.9009 | LR: 6.76e-05 | 2369.0ms/step | 8.4 samples/s +Step 33640/44000 | Loss: 2.9255 | LR: 6.75e-05 | 2396.4ms/step | 8.3 samples/s +Step 33650/44000 | Loss: 2.9020 | LR: 6.74e-05 | 2354.4ms/step | 8.5 samples/s +Step 33660/44000 | Loss: 2.8916 | LR: 6.73e-05 | 2413.9ms/step | 8.3 samples/s +Step 33670/44000 | Loss: 2.9068 | LR: 6.71e-05 | 2400.6ms/step | 8.3 samples/s +Step 33680/44000 | Loss: 2.8821 | LR: 6.70e-05 | 2377.2ms/step | 8.4 samples/s +Step 33690/44000 | Loss: 2.8519 | LR: 6.69e-05 | 2361.3ms/step | 8.5 samples/s +Step 33700/44000 | Loss: 2.8775 | LR: 6.68e-05 | 2363.4ms/step | 8.5 samples/s +Step 33710/44000 | Loss: 2.8890 | LR: 6.66e-05 | 2352.8ms/step | 8.5 samples/s +Step 33720/44000 | Loss: 2.9161 | LR: 6.65e-05 | 2390.0ms/step | 8.4 samples/s +Step 33730/44000 | Loss: 2.8929 | LR: 6.64e-05 | 2369.2ms/step | 8.4 samples/s +Step 33740/44000 | Loss: 2.8982 | LR: 6.63e-05 | 2337.2ms/step | 8.6 samples/s +Step 33750/44000 | Loss: 2.9174 | LR: 6.61e-05 | 2368.1ms/step | 8.4 samples/s +Step 33760/44000 | Loss: 2.8608 | LR: 6.60e-05 | 2338.4ms/step | 8.6 samples/s +Step 33770/44000 | Loss: 2.9234 | LR: 6.59e-05 | 2355.6ms/step | 8.5 samples/s +Step 33780/44000 | Loss: 2.8735 | LR: 6.58e-05 | 2332.6ms/step | 8.6 samples/s +Step 33790/44000 | Loss: 2.9234 | LR: 6.57e-05 | 2342.8ms/step | 8.5 samples/s +Step 33800/44000 | Loss: 2.8551 | LR: 6.55e-05 | 2328.1ms/step | 8.6 samples/s +Step 33810/44000 | Loss: 2.8797 | LR: 6.54e-05 | 2313.9ms/step | 8.6 samples/s +Step 33820/44000 | Loss: 2.9128 | LR: 6.53e-05 | 2374.5ms/step | 8.4 samples/s +Step 33830/44000 | Loss: 2.8839 | LR: 6.52e-05 | 2383.8ms/step | 8.4 samples/s +Step 33840/44000 | Loss: 2.9042 | LR: 6.50e-05 | 2415.9ms/step | 8.3 samples/s +Step 33850/44000 | Loss: 2.9042 | LR: 6.49e-05 | 2342.0ms/step | 8.5 samples/s +Step 33860/44000 | Loss: 2.9133 | LR: 6.48e-05 | 2353.4ms/step | 8.5 samples/s +Step 33870/44000 | Loss: 2.9088 | LR: 6.47e-05 | 2367.7ms/step | 8.4 samples/s +Step 33880/44000 | Loss: 2.9035 | LR: 6.46e-05 | 2366.6ms/step | 8.5 samples/s +Step 33890/44000 | Loss: 2.8722 | LR: 6.44e-05 | 2332.3ms/step | 8.6 samples/s +Step 33900/44000 | Loss: 2.8546 | LR: 6.43e-05 | 2379.3ms/step | 8.4 samples/s +Step 33910/44000 | Loss: 2.8946 | LR: 6.42e-05 | 2373.0ms/step | 8.4 samples/s +Step 33920/44000 | Loss: 2.8768 | LR: 6.41e-05 | 2352.8ms/step | 8.5 samples/s +Step 33930/44000 | Loss: 2.8640 | LR: 6.40e-05 | 2327.7ms/step | 8.6 samples/s +Step 33940/44000 | Loss: 2.8831 | LR: 6.38e-05 | 2452.9ms/step | 8.2 samples/s +Step 33950/44000 | Loss: 2.8945 | LR: 6.37e-05 | 2342.7ms/step | 8.5 samples/s +Step 33960/44000 | Loss: 2.8742 | LR: 6.36e-05 | 2357.9ms/step | 8.5 samples/s +Step 33970/44000 | Loss: 2.8836 | LR: 6.35e-05 | 2338.3ms/step | 8.6 samples/s +Step 33980/44000 | Loss: 2.8850 | LR: 6.33e-05 | 2355.5ms/step | 8.5 samples/s +Step 33990/44000 | Loss: 2.8961 | LR: 6.32e-05 | 2393.4ms/step | 8.4 samples/s +Step 34000/44000 | Loss: 2.8791 | LR: 6.31e-05 | 2409.5ms/step | 8.3 samples/s +Validation at step 34000: Loss = 2.8978, Perplexity = 18.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-34000 +Step 34010/44000 | Loss: 2.8817 | LR: 6.30e-05 | 4873.6ms/step | 4.1 samples/s +Step 34020/44000 | Loss: 2.8654 | LR: 6.29e-05 | 2315.6ms/step | 8.6 samples/s +Step 34030/44000 | Loss: 2.8735 | LR: 6.27e-05 | 2405.2ms/step | 8.3 samples/s +Step 34040/44000 | Loss: 2.8621 | LR: 6.26e-05 | 2417.5ms/step | 8.3 samples/s +Step 34050/44000 | Loss: 2.8615 | LR: 6.25e-05 | 2383.6ms/step | 8.4 samples/s +Step 34060/44000 | Loss: 2.8305 | LR: 6.24e-05 | 2370.4ms/step | 8.4 samples/s +Step 34070/44000 | Loss: 2.9041 | LR: 6.23e-05 | 2340.3ms/step | 8.5 samples/s +Step 34080/44000 | Loss: 2.8992 | LR: 6.21e-05 | 2186.0ms/step | 9.1 samples/s +Step 34090/44000 | Loss: 2.8834 | LR: 6.20e-05 | 2389.6ms/step | 8.4 samples/s +Step 34100/44000 | Loss: 2.8909 | LR: 6.19e-05 | 2399.9ms/step | 8.3 samples/s +Step 34110/44000 | Loss: 2.8954 | LR: 6.18e-05 | 2329.0ms/step | 8.6 samples/s +Step 34120/44000 | Loss: 2.8658 | LR: 6.17e-05 | 2413.5ms/step | 8.3 samples/s +Step 34130/44000 | Loss: 2.9002 | LR: 6.15e-05 | 2393.8ms/step | 8.4 samples/s +Step 34140/44000 | Loss: 2.8742 | LR: 6.14e-05 | 2356.7ms/step | 8.5 samples/s +Step 34150/44000 | Loss: 2.8941 | LR: 6.13e-05 | 2364.4ms/step | 8.5 samples/s +Step 34160/44000 | Loss: 2.9195 | LR: 6.12e-05 | 2369.5ms/step | 8.4 samples/s +Step 34170/44000 | Loss: 2.8586 | LR: 6.11e-05 | 2374.8ms/step | 8.4 samples/s +Step 34180/44000 | Loss: 2.9195 | LR: 6.09e-05 | 2301.5ms/step | 8.7 samples/s +Step 34190/44000 | Loss: 2.8754 | LR: 6.08e-05 | 2402.4ms/step | 8.3 samples/s +Step 34200/44000 | Loss: 2.8736 | LR: 6.07e-05 | 2364.5ms/step | 8.5 samples/s +Step 34210/44000 | Loss: 2.8905 | LR: 6.06e-05 | 2356.3ms/step | 8.5 samples/s +Step 34220/44000 | Loss: 2.8410 | LR: 6.05e-05 | 2415.2ms/step | 8.3 samples/s +Step 34230/44000 | Loss: 2.8808 | LR: 6.04e-05 | 2374.3ms/step | 8.4 samples/s +Step 34240/44000 | Loss: 2.8472 | LR: 6.02e-05 | 2395.2ms/step | 8.4 samples/s +Step 34250/44000 | Loss: 2.9259 | LR: 6.01e-05 | 2370.9ms/step | 8.4 samples/s +Step 34260/44000 | Loss: 2.8969 | LR: 6.00e-05 | 2299.6ms/step | 8.7 samples/s +Step 34270/44000 | Loss: 2.9131 | LR: 5.99e-05 | 2324.4ms/step | 8.6 samples/s +Step 34280/44000 | Loss: 2.8824 | LR: 5.98e-05 | 2368.5ms/step | 8.4 samples/s +Step 34290/44000 | Loss: 2.8922 | LR: 5.96e-05 | 2332.8ms/step | 8.6 samples/s +Step 34300/44000 | Loss: 2.8758 | LR: 5.95e-05 | 2322.8ms/step | 8.6 samples/s +Step 34310/44000 | Loss: 2.8881 | LR: 5.94e-05 | 2376.1ms/step | 8.4 samples/s +Step 34320/44000 | Loss: 2.8569 | LR: 5.93e-05 | 2381.2ms/step | 8.4 samples/s +Step 34330/44000 | Loss: 2.8435 | LR: 5.92e-05 | 2358.6ms/step | 8.5 samples/s +Step 34340/44000 | Loss: 2.9165 | LR: 5.91e-05 | 2343.1ms/step | 8.5 samples/s +Step 34350/44000 | Loss: 2.9260 | LR: 5.89e-05 | 2339.9ms/step | 8.5 samples/s +Step 34360/44000 | Loss: 2.8843 | LR: 5.88e-05 | 2356.3ms/step | 8.5 samples/s +Step 34370/44000 | Loss: 2.9033 | LR: 5.87e-05 | 2375.4ms/step | 8.4 samples/s +Step 34380/44000 | Loss: 2.9092 | LR: 5.86e-05 | 2372.7ms/step | 8.4 samples/s +Step 34390/44000 | Loss: 2.9423 | LR: 5.85e-05 | 2420.9ms/step | 8.3 samples/s +Step 34400/44000 | Loss: 2.8685 | LR: 5.84e-05 | 2386.7ms/step | 8.4 samples/s +Step 34410/44000 | Loss: 2.8773 | LR: 5.82e-05 | 2430.0ms/step | 8.2 samples/s +Step 34420/44000 | Loss: 2.8876 | LR: 5.81e-05 | 2363.4ms/step | 8.5 samples/s +Step 34430/44000 | Loss: 2.9250 | LR: 5.80e-05 | 2418.4ms/step | 8.3 samples/s +Step 34440/44000 | Loss: 2.8441 | LR: 5.79e-05 | 2362.2ms/step | 8.5 samples/s +Step 34450/44000 | Loss: 2.8960 | LR: 5.78e-05 | 2376.9ms/step | 8.4 samples/s +Step 34460/44000 | Loss: 2.8746 | LR: 5.77e-05 | 2331.0ms/step | 8.6 samples/s +Step 34470/44000 | Loss: 2.8558 | LR: 5.75e-05 | 2354.8ms/step | 8.5 samples/s +Step 34480/44000 | Loss: 2.8573 | LR: 5.74e-05 | 2346.0ms/step | 8.5 samples/s +Step 34490/44000 | Loss: 2.9033 | LR: 5.73e-05 | 2390.8ms/step | 8.4 samples/s +Step 34500/44000 | Loss: 2.8684 | LR: 5.72e-05 | 2382.7ms/step | 8.4 samples/s +Step 34510/44000 | Loss: 2.8769 | LR: 5.71e-05 | 2360.8ms/step | 8.5 samples/s +Step 34520/44000 | Loss: 2.8958 | LR: 5.70e-05 | 2388.0ms/step | 8.4 samples/s +Step 34530/44000 | Loss: 2.9056 | LR: 5.69e-05 | 2360.2ms/step | 8.5 samples/s +Step 34540/44000 | Loss: 2.9066 | LR: 5.67e-05 | 2426.8ms/step | 8.2 samples/s +Step 34550/44000 | Loss: 2.9475 | LR: 5.66e-05 | 2364.1ms/step | 8.5 samples/s +Step 34560/44000 | Loss: 2.8869 | LR: 5.65e-05 | 2375.0ms/step | 8.4 samples/s +Step 34570/44000 | Loss: 2.9014 | LR: 5.64e-05 | 2380.0ms/step | 8.4 samples/s +Step 34580/44000 | Loss: 2.8938 | LR: 5.63e-05 | 2384.8ms/step | 8.4 samples/s +Step 34590/44000 | Loss: 2.9374 | LR: 5.62e-05 | 2426.5ms/step | 8.2 samples/s +Step 34600/44000 | Loss: 2.8880 | LR: 5.60e-05 | 2420.3ms/step | 8.3 samples/s +Step 34610/44000 | Loss: 2.9312 | LR: 5.59e-05 | 2360.0ms/step | 8.5 samples/s +Step 34620/44000 | Loss: 2.9078 | LR: 5.58e-05 | 2390.0ms/step | 8.4 samples/s +Step 34630/44000 | Loss: 2.9071 | LR: 5.57e-05 | 2364.2ms/step | 8.5 samples/s +Step 34640/44000 | Loss: 2.8686 | LR: 5.56e-05 | 2390.1ms/step | 8.4 samples/s +Step 34650/44000 | Loss: 2.8779 | LR: 5.55e-05 | 2374.7ms/step | 8.4 samples/s +Step 34660/44000 | Loss: 2.9045 | LR: 5.54e-05 | 2349.2ms/step | 8.5 samples/s +Step 34670/44000 | Loss: 2.8976 | LR: 5.52e-05 | 2373.2ms/step | 8.4 samples/s +Step 34680/44000 | Loss: 2.8569 | LR: 5.51e-05 | 2377.5ms/step | 8.4 samples/s +Step 34690/44000 | Loss: 2.9155 | LR: 5.50e-05 | 2403.0ms/step | 8.3 samples/s +Step 34700/44000 | Loss: 2.8732 | LR: 5.49e-05 | 2398.6ms/step | 8.3 samples/s +Step 34710/44000 | Loss: 2.9053 | LR: 5.48e-05 | 2392.5ms/step | 8.4 samples/s +Step 34720/44000 | Loss: 2.8839 | LR: 5.47e-05 | 2406.5ms/step | 8.3 samples/s +Step 34730/44000 | Loss: 2.8704 | LR: 5.46e-05 | 2387.1ms/step | 8.4 samples/s +Step 34740/44000 | Loss: 2.8563 | LR: 5.45e-05 | 2337.4ms/step | 8.6 samples/s +Step 34750/44000 | Loss: 2.9052 | LR: 5.43e-05 | 2332.3ms/step | 8.6 samples/s +Step 34760/44000 | Loss: 2.8366 | LR: 5.42e-05 | 2395.3ms/step | 8.3 samples/s +Step 34770/44000 | Loss: 2.8990 | LR: 5.41e-05 | 2326.4ms/step | 8.6 samples/s +Step 34780/44000 | Loss: 2.8717 | LR: 5.40e-05 | 2394.6ms/step | 8.4 samples/s +Step 34790/44000 | Loss: 2.8890 | LR: 5.39e-05 | 2379.9ms/step | 8.4 samples/s +Step 34800/44000 | Loss: 2.8887 | LR: 5.38e-05 | 2410.4ms/step | 8.3 samples/s +Step 34810/44000 | Loss: 2.8268 | LR: 5.37e-05 | 2327.0ms/step | 8.6 samples/s +Step 34820/44000 | Loss: 2.9283 | LR: 5.36e-05 | 2400.4ms/step | 8.3 samples/s +Step 34830/44000 | Loss: 2.8795 | LR: 5.34e-05 | 2406.6ms/step | 8.3 samples/s +Step 34840/44000 | Loss: 2.9056 | LR: 5.33e-05 | 2380.5ms/step | 8.4 samples/s +Step 34850/44000 | Loss: 2.8602 | LR: 5.32e-05 | 2358.0ms/step | 8.5 samples/s +Step 34860/44000 | Loss: 2.9021 | LR: 5.31e-05 | 2270.7ms/step | 8.8 samples/s +Step 34870/44000 | Loss: 2.8302 | LR: 5.30e-05 | 2262.7ms/step | 8.8 samples/s +Step 34880/44000 | Loss: 2.8908 | LR: 5.29e-05 | 2242.0ms/step | 8.9 samples/s +Step 34890/44000 | Loss: 2.8792 | LR: 5.28e-05 | 2279.2ms/step | 8.8 samples/s +Step 34900/44000 | Loss: 2.9104 | LR: 5.27e-05 | 2264.7ms/step | 8.8 samples/s +Step 34910/44000 | Loss: 2.8526 | LR: 5.25e-05 | 2300.5ms/step | 8.7 samples/s +Step 34920/44000 | Loss: 2.8763 | LR: 5.24e-05 | 2287.6ms/step | 8.7 samples/s +Step 34930/44000 | Loss: 2.8712 | LR: 5.23e-05 | 2310.9ms/step | 8.7 samples/s +Step 34940/44000 | Loss: 2.8723 | LR: 5.22e-05 | 2252.6ms/step | 8.9 samples/s +Step 34950/44000 | Loss: 2.9123 | LR: 5.21e-05 | 2333.9ms/step | 8.6 samples/s +Step 34960/44000 | Loss: 2.8773 | LR: 5.20e-05 | 2299.1ms/step | 8.7 samples/s +Step 34970/44000 | Loss: 2.9196 | LR: 5.19e-05 | 2287.5ms/step | 8.7 samples/s +Step 34980/44000 | Loss: 2.9356 | LR: 5.18e-05 | 2294.5ms/step | 8.7 samples/s +Step 34990/44000 | Loss: 2.9208 | LR: 5.17e-05 | 2273.2ms/step | 8.8 samples/s +Step 35000/44000 | Loss: 2.8602 | LR: 5.15e-05 | 2306.9ms/step | 8.7 samples/s +Validation at step 35000: Loss = 2.8967, Perplexity = 18.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-35000 +Step 35010/44000 | Loss: 2.8809 | LR: 5.14e-05 | 4673.7ms/step | 4.3 samples/s +Step 35020/44000 | Loss: 2.8771 | LR: 5.13e-05 | 2299.3ms/step | 8.7 samples/s +Step 35030/44000 | Loss: 2.8997 | LR: 5.12e-05 | 2277.7ms/step | 8.8 samples/s +Step 35040/44000 | Loss: 2.8589 | LR: 5.11e-05 | 2284.0ms/step | 8.8 samples/s +Step 35050/44000 | Loss: 2.9123 | LR: 5.10e-05 | 2288.8ms/step | 8.7 samples/s +Step 35060/44000 | Loss: 2.8917 | LR: 5.09e-05 | 2146.9ms/step | 9.3 samples/s +Step 35070/44000 | Loss: 2.8940 | LR: 5.08e-05 | 2331.5ms/step | 8.6 samples/s +Step 35080/44000 | Loss: 2.9237 | LR: 5.07e-05 | 2358.0ms/step | 8.5 samples/s +Step 35090/44000 | Loss: 2.8789 | LR: 5.06e-05 | 2236.7ms/step | 8.9 samples/s +Step 35100/44000 | Loss: 2.8976 | LR: 5.04e-05 | 2243.7ms/step | 8.9 samples/s +Step 35110/44000 | Loss: 2.9017 | LR: 5.03e-05 | 2266.0ms/step | 8.8 samples/s +Step 35120/44000 | Loss: 2.8831 | LR: 5.02e-05 | 2257.9ms/step | 8.9 samples/s +Step 35130/44000 | Loss: 2.8756 | LR: 5.01e-05 | 2289.1ms/step | 8.7 samples/s +Step 35140/44000 | Loss: 2.9261 | LR: 5.00e-05 | 2268.6ms/step | 8.8 samples/s +Step 35150/44000 | Loss: 2.8971 | LR: 4.99e-05 | 2271.3ms/step | 8.8 samples/s +Step 35160/44000 | Loss: 2.8770 | LR: 4.98e-05 | 2282.7ms/step | 8.8 samples/s +Step 35170/44000 | Loss: 2.8956 | LR: 4.97e-05 | 2273.4ms/step | 8.8 samples/s +Step 35180/44000 | Loss: 2.8667 | LR: 4.96e-05 | 2256.5ms/step | 8.9 samples/s +Step 35190/44000 | Loss: 2.8704 | LR: 4.95e-05 | 2307.4ms/step | 8.7 samples/s +Step 35200/44000 | Loss: 2.8780 | LR: 4.94e-05 | 2305.4ms/step | 8.7 samples/s +Step 35210/44000 | Loss: 2.9057 | LR: 4.93e-05 | 2304.8ms/step | 8.7 samples/s +Step 35220/44000 | Loss: 2.8897 | LR: 4.91e-05 | 2259.7ms/step | 8.9 samples/s +Step 35230/44000 | Loss: 2.8677 | LR: 4.90e-05 | 2279.1ms/step | 8.8 samples/s +Step 35240/44000 | Loss: 2.8892 | LR: 4.89e-05 | 2257.7ms/step | 8.9 samples/s +Step 35250/44000 | Loss: 2.8860 | LR: 4.88e-05 | 2279.1ms/step | 8.8 samples/s +Step 35260/44000 | Loss: 2.9114 | LR: 4.87e-05 | 2320.9ms/step | 8.6 samples/s +Step 35270/44000 | Loss: 2.8620 | LR: 4.86e-05 | 2279.3ms/step | 8.8 samples/s +Step 35280/44000 | Loss: 2.8889 | LR: 4.85e-05 | 2287.1ms/step | 8.7 samples/s +Step 35290/44000 | Loss: 2.8815 | LR: 4.84e-05 | 2311.5ms/step | 8.7 samples/s +Step 35300/44000 | Loss: 2.8942 | LR: 4.83e-05 | 2276.7ms/step | 8.8 samples/s +Step 35310/44000 | Loss: 2.8926 | LR: 4.82e-05 | 2272.1ms/step | 8.8 samples/s +Step 35320/44000 | Loss: 2.9000 | LR: 4.81e-05 | 2259.3ms/step | 8.9 samples/s +Step 35330/44000 | Loss: 2.9163 | LR: 4.80e-05 | 2293.9ms/step | 8.7 samples/s +Step 35340/44000 | Loss: 2.8786 | LR: 4.79e-05 | 2322.4ms/step | 8.6 samples/s +Step 35350/44000 | Loss: 2.8870 | LR: 4.77e-05 | 2266.6ms/step | 8.8 samples/s +Step 35360/44000 | Loss: 2.9197 | LR: 4.76e-05 | 2324.6ms/step | 8.6 samples/s +Step 35370/44000 | Loss: 2.8781 | LR: 4.75e-05 | 2298.7ms/step | 8.7 samples/s +Step 35380/44000 | Loss: 2.8957 | LR: 4.74e-05 | 2256.8ms/step | 8.9 samples/s +Step 35390/44000 | Loss: 2.9296 | LR: 4.73e-05 | 2261.8ms/step | 8.8 samples/s +Step 35400/44000 | Loss: 2.9114 | LR: 4.72e-05 | 2289.1ms/step | 8.7 samples/s +Step 35410/44000 | Loss: 2.9222 | LR: 4.71e-05 | 2266.2ms/step | 8.8 samples/s +Step 35420/44000 | Loss: 2.8645 | LR: 4.70e-05 | 2281.3ms/step | 8.8 samples/s +Step 35430/44000 | Loss: 2.9032 | LR: 4.69e-05 | 2274.4ms/step | 8.8 samples/s +Step 35440/44000 | Loss: 2.8865 | LR: 4.68e-05 | 2307.1ms/step | 8.7 samples/s +Step 35450/44000 | Loss: 2.9111 | LR: 4.67e-05 | 2266.0ms/step | 8.8 samples/s +Step 35460/44000 | Loss: 2.9030 | LR: 4.66e-05 | 2312.6ms/step | 8.6 samples/s +Step 35470/44000 | Loss: 2.9033 | LR: 4.65e-05 | 2273.4ms/step | 8.8 samples/s +Step 35480/44000 | Loss: 2.9178 | LR: 4.64e-05 | 2300.1ms/step | 8.7 samples/s +Step 35490/44000 | Loss: 2.8786 | LR: 4.63e-05 | 2289.2ms/step | 8.7 samples/s +Step 35500/44000 | Loss: 2.8909 | LR: 4.62e-05 | 2267.4ms/step | 8.8 samples/s +Step 35510/44000 | Loss: 2.8817 | LR: 4.61e-05 | 2297.9ms/step | 8.7 samples/s +Step 35520/44000 | Loss: 2.8588 | LR: 4.59e-05 | 2257.0ms/step | 8.9 samples/s +Step 35530/44000 | Loss: 2.8483 | LR: 4.58e-05 | 2295.6ms/step | 8.7 samples/s +Step 35540/44000 | Loss: 2.9313 | LR: 4.57e-05 | 2304.5ms/step | 8.7 samples/s +Step 35550/44000 | Loss: 2.8551 | LR: 4.56e-05 | 2249.9ms/step | 8.9 samples/s +Step 35560/44000 | Loss: 2.9258 | LR: 4.55e-05 | 2306.2ms/step | 8.7 samples/s +Step 35570/44000 | Loss: 2.8297 | LR: 4.54e-05 | 2260.4ms/step | 8.8 samples/s +Step 35580/44000 | Loss: 2.9232 | LR: 4.53e-05 | 2322.6ms/step | 8.6 samples/s +Step 35590/44000 | Loss: 2.8997 | LR: 4.52e-05 | 2304.7ms/step | 8.7 samples/s +Step 35600/44000 | Loss: 2.8707 | LR: 4.51e-05 | 2305.4ms/step | 8.7 samples/s +Step 35610/44000 | Loss: 2.8905 | LR: 4.50e-05 | 2295.6ms/step | 8.7 samples/s +Step 35620/44000 | Loss: 2.8829 | LR: 4.49e-05 | 2256.9ms/step | 8.9 samples/s +Step 35630/44000 | Loss: 2.8667 | LR: 4.48e-05 | 2310.0ms/step | 8.7 samples/s +Step 35640/44000 | Loss: 2.9108 | LR: 4.47e-05 | 2279.0ms/step | 8.8 samples/s +Step 35650/44000 | Loss: 2.8911 | LR: 4.46e-05 | 2279.1ms/step | 8.8 samples/s +Step 35660/44000 | Loss: 2.8615 | LR: 4.45e-05 | 2320.6ms/step | 8.6 samples/s +Step 35670/44000 | Loss: 2.9007 | LR: 4.44e-05 | 2300.1ms/step | 8.7 samples/s +Step 35680/44000 | Loss: 2.8648 | LR: 4.43e-05 | 2268.8ms/step | 8.8 samples/s +Step 35690/44000 | Loss: 2.9224 | LR: 4.42e-05 | 2284.3ms/step | 8.8 samples/s +Step 35700/44000 | Loss: 2.9103 | LR: 4.41e-05 | 2289.4ms/step | 8.7 samples/s +Step 35710/44000 | Loss: 2.8693 | LR: 4.40e-05 | 2280.3ms/step | 8.8 samples/s +Step 35720/44000 | Loss: 2.8650 | LR: 4.39e-05 | 2297.8ms/step | 8.7 samples/s +Step 35730/44000 | Loss: 2.9141 | LR: 4.38e-05 | 2311.5ms/step | 8.7 samples/s +Step 35740/44000 | Loss: 2.9183 | LR: 4.37e-05 | 2308.3ms/step | 8.7 samples/s +Step 35750/44000 | Loss: 2.8469 | LR: 4.36e-05 | 2345.5ms/step | 8.5 samples/s +Step 35760/44000 | Loss: 2.9025 | LR: 4.35e-05 | 2349.2ms/step | 8.5 samples/s +Step 35770/44000 | Loss: 2.8811 | LR: 4.34e-05 | 2268.6ms/step | 8.8 samples/s +Step 35780/44000 | Loss: 2.8649 | LR: 4.33e-05 | 2315.2ms/step | 8.6 samples/s +Step 35790/44000 | Loss: 2.8621 | LR: 4.32e-05 | 2302.2ms/step | 8.7 samples/s +Step 35800/44000 | Loss: 2.8925 | LR: 4.31e-05 | 2340.3ms/step | 8.5 samples/s +Step 35810/44000 | Loss: 2.8709 | LR: 4.29e-05 | 2277.5ms/step | 8.8 samples/s +Step 35820/44000 | Loss: 2.8876 | LR: 4.28e-05 | 2298.7ms/step | 8.7 samples/s +Step 35830/44000 | Loss: 2.8742 | LR: 4.27e-05 | 2296.0ms/step | 8.7 samples/s +Step 35840/44000 | Loss: 2.9010 | LR: 4.26e-05 | 2290.5ms/step | 8.7 samples/s +Step 35850/44000 | Loss: 2.8914 | LR: 4.25e-05 | 2318.6ms/step | 8.6 samples/s +Step 35860/44000 | Loss: 2.8637 | LR: 4.24e-05 | 2311.7ms/step | 8.7 samples/s +Step 35870/44000 | Loss: 2.9041 | LR: 4.23e-05 | 2290.2ms/step | 8.7 samples/s +Step 35880/44000 | Loss: 2.9116 | LR: 4.22e-05 | 2290.5ms/step | 8.7 samples/s +Step 35890/44000 | Loss: 2.8730 | LR: 4.21e-05 | 2290.5ms/step | 8.7 samples/s +Step 35900/44000 | Loss: 2.9236 | LR: 4.20e-05 | 2269.1ms/step | 8.8 samples/s +Step 35910/44000 | Loss: 2.8268 | LR: 4.19e-05 | 2324.8ms/step | 8.6 samples/s +Step 35920/44000 | Loss: 2.9362 | LR: 4.18e-05 | 2301.3ms/step | 8.7 samples/s +Step 35930/44000 | Loss: 2.8840 | LR: 4.17e-05 | 2309.3ms/step | 8.7 samples/s +Step 35940/44000 | Loss: 2.9120 | LR: 4.16e-05 | 2278.6ms/step | 8.8 samples/s +Step 35950/44000 | Loss: 2.8629 | LR: 4.15e-05 | 2318.3ms/step | 8.6 samples/s +Step 35960/44000 | Loss: 2.9005 | LR: 4.14e-05 | 2301.9ms/step | 8.7 samples/s +Step 35970/44000 | Loss: 2.8730 | LR: 4.13e-05 | 2305.2ms/step | 8.7 samples/s +Step 35980/44000 | Loss: 2.8629 | LR: 4.12e-05 | 2324.8ms/step | 8.6 samples/s +Step 35990/44000 | Loss: 2.8694 | LR: 4.11e-05 | 2338.2ms/step | 8.6 samples/s +Step 36000/44000 | Loss: 2.8887 | LR: 4.10e-05 | 2259.1ms/step | 8.9 samples/s +Validation at step 36000: Loss = 2.8958, Perplexity = 18.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-36000 +Step 36010/44000 | Loss: 2.8772 | LR: 4.09e-05 | 4674.8ms/step | 4.3 samples/s +Step 36020/44000 | Loss: 2.8644 | LR: 4.08e-05 | 2258.1ms/step | 8.9 samples/s +Step 36030/44000 | Loss: 2.9041 | LR: 4.07e-05 | 2263.7ms/step | 8.8 samples/s +Step 36040/44000 | Loss: 2.8765 | LR: 4.06e-05 | 2278.8ms/step | 8.8 samples/s +Step 36050/44000 | Loss: 2.8738 | LR: 4.05e-05 | 2122.2ms/step | 9.4 samples/s +Step 36060/44000 | Loss: 2.9076 | LR: 4.04e-05 | 2277.9ms/step | 8.8 samples/s +Step 36070/44000 | Loss: 2.8747 | LR: 4.03e-05 | 2333.2ms/step | 8.6 samples/s +Step 36080/44000 | Loss: 2.8747 | LR: 4.02e-05 | 2353.4ms/step | 8.5 samples/s +Step 36090/44000 | Loss: 2.9002 | LR: 4.01e-05 | 2310.9ms/step | 8.7 samples/s +Step 36100/44000 | Loss: 2.8743 | LR: 4.00e-05 | 2264.6ms/step | 8.8 samples/s +Step 36110/44000 | Loss: 2.8656 | LR: 3.99e-05 | 2274.0ms/step | 8.8 samples/s +Step 36120/44000 | Loss: 2.9033 | LR: 3.98e-05 | 2261.2ms/step | 8.8 samples/s +Step 36130/44000 | Loss: 2.8798 | LR: 3.97e-05 | 2258.5ms/step | 8.9 samples/s +Step 36140/44000 | Loss: 2.8797 | LR: 3.97e-05 | 2318.5ms/step | 8.6 samples/s +Step 36150/44000 | Loss: 2.8921 | LR: 3.96e-05 | 2269.1ms/step | 8.8 samples/s +Step 36160/44000 | Loss: 2.8910 | LR: 3.95e-05 | 2295.0ms/step | 8.7 samples/s +Step 36170/44000 | Loss: 2.8654 | LR: 3.94e-05 | 2277.5ms/step | 8.8 samples/s +Step 36180/44000 | Loss: 2.8865 | LR: 3.93e-05 | 2279.6ms/step | 8.8 samples/s +Step 36190/44000 | Loss: 2.9272 | LR: 3.92e-05 | 2269.0ms/step | 8.8 samples/s +Step 36200/44000 | Loss: 2.8988 | LR: 3.91e-05 | 2298.3ms/step | 8.7 samples/s +Step 36210/44000 | Loss: 2.8968 | LR: 3.90e-05 | 2299.6ms/step | 8.7 samples/s +Step 36220/44000 | Loss: 2.8842 | LR: 3.89e-05 | 2252.5ms/step | 8.9 samples/s +Step 36230/44000 | Loss: 2.8942 | LR: 3.88e-05 | 2267.8ms/step | 8.8 samples/s +Step 36240/44000 | Loss: 2.9206 | LR: 3.87e-05 | 2297.1ms/step | 8.7 samples/s +Step 36250/44000 | Loss: 2.8599 | LR: 3.86e-05 | 2348.8ms/step | 8.5 samples/s +Step 36260/44000 | Loss: 2.9467 | LR: 3.85e-05 | 2298.5ms/step | 8.7 samples/s +Step 36270/44000 | Loss: 2.8962 | LR: 3.84e-05 | 2264.0ms/step | 8.8 samples/s +Step 36280/44000 | Loss: 2.8985 | LR: 3.83e-05 | 2304.8ms/step | 8.7 samples/s +Step 36290/44000 | Loss: 2.8994 | LR: 3.82e-05 | 2283.7ms/step | 8.8 samples/s +Step 36300/44000 | Loss: 2.8677 | LR: 3.81e-05 | 2298.0ms/step | 8.7 samples/s +Step 36310/44000 | Loss: 2.9181 | LR: 3.80e-05 | 2291.3ms/step | 8.7 samples/s +Step 36320/44000 | Loss: 2.8711 | LR: 3.79e-05 | 2322.1ms/step | 8.6 samples/s +Step 36330/44000 | Loss: 2.9074 | LR: 3.78e-05 | 2349.5ms/step | 8.5 samples/s +Step 36340/44000 | Loss: 2.8657 | LR: 3.77e-05 | 2312.0ms/step | 8.7 samples/s +Step 36350/44000 | Loss: 2.8659 | LR: 3.76e-05 | 2332.5ms/step | 8.6 samples/s +Step 36360/44000 | Loss: 2.8867 | LR: 3.75e-05 | 2322.1ms/step | 8.6 samples/s +Step 36370/44000 | Loss: 2.8686 | LR: 3.74e-05 | 2300.1ms/step | 8.7 samples/s +Step 36380/44000 | Loss: 2.8707 | LR: 3.73e-05 | 2266.9ms/step | 8.8 samples/s +Step 36390/44000 | Loss: 2.8747 | LR: 3.72e-05 | 2270.5ms/step | 8.8 samples/s +Step 36400/44000 | Loss: 2.8999 | LR: 3.71e-05 | 2283.6ms/step | 8.8 samples/s +Step 36410/44000 | Loss: 2.9190 | LR: 3.70e-05 | 2259.4ms/step | 8.9 samples/s +Step 36420/44000 | Loss: 2.8779 | LR: 3.69e-05 | 2298.1ms/step | 8.7 samples/s +Step 36430/44000 | Loss: 2.8825 | LR: 3.69e-05 | 2330.3ms/step | 8.6 samples/s +Step 36440/44000 | Loss: 2.8624 | LR: 3.68e-05 | 2245.1ms/step | 8.9 samples/s +Step 36450/44000 | Loss: 2.9207 | LR: 3.67e-05 | 2258.8ms/step | 8.9 samples/s +Step 36460/44000 | Loss: 2.9383 | LR: 3.66e-05 | 2275.1ms/step | 8.8 samples/s +Step 36470/44000 | Loss: 2.9058 | LR: 3.65e-05 | 2288.2ms/step | 8.7 samples/s +Step 36480/44000 | Loss: 2.9208 | LR: 3.64e-05 | 2252.8ms/step | 8.9 samples/s +Step 36490/44000 | Loss: 2.9205 | LR: 3.63e-05 | 2289.7ms/step | 8.7 samples/s +Step 36500/44000 | Loss: 2.8888 | LR: 3.62e-05 | 2274.1ms/step | 8.8 samples/s +Step 36510/44000 | Loss: 2.8519 | LR: 3.61e-05 | 2257.7ms/step | 8.9 samples/s +Step 36520/44000 | Loss: 2.8582 | LR: 3.60e-05 | 2295.6ms/step | 8.7 samples/s +Step 36530/44000 | Loss: 2.9285 | LR: 3.59e-05 | 2343.7ms/step | 8.5 samples/s +Step 36540/44000 | Loss: 2.8766 | LR: 3.58e-05 | 2287.1ms/step | 8.7 samples/s +Step 36550/44000 | Loss: 2.8810 | LR: 3.57e-05 | 2316.8ms/step | 8.6 samples/s +Step 36560/44000 | Loss: 2.8850 | LR: 3.56e-05 | 2233.6ms/step | 9.0 samples/s +Step 36570/44000 | Loss: 2.9257 | LR: 3.55e-05 | 2261.7ms/step | 8.8 samples/s +Step 36580/44000 | Loss: 2.8941 | LR: 3.54e-05 | 2290.1ms/step | 8.7 samples/s +Step 36590/44000 | Loss: 2.8915 | LR: 3.53e-05 | 2245.0ms/step | 8.9 samples/s +Step 36600/44000 | Loss: 2.8603 | LR: 3.53e-05 | 2317.6ms/step | 8.6 samples/s +Step 36610/44000 | Loss: 2.8929 | LR: 3.52e-05 | 2292.7ms/step | 8.7 samples/s +Step 36620/44000 | Loss: 2.8637 | LR: 3.51e-05 | 2294.5ms/step | 8.7 samples/s +Step 36630/44000 | Loss: 2.8870 | LR: 3.50e-05 | 2269.0ms/step | 8.8 samples/s +Step 36640/44000 | Loss: 2.8563 | LR: 3.49e-05 | 2246.4ms/step | 8.9 samples/s +Step 36650/44000 | Loss: 2.8542 | LR: 3.48e-05 | 2279.2ms/step | 8.8 samples/s +Step 36660/44000 | Loss: 2.8747 | LR: 3.47e-05 | 2294.1ms/step | 8.7 samples/s +Step 36670/44000 | Loss: 2.9136 | LR: 3.46e-05 | 2299.8ms/step | 8.7 samples/s +Step 36680/44000 | Loss: 2.8881 | LR: 3.45e-05 | 2283.3ms/step | 8.8 samples/s +Step 36690/44000 | Loss: 2.9469 | LR: 3.44e-05 | 2322.2ms/step | 8.6 samples/s +Step 36700/44000 | Loss: 2.9126 | LR: 3.43e-05 | 2285.8ms/step | 8.7 samples/s +Step 36710/44000 | Loss: 2.8646 | LR: 3.42e-05 | 2260.8ms/step | 8.8 samples/s +Step 36720/44000 | Loss: 2.8773 | LR: 3.41e-05 | 2310.1ms/step | 8.7 samples/s +Step 36730/44000 | Loss: 2.9119 | LR: 3.41e-05 | 2288.5ms/step | 8.7 samples/s +Step 36740/44000 | Loss: 2.9046 | LR: 3.40e-05 | 2300.3ms/step | 8.7 samples/s +Step 36750/44000 | Loss: 2.8939 | LR: 3.39e-05 | 2298.2ms/step | 8.7 samples/s +Step 36760/44000 | Loss: 2.8901 | LR: 3.38e-05 | 2301.0ms/step | 8.7 samples/s +Step 36770/44000 | Loss: 2.9176 | LR: 3.37e-05 | 2301.2ms/step | 8.7 samples/s +Step 36780/44000 | Loss: 2.8575 | LR: 3.36e-05 | 2296.0ms/step | 8.7 samples/s +Step 36790/44000 | Loss: 2.8778 | LR: 3.35e-05 | 2302.9ms/step | 8.7 samples/s +Step 36800/44000 | Loss: 2.8677 | LR: 3.34e-05 | 2251.7ms/step | 8.9 samples/s +Step 36810/44000 | Loss: 2.8876 | LR: 3.33e-05 | 2239.4ms/step | 8.9 samples/s +Step 36820/44000 | Loss: 2.8907 | LR: 3.32e-05 | 2280.8ms/step | 8.8 samples/s +Step 36830/44000 | Loss: 2.9102 | LR: 3.31e-05 | 2279.1ms/step | 8.8 samples/s +Step 36840/44000 | Loss: 2.8328 | LR: 3.31e-05 | 2258.5ms/step | 8.9 samples/s +Step 36850/44000 | Loss: 2.9340 | LR: 3.30e-05 | 2329.0ms/step | 8.6 samples/s +Step 36860/44000 | Loss: 2.9443 | LR: 3.29e-05 | 2314.4ms/step | 8.6 samples/s +Step 36870/44000 | Loss: 2.9011 | LR: 3.28e-05 | 2286.3ms/step | 8.7 samples/s +Step 36880/44000 | Loss: 2.8611 | LR: 3.27e-05 | 2290.0ms/step | 8.7 samples/s +Step 36890/44000 | Loss: 2.8921 | LR: 3.26e-05 | 2300.4ms/step | 8.7 samples/s +Step 36900/44000 | Loss: 2.9006 | LR: 3.25e-05 | 2268.1ms/step | 8.8 samples/s +Step 36910/44000 | Loss: 2.9090 | LR: 3.24e-05 | 2272.7ms/step | 8.8 samples/s +Step 36920/44000 | Loss: 2.9087 | LR: 3.23e-05 | 2304.9ms/step | 8.7 samples/s +Step 36930/44000 | Loss: 2.8594 | LR: 3.22e-05 | 2289.1ms/step | 8.7 samples/s +Step 36940/44000 | Loss: 2.9121 | LR: 3.22e-05 | 2292.6ms/step | 8.7 samples/s +Step 36950/44000 | Loss: 2.8568 | LR: 3.21e-05 | 2297.3ms/step | 8.7 samples/s +Step 36960/44000 | Loss: 2.8963 | LR: 3.20e-05 | 2344.8ms/step | 8.5 samples/s +Step 36970/44000 | Loss: 2.8672 | LR: 3.19e-05 | 2274.3ms/step | 8.8 samples/s +Step 36980/44000 | Loss: 2.8870 | LR: 3.18e-05 | 2255.6ms/step | 8.9 samples/s +Step 36990/44000 | Loss: 2.8792 | LR: 3.17e-05 | 2317.4ms/step | 8.6 samples/s +Step 37000/44000 | Loss: 2.8942 | LR: 3.16e-05 | 2335.5ms/step | 8.6 samples/s +Validation at step 37000: Loss = 2.8953, Perplexity = 18.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-37000 +Step 37010/44000 | Loss: 2.8939 | LR: 3.15e-05 | 4717.8ms/step | 4.2 samples/s +Step 37020/44000 | Loss: 2.8860 | LR: 3.15e-05 | 2341.0ms/step | 8.5 samples/s +Step 37030/44000 | Loss: 2.8434 | LR: 3.14e-05 | 2224.3ms/step | 9.0 samples/s +Step 37040/44000 | Loss: 2.8867 | LR: 3.13e-05 | 2237.4ms/step | 8.9 samples/s +Step 37050/44000 | Loss: 2.8704 | LR: 3.12e-05 | 2247.5ms/step | 8.9 samples/s +Step 37060/44000 | Loss: 2.9206 | LR: 3.11e-05 | 2275.9ms/step | 8.8 samples/s +Step 37070/44000 | Loss: 2.9115 | LR: 3.10e-05 | 2256.8ms/step | 8.9 samples/s +Step 37080/44000 | Loss: 2.9012 | LR: 3.09e-05 | 2256.1ms/step | 8.9 samples/s +Step 37090/44000 | Loss: 2.8976 | LR: 3.08e-05 | 2222.0ms/step | 9.0 samples/s +Step 37100/44000 | Loss: 2.8983 | LR: 3.07e-05 | 2306.6ms/step | 8.7 samples/s +Step 37110/44000 | Loss: 2.9145 | LR: 3.07e-05 | 2270.6ms/step | 8.8 samples/s +Step 37120/44000 | Loss: 2.8668 | LR: 3.06e-05 | 2315.1ms/step | 8.6 samples/s +Step 37130/44000 | Loss: 2.8915 | LR: 3.05e-05 | 2290.4ms/step | 8.7 samples/s +Step 37140/44000 | Loss: 2.9107 | LR: 3.04e-05 | 2288.7ms/step | 8.7 samples/s +Step 37150/44000 | Loss: 2.9097 | LR: 3.03e-05 | 2316.0ms/step | 8.6 samples/s +Step 37160/44000 | Loss: 2.8973 | LR: 3.02e-05 | 2273.5ms/step | 8.8 samples/s +Step 37170/44000 | Loss: 2.8522 | LR: 3.01e-05 | 2294.0ms/step | 8.7 samples/s +Step 37180/44000 | Loss: 2.8720 | LR: 3.01e-05 | 2310.7ms/step | 8.7 samples/s +Step 37190/44000 | Loss: 2.9284 | LR: 3.00e-05 | 2268.2ms/step | 8.8 samples/s +Step 37200/44000 | Loss: 2.8519 | LR: 2.99e-05 | 2248.7ms/step | 8.9 samples/s +Step 37210/44000 | Loss: 2.9291 | LR: 2.98e-05 | 2269.5ms/step | 8.8 samples/s +Step 37220/44000 | Loss: 2.8425 | LR: 2.97e-05 | 2261.7ms/step | 8.8 samples/s +Step 37230/44000 | Loss: 2.8825 | LR: 2.96e-05 | 2287.7ms/step | 8.7 samples/s +Step 37240/44000 | Loss: 2.9031 | LR: 2.95e-05 | 2275.8ms/step | 8.8 samples/s +Step 37250/44000 | Loss: 2.8641 | LR: 2.95e-05 | 2321.0ms/step | 8.6 samples/s +Step 37260/44000 | Loss: 2.8164 | LR: 2.94e-05 | 2323.2ms/step | 8.6 samples/s +Step 37270/44000 | Loss: 2.8574 | LR: 2.93e-05 | 2302.5ms/step | 8.7 samples/s +Step 37280/44000 | Loss: 2.8644 | LR: 2.92e-05 | 2346.9ms/step | 8.5 samples/s +Step 37290/44000 | Loss: 2.8963 | LR: 2.91e-05 | 2263.5ms/step | 8.8 samples/s +Step 37300/44000 | Loss: 2.8446 | LR: 2.90e-05 | 2284.2ms/step | 8.8 samples/s +Step 37310/44000 | Loss: 2.8985 | LR: 2.89e-05 | 2257.7ms/step | 8.9 samples/s +Step 37320/44000 | Loss: 2.8524 | LR: 2.89e-05 | 2269.2ms/step | 8.8 samples/s +Step 37330/44000 | Loss: 2.8455 | LR: 2.88e-05 | 2291.0ms/step | 8.7 samples/s +Step 37340/44000 | Loss: 2.8886 | LR: 2.87e-05 | 2326.9ms/step | 8.6 samples/s +Step 37350/44000 | Loss: 2.8978 | LR: 2.86e-05 | 2284.4ms/step | 8.8 samples/s +Step 37360/44000 | Loss: 2.8898 | LR: 2.85e-05 | 2291.3ms/step | 8.7 samples/s +Step 37370/44000 | Loss: 2.8734 | LR: 2.84e-05 | 2325.4ms/step | 8.6 samples/s +Step 37380/44000 | Loss: 2.9132 | LR: 2.84e-05 | 2272.6ms/step | 8.8 samples/s +Step 37390/44000 | Loss: 2.8713 | LR: 2.83e-05 | 2319.1ms/step | 8.6 samples/s +Step 37400/44000 | Loss: 2.8966 | LR: 2.82e-05 | 2284.8ms/step | 8.8 samples/s +Step 37410/44000 | Loss: 2.9147 | LR: 2.81e-05 | 2350.4ms/step | 8.5 samples/s +Step 37420/44000 | Loss: 2.8628 | LR: 2.80e-05 | 2325.3ms/step | 8.6 samples/s +Step 37430/44000 | Loss: 2.9245 | LR: 2.79e-05 | 2278.9ms/step | 8.8 samples/s +Step 37440/44000 | Loss: 2.8887 | LR: 2.78e-05 | 2331.9ms/step | 8.6 samples/s +Step 37450/44000 | Loss: 2.8951 | LR: 2.78e-05 | 2268.8ms/step | 8.8 samples/s +Step 37460/44000 | Loss: 2.9097 | LR: 2.77e-05 | 2263.3ms/step | 8.8 samples/s +Step 37470/44000 | Loss: 2.8733 | LR: 2.76e-05 | 2313.6ms/step | 8.6 samples/s +Step 37480/44000 | Loss: 2.8884 | LR: 2.75e-05 | 2305.5ms/step | 8.7 samples/s +Step 37490/44000 | Loss: 2.9145 | LR: 2.74e-05 | 2275.6ms/step | 8.8 samples/s +Step 37500/44000 | Loss: 2.8714 | LR: 2.74e-05 | 2324.9ms/step | 8.6 samples/s +Step 37510/44000 | Loss: 2.8836 | LR: 2.73e-05 | 2306.8ms/step | 8.7 samples/s +Step 37520/44000 | Loss: 2.8521 | LR: 2.72e-05 | 2302.8ms/step | 8.7 samples/s +Step 37530/44000 | Loss: 2.8952 | LR: 2.71e-05 | 2338.7ms/step | 8.6 samples/s +Step 37540/44000 | Loss: 2.8799 | LR: 2.70e-05 | 2265.2ms/step | 8.8 samples/s +Step 37550/44000 | Loss: 2.8148 | LR: 2.69e-05 | 2282.0ms/step | 8.8 samples/s +Step 37560/44000 | Loss: 2.9153 | LR: 2.69e-05 | 2267.9ms/step | 8.8 samples/s +Step 37570/44000 | Loss: 2.9116 | LR: 2.68e-05 | 2300.9ms/step | 8.7 samples/s +Step 37580/44000 | Loss: 2.8752 | LR: 2.67e-05 | 2269.0ms/step | 8.8 samples/s +Step 37590/44000 | Loss: 2.9061 | LR: 2.66e-05 | 2268.2ms/step | 8.8 samples/s +Step 37600/44000 | Loss: 2.8977 | LR: 2.65e-05 | 2298.0ms/step | 8.7 samples/s +Step 37610/44000 | Loss: 2.9168 | LR: 2.65e-05 | 2297.1ms/step | 8.7 samples/s +Step 37620/44000 | Loss: 2.8842 | LR: 2.64e-05 | 2311.0ms/step | 8.7 samples/s +Step 37630/44000 | Loss: 2.8994 | LR: 2.63e-05 | 2263.6ms/step | 8.8 samples/s +Step 37640/44000 | Loss: 2.9226 | LR: 2.62e-05 | 2288.3ms/step | 8.7 samples/s +Step 37650/44000 | Loss: 2.9126 | LR: 2.61e-05 | 2276.5ms/step | 8.8 samples/s +Step 37660/44000 | Loss: 2.8828 | LR: 2.60e-05 | 2278.1ms/step | 8.8 samples/s +Step 37670/44000 | Loss: 2.8965 | LR: 2.60e-05 | 2278.8ms/step | 8.8 samples/s +Step 37680/44000 | Loss: 2.8671 | LR: 2.59e-05 | 2336.5ms/step | 8.6 samples/s +Step 37690/44000 | Loss: 2.8441 | LR: 2.58e-05 | 2294.8ms/step | 8.7 samples/s +Step 37700/44000 | Loss: 2.9283 | LR: 2.57e-05 | 2290.3ms/step | 8.7 samples/s +Step 37710/44000 | Loss: 2.8889 | LR: 2.56e-05 | 2314.4ms/step | 8.6 samples/s +Step 37720/44000 | Loss: 2.8798 | LR: 2.56e-05 | 2321.6ms/step | 8.6 samples/s +Step 37730/44000 | Loss: 2.9170 | LR: 2.55e-05 | 2285.2ms/step | 8.8 samples/s +Step 37740/44000 | Loss: 2.8955 | LR: 2.54e-05 | 2284.7ms/step | 8.8 samples/s +Step 37750/44000 | Loss: 2.9298 | LR: 2.53e-05 | 2279.6ms/step | 8.8 samples/s +Step 37760/44000 | Loss: 2.8357 | LR: 2.52e-05 | 2319.7ms/step | 8.6 samples/s +Step 37770/44000 | Loss: 2.8807 | LR: 2.52e-05 | 2308.2ms/step | 8.7 samples/s +Step 37780/44000 | Loss: 2.8902 | LR: 2.51e-05 | 2300.9ms/step | 8.7 samples/s +Step 37790/44000 | Loss: 2.9068 | LR: 2.50e-05 | 2317.0ms/step | 8.6 samples/s +Step 37800/44000 | Loss: 2.9045 | LR: 2.49e-05 | 2274.2ms/step | 8.8 samples/s +Step 37810/44000 | Loss: 2.8812 | LR: 2.48e-05 | 2280.2ms/step | 8.8 samples/s +Step 37820/44000 | Loss: 2.9387 | LR: 2.48e-05 | 2274.3ms/step | 8.8 samples/s +Step 37830/44000 | Loss: 2.9096 | LR: 2.47e-05 | 2323.9ms/step | 8.6 samples/s +Step 37840/44000 | Loss: 2.8933 | LR: 2.46e-05 | 2268.7ms/step | 8.8 samples/s +Step 37850/44000 | Loss: 2.8298 | LR: 2.45e-05 | 2263.5ms/step | 8.8 samples/s +Step 37860/44000 | Loss: 2.8722 | LR: 2.45e-05 | 2222.5ms/step | 9.0 samples/s +Step 37870/44000 | Loss: 2.8910 | LR: 2.44e-05 | 2278.4ms/step | 8.8 samples/s +Step 37880/44000 | Loss: 2.8672 | LR: 2.43e-05 | 2279.2ms/step | 8.8 samples/s +Step 37890/44000 | Loss: 2.9277 | LR: 2.42e-05 | 2300.4ms/step | 8.7 samples/s +Step 37900/44000 | Loss: 2.8607 | LR: 2.41e-05 | 2331.2ms/step | 8.6 samples/s +Step 37910/44000 | Loss: 2.9030 | LR: 2.41e-05 | 2295.9ms/step | 8.7 samples/s +Step 37920/44000 | Loss: 2.9054 | LR: 2.40e-05 | 2361.4ms/step | 8.5 samples/s +Step 37930/44000 | Loss: 2.8759 | LR: 2.39e-05 | 2336.9ms/step | 8.6 samples/s +Step 37940/44000 | Loss: 2.8757 | LR: 2.38e-05 | 2289.2ms/step | 8.7 samples/s +Step 37950/44000 | Loss: 2.8496 | LR: 2.38e-05 | 2326.5ms/step | 8.6 samples/s +Step 37960/44000 | Loss: 2.8766 | LR: 2.37e-05 | 2269.0ms/step | 8.8 samples/s +Step 37970/44000 | Loss: 2.8916 | LR: 2.36e-05 | 2310.3ms/step | 8.7 samples/s +Step 37980/44000 | Loss: 2.8993 | LR: 2.35e-05 | 2321.6ms/step | 8.6 samples/s +Step 37990/44000 | Loss: 2.8964 | LR: 2.34e-05 | 2374.8ms/step | 8.4 samples/s +Step 38000/44000 | Loss: 2.9028 | LR: 2.34e-05 | 2274.6ms/step | 8.8 samples/s +Validation at step 38000: Loss = 2.8949, Perplexity = 18.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-38000 +Step 38010/44000 | Loss: 2.8908 | LR: 2.33e-05 | 4632.0ms/step | 4.3 samples/s +Step 38020/44000 | Loss: 2.8549 | LR: 2.32e-05 | 2139.1ms/step | 9.3 samples/s +Step 38030/44000 | Loss: 2.9320 | LR: 2.31e-05 | 2260.9ms/step | 8.8 samples/s +Step 38040/44000 | Loss: 2.8717 | LR: 2.31e-05 | 2285.8ms/step | 8.7 samples/s +Step 38050/44000 | Loss: 2.8903 | LR: 2.30e-05 | 2279.6ms/step | 8.8 samples/s +Step 38060/44000 | Loss: 2.8799 | LR: 2.29e-05 | 2288.1ms/step | 8.7 samples/s +Step 38070/44000 | Loss: 2.8853 | LR: 2.28e-05 | 2281.3ms/step | 8.8 samples/s +Step 38080/44000 | Loss: 2.8731 | LR: 2.28e-05 | 2309.1ms/step | 8.7 samples/s +Step 38090/44000 | Loss: 2.8408 | LR: 2.27e-05 | 2306.1ms/step | 8.7 samples/s +Step 38100/44000 | Loss: 2.8741 | LR: 2.26e-05 | 2254.1ms/step | 8.9 samples/s +Step 38110/44000 | Loss: 2.8858 | LR: 2.25e-05 | 2285.1ms/step | 8.8 samples/s +Step 38120/44000 | Loss: 2.9117 | LR: 2.25e-05 | 2261.8ms/step | 8.8 samples/s +Step 38130/44000 | Loss: 2.9235 | LR: 2.24e-05 | 2261.4ms/step | 8.8 samples/s +Step 38140/44000 | Loss: 2.8922 | LR: 2.23e-05 | 2316.5ms/step | 8.6 samples/s +Step 38150/44000 | Loss: 2.8902 | LR: 2.22e-05 | 2311.5ms/step | 8.7 samples/s +Step 38160/44000 | Loss: 2.8402 | LR: 2.22e-05 | 2312.3ms/step | 8.6 samples/s +Step 38170/44000 | Loss: 2.9185 | LR: 2.21e-05 | 2269.5ms/step | 8.8 samples/s +Step 38180/44000 | Loss: 2.8562 | LR: 2.20e-05 | 2332.4ms/step | 8.6 samples/s +Step 38190/44000 | Loss: 2.9337 | LR: 2.19e-05 | 2283.7ms/step | 8.8 samples/s +Step 38200/44000 | Loss: 2.8735 | LR: 2.19e-05 | 2283.3ms/step | 8.8 samples/s +Step 38210/44000 | Loss: 2.8864 | LR: 2.18e-05 | 2287.4ms/step | 8.7 samples/s +Step 38220/44000 | Loss: 2.8870 | LR: 2.17e-05 | 2293.0ms/step | 8.7 samples/s +Step 38230/44000 | Loss: 2.8734 | LR: 2.16e-05 | 2267.9ms/step | 8.8 samples/s +Step 38240/44000 | Loss: 2.9064 | LR: 2.16e-05 | 2258.4ms/step | 8.9 samples/s +Step 38250/44000 | Loss: 2.8541 | LR: 2.15e-05 | 2327.6ms/step | 8.6 samples/s +Step 38260/44000 | Loss: 2.8755 | LR: 2.14e-05 | 2285.4ms/step | 8.8 samples/s +Step 38270/44000 | Loss: 2.8875 | LR: 2.13e-05 | 2327.6ms/step | 8.6 samples/s +Step 38280/44000 | Loss: 2.9028 | LR: 2.13e-05 | 2327.2ms/step | 8.6 samples/s +Step 38290/44000 | Loss: 2.8892 | LR: 2.12e-05 | 2312.7ms/step | 8.6 samples/s +Step 38300/44000 | Loss: 2.8657 | LR: 2.11e-05 | 2324.3ms/step | 8.6 samples/s +Step 38310/44000 | Loss: 2.9300 | LR: 2.11e-05 | 2287.3ms/step | 8.7 samples/s +Step 38320/44000 | Loss: 2.8884 | LR: 2.10e-05 | 2280.3ms/step | 8.8 samples/s +Step 38330/44000 | Loss: 2.8859 | LR: 2.09e-05 | 2264.2ms/step | 8.8 samples/s +Step 38340/44000 | Loss: 2.9178 | LR: 2.08e-05 | 2316.1ms/step | 8.6 samples/s +Step 38350/44000 | Loss: 2.9189 | LR: 2.08e-05 | 2295.2ms/step | 8.7 samples/s +Step 38360/44000 | Loss: 2.8597 | LR: 2.07e-05 | 2338.1ms/step | 8.6 samples/s +Step 38370/44000 | Loss: 2.9017 | LR: 2.06e-05 | 2252.8ms/step | 8.9 samples/s +Step 38380/44000 | Loss: 2.9386 | LR: 2.05e-05 | 2271.3ms/step | 8.8 samples/s +Step 38390/44000 | Loss: 2.8640 | LR: 2.05e-05 | 2299.4ms/step | 8.7 samples/s +Step 38400/44000 | Loss: 2.8828 | LR: 2.04e-05 | 2272.6ms/step | 8.8 samples/s +Step 38410/44000 | Loss: 2.8664 | LR: 2.03e-05 | 2261.4ms/step | 8.8 samples/s +Step 38420/44000 | Loss: 2.8749 | LR: 2.03e-05 | 2304.1ms/step | 8.7 samples/s +Step 38430/44000 | Loss: 2.8930 | LR: 2.02e-05 | 2285.3ms/step | 8.8 samples/s +Step 38440/44000 | Loss: 2.8849 | LR: 2.01e-05 | 2312.7ms/step | 8.6 samples/s +Step 38450/44000 | Loss: 2.8807 | LR: 2.00e-05 | 2369.2ms/step | 8.4 samples/s +Step 38460/44000 | Loss: 2.8449 | LR: 2.00e-05 | 2332.7ms/step | 8.6 samples/s +Step 38470/44000 | Loss: 2.9175 | LR: 1.99e-05 | 2354.8ms/step | 8.5 samples/s +Step 38480/44000 | Loss: 2.8842 | LR: 1.98e-05 | 2280.8ms/step | 8.8 samples/s +Step 38490/44000 | Loss: 2.9193 | LR: 1.98e-05 | 2264.8ms/step | 8.8 samples/s +Step 38500/44000 | Loss: 2.9073 | LR: 1.97e-05 | 2270.4ms/step | 8.8 samples/s +Step 38510/44000 | Loss: 2.8629 | LR: 1.96e-05 | 2326.3ms/step | 8.6 samples/s +Step 38520/44000 | Loss: 2.8907 | LR: 1.95e-05 | 2282.2ms/step | 8.8 samples/s +Step 38530/44000 | Loss: 2.8937 | LR: 1.95e-05 | 2290.7ms/step | 8.7 samples/s +Step 38540/44000 | Loss: 2.8725 | LR: 1.94e-05 | 2286.9ms/step | 8.7 samples/s +Step 38550/44000 | Loss: 2.9509 | LR: 1.93e-05 | 2284.5ms/step | 8.8 samples/s +Step 38560/44000 | Loss: 2.8920 | LR: 1.93e-05 | 2333.1ms/step | 8.6 samples/s +Step 38570/44000 | Loss: 2.8869 | LR: 1.92e-05 | 2295.9ms/step | 8.7 samples/s +Step 38580/44000 | Loss: 2.8606 | LR: 1.91e-05 | 2291.5ms/step | 8.7 samples/s +Step 38590/44000 | Loss: 2.9412 | LR: 1.91e-05 | 2285.7ms/step | 8.7 samples/s +Step 38600/44000 | Loss: 2.8963 | LR: 1.90e-05 | 2352.9ms/step | 8.5 samples/s +Step 38610/44000 | Loss: 2.8995 | LR: 1.89e-05 | 2261.3ms/step | 8.8 samples/s +Step 38620/44000 | Loss: 2.9006 | LR: 1.88e-05 | 2285.5ms/step | 8.8 samples/s +Step 38630/44000 | Loss: 2.8569 | LR: 1.88e-05 | 2306.0ms/step | 8.7 samples/s +Step 38640/44000 | Loss: 2.8940 | LR: 1.87e-05 | 2304.4ms/step | 8.7 samples/s +Step 38650/44000 | Loss: 2.9011 | LR: 1.86e-05 | 2281.6ms/step | 8.8 samples/s +Step 38660/44000 | Loss: 2.8875 | LR: 1.86e-05 | 2282.8ms/step | 8.8 samples/s +Step 38670/44000 | Loss: 2.9151 | LR: 1.85e-05 | 2254.5ms/step | 8.9 samples/s +Step 38680/44000 | Loss: 2.9157 | LR: 1.84e-05 | 2312.9ms/step | 8.6 samples/s +Step 38690/44000 | Loss: 2.8724 | LR: 1.84e-05 | 2298.6ms/step | 8.7 samples/s +Step 38700/44000 | Loss: 2.8455 | LR: 1.83e-05 | 2304.3ms/step | 8.7 samples/s +Step 38710/44000 | Loss: 2.8722 | LR: 1.82e-05 | 2294.0ms/step | 8.7 samples/s +Step 38720/44000 | Loss: 2.8770 | LR: 1.82e-05 | 2245.1ms/step | 8.9 samples/s +Step 38730/44000 | Loss: 2.8883 | LR: 1.81e-05 | 2272.4ms/step | 8.8 samples/s +Step 38740/44000 | Loss: 2.9152 | LR: 1.80e-05 | 2254.1ms/step | 8.9 samples/s +Step 38750/44000 | Loss: 2.8751 | LR: 1.80e-05 | 2267.4ms/step | 8.8 samples/s +Step 38760/44000 | Loss: 2.8786 | LR: 1.79e-05 | 2281.5ms/step | 8.8 samples/s +Step 38770/44000 | Loss: 2.9064 | LR: 1.78e-05 | 2316.5ms/step | 8.6 samples/s +Step 38780/44000 | Loss: 2.8938 | LR: 1.78e-05 | 2259.1ms/step | 8.9 samples/s +Step 38790/44000 | Loss: 2.8849 | LR: 1.77e-05 | 2320.1ms/step | 8.6 samples/s +Step 38800/44000 | Loss: 2.8439 | LR: 1.76e-05 | 2293.5ms/step | 8.7 samples/s +Step 38810/44000 | Loss: 2.8699 | LR: 1.76e-05 | 2262.8ms/step | 8.8 samples/s +Step 38820/44000 | Loss: 2.8224 | LR: 1.75e-05 | 2273.0ms/step | 8.8 samples/s +Step 38830/44000 | Loss: 2.9007 | LR: 1.74e-05 | 2284.2ms/step | 8.8 samples/s +Step 38840/44000 | Loss: 2.8934 | LR: 1.74e-05 | 2281.0ms/step | 8.8 samples/s +Step 38850/44000 | Loss: 2.8483 | LR: 1.73e-05 | 2248.9ms/step | 8.9 samples/s +Step 38860/44000 | Loss: 2.8463 | LR: 1.72e-05 | 2310.0ms/step | 8.7 samples/s +Step 38870/44000 | Loss: 2.9168 | LR: 1.72e-05 | 2302.9ms/step | 8.7 samples/s +Step 38880/44000 | Loss: 2.9018 | LR: 1.71e-05 | 2251.4ms/step | 8.9 samples/s +Step 38890/44000 | Loss: 2.9008 | LR: 1.70e-05 | 2270.8ms/step | 8.8 samples/s +Step 38900/44000 | Loss: 2.8807 | LR: 1.70e-05 | 2293.0ms/step | 8.7 samples/s +Step 38910/44000 | Loss: 2.8651 | LR: 1.69e-05 | 2287.3ms/step | 8.7 samples/s +Step 38920/44000 | Loss: 2.8719 | LR: 1.68e-05 | 2243.0ms/step | 8.9 samples/s +Step 38930/44000 | Loss: 2.8609 | LR: 1.68e-05 | 2295.8ms/step | 8.7 samples/s +Step 38940/44000 | Loss: 2.9441 | LR: 1.67e-05 | 2253.6ms/step | 8.9 samples/s +Step 38950/44000 | Loss: 2.8818 | LR: 1.66e-05 | 2274.1ms/step | 8.8 samples/s +Step 38960/44000 | Loss: 2.8805 | LR: 1.66e-05 | 2291.5ms/step | 8.7 samples/s +Step 38970/44000 | Loss: 2.8697 | LR: 1.65e-05 | 2288.4ms/step | 8.7 samples/s +Step 38980/44000 | Loss: 2.9076 | LR: 1.64e-05 | 2296.9ms/step | 8.7 samples/s +Step 38990/44000 | Loss: 2.8556 | LR: 1.64e-05 | 2259.4ms/step | 8.9 samples/s +Step 39000/44000 | Loss: 2.9168 | LR: 1.63e-05 | 2291.0ms/step | 8.7 samples/s +Validation at step 39000: Loss = 2.8947, Perplexity = 18.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-39000 +Step 39010/44000 | Loss: 2.8961 | LR: 1.62e-05 | 4558.3ms/step | 4.4 samples/s +Step 39020/44000 | Loss: 2.8611 | LR: 1.62e-05 | 2256.8ms/step | 8.9 samples/s +Step 39030/44000 | Loss: 2.8770 | LR: 1.61e-05 | 2246.9ms/step | 8.9 samples/s +Step 39040/44000 | Loss: 2.8492 | LR: 1.61e-05 | 2310.1ms/step | 8.7 samples/s +Step 39050/44000 | Loss: 2.8706 | LR: 1.60e-05 | 2267.9ms/step | 8.8 samples/s +Step 39060/44000 | Loss: 2.8396 | LR: 1.59e-05 | 2315.8ms/step | 8.6 samples/s +Step 39070/44000 | Loss: 2.9294 | LR: 1.59e-05 | 2285.1ms/step | 8.8 samples/s +Step 39080/44000 | Loss: 2.9217 | LR: 1.58e-05 | 2346.7ms/step | 8.5 samples/s +Step 39090/44000 | Loss: 2.8595 | LR: 1.57e-05 | 2289.4ms/step | 8.7 samples/s +Step 39100/44000 | Loss: 2.8507 | LR: 1.57e-05 | 2360.8ms/step | 8.5 samples/s +Step 39110/44000 | Loss: 2.8915 | LR: 1.56e-05 | 2282.6ms/step | 8.8 samples/s +Step 39120/44000 | Loss: 2.8780 | LR: 1.55e-05 | 2286.2ms/step | 8.7 samples/s +Step 39130/44000 | Loss: 2.8726 | LR: 1.55e-05 | 2286.2ms/step | 8.7 samples/s +Step 39140/44000 | Loss: 2.8825 | LR: 1.54e-05 | 2251.4ms/step | 8.9 samples/s +Step 39150/44000 | Loss: 2.8789 | LR: 1.54e-05 | 2327.2ms/step | 8.6 samples/s +Step 39160/44000 | Loss: 2.8858 | LR: 1.53e-05 | 2316.5ms/step | 8.6 samples/s +Step 39170/44000 | Loss: 2.8730 | LR: 1.52e-05 | 2322.0ms/step | 8.6 samples/s +Step 39180/44000 | Loss: 2.9154 | LR: 1.52e-05 | 2295.7ms/step | 8.7 samples/s +Step 39190/44000 | Loss: 2.9006 | LR: 1.51e-05 | 2293.9ms/step | 8.7 samples/s +Step 39200/44000 | Loss: 2.8746 | LR: 1.50e-05 | 2308.0ms/step | 8.7 samples/s +Step 39210/44000 | Loss: 2.8853 | LR: 1.50e-05 | 2280.0ms/step | 8.8 samples/s +Step 39220/44000 | Loss: 2.9003 | LR: 1.49e-05 | 2253.0ms/step | 8.9 samples/s +Step 39230/44000 | Loss: 2.9139 | LR: 1.49e-05 | 2333.8ms/step | 8.6 samples/s +Step 39240/44000 | Loss: 2.8567 | LR: 1.48e-05 | 2244.1ms/step | 8.9 samples/s +Step 39250/44000 | Loss: 2.9310 | LR: 1.47e-05 | 2279.6ms/step | 8.8 samples/s +Step 39260/44000 | Loss: 2.8317 | LR: 1.47e-05 | 2272.4ms/step | 8.8 samples/s +Step 39270/44000 | Loss: 2.8505 | LR: 1.46e-05 | 2261.2ms/step | 8.8 samples/s +Step 39280/44000 | Loss: 2.8623 | LR: 1.46e-05 | 2305.7ms/step | 8.7 samples/s +Step 39290/44000 | Loss: 2.8906 | LR: 1.45e-05 | 2260.1ms/step | 8.8 samples/s +Step 39300/44000 | Loss: 2.8373 | LR: 1.44e-05 | 2281.1ms/step | 8.8 samples/s +Step 39310/44000 | Loss: 2.9552 | LR: 1.44e-05 | 2295.5ms/step | 8.7 samples/s +Step 39320/44000 | Loss: 2.9018 | LR: 1.43e-05 | 2306.1ms/step | 8.7 samples/s +Step 39330/44000 | Loss: 2.8926 | LR: 1.42e-05 | 2321.5ms/step | 8.6 samples/s +Step 39340/44000 | Loss: 2.9182 | LR: 1.42e-05 | 2316.2ms/step | 8.6 samples/s +Step 39350/44000 | Loss: 2.8982 | LR: 1.41e-05 | 2264.0ms/step | 8.8 samples/s +Step 39360/44000 | Loss: 2.8787 | LR: 1.41e-05 | 2311.2ms/step | 8.7 samples/s +Step 39370/44000 | Loss: 2.9026 | LR: 1.40e-05 | 2247.9ms/step | 8.9 samples/s +Step 39380/44000 | Loss: 2.9077 | LR: 1.39e-05 | 2243.3ms/step | 8.9 samples/s +Step 39390/44000 | Loss: 2.8566 | LR: 1.39e-05 | 2315.0ms/step | 8.6 samples/s +Step 39400/44000 | Loss: 2.8891 | LR: 1.38e-05 | 2318.9ms/step | 8.6 samples/s +Step 39410/44000 | Loss: 2.8731 | LR: 1.38e-05 | 2273.8ms/step | 8.8 samples/s +Step 39420/44000 | Loss: 2.8946 | LR: 1.37e-05 | 2327.4ms/step | 8.6 samples/s +Step 39430/44000 | Loss: 2.9090 | LR: 1.36e-05 | 2316.9ms/step | 8.6 samples/s +Step 39440/44000 | Loss: 2.8242 | LR: 1.36e-05 | 2307.6ms/step | 8.7 samples/s +Step 39450/44000 | Loss: 2.9139 | LR: 1.35e-05 | 2311.8ms/step | 8.7 samples/s +Step 39460/44000 | Loss: 2.9076 | LR: 1.35e-05 | 2264.0ms/step | 8.8 samples/s +Step 39470/44000 | Loss: 2.8989 | LR: 1.34e-05 | 2283.8ms/step | 8.8 samples/s +Step 39480/44000 | Loss: 2.8740 | LR: 1.34e-05 | 2291.6ms/step | 8.7 samples/s +Step 39490/44000 | Loss: 2.8630 | LR: 1.33e-05 | 2273.3ms/step | 8.8 samples/s +Step 39500/44000 | Loss: 2.8917 | LR: 1.32e-05 | 2302.2ms/step | 8.7 samples/s +Step 39510/44000 | Loss: 2.9519 | LR: 1.32e-05 | 2270.5ms/step | 8.8 samples/s +Step 39520/44000 | Loss: 2.8658 | LR: 1.31e-05 | 2289.3ms/step | 8.7 samples/s +Step 39530/44000 | Loss: 2.8961 | LR: 1.31e-05 | 2302.3ms/step | 8.7 samples/s +Step 39540/44000 | Loss: 2.8688 | LR: 1.30e-05 | 2297.8ms/step | 8.7 samples/s +Step 39550/44000 | Loss: 2.8824 | LR: 1.29e-05 | 2224.8ms/step | 9.0 samples/s +Step 39560/44000 | Loss: 2.8859 | LR: 1.29e-05 | 2254.0ms/step | 8.9 samples/s +Step 39570/44000 | Loss: 2.9000 | LR: 1.28e-05 | 2311.6ms/step | 8.7 samples/s +Step 39580/44000 | Loss: 2.9081 | LR: 1.28e-05 | 2327.2ms/step | 8.6 samples/s +Step 39590/44000 | Loss: 2.8930 | LR: 1.27e-05 | 2290.9ms/step | 8.7 samples/s +Step 39600/44000 | Loss: 2.9042 | LR: 1.27e-05 | 2322.0ms/step | 8.6 samples/s +Step 39610/44000 | Loss: 2.8728 | LR: 1.26e-05 | 2316.9ms/step | 8.6 samples/s +Step 39620/44000 | Loss: 2.8767 | LR: 1.25e-05 | 2307.2ms/step | 8.7 samples/s +Step 39630/44000 | Loss: 2.8941 | LR: 1.25e-05 | 2283.5ms/step | 8.8 samples/s +Step 39640/44000 | Loss: 2.9414 | LR: 1.24e-05 | 2347.9ms/step | 8.5 samples/s +Step 39650/44000 | Loss: 2.9006 | LR: 1.24e-05 | 2248.4ms/step | 8.9 samples/s +Step 39660/44000 | Loss: 2.8701 | LR: 1.23e-05 | 2285.1ms/step | 8.8 samples/s +Step 39670/44000 | Loss: 2.8508 | LR: 1.23e-05 | 2284.7ms/step | 8.8 samples/s +Step 39680/44000 | Loss: 2.8745 | LR: 1.22e-05 | 2310.3ms/step | 8.7 samples/s +Step 39690/44000 | Loss: 2.9279 | LR: 1.22e-05 | 2268.6ms/step | 8.8 samples/s +Step 39700/44000 | Loss: 2.8799 | LR: 1.21e-05 | 2288.6ms/step | 8.7 samples/s +Step 39710/44000 | Loss: 2.9057 | LR: 1.20e-05 | 2284.0ms/step | 8.8 samples/s +Step 39720/44000 | Loss: 2.9230 | LR: 1.20e-05 | 2303.2ms/step | 8.7 samples/s +Step 39730/44000 | Loss: 2.8781 | LR: 1.19e-05 | 2330.7ms/step | 8.6 samples/s +Step 39740/44000 | Loss: 2.8793 | LR: 1.19e-05 | 2331.1ms/step | 8.6 samples/s +Step 39750/44000 | Loss: 2.8719 | LR: 1.18e-05 | 2264.0ms/step | 8.8 samples/s +Step 39760/44000 | Loss: 2.8975 | LR: 1.18e-05 | 2283.2ms/step | 8.8 samples/s +Step 39770/44000 | Loss: 2.9026 | LR: 1.17e-05 | 2312.9ms/step | 8.6 samples/s +Step 39780/44000 | Loss: 2.8995 | LR: 1.17e-05 | 2260.3ms/step | 8.8 samples/s +Step 39790/44000 | Loss: 2.9220 | LR: 1.16e-05 | 2284.2ms/step | 8.8 samples/s +Step 39800/44000 | Loss: 2.9100 | LR: 1.15e-05 | 2263.6ms/step | 8.8 samples/s +Step 39810/44000 | Loss: 2.8860 | LR: 1.15e-05 | 2285.0ms/step | 8.8 samples/s +Step 39820/44000 | Loss: 2.8768 | LR: 1.14e-05 | 2252.5ms/step | 8.9 samples/s +Step 39830/44000 | Loss: 2.9271 | LR: 1.14e-05 | 2295.7ms/step | 8.7 samples/s +Step 39840/44000 | Loss: 2.8992 | LR: 1.13e-05 | 2310.7ms/step | 8.7 samples/s +Step 39850/44000 | Loss: 2.9182 | LR: 1.13e-05 | 2277.5ms/step | 8.8 samples/s +Step 39860/44000 | Loss: 2.8981 | LR: 1.12e-05 | 2323.9ms/step | 8.6 samples/s +Step 39870/44000 | Loss: 2.8653 | LR: 1.12e-05 | 2289.8ms/step | 8.7 samples/s +Step 39880/44000 | Loss: 2.9009 | LR: 1.11e-05 | 2300.7ms/step | 8.7 samples/s +Step 39890/44000 | Loss: 2.8263 | LR: 1.11e-05 | 2289.8ms/step | 8.7 samples/s +Step 39900/44000 | Loss: 2.8809 | LR: 1.10e-05 | 2274.8ms/step | 8.8 samples/s +Step 39910/44000 | Loss: 2.9177 | LR: 1.10e-05 | 2305.5ms/step | 8.7 samples/s +Step 39920/44000 | Loss: 2.8368 | LR: 1.09e-05 | 2284.4ms/step | 8.8 samples/s +Step 39930/44000 | Loss: 2.9021 | LR: 1.08e-05 | 2305.7ms/step | 8.7 samples/s +Step 39940/44000 | Loss: 2.8770 | LR: 1.08e-05 | 2305.8ms/step | 8.7 samples/s +Step 39950/44000 | Loss: 2.8563 | LR: 1.07e-05 | 2237.0ms/step | 8.9 samples/s +Step 39960/44000 | Loss: 2.9108 | LR: 1.07e-05 | 2269.8ms/step | 8.8 samples/s +Step 39970/44000 | Loss: 2.8903 | LR: 1.06e-05 | 2248.7ms/step | 8.9 samples/s +Step 39980/44000 | Loss: 2.8042 | LR: 1.06e-05 | 2306.3ms/step | 8.7 samples/s +Step 39990/44000 | Loss: 2.8358 | LR: 1.05e-05 | 2268.6ms/step | 8.8 samples/s +Step 40000/44000 | Loss: 2.9003 | LR: 1.05e-05 | 2256.2ms/step | 8.9 samples/s +Validation at step 40000: Loss = 2.8946, Perplexity = 18.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-40000 +Step 40010/44000 | Loss: 2.9081 | LR: 1.04e-05 | 4489.7ms/step | 4.5 samples/s +Step 40020/44000 | Loss: 2.8567 | LR: 1.04e-05 | 2261.6ms/step | 8.8 samples/s +Step 40030/44000 | Loss: 2.9102 | LR: 1.03e-05 | 2302.4ms/step | 8.7 samples/s +Step 40040/44000 | Loss: 2.8568 | LR: 1.03e-05 | 2257.3ms/step | 8.9 samples/s +Step 40050/44000 | Loss: 2.8907 | LR: 1.02e-05 | 2312.6ms/step | 8.6 samples/s +Step 40060/44000 | Loss: 2.8926 | LR: 1.02e-05 | 2264.4ms/step | 8.8 samples/s +Step 40070/44000 | Loss: 2.8829 | LR: 1.01e-05 | 2262.8ms/step | 8.8 samples/s +Step 40080/44000 | Loss: 2.8661 | LR: 1.01e-05 | 2279.6ms/step | 8.8 samples/s +Step 40090/44000 | Loss: 2.8583 | LR: 1.00e-05 | 2262.1ms/step | 8.8 samples/s +Step 40100/44000 | Loss: 2.8788 | LR: 9.96e-06 | 2282.0ms/step | 8.8 samples/s +Step 40110/44000 | Loss: 2.8402 | LR: 9.91e-06 | 2288.5ms/step | 8.7 samples/s +Step 40120/44000 | Loss: 2.8977 | LR: 9.86e-06 | 2282.4ms/step | 8.8 samples/s +Step 40130/44000 | Loss: 2.8647 | LR: 9.81e-06 | 2275.6ms/step | 8.8 samples/s +Step 40140/44000 | Loss: 2.8931 | LR: 9.76e-06 | 2269.4ms/step | 8.8 samples/s +Step 40150/44000 | Loss: 2.8418 | LR: 9.71e-06 | 2275.2ms/step | 8.8 samples/s +Step 40160/44000 | Loss: 2.8935 | LR: 9.66e-06 | 2270.1ms/step | 8.8 samples/s +Step 40170/44000 | Loss: 2.8894 | LR: 9.61e-06 | 2373.9ms/step | 8.4 samples/s +Step 40180/44000 | Loss: 2.9034 | LR: 9.56e-06 | 2259.2ms/step | 8.9 samples/s +Step 40190/44000 | Loss: 2.9012 | LR: 9.51e-06 | 2322.6ms/step | 8.6 samples/s +Step 40200/44000 | Loss: 2.8832 | LR: 9.46e-06 | 2290.0ms/step | 8.7 samples/s +Step 40210/44000 | Loss: 2.8649 | LR: 9.41e-06 | 2258.5ms/step | 8.9 samples/s +Step 40220/44000 | Loss: 2.8526 | LR: 9.36e-06 | 2316.5ms/step | 8.6 samples/s +Step 40230/44000 | Loss: 2.9116 | LR: 9.32e-06 | 2279.8ms/step | 8.8 samples/s +Step 40240/44000 | Loss: 2.9186 | LR: 9.27e-06 | 2299.9ms/step | 8.7 samples/s +Step 40250/44000 | Loss: 2.8811 | LR: 9.22e-06 | 2272.5ms/step | 8.8 samples/s +Step 40260/44000 | Loss: 2.8628 | LR: 9.17e-06 | 2243.3ms/step | 8.9 samples/s +Step 40270/44000 | Loss: 2.8876 | LR: 9.12e-06 | 2252.0ms/step | 8.9 samples/s +Step 40280/44000 | Loss: 2.8850 | LR: 9.07e-06 | 2251.4ms/step | 8.9 samples/s +Step 40290/44000 | Loss: 2.9145 | LR: 9.02e-06 | 2269.4ms/step | 8.8 samples/s +Step 40300/44000 | Loss: 2.8991 | LR: 8.97e-06 | 2290.1ms/step | 8.7 samples/s +Step 40310/44000 | Loss: 2.8783 | LR: 8.93e-06 | 2274.6ms/step | 8.8 samples/s +Step 40320/44000 | Loss: 2.8702 | LR: 8.88e-06 | 2279.5ms/step | 8.8 samples/s +Step 40330/44000 | Loss: 2.8885 | LR: 8.83e-06 | 2297.9ms/step | 8.7 samples/s +Step 40340/44000 | Loss: 2.8765 | LR: 8.78e-06 | 2271.6ms/step | 8.8 samples/s +Step 40350/44000 | Loss: 2.9035 | LR: 8.74e-06 | 2319.6ms/step | 8.6 samples/s +Step 40360/44000 | Loss: 2.9014 | LR: 8.69e-06 | 2272.2ms/step | 8.8 samples/s +Step 40370/44000 | Loss: 2.8796 | LR: 8.64e-06 | 2258.6ms/step | 8.9 samples/s +Step 40380/44000 | Loss: 2.8732 | LR: 8.59e-06 | 2259.6ms/step | 8.9 samples/s +Step 40390/44000 | Loss: 2.8685 | LR: 8.55e-06 | 2286.0ms/step | 8.7 samples/s +Step 40400/44000 | Loss: 2.9006 | LR: 8.50e-06 | 2243.2ms/step | 8.9 samples/s +Step 40410/44000 | Loss: 2.8694 | LR: 8.45e-06 | 2254.0ms/step | 8.9 samples/s +Step 40420/44000 | Loss: 2.8815 | LR: 8.41e-06 | 2290.3ms/step | 8.7 samples/s +Step 40430/44000 | Loss: 2.9017 | LR: 8.36e-06 | 2301.2ms/step | 8.7 samples/s +Step 40440/44000 | Loss: 2.8429 | LR: 8.31e-06 | 2286.3ms/step | 8.7 samples/s +Step 40450/44000 | Loss: 2.9092 | LR: 8.27e-06 | 2292.1ms/step | 8.7 samples/s +Step 40460/44000 | Loss: 2.8680 | LR: 8.22e-06 | 2277.0ms/step | 8.8 samples/s +Step 40470/44000 | Loss: 2.8604 | LR: 8.17e-06 | 2327.5ms/step | 8.6 samples/s +Step 40480/44000 | Loss: 2.8937 | LR: 8.13e-06 | 2280.4ms/step | 8.8 samples/s +Step 40490/44000 | Loss: 2.9269 | LR: 8.08e-06 | 2302.3ms/step | 8.7 samples/s +Step 40500/44000 | Loss: 2.8932 | LR: 8.04e-06 | 2267.2ms/step | 8.8 samples/s +Step 40510/44000 | Loss: 2.8812 | LR: 7.99e-06 | 2299.3ms/step | 8.7 samples/s +Step 40520/44000 | Loss: 2.9127 | LR: 7.94e-06 | 2325.7ms/step | 8.6 samples/s +Step 40530/44000 | Loss: 2.8971 | LR: 7.90e-06 | 2325.8ms/step | 8.6 samples/s +Step 40540/44000 | Loss: 2.9360 | LR: 7.85e-06 | 2326.1ms/step | 8.6 samples/s +Step 40550/44000 | Loss: 2.9096 | LR: 7.81e-06 | 2300.5ms/step | 8.7 samples/s +Step 40560/44000 | Loss: 2.9110 | LR: 7.76e-06 | 2318.9ms/step | 8.6 samples/s +Step 40570/44000 | Loss: 2.8746 | LR: 7.72e-06 | 2272.1ms/step | 8.8 samples/s +Step 40580/44000 | Loss: 2.8945 | LR: 7.67e-06 | 2244.3ms/step | 8.9 samples/s +Step 40590/44000 | Loss: 2.8642 | LR: 7.63e-06 | 2280.4ms/step | 8.8 samples/s +Step 40600/44000 | Loss: 2.8856 | LR: 7.59e-06 | 2262.4ms/step | 8.8 samples/s +Step 40610/44000 | Loss: 2.8687 | LR: 7.54e-06 | 2299.4ms/step | 8.7 samples/s +Step 40620/44000 | Loss: 2.9213 | LR: 7.50e-06 | 2270.3ms/step | 8.8 samples/s +Step 40630/44000 | Loss: 2.9041 | LR: 7.45e-06 | 2305.3ms/step | 8.7 samples/s +Step 40640/44000 | Loss: 2.9061 | LR: 7.41e-06 | 2305.9ms/step | 8.7 samples/s +Step 40650/44000 | Loss: 2.9067 | LR: 7.37e-06 | 2305.5ms/step | 8.7 samples/s +Step 40660/44000 | Loss: 2.9223 | LR: 7.32e-06 | 2237.4ms/step | 8.9 samples/s +Step 40670/44000 | Loss: 2.8873 | LR: 7.28e-06 | 2258.3ms/step | 8.9 samples/s +Step 40680/44000 | Loss: 2.8324 | LR: 7.23e-06 | 2305.0ms/step | 8.7 samples/s +Step 40690/44000 | Loss: 2.9188 | LR: 7.19e-06 | 2252.7ms/step | 8.9 samples/s +Step 40700/44000 | Loss: 2.8710 | LR: 7.15e-06 | 2268.6ms/step | 8.8 samples/s +Step 40710/44000 | Loss: 2.8892 | LR: 7.10e-06 | 2262.4ms/step | 8.8 samples/s +Step 40720/44000 | Loss: 2.8825 | LR: 7.06e-06 | 2274.3ms/step | 8.8 samples/s +Step 40730/44000 | Loss: 2.9192 | LR: 7.02e-06 | 2268.7ms/step | 8.8 samples/s +Step 40740/44000 | Loss: 2.8685 | LR: 6.98e-06 | 2232.1ms/step | 9.0 samples/s +Step 40750/44000 | Loss: 2.8632 | LR: 6.93e-06 | 2284.3ms/step | 8.8 samples/s +Step 40760/44000 | Loss: 2.8977 | LR: 6.89e-06 | 2357.9ms/step | 8.5 samples/s +Step 40770/44000 | Loss: 2.8397 | LR: 6.85e-06 | 2332.2ms/step | 8.6 samples/s +Step 40780/44000 | Loss: 2.9447 | LR: 6.81e-06 | 2253.1ms/step | 8.9 samples/s +Step 40790/44000 | Loss: 2.8711 | LR: 6.77e-06 | 2279.4ms/step | 8.8 samples/s +Step 40800/44000 | Loss: 2.9376 | LR: 6.72e-06 | 2301.2ms/step | 8.7 samples/s +Step 40810/44000 | Loss: 2.8831 | LR: 6.68e-06 | 2315.0ms/step | 8.6 samples/s +Step 40820/44000 | Loss: 2.8871 | LR: 6.64e-06 | 2334.6ms/step | 8.6 samples/s +Step 40830/44000 | Loss: 2.8970 | LR: 6.60e-06 | 2260.7ms/step | 8.8 samples/s +Step 40840/44000 | Loss: 2.8704 | LR: 6.56e-06 | 2309.0ms/step | 8.7 samples/s +Step 40850/44000 | Loss: 2.8876 | LR: 6.52e-06 | 2276.4ms/step | 8.8 samples/s +Step 40860/44000 | Loss: 2.8639 | LR: 6.47e-06 | 2336.4ms/step | 8.6 samples/s +Step 40870/44000 | Loss: 2.8886 | LR: 6.43e-06 | 2321.4ms/step | 8.6 samples/s +Step 40880/44000 | Loss: 2.9059 | LR: 6.39e-06 | 2256.9ms/step | 8.9 samples/s +Step 40890/44000 | Loss: 2.8738 | LR: 6.35e-06 | 2247.5ms/step | 8.9 samples/s +Step 40900/44000 | Loss: 2.9507 | LR: 6.31e-06 | 2310.5ms/step | 8.7 samples/s +Step 40910/44000 | Loss: 2.8779 | LR: 6.27e-06 | 2304.7ms/step | 8.7 samples/s +Step 40920/44000 | Loss: 2.8920 | LR: 6.23e-06 | 2268.3ms/step | 8.8 samples/s +Step 40930/44000 | Loss: 2.8678 | LR: 6.19e-06 | 2264.6ms/step | 8.8 samples/s +Step 40940/44000 | Loss: 2.9198 | LR: 6.15e-06 | 2267.2ms/step | 8.8 samples/s +Step 40950/44000 | Loss: 2.8867 | LR: 6.11e-06 | 2271.6ms/step | 8.8 samples/s +Step 40960/44000 | Loss: 2.8759 | LR: 6.07e-06 | 2283.1ms/step | 8.8 samples/s +Step 40970/44000 | Loss: 2.8854 | LR: 6.03e-06 | 2309.0ms/step | 8.7 samples/s +Step 40980/44000 | Loss: 2.8882 | LR: 5.99e-06 | 2266.6ms/step | 8.8 samples/s +Step 40990/44000 | Loss: 2.8495 | LR: 5.95e-06 | 2264.1ms/step | 8.8 samples/s +Step 41000/44000 | Loss: 2.8796 | LR: 5.91e-06 | 2115.4ms/step | 9.5 samples/s +Validation at step 41000: Loss = 2.8946, Perplexity = 18.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-41000 +Step 41010/44000 | Loss: 2.9102 | LR: 5.87e-06 | 4684.8ms/step | 4.3 samples/s +Step 41020/44000 | Loss: 2.9145 | LR: 5.83e-06 | 2322.5ms/step | 8.6 samples/s +Step 41030/44000 | Loss: 2.9620 | LR: 5.80e-06 | 2281.1ms/step | 8.8 samples/s +Step 41040/44000 | Loss: 2.8783 | LR: 5.76e-06 | 2280.7ms/step | 8.8 samples/s +Step 41050/44000 | Loss: 2.8770 | LR: 5.72e-06 | 2253.8ms/step | 8.9 samples/s +Step 41060/44000 | Loss: 2.8975 | LR: 5.68e-06 | 2255.7ms/step | 8.9 samples/s +Step 41070/44000 | Loss: 2.8787 | LR: 5.64e-06 | 2227.2ms/step | 9.0 samples/s +Step 41080/44000 | Loss: 2.8503 | LR: 5.60e-06 | 2253.2ms/step | 8.9 samples/s +Step 41090/44000 | Loss: 2.8978 | LR: 5.56e-06 | 2221.3ms/step | 9.0 samples/s +Step 41100/44000 | Loss: 2.8985 | LR: 5.53e-06 | 2269.5ms/step | 8.8 samples/s +Step 41110/44000 | Loss: 2.8787 | LR: 5.49e-06 | 2305.7ms/step | 8.7 samples/s +Step 41120/44000 | Loss: 2.9269 | LR: 5.45e-06 | 2321.5ms/step | 8.6 samples/s +Step 41130/44000 | Loss: 2.9106 | LR: 5.41e-06 | 2267.5ms/step | 8.8 samples/s +Step 41140/44000 | Loss: 2.8723 | LR: 5.38e-06 | 2306.8ms/step | 8.7 samples/s +Step 41150/44000 | Loss: 2.9097 | LR: 5.34e-06 | 2314.9ms/step | 8.6 samples/s +Step 41160/44000 | Loss: 2.8880 | LR: 5.30e-06 | 2253.3ms/step | 8.9 samples/s +Step 41170/44000 | Loss: 2.8523 | LR: 5.26e-06 | 2305.2ms/step | 8.7 samples/s +Step 41180/44000 | Loss: 2.9096 | LR: 5.23e-06 | 2287.7ms/step | 8.7 samples/s +Step 41190/44000 | Loss: 2.9296 | LR: 5.19e-06 | 2344.8ms/step | 8.5 samples/s +Step 41200/44000 | Loss: 2.9098 | LR: 5.15e-06 | 2336.9ms/step | 8.6 samples/s +Step 41210/44000 | Loss: 2.9097 | LR: 5.12e-06 | 2345.1ms/step | 8.5 samples/s +Step 41220/44000 | Loss: 2.9124 | LR: 5.08e-06 | 2313.5ms/step | 8.6 samples/s +Step 41230/44000 | Loss: 2.9085 | LR: 5.04e-06 | 2319.7ms/step | 8.6 samples/s +Step 41240/44000 | Loss: 2.8974 | LR: 5.01e-06 | 2287.6ms/step | 8.7 samples/s +Step 41250/44000 | Loss: 2.8931 | LR: 4.97e-06 | 2244.8ms/step | 8.9 samples/s +Step 41260/44000 | Loss: 2.9186 | LR: 4.94e-06 | 2272.6ms/step | 8.8 samples/s +Step 41270/44000 | Loss: 2.8695 | LR: 4.90e-06 | 2274.6ms/step | 8.8 samples/s +Step 41280/44000 | Loss: 2.8915 | LR: 4.86e-06 | 2317.8ms/step | 8.6 samples/s +Step 41290/44000 | Loss: 2.9173 | LR: 4.83e-06 | 2261.3ms/step | 8.8 samples/s +Step 41300/44000 | Loss: 2.8806 | LR: 4.79e-06 | 2307.3ms/step | 8.7 samples/s +Step 41310/44000 | Loss: 2.8630 | LR: 4.76e-06 | 2245.6ms/step | 8.9 samples/s +Step 41320/44000 | Loss: 2.8664 | LR: 4.72e-06 | 2318.0ms/step | 8.6 samples/s +Step 41330/44000 | Loss: 2.8847 | LR: 4.69e-06 | 2284.1ms/step | 8.8 samples/s +Step 41340/44000 | Loss: 2.9142 | LR: 4.65e-06 | 2290.4ms/step | 8.7 samples/s +Step 41350/44000 | Loss: 2.8726 | LR: 4.62e-06 | 2288.5ms/step | 8.7 samples/s +Step 41360/44000 | Loss: 2.8646 | LR: 4.58e-06 | 2321.6ms/step | 8.6 samples/s +Step 41370/44000 | Loss: 2.8964 | LR: 4.55e-06 | 2264.6ms/step | 8.8 samples/s +Step 41380/44000 | Loss: 2.9004 | LR: 4.51e-06 | 2352.9ms/step | 8.5 samples/s +Step 41390/44000 | Loss: 2.8878 | LR: 4.48e-06 | 2286.3ms/step | 8.7 samples/s +Step 41400/44000 | Loss: 2.8633 | LR: 4.45e-06 | 2319.8ms/step | 8.6 samples/s +Step 41410/44000 | Loss: 2.8890 | LR: 4.41e-06 | 2317.2ms/step | 8.6 samples/s +Step 41420/44000 | Loss: 2.8874 | LR: 4.38e-06 | 2372.7ms/step | 8.4 samples/s +Step 41430/44000 | Loss: 2.9082 | LR: 4.34e-06 | 2317.0ms/step | 8.6 samples/s +Step 41440/44000 | Loss: 2.9155 | LR: 4.31e-06 | 2322.5ms/step | 8.6 samples/s +Step 41450/44000 | Loss: 2.8871 | LR: 4.28e-06 | 2316.1ms/step | 8.6 samples/s +Step 41460/44000 | Loss: 2.8794 | LR: 4.24e-06 | 2273.9ms/step | 8.8 samples/s +Step 41470/44000 | Loss: 2.8714 | LR: 4.21e-06 | 2270.1ms/step | 8.8 samples/s +Step 41480/44000 | Loss: 2.8988 | LR: 4.18e-06 | 2298.6ms/step | 8.7 samples/s +Step 41490/44000 | Loss: 2.8683 | LR: 4.14e-06 | 2266.6ms/step | 8.8 samples/s +Step 41500/44000 | Loss: 2.8844 | LR: 4.11e-06 | 2304.2ms/step | 8.7 samples/s +Step 41510/44000 | Loss: 2.8855 | LR: 4.08e-06 | 2333.9ms/step | 8.6 samples/s +Step 41520/44000 | Loss: 2.8437 | LR: 4.05e-06 | 2307.6ms/step | 8.7 samples/s +Step 41530/44000 | Loss: 2.8852 | LR: 4.01e-06 | 2304.3ms/step | 8.7 samples/s +Step 41540/44000 | Loss: 2.8769 | LR: 3.98e-06 | 2284.1ms/step | 8.8 samples/s +Step 41550/44000 | Loss: 2.8686 | LR: 3.95e-06 | 2286.4ms/step | 8.7 samples/s +Step 41560/44000 | Loss: 2.8752 | LR: 3.92e-06 | 2319.7ms/step | 8.6 samples/s +Step 41570/44000 | Loss: 2.8822 | LR: 3.88e-06 | 2305.5ms/step | 8.7 samples/s +Step 41580/44000 | Loss: 2.8648 | LR: 3.85e-06 | 2327.1ms/step | 8.6 samples/s +Step 41590/44000 | Loss: 2.8775 | LR: 3.82e-06 | 2345.1ms/step | 8.5 samples/s +Step 41600/44000 | Loss: 2.8715 | LR: 3.79e-06 | 2308.4ms/step | 8.7 samples/s +Step 41610/44000 | Loss: 2.9259 | LR: 3.76e-06 | 2286.3ms/step | 8.7 samples/s +Step 41620/44000 | Loss: 2.8417 | LR: 3.73e-06 | 2361.7ms/step | 8.5 samples/s +Step 41630/44000 | Loss: 2.8862 | LR: 3.70e-06 | 2299.9ms/step | 8.7 samples/s +Step 41640/44000 | Loss: 2.9020 | LR: 3.66e-06 | 2343.5ms/step | 8.5 samples/s +Step 41650/44000 | Loss: 2.9219 | LR: 3.63e-06 | 2337.2ms/step | 8.6 samples/s +Step 41660/44000 | Loss: 2.8677 | LR: 3.60e-06 | 2305.7ms/step | 8.7 samples/s +Step 41670/44000 | Loss: 2.8864 | LR: 3.57e-06 | 2352.7ms/step | 8.5 samples/s +Step 41680/44000 | Loss: 2.8875 | LR: 3.54e-06 | 2300.8ms/step | 8.7 samples/s +Step 41690/44000 | Loss: 2.9158 | LR: 3.51e-06 | 2304.8ms/step | 8.7 samples/s +Step 41700/44000 | Loss: 2.8756 | LR: 3.48e-06 | 2333.2ms/step | 8.6 samples/s +Step 41710/44000 | Loss: 2.8558 | LR: 3.45e-06 | 2326.6ms/step | 8.6 samples/s +Step 41720/44000 | Loss: 2.8606 | LR: 3.42e-06 | 2305.5ms/step | 8.7 samples/s +Step 41730/44000 | Loss: 2.8853 | LR: 3.39e-06 | 2341.9ms/step | 8.5 samples/s +Step 41740/44000 | Loss: 2.8903 | LR: 3.36e-06 | 2334.8ms/step | 8.6 samples/s +Step 41750/44000 | Loss: 2.8762 | LR: 3.33e-06 | 2321.6ms/step | 8.6 samples/s +Step 41760/44000 | Loss: 2.8801 | LR: 3.30e-06 | 2296.3ms/step | 8.7 samples/s +Step 41770/44000 | Loss: 2.8694 | LR: 3.27e-06 | 2316.2ms/step | 8.6 samples/s +Step 41780/44000 | Loss: 2.9224 | LR: 3.24e-06 | 2331.5ms/step | 8.6 samples/s +Step 41790/44000 | Loss: 2.9126 | LR: 3.21e-06 | 2294.6ms/step | 8.7 samples/s +Step 41800/44000 | Loss: 2.9098 | LR: 3.19e-06 | 2266.7ms/step | 8.8 samples/s +Step 41810/44000 | Loss: 2.8771 | LR: 3.16e-06 | 2275.9ms/step | 8.8 samples/s +Step 41820/44000 | Loss: 2.9013 | LR: 3.13e-06 | 2330.9ms/step | 8.6 samples/s +Step 41830/44000 | Loss: 2.8696 | LR: 3.10e-06 | 2344.0ms/step | 8.5 samples/s +Step 41840/44000 | Loss: 2.9192 | LR: 3.07e-06 | 2317.3ms/step | 8.6 samples/s +Step 41850/44000 | Loss: 2.8844 | LR: 3.04e-06 | 2324.2ms/step | 8.6 samples/s +Step 41860/44000 | Loss: 2.8518 | LR: 3.01e-06 | 2300.8ms/step | 8.7 samples/s +Step 41870/44000 | Loss: 2.8358 | LR: 2.99e-06 | 2311.5ms/step | 8.7 samples/s +Step 41880/44000 | Loss: 2.9329 | LR: 2.96e-06 | 2316.3ms/step | 8.6 samples/s +Step 41890/44000 | Loss: 2.8993 | LR: 2.93e-06 | 2284.5ms/step | 8.8 samples/s +Step 41900/44000 | Loss: 2.8697 | LR: 2.90e-06 | 2331.8ms/step | 8.6 samples/s +Step 41910/44000 | Loss: 2.9007 | LR: 2.88e-06 | 2277.5ms/step | 8.8 samples/s +Step 41920/44000 | Loss: 2.8926 | LR: 2.85e-06 | 2336.7ms/step | 8.6 samples/s +Step 41930/44000 | Loss: 2.8746 | LR: 2.82e-06 | 2278.6ms/step | 8.8 samples/s +Step 41940/44000 | Loss: 2.8901 | LR: 2.79e-06 | 2274.3ms/step | 8.8 samples/s +Step 41950/44000 | Loss: 2.8603 | LR: 2.77e-06 | 2306.1ms/step | 8.7 samples/s +Step 41960/44000 | Loss: 2.9211 | LR: 2.74e-06 | 2298.0ms/step | 8.7 samples/s +Step 41970/44000 | Loss: 2.9045 | LR: 2.71e-06 | 2274.6ms/step | 8.8 samples/s +Step 41980/44000 | Loss: 2.9158 | LR: 2.69e-06 | 2316.1ms/step | 8.6 samples/s +Step 41990/44000 | Loss: 2.8802 | LR: 2.66e-06 | 2135.9ms/step | 9.4 samples/s +Step 42000/44000 | Loss: 2.8633 | LR: 2.63e-06 | 2270.8ms/step | 8.8 samples/s +Validation at step 42000: Loss = 2.8945, Perplexity = 18.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-42000 +Step 42010/44000 | Loss: 2.8971 | LR: 2.61e-06 | 4680.6ms/step | 4.3 samples/s +Step 42020/44000 | Loss: 2.8799 | LR: 2.58e-06 | 2289.7ms/step | 8.7 samples/s +Step 42030/44000 | Loss: 2.8983 | LR: 2.56e-06 | 2328.5ms/step | 8.6 samples/s +Step 42040/44000 | Loss: 2.9064 | LR: 2.53e-06 | 2303.7ms/step | 8.7 samples/s +Step 42050/44000 | Loss: 2.9364 | LR: 2.50e-06 | 2304.4ms/step | 8.7 samples/s +Step 42060/44000 | Loss: 2.9194 | LR: 2.48e-06 | 2269.2ms/step | 8.8 samples/s +Step 42070/44000 | Loss: 2.8802 | LR: 2.45e-06 | 2343.4ms/step | 8.5 samples/s +Step 42080/44000 | Loss: 2.8810 | LR: 2.43e-06 | 2295.9ms/step | 8.7 samples/s +Step 42090/44000 | Loss: 2.8994 | LR: 2.40e-06 | 2300.7ms/step | 8.7 samples/s +Step 42100/44000 | Loss: 2.8612 | LR: 2.38e-06 | 2310.7ms/step | 8.7 samples/s +Step 42110/44000 | Loss: 2.8751 | LR: 2.35e-06 | 2274.4ms/step | 8.8 samples/s +Step 42120/44000 | Loss: 2.9358 | LR: 2.33e-06 | 2279.7ms/step | 8.8 samples/s +Step 42130/44000 | Loss: 2.8984 | LR: 2.30e-06 | 2305.6ms/step | 8.7 samples/s +Step 42140/44000 | Loss: 2.8678 | LR: 2.28e-06 | 2284.8ms/step | 8.8 samples/s +Step 42150/44000 | Loss: 2.9131 | LR: 2.25e-06 | 2267.5ms/step | 8.8 samples/s +Step 42160/44000 | Loss: 2.9431 | LR: 2.23e-06 | 2291.2ms/step | 8.7 samples/s +Step 42170/44000 | Loss: 2.8667 | LR: 2.21e-06 | 2319.3ms/step | 8.6 samples/s +Step 42180/44000 | Loss: 2.8975 | LR: 2.18e-06 | 2301.5ms/step | 8.7 samples/s +Step 42190/44000 | Loss: 2.8898 | LR: 2.16e-06 | 2285.3ms/step | 8.8 samples/s +Step 42200/44000 | Loss: 2.8790 | LR: 2.13e-06 | 2289.1ms/step | 8.7 samples/s +Step 42210/44000 | Loss: 2.8318 | LR: 2.11e-06 | 2295.1ms/step | 8.7 samples/s +Step 42220/44000 | Loss: 2.8693 | LR: 2.09e-06 | 2283.5ms/step | 8.8 samples/s +Step 42230/44000 | Loss: 2.8800 | LR: 2.06e-06 | 2286.2ms/step | 8.7 samples/s +Step 42240/44000 | Loss: 2.8841 | LR: 2.04e-06 | 2327.7ms/step | 8.6 samples/s +Step 42250/44000 | Loss: 2.8792 | LR: 2.02e-06 | 2315.4ms/step | 8.6 samples/s +Step 42260/44000 | Loss: 2.9046 | LR: 1.99e-06 | 2326.1ms/step | 8.6 samples/s +Step 42270/44000 | Loss: 2.9066 | LR: 1.97e-06 | 2303.3ms/step | 8.7 samples/s +Step 42280/44000 | Loss: 2.8968 | LR: 1.95e-06 | 2298.9ms/step | 8.7 samples/s +Step 42290/44000 | Loss: 2.8595 | LR: 1.93e-06 | 2288.8ms/step | 8.7 samples/s +Step 42300/44000 | Loss: 2.9023 | LR: 1.90e-06 | 2319.8ms/step | 8.6 samples/s +Step 42310/44000 | Loss: 2.8831 | LR: 1.88e-06 | 2321.2ms/step | 8.6 samples/s +Step 42320/44000 | Loss: 2.9011 | LR: 1.86e-06 | 2293.8ms/step | 8.7 samples/s +Step 42330/44000 | Loss: 2.8816 | LR: 1.84e-06 | 2320.8ms/step | 8.6 samples/s +Step 42340/44000 | Loss: 2.8999 | LR: 1.82e-06 | 2320.6ms/step | 8.6 samples/s +Step 42350/44000 | Loss: 2.9005 | LR: 1.79e-06 | 2318.6ms/step | 8.6 samples/s +Step 42360/44000 | Loss: 2.9183 | LR: 1.77e-06 | 2333.4ms/step | 8.6 samples/s +Step 42370/44000 | Loss: 2.8890 | LR: 1.75e-06 | 2321.4ms/step | 8.6 samples/s +Step 42380/44000 | Loss: 2.8596 | LR: 1.73e-06 | 2278.6ms/step | 8.8 samples/s +Step 42390/44000 | Loss: 2.9235 | LR: 1.71e-06 | 2299.3ms/step | 8.7 samples/s +Step 42400/44000 | Loss: 2.8682 | LR: 1.69e-06 | 2280.3ms/step | 8.8 samples/s +Step 42410/44000 | Loss: 2.8657 | LR: 1.67e-06 | 2302.9ms/step | 8.7 samples/s +Step 42420/44000 | Loss: 2.8810 | LR: 1.64e-06 | 2288.6ms/step | 8.7 samples/s +Step 42430/44000 | Loss: 2.8707 | LR: 1.62e-06 | 2309.7ms/step | 8.7 samples/s +Step 42440/44000 | Loss: 2.8816 | LR: 1.60e-06 | 2305.8ms/step | 8.7 samples/s +Step 42450/44000 | Loss: 2.8765 | LR: 1.58e-06 | 2330.7ms/step | 8.6 samples/s +Step 42460/44000 | Loss: 2.8668 | LR: 1.56e-06 | 2354.4ms/step | 8.5 samples/s +Step 42470/44000 | Loss: 2.8692 | LR: 1.54e-06 | 2310.5ms/step | 8.7 samples/s +Step 42480/44000 | Loss: 2.9084 | LR: 1.52e-06 | 2326.3ms/step | 8.6 samples/s +Step 42490/44000 | Loss: 2.8981 | LR: 1.50e-06 | 2305.5ms/step | 8.7 samples/s +Step 42500/44000 | Loss: 2.9018 | LR: 1.48e-06 | 2310.5ms/step | 8.7 samples/s +Step 42510/44000 | Loss: 2.9276 | LR: 1.46e-06 | 2300.4ms/step | 8.7 samples/s +Step 42520/44000 | Loss: 2.8998 | LR: 1.44e-06 | 2263.8ms/step | 8.8 samples/s +Step 42530/44000 | Loss: 2.8968 | LR: 1.42e-06 | 2300.7ms/step | 8.7 samples/s +Step 42540/44000 | Loss: 2.9310 | LR: 1.40e-06 | 2265.6ms/step | 8.8 samples/s +Step 42550/44000 | Loss: 2.8741 | LR: 1.39e-06 | 2305.2ms/step | 8.7 samples/s +Step 42560/44000 | Loss: 2.8847 | LR: 1.37e-06 | 2360.4ms/step | 8.5 samples/s +Step 42570/44000 | Loss: 2.9049 | LR: 1.35e-06 | 2287.8ms/step | 8.7 samples/s +Step 42580/44000 | Loss: 2.9472 | LR: 1.33e-06 | 2292.4ms/step | 8.7 samples/s +Step 42590/44000 | Loss: 2.8730 | LR: 1.31e-06 | 2315.5ms/step | 8.6 samples/s +Step 42600/44000 | Loss: 2.8806 | LR: 1.29e-06 | 2281.0ms/step | 8.8 samples/s +Step 42610/44000 | Loss: 2.9019 | LR: 1.27e-06 | 2352.4ms/step | 8.5 samples/s +Step 42620/44000 | Loss: 2.8811 | LR: 1.25e-06 | 2347.0ms/step | 8.5 samples/s +Step 42630/44000 | Loss: 2.9089 | LR: 1.24e-06 | 2303.3ms/step | 8.7 samples/s +Step 42640/44000 | Loss: 2.9035 | LR: 1.22e-06 | 2295.7ms/step | 8.7 samples/s +Step 42650/44000 | Loss: 2.9126 | LR: 1.20e-06 | 2290.4ms/step | 8.7 samples/s +Step 42660/44000 | Loss: 2.8670 | LR: 1.18e-06 | 2295.1ms/step | 8.7 samples/s +Step 42670/44000 | Loss: 2.8866 | LR: 1.17e-06 | 2284.5ms/step | 8.8 samples/s +Step 42680/44000 | Loss: 2.8904 | LR: 1.15e-06 | 2305.7ms/step | 8.7 samples/s +Step 42690/44000 | Loss: 2.9208 | LR: 1.13e-06 | 2338.6ms/step | 8.6 samples/s +Step 42700/44000 | Loss: 2.8656 | LR: 1.11e-06 | 2290.4ms/step | 8.7 samples/s +Step 42710/44000 | Loss: 2.8630 | LR: 1.10e-06 | 2274.5ms/step | 8.8 samples/s +Step 42720/44000 | Loss: 2.8892 | LR: 1.08e-06 | 2303.0ms/step | 8.7 samples/s +Step 42730/44000 | Loss: 2.8680 | LR: 1.06e-06 | 2308.3ms/step | 8.7 samples/s +Step 42740/44000 | Loss: 2.8764 | LR: 1.05e-06 | 2306.8ms/step | 8.7 samples/s +Step 42750/44000 | Loss: 2.9289 | LR: 1.03e-06 | 2327.3ms/step | 8.6 samples/s +Step 42760/44000 | Loss: 2.8778 | LR: 1.01e-06 | 2270.4ms/step | 8.8 samples/s +Step 42770/44000 | Loss: 2.8831 | LR: 9.97e-07 | 2333.0ms/step | 8.6 samples/s +Step 42780/44000 | Loss: 2.9150 | LR: 9.81e-07 | 2269.3ms/step | 8.8 samples/s +Step 42790/44000 | Loss: 2.9127 | LR: 9.65e-07 | 2306.4ms/step | 8.7 samples/s +Step 42800/44000 | Loss: 2.8869 | LR: 9.49e-07 | 2274.3ms/step | 8.8 samples/s +Step 42810/44000 | Loss: 2.9273 | LR: 9.33e-07 | 2307.0ms/step | 8.7 samples/s +Step 42820/44000 | Loss: 2.8940 | LR: 9.18e-07 | 2316.3ms/step | 8.6 samples/s +Step 42830/44000 | Loss: 2.8726 | LR: 9.02e-07 | 2295.8ms/step | 8.7 samples/s +Step 42840/44000 | Loss: 2.8843 | LR: 8.87e-07 | 2264.2ms/step | 8.8 samples/s +Step 42850/44000 | Loss: 2.9207 | LR: 8.72e-07 | 2342.6ms/step | 8.5 samples/s +Step 42860/44000 | Loss: 2.9013 | LR: 8.57e-07 | 2328.0ms/step | 8.6 samples/s +Step 42870/44000 | Loss: 2.8515 | LR: 8.42e-07 | 2274.5ms/step | 8.8 samples/s +Step 42880/44000 | Loss: 2.8529 | LR: 8.27e-07 | 2279.8ms/step | 8.8 samples/s +Step 42890/44000 | Loss: 2.8883 | LR: 8.12e-07 | 2294.4ms/step | 8.7 samples/s +Step 42900/44000 | Loss: 2.9280 | LR: 7.98e-07 | 2274.4ms/step | 8.8 samples/s +Step 42910/44000 | Loss: 2.8832 | LR: 7.83e-07 | 2282.1ms/step | 8.8 samples/s +Step 42920/44000 | Loss: 2.9137 | LR: 7.69e-07 | 2302.8ms/step | 8.7 samples/s +Step 42930/44000 | Loss: 2.8526 | LR: 7.55e-07 | 2324.0ms/step | 8.6 samples/s +Step 42940/44000 | Loss: 2.8827 | LR: 7.41e-07 | 2324.9ms/step | 8.6 samples/s +Step 42950/44000 | Loss: 2.8620 | LR: 7.27e-07 | 2287.3ms/step | 8.7 samples/s +Step 42960/44000 | Loss: 2.8806 | LR: 7.13e-07 | 2309.9ms/step | 8.7 samples/s +Step 42970/44000 | Loss: 2.9210 | LR: 6.99e-07 | 2308.1ms/step | 8.7 samples/s +Step 42980/44000 | Loss: 2.8711 | LR: 6.86e-07 | 2143.2ms/step | 9.3 samples/s +Step 42990/44000 | Loss: 2.8587 | LR: 6.72e-07 | 2279.2ms/step | 8.8 samples/s +Step 43000/44000 | Loss: 2.8667 | LR: 6.59e-07 | 2288.5ms/step | 8.7 samples/s +Validation at step 43000: Loss = 2.8945, Perplexity = 18.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-43000 +Step 43010/44000 | Loss: 2.8685 | LR: 6.46e-07 | 4775.3ms/step | 4.2 samples/s +Step 43020/44000 | Loss: 2.8745 | LR: 6.33e-07 | 2294.9ms/step | 8.7 samples/s +Step 43030/44000 | Loss: 2.8230 | LR: 6.20e-07 | 2317.3ms/step | 8.6 samples/s +Step 43040/44000 | Loss: 2.8717 | LR: 6.08e-07 | 2306.7ms/step | 8.7 samples/s +Step 43050/44000 | Loss: 2.9121 | LR: 5.95e-07 | 2298.6ms/step | 8.7 samples/s +Step 43060/44000 | Loss: 2.8911 | LR: 5.83e-07 | 2310.6ms/step | 8.7 samples/s +Step 43070/44000 | Loss: 2.9059 | LR: 5.70e-07 | 2290.5ms/step | 8.7 samples/s +Step 43080/44000 | Loss: 2.8829 | LR: 5.58e-07 | 2333.5ms/step | 8.6 samples/s +Step 43090/44000 | Loss: 2.8389 | LR: 5.46e-07 | 2239.3ms/step | 8.9 samples/s +Step 43100/44000 | Loss: 2.8859 | LR: 5.34e-07 | 2294.8ms/step | 8.7 samples/s +Step 43110/44000 | Loss: 2.9028 | LR: 5.22e-07 | 2331.4ms/step | 8.6 samples/s +Step 43120/44000 | Loss: 2.8974 | LR: 5.11e-07 | 2305.4ms/step | 8.7 samples/s +Step 43130/44000 | Loss: 2.9531 | LR: 4.99e-07 | 2366.6ms/step | 8.5 samples/s +Step 43140/44000 | Loss: 2.8706 | LR: 4.88e-07 | 2275.9ms/step | 8.8 samples/s +Step 43150/44000 | Loss: 2.9294 | LR: 4.76e-07 | 2305.4ms/step | 8.7 samples/s +Step 43160/44000 | Loss: 2.8549 | LR: 4.65e-07 | 2290.0ms/step | 8.7 samples/s +Step 43170/44000 | Loss: 2.9336 | LR: 4.54e-07 | 2279.8ms/step | 8.8 samples/s +Step 43180/44000 | Loss: 2.9274 | LR: 4.43e-07 | 2292.5ms/step | 8.7 samples/s +Step 43190/44000 | Loss: 2.9058 | LR: 4.33e-07 | 2261.4ms/step | 8.8 samples/s +Step 43200/44000 | Loss: 2.8795 | LR: 4.22e-07 | 2305.4ms/step | 8.7 samples/s +Step 43210/44000 | Loss: 2.8728 | LR: 4.12e-07 | 2264.2ms/step | 8.8 samples/s +Step 43220/44000 | Loss: 2.8947 | LR: 4.01e-07 | 2306.0ms/step | 8.7 samples/s +Step 43230/44000 | Loss: 2.8919 | LR: 3.91e-07 | 2354.9ms/step | 8.5 samples/s +Step 43240/44000 | Loss: 2.8804 | LR: 3.81e-07 | 2316.2ms/step | 8.6 samples/s +Step 43250/44000 | Loss: 2.9181 | LR: 3.71e-07 | 2347.6ms/step | 8.5 samples/s +Step 43260/44000 | Loss: 2.8782 | LR: 3.61e-07 | 2353.6ms/step | 8.5 samples/s +Step 43270/44000 | Loss: 2.8798 | LR: 3.51e-07 | 2279.8ms/step | 8.8 samples/s +Step 43280/44000 | Loss: 2.9345 | LR: 3.42e-07 | 2285.2ms/step | 8.8 samples/s +Step 43290/44000 | Loss: 2.8603 | LR: 3.32e-07 | 2271.4ms/step | 8.8 samples/s +Step 43300/44000 | Loss: 2.9087 | LR: 3.23e-07 | 2383.4ms/step | 8.4 samples/s +Step 43310/44000 | Loss: 2.9056 | LR: 3.14e-07 | 2263.4ms/step | 8.8 samples/s +Step 43320/44000 | Loss: 2.9023 | LR: 3.05e-07 | 2267.5ms/step | 8.8 samples/s +Step 43330/44000 | Loss: 2.8791 | LR: 2.96e-07 | 2341.1ms/step | 8.5 samples/s +Step 43340/44000 | Loss: 2.9159 | LR: 2.87e-07 | 2315.6ms/step | 8.6 samples/s +Step 43350/44000 | Loss: 2.9024 | LR: 2.79e-07 | 2311.4ms/step | 8.7 samples/s +Step 43360/44000 | Loss: 2.9125 | LR: 2.70e-07 | 2259.1ms/step | 8.9 samples/s +Step 43370/44000 | Loss: 2.8613 | LR: 2.62e-07 | 2299.4ms/step | 8.7 samples/s +Step 43380/44000 | Loss: 2.8801 | LR: 2.53e-07 | 2304.9ms/step | 8.7 samples/s +Step 43390/44000 | Loss: 2.9187 | LR: 2.45e-07 | 2314.6ms/step | 8.6 samples/s +Step 43400/44000 | Loss: 2.8602 | LR: 2.37e-07 | 2330.5ms/step | 8.6 samples/s +Step 43410/44000 | Loss: 2.8828 | LR: 2.30e-07 | 2287.7ms/step | 8.7 samples/s +Step 43420/44000 | Loss: 2.8641 | LR: 2.22e-07 | 2258.4ms/step | 8.9 samples/s +Step 43430/44000 | Loss: 2.8823 | LR: 2.14e-07 | 2331.6ms/step | 8.6 samples/s +Step 43440/44000 | Loss: 2.9075 | LR: 2.07e-07 | 2294.3ms/step | 8.7 samples/s +Step 43450/44000 | Loss: 2.9180 | LR: 1.99e-07 | 2316.7ms/step | 8.6 samples/s +Step 43460/44000 | Loss: 2.9127 | LR: 1.92e-07 | 2324.8ms/step | 8.6 samples/s +Step 43470/44000 | Loss: 2.9174 | LR: 1.85e-07 | 2258.4ms/step | 8.9 samples/s +Step 43480/44000 | Loss: 2.8547 | LR: 1.78e-07 | 2293.9ms/step | 8.7 samples/s +Step 43490/44000 | Loss: 2.8810 | LR: 1.72e-07 | 2287.7ms/step | 8.7 samples/s +Step 43500/44000 | Loss: 2.9049 | LR: 1.65e-07 | 2331.3ms/step | 8.6 samples/s +Step 43510/44000 | Loss: 2.9189 | LR: 1.58e-07 | 2355.1ms/step | 8.5 samples/s +Step 43520/44000 | Loss: 2.8906 | LR: 1.52e-07 | 2363.0ms/step | 8.5 samples/s +Step 43530/44000 | Loss: 2.8534 | LR: 1.46e-07 | 2273.9ms/step | 8.8 samples/s +Step 43540/44000 | Loss: 2.8892 | LR: 1.40e-07 | 2285.1ms/step | 8.8 samples/s +Step 43550/44000 | Loss: 2.8991 | LR: 1.34e-07 | 2305.6ms/step | 8.7 samples/s +Step 43560/44000 | Loss: 2.9156 | LR: 1.28e-07 | 2280.0ms/step | 8.8 samples/s +Step 43570/44000 | Loss: 2.8773 | LR: 1.22e-07 | 2335.7ms/step | 8.6 samples/s +Step 43580/44000 | Loss: 2.8999 | LR: 1.16e-07 | 2366.8ms/step | 8.5 samples/s +Step 43590/44000 | Loss: 2.8863 | LR: 1.11e-07 | 2294.7ms/step | 8.7 samples/s +Step 43600/44000 | Loss: 2.8477 | LR: 1.06e-07 | 2308.1ms/step | 8.7 samples/s +Step 43610/44000 | Loss: 2.8780 | LR: 1.00e-07 | 2298.2ms/step | 8.7 samples/s +Step 43620/44000 | Loss: 2.9008 | LR: 9.52e-08 | 2306.7ms/step | 8.7 samples/s +Step 43630/44000 | Loss: 2.8442 | LR: 9.03e-08 | 2321.5ms/step | 8.6 samples/s +Step 43640/44000 | Loss: 2.8909 | LR: 8.55e-08 | 2288.9ms/step | 8.7 samples/s +Step 43650/44000 | Loss: 2.8914 | LR: 8.08e-08 | 2310.7ms/step | 8.7 samples/s +Step 43660/44000 | Loss: 2.9065 | LR: 7.62e-08 | 2310.8ms/step | 8.7 samples/s +Step 43670/44000 | Loss: 2.9074 | LR: 7.18e-08 | 2274.9ms/step | 8.8 samples/s +Step 43680/44000 | Loss: 2.8797 | LR: 6.75e-08 | 2269.3ms/step | 8.8 samples/s +Step 43690/44000 | Loss: 2.8927 | LR: 6.34e-08 | 2290.3ms/step | 8.7 samples/s +Step 43700/44000 | Loss: 2.8798 | LR: 5.94e-08 | 2361.8ms/step | 8.5 samples/s +Step 43710/44000 | Loss: 2.8436 | LR: 5.55e-08 | 2302.2ms/step | 8.7 samples/s +Step 43720/44000 | Loss: 2.8517 | LR: 5.17e-08 | 2291.1ms/step | 8.7 samples/s +Step 43730/44000 | Loss: 2.8563 | LR: 4.81e-08 | 2263.3ms/step | 8.8 samples/s +Step 43740/44000 | Loss: 2.8551 | LR: 4.46e-08 | 2327.4ms/step | 8.6 samples/s +Step 43750/44000 | Loss: 2.8998 | LR: 4.12e-08 | 2280.0ms/step | 8.8 samples/s +Step 43760/44000 | Loss: 2.8663 | LR: 3.80e-08 | 2279.6ms/step | 8.8 samples/s +Step 43770/44000 | Loss: 2.8571 | LR: 3.49e-08 | 2278.8ms/step | 8.8 samples/s +Step 43780/44000 | Loss: 2.8369 | LR: 3.19e-08 | 2316.5ms/step | 8.6 samples/s +Step 43790/44000 | Loss: 2.8778 | LR: 2.91e-08 | 2323.6ms/step | 8.6 samples/s +Step 43800/44000 | Loss: 2.9006 | LR: 2.64e-08 | 2293.8ms/step | 8.7 samples/s +Step 43810/44000 | Loss: 2.8455 | LR: 2.38e-08 | 2311.0ms/step | 8.7 samples/s +Step 43820/44000 | Loss: 2.8968 | LR: 2.14e-08 | 2319.5ms/step | 8.6 samples/s +Step 43830/44000 | Loss: 2.9075 | LR: 1.91e-08 | 2316.4ms/step | 8.6 samples/s +Step 43840/44000 | Loss: 2.8757 | LR: 1.69e-08 | 2285.5ms/step | 8.8 samples/s +Step 43850/44000 | Loss: 2.8732 | LR: 1.48e-08 | 2275.0ms/step | 8.8 samples/s +Step 43860/44000 | Loss: 2.8715 | LR: 1.29e-08 | 2298.9ms/step | 8.7 samples/s +Step 43870/44000 | Loss: 2.8838 | LR: 1.11e-08 | 2298.3ms/step | 8.7 samples/s +Step 43880/44000 | Loss: 2.9411 | LR: 9.50e-09 | 2302.4ms/step | 8.7 samples/s +Step 43890/44000 | Loss: 2.9216 | LR: 7.98e-09 | 2308.5ms/step | 8.7 samples/s +Step 43900/44000 | Loss: 2.8636 | LR: 6.60e-09 | 2278.8ms/step | 8.8 samples/s +Step 43910/44000 | Loss: 2.9062 | LR: 5.34e-09 | 2330.8ms/step | 8.6 samples/s +Step 43920/44000 | Loss: 2.8977 | LR: 4.22e-09 | 2339.8ms/step | 8.5 samples/s +Step 43930/44000 | Loss: 2.8767 | LR: 3.23e-09 | 2271.0ms/step | 8.8 samples/s +Step 43940/44000 | Loss: 2.9109 | LR: 2.37e-09 | 2289.6ms/step | 8.7 samples/s +Step 43950/44000 | Loss: 2.9022 | LR: 1.65e-09 | 2301.4ms/step | 8.7 samples/s +Step 43960/44000 | Loss: 2.9382 | LR: 1.06e-09 | 2325.9ms/step | 8.6 samples/s +Step 43970/44000 | Loss: 2.8980 | LR: 5.94e-10 | 2070.8ms/step | 9.7 samples/s +Step 43980/44000 | Loss: 2.9181 | LR: 2.64e-10 | 1913.9ms/step | 10.4 samples/s +Step 43990/44000 | Loss: 2.8688 | LR: 6.60e-11 | 1915.2ms/step | 10.4 samples/s +Step 44000/44000 | Loss: 2.8571 | LR: 0.00e+00 | 1914.5ms/step | 10.4 samples/s +Validation at step 44000: Loss = 2.8946, Perplexity = 18.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-44000 +wandb: updating run metadata +wandb: uploading config.yaml; uploading output.log +wandb: uploading output.log +wandb: uploading data +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▆▆▅▅▄▄▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▆▆▅▄▄▃▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ███▇▁▂▂▂▂▂▂▂▃▄▂▄▂▂▂▁▁▄▂▅▅▅▄▄▃▄▅▄▄▃▄▄▄▄▄▄ +wandb: speed/samples_per_sec ▄▄▁▂▂▂▂█▆▆▆█▆▆▆▆▅▆▆▅▇▇▇▅▅▃▃▄▅▄▄▅▄▅▅▄▅▅▅▄ +wandb: speed/steps_per_sec ▄▂▂▁▁▃▇█▇▇▆▇▇▇▇▇▇█▆▆▄▄▃▃▅▅▅▅▄▅▅▅▅▅▄▅▅▄▅▅ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.89455 +wandb: eval/perplexity 18.07544 +wandb: eval/uses_dense_attention 1 +wandb: eval/uses_sparse_attention 0 +wandb: speed/ms_per_step 1914.46338 +wandb: speed/samples_per_sec 10.44679 +wandb: speed/steps_per_sec 0.52234 +wandb: +4 ... +wandb: +wandb: 🚀 View run l8k365m_dense_ropectl_a1p0_seed44 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/bajm6b15 +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260820_140814-bajm6b15/logs +Training complete! diff --git a/epfl_final/train_logs/ropectl_a1p0_a10040_ddp2/seed45.20260820T140639Z.log b/epfl_final/train_logs/ropectl_a1p0_a10040_ddp2/seed45.20260820T140639Z.log new file mode 100644 index 0000000000000000000000000000000000000000..72f09383ed571b7a7ff3fff349072c14001752c6 --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_a10040_ddp2/seed45.20260820T140639Z.log @@ -0,0 +1,3029 @@ +W0820 14:06:49.821000 216 site-packages/torch/distributed/run.py:851] +W0820 14:06:49.821000 216 site-packages/torch/distributed/run.py:851] ***************************************** +W0820 14:06:49.821000 216 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0820 14:06:49.821000 216 site-packages/torch/distributed/run.py:851] ***************************************** +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-17000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-17000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 45, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed45", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control", + "ddp2-a100-40gb" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-17000 +Checkpoint activations: False +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-17000... +Preserving checkpoint LR state for exact resume: current=[0.0003451463250063146], schedule_base=[0.0005] +Resumed from checkpoint at step 17000 (epoch 0, micro_step 170000) +Resuming training from step 17000 +Resuming wandb run xfnxdn24 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run xfnxdn24 +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140811-xfnxdn24 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed45 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/xfnxdn24 +Starting training... +Validation at step 17000: Loss = 2.9929, Perplexity = 19.94 +wandb: WARNING Tried to log to step 17000 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 17001: dense (dense step interval) +Step 17010/44000 | Loss: 2.9731 | LR: 3.45e-04 | 48112.3ms/step | 0.4 samples/s +wandb: WARNING Tried to log to step 17010 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17020/44000 | Loss: 2.9369 | LR: 3.45e-04 | 2360.9ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17020 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17030/44000 | Loss: 3.0392 | LR: 3.45e-04 | 2445.4ms/step | 8.2 samples/s +wandb: WARNING Tried to log to step 17030 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17040/44000 | Loss: 3.0175 | LR: 3.44e-04 | 2391.5ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17040 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17050/44000 | Loss: 3.0138 | LR: 3.44e-04 | 2338.6ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17050 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17060/44000 | Loss: 3.0429 | LR: 3.44e-04 | 2314.5ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17060 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17070/44000 | Loss: 2.9814 | LR: 3.44e-04 | 2316.2ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17070 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17080/44000 | Loss: 3.0132 | LR: 3.44e-04 | 2254.9ms/step | 8.9 samples/s +wandb: WARNING Tried to log to step 17080 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17090/44000 | Loss: 2.9871 | LR: 3.44e-04 | 2234.2ms/step | 9.0 samples/s +wandb: WARNING Tried to log to step 17090 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17100/44000 | Loss: 3.0089 | LR: 3.43e-04 | 2219.5ms/step | 9.0 samples/s +wandb: WARNING Tried to log to step 17100 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17110/44000 | Loss: 3.0362 | LR: 3.43e-04 | 2233.7ms/step | 9.0 samples/s +wandb: WARNING Tried to log to step 17110 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17120/44000 | Loss: 2.9951 | LR: 3.43e-04 | 2246.3ms/step | 8.9 samples/s +wandb: WARNING Tried to log to step 17120 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17130/44000 | Loss: 3.0046 | LR: 3.43e-04 | 2244.8ms/step | 8.9 samples/s +wandb: WARNING Tried to log to step 17130 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17140/44000 | Loss: 3.0201 | LR: 3.43e-04 | 2253.8ms/step | 8.9 samples/s +wandb: WARNING Tried to log to step 17140 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17150/44000 | Loss: 2.9863 | LR: 3.43e-04 | 2247.9ms/step | 8.9 samples/s +wandb: WARNING Tried to log to step 17150 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17160/44000 | Loss: 3.0217 | LR: 3.42e-04 | 2264.1ms/step | 8.8 samples/s +wandb: WARNING Tried to log to step 17160 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17170/44000 | Loss: 3.0199 | LR: 3.42e-04 | 2253.4ms/step | 8.9 samples/s +wandb: WARNING Tried to log to step 17170 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17180/44000 | Loss: 2.9797 | LR: 3.42e-04 | 2275.9ms/step | 8.8 samples/s +wandb: WARNING Tried to log to step 17180 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17190/44000 | Loss: 3.0633 | LR: 3.42e-04 | 2382.9ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17190 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17200/44000 | Loss: 2.9793 | LR: 3.42e-04 | 2332.7ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17200 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17210/44000 | Loss: 3.0258 | LR: 3.42e-04 | 2319.9ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17210 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17220/44000 | Loss: 2.9968 | LR: 3.41e-04 | 2309.9ms/step | 8.7 samples/s +wandb: WARNING Tried to log to step 17220 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17230/44000 | Loss: 3.0086 | LR: 3.41e-04 | 2375.6ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17230 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17240/44000 | Loss: 2.9768 | LR: 3.41e-04 | 2300.0ms/step | 8.7 samples/s +wandb: WARNING Tried to log to step 17240 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17250/44000 | Loss: 3.0157 | LR: 3.41e-04 | 2358.9ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17250 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17260/44000 | Loss: 3.0178 | LR: 3.41e-04 | 2351.3ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17260 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17270/44000 | Loss: 2.9877 | LR: 3.41e-04 | 2331.1ms/step | 8.6 samples/s +wandb: WARNING Tried to log to step 17270 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17280/44000 | Loss: 2.9893 | LR: 3.40e-04 | 2362.2ms/step | 8.5 samples/s +wandb: WARNING Tried to log to step 17280 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17290/44000 | Loss: 2.9890 | LR: 3.40e-04 | 2390.2ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17290 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17300/44000 | Loss: 3.0214 | LR: 3.40e-04 | 2369.0ms/step | 8.4 samples/s +wandb: WARNING Tried to log to step 17300 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17310/44000 | Loss: 2.9863 | LR: 3.40e-04 | 2395.4ms/step | 8.3 samples/s +wandb: WARNING Tried to log to step 17310 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17320/44000 | Loss: 3.0049 | LR: 3.40e-04 | 2419.6ms/step | 8.3 samples/s +wandb: WARNING Tried to log to step 17320 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17330/44000 | Loss: 3.0394 | LR: 3.40e-04 | 2483.2ms/step | 8.1 samples/s +wandb: WARNING Tried to log to step 17330 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17340/44000 | Loss: 2.9929 | LR: 3.39e-04 | 2441.9ms/step | 8.2 samples/s +wandb: WARNING Tried to log to step 17340 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17350/44000 | Loss: 2.9633 | LR: 3.39e-04 | 2507.8ms/step | 8.0 samples/s +wandb: WARNING Tried to log to step 17350 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17360/44000 | Loss: 2.9831 | LR: 3.39e-04 | 2482.9ms/step | 8.1 samples/s +wandb: WARNING Tried to log to step 17360 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17370/44000 | Loss: 3.0327 | LR: 3.39e-04 | 2524.8ms/step | 7.9 samples/s +wandb: WARNING Tried to log to step 17370 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17380/44000 | Loss: 2.9942 | LR: 3.39e-04 | 2503.6ms/step | 8.0 samples/s +wandb: WARNING Tried to log to step 17380 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17390/44000 | Loss: 3.0283 | LR: 3.39e-04 | 2519.1ms/step | 7.9 samples/s +wandb: WARNING Tried to log to step 17390 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17400/44000 | Loss: 2.9490 | LR: 3.38e-04 | 2435.6ms/step | 8.2 samples/s +wandb: WARNING Tried to log to step 17400 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17410/44000 | Loss: 3.0063 | LR: 3.38e-04 | 2482.8ms/step | 8.1 samples/s +wandb: WARNING Tried to log to step 17410 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17420/44000 | Loss: 2.9595 | LR: 3.38e-04 | 2446.2ms/step | 8.2 samples/s +wandb: WARNING Tried to log to step 17420 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17430/44000 | Loss: 2.9976 | LR: 3.38e-04 | 2455.4ms/step | 8.1 samples/s +wandb: WARNING Tried to log to step 17430 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17440/44000 | Loss: 2.9982 | LR: 3.38e-04 | 2491.5ms/step | 8.0 samples/s +wandb: WARNING Tried to log to step 17440 that is less than the current step 17441. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17450/44000 | Loss: 3.0062 | LR: 3.38e-04 | 2537.7ms/step | 7.9 samples/s +Step 17460/44000 | Loss: 2.9983 | LR: 3.37e-04 | 2399.9ms/step | 8.3 samples/s +Step 17470/44000 | Loss: 2.9704 | LR: 3.37e-04 | 2500.0ms/step | 8.0 samples/s +Step 17480/44000 | Loss: 2.9951 | LR: 3.37e-04 | 2514.3ms/step | 8.0 samples/s +Step 17490/44000 | Loss: 2.9611 | LR: 3.37e-04 | 2453.6ms/step | 8.2 samples/s +Step 17500/44000 | Loss: 2.9917 | LR: 3.37e-04 | 2517.3ms/step | 7.9 samples/s +Step 17510/44000 | Loss: 2.9786 | LR: 3.37e-04 | 2503.4ms/step | 8.0 samples/s +Step 17520/44000 | Loss: 2.9583 | LR: 3.36e-04 | 2457.8ms/step | 8.1 samples/s +Step 17530/44000 | Loss: 3.0167 | LR: 3.36e-04 | 2482.8ms/step | 8.1 samples/s +Step 17540/44000 | Loss: 2.9789 | LR: 3.36e-04 | 2499.6ms/step | 8.0 samples/s +Step 17550/44000 | Loss: 2.9622 | LR: 3.36e-04 | 2456.9ms/step | 8.1 samples/s +Step 17560/44000 | Loss: 2.9949 | LR: 3.36e-04 | 2487.5ms/step | 8.0 samples/s +Step 17570/44000 | Loss: 3.0287 | LR: 3.35e-04 | 2461.7ms/step | 8.1 samples/s +Step 17580/44000 | Loss: 2.9696 | LR: 3.35e-04 | 2483.1ms/step | 8.1 samples/s +Step 17590/44000 | Loss: 3.0140 | LR: 3.35e-04 | 2492.5ms/step | 8.0 samples/s +Step 17600/44000 | Loss: 3.0125 | LR: 3.35e-04 | 2398.8ms/step | 8.3 samples/s +Step 17610/44000 | Loss: 2.9833 | LR: 3.35e-04 | 2435.1ms/step | 8.2 samples/s +Step 17620/44000 | Loss: 3.0007 | LR: 3.35e-04 | 2416.3ms/step | 8.3 samples/s +Step 17630/44000 | Loss: 3.0569 | LR: 3.34e-04 | 2444.3ms/step | 8.2 samples/s +Step 17640/44000 | Loss: 3.0115 | LR: 3.34e-04 | 2388.6ms/step | 8.4 samples/s +Step 17650/44000 | Loss: 3.0168 | LR: 3.34e-04 | 2425.0ms/step | 8.2 samples/s +Step 17660/44000 | Loss: 2.9794 | LR: 3.34e-04 | 2432.8ms/step | 8.2 samples/s +Step 17670/44000 | Loss: 2.9947 | LR: 3.34e-04 | 2465.2ms/step | 8.1 samples/s +Step 17680/44000 | Loss: 3.0078 | LR: 3.34e-04 | 2477.9ms/step | 8.1 samples/s +Step 17690/44000 | Loss: 2.9553 | LR: 3.33e-04 | 2451.0ms/step | 8.2 samples/s +Step 17700/44000 | Loss: 2.9951 | LR: 3.33e-04 | 2423.8ms/step | 8.3 samples/s +Step 17710/44000 | Loss: 2.9651 | LR: 3.33e-04 | 2406.8ms/step | 8.3 samples/s +Step 17720/44000 | Loss: 2.9949 | LR: 3.33e-04 | 2399.1ms/step | 8.3 samples/s +Step 17730/44000 | Loss: 2.9711 | LR: 3.33e-04 | 2398.8ms/step | 8.3 samples/s +Step 17740/44000 | Loss: 2.9958 | LR: 3.33e-04 | 2398.6ms/step | 8.3 samples/s +Step 17750/44000 | Loss: 3.0374 | LR: 3.32e-04 | 2375.9ms/step | 8.4 samples/s +Step 17760/44000 | Loss: 3.0348 | LR: 3.32e-04 | 2427.4ms/step | 8.2 samples/s +Step 17770/44000 | Loss: 2.9895 | LR: 3.32e-04 | 2482.2ms/step | 8.1 samples/s +Step 17780/44000 | Loss: 2.9774 | LR: 3.32e-04 | 2547.8ms/step | 7.8 samples/s +Step 17790/44000 | Loss: 2.9850 | LR: 3.32e-04 | 2507.5ms/step | 8.0 samples/s +Step 17800/44000 | Loss: 2.9731 | LR: 3.32e-04 | 2400.5ms/step | 8.3 samples/s +Step 17810/44000 | Loss: 2.9808 | LR: 3.31e-04 | 2415.2ms/step | 8.3 samples/s +Step 17820/44000 | Loss: 2.9786 | LR: 3.31e-04 | 2385.3ms/step | 8.4 samples/s +Step 17830/44000 | Loss: 3.0022 | LR: 3.31e-04 | 2402.9ms/step | 8.3 samples/s +Step 17840/44000 | Loss: 2.9892 | LR: 3.31e-04 | 2418.5ms/step | 8.3 samples/s +Step 17850/44000 | Loss: 2.9553 | LR: 3.31e-04 | 2427.8ms/step | 8.2 samples/s +Step 17860/44000 | Loss: 3.0505 | LR: 3.31e-04 | 2456.4ms/step | 8.1 samples/s +Step 17870/44000 | Loss: 3.0114 | LR: 3.30e-04 | 2367.8ms/step | 8.4 samples/s +Step 17880/44000 | Loss: 2.9769 | LR: 3.30e-04 | 2360.8ms/step | 8.5 samples/s +Step 17890/44000 | Loss: 3.0060 | LR: 3.30e-04 | 2370.4ms/step | 8.4 samples/s +Step 17900/44000 | Loss: 2.9745 | LR: 3.30e-04 | 2378.7ms/step | 8.4 samples/s +Step 17910/44000 | Loss: 2.9279 | LR: 3.30e-04 | 2367.4ms/step | 8.4 samples/s +Step 17920/44000 | Loss: 2.9740 | LR: 3.29e-04 | 2390.7ms/step | 8.4 samples/s +Step 17930/44000 | Loss: 2.9924 | LR: 3.29e-04 | 2479.2ms/step | 8.1 samples/s +Step 17940/44000 | Loss: 2.9854 | LR: 3.29e-04 | 2497.9ms/step | 8.0 samples/s +Step 17950/44000 | Loss: 2.9807 | LR: 3.29e-04 | 2499.1ms/step | 8.0 samples/s +Step 17960/44000 | Loss: 2.9897 | LR: 3.29e-04 | 2335.1ms/step | 8.6 samples/s +Step 17970/44000 | Loss: 3.0139 | LR: 3.29e-04 | 2357.9ms/step | 8.5 samples/s +Step 17980/44000 | Loss: 3.0083 | LR: 3.28e-04 | 2384.3ms/step | 8.4 samples/s +Step 17990/44000 | Loss: 3.0065 | LR: 3.28e-04 | 2322.7ms/step | 8.6 samples/s +Step 18000/44000 | Loss: 2.9859 | LR: 3.28e-04 | 2124.4ms/step | 9.4 samples/s +Validation at step 18000: Loss = 2.9818, Perplexity = 19.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-18000 +Step 18010/44000 | Loss: 2.9525 | LR: 3.28e-04 | 4973.2ms/step | 4.0 samples/s +Step 18020/44000 | Loss: 2.9962 | LR: 3.28e-04 | 2398.6ms/step | 8.3 samples/s +Step 18030/44000 | Loss: 2.9435 | LR: 3.28e-04 | 2410.7ms/step | 8.3 samples/s +Step 18040/44000 | Loss: 2.9994 | LR: 3.27e-04 | 2399.8ms/step | 8.3 samples/s +Step 18050/44000 | Loss: 2.9605 | LR: 3.27e-04 | 2441.2ms/step | 8.2 samples/s +Step 18060/44000 | Loss: 3.0303 | LR: 3.27e-04 | 2368.2ms/step | 8.4 samples/s +Step 18070/44000 | Loss: 2.9967 | LR: 3.27e-04 | 2348.0ms/step | 8.5 samples/s +Step 18080/44000 | Loss: 2.9871 | LR: 3.27e-04 | 2384.9ms/step | 8.4 samples/s +Step 18090/44000 | Loss: 3.0037 | LR: 3.27e-04 | 2357.9ms/step | 8.5 samples/s +Step 18100/44000 | Loss: 3.0049 | LR: 3.26e-04 | 2456.5ms/step | 8.1 samples/s +Step 18110/44000 | Loss: 3.0352 | LR: 3.26e-04 | 2488.4ms/step | 8.0 samples/s +Step 18120/44000 | Loss: 2.9619 | LR: 3.26e-04 | 2422.6ms/step | 8.3 samples/s +Step 18130/44000 | Loss: 2.9918 | LR: 3.26e-04 | 2444.0ms/step | 8.2 samples/s +Step 18140/44000 | Loss: 3.0346 | LR: 3.26e-04 | 2466.4ms/step | 8.1 samples/s +Step 18150/44000 | Loss: 2.9827 | LR: 3.26e-04 | 2441.5ms/step | 8.2 samples/s +Step 18160/44000 | Loss: 3.0177 | LR: 3.25e-04 | 2344.8ms/step | 8.5 samples/s +Step 18170/44000 | Loss: 3.0311 | LR: 3.25e-04 | 2360.2ms/step | 8.5 samples/s +Step 18180/44000 | Loss: 2.9901 | LR: 3.25e-04 | 2372.0ms/step | 8.4 samples/s +Step 18190/44000 | Loss: 2.9921 | LR: 3.25e-04 | 2369.0ms/step | 8.4 samples/s +Step 18200/44000 | Loss: 3.0286 | LR: 3.25e-04 | 2372.8ms/step | 8.4 samples/s +Step 18210/44000 | Loss: 3.0118 | LR: 3.24e-04 | 2329.7ms/step | 8.6 samples/s +Step 18220/44000 | Loss: 2.9495 | LR: 3.24e-04 | 2336.6ms/step | 8.6 samples/s +Step 18230/44000 | Loss: 2.9521 | LR: 3.24e-04 | 2386.7ms/step | 8.4 samples/s +Step 18240/44000 | Loss: 2.9709 | LR: 3.24e-04 | 2400.5ms/step | 8.3 samples/s +Step 18250/44000 | Loss: 3.0211 | LR: 3.24e-04 | 2401.5ms/step | 8.3 samples/s +Step 18260/44000 | Loss: 3.0048 | LR: 3.24e-04 | 2385.9ms/step | 8.4 samples/s +Step 18270/44000 | Loss: 2.9820 | LR: 3.23e-04 | 2667.5ms/step | 7.5 samples/s +Step 18280/44000 | Loss: 3.0050 | LR: 3.23e-04 | 2751.6ms/step | 7.3 samples/s +Step 18290/44000 | Loss: 3.0215 | LR: 3.23e-04 | 2642.2ms/step | 7.6 samples/s +Step 18300/44000 | Loss: 2.9761 | LR: 3.23e-04 | 2556.1ms/step | 7.8 samples/s +Step 18310/44000 | Loss: 3.0097 | LR: 3.23e-04 | 2576.8ms/step | 7.8 samples/s +Step 18320/44000 | Loss: 2.9825 | LR: 3.23e-04 | 2547.6ms/step | 7.9 samples/s +Step 18330/44000 | Loss: 3.0108 | LR: 3.22e-04 | 2592.8ms/step | 7.7 samples/s +Step 18340/44000 | Loss: 2.9654 | LR: 3.22e-04 | 2568.0ms/step | 7.8 samples/s +Step 18350/44000 | Loss: 2.9749 | LR: 3.22e-04 | 2788.3ms/step | 7.2 samples/s +Step 18360/44000 | Loss: 2.9666 | LR: 3.22e-04 | 2879.4ms/step | 6.9 samples/s +Step 18370/44000 | Loss: 2.9887 | LR: 3.22e-04 | 2862.0ms/step | 7.0 samples/s +Step 18380/44000 | Loss: 2.9433 | LR: 3.22e-04 | 2770.8ms/step | 7.2 samples/s +Step 18390/44000 | Loss: 3.0112 | LR: 3.21e-04 | 2858.4ms/step | 7.0 samples/s +Step 18400/44000 | Loss: 3.0351 | LR: 3.21e-04 | 2683.9ms/step | 7.5 samples/s +Step 18410/44000 | Loss: 2.9823 | LR: 3.21e-04 | 2677.3ms/step | 7.5 samples/s +Step 18420/44000 | Loss: 3.0248 | LR: 3.21e-04 | 2594.9ms/step | 7.7 samples/s +Step 18430/44000 | Loss: 3.0081 | LR: 3.21e-04 | 2703.3ms/step | 7.4 samples/s +Step 18440/44000 | Loss: 3.0104 | LR: 3.20e-04 | 2574.7ms/step | 7.8 samples/s +Step 18450/44000 | Loss: 2.9947 | LR: 3.20e-04 | 2586.2ms/step | 7.7 samples/s +Step 18460/44000 | Loss: 3.0043 | LR: 3.20e-04 | 2588.6ms/step | 7.7 samples/s +Step 18470/44000 | Loss: 2.9782 | LR: 3.20e-04 | 2657.7ms/step | 7.5 samples/s +Step 18480/44000 | Loss: 3.0168 | LR: 3.20e-04 | 2677.3ms/step | 7.5 samples/s +Step 18490/44000 | Loss: 3.0176 | LR: 3.20e-04 | 2636.9ms/step | 7.6 samples/s +Step 18500/44000 | Loss: 2.9407 | LR: 3.19e-04 | 2758.5ms/step | 7.3 samples/s +Step 18510/44000 | Loss: 2.9337 | LR: 3.19e-04 | 2618.8ms/step | 7.6 samples/s +Step 18520/44000 | Loss: 3.0120 | LR: 3.19e-04 | 2705.3ms/step | 7.4 samples/s +Step 18530/44000 | Loss: 2.9719 | LR: 3.19e-04 | 2715.7ms/step | 7.4 samples/s +Step 18540/44000 | Loss: 3.0478 | LR: 3.19e-04 | 2709.6ms/step | 7.4 samples/s +Step 18550/44000 | Loss: 2.9826 | LR: 3.19e-04 | 2657.6ms/step | 7.5 samples/s +Step 18560/44000 | Loss: 2.9858 | LR: 3.18e-04 | 2850.5ms/step | 7.0 samples/s +Step 18570/44000 | Loss: 3.0212 | LR: 3.18e-04 | 2588.6ms/step | 7.7 samples/s +Step 18580/44000 | Loss: 2.9716 | LR: 3.18e-04 | 2725.5ms/step | 7.3 samples/s +Step 18590/44000 | Loss: 3.0038 | LR: 3.18e-04 | 2682.5ms/step | 7.5 samples/s +Step 18600/44000 | Loss: 2.9628 | LR: 3.18e-04 | 2819.4ms/step | 7.1 samples/s +Step 18610/44000 | Loss: 2.9329 | LR: 3.18e-04 | 2735.4ms/step | 7.3 samples/s +Step 18620/44000 | Loss: 2.9590 | LR: 3.17e-04 | 2704.7ms/step | 7.4 samples/s +Step 18630/44000 | Loss: 2.9556 | LR: 3.17e-04 | 2605.6ms/step | 7.7 samples/s +Step 18640/44000 | Loss: 3.0105 | LR: 3.17e-04 | 2606.6ms/step | 7.7 samples/s +Step 18650/44000 | Loss: 3.0033 | LR: 3.17e-04 | 2708.6ms/step | 7.4 samples/s +Step 18660/44000 | Loss: 2.9950 | LR: 3.17e-04 | 2745.6ms/step | 7.3 samples/s +Step 18670/44000 | Loss: 2.9751 | LR: 3.16e-04 | 2719.6ms/step | 7.4 samples/s +Step 18680/44000 | Loss: 3.0019 | LR: 3.16e-04 | 2632.2ms/step | 7.6 samples/s +Step 18690/44000 | Loss: 2.9591 | LR: 3.16e-04 | 2627.1ms/step | 7.6 samples/s +Step 18700/44000 | Loss: 2.9642 | LR: 3.16e-04 | 2693.1ms/step | 7.4 samples/s +Step 18710/44000 | Loss: 2.9656 | LR: 3.16e-04 | 2663.3ms/step | 7.5 samples/s +Step 18720/44000 | Loss: 2.9804 | LR: 3.16e-04 | 2522.0ms/step | 7.9 samples/s +Step 18730/44000 | Loss: 2.9812 | LR: 3.15e-04 | 2397.0ms/step | 8.3 samples/s +Step 18740/44000 | Loss: 2.9972 | LR: 3.15e-04 | 2393.9ms/step | 8.4 samples/s +Step 18750/44000 | Loss: 2.9667 | LR: 3.15e-04 | 2399.2ms/step | 8.3 samples/s +Step 18760/44000 | Loss: 2.9514 | LR: 3.15e-04 | 2470.7ms/step | 8.1 samples/s +Step 18770/44000 | Loss: 2.9686 | LR: 3.15e-04 | 2570.4ms/step | 7.8 samples/s +Step 18780/44000 | Loss: 2.9777 | LR: 3.15e-04 | 2628.7ms/step | 7.6 samples/s +Step 18790/44000 | Loss: 2.9648 | LR: 3.14e-04 | 2766.8ms/step | 7.2 samples/s +Step 18800/44000 | Loss: 2.9783 | LR: 3.14e-04 | 2662.2ms/step | 7.5 samples/s +Step 18810/44000 | Loss: 2.9680 | LR: 3.14e-04 | 2766.6ms/step | 7.2 samples/s +Step 18820/44000 | Loss: 3.0275 | LR: 3.14e-04 | 2493.5ms/step | 8.0 samples/s +Step 18830/44000 | Loss: 2.9714 | LR: 3.14e-04 | 2494.2ms/step | 8.0 samples/s +Step 18840/44000 | Loss: 2.9726 | LR: 3.13e-04 | 2611.8ms/step | 7.7 samples/s +Step 18850/44000 | Loss: 2.9532 | LR: 3.13e-04 | 2760.8ms/step | 7.2 samples/s +Step 18860/44000 | Loss: 2.9469 | LR: 3.13e-04 | 2655.6ms/step | 7.5 samples/s +Step 18870/44000 | Loss: 2.9784 | LR: 3.13e-04 | 2552.5ms/step | 7.8 samples/s +Step 18880/44000 | Loss: 2.9337 | LR: 3.13e-04 | 2651.8ms/step | 7.5 samples/s +Step 18890/44000 | Loss: 2.9731 | LR: 3.13e-04 | 2791.0ms/step | 7.2 samples/s +Step 18900/44000 | Loss: 2.9527 | LR: 3.12e-04 | 2822.7ms/step | 7.1 samples/s +Step 18910/44000 | Loss: 3.0113 | LR: 3.12e-04 | 2789.3ms/step | 7.2 samples/s +Step 18920/44000 | Loss: 3.0171 | LR: 3.12e-04 | 2563.4ms/step | 7.8 samples/s +Step 18930/44000 | Loss: 2.9565 | LR: 3.12e-04 | 2652.2ms/step | 7.5 samples/s +Step 18940/44000 | Loss: 2.9478 | LR: 3.12e-04 | 2613.0ms/step | 7.7 samples/s +Step 18950/44000 | Loss: 2.9234 | LR: 3.12e-04 | 2621.9ms/step | 7.6 samples/s +Step 18960/44000 | Loss: 2.9553 | LR: 3.11e-04 | 2694.5ms/step | 7.4 samples/s +Step 18970/44000 | Loss: 2.9711 | LR: 3.11e-04 | 2415.7ms/step | 8.3 samples/s +Step 18980/44000 | Loss: 3.0417 | LR: 3.11e-04 | 2825.8ms/step | 7.1 samples/s +Step 18990/44000 | Loss: 2.9550 | LR: 3.11e-04 | 2788.5ms/step | 7.2 samples/s +Step 19000/44000 | Loss: 2.9866 | LR: 3.11e-04 | 2672.0ms/step | 7.5 samples/s +Validation at step 19000: Loss = 2.9710, Perplexity = 19.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-19000 +Step 19010/44000 | Loss: 2.9325 | LR: 3.11e-04 | 5504.9ms/step | 3.6 samples/s +Step 19020/44000 | Loss: 3.0251 | LR: 3.10e-04 | 2749.0ms/step | 7.3 samples/s +Step 19030/44000 | Loss: 2.9741 | LR: 3.10e-04 | 2725.2ms/step | 7.3 samples/s +Step 19040/44000 | Loss: 2.9778 | LR: 3.10e-04 | 2630.3ms/step | 7.6 samples/s +Step 19050/44000 | Loss: 2.9992 | LR: 3.10e-04 | 2693.9ms/step | 7.4 samples/s +Step 19060/44000 | Loss: 3.0248 | LR: 3.10e-04 | 2761.9ms/step | 7.2 samples/s +Step 19070/44000 | Loss: 3.0379 | LR: 3.09e-04 | 2904.1ms/step | 6.9 samples/s +Step 19080/44000 | Loss: 2.9399 | LR: 3.09e-04 | 2898.1ms/step | 6.9 samples/s +Step 19090/44000 | Loss: 2.9393 | LR: 3.09e-04 | 2636.0ms/step | 7.6 samples/s +Step 19100/44000 | Loss: 2.9844 | LR: 3.09e-04 | 2643.0ms/step | 7.6 samples/s +Step 19110/44000 | Loss: 2.9931 | LR: 3.09e-04 | 2526.1ms/step | 7.9 samples/s +Step 19120/44000 | Loss: 2.9523 | LR: 3.09e-04 | 2579.7ms/step | 7.8 samples/s +Step 19130/44000 | Loss: 2.9720 | LR: 3.08e-04 | 2603.4ms/step | 7.7 samples/s +Step 19140/44000 | Loss: 2.9761 | LR: 3.08e-04 | 2609.6ms/step | 7.7 samples/s +Step 19150/44000 | Loss: 2.9734 | LR: 3.08e-04 | 2704.2ms/step | 7.4 samples/s +Step 19160/44000 | Loss: 2.9819 | LR: 3.08e-04 | 2836.4ms/step | 7.1 samples/s +Step 19170/44000 | Loss: 2.9436 | LR: 3.08e-04 | 2458.4ms/step | 8.1 samples/s +Step 19180/44000 | Loss: 3.0193 | LR: 3.08e-04 | 2271.8ms/step | 8.8 samples/s +Step 19190/44000 | Loss: 2.9824 | LR: 3.07e-04 | 2316.4ms/step | 8.6 samples/s +Step 19200/44000 | Loss: 2.9562 | LR: 3.07e-04 | 2496.0ms/step | 8.0 samples/s +Step 19210/44000 | Loss: 3.0019 | LR: 3.07e-04 | 2670.6ms/step | 7.5 samples/s +Step 19220/44000 | Loss: 2.9788 | LR: 3.07e-04 | 2542.9ms/step | 7.9 samples/s +Step 19230/44000 | Loss: 3.0023 | LR: 3.07e-04 | 2631.4ms/step | 7.6 samples/s +Step 19240/44000 | Loss: 2.9826 | LR: 3.06e-04 | 2631.9ms/step | 7.6 samples/s +Step 19250/44000 | Loss: 2.9883 | LR: 3.06e-04 | 2595.4ms/step | 7.7 samples/s +Step 19260/44000 | Loss: 3.0084 | LR: 3.06e-04 | 2657.1ms/step | 7.5 samples/s +Step 19270/44000 | Loss: 3.0090 | LR: 3.06e-04 | 2689.6ms/step | 7.4 samples/s +Step 19280/44000 | Loss: 2.9614 | LR: 3.06e-04 | 2662.3ms/step | 7.5 samples/s +Step 19290/44000 | Loss: 2.9573 | LR: 3.06e-04 | 2573.4ms/step | 7.8 samples/s +Step 19300/44000 | Loss: 2.9760 | LR: 3.05e-04 | 2706.3ms/step | 7.4 samples/s +Step 19310/44000 | Loss: 2.9988 | LR: 3.05e-04 | 2723.6ms/step | 7.3 samples/s +Step 19320/44000 | Loss: 3.0100 | LR: 3.05e-04 | 2733.1ms/step | 7.3 samples/s +Step 19330/44000 | Loss: 2.9352 | LR: 3.05e-04 | 2694.5ms/step | 7.4 samples/s +Step 19340/44000 | Loss: 2.9780 | LR: 3.05e-04 | 2677.2ms/step | 7.5 samples/s +Step 19350/44000 | Loss: 2.9804 | LR: 3.04e-04 | 2720.1ms/step | 7.4 samples/s +Step 19360/44000 | Loss: 2.9556 | LR: 3.04e-04 | 2765.4ms/step | 7.2 samples/s +Step 19370/44000 | Loss: 2.8815 | LR: 3.04e-04 | 2898.2ms/step | 6.9 samples/s +Step 19380/44000 | Loss: 2.9728 | LR: 3.04e-04 | 2690.4ms/step | 7.4 samples/s +Step 19390/44000 | Loss: 2.9812 | LR: 3.04e-04 | 2644.2ms/step | 7.6 samples/s +Step 19400/44000 | Loss: 2.9533 | LR: 3.04e-04 | 2706.6ms/step | 7.4 samples/s +Step 19410/44000 | Loss: 2.9407 | LR: 3.03e-04 | 2666.8ms/step | 7.5 samples/s +Step 19420/44000 | Loss: 2.9725 | LR: 3.03e-04 | 2762.7ms/step | 7.2 samples/s +Step 19430/44000 | Loss: 2.9444 | LR: 3.03e-04 | 2727.3ms/step | 7.3 samples/s +Step 19440/44000 | Loss: 2.9490 | LR: 3.03e-04 | 2736.2ms/step | 7.3 samples/s +Step 19450/44000 | Loss: 2.9612 | LR: 3.03e-04 | 2713.3ms/step | 7.4 samples/s +Step 19460/44000 | Loss: 3.0180 | LR: 3.03e-04 | 2514.2ms/step | 8.0 samples/s +Step 19470/44000 | Loss: 2.9687 | LR: 3.02e-04 | 2508.8ms/step | 8.0 samples/s +Step 19480/44000 | Loss: 2.9852 | LR: 3.02e-04 | 2578.0ms/step | 7.8 samples/s +Step 19490/44000 | Loss: 2.9869 | LR: 3.02e-04 | 2646.0ms/step | 7.6 samples/s +Step 19500/44000 | Loss: 2.9900 | LR: 3.02e-04 | 2641.0ms/step | 7.6 samples/s +Step 19510/44000 | Loss: 2.9711 | LR: 3.02e-04 | 2815.0ms/step | 7.1 samples/s +Step 19520/44000 | Loss: 2.9753 | LR: 3.01e-04 | 2677.5ms/step | 7.5 samples/s +Step 19530/44000 | Loss: 2.9466 | LR: 3.01e-04 | 2652.2ms/step | 7.5 samples/s +Step 19540/44000 | Loss: 2.9923 | LR: 3.01e-04 | 2641.9ms/step | 7.6 samples/s +Step 19550/44000 | Loss: 3.0004 | LR: 3.01e-04 | 2555.6ms/step | 7.8 samples/s +Step 19560/44000 | Loss: 2.9837 | LR: 3.01e-04 | 2671.1ms/step | 7.5 samples/s +Step 19570/44000 | Loss: 2.9929 | LR: 3.01e-04 | 2680.6ms/step | 7.5 samples/s +Step 19580/44000 | Loss: 2.9998 | LR: 3.00e-04 | 2625.2ms/step | 7.6 samples/s +Step 19590/44000 | Loss: 2.9450 | LR: 3.00e-04 | 2666.7ms/step | 7.5 samples/s +Step 19600/44000 | Loss: 2.9150 | LR: 3.00e-04 | 2683.6ms/step | 7.5 samples/s +Step 19610/44000 | Loss: 2.9615 | LR: 3.00e-04 | 2659.3ms/step | 7.5 samples/s +Step 19620/44000 | Loss: 2.9824 | LR: 3.00e-04 | 2722.0ms/step | 7.3 samples/s +Step 19630/44000 | Loss: 2.9731 | LR: 3.00e-04 | 2511.4ms/step | 8.0 samples/s +Step 19640/44000 | Loss: 2.9430 | LR: 2.99e-04 | 2421.5ms/step | 8.3 samples/s +Step 19650/44000 | Loss: 2.9512 | LR: 2.99e-04 | 2442.2ms/step | 8.2 samples/s +Step 19660/44000 | Loss: 2.9448 | LR: 2.99e-04 | 2505.3ms/step | 8.0 samples/s +Step 19670/44000 | Loss: 2.9483 | LR: 2.99e-04 | 2861.5ms/step | 7.0 samples/s +Step 19680/44000 | Loss: 2.9626 | LR: 2.99e-04 | 2717.6ms/step | 7.4 samples/s +Step 19690/44000 | Loss: 2.9212 | LR: 2.98e-04 | 2782.4ms/step | 7.2 samples/s +Step 19700/44000 | Loss: 2.9940 | LR: 2.98e-04 | 2772.8ms/step | 7.2 samples/s +Step 19710/44000 | Loss: 2.9506 | LR: 2.98e-04 | 2743.7ms/step | 7.3 samples/s +Step 19720/44000 | Loss: 2.9908 | LR: 2.98e-04 | 2709.7ms/step | 7.4 samples/s +Step 19730/44000 | Loss: 2.9874 | LR: 2.98e-04 | 2841.7ms/step | 7.0 samples/s +Step 19740/44000 | Loss: 2.9486 | LR: 2.98e-04 | 2733.6ms/step | 7.3 samples/s +Step 19750/44000 | Loss: 3.0186 | LR: 2.97e-04 | 2681.7ms/step | 7.5 samples/s +Step 19760/44000 | Loss: 2.9289 | LR: 2.97e-04 | 2776.9ms/step | 7.2 samples/s +Step 19770/44000 | Loss: 3.0183 | LR: 2.97e-04 | 2844.2ms/step | 7.0 samples/s +Step 19780/44000 | Loss: 2.9655 | LR: 2.97e-04 | 2848.5ms/step | 7.0 samples/s +Step 19790/44000 | Loss: 2.9980 | LR: 2.97e-04 | 2748.5ms/step | 7.3 samples/s +Step 19800/44000 | Loss: 2.9595 | LR: 2.96e-04 | 2770.3ms/step | 7.2 samples/s +Step 19810/44000 | Loss: 2.9917 | LR: 2.96e-04 | 2754.0ms/step | 7.3 samples/s +Step 19820/44000 | Loss: 2.9639 | LR: 2.96e-04 | 2682.8ms/step | 7.5 samples/s +Step 19830/44000 | Loss: 2.9457 | LR: 2.96e-04 | 2542.2ms/step | 7.9 samples/s +Step 19840/44000 | Loss: 2.9691 | LR: 2.96e-04 | 2578.2ms/step | 7.8 samples/s +Step 19850/44000 | Loss: 2.9699 | LR: 2.96e-04 | 2620.2ms/step | 7.6 samples/s +Step 19860/44000 | Loss: 2.9894 | LR: 2.95e-04 | 2651.5ms/step | 7.5 samples/s +Step 19870/44000 | Loss: 2.9234 | LR: 2.95e-04 | 2728.2ms/step | 7.3 samples/s +Step 19880/44000 | Loss: 3.0250 | LR: 2.95e-04 | 2580.3ms/step | 7.8 samples/s +Step 19890/44000 | Loss: 2.9914 | LR: 2.95e-04 | 2710.6ms/step | 7.4 samples/s +Step 19900/44000 | Loss: 2.9207 | LR: 2.95e-04 | 2631.1ms/step | 7.6 samples/s +Step 19910/44000 | Loss: 2.9641 | LR: 2.95e-04 | 2752.2ms/step | 7.3 samples/s +Step 19920/44000 | Loss: 2.9771 | LR: 2.94e-04 | 2668.8ms/step | 7.5 samples/s +Step 19930/44000 | Loss: 2.9542 | LR: 2.94e-04 | 2562.8ms/step | 7.8 samples/s +Step 19940/44000 | Loss: 3.0007 | LR: 2.94e-04 | 2367.4ms/step | 8.4 samples/s +Step 19950/44000 | Loss: 2.9885 | LR: 2.94e-04 | 2567.2ms/step | 7.8 samples/s +Step 19960/44000 | Loss: 2.9543 | LR: 2.94e-04 | 2701.0ms/step | 7.4 samples/s +Step 19970/44000 | Loss: 2.9864 | LR: 2.93e-04 | 2640.8ms/step | 7.6 samples/s +Step 19980/44000 | Loss: 2.9698 | LR: 2.93e-04 | 2688.7ms/step | 7.4 samples/s +Step 19990/44000 | Loss: 2.9782 | LR: 2.93e-04 | 2576.5ms/step | 7.8 samples/s +Step 20000/44000 | Loss: 2.9547 | LR: 2.93e-04 | 2664.5ms/step | 7.5 samples/s +Validation at step 20000: Loss = 2.9593, Perplexity = 19.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-20000 +Step 20010/44000 | Loss: 2.9608 | LR: 2.93e-04 | 5608.5ms/step | 3.6 samples/s +Step 20020/44000 | Loss: 2.9620 | LR: 2.93e-04 | 2613.3ms/step | 7.7 samples/s +Step 20030/44000 | Loss: 2.9923 | LR: 2.92e-04 | 2689.5ms/step | 7.4 samples/s +Step 20040/44000 | Loss: 2.9355 | LR: 2.92e-04 | 2665.9ms/step | 7.5 samples/s +Step 20050/44000 | Loss: 2.9949 | LR: 2.92e-04 | 2720.6ms/step | 7.4 samples/s +Step 20060/44000 | Loss: 2.9474 | LR: 2.92e-04 | 2598.7ms/step | 7.7 samples/s +Step 20070/44000 | Loss: 3.0142 | LR: 2.92e-04 | 2800.3ms/step | 7.1 samples/s +Step 20080/44000 | Loss: 2.9557 | LR: 2.91e-04 | 2569.1ms/step | 7.8 samples/s +Step 20090/44000 | Loss: 2.9407 | LR: 2.91e-04 | 2453.7ms/step | 8.2 samples/s +Step 20100/44000 | Loss: 2.9706 | LR: 2.91e-04 | 2489.3ms/step | 8.0 samples/s +Step 20110/44000 | Loss: 2.9241 | LR: 2.91e-04 | 2699.9ms/step | 7.4 samples/s +Step 20120/44000 | Loss: 2.9745 | LR: 2.91e-04 | 2678.8ms/step | 7.5 samples/s +Step 20130/44000 | Loss: 2.9637 | LR: 2.91e-04 | 2600.5ms/step | 7.7 samples/s +Step 20140/44000 | Loss: 2.9249 | LR: 2.90e-04 | 2670.5ms/step | 7.5 samples/s +Step 20150/44000 | Loss: 2.9995 | LR: 2.90e-04 | 2635.6ms/step | 7.6 samples/s +Step 20160/44000 | Loss: 2.9606 | LR: 2.90e-04 | 2562.1ms/step | 7.8 samples/s +Step 20170/44000 | Loss: 2.9843 | LR: 2.90e-04 | 2593.3ms/step | 7.7 samples/s +Step 20180/44000 | Loss: 2.9623 | LR: 2.90e-04 | 2660.4ms/step | 7.5 samples/s +Step 20190/44000 | Loss: 3.0278 | LR: 2.90e-04 | 2673.4ms/step | 7.5 samples/s +Step 20200/44000 | Loss: 2.9883 | LR: 2.89e-04 | 2799.3ms/step | 7.1 samples/s +Step 20210/44000 | Loss: 2.9497 | LR: 2.89e-04 | 2689.7ms/step | 7.4 samples/s +Step 20220/44000 | Loss: 2.9850 | LR: 2.89e-04 | 2652.1ms/step | 7.5 samples/s +Step 20230/44000 | Loss: 2.9397 | LR: 2.89e-04 | 2668.1ms/step | 7.5 samples/s +Step 20240/44000 | Loss: 2.9244 | LR: 2.89e-04 | 2705.5ms/step | 7.4 samples/s +Step 20250/44000 | Loss: 2.9550 | LR: 2.88e-04 | 2649.4ms/step | 7.5 samples/s +Step 20260/44000 | Loss: 2.9296 | LR: 2.88e-04 | 2669.3ms/step | 7.5 samples/s +Step 20270/44000 | Loss: 2.9924 | LR: 2.88e-04 | 2683.8ms/step | 7.5 samples/s +Step 20280/44000 | Loss: 2.9659 | LR: 2.88e-04 | 2641.5ms/step | 7.6 samples/s +Step 20290/44000 | Loss: 2.9707 | LR: 2.88e-04 | 2625.9ms/step | 7.6 samples/s +Step 20300/44000 | Loss: 2.9584 | LR: 2.88e-04 | 2657.8ms/step | 7.5 samples/s +Step 20310/44000 | Loss: 2.9606 | LR: 2.87e-04 | 2626.2ms/step | 7.6 samples/s +Step 20320/44000 | Loss: 2.9811 | LR: 2.87e-04 | 2631.6ms/step | 7.6 samples/s +Step 20330/44000 | Loss: 2.9834 | LR: 2.87e-04 | 2642.9ms/step | 7.6 samples/s +Step 20340/44000 | Loss: 2.9451 | LR: 2.87e-04 | 2562.6ms/step | 7.8 samples/s +Step 20350/44000 | Loss: 2.9242 | LR: 2.87e-04 | 2747.6ms/step | 7.3 samples/s +Step 20360/44000 | Loss: 2.9900 | LR: 2.86e-04 | 2725.8ms/step | 7.3 samples/s +Step 20370/44000 | Loss: 2.9500 | LR: 2.86e-04 | 2671.6ms/step | 7.5 samples/s +Step 20380/44000 | Loss: 2.9476 | LR: 2.86e-04 | 2631.8ms/step | 7.6 samples/s +Step 20390/44000 | Loss: 2.9587 | LR: 2.86e-04 | 2614.5ms/step | 7.6 samples/s +Step 20400/44000 | Loss: 2.9827 | LR: 2.86e-04 | 2585.8ms/step | 7.7 samples/s +Step 20410/44000 | Loss: 2.9942 | LR: 2.86e-04 | 2602.7ms/step | 7.7 samples/s +Step 20420/44000 | Loss: 2.9332 | LR: 2.85e-04 | 2841.6ms/step | 7.0 samples/s +Step 20430/44000 | Loss: 2.9740 | LR: 2.85e-04 | 2651.4ms/step | 7.5 samples/s +Step 20440/44000 | Loss: 2.9530 | LR: 2.85e-04 | 2738.1ms/step | 7.3 samples/s +Step 20450/44000 | Loss: 3.0246 | LR: 2.85e-04 | 2641.2ms/step | 7.6 samples/s +Step 20460/44000 | Loss: 2.9524 | LR: 2.85e-04 | 2531.4ms/step | 7.9 samples/s +Step 20470/44000 | Loss: 2.9486 | LR: 2.84e-04 | 2568.1ms/step | 7.8 samples/s +Step 20480/44000 | Loss: 3.0178 | LR: 2.84e-04 | 2604.2ms/step | 7.7 samples/s +Step 20490/44000 | Loss: 2.9596 | LR: 2.84e-04 | 2576.1ms/step | 7.8 samples/s +Step 20500/44000 | Loss: 2.9963 | LR: 2.84e-04 | 2538.8ms/step | 7.9 samples/s +Step 20510/44000 | Loss: 2.9900 | LR: 2.84e-04 | 2611.1ms/step | 7.7 samples/s +Step 20520/44000 | Loss: 2.9981 | LR: 2.84e-04 | 2640.7ms/step | 7.6 samples/s +Step 20530/44000 | Loss: 3.0272 | LR: 2.83e-04 | 2527.9ms/step | 7.9 samples/s +Step 20540/44000 | Loss: 2.9531 | LR: 2.83e-04 | 2583.2ms/step | 7.7 samples/s +Step 20550/44000 | Loss: 2.9611 | LR: 2.83e-04 | 2719.6ms/step | 7.4 samples/s +Step 20560/44000 | Loss: 2.9350 | LR: 2.83e-04 | 2550.5ms/step | 7.8 samples/s +Step 20570/44000 | Loss: 2.9483 | LR: 2.83e-04 | 2466.2ms/step | 8.1 samples/s +Step 20580/44000 | Loss: 2.9393 | LR: 2.83e-04 | 2404.2ms/step | 8.3 samples/s +Step 20590/44000 | Loss: 2.9846 | LR: 2.82e-04 | 2560.4ms/step | 7.8 samples/s +Step 20600/44000 | Loss: 2.9587 | LR: 2.82e-04 | 2570.2ms/step | 7.8 samples/s +Step 20610/44000 | Loss: 2.9244 | LR: 2.82e-04 | 2644.2ms/step | 7.6 samples/s +Step 20620/44000 | Loss: 2.9468 | LR: 2.82e-04 | 2598.8ms/step | 7.7 samples/s +Step 20630/44000 | Loss: 2.9400 | LR: 2.82e-04 | 2701.5ms/step | 7.4 samples/s +Step 20640/44000 | Loss: 2.9384 | LR: 2.81e-04 | 2768.1ms/step | 7.2 samples/s +Step 20650/44000 | Loss: 2.9500 | LR: 2.81e-04 | 2756.5ms/step | 7.3 samples/s +Step 20660/44000 | Loss: 2.9438 | LR: 2.81e-04 | 2727.1ms/step | 7.3 samples/s +Step 20670/44000 | Loss: 2.9668 | LR: 2.81e-04 | 2579.3ms/step | 7.8 samples/s +Step 20680/44000 | Loss: 2.9589 | LR: 2.81e-04 | 2594.5ms/step | 7.7 samples/s +Step 20690/44000 | Loss: 2.9322 | LR: 2.81e-04 | 2607.0ms/step | 7.7 samples/s +Step 20700/44000 | Loss: 2.9545 | LR: 2.80e-04 | 2603.4ms/step | 7.7 samples/s +Step 20710/44000 | Loss: 2.9542 | LR: 2.80e-04 | 2587.4ms/step | 7.7 samples/s +Step 20720/44000 | Loss: 2.9411 | LR: 2.80e-04 | 2581.5ms/step | 7.7 samples/s +Step 20730/44000 | Loss: 2.9034 | LR: 2.80e-04 | 2600.0ms/step | 7.7 samples/s +Step 20740/44000 | Loss: 2.9511 | LR: 2.80e-04 | 2579.2ms/step | 7.8 samples/s +Step 20750/44000 | Loss: 2.9716 | LR: 2.79e-04 | 2673.4ms/step | 7.5 samples/s +Step 20760/44000 | Loss: 2.9633 | LR: 2.79e-04 | 2799.4ms/step | 7.1 samples/s +Step 20770/44000 | Loss: 3.0128 | LR: 2.79e-04 | 2672.8ms/step | 7.5 samples/s +Step 20780/44000 | Loss: 2.9570 | LR: 2.79e-04 | 2620.1ms/step | 7.6 samples/s +Step 20790/44000 | Loss: 2.9721 | LR: 2.79e-04 | 2803.9ms/step | 7.1 samples/s +Step 20800/44000 | Loss: 2.9693 | LR: 2.79e-04 | 2646.9ms/step | 7.6 samples/s +Step 20810/44000 | Loss: 2.9834 | LR: 2.78e-04 | 2572.4ms/step | 7.8 samples/s +Step 20820/44000 | Loss: 2.9603 | LR: 2.78e-04 | 2677.7ms/step | 7.5 samples/s +Step 20830/44000 | Loss: 2.9189 | LR: 2.78e-04 | 2751.8ms/step | 7.3 samples/s +Step 20840/44000 | Loss: 2.9579 | LR: 2.78e-04 | 2707.8ms/step | 7.4 samples/s +Step 20850/44000 | Loss: 2.9506 | LR: 2.78e-04 | 2698.5ms/step | 7.4 samples/s +Step 20860/44000 | Loss: 3.0027 | LR: 2.77e-04 | 2727.7ms/step | 7.3 samples/s +Step 20870/44000 | Loss: 2.9710 | LR: 2.77e-04 | 2696.0ms/step | 7.4 samples/s +Step 20880/44000 | Loss: 2.9542 | LR: 2.77e-04 | 2535.2ms/step | 7.9 samples/s +Step 20890/44000 | Loss: 2.9886 | LR: 2.77e-04 | 2360.8ms/step | 8.5 samples/s +Step 20900/44000 | Loss: 2.9680 | LR: 2.77e-04 | 2236.0ms/step | 8.9 samples/s +Step 20910/44000 | Loss: 2.9688 | LR: 2.77e-04 | 2435.6ms/step | 8.2 samples/s +Step 20920/44000 | Loss: 2.9728 | LR: 2.76e-04 | 2409.6ms/step | 8.3 samples/s +Step 20930/44000 | Loss: 2.9073 | LR: 2.76e-04 | 2152.5ms/step | 9.3 samples/s +Step 20940/44000 | Loss: 2.9647 | LR: 2.76e-04 | 2178.6ms/step | 9.2 samples/s +Step 20950/44000 | Loss: 2.9030 | LR: 2.76e-04 | 2183.8ms/step | 9.2 samples/s +Step 20960/44000 | Loss: 2.9732 | LR: 2.76e-04 | 2168.7ms/step | 9.2 samples/s +Step 20970/44000 | Loss: 2.9428 | LR: 2.75e-04 | 2147.6ms/step | 9.3 samples/s +Step 20980/44000 | Loss: 2.9611 | LR: 2.75e-04 | 2210.0ms/step | 9.0 samples/s +Step 20990/44000 | Loss: 2.9550 | LR: 2.75e-04 | 2150.8ms/step | 9.3 samples/s +Step 21000/44000 | Loss: 2.9815 | LR: 2.75e-04 | 2157.9ms/step | 9.3 samples/s +Validation at step 21000: Loss = 2.9493, Perplexity = 19.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-21000 +Step 21010/44000 | Loss: 2.9321 | LR: 2.75e-04 | 4440.8ms/step | 4.5 samples/s +Step 21020/44000 | Loss: 2.9510 | LR: 2.75e-04 | 2147.3ms/step | 9.3 samples/s +Step 21030/44000 | Loss: 2.9662 | LR: 2.74e-04 | 2150.8ms/step | 9.3 samples/s +Step 21040/44000 | Loss: 2.9584 | LR: 2.74e-04 | 2116.8ms/step | 9.4 samples/s +Step 21050/44000 | Loss: 2.9312 | LR: 2.74e-04 | 2183.2ms/step | 9.2 samples/s +Step 21060/44000 | Loss: 2.9428 | LR: 2.74e-04 | 2164.7ms/step | 9.2 samples/s +Step 21070/44000 | Loss: 2.9857 | LR: 2.74e-04 | 2168.9ms/step | 9.2 samples/s +Step 21080/44000 | Loss: 2.9222 | LR: 2.73e-04 | 2162.5ms/step | 9.2 samples/s +Step 21090/44000 | Loss: 2.9872 | LR: 2.73e-04 | 2177.4ms/step | 9.2 samples/s +Step 21100/44000 | Loss: 2.9620 | LR: 2.73e-04 | 2157.6ms/step | 9.3 samples/s +Step 21110/44000 | Loss: 2.9706 | LR: 2.73e-04 | 2148.8ms/step | 9.3 samples/s +Step 21120/44000 | Loss: 3.0067 | LR: 2.73e-04 | 2166.1ms/step | 9.2 samples/s +Step 21130/44000 | Loss: 2.9745 | LR: 2.73e-04 | 2149.2ms/step | 9.3 samples/s +Step 21140/44000 | Loss: 2.9460 | LR: 2.72e-04 | 2163.2ms/step | 9.2 samples/s +Step 21150/44000 | Loss: 2.9953 | LR: 2.72e-04 | 2123.1ms/step | 9.4 samples/s +Step 21160/44000 | Loss: 2.9573 | LR: 2.72e-04 | 2163.5ms/step | 9.2 samples/s +Step 21170/44000 | Loss: 2.9384 | LR: 2.72e-04 | 2153.2ms/step | 9.3 samples/s +Step 21180/44000 | Loss: 2.8869 | LR: 2.72e-04 | 2158.4ms/step | 9.3 samples/s +Step 21190/44000 | Loss: 2.9113 | LR: 2.71e-04 | 2174.1ms/step | 9.2 samples/s +Step 21200/44000 | Loss: 2.9334 | LR: 2.71e-04 | 2184.8ms/step | 9.2 samples/s +Step 21210/44000 | Loss: 2.9391 | LR: 2.71e-04 | 2138.0ms/step | 9.4 samples/s +Step 21220/44000 | Loss: 2.9619 | LR: 2.71e-04 | 2151.4ms/step | 9.3 samples/s +Step 21230/44000 | Loss: 2.9910 | LR: 2.71e-04 | 2149.1ms/step | 9.3 samples/s +Step 21240/44000 | Loss: 2.9157 | LR: 2.71e-04 | 2148.4ms/step | 9.3 samples/s +Step 21250/44000 | Loss: 2.9533 | LR: 2.70e-04 | 2149.8ms/step | 9.3 samples/s +Step 21260/44000 | Loss: 2.9330 | LR: 2.70e-04 | 2138.8ms/step | 9.4 samples/s +Step 21270/44000 | Loss: 2.9444 | LR: 2.70e-04 | 2160.0ms/step | 9.3 samples/s +Step 21280/44000 | Loss: 2.9157 | LR: 2.70e-04 | 2162.9ms/step | 9.2 samples/s +Step 21290/44000 | Loss: 2.9136 | LR: 2.70e-04 | 2168.7ms/step | 9.2 samples/s +Step 21300/44000 | Loss: 2.9195 | LR: 2.70e-04 | 2142.0ms/step | 9.3 samples/s +Step 21310/44000 | Loss: 2.9604 | LR: 2.69e-04 | 2145.9ms/step | 9.3 samples/s +Step 21320/44000 | Loss: 2.9292 | LR: 2.69e-04 | 2144.2ms/step | 9.3 samples/s +Step 21330/44000 | Loss: 2.9622 | LR: 2.69e-04 | 2136.0ms/step | 9.4 samples/s +Step 21340/44000 | Loss: 2.9674 | LR: 2.69e-04 | 2143.9ms/step | 9.3 samples/s +Step 21350/44000 | Loss: 2.9959 | LR: 2.69e-04 | 2205.2ms/step | 9.1 samples/s +Step 21360/44000 | Loss: 2.9421 | LR: 2.68e-04 | 2143.3ms/step | 9.3 samples/s +Step 21370/44000 | Loss: 2.9705 | LR: 2.68e-04 | 2141.9ms/step | 9.3 samples/s +Step 21380/44000 | Loss: 2.9682 | LR: 2.68e-04 | 2158.4ms/step | 9.3 samples/s +Step 21390/44000 | Loss: 2.9776 | LR: 2.68e-04 | 2142.5ms/step | 9.3 samples/s +Step 21400/44000 | Loss: 2.9668 | LR: 2.68e-04 | 2150.1ms/step | 9.3 samples/s +Step 21410/44000 | Loss: 2.9519 | LR: 2.68e-04 | 2171.8ms/step | 9.2 samples/s +Step 21420/44000 | Loss: 2.9452 | LR: 2.67e-04 | 2165.9ms/step | 9.2 samples/s +Step 21430/44000 | Loss: 2.9339 | LR: 2.67e-04 | 2186.3ms/step | 9.1 samples/s +Step 21440/44000 | Loss: 2.9259 | LR: 2.67e-04 | 2160.0ms/step | 9.3 samples/s +Step 21450/44000 | Loss: 2.9280 | LR: 2.67e-04 | 2183.2ms/step | 9.2 samples/s +Step 21460/44000 | Loss: 2.9488 | LR: 2.67e-04 | 2126.7ms/step | 9.4 samples/s +Step 21470/44000 | Loss: 2.9787 | LR: 2.66e-04 | 2110.4ms/step | 9.5 samples/s +Step 21480/44000 | Loss: 2.9408 | LR: 2.66e-04 | 2094.1ms/step | 9.6 samples/s +Step 21490/44000 | Loss: 2.9481 | LR: 2.66e-04 | 2116.4ms/step | 9.5 samples/s +Step 21500/44000 | Loss: 2.9226 | LR: 2.66e-04 | 2080.3ms/step | 9.6 samples/s +Step 21510/44000 | Loss: 2.9294 | LR: 2.66e-04 | 2104.5ms/step | 9.5 samples/s +Step 21520/44000 | Loss: 2.8839 | LR: 2.66e-04 | 2115.9ms/step | 9.5 samples/s +Step 21530/44000 | Loss: 3.0305 | LR: 2.65e-04 | 2095.6ms/step | 9.5 samples/s +Step 21540/44000 | Loss: 2.9232 | LR: 2.65e-04 | 2121.5ms/step | 9.4 samples/s +Step 21550/44000 | Loss: 2.9135 | LR: 2.65e-04 | 2105.7ms/step | 9.5 samples/s +Step 21560/44000 | Loss: 2.9370 | LR: 2.65e-04 | 2116.3ms/step | 9.5 samples/s +Step 21570/44000 | Loss: 2.9677 | LR: 2.65e-04 | 2095.2ms/step | 9.5 samples/s +Step 21580/44000 | Loss: 2.9199 | LR: 2.64e-04 | 2126.5ms/step | 9.4 samples/s +Step 21590/44000 | Loss: 2.9467 | LR: 2.64e-04 | 2105.4ms/step | 9.5 samples/s +Step 21600/44000 | Loss: 2.9793 | LR: 2.64e-04 | 2111.1ms/step | 9.5 samples/s +Step 21610/44000 | Loss: 2.9350 | LR: 2.64e-04 | 2095.4ms/step | 9.5 samples/s +Step 21620/44000 | Loss: 2.9556 | LR: 2.64e-04 | 2116.4ms/step | 9.5 samples/s +Step 21630/44000 | Loss: 2.9449 | LR: 2.64e-04 | 2158.5ms/step | 9.3 samples/s +Step 21640/44000 | Loss: 2.9354 | LR: 2.63e-04 | 2152.3ms/step | 9.3 samples/s +Step 21650/44000 | Loss: 2.9509 | LR: 2.63e-04 | 2160.6ms/step | 9.3 samples/s +Step 21660/44000 | Loss: 2.9818 | LR: 2.63e-04 | 2182.2ms/step | 9.2 samples/s +Step 21670/44000 | Loss: 2.9718 | LR: 2.63e-04 | 2184.4ms/step | 9.2 samples/s +Step 21680/44000 | Loss: 2.9529 | LR: 2.63e-04 | 2168.7ms/step | 9.2 samples/s +Step 21690/44000 | Loss: 2.9508 | LR: 2.62e-04 | 2194.5ms/step | 9.1 samples/s +Step 21700/44000 | Loss: 2.9716 | LR: 2.62e-04 | 2184.6ms/step | 9.2 samples/s +Step 21710/44000 | Loss: 2.9327 | LR: 2.62e-04 | 2183.8ms/step | 9.2 samples/s +Step 21720/44000 | Loss: 2.9254 | LR: 2.62e-04 | 2167.8ms/step | 9.2 samples/s +Step 21730/44000 | Loss: 2.9433 | LR: 2.62e-04 | 2178.8ms/step | 9.2 samples/s +Step 21740/44000 | Loss: 2.9385 | LR: 2.62e-04 | 2216.2ms/step | 9.0 samples/s +Step 21750/44000 | Loss: 2.9367 | LR: 2.61e-04 | 2195.0ms/step | 9.1 samples/s +Step 21760/44000 | Loss: 2.9536 | LR: 2.61e-04 | 2195.2ms/step | 9.1 samples/s +Step 21770/44000 | Loss: 2.9360 | LR: 2.61e-04 | 2152.8ms/step | 9.3 samples/s +Step 21780/44000 | Loss: 2.9885 | LR: 2.61e-04 | 2136.8ms/step | 9.4 samples/s +Step 21790/44000 | Loss: 2.9815 | LR: 2.61e-04 | 2168.6ms/step | 9.2 samples/s +Step 21800/44000 | Loss: 2.9370 | LR: 2.60e-04 | 2154.3ms/step | 9.3 samples/s +Step 21810/44000 | Loss: 2.9542 | LR: 2.60e-04 | 2161.9ms/step | 9.3 samples/s +Step 21820/44000 | Loss: 2.9456 | LR: 2.60e-04 | 2147.8ms/step | 9.3 samples/s +Step 21830/44000 | Loss: 2.9347 | LR: 2.60e-04 | 2189.5ms/step | 9.1 samples/s +Step 21840/44000 | Loss: 2.9526 | LR: 2.60e-04 | 2145.8ms/step | 9.3 samples/s +Step 21850/44000 | Loss: 2.9197 | LR: 2.60e-04 | 2170.2ms/step | 9.2 samples/s +Step 21860/44000 | Loss: 2.9663 | LR: 2.59e-04 | 2143.1ms/step | 9.3 samples/s +Step 21870/44000 | Loss: 2.9704 | LR: 2.59e-04 | 2189.1ms/step | 9.1 samples/s +Step 21880/44000 | Loss: 2.9494 | LR: 2.59e-04 | 2218.9ms/step | 9.0 samples/s +Step 21890/44000 | Loss: 2.9573 | LR: 2.59e-04 | 2140.5ms/step | 9.3 samples/s +Step 21900/44000 | Loss: 2.9626 | LR: 2.59e-04 | 2055.1ms/step | 9.7 samples/s +Step 21910/44000 | Loss: 2.9443 | LR: 2.58e-04 | 2162.1ms/step | 9.3 samples/s +Step 21920/44000 | Loss: 2.9584 | LR: 2.58e-04 | 2143.9ms/step | 9.3 samples/s +Step 21930/44000 | Loss: 2.9272 | LR: 2.58e-04 | 2134.7ms/step | 9.4 samples/s +Step 21940/44000 | Loss: 2.9230 | LR: 2.58e-04 | 2161.5ms/step | 9.3 samples/s +Step 21950/44000 | Loss: 2.9615 | LR: 2.58e-04 | 2196.8ms/step | 9.1 samples/s +Step 21960/44000 | Loss: 2.9792 | LR: 2.58e-04 | 2190.1ms/step | 9.1 samples/s +Step 21970/44000 | Loss: 2.9295 | LR: 2.57e-04 | 2184.3ms/step | 9.2 samples/s +Step 21980/44000 | Loss: 2.9469 | LR: 2.57e-04 | 2155.3ms/step | 9.3 samples/s +Step 21990/44000 | Loss: 2.9476 | LR: 2.57e-04 | 2185.6ms/step | 9.2 samples/s +Step 22000/44000 | Loss: 2.9724 | LR: 2.57e-04 | 2153.4ms/step | 9.3 samples/s +Validation at step 22000: Loss = 2.9392, Perplexity = 18.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-22000 +Step 22010/44000 | Loss: 2.9337 | LR: 2.57e-04 | 4505.9ms/step | 4.4 samples/s +Step 22020/44000 | Loss: 2.9640 | LR: 2.56e-04 | 2148.9ms/step | 9.3 samples/s +Step 22030/44000 | Loss: 2.9909 | LR: 2.56e-04 | 2143.1ms/step | 9.3 samples/s +Step 22040/44000 | Loss: 2.9331 | LR: 2.56e-04 | 2174.4ms/step | 9.2 samples/s +Step 22050/44000 | Loss: 2.9560 | LR: 2.56e-04 | 2170.7ms/step | 9.2 samples/s +Step 22060/44000 | Loss: 2.9783 | LR: 2.56e-04 | 2190.3ms/step | 9.1 samples/s +Step 22070/44000 | Loss: 2.9625 | LR: 2.56e-04 | 2183.4ms/step | 9.2 samples/s +Step 22080/44000 | Loss: 2.9095 | LR: 2.55e-04 | 2153.1ms/step | 9.3 samples/s +Step 22090/44000 | Loss: 2.9321 | LR: 2.55e-04 | 2171.5ms/step | 9.2 samples/s +Step 22100/44000 | Loss: 2.9432 | LR: 2.55e-04 | 2162.0ms/step | 9.3 samples/s +Step 22110/44000 | Loss: 2.9518 | LR: 2.55e-04 | 2125.3ms/step | 9.4 samples/s +Step 22120/44000 | Loss: 2.9568 | LR: 2.55e-04 | 2168.7ms/step | 9.2 samples/s +Step 22130/44000 | Loss: 2.9401 | LR: 2.54e-04 | 2218.6ms/step | 9.0 samples/s +Step 22140/44000 | Loss: 2.8972 | LR: 2.54e-04 | 2172.2ms/step | 9.2 samples/s +Step 22150/44000 | Loss: 2.9664 | LR: 2.54e-04 | 2143.6ms/step | 9.3 samples/s +Step 22160/44000 | Loss: 2.9487 | LR: 2.54e-04 | 2147.9ms/step | 9.3 samples/s +Step 22170/44000 | Loss: 2.9648 | LR: 2.54e-04 | 2159.2ms/step | 9.3 samples/s +Step 22180/44000 | Loss: 2.9204 | LR: 2.54e-04 | 2158.2ms/step | 9.3 samples/s +Step 22190/44000 | Loss: 2.9457 | LR: 2.53e-04 | 2183.6ms/step | 9.2 samples/s +Step 22200/44000 | Loss: 2.8725 | LR: 2.53e-04 | 2165.9ms/step | 9.2 samples/s +Step 22210/44000 | Loss: 2.9479 | LR: 2.53e-04 | 2137.3ms/step | 9.4 samples/s +Step 22220/44000 | Loss: 2.9339 | LR: 2.53e-04 | 2142.0ms/step | 9.3 samples/s +Step 22230/44000 | Loss: 2.9525 | LR: 2.53e-04 | 2185.9ms/step | 9.1 samples/s +Step 22240/44000 | Loss: 2.9394 | LR: 2.52e-04 | 2208.8ms/step | 9.1 samples/s +Step 22250/44000 | Loss: 2.9806 | LR: 2.52e-04 | 2197.4ms/step | 9.1 samples/s +Step 22260/44000 | Loss: 2.9550 | LR: 2.52e-04 | 2153.9ms/step | 9.3 samples/s +Step 22270/44000 | Loss: 2.9578 | LR: 2.52e-04 | 2166.7ms/step | 9.2 samples/s +Step 22280/44000 | Loss: 2.9217 | LR: 2.52e-04 | 2137.3ms/step | 9.4 samples/s +Step 22290/44000 | Loss: 2.9333 | LR: 2.52e-04 | 2128.1ms/step | 9.4 samples/s +Step 22300/44000 | Loss: 2.9301 | LR: 2.51e-04 | 2154.9ms/step | 9.3 samples/s +Step 22310/44000 | Loss: 2.9543 | LR: 2.51e-04 | 2136.5ms/step | 9.4 samples/s +Step 22320/44000 | Loss: 2.9680 | LR: 2.51e-04 | 2140.0ms/step | 9.3 samples/s +Step 22330/44000 | Loss: 2.9409 | LR: 2.51e-04 | 2179.1ms/step | 9.2 samples/s +Step 22340/44000 | Loss: 2.9621 | LR: 2.51e-04 | 2180.9ms/step | 9.2 samples/s +Step 22350/44000 | Loss: 2.9347 | LR: 2.50e-04 | 2137.7ms/step | 9.4 samples/s +Step 22360/44000 | Loss: 2.9606 | LR: 2.50e-04 | 2140.8ms/step | 9.3 samples/s +Step 22370/44000 | Loss: 2.9409 | LR: 2.50e-04 | 2178.7ms/step | 9.2 samples/s +Step 22380/44000 | Loss: 2.9534 | LR: 2.50e-04 | 2164.7ms/step | 9.2 samples/s +Step 22390/44000 | Loss: 2.9680 | LR: 2.50e-04 | 2160.3ms/step | 9.3 samples/s +Step 22400/44000 | Loss: 2.9289 | LR: 2.50e-04 | 2147.8ms/step | 9.3 samples/s +Step 22410/44000 | Loss: 2.9333 | LR: 2.49e-04 | 2159.1ms/step | 9.3 samples/s +Step 22420/44000 | Loss: 2.9419 | LR: 2.49e-04 | 2158.5ms/step | 9.3 samples/s +Step 22430/44000 | Loss: 2.9664 | LR: 2.49e-04 | 2147.8ms/step | 9.3 samples/s +Step 22440/44000 | Loss: 2.9571 | LR: 2.49e-04 | 2152.9ms/step | 9.3 samples/s +Step 22450/44000 | Loss: 2.9630 | LR: 2.49e-04 | 2211.0ms/step | 9.0 samples/s +Step 22460/44000 | Loss: 2.9402 | LR: 2.48e-04 | 2192.6ms/step | 9.1 samples/s +Step 22470/44000 | Loss: 2.9511 | LR: 2.48e-04 | 2144.9ms/step | 9.3 samples/s +Step 22480/44000 | Loss: 2.9255 | LR: 2.48e-04 | 2205.2ms/step | 9.1 samples/s +Step 22490/44000 | Loss: 2.9285 | LR: 2.48e-04 | 2162.8ms/step | 9.2 samples/s +Step 22500/44000 | Loss: 2.9200 | LR: 2.48e-04 | 2145.0ms/step | 9.3 samples/s +Step 22510/44000 | Loss: 2.9578 | LR: 2.48e-04 | 2161.6ms/step | 9.3 samples/s +Step 22520/44000 | Loss: 2.9290 | LR: 2.47e-04 | 2191.3ms/step | 9.1 samples/s +Step 22530/44000 | Loss: 2.9604 | LR: 2.47e-04 | 2158.3ms/step | 9.3 samples/s +Step 22540/44000 | Loss: 2.9532 | LR: 2.47e-04 | 2174.2ms/step | 9.2 samples/s +Step 22550/44000 | Loss: 2.8868 | LR: 2.47e-04 | 2163.4ms/step | 9.2 samples/s +Step 22560/44000 | Loss: 2.9357 | LR: 2.47e-04 | 2161.4ms/step | 9.3 samples/s +Step 22570/44000 | Loss: 2.9470 | LR: 2.46e-04 | 2161.1ms/step | 9.3 samples/s +Step 22580/44000 | Loss: 2.9669 | LR: 2.46e-04 | 2167.0ms/step | 9.2 samples/s +Step 22590/44000 | Loss: 2.9487 | LR: 2.46e-04 | 2151.9ms/step | 9.3 samples/s +Step 22600/44000 | Loss: 2.9207 | LR: 2.46e-04 | 2195.5ms/step | 9.1 samples/s +Step 22610/44000 | Loss: 2.9172 | LR: 2.46e-04 | 2183.2ms/step | 9.2 samples/s +Step 22620/44000 | Loss: 2.9211 | LR: 2.46e-04 | 2159.2ms/step | 9.3 samples/s +Step 22630/44000 | Loss: 2.9307 | LR: 2.45e-04 | 2180.2ms/step | 9.2 samples/s +Step 22640/44000 | Loss: 2.9701 | LR: 2.45e-04 | 2182.5ms/step | 9.2 samples/s +Step 22650/44000 | Loss: 2.9460 | LR: 2.45e-04 | 2158.4ms/step | 9.3 samples/s +Step 22660/44000 | Loss: 2.9797 | LR: 2.45e-04 | 2168.3ms/step | 9.2 samples/s +Step 22670/44000 | Loss: 2.9295 | LR: 2.45e-04 | 2147.5ms/step | 9.3 samples/s +Step 22680/44000 | Loss: 2.9406 | LR: 2.44e-04 | 2152.8ms/step | 9.3 samples/s +Step 22690/44000 | Loss: 3.0060 | LR: 2.44e-04 | 2168.8ms/step | 9.2 samples/s +Step 22700/44000 | Loss: 2.9602 | LR: 2.44e-04 | 2137.0ms/step | 9.4 samples/s +Step 22710/44000 | Loss: 2.9179 | LR: 2.44e-04 | 2156.0ms/step | 9.3 samples/s +Step 22720/44000 | Loss: 2.9303 | LR: 2.44e-04 | 2176.2ms/step | 9.2 samples/s +Step 22730/44000 | Loss: 2.9246 | LR: 2.44e-04 | 2137.2ms/step | 9.4 samples/s +Step 22740/44000 | Loss: 2.9619 | LR: 2.43e-04 | 2180.2ms/step | 9.2 samples/s +Step 22750/44000 | Loss: 2.9317 | LR: 2.43e-04 | 2178.7ms/step | 9.2 samples/s +Step 22760/44000 | Loss: 2.9332 | LR: 2.43e-04 | 2134.0ms/step | 9.4 samples/s +Step 22770/44000 | Loss: 2.9473 | LR: 2.43e-04 | 2214.2ms/step | 9.0 samples/s +Step 22780/44000 | Loss: 2.9298 | LR: 2.43e-04 | 2148.7ms/step | 9.3 samples/s +Step 22790/44000 | Loss: 2.8924 | LR: 2.42e-04 | 2174.4ms/step | 9.2 samples/s +Step 22800/44000 | Loss: 2.9268 | LR: 2.42e-04 | 2169.4ms/step | 9.2 samples/s +Step 22810/44000 | Loss: 2.9050 | LR: 2.42e-04 | 2154.2ms/step | 9.3 samples/s +Step 22820/44000 | Loss: 2.9287 | LR: 2.42e-04 | 2149.5ms/step | 9.3 samples/s +Step 22830/44000 | Loss: 2.9176 | LR: 2.42e-04 | 2156.9ms/step | 9.3 samples/s +Step 22840/44000 | Loss: 2.9139 | LR: 2.42e-04 | 2184.9ms/step | 9.2 samples/s +Step 22850/44000 | Loss: 2.9197 | LR: 2.41e-04 | 2150.4ms/step | 9.3 samples/s +Step 22860/44000 | Loss: 2.9833 | LR: 2.41e-04 | 2150.4ms/step | 9.3 samples/s +Step 22870/44000 | Loss: 2.9409 | LR: 2.41e-04 | 2156.4ms/step | 9.3 samples/s +Step 22880/44000 | Loss: 2.9498 | LR: 2.41e-04 | 2157.2ms/step | 9.3 samples/s +Step 22890/44000 | Loss: 2.9317 | LR: 2.41e-04 | 2164.3ms/step | 9.2 samples/s +Step 22900/44000 | Loss: 2.9147 | LR: 2.40e-04 | 2065.0ms/step | 9.7 samples/s +Step 22910/44000 | Loss: 2.9361 | LR: 2.40e-04 | 2142.5ms/step | 9.3 samples/s +Step 22920/44000 | Loss: 2.9476 | LR: 2.40e-04 | 2174.6ms/step | 9.2 samples/s +Step 22930/44000 | Loss: 2.9295 | LR: 2.40e-04 | 2164.0ms/step | 9.2 samples/s +Step 22940/44000 | Loss: 2.9959 | LR: 2.40e-04 | 2169.7ms/step | 9.2 samples/s +Step 22950/44000 | Loss: 2.9162 | LR: 2.40e-04 | 2146.5ms/step | 9.3 samples/s +Step 22960/44000 | Loss: 2.9142 | LR: 2.39e-04 | 2197.6ms/step | 9.1 samples/s +Step 22970/44000 | Loss: 2.9360 | LR: 2.39e-04 | 2163.3ms/step | 9.2 samples/s +Step 22980/44000 | Loss: 2.9416 | LR: 2.39e-04 | 2182.8ms/step | 9.2 samples/s +Step 22990/44000 | Loss: 2.9172 | LR: 2.39e-04 | 2158.0ms/step | 9.3 samples/s +Step 23000/44000 | Loss: 2.8744 | LR: 2.39e-04 | 2148.5ms/step | 9.3 samples/s +Validation at step 23000: Loss = 2.9296, Perplexity = 18.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-23000 +Step 23010/44000 | Loss: 2.9525 | LR: 2.38e-04 | 4474.3ms/step | 4.5 samples/s +Step 23020/44000 | Loss: 2.9466 | LR: 2.38e-04 | 2189.8ms/step | 9.1 samples/s +Step 23030/44000 | Loss: 2.8955 | LR: 2.38e-04 | 2178.9ms/step | 9.2 samples/s +Step 23040/44000 | Loss: 2.9149 | LR: 2.38e-04 | 2131.6ms/step | 9.4 samples/s +Step 23050/44000 | Loss: 2.9363 | LR: 2.38e-04 | 2174.1ms/step | 9.2 samples/s +Step 23060/44000 | Loss: 2.9538 | LR: 2.38e-04 | 2208.0ms/step | 9.1 samples/s +Step 23070/44000 | Loss: 2.9297 | LR: 2.37e-04 | 2120.2ms/step | 9.4 samples/s +Step 23080/44000 | Loss: 2.9788 | LR: 2.37e-04 | 2184.4ms/step | 9.2 samples/s +Step 23090/44000 | Loss: 2.9443 | LR: 2.37e-04 | 2137.8ms/step | 9.4 samples/s +Step 23100/44000 | Loss: 2.9503 | LR: 2.37e-04 | 2219.4ms/step | 9.0 samples/s +Step 23110/44000 | Loss: 2.8982 | LR: 2.37e-04 | 2143.9ms/step | 9.3 samples/s +Step 23120/44000 | Loss: 2.9686 | LR: 2.36e-04 | 2147.2ms/step | 9.3 samples/s +Step 23130/44000 | Loss: 2.9637 | LR: 2.36e-04 | 2172.2ms/step | 9.2 samples/s +Step 23140/44000 | Loss: 2.9404 | LR: 2.36e-04 | 2234.2ms/step | 9.0 samples/s +Step 23150/44000 | Loss: 2.9175 | LR: 2.36e-04 | 2146.0ms/step | 9.3 samples/s +Step 23160/44000 | Loss: 2.9509 | LR: 2.36e-04 | 2178.7ms/step | 9.2 samples/s +Step 23170/44000 | Loss: 2.9711 | LR: 2.36e-04 | 2201.0ms/step | 9.1 samples/s +Step 23180/44000 | Loss: 2.9094 | LR: 2.35e-04 | 2249.6ms/step | 8.9 samples/s +Step 23190/44000 | Loss: 2.9137 | LR: 2.35e-04 | 2154.8ms/step | 9.3 samples/s +Step 23200/44000 | Loss: 2.8874 | LR: 2.35e-04 | 2164.0ms/step | 9.2 samples/s +Step 23210/44000 | Loss: 2.9759 | LR: 2.35e-04 | 2222.1ms/step | 9.0 samples/s +Step 23220/44000 | Loss: 2.9895 | LR: 2.35e-04 | 2221.1ms/step | 9.0 samples/s +Step 23230/44000 | Loss: 2.9289 | LR: 2.34e-04 | 2165.3ms/step | 9.2 samples/s +Step 23240/44000 | Loss: 2.9391 | LR: 2.34e-04 | 2221.8ms/step | 9.0 samples/s +Step 23250/44000 | Loss: 2.8896 | LR: 2.34e-04 | 2166.4ms/step | 9.2 samples/s +Step 23260/44000 | Loss: 2.9400 | LR: 2.34e-04 | 2247.4ms/step | 8.9 samples/s +Step 23270/44000 | Loss: 2.9737 | LR: 2.34e-04 | 2139.6ms/step | 9.3 samples/s +Step 23280/44000 | Loss: 2.9681 | LR: 2.34e-04 | 2172.2ms/step | 9.2 samples/s +Step 23290/44000 | Loss: 2.9028 | LR: 2.33e-04 | 2180.1ms/step | 9.2 samples/s +Step 23300/44000 | Loss: 2.9164 | LR: 2.33e-04 | 2253.6ms/step | 8.9 samples/s +Step 23310/44000 | Loss: 2.9696 | LR: 2.33e-04 | 2153.7ms/step | 9.3 samples/s +Step 23320/44000 | Loss: 2.9412 | LR: 2.33e-04 | 2159.3ms/step | 9.3 samples/s +Step 23330/44000 | Loss: 2.9109 | LR: 2.33e-04 | 2170.0ms/step | 9.2 samples/s +Step 23340/44000 | Loss: 2.9250 | LR: 2.32e-04 | 2211.8ms/step | 9.0 samples/s +Step 23350/44000 | Loss: 2.8971 | LR: 2.32e-04 | 2205.6ms/step | 9.1 samples/s +Step 23360/44000 | Loss: 2.8671 | LR: 2.32e-04 | 2138.6ms/step | 9.4 samples/s +Step 23370/44000 | Loss: 2.8988 | LR: 2.32e-04 | 2141.5ms/step | 9.3 samples/s +Step 23380/44000 | Loss: 2.9045 | LR: 2.32e-04 | 2199.5ms/step | 9.1 samples/s +Step 23390/44000 | Loss: 2.8919 | LR: 2.32e-04 | 2226.8ms/step | 9.0 samples/s +Step 23400/44000 | Loss: 2.9456 | LR: 2.31e-04 | 2228.6ms/step | 9.0 samples/s +Step 23410/44000 | Loss: 2.9330 | LR: 2.31e-04 | 2203.9ms/step | 9.1 samples/s +Step 23420/44000 | Loss: 2.9615 | LR: 2.31e-04 | 2212.6ms/step | 9.0 samples/s +Step 23430/44000 | Loss: 2.9375 | LR: 2.31e-04 | 2185.0ms/step | 9.2 samples/s +Step 23440/44000 | Loss: 2.9399 | LR: 2.31e-04 | 2153.1ms/step | 9.3 samples/s +Step 23450/44000 | Loss: 2.9387 | LR: 2.30e-04 | 2189.1ms/step | 9.1 samples/s +Step 23460/44000 | Loss: 2.9165 | LR: 2.30e-04 | 2159.0ms/step | 9.3 samples/s +Step 23470/44000 | Loss: 2.9857 | LR: 2.30e-04 | 2213.0ms/step | 9.0 samples/s +Step 23480/44000 | Loss: 2.9344 | LR: 2.30e-04 | 2185.1ms/step | 9.2 samples/s +Step 23490/44000 | Loss: 2.9537 | LR: 2.30e-04 | 2138.1ms/step | 9.4 samples/s +Step 23500/44000 | Loss: 2.9410 | LR: 2.30e-04 | 2228.9ms/step | 9.0 samples/s +Step 23510/44000 | Loss: 2.8933 | LR: 2.29e-04 | 2329.2ms/step | 8.6 samples/s +Step 23520/44000 | Loss: 2.9427 | LR: 2.29e-04 | 2208.9ms/step | 9.1 samples/s +Step 23530/44000 | Loss: 2.9036 | LR: 2.29e-04 | 2171.0ms/step | 9.2 samples/s +Step 23540/44000 | Loss: 2.9508 | LR: 2.29e-04 | 2141.3ms/step | 9.3 samples/s +Step 23550/44000 | Loss: 2.9186 | LR: 2.29e-04 | 2156.0ms/step | 9.3 samples/s +Step 23560/44000 | Loss: 2.9462 | LR: 2.29e-04 | 2131.8ms/step | 9.4 samples/s +Step 23570/44000 | Loss: 2.9429 | LR: 2.28e-04 | 2346.4ms/step | 8.5 samples/s +Step 23580/44000 | Loss: 2.9422 | LR: 2.28e-04 | 2222.5ms/step | 9.0 samples/s +Step 23590/44000 | Loss: 2.9628 | LR: 2.28e-04 | 2148.6ms/step | 9.3 samples/s +Step 23600/44000 | Loss: 2.9562 | LR: 2.28e-04 | 2149.2ms/step | 9.3 samples/s +Step 23610/44000 | Loss: 2.9415 | LR: 2.28e-04 | 2166.6ms/step | 9.2 samples/s +Step 23620/44000 | Loss: 2.9442 | LR: 2.27e-04 | 2167.0ms/step | 9.2 samples/s +Step 23630/44000 | Loss: 2.8882 | LR: 2.27e-04 | 2154.6ms/step | 9.3 samples/s +Step 23640/44000 | Loss: 2.9521 | LR: 2.27e-04 | 2196.5ms/step | 9.1 samples/s +Step 23650/44000 | Loss: 2.9509 | LR: 2.27e-04 | 2232.2ms/step | 9.0 samples/s +Step 23660/44000 | Loss: 2.8945 | LR: 2.27e-04 | 2175.4ms/step | 9.2 samples/s +Step 23670/44000 | Loss: 2.9346 | LR: 2.27e-04 | 2212.5ms/step | 9.0 samples/s +Step 23680/44000 | Loss: 2.9375 | LR: 2.26e-04 | 2154.3ms/step | 9.3 samples/s +Step 23690/44000 | Loss: 2.9253 | LR: 2.26e-04 | 2206.8ms/step | 9.1 samples/s +Step 23700/44000 | Loss: 2.9178 | LR: 2.26e-04 | 2162.4ms/step | 9.2 samples/s +Step 23710/44000 | Loss: 2.9142 | LR: 2.26e-04 | 2150.6ms/step | 9.3 samples/s +Step 23720/44000 | Loss: 2.8846 | LR: 2.26e-04 | 2157.1ms/step | 9.3 samples/s +Step 23730/44000 | Loss: 2.9464 | LR: 2.25e-04 | 2120.8ms/step | 9.4 samples/s +Step 23740/44000 | Loss: 2.9314 | LR: 2.25e-04 | 2139.5ms/step | 9.3 samples/s +Step 23750/44000 | Loss: 2.9274 | LR: 2.25e-04 | 2151.3ms/step | 9.3 samples/s +Step 23760/44000 | Loss: 2.9320 | LR: 2.25e-04 | 2148.4ms/step | 9.3 samples/s +Step 23770/44000 | Loss: 2.9350 | LR: 2.25e-04 | 2143.8ms/step | 9.3 samples/s +Step 23780/44000 | Loss: 2.9217 | LR: 2.25e-04 | 2144.4ms/step | 9.3 samples/s +Step 23790/44000 | Loss: 2.9694 | LR: 2.24e-04 | 2137.0ms/step | 9.4 samples/s +Step 23800/44000 | Loss: 2.9168 | LR: 2.24e-04 | 2140.5ms/step | 9.3 samples/s +Step 23810/44000 | Loss: 2.9713 | LR: 2.24e-04 | 2140.7ms/step | 9.3 samples/s +Step 23820/44000 | Loss: 2.8834 | LR: 2.24e-04 | 2158.1ms/step | 9.3 samples/s +Step 23830/44000 | Loss: 2.8985 | LR: 2.24e-04 | 2118.2ms/step | 9.4 samples/s +Step 23840/44000 | Loss: 2.9170 | LR: 2.23e-04 | 2131.1ms/step | 9.4 samples/s +Step 23850/44000 | Loss: 2.9342 | LR: 2.23e-04 | 2149.1ms/step | 9.3 samples/s +Step 23860/44000 | Loss: 2.8986 | LR: 2.23e-04 | 2183.6ms/step | 9.2 samples/s +Step 23870/44000 | Loss: 2.9260 | LR: 2.23e-04 | 2160.3ms/step | 9.3 samples/s +Step 23880/44000 | Loss: 2.9373 | LR: 2.23e-04 | 2159.1ms/step | 9.3 samples/s +Step 23890/44000 | Loss: 2.9066 | LR: 2.23e-04 | 2026.2ms/step | 9.9 samples/s +Step 23900/44000 | Loss: 2.9259 | LR: 2.22e-04 | 2158.0ms/step | 9.3 samples/s +Step 23910/44000 | Loss: 2.9482 | LR: 2.22e-04 | 2147.1ms/step | 9.3 samples/s +Step 23920/44000 | Loss: 2.9229 | LR: 2.22e-04 | 2133.0ms/step | 9.4 samples/s +Step 23930/44000 | Loss: 2.9735 | LR: 2.22e-04 | 2139.6ms/step | 9.3 samples/s +Step 23940/44000 | Loss: 2.9165 | LR: 2.22e-04 | 2155.9ms/step | 9.3 samples/s +Step 23950/44000 | Loss: 2.9348 | LR: 2.21e-04 | 2144.4ms/step | 9.3 samples/s +Step 23960/44000 | Loss: 2.9110 | LR: 2.21e-04 | 2167.3ms/step | 9.2 samples/s +Step 23970/44000 | Loss: 2.9627 | LR: 2.21e-04 | 2158.9ms/step | 9.3 samples/s +Step 23980/44000 | Loss: 2.9338 | LR: 2.21e-04 | 2150.8ms/step | 9.3 samples/s +Step 23990/44000 | Loss: 2.8952 | LR: 2.21e-04 | 2156.5ms/step | 9.3 samples/s +Step 24000/44000 | Loss: 2.8998 | LR: 2.21e-04 | 2132.5ms/step | 9.4 samples/s +Validation at step 24000: Loss = 2.9198, Perplexity = 18.54 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-24000 +Step 24010/44000 | Loss: 2.9314 | LR: 2.20e-04 | 4353.7ms/step | 4.6 samples/s +Step 24020/44000 | Loss: 2.9486 | LR: 2.20e-04 | 2142.3ms/step | 9.3 samples/s +Step 24030/44000 | Loss: 2.9248 | LR: 2.20e-04 | 2152.6ms/step | 9.3 samples/s +Step 24040/44000 | Loss: 2.8900 | LR: 2.20e-04 | 2171.2ms/step | 9.2 samples/s +Step 24050/44000 | Loss: 2.9211 | LR: 2.20e-04 | 2158.0ms/step | 9.3 samples/s +Step 24060/44000 | Loss: 2.9161 | LR: 2.19e-04 | 2196.7ms/step | 9.1 samples/s +Step 24070/44000 | Loss: 2.9419 | LR: 2.19e-04 | 2185.3ms/step | 9.2 samples/s +Step 24080/44000 | Loss: 2.9439 | LR: 2.19e-04 | 2157.0ms/step | 9.3 samples/s +Step 24090/44000 | Loss: 2.9013 | LR: 2.19e-04 | 2174.2ms/step | 9.2 samples/s +Step 24100/44000 | Loss: 2.9168 | LR: 2.19e-04 | 2141.9ms/step | 9.3 samples/s +Step 24110/44000 | Loss: 2.9497 | LR: 2.19e-04 | 2147.3ms/step | 9.3 samples/s +Step 24120/44000 | Loss: 2.9121 | LR: 2.18e-04 | 2181.1ms/step | 9.2 samples/s +Step 24130/44000 | Loss: 2.9376 | LR: 2.18e-04 | 2201.5ms/step | 9.1 samples/s +Step 24140/44000 | Loss: 2.9174 | LR: 2.18e-04 | 2170.7ms/step | 9.2 samples/s +Step 24150/44000 | Loss: 2.9082 | LR: 2.18e-04 | 2163.6ms/step | 9.2 samples/s +Step 24160/44000 | Loss: 2.9495 | LR: 2.18e-04 | 2147.5ms/step | 9.3 samples/s +Step 24170/44000 | Loss: 2.9047 | LR: 2.17e-04 | 2136.3ms/step | 9.4 samples/s +Step 24180/44000 | Loss: 2.9040 | LR: 2.17e-04 | 2139.7ms/step | 9.3 samples/s +Step 24190/44000 | Loss: 2.9334 | LR: 2.17e-04 | 2141.3ms/step | 9.3 samples/s +Step 24200/44000 | Loss: 2.9184 | LR: 2.17e-04 | 2142.6ms/step | 9.3 samples/s +Step 24210/44000 | Loss: 2.8850 | LR: 2.17e-04 | 2202.6ms/step | 9.1 samples/s +Step 24220/44000 | Loss: 2.9603 | LR: 2.17e-04 | 2196.5ms/step | 9.1 samples/s +Step 24230/44000 | Loss: 2.9629 | LR: 2.16e-04 | 2184.8ms/step | 9.2 samples/s +Step 24240/44000 | Loss: 2.9357 | LR: 2.16e-04 | 2166.2ms/step | 9.2 samples/s +Step 24250/44000 | Loss: 2.9133 | LR: 2.16e-04 | 2138.7ms/step | 9.4 samples/s +Step 24260/44000 | Loss: 2.9291 | LR: 2.16e-04 | 2127.5ms/step | 9.4 samples/s +Step 24270/44000 | Loss: 2.8885 | LR: 2.16e-04 | 2142.5ms/step | 9.3 samples/s +Step 24280/44000 | Loss: 2.8813 | LR: 2.16e-04 | 2161.3ms/step | 9.3 samples/s +Step 24290/44000 | Loss: 2.9221 | LR: 2.15e-04 | 2159.7ms/step | 9.3 samples/s +Step 24300/44000 | Loss: 2.9206 | LR: 2.15e-04 | 2146.2ms/step | 9.3 samples/s +Step 24310/44000 | Loss: 2.9353 | LR: 2.15e-04 | 2151.1ms/step | 9.3 samples/s +Step 24320/44000 | Loss: 2.8887 | LR: 2.15e-04 | 2175.2ms/step | 9.2 samples/s +Step 24330/44000 | Loss: 2.9541 | LR: 2.15e-04 | 2165.5ms/step | 9.2 samples/s +Step 24340/44000 | Loss: 2.9024 | LR: 2.14e-04 | 2141.6ms/step | 9.3 samples/s +Step 24350/44000 | Loss: 2.9433 | LR: 2.14e-04 | 2152.9ms/step | 9.3 samples/s +Step 24360/44000 | Loss: 2.9364 | LR: 2.14e-04 | 2156.5ms/step | 9.3 samples/s +Step 24370/44000 | Loss: 2.9501 | LR: 2.14e-04 | 2184.1ms/step | 9.2 samples/s +Step 24380/44000 | Loss: 2.9321 | LR: 2.14e-04 | 2148.0ms/step | 9.3 samples/s +Step 24390/44000 | Loss: 2.9262 | LR: 2.14e-04 | 2148.0ms/step | 9.3 samples/s +Step 24400/44000 | Loss: 2.9115 | LR: 2.13e-04 | 2150.8ms/step | 9.3 samples/s +Step 24410/44000 | Loss: 2.9243 | LR: 2.13e-04 | 2154.2ms/step | 9.3 samples/s +Step 24420/44000 | Loss: 2.9450 | LR: 2.13e-04 | 2154.3ms/step | 9.3 samples/s +Step 24430/44000 | Loss: 2.9146 | LR: 2.13e-04 | 2147.0ms/step | 9.3 samples/s +Step 24440/44000 | Loss: 2.9504 | LR: 2.13e-04 | 2171.3ms/step | 9.2 samples/s +Step 24450/44000 | Loss: 2.8925 | LR: 2.12e-04 | 2151.1ms/step | 9.3 samples/s +Step 24460/44000 | Loss: 2.8940 | LR: 2.12e-04 | 2167.6ms/step | 9.2 samples/s +Step 24470/44000 | Loss: 2.9341 | LR: 2.12e-04 | 2112.8ms/step | 9.5 samples/s +Step 24480/44000 | Loss: 2.9137 | LR: 2.12e-04 | 2141.2ms/step | 9.3 samples/s +Step 24490/44000 | Loss: 2.8783 | LR: 2.12e-04 | 2159.4ms/step | 9.3 samples/s +Step 24500/44000 | Loss: 2.9488 | LR: 2.12e-04 | 2148.8ms/step | 9.3 samples/s +Step 24510/44000 | Loss: 2.9493 | LR: 2.11e-04 | 2153.3ms/step | 9.3 samples/s +Step 24520/44000 | Loss: 2.9198 | LR: 2.11e-04 | 2179.2ms/step | 9.2 samples/s +Step 24530/44000 | Loss: 2.9118 | LR: 2.11e-04 | 2161.4ms/step | 9.3 samples/s +Step 24540/44000 | Loss: 2.9559 | LR: 2.11e-04 | 2167.4ms/step | 9.2 samples/s +Step 24550/44000 | Loss: 2.9260 | LR: 2.11e-04 | 2155.4ms/step | 9.3 samples/s +Step 24560/44000 | Loss: 2.8966 | LR: 2.10e-04 | 2128.0ms/step | 9.4 samples/s +Step 24570/44000 | Loss: 2.9619 | LR: 2.10e-04 | 2129.3ms/step | 9.4 samples/s +Step 24580/44000 | Loss: 2.8933 | LR: 2.10e-04 | 2179.3ms/step | 9.2 samples/s +Step 24590/44000 | Loss: 2.9521 | LR: 2.10e-04 | 2139.9ms/step | 9.3 samples/s +Step 24600/44000 | Loss: 2.8786 | LR: 2.10e-04 | 2148.2ms/step | 9.3 samples/s +Step 24610/44000 | Loss: 2.9325 | LR: 2.10e-04 | 2118.7ms/step | 9.4 samples/s +Step 24620/44000 | Loss: 2.8949 | LR: 2.09e-04 | 2124.1ms/step | 9.4 samples/s +Step 24630/44000 | Loss: 2.8963 | LR: 2.09e-04 | 2132.2ms/step | 9.4 samples/s +Step 24640/44000 | Loss: 2.9442 | LR: 2.09e-04 | 2144.2ms/step | 9.3 samples/s +Step 24650/44000 | Loss: 2.9507 | LR: 2.09e-04 | 2149.8ms/step | 9.3 samples/s +Step 24660/44000 | Loss: 2.9194 | LR: 2.09e-04 | 2138.6ms/step | 9.4 samples/s +Step 24670/44000 | Loss: 2.9285 | LR: 2.09e-04 | 2149.8ms/step | 9.3 samples/s +Step 24680/44000 | Loss: 2.9505 | LR: 2.08e-04 | 2162.2ms/step | 9.2 samples/s +Step 24690/44000 | Loss: 2.9251 | LR: 2.08e-04 | 2196.4ms/step | 9.1 samples/s +Step 24700/44000 | Loss: 2.9326 | LR: 2.08e-04 | 2171.7ms/step | 9.2 samples/s +Step 24710/44000 | Loss: 2.9535 | LR: 2.08e-04 | 2143.2ms/step | 9.3 samples/s +Step 24720/44000 | Loss: 2.8925 | LR: 2.08e-04 | 2144.3ms/step | 9.3 samples/s +Step 24730/44000 | Loss: 2.9367 | LR: 2.07e-04 | 2165.4ms/step | 9.2 samples/s +Step 24740/44000 | Loss: 2.8904 | LR: 2.07e-04 | 2173.4ms/step | 9.2 samples/s +Step 24750/44000 | Loss: 2.9213 | LR: 2.07e-04 | 2155.8ms/step | 9.3 samples/s +Step 24760/44000 | Loss: 2.9236 | LR: 2.07e-04 | 2196.7ms/step | 9.1 samples/s +Step 24770/44000 | Loss: 2.9274 | LR: 2.07e-04 | 2166.5ms/step | 9.2 samples/s +Step 24780/44000 | Loss: 2.9180 | LR: 2.07e-04 | 2178.0ms/step | 9.2 samples/s +Step 24790/44000 | Loss: 2.9196 | LR: 2.06e-04 | 2146.2ms/step | 9.3 samples/s +Step 24800/44000 | Loss: 2.9051 | LR: 2.06e-04 | 2174.9ms/step | 9.2 samples/s +Step 24810/44000 | Loss: 2.8801 | LR: 2.06e-04 | 2179.9ms/step | 9.2 samples/s +Step 24820/44000 | Loss: 2.9006 | LR: 2.06e-04 | 2147.4ms/step | 9.3 samples/s +Step 24830/44000 | Loss: 2.8987 | LR: 2.06e-04 | 2205.0ms/step | 9.1 samples/s +Step 24840/44000 | Loss: 2.9507 | LR: 2.05e-04 | 2177.1ms/step | 9.2 samples/s +Step 24850/44000 | Loss: 2.9009 | LR: 2.05e-04 | 2140.7ms/step | 9.3 samples/s +Step 24860/44000 | Loss: 2.9195 | LR: 2.05e-04 | 2220.9ms/step | 9.0 samples/s +Step 24870/44000 | Loss: 2.8948 | LR: 2.05e-04 | 2144.1ms/step | 9.3 samples/s +Step 24880/44000 | Loss: 2.9309 | LR: 2.05e-04 | 2116.5ms/step | 9.4 samples/s +Step 24890/44000 | Loss: 2.9153 | LR: 2.05e-04 | 2038.1ms/step | 9.8 samples/s +Step 24900/44000 | Loss: 2.9232 | LR: 2.04e-04 | 2144.0ms/step | 9.3 samples/s +Step 24910/44000 | Loss: 2.9127 | LR: 2.04e-04 | 2169.4ms/step | 9.2 samples/s +Step 24920/44000 | Loss: 2.9313 | LR: 2.04e-04 | 2160.2ms/step | 9.3 samples/s +Step 24930/44000 | Loss: 2.9257 | LR: 2.04e-04 | 2174.7ms/step | 9.2 samples/s +Step 24940/44000 | Loss: 2.8861 | LR: 2.04e-04 | 2135.4ms/step | 9.4 samples/s +Step 24950/44000 | Loss: 2.9043 | LR: 2.04e-04 | 2147.7ms/step | 9.3 samples/s +Step 24960/44000 | Loss: 2.8927 | LR: 2.03e-04 | 2151.6ms/step | 9.3 samples/s +Step 24970/44000 | Loss: 2.9275 | LR: 2.03e-04 | 2148.0ms/step | 9.3 samples/s +Step 24980/44000 | Loss: 2.9318 | LR: 2.03e-04 | 2189.6ms/step | 9.1 samples/s +Step 24990/44000 | Loss: 2.9330 | LR: 2.03e-04 | 2138.7ms/step | 9.4 samples/s +Step 25000/44000 | Loss: 2.9044 | LR: 2.03e-04 | 2143.7ms/step | 9.3 samples/s +Validation at step 25000: Loss = 2.9129, Perplexity = 18.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-25000 +Step 25010/44000 | Loss: 2.9092 | LR: 2.02e-04 | 4426.1ms/step | 4.5 samples/s +Step 25020/44000 | Loss: 2.9486 | LR: 2.02e-04 | 2152.8ms/step | 9.3 samples/s +Step 25030/44000 | Loss: 2.9800 | LR: 2.02e-04 | 2177.3ms/step | 9.2 samples/s +Step 25040/44000 | Loss: 2.9225 | LR: 2.02e-04 | 2173.9ms/step | 9.2 samples/s +Step 25050/44000 | Loss: 2.9195 | LR: 2.02e-04 | 2154.0ms/step | 9.3 samples/s +Step 25060/44000 | Loss: 2.9469 | LR: 2.02e-04 | 2184.8ms/step | 9.2 samples/s +Step 25070/44000 | Loss: 2.9185 | LR: 2.01e-04 | 2138.2ms/step | 9.4 samples/s +Step 25080/44000 | Loss: 2.9206 | LR: 2.01e-04 | 2138.3ms/step | 9.4 samples/s +Step 25090/44000 | Loss: 2.9592 | LR: 2.01e-04 | 2153.5ms/step | 9.3 samples/s +Step 25100/44000 | Loss: 2.9089 | LR: 2.01e-04 | 2141.1ms/step | 9.3 samples/s +Step 25110/44000 | Loss: 2.9323 | LR: 2.01e-04 | 2153.6ms/step | 9.3 samples/s +Step 25120/44000 | Loss: 2.9084 | LR: 2.00e-04 | 2174.0ms/step | 9.2 samples/s +Step 25130/44000 | Loss: 2.9477 | LR: 2.00e-04 | 2168.7ms/step | 9.2 samples/s +Step 25140/44000 | Loss: 2.9256 | LR: 2.00e-04 | 2203.9ms/step | 9.1 samples/s +Step 25150/44000 | Loss: 2.8990 | LR: 2.00e-04 | 2174.7ms/step | 9.2 samples/s +Step 25160/44000 | Loss: 2.9125 | LR: 2.00e-04 | 2144.2ms/step | 9.3 samples/s +Step 25170/44000 | Loss: 2.8802 | LR: 2.00e-04 | 2174.5ms/step | 9.2 samples/s +Step 25180/44000 | Loss: 2.8898 | LR: 1.99e-04 | 2170.2ms/step | 9.2 samples/s +Step 25190/44000 | Loss: 2.9108 | LR: 1.99e-04 | 2156.7ms/step | 9.3 samples/s +Step 25200/44000 | Loss: 2.9340 | LR: 1.99e-04 | 2139.7ms/step | 9.3 samples/s +Step 25210/44000 | Loss: 2.9303 | LR: 1.99e-04 | 2159.1ms/step | 9.3 samples/s +Step 25220/44000 | Loss: 2.9240 | LR: 1.99e-04 | 2222.9ms/step | 9.0 samples/s +Step 25230/44000 | Loss: 2.9664 | LR: 1.99e-04 | 2153.0ms/step | 9.3 samples/s +Step 25240/44000 | Loss: 2.9449 | LR: 1.98e-04 | 2152.9ms/step | 9.3 samples/s +Step 25250/44000 | Loss: 2.9269 | LR: 1.98e-04 | 2143.2ms/step | 9.3 samples/s +Step 25260/44000 | Loss: 2.9221 | LR: 1.98e-04 | 2158.6ms/step | 9.3 samples/s +Step 25270/44000 | Loss: 2.9190 | LR: 1.98e-04 | 2209.7ms/step | 9.1 samples/s +Step 25280/44000 | Loss: 2.9359 | LR: 1.98e-04 | 2165.6ms/step | 9.2 samples/s +Step 25290/44000 | Loss: 2.9794 | LR: 1.97e-04 | 2152.7ms/step | 9.3 samples/s +Step 25300/44000 | Loss: 2.9412 | LR: 1.97e-04 | 2155.0ms/step | 9.3 samples/s +Step 25310/44000 | Loss: 2.9099 | LR: 1.97e-04 | 2162.1ms/step | 9.3 samples/s +Step 25320/44000 | Loss: 2.9186 | LR: 1.97e-04 | 2154.2ms/step | 9.3 samples/s +Step 25330/44000 | Loss: 2.8844 | LR: 1.97e-04 | 2195.5ms/step | 9.1 samples/s +Step 25340/44000 | Loss: 2.8683 | LR: 1.97e-04 | 2200.4ms/step | 9.1 samples/s +Step 25350/44000 | Loss: 2.9826 | LR: 1.96e-04 | 2190.6ms/step | 9.1 samples/s +Step 25360/44000 | Loss: 2.9515 | LR: 1.96e-04 | 2121.8ms/step | 9.4 samples/s +Step 25370/44000 | Loss: 2.9475 | LR: 1.96e-04 | 2162.6ms/step | 9.2 samples/s +Step 25380/44000 | Loss: 2.9049 | LR: 1.96e-04 | 2132.0ms/step | 9.4 samples/s +Step 25390/44000 | Loss: 2.9585 | LR: 1.96e-04 | 2178.0ms/step | 9.2 samples/s +Step 25400/44000 | Loss: 2.9321 | LR: 1.96e-04 | 2148.4ms/step | 9.3 samples/s +Step 25410/44000 | Loss: 2.9408 | LR: 1.95e-04 | 2154.6ms/step | 9.3 samples/s +Step 25420/44000 | Loss: 2.9175 | LR: 1.95e-04 | 2149.9ms/step | 9.3 samples/s +Step 25430/44000 | Loss: 2.9315 | LR: 1.95e-04 | 2153.7ms/step | 9.3 samples/s +Step 25440/44000 | Loss: 2.8928 | LR: 1.95e-04 | 2164.0ms/step | 9.2 samples/s +Step 25450/44000 | Loss: 2.9405 | LR: 1.95e-04 | 2135.2ms/step | 9.4 samples/s +Step 25460/44000 | Loss: 2.9019 | LR: 1.94e-04 | 2141.4ms/step | 9.3 samples/s +Step 25470/44000 | Loss: 2.9121 | LR: 1.94e-04 | 2161.5ms/step | 9.3 samples/s +Step 25480/44000 | Loss: 2.8921 | LR: 1.94e-04 | 2156.6ms/step | 9.3 samples/s +Step 25490/44000 | Loss: 2.8880 | LR: 1.94e-04 | 2147.8ms/step | 9.3 samples/s +Step 25500/44000 | Loss: 2.9300 | LR: 1.94e-04 | 2183.7ms/step | 9.2 samples/s +Step 25510/44000 | Loss: 2.9010 | LR: 1.94e-04 | 2316.6ms/step | 8.6 samples/s +Step 25520/44000 | Loss: 2.9212 | LR: 1.93e-04 | 2197.1ms/step | 9.1 samples/s +Step 25530/44000 | Loss: 2.8876 | LR: 1.93e-04 | 2151.5ms/step | 9.3 samples/s +Step 25540/44000 | Loss: 2.9186 | LR: 1.93e-04 | 2199.0ms/step | 9.1 samples/s +Step 25550/44000 | Loss: 2.9147 | LR: 1.93e-04 | 2170.0ms/step | 9.2 samples/s +Step 25560/44000 | Loss: 2.9734 | LR: 1.93e-04 | 2157.0ms/step | 9.3 samples/s +Step 25570/44000 | Loss: 2.9317 | LR: 1.92e-04 | 2142.6ms/step | 9.3 samples/s +Step 25580/44000 | Loss: 2.9055 | LR: 1.92e-04 | 2147.5ms/step | 9.3 samples/s +Step 25590/44000 | Loss: 2.9267 | LR: 1.92e-04 | 2176.0ms/step | 9.2 samples/s +Step 25600/44000 | Loss: 2.9457 | LR: 1.92e-04 | 2163.3ms/step | 9.2 samples/s +Step 25610/44000 | Loss: 2.8856 | LR: 1.92e-04 | 2136.4ms/step | 9.4 samples/s +Step 25620/44000 | Loss: 2.9363 | LR: 1.92e-04 | 2148.1ms/step | 9.3 samples/s +Step 25630/44000 | Loss: 2.8995 | LR: 1.91e-04 | 2195.0ms/step | 9.1 samples/s +Step 25640/44000 | Loss: 2.9088 | LR: 1.91e-04 | 2151.4ms/step | 9.3 samples/s +Step 25650/44000 | Loss: 2.8973 | LR: 1.91e-04 | 2166.2ms/step | 9.2 samples/s +Step 25660/44000 | Loss: 2.9664 | LR: 1.91e-04 | 2164.4ms/step | 9.2 samples/s +Step 25670/44000 | Loss: 2.9271 | LR: 1.91e-04 | 2144.1ms/step | 9.3 samples/s +Step 25680/44000 | Loss: 2.8841 | LR: 1.91e-04 | 2141.1ms/step | 9.3 samples/s +Step 25690/44000 | Loss: 2.9559 | LR: 1.90e-04 | 2178.8ms/step | 9.2 samples/s +Step 25700/44000 | Loss: 2.9482 | LR: 1.90e-04 | 2216.4ms/step | 9.0 samples/s +Step 25710/44000 | Loss: 2.8926 | LR: 1.90e-04 | 2160.9ms/step | 9.3 samples/s +Step 25720/44000 | Loss: 2.9382 | LR: 1.90e-04 | 2172.3ms/step | 9.2 samples/s +Step 25730/44000 | Loss: 2.9113 | LR: 1.90e-04 | 2159.4ms/step | 9.3 samples/s +Step 25740/44000 | Loss: 2.9269 | LR: 1.89e-04 | 2182.3ms/step | 9.2 samples/s +Step 25750/44000 | Loss: 2.9229 | LR: 1.89e-04 | 2155.9ms/step | 9.3 samples/s +Step 25760/44000 | Loss: 2.9209 | LR: 1.89e-04 | 2131.1ms/step | 9.4 samples/s +Step 25770/44000 | Loss: 2.9503 | LR: 1.89e-04 | 2155.0ms/step | 9.3 samples/s +Step 25780/44000 | Loss: 2.8892 | LR: 1.89e-04 | 2153.2ms/step | 9.3 samples/s +Step 25790/44000 | Loss: 2.9200 | LR: 1.89e-04 | 2148.2ms/step | 9.3 samples/s +Step 25800/44000 | Loss: 2.8867 | LR: 1.88e-04 | 2155.2ms/step | 9.3 samples/s +Step 25810/44000 | Loss: 2.9567 | LR: 1.88e-04 | 2158.4ms/step | 9.3 samples/s +Step 25820/44000 | Loss: 2.9688 | LR: 1.88e-04 | 2191.7ms/step | 9.1 samples/s +Step 25830/44000 | Loss: 2.8931 | LR: 1.88e-04 | 2181.3ms/step | 9.2 samples/s +Step 25840/44000 | Loss: 2.9401 | LR: 1.88e-04 | 2155.8ms/step | 9.3 samples/s +Step 25850/44000 | Loss: 2.8723 | LR: 1.88e-04 | 2151.7ms/step | 9.3 samples/s +Step 25860/44000 | Loss: 2.9265 | LR: 1.87e-04 | 2167.2ms/step | 9.2 samples/s +Step 25870/44000 | Loss: 2.9160 | LR: 1.87e-04 | 2168.6ms/step | 9.2 samples/s +Step 25880/44000 | Loss: 2.9268 | LR: 1.87e-04 | 2124.5ms/step | 9.4 samples/s +Step 25890/44000 | Loss: 2.9031 | LR: 1.87e-04 | 2077.5ms/step | 9.6 samples/s +Step 25900/44000 | Loss: 2.9216 | LR: 1.87e-04 | 2213.4ms/step | 9.0 samples/s +Step 25910/44000 | Loss: 2.9231 | LR: 1.87e-04 | 2146.3ms/step | 9.3 samples/s +Step 25920/44000 | Loss: 2.8881 | LR: 1.86e-04 | 2151.3ms/step | 9.3 samples/s +Step 25930/44000 | Loss: 2.9016 | LR: 1.86e-04 | 2149.0ms/step | 9.3 samples/s +Step 25940/44000 | Loss: 2.9405 | LR: 1.86e-04 | 2164.1ms/step | 9.2 samples/s +Step 25950/44000 | Loss: 2.9172 | LR: 1.86e-04 | 2150.4ms/step | 9.3 samples/s +Step 25960/44000 | Loss: 2.9306 | LR: 1.86e-04 | 2127.7ms/step | 9.4 samples/s +Step 25970/44000 | Loss: 2.9257 | LR: 1.85e-04 | 2205.4ms/step | 9.1 samples/s +Step 25980/44000 | Loss: 2.9114 | LR: 1.85e-04 | 2153.9ms/step | 9.3 samples/s +Step 25990/44000 | Loss: 2.9021 | LR: 1.85e-04 | 2206.1ms/step | 9.1 samples/s +Step 26000/44000 | Loss: 2.9231 | LR: 1.85e-04 | 2201.9ms/step | 9.1 samples/s +Validation at step 26000: Loss = 2.9057, Perplexity = 18.28 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-26000 +Step 26010/44000 | Loss: 2.9226 | LR: 1.85e-04 | 4447.5ms/step | 4.5 samples/s +Step 26020/44000 | Loss: 2.9117 | LR: 1.85e-04 | 2141.8ms/step | 9.3 samples/s +Step 26030/44000 | Loss: 2.9242 | LR: 1.84e-04 | 2179.9ms/step | 9.2 samples/s +Step 26040/44000 | Loss: 2.9295 | LR: 1.84e-04 | 2138.2ms/step | 9.4 samples/s +Step 26050/44000 | Loss: 2.9477 | LR: 1.84e-04 | 2184.1ms/step | 9.2 samples/s +Step 26060/44000 | Loss: 2.9402 | LR: 1.84e-04 | 2153.1ms/step | 9.3 samples/s +Step 26070/44000 | Loss: 2.8946 | LR: 1.84e-04 | 2164.9ms/step | 9.2 samples/s +Step 26080/44000 | Loss: 2.9293 | LR: 1.84e-04 | 2172.8ms/step | 9.2 samples/s +Step 26090/44000 | Loss: 2.9271 | LR: 1.83e-04 | 2168.6ms/step | 9.2 samples/s +Step 26100/44000 | Loss: 2.9345 | LR: 1.83e-04 | 2145.4ms/step | 9.3 samples/s +Step 26110/44000 | Loss: 2.8726 | LR: 1.83e-04 | 2164.1ms/step | 9.2 samples/s +Step 26120/44000 | Loss: 2.9228 | LR: 1.83e-04 | 2148.7ms/step | 9.3 samples/s +Step 26130/44000 | Loss: 2.8746 | LR: 1.83e-04 | 2164.1ms/step | 9.2 samples/s +Step 26140/44000 | Loss: 2.9070 | LR: 1.82e-04 | 2173.7ms/step | 9.2 samples/s +Step 26150/44000 | Loss: 2.9352 | LR: 1.82e-04 | 2132.4ms/step | 9.4 samples/s +Step 26160/44000 | Loss: 2.9065 | LR: 1.82e-04 | 2188.5ms/step | 9.1 samples/s +Step 26170/44000 | Loss: 2.9398 | LR: 1.82e-04 | 2168.4ms/step | 9.2 samples/s +Step 26180/44000 | Loss: 2.9306 | LR: 1.82e-04 | 2142.3ms/step | 9.3 samples/s +Step 26190/44000 | Loss: 2.9500 | LR: 1.82e-04 | 2128.3ms/step | 9.4 samples/s +Step 26200/44000 | Loss: 2.8915 | LR: 1.81e-04 | 2154.8ms/step | 9.3 samples/s +Step 26210/44000 | Loss: 2.8904 | LR: 1.81e-04 | 2131.6ms/step | 9.4 samples/s +Step 26220/44000 | Loss: 2.8889 | LR: 1.81e-04 | 2155.3ms/step | 9.3 samples/s +Step 26230/44000 | Loss: 2.9265 | LR: 1.81e-04 | 2144.3ms/step | 9.3 samples/s +Step 26240/44000 | Loss: 2.8832 | LR: 1.81e-04 | 2157.4ms/step | 9.3 samples/s +Step 26250/44000 | Loss: 2.9215 | LR: 1.81e-04 | 2204.8ms/step | 9.1 samples/s +Step 26260/44000 | Loss: 2.9306 | LR: 1.80e-04 | 2172.8ms/step | 9.2 samples/s +Step 26270/44000 | Loss: 2.9248 | LR: 1.80e-04 | 2155.9ms/step | 9.3 samples/s +Step 26280/44000 | Loss: 2.9222 | LR: 1.80e-04 | 2166.4ms/step | 9.2 samples/s +Step 26290/44000 | Loss: 2.9019 | LR: 1.80e-04 | 2157.1ms/step | 9.3 samples/s +Step 26300/44000 | Loss: 2.9070 | LR: 1.80e-04 | 2151.1ms/step | 9.3 samples/s +Step 26310/44000 | Loss: 2.8996 | LR: 1.80e-04 | 2194.5ms/step | 9.1 samples/s +Step 26320/44000 | Loss: 2.8918 | LR: 1.79e-04 | 2167.6ms/step | 9.2 samples/s +Step 26330/44000 | Loss: 2.9183 | LR: 1.79e-04 | 2147.0ms/step | 9.3 samples/s +Step 26340/44000 | Loss: 2.9080 | LR: 1.79e-04 | 2195.0ms/step | 9.1 samples/s +Step 26350/44000 | Loss: 2.9134 | LR: 1.79e-04 | 2147.7ms/step | 9.3 samples/s +Step 26360/44000 | Loss: 2.9255 | LR: 1.79e-04 | 2157.8ms/step | 9.3 samples/s +Step 26370/44000 | Loss: 2.9498 | LR: 1.78e-04 | 2158.0ms/step | 9.3 samples/s +Step 26380/44000 | Loss: 2.9001 | LR: 1.78e-04 | 2178.2ms/step | 9.2 samples/s +Step 26390/44000 | Loss: 2.9140 | LR: 1.78e-04 | 2166.1ms/step | 9.2 samples/s +Step 26400/44000 | Loss: 2.9436 | LR: 1.78e-04 | 2160.3ms/step | 9.3 samples/s +Step 26410/44000 | Loss: 2.9062 | LR: 1.78e-04 | 2155.2ms/step | 9.3 samples/s +Step 26420/44000 | Loss: 2.9037 | LR: 1.78e-04 | 2183.0ms/step | 9.2 samples/s +Step 26430/44000 | Loss: 2.9530 | LR: 1.77e-04 | 2191.6ms/step | 9.1 samples/s +Step 26440/44000 | Loss: 2.8901 | LR: 1.77e-04 | 2182.8ms/step | 9.2 samples/s +Step 26450/44000 | Loss: 2.8800 | LR: 1.77e-04 | 2172.3ms/step | 9.2 samples/s +Step 26460/44000 | Loss: 2.9246 | LR: 1.77e-04 | 2147.3ms/step | 9.3 samples/s +Step 26470/44000 | Loss: 2.8913 | LR: 1.77e-04 | 2189.4ms/step | 9.1 samples/s +Step 26480/44000 | Loss: 2.9156 | LR: 1.77e-04 | 2159.5ms/step | 9.3 samples/s +Step 26490/44000 | Loss: 2.9207 | LR: 1.76e-04 | 2130.3ms/step | 9.4 samples/s +Step 26500/44000 | Loss: 2.8467 | LR: 1.76e-04 | 2136.8ms/step | 9.4 samples/s +Step 26510/44000 | Loss: 2.9237 | LR: 1.76e-04 | 2157.1ms/step | 9.3 samples/s +Step 26520/44000 | Loss: 2.8769 | LR: 1.76e-04 | 2165.1ms/step | 9.2 samples/s +Step 26530/44000 | Loss: 2.9416 | LR: 1.76e-04 | 2152.0ms/step | 9.3 samples/s +Step 26540/44000 | Loss: 2.9013 | LR: 1.76e-04 | 2174.8ms/step | 9.2 samples/s +Step 26550/44000 | Loss: 2.9017 | LR: 1.75e-04 | 2168.5ms/step | 9.2 samples/s +Step 26560/44000 | Loss: 2.9069 | LR: 1.75e-04 | 2152.5ms/step | 9.3 samples/s +Step 26570/44000 | Loss: 2.9053 | LR: 1.75e-04 | 2153.8ms/step | 9.3 samples/s +Step 26580/44000 | Loss: 2.8894 | LR: 1.75e-04 | 2183.0ms/step | 9.2 samples/s +Step 26590/44000 | Loss: 2.9335 | LR: 1.75e-04 | 2157.9ms/step | 9.3 samples/s +Step 26600/44000 | Loss: 2.9295 | LR: 1.74e-04 | 2199.4ms/step | 9.1 samples/s +Step 26610/44000 | Loss: 2.9304 | LR: 1.74e-04 | 2137.0ms/step | 9.4 samples/s +Step 26620/44000 | Loss: 2.8927 | LR: 1.74e-04 | 2209.8ms/step | 9.1 samples/s +Step 26630/44000 | Loss: 2.9256 | LR: 1.74e-04 | 2188.9ms/step | 9.1 samples/s +Step 26640/44000 | Loss: 2.8820 | LR: 1.74e-04 | 2140.5ms/step | 9.3 samples/s +Step 26650/44000 | Loss: 2.9082 | LR: 1.74e-04 | 2153.7ms/step | 9.3 samples/s +Step 26660/44000 | Loss: 2.8846 | LR: 1.73e-04 | 2153.8ms/step | 9.3 samples/s +Step 26670/44000 | Loss: 2.9346 | LR: 1.73e-04 | 2152.6ms/step | 9.3 samples/s +Step 26680/44000 | Loss: 2.9082 | LR: 1.73e-04 | 2146.3ms/step | 9.3 samples/s +Step 26690/44000 | Loss: 2.9419 | LR: 1.73e-04 | 2144.1ms/step | 9.3 samples/s +Step 26700/44000 | Loss: 2.8999 | LR: 1.73e-04 | 2148.0ms/step | 9.3 samples/s +Step 26710/44000 | Loss: 2.9192 | LR: 1.73e-04 | 2149.3ms/step | 9.3 samples/s +Step 26720/44000 | Loss: 2.9392 | LR: 1.72e-04 | 2148.1ms/step | 9.3 samples/s +Step 26730/44000 | Loss: 2.9285 | LR: 1.72e-04 | 2152.2ms/step | 9.3 samples/s +Step 26740/44000 | Loss: 2.9368 | LR: 1.72e-04 | 2156.9ms/step | 9.3 samples/s +Step 26750/44000 | Loss: 2.8799 | LR: 1.72e-04 | 2145.6ms/step | 9.3 samples/s +Step 26760/44000 | Loss: 2.8978 | LR: 1.72e-04 | 2179.9ms/step | 9.2 samples/s +Step 26770/44000 | Loss: 2.8736 | LR: 1.72e-04 | 2131.8ms/step | 9.4 samples/s +Step 26780/44000 | Loss: 2.9081 | LR: 1.71e-04 | 2174.4ms/step | 9.2 samples/s +Step 26790/44000 | Loss: 2.9235 | LR: 1.71e-04 | 2143.9ms/step | 9.3 samples/s +Step 26800/44000 | Loss: 2.8482 | LR: 1.71e-04 | 2154.1ms/step | 9.3 samples/s +Step 26810/44000 | Loss: 2.9290 | LR: 1.71e-04 | 2157.0ms/step | 9.3 samples/s +Step 26820/44000 | Loss: 2.9115 | LR: 1.71e-04 | 2188.2ms/step | 9.1 samples/s +Step 26830/44000 | Loss: 2.9005 | LR: 1.71e-04 | 2158.2ms/step | 9.3 samples/s +Step 26840/44000 | Loss: 2.8918 | LR: 1.70e-04 | 2163.3ms/step | 9.2 samples/s +Step 26850/44000 | Loss: 2.9345 | LR: 1.70e-04 | 2152.9ms/step | 9.3 samples/s +Step 26860/44000 | Loss: 2.8591 | LR: 1.70e-04 | 2179.5ms/step | 9.2 samples/s +Step 26870/44000 | Loss: 2.9240 | LR: 1.70e-04 | 2159.5ms/step | 9.3 samples/s +Step 26880/44000 | Loss: 2.9324 | LR: 1.70e-04 | 2048.1ms/step | 9.8 samples/s +Step 26890/44000 | Loss: 2.9462 | LR: 1.69e-04 | 2138.1ms/step | 9.4 samples/s +Step 26900/44000 | Loss: 2.8822 | LR: 1.69e-04 | 2142.4ms/step | 9.3 samples/s +Step 26910/44000 | Loss: 2.9031 | LR: 1.69e-04 | 2164.8ms/step | 9.2 samples/s +Step 26920/44000 | Loss: 2.9438 | LR: 1.69e-04 | 2159.9ms/step | 9.3 samples/s +Step 26930/44000 | Loss: 2.8943 | LR: 1.69e-04 | 2163.7ms/step | 9.2 samples/s +Step 26940/44000 | Loss: 2.8965 | LR: 1.69e-04 | 2212.9ms/step | 9.0 samples/s +Step 26950/44000 | Loss: 2.9416 | LR: 1.68e-04 | 2158.2ms/step | 9.3 samples/s +Step 26960/44000 | Loss: 2.9247 | LR: 1.68e-04 | 2185.2ms/step | 9.2 samples/s +Step 26970/44000 | Loss: 2.9195 | LR: 1.68e-04 | 2164.3ms/step | 9.2 samples/s +Step 26980/44000 | Loss: 2.9208 | LR: 1.68e-04 | 2161.2ms/step | 9.3 samples/s +Step 26990/44000 | Loss: 2.9451 | LR: 1.68e-04 | 2167.2ms/step | 9.2 samples/s +Step 27000/44000 | Loss: 2.8545 | LR: 1.68e-04 | 2197.3ms/step | 9.1 samples/s +Validation at step 27000: Loss = 2.8998, Perplexity = 18.17 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-27000 +Step 27010/44000 | Loss: 2.9503 | LR: 1.67e-04 | 4445.9ms/step | 4.5 samples/s +Step 27020/44000 | Loss: 2.9136 | LR: 1.67e-04 | 2158.9ms/step | 9.3 samples/s +Step 27030/44000 | Loss: 2.8988 | LR: 1.67e-04 | 2153.8ms/step | 9.3 samples/s +Step 27040/44000 | Loss: 2.9250 | LR: 1.67e-04 | 2171.3ms/step | 9.2 samples/s +Step 27050/44000 | Loss: 2.9296 | LR: 1.67e-04 | 2159.8ms/step | 9.3 samples/s +Step 27060/44000 | Loss: 2.9271 | LR: 1.67e-04 | 2162.0ms/step | 9.3 samples/s +Step 27070/44000 | Loss: 2.8905 | LR: 1.66e-04 | 2174.9ms/step | 9.2 samples/s +Step 27080/44000 | Loss: 2.8792 | LR: 1.66e-04 | 2179.3ms/step | 9.2 samples/s +Step 27090/44000 | Loss: 2.8961 | LR: 1.66e-04 | 2190.6ms/step | 9.1 samples/s +Step 27100/44000 | Loss: 2.8915 | LR: 1.66e-04 | 2147.3ms/step | 9.3 samples/s +Step 27110/44000 | Loss: 2.9147 | LR: 1.66e-04 | 2150.8ms/step | 9.3 samples/s +Step 27120/44000 | Loss: 2.9160 | LR: 1.66e-04 | 2175.1ms/step | 9.2 samples/s +Step 27130/44000 | Loss: 2.8715 | LR: 1.65e-04 | 2141.7ms/step | 9.3 samples/s +Step 27140/44000 | Loss: 2.8851 | LR: 1.65e-04 | 2143.4ms/step | 9.3 samples/s +Step 27150/44000 | Loss: 2.9053 | LR: 1.65e-04 | 2166.2ms/step | 9.2 samples/s +Step 27160/44000 | Loss: 2.9480 | LR: 1.65e-04 | 2170.4ms/step | 9.2 samples/s +Step 27170/44000 | Loss: 2.8815 | LR: 1.65e-04 | 2181.5ms/step | 9.2 samples/s +Step 27180/44000 | Loss: 2.9264 | LR: 1.65e-04 | 2190.8ms/step | 9.1 samples/s +Step 27190/44000 | Loss: 2.8801 | LR: 1.64e-04 | 2179.2ms/step | 9.2 samples/s +Step 27200/44000 | Loss: 2.8751 | LR: 1.64e-04 | 2142.2ms/step | 9.3 samples/s +Step 27210/44000 | Loss: 2.9583 | LR: 1.64e-04 | 2160.0ms/step | 9.3 samples/s +Step 27220/44000 | Loss: 2.9136 | LR: 1.64e-04 | 2154.4ms/step | 9.3 samples/s +Step 27230/44000 | Loss: 2.9161 | LR: 1.64e-04 | 2169.6ms/step | 9.2 samples/s +Step 27240/44000 | Loss: 2.8955 | LR: 1.63e-04 | 2153.9ms/step | 9.3 samples/s +Step 27250/44000 | Loss: 2.8856 | LR: 1.63e-04 | 2170.6ms/step | 9.2 samples/s +Step 27260/44000 | Loss: 2.9125 | LR: 1.63e-04 | 2188.8ms/step | 9.1 samples/s +Step 27270/44000 | Loss: 2.8843 | LR: 1.63e-04 | 2153.2ms/step | 9.3 samples/s +Step 27280/44000 | Loss: 2.9195 | LR: 1.63e-04 | 2143.6ms/step | 9.3 samples/s +Step 27290/44000 | Loss: 2.9097 | LR: 1.63e-04 | 2147.7ms/step | 9.3 samples/s +Step 27300/44000 | Loss: 2.8844 | LR: 1.62e-04 | 2142.2ms/step | 9.3 samples/s +Step 27310/44000 | Loss: 2.8965 | LR: 1.62e-04 | 2151.8ms/step | 9.3 samples/s +Step 27320/44000 | Loss: 2.8729 | LR: 1.62e-04 | 2152.9ms/step | 9.3 samples/s +Step 27330/44000 | Loss: 2.9053 | LR: 1.62e-04 | 2148.3ms/step | 9.3 samples/s +Step 27340/44000 | Loss: 2.9082 | LR: 1.62e-04 | 2153.7ms/step | 9.3 samples/s +Step 27350/44000 | Loss: 2.9142 | LR: 1.62e-04 | 2165.3ms/step | 9.2 samples/s +Step 27360/44000 | Loss: 2.9416 | LR: 1.61e-04 | 2167.5ms/step | 9.2 samples/s +Step 27370/44000 | Loss: 2.8943 | LR: 1.61e-04 | 2221.3ms/step | 9.0 samples/s +Step 27380/44000 | Loss: 2.9634 | LR: 1.61e-04 | 2161.7ms/step | 9.3 samples/s +Step 27390/44000 | Loss: 2.9134 | LR: 1.61e-04 | 2250.2ms/step | 8.9 samples/s +Step 27400/44000 | Loss: 2.9271 | LR: 1.61e-04 | 2221.3ms/step | 9.0 samples/s +Step 27410/44000 | Loss: 2.8790 | LR: 1.61e-04 | 2285.7ms/step | 8.8 samples/s +Step 27420/44000 | Loss: 2.8865 | LR: 1.60e-04 | 2216.2ms/step | 9.0 samples/s +Step 27430/44000 | Loss: 2.8991 | LR: 1.60e-04 | 2240.6ms/step | 8.9 samples/s +Step 27440/44000 | Loss: 2.9200 | LR: 1.60e-04 | 2277.0ms/step | 8.8 samples/s +Step 27450/44000 | Loss: 2.8931 | LR: 1.60e-04 | 2245.3ms/step | 8.9 samples/s +Step 27460/44000 | Loss: 2.9308 | LR: 1.60e-04 | 2276.9ms/step | 8.8 samples/s +Step 27470/44000 | Loss: 2.9431 | LR: 1.60e-04 | 2281.0ms/step | 8.8 samples/s +Step 27480/44000 | Loss: 2.8939 | LR: 1.59e-04 | 2306.0ms/step | 8.7 samples/s +Step 27490/44000 | Loss: 2.9153 | LR: 1.59e-04 | 2271.1ms/step | 8.8 samples/s +Step 27500/44000 | Loss: 2.9426 | LR: 1.59e-04 | 2215.4ms/step | 9.0 samples/s +Step 27510/44000 | Loss: 2.8980 | LR: 1.59e-04 | 2207.8ms/step | 9.1 samples/s +Step 27520/44000 | Loss: 2.9066 | LR: 1.59e-04 | 2285.4ms/step | 8.8 samples/s +Step 27530/44000 | Loss: 2.9269 | LR: 1.59e-04 | 2237.7ms/step | 8.9 samples/s +Step 27540/44000 | Loss: 2.8929 | LR: 1.58e-04 | 2266.8ms/step | 8.8 samples/s +Step 27550/44000 | Loss: 2.8922 | LR: 1.58e-04 | 2238.3ms/step | 8.9 samples/s +Step 27560/44000 | Loss: 2.8367 | LR: 1.58e-04 | 2235.2ms/step | 8.9 samples/s +Step 27570/44000 | Loss: 2.9395 | LR: 1.58e-04 | 2252.5ms/step | 8.9 samples/s +Step 27580/44000 | Loss: 2.8838 | LR: 1.58e-04 | 2278.5ms/step | 8.8 samples/s +Step 27590/44000 | Loss: 2.9171 | LR: 1.58e-04 | 2310.3ms/step | 8.7 samples/s +Step 27600/44000 | Loss: 2.8893 | LR: 1.57e-04 | 2312.7ms/step | 8.6 samples/s +Step 27610/44000 | Loss: 2.9036 | LR: 1.57e-04 | 2276.8ms/step | 8.8 samples/s +Step 27620/44000 | Loss: 2.8978 | LR: 1.57e-04 | 2268.6ms/step | 8.8 samples/s +Step 27630/44000 | Loss: 2.9113 | LR: 1.57e-04 | 2259.8ms/step | 8.9 samples/s +Step 27640/44000 | Loss: 2.9140 | LR: 1.57e-04 | 2257.2ms/step | 8.9 samples/s +Step 27650/44000 | Loss: 2.8784 | LR: 1.57e-04 | 2234.9ms/step | 8.9 samples/s +Step 27660/44000 | Loss: 2.8939 | LR: 1.56e-04 | 2271.8ms/step | 8.8 samples/s +Step 27670/44000 | Loss: 2.9393 | LR: 1.56e-04 | 2227.3ms/step | 9.0 samples/s +Step 27680/44000 | Loss: 2.8897 | LR: 1.56e-04 | 2212.3ms/step | 9.0 samples/s +Step 27690/44000 | Loss: 2.8773 | LR: 1.56e-04 | 2245.3ms/step | 8.9 samples/s +Step 27700/44000 | Loss: 2.9018 | LR: 1.56e-04 | 2262.4ms/step | 8.8 samples/s +Step 27710/44000 | Loss: 2.9004 | LR: 1.56e-04 | 2227.8ms/step | 9.0 samples/s +Step 27720/44000 | Loss: 2.9400 | LR: 1.55e-04 | 2249.6ms/step | 8.9 samples/s +Step 27730/44000 | Loss: 2.8958 | LR: 1.55e-04 | 2238.1ms/step | 8.9 samples/s +Step 27740/44000 | Loss: 2.8937 | LR: 1.55e-04 | 2216.9ms/step | 9.0 samples/s +Step 27750/44000 | Loss: 2.9128 | LR: 1.55e-04 | 2258.7ms/step | 8.9 samples/s +Step 27760/44000 | Loss: 2.9121 | LR: 1.55e-04 | 2247.0ms/step | 8.9 samples/s +Step 27770/44000 | Loss: 2.9283 | LR: 1.55e-04 | 2331.6ms/step | 8.6 samples/s +Step 27780/44000 | Loss: 2.8581 | LR: 1.54e-04 | 2259.7ms/step | 8.9 samples/s +Step 27790/44000 | Loss: 2.8798 | LR: 1.54e-04 | 2269.4ms/step | 8.8 samples/s +Step 27800/44000 | Loss: 2.8715 | LR: 1.54e-04 | 2229.7ms/step | 9.0 samples/s +Step 27810/44000 | Loss: 2.9278 | LR: 1.54e-04 | 2232.5ms/step | 9.0 samples/s +Step 27820/44000 | Loss: 2.9001 | LR: 1.54e-04 | 2253.1ms/step | 8.9 samples/s +Step 27830/44000 | Loss: 2.9392 | LR: 1.54e-04 | 2242.4ms/step | 8.9 samples/s +Step 27840/44000 | Loss: 2.9143 | LR: 1.53e-04 | 2231.0ms/step | 9.0 samples/s +Step 27850/44000 | Loss: 2.9371 | LR: 1.53e-04 | 2267.5ms/step | 8.8 samples/s +Step 27860/44000 | Loss: 2.8683 | LR: 1.53e-04 | 2243.9ms/step | 8.9 samples/s +Step 27870/44000 | Loss: 2.8754 | LR: 1.53e-04 | 2196.1ms/step | 9.1 samples/s +Step 27880/44000 | Loss: 2.8946 | LR: 1.53e-04 | 2094.2ms/step | 9.6 samples/s +Step 27890/44000 | Loss: 2.9109 | LR: 1.53e-04 | 2195.5ms/step | 9.1 samples/s +Step 27900/44000 | Loss: 2.8957 | LR: 1.52e-04 | 2155.4ms/step | 9.3 samples/s +Step 27910/44000 | Loss: 2.9128 | LR: 1.52e-04 | 2148.1ms/step | 9.3 samples/s +Step 27920/44000 | Loss: 2.8744 | LR: 1.52e-04 | 2157.2ms/step | 9.3 samples/s +Step 27930/44000 | Loss: 2.9277 | LR: 1.52e-04 | 2191.7ms/step | 9.1 samples/s +Step 27940/44000 | Loss: 2.9413 | LR: 1.52e-04 | 2168.9ms/step | 9.2 samples/s +Step 27950/44000 | Loss: 2.9284 | LR: 1.52e-04 | 2138.4ms/step | 9.4 samples/s +Step 27960/44000 | Loss: 2.9307 | LR: 1.51e-04 | 2166.2ms/step | 9.2 samples/s +Step 27970/44000 | Loss: 2.9115 | LR: 1.51e-04 | 2173.4ms/step | 9.2 samples/s +Step 27980/44000 | Loss: 2.9113 | LR: 1.51e-04 | 2155.1ms/step | 9.3 samples/s +Step 27990/44000 | Loss: 2.9211 | LR: 1.51e-04 | 2163.2ms/step | 9.2 samples/s +Step 28000/44000 | Loss: 2.8657 | LR: 1.51e-04 | 2170.1ms/step | 9.2 samples/s +Validation at step 28000: Loss = 2.8952, Perplexity = 18.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-28000 +Step 28010/44000 | Loss: 2.9185 | LR: 1.51e-04 | 4445.6ms/step | 4.5 samples/s +Step 28020/44000 | Loss: 2.8981 | LR: 1.50e-04 | 2111.5ms/step | 9.5 samples/s +Step 28030/44000 | Loss: 2.9271 | LR: 1.50e-04 | 2155.0ms/step | 9.3 samples/s +Step 28040/44000 | Loss: 2.8851 | LR: 1.50e-04 | 2170.0ms/step | 9.2 samples/s +Step 28050/44000 | Loss: 2.9328 | LR: 1.50e-04 | 2151.7ms/step | 9.3 samples/s +Step 28060/44000 | Loss: 2.9188 | LR: 1.50e-04 | 2139.3ms/step | 9.3 samples/s +Step 28070/44000 | Loss: 2.8903 | LR: 1.50e-04 | 2157.1ms/step | 9.3 samples/s +Step 28080/44000 | Loss: 2.9189 | LR: 1.49e-04 | 2132.2ms/step | 9.4 samples/s +Step 28090/44000 | Loss: 2.8905 | LR: 1.49e-04 | 2133.6ms/step | 9.4 samples/s +Step 28100/44000 | Loss: 2.9172 | LR: 1.49e-04 | 2150.8ms/step | 9.3 samples/s +Step 28110/44000 | Loss: 2.8890 | LR: 1.49e-04 | 2146.2ms/step | 9.3 samples/s +Step 28120/44000 | Loss: 2.9119 | LR: 1.49e-04 | 2166.7ms/step | 9.2 samples/s +Step 28130/44000 | Loss: 2.9187 | LR: 1.49e-04 | 2180.0ms/step | 9.2 samples/s +Step 28140/44000 | Loss: 2.8975 | LR: 1.48e-04 | 2128.9ms/step | 9.4 samples/s +Step 28150/44000 | Loss: 2.9083 | LR: 1.48e-04 | 2154.1ms/step | 9.3 samples/s +Step 28160/44000 | Loss: 2.8910 | LR: 1.48e-04 | 2127.6ms/step | 9.4 samples/s +Step 28170/44000 | Loss: 2.9113 | LR: 1.48e-04 | 2132.2ms/step | 9.4 samples/s +Step 28180/44000 | Loss: 2.9093 | LR: 1.48e-04 | 2143.6ms/step | 9.3 samples/s +Step 28190/44000 | Loss: 2.9144 | LR: 1.48e-04 | 2142.2ms/step | 9.3 samples/s +Step 28200/44000 | Loss: 2.9347 | LR: 1.47e-04 | 2148.5ms/step | 9.3 samples/s +Step 28210/44000 | Loss: 2.9144 | LR: 1.47e-04 | 2135.6ms/step | 9.4 samples/s +Step 28220/44000 | Loss: 2.9223 | LR: 1.47e-04 | 2156.8ms/step | 9.3 samples/s +Step 28230/44000 | Loss: 2.9112 | LR: 1.47e-04 | 2149.8ms/step | 9.3 samples/s +Step 28240/44000 | Loss: 2.9005 | LR: 1.47e-04 | 2146.2ms/step | 9.3 samples/s +Step 28250/44000 | Loss: 2.9130 | LR: 1.47e-04 | 2157.4ms/step | 9.3 samples/s +Step 28260/44000 | Loss: 2.8976 | LR: 1.46e-04 | 2138.5ms/step | 9.4 samples/s +Step 28270/44000 | Loss: 2.8844 | LR: 1.46e-04 | 2152.1ms/step | 9.3 samples/s +Step 28280/44000 | Loss: 2.8910 | LR: 1.46e-04 | 2181.7ms/step | 9.2 samples/s +Step 28290/44000 | Loss: 2.9113 | LR: 1.46e-04 | 2156.4ms/step | 9.3 samples/s +Step 28300/44000 | Loss: 2.8848 | LR: 1.46e-04 | 2119.9ms/step | 9.4 samples/s +Step 28310/44000 | Loss: 2.8926 | LR: 1.46e-04 | 2160.3ms/step | 9.3 samples/s +Step 28320/44000 | Loss: 2.9141 | LR: 1.45e-04 | 2157.0ms/step | 9.3 samples/s +Step 28330/44000 | Loss: 2.8963 | LR: 1.45e-04 | 2131.6ms/step | 9.4 samples/s +Step 28340/44000 | Loss: 2.8820 | LR: 1.45e-04 | 2139.2ms/step | 9.3 samples/s +Step 28350/44000 | Loss: 2.9144 | LR: 1.45e-04 | 2171.5ms/step | 9.2 samples/s +Step 28360/44000 | Loss: 2.9434 | LR: 1.45e-04 | 2155.9ms/step | 9.3 samples/s +Step 28370/44000 | Loss: 2.8116 | LR: 1.45e-04 | 2153.6ms/step | 9.3 samples/s +Step 28380/44000 | Loss: 2.8956 | LR: 1.44e-04 | 2147.6ms/step | 9.3 samples/s +Step 28390/44000 | Loss: 2.9361 | LR: 1.44e-04 | 2155.5ms/step | 9.3 samples/s +Step 28400/44000 | Loss: 2.9100 | LR: 1.44e-04 | 2173.8ms/step | 9.2 samples/s +Step 28410/44000 | Loss: 2.9214 | LR: 1.44e-04 | 2151.5ms/step | 9.3 samples/s +Step 28420/44000 | Loss: 2.8677 | LR: 1.44e-04 | 2135.2ms/step | 9.4 samples/s +Step 28430/44000 | Loss: 2.8831 | LR: 1.44e-04 | 2130.5ms/step | 9.4 samples/s +Step 28440/44000 | Loss: 2.9030 | LR: 1.43e-04 | 2143.8ms/step | 9.3 samples/s +Step 28450/44000 | Loss: 2.9295 | LR: 1.43e-04 | 2171.7ms/step | 9.2 samples/s +Step 28460/44000 | Loss: 2.9119 | LR: 1.43e-04 | 2177.3ms/step | 9.2 samples/s +Step 28470/44000 | Loss: 2.8993 | LR: 1.43e-04 | 2179.2ms/step | 9.2 samples/s +Step 28480/44000 | Loss: 2.9061 | LR: 1.43e-04 | 2158.6ms/step | 9.3 samples/s +Step 28490/44000 | Loss: 2.8965 | LR: 1.43e-04 | 2135.3ms/step | 9.4 samples/s +Step 28500/44000 | Loss: 2.9052 | LR: 1.42e-04 | 2156.2ms/step | 9.3 samples/s +Step 28510/44000 | Loss: 2.9144 | LR: 1.42e-04 | 2148.2ms/step | 9.3 samples/s +Step 28520/44000 | Loss: 2.9127 | LR: 1.42e-04 | 2161.8ms/step | 9.3 samples/s +Step 28530/44000 | Loss: 2.8309 | LR: 1.42e-04 | 2156.8ms/step | 9.3 samples/s +Step 28540/44000 | Loss: 2.9245 | LR: 1.42e-04 | 2148.1ms/step | 9.3 samples/s +Step 28550/44000 | Loss: 2.9262 | LR: 1.42e-04 | 2151.5ms/step | 9.3 samples/s +Step 28560/44000 | Loss: 2.9457 | LR: 1.41e-04 | 2142.0ms/step | 9.3 samples/s +Step 28570/44000 | Loss: 2.9160 | LR: 1.41e-04 | 2165.8ms/step | 9.2 samples/s +Step 28580/44000 | Loss: 2.9175 | LR: 1.41e-04 | 2158.8ms/step | 9.3 samples/s +Step 28590/44000 | Loss: 2.9165 | LR: 1.41e-04 | 2160.8ms/step | 9.3 samples/s +Step 28600/44000 | Loss: 2.8955 | LR: 1.41e-04 | 2153.9ms/step | 9.3 samples/s +Step 28610/44000 | Loss: 2.8887 | LR: 1.41e-04 | 2145.7ms/step | 9.3 samples/s +Step 28620/44000 | Loss: 2.8951 | LR: 1.40e-04 | 2146.0ms/step | 9.3 samples/s +Step 28630/44000 | Loss: 2.8918 | LR: 1.40e-04 | 2141.2ms/step | 9.3 samples/s +Step 28640/44000 | Loss: 2.8794 | LR: 1.40e-04 | 2172.1ms/step | 9.2 samples/s +Step 28650/44000 | Loss: 2.8574 | LR: 1.40e-04 | 2159.0ms/step | 9.3 samples/s +Step 28660/44000 | Loss: 2.9490 | LR: 1.40e-04 | 2174.3ms/step | 9.2 samples/s +Step 28670/44000 | Loss: 2.8998 | LR: 1.40e-04 | 2161.1ms/step | 9.3 samples/s +Step 28680/44000 | Loss: 2.9044 | LR: 1.39e-04 | 2128.9ms/step | 9.4 samples/s +Step 28690/44000 | Loss: 2.9173 | LR: 1.39e-04 | 2152.5ms/step | 9.3 samples/s +Step 28700/44000 | Loss: 2.9100 | LR: 1.39e-04 | 2142.9ms/step | 9.3 samples/s +Step 28710/44000 | Loss: 2.8952 | LR: 1.39e-04 | 2165.8ms/step | 9.2 samples/s +Step 28720/44000 | Loss: 2.8923 | LR: 1.39e-04 | 2147.5ms/step | 9.3 samples/s +Step 28730/44000 | Loss: 2.8776 | LR: 1.39e-04 | 2145.6ms/step | 9.3 samples/s +Step 28740/44000 | Loss: 2.9396 | LR: 1.38e-04 | 2158.4ms/step | 9.3 samples/s +Step 28750/44000 | Loss: 2.8890 | LR: 1.38e-04 | 2155.7ms/step | 9.3 samples/s +Step 28760/44000 | Loss: 2.8810 | LR: 1.38e-04 | 2152.1ms/step | 9.3 samples/s +Step 28770/44000 | Loss: 2.8755 | LR: 1.38e-04 | 2155.1ms/step | 9.3 samples/s +Step 28780/44000 | Loss: 2.9309 | LR: 1.38e-04 | 2146.5ms/step | 9.3 samples/s +Step 28790/44000 | Loss: 2.8795 | LR: 1.38e-04 | 2158.5ms/step | 9.3 samples/s +Step 28800/44000 | Loss: 2.8885 | LR: 1.38e-04 | 2136.0ms/step | 9.4 samples/s +Step 28810/44000 | Loss: 2.9525 | LR: 1.37e-04 | 2164.0ms/step | 9.2 samples/s +Step 28820/44000 | Loss: 2.9184 | LR: 1.37e-04 | 2230.7ms/step | 9.0 samples/s +Step 28830/44000 | Loss: 2.8975 | LR: 1.37e-04 | 2278.3ms/step | 8.8 samples/s +Step 28840/44000 | Loss: 2.9170 | LR: 1.37e-04 | 2205.7ms/step | 9.1 samples/s +Step 28850/44000 | Loss: 2.8897 | LR: 1.37e-04 | 2294.8ms/step | 8.7 samples/s +Step 28860/44000 | Loss: 2.8805 | LR: 1.37e-04 | 2319.9ms/step | 8.6 samples/s +Step 28870/44000 | Loss: 2.8967 | LR: 1.36e-04 | 2197.3ms/step | 9.1 samples/s +Step 28880/44000 | Loss: 2.8882 | LR: 1.36e-04 | 2180.4ms/step | 9.2 samples/s +Step 28890/44000 | Loss: 2.8748 | LR: 1.36e-04 | 2249.8ms/step | 8.9 samples/s +Step 28900/44000 | Loss: 2.9196 | LR: 1.36e-04 | 2274.0ms/step | 8.8 samples/s +Step 28910/44000 | Loss: 2.9023 | LR: 1.36e-04 | 2290.2ms/step | 8.7 samples/s +Step 28920/44000 | Loss: 2.9162 | LR: 1.36e-04 | 2172.1ms/step | 9.2 samples/s +Step 28930/44000 | Loss: 2.8876 | LR: 1.35e-04 | 2252.0ms/step | 8.9 samples/s +Step 28940/44000 | Loss: 2.9294 | LR: 1.35e-04 | 2231.1ms/step | 9.0 samples/s +Step 28950/44000 | Loss: 2.9248 | LR: 1.35e-04 | 2241.3ms/step | 8.9 samples/s +Step 28960/44000 | Loss: 2.8890 | LR: 1.35e-04 | 2247.2ms/step | 8.9 samples/s +Step 28970/44000 | Loss: 2.9059 | LR: 1.35e-04 | 2293.9ms/step | 8.7 samples/s +Step 28980/44000 | Loss: 2.9091 | LR: 1.35e-04 | 2137.4ms/step | 9.4 samples/s +Step 28990/44000 | Loss: 2.9180 | LR: 1.34e-04 | 2139.7ms/step | 9.3 samples/s +Step 29000/44000 | Loss: 2.8819 | LR: 1.34e-04 | 2166.1ms/step | 9.2 samples/s +Validation at step 29000: Loss = 2.8897, Perplexity = 17.99 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-29000 +Step 29010/44000 | Loss: 2.9011 | LR: 1.34e-04 | 4397.1ms/step | 4.5 samples/s +Step 29020/44000 | Loss: 2.8768 | LR: 1.34e-04 | 2172.0ms/step | 9.2 samples/s +Step 29030/44000 | Loss: 2.8921 | LR: 1.34e-04 | 2152.6ms/step | 9.3 samples/s +Step 29040/44000 | Loss: 2.9297 | LR: 1.34e-04 | 2163.5ms/step | 9.2 samples/s +Step 29050/44000 | Loss: 2.8854 | LR: 1.33e-04 | 2131.9ms/step | 9.4 samples/s +Step 29060/44000 | Loss: 2.9234 | LR: 1.33e-04 | 2154.3ms/step | 9.3 samples/s +Step 29070/44000 | Loss: 2.8796 | LR: 1.33e-04 | 2161.4ms/step | 9.3 samples/s +Step 29080/44000 | Loss: 2.9159 | LR: 1.33e-04 | 2173.4ms/step | 9.2 samples/s +Step 29090/44000 | Loss: 2.9442 | LR: 1.33e-04 | 2140.9ms/step | 9.3 samples/s +Step 29100/44000 | Loss: 2.9002 | LR: 1.33e-04 | 2135.5ms/step | 9.4 samples/s +Step 29110/44000 | Loss: 2.9163 | LR: 1.33e-04 | 2143.7ms/step | 9.3 samples/s +Step 29120/44000 | Loss: 2.8907 | LR: 1.32e-04 | 2149.3ms/step | 9.3 samples/s +Step 29130/44000 | Loss: 2.9060 | LR: 1.32e-04 | 2141.7ms/step | 9.3 samples/s +Step 29140/44000 | Loss: 2.8800 | LR: 1.32e-04 | 2145.0ms/step | 9.3 samples/s +Step 29150/44000 | Loss: 2.8978 | LR: 1.32e-04 | 2148.4ms/step | 9.3 samples/s +Step 29160/44000 | Loss: 2.8777 | LR: 1.32e-04 | 2164.7ms/step | 9.2 samples/s +Step 29170/44000 | Loss: 2.9121 | LR: 1.32e-04 | 2138.8ms/step | 9.4 samples/s +Step 29180/44000 | Loss: 2.9286 | LR: 1.31e-04 | 2163.0ms/step | 9.2 samples/s +Step 29190/44000 | Loss: 2.8762 | LR: 1.31e-04 | 2138.4ms/step | 9.4 samples/s +Step 29200/44000 | Loss: 2.8986 | LR: 1.31e-04 | 2138.0ms/step | 9.4 samples/s +Step 29210/44000 | Loss: 2.8952 | LR: 1.31e-04 | 2143.3ms/step | 9.3 samples/s +Step 29220/44000 | Loss: 2.8743 | LR: 1.31e-04 | 2167.6ms/step | 9.2 samples/s +Step 29230/44000 | Loss: 2.9259 | LR: 1.31e-04 | 2165.1ms/step | 9.2 samples/s +Step 29240/44000 | Loss: 2.9279 | LR: 1.30e-04 | 2146.6ms/step | 9.3 samples/s +Step 29250/44000 | Loss: 2.9437 | LR: 1.30e-04 | 2127.1ms/step | 9.4 samples/s +Step 29260/44000 | Loss: 2.8689 | LR: 1.30e-04 | 2149.2ms/step | 9.3 samples/s +Step 29270/44000 | Loss: 2.9021 | LR: 1.30e-04 | 2160.7ms/step | 9.3 samples/s +Step 29280/44000 | Loss: 2.8688 | LR: 1.30e-04 | 2159.1ms/step | 9.3 samples/s +Step 29290/44000 | Loss: 2.9090 | LR: 1.30e-04 | 2153.7ms/step | 9.3 samples/s +Step 29300/44000 | Loss: 2.9375 | LR: 1.29e-04 | 2145.9ms/step | 9.3 samples/s +Step 29310/44000 | Loss: 2.8860 | LR: 1.29e-04 | 2140.1ms/step | 9.3 samples/s +Step 29320/44000 | Loss: 2.8590 | LR: 1.29e-04 | 2128.8ms/step | 9.4 samples/s +Step 29330/44000 | Loss: 2.9065 | LR: 1.29e-04 | 2152.7ms/step | 9.3 samples/s +Step 29340/44000 | Loss: 2.8408 | LR: 1.29e-04 | 2161.3ms/step | 9.3 samples/s +Step 29350/44000 | Loss: 2.8946 | LR: 1.29e-04 | 2144.1ms/step | 9.3 samples/s +Step 29360/44000 | Loss: 2.9248 | LR: 1.29e-04 | 2153.5ms/step | 9.3 samples/s +Step 29370/44000 | Loss: 2.9321 | LR: 1.28e-04 | 2141.8ms/step | 9.3 samples/s +Step 29380/44000 | Loss: 2.8875 | LR: 1.28e-04 | 2138.5ms/step | 9.4 samples/s +Step 29390/44000 | Loss: 2.8764 | LR: 1.28e-04 | 2146.3ms/step | 9.3 samples/s +Step 29400/44000 | Loss: 2.9297 | LR: 1.28e-04 | 2285.5ms/step | 8.8 samples/s +Step 29410/44000 | Loss: 2.9087 | LR: 1.28e-04 | 2216.1ms/step | 9.0 samples/s +Step 29420/44000 | Loss: 2.8785 | LR: 1.28e-04 | 2302.7ms/step | 8.7 samples/s +Step 29430/44000 | Loss: 2.8989 | LR: 1.27e-04 | 2224.8ms/step | 9.0 samples/s +Step 29440/44000 | Loss: 2.9352 | LR: 1.27e-04 | 2278.9ms/step | 8.8 samples/s +Step 29450/44000 | Loss: 2.8809 | LR: 1.27e-04 | 2346.6ms/step | 8.5 samples/s +Step 29460/44000 | Loss: 2.8712 | LR: 1.27e-04 | 2173.2ms/step | 9.2 samples/s +Step 29470/44000 | Loss: 2.9218 | LR: 1.27e-04 | 2315.7ms/step | 8.6 samples/s +Step 29480/44000 | Loss: 2.8663 | LR: 1.27e-04 | 2192.3ms/step | 9.1 samples/s +Step 29490/44000 | Loss: 2.9218 | LR: 1.26e-04 | 2271.5ms/step | 8.8 samples/s +Step 29500/44000 | Loss: 2.9653 | LR: 1.26e-04 | 2226.5ms/step | 9.0 samples/s +Step 29510/44000 | Loss: 2.8910 | LR: 1.26e-04 | 2273.6ms/step | 8.8 samples/s +Step 29520/44000 | Loss: 2.8939 | LR: 1.26e-04 | 2251.0ms/step | 8.9 samples/s +Step 29530/44000 | Loss: 2.8871 | LR: 1.26e-04 | 2259.4ms/step | 8.9 samples/s +Step 29540/44000 | Loss: 2.9139 | LR: 1.26e-04 | 2299.5ms/step | 8.7 samples/s +Step 29550/44000 | Loss: 2.8824 | LR: 1.26e-04 | 2196.0ms/step | 9.1 samples/s +Step 29560/44000 | Loss: 2.8956 | LR: 1.25e-04 | 2285.1ms/step | 8.8 samples/s +Step 29570/44000 | Loss: 2.8940 | LR: 1.25e-04 | 2140.1ms/step | 9.3 samples/s +Step 29580/44000 | Loss: 2.8777 | LR: 1.25e-04 | 2166.7ms/step | 9.2 samples/s +Step 29590/44000 | Loss: 2.8941 | LR: 1.25e-04 | 2153.9ms/step | 9.3 samples/s +Step 29600/44000 | Loss: 2.9325 | LR: 1.25e-04 | 2142.2ms/step | 9.3 samples/s +Step 29610/44000 | Loss: 2.9089 | LR: 1.25e-04 | 2136.0ms/step | 9.4 samples/s +Step 29620/44000 | Loss: 2.8967 | LR: 1.24e-04 | 2154.1ms/step | 9.3 samples/s +Step 29630/44000 | Loss: 2.8826 | LR: 1.24e-04 | 2128.6ms/step | 9.4 samples/s +Step 29640/44000 | Loss: 2.8718 | LR: 1.24e-04 | 2150.4ms/step | 9.3 samples/s +Step 29650/44000 | Loss: 2.8975 | LR: 1.24e-04 | 2174.4ms/step | 9.2 samples/s +Step 29660/44000 | Loss: 2.9486 | LR: 1.24e-04 | 2132.7ms/step | 9.4 samples/s +Step 29670/44000 | Loss: 2.8720 | LR: 1.24e-04 | 2142.5ms/step | 9.3 samples/s +Step 29680/44000 | Loss: 2.9231 | LR: 1.23e-04 | 2163.6ms/step | 9.2 samples/s +Step 29690/44000 | Loss: 2.9028 | LR: 1.23e-04 | 2152.0ms/step | 9.3 samples/s +Step 29700/44000 | Loss: 2.9338 | LR: 1.23e-04 | 2113.3ms/step | 9.5 samples/s +Step 29710/44000 | Loss: 2.8694 | LR: 1.23e-04 | 2098.5ms/step | 9.5 samples/s +Step 29720/44000 | Loss: 2.8901 | LR: 1.23e-04 | 2088.3ms/step | 9.6 samples/s +Step 29730/44000 | Loss: 2.8781 | LR: 1.23e-04 | 2083.8ms/step | 9.6 samples/s +Step 29740/44000 | Loss: 2.8614 | LR: 1.23e-04 | 2085.8ms/step | 9.6 samples/s +Step 29750/44000 | Loss: 2.9148 | LR: 1.22e-04 | 2114.1ms/step | 9.5 samples/s +Step 29760/44000 | Loss: 2.8339 | LR: 1.22e-04 | 2091.8ms/step | 9.6 samples/s +Step 29770/44000 | Loss: 2.9153 | LR: 1.22e-04 | 2106.2ms/step | 9.5 samples/s +Step 29780/44000 | Loss: 2.9065 | LR: 1.22e-04 | 2105.7ms/step | 9.5 samples/s +Step 29790/44000 | Loss: 2.8938 | LR: 1.22e-04 | 2112.7ms/step | 9.5 samples/s +Step 29800/44000 | Loss: 2.9007 | LR: 1.22e-04 | 2119.9ms/step | 9.4 samples/s +Step 29810/44000 | Loss: 2.9433 | LR: 1.21e-04 | 2111.7ms/step | 9.5 samples/s +Step 29820/44000 | Loss: 2.8601 | LR: 1.21e-04 | 2069.8ms/step | 9.7 samples/s +Step 29830/44000 | Loss: 2.9105 | LR: 1.21e-04 | 2115.4ms/step | 9.5 samples/s +Step 29840/44000 | Loss: 2.8917 | LR: 1.21e-04 | 2070.1ms/step | 9.7 samples/s +Step 29850/44000 | Loss: 2.8589 | LR: 1.21e-04 | 2103.0ms/step | 9.5 samples/s +Step 29860/44000 | Loss: 2.8680 | LR: 1.21e-04 | 2107.4ms/step | 9.5 samples/s +Step 29870/44000 | Loss: 2.8715 | LR: 1.21e-04 | 2077.3ms/step | 9.6 samples/s +Step 29880/44000 | Loss: 2.9041 | LR: 1.20e-04 | 1971.8ms/step | 10.1 samples/s +Step 29890/44000 | Loss: 2.9131 | LR: 1.20e-04 | 2014.9ms/step | 9.9 samples/s +Step 29900/44000 | Loss: 2.8726 | LR: 1.20e-04 | 2111.6ms/step | 9.5 samples/s +Step 29910/44000 | Loss: 2.8874 | LR: 1.20e-04 | 2101.3ms/step | 9.5 samples/s +Step 29920/44000 | Loss: 2.9063 | LR: 1.20e-04 | 2110.4ms/step | 9.5 samples/s +Step 29930/44000 | Loss: 2.9016 | LR: 1.20e-04 | 2100.7ms/step | 9.5 samples/s +Step 29940/44000 | Loss: 2.8766 | LR: 1.19e-04 | 2113.8ms/step | 9.5 samples/s +Step 29950/44000 | Loss: 2.8645 | LR: 1.19e-04 | 2114.5ms/step | 9.5 samples/s +Step 29960/44000 | Loss: 2.8867 | LR: 1.19e-04 | 2094.5ms/step | 9.5 samples/s +Step 29970/44000 | Loss: 2.8646 | LR: 1.19e-04 | 2099.8ms/step | 9.5 samples/s +Step 29980/44000 | Loss: 2.8747 | LR: 1.19e-04 | 2118.1ms/step | 9.4 samples/s +Step 29990/44000 | Loss: 2.8838 | LR: 1.19e-04 | 2115.0ms/step | 9.5 samples/s +Step 30000/44000 | Loss: 2.9112 | LR: 1.19e-04 | 2092.0ms/step | 9.6 samples/s +Validation at step 30000: Loss = 2.8861, Perplexity = 17.92 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-30000 +Step 30010/44000 | Loss: 2.8790 | LR: 1.18e-04 | 4296.4ms/step | 4.7 samples/s +Step 30020/44000 | Loss: 2.8861 | LR: 1.18e-04 | 2106.0ms/step | 9.5 samples/s +Step 30030/44000 | Loss: 2.8930 | LR: 1.18e-04 | 2110.2ms/step | 9.5 samples/s +Step 30040/44000 | Loss: 2.9343 | LR: 1.18e-04 | 2110.5ms/step | 9.5 samples/s +Step 30050/44000 | Loss: 2.8936 | LR: 1.18e-04 | 2106.3ms/step | 9.5 samples/s +Step 30060/44000 | Loss: 2.8880 | LR: 1.18e-04 | 2104.4ms/step | 9.5 samples/s +Step 30070/44000 | Loss: 2.9253 | LR: 1.17e-04 | 2116.0ms/step | 9.5 samples/s +Step 30080/44000 | Loss: 2.9442 | LR: 1.17e-04 | 2102.6ms/step | 9.5 samples/s +Step 30090/44000 | Loss: 2.8898 | LR: 1.17e-04 | 2098.2ms/step | 9.5 samples/s +Step 30100/44000 | Loss: 2.8836 | LR: 1.17e-04 | 2093.8ms/step | 9.6 samples/s +Step 30110/44000 | Loss: 2.8496 | LR: 1.17e-04 | 2112.2ms/step | 9.5 samples/s +Step 30120/44000 | Loss: 2.8912 | LR: 1.17e-04 | 2115.5ms/step | 9.5 samples/s +Step 30130/44000 | Loss: 2.8950 | LR: 1.17e-04 | 2098.4ms/step | 9.5 samples/s +Step 30140/44000 | Loss: 2.9151 | LR: 1.16e-04 | 2097.4ms/step | 9.5 samples/s +Step 30150/44000 | Loss: 2.8615 | LR: 1.16e-04 | 2106.5ms/step | 9.5 samples/s +Step 30160/44000 | Loss: 2.9082 | LR: 1.16e-04 | 2100.2ms/step | 9.5 samples/s +Step 30170/44000 | Loss: 2.9130 | LR: 1.16e-04 | 2101.4ms/step | 9.5 samples/s +Step 30180/44000 | Loss: 2.8114 | LR: 1.16e-04 | 2101.3ms/step | 9.5 samples/s +Step 30190/44000 | Loss: 2.8927 | LR: 1.16e-04 | 2121.1ms/step | 9.4 samples/s +Step 30200/44000 | Loss: 2.8474 | LR: 1.15e-04 | 2107.3ms/step | 9.5 samples/s +Step 30210/44000 | Loss: 2.9137 | LR: 1.15e-04 | 2078.8ms/step | 9.6 samples/s +Step 30220/44000 | Loss: 2.8798 | LR: 1.15e-04 | 2116.4ms/step | 9.4 samples/s +Step 30230/44000 | Loss: 2.9010 | LR: 1.15e-04 | 2103.5ms/step | 9.5 samples/s +Step 30240/44000 | Loss: 2.8887 | LR: 1.15e-04 | 2107.6ms/step | 9.5 samples/s +Step 30250/44000 | Loss: 2.9177 | LR: 1.15e-04 | 2111.1ms/step | 9.5 samples/s +Step 30260/44000 | Loss: 2.8704 | LR: 1.15e-04 | 2115.0ms/step | 9.5 samples/s +Step 30270/44000 | Loss: 2.9039 | LR: 1.14e-04 | 2104.5ms/step | 9.5 samples/s +Step 30280/44000 | Loss: 2.9456 | LR: 1.14e-04 | 2089.0ms/step | 9.6 samples/s +Step 30290/44000 | Loss: 2.8629 | LR: 1.14e-04 | 2096.8ms/step | 9.5 samples/s +Step 30300/44000 | Loss: 2.9023 | LR: 1.14e-04 | 2096.2ms/step | 9.5 samples/s +Step 30310/44000 | Loss: 2.8650 | LR: 1.14e-04 | 2103.2ms/step | 9.5 samples/s +Step 30320/44000 | Loss: 2.9107 | LR: 1.14e-04 | 2104.6ms/step | 9.5 samples/s +Step 30330/44000 | Loss: 2.8815 | LR: 1.13e-04 | 2096.1ms/step | 9.5 samples/s +Step 30340/44000 | Loss: 2.8715 | LR: 1.13e-04 | 2099.2ms/step | 9.5 samples/s +Step 30350/44000 | Loss: 2.8964 | LR: 1.13e-04 | 2092.8ms/step | 9.6 samples/s +Step 30360/44000 | Loss: 2.8940 | LR: 1.13e-04 | 2104.2ms/step | 9.5 samples/s +Step 30370/44000 | Loss: 2.9409 | LR: 1.13e-04 | 2117.7ms/step | 9.4 samples/s +Step 30380/44000 | Loss: 2.9004 | LR: 1.13e-04 | 2097.2ms/step | 9.5 samples/s +Step 30390/44000 | Loss: 2.8985 | LR: 1.13e-04 | 2103.2ms/step | 9.5 samples/s +Step 30400/44000 | Loss: 2.8689 | LR: 1.12e-04 | 2110.1ms/step | 9.5 samples/s +Step 30410/44000 | Loss: 2.8957 | LR: 1.12e-04 | 2111.7ms/step | 9.5 samples/s +Step 30420/44000 | Loss: 2.8899 | LR: 1.12e-04 | 2090.3ms/step | 9.6 samples/s +Step 30430/44000 | Loss: 2.8933 | LR: 1.12e-04 | 2111.3ms/step | 9.5 samples/s +Step 30440/44000 | Loss: 2.9258 | LR: 1.12e-04 | 2121.2ms/step | 9.4 samples/s +Step 30450/44000 | Loss: 2.8557 | LR: 1.12e-04 | 2076.1ms/step | 9.6 samples/s +Step 30460/44000 | Loss: 2.8957 | LR: 1.11e-04 | 2121.0ms/step | 9.4 samples/s +Step 30470/44000 | Loss: 2.8955 | LR: 1.11e-04 | 2074.6ms/step | 9.6 samples/s +Step 30480/44000 | Loss: 2.8849 | LR: 1.11e-04 | 2106.6ms/step | 9.5 samples/s +Step 30490/44000 | Loss: 2.8845 | LR: 1.11e-04 | 2104.4ms/step | 9.5 samples/s +Step 30500/44000 | Loss: 2.9059 | LR: 1.11e-04 | 2109.2ms/step | 9.5 samples/s +Step 30510/44000 | Loss: 2.8976 | LR: 1.11e-04 | 2125.1ms/step | 9.4 samples/s +Step 30520/44000 | Loss: 2.8650 | LR: 1.11e-04 | 2101.2ms/step | 9.5 samples/s +Step 30530/44000 | Loss: 2.9335 | LR: 1.10e-04 | 2132.8ms/step | 9.4 samples/s +Step 30540/44000 | Loss: 2.8884 | LR: 1.10e-04 | 2111.6ms/step | 9.5 samples/s +Step 30550/44000 | Loss: 2.9375 | LR: 1.10e-04 | 2112.0ms/step | 9.5 samples/s +Step 30560/44000 | Loss: 2.8892 | LR: 1.10e-04 | 2107.3ms/step | 9.5 samples/s +Step 30570/44000 | Loss: 2.8994 | LR: 1.10e-04 | 2090.8ms/step | 9.6 samples/s +Step 30580/44000 | Loss: 2.8609 | LR: 1.10e-04 | 2114.0ms/step | 9.5 samples/s +Step 30590/44000 | Loss: 2.8901 | LR: 1.10e-04 | 2105.0ms/step | 9.5 samples/s +Step 30600/44000 | Loss: 2.8634 | LR: 1.09e-04 | 2108.1ms/step | 9.5 samples/s +Step 30610/44000 | Loss: 2.8592 | LR: 1.09e-04 | 2096.6ms/step | 9.5 samples/s +Step 30620/44000 | Loss: 2.8780 | LR: 1.09e-04 | 2097.7ms/step | 9.5 samples/s +Step 30630/44000 | Loss: 2.9105 | LR: 1.09e-04 | 2100.4ms/step | 9.5 samples/s +Step 30640/44000 | Loss: 2.8953 | LR: 1.09e-04 | 2111.2ms/step | 9.5 samples/s +Step 30650/44000 | Loss: 2.9284 | LR: 1.09e-04 | 2106.5ms/step | 9.5 samples/s +Step 30660/44000 | Loss: 2.8986 | LR: 1.08e-04 | 2095.9ms/step | 9.5 samples/s +Step 30670/44000 | Loss: 2.9053 | LR: 1.08e-04 | 2086.5ms/step | 9.6 samples/s +Step 30680/44000 | Loss: 2.8970 | LR: 1.08e-04 | 2087.2ms/step | 9.6 samples/s +Step 30690/44000 | Loss: 2.8861 | LR: 1.08e-04 | 2109.6ms/step | 9.5 samples/s +Step 30700/44000 | Loss: 2.8746 | LR: 1.08e-04 | 2111.9ms/step | 9.5 samples/s +Step 30710/44000 | Loss: 2.9020 | LR: 1.08e-04 | 2131.1ms/step | 9.4 samples/s +Step 30720/44000 | Loss: 2.9262 | LR: 1.08e-04 | 2114.4ms/step | 9.5 samples/s +Step 30730/44000 | Loss: 2.9081 | LR: 1.07e-04 | 2114.2ms/step | 9.5 samples/s +Step 30740/44000 | Loss: 2.8859 | LR: 1.07e-04 | 2132.8ms/step | 9.4 samples/s +Step 30750/44000 | Loss: 2.8900 | LR: 1.07e-04 | 2109.4ms/step | 9.5 samples/s +Step 30760/44000 | Loss: 2.8548 | LR: 1.07e-04 | 2095.3ms/step | 9.5 samples/s +Step 30770/44000 | Loss: 2.9059 | LR: 1.07e-04 | 2090.3ms/step | 9.6 samples/s +Step 30780/44000 | Loss: 2.8896 | LR: 1.07e-04 | 2086.1ms/step | 9.6 samples/s +Step 30790/44000 | Loss: 2.9407 | LR: 1.07e-04 | 2106.4ms/step | 9.5 samples/s +Step 30800/44000 | Loss: 2.8900 | LR: 1.06e-04 | 2107.3ms/step | 9.5 samples/s +Step 30810/44000 | Loss: 2.8924 | LR: 1.06e-04 | 2107.4ms/step | 9.5 samples/s +Step 30820/44000 | Loss: 2.8902 | LR: 1.06e-04 | 2112.2ms/step | 9.5 samples/s +Step 30830/44000 | Loss: 2.8909 | LR: 1.06e-04 | 2111.3ms/step | 9.5 samples/s +Step 30840/44000 | Loss: 2.8835 | LR: 1.06e-04 | 2092.0ms/step | 9.6 samples/s +Step 30850/44000 | Loss: 2.8693 | LR: 1.06e-04 | 2092.7ms/step | 9.6 samples/s +Step 30860/44000 | Loss: 2.8972 | LR: 1.05e-04 | 2116.5ms/step | 9.4 samples/s +Step 30870/44000 | Loss: 2.9146 | LR: 1.05e-04 | 2110.3ms/step | 9.5 samples/s +Step 30880/44000 | Loss: 2.9086 | LR: 1.05e-04 | 2002.7ms/step | 10.0 samples/s +Step 30890/44000 | Loss: 2.8643 | LR: 1.05e-04 | 2089.1ms/step | 9.6 samples/s +Step 30900/44000 | Loss: 2.8908 | LR: 1.05e-04 | 2110.6ms/step | 9.5 samples/s +Step 30910/44000 | Loss: 2.8840 | LR: 1.05e-04 | 2100.1ms/step | 9.5 samples/s +Step 30920/44000 | Loss: 2.9138 | LR: 1.05e-04 | 2095.5ms/step | 9.5 samples/s +Step 30930/44000 | Loss: 2.8903 | LR: 1.04e-04 | 2112.0ms/step | 9.5 samples/s +Step 30940/44000 | Loss: 2.9243 | LR: 1.04e-04 | 2091.7ms/step | 9.6 samples/s +Step 30950/44000 | Loss: 2.8983 | LR: 1.04e-04 | 2132.2ms/step | 9.4 samples/s +Step 30960/44000 | Loss: 2.8983 | LR: 1.04e-04 | 2206.7ms/step | 9.1 samples/s +Step 30970/44000 | Loss: 2.9227 | LR: 1.04e-04 | 2223.4ms/step | 9.0 samples/s +Step 30980/44000 | Loss: 2.9051 | LR: 1.04e-04 | 2265.0ms/step | 8.8 samples/s +Step 30990/44000 | Loss: 2.8953 | LR: 1.04e-04 | 2223.0ms/step | 9.0 samples/s +Step 31000/44000 | Loss: 2.9309 | LR: 1.03e-04 | 2215.4ms/step | 9.0 samples/s +Validation at step 31000: Loss = 2.8827, Perplexity = 17.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-31000 +Step 31010/44000 | Loss: 2.8754 | LR: 1.03e-04 | 4592.0ms/step | 4.4 samples/s +Step 31020/44000 | Loss: 2.9210 | LR: 1.03e-04 | 2201.5ms/step | 9.1 samples/s +Step 31030/44000 | Loss: 2.8827 | LR: 1.03e-04 | 2188.5ms/step | 9.1 samples/s +Step 31040/44000 | Loss: 2.8573 | LR: 1.03e-04 | 2212.0ms/step | 9.0 samples/s +Step 31050/44000 | Loss: 2.8404 | LR: 1.03e-04 | 2225.8ms/step | 9.0 samples/s +Step 31060/44000 | Loss: 2.8154 | LR: 1.03e-04 | 2214.1ms/step | 9.0 samples/s +Step 31070/44000 | Loss: 2.8633 | LR: 1.02e-04 | 2204.9ms/step | 9.1 samples/s +Step 31080/44000 | Loss: 2.8963 | LR: 1.02e-04 | 2234.5ms/step | 9.0 samples/s +Step 31090/44000 | Loss: 2.8832 | LR: 1.02e-04 | 2213.5ms/step | 9.0 samples/s +Step 31100/44000 | Loss: 2.8769 | LR: 1.02e-04 | 2217.7ms/step | 9.0 samples/s +Step 31110/44000 | Loss: 2.9138 | LR: 1.02e-04 | 2211.9ms/step | 9.0 samples/s +Step 31120/44000 | Loss: 2.8820 | LR: 1.02e-04 | 2229.8ms/step | 9.0 samples/s +Step 31130/44000 | Loss: 2.8725 | LR: 1.02e-04 | 2204.5ms/step | 9.1 samples/s +Step 31140/44000 | Loss: 2.8942 | LR: 1.01e-04 | 2211.5ms/step | 9.0 samples/s +Step 31150/44000 | Loss: 2.9196 | LR: 1.01e-04 | 2218.3ms/step | 9.0 samples/s +Step 31160/44000 | Loss: 2.9035 | LR: 1.01e-04 | 2217.8ms/step | 9.0 samples/s +Step 31170/44000 | Loss: 2.9326 | LR: 1.01e-04 | 2220.6ms/step | 9.0 samples/s +Step 31180/44000 | Loss: 2.8539 | LR: 1.01e-04 | 2220.0ms/step | 9.0 samples/s +Step 31190/44000 | Loss: 2.8919 | LR: 1.01e-04 | 2213.1ms/step | 9.0 samples/s +Step 31200/44000 | Loss: 2.8776 | LR: 1.00e-04 | 2245.3ms/step | 8.9 samples/s +Step 31210/44000 | Loss: 2.8750 | LR: 1.00e-04 | 2233.6ms/step | 9.0 samples/s +Step 31220/44000 | Loss: 2.8796 | LR: 1.00e-04 | 2216.9ms/step | 9.0 samples/s +Step 31230/44000 | Loss: 2.9205 | LR: 1.00e-04 | 2231.6ms/step | 9.0 samples/s +Step 31240/44000 | Loss: 2.8564 | LR: 9.99e-05 | 2224.0ms/step | 9.0 samples/s +Step 31250/44000 | Loss: 2.8789 | LR: 9.98e-05 | 2236.7ms/step | 8.9 samples/s +Step 31260/44000 | Loss: 2.8631 | LR: 9.96e-05 | 2234.5ms/step | 9.0 samples/s +Step 31270/44000 | Loss: 2.8613 | LR: 9.95e-05 | 2233.5ms/step | 9.0 samples/s +Step 31280/44000 | Loss: 2.9296 | LR: 9.93e-05 | 2207.5ms/step | 9.1 samples/s +Step 31290/44000 | Loss: 2.9037 | LR: 9.92e-05 | 2201.8ms/step | 9.1 samples/s +Step 31300/44000 | Loss: 2.9038 | LR: 9.90e-05 | 2220.0ms/step | 9.0 samples/s +Step 31310/44000 | Loss: 2.9097 | LR: 9.89e-05 | 2225.6ms/step | 9.0 samples/s +Step 31320/44000 | Loss: 2.9103 | LR: 9.88e-05 | 2218.1ms/step | 9.0 samples/s +Step 31330/44000 | Loss: 2.8824 | LR: 9.86e-05 | 2200.4ms/step | 9.1 samples/s +Step 31340/44000 | Loss: 2.8973 | LR: 9.85e-05 | 2179.3ms/step | 9.2 samples/s +Step 31350/44000 | Loss: 2.8643 | LR: 9.83e-05 | 2182.0ms/step | 9.2 samples/s +Step 31360/44000 | Loss: 2.8823 | LR: 9.82e-05 | 2140.4ms/step | 9.3 samples/s +Step 31370/44000 | Loss: 2.9055 | LR: 9.80e-05 | 2122.9ms/step | 9.4 samples/s +Step 31380/44000 | Loss: 2.9214 | LR: 9.79e-05 | 2095.6ms/step | 9.5 samples/s +Step 31390/44000 | Loss: 2.9221 | LR: 9.77e-05 | 2092.5ms/step | 9.6 samples/s +Step 31400/44000 | Loss: 2.8941 | LR: 9.76e-05 | 2142.1ms/step | 9.3 samples/s +Step 31410/44000 | Loss: 2.8648 | LR: 9.75e-05 | 2150.9ms/step | 9.3 samples/s +Step 31420/44000 | Loss: 2.8736 | LR: 9.73e-05 | 2185.3ms/step | 9.2 samples/s +Step 31430/44000 | Loss: 2.9333 | LR: 9.72e-05 | 2140.7ms/step | 9.3 samples/s +Step 31440/44000 | Loss: 2.8986 | LR: 9.70e-05 | 2109.7ms/step | 9.5 samples/s +Step 31450/44000 | Loss: 2.9046 | LR: 9.69e-05 | 2107.8ms/step | 9.5 samples/s +Step 31460/44000 | Loss: 2.8714 | LR: 9.67e-05 | 2122.0ms/step | 9.4 samples/s +Step 31470/44000 | Loss: 2.8972 | LR: 9.66e-05 | 2127.0ms/step | 9.4 samples/s +Step 31480/44000 | Loss: 2.8770 | LR: 9.65e-05 | 2161.5ms/step | 9.3 samples/s +Step 31490/44000 | Loss: 2.8940 | LR: 9.63e-05 | 2135.8ms/step | 9.4 samples/s +Step 31500/44000 | Loss: 2.8545 | LR: 9.62e-05 | 2157.2ms/step | 9.3 samples/s +Step 31510/44000 | Loss: 2.9154 | LR: 9.60e-05 | 2148.0ms/step | 9.3 samples/s +Step 31520/44000 | Loss: 2.8761 | LR: 9.59e-05 | 2095.9ms/step | 9.5 samples/s +Step 31530/44000 | Loss: 2.9005 | LR: 9.57e-05 | 2120.4ms/step | 9.4 samples/s +Step 31540/44000 | Loss: 2.8822 | LR: 9.56e-05 | 2123.9ms/step | 9.4 samples/s +Step 31550/44000 | Loss: 2.8519 | LR: 9.54e-05 | 2204.1ms/step | 9.1 samples/s +Step 31560/44000 | Loss: 2.8607 | LR: 9.53e-05 | 2170.9ms/step | 9.2 samples/s +Step 31570/44000 | Loss: 2.9371 | LR: 9.52e-05 | 2163.6ms/step | 9.2 samples/s +Step 31580/44000 | Loss: 2.8950 | LR: 9.50e-05 | 2158.2ms/step | 9.3 samples/s +Step 31590/44000 | Loss: 2.9068 | LR: 9.49e-05 | 2180.5ms/step | 9.2 samples/s +Step 31600/44000 | Loss: 2.8790 | LR: 9.47e-05 | 2147.7ms/step | 9.3 samples/s +Step 31610/44000 | Loss: 2.9172 | LR: 9.46e-05 | 2155.2ms/step | 9.3 samples/s +Step 31620/44000 | Loss: 2.8717 | LR: 9.45e-05 | 2148.5ms/step | 9.3 samples/s +Step 31630/44000 | Loss: 2.8640 | LR: 9.43e-05 | 2116.8ms/step | 9.4 samples/s +Step 31640/44000 | Loss: 2.9253 | LR: 9.42e-05 | 2183.4ms/step | 9.2 samples/s +Step 31650/44000 | Loss: 2.8260 | LR: 9.40e-05 | 2193.1ms/step | 9.1 samples/s +Step 31660/44000 | Loss: 2.9180 | LR: 9.39e-05 | 2151.5ms/step | 9.3 samples/s +Step 31670/44000 | Loss: 2.9544 | LR: 9.37e-05 | 2153.7ms/step | 9.3 samples/s +Step 31680/44000 | Loss: 2.9225 | LR: 9.36e-05 | 2137.9ms/step | 9.4 samples/s +Step 31690/44000 | Loss: 2.9132 | LR: 9.35e-05 | 2175.0ms/step | 9.2 samples/s +Step 31700/44000 | Loss: 2.8746 | LR: 9.33e-05 | 2132.1ms/step | 9.4 samples/s +Step 31710/44000 | Loss: 2.8669 | LR: 9.32e-05 | 2124.0ms/step | 9.4 samples/s +Step 31720/44000 | Loss: 2.9019 | LR: 9.30e-05 | 2146.4ms/step | 9.3 samples/s +Step 31730/44000 | Loss: 2.9098 | LR: 9.29e-05 | 2140.9ms/step | 9.3 samples/s +Step 31740/44000 | Loss: 2.8948 | LR: 9.28e-05 | 2143.6ms/step | 9.3 samples/s +Step 31750/44000 | Loss: 2.9242 | LR: 9.26e-05 | 2118.7ms/step | 9.4 samples/s +Step 31760/44000 | Loss: 2.9054 | LR: 9.25e-05 | 2136.2ms/step | 9.4 samples/s +Step 31770/44000 | Loss: 2.8834 | LR: 9.23e-05 | 2119.9ms/step | 9.4 samples/s +Step 31780/44000 | Loss: 2.8539 | LR: 9.22e-05 | 2164.7ms/step | 9.2 samples/s +Step 31790/44000 | Loss: 2.8748 | LR: 9.20e-05 | 2154.2ms/step | 9.3 samples/s +Step 31800/44000 | Loss: 2.9135 | LR: 9.19e-05 | 2127.8ms/step | 9.4 samples/s +Step 31810/44000 | Loss: 2.8627 | LR: 9.18e-05 | 2132.6ms/step | 9.4 samples/s +Step 31820/44000 | Loss: 2.9137 | LR: 9.16e-05 | 2131.3ms/step | 9.4 samples/s +Step 31830/44000 | Loss: 2.8925 | LR: 9.15e-05 | 2150.1ms/step | 9.3 samples/s +Step 31840/44000 | Loss: 2.8880 | LR: 9.13e-05 | 2168.6ms/step | 9.2 samples/s +Step 31850/44000 | Loss: 2.8962 | LR: 9.12e-05 | 2154.6ms/step | 9.3 samples/s +Step 31860/44000 | Loss: 2.8507 | LR: 9.11e-05 | 2152.2ms/step | 9.3 samples/s +Step 31870/44000 | Loss: 2.8756 | LR: 9.09e-05 | 2156.4ms/step | 9.3 samples/s +Step 31880/44000 | Loss: 2.9127 | LR: 9.08e-05 | 2148.5ms/step | 9.3 samples/s +Step 31890/44000 | Loss: 2.8728 | LR: 9.06e-05 | 2100.0ms/step | 9.5 samples/s +Step 31900/44000 | Loss: 2.8522 | LR: 9.05e-05 | 2054.2ms/step | 9.7 samples/s +Step 31910/44000 | Loss: 2.8644 | LR: 9.04e-05 | 2172.0ms/step | 9.2 samples/s +Step 31920/44000 | Loss: 2.8962 | LR: 9.02e-05 | 2229.6ms/step | 9.0 samples/s +Step 31930/44000 | Loss: 2.9210 | LR: 9.01e-05 | 2307.0ms/step | 8.7 samples/s +Step 31940/44000 | Loss: 2.9169 | LR: 8.99e-05 | 2330.5ms/step | 8.6 samples/s +Step 31950/44000 | Loss: 2.9283 | LR: 8.98e-05 | 2317.7ms/step | 8.6 samples/s +Step 31960/44000 | Loss: 2.9114 | LR: 8.97e-05 | 2353.0ms/step | 8.5 samples/s +Step 31970/44000 | Loss: 2.8890 | LR: 8.95e-05 | 2287.1ms/step | 8.7 samples/s +Step 31980/44000 | Loss: 2.8821 | LR: 8.94e-05 | 2314.5ms/step | 8.6 samples/s +Step 31990/44000 | Loss: 2.9044 | LR: 8.92e-05 | 2333.2ms/step | 8.6 samples/s +Step 32000/44000 | Loss: 2.8904 | LR: 8.91e-05 | 2324.8ms/step | 8.6 samples/s +Validation at step 32000: Loss = 2.8800, Perplexity = 17.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-32000 +Step 32010/44000 | Loss: 2.8962 | LR: 8.90e-05 | 4807.2ms/step | 4.2 samples/s +Step 32020/44000 | Loss: 2.8919 | LR: 8.88e-05 | 2338.3ms/step | 8.6 samples/s +Step 32030/44000 | Loss: 2.9023 | LR: 8.87e-05 | 2316.8ms/step | 8.6 samples/s +Step 32040/44000 | Loss: 2.9088 | LR: 8.86e-05 | 2316.5ms/step | 8.6 samples/s +Step 32050/44000 | Loss: 2.9023 | LR: 8.84e-05 | 2319.9ms/step | 8.6 samples/s +Step 32060/44000 | Loss: 2.8821 | LR: 8.83e-05 | 2310.1ms/step | 8.7 samples/s +Step 32070/44000 | Loss: 2.9045 | LR: 8.81e-05 | 2341.5ms/step | 8.5 samples/s +Step 32080/44000 | Loss: 2.8849 | LR: 8.80e-05 | 2327.8ms/step | 8.6 samples/s +Step 32090/44000 | Loss: 2.8734 | LR: 8.79e-05 | 2300.3ms/step | 8.7 samples/s +Step 32100/44000 | Loss: 2.8640 | LR: 8.77e-05 | 2316.3ms/step | 8.6 samples/s +Step 32110/44000 | Loss: 2.8724 | LR: 8.76e-05 | 2321.2ms/step | 8.6 samples/s +Step 32120/44000 | Loss: 2.9289 | LR: 8.74e-05 | 2316.5ms/step | 8.6 samples/s +Step 32130/44000 | Loss: 2.8999 | LR: 8.73e-05 | 2316.5ms/step | 8.6 samples/s +Step 32140/44000 | Loss: 2.9134 | LR: 8.72e-05 | 2315.8ms/step | 8.6 samples/s +Step 32150/44000 | Loss: 2.9049 | LR: 8.70e-05 | 2346.9ms/step | 8.5 samples/s +Step 32160/44000 | Loss: 2.9209 | LR: 8.69e-05 | 2310.7ms/step | 8.7 samples/s +Step 32170/44000 | Loss: 2.8999 | LR: 8.68e-05 | 2310.3ms/step | 8.7 samples/s +Step 32180/44000 | Loss: 2.8417 | LR: 8.66e-05 | 2316.6ms/step | 8.6 samples/s +Step 32190/44000 | Loss: 2.9553 | LR: 8.65e-05 | 2328.1ms/step | 8.6 samples/s +Step 32200/44000 | Loss: 2.9276 | LR: 8.63e-05 | 2318.7ms/step | 8.6 samples/s +Step 32210/44000 | Loss: 2.8991 | LR: 8.62e-05 | 2323.7ms/step | 8.6 samples/s +Step 32220/44000 | Loss: 2.8768 | LR: 8.61e-05 | 2322.3ms/step | 8.6 samples/s +Step 32230/44000 | Loss: 2.8728 | LR: 8.59e-05 | 2354.5ms/step | 8.5 samples/s +Step 32240/44000 | Loss: 2.9325 | LR: 8.58e-05 | 2305.9ms/step | 8.7 samples/s +Step 32250/44000 | Loss: 2.8704 | LR: 8.57e-05 | 2285.5ms/step | 8.8 samples/s +Step 32260/44000 | Loss: 2.9359 | LR: 8.55e-05 | 2309.9ms/step | 8.7 samples/s +Step 32270/44000 | Loss: 2.8948 | LR: 8.54e-05 | 2344.9ms/step | 8.5 samples/s +Step 32280/44000 | Loss: 2.8567 | LR: 8.53e-05 | 2323.3ms/step | 8.6 samples/s +Step 32290/44000 | Loss: 2.8496 | LR: 8.51e-05 | 2300.1ms/step | 8.7 samples/s +Step 32300/44000 | Loss: 2.8540 | LR: 8.50e-05 | 2294.8ms/step | 8.7 samples/s +Step 32310/44000 | Loss: 2.8559 | LR: 8.48e-05 | 2311.5ms/step | 8.7 samples/s +Step 32320/44000 | Loss: 2.8846 | LR: 8.47e-05 | 2342.8ms/step | 8.5 samples/s +Step 32330/44000 | Loss: 2.8899 | LR: 8.46e-05 | 2343.4ms/step | 8.5 samples/s +Step 32340/44000 | Loss: 2.9091 | LR: 8.44e-05 | 2296.0ms/step | 8.7 samples/s +Step 32350/44000 | Loss: 2.8605 | LR: 8.43e-05 | 2322.2ms/step | 8.6 samples/s +Step 32360/44000 | Loss: 2.9017 | LR: 8.42e-05 | 2308.8ms/step | 8.7 samples/s +Step 32370/44000 | Loss: 2.9091 | LR: 8.40e-05 | 2303.9ms/step | 8.7 samples/s +Step 32380/44000 | Loss: 2.8950 | LR: 8.39e-05 | 2311.5ms/step | 8.7 samples/s +Step 32390/44000 | Loss: 2.9131 | LR: 8.38e-05 | 2328.6ms/step | 8.6 samples/s +Step 32400/44000 | Loss: 2.8685 | LR: 8.36e-05 | 2310.5ms/step | 8.7 samples/s +Step 32410/44000 | Loss: 2.8787 | LR: 8.35e-05 | 2321.5ms/step | 8.6 samples/s +Step 32420/44000 | Loss: 2.9277 | LR: 8.33e-05 | 2311.2ms/step | 8.7 samples/s +Step 32430/44000 | Loss: 2.8557 | LR: 8.32e-05 | 2296.3ms/step | 8.7 samples/s +Step 32440/44000 | Loss: 2.9035 | LR: 8.31e-05 | 2290.3ms/step | 8.7 samples/s +Step 32450/44000 | Loss: 2.8867 | LR: 8.29e-05 | 2337.3ms/step | 8.6 samples/s +Step 32460/44000 | Loss: 2.9055 | LR: 8.28e-05 | 2313.5ms/step | 8.6 samples/s +Step 32470/44000 | Loss: 2.8863 | LR: 8.27e-05 | 2320.1ms/step | 8.6 samples/s +Step 32480/44000 | Loss: 2.9041 | LR: 8.25e-05 | 2311.8ms/step | 8.7 samples/s +Step 32490/44000 | Loss: 2.8744 | LR: 8.24e-05 | 2312.0ms/step | 8.7 samples/s +Step 32500/44000 | Loss: 2.8965 | LR: 8.23e-05 | 2311.6ms/step | 8.7 samples/s +Step 32510/44000 | Loss: 2.8731 | LR: 8.21e-05 | 2306.9ms/step | 8.7 samples/s +Step 32520/44000 | Loss: 2.8573 | LR: 8.20e-05 | 2320.0ms/step | 8.6 samples/s +Step 32530/44000 | Loss: 2.8551 | LR: 8.19e-05 | 2313.8ms/step | 8.6 samples/s +Step 32540/44000 | Loss: 2.8878 | LR: 8.17e-05 | 2337.2ms/step | 8.6 samples/s +Step 32550/44000 | Loss: 2.8516 | LR: 8.16e-05 | 2299.9ms/step | 8.7 samples/s +Step 32560/44000 | Loss: 2.8328 | LR: 8.15e-05 | 2295.6ms/step | 8.7 samples/s +Step 32570/44000 | Loss: 2.8683 | LR: 8.13e-05 | 2317.8ms/step | 8.6 samples/s +Step 32580/44000 | Loss: 2.8863 | LR: 8.12e-05 | 2326.7ms/step | 8.6 samples/s +Step 32590/44000 | Loss: 2.8469 | LR: 8.11e-05 | 2382.4ms/step | 8.4 samples/s +Step 32600/44000 | Loss: 2.9107 | LR: 8.09e-05 | 2307.3ms/step | 8.7 samples/s +Step 32610/44000 | Loss: 2.9464 | LR: 8.08e-05 | 2309.0ms/step | 8.7 samples/s +Step 32620/44000 | Loss: 2.9017 | LR: 8.07e-05 | 2353.8ms/step | 8.5 samples/s +Step 32630/44000 | Loss: 2.9126 | LR: 8.05e-05 | 2350.8ms/step | 8.5 samples/s +Step 32640/44000 | Loss: 2.8962 | LR: 8.04e-05 | 2343.4ms/step | 8.5 samples/s +Step 32650/44000 | Loss: 2.9192 | LR: 8.03e-05 | 2322.3ms/step | 8.6 samples/s +Step 32660/44000 | Loss: 2.8852 | LR: 8.01e-05 | 2337.5ms/step | 8.6 samples/s +Step 32670/44000 | Loss: 2.8558 | LR: 8.00e-05 | 2353.0ms/step | 8.5 samples/s +Step 32680/44000 | Loss: 2.8804 | LR: 7.99e-05 | 2356.7ms/step | 8.5 samples/s +Step 32690/44000 | Loss: 2.9562 | LR: 7.97e-05 | 2315.1ms/step | 8.6 samples/s +Step 32700/44000 | Loss: 2.9024 | LR: 7.96e-05 | 2349.5ms/step | 8.5 samples/s +Step 32710/44000 | Loss: 2.8453 | LR: 7.95e-05 | 2353.0ms/step | 8.5 samples/s +Step 32720/44000 | Loss: 2.9292 | LR: 7.93e-05 | 2342.1ms/step | 8.5 samples/s +Step 32730/44000 | Loss: 2.8874 | LR: 7.92e-05 | 2326.6ms/step | 8.6 samples/s +Step 32740/44000 | Loss: 2.8701 | LR: 7.91e-05 | 2344.0ms/step | 8.5 samples/s +Step 32750/44000 | Loss: 2.8851 | LR: 7.89e-05 | 2372.5ms/step | 8.4 samples/s +Step 32760/44000 | Loss: 2.9102 | LR: 7.88e-05 | 2295.0ms/step | 8.7 samples/s +Step 32770/44000 | Loss: 2.8363 | LR: 7.87e-05 | 2303.8ms/step | 8.7 samples/s +Step 32780/44000 | Loss: 2.8910 | LR: 7.85e-05 | 2340.6ms/step | 8.5 samples/s +Step 32790/44000 | Loss: 2.8771 | LR: 7.84e-05 | 2317.0ms/step | 8.6 samples/s +Step 32800/44000 | Loss: 2.8735 | LR: 7.83e-05 | 2326.8ms/step | 8.6 samples/s +Step 32810/44000 | Loss: 2.9166 | LR: 7.81e-05 | 2379.8ms/step | 8.4 samples/s +Step 32820/44000 | Loss: 2.8796 | LR: 7.80e-05 | 2321.7ms/step | 8.6 samples/s +Step 32830/44000 | Loss: 2.9090 | LR: 7.79e-05 | 2314.8ms/step | 8.6 samples/s +Step 32840/44000 | Loss: 2.8557 | LR: 7.77e-05 | 2317.9ms/step | 8.6 samples/s +Step 32850/44000 | Loss: 2.8961 | LR: 7.76e-05 | 2286.1ms/step | 8.7 samples/s +Step 32860/44000 | Loss: 2.9128 | LR: 7.75e-05 | 2346.5ms/step | 8.5 samples/s +Step 32870/44000 | Loss: 2.9046 | LR: 7.73e-05 | 2337.3ms/step | 8.6 samples/s +Step 32880/44000 | Loss: 2.8753 | LR: 7.72e-05 | 2313.2ms/step | 8.6 samples/s +Step 32890/44000 | Loss: 2.8892 | LR: 7.71e-05 | 2340.0ms/step | 8.5 samples/s +Step 32900/44000 | Loss: 2.8838 | LR: 7.70e-05 | 2278.0ms/step | 8.8 samples/s +Step 32910/44000 | Loss: 2.8844 | LR: 7.68e-05 | 2254.3ms/step | 8.9 samples/s +Step 32920/44000 | Loss: 2.8845 | LR: 7.67e-05 | 2169.0ms/step | 9.2 samples/s +Step 32930/44000 | Loss: 2.8550 | LR: 7.66e-05 | 2312.4ms/step | 8.6 samples/s +Step 32940/44000 | Loss: 2.8869 | LR: 7.64e-05 | 2389.9ms/step | 8.4 samples/s +Step 32950/44000 | Loss: 2.8969 | LR: 7.63e-05 | 2337.0ms/step | 8.6 samples/s +Step 32960/44000 | Loss: 2.8572 | LR: 7.62e-05 | 2336.7ms/step | 8.6 samples/s +Step 32970/44000 | Loss: 2.8865 | LR: 7.60e-05 | 2305.7ms/step | 8.7 samples/s +Step 32980/44000 | Loss: 2.8637 | LR: 7.59e-05 | 2323.5ms/step | 8.6 samples/s +Step 32990/44000 | Loss: 2.8922 | LR: 7.58e-05 | 2329.7ms/step | 8.6 samples/s +Step 33000/44000 | Loss: 2.8886 | LR: 7.56e-05 | 2289.9ms/step | 8.7 samples/s +Validation at step 33000: Loss = 2.8779, Perplexity = 17.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-33000 +Step 33010/44000 | Loss: 2.8307 | LR: 7.55e-05 | 4801.1ms/step | 4.2 samples/s +Step 33020/44000 | Loss: 2.8970 | LR: 7.54e-05 | 2332.3ms/step | 8.6 samples/s +Step 33030/44000 | Loss: 2.8711 | LR: 7.53e-05 | 2305.6ms/step | 8.7 samples/s +Step 33040/44000 | Loss: 2.8833 | LR: 7.51e-05 | 2322.2ms/step | 8.6 samples/s +Step 33050/44000 | Loss: 2.9030 | LR: 7.50e-05 | 2321.9ms/step | 8.6 samples/s +Step 33060/44000 | Loss: 2.8819 | LR: 7.49e-05 | 2326.7ms/step | 8.6 samples/s +Step 33070/44000 | Loss: 2.8981 | LR: 7.47e-05 | 2320.9ms/step | 8.6 samples/s +Step 33080/44000 | Loss: 2.9452 | LR: 7.46e-05 | 2300.8ms/step | 8.7 samples/s +Step 33090/44000 | Loss: 2.9063 | LR: 7.45e-05 | 2316.9ms/step | 8.6 samples/s +Step 33100/44000 | Loss: 2.8755 | LR: 7.44e-05 | 2328.2ms/step | 8.6 samples/s +Step 33110/44000 | Loss: 2.8710 | LR: 7.42e-05 | 2290.6ms/step | 8.7 samples/s +Step 33120/44000 | Loss: 2.8863 | LR: 7.41e-05 | 2321.4ms/step | 8.6 samples/s +Step 33130/44000 | Loss: 2.9123 | LR: 7.40e-05 | 2321.0ms/step | 8.6 samples/s +Step 33140/44000 | Loss: 2.8786 | LR: 7.38e-05 | 2305.7ms/step | 8.7 samples/s +Step 33150/44000 | Loss: 2.8591 | LR: 7.37e-05 | 2306.7ms/step | 8.7 samples/s +Step 33160/44000 | Loss: 2.8469 | LR: 7.36e-05 | 2358.4ms/step | 8.5 samples/s +Step 33170/44000 | Loss: 2.8984 | LR: 7.34e-05 | 2314.1ms/step | 8.6 samples/s +Step 33180/44000 | Loss: 2.8955 | LR: 7.33e-05 | 2326.8ms/step | 8.6 samples/s +Step 33190/44000 | Loss: 2.9191 | LR: 7.32e-05 | 2347.5ms/step | 8.5 samples/s +Step 33200/44000 | Loss: 2.9136 | LR: 7.31e-05 | 2298.4ms/step | 8.7 samples/s +Step 33210/44000 | Loss: 2.8797 | LR: 7.29e-05 | 2340.5ms/step | 8.5 samples/s +Step 33220/44000 | Loss: 2.8650 | LR: 7.28e-05 | 2344.7ms/step | 8.5 samples/s +Step 33230/44000 | Loss: 2.8973 | LR: 7.27e-05 | 2299.2ms/step | 8.7 samples/s +Step 33240/44000 | Loss: 2.9033 | LR: 7.26e-05 | 2314.2ms/step | 8.6 samples/s +Step 33250/44000 | Loss: 2.8825 | LR: 7.24e-05 | 2337.5ms/step | 8.6 samples/s +Step 33260/44000 | Loss: 2.8293 | LR: 7.23e-05 | 2328.6ms/step | 8.6 samples/s +Step 33270/44000 | Loss: 2.8625 | LR: 7.22e-05 | 2348.4ms/step | 8.5 samples/s +Step 33280/44000 | Loss: 2.9079 | LR: 7.20e-05 | 2327.4ms/step | 8.6 samples/s +Step 33290/44000 | Loss: 2.8961 | LR: 7.19e-05 | 2319.8ms/step | 8.6 samples/s +Step 33300/44000 | Loss: 2.8642 | LR: 7.18e-05 | 2283.0ms/step | 8.8 samples/s +Step 33310/44000 | Loss: 2.8985 | LR: 7.17e-05 | 2317.2ms/step | 8.6 samples/s +Step 33320/44000 | Loss: 2.8888 | LR: 7.15e-05 | 2347.4ms/step | 8.5 samples/s +Step 33330/44000 | Loss: 2.8992 | LR: 7.14e-05 | 2345.5ms/step | 8.5 samples/s +Step 33340/44000 | Loss: 2.9419 | LR: 7.13e-05 | 2324.6ms/step | 8.6 samples/s +Step 33350/44000 | Loss: 2.8566 | LR: 7.11e-05 | 2346.6ms/step | 8.5 samples/s +Step 33360/44000 | Loss: 2.8965 | LR: 7.10e-05 | 2360.0ms/step | 8.5 samples/s +Step 33370/44000 | Loss: 2.8789 | LR: 7.09e-05 | 2342.5ms/step | 8.5 samples/s +Step 33380/44000 | Loss: 2.8998 | LR: 7.08e-05 | 2342.9ms/step | 8.5 samples/s +Step 33390/44000 | Loss: 2.9161 | LR: 7.06e-05 | 2361.1ms/step | 8.5 samples/s +Step 33400/44000 | Loss: 2.8784 | LR: 7.05e-05 | 2328.8ms/step | 8.6 samples/s +Step 33410/44000 | Loss: 2.8902 | LR: 7.04e-05 | 2353.5ms/step | 8.5 samples/s +Step 33420/44000 | Loss: 2.8718 | LR: 7.03e-05 | 2323.9ms/step | 8.6 samples/s +Step 33430/44000 | Loss: 2.8959 | LR: 7.01e-05 | 2278.2ms/step | 8.8 samples/s +Step 33440/44000 | Loss: 2.8560 | LR: 7.00e-05 | 2300.6ms/step | 8.7 samples/s +Step 33450/44000 | Loss: 2.8794 | LR: 6.99e-05 | 2342.9ms/step | 8.5 samples/s +Step 33460/44000 | Loss: 2.8843 | LR: 6.98e-05 | 2344.0ms/step | 8.5 samples/s +Step 33470/44000 | Loss: 2.8786 | LR: 6.96e-05 | 2337.3ms/step | 8.6 samples/s +Step 33480/44000 | Loss: 2.9106 | LR: 6.95e-05 | 2337.5ms/step | 8.6 samples/s +Step 33490/44000 | Loss: 2.9210 | LR: 6.94e-05 | 2344.9ms/step | 8.5 samples/s +Step 33500/44000 | Loss: 2.9217 | LR: 6.93e-05 | 2332.9ms/step | 8.6 samples/s +Step 33510/44000 | Loss: 2.8960 | LR: 6.91e-05 | 2289.7ms/step | 8.7 samples/s +Step 33520/44000 | Loss: 2.8659 | LR: 6.90e-05 | 2324.5ms/step | 8.6 samples/s +Step 33530/44000 | Loss: 2.8636 | LR: 6.89e-05 | 2328.5ms/step | 8.6 samples/s +Step 33540/44000 | Loss: 2.8660 | LR: 6.88e-05 | 2312.8ms/step | 8.6 samples/s +Step 33550/44000 | Loss: 2.8706 | LR: 6.86e-05 | 2358.8ms/step | 8.5 samples/s +Step 33560/44000 | Loss: 2.8486 | LR: 6.85e-05 | 2318.1ms/step | 8.6 samples/s +Step 33570/44000 | Loss: 2.8839 | LR: 6.84e-05 | 2349.4ms/step | 8.5 samples/s +Step 33580/44000 | Loss: 2.9108 | LR: 6.83e-05 | 2337.6ms/step | 8.6 samples/s +Step 33590/44000 | Loss: 2.9199 | LR: 6.81e-05 | 2290.0ms/step | 8.7 samples/s +Step 33600/44000 | Loss: 2.8962 | LR: 6.80e-05 | 2295.5ms/step | 8.7 samples/s +Step 33610/44000 | Loss: 2.8770 | LR: 6.79e-05 | 2353.5ms/step | 8.5 samples/s +Step 33620/44000 | Loss: 2.9215 | LR: 6.78e-05 | 2316.1ms/step | 8.6 samples/s +Step 33630/44000 | Loss: 2.9039 | LR: 6.76e-05 | 2320.7ms/step | 8.6 samples/s +Step 33640/44000 | Loss: 2.9041 | LR: 6.75e-05 | 2334.9ms/step | 8.6 samples/s +Step 33650/44000 | Loss: 2.8842 | LR: 6.74e-05 | 2315.0ms/step | 8.6 samples/s +Step 33660/44000 | Loss: 2.8854 | LR: 6.73e-05 | 2284.5ms/step | 8.8 samples/s +Step 33670/44000 | Loss: 2.8898 | LR: 6.71e-05 | 2320.9ms/step | 8.6 samples/s +Step 33680/44000 | Loss: 2.8913 | LR: 6.70e-05 | 2357.8ms/step | 8.5 samples/s +Step 33690/44000 | Loss: 2.8977 | LR: 6.69e-05 | 2314.8ms/step | 8.6 samples/s +Step 33700/44000 | Loss: 2.8709 | LR: 6.68e-05 | 2311.7ms/step | 8.7 samples/s +Step 33710/44000 | Loss: 2.8880 | LR: 6.66e-05 | 2321.3ms/step | 8.6 samples/s +Step 33720/44000 | Loss: 2.8954 | LR: 6.65e-05 | 2310.4ms/step | 8.7 samples/s +Step 33730/44000 | Loss: 2.8513 | LR: 6.64e-05 | 2337.3ms/step | 8.6 samples/s +Step 33740/44000 | Loss: 2.8995 | LR: 6.63e-05 | 2319.2ms/step | 8.6 samples/s +Step 33750/44000 | Loss: 2.8543 | LR: 6.61e-05 | 2343.9ms/step | 8.5 samples/s +Step 33760/44000 | Loss: 2.9153 | LR: 6.60e-05 | 2311.6ms/step | 8.7 samples/s +Step 33770/44000 | Loss: 2.8310 | LR: 6.59e-05 | 2284.3ms/step | 8.8 samples/s +Step 33780/44000 | Loss: 2.8905 | LR: 6.58e-05 | 2358.4ms/step | 8.5 samples/s +Step 33790/44000 | Loss: 2.8785 | LR: 6.57e-05 | 2368.2ms/step | 8.4 samples/s +Step 33800/44000 | Loss: 2.8759 | LR: 6.55e-05 | 2313.9ms/step | 8.6 samples/s +Step 33810/44000 | Loss: 2.8279 | LR: 6.54e-05 | 2316.4ms/step | 8.6 samples/s +Step 33820/44000 | Loss: 2.9115 | LR: 6.53e-05 | 2316.7ms/step | 8.6 samples/s +Step 33830/44000 | Loss: 2.8871 | LR: 6.52e-05 | 2311.3ms/step | 8.7 samples/s +Step 33840/44000 | Loss: 2.8986 | LR: 6.50e-05 | 2296.1ms/step | 8.7 samples/s +Step 33850/44000 | Loss: 2.8736 | LR: 6.49e-05 | 2285.2ms/step | 8.8 samples/s +Step 33860/44000 | Loss: 2.8814 | LR: 6.48e-05 | 2325.0ms/step | 8.6 samples/s +Step 33870/44000 | Loss: 2.8469 | LR: 6.47e-05 | 2333.3ms/step | 8.6 samples/s +Step 33880/44000 | Loss: 2.8454 | LR: 6.46e-05 | 2325.0ms/step | 8.6 samples/s +Step 33890/44000 | Loss: 2.8469 | LR: 6.44e-05 | 2321.9ms/step | 8.6 samples/s +Step 33900/44000 | Loss: 2.8732 | LR: 6.43e-05 | 2265.0ms/step | 8.8 samples/s +Step 33910/44000 | Loss: 2.8758 | LR: 6.42e-05 | 2320.1ms/step | 8.6 samples/s +Step 33920/44000 | Loss: 2.8450 | LR: 6.41e-05 | 2343.5ms/step | 8.5 samples/s +Step 33930/44000 | Loss: 2.8785 | LR: 6.40e-05 | 2217.2ms/step | 9.0 samples/s +Step 33940/44000 | Loss: 2.8926 | LR: 6.38e-05 | 2204.0ms/step | 9.1 samples/s +Step 33950/44000 | Loss: 2.8897 | LR: 6.37e-05 | 2311.0ms/step | 8.7 samples/s +Step 33960/44000 | Loss: 2.8779 | LR: 6.36e-05 | 2304.4ms/step | 8.7 samples/s +Step 33970/44000 | Loss: 2.9082 | LR: 6.35e-05 | 2300.8ms/step | 8.7 samples/s +Step 33980/44000 | Loss: 2.8273 | LR: 6.33e-05 | 2323.5ms/step | 8.6 samples/s +Step 33990/44000 | Loss: 2.8956 | LR: 6.32e-05 | 2337.2ms/step | 8.6 samples/s +Step 34000/44000 | Loss: 2.9119 | LR: 6.31e-05 | 2310.5ms/step | 8.7 samples/s +Validation at step 34000: Loss = 2.8763, Perplexity = 17.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-34000 +Step 34010/44000 | Loss: 2.8765 | LR: 6.30e-05 | 4749.2ms/step | 4.2 samples/s +Step 34020/44000 | Loss: 2.8843 | LR: 6.29e-05 | 2327.7ms/step | 8.6 samples/s +Step 34030/44000 | Loss: 2.9067 | LR: 6.27e-05 | 2334.3ms/step | 8.6 samples/s +Step 34040/44000 | Loss: 2.8546 | LR: 6.26e-05 | 2329.2ms/step | 8.6 samples/s +Step 34050/44000 | Loss: 2.9003 | LR: 6.25e-05 | 2338.2ms/step | 8.6 samples/s +Step 34060/44000 | Loss: 2.8795 | LR: 6.24e-05 | 2311.1ms/step | 8.7 samples/s +Step 34070/44000 | Loss: 2.8735 | LR: 6.23e-05 | 2329.2ms/step | 8.6 samples/s +Step 34080/44000 | Loss: 2.8386 | LR: 6.21e-05 | 2324.4ms/step | 8.6 samples/s +Step 34090/44000 | Loss: 2.8585 | LR: 6.20e-05 | 2300.8ms/step | 8.7 samples/s +Step 34100/44000 | Loss: 2.9238 | LR: 6.19e-05 | 2280.2ms/step | 8.8 samples/s +Step 34110/44000 | Loss: 2.8781 | LR: 6.18e-05 | 2331.4ms/step | 8.6 samples/s +Step 34120/44000 | Loss: 2.8443 | LR: 6.17e-05 | 2317.3ms/step | 8.6 samples/s +Step 34130/44000 | Loss: 2.8983 | LR: 6.15e-05 | 2341.5ms/step | 8.5 samples/s +Step 34140/44000 | Loss: 2.9125 | LR: 6.14e-05 | 2337.5ms/step | 8.6 samples/s +Step 34150/44000 | Loss: 2.9017 | LR: 6.13e-05 | 2326.8ms/step | 8.6 samples/s +Step 34160/44000 | Loss: 2.8985 | LR: 6.12e-05 | 2300.6ms/step | 8.7 samples/s +Step 34170/44000 | Loss: 2.8990 | LR: 6.11e-05 | 2300.8ms/step | 8.7 samples/s +Step 34180/44000 | Loss: 2.8822 | LR: 6.09e-05 | 2305.9ms/step | 8.7 samples/s +Step 34190/44000 | Loss: 2.8810 | LR: 6.08e-05 | 2316.4ms/step | 8.6 samples/s +Step 34200/44000 | Loss: 2.8912 | LR: 6.07e-05 | 2320.8ms/step | 8.6 samples/s +Step 34210/44000 | Loss: 2.8890 | LR: 6.06e-05 | 2311.8ms/step | 8.7 samples/s +Step 34220/44000 | Loss: 2.8708 | LR: 6.05e-05 | 2358.3ms/step | 8.5 samples/s +Step 34230/44000 | Loss: 2.8622 | LR: 6.04e-05 | 2321.7ms/step | 8.6 samples/s +Step 34240/44000 | Loss: 2.8488 | LR: 6.02e-05 | 2329.3ms/step | 8.6 samples/s +Step 34250/44000 | Loss: 2.8744 | LR: 6.01e-05 | 2328.5ms/step | 8.6 samples/s +Step 34260/44000 | Loss: 2.8813 | LR: 6.00e-05 | 2356.6ms/step | 8.5 samples/s +Step 34270/44000 | Loss: 2.8906 | LR: 5.99e-05 | 2344.1ms/step | 8.5 samples/s +Step 34280/44000 | Loss: 2.9140 | LR: 5.98e-05 | 2305.2ms/step | 8.7 samples/s +Step 34290/44000 | Loss: 2.8627 | LR: 5.96e-05 | 2323.6ms/step | 8.6 samples/s +Step 34300/44000 | Loss: 2.8861 | LR: 5.95e-05 | 2316.8ms/step | 8.6 samples/s +Step 34310/44000 | Loss: 2.9294 | LR: 5.94e-05 | 2314.3ms/step | 8.6 samples/s +Step 34320/44000 | Loss: 2.8687 | LR: 5.93e-05 | 2337.1ms/step | 8.6 samples/s +Step 34330/44000 | Loss: 2.8542 | LR: 5.92e-05 | 2324.4ms/step | 8.6 samples/s +Step 34340/44000 | Loss: 2.8845 | LR: 5.91e-05 | 2305.5ms/step | 8.7 samples/s +Step 34350/44000 | Loss: 2.8705 | LR: 5.89e-05 | 2345.5ms/step | 8.5 samples/s +Step 34360/44000 | Loss: 2.8884 | LR: 5.88e-05 | 2332.2ms/step | 8.6 samples/s +Step 34370/44000 | Loss: 2.9056 | LR: 5.87e-05 | 2317.0ms/step | 8.6 samples/s +Step 34380/44000 | Loss: 2.8996 | LR: 5.86e-05 | 2317.2ms/step | 8.6 samples/s +Step 34390/44000 | Loss: 2.8604 | LR: 5.85e-05 | 2299.4ms/step | 8.7 samples/s +Step 34400/44000 | Loss: 2.8861 | LR: 5.84e-05 | 2329.4ms/step | 8.6 samples/s +Step 34410/44000 | Loss: 2.9189 | LR: 5.82e-05 | 2319.5ms/step | 8.6 samples/s +Step 34420/44000 | Loss: 2.8919 | LR: 5.81e-05 | 2297.9ms/step | 8.7 samples/s +Step 34430/44000 | Loss: 2.9069 | LR: 5.80e-05 | 2333.0ms/step | 8.6 samples/s +Step 34440/44000 | Loss: 2.8728 | LR: 5.79e-05 | 2324.4ms/step | 8.6 samples/s +Step 34450/44000 | Loss: 2.9282 | LR: 5.78e-05 | 2315.5ms/step | 8.6 samples/s +Step 34460/44000 | Loss: 2.8291 | LR: 5.77e-05 | 2331.7ms/step | 8.6 samples/s +Step 34470/44000 | Loss: 2.8786 | LR: 5.75e-05 | 2327.0ms/step | 8.6 samples/s +Step 34480/44000 | Loss: 2.8792 | LR: 5.74e-05 | 2332.8ms/step | 8.6 samples/s +Step 34490/44000 | Loss: 2.9014 | LR: 5.73e-05 | 2308.8ms/step | 8.7 samples/s +Step 34500/44000 | Loss: 2.8899 | LR: 5.72e-05 | 2346.9ms/step | 8.5 samples/s +Step 34510/44000 | Loss: 2.9244 | LR: 5.71e-05 | 2315.9ms/step | 8.6 samples/s +Step 34520/44000 | Loss: 2.8711 | LR: 5.70e-05 | 2312.4ms/step | 8.6 samples/s +Step 34530/44000 | Loss: 2.9000 | LR: 5.69e-05 | 2315.6ms/step | 8.6 samples/s +Step 34540/44000 | Loss: 2.8815 | LR: 5.67e-05 | 2264.6ms/step | 8.8 samples/s +Step 34550/44000 | Loss: 2.8695 | LR: 5.66e-05 | 2300.5ms/step | 8.7 samples/s +Step 34560/44000 | Loss: 2.8547 | LR: 5.65e-05 | 2305.4ms/step | 8.7 samples/s +Step 34570/44000 | Loss: 2.8387 | LR: 5.64e-05 | 2310.3ms/step | 8.7 samples/s +Step 34580/44000 | Loss: 2.9043 | LR: 5.63e-05 | 2325.6ms/step | 8.6 samples/s +Step 34590/44000 | Loss: 2.9152 | LR: 5.62e-05 | 2325.3ms/step | 8.6 samples/s +Step 34600/44000 | Loss: 2.9258 | LR: 5.60e-05 | 2338.7ms/step | 8.6 samples/s +Step 34610/44000 | Loss: 2.8935 | LR: 5.59e-05 | 2347.8ms/step | 8.5 samples/s +Step 34620/44000 | Loss: 2.9159 | LR: 5.58e-05 | 2317.1ms/step | 8.6 samples/s +Step 34630/44000 | Loss: 2.8703 | LR: 5.57e-05 | 2311.9ms/step | 8.7 samples/s +Step 34640/44000 | Loss: 2.8764 | LR: 5.56e-05 | 2322.1ms/step | 8.6 samples/s +Step 34650/44000 | Loss: 2.8719 | LR: 5.55e-05 | 2291.0ms/step | 8.7 samples/s +Step 34660/44000 | Loss: 2.8843 | LR: 5.54e-05 | 2302.5ms/step | 8.7 samples/s +Step 34670/44000 | Loss: 2.9238 | LR: 5.52e-05 | 2299.3ms/step | 8.7 samples/s +Step 34680/44000 | Loss: 2.9162 | LR: 5.51e-05 | 2305.6ms/step | 8.7 samples/s +Step 34690/44000 | Loss: 2.8597 | LR: 5.50e-05 | 2316.6ms/step | 8.6 samples/s +Step 34700/44000 | Loss: 2.8839 | LR: 5.49e-05 | 2321.0ms/step | 8.6 samples/s +Step 34710/44000 | Loss: 2.8248 | LR: 5.48e-05 | 2335.7ms/step | 8.6 samples/s +Step 34720/44000 | Loss: 2.8735 | LR: 5.47e-05 | 2302.1ms/step | 8.7 samples/s +Step 34730/44000 | Loss: 2.8924 | LR: 5.46e-05 | 2320.4ms/step | 8.6 samples/s +Step 34740/44000 | Loss: 2.9005 | LR: 5.45e-05 | 2317.3ms/step | 8.6 samples/s +Step 34750/44000 | Loss: 2.8660 | LR: 5.43e-05 | 2298.6ms/step | 8.7 samples/s +Step 34760/44000 | Loss: 2.8899 | LR: 5.42e-05 | 2297.6ms/step | 8.7 samples/s +Step 34770/44000 | Loss: 2.8817 | LR: 5.41e-05 | 2303.5ms/step | 8.7 samples/s +Step 34780/44000 | Loss: 2.8980 | LR: 5.40e-05 | 2315.0ms/step | 8.6 samples/s +Step 34790/44000 | Loss: 2.9137 | LR: 5.39e-05 | 2275.4ms/step | 8.8 samples/s +Step 34800/44000 | Loss: 2.8863 | LR: 5.38e-05 | 2254.2ms/step | 8.9 samples/s +Step 34810/44000 | Loss: 2.8807 | LR: 5.37e-05 | 2237.1ms/step | 8.9 samples/s +Step 34820/44000 | Loss: 2.9231 | LR: 5.36e-05 | 2236.4ms/step | 8.9 samples/s +Step 34830/44000 | Loss: 2.9017 | LR: 5.34e-05 | 2256.2ms/step | 8.9 samples/s +Step 34840/44000 | Loss: 2.8666 | LR: 5.33e-05 | 2253.8ms/step | 8.9 samples/s +Step 34850/44000 | Loss: 2.8773 | LR: 5.32e-05 | 2280.0ms/step | 8.8 samples/s +Step 34860/44000 | Loss: 2.9101 | LR: 5.31e-05 | 2253.2ms/step | 8.9 samples/s +Step 34870/44000 | Loss: 2.8911 | LR: 5.30e-05 | 2268.4ms/step | 8.8 samples/s +Step 34880/44000 | Loss: 2.8651 | LR: 5.29e-05 | 2253.9ms/step | 8.9 samples/s +Step 34890/44000 | Loss: 2.8716 | LR: 5.28e-05 | 2266.7ms/step | 8.8 samples/s +Step 34900/44000 | Loss: 2.8860 | LR: 5.27e-05 | 2282.6ms/step | 8.8 samples/s +Step 34910/44000 | Loss: 2.9119 | LR: 5.25e-05 | 2271.2ms/step | 8.8 samples/s +Step 34920/44000 | Loss: 2.8772 | LR: 5.24e-05 | 2271.1ms/step | 8.8 samples/s +Step 34930/44000 | Loss: 2.8673 | LR: 5.23e-05 | 2267.0ms/step | 8.8 samples/s +Step 34940/44000 | Loss: 2.8628 | LR: 5.22e-05 | 2272.3ms/step | 8.8 samples/s +Step 34950/44000 | Loss: 2.8507 | LR: 5.21e-05 | 2217.8ms/step | 9.0 samples/s +Step 34960/44000 | Loss: 2.8852 | LR: 5.20e-05 | 2114.5ms/step | 9.5 samples/s +Step 34970/44000 | Loss: 2.9329 | LR: 5.19e-05 | 2245.9ms/step | 8.9 samples/s +Step 34980/44000 | Loss: 2.8739 | LR: 5.18e-05 | 2267.7ms/step | 8.8 samples/s +Step 34990/44000 | Loss: 2.8797 | LR: 5.17e-05 | 2225.3ms/step | 9.0 samples/s +Step 35000/44000 | Loss: 2.9254 | LR: 5.15e-05 | 2262.1ms/step | 8.8 samples/s +Validation at step 35000: Loss = 2.8752, Perplexity = 17.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-35000 +Step 35010/44000 | Loss: 2.8511 | LR: 5.14e-05 | 4769.4ms/step | 4.2 samples/s +Step 35020/44000 | Loss: 2.8470 | LR: 5.13e-05 | 2263.2ms/step | 8.8 samples/s +Step 35030/44000 | Loss: 2.8493 | LR: 5.12e-05 | 2262.7ms/step | 8.8 samples/s +Step 35040/44000 | Loss: 2.8611 | LR: 5.11e-05 | 2236.5ms/step | 8.9 samples/s +Step 35050/44000 | Loss: 2.9033 | LR: 5.10e-05 | 2262.9ms/step | 8.8 samples/s +Step 35060/44000 | Loss: 2.8609 | LR: 5.09e-05 | 2255.9ms/step | 8.9 samples/s +Step 35070/44000 | Loss: 2.9261 | LR: 5.08e-05 | 2235.2ms/step | 8.9 samples/s +Step 35080/44000 | Loss: 2.8809 | LR: 5.07e-05 | 2282.3ms/step | 8.8 samples/s +Step 35090/44000 | Loss: 2.8589 | LR: 5.06e-05 | 2262.0ms/step | 8.8 samples/s +Step 35100/44000 | Loss: 2.8938 | LR: 5.04e-05 | 2223.1ms/step | 9.0 samples/s +Step 35110/44000 | Loss: 2.8684 | LR: 5.03e-05 | 2250.5ms/step | 8.9 samples/s +Step 35120/44000 | Loss: 2.8849 | LR: 5.02e-05 | 2237.8ms/step | 8.9 samples/s +Step 35130/44000 | Loss: 2.8821 | LR: 5.01e-05 | 2259.5ms/step | 8.9 samples/s +Step 35140/44000 | Loss: 2.9223 | LR: 5.00e-05 | 2283.7ms/step | 8.8 samples/s +Step 35150/44000 | Loss: 2.8281 | LR: 4.99e-05 | 2259.1ms/step | 8.9 samples/s +Step 35160/44000 | Loss: 2.9011 | LR: 4.98e-05 | 2252.5ms/step | 8.9 samples/s +Step 35170/44000 | Loss: 2.8840 | LR: 4.97e-05 | 2277.5ms/step | 8.8 samples/s +Step 35180/44000 | Loss: 2.8844 | LR: 4.96e-05 | 2263.8ms/step | 8.8 samples/s +Step 35190/44000 | Loss: 2.8503 | LR: 4.95e-05 | 2260.9ms/step | 8.8 samples/s +Step 35200/44000 | Loss: 2.8661 | LR: 4.94e-05 | 2278.3ms/step | 8.8 samples/s +Step 35210/44000 | Loss: 2.8812 | LR: 4.93e-05 | 2279.7ms/step | 8.8 samples/s +Step 35220/44000 | Loss: 2.9299 | LR: 4.91e-05 | 2295.0ms/step | 8.7 samples/s +Step 35230/44000 | Loss: 2.9016 | LR: 4.90e-05 | 2262.1ms/step | 8.8 samples/s +Step 35240/44000 | Loss: 2.8568 | LR: 4.89e-05 | 2249.5ms/step | 8.9 samples/s +Step 35250/44000 | Loss: 2.8627 | LR: 4.88e-05 | 2236.6ms/step | 8.9 samples/s +Step 35260/44000 | Loss: 2.8215 | LR: 4.87e-05 | 2273.9ms/step | 8.8 samples/s +Step 35270/44000 | Loss: 2.9062 | LR: 4.86e-05 | 2269.4ms/step | 8.8 samples/s +Step 35280/44000 | Loss: 2.8890 | LR: 4.85e-05 | 2295.4ms/step | 8.7 samples/s +Step 35290/44000 | Loss: 2.8774 | LR: 4.84e-05 | 2294.3ms/step | 8.7 samples/s +Step 35300/44000 | Loss: 2.8836 | LR: 4.83e-05 | 2238.7ms/step | 8.9 samples/s +Step 35310/44000 | Loss: 2.8497 | LR: 4.82e-05 | 2250.6ms/step | 8.9 samples/s +Step 35320/44000 | Loss: 2.8890 | LR: 4.81e-05 | 2227.8ms/step | 9.0 samples/s +Step 35330/44000 | Loss: 2.8831 | LR: 4.80e-05 | 2277.0ms/step | 8.8 samples/s +Step 35340/44000 | Loss: 2.9171 | LR: 4.79e-05 | 2264.7ms/step | 8.8 samples/s +Step 35350/44000 | Loss: 2.8674 | LR: 4.77e-05 | 2276.1ms/step | 8.8 samples/s +Step 35360/44000 | Loss: 2.8697 | LR: 4.76e-05 | 2280.0ms/step | 8.8 samples/s +Step 35370/44000 | Loss: 2.9007 | LR: 4.75e-05 | 2273.9ms/step | 8.8 samples/s +Step 35380/44000 | Loss: 2.9178 | LR: 4.74e-05 | 2285.1ms/step | 8.8 samples/s +Step 35390/44000 | Loss: 2.8404 | LR: 4.73e-05 | 2252.9ms/step | 8.9 samples/s +Step 35400/44000 | Loss: 2.8961 | LR: 4.72e-05 | 2279.2ms/step | 8.8 samples/s +Step 35410/44000 | Loss: 2.8548 | LR: 4.71e-05 | 2262.5ms/step | 8.8 samples/s +Step 35420/44000 | Loss: 2.8642 | LR: 4.70e-05 | 2267.3ms/step | 8.8 samples/s +Step 35430/44000 | Loss: 2.8587 | LR: 4.69e-05 | 2240.8ms/step | 8.9 samples/s +Step 35440/44000 | Loss: 2.9555 | LR: 4.68e-05 | 2269.6ms/step | 8.8 samples/s +Step 35450/44000 | Loss: 2.9014 | LR: 4.67e-05 | 2254.9ms/step | 8.9 samples/s +Step 35460/44000 | Loss: 2.9346 | LR: 4.66e-05 | 2276.0ms/step | 8.8 samples/s +Step 35470/44000 | Loss: 2.8494 | LR: 4.65e-05 | 2228.4ms/step | 9.0 samples/s +Step 35480/44000 | Loss: 2.8892 | LR: 4.64e-05 | 2292.1ms/step | 8.7 samples/s +Step 35490/44000 | Loss: 2.8999 | LR: 4.63e-05 | 2267.3ms/step | 8.8 samples/s +Step 35500/44000 | Loss: 2.8741 | LR: 4.62e-05 | 2254.6ms/step | 8.9 samples/s +Step 35510/44000 | Loss: 2.8922 | LR: 4.61e-05 | 2259.3ms/step | 8.9 samples/s +Step 35520/44000 | Loss: 2.8587 | LR: 4.59e-05 | 2257.6ms/step | 8.9 samples/s +Step 35530/44000 | Loss: 2.8856 | LR: 4.58e-05 | 2278.7ms/step | 8.8 samples/s +Step 35540/44000 | Loss: 2.9154 | LR: 4.57e-05 | 2257.7ms/step | 8.9 samples/s +Step 35550/44000 | Loss: 2.8686 | LR: 4.56e-05 | 2265.0ms/step | 8.8 samples/s +Step 35560/44000 | Loss: 2.9144 | LR: 4.55e-05 | 2287.5ms/step | 8.7 samples/s +Step 35570/44000 | Loss: 2.8388 | LR: 4.54e-05 | 2263.3ms/step | 8.8 samples/s +Step 35580/44000 | Loss: 2.8970 | LR: 4.53e-05 | 2254.5ms/step | 8.9 samples/s +Step 35590/44000 | Loss: 2.9186 | LR: 4.52e-05 | 2240.5ms/step | 8.9 samples/s +Step 35600/44000 | Loss: 2.8857 | LR: 4.51e-05 | 2278.2ms/step | 8.8 samples/s +Step 35610/44000 | Loss: 2.8761 | LR: 4.50e-05 | 2277.4ms/step | 8.8 samples/s +Step 35620/44000 | Loss: 2.8879 | LR: 4.49e-05 | 2275.3ms/step | 8.8 samples/s +Step 35630/44000 | Loss: 2.9011 | LR: 4.48e-05 | 2257.8ms/step | 8.9 samples/s +Step 35640/44000 | Loss: 2.9074 | LR: 4.47e-05 | 2281.0ms/step | 8.8 samples/s +Step 35650/44000 | Loss: 2.8867 | LR: 4.46e-05 | 2271.4ms/step | 8.8 samples/s +Step 35660/44000 | Loss: 2.8926 | LR: 4.45e-05 | 2263.1ms/step | 8.8 samples/s +Step 35670/44000 | Loss: 2.8872 | LR: 4.44e-05 | 2278.9ms/step | 8.8 samples/s +Step 35680/44000 | Loss: 2.8956 | LR: 4.43e-05 | 2270.2ms/step | 8.8 samples/s +Step 35690/44000 | Loss: 2.9008 | LR: 4.42e-05 | 2289.1ms/step | 8.7 samples/s +Step 35700/44000 | Loss: 2.8765 | LR: 4.41e-05 | 2268.5ms/step | 8.8 samples/s +Step 35710/44000 | Loss: 2.8929 | LR: 4.40e-05 | 2263.7ms/step | 8.8 samples/s +Step 35720/44000 | Loss: 2.8872 | LR: 4.39e-05 | 2258.9ms/step | 8.9 samples/s +Step 35730/44000 | Loss: 2.8461 | LR: 4.38e-05 | 2237.3ms/step | 8.9 samples/s +Step 35740/44000 | Loss: 2.8946 | LR: 4.37e-05 | 2211.1ms/step | 9.0 samples/s +Step 35750/44000 | Loss: 2.8659 | LR: 4.36e-05 | 2238.2ms/step | 8.9 samples/s +Step 35760/44000 | Loss: 2.8821 | LR: 4.35e-05 | 2232.6ms/step | 9.0 samples/s +Step 35770/44000 | Loss: 2.8836 | LR: 4.34e-05 | 2252.3ms/step | 8.9 samples/s +Step 35780/44000 | Loss: 2.8497 | LR: 4.33e-05 | 2249.6ms/step | 8.9 samples/s +Step 35790/44000 | Loss: 2.8877 | LR: 4.32e-05 | 2299.7ms/step | 8.7 samples/s +Step 35800/44000 | Loss: 2.8653 | LR: 4.31e-05 | 2268.9ms/step | 8.8 samples/s +Step 35810/44000 | Loss: 2.8672 | LR: 4.29e-05 | 2263.3ms/step | 8.8 samples/s +Step 35820/44000 | Loss: 2.8468 | LR: 4.28e-05 | 2248.5ms/step | 8.9 samples/s +Step 35830/44000 | Loss: 2.9105 | LR: 4.27e-05 | 2253.6ms/step | 8.9 samples/s +Step 35840/44000 | Loss: 2.8823 | LR: 4.26e-05 | 2245.3ms/step | 8.9 samples/s +Step 35850/44000 | Loss: 2.8774 | LR: 4.25e-05 | 2266.7ms/step | 8.8 samples/s +Step 35860/44000 | Loss: 2.9446 | LR: 4.24e-05 | 2269.8ms/step | 8.8 samples/s +Step 35870/44000 | Loss: 2.8722 | LR: 4.23e-05 | 2282.0ms/step | 8.8 samples/s +Step 35880/44000 | Loss: 2.9251 | LR: 4.22e-05 | 2252.2ms/step | 8.9 samples/s +Step 35890/44000 | Loss: 2.8878 | LR: 4.21e-05 | 2278.2ms/step | 8.8 samples/s +Step 35900/44000 | Loss: 2.8922 | LR: 4.20e-05 | 2286.4ms/step | 8.7 samples/s +Step 35910/44000 | Loss: 2.8660 | LR: 4.19e-05 | 2244.2ms/step | 8.9 samples/s +Step 35920/44000 | Loss: 2.8354 | LR: 4.18e-05 | 2256.1ms/step | 8.9 samples/s +Step 35930/44000 | Loss: 2.8556 | LR: 4.17e-05 | 2249.7ms/step | 8.9 samples/s +Step 35940/44000 | Loss: 2.8837 | LR: 4.16e-05 | 2271.6ms/step | 8.8 samples/s +Step 35950/44000 | Loss: 2.8954 | LR: 4.15e-05 | 2284.3ms/step | 8.8 samples/s +Step 35960/44000 | Loss: 2.8673 | LR: 4.14e-05 | 2197.9ms/step | 9.1 samples/s +Step 35970/44000 | Loss: 2.9256 | LR: 4.13e-05 | 2107.7ms/step | 9.5 samples/s +Step 35980/44000 | Loss: 2.8751 | LR: 4.12e-05 | 2254.8ms/step | 8.9 samples/s +Step 35990/44000 | Loss: 2.8637 | LR: 4.11e-05 | 2251.8ms/step | 8.9 samples/s +Step 36000/44000 | Loss: 2.8947 | LR: 4.10e-05 | 2237.0ms/step | 8.9 samples/s +Validation at step 36000: Loss = 2.8742, Perplexity = 17.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-36000 +Step 36010/44000 | Loss: 2.8640 | LR: 4.09e-05 | 4700.0ms/step | 4.3 samples/s +Step 36020/44000 | Loss: 2.8639 | LR: 4.08e-05 | 2250.1ms/step | 8.9 samples/s +Step 36030/44000 | Loss: 2.8986 | LR: 4.07e-05 | 2277.7ms/step | 8.8 samples/s +Step 36040/44000 | Loss: 2.8447 | LR: 4.06e-05 | 2279.7ms/step | 8.8 samples/s +Step 36050/44000 | Loss: 2.8779 | LR: 4.05e-05 | 2257.6ms/step | 8.9 samples/s +Step 36060/44000 | Loss: 2.8635 | LR: 4.04e-05 | 2243.9ms/step | 8.9 samples/s +Step 36070/44000 | Loss: 2.8924 | LR: 4.03e-05 | 2253.7ms/step | 8.9 samples/s +Step 36080/44000 | Loss: 2.8925 | LR: 4.02e-05 | 2237.0ms/step | 8.9 samples/s +Step 36090/44000 | Loss: 2.8975 | LR: 4.01e-05 | 2269.3ms/step | 8.8 samples/s +Step 36100/44000 | Loss: 2.9352 | LR: 4.00e-05 | 2284.2ms/step | 8.8 samples/s +Step 36110/44000 | Loss: 2.8752 | LR: 3.99e-05 | 2241.8ms/step | 8.9 samples/s +Step 36120/44000 | Loss: 2.8963 | LR: 3.98e-05 | 2268.0ms/step | 8.8 samples/s +Step 36130/44000 | Loss: 2.8773 | LR: 3.97e-05 | 2232.6ms/step | 9.0 samples/s +Step 36140/44000 | Loss: 2.8955 | LR: 3.97e-05 | 2259.6ms/step | 8.9 samples/s +Step 36150/44000 | Loss: 2.8874 | LR: 3.96e-05 | 2287.3ms/step | 8.7 samples/s +Step 36160/44000 | Loss: 2.8972 | LR: 3.95e-05 | 2280.8ms/step | 8.8 samples/s +Step 36170/44000 | Loss: 2.8729 | LR: 3.94e-05 | 2255.1ms/step | 8.9 samples/s +Step 36180/44000 | Loss: 2.9142 | LR: 3.93e-05 | 2260.9ms/step | 8.8 samples/s +Step 36190/44000 | Loss: 2.9322 | LR: 3.92e-05 | 2279.8ms/step | 8.8 samples/s +Step 36200/44000 | Loss: 2.9006 | LR: 3.91e-05 | 2242.7ms/step | 8.9 samples/s +Step 36210/44000 | Loss: 2.8366 | LR: 3.90e-05 | 2283.2ms/step | 8.8 samples/s +Step 36220/44000 | Loss: 2.9182 | LR: 3.89e-05 | 2236.8ms/step | 8.9 samples/s +Step 36230/44000 | Loss: 2.8737 | LR: 3.88e-05 | 2257.2ms/step | 8.9 samples/s +Step 36240/44000 | Loss: 2.8960 | LR: 3.87e-05 | 2304.5ms/step | 8.7 samples/s +Step 36250/44000 | Loss: 2.8655 | LR: 3.86e-05 | 2252.6ms/step | 8.9 samples/s +Step 36260/44000 | Loss: 2.9128 | LR: 3.85e-05 | 2252.1ms/step | 8.9 samples/s +Step 36270/44000 | Loss: 2.9171 | LR: 3.84e-05 | 2237.4ms/step | 8.9 samples/s +Step 36280/44000 | Loss: 2.8218 | LR: 3.83e-05 | 2249.7ms/step | 8.9 samples/s +Step 36290/44000 | Loss: 2.8737 | LR: 3.82e-05 | 2239.8ms/step | 8.9 samples/s +Step 36300/44000 | Loss: 2.8476 | LR: 3.81e-05 | 2281.9ms/step | 8.8 samples/s +Step 36310/44000 | Loss: 2.8877 | LR: 3.80e-05 | 2266.8ms/step | 8.8 samples/s +Step 36320/44000 | Loss: 2.8879 | LR: 3.79e-05 | 2255.6ms/step | 8.9 samples/s +Step 36330/44000 | Loss: 2.9046 | LR: 3.78e-05 | 2240.3ms/step | 8.9 samples/s +Step 36340/44000 | Loss: 2.9273 | LR: 3.77e-05 | 2263.0ms/step | 8.8 samples/s +Step 36350/44000 | Loss: 2.9019 | LR: 3.76e-05 | 2260.0ms/step | 8.8 samples/s +Step 36360/44000 | Loss: 2.8989 | LR: 3.75e-05 | 2276.9ms/step | 8.8 samples/s +Step 36370/44000 | Loss: 2.8681 | LR: 3.74e-05 | 2270.2ms/step | 8.8 samples/s +Step 36380/44000 | Loss: 2.8866 | LR: 3.73e-05 | 2258.3ms/step | 8.9 samples/s +Step 36390/44000 | Loss: 2.9186 | LR: 3.72e-05 | 2293.9ms/step | 8.7 samples/s +Step 36400/44000 | Loss: 2.8915 | LR: 3.71e-05 | 2238.6ms/step | 8.9 samples/s +Step 36410/44000 | Loss: 2.8597 | LR: 3.70e-05 | 2245.4ms/step | 8.9 samples/s +Step 36420/44000 | Loss: 2.8486 | LR: 3.69e-05 | 2268.1ms/step | 8.8 samples/s +Step 36430/44000 | Loss: 2.9334 | LR: 3.69e-05 | 2252.7ms/step | 8.9 samples/s +Step 36440/44000 | Loss: 2.8504 | LR: 3.68e-05 | 2254.2ms/step | 8.9 samples/s +Step 36450/44000 | Loss: 2.9111 | LR: 3.67e-05 | 2284.7ms/step | 8.8 samples/s +Step 36460/44000 | Loss: 2.8726 | LR: 3.66e-05 | 2284.4ms/step | 8.8 samples/s +Step 36470/44000 | Loss: 2.9075 | LR: 3.65e-05 | 2258.7ms/step | 8.9 samples/s +Step 36480/44000 | Loss: 2.8824 | LR: 3.64e-05 | 2280.9ms/step | 8.8 samples/s +Step 36490/44000 | Loss: 2.9061 | LR: 3.63e-05 | 2260.3ms/step | 8.8 samples/s +Step 36500/44000 | Loss: 2.8795 | LR: 3.62e-05 | 2291.9ms/step | 8.7 samples/s +Step 36510/44000 | Loss: 2.8401 | LR: 3.61e-05 | 2274.1ms/step | 8.8 samples/s +Step 36520/44000 | Loss: 2.8427 | LR: 3.60e-05 | 2241.9ms/step | 8.9 samples/s +Step 36530/44000 | Loss: 2.9200 | LR: 3.59e-05 | 2232.7ms/step | 9.0 samples/s +Step 36540/44000 | Loss: 2.8639 | LR: 3.58e-05 | 2257.0ms/step | 8.9 samples/s +Step 36550/44000 | Loss: 2.8467 | LR: 3.57e-05 | 2257.2ms/step | 8.9 samples/s +Step 36560/44000 | Loss: 2.8801 | LR: 3.56e-05 | 2234.8ms/step | 8.9 samples/s +Step 36570/44000 | Loss: 2.9034 | LR: 3.55e-05 | 2237.8ms/step | 8.9 samples/s +Step 36580/44000 | Loss: 2.8747 | LR: 3.54e-05 | 2241.8ms/step | 8.9 samples/s +Step 36590/44000 | Loss: 2.9067 | LR: 3.53e-05 | 2268.4ms/step | 8.8 samples/s +Step 36600/44000 | Loss: 2.8834 | LR: 3.53e-05 | 2257.5ms/step | 8.9 samples/s +Step 36610/44000 | Loss: 2.8777 | LR: 3.52e-05 | 2257.5ms/step | 8.9 samples/s +Step 36620/44000 | Loss: 2.9182 | LR: 3.51e-05 | 2226.0ms/step | 9.0 samples/s +Step 36630/44000 | Loss: 2.8912 | LR: 3.50e-05 | 2210.5ms/step | 9.0 samples/s +Step 36640/44000 | Loss: 2.8469 | LR: 3.49e-05 | 2256.3ms/step | 8.9 samples/s +Step 36650/44000 | Loss: 2.8489 | LR: 3.48e-05 | 2260.2ms/step | 8.8 samples/s +Step 36660/44000 | Loss: 2.8940 | LR: 3.47e-05 | 2266.5ms/step | 8.8 samples/s +Step 36670/44000 | Loss: 2.9100 | LR: 3.46e-05 | 2264.0ms/step | 8.8 samples/s +Step 36680/44000 | Loss: 2.8802 | LR: 3.45e-05 | 2263.0ms/step | 8.8 samples/s +Step 36690/44000 | Loss: 2.8834 | LR: 3.44e-05 | 2240.1ms/step | 8.9 samples/s +Step 36700/44000 | Loss: 2.8357 | LR: 3.43e-05 | 2263.2ms/step | 8.8 samples/s +Step 36710/44000 | Loss: 2.8967 | LR: 3.42e-05 | 2247.0ms/step | 8.9 samples/s +Step 36720/44000 | Loss: 2.8620 | LR: 3.41e-05 | 2270.6ms/step | 8.8 samples/s +Step 36730/44000 | Loss: 2.8575 | LR: 3.41e-05 | 2247.1ms/step | 8.9 samples/s +Step 36740/44000 | Loss: 2.8636 | LR: 3.40e-05 | 2262.4ms/step | 8.8 samples/s +Step 36750/44000 | Loss: 2.8774 | LR: 3.39e-05 | 2261.6ms/step | 8.8 samples/s +Step 36760/44000 | Loss: 2.8612 | LR: 3.38e-05 | 2248.3ms/step | 8.9 samples/s +Step 36770/44000 | Loss: 2.8757 | LR: 3.37e-05 | 2236.7ms/step | 8.9 samples/s +Step 36780/44000 | Loss: 2.8756 | LR: 3.36e-05 | 2257.9ms/step | 8.9 samples/s +Step 36790/44000 | Loss: 2.8619 | LR: 3.35e-05 | 2268.3ms/step | 8.8 samples/s +Step 36800/44000 | Loss: 2.9452 | LR: 3.34e-05 | 2269.0ms/step | 8.8 samples/s +Step 36810/44000 | Loss: 2.8502 | LR: 3.33e-05 | 2214.9ms/step | 9.0 samples/s +Step 36820/44000 | Loss: 2.8919 | LR: 3.32e-05 | 2275.6ms/step | 8.8 samples/s +Step 36830/44000 | Loss: 2.8920 | LR: 3.31e-05 | 2276.7ms/step | 8.8 samples/s +Step 36840/44000 | Loss: 2.8755 | LR: 3.31e-05 | 2248.2ms/step | 8.9 samples/s +Step 36850/44000 | Loss: 2.8707 | LR: 3.30e-05 | 2239.1ms/step | 8.9 samples/s +Step 36860/44000 | Loss: 2.8633 | LR: 3.29e-05 | 2260.4ms/step | 8.8 samples/s +Step 36870/44000 | Loss: 2.8714 | LR: 3.28e-05 | 2276.7ms/step | 8.8 samples/s +Step 36880/44000 | Loss: 2.8938 | LR: 3.27e-05 | 2272.6ms/step | 8.8 samples/s +Step 36890/44000 | Loss: 2.8664 | LR: 3.26e-05 | 2259.5ms/step | 8.9 samples/s +Step 36900/44000 | Loss: 2.8332 | LR: 3.25e-05 | 2259.5ms/step | 8.9 samples/s +Step 36910/44000 | Loss: 2.8849 | LR: 3.24e-05 | 2278.8ms/step | 8.8 samples/s +Step 36920/44000 | Loss: 2.9041 | LR: 3.23e-05 | 2271.1ms/step | 8.8 samples/s +Step 36930/44000 | Loss: 2.8943 | LR: 3.22e-05 | 2275.8ms/step | 8.8 samples/s +Step 36940/44000 | Loss: 2.9122 | LR: 3.22e-05 | 2237.2ms/step | 8.9 samples/s +Step 36950/44000 | Loss: 2.9162 | LR: 3.21e-05 | 2252.6ms/step | 8.9 samples/s +Step 36960/44000 | Loss: 2.8802 | LR: 3.20e-05 | 2285.9ms/step | 8.7 samples/s +Step 36970/44000 | Loss: 2.8561 | LR: 3.19e-05 | 2263.1ms/step | 8.8 samples/s +Step 36980/44000 | Loss: 2.8557 | LR: 3.18e-05 | 2117.0ms/step | 9.4 samples/s +Step 36990/44000 | Loss: 2.8350 | LR: 3.17e-05 | 2196.9ms/step | 9.1 samples/s +Step 37000/44000 | Loss: 2.9141 | LR: 3.16e-05 | 2261.7ms/step | 8.8 samples/s +Validation at step 37000: Loss = 2.8737, Perplexity = 17.70 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-37000 +Step 37010/44000 | Loss: 2.9364 | LR: 3.15e-05 | 4685.4ms/step | 4.3 samples/s +Step 37020/44000 | Loss: 2.8612 | LR: 3.15e-05 | 2238.3ms/step | 8.9 samples/s +Step 37030/44000 | Loss: 2.9164 | LR: 3.14e-05 | 2287.5ms/step | 8.7 samples/s +Step 37040/44000 | Loss: 2.8339 | LR: 3.13e-05 | 2259.7ms/step | 8.9 samples/s +Step 37050/44000 | Loss: 2.8335 | LR: 3.12e-05 | 2257.2ms/step | 8.9 samples/s +Step 37060/44000 | Loss: 2.8556 | LR: 3.11e-05 | 2219.9ms/step | 9.0 samples/s +Step 37070/44000 | Loss: 2.9039 | LR: 3.10e-05 | 2254.6ms/step | 8.9 samples/s +Step 37080/44000 | Loss: 2.8976 | LR: 3.09e-05 | 2277.3ms/step | 8.8 samples/s +Step 37090/44000 | Loss: 2.9112 | LR: 3.08e-05 | 2267.9ms/step | 8.8 samples/s +Step 37100/44000 | Loss: 2.9246 | LR: 3.07e-05 | 2295.4ms/step | 8.7 samples/s +Step 37110/44000 | Loss: 2.8939 | LR: 3.07e-05 | 2280.3ms/step | 8.8 samples/s +Step 37120/44000 | Loss: 2.8763 | LR: 3.06e-05 | 2264.3ms/step | 8.8 samples/s +Step 37130/44000 | Loss: 2.8714 | LR: 3.05e-05 | 2234.7ms/step | 8.9 samples/s +Step 37140/44000 | Loss: 2.9040 | LR: 3.04e-05 | 2252.7ms/step | 8.9 samples/s +Step 37150/44000 | Loss: 2.8630 | LR: 3.03e-05 | 2283.7ms/step | 8.8 samples/s +Step 37160/44000 | Loss: 2.8995 | LR: 3.02e-05 | 2247.3ms/step | 8.9 samples/s +Step 37170/44000 | Loss: 2.8741 | LR: 3.01e-05 | 2246.4ms/step | 8.9 samples/s +Step 37180/44000 | Loss: 2.8738 | LR: 3.01e-05 | 2279.9ms/step | 8.8 samples/s +Step 37190/44000 | Loss: 2.8691 | LR: 3.00e-05 | 2253.3ms/step | 8.9 samples/s +Step 37200/44000 | Loss: 2.9013 | LR: 2.99e-05 | 2243.9ms/step | 8.9 samples/s +Step 37210/44000 | Loss: 2.8935 | LR: 2.98e-05 | 2235.2ms/step | 8.9 samples/s +Step 37220/44000 | Loss: 2.8449 | LR: 2.97e-05 | 2268.2ms/step | 8.8 samples/s +Step 37230/44000 | Loss: 2.8849 | LR: 2.96e-05 | 2273.1ms/step | 8.8 samples/s +Step 37240/44000 | Loss: 2.9024 | LR: 2.95e-05 | 2247.4ms/step | 8.9 samples/s +Step 37250/44000 | Loss: 2.8564 | LR: 2.95e-05 | 2277.9ms/step | 8.8 samples/s +Step 37260/44000 | Loss: 2.8556 | LR: 2.94e-05 | 2264.0ms/step | 8.8 samples/s +Step 37270/44000 | Loss: 2.9103 | LR: 2.93e-05 | 2251.7ms/step | 8.9 samples/s +Step 37280/44000 | Loss: 2.8664 | LR: 2.92e-05 | 2257.5ms/step | 8.9 samples/s +Step 37290/44000 | Loss: 2.8556 | LR: 2.91e-05 | 2250.3ms/step | 8.9 samples/s +Step 37300/44000 | Loss: 2.8681 | LR: 2.90e-05 | 2245.9ms/step | 8.9 samples/s +Step 37310/44000 | Loss: 2.9292 | LR: 2.89e-05 | 2267.6ms/step | 8.8 samples/s +Step 37320/44000 | Loss: 2.9068 | LR: 2.89e-05 | 2269.8ms/step | 8.8 samples/s +Step 37330/44000 | Loss: 2.8906 | LR: 2.88e-05 | 2246.4ms/step | 8.9 samples/s +Step 37340/44000 | Loss: 2.9189 | LR: 2.87e-05 | 2255.8ms/step | 8.9 samples/s +Step 37350/44000 | Loss: 2.8845 | LR: 2.86e-05 | 2279.9ms/step | 8.8 samples/s +Step 37360/44000 | Loss: 2.8769 | LR: 2.85e-05 | 2279.0ms/step | 8.8 samples/s +Step 37370/44000 | Loss: 2.8953 | LR: 2.84e-05 | 2249.0ms/step | 8.9 samples/s +Step 37380/44000 | Loss: 2.8904 | LR: 2.84e-05 | 2249.0ms/step | 8.9 samples/s +Step 37390/44000 | Loss: 2.8627 | LR: 2.83e-05 | 2260.6ms/step | 8.8 samples/s +Step 37400/44000 | Loss: 2.8366 | LR: 2.82e-05 | 2226.9ms/step | 9.0 samples/s +Step 37410/44000 | Loss: 2.8874 | LR: 2.81e-05 | 2263.9ms/step | 8.8 samples/s +Step 37420/44000 | Loss: 2.8769 | LR: 2.80e-05 | 2248.0ms/step | 8.9 samples/s +Step 37430/44000 | Loss: 2.8746 | LR: 2.79e-05 | 2263.5ms/step | 8.8 samples/s +Step 37440/44000 | Loss: 2.8720 | LR: 2.78e-05 | 2257.4ms/step | 8.9 samples/s +Step 37450/44000 | Loss: 2.8553 | LR: 2.78e-05 | 2282.6ms/step | 8.8 samples/s +Step 37460/44000 | Loss: 2.8874 | LR: 2.77e-05 | 2270.8ms/step | 8.8 samples/s +Step 37470/44000 | Loss: 2.9233 | LR: 2.76e-05 | 2295.4ms/step | 8.7 samples/s +Step 37480/44000 | Loss: 2.8848 | LR: 2.75e-05 | 2252.0ms/step | 8.9 samples/s +Step 37490/44000 | Loss: 2.8730 | LR: 2.74e-05 | 2268.3ms/step | 8.8 samples/s +Step 37500/44000 | Loss: 2.8279 | LR: 2.74e-05 | 2273.7ms/step | 8.8 samples/s +Step 37510/44000 | Loss: 2.8787 | LR: 2.73e-05 | 2288.7ms/step | 8.7 samples/s +Step 37520/44000 | Loss: 2.8882 | LR: 2.72e-05 | 2252.3ms/step | 8.9 samples/s +Step 37530/44000 | Loss: 2.9048 | LR: 2.71e-05 | 2266.4ms/step | 8.8 samples/s +Step 37540/44000 | Loss: 2.9049 | LR: 2.70e-05 | 2239.6ms/step | 8.9 samples/s +Step 37550/44000 | Loss: 2.8945 | LR: 2.69e-05 | 2264.1ms/step | 8.8 samples/s +Step 37560/44000 | Loss: 2.8641 | LR: 2.69e-05 | 2264.7ms/step | 8.8 samples/s +Step 37570/44000 | Loss: 2.8299 | LR: 2.68e-05 | 2272.6ms/step | 8.8 samples/s +Step 37580/44000 | Loss: 2.8648 | LR: 2.67e-05 | 2252.5ms/step | 8.9 samples/s +Step 37590/44000 | Loss: 2.8445 | LR: 2.66e-05 | 2253.4ms/step | 8.9 samples/s +Step 37600/44000 | Loss: 2.8660 | LR: 2.65e-05 | 2261.8ms/step | 8.8 samples/s +Step 37610/44000 | Loss: 2.8502 | LR: 2.65e-05 | 2259.6ms/step | 8.9 samples/s +Step 37620/44000 | Loss: 2.8537 | LR: 2.64e-05 | 2253.6ms/step | 8.9 samples/s +Step 37630/44000 | Loss: 2.8376 | LR: 2.63e-05 | 2263.5ms/step | 8.8 samples/s +Step 37640/44000 | Loss: 2.9440 | LR: 2.62e-05 | 2284.2ms/step | 8.8 samples/s +Step 37650/44000 | Loss: 2.8686 | LR: 2.61e-05 | 2258.0ms/step | 8.9 samples/s +Step 37660/44000 | Loss: 2.8904 | LR: 2.60e-05 | 2294.1ms/step | 8.7 samples/s +Step 37670/44000 | Loss: 2.8678 | LR: 2.60e-05 | 2264.0ms/step | 8.8 samples/s +Step 37680/44000 | Loss: 2.8742 | LR: 2.59e-05 | 2234.8ms/step | 8.9 samples/s +Step 37690/44000 | Loss: 2.8678 | LR: 2.58e-05 | 2254.8ms/step | 8.9 samples/s +Step 37700/44000 | Loss: 2.9100 | LR: 2.57e-05 | 2255.5ms/step | 8.9 samples/s +Step 37710/44000 | Loss: 2.8850 | LR: 2.56e-05 | 2237.3ms/step | 8.9 samples/s +Step 37720/44000 | Loss: 2.8679 | LR: 2.56e-05 | 2240.5ms/step | 8.9 samples/s +Step 37730/44000 | Loss: 2.9306 | LR: 2.55e-05 | 2290.1ms/step | 8.7 samples/s +Step 37740/44000 | Loss: 2.8659 | LR: 2.54e-05 | 2254.0ms/step | 8.9 samples/s +Step 37750/44000 | Loss: 2.8927 | LR: 2.53e-05 | 2268.1ms/step | 8.8 samples/s +Step 37760/44000 | Loss: 2.8575 | LR: 2.52e-05 | 2265.1ms/step | 8.8 samples/s +Step 37770/44000 | Loss: 2.8556 | LR: 2.52e-05 | 2272.3ms/step | 8.8 samples/s +Step 37780/44000 | Loss: 2.8621 | LR: 2.51e-05 | 2271.8ms/step | 8.8 samples/s +Step 37790/44000 | Loss: 2.8880 | LR: 2.50e-05 | 2272.9ms/step | 8.8 samples/s +Step 37800/44000 | Loss: 2.9320 | LR: 2.49e-05 | 2281.7ms/step | 8.8 samples/s +Step 37810/44000 | Loss: 2.8911 | LR: 2.48e-05 | 2281.5ms/step | 8.8 samples/s +Step 37820/44000 | Loss: 2.8404 | LR: 2.48e-05 | 2257.1ms/step | 8.9 samples/s +Step 37830/44000 | Loss: 2.8816 | LR: 2.47e-05 | 2269.9ms/step | 8.8 samples/s +Step 37840/44000 | Loss: 2.8754 | LR: 2.46e-05 | 2226.5ms/step | 9.0 samples/s +Step 37850/44000 | Loss: 2.8557 | LR: 2.45e-05 | 2274.4ms/step | 8.8 samples/s +Step 37860/44000 | Loss: 2.8618 | LR: 2.45e-05 | 2277.3ms/step | 8.8 samples/s +Step 37870/44000 | Loss: 2.8899 | LR: 2.44e-05 | 2254.6ms/step | 8.9 samples/s +Step 37880/44000 | Loss: 2.9073 | LR: 2.43e-05 | 2273.3ms/step | 8.8 samples/s +Step 37890/44000 | Loss: 2.8760 | LR: 2.42e-05 | 2238.5ms/step | 8.9 samples/s +Step 37900/44000 | Loss: 2.8661 | LR: 2.41e-05 | 2247.2ms/step | 8.9 samples/s +Step 37910/44000 | Loss: 2.8860 | LR: 2.41e-05 | 2279.7ms/step | 8.8 samples/s +Step 37920/44000 | Loss: 2.8670 | LR: 2.40e-05 | 2269.6ms/step | 8.8 samples/s +Step 37930/44000 | Loss: 2.9038 | LR: 2.39e-05 | 2258.2ms/step | 8.9 samples/s +Step 37940/44000 | Loss: 2.8797 | LR: 2.38e-05 | 2222.4ms/step | 9.0 samples/s +Step 37950/44000 | Loss: 2.9219 | LR: 2.38e-05 | 2249.5ms/step | 8.9 samples/s +Step 37960/44000 | Loss: 2.8738 | LR: 2.37e-05 | 2247.2ms/step | 8.9 samples/s +Step 37970/44000 | Loss: 2.9027 | LR: 2.36e-05 | 2258.5ms/step | 8.9 samples/s +Step 37980/44000 | Loss: 2.8533 | LR: 2.35e-05 | 2249.0ms/step | 8.9 samples/s +Step 37990/44000 | Loss: 2.9255 | LR: 2.34e-05 | 2223.3ms/step | 9.0 samples/s +Step 38000/44000 | Loss: 2.9127 | LR: 2.34e-05 | 2130.5ms/step | 9.4 samples/s +Validation at step 38000: Loss = 2.8734, Perplexity = 17.70 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-38000 +Step 38010/44000 | Loss: 2.8835 | LR: 2.33e-05 | 4654.5ms/step | 4.3 samples/s +Step 38020/44000 | Loss: 2.8569 | LR: 2.32e-05 | 2245.6ms/step | 8.9 samples/s +Step 38030/44000 | Loss: 2.8652 | LR: 2.31e-05 | 2269.8ms/step | 8.8 samples/s +Step 38040/44000 | Loss: 2.8817 | LR: 2.31e-05 | 2301.7ms/step | 8.7 samples/s +Step 38050/44000 | Loss: 2.8548 | LR: 2.30e-05 | 2284.9ms/step | 8.8 samples/s +Step 38060/44000 | Loss: 2.8843 | LR: 2.29e-05 | 2272.6ms/step | 8.8 samples/s +Step 38070/44000 | Loss: 2.8985 | LR: 2.28e-05 | 2252.5ms/step | 8.9 samples/s +Step 38080/44000 | Loss: 2.8751 | LR: 2.28e-05 | 2257.3ms/step | 8.9 samples/s +Step 38090/44000 | Loss: 2.8944 | LR: 2.27e-05 | 2244.1ms/step | 8.9 samples/s +Step 38100/44000 | Loss: 2.8488 | LR: 2.26e-05 | 2248.6ms/step | 8.9 samples/s +Step 38110/44000 | Loss: 2.8702 | LR: 2.25e-05 | 2250.0ms/step | 8.9 samples/s +Step 38120/44000 | Loss: 2.8792 | LR: 2.25e-05 | 2262.7ms/step | 8.8 samples/s +Step 38130/44000 | Loss: 2.9064 | LR: 2.24e-05 | 2299.1ms/step | 8.7 samples/s +Step 38140/44000 | Loss: 2.8972 | LR: 2.23e-05 | 2264.6ms/step | 8.8 samples/s +Step 38150/44000 | Loss: 2.8962 | LR: 2.22e-05 | 2291.1ms/step | 8.7 samples/s +Step 38160/44000 | Loss: 2.8711 | LR: 2.22e-05 | 2265.8ms/step | 8.8 samples/s +Step 38170/44000 | Loss: 2.8821 | LR: 2.21e-05 | 2290.6ms/step | 8.7 samples/s +Step 38180/44000 | Loss: 2.8724 | LR: 2.20e-05 | 2300.7ms/step | 8.7 samples/s +Step 38190/44000 | Loss: 2.8808 | LR: 2.19e-05 | 2249.7ms/step | 8.9 samples/s +Step 38200/44000 | Loss: 2.8880 | LR: 2.19e-05 | 2284.4ms/step | 8.8 samples/s +Step 38210/44000 | Loss: 2.8557 | LR: 2.18e-05 | 2274.1ms/step | 8.8 samples/s +Step 38220/44000 | Loss: 2.9158 | LR: 2.17e-05 | 2262.3ms/step | 8.8 samples/s +Step 38230/44000 | Loss: 2.9045 | LR: 2.16e-05 | 2281.4ms/step | 8.8 samples/s +Step 38240/44000 | Loss: 2.8833 | LR: 2.16e-05 | 2242.0ms/step | 8.9 samples/s +Step 38250/44000 | Loss: 2.8893 | LR: 2.15e-05 | 2254.1ms/step | 8.9 samples/s +Step 38260/44000 | Loss: 2.8305 | LR: 2.14e-05 | 2275.5ms/step | 8.8 samples/s +Step 38270/44000 | Loss: 2.8635 | LR: 2.13e-05 | 2233.4ms/step | 9.0 samples/s +Step 38280/44000 | Loss: 2.8770 | LR: 2.13e-05 | 2291.1ms/step | 8.7 samples/s +Step 38290/44000 | Loss: 2.8521 | LR: 2.12e-05 | 2244.5ms/step | 8.9 samples/s +Step 38300/44000 | Loss: 2.8967 | LR: 2.11e-05 | 2267.9ms/step | 8.8 samples/s +Step 38310/44000 | Loss: 2.9089 | LR: 2.11e-05 | 2269.4ms/step | 8.8 samples/s +Step 38320/44000 | Loss: 2.8804 | LR: 2.10e-05 | 2269.1ms/step | 8.8 samples/s +Step 38330/44000 | Loss: 2.8861 | LR: 2.09e-05 | 2222.5ms/step | 9.0 samples/s +Step 38340/44000 | Loss: 2.8985 | LR: 2.08e-05 | 2276.2ms/step | 8.8 samples/s +Step 38350/44000 | Loss: 2.9028 | LR: 2.08e-05 | 2262.3ms/step | 8.8 samples/s +Step 38360/44000 | Loss: 2.9073 | LR: 2.07e-05 | 2265.2ms/step | 8.8 samples/s +Step 38370/44000 | Loss: 2.8794 | LR: 2.06e-05 | 2285.3ms/step | 8.8 samples/s +Step 38380/44000 | Loss: 2.8910 | LR: 2.05e-05 | 2259.7ms/step | 8.9 samples/s +Step 38390/44000 | Loss: 2.8512 | LR: 2.05e-05 | 2248.7ms/step | 8.9 samples/s +Step 38400/44000 | Loss: 2.8864 | LR: 2.04e-05 | 2248.5ms/step | 8.9 samples/s +Step 38410/44000 | Loss: 2.8964 | LR: 2.03e-05 | 2272.3ms/step | 8.8 samples/s +Step 38420/44000 | Loss: 2.9124 | LR: 2.03e-05 | 2251.6ms/step | 8.9 samples/s +Step 38430/44000 | Loss: 2.9600 | LR: 2.02e-05 | 2238.5ms/step | 8.9 samples/s +Step 38440/44000 | Loss: 2.9272 | LR: 2.01e-05 | 2248.9ms/step | 8.9 samples/s +Step 38450/44000 | Loss: 2.9429 | LR: 2.00e-05 | 2239.3ms/step | 8.9 samples/s +Step 38460/44000 | Loss: 2.8767 | LR: 2.00e-05 | 2253.4ms/step | 8.9 samples/s +Step 38470/44000 | Loss: 2.8995 | LR: 1.99e-05 | 2212.9ms/step | 9.0 samples/s +Step 38480/44000 | Loss: 2.8702 | LR: 1.98e-05 | 2259.5ms/step | 8.9 samples/s +Step 38490/44000 | Loss: 2.8841 | LR: 1.98e-05 | 2255.6ms/step | 8.9 samples/s +Step 38500/44000 | Loss: 2.8865 | LR: 1.97e-05 | 2263.2ms/step | 8.8 samples/s +Step 38510/44000 | Loss: 2.8481 | LR: 1.96e-05 | 2273.5ms/step | 8.8 samples/s +Step 38520/44000 | Loss: 2.9121 | LR: 1.95e-05 | 2293.1ms/step | 8.7 samples/s +Step 38530/44000 | Loss: 2.9196 | LR: 1.95e-05 | 2262.8ms/step | 8.8 samples/s +Step 38540/44000 | Loss: 2.8481 | LR: 1.94e-05 | 2228.2ms/step | 9.0 samples/s +Step 38550/44000 | Loss: 2.8830 | LR: 1.93e-05 | 2285.8ms/step | 8.7 samples/s +Step 38560/44000 | Loss: 2.9072 | LR: 1.93e-05 | 2250.7ms/step | 8.9 samples/s +Step 38570/44000 | Loss: 2.8526 | LR: 1.92e-05 | 2238.9ms/step | 8.9 samples/s +Step 38580/44000 | Loss: 2.8905 | LR: 1.91e-05 | 2223.7ms/step | 9.0 samples/s +Step 38590/44000 | Loss: 2.8499 | LR: 1.91e-05 | 2252.2ms/step | 8.9 samples/s +Step 38600/44000 | Loss: 2.8285 | LR: 1.90e-05 | 2277.9ms/step | 8.8 samples/s +Step 38610/44000 | Loss: 2.9129 | LR: 1.89e-05 | 2287.1ms/step | 8.7 samples/s +Step 38620/44000 | Loss: 2.8527 | LR: 1.88e-05 | 2247.7ms/step | 8.9 samples/s +Step 38630/44000 | Loss: 2.9196 | LR: 1.88e-05 | 2246.7ms/step | 8.9 samples/s +Step 38640/44000 | Loss: 2.8587 | LR: 1.87e-05 | 2261.9ms/step | 8.8 samples/s +Step 38650/44000 | Loss: 2.8802 | LR: 1.86e-05 | 2243.0ms/step | 8.9 samples/s +Step 38660/44000 | Loss: 2.9339 | LR: 1.86e-05 | 2253.8ms/step | 8.9 samples/s +Step 38670/44000 | Loss: 2.8992 | LR: 1.85e-05 | 2264.8ms/step | 8.8 samples/s +Step 38680/44000 | Loss: 2.8709 | LR: 1.84e-05 | 2260.2ms/step | 8.8 samples/s +Step 38690/44000 | Loss: 2.9068 | LR: 1.84e-05 | 2258.7ms/step | 8.9 samples/s +Step 38700/44000 | Loss: 2.9065 | LR: 1.83e-05 | 2267.3ms/step | 8.8 samples/s +Step 38710/44000 | Loss: 2.8601 | LR: 1.82e-05 | 2237.8ms/step | 8.9 samples/s +Step 38720/44000 | Loss: 2.8705 | LR: 1.82e-05 | 2262.9ms/step | 8.8 samples/s +Step 38730/44000 | Loss: 2.9017 | LR: 1.81e-05 | 2256.0ms/step | 8.9 samples/s +Step 38740/44000 | Loss: 2.8594 | LR: 1.80e-05 | 2261.0ms/step | 8.8 samples/s +Step 38750/44000 | Loss: 2.9241 | LR: 1.80e-05 | 2252.9ms/step | 8.9 samples/s +Step 38760/44000 | Loss: 2.8962 | LR: 1.79e-05 | 2270.8ms/step | 8.8 samples/s +Step 38770/44000 | Loss: 2.8622 | LR: 1.78e-05 | 2248.7ms/step | 8.9 samples/s +Step 38780/44000 | Loss: 2.8555 | LR: 1.78e-05 | 2260.7ms/step | 8.8 samples/s +Step 38790/44000 | Loss: 2.8735 | LR: 1.77e-05 | 2253.8ms/step | 8.9 samples/s +Step 38800/44000 | Loss: 2.9019 | LR: 1.76e-05 | 2226.5ms/step | 9.0 samples/s +Step 38810/44000 | Loss: 2.9272 | LR: 1.76e-05 | 2276.0ms/step | 8.8 samples/s +Step 38820/44000 | Loss: 2.9099 | LR: 1.75e-05 | 2272.8ms/step | 8.8 samples/s +Step 38830/44000 | Loss: 2.8831 | LR: 1.74e-05 | 2243.4ms/step | 8.9 samples/s +Step 38840/44000 | Loss: 2.8773 | LR: 1.74e-05 | 2258.7ms/step | 8.9 samples/s +Step 38850/44000 | Loss: 2.9029 | LR: 1.73e-05 | 2226.4ms/step | 9.0 samples/s +Step 38860/44000 | Loss: 2.9597 | LR: 1.72e-05 | 2268.4ms/step | 8.8 samples/s +Step 38870/44000 | Loss: 2.8838 | LR: 1.72e-05 | 2264.7ms/step | 8.8 samples/s +Step 38880/44000 | Loss: 2.8798 | LR: 1.71e-05 | 2266.1ms/step | 8.8 samples/s +Step 38890/44000 | Loss: 2.8735 | LR: 1.70e-05 | 2255.7ms/step | 8.9 samples/s +Step 38900/44000 | Loss: 2.9022 | LR: 1.70e-05 | 2256.1ms/step | 8.9 samples/s +Step 38910/44000 | Loss: 2.8603 | LR: 1.69e-05 | 2279.1ms/step | 8.8 samples/s +Step 38920/44000 | Loss: 2.8932 | LR: 1.68e-05 | 2246.4ms/step | 8.9 samples/s +Step 38930/44000 | Loss: 2.9388 | LR: 1.68e-05 | 2251.0ms/step | 8.9 samples/s +Step 38940/44000 | Loss: 2.9025 | LR: 1.67e-05 | 2235.5ms/step | 8.9 samples/s +Step 38950/44000 | Loss: 2.8811 | LR: 1.66e-05 | 2228.2ms/step | 9.0 samples/s +Step 38960/44000 | Loss: 2.8909 | LR: 1.66e-05 | 2259.0ms/step | 8.9 samples/s +Step 38970/44000 | Loss: 2.8885 | LR: 1.65e-05 | 2243.9ms/step | 8.9 samples/s +Step 38980/44000 | Loss: 2.8846 | LR: 1.64e-05 | 2254.4ms/step | 8.9 samples/s +Step 38990/44000 | Loss: 2.8904 | LR: 1.64e-05 | 2264.6ms/step | 8.8 samples/s +Step 39000/44000 | Loss: 2.9009 | LR: 1.63e-05 | 2228.0ms/step | 9.0 samples/s +Validation at step 39000: Loss = 2.8731, Perplexity = 17.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-39000 +Step 39010/44000 | Loss: 2.9014 | LR: 1.62e-05 | 4453.8ms/step | 4.5 samples/s +Step 39020/44000 | Loss: 2.8985 | LR: 1.62e-05 | 2253.5ms/step | 8.9 samples/s +Step 39030/44000 | Loss: 2.7947 | LR: 1.61e-05 | 2239.5ms/step | 8.9 samples/s +Step 39040/44000 | Loss: 2.9126 | LR: 1.61e-05 | 2256.3ms/step | 8.9 samples/s +Step 39050/44000 | Loss: 2.8785 | LR: 1.60e-05 | 2279.0ms/step | 8.8 samples/s +Step 39060/44000 | Loss: 2.8152 | LR: 1.59e-05 | 2261.0ms/step | 8.8 samples/s +Step 39070/44000 | Loss: 2.8622 | LR: 1.59e-05 | 2245.6ms/step | 8.9 samples/s +Step 39080/44000 | Loss: 2.8796 | LR: 1.58e-05 | 2252.3ms/step | 8.9 samples/s +Step 39090/44000 | Loss: 2.9240 | LR: 1.57e-05 | 2253.0ms/step | 8.9 samples/s +Step 39100/44000 | Loss: 2.8642 | LR: 1.57e-05 | 2289.8ms/step | 8.7 samples/s +Step 39110/44000 | Loss: 2.9445 | LR: 1.56e-05 | 2247.3ms/step | 8.9 samples/s +Step 39120/44000 | Loss: 2.8654 | LR: 1.55e-05 | 2261.3ms/step | 8.8 samples/s +Step 39130/44000 | Loss: 2.8868 | LR: 1.55e-05 | 2239.3ms/step | 8.9 samples/s +Step 39140/44000 | Loss: 2.8701 | LR: 1.54e-05 | 2246.9ms/step | 8.9 samples/s +Step 39150/44000 | Loss: 2.9081 | LR: 1.54e-05 | 2223.6ms/step | 9.0 samples/s +Step 39160/44000 | Loss: 2.9316 | LR: 1.53e-05 | 2239.6ms/step | 8.9 samples/s +Step 39170/44000 | Loss: 2.8325 | LR: 1.52e-05 | 2261.3ms/step | 8.8 samples/s +Step 39180/44000 | Loss: 2.8422 | LR: 1.52e-05 | 2255.1ms/step | 8.9 samples/s +Step 39190/44000 | Loss: 2.8776 | LR: 1.51e-05 | 2265.4ms/step | 8.8 samples/s +Step 39200/44000 | Loss: 2.9081 | LR: 1.50e-05 | 2285.0ms/step | 8.8 samples/s +Step 39210/44000 | Loss: 2.8907 | LR: 1.50e-05 | 2294.2ms/step | 8.7 samples/s +Step 39220/44000 | Loss: 2.8959 | LR: 1.49e-05 | 2261.4ms/step | 8.8 samples/s +Step 39230/44000 | Loss: 2.8425 | LR: 1.49e-05 | 2285.2ms/step | 8.8 samples/s +Step 39240/44000 | Loss: 2.8502 | LR: 1.48e-05 | 2249.0ms/step | 8.9 samples/s +Step 39250/44000 | Loss: 2.8808 | LR: 1.47e-05 | 2273.2ms/step | 8.8 samples/s +Step 39260/44000 | Loss: 2.8489 | LR: 1.47e-05 | 2275.5ms/step | 8.8 samples/s +Step 39270/44000 | Loss: 2.9078 | LR: 1.46e-05 | 2249.4ms/step | 8.9 samples/s +Step 39280/44000 | Loss: 2.8136 | LR: 1.46e-05 | 2305.9ms/step | 8.7 samples/s +Step 39290/44000 | Loss: 2.9186 | LR: 1.45e-05 | 2273.0ms/step | 8.8 samples/s +Step 39300/44000 | Loss: 2.8624 | LR: 1.44e-05 | 2267.5ms/step | 8.8 samples/s +Step 39310/44000 | Loss: 2.8617 | LR: 1.44e-05 | 2277.6ms/step | 8.8 samples/s +Step 39320/44000 | Loss: 2.8343 | LR: 1.43e-05 | 2235.1ms/step | 8.9 samples/s +Step 39330/44000 | Loss: 2.8546 | LR: 1.42e-05 | 2256.5ms/step | 8.9 samples/s +Step 39340/44000 | Loss: 2.9024 | LR: 1.42e-05 | 2242.9ms/step | 8.9 samples/s +Step 39350/44000 | Loss: 2.8785 | LR: 1.41e-05 | 2275.8ms/step | 8.8 samples/s +Step 39360/44000 | Loss: 2.8636 | LR: 1.41e-05 | 2283.3ms/step | 8.8 samples/s +Step 39370/44000 | Loss: 2.8393 | LR: 1.40e-05 | 2281.4ms/step | 8.8 samples/s +Step 39380/44000 | Loss: 2.9373 | LR: 1.39e-05 | 2245.0ms/step | 8.9 samples/s +Step 39390/44000 | Loss: 2.8804 | LR: 1.39e-05 | 2261.2ms/step | 8.8 samples/s +Step 39400/44000 | Loss: 2.9157 | LR: 1.38e-05 | 2296.6ms/step | 8.7 samples/s +Step 39410/44000 | Loss: 2.8670 | LR: 1.38e-05 | 2246.5ms/step | 8.9 samples/s +Step 39420/44000 | Loss: 2.8956 | LR: 1.37e-05 | 2281.7ms/step | 8.8 samples/s +Step 39430/44000 | Loss: 2.8662 | LR: 1.36e-05 | 2269.0ms/step | 8.8 samples/s +Step 39440/44000 | Loss: 2.9171 | LR: 1.36e-05 | 2244.1ms/step | 8.9 samples/s +Step 39450/44000 | Loss: 2.8843 | LR: 1.35e-05 | 2254.7ms/step | 8.9 samples/s +Step 39460/44000 | Loss: 2.8722 | LR: 1.35e-05 | 2244.8ms/step | 8.9 samples/s +Step 39470/44000 | Loss: 2.8523 | LR: 1.34e-05 | 2242.9ms/step | 8.9 samples/s +Step 39480/44000 | Loss: 2.8677 | LR: 1.34e-05 | 2277.6ms/step | 8.8 samples/s +Step 39490/44000 | Loss: 2.9172 | LR: 1.33e-05 | 2257.6ms/step | 8.9 samples/s +Step 39500/44000 | Loss: 2.8887 | LR: 1.32e-05 | 2260.2ms/step | 8.8 samples/s +Step 39510/44000 | Loss: 2.8402 | LR: 1.32e-05 | 2248.4ms/step | 8.9 samples/s +Step 39520/44000 | Loss: 2.9595 | LR: 1.31e-05 | 2274.6ms/step | 8.8 samples/s +Step 39530/44000 | Loss: 2.8771 | LR: 1.31e-05 | 2240.0ms/step | 8.9 samples/s +Step 39540/44000 | Loss: 2.8433 | LR: 1.30e-05 | 2243.5ms/step | 8.9 samples/s +Step 39550/44000 | Loss: 2.8681 | LR: 1.29e-05 | 2236.1ms/step | 8.9 samples/s +Step 39560/44000 | Loss: 2.8748 | LR: 1.29e-05 | 2241.9ms/step | 8.9 samples/s +Step 39570/44000 | Loss: 2.8624 | LR: 1.28e-05 | 2287.3ms/step | 8.7 samples/s +Step 39580/44000 | Loss: 2.9019 | LR: 1.28e-05 | 2231.6ms/step | 9.0 samples/s +Step 39590/44000 | Loss: 2.8580 | LR: 1.27e-05 | 2229.4ms/step | 9.0 samples/s +Step 39600/44000 | Loss: 2.8736 | LR: 1.27e-05 | 2227.0ms/step | 9.0 samples/s +Step 39610/44000 | Loss: 2.9121 | LR: 1.26e-05 | 2244.0ms/step | 8.9 samples/s +Step 39620/44000 | Loss: 2.8868 | LR: 1.25e-05 | 2253.4ms/step | 8.9 samples/s +Step 39630/44000 | Loss: 2.8877 | LR: 1.25e-05 | 2262.0ms/step | 8.8 samples/s +Step 39640/44000 | Loss: 2.8984 | LR: 1.24e-05 | 2265.8ms/step | 8.8 samples/s +Step 39650/44000 | Loss: 2.8955 | LR: 1.24e-05 | 2248.3ms/step | 8.9 samples/s +Step 39660/44000 | Loss: 2.8748 | LR: 1.23e-05 | 2237.7ms/step | 8.9 samples/s +Step 39670/44000 | Loss: 2.9282 | LR: 1.23e-05 | 2299.5ms/step | 8.7 samples/s +Step 39680/44000 | Loss: 2.9138 | LR: 1.22e-05 | 2255.0ms/step | 8.9 samples/s +Step 39690/44000 | Loss: 2.8801 | LR: 1.22e-05 | 2270.2ms/step | 8.8 samples/s +Step 39700/44000 | Loss: 2.8788 | LR: 1.21e-05 | 2267.7ms/step | 8.8 samples/s +Step 39710/44000 | Loss: 2.9025 | LR: 1.20e-05 | 2282.2ms/step | 8.8 samples/s +Step 39720/44000 | Loss: 2.8753 | LR: 1.20e-05 | 2242.5ms/step | 8.9 samples/s +Step 39730/44000 | Loss: 2.9007 | LR: 1.19e-05 | 2268.4ms/step | 8.8 samples/s +Step 39740/44000 | Loss: 2.8772 | LR: 1.19e-05 | 2282.0ms/step | 8.8 samples/s +Step 39750/44000 | Loss: 2.8363 | LR: 1.18e-05 | 2291.2ms/step | 8.7 samples/s +Step 39760/44000 | Loss: 2.8859 | LR: 1.18e-05 | 2259.5ms/step | 8.9 samples/s +Step 39770/44000 | Loss: 2.9014 | LR: 1.17e-05 | 2297.6ms/step | 8.7 samples/s +Step 39780/44000 | Loss: 2.9075 | LR: 1.17e-05 | 2291.8ms/step | 8.7 samples/s +Step 39790/44000 | Loss: 2.8794 | LR: 1.16e-05 | 2272.9ms/step | 8.8 samples/s +Step 39800/44000 | Loss: 2.8952 | LR: 1.15e-05 | 2282.0ms/step | 8.8 samples/s +Step 39810/44000 | Loss: 2.8633 | LR: 1.15e-05 | 2249.1ms/step | 8.9 samples/s +Step 39820/44000 | Loss: 2.8618 | LR: 1.14e-05 | 2294.8ms/step | 8.7 samples/s +Step 39830/44000 | Loss: 2.8925 | LR: 1.14e-05 | 2265.4ms/step | 8.8 samples/s +Step 39840/44000 | Loss: 2.9101 | LR: 1.13e-05 | 2242.4ms/step | 8.9 samples/s +Step 39850/44000 | Loss: 2.8573 | LR: 1.13e-05 | 2253.3ms/step | 8.9 samples/s +Step 39860/44000 | Loss: 2.8418 | LR: 1.12e-05 | 2274.3ms/step | 8.8 samples/s +Step 39870/44000 | Loss: 2.8860 | LR: 1.12e-05 | 2289.9ms/step | 8.7 samples/s +Step 39880/44000 | Loss: 2.8821 | LR: 1.11e-05 | 2269.2ms/step | 8.8 samples/s +Step 39890/44000 | Loss: 2.9048 | LR: 1.11e-05 | 2279.5ms/step | 8.8 samples/s +Step 39900/44000 | Loss: 2.8689 | LR: 1.10e-05 | 2242.9ms/step | 8.9 samples/s +Step 39910/44000 | Loss: 2.9014 | LR: 1.10e-05 | 2264.1ms/step | 8.8 samples/s +Step 39920/44000 | Loss: 2.8818 | LR: 1.09e-05 | 2284.9ms/step | 8.8 samples/s +Step 39930/44000 | Loss: 2.8786 | LR: 1.08e-05 | 2268.0ms/step | 8.8 samples/s +Step 39940/44000 | Loss: 2.8366 | LR: 1.08e-05 | 2255.5ms/step | 8.9 samples/s +Step 39950/44000 | Loss: 2.8262 | LR: 1.07e-05 | 2259.3ms/step | 8.9 samples/s +Step 39960/44000 | Loss: 2.8858 | LR: 1.07e-05 | 2239.5ms/step | 8.9 samples/s +Step 39970/44000 | Loss: 2.8668 | LR: 1.06e-05 | 2275.4ms/step | 8.8 samples/s +Step 39980/44000 | Loss: 2.8536 | LR: 1.06e-05 | 2248.0ms/step | 8.9 samples/s +Step 39990/44000 | Loss: 2.8895 | LR: 1.05e-05 | 2299.7ms/step | 8.7 samples/s +Step 40000/44000 | Loss: 2.8813 | LR: 1.05e-05 | 2279.5ms/step | 8.8 samples/s +Validation at step 40000: Loss = 2.8731, Perplexity = 17.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-40000 +Step 40010/44000 | Loss: 2.8928 | LR: 1.04e-05 | 4532.9ms/step | 4.4 samples/s +Step 40020/44000 | Loss: 2.8936 | LR: 1.04e-05 | 2252.2ms/step | 8.9 samples/s +Step 40030/44000 | Loss: 2.8776 | LR: 1.03e-05 | 2261.6ms/step | 8.8 samples/s +Step 40040/44000 | Loss: 2.8929 | LR: 1.03e-05 | 2257.1ms/step | 8.9 samples/s +Step 40050/44000 | Loss: 2.8957 | LR: 1.02e-05 | 2264.9ms/step | 8.8 samples/s +Step 40060/44000 | Loss: 2.8910 | LR: 1.02e-05 | 2267.9ms/step | 8.8 samples/s +Step 40070/44000 | Loss: 2.9171 | LR: 1.01e-05 | 2260.4ms/step | 8.8 samples/s +Step 40080/44000 | Loss: 2.8551 | LR: 1.01e-05 | 2253.2ms/step | 8.9 samples/s +Step 40090/44000 | Loss: 2.8987 | LR: 1.00e-05 | 2290.5ms/step | 8.7 samples/s +Step 40100/44000 | Loss: 2.8678 | LR: 9.96e-06 | 2253.0ms/step | 8.9 samples/s +Step 40110/44000 | Loss: 2.8990 | LR: 9.91e-06 | 2270.0ms/step | 8.8 samples/s +Step 40120/44000 | Loss: 2.8733 | LR: 9.86e-06 | 2256.4ms/step | 8.9 samples/s +Step 40130/44000 | Loss: 2.8552 | LR: 9.81e-06 | 2288.4ms/step | 8.7 samples/s +Step 40140/44000 | Loss: 2.9044 | LR: 9.76e-06 | 2264.0ms/step | 8.8 samples/s +Step 40150/44000 | Loss: 2.8963 | LR: 9.71e-06 | 2253.9ms/step | 8.9 samples/s +Step 40160/44000 | Loss: 2.8860 | LR: 9.66e-06 | 2249.8ms/step | 8.9 samples/s +Step 40170/44000 | Loss: 2.8712 | LR: 9.61e-06 | 2274.4ms/step | 8.8 samples/s +Step 40180/44000 | Loss: 2.8590 | LR: 9.56e-06 | 2276.3ms/step | 8.8 samples/s +Step 40190/44000 | Loss: 2.8864 | LR: 9.51e-06 | 2267.8ms/step | 8.8 samples/s +Step 40200/44000 | Loss: 2.8753 | LR: 9.46e-06 | 2317.2ms/step | 8.6 samples/s +Step 40210/44000 | Loss: 2.8385 | LR: 9.41e-06 | 2262.1ms/step | 8.8 samples/s +Step 40220/44000 | Loss: 2.8755 | LR: 9.36e-06 | 2249.9ms/step | 8.9 samples/s +Step 40230/44000 | Loss: 2.8799 | LR: 9.32e-06 | 2256.4ms/step | 8.9 samples/s +Step 40240/44000 | Loss: 2.8024 | LR: 9.27e-06 | 2268.7ms/step | 8.8 samples/s +Step 40250/44000 | Loss: 2.8410 | LR: 9.22e-06 | 2264.3ms/step | 8.8 samples/s +Step 40260/44000 | Loss: 2.8882 | LR: 9.17e-06 | 2281.5ms/step | 8.8 samples/s +Step 40270/44000 | Loss: 2.8989 | LR: 9.12e-06 | 2250.9ms/step | 8.9 samples/s +Step 40280/44000 | Loss: 2.9121 | LR: 9.07e-06 | 2239.3ms/step | 8.9 samples/s +Step 40290/44000 | Loss: 2.8454 | LR: 9.02e-06 | 2265.0ms/step | 8.8 samples/s +Step 40300/44000 | Loss: 2.8568 | LR: 8.97e-06 | 2253.9ms/step | 8.9 samples/s +Step 40310/44000 | Loss: 2.8860 | LR: 8.93e-06 | 2274.5ms/step | 8.8 samples/s +Step 40320/44000 | Loss: 2.9127 | LR: 8.88e-06 | 2251.9ms/step | 8.9 samples/s +Step 40330/44000 | Loss: 2.8774 | LR: 8.83e-06 | 2245.3ms/step | 8.9 samples/s +Step 40340/44000 | Loss: 2.8970 | LR: 8.78e-06 | 2271.3ms/step | 8.8 samples/s +Step 40350/44000 | Loss: 2.9050 | LR: 8.74e-06 | 2270.2ms/step | 8.8 samples/s +Step 40360/44000 | Loss: 2.8488 | LR: 8.69e-06 | 2249.6ms/step | 8.9 samples/s +Step 40370/44000 | Loss: 2.9214 | LR: 8.64e-06 | 2233.1ms/step | 9.0 samples/s +Step 40380/44000 | Loss: 2.8836 | LR: 8.59e-06 | 2258.4ms/step | 8.9 samples/s +Step 40390/44000 | Loss: 2.8678 | LR: 8.55e-06 | 2254.0ms/step | 8.9 samples/s +Step 40400/44000 | Loss: 2.8925 | LR: 8.50e-06 | 2238.5ms/step | 8.9 samples/s +Step 40410/44000 | Loss: 2.8604 | LR: 8.45e-06 | 2236.0ms/step | 8.9 samples/s +Step 40420/44000 | Loss: 2.8972 | LR: 8.41e-06 | 2266.6ms/step | 8.8 samples/s +Step 40430/44000 | Loss: 2.8879 | LR: 8.36e-06 | 2255.3ms/step | 8.9 samples/s +Step 40440/44000 | Loss: 2.9054 | LR: 8.31e-06 | 2273.1ms/step | 8.8 samples/s +Step 40450/44000 | Loss: 2.8236 | LR: 8.27e-06 | 2265.6ms/step | 8.8 samples/s +Step 40460/44000 | Loss: 2.8159 | LR: 8.22e-06 | 2251.9ms/step | 8.9 samples/s +Step 40470/44000 | Loss: 2.9263 | LR: 8.17e-06 | 2258.6ms/step | 8.9 samples/s +Step 40480/44000 | Loss: 2.8818 | LR: 8.13e-06 | 2291.3ms/step | 8.7 samples/s +Step 40490/44000 | Loss: 2.9109 | LR: 8.08e-06 | 2262.9ms/step | 8.8 samples/s +Step 40500/44000 | Loss: 2.9415 | LR: 8.04e-06 | 2256.6ms/step | 8.9 samples/s +Step 40510/44000 | Loss: 2.8599 | LR: 7.99e-06 | 2261.7ms/step | 8.8 samples/s +Step 40520/44000 | Loss: 2.8529 | LR: 7.94e-06 | 2257.7ms/step | 8.9 samples/s +Step 40530/44000 | Loss: 2.8741 | LR: 7.90e-06 | 2243.5ms/step | 8.9 samples/s +Step 40540/44000 | Loss: 2.8769 | LR: 7.85e-06 | 2285.5ms/step | 8.8 samples/s +Step 40550/44000 | Loss: 2.8500 | LR: 7.81e-06 | 2277.3ms/step | 8.8 samples/s +Step 40560/44000 | Loss: 2.8920 | LR: 7.76e-06 | 2256.6ms/step | 8.9 samples/s +Step 40570/44000 | Loss: 2.8720 | LR: 7.72e-06 | 2258.9ms/step | 8.9 samples/s +Step 40580/44000 | Loss: 2.9108 | LR: 7.67e-06 | 2248.4ms/step | 8.9 samples/s +Step 40590/44000 | Loss: 2.8869 | LR: 7.63e-06 | 2243.3ms/step | 8.9 samples/s +Step 40600/44000 | Loss: 2.8827 | LR: 7.59e-06 | 2252.0ms/step | 8.9 samples/s +Step 40610/44000 | Loss: 2.9113 | LR: 7.54e-06 | 2301.0ms/step | 8.7 samples/s +Step 40620/44000 | Loss: 2.8556 | LR: 7.50e-06 | 2265.5ms/step | 8.8 samples/s +Step 40630/44000 | Loss: 2.8826 | LR: 7.45e-06 | 2295.2ms/step | 8.7 samples/s +Step 40640/44000 | Loss: 2.8633 | LR: 7.41e-06 | 2251.3ms/step | 8.9 samples/s +Step 40650/44000 | Loss: 2.8818 | LR: 7.37e-06 | 2265.2ms/step | 8.8 samples/s +Step 40660/44000 | Loss: 2.9023 | LR: 7.32e-06 | 2263.3ms/step | 8.8 samples/s +Step 40670/44000 | Loss: 2.9198 | LR: 7.28e-06 | 2247.6ms/step | 8.9 samples/s +Step 40680/44000 | Loss: 2.8771 | LR: 7.23e-06 | 2247.6ms/step | 8.9 samples/s +Step 40690/44000 | Loss: 2.9100 | LR: 7.19e-06 | 2284.0ms/step | 8.8 samples/s +Step 40700/44000 | Loss: 2.8679 | LR: 7.15e-06 | 2272.2ms/step | 8.8 samples/s +Step 40710/44000 | Loss: 2.8876 | LR: 7.10e-06 | 2259.7ms/step | 8.9 samples/s +Step 40720/44000 | Loss: 2.8426 | LR: 7.06e-06 | 2248.8ms/step | 8.9 samples/s +Step 40730/44000 | Loss: 2.8700 | LR: 7.02e-06 | 2291.4ms/step | 8.7 samples/s +Step 40740/44000 | Loss: 2.9060 | LR: 6.98e-06 | 2265.3ms/step | 8.8 samples/s +Step 40750/44000 | Loss: 2.8559 | LR: 6.93e-06 | 2266.9ms/step | 8.8 samples/s +Step 40760/44000 | Loss: 2.8666 | LR: 6.89e-06 | 2238.8ms/step | 8.9 samples/s +Step 40770/44000 | Loss: 2.8776 | LR: 6.85e-06 | 2279.3ms/step | 8.8 samples/s +Step 40780/44000 | Loss: 2.8517 | LR: 6.81e-06 | 2263.7ms/step | 8.8 samples/s +Step 40790/44000 | Loss: 2.8820 | LR: 6.77e-06 | 2244.5ms/step | 8.9 samples/s +Step 40800/44000 | Loss: 2.9321 | LR: 6.72e-06 | 2264.2ms/step | 8.8 samples/s +Step 40810/44000 | Loss: 2.8975 | LR: 6.68e-06 | 2251.2ms/step | 8.9 samples/s +Step 40820/44000 | Loss: 2.8563 | LR: 6.64e-06 | 2278.7ms/step | 8.8 samples/s +Step 40830/44000 | Loss: 2.9121 | LR: 6.60e-06 | 2267.7ms/step | 8.8 samples/s +Step 40840/44000 | Loss: 2.9074 | LR: 6.56e-06 | 2243.8ms/step | 8.9 samples/s +Step 40850/44000 | Loss: 2.8799 | LR: 6.52e-06 | 2243.0ms/step | 8.9 samples/s +Step 40860/44000 | Loss: 2.8766 | LR: 6.47e-06 | 2257.8ms/step | 8.9 samples/s +Step 40870/44000 | Loss: 2.8843 | LR: 6.43e-06 | 2242.6ms/step | 8.9 samples/s +Step 40880/44000 | Loss: 2.9346 | LR: 6.39e-06 | 2273.0ms/step | 8.8 samples/s +Step 40890/44000 | Loss: 2.9338 | LR: 6.35e-06 | 2265.7ms/step | 8.8 samples/s +Step 40900/44000 | Loss: 2.8727 | LR: 6.31e-06 | 2246.9ms/step | 8.9 samples/s +Step 40910/44000 | Loss: 2.9187 | LR: 6.27e-06 | 2249.1ms/step | 8.9 samples/s +Step 40920/44000 | Loss: 2.8723 | LR: 6.23e-06 | 2275.1ms/step | 8.8 samples/s +Step 40930/44000 | Loss: 2.9351 | LR: 6.19e-06 | 2252.6ms/step | 8.9 samples/s +Step 40940/44000 | Loss: 2.8725 | LR: 6.15e-06 | 2263.4ms/step | 8.8 samples/s +Step 40950/44000 | Loss: 2.9030 | LR: 6.11e-06 | 2263.0ms/step | 8.8 samples/s +Step 40960/44000 | Loss: 2.9063 | LR: 6.07e-06 | 2258.2ms/step | 8.9 samples/s +Step 40970/44000 | Loss: 2.8827 | LR: 6.03e-06 | 2276.2ms/step | 8.8 samples/s +Step 40980/44000 | Loss: 2.9322 | LR: 5.99e-06 | 2233.8ms/step | 9.0 samples/s +Step 40990/44000 | Loss: 2.8682 | LR: 5.95e-06 | 2268.6ms/step | 8.8 samples/s +Step 41000/44000 | Loss: 2.8806 | LR: 5.91e-06 | 2271.8ms/step | 8.8 samples/s +Validation at step 41000: Loss = 2.8730, Perplexity = 17.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-41000 +Step 41010/44000 | Loss: 2.8612 | LR: 5.87e-06 | 4675.5ms/step | 4.3 samples/s +Step 41020/44000 | Loss: 2.8607 | LR: 5.83e-06 | 2097.6ms/step | 9.5 samples/s +Step 41030/44000 | Loss: 2.8879 | LR: 5.80e-06 | 2243.3ms/step | 8.9 samples/s +Step 41040/44000 | Loss: 2.8489 | LR: 5.76e-06 | 2252.0ms/step | 8.9 samples/s +Step 41050/44000 | Loss: 2.8646 | LR: 5.72e-06 | 2268.4ms/step | 8.8 samples/s +Step 41060/44000 | Loss: 2.8760 | LR: 5.68e-06 | 2279.9ms/step | 8.8 samples/s +Step 41070/44000 | Loss: 2.9200 | LR: 5.64e-06 | 2256.6ms/step | 8.9 samples/s +Step 41080/44000 | Loss: 2.9051 | LR: 5.60e-06 | 2254.3ms/step | 8.9 samples/s +Step 41090/44000 | Loss: 2.8795 | LR: 5.56e-06 | 2227.2ms/step | 9.0 samples/s +Step 41100/44000 | Loss: 2.9308 | LR: 5.53e-06 | 2248.2ms/step | 8.9 samples/s +Step 41110/44000 | Loss: 2.8634 | LR: 5.49e-06 | 2216.3ms/step | 9.0 samples/s +Step 41120/44000 | Loss: 2.8588 | LR: 5.45e-06 | 2267.9ms/step | 8.8 samples/s +Step 41130/44000 | Loss: 2.8817 | LR: 5.41e-06 | 2274.2ms/step | 8.8 samples/s +Step 41140/44000 | Loss: 2.9118 | LR: 5.38e-06 | 2316.2ms/step | 8.6 samples/s +Step 41150/44000 | Loss: 2.8928 | LR: 5.34e-06 | 2268.7ms/step | 8.8 samples/s +Step 41160/44000 | Loss: 2.9040 | LR: 5.30e-06 | 2258.6ms/step | 8.9 samples/s +Step 41170/44000 | Loss: 2.8650 | LR: 5.26e-06 | 2310.5ms/step | 8.7 samples/s +Step 41180/44000 | Loss: 2.9298 | LR: 5.23e-06 | 2264.3ms/step | 8.8 samples/s +Step 41190/44000 | Loss: 2.8505 | LR: 5.19e-06 | 2309.9ms/step | 8.7 samples/s +Step 41200/44000 | Loss: 2.9178 | LR: 5.15e-06 | 2274.4ms/step | 8.8 samples/s +Step 41210/44000 | Loss: 2.8675 | LR: 5.12e-06 | 2298.4ms/step | 8.7 samples/s +Step 41220/44000 | Loss: 2.8950 | LR: 5.08e-06 | 2254.8ms/step | 8.9 samples/s +Step 41230/44000 | Loss: 2.8677 | LR: 5.04e-06 | 2284.9ms/step | 8.8 samples/s +Step 41240/44000 | Loss: 2.8913 | LR: 5.01e-06 | 2289.7ms/step | 8.7 samples/s +Step 41250/44000 | Loss: 2.8907 | LR: 4.97e-06 | 2273.9ms/step | 8.8 samples/s +Step 41260/44000 | Loss: 2.8784 | LR: 4.94e-06 | 2280.9ms/step | 8.8 samples/s +Step 41270/44000 | Loss: 2.8474 | LR: 4.90e-06 | 2242.4ms/step | 8.9 samples/s +Step 41280/44000 | Loss: 2.8973 | LR: 4.86e-06 | 2275.6ms/step | 8.8 samples/s +Step 41290/44000 | Loss: 2.9006 | LR: 4.83e-06 | 2273.6ms/step | 8.8 samples/s +Step 41300/44000 | Loss: 2.8711 | LR: 4.79e-06 | 2300.9ms/step | 8.7 samples/s +Step 41310/44000 | Loss: 2.8831 | LR: 4.76e-06 | 2279.4ms/step | 8.8 samples/s +Step 41320/44000 | Loss: 2.8977 | LR: 4.72e-06 | 2294.9ms/step | 8.7 samples/s +Step 41330/44000 | Loss: 2.8843 | LR: 4.69e-06 | 2257.6ms/step | 8.9 samples/s +Step 41340/44000 | Loss: 2.8664 | LR: 4.65e-06 | 2271.7ms/step | 8.8 samples/s +Step 41350/44000 | Loss: 2.9184 | LR: 4.62e-06 | 2283.1ms/step | 8.8 samples/s +Step 41360/44000 | Loss: 2.8780 | LR: 4.58e-06 | 2273.0ms/step | 8.8 samples/s +Step 41370/44000 | Loss: 2.8530 | LR: 4.55e-06 | 2291.2ms/step | 8.7 samples/s +Step 41380/44000 | Loss: 2.8861 | LR: 4.51e-06 | 2289.6ms/step | 8.7 samples/s +Step 41390/44000 | Loss: 2.8766 | LR: 4.48e-06 | 2258.5ms/step | 8.9 samples/s +Step 41400/44000 | Loss: 2.8673 | LR: 4.45e-06 | 2306.7ms/step | 8.7 samples/s +Step 41410/44000 | Loss: 2.8509 | LR: 4.41e-06 | 2285.5ms/step | 8.8 samples/s +Step 41420/44000 | Loss: 2.8728 | LR: 4.38e-06 | 2266.9ms/step | 8.8 samples/s +Step 41430/44000 | Loss: 2.8522 | LR: 4.34e-06 | 2282.6ms/step | 8.8 samples/s +Step 41440/44000 | Loss: 2.8485 | LR: 4.31e-06 | 2290.2ms/step | 8.7 samples/s +Step 41450/44000 | Loss: 2.8506 | LR: 4.28e-06 | 2295.7ms/step | 8.7 samples/s +Step 41460/44000 | Loss: 2.8994 | LR: 4.24e-06 | 2266.1ms/step | 8.8 samples/s +Step 41470/44000 | Loss: 2.8962 | LR: 4.21e-06 | 2258.7ms/step | 8.9 samples/s +Step 41480/44000 | Loss: 2.8638 | LR: 4.18e-06 | 2269.5ms/step | 8.8 samples/s +Step 41490/44000 | Loss: 2.8598 | LR: 4.14e-06 | 2259.7ms/step | 8.9 samples/s +Step 41500/44000 | Loss: 2.8746 | LR: 4.11e-06 | 2256.4ms/step | 8.9 samples/s +Step 41510/44000 | Loss: 2.8892 | LR: 4.08e-06 | 2278.5ms/step | 8.8 samples/s +Step 41520/44000 | Loss: 2.8790 | LR: 4.05e-06 | 2261.2ms/step | 8.8 samples/s +Step 41530/44000 | Loss: 2.9012 | LR: 4.01e-06 | 2296.3ms/step | 8.7 samples/s +Step 41540/44000 | Loss: 2.8938 | LR: 3.98e-06 | 2254.1ms/step | 8.9 samples/s +Step 41550/44000 | Loss: 2.8692 | LR: 3.95e-06 | 2274.1ms/step | 8.8 samples/s +Step 41560/44000 | Loss: 2.8824 | LR: 3.92e-06 | 2274.1ms/step | 8.8 samples/s +Step 41570/44000 | Loss: 2.8739 | LR: 3.88e-06 | 2269.9ms/step | 8.8 samples/s +Step 41580/44000 | Loss: 2.8609 | LR: 3.85e-06 | 2273.1ms/step | 8.8 samples/s +Step 41590/44000 | Loss: 2.8939 | LR: 3.82e-06 | 2293.2ms/step | 8.7 samples/s +Step 41600/44000 | Loss: 2.8474 | LR: 3.79e-06 | 2286.2ms/step | 8.7 samples/s +Step 41610/44000 | Loss: 2.9054 | LR: 3.76e-06 | 2343.3ms/step | 8.5 samples/s +Step 41620/44000 | Loss: 2.8494 | LR: 3.73e-06 | 2289.2ms/step | 8.7 samples/s +Step 41630/44000 | Loss: 2.8300 | LR: 3.70e-06 | 2253.7ms/step | 8.9 samples/s +Step 41640/44000 | Loss: 2.8857 | LR: 3.66e-06 | 2295.7ms/step | 8.7 samples/s +Step 41650/44000 | Loss: 2.8934 | LR: 3.63e-06 | 2293.9ms/step | 8.7 samples/s +Step 41660/44000 | Loss: 2.8804 | LR: 3.60e-06 | 2290.2ms/step | 8.7 samples/s +Step 41670/44000 | Loss: 2.8287 | LR: 3.57e-06 | 2279.7ms/step | 8.8 samples/s +Step 41680/44000 | Loss: 2.8742 | LR: 3.54e-06 | 2309.6ms/step | 8.7 samples/s +Step 41690/44000 | Loss: 2.8595 | LR: 3.51e-06 | 2317.8ms/step | 8.6 samples/s +Step 41700/44000 | Loss: 2.8954 | LR: 3.48e-06 | 2300.8ms/step | 8.7 samples/s +Step 41710/44000 | Loss: 2.8432 | LR: 3.45e-06 | 2297.7ms/step | 8.7 samples/s +Step 41720/44000 | Loss: 2.8654 | LR: 3.42e-06 | 2304.5ms/step | 8.7 samples/s +Step 41730/44000 | Loss: 2.9165 | LR: 3.39e-06 | 2267.0ms/step | 8.8 samples/s +Step 41740/44000 | Loss: 2.9199 | LR: 3.36e-06 | 2259.7ms/step | 8.9 samples/s +Step 41750/44000 | Loss: 2.8535 | LR: 3.33e-06 | 2300.0ms/step | 8.7 samples/s +Step 41760/44000 | Loss: 2.9121 | LR: 3.30e-06 | 2304.9ms/step | 8.7 samples/s +Step 41770/44000 | Loss: 2.8268 | LR: 3.27e-06 | 2268.4ms/step | 8.8 samples/s +Step 41780/44000 | Loss: 2.8665 | LR: 3.24e-06 | 2271.3ms/step | 8.8 samples/s +Step 41790/44000 | Loss: 2.8970 | LR: 3.21e-06 | 2260.9ms/step | 8.8 samples/s +Step 41800/44000 | Loss: 2.8535 | LR: 3.19e-06 | 2288.2ms/step | 8.7 samples/s +Step 41810/44000 | Loss: 2.8583 | LR: 3.16e-06 | 2248.7ms/step | 8.9 samples/s +Step 41820/44000 | Loss: 2.9127 | LR: 3.13e-06 | 2252.7ms/step | 8.9 samples/s +Step 41830/44000 | Loss: 2.8485 | LR: 3.10e-06 | 2273.9ms/step | 8.8 samples/s +Step 41840/44000 | Loss: 2.8478 | LR: 3.07e-06 | 2310.8ms/step | 8.7 samples/s +Step 41850/44000 | Loss: 2.8924 | LR: 3.04e-06 | 2310.7ms/step | 8.7 samples/s +Step 41860/44000 | Loss: 2.8636 | LR: 3.01e-06 | 2276.6ms/step | 8.8 samples/s +Step 41870/44000 | Loss: 2.8624 | LR: 2.99e-06 | 2281.3ms/step | 8.8 samples/s +Step 41880/44000 | Loss: 2.9188 | LR: 2.96e-06 | 2306.3ms/step | 8.7 samples/s +Step 41890/44000 | Loss: 2.8467 | LR: 2.93e-06 | 2271.9ms/step | 8.8 samples/s +Step 41900/44000 | Loss: 2.8413 | LR: 2.90e-06 | 2271.5ms/step | 8.8 samples/s +Step 41910/44000 | Loss: 2.8904 | LR: 2.88e-06 | 2274.0ms/step | 8.8 samples/s +Step 41920/44000 | Loss: 2.8912 | LR: 2.85e-06 | 2274.8ms/step | 8.8 samples/s +Step 41930/44000 | Loss: 2.8711 | LR: 2.82e-06 | 2285.1ms/step | 8.8 samples/s +Step 41940/44000 | Loss: 2.8676 | LR: 2.79e-06 | 2289.4ms/step | 8.7 samples/s +Step 41950/44000 | Loss: 2.8565 | LR: 2.77e-06 | 2286.9ms/step | 8.7 samples/s +Step 41960/44000 | Loss: 2.8828 | LR: 2.74e-06 | 2271.4ms/step | 8.8 samples/s +Step 41970/44000 | Loss: 2.8766 | LR: 2.71e-06 | 2284.7ms/step | 8.8 samples/s +Step 41980/44000 | Loss: 2.8490 | LR: 2.69e-06 | 2301.6ms/step | 8.7 samples/s +Step 41990/44000 | Loss: 2.8189 | LR: 2.66e-06 | 2294.7ms/step | 8.7 samples/s +Step 42000/44000 | Loss: 2.9134 | LR: 2.63e-06 | 2279.5ms/step | 8.8 samples/s +Validation at step 42000: Loss = 2.8730, Perplexity = 17.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-42000 +Step 42010/44000 | Loss: 2.9071 | LR: 2.61e-06 | 4752.0ms/step | 4.2 samples/s +Step 42020/44000 | Loss: 2.8885 | LR: 2.58e-06 | 2265.5ms/step | 8.8 samples/s +Step 42030/44000 | Loss: 2.8997 | LR: 2.56e-06 | 2111.4ms/step | 9.5 samples/s +Step 42040/44000 | Loss: 2.8782 | LR: 2.53e-06 | 2284.6ms/step | 8.8 samples/s +Step 42050/44000 | Loss: 2.9189 | LR: 2.50e-06 | 2274.5ms/step | 8.8 samples/s +Step 42060/44000 | Loss: 2.9161 | LR: 2.48e-06 | 2292.8ms/step | 8.7 samples/s +Step 42070/44000 | Loss: 2.8605 | LR: 2.45e-06 | 2311.1ms/step | 8.7 samples/s +Step 42080/44000 | Loss: 2.8767 | LR: 2.43e-06 | 2285.5ms/step | 8.8 samples/s +Step 42090/44000 | Loss: 2.8843 | LR: 2.40e-06 | 2275.6ms/step | 8.8 samples/s +Step 42100/44000 | Loss: 2.9030 | LR: 2.38e-06 | 2300.1ms/step | 8.7 samples/s +Step 42110/44000 | Loss: 2.8835 | LR: 2.35e-06 | 2301.0ms/step | 8.7 samples/s +Step 42120/44000 | Loss: 2.8668 | LR: 2.33e-06 | 2295.1ms/step | 8.7 samples/s +Step 42130/44000 | Loss: 2.8936 | LR: 2.30e-06 | 2305.4ms/step | 8.7 samples/s +Step 42140/44000 | Loss: 2.8762 | LR: 2.28e-06 | 2274.9ms/step | 8.8 samples/s +Step 42150/44000 | Loss: 2.8447 | LR: 2.25e-06 | 2284.3ms/step | 8.8 samples/s +Step 42160/44000 | Loss: 2.8537 | LR: 2.23e-06 | 2306.3ms/step | 8.7 samples/s +Step 42170/44000 | Loss: 2.8447 | LR: 2.21e-06 | 2284.3ms/step | 8.8 samples/s +Step 42180/44000 | Loss: 2.8715 | LR: 2.18e-06 | 2266.6ms/step | 8.8 samples/s +Step 42190/44000 | Loss: 2.8398 | LR: 2.16e-06 | 2290.3ms/step | 8.7 samples/s +Step 42200/44000 | Loss: 2.8749 | LR: 2.13e-06 | 2308.0ms/step | 8.7 samples/s +Step 42210/44000 | Loss: 2.8890 | LR: 2.11e-06 | 2267.3ms/step | 8.8 samples/s +Step 42220/44000 | Loss: 2.8571 | LR: 2.09e-06 | 2284.9ms/step | 8.8 samples/s +Step 42230/44000 | Loss: 2.8856 | LR: 2.06e-06 | 2281.2ms/step | 8.8 samples/s +Step 42240/44000 | Loss: 2.8649 | LR: 2.04e-06 | 2261.1ms/step | 8.8 samples/s +Step 42250/44000 | Loss: 2.9159 | LR: 2.02e-06 | 2278.9ms/step | 8.8 samples/s +Step 42260/44000 | Loss: 2.8523 | LR: 1.99e-06 | 2270.7ms/step | 8.8 samples/s +Step 42270/44000 | Loss: 2.9205 | LR: 1.97e-06 | 2285.7ms/step | 8.8 samples/s +Step 42280/44000 | Loss: 2.8588 | LR: 1.95e-06 | 2272.9ms/step | 8.8 samples/s +Step 42290/44000 | Loss: 2.8717 | LR: 1.93e-06 | 2275.8ms/step | 8.8 samples/s +Step 42300/44000 | Loss: 2.9104 | LR: 1.90e-06 | 2317.2ms/step | 8.6 samples/s +Step 42310/44000 | Loss: 2.8432 | LR: 1.88e-06 | 2294.1ms/step | 8.7 samples/s +Step 42320/44000 | Loss: 2.9021 | LR: 1.86e-06 | 2293.6ms/step | 8.7 samples/s +Step 42330/44000 | Loss: 2.9273 | LR: 1.84e-06 | 2268.2ms/step | 8.8 samples/s +Step 42340/44000 | Loss: 2.8967 | LR: 1.82e-06 | 2319.9ms/step | 8.6 samples/s +Step 42350/44000 | Loss: 2.8690 | LR: 1.79e-06 | 2299.8ms/step | 8.7 samples/s +Step 42360/44000 | Loss: 2.8537 | LR: 1.77e-06 | 2309.7ms/step | 8.7 samples/s +Step 42370/44000 | Loss: 2.8728 | LR: 1.75e-06 | 2294.0ms/step | 8.7 samples/s +Step 42380/44000 | Loss: 2.8416 | LR: 1.73e-06 | 2274.9ms/step | 8.8 samples/s +Step 42390/44000 | Loss: 2.9007 | LR: 1.71e-06 | 2314.7ms/step | 8.6 samples/s +Step 42400/44000 | Loss: 2.9037 | LR: 1.69e-06 | 2284.6ms/step | 8.8 samples/s +Step 42410/44000 | Loss: 2.8416 | LR: 1.67e-06 | 2269.7ms/step | 8.8 samples/s +Step 42420/44000 | Loss: 2.8696 | LR: 1.64e-06 | 2283.8ms/step | 8.8 samples/s +Step 42430/44000 | Loss: 2.8714 | LR: 1.62e-06 | 2260.1ms/step | 8.8 samples/s +Step 42440/44000 | Loss: 2.8972 | LR: 1.60e-06 | 2281.9ms/step | 8.8 samples/s +Step 42450/44000 | Loss: 2.8706 | LR: 1.58e-06 | 2252.9ms/step | 8.9 samples/s +Step 42460/44000 | Loss: 2.8248 | LR: 1.56e-06 | 2320.2ms/step | 8.6 samples/s +Step 42470/44000 | Loss: 2.8980 | LR: 1.54e-06 | 2289.4ms/step | 8.7 samples/s +Step 42480/44000 | Loss: 2.8627 | LR: 1.52e-06 | 2261.8ms/step | 8.8 samples/s +Step 42490/44000 | Loss: 2.8997 | LR: 1.50e-06 | 2280.7ms/step | 8.8 samples/s +Step 42500/44000 | Loss: 2.8959 | LR: 1.48e-06 | 2283.5ms/step | 8.8 samples/s +Step 42510/44000 | Loss: 2.8774 | LR: 1.46e-06 | 2288.5ms/step | 8.7 samples/s +Step 42520/44000 | Loss: 2.8910 | LR: 1.44e-06 | 2288.3ms/step | 8.7 samples/s +Step 42530/44000 | Loss: 2.8507 | LR: 1.42e-06 | 2285.3ms/step | 8.8 samples/s +Step 42540/44000 | Loss: 2.8739 | LR: 1.40e-06 | 2289.5ms/step | 8.7 samples/s +Step 42550/44000 | Loss: 2.9057 | LR: 1.39e-06 | 2274.7ms/step | 8.8 samples/s +Step 42560/44000 | Loss: 2.8973 | LR: 1.37e-06 | 2285.8ms/step | 8.7 samples/s +Step 42570/44000 | Loss: 2.8539 | LR: 1.35e-06 | 2279.4ms/step | 8.8 samples/s +Step 42580/44000 | Loss: 2.8884 | LR: 1.33e-06 | 2295.5ms/step | 8.7 samples/s +Step 42590/44000 | Loss: 2.8679 | LR: 1.31e-06 | 2273.1ms/step | 8.8 samples/s +Step 42600/44000 | Loss: 2.8710 | LR: 1.29e-06 | 2254.4ms/step | 8.9 samples/s +Step 42610/44000 | Loss: 2.8919 | LR: 1.27e-06 | 2275.7ms/step | 8.8 samples/s +Step 42620/44000 | Loss: 2.8750 | LR: 1.25e-06 | 2295.4ms/step | 8.7 samples/s +Step 42630/44000 | Loss: 2.9051 | LR: 1.24e-06 | 2308.4ms/step | 8.7 samples/s +Step 42640/44000 | Loss: 2.8969 | LR: 1.22e-06 | 2299.1ms/step | 8.7 samples/s +Step 42650/44000 | Loss: 2.8799 | LR: 1.20e-06 | 2300.2ms/step | 8.7 samples/s +Step 42660/44000 | Loss: 2.9040 | LR: 1.18e-06 | 2294.8ms/step | 8.7 samples/s +Step 42670/44000 | Loss: 2.9022 | LR: 1.17e-06 | 2308.7ms/step | 8.7 samples/s +Step 42680/44000 | Loss: 2.8888 | LR: 1.15e-06 | 2289.7ms/step | 8.7 samples/s +Step 42690/44000 | Loss: 2.9230 | LR: 1.13e-06 | 2290.0ms/step | 8.7 samples/s +Step 42700/44000 | Loss: 2.9102 | LR: 1.11e-06 | 2290.8ms/step | 8.7 samples/s +Step 42710/44000 | Loss: 2.8677 | LR: 1.10e-06 | 2293.5ms/step | 8.7 samples/s +Step 42720/44000 | Loss: 2.8507 | LR: 1.08e-06 | 2305.0ms/step | 8.7 samples/s +Step 42730/44000 | Loss: 2.8834 | LR: 1.06e-06 | 2291.6ms/step | 8.7 samples/s +Step 42740/44000 | Loss: 2.8973 | LR: 1.05e-06 | 2285.7ms/step | 8.7 samples/s +Step 42750/44000 | Loss: 2.8794 | LR: 1.03e-06 | 2275.3ms/step | 8.8 samples/s +Step 42760/44000 | Loss: 2.8387 | LR: 1.01e-06 | 2291.0ms/step | 8.7 samples/s +Step 42770/44000 | Loss: 2.8444 | LR: 9.97e-07 | 2277.6ms/step | 8.8 samples/s +Step 42780/44000 | Loss: 2.8869 | LR: 9.81e-07 | 2281.9ms/step | 8.8 samples/s +Step 42790/44000 | Loss: 2.8491 | LR: 9.65e-07 | 2297.0ms/step | 8.7 samples/s +Step 42800/44000 | Loss: 2.8655 | LR: 9.49e-07 | 2273.7ms/step | 8.8 samples/s +Step 42810/44000 | Loss: 2.9140 | LR: 9.33e-07 | 2259.5ms/step | 8.9 samples/s +Step 42820/44000 | Loss: 2.8964 | LR: 9.18e-07 | 2317.3ms/step | 8.6 samples/s +Step 42830/44000 | Loss: 2.8849 | LR: 9.02e-07 | 2279.7ms/step | 8.8 samples/s +Step 42840/44000 | Loss: 2.9020 | LR: 8.87e-07 | 2280.4ms/step | 8.8 samples/s +Step 42850/44000 | Loss: 2.9176 | LR: 8.72e-07 | 2279.6ms/step | 8.8 samples/s +Step 42860/44000 | Loss: 2.8816 | LR: 8.57e-07 | 2298.7ms/step | 8.7 samples/s +Step 42870/44000 | Loss: 2.8531 | LR: 8.42e-07 | 2285.3ms/step | 8.8 samples/s +Step 42880/44000 | Loss: 2.8261 | LR: 8.27e-07 | 2302.9ms/step | 8.7 samples/s +Step 42890/44000 | Loss: 2.8885 | LR: 8.12e-07 | 2271.6ms/step | 8.8 samples/s +Step 42900/44000 | Loss: 2.8422 | LR: 7.98e-07 | 2266.5ms/step | 8.8 samples/s +Step 42910/44000 | Loss: 2.8509 | LR: 7.83e-07 | 2275.9ms/step | 8.8 samples/s +Step 42920/44000 | Loss: 2.8818 | LR: 7.69e-07 | 2273.9ms/step | 8.8 samples/s +Step 42930/44000 | Loss: 2.8926 | LR: 7.55e-07 | 2243.5ms/step | 8.9 samples/s +Step 42940/44000 | Loss: 2.8476 | LR: 7.41e-07 | 2276.8ms/step | 8.8 samples/s +Step 42950/44000 | Loss: 2.8620 | LR: 7.27e-07 | 2294.6ms/step | 8.7 samples/s +Step 42960/44000 | Loss: 2.8641 | LR: 7.13e-07 | 2304.8ms/step | 8.7 samples/s +Step 42970/44000 | Loss: 2.8910 | LR: 6.99e-07 | 2279.3ms/step | 8.8 samples/s +Step 42980/44000 | Loss: 2.8666 | LR: 6.86e-07 | 2269.2ms/step | 8.8 samples/s +Step 42990/44000 | Loss: 2.8885 | LR: 6.72e-07 | 2265.0ms/step | 8.8 samples/s +Step 43000/44000 | Loss: 2.8804 | LR: 6.59e-07 | 2279.8ms/step | 8.8 samples/s +Validation at step 43000: Loss = 2.8730, Perplexity = 17.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-43000 +Step 43010/44000 | Loss: 2.8439 | LR: 6.46e-07 | 4732.8ms/step | 4.2 samples/s +Step 43020/44000 | Loss: 2.8503 | LR: 6.33e-07 | 2278.4ms/step | 8.8 samples/s +Step 43030/44000 | Loss: 2.8378 | LR: 6.20e-07 | 2285.1ms/step | 8.8 samples/s +Step 43040/44000 | Loss: 2.8800 | LR: 6.08e-07 | 2125.7ms/step | 9.4 samples/s +Step 43050/44000 | Loss: 2.8706 | LR: 5.95e-07 | 2306.2ms/step | 8.7 samples/s +Step 43060/44000 | Loss: 2.9412 | LR: 5.83e-07 | 2290.4ms/step | 8.7 samples/s +Step 43070/44000 | Loss: 2.8930 | LR: 5.70e-07 | 2301.1ms/step | 8.7 samples/s +Step 43080/44000 | Loss: 2.8966 | LR: 5.58e-07 | 2289.6ms/step | 8.7 samples/s +Step 43090/44000 | Loss: 2.9311 | LR: 5.46e-07 | 2296.0ms/step | 8.7 samples/s +Step 43100/44000 | Loss: 2.9347 | LR: 5.34e-07 | 2314.9ms/step | 8.6 samples/s +Step 43110/44000 | Loss: 2.8533 | LR: 5.22e-07 | 2278.9ms/step | 8.8 samples/s +Step 43120/44000 | Loss: 2.8550 | LR: 5.11e-07 | 2257.1ms/step | 8.9 samples/s +Step 43130/44000 | Loss: 2.8915 | LR: 4.99e-07 | 2274.4ms/step | 8.8 samples/s +Step 43140/44000 | Loss: 2.9099 | LR: 4.88e-07 | 2278.4ms/step | 8.8 samples/s +Step 43150/44000 | Loss: 2.8626 | LR: 4.76e-07 | 2295.8ms/step | 8.7 samples/s +Step 43160/44000 | Loss: 2.8515 | LR: 4.65e-07 | 2310.8ms/step | 8.7 samples/s +Step 43170/44000 | Loss: 2.8679 | LR: 4.54e-07 | 2256.3ms/step | 8.9 samples/s +Step 43180/44000 | Loss: 2.8739 | LR: 4.43e-07 | 2295.0ms/step | 8.7 samples/s +Step 43190/44000 | Loss: 2.8784 | LR: 4.33e-07 | 2302.9ms/step | 8.7 samples/s +Step 43200/44000 | Loss: 2.8907 | LR: 4.22e-07 | 2277.9ms/step | 8.8 samples/s +Step 43210/44000 | Loss: 2.8665 | LR: 4.12e-07 | 2301.4ms/step | 8.7 samples/s +Step 43220/44000 | Loss: 2.8912 | LR: 4.01e-07 | 2262.3ms/step | 8.8 samples/s +Step 43230/44000 | Loss: 2.8895 | LR: 3.91e-07 | 2275.9ms/step | 8.8 samples/s +Step 43240/44000 | Loss: 2.8607 | LR: 3.81e-07 | 2303.2ms/step | 8.7 samples/s +Step 43250/44000 | Loss: 2.8897 | LR: 3.71e-07 | 2261.5ms/step | 8.8 samples/s +Step 43260/44000 | Loss: 2.8877 | LR: 3.61e-07 | 2295.6ms/step | 8.7 samples/s +Step 43270/44000 | Loss: 2.8770 | LR: 3.51e-07 | 2279.9ms/step | 8.8 samples/s +Step 43280/44000 | Loss: 2.8599 | LR: 3.42e-07 | 2267.3ms/step | 8.8 samples/s +Step 43290/44000 | Loss: 2.8859 | LR: 3.32e-07 | 2301.5ms/step | 8.7 samples/s +Step 43300/44000 | Loss: 2.8572 | LR: 3.23e-07 | 2302.9ms/step | 8.7 samples/s +Step 43310/44000 | Loss: 2.8773 | LR: 3.14e-07 | 2289.9ms/step | 8.7 samples/s +Step 43320/44000 | Loss: 2.8605 | LR: 3.05e-07 | 2279.6ms/step | 8.8 samples/s +Step 43330/44000 | Loss: 2.8814 | LR: 2.96e-07 | 2270.6ms/step | 8.8 samples/s +Step 43340/44000 | Loss: 2.9204 | LR: 2.87e-07 | 2237.4ms/step | 8.9 samples/s +Step 43350/44000 | Loss: 2.8964 | LR: 2.79e-07 | 2274.4ms/step | 8.8 samples/s +Step 43360/44000 | Loss: 2.8809 | LR: 2.70e-07 | 2257.9ms/step | 8.9 samples/s +Step 43370/44000 | Loss: 2.8933 | LR: 2.62e-07 | 2250.6ms/step | 8.9 samples/s +Step 43380/44000 | Loss: 2.9049 | LR: 2.53e-07 | 2295.3ms/step | 8.7 samples/s +Step 43390/44000 | Loss: 2.8628 | LR: 2.45e-07 | 2289.8ms/step | 8.7 samples/s +Step 43400/44000 | Loss: 2.8815 | LR: 2.37e-07 | 2278.8ms/step | 8.8 samples/s +Step 43410/44000 | Loss: 2.8429 | LR: 2.30e-07 | 2290.7ms/step | 8.7 samples/s +Step 43420/44000 | Loss: 2.8476 | LR: 2.22e-07 | 2306.0ms/step | 8.7 samples/s +Step 43430/44000 | Loss: 2.9331 | LR: 2.14e-07 | 2315.9ms/step | 8.6 samples/s +Step 43440/44000 | Loss: 2.8789 | LR: 2.07e-07 | 2288.7ms/step | 8.7 samples/s +Step 43450/44000 | Loss: 2.8522 | LR: 1.99e-07 | 2279.2ms/step | 8.8 samples/s +Step 43460/44000 | Loss: 2.8798 | LR: 1.92e-07 | 2283.9ms/step | 8.8 samples/s +Step 43470/44000 | Loss: 2.8896 | LR: 1.85e-07 | 2316.7ms/step | 8.6 samples/s +Step 43480/44000 | Loss: 2.8860 | LR: 1.78e-07 | 2278.9ms/step | 8.8 samples/s +Step 43490/44000 | Loss: 2.9009 | LR: 1.72e-07 | 2268.5ms/step | 8.8 samples/s +Step 43500/44000 | Loss: 2.8863 | LR: 1.65e-07 | 2275.0ms/step | 8.8 samples/s +Step 43510/44000 | Loss: 2.8796 | LR: 1.58e-07 | 2271.2ms/step | 8.8 samples/s +Step 43520/44000 | Loss: 2.8967 | LR: 1.52e-07 | 2288.8ms/step | 8.7 samples/s +Step 43530/44000 | Loss: 2.8674 | LR: 1.46e-07 | 2281.6ms/step | 8.8 samples/s +Step 43540/44000 | Loss: 2.8771 | LR: 1.40e-07 | 2314.3ms/step | 8.6 samples/s +Step 43550/44000 | Loss: 2.8807 | LR: 1.34e-07 | 2284.5ms/step | 8.8 samples/s +Step 43560/44000 | Loss: 2.8860 | LR: 1.28e-07 | 2273.7ms/step | 8.8 samples/s +Step 43570/44000 | Loss: 2.9036 | LR: 1.22e-07 | 2280.2ms/step | 8.8 samples/s +Step 43580/44000 | Loss: 2.8739 | LR: 1.16e-07 | 2284.0ms/step | 8.8 samples/s +Step 43590/44000 | Loss: 2.9053 | LR: 1.11e-07 | 2303.3ms/step | 8.7 samples/s +Step 43600/44000 | Loss: 2.8713 | LR: 1.06e-07 | 2277.0ms/step | 8.8 samples/s +Step 43610/44000 | Loss: 2.8310 | LR: 1.00e-07 | 2253.5ms/step | 8.9 samples/s +Step 43620/44000 | Loss: 2.8463 | LR: 9.52e-08 | 2268.9ms/step | 8.8 samples/s +Step 43630/44000 | Loss: 2.8745 | LR: 9.03e-08 | 2290.1ms/step | 8.7 samples/s +Step 43640/44000 | Loss: 2.8667 | LR: 8.55e-08 | 2258.9ms/step | 8.9 samples/s +Step 43650/44000 | Loss: 2.8308 | LR: 8.08e-08 | 2254.2ms/step | 8.9 samples/s +Step 43660/44000 | Loss: 2.8446 | LR: 7.62e-08 | 2280.4ms/step | 8.8 samples/s +Step 43670/44000 | Loss: 2.8807 | LR: 7.18e-08 | 2263.1ms/step | 8.8 samples/s +Step 43680/44000 | Loss: 2.8598 | LR: 6.75e-08 | 2273.7ms/step | 8.8 samples/s +Step 43690/44000 | Loss: 2.9169 | LR: 6.34e-08 | 2289.4ms/step | 8.7 samples/s +Step 43700/44000 | Loss: 2.8583 | LR: 5.94e-08 | 2258.0ms/step | 8.9 samples/s +Step 43710/44000 | Loss: 2.8778 | LR: 5.55e-08 | 2280.2ms/step | 8.8 samples/s +Step 43720/44000 | Loss: 2.8783 | LR: 5.17e-08 | 2255.0ms/step | 8.9 samples/s +Step 43730/44000 | Loss: 2.9043 | LR: 4.81e-08 | 2277.1ms/step | 8.8 samples/s +Step 43740/44000 | Loss: 2.8776 | LR: 4.46e-08 | 2291.2ms/step | 8.7 samples/s +Step 43750/44000 | Loss: 2.8954 | LR: 4.12e-08 | 2284.6ms/step | 8.8 samples/s +Step 43760/44000 | Loss: 2.8842 | LR: 3.80e-08 | 2313.3ms/step | 8.6 samples/s +Step 43770/44000 | Loss: 2.9176 | LR: 3.49e-08 | 2277.7ms/step | 8.8 samples/s +Step 43780/44000 | Loss: 2.8788 | LR: 3.19e-08 | 2296.5ms/step | 8.7 samples/s +Step 43790/44000 | Loss: 2.9006 | LR: 2.91e-08 | 2284.5ms/step | 8.8 samples/s +Step 43800/44000 | Loss: 2.8586 | LR: 2.64e-08 | 2276.2ms/step | 8.8 samples/s +Step 43810/44000 | Loss: 2.8299 | LR: 2.38e-08 | 2283.7ms/step | 8.8 samples/s +Step 43820/44000 | Loss: 2.8425 | LR: 2.14e-08 | 2288.1ms/step | 8.7 samples/s +Step 43830/44000 | Loss: 2.8497 | LR: 1.91e-08 | 2266.5ms/step | 8.8 samples/s +Step 43840/44000 | Loss: 2.8785 | LR: 1.69e-08 | 2280.3ms/step | 8.8 samples/s +Step 43850/44000 | Loss: 2.8717 | LR: 1.48e-08 | 2292.0ms/step | 8.7 samples/s +Step 43860/44000 | Loss: 2.9151 | LR: 1.29e-08 | 2311.1ms/step | 8.7 samples/s +Step 43870/44000 | Loss: 2.8889 | LR: 1.11e-08 | 2267.4ms/step | 8.8 samples/s +Step 43880/44000 | Loss: 2.8857 | LR: 9.50e-09 | 2326.4ms/step | 8.6 samples/s +Step 43890/44000 | Loss: 2.8601 | LR: 7.98e-09 | 2292.7ms/step | 8.7 samples/s +Step 43900/44000 | Loss: 2.8377 | LR: 6.60e-09 | 2285.6ms/step | 8.8 samples/s +Step 43910/44000 | Loss: 2.8842 | LR: 5.34e-09 | 2296.0ms/step | 8.7 samples/s +Step 43920/44000 | Loss: 2.8317 | LR: 4.22e-09 | 2300.6ms/step | 8.7 samples/s +Step 43930/44000 | Loss: 2.8562 | LR: 3.23e-09 | 2273.7ms/step | 8.8 samples/s +Step 43940/44000 | Loss: 2.8956 | LR: 2.37e-09 | 2261.7ms/step | 8.8 samples/s +Step 43950/44000 | Loss: 2.9135 | LR: 1.65e-09 | 2285.5ms/step | 8.8 samples/s +Step 43960/44000 | Loss: 2.9154 | LR: 1.06e-09 | 2300.2ms/step | 8.7 samples/s +Step 43970/44000 | Loss: 2.8724 | LR: 5.94e-10 | 2277.7ms/step | 8.8 samples/s +Step 43980/44000 | Loss: 2.9022 | LR: 2.64e-10 | 2279.7ms/step | 8.8 samples/s +Step 43990/44000 | Loss: 2.8514 | LR: 6.60e-11 | 2282.9ms/step | 8.8 samples/s +Step 44000/44000 | Loss: 2.8737 | LR: 0.00e+00 | 2332.5ms/step | 8.6 samples/s +Validation at step 44000: Loss = 2.8730, Perplexity = 17.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-44000 +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading history steps 3800-3800, summary, console lines 2800-2802 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▇▆▅▅▄▄▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▆▆▅▅▄▃▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▅▄▇██▇▇▁▂▂▂▂▂▂▃▃▂▁▁▂▁▄▄▄▃▃▃▃▃▃▃▃▃▃▃▃▄▃▃▃ +wandb: speed/samples_per_sec ▂▁▁▄▂▂▇▇▇▇█▇▇▆▇▆▇▇██▇▆▆▅▆▆▆▆▅▆▅▆▆▆▆▇▅▆▆▆ +wandb: speed/steps_per_sec ▃▁▁▁▃██▇▇██▇▇█▇█▆███▇▇▅▅▅▅▅▆▇▆▆▇▆▇▆▆▆▆▆▅ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.87302 +wandb: eval/perplexity 17.69029 +wandb: eval/uses_dense_attention 1 +wandb: eval/uses_sparse_attention 0 +wandb: speed/ms_per_step 2332.46992 +wandb: speed/samples_per_sec 8.5746 +wandb: speed/steps_per_sec 0.42873 +wandb: +4 ... +wandb: +wandb: 🚀 View run l8k365m_dense_ropectl_a1p0_seed45 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/xfnxdn24 +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260820_140811-xfnxdn24/logs +Training complete! diff --git a/epfl_final/train_logs/ropectl_a1p0_a100_ddp2/seed42.20260820T140226Z.log b/epfl_final/train_logs/ropectl_a1p0_a100_ddp2/seed42.20260820T140226Z.log new file mode 100644 index 0000000000000000000000000000000000000000..9965d027f8e8aabe24ee88f18accad8d816da671 --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_a100_ddp2/seed42.20260820T140226Z.log @@ -0,0 +1,847 @@ +W0820 14:02:40.078000 229 site-packages/torch/distributed/run.py:851] +W0820 14:02:40.078000 229 site-packages/torch/distributed/run.py:851] ***************************************** +W0820 14:02:40.078000 229 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0820 14:02:40.078000 229 site-packages/torch/distributed/run.py:851] ***************************************** +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-17000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-17000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed42", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control", + "ddp2-a100" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-17000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... + +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-17000... +Preserving checkpoint LR state for exact resume: current=[0.0003451463250063146], schedule_base=[0.0005] +Resumed from checkpoint at step 17000 (epoch 0, micro_step 170000) +Resuming training from step 17000 +Resuming wandb run jqstnvej +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run jqstnvej +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140401-jqstnvej +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed42 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/jqstnvej +Starting training... +Validation at step 17000: Loss = 3.0074, Perplexity = 20.23 +wandb: WARNING Tried to log to step 17000 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 17001: dense (dense step interval) +Step 17010/44000 | Loss: 2.9953 | LR: 3.45e-04 | 42298.8ms/step | 0.5 samples/s +wandb: WARNING Tried to log to step 17010 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17020/44000 | Loss: 3.0000 | LR: 3.45e-04 | 1764.7ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17020 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17030/44000 | Loss: 2.9989 | LR: 3.45e-04 | 1765.3ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17030 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17040/44000 | Loss: 2.9807 | LR: 3.44e-04 | 1767.9ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17040 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17050/44000 | Loss: 2.9804 | LR: 3.44e-04 | 1767.1ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17050 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17060/44000 | Loss: 3.0394 | LR: 3.44e-04 | 1766.2ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17060 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17070/44000 | Loss: 3.0309 | LR: 3.44e-04 | 1768.2ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17070 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17080/44000 | Loss: 3.0213 | LR: 3.44e-04 | 1768.4ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17080 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17090/44000 | Loss: 3.0357 | LR: 3.44e-04 | 1769.2ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17090 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17100/44000 | Loss: 2.9922 | LR: 3.43e-04 | 1769.5ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17100 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17110/44000 | Loss: 2.9951 | LR: 3.43e-04 | 1769.3ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17110 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17120/44000 | Loss: 2.9471 | LR: 3.43e-04 | 1768.7ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17120 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17130/44000 | Loss: 2.9969 | LR: 3.43e-04 | 1767.5ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17130 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17140/44000 | Loss: 2.9777 | LR: 3.43e-04 | 1767.6ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17140 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17150/44000 | Loss: 2.9867 | LR: 3.43e-04 | 1768.0ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17150 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17160/44000 | Loss: 2.9971 | LR: 3.42e-04 | 1768.9ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17160 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17170/44000 | Loss: 2.9694 | LR: 3.42e-04 | 1768.6ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17170 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17180/44000 | Loss: 2.9811 | LR: 3.42e-04 | 1768.6ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17180 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17190/44000 | Loss: 3.0168 | LR: 3.42e-04 | 1768.7ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17190 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17200/44000 | Loss: 2.9846 | LR: 3.42e-04 | 1769.9ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17200 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17210/44000 | Loss: 2.9550 | LR: 3.42e-04 | 1768.8ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17210 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17220/44000 | Loss: 2.9769 | LR: 3.41e-04 | 1769.4ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17220 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17230/44000 | Loss: 3.0017 | LR: 3.41e-04 | 1769.2ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17230 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17240/44000 | Loss: 2.9902 | LR: 3.41e-04 | 1769.0ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17240 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17250/44000 | Loss: 3.0014 | LR: 3.41e-04 | 1768.4ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17250 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17260/44000 | Loss: 3.0338 | LR: 3.41e-04 | 1769.7ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17260 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17270/44000 | Loss: 3.0320 | LR: 3.41e-04 | 1769.6ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17270 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17280/44000 | Loss: 2.9712 | LR: 3.40e-04 | 1769.3ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17280 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17290/44000 | Loss: 3.0376 | LR: 3.40e-04 | 1770.7ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17290 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17300/44000 | Loss: 3.0024 | LR: 3.40e-04 | 1770.3ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17300 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17310/44000 | Loss: 2.9896 | LR: 3.40e-04 | 1770.0ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17310 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17320/44000 | Loss: 3.0202 | LR: 3.40e-04 | 1772.8ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17320 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17330/44000 | Loss: 2.9877 | LR: 3.40e-04 | 1769.8ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17330 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17340/44000 | Loss: 3.0100 | LR: 3.39e-04 | 1768.7ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17340 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17350/44000 | Loss: 2.9876 | LR: 3.39e-04 | 1768.4ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17350 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17360/44000 | Loss: 2.9364 | LR: 3.39e-04 | 1768.5ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17360 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17370/44000 | Loss: 3.0425 | LR: 3.39e-04 | 1767.9ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17370 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17380/44000 | Loss: 2.9622 | LR: 3.39e-04 | 1768.4ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17380 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17390/44000 | Loss: 3.0031 | LR: 3.39e-04 | 1767.7ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17390 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17400/44000 | Loss: 2.9896 | LR: 3.38e-04 | 1768.1ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17400 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17410/44000 | Loss: 2.9699 | LR: 3.38e-04 | 1769.6ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17410 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17420/44000 | Loss: 2.9919 | LR: 3.38e-04 | 1769.2ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17420 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17430/44000 | Loss: 2.9849 | LR: 3.38e-04 | 1767.6ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17430 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17440/44000 | Loss: 2.9565 | LR: 3.38e-04 | 1767.9ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17440 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17450/44000 | Loss: 3.0010 | LR: 3.38e-04 | 1767.9ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17450 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17460/44000 | Loss: 2.9859 | LR: 3.37e-04 | 1768.2ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17460 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17470/44000 | Loss: 2.9710 | LR: 3.37e-04 | 1769.0ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 17470 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17480/44000 | Loss: 2.9962 | LR: 3.37e-04 | 1769.1ms/step | 11.3 samples/s +Step 17490/44000 | Loss: 3.0029 | LR: 3.37e-04 | 1770.1ms/step | 11.3 samples/s +Step 17500/44000 | Loss: 2.9515 | LR: 3.37e-04 | 1769.6ms/step | 11.3 samples/s +Step 17510/44000 | Loss: 2.9864 | LR: 3.37e-04 | 1769.0ms/step | 11.3 samples/s +Step 17520/44000 | Loss: 3.0140 | LR: 3.36e-04 | 1769.8ms/step | 11.3 samples/s +Step 17530/44000 | Loss: 2.9341 | LR: 3.36e-04 | 1769.9ms/step | 11.3 samples/s +Step 17540/44000 | Loss: 2.9562 | LR: 3.36e-04 | 1768.0ms/step | 11.3 samples/s +Step 17550/44000 | Loss: 2.9230 | LR: 3.36e-04 | 1768.3ms/step | 11.3 samples/s +Step 17560/44000 | Loss: 2.9470 | LR: 3.36e-04 | 1768.6ms/step | 11.3 samples/s +Step 17570/44000 | Loss: 2.9793 | LR: 3.35e-04 | 1768.6ms/step | 11.3 samples/s +Step 17580/44000 | Loss: 2.9684 | LR: 3.35e-04 | 1769.8ms/step | 11.3 samples/s +Step 17590/44000 | Loss: 3.0173 | LR: 3.35e-04 | 1771.3ms/step | 11.3 samples/s +Step 17600/44000 | Loss: 3.0286 | LR: 3.35e-04 | 1770.7ms/step | 11.3 samples/s +Step 17610/44000 | Loss: 3.0003 | LR: 3.35e-04 | 1769.6ms/step | 11.3 samples/s +Step 17620/44000 | Loss: 2.9908 | LR: 3.35e-04 | 1768.5ms/step | 11.3 samples/s +Step 17630/44000 | Loss: 3.0167 | LR: 3.34e-04 | 1770.4ms/step | 11.3 samples/s +Step 17640/44000 | Loss: 3.0106 | LR: 3.34e-04 | 1769.1ms/step | 11.3 samples/s +Step 17650/44000 | Loss: 2.9579 | LR: 3.34e-04 | 1768.7ms/step | 11.3 samples/s +Step 17660/44000 | Loss: 2.9935 | LR: 3.34e-04 | 1769.7ms/step | 11.3 samples/s +Step 17670/44000 | Loss: 3.0074 | LR: 3.34e-04 | 1770.4ms/step | 11.3 samples/s +Step 17680/44000 | Loss: 2.9855 | LR: 3.34e-04 | 1769.7ms/step | 11.3 samples/s +Step 17690/44000 | Loss: 2.9490 | LR: 3.33e-04 | 1769.4ms/step | 11.3 samples/s +Step 17700/44000 | Loss: 3.0226 | LR: 3.33e-04 | 1769.8ms/step | 11.3 samples/s +Step 17710/44000 | Loss: 3.0278 | LR: 3.33e-04 | 1772.0ms/step | 11.3 samples/s +Step 17720/44000 | Loss: 2.9615 | LR: 3.33e-04 | 1770.4ms/step | 11.3 samples/s +Step 17730/44000 | Loss: 3.0084 | LR: 3.33e-04 | 1771.7ms/step | 11.3 samples/s +Step 17740/44000 | Loss: 2.9796 | LR: 3.33e-04 | 1769.3ms/step | 11.3 samples/s +Step 17750/44000 | Loss: 2.9873 | LR: 3.32e-04 | 1770.7ms/step | 11.3 samples/s +Step 17760/44000 | Loss: 2.9708 | LR: 3.32e-04 | 1773.3ms/step | 11.3 samples/s +Step 17770/44000 | Loss: 2.9688 | LR: 3.32e-04 | 1772.8ms/step | 11.3 samples/s +Step 17780/44000 | Loss: 2.9390 | LR: 3.32e-04 | 1774.8ms/step | 11.3 samples/s +Step 17790/44000 | Loss: 2.9552 | LR: 3.32e-04 | 1770.7ms/step | 11.3 samples/s +Step 17800/44000 | Loss: 3.0171 | LR: 3.32e-04 | 1769.5ms/step | 11.3 samples/s +Step 17810/44000 | Loss: 3.0365 | LR: 3.31e-04 | 1771.1ms/step | 11.3 samples/s +Step 17820/44000 | Loss: 3.0011 | LR: 3.31e-04 | 1770.9ms/step | 11.3 samples/s +Step 17830/44000 | Loss: 3.0162 | LR: 3.31e-04 | 1768.5ms/step | 11.3 samples/s +Step 17840/44000 | Loss: 2.9913 | LR: 3.31e-04 | 1772.6ms/step | 11.3 samples/s +Step 17850/44000 | Loss: 2.9622 | LR: 3.31e-04 | 1772.4ms/step | 11.3 samples/s +Step 17860/44000 | Loss: 3.0152 | LR: 3.31e-04 | 1770.3ms/step | 11.3 samples/s +Step 17870/44000 | Loss: 2.9732 | LR: 3.30e-04 | 1769.4ms/step | 11.3 samples/s +Step 17880/44000 | Loss: 2.9904 | LR: 3.30e-04 | 1768.9ms/step | 11.3 samples/s +Step 17890/44000 | Loss: 2.9925 | LR: 3.30e-04 | 1770.4ms/step | 11.3 samples/s +Step 17900/44000 | Loss: 2.9819 | LR: 3.30e-04 | 1769.9ms/step | 11.3 samples/s +Step 17910/44000 | Loss: 2.9529 | LR: 3.30e-04 | 1770.6ms/step | 11.3 samples/s +Step 17920/44000 | Loss: 3.0158 | LR: 3.29e-04 | 1769.9ms/step | 11.3 samples/s +Step 17930/44000 | Loss: 2.9927 | LR: 3.29e-04 | 1770.5ms/step | 11.3 samples/s +Step 17940/44000 | Loss: 2.9965 | LR: 3.29e-04 | 1769.9ms/step | 11.3 samples/s +Step 17950/44000 | Loss: 2.9844 | LR: 3.29e-04 | 1767.6ms/step | 11.3 samples/s +Step 17960/44000 | Loss: 2.9831 | LR: 3.29e-04 | 1769.0ms/step | 11.3 samples/s +Step 17970/44000 | Loss: 2.9568 | LR: 3.29e-04 | 1770.8ms/step | 11.3 samples/s +Step 17980/44000 | Loss: 2.9801 | LR: 3.28e-04 | 1771.5ms/step | 11.3 samples/s +Step 17990/44000 | Loss: 2.9367 | LR: 3.28e-04 | 1771.4ms/step | 11.3 samples/s +Step 18000/44000 | Loss: 3.0193 | LR: 3.28e-04 | 1769.5ms/step | 11.3 samples/s +Validation at step 18000: Loss = 2.9956, Perplexity = 20.00 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-18000 +Step 18010/44000 | Loss: 2.9897 | LR: 3.28e-04 | 3629.0ms/step | 5.5 samples/s +Step 18020/44000 | Loss: 2.9957 | LR: 3.28e-04 | 1768.1ms/step | 11.3 samples/s +Step 18030/44000 | Loss: 2.9941 | LR: 3.28e-04 | 1769.8ms/step | 11.3 samples/s +Step 18040/44000 | Loss: 2.9955 | LR: 3.27e-04 | 1772.3ms/step | 11.3 samples/s +Step 18050/44000 | Loss: 3.0196 | LR: 3.27e-04 | 1769.6ms/step | 11.3 samples/s +Step 18060/44000 | Loss: 2.9550 | LR: 3.27e-04 | 1768.9ms/step | 11.3 samples/s +Step 18070/44000 | Loss: 2.9920 | LR: 3.27e-04 | 1770.0ms/step | 11.3 samples/s +Step 18080/44000 | Loss: 2.9590 | LR: 3.27e-04 | 1771.4ms/step | 11.3 samples/s +Step 18090/44000 | Loss: 2.9606 | LR: 3.27e-04 | 1770.9ms/step | 11.3 samples/s +Step 18100/44000 | Loss: 2.9588 | LR: 3.26e-04 | 1771.8ms/step | 11.3 samples/s +Step 18110/44000 | Loss: 2.9402 | LR: 3.26e-04 | 1772.2ms/step | 11.3 samples/s +Step 18120/44000 | Loss: 3.0048 | LR: 3.26e-04 | 1772.8ms/step | 11.3 samples/s +Step 18130/44000 | Loss: 2.9780 | LR: 3.26e-04 | 1773.0ms/step | 11.3 samples/s +Step 18140/44000 | Loss: 2.9664 | LR: 3.26e-04 | 1769.4ms/step | 11.3 samples/s +Step 18150/44000 | Loss: 2.9470 | LR: 3.26e-04 | 1769.9ms/step | 11.3 samples/s +Step 18160/44000 | Loss: 2.9995 | LR: 3.25e-04 | 1770.8ms/step | 11.3 samples/s +Step 18170/44000 | Loss: 2.9699 | LR: 3.25e-04 | 1769.8ms/step | 11.3 samples/s +Step 18180/44000 | Loss: 2.9892 | LR: 3.25e-04 | 1769.9ms/step | 11.3 samples/s +Step 18190/44000 | Loss: 2.9809 | LR: 3.25e-04 | 1772.5ms/step | 11.3 samples/s +Step 18200/44000 | Loss: 2.9980 | LR: 3.25e-04 | 1771.5ms/step | 11.3 samples/s +Step 18210/44000 | Loss: 2.9697 | LR: 3.24e-04 | 1771.8ms/step | 11.3 samples/s +Step 18220/44000 | Loss: 2.9545 | LR: 3.24e-04 | 1770.8ms/step | 11.3 samples/s +Step 18230/44000 | Loss: 2.9375 | LR: 3.24e-04 | 1770.8ms/step | 11.3 samples/s +Step 18240/44000 | Loss: 2.9661 | LR: 3.24e-04 | 1772.6ms/step | 11.3 samples/s +Step 18250/44000 | Loss: 2.9907 | LR: 3.24e-04 | 1771.9ms/step | 11.3 samples/s +Step 18260/44000 | Loss: 2.9708 | LR: 3.24e-04 | 1772.0ms/step | 11.3 samples/s +Step 18270/44000 | Loss: 2.9797 | LR: 3.23e-04 | 1771.2ms/step | 11.3 samples/s +Step 18280/44000 | Loss: 2.9829 | LR: 3.23e-04 | 1771.5ms/step | 11.3 samples/s +Step 18290/44000 | Loss: 2.9986 | LR: 3.23e-04 | 1770.7ms/step | 11.3 samples/s +Step 18300/44000 | Loss: 2.9809 | LR: 3.23e-04 | 1771.9ms/step | 11.3 samples/s +Step 18310/44000 | Loss: 2.9801 | LR: 3.23e-04 | 1772.4ms/step | 11.3 samples/s +Step 18320/44000 | Loss: 2.9896 | LR: 3.23e-04 | 1772.6ms/step | 11.3 samples/s +Step 18330/44000 | Loss: 2.9907 | LR: 3.22e-04 | 1772.7ms/step | 11.3 samples/s +Step 18340/44000 | Loss: 3.0383 | LR: 3.22e-04 | 1770.9ms/step | 11.3 samples/s +Step 18350/44000 | Loss: 2.9696 | LR: 3.22e-04 | 1769.9ms/step | 11.3 samples/s +Step 18360/44000 | Loss: 2.9360 | LR: 3.22e-04 | 1770.6ms/step | 11.3 samples/s +Step 18370/44000 | Loss: 2.9669 | LR: 3.22e-04 | 1768.4ms/step | 11.3 samples/s +Step 18380/44000 | Loss: 2.9547 | LR: 3.22e-04 | 1769.3ms/step | 11.3 samples/s +Step 18390/44000 | Loss: 2.9836 | LR: 3.21e-04 | 1769.7ms/step | 11.3 samples/s +Step 18400/44000 | Loss: 2.9594 | LR: 3.21e-04 | 1770.6ms/step | 11.3 samples/s +Step 18410/44000 | Loss: 2.9419 | LR: 3.21e-04 | 1771.6ms/step | 11.3 samples/s +Step 18420/44000 | Loss: 3.0278 | LR: 3.21e-04 | 1769.8ms/step | 11.3 samples/s +Step 18430/44000 | Loss: 3.0356 | LR: 3.21e-04 | 1770.7ms/step | 11.3 samples/s +Step 18440/44000 | Loss: 2.9890 | LR: 3.20e-04 | 1770.9ms/step | 11.3 samples/s +Step 18450/44000 | Loss: 2.9739 | LR: 3.20e-04 | 1770.5ms/step | 11.3 samples/s +Step 18460/44000 | Loss: 2.9957 | LR: 3.20e-04 | 1772.4ms/step | 11.3 samples/s +Step 18470/44000 | Loss: 2.9845 | LR: 3.20e-04 | 1770.9ms/step | 11.3 samples/s +Step 18480/44000 | Loss: 2.9698 | LR: 3.20e-04 | 1771.4ms/step | 11.3 samples/s +Step 18490/44000 | Loss: 2.9566 | LR: 3.20e-04 | 1770.4ms/step | 11.3 samples/s +Step 18500/44000 | Loss: 2.9605 | LR: 3.19e-04 | 1771.3ms/step | 11.3 samples/s +Step 18510/44000 | Loss: 2.9494 | LR: 3.19e-04 | 1771.1ms/step | 11.3 samples/s +Step 18520/44000 | Loss: 2.9192 | LR: 3.19e-04 | 1770.7ms/step | 11.3 samples/s +Step 18530/44000 | Loss: 2.9784 | LR: 3.19e-04 | 1770.9ms/step | 11.3 samples/s +Step 18540/44000 | Loss: 2.9874 | LR: 3.19e-04 | 1771.8ms/step | 11.3 samples/s +Step 18550/44000 | Loss: 2.9844 | LR: 3.19e-04 | 1769.3ms/step | 11.3 samples/s +Step 18560/44000 | Loss: 2.9353 | LR: 3.18e-04 | 1769.7ms/step | 11.3 samples/s +Step 18570/44000 | Loss: 2.9711 | LR: 3.18e-04 | 1770.9ms/step | 11.3 samples/s +Step 18580/44000 | Loss: 3.0260 | LR: 3.18e-04 | 1770.8ms/step | 11.3 samples/s +Step 18590/44000 | Loss: 2.9768 | LR: 3.18e-04 | 1772.7ms/step | 11.3 samples/s +Step 18600/44000 | Loss: 2.9523 | LR: 3.18e-04 | 1770.3ms/step | 11.3 samples/s +Step 18610/44000 | Loss: 3.0193 | LR: 3.18e-04 | 1771.4ms/step | 11.3 samples/s +Step 18620/44000 | Loss: 2.9781 | LR: 3.17e-04 | 1769.3ms/step | 11.3 samples/s +Step 18630/44000 | Loss: 2.9890 | LR: 3.17e-04 | 1770.0ms/step | 11.3 samples/s +Step 18640/44000 | Loss: 2.9644 | LR: 3.17e-04 | 1770.3ms/step | 11.3 samples/s +Step 18650/44000 | Loss: 3.0036 | LR: 3.17e-04 | 1771.1ms/step | 11.3 samples/s +Step 18660/44000 | Loss: 2.9546 | LR: 3.17e-04 | 1769.6ms/step | 11.3 samples/s +Step 18670/44000 | Loss: 2.9598 | LR: 3.16e-04 | 1771.4ms/step | 11.3 samples/s +Step 18680/44000 | Loss: 3.0120 | LR: 3.16e-04 | 1770.7ms/step | 11.3 samples/s +Step 18690/44000 | Loss: 2.9864 | LR: 3.16e-04 | 1769.8ms/step | 11.3 samples/s +Step 18700/44000 | Loss: 2.9835 | LR: 3.16e-04 | 1770.4ms/step | 11.3 samples/s +Step 18710/44000 | Loss: 2.9747 | LR: 3.16e-04 | 1770.7ms/step | 11.3 samples/s +Step 18720/44000 | Loss: 2.9969 | LR: 3.16e-04 | 1771.5ms/step | 11.3 samples/s +Step 18730/44000 | Loss: 2.9643 | LR: 3.15e-04 | 1771.6ms/step | 11.3 samples/s +Step 18740/44000 | Loss: 2.9840 | LR: 3.15e-04 | 1771.7ms/step | 11.3 samples/s +Step 18750/44000 | Loss: 2.9541 | LR: 3.15e-04 | 1771.9ms/step | 11.3 samples/s +Step 18760/44000 | Loss: 2.9484 | LR: 3.15e-04 | 1768.8ms/step | 11.3 samples/s +Step 18770/44000 | Loss: 2.9171 | LR: 3.15e-04 | 1770.0ms/step | 11.3 samples/s +Step 18780/44000 | Loss: 2.9861 | LR: 3.15e-04 | 1769.6ms/step | 11.3 samples/s +Step 18790/44000 | Loss: 2.9754 | LR: 3.14e-04 | 1769.6ms/step | 11.3 samples/s +Step 18800/44000 | Loss: 2.9667 | LR: 3.14e-04 | 1770.9ms/step | 11.3 samples/s +Step 18810/44000 | Loss: 2.9540 | LR: 3.14e-04 | 1770.4ms/step | 11.3 samples/s +Step 18820/44000 | Loss: 2.9444 | LR: 3.14e-04 | 1771.2ms/step | 11.3 samples/s +Step 18830/44000 | Loss: 2.9815 | LR: 3.14e-04 | 1770.9ms/step | 11.3 samples/s +Step 18840/44000 | Loss: 2.9418 | LR: 3.13e-04 | 1770.7ms/step | 11.3 samples/s +Step 18850/44000 | Loss: 2.9511 | LR: 3.13e-04 | 1770.5ms/step | 11.3 samples/s +Step 18860/44000 | Loss: 2.9654 | LR: 3.13e-04 | 1770.4ms/step | 11.3 samples/s +Step 18870/44000 | Loss: 2.9814 | LR: 3.13e-04 | 1770.9ms/step | 11.3 samples/s +Step 18880/44000 | Loss: 2.9764 | LR: 3.13e-04 | 1769.8ms/step | 11.3 samples/s +Step 18890/44000 | Loss: 2.9910 | LR: 3.13e-04 | 1769.6ms/step | 11.3 samples/s +Step 18900/44000 | Loss: 3.0031 | LR: 3.12e-04 | 1770.1ms/step | 11.3 samples/s +Step 18910/44000 | Loss: 2.9752 | LR: 3.12e-04 | 1771.0ms/step | 11.3 samples/s +Step 18920/44000 | Loss: 3.0025 | LR: 3.12e-04 | 1770.4ms/step | 11.3 samples/s +Step 18930/44000 | Loss: 3.0012 | LR: 3.12e-04 | 1771.5ms/step | 11.3 samples/s +Step 18940/44000 | Loss: 2.9471 | LR: 3.12e-04 | 1771.2ms/step | 11.3 samples/s +Step 18950/44000 | Loss: 2.9523 | LR: 3.12e-04 | 1772.1ms/step | 11.3 samples/s +Step 18960/44000 | Loss: 2.9595 | LR: 3.11e-04 | 1769.2ms/step | 11.3 samples/s +Step 18970/44000 | Loss: 2.9815 | LR: 3.11e-04 | 1770.6ms/step | 11.3 samples/s +Step 18980/44000 | Loss: 2.9753 | LR: 3.11e-04 | 1769.9ms/step | 11.3 samples/s +Step 18990/44000 | Loss: 3.0137 | LR: 3.11e-04 | 1769.1ms/step | 11.3 samples/s +Step 19000/44000 | Loss: 3.0280 | LR: 3.11e-04 | 1769.0ms/step | 11.3 samples/s +Validation at step 19000: Loss = 2.9844, Perplexity = 19.77 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-19000 +Step 19010/44000 | Loss: 2.9494 | LR: 3.11e-04 | 3549.4ms/step | 5.6 samples/s +Step 19020/44000 | Loss: 2.9590 | LR: 3.10e-04 | 1768.8ms/step | 11.3 samples/s +Step 19030/44000 | Loss: 2.9950 | LR: 3.10e-04 | 1771.1ms/step | 11.3 samples/s +Step 19040/44000 | Loss: 2.9828 | LR: 3.10e-04 | 1770.4ms/step | 11.3 samples/s +Step 19050/44000 | Loss: 2.9433 | LR: 3.10e-04 | 1771.6ms/step | 11.3 samples/s +Step 19060/44000 | Loss: 3.0023 | LR: 3.10e-04 | 1771.6ms/step | 11.3 samples/s +Step 19070/44000 | Loss: 2.9470 | LR: 3.09e-04 | 1771.7ms/step | 11.3 samples/s +Step 19080/44000 | Loss: 2.9568 | LR: 3.09e-04 | 1771.3ms/step | 11.3 samples/s +Step 19090/44000 | Loss: 2.9839 | LR: 3.09e-04 | 1773.3ms/step | 11.3 samples/s +Step 19100/44000 | Loss: 2.9568 | LR: 3.09e-04 | 1772.0ms/step | 11.3 samples/s +Step 19110/44000 | Loss: 2.9778 | LR: 3.09e-04 | 1771.4ms/step | 11.3 samples/s +Step 19120/44000 | Loss: 2.9881 | LR: 3.09e-04 | 1771.3ms/step | 11.3 samples/s +Step 19130/44000 | Loss: 2.9526 | LR: 3.08e-04 | 1772.5ms/step | 11.3 samples/s +Step 19140/44000 | Loss: 2.9738 | LR: 3.08e-04 | 1771.2ms/step | 11.3 samples/s +Step 19150/44000 | Loss: 2.9667 | LR: 3.08e-04 | 1772.0ms/step | 11.3 samples/s +Step 19160/44000 | Loss: 2.9943 | LR: 3.08e-04 | 1769.6ms/step | 11.3 samples/s +Step 19170/44000 | Loss: 2.9402 | LR: 3.08e-04 | 1770.2ms/step | 11.3 samples/s +Step 19180/44000 | Loss: 2.9953 | LR: 3.08e-04 | 1771.5ms/step | 11.3 samples/s +Step 19190/44000 | Loss: 3.0067 | LR: 3.07e-04 | 1771.0ms/step | 11.3 samples/s +Step 19200/44000 | Loss: 2.9986 | LR: 3.07e-04 | 1770.2ms/step | 11.3 samples/s +Step 19210/44000 | Loss: 2.9489 | LR: 3.07e-04 | 1770.3ms/step | 11.3 samples/s +Step 19220/44000 | Loss: 2.9948 | LR: 3.07e-04 | 1770.9ms/step | 11.3 samples/s +Step 19230/44000 | Loss: 3.0039 | LR: 3.07e-04 | 1768.9ms/step | 11.3 samples/s +Step 19240/44000 | Loss: 2.9916 | LR: 3.06e-04 | 1770.4ms/step | 11.3 samples/s +Step 19250/44000 | Loss: 2.9411 | LR: 3.06e-04 | 1770.0ms/step | 11.3 samples/s +Step 19260/44000 | Loss: 2.9537 | LR: 3.06e-04 | 1770.7ms/step | 11.3 samples/s +Step 19270/44000 | Loss: 2.9491 | LR: 3.06e-04 | 1770.5ms/step | 11.3 samples/s +Step 19280/44000 | Loss: 2.9620 | LR: 3.06e-04 | 1771.3ms/step | 11.3 samples/s +Step 19290/44000 | Loss: 2.9807 | LR: 3.06e-04 | 1770.6ms/step | 11.3 samples/s +Step 19300/44000 | Loss: 2.9847 | LR: 3.05e-04 | 1770.7ms/step | 11.3 samples/s +Step 19310/44000 | Loss: 2.9413 | LR: 3.05e-04 | 1772.0ms/step | 11.3 samples/s +Step 19320/44000 | Loss: 3.0060 | LR: 3.05e-04 | 1772.5ms/step | 11.3 samples/s +Step 19330/44000 | Loss: 2.9936 | LR: 3.05e-04 | 1772.4ms/step | 11.3 samples/s +Step 19340/44000 | Loss: 2.9193 | LR: 3.05e-04 | 1771.9ms/step | 11.3 samples/s +Step 19350/44000 | Loss: 3.0031 | LR: 3.04e-04 | 1771.0ms/step | 11.3 samples/s +Step 19360/44000 | Loss: 2.9996 | LR: 3.04e-04 | 1770.7ms/step | 11.3 samples/s +Step 19370/44000 | Loss: 2.9783 | LR: 3.04e-04 | 1769.4ms/step | 11.3 samples/s +Step 19380/44000 | Loss: 2.9958 | LR: 3.04e-04 | 1769.1ms/step | 11.3 samples/s +Step 19390/44000 | Loss: 2.9684 | LR: 3.04e-04 | 1770.7ms/step | 11.3 samples/s +Step 19400/44000 | Loss: 2.9573 | LR: 3.04e-04 | 1771.1ms/step | 11.3 samples/s +Step 19410/44000 | Loss: 2.9659 | LR: 3.03e-04 | 1771.0ms/step | 11.3 samples/s +Step 19420/44000 | Loss: 2.9652 | LR: 3.03e-04 | 1771.9ms/step | 11.3 samples/s +Step 19430/44000 | Loss: 2.9947 | LR: 3.03e-04 | 1771.3ms/step | 11.3 samples/s +Step 19440/44000 | Loss: 2.9503 | LR: 3.03e-04 | 1772.2ms/step | 11.3 samples/s +Step 19450/44000 | Loss: 2.9506 | LR: 3.03e-04 | 1770.6ms/step | 11.3 samples/s +Step 19460/44000 | Loss: 2.9538 | LR: 3.03e-04 | 1770.8ms/step | 11.3 samples/s +Step 19470/44000 | Loss: 2.9783 | LR: 3.02e-04 | 1770.5ms/step | 11.3 samples/s +Step 19480/44000 | Loss: 3.0068 | LR: 3.02e-04 | 1770.4ms/step | 11.3 samples/s +Step 19490/44000 | Loss: 2.9531 | LR: 3.02e-04 | 1769.5ms/step | 11.3 samples/s +Step 19500/44000 | Loss: 2.9721 | LR: 3.02e-04 | 1770.5ms/step | 11.3 samples/s +Step 19510/44000 | Loss: 2.9562 | LR: 3.02e-04 | 1773.0ms/step | 11.3 samples/s +Step 19520/44000 | Loss: 2.9827 | LR: 3.01e-04 | 1771.8ms/step | 11.3 samples/s +Step 19530/44000 | Loss: 2.9471 | LR: 3.01e-04 | 1772.0ms/step | 11.3 samples/s +Step 19540/44000 | Loss: 2.9769 | LR: 3.01e-04 | 1770.6ms/step | 11.3 samples/s +Step 19550/44000 | Loss: 2.9495 | LR: 3.01e-04 | 1771.2ms/step | 11.3 samples/s +Step 19560/44000 | Loss: 2.9559 | LR: 3.01e-04 | 1772.4ms/step | 11.3 samples/s +Step 19570/44000 | Loss: 2.9899 | LR: 3.01e-04 | 1771.6ms/step | 11.3 samples/s +Step 19580/44000 | Loss: 2.9869 | LR: 3.00e-04 | 1771.4ms/step | 11.3 samples/s +Step 19590/44000 | Loss: 2.9162 | LR: 3.00e-04 | 1771.4ms/step | 11.3 samples/s +Step 19600/44000 | Loss: 2.9708 | LR: 3.00e-04 | 1772.5ms/step | 11.3 samples/s +Step 19610/44000 | Loss: 2.9644 | LR: 3.00e-04 | 1771.6ms/step | 11.3 samples/s +Step 19620/44000 | Loss: 2.9560 | LR: 3.00e-04 | 1771.7ms/step | 11.3 samples/s +Step 19630/44000 | Loss: 2.9680 | LR: 3.00e-04 | 1770.0ms/step | 11.3 samples/s +Step 19640/44000 | Loss: 2.9741 | LR: 2.99e-04 | 1770.5ms/step | 11.3 samples/s +Step 19650/44000 | Loss: 2.9759 | LR: 2.99e-04 | 1769.5ms/step | 11.3 samples/s +Step 19660/44000 | Loss: 2.9623 | LR: 2.99e-04 | 1770.7ms/step | 11.3 samples/s +Step 19670/44000 | Loss: 2.9420 | LR: 2.99e-04 | 1770.9ms/step | 11.3 samples/s +Step 19680/44000 | Loss: 2.9692 | LR: 2.99e-04 | 1770.8ms/step | 11.3 samples/s +Step 19690/44000 | Loss: 2.9437 | LR: 2.98e-04 | 1769.9ms/step | 11.3 samples/s +Step 19700/44000 | Loss: 2.9444 | LR: 2.98e-04 | 1770.1ms/step | 11.3 samples/s +Step 19710/44000 | Loss: 2.9681 | LR: 2.98e-04 | 1771.0ms/step | 11.3 samples/s +Step 19720/44000 | Loss: 2.9803 | LR: 2.98e-04 | 1771.2ms/step | 11.3 samples/s +Step 19730/44000 | Loss: 2.9701 | LR: 2.98e-04 | 1770.4ms/step | 11.3 samples/s +Step 19740/44000 | Loss: 2.9696 | LR: 2.98e-04 | 1770.8ms/step | 11.3 samples/s +Step 19750/44000 | Loss: 2.9536 | LR: 2.97e-04 | 1771.4ms/step | 11.3 samples/s +Step 19760/44000 | Loss: 2.9481 | LR: 2.97e-04 | 1771.4ms/step | 11.3 samples/s +Step 19770/44000 | Loss: 2.9471 | LR: 2.97e-04 | 1770.4ms/step | 11.3 samples/s +Step 19780/44000 | Loss: 2.9547 | LR: 2.97e-04 | 1769.1ms/step | 11.3 samples/s +Step 19790/44000 | Loss: 2.9828 | LR: 2.97e-04 | 1769.1ms/step | 11.3 samples/s +Step 19800/44000 | Loss: 2.9386 | LR: 2.96e-04 | 1769.8ms/step | 11.3 samples/s +Step 19810/44000 | Loss: 2.9634 | LR: 2.96e-04 | 1771.0ms/step | 11.3 samples/s +Step 19820/44000 | Loss: 2.9613 | LR: 2.96e-04 | 1769.8ms/step | 11.3 samples/s +Step 19830/44000 | Loss: 3.0163 | LR: 2.96e-04 | 1770.3ms/step | 11.3 samples/s +Step 19840/44000 | Loss: 2.9451 | LR: 2.96e-04 | 1770.3ms/step | 11.3 samples/s +Step 19850/44000 | Loss: 2.9251 | LR: 2.96e-04 | 1772.6ms/step | 11.3 samples/s +Step 19860/44000 | Loss: 2.9628 | LR: 2.95e-04 | 1771.3ms/step | 11.3 samples/s +Step 19870/44000 | Loss: 2.9556 | LR: 2.95e-04 | 1771.5ms/step | 11.3 samples/s +Step 19880/44000 | Loss: 3.0099 | LR: 2.95e-04 | 1770.1ms/step | 11.3 samples/s +Step 19890/44000 | Loss: 2.9379 | LR: 2.95e-04 | 1776.5ms/step | 11.3 samples/s +Step 19900/44000 | Loss: 3.0089 | LR: 2.95e-04 | 1773.8ms/step | 11.3 samples/s +Step 19910/44000 | Loss: 2.9912 | LR: 2.95e-04 | 1770.9ms/step | 11.3 samples/s +Step 19920/44000 | Loss: 2.9984 | LR: 2.94e-04 | 1772.6ms/step | 11.3 samples/s +Step 19930/44000 | Loss: 2.9582 | LR: 2.94e-04 | 1771.5ms/step | 11.3 samples/s +Step 19940/44000 | Loss: 2.9479 | LR: 2.94e-04 | 1772.7ms/step | 11.3 samples/s +Step 19950/44000 | Loss: 2.9565 | LR: 2.94e-04 | 1772.1ms/step | 11.3 samples/s +Step 19960/44000 | Loss: 2.9890 | LR: 2.94e-04 | 1773.3ms/step | 11.3 samples/s +Step 19970/44000 | Loss: 2.9651 | LR: 2.93e-04 | 1772.6ms/step | 11.3 samples/s +Step 19980/44000 | Loss: 2.9537 | LR: 2.93e-04 | 1770.7ms/step | 11.3 samples/s +Step 19990/44000 | Loss: 2.9003 | LR: 2.93e-04 | 1770.5ms/step | 11.3 samples/s +Step 20000/44000 | Loss: 2.9248 | LR: 2.93e-04 | 1772.2ms/step | 11.3 samples/s +Validation at step 20000: Loss = 2.9722, Perplexity = 19.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-20000 +Step 20010/44000 | Loss: 2.9816 | LR: 2.93e-04 | 3625.9ms/step | 5.5 samples/s +Step 20020/44000 | Loss: 2.9778 | LR: 2.93e-04 | 1770.1ms/step | 11.3 samples/s +Step 20030/44000 | Loss: 2.9212 | LR: 2.92e-04 | 1770.6ms/step | 11.3 samples/s +Step 20040/44000 | Loss: 2.9739 | LR: 2.92e-04 | 1770.1ms/step | 11.3 samples/s +Step 20050/44000 | Loss: 2.9567 | LR: 2.92e-04 | 1770.6ms/step | 11.3 samples/s +Step 20060/44000 | Loss: 2.9815 | LR: 2.92e-04 | 1769.9ms/step | 11.3 samples/s +Step 20070/44000 | Loss: 2.9755 | LR: 2.92e-04 | 1769.5ms/step | 11.3 samples/s +Step 20080/44000 | Loss: 2.9633 | LR: 2.91e-04 | 1770.1ms/step | 11.3 samples/s +Step 20090/44000 | Loss: 2.9590 | LR: 2.91e-04 | 1771.7ms/step | 11.3 samples/s +Step 20100/44000 | Loss: 2.9741 | LR: 2.91e-04 | 1769.4ms/step | 11.3 samples/s +Step 20110/44000 | Loss: 2.9806 | LR: 2.91e-04 | 1770.4ms/step | 11.3 samples/s +Step 20120/44000 | Loss: 2.9475 | LR: 2.91e-04 | 1770.5ms/step | 11.3 samples/s +Step 20130/44000 | Loss: 2.9405 | LR: 2.91e-04 | 1771.7ms/step | 11.3 samples/s +Step 20140/44000 | Loss: 2.9664 | LR: 2.90e-04 | 1772.1ms/step | 11.3 samples/s +Step 20150/44000 | Loss: 2.9486 | LR: 2.90e-04 | 1771.5ms/step | 11.3 samples/s +Step 20160/44000 | Loss: 3.0032 | LR: 2.90e-04 | 1772.0ms/step | 11.3 samples/s +Step 20170/44000 | Loss: 2.9958 | LR: 2.90e-04 | 1771.3ms/step | 11.3 samples/s +Step 20180/44000 | Loss: 2.9919 | LR: 2.90e-04 | 1768.5ms/step | 11.3 samples/s +Step 20190/44000 | Loss: 2.9245 | LR: 2.90e-04 | 1768.7ms/step | 11.3 samples/s +Step 20200/44000 | Loss: 2.9556 | LR: 2.89e-04 | 1769.2ms/step | 11.3 samples/s +Step 20210/44000 | Loss: 2.9703 | LR: 2.89e-04 | 1769.3ms/step | 11.3 samples/s +Step 20220/44000 | Loss: 2.9727 | LR: 2.89e-04 | 1772.1ms/step | 11.3 samples/s +Step 20230/44000 | Loss: 2.9500 | LR: 2.89e-04 | 1771.0ms/step | 11.3 samples/s +Step 20240/44000 | Loss: 2.9612 | LR: 2.89e-04 | 1771.3ms/step | 11.3 samples/s +Step 20250/44000 | Loss: 2.9833 | LR: 2.88e-04 | 1771.6ms/step | 11.3 samples/s +Step 20260/44000 | Loss: 3.0002 | LR: 2.88e-04 | 1770.6ms/step | 11.3 samples/s +Step 20270/44000 | Loss: 2.9567 | LR: 2.88e-04 | 1770.1ms/step | 11.3 samples/s +Step 20280/44000 | Loss: 2.9705 | LR: 2.88e-04 | 1770.6ms/step | 11.3 samples/s +Step 20290/44000 | Loss: 2.9384 | LR: 2.88e-04 | 1770.4ms/step | 11.3 samples/s +Step 20300/44000 | Loss: 2.9583 | LR: 2.88e-04 | 1770.1ms/step | 11.3 samples/s +Step 20310/44000 | Loss: 2.9347 | LR: 2.87e-04 | 1770.4ms/step | 11.3 samples/s +Step 20320/44000 | Loss: 2.9550 | LR: 2.87e-04 | 1769.8ms/step | 11.3 samples/s +Step 20330/44000 | Loss: 2.9434 | LR: 2.87e-04 | 1770.9ms/step | 11.3 samples/s +Step 20340/44000 | Loss: 2.9719 | LR: 2.87e-04 | 1771.2ms/step | 11.3 samples/s +Step 20350/44000 | Loss: 2.9250 | LR: 2.87e-04 | 1772.5ms/step | 11.3 samples/s +Step 20360/44000 | Loss: 2.9488 | LR: 2.86e-04 | 1771.2ms/step | 11.3 samples/s +Step 20370/44000 | Loss: 2.9819 | LR: 2.86e-04 | 1771.0ms/step | 11.3 samples/s +Step 20380/44000 | Loss: 2.9286 | LR: 2.86e-04 | 1769.5ms/step | 11.3 samples/s +Step 20390/44000 | Loss: 3.0053 | LR: 2.86e-04 | 1771.1ms/step | 11.3 samples/s +Step 20400/44000 | Loss: 2.9318 | LR: 2.86e-04 | 1769.1ms/step | 11.3 samples/s +Step 20410/44000 | Loss: 2.9306 | LR: 2.86e-04 | 1768.1ms/step | 11.3 samples/s +Step 20420/44000 | Loss: 2.9649 | LR: 2.85e-04 | 1767.8ms/step | 11.3 samples/s +Step 20430/44000 | Loss: 2.9844 | LR: 2.85e-04 | 1769.2ms/step | 11.3 samples/s +Step 20440/44000 | Loss: 2.9624 | LR: 2.85e-04 | 1769.8ms/step | 11.3 samples/s +Step 20450/44000 | Loss: 3.0124 | LR: 2.85e-04 | 1769.3ms/step | 11.3 samples/s +Step 20460/44000 | Loss: 3.0169 | LR: 2.85e-04 | 1769.4ms/step | 11.3 samples/s +Step 20470/44000 | Loss: 2.9604 | LR: 2.84e-04 | 1770.5ms/step | 11.3 samples/s +Step 20480/44000 | Loss: 2.9868 | LR: 2.84e-04 | 1771.8ms/step | 11.3 samples/s +Step 20490/44000 | Loss: 2.9644 | LR: 2.84e-04 | 1770.6ms/step | 11.3 samples/s +Step 20500/44000 | Loss: 2.9266 | LR: 2.84e-04 | 1771.9ms/step | 11.3 samples/s +Step 20510/44000 | Loss: 2.9403 | LR: 2.84e-04 | 1771.8ms/step | 11.3 samples/s +Step 20520/44000 | Loss: 2.9423 | LR: 2.84e-04 | 1771.4ms/step | 11.3 samples/s +Step 20530/44000 | Loss: 2.9933 | LR: 2.83e-04 | 1770.2ms/step | 11.3 samples/s +Step 20540/44000 | Loss: 2.9370 | LR: 2.83e-04 | 1770.2ms/step | 11.3 samples/s +Step 20550/44000 | Loss: 2.9665 | LR: 2.83e-04 | 1770.3ms/step | 11.3 samples/s +Step 20560/44000 | Loss: 2.9544 | LR: 2.83e-04 | 1770.3ms/step | 11.3 samples/s +Step 20570/44000 | Loss: 2.9111 | LR: 2.83e-04 | 1772.4ms/step | 11.3 samples/s +Step 20580/44000 | Loss: 2.9822 | LR: 2.83e-04 | 1771.8ms/step | 11.3 samples/s +Step 20590/44000 | Loss: 2.9426 | LR: 2.82e-04 | 1770.4ms/step | 11.3 samples/s +Step 20600/44000 | Loss: 2.9467 | LR: 2.82e-04 | 1771.4ms/step | 11.3 samples/s +Step 20610/44000 | Loss: 2.9570 | LR: 2.82e-04 | 1769.2ms/step | 11.3 samples/s +Step 20620/44000 | Loss: 2.9540 | LR: 2.82e-04 | 1770.1ms/step | 11.3 samples/s +Step 20630/44000 | Loss: 2.9329 | LR: 2.82e-04 | 1769.8ms/step | 11.3 samples/s +Step 20640/44000 | Loss: 2.9869 | LR: 2.81e-04 | 1769.6ms/step | 11.3 samples/s +Step 20650/44000 | Loss: 2.9165 | LR: 2.81e-04 | 1770.8ms/step | 11.3 samples/s +Step 20660/44000 | Loss: 2.9320 | LR: 2.81e-04 | 1770.4ms/step | 11.3 samples/s +Step 20670/44000 | Loss: 2.9362 | LR: 2.81e-04 | 1771.0ms/step | 11.3 samples/s +Step 20680/44000 | Loss: 2.9663 | LR: 2.81e-04 | 1770.9ms/step | 11.3 samples/s +Step 20690/44000 | Loss: 2.8882 | LR: 2.81e-04 | 1770.5ms/step | 11.3 samples/s +Step 20700/44000 | Loss: 2.9963 | LR: 2.80e-04 | 1769.9ms/step | 11.3 samples/s +Step 20710/44000 | Loss: 2.9678 | LR: 2.80e-04 | 1771.0ms/step | 11.3 samples/s +Step 20720/44000 | Loss: 2.9527 | LR: 2.80e-04 | 1771.0ms/step | 11.3 samples/s +Step 20730/44000 | Loss: 2.9973 | LR: 2.80e-04 | 1771.2ms/step | 11.3 samples/s +Step 20740/44000 | Loss: 3.0062 | LR: 2.80e-04 | 1770.5ms/step | 11.3 samples/s +Step 20750/44000 | Loss: 2.9608 | LR: 2.79e-04 | 1772.0ms/step | 11.3 samples/s +Step 20760/44000 | Loss: 2.9559 | LR: 2.79e-04 | 1771.7ms/step | 11.3 samples/s +Step 20770/44000 | Loss: 2.9498 | LR: 2.79e-04 | 1772.1ms/step | 11.3 samples/s +Step 20780/44000 | Loss: 2.9675 | LR: 2.79e-04 | 1771.2ms/step | 11.3 samples/s +Step 20790/44000 | Loss: 2.9515 | LR: 2.79e-04 | 1769.0ms/step | 11.3 samples/s +Step 20800/44000 | Loss: 2.9370 | LR: 2.79e-04 | 1769.6ms/step | 11.3 samples/s +Step 20810/44000 | Loss: 2.9494 | LR: 2.78e-04 | 1770.3ms/step | 11.3 samples/s +Step 20820/44000 | Loss: 2.9558 | LR: 2.78e-04 | 1770.0ms/step | 11.3 samples/s +Step 20830/44000 | Loss: 2.9733 | LR: 2.78e-04 | 1770.6ms/step | 11.3 samples/s +Step 20840/44000 | Loss: 2.9305 | LR: 2.78e-04 | 1770.8ms/step | 11.3 samples/s +Step 20850/44000 | Loss: 2.9401 | LR: 2.78e-04 | 1773.0ms/step | 11.3 samples/s +Step 20860/44000 | Loss: 2.9898 | LR: 2.77e-04 | 1770.0ms/step | 11.3 samples/s +Step 20870/44000 | Loss: 2.9470 | LR: 2.77e-04 | 1771.4ms/step | 11.3 samples/s +Step 20880/44000 | Loss: 2.9657 | LR: 2.77e-04 | 1772.3ms/step | 11.3 samples/s +Step 20890/44000 | Loss: 2.9494 | LR: 2.77e-04 | 1770.9ms/step | 11.3 samples/s +Step 20900/44000 | Loss: 2.9441 | LR: 2.77e-04 | 1771.1ms/step | 11.3 samples/s +Step 20910/44000 | Loss: 2.9343 | LR: 2.77e-04 | 1771.2ms/step | 11.3 samples/s +Step 20920/44000 | Loss: 2.9755 | LR: 2.76e-04 | 1771.0ms/step | 11.3 samples/s +Step 20930/44000 | Loss: 2.9439 | LR: 2.76e-04 | 1771.4ms/step | 11.3 samples/s +Step 20940/44000 | Loss: 2.9537 | LR: 2.76e-04 | 1770.7ms/step | 11.3 samples/s +Step 20950/44000 | Loss: 2.9685 | LR: 2.76e-04 | 1772.1ms/step | 11.3 samples/s +Step 20960/44000 | Loss: 2.9687 | LR: 2.76e-04 | 1772.2ms/step | 11.3 samples/s +Step 20970/44000 | Loss: 2.9550 | LR: 2.75e-04 | 1772.4ms/step | 11.3 samples/s +Step 20980/44000 | Loss: 2.9290 | LR: 2.75e-04 | 1773.5ms/step | 11.3 samples/s +Step 20990/44000 | Loss: 2.9885 | LR: 2.75e-04 | 1770.9ms/step | 11.3 samples/s +Step 21000/44000 | Loss: 2.9418 | LR: 2.75e-04 | 1769.5ms/step | 11.3 samples/s +Validation at step 21000: Loss = 2.9617, Perplexity = 19.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-21000 +Step 21010/44000 | Loss: 2.9727 | LR: 2.75e-04 | 3547.8ms/step | 5.6 samples/s +Step 21020/44000 | Loss: 2.9407 | LR: 2.75e-04 | 1769.0ms/step | 11.3 samples/s +Step 21030/44000 | Loss: 2.9414 | LR: 2.74e-04 | 1770.3ms/step | 11.3 samples/s +Step 21040/44000 | Loss: 2.9263 | LR: 2.74e-04 | 1771.0ms/step | 11.3 samples/s +Step 21050/44000 | Loss: 2.9586 | LR: 2.74e-04 | 1770.9ms/step | 11.3 samples/s +Step 21060/44000 | Loss: 2.9516 | LR: 2.74e-04 | 1769.1ms/step | 11.3 samples/s +Step 21070/44000 | Loss: 2.8871 | LR: 2.74e-04 | 1769.7ms/step | 11.3 samples/s +Step 21080/44000 | Loss: 2.9939 | LR: 2.73e-04 | 1770.1ms/step | 11.3 samples/s +Step 21090/44000 | Loss: 2.9073 | LR: 2.73e-04 | 1770.5ms/step | 11.3 samples/s +Step 21100/44000 | Loss: 2.9544 | LR: 2.73e-04 | 1771.9ms/step | 11.3 samples/s +Step 21110/44000 | Loss: 2.9470 | LR: 2.73e-04 | 1771.6ms/step | 11.3 samples/s +Step 21120/44000 | Loss: 2.9368 | LR: 2.73e-04 | 1769.9ms/step | 11.3 samples/s +Step 21130/44000 | Loss: 2.9874 | LR: 2.73e-04 | 1770.6ms/step | 11.3 samples/s +Step 21140/44000 | Loss: 2.8958 | LR: 2.72e-04 | 1771.4ms/step | 11.3 samples/s +Step 21150/44000 | Loss: 2.9277 | LR: 2.72e-04 | 1770.2ms/step | 11.3 samples/s +Step 21160/44000 | Loss: 2.9177 | LR: 2.72e-04 | 1772.2ms/step | 11.3 samples/s +Step 21170/44000 | Loss: 2.9782 | LR: 2.72e-04 | 1772.7ms/step | 11.3 samples/s +Step 21180/44000 | Loss: 2.9245 | LR: 2.72e-04 | 1773.4ms/step | 11.3 samples/s +Step 21190/44000 | Loss: 2.9490 | LR: 2.71e-04 | 1769.9ms/step | 11.3 samples/s +Step 21200/44000 | Loss: 2.9321 | LR: 2.71e-04 | 1770.4ms/step | 11.3 samples/s +Step 21210/44000 | Loss: 2.9134 | LR: 2.71e-04 | 1770.8ms/step | 11.3 samples/s +Step 21220/44000 | Loss: 2.9325 | LR: 2.71e-04 | 1771.3ms/step | 11.3 samples/s +Step 21230/44000 | Loss: 2.9484 | LR: 2.71e-04 | 1772.9ms/step | 11.3 samples/s +Step 21240/44000 | Loss: 2.9883 | LR: 2.71e-04 | 1771.5ms/step | 11.3 samples/s +Step 21250/44000 | Loss: 2.9579 | LR: 2.70e-04 | 1771.2ms/step | 11.3 samples/s +Step 21260/44000 | Loss: 2.9837 | LR: 2.70e-04 | 1770.7ms/step | 11.3 samples/s +Step 21270/44000 | Loss: 2.9694 | LR: 2.70e-04 | 1770.3ms/step | 11.3 samples/s +Step 21280/44000 | Loss: 2.9666 | LR: 2.70e-04 | 1771.0ms/step | 11.3 samples/s +Step 21290/44000 | Loss: 2.9292 | LR: 2.70e-04 | 1771.3ms/step | 11.3 samples/s +Step 21300/44000 | Loss: 2.9695 | LR: 2.70e-04 | 1771.5ms/step | 11.3 samples/s +Step 21310/44000 | Loss: 2.9305 | LR: 2.69e-04 | 1771.9ms/step | 11.3 samples/s +Step 21320/44000 | Loss: 2.9512 | LR: 2.69e-04 | 1772.1ms/step | 11.3 samples/s +Step 21330/44000 | Loss: 2.9623 | LR: 2.69e-04 | 1772.0ms/step | 11.3 samples/s +Step 21340/44000 | Loss: 2.9600 | LR: 2.69e-04 | 1772.6ms/step | 11.3 samples/s +Step 21350/44000 | Loss: 2.9760 | LR: 2.69e-04 | 1773.5ms/step | 11.3 samples/s +Step 21360/44000 | Loss: 2.9180 | LR: 2.68e-04 | 1771.9ms/step | 11.3 samples/s +Step 21370/44000 | Loss: 2.9145 | LR: 2.68e-04 | 1772.0ms/step | 11.3 samples/s +Step 21380/44000 | Loss: 2.9765 | LR: 2.68e-04 | 1772.4ms/step | 11.3 samples/s +Step 21390/44000 | Loss: 2.9132 | LR: 2.68e-04 | 1771.6ms/step | 11.3 samples/s +Step 21400/44000 | Loss: 2.9095 | LR: 2.68e-04 | 1769.1ms/step | 11.3 samples/s +Step 21410/44000 | Loss: 2.9186 | LR: 2.68e-04 | 1768.8ms/step | 11.3 samples/s +Step 21420/44000 | Loss: 2.9608 | LR: 2.67e-04 | 1769.8ms/step | 11.3 samples/s +Step 21430/44000 | Loss: 2.9342 | LR: 2.67e-04 | 1770.2ms/step | 11.3 samples/s +Step 21440/44000 | Loss: 2.9320 | LR: 2.67e-04 | 1770.1ms/step | 11.3 samples/s +Step 21450/44000 | Loss: 2.9778 | LR: 2.67e-04 | 1769.6ms/step | 11.3 samples/s +Step 21460/44000 | Loss: 2.9842 | LR: 2.67e-04 | 1769.1ms/step | 11.3 samples/s +Step 21470/44000 | Loss: 2.9218 | LR: 2.66e-04 | 1769.9ms/step | 11.3 samples/s +Step 21480/44000 | Loss: 2.9486 | LR: 2.66e-04 | 1769.7ms/step | 11.3 samples/s +Step 21490/44000 | Loss: 2.9668 | LR: 2.66e-04 | 1769.3ms/step | 11.3 samples/s +Step 21500/44000 | Loss: 2.9292 | LR: 2.66e-04 | 1770.5ms/step | 11.3 samples/s +Step 21510/44000 | Loss: 2.9646 | LR: 2.66e-04 | 1770.9ms/step | 11.3 samples/s +Step 21520/44000 | Loss: 2.9676 | LR: 2.66e-04 | 1770.2ms/step | 11.3 samples/s +Step 21530/44000 | Loss: 2.9028 | LR: 2.65e-04 | 1771.7ms/step | 11.3 samples/s +Step 21540/44000 | Loss: 2.9628 | LR: 2.65e-04 | 1771.6ms/step | 11.3 samples/s +Step 21550/44000 | Loss: 2.9413 | LR: 2.65e-04 | 1771.8ms/step | 11.3 samples/s +Step 21560/44000 | Loss: 2.9478 | LR: 2.65e-04 | 1771.2ms/step | 11.3 samples/s +Step 21570/44000 | Loss: 2.9693 | LR: 2.65e-04 | 1771.1ms/step | 11.3 samples/s +Step 21580/44000 | Loss: 2.9467 | LR: 2.64e-04 | 1772.8ms/step | 11.3 samples/s +Step 21590/44000 | Loss: 2.9269 | LR: 2.64e-04 | 1773.4ms/step | 11.3 samples/s +Step 21600/44000 | Loss: 2.9671 | LR: 2.64e-04 | 1771.1ms/step | 11.3 samples/s +Step 21610/44000 | Loss: 2.9265 | LR: 2.64e-04 | 1771.8ms/step | 11.3 samples/s +Step 21620/44000 | Loss: 2.9045 | LR: 2.64e-04 | 1770.5ms/step | 11.3 samples/s +Step 21630/44000 | Loss: 2.9044 | LR: 2.64e-04 | 1771.2ms/step | 11.3 samples/s +Step 21640/44000 | Loss: 2.9569 | LR: 2.63e-04 | 1771.3ms/step | 11.3 samples/s +Step 21650/44000 | Loss: 2.9823 | LR: 2.63e-04 | 1773.1ms/step | 11.3 samples/s +Step 21660/44000 | Loss: 2.9459 | LR: 2.63e-04 | 1773.0ms/step | 11.3 samples/s +Step 21670/44000 | Loss: 2.9741 | LR: 2.63e-04 | 1772.1ms/step | 11.3 samples/s +Step 21680/44000 | Loss: 2.9790 | LR: 2.63e-04 | 1770.7ms/step | 11.3 samples/s +Step 21690/44000 | Loss: 2.9701 | LR: 2.62e-04 | 1771.1ms/step | 11.3 samples/s +Step 21700/44000 | Loss: 2.9565 | LR: 2.62e-04 | 1771.6ms/step | 11.3 samples/s +Step 21710/44000 | Loss: 2.9762 | LR: 2.62e-04 | 1770.4ms/step | 11.3 samples/s +Step 21720/44000 | Loss: 2.9282 | LR: 2.62e-04 | 1770.9ms/step | 11.3 samples/s +Step 21730/44000 | Loss: 2.9703 | LR: 2.62e-04 | 1772.2ms/step | 11.3 samples/s +Step 21740/44000 | Loss: 2.9507 | LR: 2.62e-04 | 1771.4ms/step | 11.3 samples/s +Step 21750/44000 | Loss: 2.9642 | LR: 2.61e-04 | 1770.8ms/step | 11.3 samples/s +Step 21760/44000 | Loss: 2.9588 | LR: 2.61e-04 | 1772.7ms/step | 11.3 samples/s +Step 21770/44000 | Loss: 2.8906 | LR: 2.61e-04 | 1773.6ms/step | 11.3 samples/s +Step 21780/44000 | Loss: 2.9805 | LR: 2.61e-04 | 1772.7ms/step | 11.3 samples/s +Step 21790/44000 | Loss: 2.8840 | LR: 2.61e-04 | 1773.2ms/step | 11.3 samples/s +Step 21800/44000 | Loss: 2.9755 | LR: 2.60e-04 | 1772.4ms/step | 11.3 samples/s +Step 21810/44000 | Loss: 2.9098 | LR: 2.60e-04 | 1770.4ms/step | 11.3 samples/s +Step 21820/44000 | Loss: 2.9524 | LR: 2.60e-04 | 1771.0ms/step | 11.3 samples/s +Step 21830/44000 | Loss: 2.9297 | LR: 2.60e-04 | 1770.7ms/step | 11.3 samples/s +Step 21840/44000 | Loss: 2.9537 | LR: 2.60e-04 | 1769.3ms/step | 11.3 samples/s +Step 21850/44000 | Loss: 2.9244 | LR: 2.60e-04 | 1768.5ms/step | 11.3 samples/s +Step 21860/44000 | Loss: 2.9163 | LR: 2.59e-04 | 1771.6ms/step | 11.3 samples/s +Step 21870/44000 | Loss: 2.9120 | LR: 2.59e-04 | 1771.2ms/step | 11.3 samples/s +Step 21880/44000 | Loss: 2.9423 | LR: 2.59e-04 | 1771.1ms/step | 11.3 samples/s +Step 21890/44000 | Loss: 2.9102 | LR: 2.59e-04 | 1773.4ms/step | 11.3 samples/s +Step 21900/44000 | Loss: 2.9573 | LR: 2.59e-04 | 1774.7ms/step | 11.3 samples/s +Step 21910/44000 | Loss: 2.9441 | LR: 2.58e-04 | 1772.7ms/step | 11.3 samples/s +Step 21920/44000 | Loss: 2.9309 | LR: 2.58e-04 | 1770.5ms/step | 11.3 samples/s +Step 21930/44000 | Loss: 2.9375 | LR: 2.58e-04 | 1770.8ms/step | 11.3 samples/s +Step 21940/44000 | Loss: 2.9143 | LR: 2.58e-04 | 1771.7ms/step | 11.3 samples/s +Step 21950/44000 | Loss: 2.9301 | LR: 2.58e-04 | 1771.5ms/step | 11.3 samples/s +Step 21960/44000 | Loss: 2.9244 | LR: 2.58e-04 | 1772.3ms/step | 11.3 samples/s +Step 21970/44000 | Loss: 2.9328 | LR: 2.57e-04 | 1772.3ms/step | 11.3 samples/s +Step 21980/44000 | Loss: 2.9116 | LR: 2.57e-04 | 1772.6ms/step | 11.3 samples/s +Step 21990/44000 | Loss: 2.9667 | LR: 2.57e-04 | 1773.9ms/step | 11.3 samples/s +Step 22000/44000 | Loss: 2.9800 | LR: 2.57e-04 | 1772.9ms/step | 11.3 samples/s +Validation at step 22000: Loss = 2.9528, Perplexity = 19.16 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-22000 +Step 22010/44000 | Loss: 2.9338 | LR: 2.57e-04 | 3560.1ms/step | 5.6 samples/s +Step 22020/44000 | Loss: 2.8921 | LR: 2.56e-04 | 1771.7ms/step | 11.3 samples/s +Step 22030/44000 | Loss: 2.9259 | LR: 2.56e-04 | 1772.4ms/step | 11.3 samples/s +Step 22040/44000 | Loss: 2.9486 | LR: 2.56e-04 | 1772.0ms/step | 11.3 samples/s +Step 22050/44000 | Loss: 2.9976 | LR: 2.56e-04 | 1771.1ms/step | 11.3 samples/s +Step 22060/44000 | Loss: 2.9513 | LR: 2.56e-04 | 1771.6ms/step | 11.3 samples/s +Step 22070/44000 | Loss: 2.9246 | LR: 2.56e-04 | 1772.0ms/step | 11.3 samples/s +Step 22080/44000 | Loss: 2.9267 | LR: 2.55e-04 | 1773.3ms/step | 11.3 samples/s +Step 22090/44000 | Loss: 2.9569 | LR: 2.55e-04 | 1772.1ms/step | 11.3 samples/s +Step 22100/44000 | Loss: 2.9295 | LR: 2.55e-04 | 1774.0ms/step | 11.3 samples/s +Step 22110/44000 | Loss: 2.9712 | LR: 2.55e-04 | 1773.3ms/step | 11.3 samples/s +Step 22120/44000 | Loss: 2.9615 | LR: 2.55e-04 | 1772.8ms/step | 11.3 samples/s +Step 22130/44000 | Loss: 2.9275 | LR: 2.54e-04 | 1771.5ms/step | 11.3 samples/s +Step 22140/44000 | Loss: 2.9286 | LR: 2.54e-04 | 1771.4ms/step | 11.3 samples/s +Step 22150/44000 | Loss: 2.9220 | LR: 2.54e-04 | 1771.9ms/step | 11.3 samples/s +Step 22160/44000 | Loss: 2.9725 | LR: 2.54e-04 | 1772.6ms/step | 11.3 samples/s +Step 22170/44000 | Loss: 3.0177 | LR: 2.54e-04 | 1773.2ms/step | 11.3 samples/s +Step 22180/44000 | Loss: 2.9142 | LR: 2.54e-04 | 1773.9ms/step | 11.3 samples/s +Step 22190/44000 | Loss: 2.9659 | LR: 2.53e-04 | 1773.6ms/step | 11.3 samples/s +Step 22200/44000 | Loss: 2.9481 | LR: 2.53e-04 | 1775.0ms/step | 11.3 samples/s +Step 22210/44000 | Loss: 2.9754 | LR: 2.53e-04 | 1771.7ms/step | 11.3 samples/s +Step 22220/44000 | Loss: 2.8976 | LR: 2.53e-04 | 1772.3ms/step | 11.3 samples/s +Step 22230/44000 | Loss: 2.9833 | LR: 2.53e-04 | 1773.7ms/step | 11.3 samples/s +Step 22240/44000 | Loss: 2.9172 | LR: 2.52e-04 | 1772.4ms/step | 11.3 samples/s +Step 22250/44000 | Loss: 2.9282 | LR: 2.52e-04 | 1772.6ms/step | 11.3 samples/s +Step 22260/44000 | Loss: 2.9525 | LR: 2.52e-04 | 1772.5ms/step | 11.3 samples/s +Step 22270/44000 | Loss: 2.9295 | LR: 2.52e-04 | 1772.0ms/step | 11.3 samples/s +Step 22280/44000 | Loss: 2.9373 | LR: 2.52e-04 | 1772.4ms/step | 11.3 samples/s +Step 22290/44000 | Loss: 2.8460 | LR: 2.52e-04 | 1772.9ms/step | 11.3 samples/s +Step 22300/44000 | Loss: 2.8991 | LR: 2.51e-04 | 1770.6ms/step | 11.3 samples/s +Step 22310/44000 | Loss: 2.9836 | LR: 2.51e-04 | 1771.5ms/step | 11.3 samples/s +Step 22320/44000 | Loss: 2.9604 | LR: 2.51e-04 | 1772.5ms/step | 11.3 samples/s +Step 22330/44000 | Loss: 2.8921 | LR: 2.51e-04 | 1771.1ms/step | 11.3 samples/s +Step 22340/44000 | Loss: 2.9545 | LR: 2.51e-04 | 1770.8ms/step | 11.3 samples/s +Step 22350/44000 | Loss: 2.9822 | LR: 2.50e-04 | 1771.8ms/step | 11.3 samples/s +Step 22360/44000 | Loss: 2.9276 | LR: 2.50e-04 | 1771.7ms/step | 11.3 samples/s +Step 22370/44000 | Loss: 2.9170 | LR: 2.50e-04 | 1772.4ms/step | 11.3 samples/s +Step 22380/44000 | Loss: 2.9609 | LR: 2.50e-04 | 1773.5ms/step | 11.3 samples/s +Step 22390/44000 | Loss: 2.9126 | LR: 2.50e-04 | 1773.8ms/step | 11.3 samples/s +Step 22400/44000 | Loss: 2.9267 | LR: 2.50e-04 | 1773.5ms/step | 11.3 samples/s +Step 22410/44000 | Loss: 2.9545 | LR: 2.49e-04 | 1773.4ms/step | 11.3 samples/s +Step 22420/44000 | Loss: 2.9243 | LR: 2.49e-04 | 1771.0ms/step | 11.3 samples/s +Step 22430/44000 | Loss: 2.9332 | LR: 2.49e-04 | 1771.4ms/step | 11.3 samples/s +Step 22440/44000 | Loss: 2.9683 | LR: 2.49e-04 | 1771.3ms/step | 11.3 samples/s +Step 22450/44000 | Loss: 2.9830 | LR: 2.49e-04 | 1770.6ms/step | 11.3 samples/s +Step 22460/44000 | Loss: 2.8939 | LR: 2.48e-04 | 1770.3ms/step | 11.3 samples/s +Step 22470/44000 | Loss: 2.9308 | LR: 2.48e-04 | 1770.8ms/step | 11.3 samples/s +Step 22480/44000 | Loss: 2.9259 | LR: 2.48e-04 | 1771.5ms/step | 11.3 samples/s +Step 22490/44000 | Loss: 2.9606 | LR: 2.48e-04 | 1774.1ms/step | 11.3 samples/s +Step 22500/44000 | Loss: 2.9183 | LR: 2.48e-04 | 1773.6ms/step | 11.3 samples/s +Step 22510/44000 | Loss: 2.9498 | LR: 2.48e-04 | 1771.7ms/step | 11.3 samples/s +Step 22520/44000 | Loss: 2.9181 | LR: 2.47e-04 | 1773.5ms/step | 11.3 samples/s +Step 22530/44000 | Loss: 2.9146 | LR: 2.47e-04 | 1772.8ms/step | 11.3 samples/s +Step 22540/44000 | Loss: 2.9389 | LR: 2.47e-04 | 1773.3ms/step | 11.3 samples/s +Step 22550/44000 | Loss: 2.9228 | LR: 2.47e-04 | 1773.0ms/step | 11.3 samples/s +Step 22560/44000 | Loss: 2.9578 | LR: 2.47e-04 | 1772.6ms/step | 11.3 samples/s +Step 22570/44000 | Loss: 2.9510 | LR: 2.46e-04 | 1773.7ms/step | 11.3 samples/s +Step 22580/44000 | Loss: 2.9165 | LR: 2.46e-04 | 1772.8ms/step | 11.3 samples/s +Step 22590/44000 | Loss: 2.9103 | LR: 2.46e-04 | 1774.3ms/step | 11.3 samples/s +Step 22600/44000 | Loss: 2.9526 | LR: 2.46e-04 | 1773.1ms/step | 11.3 samples/s +Step 22610/44000 | Loss: 2.8903 | LR: 2.46e-04 | 1774.3ms/step | 11.3 samples/s +Step 22620/44000 | Loss: 2.9446 | LR: 2.46e-04 | 1772.4ms/step | 11.3 samples/s +Step 22630/44000 | Loss: 2.9834 | LR: 2.45e-04 | 1771.9ms/step | 11.3 samples/s +Step 22640/44000 | Loss: 2.9538 | LR: 2.45e-04 | 1771.7ms/step | 11.3 samples/s +Step 22650/44000 | Loss: 2.9311 | LR: 2.45e-04 | 1772.9ms/step | 11.3 samples/s +Step 22660/44000 | Loss: 2.9542 | LR: 2.45e-04 | 1771.9ms/step | 11.3 samples/s +Step 22670/44000 | Loss: 2.9533 | LR: 2.45e-04 | 1771.4ms/step | 11.3 samples/s +Step 22680/44000 | Loss: 2.8716 | LR: 2.44e-04 | 1771.6ms/step | 11.3 samples/s +Step 22690/44000 | Loss: 2.9154 | LR: 2.44e-04 | 1772.4ms/step | 11.3 samples/s +Step 22700/44000 | Loss: 2.9791 | LR: 2.44e-04 | 1772.3ms/step | 11.3 samples/s +Step 22710/44000 | Loss: 2.9177 | LR: 2.44e-04 | 1772.2ms/step | 11.3 samples/s +Step 22720/44000 | Loss: 2.9510 | LR: 2.44e-04 | 1773.0ms/step | 11.3 samples/s +Step 22730/44000 | Loss: 2.9412 | LR: 2.44e-04 | 1772.1ms/step | 11.3 samples/s +Step 22740/44000 | Loss: 2.9281 | LR: 2.43e-04 | 1772.3ms/step | 11.3 samples/s +Step 22750/44000 | Loss: 2.8959 | LR: 2.43e-04 | 1772.1ms/step | 11.3 samples/s +Step 22760/44000 | Loss: 2.9371 | LR: 2.43e-04 | 1772.2ms/step | 11.3 samples/s +Step 22770/44000 | Loss: 2.9628 | LR: 2.43e-04 | 1771.5ms/step | 11.3 samples/s +Step 22780/44000 | Loss: 2.9025 | LR: 2.43e-04 | 1770.6ms/step | 11.3 samples/s +Step 22790/44000 | Loss: 2.9476 | LR: 2.42e-04 | 1771.8ms/step | 11.3 samples/s +Step 22800/44000 | Loss: 2.9513 | LR: 2.42e-04 | 1772.6ms/step | 11.3 samples/s +Step 22810/44000 | Loss: 2.9639 | LR: 2.42e-04 | 1771.8ms/step | 11.3 samples/s +Step 22820/44000 | Loss: 2.9381 | LR: 2.42e-04 | 1773.2ms/step | 11.3 samples/s +Step 22830/44000 | Loss: 2.9468 | LR: 2.42e-04 | 1770.6ms/step | 11.3 samples/s +Step 22840/44000 | Loss: 2.9237 | LR: 2.42e-04 | 1771.5ms/step | 11.3 samples/s +Step 22850/44000 | Loss: 2.9807 | LR: 2.41e-04 | 1771.2ms/step | 11.3 samples/s +Step 22860/44000 | Loss: 2.9457 | LR: 2.41e-04 | 1770.2ms/step | 11.3 samples/s +Step 22870/44000 | Loss: 2.9746 | LR: 2.41e-04 | 1770.5ms/step | 11.3 samples/s +Step 22880/44000 | Loss: 2.9520 | LR: 2.41e-04 | 1770.1ms/step | 11.3 samples/s +Step 22890/44000 | Loss: 2.9710 | LR: 2.41e-04 | 1772.0ms/step | 11.3 samples/s +Step 22900/44000 | Loss: 2.9509 | LR: 2.40e-04 | 1771.6ms/step | 11.3 samples/s +Step 22910/44000 | Loss: 2.9411 | LR: 2.40e-04 | 1771.8ms/step | 11.3 samples/s +Step 22920/44000 | Loss: 2.9636 | LR: 2.40e-04 | 1771.6ms/step | 11.3 samples/s +Step 22930/44000 | Loss: 2.9357 | LR: 2.40e-04 | 1772.1ms/step | 11.3 samples/s +Step 22940/44000 | Loss: 2.9113 | LR: 2.40e-04 | 1771.4ms/step | 11.3 samples/s diff --git a/epfl_final/train_logs/ropectl_a1p0_a100_ddp2/seed43.20260820T140226Z.log b/epfl_final/train_logs/ropectl_a1p0_a100_ddp2/seed43.20260820T140226Z.log new file mode 100644 index 0000000000000000000000000000000000000000..b98660856ce5fb5a54f2d585c3257f13319a6d5f --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_a100_ddp2/seed43.20260820T140226Z.log @@ -0,0 +1,838 @@ +W0820 14:02:40.078000 230 site-packages/torch/distributed/run.py:851] +W0820 14:02:40.078000 230 site-packages/torch/distributed/run.py:851] ***************************************** +W0820 14:02:40.078000 230 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0820 14:02:40.078000 230 site-packages/torch/distributed/run.py:851] ***************************************** +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-18000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-18000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 43, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed43", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control", + "ddp2-a100" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-18000 +Checkpoint activations: FalseCheckpoint activations: False + +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... + +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-18000... +Preserving checkpoint LR state for exact resume: current=[0.00032811727248514757], schedule_base=[0.0005] +Resumed from checkpoint at step 18000 (epoch 0, micro_step 180000) +Resuming training from step 18000 +Resuming wandb run ysa9iyov +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run ysa9iyov +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140401-ysa9iyov +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed43 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ysa9iyov +Starting training... +Validation at step 18000: Loss = 2.9728, Perplexity = 19.55 +wandb: WARNING Tried to log to step 18000 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 18001: dense (dense step interval) +Step 18010/44000 | Loss: 2.9971 | LR: 3.28e-04 | 42307.5ms/step | 0.5 samples/s +wandb: WARNING Tried to log to step 18010 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18020/44000 | Loss: 2.9907 | LR: 3.28e-04 | 1773.9ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 18020 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18030/44000 | Loss: 3.0001 | LR: 3.28e-04 | 1777.2ms/step | 11.3 samples/s +wandb: WARNING Tried to log to step 18030 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18040/44000 | Loss: 2.9561 | LR: 3.27e-04 | 1778.8ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18040 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18050/44000 | Loss: 2.9921 | LR: 3.27e-04 | 1779.0ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18050 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18060/44000 | Loss: 3.0097 | LR: 3.27e-04 | 1781.8ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18060 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18070/44000 | Loss: 2.9773 | LR: 3.27e-04 | 1781.6ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18070 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18080/44000 | Loss: 2.9883 | LR: 3.27e-04 | 1781.0ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18080 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18090/44000 | Loss: 2.9542 | LR: 3.27e-04 | 1780.5ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18090 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18100/44000 | Loss: 2.9393 | LR: 3.26e-04 | 1781.5ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18100 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18110/44000 | Loss: 3.0064 | LR: 3.26e-04 | 1781.2ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18110 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18120/44000 | Loss: 2.9451 | LR: 3.26e-04 | 1782.0ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18120 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18130/44000 | Loss: 2.9749 | LR: 3.26e-04 | 1781.3ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18130 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18140/44000 | Loss: 3.0306 | LR: 3.26e-04 | 1781.8ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18140 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18150/44000 | Loss: 3.0022 | LR: 3.26e-04 | 1781.8ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18150 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18160/44000 | Loss: 2.9571 | LR: 3.25e-04 | 1782.0ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18160 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18170/44000 | Loss: 2.9630 | LR: 3.25e-04 | 1783.4ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18170 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18180/44000 | Loss: 2.9619 | LR: 3.25e-04 | 1783.4ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18180 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18190/44000 | Loss: 3.0241 | LR: 3.25e-04 | 1782.4ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18190 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18200/44000 | Loss: 2.9990 | LR: 3.25e-04 | 1785.0ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18200 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18210/44000 | Loss: 3.0164 | LR: 3.24e-04 | 1783.6ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18210 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18220/44000 | Loss: 2.9772 | LR: 3.24e-04 | 1782.7ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18220 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18230/44000 | Loss: 2.9828 | LR: 3.24e-04 | 1782.7ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18230 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18240/44000 | Loss: 3.0113 | LR: 3.24e-04 | 1783.1ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18240 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18250/44000 | Loss: 2.9762 | LR: 3.24e-04 | 1783.6ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18250 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18260/44000 | Loss: 3.0124 | LR: 3.24e-04 | 1783.3ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18260 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18270/44000 | Loss: 2.9906 | LR: 3.23e-04 | 1783.6ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18270 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18280/44000 | Loss: 2.9492 | LR: 3.23e-04 | 1784.0ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18280 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18290/44000 | Loss: 3.0016 | LR: 3.23e-04 | 1781.8ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18290 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18300/44000 | Loss: 3.0241 | LR: 3.23e-04 | 1782.7ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18300 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18310/44000 | Loss: 3.0117 | LR: 3.23e-04 | 1782.7ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18310 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18320/44000 | Loss: 3.0382 | LR: 3.23e-04 | 1783.2ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18320 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18330/44000 | Loss: 3.0040 | LR: 3.22e-04 | 1782.2ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18330 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18340/44000 | Loss: 2.9568 | LR: 3.22e-04 | 1782.7ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18340 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18350/44000 | Loss: 2.9752 | LR: 3.22e-04 | 1782.5ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18350 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18360/44000 | Loss: 2.9717 | LR: 3.22e-04 | 1782.6ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18360 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18370/44000 | Loss: 3.0086 | LR: 3.22e-04 | 1781.7ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18370 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18380/44000 | Loss: 2.9834 | LR: 3.22e-04 | 1782.6ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18380 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18390/44000 | Loss: 3.0108 | LR: 3.21e-04 | 1782.5ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18390 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18400/44000 | Loss: 3.0106 | LR: 3.21e-04 | 1783.0ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18400 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18410/44000 | Loss: 2.9893 | LR: 3.21e-04 | 1781.9ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18410 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18420/44000 | Loss: 2.9824 | LR: 3.21e-04 | 1781.9ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18420 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18430/44000 | Loss: 2.9528 | LR: 3.21e-04 | 1780.9ms/step | 11.2 samples/s +wandb: WARNING Tried to log to step 18430 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 18440/44000 | Loss: 2.9364 | LR: 3.20e-04 | 1781.7ms/step | 11.2 samples/s +Step 18450/44000 | Loss: 2.9737 | LR: 3.20e-04 | 1783.2ms/step | 11.2 samples/s +Step 18460/44000 | Loss: 2.9931 | LR: 3.20e-04 | 1782.8ms/step | 11.2 samples/s +Step 18470/44000 | Loss: 3.0209 | LR: 3.20e-04 | 1782.0ms/step | 11.2 samples/s +Step 18480/44000 | Loss: 2.9502 | LR: 3.20e-04 | 1782.9ms/step | 11.2 samples/s +Step 18490/44000 | Loss: 2.9597 | LR: 3.20e-04 | 1783.0ms/step | 11.2 samples/s +Step 18500/44000 | Loss: 2.9759 | LR: 3.19e-04 | 1783.1ms/step | 11.2 samples/s +Step 18510/44000 | Loss: 2.9690 | LR: 3.19e-04 | 1783.1ms/step | 11.2 samples/s +Step 18520/44000 | Loss: 2.9974 | LR: 3.19e-04 | 1782.9ms/step | 11.2 samples/s +Step 18530/44000 | Loss: 3.0094 | LR: 3.19e-04 | 1783.1ms/step | 11.2 samples/s +Step 18540/44000 | Loss: 3.0191 | LR: 3.19e-04 | 1784.2ms/step | 11.2 samples/s +Step 18550/44000 | Loss: 2.9463 | LR: 3.19e-04 | 1782.2ms/step | 11.2 samples/s +Step 18560/44000 | Loss: 3.0022 | LR: 3.18e-04 | 1782.6ms/step | 11.2 samples/s +Step 18570/44000 | Loss: 3.0116 | LR: 3.18e-04 | 1781.9ms/step | 11.2 samples/s +Step 18580/44000 | Loss: 3.0194 | LR: 3.18e-04 | 1783.9ms/step | 11.2 samples/s +Step 18590/44000 | Loss: 2.9816 | LR: 3.18e-04 | 1783.1ms/step | 11.2 samples/s +Step 18600/44000 | Loss: 2.9943 | LR: 3.18e-04 | 1783.7ms/step | 11.2 samples/s +Step 18610/44000 | Loss: 2.9607 | LR: 3.18e-04 | 1782.4ms/step | 11.2 samples/s +Step 18620/44000 | Loss: 3.0022 | LR: 3.17e-04 | 1782.5ms/step | 11.2 samples/s +Step 18630/44000 | Loss: 3.0042 | LR: 3.17e-04 | 1782.9ms/step | 11.2 samples/s +Step 18640/44000 | Loss: 3.0192 | LR: 3.17e-04 | 1782.6ms/step | 11.2 samples/s +Step 18650/44000 | Loss: 2.9810 | LR: 3.17e-04 | 1782.3ms/step | 11.2 samples/s +Step 18660/44000 | Loss: 2.9998 | LR: 3.17e-04 | 1782.8ms/step | 11.2 samples/s +Step 18670/44000 | Loss: 2.9850 | LR: 3.16e-04 | 1782.9ms/step | 11.2 samples/s +Step 18680/44000 | Loss: 3.0212 | LR: 3.16e-04 | 1783.8ms/step | 11.2 samples/s +Step 18690/44000 | Loss: 2.9495 | LR: 3.16e-04 | 1783.3ms/step | 11.2 samples/s +Step 18700/44000 | Loss: 2.9738 | LR: 3.16e-04 | 1781.9ms/step | 11.2 samples/s +Step 18710/44000 | Loss: 3.0018 | LR: 3.16e-04 | 1783.0ms/step | 11.2 samples/s +Step 18720/44000 | Loss: 2.9478 | LR: 3.16e-04 | 1782.1ms/step | 11.2 samples/s +Step 18730/44000 | Loss: 3.0106 | LR: 3.15e-04 | 1782.9ms/step | 11.2 samples/s +Step 18740/44000 | Loss: 2.9539 | LR: 3.15e-04 | 1781.9ms/step | 11.2 samples/s +Step 18750/44000 | Loss: 2.9888 | LR: 3.15e-04 | 1786.6ms/step | 11.2 samples/s +Step 18760/44000 | Loss: 2.9621 | LR: 3.15e-04 | 1784.1ms/step | 11.2 samples/s +Step 18770/44000 | Loss: 2.9700 | LR: 3.15e-04 | 1784.5ms/step | 11.2 samples/s +Step 18780/44000 | Loss: 2.9620 | LR: 3.15e-04 | 1783.6ms/step | 11.2 samples/s +Step 18790/44000 | Loss: 2.9962 | LR: 3.14e-04 | 1783.3ms/step | 11.2 samples/s +Step 18800/44000 | Loss: 2.9913 | LR: 3.14e-04 | 1786.7ms/step | 11.2 samples/s +Step 18810/44000 | Loss: 2.9776 | LR: 3.14e-04 | 1786.7ms/step | 11.2 samples/s +Step 18820/44000 | Loss: 2.9986 | LR: 3.14e-04 | 1782.5ms/step | 11.2 samples/s +Step 18830/44000 | Loss: 2.9464 | LR: 3.14e-04 | 1783.4ms/step | 11.2 samples/s +Step 18840/44000 | Loss: 2.9397 | LR: 3.13e-04 | 1783.2ms/step | 11.2 samples/s +Step 18850/44000 | Loss: 2.9791 | LR: 3.13e-04 | 1783.6ms/step | 11.2 samples/s +Step 18860/44000 | Loss: 2.9596 | LR: 3.13e-04 | 1783.5ms/step | 11.2 samples/s +Step 18870/44000 | Loss: 2.9497 | LR: 3.13e-04 | 1783.7ms/step | 11.2 samples/s +Step 18880/44000 | Loss: 2.9688 | LR: 3.13e-04 | 1783.8ms/step | 11.2 samples/s +Step 18890/44000 | Loss: 2.9681 | LR: 3.13e-04 | 1784.2ms/step | 11.2 samples/s +Step 18900/44000 | Loss: 2.9671 | LR: 3.12e-04 | 1784.4ms/step | 11.2 samples/s +Step 18910/44000 | Loss: 2.9530 | LR: 3.12e-04 | 1784.7ms/step | 11.2 samples/s +Step 18920/44000 | Loss: 2.9778 | LR: 3.12e-04 | 1784.4ms/step | 11.2 samples/s +Step 18930/44000 | Loss: 3.0043 | LR: 3.12e-04 | 1783.4ms/step | 11.2 samples/s +Step 18940/44000 | Loss: 2.9871 | LR: 3.12e-04 | 1782.7ms/step | 11.2 samples/s +Step 18950/44000 | Loss: 2.9887 | LR: 3.12e-04 | 1782.5ms/step | 11.2 samples/s +Step 18960/44000 | Loss: 2.9920 | LR: 3.11e-04 | 1783.0ms/step | 11.2 samples/s +Step 18970/44000 | Loss: 2.9774 | LR: 3.11e-04 | 1783.8ms/step | 11.2 samples/s +Step 18980/44000 | Loss: 2.9757 | LR: 3.11e-04 | 1782.9ms/step | 11.2 samples/s +Step 18990/44000 | Loss: 2.9514 | LR: 3.11e-04 | 1783.4ms/step | 11.2 samples/s +Step 19000/44000 | Loss: 2.9648 | LR: 3.11e-04 | 1782.4ms/step | 11.2 samples/s +Validation at step 19000: Loss = 2.9626, Perplexity = 19.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-19000 +Step 19010/44000 | Loss: 2.9803 | LR: 3.11e-04 | 3584.0ms/step | 5.6 samples/s +Step 19020/44000 | Loss: 2.9681 | LR: 3.10e-04 | 1785.4ms/step | 11.2 samples/s +Step 19030/44000 | Loss: 2.9647 | LR: 3.10e-04 | 1786.4ms/step | 11.2 samples/s +Step 19040/44000 | Loss: 2.9520 | LR: 3.10e-04 | 1786.3ms/step | 11.2 samples/s +Step 19050/44000 | Loss: 3.0237 | LR: 3.10e-04 | 1787.6ms/step | 11.2 samples/s +Step 19060/44000 | Loss: 2.9959 | LR: 3.10e-04 | 1785.9ms/step | 11.2 samples/s +Step 19070/44000 | Loss: 2.9645 | LR: 3.09e-04 | 1787.2ms/step | 11.2 samples/s +Step 19080/44000 | Loss: 3.0128 | LR: 3.09e-04 | 1784.9ms/step | 11.2 samples/s +Step 19090/44000 | Loss: 2.9736 | LR: 3.09e-04 | 1786.8ms/step | 11.2 samples/s +Step 19100/44000 | Loss: 3.0043 | LR: 3.09e-04 | 1786.2ms/step | 11.2 samples/s +Step 19110/44000 | Loss: 3.0010 | LR: 3.09e-04 | 1787.1ms/step | 11.2 samples/s +Step 19120/44000 | Loss: 3.0102 | LR: 3.09e-04 | 1786.1ms/step | 11.2 samples/s +Step 19130/44000 | Loss: 3.0016 | LR: 3.08e-04 | 1786.9ms/step | 11.2 samples/s +Step 19140/44000 | Loss: 3.0037 | LR: 3.08e-04 | 1785.6ms/step | 11.2 samples/s +Step 19150/44000 | Loss: 2.9993 | LR: 3.08e-04 | 1787.7ms/step | 11.2 samples/s +Step 19160/44000 | Loss: 2.9615 | LR: 3.08e-04 | 1787.0ms/step | 11.2 samples/s +Step 19170/44000 | Loss: 3.0020 | LR: 3.08e-04 | 1786.2ms/step | 11.2 samples/s +Step 19180/44000 | Loss: 3.0121 | LR: 3.08e-04 | 1785.0ms/step | 11.2 samples/s +Step 19190/44000 | Loss: 2.9320 | LR: 3.07e-04 | 1786.2ms/step | 11.2 samples/s +Step 19200/44000 | Loss: 2.9542 | LR: 3.07e-04 | 1786.1ms/step | 11.2 samples/s +Step 19210/44000 | Loss: 2.9760 | LR: 3.07e-04 | 1785.5ms/step | 11.2 samples/s +Step 19220/44000 | Loss: 3.0029 | LR: 3.07e-04 | 1785.0ms/step | 11.2 samples/s +Step 19230/44000 | Loss: 2.9765 | LR: 3.07e-04 | 1786.1ms/step | 11.2 samples/s +Step 19240/44000 | Loss: 2.9915 | LR: 3.06e-04 | 1786.2ms/step | 11.2 samples/s +Step 19250/44000 | Loss: 2.9664 | LR: 3.06e-04 | 1786.1ms/step | 11.2 samples/s +Step 19260/44000 | Loss: 2.9532 | LR: 3.06e-04 | 1785.8ms/step | 11.2 samples/s +Step 19270/44000 | Loss: 2.9631 | LR: 3.06e-04 | 1784.7ms/step | 11.2 samples/s +Step 19280/44000 | Loss: 3.0000 | LR: 3.06e-04 | 1786.2ms/step | 11.2 samples/s +Step 19290/44000 | Loss: 2.9515 | LR: 3.06e-04 | 1784.9ms/step | 11.2 samples/s +Step 19300/44000 | Loss: 2.9979 | LR: 3.05e-04 | 1786.6ms/step | 11.2 samples/s +Step 19310/44000 | Loss: 2.9852 | LR: 3.05e-04 | 1786.5ms/step | 11.2 samples/s +Step 19320/44000 | Loss: 2.9740 | LR: 3.05e-04 | 1786.5ms/step | 11.2 samples/s +Step 19330/44000 | Loss: 2.9537 | LR: 3.05e-04 | 1786.3ms/step | 11.2 samples/s +Step 19340/44000 | Loss: 2.9845 | LR: 3.05e-04 | 1786.3ms/step | 11.2 samples/s +Step 19350/44000 | Loss: 2.9603 | LR: 3.04e-04 | 1785.3ms/step | 11.2 samples/s +Step 19360/44000 | Loss: 2.9405 | LR: 3.04e-04 | 1784.9ms/step | 11.2 samples/s +Step 19370/44000 | Loss: 2.9557 | LR: 3.04e-04 | 1784.2ms/step | 11.2 samples/s +Step 19380/44000 | Loss: 2.9809 | LR: 3.04e-04 | 1785.4ms/step | 11.2 samples/s +Step 19390/44000 | Loss: 2.9811 | LR: 3.04e-04 | 1784.9ms/step | 11.2 samples/s +Step 19400/44000 | Loss: 2.9769 | LR: 3.04e-04 | 1787.2ms/step | 11.2 samples/s +Step 19410/44000 | Loss: 2.9840 | LR: 3.03e-04 | 1786.3ms/step | 11.2 samples/s +Step 19420/44000 | Loss: 2.9710 | LR: 3.03e-04 | 1786.6ms/step | 11.2 samples/s +Step 19430/44000 | Loss: 2.9264 | LR: 3.03e-04 | 1784.7ms/step | 11.2 samples/s +Step 19440/44000 | Loss: 2.9834 | LR: 3.03e-04 | 1785.9ms/step | 11.2 samples/s +Step 19450/44000 | Loss: 3.0116 | LR: 3.03e-04 | 1785.7ms/step | 11.2 samples/s +Step 19460/44000 | Loss: 2.9537 | LR: 3.03e-04 | 1787.3ms/step | 11.2 samples/s +Step 19470/44000 | Loss: 2.9745 | LR: 3.02e-04 | 1788.1ms/step | 11.2 samples/s +Step 19480/44000 | Loss: 2.9615 | LR: 3.02e-04 | 1786.2ms/step | 11.2 samples/s +Step 19490/44000 | Loss: 2.9910 | LR: 3.02e-04 | 1786.4ms/step | 11.2 samples/s +Step 19500/44000 | Loss: 3.0047 | LR: 3.02e-04 | 1787.2ms/step | 11.2 samples/s +Step 19510/44000 | Loss: 2.9671 | LR: 3.02e-04 | 1785.6ms/step | 11.2 samples/s +Step 19520/44000 | Loss: 2.9678 | LR: 3.01e-04 | 1785.5ms/step | 11.2 samples/s +Step 19530/44000 | Loss: 2.9600 | LR: 3.01e-04 | 1785.8ms/step | 11.2 samples/s +Step 19540/44000 | Loss: 2.9711 | LR: 3.01e-04 | 1785.8ms/step | 11.2 samples/s +Step 19550/44000 | Loss: 2.9762 | LR: 3.01e-04 | 1786.3ms/step | 11.2 samples/s +Step 19560/44000 | Loss: 2.9826 | LR: 3.01e-04 | 1785.5ms/step | 11.2 samples/s +Step 19570/44000 | Loss: 2.9861 | LR: 3.01e-04 | 1785.6ms/step | 11.2 samples/s +Step 19580/44000 | Loss: 2.9780 | LR: 3.00e-04 | 1785.7ms/step | 11.2 samples/s +Step 19590/44000 | Loss: 2.9711 | LR: 3.00e-04 | 1785.8ms/step | 11.2 samples/s +Step 19600/44000 | Loss: 2.9793 | LR: 3.00e-04 | 1785.5ms/step | 11.2 samples/s +Step 19610/44000 | Loss: 3.0148 | LR: 3.00e-04 | 1785.2ms/step | 11.2 samples/s +Step 19620/44000 | Loss: 3.0110 | LR: 3.00e-04 | 1782.6ms/step | 11.2 samples/s +Step 19630/44000 | Loss: 2.9517 | LR: 3.00e-04 | 1785.6ms/step | 11.2 samples/s +Step 19640/44000 | Loss: 2.9868 | LR: 2.99e-04 | 1785.3ms/step | 11.2 samples/s +Step 19650/44000 | Loss: 2.9841 | LR: 2.99e-04 | 1786.1ms/step | 11.2 samples/s +Step 19660/44000 | Loss: 2.9842 | LR: 2.99e-04 | 1784.9ms/step | 11.2 samples/s +Step 19670/44000 | Loss: 3.0168 | LR: 2.99e-04 | 1786.1ms/step | 11.2 samples/s +Step 19680/44000 | Loss: 2.9891 | LR: 2.99e-04 | 1785.7ms/step | 11.2 samples/s +Step 19690/44000 | Loss: 2.9710 | LR: 2.98e-04 | 1786.8ms/step | 11.2 samples/s +Step 19700/44000 | Loss: 2.9777 | LR: 2.98e-04 | 1786.5ms/step | 11.2 samples/s +Step 19710/44000 | Loss: 2.9780 | LR: 2.98e-04 | 1787.1ms/step | 11.2 samples/s +Step 19720/44000 | Loss: 3.0133 | LR: 2.98e-04 | 1787.6ms/step | 11.2 samples/s +Step 19730/44000 | Loss: 2.9665 | LR: 2.98e-04 | 1786.6ms/step | 11.2 samples/s +Step 19740/44000 | Loss: 2.9740 | LR: 2.98e-04 | 1785.4ms/step | 11.2 samples/s +Step 19750/44000 | Loss: 2.9987 | LR: 2.97e-04 | 1786.4ms/step | 11.2 samples/s +Step 19760/44000 | Loss: 2.9649 | LR: 2.97e-04 | 1785.2ms/step | 11.2 samples/s +Step 19770/44000 | Loss: 2.9672 | LR: 2.97e-04 | 1786.0ms/step | 11.2 samples/s +Step 19780/44000 | Loss: 2.9803 | LR: 2.97e-04 | 1785.9ms/step | 11.2 samples/s +Step 19790/44000 | Loss: 3.0131 | LR: 2.97e-04 | 1785.7ms/step | 11.2 samples/s +Step 19800/44000 | Loss: 2.9232 | LR: 2.96e-04 | 1785.6ms/step | 11.2 samples/s +Step 19810/44000 | Loss: 2.9741 | LR: 2.96e-04 | 1785.9ms/step | 11.2 samples/s +Step 19820/44000 | Loss: 3.0011 | LR: 2.96e-04 | 1785.4ms/step | 11.2 samples/s +Step 19830/44000 | Loss: 2.9685 | LR: 2.96e-04 | 1785.9ms/step | 11.2 samples/s +Step 19840/44000 | Loss: 3.0102 | LR: 2.96e-04 | 1785.9ms/step | 11.2 samples/s +Step 19850/44000 | Loss: 2.9582 | LR: 2.96e-04 | 1786.0ms/step | 11.2 samples/s +Step 19860/44000 | Loss: 3.0023 | LR: 2.95e-04 | 1786.2ms/step | 11.2 samples/s +Step 19870/44000 | Loss: 2.9845 | LR: 2.95e-04 | 1785.1ms/step | 11.2 samples/s +Step 19880/44000 | Loss: 3.0251 | LR: 2.95e-04 | 1785.3ms/step | 11.2 samples/s +Step 19890/44000 | Loss: 2.9631 | LR: 2.95e-04 | 1785.0ms/step | 11.2 samples/s +Step 19900/44000 | Loss: 2.9846 | LR: 2.95e-04 | 1786.8ms/step | 11.2 samples/s +Step 19910/44000 | Loss: 2.9572 | LR: 2.95e-04 | 1785.3ms/step | 11.2 samples/s +Step 19920/44000 | Loss: 2.9981 | LR: 2.94e-04 | 1784.7ms/step | 11.2 samples/s +Step 19930/44000 | Loss: 2.9439 | LR: 2.94e-04 | 1784.0ms/step | 11.2 samples/s +Step 19940/44000 | Loss: 2.9729 | LR: 2.94e-04 | 1785.8ms/step | 11.2 samples/s +Step 19950/44000 | Loss: 2.9531 | LR: 2.94e-04 | 1784.2ms/step | 11.2 samples/s +Step 19960/44000 | Loss: 2.9860 | LR: 2.94e-04 | 1784.4ms/step | 11.2 samples/s +Step 19970/44000 | Loss: 2.9534 | LR: 2.93e-04 | 1783.5ms/step | 11.2 samples/s +Step 19980/44000 | Loss: 2.9586 | LR: 2.93e-04 | 1784.1ms/step | 11.2 samples/s +Step 19990/44000 | Loss: 2.9507 | LR: 2.93e-04 | 1784.3ms/step | 11.2 samples/s +Step 20000/44000 | Loss: 2.9806 | LR: 2.93e-04 | 1784.9ms/step | 11.2 samples/s +Validation at step 20000: Loss = 2.9514, Perplexity = 19.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-20000 +Step 20010/44000 | Loss: 2.9525 | LR: 2.93e-04 | 3588.2ms/step | 5.6 samples/s +Step 20020/44000 | Loss: 2.9913 | LR: 2.93e-04 | 1784.6ms/step | 11.2 samples/s +Step 20030/44000 | Loss: 2.9715 | LR: 2.92e-04 | 1785.8ms/step | 11.2 samples/s +Step 20040/44000 | Loss: 2.9938 | LR: 2.92e-04 | 1784.2ms/step | 11.2 samples/s +Step 20050/44000 | Loss: 2.9420 | LR: 2.92e-04 | 1784.0ms/step | 11.2 samples/s +Step 20060/44000 | Loss: 2.9639 | LR: 2.92e-04 | 1783.9ms/step | 11.2 samples/s +Step 20070/44000 | Loss: 3.0013 | LR: 2.92e-04 | 1784.7ms/step | 11.2 samples/s +Step 20080/44000 | Loss: 2.9357 | LR: 2.91e-04 | 1784.5ms/step | 11.2 samples/s +Step 20090/44000 | Loss: 2.9849 | LR: 2.91e-04 | 1784.5ms/step | 11.2 samples/s +Step 20100/44000 | Loss: 2.9498 | LR: 2.91e-04 | 1784.7ms/step | 11.2 samples/s +Step 20110/44000 | Loss: 2.9376 | LR: 2.91e-04 | 1785.1ms/step | 11.2 samples/s +Step 20120/44000 | Loss: 2.9287 | LR: 2.91e-04 | 1784.4ms/step | 11.2 samples/s +Step 20130/44000 | Loss: 2.9613 | LR: 2.91e-04 | 1784.2ms/step | 11.2 samples/s +Step 20140/44000 | Loss: 2.9598 | LR: 2.90e-04 | 1786.0ms/step | 11.2 samples/s +Step 20150/44000 | Loss: 2.9421 | LR: 2.90e-04 | 1785.0ms/step | 11.2 samples/s +Step 20160/44000 | Loss: 3.0087 | LR: 2.90e-04 | 1784.8ms/step | 11.2 samples/s +Step 20170/44000 | Loss: 2.9875 | LR: 2.90e-04 | 1784.1ms/step | 11.2 samples/s +Step 20180/44000 | Loss: 2.9414 | LR: 2.90e-04 | 1784.9ms/step | 11.2 samples/s +Step 20190/44000 | Loss: 2.9603 | LR: 2.90e-04 | 1784.0ms/step | 11.2 samples/s +Step 20200/44000 | Loss: 2.9277 | LR: 2.89e-04 | 1786.8ms/step | 11.2 samples/s +Step 20210/44000 | Loss: 2.9626 | LR: 2.89e-04 | 1784.3ms/step | 11.2 samples/s +Step 20220/44000 | Loss: 2.9394 | LR: 2.89e-04 | 1788.5ms/step | 11.2 samples/s +Step 20230/44000 | Loss: 2.9337 | LR: 2.89e-04 | 1784.7ms/step | 11.2 samples/s +Step 20240/44000 | Loss: 2.9535 | LR: 2.89e-04 | 1786.2ms/step | 11.2 samples/s +Step 20250/44000 | Loss: 2.9984 | LR: 2.88e-04 | 1785.6ms/step | 11.2 samples/s +Step 20260/44000 | Loss: 2.9629 | LR: 2.88e-04 | 1787.2ms/step | 11.2 samples/s +Step 20270/44000 | Loss: 2.9640 | LR: 2.88e-04 | 1784.1ms/step | 11.2 samples/s +Step 20280/44000 | Loss: 2.9570 | LR: 2.88e-04 | 1786.5ms/step | 11.2 samples/s +Step 20290/44000 | Loss: 2.9456 | LR: 2.88e-04 | 1786.6ms/step | 11.2 samples/s +Step 20300/44000 | Loss: 2.9794 | LR: 2.88e-04 | 1784.1ms/step | 11.2 samples/s +Step 20310/44000 | Loss: 2.9673 | LR: 2.87e-04 | 1785.3ms/step | 11.2 samples/s +Step 20320/44000 | Loss: 2.9821 | LR: 2.87e-04 | 1785.3ms/step | 11.2 samples/s +Step 20330/44000 | Loss: 2.9740 | LR: 2.87e-04 | 1784.6ms/step | 11.2 samples/s +Step 20340/44000 | Loss: 2.9713 | LR: 2.87e-04 | 1784.0ms/step | 11.2 samples/s +Step 20350/44000 | Loss: 2.9644 | LR: 2.87e-04 | 1786.9ms/step | 11.2 samples/s +Step 20360/44000 | Loss: 2.9474 | LR: 2.86e-04 | 1785.9ms/step | 11.2 samples/s +Step 20370/44000 | Loss: 2.9757 | LR: 2.86e-04 | 1786.2ms/step | 11.2 samples/s +Step 20380/44000 | Loss: 2.9547 | LR: 2.86e-04 | 1784.9ms/step | 11.2 samples/s +Step 20390/44000 | Loss: 2.9301 | LR: 2.86e-04 | 1784.8ms/step | 11.2 samples/s +Step 20400/44000 | Loss: 2.9666 | LR: 2.86e-04 | 1784.2ms/step | 11.2 samples/s +Step 20410/44000 | Loss: 2.9617 | LR: 2.86e-04 | 1785.0ms/step | 11.2 samples/s +Step 20420/44000 | Loss: 2.9757 | LR: 2.85e-04 | 1786.5ms/step | 11.2 samples/s +Step 20430/44000 | Loss: 2.9885 | LR: 2.85e-04 | 1785.8ms/step | 11.2 samples/s +Step 20440/44000 | Loss: 2.9902 | LR: 2.85e-04 | 1784.2ms/step | 11.2 samples/s +Step 20450/44000 | Loss: 2.9591 | LR: 2.85e-04 | 1785.5ms/step | 11.2 samples/s +Step 20460/44000 | Loss: 2.9288 | LR: 2.85e-04 | 1784.3ms/step | 11.2 samples/s +Step 20470/44000 | Loss: 2.9821 | LR: 2.84e-04 | 1785.2ms/step | 11.2 samples/s +Step 20480/44000 | Loss: 2.9736 | LR: 2.84e-04 | 1784.5ms/step | 11.2 samples/s +Step 20490/44000 | Loss: 2.9740 | LR: 2.84e-04 | 1785.8ms/step | 11.2 samples/s +Step 20500/44000 | Loss: 2.9178 | LR: 2.84e-04 | 1784.6ms/step | 11.2 samples/s +Step 20510/44000 | Loss: 3.0172 | LR: 2.84e-04 | 1785.1ms/step | 11.2 samples/s +Step 20520/44000 | Loss: 2.9671 | LR: 2.84e-04 | 1784.5ms/step | 11.2 samples/s +Step 20530/44000 | Loss: 2.9320 | LR: 2.83e-04 | 1784.8ms/step | 11.2 samples/s +Step 20540/44000 | Loss: 2.9589 | LR: 2.83e-04 | 1785.2ms/step | 11.2 samples/s +Step 20550/44000 | Loss: 2.9659 | LR: 2.83e-04 | 1783.8ms/step | 11.2 samples/s +Step 20560/44000 | Loss: 2.9420 | LR: 2.83e-04 | 1784.0ms/step | 11.2 samples/s +Step 20570/44000 | Loss: 2.9842 | LR: 2.83e-04 | 1783.3ms/step | 11.2 samples/s +Step 20580/44000 | Loss: 2.9633 | LR: 2.83e-04 | 1784.6ms/step | 11.2 samples/s +Step 20590/44000 | Loss: 2.9772 | LR: 2.82e-04 | 1784.5ms/step | 11.2 samples/s +Step 20600/44000 | Loss: 2.9824 | LR: 2.82e-04 | 1785.4ms/step | 11.2 samples/s +Step 20610/44000 | Loss: 2.9429 | LR: 2.82e-04 | 1784.3ms/step | 11.2 samples/s +Step 20620/44000 | Loss: 2.9572 | LR: 2.82e-04 | 1784.1ms/step | 11.2 samples/s +Step 20630/44000 | Loss: 2.9799 | LR: 2.82e-04 | 1783.6ms/step | 11.2 samples/s +Step 20640/44000 | Loss: 2.9444 | LR: 2.81e-04 | 1784.2ms/step | 11.2 samples/s +Step 20650/44000 | Loss: 2.9568 | LR: 2.81e-04 | 1783.0ms/step | 11.2 samples/s +Step 20660/44000 | Loss: 2.9536 | LR: 2.81e-04 | 1784.0ms/step | 11.2 samples/s +Step 20670/44000 | Loss: 2.9776 | LR: 2.81e-04 | 1783.2ms/step | 11.2 samples/s +Step 20680/44000 | Loss: 2.9186 | LR: 2.81e-04 | 1783.7ms/step | 11.2 samples/s +Step 20690/44000 | Loss: 2.9539 | LR: 2.81e-04 | 1783.3ms/step | 11.2 samples/s +Step 20700/44000 | Loss: 2.9371 | LR: 2.80e-04 | 1785.0ms/step | 11.2 samples/s +Step 20710/44000 | Loss: 2.9530 | LR: 2.80e-04 | 1784.1ms/step | 11.2 samples/s +Step 20720/44000 | Loss: 2.9832 | LR: 2.80e-04 | 1786.5ms/step | 11.2 samples/s +Step 20730/44000 | Loss: 2.9093 | LR: 2.80e-04 | 1786.4ms/step | 11.2 samples/s +Step 20740/44000 | Loss: 2.9929 | LR: 2.80e-04 | 1784.8ms/step | 11.2 samples/s +Step 20750/44000 | Loss: 2.9719 | LR: 2.79e-04 | 1784.1ms/step | 11.2 samples/s +Step 20760/44000 | Loss: 3.0116 | LR: 2.79e-04 | 1783.6ms/step | 11.2 samples/s +Step 20770/44000 | Loss: 2.9461 | LR: 2.79e-04 | 1783.8ms/step | 11.2 samples/s +Step 20780/44000 | Loss: 2.9501 | LR: 2.79e-04 | 1783.9ms/step | 11.2 samples/s +Step 20790/44000 | Loss: 2.9585 | LR: 2.79e-04 | 1783.8ms/step | 11.2 samples/s +Step 20800/44000 | Loss: 2.9744 | LR: 2.79e-04 | 1783.7ms/step | 11.2 samples/s +Step 20810/44000 | Loss: 2.9439 | LR: 2.78e-04 | 1784.1ms/step | 11.2 samples/s +Step 20820/44000 | Loss: 2.9059 | LR: 2.78e-04 | 1783.4ms/step | 11.2 samples/s +Step 20830/44000 | Loss: 2.9289 | LR: 2.78e-04 | 1785.1ms/step | 11.2 samples/s +Step 20840/44000 | Loss: 2.9486 | LR: 2.78e-04 | 1783.8ms/step | 11.2 samples/s +Step 20850/44000 | Loss: 2.9122 | LR: 2.78e-04 | 1785.2ms/step | 11.2 samples/s +Step 20860/44000 | Loss: 3.0248 | LR: 2.77e-04 | 1784.2ms/step | 11.2 samples/s +Step 20870/44000 | Loss: 2.9592 | LR: 2.77e-04 | 1792.6ms/step | 11.2 samples/s +Step 20880/44000 | Loss: 2.9901 | LR: 2.77e-04 | 1788.2ms/step | 11.2 samples/s +Step 20890/44000 | Loss: 2.9332 | LR: 2.77e-04 | 1785.5ms/step | 11.2 samples/s +Step 20900/44000 | Loss: 2.9794 | LR: 2.77e-04 | 1784.2ms/step | 11.2 samples/s +Step 20910/44000 | Loss: 2.9616 | LR: 2.77e-04 | 1784.8ms/step | 11.2 samples/s +Step 20920/44000 | Loss: 2.9739 | LR: 2.76e-04 | 1784.1ms/step | 11.2 samples/s +Step 20930/44000 | Loss: 2.9450 | LR: 2.76e-04 | 1785.6ms/step | 11.2 samples/s +Step 20940/44000 | Loss: 2.9973 | LR: 2.76e-04 | 1786.0ms/step | 11.2 samples/s +Step 20950/44000 | Loss: 2.9661 | LR: 2.76e-04 | 1787.1ms/step | 11.2 samples/s +Step 20960/44000 | Loss: 2.9288 | LR: 2.76e-04 | 1784.9ms/step | 11.2 samples/s +Step 20970/44000 | Loss: 2.9681 | LR: 2.75e-04 | 1785.0ms/step | 11.2 samples/s +Step 20980/44000 | Loss: 2.9672 | LR: 2.75e-04 | 1785.5ms/step | 11.2 samples/s +Step 20990/44000 | Loss: 2.9756 | LR: 2.75e-04 | 1784.5ms/step | 11.2 samples/s +Step 21000/44000 | Loss: 2.9553 | LR: 2.75e-04 | 1785.9ms/step | 11.2 samples/s +Validation at step 21000: Loss = 2.9396, Perplexity = 18.91 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-21000 +Step 21010/44000 | Loss: 2.9004 | LR: 2.75e-04 | 3591.0ms/step | 5.6 samples/s +Step 21020/44000 | Loss: 2.9762 | LR: 2.75e-04 | 1782.9ms/step | 11.2 samples/s +Step 21030/44000 | Loss: 2.9528 | LR: 2.74e-04 | 1783.8ms/step | 11.2 samples/s +Step 21040/44000 | Loss: 2.9749 | LR: 2.74e-04 | 1784.6ms/step | 11.2 samples/s +Step 21050/44000 | Loss: 2.9377 | LR: 2.74e-04 | 1784.3ms/step | 11.2 samples/s +Step 21060/44000 | Loss: 2.9398 | LR: 2.74e-04 | 1783.0ms/step | 11.2 samples/s +Step 21070/44000 | Loss: 2.9675 | LR: 2.74e-04 | 1784.6ms/step | 11.2 samples/s +Step 21080/44000 | Loss: 2.9737 | LR: 2.73e-04 | 1783.6ms/step | 11.2 samples/s +Step 21090/44000 | Loss: 3.0046 | LR: 2.73e-04 | 1784.6ms/step | 11.2 samples/s +Step 21100/44000 | Loss: 2.9257 | LR: 2.73e-04 | 1783.4ms/step | 11.2 samples/s +Step 21110/44000 | Loss: 2.9474 | LR: 2.73e-04 | 1784.4ms/step | 11.2 samples/s +Step 21120/44000 | Loss: 2.9563 | LR: 2.73e-04 | 1783.2ms/step | 11.2 samples/s +Step 21130/44000 | Loss: 2.9624 | LR: 2.73e-04 | 1783.9ms/step | 11.2 samples/s +Step 21140/44000 | Loss: 2.9489 | LR: 2.72e-04 | 1784.2ms/step | 11.2 samples/s +Step 21150/44000 | Loss: 2.9244 | LR: 2.72e-04 | 1785.5ms/step | 11.2 samples/s +Step 21160/44000 | Loss: 2.9642 | LR: 2.72e-04 | 1784.0ms/step | 11.2 samples/s +Step 21170/44000 | Loss: 2.9359 | LR: 2.72e-04 | 1783.6ms/step | 11.2 samples/s +Step 21180/44000 | Loss: 2.9125 | LR: 2.72e-04 | 1782.5ms/step | 11.2 samples/s +Step 21190/44000 | Loss: 2.9541 | LR: 2.71e-04 | 1783.9ms/step | 11.2 samples/s +Step 21200/44000 | Loss: 2.9062 | LR: 2.71e-04 | 1784.2ms/step | 11.2 samples/s +Step 21210/44000 | Loss: 2.9329 | LR: 2.71e-04 | 1784.6ms/step | 11.2 samples/s +Step 21220/44000 | Loss: 2.9098 | LR: 2.71e-04 | 1784.3ms/step | 11.2 samples/s +Step 21230/44000 | Loss: 2.9342 | LR: 2.71e-04 | 1785.1ms/step | 11.2 samples/s +Step 21240/44000 | Loss: 2.9464 | LR: 2.71e-04 | 1785.3ms/step | 11.2 samples/s +Step 21250/44000 | Loss: 2.9395 | LR: 2.70e-04 | 1786.7ms/step | 11.2 samples/s +Step 21260/44000 | Loss: 2.9534 | LR: 2.70e-04 | 1785.1ms/step | 11.2 samples/s +Step 21270/44000 | Loss: 2.9706 | LR: 2.70e-04 | 1784.1ms/step | 11.2 samples/s +Step 21280/44000 | Loss: 2.9487 | LR: 2.70e-04 | 1784.4ms/step | 11.2 samples/s +Step 21290/44000 | Loss: 2.9144 | LR: 2.70e-04 | 1783.1ms/step | 11.2 samples/s +Step 21300/44000 | Loss: 2.9522 | LR: 2.70e-04 | 1784.1ms/step | 11.2 samples/s +Step 21310/44000 | Loss: 2.9817 | LR: 2.69e-04 | 1783.2ms/step | 11.2 samples/s +Step 21320/44000 | Loss: 2.9461 | LR: 2.69e-04 | 1785.1ms/step | 11.2 samples/s +Step 21330/44000 | Loss: 2.9692 | LR: 2.69e-04 | 1782.8ms/step | 11.2 samples/s +Step 21340/44000 | Loss: 2.9745 | LR: 2.69e-04 | 1783.7ms/step | 11.2 samples/s +Step 21350/44000 | Loss: 2.9756 | LR: 2.69e-04 | 1783.4ms/step | 11.2 samples/s +Step 21360/44000 | Loss: 2.9168 | LR: 2.68e-04 | 1784.6ms/step | 11.2 samples/s +Step 21370/44000 | Loss: 2.9945 | LR: 2.68e-04 | 1784.1ms/step | 11.2 samples/s +Step 21380/44000 | Loss: 2.9426 | LR: 2.68e-04 | 1782.8ms/step | 11.2 samples/s +Step 21390/44000 | Loss: 2.9864 | LR: 2.68e-04 | 1782.4ms/step | 11.2 samples/s +Step 21400/44000 | Loss: 3.0251 | LR: 2.68e-04 | 1782.9ms/step | 11.2 samples/s +Step 21410/44000 | Loss: 2.9505 | LR: 2.68e-04 | 1783.3ms/step | 11.2 samples/s +Step 21420/44000 | Loss: 2.9903 | LR: 2.67e-04 | 1784.0ms/step | 11.2 samples/s +Step 21430/44000 | Loss: 2.9451 | LR: 2.67e-04 | 1783.5ms/step | 11.2 samples/s +Step 21440/44000 | Loss: 2.9615 | LR: 2.67e-04 | 1784.3ms/step | 11.2 samples/s +Step 21450/44000 | Loss: 2.9595 | LR: 2.67e-04 | 1783.8ms/step | 11.2 samples/s +Step 21460/44000 | Loss: 2.9730 | LR: 2.67e-04 | 1783.3ms/step | 11.2 samples/s +Step 21470/44000 | Loss: 2.9147 | LR: 2.66e-04 | 1784.8ms/step | 11.2 samples/s +Step 21480/44000 | Loss: 2.9483 | LR: 2.66e-04 | 1783.7ms/step | 11.2 samples/s +Step 21490/44000 | Loss: 2.9476 | LR: 2.66e-04 | 1783.9ms/step | 11.2 samples/s +Step 21500/44000 | Loss: 2.9503 | LR: 2.66e-04 | 1783.3ms/step | 11.2 samples/s +Step 21510/44000 | Loss: 2.9316 | LR: 2.66e-04 | 1783.8ms/step | 11.2 samples/s +Step 21520/44000 | Loss: 2.9278 | LR: 2.66e-04 | 1783.0ms/step | 11.2 samples/s +Step 21530/44000 | Loss: 2.9478 | LR: 2.65e-04 | 1784.2ms/step | 11.2 samples/s +Step 21540/44000 | Loss: 2.9481 | LR: 2.65e-04 | 1783.2ms/step | 11.2 samples/s +Step 21550/44000 | Loss: 2.9387 | LR: 2.65e-04 | 1784.6ms/step | 11.2 samples/s +Step 21560/44000 | Loss: 2.9281 | LR: 2.65e-04 | 1784.4ms/step | 11.2 samples/s +Step 21570/44000 | Loss: 2.9686 | LR: 2.65e-04 | 1783.5ms/step | 11.2 samples/s +Step 21580/44000 | Loss: 2.9503 | LR: 2.64e-04 | 1783.2ms/step | 11.2 samples/s +Step 21590/44000 | Loss: 2.9576 | LR: 2.64e-04 | 1784.0ms/step | 11.2 samples/s +Step 21600/44000 | Loss: 2.9265 | LR: 2.64e-04 | 1783.0ms/step | 11.2 samples/s +Step 21610/44000 | Loss: 2.9480 | LR: 2.64e-04 | 1783.4ms/step | 11.2 samples/s +Step 21620/44000 | Loss: 2.9793 | LR: 2.64e-04 | 1782.8ms/step | 11.2 samples/s +Step 21630/44000 | Loss: 2.9569 | LR: 2.64e-04 | 1782.6ms/step | 11.2 samples/s +Step 21640/44000 | Loss: 2.9689 | LR: 2.63e-04 | 1782.5ms/step | 11.2 samples/s +Step 21650/44000 | Loss: 2.9492 | LR: 2.63e-04 | 1782.6ms/step | 11.2 samples/s +Step 21660/44000 | Loss: 2.9155 | LR: 2.63e-04 | 1784.1ms/step | 11.2 samples/s +Step 21670/44000 | Loss: 2.9601 | LR: 2.63e-04 | 1783.5ms/step | 11.2 samples/s +Step 21680/44000 | Loss: 2.9736 | LR: 2.63e-04 | 1783.7ms/step | 11.2 samples/s +Step 21690/44000 | Loss: 2.9557 | LR: 2.62e-04 | 1783.2ms/step | 11.2 samples/s +Step 21700/44000 | Loss: 2.9446 | LR: 2.62e-04 | 1784.6ms/step | 11.2 samples/s +Step 21710/44000 | Loss: 2.9369 | LR: 2.62e-04 | 1784.9ms/step | 11.2 samples/s +Step 21720/44000 | Loss: 3.0092 | LR: 2.62e-04 | 1784.3ms/step | 11.2 samples/s +Step 21730/44000 | Loss: 2.9173 | LR: 2.62e-04 | 1784.2ms/step | 11.2 samples/s +Step 21740/44000 | Loss: 2.9280 | LR: 2.62e-04 | 1784.7ms/step | 11.2 samples/s +Step 21750/44000 | Loss: 2.9393 | LR: 2.61e-04 | 1784.1ms/step | 11.2 samples/s +Step 21760/44000 | Loss: 2.9398 | LR: 2.61e-04 | 1785.0ms/step | 11.2 samples/s +Step 21770/44000 | Loss: 2.9675 | LR: 2.61e-04 | 1785.3ms/step | 11.2 samples/s +Step 21780/44000 | Loss: 2.9116 | LR: 2.61e-04 | 1785.5ms/step | 11.2 samples/s +Step 21790/44000 | Loss: 2.9578 | LR: 2.61e-04 | 1784.4ms/step | 11.2 samples/s +Step 21800/44000 | Loss: 2.8972 | LR: 2.60e-04 | 1785.4ms/step | 11.2 samples/s +Step 21810/44000 | Loss: 2.9899 | LR: 2.60e-04 | 1784.3ms/step | 11.2 samples/s +Step 21820/44000 | Loss: 2.9564 | LR: 2.60e-04 | 1784.9ms/step | 11.2 samples/s +Step 21830/44000 | Loss: 2.9341 | LR: 2.60e-04 | 1784.9ms/step | 11.2 samples/s +Step 21840/44000 | Loss: 2.9223 | LR: 2.60e-04 | 1784.8ms/step | 11.2 samples/s +Step 21850/44000 | Loss: 2.9339 | LR: 2.60e-04 | 1784.1ms/step | 11.2 samples/s +Step 21860/44000 | Loss: 2.9618 | LR: 2.59e-04 | 1784.8ms/step | 11.2 samples/s +Step 21870/44000 | Loss: 2.9161 | LR: 2.59e-04 | 1785.1ms/step | 11.2 samples/s +Step 21880/44000 | Loss: 2.9389 | LR: 2.59e-04 | 1784.0ms/step | 11.2 samples/s +Step 21890/44000 | Loss: 2.9089 | LR: 2.59e-04 | 1784.0ms/step | 11.2 samples/s +Step 21900/44000 | Loss: 2.9641 | LR: 2.59e-04 | 1784.0ms/step | 11.2 samples/s +Step 21910/44000 | Loss: 2.9514 | LR: 2.58e-04 | 1784.2ms/step | 11.2 samples/s +Step 21920/44000 | Loss: 2.9250 | LR: 2.58e-04 | 1783.8ms/step | 11.2 samples/s +Step 21930/44000 | Loss: 2.9058 | LR: 2.58e-04 | 1784.1ms/step | 11.2 samples/s +Step 21940/44000 | Loss: 2.9537 | LR: 2.58e-04 | 1784.4ms/step | 11.2 samples/s +Step 21950/44000 | Loss: 2.9575 | LR: 2.58e-04 | 1785.0ms/step | 11.2 samples/s +Step 21960/44000 | Loss: 2.9640 | LR: 2.58e-04 | 1784.0ms/step | 11.2 samples/s +Step 21970/44000 | Loss: 2.9483 | LR: 2.57e-04 | 1784.1ms/step | 11.2 samples/s +Step 21980/44000 | Loss: 2.9637 | LR: 2.57e-04 | 1783.8ms/step | 11.2 samples/s +Step 21990/44000 | Loss: 2.9575 | LR: 2.57e-04 | 1783.3ms/step | 11.2 samples/s +Step 22000/44000 | Loss: 2.9742 | LR: 2.57e-04 | 1782.5ms/step | 11.2 samples/s +Validation at step 22000: Loss = 2.9299, Perplexity = 18.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-22000 +Step 22010/44000 | Loss: 2.9529 | LR: 2.57e-04 | 3598.0ms/step | 5.6 samples/s +Step 22020/44000 | Loss: 2.9475 | LR: 2.56e-04 | 1782.7ms/step | 11.2 samples/s +Step 22030/44000 | Loss: 2.9249 | LR: 2.56e-04 | 1783.3ms/step | 11.2 samples/s +Step 22040/44000 | Loss: 2.9355 | LR: 2.56e-04 | 1783.9ms/step | 11.2 samples/s +Step 22050/44000 | Loss: 2.9719 | LR: 2.56e-04 | 1783.8ms/step | 11.2 samples/s +Step 22060/44000 | Loss: 2.9282 | LR: 2.56e-04 | 1784.5ms/step | 11.2 samples/s +Step 22070/44000 | Loss: 2.9475 | LR: 2.56e-04 | 1783.7ms/step | 11.2 samples/s +Step 22080/44000 | Loss: 2.9194 | LR: 2.55e-04 | 1784.2ms/step | 11.2 samples/s +Step 22090/44000 | Loss: 2.9284 | LR: 2.55e-04 | 1784.9ms/step | 11.2 samples/s +Step 22100/44000 | Loss: 2.9592 | LR: 2.55e-04 | 1784.2ms/step | 11.2 samples/s +Step 22110/44000 | Loss: 2.9502 | LR: 2.55e-04 | 1784.3ms/step | 11.2 samples/s +Step 22120/44000 | Loss: 2.9309 | LR: 2.55e-04 | 1784.9ms/step | 11.2 samples/s +Step 22130/44000 | Loss: 2.9673 | LR: 2.54e-04 | 1785.1ms/step | 11.2 samples/s +Step 22140/44000 | Loss: 2.9595 | LR: 2.54e-04 | 1786.3ms/step | 11.2 samples/s +Step 22150/44000 | Loss: 2.9657 | LR: 2.54e-04 | 1785.5ms/step | 11.2 samples/s +Step 22160/44000 | Loss: 2.9435 | LR: 2.54e-04 | 1783.9ms/step | 11.2 samples/s +Step 22170/44000 | Loss: 2.9544 | LR: 2.54e-04 | 1784.4ms/step | 11.2 samples/s +Step 22180/44000 | Loss: 2.9317 | LR: 2.54e-04 | 1785.0ms/step | 11.2 samples/s +Step 22190/44000 | Loss: 2.9142 | LR: 2.53e-04 | 1785.3ms/step | 11.2 samples/s +Step 22200/44000 | Loss: 2.9581 | LR: 2.53e-04 | 1784.3ms/step | 11.2 samples/s +Step 22210/44000 | Loss: 2.8973 | LR: 2.53e-04 | 1784.9ms/step | 11.2 samples/s +Step 22220/44000 | Loss: 2.9223 | LR: 2.53e-04 | 1784.1ms/step | 11.2 samples/s +Step 22230/44000 | Loss: 2.9138 | LR: 2.53e-04 | 1784.9ms/step | 11.2 samples/s +Step 22240/44000 | Loss: 2.9175 | LR: 2.52e-04 | 1783.4ms/step | 11.2 samples/s +Step 22250/44000 | Loss: 2.9396 | LR: 2.52e-04 | 1784.4ms/step | 11.2 samples/s +Step 22260/44000 | Loss: 2.9605 | LR: 2.52e-04 | 1783.5ms/step | 11.2 samples/s +Step 22270/44000 | Loss: 2.9301 | LR: 2.52e-04 | 1785.1ms/step | 11.2 samples/s +Step 22280/44000 | Loss: 2.9358 | LR: 2.52e-04 | 1785.0ms/step | 11.2 samples/s +Step 22290/44000 | Loss: 2.9451 | LR: 2.52e-04 | 1785.5ms/step | 11.2 samples/s +Step 22300/44000 | Loss: 2.9260 | LR: 2.51e-04 | 1785.0ms/step | 11.2 samples/s +Step 22310/44000 | Loss: 2.9436 | LR: 2.51e-04 | 1785.3ms/step | 11.2 samples/s +Step 22320/44000 | Loss: 2.8736 | LR: 2.51e-04 | 1786.8ms/step | 11.2 samples/s +Step 22330/44000 | Loss: 2.9177 | LR: 2.51e-04 | 1786.5ms/step | 11.2 samples/s +Step 22340/44000 | Loss: 2.9895 | LR: 2.51e-04 | 1786.3ms/step | 11.2 samples/s +Step 22350/44000 | Loss: 2.9402 | LR: 2.50e-04 | 1786.7ms/step | 11.2 samples/s +Step 22360/44000 | Loss: 2.8859 | LR: 2.50e-04 | 1787.8ms/step | 11.2 samples/s +Step 22370/44000 | Loss: 2.9543 | LR: 2.50e-04 | 1784.5ms/step | 11.2 samples/s +Step 22380/44000 | Loss: 2.9313 | LR: 2.50e-04 | 1784.7ms/step | 11.2 samples/s +Step 22390/44000 | Loss: 2.9229 | LR: 2.50e-04 | 1785.1ms/step | 11.2 samples/s +Step 22400/44000 | Loss: 2.9187 | LR: 2.50e-04 | 1785.9ms/step | 11.2 samples/s +Step 22410/44000 | Loss: 2.9282 | LR: 2.49e-04 | 1783.9ms/step | 11.2 samples/s +Step 22420/44000 | Loss: 2.9421 | LR: 2.49e-04 | 1783.1ms/step | 11.2 samples/s +Step 22430/44000 | Loss: 2.9235 | LR: 2.49e-04 | 1783.7ms/step | 11.2 samples/s +Step 22440/44000 | Loss: 2.9476 | LR: 2.49e-04 | 1784.7ms/step | 11.2 samples/s +Step 22450/44000 | Loss: 2.9239 | LR: 2.49e-04 | 1785.0ms/step | 11.2 samples/s +Step 22460/44000 | Loss: 2.9868 | LR: 2.48e-04 | 1784.7ms/step | 11.2 samples/s +Step 22470/44000 | Loss: 2.9359 | LR: 2.48e-04 | 1785.0ms/step | 11.2 samples/s +Step 22480/44000 | Loss: 2.9391 | LR: 2.48e-04 | 1786.1ms/step | 11.2 samples/s +Step 22490/44000 | Loss: 2.9371 | LR: 2.48e-04 | 1784.2ms/step | 11.2 samples/s +Step 22500/44000 | Loss: 2.9759 | LR: 2.48e-04 | 1784.7ms/step | 11.2 samples/s +Step 22510/44000 | Loss: 2.9478 | LR: 2.48e-04 | 1784.6ms/step | 11.2 samples/s +Step 22520/44000 | Loss: 2.9331 | LR: 2.47e-04 | 1784.4ms/step | 11.2 samples/s +Step 22530/44000 | Loss: 2.9508 | LR: 2.47e-04 | 1784.3ms/step | 11.2 samples/s +Step 22540/44000 | Loss: 2.9616 | LR: 2.47e-04 | 1785.1ms/step | 11.2 samples/s +Step 22550/44000 | Loss: 2.9633 | LR: 2.47e-04 | 1786.1ms/step | 11.2 samples/s +Step 22560/44000 | Loss: 2.9919 | LR: 2.47e-04 | 1784.5ms/step | 11.2 samples/s +Step 22570/44000 | Loss: 2.9654 | LR: 2.46e-04 | 1786.2ms/step | 11.2 samples/s +Step 22580/44000 | Loss: 2.9429 | LR: 2.46e-04 | 1786.2ms/step | 11.2 samples/s +Step 22590/44000 | Loss: 3.0019 | LR: 2.46e-04 | 1788.8ms/step | 11.2 samples/s +Step 22600/44000 | Loss: 2.9478 | LR: 2.46e-04 | 1785.6ms/step | 11.2 samples/s +Step 22610/44000 | Loss: 2.9274 | LR: 2.46e-04 | 1785.8ms/step | 11.2 samples/s +Step 22620/44000 | Loss: 2.9556 | LR: 2.46e-04 | 1785.0ms/step | 11.2 samples/s +Step 22630/44000 | Loss: 2.9437 | LR: 2.45e-04 | 1785.8ms/step | 11.2 samples/s +Step 22640/44000 | Loss: 2.9011 | LR: 2.45e-04 | 1785.5ms/step | 11.2 samples/s +Step 22650/44000 | Loss: 2.9521 | LR: 2.45e-04 | 1787.7ms/step | 11.2 samples/s +Step 22660/44000 | Loss: 2.9198 | LR: 2.45e-04 | 1786.1ms/step | 11.2 samples/s +Step 22670/44000 | Loss: 2.9658 | LR: 2.45e-04 | 1785.7ms/step | 11.2 samples/s +Step 22680/44000 | Loss: 2.9353 | LR: 2.44e-04 | 1785.8ms/step | 11.2 samples/s +Step 22690/44000 | Loss: 2.9288 | LR: 2.44e-04 | 1785.4ms/step | 11.2 samples/s +Step 22700/44000 | Loss: 2.9867 | LR: 2.44e-04 | 1784.9ms/step | 11.2 samples/s +Step 22710/44000 | Loss: 2.9050 | LR: 2.44e-04 | 1784.9ms/step | 11.2 samples/s +Step 22720/44000 | Loss: 3.0138 | LR: 2.44e-04 | 1785.8ms/step | 11.2 samples/s +Step 22730/44000 | Loss: 2.9125 | LR: 2.44e-04 | 1786.2ms/step | 11.2 samples/s +Step 22740/44000 | Loss: 2.9592 | LR: 2.43e-04 | 1784.8ms/step | 11.2 samples/s +Step 22750/44000 | Loss: 2.9658 | LR: 2.43e-04 | 1785.7ms/step | 11.2 samples/s +Step 22760/44000 | Loss: 2.9370 | LR: 2.43e-04 | 1785.6ms/step | 11.2 samples/s +Step 22770/44000 | Loss: 2.9834 | LR: 2.43e-04 | 1784.4ms/step | 11.2 samples/s +Step 22780/44000 | Loss: 2.9316 | LR: 2.43e-04 | 1783.8ms/step | 11.2 samples/s +Step 22790/44000 | Loss: 2.9402 | LR: 2.42e-04 | 1783.8ms/step | 11.2 samples/s +Step 22800/44000 | Loss: 2.9216 | LR: 2.42e-04 | 1782.9ms/step | 11.2 samples/s +Step 22810/44000 | Loss: 2.9279 | LR: 2.42e-04 | 1782.6ms/step | 11.2 samples/s +Step 22820/44000 | Loss: 2.9985 | LR: 2.42e-04 | 1785.0ms/step | 11.2 samples/s +Step 22830/44000 | Loss: 2.9255 | LR: 2.42e-04 | 1784.7ms/step | 11.2 samples/s +Step 22840/44000 | Loss: 2.9068 | LR: 2.42e-04 | 1785.3ms/step | 11.2 samples/s +Step 22850/44000 | Loss: 2.9259 | LR: 2.41e-04 | 1785.6ms/step | 11.2 samples/s +Step 22860/44000 | Loss: 2.9009 | LR: 2.41e-04 | 1790.5ms/step | 11.2 samples/s +Step 22870/44000 | Loss: 2.9616 | LR: 2.41e-04 | 1788.9ms/step | 11.2 samples/s +Step 22880/44000 | Loss: 2.9366 | LR: 2.41e-04 | 1786.5ms/step | 11.2 samples/s +Step 22890/44000 | Loss: 2.9706 | LR: 2.41e-04 | 1784.9ms/step | 11.2 samples/s +Step 22900/44000 | Loss: 2.9555 | LR: 2.40e-04 | 1787.0ms/step | 11.2 samples/s +Step 22910/44000 | Loss: 2.9683 | LR: 2.40e-04 | 1786.4ms/step | 11.2 samples/s +Step 22920/44000 | Loss: 2.9725 | LR: 2.40e-04 | 1785.6ms/step | 11.2 samples/s +Step 22930/44000 | Loss: 2.9169 | LR: 2.40e-04 | 1784.8ms/step | 11.2 samples/s +Step 22940/44000 | Loss: 2.9324 | LR: 2.40e-04 | 1785.0ms/step | 11.2 samples/s +Step 22950/44000 | Loss: 2.9440 | LR: 2.40e-04 | 1785.0ms/step | 11.2 samples/s +Step 22960/44000 | Loss: 2.9002 | LR: 2.39e-04 | 1785.5ms/step | 11.2 samples/s +Step 22970/44000 | Loss: 2.8904 | LR: 2.39e-04 | 1786.8ms/step | 11.2 samples/s +Step 22980/44000 | Loss: 2.9205 | LR: 2.39e-04 | 1786.6ms/step | 11.2 samples/s +Step 22990/44000 | Loss: 2.9192 | LR: 2.39e-04 | 1785.3ms/step | 11.2 samples/s +Step 23000/44000 | Loss: 2.9285 | LR: 2.39e-04 | 1785.5ms/step | 11.2 samples/s +Validation at step 23000: Loss = 2.9212, Perplexity = 18.56 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-23000 +Step 23010/44000 | Loss: 2.9101 | LR: 2.38e-04 | 3602.2ms/step | 5.6 samples/s +Step 23020/44000 | Loss: 2.8920 | LR: 2.38e-04 | 1785.4ms/step | 11.2 samples/s +Step 23030/44000 | Loss: 2.8955 | LR: 2.38e-04 | 1784.7ms/step | 11.2 samples/s +Step 23040/44000 | Loss: 2.9346 | LR: 2.38e-04 | 1786.1ms/step | 11.2 samples/s +Step 23050/44000 | Loss: 2.9574 | LR: 2.38e-04 | 1786.2ms/step | 11.2 samples/s +Step 23060/44000 | Loss: 2.9709 | LR: 2.38e-04 | 1786.1ms/step | 11.2 samples/s +Step 23070/44000 | Loss: 2.9376 | LR: 2.37e-04 | 1786.7ms/step | 11.2 samples/s +Step 23080/44000 | Loss: 2.9551 | LR: 2.37e-04 | 1786.4ms/step | 11.2 samples/s +Step 23090/44000 | Loss: 2.9186 | LR: 2.37e-04 | 1786.1ms/step | 11.2 samples/s +Step 23100/44000 | Loss: 2.9119 | LR: 2.37e-04 | 1786.6ms/step | 11.2 samples/s +Step 23110/44000 | Loss: 2.9158 | LR: 2.37e-04 | 1787.6ms/step | 11.2 samples/s +Step 23120/44000 | Loss: 2.9540 | LR: 2.36e-04 | 1787.1ms/step | 11.2 samples/s +Step 23130/44000 | Loss: 2.9618 | LR: 2.36e-04 | 1788.0ms/step | 11.2 samples/s +Step 23140/44000 | Loss: 2.9273 | LR: 2.36e-04 | 1786.1ms/step | 11.2 samples/s +Step 23150/44000 | Loss: 2.9128 | LR: 2.36e-04 | 1786.9ms/step | 11.2 samples/s +Step 23160/44000 | Loss: 2.9295 | LR: 2.36e-04 | 1786.9ms/step | 11.2 samples/s +Step 23170/44000 | Loss: 2.9265 | LR: 2.36e-04 | 1788.0ms/step | 11.2 samples/s +Step 23180/44000 | Loss: 2.9517 | LR: 2.35e-04 | 1785.6ms/step | 11.2 samples/s +Step 23190/44000 | Loss: 2.9260 | LR: 2.35e-04 | 1784.9ms/step | 11.2 samples/s +Step 23200/44000 | Loss: 2.9469 | LR: 2.35e-04 | 1787.5ms/step | 11.2 samples/s +Step 23210/44000 | Loss: 2.9487 | LR: 2.35e-04 | 1787.4ms/step | 11.2 samples/s +Step 23220/44000 | Loss: 2.9608 | LR: 2.35e-04 | 1786.7ms/step | 11.2 samples/s +Step 23230/44000 | Loss: 2.9214 | LR: 2.34e-04 | 1786.5ms/step | 11.2 samples/s +Step 23240/44000 | Loss: 2.9871 | LR: 2.34e-04 | 1787.2ms/step | 11.2 samples/s +Step 23250/44000 | Loss: 2.9293 | LR: 2.34e-04 | 1787.2ms/step | 11.2 samples/s +Step 23260/44000 | Loss: 2.9407 | LR: 2.34e-04 | 1786.7ms/step | 11.2 samples/s +Step 23270/44000 | Loss: 2.9158 | LR: 2.34e-04 | 1787.1ms/step | 11.2 samples/s +Step 23280/44000 | Loss: 2.8751 | LR: 2.34e-04 | 1786.5ms/step | 11.2 samples/s +Step 23290/44000 | Loss: 2.8713 | LR: 2.33e-04 | 1786.0ms/step | 11.2 samples/s +Step 23300/44000 | Loss: 2.8930 | LR: 2.33e-04 | 1786.0ms/step | 11.2 samples/s +Step 23310/44000 | Loss: 2.9444 | LR: 2.33e-04 | 1787.5ms/step | 11.2 samples/s +Step 23320/44000 | Loss: 2.9697 | LR: 2.33e-04 | 1786.9ms/step | 11.2 samples/s +Step 23330/44000 | Loss: 2.9259 | LR: 2.33e-04 | 1786.5ms/step | 11.2 samples/s +Step 23340/44000 | Loss: 2.8984 | LR: 2.32e-04 | 1786.5ms/step | 11.2 samples/s +Step 23350/44000 | Loss: 2.9234 | LR: 2.32e-04 | 1787.3ms/step | 11.2 samples/s +Step 23360/44000 | Loss: 2.8995 | LR: 2.32e-04 | 1785.6ms/step | 11.2 samples/s +Step 23370/44000 | Loss: 2.9135 | LR: 2.32e-04 | 1787.1ms/step | 11.2 samples/s +Step 23380/44000 | Loss: 2.9199 | LR: 2.32e-04 | 1786.7ms/step | 11.2 samples/s +Step 23390/44000 | Loss: 2.9365 | LR: 2.32e-04 | 1787.1ms/step | 11.2 samples/s +Step 23400/44000 | Loss: 2.9408 | LR: 2.31e-04 | 1784.5ms/step | 11.2 samples/s +Step 23410/44000 | Loss: 2.9225 | LR: 2.31e-04 | 1785.2ms/step | 11.2 samples/s +Step 23420/44000 | Loss: 2.9611 | LR: 2.31e-04 | 1786.1ms/step | 11.2 samples/s +Step 23430/44000 | Loss: 2.9693 | LR: 2.31e-04 | 1787.0ms/step | 11.2 samples/s +Step 23440/44000 | Loss: 2.9681 | LR: 2.31e-04 | 1785.6ms/step | 11.2 samples/s +Step 23450/44000 | Loss: 2.9283 | LR: 2.30e-04 | 1786.4ms/step | 11.2 samples/s +Step 23460/44000 | Loss: 2.8931 | LR: 2.30e-04 | 1786.5ms/step | 11.2 samples/s +Step 23470/44000 | Loss: 2.8925 | LR: 2.30e-04 | 1786.6ms/step | 11.2 samples/s +Step 23480/44000 | Loss: 2.9322 | LR: 2.30e-04 | 1785.6ms/step | 11.2 samples/s +Step 23490/44000 | Loss: 2.9570 | LR: 2.30e-04 | 1786.6ms/step | 11.2 samples/s +Step 23500/44000 | Loss: 2.9036 | LR: 2.30e-04 | 1787.3ms/step | 11.2 samples/s +Step 23510/44000 | Loss: 2.9557 | LR: 2.29e-04 | 1787.3ms/step | 11.2 samples/s +Step 23520/44000 | Loss: 2.9466 | LR: 2.29e-04 | 1788.2ms/step | 11.2 samples/s +Step 23530/44000 | Loss: 2.9419 | LR: 2.29e-04 | 1787.3ms/step | 11.2 samples/s +Step 23540/44000 | Loss: 2.9325 | LR: 2.29e-04 | 1787.0ms/step | 11.2 samples/s +Step 23550/44000 | Loss: 2.9084 | LR: 2.29e-04 | 1786.5ms/step | 11.2 samples/s +Step 23560/44000 | Loss: 2.9623 | LR: 2.29e-04 | 1787.9ms/step | 11.2 samples/s +Step 23570/44000 | Loss: 2.9347 | LR: 2.28e-04 | 1786.7ms/step | 11.2 samples/s +Step 23580/44000 | Loss: 2.9231 | LR: 2.28e-04 | 1787.4ms/step | 11.2 samples/s +Step 23590/44000 | Loss: 2.9382 | LR: 2.28e-04 | 1786.6ms/step | 11.2 samples/s +Step 23600/44000 | Loss: 2.9333 | LR: 2.28e-04 | 1786.1ms/step | 11.2 samples/s +Step 23610/44000 | Loss: 2.9002 | LR: 2.28e-04 | 1786.1ms/step | 11.2 samples/s +Step 23620/44000 | Loss: 2.9147 | LR: 2.27e-04 | 1786.8ms/step | 11.2 samples/s +Step 23630/44000 | Loss: 2.9571 | LR: 2.27e-04 | 1786.0ms/step | 11.2 samples/s +Step 23640/44000 | Loss: 2.9667 | LR: 2.27e-04 | 1788.0ms/step | 11.2 samples/s +Step 23650/44000 | Loss: 2.9068 | LR: 2.27e-04 | 1787.4ms/step | 11.2 samples/s +Step 23660/44000 | Loss: 2.9729 | LR: 2.27e-04 | 1789.0ms/step | 11.2 samples/s +Step 23670/44000 | Loss: 2.9478 | LR: 2.27e-04 | 1787.2ms/step | 11.2 samples/s +Step 23680/44000 | Loss: 2.9451 | LR: 2.26e-04 | 1787.1ms/step | 11.2 samples/s +Step 23690/44000 | Loss: 2.9159 | LR: 2.26e-04 | 1787.7ms/step | 11.2 samples/s +Step 23700/44000 | Loss: 2.9361 | LR: 2.26e-04 | 1788.2ms/step | 11.2 samples/s +Step 23710/44000 | Loss: 2.9220 | LR: 2.26e-04 | 1787.5ms/step | 11.2 samples/s +Step 23720/44000 | Loss: 2.9507 | LR: 2.26e-04 | 1786.4ms/step | 11.2 samples/s +Step 23730/44000 | Loss: 2.9063 | LR: 2.25e-04 | 1785.3ms/step | 11.2 samples/s +Step 23740/44000 | Loss: 2.9326 | LR: 2.25e-04 | 1783.9ms/step | 11.2 samples/s +Step 23750/44000 | Loss: 2.9249 | LR: 2.25e-04 | 1786.6ms/step | 11.2 samples/s +Step 23760/44000 | Loss: 2.9178 | LR: 2.25e-04 | 1786.6ms/step | 11.2 samples/s +Step 23770/44000 | Loss: 2.9238 | LR: 2.25e-04 | 1787.6ms/step | 11.2 samples/s +Step 23780/44000 | Loss: 2.9490 | LR: 2.25e-04 | 1787.0ms/step | 11.2 samples/s +Step 23790/44000 | Loss: 2.9312 | LR: 2.24e-04 | 1787.2ms/step | 11.2 samples/s +Step 23800/44000 | Loss: 2.9404 | LR: 2.24e-04 | 1786.2ms/step | 11.2 samples/s +Step 23810/44000 | Loss: 2.9420 | LR: 2.24e-04 | 1786.6ms/step | 11.2 samples/s +Step 23820/44000 | Loss: 2.9315 | LR: 2.24e-04 | 1785.1ms/step | 11.2 samples/s +Step 23830/44000 | Loss: 2.9264 | LR: 2.24e-04 | 1784.9ms/step | 11.2 samples/s +Step 23840/44000 | Loss: 2.9111 | LR: 2.23e-04 | 1784.6ms/step | 11.2 samples/s +Step 23850/44000 | Loss: 2.9539 | LR: 2.23e-04 | 1785.7ms/step | 11.2 samples/s +Step 23860/44000 | Loss: 2.9504 | LR: 2.23e-04 | 1786.9ms/step | 11.2 samples/s +Step 23870/44000 | Loss: 2.9326 | LR: 2.23e-04 | 1786.3ms/step | 11.2 samples/s +Step 23880/44000 | Loss: 2.9230 | LR: 2.23e-04 | 1786.0ms/step | 11.2 samples/s +Step 23890/44000 | Loss: 2.9233 | LR: 2.23e-04 | 1786.8ms/step | 11.2 samples/s diff --git a/epfl_final/train_logs/ropectl_a1p0_h100_42_43_resume/seed42.20260820T194037Z.log b/epfl_final/train_logs/ropectl_a1p0_h100_42_43_resume/seed42.20260820T194037Z.log new file mode 100644 index 0000000000000000000000000000000000000000..a8377a2f784ad6fb2969e318c1e42a8403f27e8d --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_h100_42_43_resume/seed42.20260820T194037Z.log @@ -0,0 +1,2560 @@ +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-22000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-22000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed42", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-22000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-22000... +Preserving checkpoint LR state for exact resume: current=[0.0002568089685768038], schedule_base=[0.0005] +Resumed from checkpoint at step 22000 (epoch 0, micro_step 195000) +Resuming training from step 22000 +Resuming wandb run jqstnvej +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run jqstnvej +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_194209-jqstnvej +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed42 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/jqstnvej +Starting training... +Validation at step 22000: Loss = 2.9576, Perplexity = 19.25 +wandb: WARNING Tried to log to step 22000 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 22001: dense (dense step interval) +Step 22010/44000 | Loss: 2.9583 | LR: 2.57e-04 | 47512.8ms/step | 0.4 samples/s +wandb: WARNING Tried to log to step 22010 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22020/44000 | Loss: 2.9671 | LR: 2.56e-04 | 1369.4ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 22020 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22030/44000 | Loss: 2.9344 | LR: 2.56e-04 | 1365.5ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 22030 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22040/44000 | Loss: 2.9191 | LR: 2.56e-04 | 1366.6ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 22040 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22050/44000 | Loss: 2.9365 | LR: 2.56e-04 | 1371.7ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 22050 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22060/44000 | Loss: 2.9609 | LR: 2.56e-04 | 1373.3ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 22060 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22070/44000 | Loss: 2.8936 | LR: 2.56e-04 | 1377.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22070 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22080/44000 | Loss: 2.9269 | LR: 2.55e-04 | 1380.4ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22080 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22090/44000 | Loss: 2.9281 | LR: 2.55e-04 | 1383.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22090 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22100/44000 | Loss: 2.9367 | LR: 2.55e-04 | 1384.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22100 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22110/44000 | Loss: 2.9426 | LR: 2.55e-04 | 1381.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22110 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22120/44000 | Loss: 2.9360 | LR: 2.55e-04 | 1382.6ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22120 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22130/44000 | Loss: 2.9396 | LR: 2.54e-04 | 1382.4ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22130 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22140/44000 | Loss: 2.9326 | LR: 2.54e-04 | 1383.3ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22140 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22150/44000 | Loss: 2.9480 | LR: 2.54e-04 | 1382.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22150 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22160/44000 | Loss: 2.9249 | LR: 2.54e-04 | 1385.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22160 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22170/44000 | Loss: 2.9553 | LR: 2.54e-04 | 1384.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22170 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22180/44000 | Loss: 2.9451 | LR: 2.54e-04 | 1378.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22180 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22190/44000 | Loss: 2.9241 | LR: 2.53e-04 | 1380.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22190 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22200/44000 | Loss: 2.9436 | LR: 2.53e-04 | 1382.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22200 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22210/44000 | Loss: 2.9428 | LR: 2.53e-04 | 1383.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22210 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22220/44000 | Loss: 2.9152 | LR: 2.53e-04 | 1383.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22220 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22230/44000 | Loss: 2.9386 | LR: 2.53e-04 | 1385.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22230 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22240/44000 | Loss: 2.9109 | LR: 2.52e-04 | 1385.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22240 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22250/44000 | Loss: 2.9156 | LR: 2.52e-04 | 1381.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22250 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22260/44000 | Loss: 2.9508 | LR: 2.52e-04 | 1383.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22260 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22270/44000 | Loss: 2.9014 | LR: 2.52e-04 | 1382.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22270 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22280/44000 | Loss: 2.9147 | LR: 2.52e-04 | 1384.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22280 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22290/44000 | Loss: 2.9359 | LR: 2.52e-04 | 1385.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22290 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22300/44000 | Loss: 2.9341 | LR: 2.51e-04 | 1384.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22300 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22310/44000 | Loss: 2.9306 | LR: 2.51e-04 | 1385.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22310 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22320/44000 | Loss: 2.9498 | LR: 2.51e-04 | 1385.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22320 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22330/44000 | Loss: 2.9246 | LR: 2.51e-04 | 1385.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22330 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22340/44000 | Loss: 2.9240 | LR: 2.51e-04 | 1384.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22340 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22350/44000 | Loss: 2.9454 | LR: 2.50e-04 | 1386.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22350 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22360/44000 | Loss: 2.9321 | LR: 2.50e-04 | 1385.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22360 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22370/44000 | Loss: 2.9268 | LR: 2.50e-04 | 1385.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22370 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22380/44000 | Loss: 2.9323 | LR: 2.50e-04 | 1384.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22380 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22390/44000 | Loss: 2.9364 | LR: 2.50e-04 | 1384.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22390 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22400/44000 | Loss: 2.9259 | LR: 2.50e-04 | 1387.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22400 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22410/44000 | Loss: 2.9238 | LR: 2.49e-04 | 1384.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22410 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22420/44000 | Loss: 2.9443 | LR: 2.49e-04 | 1387.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22420 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22430/44000 | Loss: 2.9635 | LR: 2.49e-04 | 1384.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22430 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22440/44000 | Loss: 2.9143 | LR: 2.49e-04 | 1383.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22440 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22450/44000 | Loss: 2.9323 | LR: 2.49e-04 | 1383.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22450 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22460/44000 | Loss: 2.9562 | LR: 2.48e-04 | 1380.6ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22460 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22470/44000 | Loss: 2.9237 | LR: 2.48e-04 | 1382.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22470 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22480/44000 | Loss: 2.9097 | LR: 2.48e-04 | 1383.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22480 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22490/44000 | Loss: 2.9493 | LR: 2.48e-04 | 1383.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22490 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22500/44000 | Loss: 2.9419 | LR: 2.48e-04 | 1382.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22500 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22510/44000 | Loss: 2.9238 | LR: 2.48e-04 | 1385.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22510 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22520/44000 | Loss: 2.9178 | LR: 2.47e-04 | 1384.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22520 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22530/44000 | Loss: 2.9471 | LR: 2.47e-04 | 1382.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22530 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22540/44000 | Loss: 2.9163 | LR: 2.47e-04 | 1379.4ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22540 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22550/44000 | Loss: 2.9470 | LR: 2.47e-04 | 1381.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22550 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22560/44000 | Loss: 2.9108 | LR: 2.47e-04 | 1382.4ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22560 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22570/44000 | Loss: 2.8929 | LR: 2.46e-04 | 1384.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22570 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22580/44000 | Loss: 2.9301 | LR: 2.46e-04 | 1384.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22580 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22590/44000 | Loss: 2.9370 | LR: 2.46e-04 | 1384.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22590 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22600/44000 | Loss: 2.9269 | LR: 2.46e-04 | 1384.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22600 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22610/44000 | Loss: 2.9412 | LR: 2.46e-04 | 1385.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22610 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22620/44000 | Loss: 2.9180 | LR: 2.46e-04 | 1385.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22620 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22630/44000 | Loss: 2.9418 | LR: 2.45e-04 | 1384.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22630 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22640/44000 | Loss: 2.9018 | LR: 2.45e-04 | 1385.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22640 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22650/44000 | Loss: 2.9335 | LR: 2.45e-04 | 1383.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22650 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22660/44000 | Loss: 2.9161 | LR: 2.45e-04 | 1383.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22660 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22670/44000 | Loss: 2.9430 | LR: 2.45e-04 | 1384.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22670 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22680/44000 | Loss: 2.9121 | LR: 2.44e-04 | 1387.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22680 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22690/44000 | Loss: 2.9462 | LR: 2.44e-04 | 1386.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22690 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22700/44000 | Loss: 2.9138 | LR: 2.44e-04 | 1387.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22700 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22710/44000 | Loss: 2.9539 | LR: 2.44e-04 | 1386.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22710 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22720/44000 | Loss: 2.9345 | LR: 2.44e-04 | 1384.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22720 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22730/44000 | Loss: 2.9517 | LR: 2.44e-04 | 1385.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22730 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22740/44000 | Loss: 2.9236 | LR: 2.43e-04 | 1385.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22740 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22750/44000 | Loss: 2.9150 | LR: 2.43e-04 | 1383.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22750 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22760/44000 | Loss: 2.9197 | LR: 2.43e-04 | 1381.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22760 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22770/44000 | Loss: 2.9197 | LR: 2.43e-04 | 1379.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22770 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22780/44000 | Loss: 2.9143 | LR: 2.43e-04 | 1382.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22780 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22790/44000 | Loss: 2.9194 | LR: 2.42e-04 | 1383.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22790 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22800/44000 | Loss: 2.8953 | LR: 2.42e-04 | 1384.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22800 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22810/44000 | Loss: 2.9446 | LR: 2.42e-04 | 1384.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22810 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22820/44000 | Loss: 2.9264 | LR: 2.42e-04 | 1382.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22820 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22830/44000 | Loss: 2.9297 | LR: 2.42e-04 | 1385.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22830 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22840/44000 | Loss: 2.9165 | LR: 2.42e-04 | 1382.4ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22840 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22850/44000 | Loss: 2.9333 | LR: 2.41e-04 | 1383.2ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22850 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22860/44000 | Loss: 2.9491 | LR: 2.41e-04 | 1384.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22860 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22870/44000 | Loss: 2.8830 | LR: 2.41e-04 | 1384.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22870 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22880/44000 | Loss: 2.9108 | LR: 2.41e-04 | 1382.2ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22880 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22890/44000 | Loss: 2.9332 | LR: 2.41e-04 | 1386.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22890 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22900/44000 | Loss: 2.9537 | LR: 2.40e-04 | 1387.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22900 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22910/44000 | Loss: 2.9066 | LR: 2.40e-04 | 1384.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 22910 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22920/44000 | Loss: 2.9458 | LR: 2.40e-04 | 1383.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 22920 that is less than the current step 22921. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 22930/44000 | Loss: 2.9055 | LR: 2.40e-04 | 1385.5ms/step | 14.4 samples/s +Step 22940/44000 | Loss: 2.9471 | LR: 2.40e-04 | 1386.2ms/step | 14.4 samples/s +Step 22950/44000 | Loss: 2.9295 | LR: 2.40e-04 | 1384.5ms/step | 14.4 samples/s +Step 22960/44000 | Loss: 2.9438 | LR: 2.39e-04 | 1385.1ms/step | 14.4 samples/s +Step 22970/44000 | Loss: 2.9484 | LR: 2.39e-04 | 1386.3ms/step | 14.4 samples/s +Step 22980/44000 | Loss: 2.9374 | LR: 2.39e-04 | 1384.8ms/step | 14.4 samples/s +Step 22990/44000 | Loss: 2.9610 | LR: 2.39e-04 | 1385.8ms/step | 14.4 samples/s +Step 23000/44000 | Loss: 2.9141 | LR: 2.39e-04 | 1385.7ms/step | 14.4 samples/s +Validation at step 23000: Loss = 2.9475, Perplexity = 19.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-23000 +Step 23010/44000 | Loss: 2.9340 | LR: 2.38e-04 | 3882.3ms/step | 5.2 samples/s +Step 23020/44000 | Loss: 2.9493 | LR: 2.38e-04 | 1383.0ms/step | 14.5 samples/s +Step 23030/44000 | Loss: 2.9375 | LR: 2.38e-04 | 1383.7ms/step | 14.5 samples/s +Step 23040/44000 | Loss: 2.9192 | LR: 2.38e-04 | 1381.6ms/step | 14.5 samples/s +Step 23050/44000 | Loss: 2.9307 | LR: 2.38e-04 | 1383.8ms/step | 14.5 samples/s +Step 23060/44000 | Loss: 2.9251 | LR: 2.38e-04 | 1387.8ms/step | 14.4 samples/s +Step 23070/44000 | Loss: 2.9253 | LR: 2.37e-04 | 1387.4ms/step | 14.4 samples/s +Step 23080/44000 | Loss: 2.9341 | LR: 2.37e-04 | 1389.0ms/step | 14.4 samples/s +Step 23090/44000 | Loss: 2.9195 | LR: 2.37e-04 | 1390.6ms/step | 14.4 samples/s +Step 23100/44000 | Loss: 2.9213 | LR: 2.37e-04 | 1389.5ms/step | 14.4 samples/s +Step 23110/44000 | Loss: 2.9005 | LR: 2.37e-04 | 1388.8ms/step | 14.4 samples/s +Step 23120/44000 | Loss: 2.8915 | LR: 2.36e-04 | 1388.5ms/step | 14.4 samples/s +Step 23130/44000 | Loss: 2.9503 | LR: 2.36e-04 | 1388.8ms/step | 14.4 samples/s +Step 23140/44000 | Loss: 2.9416 | LR: 2.36e-04 | 1389.7ms/step | 14.4 samples/s +Step 23150/44000 | Loss: 2.9287 | LR: 2.36e-04 | 1390.2ms/step | 14.4 samples/s +Step 23160/44000 | Loss: 2.9524 | LR: 2.36e-04 | 1389.9ms/step | 14.4 samples/s +Step 23170/44000 | Loss: 2.9237 | LR: 2.36e-04 | 1390.3ms/step | 14.4 samples/s +Step 23180/44000 | Loss: 2.9246 | LR: 2.35e-04 | 1388.7ms/step | 14.4 samples/s +Step 23190/44000 | Loss: 2.9266 | LR: 2.35e-04 | 1391.2ms/step | 14.4 samples/s +Step 23200/44000 | Loss: 2.9431 | LR: 2.35e-04 | 1389.9ms/step | 14.4 samples/s +Step 23210/44000 | Loss: 2.9405 | LR: 2.35e-04 | 1391.1ms/step | 14.4 samples/s +Step 23220/44000 | Loss: 2.9139 | LR: 2.35e-04 | 1388.8ms/step | 14.4 samples/s +Step 23230/44000 | Loss: 2.9178 | LR: 2.34e-04 | 1390.2ms/step | 14.4 samples/s +Step 23240/44000 | Loss: 2.9313 | LR: 2.34e-04 | 1387.4ms/step | 14.4 samples/s +Step 23250/44000 | Loss: 2.9137 | LR: 2.34e-04 | 1386.7ms/step | 14.4 samples/s +Step 23260/44000 | Loss: 2.9505 | LR: 2.34e-04 | 1387.1ms/step | 14.4 samples/s +Step 23270/44000 | Loss: 2.9194 | LR: 2.34e-04 | 1390.1ms/step | 14.4 samples/s +Step 23280/44000 | Loss: 2.9090 | LR: 2.34e-04 | 1388.4ms/step | 14.4 samples/s +Step 23290/44000 | Loss: 2.9271 | LR: 2.33e-04 | 1392.4ms/step | 14.4 samples/s +Step 23300/44000 | Loss: 2.9216 | LR: 2.33e-04 | 1387.8ms/step | 14.4 samples/s +Step 23310/44000 | Loss: 2.9164 | LR: 2.33e-04 | 1385.9ms/step | 14.4 samples/s +Step 23320/44000 | Loss: 2.9133 | LR: 2.33e-04 | 1387.0ms/step | 14.4 samples/s +Step 23330/44000 | Loss: 2.9350 | LR: 2.33e-04 | 1387.3ms/step | 14.4 samples/s +Step 23340/44000 | Loss: 2.9132 | LR: 2.32e-04 | 1388.2ms/step | 14.4 samples/s +Step 23350/44000 | Loss: 2.9419 | LR: 2.32e-04 | 1388.8ms/step | 14.4 samples/s +Step 23360/44000 | Loss: 2.9110 | LR: 2.32e-04 | 1388.0ms/step | 14.4 samples/s +Step 23370/44000 | Loss: 2.9118 | LR: 2.32e-04 | 1388.9ms/step | 14.4 samples/s +Step 23380/44000 | Loss: 2.9236 | LR: 2.32e-04 | 1384.5ms/step | 14.4 samples/s +Step 23390/44000 | Loss: 2.9466 | LR: 2.32e-04 | 1381.6ms/step | 14.5 samples/s +Step 23400/44000 | Loss: 2.9121 | LR: 2.31e-04 | 1382.8ms/step | 14.5 samples/s +Step 23410/44000 | Loss: 2.9515 | LR: 2.31e-04 | 1385.1ms/step | 14.4 samples/s +Step 23420/44000 | Loss: 2.8933 | LR: 2.31e-04 | 1388.1ms/step | 14.4 samples/s +Step 23430/44000 | Loss: 2.9176 | LR: 2.31e-04 | 1387.7ms/step | 14.4 samples/s +Step 23440/44000 | Loss: 2.9669 | LR: 2.31e-04 | 1387.0ms/step | 14.4 samples/s +Step 23450/44000 | Loss: 2.9279 | LR: 2.30e-04 | 1389.0ms/step | 14.4 samples/s +Step 23460/44000 | Loss: 2.9332 | LR: 2.30e-04 | 1387.6ms/step | 14.4 samples/s +Step 23470/44000 | Loss: 2.9133 | LR: 2.30e-04 | 1386.7ms/step | 14.4 samples/s +Step 23480/44000 | Loss: 2.9361 | LR: 2.30e-04 | 1387.0ms/step | 14.4 samples/s +Step 23490/44000 | Loss: 2.9488 | LR: 2.30e-04 | 1387.2ms/step | 14.4 samples/s +Step 23500/44000 | Loss: 2.9481 | LR: 2.30e-04 | 1387.1ms/step | 14.4 samples/s +Step 23510/44000 | Loss: 2.9310 | LR: 2.29e-04 | 1387.9ms/step | 14.4 samples/s +Step 23520/44000 | Loss: 2.9234 | LR: 2.29e-04 | 1388.5ms/step | 14.4 samples/s +Step 23530/44000 | Loss: 2.9317 | LR: 2.29e-04 | 1386.2ms/step | 14.4 samples/s +Step 23540/44000 | Loss: 2.8965 | LR: 2.29e-04 | 1391.8ms/step | 14.4 samples/s +Step 23550/44000 | Loss: 2.9559 | LR: 2.29e-04 | 1389.3ms/step | 14.4 samples/s +Step 23560/44000 | Loss: 2.9219 | LR: 2.29e-04 | 1390.8ms/step | 14.4 samples/s +Step 23570/44000 | Loss: 2.9225 | LR: 2.28e-04 | 1388.7ms/step | 14.4 samples/s +Step 23580/44000 | Loss: 2.9327 | LR: 2.28e-04 | 1390.3ms/step | 14.4 samples/s +Step 23590/44000 | Loss: 2.9100 | LR: 2.28e-04 | 1389.1ms/step | 14.4 samples/s +Step 23600/44000 | Loss: 2.9045 | LR: 2.28e-04 | 1390.1ms/step | 14.4 samples/s +Step 23610/44000 | Loss: 2.9470 | LR: 2.28e-04 | 1388.8ms/step | 14.4 samples/s +Step 23620/44000 | Loss: 2.9304 | LR: 2.27e-04 | 1389.3ms/step | 14.4 samples/s +Step 23630/44000 | Loss: 2.9065 | LR: 2.27e-04 | 1389.8ms/step | 14.4 samples/s +Step 23640/44000 | Loss: 2.9457 | LR: 2.27e-04 | 1388.8ms/step | 14.4 samples/s +Step 23650/44000 | Loss: 2.9063 | LR: 2.27e-04 | 1387.7ms/step | 14.4 samples/s +Step 23660/44000 | Loss: 2.9299 | LR: 2.27e-04 | 1385.5ms/step | 14.4 samples/s +Step 23670/44000 | Loss: 2.9211 | LR: 2.27e-04 | 1386.7ms/step | 14.4 samples/s +Step 23680/44000 | Loss: 2.9071 | LR: 2.26e-04 | 1387.2ms/step | 14.4 samples/s +Step 23690/44000 | Loss: 2.9381 | LR: 2.26e-04 | 1390.1ms/step | 14.4 samples/s +Step 23700/44000 | Loss: 2.8940 | LR: 2.26e-04 | 1390.0ms/step | 14.4 samples/s +Step 23710/44000 | Loss: 2.9449 | LR: 2.26e-04 | 1389.4ms/step | 14.4 samples/s +Step 23720/44000 | Loss: 2.9508 | LR: 2.26e-04 | 1390.7ms/step | 14.4 samples/s +Step 23730/44000 | Loss: 2.9316 | LR: 2.25e-04 | 1390.0ms/step | 14.4 samples/s +Step 23740/44000 | Loss: 2.9047 | LR: 2.25e-04 | 1382.4ms/step | 14.5 samples/s +Step 23750/44000 | Loss: 2.9200 | LR: 2.25e-04 | 1381.9ms/step | 14.5 samples/s +Step 23760/44000 | Loss: 2.9388 | LR: 2.25e-04 | 1383.3ms/step | 14.5 samples/s +Step 23770/44000 | Loss: 2.8975 | LR: 2.25e-04 | 1384.6ms/step | 14.4 samples/s +Step 23780/44000 | Loss: 2.9481 | LR: 2.25e-04 | 1385.4ms/step | 14.4 samples/s +Step 23790/44000 | Loss: 2.9033 | LR: 2.24e-04 | 1386.2ms/step | 14.4 samples/s +Step 23800/44000 | Loss: 2.9141 | LR: 2.24e-04 | 1386.2ms/step | 14.4 samples/s +Step 23810/44000 | Loss: 2.9210 | LR: 2.24e-04 | 1387.1ms/step | 14.4 samples/s +Step 23820/44000 | Loss: 2.9363 | LR: 2.24e-04 | 1386.3ms/step | 14.4 samples/s +Step 23830/44000 | Loss: 2.9153 | LR: 2.24e-04 | 1387.3ms/step | 14.4 samples/s +Step 23840/44000 | Loss: 2.9070 | LR: 2.23e-04 | 1389.4ms/step | 14.4 samples/s +Step 23850/44000 | Loss: 2.8999 | LR: 2.23e-04 | 1390.0ms/step | 14.4 samples/s +Step 23860/44000 | Loss: 2.9243 | LR: 2.23e-04 | 1389.4ms/step | 14.4 samples/s +Step 23870/44000 | Loss: 2.9140 | LR: 2.23e-04 | 1390.6ms/step | 14.4 samples/s +Step 23880/44000 | Loss: 2.9351 | LR: 2.23e-04 | 1390.3ms/step | 14.4 samples/s +Step 23890/44000 | Loss: 2.9186 | LR: 2.23e-04 | 1389.6ms/step | 14.4 samples/s +Step 23900/44000 | Loss: 2.9155 | LR: 2.22e-04 | 1390.3ms/step | 14.4 samples/s +Step 23910/44000 | Loss: 2.9082 | LR: 2.22e-04 | 1389.9ms/step | 14.4 samples/s +Step 23920/44000 | Loss: 2.9228 | LR: 2.22e-04 | 1389.8ms/step | 14.4 samples/s +Step 23930/44000 | Loss: 2.9156 | LR: 2.22e-04 | 1391.0ms/step | 14.4 samples/s +Step 23940/44000 | Loss: 2.9283 | LR: 2.22e-04 | 1389.2ms/step | 14.4 samples/s +Step 23950/44000 | Loss: 2.8991 | LR: 2.21e-04 | 1391.6ms/step | 14.4 samples/s +Step 23960/44000 | Loss: 2.8779 | LR: 2.21e-04 | 1389.5ms/step | 14.4 samples/s +Step 23970/44000 | Loss: 2.9447 | LR: 2.21e-04 | 1387.6ms/step | 14.4 samples/s +Step 23980/44000 | Loss: 2.9814 | LR: 2.21e-04 | 1385.7ms/step | 14.4 samples/s +Step 23990/44000 | Loss: 2.9057 | LR: 2.21e-04 | 1387.5ms/step | 14.4 samples/s +Step 24000/44000 | Loss: 2.9085 | LR: 2.21e-04 | 1386.0ms/step | 14.4 samples/s +Validation at step 24000: Loss = 2.9391, Perplexity = 18.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-24000 +Step 24010/44000 | Loss: 2.9250 | LR: 2.20e-04 | 3870.1ms/step | 5.2 samples/s +Step 24020/44000 | Loss: 2.8959 | LR: 2.20e-04 | 1384.1ms/step | 14.4 samples/s +Step 24030/44000 | Loss: 2.9137 | LR: 2.20e-04 | 1385.3ms/step | 14.4 samples/s +Step 24040/44000 | Loss: 2.9466 | LR: 2.20e-04 | 1385.9ms/step | 14.4 samples/s +Step 24050/44000 | Loss: 2.9173 | LR: 2.20e-04 | 1387.6ms/step | 14.4 samples/s +Step 24060/44000 | Loss: 2.9159 | LR: 2.19e-04 | 1386.9ms/step | 14.4 samples/s +Step 24070/44000 | Loss: 2.9349 | LR: 2.19e-04 | 1386.1ms/step | 14.4 samples/s +Step 24080/44000 | Loss: 2.9244 | LR: 2.19e-04 | 1385.6ms/step | 14.4 samples/s +Step 24090/44000 | Loss: 2.9038 | LR: 2.19e-04 | 1386.5ms/step | 14.4 samples/s +Step 24100/44000 | Loss: 2.8829 | LR: 2.19e-04 | 1386.1ms/step | 14.4 samples/s +Step 24110/44000 | Loss: 2.8918 | LR: 2.19e-04 | 1388.3ms/step | 14.4 samples/s +Step 24120/44000 | Loss: 2.9218 | LR: 2.18e-04 | 1386.9ms/step | 14.4 samples/s +Step 24130/44000 | Loss: 2.8924 | LR: 2.18e-04 | 1388.7ms/step | 14.4 samples/s +Step 24140/44000 | Loss: 2.9274 | LR: 2.18e-04 | 1389.9ms/step | 14.4 samples/s +Step 24150/44000 | Loss: 2.9251 | LR: 2.18e-04 | 1383.7ms/step | 14.5 samples/s +Step 24160/44000 | Loss: 2.8993 | LR: 2.18e-04 | 1382.6ms/step | 14.5 samples/s +Step 24170/44000 | Loss: 2.9511 | LR: 2.17e-04 | 1386.1ms/step | 14.4 samples/s +Step 24180/44000 | Loss: 2.9399 | LR: 2.17e-04 | 1386.0ms/step | 14.4 samples/s +Step 24190/44000 | Loss: 2.9200 | LR: 2.17e-04 | 1387.1ms/step | 14.4 samples/s +Step 24200/44000 | Loss: 2.9324 | LR: 2.17e-04 | 1386.9ms/step | 14.4 samples/s +Step 24210/44000 | Loss: 2.9246 | LR: 2.17e-04 | 1387.9ms/step | 14.4 samples/s +Step 24220/44000 | Loss: 2.9408 | LR: 2.17e-04 | 1386.7ms/step | 14.4 samples/s +Step 24230/44000 | Loss: 2.9295 | LR: 2.16e-04 | 1389.3ms/step | 14.4 samples/s +Step 24240/44000 | Loss: 2.9220 | LR: 2.16e-04 | 1387.1ms/step | 14.4 samples/s +Step 24250/44000 | Loss: 2.9145 | LR: 2.16e-04 | 1386.9ms/step | 14.4 samples/s +Step 24260/44000 | Loss: 2.9081 | LR: 2.16e-04 | 1387.4ms/step | 14.4 samples/s +Step 24270/44000 | Loss: 2.9106 | LR: 2.16e-04 | 1387.4ms/step | 14.4 samples/s +Step 24280/44000 | Loss: 2.8990 | LR: 2.16e-04 | 1387.5ms/step | 14.4 samples/s +Step 24290/44000 | Loss: 2.9165 | LR: 2.15e-04 | 1388.6ms/step | 14.4 samples/s +Step 24300/44000 | Loss: 2.9363 | LR: 2.15e-04 | 1385.4ms/step | 14.4 samples/s +Step 24310/44000 | Loss: 2.9121 | LR: 2.15e-04 | 1387.8ms/step | 14.4 samples/s +Step 24320/44000 | Loss: 2.9214 | LR: 2.15e-04 | 1388.9ms/step | 14.4 samples/s +Step 24330/44000 | Loss: 2.9263 | LR: 2.15e-04 | 1387.2ms/step | 14.4 samples/s +Step 24340/44000 | Loss: 2.9328 | LR: 2.14e-04 | 1388.1ms/step | 14.4 samples/s +Step 24350/44000 | Loss: 2.9066 | LR: 2.14e-04 | 1387.9ms/step | 14.4 samples/s +Step 24360/44000 | Loss: 2.8865 | LR: 2.14e-04 | 1382.9ms/step | 14.5 samples/s +Step 24370/44000 | Loss: 2.9502 | LR: 2.14e-04 | 1383.9ms/step | 14.5 samples/s +Step 24380/44000 | Loss: 2.8976 | LR: 2.14e-04 | 1386.5ms/step | 14.4 samples/s +Step 24390/44000 | Loss: 2.9116 | LR: 2.14e-04 | 1387.1ms/step | 14.4 samples/s +Step 24400/44000 | Loss: 2.9074 | LR: 2.13e-04 | 1387.1ms/step | 14.4 samples/s +Step 24410/44000 | Loss: 2.9226 | LR: 2.13e-04 | 1386.6ms/step | 14.4 samples/s +Step 24420/44000 | Loss: 2.9088 | LR: 2.13e-04 | 1388.0ms/step | 14.4 samples/s +Step 24430/44000 | Loss: 2.9394 | LR: 2.13e-04 | 1386.6ms/step | 14.4 samples/s +Step 24440/44000 | Loss: 2.8995 | LR: 2.13e-04 | 1389.4ms/step | 14.4 samples/s +Step 24450/44000 | Loss: 2.8889 | LR: 2.12e-04 | 1387.6ms/step | 14.4 samples/s +Step 24460/44000 | Loss: 2.8944 | LR: 2.12e-04 | 1388.0ms/step | 14.4 samples/s +Step 24470/44000 | Loss: 2.8970 | LR: 2.12e-04 | 1389.3ms/step | 14.4 samples/s +Step 24480/44000 | Loss: 2.8981 | LR: 2.12e-04 | 1387.2ms/step | 14.4 samples/s +Step 24490/44000 | Loss: 2.9282 | LR: 2.12e-04 | 1387.4ms/step | 14.4 samples/s +Step 24500/44000 | Loss: 2.9314 | LR: 2.12e-04 | 1387.7ms/step | 14.4 samples/s +Step 24510/44000 | Loss: 2.9201 | LR: 2.11e-04 | 1387.0ms/step | 14.4 samples/s +Step 24520/44000 | Loss: 2.9098 | LR: 2.11e-04 | 1388.0ms/step | 14.4 samples/s +Step 24530/44000 | Loss: 2.9267 | LR: 2.11e-04 | 1390.0ms/step | 14.4 samples/s +Step 24540/44000 | Loss: 2.9028 | LR: 2.11e-04 | 1387.6ms/step | 14.4 samples/s +Step 24550/44000 | Loss: 2.9210 | LR: 2.11e-04 | 1390.4ms/step | 14.4 samples/s +Step 24560/44000 | Loss: 2.9283 | LR: 2.10e-04 | 1390.9ms/step | 14.4 samples/s +Step 24570/44000 | Loss: 2.9127 | LR: 2.10e-04 | 1387.6ms/step | 14.4 samples/s +Step 24580/44000 | Loss: 2.9099 | LR: 2.10e-04 | 1384.7ms/step | 14.4 samples/s +Step 24590/44000 | Loss: 2.9035 | LR: 2.10e-04 | 1383.9ms/step | 14.5 samples/s +Step 24600/44000 | Loss: 2.9272 | LR: 2.10e-04 | 1384.5ms/step | 14.4 samples/s +Step 24610/44000 | Loss: 2.8836 | LR: 2.10e-04 | 1384.2ms/step | 14.4 samples/s +Step 24620/44000 | Loss: 2.9030 | LR: 2.09e-04 | 1385.4ms/step | 14.4 samples/s +Step 24630/44000 | Loss: 2.9202 | LR: 2.09e-04 | 1387.1ms/step | 14.4 samples/s +Step 24640/44000 | Loss: 2.8949 | LR: 2.09e-04 | 1389.1ms/step | 14.4 samples/s +Step 24650/44000 | Loss: 2.9196 | LR: 2.09e-04 | 1385.0ms/step | 14.4 samples/s +Step 24660/44000 | Loss: 2.8956 | LR: 2.09e-04 | 1386.8ms/step | 14.4 samples/s +Step 24670/44000 | Loss: 2.9199 | LR: 2.09e-04 | 1384.3ms/step | 14.4 samples/s +Step 24680/44000 | Loss: 2.9073 | LR: 2.08e-04 | 1388.2ms/step | 14.4 samples/s +Step 24690/44000 | Loss: 2.9003 | LR: 2.08e-04 | 1386.0ms/step | 14.4 samples/s +Step 24700/44000 | Loss: 2.9151 | LR: 2.08e-04 | 1385.2ms/step | 14.4 samples/s +Step 24710/44000 | Loss: 2.9242 | LR: 2.08e-04 | 1388.4ms/step | 14.4 samples/s +Step 24720/44000 | Loss: 2.8913 | LR: 2.08e-04 | 1388.6ms/step | 14.4 samples/s +Step 24730/44000 | Loss: 2.9089 | LR: 2.07e-04 | 1389.7ms/step | 14.4 samples/s +Step 24740/44000 | Loss: 2.9021 | LR: 2.07e-04 | 1388.6ms/step | 14.4 samples/s +Step 24750/44000 | Loss: 2.9384 | LR: 2.07e-04 | 1388.0ms/step | 14.4 samples/s +Step 24760/44000 | Loss: 2.8863 | LR: 2.07e-04 | 1390.5ms/step | 14.4 samples/s +Step 24770/44000 | Loss: 2.8991 | LR: 2.07e-04 | 1387.3ms/step | 14.4 samples/s +Step 24780/44000 | Loss: 2.9088 | LR: 2.07e-04 | 1390.3ms/step | 14.4 samples/s +Step 24790/44000 | Loss: 2.9498 | LR: 2.06e-04 | 1390.5ms/step | 14.4 samples/s +Step 24800/44000 | Loss: 2.9012 | LR: 2.06e-04 | 1389.3ms/step | 14.4 samples/s +Step 24810/44000 | Loss: 2.9479 | LR: 2.06e-04 | 1389.5ms/step | 14.4 samples/s +Step 24820/44000 | Loss: 2.8999 | LR: 2.06e-04 | 1390.6ms/step | 14.4 samples/s +Step 24830/44000 | Loss: 2.8973 | LR: 2.06e-04 | 1391.8ms/step | 14.4 samples/s +Step 24840/44000 | Loss: 2.9375 | LR: 2.05e-04 | 1388.3ms/step | 14.4 samples/s +Step 24850/44000 | Loss: 2.9278 | LR: 2.05e-04 | 1390.0ms/step | 14.4 samples/s +Step 24860/44000 | Loss: 2.8944 | LR: 2.05e-04 | 1388.7ms/step | 14.4 samples/s +Step 24870/44000 | Loss: 2.9049 | LR: 2.05e-04 | 1388.1ms/step | 14.4 samples/s +Step 24880/44000 | Loss: 2.8999 | LR: 2.05e-04 | 1387.8ms/step | 14.4 samples/s +Step 24890/44000 | Loss: 2.9290 | LR: 2.05e-04 | 1389.1ms/step | 14.4 samples/s +Step 24900/44000 | Loss: 2.9158 | LR: 2.04e-04 | 1386.8ms/step | 14.4 samples/s +Step 24910/44000 | Loss: 2.9076 | LR: 2.04e-04 | 1389.3ms/step | 14.4 samples/s +Step 24920/44000 | Loss: 2.9003 | LR: 2.04e-04 | 1389.7ms/step | 14.4 samples/s +Step 24930/44000 | Loss: 2.9170 | LR: 2.04e-04 | 1388.0ms/step | 14.4 samples/s +Step 24940/44000 | Loss: 2.9083 | LR: 2.04e-04 | 1386.0ms/step | 14.4 samples/s +Step 24950/44000 | Loss: 2.9182 | LR: 2.04e-04 | 1388.0ms/step | 14.4 samples/s +Step 24960/44000 | Loss: 2.9180 | LR: 2.03e-04 | 1388.6ms/step | 14.4 samples/s +Step 24970/44000 | Loss: 2.9220 | LR: 2.03e-04 | 1388.9ms/step | 14.4 samples/s +Step 24980/44000 | Loss: 2.9141 | LR: 2.03e-04 | 1388.5ms/step | 14.4 samples/s +Step 24990/44000 | Loss: 2.9060 | LR: 2.03e-04 | 1386.4ms/step | 14.4 samples/s +Step 25000/44000 | Loss: 2.9404 | LR: 2.03e-04 | 1390.2ms/step | 14.4 samples/s +Validation at step 25000: Loss = 2.9313, Perplexity = 18.75 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-25000 +Step 25010/44000 | Loss: 2.8940 | LR: 2.02e-04 | 3877.0ms/step | 5.2 samples/s +Step 25020/44000 | Loss: 2.9154 | LR: 2.02e-04 | 1383.0ms/step | 14.5 samples/s +Step 25030/44000 | Loss: 2.8896 | LR: 2.02e-04 | 1385.9ms/step | 14.4 samples/s +Step 25040/44000 | Loss: 2.9064 | LR: 2.02e-04 | 1384.1ms/step | 14.4 samples/s +Step 25050/44000 | Loss: 2.8936 | LR: 2.02e-04 | 1383.4ms/step | 14.5 samples/s +Step 25060/44000 | Loss: 2.9050 | LR: 2.02e-04 | 1385.2ms/step | 14.4 samples/s +Step 25070/44000 | Loss: 2.9195 | LR: 2.01e-04 | 1386.7ms/step | 14.4 samples/s +Step 25080/44000 | Loss: 2.9314 | LR: 2.01e-04 | 1386.7ms/step | 14.4 samples/s +Step 25090/44000 | Loss: 2.8992 | LR: 2.01e-04 | 1386.3ms/step | 14.4 samples/s +Step 25100/44000 | Loss: 2.9127 | LR: 2.01e-04 | 1390.0ms/step | 14.4 samples/s +Step 25110/44000 | Loss: 2.9463 | LR: 2.01e-04 | 1389.8ms/step | 14.4 samples/s +Step 25120/44000 | Loss: 2.8897 | LR: 2.00e-04 | 1388.9ms/step | 14.4 samples/s +Step 25130/44000 | Loss: 2.9295 | LR: 2.00e-04 | 1391.4ms/step | 14.4 samples/s +Step 25140/44000 | Loss: 2.9106 | LR: 2.00e-04 | 1389.0ms/step | 14.4 samples/s +Step 25150/44000 | Loss: 2.8911 | LR: 2.00e-04 | 1382.9ms/step | 14.5 samples/s +Step 25160/44000 | Loss: 2.9347 | LR: 2.00e-04 | 1382.0ms/step | 14.5 samples/s +Step 25170/44000 | Loss: 2.9319 | LR: 2.00e-04 | 1383.6ms/step | 14.5 samples/s +Step 25180/44000 | Loss: 2.8915 | LR: 1.99e-04 | 1386.2ms/step | 14.4 samples/s +Step 25190/44000 | Loss: 2.8948 | LR: 1.99e-04 | 1385.8ms/step | 14.4 samples/s +Step 25200/44000 | Loss: 2.9195 | LR: 1.99e-04 | 1387.6ms/step | 14.4 samples/s +Step 25210/44000 | Loss: 2.9094 | LR: 1.99e-04 | 1386.7ms/step | 14.4 samples/s +Step 25220/44000 | Loss: 2.8831 | LR: 1.99e-04 | 1381.0ms/step | 14.5 samples/s +Step 25230/44000 | Loss: 2.9342 | LR: 1.99e-04 | 1384.7ms/step | 14.4 samples/s +Step 25240/44000 | Loss: 2.8970 | LR: 1.98e-04 | 1384.7ms/step | 14.4 samples/s +Step 25250/44000 | Loss: 2.9152 | LR: 1.98e-04 | 1387.8ms/step | 14.4 samples/s +Step 25260/44000 | Loss: 2.9181 | LR: 1.98e-04 | 1388.1ms/step | 14.4 samples/s +Step 25270/44000 | Loss: 2.8759 | LR: 1.98e-04 | 1388.3ms/step | 14.4 samples/s +Step 25280/44000 | Loss: 2.9461 | LR: 1.98e-04 | 1387.7ms/step | 14.4 samples/s +Step 25290/44000 | Loss: 2.9380 | LR: 1.97e-04 | 1390.1ms/step | 14.4 samples/s +Step 25300/44000 | Loss: 2.9265 | LR: 1.97e-04 | 1387.6ms/step | 14.4 samples/s +Step 25310/44000 | Loss: 2.8727 | LR: 1.97e-04 | 1388.9ms/step | 14.4 samples/s +Step 25320/44000 | Loss: 2.9119 | LR: 1.97e-04 | 1390.7ms/step | 14.4 samples/s +Step 25330/44000 | Loss: 2.9071 | LR: 1.97e-04 | 1388.6ms/step | 14.4 samples/s +Step 25340/44000 | Loss: 2.9267 | LR: 1.97e-04 | 1391.1ms/step | 14.4 samples/s +Step 25350/44000 | Loss: 2.8874 | LR: 1.96e-04 | 1391.2ms/step | 14.4 samples/s +Step 25360/44000 | Loss: 2.9121 | LR: 1.96e-04 | 1387.0ms/step | 14.4 samples/s +Step 25370/44000 | Loss: 2.8990 | LR: 1.96e-04 | 1385.6ms/step | 14.4 samples/s +Step 25380/44000 | Loss: 2.9292 | LR: 1.96e-04 | 1384.4ms/step | 14.4 samples/s +Step 25390/44000 | Loss: 2.9040 | LR: 1.96e-04 | 1386.0ms/step | 14.4 samples/s +Step 25400/44000 | Loss: 2.9383 | LR: 1.96e-04 | 1386.4ms/step | 14.4 samples/s +Step 25410/44000 | Loss: 2.9287 | LR: 1.95e-04 | 1389.2ms/step | 14.4 samples/s +Step 25420/44000 | Loss: 2.9242 | LR: 1.95e-04 | 1388.8ms/step | 14.4 samples/s +Step 25430/44000 | Loss: 2.9230 | LR: 1.95e-04 | 1383.1ms/step | 14.5 samples/s +Step 25440/44000 | Loss: 2.8979 | LR: 1.95e-04 | 1383.1ms/step | 14.5 samples/s +Step 25450/44000 | Loss: 2.9264 | LR: 1.95e-04 | 1382.8ms/step | 14.5 samples/s +Step 25460/44000 | Loss: 2.9233 | LR: 1.94e-04 | 1387.5ms/step | 14.4 samples/s +Step 25470/44000 | Loss: 2.8961 | LR: 1.94e-04 | 1386.5ms/step | 14.4 samples/s +Step 25480/44000 | Loss: 2.9183 | LR: 1.94e-04 | 1387.6ms/step | 14.4 samples/s +Step 25490/44000 | Loss: 2.9025 | LR: 1.94e-04 | 1388.4ms/step | 14.4 samples/s +Step 25500/44000 | Loss: 2.8950 | LR: 1.94e-04 | 1385.3ms/step | 14.4 samples/s +Step 25510/44000 | Loss: 2.9057 | LR: 1.94e-04 | 1382.5ms/step | 14.5 samples/s +Step 25520/44000 | Loss: 2.9071 | LR: 1.93e-04 | 1385.2ms/step | 14.4 samples/s +Step 25530/44000 | Loss: 2.8836 | LR: 1.93e-04 | 1389.4ms/step | 14.4 samples/s +Step 25540/44000 | Loss: 2.8866 | LR: 1.93e-04 | 1388.3ms/step | 14.4 samples/s +Step 25550/44000 | Loss: 2.8971 | LR: 1.93e-04 | 1389.5ms/step | 14.4 samples/s +Step 25560/44000 | Loss: 2.9332 | LR: 1.93e-04 | 1388.3ms/step | 14.4 samples/s +Step 25570/44000 | Loss: 2.9446 | LR: 1.92e-04 | 1386.3ms/step | 14.4 samples/s +Step 25580/44000 | Loss: 2.8976 | LR: 1.92e-04 | 1386.5ms/step | 14.4 samples/s +Step 25590/44000 | Loss: 2.8964 | LR: 1.92e-04 | 1390.1ms/step | 14.4 samples/s +Step 25600/44000 | Loss: 2.9022 | LR: 1.92e-04 | 1386.6ms/step | 14.4 samples/s +Step 25610/44000 | Loss: 2.8939 | LR: 1.92e-04 | 1385.2ms/step | 14.4 samples/s +Step 25620/44000 | Loss: 2.9185 | LR: 1.92e-04 | 1382.4ms/step | 14.5 samples/s +Step 25630/44000 | Loss: 2.9158 | LR: 1.91e-04 | 1382.7ms/step | 14.5 samples/s +Step 25640/44000 | Loss: 2.9090 | LR: 1.91e-04 | 1381.9ms/step | 14.5 samples/s +Step 25650/44000 | Loss: 2.9012 | LR: 1.91e-04 | 1385.4ms/step | 14.4 samples/s +Step 25660/44000 | Loss: 2.9006 | LR: 1.91e-04 | 1389.6ms/step | 14.4 samples/s +Step 25670/44000 | Loss: 2.9123 | LR: 1.91e-04 | 1391.7ms/step | 14.4 samples/s +Step 25680/44000 | Loss: 2.9093 | LR: 1.91e-04 | 1388.6ms/step | 14.4 samples/s +Step 25690/44000 | Loss: 2.8681 | LR: 1.90e-04 | 1386.3ms/step | 14.4 samples/s +Step 25700/44000 | Loss: 2.9207 | LR: 1.90e-04 | 1389.5ms/step | 14.4 samples/s +Step 25710/44000 | Loss: 2.8756 | LR: 1.90e-04 | 1388.2ms/step | 14.4 samples/s +Step 25720/44000 | Loss: 2.8841 | LR: 1.90e-04 | 1388.1ms/step | 14.4 samples/s +Step 25730/44000 | Loss: 2.9116 | LR: 1.90e-04 | 1389.6ms/step | 14.4 samples/s +Step 25740/44000 | Loss: 2.9168 | LR: 1.89e-04 | 1390.3ms/step | 14.4 samples/s +Step 25750/44000 | Loss: 2.9027 | LR: 1.89e-04 | 1388.1ms/step | 14.4 samples/s +Step 25760/44000 | Loss: 2.8945 | LR: 1.89e-04 | 1389.7ms/step | 14.4 samples/s +Step 25770/44000 | Loss: 2.9072 | LR: 1.89e-04 | 1388.8ms/step | 14.4 samples/s +Step 25780/44000 | Loss: 2.8752 | LR: 1.89e-04 | 1386.0ms/step | 14.4 samples/s +Step 25790/44000 | Loss: 2.9141 | LR: 1.89e-04 | 1382.9ms/step | 14.5 samples/s +Step 25800/44000 | Loss: 2.9038 | LR: 1.88e-04 | 1386.4ms/step | 14.4 samples/s +Step 25810/44000 | Loss: 2.8994 | LR: 1.88e-04 | 1387.6ms/step | 14.4 samples/s +Step 25820/44000 | Loss: 2.9121 | LR: 1.88e-04 | 1388.5ms/step | 14.4 samples/s +Step 25830/44000 | Loss: 2.9106 | LR: 1.88e-04 | 1389.6ms/step | 14.4 samples/s +Step 25840/44000 | Loss: 2.8807 | LR: 1.88e-04 | 1384.4ms/step | 14.4 samples/s +Step 25850/44000 | Loss: 2.9465 | LR: 1.88e-04 | 1381.8ms/step | 14.5 samples/s +Step 25860/44000 | Loss: 2.9061 | LR: 1.87e-04 | 1385.8ms/step | 14.4 samples/s +Step 25870/44000 | Loss: 2.9142 | LR: 1.87e-04 | 1385.3ms/step | 14.4 samples/s +Step 25880/44000 | Loss: 2.9020 | LR: 1.87e-04 | 1388.4ms/step | 14.4 samples/s +Step 25890/44000 | Loss: 2.9544 | LR: 1.87e-04 | 1387.3ms/step | 14.4 samples/s +Step 25900/44000 | Loss: 2.9231 | LR: 1.87e-04 | 1387.4ms/step | 14.4 samples/s +Step 25910/44000 | Loss: 2.9154 | LR: 1.87e-04 | 1387.8ms/step | 14.4 samples/s +Step 25920/44000 | Loss: 2.9220 | LR: 1.86e-04 | 1389.1ms/step | 14.4 samples/s +Step 25930/44000 | Loss: 2.9108 | LR: 1.86e-04 | 1387.0ms/step | 14.4 samples/s +Step 25940/44000 | Loss: 2.8775 | LR: 1.86e-04 | 1388.2ms/step | 14.4 samples/s +Step 25950/44000 | Loss: 2.9198 | LR: 1.86e-04 | 1389.7ms/step | 14.4 samples/s +Step 25960/44000 | Loss: 2.8903 | LR: 1.86e-04 | 1390.0ms/step | 14.4 samples/s +Step 25970/44000 | Loss: 2.8955 | LR: 1.85e-04 | 1388.4ms/step | 14.4 samples/s +Step 25980/44000 | Loss: 2.8876 | LR: 1.85e-04 | 1389.5ms/step | 14.4 samples/s +Step 25990/44000 | Loss: 2.9075 | LR: 1.85e-04 | 1389.7ms/step | 14.4 samples/s +Step 26000/44000 | Loss: 2.9409 | LR: 1.85e-04 | 1388.8ms/step | 14.4 samples/s +Validation at step 26000: Loss = 2.9242, Perplexity = 18.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-26000 +Step 26010/44000 | Loss: 2.8978 | LR: 1.85e-04 | 3879.1ms/step | 5.2 samples/s +Step 26020/44000 | Loss: 2.9204 | LR: 1.85e-04 | 1382.8ms/step | 14.5 samples/s +Step 26030/44000 | Loss: 2.8974 | LR: 1.84e-04 | 1381.0ms/step | 14.5 samples/s +Step 26040/44000 | Loss: 2.9172 | LR: 1.84e-04 | 1384.3ms/step | 14.4 samples/s +Step 26050/44000 | Loss: 2.9038 | LR: 1.84e-04 | 1383.0ms/step | 14.5 samples/s +Step 26060/44000 | Loss: 2.9146 | LR: 1.84e-04 | 1381.6ms/step | 14.5 samples/s +Step 26070/44000 | Loss: 2.9192 | LR: 1.84e-04 | 1385.1ms/step | 14.4 samples/s +Step 26080/44000 | Loss: 2.9149 | LR: 1.84e-04 | 1387.0ms/step | 14.4 samples/s +Step 26090/44000 | Loss: 2.9047 | LR: 1.83e-04 | 1390.4ms/step | 14.4 samples/s +Step 26100/44000 | Loss: 2.8820 | LR: 1.83e-04 | 1387.7ms/step | 14.4 samples/s +Step 26110/44000 | Loss: 2.9241 | LR: 1.83e-04 | 1391.1ms/step | 14.4 samples/s +Step 26120/44000 | Loss: 2.9272 | LR: 1.83e-04 | 1388.1ms/step | 14.4 samples/s +Step 26130/44000 | Loss: 2.8681 | LR: 1.83e-04 | 1385.1ms/step | 14.4 samples/s +Step 26140/44000 | Loss: 2.9112 | LR: 1.82e-04 | 1387.9ms/step | 14.4 samples/s +Step 26150/44000 | Loss: 2.9228 | LR: 1.82e-04 | 1386.2ms/step | 14.4 samples/s +Step 26160/44000 | Loss: 2.8907 | LR: 1.82e-04 | 1387.5ms/step | 14.4 samples/s +Step 26170/44000 | Loss: 2.9081 | LR: 1.82e-04 | 1388.1ms/step | 14.4 samples/s +Step 26180/44000 | Loss: 2.9241 | LR: 1.82e-04 | 1389.6ms/step | 14.4 samples/s +Step 26190/44000 | Loss: 2.9203 | LR: 1.82e-04 | 1390.9ms/step | 14.4 samples/s +Step 26200/44000 | Loss: 2.9110 | LR: 1.81e-04 | 1384.9ms/step | 14.4 samples/s +Step 26210/44000 | Loss: 2.9037 | LR: 1.81e-04 | 1387.7ms/step | 14.4 samples/s +Step 26220/44000 | Loss: 2.8710 | LR: 1.81e-04 | 1389.0ms/step | 14.4 samples/s +Step 26230/44000 | Loss: 2.8933 | LR: 1.81e-04 | 1391.6ms/step | 14.4 samples/s +Step 26240/44000 | Loss: 2.8805 | LR: 1.81e-04 | 1391.0ms/step | 14.4 samples/s +Step 26250/44000 | Loss: 2.9260 | LR: 1.81e-04 | 1389.4ms/step | 14.4 samples/s +Step 26260/44000 | Loss: 2.9194 | LR: 1.80e-04 | 1388.9ms/step | 14.4 samples/s +Step 26270/44000 | Loss: 2.9159 | LR: 1.80e-04 | 1384.1ms/step | 14.4 samples/s +Step 26280/44000 | Loss: 2.8634 | LR: 1.80e-04 | 1385.3ms/step | 14.4 samples/s +Step 26290/44000 | Loss: 2.8985 | LR: 1.80e-04 | 1389.7ms/step | 14.4 samples/s +Step 26300/44000 | Loss: 2.8969 | LR: 1.80e-04 | 1389.9ms/step | 14.4 samples/s +Step 26310/44000 | Loss: 2.9211 | LR: 1.80e-04 | 1394.2ms/step | 14.3 samples/s +Step 26320/44000 | Loss: 2.8555 | LR: 1.79e-04 | 1390.3ms/step | 14.4 samples/s +Step 26330/44000 | Loss: 2.9124 | LR: 1.79e-04 | 1389.8ms/step | 14.4 samples/s +Step 26340/44000 | Loss: 2.9089 | LR: 1.79e-04 | 1387.6ms/step | 14.4 samples/s +Step 26350/44000 | Loss: 2.9061 | LR: 1.79e-04 | 1385.4ms/step | 14.4 samples/s +Step 26360/44000 | Loss: 2.9054 | LR: 1.79e-04 | 1385.1ms/step | 14.4 samples/s +Step 26370/44000 | Loss: 2.9187 | LR: 1.78e-04 | 1390.0ms/step | 14.4 samples/s +Step 26380/44000 | Loss: 2.9012 | LR: 1.78e-04 | 1391.7ms/step | 14.4 samples/s +Step 26390/44000 | Loss: 2.9080 | LR: 1.78e-04 | 1390.1ms/step | 14.4 samples/s +Step 26400/44000 | Loss: 2.9144 | LR: 1.78e-04 | 1391.9ms/step | 14.4 samples/s +Step 26410/44000 | Loss: 2.9029 | LR: 1.78e-04 | 1386.6ms/step | 14.4 samples/s +Step 26420/44000 | Loss: 2.9129 | LR: 1.78e-04 | 1387.6ms/step | 14.4 samples/s +Step 26430/44000 | Loss: 2.8996 | LR: 1.77e-04 | 1385.9ms/step | 14.4 samples/s +Step 26440/44000 | Loss: 2.8625 | LR: 1.77e-04 | 1388.1ms/step | 14.4 samples/s +Step 26450/44000 | Loss: 2.8722 | LR: 1.77e-04 | 1388.8ms/step | 14.4 samples/s +Step 26460/44000 | Loss: 2.9066 | LR: 1.77e-04 | 1390.4ms/step | 14.4 samples/s +Step 26470/44000 | Loss: 2.9077 | LR: 1.77e-04 | 1390.9ms/step | 14.4 samples/s +Step 26480/44000 | Loss: 2.8982 | LR: 1.77e-04 | 1390.0ms/step | 14.4 samples/s +Step 26490/44000 | Loss: 2.9130 | LR: 1.76e-04 | 1395.9ms/step | 14.3 samples/s +Step 26500/44000 | Loss: 2.9036 | LR: 1.76e-04 | 1391.9ms/step | 14.4 samples/s +Step 26510/44000 | Loss: 2.8873 | LR: 1.76e-04 | 1393.0ms/step | 14.4 samples/s +Step 26520/44000 | Loss: 2.8863 | LR: 1.76e-04 | 1394.2ms/step | 14.3 samples/s +Step 26530/44000 | Loss: 2.9155 | LR: 1.76e-04 | 1390.8ms/step | 14.4 samples/s +Step 26540/44000 | Loss: 2.9291 | LR: 1.76e-04 | 1389.1ms/step | 14.4 samples/s +Step 26550/44000 | Loss: 2.8675 | LR: 1.75e-04 | 1390.8ms/step | 14.4 samples/s +Step 26560/44000 | Loss: 2.9215 | LR: 1.75e-04 | 1390.6ms/step | 14.4 samples/s +Step 26570/44000 | Loss: 2.9080 | LR: 1.75e-04 | 1391.6ms/step | 14.4 samples/s +Step 26580/44000 | Loss: 2.9089 | LR: 1.75e-04 | 1391.8ms/step | 14.4 samples/s +Step 26590/44000 | Loss: 2.9008 | LR: 1.75e-04 | 1394.1ms/step | 14.3 samples/s +Step 26600/44000 | Loss: 2.8793 | LR: 1.74e-04 | 1388.1ms/step | 14.4 samples/s +Step 26610/44000 | Loss: 2.8875 | LR: 1.74e-04 | 1389.1ms/step | 14.4 samples/s +Step 26620/44000 | Loss: 2.9202 | LR: 1.74e-04 | 1392.5ms/step | 14.4 samples/s +Step 26630/44000 | Loss: 2.9346 | LR: 1.74e-04 | 1387.6ms/step | 14.4 samples/s +Step 26640/44000 | Loss: 2.8946 | LR: 1.74e-04 | 1393.2ms/step | 14.4 samples/s +Step 26650/44000 | Loss: 2.9078 | LR: 1.74e-04 | 1390.3ms/step | 14.4 samples/s +Step 26660/44000 | Loss: 2.8875 | LR: 1.73e-04 | 1390.2ms/step | 14.4 samples/s +Step 26670/44000 | Loss: 2.9096 | LR: 1.73e-04 | 1389.7ms/step | 14.4 samples/s +Step 26680/44000 | Loss: 2.9150 | LR: 1.73e-04 | 1391.4ms/step | 14.4 samples/s +Step 26690/44000 | Loss: 2.9120 | LR: 1.73e-04 | 1387.9ms/step | 14.4 samples/s +Step 26700/44000 | Loss: 2.8745 | LR: 1.73e-04 | 1382.2ms/step | 14.5 samples/s +Step 26710/44000 | Loss: 2.9310 | LR: 1.73e-04 | 1382.3ms/step | 14.5 samples/s +Step 26720/44000 | Loss: 2.9244 | LR: 1.72e-04 | 1383.7ms/step | 14.5 samples/s +Step 26730/44000 | Loss: 2.9050 | LR: 1.72e-04 | 1385.9ms/step | 14.4 samples/s +Step 26740/44000 | Loss: 2.9233 | LR: 1.72e-04 | 1389.2ms/step | 14.4 samples/s +Step 26750/44000 | Loss: 2.8961 | LR: 1.72e-04 | 1390.1ms/step | 14.4 samples/s +Step 26760/44000 | Loss: 2.9166 | LR: 1.72e-04 | 1388.0ms/step | 14.4 samples/s +Step 26770/44000 | Loss: 2.9170 | LR: 1.72e-04 | 1383.8ms/step | 14.5 samples/s +Step 26780/44000 | Loss: 2.9039 | LR: 1.71e-04 | 1384.1ms/step | 14.5 samples/s +Step 26790/44000 | Loss: 2.9118 | LR: 1.71e-04 | 1387.5ms/step | 14.4 samples/s +Step 26800/44000 | Loss: 2.8748 | LR: 1.71e-04 | 1391.8ms/step | 14.4 samples/s +Step 26810/44000 | Loss: 2.9100 | LR: 1.71e-04 | 1389.9ms/step | 14.4 samples/s +Step 26820/44000 | Loss: 2.8856 | LR: 1.71e-04 | 1391.7ms/step | 14.4 samples/s +Step 26830/44000 | Loss: 2.9272 | LR: 1.71e-04 | 1386.2ms/step | 14.4 samples/s +Step 26840/44000 | Loss: 2.9023 | LR: 1.70e-04 | 1385.2ms/step | 14.4 samples/s +Step 26850/44000 | Loss: 2.9140 | LR: 1.70e-04 | 1385.2ms/step | 14.4 samples/s +Step 26860/44000 | Loss: 2.8883 | LR: 1.70e-04 | 1388.7ms/step | 14.4 samples/s +Step 26870/44000 | Loss: 2.8856 | LR: 1.70e-04 | 1386.1ms/step | 14.4 samples/s +Step 26880/44000 | Loss: 2.8741 | LR: 1.70e-04 | 1386.8ms/step | 14.4 samples/s +Step 26890/44000 | Loss: 2.9033 | LR: 1.69e-04 | 1388.5ms/step | 14.4 samples/s +Step 26900/44000 | Loss: 2.8832 | LR: 1.69e-04 | 1390.7ms/step | 14.4 samples/s +Step 26910/44000 | Loss: 2.8877 | LR: 1.69e-04 | 1386.1ms/step | 14.4 samples/s +Step 26920/44000 | Loss: 2.8863 | LR: 1.69e-04 | 1388.0ms/step | 14.4 samples/s +Step 26930/44000 | Loss: 2.9310 | LR: 1.69e-04 | 1387.1ms/step | 14.4 samples/s +Step 26940/44000 | Loss: 2.9075 | LR: 1.69e-04 | 1389.3ms/step | 14.4 samples/s +Step 26950/44000 | Loss: 2.9327 | LR: 1.68e-04 | 1389.8ms/step | 14.4 samples/s +Step 26960/44000 | Loss: 2.9021 | LR: 1.68e-04 | 1387.6ms/step | 14.4 samples/s +Step 26970/44000 | Loss: 2.9063 | LR: 1.68e-04 | 1389.4ms/step | 14.4 samples/s +Step 26980/44000 | Loss: 2.9122 | LR: 1.68e-04 | 1384.1ms/step | 14.4 samples/s +Step 26990/44000 | Loss: 2.8942 | LR: 1.68e-04 | 1382.8ms/step | 14.5 samples/s +Step 27000/44000 | Loss: 2.8919 | LR: 1.68e-04 | 1384.7ms/step | 14.4 samples/s +Validation at step 27000: Loss = 2.9191, Perplexity = 18.52 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-27000 +Step 27010/44000 | Loss: 2.8957 | LR: 1.67e-04 | 3895.4ms/step | 5.1 samples/s +Step 27020/44000 | Loss: 2.8926 | LR: 1.67e-04 | 1383.1ms/step | 14.5 samples/s +Step 27030/44000 | Loss: 2.9187 | LR: 1.67e-04 | 1383.8ms/step | 14.5 samples/s +Step 27040/44000 | Loss: 2.8965 | LR: 1.67e-04 | 1381.9ms/step | 14.5 samples/s +Step 27050/44000 | Loss: 2.8712 | LR: 1.67e-04 | 1384.3ms/step | 14.4 samples/s +Step 27060/44000 | Loss: 2.8714 | LR: 1.67e-04 | 1385.4ms/step | 14.4 samples/s +Step 27070/44000 | Loss: 2.8926 | LR: 1.66e-04 | 1386.5ms/step | 14.4 samples/s +Step 27080/44000 | Loss: 2.9169 | LR: 1.66e-04 | 1386.3ms/step | 14.4 samples/s +Step 27090/44000 | Loss: 2.9079 | LR: 1.66e-04 | 1386.6ms/step | 14.4 samples/s +Step 27100/44000 | Loss: 2.9144 | LR: 1.66e-04 | 1388.5ms/step | 14.4 samples/s +Step 27110/44000 | Loss: 2.8956 | LR: 1.66e-04 | 1388.5ms/step | 14.4 samples/s +Step 27120/44000 | Loss: 2.9192 | LR: 1.66e-04 | 1389.6ms/step | 14.4 samples/s +Step 27130/44000 | Loss: 2.9150 | LR: 1.65e-04 | 1389.4ms/step | 14.4 samples/s +Step 27140/44000 | Loss: 2.8855 | LR: 1.65e-04 | 1391.2ms/step | 14.4 samples/s +Step 27150/44000 | Loss: 2.9002 | LR: 1.65e-04 | 1388.9ms/step | 14.4 samples/s +Step 27160/44000 | Loss: 2.8923 | LR: 1.65e-04 | 1387.9ms/step | 14.4 samples/s +Step 27170/44000 | Loss: 2.9203 | LR: 1.65e-04 | 1386.5ms/step | 14.4 samples/s +Step 27180/44000 | Loss: 2.9109 | LR: 1.65e-04 | 1388.4ms/step | 14.4 samples/s +Step 27190/44000 | Loss: 2.8886 | LR: 1.64e-04 | 1386.4ms/step | 14.4 samples/s +Step 27200/44000 | Loss: 2.9301 | LR: 1.64e-04 | 1386.7ms/step | 14.4 samples/s +Step 27210/44000 | Loss: 2.9171 | LR: 1.64e-04 | 1387.9ms/step | 14.4 samples/s +Step 27220/44000 | Loss: 2.9352 | LR: 1.64e-04 | 1387.4ms/step | 14.4 samples/s +Step 27230/44000 | Loss: 2.8672 | LR: 1.64e-04 | 1387.6ms/step | 14.4 samples/s +Step 27240/44000 | Loss: 2.8860 | LR: 1.63e-04 | 1388.5ms/step | 14.4 samples/s +Step 27250/44000 | Loss: 2.8956 | LR: 1.63e-04 | 1384.6ms/step | 14.4 samples/s +Step 27260/44000 | Loss: 2.8551 | LR: 1.63e-04 | 1385.6ms/step | 14.4 samples/s +Step 27270/44000 | Loss: 2.9050 | LR: 1.63e-04 | 1381.5ms/step | 14.5 samples/s +Step 27280/44000 | Loss: 2.8731 | LR: 1.63e-04 | 1386.0ms/step | 14.4 samples/s +Step 27290/44000 | Loss: 2.9109 | LR: 1.63e-04 | 1385.5ms/step | 14.4 samples/s +Step 27300/44000 | Loss: 2.9064 | LR: 1.62e-04 | 1388.7ms/step | 14.4 samples/s +Step 27310/44000 | Loss: 2.8730 | LR: 1.62e-04 | 1389.7ms/step | 14.4 samples/s +Step 27320/44000 | Loss: 2.8917 | LR: 1.62e-04 | 1386.0ms/step | 14.4 samples/s +Step 27330/44000 | Loss: 2.8751 | LR: 1.62e-04 | 1382.6ms/step | 14.5 samples/s +Step 27340/44000 | Loss: 2.9169 | LR: 1.62e-04 | 1384.4ms/step | 14.4 samples/s +Step 27350/44000 | Loss: 2.9129 | LR: 1.62e-04 | 1386.0ms/step | 14.4 samples/s +Step 27360/44000 | Loss: 2.8831 | LR: 1.61e-04 | 1384.8ms/step | 14.4 samples/s +Step 27370/44000 | Loss: 2.9072 | LR: 1.61e-04 | 1386.6ms/step | 14.4 samples/s +Step 27380/44000 | Loss: 2.9111 | LR: 1.61e-04 | 1390.4ms/step | 14.4 samples/s +Step 27390/44000 | Loss: 2.9265 | LR: 1.61e-04 | 1388.6ms/step | 14.4 samples/s +Step 27400/44000 | Loss: 2.8628 | LR: 1.61e-04 | 1387.1ms/step | 14.4 samples/s +Step 27410/44000 | Loss: 2.8955 | LR: 1.61e-04 | 1386.8ms/step | 14.4 samples/s +Step 27420/44000 | Loss: 2.8877 | LR: 1.60e-04 | 1386.6ms/step | 14.4 samples/s +Step 27430/44000 | Loss: 2.8757 | LR: 1.60e-04 | 1389.7ms/step | 14.4 samples/s +Step 27440/44000 | Loss: 2.9053 | LR: 1.60e-04 | 1388.4ms/step | 14.4 samples/s +Step 27450/44000 | Loss: 2.8941 | LR: 1.60e-04 | 1387.6ms/step | 14.4 samples/s +Step 27460/44000 | Loss: 2.9038 | LR: 1.60e-04 | 1387.6ms/step | 14.4 samples/s +Step 27470/44000 | Loss: 2.8813 | LR: 1.60e-04 | 1385.0ms/step | 14.4 samples/s +Step 27480/44000 | Loss: 2.9181 | LR: 1.59e-04 | 1388.1ms/step | 14.4 samples/s +Step 27490/44000 | Loss: 2.9032 | LR: 1.59e-04 | 1385.8ms/step | 14.4 samples/s +Step 27500/44000 | Loss: 2.9017 | LR: 1.59e-04 | 1386.7ms/step | 14.4 samples/s +Step 27510/44000 | Loss: 2.9155 | LR: 1.59e-04 | 1387.6ms/step | 14.4 samples/s +Step 27520/44000 | Loss: 2.9154 | LR: 1.59e-04 | 1386.4ms/step | 14.4 samples/s +Step 27530/44000 | Loss: 2.9061 | LR: 1.59e-04 | 1385.8ms/step | 14.4 samples/s +Step 27540/44000 | Loss: 2.9301 | LR: 1.58e-04 | 1383.4ms/step | 14.5 samples/s +Step 27550/44000 | Loss: 2.9007 | LR: 1.58e-04 | 1380.5ms/step | 14.5 samples/s +Step 27560/44000 | Loss: 2.9056 | LR: 1.58e-04 | 1383.5ms/step | 14.5 samples/s +Step 27570/44000 | Loss: 2.8961 | LR: 1.58e-04 | 1386.9ms/step | 14.4 samples/s +Step 27580/44000 | Loss: 2.8942 | LR: 1.58e-04 | 1387.5ms/step | 14.4 samples/s +Step 27590/44000 | Loss: 2.8724 | LR: 1.58e-04 | 1388.8ms/step | 14.4 samples/s +Step 27600/44000 | Loss: 2.8887 | LR: 1.57e-04 | 1389.8ms/step | 14.4 samples/s +Step 27610/44000 | Loss: 2.8992 | LR: 1.57e-04 | 1391.4ms/step | 14.4 samples/s +Step 27620/44000 | Loss: 2.9247 | LR: 1.57e-04 | 1388.0ms/step | 14.4 samples/s +Step 27630/44000 | Loss: 2.8815 | LR: 1.57e-04 | 1389.7ms/step | 14.4 samples/s +Step 27640/44000 | Loss: 2.9177 | LR: 1.57e-04 | 1387.5ms/step | 14.4 samples/s +Step 27650/44000 | Loss: 2.9229 | LR: 1.57e-04 | 1387.7ms/step | 14.4 samples/s +Step 27660/44000 | Loss: 2.8957 | LR: 1.56e-04 | 1385.9ms/step | 14.4 samples/s +Step 27670/44000 | Loss: 2.9035 | LR: 1.56e-04 | 1389.1ms/step | 14.4 samples/s +Step 27680/44000 | Loss: 2.8998 | LR: 1.56e-04 | 1388.8ms/step | 14.4 samples/s +Step 27690/44000 | Loss: 2.9314 | LR: 1.56e-04 | 1386.2ms/step | 14.4 samples/s +Step 27700/44000 | Loss: 2.8857 | LR: 1.56e-04 | 1389.8ms/step | 14.4 samples/s +Step 27710/44000 | Loss: 2.8886 | LR: 1.56e-04 | 1390.0ms/step | 14.4 samples/s +Step 27720/44000 | Loss: 2.8889 | LR: 1.55e-04 | 1389.5ms/step | 14.4 samples/s +Step 27730/44000 | Loss: 2.8982 | LR: 1.55e-04 | 1390.5ms/step | 14.4 samples/s +Step 27740/44000 | Loss: 2.9031 | LR: 1.55e-04 | 1387.9ms/step | 14.4 samples/s +Step 27750/44000 | Loss: 2.8981 | LR: 1.55e-04 | 1390.3ms/step | 14.4 samples/s +Step 27760/44000 | Loss: 2.8989 | LR: 1.55e-04 | 1388.2ms/step | 14.4 samples/s +Step 27770/44000 | Loss: 2.9058 | LR: 1.55e-04 | 1389.5ms/step | 14.4 samples/s +Step 27780/44000 | Loss: 2.8855 | LR: 1.54e-04 | 1387.1ms/step | 14.4 samples/s +Step 27790/44000 | Loss: 2.9117 | LR: 1.54e-04 | 1387.2ms/step | 14.4 samples/s +Step 27800/44000 | Loss: 2.8928 | LR: 1.54e-04 | 1388.4ms/step | 14.4 samples/s +Step 27810/44000 | Loss: 2.8803 | LR: 1.54e-04 | 1389.0ms/step | 14.4 samples/s +Step 27820/44000 | Loss: 2.8904 | LR: 1.54e-04 | 1386.7ms/step | 14.4 samples/s +Step 27830/44000 | Loss: 2.9310 | LR: 1.54e-04 | 1387.5ms/step | 14.4 samples/s +Step 27840/44000 | Loss: 2.9210 | LR: 1.53e-04 | 1388.6ms/step | 14.4 samples/s +Step 27850/44000 | Loss: 2.9085 | LR: 1.53e-04 | 1387.9ms/step | 14.4 samples/s +Step 27860/44000 | Loss: 2.8914 | LR: 1.53e-04 | 1391.4ms/step | 14.4 samples/s +Step 27870/44000 | Loss: 2.8737 | LR: 1.53e-04 | 1388.4ms/step | 14.4 samples/s +Step 27880/44000 | Loss: 2.8976 | LR: 1.53e-04 | 1390.1ms/step | 14.4 samples/s +Step 27890/44000 | Loss: 2.8877 | LR: 1.53e-04 | 1387.5ms/step | 14.4 samples/s +Step 27900/44000 | Loss: 2.8999 | LR: 1.52e-04 | 1390.2ms/step | 14.4 samples/s +Step 27910/44000 | Loss: 2.8934 | LR: 1.52e-04 | 1383.1ms/step | 14.5 samples/s +Step 27920/44000 | Loss: 2.9019 | LR: 1.52e-04 | 1381.6ms/step | 14.5 samples/s +Step 27930/44000 | Loss: 2.9347 | LR: 1.52e-04 | 1383.9ms/step | 14.5 samples/s +Step 27940/44000 | Loss: 2.8792 | LR: 1.52e-04 | 1384.5ms/step | 14.4 samples/s +Step 27950/44000 | Loss: 2.9169 | LR: 1.52e-04 | 1385.3ms/step | 14.4 samples/s +Step 27960/44000 | Loss: 2.8785 | LR: 1.51e-04 | 1388.2ms/step | 14.4 samples/s +Step 27970/44000 | Loss: 2.9161 | LR: 1.51e-04 | 1388.5ms/step | 14.4 samples/s +Step 27980/44000 | Loss: 2.8752 | LR: 1.51e-04 | 1387.4ms/step | 14.4 samples/s +Step 27990/44000 | Loss: 2.9100 | LR: 1.51e-04 | 1388.8ms/step | 14.4 samples/s +Step 28000/44000 | Loss: 2.9041 | LR: 1.51e-04 | 1390.2ms/step | 14.4 samples/s +Validation at step 28000: Loss = 2.9139, Perplexity = 18.43 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-28000 +Step 28010/44000 | Loss: 2.8903 | LR: 1.51e-04 | 3872.8ms/step | 5.2 samples/s +Step 28020/44000 | Loss: 2.8792 | LR: 1.50e-04 | 1384.5ms/step | 14.4 samples/s +Step 28030/44000 | Loss: 2.8772 | LR: 1.50e-04 | 1385.8ms/step | 14.4 samples/s +Step 28040/44000 | Loss: 2.9011 | LR: 1.50e-04 | 1387.4ms/step | 14.4 samples/s +Step 28050/44000 | Loss: 2.8814 | LR: 1.50e-04 | 1386.3ms/step | 14.4 samples/s +Step 28060/44000 | Loss: 2.9152 | LR: 1.50e-04 | 1384.3ms/step | 14.4 samples/s +Step 28070/44000 | Loss: 2.8903 | LR: 1.50e-04 | 1385.1ms/step | 14.4 samples/s +Step 28080/44000 | Loss: 2.9020 | LR: 1.49e-04 | 1384.0ms/step | 14.5 samples/s +Step 28090/44000 | Loss: 2.8717 | LR: 1.49e-04 | 1385.6ms/step | 14.4 samples/s +Step 28100/44000 | Loss: 2.8887 | LR: 1.49e-04 | 1387.1ms/step | 14.4 samples/s +Step 28110/44000 | Loss: 2.8857 | LR: 1.49e-04 | 1388.5ms/step | 14.4 samples/s +Step 28120/44000 | Loss: 2.8994 | LR: 1.49e-04 | 1388.0ms/step | 14.4 samples/s +Step 28130/44000 | Loss: 2.8838 | LR: 1.49e-04 | 1387.6ms/step | 14.4 samples/s +Step 28140/44000 | Loss: 2.8879 | LR: 1.48e-04 | 1386.4ms/step | 14.4 samples/s +Step 28150/44000 | Loss: 2.8870 | LR: 1.48e-04 | 1387.6ms/step | 14.4 samples/s +Step 28160/44000 | Loss: 2.9044 | LR: 1.48e-04 | 1388.1ms/step | 14.4 samples/s +Step 28170/44000 | Loss: 2.8992 | LR: 1.48e-04 | 1390.1ms/step | 14.4 samples/s +Step 28180/44000 | Loss: 2.8794 | LR: 1.48e-04 | 1388.7ms/step | 14.4 samples/s +Step 28190/44000 | Loss: 2.9097 | LR: 1.48e-04 | 1391.4ms/step | 14.4 samples/s +Step 28200/44000 | Loss: 2.9236 | LR: 1.47e-04 | 1389.0ms/step | 14.4 samples/s +Step 28210/44000 | Loss: 2.8994 | LR: 1.47e-04 | 1387.0ms/step | 14.4 samples/s +Step 28220/44000 | Loss: 2.8921 | LR: 1.47e-04 | 1387.2ms/step | 14.4 samples/s +Step 28230/44000 | Loss: 2.9065 | LR: 1.47e-04 | 1387.9ms/step | 14.4 samples/s +Step 28240/44000 | Loss: 2.8871 | LR: 1.47e-04 | 1388.5ms/step | 14.4 samples/s +Step 28250/44000 | Loss: 2.9248 | LR: 1.47e-04 | 1388.4ms/step | 14.4 samples/s +Step 28260/44000 | Loss: 2.8884 | LR: 1.46e-04 | 1388.0ms/step | 14.4 samples/s +Step 28270/44000 | Loss: 2.8830 | LR: 1.46e-04 | 1388.6ms/step | 14.4 samples/s +Step 28280/44000 | Loss: 2.9037 | LR: 1.46e-04 | 1386.5ms/step | 14.4 samples/s +Step 28290/44000 | Loss: 2.8891 | LR: 1.46e-04 | 1386.9ms/step | 14.4 samples/s +Step 28300/44000 | Loss: 2.8547 | LR: 1.46e-04 | 1389.9ms/step | 14.4 samples/s +Step 28310/44000 | Loss: 2.9036 | LR: 1.46e-04 | 1390.3ms/step | 14.4 samples/s +Step 28320/44000 | Loss: 2.8940 | LR: 1.45e-04 | 1388.7ms/step | 14.4 samples/s +Step 28330/44000 | Loss: 2.8740 | LR: 1.45e-04 | 1387.4ms/step | 14.4 samples/s +Step 28340/44000 | Loss: 2.9064 | LR: 1.45e-04 | 1387.5ms/step | 14.4 samples/s +Step 28350/44000 | Loss: 2.9238 | LR: 1.45e-04 | 1388.3ms/step | 14.4 samples/s +Step 28360/44000 | Loss: 2.9078 | LR: 1.45e-04 | 1390.0ms/step | 14.4 samples/s +Step 28370/44000 | Loss: 2.8919 | LR: 1.45e-04 | 1388.4ms/step | 14.4 samples/s +Step 28380/44000 | Loss: 2.9038 | LR: 1.44e-04 | 1391.0ms/step | 14.4 samples/s +Step 28390/44000 | Loss: 2.9285 | LR: 1.44e-04 | 1388.2ms/step | 14.4 samples/s +Step 28400/44000 | Loss: 2.8904 | LR: 1.44e-04 | 1386.5ms/step | 14.4 samples/s +Step 28410/44000 | Loss: 2.8781 | LR: 1.44e-04 | 1389.8ms/step | 14.4 samples/s +Step 28420/44000 | Loss: 2.8929 | LR: 1.44e-04 | 1387.3ms/step | 14.4 samples/s +Step 28430/44000 | Loss: 2.8968 | LR: 1.44e-04 | 1390.3ms/step | 14.4 samples/s +Step 28440/44000 | Loss: 2.9232 | LR: 1.43e-04 | 1388.4ms/step | 14.4 samples/s +Step 28450/44000 | Loss: 2.8913 | LR: 1.43e-04 | 1388.2ms/step | 14.4 samples/s +Step 28460/44000 | Loss: 2.8802 | LR: 1.43e-04 | 1389.8ms/step | 14.4 samples/s +Step 28470/44000 | Loss: 2.8744 | LR: 1.43e-04 | 1390.1ms/step | 14.4 samples/s +Step 28480/44000 | Loss: 2.8916 | LR: 1.43e-04 | 1388.4ms/step | 14.4 samples/s +Step 28490/44000 | Loss: 2.8928 | LR: 1.43e-04 | 1388.6ms/step | 14.4 samples/s +Step 28500/44000 | Loss: 2.8918 | LR: 1.42e-04 | 1388.1ms/step | 14.4 samples/s +Step 28510/44000 | Loss: 2.9014 | LR: 1.42e-04 | 1388.3ms/step | 14.4 samples/s +Step 28520/44000 | Loss: 2.8855 | LR: 1.42e-04 | 1387.4ms/step | 14.4 samples/s +Step 28530/44000 | Loss: 2.8978 | LR: 1.42e-04 | 1390.9ms/step | 14.4 samples/s +Step 28540/44000 | Loss: 2.8654 | LR: 1.42e-04 | 1389.5ms/step | 14.4 samples/s +Step 28550/44000 | Loss: 2.8617 | LR: 1.42e-04 | 1389.6ms/step | 14.4 samples/s +Step 28560/44000 | Loss: 2.8934 | LR: 1.41e-04 | 1392.7ms/step | 14.4 samples/s +Step 28570/44000 | Loss: 2.8677 | LR: 1.41e-04 | 1389.6ms/step | 14.4 samples/s +Step 28580/44000 | Loss: 2.9139 | LR: 1.41e-04 | 1389.0ms/step | 14.4 samples/s +Step 28590/44000 | Loss: 2.8514 | LR: 1.41e-04 | 1389.0ms/step | 14.4 samples/s +Step 28600/44000 | Loss: 2.8896 | LR: 1.41e-04 | 1387.1ms/step | 14.4 samples/s +Step 28610/44000 | Loss: 2.8986 | LR: 1.41e-04 | 1388.9ms/step | 14.4 samples/s +Step 28620/44000 | Loss: 2.8613 | LR: 1.40e-04 | 1388.8ms/step | 14.4 samples/s +Step 28630/44000 | Loss: 2.9111 | LR: 1.40e-04 | 1390.3ms/step | 14.4 samples/s +Step 28640/44000 | Loss: 2.8862 | LR: 1.40e-04 | 1390.0ms/step | 14.4 samples/s +Step 28650/44000 | Loss: 2.8828 | LR: 1.40e-04 | 1389.2ms/step | 14.4 samples/s +Step 28660/44000 | Loss: 2.9386 | LR: 1.40e-04 | 1389.7ms/step | 14.4 samples/s +Step 28670/44000 | Loss: 2.9358 | LR: 1.40e-04 | 1388.3ms/step | 14.4 samples/s +Step 28680/44000 | Loss: 2.8718 | LR: 1.39e-04 | 1389.3ms/step | 14.4 samples/s +Step 28690/44000 | Loss: 2.8894 | LR: 1.39e-04 | 1390.5ms/step | 14.4 samples/s +Step 28700/44000 | Loss: 2.8986 | LR: 1.39e-04 | 1388.7ms/step | 14.4 samples/s +Step 28710/44000 | Loss: 2.9227 | LR: 1.39e-04 | 1386.3ms/step | 14.4 samples/s +Step 28720/44000 | Loss: 2.8455 | LR: 1.39e-04 | 1390.2ms/step | 14.4 samples/s +Step 28730/44000 | Loss: 2.8641 | LR: 1.39e-04 | 1387.7ms/step | 14.4 samples/s +Step 28740/44000 | Loss: 2.8677 | LR: 1.38e-04 | 1389.2ms/step | 14.4 samples/s +Step 28750/44000 | Loss: 2.8918 | LR: 1.38e-04 | 1386.1ms/step | 14.4 samples/s +Step 28760/44000 | Loss: 2.9200 | LR: 1.38e-04 | 1386.1ms/step | 14.4 samples/s +Step 28770/44000 | Loss: 2.8805 | LR: 1.38e-04 | 1386.7ms/step | 14.4 samples/s +Step 28780/44000 | Loss: 2.9067 | LR: 1.38e-04 | 1388.3ms/step | 14.4 samples/s +Step 28790/44000 | Loss: 2.9107 | LR: 1.38e-04 | 1386.4ms/step | 14.4 samples/s +Step 28800/44000 | Loss: 2.9115 | LR: 1.38e-04 | 1388.5ms/step | 14.4 samples/s +Step 28810/44000 | Loss: 2.9154 | LR: 1.37e-04 | 1389.7ms/step | 14.4 samples/s +Step 28820/44000 | Loss: 2.8951 | LR: 1.37e-04 | 1388.9ms/step | 14.4 samples/s +Step 28830/44000 | Loss: 2.8989 | LR: 1.37e-04 | 1390.2ms/step | 14.4 samples/s +Step 28840/44000 | Loss: 2.8786 | LR: 1.37e-04 | 1389.8ms/step | 14.4 samples/s +Step 28850/44000 | Loss: 2.8838 | LR: 1.37e-04 | 1388.0ms/step | 14.4 samples/s +Step 28860/44000 | Loss: 2.8934 | LR: 1.37e-04 | 1390.4ms/step | 14.4 samples/s +Step 28870/44000 | Loss: 2.8777 | LR: 1.36e-04 | 1388.0ms/step | 14.4 samples/s +Step 28880/44000 | Loss: 2.9050 | LR: 1.36e-04 | 1389.1ms/step | 14.4 samples/s +Step 28890/44000 | Loss: 2.9043 | LR: 1.36e-04 | 1390.7ms/step | 14.4 samples/s +Step 28900/44000 | Loss: 2.9026 | LR: 1.36e-04 | 1389.2ms/step | 14.4 samples/s +Step 28910/44000 | Loss: 2.8989 | LR: 1.36e-04 | 1388.2ms/step | 14.4 samples/s +Step 28920/44000 | Loss: 2.8994 | LR: 1.36e-04 | 1390.9ms/step | 14.4 samples/s +Step 28930/44000 | Loss: 2.8867 | LR: 1.35e-04 | 1390.7ms/step | 14.4 samples/s +Step 28940/44000 | Loss: 2.9041 | LR: 1.35e-04 | 1389.3ms/step | 14.4 samples/s +Step 28950/44000 | Loss: 2.8846 | LR: 1.35e-04 | 1387.6ms/step | 14.4 samples/s +Step 28960/44000 | Loss: 2.8940 | LR: 1.35e-04 | 1387.6ms/step | 14.4 samples/s +Step 28970/44000 | Loss: 2.8953 | LR: 1.35e-04 | 1388.6ms/step | 14.4 samples/s +Step 28980/44000 | Loss: 2.9191 | LR: 1.35e-04 | 1388.2ms/step | 14.4 samples/s +Step 28990/44000 | Loss: 2.8847 | LR: 1.34e-04 | 1387.6ms/step | 14.4 samples/s +Step 29000/44000 | Loss: 2.8841 | LR: 1.34e-04 | 1388.2ms/step | 14.4 samples/s +Validation at step 29000: Loss = 2.9101, Perplexity = 18.36 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-29000 +Step 29010/44000 | Loss: 2.9114 | LR: 1.34e-04 | 3870.0ms/step | 5.2 samples/s +Step 29020/44000 | Loss: 2.8950 | LR: 1.34e-04 | 1386.2ms/step | 14.4 samples/s +Step 29030/44000 | Loss: 2.8918 | LR: 1.34e-04 | 1389.7ms/step | 14.4 samples/s +Step 29040/44000 | Loss: 2.9111 | LR: 1.34e-04 | 1392.2ms/step | 14.4 samples/s +Step 29050/44000 | Loss: 2.9091 | LR: 1.33e-04 | 1391.5ms/step | 14.4 samples/s +Step 29060/44000 | Loss: 2.9006 | LR: 1.33e-04 | 1394.6ms/step | 14.3 samples/s +Step 29070/44000 | Loss: 2.8924 | LR: 1.33e-04 | 1391.8ms/step | 14.4 samples/s +Step 29080/44000 | Loss: 2.9101 | LR: 1.33e-04 | 1392.1ms/step | 14.4 samples/s +Step 29090/44000 | Loss: 2.9258 | LR: 1.33e-04 | 1391.9ms/step | 14.4 samples/s +Step 29100/44000 | Loss: 2.8619 | LR: 1.33e-04 | 1390.3ms/step | 14.4 samples/s +Step 29110/44000 | Loss: 2.8986 | LR: 1.33e-04 | 1392.2ms/step | 14.4 samples/s +Step 29120/44000 | Loss: 2.9033 | LR: 1.32e-04 | 1390.7ms/step | 14.4 samples/s +Step 29130/44000 | Loss: 2.8942 | LR: 1.32e-04 | 1391.3ms/step | 14.4 samples/s +Step 29140/44000 | Loss: 2.8680 | LR: 1.32e-04 | 1389.6ms/step | 14.4 samples/s +Step 29150/44000 | Loss: 2.8945 | LR: 1.32e-04 | 1390.1ms/step | 14.4 samples/s +Step 29160/44000 | Loss: 2.9038 | LR: 1.32e-04 | 1391.6ms/step | 14.4 samples/s +Step 29170/44000 | Loss: 2.8750 | LR: 1.32e-04 | 1391.6ms/step | 14.4 samples/s +Step 29180/44000 | Loss: 2.8602 | LR: 1.31e-04 | 1389.5ms/step | 14.4 samples/s +Step 29190/44000 | Loss: 2.9160 | LR: 1.31e-04 | 1387.7ms/step | 14.4 samples/s +Step 29200/44000 | Loss: 2.8628 | LR: 1.31e-04 | 1389.7ms/step | 14.4 samples/s +Step 29210/44000 | Loss: 2.8926 | LR: 1.31e-04 | 1394.3ms/step | 14.3 samples/s +Step 29220/44000 | Loss: 2.9184 | LR: 1.31e-04 | 1391.8ms/step | 14.4 samples/s +Step 29230/44000 | Loss: 2.9383 | LR: 1.31e-04 | 1393.7ms/step | 14.3 samples/s +Step 29240/44000 | Loss: 2.9057 | LR: 1.30e-04 | 1393.2ms/step | 14.4 samples/s +Step 29250/44000 | Loss: 2.9065 | LR: 1.30e-04 | 1390.8ms/step | 14.4 samples/s +Step 29260/44000 | Loss: 2.8726 | LR: 1.30e-04 | 1390.2ms/step | 14.4 samples/s +Step 29270/44000 | Loss: 2.8713 | LR: 1.30e-04 | 1390.2ms/step | 14.4 samples/s +Step 29280/44000 | Loss: 2.8890 | LR: 1.30e-04 | 1389.8ms/step | 14.4 samples/s +Step 29290/44000 | Loss: 2.8910 | LR: 1.30e-04 | 1389.9ms/step | 14.4 samples/s +Step 29300/44000 | Loss: 2.8895 | LR: 1.29e-04 | 1390.1ms/step | 14.4 samples/s +Step 29310/44000 | Loss: 2.9014 | LR: 1.29e-04 | 1392.0ms/step | 14.4 samples/s +Step 29320/44000 | Loss: 2.9060 | LR: 1.29e-04 | 1390.0ms/step | 14.4 samples/s +Step 29330/44000 | Loss: 2.8936 | LR: 1.29e-04 | 1390.6ms/step | 14.4 samples/s +Step 29340/44000 | Loss: 2.8685 | LR: 1.29e-04 | 1389.5ms/step | 14.4 samples/s +Step 29350/44000 | Loss: 2.8956 | LR: 1.29e-04 | 1389.8ms/step | 14.4 samples/s +Step 29360/44000 | Loss: 2.8908 | LR: 1.29e-04 | 1390.4ms/step | 14.4 samples/s +Step 29370/44000 | Loss: 2.9028 | LR: 1.28e-04 | 1392.0ms/step | 14.4 samples/s +Step 29380/44000 | Loss: 2.8984 | LR: 1.28e-04 | 1389.2ms/step | 14.4 samples/s +Step 29390/44000 | Loss: 2.8711 | LR: 1.28e-04 | 1387.2ms/step | 14.4 samples/s +Step 29400/44000 | Loss: 2.8709 | LR: 1.28e-04 | 1391.5ms/step | 14.4 samples/s +Step 29410/44000 | Loss: 2.8617 | LR: 1.28e-04 | 1393.7ms/step | 14.4 samples/s +Step 29420/44000 | Loss: 2.8856 | LR: 1.28e-04 | 1391.6ms/step | 14.4 samples/s +Step 29430/44000 | Loss: 2.8784 | LR: 1.27e-04 | 1392.6ms/step | 14.4 samples/s +Step 29440/44000 | Loss: 2.8949 | LR: 1.27e-04 | 1392.1ms/step | 14.4 samples/s +Step 29450/44000 | Loss: 2.8836 | LR: 1.27e-04 | 1393.0ms/step | 14.4 samples/s +Step 29460/44000 | Loss: 2.8763 | LR: 1.27e-04 | 1393.0ms/step | 14.4 samples/s +Step 29470/44000 | Loss: 2.8637 | LR: 1.27e-04 | 1391.7ms/step | 14.4 samples/s +Step 29480/44000 | Loss: 2.8890 | LR: 1.27e-04 | 1390.9ms/step | 14.4 samples/s +Step 29490/44000 | Loss: 2.8724 | LR: 1.26e-04 | 1390.2ms/step | 14.4 samples/s +Step 29500/44000 | Loss: 2.8640 | LR: 1.26e-04 | 1393.0ms/step | 14.4 samples/s +Step 29510/44000 | Loss: 2.8967 | LR: 1.26e-04 | 1394.4ms/step | 14.3 samples/s +Step 29520/44000 | Loss: 2.9036 | LR: 1.26e-04 | 1390.2ms/step | 14.4 samples/s +Step 29530/44000 | Loss: 2.8940 | LR: 1.26e-04 | 1389.5ms/step | 14.4 samples/s +Step 29540/44000 | Loss: 2.8883 | LR: 1.26e-04 | 1391.0ms/step | 14.4 samples/s +Step 29550/44000 | Loss: 2.8595 | LR: 1.26e-04 | 1391.4ms/step | 14.4 samples/s +Step 29560/44000 | Loss: 2.8839 | LR: 1.25e-04 | 1393.5ms/step | 14.4 samples/s +Step 29570/44000 | Loss: 2.8899 | LR: 1.25e-04 | 1394.3ms/step | 14.3 samples/s +Step 29580/44000 | Loss: 2.8870 | LR: 1.25e-04 | 1392.0ms/step | 14.4 samples/s +Step 29590/44000 | Loss: 2.8852 | LR: 1.25e-04 | 1390.2ms/step | 14.4 samples/s +Step 29600/44000 | Loss: 2.8603 | LR: 1.25e-04 | 1390.2ms/step | 14.4 samples/s +Step 29610/44000 | Loss: 2.8868 | LR: 1.25e-04 | 1391.1ms/step | 14.4 samples/s +Step 29620/44000 | Loss: 2.8879 | LR: 1.24e-04 | 1391.4ms/step | 14.4 samples/s +Step 29630/44000 | Loss: 2.8813 | LR: 1.24e-04 | 1390.6ms/step | 14.4 samples/s +Step 29640/44000 | Loss: 2.8419 | LR: 1.24e-04 | 1392.5ms/step | 14.4 samples/s +Step 29650/44000 | Loss: 2.8661 | LR: 1.24e-04 | 1394.7ms/step | 14.3 samples/s +Step 29660/44000 | Loss: 2.9160 | LR: 1.24e-04 | 1393.6ms/step | 14.4 samples/s +Step 29670/44000 | Loss: 2.8800 | LR: 1.24e-04 | 1394.4ms/step | 14.3 samples/s +Step 29680/44000 | Loss: 2.8881 | LR: 1.23e-04 | 1391.5ms/step | 14.4 samples/s +Step 29690/44000 | Loss: 2.8894 | LR: 1.23e-04 | 1392.1ms/step | 14.4 samples/s +Step 29700/44000 | Loss: 2.8863 | LR: 1.23e-04 | 1391.7ms/step | 14.4 samples/s +Step 29710/44000 | Loss: 2.8972 | LR: 1.23e-04 | 1388.1ms/step | 14.4 samples/s +Step 29720/44000 | Loss: 2.8919 | LR: 1.23e-04 | 1391.4ms/step | 14.4 samples/s +Step 29730/44000 | Loss: 2.8928 | LR: 1.23e-04 | 1388.1ms/step | 14.4 samples/s +Step 29740/44000 | Loss: 2.8495 | LR: 1.23e-04 | 1390.8ms/step | 14.4 samples/s +Step 29750/44000 | Loss: 2.8779 | LR: 1.22e-04 | 1389.7ms/step | 14.4 samples/s +Step 29760/44000 | Loss: 2.8786 | LR: 1.22e-04 | 1389.0ms/step | 14.4 samples/s +Step 29770/44000 | Loss: 2.9072 | LR: 1.22e-04 | 1381.7ms/step | 14.5 samples/s +Step 29780/44000 | Loss: 2.9113 | LR: 1.22e-04 | 1383.7ms/step | 14.5 samples/s +Step 29790/44000 | Loss: 2.8857 | LR: 1.22e-04 | 1385.9ms/step | 14.4 samples/s +Step 29800/44000 | Loss: 2.9218 | LR: 1.22e-04 | 1387.1ms/step | 14.4 samples/s +Step 29810/44000 | Loss: 2.8822 | LR: 1.21e-04 | 1384.8ms/step | 14.4 samples/s +Step 29820/44000 | Loss: 2.8558 | LR: 1.21e-04 | 1387.9ms/step | 14.4 samples/s +Step 29830/44000 | Loss: 2.8728 | LR: 1.21e-04 | 1392.2ms/step | 14.4 samples/s +Step 29840/44000 | Loss: 2.8763 | LR: 1.21e-04 | 1389.6ms/step | 14.4 samples/s +Step 29850/44000 | Loss: 2.8480 | LR: 1.21e-04 | 1388.9ms/step | 14.4 samples/s +Step 29860/44000 | Loss: 2.9055 | LR: 1.21e-04 | 1389.3ms/step | 14.4 samples/s +Step 29870/44000 | Loss: 2.9019 | LR: 1.21e-04 | 1392.4ms/step | 14.4 samples/s +Step 29880/44000 | Loss: 2.8795 | LR: 1.20e-04 | 1389.9ms/step | 14.4 samples/s +Step 29890/44000 | Loss: 2.8969 | LR: 1.20e-04 | 1390.8ms/step | 14.4 samples/s +Step 29900/44000 | Loss: 2.9183 | LR: 1.20e-04 | 1393.2ms/step | 14.4 samples/s +Step 29910/44000 | Loss: 2.9204 | LR: 1.20e-04 | 1390.2ms/step | 14.4 samples/s +Step 29920/44000 | Loss: 2.8996 | LR: 1.20e-04 | 1390.0ms/step | 14.4 samples/s +Step 29930/44000 | Loss: 2.8786 | LR: 1.20e-04 | 1390.0ms/step | 14.4 samples/s +Step 29940/44000 | Loss: 2.9008 | LR: 1.19e-04 | 1388.1ms/step | 14.4 samples/s +Step 29950/44000 | Loss: 2.8870 | LR: 1.19e-04 | 1389.8ms/step | 14.4 samples/s +Step 29960/44000 | Loss: 2.9071 | LR: 1.19e-04 | 1389.7ms/step | 14.4 samples/s +Step 29970/44000 | Loss: 2.8632 | LR: 1.19e-04 | 1388.9ms/step | 14.4 samples/s +Step 29980/44000 | Loss: 2.8613 | LR: 1.19e-04 | 1389.0ms/step | 14.4 samples/s +Step 29990/44000 | Loss: 2.8889 | LR: 1.19e-04 | 1394.3ms/step | 14.3 samples/s +Step 30000/44000 | Loss: 2.8997 | LR: 1.19e-04 | 1392.8ms/step | 14.4 samples/s +Validation at step 30000: Loss = 2.9055, Perplexity = 18.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-30000 +Step 30010/44000 | Loss: 2.8375 | LR: 1.18e-04 | 3889.5ms/step | 5.1 samples/s +Step 30020/44000 | Loss: 2.8893 | LR: 1.18e-04 | 1384.1ms/step | 14.5 samples/s +Step 30030/44000 | Loss: 2.8680 | LR: 1.18e-04 | 1383.9ms/step | 14.5 samples/s +Step 30040/44000 | Loss: 2.8942 | LR: 1.18e-04 | 1385.8ms/step | 14.4 samples/s +Step 30050/44000 | Loss: 2.8807 | LR: 1.18e-04 | 1386.2ms/step | 14.4 samples/s +Step 30060/44000 | Loss: 2.9115 | LR: 1.18e-04 | 1388.5ms/step | 14.4 samples/s +Step 30070/44000 | Loss: 2.9025 | LR: 1.17e-04 | 1388.1ms/step | 14.4 samples/s +Step 30080/44000 | Loss: 2.9155 | LR: 1.17e-04 | 1390.3ms/step | 14.4 samples/s +Step 30090/44000 | Loss: 2.8823 | LR: 1.17e-04 | 1387.3ms/step | 14.4 samples/s +Step 30100/44000 | Loss: 2.8566 | LR: 1.17e-04 | 1387.9ms/step | 14.4 samples/s +Step 30110/44000 | Loss: 2.8638 | LR: 1.17e-04 | 1387.0ms/step | 14.4 samples/s +Step 30120/44000 | Loss: 2.8741 | LR: 1.17e-04 | 1387.1ms/step | 14.4 samples/s +Step 30130/44000 | Loss: 2.8752 | LR: 1.17e-04 | 1386.7ms/step | 14.4 samples/s +Step 30140/44000 | Loss: 2.8715 | LR: 1.16e-04 | 1387.6ms/step | 14.4 samples/s +Step 30150/44000 | Loss: 2.8878 | LR: 1.16e-04 | 1388.5ms/step | 14.4 samples/s +Step 30160/44000 | Loss: 2.8867 | LR: 1.16e-04 | 1388.6ms/step | 14.4 samples/s +Step 30170/44000 | Loss: 2.8818 | LR: 1.16e-04 | 1388.4ms/step | 14.4 samples/s +Step 30180/44000 | Loss: 2.8944 | LR: 1.16e-04 | 1390.1ms/step | 14.4 samples/s +Step 30190/44000 | Loss: 2.8701 | LR: 1.16e-04 | 1389.0ms/step | 14.4 samples/s +Step 30200/44000 | Loss: 2.8962 | LR: 1.15e-04 | 1387.2ms/step | 14.4 samples/s +Step 30210/44000 | Loss: 2.8872 | LR: 1.15e-04 | 1391.6ms/step | 14.4 samples/s +Step 30220/44000 | Loss: 2.8797 | LR: 1.15e-04 | 1387.2ms/step | 14.4 samples/s +Step 30230/44000 | Loss: 2.8959 | LR: 1.15e-04 | 1390.9ms/step | 14.4 samples/s +Step 30240/44000 | Loss: 2.8836 | LR: 1.15e-04 | 1390.8ms/step | 14.4 samples/s +Step 30250/44000 | Loss: 2.8929 | LR: 1.15e-04 | 1389.5ms/step | 14.4 samples/s +Step 30260/44000 | Loss: 2.8959 | LR: 1.15e-04 | 1390.1ms/step | 14.4 samples/s +Step 30270/44000 | Loss: 2.8810 | LR: 1.14e-04 | 1389.9ms/step | 14.4 samples/s +Step 30280/44000 | Loss: 2.8956 | LR: 1.14e-04 | 1388.1ms/step | 14.4 samples/s +Step 30290/44000 | Loss: 2.8955 | LR: 1.14e-04 | 1390.1ms/step | 14.4 samples/s +Step 30300/44000 | Loss: 2.8862 | LR: 1.14e-04 | 1389.6ms/step | 14.4 samples/s +Step 30310/44000 | Loss: 2.8888 | LR: 1.14e-04 | 1387.4ms/step | 14.4 samples/s +Step 30320/44000 | Loss: 2.9055 | LR: 1.14e-04 | 1387.0ms/step | 14.4 samples/s +Step 30330/44000 | Loss: 2.8977 | LR: 1.13e-04 | 1387.9ms/step | 14.4 samples/s +Step 30340/44000 | Loss: 2.8795 | LR: 1.13e-04 | 1389.6ms/step | 14.4 samples/s +Step 30350/44000 | Loss: 2.8914 | LR: 1.13e-04 | 1387.7ms/step | 14.4 samples/s +Step 30360/44000 | Loss: 2.8886 | LR: 1.13e-04 | 1389.8ms/step | 14.4 samples/s +Step 30370/44000 | Loss: 2.9092 | LR: 1.13e-04 | 1391.8ms/step | 14.4 samples/s +Step 30380/44000 | Loss: 2.8974 | LR: 1.13e-04 | 1389.6ms/step | 14.4 samples/s +Step 30390/44000 | Loss: 2.8653 | LR: 1.13e-04 | 1386.4ms/step | 14.4 samples/s +Step 30400/44000 | Loss: 2.8737 | LR: 1.12e-04 | 1389.3ms/step | 14.4 samples/s +Step 30410/44000 | Loss: 2.9015 | LR: 1.12e-04 | 1389.8ms/step | 14.4 samples/s +Step 30420/44000 | Loss: 2.9023 | LR: 1.12e-04 | 1387.6ms/step | 14.4 samples/s +Step 30430/44000 | Loss: 2.8922 | LR: 1.12e-04 | 1388.6ms/step | 14.4 samples/s +Step 30440/44000 | Loss: 2.9127 | LR: 1.12e-04 | 1388.2ms/step | 14.4 samples/s +Step 30450/44000 | Loss: 2.8889 | LR: 1.12e-04 | 1387.6ms/step | 14.4 samples/s +Step 30460/44000 | Loss: 2.8951 | LR: 1.11e-04 | 1387.8ms/step | 14.4 samples/s +Step 30470/44000 | Loss: 2.8690 | LR: 1.11e-04 | 1387.2ms/step | 14.4 samples/s +Step 30480/44000 | Loss: 2.9215 | LR: 1.11e-04 | 1387.3ms/step | 14.4 samples/s +Step 30490/44000 | Loss: 2.9014 | LR: 1.11e-04 | 1387.6ms/step | 14.4 samples/s +Step 30500/44000 | Loss: 2.8717 | LR: 1.11e-04 | 1386.5ms/step | 14.4 samples/s +Step 30510/44000 | Loss: 2.8900 | LR: 1.11e-04 | 1385.8ms/step | 14.4 samples/s +Step 30520/44000 | Loss: 2.8814 | LR: 1.11e-04 | 1387.5ms/step | 14.4 samples/s +Step 30530/44000 | Loss: 2.8928 | LR: 1.10e-04 | 1388.5ms/step | 14.4 samples/s +Step 30540/44000 | Loss: 2.9057 | LR: 1.10e-04 | 1386.2ms/step | 14.4 samples/s +Step 30550/44000 | Loss: 2.8901 | LR: 1.10e-04 | 1383.7ms/step | 14.5 samples/s +Step 30560/44000 | Loss: 2.8716 | LR: 1.10e-04 | 1382.4ms/step | 14.5 samples/s +Step 30570/44000 | Loss: 2.8672 | LR: 1.10e-04 | 1384.3ms/step | 14.4 samples/s +Step 30580/44000 | Loss: 2.8989 | LR: 1.10e-04 | 1384.4ms/step | 14.4 samples/s +Step 30590/44000 | Loss: 2.8541 | LR: 1.10e-04 | 1384.5ms/step | 14.4 samples/s +Step 30600/44000 | Loss: 2.8897 | LR: 1.09e-04 | 1386.6ms/step | 14.4 samples/s +Step 30610/44000 | Loss: 2.8698 | LR: 1.09e-04 | 1385.3ms/step | 14.4 samples/s +Step 30620/44000 | Loss: 2.8835 | LR: 1.09e-04 | 1387.2ms/step | 14.4 samples/s +Step 30630/44000 | Loss: 2.8691 | LR: 1.09e-04 | 1387.4ms/step | 14.4 samples/s +Step 30640/44000 | Loss: 2.8879 | LR: 1.09e-04 | 1386.6ms/step | 14.4 samples/s +Step 30650/44000 | Loss: 2.8941 | LR: 1.09e-04 | 1387.3ms/step | 14.4 samples/s +Step 30660/44000 | Loss: 2.8950 | LR: 1.08e-04 | 1388.2ms/step | 14.4 samples/s +Step 30670/44000 | Loss: 2.8697 | LR: 1.08e-04 | 1389.2ms/step | 14.4 samples/s +Step 30680/44000 | Loss: 2.8795 | LR: 1.08e-04 | 1389.1ms/step | 14.4 samples/s +Step 30690/44000 | Loss: 2.9185 | LR: 1.08e-04 | 1389.7ms/step | 14.4 samples/s +Step 30700/44000 | Loss: 2.8992 | LR: 1.08e-04 | 1393.7ms/step | 14.4 samples/s +Step 30710/44000 | Loss: 2.8831 | LR: 1.08e-04 | 1390.2ms/step | 14.4 samples/s +Step 30720/44000 | Loss: 2.8670 | LR: 1.08e-04 | 1390.2ms/step | 14.4 samples/s +Step 30730/44000 | Loss: 2.8977 | LR: 1.07e-04 | 1386.7ms/step | 14.4 samples/s +Step 30740/44000 | Loss: 2.8798 | LR: 1.07e-04 | 1388.8ms/step | 14.4 samples/s +Step 30750/44000 | Loss: 2.8600 | LR: 1.07e-04 | 1388.3ms/step | 14.4 samples/s +Step 30760/44000 | Loss: 2.8990 | LR: 1.07e-04 | 1388.0ms/step | 14.4 samples/s +Step 30770/44000 | Loss: 2.9046 | LR: 1.07e-04 | 1388.5ms/step | 14.4 samples/s +Step 30780/44000 | Loss: 2.8626 | LR: 1.07e-04 | 1389.6ms/step | 14.4 samples/s +Step 30790/44000 | Loss: 2.8802 | LR: 1.07e-04 | 1389.8ms/step | 14.4 samples/s +Step 30800/44000 | Loss: 2.8739 | LR: 1.06e-04 | 1388.6ms/step | 14.4 samples/s +Step 30810/44000 | Loss: 2.8996 | LR: 1.06e-04 | 1389.9ms/step | 14.4 samples/s +Step 30820/44000 | Loss: 2.9075 | LR: 1.06e-04 | 1388.1ms/step | 14.4 samples/s +Step 30830/44000 | Loss: 2.8712 | LR: 1.06e-04 | 1389.7ms/step | 14.4 samples/s +Step 30840/44000 | Loss: 2.9017 | LR: 1.06e-04 | 1389.6ms/step | 14.4 samples/s +Step 30850/44000 | Loss: 2.8828 | LR: 1.06e-04 | 1385.7ms/step | 14.4 samples/s +Step 30860/44000 | Loss: 2.8802 | LR: 1.05e-04 | 1387.0ms/step | 14.4 samples/s +Step 30870/44000 | Loss: 2.9094 | LR: 1.05e-04 | 1389.9ms/step | 14.4 samples/s +Step 30880/44000 | Loss: 2.9025 | LR: 1.05e-04 | 1387.9ms/step | 14.4 samples/s +Step 30890/44000 | Loss: 2.8913 | LR: 1.05e-04 | 1387.9ms/step | 14.4 samples/s +Step 30900/44000 | Loss: 2.8944 | LR: 1.05e-04 | 1390.3ms/step | 14.4 samples/s +Step 30910/44000 | Loss: 2.8731 | LR: 1.05e-04 | 1390.8ms/step | 14.4 samples/s +Step 30920/44000 | Loss: 2.8818 | LR: 1.05e-04 | 1389.2ms/step | 14.4 samples/s +Step 30930/44000 | Loss: 2.8713 | LR: 1.04e-04 | 1389.9ms/step | 14.4 samples/s +Step 30940/44000 | Loss: 2.8861 | LR: 1.04e-04 | 1389.8ms/step | 14.4 samples/s +Step 30950/44000 | Loss: 2.8797 | LR: 1.04e-04 | 1386.7ms/step | 14.4 samples/s +Step 30960/44000 | Loss: 2.9038 | LR: 1.04e-04 | 1387.5ms/step | 14.4 samples/s +Step 30970/44000 | Loss: 2.8796 | LR: 1.04e-04 | 1384.6ms/step | 14.4 samples/s +Step 30980/44000 | Loss: 2.8631 | LR: 1.04e-04 | 1381.1ms/step | 14.5 samples/s +Step 30990/44000 | Loss: 2.8849 | LR: 1.04e-04 | 1386.4ms/step | 14.4 samples/s +Step 31000/44000 | Loss: 2.9002 | LR: 1.03e-04 | 1387.1ms/step | 14.4 samples/s +Validation at step 31000: Loss = 2.9021, Perplexity = 18.21 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-31000 +Step 31010/44000 | Loss: 2.8710 | LR: 1.03e-04 | 3870.9ms/step | 5.2 samples/s +Step 31020/44000 | Loss: 2.8847 | LR: 1.03e-04 | 1380.6ms/step | 14.5 samples/s +Step 31030/44000 | Loss: 2.8715 | LR: 1.03e-04 | 1379.7ms/step | 14.5 samples/s +Step 31040/44000 | Loss: 2.9058 | LR: 1.03e-04 | 1384.3ms/step | 14.4 samples/s +Step 31050/44000 | Loss: 2.8673 | LR: 1.03e-04 | 1383.9ms/step | 14.5 samples/s +Step 31060/44000 | Loss: 2.9049 | LR: 1.03e-04 | 1385.2ms/step | 14.4 samples/s +Step 31070/44000 | Loss: 2.8600 | LR: 1.02e-04 | 1388.4ms/step | 14.4 samples/s +Step 31080/44000 | Loss: 2.8839 | LR: 1.02e-04 | 1387.9ms/step | 14.4 samples/s +Step 31090/44000 | Loss: 2.8774 | LR: 1.02e-04 | 1386.0ms/step | 14.4 samples/s +Step 31100/44000 | Loss: 2.8729 | LR: 1.02e-04 | 1386.8ms/step | 14.4 samples/s +Step 31110/44000 | Loss: 2.8844 | LR: 1.02e-04 | 1387.2ms/step | 14.4 samples/s +Step 31120/44000 | Loss: 2.8856 | LR: 1.02e-04 | 1388.2ms/step | 14.4 samples/s +Step 31130/44000 | Loss: 2.8547 | LR: 1.02e-04 | 1387.7ms/step | 14.4 samples/s +Step 31140/44000 | Loss: 2.8889 | LR: 1.01e-04 | 1391.3ms/step | 14.4 samples/s +Step 31150/44000 | Loss: 2.8796 | LR: 1.01e-04 | 1390.1ms/step | 14.4 samples/s +Step 31160/44000 | Loss: 2.8610 | LR: 1.01e-04 | 1388.6ms/step | 14.4 samples/s +Step 31170/44000 | Loss: 2.8617 | LR: 1.01e-04 | 1390.2ms/step | 14.4 samples/s +Step 31180/44000 | Loss: 2.8988 | LR: 1.01e-04 | 1390.6ms/step | 14.4 samples/s +Step 31190/44000 | Loss: 2.8807 | LR: 1.01e-04 | 1389.5ms/step | 14.4 samples/s +Step 31200/44000 | Loss: 2.8703 | LR: 1.00e-04 | 1389.0ms/step | 14.4 samples/s +Step 31210/44000 | Loss: 2.8781 | LR: 1.00e-04 | 1390.9ms/step | 14.4 samples/s +Step 31220/44000 | Loss: 2.8824 | LR: 1.00e-04 | 1391.7ms/step | 14.4 samples/s +Step 31230/44000 | Loss: 2.8657 | LR: 1.00e-04 | 1387.2ms/step | 14.4 samples/s +Step 31240/44000 | Loss: 2.8647 | LR: 9.99e-05 | 1388.5ms/step | 14.4 samples/s +Step 31250/44000 | Loss: 2.8766 | LR: 9.98e-05 | 1388.1ms/step | 14.4 samples/s +Step 31260/44000 | Loss: 2.8960 | LR: 9.96e-05 | 1390.5ms/step | 14.4 samples/s +Step 31270/44000 | Loss: 2.8820 | LR: 9.95e-05 | 1389.6ms/step | 14.4 samples/s +Step 31280/44000 | Loss: 2.8885 | LR: 9.93e-05 | 1389.6ms/step | 14.4 samples/s +Step 31290/44000 | Loss: 2.8765 | LR: 9.92e-05 | 1387.6ms/step | 14.4 samples/s +Step 31300/44000 | Loss: 2.9010 | LR: 9.90e-05 | 1388.3ms/step | 14.4 samples/s +Step 31310/44000 | Loss: 2.8738 | LR: 9.89e-05 | 1386.5ms/step | 14.4 samples/s +Step 31320/44000 | Loss: 2.8699 | LR: 9.88e-05 | 1388.0ms/step | 14.4 samples/s +Step 31330/44000 | Loss: 2.8863 | LR: 9.86e-05 | 1388.1ms/step | 14.4 samples/s +Step 31340/44000 | Loss: 2.8888 | LR: 9.85e-05 | 1391.4ms/step | 14.4 samples/s +Step 31350/44000 | Loss: 2.8926 | LR: 9.83e-05 | 1387.6ms/step | 14.4 samples/s +Step 31360/44000 | Loss: 2.8904 | LR: 9.82e-05 | 1389.7ms/step | 14.4 samples/s +Step 31370/44000 | Loss: 2.8816 | LR: 9.80e-05 | 1388.7ms/step | 14.4 samples/s +Step 31380/44000 | Loss: 2.8486 | LR: 9.79e-05 | 1390.5ms/step | 14.4 samples/s +Step 31390/44000 | Loss: 2.8640 | LR: 9.77e-05 | 1388.0ms/step | 14.4 samples/s +Step 31400/44000 | Loss: 2.8953 | LR: 9.76e-05 | 1388.1ms/step | 14.4 samples/s +Step 31410/44000 | Loss: 2.8631 | LR: 9.75e-05 | 1388.7ms/step | 14.4 samples/s +Step 31420/44000 | Loss: 2.8873 | LR: 9.73e-05 | 1389.2ms/step | 14.4 samples/s +Step 31430/44000 | Loss: 2.8792 | LR: 9.72e-05 | 1388.6ms/step | 14.4 samples/s +Step 31440/44000 | Loss: 2.8629 | LR: 9.70e-05 | 1389.6ms/step | 14.4 samples/s +Step 31450/44000 | Loss: 2.8645 | LR: 9.69e-05 | 1386.8ms/step | 14.4 samples/s +Step 31460/44000 | Loss: 2.8998 | LR: 9.67e-05 | 1389.2ms/step | 14.4 samples/s +Step 31470/44000 | Loss: 2.8567 | LR: 9.66e-05 | 1390.6ms/step | 14.4 samples/s +Step 31480/44000 | Loss: 2.8650 | LR: 9.65e-05 | 1389.6ms/step | 14.4 samples/s +Step 31490/44000 | Loss: 2.8957 | LR: 9.63e-05 | 1389.9ms/step | 14.4 samples/s +Step 31500/44000 | Loss: 2.8600 | LR: 9.62e-05 | 1388.4ms/step | 14.4 samples/s +Step 31510/44000 | Loss: 2.8882 | LR: 9.60e-05 | 1391.1ms/step | 14.4 samples/s +Step 31520/44000 | Loss: 2.8774 | LR: 9.59e-05 | 1387.9ms/step | 14.4 samples/s +Step 31530/44000 | Loss: 2.8914 | LR: 9.57e-05 | 1388.4ms/step | 14.4 samples/s +Step 31540/44000 | Loss: 2.8734 | LR: 9.56e-05 | 1389.6ms/step | 14.4 samples/s +Step 31550/44000 | Loss: 2.8738 | LR: 9.54e-05 | 1390.9ms/step | 14.4 samples/s +Step 31560/44000 | Loss: 2.8673 | LR: 9.53e-05 | 1387.2ms/step | 14.4 samples/s +Step 31570/44000 | Loss: 2.8752 | LR: 9.52e-05 | 1389.0ms/step | 14.4 samples/s +Step 31580/44000 | Loss: 2.9203 | LR: 9.50e-05 | 1390.1ms/step | 14.4 samples/s +Step 31590/44000 | Loss: 2.8932 | LR: 9.49e-05 | 1389.7ms/step | 14.4 samples/s +Step 31600/44000 | Loss: 2.8756 | LR: 9.47e-05 | 1388.6ms/step | 14.4 samples/s +Step 31610/44000 | Loss: 2.8560 | LR: 9.46e-05 | 1387.7ms/step | 14.4 samples/s +Step 31620/44000 | Loss: 2.8843 | LR: 9.45e-05 | 1388.6ms/step | 14.4 samples/s +Step 31630/44000 | Loss: 2.8744 | LR: 9.43e-05 | 1387.9ms/step | 14.4 samples/s +Step 31640/44000 | Loss: 2.9025 | LR: 9.42e-05 | 1385.7ms/step | 14.4 samples/s +Step 31650/44000 | Loss: 2.8749 | LR: 9.40e-05 | 1387.4ms/step | 14.4 samples/s +Step 31660/44000 | Loss: 2.8778 | LR: 9.39e-05 | 1386.6ms/step | 14.4 samples/s +Step 31670/44000 | Loss: 2.8481 | LR: 9.37e-05 | 1385.7ms/step | 14.4 samples/s +Step 31680/44000 | Loss: 2.8957 | LR: 9.36e-05 | 1386.6ms/step | 14.4 samples/s +Step 31690/44000 | Loss: 2.8642 | LR: 9.35e-05 | 1387.8ms/step | 14.4 samples/s +Step 31700/44000 | Loss: 2.9018 | LR: 9.33e-05 | 1387.5ms/step | 14.4 samples/s +Step 31710/44000 | Loss: 2.8890 | LR: 9.32e-05 | 1388.1ms/step | 14.4 samples/s +Step 31720/44000 | Loss: 2.8939 | LR: 9.30e-05 | 1389.1ms/step | 14.4 samples/s +Step 31730/44000 | Loss: 2.8917 | LR: 9.29e-05 | 1388.2ms/step | 14.4 samples/s +Step 31740/44000 | Loss: 2.8620 | LR: 9.28e-05 | 1390.8ms/step | 14.4 samples/s +Step 31750/44000 | Loss: 2.8850 | LR: 9.26e-05 | 1387.8ms/step | 14.4 samples/s +Step 31760/44000 | Loss: 2.8831 | LR: 9.25e-05 | 1385.9ms/step | 14.4 samples/s +Step 31770/44000 | Loss: 2.8967 | LR: 9.23e-05 | 1388.0ms/step | 14.4 samples/s +Step 31780/44000 | Loss: 2.9068 | LR: 9.22e-05 | 1388.0ms/step | 14.4 samples/s +Step 31790/44000 | Loss: 2.8982 | LR: 9.20e-05 | 1388.1ms/step | 14.4 samples/s +Step 31800/44000 | Loss: 2.9102 | LR: 9.19e-05 | 1389.9ms/step | 14.4 samples/s +Step 31810/44000 | Loss: 2.9484 | LR: 9.18e-05 | 1389.9ms/step | 14.4 samples/s +Step 31820/44000 | Loss: 2.8768 | LR: 9.16e-05 | 1386.5ms/step | 14.4 samples/s +Step 31830/44000 | Loss: 2.8722 | LR: 9.15e-05 | 1389.9ms/step | 14.4 samples/s +Step 31840/44000 | Loss: 2.8852 | LR: 9.13e-05 | 1385.7ms/step | 14.4 samples/s +Step 31850/44000 | Loss: 2.8771 | LR: 9.12e-05 | 1389.1ms/step | 14.4 samples/s +Step 31860/44000 | Loss: 2.8992 | LR: 9.11e-05 | 1389.8ms/step | 14.4 samples/s +Step 31870/44000 | Loss: 2.9042 | LR: 9.09e-05 | 1387.6ms/step | 14.4 samples/s +Step 31880/44000 | Loss: 2.8933 | LR: 9.08e-05 | 1393.0ms/step | 14.4 samples/s +Step 31890/44000 | Loss: 2.9126 | LR: 9.06e-05 | 1391.4ms/step | 14.4 samples/s +Step 31900/44000 | Loss: 2.8699 | LR: 9.05e-05 | 1387.7ms/step | 14.4 samples/s +Step 31910/44000 | Loss: 2.8897 | LR: 9.04e-05 | 1389.6ms/step | 14.4 samples/s +Step 31920/44000 | Loss: 2.8826 | LR: 9.02e-05 | 1388.3ms/step | 14.4 samples/s +Step 31930/44000 | Loss: 2.9148 | LR: 9.01e-05 | 1389.8ms/step | 14.4 samples/s +Step 31940/44000 | Loss: 2.8967 | LR: 8.99e-05 | 1388.2ms/step | 14.4 samples/s +Step 31950/44000 | Loss: 2.8909 | LR: 8.98e-05 | 1390.2ms/step | 14.4 samples/s +Step 31960/44000 | Loss: 2.8776 | LR: 8.97e-05 | 1389.4ms/step | 14.4 samples/s +Step 31970/44000 | Loss: 2.8676 | LR: 8.95e-05 | 1388.2ms/step | 14.4 samples/s +Step 31980/44000 | Loss: 2.8986 | LR: 8.94e-05 | 1390.6ms/step | 14.4 samples/s +Step 31990/44000 | Loss: 2.8912 | LR: 8.92e-05 | 1388.4ms/step | 14.4 samples/s +Step 32000/44000 | Loss: 2.8907 | LR: 8.91e-05 | 1388.3ms/step | 14.4 samples/s +Validation at step 32000: Loss = 2.9000, Perplexity = 18.17 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-32000 +Step 32010/44000 | Loss: 2.8957 | LR: 8.90e-05 | 3871.4ms/step | 5.2 samples/s +Step 32020/44000 | Loss: 2.8994 | LR: 8.88e-05 | 1388.3ms/step | 14.4 samples/s +Step 32030/44000 | Loss: 2.8800 | LR: 8.87e-05 | 1387.2ms/step | 14.4 samples/s +Step 32040/44000 | Loss: 2.8857 | LR: 8.86e-05 | 1388.8ms/step | 14.4 samples/s +Step 32050/44000 | Loss: 2.8947 | LR: 8.84e-05 | 1389.4ms/step | 14.4 samples/s +Step 32060/44000 | Loss: 2.9213 | LR: 8.83e-05 | 1389.8ms/step | 14.4 samples/s +Step 32070/44000 | Loss: 2.8721 | LR: 8.81e-05 | 1393.7ms/step | 14.4 samples/s +Step 32080/44000 | Loss: 2.8848 | LR: 8.80e-05 | 1393.9ms/step | 14.3 samples/s +Step 32090/44000 | Loss: 2.8793 | LR: 8.79e-05 | 1392.3ms/step | 14.4 samples/s +Step 32100/44000 | Loss: 2.8476 | LR: 8.77e-05 | 1381.3ms/step | 14.5 samples/s +Step 32110/44000 | Loss: 2.8983 | LR: 8.76e-05 | 1377.9ms/step | 14.5 samples/s +Step 32120/44000 | Loss: 2.9057 | LR: 8.74e-05 | 1382.2ms/step | 14.5 samples/s +Step 32130/44000 | Loss: 2.8994 | LR: 8.73e-05 | 1383.4ms/step | 14.5 samples/s +Step 32140/44000 | Loss: 2.8674 | LR: 8.72e-05 | 1387.1ms/step | 14.4 samples/s +Step 32150/44000 | Loss: 2.8452 | LR: 8.70e-05 | 1384.3ms/step | 14.4 samples/s +Step 32160/44000 | Loss: 2.8705 | LR: 8.69e-05 | 1386.1ms/step | 14.4 samples/s +Step 32170/44000 | Loss: 2.8451 | LR: 8.68e-05 | 1386.4ms/step | 14.4 samples/s +Step 32180/44000 | Loss: 2.8771 | LR: 8.66e-05 | 1387.9ms/step | 14.4 samples/s +Step 32190/44000 | Loss: 2.9090 | LR: 8.65e-05 | 1387.9ms/step | 14.4 samples/s +Step 32200/44000 | Loss: 2.8988 | LR: 8.63e-05 | 1389.8ms/step | 14.4 samples/s +Step 32210/44000 | Loss: 2.8853 | LR: 8.62e-05 | 1389.6ms/step | 14.4 samples/s +Step 32220/44000 | Loss: 2.8977 | LR: 8.61e-05 | 1389.7ms/step | 14.4 samples/s +Step 32230/44000 | Loss: 2.8862 | LR: 8.59e-05 | 1386.9ms/step | 14.4 samples/s +Step 32240/44000 | Loss: 2.8511 | LR: 8.58e-05 | 1387.8ms/step | 14.4 samples/s +Step 32250/44000 | Loss: 2.8916 | LR: 8.57e-05 | 1387.3ms/step | 14.4 samples/s +Step 32260/44000 | Loss: 2.8569 | LR: 8.55e-05 | 1390.7ms/step | 14.4 samples/s +Step 32270/44000 | Loss: 2.8808 | LR: 8.54e-05 | 1388.1ms/step | 14.4 samples/s +Step 32280/44000 | Loss: 2.8503 | LR: 8.53e-05 | 1389.6ms/step | 14.4 samples/s +Step 32290/44000 | Loss: 2.8662 | LR: 8.51e-05 | 1390.0ms/step | 14.4 samples/s +Step 32300/44000 | Loss: 2.8385 | LR: 8.50e-05 | 1388.2ms/step | 14.4 samples/s +Step 32310/44000 | Loss: 2.8813 | LR: 8.48e-05 | 1390.0ms/step | 14.4 samples/s +Step 32320/44000 | Loss: 2.8826 | LR: 8.47e-05 | 1386.2ms/step | 14.4 samples/s +Step 32330/44000 | Loss: 2.9109 | LR: 8.46e-05 | 1386.4ms/step | 14.4 samples/s +Step 32340/44000 | Loss: 2.8816 | LR: 8.44e-05 | 1389.3ms/step | 14.4 samples/s +Step 32350/44000 | Loss: 2.8842 | LR: 8.43e-05 | 1388.4ms/step | 14.4 samples/s +Step 32360/44000 | Loss: 2.8847 | LR: 8.42e-05 | 1389.6ms/step | 14.4 samples/s +Step 32370/44000 | Loss: 2.8683 | LR: 8.40e-05 | 1389.6ms/step | 14.4 samples/s +Step 32380/44000 | Loss: 2.8444 | LR: 8.39e-05 | 1386.6ms/step | 14.4 samples/s +Step 32390/44000 | Loss: 2.8697 | LR: 8.38e-05 | 1387.8ms/step | 14.4 samples/s +Step 32400/44000 | Loss: 2.8971 | LR: 8.36e-05 | 1387.1ms/step | 14.4 samples/s +Step 32410/44000 | Loss: 2.8757 | LR: 8.35e-05 | 1387.1ms/step | 14.4 samples/s +Step 32420/44000 | Loss: 2.8640 | LR: 8.33e-05 | 1386.0ms/step | 14.4 samples/s +Step 32430/44000 | Loss: 2.8853 | LR: 8.32e-05 | 1384.2ms/step | 14.4 samples/s +Step 32440/44000 | Loss: 2.8977 | LR: 8.31e-05 | 1383.0ms/step | 14.5 samples/s +Step 32450/44000 | Loss: 2.9129 | LR: 8.29e-05 | 1386.5ms/step | 14.4 samples/s +Step 32460/44000 | Loss: 2.8739 | LR: 8.28e-05 | 1388.0ms/step | 14.4 samples/s +Step 32470/44000 | Loss: 2.8885 | LR: 8.27e-05 | 1386.2ms/step | 14.4 samples/s +Step 32480/44000 | Loss: 2.9002 | LR: 8.25e-05 | 1387.2ms/step | 14.4 samples/s +Step 32490/44000 | Loss: 2.8924 | LR: 8.24e-05 | 1388.8ms/step | 14.4 samples/s +Step 32500/44000 | Loss: 2.9139 | LR: 8.23e-05 | 1389.1ms/step | 14.4 samples/s +Step 32510/44000 | Loss: 2.8780 | LR: 8.21e-05 | 1388.3ms/step | 14.4 samples/s +Step 32520/44000 | Loss: 2.8933 | LR: 8.20e-05 | 1387.0ms/step | 14.4 samples/s +Step 32530/44000 | Loss: 2.8807 | LR: 8.19e-05 | 1386.1ms/step | 14.4 samples/s +Step 32540/44000 | Loss: 2.8600 | LR: 8.17e-05 | 1387.9ms/step | 14.4 samples/s +Step 32550/44000 | Loss: 2.8820 | LR: 8.16e-05 | 1390.3ms/step | 14.4 samples/s +Step 32560/44000 | Loss: 2.8584 | LR: 8.15e-05 | 1390.1ms/step | 14.4 samples/s +Step 32570/44000 | Loss: 2.8925 | LR: 8.13e-05 | 1391.2ms/step | 14.4 samples/s +Step 32580/44000 | Loss: 2.8644 | LR: 8.12e-05 | 1390.9ms/step | 14.4 samples/s +Step 32590/44000 | Loss: 2.9014 | LR: 8.11e-05 | 1389.3ms/step | 14.4 samples/s +Step 32600/44000 | Loss: 2.8689 | LR: 8.09e-05 | 1387.3ms/step | 14.4 samples/s +Step 32610/44000 | Loss: 2.8907 | LR: 8.08e-05 | 1380.6ms/step | 14.5 samples/s +Step 32620/44000 | Loss: 2.8770 | LR: 8.07e-05 | 1382.2ms/step | 14.5 samples/s +Step 32630/44000 | Loss: 2.8929 | LR: 8.05e-05 | 1386.4ms/step | 14.4 samples/s +Step 32640/44000 | Loss: 2.8419 | LR: 8.04e-05 | 1388.7ms/step | 14.4 samples/s +Step 32650/44000 | Loss: 2.8729 | LR: 8.03e-05 | 1386.8ms/step | 14.4 samples/s +Step 32660/44000 | Loss: 2.8688 | LR: 8.01e-05 | 1387.6ms/step | 14.4 samples/s +Step 32670/44000 | Loss: 2.8957 | LR: 8.00e-05 | 1386.9ms/step | 14.4 samples/s +Step 32680/44000 | Loss: 2.8906 | LR: 7.99e-05 | 1386.2ms/step | 14.4 samples/s +Step 32690/44000 | Loss: 2.8457 | LR: 7.97e-05 | 1387.3ms/step | 14.4 samples/s +Step 32700/44000 | Loss: 2.8814 | LR: 7.96e-05 | 1390.1ms/step | 14.4 samples/s +Step 32710/44000 | Loss: 2.8744 | LR: 7.95e-05 | 1388.2ms/step | 14.4 samples/s +Step 32720/44000 | Loss: 2.8814 | LR: 7.93e-05 | 1389.7ms/step | 14.4 samples/s +Step 32730/44000 | Loss: 2.8694 | LR: 7.92e-05 | 1389.8ms/step | 14.4 samples/s +Step 32740/44000 | Loss: 2.8758 | LR: 7.91e-05 | 1386.8ms/step | 14.4 samples/s +Step 32750/44000 | Loss: 2.8714 | LR: 7.89e-05 | 1387.6ms/step | 14.4 samples/s +Step 32760/44000 | Loss: 2.8892 | LR: 7.88e-05 | 1387.3ms/step | 14.4 samples/s +Step 32770/44000 | Loss: 2.8884 | LR: 7.87e-05 | 1389.6ms/step | 14.4 samples/s +Step 32780/44000 | Loss: 2.8934 | LR: 7.85e-05 | 1389.0ms/step | 14.4 samples/s +Step 32790/44000 | Loss: 2.8814 | LR: 7.84e-05 | 1388.3ms/step | 14.4 samples/s +Step 32800/44000 | Loss: 2.8941 | LR: 7.83e-05 | 1385.5ms/step | 14.4 samples/s +Step 32810/44000 | Loss: 2.8895 | LR: 7.81e-05 | 1385.7ms/step | 14.4 samples/s +Step 32820/44000 | Loss: 2.9053 | LR: 7.80e-05 | 1387.8ms/step | 14.4 samples/s +Step 32830/44000 | Loss: 2.8670 | LR: 7.79e-05 | 1384.8ms/step | 14.4 samples/s +Step 32840/44000 | Loss: 2.9140 | LR: 7.77e-05 | 1383.8ms/step | 14.5 samples/s +Step 32850/44000 | Loss: 2.8739 | LR: 7.76e-05 | 1384.2ms/step | 14.4 samples/s +Step 32860/44000 | Loss: 2.8665 | LR: 7.75e-05 | 1384.1ms/step | 14.4 samples/s +Step 32870/44000 | Loss: 2.8594 | LR: 7.73e-05 | 1390.5ms/step | 14.4 samples/s +Step 32880/44000 | Loss: 2.8813 | LR: 7.72e-05 | 1386.3ms/step | 14.4 samples/s +Step 32890/44000 | Loss: 2.8679 | LR: 7.71e-05 | 1386.8ms/step | 14.4 samples/s +Step 32900/44000 | Loss: 2.8839 | LR: 7.70e-05 | 1388.6ms/step | 14.4 samples/s +Step 32910/44000 | Loss: 2.8577 | LR: 7.68e-05 | 1389.4ms/step | 14.4 samples/s +Step 32920/44000 | Loss: 2.8843 | LR: 7.67e-05 | 1386.4ms/step | 14.4 samples/s +Step 32930/44000 | Loss: 2.8524 | LR: 7.66e-05 | 1388.0ms/step | 14.4 samples/s +Step 32940/44000 | Loss: 2.8742 | LR: 7.64e-05 | 1388.4ms/step | 14.4 samples/s +Step 32950/44000 | Loss: 2.8936 | LR: 7.63e-05 | 1387.3ms/step | 14.4 samples/s +Step 32960/44000 | Loss: 2.8892 | LR: 7.62e-05 | 1387.4ms/step | 14.4 samples/s +Step 32970/44000 | Loss: 2.8579 | LR: 7.60e-05 | 1388.7ms/step | 14.4 samples/s +Step 32980/44000 | Loss: 2.8501 | LR: 7.59e-05 | 1391.9ms/step | 14.4 samples/s +Step 32990/44000 | Loss: 2.8855 | LR: 7.58e-05 | 1388.2ms/step | 14.4 samples/s +Step 33000/44000 | Loss: 2.8921 | LR: 7.56e-05 | 1391.3ms/step | 14.4 samples/s +Validation at step 33000: Loss = 2.8979, Perplexity = 18.14 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-33000 +Step 33010/44000 | Loss: 2.8589 | LR: 7.55e-05 | 3951.1ms/step | 5.1 samples/s +Step 33020/44000 | Loss: 2.8912 | LR: 7.54e-05 | 1385.6ms/step | 14.4 samples/s +Step 33030/44000 | Loss: 2.8464 | LR: 7.53e-05 | 1385.9ms/step | 14.4 samples/s +Step 33040/44000 | Loss: 2.8880 | LR: 7.51e-05 | 1388.3ms/step | 14.4 samples/s +Step 33050/44000 | Loss: 2.9020 | LR: 7.50e-05 | 1387.7ms/step | 14.4 samples/s +Step 33060/44000 | Loss: 2.9202 | LR: 7.49e-05 | 1388.2ms/step | 14.4 samples/s +Step 33070/44000 | Loss: 2.8864 | LR: 7.47e-05 | 1391.8ms/step | 14.4 samples/s +Step 33080/44000 | Loss: 2.8542 | LR: 7.46e-05 | 1388.8ms/step | 14.4 samples/s +Step 33090/44000 | Loss: 2.8590 | LR: 7.45e-05 | 1383.7ms/step | 14.5 samples/s +Step 33100/44000 | Loss: 2.8682 | LR: 7.44e-05 | 1379.7ms/step | 14.5 samples/s +Step 33110/44000 | Loss: 2.8799 | LR: 7.42e-05 | 1380.1ms/step | 14.5 samples/s +Step 33120/44000 | Loss: 2.8675 | LR: 7.41e-05 | 1381.9ms/step | 14.5 samples/s +Step 33130/44000 | Loss: 2.8900 | LR: 7.40e-05 | 1383.6ms/step | 14.5 samples/s +Step 33140/44000 | Loss: 2.8862 | LR: 7.38e-05 | 1384.6ms/step | 14.4 samples/s +Step 33150/44000 | Loss: 2.8812 | LR: 7.37e-05 | 1386.1ms/step | 14.4 samples/s +Step 33160/44000 | Loss: 2.8858 | LR: 7.36e-05 | 1387.7ms/step | 14.4 samples/s +Step 33170/44000 | Loss: 2.8848 | LR: 7.34e-05 | 1387.9ms/step | 14.4 samples/s +Step 33180/44000 | Loss: 2.8987 | LR: 7.33e-05 | 1387.1ms/step | 14.4 samples/s +Step 33190/44000 | Loss: 2.8795 | LR: 7.32e-05 | 1389.7ms/step | 14.4 samples/s +Step 33200/44000 | Loss: 2.8937 | LR: 7.31e-05 | 1389.2ms/step | 14.4 samples/s +Step 33210/44000 | Loss: 2.8922 | LR: 7.29e-05 | 1388.6ms/step | 14.4 samples/s +Step 33220/44000 | Loss: 2.8767 | LR: 7.28e-05 | 1391.5ms/step | 14.4 samples/s +Step 33230/44000 | Loss: 2.9103 | LR: 7.27e-05 | 1391.9ms/step | 14.4 samples/s +Step 33240/44000 | Loss: 2.8982 | LR: 7.26e-05 | 1389.8ms/step | 14.4 samples/s +Step 33250/44000 | Loss: 2.8619 | LR: 7.24e-05 | 1392.5ms/step | 14.4 samples/s +Step 33260/44000 | Loss: 2.8559 | LR: 7.23e-05 | 1392.1ms/step | 14.4 samples/s +Step 33270/44000 | Loss: 2.8498 | LR: 7.22e-05 | 1397.9ms/step | 14.3 samples/s +Step 33280/44000 | Loss: 2.8712 | LR: 7.20e-05 | 1393.3ms/step | 14.4 samples/s +Step 33290/44000 | Loss: 2.8983 | LR: 7.19e-05 | 1387.2ms/step | 14.4 samples/s +Step 33300/44000 | Loss: 2.8878 | LR: 7.18e-05 | 1391.3ms/step | 14.4 samples/s +Step 33310/44000 | Loss: 2.8840 | LR: 7.17e-05 | 1388.7ms/step | 14.4 samples/s +Step 33320/44000 | Loss: 2.8569 | LR: 7.15e-05 | 1391.8ms/step | 14.4 samples/s +Step 33330/44000 | Loss: 2.8986 | LR: 7.14e-05 | 1390.0ms/step | 14.4 samples/s +Step 33340/44000 | Loss: 2.8678 | LR: 7.13e-05 | 1391.2ms/step | 14.4 samples/s +Step 33350/44000 | Loss: 2.8866 | LR: 7.11e-05 | 1389.6ms/step | 14.4 samples/s +Step 33360/44000 | Loss: 2.8772 | LR: 7.10e-05 | 1390.8ms/step | 14.4 samples/s +Step 33370/44000 | Loss: 2.8926 | LR: 7.09e-05 | 1388.8ms/step | 14.4 samples/s +Step 33380/44000 | Loss: 2.8654 | LR: 7.08e-05 | 1391.6ms/step | 14.4 samples/s +Step 33390/44000 | Loss: 2.8861 | LR: 7.06e-05 | 1387.4ms/step | 14.4 samples/s +Step 33400/44000 | Loss: 2.8704 | LR: 7.05e-05 | 1388.6ms/step | 14.4 samples/s +Step 33410/44000 | Loss: 2.8680 | LR: 7.04e-05 | 1387.9ms/step | 14.4 samples/s +Step 33420/44000 | Loss: 2.8868 | LR: 7.03e-05 | 1388.9ms/step | 14.4 samples/s +Step 33430/44000 | Loss: 2.8680 | LR: 7.01e-05 | 1390.5ms/step | 14.4 samples/s +Step 33440/44000 | Loss: 2.8459 | LR: 7.00e-05 | 1386.3ms/step | 14.4 samples/s +Step 33450/44000 | Loss: 2.8798 | LR: 6.99e-05 | 1387.1ms/step | 14.4 samples/s +Step 33460/44000 | Loss: 2.8822 | LR: 6.98e-05 | 1387.5ms/step | 14.4 samples/s +Step 33470/44000 | Loss: 2.9203 | LR: 6.96e-05 | 1389.7ms/step | 14.4 samples/s +Step 33480/44000 | Loss: 2.8798 | LR: 6.95e-05 | 1390.1ms/step | 14.4 samples/s +Step 33490/44000 | Loss: 2.8674 | LR: 6.94e-05 | 1388.0ms/step | 14.4 samples/s +Step 33500/44000 | Loss: 2.9191 | LR: 6.93e-05 | 1388.5ms/step | 14.4 samples/s +Step 33510/44000 | Loss: 2.8922 | LR: 6.91e-05 | 1387.4ms/step | 14.4 samples/s +Step 33520/44000 | Loss: 2.8512 | LR: 6.90e-05 | 1387.8ms/step | 14.4 samples/s +Step 33530/44000 | Loss: 2.8735 | LR: 6.89e-05 | 1388.1ms/step | 14.4 samples/s +Step 33540/44000 | Loss: 2.8292 | LR: 6.88e-05 | 1389.3ms/step | 14.4 samples/s +Step 33550/44000 | Loss: 2.8954 | LR: 6.86e-05 | 1390.9ms/step | 14.4 samples/s +Step 33560/44000 | Loss: 2.8862 | LR: 6.85e-05 | 1393.4ms/step | 14.4 samples/s +Step 33570/44000 | Loss: 2.8742 | LR: 6.84e-05 | 1390.2ms/step | 14.4 samples/s +Step 33580/44000 | Loss: 2.8807 | LR: 6.83e-05 | 1392.4ms/step | 14.4 samples/s +Step 33590/44000 | Loss: 2.9003 | LR: 6.81e-05 | 1393.7ms/step | 14.3 samples/s +Step 33600/44000 | Loss: 2.8779 | LR: 6.80e-05 | 1388.4ms/step | 14.4 samples/s +Step 33610/44000 | Loss: 2.8675 | LR: 6.79e-05 | 1389.8ms/step | 14.4 samples/s +Step 33620/44000 | Loss: 2.8822 | LR: 6.78e-05 | 1391.7ms/step | 14.4 samples/s +Step 33630/44000 | Loss: 2.8955 | LR: 6.76e-05 | 1389.6ms/step | 14.4 samples/s +Step 33640/44000 | Loss: 2.9068 | LR: 6.75e-05 | 1386.2ms/step | 14.4 samples/s +Step 33650/44000 | Loss: 2.8588 | LR: 6.74e-05 | 1387.8ms/step | 14.4 samples/s +Step 33660/44000 | Loss: 2.8661 | LR: 6.73e-05 | 1386.5ms/step | 14.4 samples/s +Step 33670/44000 | Loss: 2.8690 | LR: 6.71e-05 | 1390.4ms/step | 14.4 samples/s +Step 33680/44000 | Loss: 2.8796 | LR: 6.70e-05 | 1384.9ms/step | 14.4 samples/s +Step 33690/44000 | Loss: 2.8649 | LR: 6.69e-05 | 1384.7ms/step | 14.4 samples/s +Step 33700/44000 | Loss: 2.8658 | LR: 6.68e-05 | 1384.1ms/step | 14.4 samples/s +Step 33710/44000 | Loss: 2.8813 | LR: 6.66e-05 | 1384.0ms/step | 14.5 samples/s +Step 33720/44000 | Loss: 2.9005 | LR: 6.65e-05 | 1386.1ms/step | 14.4 samples/s +Step 33730/44000 | Loss: 2.8819 | LR: 6.64e-05 | 1385.5ms/step | 14.4 samples/s +Step 33740/44000 | Loss: 2.8712 | LR: 6.63e-05 | 1388.9ms/step | 14.4 samples/s +Step 33750/44000 | Loss: 2.8923 | LR: 6.61e-05 | 1388.1ms/step | 14.4 samples/s +Step 33760/44000 | Loss: 2.8978 | LR: 6.60e-05 | 1388.7ms/step | 14.4 samples/s +Step 33770/44000 | Loss: 2.8791 | LR: 6.59e-05 | 1388.9ms/step | 14.4 samples/s +Step 33780/44000 | Loss: 2.8657 | LR: 6.58e-05 | 1388.5ms/step | 14.4 samples/s +Step 33790/44000 | Loss: 2.8807 | LR: 6.57e-05 | 1387.4ms/step | 14.4 samples/s +Step 33800/44000 | Loss: 2.8487 | LR: 6.55e-05 | 1386.4ms/step | 14.4 samples/s +Step 33810/44000 | Loss: 2.8738 | LR: 6.54e-05 | 1384.8ms/step | 14.4 samples/s +Step 33820/44000 | Loss: 2.8522 | LR: 6.53e-05 | 1386.6ms/step | 14.4 samples/s +Step 33830/44000 | Loss: 2.8923 | LR: 6.52e-05 | 1385.6ms/step | 14.4 samples/s +Step 33840/44000 | Loss: 2.9078 | LR: 6.50e-05 | 1383.8ms/step | 14.5 samples/s +Step 33850/44000 | Loss: 2.9077 | LR: 6.49e-05 | 1387.5ms/step | 14.4 samples/s +Step 33860/44000 | Loss: 2.8954 | LR: 6.48e-05 | 1387.5ms/step | 14.4 samples/s +Step 33870/44000 | Loss: 2.8665 | LR: 6.47e-05 | 1385.9ms/step | 14.4 samples/s +Step 33880/44000 | Loss: 2.8760 | LR: 6.46e-05 | 1387.9ms/step | 14.4 samples/s +Step 33890/44000 | Loss: 2.8716 | LR: 6.44e-05 | 1388.5ms/step | 14.4 samples/s +Step 33900/44000 | Loss: 2.8897 | LR: 6.43e-05 | 1385.4ms/step | 14.4 samples/s +Step 33910/44000 | Loss: 2.8897 | LR: 6.42e-05 | 1386.4ms/step | 14.4 samples/s +Step 33920/44000 | Loss: 2.8765 | LR: 6.41e-05 | 1386.9ms/step | 14.4 samples/s +Step 33930/44000 | Loss: 2.9081 | LR: 6.40e-05 | 1387.9ms/step | 14.4 samples/s +Step 33940/44000 | Loss: 2.8913 | LR: 6.38e-05 | 1384.1ms/step | 14.4 samples/s +Step 33950/44000 | Loss: 2.8927 | LR: 6.37e-05 | 1387.6ms/step | 14.4 samples/s +Step 33960/44000 | Loss: 2.8908 | LR: 6.36e-05 | 1387.4ms/step | 14.4 samples/s +Step 33970/44000 | Loss: 2.8539 | LR: 6.35e-05 | 1388.2ms/step | 14.4 samples/s +Step 33980/44000 | Loss: 2.8731 | LR: 6.33e-05 | 1387.9ms/step | 14.4 samples/s +Step 33990/44000 | Loss: 2.8965 | LR: 6.32e-05 | 1390.1ms/step | 14.4 samples/s +Step 34000/44000 | Loss: 2.8592 | LR: 6.31e-05 | 1391.9ms/step | 14.4 samples/s +Validation at step 34000: Loss = 2.8964, Perplexity = 18.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-34000 +Step 34010/44000 | Loss: 2.8976 | LR: 6.30e-05 | 3897.9ms/step | 5.1 samples/s +Step 34020/44000 | Loss: 2.8771 | LR: 6.29e-05 | 1386.7ms/step | 14.4 samples/s +Step 34030/44000 | Loss: 2.8458 | LR: 6.27e-05 | 1383.3ms/step | 14.5 samples/s +Step 34040/44000 | Loss: 2.8732 | LR: 6.26e-05 | 1385.5ms/step | 14.4 samples/s +Step 34050/44000 | Loss: 2.8919 | LR: 6.25e-05 | 1384.9ms/step | 14.4 samples/s +Step 34060/44000 | Loss: 2.8759 | LR: 6.24e-05 | 1385.2ms/step | 14.4 samples/s +Step 34070/44000 | Loss: 2.8685 | LR: 6.23e-05 | 1384.5ms/step | 14.4 samples/s +Step 34080/44000 | Loss: 2.8746 | LR: 6.21e-05 | 1386.7ms/step | 14.4 samples/s +Step 34090/44000 | Loss: 2.8856 | LR: 6.20e-05 | 1387.2ms/step | 14.4 samples/s +Step 34100/44000 | Loss: 2.8633 | LR: 6.19e-05 | 1384.3ms/step | 14.4 samples/s +Step 34110/44000 | Loss: 2.8659 | LR: 6.18e-05 | 1384.0ms/step | 14.5 samples/s +Step 34120/44000 | Loss: 2.8773 | LR: 6.17e-05 | 1385.4ms/step | 14.4 samples/s +Step 34130/44000 | Loss: 2.8954 | LR: 6.15e-05 | 1385.6ms/step | 14.4 samples/s +Step 34140/44000 | Loss: 2.8640 | LR: 6.14e-05 | 1383.0ms/step | 14.5 samples/s +Step 34150/44000 | Loss: 2.8770 | LR: 6.13e-05 | 1383.1ms/step | 14.5 samples/s +Step 34160/44000 | Loss: 2.8636 | LR: 6.12e-05 | 1386.4ms/step | 14.4 samples/s +Step 34170/44000 | Loss: 2.8972 | LR: 6.11e-05 | 1386.5ms/step | 14.4 samples/s +Step 34180/44000 | Loss: 2.8874 | LR: 6.09e-05 | 1384.9ms/step | 14.4 samples/s +Step 34190/44000 | Loss: 2.8886 | LR: 6.08e-05 | 1386.6ms/step | 14.4 samples/s +Step 34200/44000 | Loss: 2.9075 | LR: 6.07e-05 | 1386.1ms/step | 14.4 samples/s +Step 34210/44000 | Loss: 2.8864 | LR: 6.06e-05 | 1387.4ms/step | 14.4 samples/s +Step 34220/44000 | Loss: 2.8992 | LR: 6.05e-05 | 1389.9ms/step | 14.4 samples/s +Step 34230/44000 | Loss: 2.8440 | LR: 6.04e-05 | 1390.7ms/step | 14.4 samples/s +Step 34240/44000 | Loss: 2.8911 | LR: 6.02e-05 | 1389.5ms/step | 14.4 samples/s +Step 34250/44000 | Loss: 2.8992 | LR: 6.01e-05 | 1390.7ms/step | 14.4 samples/s +Step 34260/44000 | Loss: 2.8731 | LR: 6.00e-05 | 1391.2ms/step | 14.4 samples/s +Step 34270/44000 | Loss: 2.8771 | LR: 5.99e-05 | 1388.8ms/step | 14.4 samples/s +Step 34280/44000 | Loss: 2.8454 | LR: 5.98e-05 | 1386.2ms/step | 14.4 samples/s +Step 34290/44000 | Loss: 2.8929 | LR: 5.96e-05 | 1389.8ms/step | 14.4 samples/s +Step 34300/44000 | Loss: 2.8379 | LR: 5.95e-05 | 1385.8ms/step | 14.4 samples/s +Step 34310/44000 | Loss: 2.9047 | LR: 5.94e-05 | 1386.3ms/step | 14.4 samples/s +Step 34320/44000 | Loss: 2.8628 | LR: 5.93e-05 | 1386.4ms/step | 14.4 samples/s +Step 34330/44000 | Loss: 2.8597 | LR: 5.92e-05 | 1389.6ms/step | 14.4 samples/s +Step 34340/44000 | Loss: 2.8668 | LR: 5.91e-05 | 1393.6ms/step | 14.4 samples/s +Step 34350/44000 | Loss: 2.8700 | LR: 5.89e-05 | 1390.2ms/step | 14.4 samples/s +Step 34360/44000 | Loss: 2.8771 | LR: 5.88e-05 | 1388.2ms/step | 14.4 samples/s +Step 34370/44000 | Loss: 2.8900 | LR: 5.87e-05 | 1390.0ms/step | 14.4 samples/s +Step 34380/44000 | Loss: 2.8776 | LR: 5.86e-05 | 1387.7ms/step | 14.4 samples/s +Step 34390/44000 | Loss: 2.8713 | LR: 5.85e-05 | 1389.9ms/step | 14.4 samples/s +Step 34400/44000 | Loss: 2.8270 | LR: 5.84e-05 | 1388.0ms/step | 14.4 samples/s +Step 34410/44000 | Loss: 2.8604 | LR: 5.82e-05 | 1390.6ms/step | 14.4 samples/s +Step 34420/44000 | Loss: 2.8790 | LR: 5.81e-05 | 1386.6ms/step | 14.4 samples/s +Step 34430/44000 | Loss: 2.8755 | LR: 5.80e-05 | 1381.1ms/step | 14.5 samples/s +Step 34440/44000 | Loss: 2.8752 | LR: 5.79e-05 | 1382.8ms/step | 14.5 samples/s +Step 34450/44000 | Loss: 2.8969 | LR: 5.78e-05 | 1384.0ms/step | 14.5 samples/s +Step 34460/44000 | Loss: 2.8837 | LR: 5.77e-05 | 1385.2ms/step | 14.4 samples/s +Step 34470/44000 | Loss: 2.8835 | LR: 5.75e-05 | 1383.9ms/step | 14.5 samples/s +Step 34480/44000 | Loss: 2.8828 | LR: 5.74e-05 | 1385.2ms/step | 14.4 samples/s +Step 34490/44000 | Loss: 2.8833 | LR: 5.73e-05 | 1387.2ms/step | 14.4 samples/s +Step 34500/44000 | Loss: 2.8611 | LR: 5.72e-05 | 1388.0ms/step | 14.4 samples/s +Step 34510/44000 | Loss: 2.8820 | LR: 5.71e-05 | 1385.8ms/step | 14.4 samples/s +Step 34520/44000 | Loss: 2.8809 | LR: 5.70e-05 | 1386.3ms/step | 14.4 samples/s +Step 34530/44000 | Loss: 2.8701 | LR: 5.69e-05 | 1388.4ms/step | 14.4 samples/s +Step 34540/44000 | Loss: 2.8761 | LR: 5.67e-05 | 1387.0ms/step | 14.4 samples/s +Step 34550/44000 | Loss: 2.8948 | LR: 5.66e-05 | 1389.0ms/step | 14.4 samples/s +Step 34560/44000 | Loss: 2.8670 | LR: 5.65e-05 | 1383.4ms/step | 14.5 samples/s +Step 34570/44000 | Loss: 2.8554 | LR: 5.64e-05 | 1383.1ms/step | 14.5 samples/s +Step 34580/44000 | Loss: 2.8369 | LR: 5.63e-05 | 1384.7ms/step | 14.4 samples/s +Step 34590/44000 | Loss: 2.8902 | LR: 5.62e-05 | 1385.5ms/step | 14.4 samples/s +Step 34600/44000 | Loss: 2.9033 | LR: 5.60e-05 | 1388.2ms/step | 14.4 samples/s +Step 34610/44000 | Loss: 2.8789 | LR: 5.59e-05 | 1390.0ms/step | 14.4 samples/s +Step 34620/44000 | Loss: 2.8545 | LR: 5.58e-05 | 1390.6ms/step | 14.4 samples/s +Step 34630/44000 | Loss: 2.8844 | LR: 5.57e-05 | 1384.5ms/step | 14.4 samples/s +Step 34640/44000 | Loss: 2.8629 | LR: 5.56e-05 | 1379.4ms/step | 14.5 samples/s +Step 34650/44000 | Loss: 2.8700 | LR: 5.55e-05 | 1383.3ms/step | 14.5 samples/s +Step 34660/44000 | Loss: 2.8464 | LR: 5.54e-05 | 1384.9ms/step | 14.4 samples/s +Step 34670/44000 | Loss: 2.8968 | LR: 5.52e-05 | 1382.7ms/step | 14.5 samples/s +Step 34680/44000 | Loss: 2.8696 | LR: 5.51e-05 | 1383.4ms/step | 14.5 samples/s +Step 34690/44000 | Loss: 2.9061 | LR: 5.50e-05 | 1383.5ms/step | 14.5 samples/s +Step 34700/44000 | Loss: 2.8843 | LR: 5.49e-05 | 1386.6ms/step | 14.4 samples/s +Step 34710/44000 | Loss: 2.8641 | LR: 5.48e-05 | 1385.8ms/step | 14.4 samples/s +Step 34720/44000 | Loss: 2.8644 | LR: 5.47e-05 | 1386.2ms/step | 14.4 samples/s +Step 34730/44000 | Loss: 2.8869 | LR: 5.46e-05 | 1391.6ms/step | 14.4 samples/s +Step 34740/44000 | Loss: 2.8805 | LR: 5.45e-05 | 1387.9ms/step | 14.4 samples/s +Step 34750/44000 | Loss: 2.8490 | LR: 5.43e-05 | 1388.4ms/step | 14.4 samples/s +Step 34760/44000 | Loss: 2.8678 | LR: 5.42e-05 | 1388.7ms/step | 14.4 samples/s +Step 34770/44000 | Loss: 2.8836 | LR: 5.41e-05 | 1390.9ms/step | 14.4 samples/s +Step 34780/44000 | Loss: 2.8526 | LR: 5.40e-05 | 1391.6ms/step | 14.4 samples/s +Step 34790/44000 | Loss: 2.9066 | LR: 5.39e-05 | 1389.4ms/step | 14.4 samples/s +Step 34800/44000 | Loss: 2.8859 | LR: 5.38e-05 | 1387.3ms/step | 14.4 samples/s +Step 34810/44000 | Loss: 2.8651 | LR: 5.37e-05 | 1386.7ms/step | 14.4 samples/s +Step 34820/44000 | Loss: 2.8737 | LR: 5.36e-05 | 1389.3ms/step | 14.4 samples/s +Step 34830/44000 | Loss: 2.8887 | LR: 5.34e-05 | 1388.3ms/step | 14.4 samples/s +Step 34840/44000 | Loss: 2.8513 | LR: 5.33e-05 | 1390.6ms/step | 14.4 samples/s +Step 34850/44000 | Loss: 2.8953 | LR: 5.32e-05 | 1391.2ms/step | 14.4 samples/s +Step 34860/44000 | Loss: 2.8701 | LR: 5.31e-05 | 1391.4ms/step | 14.4 samples/s +Step 34870/44000 | Loss: 2.8734 | LR: 5.30e-05 | 1388.3ms/step | 14.4 samples/s +Step 34880/44000 | Loss: 2.8512 | LR: 5.29e-05 | 1389.6ms/step | 14.4 samples/s +Step 34890/44000 | Loss: 2.8637 | LR: 5.28e-05 | 1392.1ms/step | 14.4 samples/s +Step 34900/44000 | Loss: 2.8884 | LR: 5.27e-05 | 1388.6ms/step | 14.4 samples/s +Step 34910/44000 | Loss: 2.9033 | LR: 5.25e-05 | 1387.8ms/step | 14.4 samples/s +Step 34920/44000 | Loss: 2.8867 | LR: 5.24e-05 | 1388.2ms/step | 14.4 samples/s +Step 34930/44000 | Loss: 2.8728 | LR: 5.23e-05 | 1387.9ms/step | 14.4 samples/s +Step 34940/44000 | Loss: 2.8778 | LR: 5.22e-05 | 1386.2ms/step | 14.4 samples/s +Step 34950/44000 | Loss: 2.8676 | LR: 5.21e-05 | 1388.7ms/step | 14.4 samples/s +Step 34960/44000 | Loss: 2.8997 | LR: 5.20e-05 | 1388.5ms/step | 14.4 samples/s +Step 34970/44000 | Loss: 2.8654 | LR: 5.19e-05 | 1389.3ms/step | 14.4 samples/s +Step 34980/44000 | Loss: 2.8897 | LR: 5.18e-05 | 1387.8ms/step | 14.4 samples/s +Step 34990/44000 | Loss: 2.8649 | LR: 5.17e-05 | 1389.9ms/step | 14.4 samples/s +Step 35000/44000 | Loss: 2.8808 | LR: 5.15e-05 | 1390.4ms/step | 14.4 samples/s +Validation at step 35000: Loss = 2.8953, Perplexity = 18.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-35000 +Step 35010/44000 | Loss: 2.8775 | LR: 5.14e-05 | 3866.1ms/step | 5.2 samples/s +Step 35020/44000 | Loss: 2.8696 | LR: 5.13e-05 | 1384.1ms/step | 14.5 samples/s +Step 35030/44000 | Loss: 2.8647 | LR: 5.12e-05 | 1386.8ms/step | 14.4 samples/s +Step 35040/44000 | Loss: 2.8653 | LR: 5.11e-05 | 1386.8ms/step | 14.4 samples/s +Step 35050/44000 | Loss: 2.8739 | LR: 5.10e-05 | 1389.7ms/step | 14.4 samples/s +Step 35060/44000 | Loss: 2.8996 | LR: 5.09e-05 | 1388.3ms/step | 14.4 samples/s +Step 35070/44000 | Loss: 2.8713 | LR: 5.08e-05 | 1388.6ms/step | 14.4 samples/s +Step 35080/44000 | Loss: 2.8683 | LR: 5.07e-05 | 1390.0ms/step | 14.4 samples/s +Step 35090/44000 | Loss: 2.8913 | LR: 5.06e-05 | 1387.1ms/step | 14.4 samples/s +Step 35100/44000 | Loss: 2.9003 | LR: 5.04e-05 | 1388.1ms/step | 14.4 samples/s +Step 35110/44000 | Loss: 2.8673 | LR: 5.03e-05 | 1386.1ms/step | 14.4 samples/s +Step 35120/44000 | Loss: 2.8740 | LR: 5.02e-05 | 1386.0ms/step | 14.4 samples/s +Step 35130/44000 | Loss: 2.8851 | LR: 5.01e-05 | 1387.6ms/step | 14.4 samples/s +Step 35140/44000 | Loss: 2.8800 | LR: 5.00e-05 | 1384.8ms/step | 14.4 samples/s +Step 35150/44000 | Loss: 2.8609 | LR: 4.99e-05 | 1385.1ms/step | 14.4 samples/s +Step 35160/44000 | Loss: 2.8701 | LR: 4.98e-05 | 1386.7ms/step | 14.4 samples/s +Step 35170/44000 | Loss: 2.8588 | LR: 4.97e-05 | 1389.8ms/step | 14.4 samples/s +Step 35180/44000 | Loss: 2.8874 | LR: 4.96e-05 | 1386.0ms/step | 14.4 samples/s +Step 35190/44000 | Loss: 2.8755 | LR: 4.95e-05 | 1387.9ms/step | 14.4 samples/s +Step 35200/44000 | Loss: 2.8962 | LR: 4.94e-05 | 1388.4ms/step | 14.4 samples/s +Step 35210/44000 | Loss: 2.8996 | LR: 4.93e-05 | 1388.0ms/step | 14.4 samples/s +Step 35220/44000 | Loss: 2.8918 | LR: 4.91e-05 | 1387.0ms/step | 14.4 samples/s +Step 35230/44000 | Loss: 2.8518 | LR: 4.90e-05 | 1387.7ms/step | 14.4 samples/s +Step 35240/44000 | Loss: 2.8674 | LR: 4.89e-05 | 1388.3ms/step | 14.4 samples/s +Step 35250/44000 | Loss: 2.8683 | LR: 4.88e-05 | 1389.0ms/step | 14.4 samples/s +Step 35260/44000 | Loss: 2.8763 | LR: 4.87e-05 | 1387.7ms/step | 14.4 samples/s +Step 35270/44000 | Loss: 2.8784 | LR: 4.86e-05 | 1386.9ms/step | 14.4 samples/s +Step 35280/44000 | Loss: 2.8768 | LR: 4.85e-05 | 1389.2ms/step | 14.4 samples/s +Step 35290/44000 | Loss: 2.8335 | LR: 4.84e-05 | 1388.2ms/step | 14.4 samples/s +Step 35300/44000 | Loss: 2.8998 | LR: 4.83e-05 | 1385.7ms/step | 14.4 samples/s +Step 35310/44000 | Loss: 2.8811 | LR: 4.82e-05 | 1386.3ms/step | 14.4 samples/s +Step 35320/44000 | Loss: 2.8809 | LR: 4.81e-05 | 1387.1ms/step | 14.4 samples/s +Step 35330/44000 | Loss: 2.8791 | LR: 4.80e-05 | 1386.7ms/step | 14.4 samples/s +Step 35340/44000 | Loss: 2.8838 | LR: 4.79e-05 | 1386.1ms/step | 14.4 samples/s +Step 35350/44000 | Loss: 2.8602 | LR: 4.77e-05 | 1386.7ms/step | 14.4 samples/s +Step 35360/44000 | Loss: 2.8949 | LR: 4.76e-05 | 1385.9ms/step | 14.4 samples/s +Step 35370/44000 | Loss: 2.9239 | LR: 4.75e-05 | 1390.5ms/step | 14.4 samples/s +Step 35380/44000 | Loss: 2.8625 | LR: 4.74e-05 | 1389.0ms/step | 14.4 samples/s +Step 35390/44000 | Loss: 2.8833 | LR: 4.73e-05 | 1388.4ms/step | 14.4 samples/s +Step 35400/44000 | Loss: 2.9221 | LR: 4.72e-05 | 1389.1ms/step | 14.4 samples/s +Step 35410/44000 | Loss: 2.8653 | LR: 4.71e-05 | 1388.3ms/step | 14.4 samples/s +Step 35420/44000 | Loss: 2.8611 | LR: 4.70e-05 | 1390.0ms/step | 14.4 samples/s +Step 35430/44000 | Loss: 2.8657 | LR: 4.69e-05 | 1387.9ms/step | 14.4 samples/s +Step 35440/44000 | Loss: 2.9124 | LR: 4.68e-05 | 1385.8ms/step | 14.4 samples/s +Step 35450/44000 | Loss: 2.8945 | LR: 4.67e-05 | 1388.4ms/step | 14.4 samples/s +Step 35460/44000 | Loss: 2.8844 | LR: 4.66e-05 | 1388.5ms/step | 14.4 samples/s +Step 35470/44000 | Loss: 2.8936 | LR: 4.65e-05 | 1386.1ms/step | 14.4 samples/s +Step 35480/44000 | Loss: 2.8998 | LR: 4.64e-05 | 1386.4ms/step | 14.4 samples/s +Step 35490/44000 | Loss: 2.8955 | LR: 4.63e-05 | 1384.9ms/step | 14.4 samples/s +Step 35500/44000 | Loss: 2.8911 | LR: 4.62e-05 | 1385.5ms/step | 14.4 samples/s +Step 35510/44000 | Loss: 2.9132 | LR: 4.61e-05 | 1384.5ms/step | 14.4 samples/s +Step 35520/44000 | Loss: 2.8840 | LR: 4.59e-05 | 1385.0ms/step | 14.4 samples/s +Step 35530/44000 | Loss: 2.8902 | LR: 4.58e-05 | 1387.8ms/step | 14.4 samples/s +Step 35540/44000 | Loss: 2.9003 | LR: 4.57e-05 | 1384.1ms/step | 14.4 samples/s +Step 35550/44000 | Loss: 2.8939 | LR: 4.56e-05 | 1386.2ms/step | 14.4 samples/s +Step 35560/44000 | Loss: 2.8510 | LR: 4.55e-05 | 1386.2ms/step | 14.4 samples/s +Step 35570/44000 | Loss: 2.8725 | LR: 4.54e-05 | 1384.7ms/step | 14.4 samples/s +Step 35580/44000 | Loss: 2.8915 | LR: 4.53e-05 | 1381.7ms/step | 14.5 samples/s +Step 35590/44000 | Loss: 2.8758 | LR: 4.52e-05 | 1386.0ms/step | 14.4 samples/s +Step 35600/44000 | Loss: 2.8682 | LR: 4.51e-05 | 1385.9ms/step | 14.4 samples/s +Step 35610/44000 | Loss: 2.8637 | LR: 4.50e-05 | 1386.6ms/step | 14.4 samples/s +Step 35620/44000 | Loss: 2.8906 | LR: 4.49e-05 | 1387.2ms/step | 14.4 samples/s +Step 35630/44000 | Loss: 2.8711 | LR: 4.48e-05 | 1387.9ms/step | 14.4 samples/s +Step 35640/44000 | Loss: 2.8626 | LR: 4.47e-05 | 1385.9ms/step | 14.4 samples/s +Step 35650/44000 | Loss: 2.8467 | LR: 4.46e-05 | 1386.9ms/step | 14.4 samples/s +Step 35660/44000 | Loss: 2.8830 | LR: 4.45e-05 | 1384.8ms/step | 14.4 samples/s +Step 35670/44000 | Loss: 2.8871 | LR: 4.44e-05 | 1385.4ms/step | 14.4 samples/s +Step 35680/44000 | Loss: 2.8975 | LR: 4.43e-05 | 1385.9ms/step | 14.4 samples/s +Step 35690/44000 | Loss: 2.8685 | LR: 4.42e-05 | 1387.5ms/step | 14.4 samples/s +Step 35700/44000 | Loss: 2.8862 | LR: 4.41e-05 | 1385.5ms/step | 14.4 samples/s +Step 35710/44000 | Loss: 2.9071 | LR: 4.40e-05 | 1381.9ms/step | 14.5 samples/s +Step 35720/44000 | Loss: 2.8830 | LR: 4.39e-05 | 1380.6ms/step | 14.5 samples/s +Step 35730/44000 | Loss: 2.8391 | LR: 4.38e-05 | 1384.6ms/step | 14.4 samples/s +Step 35740/44000 | Loss: 2.8704 | LR: 4.37e-05 | 1385.9ms/step | 14.4 samples/s +Step 35750/44000 | Loss: 2.8559 | LR: 4.36e-05 | 1387.9ms/step | 14.4 samples/s +Step 35760/44000 | Loss: 2.8639 | LR: 4.35e-05 | 1387.6ms/step | 14.4 samples/s +Step 35770/44000 | Loss: 2.8855 | LR: 4.34e-05 | 1388.0ms/step | 14.4 samples/s +Step 35780/44000 | Loss: 2.8777 | LR: 4.33e-05 | 1387.5ms/step | 14.4 samples/s +Step 35790/44000 | Loss: 2.8773 | LR: 4.32e-05 | 1386.3ms/step | 14.4 samples/s +Step 35800/44000 | Loss: 2.8695 | LR: 4.31e-05 | 1387.0ms/step | 14.4 samples/s +Step 35810/44000 | Loss: 2.8973 | LR: 4.29e-05 | 1385.2ms/step | 14.4 samples/s +Step 35820/44000 | Loss: 2.9211 | LR: 4.28e-05 | 1388.5ms/step | 14.4 samples/s +Step 35830/44000 | Loss: 2.8468 | LR: 4.27e-05 | 1388.2ms/step | 14.4 samples/s +Step 35840/44000 | Loss: 2.8605 | LR: 4.26e-05 | 1385.9ms/step | 14.4 samples/s +Step 35850/44000 | Loss: 2.8619 | LR: 4.25e-05 | 1385.0ms/step | 14.4 samples/s +Step 35860/44000 | Loss: 2.8362 | LR: 4.24e-05 | 1385.2ms/step | 14.4 samples/s +Step 35870/44000 | Loss: 2.8618 | LR: 4.23e-05 | 1386.7ms/step | 14.4 samples/s +Step 35880/44000 | Loss: 2.8547 | LR: 4.22e-05 | 1387.6ms/step | 14.4 samples/s +Step 35890/44000 | Loss: 2.8796 | LR: 4.21e-05 | 1386.5ms/step | 14.4 samples/s +Step 35900/44000 | Loss: 2.9049 | LR: 4.20e-05 | 1384.9ms/step | 14.4 samples/s +Step 35910/44000 | Loss: 2.8586 | LR: 4.19e-05 | 1388.3ms/step | 14.4 samples/s +Step 35920/44000 | Loss: 2.8632 | LR: 4.18e-05 | 1388.6ms/step | 14.4 samples/s +Step 35930/44000 | Loss: 2.8826 | LR: 4.17e-05 | 1387.1ms/step | 14.4 samples/s +Step 35940/44000 | Loss: 2.8865 | LR: 4.16e-05 | 1390.4ms/step | 14.4 samples/s +Step 35950/44000 | Loss: 2.8538 | LR: 4.15e-05 | 1386.0ms/step | 14.4 samples/s +Step 35960/44000 | Loss: 2.8799 | LR: 4.14e-05 | 1389.2ms/step | 14.4 samples/s +Step 35970/44000 | Loss: 2.8923 | LR: 4.13e-05 | 1388.3ms/step | 14.4 samples/s +Step 35980/44000 | Loss: 2.8884 | LR: 4.12e-05 | 1388.3ms/step | 14.4 samples/s +Step 35990/44000 | Loss: 2.9025 | LR: 4.11e-05 | 1386.2ms/step | 14.4 samples/s +Step 36000/44000 | Loss: 2.9106 | LR: 4.10e-05 | 1387.8ms/step | 14.4 samples/s +Validation at step 36000: Loss = 2.8944, Perplexity = 18.07 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-36000 +Step 36010/44000 | Loss: 2.8724 | LR: 4.09e-05 | 3872.0ms/step | 5.2 samples/s +Step 36020/44000 | Loss: 2.8643 | LR: 4.08e-05 | 1386.1ms/step | 14.4 samples/s +Step 36030/44000 | Loss: 2.8746 | LR: 4.07e-05 | 1386.4ms/step | 14.4 samples/s +Step 36040/44000 | Loss: 2.8636 | LR: 4.06e-05 | 1388.4ms/step | 14.4 samples/s +Step 36050/44000 | Loss: 2.8868 | LR: 4.05e-05 | 1386.5ms/step | 14.4 samples/s +Step 36060/44000 | Loss: 2.8924 | LR: 4.04e-05 | 1385.2ms/step | 14.4 samples/s +Step 36070/44000 | Loss: 2.8794 | LR: 4.03e-05 | 1388.7ms/step | 14.4 samples/s +Step 36080/44000 | Loss: 2.8659 | LR: 4.02e-05 | 1385.0ms/step | 14.4 samples/s +Step 36090/44000 | Loss: 2.8766 | LR: 4.01e-05 | 1376.9ms/step | 14.5 samples/s +Step 36100/44000 | Loss: 2.8961 | LR: 4.00e-05 | 1377.6ms/step | 14.5 samples/s +Step 36110/44000 | Loss: 2.8739 | LR: 3.99e-05 | 1382.8ms/step | 14.5 samples/s +Step 36120/44000 | Loss: 2.8873 | LR: 3.98e-05 | 1383.0ms/step | 14.5 samples/s +Step 36130/44000 | Loss: 2.8888 | LR: 3.97e-05 | 1385.1ms/step | 14.4 samples/s +Step 36140/44000 | Loss: 2.8773 | LR: 3.97e-05 | 1384.9ms/step | 14.4 samples/s +Step 36150/44000 | Loss: 2.8703 | LR: 3.96e-05 | 1386.9ms/step | 14.4 samples/s +Step 36160/44000 | Loss: 2.9119 | LR: 3.95e-05 | 1386.7ms/step | 14.4 samples/s +Step 36170/44000 | Loss: 2.8808 | LR: 3.94e-05 | 1393.4ms/step | 14.4 samples/s +Step 36180/44000 | Loss: 2.8676 | LR: 3.93e-05 | 1387.0ms/step | 14.4 samples/s +Step 36190/44000 | Loss: 2.8855 | LR: 3.92e-05 | 1386.1ms/step | 14.4 samples/s +Step 36200/44000 | Loss: 2.8517 | LR: 3.91e-05 | 1389.5ms/step | 14.4 samples/s +Step 36210/44000 | Loss: 2.8439 | LR: 3.90e-05 | 1382.0ms/step | 14.5 samples/s +Step 36220/44000 | Loss: 2.8994 | LR: 3.89e-05 | 1379.6ms/step | 14.5 samples/s +Step 36230/44000 | Loss: 2.8882 | LR: 3.88e-05 | 1382.4ms/step | 14.5 samples/s +Step 36240/44000 | Loss: 2.8717 | LR: 3.87e-05 | 1395.8ms/step | 14.3 samples/s +Step 36250/44000 | Loss: 2.8725 | LR: 3.86e-05 | 1390.3ms/step | 14.4 samples/s +Step 36260/44000 | Loss: 2.8872 | LR: 3.85e-05 | 1392.2ms/step | 14.4 samples/s +Step 36270/44000 | Loss: 2.8849 | LR: 3.84e-05 | 1396.1ms/step | 14.3 samples/s +Step 36280/44000 | Loss: 2.8724 | LR: 3.83e-05 | 1399.8ms/step | 14.3 samples/s +Step 36290/44000 | Loss: 2.8683 | LR: 3.82e-05 | 1399.1ms/step | 14.3 samples/s +Step 36300/44000 | Loss: 2.8778 | LR: 3.81e-05 | 1399.2ms/step | 14.3 samples/s +Step 36310/44000 | Loss: 2.8703 | LR: 3.80e-05 | 1399.7ms/step | 14.3 samples/s +Step 36320/44000 | Loss: 2.8300 | LR: 3.79e-05 | 1397.9ms/step | 14.3 samples/s +Step 36330/44000 | Loss: 2.8445 | LR: 3.78e-05 | 1398.1ms/step | 14.3 samples/s +Step 36340/44000 | Loss: 2.8659 | LR: 3.77e-05 | 1395.4ms/step | 14.3 samples/s +Step 36350/44000 | Loss: 2.8618 | LR: 3.76e-05 | 1394.6ms/step | 14.3 samples/s +Step 36360/44000 | Loss: 2.9026 | LR: 3.75e-05 | 1398.1ms/step | 14.3 samples/s +Step 36370/44000 | Loss: 2.8609 | LR: 3.74e-05 | 1399.8ms/step | 14.3 samples/s +Step 36380/44000 | Loss: 2.8805 | LR: 3.73e-05 | 1398.5ms/step | 14.3 samples/s +Step 36390/44000 | Loss: 2.8652 | LR: 3.72e-05 | 1399.6ms/step | 14.3 samples/s +Step 36400/44000 | Loss: 2.8861 | LR: 3.71e-05 | 1397.1ms/step | 14.3 samples/s +Step 36410/44000 | Loss: 2.8740 | LR: 3.70e-05 | 1397.0ms/step | 14.3 samples/s +Step 36420/44000 | Loss: 2.8898 | LR: 3.69e-05 | 1397.7ms/step | 14.3 samples/s +Step 36430/44000 | Loss: 2.8879 | LR: 3.69e-05 | 1396.4ms/step | 14.3 samples/s +Step 36440/44000 | Loss: 2.8783 | LR: 3.68e-05 | 1396.2ms/step | 14.3 samples/s +Step 36450/44000 | Loss: 2.8686 | LR: 3.67e-05 | 1395.8ms/step | 14.3 samples/s +Step 36460/44000 | Loss: 2.8975 | LR: 3.66e-05 | 1391.8ms/step | 14.4 samples/s +Step 36470/44000 | Loss: 2.8620 | LR: 3.65e-05 | 1396.0ms/step | 14.3 samples/s +Step 36480/44000 | Loss: 2.8764 | LR: 3.64e-05 | 1395.3ms/step | 14.3 samples/s +Step 36490/44000 | Loss: 2.8717 | LR: 3.63e-05 | 1394.0ms/step | 14.3 samples/s +Step 36500/44000 | Loss: 2.8842 | LR: 3.62e-05 | 1396.8ms/step | 14.3 samples/s +Step 36510/44000 | Loss: 2.8509 | LR: 3.61e-05 | 1398.3ms/step | 14.3 samples/s +Step 36520/44000 | Loss: 2.8739 | LR: 3.60e-05 | 1398.9ms/step | 14.3 samples/s +Step 36530/44000 | Loss: 2.8722 | LR: 3.59e-05 | 1399.2ms/step | 14.3 samples/s +Step 36540/44000 | Loss: 2.8604 | LR: 3.58e-05 | 1395.7ms/step | 14.3 samples/s +Step 36550/44000 | Loss: 2.8976 | LR: 3.57e-05 | 1396.0ms/step | 14.3 samples/s +Step 36560/44000 | Loss: 2.8928 | LR: 3.56e-05 | 1399.3ms/step | 14.3 samples/s +Step 36570/44000 | Loss: 2.9031 | LR: 3.55e-05 | 1396.2ms/step | 14.3 samples/s +Step 36580/44000 | Loss: 2.8734 | LR: 3.54e-05 | 1396.8ms/step | 14.3 samples/s +Step 36590/44000 | Loss: 2.8814 | LR: 3.53e-05 | 1399.7ms/step | 14.3 samples/s +Step 36600/44000 | Loss: 2.8668 | LR: 3.53e-05 | 1395.9ms/step | 14.3 samples/s +Step 36610/44000 | Loss: 2.8945 | LR: 3.52e-05 | 1398.5ms/step | 14.3 samples/s +Step 36620/44000 | Loss: 2.8842 | LR: 3.51e-05 | 1395.8ms/step | 14.3 samples/s +Step 36630/44000 | Loss: 2.8814 | LR: 3.50e-05 | 1391.9ms/step | 14.4 samples/s +Step 36640/44000 | Loss: 2.8690 | LR: 3.49e-05 | 1398.2ms/step | 14.3 samples/s +Step 36650/44000 | Loss: 2.8659 | LR: 3.48e-05 | 1397.9ms/step | 14.3 samples/s +Step 36660/44000 | Loss: 2.9269 | LR: 3.47e-05 | 1400.0ms/step | 14.3 samples/s +Step 36670/44000 | Loss: 2.8710 | LR: 3.46e-05 | 1395.0ms/step | 14.3 samples/s +Step 36680/44000 | Loss: 2.8941 | LR: 3.45e-05 | 1399.3ms/step | 14.3 samples/s +Step 36690/44000 | Loss: 2.8945 | LR: 3.44e-05 | 1396.2ms/step | 14.3 samples/s +Step 36700/44000 | Loss: 2.8700 | LR: 3.43e-05 | 1395.9ms/step | 14.3 samples/s +Step 36710/44000 | Loss: 2.8617 | LR: 3.42e-05 | 1396.0ms/step | 14.3 samples/s +Step 36720/44000 | Loss: 2.8748 | LR: 3.41e-05 | 1400.0ms/step | 14.3 samples/s +Step 36730/44000 | Loss: 2.8687 | LR: 3.41e-05 | 1391.2ms/step | 14.4 samples/s +Step 36740/44000 | Loss: 2.9037 | LR: 3.40e-05 | 1392.3ms/step | 14.4 samples/s +Step 36750/44000 | Loss: 2.8764 | LR: 3.39e-05 | 1392.7ms/step | 14.4 samples/s +Step 36760/44000 | Loss: 2.8746 | LR: 3.38e-05 | 1395.0ms/step | 14.3 samples/s +Step 36770/44000 | Loss: 2.9035 | LR: 3.37e-05 | 1396.5ms/step | 14.3 samples/s +Step 36780/44000 | Loss: 2.8756 | LR: 3.36e-05 | 1394.4ms/step | 14.3 samples/s +Step 36790/44000 | Loss: 2.8562 | LR: 3.35e-05 | 1396.1ms/step | 14.3 samples/s +Step 36800/44000 | Loss: 2.8787 | LR: 3.34e-05 | 1394.0ms/step | 14.3 samples/s +Step 36810/44000 | Loss: 2.8978 | LR: 3.33e-05 | 1397.5ms/step | 14.3 samples/s +Step 36820/44000 | Loss: 2.9138 | LR: 3.32e-05 | 1394.4ms/step | 14.3 samples/s +Step 36830/44000 | Loss: 2.9074 | LR: 3.31e-05 | 1396.1ms/step | 14.3 samples/s +Step 36840/44000 | Loss: 2.9045 | LR: 3.31e-05 | 1393.7ms/step | 14.4 samples/s +Step 36850/44000 | Loss: 2.8979 | LR: 3.30e-05 | 1396.4ms/step | 14.3 samples/s +Step 36860/44000 | Loss: 2.8673 | LR: 3.29e-05 | 1398.1ms/step | 14.3 samples/s +Step 36870/44000 | Loss: 2.8732 | LR: 3.28e-05 | 1396.7ms/step | 14.3 samples/s +Step 36880/44000 | Loss: 2.8964 | LR: 3.27e-05 | 1395.5ms/step | 14.3 samples/s +Step 36890/44000 | Loss: 2.8973 | LR: 3.26e-05 | 1395.2ms/step | 14.3 samples/s +Step 36900/44000 | Loss: 2.8590 | LR: 3.25e-05 | 1396.6ms/step | 14.3 samples/s +Step 36910/44000 | Loss: 2.8748 | LR: 3.24e-05 | 1398.2ms/step | 14.3 samples/s +Step 36920/44000 | Loss: 2.8576 | LR: 3.23e-05 | 1399.6ms/step | 14.3 samples/s +Step 36930/44000 | Loss: 2.8741 | LR: 3.22e-05 | 1398.2ms/step | 14.3 samples/s +Step 36940/44000 | Loss: 2.8940 | LR: 3.22e-05 | 1394.1ms/step | 14.3 samples/s +Step 36950/44000 | Loss: 2.8864 | LR: 3.21e-05 | 1397.8ms/step | 14.3 samples/s +Step 36960/44000 | Loss: 2.8891 | LR: 3.20e-05 | 1397.9ms/step | 14.3 samples/s +Step 36970/44000 | Loss: 2.8831 | LR: 3.19e-05 | 1394.4ms/step | 14.3 samples/s +Step 36980/44000 | Loss: 2.9009 | LR: 3.18e-05 | 1399.0ms/step | 14.3 samples/s +Step 36990/44000 | Loss: 2.8759 | LR: 3.17e-05 | 1395.5ms/step | 14.3 samples/s +Step 37000/44000 | Loss: 2.8833 | LR: 3.16e-05 | 1397.2ms/step | 14.3 samples/s +Validation at step 37000: Loss = 2.8939, Perplexity = 18.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-37000 +Step 37010/44000 | Loss: 2.8575 | LR: 3.15e-05 | 3934.0ms/step | 5.1 samples/s +Step 37020/44000 | Loss: 2.8864 | LR: 3.15e-05 | 1390.1ms/step | 14.4 samples/s +Step 37030/44000 | Loss: 2.8456 | LR: 3.14e-05 | 1396.2ms/step | 14.3 samples/s +Step 37040/44000 | Loss: 2.8619 | LR: 3.13e-05 | 1396.0ms/step | 14.3 samples/s +Step 37050/44000 | Loss: 2.8784 | LR: 3.12e-05 | 1395.9ms/step | 14.3 samples/s +Step 37060/44000 | Loss: 2.8555 | LR: 3.11e-05 | 1398.1ms/step | 14.3 samples/s +Step 37070/44000 | Loss: 2.8884 | LR: 3.10e-05 | 1397.8ms/step | 14.3 samples/s +Step 37080/44000 | Loss: 2.8519 | LR: 3.09e-05 | 1398.2ms/step | 14.3 samples/s +Step 37090/44000 | Loss: 2.8587 | LR: 3.08e-05 | 1396.0ms/step | 14.3 samples/s +Step 37100/44000 | Loss: 2.8916 | LR: 3.07e-05 | 1394.1ms/step | 14.3 samples/s +Step 37110/44000 | Loss: 2.9098 | LR: 3.07e-05 | 1391.6ms/step | 14.4 samples/s +Step 37120/44000 | Loss: 2.8573 | LR: 3.06e-05 | 1394.4ms/step | 14.3 samples/s +Step 37130/44000 | Loss: 2.8747 | LR: 3.05e-05 | 1389.6ms/step | 14.4 samples/s +Step 37140/44000 | Loss: 2.8856 | LR: 3.04e-05 | 1391.8ms/step | 14.4 samples/s +Step 37150/44000 | Loss: 2.9027 | LR: 3.03e-05 | 1390.5ms/step | 14.4 samples/s +Step 37160/44000 | Loss: 2.8784 | LR: 3.02e-05 | 1393.0ms/step | 14.4 samples/s +Step 37170/44000 | Loss: 2.8813 | LR: 3.01e-05 | 1390.7ms/step | 14.4 samples/s +Step 37180/44000 | Loss: 2.8662 | LR: 3.01e-05 | 1388.7ms/step | 14.4 samples/s +Step 37190/44000 | Loss: 2.8911 | LR: 3.00e-05 | 1389.3ms/step | 14.4 samples/s +Step 37200/44000 | Loss: 2.8746 | LR: 2.99e-05 | 1389.6ms/step | 14.4 samples/s +Step 37210/44000 | Loss: 2.8460 | LR: 2.98e-05 | 1392.6ms/step | 14.4 samples/s +Step 37220/44000 | Loss: 2.8648 | LR: 2.97e-05 | 1392.1ms/step | 14.4 samples/s +Step 37230/44000 | Loss: 2.8647 | LR: 2.96e-05 | 1390.6ms/step | 14.4 samples/s +Step 37240/44000 | Loss: 2.8473 | LR: 2.95e-05 | 1386.4ms/step | 14.4 samples/s +Step 37250/44000 | Loss: 2.8457 | LR: 2.95e-05 | 1384.9ms/step | 14.4 samples/s +Step 37260/44000 | Loss: 2.8714 | LR: 2.94e-05 | 1387.6ms/step | 14.4 samples/s +Step 37270/44000 | Loss: 2.8803 | LR: 2.93e-05 | 1388.2ms/step | 14.4 samples/s +Step 37280/44000 | Loss: 2.8754 | LR: 2.92e-05 | 1388.6ms/step | 14.4 samples/s +Step 37290/44000 | Loss: 2.9006 | LR: 2.91e-05 | 1391.9ms/step | 14.4 samples/s +Step 37300/44000 | Loss: 2.8685 | LR: 2.90e-05 | 1391.0ms/step | 14.4 samples/s +Step 37310/44000 | Loss: 2.8874 | LR: 2.89e-05 | 1390.7ms/step | 14.4 samples/s +Step 37320/44000 | Loss: 2.8800 | LR: 2.89e-05 | 1393.7ms/step | 14.4 samples/s +Step 37330/44000 | Loss: 2.8351 | LR: 2.88e-05 | 1391.1ms/step | 14.4 samples/s +Step 37340/44000 | Loss: 2.8762 | LR: 2.87e-05 | 1391.4ms/step | 14.4 samples/s +Step 37350/44000 | Loss: 2.8645 | LR: 2.86e-05 | 1391.6ms/step | 14.4 samples/s +Step 37360/44000 | Loss: 2.8797 | LR: 2.85e-05 | 1392.4ms/step | 14.4 samples/s +Step 37370/44000 | Loss: 2.8752 | LR: 2.84e-05 | 1392.2ms/step | 14.4 samples/s +Step 37380/44000 | Loss: 2.8611 | LR: 2.84e-05 | 1388.4ms/step | 14.4 samples/s +Step 37390/44000 | Loss: 2.8496 | LR: 2.83e-05 | 1388.8ms/step | 14.4 samples/s +Step 37400/44000 | Loss: 2.9013 | LR: 2.82e-05 | 1390.8ms/step | 14.4 samples/s +Step 37410/44000 | Loss: 2.8810 | LR: 2.81e-05 | 1392.8ms/step | 14.4 samples/s +Step 37420/44000 | Loss: 2.8903 | LR: 2.80e-05 | 1393.7ms/step | 14.3 samples/s +Step 37430/44000 | Loss: 2.9075 | LR: 2.79e-05 | 1391.3ms/step | 14.4 samples/s +Step 37440/44000 | Loss: 2.8619 | LR: 2.78e-05 | 1389.9ms/step | 14.4 samples/s +Step 37450/44000 | Loss: 2.8550 | LR: 2.78e-05 | 1390.5ms/step | 14.4 samples/s +Step 37460/44000 | Loss: 2.8830 | LR: 2.77e-05 | 1391.6ms/step | 14.4 samples/s +Step 37470/44000 | Loss: 2.8695 | LR: 2.76e-05 | 1389.5ms/step | 14.4 samples/s +Step 37480/44000 | Loss: 2.8697 | LR: 2.75e-05 | 1390.0ms/step | 14.4 samples/s +Step 37490/44000 | Loss: 2.8923 | LR: 2.74e-05 | 1392.1ms/step | 14.4 samples/s +Step 37500/44000 | Loss: 2.8658 | LR: 2.74e-05 | 1394.1ms/step | 14.3 samples/s +Step 37510/44000 | Loss: 2.8941 | LR: 2.73e-05 | 1392.1ms/step | 14.4 samples/s +Step 37520/44000 | Loss: 2.8868 | LR: 2.72e-05 | 1394.0ms/step | 14.3 samples/s +Step 37530/44000 | Loss: 2.8989 | LR: 2.71e-05 | 1396.1ms/step | 14.3 samples/s +Step 37540/44000 | Loss: 2.8726 | LR: 2.70e-05 | 1392.0ms/step | 14.4 samples/s +Step 37550/44000 | Loss: 2.8523 | LR: 2.69e-05 | 1385.0ms/step | 14.4 samples/s +Step 37560/44000 | Loss: 2.8610 | LR: 2.69e-05 | 1387.7ms/step | 14.4 samples/s +Step 37570/44000 | Loss: 2.8671 | LR: 2.68e-05 | 1387.1ms/step | 14.4 samples/s +Step 37580/44000 | Loss: 2.8868 | LR: 2.67e-05 | 1389.2ms/step | 14.4 samples/s +Step 37590/44000 | Loss: 2.8709 | LR: 2.66e-05 | 1388.9ms/step | 14.4 samples/s +Step 37600/44000 | Loss: 2.8567 | LR: 2.65e-05 | 1388.2ms/step | 14.4 samples/s +Step 37610/44000 | Loss: 2.8926 | LR: 2.65e-05 | 1392.2ms/step | 14.4 samples/s +Step 37620/44000 | Loss: 2.8762 | LR: 2.64e-05 | 1391.9ms/step | 14.4 samples/s +Step 37630/44000 | Loss: 2.8875 | LR: 2.63e-05 | 1391.9ms/step | 14.4 samples/s +Step 37640/44000 | Loss: 2.8970 | LR: 2.62e-05 | 1392.5ms/step | 14.4 samples/s +Step 37650/44000 | Loss: 2.8923 | LR: 2.61e-05 | 1392.1ms/step | 14.4 samples/s +Step 37660/44000 | Loss: 2.8622 | LR: 2.60e-05 | 1389.4ms/step | 14.4 samples/s +Step 37670/44000 | Loss: 2.8939 | LR: 2.60e-05 | 1390.3ms/step | 14.4 samples/s +Step 37680/44000 | Loss: 2.8916 | LR: 2.59e-05 | 1397.3ms/step | 14.3 samples/s +Step 37690/44000 | Loss: 2.8935 | LR: 2.58e-05 | 1392.5ms/step | 14.4 samples/s +Step 37700/44000 | Loss: 2.8979 | LR: 2.57e-05 | 1393.7ms/step | 14.4 samples/s +Step 37710/44000 | Loss: 2.8620 | LR: 2.56e-05 | 1392.1ms/step | 14.4 samples/s +Step 37720/44000 | Loss: 2.8706 | LR: 2.56e-05 | 1386.9ms/step | 14.4 samples/s +Step 37730/44000 | Loss: 2.8522 | LR: 2.55e-05 | 1388.2ms/step | 14.4 samples/s +Step 37740/44000 | Loss: 2.8663 | LR: 2.54e-05 | 1390.1ms/step | 14.4 samples/s +Step 37750/44000 | Loss: 2.8821 | LR: 2.53e-05 | 1389.0ms/step | 14.4 samples/s +Step 37760/44000 | Loss: 2.8973 | LR: 2.52e-05 | 1384.1ms/step | 14.5 samples/s +Step 37770/44000 | Loss: 2.8565 | LR: 2.52e-05 | 1385.6ms/step | 14.4 samples/s +Step 37780/44000 | Loss: 2.8717 | LR: 2.51e-05 | 1385.7ms/step | 14.4 samples/s +Step 37790/44000 | Loss: 2.8501 | LR: 2.50e-05 | 1388.1ms/step | 14.4 samples/s +Step 37800/44000 | Loss: 2.8845 | LR: 2.49e-05 | 1388.3ms/step | 14.4 samples/s +Step 37810/44000 | Loss: 2.8881 | LR: 2.48e-05 | 1391.4ms/step | 14.4 samples/s +Step 37820/44000 | Loss: 2.8721 | LR: 2.48e-05 | 1390.1ms/step | 14.4 samples/s +Step 37830/44000 | Loss: 2.8687 | LR: 2.47e-05 | 1391.4ms/step | 14.4 samples/s +Step 37840/44000 | Loss: 2.9119 | LR: 2.46e-05 | 1390.1ms/step | 14.4 samples/s +Step 37850/44000 | Loss: 2.8716 | LR: 2.45e-05 | 1393.3ms/step | 14.4 samples/s +Step 37860/44000 | Loss: 2.8677 | LR: 2.45e-05 | 1391.9ms/step | 14.4 samples/s +Step 37870/44000 | Loss: 2.8863 | LR: 2.44e-05 | 1394.4ms/step | 14.3 samples/s +Step 37880/44000 | Loss: 2.8665 | LR: 2.43e-05 | 1394.0ms/step | 14.3 samples/s +Step 37890/44000 | Loss: 2.8838 | LR: 2.42e-05 | 1396.0ms/step | 14.3 samples/s +Step 37900/44000 | Loss: 2.8767 | LR: 2.41e-05 | 1399.9ms/step | 14.3 samples/s +Step 37910/44000 | Loss: 2.8772 | LR: 2.41e-05 | 1390.2ms/step | 14.4 samples/s +Step 37920/44000 | Loss: 2.8485 | LR: 2.40e-05 | 1387.5ms/step | 14.4 samples/s +Step 37930/44000 | Loss: 2.8881 | LR: 2.39e-05 | 1390.1ms/step | 14.4 samples/s +Step 37940/44000 | Loss: 2.8664 | LR: 2.38e-05 | 1390.0ms/step | 14.4 samples/s +Step 37950/44000 | Loss: 2.8580 | LR: 2.38e-05 | 1390.4ms/step | 14.4 samples/s +Step 37960/44000 | Loss: 2.8657 | LR: 2.37e-05 | 1394.1ms/step | 14.3 samples/s +Step 37970/44000 | Loss: 2.8535 | LR: 2.36e-05 | 1395.9ms/step | 14.3 samples/s +Step 37980/44000 | Loss: 2.8896 | LR: 2.35e-05 | 1395.6ms/step | 14.3 samples/s +Step 37990/44000 | Loss: 2.8799 | LR: 2.34e-05 | 1395.3ms/step | 14.3 samples/s +Step 38000/44000 | Loss: 2.8806 | LR: 2.34e-05 | 1395.0ms/step | 14.3 samples/s +Validation at step 38000: Loss = 2.8935, Perplexity = 18.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-38000 +Step 38010/44000 | Loss: 2.8892 | LR: 2.33e-05 | 3901.8ms/step | 5.1 samples/s +Step 38020/44000 | Loss: 2.8364 | LR: 2.32e-05 | 1384.3ms/step | 14.4 samples/s +Step 38030/44000 | Loss: 2.8455 | LR: 2.31e-05 | 1389.3ms/step | 14.4 samples/s +Step 38040/44000 | Loss: 2.8794 | LR: 2.31e-05 | 1387.7ms/step | 14.4 samples/s +Step 38050/44000 | Loss: 2.8755 | LR: 2.30e-05 | 1390.3ms/step | 14.4 samples/s +Step 38060/44000 | Loss: 2.8660 | LR: 2.29e-05 | 1390.6ms/step | 14.4 samples/s +Step 38070/44000 | Loss: 2.8540 | LR: 2.28e-05 | 1391.0ms/step | 14.4 samples/s +Step 38080/44000 | Loss: 2.8673 | LR: 2.28e-05 | 1389.1ms/step | 14.4 samples/s +Step 38090/44000 | Loss: 2.9043 | LR: 2.27e-05 | 1391.8ms/step | 14.4 samples/s +Step 38100/44000 | Loss: 2.8630 | LR: 2.26e-05 | 1396.1ms/step | 14.3 samples/s +Step 38110/44000 | Loss: 2.8862 | LR: 2.25e-05 | 1393.3ms/step | 14.4 samples/s +Step 38120/44000 | Loss: 2.8194 | LR: 2.25e-05 | 1394.6ms/step | 14.3 samples/s +Step 38130/44000 | Loss: 2.8885 | LR: 2.24e-05 | 1392.2ms/step | 14.4 samples/s +Step 38140/44000 | Loss: 2.8894 | LR: 2.23e-05 | 1389.8ms/step | 14.4 samples/s +Step 38150/44000 | Loss: 2.8763 | LR: 2.22e-05 | 1392.0ms/step | 14.4 samples/s +Step 38160/44000 | Loss: 2.8759 | LR: 2.22e-05 | 1390.9ms/step | 14.4 samples/s +Step 38170/44000 | Loss: 2.8806 | LR: 2.21e-05 | 1388.4ms/step | 14.4 samples/s +Step 38180/44000 | Loss: 2.8766 | LR: 2.20e-05 | 1392.2ms/step | 14.4 samples/s +Step 38190/44000 | Loss: 2.8642 | LR: 2.19e-05 | 1389.8ms/step | 14.4 samples/s +Step 38200/44000 | Loss: 2.8764 | LR: 2.19e-05 | 1387.5ms/step | 14.4 samples/s +Step 38210/44000 | Loss: 2.9138 | LR: 2.18e-05 | 1389.3ms/step | 14.4 samples/s +Step 38220/44000 | Loss: 2.8997 | LR: 2.17e-05 | 1392.9ms/step | 14.4 samples/s +Step 38230/44000 | Loss: 2.8687 | LR: 2.16e-05 | 1392.2ms/step | 14.4 samples/s +Step 38240/44000 | Loss: 2.8941 | LR: 2.16e-05 | 1389.7ms/step | 14.4 samples/s +Step 38250/44000 | Loss: 2.8678 | LR: 2.15e-05 | 1393.2ms/step | 14.4 samples/s +Step 38260/44000 | Loss: 2.8873 | LR: 2.14e-05 | 1392.3ms/step | 14.4 samples/s +Step 38270/44000 | Loss: 2.8705 | LR: 2.13e-05 | 1390.4ms/step | 14.4 samples/s +Step 38280/44000 | Loss: 2.9018 | LR: 2.13e-05 | 1387.3ms/step | 14.4 samples/s +Step 38290/44000 | Loss: 2.8759 | LR: 2.12e-05 | 1387.3ms/step | 14.4 samples/s +Step 38300/44000 | Loss: 2.8564 | LR: 2.11e-05 | 1391.2ms/step | 14.4 samples/s +Step 38310/44000 | Loss: 2.8983 | LR: 2.11e-05 | 1391.3ms/step | 14.4 samples/s +Step 38320/44000 | Loss: 2.8798 | LR: 2.10e-05 | 1388.5ms/step | 14.4 samples/s +Step 38330/44000 | Loss: 2.8771 | LR: 2.09e-05 | 1389.8ms/step | 14.4 samples/s +Step 38340/44000 | Loss: 2.8557 | LR: 2.08e-05 | 1390.4ms/step | 14.4 samples/s +Step 38350/44000 | Loss: 2.8748 | LR: 2.08e-05 | 1391.8ms/step | 14.4 samples/s +Step 38360/44000 | Loss: 2.8501 | LR: 2.07e-05 | 1391.9ms/step | 14.4 samples/s +Step 38370/44000 | Loss: 2.9411 | LR: 2.06e-05 | 1393.6ms/step | 14.4 samples/s +Step 38380/44000 | Loss: 2.8875 | LR: 2.05e-05 | 1388.4ms/step | 14.4 samples/s +Step 38390/44000 | Loss: 2.8605 | LR: 2.05e-05 | 1392.1ms/step | 14.4 samples/s +Step 38400/44000 | Loss: 2.8584 | LR: 2.04e-05 | 1393.6ms/step | 14.4 samples/s +Step 38410/44000 | Loss: 2.8557 | LR: 2.03e-05 | 1392.1ms/step | 14.4 samples/s +Step 38420/44000 | Loss: 2.8600 | LR: 2.03e-05 | 1388.4ms/step | 14.4 samples/s +Step 38430/44000 | Loss: 2.8199 | LR: 2.02e-05 | 1390.1ms/step | 14.4 samples/s +Step 38440/44000 | Loss: 2.8869 | LR: 2.01e-05 | 1391.4ms/step | 14.4 samples/s +Step 38450/44000 | Loss: 2.8897 | LR: 2.00e-05 | 1390.2ms/step | 14.4 samples/s +Step 38460/44000 | Loss: 2.8698 | LR: 2.00e-05 | 1390.1ms/step | 14.4 samples/s +Step 38470/44000 | Loss: 2.8766 | LR: 1.99e-05 | 1391.9ms/step | 14.4 samples/s +Step 38480/44000 | Loss: 2.8886 | LR: 1.98e-05 | 1392.3ms/step | 14.4 samples/s +Step 38490/44000 | Loss: 2.8543 | LR: 1.98e-05 | 1392.8ms/step | 14.4 samples/s +Step 38500/44000 | Loss: 2.8902 | LR: 1.97e-05 | 1391.2ms/step | 14.4 samples/s +Step 38510/44000 | Loss: 2.8630 | LR: 1.96e-05 | 1393.4ms/step | 14.4 samples/s +Step 38520/44000 | Loss: 2.8968 | LR: 1.95e-05 | 1390.3ms/step | 14.4 samples/s +Step 38530/44000 | Loss: 2.8799 | LR: 1.95e-05 | 1388.3ms/step | 14.4 samples/s +Step 38540/44000 | Loss: 2.8537 | LR: 1.94e-05 | 1389.6ms/step | 14.4 samples/s +Step 38550/44000 | Loss: 2.8794 | LR: 1.93e-05 | 1389.4ms/step | 14.4 samples/s +Step 38560/44000 | Loss: 2.8836 | LR: 1.93e-05 | 1388.7ms/step | 14.4 samples/s +Step 38570/44000 | Loss: 2.8803 | LR: 1.92e-05 | 1391.3ms/step | 14.4 samples/s +Step 38580/44000 | Loss: 2.9048 | LR: 1.91e-05 | 1389.2ms/step | 14.4 samples/s +Step 38590/44000 | Loss: 2.8557 | LR: 1.91e-05 | 1393.2ms/step | 14.4 samples/s +Step 38600/44000 | Loss: 2.8697 | LR: 1.90e-05 | 1389.9ms/step | 14.4 samples/s +Step 38610/44000 | Loss: 2.8523 | LR: 1.89e-05 | 1389.2ms/step | 14.4 samples/s +Step 38620/44000 | Loss: 2.8515 | LR: 1.88e-05 | 1393.6ms/step | 14.4 samples/s +Step 38630/44000 | Loss: 2.8926 | LR: 1.88e-05 | 1392.9ms/step | 14.4 samples/s +Step 38640/44000 | Loss: 2.8693 | LR: 1.87e-05 | 1391.0ms/step | 14.4 samples/s +Step 38650/44000 | Loss: 2.8489 | LR: 1.86e-05 | 1392.8ms/step | 14.4 samples/s +Step 38660/44000 | Loss: 2.8792 | LR: 1.86e-05 | 1389.1ms/step | 14.4 samples/s +Step 38670/44000 | Loss: 2.8810 | LR: 1.85e-05 | 1391.8ms/step | 14.4 samples/s +Step 38680/44000 | Loss: 2.8855 | LR: 1.84e-05 | 1390.2ms/step | 14.4 samples/s +Step 38690/44000 | Loss: 2.8816 | LR: 1.84e-05 | 1391.9ms/step | 14.4 samples/s +Step 38700/44000 | Loss: 2.8587 | LR: 1.83e-05 | 1387.2ms/step | 14.4 samples/s +Step 38710/44000 | Loss: 2.8598 | LR: 1.82e-05 | 1392.8ms/step | 14.4 samples/s +Step 38720/44000 | Loss: 2.8584 | LR: 1.82e-05 | 1391.5ms/step | 14.4 samples/s +Step 38730/44000 | Loss: 2.9088 | LR: 1.81e-05 | 1392.1ms/step | 14.4 samples/s +Step 38740/44000 | Loss: 2.8452 | LR: 1.80e-05 | 1388.4ms/step | 14.4 samples/s +Step 38750/44000 | Loss: 2.8768 | LR: 1.80e-05 | 1388.7ms/step | 14.4 samples/s +Step 38760/44000 | Loss: 2.8517 | LR: 1.79e-05 | 1389.0ms/step | 14.4 samples/s +Step 38770/44000 | Loss: 2.8887 | LR: 1.78e-05 | 1389.8ms/step | 14.4 samples/s +Step 38780/44000 | Loss: 2.8755 | LR: 1.78e-05 | 1388.3ms/step | 14.4 samples/s +Step 38790/44000 | Loss: 2.8613 | LR: 1.77e-05 | 1388.5ms/step | 14.4 samples/s +Step 38800/44000 | Loss: 2.8962 | LR: 1.76e-05 | 1390.1ms/step | 14.4 samples/s +Step 38810/44000 | Loss: 2.8870 | LR: 1.76e-05 | 1391.5ms/step | 14.4 samples/s +Step 38820/44000 | Loss: 2.8775 | LR: 1.75e-05 | 1386.4ms/step | 14.4 samples/s +Step 38830/44000 | Loss: 2.8669 | LR: 1.74e-05 | 1389.8ms/step | 14.4 samples/s +Step 38840/44000 | Loss: 2.8769 | LR: 1.74e-05 | 1391.2ms/step | 14.4 samples/s +Step 38850/44000 | Loss: 2.8293 | LR: 1.73e-05 | 1392.7ms/step | 14.4 samples/s +Step 38860/44000 | Loss: 2.9117 | LR: 1.72e-05 | 1394.1ms/step | 14.3 samples/s +Step 38870/44000 | Loss: 2.8687 | LR: 1.72e-05 | 1390.8ms/step | 14.4 samples/s +Step 38880/44000 | Loss: 2.8892 | LR: 1.71e-05 | 1389.7ms/step | 14.4 samples/s +Step 38890/44000 | Loss: 2.8772 | LR: 1.70e-05 | 1390.3ms/step | 14.4 samples/s +Step 38900/44000 | Loss: 2.8818 | LR: 1.70e-05 | 1391.6ms/step | 14.4 samples/s +Step 38910/44000 | Loss: 2.8464 | LR: 1.69e-05 | 1391.9ms/step | 14.4 samples/s +Step 38920/44000 | Loss: 2.8989 | LR: 1.68e-05 | 1391.3ms/step | 14.4 samples/s +Step 38930/44000 | Loss: 2.9211 | LR: 1.68e-05 | 1392.8ms/step | 14.4 samples/s +Step 38940/44000 | Loss: 2.8595 | LR: 1.67e-05 | 1392.6ms/step | 14.4 samples/s +Step 38950/44000 | Loss: 2.9032 | LR: 1.66e-05 | 1393.3ms/step | 14.4 samples/s +Step 38960/44000 | Loss: 2.8701 | LR: 1.66e-05 | 1393.4ms/step | 14.4 samples/s +Step 38970/44000 | Loss: 2.9141 | LR: 1.65e-05 | 1392.6ms/step | 14.4 samples/s +Step 38980/44000 | Loss: 2.8572 | LR: 1.64e-05 | 1395.9ms/step | 14.3 samples/s +Step 38990/44000 | Loss: 2.9012 | LR: 1.64e-05 | 1394.7ms/step | 14.3 samples/s +Step 39000/44000 | Loss: 2.8568 | LR: 1.63e-05 | 1391.1ms/step | 14.4 samples/s +Validation at step 39000: Loss = 2.8933, Perplexity = 18.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-39000 +Step 39010/44000 | Loss: 2.8947 | LR: 1.62e-05 | 3896.6ms/step | 5.1 samples/s +Step 39020/44000 | Loss: 2.8652 | LR: 1.62e-05 | 1383.8ms/step | 14.5 samples/s +Step 39030/44000 | Loss: 2.8546 | LR: 1.61e-05 | 1385.9ms/step | 14.4 samples/s +Step 39040/44000 | Loss: 2.8962 | LR: 1.61e-05 | 1386.9ms/step | 14.4 samples/s +Step 39050/44000 | Loss: 2.8814 | LR: 1.60e-05 | 1386.0ms/step | 14.4 samples/s +Step 39060/44000 | Loss: 2.8731 | LR: 1.59e-05 | 1386.4ms/step | 14.4 samples/s +Step 39070/44000 | Loss: 2.8751 | LR: 1.59e-05 | 1390.7ms/step | 14.4 samples/s +Step 39080/44000 | Loss: 2.9002 | LR: 1.58e-05 | 1391.6ms/step | 14.4 samples/s +Step 39090/44000 | Loss: 2.8531 | LR: 1.57e-05 | 1384.7ms/step | 14.4 samples/s +Step 39100/44000 | Loss: 2.8646 | LR: 1.57e-05 | 1381.3ms/step | 14.5 samples/s +Step 39110/44000 | Loss: 2.8874 | LR: 1.56e-05 | 1386.3ms/step | 14.4 samples/s +Step 39120/44000 | Loss: 2.8493 | LR: 1.55e-05 | 1388.4ms/step | 14.4 samples/s +Step 39130/44000 | Loss: 2.8916 | LR: 1.55e-05 | 1389.6ms/step | 14.4 samples/s +Step 39140/44000 | Loss: 2.8829 | LR: 1.54e-05 | 1387.0ms/step | 14.4 samples/s +Step 39150/44000 | Loss: 2.8668 | LR: 1.54e-05 | 1384.6ms/step | 14.4 samples/s +Step 39160/44000 | Loss: 2.8911 | LR: 1.53e-05 | 1384.5ms/step | 14.4 samples/s +Step 39170/44000 | Loss: 2.8927 | LR: 1.52e-05 | 1390.2ms/step | 14.4 samples/s +Step 39180/44000 | Loss: 2.8913 | LR: 1.52e-05 | 1391.8ms/step | 14.4 samples/s +Step 39190/44000 | Loss: 2.8934 | LR: 1.51e-05 | 1390.8ms/step | 14.4 samples/s +Step 39200/44000 | Loss: 2.8883 | LR: 1.50e-05 | 1391.3ms/step | 14.4 samples/s +Step 39210/44000 | Loss: 2.8897 | LR: 1.50e-05 | 1391.7ms/step | 14.4 samples/s +Step 39220/44000 | Loss: 2.8841 | LR: 1.49e-05 | 1391.4ms/step | 14.4 samples/s +Step 39230/44000 | Loss: 2.8784 | LR: 1.49e-05 | 1392.9ms/step | 14.4 samples/s +Step 39240/44000 | Loss: 2.8853 | LR: 1.48e-05 | 1391.9ms/step | 14.4 samples/s +Step 39250/44000 | Loss: 2.8860 | LR: 1.47e-05 | 1391.4ms/step | 14.4 samples/s +Step 39260/44000 | Loss: 2.8570 | LR: 1.47e-05 | 1390.8ms/step | 14.4 samples/s +Step 39270/44000 | Loss: 2.8857 | LR: 1.46e-05 | 1392.5ms/step | 14.4 samples/s +Step 39280/44000 | Loss: 2.8866 | LR: 1.46e-05 | 1391.3ms/step | 14.4 samples/s +Step 39290/44000 | Loss: 2.8972 | LR: 1.45e-05 | 1390.4ms/step | 14.4 samples/s +Step 39300/44000 | Loss: 2.8952 | LR: 1.44e-05 | 1395.4ms/step | 14.3 samples/s +Step 39310/44000 | Loss: 2.9174 | LR: 1.44e-05 | 1394.5ms/step | 14.3 samples/s +Step 39320/44000 | Loss: 2.8685 | LR: 1.43e-05 | 1398.1ms/step | 14.3 samples/s +Step 39330/44000 | Loss: 2.8524 | LR: 1.42e-05 | 1396.1ms/step | 14.3 samples/s +Step 39340/44000 | Loss: 2.9131 | LR: 1.42e-05 | 1393.7ms/step | 14.4 samples/s +Step 39350/44000 | Loss: 2.8874 | LR: 1.41e-05 | 1392.6ms/step | 14.4 samples/s +Step 39360/44000 | Loss: 2.8949 | LR: 1.41e-05 | 1393.6ms/step | 14.4 samples/s +Step 39370/44000 | Loss: 2.8901 | LR: 1.40e-05 | 1397.7ms/step | 14.3 samples/s +Step 39380/44000 | Loss: 2.8821 | LR: 1.39e-05 | 1394.3ms/step | 14.3 samples/s +Step 39390/44000 | Loss: 2.8509 | LR: 1.39e-05 | 1396.1ms/step | 14.3 samples/s +Step 39400/44000 | Loss: 2.8711 | LR: 1.38e-05 | 1394.0ms/step | 14.3 samples/s +Step 39410/44000 | Loss: 2.8925 | LR: 1.38e-05 | 1396.2ms/step | 14.3 samples/s +Step 39420/44000 | Loss: 2.8488 | LR: 1.37e-05 | 1397.1ms/step | 14.3 samples/s +Step 39430/44000 | Loss: 2.8585 | LR: 1.36e-05 | 1390.5ms/step | 14.4 samples/s +Step 39440/44000 | Loss: 2.8940 | LR: 1.36e-05 | 1389.9ms/step | 14.4 samples/s +Step 39450/44000 | Loss: 2.8702 | LR: 1.35e-05 | 1391.1ms/step | 14.4 samples/s +Step 39460/44000 | Loss: 2.8502 | LR: 1.35e-05 | 1391.4ms/step | 14.4 samples/s +Step 39470/44000 | Loss: 2.8427 | LR: 1.34e-05 | 1389.8ms/step | 14.4 samples/s +Step 39480/44000 | Loss: 2.9046 | LR: 1.34e-05 | 1391.0ms/step | 14.4 samples/s +Step 39490/44000 | Loss: 2.8894 | LR: 1.33e-05 | 1384.9ms/step | 14.4 samples/s +Step 39500/44000 | Loss: 2.8542 | LR: 1.32e-05 | 1383.5ms/step | 14.5 samples/s +Step 39510/44000 | Loss: 2.9126 | LR: 1.32e-05 | 1388.3ms/step | 14.4 samples/s +Step 39520/44000 | Loss: 2.8939 | LR: 1.31e-05 | 1390.4ms/step | 14.4 samples/s +Step 39530/44000 | Loss: 2.9131 | LR: 1.31e-05 | 1393.0ms/step | 14.4 samples/s +Step 39540/44000 | Loss: 2.8801 | LR: 1.30e-05 | 1389.1ms/step | 14.4 samples/s +Step 39550/44000 | Loss: 2.8511 | LR: 1.29e-05 | 1391.6ms/step | 14.4 samples/s +Step 39560/44000 | Loss: 2.8858 | LR: 1.29e-05 | 1392.2ms/step | 14.4 samples/s +Step 39570/44000 | Loss: 2.8603 | LR: 1.28e-05 | 1394.9ms/step | 14.3 samples/s +Step 39580/44000 | Loss: 2.8594 | LR: 1.28e-05 | 1388.9ms/step | 14.4 samples/s +Step 39590/44000 | Loss: 2.8874 | LR: 1.27e-05 | 1388.6ms/step | 14.4 samples/s +Step 39600/44000 | Loss: 2.8752 | LR: 1.27e-05 | 1390.3ms/step | 14.4 samples/s +Step 39610/44000 | Loss: 2.8531 | LR: 1.26e-05 | 1389.5ms/step | 14.4 samples/s +Step 39620/44000 | Loss: 2.8864 | LR: 1.25e-05 | 1389.0ms/step | 14.4 samples/s +Step 39630/44000 | Loss: 2.8838 | LR: 1.25e-05 | 1388.9ms/step | 14.4 samples/s +Step 39640/44000 | Loss: 2.8570 | LR: 1.24e-05 | 1393.7ms/step | 14.4 samples/s +Step 39650/44000 | Loss: 2.9064 | LR: 1.24e-05 | 1388.9ms/step | 14.4 samples/s +Step 39660/44000 | Loss: 2.8450 | LR: 1.23e-05 | 1393.6ms/step | 14.4 samples/s +Step 39670/44000 | Loss: 2.8757 | LR: 1.23e-05 | 1391.6ms/step | 14.4 samples/s +Step 39680/44000 | Loss: 2.8856 | LR: 1.22e-05 | 1394.3ms/step | 14.3 samples/s +Step 39690/44000 | Loss: 2.9022 | LR: 1.22e-05 | 1393.3ms/step | 14.4 samples/s +Step 39700/44000 | Loss: 2.8743 | LR: 1.21e-05 | 1391.0ms/step | 14.4 samples/s +Step 39710/44000 | Loss: 2.8794 | LR: 1.20e-05 | 1389.7ms/step | 14.4 samples/s +Step 39720/44000 | Loss: 2.8556 | LR: 1.20e-05 | 1391.8ms/step | 14.4 samples/s +Step 39730/44000 | Loss: 2.8915 | LR: 1.19e-05 | 1390.3ms/step | 14.4 samples/s +Step 39740/44000 | Loss: 2.8625 | LR: 1.19e-05 | 1389.6ms/step | 14.4 samples/s +Step 39750/44000 | Loss: 2.9093 | LR: 1.18e-05 | 1385.8ms/step | 14.4 samples/s +Step 39760/44000 | Loss: 2.8546 | LR: 1.18e-05 | 1390.1ms/step | 14.4 samples/s +Step 39770/44000 | Loss: 2.9094 | LR: 1.17e-05 | 1390.1ms/step | 14.4 samples/s +Step 39780/44000 | Loss: 2.8895 | LR: 1.17e-05 | 1392.0ms/step | 14.4 samples/s +Step 39790/44000 | Loss: 2.8696 | LR: 1.16e-05 | 1390.5ms/step | 14.4 samples/s +Step 39800/44000 | Loss: 2.8621 | LR: 1.15e-05 | 1390.1ms/step | 14.4 samples/s +Step 39810/44000 | Loss: 2.8592 | LR: 1.15e-05 | 1391.6ms/step | 14.4 samples/s +Step 39820/44000 | Loss: 2.8746 | LR: 1.14e-05 | 1389.8ms/step | 14.4 samples/s +Step 39830/44000 | Loss: 2.8728 | LR: 1.14e-05 | 1393.9ms/step | 14.3 samples/s +Step 39840/44000 | Loss: 2.8724 | LR: 1.13e-05 | 1391.7ms/step | 14.4 samples/s +Step 39850/44000 | Loss: 2.8997 | LR: 1.13e-05 | 1390.8ms/step | 14.4 samples/s +Step 39860/44000 | Loss: 2.8718 | LR: 1.12e-05 | 1392.6ms/step | 14.4 samples/s +Step 39870/44000 | Loss: 2.8755 | LR: 1.12e-05 | 1393.8ms/step | 14.3 samples/s +Step 39880/44000 | Loss: 2.8823 | LR: 1.11e-05 | 1393.5ms/step | 14.4 samples/s +Step 39890/44000 | Loss: 2.8768 | LR: 1.11e-05 | 1391.9ms/step | 14.4 samples/s +Step 39900/44000 | Loss: 2.8586 | LR: 1.10e-05 | 1388.0ms/step | 14.4 samples/s +Step 39910/44000 | Loss: 2.8533 | LR: 1.10e-05 | 1389.9ms/step | 14.4 samples/s +Step 39920/44000 | Loss: 2.8738 | LR: 1.09e-05 | 1389.9ms/step | 14.4 samples/s +Step 39930/44000 | Loss: 2.8912 | LR: 1.08e-05 | 1388.6ms/step | 14.4 samples/s +Step 39940/44000 | Loss: 2.8589 | LR: 1.08e-05 | 1391.1ms/step | 14.4 samples/s +Step 39950/44000 | Loss: 2.8503 | LR: 1.07e-05 | 1388.9ms/step | 14.4 samples/s +Step 39960/44000 | Loss: 2.8661 | LR: 1.07e-05 | 1388.0ms/step | 14.4 samples/s +Step 39970/44000 | Loss: 2.8953 | LR: 1.06e-05 | 1389.8ms/step | 14.4 samples/s +Step 39980/44000 | Loss: 2.8817 | LR: 1.06e-05 | 1389.4ms/step | 14.4 samples/s +Step 39990/44000 | Loss: 2.8851 | LR: 1.05e-05 | 1388.7ms/step | 14.4 samples/s +Step 40000/44000 | Loss: 2.8925 | LR: 1.05e-05 | 1390.0ms/step | 14.4 samples/s +Validation at step 40000: Loss = 2.8932, Perplexity = 18.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-40000 +Step 40010/44000 | Loss: 2.8844 | LR: 1.04e-05 | 3907.1ms/step | 5.1 samples/s +Step 40020/44000 | Loss: 2.8873 | LR: 1.04e-05 | 1383.3ms/step | 14.5 samples/s +Step 40030/44000 | Loss: 2.8781 | LR: 1.03e-05 | 1383.9ms/step | 14.5 samples/s +Step 40040/44000 | Loss: 2.8222 | LR: 1.03e-05 | 1383.6ms/step | 14.5 samples/s +Step 40050/44000 | Loss: 2.9019 | LR: 1.02e-05 | 1386.7ms/step | 14.4 samples/s +Step 40060/44000 | Loss: 2.8504 | LR: 1.02e-05 | 1384.8ms/step | 14.4 samples/s +Step 40070/44000 | Loss: 2.8552 | LR: 1.01e-05 | 1388.9ms/step | 14.4 samples/s +Step 40080/44000 | Loss: 2.8806 | LR: 1.01e-05 | 1390.9ms/step | 14.4 samples/s +Step 40090/44000 | Loss: 2.8492 | LR: 1.00e-05 | 1391.1ms/step | 14.4 samples/s +Step 40100/44000 | Loss: 2.8780 | LR: 9.96e-06 | 1393.3ms/step | 14.4 samples/s +Step 40110/44000 | Loss: 2.8836 | LR: 9.91e-06 | 1392.7ms/step | 14.4 samples/s +Step 40120/44000 | Loss: 2.8806 | LR: 9.86e-06 | 1394.0ms/step | 14.3 samples/s +Step 40130/44000 | Loss: 2.9010 | LR: 9.81e-06 | 1387.5ms/step | 14.4 samples/s +Step 40140/44000 | Loss: 2.8649 | LR: 9.76e-06 | 1385.9ms/step | 14.4 samples/s +Step 40150/44000 | Loss: 2.8951 | LR: 9.71e-06 | 1384.2ms/step | 14.4 samples/s +Step 40160/44000 | Loss: 2.8933 | LR: 9.66e-06 | 1385.5ms/step | 14.4 samples/s +Step 40170/44000 | Loss: 2.8978 | LR: 9.61e-06 | 1387.5ms/step | 14.4 samples/s +Step 40180/44000 | Loss: 2.8556 | LR: 9.56e-06 | 1389.8ms/step | 14.4 samples/s +Step 40190/44000 | Loss: 2.8769 | LR: 9.51e-06 | 1390.6ms/step | 14.4 samples/s +Step 40200/44000 | Loss: 2.8725 | LR: 9.46e-06 | 1390.3ms/step | 14.4 samples/s +Step 40210/44000 | Loss: 2.8702 | LR: 9.41e-06 | 1390.6ms/step | 14.4 samples/s +Step 40220/44000 | Loss: 2.8599 | LR: 9.36e-06 | 1390.0ms/step | 14.4 samples/s +Step 40230/44000 | Loss: 2.8580 | LR: 9.32e-06 | 1391.6ms/step | 14.4 samples/s +Step 40240/44000 | Loss: 2.8529 | LR: 9.27e-06 | 1392.2ms/step | 14.4 samples/s +Step 40250/44000 | Loss: 2.8539 | LR: 9.22e-06 | 1390.3ms/step | 14.4 samples/s +Step 40260/44000 | Loss: 2.8881 | LR: 9.17e-06 | 1392.7ms/step | 14.4 samples/s +Step 40270/44000 | Loss: 2.8714 | LR: 9.12e-06 | 1392.5ms/step | 14.4 samples/s +Step 40280/44000 | Loss: 2.8898 | LR: 9.07e-06 | 1393.9ms/step | 14.3 samples/s +Step 40290/44000 | Loss: 2.8865 | LR: 9.02e-06 | 1390.9ms/step | 14.4 samples/s +Step 40300/44000 | Loss: 2.8628 | LR: 8.97e-06 | 1393.3ms/step | 14.4 samples/s +Step 40310/44000 | Loss: 2.8938 | LR: 8.93e-06 | 1393.9ms/step | 14.3 samples/s +Step 40320/44000 | Loss: 2.8454 | LR: 8.88e-06 | 1390.7ms/step | 14.4 samples/s +Step 40330/44000 | Loss: 2.8895 | LR: 8.83e-06 | 1392.5ms/step | 14.4 samples/s +Step 40340/44000 | Loss: 2.8826 | LR: 8.78e-06 | 1390.8ms/step | 14.4 samples/s +Step 40350/44000 | Loss: 2.8875 | LR: 8.74e-06 | 1390.5ms/step | 14.4 samples/s +Step 40360/44000 | Loss: 2.8998 | LR: 8.69e-06 | 1390.2ms/step | 14.4 samples/s +Step 40370/44000 | Loss: 2.8901 | LR: 8.64e-06 | 1390.1ms/step | 14.4 samples/s +Step 40380/44000 | Loss: 2.8822 | LR: 8.59e-06 | 1391.5ms/step | 14.4 samples/s +Step 40390/44000 | Loss: 2.8661 | LR: 8.55e-06 | 1391.7ms/step | 14.4 samples/s +Step 40400/44000 | Loss: 2.8632 | LR: 8.50e-06 | 1393.0ms/step | 14.4 samples/s +Step 40410/44000 | Loss: 2.8757 | LR: 8.45e-06 | 1391.8ms/step | 14.4 samples/s +Step 40420/44000 | Loss: 2.8851 | LR: 8.41e-06 | 1395.7ms/step | 14.3 samples/s +Step 40430/44000 | Loss: 2.8831 | LR: 8.36e-06 | 1392.2ms/step | 14.4 samples/s +Step 40440/44000 | Loss: 2.8908 | LR: 8.31e-06 | 1391.5ms/step | 14.4 samples/s +Step 40450/44000 | Loss: 2.8610 | LR: 8.27e-06 | 1390.6ms/step | 14.4 samples/s +Step 40460/44000 | Loss: 2.8849 | LR: 8.22e-06 | 1391.6ms/step | 14.4 samples/s +Step 40470/44000 | Loss: 2.8606 | LR: 8.17e-06 | 1389.9ms/step | 14.4 samples/s +Step 40480/44000 | Loss: 2.8727 | LR: 8.13e-06 | 1392.0ms/step | 14.4 samples/s +Step 40490/44000 | Loss: 2.8681 | LR: 8.08e-06 | 1390.6ms/step | 14.4 samples/s +Step 40500/44000 | Loss: 2.8676 | LR: 8.04e-06 | 1390.6ms/step | 14.4 samples/s +Step 40510/44000 | Loss: 2.8815 | LR: 7.99e-06 | 1390.1ms/step | 14.4 samples/s +Step 40520/44000 | Loss: 2.8755 | LR: 7.94e-06 | 1391.0ms/step | 14.4 samples/s +Step 40530/44000 | Loss: 2.9016 | LR: 7.90e-06 | 1392.8ms/step | 14.4 samples/s +Step 40540/44000 | Loss: 2.8892 | LR: 7.85e-06 | 1391.7ms/step | 14.4 samples/s +Step 40550/44000 | Loss: 2.8689 | LR: 7.81e-06 | 1392.7ms/step | 14.4 samples/s +Step 40560/44000 | Loss: 2.9003 | LR: 7.76e-06 | 1391.6ms/step | 14.4 samples/s +Step 40570/44000 | Loss: 2.8794 | LR: 7.72e-06 | 1393.1ms/step | 14.4 samples/s +Step 40580/44000 | Loss: 2.9050 | LR: 7.67e-06 | 1391.8ms/step | 14.4 samples/s +Step 40590/44000 | Loss: 2.8815 | LR: 7.63e-06 | 1385.3ms/step | 14.4 samples/s +Step 40600/44000 | Loss: 2.8516 | LR: 7.59e-06 | 1391.2ms/step | 14.4 samples/s +Step 40610/44000 | Loss: 2.9093 | LR: 7.54e-06 | 1390.9ms/step | 14.4 samples/s +Step 40620/44000 | Loss: 2.8574 | LR: 7.50e-06 | 1392.7ms/step | 14.4 samples/s +Step 40630/44000 | Loss: 2.8951 | LR: 7.45e-06 | 1394.4ms/step | 14.3 samples/s +Step 40640/44000 | Loss: 2.8896 | LR: 7.41e-06 | 1393.1ms/step | 14.4 samples/s +Step 40650/44000 | Loss: 2.8713 | LR: 7.37e-06 | 1394.9ms/step | 14.3 samples/s +Step 40660/44000 | Loss: 2.8839 | LR: 7.32e-06 | 1394.0ms/step | 14.3 samples/s +Step 40670/44000 | Loss: 2.8707 | LR: 7.28e-06 | 1393.7ms/step | 14.4 samples/s +Step 40680/44000 | Loss: 2.8698 | LR: 7.23e-06 | 1391.9ms/step | 14.4 samples/s +Step 40690/44000 | Loss: 2.8515 | LR: 7.19e-06 | 1386.7ms/step | 14.4 samples/s +Step 40700/44000 | Loss: 2.8526 | LR: 7.15e-06 | 1389.1ms/step | 14.4 samples/s +Step 40710/44000 | Loss: 2.8913 | LR: 7.10e-06 | 1389.4ms/step | 14.4 samples/s +Step 40720/44000 | Loss: 2.8828 | LR: 7.06e-06 | 1389.7ms/step | 14.4 samples/s +Step 40730/44000 | Loss: 2.8367 | LR: 7.02e-06 | 1391.2ms/step | 14.4 samples/s +Step 40740/44000 | Loss: 2.8955 | LR: 6.98e-06 | 1391.6ms/step | 14.4 samples/s +Step 40750/44000 | Loss: 2.8749 | LR: 6.93e-06 | 1394.5ms/step | 14.3 samples/s +Step 40760/44000 | Loss: 2.8540 | LR: 6.89e-06 | 1391.2ms/step | 14.4 samples/s +Step 40770/44000 | Loss: 2.8693 | LR: 6.85e-06 | 1383.6ms/step | 14.5 samples/s +Step 40780/44000 | Loss: 2.8643 | LR: 6.81e-06 | 1386.4ms/step | 14.4 samples/s +Step 40790/44000 | Loss: 2.8642 | LR: 6.77e-06 | 1386.4ms/step | 14.4 samples/s +Step 40800/44000 | Loss: 2.8988 | LR: 6.72e-06 | 1389.4ms/step | 14.4 samples/s +Step 40810/44000 | Loss: 2.8759 | LR: 6.68e-06 | 1391.0ms/step | 14.4 samples/s +Step 40820/44000 | Loss: 2.9023 | LR: 6.64e-06 | 1391.9ms/step | 14.4 samples/s +Step 40830/44000 | Loss: 2.9053 | LR: 6.60e-06 | 1388.5ms/step | 14.4 samples/s +Step 40840/44000 | Loss: 2.8831 | LR: 6.56e-06 | 1387.2ms/step | 14.4 samples/s +Step 40850/44000 | Loss: 2.8747 | LR: 6.52e-06 | 1385.8ms/step | 14.4 samples/s +Step 40860/44000 | Loss: 2.8709 | LR: 6.47e-06 | 1392.0ms/step | 14.4 samples/s +Step 40870/44000 | Loss: 2.8637 | LR: 6.43e-06 | 1389.7ms/step | 14.4 samples/s +Step 40880/44000 | Loss: 2.8730 | LR: 6.39e-06 | 1389.1ms/step | 14.4 samples/s +Step 40890/44000 | Loss: 2.9075 | LR: 6.35e-06 | 1390.5ms/step | 14.4 samples/s +Step 40900/44000 | Loss: 2.8544 | LR: 6.31e-06 | 1390.8ms/step | 14.4 samples/s +Step 40910/44000 | Loss: 2.8381 | LR: 6.27e-06 | 1390.7ms/step | 14.4 samples/s +Step 40920/44000 | Loss: 2.8986 | LR: 6.23e-06 | 1389.3ms/step | 14.4 samples/s +Step 40930/44000 | Loss: 2.8585 | LR: 6.19e-06 | 1394.6ms/step | 14.3 samples/s +Step 40940/44000 | Loss: 2.8782 | LR: 6.15e-06 | 1390.3ms/step | 14.4 samples/s +Step 40950/44000 | Loss: 2.8861 | LR: 6.11e-06 | 1389.9ms/step | 14.4 samples/s +Step 40960/44000 | Loss: 2.8602 | LR: 6.07e-06 | 1390.0ms/step | 14.4 samples/s +Step 40970/44000 | Loss: 2.8908 | LR: 6.03e-06 | 1393.7ms/step | 14.4 samples/s +Step 40980/44000 | Loss: 2.8388 | LR: 5.99e-06 | 1390.2ms/step | 14.4 samples/s +Step 40990/44000 | Loss: 2.8564 | LR: 5.95e-06 | 1392.0ms/step | 14.4 samples/s +Step 41000/44000 | Loss: 2.9035 | LR: 5.91e-06 | 1389.9ms/step | 14.4 samples/s +Validation at step 41000: Loss = 2.8932, Perplexity = 18.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-41000 +Step 41010/44000 | Loss: 2.8622 | LR: 5.87e-06 | 3917.6ms/step | 5.1 samples/s +Step 41020/44000 | Loss: 2.8813 | LR: 5.83e-06 | 1390.6ms/step | 14.4 samples/s +Step 41030/44000 | Loss: 2.8954 | LR: 5.80e-06 | 1392.1ms/step | 14.4 samples/s +Step 41040/44000 | Loss: 2.8731 | LR: 5.76e-06 | 1389.9ms/step | 14.4 samples/s +Step 41050/44000 | Loss: 2.9039 | LR: 5.72e-06 | 1394.0ms/step | 14.3 samples/s +Step 41060/44000 | Loss: 2.8496 | LR: 5.68e-06 | 1393.5ms/step | 14.4 samples/s +Step 41070/44000 | Loss: 2.8844 | LR: 5.64e-06 | 1390.0ms/step | 14.4 samples/s +Step 41080/44000 | Loss: 2.8594 | LR: 5.60e-06 | 1390.2ms/step | 14.4 samples/s +Step 41090/44000 | Loss: 2.8742 | LR: 5.56e-06 | 1393.8ms/step | 14.3 samples/s +Step 41100/44000 | Loss: 2.8853 | LR: 5.53e-06 | 1392.1ms/step | 14.4 samples/s +Step 41110/44000 | Loss: 2.8846 | LR: 5.49e-06 | 1390.7ms/step | 14.4 samples/s +Step 41120/44000 | Loss: 2.8738 | LR: 5.45e-06 | 1394.9ms/step | 14.3 samples/s +Step 41130/44000 | Loss: 2.9037 | LR: 5.41e-06 | 1392.7ms/step | 14.4 samples/s +Step 41140/44000 | Loss: 2.8909 | LR: 5.38e-06 | 1393.7ms/step | 14.3 samples/s +Step 41150/44000 | Loss: 2.8785 | LR: 5.34e-06 | 1386.9ms/step | 14.4 samples/s +Step 41160/44000 | Loss: 2.9106 | LR: 5.30e-06 | 1385.4ms/step | 14.4 samples/s +Step 41170/44000 | Loss: 2.8591 | LR: 5.26e-06 | 1386.3ms/step | 14.4 samples/s +Step 41180/44000 | Loss: 2.8833 | LR: 5.23e-06 | 1387.2ms/step | 14.4 samples/s +Step 41190/44000 | Loss: 2.8992 | LR: 5.19e-06 | 1388.7ms/step | 14.4 samples/s +Step 41200/44000 | Loss: 2.8267 | LR: 5.15e-06 | 1387.4ms/step | 14.4 samples/s +Step 41210/44000 | Loss: 2.8574 | LR: 5.12e-06 | 1387.5ms/step | 14.4 samples/s +Step 41220/44000 | Loss: 2.8276 | LR: 5.08e-06 | 1387.7ms/step | 14.4 samples/s +Step 41230/44000 | Loss: 2.8602 | LR: 5.04e-06 | 1389.2ms/step | 14.4 samples/s +Step 41240/44000 | Loss: 2.8704 | LR: 5.01e-06 | 1394.0ms/step | 14.3 samples/s +Step 41250/44000 | Loss: 2.8730 | LR: 4.97e-06 | 1390.2ms/step | 14.4 samples/s +Step 41260/44000 | Loss: 2.8634 | LR: 4.94e-06 | 1393.6ms/step | 14.4 samples/s +Step 41270/44000 | Loss: 2.8769 | LR: 4.90e-06 | 1395.9ms/step | 14.3 samples/s +Step 41280/44000 | Loss: 2.8915 | LR: 4.86e-06 | 1389.2ms/step | 14.4 samples/s +Step 41290/44000 | Loss: 2.8834 | LR: 4.83e-06 | 1385.4ms/step | 14.4 samples/s +Step 41300/44000 | Loss: 2.8558 | LR: 4.79e-06 | 1383.0ms/step | 14.5 samples/s +Step 41310/44000 | Loss: 2.8556 | LR: 4.76e-06 | 1383.9ms/step | 14.5 samples/s +Step 41320/44000 | Loss: 2.8908 | LR: 4.72e-06 | 1386.7ms/step | 14.4 samples/s +Step 41330/44000 | Loss: 2.8667 | LR: 4.69e-06 | 1389.9ms/step | 14.4 samples/s +Step 41340/44000 | Loss: 2.8603 | LR: 4.65e-06 | 1390.3ms/step | 14.4 samples/s +Step 41350/44000 | Loss: 2.8766 | LR: 4.62e-06 | 1385.9ms/step | 14.4 samples/s +Step 41360/44000 | Loss: 2.8372 | LR: 4.58e-06 | 1387.9ms/step | 14.4 samples/s +Step 41370/44000 | Loss: 2.8705 | LR: 4.55e-06 | 1390.3ms/step | 14.4 samples/s +Step 41380/44000 | Loss: 2.8552 | LR: 4.51e-06 | 1389.5ms/step | 14.4 samples/s +Step 41390/44000 | Loss: 2.8883 | LR: 4.48e-06 | 1390.7ms/step | 14.4 samples/s +Step 41400/44000 | Loss: 2.8757 | LR: 4.45e-06 | 1391.0ms/step | 14.4 samples/s +Step 41410/44000 | Loss: 2.8729 | LR: 4.41e-06 | 1392.7ms/step | 14.4 samples/s +Step 41420/44000 | Loss: 2.8896 | LR: 4.38e-06 | 1390.7ms/step | 14.4 samples/s +Step 41430/44000 | Loss: 2.8997 | LR: 4.34e-06 | 1395.0ms/step | 14.3 samples/s +Step 41440/44000 | Loss: 2.8649 | LR: 4.31e-06 | 1390.2ms/step | 14.4 samples/s +Step 41450/44000 | Loss: 2.8786 | LR: 4.28e-06 | 1391.0ms/step | 14.4 samples/s +Step 41460/44000 | Loss: 2.8738 | LR: 4.24e-06 | 1393.2ms/step | 14.4 samples/s +Step 41470/44000 | Loss: 2.8786 | LR: 4.21e-06 | 1393.9ms/step | 14.3 samples/s +Step 41480/44000 | Loss: 2.8707 | LR: 4.18e-06 | 1398.2ms/step | 14.3 samples/s +Step 41490/44000 | Loss: 2.8933 | LR: 4.14e-06 | 1395.8ms/step | 14.3 samples/s +Step 41500/44000 | Loss: 2.8739 | LR: 4.11e-06 | 1394.1ms/step | 14.3 samples/s +Step 41510/44000 | Loss: 2.8782 | LR: 4.08e-06 | 1391.9ms/step | 14.4 samples/s +Step 41520/44000 | Loss: 2.8912 | LR: 4.05e-06 | 1392.3ms/step | 14.4 samples/s +Step 41530/44000 | Loss: 2.8707 | LR: 4.01e-06 | 1392.0ms/step | 14.4 samples/s +Step 41540/44000 | Loss: 2.8728 | LR: 3.98e-06 | 1393.7ms/step | 14.4 samples/s +Step 41550/44000 | Loss: 2.9004 | LR: 3.95e-06 | 1393.0ms/step | 14.4 samples/s +Step 41560/44000 | Loss: 2.8553 | LR: 3.92e-06 | 1394.4ms/step | 14.3 samples/s +Step 41570/44000 | Loss: 2.8651 | LR: 3.88e-06 | 1394.4ms/step | 14.3 samples/s +Step 41580/44000 | Loss: 2.8620 | LR: 3.85e-06 | 1394.2ms/step | 14.3 samples/s +Step 41590/44000 | Loss: 2.8442 | LR: 3.82e-06 | 1396.0ms/step | 14.3 samples/s +Step 41600/44000 | Loss: 2.8576 | LR: 3.79e-06 | 1392.5ms/step | 14.4 samples/s +Step 41610/44000 | Loss: 2.8672 | LR: 3.76e-06 | 1389.2ms/step | 14.4 samples/s +Step 41620/44000 | Loss: 2.8788 | LR: 3.73e-06 | 1392.2ms/step | 14.4 samples/s +Step 41630/44000 | Loss: 2.8308 | LR: 3.70e-06 | 1395.5ms/step | 14.3 samples/s +Step 41640/44000 | Loss: 2.8832 | LR: 3.66e-06 | 1394.4ms/step | 14.3 samples/s +Step 41650/44000 | Loss: 2.9099 | LR: 3.63e-06 | 1392.4ms/step | 14.4 samples/s +Step 41660/44000 | Loss: 2.8851 | LR: 3.60e-06 | 1395.8ms/step | 14.3 samples/s +Step 41670/44000 | Loss: 2.8899 | LR: 3.57e-06 | 1394.2ms/step | 14.3 samples/s +Step 41680/44000 | Loss: 2.8720 | LR: 3.54e-06 | 1391.3ms/step | 14.4 samples/s +Step 41690/44000 | Loss: 2.8552 | LR: 3.51e-06 | 1394.7ms/step | 14.3 samples/s +Step 41700/44000 | Loss: 2.8746 | LR: 3.48e-06 | 1398.0ms/step | 14.3 samples/s +Step 41710/44000 | Loss: 2.8527 | LR: 3.45e-06 | 1396.3ms/step | 14.3 samples/s +Step 41720/44000 | Loss: 2.9077 | LR: 3.42e-06 | 1393.9ms/step | 14.3 samples/s +Step 41730/44000 | Loss: 2.8822 | LR: 3.39e-06 | 1392.0ms/step | 14.4 samples/s +Step 41740/44000 | Loss: 2.8577 | LR: 3.36e-06 | 1394.0ms/step | 14.3 samples/s +Step 41750/44000 | Loss: 2.8634 | LR: 3.33e-06 | 1390.2ms/step | 14.4 samples/s +Step 41760/44000 | Loss: 2.8572 | LR: 3.30e-06 | 1391.9ms/step | 14.4 samples/s +Step 41770/44000 | Loss: 2.8718 | LR: 3.27e-06 | 1389.7ms/step | 14.4 samples/s +Step 41780/44000 | Loss: 2.8746 | LR: 3.24e-06 | 1392.1ms/step | 14.4 samples/s +Step 41790/44000 | Loss: 2.8719 | LR: 3.21e-06 | 1387.9ms/step | 14.4 samples/s +Step 41800/44000 | Loss: 2.8734 | LR: 3.19e-06 | 1384.5ms/step | 14.4 samples/s +Step 41810/44000 | Loss: 2.8676 | LR: 3.16e-06 | 1387.2ms/step | 14.4 samples/s +Step 41820/44000 | Loss: 2.8695 | LR: 3.13e-06 | 1386.7ms/step | 14.4 samples/s +Step 41830/44000 | Loss: 2.8578 | LR: 3.10e-06 | 1385.0ms/step | 14.4 samples/s +Step 41840/44000 | Loss: 2.8810 | LR: 3.07e-06 | 1380.9ms/step | 14.5 samples/s +Step 41850/44000 | Loss: 2.8821 | LR: 3.04e-06 | 1381.7ms/step | 14.5 samples/s +Step 41860/44000 | Loss: 2.8620 | LR: 3.01e-06 | 1384.0ms/step | 14.5 samples/s +Step 41870/44000 | Loss: 2.8584 | LR: 2.99e-06 | 1385.8ms/step | 14.4 samples/s +Step 41880/44000 | Loss: 2.8582 | LR: 2.96e-06 | 1385.9ms/step | 14.4 samples/s +Step 41890/44000 | Loss: 2.8535 | LR: 2.93e-06 | 1388.4ms/step | 14.4 samples/s +Step 41900/44000 | Loss: 2.8705 | LR: 2.90e-06 | 1388.1ms/step | 14.4 samples/s +Step 41910/44000 | Loss: 2.8750 | LR: 2.88e-06 | 1388.0ms/step | 14.4 samples/s +Step 41920/44000 | Loss: 2.8837 | LR: 2.85e-06 | 1384.5ms/step | 14.4 samples/s +Step 41930/44000 | Loss: 2.9012 | LR: 2.82e-06 | 1387.8ms/step | 14.4 samples/s +Step 41940/44000 | Loss: 2.8890 | LR: 2.79e-06 | 1386.2ms/step | 14.4 samples/s +Step 41950/44000 | Loss: 2.8545 | LR: 2.77e-06 | 1386.9ms/step | 14.4 samples/s +Step 41960/44000 | Loss: 2.8651 | LR: 2.74e-06 | 1386.5ms/step | 14.4 samples/s +Step 41970/44000 | Loss: 2.8553 | LR: 2.71e-06 | 1384.1ms/step | 14.4 samples/s +Step 41980/44000 | Loss: 2.8760 | LR: 2.69e-06 | 1384.7ms/step | 14.4 samples/s +Step 41990/44000 | Loss: 2.8580 | LR: 2.66e-06 | 1383.6ms/step | 14.5 samples/s +Step 42000/44000 | Loss: 2.8513 | LR: 2.63e-06 | 1383.8ms/step | 14.5 samples/s +Validation at step 42000: Loss = 2.8932, Perplexity = 18.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-42000 +Step 42010/44000 | Loss: 2.8927 | LR: 2.61e-06 | 3882.0ms/step | 5.2 samples/s +Step 42020/44000 | Loss: 2.8725 | LR: 2.58e-06 | 1382.4ms/step | 14.5 samples/s +Step 42030/44000 | Loss: 2.8596 | LR: 2.56e-06 | 1385.4ms/step | 14.4 samples/s +Step 42040/44000 | Loss: 2.8512 | LR: 2.53e-06 | 1389.2ms/step | 14.4 samples/s +Step 42050/44000 | Loss: 2.8824 | LR: 2.50e-06 | 1384.9ms/step | 14.4 samples/s +Step 42060/44000 | Loss: 2.8893 | LR: 2.48e-06 | 1380.4ms/step | 14.5 samples/s +Step 42070/44000 | Loss: 2.9021 | LR: 2.45e-06 | 1380.9ms/step | 14.5 samples/s +Step 42080/44000 | Loss: 2.8750 | LR: 2.43e-06 | 1383.1ms/step | 14.5 samples/s +Step 42090/44000 | Loss: 2.8624 | LR: 2.40e-06 | 1383.9ms/step | 14.5 samples/s +Step 42100/44000 | Loss: 2.8681 | LR: 2.38e-06 | 1389.4ms/step | 14.4 samples/s +Step 42110/44000 | Loss: 2.9014 | LR: 2.35e-06 | 1391.3ms/step | 14.4 samples/s +Step 42120/44000 | Loss: 2.8877 | LR: 2.33e-06 | 1388.8ms/step | 14.4 samples/s +Step 42130/44000 | Loss: 2.8834 | LR: 2.30e-06 | 1384.5ms/step | 14.4 samples/s +Step 42140/44000 | Loss: 2.8532 | LR: 2.28e-06 | 1388.8ms/step | 14.4 samples/s +Step 42150/44000 | Loss: 2.8798 | LR: 2.25e-06 | 1387.4ms/step | 14.4 samples/s +Step 42160/44000 | Loss: 2.8928 | LR: 2.23e-06 | 1386.2ms/step | 14.4 samples/s +Step 42170/44000 | Loss: 2.8673 | LR: 2.21e-06 | 1386.7ms/step | 14.4 samples/s +Step 42180/44000 | Loss: 2.8920 | LR: 2.18e-06 | 1390.2ms/step | 14.4 samples/s +Step 42190/44000 | Loss: 2.9013 | LR: 2.16e-06 | 1386.8ms/step | 14.4 samples/s +Step 42200/44000 | Loss: 2.8541 | LR: 2.13e-06 | 1389.8ms/step | 14.4 samples/s +Step 42210/44000 | Loss: 2.9160 | LR: 2.11e-06 | 1389.9ms/step | 14.4 samples/s +Step 42220/44000 | Loss: 2.8863 | LR: 2.09e-06 | 1390.8ms/step | 14.4 samples/s +Step 42230/44000 | Loss: 2.8585 | LR: 2.06e-06 | 1387.5ms/step | 14.4 samples/s +Step 42240/44000 | Loss: 2.8588 | LR: 2.04e-06 | 1384.6ms/step | 14.4 samples/s +Step 42250/44000 | Loss: 2.8202 | LR: 2.02e-06 | 1387.0ms/step | 14.4 samples/s +Step 42260/44000 | Loss: 2.8412 | LR: 1.99e-06 | 1384.5ms/step | 14.4 samples/s +Step 42270/44000 | Loss: 2.8860 | LR: 1.97e-06 | 1389.3ms/step | 14.4 samples/s +Step 42280/44000 | Loss: 2.8573 | LR: 1.95e-06 | 1387.2ms/step | 14.4 samples/s +Step 42290/44000 | Loss: 2.8346 | LR: 1.93e-06 | 1388.1ms/step | 14.4 samples/s +Step 42300/44000 | Loss: 2.8943 | LR: 1.90e-06 | 1386.6ms/step | 14.4 samples/s +Step 42310/44000 | Loss: 2.8987 | LR: 1.88e-06 | 1389.1ms/step | 14.4 samples/s +Step 42320/44000 | Loss: 2.8600 | LR: 1.86e-06 | 1388.5ms/step | 14.4 samples/s +Step 42330/44000 | Loss: 2.8971 | LR: 1.84e-06 | 1388.1ms/step | 14.4 samples/s +Step 42340/44000 | Loss: 2.8737 | LR: 1.82e-06 | 1388.7ms/step | 14.4 samples/s +Step 42350/44000 | Loss: 2.8633 | LR: 1.79e-06 | 1387.2ms/step | 14.4 samples/s +Step 42360/44000 | Loss: 2.8974 | LR: 1.77e-06 | 1389.6ms/step | 14.4 samples/s +Step 42370/44000 | Loss: 2.8771 | LR: 1.75e-06 | 1387.8ms/step | 14.4 samples/s +Step 42380/44000 | Loss: 2.8636 | LR: 1.73e-06 | 1383.0ms/step | 14.5 samples/s +Step 42390/44000 | Loss: 2.8558 | LR: 1.71e-06 | 1385.7ms/step | 14.4 samples/s +Step 42400/44000 | Loss: 2.8882 | LR: 1.69e-06 | 1386.3ms/step | 14.4 samples/s +Step 42410/44000 | Loss: 2.8562 | LR: 1.67e-06 | 1390.2ms/step | 14.4 samples/s +Step 42420/44000 | Loss: 2.8590 | LR: 1.64e-06 | 1387.2ms/step | 14.4 samples/s +Step 42430/44000 | Loss: 2.8724 | LR: 1.62e-06 | 1390.8ms/step | 14.4 samples/s +Step 42440/44000 | Loss: 2.9016 | LR: 1.60e-06 | 1386.0ms/step | 14.4 samples/s +Step 42450/44000 | Loss: 2.8811 | LR: 1.58e-06 | 1386.8ms/step | 14.4 samples/s +Step 42460/44000 | Loss: 2.8782 | LR: 1.56e-06 | 1387.0ms/step | 14.4 samples/s +Step 42470/44000 | Loss: 2.8933 | LR: 1.54e-06 | 1388.9ms/step | 14.4 samples/s +Step 42480/44000 | Loss: 2.8676 | LR: 1.52e-06 | 1388.1ms/step | 14.4 samples/s +Step 42490/44000 | Loss: 2.8768 | LR: 1.50e-06 | 1386.4ms/step | 14.4 samples/s +Step 42500/44000 | Loss: 2.8437 | LR: 1.48e-06 | 1389.9ms/step | 14.4 samples/s +Step 42510/44000 | Loss: 2.8689 | LR: 1.46e-06 | 1387.9ms/step | 14.4 samples/s +Step 42520/44000 | Loss: 2.8593 | LR: 1.44e-06 | 1390.1ms/step | 14.4 samples/s +Step 42530/44000 | Loss: 2.8789 | LR: 1.42e-06 | 1389.4ms/step | 14.4 samples/s +Step 42540/44000 | Loss: 2.8812 | LR: 1.40e-06 | 1385.1ms/step | 14.4 samples/s +Step 42550/44000 | Loss: 2.8786 | LR: 1.39e-06 | 1387.5ms/step | 14.4 samples/s +Step 42560/44000 | Loss: 2.8770 | LR: 1.37e-06 | 1385.8ms/step | 14.4 samples/s +Step 42570/44000 | Loss: 2.8566 | LR: 1.35e-06 | 1386.4ms/step | 14.4 samples/s +Step 42580/44000 | Loss: 2.8683 | LR: 1.33e-06 | 1387.8ms/step | 14.4 samples/s +Step 42590/44000 | Loss: 2.8722 | LR: 1.31e-06 | 1388.2ms/step | 14.4 samples/s +Step 42600/44000 | Loss: 2.8686 | LR: 1.29e-06 | 1387.6ms/step | 14.4 samples/s +Step 42610/44000 | Loss: 2.8871 | LR: 1.27e-06 | 1384.6ms/step | 14.4 samples/s +Step 42620/44000 | Loss: 2.8680 | LR: 1.25e-06 | 1385.7ms/step | 14.4 samples/s +Step 42630/44000 | Loss: 2.8857 | LR: 1.24e-06 | 1383.6ms/step | 14.5 samples/s +Step 42640/44000 | Loss: 2.8685 | LR: 1.22e-06 | 1386.6ms/step | 14.4 samples/s +Step 42650/44000 | Loss: 2.8544 | LR: 1.20e-06 | 1387.7ms/step | 14.4 samples/s +Step 42660/44000 | Loss: 2.8800 | LR: 1.18e-06 | 1388.0ms/step | 14.4 samples/s +Step 42670/44000 | Loss: 2.8842 | LR: 1.17e-06 | 1389.8ms/step | 14.4 samples/s +Step 42680/44000 | Loss: 2.9042 | LR: 1.15e-06 | 1390.4ms/step | 14.4 samples/s +Step 42690/44000 | Loss: 2.9117 | LR: 1.13e-06 | 1389.8ms/step | 14.4 samples/s +Step 42700/44000 | Loss: 2.8790 | LR: 1.11e-06 | 1391.2ms/step | 14.4 samples/s +Step 42710/44000 | Loss: 2.8680 | LR: 1.10e-06 | 1389.2ms/step | 14.4 samples/s +Step 42720/44000 | Loss: 2.8622 | LR: 1.08e-06 | 1388.7ms/step | 14.4 samples/s +Step 42730/44000 | Loss: 2.8868 | LR: 1.06e-06 | 1388.5ms/step | 14.4 samples/s +Step 42740/44000 | Loss: 2.8803 | LR: 1.05e-06 | 1388.4ms/step | 14.4 samples/s +Step 42750/44000 | Loss: 2.8942 | LR: 1.03e-06 | 1390.9ms/step | 14.4 samples/s +Step 42760/44000 | Loss: 2.8804 | LR: 1.01e-06 | 1386.3ms/step | 14.4 samples/s +Step 42770/44000 | Loss: 2.8254 | LR: 9.97e-07 | 1390.2ms/step | 14.4 samples/s +Step 42780/44000 | Loss: 2.9064 | LR: 9.81e-07 | 1385.0ms/step | 14.4 samples/s +Step 42790/44000 | Loss: 2.8707 | LR: 9.65e-07 | 1384.3ms/step | 14.4 samples/s +Step 42800/44000 | Loss: 2.8858 | LR: 9.49e-07 | 1383.4ms/step | 14.5 samples/s +Step 42810/44000 | Loss: 2.8744 | LR: 9.33e-07 | 1386.0ms/step | 14.4 samples/s +Step 42820/44000 | Loss: 2.8698 | LR: 9.18e-07 | 1383.0ms/step | 14.5 samples/s +Step 42830/44000 | Loss: 2.8646 | LR: 9.02e-07 | 1387.8ms/step | 14.4 samples/s +Step 42840/44000 | Loss: 2.8882 | LR: 8.87e-07 | 1387.0ms/step | 14.4 samples/s +Step 42850/44000 | Loss: 2.8694 | LR: 8.72e-07 | 1383.4ms/step | 14.5 samples/s +Step 42860/44000 | Loss: 2.8608 | LR: 8.57e-07 | 1383.1ms/step | 14.5 samples/s +Step 42870/44000 | Loss: 2.9129 | LR: 8.42e-07 | 1383.5ms/step | 14.5 samples/s +Step 42880/44000 | Loss: 2.8380 | LR: 8.27e-07 | 1386.8ms/step | 14.4 samples/s +Step 42890/44000 | Loss: 2.8471 | LR: 8.12e-07 | 1389.2ms/step | 14.4 samples/s +Step 42900/44000 | Loss: 2.8894 | LR: 7.98e-07 | 1390.6ms/step | 14.4 samples/s +Step 42910/44000 | Loss: 2.8903 | LR: 7.83e-07 | 1389.9ms/step | 14.4 samples/s +Step 42920/44000 | Loss: 2.9041 | LR: 7.69e-07 | 1384.3ms/step | 14.4 samples/s +Step 42930/44000 | Loss: 2.8915 | LR: 7.55e-07 | 1385.7ms/step | 14.4 samples/s +Step 42940/44000 | Loss: 2.8768 | LR: 7.41e-07 | 1387.9ms/step | 14.4 samples/s +Step 42950/44000 | Loss: 2.8448 | LR: 7.27e-07 | 1385.5ms/step | 14.4 samples/s +Step 42960/44000 | Loss: 2.8629 | LR: 7.13e-07 | 1388.4ms/step | 14.4 samples/s +Step 42970/44000 | Loss: 2.8638 | LR: 6.99e-07 | 1387.0ms/step | 14.4 samples/s +Step 42980/44000 | Loss: 2.8687 | LR: 6.86e-07 | 1384.9ms/step | 14.4 samples/s +Step 42990/44000 | Loss: 2.8776 | LR: 6.72e-07 | 1382.9ms/step | 14.5 samples/s +Step 43000/44000 | Loss: 2.8482 | LR: 6.59e-07 | 1387.4ms/step | 14.4 samples/s +Validation at step 43000: Loss = 2.8932, Perplexity = 18.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-43000 +Step 43010/44000 | Loss: 2.8762 | LR: 6.46e-07 | 3878.0ms/step | 5.2 samples/s +Step 43020/44000 | Loss: 2.8538 | LR: 6.33e-07 | 1386.2ms/step | 14.4 samples/s +Step 43030/44000 | Loss: 2.8983 | LR: 6.20e-07 | 1387.9ms/step | 14.4 samples/s +Step 43040/44000 | Loss: 2.8662 | LR: 6.08e-07 | 1387.1ms/step | 14.4 samples/s +Step 43050/44000 | Loss: 2.8993 | LR: 5.95e-07 | 1384.8ms/step | 14.4 samples/s +Step 43060/44000 | Loss: 2.8743 | LR: 5.83e-07 | 1385.8ms/step | 14.4 samples/s +Step 43070/44000 | Loss: 2.8635 | LR: 5.70e-07 | 1386.5ms/step | 14.4 samples/s +Step 43080/44000 | Loss: 2.8953 | LR: 5.58e-07 | 1385.6ms/step | 14.4 samples/s +Step 43090/44000 | Loss: 2.8613 | LR: 5.46e-07 | 1383.7ms/step | 14.5 samples/s +Step 43100/44000 | Loss: 2.8966 | LR: 5.34e-07 | 1382.4ms/step | 14.5 samples/s +Step 43110/44000 | Loss: 2.8646 | LR: 5.22e-07 | 1386.1ms/step | 14.4 samples/s +Step 43120/44000 | Loss: 2.8743 | LR: 5.11e-07 | 1383.9ms/step | 14.5 samples/s +Step 43130/44000 | Loss: 2.8799 | LR: 4.99e-07 | 1383.7ms/step | 14.5 samples/s +Step 43140/44000 | Loss: 2.8943 | LR: 4.88e-07 | 1384.1ms/step | 14.4 samples/s +Step 43150/44000 | Loss: 2.8780 | LR: 4.76e-07 | 1384.6ms/step | 14.4 samples/s +Step 43160/44000 | Loss: 2.8713 | LR: 4.65e-07 | 1380.4ms/step | 14.5 samples/s +Step 43170/44000 | Loss: 2.8352 | LR: 4.54e-07 | 1379.3ms/step | 14.5 samples/s +Step 43180/44000 | Loss: 2.8809 | LR: 4.43e-07 | 1379.5ms/step | 14.5 samples/s +Step 43190/44000 | Loss: 2.8520 | LR: 4.33e-07 | 1376.5ms/step | 14.5 samples/s +Step 43200/44000 | Loss: 2.8446 | LR: 4.22e-07 | 1376.0ms/step | 14.5 samples/s +Step 43210/44000 | Loss: 2.8762 | LR: 4.12e-07 | 1374.3ms/step | 14.6 samples/s +Step 43220/44000 | Loss: 2.8931 | LR: 4.01e-07 | 1374.5ms/step | 14.6 samples/s +Step 43230/44000 | Loss: 2.8956 | LR: 3.91e-07 | 1373.3ms/step | 14.6 samples/s +Step 43240/44000 | Loss: 2.8759 | LR: 3.81e-07 | 1372.2ms/step | 14.6 samples/s +Step 43250/44000 | Loss: 2.8894 | LR: 3.71e-07 | 1370.6ms/step | 14.6 samples/s +Step 43260/44000 | Loss: 2.8815 | LR: 3.61e-07 | 1372.1ms/step | 14.6 samples/s +Step 43270/44000 | Loss: 2.8743 | LR: 3.51e-07 | 1370.9ms/step | 14.6 samples/s +Step 43280/44000 | Loss: 2.8719 | LR: 3.42e-07 | 1370.7ms/step | 14.6 samples/s +Step 43290/44000 | Loss: 2.8461 | LR: 3.32e-07 | 1371.1ms/step | 14.6 samples/s +Step 43300/44000 | Loss: 2.8490 | LR: 3.23e-07 | 1372.2ms/step | 14.6 samples/s +Step 43310/44000 | Loss: 2.8785 | LR: 3.14e-07 | 1374.1ms/step | 14.6 samples/s +Step 43320/44000 | Loss: 2.8808 | LR: 3.05e-07 | 1372.4ms/step | 14.6 samples/s +Step 43330/44000 | Loss: 2.8779 | LR: 2.96e-07 | 1371.4ms/step | 14.6 samples/s +Step 43340/44000 | Loss: 2.8851 | LR: 2.87e-07 | 1369.7ms/step | 14.6 samples/s +Step 43350/44000 | Loss: 2.8918 | LR: 2.79e-07 | 1370.7ms/step | 14.6 samples/s +Step 43360/44000 | Loss: 2.8739 | LR: 2.70e-07 | 1370.5ms/step | 14.6 samples/s +Step 43370/44000 | Loss: 2.8704 | LR: 2.62e-07 | 1368.3ms/step | 14.6 samples/s +Step 43380/44000 | Loss: 2.8356 | LR: 2.53e-07 | 1370.3ms/step | 14.6 samples/s +Step 43390/44000 | Loss: 2.8735 | LR: 2.45e-07 | 1368.9ms/step | 14.6 samples/s +Step 43400/44000 | Loss: 2.8840 | LR: 2.37e-07 | 1370.7ms/step | 14.6 samples/s +Step 43410/44000 | Loss: 2.8932 | LR: 2.30e-07 | 1369.4ms/step | 14.6 samples/s +Step 43420/44000 | Loss: 2.8463 | LR: 2.22e-07 | 1369.4ms/step | 14.6 samples/s +Step 43430/44000 | Loss: 2.8533 | LR: 2.14e-07 | 1370.0ms/step | 14.6 samples/s +Step 43440/44000 | Loss: 2.8526 | LR: 2.07e-07 | 1368.2ms/step | 14.6 samples/s +Step 43450/44000 | Loss: 2.8406 | LR: 1.99e-07 | 1369.0ms/step | 14.6 samples/s +Step 43460/44000 | Loss: 2.8534 | LR: 1.92e-07 | 1371.6ms/step | 14.6 samples/s +Step 43470/44000 | Loss: 2.8994 | LR: 1.85e-07 | 1370.4ms/step | 14.6 samples/s +Step 43480/44000 | Loss: 2.9044 | LR: 1.78e-07 | 1370.0ms/step | 14.6 samples/s +Step 43490/44000 | Loss: 2.8706 | LR: 1.72e-07 | 1370.4ms/step | 14.6 samples/s +Step 43500/44000 | Loss: 2.8768 | LR: 1.65e-07 | 1369.8ms/step | 14.6 samples/s +Step 43510/44000 | Loss: 2.8626 | LR: 1.58e-07 | 1372.6ms/step | 14.6 samples/s +Step 43520/44000 | Loss: 2.8787 | LR: 1.52e-07 | 1370.5ms/step | 14.6 samples/s +Step 43530/44000 | Loss: 2.8932 | LR: 1.46e-07 | 1370.1ms/step | 14.6 samples/s +Step 43540/44000 | Loss: 2.8739 | LR: 1.40e-07 | 1371.1ms/step | 14.6 samples/s +Step 43550/44000 | Loss: 2.8532 | LR: 1.34e-07 | 1370.8ms/step | 14.6 samples/s +Step 43560/44000 | Loss: 2.8791 | LR: 1.28e-07 | 1371.9ms/step | 14.6 samples/s +Step 43570/44000 | Loss: 2.9063 | LR: 1.22e-07 | 1370.2ms/step | 14.6 samples/s +Step 43580/44000 | Loss: 2.8655 | LR: 1.16e-07 | 1370.2ms/step | 14.6 samples/s +Step 43590/44000 | Loss: 2.8370 | LR: 1.11e-07 | 1371.8ms/step | 14.6 samples/s +Step 43600/44000 | Loss: 2.8959 | LR: 1.06e-07 | 1371.3ms/step | 14.6 samples/s +Step 43610/44000 | Loss: 2.8245 | LR: 1.00e-07 | 1370.7ms/step | 14.6 samples/s +Step 43620/44000 | Loss: 2.9060 | LR: 9.52e-08 | 1370.7ms/step | 14.6 samples/s +Step 43630/44000 | Loss: 2.8973 | LR: 9.03e-08 | 1369.7ms/step | 14.6 samples/s +Step 43640/44000 | Loss: 2.8711 | LR: 8.55e-08 | 1369.8ms/step | 14.6 samples/s +Step 43650/44000 | Loss: 2.8787 | LR: 8.08e-08 | 1372.1ms/step | 14.6 samples/s +Step 43660/44000 | Loss: 2.9128 | LR: 7.62e-08 | 1377.3ms/step | 14.5 samples/s +Step 43670/44000 | Loss: 2.8480 | LR: 7.18e-08 | 1370.3ms/step | 14.6 samples/s +Step 43680/44000 | Loss: 2.8817 | LR: 6.75e-08 | 1377.9ms/step | 14.5 samples/s +Step 43690/44000 | Loss: 2.8719 | LR: 6.34e-08 | 1373.0ms/step | 14.6 samples/s +Step 43700/44000 | Loss: 2.8675 | LR: 5.94e-08 | 1373.3ms/step | 14.6 samples/s +Step 43710/44000 | Loss: 2.8751 | LR: 5.55e-08 | 1372.2ms/step | 14.6 samples/s +Step 43720/44000 | Loss: 2.8479 | LR: 5.17e-08 | 1371.9ms/step | 14.6 samples/s +Step 43730/44000 | Loss: 2.8505 | LR: 4.81e-08 | 1373.7ms/step | 14.6 samples/s +Step 43740/44000 | Loss: 2.8878 | LR: 4.46e-08 | 1377.0ms/step | 14.5 samples/s +Step 43750/44000 | Loss: 2.8788 | LR: 4.12e-08 | 1368.4ms/step | 14.6 samples/s +Step 43760/44000 | Loss: 2.8777 | LR: 3.80e-08 | 1369.1ms/step | 14.6 samples/s +Step 43770/44000 | Loss: 2.8844 | LR: 3.49e-08 | 1369.5ms/step | 14.6 samples/s +Step 43780/44000 | Loss: 2.8793 | LR: 3.19e-08 | 1371.7ms/step | 14.6 samples/s +Step 43790/44000 | Loss: 2.8675 | LR: 2.91e-08 | 1373.1ms/step | 14.6 samples/s +Step 43800/44000 | Loss: 2.9167 | LR: 2.64e-08 | 1375.2ms/step | 14.5 samples/s +Step 43810/44000 | Loss: 2.8641 | LR: 2.38e-08 | 1373.2ms/step | 14.6 samples/s +Step 43820/44000 | Loss: 2.8749 | LR: 2.14e-08 | 1369.3ms/step | 14.6 samples/s +Step 43830/44000 | Loss: 2.8866 | LR: 1.91e-08 | 1370.4ms/step | 14.6 samples/s +Step 43840/44000 | Loss: 2.8847 | LR: 1.69e-08 | 1371.3ms/step | 14.6 samples/s +Step 43850/44000 | Loss: 2.8743 | LR: 1.48e-08 | 1370.7ms/step | 14.6 samples/s +Step 43860/44000 | Loss: 2.8603 | LR: 1.29e-08 | 1371.6ms/step | 14.6 samples/s +Step 43870/44000 | Loss: 2.8841 | LR: 1.11e-08 | 1376.0ms/step | 14.5 samples/s +Step 43880/44000 | Loss: 2.8829 | LR: 9.50e-09 | 1372.9ms/step | 14.6 samples/s +Step 43890/44000 | Loss: 2.8568 | LR: 7.98e-09 | 1371.5ms/step | 14.6 samples/s +Step 43900/44000 | Loss: 2.8854 | LR: 6.60e-09 | 1372.4ms/step | 14.6 samples/s +Step 43910/44000 | Loss: 2.8784 | LR: 5.34e-09 | 1368.9ms/step | 14.6 samples/s +Step 43920/44000 | Loss: 2.8630 | LR: 4.22e-09 | 1370.1ms/step | 14.6 samples/s +Step 43930/44000 | Loss: 2.8772 | LR: 3.23e-09 | 1371.9ms/step | 14.6 samples/s +Step 43940/44000 | Loss: 2.8581 | LR: 2.37e-09 | 1370.2ms/step | 14.6 samples/s +Step 43950/44000 | Loss: 2.8573 | LR: 1.65e-09 | 1372.4ms/step | 14.6 samples/s +Step 43960/44000 | Loss: 2.8817 | LR: 1.06e-09 | 1371.3ms/step | 14.6 samples/s +Step 43970/44000 | Loss: 2.8890 | LR: 5.94e-10 | 1370.4ms/step | 14.6 samples/s +Step 43980/44000 | Loss: 2.8942 | LR: 2.64e-10 | 1371.2ms/step | 14.6 samples/s +Step 43990/44000 | Loss: 2.8547 | LR: 6.60e-11 | 1371.0ms/step | 14.6 samples/s +Step 44000/44000 | Loss: 2.8999 | LR: 0.00e+00 | 1369.6ms/step | 14.6 samples/s +Validation at step 44000: Loss = 2.8932, Perplexity = 18.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▆▅▄▄▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▆▅▄▄▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▅▆▅▄▅▅▄▅▆▆▅▅▅▅▆▅▆▆▆▅▅▆▄▄▅▃▅▅█▆▇▆▆▆▆▆▅▅▅▁ +wandb: speed/samples_per_sec ▄▄▄▄▄▄▃▄▄▄▄▃▄▄▃▃▅▄▄▄▅▄▄▁▁▃▄▃▂▃▃▃▄▃▂▃▅▅▇█ +wandb: speed/steps_per_sec ██████████▁█████████████████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.89316 +wandb: eval/perplexity 18.05025 +wandb: eval/uses_dense_attention 1 +wandb: eval/uses_sparse_attention 0 +wandb: speed/ms_per_step 1369.60702 +wandb: speed/samples_per_sec 14.60273 +wandb: speed/steps_per_sec 0.73014 +wandb: +4 ... +wandb: +wandb: 🚀 View run l8k365m_dense_ropectl_a1p0_seed42 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/jqstnvej +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260820_194209-jqstnvej/logs +Training complete! diff --git a/epfl_final/train_logs/ropectl_a1p0_h100_42_43_resume/seed43.20260820T194037Z.log b/epfl_final/train_logs/ropectl_a1p0_h100_42_43_resume/seed43.20260820T194037Z.log new file mode 100644 index 0000000000000000000000000000000000000000..085481ef61c4a1f279cd68b6b606f059a81801fa --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_h100_42_43_resume/seed43.20260820T194037Z.log @@ -0,0 +1,2454 @@ +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-23000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-23000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 43, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed43", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-23000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-23000... +Preserving checkpoint LR state for exact resume: current=[0.00023865421385855804], schedule_base=[0.0005] +Resumed from checkpoint at step 23000 (epoch 0, micro_step 205000) +Resuming training from step 23000 +Resuming wandb run ysa9iyov +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run ysa9iyov +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_194210-ysa9iyov +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed43 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ysa9iyov +Starting training... +Validation at step 23000: Loss = 2.9266, Perplexity = 18.66 +wandb: WARNING Tried to log to step 23000 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 23001: dense (dense step interval) +Step 23010/44000 | Loss: 2.9392 | LR: 2.38e-04 | 48434.0ms/step | 0.4 samples/s +wandb: WARNING Tried to log to step 23010 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23020/44000 | Loss: 2.9493 | LR: 2.38e-04 | 1396.6ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 23020 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23030/44000 | Loss: 2.9451 | LR: 2.38e-04 | 1399.8ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 23030 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23040/44000 | Loss: 2.9360 | LR: 2.38e-04 | 1400.1ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 23040 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23050/44000 | Loss: 2.9592 | LR: 2.38e-04 | 1399.7ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 23050 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23060/44000 | Loss: 2.9357 | LR: 2.38e-04 | 1400.3ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 23060 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23070/44000 | Loss: 2.9316 | LR: 2.37e-04 | 1399.8ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 23070 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23080/44000 | Loss: 2.8990 | LR: 2.37e-04 | 1400.6ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 23080 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23090/44000 | Loss: 2.9404 | LR: 2.37e-04 | 1391.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23090 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23100/44000 | Loss: 2.9341 | LR: 2.37e-04 | 1391.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23100 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23110/44000 | Loss: 2.8974 | LR: 2.37e-04 | 1392.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23110 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23120/44000 | Loss: 2.9224 | LR: 2.36e-04 | 1393.9ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 23120 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23130/44000 | Loss: 2.9201 | LR: 2.36e-04 | 1392.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23130 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23140/44000 | Loss: 2.9415 | LR: 2.36e-04 | 1386.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23140 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23150/44000 | Loss: 2.9414 | LR: 2.36e-04 | 1389.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23150 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23160/44000 | Loss: 2.9029 | LR: 2.36e-04 | 1390.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23160 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23170/44000 | Loss: 2.9170 | LR: 2.36e-04 | 1387.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23170 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23180/44000 | Loss: 2.9293 | LR: 2.35e-04 | 1388.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23180 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23190/44000 | Loss: 2.9712 | LR: 2.35e-04 | 1391.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23190 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23200/44000 | Loss: 2.9125 | LR: 2.35e-04 | 1388.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23200 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23210/44000 | Loss: 2.9473 | LR: 2.35e-04 | 1391.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23210 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23220/44000 | Loss: 2.9610 | LR: 2.35e-04 | 1390.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23220 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23230/44000 | Loss: 2.9241 | LR: 2.34e-04 | 1389.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23230 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23240/44000 | Loss: 2.9761 | LR: 2.34e-04 | 1388.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23240 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23250/44000 | Loss: 2.9381 | LR: 2.34e-04 | 1389.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23250 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23260/44000 | Loss: 2.9557 | LR: 2.34e-04 | 1391.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23260 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23270/44000 | Loss: 2.8940 | LR: 2.34e-04 | 1388.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23270 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23280/44000 | Loss: 2.9496 | LR: 2.34e-04 | 1391.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23280 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23290/44000 | Loss: 2.9315 | LR: 2.33e-04 | 1388.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23290 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23300/44000 | Loss: 2.9151 | LR: 2.33e-04 | 1389.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23300 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23310/44000 | Loss: 2.9061 | LR: 2.33e-04 | 1389.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23310 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23320/44000 | Loss: 2.9578 | LR: 2.33e-04 | 1391.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23320 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23330/44000 | Loss: 2.9434 | LR: 2.33e-04 | 1389.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23330 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23340/44000 | Loss: 2.9280 | LR: 2.32e-04 | 1390.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23340 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23350/44000 | Loss: 2.9517 | LR: 2.32e-04 | 1391.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23350 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23360/44000 | Loss: 2.9301 | LR: 2.32e-04 | 1391.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23360 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23370/44000 | Loss: 2.8972 | LR: 2.32e-04 | 1391.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23370 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23380/44000 | Loss: 2.9351 | LR: 2.32e-04 | 1391.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23380 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23390/44000 | Loss: 2.9329 | LR: 2.32e-04 | 1389.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23390 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23400/44000 | Loss: 2.9500 | LR: 2.31e-04 | 1390.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23400 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23410/44000 | Loss: 2.9303 | LR: 2.31e-04 | 1390.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23410 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23420/44000 | Loss: 2.9259 | LR: 2.31e-04 | 1390.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23420 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23430/44000 | Loss: 2.9270 | LR: 2.31e-04 | 1387.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23430 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23440/44000 | Loss: 2.9283 | LR: 2.31e-04 | 1389.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23440 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23450/44000 | Loss: 2.9348 | LR: 2.30e-04 | 1389.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23450 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23460/44000 | Loss: 2.9055 | LR: 2.30e-04 | 1390.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23460 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23470/44000 | Loss: 2.9402 | LR: 2.30e-04 | 1390.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23470 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23480/44000 | Loss: 2.9060 | LR: 2.30e-04 | 1389.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23480 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23490/44000 | Loss: 2.9621 | LR: 2.30e-04 | 1391.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23490 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23500/44000 | Loss: 2.9367 | LR: 2.30e-04 | 1389.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23500 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23510/44000 | Loss: 2.9450 | LR: 2.29e-04 | 1391.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23510 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23520/44000 | Loss: 2.8932 | LR: 2.29e-04 | 1389.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23520 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23530/44000 | Loss: 2.8914 | LR: 2.29e-04 | 1386.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23530 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23540/44000 | Loss: 2.9409 | LR: 2.29e-04 | 1390.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23540 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23550/44000 | Loss: 2.9674 | LR: 2.29e-04 | 1391.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23550 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23560/44000 | Loss: 2.9313 | LR: 2.29e-04 | 1392.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23560 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23570/44000 | Loss: 2.9070 | LR: 2.28e-04 | 1388.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23570 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23580/44000 | Loss: 2.9113 | LR: 2.28e-04 | 1389.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23580 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23590/44000 | Loss: 2.9247 | LR: 2.28e-04 | 1391.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23590 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23600/44000 | Loss: 2.9572 | LR: 2.28e-04 | 1388.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23600 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23610/44000 | Loss: 2.9392 | LR: 2.28e-04 | 1389.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23610 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23620/44000 | Loss: 2.9352 | LR: 2.27e-04 | 1391.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23620 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23630/44000 | Loss: 2.9384 | LR: 2.27e-04 | 1389.2ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23630 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23640/44000 | Loss: 2.9469 | LR: 2.27e-04 | 1391.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23640 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23650/44000 | Loss: 2.9300 | LR: 2.27e-04 | 1392.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23650 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23660/44000 | Loss: 2.9395 | LR: 2.27e-04 | 1389.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23660 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23670/44000 | Loss: 2.9346 | LR: 2.27e-04 | 1390.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23670 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23680/44000 | Loss: 2.9118 | LR: 2.26e-04 | 1393.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23680 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23690/44000 | Loss: 2.9217 | LR: 2.26e-04 | 1390.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23690 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23700/44000 | Loss: 2.9253 | LR: 2.26e-04 | 1390.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23700 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23710/44000 | Loss: 2.9481 | LR: 2.26e-04 | 1389.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23710 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23720/44000 | Loss: 2.9442 | LR: 2.26e-04 | 1391.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23720 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23730/44000 | Loss: 2.9224 | LR: 2.25e-04 | 1390.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23730 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23740/44000 | Loss: 2.9213 | LR: 2.25e-04 | 1389.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23740 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23750/44000 | Loss: 2.9586 | LR: 2.25e-04 | 1389.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23750 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23760/44000 | Loss: 2.9280 | LR: 2.25e-04 | 1388.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23760 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23770/44000 | Loss: 2.9257 | LR: 2.25e-04 | 1392.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23770 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23780/44000 | Loss: 2.9163 | LR: 2.25e-04 | 1389.9ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23780 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23790/44000 | Loss: 2.9254 | LR: 2.24e-04 | 1391.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23790 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23800/44000 | Loss: 2.9139 | LR: 2.24e-04 | 1390.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23800 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23810/44000 | Loss: 2.9332 | LR: 2.24e-04 | 1391.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23810 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23820/44000 | Loss: 2.9617 | LR: 2.24e-04 | 1388.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23820 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23830/44000 | Loss: 2.9092 | LR: 2.24e-04 | 1391.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23830 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23840/44000 | Loss: 2.9291 | LR: 2.23e-04 | 1393.5ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23840 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23850/44000 | Loss: 2.9045 | LR: 2.23e-04 | 1389.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23850 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23860/44000 | Loss: 2.9026 | LR: 2.23e-04 | 1388.6ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23860 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23870/44000 | Loss: 2.9355 | LR: 2.23e-04 | 1392.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23870 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23880/44000 | Loss: 2.9097 | LR: 2.23e-04 | 1389.4ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 23880 that is less than the current step 23881. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 23890/44000 | Loss: 2.9264 | LR: 2.23e-04 | 1391.1ms/step | 14.4 samples/s +Step 23900/44000 | Loss: 2.9087 | LR: 2.22e-04 | 1392.5ms/step | 14.4 samples/s +Step 23910/44000 | Loss: 2.9359 | LR: 2.22e-04 | 1393.5ms/step | 14.4 samples/s +Step 23920/44000 | Loss: 2.9091 | LR: 2.22e-04 | 1391.0ms/step | 14.4 samples/s +Step 23930/44000 | Loss: 2.9321 | LR: 2.22e-04 | 1392.0ms/step | 14.4 samples/s +Step 23940/44000 | Loss: 2.9294 | LR: 2.22e-04 | 1388.3ms/step | 14.4 samples/s +Step 23950/44000 | Loss: 2.9039 | LR: 2.21e-04 | 1391.5ms/step | 14.4 samples/s +Step 23960/44000 | Loss: 2.9153 | LR: 2.21e-04 | 1389.6ms/step | 14.4 samples/s +Step 23970/44000 | Loss: 2.9399 | LR: 2.21e-04 | 1388.9ms/step | 14.4 samples/s +Step 23980/44000 | Loss: 2.9313 | LR: 2.21e-04 | 1387.9ms/step | 14.4 samples/s +Step 23990/44000 | Loss: 2.9354 | LR: 2.21e-04 | 1390.4ms/step | 14.4 samples/s +Step 24000/44000 | Loss: 2.9362 | LR: 2.21e-04 | 1389.6ms/step | 14.4 samples/s +Validation at step 24000: Loss = 2.9173, Perplexity = 18.49 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-24000 +Step 24010/44000 | Loss: 2.9276 | LR: 2.20e-04 | 3902.0ms/step | 5.1 samples/s +Step 24020/44000 | Loss: 2.9045 | LR: 2.20e-04 | 1400.0ms/step | 14.3 samples/s +Step 24030/44000 | Loss: 2.9234 | LR: 2.20e-04 | 1400.0ms/step | 14.3 samples/s +Step 24040/44000 | Loss: 2.9248 | LR: 2.20e-04 | 1399.8ms/step | 14.3 samples/s +Step 24050/44000 | Loss: 2.9101 | LR: 2.20e-04 | 1400.1ms/step | 14.3 samples/s +Step 24060/44000 | Loss: 2.9267 | LR: 2.19e-04 | 1400.0ms/step | 14.3 samples/s +Step 24070/44000 | Loss: 2.9637 | LR: 2.19e-04 | 1400.4ms/step | 14.3 samples/s +Step 24080/44000 | Loss: 2.9212 | LR: 2.19e-04 | 1399.6ms/step | 14.3 samples/s +Step 24090/44000 | Loss: 2.9421 | LR: 2.19e-04 | 1400.1ms/step | 14.3 samples/s +Step 24100/44000 | Loss: 2.9139 | LR: 2.19e-04 | 1395.8ms/step | 14.3 samples/s +Step 24110/44000 | Loss: 2.9664 | LR: 2.19e-04 | 1397.6ms/step | 14.3 samples/s +Step 24120/44000 | Loss: 2.9452 | LR: 2.18e-04 | 1394.5ms/step | 14.3 samples/s +Step 24130/44000 | Loss: 2.9286 | LR: 2.18e-04 | 1396.1ms/step | 14.3 samples/s +Step 24140/44000 | Loss: 2.9348 | LR: 2.18e-04 | 1395.6ms/step | 14.3 samples/s +Step 24150/44000 | Loss: 2.9487 | LR: 2.18e-04 | 1398.8ms/step | 14.3 samples/s +Step 24160/44000 | Loss: 2.8972 | LR: 2.18e-04 | 1394.3ms/step | 14.3 samples/s +Step 24170/44000 | Loss: 2.9253 | LR: 2.17e-04 | 1395.1ms/step | 14.3 samples/s +Step 24180/44000 | Loss: 2.9490 | LR: 2.17e-04 | 1395.3ms/step | 14.3 samples/s +Step 24190/44000 | Loss: 2.9083 | LR: 2.17e-04 | 1392.1ms/step | 14.4 samples/s +Step 24200/44000 | Loss: 2.9146 | LR: 2.17e-04 | 1394.4ms/step | 14.3 samples/s +Step 24210/44000 | Loss: 2.9098 | LR: 2.17e-04 | 1396.2ms/step | 14.3 samples/s +Step 24220/44000 | Loss: 2.9004 | LR: 2.17e-04 | 1397.4ms/step | 14.3 samples/s +Step 24230/44000 | Loss: 2.8906 | LR: 2.16e-04 | 1392.1ms/step | 14.4 samples/s +Step 24240/44000 | Loss: 2.9550 | LR: 2.16e-04 | 1393.6ms/step | 14.4 samples/s +Step 24250/44000 | Loss: 2.9348 | LR: 2.16e-04 | 1392.8ms/step | 14.4 samples/s +Step 24260/44000 | Loss: 2.9425 | LR: 2.16e-04 | 1394.7ms/step | 14.3 samples/s +Step 24270/44000 | Loss: 2.9252 | LR: 2.16e-04 | 1395.2ms/step | 14.3 samples/s +Step 24280/44000 | Loss: 2.9185 | LR: 2.16e-04 | 1394.2ms/step | 14.3 samples/s +Step 24290/44000 | Loss: 2.8858 | LR: 2.15e-04 | 1391.4ms/step | 14.4 samples/s +Step 24300/44000 | Loss: 2.9515 | LR: 2.15e-04 | 1396.8ms/step | 14.3 samples/s +Step 24310/44000 | Loss: 2.9668 | LR: 2.15e-04 | 1399.8ms/step | 14.3 samples/s +Step 24320/44000 | Loss: 2.9196 | LR: 2.15e-04 | 1400.1ms/step | 14.3 samples/s +Step 24330/44000 | Loss: 2.8937 | LR: 2.15e-04 | 1400.1ms/step | 14.3 samples/s +Step 24340/44000 | Loss: 2.9216 | LR: 2.14e-04 | 1400.1ms/step | 14.3 samples/s +Step 24350/44000 | Loss: 2.9169 | LR: 2.14e-04 | 1395.3ms/step | 14.3 samples/s +Step 24360/44000 | Loss: 2.9305 | LR: 2.14e-04 | 1392.4ms/step | 14.4 samples/s +Step 24370/44000 | Loss: 2.9442 | LR: 2.14e-04 | 1394.2ms/step | 14.3 samples/s +Step 24380/44000 | Loss: 2.8921 | LR: 2.14e-04 | 1399.9ms/step | 14.3 samples/s +Step 24390/44000 | Loss: 2.9499 | LR: 2.14e-04 | 1399.9ms/step | 14.3 samples/s +Step 24400/44000 | Loss: 2.9341 | LR: 2.13e-04 | 1399.9ms/step | 14.3 samples/s +Step 24410/44000 | Loss: 2.9210 | LR: 2.13e-04 | 1400.2ms/step | 14.3 samples/s +Step 24420/44000 | Loss: 2.9169 | LR: 2.13e-04 | 1399.9ms/step | 14.3 samples/s +Step 24430/44000 | Loss: 2.9347 | LR: 2.13e-04 | 1400.0ms/step | 14.3 samples/s +Step 24440/44000 | Loss: 2.9374 | LR: 2.13e-04 | 1400.2ms/step | 14.3 samples/s +Step 24450/44000 | Loss: 2.9378 | LR: 2.12e-04 | 1399.8ms/step | 14.3 samples/s +Step 24460/44000 | Loss: 2.9278 | LR: 2.12e-04 | 1399.9ms/step | 14.3 samples/s +Step 24470/44000 | Loss: 2.9187 | LR: 2.12e-04 | 1400.0ms/step | 14.3 samples/s +Step 24480/44000 | Loss: 2.9148 | LR: 2.12e-04 | 1400.0ms/step | 14.3 samples/s +Step 24490/44000 | Loss: 2.8969 | LR: 2.12e-04 | 1399.9ms/step | 14.3 samples/s +Step 24500/44000 | Loss: 2.9364 | LR: 2.12e-04 | 1400.1ms/step | 14.3 samples/s +Step 24510/44000 | Loss: 2.8936 | LR: 2.11e-04 | 1399.9ms/step | 14.3 samples/s +Step 24520/44000 | Loss: 2.9561 | LR: 2.11e-04 | 1400.2ms/step | 14.3 samples/s +Step 24530/44000 | Loss: 2.8984 | LR: 2.11e-04 | 1398.1ms/step | 14.3 samples/s +Step 24540/44000 | Loss: 2.8883 | LR: 2.11e-04 | 1399.5ms/step | 14.3 samples/s +Step 24550/44000 | Loss: 2.9053 | LR: 2.11e-04 | 1400.4ms/step | 14.3 samples/s +Step 24560/44000 | Loss: 2.9197 | LR: 2.10e-04 | 1395.8ms/step | 14.3 samples/s +Step 24570/44000 | Loss: 2.9142 | LR: 2.10e-04 | 1395.7ms/step | 14.3 samples/s +Step 24580/44000 | Loss: 2.9190 | LR: 2.10e-04 | 1392.3ms/step | 14.4 samples/s +Step 24590/44000 | Loss: 2.9026 | LR: 2.10e-04 | 1392.2ms/step | 14.4 samples/s +Step 24600/44000 | Loss: 2.8870 | LR: 2.10e-04 | 1393.9ms/step | 14.3 samples/s +Step 24610/44000 | Loss: 2.9353 | LR: 2.10e-04 | 1393.9ms/step | 14.3 samples/s +Step 24620/44000 | Loss: 2.8962 | LR: 2.09e-04 | 1393.7ms/step | 14.4 samples/s +Step 24630/44000 | Loss: 2.9108 | LR: 2.09e-04 | 1391.4ms/step | 14.4 samples/s +Step 24640/44000 | Loss: 2.9201 | LR: 2.09e-04 | 1395.1ms/step | 14.3 samples/s +Step 24650/44000 | Loss: 2.9101 | LR: 2.09e-04 | 1393.8ms/step | 14.3 samples/s +Step 24660/44000 | Loss: 2.9058 | LR: 2.09e-04 | 1392.3ms/step | 14.4 samples/s +Step 24670/44000 | Loss: 2.9040 | LR: 2.09e-04 | 1393.1ms/step | 14.4 samples/s +Step 24680/44000 | Loss: 2.9415 | LR: 2.08e-04 | 1394.3ms/step | 14.3 samples/s +Step 24690/44000 | Loss: 2.9134 | LR: 2.08e-04 | 1395.0ms/step | 14.3 samples/s +Step 24700/44000 | Loss: 2.9186 | LR: 2.08e-04 | 1395.4ms/step | 14.3 samples/s +Step 24710/44000 | Loss: 2.8991 | LR: 2.08e-04 | 1393.7ms/step | 14.4 samples/s +Step 24720/44000 | Loss: 2.9408 | LR: 2.08e-04 | 1394.2ms/step | 14.3 samples/s +Step 24730/44000 | Loss: 2.9463 | LR: 2.07e-04 | 1400.0ms/step | 14.3 samples/s +Step 24740/44000 | Loss: 2.9235 | LR: 2.07e-04 | 1400.3ms/step | 14.3 samples/s +Step 24750/44000 | Loss: 2.9121 | LR: 2.07e-04 | 1399.8ms/step | 14.3 samples/s +Step 24760/44000 | Loss: 2.9457 | LR: 2.07e-04 | 1399.9ms/step | 14.3 samples/s +Step 24770/44000 | Loss: 2.9368 | LR: 2.07e-04 | 1400.1ms/step | 14.3 samples/s +Step 24780/44000 | Loss: 2.8845 | LR: 2.07e-04 | 1400.0ms/step | 14.3 samples/s +Step 24790/44000 | Loss: 2.9052 | LR: 2.06e-04 | 1400.1ms/step | 14.3 samples/s +Step 24800/44000 | Loss: 2.9069 | LR: 2.06e-04 | 1400.1ms/step | 14.3 samples/s +Step 24810/44000 | Loss: 2.9195 | LR: 2.06e-04 | 1399.9ms/step | 14.3 samples/s +Step 24820/44000 | Loss: 2.9301 | LR: 2.06e-04 | 1395.6ms/step | 14.3 samples/s +Step 24830/44000 | Loss: 2.9252 | LR: 2.06e-04 | 1396.4ms/step | 14.3 samples/s +Step 24840/44000 | Loss: 2.9226 | LR: 2.05e-04 | 1393.2ms/step | 14.4 samples/s +Step 24850/44000 | Loss: 2.9049 | LR: 2.05e-04 | 1392.8ms/step | 14.4 samples/s +Step 24860/44000 | Loss: 2.8967 | LR: 2.05e-04 | 1392.1ms/step | 14.4 samples/s +Step 24870/44000 | Loss: 2.9115 | LR: 2.05e-04 | 1393.6ms/step | 14.4 samples/s +Step 24880/44000 | Loss: 2.9408 | LR: 2.05e-04 | 1394.8ms/step | 14.3 samples/s +Step 24890/44000 | Loss: 2.9163 | LR: 2.05e-04 | 1393.6ms/step | 14.4 samples/s +Step 24900/44000 | Loss: 2.9692 | LR: 2.04e-04 | 1394.7ms/step | 14.3 samples/s +Step 24910/44000 | Loss: 2.9530 | LR: 2.04e-04 | 1393.3ms/step | 14.4 samples/s +Step 24920/44000 | Loss: 2.9272 | LR: 2.04e-04 | 1395.9ms/step | 14.3 samples/s +Step 24930/44000 | Loss: 2.9023 | LR: 2.04e-04 | 1395.9ms/step | 14.3 samples/s +Step 24940/44000 | Loss: 2.9421 | LR: 2.04e-04 | 1396.2ms/step | 14.3 samples/s +Step 24950/44000 | Loss: 2.9025 | LR: 2.04e-04 | 1400.2ms/step | 14.3 samples/s +Step 24960/44000 | Loss: 2.9097 | LR: 2.03e-04 | 1399.8ms/step | 14.3 samples/s +Step 24970/44000 | Loss: 2.9059 | LR: 2.03e-04 | 1399.7ms/step | 14.3 samples/s +Step 24980/44000 | Loss: 2.9377 | LR: 2.03e-04 | 1396.1ms/step | 14.3 samples/s +Step 24990/44000 | Loss: 2.9184 | LR: 2.03e-04 | 1393.9ms/step | 14.3 samples/s +Step 25000/44000 | Loss: 2.9149 | LR: 2.03e-04 | 1391.9ms/step | 14.4 samples/s +Validation at step 25000: Loss = 2.9096, Perplexity = 18.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-25000 +Step 25010/44000 | Loss: 2.9180 | LR: 2.02e-04 | 3910.5ms/step | 5.1 samples/s +Step 25020/44000 | Loss: 2.9231 | LR: 2.02e-04 | 1399.9ms/step | 14.3 samples/s +Step 25030/44000 | Loss: 2.8954 | LR: 2.02e-04 | 1400.1ms/step | 14.3 samples/s +Step 25040/44000 | Loss: 2.9447 | LR: 2.02e-04 | 1399.9ms/step | 14.3 samples/s +Step 25050/44000 | Loss: 2.9176 | LR: 2.02e-04 | 1399.9ms/step | 14.3 samples/s +Step 25060/44000 | Loss: 2.8925 | LR: 2.02e-04 | 1400.1ms/step | 14.3 samples/s +Step 25070/44000 | Loss: 2.9253 | LR: 2.01e-04 | 1400.1ms/step | 14.3 samples/s +Step 25080/44000 | Loss: 2.9118 | LR: 2.01e-04 | 1399.9ms/step | 14.3 samples/s +Step 25090/44000 | Loss: 2.8949 | LR: 2.01e-04 | 1400.0ms/step | 14.3 samples/s +Step 25100/44000 | Loss: 2.9067 | LR: 2.01e-04 | 1400.0ms/step | 14.3 samples/s +Step 25110/44000 | Loss: 2.9229 | LR: 2.01e-04 | 1400.2ms/step | 14.3 samples/s +Step 25120/44000 | Loss: 2.9485 | LR: 2.00e-04 | 1399.7ms/step | 14.3 samples/s +Step 25130/44000 | Loss: 2.9282 | LR: 2.00e-04 | 1400.0ms/step | 14.3 samples/s +Step 25140/44000 | Loss: 2.8947 | LR: 2.00e-04 | 1399.8ms/step | 14.3 samples/s +Step 25150/44000 | Loss: 2.9064 | LR: 2.00e-04 | 1400.2ms/step | 14.3 samples/s +Step 25160/44000 | Loss: 2.9227 | LR: 2.00e-04 | 1399.9ms/step | 14.3 samples/s +Step 25170/44000 | Loss: 2.9013 | LR: 2.00e-04 | 1400.1ms/step | 14.3 samples/s +Step 25180/44000 | Loss: 2.9490 | LR: 1.99e-04 | 1399.9ms/step | 14.3 samples/s +Step 25190/44000 | Loss: 2.8863 | LR: 1.99e-04 | 1400.0ms/step | 14.3 samples/s +Step 25200/44000 | Loss: 2.9315 | LR: 1.99e-04 | 1400.1ms/step | 14.3 samples/s +Step 25210/44000 | Loss: 2.8976 | LR: 1.99e-04 | 1399.9ms/step | 14.3 samples/s +Step 25220/44000 | Loss: 2.8806 | LR: 1.99e-04 | 1400.0ms/step | 14.3 samples/s +Step 25230/44000 | Loss: 2.8958 | LR: 1.99e-04 | 1399.9ms/step | 14.3 samples/s +Step 25240/44000 | Loss: 2.9367 | LR: 1.98e-04 | 1400.2ms/step | 14.3 samples/s +Step 25250/44000 | Loss: 2.9292 | LR: 1.98e-04 | 1400.1ms/step | 14.3 samples/s +Step 25260/44000 | Loss: 2.9232 | LR: 1.98e-04 | 1399.8ms/step | 14.3 samples/s +Step 25270/44000 | Loss: 2.8992 | LR: 1.98e-04 | 1400.2ms/step | 14.3 samples/s +Step 25280/44000 | Loss: 2.9279 | LR: 1.98e-04 | 1399.9ms/step | 14.3 samples/s +Step 25290/44000 | Loss: 2.9336 | LR: 1.97e-04 | 1399.9ms/step | 14.3 samples/s +Step 25300/44000 | Loss: 2.9318 | LR: 1.97e-04 | 1399.9ms/step | 14.3 samples/s +Step 25310/44000 | Loss: 2.9372 | LR: 1.97e-04 | 1400.1ms/step | 14.3 samples/s +Step 25320/44000 | Loss: 2.9582 | LR: 1.97e-04 | 1400.3ms/step | 14.3 samples/s +Step 25330/44000 | Loss: 2.9259 | LR: 1.97e-04 | 1399.7ms/step | 14.3 samples/s +Step 25340/44000 | Loss: 2.8846 | LR: 1.97e-04 | 1400.1ms/step | 14.3 samples/s +Step 25350/44000 | Loss: 2.8964 | LR: 1.96e-04 | 1400.1ms/step | 14.3 samples/s +Step 25360/44000 | Loss: 2.9024 | LR: 1.96e-04 | 1400.1ms/step | 14.3 samples/s +Step 25370/44000 | Loss: 2.9016 | LR: 1.96e-04 | 1399.8ms/step | 14.3 samples/s +Step 25380/44000 | Loss: 2.8913 | LR: 1.96e-04 | 1400.3ms/step | 14.3 samples/s +Step 25390/44000 | Loss: 2.9050 | LR: 1.96e-04 | 1399.7ms/step | 14.3 samples/s +Step 25400/44000 | Loss: 2.9379 | LR: 1.96e-04 | 1400.4ms/step | 14.3 samples/s +Step 25410/44000 | Loss: 2.9204 | LR: 1.95e-04 | 1399.8ms/step | 14.3 samples/s +Step 25420/44000 | Loss: 2.9427 | LR: 1.95e-04 | 1400.0ms/step | 14.3 samples/s +Step 25430/44000 | Loss: 2.9184 | LR: 1.95e-04 | 1399.9ms/step | 14.3 samples/s +Step 25440/44000 | Loss: 2.9047 | LR: 1.95e-04 | 1400.0ms/step | 14.3 samples/s +Step 25450/44000 | Loss: 2.9175 | LR: 1.95e-04 | 1400.1ms/step | 14.3 samples/s +Step 25460/44000 | Loss: 2.9057 | LR: 1.94e-04 | 1400.0ms/step | 14.3 samples/s +Step 25470/44000 | Loss: 2.9151 | LR: 1.94e-04 | 1400.1ms/step | 14.3 samples/s +Step 25480/44000 | Loss: 2.9028 | LR: 1.94e-04 | 1400.0ms/step | 14.3 samples/s +Step 25490/44000 | Loss: 2.9377 | LR: 1.94e-04 | 1399.9ms/step | 14.3 samples/s +Step 25500/44000 | Loss: 2.9447 | LR: 1.94e-04 | 1399.9ms/step | 14.3 samples/s +Step 25510/44000 | Loss: 2.9222 | LR: 1.94e-04 | 1400.0ms/step | 14.3 samples/s +Step 25520/44000 | Loss: 2.9154 | LR: 1.93e-04 | 1400.2ms/step | 14.3 samples/s +Step 25530/44000 | Loss: 2.9238 | LR: 1.93e-04 | 1399.8ms/step | 14.3 samples/s +Step 25540/44000 | Loss: 2.9462 | LR: 1.93e-04 | 1400.0ms/step | 14.3 samples/s +Step 25550/44000 | Loss: 2.9187 | LR: 1.93e-04 | 1399.9ms/step | 14.3 samples/s +Step 25560/44000 | Loss: 2.8963 | LR: 1.93e-04 | 1400.1ms/step | 14.3 samples/s +Step 25570/44000 | Loss: 2.9230 | LR: 1.92e-04 | 1400.0ms/step | 14.3 samples/s +Step 25580/44000 | Loss: 2.9089 | LR: 1.92e-04 | 1400.0ms/step | 14.3 samples/s +Step 25590/44000 | Loss: 2.9154 | LR: 1.92e-04 | 1399.8ms/step | 14.3 samples/s +Step 25600/44000 | Loss: 2.9321 | LR: 1.92e-04 | 1400.1ms/step | 14.3 samples/s +Step 25610/44000 | Loss: 2.9334 | LR: 1.92e-04 | 1400.1ms/step | 14.3 samples/s +Step 25620/44000 | Loss: 2.9110 | LR: 1.92e-04 | 1399.8ms/step | 14.3 samples/s +Step 25630/44000 | Loss: 2.9274 | LR: 1.91e-04 | 1400.1ms/step | 14.3 samples/s +Step 25640/44000 | Loss: 2.9389 | LR: 1.91e-04 | 1400.2ms/step | 14.3 samples/s +Step 25650/44000 | Loss: 2.9012 | LR: 1.91e-04 | 1399.9ms/step | 14.3 samples/s +Step 25660/44000 | Loss: 2.9079 | LR: 1.91e-04 | 1400.0ms/step | 14.3 samples/s +Step 25670/44000 | Loss: 2.9014 | LR: 1.91e-04 | 1400.1ms/step | 14.3 samples/s +Step 25680/44000 | Loss: 2.9205 | LR: 1.91e-04 | 1399.9ms/step | 14.3 samples/s +Step 25690/44000 | Loss: 2.9243 | LR: 1.90e-04 | 1399.8ms/step | 14.3 samples/s +Step 25700/44000 | Loss: 2.8535 | LR: 1.90e-04 | 1400.2ms/step | 14.3 samples/s +Step 25710/44000 | Loss: 2.9212 | LR: 1.90e-04 | 1400.0ms/step | 14.3 samples/s +Step 25720/44000 | Loss: 2.9230 | LR: 1.90e-04 | 1399.9ms/step | 14.3 samples/s +Step 25730/44000 | Loss: 2.9308 | LR: 1.90e-04 | 1400.1ms/step | 14.3 samples/s +Step 25740/44000 | Loss: 2.9145 | LR: 1.89e-04 | 1399.9ms/step | 14.3 samples/s +Step 25750/44000 | Loss: 2.8902 | LR: 1.89e-04 | 1399.9ms/step | 14.3 samples/s +Step 25760/44000 | Loss: 2.9473 | LR: 1.89e-04 | 1400.0ms/step | 14.3 samples/s +Step 25770/44000 | Loss: 2.9341 | LR: 1.89e-04 | 1400.0ms/step | 14.3 samples/s +Step 25780/44000 | Loss: 2.9236 | LR: 1.89e-04 | 1400.0ms/step | 14.3 samples/s +Step 25790/44000 | Loss: 2.9106 | LR: 1.89e-04 | 1399.8ms/step | 14.3 samples/s +Step 25800/44000 | Loss: 2.8903 | LR: 1.88e-04 | 1400.2ms/step | 14.3 samples/s +Step 25810/44000 | Loss: 2.9178 | LR: 1.88e-04 | 1400.1ms/step | 14.3 samples/s +Step 25820/44000 | Loss: 2.9282 | LR: 1.88e-04 | 1399.8ms/step | 14.3 samples/s +Step 25830/44000 | Loss: 2.9369 | LR: 1.88e-04 | 1400.1ms/step | 14.3 samples/s +Step 25840/44000 | Loss: 2.9120 | LR: 1.88e-04 | 1400.1ms/step | 14.3 samples/s +Step 25850/44000 | Loss: 2.8918 | LR: 1.88e-04 | 1399.8ms/step | 14.3 samples/s +Step 25860/44000 | Loss: 2.8907 | LR: 1.87e-04 | 1400.2ms/step | 14.3 samples/s +Step 25870/44000 | Loss: 2.9183 | LR: 1.87e-04 | 1399.8ms/step | 14.3 samples/s +Step 25880/44000 | Loss: 2.8854 | LR: 1.87e-04 | 1400.0ms/step | 14.3 samples/s +Step 25890/44000 | Loss: 2.9170 | LR: 1.87e-04 | 1400.0ms/step | 14.3 samples/s +Step 25900/44000 | Loss: 2.9258 | LR: 1.87e-04 | 1400.0ms/step | 14.3 samples/s +Step 25910/44000 | Loss: 2.9143 | LR: 1.87e-04 | 1400.1ms/step | 14.3 samples/s +Step 25920/44000 | Loss: 2.9499 | LR: 1.86e-04 | 1400.0ms/step | 14.3 samples/s +Step 25930/44000 | Loss: 2.8807 | LR: 1.86e-04 | 1399.9ms/step | 14.3 samples/s +Step 25940/44000 | Loss: 2.9150 | LR: 1.86e-04 | 1400.1ms/step | 14.3 samples/s +Step 25950/44000 | Loss: 2.9175 | LR: 1.86e-04 | 1399.9ms/step | 14.3 samples/s +Step 25960/44000 | Loss: 2.9114 | LR: 1.86e-04 | 1400.1ms/step | 14.3 samples/s +Step 25970/44000 | Loss: 2.9051 | LR: 1.85e-04 | 1400.0ms/step | 14.3 samples/s +Step 25980/44000 | Loss: 2.9303 | LR: 1.85e-04 | 1399.9ms/step | 14.3 samples/s +Step 25990/44000 | Loss: 2.9180 | LR: 1.85e-04 | 1400.1ms/step | 14.3 samples/s +Step 26000/44000 | Loss: 2.9013 | LR: 1.85e-04 | 1399.7ms/step | 14.3 samples/s +Validation at step 26000: Loss = 2.9029, Perplexity = 18.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-26000 +Step 26010/44000 | Loss: 2.9361 | LR: 1.85e-04 | 3898.1ms/step | 5.1 samples/s +Step 26020/44000 | Loss: 2.9187 | LR: 1.85e-04 | 1400.2ms/step | 14.3 samples/s +Step 26030/44000 | Loss: 2.9096 | LR: 1.84e-04 | 1399.7ms/step | 14.3 samples/s +Step 26040/44000 | Loss: 2.9062 | LR: 1.84e-04 | 1400.1ms/step | 14.3 samples/s +Step 26050/44000 | Loss: 2.9099 | LR: 1.84e-04 | 1400.1ms/step | 14.3 samples/s +Step 26060/44000 | Loss: 2.9303 | LR: 1.84e-04 | 1400.1ms/step | 14.3 samples/s +Step 26070/44000 | Loss: 2.8953 | LR: 1.84e-04 | 1400.0ms/step | 14.3 samples/s +Step 26080/44000 | Loss: 2.9080 | LR: 1.84e-04 | 1400.0ms/step | 14.3 samples/s +Step 26090/44000 | Loss: 2.9128 | LR: 1.83e-04 | 1399.7ms/step | 14.3 samples/s +Step 26100/44000 | Loss: 2.9275 | LR: 1.83e-04 | 1400.3ms/step | 14.3 samples/s +Step 26110/44000 | Loss: 2.9109 | LR: 1.83e-04 | 1400.0ms/step | 14.3 samples/s +Step 26120/44000 | Loss: 2.9155 | LR: 1.83e-04 | 1399.9ms/step | 14.3 samples/s +Step 26130/44000 | Loss: 2.8990 | LR: 1.83e-04 | 1399.9ms/step | 14.3 samples/s +Step 26140/44000 | Loss: 2.9114 | LR: 1.82e-04 | 1400.1ms/step | 14.3 samples/s +Step 26150/44000 | Loss: 2.8882 | LR: 1.82e-04 | 1400.0ms/step | 14.3 samples/s +Step 26160/44000 | Loss: 2.8995 | LR: 1.82e-04 | 1400.0ms/step | 14.3 samples/s +Step 26170/44000 | Loss: 2.8921 | LR: 1.82e-04 | 1400.2ms/step | 14.3 samples/s +Step 26180/44000 | Loss: 2.9277 | LR: 1.82e-04 | 1399.8ms/step | 14.3 samples/s +Step 26190/44000 | Loss: 2.8895 | LR: 1.82e-04 | 1399.9ms/step | 14.3 samples/s +Step 26200/44000 | Loss: 2.9478 | LR: 1.81e-04 | 1400.3ms/step | 14.3 samples/s +Step 26210/44000 | Loss: 2.8967 | LR: 1.81e-04 | 1400.0ms/step | 14.3 samples/s +Step 26220/44000 | Loss: 2.9155 | LR: 1.81e-04 | 1399.9ms/step | 14.3 samples/s +Step 26230/44000 | Loss: 2.9044 | LR: 1.81e-04 | 1400.0ms/step | 14.3 samples/s +Step 26240/44000 | Loss: 2.9336 | LR: 1.81e-04 | 1399.9ms/step | 14.3 samples/s +Step 26250/44000 | Loss: 2.8993 | LR: 1.81e-04 | 1400.2ms/step | 14.3 samples/s +Step 26260/44000 | Loss: 2.9083 | LR: 1.80e-04 | 1399.9ms/step | 14.3 samples/s +Step 26270/44000 | Loss: 2.8999 | LR: 1.80e-04 | 1400.0ms/step | 14.3 samples/s +Step 26280/44000 | Loss: 2.8813 | LR: 1.80e-04 | 1400.2ms/step | 14.3 samples/s +Step 26290/44000 | Loss: 2.9058 | LR: 1.80e-04 | 1400.0ms/step | 14.3 samples/s +Step 26300/44000 | Loss: 2.9150 | LR: 1.80e-04 | 1400.0ms/step | 14.3 samples/s +Step 26310/44000 | Loss: 2.9253 | LR: 1.80e-04 | 1400.0ms/step | 14.3 samples/s +Step 26320/44000 | Loss: 2.9116 | LR: 1.79e-04 | 1399.8ms/step | 14.3 samples/s +Step 26330/44000 | Loss: 2.9113 | LR: 1.79e-04 | 1400.1ms/step | 14.3 samples/s +Step 26340/44000 | Loss: 2.8727 | LR: 1.79e-04 | 1399.8ms/step | 14.3 samples/s +Step 26350/44000 | Loss: 2.9058 | LR: 1.79e-04 | 1400.2ms/step | 14.3 samples/s +Step 26360/44000 | Loss: 2.9370 | LR: 1.79e-04 | 1399.9ms/step | 14.3 samples/s +Step 26370/44000 | Loss: 2.9373 | LR: 1.78e-04 | 1400.2ms/step | 14.3 samples/s +Step 26380/44000 | Loss: 2.9317 | LR: 1.78e-04 | 1399.9ms/step | 14.3 samples/s +Step 26390/44000 | Loss: 2.9116 | LR: 1.78e-04 | 1399.9ms/step | 14.3 samples/s +Step 26400/44000 | Loss: 2.9152 | LR: 1.78e-04 | 1400.0ms/step | 14.3 samples/s +Step 26410/44000 | Loss: 2.8973 | LR: 1.78e-04 | 1400.1ms/step | 14.3 samples/s +Step 26420/44000 | Loss: 2.9040 | LR: 1.78e-04 | 1399.9ms/step | 14.3 samples/s +Step 26430/44000 | Loss: 2.9192 | LR: 1.77e-04 | 1399.9ms/step | 14.3 samples/s +Step 26440/44000 | Loss: 2.8862 | LR: 1.77e-04 | 1400.0ms/step | 14.3 samples/s +Step 26450/44000 | Loss: 2.8955 | LR: 1.77e-04 | 1400.1ms/step | 14.3 samples/s +Step 26460/44000 | Loss: 2.8814 | LR: 1.77e-04 | 1399.9ms/step | 14.3 samples/s +Step 26470/44000 | Loss: 2.9171 | LR: 1.77e-04 | 1400.0ms/step | 14.3 samples/s +Step 26480/44000 | Loss: 2.8960 | LR: 1.77e-04 | 1400.2ms/step | 14.3 samples/s +Step 26490/44000 | Loss: 2.8976 | LR: 1.76e-04 | 1399.8ms/step | 14.3 samples/s +Step 26500/44000 | Loss: 2.9134 | LR: 1.76e-04 | 1400.1ms/step | 14.3 samples/s +Step 26510/44000 | Loss: 2.9165 | LR: 1.76e-04 | 1400.0ms/step | 14.3 samples/s +Step 26520/44000 | Loss: 2.8954 | LR: 1.76e-04 | 1400.0ms/step | 14.3 samples/s +Step 26530/44000 | Loss: 2.8919 | LR: 1.76e-04 | 1400.1ms/step | 14.3 samples/s +Step 26540/44000 | Loss: 2.9211 | LR: 1.76e-04 | 1400.0ms/step | 14.3 samples/s +Step 26550/44000 | Loss: 2.9150 | LR: 1.75e-04 | 1399.7ms/step | 14.3 samples/s +Step 26560/44000 | Loss: 2.9301 | LR: 1.75e-04 | 1400.1ms/step | 14.3 samples/s +Step 26570/44000 | Loss: 2.9285 | LR: 1.75e-04 | 1400.1ms/step | 14.3 samples/s +Step 26580/44000 | Loss: 2.9140 | LR: 1.75e-04 | 1399.8ms/step | 14.3 samples/s +Step 26590/44000 | Loss: 2.9024 | LR: 1.75e-04 | 1400.2ms/step | 14.3 samples/s +Step 26600/44000 | Loss: 2.8695 | LR: 1.74e-04 | 1400.0ms/step | 14.3 samples/s +Step 26610/44000 | Loss: 2.8682 | LR: 1.74e-04 | 1400.0ms/step | 14.3 samples/s +Step 26620/44000 | Loss: 2.8859 | LR: 1.74e-04 | 1400.0ms/step | 14.3 samples/s +Step 26630/44000 | Loss: 2.8882 | LR: 1.74e-04 | 1400.1ms/step | 14.3 samples/s +Step 26640/44000 | Loss: 2.9247 | LR: 1.74e-04 | 1399.9ms/step | 14.3 samples/s +Step 26650/44000 | Loss: 2.9025 | LR: 1.74e-04 | 1399.8ms/step | 14.3 samples/s +Step 26660/44000 | Loss: 2.9062 | LR: 1.73e-04 | 1400.0ms/step | 14.3 samples/s +Step 26670/44000 | Loss: 2.8916 | LR: 1.73e-04 | 1400.0ms/step | 14.3 samples/s +Step 26680/44000 | Loss: 2.9356 | LR: 1.73e-04 | 1399.9ms/step | 14.3 samples/s +Step 26690/44000 | Loss: 2.8915 | LR: 1.73e-04 | 1400.2ms/step | 14.3 samples/s +Step 26700/44000 | Loss: 2.8800 | LR: 1.73e-04 | 1399.9ms/step | 14.3 samples/s +Step 26710/44000 | Loss: 2.8746 | LR: 1.73e-04 | 1400.2ms/step | 14.3 samples/s +Step 26720/44000 | Loss: 2.9146 | LR: 1.72e-04 | 1399.9ms/step | 14.3 samples/s +Step 26730/44000 | Loss: 2.8957 | LR: 1.72e-04 | 1400.1ms/step | 14.3 samples/s +Step 26740/44000 | Loss: 2.9181 | LR: 1.72e-04 | 1400.0ms/step | 14.3 samples/s +Step 26750/44000 | Loss: 2.9122 | LR: 1.72e-04 | 1400.1ms/step | 14.3 samples/s +Step 26760/44000 | Loss: 2.8893 | LR: 1.72e-04 | 1400.0ms/step | 14.3 samples/s +Step 26770/44000 | Loss: 2.9230 | LR: 1.72e-04 | 1399.8ms/step | 14.3 samples/s +Step 26780/44000 | Loss: 2.9069 | LR: 1.71e-04 | 1400.0ms/step | 14.3 samples/s +Step 26790/44000 | Loss: 2.8971 | LR: 1.71e-04 | 1400.0ms/step | 14.3 samples/s +Step 26800/44000 | Loss: 2.9199 | LR: 1.71e-04 | 1400.1ms/step | 14.3 samples/s +Step 26810/44000 | Loss: 2.9080 | LR: 1.71e-04 | 1400.1ms/step | 14.3 samples/s +Step 26820/44000 | Loss: 2.9098 | LR: 1.71e-04 | 1400.0ms/step | 14.3 samples/s +Step 26830/44000 | Loss: 2.8895 | LR: 1.71e-04 | 1399.8ms/step | 14.3 samples/s +Step 26840/44000 | Loss: 2.8899 | LR: 1.70e-04 | 1400.2ms/step | 14.3 samples/s +Step 26850/44000 | Loss: 2.9267 | LR: 1.70e-04 | 1400.3ms/step | 14.3 samples/s +Step 26860/44000 | Loss: 2.9393 | LR: 1.70e-04 | 1399.5ms/step | 14.3 samples/s +Step 26870/44000 | Loss: 2.9305 | LR: 1.70e-04 | 1400.0ms/step | 14.3 samples/s +Step 26880/44000 | Loss: 2.9028 | LR: 1.70e-04 | 1399.9ms/step | 14.3 samples/s +Step 26890/44000 | Loss: 2.9087 | LR: 1.69e-04 | 1400.1ms/step | 14.3 samples/s +Step 26900/44000 | Loss: 2.8965 | LR: 1.69e-04 | 1400.2ms/step | 14.3 samples/s +Step 26910/44000 | Loss: 2.9172 | LR: 1.69e-04 | 1399.7ms/step | 14.3 samples/s +Step 26920/44000 | Loss: 2.8914 | LR: 1.69e-04 | 1400.1ms/step | 14.3 samples/s +Step 26930/44000 | Loss: 2.8941 | LR: 1.69e-04 | 1399.9ms/step | 14.3 samples/s +Step 26940/44000 | Loss: 2.8866 | LR: 1.69e-04 | 1400.3ms/step | 14.3 samples/s +Step 26950/44000 | Loss: 2.8855 | LR: 1.68e-04 | 1399.7ms/step | 14.3 samples/s +Step 26960/44000 | Loss: 2.9147 | LR: 1.68e-04 | 1400.2ms/step | 14.3 samples/s +Step 26970/44000 | Loss: 2.9338 | LR: 1.68e-04 | 1399.7ms/step | 14.3 samples/s +Step 26980/44000 | Loss: 2.9002 | LR: 1.68e-04 | 1400.1ms/step | 14.3 samples/s +Step 26990/44000 | Loss: 2.9117 | LR: 1.68e-04 | 1400.2ms/step | 14.3 samples/s +Step 27000/44000 | Loss: 2.9229 | LR: 1.68e-04 | 1399.7ms/step | 14.3 samples/s +Validation at step 27000: Loss = 2.8973, Perplexity = 18.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-27000 +Step 27010/44000 | Loss: 2.9088 | LR: 1.67e-04 | 3902.2ms/step | 5.1 samples/s +Step 27020/44000 | Loss: 2.9313 | LR: 1.67e-04 | 1400.0ms/step | 14.3 samples/s +Step 27030/44000 | Loss: 2.9094 | LR: 1.67e-04 | 1397.6ms/step | 14.3 samples/s +Step 27040/44000 | Loss: 2.8860 | LR: 1.67e-04 | 1392.2ms/step | 14.4 samples/s +Step 27050/44000 | Loss: 2.8615 | LR: 1.67e-04 | 1391.9ms/step | 14.4 samples/s +Step 27060/44000 | Loss: 2.8882 | LR: 1.67e-04 | 1394.1ms/step | 14.3 samples/s +Step 27070/44000 | Loss: 2.9270 | LR: 1.66e-04 | 1390.9ms/step | 14.4 samples/s +Step 27080/44000 | Loss: 2.9141 | LR: 1.66e-04 | 1395.0ms/step | 14.3 samples/s +Step 27090/44000 | Loss: 2.9078 | LR: 1.66e-04 | 1393.8ms/step | 14.3 samples/s +Step 27100/44000 | Loss: 2.8685 | LR: 1.66e-04 | 1392.1ms/step | 14.4 samples/s +Step 27110/44000 | Loss: 2.9306 | LR: 1.66e-04 | 1392.1ms/step | 14.4 samples/s +Step 27120/44000 | Loss: 2.8948 | LR: 1.66e-04 | 1394.1ms/step | 14.3 samples/s +Step 27130/44000 | Loss: 2.8759 | LR: 1.65e-04 | 1393.7ms/step | 14.4 samples/s +Step 27140/44000 | Loss: 2.9127 | LR: 1.65e-04 | 1392.8ms/step | 14.4 samples/s +Step 27150/44000 | Loss: 2.8954 | LR: 1.65e-04 | 1392.4ms/step | 14.4 samples/s +Step 27160/44000 | Loss: 2.9324 | LR: 1.65e-04 | 1393.0ms/step | 14.4 samples/s +Step 27170/44000 | Loss: 2.9132 | LR: 1.65e-04 | 1392.1ms/step | 14.4 samples/s +Step 27180/44000 | Loss: 2.8796 | LR: 1.65e-04 | 1393.7ms/step | 14.3 samples/s +Step 27190/44000 | Loss: 2.9002 | LR: 1.64e-04 | 1394.3ms/step | 14.3 samples/s +Step 27200/44000 | Loss: 2.8917 | LR: 1.64e-04 | 1396.5ms/step | 14.3 samples/s +Step 27210/44000 | Loss: 2.9262 | LR: 1.64e-04 | 1395.5ms/step | 14.3 samples/s +Step 27220/44000 | Loss: 2.8913 | LR: 1.64e-04 | 1392.8ms/step | 14.4 samples/s +Step 27230/44000 | Loss: 2.9096 | LR: 1.64e-04 | 1393.0ms/step | 14.4 samples/s +Step 27240/44000 | Loss: 2.9301 | LR: 1.63e-04 | 1393.5ms/step | 14.4 samples/s +Step 27250/44000 | Loss: 2.9386 | LR: 1.63e-04 | 1394.4ms/step | 14.3 samples/s +Step 27260/44000 | Loss: 2.9335 | LR: 1.63e-04 | 1396.8ms/step | 14.3 samples/s +Step 27270/44000 | Loss: 2.9033 | LR: 1.63e-04 | 1393.4ms/step | 14.4 samples/s +Step 27280/44000 | Loss: 2.9168 | LR: 1.63e-04 | 1393.9ms/step | 14.3 samples/s +Step 27290/44000 | Loss: 2.9239 | LR: 1.63e-04 | 1392.6ms/step | 14.4 samples/s +Step 27300/44000 | Loss: 2.9183 | LR: 1.62e-04 | 1392.9ms/step | 14.4 samples/s +Step 27310/44000 | Loss: 2.9163 | LR: 1.62e-04 | 1393.2ms/step | 14.4 samples/s +Step 27320/44000 | Loss: 2.8929 | LR: 1.62e-04 | 1393.9ms/step | 14.3 samples/s +Step 27330/44000 | Loss: 2.8889 | LR: 1.62e-04 | 1391.2ms/step | 14.4 samples/s +Step 27340/44000 | Loss: 2.8775 | LR: 1.62e-04 | 1394.3ms/step | 14.3 samples/s +Step 27350/44000 | Loss: 2.8854 | LR: 1.62e-04 | 1392.1ms/step | 14.4 samples/s +Step 27360/44000 | Loss: 2.9289 | LR: 1.61e-04 | 1394.0ms/step | 14.3 samples/s +Step 27370/44000 | Loss: 2.8930 | LR: 1.61e-04 | 1390.0ms/step | 14.4 samples/s +Step 27380/44000 | Loss: 2.9243 | LR: 1.61e-04 | 1391.4ms/step | 14.4 samples/s +Step 27390/44000 | Loss: 2.8945 | LR: 1.61e-04 | 1393.8ms/step | 14.3 samples/s +Step 27400/44000 | Loss: 2.9158 | LR: 1.61e-04 | 1391.0ms/step | 14.4 samples/s +Step 27410/44000 | Loss: 2.9068 | LR: 1.61e-04 | 1393.9ms/step | 14.3 samples/s +Step 27420/44000 | Loss: 2.9396 | LR: 1.60e-04 | 1393.2ms/step | 14.4 samples/s +Step 27430/44000 | Loss: 2.9286 | LR: 1.60e-04 | 1394.5ms/step | 14.3 samples/s +Step 27440/44000 | Loss: 2.9014 | LR: 1.60e-04 | 1391.3ms/step | 14.4 samples/s +Step 27450/44000 | Loss: 2.8817 | LR: 1.60e-04 | 1397.0ms/step | 14.3 samples/s +Step 27460/44000 | Loss: 2.9149 | LR: 1.60e-04 | 1394.0ms/step | 14.3 samples/s +Step 27470/44000 | Loss: 2.9099 | LR: 1.60e-04 | 1395.8ms/step | 14.3 samples/s +Step 27480/44000 | Loss: 2.9198 | LR: 1.59e-04 | 1394.1ms/step | 14.3 samples/s +Step 27490/44000 | Loss: 2.8928 | LR: 1.59e-04 | 1395.8ms/step | 14.3 samples/s +Step 27500/44000 | Loss: 2.9095 | LR: 1.59e-04 | 1396.3ms/step | 14.3 samples/s +Step 27510/44000 | Loss: 2.9116 | LR: 1.59e-04 | 1392.3ms/step | 14.4 samples/s +Step 27520/44000 | Loss: 2.8780 | LR: 1.59e-04 | 1395.5ms/step | 14.3 samples/s +Step 27530/44000 | Loss: 2.8982 | LR: 1.59e-04 | 1392.2ms/step | 14.4 samples/s +Step 27540/44000 | Loss: 2.9155 | LR: 1.58e-04 | 1396.0ms/step | 14.3 samples/s +Step 27550/44000 | Loss: 2.9110 | LR: 1.58e-04 | 1394.0ms/step | 14.3 samples/s +Step 27560/44000 | Loss: 2.8834 | LR: 1.58e-04 | 1395.0ms/step | 14.3 samples/s +Step 27570/44000 | Loss: 2.9187 | LR: 1.58e-04 | 1393.7ms/step | 14.4 samples/s +Step 27580/44000 | Loss: 2.9167 | LR: 1.58e-04 | 1395.1ms/step | 14.3 samples/s +Step 27590/44000 | Loss: 2.9026 | LR: 1.58e-04 | 1393.8ms/step | 14.3 samples/s +Step 27600/44000 | Loss: 2.8998 | LR: 1.57e-04 | 1394.3ms/step | 14.3 samples/s +Step 27610/44000 | Loss: 2.8991 | LR: 1.57e-04 | 1394.9ms/step | 14.3 samples/s +Step 27620/44000 | Loss: 2.8936 | LR: 1.57e-04 | 1395.2ms/step | 14.3 samples/s +Step 27630/44000 | Loss: 2.9042 | LR: 1.57e-04 | 1392.1ms/step | 14.4 samples/s +Step 27640/44000 | Loss: 2.8957 | LR: 1.57e-04 | 1396.9ms/step | 14.3 samples/s +Step 27650/44000 | Loss: 2.9259 | LR: 1.57e-04 | 1394.0ms/step | 14.3 samples/s +Step 27660/44000 | Loss: 2.8982 | LR: 1.56e-04 | 1390.3ms/step | 14.4 samples/s +Step 27670/44000 | Loss: 2.9209 | LR: 1.56e-04 | 1389.3ms/step | 14.4 samples/s +Step 27680/44000 | Loss: 2.8969 | LR: 1.56e-04 | 1391.1ms/step | 14.4 samples/s +Step 27690/44000 | Loss: 2.8929 | LR: 1.56e-04 | 1391.2ms/step | 14.4 samples/s +Step 27700/44000 | Loss: 2.9340 | LR: 1.56e-04 | 1392.7ms/step | 14.4 samples/s +Step 27710/44000 | Loss: 2.9032 | LR: 1.56e-04 | 1390.0ms/step | 14.4 samples/s +Step 27720/44000 | Loss: 2.8885 | LR: 1.55e-04 | 1391.2ms/step | 14.4 samples/s +Step 27730/44000 | Loss: 2.9151 | LR: 1.55e-04 | 1386.5ms/step | 14.4 samples/s +Step 27740/44000 | Loss: 2.9176 | LR: 1.55e-04 | 1391.3ms/step | 14.4 samples/s +Step 27750/44000 | Loss: 2.9204 | LR: 1.55e-04 | 1393.4ms/step | 14.4 samples/s +Step 27760/44000 | Loss: 2.9306 | LR: 1.55e-04 | 1394.1ms/step | 14.3 samples/s +Step 27770/44000 | Loss: 2.9212 | LR: 1.55e-04 | 1393.6ms/step | 14.4 samples/s +Step 27780/44000 | Loss: 2.8861 | LR: 1.54e-04 | 1394.4ms/step | 14.3 samples/s +Step 27790/44000 | Loss: 2.9389 | LR: 1.54e-04 | 1391.7ms/step | 14.4 samples/s +Step 27800/44000 | Loss: 2.9195 | LR: 1.54e-04 | 1394.4ms/step | 14.3 samples/s +Step 27810/44000 | Loss: 2.9107 | LR: 1.54e-04 | 1393.1ms/step | 14.4 samples/s +Step 27820/44000 | Loss: 2.9054 | LR: 1.54e-04 | 1393.0ms/step | 14.4 samples/s +Step 27830/44000 | Loss: 2.8957 | LR: 1.54e-04 | 1395.6ms/step | 14.3 samples/s +Step 27840/44000 | Loss: 2.8887 | LR: 1.53e-04 | 1393.9ms/step | 14.3 samples/s +Step 27850/44000 | Loss: 2.8517 | LR: 1.53e-04 | 1393.5ms/step | 14.4 samples/s +Step 27860/44000 | Loss: 2.9005 | LR: 1.53e-04 | 1392.7ms/step | 14.4 samples/s +Step 27870/44000 | Loss: 2.9101 | LR: 1.53e-04 | 1398.2ms/step | 14.3 samples/s +Step 27880/44000 | Loss: 2.9139 | LR: 1.53e-04 | 1395.7ms/step | 14.3 samples/s +Step 27890/44000 | Loss: 2.8941 | LR: 1.53e-04 | 1396.2ms/step | 14.3 samples/s +Step 27900/44000 | Loss: 2.9008 | LR: 1.52e-04 | 1394.1ms/step | 14.3 samples/s +Step 27910/44000 | Loss: 2.8998 | LR: 1.52e-04 | 1393.3ms/step | 14.4 samples/s +Step 27920/44000 | Loss: 2.8975 | LR: 1.52e-04 | 1392.4ms/step | 14.4 samples/s +Step 27930/44000 | Loss: 2.8915 | LR: 1.52e-04 | 1393.1ms/step | 14.4 samples/s +Step 27940/44000 | Loss: 2.9166 | LR: 1.52e-04 | 1391.0ms/step | 14.4 samples/s +Step 27950/44000 | Loss: 2.8593 | LR: 1.52e-04 | 1391.9ms/step | 14.4 samples/s +Step 27960/44000 | Loss: 2.9018 | LR: 1.51e-04 | 1395.3ms/step | 14.3 samples/s +Step 27970/44000 | Loss: 2.9039 | LR: 1.51e-04 | 1390.6ms/step | 14.4 samples/s +Step 27980/44000 | Loss: 2.9093 | LR: 1.51e-04 | 1394.3ms/step | 14.3 samples/s +Step 27990/44000 | Loss: 2.9036 | LR: 1.51e-04 | 1392.2ms/step | 14.4 samples/s +Step 28000/44000 | Loss: 2.8568 | LR: 1.51e-04 | 1391.9ms/step | 14.4 samples/s +Validation at step 28000: Loss = 2.8920, Perplexity = 18.03 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-28000 +Step 28010/44000 | Loss: 2.8964 | LR: 1.51e-04 | 3890.2ms/step | 5.1 samples/s +Step 28020/44000 | Loss: 2.8907 | LR: 1.50e-04 | 1399.9ms/step | 14.3 samples/s +Step 28030/44000 | Loss: 2.9120 | LR: 1.50e-04 | 1399.5ms/step | 14.3 samples/s +Step 28040/44000 | Loss: 2.8961 | LR: 1.50e-04 | 1393.6ms/step | 14.4 samples/s +Step 28050/44000 | Loss: 2.9189 | LR: 1.50e-04 | 1396.5ms/step | 14.3 samples/s +Step 28060/44000 | Loss: 2.9027 | LR: 1.50e-04 | 1396.1ms/step | 14.3 samples/s +Step 28070/44000 | Loss: 2.8899 | LR: 1.50e-04 | 1398.0ms/step | 14.3 samples/s +Step 28080/44000 | Loss: 2.9272 | LR: 1.49e-04 | 1391.9ms/step | 14.4 samples/s +Step 28090/44000 | Loss: 2.9019 | LR: 1.49e-04 | 1393.9ms/step | 14.3 samples/s +Step 28100/44000 | Loss: 2.9083 | LR: 1.49e-04 | 1394.4ms/step | 14.3 samples/s +Step 28110/44000 | Loss: 2.8954 | LR: 1.49e-04 | 1397.4ms/step | 14.3 samples/s +Step 28120/44000 | Loss: 2.8963 | LR: 1.49e-04 | 1393.7ms/step | 14.4 samples/s +Step 28130/44000 | Loss: 2.8885 | LR: 1.49e-04 | 1394.8ms/step | 14.3 samples/s +Step 28140/44000 | Loss: 2.9102 | LR: 1.48e-04 | 1391.9ms/step | 14.4 samples/s +Step 28150/44000 | Loss: 2.8712 | LR: 1.48e-04 | 1392.0ms/step | 14.4 samples/s +Step 28160/44000 | Loss: 2.8809 | LR: 1.48e-04 | 1398.2ms/step | 14.3 samples/s +Step 28170/44000 | Loss: 2.8765 | LR: 1.48e-04 | 1393.9ms/step | 14.3 samples/s +Step 28180/44000 | Loss: 2.9140 | LR: 1.48e-04 | 1393.0ms/step | 14.4 samples/s +Step 28190/44000 | Loss: 2.9029 | LR: 1.48e-04 | 1392.8ms/step | 14.4 samples/s +Step 28200/44000 | Loss: 2.8835 | LR: 1.47e-04 | 1393.9ms/step | 14.3 samples/s +Step 28210/44000 | Loss: 2.9246 | LR: 1.47e-04 | 1394.3ms/step | 14.3 samples/s +Step 28220/44000 | Loss: 2.8943 | LR: 1.47e-04 | 1400.0ms/step | 14.3 samples/s +Step 28230/44000 | Loss: 2.9517 | LR: 1.47e-04 | 1399.9ms/step | 14.3 samples/s +Step 28240/44000 | Loss: 2.8656 | LR: 1.47e-04 | 1399.9ms/step | 14.3 samples/s +Step 28250/44000 | Loss: 2.8849 | LR: 1.47e-04 | 1400.1ms/step | 14.3 samples/s +Step 28260/44000 | Loss: 2.8975 | LR: 1.46e-04 | 1399.9ms/step | 14.3 samples/s +Step 28270/44000 | Loss: 2.9123 | LR: 1.46e-04 | 1400.1ms/step | 14.3 samples/s +Step 28280/44000 | Loss: 2.8840 | LR: 1.46e-04 | 1399.9ms/step | 14.3 samples/s +Step 28290/44000 | Loss: 2.9201 | LR: 1.46e-04 | 1400.1ms/step | 14.3 samples/s +Step 28300/44000 | Loss: 2.9107 | LR: 1.46e-04 | 1400.1ms/step | 14.3 samples/s +Step 28310/44000 | Loss: 2.9150 | LR: 1.46e-04 | 1399.8ms/step | 14.3 samples/s +Step 28320/44000 | Loss: 2.8790 | LR: 1.45e-04 | 1400.1ms/step | 14.3 samples/s +Step 28330/44000 | Loss: 2.8898 | LR: 1.45e-04 | 1399.7ms/step | 14.3 samples/s +Step 28340/44000 | Loss: 2.8973 | LR: 1.45e-04 | 1399.6ms/step | 14.3 samples/s +Step 28350/44000 | Loss: 2.8690 | LR: 1.45e-04 | 1392.3ms/step | 14.4 samples/s +Step 28360/44000 | Loss: 2.8822 | LR: 1.45e-04 | 1391.2ms/step | 14.4 samples/s +Step 28370/44000 | Loss: 2.8946 | LR: 1.45e-04 | 1394.6ms/step | 14.3 samples/s +Step 28380/44000 | Loss: 2.9037 | LR: 1.44e-04 | 1396.1ms/step | 14.3 samples/s +Step 28390/44000 | Loss: 2.9104 | LR: 1.44e-04 | 1396.3ms/step | 14.3 samples/s +Step 28400/44000 | Loss: 2.9246 | LR: 1.44e-04 | 1391.8ms/step | 14.4 samples/s +Step 28410/44000 | Loss: 2.9293 | LR: 1.44e-04 | 1392.3ms/step | 14.4 samples/s +Step 28420/44000 | Loss: 2.8610 | LR: 1.44e-04 | 1393.6ms/step | 14.4 samples/s +Step 28430/44000 | Loss: 2.8721 | LR: 1.44e-04 | 1395.5ms/step | 14.3 samples/s +Step 28440/44000 | Loss: 2.9097 | LR: 1.43e-04 | 1390.7ms/step | 14.4 samples/s +Step 28450/44000 | Loss: 2.9093 | LR: 1.43e-04 | 1392.6ms/step | 14.4 samples/s +Step 28460/44000 | Loss: 2.9040 | LR: 1.43e-04 | 1393.1ms/step | 14.4 samples/s +Step 28470/44000 | Loss: 2.8994 | LR: 1.43e-04 | 1394.3ms/step | 14.3 samples/s +Step 28480/44000 | Loss: 2.9022 | LR: 1.43e-04 | 1393.1ms/step | 14.4 samples/s +Step 28490/44000 | Loss: 2.8905 | LR: 1.43e-04 | 1392.8ms/step | 14.4 samples/s +Step 28500/44000 | Loss: 2.9020 | LR: 1.42e-04 | 1398.2ms/step | 14.3 samples/s +Step 28510/44000 | Loss: 2.9102 | LR: 1.42e-04 | 1399.8ms/step | 14.3 samples/s +Step 28520/44000 | Loss: 2.8906 | LR: 1.42e-04 | 1400.3ms/step | 14.3 samples/s +Step 28530/44000 | Loss: 2.8739 | LR: 1.42e-04 | 1399.2ms/step | 14.3 samples/s +Step 28540/44000 | Loss: 2.9093 | LR: 1.42e-04 | 1392.6ms/step | 14.4 samples/s +Step 28550/44000 | Loss: 2.9042 | LR: 1.42e-04 | 1398.0ms/step | 14.3 samples/s +Step 28560/44000 | Loss: 2.8932 | LR: 1.41e-04 | 1393.0ms/step | 14.4 samples/s +Step 28570/44000 | Loss: 2.8821 | LR: 1.41e-04 | 1390.9ms/step | 14.4 samples/s +Step 28580/44000 | Loss: 2.9083 | LR: 1.41e-04 | 1394.1ms/step | 14.3 samples/s +Step 28590/44000 | Loss: 2.9098 | LR: 1.41e-04 | 1392.2ms/step | 14.4 samples/s +Step 28600/44000 | Loss: 2.9062 | LR: 1.41e-04 | 1397.2ms/step | 14.3 samples/s +Step 28610/44000 | Loss: 2.9148 | LR: 1.41e-04 | 1395.7ms/step | 14.3 samples/s +Step 28620/44000 | Loss: 2.9168 | LR: 1.40e-04 | 1393.1ms/step | 14.4 samples/s +Step 28630/44000 | Loss: 2.8489 | LR: 1.40e-04 | 1391.5ms/step | 14.4 samples/s +Step 28640/44000 | Loss: 2.8856 | LR: 1.40e-04 | 1391.8ms/step | 14.4 samples/s +Step 28650/44000 | Loss: 2.9125 | LR: 1.40e-04 | 1394.3ms/step | 14.3 samples/s +Step 28660/44000 | Loss: 2.8960 | LR: 1.40e-04 | 1392.3ms/step | 14.4 samples/s +Step 28670/44000 | Loss: 2.9034 | LR: 1.40e-04 | 1395.1ms/step | 14.3 samples/s +Step 28680/44000 | Loss: 2.8959 | LR: 1.39e-04 | 1394.8ms/step | 14.3 samples/s +Step 28690/44000 | Loss: 2.9019 | LR: 1.39e-04 | 1395.8ms/step | 14.3 samples/s +Step 28700/44000 | Loss: 2.8724 | LR: 1.39e-04 | 1396.2ms/step | 14.3 samples/s +Step 28710/44000 | Loss: 2.9219 | LR: 1.39e-04 | 1393.8ms/step | 14.3 samples/s +Step 28720/44000 | Loss: 2.9249 | LR: 1.39e-04 | 1395.0ms/step | 14.3 samples/s +Step 28730/44000 | Loss: 2.8958 | LR: 1.39e-04 | 1392.7ms/step | 14.4 samples/s +Step 28740/44000 | Loss: 2.8938 | LR: 1.38e-04 | 1390.3ms/step | 14.4 samples/s +Step 28750/44000 | Loss: 2.9146 | LR: 1.38e-04 | 1391.9ms/step | 14.4 samples/s +Step 28760/44000 | Loss: 2.9157 | LR: 1.38e-04 | 1390.6ms/step | 14.4 samples/s +Step 28770/44000 | Loss: 2.9036 | LR: 1.38e-04 | 1391.2ms/step | 14.4 samples/s +Step 28780/44000 | Loss: 2.9101 | LR: 1.38e-04 | 1393.3ms/step | 14.4 samples/s +Step 28790/44000 | Loss: 2.8964 | LR: 1.38e-04 | 1393.0ms/step | 14.4 samples/s +Step 28800/44000 | Loss: 2.8887 | LR: 1.38e-04 | 1391.8ms/step | 14.4 samples/s +Step 28810/44000 | Loss: 2.8891 | LR: 1.37e-04 | 1398.6ms/step | 14.3 samples/s +Step 28820/44000 | Loss: 2.8753 | LR: 1.37e-04 | 1397.4ms/step | 14.3 samples/s +Step 28830/44000 | Loss: 2.8975 | LR: 1.37e-04 | 1396.3ms/step | 14.3 samples/s +Step 28840/44000 | Loss: 2.8867 | LR: 1.37e-04 | 1397.9ms/step | 14.3 samples/s +Step 28850/44000 | Loss: 2.8871 | LR: 1.37e-04 | 1394.1ms/step | 14.3 samples/s +Step 28860/44000 | Loss: 2.8845 | LR: 1.37e-04 | 1393.8ms/step | 14.3 samples/s +Step 28870/44000 | Loss: 2.9021 | LR: 1.36e-04 | 1392.9ms/step | 14.4 samples/s +Step 28880/44000 | Loss: 2.8793 | LR: 1.36e-04 | 1399.3ms/step | 14.3 samples/s +Step 28890/44000 | Loss: 2.9032 | LR: 1.36e-04 | 1400.0ms/step | 14.3 samples/s +Step 28900/44000 | Loss: 2.9405 | LR: 1.36e-04 | 1393.0ms/step | 14.4 samples/s +Step 28910/44000 | Loss: 2.8807 | LR: 1.36e-04 | 1394.9ms/step | 14.3 samples/s +Step 28920/44000 | Loss: 2.9197 | LR: 1.36e-04 | 1394.0ms/step | 14.3 samples/s +Step 28930/44000 | Loss: 2.8715 | LR: 1.35e-04 | 1397.1ms/step | 14.3 samples/s +Step 28940/44000 | Loss: 2.8843 | LR: 1.35e-04 | 1394.5ms/step | 14.3 samples/s +Step 28950/44000 | Loss: 2.8929 | LR: 1.35e-04 | 1396.4ms/step | 14.3 samples/s +Step 28960/44000 | Loss: 2.8974 | LR: 1.35e-04 | 1394.0ms/step | 14.3 samples/s +Step 28970/44000 | Loss: 2.8990 | LR: 1.35e-04 | 1395.6ms/step | 14.3 samples/s +Step 28980/44000 | Loss: 2.9234 | LR: 1.35e-04 | 1394.3ms/step | 14.3 samples/s +Step 28990/44000 | Loss: 2.8734 | LR: 1.34e-04 | 1391.2ms/step | 14.4 samples/s +Step 29000/44000 | Loss: 2.8733 | LR: 1.34e-04 | 1392.9ms/step | 14.4 samples/s +Validation at step 29000: Loss = 2.8876, Perplexity = 17.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-29000 +Step 29010/44000 | Loss: 2.8928 | LR: 1.34e-04 | 3922.1ms/step | 5.1 samples/s +Step 29020/44000 | Loss: 2.9074 | LR: 1.34e-04 | 1400.0ms/step | 14.3 samples/s +Step 29030/44000 | Loss: 2.8787 | LR: 1.34e-04 | 1400.1ms/step | 14.3 samples/s +Step 29040/44000 | Loss: 2.9095 | LR: 1.34e-04 | 1400.1ms/step | 14.3 samples/s +Step 29050/44000 | Loss: 2.8860 | LR: 1.33e-04 | 1400.0ms/step | 14.3 samples/s +Step 29060/44000 | Loss: 2.8955 | LR: 1.33e-04 | 1399.8ms/step | 14.3 samples/s +Step 29070/44000 | Loss: 2.8468 | LR: 1.33e-04 | 1400.1ms/step | 14.3 samples/s +Step 29080/44000 | Loss: 2.9046 | LR: 1.33e-04 | 1400.0ms/step | 14.3 samples/s +Step 29090/44000 | Loss: 2.9302 | LR: 1.33e-04 | 1400.1ms/step | 14.3 samples/s +Step 29100/44000 | Loss: 2.9027 | LR: 1.33e-04 | 1400.0ms/step | 14.3 samples/s +Step 29110/44000 | Loss: 2.9099 | LR: 1.33e-04 | 1399.6ms/step | 14.3 samples/s +Step 29120/44000 | Loss: 2.8899 | LR: 1.32e-04 | 1400.3ms/step | 14.3 samples/s +Step 29130/44000 | Loss: 2.9071 | LR: 1.32e-04 | 1400.0ms/step | 14.3 samples/s +Step 29140/44000 | Loss: 2.9059 | LR: 1.32e-04 | 1399.9ms/step | 14.3 samples/s +Step 29150/44000 | Loss: 2.8758 | LR: 1.32e-04 | 1400.1ms/step | 14.3 samples/s +Step 29160/44000 | Loss: 2.9015 | LR: 1.32e-04 | 1400.0ms/step | 14.3 samples/s +Step 29170/44000 | Loss: 2.8987 | LR: 1.32e-04 | 1400.1ms/step | 14.3 samples/s +Step 29180/44000 | Loss: 2.8702 | LR: 1.31e-04 | 1399.8ms/step | 14.3 samples/s +Step 29190/44000 | Loss: 2.8890 | LR: 1.31e-04 | 1400.0ms/step | 14.3 samples/s +Step 29200/44000 | Loss: 2.9307 | LR: 1.31e-04 | 1400.1ms/step | 14.3 samples/s +Step 29210/44000 | Loss: 2.9160 | LR: 1.31e-04 | 1399.8ms/step | 14.3 samples/s +Step 29220/44000 | Loss: 2.9101 | LR: 1.31e-04 | 1400.2ms/step | 14.3 samples/s +Step 29230/44000 | Loss: 2.8789 | LR: 1.31e-04 | 1399.9ms/step | 14.3 samples/s +Step 29240/44000 | Loss: 2.8991 | LR: 1.30e-04 | 1400.2ms/step | 14.3 samples/s +Step 29250/44000 | Loss: 2.9010 | LR: 1.30e-04 | 1399.8ms/step | 14.3 samples/s +Step 29260/44000 | Loss: 2.9006 | LR: 1.30e-04 | 1400.0ms/step | 14.3 samples/s +Step 29270/44000 | Loss: 2.9078 | LR: 1.30e-04 | 1400.1ms/step | 14.3 samples/s +Step 29280/44000 | Loss: 2.8918 | LR: 1.30e-04 | 1399.9ms/step | 14.3 samples/s +Step 29290/44000 | Loss: 2.8922 | LR: 1.30e-04 | 1400.1ms/step | 14.3 samples/s +Step 29300/44000 | Loss: 2.8721 | LR: 1.29e-04 | 1399.8ms/step | 14.3 samples/s +Step 29310/44000 | Loss: 2.9153 | LR: 1.29e-04 | 1400.1ms/step | 14.3 samples/s +Step 29320/44000 | Loss: 2.9129 | LR: 1.29e-04 | 1400.1ms/step | 14.3 samples/s +Step 29330/44000 | Loss: 2.8948 | LR: 1.29e-04 | 1400.1ms/step | 14.3 samples/s +Step 29340/44000 | Loss: 2.8709 | LR: 1.29e-04 | 1400.1ms/step | 14.3 samples/s +Step 29350/44000 | Loss: 2.8978 | LR: 1.29e-04 | 1399.9ms/step | 14.3 samples/s +Step 29360/44000 | Loss: 2.9283 | LR: 1.29e-04 | 1399.8ms/step | 14.3 samples/s +Step 29370/44000 | Loss: 2.9184 | LR: 1.28e-04 | 1400.2ms/step | 14.3 samples/s +Step 29380/44000 | Loss: 2.8922 | LR: 1.28e-04 | 1400.0ms/step | 14.3 samples/s +Step 29390/44000 | Loss: 2.9190 | LR: 1.28e-04 | 1399.9ms/step | 14.3 samples/s +Step 29400/44000 | Loss: 2.9011 | LR: 1.28e-04 | 1400.2ms/step | 14.3 samples/s +Step 29410/44000 | Loss: 2.8917 | LR: 1.28e-04 | 1399.9ms/step | 14.3 samples/s +Step 29420/44000 | Loss: 2.8841 | LR: 1.28e-04 | 1399.9ms/step | 14.3 samples/s +Step 29430/44000 | Loss: 2.9029 | LR: 1.27e-04 | 1400.2ms/step | 14.3 samples/s +Step 29440/44000 | Loss: 2.8901 | LR: 1.27e-04 | 1400.0ms/step | 14.3 samples/s +Step 29450/44000 | Loss: 2.8919 | LR: 1.27e-04 | 1400.0ms/step | 14.3 samples/s +Step 29460/44000 | Loss: 2.8980 | LR: 1.27e-04 | 1399.9ms/step | 14.3 samples/s +Step 29470/44000 | Loss: 2.9180 | LR: 1.27e-04 | 1400.1ms/step | 14.3 samples/s +Step 29480/44000 | Loss: 2.8959 | LR: 1.27e-04 | 1400.1ms/step | 14.3 samples/s +Step 29490/44000 | Loss: 2.8982 | LR: 1.26e-04 | 1400.1ms/step | 14.3 samples/s +Step 29500/44000 | Loss: 2.8814 | LR: 1.26e-04 | 1399.9ms/step | 14.3 samples/s +Step 29510/44000 | Loss: 2.8983 | LR: 1.26e-04 | 1400.0ms/step | 14.3 samples/s +Step 29520/44000 | Loss: 2.8841 | LR: 1.26e-04 | 1399.9ms/step | 14.3 samples/s +Step 29530/44000 | Loss: 2.9039 | LR: 1.26e-04 | 1400.0ms/step | 14.3 samples/s +Step 29540/44000 | Loss: 2.8878 | LR: 1.26e-04 | 1400.1ms/step | 14.3 samples/s +Step 29550/44000 | Loss: 2.8855 | LR: 1.26e-04 | 1399.8ms/step | 14.3 samples/s +Step 29560/44000 | Loss: 2.9060 | LR: 1.25e-04 | 1400.1ms/step | 14.3 samples/s +Step 29570/44000 | Loss: 2.9089 | LR: 1.25e-04 | 1399.8ms/step | 14.3 samples/s +Step 29580/44000 | Loss: 2.9023 | LR: 1.25e-04 | 1400.2ms/step | 14.3 samples/s +Step 29590/44000 | Loss: 2.8682 | LR: 1.25e-04 | 1400.1ms/step | 14.3 samples/s +Step 29600/44000 | Loss: 2.8910 | LR: 1.25e-04 | 1400.0ms/step | 14.3 samples/s +Step 29610/44000 | Loss: 2.8877 | LR: 1.25e-04 | 1399.6ms/step | 14.3 samples/s +Step 29620/44000 | Loss: 2.8826 | LR: 1.24e-04 | 1400.4ms/step | 14.3 samples/s +Step 29630/44000 | Loss: 2.9149 | LR: 1.24e-04 | 1399.9ms/step | 14.3 samples/s +Step 29640/44000 | Loss: 2.9200 | LR: 1.24e-04 | 1400.0ms/step | 14.3 samples/s +Step 29650/44000 | Loss: 2.9195 | LR: 1.24e-04 | 1400.0ms/step | 14.3 samples/s +Step 29660/44000 | Loss: 2.8966 | LR: 1.24e-04 | 1399.9ms/step | 14.3 samples/s +Step 29670/44000 | Loss: 2.8561 | LR: 1.24e-04 | 1400.1ms/step | 14.3 samples/s +Step 29680/44000 | Loss: 2.8756 | LR: 1.23e-04 | 1399.9ms/step | 14.3 samples/s +Step 29690/44000 | Loss: 2.8883 | LR: 1.23e-04 | 1399.9ms/step | 14.3 samples/s +Step 29700/44000 | Loss: 2.9256 | LR: 1.23e-04 | 1400.1ms/step | 14.3 samples/s +Step 29710/44000 | Loss: 2.8746 | LR: 1.23e-04 | 1400.1ms/step | 14.3 samples/s +Step 29720/44000 | Loss: 2.8899 | LR: 1.23e-04 | 1400.0ms/step | 14.3 samples/s +Step 29730/44000 | Loss: 2.8801 | LR: 1.23e-04 | 1400.1ms/step | 14.3 samples/s +Step 29740/44000 | Loss: 2.8937 | LR: 1.23e-04 | 1399.9ms/step | 14.3 samples/s +Step 29750/44000 | Loss: 2.8549 | LR: 1.22e-04 | 1400.1ms/step | 14.3 samples/s +Step 29760/44000 | Loss: 2.8631 | LR: 1.22e-04 | 1399.8ms/step | 14.3 samples/s +Step 29770/44000 | Loss: 2.8738 | LR: 1.22e-04 | 1400.2ms/step | 14.3 samples/s +Step 29780/44000 | Loss: 2.8463 | LR: 1.22e-04 | 1400.0ms/step | 14.3 samples/s +Step 29790/44000 | Loss: 2.8709 | LR: 1.22e-04 | 1400.1ms/step | 14.3 samples/s +Step 29800/44000 | Loss: 2.9117 | LR: 1.22e-04 | 1400.0ms/step | 14.3 samples/s +Step 29810/44000 | Loss: 2.8927 | LR: 1.21e-04 | 1399.6ms/step | 14.3 samples/s +Step 29820/44000 | Loss: 2.8951 | LR: 1.21e-04 | 1400.2ms/step | 14.3 samples/s +Step 29830/44000 | Loss: 2.9402 | LR: 1.21e-04 | 1400.0ms/step | 14.3 samples/s +Step 29840/44000 | Loss: 2.8904 | LR: 1.21e-04 | 1399.8ms/step | 14.3 samples/s +Step 29850/44000 | Loss: 2.8806 | LR: 1.21e-04 | 1400.3ms/step | 14.3 samples/s +Step 29860/44000 | Loss: 2.8999 | LR: 1.21e-04 | 1399.9ms/step | 14.3 samples/s +Step 29870/44000 | Loss: 2.9064 | LR: 1.21e-04 | 1400.0ms/step | 14.3 samples/s +Step 29880/44000 | Loss: 2.8895 | LR: 1.20e-04 | 1400.1ms/step | 14.3 samples/s +Step 29890/44000 | Loss: 2.8813 | LR: 1.20e-04 | 1400.1ms/step | 14.3 samples/s +Step 29900/44000 | Loss: 2.8885 | LR: 1.20e-04 | 1399.8ms/step | 14.3 samples/s +Step 29910/44000 | Loss: 2.9151 | LR: 1.20e-04 | 1400.1ms/step | 14.3 samples/s +Step 29920/44000 | Loss: 2.9068 | LR: 1.20e-04 | 1399.9ms/step | 14.3 samples/s +Step 29930/44000 | Loss: 2.8995 | LR: 1.20e-04 | 1400.1ms/step | 14.3 samples/s +Step 29940/44000 | Loss: 2.8321 | LR: 1.19e-04 | 1399.7ms/step | 14.3 samples/s +Step 29950/44000 | Loss: 2.8965 | LR: 1.19e-04 | 1400.1ms/step | 14.3 samples/s +Step 29960/44000 | Loss: 2.8851 | LR: 1.19e-04 | 1400.2ms/step | 14.3 samples/s +Step 29970/44000 | Loss: 2.9319 | LR: 1.19e-04 | 1399.8ms/step | 14.3 samples/s +Step 29980/44000 | Loss: 2.9085 | LR: 1.19e-04 | 1400.1ms/step | 14.3 samples/s +Step 29990/44000 | Loss: 2.8802 | LR: 1.19e-04 | 1400.0ms/step | 14.3 samples/s +Step 30000/44000 | Loss: 2.8602 | LR: 1.19e-04 | 1400.0ms/step | 14.3 samples/s +Validation at step 30000: Loss = 2.8841, Perplexity = 17.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-30000 +Step 30010/44000 | Loss: 2.8751 | LR: 1.18e-04 | 3913.8ms/step | 5.1 samples/s +Step 30020/44000 | Loss: 2.8919 | LR: 1.18e-04 | 1400.1ms/step | 14.3 samples/s +Step 30030/44000 | Loss: 2.9028 | LR: 1.18e-04 | 1399.9ms/step | 14.3 samples/s +Step 30040/44000 | Loss: 2.8837 | LR: 1.18e-04 | 1400.0ms/step | 14.3 samples/s +Step 30050/44000 | Loss: 2.8979 | LR: 1.18e-04 | 1400.0ms/step | 14.3 samples/s +Step 30060/44000 | Loss: 2.9145 | LR: 1.18e-04 | 1400.0ms/step | 14.3 samples/s +Step 30070/44000 | Loss: 2.8924 | LR: 1.17e-04 | 1399.9ms/step | 14.3 samples/s +Step 30080/44000 | Loss: 2.9071 | LR: 1.17e-04 | 1400.1ms/step | 14.3 samples/s +Step 30090/44000 | Loss: 2.9081 | LR: 1.17e-04 | 1400.0ms/step | 14.3 samples/s +Step 30100/44000 | Loss: 2.8980 | LR: 1.17e-04 | 1399.9ms/step | 14.3 samples/s +Step 30110/44000 | Loss: 2.8598 | LR: 1.17e-04 | 1399.9ms/step | 14.3 samples/s +Step 30120/44000 | Loss: 2.9091 | LR: 1.17e-04 | 1400.2ms/step | 14.3 samples/s +Step 30130/44000 | Loss: 2.8678 | LR: 1.17e-04 | 1400.0ms/step | 14.3 samples/s +Step 30140/44000 | Loss: 2.9069 | LR: 1.16e-04 | 1400.0ms/step | 14.3 samples/s +Step 30150/44000 | Loss: 2.9150 | LR: 1.16e-04 | 1399.9ms/step | 14.3 samples/s +Step 30160/44000 | Loss: 2.8818 | LR: 1.16e-04 | 1400.1ms/step | 14.3 samples/s +Step 30170/44000 | Loss: 2.8934 | LR: 1.16e-04 | 1400.0ms/step | 14.3 samples/s +Step 30180/44000 | Loss: 2.8963 | LR: 1.16e-04 | 1400.0ms/step | 14.3 samples/s +Step 30190/44000 | Loss: 2.9089 | LR: 1.16e-04 | 1399.8ms/step | 14.3 samples/s +Step 30200/44000 | Loss: 2.8722 | LR: 1.15e-04 | 1400.1ms/step | 14.3 samples/s +Step 30210/44000 | Loss: 2.8850 | LR: 1.15e-04 | 1399.9ms/step | 14.3 samples/s +Step 30220/44000 | Loss: 2.9172 | LR: 1.15e-04 | 1400.1ms/step | 14.3 samples/s +Step 30230/44000 | Loss: 2.9045 | LR: 1.15e-04 | 1400.0ms/step | 14.3 samples/s +Step 30240/44000 | Loss: 2.9025 | LR: 1.15e-04 | 1400.1ms/step | 14.3 samples/s +Step 30250/44000 | Loss: 2.8844 | LR: 1.15e-04 | 1399.7ms/step | 14.3 samples/s +Step 30260/44000 | Loss: 2.8806 | LR: 1.15e-04 | 1400.1ms/step | 14.3 samples/s +Step 30270/44000 | Loss: 2.8748 | LR: 1.14e-04 | 1399.9ms/step | 14.3 samples/s +Step 30280/44000 | Loss: 2.8907 | LR: 1.14e-04 | 1399.9ms/step | 14.3 samples/s +Step 30290/44000 | Loss: 2.8754 | LR: 1.14e-04 | 1400.1ms/step | 14.3 samples/s +Step 30300/44000 | Loss: 2.8862 | LR: 1.14e-04 | 1400.1ms/step | 14.3 samples/s +Step 30310/44000 | Loss: 2.9014 | LR: 1.14e-04 | 1399.9ms/step | 14.3 samples/s +Step 30320/44000 | Loss: 2.8925 | LR: 1.14e-04 | 1400.0ms/step | 14.3 samples/s +Step 30330/44000 | Loss: 2.8816 | LR: 1.13e-04 | 1400.1ms/step | 14.3 samples/s +Step 30340/44000 | Loss: 2.8867 | LR: 1.13e-04 | 1400.1ms/step | 14.3 samples/s +Step 30350/44000 | Loss: 2.8927 | LR: 1.13e-04 | 1400.2ms/step | 14.3 samples/s +Step 30360/44000 | Loss: 2.9089 | LR: 1.13e-04 | 1399.8ms/step | 14.3 samples/s +Step 30370/44000 | Loss: 2.8802 | LR: 1.13e-04 | 1400.0ms/step | 14.3 samples/s +Step 30380/44000 | Loss: 2.9032 | LR: 1.13e-04 | 1400.0ms/step | 14.3 samples/s +Step 30390/44000 | Loss: 2.8608 | LR: 1.13e-04 | 1399.9ms/step | 14.3 samples/s +Step 30400/44000 | Loss: 2.9142 | LR: 1.12e-04 | 1400.0ms/step | 14.3 samples/s +Step 30410/44000 | Loss: 2.8906 | LR: 1.12e-04 | 1399.8ms/step | 14.3 samples/s +Step 30420/44000 | Loss: 2.8810 | LR: 1.12e-04 | 1400.1ms/step | 14.3 samples/s +Step 30430/44000 | Loss: 2.8603 | LR: 1.12e-04 | 1400.0ms/step | 14.3 samples/s +Step 30440/44000 | Loss: 2.9101 | LR: 1.12e-04 | 1400.0ms/step | 14.3 samples/s +Step 30450/44000 | Loss: 2.9078 | LR: 1.12e-04 | 1400.1ms/step | 14.3 samples/s +Step 30460/44000 | Loss: 2.8914 | LR: 1.11e-04 | 1399.9ms/step | 14.3 samples/s +Step 30470/44000 | Loss: 2.8522 | LR: 1.11e-04 | 1400.2ms/step | 14.3 samples/s +Step 30480/44000 | Loss: 2.9110 | LR: 1.11e-04 | 1399.7ms/step | 14.3 samples/s +Step 30490/44000 | Loss: 2.9085 | LR: 1.11e-04 | 1400.3ms/step | 14.3 samples/s +Step 30500/44000 | Loss: 2.8636 | LR: 1.11e-04 | 1400.1ms/step | 14.3 samples/s +Step 30510/44000 | Loss: 2.8872 | LR: 1.11e-04 | 1399.9ms/step | 14.3 samples/s +Step 30520/44000 | Loss: 2.9008 | LR: 1.11e-04 | 1400.2ms/step | 14.3 samples/s +Step 30530/44000 | Loss: 2.8858 | LR: 1.10e-04 | 1399.7ms/step | 14.3 samples/s +Step 30540/44000 | Loss: 2.8519 | LR: 1.10e-04 | 1400.2ms/step | 14.3 samples/s +Step 30550/44000 | Loss: 2.9079 | LR: 1.10e-04 | 1399.7ms/step | 14.3 samples/s +Step 30560/44000 | Loss: 2.8997 | LR: 1.10e-04 | 1400.3ms/step | 14.3 samples/s +Step 30570/44000 | Loss: 2.9044 | LR: 1.10e-04 | 1400.1ms/step | 14.3 samples/s +Step 30580/44000 | Loss: 2.8783 | LR: 1.10e-04 | 1399.8ms/step | 14.3 samples/s +Step 30590/44000 | Loss: 2.9034 | LR: 1.10e-04 | 1400.0ms/step | 14.3 samples/s +Step 30600/44000 | Loss: 2.9061 | LR: 1.09e-04 | 1400.2ms/step | 14.3 samples/s +Step 30610/44000 | Loss: 2.8737 | LR: 1.09e-04 | 1399.8ms/step | 14.3 samples/s +Step 30620/44000 | Loss: 2.8939 | LR: 1.09e-04 | 1400.0ms/step | 14.3 samples/s +Step 30630/44000 | Loss: 2.8897 | LR: 1.09e-04 | 1400.0ms/step | 14.3 samples/s +Step 30640/44000 | Loss: 2.8889 | LR: 1.09e-04 | 1400.0ms/step | 14.3 samples/s +Step 30650/44000 | Loss: 2.8776 | LR: 1.09e-04 | 1400.1ms/step | 14.3 samples/s +Step 30660/44000 | Loss: 2.8730 | LR: 1.08e-04 | 1400.0ms/step | 14.3 samples/s +Step 30670/44000 | Loss: 2.9199 | LR: 1.08e-04 | 1400.0ms/step | 14.3 samples/s +Step 30680/44000 | Loss: 2.8679 | LR: 1.08e-04 | 1399.9ms/step | 14.3 samples/s +Step 30690/44000 | Loss: 2.8976 | LR: 1.08e-04 | 1400.1ms/step | 14.3 samples/s +Step 30700/44000 | Loss: 2.8763 | LR: 1.08e-04 | 1400.1ms/step | 14.3 samples/s +Step 30710/44000 | Loss: 2.8946 | LR: 1.08e-04 | 1400.0ms/step | 14.3 samples/s +Step 30720/44000 | Loss: 2.9027 | LR: 1.08e-04 | 1399.9ms/step | 14.3 samples/s +Step 30730/44000 | Loss: 2.8602 | LR: 1.07e-04 | 1400.0ms/step | 14.3 samples/s +Step 30740/44000 | Loss: 2.8753 | LR: 1.07e-04 | 1400.1ms/step | 14.3 samples/s +Step 30750/44000 | Loss: 2.9037 | LR: 1.07e-04 | 1399.8ms/step | 14.3 samples/s +Step 30760/44000 | Loss: 2.9069 | LR: 1.07e-04 | 1400.2ms/step | 14.3 samples/s +Step 30770/44000 | Loss: 2.8804 | LR: 1.07e-04 | 1400.0ms/step | 14.3 samples/s +Step 30780/44000 | Loss: 2.8718 | LR: 1.07e-04 | 1400.0ms/step | 14.3 samples/s +Step 30790/44000 | Loss: 2.8978 | LR: 1.07e-04 | 1399.9ms/step | 14.3 samples/s +Step 30800/44000 | Loss: 2.8995 | LR: 1.06e-04 | 1400.0ms/step | 14.3 samples/s +Step 30810/44000 | Loss: 2.8575 | LR: 1.06e-04 | 1400.0ms/step | 14.3 samples/s +Step 30820/44000 | Loss: 2.8892 | LR: 1.06e-04 | 1399.9ms/step | 14.3 samples/s +Step 30830/44000 | Loss: 2.9325 | LR: 1.06e-04 | 1400.1ms/step | 14.3 samples/s +Step 30840/44000 | Loss: 2.8621 | LR: 1.06e-04 | 1400.0ms/step | 14.3 samples/s +Step 30850/44000 | Loss: 2.8924 | LR: 1.06e-04 | 1400.1ms/step | 14.3 samples/s +Step 30860/44000 | Loss: 2.8773 | LR: 1.05e-04 | 1399.9ms/step | 14.3 samples/s +Step 30870/44000 | Loss: 2.8680 | LR: 1.05e-04 | 1400.1ms/step | 14.3 samples/s +Step 30880/44000 | Loss: 2.9171 | LR: 1.05e-04 | 1399.8ms/step | 14.3 samples/s +Step 30890/44000 | Loss: 2.8495 | LR: 1.05e-04 | 1400.1ms/step | 14.3 samples/s +Step 30900/44000 | Loss: 2.8861 | LR: 1.05e-04 | 1400.0ms/step | 14.3 samples/s +Step 30910/44000 | Loss: 2.8745 | LR: 1.05e-04 | 1399.9ms/step | 14.3 samples/s +Step 30920/44000 | Loss: 2.8665 | LR: 1.05e-04 | 1400.0ms/step | 14.3 samples/s +Step 30930/44000 | Loss: 2.8816 | LR: 1.04e-04 | 1399.8ms/step | 14.3 samples/s +Step 30940/44000 | Loss: 2.8377 | LR: 1.04e-04 | 1400.0ms/step | 14.3 samples/s +Step 30950/44000 | Loss: 2.8894 | LR: 1.04e-04 | 1400.0ms/step | 14.3 samples/s +Step 30960/44000 | Loss: 2.9006 | LR: 1.04e-04 | 1400.1ms/step | 14.3 samples/s +Step 30970/44000 | Loss: 2.8877 | LR: 1.04e-04 | 1400.0ms/step | 14.3 samples/s +Step 30980/44000 | Loss: 2.8637 | LR: 1.04e-04 | 1399.9ms/step | 14.3 samples/s +Step 30990/44000 | Loss: 2.8918 | LR: 1.04e-04 | 1400.0ms/step | 14.3 samples/s +Step 31000/44000 | Loss: 2.8706 | LR: 1.03e-04 | 1400.1ms/step | 14.3 samples/s +Validation at step 31000: Loss = 2.8804, Perplexity = 17.82 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-31000 +Step 31010/44000 | Loss: 2.8741 | LR: 1.03e-04 | 3899.9ms/step | 5.1 samples/s +Step 31020/44000 | Loss: 2.9003 | LR: 1.03e-04 | 1399.9ms/step | 14.3 samples/s +Step 31030/44000 | Loss: 2.8896 | LR: 1.03e-04 | 1400.1ms/step | 14.3 samples/s +Step 31040/44000 | Loss: 2.8966 | LR: 1.03e-04 | 1400.1ms/step | 14.3 samples/s +Step 31050/44000 | Loss: 2.8828 | LR: 1.03e-04 | 1399.9ms/step | 14.3 samples/s +Step 31060/44000 | Loss: 2.8798 | LR: 1.03e-04 | 1400.0ms/step | 14.3 samples/s +Step 31070/44000 | Loss: 2.8922 | LR: 1.02e-04 | 1400.0ms/step | 14.3 samples/s +Step 31080/44000 | Loss: 2.9131 | LR: 1.02e-04 | 1400.1ms/step | 14.3 samples/s +Step 31090/44000 | Loss: 2.8748 | LR: 1.02e-04 | 1399.9ms/step | 14.3 samples/s +Step 31100/44000 | Loss: 2.8741 | LR: 1.02e-04 | 1400.0ms/step | 14.3 samples/s +Step 31110/44000 | Loss: 2.8852 | LR: 1.02e-04 | 1400.0ms/step | 14.3 samples/s +Step 31120/44000 | Loss: 2.9113 | LR: 1.02e-04 | 1400.1ms/step | 14.3 samples/s +Step 31130/44000 | Loss: 2.8684 | LR: 1.02e-04 | 1399.8ms/step | 14.3 samples/s +Step 31140/44000 | Loss: 2.8897 | LR: 1.01e-04 | 1400.1ms/step | 14.3 samples/s +Step 31150/44000 | Loss: 2.8691 | LR: 1.01e-04 | 1400.0ms/step | 14.3 samples/s +Step 31160/44000 | Loss: 2.9094 | LR: 1.01e-04 | 1400.0ms/step | 14.3 samples/s +Step 31170/44000 | Loss: 2.8655 | LR: 1.01e-04 | 1400.1ms/step | 14.3 samples/s +Step 31180/44000 | Loss: 2.9037 | LR: 1.01e-04 | 1399.9ms/step | 14.3 samples/s +Step 31190/44000 | Loss: 2.9156 | LR: 1.01e-04 | 1400.1ms/step | 14.3 samples/s +Step 31200/44000 | Loss: 2.8833 | LR: 1.00e-04 | 1400.0ms/step | 14.3 samples/s +Step 31210/44000 | Loss: 2.8867 | LR: 1.00e-04 | 1399.8ms/step | 14.3 samples/s +Step 31220/44000 | Loss: 2.8945 | LR: 1.00e-04 | 1400.2ms/step | 14.3 samples/s +Step 31230/44000 | Loss: 2.8621 | LR: 1.00e-04 | 1400.1ms/step | 14.3 samples/s +Step 31240/44000 | Loss: 2.8871 | LR: 9.99e-05 | 1400.1ms/step | 14.3 samples/s +Step 31250/44000 | Loss: 2.8984 | LR: 9.98e-05 | 1399.8ms/step | 14.3 samples/s +Step 31260/44000 | Loss: 2.8702 | LR: 9.96e-05 | 1399.9ms/step | 14.3 samples/s +Step 31270/44000 | Loss: 2.8820 | LR: 9.95e-05 | 1400.1ms/step | 14.3 samples/s +Step 31280/44000 | Loss: 2.9041 | LR: 9.93e-05 | 1400.0ms/step | 14.3 samples/s +Step 31290/44000 | Loss: 2.8897 | LR: 9.92e-05 | 1400.0ms/step | 14.3 samples/s +Step 31300/44000 | Loss: 2.9013 | LR: 9.90e-05 | 1400.2ms/step | 14.3 samples/s +Step 31310/44000 | Loss: 2.8731 | LR: 9.89e-05 | 1399.8ms/step | 14.3 samples/s +Step 31320/44000 | Loss: 2.8696 | LR: 9.88e-05 | 1400.1ms/step | 14.3 samples/s +Step 31330/44000 | Loss: 2.9065 | LR: 9.86e-05 | 1399.9ms/step | 14.3 samples/s +Step 31340/44000 | Loss: 2.8925 | LR: 9.85e-05 | 1400.0ms/step | 14.3 samples/s +Step 31350/44000 | Loss: 2.9298 | LR: 9.83e-05 | 1400.1ms/step | 14.3 samples/s +Step 31360/44000 | Loss: 2.9030 | LR: 9.82e-05 | 1400.0ms/step | 14.3 samples/s +Step 31370/44000 | Loss: 2.8862 | LR: 9.80e-05 | 1399.8ms/step | 14.3 samples/s +Step 31380/44000 | Loss: 2.9280 | LR: 9.79e-05 | 1400.1ms/step | 14.3 samples/s +Step 31390/44000 | Loss: 2.8738 | LR: 9.77e-05 | 1400.0ms/step | 14.3 samples/s +Step 31400/44000 | Loss: 2.8745 | LR: 9.76e-05 | 1400.1ms/step | 14.3 samples/s +Step 31410/44000 | Loss: 2.8713 | LR: 9.75e-05 | 1399.9ms/step | 14.3 samples/s +Step 31420/44000 | Loss: 2.8809 | LR: 9.73e-05 | 1399.9ms/step | 14.3 samples/s +Step 31430/44000 | Loss: 2.8915 | LR: 9.72e-05 | 1400.1ms/step | 14.3 samples/s +Step 31440/44000 | Loss: 2.8713 | LR: 9.70e-05 | 1400.1ms/step | 14.3 samples/s +Step 31450/44000 | Loss: 2.8868 | LR: 9.69e-05 | 1400.0ms/step | 14.3 samples/s +Step 31460/44000 | Loss: 2.8828 | LR: 9.67e-05 | 1400.0ms/step | 14.3 samples/s +Step 31470/44000 | Loss: 2.8765 | LR: 9.66e-05 | 1399.9ms/step | 14.3 samples/s +Step 31480/44000 | Loss: 2.8889 | LR: 9.65e-05 | 1399.9ms/step | 14.3 samples/s +Step 31490/44000 | Loss: 2.8841 | LR: 9.63e-05 | 1400.1ms/step | 14.3 samples/s +Step 31500/44000 | Loss: 2.9128 | LR: 9.62e-05 | 1400.1ms/step | 14.3 samples/s +Step 31510/44000 | Loss: 2.9080 | LR: 9.60e-05 | 1399.7ms/step | 14.3 samples/s +Step 31520/44000 | Loss: 2.8827 | LR: 9.59e-05 | 1400.1ms/step | 14.3 samples/s +Step 31530/44000 | Loss: 2.9140 | LR: 9.57e-05 | 1400.1ms/step | 14.3 samples/s +Step 31540/44000 | Loss: 2.8929 | LR: 9.56e-05 | 1400.0ms/step | 14.3 samples/s +Step 31550/44000 | Loss: 2.8870 | LR: 9.54e-05 | 1399.8ms/step | 14.3 samples/s +Step 31560/44000 | Loss: 2.8837 | LR: 9.53e-05 | 1400.2ms/step | 14.3 samples/s +Step 31570/44000 | Loss: 2.9247 | LR: 9.52e-05 | 1400.1ms/step | 14.3 samples/s +Step 31580/44000 | Loss: 2.9316 | LR: 9.50e-05 | 1399.8ms/step | 14.3 samples/s +Step 31590/44000 | Loss: 2.8384 | LR: 9.49e-05 | 1400.0ms/step | 14.3 samples/s +Step 31600/44000 | Loss: 2.9012 | LR: 9.47e-05 | 1399.9ms/step | 14.3 samples/s +Step 31610/44000 | Loss: 2.8939 | LR: 9.46e-05 | 1400.0ms/step | 14.3 samples/s +Step 31620/44000 | Loss: 2.9143 | LR: 9.45e-05 | 1399.9ms/step | 14.3 samples/s +Step 31630/44000 | Loss: 2.8605 | LR: 9.43e-05 | 1400.1ms/step | 14.3 samples/s +Step 31640/44000 | Loss: 2.8877 | LR: 9.42e-05 | 1400.0ms/step | 14.3 samples/s +Step 31650/44000 | Loss: 2.8926 | LR: 9.40e-05 | 1400.0ms/step | 14.3 samples/s +Step 31660/44000 | Loss: 2.8911 | LR: 9.39e-05 | 1400.1ms/step | 14.3 samples/s +Step 31670/44000 | Loss: 2.9064 | LR: 9.37e-05 | 1399.9ms/step | 14.3 samples/s +Step 31680/44000 | Loss: 2.8854 | LR: 9.36e-05 | 1400.0ms/step | 14.3 samples/s +Step 31690/44000 | Loss: 2.8834 | LR: 9.35e-05 | 1400.2ms/step | 14.3 samples/s +Step 31700/44000 | Loss: 2.8733 | LR: 9.33e-05 | 1399.7ms/step | 14.3 samples/s +Step 31710/44000 | Loss: 2.8855 | LR: 9.32e-05 | 1400.2ms/step | 14.3 samples/s +Step 31720/44000 | Loss: 2.8625 | LR: 9.30e-05 | 1400.1ms/step | 14.3 samples/s +Step 31730/44000 | Loss: 2.8686 | LR: 9.29e-05 | 1400.0ms/step | 14.3 samples/s +Step 31740/44000 | Loss: 2.9127 | LR: 9.28e-05 | 1399.7ms/step | 14.3 samples/s +Step 31750/44000 | Loss: 2.8970 | LR: 9.26e-05 | 1400.1ms/step | 14.3 samples/s +Step 31760/44000 | Loss: 2.8695 | LR: 9.25e-05 | 1400.1ms/step | 14.3 samples/s +Step 31770/44000 | Loss: 2.9001 | LR: 9.23e-05 | 1399.9ms/step | 14.3 samples/s +Step 31780/44000 | Loss: 2.8910 | LR: 9.22e-05 | 1400.1ms/step | 14.3 samples/s +Step 31790/44000 | Loss: 2.9101 | LR: 9.20e-05 | 1400.0ms/step | 14.3 samples/s +Step 31800/44000 | Loss: 2.8969 | LR: 9.19e-05 | 1400.1ms/step | 14.3 samples/s +Step 31810/44000 | Loss: 2.8999 | LR: 9.18e-05 | 1400.0ms/step | 14.3 samples/s +Step 31820/44000 | Loss: 2.8759 | LR: 9.16e-05 | 1400.0ms/step | 14.3 samples/s +Step 31830/44000 | Loss: 2.8923 | LR: 9.15e-05 | 1399.8ms/step | 14.3 samples/s +Step 31840/44000 | Loss: 2.8826 | LR: 9.13e-05 | 1400.0ms/step | 14.3 samples/s +Step 31850/44000 | Loss: 2.9161 | LR: 9.12e-05 | 1400.1ms/step | 14.3 samples/s +Step 31860/44000 | Loss: 2.8899 | LR: 9.11e-05 | 1400.0ms/step | 14.3 samples/s +Step 31870/44000 | Loss: 2.9024 | LR: 9.09e-05 | 1400.1ms/step | 14.3 samples/s +Step 31880/44000 | Loss: 2.8945 | LR: 9.08e-05 | 1399.8ms/step | 14.3 samples/s +Step 31890/44000 | Loss: 2.8970 | LR: 9.06e-05 | 1400.2ms/step | 14.3 samples/s +Step 31900/44000 | Loss: 2.8674 | LR: 9.05e-05 | 1399.8ms/step | 14.3 samples/s +Step 31910/44000 | Loss: 2.8744 | LR: 9.04e-05 | 1400.0ms/step | 14.3 samples/s +Step 31920/44000 | Loss: 2.8647 | LR: 9.02e-05 | 1400.1ms/step | 14.3 samples/s +Step 31930/44000 | Loss: 2.8729 | LR: 9.01e-05 | 1399.9ms/step | 14.3 samples/s +Step 31940/44000 | Loss: 2.8729 | LR: 8.99e-05 | 1399.9ms/step | 14.3 samples/s +Step 31950/44000 | Loss: 2.8684 | LR: 8.98e-05 | 1400.2ms/step | 14.3 samples/s +Step 31960/44000 | Loss: 2.8921 | LR: 8.97e-05 | 1400.0ms/step | 14.3 samples/s +Step 31970/44000 | Loss: 2.8789 | LR: 8.95e-05 | 1400.0ms/step | 14.3 samples/s +Step 31980/44000 | Loss: 2.8775 | LR: 8.94e-05 | 1399.9ms/step | 14.3 samples/s +Step 31990/44000 | Loss: 2.8876 | LR: 8.92e-05 | 1400.0ms/step | 14.3 samples/s +Step 32000/44000 | Loss: 2.8836 | LR: 8.91e-05 | 1400.1ms/step | 14.3 samples/s +Validation at step 32000: Loss = 2.8777, Perplexity = 17.77 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-32000 +Step 32010/44000 | Loss: 2.8550 | LR: 8.90e-05 | 3925.9ms/step | 5.1 samples/s +Step 32020/44000 | Loss: 2.8814 | LR: 8.88e-05 | 1400.0ms/step | 14.3 samples/s +Step 32030/44000 | Loss: 2.8885 | LR: 8.87e-05 | 1400.1ms/step | 14.3 samples/s +Step 32040/44000 | Loss: 2.8814 | LR: 8.86e-05 | 1399.9ms/step | 14.3 samples/s +Step 32050/44000 | Loss: 2.8724 | LR: 8.84e-05 | 1400.0ms/step | 14.3 samples/s +Step 32060/44000 | Loss: 2.8804 | LR: 8.83e-05 | 1400.2ms/step | 14.3 samples/s +Step 32070/44000 | Loss: 2.8841 | LR: 8.81e-05 | 1399.8ms/step | 14.3 samples/s +Step 32080/44000 | Loss: 2.8720 | LR: 8.80e-05 | 1399.9ms/step | 14.3 samples/s +Step 32090/44000 | Loss: 2.9011 | LR: 8.79e-05 | 1400.2ms/step | 14.3 samples/s +Step 32100/44000 | Loss: 2.9094 | LR: 8.77e-05 | 1399.9ms/step | 14.3 samples/s +Step 32110/44000 | Loss: 2.8762 | LR: 8.76e-05 | 1400.2ms/step | 14.3 samples/s +Step 32120/44000 | Loss: 2.9094 | LR: 8.74e-05 | 1400.0ms/step | 14.3 samples/s +Step 32130/44000 | Loss: 2.8904 | LR: 8.73e-05 | 1399.8ms/step | 14.3 samples/s +Step 32140/44000 | Loss: 2.8956 | LR: 8.72e-05 | 1400.1ms/step | 14.3 samples/s +Step 32150/44000 | Loss: 2.8861 | LR: 8.70e-05 | 1400.1ms/step | 14.3 samples/s +Step 32160/44000 | Loss: 2.9148 | LR: 8.69e-05 | 1399.9ms/step | 14.3 samples/s +Step 32170/44000 | Loss: 2.9119 | LR: 8.68e-05 | 1398.2ms/step | 14.3 samples/s +Step 32180/44000 | Loss: 2.8811 | LR: 8.66e-05 | 1399.7ms/step | 14.3 samples/s +Step 32190/44000 | Loss: 2.8925 | LR: 8.65e-05 | 1400.4ms/step | 14.3 samples/s +Step 32200/44000 | Loss: 2.8466 | LR: 8.63e-05 | 1399.8ms/step | 14.3 samples/s +Step 32210/44000 | Loss: 2.8622 | LR: 8.62e-05 | 1400.0ms/step | 14.3 samples/s +Step 32220/44000 | Loss: 2.8769 | LR: 8.61e-05 | 1399.9ms/step | 14.3 samples/s +Step 32230/44000 | Loss: 2.8850 | LR: 8.59e-05 | 1400.0ms/step | 14.3 samples/s +Step 32240/44000 | Loss: 2.8668 | LR: 8.58e-05 | 1400.1ms/step | 14.3 samples/s +Step 32250/44000 | Loss: 2.8901 | LR: 8.57e-05 | 1399.9ms/step | 14.3 samples/s +Step 32260/44000 | Loss: 2.8725 | LR: 8.55e-05 | 1400.1ms/step | 14.3 samples/s +Step 32270/44000 | Loss: 2.8999 | LR: 8.54e-05 | 1399.9ms/step | 14.3 samples/s +Step 32280/44000 | Loss: 2.9009 | LR: 8.53e-05 | 1400.1ms/step | 14.3 samples/s +Step 32290/44000 | Loss: 2.8872 | LR: 8.51e-05 | 1399.9ms/step | 14.3 samples/s +Step 32300/44000 | Loss: 2.9023 | LR: 8.50e-05 | 1400.0ms/step | 14.3 samples/s +Step 32310/44000 | Loss: 2.8911 | LR: 8.48e-05 | 1400.3ms/step | 14.3 samples/s +Step 32320/44000 | Loss: 2.8897 | LR: 8.47e-05 | 1399.9ms/step | 14.3 samples/s +Step 32330/44000 | Loss: 2.9070 | LR: 8.46e-05 | 1399.9ms/step | 14.3 samples/s +Step 32340/44000 | Loss: 2.8960 | LR: 8.44e-05 | 1400.0ms/step | 14.3 samples/s +Step 32350/44000 | Loss: 2.9401 | LR: 8.43e-05 | 1399.9ms/step | 14.3 samples/s +Step 32360/44000 | Loss: 2.8972 | LR: 8.42e-05 | 1400.0ms/step | 14.3 samples/s +Step 32370/44000 | Loss: 2.8804 | LR: 8.40e-05 | 1400.1ms/step | 14.3 samples/s +Step 32380/44000 | Loss: 2.8809 | LR: 8.39e-05 | 1399.9ms/step | 14.3 samples/s +Step 32390/44000 | Loss: 2.8815 | LR: 8.38e-05 | 1399.8ms/step | 14.3 samples/s +Step 32400/44000 | Loss: 2.8799 | LR: 8.36e-05 | 1400.1ms/step | 14.3 samples/s +Step 32410/44000 | Loss: 2.8703 | LR: 8.35e-05 | 1400.1ms/step | 14.3 samples/s +Step 32420/44000 | Loss: 2.8750 | LR: 8.33e-05 | 1400.0ms/step | 14.3 samples/s +Step 32430/44000 | Loss: 2.8882 | LR: 8.32e-05 | 1399.9ms/step | 14.3 samples/s +Step 32440/44000 | Loss: 2.8620 | LR: 8.31e-05 | 1400.2ms/step | 14.3 samples/s +Step 32450/44000 | Loss: 2.8666 | LR: 8.29e-05 | 1400.0ms/step | 14.3 samples/s +Step 32460/44000 | Loss: 2.8873 | LR: 8.28e-05 | 1399.9ms/step | 14.3 samples/s +Step 32470/44000 | Loss: 2.9144 | LR: 8.27e-05 | 1400.0ms/step | 14.3 samples/s +Step 32480/44000 | Loss: 2.8821 | LR: 8.25e-05 | 1400.1ms/step | 14.3 samples/s +Step 32490/44000 | Loss: 2.8535 | LR: 8.24e-05 | 1400.0ms/step | 14.3 samples/s +Step 32500/44000 | Loss: 2.8841 | LR: 8.23e-05 | 1399.8ms/step | 14.3 samples/s +Step 32510/44000 | Loss: 2.8905 | LR: 8.21e-05 | 1400.3ms/step | 14.3 samples/s +Step 32520/44000 | Loss: 2.9090 | LR: 8.20e-05 | 1399.9ms/step | 14.3 samples/s +Step 32530/44000 | Loss: 2.8530 | LR: 8.19e-05 | 1399.9ms/step | 14.3 samples/s +Step 32540/44000 | Loss: 2.8719 | LR: 8.17e-05 | 1400.1ms/step | 14.3 samples/s +Step 32550/44000 | Loss: 2.8833 | LR: 8.16e-05 | 1399.7ms/step | 14.3 samples/s +Step 32560/44000 | Loss: 2.8968 | LR: 8.15e-05 | 1400.1ms/step | 14.3 samples/s +Step 32570/44000 | Loss: 2.8880 | LR: 8.13e-05 | 1400.0ms/step | 14.3 samples/s +Step 32580/44000 | Loss: 2.9189 | LR: 8.12e-05 | 1400.2ms/step | 14.3 samples/s +Step 32590/44000 | Loss: 2.9086 | LR: 8.11e-05 | 1400.0ms/step | 14.3 samples/s +Step 32600/44000 | Loss: 2.9067 | LR: 8.09e-05 | 1399.9ms/step | 14.3 samples/s +Step 32610/44000 | Loss: 2.9229 | LR: 8.08e-05 | 1400.0ms/step | 14.3 samples/s +Step 32620/44000 | Loss: 2.8763 | LR: 8.07e-05 | 1399.8ms/step | 14.3 samples/s +Step 32630/44000 | Loss: 2.8912 | LR: 8.05e-05 | 1400.2ms/step | 14.3 samples/s +Step 32640/44000 | Loss: 2.8907 | LR: 8.04e-05 | 1400.0ms/step | 14.3 samples/s +Step 32650/44000 | Loss: 2.8667 | LR: 8.03e-05 | 1399.8ms/step | 14.3 samples/s +Step 32660/44000 | Loss: 2.9142 | LR: 8.01e-05 | 1400.2ms/step | 14.3 samples/s +Step 32670/44000 | Loss: 2.9068 | LR: 8.00e-05 | 1399.9ms/step | 14.3 samples/s +Step 32680/44000 | Loss: 2.8867 | LR: 7.99e-05 | 1400.0ms/step | 14.3 samples/s +Step 32690/44000 | Loss: 2.9139 | LR: 7.97e-05 | 1400.2ms/step | 14.3 samples/s +Step 32700/44000 | Loss: 2.8929 | LR: 7.96e-05 | 1399.7ms/step | 14.3 samples/s +Step 32710/44000 | Loss: 2.9097 | LR: 7.95e-05 | 1400.2ms/step | 14.3 samples/s +Step 32720/44000 | Loss: 2.8818 | LR: 7.93e-05 | 1400.0ms/step | 14.3 samples/s +Step 32730/44000 | Loss: 2.8799 | LR: 7.92e-05 | 1399.8ms/step | 14.3 samples/s +Step 32740/44000 | Loss: 2.8727 | LR: 7.91e-05 | 1400.3ms/step | 14.3 samples/s +Step 32750/44000 | Loss: 2.9041 | LR: 7.89e-05 | 1399.6ms/step | 14.3 samples/s +Step 32760/44000 | Loss: 2.8776 | LR: 7.88e-05 | 1400.4ms/step | 14.3 samples/s +Step 32770/44000 | Loss: 2.8963 | LR: 7.87e-05 | 1399.9ms/step | 14.3 samples/s +Step 32780/44000 | Loss: 2.9080 | LR: 7.85e-05 | 1400.1ms/step | 14.3 samples/s +Step 32790/44000 | Loss: 2.8762 | LR: 7.84e-05 | 1399.8ms/step | 14.3 samples/s +Step 32800/44000 | Loss: 2.8628 | LR: 7.83e-05 | 1399.9ms/step | 14.3 samples/s +Step 32810/44000 | Loss: 2.8896 | LR: 7.81e-05 | 1400.0ms/step | 14.3 samples/s +Step 32820/44000 | Loss: 2.8741 | LR: 7.80e-05 | 1400.0ms/step | 14.3 samples/s +Step 32830/44000 | Loss: 2.8776 | LR: 7.79e-05 | 1400.1ms/step | 14.3 samples/s +Step 32840/44000 | Loss: 2.8912 | LR: 7.77e-05 | 1399.9ms/step | 14.3 samples/s +Step 32850/44000 | Loss: 2.9027 | LR: 7.76e-05 | 1400.1ms/step | 14.3 samples/s +Step 32860/44000 | Loss: 2.8919 | LR: 7.75e-05 | 1400.0ms/step | 14.3 samples/s +Step 32870/44000 | Loss: 2.8636 | LR: 7.73e-05 | 1400.0ms/step | 14.3 samples/s +Step 32880/44000 | Loss: 2.9112 | LR: 7.72e-05 | 1399.9ms/step | 14.3 samples/s +Step 32890/44000 | Loss: 2.8708 | LR: 7.71e-05 | 1400.1ms/step | 14.3 samples/s +Step 32900/44000 | Loss: 2.8754 | LR: 7.70e-05 | 1400.0ms/step | 14.3 samples/s +Step 32910/44000 | Loss: 2.8928 | LR: 7.68e-05 | 1399.9ms/step | 14.3 samples/s +Step 32920/44000 | Loss: 2.9037 | LR: 7.67e-05 | 1400.3ms/step | 14.3 samples/s +Step 32930/44000 | Loss: 2.9206 | LR: 7.66e-05 | 1400.0ms/step | 14.3 samples/s +Step 32940/44000 | Loss: 2.8917 | LR: 7.64e-05 | 1399.8ms/step | 14.3 samples/s +Step 32950/44000 | Loss: 2.8933 | LR: 7.63e-05 | 1400.2ms/step | 14.3 samples/s +Step 32960/44000 | Loss: 2.8729 | LR: 7.62e-05 | 1399.8ms/step | 14.3 samples/s +Step 32970/44000 | Loss: 2.8852 | LR: 7.60e-05 | 1399.9ms/step | 14.3 samples/s +Step 32980/44000 | Loss: 2.8674 | LR: 7.59e-05 | 1399.9ms/step | 14.3 samples/s +Step 32990/44000 | Loss: 2.8662 | LR: 7.58e-05 | 1400.1ms/step | 14.3 samples/s +Step 33000/44000 | Loss: 2.8996 | LR: 7.56e-05 | 1400.0ms/step | 14.3 samples/s +Validation at step 33000: Loss = 2.8759, Perplexity = 17.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-33000 +Step 33010/44000 | Loss: 2.9048 | LR: 7.55e-05 | 3925.9ms/step | 5.1 samples/s +Step 33020/44000 | Loss: 2.8873 | LR: 7.54e-05 | 1397.9ms/step | 14.3 samples/s +Step 33030/44000 | Loss: 2.9072 | LR: 7.53e-05 | 1399.9ms/step | 14.3 samples/s +Step 33040/44000 | Loss: 2.8755 | LR: 7.51e-05 | 1400.1ms/step | 14.3 samples/s +Step 33050/44000 | Loss: 2.8698 | LR: 7.50e-05 | 1393.7ms/step | 14.4 samples/s +Step 33060/44000 | Loss: 2.9011 | LR: 7.49e-05 | 1393.7ms/step | 14.4 samples/s +Step 33070/44000 | Loss: 2.8612 | LR: 7.47e-05 | 1392.6ms/step | 14.4 samples/s +Step 33080/44000 | Loss: 2.8755 | LR: 7.46e-05 | 1394.4ms/step | 14.3 samples/s +Step 33090/44000 | Loss: 2.9052 | LR: 7.45e-05 | 1393.6ms/step | 14.4 samples/s +Step 33100/44000 | Loss: 2.8795 | LR: 7.44e-05 | 1394.0ms/step | 14.3 samples/s +Step 33110/44000 | Loss: 2.8780 | LR: 7.42e-05 | 1396.0ms/step | 14.3 samples/s +Step 33120/44000 | Loss: 2.8850 | LR: 7.41e-05 | 1397.9ms/step | 14.3 samples/s +Step 33130/44000 | Loss: 2.9174 | LR: 7.40e-05 | 1394.0ms/step | 14.3 samples/s +Step 33140/44000 | Loss: 2.8680 | LR: 7.38e-05 | 1394.6ms/step | 14.3 samples/s +Step 33150/44000 | Loss: 2.8625 | LR: 7.37e-05 | 1395.5ms/step | 14.3 samples/s +Step 33160/44000 | Loss: 2.8505 | LR: 7.36e-05 | 1393.0ms/step | 14.4 samples/s +Step 33170/44000 | Loss: 2.8997 | LR: 7.34e-05 | 1398.6ms/step | 14.3 samples/s +Step 33180/44000 | Loss: 2.9174 | LR: 7.33e-05 | 1396.4ms/step | 14.3 samples/s +Step 33190/44000 | Loss: 2.8876 | LR: 7.32e-05 | 1393.5ms/step | 14.4 samples/s +Step 33200/44000 | Loss: 2.8617 | LR: 7.31e-05 | 1394.1ms/step | 14.3 samples/s +Step 33210/44000 | Loss: 2.8737 | LR: 7.29e-05 | 1394.6ms/step | 14.3 samples/s +Step 33220/44000 | Loss: 2.8903 | LR: 7.28e-05 | 1391.9ms/step | 14.4 samples/s +Step 33230/44000 | Loss: 2.8590 | LR: 7.27e-05 | 1396.0ms/step | 14.3 samples/s +Step 33240/44000 | Loss: 2.8891 | LR: 7.26e-05 | 1394.0ms/step | 14.3 samples/s +Step 33250/44000 | Loss: 2.8920 | LR: 7.24e-05 | 1393.8ms/step | 14.3 samples/s +Step 33260/44000 | Loss: 2.8932 | LR: 7.23e-05 | 1396.1ms/step | 14.3 samples/s +Step 33270/44000 | Loss: 2.8994 | LR: 7.22e-05 | 1394.1ms/step | 14.3 samples/s +Step 33280/44000 | Loss: 2.8755 | LR: 7.20e-05 | 1392.7ms/step | 14.4 samples/s +Step 33290/44000 | Loss: 2.8665 | LR: 7.19e-05 | 1396.2ms/step | 14.3 samples/s +Step 33300/44000 | Loss: 2.9029 | LR: 7.18e-05 | 1395.2ms/step | 14.3 samples/s +Step 33310/44000 | Loss: 2.8635 | LR: 7.17e-05 | 1393.8ms/step | 14.3 samples/s +Step 33320/44000 | Loss: 2.8744 | LR: 7.15e-05 | 1396.2ms/step | 14.3 samples/s +Step 33330/44000 | Loss: 2.8977 | LR: 7.14e-05 | 1393.8ms/step | 14.3 samples/s +Step 33340/44000 | Loss: 2.9067 | LR: 7.13e-05 | 1394.1ms/step | 14.3 samples/s +Step 33350/44000 | Loss: 2.8893 | LR: 7.11e-05 | 1392.1ms/step | 14.4 samples/s +Step 33360/44000 | Loss: 2.8896 | LR: 7.10e-05 | 1392.0ms/step | 14.4 samples/s +Step 33370/44000 | Loss: 2.8519 | LR: 7.09e-05 | 1392.7ms/step | 14.4 samples/s +Step 33380/44000 | Loss: 2.8813 | LR: 7.08e-05 | 1391.1ms/step | 14.4 samples/s +Step 33390/44000 | Loss: 2.8886 | LR: 7.06e-05 | 1394.1ms/step | 14.3 samples/s +Step 33400/44000 | Loss: 2.8872 | LR: 7.05e-05 | 1392.0ms/step | 14.4 samples/s +Step 33410/44000 | Loss: 2.9066 | LR: 7.04e-05 | 1396.2ms/step | 14.3 samples/s +Step 33420/44000 | Loss: 2.8845 | LR: 7.03e-05 | 1393.6ms/step | 14.4 samples/s +Step 33430/44000 | Loss: 2.8689 | LR: 7.01e-05 | 1394.4ms/step | 14.3 samples/s +Step 33440/44000 | Loss: 2.8803 | LR: 7.00e-05 | 1396.1ms/step | 14.3 samples/s +Step 33450/44000 | Loss: 2.8876 | LR: 6.99e-05 | 1397.8ms/step | 14.3 samples/s +Step 33460/44000 | Loss: 2.9012 | LR: 6.98e-05 | 1393.8ms/step | 14.3 samples/s +Step 33470/44000 | Loss: 2.8655 | LR: 6.96e-05 | 1394.1ms/step | 14.3 samples/s +Step 33480/44000 | Loss: 2.8981 | LR: 6.95e-05 | 1393.9ms/step | 14.3 samples/s +Step 33490/44000 | Loss: 2.9080 | LR: 6.94e-05 | 1393.9ms/step | 14.3 samples/s +Step 33500/44000 | Loss: 2.9102 | LR: 6.93e-05 | 1392.4ms/step | 14.4 samples/s +Step 33510/44000 | Loss: 2.8952 | LR: 6.91e-05 | 1397.8ms/step | 14.3 samples/s +Step 33520/44000 | Loss: 2.8726 | LR: 6.90e-05 | 1390.0ms/step | 14.4 samples/s +Step 33530/44000 | Loss: 2.8543 | LR: 6.89e-05 | 1394.4ms/step | 14.3 samples/s +Step 33540/44000 | Loss: 2.8922 | LR: 6.88e-05 | 1397.7ms/step | 14.3 samples/s +Step 33550/44000 | Loss: 2.8866 | LR: 6.86e-05 | 1394.0ms/step | 14.3 samples/s +Step 33560/44000 | Loss: 2.8806 | LR: 6.85e-05 | 1392.1ms/step | 14.4 samples/s +Step 33570/44000 | Loss: 2.8625 | LR: 6.84e-05 | 1393.0ms/step | 14.4 samples/s +Step 33580/44000 | Loss: 2.8808 | LR: 6.83e-05 | 1393.0ms/step | 14.4 samples/s +Step 33590/44000 | Loss: 2.8990 | LR: 6.81e-05 | 1390.6ms/step | 14.4 samples/s +Step 33600/44000 | Loss: 2.8970 | LR: 6.80e-05 | 1393.4ms/step | 14.4 samples/s +Step 33610/44000 | Loss: 2.9089 | LR: 6.79e-05 | 1393.7ms/step | 14.4 samples/s +Step 33620/44000 | Loss: 2.9134 | LR: 6.78e-05 | 1394.3ms/step | 14.3 samples/s +Step 33630/44000 | Loss: 2.8612 | LR: 6.76e-05 | 1393.9ms/step | 14.3 samples/s +Step 33640/44000 | Loss: 2.8858 | LR: 6.75e-05 | 1393.2ms/step | 14.4 samples/s +Step 33650/44000 | Loss: 2.8905 | LR: 6.74e-05 | 1395.0ms/step | 14.3 samples/s +Step 33660/44000 | Loss: 2.9117 | LR: 6.73e-05 | 1395.9ms/step | 14.3 samples/s +Step 33670/44000 | Loss: 2.8797 | LR: 6.71e-05 | 1390.4ms/step | 14.4 samples/s +Step 33680/44000 | Loss: 2.8942 | LR: 6.70e-05 | 1395.3ms/step | 14.3 samples/s +Step 33690/44000 | Loss: 2.9084 | LR: 6.69e-05 | 1395.9ms/step | 14.3 samples/s +Step 33700/44000 | Loss: 2.8705 | LR: 6.68e-05 | 1392.5ms/step | 14.4 samples/s +Step 33710/44000 | Loss: 2.8603 | LR: 6.66e-05 | 1393.9ms/step | 14.3 samples/s +Step 33720/44000 | Loss: 2.9148 | LR: 6.65e-05 | 1390.2ms/step | 14.4 samples/s +Step 33730/44000 | Loss: 2.8852 | LR: 6.64e-05 | 1393.5ms/step | 14.4 samples/s +Step 33740/44000 | Loss: 2.8805 | LR: 6.63e-05 | 1395.8ms/step | 14.3 samples/s +Step 33750/44000 | Loss: 2.8852 | LR: 6.61e-05 | 1394.6ms/step | 14.3 samples/s +Step 33760/44000 | Loss: 2.8578 | LR: 6.60e-05 | 1388.0ms/step | 14.4 samples/s +Step 33770/44000 | Loss: 2.8891 | LR: 6.59e-05 | 1389.6ms/step | 14.4 samples/s +Step 33780/44000 | Loss: 2.8744 | LR: 6.58e-05 | 1392.0ms/step | 14.4 samples/s +Step 33790/44000 | Loss: 2.8794 | LR: 6.57e-05 | 1392.3ms/step | 14.4 samples/s +Step 33800/44000 | Loss: 2.8685 | LR: 6.55e-05 | 1393.1ms/step | 14.4 samples/s +Step 33810/44000 | Loss: 2.8828 | LR: 6.54e-05 | 1398.4ms/step | 14.3 samples/s +Step 33820/44000 | Loss: 2.8851 | LR: 6.53e-05 | 1400.5ms/step | 14.3 samples/s +Step 33830/44000 | Loss: 2.8654 | LR: 6.52e-05 | 1400.2ms/step | 14.3 samples/s +Step 33840/44000 | Loss: 2.8835 | LR: 6.50e-05 | 1397.1ms/step | 14.3 samples/s +Step 33850/44000 | Loss: 2.8921 | LR: 6.49e-05 | 1392.8ms/step | 14.4 samples/s +Step 33860/44000 | Loss: 2.8989 | LR: 6.48e-05 | 1393.9ms/step | 14.3 samples/s +Step 33870/44000 | Loss: 2.8622 | LR: 6.47e-05 | 1394.8ms/step | 14.3 samples/s +Step 33880/44000 | Loss: 2.8916 | LR: 6.46e-05 | 1391.3ms/step | 14.4 samples/s +Step 33890/44000 | Loss: 2.8907 | LR: 6.44e-05 | 1393.4ms/step | 14.4 samples/s +Step 33900/44000 | Loss: 2.9025 | LR: 6.43e-05 | 1393.4ms/step | 14.4 samples/s +Step 33910/44000 | Loss: 2.8881 | LR: 6.42e-05 | 1392.7ms/step | 14.4 samples/s +Step 33920/44000 | Loss: 2.9209 | LR: 6.41e-05 | 1396.4ms/step | 14.3 samples/s +Step 33930/44000 | Loss: 2.8674 | LR: 6.40e-05 | 1395.9ms/step | 14.3 samples/s +Step 33940/44000 | Loss: 2.9036 | LR: 6.38e-05 | 1397.3ms/step | 14.3 samples/s +Step 33950/44000 | Loss: 2.9015 | LR: 6.37e-05 | 1393.1ms/step | 14.4 samples/s +Step 33960/44000 | Loss: 2.9073 | LR: 6.36e-05 | 1394.0ms/step | 14.3 samples/s +Step 33970/44000 | Loss: 2.9054 | LR: 6.35e-05 | 1393.2ms/step | 14.4 samples/s +Step 33980/44000 | Loss: 2.8844 | LR: 6.33e-05 | 1394.3ms/step | 14.3 samples/s +Step 33990/44000 | Loss: 2.9045 | LR: 6.32e-05 | 1392.0ms/step | 14.4 samples/s +Step 34000/44000 | Loss: 2.8898 | LR: 6.31e-05 | 1396.4ms/step | 14.3 samples/s +Validation at step 34000: Loss = 2.8743, Perplexity = 17.71 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-34000 +Step 34010/44000 | Loss: 2.8691 | LR: 6.30e-05 | 3905.8ms/step | 5.1 samples/s +Step 34020/44000 | Loss: 2.9095 | LR: 6.29e-05 | 1400.2ms/step | 14.3 samples/s +Step 34030/44000 | Loss: 2.9167 | LR: 6.27e-05 | 1400.0ms/step | 14.3 samples/s +Step 34040/44000 | Loss: 2.8803 | LR: 6.26e-05 | 1400.0ms/step | 14.3 samples/s +Step 34050/44000 | Loss: 2.8810 | LR: 6.25e-05 | 1399.8ms/step | 14.3 samples/s +Step 34060/44000 | Loss: 2.8708 | LR: 6.24e-05 | 1400.2ms/step | 14.3 samples/s +Step 34070/44000 | Loss: 2.8562 | LR: 6.23e-05 | 1399.9ms/step | 14.3 samples/s +Step 34080/44000 | Loss: 2.8701 | LR: 6.21e-05 | 1399.5ms/step | 14.3 samples/s +Step 34090/44000 | Loss: 2.8710 | LR: 6.20e-05 | 1392.1ms/step | 14.4 samples/s +Step 34100/44000 | Loss: 2.8769 | LR: 6.19e-05 | 1393.5ms/step | 14.4 samples/s +Step 34110/44000 | Loss: 2.8991 | LR: 6.18e-05 | 1392.5ms/step | 14.4 samples/s +Step 34120/44000 | Loss: 2.8843 | LR: 6.17e-05 | 1392.5ms/step | 14.4 samples/s +Step 34130/44000 | Loss: 2.8654 | LR: 6.15e-05 | 1393.6ms/step | 14.4 samples/s +Step 34140/44000 | Loss: 2.8858 | LR: 6.14e-05 | 1392.3ms/step | 14.4 samples/s +Step 34150/44000 | Loss: 2.8497 | LR: 6.13e-05 | 1395.7ms/step | 14.3 samples/s +Step 34160/44000 | Loss: 2.8713 | LR: 6.12e-05 | 1392.0ms/step | 14.4 samples/s +Step 34170/44000 | Loss: 2.8907 | LR: 6.11e-05 | 1393.1ms/step | 14.4 samples/s +Step 34180/44000 | Loss: 2.8834 | LR: 6.09e-05 | 1390.9ms/step | 14.4 samples/s +Step 34190/44000 | Loss: 2.8817 | LR: 6.08e-05 | 1395.5ms/step | 14.3 samples/s +Step 34200/44000 | Loss: 2.8826 | LR: 6.07e-05 | 1392.5ms/step | 14.4 samples/s +Step 34210/44000 | Loss: 2.9035 | LR: 6.06e-05 | 1398.0ms/step | 14.3 samples/s +Step 34220/44000 | Loss: 2.9118 | LR: 6.05e-05 | 1395.2ms/step | 14.3 samples/s +Step 34230/44000 | Loss: 2.9066 | LR: 6.04e-05 | 1396.4ms/step | 14.3 samples/s +Step 34240/44000 | Loss: 2.8756 | LR: 6.02e-05 | 1392.1ms/step | 14.4 samples/s +Step 34250/44000 | Loss: 2.8910 | LR: 6.01e-05 | 1396.4ms/step | 14.3 samples/s +Step 34260/44000 | Loss: 2.8953 | LR: 6.00e-05 | 1393.4ms/step | 14.4 samples/s +Step 34270/44000 | Loss: 2.8866 | LR: 5.99e-05 | 1392.3ms/step | 14.4 samples/s +Step 34280/44000 | Loss: 2.9037 | LR: 5.98e-05 | 1395.2ms/step | 14.3 samples/s +Step 34290/44000 | Loss: 2.8871 | LR: 5.96e-05 | 1396.6ms/step | 14.3 samples/s +Step 34300/44000 | Loss: 2.8867 | LR: 5.95e-05 | 1393.9ms/step | 14.3 samples/s +Step 34310/44000 | Loss: 2.8819 | LR: 5.94e-05 | 1391.3ms/step | 14.4 samples/s +Step 34320/44000 | Loss: 2.8813 | LR: 5.93e-05 | 1393.0ms/step | 14.4 samples/s +Step 34330/44000 | Loss: 2.8967 | LR: 5.92e-05 | 1392.0ms/step | 14.4 samples/s +Step 34340/44000 | Loss: 2.8746 | LR: 5.91e-05 | 1391.9ms/step | 14.4 samples/s +Step 34350/44000 | Loss: 2.8508 | LR: 5.89e-05 | 1395.6ms/step | 14.3 samples/s +Step 34360/44000 | Loss: 2.8778 | LR: 5.88e-05 | 1390.4ms/step | 14.4 samples/s +Step 34370/44000 | Loss: 2.8930 | LR: 5.87e-05 | 1390.7ms/step | 14.4 samples/s +Step 34380/44000 | Loss: 2.8721 | LR: 5.86e-05 | 1395.2ms/step | 14.3 samples/s +Step 34390/44000 | Loss: 2.8690 | LR: 5.85e-05 | 1392.2ms/step | 14.4 samples/s +Step 34400/44000 | Loss: 2.8763 | LR: 5.84e-05 | 1392.7ms/step | 14.4 samples/s +Step 34410/44000 | Loss: 2.8646 | LR: 5.82e-05 | 1392.3ms/step | 14.4 samples/s +Step 34420/44000 | Loss: 2.8904 | LR: 5.81e-05 | 1393.0ms/step | 14.4 samples/s +Step 34430/44000 | Loss: 2.8973 | LR: 5.80e-05 | 1394.1ms/step | 14.3 samples/s +Step 34440/44000 | Loss: 2.8573 | LR: 5.79e-05 | 1393.7ms/step | 14.4 samples/s +Step 34450/44000 | Loss: 2.8898 | LR: 5.78e-05 | 1393.9ms/step | 14.3 samples/s +Step 34460/44000 | Loss: 2.9213 | LR: 5.77e-05 | 1396.0ms/step | 14.3 samples/s +Step 34470/44000 | Loss: 2.8721 | LR: 5.75e-05 | 1396.5ms/step | 14.3 samples/s +Step 34480/44000 | Loss: 2.8940 | LR: 5.74e-05 | 1395.6ms/step | 14.3 samples/s +Step 34490/44000 | Loss: 2.8539 | LR: 5.73e-05 | 1392.1ms/step | 14.4 samples/s +Step 34500/44000 | Loss: 2.8511 | LR: 5.72e-05 | 1391.9ms/step | 14.4 samples/s +Step 34510/44000 | Loss: 2.8894 | LR: 5.71e-05 | 1396.2ms/step | 14.3 samples/s +Step 34520/44000 | Loss: 2.8902 | LR: 5.70e-05 | 1394.0ms/step | 14.3 samples/s +Step 34530/44000 | Loss: 2.8910 | LR: 5.69e-05 | 1391.8ms/step | 14.4 samples/s +Step 34540/44000 | Loss: 2.8990 | LR: 5.67e-05 | 1392.0ms/step | 14.4 samples/s +Step 34550/44000 | Loss: 2.9259 | LR: 5.66e-05 | 1392.0ms/step | 14.4 samples/s +Step 34560/44000 | Loss: 2.8592 | LR: 5.65e-05 | 1394.8ms/step | 14.3 samples/s +Step 34570/44000 | Loss: 2.8959 | LR: 5.64e-05 | 1390.4ms/step | 14.4 samples/s +Step 34580/44000 | Loss: 2.8917 | LR: 5.63e-05 | 1392.8ms/step | 14.4 samples/s +Step 34590/44000 | Loss: 2.9023 | LR: 5.62e-05 | 1392.0ms/step | 14.4 samples/s +Step 34600/44000 | Loss: 2.8933 | LR: 5.60e-05 | 1390.1ms/step | 14.4 samples/s +Step 34610/44000 | Loss: 2.8875 | LR: 5.59e-05 | 1393.1ms/step | 14.4 samples/s +Step 34620/44000 | Loss: 2.8863 | LR: 5.58e-05 | 1391.0ms/step | 14.4 samples/s +Step 34630/44000 | Loss: 2.8588 | LR: 5.57e-05 | 1395.4ms/step | 14.3 samples/s +Step 34640/44000 | Loss: 2.8760 | LR: 5.56e-05 | 1392.7ms/step | 14.4 samples/s +Step 34650/44000 | Loss: 2.8776 | LR: 5.55e-05 | 1391.8ms/step | 14.4 samples/s +Step 34660/44000 | Loss: 2.8686 | LR: 5.54e-05 | 1393.7ms/step | 14.3 samples/s +Step 34670/44000 | Loss: 2.8491 | LR: 5.52e-05 | 1394.4ms/step | 14.3 samples/s +Step 34680/44000 | Loss: 2.8632 | LR: 5.51e-05 | 1394.0ms/step | 14.3 samples/s +Step 34690/44000 | Loss: 2.8454 | LR: 5.50e-05 | 1393.4ms/step | 14.4 samples/s +Step 34700/44000 | Loss: 2.8878 | LR: 5.49e-05 | 1394.6ms/step | 14.3 samples/s +Step 34710/44000 | Loss: 2.8744 | LR: 5.48e-05 | 1393.6ms/step | 14.4 samples/s +Step 34720/44000 | Loss: 2.8926 | LR: 5.47e-05 | 1391.8ms/step | 14.4 samples/s +Step 34730/44000 | Loss: 2.8948 | LR: 5.46e-05 | 1394.1ms/step | 14.3 samples/s +Step 34740/44000 | Loss: 2.8777 | LR: 5.45e-05 | 1389.6ms/step | 14.4 samples/s +Step 34750/44000 | Loss: 2.8857 | LR: 5.43e-05 | 1391.8ms/step | 14.4 samples/s +Step 34760/44000 | Loss: 2.8741 | LR: 5.42e-05 | 1394.4ms/step | 14.3 samples/s +Step 34770/44000 | Loss: 2.8918 | LR: 5.41e-05 | 1392.3ms/step | 14.4 samples/s +Step 34780/44000 | Loss: 2.8789 | LR: 5.40e-05 | 1392.0ms/step | 14.4 samples/s +Step 34790/44000 | Loss: 2.8958 | LR: 5.39e-05 | 1392.0ms/step | 14.4 samples/s +Step 34800/44000 | Loss: 2.8888 | LR: 5.38e-05 | 1391.6ms/step | 14.4 samples/s +Step 34810/44000 | Loss: 2.8797 | LR: 5.37e-05 | 1391.8ms/step | 14.4 samples/s +Step 34820/44000 | Loss: 2.8672 | LR: 5.36e-05 | 1390.8ms/step | 14.4 samples/s +Step 34830/44000 | Loss: 2.8597 | LR: 5.34e-05 | 1394.2ms/step | 14.3 samples/s +Step 34840/44000 | Loss: 2.8911 | LR: 5.33e-05 | 1393.7ms/step | 14.4 samples/s +Step 34850/44000 | Loss: 2.8616 | LR: 5.32e-05 | 1391.9ms/step | 14.4 samples/s +Step 34860/44000 | Loss: 2.8953 | LR: 5.31e-05 | 1394.2ms/step | 14.3 samples/s +Step 34870/44000 | Loss: 2.8896 | LR: 5.30e-05 | 1394.0ms/step | 14.3 samples/s +Step 34880/44000 | Loss: 2.8649 | LR: 5.29e-05 | 1396.6ms/step | 14.3 samples/s +Step 34890/44000 | Loss: 2.8858 | LR: 5.28e-05 | 1395.2ms/step | 14.3 samples/s +Step 34900/44000 | Loss: 2.8647 | LR: 5.27e-05 | 1392.7ms/step | 14.4 samples/s +Step 34910/44000 | Loss: 2.8744 | LR: 5.25e-05 | 1393.7ms/step | 14.4 samples/s +Step 34920/44000 | Loss: 2.8635 | LR: 5.24e-05 | 1396.6ms/step | 14.3 samples/s +Step 34930/44000 | Loss: 2.9101 | LR: 5.23e-05 | 1397.1ms/step | 14.3 samples/s +Step 34940/44000 | Loss: 2.8676 | LR: 5.22e-05 | 1394.2ms/step | 14.3 samples/s +Step 34950/44000 | Loss: 2.8941 | LR: 5.21e-05 | 1390.0ms/step | 14.4 samples/s +Step 34960/44000 | Loss: 2.9016 | LR: 5.20e-05 | 1394.7ms/step | 14.3 samples/s +Step 34970/44000 | Loss: 2.8760 | LR: 5.19e-05 | 1396.6ms/step | 14.3 samples/s +Step 34980/44000 | Loss: 2.8662 | LR: 5.18e-05 | 1393.5ms/step | 14.4 samples/s +Step 34990/44000 | Loss: 2.8923 | LR: 5.17e-05 | 1391.4ms/step | 14.4 samples/s +Step 35000/44000 | Loss: 2.8980 | LR: 5.15e-05 | 1399.9ms/step | 14.3 samples/s +Validation at step 35000: Loss = 2.8731, Perplexity = 17.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-35000 +Step 35010/44000 | Loss: 2.8845 | LR: 5.14e-05 | 3908.0ms/step | 5.1 samples/s +Step 35020/44000 | Loss: 2.8663 | LR: 5.13e-05 | 1399.9ms/step | 14.3 samples/s +Step 35030/44000 | Loss: 2.8759 | LR: 5.12e-05 | 1400.2ms/step | 14.3 samples/s +Step 35040/44000 | Loss: 2.8800 | LR: 5.11e-05 | 1399.9ms/step | 14.3 samples/s +Step 35050/44000 | Loss: 2.8753 | LR: 5.10e-05 | 1400.0ms/step | 14.3 samples/s +Step 35060/44000 | Loss: 2.8610 | LR: 5.09e-05 | 1400.1ms/step | 14.3 samples/s +Step 35070/44000 | Loss: 2.8818 | LR: 5.08e-05 | 1399.8ms/step | 14.3 samples/s +Step 35080/44000 | Loss: 2.8901 | LR: 5.07e-05 | 1400.1ms/step | 14.3 samples/s +Step 35090/44000 | Loss: 2.8971 | LR: 5.06e-05 | 1399.8ms/step | 14.3 samples/s +Step 35100/44000 | Loss: 2.8996 | LR: 5.04e-05 | 1400.3ms/step | 14.3 samples/s +Step 35110/44000 | Loss: 2.8891 | LR: 5.03e-05 | 1400.0ms/step | 14.3 samples/s +Step 35120/44000 | Loss: 2.8851 | LR: 5.02e-05 | 1400.0ms/step | 14.3 samples/s +Step 35130/44000 | Loss: 2.8856 | LR: 5.01e-05 | 1399.8ms/step | 14.3 samples/s +Step 35140/44000 | Loss: 2.8753 | LR: 5.00e-05 | 1400.1ms/step | 14.3 samples/s +Step 35150/44000 | Loss: 2.8960 | LR: 4.99e-05 | 1400.0ms/step | 14.3 samples/s +Step 35160/44000 | Loss: 2.9231 | LR: 4.98e-05 | 1400.2ms/step | 14.3 samples/s +Step 35170/44000 | Loss: 2.8660 | LR: 4.97e-05 | 1399.9ms/step | 14.3 samples/s +Step 35180/44000 | Loss: 2.8533 | LR: 4.96e-05 | 1400.0ms/step | 14.3 samples/s +Step 35190/44000 | Loss: 2.8869 | LR: 4.95e-05 | 1399.8ms/step | 14.3 samples/s +Step 35200/44000 | Loss: 2.9111 | LR: 4.94e-05 | 1400.1ms/step | 14.3 samples/s +Step 35210/44000 | Loss: 2.8520 | LR: 4.93e-05 | 1400.0ms/step | 14.3 samples/s +Step 35220/44000 | Loss: 2.8472 | LR: 4.91e-05 | 1400.1ms/step | 14.3 samples/s +Step 35230/44000 | Loss: 2.8822 | LR: 4.90e-05 | 1400.0ms/step | 14.3 samples/s +Step 35240/44000 | Loss: 2.8644 | LR: 4.89e-05 | 1400.1ms/step | 14.3 samples/s +Step 35250/44000 | Loss: 2.8926 | LR: 4.88e-05 | 1399.8ms/step | 14.3 samples/s +Step 35260/44000 | Loss: 2.8936 | LR: 4.87e-05 | 1399.9ms/step | 14.3 samples/s +Step 35270/44000 | Loss: 2.8819 | LR: 4.86e-05 | 1400.1ms/step | 14.3 samples/s +Step 35280/44000 | Loss: 2.8693 | LR: 4.85e-05 | 1400.0ms/step | 14.3 samples/s +Step 35290/44000 | Loss: 2.9100 | LR: 4.84e-05 | 1399.9ms/step | 14.3 samples/s +Step 35300/44000 | Loss: 2.8560 | LR: 4.83e-05 | 1400.3ms/step | 14.3 samples/s +Step 35310/44000 | Loss: 2.8617 | LR: 4.82e-05 | 1399.7ms/step | 14.3 samples/s +Step 35320/44000 | Loss: 2.8964 | LR: 4.81e-05 | 1400.0ms/step | 14.3 samples/s +Step 35330/44000 | Loss: 2.8564 | LR: 4.80e-05 | 1399.8ms/step | 14.3 samples/s +Step 35340/44000 | Loss: 2.8460 | LR: 4.79e-05 | 1400.0ms/step | 14.3 samples/s +Step 35350/44000 | Loss: 2.8685 | LR: 4.77e-05 | 1400.3ms/step | 14.3 samples/s +Step 35360/44000 | Loss: 2.8638 | LR: 4.76e-05 | 1404.3ms/step | 14.2 samples/s +Step 35370/44000 | Loss: 2.8848 | LR: 4.75e-05 | 1399.6ms/step | 14.3 samples/s +Step 35380/44000 | Loss: 2.8740 | LR: 4.74e-05 | 1399.9ms/step | 14.3 samples/s +Step 35390/44000 | Loss: 2.8627 | LR: 4.73e-05 | 1400.0ms/step | 14.3 samples/s +Step 35400/44000 | Loss: 2.8838 | LR: 4.72e-05 | 1400.0ms/step | 14.3 samples/s +Step 35410/44000 | Loss: 2.8585 | LR: 4.71e-05 | 1400.0ms/step | 14.3 samples/s +Step 35420/44000 | Loss: 2.8832 | LR: 4.70e-05 | 1400.2ms/step | 14.3 samples/s +Step 35430/44000 | Loss: 2.8821 | LR: 4.69e-05 | 1399.9ms/step | 14.3 samples/s +Step 35440/44000 | Loss: 2.8691 | LR: 4.68e-05 | 1399.9ms/step | 14.3 samples/s +Step 35450/44000 | Loss: 2.8652 | LR: 4.67e-05 | 1399.9ms/step | 14.3 samples/s +Step 35460/44000 | Loss: 2.8850 | LR: 4.66e-05 | 1400.1ms/step | 14.3 samples/s +Step 35470/44000 | Loss: 2.8704 | LR: 4.65e-05 | 1400.0ms/step | 14.3 samples/s +Step 35480/44000 | Loss: 2.9031 | LR: 4.64e-05 | 1400.0ms/step | 14.3 samples/s +Step 35490/44000 | Loss: 2.8934 | LR: 4.63e-05 | 1400.0ms/step | 14.3 samples/s +Step 35500/44000 | Loss: 2.8994 | LR: 4.62e-05 | 1400.0ms/step | 14.3 samples/s +Step 35510/44000 | Loss: 2.8969 | LR: 4.61e-05 | 1400.0ms/step | 14.3 samples/s +Step 35520/44000 | Loss: 2.8882 | LR: 4.59e-05 | 1400.1ms/step | 14.3 samples/s +Step 35530/44000 | Loss: 2.8765 | LR: 4.58e-05 | 1399.9ms/step | 14.3 samples/s +Step 35540/44000 | Loss: 2.9241 | LR: 4.57e-05 | 1399.9ms/step | 14.3 samples/s +Step 35550/44000 | Loss: 2.8795 | LR: 4.56e-05 | 1400.1ms/step | 14.3 samples/s +Step 35560/44000 | Loss: 2.8745 | LR: 4.55e-05 | 1400.1ms/step | 14.3 samples/s +Step 35570/44000 | Loss: 2.8560 | LR: 4.54e-05 | 1399.7ms/step | 14.3 samples/s +Step 35580/44000 | Loss: 2.8872 | LR: 4.53e-05 | 1400.1ms/step | 14.3 samples/s +Step 35590/44000 | Loss: 2.9026 | LR: 4.52e-05 | 1400.0ms/step | 14.3 samples/s +Step 35600/44000 | Loss: 2.8591 | LR: 4.51e-05 | 1400.3ms/step | 14.3 samples/s +Step 35610/44000 | Loss: 2.8615 | LR: 4.50e-05 | 1399.8ms/step | 14.3 samples/s +Step 35620/44000 | Loss: 2.8852 | LR: 4.49e-05 | 1400.0ms/step | 14.3 samples/s +Step 35630/44000 | Loss: 2.8826 | LR: 4.48e-05 | 1400.1ms/step | 14.3 samples/s +Step 35640/44000 | Loss: 2.8701 | LR: 4.47e-05 | 1399.7ms/step | 14.3 samples/s +Step 35650/44000 | Loss: 2.8466 | LR: 4.46e-05 | 1400.3ms/step | 14.3 samples/s +Step 35660/44000 | Loss: 2.8692 | LR: 4.45e-05 | 1399.8ms/step | 14.3 samples/s +Step 35670/44000 | Loss: 2.8813 | LR: 4.44e-05 | 1400.1ms/step | 14.3 samples/s +Step 35680/44000 | Loss: 2.8541 | LR: 4.43e-05 | 1400.0ms/step | 14.3 samples/s +Step 35690/44000 | Loss: 2.8791 | LR: 4.42e-05 | 1400.0ms/step | 14.3 samples/s +Step 35700/44000 | Loss: 2.8672 | LR: 4.41e-05 | 1399.8ms/step | 14.3 samples/s +Step 35710/44000 | Loss: 2.9318 | LR: 4.40e-05 | 1400.2ms/step | 14.3 samples/s +Step 35720/44000 | Loss: 2.8879 | LR: 4.39e-05 | 1400.0ms/step | 14.3 samples/s +Step 35730/44000 | Loss: 2.8757 | LR: 4.38e-05 | 1400.0ms/step | 14.3 samples/s +Step 35740/44000 | Loss: 2.8863 | LR: 4.37e-05 | 1400.1ms/step | 14.3 samples/s +Step 35750/44000 | Loss: 2.8620 | LR: 4.36e-05 | 1399.9ms/step | 14.3 samples/s +Step 35760/44000 | Loss: 2.9189 | LR: 4.35e-05 | 1399.9ms/step | 14.3 samples/s +Step 35770/44000 | Loss: 2.8910 | LR: 4.34e-05 | 1400.1ms/step | 14.3 samples/s +Step 35780/44000 | Loss: 2.8714 | LR: 4.33e-05 | 1400.1ms/step | 14.3 samples/s +Step 35790/44000 | Loss: 2.8656 | LR: 4.32e-05 | 1400.0ms/step | 14.3 samples/s +Step 35800/44000 | Loss: 2.8939 | LR: 4.31e-05 | 1400.0ms/step | 14.3 samples/s +Step 35810/44000 | Loss: 2.9360 | LR: 4.29e-05 | 1399.9ms/step | 14.3 samples/s +Step 35820/44000 | Loss: 2.8875 | LR: 4.28e-05 | 1400.1ms/step | 14.3 samples/s +Step 35830/44000 | Loss: 2.8765 | LR: 4.27e-05 | 1400.0ms/step | 14.3 samples/s +Step 35840/44000 | Loss: 2.9027 | LR: 4.26e-05 | 1400.0ms/step | 14.3 samples/s +Step 35850/44000 | Loss: 2.8811 | LR: 4.25e-05 | 1400.0ms/step | 14.3 samples/s +Step 35860/44000 | Loss: 2.8822 | LR: 4.24e-05 | 1399.9ms/step | 14.3 samples/s +Step 35870/44000 | Loss: 2.8856 | LR: 4.23e-05 | 1399.9ms/step | 14.3 samples/s +Step 35880/44000 | Loss: 2.8785 | LR: 4.22e-05 | 1400.1ms/step | 14.3 samples/s +Step 35890/44000 | Loss: 2.8664 | LR: 4.21e-05 | 1400.0ms/step | 14.3 samples/s +Step 35900/44000 | Loss: 2.8875 | LR: 4.20e-05 | 1399.8ms/step | 14.3 samples/s +Step 35910/44000 | Loss: 2.8971 | LR: 4.19e-05 | 1400.2ms/step | 14.3 samples/s +Step 35920/44000 | Loss: 2.8799 | LR: 4.18e-05 | 1400.1ms/step | 14.3 samples/s +Step 35930/44000 | Loss: 2.8786 | LR: 4.17e-05 | 1399.8ms/step | 14.3 samples/s +Step 35940/44000 | Loss: 2.9048 | LR: 4.16e-05 | 1400.0ms/step | 14.3 samples/s +Step 35950/44000 | Loss: 2.8718 | LR: 4.15e-05 | 1400.0ms/step | 14.3 samples/s +Step 35960/44000 | Loss: 2.8950 | LR: 4.14e-05 | 1400.0ms/step | 14.3 samples/s +Step 35970/44000 | Loss: 2.8654 | LR: 4.13e-05 | 1400.1ms/step | 14.3 samples/s +Step 35980/44000 | Loss: 2.9018 | LR: 4.12e-05 | 1400.0ms/step | 14.3 samples/s +Step 35990/44000 | Loss: 2.8976 | LR: 4.11e-05 | 1400.1ms/step | 14.3 samples/s +Step 36000/44000 | Loss: 2.8881 | LR: 4.10e-05 | 1400.0ms/step | 14.3 samples/s +Validation at step 36000: Loss = 2.8724, Perplexity = 17.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-36000 +Step 36010/44000 | Loss: 2.9050 | LR: 4.09e-05 | 3913.8ms/step | 5.1 samples/s +Step 36020/44000 | Loss: 2.8968 | LR: 4.08e-05 | 1400.1ms/step | 14.3 samples/s +Step 36030/44000 | Loss: 2.9044 | LR: 4.07e-05 | 1400.0ms/step | 14.3 samples/s +Step 36040/44000 | Loss: 2.9127 | LR: 4.06e-05 | 1400.2ms/step | 14.3 samples/s +Step 36050/44000 | Loss: 2.8837 | LR: 4.05e-05 | 1399.8ms/step | 14.3 samples/s +Step 36060/44000 | Loss: 2.8829 | LR: 4.04e-05 | 1400.1ms/step | 14.3 samples/s +Step 36070/44000 | Loss: 2.8918 | LR: 4.03e-05 | 1399.9ms/step | 14.3 samples/s +Step 36080/44000 | Loss: 2.8954 | LR: 4.02e-05 | 1399.9ms/step | 14.3 samples/s +Step 36090/44000 | Loss: 2.8944 | LR: 4.01e-05 | 1400.1ms/step | 14.3 samples/s +Step 36100/44000 | Loss: 2.9103 | LR: 4.00e-05 | 1400.1ms/step | 14.3 samples/s +Step 36110/44000 | Loss: 2.8803 | LR: 3.99e-05 | 1399.9ms/step | 14.3 samples/s +Step 36120/44000 | Loss: 2.8895 | LR: 3.98e-05 | 1400.0ms/step | 14.3 samples/s +Step 36130/44000 | Loss: 2.8805 | LR: 3.97e-05 | 1400.1ms/step | 14.3 samples/s +Step 36140/44000 | Loss: 2.8945 | LR: 3.97e-05 | 1400.0ms/step | 14.3 samples/s +Step 36150/44000 | Loss: 2.8694 | LR: 3.96e-05 | 1400.0ms/step | 14.3 samples/s +Step 36160/44000 | Loss: 2.8651 | LR: 3.95e-05 | 1400.0ms/step | 14.3 samples/s +Step 36170/44000 | Loss: 2.8782 | LR: 3.94e-05 | 1400.1ms/step | 14.3 samples/s +Step 36180/44000 | Loss: 2.8824 | LR: 3.93e-05 | 1399.9ms/step | 14.3 samples/s +Step 36190/44000 | Loss: 2.8972 | LR: 3.92e-05 | 1400.0ms/step | 14.3 samples/s +Step 36200/44000 | Loss: 2.8486 | LR: 3.91e-05 | 1400.0ms/step | 14.3 samples/s +Step 36210/44000 | Loss: 2.8811 | LR: 3.90e-05 | 1400.0ms/step | 14.3 samples/s +Step 36220/44000 | Loss: 2.8882 | LR: 3.89e-05 | 1400.0ms/step | 14.3 samples/s +Step 36230/44000 | Loss: 2.8750 | LR: 3.88e-05 | 1399.9ms/step | 14.3 samples/s +Step 36240/44000 | Loss: 2.8940 | LR: 3.87e-05 | 1400.1ms/step | 14.3 samples/s +Step 36250/44000 | Loss: 2.8765 | LR: 3.86e-05 | 1400.2ms/step | 14.3 samples/s +Step 36260/44000 | Loss: 2.8940 | LR: 3.85e-05 | 1399.9ms/step | 14.3 samples/s +Step 36270/44000 | Loss: 2.8731 | LR: 3.84e-05 | 1399.9ms/step | 14.3 samples/s +Step 36280/44000 | Loss: 2.8862 | LR: 3.83e-05 | 1400.1ms/step | 14.3 samples/s +Step 36290/44000 | Loss: 2.8608 | LR: 3.82e-05 | 1400.0ms/step | 14.3 samples/s +Step 36300/44000 | Loss: 2.8845 | LR: 3.81e-05 | 1400.0ms/step | 14.3 samples/s +Step 36310/44000 | Loss: 2.8468 | LR: 3.80e-05 | 1400.2ms/step | 14.3 samples/s +Step 36320/44000 | Loss: 2.8747 | LR: 3.79e-05 | 1399.8ms/step | 14.3 samples/s +Step 36330/44000 | Loss: 2.8801 | LR: 3.78e-05 | 1399.8ms/step | 14.3 samples/s +Step 36340/44000 | Loss: 2.8771 | LR: 3.77e-05 | 1400.4ms/step | 14.3 samples/s +Step 36350/44000 | Loss: 2.9063 | LR: 3.76e-05 | 1399.8ms/step | 14.3 samples/s +Step 36360/44000 | Loss: 2.8966 | LR: 3.75e-05 | 1400.0ms/step | 14.3 samples/s +Step 36370/44000 | Loss: 2.8838 | LR: 3.74e-05 | 1400.1ms/step | 14.3 samples/s +Step 36380/44000 | Loss: 2.9052 | LR: 3.73e-05 | 1400.1ms/step | 14.3 samples/s +Step 36390/44000 | Loss: 2.8569 | LR: 3.72e-05 | 1400.1ms/step | 14.3 samples/s +Step 36400/44000 | Loss: 2.8573 | LR: 3.71e-05 | 1399.8ms/step | 14.3 samples/s +Step 36410/44000 | Loss: 2.8940 | LR: 3.70e-05 | 1399.9ms/step | 14.3 samples/s +Step 36420/44000 | Loss: 2.8748 | LR: 3.69e-05 | 1400.1ms/step | 14.3 samples/s +Step 36430/44000 | Loss: 2.8720 | LR: 3.69e-05 | 1400.0ms/step | 14.3 samples/s +Step 36440/44000 | Loss: 2.8505 | LR: 3.68e-05 | 1400.1ms/step | 14.3 samples/s +Step 36450/44000 | Loss: 2.8712 | LR: 3.67e-05 | 1400.0ms/step | 14.3 samples/s +Step 36460/44000 | Loss: 2.9247 | LR: 3.66e-05 | 1399.9ms/step | 14.3 samples/s +Step 36470/44000 | Loss: 2.9024 | LR: 3.65e-05 | 1399.9ms/step | 14.3 samples/s +Step 36480/44000 | Loss: 2.8922 | LR: 3.64e-05 | 1400.4ms/step | 14.3 samples/s +Step 36490/44000 | Loss: 2.8881 | LR: 3.63e-05 | 1399.8ms/step | 14.3 samples/s +Step 36500/44000 | Loss: 2.8886 | LR: 3.62e-05 | 1400.0ms/step | 14.3 samples/s +Step 36510/44000 | Loss: 2.8730 | LR: 3.61e-05 | 1400.1ms/step | 14.3 samples/s +Step 36520/44000 | Loss: 2.8823 | LR: 3.60e-05 | 1399.8ms/step | 14.3 samples/s +Step 36530/44000 | Loss: 2.8835 | LR: 3.59e-05 | 1400.0ms/step | 14.3 samples/s +Step 36540/44000 | Loss: 2.8910 | LR: 3.58e-05 | 1400.1ms/step | 14.3 samples/s +Step 36550/44000 | Loss: 2.8780 | LR: 3.57e-05 | 1399.9ms/step | 14.3 samples/s +Step 36560/44000 | Loss: 2.8885 | LR: 3.56e-05 | 1400.1ms/step | 14.3 samples/s +Step 36570/44000 | Loss: 2.8699 | LR: 3.55e-05 | 1399.8ms/step | 14.3 samples/s +Step 36580/44000 | Loss: 2.8903 | LR: 3.54e-05 | 1400.2ms/step | 14.3 samples/s +Step 36590/44000 | Loss: 2.8909 | LR: 3.53e-05 | 1400.0ms/step | 14.3 samples/s +Step 36600/44000 | Loss: 2.8884 | LR: 3.53e-05 | 1399.8ms/step | 14.3 samples/s +Step 36610/44000 | Loss: 2.8853 | LR: 3.52e-05 | 1400.1ms/step | 14.3 samples/s +Step 36620/44000 | Loss: 2.8657 | LR: 3.51e-05 | 1399.9ms/step | 14.3 samples/s +Step 36630/44000 | Loss: 2.8700 | LR: 3.50e-05 | 1400.1ms/step | 14.3 samples/s +Step 36640/44000 | Loss: 2.8808 | LR: 3.49e-05 | 1399.9ms/step | 14.3 samples/s +Step 36650/44000 | Loss: 2.8542 | LR: 3.48e-05 | 1399.9ms/step | 14.3 samples/s +Step 36660/44000 | Loss: 2.8804 | LR: 3.47e-05 | 1400.2ms/step | 14.3 samples/s +Step 36670/44000 | Loss: 2.8905 | LR: 3.46e-05 | 1399.8ms/step | 14.3 samples/s +Step 36680/44000 | Loss: 2.8806 | LR: 3.45e-05 | 1400.1ms/step | 14.3 samples/s +Step 36690/44000 | Loss: 2.8604 | LR: 3.44e-05 | 1400.1ms/step | 14.3 samples/s +Step 36700/44000 | Loss: 2.8844 | LR: 3.43e-05 | 1399.8ms/step | 14.3 samples/s +Step 36710/44000 | Loss: 2.8809 | LR: 3.42e-05 | 1400.1ms/step | 14.3 samples/s +Step 36720/44000 | Loss: 2.8676 | LR: 3.41e-05 | 1400.1ms/step | 14.3 samples/s +Step 36730/44000 | Loss: 2.8713 | LR: 3.41e-05 | 1399.9ms/step | 14.3 samples/s +Step 36740/44000 | Loss: 2.9108 | LR: 3.40e-05 | 1400.0ms/step | 14.3 samples/s +Step 36750/44000 | Loss: 2.8690 | LR: 3.39e-05 | 1399.9ms/step | 14.3 samples/s +Step 36760/44000 | Loss: 2.8537 | LR: 3.38e-05 | 1399.9ms/step | 14.3 samples/s +Step 36770/44000 | Loss: 2.8933 | LR: 3.37e-05 | 1400.1ms/step | 14.3 samples/s +Step 36780/44000 | Loss: 2.8653 | LR: 3.36e-05 | 1400.2ms/step | 14.3 samples/s +Step 36790/44000 | Loss: 2.8980 | LR: 3.35e-05 | 1399.9ms/step | 14.3 samples/s +Step 36800/44000 | Loss: 2.8706 | LR: 3.34e-05 | 1400.0ms/step | 14.3 samples/s +Step 36810/44000 | Loss: 2.8886 | LR: 3.33e-05 | 1400.0ms/step | 14.3 samples/s +Step 36820/44000 | Loss: 2.8823 | LR: 3.32e-05 | 1399.9ms/step | 14.3 samples/s +Step 36830/44000 | Loss: 2.8661 | LR: 3.31e-05 | 1400.2ms/step | 14.3 samples/s +Step 36840/44000 | Loss: 2.8832 | LR: 3.31e-05 | 1399.9ms/step | 14.3 samples/s +Step 36850/44000 | Loss: 2.8889 | LR: 3.30e-05 | 1400.1ms/step | 14.3 samples/s +Step 36860/44000 | Loss: 2.8955 | LR: 3.29e-05 | 1400.0ms/step | 14.3 samples/s +Step 36870/44000 | Loss: 2.8973 | LR: 3.28e-05 | 1400.0ms/step | 14.3 samples/s +Step 36880/44000 | Loss: 2.8856 | LR: 3.27e-05 | 1399.9ms/step | 14.3 samples/s +Step 36890/44000 | Loss: 2.9182 | LR: 3.26e-05 | 1400.0ms/step | 14.3 samples/s +Step 36900/44000 | Loss: 2.8605 | LR: 3.25e-05 | 1400.1ms/step | 14.3 samples/s +Step 36910/44000 | Loss: 2.8623 | LR: 3.24e-05 | 1400.1ms/step | 14.3 samples/s +Step 36920/44000 | Loss: 2.9037 | LR: 3.23e-05 | 1399.8ms/step | 14.3 samples/s +Step 36930/44000 | Loss: 2.8840 | LR: 3.22e-05 | 1400.1ms/step | 14.3 samples/s +Step 36940/44000 | Loss: 2.8863 | LR: 3.22e-05 | 1399.9ms/step | 14.3 samples/s +Step 36950/44000 | Loss: 2.8937 | LR: 3.21e-05 | 1399.8ms/step | 14.3 samples/s +Step 36960/44000 | Loss: 2.8700 | LR: 3.20e-05 | 1400.2ms/step | 14.3 samples/s +Step 36970/44000 | Loss: 2.8767 | LR: 3.19e-05 | 1400.1ms/step | 14.3 samples/s +Step 36980/44000 | Loss: 2.8900 | LR: 3.18e-05 | 1399.9ms/step | 14.3 samples/s +Step 36990/44000 | Loss: 2.9085 | LR: 3.17e-05 | 1399.7ms/step | 14.3 samples/s +Step 37000/44000 | Loss: 2.8530 | LR: 3.16e-05 | 1399.9ms/step | 14.3 samples/s +Validation at step 37000: Loss = 2.8718, Perplexity = 17.67 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-37000 +Step 37010/44000 | Loss: 2.9004 | LR: 3.15e-05 | 3910.2ms/step | 5.1 samples/s +Step 37020/44000 | Loss: 2.8803 | LR: 3.15e-05 | 1395.5ms/step | 14.3 samples/s +Step 37030/44000 | Loss: 2.9167 | LR: 3.14e-05 | 1395.1ms/step | 14.3 samples/s +Step 37040/44000 | Loss: 2.8532 | LR: 3.13e-05 | 1399.4ms/step | 14.3 samples/s +Step 37050/44000 | Loss: 2.8751 | LR: 3.12e-05 | 1399.9ms/step | 14.3 samples/s +Step 37060/44000 | Loss: 2.9077 | LR: 3.11e-05 | 1400.2ms/step | 14.3 samples/s +Step 37070/44000 | Loss: 2.8452 | LR: 3.10e-05 | 1399.9ms/step | 14.3 samples/s +Step 37080/44000 | Loss: 2.8667 | LR: 3.09e-05 | 1400.2ms/step | 14.3 samples/s +Step 37090/44000 | Loss: 2.8909 | LR: 3.08e-05 | 1399.9ms/step | 14.3 samples/s +Step 37100/44000 | Loss: 2.8880 | LR: 3.07e-05 | 1395.9ms/step | 14.3 samples/s +Step 37110/44000 | Loss: 2.9162 | LR: 3.07e-05 | 1400.2ms/step | 14.3 samples/s +Step 37120/44000 | Loss: 2.8873 | LR: 3.06e-05 | 1399.9ms/step | 14.3 samples/s +Step 37130/44000 | Loss: 2.8682 | LR: 3.05e-05 | 1400.1ms/step | 14.3 samples/s +Step 37140/44000 | Loss: 2.8775 | LR: 3.04e-05 | 1399.9ms/step | 14.3 samples/s +Step 37150/44000 | Loss: 2.8675 | LR: 3.03e-05 | 1400.0ms/step | 14.3 samples/s +Step 37160/44000 | Loss: 2.8913 | LR: 3.02e-05 | 1399.9ms/step | 14.3 samples/s +Step 37170/44000 | Loss: 2.8647 | LR: 3.01e-05 | 1400.2ms/step | 14.3 samples/s +Step 37180/44000 | Loss: 2.8934 | LR: 3.01e-05 | 1400.1ms/step | 14.3 samples/s +Step 37190/44000 | Loss: 2.9245 | LR: 3.00e-05 | 1399.9ms/step | 14.3 samples/s +Step 37200/44000 | Loss: 2.8740 | LR: 2.99e-05 | 1400.1ms/step | 14.3 samples/s +Step 37210/44000 | Loss: 2.8557 | LR: 2.98e-05 | 1400.0ms/step | 14.3 samples/s +Step 37220/44000 | Loss: 2.8942 | LR: 2.97e-05 | 1399.9ms/step | 14.3 samples/s +Step 37230/44000 | Loss: 2.8715 | LR: 2.96e-05 | 1400.2ms/step | 14.3 samples/s +Step 37240/44000 | Loss: 2.8896 | LR: 2.95e-05 | 1400.1ms/step | 14.3 samples/s +Step 37250/44000 | Loss: 2.8819 | LR: 2.95e-05 | 1399.8ms/step | 14.3 samples/s +Step 37260/44000 | Loss: 2.8966 | LR: 2.94e-05 | 1400.3ms/step | 14.3 samples/s +Step 37270/44000 | Loss: 2.8600 | LR: 2.93e-05 | 1399.9ms/step | 14.3 samples/s +Step 37280/44000 | Loss: 2.9020 | LR: 2.92e-05 | 1400.0ms/step | 14.3 samples/s +Step 37290/44000 | Loss: 2.8860 | LR: 2.91e-05 | 1400.0ms/step | 14.3 samples/s +Step 37300/44000 | Loss: 2.8490 | LR: 2.90e-05 | 1399.9ms/step | 14.3 samples/s +Step 37310/44000 | Loss: 2.9012 | LR: 2.89e-05 | 1400.0ms/step | 14.3 samples/s +Step 37320/44000 | Loss: 2.8541 | LR: 2.89e-05 | 1399.8ms/step | 14.3 samples/s +Step 37330/44000 | Loss: 2.8766 | LR: 2.88e-05 | 1400.3ms/step | 14.3 samples/s +Step 37340/44000 | Loss: 2.8931 | LR: 2.87e-05 | 1400.0ms/step | 14.3 samples/s +Step 37350/44000 | Loss: 2.8820 | LR: 2.86e-05 | 1399.8ms/step | 14.3 samples/s +Step 37360/44000 | Loss: 2.8963 | LR: 2.85e-05 | 1400.1ms/step | 14.3 samples/s +Step 37370/44000 | Loss: 2.8602 | LR: 2.84e-05 | 1399.9ms/step | 14.3 samples/s +Step 37380/44000 | Loss: 2.8835 | LR: 2.84e-05 | 1400.0ms/step | 14.3 samples/s +Step 37390/44000 | Loss: 2.8902 | LR: 2.83e-05 | 1399.9ms/step | 14.3 samples/s +Step 37400/44000 | Loss: 2.8653 | LR: 2.82e-05 | 1400.2ms/step | 14.3 samples/s +Step 37410/44000 | Loss: 2.8911 | LR: 2.81e-05 | 1399.9ms/step | 14.3 samples/s +Step 37420/44000 | Loss: 2.8645 | LR: 2.80e-05 | 1400.1ms/step | 14.3 samples/s +Step 37430/44000 | Loss: 2.8834 | LR: 2.79e-05 | 1400.1ms/step | 14.3 samples/s +Step 37440/44000 | Loss: 2.8856 | LR: 2.78e-05 | 1399.7ms/step | 14.3 samples/s +Step 37450/44000 | Loss: 2.8822 | LR: 2.78e-05 | 1400.2ms/step | 14.3 samples/s +Step 37460/44000 | Loss: 2.8690 | LR: 2.77e-05 | 1399.9ms/step | 14.3 samples/s +Step 37470/44000 | Loss: 2.9137 | LR: 2.76e-05 | 1400.1ms/step | 14.3 samples/s +Step 37480/44000 | Loss: 2.8570 | LR: 2.75e-05 | 1400.1ms/step | 14.3 samples/s +Step 37490/44000 | Loss: 2.8682 | LR: 2.74e-05 | 1399.9ms/step | 14.3 samples/s +Step 37500/44000 | Loss: 2.9139 | LR: 2.74e-05 | 1400.0ms/step | 14.3 samples/s +Step 37510/44000 | Loss: 2.8865 | LR: 2.73e-05 | 1400.0ms/step | 14.3 samples/s +Step 37520/44000 | Loss: 2.8792 | LR: 2.72e-05 | 1400.1ms/step | 14.3 samples/s +Step 37530/44000 | Loss: 2.8805 | LR: 2.71e-05 | 1399.9ms/step | 14.3 samples/s +Step 37540/44000 | Loss: 2.8795 | LR: 2.70e-05 | 1400.1ms/step | 14.3 samples/s +Step 37550/44000 | Loss: 2.8904 | LR: 2.69e-05 | 1400.0ms/step | 14.3 samples/s +Step 37560/44000 | Loss: 2.9206 | LR: 2.69e-05 | 1399.8ms/step | 14.3 samples/s +Step 37570/44000 | Loss: 2.8625 | LR: 2.68e-05 | 1400.3ms/step | 14.3 samples/s +Step 37580/44000 | Loss: 2.8755 | LR: 2.67e-05 | 1400.1ms/step | 14.3 samples/s +Step 37590/44000 | Loss: 2.8804 | LR: 2.66e-05 | 1399.9ms/step | 14.3 samples/s +Step 37600/44000 | Loss: 2.8293 | LR: 2.65e-05 | 1400.0ms/step | 14.3 samples/s +Step 37610/44000 | Loss: 2.8677 | LR: 2.65e-05 | 1400.0ms/step | 14.3 samples/s +Step 37620/44000 | Loss: 2.8907 | LR: 2.64e-05 | 1399.8ms/step | 14.3 samples/s +Step 37630/44000 | Loss: 2.9012 | LR: 2.63e-05 | 1400.0ms/step | 14.3 samples/s +Step 37640/44000 | Loss: 2.9040 | LR: 2.62e-05 | 1400.3ms/step | 14.3 samples/s +Step 37650/44000 | Loss: 2.9089 | LR: 2.61e-05 | 1399.8ms/step | 14.3 samples/s +Step 37660/44000 | Loss: 2.8776 | LR: 2.60e-05 | 1400.0ms/step | 14.3 samples/s +Step 37670/44000 | Loss: 2.8935 | LR: 2.60e-05 | 1400.0ms/step | 14.3 samples/s +Step 37680/44000 | Loss: 2.8637 | LR: 2.59e-05 | 1400.1ms/step | 14.3 samples/s +Step 37690/44000 | Loss: 2.8643 | LR: 2.58e-05 | 1399.9ms/step | 14.3 samples/s +Step 37700/44000 | Loss: 2.8546 | LR: 2.57e-05 | 1399.7ms/step | 14.3 samples/s +Step 37710/44000 | Loss: 2.8956 | LR: 2.56e-05 | 1400.3ms/step | 14.3 samples/s +Step 37720/44000 | Loss: 2.9029 | LR: 2.56e-05 | 1400.0ms/step | 14.3 samples/s +Step 37730/44000 | Loss: 2.8792 | LR: 2.55e-05 | 1399.8ms/step | 14.3 samples/s +Step 37740/44000 | Loss: 2.8873 | LR: 2.54e-05 | 1400.2ms/step | 14.3 samples/s +Step 37750/44000 | Loss: 2.8816 | LR: 2.53e-05 | 1399.8ms/step | 14.3 samples/s +Step 37760/44000 | Loss: 2.8873 | LR: 2.52e-05 | 1400.2ms/step | 14.3 samples/s +Step 37770/44000 | Loss: 2.9019 | LR: 2.52e-05 | 1400.1ms/step | 14.3 samples/s +Step 37780/44000 | Loss: 2.9293 | LR: 2.51e-05 | 1399.9ms/step | 14.3 samples/s +Step 37790/44000 | Loss: 2.8969 | LR: 2.50e-05 | 1400.0ms/step | 14.3 samples/s +Step 37800/44000 | Loss: 2.9032 | LR: 2.49e-05 | 1400.0ms/step | 14.3 samples/s +Step 37810/44000 | Loss: 2.8694 | LR: 2.48e-05 | 1400.1ms/step | 14.3 samples/s +Step 37820/44000 | Loss: 2.8587 | LR: 2.48e-05 | 1399.8ms/step | 14.3 samples/s +Step 37830/44000 | Loss: 2.8600 | LR: 2.47e-05 | 1400.1ms/step | 14.3 samples/s +Step 37840/44000 | Loss: 2.8711 | LR: 2.46e-05 | 1400.1ms/step | 14.3 samples/s +Step 37850/44000 | Loss: 2.8695 | LR: 2.45e-05 | 1399.8ms/step | 14.3 samples/s +Step 37860/44000 | Loss: 2.8729 | LR: 2.45e-05 | 1400.0ms/step | 14.3 samples/s +Step 37870/44000 | Loss: 2.8891 | LR: 2.44e-05 | 1400.1ms/step | 14.3 samples/s +Step 37880/44000 | Loss: 2.8975 | LR: 2.43e-05 | 1400.1ms/step | 14.3 samples/s +Step 37890/44000 | Loss: 2.8653 | LR: 2.42e-05 | 1399.9ms/step | 14.3 samples/s +Step 37900/44000 | Loss: 2.8829 | LR: 2.41e-05 | 1400.0ms/step | 14.3 samples/s +Step 37910/44000 | Loss: 2.8768 | LR: 2.41e-05 | 1399.9ms/step | 14.3 samples/s +Step 37920/44000 | Loss: 2.8885 | LR: 2.40e-05 | 1400.0ms/step | 14.3 samples/s +Step 37930/44000 | Loss: 2.9038 | LR: 2.39e-05 | 1399.9ms/step | 14.3 samples/s +Step 37940/44000 | Loss: 2.8849 | LR: 2.38e-05 | 1400.0ms/step | 14.3 samples/s +Step 37950/44000 | Loss: 2.9056 | LR: 2.38e-05 | 1400.2ms/step | 14.3 samples/s +Step 37960/44000 | Loss: 2.8686 | LR: 2.37e-05 | 1399.9ms/step | 14.3 samples/s +Step 37970/44000 | Loss: 2.8338 | LR: 2.36e-05 | 1399.9ms/step | 14.3 samples/s +Step 37980/44000 | Loss: 2.8845 | LR: 2.35e-05 | 1400.1ms/step | 14.3 samples/s +Step 37990/44000 | Loss: 2.8490 | LR: 2.34e-05 | 1399.9ms/step | 14.3 samples/s +Step 38000/44000 | Loss: 2.8792 | LR: 2.34e-05 | 1400.0ms/step | 14.3 samples/s +Validation at step 38000: Loss = 2.8714, Perplexity = 17.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-38000 +Step 38010/44000 | Loss: 2.8448 | LR: 2.33e-05 | 3908.0ms/step | 5.1 samples/s +Step 38020/44000 | Loss: 2.8891 | LR: 2.32e-05 | 1399.8ms/step | 14.3 samples/s +Step 38030/44000 | Loss: 2.8707 | LR: 2.31e-05 | 1400.1ms/step | 14.3 samples/s +Step 38040/44000 | Loss: 2.8553 | LR: 2.31e-05 | 1399.8ms/step | 14.3 samples/s +Step 38050/44000 | Loss: 2.8487 | LR: 2.30e-05 | 1400.3ms/step | 14.3 samples/s +Step 38060/44000 | Loss: 2.8649 | LR: 2.29e-05 | 1400.1ms/step | 14.3 samples/s +Step 38070/44000 | Loss: 2.8792 | LR: 2.28e-05 | 1400.0ms/step | 14.3 samples/s +Step 38080/44000 | Loss: 2.8779 | LR: 2.28e-05 | 1399.9ms/step | 14.3 samples/s +Step 38090/44000 | Loss: 2.8492 | LR: 2.27e-05 | 1400.2ms/step | 14.3 samples/s +Step 38100/44000 | Loss: 2.9010 | LR: 2.26e-05 | 1399.7ms/step | 14.3 samples/s +Step 38110/44000 | Loss: 2.8593 | LR: 2.25e-05 | 1400.1ms/step | 14.3 samples/s +Step 38120/44000 | Loss: 2.8863 | LR: 2.25e-05 | 1399.8ms/step | 14.3 samples/s +Step 38130/44000 | Loss: 2.8676 | LR: 2.24e-05 | 1400.1ms/step | 14.3 samples/s +Step 38140/44000 | Loss: 2.9112 | LR: 2.23e-05 | 1399.9ms/step | 14.3 samples/s +Step 38150/44000 | Loss: 2.9008 | LR: 2.22e-05 | 1400.1ms/step | 14.3 samples/s +Step 38160/44000 | Loss: 2.8733 | LR: 2.22e-05 | 1400.3ms/step | 14.3 samples/s +Step 38170/44000 | Loss: 2.8780 | LR: 2.21e-05 | 1399.8ms/step | 14.3 samples/s +Step 38180/44000 | Loss: 2.8468 | LR: 2.20e-05 | 1399.9ms/step | 14.3 samples/s +Step 38190/44000 | Loss: 2.8862 | LR: 2.19e-05 | 1400.0ms/step | 14.3 samples/s +Step 38200/44000 | Loss: 2.8950 | LR: 2.19e-05 | 1400.0ms/step | 14.3 samples/s +Step 38210/44000 | Loss: 2.8752 | LR: 2.18e-05 | 1399.9ms/step | 14.3 samples/s +Step 38220/44000 | Loss: 2.8639 | LR: 2.17e-05 | 1399.9ms/step | 14.3 samples/s +Step 38230/44000 | Loss: 2.8939 | LR: 2.16e-05 | 1400.1ms/step | 14.3 samples/s +Step 38240/44000 | Loss: 2.8847 | LR: 2.16e-05 | 1400.0ms/step | 14.3 samples/s +Step 38250/44000 | Loss: 2.8670 | LR: 2.15e-05 | 1399.9ms/step | 14.3 samples/s +Step 38260/44000 | Loss: 2.8502 | LR: 2.14e-05 | 1400.0ms/step | 14.3 samples/s +Step 38270/44000 | Loss: 2.9130 | LR: 2.13e-05 | 1400.0ms/step | 14.3 samples/s +Step 38280/44000 | Loss: 2.8928 | LR: 2.13e-05 | 1400.1ms/step | 14.3 samples/s +Step 38290/44000 | Loss: 2.8929 | LR: 2.12e-05 | 1400.1ms/step | 14.3 samples/s +Step 38300/44000 | Loss: 2.8854 | LR: 2.11e-05 | 1399.8ms/step | 14.3 samples/s +Step 38310/44000 | Loss: 2.8681 | LR: 2.11e-05 | 1400.0ms/step | 14.3 samples/s +Step 38320/44000 | Loss: 2.8707 | LR: 2.10e-05 | 1400.0ms/step | 14.3 samples/s +Step 38330/44000 | Loss: 2.8828 | LR: 2.09e-05 | 1399.8ms/step | 14.3 samples/s +Step 38340/44000 | Loss: 2.8748 | LR: 2.08e-05 | 1400.2ms/step | 14.3 samples/s +Step 38350/44000 | Loss: 2.8788 | LR: 2.08e-05 | 1400.0ms/step | 14.3 samples/s +Step 38360/44000 | Loss: 2.8843 | LR: 2.07e-05 | 1400.0ms/step | 14.3 samples/s +Step 38370/44000 | Loss: 2.9093 | LR: 2.06e-05 | 1400.1ms/step | 14.3 samples/s +Step 38380/44000 | Loss: 2.8644 | LR: 2.05e-05 | 1400.0ms/step | 14.3 samples/s +Step 38390/44000 | Loss: 2.8645 | LR: 2.05e-05 | 1399.9ms/step | 14.3 samples/s +Step 38400/44000 | Loss: 2.8658 | LR: 2.04e-05 | 1400.0ms/step | 14.3 samples/s +Step 38410/44000 | Loss: 2.8703 | LR: 2.03e-05 | 1400.0ms/step | 14.3 samples/s +Step 38420/44000 | Loss: 2.9054 | LR: 2.03e-05 | 1400.0ms/step | 14.3 samples/s +Step 38430/44000 | Loss: 2.8729 | LR: 2.02e-05 | 1399.8ms/step | 14.3 samples/s +Step 38440/44000 | Loss: 2.8964 | LR: 2.01e-05 | 1400.0ms/step | 14.3 samples/s +Step 38450/44000 | Loss: 2.8329 | LR: 2.00e-05 | 1400.0ms/step | 14.3 samples/s +Step 38460/44000 | Loss: 2.8670 | LR: 2.00e-05 | 1400.1ms/step | 14.3 samples/s +Step 38470/44000 | Loss: 2.8981 | LR: 1.99e-05 | 1400.0ms/step | 14.3 samples/s +Step 38480/44000 | Loss: 2.8998 | LR: 1.98e-05 | 1399.9ms/step | 14.3 samples/s +Step 38490/44000 | Loss: 2.8975 | LR: 1.98e-05 | 1400.2ms/step | 14.3 samples/s +Step 38500/44000 | Loss: 2.8689 | LR: 1.97e-05 | 1399.9ms/step | 14.3 samples/s +Step 38510/44000 | Loss: 2.9003 | LR: 1.96e-05 | 1399.8ms/step | 14.3 samples/s +Step 38520/44000 | Loss: 2.8586 | LR: 1.95e-05 | 1400.1ms/step | 14.3 samples/s +Step 38530/44000 | Loss: 2.8896 | LR: 1.95e-05 | 1400.0ms/step | 14.3 samples/s +Step 38540/44000 | Loss: 2.8847 | LR: 1.94e-05 | 1400.0ms/step | 14.3 samples/s +Step 38550/44000 | Loss: 2.8780 | LR: 1.93e-05 | 1400.0ms/step | 14.3 samples/s +Step 38560/44000 | Loss: 2.9097 | LR: 1.93e-05 | 1400.0ms/step | 14.3 samples/s +Step 38570/44000 | Loss: 2.8472 | LR: 1.92e-05 | 1400.0ms/step | 14.3 samples/s +Step 38580/44000 | Loss: 2.9014 | LR: 1.91e-05 | 1400.0ms/step | 14.3 samples/s +Step 38590/44000 | Loss: 2.8863 | LR: 1.91e-05 | 1400.2ms/step | 14.3 samples/s +Step 38600/44000 | Loss: 2.9114 | LR: 1.90e-05 | 1399.9ms/step | 14.3 samples/s +Step 38610/44000 | Loss: 2.8711 | LR: 1.89e-05 | 1399.9ms/step | 14.3 samples/s +Step 38620/44000 | Loss: 2.8905 | LR: 1.88e-05 | 1400.0ms/step | 14.3 samples/s +Step 38630/44000 | Loss: 2.8873 | LR: 1.88e-05 | 1399.9ms/step | 14.3 samples/s +Step 38640/44000 | Loss: 2.8932 | LR: 1.87e-05 | 1400.0ms/step | 14.3 samples/s +Step 38650/44000 | Loss: 2.8858 | LR: 1.86e-05 | 1400.0ms/step | 14.3 samples/s +Step 38660/44000 | Loss: 2.8840 | LR: 1.86e-05 | 1400.1ms/step | 14.3 samples/s +Step 38670/44000 | Loss: 2.9156 | LR: 1.85e-05 | 1400.2ms/step | 14.3 samples/s +Step 38680/44000 | Loss: 2.8828 | LR: 1.84e-05 | 1400.0ms/step | 14.3 samples/s +Step 38690/44000 | Loss: 2.8981 | LR: 1.84e-05 | 1399.8ms/step | 14.3 samples/s +Step 38700/44000 | Loss: 2.8704 | LR: 1.83e-05 | 1400.0ms/step | 14.3 samples/s +Step 38710/44000 | Loss: 2.8801 | LR: 1.82e-05 | 1399.9ms/step | 14.3 samples/s +Step 38720/44000 | Loss: 2.8514 | LR: 1.82e-05 | 1400.0ms/step | 14.3 samples/s +Step 38730/44000 | Loss: 2.8693 | LR: 1.81e-05 | 1400.1ms/step | 14.3 samples/s +Step 38740/44000 | Loss: 2.8943 | LR: 1.80e-05 | 1400.0ms/step | 14.3 samples/s +Step 38750/44000 | Loss: 2.8645 | LR: 1.80e-05 | 1400.0ms/step | 14.3 samples/s +Step 38760/44000 | Loss: 2.8954 | LR: 1.79e-05 | 1400.0ms/step | 14.3 samples/s +Step 38770/44000 | Loss: 2.8688 | LR: 1.78e-05 | 1400.0ms/step | 14.3 samples/s +Step 38780/44000 | Loss: 2.9122 | LR: 1.78e-05 | 1399.9ms/step | 14.3 samples/s +Step 38790/44000 | Loss: 2.8661 | LR: 1.77e-05 | 1400.1ms/step | 14.3 samples/s +Step 38800/44000 | Loss: 2.8500 | LR: 1.76e-05 | 1399.8ms/step | 14.3 samples/s +Step 38810/44000 | Loss: 2.8736 | LR: 1.76e-05 | 1400.0ms/step | 14.3 samples/s +Step 38820/44000 | Loss: 2.8827 | LR: 1.75e-05 | 1399.9ms/step | 14.3 samples/s +Step 38830/44000 | Loss: 2.8943 | LR: 1.74e-05 | 1400.1ms/step | 14.3 samples/s +Step 38840/44000 | Loss: 2.8565 | LR: 1.74e-05 | 1400.1ms/step | 14.3 samples/s +Step 38850/44000 | Loss: 2.8874 | LR: 1.73e-05 | 1399.9ms/step | 14.3 samples/s +Step 38860/44000 | Loss: 2.8688 | LR: 1.72e-05 | 1400.1ms/step | 14.3 samples/s +Step 38870/44000 | Loss: 2.8567 | LR: 1.72e-05 | 1400.0ms/step | 14.3 samples/s +Step 38880/44000 | Loss: 2.8725 | LR: 1.71e-05 | 1399.9ms/step | 14.3 samples/s +Step 38890/44000 | Loss: 2.9141 | LR: 1.70e-05 | 1400.0ms/step | 14.3 samples/s +Step 38900/44000 | Loss: 2.9115 | LR: 1.70e-05 | 1400.1ms/step | 14.3 samples/s +Step 38910/44000 | Loss: 2.8684 | LR: 1.69e-05 | 1400.0ms/step | 14.3 samples/s +Step 38920/44000 | Loss: 2.8830 | LR: 1.68e-05 | 1399.9ms/step | 14.3 samples/s +Step 38930/44000 | Loss: 2.8486 | LR: 1.68e-05 | 1400.1ms/step | 14.3 samples/s +Step 38940/44000 | Loss: 2.8576 | LR: 1.67e-05 | 1400.1ms/step | 14.3 samples/s +Step 38950/44000 | Loss: 2.8853 | LR: 1.66e-05 | 1399.7ms/step | 14.3 samples/s +Step 38960/44000 | Loss: 2.8839 | LR: 1.66e-05 | 1400.1ms/step | 14.3 samples/s +Step 38970/44000 | Loss: 2.8880 | LR: 1.65e-05 | 1400.0ms/step | 14.3 samples/s +Step 38980/44000 | Loss: 2.8629 | LR: 1.64e-05 | 1400.0ms/step | 14.3 samples/s +Step 38990/44000 | Loss: 2.8603 | LR: 1.64e-05 | 1399.9ms/step | 14.3 samples/s +Step 39000/44000 | Loss: 2.8737 | LR: 1.63e-05 | 1400.1ms/step | 14.3 samples/s +Validation at step 39000: Loss = 2.8712, Perplexity = 17.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-39000 +Step 39010/44000 | Loss: 2.8781 | LR: 1.62e-05 | 3905.6ms/step | 5.1 samples/s +Step 39020/44000 | Loss: 2.8881 | LR: 1.62e-05 | 1394.4ms/step | 14.3 samples/s +Step 39030/44000 | Loss: 2.9120 | LR: 1.61e-05 | 1395.9ms/step | 14.3 samples/s +Step 39040/44000 | Loss: 2.8977 | LR: 1.61e-05 | 1398.0ms/step | 14.3 samples/s +Step 39050/44000 | Loss: 2.8671 | LR: 1.60e-05 | 1400.2ms/step | 14.3 samples/s +Step 39060/44000 | Loss: 2.8735 | LR: 1.59e-05 | 1395.7ms/step | 14.3 samples/s +Step 39070/44000 | Loss: 2.8931 | LR: 1.59e-05 | 1395.9ms/step | 14.3 samples/s +Step 39080/44000 | Loss: 2.8629 | LR: 1.58e-05 | 1393.7ms/step | 14.3 samples/s +Step 39090/44000 | Loss: 2.8760 | LR: 1.57e-05 | 1397.9ms/step | 14.3 samples/s +Step 39100/44000 | Loss: 2.8889 | LR: 1.57e-05 | 1398.5ms/step | 14.3 samples/s +Step 39110/44000 | Loss: 2.8649 | LR: 1.56e-05 | 1399.7ms/step | 14.3 samples/s +Step 39120/44000 | Loss: 2.8495 | LR: 1.55e-05 | 1400.1ms/step | 14.3 samples/s +Step 39130/44000 | Loss: 2.8898 | LR: 1.55e-05 | 1398.1ms/step | 14.3 samples/s +Step 39140/44000 | Loss: 2.8614 | LR: 1.54e-05 | 1400.1ms/step | 14.3 samples/s +Step 39150/44000 | Loss: 2.8663 | LR: 1.54e-05 | 1400.2ms/step | 14.3 samples/s +Step 39160/44000 | Loss: 2.9113 | LR: 1.53e-05 | 1399.8ms/step | 14.3 samples/s +Step 39170/44000 | Loss: 2.8755 | LR: 1.52e-05 | 1400.1ms/step | 14.3 samples/s +Step 39180/44000 | Loss: 2.9056 | LR: 1.52e-05 | 1399.8ms/step | 14.3 samples/s +Step 39190/44000 | Loss: 2.8587 | LR: 1.51e-05 | 1399.8ms/step | 14.3 samples/s +Step 39200/44000 | Loss: 2.8752 | LR: 1.50e-05 | 1400.3ms/step | 14.3 samples/s +Step 39210/44000 | Loss: 2.8792 | LR: 1.50e-05 | 1399.8ms/step | 14.3 samples/s +Step 39220/44000 | Loss: 2.8687 | LR: 1.49e-05 | 1400.3ms/step | 14.3 samples/s +Step 39230/44000 | Loss: 2.8686 | LR: 1.49e-05 | 1399.7ms/step | 14.3 samples/s +Step 39240/44000 | Loss: 2.8513 | LR: 1.48e-05 | 1398.1ms/step | 14.3 samples/s +Step 39250/44000 | Loss: 2.8520 | LR: 1.47e-05 | 1399.8ms/step | 14.3 samples/s +Step 39260/44000 | Loss: 2.8660 | LR: 1.47e-05 | 1400.2ms/step | 14.3 samples/s +Step 39270/44000 | Loss: 2.8795 | LR: 1.46e-05 | 1399.6ms/step | 14.3 samples/s +Step 39280/44000 | Loss: 2.8839 | LR: 1.46e-05 | 1398.3ms/step | 14.3 samples/s +Step 39290/44000 | Loss: 2.8403 | LR: 1.45e-05 | 1400.1ms/step | 14.3 samples/s +Step 39300/44000 | Loss: 2.8449 | LR: 1.44e-05 | 1397.7ms/step | 14.3 samples/s +Step 39310/44000 | Loss: 2.8472 | LR: 1.44e-05 | 1398.2ms/step | 14.3 samples/s +Step 39320/44000 | Loss: 2.8882 | LR: 1.43e-05 | 1398.1ms/step | 14.3 samples/s +Step 39330/44000 | Loss: 2.8809 | LR: 1.42e-05 | 1398.0ms/step | 14.3 samples/s +Step 39340/44000 | Loss: 2.8876 | LR: 1.42e-05 | 1400.1ms/step | 14.3 samples/s +Step 39350/44000 | Loss: 2.8530 | LR: 1.41e-05 | 1400.3ms/step | 14.3 samples/s +Step 39360/44000 | Loss: 2.8511 | LR: 1.41e-05 | 1399.8ms/step | 14.3 samples/s +Step 39370/44000 | Loss: 2.8780 | LR: 1.40e-05 | 1399.9ms/step | 14.3 samples/s +Step 39380/44000 | Loss: 2.8939 | LR: 1.39e-05 | 1400.2ms/step | 14.3 samples/s +Step 39390/44000 | Loss: 2.8532 | LR: 1.39e-05 | 1399.8ms/step | 14.3 samples/s +Step 39400/44000 | Loss: 2.8649 | LR: 1.38e-05 | 1398.0ms/step | 14.3 samples/s +Step 39410/44000 | Loss: 2.8802 | LR: 1.38e-05 | 1398.1ms/step | 14.3 samples/s +Step 39420/44000 | Loss: 2.8486 | LR: 1.37e-05 | 1400.1ms/step | 14.3 samples/s +Step 39430/44000 | Loss: 2.8789 | LR: 1.36e-05 | 1400.0ms/step | 14.3 samples/s +Step 39440/44000 | Loss: 2.8894 | LR: 1.36e-05 | 1399.7ms/step | 14.3 samples/s +Step 39450/44000 | Loss: 2.8879 | LR: 1.35e-05 | 1400.4ms/step | 14.3 samples/s +Step 39460/44000 | Loss: 2.8918 | LR: 1.35e-05 | 1397.5ms/step | 14.3 samples/s +Step 39470/44000 | Loss: 2.8808 | LR: 1.34e-05 | 1398.6ms/step | 14.3 samples/s +Step 39480/44000 | Loss: 2.8957 | LR: 1.34e-05 | 1399.7ms/step | 14.3 samples/s +Step 39490/44000 | Loss: 2.8864 | LR: 1.33e-05 | 1400.0ms/step | 14.3 samples/s +Step 39500/44000 | Loss: 2.8999 | LR: 1.32e-05 | 1397.7ms/step | 14.3 samples/s +Step 39510/44000 | Loss: 2.8756 | LR: 1.32e-05 | 1400.3ms/step | 14.3 samples/s +Step 39520/44000 | Loss: 2.8633 | LR: 1.31e-05 | 1400.0ms/step | 14.3 samples/s +Step 39530/44000 | Loss: 2.8569 | LR: 1.31e-05 | 1400.1ms/step | 14.3 samples/s +Step 39540/44000 | Loss: 2.9014 | LR: 1.30e-05 | 1399.8ms/step | 14.3 samples/s +Step 39550/44000 | Loss: 2.8544 | LR: 1.29e-05 | 1400.0ms/step | 14.3 samples/s +Step 39560/44000 | Loss: 2.8390 | LR: 1.29e-05 | 1399.9ms/step | 14.3 samples/s +Step 39570/44000 | Loss: 2.8870 | LR: 1.28e-05 | 1400.2ms/step | 14.3 samples/s +Step 39580/44000 | Loss: 2.8813 | LR: 1.28e-05 | 1400.0ms/step | 14.3 samples/s +Step 39590/44000 | Loss: 2.8623 | LR: 1.27e-05 | 1399.9ms/step | 14.3 samples/s +Step 39600/44000 | Loss: 2.8364 | LR: 1.27e-05 | 1399.9ms/step | 14.3 samples/s +Step 39610/44000 | Loss: 2.8790 | LR: 1.26e-05 | 1400.1ms/step | 14.3 samples/s +Step 39620/44000 | Loss: 2.8563 | LR: 1.25e-05 | 1400.0ms/step | 14.3 samples/s +Step 39630/44000 | Loss: 2.8689 | LR: 1.25e-05 | 1400.0ms/step | 14.3 samples/s +Step 39640/44000 | Loss: 2.8965 | LR: 1.24e-05 | 1400.0ms/step | 14.3 samples/s +Step 39650/44000 | Loss: 2.8719 | LR: 1.24e-05 | 1400.0ms/step | 14.3 samples/s +Step 39660/44000 | Loss: 2.8896 | LR: 1.23e-05 | 1399.7ms/step | 14.3 samples/s +Step 39670/44000 | Loss: 2.8640 | LR: 1.23e-05 | 1400.4ms/step | 14.3 samples/s +Step 39680/44000 | Loss: 2.8777 | LR: 1.22e-05 | 1400.0ms/step | 14.3 samples/s +Step 39690/44000 | Loss: 2.8755 | LR: 1.22e-05 | 1400.0ms/step | 14.3 samples/s +Step 39700/44000 | Loss: 2.8802 | LR: 1.21e-05 | 1400.0ms/step | 14.3 samples/s +Step 39710/44000 | Loss: 2.9109 | LR: 1.20e-05 | 1399.9ms/step | 14.3 samples/s +Step 39720/44000 | Loss: 2.9006 | LR: 1.20e-05 | 1399.9ms/step | 14.3 samples/s +Step 39730/44000 | Loss: 2.8916 | LR: 1.19e-05 | 1400.0ms/step | 14.3 samples/s +Step 39740/44000 | Loss: 2.8954 | LR: 1.19e-05 | 1399.9ms/step | 14.3 samples/s +Step 39750/44000 | Loss: 2.8742 | LR: 1.18e-05 | 1400.3ms/step | 14.3 samples/s +Step 39760/44000 | Loss: 2.8837 | LR: 1.18e-05 | 1399.9ms/step | 14.3 samples/s +Step 39770/44000 | Loss: 2.8776 | LR: 1.17e-05 | 1400.2ms/step | 14.3 samples/s +Step 39780/44000 | Loss: 2.9176 | LR: 1.17e-05 | 1400.0ms/step | 14.3 samples/s +Step 39790/44000 | Loss: 2.8711 | LR: 1.16e-05 | 1397.5ms/step | 14.3 samples/s +Step 39800/44000 | Loss: 2.8905 | LR: 1.15e-05 | 1398.3ms/step | 14.3 samples/s +Step 39810/44000 | Loss: 2.8794 | LR: 1.15e-05 | 1397.9ms/step | 14.3 samples/s +Step 39820/44000 | Loss: 2.8698 | LR: 1.14e-05 | 1398.5ms/step | 14.3 samples/s +Step 39830/44000 | Loss: 2.8573 | LR: 1.14e-05 | 1399.3ms/step | 14.3 samples/s +Step 39840/44000 | Loss: 2.8643 | LR: 1.13e-05 | 1398.3ms/step | 14.3 samples/s +Step 39850/44000 | Loss: 2.8842 | LR: 1.13e-05 | 1397.8ms/step | 14.3 samples/s +Step 39860/44000 | Loss: 2.8880 | LR: 1.12e-05 | 1400.0ms/step | 14.3 samples/s +Step 39870/44000 | Loss: 2.8963 | LR: 1.12e-05 | 1398.1ms/step | 14.3 samples/s +Step 39880/44000 | Loss: 2.8770 | LR: 1.11e-05 | 1400.2ms/step | 14.3 samples/s +Step 39890/44000 | Loss: 2.8988 | LR: 1.11e-05 | 1395.8ms/step | 14.3 samples/s +Step 39900/44000 | Loss: 2.8702 | LR: 1.10e-05 | 1399.6ms/step | 14.3 samples/s +Step 39910/44000 | Loss: 2.8816 | LR: 1.10e-05 | 1394.7ms/step | 14.3 samples/s +Step 39920/44000 | Loss: 2.9046 | LR: 1.09e-05 | 1397.6ms/step | 14.3 samples/s +Step 39930/44000 | Loss: 2.8879 | LR: 1.08e-05 | 1398.1ms/step | 14.3 samples/s +Step 39940/44000 | Loss: 2.8753 | LR: 1.08e-05 | 1398.1ms/step | 14.3 samples/s +Step 39950/44000 | Loss: 2.8741 | LR: 1.07e-05 | 1399.1ms/step | 14.3 samples/s +Step 39960/44000 | Loss: 2.8674 | LR: 1.07e-05 | 1399.1ms/step | 14.3 samples/s +Step 39970/44000 | Loss: 2.8751 | LR: 1.06e-05 | 1399.7ms/step | 14.3 samples/s +Step 39980/44000 | Loss: 2.8890 | LR: 1.06e-05 | 1396.5ms/step | 14.3 samples/s +Step 39990/44000 | Loss: 2.8803 | LR: 1.05e-05 | 1400.0ms/step | 14.3 samples/s +Step 40000/44000 | Loss: 2.9044 | LR: 1.05e-05 | 1399.7ms/step | 14.3 samples/s +Validation at step 40000: Loss = 2.8712, Perplexity = 17.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-40000 +Step 40010/44000 | Loss: 2.9256 | LR: 1.04e-05 | 3882.3ms/step | 5.2 samples/s +Step 40020/44000 | Loss: 2.9018 | LR: 1.04e-05 | 1400.0ms/step | 14.3 samples/s +Step 40030/44000 | Loss: 2.8629 | LR: 1.03e-05 | 1399.7ms/step | 14.3 samples/s +Step 40040/44000 | Loss: 2.8748 | LR: 1.03e-05 | 1400.3ms/step | 14.3 samples/s +Step 40050/44000 | Loss: 2.8825 | LR: 1.02e-05 | 1400.1ms/step | 14.3 samples/s +Step 40060/44000 | Loss: 2.8725 | LR: 1.02e-05 | 1400.2ms/step | 14.3 samples/s +Step 40070/44000 | Loss: 2.8960 | LR: 1.01e-05 | 1399.6ms/step | 14.3 samples/s +Step 40080/44000 | Loss: 2.8669 | LR: 1.01e-05 | 1399.9ms/step | 14.3 samples/s +Step 40090/44000 | Loss: 2.8490 | LR: 1.00e-05 | 1400.3ms/step | 14.3 samples/s +Step 40100/44000 | Loss: 2.8796 | LR: 9.96e-06 | 1399.8ms/step | 14.3 samples/s +Step 40110/44000 | Loss: 2.8428 | LR: 9.91e-06 | 1400.1ms/step | 14.3 samples/s +Step 40120/44000 | Loss: 2.8721 | LR: 9.86e-06 | 1400.0ms/step | 14.3 samples/s +Step 40130/44000 | Loss: 2.8821 | LR: 9.81e-06 | 1400.2ms/step | 14.3 samples/s +Step 40140/44000 | Loss: 2.8819 | LR: 9.76e-06 | 1399.7ms/step | 14.3 samples/s +Step 40150/44000 | Loss: 2.9044 | LR: 9.71e-06 | 1400.2ms/step | 14.3 samples/s +Step 40160/44000 | Loss: 2.8735 | LR: 9.66e-06 | 1400.1ms/step | 14.3 samples/s +Step 40170/44000 | Loss: 2.8806 | LR: 9.61e-06 | 1399.9ms/step | 14.3 samples/s +Step 40180/44000 | Loss: 2.8697 | LR: 9.56e-06 | 1400.1ms/step | 14.3 samples/s +Step 40190/44000 | Loss: 2.8632 | LR: 9.51e-06 | 1399.9ms/step | 14.3 samples/s +Step 40200/44000 | Loss: 2.8906 | LR: 9.46e-06 | 1400.0ms/step | 14.3 samples/s +Step 40210/44000 | Loss: 2.8941 | LR: 9.41e-06 | 1399.9ms/step | 14.3 samples/s +Step 40220/44000 | Loss: 2.8716 | LR: 9.36e-06 | 1400.0ms/step | 14.3 samples/s +Step 40230/44000 | Loss: 2.8745 | LR: 9.32e-06 | 1400.0ms/step | 14.3 samples/s +Step 40240/44000 | Loss: 2.8552 | LR: 9.27e-06 | 1400.1ms/step | 14.3 samples/s +Step 40250/44000 | Loss: 2.8809 | LR: 9.22e-06 | 1399.9ms/step | 14.3 samples/s +Step 40260/44000 | Loss: 2.8661 | LR: 9.17e-06 | 1399.9ms/step | 14.3 samples/s +Step 40270/44000 | Loss: 2.8997 | LR: 9.12e-06 | 1400.1ms/step | 14.3 samples/s +Step 40280/44000 | Loss: 2.8918 | LR: 9.07e-06 | 1400.0ms/step | 14.3 samples/s +Step 40290/44000 | Loss: 2.8530 | LR: 9.02e-06 | 1400.2ms/step | 14.3 samples/s +Step 40300/44000 | Loss: 2.8884 | LR: 8.97e-06 | 1400.0ms/step | 14.3 samples/s +Step 40310/44000 | Loss: 2.8655 | LR: 8.93e-06 | 1400.0ms/step | 14.3 samples/s +Step 40320/44000 | Loss: 2.8664 | LR: 8.88e-06 | 1399.9ms/step | 14.3 samples/s +Step 40330/44000 | Loss: 2.8775 | LR: 8.83e-06 | 1400.1ms/step | 14.3 samples/s +Step 40340/44000 | Loss: 2.8796 | LR: 8.78e-06 | 1399.9ms/step | 14.3 samples/s +Step 40350/44000 | Loss: 2.8638 | LR: 8.74e-06 | 1400.4ms/step | 14.3 samples/s +Step 40360/44000 | Loss: 2.8836 | LR: 8.69e-06 | 1399.6ms/step | 14.3 samples/s +Step 40370/44000 | Loss: 2.8762 | LR: 8.64e-06 | 1400.1ms/step | 14.3 samples/s +Step 40380/44000 | Loss: 2.9034 | LR: 8.59e-06 | 1399.8ms/step | 14.3 samples/s +Step 40390/44000 | Loss: 2.8656 | LR: 8.55e-06 | 1400.3ms/step | 14.3 samples/s +Step 40400/44000 | Loss: 2.8634 | LR: 8.50e-06 | 1399.9ms/step | 14.3 samples/s +Step 40410/44000 | Loss: 2.9277 | LR: 8.45e-06 | 1400.1ms/step | 14.3 samples/s +Step 40420/44000 | Loss: 2.8666 | LR: 8.41e-06 | 1400.0ms/step | 14.3 samples/s +Step 40430/44000 | Loss: 2.9183 | LR: 8.36e-06 | 1399.7ms/step | 14.3 samples/s +Step 40440/44000 | Loss: 2.8773 | LR: 8.31e-06 | 1399.9ms/step | 14.3 samples/s +Step 40450/44000 | Loss: 2.8894 | LR: 8.27e-06 | 1400.2ms/step | 14.3 samples/s +Step 40460/44000 | Loss: 2.8815 | LR: 8.22e-06 | 1399.9ms/step | 14.3 samples/s +Step 40470/44000 | Loss: 2.8680 | LR: 8.17e-06 | 1399.9ms/step | 14.3 samples/s +Step 40480/44000 | Loss: 2.8945 | LR: 8.13e-06 | 1400.0ms/step | 14.3 samples/s +Step 40490/44000 | Loss: 2.8470 | LR: 8.08e-06 | 1399.9ms/step | 14.3 samples/s +Step 40500/44000 | Loss: 2.8812 | LR: 8.04e-06 | 1400.1ms/step | 14.3 samples/s +Step 40510/44000 | Loss: 2.8798 | LR: 7.99e-06 | 1400.3ms/step | 14.3 samples/s +Step 40520/44000 | Loss: 2.8588 | LR: 7.94e-06 | 1399.9ms/step | 14.3 samples/s +Step 40530/44000 | Loss: 2.8724 | LR: 7.90e-06 | 1399.9ms/step | 14.3 samples/s +Step 40540/44000 | Loss: 2.9101 | LR: 7.85e-06 | 1399.9ms/step | 14.3 samples/s +Step 40550/44000 | Loss: 2.8720 | LR: 7.81e-06 | 1400.3ms/step | 14.3 samples/s +Step 40560/44000 | Loss: 2.8903 | LR: 7.76e-06 | 1399.7ms/step | 14.3 samples/s +Step 40570/44000 | Loss: 2.8613 | LR: 7.72e-06 | 1400.1ms/step | 14.3 samples/s +Step 40580/44000 | Loss: 2.9095 | LR: 7.67e-06 | 1400.0ms/step | 14.3 samples/s +Step 40590/44000 | Loss: 2.8633 | LR: 7.63e-06 | 1400.2ms/step | 14.3 samples/s +Step 40600/44000 | Loss: 2.9148 | LR: 7.59e-06 | 1400.0ms/step | 14.3 samples/s +Step 40610/44000 | Loss: 2.8512 | LR: 7.54e-06 | 1399.7ms/step | 14.3 samples/s +Step 40620/44000 | Loss: 2.8865 | LR: 7.50e-06 | 1400.4ms/step | 14.3 samples/s +Step 40630/44000 | Loss: 2.8923 | LR: 7.45e-06 | 1399.8ms/step | 14.3 samples/s +Step 40640/44000 | Loss: 2.9084 | LR: 7.41e-06 | 1400.1ms/step | 14.3 samples/s +Step 40650/44000 | Loss: 2.8976 | LR: 7.37e-06 | 1399.9ms/step | 14.3 samples/s +Step 40660/44000 | Loss: 2.8755 | LR: 7.32e-06 | 1400.1ms/step | 14.3 samples/s +Step 40670/44000 | Loss: 2.8613 | LR: 7.28e-06 | 1400.1ms/step | 14.3 samples/s +Step 40680/44000 | Loss: 2.8772 | LR: 7.23e-06 | 1399.8ms/step | 14.3 samples/s +Step 40690/44000 | Loss: 2.8654 | LR: 7.19e-06 | 1400.1ms/step | 14.3 samples/s +Step 40700/44000 | Loss: 2.8807 | LR: 7.15e-06 | 1400.0ms/step | 14.3 samples/s +Step 40710/44000 | Loss: 2.8890 | LR: 7.10e-06 | 1399.8ms/step | 14.3 samples/s +Step 40720/44000 | Loss: 2.8602 | LR: 7.06e-06 | 1400.2ms/step | 14.3 samples/s +Step 40730/44000 | Loss: 2.8768 | LR: 7.02e-06 | 1399.9ms/step | 14.3 samples/s +Step 40740/44000 | Loss: 2.8879 | LR: 6.98e-06 | 1400.1ms/step | 14.3 samples/s +Step 40750/44000 | Loss: 2.8898 | LR: 6.93e-06 | 1400.0ms/step | 14.3 samples/s +Step 40760/44000 | Loss: 2.8661 | LR: 6.89e-06 | 1400.0ms/step | 14.3 samples/s +Step 40770/44000 | Loss: 2.8946 | LR: 6.85e-06 | 1400.0ms/step | 14.3 samples/s +Step 40780/44000 | Loss: 2.8869 | LR: 6.81e-06 | 1400.0ms/step | 14.3 samples/s +Step 40790/44000 | Loss: 2.8673 | LR: 6.77e-06 | 1399.9ms/step | 14.3 samples/s +Step 40800/44000 | Loss: 2.8595 | LR: 6.72e-06 | 1400.2ms/step | 14.3 samples/s +Step 40810/44000 | Loss: 2.8998 | LR: 6.68e-06 | 1399.9ms/step | 14.3 samples/s +Step 40820/44000 | Loss: 2.8805 | LR: 6.64e-06 | 1400.1ms/step | 14.3 samples/s +Step 40830/44000 | Loss: 2.8675 | LR: 6.60e-06 | 1399.9ms/step | 14.3 samples/s +Step 40840/44000 | Loss: 2.8855 | LR: 6.56e-06 | 1400.1ms/step | 14.3 samples/s +Step 40850/44000 | Loss: 2.8663 | LR: 6.52e-06 | 1400.0ms/step | 14.3 samples/s +Step 40860/44000 | Loss: 2.8527 | LR: 6.47e-06 | 1399.8ms/step | 14.3 samples/s +Step 40870/44000 | Loss: 2.8838 | LR: 6.43e-06 | 1400.0ms/step | 14.3 samples/s +Step 40880/44000 | Loss: 2.8666 | LR: 6.39e-06 | 1400.2ms/step | 14.3 samples/s +Step 40890/44000 | Loss: 2.8747 | LR: 6.35e-06 | 1399.8ms/step | 14.3 samples/s +Step 40900/44000 | Loss: 2.8841 | LR: 6.31e-06 | 1399.9ms/step | 14.3 samples/s +Step 40910/44000 | Loss: 2.8942 | LR: 6.27e-06 | 1400.1ms/step | 14.3 samples/s +Step 40920/44000 | Loss: 2.8660 | LR: 6.23e-06 | 1400.1ms/step | 14.3 samples/s +Step 40930/44000 | Loss: 2.9052 | LR: 6.19e-06 | 1399.8ms/step | 14.3 samples/s +Step 40940/44000 | Loss: 2.8735 | LR: 6.15e-06 | 1400.1ms/step | 14.3 samples/s +Step 40950/44000 | Loss: 2.8823 | LR: 6.11e-06 | 1400.2ms/step | 14.3 samples/s +Step 40960/44000 | Loss: 2.8551 | LR: 6.07e-06 | 1399.8ms/step | 14.3 samples/s +Step 40970/44000 | Loss: 2.8809 | LR: 6.03e-06 | 1399.9ms/step | 14.3 samples/s +Step 40980/44000 | Loss: 2.8764 | LR: 5.99e-06 | 1400.0ms/step | 14.3 samples/s +Step 40990/44000 | Loss: 2.8703 | LR: 5.95e-06 | 1400.0ms/step | 14.3 samples/s +Step 41000/44000 | Loss: 2.8549 | LR: 5.91e-06 | 1400.0ms/step | 14.3 samples/s +Validation at step 41000: Loss = 2.8711, Perplexity = 17.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-41000 +Step 41010/44000 | Loss: 2.9187 | LR: 5.87e-06 | 3905.9ms/step | 5.1 samples/s +Step 41020/44000 | Loss: 2.8838 | LR: 5.83e-06 | 1400.0ms/step | 14.3 samples/s +Step 41030/44000 | Loss: 2.8825 | LR: 5.80e-06 | 1400.1ms/step | 14.3 samples/s +Step 41040/44000 | Loss: 2.8585 | LR: 5.76e-06 | 1391.7ms/step | 14.4 samples/s +Step 41050/44000 | Loss: 2.8680 | LR: 5.72e-06 | 1392.2ms/step | 14.4 samples/s +Step 41060/44000 | Loss: 2.8563 | LR: 5.68e-06 | 1390.0ms/step | 14.4 samples/s +Step 41070/44000 | Loss: 2.9105 | LR: 5.64e-06 | 1394.2ms/step | 14.3 samples/s +Step 41080/44000 | Loss: 2.8783 | LR: 5.60e-06 | 1391.4ms/step | 14.4 samples/s +Step 41090/44000 | Loss: 2.8650 | LR: 5.56e-06 | 1390.4ms/step | 14.4 samples/s +Step 41100/44000 | Loss: 2.8919 | LR: 5.53e-06 | 1391.8ms/step | 14.4 samples/s +Step 41110/44000 | Loss: 2.9157 | LR: 5.49e-06 | 1393.5ms/step | 14.4 samples/s +Step 41120/44000 | Loss: 2.9103 | LR: 5.45e-06 | 1390.3ms/step | 14.4 samples/s +Step 41130/44000 | Loss: 2.8542 | LR: 5.41e-06 | 1393.5ms/step | 14.4 samples/s +Step 41140/44000 | Loss: 2.8857 | LR: 5.38e-06 | 1395.0ms/step | 14.3 samples/s +Step 41150/44000 | Loss: 2.8847 | LR: 5.34e-06 | 1394.3ms/step | 14.3 samples/s +Step 41160/44000 | Loss: 2.8764 | LR: 5.30e-06 | 1396.3ms/step | 14.3 samples/s +Step 41170/44000 | Loss: 2.8840 | LR: 5.26e-06 | 1401.8ms/step | 14.3 samples/s +Step 41180/44000 | Loss: 2.8904 | LR: 5.23e-06 | 1400.0ms/step | 14.3 samples/s +Step 41190/44000 | Loss: 2.8769 | LR: 5.19e-06 | 1399.9ms/step | 14.3 samples/s +Step 41200/44000 | Loss: 2.8700 | LR: 5.15e-06 | 1394.3ms/step | 14.3 samples/s +Step 41210/44000 | Loss: 2.8980 | LR: 5.12e-06 | 1395.6ms/step | 14.3 samples/s +Step 41220/44000 | Loss: 2.8744 | LR: 5.08e-06 | 1400.5ms/step | 14.3 samples/s +Step 41230/44000 | Loss: 2.8635 | LR: 5.04e-06 | 1394.3ms/step | 14.3 samples/s +Step 41240/44000 | Loss: 2.8656 | LR: 5.01e-06 | 1395.5ms/step | 14.3 samples/s +Step 41250/44000 | Loss: 2.8451 | LR: 4.97e-06 | 1393.9ms/step | 14.3 samples/s +Step 41260/44000 | Loss: 2.8949 | LR: 4.94e-06 | 1393.9ms/step | 14.3 samples/s +Step 41270/44000 | Loss: 2.8692 | LR: 4.90e-06 | 1394.1ms/step | 14.3 samples/s +Step 41280/44000 | Loss: 2.8812 | LR: 4.86e-06 | 1396.0ms/step | 14.3 samples/s +Step 41290/44000 | Loss: 2.8694 | LR: 4.83e-06 | 1397.6ms/step | 14.3 samples/s +Step 41300/44000 | Loss: 2.9146 | LR: 4.79e-06 | 1393.5ms/step | 14.4 samples/s +Step 41310/44000 | Loss: 2.8820 | LR: 4.76e-06 | 1392.8ms/step | 14.4 samples/s +Step 41320/44000 | Loss: 2.8832 | LR: 4.72e-06 | 1396.1ms/step | 14.3 samples/s +Step 41330/44000 | Loss: 2.8980 | LR: 4.69e-06 | 1395.4ms/step | 14.3 samples/s +Step 41340/44000 | Loss: 2.8776 | LR: 4.65e-06 | 1390.6ms/step | 14.4 samples/s +Step 41350/44000 | Loss: 2.8579 | LR: 4.62e-06 | 1393.4ms/step | 14.4 samples/s +Step 41360/44000 | Loss: 2.8909 | LR: 4.58e-06 | 1392.4ms/step | 14.4 samples/s +Step 41370/44000 | Loss: 2.9084 | LR: 4.55e-06 | 1393.0ms/step | 14.4 samples/s +Step 41380/44000 | Loss: 2.8813 | LR: 4.51e-06 | 1391.0ms/step | 14.4 samples/s +Step 41390/44000 | Loss: 2.8733 | LR: 4.48e-06 | 1396.2ms/step | 14.3 samples/s +Step 41400/44000 | Loss: 2.8584 | LR: 4.45e-06 | 1395.5ms/step | 14.3 samples/s +Step 41410/44000 | Loss: 2.8579 | LR: 4.41e-06 | 1392.0ms/step | 14.4 samples/s +Step 41420/44000 | Loss: 2.9051 | LR: 4.38e-06 | 1394.2ms/step | 14.3 samples/s +Step 41430/44000 | Loss: 2.8663 | LR: 4.34e-06 | 1396.1ms/step | 14.3 samples/s +Step 41440/44000 | Loss: 2.9304 | LR: 4.31e-06 | 1395.9ms/step | 14.3 samples/s +Step 41450/44000 | Loss: 2.8830 | LR: 4.28e-06 | 1394.0ms/step | 14.3 samples/s +Step 41460/44000 | Loss: 2.8876 | LR: 4.24e-06 | 1392.9ms/step | 14.4 samples/s +Step 41470/44000 | Loss: 2.8772 | LR: 4.21e-06 | 1391.1ms/step | 14.4 samples/s +Step 41480/44000 | Loss: 2.8688 | LR: 4.18e-06 | 1395.7ms/step | 14.3 samples/s +Step 41490/44000 | Loss: 2.8890 | LR: 4.14e-06 | 1393.4ms/step | 14.4 samples/s +Step 41500/44000 | Loss: 2.8993 | LR: 4.11e-06 | 1396.9ms/step | 14.3 samples/s +Step 41510/44000 | Loss: 2.8516 | LR: 4.08e-06 | 1395.8ms/step | 14.3 samples/s +Step 41520/44000 | Loss: 2.8587 | LR: 4.05e-06 | 1394.0ms/step | 14.3 samples/s +Step 41530/44000 | Loss: 2.8730 | LR: 4.01e-06 | 1396.0ms/step | 14.3 samples/s +Step 41540/44000 | Loss: 2.8934 | LR: 3.98e-06 | 1397.2ms/step | 14.3 samples/s +Step 41550/44000 | Loss: 2.8819 | LR: 3.95e-06 | 1390.7ms/step | 14.4 samples/s +Step 41560/44000 | Loss: 2.8830 | LR: 3.92e-06 | 1390.4ms/step | 14.4 samples/s +Step 41570/44000 | Loss: 2.8710 | LR: 3.88e-06 | 1392.0ms/step | 14.4 samples/s +Step 41580/44000 | Loss: 2.8977 | LR: 3.85e-06 | 1391.2ms/step | 14.4 samples/s +Step 41590/44000 | Loss: 2.8547 | LR: 3.82e-06 | 1394.5ms/step | 14.3 samples/s +Step 41600/44000 | Loss: 2.8662 | LR: 3.79e-06 | 1392.4ms/step | 14.4 samples/s +Step 41610/44000 | Loss: 2.9184 | LR: 3.76e-06 | 1393.8ms/step | 14.3 samples/s +Step 41620/44000 | Loss: 2.8899 | LR: 3.73e-06 | 1394.2ms/step | 14.3 samples/s +Step 41630/44000 | Loss: 2.9099 | LR: 3.70e-06 | 1394.0ms/step | 14.3 samples/s +Step 41640/44000 | Loss: 2.8827 | LR: 3.66e-06 | 1399.6ms/step | 14.3 samples/s +Step 41650/44000 | Loss: 2.8840 | LR: 3.63e-06 | 1399.0ms/step | 14.3 samples/s +Step 41660/44000 | Loss: 2.8820 | LR: 3.60e-06 | 1391.2ms/step | 14.4 samples/s +Step 41670/44000 | Loss: 2.8642 | LR: 3.57e-06 | 1393.6ms/step | 14.4 samples/s +Step 41680/44000 | Loss: 2.9065 | LR: 3.54e-06 | 1392.4ms/step | 14.4 samples/s +Step 41690/44000 | Loss: 2.8941 | LR: 3.51e-06 | 1393.3ms/step | 14.4 samples/s +Step 41700/44000 | Loss: 2.8333 | LR: 3.48e-06 | 1392.0ms/step | 14.4 samples/s +Step 41710/44000 | Loss: 2.8714 | LR: 3.45e-06 | 1390.8ms/step | 14.4 samples/s +Step 41720/44000 | Loss: 2.8704 | LR: 3.42e-06 | 1392.8ms/step | 14.4 samples/s +Step 41730/44000 | Loss: 2.8996 | LR: 3.39e-06 | 1390.7ms/step | 14.4 samples/s +Step 41740/44000 | Loss: 2.8928 | LR: 3.36e-06 | 1392.7ms/step | 14.4 samples/s +Step 41750/44000 | Loss: 2.8620 | LR: 3.33e-06 | 1394.3ms/step | 14.3 samples/s +Step 41760/44000 | Loss: 2.9008 | LR: 3.30e-06 | 1394.4ms/step | 14.3 samples/s +Step 41770/44000 | Loss: 2.8997 | LR: 3.27e-06 | 1392.7ms/step | 14.4 samples/s +Step 41780/44000 | Loss: 2.8675 | LR: 3.24e-06 | 1392.4ms/step | 14.4 samples/s +Step 41790/44000 | Loss: 2.8494 | LR: 3.21e-06 | 1397.6ms/step | 14.3 samples/s +Step 41800/44000 | Loss: 2.8934 | LR: 3.19e-06 | 1395.6ms/step | 14.3 samples/s +Step 41810/44000 | Loss: 2.8934 | LR: 3.16e-06 | 1396.6ms/step | 14.3 samples/s +Step 41820/44000 | Loss: 2.8885 | LR: 3.13e-06 | 1396.0ms/step | 14.3 samples/s +Step 41830/44000 | Loss: 2.8770 | LR: 3.10e-06 | 1393.8ms/step | 14.3 samples/s +Step 41840/44000 | Loss: 2.8880 | LR: 3.07e-06 | 1394.1ms/step | 14.3 samples/s +Step 41850/44000 | Loss: 2.9183 | LR: 3.04e-06 | 1394.1ms/step | 14.3 samples/s +Step 41860/44000 | Loss: 2.8675 | LR: 3.01e-06 | 1393.9ms/step | 14.3 samples/s +Step 41870/44000 | Loss: 2.8937 | LR: 2.99e-06 | 1391.8ms/step | 14.4 samples/s +Step 41880/44000 | Loss: 2.8693 | LR: 2.96e-06 | 1395.7ms/step | 14.3 samples/s +Step 41890/44000 | Loss: 2.8581 | LR: 2.93e-06 | 1392.3ms/step | 14.4 samples/s +Step 41900/44000 | Loss: 2.8913 | LR: 2.90e-06 | 1392.1ms/step | 14.4 samples/s +Step 41910/44000 | Loss: 2.8800 | LR: 2.88e-06 | 1393.9ms/step | 14.3 samples/s +Step 41920/44000 | Loss: 2.9031 | LR: 2.85e-06 | 1396.2ms/step | 14.3 samples/s +Step 41930/44000 | Loss: 2.8723 | LR: 2.82e-06 | 1393.9ms/step | 14.3 samples/s +Step 41940/44000 | Loss: 2.8693 | LR: 2.79e-06 | 1393.9ms/step | 14.3 samples/s +Step 41950/44000 | Loss: 2.8899 | LR: 2.77e-06 | 1392.0ms/step | 14.4 samples/s +Step 41960/44000 | Loss: 2.8806 | LR: 2.74e-06 | 1394.2ms/step | 14.3 samples/s +Step 41970/44000 | Loss: 2.8989 | LR: 2.71e-06 | 1393.2ms/step | 14.4 samples/s +Step 41980/44000 | Loss: 2.8661 | LR: 2.69e-06 | 1396.8ms/step | 14.3 samples/s +Step 41990/44000 | Loss: 2.9095 | LR: 2.66e-06 | 1394.8ms/step | 14.3 samples/s +Step 42000/44000 | Loss: 2.8986 | LR: 2.63e-06 | 1392.8ms/step | 14.4 samples/s +Validation at step 42000: Loss = 2.8711, Perplexity = 17.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-42000 +Step 42010/44000 | Loss: 2.8858 | LR: 2.61e-06 | 3888.4ms/step | 5.1 samples/s +Step 42020/44000 | Loss: 2.8740 | LR: 2.58e-06 | 1400.0ms/step | 14.3 samples/s +Step 42030/44000 | Loss: 2.9068 | LR: 2.56e-06 | 1400.1ms/step | 14.3 samples/s +Step 42040/44000 | Loss: 2.8956 | LR: 2.53e-06 | 1399.8ms/step | 14.3 samples/s +Step 42050/44000 | Loss: 2.8894 | LR: 2.50e-06 | 1400.1ms/step | 14.3 samples/s +Step 42060/44000 | Loss: 2.8888 | LR: 2.48e-06 | 1400.0ms/step | 14.3 samples/s +Step 42070/44000 | Loss: 2.8894 | LR: 2.45e-06 | 1400.2ms/step | 14.3 samples/s +Step 42080/44000 | Loss: 2.8896 | LR: 2.43e-06 | 1399.9ms/step | 14.3 samples/s +Step 42090/44000 | Loss: 2.8726 | LR: 2.40e-06 | 1400.0ms/step | 14.3 samples/s +Step 42100/44000 | Loss: 2.9162 | LR: 2.38e-06 | 1399.9ms/step | 14.3 samples/s +Step 42110/44000 | Loss: 2.8410 | LR: 2.35e-06 | 1400.1ms/step | 14.3 samples/s +Step 42120/44000 | Loss: 2.8798 | LR: 2.33e-06 | 1400.1ms/step | 14.3 samples/s +Step 42130/44000 | Loss: 2.8898 | LR: 2.30e-06 | 1400.2ms/step | 14.3 samples/s +Step 42140/44000 | Loss: 2.8555 | LR: 2.28e-06 | 1399.7ms/step | 14.3 samples/s +Step 42150/44000 | Loss: 2.8809 | LR: 2.25e-06 | 1399.8ms/step | 14.3 samples/s +Step 42160/44000 | Loss: 2.8913 | LR: 2.23e-06 | 1400.1ms/step | 14.3 samples/s +Step 42170/44000 | Loss: 2.9125 | LR: 2.21e-06 | 1400.0ms/step | 14.3 samples/s +Step 42180/44000 | Loss: 2.8786 | LR: 2.18e-06 | 1399.9ms/step | 14.3 samples/s +Step 42190/44000 | Loss: 2.9032 | LR: 2.16e-06 | 1400.1ms/step | 14.3 samples/s +Step 42200/44000 | Loss: 2.8600 | LR: 2.13e-06 | 1400.0ms/step | 14.3 samples/s +Step 42210/44000 | Loss: 2.8729 | LR: 2.11e-06 | 1399.7ms/step | 14.3 samples/s +Step 42220/44000 | Loss: 2.8888 | LR: 2.09e-06 | 1400.3ms/step | 14.3 samples/s +Step 42230/44000 | Loss: 2.8717 | LR: 2.06e-06 | 1399.8ms/step | 14.3 samples/s +Step 42240/44000 | Loss: 2.8817 | LR: 2.04e-06 | 1400.5ms/step | 14.3 samples/s +Step 42250/44000 | Loss: 2.8506 | LR: 2.02e-06 | 1399.6ms/step | 14.3 samples/s +Step 42260/44000 | Loss: 2.8660 | LR: 1.99e-06 | 1400.2ms/step | 14.3 samples/s +Step 42270/44000 | Loss: 2.8614 | LR: 1.97e-06 | 1399.9ms/step | 14.3 samples/s +Step 42280/44000 | Loss: 2.8892 | LR: 1.95e-06 | 1400.0ms/step | 14.3 samples/s +Step 42290/44000 | Loss: 2.8547 | LR: 1.93e-06 | 1400.1ms/step | 14.3 samples/s +Step 42300/44000 | Loss: 2.8663 | LR: 1.90e-06 | 1399.9ms/step | 14.3 samples/s +Step 42310/44000 | Loss: 2.9001 | LR: 1.88e-06 | 1400.2ms/step | 14.3 samples/s +Step 42320/44000 | Loss: 2.8874 | LR: 1.86e-06 | 1400.0ms/step | 14.3 samples/s +Step 42330/44000 | Loss: 2.9012 | LR: 1.84e-06 | 1399.8ms/step | 14.3 samples/s +Step 42340/44000 | Loss: 2.8850 | LR: 1.82e-06 | 1400.2ms/step | 14.3 samples/s +Step 42350/44000 | Loss: 2.9056 | LR: 1.79e-06 | 1399.9ms/step | 14.3 samples/s +Step 42360/44000 | Loss: 2.8740 | LR: 1.77e-06 | 1400.1ms/step | 14.3 samples/s +Step 42370/44000 | Loss: 2.8876 | LR: 1.75e-06 | 1400.0ms/step | 14.3 samples/s +Step 42380/44000 | Loss: 2.8834 | LR: 1.73e-06 | 1399.8ms/step | 14.3 samples/s +Step 42390/44000 | Loss: 2.8896 | LR: 1.71e-06 | 1399.9ms/step | 14.3 samples/s +Step 42400/44000 | Loss: 2.8637 | LR: 1.69e-06 | 1400.0ms/step | 14.3 samples/s +Step 42410/44000 | Loss: 2.8891 | LR: 1.67e-06 | 1400.2ms/step | 14.3 samples/s +Step 42420/44000 | Loss: 2.8787 | LR: 1.64e-06 | 1400.0ms/step | 14.3 samples/s +Step 42430/44000 | Loss: 2.8760 | LR: 1.62e-06 | 1399.9ms/step | 14.3 samples/s +Step 42440/44000 | Loss: 2.8892 | LR: 1.60e-06 | 1400.1ms/step | 14.3 samples/s +Step 42450/44000 | Loss: 2.8863 | LR: 1.58e-06 | 1399.9ms/step | 14.3 samples/s +Step 42460/44000 | Loss: 2.9123 | LR: 1.56e-06 | 1400.0ms/step | 14.3 samples/s +Step 42470/44000 | Loss: 2.8738 | LR: 1.54e-06 | 1399.9ms/step | 14.3 samples/s +Step 42480/44000 | Loss: 2.8913 | LR: 1.52e-06 | 1400.1ms/step | 14.3 samples/s +Step 42490/44000 | Loss: 2.8781 | LR: 1.50e-06 | 1400.0ms/step | 14.3 samples/s +Step 42500/44000 | Loss: 2.8599 | LR: 1.48e-06 | 1400.0ms/step | 14.3 samples/s +Step 42510/44000 | Loss: 2.8894 | LR: 1.46e-06 | 1400.0ms/step | 14.3 samples/s +Step 42520/44000 | Loss: 2.8781 | LR: 1.44e-06 | 1399.9ms/step | 14.3 samples/s +Step 42530/44000 | Loss: 2.9069 | LR: 1.42e-06 | 1400.1ms/step | 14.3 samples/s +Step 42540/44000 | Loss: 2.8848 | LR: 1.40e-06 | 1399.8ms/step | 14.3 samples/s +Step 42550/44000 | Loss: 2.8921 | LR: 1.39e-06 | 1400.0ms/step | 14.3 samples/s +Step 42560/44000 | Loss: 2.8786 | LR: 1.37e-06 | 1400.2ms/step | 14.3 samples/s +Step 42570/44000 | Loss: 2.8591 | LR: 1.35e-06 | 1399.8ms/step | 14.3 samples/s +Step 42580/44000 | Loss: 2.8993 | LR: 1.33e-06 | 1400.2ms/step | 14.3 samples/s +Step 42590/44000 | Loss: 2.9049 | LR: 1.31e-06 | 1399.7ms/step | 14.3 samples/s +Step 42600/44000 | Loss: 2.8716 | LR: 1.29e-06 | 1400.3ms/step | 14.3 samples/s +Step 42610/44000 | Loss: 2.8933 | LR: 1.27e-06 | 1399.9ms/step | 14.3 samples/s +Step 42620/44000 | Loss: 2.8656 | LR: 1.25e-06 | 1399.9ms/step | 14.3 samples/s +Step 42630/44000 | Loss: 2.8645 | LR: 1.24e-06 | 1400.3ms/step | 14.3 samples/s +Step 42640/44000 | Loss: 2.8883 | LR: 1.22e-06 | 1399.6ms/step | 14.3 samples/s +Step 42650/44000 | Loss: 2.8928 | LR: 1.20e-06 | 1400.2ms/step | 14.3 samples/s +Step 42660/44000 | Loss: 2.8668 | LR: 1.18e-06 | 1399.6ms/step | 14.3 samples/s +Step 42670/44000 | Loss: 2.8811 | LR: 1.17e-06 | 1400.4ms/step | 14.3 samples/s +Step 42680/44000 | Loss: 2.8791 | LR: 1.15e-06 | 1400.0ms/step | 14.3 samples/s +Step 42690/44000 | Loss: 2.9041 | LR: 1.13e-06 | 1399.9ms/step | 14.3 samples/s +Step 42700/44000 | Loss: 2.8724 | LR: 1.11e-06 | 1400.0ms/step | 14.3 samples/s +Step 42710/44000 | Loss: 2.8842 | LR: 1.10e-06 | 1400.2ms/step | 14.3 samples/s +Step 42720/44000 | Loss: 2.8874 | LR: 1.08e-06 | 1399.6ms/step | 14.3 samples/s +Step 42730/44000 | Loss: 2.9053 | LR: 1.06e-06 | 1400.1ms/step | 14.3 samples/s +Step 42740/44000 | Loss: 2.8618 | LR: 1.05e-06 | 1400.1ms/step | 14.3 samples/s +Step 42750/44000 | Loss: 2.8254 | LR: 1.03e-06 | 1400.0ms/step | 14.3 samples/s +Step 42760/44000 | Loss: 2.9050 | LR: 1.01e-06 | 1400.1ms/step | 14.3 samples/s +Step 42770/44000 | Loss: 2.8488 | LR: 9.97e-07 | 1399.8ms/step | 14.3 samples/s +Step 42780/44000 | Loss: 2.8606 | LR: 9.81e-07 | 1400.2ms/step | 14.3 samples/s +Step 42790/44000 | Loss: 2.8832 | LR: 9.65e-07 | 1399.9ms/step | 14.3 samples/s +Step 42800/44000 | Loss: 2.8877 | LR: 9.49e-07 | 1400.1ms/step | 14.3 samples/s +Step 42810/44000 | Loss: 2.8867 | LR: 9.33e-07 | 1400.2ms/step | 14.3 samples/s +Step 42820/44000 | Loss: 2.8810 | LR: 9.18e-07 | 1399.6ms/step | 14.3 samples/s +Step 42830/44000 | Loss: 2.8777 | LR: 9.02e-07 | 1400.2ms/step | 14.3 samples/s +Step 42840/44000 | Loss: 2.8675 | LR: 8.87e-07 | 1399.8ms/step | 14.3 samples/s +Step 42850/44000 | Loss: 2.8796 | LR: 8.72e-07 | 1400.1ms/step | 14.3 samples/s +Step 42860/44000 | Loss: 2.8615 | LR: 8.57e-07 | 1400.0ms/step | 14.3 samples/s +Step 42870/44000 | Loss: 2.8991 | LR: 8.42e-07 | 1400.1ms/step | 14.3 samples/s +Step 42880/44000 | Loss: 2.9042 | LR: 8.27e-07 | 1400.2ms/step | 14.3 samples/s +Step 42890/44000 | Loss: 2.8797 | LR: 8.12e-07 | 1399.7ms/step | 14.3 samples/s +Step 42900/44000 | Loss: 2.8786 | LR: 7.98e-07 | 1400.1ms/step | 14.3 samples/s +Step 42910/44000 | Loss: 2.8773 | LR: 7.83e-07 | 1400.0ms/step | 14.3 samples/s +Step 42920/44000 | Loss: 2.9073 | LR: 7.69e-07 | 1400.0ms/step | 14.3 samples/s +Step 42930/44000 | Loss: 2.8820 | LR: 7.55e-07 | 1400.0ms/step | 14.3 samples/s +Step 42940/44000 | Loss: 2.8958 | LR: 7.41e-07 | 1399.9ms/step | 14.3 samples/s +Step 42950/44000 | Loss: 2.8460 | LR: 7.27e-07 | 1400.2ms/step | 14.3 samples/s +Step 42960/44000 | Loss: 2.8568 | LR: 7.13e-07 | 1399.7ms/step | 14.3 samples/s +Step 42970/44000 | Loss: 2.8680 | LR: 6.99e-07 | 1400.3ms/step | 14.3 samples/s +Step 42980/44000 | Loss: 2.8828 | LR: 6.86e-07 | 1399.9ms/step | 14.3 samples/s +Step 42990/44000 | Loss: 2.9103 | LR: 6.72e-07 | 1399.9ms/step | 14.3 samples/s +Step 43000/44000 | Loss: 2.8858 | LR: 6.59e-07 | 1400.0ms/step | 14.3 samples/s +Validation at step 43000: Loss = 2.8711, Perplexity = 17.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-43000 +Step 43010/44000 | Loss: 2.8767 | LR: 6.46e-07 | 3896.0ms/step | 5.1 samples/s +Step 43020/44000 | Loss: 2.8402 | LR: 6.33e-07 | 1400.0ms/step | 14.3 samples/s +Step 43030/44000 | Loss: 2.8740 | LR: 6.20e-07 | 1399.9ms/step | 14.3 samples/s +Step 43040/44000 | Loss: 2.8689 | LR: 6.08e-07 | 1400.0ms/step | 14.3 samples/s +Step 43050/44000 | Loss: 2.8508 | LR: 5.95e-07 | 1399.9ms/step | 14.3 samples/s +Step 43060/44000 | Loss: 2.8829 | LR: 5.83e-07 | 1400.1ms/step | 14.3 samples/s +Step 43070/44000 | Loss: 2.8853 | LR: 5.70e-07 | 1400.2ms/step | 14.3 samples/s +Step 43080/44000 | Loss: 2.8605 | LR: 5.58e-07 | 1399.9ms/step | 14.3 samples/s +Step 43090/44000 | Loss: 2.8847 | LR: 5.46e-07 | 1399.9ms/step | 14.3 samples/s +Step 43100/44000 | Loss: 2.8949 | LR: 5.34e-07 | 1400.1ms/step | 14.3 samples/s +Step 43110/44000 | Loss: 2.8565 | LR: 5.22e-07 | 1399.8ms/step | 14.3 samples/s +Step 43120/44000 | Loss: 2.8801 | LR: 5.11e-07 | 1400.1ms/step | 14.3 samples/s +Step 43130/44000 | Loss: 2.8914 | LR: 4.99e-07 | 1400.3ms/step | 14.3 samples/s +Step 43140/44000 | Loss: 2.8925 | LR: 4.88e-07 | 1399.8ms/step | 14.3 samples/s +Step 43150/44000 | Loss: 2.8864 | LR: 4.76e-07 | 1400.0ms/step | 14.3 samples/s +Step 43160/44000 | Loss: 2.8917 | LR: 4.65e-07 | 1400.0ms/step | 14.3 samples/s +Step 43170/44000 | Loss: 2.8918 | LR: 4.54e-07 | 1399.6ms/step | 14.3 samples/s +Step 43180/44000 | Loss: 2.8839 | LR: 4.43e-07 | 1400.2ms/step | 14.3 samples/s +Step 43190/44000 | Loss: 2.8746 | LR: 4.33e-07 | 1400.3ms/step | 14.3 samples/s +Step 43200/44000 | Loss: 2.8900 | LR: 4.22e-07 | 1400.1ms/step | 14.3 samples/s +Step 43210/44000 | Loss: 2.8780 | LR: 4.12e-07 | 1399.8ms/step | 14.3 samples/s +Step 43220/44000 | Loss: 2.8876 | LR: 4.01e-07 | 1399.9ms/step | 14.3 samples/s +Step 43230/44000 | Loss: 2.8552 | LR: 3.91e-07 | 1400.1ms/step | 14.3 samples/s +Step 43240/44000 | Loss: 2.8599 | LR: 3.81e-07 | 1400.0ms/step | 14.3 samples/s +Step 43250/44000 | Loss: 2.8767 | LR: 3.71e-07 | 1399.9ms/step | 14.3 samples/s +Step 43260/44000 | Loss: 2.8815 | LR: 3.61e-07 | 1400.2ms/step | 14.3 samples/s +Step 43270/44000 | Loss: 2.8922 | LR: 3.51e-07 | 1400.0ms/step | 14.3 samples/s +Step 43280/44000 | Loss: 2.8865 | LR: 3.42e-07 | 1400.1ms/step | 14.3 samples/s +Step 43290/44000 | Loss: 2.8804 | LR: 3.32e-07 | 1399.9ms/step | 14.3 samples/s +Step 43300/44000 | Loss: 2.9073 | LR: 3.23e-07 | 1399.9ms/step | 14.3 samples/s +Step 43310/44000 | Loss: 2.8781 | LR: 3.14e-07 | 1400.1ms/step | 14.3 samples/s +Step 43320/44000 | Loss: 2.9141 | LR: 3.05e-07 | 1400.0ms/step | 14.3 samples/s +Step 43330/44000 | Loss: 2.8566 | LR: 2.96e-07 | 1400.2ms/step | 14.3 samples/s +Step 43340/44000 | Loss: 2.8661 | LR: 2.87e-07 | 1399.9ms/step | 14.3 samples/s +Step 43350/44000 | Loss: 2.8898 | LR: 2.79e-07 | 1400.0ms/step | 14.3 samples/s +Step 43360/44000 | Loss: 2.8805 | LR: 2.70e-07 | 1400.1ms/step | 14.3 samples/s +Step 43370/44000 | Loss: 2.9129 | LR: 2.62e-07 | 1400.8ms/step | 14.3 samples/s +Step 43380/44000 | Loss: 2.8711 | LR: 2.53e-07 | 1399.0ms/step | 14.3 samples/s +Step 43390/44000 | Loss: 2.8626 | LR: 2.45e-07 | 1400.0ms/step | 14.3 samples/s +Step 43400/44000 | Loss: 2.8740 | LR: 2.37e-07 | 1400.1ms/step | 14.3 samples/s +Step 43410/44000 | Loss: 2.8727 | LR: 2.30e-07 | 1400.1ms/step | 14.3 samples/s +Step 43420/44000 | Loss: 2.8931 | LR: 2.22e-07 | 1399.8ms/step | 14.3 samples/s +Step 43430/44000 | Loss: 2.8907 | LR: 2.14e-07 | 1400.1ms/step | 14.3 samples/s +Step 43440/44000 | Loss: 2.8746 | LR: 2.07e-07 | 1399.8ms/step | 14.3 samples/s +Step 43450/44000 | Loss: 2.8563 | LR: 1.99e-07 | 1400.1ms/step | 14.3 samples/s +Step 43460/44000 | Loss: 2.9005 | LR: 1.92e-07 | 1399.9ms/step | 14.3 samples/s +Step 43470/44000 | Loss: 2.8802 | LR: 1.85e-07 | 1400.0ms/step | 14.3 samples/s +Step 43480/44000 | Loss: 2.8844 | LR: 1.78e-07 | 1400.0ms/step | 14.3 samples/s +Step 43490/44000 | Loss: 2.8669 | LR: 1.72e-07 | 1400.1ms/step | 14.3 samples/s +Step 43500/44000 | Loss: 2.8991 | LR: 1.65e-07 | 1400.1ms/step | 14.3 samples/s +Step 43510/44000 | Loss: 2.8901 | LR: 1.58e-07 | 1399.9ms/step | 14.3 samples/s +Step 43520/44000 | Loss: 2.8714 | LR: 1.52e-07 | 1399.9ms/step | 14.3 samples/s +Step 43530/44000 | Loss: 2.8808 | LR: 1.46e-07 | 1400.0ms/step | 14.3 samples/s +Step 43540/44000 | Loss: 2.8802 | LR: 1.40e-07 | 1400.2ms/step | 14.3 samples/s +Step 43550/44000 | Loss: 2.8584 | LR: 1.34e-07 | 1399.9ms/step | 14.3 samples/s +Step 43560/44000 | Loss: 2.9329 | LR: 1.28e-07 | 1400.1ms/step | 14.3 samples/s +Step 43570/44000 | Loss: 2.8733 | LR: 1.22e-07 | 1400.0ms/step | 14.3 samples/s +Step 43580/44000 | Loss: 2.8842 | LR: 1.16e-07 | 1399.9ms/step | 14.3 samples/s +Step 43590/44000 | Loss: 2.8763 | LR: 1.11e-07 | 1400.2ms/step | 14.3 samples/s +Step 43600/44000 | Loss: 2.8631 | LR: 1.06e-07 | 1399.8ms/step | 14.3 samples/s +Step 43610/44000 | Loss: 2.8633 | LR: 1.00e-07 | 1400.1ms/step | 14.3 samples/s +Step 43620/44000 | Loss: 2.8581 | LR: 9.52e-08 | 1400.0ms/step | 14.3 samples/s +Step 43630/44000 | Loss: 2.8752 | LR: 9.03e-08 | 1400.1ms/step | 14.3 samples/s +Step 43640/44000 | Loss: 2.8907 | LR: 8.55e-08 | 1399.8ms/step | 14.3 samples/s +Step 43650/44000 | Loss: 2.8776 | LR: 8.08e-08 | 1400.0ms/step | 14.3 samples/s +Step 43660/44000 | Loss: 2.8920 | LR: 7.62e-08 | 1399.9ms/step | 14.3 samples/s +Step 43670/44000 | Loss: 2.8776 | LR: 7.18e-08 | 1400.1ms/step | 14.3 samples/s +Step 43680/44000 | Loss: 2.8685 | LR: 6.75e-08 | 1400.1ms/step | 14.3 samples/s +Step 43690/44000 | Loss: 2.9165 | LR: 6.34e-08 | 1399.8ms/step | 14.3 samples/s +Step 43700/44000 | Loss: 2.8728 | LR: 5.94e-08 | 1400.0ms/step | 14.3 samples/s +Step 43710/44000 | Loss: 2.8781 | LR: 5.55e-08 | 1400.1ms/step | 14.3 samples/s +Step 43720/44000 | Loss: 2.8732 | LR: 5.17e-08 | 1399.9ms/step | 14.3 samples/s +Step 43730/44000 | Loss: 2.8729 | LR: 4.81e-08 | 1400.0ms/step | 14.3 samples/s +Step 43740/44000 | Loss: 2.8843 | LR: 4.46e-08 | 1400.0ms/step | 14.3 samples/s +Step 43750/44000 | Loss: 2.8850 | LR: 4.12e-08 | 1400.0ms/step | 14.3 samples/s +Step 43760/44000 | Loss: 2.8892 | LR: 3.80e-08 | 1400.0ms/step | 14.3 samples/s +Step 43770/44000 | Loss: 2.8714 | LR: 3.49e-08 | 1400.1ms/step | 14.3 samples/s +Step 43780/44000 | Loss: 2.8615 | LR: 3.19e-08 | 1399.9ms/step | 14.3 samples/s +Step 43790/44000 | Loss: 2.8643 | LR: 2.91e-08 | 1400.1ms/step | 14.3 samples/s +Step 43800/44000 | Loss: 2.9160 | LR: 2.64e-08 | 1400.0ms/step | 14.3 samples/s +Step 43810/44000 | Loss: 2.9129 | LR: 2.38e-08 | 1399.8ms/step | 14.3 samples/s +Step 43820/44000 | Loss: 2.8907 | LR: 2.14e-08 | 1400.1ms/step | 14.3 samples/s +Step 43830/44000 | Loss: 2.8713 | LR: 1.91e-08 | 1399.8ms/step | 14.3 samples/s +Step 43840/44000 | Loss: 2.8879 | LR: 1.69e-08 | 1400.1ms/step | 14.3 samples/s +Step 43850/44000 | Loss: 2.8996 | LR: 1.48e-08 | 1400.1ms/step | 14.3 samples/s +Step 43860/44000 | Loss: 2.8812 | LR: 1.29e-08 | 1400.1ms/step | 14.3 samples/s +Step 43870/44000 | Loss: 2.8339 | LR: 1.11e-08 | 1399.9ms/step | 14.3 samples/s +Step 43880/44000 | Loss: 2.8890 | LR: 9.50e-09 | 1400.2ms/step | 14.3 samples/s +Step 43890/44000 | Loss: 2.8626 | LR: 7.98e-09 | 1399.7ms/step | 14.3 samples/s +Step 43900/44000 | Loss: 2.8814 | LR: 6.60e-09 | 1400.2ms/step | 14.3 samples/s +Step 43910/44000 | Loss: 2.9025 | LR: 5.34e-09 | 1400.1ms/step | 14.3 samples/s +Step 43920/44000 | Loss: 2.8664 | LR: 4.22e-09 | 1399.8ms/step | 14.3 samples/s +Step 43930/44000 | Loss: 2.8904 | LR: 3.23e-09 | 1399.9ms/step | 14.3 samples/s +Step 43940/44000 | Loss: 2.8862 | LR: 2.37e-09 | 1399.9ms/step | 14.3 samples/s +Step 43950/44000 | Loss: 2.8778 | LR: 1.65e-09 | 1400.0ms/step | 14.3 samples/s +Step 43960/44000 | Loss: 2.8992 | LR: 1.06e-09 | 1400.2ms/step | 14.3 samples/s +Step 43970/44000 | Loss: 2.8599 | LR: 5.94e-10 | 1399.8ms/step | 14.3 samples/s +Step 43980/44000 | Loss: 2.8690 | LR: 2.64e-10 | 1400.1ms/step | 14.3 samples/s +Step 43990/44000 | Loss: 2.8749 | LR: 6.60e-11 | 1400.0ms/step | 14.3 samples/s +Step 44000/44000 | Loss: 2.8414 | LR: 0.00e+00 | 1400.2ms/step | 14.3 samples/s +Validation at step 44000: Loss = 2.8711, Perplexity = 17.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-44000 +wandb: updating run metadata +wandb: uploading wandb-summary.json; uploading config.yaml; uploading output.log +wandb: uploading data +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▆▅▄▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▆▅▄▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ▃▁▁▁▁▄▄▇▆▇█▅▁▁▁▁▁▁▁▁▇▇▄▄▁▁▁▁▁▁▁▁▁▁▇▁▂▁▁▁ +wandb: speed/steps_per_sec ███████████████████▁████████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.87111 +wandb: eval/perplexity 17.65666 +wandb: eval/uses_dense_attention 1 +wandb: eval/uses_sparse_attention 0 +wandb: speed/ms_per_step 1400.19476 +wandb: speed/samples_per_sec 14.28373 +wandb: speed/steps_per_sec 0.71419 +wandb: +4 ... +wandb: +wandb: 🚀 View run l8k365m_dense_ropectl_a1p0_seed43 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ysa9iyov +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260820_194210-ysa9iyov/logs +Training complete! diff --git a/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed42.20260820T041452Z.log b/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed42.20260820T041452Z.log new file mode 100644 index 0000000000000000000000000000000000000000..270f33c509ef0be9640eac47a9b94696ae764b21 --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed42.20260820T041452Z.log @@ -0,0 +1,647 @@ +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-13000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-13000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 42, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed42", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-13000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-13000... +Preserving checkpoint LR state for exact resume: current=[0.0004073888929652048], schedule_base=[0.0005] +Resumed from checkpoint at step 13000 (epoch 0, micro_step 130000) +Resuming training from step 13000 +Resuming wandb run jqstnvej +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run jqstnvej +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-jqstnvej +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed42 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/jqstnvej +Starting training... +Validation at step 13000: Loss = 3.0665, Perplexity = 21.47 +wandb: WARNING Tried to log to step 13000 that is less than the current step 13021. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 13001: dense (dense step interval) +Step 13010/44000 | Loss: 3.0287 | LR: 4.07e-04 | 30165.3ms/step | 0.7 samples/s +wandb: WARNING Tried to log to step 13010 that is less than the current step 13021. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13020/44000 | Loss: 3.0440 | LR: 4.07e-04 | 1375.2ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13020 that is less than the current step 13021. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13030/44000 | Loss: 3.0257 | LR: 4.07e-04 | 1378.1ms/step | 14.5 samples/s +Step 13040/44000 | Loss: 3.0862 | LR: 4.07e-04 | 1380.1ms/step | 14.5 samples/s +Step 13050/44000 | Loss: 3.0640 | LR: 4.07e-04 | 1380.4ms/step | 14.5 samples/s +Step 13060/44000 | Loss: 3.0764 | LR: 4.07e-04 | 1379.9ms/step | 14.5 samples/s +Step 13070/44000 | Loss: 3.0550 | LR: 4.06e-04 | 1384.0ms/step | 14.5 samples/s +Step 13080/44000 | Loss: 3.0384 | LR: 4.06e-04 | 1386.1ms/step | 14.4 samples/s +Step 13090/44000 | Loss: 3.0527 | LR: 4.06e-04 | 1385.1ms/step | 14.4 samples/s +Step 13100/44000 | Loss: 3.0500 | LR: 4.06e-04 | 1386.4ms/step | 14.4 samples/s +Step 13110/44000 | Loss: 3.0585 | LR: 4.06e-04 | 1385.3ms/step | 14.4 samples/s +Step 13120/44000 | Loss: 3.0356 | LR: 4.06e-04 | 1385.5ms/step | 14.4 samples/s +Step 13130/44000 | Loss: 3.0393 | LR: 4.06e-04 | 1383.5ms/step | 14.5 samples/s +Step 13140/44000 | Loss: 3.0439 | LR: 4.05e-04 | 1385.2ms/step | 14.4 samples/s +Step 13150/44000 | Loss: 3.0256 | LR: 4.05e-04 | 1388.8ms/step | 14.4 samples/s +Step 13160/44000 | Loss: 3.0225 | LR: 4.05e-04 | 1388.1ms/step | 14.4 samples/s +Step 13170/44000 | Loss: 3.0341 | LR: 4.05e-04 | 1387.4ms/step | 14.4 samples/s +Step 13180/44000 | Loss: 3.0508 | LR: 4.05e-04 | 1387.3ms/step | 14.4 samples/s +Step 13190/44000 | Loss: 3.0768 | LR: 4.05e-04 | 1387.4ms/step | 14.4 samples/s +Step 13200/44000 | Loss: 3.0454 | LR: 4.05e-04 | 1388.1ms/step | 14.4 samples/s +Step 13210/44000 | Loss: 3.0508 | LR: 4.04e-04 | 1387.9ms/step | 14.4 samples/s +Step 13220/44000 | Loss: 3.0491 | LR: 4.04e-04 | 1389.3ms/step | 14.4 samples/s +Step 13230/44000 | Loss: 3.0392 | LR: 4.04e-04 | 1390.9ms/step | 14.4 samples/s +Step 13240/44000 | Loss: 3.0731 | LR: 4.04e-04 | 1387.6ms/step | 14.4 samples/s +Step 13250/44000 | Loss: 3.0547 | LR: 4.04e-04 | 1386.2ms/step | 14.4 samples/s +Step 13260/44000 | Loss: 3.0456 | LR: 4.04e-04 | 1387.3ms/step | 14.4 samples/s +Step 13270/44000 | Loss: 3.0193 | LR: 4.04e-04 | 1386.3ms/step | 14.4 samples/s +Step 13280/44000 | Loss: 3.0294 | LR: 4.03e-04 | 1388.8ms/step | 14.4 samples/s +Step 13290/44000 | Loss: 3.0476 | LR: 4.03e-04 | 1386.3ms/step | 14.4 samples/s +Step 13300/44000 | Loss: 3.0134 | LR: 4.03e-04 | 1387.1ms/step | 14.4 samples/s +Step 13310/44000 | Loss: 3.0769 | LR: 4.03e-04 | 1388.9ms/step | 14.4 samples/s +Step 13320/44000 | Loss: 3.0242 | LR: 4.03e-04 | 1391.2ms/step | 14.4 samples/s +Step 13330/44000 | Loss: 3.0419 | LR: 4.03e-04 | 1390.2ms/step | 14.4 samples/s +Step 13340/44000 | Loss: 3.0476 | LR: 4.03e-04 | 1389.4ms/step | 14.4 samples/s +Step 13350/44000 | Loss: 3.0307 | LR: 4.02e-04 | 1387.8ms/step | 14.4 samples/s +Step 13360/44000 | Loss: 3.0297 | LR: 4.02e-04 | 1388.4ms/step | 14.4 samples/s +Step 13370/44000 | Loss: 3.0360 | LR: 4.02e-04 | 1386.4ms/step | 14.4 samples/s +Step 13380/44000 | Loss: 3.0373 | LR: 4.02e-04 | 1387.7ms/step | 14.4 samples/s +Step 13390/44000 | Loss: 3.0441 | LR: 4.02e-04 | 1389.9ms/step | 14.4 samples/s +Step 13400/44000 | Loss: 3.0488 | LR: 4.02e-04 | 1388.0ms/step | 14.4 samples/s +Step 13410/44000 | Loss: 3.0572 | LR: 4.02e-04 | 1384.0ms/step | 14.5 samples/s +Step 13420/44000 | Loss: 3.0663 | LR: 4.01e-04 | 1384.1ms/step | 14.5 samples/s +Step 13430/44000 | Loss: 3.0234 | LR: 4.01e-04 | 1385.7ms/step | 14.4 samples/s +Step 13440/44000 | Loss: 3.0172 | LR: 4.01e-04 | 1389.9ms/step | 14.4 samples/s +Step 13450/44000 | Loss: 3.0330 | LR: 4.01e-04 | 1386.2ms/step | 14.4 samples/s +Step 13460/44000 | Loss: 3.0628 | LR: 4.01e-04 | 1386.8ms/step | 14.4 samples/s +Step 13470/44000 | Loss: 3.0356 | LR: 4.01e-04 | 1389.4ms/step | 14.4 samples/s +Step 13480/44000 | Loss: 3.0476 | LR: 4.01e-04 | 1392.8ms/step | 14.4 samples/s +Step 13490/44000 | Loss: 3.0474 | LR: 4.00e-04 | 1387.1ms/step | 14.4 samples/s +Step 13500/44000 | Loss: 3.0593 | LR: 4.00e-04 | 1387.7ms/step | 14.4 samples/s +Step 13510/44000 | Loss: 3.0408 | LR: 4.00e-04 | 1386.8ms/step | 14.4 samples/s +Step 13520/44000 | Loss: 3.0588 | LR: 4.00e-04 | 1386.9ms/step | 14.4 samples/s +Step 13530/44000 | Loss: 3.0276 | LR: 4.00e-04 | 1387.7ms/step | 14.4 samples/s +Step 13540/44000 | Loss: 3.0556 | LR: 4.00e-04 | 1385.9ms/step | 14.4 samples/s +Step 13550/44000 | Loss: 3.0772 | LR: 4.00e-04 | 1387.1ms/step | 14.4 samples/s +Step 13560/44000 | Loss: 3.0487 | LR: 3.99e-04 | 1384.6ms/step | 14.4 samples/s +Step 13570/44000 | Loss: 3.0246 | LR: 3.99e-04 | 1385.4ms/step | 14.4 samples/s +Step 13580/44000 | Loss: 3.0007 | LR: 3.99e-04 | 1387.3ms/step | 14.4 samples/s +Step 13590/44000 | Loss: 3.0154 | LR: 3.99e-04 | 1387.3ms/step | 14.4 samples/s +Step 13600/44000 | Loss: 3.0443 | LR: 3.99e-04 | 1387.9ms/step | 14.4 samples/s +Step 13610/44000 | Loss: 3.0471 | LR: 3.99e-04 | 1388.1ms/step | 14.4 samples/s +Step 13620/44000 | Loss: 3.0051 | LR: 3.98e-04 | 1389.2ms/step | 14.4 samples/s +Step 13630/44000 | Loss: 3.0344 | LR: 3.98e-04 | 1389.6ms/step | 14.4 samples/s +Step 13640/44000 | Loss: 3.0839 | LR: 3.98e-04 | 1386.4ms/step | 14.4 samples/s +Step 13650/44000 | Loss: 3.0618 | LR: 3.98e-04 | 1385.2ms/step | 14.4 samples/s +Step 13660/44000 | Loss: 3.0412 | LR: 3.98e-04 | 1385.2ms/step | 14.4 samples/s +Step 13670/44000 | Loss: 3.0370 | LR: 3.98e-04 | 1386.4ms/step | 14.4 samples/s +Step 13680/44000 | Loss: 3.0431 | LR: 3.98e-04 | 1387.5ms/step | 14.4 samples/s +Step 13690/44000 | Loss: 3.0439 | LR: 3.97e-04 | 1387.2ms/step | 14.4 samples/s +Step 13700/44000 | Loss: 3.0576 | LR: 3.97e-04 | 1386.3ms/step | 14.4 samples/s +Step 13710/44000 | Loss: 3.0240 | LR: 3.97e-04 | 1388.1ms/step | 14.4 samples/s +Step 13720/44000 | Loss: 3.0602 | LR: 3.97e-04 | 1389.9ms/step | 14.4 samples/s +Step 13730/44000 | Loss: 3.0537 | LR: 3.97e-04 | 1387.3ms/step | 14.4 samples/s +Step 13740/44000 | Loss: 3.0484 | LR: 3.97e-04 | 1387.1ms/step | 14.4 samples/s +Step 13750/44000 | Loss: 3.0405 | LR: 3.97e-04 | 1388.1ms/step | 14.4 samples/s +Step 13760/44000 | Loss: 3.0393 | LR: 3.96e-04 | 1389.5ms/step | 14.4 samples/s +Step 13770/44000 | Loss: 3.0233 | LR: 3.96e-04 | 1386.4ms/step | 14.4 samples/s +Step 13780/44000 | Loss: 3.0353 | LR: 3.96e-04 | 1390.2ms/step | 14.4 samples/s +Step 13790/44000 | Loss: 3.0609 | LR: 3.96e-04 | 1388.8ms/step | 14.4 samples/s +Step 13800/44000 | Loss: 3.0408 | LR: 3.96e-04 | 1384.2ms/step | 14.4 samples/s +Step 13810/44000 | Loss: 3.0345 | LR: 3.96e-04 | 1385.0ms/step | 14.4 samples/s +Step 13820/44000 | Loss: 3.0194 | LR: 3.96e-04 | 1387.9ms/step | 14.4 samples/s +Step 13830/44000 | Loss: 3.0231 | LR: 3.95e-04 | 1386.2ms/step | 14.4 samples/s +Step 13840/44000 | Loss: 3.0142 | LR: 3.95e-04 | 1387.7ms/step | 14.4 samples/s +Step 13850/44000 | Loss: 3.0426 | LR: 3.95e-04 | 1388.0ms/step | 14.4 samples/s +Step 13860/44000 | Loss: 3.0148 | LR: 3.95e-04 | 1385.9ms/step | 14.4 samples/s +Step 13870/44000 | Loss: 3.0317 | LR: 3.95e-04 | 1389.2ms/step | 14.4 samples/s +Step 13880/44000 | Loss: 3.0121 | LR: 3.95e-04 | 1384.5ms/step | 14.4 samples/s +Step 13890/44000 | Loss: 3.0297 | LR: 3.95e-04 | 1385.9ms/step | 14.4 samples/s +Step 13900/44000 | Loss: 3.0451 | LR: 3.94e-04 | 1387.3ms/step | 14.4 samples/s +Step 13910/44000 | Loss: 2.9991 | LR: 3.94e-04 | 1388.2ms/step | 14.4 samples/s +Step 13920/44000 | Loss: 3.0000 | LR: 3.94e-04 | 1387.5ms/step | 14.4 samples/s +Step 13930/44000 | Loss: 3.0256 | LR: 3.94e-04 | 1387.4ms/step | 14.4 samples/s +Step 13940/44000 | Loss: 3.0589 | LR: 3.94e-04 | 1390.6ms/step | 14.4 samples/s +Step 13950/44000 | Loss: 3.0292 | LR: 3.94e-04 | 1387.7ms/step | 14.4 samples/s +Step 13960/44000 | Loss: 3.0183 | LR: 3.93e-04 | 1388.6ms/step | 14.4 samples/s +Step 13970/44000 | Loss: 3.0254 | LR: 3.93e-04 | 1386.7ms/step | 14.4 samples/s +Step 13980/44000 | Loss: 3.0400 | LR: 3.93e-04 | 1388.6ms/step | 14.4 samples/s +Step 13990/44000 | Loss: 3.0382 | LR: 3.93e-04 | 1389.5ms/step | 14.4 samples/s +Step 14000/44000 | Loss: 3.0055 | LR: 3.93e-04 | 1386.5ms/step | 14.4 samples/s +Validation at step 14000: Loss = 3.0514, Perplexity = 21.14 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-14000 +Step 14010/44000 | Loss: 3.0195 | LR: 3.93e-04 | 3910.8ms/step | 5.1 samples/s +Step 14020/44000 | Loss: 3.0264 | LR: 3.93e-04 | 1377.6ms/step | 14.5 samples/s +Step 14030/44000 | Loss: 3.0339 | LR: 3.92e-04 | 1379.4ms/step | 14.5 samples/s +Step 14040/44000 | Loss: 3.0399 | LR: 3.92e-04 | 1384.5ms/step | 14.4 samples/s +Step 14050/44000 | Loss: 3.0264 | LR: 3.92e-04 | 1383.6ms/step | 14.5 samples/s +Step 14060/44000 | Loss: 3.0510 | LR: 3.92e-04 | 1388.0ms/step | 14.4 samples/s +Step 14070/44000 | Loss: 3.0416 | LR: 3.92e-04 | 1388.5ms/step | 14.4 samples/s +Step 14080/44000 | Loss: 2.9989 | LR: 3.92e-04 | 1392.9ms/step | 14.4 samples/s +Step 14090/44000 | Loss: 3.0299 | LR: 3.92e-04 | 1390.4ms/step | 14.4 samples/s +Step 14100/44000 | Loss: 3.0058 | LR: 3.91e-04 | 1394.3ms/step | 14.3 samples/s +Step 14110/44000 | Loss: 3.0750 | LR: 3.91e-04 | 1393.8ms/step | 14.3 samples/s +Step 14120/44000 | Loss: 3.0350 | LR: 3.91e-04 | 1395.7ms/step | 14.3 samples/s +Step 14130/44000 | Loss: 3.0346 | LR: 3.91e-04 | 1391.6ms/step | 14.4 samples/s +Step 14140/44000 | Loss: 3.0181 | LR: 3.91e-04 | 1394.6ms/step | 14.3 samples/s +Step 14150/44000 | Loss: 3.0178 | LR: 3.91e-04 | 1392.2ms/step | 14.4 samples/s +Step 14160/44000 | Loss: 3.0493 | LR: 3.90e-04 | 1393.9ms/step | 14.3 samples/s +Step 14170/44000 | Loss: 3.0609 | LR: 3.90e-04 | 1398.1ms/step | 14.3 samples/s +Step 14180/44000 | Loss: 3.0742 | LR: 3.90e-04 | 1399.6ms/step | 14.3 samples/s +Step 14190/44000 | Loss: 3.0194 | LR: 3.90e-04 | 1394.1ms/step | 14.3 samples/s +Step 14200/44000 | Loss: 3.0512 | LR: 3.90e-04 | 1400.2ms/step | 14.3 samples/s +Step 14210/44000 | Loss: 3.0552 | LR: 3.90e-04 | 1399.9ms/step | 14.3 samples/s +Step 14220/44000 | Loss: 3.0104 | LR: 3.90e-04 | 1400.0ms/step | 14.3 samples/s +Step 14230/44000 | Loss: 3.0535 | LR: 3.89e-04 | 1400.0ms/step | 14.3 samples/s +Step 14240/44000 | Loss: 3.0235 | LR: 3.89e-04 | 1398.5ms/step | 14.3 samples/s +Step 14250/44000 | Loss: 3.0587 | LR: 3.89e-04 | 1397.3ms/step | 14.3 samples/s +Step 14260/44000 | Loss: 3.0575 | LR: 3.89e-04 | 1396.4ms/step | 14.3 samples/s +Step 14270/44000 | Loss: 2.9848 | LR: 3.89e-04 | 1397.7ms/step | 14.3 samples/s +Step 14280/44000 | Loss: 3.0336 | LR: 3.89e-04 | 1396.2ms/step | 14.3 samples/s +Step 14290/44000 | Loss: 3.0208 | LR: 3.89e-04 | 1398.0ms/step | 14.3 samples/s +Step 14300/44000 | Loss: 3.0309 | LR: 3.88e-04 | 1399.8ms/step | 14.3 samples/s +Step 14310/44000 | Loss: 3.0246 | LR: 3.88e-04 | 1397.8ms/step | 14.3 samples/s +Step 14320/44000 | Loss: 3.0096 | LR: 3.88e-04 | 1400.3ms/step | 14.3 samples/s +Step 14330/44000 | Loss: 3.0129 | LR: 3.88e-04 | 1400.0ms/step | 14.3 samples/s +Step 14340/44000 | Loss: 3.0275 | LR: 3.88e-04 | 1399.5ms/step | 14.3 samples/s +Step 14350/44000 | Loss: 3.0316 | LR: 3.88e-04 | 1396.6ms/step | 14.3 samples/s +Step 14360/44000 | Loss: 3.0323 | LR: 3.87e-04 | 1399.7ms/step | 14.3 samples/s +Step 14370/44000 | Loss: 3.0338 | LR: 3.87e-04 | 1394.3ms/step | 14.3 samples/s +Step 14380/44000 | Loss: 3.0403 | LR: 3.87e-04 | 1393.4ms/step | 14.4 samples/s +Step 14390/44000 | Loss: 3.0455 | LR: 3.87e-04 | 1398.5ms/step | 14.3 samples/s +Step 14400/44000 | Loss: 3.0131 | LR: 3.87e-04 | 1399.9ms/step | 14.3 samples/s +Step 14410/44000 | Loss: 3.0276 | LR: 3.87e-04 | 1396.1ms/step | 14.3 samples/s +Step 14420/44000 | Loss: 3.0622 | LR: 3.87e-04 | 1398.2ms/step | 14.3 samples/s +Step 14430/44000 | Loss: 3.0231 | LR: 3.86e-04 | 1397.6ms/step | 14.3 samples/s +Step 14440/44000 | Loss: 3.0176 | LR: 3.86e-04 | 1396.1ms/step | 14.3 samples/s +Step 14450/44000 | Loss: 3.0216 | LR: 3.86e-04 | 1396.5ms/step | 14.3 samples/s +Step 14460/44000 | Loss: 2.9962 | LR: 3.86e-04 | 1399.7ms/step | 14.3 samples/s +Step 14470/44000 | Loss: 2.9985 | LR: 3.86e-04 | 1397.9ms/step | 14.3 samples/s +Step 14480/44000 | Loss: 3.0100 | LR: 3.86e-04 | 1400.2ms/step | 14.3 samples/s +Step 14490/44000 | Loss: 3.0250 | LR: 3.85e-04 | 1399.3ms/step | 14.3 samples/s +Step 14500/44000 | Loss: 3.0494 | LR: 3.85e-04 | 1398.6ms/step | 14.3 samples/s +Step 14510/44000 | Loss: 3.0211 | LR: 3.85e-04 | 1400.1ms/step | 14.3 samples/s +Step 14520/44000 | Loss: 3.0304 | LR: 3.85e-04 | 1399.8ms/step | 14.3 samples/s +Step 14530/44000 | Loss: 3.0245 | LR: 3.85e-04 | 1398.1ms/step | 14.3 samples/s +Step 14540/44000 | Loss: 3.0155 | LR: 3.85e-04 | 1397.9ms/step | 14.3 samples/s +Step 14550/44000 | Loss: 3.0204 | LR: 3.85e-04 | 1400.1ms/step | 14.3 samples/s +Step 14560/44000 | Loss: 3.0063 | LR: 3.84e-04 | 1397.6ms/step | 14.3 samples/s +Step 14570/44000 | Loss: 3.0307 | LR: 3.84e-04 | 1398.2ms/step | 14.3 samples/s +Step 14580/44000 | Loss: 3.0312 | LR: 3.84e-04 | 1397.9ms/step | 14.3 samples/s +Step 14590/44000 | Loss: 3.0091 | LR: 3.84e-04 | 1398.2ms/step | 14.3 samples/s +Step 14600/44000 | Loss: 3.0276 | LR: 3.84e-04 | 1398.7ms/step | 14.3 samples/s +Step 14610/44000 | Loss: 3.0123 | LR: 3.84e-04 | 1399.6ms/step | 14.3 samples/s +Step 14620/44000 | Loss: 3.0115 | LR: 3.83e-04 | 1397.8ms/step | 14.3 samples/s +Step 14630/44000 | Loss: 3.0249 | LR: 3.83e-04 | 1399.8ms/step | 14.3 samples/s +Step 14640/44000 | Loss: 3.0200 | LR: 3.83e-04 | 1399.6ms/step | 14.3 samples/s +Step 14650/44000 | Loss: 3.0181 | LR: 3.83e-04 | 1395.6ms/step | 14.3 samples/s +Step 14660/44000 | Loss: 2.9862 | LR: 3.83e-04 | 1398.8ms/step | 14.3 samples/s +Step 14670/44000 | Loss: 3.0156 | LR: 3.83e-04 | 1398.1ms/step | 14.3 samples/s +Step 14680/44000 | Loss: 3.0400 | LR: 3.83e-04 | 1394.5ms/step | 14.3 samples/s +Step 14690/44000 | Loss: 3.0197 | LR: 3.82e-04 | 1397.5ms/step | 14.3 samples/s +Step 14700/44000 | Loss: 3.0294 | LR: 3.82e-04 | 1398.0ms/step | 14.3 samples/s +Step 14710/44000 | Loss: 3.0228 | LR: 3.82e-04 | 1400.3ms/step | 14.3 samples/s +Step 14720/44000 | Loss: 3.0467 | LR: 3.82e-04 | 1395.8ms/step | 14.3 samples/s +Step 14730/44000 | Loss: 2.9995 | LR: 3.82e-04 | 1400.1ms/step | 14.3 samples/s +Step 14740/44000 | Loss: 3.0405 | LR: 3.82e-04 | 1399.7ms/step | 14.3 samples/s +Step 14750/44000 | Loss: 3.0521 | LR: 3.81e-04 | 1398.2ms/step | 14.3 samples/s +Step 14760/44000 | Loss: 3.0057 | LR: 3.81e-04 | 1397.8ms/step | 14.3 samples/s +Step 14770/44000 | Loss: 3.0283 | LR: 3.81e-04 | 1398.2ms/step | 14.3 samples/s +Step 14780/44000 | Loss: 2.9995 | LR: 3.81e-04 | 1398.1ms/step | 14.3 samples/s +Step 14790/44000 | Loss: 3.0225 | LR: 3.81e-04 | 1395.9ms/step | 14.3 samples/s +Step 14800/44000 | Loss: 2.9918 | LR: 3.81e-04 | 1398.2ms/step | 14.3 samples/s +Step 14810/44000 | Loss: 3.0154 | LR: 3.81e-04 | 1400.2ms/step | 14.3 samples/s +Step 14820/44000 | Loss: 3.0140 | LR: 3.80e-04 | 1398.7ms/step | 14.3 samples/s +Step 14830/44000 | Loss: 3.0100 | LR: 3.80e-04 | 1398.9ms/step | 14.3 samples/s +Step 14840/44000 | Loss: 2.9869 | LR: 3.80e-04 | 1398.1ms/step | 14.3 samples/s +Step 14850/44000 | Loss: 3.0370 | LR: 3.80e-04 | 1399.9ms/step | 14.3 samples/s +Step 14860/44000 | Loss: 3.0687 | LR: 3.80e-04 | 1400.1ms/step | 14.3 samples/s +Step 14870/44000 | Loss: 3.0336 | LR: 3.80e-04 | 1395.7ms/step | 14.3 samples/s +Step 14880/44000 | Loss: 3.0418 | LR: 3.79e-04 | 1396.1ms/step | 14.3 samples/s +Step 14890/44000 | Loss: 2.9973 | LR: 3.79e-04 | 1398.5ms/step | 14.3 samples/s +Step 14900/44000 | Loss: 3.0477 | LR: 3.79e-04 | 1397.7ms/step | 14.3 samples/s +Step 14910/44000 | Loss: 3.0226 | LR: 3.79e-04 | 1400.0ms/step | 14.3 samples/s +Step 14920/44000 | Loss: 3.0062 | LR: 3.79e-04 | 1397.9ms/step | 14.3 samples/s +Step 14930/44000 | Loss: 3.0351 | LR: 3.79e-04 | 1395.7ms/step | 14.3 samples/s +Step 14940/44000 | Loss: 2.9710 | LR: 3.79e-04 | 1400.3ms/step | 14.3 samples/s +Step 14950/44000 | Loss: 3.0007 | LR: 3.78e-04 | 1398.0ms/step | 14.3 samples/s +Step 14960/44000 | Loss: 3.0032 | LR: 3.78e-04 | 1398.0ms/step | 14.3 samples/s +Step 14970/44000 | Loss: 3.0145 | LR: 3.78e-04 | 1395.5ms/step | 14.3 samples/s +Step 14980/44000 | Loss: 2.9904 | LR: 3.78e-04 | 1398.3ms/step | 14.3 samples/s +Step 14990/44000 | Loss: 3.0434 | LR: 3.78e-04 | 1400.0ms/step | 14.3 samples/s +Step 15000/44000 | Loss: 3.0264 | LR: 3.78e-04 | 1400.3ms/step | 14.3 samples/s +Validation at step 15000: Loss = 3.0379, Perplexity = 20.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-15000 +Step 15010/44000 | Loss: 3.0395 | LR: 3.77e-04 | 3905.4ms/step | 5.1 samples/s +Step 15020/44000 | Loss: 2.9784 | LR: 3.77e-04 | 1383.4ms/step | 14.5 samples/s +Step 15030/44000 | Loss: 3.0472 | LR: 3.77e-04 | 1383.1ms/step | 14.5 samples/s +Step 15040/44000 | Loss: 3.0234 | LR: 3.77e-04 | 1381.6ms/step | 14.5 samples/s +Step 15050/44000 | Loss: 3.0335 | LR: 3.77e-04 | 1384.9ms/step | 14.4 samples/s +Step 15060/44000 | Loss: 3.0408 | LR: 3.77e-04 | 1388.4ms/step | 14.4 samples/s +Step 15070/44000 | Loss: 3.0389 | LR: 3.77e-04 | 1391.9ms/step | 14.4 samples/s +Step 15080/44000 | Loss: 3.0075 | LR: 3.76e-04 | 1397.3ms/step | 14.3 samples/s +Step 15090/44000 | Loss: 3.0260 | LR: 3.76e-04 | 1396.1ms/step | 14.3 samples/s +Step 15100/44000 | Loss: 3.0122 | LR: 3.76e-04 | 1399.7ms/step | 14.3 samples/s +Step 15110/44000 | Loss: 3.0386 | LR: 3.76e-04 | 1398.0ms/step | 14.3 samples/s +Step 15120/44000 | Loss: 3.0303 | LR: 3.76e-04 | 1396.0ms/step | 14.3 samples/s +Step 15130/44000 | Loss: 3.0005 | LR: 3.76e-04 | 1396.0ms/step | 14.3 samples/s +Step 15140/44000 | Loss: 3.0191 | LR: 3.75e-04 | 1396.2ms/step | 14.3 samples/s +Step 15150/44000 | Loss: 3.0069 | LR: 3.75e-04 | 1396.5ms/step | 14.3 samples/s +Step 15160/44000 | Loss: 3.0318 | LR: 3.75e-04 | 1393.4ms/step | 14.4 samples/s +Step 15170/44000 | Loss: 3.0635 | LR: 3.75e-04 | 1394.1ms/step | 14.3 samples/s +Step 15180/44000 | Loss: 2.9948 | LR: 3.75e-04 | 1394.7ms/step | 14.3 samples/s +Step 15190/44000 | Loss: 3.0487 | LR: 3.75e-04 | 1396.7ms/step | 14.3 samples/s +Step 15200/44000 | Loss: 3.0296 | LR: 3.74e-04 | 1394.4ms/step | 14.3 samples/s +Step 15210/44000 | Loss: 3.0246 | LR: 3.74e-04 | 1395.4ms/step | 14.3 samples/s +Step 15220/44000 | Loss: 2.9780 | LR: 3.74e-04 | 1396.0ms/step | 14.3 samples/s +Step 15230/44000 | Loss: 3.0285 | LR: 3.74e-04 | 1396.4ms/step | 14.3 samples/s +Step 15240/44000 | Loss: 3.0217 | LR: 3.74e-04 | 1398.1ms/step | 14.3 samples/s +Step 15250/44000 | Loss: 3.0170 | LR: 3.74e-04 | 1400.0ms/step | 14.3 samples/s +Step 15260/44000 | Loss: 3.0567 | LR: 3.74e-04 | 1398.2ms/step | 14.3 samples/s +Step 15270/44000 | Loss: 3.0121 | LR: 3.73e-04 | 1400.0ms/step | 14.3 samples/s +Step 15280/44000 | Loss: 2.9865 | LR: 3.73e-04 | 1400.2ms/step | 14.3 samples/s +Step 15290/44000 | Loss: 3.0229 | LR: 3.73e-04 | 1399.9ms/step | 14.3 samples/s +Step 15300/44000 | Loss: 2.9984 | LR: 3.73e-04 | 1399.8ms/step | 14.3 samples/s +Step 15310/44000 | Loss: 3.0261 | LR: 3.73e-04 | 1397.9ms/step | 14.3 samples/s +Step 15320/44000 | Loss: 2.9886 | LR: 3.73e-04 | 1396.9ms/step | 14.3 samples/s +Step 15330/44000 | Loss: 2.9936 | LR: 3.72e-04 | 1396.5ms/step | 14.3 samples/s +Step 15340/44000 | Loss: 3.0132 | LR: 3.72e-04 | 1394.6ms/step | 14.3 samples/s +Step 15350/44000 | Loss: 3.0146 | LR: 3.72e-04 | 1394.9ms/step | 14.3 samples/s +Step 15360/44000 | Loss: 3.0296 | LR: 3.72e-04 | 1393.0ms/step | 14.4 samples/s +Step 15370/44000 | Loss: 3.0021 | LR: 3.72e-04 | 1395.6ms/step | 14.3 samples/s +Step 15380/44000 | Loss: 2.9956 | LR: 3.72e-04 | 1392.9ms/step | 14.4 samples/s +Step 15390/44000 | Loss: 2.9975 | LR: 3.71e-04 | 1395.1ms/step | 14.3 samples/s +Step 15400/44000 | Loss: 3.0158 | LR: 3.71e-04 | 1398.6ms/step | 14.3 samples/s +Step 15410/44000 | Loss: 3.0168 | LR: 3.71e-04 | 1395.9ms/step | 14.3 samples/s +Step 15420/44000 | Loss: 3.0116 | LR: 3.71e-04 | 1398.4ms/step | 14.3 samples/s +Step 15430/44000 | Loss: 3.0107 | LR: 3.71e-04 | 1398.0ms/step | 14.3 samples/s +Step 15440/44000 | Loss: 2.9948 | LR: 3.71e-04 | 1396.0ms/step | 14.3 samples/s +Step 15450/44000 | Loss: 3.0260 | LR: 3.71e-04 | 1397.8ms/step | 14.3 samples/s +Step 15460/44000 | Loss: 3.0192 | LR: 3.70e-04 | 1396.0ms/step | 14.3 samples/s +Step 15470/44000 | Loss: 2.9916 | LR: 3.70e-04 | 1400.0ms/step | 14.3 samples/s +Step 15480/44000 | Loss: 3.0190 | LR: 3.70e-04 | 1399.9ms/step | 14.3 samples/s +Step 15490/44000 | Loss: 3.0406 | LR: 3.70e-04 | 1398.2ms/step | 14.3 samples/s +Step 15500/44000 | Loss: 3.0192 | LR: 3.70e-04 | 1399.9ms/step | 14.3 samples/s +Step 15510/44000 | Loss: 2.9820 | LR: 3.70e-04 | 1400.0ms/step | 14.3 samples/s +Step 15520/44000 | Loss: 3.0317 | LR: 3.69e-04 | 1400.0ms/step | 14.3 samples/s +Step 15530/44000 | Loss: 2.9941 | LR: 3.69e-04 | 1397.6ms/step | 14.3 samples/s +Step 15540/44000 | Loss: 3.0270 | LR: 3.69e-04 | 1398.6ms/step | 14.3 samples/s +Step 15550/44000 | Loss: 3.0610 | LR: 3.69e-04 | 1398.1ms/step | 14.3 samples/s +Step 15560/44000 | Loss: 3.0169 | LR: 3.69e-04 | 1399.8ms/step | 14.3 samples/s +Step 15570/44000 | Loss: 3.0076 | LR: 3.69e-04 | 1400.2ms/step | 14.3 samples/s +Step 15580/44000 | Loss: 2.9940 | LR: 3.68e-04 | 1393.8ms/step | 14.3 samples/s +Step 15590/44000 | Loss: 3.0269 | LR: 3.68e-04 | 1398.6ms/step | 14.3 samples/s +Step 15600/44000 | Loss: 3.0211 | LR: 3.68e-04 | 1397.5ms/step | 14.3 samples/s +Step 15610/44000 | Loss: 3.0502 | LR: 3.68e-04 | 1399.8ms/step | 14.3 samples/s +Step 15620/44000 | Loss: 3.0169 | LR: 3.68e-04 | 1393.8ms/step | 14.3 samples/s +Step 15630/44000 | Loss: 3.0036 | LR: 3.68e-04 | 1394.1ms/step | 14.3 samples/s +Step 15640/44000 | Loss: 3.0207 | LR: 3.67e-04 | 1394.0ms/step | 14.3 samples/s +Step 15650/44000 | Loss: 3.0254 | LR: 3.67e-04 | 1396.2ms/step | 14.3 samples/s +Step 15660/44000 | Loss: 3.0146 | LR: 3.67e-04 | 1400.1ms/step | 14.3 samples/s +Step 15670/44000 | Loss: 3.0043 | LR: 3.67e-04 | 1397.8ms/step | 14.3 samples/s +Step 15680/44000 | Loss: 2.9986 | LR: 3.67e-04 | 1400.4ms/step | 14.3 samples/s +Step 15690/44000 | Loss: 3.0438 | LR: 3.67e-04 | 1397.4ms/step | 14.3 samples/s +Step 15700/44000 | Loss: 2.9976 | LR: 3.67e-04 | 1398.1ms/step | 14.3 samples/s +Step 15710/44000 | Loss: 2.9913 | LR: 3.66e-04 | 1396.1ms/step | 14.3 samples/s +Step 15720/44000 | Loss: 3.0242 | LR: 3.66e-04 | 1396.5ms/step | 14.3 samples/s +Step 15730/44000 | Loss: 3.0170 | LR: 3.66e-04 | 1395.9ms/step | 14.3 samples/s +Step 15740/44000 | Loss: 3.0116 | LR: 3.66e-04 | 1397.7ms/step | 14.3 samples/s +Step 15750/44000 | Loss: 3.0043 | LR: 3.66e-04 | 1396.1ms/step | 14.3 samples/s +Step 15760/44000 | Loss: 2.9945 | LR: 3.66e-04 | 1399.8ms/step | 14.3 samples/s +Step 15770/44000 | Loss: 3.0071 | LR: 3.65e-04 | 1398.2ms/step | 14.3 samples/s +Step 15780/44000 | Loss: 3.0274 | LR: 3.65e-04 | 1400.2ms/step | 14.3 samples/s +Step 15790/44000 | Loss: 3.0277 | LR: 3.65e-04 | 1397.5ms/step | 14.3 samples/s +Step 15800/44000 | Loss: 2.9978 | LR: 3.65e-04 | 1400.1ms/step | 14.3 samples/s +Step 15810/44000 | Loss: 3.0099 | LR: 3.65e-04 | 1397.7ms/step | 14.3 samples/s +Step 15820/44000 | Loss: 3.0154 | LR: 3.65e-04 | 1397.9ms/step | 14.3 samples/s +Step 15830/44000 | Loss: 3.0436 | LR: 3.64e-04 | 1394.5ms/step | 14.3 samples/s +Step 15840/44000 | Loss: 2.9938 | LR: 3.64e-04 | 1397.8ms/step | 14.3 samples/s +Step 15850/44000 | Loss: 2.9820 | LR: 3.64e-04 | 1397.6ms/step | 14.3 samples/s +Step 15860/44000 | Loss: 3.0010 | LR: 3.64e-04 | 1396.4ms/step | 14.3 samples/s +Step 15870/44000 | Loss: 3.0045 | LR: 3.64e-04 | 1394.4ms/step | 14.3 samples/s +Step 15880/44000 | Loss: 3.0196 | LR: 3.64e-04 | 1399.9ms/step | 14.3 samples/s +Step 15890/44000 | Loss: 3.0452 | LR: 3.63e-04 | 1397.9ms/step | 14.3 samples/s +Step 15900/44000 | Loss: 2.9920 | LR: 3.63e-04 | 1397.0ms/step | 14.3 samples/s +Step 15910/44000 | Loss: 3.0221 | LR: 3.63e-04 | 1393.3ms/step | 14.4 samples/s +Step 15920/44000 | Loss: 3.0163 | LR: 3.63e-04 | 1399.8ms/step | 14.3 samples/s +Step 15930/44000 | Loss: 3.0266 | LR: 3.63e-04 | 1400.0ms/step | 14.3 samples/s +Step 15940/44000 | Loss: 3.0067 | LR: 3.63e-04 | 1393.8ms/step | 14.3 samples/s +Step 15950/44000 | Loss: 3.0263 | LR: 3.62e-04 | 1396.5ms/step | 14.3 samples/s +Step 15960/44000 | Loss: 3.0358 | LR: 3.62e-04 | 1395.6ms/step | 14.3 samples/s +Step 15970/44000 | Loss: 3.0408 | LR: 3.62e-04 | 1398.1ms/step | 14.3 samples/s +Step 15980/44000 | Loss: 2.9845 | LR: 3.62e-04 | 1398.0ms/step | 14.3 samples/s +Step 15990/44000 | Loss: 3.0149 | LR: 3.62e-04 | 1398.1ms/step | 14.3 samples/s +Step 16000/44000 | Loss: 3.0248 | LR: 3.62e-04 | 1399.4ms/step | 14.3 samples/s +Validation at step 16000: Loss = 3.0281, Perplexity = 20.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-16000 +Step 16010/44000 | Loss: 2.9864 | LR: 3.62e-04 | 3888.2ms/step | 5.1 samples/s +Step 16020/44000 | Loss: 2.9857 | LR: 3.61e-04 | 1379.9ms/step | 14.5 samples/s +Step 16030/44000 | Loss: 3.0181 | LR: 3.61e-04 | 1384.4ms/step | 14.4 samples/s +Step 16040/44000 | Loss: 3.0149 | LR: 3.61e-04 | 1386.6ms/step | 14.4 samples/s +Step 16050/44000 | Loss: 3.0004 | LR: 3.61e-04 | 1382.8ms/step | 14.5 samples/s +Step 16060/44000 | Loss: 3.0185 | LR: 3.61e-04 | 1386.5ms/step | 14.4 samples/s +Step 16070/44000 | Loss: 3.0414 | LR: 3.61e-04 | 1387.2ms/step | 14.4 samples/s +Step 16080/44000 | Loss: 3.0259 | LR: 3.60e-04 | 1392.0ms/step | 14.4 samples/s +Step 16090/44000 | Loss: 2.9867 | LR: 3.60e-04 | 1393.1ms/step | 14.4 samples/s +Step 16100/44000 | Loss: 3.0156 | LR: 3.60e-04 | 1391.6ms/step | 14.4 samples/s +Step 16110/44000 | Loss: 2.9859 | LR: 3.60e-04 | 1393.9ms/step | 14.3 samples/s +Step 16120/44000 | Loss: 2.9940 | LR: 3.60e-04 | 1394.4ms/step | 14.3 samples/s +Step 16130/44000 | Loss: 3.0248 | LR: 3.60e-04 | 1393.7ms/step | 14.4 samples/s +Step 16140/44000 | Loss: 2.9950 | LR: 3.59e-04 | 1394.0ms/step | 14.3 samples/s +Step 16150/44000 | Loss: 2.9980 | LR: 3.59e-04 | 1392.2ms/step | 14.4 samples/s +Step 16160/44000 | Loss: 3.0314 | LR: 3.59e-04 | 1395.5ms/step | 14.3 samples/s +Step 16170/44000 | Loss: 3.0048 | LR: 3.59e-04 | 1395.9ms/step | 14.3 samples/s +Step 16180/44000 | Loss: 2.9694 | LR: 3.59e-04 | 1391.4ms/step | 14.4 samples/s +Step 16190/44000 | Loss: 3.0218 | LR: 3.59e-04 | 1395.1ms/step | 14.3 samples/s +Step 16200/44000 | Loss: 3.0063 | LR: 3.58e-04 | 1400.2ms/step | 14.3 samples/s +Step 16210/44000 | Loss: 3.0114 | LR: 3.58e-04 | 1395.8ms/step | 14.3 samples/s +Step 16220/44000 | Loss: 2.9844 | LR: 3.58e-04 | 1394.1ms/step | 14.3 samples/s +Step 16230/44000 | Loss: 3.0263 | LR: 3.58e-04 | 1395.9ms/step | 14.3 samples/s +Step 16240/44000 | Loss: 3.0202 | LR: 3.58e-04 | 1396.3ms/step | 14.3 samples/s +Step 16250/44000 | Loss: 2.9751 | LR: 3.58e-04 | 1397.7ms/step | 14.3 samples/s +Step 16260/44000 | Loss: 3.0133 | LR: 3.57e-04 | 1394.7ms/step | 14.3 samples/s +Step 16270/44000 | Loss: 3.0227 | LR: 3.57e-04 | 1395.3ms/step | 14.3 samples/s +Step 16280/44000 | Loss: 3.0159 | LR: 3.57e-04 | 1398.3ms/step | 14.3 samples/s +Step 16290/44000 | Loss: 3.0043 | LR: 3.57e-04 | 1395.9ms/step | 14.3 samples/s +Step 16300/44000 | Loss: 3.0060 | LR: 3.57e-04 | 1395.7ms/step | 14.3 samples/s +Step 16310/44000 | Loss: 2.9945 | LR: 3.57e-04 | 1398.3ms/step | 14.3 samples/s +Step 16320/44000 | Loss: 2.9799 | LR: 3.56e-04 | 1396.1ms/step | 14.3 samples/s +Step 16330/44000 | Loss: 3.0184 | LR: 3.56e-04 | 1397.7ms/step | 14.3 samples/s +Step 16340/44000 | Loss: 3.0043 | LR: 3.56e-04 | 1396.3ms/step | 14.3 samples/s +Step 16350/44000 | Loss: 3.0138 | LR: 3.56e-04 | 1393.5ms/step | 14.4 samples/s +Step 16360/44000 | Loss: 2.9752 | LR: 3.56e-04 | 1394.3ms/step | 14.3 samples/s +Step 16370/44000 | Loss: 2.9994 | LR: 3.56e-04 | 1398.1ms/step | 14.3 samples/s +Step 16380/44000 | Loss: 2.9924 | LR: 3.55e-04 | 1397.9ms/step | 14.3 samples/s +Step 16390/44000 | Loss: 2.9954 | LR: 3.55e-04 | 1397.2ms/step | 14.3 samples/s +Step 16400/44000 | Loss: 2.9960 | LR: 3.55e-04 | 1398.7ms/step | 14.3 samples/s +Step 16410/44000 | Loss: 2.9906 | LR: 3.55e-04 | 1396.2ms/step | 14.3 samples/s +Step 16420/44000 | Loss: 3.0041 | LR: 3.55e-04 | 1399.7ms/step | 14.3 samples/s +Step 16430/44000 | Loss: 3.0090 | LR: 3.55e-04 | 1400.4ms/step | 14.3 samples/s +Step 16440/44000 | Loss: 3.0280 | LR: 3.54e-04 | 1395.8ms/step | 14.3 samples/s +Step 16450/44000 | Loss: 3.0094 | LR: 3.54e-04 | 1395.9ms/step | 14.3 samples/s +Step 16460/44000 | Loss: 2.9559 | LR: 3.54e-04 | 1396.1ms/step | 14.3 samples/s +Step 16470/44000 | Loss: 2.9743 | LR: 3.54e-04 | 1396.1ms/step | 14.3 samples/s +Step 16480/44000 | Loss: 3.0436 | LR: 3.54e-04 | 1396.0ms/step | 14.3 samples/s +Step 16490/44000 | Loss: 2.9995 | LR: 3.54e-04 | 1398.2ms/step | 14.3 samples/s +Step 16500/44000 | Loss: 3.0170 | LR: 3.53e-04 | 1393.9ms/step | 14.3 samples/s +Step 16510/44000 | Loss: 3.0246 | LR: 3.53e-04 | 1395.8ms/step | 14.3 samples/s +Step 16520/44000 | Loss: 3.0180 | LR: 3.53e-04 | 1392.1ms/step | 14.4 samples/s +Step 16530/44000 | Loss: 2.9973 | LR: 3.53e-04 | 1400.1ms/step | 14.3 samples/s +Step 16540/44000 | Loss: 3.0152 | LR: 3.53e-04 | 1400.2ms/step | 14.3 samples/s +Step 16550/44000 | Loss: 2.9887 | LR: 3.53e-04 | 1397.9ms/step | 14.3 samples/s +Step 16560/44000 | Loss: 2.9889 | LR: 3.52e-04 | 1399.9ms/step | 14.3 samples/s +Step 16570/44000 | Loss: 3.0022 | LR: 3.52e-04 | 1395.7ms/step | 14.3 samples/s +Step 16580/44000 | Loss: 2.9782 | LR: 3.52e-04 | 1395.7ms/step | 14.3 samples/s +Step 16590/44000 | Loss: 2.9683 | LR: 3.52e-04 | 1396.8ms/step | 14.3 samples/s +Step 16600/44000 | Loss: 3.0078 | LR: 3.52e-04 | 1399.9ms/step | 14.3 samples/s +Step 16610/44000 | Loss: 2.9827 | LR: 3.52e-04 | 1398.1ms/step | 14.3 samples/s +Step 16620/44000 | Loss: 2.9881 | LR: 3.51e-04 | 1396.6ms/step | 14.3 samples/s +Step 16630/44000 | Loss: 2.9986 | LR: 3.51e-04 | 1391.3ms/step | 14.4 samples/s +Step 16640/44000 | Loss: 3.0085 | LR: 3.51e-04 | 1399.7ms/step | 14.3 samples/s +Step 16650/44000 | Loss: 3.0182 | LR: 3.51e-04 | 1399.6ms/step | 14.3 samples/s +Step 16660/44000 | Loss: 3.0038 | LR: 3.51e-04 | 1392.8ms/step | 14.4 samples/s +Step 16670/44000 | Loss: 3.0267 | LR: 3.51e-04 | 1397.5ms/step | 14.3 samples/s +Step 16680/44000 | Loss: 3.0212 | LR: 3.50e-04 | 1400.3ms/step | 14.3 samples/s +Step 16690/44000 | Loss: 3.0109 | LR: 3.50e-04 | 1398.2ms/step | 14.3 samples/s +Step 16700/44000 | Loss: 2.9904 | LR: 3.50e-04 | 1397.8ms/step | 14.3 samples/s +Step 16710/44000 | Loss: 2.9792 | LR: 3.50e-04 | 1398.0ms/step | 14.3 samples/s +Step 16720/44000 | Loss: 2.9861 | LR: 3.50e-04 | 1394.3ms/step | 14.3 samples/s +Step 16730/44000 | Loss: 3.0105 | LR: 3.50e-04 | 1396.1ms/step | 14.3 samples/s +Step 16740/44000 | Loss: 3.0149 | LR: 3.49e-04 | 1399.5ms/step | 14.3 samples/s +Step 16750/44000 | Loss: 3.0014 | LR: 3.49e-04 | 1396.1ms/step | 14.3 samples/s +Step 16760/44000 | Loss: 3.0067 | LR: 3.49e-04 | 1394.1ms/step | 14.3 samples/s +Step 16770/44000 | Loss: 2.9674 | LR: 3.49e-04 | 1397.6ms/step | 14.3 samples/s +Step 16780/44000 | Loss: 3.0223 | LR: 3.49e-04 | 1398.1ms/step | 14.3 samples/s +Step 16790/44000 | Loss: 2.9809 | LR: 3.49e-04 | 1398.1ms/step | 14.3 samples/s +Step 16800/44000 | Loss: 3.0072 | LR: 3.48e-04 | 1395.4ms/step | 14.3 samples/s +Step 16810/44000 | Loss: 2.9969 | LR: 3.48e-04 | 1396.0ms/step | 14.3 samples/s +Step 16820/44000 | Loss: 2.9723 | LR: 3.48e-04 | 1390.7ms/step | 14.4 samples/s +Step 16830/44000 | Loss: 3.0121 | LR: 3.48e-04 | 1395.6ms/step | 14.3 samples/s +Step 16840/44000 | Loss: 2.9837 | LR: 3.48e-04 | 1395.5ms/step | 14.3 samples/s +Step 16850/44000 | Loss: 3.0045 | LR: 3.48e-04 | 1395.0ms/step | 14.3 samples/s +Step 16860/44000 | Loss: 3.0111 | LR: 3.47e-04 | 1399.9ms/step | 14.3 samples/s +Step 16870/44000 | Loss: 2.9940 | LR: 3.47e-04 | 1396.0ms/step | 14.3 samples/s +Step 16880/44000 | Loss: 2.9770 | LR: 3.47e-04 | 1399.6ms/step | 14.3 samples/s +Step 16890/44000 | Loss: 2.9884 | LR: 3.47e-04 | 1396.5ms/step | 14.3 samples/s +Step 16900/44000 | Loss: 2.9774 | LR: 3.47e-04 | 1397.8ms/step | 14.3 samples/s +Step 16910/44000 | Loss: 3.0140 | LR: 3.47e-04 | 1398.3ms/step | 14.3 samples/s +Step 16920/44000 | Loss: 3.0135 | LR: 3.46e-04 | 1397.7ms/step | 14.3 samples/s +Step 16930/44000 | Loss: 3.0102 | LR: 3.46e-04 | 1400.4ms/step | 14.3 samples/s +Step 16940/44000 | Loss: 3.0144 | LR: 3.46e-04 | 1399.9ms/step | 14.3 samples/s +Step 16950/44000 | Loss: 2.9817 | LR: 3.46e-04 | 1397.9ms/step | 14.3 samples/s +Step 16960/44000 | Loss: 3.0258 | LR: 3.46e-04 | 1395.6ms/step | 14.3 samples/s +Step 16970/44000 | Loss: 2.9987 | LR: 3.46e-04 | 1398.4ms/step | 14.3 samples/s +Step 16980/44000 | Loss: 2.9798 | LR: 3.45e-04 | 1399.7ms/step | 14.3 samples/s +Step 16990/44000 | Loss: 3.0147 | LR: 3.45e-04 | 1398.4ms/step | 14.3 samples/s +Step 17000/44000 | Loss: 3.0201 | LR: 3.45e-04 | 1391.1ms/step | 14.4 samples/s +Validation at step 17000: Loss = 3.0121, Perplexity = 20.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-17000 +Step 17010/44000 | Loss: 2.9965 | LR: 3.45e-04 | 3968.6ms/step | 5.0 samples/s +Step 17020/44000 | Loss: 2.9838 | LR: 3.45e-04 | 1384.7ms/step | 14.4 samples/s +Step 17030/44000 | Loss: 2.9773 | LR: 3.45e-04 | 1383.2ms/step | 14.5 samples/s +Step 17040/44000 | Loss: 2.9716 | LR: 3.44e-04 | 1386.5ms/step | 14.4 samples/s +Step 17050/44000 | Loss: 3.0080 | LR: 3.44e-04 | 1384.9ms/step | 14.4 samples/s +Step 17060/44000 | Loss: 3.0121 | LR: 3.44e-04 | 1387.2ms/step | 14.4 samples/s +Step 17070/44000 | Loss: 2.9939 | LR: 3.44e-04 | 1389.7ms/step | 14.4 samples/s +Step 17080/44000 | Loss: 3.0381 | LR: 3.44e-04 | 1391.7ms/step | 14.4 samples/s +Step 17090/44000 | Loss: 2.9735 | LR: 3.44e-04 | 1390.1ms/step | 14.4 samples/s +Step 17100/44000 | Loss: 2.9871 | LR: 3.43e-04 | 1390.1ms/step | 14.4 samples/s +Step 17110/44000 | Loss: 3.0153 | LR: 3.43e-04 | 1393.7ms/step | 14.4 samples/s +Step 17120/44000 | Loss: 3.0041 | LR: 3.43e-04 | 1389.3ms/step | 14.4 samples/s +Step 17130/44000 | Loss: 2.9881 | LR: 3.43e-04 | 1391.1ms/step | 14.4 samples/s +Step 17140/44000 | Loss: 3.0179 | LR: 3.43e-04 | 1389.9ms/step | 14.4 samples/s +Step 17150/44000 | Loss: 2.9896 | LR: 3.43e-04 | 1390.6ms/step | 14.4 samples/s +Step 17160/44000 | Loss: 3.0055 | LR: 3.42e-04 | 1391.1ms/step | 14.4 samples/s +Step 17170/44000 | Loss: 3.0043 | LR: 3.42e-04 | 1389.0ms/step | 14.4 samples/s +Step 17180/44000 | Loss: 3.0208 | LR: 3.42e-04 | 1388.5ms/step | 14.4 samples/s +Step 17190/44000 | Loss: 2.9791 | LR: 3.42e-04 | 1390.5ms/step | 14.4 samples/s +Step 17200/44000 | Loss: 2.9915 | LR: 3.42e-04 | 1391.8ms/step | 14.4 samples/s +Step 17210/44000 | Loss: 2.9879 | LR: 3.42e-04 | 1389.7ms/step | 14.4 samples/s +Step 17220/44000 | Loss: 2.9841 | LR: 3.41e-04 | 1389.6ms/step | 14.4 samples/s +Step 17230/44000 | Loss: 2.9995 | LR: 3.41e-04 | 1391.6ms/step | 14.4 samples/s +Step 17240/44000 | Loss: 2.9744 | LR: 3.41e-04 | 1393.3ms/step | 14.4 samples/s +Step 17250/44000 | Loss: 2.9778 | LR: 3.41e-04 | 1392.0ms/step | 14.4 samples/s +Step 17260/44000 | Loss: 2.9987 | LR: 3.41e-04 | 1392.1ms/step | 14.4 samples/s +Step 17270/44000 | Loss: 2.9892 | LR: 3.41e-04 | 1392.3ms/step | 14.4 samples/s +Step 17280/44000 | Loss: 3.0014 | LR: 3.40e-04 | 1395.5ms/step | 14.3 samples/s +Step 17290/44000 | Loss: 3.0176 | LR: 3.40e-04 | 1393.5ms/step | 14.4 samples/s +Step 17300/44000 | Loss: 2.9963 | LR: 3.40e-04 | 1390.0ms/step | 14.4 samples/s +Step 17310/44000 | Loss: 3.0070 | LR: 3.40e-04 | 1391.2ms/step | 14.4 samples/s +Step 17320/44000 | Loss: 3.0012 | LR: 3.40e-04 | 1389.7ms/step | 14.4 samples/s +Step 17330/44000 | Loss: 3.0012 | LR: 3.40e-04 | 1395.8ms/step | 14.3 samples/s +Step 17340/44000 | Loss: 2.9895 | LR: 3.39e-04 | 1392.9ms/step | 14.4 samples/s +Step 17350/44000 | Loss: 2.9804 | LR: 3.39e-04 | 1391.0ms/step | 14.4 samples/s +Step 17360/44000 | Loss: 2.9887 | LR: 3.39e-04 | 1392.4ms/step | 14.4 samples/s +Step 17370/44000 | Loss: 2.9760 | LR: 3.39e-04 | 1393.4ms/step | 14.4 samples/s +Step 17380/44000 | Loss: 2.9965 | LR: 3.39e-04 | 1396.1ms/step | 14.3 samples/s +Step 17390/44000 | Loss: 2.9764 | LR: 3.39e-04 | 1392.5ms/step | 14.4 samples/s +Step 17400/44000 | Loss: 2.9813 | LR: 3.38e-04 | 1392.2ms/step | 14.4 samples/s +Step 17410/44000 | Loss: 2.9788 | LR: 3.38e-04 | 1391.5ms/step | 14.4 samples/s +Step 17420/44000 | Loss: 2.9823 | LR: 3.38e-04 | 1392.0ms/step | 14.4 samples/s +Step 17430/44000 | Loss: 3.0083 | LR: 3.38e-04 | 1393.0ms/step | 14.4 samples/s +Step 17440/44000 | Loss: 2.9813 | LR: 3.38e-04 | 1393.6ms/step | 14.4 samples/s +Step 17450/44000 | Loss: 3.0100 | LR: 3.38e-04 | 1394.9ms/step | 14.3 samples/s +Step 17460/44000 | Loss: 2.9793 | LR: 3.37e-04 | 1392.2ms/step | 14.4 samples/s +Step 17470/44000 | Loss: 2.9795 | LR: 3.37e-04 | 1388.9ms/step | 14.4 samples/s +Step 17480/44000 | Loss: 2.9965 | LR: 3.37e-04 | 1391.6ms/step | 14.4 samples/s +Step 17490/44000 | Loss: 2.9569 | LR: 3.37e-04 | 1391.1ms/step | 14.4 samples/s diff --git a/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed43.20260820T041452Z.log b/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed43.20260820T041452Z.log new file mode 100644 index 0000000000000000000000000000000000000000..3b26852f516fc853bff63a166c55092b62fba949 --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed43.20260820T041452Z.log @@ -0,0 +1,640 @@ +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-14000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-14000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 43, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed43", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-14000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-14000... +Preserving checkpoint LR state for exact resume: current=[0.00039287714704080913], schedule_base=[0.0005] +Resumed from checkpoint at step 14000 (epoch 0, micro_step 140000) +Resuming training from step 14000 +Resuming wandb run ysa9iyov +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run ysa9iyov +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-ysa9iyov +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed43 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ysa9iyov +Starting training... +Validation at step 14000: Loss = 3.0315, Perplexity = 20.73 +Switching attention mode for step 14001: dense (dense step interval) +Step 14010/44000 | Loss: 3.0521 | LR: 3.93e-04 | 30764.5ms/step | 0.7 samples/s +Step 14020/44000 | Loss: 3.0747 | LR: 3.93e-04 | 1400.4ms/step | 14.3 samples/s +Step 14030/44000 | Loss: 3.0141 | LR: 3.92e-04 | 1401.7ms/step | 14.3 samples/s +Step 14040/44000 | Loss: 3.0482 | LR: 3.92e-04 | 1404.4ms/step | 14.2 samples/s +Step 14050/44000 | Loss: 3.0244 | LR: 3.92e-04 | 1402.5ms/step | 14.3 samples/s +Step 14060/44000 | Loss: 2.9957 | LR: 3.92e-04 | 1405.5ms/step | 14.2 samples/s +Step 14070/44000 | Loss: 3.0557 | LR: 3.92e-04 | 1404.4ms/step | 14.2 samples/s +Step 14080/44000 | Loss: 3.0433 | LR: 3.92e-04 | 1407.5ms/step | 14.2 samples/s +Step 14090/44000 | Loss: 3.0255 | LR: 3.92e-04 | 1406.2ms/step | 14.2 samples/s +Step 14100/44000 | Loss: 3.0300 | LR: 3.91e-04 | 1401.6ms/step | 14.3 samples/s +Step 14110/44000 | Loss: 3.0152 | LR: 3.91e-04 | 1400.1ms/step | 14.3 samples/s +Step 14120/44000 | Loss: 3.0562 | LR: 3.91e-04 | 1400.1ms/step | 14.3 samples/s +Step 14130/44000 | Loss: 3.0295 | LR: 3.91e-04 | 1399.9ms/step | 14.3 samples/s +Step 14140/44000 | Loss: 3.0463 | LR: 3.91e-04 | 1400.0ms/step | 14.3 samples/s +Step 14150/44000 | Loss: 3.0376 | LR: 3.91e-04 | 1400.0ms/step | 14.3 samples/s +Step 14160/44000 | Loss: 3.0425 | LR: 3.90e-04 | 1399.9ms/step | 14.3 samples/s +Step 14170/44000 | Loss: 3.0176 | LR: 3.90e-04 | 1400.3ms/step | 14.3 samples/s +Step 14180/44000 | Loss: 3.0226 | LR: 3.90e-04 | 1399.5ms/step | 14.3 samples/s +Step 14190/44000 | Loss: 3.0413 | LR: 3.90e-04 | 1400.6ms/step | 14.3 samples/s +Step 14200/44000 | Loss: 3.0287 | LR: 3.90e-04 | 1399.7ms/step | 14.3 samples/s +Step 14210/44000 | Loss: 3.0243 | LR: 3.90e-04 | 1399.9ms/step | 14.3 samples/s +Step 14220/44000 | Loss: 3.0587 | LR: 3.90e-04 | 1400.2ms/step | 14.3 samples/s +Step 14230/44000 | Loss: 3.0377 | LR: 3.89e-04 | 1399.9ms/step | 14.3 samples/s +Step 14240/44000 | Loss: 3.0638 | LR: 3.89e-04 | 1400.3ms/step | 14.3 samples/s +Step 14250/44000 | Loss: 3.0836 | LR: 3.89e-04 | 1401.7ms/step | 14.3 samples/s +Step 14260/44000 | Loss: 3.0346 | LR: 3.89e-04 | 1400.8ms/step | 14.3 samples/s +Step 14270/44000 | Loss: 3.0117 | LR: 3.89e-04 | 1401.1ms/step | 14.3 samples/s +Step 14280/44000 | Loss: 3.0250 | LR: 3.89e-04 | 1400.0ms/step | 14.3 samples/s +Step 14290/44000 | Loss: 3.0364 | LR: 3.89e-04 | 1399.8ms/step | 14.3 samples/s +Step 14300/44000 | Loss: 3.0342 | LR: 3.88e-04 | 1400.3ms/step | 14.3 samples/s +Step 14310/44000 | Loss: 3.0343 | LR: 3.88e-04 | 1399.9ms/step | 14.3 samples/s +Step 14320/44000 | Loss: 3.0531 | LR: 3.88e-04 | 1399.9ms/step | 14.3 samples/s +Step 14330/44000 | Loss: 3.0523 | LR: 3.88e-04 | 1400.1ms/step | 14.3 samples/s +Step 14340/44000 | Loss: 3.0429 | LR: 3.88e-04 | 1399.9ms/step | 14.3 samples/s +Step 14350/44000 | Loss: 3.0559 | LR: 3.88e-04 | 1400.1ms/step | 14.3 samples/s +Step 14360/44000 | Loss: 3.0261 | LR: 3.87e-04 | 1399.9ms/step | 14.3 samples/s +Step 14370/44000 | Loss: 3.0138 | LR: 3.87e-04 | 1400.5ms/step | 14.3 samples/s +Step 14380/44000 | Loss: 3.0530 | LR: 3.87e-04 | 1399.5ms/step | 14.3 samples/s +Step 14390/44000 | Loss: 3.0273 | LR: 3.87e-04 | 1400.0ms/step | 14.3 samples/s +Step 14400/44000 | Loss: 3.0345 | LR: 3.87e-04 | 1399.9ms/step | 14.3 samples/s +Step 14410/44000 | Loss: 3.0357 | LR: 3.87e-04 | 1400.2ms/step | 14.3 samples/s +Step 14420/44000 | Loss: 3.0597 | LR: 3.87e-04 | 1399.8ms/step | 14.3 samples/s +Step 14430/44000 | Loss: 3.0228 | LR: 3.86e-04 | 1400.1ms/step | 14.3 samples/s +Step 14440/44000 | Loss: 3.0280 | LR: 3.86e-04 | 1399.9ms/step | 14.3 samples/s +Step 14450/44000 | Loss: 3.0255 | LR: 3.86e-04 | 1400.3ms/step | 14.3 samples/s +Step 14460/44000 | Loss: 3.0530 | LR: 3.86e-04 | 1399.8ms/step | 14.3 samples/s +Step 14470/44000 | Loss: 3.0447 | LR: 3.86e-04 | 1399.9ms/step | 14.3 samples/s +Step 14480/44000 | Loss: 3.0540 | LR: 3.86e-04 | 1400.0ms/step | 14.3 samples/s +Step 14490/44000 | Loss: 3.0544 | LR: 3.85e-04 | 1400.3ms/step | 14.3 samples/s +Step 14500/44000 | Loss: 2.9941 | LR: 3.85e-04 | 1400.0ms/step | 14.3 samples/s +Step 14510/44000 | Loss: 3.0325 | LR: 3.85e-04 | 1399.8ms/step | 14.3 samples/s +Step 14520/44000 | Loss: 3.0357 | LR: 3.85e-04 | 1399.9ms/step | 14.3 samples/s +Step 14530/44000 | Loss: 3.0294 | LR: 3.85e-04 | 1400.3ms/step | 14.3 samples/s +Step 14540/44000 | Loss: 3.0313 | LR: 3.85e-04 | 1399.9ms/step | 14.3 samples/s +Step 14550/44000 | Loss: 3.0350 | LR: 3.85e-04 | 1399.7ms/step | 14.3 samples/s +Step 14560/44000 | Loss: 3.0122 | LR: 3.84e-04 | 1400.1ms/step | 14.3 samples/s +Step 14570/44000 | Loss: 3.0366 | LR: 3.84e-04 | 1400.1ms/step | 14.3 samples/s +Step 14580/44000 | Loss: 3.0213 | LR: 3.84e-04 | 1400.4ms/step | 14.3 samples/s +Step 14590/44000 | Loss: 3.0545 | LR: 3.84e-04 | 1399.4ms/step | 14.3 samples/s +Step 14600/44000 | Loss: 3.0290 | LR: 3.84e-04 | 1400.4ms/step | 14.3 samples/s +Step 14610/44000 | Loss: 3.0532 | LR: 3.84e-04 | 1400.1ms/step | 14.3 samples/s +Step 14620/44000 | Loss: 3.0001 | LR: 3.83e-04 | 1399.6ms/step | 14.3 samples/s +Step 14630/44000 | Loss: 3.0206 | LR: 3.83e-04 | 1399.9ms/step | 14.3 samples/s +Step 14640/44000 | Loss: 3.0220 | LR: 3.83e-04 | 1400.1ms/step | 14.3 samples/s +Step 14650/44000 | Loss: 3.0496 | LR: 3.83e-04 | 1400.2ms/step | 14.3 samples/s +Step 14660/44000 | Loss: 3.0342 | LR: 3.83e-04 | 1399.8ms/step | 14.3 samples/s +Step 14670/44000 | Loss: 3.0729 | LR: 3.83e-04 | 1399.9ms/step | 14.3 samples/s +Step 14680/44000 | Loss: 3.0262 | LR: 3.83e-04 | 1400.2ms/step | 14.3 samples/s +Step 14690/44000 | Loss: 3.0151 | LR: 3.82e-04 | 1400.0ms/step | 14.3 samples/s +Step 14700/44000 | Loss: 3.0294 | LR: 3.82e-04 | 1399.7ms/step | 14.3 samples/s +Step 14710/44000 | Loss: 3.0133 | LR: 3.82e-04 | 1400.0ms/step | 14.3 samples/s +Step 14720/44000 | Loss: 3.0168 | LR: 3.82e-04 | 1400.4ms/step | 14.3 samples/s +Step 14730/44000 | Loss: 3.0168 | LR: 3.82e-04 | 1400.1ms/step | 14.3 samples/s +Step 14740/44000 | Loss: 3.0555 | LR: 3.82e-04 | 1400.0ms/step | 14.3 samples/s +Step 14750/44000 | Loss: 3.0453 | LR: 3.81e-04 | 1399.5ms/step | 14.3 samples/s +Step 14760/44000 | Loss: 3.0187 | LR: 3.81e-04 | 1400.1ms/step | 14.3 samples/s +Step 14770/44000 | Loss: 3.0488 | LR: 3.81e-04 | 1400.2ms/step | 14.3 samples/s +Step 14780/44000 | Loss: 3.0276 | LR: 3.81e-04 | 1400.0ms/step | 14.3 samples/s +Step 14790/44000 | Loss: 3.0210 | LR: 3.81e-04 | 1399.7ms/step | 14.3 samples/s +Step 14800/44000 | Loss: 3.0516 | LR: 3.81e-04 | 1400.1ms/step | 14.3 samples/s +Step 14810/44000 | Loss: 3.0319 | LR: 3.81e-04 | 1399.9ms/step | 14.3 samples/s +Step 14820/44000 | Loss: 3.0442 | LR: 3.80e-04 | 1400.0ms/step | 14.3 samples/s +Step 14830/44000 | Loss: 3.0371 | LR: 3.80e-04 | 1400.1ms/step | 14.3 samples/s +Step 14840/44000 | Loss: 3.0045 | LR: 3.80e-04 | 1399.8ms/step | 14.3 samples/s +Step 14850/44000 | Loss: 2.9777 | LR: 3.80e-04 | 1400.3ms/step | 14.3 samples/s +Step 14860/44000 | Loss: 3.0043 | LR: 3.80e-04 | 1399.7ms/step | 14.3 samples/s +Step 14870/44000 | Loss: 3.0675 | LR: 3.80e-04 | 1400.5ms/step | 14.3 samples/s +Step 14880/44000 | Loss: 3.0157 | LR: 3.79e-04 | 1399.5ms/step | 14.3 samples/s +Step 14890/44000 | Loss: 3.0207 | LR: 3.79e-04 | 1400.1ms/step | 14.3 samples/s +Step 14900/44000 | Loss: 3.0087 | LR: 3.79e-04 | 1400.0ms/step | 14.3 samples/s +Step 14910/44000 | Loss: 3.0392 | LR: 3.79e-04 | 1399.9ms/step | 14.3 samples/s +Step 14920/44000 | Loss: 3.0563 | LR: 3.79e-04 | 1400.3ms/step | 14.3 samples/s +Step 14930/44000 | Loss: 3.0244 | LR: 3.79e-04 | 1400.0ms/step | 14.3 samples/s +Step 14940/44000 | Loss: 3.0206 | LR: 3.79e-04 | 1402.0ms/step | 14.3 samples/s +Step 14950/44000 | Loss: 3.0400 | LR: 3.78e-04 | 1399.8ms/step | 14.3 samples/s +Step 14960/44000 | Loss: 3.0309 | LR: 3.78e-04 | 1400.3ms/step | 14.3 samples/s +Step 14970/44000 | Loss: 3.0373 | LR: 3.78e-04 | 1401.5ms/step | 14.3 samples/s +Step 14980/44000 | Loss: 3.0326 | LR: 3.78e-04 | 1400.2ms/step | 14.3 samples/s +Step 14990/44000 | Loss: 3.0105 | LR: 3.78e-04 | 1400.1ms/step | 14.3 samples/s +Step 15000/44000 | Loss: 3.0185 | LR: 3.78e-04 | 1400.4ms/step | 14.3 samples/s +Validation at step 15000: Loss = 3.0168, Perplexity = 20.43 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-15000 +Step 15010/44000 | Loss: 3.0053 | LR: 3.77e-04 | 3941.5ms/step | 5.1 samples/s +Step 15020/44000 | Loss: 3.0368 | LR: 3.77e-04 | 1402.4ms/step | 14.3 samples/s +Step 15030/44000 | Loss: 3.0373 | LR: 3.77e-04 | 1409.6ms/step | 14.2 samples/s +Step 15040/44000 | Loss: 3.0137 | LR: 3.77e-04 | 1410.2ms/step | 14.2 samples/s +Step 15050/44000 | Loss: 3.0237 | LR: 3.77e-04 | 1411.9ms/step | 14.2 samples/s +Step 15060/44000 | Loss: 3.0238 | LR: 3.77e-04 | 1409.0ms/step | 14.2 samples/s +Step 15070/44000 | Loss: 3.0276 | LR: 3.77e-04 | 1412.4ms/step | 14.2 samples/s +Step 15080/44000 | Loss: 3.0443 | LR: 3.76e-04 | 1411.0ms/step | 14.2 samples/s +Step 15090/44000 | Loss: 3.0242 | LR: 3.76e-04 | 1410.4ms/step | 14.2 samples/s +Step 15100/44000 | Loss: 3.0022 | LR: 3.76e-04 | 1411.3ms/step | 14.2 samples/s +Step 15110/44000 | Loss: 3.0292 | LR: 3.76e-04 | 1412.4ms/step | 14.2 samples/s +Step 15120/44000 | Loss: 3.0013 | LR: 3.76e-04 | 1414.7ms/step | 14.1 samples/s +Step 15130/44000 | Loss: 3.0519 | LR: 3.76e-04 | 1409.6ms/step | 14.2 samples/s +Step 15140/44000 | Loss: 3.0067 | LR: 3.75e-04 | 1411.2ms/step | 14.2 samples/s +Step 15150/44000 | Loss: 3.0282 | LR: 3.75e-04 | 1412.2ms/step | 14.2 samples/s +Step 15160/44000 | Loss: 3.0164 | LR: 3.75e-04 | 1411.7ms/step | 14.2 samples/s +Step 15170/44000 | Loss: 3.0404 | LR: 3.75e-04 | 1410.0ms/step | 14.2 samples/s +Step 15180/44000 | Loss: 2.9985 | LR: 3.75e-04 | 1408.0ms/step | 14.2 samples/s +Step 15190/44000 | Loss: 3.0056 | LR: 3.75e-04 | 1412.1ms/step | 14.2 samples/s +Step 15200/44000 | Loss: 3.0177 | LR: 3.74e-04 | 1413.8ms/step | 14.1 samples/s +Step 15210/44000 | Loss: 3.0339 | LR: 3.74e-04 | 1409.1ms/step | 14.2 samples/s +Step 15220/44000 | Loss: 3.0289 | LR: 3.74e-04 | 1409.4ms/step | 14.2 samples/s +Step 15230/44000 | Loss: 3.0372 | LR: 3.74e-04 | 1409.8ms/step | 14.2 samples/s +Step 15240/44000 | Loss: 3.0108 | LR: 3.74e-04 | 1412.0ms/step | 14.2 samples/s +Step 15250/44000 | Loss: 3.0425 | LR: 3.74e-04 | 1408.8ms/step | 14.2 samples/s +Step 15260/44000 | Loss: 3.0209 | LR: 3.74e-04 | 1408.6ms/step | 14.2 samples/s +Step 15270/44000 | Loss: 3.0247 | LR: 3.73e-04 | 1412.5ms/step | 14.2 samples/s +Step 15280/44000 | Loss: 3.0363 | LR: 3.73e-04 | 1412.3ms/step | 14.2 samples/s +Step 15290/44000 | Loss: 3.0178 | LR: 3.73e-04 | 1412.4ms/step | 14.2 samples/s +Step 15300/44000 | Loss: 3.0352 | LR: 3.73e-04 | 1410.6ms/step | 14.2 samples/s +Step 15310/44000 | Loss: 3.0114 | LR: 3.73e-04 | 1413.1ms/step | 14.2 samples/s +Step 15320/44000 | Loss: 3.0030 | LR: 3.73e-04 | 1411.7ms/step | 14.2 samples/s +Step 15330/44000 | Loss: 3.0128 | LR: 3.72e-04 | 1411.8ms/step | 14.2 samples/s +Step 15340/44000 | Loss: 3.0430 | LR: 3.72e-04 | 1410.2ms/step | 14.2 samples/s +Step 15350/44000 | Loss: 3.0171 | LR: 3.72e-04 | 1410.0ms/step | 14.2 samples/s +Step 15360/44000 | Loss: 3.0435 | LR: 3.72e-04 | 1407.8ms/step | 14.2 samples/s +Step 15370/44000 | Loss: 3.0452 | LR: 3.72e-04 | 1409.5ms/step | 14.2 samples/s +Step 15380/44000 | Loss: 3.0110 | LR: 3.72e-04 | 1408.6ms/step | 14.2 samples/s +Step 15390/44000 | Loss: 3.0187 | LR: 3.71e-04 | 1410.5ms/step | 14.2 samples/s +Step 15400/44000 | Loss: 3.0359 | LR: 3.71e-04 | 1410.7ms/step | 14.2 samples/s +Step 15410/44000 | Loss: 3.0216 | LR: 3.71e-04 | 1411.2ms/step | 14.2 samples/s +Step 15420/44000 | Loss: 3.0334 | LR: 3.71e-04 | 1412.5ms/step | 14.2 samples/s +Step 15430/44000 | Loss: 3.0338 | LR: 3.71e-04 | 1411.6ms/step | 14.2 samples/s +Step 15440/44000 | Loss: 3.0213 | LR: 3.71e-04 | 1411.4ms/step | 14.2 samples/s +Step 15450/44000 | Loss: 3.0234 | LR: 3.71e-04 | 1410.6ms/step | 14.2 samples/s +Step 15460/44000 | Loss: 3.0322 | LR: 3.70e-04 | 1411.4ms/step | 14.2 samples/s +Step 15470/44000 | Loss: 3.0304 | LR: 3.70e-04 | 1412.0ms/step | 14.2 samples/s +Step 15480/44000 | Loss: 3.0119 | LR: 3.70e-04 | 1412.8ms/step | 14.2 samples/s +Step 15490/44000 | Loss: 3.0070 | LR: 3.70e-04 | 1409.2ms/step | 14.2 samples/s +Step 15500/44000 | Loss: 3.0243 | LR: 3.70e-04 | 1410.9ms/step | 14.2 samples/s +Step 15510/44000 | Loss: 3.0337 | LR: 3.70e-04 | 1410.5ms/step | 14.2 samples/s +Step 15520/44000 | Loss: 3.0178 | LR: 3.69e-04 | 1408.4ms/step | 14.2 samples/s +Step 15530/44000 | Loss: 2.9951 | LR: 3.69e-04 | 1409.1ms/step | 14.2 samples/s +Step 15540/44000 | Loss: 3.0524 | LR: 3.69e-04 | 1411.5ms/step | 14.2 samples/s +Step 15550/44000 | Loss: 3.0217 | LR: 3.69e-04 | 1408.2ms/step | 14.2 samples/s +Step 15560/44000 | Loss: 3.0148 | LR: 3.69e-04 | 1409.3ms/step | 14.2 samples/s +Step 15570/44000 | Loss: 2.9971 | LR: 3.69e-04 | 1410.2ms/step | 14.2 samples/s +Step 15580/44000 | Loss: 3.0258 | LR: 3.68e-04 | 1410.1ms/step | 14.2 samples/s +Step 15590/44000 | Loss: 3.0436 | LR: 3.68e-04 | 1412.1ms/step | 14.2 samples/s +Step 15600/44000 | Loss: 3.0186 | LR: 3.68e-04 | 1407.6ms/step | 14.2 samples/s +Step 15610/44000 | Loss: 2.9948 | LR: 3.68e-04 | 1410.4ms/step | 14.2 samples/s +Step 15620/44000 | Loss: 3.0137 | LR: 3.68e-04 | 1412.1ms/step | 14.2 samples/s +Step 15630/44000 | Loss: 3.0320 | LR: 3.68e-04 | 1412.2ms/step | 14.2 samples/s +Step 15640/44000 | Loss: 3.0359 | LR: 3.67e-04 | 1411.8ms/step | 14.2 samples/s +Step 15650/44000 | Loss: 3.0122 | LR: 3.67e-04 | 1410.7ms/step | 14.2 samples/s +Step 15660/44000 | Loss: 3.0295 | LR: 3.67e-04 | 1411.1ms/step | 14.2 samples/s +Step 15670/44000 | Loss: 3.0124 | LR: 3.67e-04 | 1411.2ms/step | 14.2 samples/s +Step 15680/44000 | Loss: 3.0503 | LR: 3.67e-04 | 1411.2ms/step | 14.2 samples/s +Step 15690/44000 | Loss: 3.0024 | LR: 3.67e-04 | 1409.5ms/step | 14.2 samples/s +Step 15700/44000 | Loss: 3.0032 | LR: 3.67e-04 | 1408.5ms/step | 14.2 samples/s +Step 15710/44000 | Loss: 3.0169 | LR: 3.66e-04 | 1412.2ms/step | 14.2 samples/s +Step 15720/44000 | Loss: 2.9977 | LR: 3.66e-04 | 1411.1ms/step | 14.2 samples/s +Step 15730/44000 | Loss: 3.0203 | LR: 3.66e-04 | 1408.1ms/step | 14.2 samples/s +Step 15740/44000 | Loss: 3.0099 | LR: 3.66e-04 | 1412.6ms/step | 14.2 samples/s +Step 15750/44000 | Loss: 3.0203 | LR: 3.66e-04 | 1409.6ms/step | 14.2 samples/s +Step 15760/44000 | Loss: 3.0059 | LR: 3.66e-04 | 1411.6ms/step | 14.2 samples/s +Step 15770/44000 | Loss: 3.0506 | LR: 3.65e-04 | 1412.3ms/step | 14.2 samples/s +Step 15780/44000 | Loss: 2.9989 | LR: 3.65e-04 | 1412.1ms/step | 14.2 samples/s +Step 15790/44000 | Loss: 3.0394 | LR: 3.65e-04 | 1412.1ms/step | 14.2 samples/s +Step 15800/44000 | Loss: 3.0058 | LR: 3.65e-04 | 1414.2ms/step | 14.1 samples/s +Step 15810/44000 | Loss: 3.0214 | LR: 3.65e-04 | 1411.8ms/step | 14.2 samples/s +Step 15820/44000 | Loss: 2.9922 | LR: 3.65e-04 | 1408.5ms/step | 14.2 samples/s +Step 15830/44000 | Loss: 2.9982 | LR: 3.64e-04 | 1407.2ms/step | 14.2 samples/s +Step 15840/44000 | Loss: 3.0272 | LR: 3.64e-04 | 1411.9ms/step | 14.2 samples/s +Step 15850/44000 | Loss: 3.0119 | LR: 3.64e-04 | 1411.0ms/step | 14.2 samples/s +Step 15860/44000 | Loss: 3.0130 | LR: 3.64e-04 | 1411.3ms/step | 14.2 samples/s +Step 15870/44000 | Loss: 3.0085 | LR: 3.64e-04 | 1409.9ms/step | 14.2 samples/s +Step 15880/44000 | Loss: 3.0284 | LR: 3.64e-04 | 1412.5ms/step | 14.2 samples/s +Step 15890/44000 | Loss: 2.9768 | LR: 3.63e-04 | 1411.4ms/step | 14.2 samples/s +Step 15900/44000 | Loss: 3.0116 | LR: 3.63e-04 | 1410.3ms/step | 14.2 samples/s +Step 15910/44000 | Loss: 3.0028 | LR: 3.63e-04 | 1408.9ms/step | 14.2 samples/s +Step 15920/44000 | Loss: 3.0251 | LR: 3.63e-04 | 1407.7ms/step | 14.2 samples/s +Step 15930/44000 | Loss: 3.0078 | LR: 3.63e-04 | 1409.8ms/step | 14.2 samples/s +Step 15940/44000 | Loss: 3.0058 | LR: 3.63e-04 | 1410.9ms/step | 14.2 samples/s +Step 15950/44000 | Loss: 3.0315 | LR: 3.62e-04 | 1410.2ms/step | 14.2 samples/s +Step 15960/44000 | Loss: 2.9918 | LR: 3.62e-04 | 1410.2ms/step | 14.2 samples/s +Step 15970/44000 | Loss: 2.9993 | LR: 3.62e-04 | 1410.8ms/step | 14.2 samples/s +Step 15980/44000 | Loss: 3.0220 | LR: 3.62e-04 | 1409.8ms/step | 14.2 samples/s +Step 15990/44000 | Loss: 3.0168 | LR: 3.62e-04 | 1408.6ms/step | 14.2 samples/s +Step 16000/44000 | Loss: 3.0033 | LR: 3.62e-04 | 1407.0ms/step | 14.2 samples/s +Validation at step 16000: Loss = 3.0032, Perplexity = 20.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-16000 +Step 16010/44000 | Loss: 3.0004 | LR: 3.62e-04 | 3962.0ms/step | 5.0 samples/s +Step 16020/44000 | Loss: 3.0341 | LR: 3.61e-04 | 1403.8ms/step | 14.2 samples/s +Step 16030/44000 | Loss: 2.9863 | LR: 3.61e-04 | 1406.9ms/step | 14.2 samples/s +Step 16040/44000 | Loss: 3.0186 | LR: 3.61e-04 | 1400.9ms/step | 14.3 samples/s +Step 16050/44000 | Loss: 3.0309 | LR: 3.61e-04 | 1402.2ms/step | 14.3 samples/s +Step 16060/44000 | Loss: 3.0071 | LR: 3.61e-04 | 1404.0ms/step | 14.2 samples/s +Step 16070/44000 | Loss: 2.9967 | LR: 3.61e-04 | 1401.9ms/step | 14.3 samples/s +Step 16080/44000 | Loss: 3.0232 | LR: 3.60e-04 | 1400.0ms/step | 14.3 samples/s +Step 16090/44000 | Loss: 3.0047 | LR: 3.60e-04 | 1400.3ms/step | 14.3 samples/s +Step 16100/44000 | Loss: 3.0289 | LR: 3.60e-04 | 1401.8ms/step | 14.3 samples/s +Step 16110/44000 | Loss: 2.9921 | LR: 3.60e-04 | 1399.6ms/step | 14.3 samples/s +Step 16120/44000 | Loss: 3.0093 | LR: 3.60e-04 | 1400.4ms/step | 14.3 samples/s +Step 16130/44000 | Loss: 2.9896 | LR: 3.60e-04 | 1400.0ms/step | 14.3 samples/s +Step 16140/44000 | Loss: 2.9900 | LR: 3.59e-04 | 1402.5ms/step | 14.3 samples/s +Step 16150/44000 | Loss: 3.0304 | LR: 3.59e-04 | 1404.2ms/step | 14.2 samples/s +Step 16160/44000 | Loss: 3.0094 | LR: 3.59e-04 | 1403.2ms/step | 14.3 samples/s +Step 16170/44000 | Loss: 2.9771 | LR: 3.59e-04 | 1402.0ms/step | 14.3 samples/s +Step 16180/44000 | Loss: 2.9958 | LR: 3.59e-04 | 1404.3ms/step | 14.2 samples/s +Step 16190/44000 | Loss: 2.9992 | LR: 3.59e-04 | 1401.7ms/step | 14.3 samples/s +Step 16200/44000 | Loss: 3.0221 | LR: 3.58e-04 | 1403.7ms/step | 14.2 samples/s +Step 16210/44000 | Loss: 3.0118 | LR: 3.58e-04 | 1400.2ms/step | 14.3 samples/s +Step 16220/44000 | Loss: 3.0274 | LR: 3.58e-04 | 1402.1ms/step | 14.3 samples/s +Step 16230/44000 | Loss: 3.0129 | LR: 3.58e-04 | 1402.1ms/step | 14.3 samples/s +Step 16240/44000 | Loss: 2.9960 | LR: 3.58e-04 | 1404.5ms/step | 14.2 samples/s +Step 16250/44000 | Loss: 3.0097 | LR: 3.58e-04 | 1400.9ms/step | 14.3 samples/s +Step 16260/44000 | Loss: 3.0072 | LR: 3.57e-04 | 1404.0ms/step | 14.2 samples/s +Step 16270/44000 | Loss: 2.9829 | LR: 3.57e-04 | 1402.4ms/step | 14.3 samples/s +Step 16280/44000 | Loss: 2.9928 | LR: 3.57e-04 | 1401.7ms/step | 14.3 samples/s +Step 16290/44000 | Loss: 3.0136 | LR: 3.57e-04 | 1400.4ms/step | 14.3 samples/s +Step 16300/44000 | Loss: 3.0019 | LR: 3.57e-04 | 1401.5ms/step | 14.3 samples/s +Step 16310/44000 | Loss: 3.0136 | LR: 3.57e-04 | 1402.7ms/step | 14.3 samples/s +Step 16320/44000 | Loss: 3.0140 | LR: 3.56e-04 | 1401.6ms/step | 14.3 samples/s +Step 16330/44000 | Loss: 3.0250 | LR: 3.56e-04 | 1401.0ms/step | 14.3 samples/s +Step 16340/44000 | Loss: 2.9840 | LR: 3.56e-04 | 1401.0ms/step | 14.3 samples/s +Step 16350/44000 | Loss: 3.0017 | LR: 3.56e-04 | 1399.6ms/step | 14.3 samples/s +Step 16360/44000 | Loss: 3.0169 | LR: 3.56e-04 | 1401.0ms/step | 14.3 samples/s +Step 16370/44000 | Loss: 3.0185 | LR: 3.56e-04 | 1399.2ms/step | 14.3 samples/s +Step 16380/44000 | Loss: 2.9950 | LR: 3.55e-04 | 1402.1ms/step | 14.3 samples/s +Step 16390/44000 | Loss: 3.0218 | LR: 3.55e-04 | 1400.4ms/step | 14.3 samples/s +Step 16400/44000 | Loss: 2.9841 | LR: 3.55e-04 | 1399.6ms/step | 14.3 samples/s +Step 16410/44000 | Loss: 3.0440 | LR: 3.55e-04 | 1400.1ms/step | 14.3 samples/s +Step 16420/44000 | Loss: 2.9912 | LR: 3.55e-04 | 1402.4ms/step | 14.3 samples/s +Step 16430/44000 | Loss: 2.9887 | LR: 3.55e-04 | 1403.3ms/step | 14.3 samples/s +Step 16440/44000 | Loss: 3.0123 | LR: 3.54e-04 | 1400.6ms/step | 14.3 samples/s +Step 16450/44000 | Loss: 2.9977 | LR: 3.54e-04 | 1402.2ms/step | 14.3 samples/s +Step 16460/44000 | Loss: 2.9785 | LR: 3.54e-04 | 1402.2ms/step | 14.3 samples/s +Step 16470/44000 | Loss: 2.9969 | LR: 3.54e-04 | 1401.2ms/step | 14.3 samples/s +Step 16480/44000 | Loss: 3.0249 | LR: 3.54e-04 | 1400.0ms/step | 14.3 samples/s +Step 16490/44000 | Loss: 3.0014 | LR: 3.54e-04 | 1400.1ms/step | 14.3 samples/s +Step 16500/44000 | Loss: 3.0037 | LR: 3.53e-04 | 1400.3ms/step | 14.3 samples/s +Step 16510/44000 | Loss: 3.0069 | LR: 3.53e-04 | 1399.6ms/step | 14.3 samples/s +Step 16520/44000 | Loss: 2.9786 | LR: 3.53e-04 | 1400.2ms/step | 14.3 samples/s +Step 16530/44000 | Loss: 3.0031 | LR: 3.53e-04 | 1402.0ms/step | 14.3 samples/s +Step 16540/44000 | Loss: 3.0183 | LR: 3.53e-04 | 1400.4ms/step | 14.3 samples/s +Step 16550/44000 | Loss: 3.0102 | LR: 3.53e-04 | 1401.2ms/step | 14.3 samples/s +Step 16560/44000 | Loss: 3.0114 | LR: 3.52e-04 | 1400.0ms/step | 14.3 samples/s +Step 16570/44000 | Loss: 2.9928 | LR: 3.52e-04 | 1400.5ms/step | 14.3 samples/s +Step 16580/44000 | Loss: 2.9866 | LR: 3.52e-04 | 1400.1ms/step | 14.3 samples/s +Step 16590/44000 | Loss: 3.0028 | LR: 3.52e-04 | 1402.4ms/step | 14.3 samples/s +Step 16600/44000 | Loss: 2.9975 | LR: 3.52e-04 | 1403.3ms/step | 14.3 samples/s +Step 16610/44000 | Loss: 3.0065 | LR: 3.52e-04 | 1402.0ms/step | 14.3 samples/s +Step 16620/44000 | Loss: 3.0008 | LR: 3.51e-04 | 1403.8ms/step | 14.2 samples/s +Step 16630/44000 | Loss: 3.0111 | LR: 3.51e-04 | 1402.6ms/step | 14.3 samples/s +Step 16640/44000 | Loss: 2.9828 | LR: 3.51e-04 | 1405.3ms/step | 14.2 samples/s +Step 16650/44000 | Loss: 3.0010 | LR: 3.51e-04 | 1404.1ms/step | 14.2 samples/s +Step 16660/44000 | Loss: 3.0140 | LR: 3.51e-04 | 1404.2ms/step | 14.2 samples/s +Step 16670/44000 | Loss: 2.9953 | LR: 3.51e-04 | 1402.2ms/step | 14.3 samples/s +Step 16680/44000 | Loss: 3.0067 | LR: 3.50e-04 | 1401.5ms/step | 14.3 samples/s +Step 16690/44000 | Loss: 3.0189 | LR: 3.50e-04 | 1402.4ms/step | 14.3 samples/s +Step 16700/44000 | Loss: 3.0106 | LR: 3.50e-04 | 1406.0ms/step | 14.2 samples/s +Step 16710/44000 | Loss: 3.0117 | LR: 3.50e-04 | 1403.8ms/step | 14.2 samples/s +Step 16720/44000 | Loss: 3.0164 | LR: 3.50e-04 | 1404.1ms/step | 14.2 samples/s +Step 16730/44000 | Loss: 3.0528 | LR: 3.50e-04 | 1403.9ms/step | 14.2 samples/s +Step 16740/44000 | Loss: 2.9955 | LR: 3.49e-04 | 1402.1ms/step | 14.3 samples/s +Step 16750/44000 | Loss: 3.0083 | LR: 3.49e-04 | 1399.5ms/step | 14.3 samples/s +Step 16760/44000 | Loss: 2.9814 | LR: 3.49e-04 | 1400.8ms/step | 14.3 samples/s +Step 16770/44000 | Loss: 2.9909 | LR: 3.49e-04 | 1403.7ms/step | 14.2 samples/s +Step 16780/44000 | Loss: 2.9880 | LR: 3.49e-04 | 1402.2ms/step | 14.3 samples/s +Step 16790/44000 | Loss: 3.0199 | LR: 3.49e-04 | 1404.3ms/step | 14.2 samples/s +Step 16800/44000 | Loss: 2.9944 | LR: 3.48e-04 | 1404.6ms/step | 14.2 samples/s +Step 16810/44000 | Loss: 3.0162 | LR: 3.48e-04 | 1400.8ms/step | 14.3 samples/s +Step 16820/44000 | Loss: 2.9954 | LR: 3.48e-04 | 1401.6ms/step | 14.3 samples/s +Step 16830/44000 | Loss: 3.0058 | LR: 3.48e-04 | 1400.2ms/step | 14.3 samples/s +Step 16840/44000 | Loss: 3.0278 | LR: 3.48e-04 | 1402.3ms/step | 14.3 samples/s +Step 16850/44000 | Loss: 3.0013 | LR: 3.48e-04 | 1401.7ms/step | 14.3 samples/s +Step 16860/44000 | Loss: 3.0208 | LR: 3.47e-04 | 1400.3ms/step | 14.3 samples/s +Step 16870/44000 | Loss: 3.0287 | LR: 3.47e-04 | 1403.6ms/step | 14.2 samples/s +Step 16880/44000 | Loss: 2.9802 | LR: 3.47e-04 | 1400.0ms/step | 14.3 samples/s +Step 16890/44000 | Loss: 3.0092 | LR: 3.47e-04 | 1402.5ms/step | 14.3 samples/s +Step 16900/44000 | Loss: 3.0112 | LR: 3.47e-04 | 1399.4ms/step | 14.3 samples/s +Step 16910/44000 | Loss: 2.9900 | LR: 3.47e-04 | 1402.6ms/step | 14.3 samples/s +Step 16920/44000 | Loss: 3.0025 | LR: 3.46e-04 | 1400.0ms/step | 14.3 samples/s +Step 16930/44000 | Loss: 3.0420 | LR: 3.46e-04 | 1402.6ms/step | 14.3 samples/s +Step 16940/44000 | Loss: 2.9802 | LR: 3.46e-04 | 1403.0ms/step | 14.3 samples/s +Step 16950/44000 | Loss: 3.0236 | LR: 3.46e-04 | 1402.2ms/step | 14.3 samples/s +Step 16960/44000 | Loss: 2.9748 | LR: 3.46e-04 | 1399.9ms/step | 14.3 samples/s +Step 16970/44000 | Loss: 2.9966 | LR: 3.46e-04 | 1402.0ms/step | 14.3 samples/s +Step 16980/44000 | Loss: 3.0043 | LR: 3.45e-04 | 1401.6ms/step | 14.3 samples/s +Step 16990/44000 | Loss: 3.0047 | LR: 3.45e-04 | 1400.0ms/step | 14.3 samples/s +Step 17000/44000 | Loss: 3.0066 | LR: 3.45e-04 | 1399.9ms/step | 14.3 samples/s +Validation at step 17000: Loss = 2.9912, Perplexity = 19.91 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-17000 +Step 17010/44000 | Loss: 2.9859 | LR: 3.45e-04 | 3948.6ms/step | 5.1 samples/s +Step 17020/44000 | Loss: 3.0082 | LR: 3.45e-04 | 1403.8ms/step | 14.2 samples/s +Step 17030/44000 | Loss: 2.9998 | LR: 3.45e-04 | 1401.9ms/step | 14.3 samples/s +Step 17040/44000 | Loss: 3.0025 | LR: 3.44e-04 | 1407.1ms/step | 14.2 samples/s +Step 17050/44000 | Loss: 3.0099 | LR: 3.44e-04 | 1408.8ms/step | 14.2 samples/s +Step 17060/44000 | Loss: 3.0021 | LR: 3.44e-04 | 1409.4ms/step | 14.2 samples/s +Step 17070/44000 | Loss: 2.9977 | LR: 3.44e-04 | 1410.9ms/step | 14.2 samples/s +Step 17080/44000 | Loss: 3.0071 | LR: 3.44e-04 | 1409.5ms/step | 14.2 samples/s +Step 17090/44000 | Loss: 2.9980 | LR: 3.44e-04 | 1407.8ms/step | 14.2 samples/s +Step 17100/44000 | Loss: 2.9998 | LR: 3.43e-04 | 1408.5ms/step | 14.2 samples/s +Step 17110/44000 | Loss: 3.0134 | LR: 3.43e-04 | 1408.3ms/step | 14.2 samples/s +Step 17120/44000 | Loss: 3.0028 | LR: 3.43e-04 | 1412.4ms/step | 14.2 samples/s +Step 17130/44000 | Loss: 3.0126 | LR: 3.43e-04 | 1409.1ms/step | 14.2 samples/s +Step 17140/44000 | Loss: 2.9939 | LR: 3.43e-04 | 1408.1ms/step | 14.2 samples/s +Step 17150/44000 | Loss: 3.0095 | LR: 3.43e-04 | 1409.9ms/step | 14.2 samples/s +Step 17160/44000 | Loss: 2.9964 | LR: 3.42e-04 | 1410.6ms/step | 14.2 samples/s +Step 17170/44000 | Loss: 2.9783 | LR: 3.42e-04 | 1411.3ms/step | 14.2 samples/s +Step 17180/44000 | Loss: 3.0097 | LR: 3.42e-04 | 1410.4ms/step | 14.2 samples/s +Step 17190/44000 | Loss: 2.9733 | LR: 3.42e-04 | 1407.8ms/step | 14.2 samples/s +Step 17200/44000 | Loss: 3.0141 | LR: 3.42e-04 | 1409.9ms/step | 14.2 samples/s +Step 17210/44000 | Loss: 3.0139 | LR: 3.42e-04 | 1406.3ms/step | 14.2 samples/s +Step 17220/44000 | Loss: 2.9842 | LR: 3.41e-04 | 1405.9ms/step | 14.2 samples/s +Step 17230/44000 | Loss: 2.9903 | LR: 3.41e-04 | 1406.2ms/step | 14.2 samples/s +Step 17240/44000 | Loss: 3.0228 | LR: 3.41e-04 | 1410.1ms/step | 14.2 samples/s +Step 17250/44000 | Loss: 3.0173 | LR: 3.41e-04 | 1410.3ms/step | 14.2 samples/s +Step 17260/44000 | Loss: 2.9775 | LR: 3.41e-04 | 1413.2ms/step | 14.2 samples/s +Step 17270/44000 | Loss: 2.9755 | LR: 3.41e-04 | 1408.4ms/step | 14.2 samples/s +Step 17280/44000 | Loss: 3.0070 | LR: 3.40e-04 | 1410.0ms/step | 14.2 samples/s +Step 17290/44000 | Loss: 2.9774 | LR: 3.40e-04 | 1409.8ms/step | 14.2 samples/s +Step 17300/44000 | Loss: 2.9916 | LR: 3.40e-04 | 1406.9ms/step | 14.2 samples/s +Step 17310/44000 | Loss: 3.0065 | LR: 3.40e-04 | 1406.8ms/step | 14.2 samples/s +Step 17320/44000 | Loss: 3.0215 | LR: 3.40e-04 | 1410.0ms/step | 14.2 samples/s +Step 17330/44000 | Loss: 3.0171 | LR: 3.40e-04 | 1409.5ms/step | 14.2 samples/s +Step 17340/44000 | Loss: 3.0105 | LR: 3.39e-04 | 1409.0ms/step | 14.2 samples/s +Step 17350/44000 | Loss: 3.0224 | LR: 3.39e-04 | 1405.4ms/step | 14.2 samples/s +Step 17360/44000 | Loss: 2.9547 | LR: 3.39e-04 | 1399.9ms/step | 14.3 samples/s +Step 17370/44000 | Loss: 3.0315 | LR: 3.39e-04 | 1402.9ms/step | 14.3 samples/s +Step 17380/44000 | Loss: 2.9934 | LR: 3.39e-04 | 1401.7ms/step | 14.3 samples/s +Step 17390/44000 | Loss: 3.0234 | LR: 3.39e-04 | 1402.0ms/step | 14.3 samples/s +Step 17400/44000 | Loss: 3.0135 | LR: 3.38e-04 | 1401.7ms/step | 14.3 samples/s +Step 17410/44000 | Loss: 2.9675 | LR: 3.38e-04 | 1401.7ms/step | 14.3 samples/s +Step 17420/44000 | Loss: 2.9944 | LR: 3.38e-04 | 1402.1ms/step | 14.3 samples/s +Step 17430/44000 | Loss: 2.9855 | LR: 3.38e-04 | 1400.3ms/step | 14.3 samples/s +Step 17440/44000 | Loss: 3.0193 | LR: 3.38e-04 | 1400.6ms/step | 14.3 samples/s +Step 17450/44000 | Loss: 3.0119 | LR: 3.38e-04 | 1401.8ms/step | 14.3 samples/s +Step 17460/44000 | Loss: 2.9935 | LR: 3.37e-04 | 1401.1ms/step | 14.3 samples/s +Step 17470/44000 | Loss: 2.9749 | LR: 3.37e-04 | 1399.9ms/step | 14.3 samples/s +Step 17480/44000 | Loss: 3.0113 | LR: 3.37e-04 | 1400.2ms/step | 14.3 samples/s +Step 17490/44000 | Loss: 2.9957 | LR: 3.37e-04 | 1406.0ms/step | 14.2 samples/s +Step 17500/44000 | Loss: 2.9999 | LR: 3.37e-04 | 1402.3ms/step | 14.3 samples/s +Step 17510/44000 | Loss: 2.9788 | LR: 3.37e-04 | 1401.7ms/step | 14.3 samples/s +Step 17520/44000 | Loss: 2.9929 | LR: 3.36e-04 | 1404.1ms/step | 14.2 samples/s +Step 17530/44000 | Loss: 2.9834 | LR: 3.36e-04 | 1404.5ms/step | 14.2 samples/s +Step 17540/44000 | Loss: 2.9702 | LR: 3.36e-04 | 1402.0ms/step | 14.3 samples/s +Step 17550/44000 | Loss: 2.9899 | LR: 3.36e-04 | 1402.8ms/step | 14.3 samples/s +Step 17560/44000 | Loss: 2.9560 | LR: 3.36e-04 | 1402.6ms/step | 14.3 samples/s +Step 17570/44000 | Loss: 2.9599 | LR: 3.35e-04 | 1404.3ms/step | 14.2 samples/s +Step 17580/44000 | Loss: 2.9956 | LR: 3.35e-04 | 1402.1ms/step | 14.3 samples/s +Step 17590/44000 | Loss: 2.9956 | LR: 3.35e-04 | 1399.8ms/step | 14.3 samples/s +Step 17600/44000 | Loss: 3.0255 | LR: 3.35e-04 | 1402.0ms/step | 14.3 samples/s +Step 17610/44000 | Loss: 2.9929 | LR: 3.35e-04 | 1403.8ms/step | 14.2 samples/s +Step 17620/44000 | Loss: 2.9550 | LR: 3.35e-04 | 1400.6ms/step | 14.3 samples/s +Step 17630/44000 | Loss: 3.0227 | LR: 3.34e-04 | 1403.5ms/step | 14.3 samples/s +Step 17640/44000 | Loss: 2.9923 | LR: 3.34e-04 | 1402.5ms/step | 14.3 samples/s +Step 17650/44000 | Loss: 2.9923 | LR: 3.34e-04 | 1401.3ms/step | 14.3 samples/s +Step 17660/44000 | Loss: 2.9837 | LR: 3.34e-04 | 1402.2ms/step | 14.3 samples/s +Step 17670/44000 | Loss: 2.9928 | LR: 3.34e-04 | 1402.8ms/step | 14.3 samples/s +Step 17680/44000 | Loss: 2.9856 | LR: 3.34e-04 | 1401.3ms/step | 14.3 samples/s +Step 17690/44000 | Loss: 2.9979 | LR: 3.33e-04 | 1400.2ms/step | 14.3 samples/s +Step 17700/44000 | Loss: 2.9534 | LR: 3.33e-04 | 1401.7ms/step | 14.3 samples/s +Step 17710/44000 | Loss: 2.9818 | LR: 3.33e-04 | 1402.3ms/step | 14.3 samples/s +Step 17720/44000 | Loss: 2.9715 | LR: 3.33e-04 | 1399.6ms/step | 14.3 samples/s +Step 17730/44000 | Loss: 2.9720 | LR: 3.33e-04 | 1400.6ms/step | 14.3 samples/s +Step 17740/44000 | Loss: 2.9461 | LR: 3.33e-04 | 1401.3ms/step | 14.3 samples/s +Step 17750/44000 | Loss: 3.0006 | LR: 3.32e-04 | 1400.1ms/step | 14.3 samples/s +Step 17760/44000 | Loss: 2.9835 | LR: 3.32e-04 | 1399.8ms/step | 14.3 samples/s +Step 17770/44000 | Loss: 2.9829 | LR: 3.32e-04 | 1402.0ms/step | 14.3 samples/s +Step 17780/44000 | Loss: 2.9943 | LR: 3.32e-04 | 1402.2ms/step | 14.3 samples/s +Step 17790/44000 | Loss: 3.0138 | LR: 3.32e-04 | 1400.1ms/step | 14.3 samples/s +Step 17800/44000 | Loss: 2.9886 | LR: 3.32e-04 | 1401.8ms/step | 14.3 samples/s +Step 17810/44000 | Loss: 2.9793 | LR: 3.31e-04 | 1400.0ms/step | 14.3 samples/s +Step 17820/44000 | Loss: 2.9885 | LR: 3.31e-04 | 1402.2ms/step | 14.3 samples/s +Step 17830/44000 | Loss: 2.9953 | LR: 3.31e-04 | 1402.1ms/step | 14.3 samples/s +Step 17840/44000 | Loss: 3.0073 | LR: 3.31e-04 | 1401.9ms/step | 14.3 samples/s +Step 17850/44000 | Loss: 3.0087 | LR: 3.31e-04 | 1402.1ms/step | 14.3 samples/s +Step 17860/44000 | Loss: 3.0400 | LR: 3.31e-04 | 1402.3ms/step | 14.3 samples/s +Step 17870/44000 | Loss: 2.9774 | LR: 3.30e-04 | 1401.5ms/step | 14.3 samples/s +Step 17880/44000 | Loss: 2.9930 | LR: 3.30e-04 | 1401.9ms/step | 14.3 samples/s +Step 17890/44000 | Loss: 2.9595 | LR: 3.30e-04 | 1401.9ms/step | 14.3 samples/s +Step 17900/44000 | Loss: 2.9887 | LR: 3.30e-04 | 1402.2ms/step | 14.3 samples/s +Step 17910/44000 | Loss: 2.9930 | LR: 3.30e-04 | 1404.0ms/step | 14.2 samples/s +Step 17920/44000 | Loss: 2.9938 | LR: 3.29e-04 | 1404.1ms/step | 14.2 samples/s +Step 17930/44000 | Loss: 2.9668 | LR: 3.29e-04 | 1402.2ms/step | 14.3 samples/s +Step 17940/44000 | Loss: 3.0049 | LR: 3.29e-04 | 1401.7ms/step | 14.3 samples/s +Step 17950/44000 | Loss: 2.9615 | LR: 3.29e-04 | 1400.1ms/step | 14.3 samples/s +Step 17960/44000 | Loss: 2.9870 | LR: 3.29e-04 | 1401.7ms/step | 14.3 samples/s +Step 17970/44000 | Loss: 2.9579 | LR: 3.29e-04 | 1402.6ms/step | 14.3 samples/s +Step 17980/44000 | Loss: 3.0065 | LR: 3.28e-04 | 1405.4ms/step | 14.2 samples/s +Step 17990/44000 | Loss: 2.9547 | LR: 3.28e-04 | 1409.2ms/step | 14.2 samples/s +Step 18000/44000 | Loss: 3.0120 | LR: 3.28e-04 | 1408.9ms/step | 14.2 samples/s +Validation at step 18000: Loss = 2.9788, Perplexity = 19.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-18000 +Step 18010/44000 | Loss: 3.0195 | LR: 3.28e-04 | 4022.0ms/step | 5.0 samples/s +Step 18020/44000 | Loss: 2.9689 | LR: 3.28e-04 | 1405.7ms/step | 14.2 samples/s +Step 18030/44000 | Loss: 2.9515 | LR: 3.28e-04 | 1410.2ms/step | 14.2 samples/s +Step 18040/44000 | Loss: 2.9922 | LR: 3.27e-04 | 1410.7ms/step | 14.2 samples/s +Step 18050/44000 | Loss: 2.9694 | LR: 3.27e-04 | 1409.9ms/step | 14.2 samples/s +Step 18060/44000 | Loss: 2.9700 | LR: 3.27e-04 | 1412.4ms/step | 14.2 samples/s +Step 18070/44000 | Loss: 3.0065 | LR: 3.27e-04 | 1412.9ms/step | 14.2 samples/s +Step 18080/44000 | Loss: 3.0038 | LR: 3.27e-04 | 1410.6ms/step | 14.2 samples/s +Step 18090/44000 | Loss: 2.9937 | LR: 3.27e-04 | 1411.3ms/step | 14.2 samples/s +Step 18100/44000 | Loss: 2.9676 | LR: 3.26e-04 | 1409.3ms/step | 14.2 samples/s +Step 18110/44000 | Loss: 2.9604 | LR: 3.26e-04 | 1410.5ms/step | 14.2 samples/s +Step 18120/44000 | Loss: 2.9780 | LR: 3.26e-04 | 1410.3ms/step | 14.2 samples/s +Step 18130/44000 | Loss: 2.9988 | LR: 3.26e-04 | 1410.0ms/step | 14.2 samples/s +Step 18140/44000 | Loss: 2.9800 | LR: 3.26e-04 | 1409.8ms/step | 14.2 samples/s +Step 18150/44000 | Loss: 3.0062 | LR: 3.26e-04 | 1411.2ms/step | 14.2 samples/s +Step 18160/44000 | Loss: 2.9385 | LR: 3.25e-04 | 1409.3ms/step | 14.2 samples/s +Step 18170/44000 | Loss: 2.9684 | LR: 3.25e-04 | 1409.8ms/step | 14.2 samples/s +Step 18180/44000 | Loss: 3.0144 | LR: 3.25e-04 | 1411.7ms/step | 14.2 samples/s +Step 18190/44000 | Loss: 2.9787 | LR: 3.25e-04 | 1408.5ms/step | 14.2 samples/s +Step 18200/44000 | Loss: 3.0038 | LR: 3.25e-04 | 1410.3ms/step | 14.2 samples/s +Step 18210/44000 | Loss: 2.9708 | LR: 3.24e-04 | 1409.8ms/step | 14.2 samples/s +Step 18220/44000 | Loss: 2.9810 | LR: 3.24e-04 | 1411.5ms/step | 14.2 samples/s +Step 18230/44000 | Loss: 2.9815 | LR: 3.24e-04 | 1408.3ms/step | 14.2 samples/s +Step 18240/44000 | Loss: 3.0291 | LR: 3.24e-04 | 1409.5ms/step | 14.2 samples/s +Step 18250/44000 | Loss: 2.9642 | LR: 3.24e-04 | 1407.2ms/step | 14.2 samples/s +Step 18260/44000 | Loss: 3.0108 | LR: 3.24e-04 | 1408.9ms/step | 14.2 samples/s +Step 18270/44000 | Loss: 2.9636 | LR: 3.23e-04 | 1410.6ms/step | 14.2 samples/s +Step 18280/44000 | Loss: 2.9813 | LR: 3.23e-04 | 1408.1ms/step | 14.2 samples/s +Step 18290/44000 | Loss: 3.0170 | LR: 3.23e-04 | 1410.9ms/step | 14.2 samples/s +Step 18300/44000 | Loss: 3.0146 | LR: 3.23e-04 | 1410.5ms/step | 14.2 samples/s +Step 18310/44000 | Loss: 2.9883 | LR: 3.23e-04 | 1409.9ms/step | 14.2 samples/s +Step 18320/44000 | Loss: 2.9947 | LR: 3.23e-04 | 1408.7ms/step | 14.2 samples/s +Step 18330/44000 | Loss: 2.9648 | LR: 3.22e-04 | 1411.8ms/step | 14.2 samples/s +Step 18340/44000 | Loss: 3.0022 | LR: 3.22e-04 | 1413.6ms/step | 14.1 samples/s +Step 18350/44000 | Loss: 2.9705 | LR: 3.22e-04 | 1412.3ms/step | 14.2 samples/s +Step 18360/44000 | Loss: 3.0002 | LR: 3.22e-04 | 1409.5ms/step | 14.2 samples/s +Step 18370/44000 | Loss: 2.9743 | LR: 3.22e-04 | 1410.3ms/step | 14.2 samples/s +Step 18380/44000 | Loss: 3.0053 | LR: 3.22e-04 | 1407.6ms/step | 14.2 samples/s +Step 18390/44000 | Loss: 2.9929 | LR: 3.21e-04 | 1410.0ms/step | 14.2 samples/s +Step 18400/44000 | Loss: 2.9745 | LR: 3.21e-04 | 1408.1ms/step | 14.2 samples/s +Step 18410/44000 | Loss: 2.9821 | LR: 3.21e-04 | 1409.8ms/step | 14.2 samples/s +Step 18420/44000 | Loss: 2.9775 | LR: 3.21e-04 | 1406.0ms/step | 14.2 samples/s +Step 18430/44000 | Loss: 2.9899 | LR: 3.21e-04 | 1410.0ms/step | 14.2 samples/s +Step 18440/44000 | Loss: 2.9887 | LR: 3.20e-04 | 1408.6ms/step | 14.2 samples/s +Step 18450/44000 | Loss: 2.9978 | LR: 3.20e-04 | 1410.2ms/step | 14.2 samples/s diff --git a/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed44.20260820T041452Z.log b/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed44.20260820T041452Z.log new file mode 100644 index 0000000000000000000000000000000000000000..aa0e99839f0f04bd7da290b35e3ff06ea72956ae --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed44.20260820T041452Z.log @@ -0,0 +1,666 @@ +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-13000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-13000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 44, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed44", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-13000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-13000... +Preserving checkpoint LR state for exact resume: current=[0.0004073888929652048], schedule_base=[0.0005] +Resumed from checkpoint at step 13000 (epoch 0, micro_step 130000) +Resuming training from step 13000 +Resuming wandb run bajm6b15 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run bajm6b15 +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041558-bajm6b15 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed44 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/bajm6b15 +Starting training... +Validation at step 13000: Loss = 3.0696, Perplexity = 21.53 +wandb: WARNING Tried to log to step 13000 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 13001: dense (dense step interval) +Step 13010/44000 | Loss: 3.0583 | LR: 4.07e-04 | 30145.6ms/step | 0.7 samples/s +wandb: WARNING Tried to log to step 13010 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13020/44000 | Loss: 3.0539 | LR: 4.07e-04 | 1368.0ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 13020 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13030/44000 | Loss: 3.0645 | LR: 4.07e-04 | 1371.0ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 13030 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13040/44000 | Loss: 3.0438 | LR: 4.07e-04 | 1375.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13040 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13050/44000 | Loss: 3.0602 | LR: 4.07e-04 | 1378.6ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13050 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13060/44000 | Loss: 3.0601 | LR: 4.07e-04 | 1376.2ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13060 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13070/44000 | Loss: 3.0745 | LR: 4.06e-04 | 1380.3ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13070 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13080/44000 | Loss: 3.0356 | LR: 4.06e-04 | 1379.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13080 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13090/44000 | Loss: 3.0677 | LR: 4.06e-04 | 1377.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13090 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13100/44000 | Loss: 3.0657 | LR: 4.06e-04 | 1379.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13100 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13110/44000 | Loss: 3.0377 | LR: 4.06e-04 | 1382.3ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13110 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13120/44000 | Loss: 3.0419 | LR: 4.06e-04 | 1377.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13120 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13130/44000 | Loss: 3.0623 | LR: 4.06e-04 | 1374.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13130 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13140/44000 | Loss: 3.0456 | LR: 4.05e-04 | 1373.6ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 13140 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13150/44000 | Loss: 3.0531 | LR: 4.05e-04 | 1375.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13150 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13160/44000 | Loss: 3.0258 | LR: 4.05e-04 | 1375.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13160 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13170/44000 | Loss: 3.0970 | LR: 4.05e-04 | 1373.9ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 13170 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13180/44000 | Loss: 3.0733 | LR: 4.05e-04 | 1372.2ms/step | 14.6 samples/s +Step 13190/44000 | Loss: 3.0473 | LR: 4.05e-04 | 1373.9ms/step | 14.6 samples/s +Step 13200/44000 | Loss: 3.0934 | LR: 4.05e-04 | 1375.0ms/step | 14.5 samples/s +Step 13210/44000 | Loss: 3.0429 | LR: 4.04e-04 | 1372.5ms/step | 14.6 samples/s +Step 13220/44000 | Loss: 3.0658 | LR: 4.04e-04 | 1372.0ms/step | 14.6 samples/s +Step 13230/44000 | Loss: 3.0785 | LR: 4.04e-04 | 1373.3ms/step | 14.6 samples/s +Step 13240/44000 | Loss: 3.0605 | LR: 4.04e-04 | 1373.4ms/step | 14.6 samples/s +Step 13250/44000 | Loss: 3.0669 | LR: 4.04e-04 | 1373.5ms/step | 14.6 samples/s +Step 13260/44000 | Loss: 3.0628 | LR: 4.04e-04 | 1372.4ms/step | 14.6 samples/s +Step 13270/44000 | Loss: 3.0632 | LR: 4.04e-04 | 1373.9ms/step | 14.6 samples/s +Step 13280/44000 | Loss: 3.0612 | LR: 4.03e-04 | 1376.4ms/step | 14.5 samples/s +Step 13290/44000 | Loss: 3.0461 | LR: 4.03e-04 | 1373.5ms/step | 14.6 samples/s +Step 13300/44000 | Loss: 3.0573 | LR: 4.03e-04 | 1374.0ms/step | 14.6 samples/s +Step 13310/44000 | Loss: 3.0515 | LR: 4.03e-04 | 1373.8ms/step | 14.6 samples/s +Step 13320/44000 | Loss: 3.0342 | LR: 4.03e-04 | 1374.6ms/step | 14.5 samples/s +Step 13330/44000 | Loss: 3.0614 | LR: 4.03e-04 | 1372.6ms/step | 14.6 samples/s +Step 13340/44000 | Loss: 3.0749 | LR: 4.03e-04 | 1372.3ms/step | 14.6 samples/s +Step 13350/44000 | Loss: 3.0320 | LR: 4.02e-04 | 1372.7ms/step | 14.6 samples/s +Step 13360/44000 | Loss: 3.1017 | LR: 4.02e-04 | 1374.6ms/step | 14.6 samples/s +Step 13370/44000 | Loss: 3.0585 | LR: 4.02e-04 | 1373.5ms/step | 14.6 samples/s +Step 13380/44000 | Loss: 3.0661 | LR: 4.02e-04 | 1373.5ms/step | 14.6 samples/s +Step 13390/44000 | Loss: 3.0359 | LR: 4.02e-04 | 1376.2ms/step | 14.5 samples/s +Step 13400/44000 | Loss: 3.0451 | LR: 4.02e-04 | 1377.0ms/step | 14.5 samples/s +Step 13410/44000 | Loss: 3.0756 | LR: 4.02e-04 | 1375.1ms/step | 14.5 samples/s +Step 13420/44000 | Loss: 3.0443 | LR: 4.01e-04 | 1376.3ms/step | 14.5 samples/s +Step 13430/44000 | Loss: 3.0581 | LR: 4.01e-04 | 1375.4ms/step | 14.5 samples/s +Step 13440/44000 | Loss: 3.0353 | LR: 4.01e-04 | 1375.8ms/step | 14.5 samples/s +Step 13450/44000 | Loss: 3.0295 | LR: 4.01e-04 | 1372.9ms/step | 14.6 samples/s +Step 13460/44000 | Loss: 3.0325 | LR: 4.01e-04 | 1373.6ms/step | 14.6 samples/s +Step 13470/44000 | Loss: 3.0720 | LR: 4.01e-04 | 1374.7ms/step | 14.5 samples/s +Step 13480/44000 | Loss: 3.0366 | LR: 4.01e-04 | 1375.3ms/step | 14.5 samples/s +Step 13490/44000 | Loss: 3.0641 | LR: 4.00e-04 | 1374.6ms/step | 14.6 samples/s +Step 13500/44000 | Loss: 3.0567 | LR: 4.00e-04 | 1375.5ms/step | 14.5 samples/s +Step 13510/44000 | Loss: 3.0549 | LR: 4.00e-04 | 1374.9ms/step | 14.5 samples/s +Step 13520/44000 | Loss: 3.0648 | LR: 4.00e-04 | 1376.7ms/step | 14.5 samples/s +Step 13530/44000 | Loss: 3.0518 | LR: 4.00e-04 | 1374.2ms/step | 14.6 samples/s +Step 13540/44000 | Loss: 3.0569 | LR: 4.00e-04 | 1374.2ms/step | 14.6 samples/s +Step 13550/44000 | Loss: 3.0538 | LR: 4.00e-04 | 1374.4ms/step | 14.6 samples/s +Step 13560/44000 | Loss: 3.0502 | LR: 3.99e-04 | 1374.0ms/step | 14.6 samples/s +Step 13570/44000 | Loss: 3.0712 | LR: 3.99e-04 | 1373.0ms/step | 14.6 samples/s +Step 13580/44000 | Loss: 3.0557 | LR: 3.99e-04 | 1372.4ms/step | 14.6 samples/s +Step 13590/44000 | Loss: 3.0451 | LR: 3.99e-04 | 1376.1ms/step | 14.5 samples/s +Step 13600/44000 | Loss: 3.0511 | LR: 3.99e-04 | 1374.8ms/step | 14.5 samples/s +Step 13610/44000 | Loss: 3.0764 | LR: 3.99e-04 | 1373.4ms/step | 14.6 samples/s +Step 13620/44000 | Loss: 3.0381 | LR: 3.98e-04 | 1373.0ms/step | 14.6 samples/s +Step 13630/44000 | Loss: 3.0307 | LR: 3.98e-04 | 1374.2ms/step | 14.6 samples/s +Step 13640/44000 | Loss: 3.0425 | LR: 3.98e-04 | 1376.8ms/step | 14.5 samples/s +Step 13650/44000 | Loss: 3.0684 | LR: 3.98e-04 | 1372.6ms/step | 14.6 samples/s +Step 13660/44000 | Loss: 3.0299 | LR: 3.98e-04 | 1371.7ms/step | 14.6 samples/s +Step 13670/44000 | Loss: 3.0322 | LR: 3.98e-04 | 1374.8ms/step | 14.5 samples/s +Step 13680/44000 | Loss: 3.0632 | LR: 3.98e-04 | 1375.3ms/step | 14.5 samples/s +Step 13690/44000 | Loss: 3.0209 | LR: 3.97e-04 | 1372.8ms/step | 14.6 samples/s +Step 13700/44000 | Loss: 3.0666 | LR: 3.97e-04 | 1372.7ms/step | 14.6 samples/s +Step 13710/44000 | Loss: 3.0531 | LR: 3.97e-04 | 1375.3ms/step | 14.5 samples/s +Step 13720/44000 | Loss: 3.0627 | LR: 3.97e-04 | 1374.7ms/step | 14.5 samples/s +Step 13730/44000 | Loss: 3.0650 | LR: 3.97e-04 | 1373.7ms/step | 14.6 samples/s +Step 13740/44000 | Loss: 3.0528 | LR: 3.97e-04 | 1372.0ms/step | 14.6 samples/s +Step 13750/44000 | Loss: 3.0296 | LR: 3.97e-04 | 1377.0ms/step | 14.5 samples/s +Step 13760/44000 | Loss: 3.0609 | LR: 3.96e-04 | 1377.0ms/step | 14.5 samples/s +Step 13770/44000 | Loss: 3.0541 | LR: 3.96e-04 | 1373.5ms/step | 14.6 samples/s +Step 13780/44000 | Loss: 3.0327 | LR: 3.96e-04 | 1372.9ms/step | 14.6 samples/s +Step 13790/44000 | Loss: 3.0562 | LR: 3.96e-04 | 1372.0ms/step | 14.6 samples/s +Step 13800/44000 | Loss: 3.0464 | LR: 3.96e-04 | 1373.8ms/step | 14.6 samples/s +Step 13810/44000 | Loss: 3.0374 | LR: 3.96e-04 | 1373.6ms/step | 14.6 samples/s +Step 13820/44000 | Loss: 3.0165 | LR: 3.96e-04 | 1372.0ms/step | 14.6 samples/s +Step 13830/44000 | Loss: 3.0255 | LR: 3.95e-04 | 1373.2ms/step | 14.6 samples/s +Step 13840/44000 | Loss: 3.0070 | LR: 3.95e-04 | 1374.4ms/step | 14.6 samples/s +Step 13850/44000 | Loss: 3.0490 | LR: 3.95e-04 | 1371.8ms/step | 14.6 samples/s +Step 13860/44000 | Loss: 3.0623 | LR: 3.95e-04 | 1372.3ms/step | 14.6 samples/s +Step 13870/44000 | Loss: 3.0529 | LR: 3.95e-04 | 1375.4ms/step | 14.5 samples/s +Step 13880/44000 | Loss: 3.0477 | LR: 3.95e-04 | 1370.7ms/step | 14.6 samples/s +Step 13890/44000 | Loss: 3.0355 | LR: 3.95e-04 | 1371.6ms/step | 14.6 samples/s +Step 13900/44000 | Loss: 3.0340 | LR: 3.94e-04 | 1371.9ms/step | 14.6 samples/s +Step 13910/44000 | Loss: 3.0398 | LR: 3.94e-04 | 1375.2ms/step | 14.5 samples/s +Step 13920/44000 | Loss: 3.0894 | LR: 3.94e-04 | 1373.9ms/step | 14.6 samples/s +Step 13930/44000 | Loss: 3.0537 | LR: 3.94e-04 | 1373.3ms/step | 14.6 samples/s +Step 13940/44000 | Loss: 3.0667 | LR: 3.94e-04 | 1374.1ms/step | 14.6 samples/s +Step 13950/44000 | Loss: 3.0609 | LR: 3.94e-04 | 1374.8ms/step | 14.5 samples/s +Step 13960/44000 | Loss: 3.0364 | LR: 3.93e-04 | 1374.3ms/step | 14.6 samples/s +Step 13970/44000 | Loss: 3.0370 | LR: 3.93e-04 | 1373.4ms/step | 14.6 samples/s +Step 13980/44000 | Loss: 3.0801 | LR: 3.93e-04 | 1375.5ms/step | 14.5 samples/s +Step 13990/44000 | Loss: 3.0125 | LR: 3.93e-04 | 1377.0ms/step | 14.5 samples/s +Step 14000/44000 | Loss: 3.0749 | LR: 3.93e-04 | 1375.5ms/step | 14.5 samples/s +Validation at step 14000: Loss = 3.0537, Perplexity = 21.19 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-14000 +Step 14010/44000 | Loss: 3.0348 | LR: 3.93e-04 | 3894.1ms/step | 5.1 samples/s +Step 14020/44000 | Loss: 3.0500 | LR: 3.93e-04 | 1385.9ms/step | 14.4 samples/s +Step 14030/44000 | Loss: 3.0532 | LR: 3.92e-04 | 1377.6ms/step | 14.5 samples/s +Step 14040/44000 | Loss: 3.0203 | LR: 3.92e-04 | 1378.8ms/step | 14.5 samples/s +Step 14050/44000 | Loss: 3.0323 | LR: 3.92e-04 | 1381.5ms/step | 14.5 samples/s +Step 14060/44000 | Loss: 3.0529 | LR: 3.92e-04 | 1381.6ms/step | 14.5 samples/s +Step 14070/44000 | Loss: 3.0459 | LR: 3.92e-04 | 1384.4ms/step | 14.4 samples/s +Step 14080/44000 | Loss: 3.0383 | LR: 3.92e-04 | 1385.9ms/step | 14.4 samples/s +Step 14090/44000 | Loss: 3.0212 | LR: 3.92e-04 | 1382.8ms/step | 14.5 samples/s +Step 14100/44000 | Loss: 3.0382 | LR: 3.91e-04 | 1382.0ms/step | 14.5 samples/s +Step 14110/44000 | Loss: 3.0276 | LR: 3.91e-04 | 1385.4ms/step | 14.4 samples/s +Step 14120/44000 | Loss: 3.0614 | LR: 3.91e-04 | 1386.5ms/step | 14.4 samples/s +Step 14130/44000 | Loss: 3.0314 | LR: 3.91e-04 | 1385.7ms/step | 14.4 samples/s +Step 14140/44000 | Loss: 3.0562 | LR: 3.91e-04 | 1384.5ms/step | 14.4 samples/s +Step 14150/44000 | Loss: 3.0508 | LR: 3.91e-04 | 1387.3ms/step | 14.4 samples/s +Step 14160/44000 | Loss: 3.0385 | LR: 3.90e-04 | 1384.0ms/step | 14.5 samples/s +Step 14170/44000 | Loss: 3.0346 | LR: 3.90e-04 | 1387.1ms/step | 14.4 samples/s +Step 14180/44000 | Loss: 3.0018 | LR: 3.90e-04 | 1388.9ms/step | 14.4 samples/s +Step 14190/44000 | Loss: 3.0360 | LR: 3.90e-04 | 1384.1ms/step | 14.4 samples/s +Step 14200/44000 | Loss: 3.0175 | LR: 3.90e-04 | 1385.5ms/step | 14.4 samples/s +Step 14210/44000 | Loss: 3.0325 | LR: 3.90e-04 | 1385.8ms/step | 14.4 samples/s +Step 14220/44000 | Loss: 3.0249 | LR: 3.90e-04 | 1384.6ms/step | 14.4 samples/s +Step 14230/44000 | Loss: 3.0705 | LR: 3.89e-04 | 1385.9ms/step | 14.4 samples/s +Step 14240/44000 | Loss: 3.0322 | LR: 3.89e-04 | 1385.6ms/step | 14.4 samples/s +Step 14250/44000 | Loss: 3.0173 | LR: 3.89e-04 | 1383.4ms/step | 14.5 samples/s +Step 14260/44000 | Loss: 3.0340 | LR: 3.89e-04 | 1384.7ms/step | 14.4 samples/s +Step 14270/44000 | Loss: 3.0348 | LR: 3.89e-04 | 1385.8ms/step | 14.4 samples/s +Step 14280/44000 | Loss: 3.0256 | LR: 3.89e-04 | 1385.2ms/step | 14.4 samples/s +Step 14290/44000 | Loss: 3.0314 | LR: 3.89e-04 | 1382.8ms/step | 14.5 samples/s +Step 14300/44000 | Loss: 3.0434 | LR: 3.88e-04 | 1383.6ms/step | 14.5 samples/s +Step 14310/44000 | Loss: 3.0538 | LR: 3.88e-04 | 1384.8ms/step | 14.4 samples/s +Step 14320/44000 | Loss: 3.0534 | LR: 3.88e-04 | 1383.7ms/step | 14.5 samples/s +Step 14330/44000 | Loss: 3.0495 | LR: 3.88e-04 | 1383.4ms/step | 14.5 samples/s +Step 14340/44000 | Loss: 3.0320 | LR: 3.88e-04 | 1384.7ms/step | 14.4 samples/s +Step 14350/44000 | Loss: 3.0422 | LR: 3.88e-04 | 1382.5ms/step | 14.5 samples/s +Step 14360/44000 | Loss: 2.9975 | LR: 3.87e-04 | 1383.7ms/step | 14.5 samples/s +Step 14370/44000 | Loss: 3.0624 | LR: 3.87e-04 | 1385.1ms/step | 14.4 samples/s +Step 14380/44000 | Loss: 3.0577 | LR: 3.87e-04 | 1386.6ms/step | 14.4 samples/s +Step 14390/44000 | Loss: 3.0378 | LR: 3.87e-04 | 1388.2ms/step | 14.4 samples/s +Step 14400/44000 | Loss: 3.0463 | LR: 3.87e-04 | 1389.3ms/step | 14.4 samples/s +Step 14410/44000 | Loss: 3.0376 | LR: 3.87e-04 | 1381.8ms/step | 14.5 samples/s +Step 14420/44000 | Loss: 3.0484 | LR: 3.87e-04 | 1384.2ms/step | 14.4 samples/s +Step 14430/44000 | Loss: 3.0090 | LR: 3.86e-04 | 1384.3ms/step | 14.4 samples/s +Step 14440/44000 | Loss: 3.0306 | LR: 3.86e-04 | 1387.3ms/step | 14.4 samples/s +Step 14450/44000 | Loss: 3.0626 | LR: 3.86e-04 | 1384.5ms/step | 14.4 samples/s +Step 14460/44000 | Loss: 3.0276 | LR: 3.86e-04 | 1383.1ms/step | 14.5 samples/s +Step 14470/44000 | Loss: 3.0396 | LR: 3.86e-04 | 1385.3ms/step | 14.4 samples/s +Step 14480/44000 | Loss: 3.0202 | LR: 3.86e-04 | 1385.1ms/step | 14.4 samples/s +Step 14490/44000 | Loss: 3.0549 | LR: 3.85e-04 | 1385.3ms/step | 14.4 samples/s +Step 14500/44000 | Loss: 3.0393 | LR: 3.85e-04 | 1390.2ms/step | 14.4 samples/s +Step 14510/44000 | Loss: 3.0490 | LR: 3.85e-04 | 1388.0ms/step | 14.4 samples/s +Step 14520/44000 | Loss: 3.0261 | LR: 3.85e-04 | 1386.4ms/step | 14.4 samples/s +Step 14530/44000 | Loss: 3.0523 | LR: 3.85e-04 | 1384.2ms/step | 14.4 samples/s +Step 14540/44000 | Loss: 3.0644 | LR: 3.85e-04 | 1386.2ms/step | 14.4 samples/s +Step 14550/44000 | Loss: 3.0110 | LR: 3.85e-04 | 1381.8ms/step | 14.5 samples/s +Step 14560/44000 | Loss: 3.0658 | LR: 3.84e-04 | 1386.2ms/step | 14.4 samples/s +Step 14570/44000 | Loss: 3.0399 | LR: 3.84e-04 | 1383.6ms/step | 14.5 samples/s +Step 14580/44000 | Loss: 3.0060 | LR: 3.84e-04 | 1382.1ms/step | 14.5 samples/s +Step 14590/44000 | Loss: 3.0186 | LR: 3.84e-04 | 1384.6ms/step | 14.4 samples/s +Step 14600/44000 | Loss: 3.0539 | LR: 3.84e-04 | 1383.9ms/step | 14.5 samples/s +Step 14610/44000 | Loss: 3.0583 | LR: 3.84e-04 | 1382.9ms/step | 14.5 samples/s +Step 14620/44000 | Loss: 3.0253 | LR: 3.83e-04 | 1385.6ms/step | 14.4 samples/s +Step 14630/44000 | Loss: 3.0322 | LR: 3.83e-04 | 1383.8ms/step | 14.5 samples/s +Step 14640/44000 | Loss: 3.0508 | LR: 3.83e-04 | 1384.9ms/step | 14.4 samples/s +Step 14650/44000 | Loss: 3.0502 | LR: 3.83e-04 | 1388.3ms/step | 14.4 samples/s +Step 14660/44000 | Loss: 3.0512 | LR: 3.83e-04 | 1385.5ms/step | 14.4 samples/s +Step 14670/44000 | Loss: 3.0824 | LR: 3.83e-04 | 1384.4ms/step | 14.4 samples/s +Step 14680/44000 | Loss: 3.0295 | LR: 3.83e-04 | 1385.3ms/step | 14.4 samples/s +Step 14690/44000 | Loss: 3.0247 | LR: 3.82e-04 | 1384.1ms/step | 14.4 samples/s +Step 14700/44000 | Loss: 3.0529 | LR: 3.82e-04 | 1385.0ms/step | 14.4 samples/s +Step 14710/44000 | Loss: 3.0174 | LR: 3.82e-04 | 1382.8ms/step | 14.5 samples/s +Step 14720/44000 | Loss: 3.0467 | LR: 3.82e-04 | 1384.0ms/step | 14.5 samples/s +Step 14730/44000 | Loss: 3.0135 | LR: 3.82e-04 | 1383.8ms/step | 14.5 samples/s +Step 14740/44000 | Loss: 3.0235 | LR: 3.82e-04 | 1384.4ms/step | 14.4 samples/s +Step 14750/44000 | Loss: 3.0217 | LR: 3.81e-04 | 1385.0ms/step | 14.4 samples/s +Step 14760/44000 | Loss: 3.0297 | LR: 3.81e-04 | 1383.8ms/step | 14.5 samples/s +Step 14770/44000 | Loss: 3.0435 | LR: 3.81e-04 | 1385.4ms/step | 14.4 samples/s +Step 14780/44000 | Loss: 3.0377 | LR: 3.81e-04 | 1388.8ms/step | 14.4 samples/s +Step 14790/44000 | Loss: 3.0492 | LR: 3.81e-04 | 1383.7ms/step | 14.5 samples/s +Step 14800/44000 | Loss: 3.0522 | LR: 3.81e-04 | 1385.4ms/step | 14.4 samples/s +Step 14810/44000 | Loss: 3.0323 | LR: 3.81e-04 | 1385.8ms/step | 14.4 samples/s +Step 14820/44000 | Loss: 3.0518 | LR: 3.80e-04 | 1384.0ms/step | 14.5 samples/s +Step 14830/44000 | Loss: 3.0578 | LR: 3.80e-04 | 1385.3ms/step | 14.4 samples/s +Step 14840/44000 | Loss: 3.0104 | LR: 3.80e-04 | 1388.0ms/step | 14.4 samples/s +Step 14850/44000 | Loss: 3.0387 | LR: 3.80e-04 | 1384.8ms/step | 14.4 samples/s +Step 14860/44000 | Loss: 3.0327 | LR: 3.80e-04 | 1385.9ms/step | 14.4 samples/s +Step 14870/44000 | Loss: 3.0378 | LR: 3.80e-04 | 1388.7ms/step | 14.4 samples/s +Step 14880/44000 | Loss: 3.0459 | LR: 3.79e-04 | 1385.7ms/step | 14.4 samples/s +Step 14890/44000 | Loss: 3.0603 | LR: 3.79e-04 | 1383.2ms/step | 14.5 samples/s +Step 14900/44000 | Loss: 3.0527 | LR: 3.79e-04 | 1384.5ms/step | 14.4 samples/s +Step 14910/44000 | Loss: 3.0334 | LR: 3.79e-04 | 1387.1ms/step | 14.4 samples/s +Step 14920/44000 | Loss: 3.0379 | LR: 3.79e-04 | 1384.0ms/step | 14.5 samples/s +Step 14930/44000 | Loss: 3.0499 | LR: 3.79e-04 | 1383.6ms/step | 14.5 samples/s +Step 14940/44000 | Loss: 3.0472 | LR: 3.79e-04 | 1386.1ms/step | 14.4 samples/s +Step 14950/44000 | Loss: 3.0311 | LR: 3.78e-04 | 1382.3ms/step | 14.5 samples/s +Step 14960/44000 | Loss: 3.0361 | LR: 3.78e-04 | 1384.8ms/step | 14.4 samples/s +Step 14970/44000 | Loss: 3.0268 | LR: 3.78e-04 | 1384.9ms/step | 14.4 samples/s +Step 14980/44000 | Loss: 3.0461 | LR: 3.78e-04 | 1382.4ms/step | 14.5 samples/s +Step 14990/44000 | Loss: 3.0686 | LR: 3.78e-04 | 1382.4ms/step | 14.5 samples/s +Step 15000/44000 | Loss: 3.0209 | LR: 3.78e-04 | 1384.0ms/step | 14.5 samples/s +Validation at step 15000: Loss = 3.0394, Perplexity = 20.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-15000 +Step 15010/44000 | Loss: 3.0339 | LR: 3.77e-04 | 3917.6ms/step | 5.1 samples/s +Step 15020/44000 | Loss: 3.0625 | LR: 3.77e-04 | 1381.9ms/step | 14.5 samples/s +Step 15030/44000 | Loss: 3.0331 | LR: 3.77e-04 | 1378.7ms/step | 14.5 samples/s +Step 15040/44000 | Loss: 3.0055 | LR: 3.77e-04 | 1380.4ms/step | 14.5 samples/s +Step 15050/44000 | Loss: 3.0099 | LR: 3.77e-04 | 1380.6ms/step | 14.5 samples/s +Step 15060/44000 | Loss: 3.0344 | LR: 3.77e-04 | 1381.2ms/step | 14.5 samples/s +Step 15070/44000 | Loss: 3.0197 | LR: 3.77e-04 | 1381.9ms/step | 14.5 samples/s +Step 15080/44000 | Loss: 3.0658 | LR: 3.76e-04 | 1382.8ms/step | 14.5 samples/s +Step 15090/44000 | Loss: 3.0302 | LR: 3.76e-04 | 1383.3ms/step | 14.5 samples/s +Step 15100/44000 | Loss: 3.0547 | LR: 3.76e-04 | 1383.9ms/step | 14.5 samples/s +Step 15110/44000 | Loss: 3.0684 | LR: 3.76e-04 | 1382.7ms/step | 14.5 samples/s +Step 15120/44000 | Loss: 3.0258 | LR: 3.76e-04 | 1384.3ms/step | 14.4 samples/s +Step 15130/44000 | Loss: 3.0248 | LR: 3.76e-04 | 1385.1ms/step | 14.4 samples/s +Step 15140/44000 | Loss: 3.0363 | LR: 3.75e-04 | 1387.2ms/step | 14.4 samples/s +Step 15150/44000 | Loss: 3.0476 | LR: 3.75e-04 | 1384.4ms/step | 14.4 samples/s +Step 15160/44000 | Loss: 3.0110 | LR: 3.75e-04 | 1383.2ms/step | 14.5 samples/s +Step 15170/44000 | Loss: 3.0182 | LR: 3.75e-04 | 1383.7ms/step | 14.5 samples/s +Step 15180/44000 | Loss: 3.0260 | LR: 3.75e-04 | 1382.6ms/step | 14.5 samples/s +Step 15190/44000 | Loss: 3.0572 | LR: 3.75e-04 | 1384.5ms/step | 14.4 samples/s +Step 15200/44000 | Loss: 3.0297 | LR: 3.74e-04 | 1384.2ms/step | 14.4 samples/s +Step 15210/44000 | Loss: 3.0048 | LR: 3.74e-04 | 1384.0ms/step | 14.5 samples/s +Step 15220/44000 | Loss: 3.0268 | LR: 3.74e-04 | 1385.3ms/step | 14.4 samples/s +Step 15230/44000 | Loss: 3.0351 | LR: 3.74e-04 | 1385.4ms/step | 14.4 samples/s +Step 15240/44000 | Loss: 3.0413 | LR: 3.74e-04 | 1384.1ms/step | 14.4 samples/s +Step 15250/44000 | Loss: 3.0270 | LR: 3.74e-04 | 1384.8ms/step | 14.4 samples/s +Step 15260/44000 | Loss: 2.9857 | LR: 3.74e-04 | 1385.5ms/step | 14.4 samples/s +Step 15270/44000 | Loss: 3.0281 | LR: 3.73e-04 | 1383.4ms/step | 14.5 samples/s +Step 15280/44000 | Loss: 3.0324 | LR: 3.73e-04 | 1384.1ms/step | 14.4 samples/s +Step 15290/44000 | Loss: 3.0523 | LR: 3.73e-04 | 1387.1ms/step | 14.4 samples/s +Step 15300/44000 | Loss: 3.0513 | LR: 3.73e-04 | 1388.0ms/step | 14.4 samples/s +Step 15310/44000 | Loss: 3.0282 | LR: 3.73e-04 | 1387.5ms/step | 14.4 samples/s +Step 15320/44000 | Loss: 3.0610 | LR: 3.73e-04 | 1384.0ms/step | 14.5 samples/s +Step 15330/44000 | Loss: 3.0312 | LR: 3.72e-04 | 1385.6ms/step | 14.4 samples/s +Step 15340/44000 | Loss: 3.0255 | LR: 3.72e-04 | 1384.6ms/step | 14.4 samples/s +Step 15350/44000 | Loss: 3.0519 | LR: 3.72e-04 | 1384.7ms/step | 14.4 samples/s +Step 15360/44000 | Loss: 3.0079 | LR: 3.72e-04 | 1385.1ms/step | 14.4 samples/s +Step 15370/44000 | Loss: 3.0117 | LR: 3.72e-04 | 1387.6ms/step | 14.4 samples/s +Step 15380/44000 | Loss: 3.0427 | LR: 3.72e-04 | 1385.4ms/step | 14.4 samples/s +Step 15390/44000 | Loss: 3.0269 | LR: 3.71e-04 | 1384.9ms/step | 14.4 samples/s +Step 15400/44000 | Loss: 3.0333 | LR: 3.71e-04 | 1382.6ms/step | 14.5 samples/s +Step 15410/44000 | Loss: 3.0421 | LR: 3.71e-04 | 1383.0ms/step | 14.5 samples/s +Step 15420/44000 | Loss: 3.0108 | LR: 3.71e-04 | 1384.4ms/step | 14.4 samples/s +Step 15430/44000 | Loss: 3.0731 | LR: 3.71e-04 | 1385.8ms/step | 14.4 samples/s +Step 15440/44000 | Loss: 3.0163 | LR: 3.71e-04 | 1383.8ms/step | 14.5 samples/s +Step 15450/44000 | Loss: 3.0307 | LR: 3.71e-04 | 1387.6ms/step | 14.4 samples/s +Step 15460/44000 | Loss: 3.0323 | LR: 3.70e-04 | 1385.2ms/step | 14.4 samples/s +Step 15470/44000 | Loss: 3.0277 | LR: 3.70e-04 | 1386.1ms/step | 14.4 samples/s +Step 15480/44000 | Loss: 3.0237 | LR: 3.70e-04 | 1385.1ms/step | 14.4 samples/s +Step 15490/44000 | Loss: 3.0137 | LR: 3.70e-04 | 1385.3ms/step | 14.4 samples/s +Step 15500/44000 | Loss: 3.0038 | LR: 3.70e-04 | 1384.8ms/step | 14.4 samples/s +Step 15510/44000 | Loss: 3.0320 | LR: 3.70e-04 | 1386.4ms/step | 14.4 samples/s +Step 15520/44000 | Loss: 3.0344 | LR: 3.69e-04 | 1386.7ms/step | 14.4 samples/s +Step 15530/44000 | Loss: 3.0344 | LR: 3.69e-04 | 1385.9ms/step | 14.4 samples/s +Step 15540/44000 | Loss: 3.0105 | LR: 3.69e-04 | 1384.8ms/step | 14.4 samples/s +Step 15550/44000 | Loss: 3.0442 | LR: 3.69e-04 | 1386.5ms/step | 14.4 samples/s +Step 15560/44000 | Loss: 3.0318 | LR: 3.69e-04 | 1384.8ms/step | 14.4 samples/s +Step 15570/44000 | Loss: 3.0118 | LR: 3.69e-04 | 1385.8ms/step | 14.4 samples/s +Step 15580/44000 | Loss: 3.0386 | LR: 3.68e-04 | 1386.3ms/step | 14.4 samples/s +Step 15590/44000 | Loss: 2.9988 | LR: 3.68e-04 | 1385.2ms/step | 14.4 samples/s +Step 15600/44000 | Loss: 3.0204 | LR: 3.68e-04 | 1386.0ms/step | 14.4 samples/s +Step 15610/44000 | Loss: 3.0008 | LR: 3.68e-04 | 1387.0ms/step | 14.4 samples/s +Step 15620/44000 | Loss: 3.0103 | LR: 3.68e-04 | 1385.4ms/step | 14.4 samples/s +Step 15630/44000 | Loss: 3.0302 | LR: 3.68e-04 | 1386.4ms/step | 14.4 samples/s +Step 15640/44000 | Loss: 3.0298 | LR: 3.67e-04 | 1386.8ms/step | 14.4 samples/s +Step 15650/44000 | Loss: 3.0513 | LR: 3.67e-04 | 1385.2ms/step | 14.4 samples/s +Step 15660/44000 | Loss: 3.0410 | LR: 3.67e-04 | 1386.8ms/step | 14.4 samples/s +Step 15670/44000 | Loss: 3.0181 | LR: 3.67e-04 | 1385.4ms/step | 14.4 samples/s +Step 15680/44000 | Loss: 3.0058 | LR: 3.67e-04 | 1387.5ms/step | 14.4 samples/s +Step 15690/44000 | Loss: 3.0076 | LR: 3.67e-04 | 1386.5ms/step | 14.4 samples/s +Step 15700/44000 | Loss: 3.0186 | LR: 3.67e-04 | 1385.2ms/step | 14.4 samples/s +Step 15710/44000 | Loss: 3.0103 | LR: 3.66e-04 | 1384.5ms/step | 14.4 samples/s +Step 15720/44000 | Loss: 3.0278 | LR: 3.66e-04 | 1385.5ms/step | 14.4 samples/s +Step 15730/44000 | Loss: 3.0192 | LR: 3.66e-04 | 1386.3ms/step | 14.4 samples/s +Step 15740/44000 | Loss: 3.0188 | LR: 3.66e-04 | 1384.6ms/step | 14.4 samples/s +Step 15750/44000 | Loss: 3.0381 | LR: 3.66e-04 | 1383.5ms/step | 14.5 samples/s +Step 15760/44000 | Loss: 3.0392 | LR: 3.66e-04 | 1384.4ms/step | 14.4 samples/s +Step 15770/44000 | Loss: 3.0306 | LR: 3.65e-04 | 1383.2ms/step | 14.5 samples/s +Step 15780/44000 | Loss: 3.0264 | LR: 3.65e-04 | 1384.6ms/step | 14.4 samples/s +Step 15790/44000 | Loss: 3.0023 | LR: 3.65e-04 | 1383.5ms/step | 14.5 samples/s +Step 15800/44000 | Loss: 3.0266 | LR: 3.65e-04 | 1384.8ms/step | 14.4 samples/s +Step 15810/44000 | Loss: 3.0322 | LR: 3.65e-04 | 1384.9ms/step | 14.4 samples/s +Step 15820/44000 | Loss: 3.0017 | LR: 3.65e-04 | 1385.5ms/step | 14.4 samples/s +Step 15830/44000 | Loss: 3.0294 | LR: 3.64e-04 | 1382.9ms/step | 14.5 samples/s +Step 15840/44000 | Loss: 3.0042 | LR: 3.64e-04 | 1384.3ms/step | 14.4 samples/s +Step 15850/44000 | Loss: 3.0312 | LR: 3.64e-04 | 1387.2ms/step | 14.4 samples/s +Step 15860/44000 | Loss: 3.0034 | LR: 3.64e-04 | 1383.6ms/step | 14.5 samples/s +Step 15870/44000 | Loss: 3.0205 | LR: 3.64e-04 | 1385.2ms/step | 14.4 samples/s +Step 15880/44000 | Loss: 3.0478 | LR: 3.64e-04 | 1384.7ms/step | 14.4 samples/s +Step 15890/44000 | Loss: 3.0021 | LR: 3.63e-04 | 1383.1ms/step | 14.5 samples/s +Step 15900/44000 | Loss: 3.0343 | LR: 3.63e-04 | 1385.2ms/step | 14.4 samples/s +Step 15910/44000 | Loss: 2.9988 | LR: 3.63e-04 | 1384.2ms/step | 14.4 samples/s +Step 15920/44000 | Loss: 2.9947 | LR: 3.63e-04 | 1384.3ms/step | 14.4 samples/s +Step 15930/44000 | Loss: 3.0073 | LR: 3.63e-04 | 1384.4ms/step | 14.4 samples/s +Step 15940/44000 | Loss: 3.0401 | LR: 3.63e-04 | 1383.6ms/step | 14.5 samples/s +Step 15950/44000 | Loss: 3.0311 | LR: 3.62e-04 | 1383.2ms/step | 14.5 samples/s +Step 15960/44000 | Loss: 3.0153 | LR: 3.62e-04 | 1383.6ms/step | 14.5 samples/s +Step 15970/44000 | Loss: 3.0067 | LR: 3.62e-04 | 1384.7ms/step | 14.4 samples/s +Step 15980/44000 | Loss: 3.0466 | LR: 3.62e-04 | 1386.7ms/step | 14.4 samples/s +Step 15990/44000 | Loss: 3.0005 | LR: 3.62e-04 | 1386.6ms/step | 14.4 samples/s +Step 16000/44000 | Loss: 3.0193 | LR: 3.62e-04 | 1385.5ms/step | 14.4 samples/s +Validation at step 16000: Loss = 3.0266, Perplexity = 20.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-16000 +Step 16010/44000 | Loss: 3.0097 | LR: 3.62e-04 | 3895.0ms/step | 5.1 samples/s +Step 16020/44000 | Loss: 3.0518 | LR: 3.61e-04 | 1379.8ms/step | 14.5 samples/s +Step 16030/44000 | Loss: 3.0150 | LR: 3.61e-04 | 1380.5ms/step | 14.5 samples/s +Step 16040/44000 | Loss: 3.0480 | LR: 3.61e-04 | 1381.2ms/step | 14.5 samples/s +Step 16050/44000 | Loss: 2.9980 | LR: 3.61e-04 | 1381.9ms/step | 14.5 samples/s +Step 16060/44000 | Loss: 3.0403 | LR: 3.61e-04 | 1382.0ms/step | 14.5 samples/s +Step 16070/44000 | Loss: 3.0418 | LR: 3.61e-04 | 1379.8ms/step | 14.5 samples/s +Step 16080/44000 | Loss: 3.0072 | LR: 3.60e-04 | 1379.8ms/step | 14.5 samples/s +Step 16090/44000 | Loss: 3.0479 | LR: 3.60e-04 | 1380.1ms/step | 14.5 samples/s +Step 16100/44000 | Loss: 3.0067 | LR: 3.60e-04 | 1378.8ms/step | 14.5 samples/s +Step 16110/44000 | Loss: 3.0202 | LR: 3.60e-04 | 1381.4ms/step | 14.5 samples/s +Step 16120/44000 | Loss: 2.9743 | LR: 3.60e-04 | 1381.9ms/step | 14.5 samples/s +Step 16130/44000 | Loss: 3.0282 | LR: 3.60e-04 | 1382.7ms/step | 14.5 samples/s +Step 16140/44000 | Loss: 3.0275 | LR: 3.59e-04 | 1382.2ms/step | 14.5 samples/s +Step 16150/44000 | Loss: 2.9930 | LR: 3.59e-04 | 1383.8ms/step | 14.5 samples/s +Step 16160/44000 | Loss: 2.9986 | LR: 3.59e-04 | 1382.8ms/step | 14.5 samples/s +Step 16170/44000 | Loss: 2.9994 | LR: 3.59e-04 | 1383.0ms/step | 14.5 samples/s +Step 16180/44000 | Loss: 3.0058 | LR: 3.59e-04 | 1381.9ms/step | 14.5 samples/s +Step 16190/44000 | Loss: 3.0110 | LR: 3.59e-04 | 1382.4ms/step | 14.5 samples/s +Step 16200/44000 | Loss: 3.0368 | LR: 3.58e-04 | 1382.9ms/step | 14.5 samples/s +Step 16210/44000 | Loss: 3.0315 | LR: 3.58e-04 | 1383.2ms/step | 14.5 samples/s +Step 16220/44000 | Loss: 2.9890 | LR: 3.58e-04 | 1381.7ms/step | 14.5 samples/s +Step 16230/44000 | Loss: 3.0031 | LR: 3.58e-04 | 1383.6ms/step | 14.5 samples/s +Step 16240/44000 | Loss: 3.0225 | LR: 3.58e-04 | 1382.5ms/step | 14.5 samples/s +Step 16250/44000 | Loss: 3.0390 | LR: 3.58e-04 | 1384.8ms/step | 14.4 samples/s +Step 16260/44000 | Loss: 3.0144 | LR: 3.57e-04 | 1382.8ms/step | 14.5 samples/s +Step 16270/44000 | Loss: 2.9970 | LR: 3.57e-04 | 1384.3ms/step | 14.4 samples/s +Step 16280/44000 | Loss: 3.0295 | LR: 3.57e-04 | 1385.6ms/step | 14.4 samples/s +Step 16290/44000 | Loss: 3.0052 | LR: 3.57e-04 | 1383.5ms/step | 14.5 samples/s +Step 16300/44000 | Loss: 3.0128 | LR: 3.57e-04 | 1381.1ms/step | 14.5 samples/s +Step 16310/44000 | Loss: 3.0132 | LR: 3.57e-04 | 1382.0ms/step | 14.5 samples/s +Step 16320/44000 | Loss: 3.0091 | LR: 3.56e-04 | 1383.8ms/step | 14.5 samples/s +Step 16330/44000 | Loss: 3.0360 | LR: 3.56e-04 | 1381.8ms/step | 14.5 samples/s +Step 16340/44000 | Loss: 2.9995 | LR: 3.56e-04 | 1385.2ms/step | 14.4 samples/s +Step 16350/44000 | Loss: 3.0521 | LR: 3.56e-04 | 1384.5ms/step | 14.4 samples/s +Step 16360/44000 | Loss: 2.9818 | LR: 3.56e-04 | 1384.4ms/step | 14.4 samples/s +Step 16370/44000 | Loss: 3.0304 | LR: 3.56e-04 | 1385.3ms/step | 14.4 samples/s +Step 16380/44000 | Loss: 3.0303 | LR: 3.55e-04 | 1382.0ms/step | 14.5 samples/s +Step 16390/44000 | Loss: 3.0041 | LR: 3.55e-04 | 1385.5ms/step | 14.4 samples/s +Step 16400/44000 | Loss: 2.9996 | LR: 3.55e-04 | 1382.7ms/step | 14.5 samples/s +Step 16410/44000 | Loss: 2.9975 | LR: 3.55e-04 | 1382.9ms/step | 14.5 samples/s +Step 16420/44000 | Loss: 2.9944 | LR: 3.55e-04 | 1383.2ms/step | 14.5 samples/s +Step 16430/44000 | Loss: 3.0136 | LR: 3.55e-04 | 1383.7ms/step | 14.5 samples/s +Step 16440/44000 | Loss: 3.0332 | LR: 3.54e-04 | 1382.2ms/step | 14.5 samples/s +Step 16450/44000 | Loss: 2.9992 | LR: 3.54e-04 | 1382.3ms/step | 14.5 samples/s +Step 16460/44000 | Loss: 3.0007 | LR: 3.54e-04 | 1381.6ms/step | 14.5 samples/s +Step 16470/44000 | Loss: 3.0298 | LR: 3.54e-04 | 1384.0ms/step | 14.5 samples/s +Step 16480/44000 | Loss: 2.9728 | LR: 3.54e-04 | 1383.8ms/step | 14.5 samples/s +Step 16490/44000 | Loss: 3.0055 | LR: 3.54e-04 | 1383.4ms/step | 14.5 samples/s +Step 16500/44000 | Loss: 3.0486 | LR: 3.53e-04 | 1383.6ms/step | 14.5 samples/s +Step 16510/44000 | Loss: 2.9800 | LR: 3.53e-04 | 1383.6ms/step | 14.5 samples/s +Step 16520/44000 | Loss: 3.0139 | LR: 3.53e-04 | 1380.3ms/step | 14.5 samples/s +Step 16530/44000 | Loss: 3.0321 | LR: 3.53e-04 | 1383.2ms/step | 14.5 samples/s +Step 16540/44000 | Loss: 3.0359 | LR: 3.53e-04 | 1385.6ms/step | 14.4 samples/s +Step 16550/44000 | Loss: 3.0076 | LR: 3.53e-04 | 1382.2ms/step | 14.5 samples/s +Step 16560/44000 | Loss: 3.0044 | LR: 3.52e-04 | 1382.8ms/step | 14.5 samples/s +Step 16570/44000 | Loss: 3.0331 | LR: 3.52e-04 | 1383.6ms/step | 14.5 samples/s +Step 16580/44000 | Loss: 3.0068 | LR: 3.52e-04 | 1383.4ms/step | 14.5 samples/s +Step 16590/44000 | Loss: 2.9892 | LR: 3.52e-04 | 1380.8ms/step | 14.5 samples/s +Step 16600/44000 | Loss: 2.9902 | LR: 3.52e-04 | 1385.8ms/step | 14.4 samples/s +Step 16610/44000 | Loss: 2.9925 | LR: 3.52e-04 | 1383.0ms/step | 14.5 samples/s +Step 16620/44000 | Loss: 3.0049 | LR: 3.51e-04 | 1384.8ms/step | 14.4 samples/s +Step 16630/44000 | Loss: 2.9694 | LR: 3.51e-04 | 1381.8ms/step | 14.5 samples/s +Step 16640/44000 | Loss: 2.9884 | LR: 3.51e-04 | 1384.6ms/step | 14.4 samples/s +Step 16650/44000 | Loss: 3.0005 | LR: 3.51e-04 | 1383.6ms/step | 14.5 samples/s +Step 16660/44000 | Loss: 3.0015 | LR: 3.51e-04 | 1382.6ms/step | 14.5 samples/s +Step 16670/44000 | Loss: 3.0329 | LR: 3.51e-04 | 1383.8ms/step | 14.5 samples/s +Step 16680/44000 | Loss: 3.0019 | LR: 3.50e-04 | 1382.3ms/step | 14.5 samples/s +Step 16690/44000 | Loss: 3.0203 | LR: 3.50e-04 | 1382.4ms/step | 14.5 samples/s +Step 16700/44000 | Loss: 2.9827 | LR: 3.50e-04 | 1382.8ms/step | 14.5 samples/s +Step 16710/44000 | Loss: 3.0042 | LR: 3.50e-04 | 1382.6ms/step | 14.5 samples/s +Step 16720/44000 | Loss: 3.0251 | LR: 3.50e-04 | 1381.6ms/step | 14.5 samples/s +Step 16730/44000 | Loss: 3.0141 | LR: 3.50e-04 | 1384.9ms/step | 14.4 samples/s +Step 16740/44000 | Loss: 3.0411 | LR: 3.49e-04 | 1382.9ms/step | 14.5 samples/s +Step 16750/44000 | Loss: 3.0324 | LR: 3.49e-04 | 1385.6ms/step | 14.4 samples/s +Step 16760/44000 | Loss: 2.9866 | LR: 3.49e-04 | 1383.3ms/step | 14.5 samples/s +Step 16770/44000 | Loss: 3.0003 | LR: 3.49e-04 | 1383.6ms/step | 14.5 samples/s +Step 16780/44000 | Loss: 3.0134 | LR: 3.49e-04 | 1386.1ms/step | 14.4 samples/s +Step 16790/44000 | Loss: 3.0487 | LR: 3.49e-04 | 1385.9ms/step | 14.4 samples/s +Step 16800/44000 | Loss: 2.9927 | LR: 3.48e-04 | 1384.0ms/step | 14.5 samples/s +Step 16810/44000 | Loss: 3.0002 | LR: 3.48e-04 | 1382.5ms/step | 14.5 samples/s +Step 16820/44000 | Loss: 3.0196 | LR: 3.48e-04 | 1386.5ms/step | 14.4 samples/s +Step 16830/44000 | Loss: 3.0025 | LR: 3.48e-04 | 1384.0ms/step | 14.5 samples/s +Step 16840/44000 | Loss: 3.0079 | LR: 3.48e-04 | 1383.4ms/step | 14.5 samples/s +Step 16850/44000 | Loss: 2.9754 | LR: 3.48e-04 | 1386.6ms/step | 14.4 samples/s +Step 16860/44000 | Loss: 3.0265 | LR: 3.47e-04 | 1384.1ms/step | 14.5 samples/s +Step 16870/44000 | Loss: 2.9931 | LR: 3.47e-04 | 1383.1ms/step | 14.5 samples/s +Step 16880/44000 | Loss: 2.9883 | LR: 3.47e-04 | 1385.4ms/step | 14.4 samples/s +Step 16890/44000 | Loss: 3.0091 | LR: 3.47e-04 | 1384.9ms/step | 14.4 samples/s +Step 16900/44000 | Loss: 3.0087 | LR: 3.47e-04 | 1385.6ms/step | 14.4 samples/s +Step 16910/44000 | Loss: 3.0072 | LR: 3.47e-04 | 1384.5ms/step | 14.4 samples/s +Step 16920/44000 | Loss: 3.0145 | LR: 3.46e-04 | 1383.8ms/step | 14.5 samples/s +Step 16930/44000 | Loss: 2.9838 | LR: 3.46e-04 | 1384.8ms/step | 14.4 samples/s +Step 16940/44000 | Loss: 2.9971 | LR: 3.46e-04 | 1385.1ms/step | 14.4 samples/s +Step 16950/44000 | Loss: 2.9736 | LR: 3.46e-04 | 1385.7ms/step | 14.4 samples/s +Step 16960/44000 | Loss: 2.9987 | LR: 3.46e-04 | 1387.3ms/step | 14.4 samples/s +Step 16970/44000 | Loss: 2.9549 | LR: 3.46e-04 | 1387.4ms/step | 14.4 samples/s +Step 16980/44000 | Loss: 3.0334 | LR: 3.45e-04 | 1387.1ms/step | 14.4 samples/s +Step 16990/44000 | Loss: 3.0090 | LR: 3.45e-04 | 1386.3ms/step | 14.4 samples/s +Step 17000/44000 | Loss: 3.0116 | LR: 3.45e-04 | 1384.1ms/step | 14.4 samples/s +Validation at step 17000: Loss = 3.0135, Perplexity = 20.36 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-17000 +Step 17010/44000 | Loss: 3.0070 | LR: 3.45e-04 | 3948.1ms/step | 5.1 samples/s +Step 17020/44000 | Loss: 2.9870 | LR: 3.45e-04 | 1378.0ms/step | 14.5 samples/s +Step 17030/44000 | Loss: 3.0102 | LR: 3.45e-04 | 1378.9ms/step | 14.5 samples/s +Step 17040/44000 | Loss: 3.0028 | LR: 3.44e-04 | 1378.5ms/step | 14.5 samples/s +Step 17050/44000 | Loss: 2.9953 | LR: 3.44e-04 | 1380.3ms/step | 14.5 samples/s +Step 17060/44000 | Loss: 2.9630 | LR: 3.44e-04 | 1379.8ms/step | 14.5 samples/s +Step 17070/44000 | Loss: 3.0029 | LR: 3.44e-04 | 1382.4ms/step | 14.5 samples/s +Step 17080/44000 | Loss: 2.9980 | LR: 3.44e-04 | 1381.6ms/step | 14.5 samples/s +Step 17090/44000 | Loss: 3.0183 | LR: 3.44e-04 | 1382.4ms/step | 14.5 samples/s +Step 17100/44000 | Loss: 3.0024 | LR: 3.43e-04 | 1385.2ms/step | 14.4 samples/s +Step 17110/44000 | Loss: 2.9932 | LR: 3.43e-04 | 1380.4ms/step | 14.5 samples/s +Step 17120/44000 | Loss: 3.0063 | LR: 3.43e-04 | 1378.4ms/step | 14.5 samples/s +Step 17130/44000 | Loss: 3.0088 | LR: 3.43e-04 | 1376.5ms/step | 14.5 samples/s +Step 17140/44000 | Loss: 2.9916 | LR: 3.43e-04 | 1374.7ms/step | 14.5 samples/s +Step 17150/44000 | Loss: 3.0089 | LR: 3.43e-04 | 1375.9ms/step | 14.5 samples/s +Step 17160/44000 | Loss: 3.0026 | LR: 3.42e-04 | 1374.2ms/step | 14.6 samples/s +Step 17170/44000 | Loss: 3.0051 | LR: 3.42e-04 | 1377.0ms/step | 14.5 samples/s +Step 17180/44000 | Loss: 3.0175 | LR: 3.42e-04 | 1376.1ms/step | 14.5 samples/s +Step 17190/44000 | Loss: 3.0205 | LR: 3.42e-04 | 1376.5ms/step | 14.5 samples/s +Step 17200/44000 | Loss: 2.9861 | LR: 3.42e-04 | 1377.6ms/step | 14.5 samples/s +Step 17210/44000 | Loss: 3.0223 | LR: 3.42e-04 | 1376.6ms/step | 14.5 samples/s +Step 17220/44000 | Loss: 3.0380 | LR: 3.41e-04 | 1376.9ms/step | 14.5 samples/s +Step 17230/44000 | Loss: 3.0040 | LR: 3.41e-04 | 1376.9ms/step | 14.5 samples/s +Step 17240/44000 | Loss: 3.0171 | LR: 3.41e-04 | 1376.8ms/step | 14.5 samples/s +Step 17250/44000 | Loss: 3.0174 | LR: 3.41e-04 | 1376.8ms/step | 14.5 samples/s +Step 17260/44000 | Loss: 3.0344 | LR: 3.41e-04 | 1376.1ms/step | 14.5 samples/s +Step 17270/44000 | Loss: 3.0230 | LR: 3.41e-04 | 1374.9ms/step | 14.5 samples/s +Step 17280/44000 | Loss: 2.9788 | LR: 3.40e-04 | 1376.9ms/step | 14.5 samples/s +Step 17290/44000 | Loss: 2.9969 | LR: 3.40e-04 | 1376.4ms/step | 14.5 samples/s +Step 17300/44000 | Loss: 3.0321 | LR: 3.40e-04 | 1378.2ms/step | 14.5 samples/s +Step 17310/44000 | Loss: 2.9923 | LR: 3.40e-04 | 1377.3ms/step | 14.5 samples/s +Step 17320/44000 | Loss: 3.0062 | LR: 3.40e-04 | 1374.5ms/step | 14.6 samples/s +Step 17330/44000 | Loss: 2.9933 | LR: 3.40e-04 | 1376.8ms/step | 14.5 samples/s +Step 17340/44000 | Loss: 3.0236 | LR: 3.39e-04 | 1376.7ms/step | 14.5 samples/s +Step 17350/44000 | Loss: 3.0148 | LR: 3.39e-04 | 1377.2ms/step | 14.5 samples/s +Step 17360/44000 | Loss: 2.9887 | LR: 3.39e-04 | 1380.1ms/step | 14.5 samples/s +Step 17370/44000 | Loss: 3.0001 | LR: 3.39e-04 | 1379.5ms/step | 14.5 samples/s +Step 17380/44000 | Loss: 2.9878 | LR: 3.39e-04 | 1380.4ms/step | 14.5 samples/s +Step 17390/44000 | Loss: 3.0254 | LR: 3.39e-04 | 1380.3ms/step | 14.5 samples/s +Step 17400/44000 | Loss: 2.9928 | LR: 3.38e-04 | 1377.3ms/step | 14.5 samples/s +Step 17410/44000 | Loss: 2.9962 | LR: 3.38e-04 | 1374.9ms/step | 14.5 samples/s +Step 17420/44000 | Loss: 2.9696 | LR: 3.38e-04 | 1377.0ms/step | 14.5 samples/s +Step 17430/44000 | Loss: 2.9984 | LR: 3.38e-04 | 1377.8ms/step | 14.5 samples/s +Step 17440/44000 | Loss: 3.0245 | LR: 3.38e-04 | 1378.0ms/step | 14.5 samples/s +Step 17450/44000 | Loss: 3.0262 | LR: 3.38e-04 | 1376.4ms/step | 14.5 samples/s +Step 17460/44000 | Loss: 2.9876 | LR: 3.37e-04 | 1375.7ms/step | 14.5 samples/s +Step 17470/44000 | Loss: 2.9730 | LR: 3.37e-04 | 1376.5ms/step | 14.5 samples/s +Step 17480/44000 | Loss: 3.0102 | LR: 3.37e-04 | 1377.3ms/step | 14.5 samples/s +Step 17490/44000 | Loss: 2.9959 | LR: 3.37e-04 | 1376.7ms/step | 14.5 samples/s +Step 17500/44000 | Loss: 3.0135 | LR: 3.37e-04 | 1377.7ms/step | 14.5 samples/s +Step 17510/44000 | Loss: 2.9908 | LR: 3.37e-04 | 1377.6ms/step | 14.5 samples/s +Step 17520/44000 | Loss: 3.0079 | LR: 3.36e-04 | 1376.3ms/step | 14.5 samples/s +Step 17530/44000 | Loss: 3.0292 | LR: 3.36e-04 | 1377.5ms/step | 14.5 samples/s diff --git a/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed45.20260820T041452Z.log b/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed45.20260820T041452Z.log new file mode 100644 index 0000000000000000000000000000000000000000..af82135fdd5e1ce387d5d2dbfed67ef368c26323 --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed45.20260820T041452Z.log @@ -0,0 +1,656 @@ +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-13000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-13000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 45, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed45", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-13000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-13000... +Preserving checkpoint LR state for exact resume: current=[0.0004073888929652048], schedule_base=[0.0005] +Resumed from checkpoint at step 13000 (epoch 0, micro_step 130000) +Resuming training from step 13000 +Resuming wandb run xfnxdn24 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run xfnxdn24 +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-xfnxdn24 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed45 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/xfnxdn24 +Starting training... +Validation at step 13000: Loss = 3.0601, Perplexity = 21.33 +wandb: WARNING Tried to log to step 13000 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 13001: dense (dense step interval) +Step 13010/44000 | Loss: 3.0530 | LR: 4.07e-04 | 30157.5ms/step | 0.7 samples/s +wandb: WARNING Tried to log to step 13010 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13020/44000 | Loss: 3.0363 | LR: 4.07e-04 | 1397.9ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13020 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13030/44000 | Loss: 3.0282 | LR: 4.07e-04 | 1395.5ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13030 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13040/44000 | Loss: 3.0404 | LR: 4.07e-04 | 1398.6ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13040 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13050/44000 | Loss: 3.0329 | LR: 4.07e-04 | 1400.2ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13050 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13060/44000 | Loss: 3.0708 | LR: 4.07e-04 | 1400.0ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13060 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13070/44000 | Loss: 3.0480 | LR: 4.06e-04 | 1399.8ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13070 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13080/44000 | Loss: 3.0513 | LR: 4.06e-04 | 1400.2ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13080 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13090/44000 | Loss: 3.0571 | LR: 4.06e-04 | 1399.9ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13090 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13100/44000 | Loss: 3.0298 | LR: 4.06e-04 | 1400.0ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13100 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13110/44000 | Loss: 3.0451 | LR: 4.06e-04 | 1400.1ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13110 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13120/44000 | Loss: 3.0519 | LR: 4.06e-04 | 1402.0ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13120 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13130/44000 | Loss: 3.0832 | LR: 4.06e-04 | 1400.0ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13130 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13140/44000 | Loss: 3.0502 | LR: 4.05e-04 | 1400.3ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13140 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13150/44000 | Loss: 3.0396 | LR: 4.05e-04 | 1401.6ms/step | 14.3 samples/s +wandb: WARNING Tried to log to step 13150 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13160/44000 | Loss: 3.0522 | LR: 4.05e-04 | 1400.8ms/step | 14.3 samples/s +Step 13170/44000 | Loss: 3.0706 | LR: 4.05e-04 | 1399.2ms/step | 14.3 samples/s +Step 13180/44000 | Loss: 3.0776 | LR: 4.05e-04 | 1400.2ms/step | 14.3 samples/s +Step 13190/44000 | Loss: 3.0760 | LR: 4.05e-04 | 1401.8ms/step | 14.3 samples/s +Step 13200/44000 | Loss: 3.0560 | LR: 4.05e-04 | 1402.3ms/step | 14.3 samples/s +Step 13210/44000 | Loss: 3.0916 | LR: 4.04e-04 | 1400.5ms/step | 14.3 samples/s +Step 13220/44000 | Loss: 3.0245 | LR: 4.04e-04 | 1399.6ms/step | 14.3 samples/s +Step 13230/44000 | Loss: 3.1008 | LR: 4.04e-04 | 1401.6ms/step | 14.3 samples/s +Step 13240/44000 | Loss: 3.0804 | LR: 4.04e-04 | 1402.0ms/step | 14.3 samples/s +Step 13250/44000 | Loss: 3.0645 | LR: 4.04e-04 | 1400.1ms/step | 14.3 samples/s +Step 13260/44000 | Loss: 3.0688 | LR: 4.04e-04 | 1400.1ms/step | 14.3 samples/s +Step 13270/44000 | Loss: 3.0776 | LR: 4.04e-04 | 1399.8ms/step | 14.3 samples/s +Step 13280/44000 | Loss: 3.0387 | LR: 4.03e-04 | 1400.2ms/step | 14.3 samples/s +Step 13290/44000 | Loss: 3.0463 | LR: 4.03e-04 | 1402.2ms/step | 14.3 samples/s +Step 13300/44000 | Loss: 3.0579 | LR: 4.03e-04 | 1402.0ms/step | 14.3 samples/s +Step 13310/44000 | Loss: 3.0696 | LR: 4.03e-04 | 1399.9ms/step | 14.3 samples/s +Step 13320/44000 | Loss: 3.0288 | LR: 4.03e-04 | 1401.8ms/step | 14.3 samples/s +Step 13330/44000 | Loss: 3.0232 | LR: 4.03e-04 | 1402.2ms/step | 14.3 samples/s +Step 13340/44000 | Loss: 3.0448 | LR: 4.03e-04 | 1400.1ms/step | 14.3 samples/s +Step 13350/44000 | Loss: 3.0312 | LR: 4.02e-04 | 1404.5ms/step | 14.2 samples/s +Step 13360/44000 | Loss: 3.0378 | LR: 4.02e-04 | 1405.4ms/step | 14.2 samples/s +Step 13370/44000 | Loss: 3.0470 | LR: 4.02e-04 | 1402.0ms/step | 14.3 samples/s +Step 13380/44000 | Loss: 3.0633 | LR: 4.02e-04 | 1403.9ms/step | 14.2 samples/s +Step 13390/44000 | Loss: 3.0842 | LR: 4.02e-04 | 1404.6ms/step | 14.2 samples/s +Step 13400/44000 | Loss: 3.0427 | LR: 4.02e-04 | 1403.9ms/step | 14.2 samples/s +Step 13410/44000 | Loss: 3.0296 | LR: 4.02e-04 | 1399.2ms/step | 14.3 samples/s +Step 13420/44000 | Loss: 3.0456 | LR: 4.01e-04 | 1401.5ms/step | 14.3 samples/s +Step 13430/44000 | Loss: 3.0520 | LR: 4.01e-04 | 1404.5ms/step | 14.2 samples/s +Step 13440/44000 | Loss: 3.0475 | LR: 4.01e-04 | 1400.1ms/step | 14.3 samples/s +Step 13450/44000 | Loss: 3.0465 | LR: 4.01e-04 | 1400.2ms/step | 14.3 samples/s +Step 13460/44000 | Loss: 3.0584 | LR: 4.01e-04 | 1400.5ms/step | 14.3 samples/s +Step 13470/44000 | Loss: 3.0524 | LR: 4.01e-04 | 1399.5ms/step | 14.3 samples/s +Step 13480/44000 | Loss: 3.0477 | LR: 4.01e-04 | 1399.7ms/step | 14.3 samples/s +Step 13490/44000 | Loss: 3.0477 | LR: 4.00e-04 | 1400.2ms/step | 14.3 samples/s +Step 13500/44000 | Loss: 3.0386 | LR: 4.00e-04 | 1399.9ms/step | 14.3 samples/s +Step 13510/44000 | Loss: 3.0418 | LR: 4.00e-04 | 1402.4ms/step | 14.3 samples/s +Step 13520/44000 | Loss: 3.0534 | LR: 4.00e-04 | 1399.8ms/step | 14.3 samples/s +Step 13530/44000 | Loss: 3.0395 | LR: 4.00e-04 | 1399.7ms/step | 14.3 samples/s +Step 13540/44000 | Loss: 3.0473 | LR: 4.00e-04 | 1400.3ms/step | 14.3 samples/s +Step 13550/44000 | Loss: 3.0782 | LR: 4.00e-04 | 1401.6ms/step | 14.3 samples/s +Step 13560/44000 | Loss: 3.0321 | LR: 3.99e-04 | 1400.3ms/step | 14.3 samples/s +Step 13570/44000 | Loss: 3.0586 | LR: 3.99e-04 | 1399.8ms/step | 14.3 samples/s +Step 13580/44000 | Loss: 3.0491 | LR: 3.99e-04 | 1400.7ms/step | 14.3 samples/s +Step 13590/44000 | Loss: 3.0513 | LR: 3.99e-04 | 1401.3ms/step | 14.3 samples/s +Step 13600/44000 | Loss: 3.0597 | LR: 3.99e-04 | 1400.1ms/step | 14.3 samples/s +Step 13610/44000 | Loss: 3.0593 | LR: 3.99e-04 | 1399.8ms/step | 14.3 samples/s +Step 13620/44000 | Loss: 3.0857 | LR: 3.98e-04 | 1402.3ms/step | 14.3 samples/s +Step 13630/44000 | Loss: 3.0691 | LR: 3.98e-04 | 1400.3ms/step | 14.3 samples/s +Step 13640/44000 | Loss: 3.0180 | LR: 3.98e-04 | 1401.5ms/step | 14.3 samples/s +Step 13650/44000 | Loss: 3.0189 | LR: 3.98e-04 | 1400.1ms/step | 14.3 samples/s +Step 13660/44000 | Loss: 3.0850 | LR: 3.98e-04 | 1404.3ms/step | 14.2 samples/s +Step 13670/44000 | Loss: 3.0420 | LR: 3.98e-04 | 1399.5ms/step | 14.3 samples/s +Step 13680/44000 | Loss: 3.0472 | LR: 3.98e-04 | 1400.0ms/step | 14.3 samples/s +Step 13690/44000 | Loss: 3.0447 | LR: 3.97e-04 | 1400.2ms/step | 14.3 samples/s +Step 13700/44000 | Loss: 3.0604 | LR: 3.97e-04 | 1404.4ms/step | 14.2 samples/s +Step 13710/44000 | Loss: 3.0303 | LR: 3.97e-04 | 1401.4ms/step | 14.3 samples/s +Step 13720/44000 | Loss: 3.0465 | LR: 3.97e-04 | 1400.6ms/step | 14.3 samples/s +Step 13730/44000 | Loss: 3.0249 | LR: 3.97e-04 | 1401.8ms/step | 14.3 samples/s +Step 13740/44000 | Loss: 3.0263 | LR: 3.97e-04 | 1403.8ms/step | 14.2 samples/s +Step 13750/44000 | Loss: 3.0443 | LR: 3.97e-04 | 1401.8ms/step | 14.3 samples/s +Step 13760/44000 | Loss: 3.0645 | LR: 3.96e-04 | 1402.2ms/step | 14.3 samples/s +Step 13770/44000 | Loss: 3.0402 | LR: 3.96e-04 | 1402.5ms/step | 14.3 samples/s +Step 13780/44000 | Loss: 3.0526 | LR: 3.96e-04 | 1403.6ms/step | 14.2 samples/s +Step 13790/44000 | Loss: 3.0623 | LR: 3.96e-04 | 1401.8ms/step | 14.3 samples/s +Step 13800/44000 | Loss: 3.0307 | LR: 3.96e-04 | 1400.5ms/step | 14.3 samples/s +Step 13810/44000 | Loss: 3.0090 | LR: 3.96e-04 | 1401.4ms/step | 14.3 samples/s +Step 13820/44000 | Loss: 3.0181 | LR: 3.96e-04 | 1399.7ms/step | 14.3 samples/s +Step 13830/44000 | Loss: 3.0469 | LR: 3.95e-04 | 1400.1ms/step | 14.3 samples/s +Step 13840/44000 | Loss: 3.0570 | LR: 3.95e-04 | 1400.1ms/step | 14.3 samples/s +Step 13850/44000 | Loss: 3.0570 | LR: 3.95e-04 | 1400.1ms/step | 14.3 samples/s +Step 13860/44000 | Loss: 3.0614 | LR: 3.95e-04 | 1399.6ms/step | 14.3 samples/s +Step 13870/44000 | Loss: 3.0264 | LR: 3.95e-04 | 1400.1ms/step | 14.3 samples/s +Step 13880/44000 | Loss: 3.0517 | LR: 3.95e-04 | 1399.9ms/step | 14.3 samples/s +Step 13890/44000 | Loss: 3.0714 | LR: 3.95e-04 | 1402.1ms/step | 14.3 samples/s +Step 13900/44000 | Loss: 3.0571 | LR: 3.94e-04 | 1400.3ms/step | 14.3 samples/s +Step 13910/44000 | Loss: 3.0328 | LR: 3.94e-04 | 1399.7ms/step | 14.3 samples/s +Step 13920/44000 | Loss: 3.0541 | LR: 3.94e-04 | 1402.1ms/step | 14.3 samples/s +Step 13930/44000 | Loss: 3.0549 | LR: 3.94e-04 | 1402.9ms/step | 14.3 samples/s +Step 13940/44000 | Loss: 3.0528 | LR: 3.94e-04 | 1402.9ms/step | 14.3 samples/s +Step 13950/44000 | Loss: 3.0231 | LR: 3.94e-04 | 1400.7ms/step | 14.3 samples/s +Step 13960/44000 | Loss: 3.0538 | LR: 3.93e-04 | 1401.3ms/step | 14.3 samples/s +Step 13970/44000 | Loss: 3.0205 | LR: 3.93e-04 | 1402.2ms/step | 14.3 samples/s +Step 13980/44000 | Loss: 3.0674 | LR: 3.93e-04 | 1399.9ms/step | 14.3 samples/s +Step 13990/44000 | Loss: 3.0573 | LR: 3.93e-04 | 1400.3ms/step | 14.3 samples/s +Step 14000/44000 | Loss: 3.0497 | LR: 3.93e-04 | 1404.1ms/step | 14.2 samples/s +Validation at step 14000: Loss = 3.0457, Perplexity = 21.02 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-14000 +Step 14010/44000 | Loss: 3.0269 | LR: 3.93e-04 | 3933.2ms/step | 5.1 samples/s +Step 14020/44000 | Loss: 3.0386 | LR: 3.93e-04 | 1399.9ms/step | 14.3 samples/s +Step 14030/44000 | Loss: 3.0397 | LR: 3.92e-04 | 1400.2ms/step | 14.3 samples/s +Step 14040/44000 | Loss: 3.0568 | LR: 3.92e-04 | 1399.9ms/step | 14.3 samples/s +Step 14050/44000 | Loss: 3.0401 | LR: 3.92e-04 | 1400.3ms/step | 14.3 samples/s +Step 14060/44000 | Loss: 3.0412 | LR: 3.92e-04 | 1400.2ms/step | 14.3 samples/s +Step 14070/44000 | Loss: 3.0348 | LR: 3.92e-04 | 1399.8ms/step | 14.3 samples/s +Step 14080/44000 | Loss: 3.0322 | LR: 3.92e-04 | 1402.5ms/step | 14.3 samples/s +Step 14090/44000 | Loss: 3.0133 | LR: 3.92e-04 | 1406.3ms/step | 14.2 samples/s +Step 14100/44000 | Loss: 3.0499 | LR: 3.91e-04 | 1403.7ms/step | 14.2 samples/s +Step 14110/44000 | Loss: 3.0379 | LR: 3.91e-04 | 1403.4ms/step | 14.3 samples/s +Step 14120/44000 | Loss: 3.0224 | LR: 3.91e-04 | 1402.5ms/step | 14.3 samples/s +Step 14130/44000 | Loss: 3.0570 | LR: 3.91e-04 | 1403.8ms/step | 14.2 samples/s +Step 14140/44000 | Loss: 3.0246 | LR: 3.91e-04 | 1404.0ms/step | 14.2 samples/s +Step 14150/44000 | Loss: 3.0318 | LR: 3.91e-04 | 1404.3ms/step | 14.2 samples/s +Step 14160/44000 | Loss: 3.0360 | LR: 3.90e-04 | 1406.8ms/step | 14.2 samples/s +Step 14170/44000 | Loss: 3.0725 | LR: 3.90e-04 | 1403.5ms/step | 14.3 samples/s +Step 14180/44000 | Loss: 3.0542 | LR: 3.90e-04 | 1404.4ms/step | 14.2 samples/s +Step 14190/44000 | Loss: 3.0367 | LR: 3.90e-04 | 1407.1ms/step | 14.2 samples/s +Step 14200/44000 | Loss: 3.0304 | LR: 3.90e-04 | 1405.9ms/step | 14.2 samples/s +Step 14210/44000 | Loss: 3.0421 | LR: 3.90e-04 | 1402.6ms/step | 14.3 samples/s +Step 14220/44000 | Loss: 3.0308 | LR: 3.90e-04 | 1406.5ms/step | 14.2 samples/s +Step 14230/44000 | Loss: 3.0301 | LR: 3.89e-04 | 1405.2ms/step | 14.2 samples/s +Step 14240/44000 | Loss: 3.0277 | LR: 3.89e-04 | 1405.7ms/step | 14.2 samples/s +Step 14250/44000 | Loss: 3.0392 | LR: 3.89e-04 | 1405.0ms/step | 14.2 samples/s +Step 14260/44000 | Loss: 3.0446 | LR: 3.89e-04 | 1405.2ms/step | 14.2 samples/s +Step 14270/44000 | Loss: 3.0438 | LR: 3.89e-04 | 1405.5ms/step | 14.2 samples/s +Step 14280/44000 | Loss: 3.0680 | LR: 3.89e-04 | 1405.0ms/step | 14.2 samples/s +Step 14290/44000 | Loss: 3.0635 | LR: 3.89e-04 | 1407.4ms/step | 14.2 samples/s +Step 14300/44000 | Loss: 3.0529 | LR: 3.88e-04 | 1406.1ms/step | 14.2 samples/s +Step 14310/44000 | Loss: 3.0404 | LR: 3.88e-04 | 1406.5ms/step | 14.2 samples/s +Step 14320/44000 | Loss: 3.0086 | LR: 3.88e-04 | 1409.2ms/step | 14.2 samples/s +Step 14330/44000 | Loss: 3.0353 | LR: 3.88e-04 | 1403.7ms/step | 14.2 samples/s +Step 14340/44000 | Loss: 3.0280 | LR: 3.88e-04 | 1408.1ms/step | 14.2 samples/s +Step 14350/44000 | Loss: 3.0095 | LR: 3.88e-04 | 1408.6ms/step | 14.2 samples/s +Step 14360/44000 | Loss: 3.0805 | LR: 3.87e-04 | 1405.8ms/step | 14.2 samples/s +Step 14370/44000 | Loss: 3.0334 | LR: 3.87e-04 | 1406.0ms/step | 14.2 samples/s +Step 14380/44000 | Loss: 3.0244 | LR: 3.87e-04 | 1408.0ms/step | 14.2 samples/s +Step 14390/44000 | Loss: 3.0272 | LR: 3.87e-04 | 1407.2ms/step | 14.2 samples/s +Step 14400/44000 | Loss: 3.0292 | LR: 3.87e-04 | 1404.3ms/step | 14.2 samples/s +Step 14410/44000 | Loss: 3.0409 | LR: 3.87e-04 | 1407.9ms/step | 14.2 samples/s +Step 14420/44000 | Loss: 3.0291 | LR: 3.87e-04 | 1404.7ms/step | 14.2 samples/s +Step 14430/44000 | Loss: 3.0118 | LR: 3.86e-04 | 1405.9ms/step | 14.2 samples/s +Step 14440/44000 | Loss: 3.0532 | LR: 3.86e-04 | 1411.6ms/step | 14.2 samples/s +Step 14450/44000 | Loss: 3.0091 | LR: 3.86e-04 | 1407.8ms/step | 14.2 samples/s +Step 14460/44000 | Loss: 3.0106 | LR: 3.86e-04 | 1404.1ms/step | 14.2 samples/s +Step 14470/44000 | Loss: 3.0369 | LR: 3.86e-04 | 1405.5ms/step | 14.2 samples/s +Step 14480/44000 | Loss: 3.0082 | LR: 3.86e-04 | 1406.6ms/step | 14.2 samples/s +Step 14490/44000 | Loss: 3.0267 | LR: 3.85e-04 | 1406.0ms/step | 14.2 samples/s +Step 14500/44000 | Loss: 3.0286 | LR: 3.85e-04 | 1407.7ms/step | 14.2 samples/s +Step 14510/44000 | Loss: 3.0183 | LR: 3.85e-04 | 1404.8ms/step | 14.2 samples/s +Step 14520/44000 | Loss: 3.0552 | LR: 3.85e-04 | 1405.8ms/step | 14.2 samples/s +Step 14530/44000 | Loss: 3.0307 | LR: 3.85e-04 | 1407.5ms/step | 14.2 samples/s +Step 14540/44000 | Loss: 3.0493 | LR: 3.85e-04 | 1408.3ms/step | 14.2 samples/s +Step 14550/44000 | Loss: 3.0265 | LR: 3.85e-04 | 1407.8ms/step | 14.2 samples/s +Step 14560/44000 | Loss: 3.0358 | LR: 3.84e-04 | 1408.7ms/step | 14.2 samples/s +Step 14570/44000 | Loss: 3.0553 | LR: 3.84e-04 | 1407.2ms/step | 14.2 samples/s +Step 14580/44000 | Loss: 3.0542 | LR: 3.84e-04 | 1405.7ms/step | 14.2 samples/s +Step 14590/44000 | Loss: 3.0441 | LR: 3.84e-04 | 1407.9ms/step | 14.2 samples/s +Step 14600/44000 | Loss: 3.0100 | LR: 3.84e-04 | 1406.5ms/step | 14.2 samples/s +Step 14610/44000 | Loss: 3.0265 | LR: 3.84e-04 | 1412.5ms/step | 14.2 samples/s +Step 14620/44000 | Loss: 3.0501 | LR: 3.83e-04 | 1409.7ms/step | 14.2 samples/s +Step 14630/44000 | Loss: 3.0324 | LR: 3.83e-04 | 1406.2ms/step | 14.2 samples/s +Step 14640/44000 | Loss: 2.9894 | LR: 3.83e-04 | 1405.6ms/step | 14.2 samples/s +Step 14650/44000 | Loss: 3.0245 | LR: 3.83e-04 | 1409.3ms/step | 14.2 samples/s +Step 14660/44000 | Loss: 3.0410 | LR: 3.83e-04 | 1406.0ms/step | 14.2 samples/s +Step 14670/44000 | Loss: 3.0240 | LR: 3.83e-04 | 1404.7ms/step | 14.2 samples/s +Step 14680/44000 | Loss: 3.0410 | LR: 3.83e-04 | 1407.7ms/step | 14.2 samples/s +Step 14690/44000 | Loss: 3.0278 | LR: 3.82e-04 | 1407.8ms/step | 14.2 samples/s +Step 14700/44000 | Loss: 2.9932 | LR: 3.82e-04 | 1404.6ms/step | 14.2 samples/s +Step 14710/44000 | Loss: 3.0281 | LR: 3.82e-04 | 1406.0ms/step | 14.2 samples/s +Step 14720/44000 | Loss: 3.0153 | LR: 3.82e-04 | 1411.6ms/step | 14.2 samples/s +Step 14730/44000 | Loss: 3.0230 | LR: 3.82e-04 | 1406.4ms/step | 14.2 samples/s +Step 14740/44000 | Loss: 3.0272 | LR: 3.82e-04 | 1406.3ms/step | 14.2 samples/s +Step 14750/44000 | Loss: 3.0469 | LR: 3.81e-04 | 1407.6ms/step | 14.2 samples/s +Step 14760/44000 | Loss: 3.0199 | LR: 3.81e-04 | 1409.7ms/step | 14.2 samples/s +Step 14770/44000 | Loss: 3.0266 | LR: 3.81e-04 | 1406.0ms/step | 14.2 samples/s +Step 14780/44000 | Loss: 3.0151 | LR: 3.81e-04 | 1404.4ms/step | 14.2 samples/s +Step 14790/44000 | Loss: 3.0456 | LR: 3.81e-04 | 1403.9ms/step | 14.2 samples/s +Step 14800/44000 | Loss: 3.0314 | LR: 3.81e-04 | 1406.3ms/step | 14.2 samples/s +Step 14810/44000 | Loss: 3.0352 | LR: 3.81e-04 | 1407.0ms/step | 14.2 samples/s +Step 14820/44000 | Loss: 3.0393 | LR: 3.80e-04 | 1404.2ms/step | 14.2 samples/s +Step 14830/44000 | Loss: 2.9928 | LR: 3.80e-04 | 1409.9ms/step | 14.2 samples/s +Step 14840/44000 | Loss: 3.0206 | LR: 3.80e-04 | 1407.5ms/step | 14.2 samples/s +Step 14850/44000 | Loss: 3.0378 | LR: 3.80e-04 | 1407.4ms/step | 14.2 samples/s +Step 14860/44000 | Loss: 3.0264 | LR: 3.80e-04 | 1405.5ms/step | 14.2 samples/s +Step 14870/44000 | Loss: 3.0054 | LR: 3.80e-04 | 1404.2ms/step | 14.2 samples/s +Step 14880/44000 | Loss: 3.0470 | LR: 3.79e-04 | 1407.6ms/step | 14.2 samples/s +Step 14890/44000 | Loss: 3.0423 | LR: 3.79e-04 | 1404.8ms/step | 14.2 samples/s +Step 14900/44000 | Loss: 3.0194 | LR: 3.79e-04 | 1405.0ms/step | 14.2 samples/s +Step 14910/44000 | Loss: 3.0062 | LR: 3.79e-04 | 1408.3ms/step | 14.2 samples/s +Step 14920/44000 | Loss: 3.0421 | LR: 3.79e-04 | 1405.9ms/step | 14.2 samples/s +Step 14930/44000 | Loss: 3.0251 | LR: 3.79e-04 | 1405.8ms/step | 14.2 samples/s +Step 14940/44000 | Loss: 3.0564 | LR: 3.79e-04 | 1408.3ms/step | 14.2 samples/s +Step 14950/44000 | Loss: 3.0071 | LR: 3.78e-04 | 1405.8ms/step | 14.2 samples/s +Step 14960/44000 | Loss: 3.0480 | LR: 3.78e-04 | 1404.5ms/step | 14.2 samples/s +Step 14970/44000 | Loss: 3.0143 | LR: 3.78e-04 | 1405.7ms/step | 14.2 samples/s +Step 14980/44000 | Loss: 3.0089 | LR: 3.78e-04 | 1406.1ms/step | 14.2 samples/s +Step 14990/44000 | Loss: 3.0396 | LR: 3.78e-04 | 1407.0ms/step | 14.2 samples/s +Step 15000/44000 | Loss: 3.0423 | LR: 3.78e-04 | 1405.0ms/step | 14.2 samples/s +Validation at step 15000: Loss = 3.0311, Perplexity = 20.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-15000 +Step 15010/44000 | Loss: 3.0371 | LR: 3.77e-04 | 3929.6ms/step | 5.1 samples/s +Step 15020/44000 | Loss: 3.0466 | LR: 3.77e-04 | 1400.1ms/step | 14.3 samples/s +Step 15030/44000 | Loss: 3.0087 | LR: 3.77e-04 | 1400.1ms/step | 14.3 samples/s +Step 15040/44000 | Loss: 3.0522 | LR: 3.77e-04 | 1402.1ms/step | 14.3 samples/s +Step 15050/44000 | Loss: 3.0278 | LR: 3.77e-04 | 1401.8ms/step | 14.3 samples/s +Step 15060/44000 | Loss: 3.0584 | LR: 3.77e-04 | 1404.2ms/step | 14.2 samples/s +Step 15070/44000 | Loss: 3.0250 | LR: 3.77e-04 | 1406.1ms/step | 14.2 samples/s +Step 15080/44000 | Loss: 3.0197 | LR: 3.76e-04 | 1403.7ms/step | 14.2 samples/s +Step 15090/44000 | Loss: 3.0212 | LR: 3.76e-04 | 1404.6ms/step | 14.2 samples/s +Step 15100/44000 | Loss: 3.0223 | LR: 3.76e-04 | 1401.9ms/step | 14.3 samples/s +Step 15110/44000 | Loss: 3.0271 | LR: 3.76e-04 | 1403.0ms/step | 14.3 samples/s +Step 15120/44000 | Loss: 3.0168 | LR: 3.76e-04 | 1402.9ms/step | 14.3 samples/s +Step 15130/44000 | Loss: 3.0303 | LR: 3.76e-04 | 1404.0ms/step | 14.2 samples/s +Step 15140/44000 | Loss: 3.0492 | LR: 3.75e-04 | 1407.5ms/step | 14.2 samples/s +Step 15150/44000 | Loss: 2.9950 | LR: 3.75e-04 | 1404.1ms/step | 14.2 samples/s +Step 15160/44000 | Loss: 3.0116 | LR: 3.75e-04 | 1405.6ms/step | 14.2 samples/s +Step 15170/44000 | Loss: 3.0240 | LR: 3.75e-04 | 1404.6ms/step | 14.2 samples/s +Step 15180/44000 | Loss: 3.0313 | LR: 3.75e-04 | 1405.3ms/step | 14.2 samples/s +Step 15190/44000 | Loss: 3.0341 | LR: 3.75e-04 | 1403.4ms/step | 14.3 samples/s +Step 15200/44000 | Loss: 3.0550 | LR: 3.74e-04 | 1404.8ms/step | 14.2 samples/s +Step 15210/44000 | Loss: 3.0267 | LR: 3.74e-04 | 1405.4ms/step | 14.2 samples/s +Step 15220/44000 | Loss: 3.0085 | LR: 3.74e-04 | 1408.8ms/step | 14.2 samples/s +Step 15230/44000 | Loss: 3.0154 | LR: 3.74e-04 | 1408.6ms/step | 14.2 samples/s +Step 15240/44000 | Loss: 3.0402 | LR: 3.74e-04 | 1405.5ms/step | 14.2 samples/s +Step 15250/44000 | Loss: 3.0188 | LR: 3.74e-04 | 1404.4ms/step | 14.2 samples/s +Step 15260/44000 | Loss: 3.0388 | LR: 3.74e-04 | 1406.5ms/step | 14.2 samples/s +Step 15270/44000 | Loss: 3.0418 | LR: 3.73e-04 | 1408.5ms/step | 14.2 samples/s +Step 15280/44000 | Loss: 3.0311 | LR: 3.73e-04 | 1408.9ms/step | 14.2 samples/s +Step 15290/44000 | Loss: 3.0193 | LR: 3.73e-04 | 1408.9ms/step | 14.2 samples/s +Step 15300/44000 | Loss: 3.0369 | LR: 3.73e-04 | 1404.6ms/step | 14.2 samples/s +Step 15310/44000 | Loss: 2.9981 | LR: 3.73e-04 | 1406.1ms/step | 14.2 samples/s +Step 15320/44000 | Loss: 3.0409 | LR: 3.73e-04 | 1406.1ms/step | 14.2 samples/s +Step 15330/44000 | Loss: 3.0352 | LR: 3.72e-04 | 1406.4ms/step | 14.2 samples/s +Step 15340/44000 | Loss: 3.0447 | LR: 3.72e-04 | 1406.2ms/step | 14.2 samples/s +Step 15350/44000 | Loss: 3.0216 | LR: 3.72e-04 | 1407.1ms/step | 14.2 samples/s +Step 15360/44000 | Loss: 3.0293 | LR: 3.72e-04 | 1402.5ms/step | 14.3 samples/s +Step 15370/44000 | Loss: 3.0100 | LR: 3.72e-04 | 1405.8ms/step | 14.2 samples/s +Step 15380/44000 | Loss: 3.0581 | LR: 3.72e-04 | 1404.8ms/step | 14.2 samples/s +Step 15390/44000 | Loss: 3.0117 | LR: 3.71e-04 | 1403.8ms/step | 14.2 samples/s +Step 15400/44000 | Loss: 3.0380 | LR: 3.71e-04 | 1405.7ms/step | 14.2 samples/s +Step 15410/44000 | Loss: 3.0102 | LR: 3.71e-04 | 1408.0ms/step | 14.2 samples/s +Step 15420/44000 | Loss: 3.0234 | LR: 3.71e-04 | 1405.6ms/step | 14.2 samples/s +Step 15430/44000 | Loss: 3.0101 | LR: 3.71e-04 | 1406.1ms/step | 14.2 samples/s +Step 15440/44000 | Loss: 3.0203 | LR: 3.71e-04 | 1405.4ms/step | 14.2 samples/s +Step 15450/44000 | Loss: 3.0209 | LR: 3.71e-04 | 1404.7ms/step | 14.2 samples/s +Step 15460/44000 | Loss: 3.0301 | LR: 3.70e-04 | 1406.3ms/step | 14.2 samples/s +Step 15470/44000 | Loss: 3.0357 | LR: 3.70e-04 | 1405.3ms/step | 14.2 samples/s +Step 15480/44000 | Loss: 3.0228 | LR: 3.70e-04 | 1404.1ms/step | 14.2 samples/s +Step 15490/44000 | Loss: 3.0229 | LR: 3.70e-04 | 1406.1ms/step | 14.2 samples/s +Step 15500/44000 | Loss: 3.0138 | LR: 3.70e-04 | 1405.9ms/step | 14.2 samples/s +Step 15510/44000 | Loss: 3.0343 | LR: 3.70e-04 | 1408.6ms/step | 14.2 samples/s +Step 15520/44000 | Loss: 3.0124 | LR: 3.69e-04 | 1407.6ms/step | 14.2 samples/s +Step 15530/44000 | Loss: 3.0176 | LR: 3.69e-04 | 1406.2ms/step | 14.2 samples/s +Step 15540/44000 | Loss: 3.0415 | LR: 3.69e-04 | 1407.5ms/step | 14.2 samples/s +Step 15550/44000 | Loss: 2.9999 | LR: 3.69e-04 | 1405.8ms/step | 14.2 samples/s +Step 15560/44000 | Loss: 3.0004 | LR: 3.69e-04 | 1406.5ms/step | 14.2 samples/s +Step 15570/44000 | Loss: 3.0474 | LR: 3.69e-04 | 1404.4ms/step | 14.2 samples/s +Step 15580/44000 | Loss: 2.9992 | LR: 3.68e-04 | 1406.0ms/step | 14.2 samples/s +Step 15590/44000 | Loss: 3.0146 | LR: 3.68e-04 | 1405.3ms/step | 14.2 samples/s +Step 15600/44000 | Loss: 3.0301 | LR: 3.68e-04 | 1404.5ms/step | 14.2 samples/s +Step 15610/44000 | Loss: 2.9960 | LR: 3.68e-04 | 1405.7ms/step | 14.2 samples/s +Step 15620/44000 | Loss: 2.9921 | LR: 3.68e-04 | 1407.9ms/step | 14.2 samples/s +Step 15630/44000 | Loss: 3.0463 | LR: 3.68e-04 | 1406.7ms/step | 14.2 samples/s +Step 15640/44000 | Loss: 3.0026 | LR: 3.67e-04 | 1407.9ms/step | 14.2 samples/s +Step 15650/44000 | Loss: 3.0406 | LR: 3.67e-04 | 1405.9ms/step | 14.2 samples/s +Step 15660/44000 | Loss: 3.0052 | LR: 3.67e-04 | 1407.0ms/step | 14.2 samples/s +Step 15670/44000 | Loss: 3.0266 | LR: 3.67e-04 | 1404.4ms/step | 14.2 samples/s +Step 15680/44000 | Loss: 3.0003 | LR: 3.67e-04 | 1408.0ms/step | 14.2 samples/s +Step 15690/44000 | Loss: 3.0145 | LR: 3.67e-04 | 1410.5ms/step | 14.2 samples/s +Step 15700/44000 | Loss: 3.0214 | LR: 3.67e-04 | 1407.5ms/step | 14.2 samples/s +Step 15710/44000 | Loss: 3.0644 | LR: 3.66e-04 | 1405.8ms/step | 14.2 samples/s +Step 15720/44000 | Loss: 2.9972 | LR: 3.66e-04 | 1405.9ms/step | 14.2 samples/s +Step 15730/44000 | Loss: 3.0144 | LR: 3.66e-04 | 1410.0ms/step | 14.2 samples/s +Step 15740/44000 | Loss: 3.0384 | LR: 3.66e-04 | 1410.4ms/step | 14.2 samples/s +Step 15750/44000 | Loss: 3.0303 | LR: 3.66e-04 | 1410.7ms/step | 14.2 samples/s +Step 15760/44000 | Loss: 3.0438 | LR: 3.66e-04 | 1406.7ms/step | 14.2 samples/s +Step 15770/44000 | Loss: 3.0205 | LR: 3.65e-04 | 1404.9ms/step | 14.2 samples/s +Step 15780/44000 | Loss: 3.0505 | LR: 3.65e-04 | 1407.6ms/step | 14.2 samples/s +Step 15790/44000 | Loss: 2.9942 | LR: 3.65e-04 | 1405.6ms/step | 14.2 samples/s +Step 15800/44000 | Loss: 3.0126 | LR: 3.65e-04 | 1405.8ms/step | 14.2 samples/s +Step 15810/44000 | Loss: 3.0108 | LR: 3.65e-04 | 1405.2ms/step | 14.2 samples/s +Step 15820/44000 | Loss: 2.9895 | LR: 3.65e-04 | 1405.7ms/step | 14.2 samples/s +Step 15830/44000 | Loss: 3.0383 | LR: 3.64e-04 | 1405.6ms/step | 14.2 samples/s +Step 15840/44000 | Loss: 3.0233 | LR: 3.64e-04 | 1406.5ms/step | 14.2 samples/s +Step 15850/44000 | Loss: 3.0265 | LR: 3.64e-04 | 1406.5ms/step | 14.2 samples/s +Step 15860/44000 | Loss: 2.9929 | LR: 3.64e-04 | 1406.4ms/step | 14.2 samples/s +Step 15870/44000 | Loss: 3.0140 | LR: 3.64e-04 | 1404.6ms/step | 14.2 samples/s +Step 15880/44000 | Loss: 3.0298 | LR: 3.64e-04 | 1408.2ms/step | 14.2 samples/s +Step 15890/44000 | Loss: 3.0192 | LR: 3.63e-04 | 1408.4ms/step | 14.2 samples/s +Step 15900/44000 | Loss: 3.0388 | LR: 3.63e-04 | 1409.1ms/step | 14.2 samples/s +Step 15910/44000 | Loss: 3.0177 | LR: 3.63e-04 | 1406.7ms/step | 14.2 samples/s +Step 15920/44000 | Loss: 3.0049 | LR: 3.63e-04 | 1407.4ms/step | 14.2 samples/s +Step 15930/44000 | Loss: 3.0327 | LR: 3.63e-04 | 1406.3ms/step | 14.2 samples/s +Step 15940/44000 | Loss: 3.0011 | LR: 3.63e-04 | 1404.4ms/step | 14.2 samples/s +Step 15950/44000 | Loss: 3.0167 | LR: 3.62e-04 | 1405.1ms/step | 14.2 samples/s +Step 15960/44000 | Loss: 2.9896 | LR: 3.62e-04 | 1406.8ms/step | 14.2 samples/s +Step 15970/44000 | Loss: 3.0153 | LR: 3.62e-04 | 1406.0ms/step | 14.2 samples/s +Step 15980/44000 | Loss: 3.0171 | LR: 3.62e-04 | 1407.1ms/step | 14.2 samples/s +Step 15990/44000 | Loss: 3.0024 | LR: 3.62e-04 | 1400.0ms/step | 14.3 samples/s +Step 16000/44000 | Loss: 3.0164 | LR: 3.62e-04 | 1400.2ms/step | 14.3 samples/s +Validation at step 16000: Loss = 3.0182, Perplexity = 20.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-16000 +Step 16010/44000 | Loss: 2.9981 | LR: 3.62e-04 | 3945.8ms/step | 5.1 samples/s +Step 16020/44000 | Loss: 2.9873 | LR: 3.61e-04 | 1399.8ms/step | 14.3 samples/s +Step 16030/44000 | Loss: 2.9954 | LR: 3.61e-04 | 1400.0ms/step | 14.3 samples/s +Step 16040/44000 | Loss: 3.0271 | LR: 3.61e-04 | 1400.3ms/step | 14.3 samples/s +Step 16050/44000 | Loss: 3.0306 | LR: 3.61e-04 | 1399.8ms/step | 14.3 samples/s +Step 16060/44000 | Loss: 3.0192 | LR: 3.61e-04 | 1400.3ms/step | 14.3 samples/s +Step 16070/44000 | Loss: 3.0197 | LR: 3.61e-04 | 1400.3ms/step | 14.3 samples/s +Step 16080/44000 | Loss: 3.0014 | LR: 3.60e-04 | 1401.9ms/step | 14.3 samples/s +Step 16090/44000 | Loss: 2.9861 | LR: 3.60e-04 | 1403.8ms/step | 14.2 samples/s +Step 16100/44000 | Loss: 3.0215 | LR: 3.60e-04 | 1401.8ms/step | 14.3 samples/s +Step 16110/44000 | Loss: 3.0163 | LR: 3.60e-04 | 1402.9ms/step | 14.3 samples/s +Step 16120/44000 | Loss: 2.9848 | LR: 3.60e-04 | 1405.4ms/step | 14.2 samples/s +Step 16130/44000 | Loss: 3.0070 | LR: 3.60e-04 | 1404.1ms/step | 14.2 samples/s +Step 16140/44000 | Loss: 3.0463 | LR: 3.59e-04 | 1405.9ms/step | 14.2 samples/s +Step 16150/44000 | Loss: 3.0108 | LR: 3.59e-04 | 1403.7ms/step | 14.2 samples/s +Step 16160/44000 | Loss: 3.0066 | LR: 3.59e-04 | 1404.4ms/step | 14.2 samples/s +Step 16170/44000 | Loss: 3.0117 | LR: 3.59e-04 | 1402.0ms/step | 14.3 samples/s +Step 16180/44000 | Loss: 3.0272 | LR: 3.59e-04 | 1404.3ms/step | 14.2 samples/s +Step 16190/44000 | Loss: 3.0013 | LR: 3.59e-04 | 1404.4ms/step | 14.2 samples/s +Step 16200/44000 | Loss: 2.9888 | LR: 3.58e-04 | 1407.0ms/step | 14.2 samples/s +Step 16210/44000 | Loss: 3.0218 | LR: 3.58e-04 | 1404.4ms/step | 14.2 samples/s +Step 16220/44000 | Loss: 3.0274 | LR: 3.58e-04 | 1402.2ms/step | 14.3 samples/s +Step 16230/44000 | Loss: 2.9845 | LR: 3.58e-04 | 1403.6ms/step | 14.2 samples/s +Step 16240/44000 | Loss: 3.0240 | LR: 3.58e-04 | 1403.7ms/step | 14.2 samples/s +Step 16250/44000 | Loss: 3.0101 | LR: 3.58e-04 | 1402.2ms/step | 14.3 samples/s +Step 16260/44000 | Loss: 2.9791 | LR: 3.57e-04 | 1401.7ms/step | 14.3 samples/s +Step 16270/44000 | Loss: 3.0358 | LR: 3.57e-04 | 1402.1ms/step | 14.3 samples/s +Step 16280/44000 | Loss: 3.0304 | LR: 3.57e-04 | 1406.9ms/step | 14.2 samples/s +Step 16290/44000 | Loss: 3.0055 | LR: 3.57e-04 | 1406.8ms/step | 14.2 samples/s +Step 16300/44000 | Loss: 3.0490 | LR: 3.57e-04 | 1406.7ms/step | 14.2 samples/s +Step 16310/44000 | Loss: 2.9993 | LR: 3.57e-04 | 1403.7ms/step | 14.2 samples/s +Step 16320/44000 | Loss: 3.0338 | LR: 3.56e-04 | 1405.5ms/step | 14.2 samples/s +Step 16330/44000 | Loss: 3.0121 | LR: 3.56e-04 | 1402.6ms/step | 14.3 samples/s +Step 16340/44000 | Loss: 3.0107 | LR: 3.56e-04 | 1404.7ms/step | 14.2 samples/s +Step 16350/44000 | Loss: 3.0077 | LR: 3.56e-04 | 1403.5ms/step | 14.3 samples/s +Step 16360/44000 | Loss: 3.0100 | LR: 3.56e-04 | 1406.1ms/step | 14.2 samples/s +Step 16370/44000 | Loss: 3.0233 | LR: 3.56e-04 | 1406.8ms/step | 14.2 samples/s +Step 16380/44000 | Loss: 2.9960 | LR: 3.55e-04 | 1407.5ms/step | 14.2 samples/s +Step 16390/44000 | Loss: 3.0204 | LR: 3.55e-04 | 1407.0ms/step | 14.2 samples/s +Step 16400/44000 | Loss: 3.0122 | LR: 3.55e-04 | 1404.4ms/step | 14.2 samples/s +Step 16410/44000 | Loss: 3.0322 | LR: 3.55e-04 | 1403.5ms/step | 14.2 samples/s +Step 16420/44000 | Loss: 3.0079 | LR: 3.55e-04 | 1402.1ms/step | 14.3 samples/s +Step 16430/44000 | Loss: 2.9612 | LR: 3.55e-04 | 1402.6ms/step | 14.3 samples/s +Step 16440/44000 | Loss: 3.0090 | LR: 3.54e-04 | 1405.5ms/step | 14.2 samples/s +Step 16450/44000 | Loss: 3.0478 | LR: 3.54e-04 | 1404.1ms/step | 14.2 samples/s +Step 16460/44000 | Loss: 2.9971 | LR: 3.54e-04 | 1402.5ms/step | 14.3 samples/s +Step 16470/44000 | Loss: 3.0192 | LR: 3.54e-04 | 1403.2ms/step | 14.3 samples/s +Step 16480/44000 | Loss: 2.9986 | LR: 3.54e-04 | 1400.5ms/step | 14.3 samples/s +Step 16490/44000 | Loss: 3.0094 | LR: 3.54e-04 | 1403.8ms/step | 14.2 samples/s +Step 16500/44000 | Loss: 2.9890 | LR: 3.53e-04 | 1404.4ms/step | 14.2 samples/s +Step 16510/44000 | Loss: 2.9981 | LR: 3.53e-04 | 1403.8ms/step | 14.2 samples/s +Step 16520/44000 | Loss: 3.0088 | LR: 3.53e-04 | 1406.4ms/step | 14.2 samples/s +Step 16530/44000 | Loss: 3.0235 | LR: 3.53e-04 | 1406.9ms/step | 14.2 samples/s +Step 16540/44000 | Loss: 3.0084 | LR: 3.53e-04 | 1404.3ms/step | 14.2 samples/s +Step 16550/44000 | Loss: 3.0437 | LR: 3.53e-04 | 1406.2ms/step | 14.2 samples/s +Step 16560/44000 | Loss: 3.0119 | LR: 3.52e-04 | 1407.5ms/step | 14.2 samples/s +Step 16570/44000 | Loss: 3.0102 | LR: 3.52e-04 | 1407.9ms/step | 14.2 samples/s +Step 16580/44000 | Loss: 2.9882 | LR: 3.52e-04 | 1404.5ms/step | 14.2 samples/s +Step 16590/44000 | Loss: 2.9727 | LR: 3.52e-04 | 1407.3ms/step | 14.2 samples/s +Step 16600/44000 | Loss: 3.0048 | LR: 3.52e-04 | 1404.1ms/step | 14.2 samples/s +Step 16610/44000 | Loss: 2.9878 | LR: 3.52e-04 | 1406.3ms/step | 14.2 samples/s +Step 16620/44000 | Loss: 3.0154 | LR: 3.51e-04 | 1408.2ms/step | 14.2 samples/s +Step 16630/44000 | Loss: 2.9833 | LR: 3.51e-04 | 1408.2ms/step | 14.2 samples/s +Step 16640/44000 | Loss: 3.0043 | LR: 3.51e-04 | 1408.2ms/step | 14.2 samples/s +Step 16650/44000 | Loss: 3.0141 | LR: 3.51e-04 | 1407.1ms/step | 14.2 samples/s +Step 16660/44000 | Loss: 3.0043 | LR: 3.51e-04 | 1406.2ms/step | 14.2 samples/s +Step 16670/44000 | Loss: 3.0042 | LR: 3.51e-04 | 1406.0ms/step | 14.2 samples/s +Step 16680/44000 | Loss: 3.0202 | LR: 3.50e-04 | 1404.1ms/step | 14.2 samples/s +Step 16690/44000 | Loss: 2.9715 | LR: 3.50e-04 | 1404.1ms/step | 14.2 samples/s +Step 16700/44000 | Loss: 2.9954 | LR: 3.50e-04 | 1403.1ms/step | 14.3 samples/s +Step 16710/44000 | Loss: 2.9902 | LR: 3.50e-04 | 1404.2ms/step | 14.2 samples/s +Step 16720/44000 | Loss: 3.0234 | LR: 3.50e-04 | 1400.8ms/step | 14.3 samples/s +Step 16730/44000 | Loss: 3.0029 | LR: 3.50e-04 | 1405.7ms/step | 14.2 samples/s +Step 16740/44000 | Loss: 3.0064 | LR: 3.49e-04 | 1407.8ms/step | 14.2 samples/s +Step 16750/44000 | Loss: 2.9984 | LR: 3.49e-04 | 1406.1ms/step | 14.2 samples/s +Step 16760/44000 | Loss: 3.0193 | LR: 3.49e-04 | 1404.6ms/step | 14.2 samples/s +Step 16770/44000 | Loss: 3.0108 | LR: 3.49e-04 | 1407.1ms/step | 14.2 samples/s +Step 16780/44000 | Loss: 3.0096 | LR: 3.49e-04 | 1406.3ms/step | 14.2 samples/s +Step 16790/44000 | Loss: 2.9972 | LR: 3.49e-04 | 1406.9ms/step | 14.2 samples/s +Step 16800/44000 | Loss: 3.0061 | LR: 3.48e-04 | 1405.3ms/step | 14.2 samples/s +Step 16810/44000 | Loss: 3.0070 | LR: 3.48e-04 | 1406.7ms/step | 14.2 samples/s +Step 16820/44000 | Loss: 3.0215 | LR: 3.48e-04 | 1407.0ms/step | 14.2 samples/s +Step 16830/44000 | Loss: 2.9923 | LR: 3.48e-04 | 1405.9ms/step | 14.2 samples/s +Step 16840/44000 | Loss: 3.0203 | LR: 3.48e-04 | 1406.4ms/step | 14.2 samples/s +Step 16850/44000 | Loss: 3.0243 | LR: 3.48e-04 | 1405.9ms/step | 14.2 samples/s +Step 16860/44000 | Loss: 3.0320 | LR: 3.47e-04 | 1407.7ms/step | 14.2 samples/s +Step 16870/44000 | Loss: 3.0149 | LR: 3.47e-04 | 1407.5ms/step | 14.2 samples/s +Step 16880/44000 | Loss: 3.0218 | LR: 3.47e-04 | 1406.8ms/step | 14.2 samples/s +Step 16890/44000 | Loss: 2.9908 | LR: 3.47e-04 | 1406.9ms/step | 14.2 samples/s +Step 16900/44000 | Loss: 2.9648 | LR: 3.47e-04 | 1407.7ms/step | 14.2 samples/s +Step 16910/44000 | Loss: 3.0129 | LR: 3.47e-04 | 1411.1ms/step | 14.2 samples/s +Step 16920/44000 | Loss: 2.9968 | LR: 3.46e-04 | 1408.1ms/step | 14.2 samples/s +Step 16930/44000 | Loss: 3.0167 | LR: 3.46e-04 | 1406.3ms/step | 14.2 samples/s +Step 16940/44000 | Loss: 3.0183 | LR: 3.46e-04 | 1402.4ms/step | 14.3 samples/s +Step 16950/44000 | Loss: 3.0071 | LR: 3.46e-04 | 1402.8ms/step | 14.3 samples/s +Step 16960/44000 | Loss: 3.0012 | LR: 3.46e-04 | 1405.0ms/step | 14.2 samples/s +Step 16970/44000 | Loss: 3.0226 | LR: 3.46e-04 | 1401.1ms/step | 14.3 samples/s +Step 16980/44000 | Loss: 3.0106 | LR: 3.45e-04 | 1400.0ms/step | 14.3 samples/s +Step 16990/44000 | Loss: 2.9798 | LR: 3.45e-04 | 1399.9ms/step | 14.3 samples/s +Step 17000/44000 | Loss: 3.0017 | LR: 3.45e-04 | 1400.2ms/step | 14.3 samples/s +Validation at step 17000: Loss = 3.0056, Perplexity = 20.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-17000 +Step 17010/44000 | Loss: 3.0082 | LR: 3.45e-04 | 4014.1ms/step | 5.0 samples/s +Step 17020/44000 | Loss: 2.9929 | LR: 3.45e-04 | 1399.8ms/step | 14.3 samples/s +Step 17030/44000 | Loss: 2.9992 | LR: 3.45e-04 | 1401.5ms/step | 14.3 samples/s +Step 17040/44000 | Loss: 3.0050 | LR: 3.44e-04 | 1402.9ms/step | 14.3 samples/s +Step 17050/44000 | Loss: 3.0076 | LR: 3.44e-04 | 1402.4ms/step | 14.3 samples/s +Step 17060/44000 | Loss: 2.9949 | LR: 3.44e-04 | 1403.8ms/step | 14.2 samples/s +Step 17070/44000 | Loss: 3.0166 | LR: 3.44e-04 | 1402.1ms/step | 14.3 samples/s +Step 17080/44000 | Loss: 3.0174 | LR: 3.44e-04 | 1406.3ms/step | 14.2 samples/s +Step 17090/44000 | Loss: 3.0093 | LR: 3.44e-04 | 1406.6ms/step | 14.2 samples/s +Step 17100/44000 | Loss: 3.0086 | LR: 3.43e-04 | 1404.8ms/step | 14.2 samples/s +Step 17110/44000 | Loss: 2.9833 | LR: 3.43e-04 | 1406.1ms/step | 14.2 samples/s +Step 17120/44000 | Loss: 3.0130 | LR: 3.43e-04 | 1406.0ms/step | 14.2 samples/s +Step 17130/44000 | Loss: 3.0143 | LR: 3.43e-04 | 1406.8ms/step | 14.2 samples/s +Step 17140/44000 | Loss: 2.9852 | LR: 3.43e-04 | 1409.3ms/step | 14.2 samples/s +Step 17150/44000 | Loss: 2.9984 | LR: 3.43e-04 | 1408.4ms/step | 14.2 samples/s +Step 17160/44000 | Loss: 2.9864 | LR: 3.42e-04 | 1407.1ms/step | 14.2 samples/s +Step 17170/44000 | Loss: 3.0419 | LR: 3.42e-04 | 1406.2ms/step | 14.2 samples/s +Step 17180/44000 | Loss: 2.9945 | LR: 3.42e-04 | 1406.3ms/step | 14.2 samples/s +Step 17190/44000 | Loss: 3.0120 | LR: 3.42e-04 | 1406.6ms/step | 14.2 samples/s +Step 17200/44000 | Loss: 3.0314 | LR: 3.42e-04 | 1409.0ms/step | 14.2 samples/s +Step 17210/44000 | Loss: 3.0222 | LR: 3.42e-04 | 1407.1ms/step | 14.2 samples/s +Step 17220/44000 | Loss: 3.0067 | LR: 3.41e-04 | 1407.3ms/step | 14.2 samples/s +Step 17230/44000 | Loss: 2.9997 | LR: 3.41e-04 | 1408.2ms/step | 14.2 samples/s +Step 17240/44000 | Loss: 3.0174 | LR: 3.41e-04 | 1409.3ms/step | 14.2 samples/s +Step 17250/44000 | Loss: 2.9681 | LR: 3.41e-04 | 1406.5ms/step | 14.2 samples/s +Step 17260/44000 | Loss: 2.9908 | LR: 3.41e-04 | 1408.4ms/step | 14.2 samples/s +Step 17270/44000 | Loss: 2.9869 | LR: 3.41e-04 | 1409.2ms/step | 14.2 samples/s +Step 17280/44000 | Loss: 2.9766 | LR: 3.40e-04 | 1407.6ms/step | 14.2 samples/s +Step 17290/44000 | Loss: 2.9601 | LR: 3.40e-04 | 1408.6ms/step | 14.2 samples/s +Step 17300/44000 | Loss: 2.9960 | LR: 3.40e-04 | 1408.3ms/step | 14.2 samples/s +Step 17310/44000 | Loss: 3.0084 | LR: 3.40e-04 | 1409.2ms/step | 14.2 samples/s +Step 17320/44000 | Loss: 3.0018 | LR: 3.40e-04 | 1409.6ms/step | 14.2 samples/s +Step 17330/44000 | Loss: 2.9974 | LR: 3.40e-04 | 1410.4ms/step | 14.2 samples/s +Step 17340/44000 | Loss: 3.0035 | LR: 3.39e-04 | 1408.6ms/step | 14.2 samples/s +Step 17350/44000 | Loss: 3.0107 | LR: 3.39e-04 | 1409.7ms/step | 14.2 samples/s +Step 17360/44000 | Loss: 3.0203 | LR: 3.39e-04 | 1410.7ms/step | 14.2 samples/s +Step 17370/44000 | Loss: 3.0077 | LR: 3.39e-04 | 1407.5ms/step | 14.2 samples/s +Step 17380/44000 | Loss: 3.0024 | LR: 3.39e-04 | 1408.4ms/step | 14.2 samples/s +Step 17390/44000 | Loss: 2.9851 | LR: 3.39e-04 | 1406.1ms/step | 14.2 samples/s +Step 17400/44000 | Loss: 3.0201 | LR: 3.38e-04 | 1408.2ms/step | 14.2 samples/s +Step 17410/44000 | Loss: 2.9925 | LR: 3.38e-04 | 1409.0ms/step | 14.2 samples/s +Step 17420/44000 | Loss: 2.9969 | LR: 3.38e-04 | 1408.1ms/step | 14.2 samples/s +Step 17430/44000 | Loss: 2.9662 | LR: 3.38e-04 | 1403.5ms/step | 14.3 samples/s +Step 17440/44000 | Loss: 2.9928 | LR: 3.38e-04 | 1408.4ms/step | 14.2 samples/s +Step 17450/44000 | Loss: 2.9656 | LR: 3.38e-04 | 1407.9ms/step | 14.2 samples/s diff --git a/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed46.20260820T041452Z.log b/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed46.20260820T041452Z.log new file mode 100644 index 0000000000000000000000000000000000000000..00b73c1e6dd5f1b3ce176fdfd93085d04969d3af --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_h100_resume/seed46.20260820T041452Z.log @@ -0,0 +1,653 @@ +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-13000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-13000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 46, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed46", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-13000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-13000... +Preserving checkpoint LR state for exact resume: current=[0.0004073888929652048], schedule_base=[0.0005] +Resumed from checkpoint at step 13000 (epoch 0, micro_step 130000) +Resuming training from step 13000 +Resuming wandb run zovy63wq +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run zovy63wq +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-zovy63wq +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed46 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/zovy63wq +Starting training... +Validation at step 13000: Loss = 3.0598, Perplexity = 21.32 +wandb: WARNING Tried to log to step 13000 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 13001: dense (dense step interval) +Step 13010/44000 | Loss: 3.0481 | LR: 4.07e-04 | 30389.3ms/step | 0.7 samples/s +wandb: WARNING Tried to log to step 13010 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13020/44000 | Loss: 3.0472 | LR: 4.07e-04 | 1376.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13020 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13030/44000 | Loss: 3.0740 | LR: 4.07e-04 | 1380.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13030 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13040/44000 | Loss: 3.0868 | LR: 4.07e-04 | 1375.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 13040 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13050/44000 | Loss: 3.0517 | LR: 4.07e-04 | 1374.0ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 13050 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 13060/44000 | Loss: 3.0409 | LR: 4.07e-04 | 1377.3ms/step | 14.5 samples/s +Step 13070/44000 | Loss: 3.0491 | LR: 4.06e-04 | 1378.9ms/step | 14.5 samples/s +Step 13080/44000 | Loss: 3.0805 | LR: 4.06e-04 | 1379.0ms/step | 14.5 samples/s +Step 13090/44000 | Loss: 3.0620 | LR: 4.06e-04 | 1378.4ms/step | 14.5 samples/s +Step 13100/44000 | Loss: 3.0623 | LR: 4.06e-04 | 1379.6ms/step | 14.5 samples/s +Step 13110/44000 | Loss: 3.0679 | LR: 4.06e-04 | 1376.4ms/step | 14.5 samples/s +Step 13120/44000 | Loss: 3.0415 | LR: 4.06e-04 | 1378.6ms/step | 14.5 samples/s +Step 13130/44000 | Loss: 3.0552 | LR: 4.06e-04 | 1377.3ms/step | 14.5 samples/s +Step 13140/44000 | Loss: 3.0384 | LR: 4.05e-04 | 1376.3ms/step | 14.5 samples/s +Step 13150/44000 | Loss: 3.0496 | LR: 4.05e-04 | 1379.6ms/step | 14.5 samples/s +Step 13160/44000 | Loss: 3.0589 | LR: 4.05e-04 | 1380.1ms/step | 14.5 samples/s +Step 13170/44000 | Loss: 3.0586 | LR: 4.05e-04 | 1377.6ms/step | 14.5 samples/s +Step 13180/44000 | Loss: 3.0326 | LR: 4.05e-04 | 1377.6ms/step | 14.5 samples/s +Step 13190/44000 | Loss: 3.0617 | LR: 4.05e-04 | 1378.9ms/step | 14.5 samples/s +Step 13200/44000 | Loss: 3.0821 | LR: 4.05e-04 | 1380.6ms/step | 14.5 samples/s +Step 13210/44000 | Loss: 3.0834 | LR: 4.04e-04 | 1376.7ms/step | 14.5 samples/s +Step 13220/44000 | Loss: 3.0198 | LR: 4.04e-04 | 1376.4ms/step | 14.5 samples/s +Step 13230/44000 | Loss: 3.0559 | LR: 4.04e-04 | 1378.8ms/step | 14.5 samples/s +Step 13240/44000 | Loss: 3.0391 | LR: 4.04e-04 | 1379.2ms/step | 14.5 samples/s +Step 13250/44000 | Loss: 3.0350 | LR: 4.04e-04 | 1378.3ms/step | 14.5 samples/s +Step 13260/44000 | Loss: 3.0151 | LR: 4.04e-04 | 1376.3ms/step | 14.5 samples/s +Step 13270/44000 | Loss: 3.0577 | LR: 4.04e-04 | 1384.5ms/step | 14.4 samples/s +Step 13280/44000 | Loss: 3.0382 | LR: 4.03e-04 | 1384.1ms/step | 14.5 samples/s +Step 13290/44000 | Loss: 3.0494 | LR: 4.03e-04 | 1383.8ms/step | 14.5 samples/s +Step 13300/44000 | Loss: 3.0422 | LR: 4.03e-04 | 1384.0ms/step | 14.5 samples/s +Step 13310/44000 | Loss: 3.0291 | LR: 4.03e-04 | 1385.1ms/step | 14.4 samples/s +Step 13320/44000 | Loss: 3.0818 | LR: 4.03e-04 | 1383.8ms/step | 14.5 samples/s +Step 13330/44000 | Loss: 3.0389 | LR: 4.03e-04 | 1381.9ms/step | 14.5 samples/s +Step 13340/44000 | Loss: 3.0393 | LR: 4.03e-04 | 1381.5ms/step | 14.5 samples/s +Step 13350/44000 | Loss: 3.0494 | LR: 4.02e-04 | 1378.8ms/step | 14.5 samples/s +Step 13360/44000 | Loss: 2.9992 | LR: 4.02e-04 | 1379.8ms/step | 14.5 samples/s +Step 13370/44000 | Loss: 3.0336 | LR: 4.02e-04 | 1377.1ms/step | 14.5 samples/s +Step 13380/44000 | Loss: 3.0688 | LR: 4.02e-04 | 1377.2ms/step | 14.5 samples/s +Step 13390/44000 | Loss: 3.0480 | LR: 4.02e-04 | 1377.6ms/step | 14.5 samples/s +Step 13400/44000 | Loss: 3.0450 | LR: 4.02e-04 | 1378.3ms/step | 14.5 samples/s +Step 13410/44000 | Loss: 3.0689 | LR: 4.02e-04 | 1376.3ms/step | 14.5 samples/s +Step 13420/44000 | Loss: 3.0442 | LR: 4.01e-04 | 1377.1ms/step | 14.5 samples/s +Step 13430/44000 | Loss: 3.0597 | LR: 4.01e-04 | 1378.5ms/step | 14.5 samples/s +Step 13440/44000 | Loss: 3.0440 | LR: 4.01e-04 | 1379.3ms/step | 14.5 samples/s +Step 13450/44000 | Loss: 3.0382 | LR: 4.01e-04 | 1375.8ms/step | 14.5 samples/s +Step 13460/44000 | Loss: 3.0512 | LR: 4.01e-04 | 1377.6ms/step | 14.5 samples/s +Step 13470/44000 | Loss: 3.0468 | LR: 4.01e-04 | 1379.4ms/step | 14.5 samples/s +Step 13480/44000 | Loss: 3.0806 | LR: 4.01e-04 | 1378.4ms/step | 14.5 samples/s +Step 13490/44000 | Loss: 3.0723 | LR: 4.00e-04 | 1375.8ms/step | 14.5 samples/s +Step 13500/44000 | Loss: 3.0526 | LR: 4.00e-04 | 1377.0ms/step | 14.5 samples/s +Step 13510/44000 | Loss: 3.0405 | LR: 4.00e-04 | 1377.5ms/step | 14.5 samples/s +Step 13520/44000 | Loss: 3.0453 | LR: 4.00e-04 | 1379.3ms/step | 14.5 samples/s +Step 13530/44000 | Loss: 3.0552 | LR: 4.00e-04 | 1378.0ms/step | 14.5 samples/s +Step 13540/44000 | Loss: 3.0549 | LR: 4.00e-04 | 1379.1ms/step | 14.5 samples/s +Step 13550/44000 | Loss: 3.0258 | LR: 4.00e-04 | 1380.2ms/step | 14.5 samples/s +Step 13560/44000 | Loss: 3.0542 | LR: 3.99e-04 | 1379.2ms/step | 14.5 samples/s +Step 13570/44000 | Loss: 3.0473 | LR: 3.99e-04 | 1376.8ms/step | 14.5 samples/s +Step 13580/44000 | Loss: 3.0352 | LR: 3.99e-04 | 1376.5ms/step | 14.5 samples/s +Step 13590/44000 | Loss: 3.0440 | LR: 3.99e-04 | 1380.5ms/step | 14.5 samples/s +Step 13600/44000 | Loss: 3.0300 | LR: 3.99e-04 | 1381.3ms/step | 14.5 samples/s +Step 13610/44000 | Loss: 3.0538 | LR: 3.99e-04 | 1380.1ms/step | 14.5 samples/s +Step 13620/44000 | Loss: 3.0784 | LR: 3.98e-04 | 1378.8ms/step | 14.5 samples/s +Step 13630/44000 | Loss: 3.0545 | LR: 3.98e-04 | 1382.0ms/step | 14.5 samples/s +Step 13640/44000 | Loss: 3.0056 | LR: 3.98e-04 | 1381.7ms/step | 14.5 samples/s +Step 13650/44000 | Loss: 3.0601 | LR: 3.98e-04 | 1378.9ms/step | 14.5 samples/s +Step 13660/44000 | Loss: 3.0521 | LR: 3.98e-04 | 1378.3ms/step | 14.5 samples/s +Step 13670/44000 | Loss: 3.0288 | LR: 3.98e-04 | 1380.8ms/step | 14.5 samples/s +Step 13680/44000 | Loss: 3.0444 | LR: 3.98e-04 | 1379.5ms/step | 14.5 samples/s +Step 13690/44000 | Loss: 3.0248 | LR: 3.97e-04 | 1378.0ms/step | 14.5 samples/s +Step 13700/44000 | Loss: 3.0686 | LR: 3.97e-04 | 1376.4ms/step | 14.5 samples/s +Step 13710/44000 | Loss: 3.0330 | LR: 3.97e-04 | 1378.9ms/step | 14.5 samples/s +Step 13720/44000 | Loss: 3.0380 | LR: 3.97e-04 | 1377.1ms/step | 14.5 samples/s +Step 13730/44000 | Loss: 2.9996 | LR: 3.97e-04 | 1377.7ms/step | 14.5 samples/s +Step 13740/44000 | Loss: 3.0181 | LR: 3.97e-04 | 1378.0ms/step | 14.5 samples/s +Step 13750/44000 | Loss: 3.0539 | LR: 3.97e-04 | 1379.1ms/step | 14.5 samples/s +Step 13760/44000 | Loss: 3.0448 | LR: 3.96e-04 | 1378.6ms/step | 14.5 samples/s +Step 13770/44000 | Loss: 3.0532 | LR: 3.96e-04 | 1378.2ms/step | 14.5 samples/s +Step 13780/44000 | Loss: 3.0652 | LR: 3.96e-04 | 1376.1ms/step | 14.5 samples/s +Step 13790/44000 | Loss: 3.0325 | LR: 3.96e-04 | 1378.2ms/step | 14.5 samples/s +Step 13800/44000 | Loss: 3.0507 | LR: 3.96e-04 | 1377.3ms/step | 14.5 samples/s +Step 13810/44000 | Loss: 3.0212 | LR: 3.96e-04 | 1376.1ms/step | 14.5 samples/s +Step 13820/44000 | Loss: 3.0419 | LR: 3.96e-04 | 1377.8ms/step | 14.5 samples/s +Step 13830/44000 | Loss: 3.0317 | LR: 3.95e-04 | 1376.6ms/step | 14.5 samples/s +Step 13840/44000 | Loss: 3.0505 | LR: 3.95e-04 | 1376.4ms/step | 14.5 samples/s +Step 13850/44000 | Loss: 3.0452 | LR: 3.95e-04 | 1376.3ms/step | 14.5 samples/s +Step 13860/44000 | Loss: 3.0308 | LR: 3.95e-04 | 1381.0ms/step | 14.5 samples/s +Step 13870/44000 | Loss: 3.0502 | LR: 3.95e-04 | 1380.3ms/step | 14.5 samples/s +Step 13880/44000 | Loss: 3.0730 | LR: 3.95e-04 | 1375.8ms/step | 14.5 samples/s +Step 13890/44000 | Loss: 3.0543 | LR: 3.95e-04 | 1377.9ms/step | 14.5 samples/s +Step 13900/44000 | Loss: 3.0624 | LR: 3.94e-04 | 1377.4ms/step | 14.5 samples/s +Step 13910/44000 | Loss: 3.0401 | LR: 3.94e-04 | 1377.8ms/step | 14.5 samples/s +Step 13920/44000 | Loss: 3.0559 | LR: 3.94e-04 | 1376.7ms/step | 14.5 samples/s +Step 13930/44000 | Loss: 3.0415 | LR: 3.94e-04 | 1377.2ms/step | 14.5 samples/s +Step 13940/44000 | Loss: 3.0456 | LR: 3.94e-04 | 1377.8ms/step | 14.5 samples/s +Step 13950/44000 | Loss: 3.0764 | LR: 3.94e-04 | 1381.1ms/step | 14.5 samples/s +Step 13960/44000 | Loss: 3.0517 | LR: 3.93e-04 | 1379.6ms/step | 14.5 samples/s +Step 13970/44000 | Loss: 3.0275 | LR: 3.93e-04 | 1378.0ms/step | 14.5 samples/s +Step 13980/44000 | Loss: 3.0508 | LR: 3.93e-04 | 1378.6ms/step | 14.5 samples/s +Step 13990/44000 | Loss: 3.0339 | LR: 3.93e-04 | 1379.9ms/step | 14.5 samples/s +Step 14000/44000 | Loss: 3.0430 | LR: 3.93e-04 | 1376.7ms/step | 14.5 samples/s +Validation at step 14000: Loss = 3.0458, Perplexity = 21.03 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-14000 +Step 14010/44000 | Loss: 3.0636 | LR: 3.93e-04 | 3907.6ms/step | 5.1 samples/s +Step 14020/44000 | Loss: 3.0477 | LR: 3.93e-04 | 1390.0ms/step | 14.4 samples/s +Step 14030/44000 | Loss: 3.0135 | LR: 3.92e-04 | 1390.0ms/step | 14.4 samples/s +Step 14040/44000 | Loss: 3.0325 | LR: 3.92e-04 | 1389.8ms/step | 14.4 samples/s +Step 14050/44000 | Loss: 3.0606 | LR: 3.92e-04 | 1393.8ms/step | 14.3 samples/s +Step 14060/44000 | Loss: 3.0371 | LR: 3.92e-04 | 1396.4ms/step | 14.3 samples/s +Step 14070/44000 | Loss: 3.0377 | LR: 3.92e-04 | 1398.0ms/step | 14.3 samples/s +Step 14080/44000 | Loss: 3.0223 | LR: 3.92e-04 | 1396.2ms/step | 14.3 samples/s +Step 14090/44000 | Loss: 3.0452 | LR: 3.92e-04 | 1393.4ms/step | 14.4 samples/s +Step 14100/44000 | Loss: 3.0447 | LR: 3.91e-04 | 1393.9ms/step | 14.3 samples/s +Step 14110/44000 | Loss: 3.0483 | LR: 3.91e-04 | 1394.4ms/step | 14.3 samples/s +Step 14120/44000 | Loss: 3.0587 | LR: 3.91e-04 | 1395.8ms/step | 14.3 samples/s +Step 14130/44000 | Loss: 3.0422 | LR: 3.91e-04 | 1394.2ms/step | 14.3 samples/s +Step 14140/44000 | Loss: 3.0681 | LR: 3.91e-04 | 1397.8ms/step | 14.3 samples/s +Step 14150/44000 | Loss: 3.0426 | LR: 3.91e-04 | 1386.5ms/step | 14.4 samples/s +Step 14160/44000 | Loss: 3.0381 | LR: 3.90e-04 | 1382.2ms/step | 14.5 samples/s +Step 14170/44000 | Loss: 3.0104 | LR: 3.90e-04 | 1382.5ms/step | 14.5 samples/s +Step 14180/44000 | Loss: 3.0441 | LR: 3.90e-04 | 1382.3ms/step | 14.5 samples/s +Step 14190/44000 | Loss: 3.0137 | LR: 3.90e-04 | 1380.7ms/step | 14.5 samples/s +Step 14200/44000 | Loss: 3.0176 | LR: 3.90e-04 | 1381.7ms/step | 14.5 samples/s +Step 14210/44000 | Loss: 3.0291 | LR: 3.90e-04 | 1383.4ms/step | 14.5 samples/s +Step 14220/44000 | Loss: 3.0395 | LR: 3.90e-04 | 1381.0ms/step | 14.5 samples/s +Step 14230/44000 | Loss: 3.0509 | LR: 3.89e-04 | 1380.3ms/step | 14.5 samples/s +Step 14240/44000 | Loss: 3.0615 | LR: 3.89e-04 | 1379.5ms/step | 14.5 samples/s +Step 14250/44000 | Loss: 3.0315 | LR: 3.89e-04 | 1380.4ms/step | 14.5 samples/s +Step 14260/44000 | Loss: 3.0270 | LR: 3.89e-04 | 1383.1ms/step | 14.5 samples/s +Step 14270/44000 | Loss: 3.0274 | LR: 3.89e-04 | 1383.9ms/step | 14.5 samples/s +Step 14280/44000 | Loss: 3.0264 | LR: 3.89e-04 | 1381.2ms/step | 14.5 samples/s +Step 14290/44000 | Loss: 3.0398 | LR: 3.89e-04 | 1380.7ms/step | 14.5 samples/s +Step 14300/44000 | Loss: 3.0369 | LR: 3.88e-04 | 1385.3ms/step | 14.4 samples/s +Step 14310/44000 | Loss: 3.0450 | LR: 3.88e-04 | 1383.8ms/step | 14.5 samples/s +Step 14320/44000 | Loss: 3.0471 | LR: 3.88e-04 | 1382.7ms/step | 14.5 samples/s +Step 14330/44000 | Loss: 3.0381 | LR: 3.88e-04 | 1381.8ms/step | 14.5 samples/s +Step 14340/44000 | Loss: 3.0571 | LR: 3.88e-04 | 1381.5ms/step | 14.5 samples/s +Step 14350/44000 | Loss: 3.0220 | LR: 3.88e-04 | 1383.6ms/step | 14.5 samples/s +Step 14360/44000 | Loss: 3.0370 | LR: 3.87e-04 | 1383.9ms/step | 14.5 samples/s +Step 14370/44000 | Loss: 3.0404 | LR: 3.87e-04 | 1384.0ms/step | 14.5 samples/s +Step 14380/44000 | Loss: 3.0496 | LR: 3.87e-04 | 1382.5ms/step | 14.5 samples/s +Step 14390/44000 | Loss: 3.0281 | LR: 3.87e-04 | 1383.2ms/step | 14.5 samples/s +Step 14400/44000 | Loss: 3.0186 | LR: 3.87e-04 | 1385.0ms/step | 14.4 samples/s +Step 14410/44000 | Loss: 3.0406 | LR: 3.87e-04 | 1378.9ms/step | 14.5 samples/s +Step 14420/44000 | Loss: 3.0558 | LR: 3.87e-04 | 1379.1ms/step | 14.5 samples/s +Step 14430/44000 | Loss: 3.0387 | LR: 3.86e-04 | 1383.4ms/step | 14.5 samples/s +Step 14440/44000 | Loss: 3.0316 | LR: 3.86e-04 | 1382.2ms/step | 14.5 samples/s +Step 14450/44000 | Loss: 3.0545 | LR: 3.86e-04 | 1382.6ms/step | 14.5 samples/s +Step 14460/44000 | Loss: 3.0381 | LR: 3.86e-04 | 1384.0ms/step | 14.5 samples/s +Step 14470/44000 | Loss: 3.0713 | LR: 3.86e-04 | 1381.9ms/step | 14.5 samples/s +Step 14480/44000 | Loss: 3.0358 | LR: 3.86e-04 | 1382.3ms/step | 14.5 samples/s +Step 14490/44000 | Loss: 3.0262 | LR: 3.85e-04 | 1382.9ms/step | 14.5 samples/s +Step 14500/44000 | Loss: 3.0348 | LR: 3.85e-04 | 1384.2ms/step | 14.4 samples/s +Step 14510/44000 | Loss: 3.0211 | LR: 3.85e-04 | 1382.2ms/step | 14.5 samples/s +Step 14520/44000 | Loss: 3.0274 | LR: 3.85e-04 | 1380.5ms/step | 14.5 samples/s +Step 14530/44000 | Loss: 3.0248 | LR: 3.85e-04 | 1381.3ms/step | 14.5 samples/s +Step 14540/44000 | Loss: 3.0323 | LR: 3.85e-04 | 1381.5ms/step | 14.5 samples/s +Step 14550/44000 | Loss: 3.0271 | LR: 3.85e-04 | 1382.8ms/step | 14.5 samples/s +Step 14560/44000 | Loss: 3.0052 | LR: 3.84e-04 | 1383.2ms/step | 14.5 samples/s +Step 14570/44000 | Loss: 3.0016 | LR: 3.84e-04 | 1381.1ms/step | 14.5 samples/s +Step 14580/44000 | Loss: 3.0015 | LR: 3.84e-04 | 1383.1ms/step | 14.5 samples/s +Step 14590/44000 | Loss: 3.0262 | LR: 3.84e-04 | 1381.2ms/step | 14.5 samples/s +Step 14600/44000 | Loss: 3.0262 | LR: 3.84e-04 | 1381.1ms/step | 14.5 samples/s +Step 14610/44000 | Loss: 3.0369 | LR: 3.84e-04 | 1381.3ms/step | 14.5 samples/s +Step 14620/44000 | Loss: 3.0198 | LR: 3.83e-04 | 1383.5ms/step | 14.5 samples/s +Step 14630/44000 | Loss: 3.0277 | LR: 3.83e-04 | 1381.6ms/step | 14.5 samples/s +Step 14640/44000 | Loss: 3.0424 | LR: 3.83e-04 | 1382.7ms/step | 14.5 samples/s +Step 14650/44000 | Loss: 3.0167 | LR: 3.83e-04 | 1386.3ms/step | 14.4 samples/s +Step 14660/44000 | Loss: 3.0428 | LR: 3.83e-04 | 1382.3ms/step | 14.5 samples/s +Step 14670/44000 | Loss: 3.0339 | LR: 3.83e-04 | 1383.4ms/step | 14.5 samples/s +Step 14680/44000 | Loss: 3.0237 | LR: 3.83e-04 | 1383.9ms/step | 14.5 samples/s +Step 14690/44000 | Loss: 3.0304 | LR: 3.82e-04 | 1382.4ms/step | 14.5 samples/s +Step 14700/44000 | Loss: 3.0316 | LR: 3.82e-04 | 1382.5ms/step | 14.5 samples/s +Step 14710/44000 | Loss: 3.0364 | LR: 3.82e-04 | 1383.0ms/step | 14.5 samples/s +Step 14720/44000 | Loss: 3.0285 | LR: 3.82e-04 | 1383.0ms/step | 14.5 samples/s +Step 14730/44000 | Loss: 3.0419 | LR: 3.82e-04 | 1383.7ms/step | 14.5 samples/s +Step 14740/44000 | Loss: 3.0382 | LR: 3.82e-04 | 1383.9ms/step | 14.5 samples/s +Step 14750/44000 | Loss: 2.9881 | LR: 3.81e-04 | 1383.9ms/step | 14.5 samples/s +Step 14760/44000 | Loss: 3.0553 | LR: 3.81e-04 | 1380.1ms/step | 14.5 samples/s +Step 14770/44000 | Loss: 3.0577 | LR: 3.81e-04 | 1382.1ms/step | 14.5 samples/s +Step 14780/44000 | Loss: 3.0383 | LR: 3.81e-04 | 1381.9ms/step | 14.5 samples/s +Step 14790/44000 | Loss: 3.0158 | LR: 3.81e-04 | 1382.7ms/step | 14.5 samples/s +Step 14800/44000 | Loss: 3.0125 | LR: 3.81e-04 | 1383.1ms/step | 14.5 samples/s +Step 14810/44000 | Loss: 3.0505 | LR: 3.81e-04 | 1384.5ms/step | 14.4 samples/s +Step 14820/44000 | Loss: 3.0259 | LR: 3.80e-04 | 1382.8ms/step | 14.5 samples/s +Step 14830/44000 | Loss: 3.0332 | LR: 3.80e-04 | 1383.2ms/step | 14.5 samples/s +Step 14840/44000 | Loss: 3.0196 | LR: 3.80e-04 | 1383.7ms/step | 14.5 samples/s +Step 14850/44000 | Loss: 3.0481 | LR: 3.80e-04 | 1380.0ms/step | 14.5 samples/s +Step 14860/44000 | Loss: 3.0521 | LR: 3.80e-04 | 1381.9ms/step | 14.5 samples/s +Step 14870/44000 | Loss: 3.0343 | LR: 3.80e-04 | 1383.7ms/step | 14.5 samples/s +Step 14880/44000 | Loss: 3.0264 | LR: 3.79e-04 | 1381.9ms/step | 14.5 samples/s +Step 14890/44000 | Loss: 3.0287 | LR: 3.79e-04 | 1382.4ms/step | 14.5 samples/s +Step 14900/44000 | Loss: 3.0374 | LR: 3.79e-04 | 1382.5ms/step | 14.5 samples/s +Step 14910/44000 | Loss: 3.0206 | LR: 3.79e-04 | 1380.1ms/step | 14.5 samples/s +Step 14920/44000 | Loss: 3.0437 | LR: 3.79e-04 | 1382.1ms/step | 14.5 samples/s +Step 14930/44000 | Loss: 3.0468 | LR: 3.79e-04 | 1383.9ms/step | 14.5 samples/s +Step 14940/44000 | Loss: 3.0035 | LR: 3.79e-04 | 1383.5ms/step | 14.5 samples/s +Step 14950/44000 | Loss: 3.0613 | LR: 3.78e-04 | 1383.9ms/step | 14.5 samples/s +Step 14960/44000 | Loss: 3.0190 | LR: 3.78e-04 | 1380.5ms/step | 14.5 samples/s +Step 14970/44000 | Loss: 3.0510 | LR: 3.78e-04 | 1380.8ms/step | 14.5 samples/s +Step 14980/44000 | Loss: 3.0112 | LR: 3.78e-04 | 1380.9ms/step | 14.5 samples/s +Step 14990/44000 | Loss: 3.0359 | LR: 3.78e-04 | 1383.6ms/step | 14.5 samples/s +Step 15000/44000 | Loss: 3.0024 | LR: 3.78e-04 | 1382.7ms/step | 14.5 samples/s +Validation at step 15000: Loss = 3.0311, Perplexity = 20.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-15000 +Step 15010/44000 | Loss: 3.0154 | LR: 3.77e-04 | 3905.7ms/step | 5.1 samples/s +Step 15020/44000 | Loss: 3.0459 | LR: 3.77e-04 | 1389.1ms/step | 14.4 samples/s +Step 15030/44000 | Loss: 3.0015 | LR: 3.77e-04 | 1387.0ms/step | 14.4 samples/s +Step 15040/44000 | Loss: 3.0281 | LR: 3.77e-04 | 1395.4ms/step | 14.3 samples/s +Step 15050/44000 | Loss: 3.0318 | LR: 3.77e-04 | 1387.9ms/step | 14.4 samples/s +Step 15060/44000 | Loss: 3.0187 | LR: 3.77e-04 | 1388.2ms/step | 14.4 samples/s +Step 15070/44000 | Loss: 3.0174 | LR: 3.77e-04 | 1392.2ms/step | 14.4 samples/s +Step 15080/44000 | Loss: 3.0162 | LR: 3.76e-04 | 1393.8ms/step | 14.3 samples/s +Step 15090/44000 | Loss: 3.0184 | LR: 3.76e-04 | 1393.9ms/step | 14.3 samples/s +Step 15100/44000 | Loss: 3.0379 | LR: 3.76e-04 | 1395.5ms/step | 14.3 samples/s +Step 15110/44000 | Loss: 3.0052 | LR: 3.76e-04 | 1396.6ms/step | 14.3 samples/s +Step 15120/44000 | Loss: 3.0370 | LR: 3.76e-04 | 1394.0ms/step | 14.3 samples/s +Step 15130/44000 | Loss: 3.0509 | LR: 3.76e-04 | 1398.1ms/step | 14.3 samples/s +Step 15140/44000 | Loss: 3.0366 | LR: 3.75e-04 | 1393.8ms/step | 14.3 samples/s +Step 15150/44000 | Loss: 3.0408 | LR: 3.75e-04 | 1397.1ms/step | 14.3 samples/s +Step 15160/44000 | Loss: 3.0499 | LR: 3.75e-04 | 1398.9ms/step | 14.3 samples/s +Step 15170/44000 | Loss: 3.0475 | LR: 3.75e-04 | 1395.9ms/step | 14.3 samples/s +Step 15180/44000 | Loss: 3.0006 | LR: 3.75e-04 | 1399.6ms/step | 14.3 samples/s +Step 15190/44000 | Loss: 3.0402 | LR: 3.75e-04 | 1394.7ms/step | 14.3 samples/s +Step 15200/44000 | Loss: 3.0069 | LR: 3.74e-04 | 1400.4ms/step | 14.3 samples/s +Step 15210/44000 | Loss: 2.9984 | LR: 3.74e-04 | 1397.4ms/step | 14.3 samples/s +Step 15220/44000 | Loss: 3.0251 | LR: 3.74e-04 | 1394.2ms/step | 14.3 samples/s +Step 15230/44000 | Loss: 3.0220 | LR: 3.74e-04 | 1393.9ms/step | 14.3 samples/s +Step 15240/44000 | Loss: 3.0309 | LR: 3.74e-04 | 1391.8ms/step | 14.4 samples/s +Step 15250/44000 | Loss: 3.0045 | LR: 3.74e-04 | 1392.3ms/step | 14.4 samples/s +Step 15260/44000 | Loss: 3.0354 | LR: 3.74e-04 | 1396.0ms/step | 14.3 samples/s +Step 15270/44000 | Loss: 3.0393 | LR: 3.73e-04 | 1398.1ms/step | 14.3 samples/s +Step 15280/44000 | Loss: 3.0223 | LR: 3.73e-04 | 1397.8ms/step | 14.3 samples/s +Step 15290/44000 | Loss: 3.0307 | LR: 3.73e-04 | 1397.5ms/step | 14.3 samples/s +Step 15300/44000 | Loss: 3.0057 | LR: 3.73e-04 | 1396.4ms/step | 14.3 samples/s +Step 15310/44000 | Loss: 3.0119 | LR: 3.73e-04 | 1395.9ms/step | 14.3 samples/s +Step 15320/44000 | Loss: 3.0289 | LR: 3.73e-04 | 1396.3ms/step | 14.3 samples/s +Step 15330/44000 | Loss: 3.0405 | LR: 3.72e-04 | 1392.0ms/step | 14.4 samples/s +Step 15340/44000 | Loss: 3.0404 | LR: 3.72e-04 | 1396.4ms/step | 14.3 samples/s +Step 15350/44000 | Loss: 3.0310 | LR: 3.72e-04 | 1391.7ms/step | 14.4 samples/s +Step 15360/44000 | Loss: 3.0393 | LR: 3.72e-04 | 1393.6ms/step | 14.4 samples/s +Step 15370/44000 | Loss: 3.0226 | LR: 3.72e-04 | 1397.8ms/step | 14.3 samples/s +Step 15380/44000 | Loss: 3.0479 | LR: 3.72e-04 | 1398.5ms/step | 14.3 samples/s +Step 15390/44000 | Loss: 3.0179 | LR: 3.71e-04 | 1397.7ms/step | 14.3 samples/s +Step 15400/44000 | Loss: 3.0449 | LR: 3.71e-04 | 1398.1ms/step | 14.3 samples/s +Step 15410/44000 | Loss: 3.0025 | LR: 3.71e-04 | 1394.2ms/step | 14.3 samples/s +Step 15420/44000 | Loss: 3.0394 | LR: 3.71e-04 | 1393.8ms/step | 14.3 samples/s +Step 15430/44000 | Loss: 3.0173 | LR: 3.71e-04 | 1396.0ms/step | 14.3 samples/s +Step 15440/44000 | Loss: 3.0320 | LR: 3.71e-04 | 1397.8ms/step | 14.3 samples/s +Step 15450/44000 | Loss: 3.0389 | LR: 3.71e-04 | 1386.7ms/step | 14.4 samples/s +Step 15460/44000 | Loss: 3.0033 | LR: 3.70e-04 | 1383.2ms/step | 14.5 samples/s +Step 15470/44000 | Loss: 3.0039 | LR: 3.70e-04 | 1380.8ms/step | 14.5 samples/s +Step 15480/44000 | Loss: 3.0047 | LR: 3.70e-04 | 1381.7ms/step | 14.5 samples/s +Step 15490/44000 | Loss: 3.0137 | LR: 3.70e-04 | 1385.5ms/step | 14.4 samples/s +Step 15500/44000 | Loss: 3.0237 | LR: 3.70e-04 | 1382.8ms/step | 14.5 samples/s +Step 15510/44000 | Loss: 3.0418 | LR: 3.70e-04 | 1383.0ms/step | 14.5 samples/s +Step 15520/44000 | Loss: 3.0105 | LR: 3.69e-04 | 1382.6ms/step | 14.5 samples/s +Step 15530/44000 | Loss: 3.0415 | LR: 3.69e-04 | 1384.1ms/step | 14.5 samples/s +Step 15540/44000 | Loss: 3.0484 | LR: 3.69e-04 | 1384.1ms/step | 14.4 samples/s +Step 15550/44000 | Loss: 3.0181 | LR: 3.69e-04 | 1381.8ms/step | 14.5 samples/s +Step 15560/44000 | Loss: 3.0018 | LR: 3.69e-04 | 1381.8ms/step | 14.5 samples/s +Step 15570/44000 | Loss: 3.0029 | LR: 3.69e-04 | 1383.0ms/step | 14.5 samples/s +Step 15580/44000 | Loss: 3.0316 | LR: 3.68e-04 | 1381.7ms/step | 14.5 samples/s +Step 15590/44000 | Loss: 3.0127 | LR: 3.68e-04 | 1381.3ms/step | 14.5 samples/s +Step 15600/44000 | Loss: 3.0162 | LR: 3.68e-04 | 1382.6ms/step | 14.5 samples/s +Step 15610/44000 | Loss: 3.0102 | LR: 3.68e-04 | 1385.5ms/step | 14.4 samples/s +Step 15620/44000 | Loss: 3.0454 | LR: 3.68e-04 | 1385.0ms/step | 14.4 samples/s +Step 15630/44000 | Loss: 3.0184 | LR: 3.68e-04 | 1383.9ms/step | 14.5 samples/s +Step 15640/44000 | Loss: 2.9881 | LR: 3.67e-04 | 1383.0ms/step | 14.5 samples/s +Step 15650/44000 | Loss: 3.0266 | LR: 3.67e-04 | 1383.6ms/step | 14.5 samples/s +Step 15660/44000 | Loss: 2.9721 | LR: 3.67e-04 | 1381.1ms/step | 14.5 samples/s +Step 15670/44000 | Loss: 3.0388 | LR: 3.67e-04 | 1379.6ms/step | 14.5 samples/s +Step 15680/44000 | Loss: 3.0331 | LR: 3.67e-04 | 1382.7ms/step | 14.5 samples/s +Step 15690/44000 | Loss: 3.0214 | LR: 3.67e-04 | 1381.8ms/step | 14.5 samples/s +Step 15700/44000 | Loss: 3.0060 | LR: 3.67e-04 | 1385.2ms/step | 14.4 samples/s +Step 15710/44000 | Loss: 3.0150 | LR: 3.66e-04 | 1381.9ms/step | 14.5 samples/s +Step 15720/44000 | Loss: 3.0232 | LR: 3.66e-04 | 1380.9ms/step | 14.5 samples/s +Step 15730/44000 | Loss: 3.0138 | LR: 3.66e-04 | 1383.2ms/step | 14.5 samples/s +Step 15740/44000 | Loss: 3.0213 | LR: 3.66e-04 | 1382.0ms/step | 14.5 samples/s +Step 15750/44000 | Loss: 3.0062 | LR: 3.66e-04 | 1380.2ms/step | 14.5 samples/s +Step 15760/44000 | Loss: 2.9919 | LR: 3.66e-04 | 1383.9ms/step | 14.5 samples/s +Step 15770/44000 | Loss: 2.9964 | LR: 3.65e-04 | 1384.2ms/step | 14.4 samples/s +Step 15780/44000 | Loss: 3.0229 | LR: 3.65e-04 | 1383.7ms/step | 14.5 samples/s +Step 15790/44000 | Loss: 3.0417 | LR: 3.65e-04 | 1382.2ms/step | 14.5 samples/s +Step 15800/44000 | Loss: 3.0243 | LR: 3.65e-04 | 1383.0ms/step | 14.5 samples/s +Step 15810/44000 | Loss: 3.0098 | LR: 3.65e-04 | 1381.5ms/step | 14.5 samples/s +Step 15820/44000 | Loss: 3.0059 | LR: 3.65e-04 | 1382.3ms/step | 14.5 samples/s +Step 15830/44000 | Loss: 3.0036 | LR: 3.64e-04 | 1381.4ms/step | 14.5 samples/s +Step 15840/44000 | Loss: 3.0112 | LR: 3.64e-04 | 1382.7ms/step | 14.5 samples/s +Step 15850/44000 | Loss: 2.9943 | LR: 3.64e-04 | 1381.7ms/step | 14.5 samples/s +Step 15860/44000 | Loss: 2.9963 | LR: 3.64e-04 | 1381.1ms/step | 14.5 samples/s +Step 15870/44000 | Loss: 3.0264 | LR: 3.64e-04 | 1382.9ms/step | 14.5 samples/s +Step 15880/44000 | Loss: 3.0167 | LR: 3.64e-04 | 1379.6ms/step | 14.5 samples/s +Step 15890/44000 | Loss: 3.0253 | LR: 3.63e-04 | 1383.2ms/step | 14.5 samples/s +Step 15900/44000 | Loss: 3.0185 | LR: 3.63e-04 | 1380.4ms/step | 14.5 samples/s +Step 15910/44000 | Loss: 3.0075 | LR: 3.63e-04 | 1380.6ms/step | 14.5 samples/s +Step 15920/44000 | Loss: 3.0237 | LR: 3.63e-04 | 1383.7ms/step | 14.5 samples/s +Step 15930/44000 | Loss: 3.0164 | LR: 3.63e-04 | 1381.5ms/step | 14.5 samples/s +Step 15940/44000 | Loss: 3.0122 | LR: 3.63e-04 | 1382.6ms/step | 14.5 samples/s +Step 15950/44000 | Loss: 3.0095 | LR: 3.62e-04 | 1381.7ms/step | 14.5 samples/s +Step 15960/44000 | Loss: 3.0093 | LR: 3.62e-04 | 1380.0ms/step | 14.5 samples/s +Step 15970/44000 | Loss: 3.0030 | LR: 3.62e-04 | 1381.9ms/step | 14.5 samples/s +Step 15980/44000 | Loss: 3.0254 | LR: 3.62e-04 | 1382.2ms/step | 14.5 samples/s +Step 15990/44000 | Loss: 3.0162 | LR: 3.62e-04 | 1380.0ms/step | 14.5 samples/s +Step 16000/44000 | Loss: 3.0118 | LR: 3.62e-04 | 1380.2ms/step | 14.5 samples/s +Validation at step 16000: Loss = 3.0173, Perplexity = 20.44 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-16000 +Step 16010/44000 | Loss: 3.0096 | LR: 3.62e-04 | 3884.3ms/step | 5.1 samples/s +Step 16020/44000 | Loss: 3.0058 | LR: 3.61e-04 | 1390.9ms/step | 14.4 samples/s +Step 16030/44000 | Loss: 3.0324 | LR: 3.61e-04 | 1390.2ms/step | 14.4 samples/s +Step 16040/44000 | Loss: 2.9985 | LR: 3.61e-04 | 1387.1ms/step | 14.4 samples/s +Step 16050/44000 | Loss: 3.0099 | LR: 3.61e-04 | 1390.9ms/step | 14.4 samples/s +Step 16060/44000 | Loss: 2.9980 | LR: 3.61e-04 | 1387.1ms/step | 14.4 samples/s +Step 16070/44000 | Loss: 3.0020 | LR: 3.61e-04 | 1389.6ms/step | 14.4 samples/s +Step 16080/44000 | Loss: 3.0256 | LR: 3.60e-04 | 1390.1ms/step | 14.4 samples/s +Step 16090/44000 | Loss: 3.0010 | LR: 3.60e-04 | 1393.2ms/step | 14.4 samples/s +Step 16100/44000 | Loss: 3.0039 | LR: 3.60e-04 | 1393.9ms/step | 14.3 samples/s +Step 16110/44000 | Loss: 3.0026 | LR: 3.60e-04 | 1394.5ms/step | 14.3 samples/s +Step 16120/44000 | Loss: 3.0100 | LR: 3.60e-04 | 1395.4ms/step | 14.3 samples/s +Step 16130/44000 | Loss: 3.0298 | LR: 3.60e-04 | 1394.4ms/step | 14.3 samples/s +Step 16140/44000 | Loss: 3.0292 | LR: 3.59e-04 | 1398.0ms/step | 14.3 samples/s +Step 16150/44000 | Loss: 3.0112 | LR: 3.59e-04 | 1393.8ms/step | 14.3 samples/s +Step 16160/44000 | Loss: 3.0259 | LR: 3.59e-04 | 1398.1ms/step | 14.3 samples/s +Step 16170/44000 | Loss: 3.0327 | LR: 3.59e-04 | 1397.1ms/step | 14.3 samples/s +Step 16180/44000 | Loss: 3.0044 | LR: 3.59e-04 | 1396.9ms/step | 14.3 samples/s +Step 16190/44000 | Loss: 2.9992 | LR: 3.59e-04 | 1388.0ms/step | 14.4 samples/s +Step 16200/44000 | Loss: 2.9916 | LR: 3.58e-04 | 1380.0ms/step | 14.5 samples/s +Step 16210/44000 | Loss: 3.0107 | LR: 3.58e-04 | 1380.0ms/step | 14.5 samples/s +Step 16220/44000 | Loss: 3.0111 | LR: 3.58e-04 | 1381.8ms/step | 14.5 samples/s +Step 16230/44000 | Loss: 3.0175 | LR: 3.58e-04 | 1382.3ms/step | 14.5 samples/s +Step 16240/44000 | Loss: 3.0278 | LR: 3.58e-04 | 1382.0ms/step | 14.5 samples/s +Step 16250/44000 | Loss: 2.9954 | LR: 3.58e-04 | 1382.4ms/step | 14.5 samples/s +Step 16260/44000 | Loss: 2.9768 | LR: 3.57e-04 | 1381.6ms/step | 14.5 samples/s +Step 16270/44000 | Loss: 2.9961 | LR: 3.57e-04 | 1381.9ms/step | 14.5 samples/s +Step 16280/44000 | Loss: 3.0008 | LR: 3.57e-04 | 1381.8ms/step | 14.5 samples/s +Step 16290/44000 | Loss: 2.9926 | LR: 3.57e-04 | 1382.8ms/step | 14.5 samples/s +Step 16300/44000 | Loss: 2.9982 | LR: 3.57e-04 | 1381.9ms/step | 14.5 samples/s +Step 16310/44000 | Loss: 3.0197 | LR: 3.57e-04 | 1383.5ms/step | 14.5 samples/s +Step 16320/44000 | Loss: 3.0188 | LR: 3.56e-04 | 1380.8ms/step | 14.5 samples/s +Step 16330/44000 | Loss: 3.0053 | LR: 3.56e-04 | 1383.5ms/step | 14.5 samples/s +Step 16340/44000 | Loss: 3.0005 | LR: 3.56e-04 | 1384.1ms/step | 14.5 samples/s +Step 16350/44000 | Loss: 3.0162 | LR: 3.56e-04 | 1381.5ms/step | 14.5 samples/s +Step 16360/44000 | Loss: 3.0381 | LR: 3.56e-04 | 1382.6ms/step | 14.5 samples/s +Step 16370/44000 | Loss: 2.9824 | LR: 3.56e-04 | 1381.8ms/step | 14.5 samples/s +Step 16380/44000 | Loss: 2.9896 | LR: 3.55e-04 | 1383.5ms/step | 14.5 samples/s +Step 16390/44000 | Loss: 2.9913 | LR: 3.55e-04 | 1382.0ms/step | 14.5 samples/s +Step 16400/44000 | Loss: 3.0306 | LR: 3.55e-04 | 1381.5ms/step | 14.5 samples/s +Step 16410/44000 | Loss: 3.0181 | LR: 3.55e-04 | 1382.6ms/step | 14.5 samples/s +Step 16420/44000 | Loss: 3.0110 | LR: 3.55e-04 | 1380.8ms/step | 14.5 samples/s +Step 16430/44000 | Loss: 2.9916 | LR: 3.55e-04 | 1382.0ms/step | 14.5 samples/s +Step 16440/44000 | Loss: 3.0059 | LR: 3.54e-04 | 1379.7ms/step | 14.5 samples/s +Step 16450/44000 | Loss: 3.0211 | LR: 3.54e-04 | 1379.7ms/step | 14.5 samples/s +Step 16460/44000 | Loss: 3.0120 | LR: 3.54e-04 | 1381.1ms/step | 14.5 samples/s +Step 16470/44000 | Loss: 2.9991 | LR: 3.54e-04 | 1380.6ms/step | 14.5 samples/s +Step 16480/44000 | Loss: 2.9927 | LR: 3.54e-04 | 1380.7ms/step | 14.5 samples/s +Step 16490/44000 | Loss: 3.0051 | LR: 3.54e-04 | 1382.2ms/step | 14.5 samples/s +Step 16500/44000 | Loss: 2.9845 | LR: 3.53e-04 | 1385.3ms/step | 14.4 samples/s +Step 16510/44000 | Loss: 3.0184 | LR: 3.53e-04 | 1382.0ms/step | 14.5 samples/s +Step 16520/44000 | Loss: 3.0196 | LR: 3.53e-04 | 1382.2ms/step | 14.5 samples/s +Step 16530/44000 | Loss: 3.0493 | LR: 3.53e-04 | 1381.9ms/step | 14.5 samples/s +Step 16540/44000 | Loss: 3.0069 | LR: 3.53e-04 | 1380.6ms/step | 14.5 samples/s +Step 16550/44000 | Loss: 3.0061 | LR: 3.53e-04 | 1382.3ms/step | 14.5 samples/s +Step 16560/44000 | Loss: 3.0258 | LR: 3.52e-04 | 1384.2ms/step | 14.4 samples/s +Step 16570/44000 | Loss: 3.0314 | LR: 3.52e-04 | 1381.7ms/step | 14.5 samples/s +Step 16580/44000 | Loss: 3.0007 | LR: 3.52e-04 | 1381.4ms/step | 14.5 samples/s +Step 16590/44000 | Loss: 2.9939 | LR: 3.52e-04 | 1381.0ms/step | 14.5 samples/s +Step 16600/44000 | Loss: 2.9923 | LR: 3.52e-04 | 1380.6ms/step | 14.5 samples/s +Step 16610/44000 | Loss: 3.0024 | LR: 3.52e-04 | 1380.1ms/step | 14.5 samples/s +Step 16620/44000 | Loss: 3.0315 | LR: 3.51e-04 | 1382.2ms/step | 14.5 samples/s +Step 16630/44000 | Loss: 2.9957 | LR: 3.51e-04 | 1383.6ms/step | 14.5 samples/s +Step 16640/44000 | Loss: 2.9889 | LR: 3.51e-04 | 1381.2ms/step | 14.5 samples/s +Step 16650/44000 | Loss: 2.9832 | LR: 3.51e-04 | 1380.9ms/step | 14.5 samples/s +Step 16660/44000 | Loss: 3.0235 | LR: 3.51e-04 | 1383.8ms/step | 14.5 samples/s +Step 16670/44000 | Loss: 3.0150 | LR: 3.51e-04 | 1382.9ms/step | 14.5 samples/s +Step 16680/44000 | Loss: 2.9943 | LR: 3.50e-04 | 1383.2ms/step | 14.5 samples/s +Step 16690/44000 | Loss: 3.0117 | LR: 3.50e-04 | 1382.0ms/step | 14.5 samples/s +Step 16700/44000 | Loss: 2.9965 | LR: 3.50e-04 | 1382.4ms/step | 14.5 samples/s +Step 16710/44000 | Loss: 3.0252 | LR: 3.50e-04 | 1381.9ms/step | 14.5 samples/s +Step 16720/44000 | Loss: 3.0122 | LR: 3.50e-04 | 1382.0ms/step | 14.5 samples/s +Step 16730/44000 | Loss: 3.0275 | LR: 3.50e-04 | 1380.4ms/step | 14.5 samples/s +Step 16740/44000 | Loss: 2.9936 | LR: 3.49e-04 | 1379.6ms/step | 14.5 samples/s +Step 16750/44000 | Loss: 3.0155 | LR: 3.49e-04 | 1381.8ms/step | 14.5 samples/s +Step 16760/44000 | Loss: 2.9755 | LR: 3.49e-04 | 1381.9ms/step | 14.5 samples/s +Step 16770/44000 | Loss: 2.9897 | LR: 3.49e-04 | 1384.0ms/step | 14.5 samples/s +Step 16780/44000 | Loss: 3.0522 | LR: 3.49e-04 | 1383.5ms/step | 14.5 samples/s +Step 16790/44000 | Loss: 3.0116 | LR: 3.49e-04 | 1385.6ms/step | 14.4 samples/s +Step 16800/44000 | Loss: 2.9814 | LR: 3.48e-04 | 1383.1ms/step | 14.5 samples/s +Step 16810/44000 | Loss: 3.0145 | LR: 3.48e-04 | 1383.1ms/step | 14.5 samples/s +Step 16820/44000 | Loss: 3.0329 | LR: 3.48e-04 | 1382.8ms/step | 14.5 samples/s +Step 16830/44000 | Loss: 2.9916 | LR: 3.48e-04 | 1383.1ms/step | 14.5 samples/s +Step 16840/44000 | Loss: 2.9916 | LR: 3.48e-04 | 1382.9ms/step | 14.5 samples/s +Step 16850/44000 | Loss: 2.9933 | LR: 3.48e-04 | 1384.2ms/step | 14.4 samples/s +Step 16860/44000 | Loss: 2.9971 | LR: 3.47e-04 | 1383.2ms/step | 14.5 samples/s +Step 16870/44000 | Loss: 3.0203 | LR: 3.47e-04 | 1381.0ms/step | 14.5 samples/s +Step 16880/44000 | Loss: 3.0168 | LR: 3.47e-04 | 1381.3ms/step | 14.5 samples/s +Step 16890/44000 | Loss: 3.0029 | LR: 3.47e-04 | 1381.6ms/step | 14.5 samples/s +Step 16900/44000 | Loss: 2.9893 | LR: 3.47e-04 | 1380.8ms/step | 14.5 samples/s +Step 16910/44000 | Loss: 3.0246 | LR: 3.47e-04 | 1381.8ms/step | 14.5 samples/s +Step 16920/44000 | Loss: 3.0200 | LR: 3.46e-04 | 1380.5ms/step | 14.5 samples/s +Step 16930/44000 | Loss: 2.9938 | LR: 3.46e-04 | 1379.7ms/step | 14.5 samples/s +Step 16940/44000 | Loss: 3.0054 | LR: 3.46e-04 | 1385.0ms/step | 14.4 samples/s +Step 16950/44000 | Loss: 2.9910 | LR: 3.46e-04 | 1382.5ms/step | 14.5 samples/s +Step 16960/44000 | Loss: 2.9779 | LR: 3.46e-04 | 1381.2ms/step | 14.5 samples/s +Step 16970/44000 | Loss: 3.0061 | LR: 3.46e-04 | 1378.7ms/step | 14.5 samples/s +Step 16980/44000 | Loss: 3.0286 | LR: 3.45e-04 | 1381.2ms/step | 14.5 samples/s +Step 16990/44000 | Loss: 3.0181 | LR: 3.45e-04 | 1383.1ms/step | 14.5 samples/s +Step 17000/44000 | Loss: 3.0214 | LR: 3.45e-04 | 1383.6ms/step | 14.5 samples/s +Validation at step 17000: Loss = 3.0033, Perplexity = 20.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-17000 +Step 17010/44000 | Loss: 3.0014 | LR: 3.45e-04 | 3960.9ms/step | 5.0 samples/s +Step 17020/44000 | Loss: 3.0023 | LR: 3.45e-04 | 1387.7ms/step | 14.4 samples/s +Step 17030/44000 | Loss: 2.9887 | LR: 3.45e-04 | 1391.8ms/step | 14.4 samples/s +Step 17040/44000 | Loss: 3.0081 | LR: 3.44e-04 | 1385.9ms/step | 14.4 samples/s +Step 17050/44000 | Loss: 3.0109 | LR: 3.44e-04 | 1378.2ms/step | 14.5 samples/s +Step 17060/44000 | Loss: 2.9994 | LR: 3.44e-04 | 1380.6ms/step | 14.5 samples/s +Step 17070/44000 | Loss: 2.9990 | LR: 3.44e-04 | 1381.2ms/step | 14.5 samples/s +Step 17080/44000 | Loss: 3.0163 | LR: 3.44e-04 | 1381.8ms/step | 14.5 samples/s +Step 17090/44000 | Loss: 3.0045 | LR: 3.44e-04 | 1382.5ms/step | 14.5 samples/s +Step 17100/44000 | Loss: 3.0048 | LR: 3.43e-04 | 1381.2ms/step | 14.5 samples/s +Step 17110/44000 | Loss: 3.0118 | LR: 3.43e-04 | 1380.7ms/step | 14.5 samples/s +Step 17120/44000 | Loss: 2.9760 | LR: 3.43e-04 | 1380.1ms/step | 14.5 samples/s +Step 17130/44000 | Loss: 3.0005 | LR: 3.43e-04 | 1382.0ms/step | 14.5 samples/s +Step 17140/44000 | Loss: 2.9831 | LR: 3.43e-04 | 1381.9ms/step | 14.5 samples/s +Step 17150/44000 | Loss: 3.0132 | LR: 3.43e-04 | 1382.4ms/step | 14.5 samples/s +Step 17160/44000 | Loss: 2.9875 | LR: 3.42e-04 | 1379.4ms/step | 14.5 samples/s +Step 17170/44000 | Loss: 3.0163 | LR: 3.42e-04 | 1379.9ms/step | 14.5 samples/s +Step 17180/44000 | Loss: 3.0253 | LR: 3.42e-04 | 1382.0ms/step | 14.5 samples/s +Step 17190/44000 | Loss: 2.9860 | LR: 3.42e-04 | 1383.0ms/step | 14.5 samples/s +Step 17200/44000 | Loss: 3.0115 | LR: 3.42e-04 | 1382.0ms/step | 14.5 samples/s +Step 17210/44000 | Loss: 3.0037 | LR: 3.42e-04 | 1380.8ms/step | 14.5 samples/s +Step 17220/44000 | Loss: 2.9925 | LR: 3.41e-04 | 1384.0ms/step | 14.5 samples/s +Step 17230/44000 | Loss: 3.0244 | LR: 3.41e-04 | 1384.4ms/step | 14.4 samples/s +Step 17240/44000 | Loss: 2.9996 | LR: 3.41e-04 | 1379.9ms/step | 14.5 samples/s +Step 17250/44000 | Loss: 3.0261 | LR: 3.41e-04 | 1381.3ms/step | 14.5 samples/s +Step 17260/44000 | Loss: 2.9982 | LR: 3.41e-04 | 1381.6ms/step | 14.5 samples/s +Step 17270/44000 | Loss: 2.9750 | LR: 3.41e-04 | 1381.2ms/step | 14.5 samples/s +Step 17280/44000 | Loss: 3.0168 | LR: 3.40e-04 | 1381.6ms/step | 14.5 samples/s +Step 17290/44000 | Loss: 2.9897 | LR: 3.40e-04 | 1383.7ms/step | 14.5 samples/s +Step 17300/44000 | Loss: 2.9955 | LR: 3.40e-04 | 1382.7ms/step | 14.5 samples/s +Step 17310/44000 | Loss: 2.9770 | LR: 3.40e-04 | 1382.5ms/step | 14.5 samples/s +Step 17320/44000 | Loss: 2.9880 | LR: 3.40e-04 | 1381.8ms/step | 14.5 samples/s +Step 17330/44000 | Loss: 2.9879 | LR: 3.40e-04 | 1382.9ms/step | 14.5 samples/s +Step 17340/44000 | Loss: 3.0105 | LR: 3.39e-04 | 1381.6ms/step | 14.5 samples/s +Step 17350/44000 | Loss: 2.9734 | LR: 3.39e-04 | 1381.4ms/step | 14.5 samples/s +Step 17360/44000 | Loss: 3.0121 | LR: 3.39e-04 | 1384.1ms/step | 14.4 samples/s +Step 17370/44000 | Loss: 2.9960 | LR: 3.39e-04 | 1384.7ms/step | 14.4 samples/s +Step 17380/44000 | Loss: 2.9806 | LR: 3.39e-04 | 1382.8ms/step | 14.5 samples/s +Step 17390/44000 | Loss: 2.9971 | LR: 3.39e-04 | 1383.6ms/step | 14.5 samples/s +Step 17400/44000 | Loss: 3.0013 | LR: 3.38e-04 | 1382.5ms/step | 14.5 samples/s +Step 17410/44000 | Loss: 2.9969 | LR: 3.38e-04 | 1380.7ms/step | 14.5 samples/s +Step 17420/44000 | Loss: 2.9929 | LR: 3.38e-04 | 1381.3ms/step | 14.5 samples/s +Step 17430/44000 | Loss: 3.0156 | LR: 3.38e-04 | 1380.6ms/step | 14.5 samples/s +Step 17440/44000 | Loss: 3.0278 | LR: 3.38e-04 | 1381.6ms/step | 14.5 samples/s +Step 17450/44000 | Loss: 2.9866 | LR: 3.38e-04 | 1380.2ms/step | 14.5 samples/s +Step 17460/44000 | Loss: 3.0027 | LR: 3.37e-04 | 1381.0ms/step | 14.5 samples/s +Step 17470/44000 | Loss: 2.9944 | LR: 3.37e-04 | 1382.1ms/step | 14.5 samples/s +Step 17480/44000 | Loss: 3.0006 | LR: 3.37e-04 | 1383.9ms/step | 14.5 samples/s +Step 17490/44000 | Loss: 2.9942 | LR: 3.37e-04 | 1382.7ms/step | 14.5 samples/s +Step 17500/44000 | Loss: 2.9898 | LR: 3.37e-04 | 1383.7ms/step | 14.5 samples/s +Step 17510/44000 | Loss: 2.9653 | LR: 3.37e-04 | 1382.1ms/step | 14.5 samples/s +Step 17520/44000 | Loss: 2.9832 | LR: 3.36e-04 | 1380.3ms/step | 14.5 samples/s diff --git a/epfl_final/train_logs/ropectl_a1p0_h100_seed46_resume/seed46.20260820T144656Z.log b/epfl_final/train_logs/ropectl_a1p0_h100_seed46_resume/seed46.20260820T144656Z.log new file mode 100644 index 0000000000000000000000000000000000000000..5ed7039d93b904a72841e76d0eecdf72dcc410d3 --- /dev/null +++ b/epfl_final/train_logs/ropectl_a1p0_h100_seed46_resume/seed46.20260820T144656Z.log @@ -0,0 +1,3029 @@ +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-17000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-17000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 10, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 1000, + "eval_batches": 50, + "logging_steps": 10, + "seed": 46, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "l8k365m_dense_ropectl_a1p0_seed46", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "rope-control" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": 0.0, + "rope_train_position_scale_set": [ + 1.0, + 0.9896339488176222, + 0.9692159692159692, + 0.9420289855072463, + 0.8730158730158729 + ], + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-17000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729] + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 1 +Gradient accumulation steps: 10 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 1513882 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-17000... +Preserving checkpoint LR state for exact resume: current=[0.0003451463250063146], schedule_base=[0.0005] +Resumed from checkpoint at step 17000 (epoch 0, micro_step 170000) +Resuming training from step 17000 +Resuming wandb run zovy63wq +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_144758-zovy63wq +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run l8k365m_dense_ropectl_a1p0_seed46 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/zovy63wq +Starting training... +Validation at step 17000: Loss = 3.0033, Perplexity = 20.15 +wandb: WARNING Tried to log to step 17000 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 17001: dense (dense step interval) +Step 17010/44000 | Loss: 2.9642 | LR: 3.45e-04 | 31919.0ms/step | 0.6 samples/s +wandb: WARNING Tried to log to step 17010 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17020/44000 | Loss: 3.0297 | LR: 3.45e-04 | 1374.6ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17020 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17030/44000 | Loss: 3.0065 | LR: 3.45e-04 | 1373.8ms/step | 14.6 samples/s +wandb: WARNING Tried to log to step 17030 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17040/44000 | Loss: 3.0082 | LR: 3.44e-04 | 1378.6ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17040 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17050/44000 | Loss: 2.9940 | LR: 3.44e-04 | 1378.9ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17050 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17060/44000 | Loss: 3.0304 | LR: 3.44e-04 | 1380.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17060 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17070/44000 | Loss: 2.9750 | LR: 3.44e-04 | 1382.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17070 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17080/44000 | Loss: 3.0238 | LR: 3.44e-04 | 1383.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17080 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17090/44000 | Loss: 2.9963 | LR: 3.44e-04 | 1381.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17090 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17100/44000 | Loss: 3.0013 | LR: 3.43e-04 | 1383.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17100 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17110/44000 | Loss: 3.0031 | LR: 3.43e-04 | 1383.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17110 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17120/44000 | Loss: 3.0088 | LR: 3.43e-04 | 1383.6ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17120 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17130/44000 | Loss: 3.0075 | LR: 3.43e-04 | 1381.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17130 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17140/44000 | Loss: 3.0001 | LR: 3.43e-04 | 1383.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17140 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17150/44000 | Loss: 3.0049 | LR: 3.43e-04 | 1382.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17150 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17160/44000 | Loss: 2.9888 | LR: 3.42e-04 | 1381.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17160 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17170/44000 | Loss: 2.9943 | LR: 3.42e-04 | 1383.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17170 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17180/44000 | Loss: 3.0046 | LR: 3.42e-04 | 1381.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17180 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17190/44000 | Loss: 3.0087 | LR: 3.42e-04 | 1381.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17190 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17200/44000 | Loss: 3.0104 | LR: 3.42e-04 | 1382.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17200 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17210/44000 | Loss: 2.9978 | LR: 3.42e-04 | 1382.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17210 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17220/44000 | Loss: 2.9936 | LR: 3.41e-04 | 1380.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17220 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17230/44000 | Loss: 2.9668 | LR: 3.41e-04 | 1383.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17230 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17240/44000 | Loss: 2.9924 | LR: 3.41e-04 | 1381.2ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17240 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17250/44000 | Loss: 2.9975 | LR: 3.41e-04 | 1383.4ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17250 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17260/44000 | Loss: 3.0018 | LR: 3.41e-04 | 1383.4ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17260 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17270/44000 | Loss: 2.9945 | LR: 3.41e-04 | 1383.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17270 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17280/44000 | Loss: 3.0231 | LR: 3.40e-04 | 1384.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17280 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17290/44000 | Loss: 2.9777 | LR: 3.40e-04 | 1383.6ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17290 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17300/44000 | Loss: 3.0020 | LR: 3.40e-04 | 1382.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17300 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17310/44000 | Loss: 3.0270 | LR: 3.40e-04 | 1383.2ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17310 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17320/44000 | Loss: 3.0028 | LR: 3.40e-04 | 1380.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17320 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17330/44000 | Loss: 2.9919 | LR: 3.40e-04 | 1383.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17330 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17340/44000 | Loss: 2.9874 | LR: 3.39e-04 | 1383.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17340 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17350/44000 | Loss: 2.9986 | LR: 3.39e-04 | 1383.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17350 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17360/44000 | Loss: 2.9831 | LR: 3.39e-04 | 1384.8ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 17360 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17370/44000 | Loss: 3.0116 | LR: 3.39e-04 | 1385.3ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 17370 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17380/44000 | Loss: 2.9833 | LR: 3.39e-04 | 1382.0ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17380 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17390/44000 | Loss: 3.0089 | LR: 3.39e-04 | 1385.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 17390 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17400/44000 | Loss: 2.9766 | LR: 3.38e-04 | 1383.6ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17400 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17410/44000 | Loss: 2.9881 | LR: 3.38e-04 | 1382.8ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17410 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17420/44000 | Loss: 2.9861 | LR: 3.38e-04 | 1381.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17420 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17430/44000 | Loss: 2.9619 | LR: 3.38e-04 | 1380.3ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17430 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17440/44000 | Loss: 2.9808 | LR: 3.38e-04 | 1380.5ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17440 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17450/44000 | Loss: 2.9751 | LR: 3.38e-04 | 1385.0ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 17450 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17460/44000 | Loss: 3.0490 | LR: 3.37e-04 | 1382.7ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17460 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17470/44000 | Loss: 2.9784 | LR: 3.37e-04 | 1382.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17470 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17480/44000 | Loss: 2.9925 | LR: 3.37e-04 | 1385.7ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 17480 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17490/44000 | Loss: 3.0050 | LR: 3.37e-04 | 1384.1ms/step | 14.4 samples/s +wandb: WARNING Tried to log to step 17490 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17500/44000 | Loss: 2.9900 | LR: 3.37e-04 | 1384.1ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17500 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17510/44000 | Loss: 2.9955 | LR: 3.37e-04 | 1383.3ms/step | 14.5 samples/s +wandb: WARNING Tried to log to step 17510 that is less than the current step 17511. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 17520/44000 | Loss: 2.9949 | LR: 3.36e-04 | 1382.7ms/step | 14.5 samples/s +Step 17530/44000 | Loss: 3.0142 | LR: 3.36e-04 | 1384.4ms/step | 14.4 samples/s +Step 17540/44000 | Loss: 3.0001 | LR: 3.36e-04 | 1382.8ms/step | 14.5 samples/s +Step 17550/44000 | Loss: 2.9748 | LR: 3.36e-04 | 1383.0ms/step | 14.5 samples/s +Step 17560/44000 | Loss: 3.0041 | LR: 3.36e-04 | 1385.9ms/step | 14.4 samples/s +Step 17570/44000 | Loss: 3.0250 | LR: 3.35e-04 | 1383.9ms/step | 14.5 samples/s +Step 17580/44000 | Loss: 3.0189 | LR: 3.35e-04 | 1383.4ms/step | 14.5 samples/s +Step 17590/44000 | Loss: 3.0325 | LR: 3.35e-04 | 1384.5ms/step | 14.4 samples/s +Step 17600/44000 | Loss: 3.0237 | LR: 3.35e-04 | 1383.6ms/step | 14.5 samples/s +Step 17610/44000 | Loss: 3.0008 | LR: 3.35e-04 | 1386.4ms/step | 14.4 samples/s +Step 17620/44000 | Loss: 2.9851 | LR: 3.35e-04 | 1383.4ms/step | 14.5 samples/s +Step 17630/44000 | Loss: 3.0243 | LR: 3.34e-04 | 1382.5ms/step | 14.5 samples/s +Step 17640/44000 | Loss: 3.0047 | LR: 3.34e-04 | 1383.8ms/step | 14.5 samples/s +Step 17650/44000 | Loss: 3.0088 | LR: 3.34e-04 | 1381.5ms/step | 14.5 samples/s +Step 17660/44000 | Loss: 2.9737 | LR: 3.34e-04 | 1384.4ms/step | 14.4 samples/s +Step 17670/44000 | Loss: 3.0270 | LR: 3.34e-04 | 1384.9ms/step | 14.4 samples/s +Step 17680/44000 | Loss: 3.0206 | LR: 3.34e-04 | 1382.7ms/step | 14.5 samples/s +Step 17690/44000 | Loss: 2.9754 | LR: 3.33e-04 | 1384.6ms/step | 14.4 samples/s +Step 17700/44000 | Loss: 2.9994 | LR: 3.33e-04 | 1383.7ms/step | 14.5 samples/s +Step 17710/44000 | Loss: 3.0253 | LR: 3.33e-04 | 1384.2ms/step | 14.4 samples/s +Step 17720/44000 | Loss: 3.0111 | LR: 3.33e-04 | 1385.2ms/step | 14.4 samples/s +Step 17730/44000 | Loss: 3.0230 | LR: 3.33e-04 | 1382.6ms/step | 14.5 samples/s +Step 17740/44000 | Loss: 2.9955 | LR: 3.33e-04 | 1384.1ms/step | 14.5 samples/s +Step 17750/44000 | Loss: 2.9599 | LR: 3.32e-04 | 1383.0ms/step | 14.5 samples/s +Step 17760/44000 | Loss: 2.9776 | LR: 3.32e-04 | 1384.8ms/step | 14.4 samples/s +Step 17770/44000 | Loss: 2.9536 | LR: 3.32e-04 | 1384.2ms/step | 14.4 samples/s +Step 17780/44000 | Loss: 2.9980 | LR: 3.32e-04 | 1381.8ms/step | 14.5 samples/s +Step 17790/44000 | Loss: 2.9850 | LR: 3.32e-04 | 1382.6ms/step | 14.5 samples/s +Step 17800/44000 | Loss: 3.0003 | LR: 3.32e-04 | 1385.3ms/step | 14.4 samples/s +Step 17810/44000 | Loss: 2.9716 | LR: 3.31e-04 | 1383.3ms/step | 14.5 samples/s +Step 17820/44000 | Loss: 2.9787 | LR: 3.31e-04 | 1383.2ms/step | 14.5 samples/s +Step 17830/44000 | Loss: 2.9653 | LR: 3.31e-04 | 1383.2ms/step | 14.5 samples/s +Step 17840/44000 | Loss: 3.0131 | LR: 3.31e-04 | 1381.1ms/step | 14.5 samples/s +Step 17850/44000 | Loss: 2.9955 | LR: 3.31e-04 | 1386.9ms/step | 14.4 samples/s +Step 17860/44000 | Loss: 2.9888 | LR: 3.31e-04 | 1384.1ms/step | 14.4 samples/s +Step 17870/44000 | Loss: 2.9737 | LR: 3.30e-04 | 1381.3ms/step | 14.5 samples/s +Step 17880/44000 | Loss: 2.9864 | LR: 3.30e-04 | 1381.3ms/step | 14.5 samples/s +Step 17890/44000 | Loss: 2.9856 | LR: 3.30e-04 | 1382.9ms/step | 14.5 samples/s +Step 17900/44000 | Loss: 3.0081 | LR: 3.30e-04 | 1381.9ms/step | 14.5 samples/s +Step 17910/44000 | Loss: 2.9843 | LR: 3.30e-04 | 1381.6ms/step | 14.5 samples/s +Step 17920/44000 | Loss: 3.0270 | LR: 3.29e-04 | 1381.8ms/step | 14.5 samples/s +Step 17930/44000 | Loss: 2.9898 | LR: 3.29e-04 | 1382.4ms/step | 14.5 samples/s +Step 17940/44000 | Loss: 2.9911 | LR: 3.29e-04 | 1381.6ms/step | 14.5 samples/s +Step 17950/44000 | Loss: 2.9890 | LR: 3.29e-04 | 1382.2ms/step | 14.5 samples/s +Step 17960/44000 | Loss: 2.9800 | LR: 3.29e-04 | 1381.4ms/step | 14.5 samples/s +Step 17970/44000 | Loss: 2.9821 | LR: 3.29e-04 | 1386.0ms/step | 14.4 samples/s +Step 17980/44000 | Loss: 2.9838 | LR: 3.28e-04 | 1382.0ms/step | 14.5 samples/s +Step 17990/44000 | Loss: 2.9913 | LR: 3.28e-04 | 1380.7ms/step | 14.5 samples/s +Step 18000/44000 | Loss: 2.9585 | LR: 3.28e-04 | 1382.7ms/step | 14.5 samples/s +Validation at step 18000: Loss = 2.9920, Perplexity = 19.92 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-18000 +Step 18010/44000 | Loss: 2.9802 | LR: 3.28e-04 | 3905.8ms/step | 5.1 samples/s +Step 18020/44000 | Loss: 2.9821 | LR: 3.28e-04 | 1385.4ms/step | 14.4 samples/s +Step 18030/44000 | Loss: 2.9952 | LR: 3.28e-04 | 1386.0ms/step | 14.4 samples/s +Step 18040/44000 | Loss: 2.9801 | LR: 3.27e-04 | 1388.9ms/step | 14.4 samples/s +Step 18050/44000 | Loss: 2.9837 | LR: 3.27e-04 | 1387.9ms/step | 14.4 samples/s +Step 18060/44000 | Loss: 2.9803 | LR: 3.27e-04 | 1389.4ms/step | 14.4 samples/s +Step 18070/44000 | Loss: 2.9901 | LR: 3.27e-04 | 1389.9ms/step | 14.4 samples/s +Step 18080/44000 | Loss: 2.9828 | LR: 3.27e-04 | 1387.8ms/step | 14.4 samples/s +Step 18090/44000 | Loss: 3.0106 | LR: 3.27e-04 | 1388.7ms/step | 14.4 samples/s +Step 18100/44000 | Loss: 3.0122 | LR: 3.26e-04 | 1388.1ms/step | 14.4 samples/s +Step 18110/44000 | Loss: 2.9898 | LR: 3.26e-04 | 1387.5ms/step | 14.4 samples/s +Step 18120/44000 | Loss: 2.9889 | LR: 3.26e-04 | 1388.0ms/step | 14.4 samples/s +Step 18130/44000 | Loss: 3.0023 | LR: 3.26e-04 | 1388.5ms/step | 14.4 samples/s +Step 18140/44000 | Loss: 2.9755 | LR: 3.26e-04 | 1389.4ms/step | 14.4 samples/s +Step 18150/44000 | Loss: 2.9809 | LR: 3.26e-04 | 1387.4ms/step | 14.4 samples/s +Step 18160/44000 | Loss: 2.9939 | LR: 3.25e-04 | 1386.7ms/step | 14.4 samples/s +Step 18170/44000 | Loss: 3.0162 | LR: 3.25e-04 | 1387.5ms/step | 14.4 samples/s +Step 18180/44000 | Loss: 2.9785 | LR: 3.25e-04 | 1386.4ms/step | 14.4 samples/s +Step 18190/44000 | Loss: 3.0018 | LR: 3.25e-04 | 1392.0ms/step | 14.4 samples/s +Step 18200/44000 | Loss: 3.0063 | LR: 3.25e-04 | 1388.1ms/step | 14.4 samples/s +Step 18210/44000 | Loss: 2.9981 | LR: 3.24e-04 | 1389.7ms/step | 14.4 samples/s +Step 18220/44000 | Loss: 2.9820 | LR: 3.24e-04 | 1389.1ms/step | 14.4 samples/s +Step 18230/44000 | Loss: 2.9877 | LR: 3.24e-04 | 1385.8ms/step | 14.4 samples/s +Step 18240/44000 | Loss: 3.0137 | LR: 3.24e-04 | 1387.5ms/step | 14.4 samples/s +Step 18250/44000 | Loss: 2.9966 | LR: 3.24e-04 | 1389.8ms/step | 14.4 samples/s +Step 18260/44000 | Loss: 2.9985 | LR: 3.24e-04 | 1388.2ms/step | 14.4 samples/s +Step 18270/44000 | Loss: 2.9773 | LR: 3.23e-04 | 1389.6ms/step | 14.4 samples/s +Step 18280/44000 | Loss: 2.9761 | LR: 3.23e-04 | 1390.3ms/step | 14.4 samples/s +Step 18290/44000 | Loss: 2.9875 | LR: 3.23e-04 | 1390.1ms/step | 14.4 samples/s +Step 18300/44000 | Loss: 2.9904 | LR: 3.23e-04 | 1391.1ms/step | 14.4 samples/s +Step 18310/44000 | Loss: 2.9841 | LR: 3.23e-04 | 1386.8ms/step | 14.4 samples/s +Step 18320/44000 | Loss: 3.0119 | LR: 3.23e-04 | 1388.6ms/step | 14.4 samples/s +Step 18330/44000 | Loss: 3.0088 | LR: 3.22e-04 | 1387.2ms/step | 14.4 samples/s +Step 18340/44000 | Loss: 2.9864 | LR: 3.22e-04 | 1386.8ms/step | 14.4 samples/s +Step 18350/44000 | Loss: 2.9942 | LR: 3.22e-04 | 1388.5ms/step | 14.4 samples/s +Step 18360/44000 | Loss: 2.9857 | LR: 3.22e-04 | 1388.2ms/step | 14.4 samples/s +Step 18370/44000 | Loss: 2.9864 | LR: 3.22e-04 | 1388.7ms/step | 14.4 samples/s +Step 18380/44000 | Loss: 2.9838 | LR: 3.22e-04 | 1390.2ms/step | 14.4 samples/s +Step 18390/44000 | Loss: 2.9909 | LR: 3.21e-04 | 1389.4ms/step | 14.4 samples/s +Step 18400/44000 | Loss: 2.9757 | LR: 3.21e-04 | 1387.9ms/step | 14.4 samples/s +Step 18410/44000 | Loss: 2.9813 | LR: 3.21e-04 | 1388.7ms/step | 14.4 samples/s +Step 18420/44000 | Loss: 2.9766 | LR: 3.21e-04 | 1387.5ms/step | 14.4 samples/s +Step 18430/44000 | Loss: 3.0099 | LR: 3.21e-04 | 1386.1ms/step | 14.4 samples/s +Step 18440/44000 | Loss: 3.0043 | LR: 3.20e-04 | 1385.9ms/step | 14.4 samples/s +Step 18450/44000 | Loss: 2.9823 | LR: 3.20e-04 | 1386.3ms/step | 14.4 samples/s +Step 18460/44000 | Loss: 2.9861 | LR: 3.20e-04 | 1385.9ms/step | 14.4 samples/s +Step 18470/44000 | Loss: 2.9916 | LR: 3.20e-04 | 1385.8ms/step | 14.4 samples/s +Step 18480/44000 | Loss: 3.0039 | LR: 3.20e-04 | 1384.4ms/step | 14.4 samples/s +Step 18490/44000 | Loss: 2.9897 | LR: 3.20e-04 | 1388.0ms/step | 14.4 samples/s +Step 18500/44000 | Loss: 2.9996 | LR: 3.19e-04 | 1388.0ms/step | 14.4 samples/s +Step 18510/44000 | Loss: 2.9897 | LR: 3.19e-04 | 1388.2ms/step | 14.4 samples/s +Step 18520/44000 | Loss: 2.9828 | LR: 3.19e-04 | 1387.5ms/step | 14.4 samples/s +Step 18530/44000 | Loss: 2.9764 | LR: 3.19e-04 | 1387.3ms/step | 14.4 samples/s +Step 18540/44000 | Loss: 2.9655 | LR: 3.19e-04 | 1386.5ms/step | 14.4 samples/s +Step 18550/44000 | Loss: 2.9607 | LR: 3.19e-04 | 1387.7ms/step | 14.4 samples/s +Step 18560/44000 | Loss: 2.9668 | LR: 3.18e-04 | 1386.1ms/step | 14.4 samples/s +Step 18570/44000 | Loss: 3.0134 | LR: 3.18e-04 | 1386.7ms/step | 14.4 samples/s +Step 18580/44000 | Loss: 2.9951 | LR: 3.18e-04 | 1386.4ms/step | 14.4 samples/s +Step 18590/44000 | Loss: 2.9632 | LR: 3.18e-04 | 1387.0ms/step | 14.4 samples/s +Step 18600/44000 | Loss: 3.0039 | LR: 3.18e-04 | 1388.1ms/step | 14.4 samples/s +Step 18610/44000 | Loss: 2.9771 | LR: 3.18e-04 | 1388.4ms/step | 14.4 samples/s +Step 18620/44000 | Loss: 2.9843 | LR: 3.17e-04 | 1387.9ms/step | 14.4 samples/s +Step 18630/44000 | Loss: 2.9889 | LR: 3.17e-04 | 1388.2ms/step | 14.4 samples/s +Step 18640/44000 | Loss: 2.9660 | LR: 3.17e-04 | 1389.2ms/step | 14.4 samples/s +Step 18650/44000 | Loss: 2.9783 | LR: 3.17e-04 | 1388.3ms/step | 14.4 samples/s +Step 18660/44000 | Loss: 2.9915 | LR: 3.17e-04 | 1390.2ms/step | 14.4 samples/s +Step 18670/44000 | Loss: 2.9759 | LR: 3.16e-04 | 1391.9ms/step | 14.4 samples/s +Step 18680/44000 | Loss: 2.9838 | LR: 3.16e-04 | 1388.2ms/step | 14.4 samples/s +Step 18690/44000 | Loss: 2.9810 | LR: 3.16e-04 | 1387.7ms/step | 14.4 samples/s +Step 18700/44000 | Loss: 2.9536 | LR: 3.16e-04 | 1387.9ms/step | 14.4 samples/s +Step 18710/44000 | Loss: 2.9682 | LR: 3.16e-04 | 1386.6ms/step | 14.4 samples/s +Step 18720/44000 | Loss: 2.9810 | LR: 3.16e-04 | 1388.2ms/step | 14.4 samples/s +Step 18730/44000 | Loss: 3.0140 | LR: 3.15e-04 | 1386.4ms/step | 14.4 samples/s +Step 18740/44000 | Loss: 2.9775 | LR: 3.15e-04 | 1387.8ms/step | 14.4 samples/s +Step 18750/44000 | Loss: 3.0039 | LR: 3.15e-04 | 1386.6ms/step | 14.4 samples/s +Step 18760/44000 | Loss: 2.9928 | LR: 3.15e-04 | 1386.9ms/step | 14.4 samples/s +Step 18770/44000 | Loss: 2.9907 | LR: 3.15e-04 | 1386.8ms/step | 14.4 samples/s +Step 18780/44000 | Loss: 2.9823 | LR: 3.15e-04 | 1390.2ms/step | 14.4 samples/s +Step 18790/44000 | Loss: 2.9675 | LR: 3.14e-04 | 1386.5ms/step | 14.4 samples/s +Step 18800/44000 | Loss: 3.0123 | LR: 3.14e-04 | 1385.6ms/step | 14.4 samples/s +Step 18810/44000 | Loss: 2.9510 | LR: 3.14e-04 | 1390.1ms/step | 14.4 samples/s +Step 18820/44000 | Loss: 2.9901 | LR: 3.14e-04 | 1387.3ms/step | 14.4 samples/s +Step 18830/44000 | Loss: 3.0001 | LR: 3.14e-04 | 1387.9ms/step | 14.4 samples/s +Step 18840/44000 | Loss: 2.9577 | LR: 3.13e-04 | 1393.3ms/step | 14.4 samples/s +Step 18850/44000 | Loss: 2.9572 | LR: 3.13e-04 | 1388.6ms/step | 14.4 samples/s +Step 18860/44000 | Loss: 2.9607 | LR: 3.13e-04 | 1386.1ms/step | 14.4 samples/s +Step 18870/44000 | Loss: 2.9848 | LR: 3.13e-04 | 1390.2ms/step | 14.4 samples/s +Step 18880/44000 | Loss: 2.9621 | LR: 3.13e-04 | 1387.8ms/step | 14.4 samples/s +Step 18890/44000 | Loss: 2.9861 | LR: 3.13e-04 | 1387.9ms/step | 14.4 samples/s +Step 18900/44000 | Loss: 2.9813 | LR: 3.12e-04 | 1392.2ms/step | 14.4 samples/s +Step 18910/44000 | Loss: 2.9477 | LR: 3.12e-04 | 1389.4ms/step | 14.4 samples/s +Step 18920/44000 | Loss: 3.0066 | LR: 3.12e-04 | 1387.9ms/step | 14.4 samples/s +Step 18930/44000 | Loss: 2.9690 | LR: 3.12e-04 | 1390.6ms/step | 14.4 samples/s +Step 18940/44000 | Loss: 2.9938 | LR: 3.12e-04 | 1390.6ms/step | 14.4 samples/s +Step 18950/44000 | Loss: 2.9886 | LR: 3.12e-04 | 1389.9ms/step | 14.4 samples/s +Step 18960/44000 | Loss: 2.9550 | LR: 3.11e-04 | 1391.5ms/step | 14.4 samples/s +Step 18970/44000 | Loss: 2.9756 | LR: 3.11e-04 | 1387.0ms/step | 14.4 samples/s +Step 18980/44000 | Loss: 2.9944 | LR: 3.11e-04 | 1385.3ms/step | 14.4 samples/s +Step 18990/44000 | Loss: 2.9704 | LR: 3.11e-04 | 1390.9ms/step | 14.4 samples/s +Step 19000/44000 | Loss: 2.9652 | LR: 3.11e-04 | 1391.1ms/step | 14.4 samples/s +Validation at step 19000: Loss = 2.9797, Perplexity = 19.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-19000 +Step 19010/44000 | Loss: 3.0037 | LR: 3.11e-04 | 3898.8ms/step | 5.1 samples/s +Step 19020/44000 | Loss: 2.9950 | LR: 3.10e-04 | 1385.1ms/step | 14.4 samples/s +Step 19030/44000 | Loss: 2.9806 | LR: 3.10e-04 | 1384.7ms/step | 14.4 samples/s +Step 19040/44000 | Loss: 2.9931 | LR: 3.10e-04 | 1387.6ms/step | 14.4 samples/s +Step 19050/44000 | Loss: 2.9810 | LR: 3.10e-04 | 1385.8ms/step | 14.4 samples/s +Step 19060/44000 | Loss: 2.9950 | LR: 3.10e-04 | 1389.3ms/step | 14.4 samples/s +Step 19070/44000 | Loss: 2.9799 | LR: 3.09e-04 | 1386.1ms/step | 14.4 samples/s +Step 19080/44000 | Loss: 2.9776 | LR: 3.09e-04 | 1386.6ms/step | 14.4 samples/s +Step 19090/44000 | Loss: 2.9460 | LR: 3.09e-04 | 1390.1ms/step | 14.4 samples/s +Step 19100/44000 | Loss: 2.9689 | LR: 3.09e-04 | 1388.6ms/step | 14.4 samples/s +Step 19110/44000 | Loss: 2.9575 | LR: 3.09e-04 | 1387.9ms/step | 14.4 samples/s +Step 19120/44000 | Loss: 2.9811 | LR: 3.09e-04 | 1393.2ms/step | 14.4 samples/s +Step 19130/44000 | Loss: 2.9770 | LR: 3.08e-04 | 1388.9ms/step | 14.4 samples/s +Step 19140/44000 | Loss: 2.9490 | LR: 3.08e-04 | 1389.1ms/step | 14.4 samples/s +Step 19150/44000 | Loss: 2.9976 | LR: 3.08e-04 | 1388.3ms/step | 14.4 samples/s +Step 19160/44000 | Loss: 2.9580 | LR: 3.08e-04 | 1383.8ms/step | 14.5 samples/s +Step 19170/44000 | Loss: 2.9737 | LR: 3.08e-04 | 1388.2ms/step | 14.4 samples/s +Step 19180/44000 | Loss: 2.9598 | LR: 3.08e-04 | 1389.7ms/step | 14.4 samples/s +Step 19190/44000 | Loss: 2.9717 | LR: 3.07e-04 | 1386.0ms/step | 14.4 samples/s +Step 19200/44000 | Loss: 2.9724 | LR: 3.07e-04 | 1392.4ms/step | 14.4 samples/s +Step 19210/44000 | Loss: 2.9601 | LR: 3.07e-04 | 1388.1ms/step | 14.4 samples/s +Step 19220/44000 | Loss: 2.9703 | LR: 3.07e-04 | 1388.1ms/step | 14.4 samples/s +Step 19230/44000 | Loss: 2.9791 | LR: 3.07e-04 | 1388.0ms/step | 14.4 samples/s +Step 19240/44000 | Loss: 3.0026 | LR: 3.06e-04 | 1385.8ms/step | 14.4 samples/s +Step 19250/44000 | Loss: 2.9683 | LR: 3.06e-04 | 1388.7ms/step | 14.4 samples/s +Step 19260/44000 | Loss: 2.9750 | LR: 3.06e-04 | 1387.4ms/step | 14.4 samples/s +Step 19270/44000 | Loss: 2.9249 | LR: 3.06e-04 | 1389.2ms/step | 14.4 samples/s +Step 19280/44000 | Loss: 2.9411 | LR: 3.06e-04 | 1390.5ms/step | 14.4 samples/s +Step 19290/44000 | Loss: 2.9596 | LR: 3.06e-04 | 1388.2ms/step | 14.4 samples/s +Step 19300/44000 | Loss: 2.9519 | LR: 3.05e-04 | 1390.1ms/step | 14.4 samples/s +Step 19310/44000 | Loss: 2.9785 | LR: 3.05e-04 | 1391.2ms/step | 14.4 samples/s +Step 19320/44000 | Loss: 2.9622 | LR: 3.05e-04 | 1387.2ms/step | 14.4 samples/s +Step 19330/44000 | Loss: 2.9544 | LR: 3.05e-04 | 1389.2ms/step | 14.4 samples/s +Step 19340/44000 | Loss: 2.9744 | LR: 3.05e-04 | 1385.7ms/step | 14.4 samples/s +Step 19350/44000 | Loss: 3.0095 | LR: 3.04e-04 | 1387.4ms/step | 14.4 samples/s +Step 19360/44000 | Loss: 2.9793 | LR: 3.04e-04 | 1387.3ms/step | 14.4 samples/s +Step 19370/44000 | Loss: 2.9790 | LR: 3.04e-04 | 1383.9ms/step | 14.5 samples/s +Step 19380/44000 | Loss: 2.9352 | LR: 3.04e-04 | 1384.0ms/step | 14.5 samples/s +Step 19390/44000 | Loss: 2.9627 | LR: 3.04e-04 | 1387.7ms/step | 14.4 samples/s +Step 19400/44000 | Loss: 2.9455 | LR: 3.04e-04 | 1384.6ms/step | 14.4 samples/s +Step 19410/44000 | Loss: 2.9523 | LR: 3.03e-04 | 1388.0ms/step | 14.4 samples/s +Step 19420/44000 | Loss: 2.9682 | LR: 3.03e-04 | 1385.4ms/step | 14.4 samples/s +Step 19430/44000 | Loss: 2.9679 | LR: 3.03e-04 | 1386.2ms/step | 14.4 samples/s +Step 19440/44000 | Loss: 2.9518 | LR: 3.03e-04 | 1388.0ms/step | 14.4 samples/s +Step 19450/44000 | Loss: 2.9967 | LR: 3.03e-04 | 1385.7ms/step | 14.4 samples/s +Step 19460/44000 | Loss: 2.9941 | LR: 3.03e-04 | 1385.8ms/step | 14.4 samples/s +Step 19470/44000 | Loss: 2.9769 | LR: 3.02e-04 | 1383.3ms/step | 14.5 samples/s +Step 19480/44000 | Loss: 2.9845 | LR: 3.02e-04 | 1383.3ms/step | 14.5 samples/s +Step 19490/44000 | Loss: 2.9843 | LR: 3.02e-04 | 1384.4ms/step | 14.4 samples/s +Step 19500/44000 | Loss: 2.9643 | LR: 3.02e-04 | 1383.9ms/step | 14.5 samples/s +Step 19510/44000 | Loss: 2.9484 | LR: 3.02e-04 | 1383.8ms/step | 14.5 samples/s +Step 19520/44000 | Loss: 2.9890 | LR: 3.01e-04 | 1387.5ms/step | 14.4 samples/s +Step 19530/44000 | Loss: 2.9832 | LR: 3.01e-04 | 1386.4ms/step | 14.4 samples/s +Step 19540/44000 | Loss: 2.9778 | LR: 3.01e-04 | 1387.9ms/step | 14.4 samples/s +Step 19550/44000 | Loss: 2.9872 | LR: 3.01e-04 | 1388.1ms/step | 14.4 samples/s +Step 19560/44000 | Loss: 2.9477 | LR: 3.01e-04 | 1386.2ms/step | 14.4 samples/s +Step 19570/44000 | Loss: 2.9751 | LR: 3.01e-04 | 1387.9ms/step | 14.4 samples/s +Step 19580/44000 | Loss: 2.9679 | LR: 3.00e-04 | 1386.8ms/step | 14.4 samples/s +Step 19590/44000 | Loss: 2.9627 | LR: 3.00e-04 | 1385.9ms/step | 14.4 samples/s +Step 19600/44000 | Loss: 2.9758 | LR: 3.00e-04 | 1385.8ms/step | 14.4 samples/s +Step 19610/44000 | Loss: 2.9647 | LR: 3.00e-04 | 1386.9ms/step | 14.4 samples/s +Step 19620/44000 | Loss: 2.9836 | LR: 3.00e-04 | 1384.8ms/step | 14.4 samples/s +Step 19630/44000 | Loss: 2.9641 | LR: 3.00e-04 | 1387.9ms/step | 14.4 samples/s +Step 19640/44000 | Loss: 2.9763 | LR: 2.99e-04 | 1384.5ms/step | 14.4 samples/s +Step 19650/44000 | Loss: 2.9842 | LR: 2.99e-04 | 1387.6ms/step | 14.4 samples/s +Step 19660/44000 | Loss: 2.9818 | LR: 2.99e-04 | 1387.1ms/step | 14.4 samples/s +Step 19670/44000 | Loss: 2.9838 | LR: 2.99e-04 | 1383.9ms/step | 14.5 samples/s +Step 19680/44000 | Loss: 2.9478 | LR: 2.99e-04 | 1388.3ms/step | 14.4 samples/s +Step 19690/44000 | Loss: 2.9491 | LR: 2.98e-04 | 1388.3ms/step | 14.4 samples/s +Step 19700/44000 | Loss: 2.9662 | LR: 2.98e-04 | 1388.3ms/step | 14.4 samples/s +Step 19710/44000 | Loss: 2.9467 | LR: 2.98e-04 | 1385.8ms/step | 14.4 samples/s +Step 19720/44000 | Loss: 2.9631 | LR: 2.98e-04 | 1384.9ms/step | 14.4 samples/s +Step 19730/44000 | Loss: 2.9647 | LR: 2.98e-04 | 1389.1ms/step | 14.4 samples/s +Step 19740/44000 | Loss: 2.9757 | LR: 2.98e-04 | 1385.8ms/step | 14.4 samples/s +Step 19750/44000 | Loss: 2.9397 | LR: 2.97e-04 | 1386.3ms/step | 14.4 samples/s +Step 19760/44000 | Loss: 2.9776 | LR: 2.97e-04 | 1389.3ms/step | 14.4 samples/s +Step 19770/44000 | Loss: 2.9926 | LR: 2.97e-04 | 1385.9ms/step | 14.4 samples/s +Step 19780/44000 | Loss: 2.9722 | LR: 2.97e-04 | 1386.4ms/step | 14.4 samples/s +Step 19790/44000 | Loss: 2.9547 | LR: 2.97e-04 | 1387.4ms/step | 14.4 samples/s +Step 19800/44000 | Loss: 2.9472 | LR: 2.96e-04 | 1387.2ms/step | 14.4 samples/s +Step 19810/44000 | Loss: 2.9753 | LR: 2.96e-04 | 1388.4ms/step | 14.4 samples/s +Step 19820/44000 | Loss: 2.9857 | LR: 2.96e-04 | 1384.6ms/step | 14.4 samples/s +Step 19830/44000 | Loss: 2.9613 | LR: 2.96e-04 | 1385.8ms/step | 14.4 samples/s +Step 19840/44000 | Loss: 2.9778 | LR: 2.96e-04 | 1385.2ms/step | 14.4 samples/s +Step 19850/44000 | Loss: 2.9636 | LR: 2.96e-04 | 1386.0ms/step | 14.4 samples/s +Step 19860/44000 | Loss: 3.0251 | LR: 2.95e-04 | 1385.5ms/step | 14.4 samples/s +Step 19870/44000 | Loss: 2.9990 | LR: 2.95e-04 | 1386.1ms/step | 14.4 samples/s +Step 19880/44000 | Loss: 2.9745 | LR: 2.95e-04 | 1389.1ms/step | 14.4 samples/s +Step 19890/44000 | Loss: 2.9759 | LR: 2.95e-04 | 1385.2ms/step | 14.4 samples/s +Step 19900/44000 | Loss: 2.9724 | LR: 2.95e-04 | 1382.3ms/step | 14.5 samples/s +Step 19910/44000 | Loss: 2.9919 | LR: 2.95e-04 | 1384.1ms/step | 14.5 samples/s +Step 19920/44000 | Loss: 2.9972 | LR: 2.94e-04 | 1384.4ms/step | 14.4 samples/s +Step 19930/44000 | Loss: 2.9488 | LR: 2.94e-04 | 1387.0ms/step | 14.4 samples/s +Step 19940/44000 | Loss: 2.9730 | LR: 2.94e-04 | 1386.2ms/step | 14.4 samples/s +Step 19950/44000 | Loss: 2.9676 | LR: 2.94e-04 | 1386.2ms/step | 14.4 samples/s +Step 19960/44000 | Loss: 2.9747 | LR: 2.94e-04 | 1386.0ms/step | 14.4 samples/s +Step 19970/44000 | Loss: 2.9552 | LR: 2.93e-04 | 1387.5ms/step | 14.4 samples/s +Step 19980/44000 | Loss: 2.9554 | LR: 2.93e-04 | 1387.2ms/step | 14.4 samples/s +Step 19990/44000 | Loss: 2.9714 | LR: 2.93e-04 | 1386.6ms/step | 14.4 samples/s +Step 20000/44000 | Loss: 2.9805 | LR: 2.93e-04 | 1385.9ms/step | 14.4 samples/s +Validation at step 20000: Loss = 2.9692, Perplexity = 19.48 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-20000 +Step 20010/44000 | Loss: 2.9443 | LR: 2.93e-04 | 3867.5ms/step | 5.2 samples/s +Step 20020/44000 | Loss: 2.9888 | LR: 2.93e-04 | 1387.0ms/step | 14.4 samples/s +Step 20030/44000 | Loss: 2.9447 | LR: 2.92e-04 | 1386.1ms/step | 14.4 samples/s +Step 20040/44000 | Loss: 2.9775 | LR: 2.92e-04 | 1390.2ms/step | 14.4 samples/s +Step 20050/44000 | Loss: 2.9484 | LR: 2.92e-04 | 1387.4ms/step | 14.4 samples/s +Step 20060/44000 | Loss: 2.9457 | LR: 2.92e-04 | 1388.2ms/step | 14.4 samples/s +Step 20070/44000 | Loss: 2.9619 | LR: 2.92e-04 | 1388.3ms/step | 14.4 samples/s +Step 20080/44000 | Loss: 2.9770 | LR: 2.91e-04 | 1388.2ms/step | 14.4 samples/s +Step 20090/44000 | Loss: 2.9603 | LR: 2.91e-04 | 1386.1ms/step | 14.4 samples/s +Step 20100/44000 | Loss: 2.9697 | LR: 2.91e-04 | 1383.8ms/step | 14.5 samples/s +Step 20110/44000 | Loss: 2.9404 | LR: 2.91e-04 | 1385.9ms/step | 14.4 samples/s +Step 20120/44000 | Loss: 2.9737 | LR: 2.91e-04 | 1386.1ms/step | 14.4 samples/s +Step 20130/44000 | Loss: 2.9349 | LR: 2.91e-04 | 1384.8ms/step | 14.4 samples/s +Step 20140/44000 | Loss: 2.9659 | LR: 2.90e-04 | 1387.7ms/step | 14.4 samples/s +Step 20150/44000 | Loss: 2.9722 | LR: 2.90e-04 | 1387.2ms/step | 14.4 samples/s +Step 20160/44000 | Loss: 2.9590 | LR: 2.90e-04 | 1386.5ms/step | 14.4 samples/s +Step 20170/44000 | Loss: 2.9918 | LR: 2.90e-04 | 1387.9ms/step | 14.4 samples/s +Step 20180/44000 | Loss: 2.9949 | LR: 2.90e-04 | 1387.5ms/step | 14.4 samples/s +Step 20190/44000 | Loss: 2.9459 | LR: 2.90e-04 | 1384.6ms/step | 14.4 samples/s +Step 20200/44000 | Loss: 2.9776 | LR: 2.89e-04 | 1387.0ms/step | 14.4 samples/s +Step 20210/44000 | Loss: 2.9880 | LR: 2.89e-04 | 1385.6ms/step | 14.4 samples/s +Step 20220/44000 | Loss: 2.9940 | LR: 2.89e-04 | 1383.6ms/step | 14.5 samples/s +Step 20230/44000 | Loss: 2.9598 | LR: 2.89e-04 | 1385.4ms/step | 14.4 samples/s +Step 20240/44000 | Loss: 2.9473 | LR: 2.89e-04 | 1385.9ms/step | 14.4 samples/s +Step 20250/44000 | Loss: 2.9543 | LR: 2.88e-04 | 1384.4ms/step | 14.4 samples/s +Step 20260/44000 | Loss: 3.0005 | LR: 2.88e-04 | 1387.0ms/step | 14.4 samples/s +Step 20270/44000 | Loss: 2.9697 | LR: 2.88e-04 | 1387.0ms/step | 14.4 samples/s +Step 20280/44000 | Loss: 2.9565 | LR: 2.88e-04 | 1387.4ms/step | 14.4 samples/s +Step 20290/44000 | Loss: 2.9779 | LR: 2.88e-04 | 1384.6ms/step | 14.4 samples/s +Step 20300/44000 | Loss: 2.9701 | LR: 2.88e-04 | 1384.2ms/step | 14.4 samples/s +Step 20310/44000 | Loss: 2.9601 | LR: 2.87e-04 | 1386.3ms/step | 14.4 samples/s +Step 20320/44000 | Loss: 2.9439 | LR: 2.87e-04 | 1386.3ms/step | 14.4 samples/s +Step 20330/44000 | Loss: 2.9569 | LR: 2.87e-04 | 1386.9ms/step | 14.4 samples/s +Step 20340/44000 | Loss: 2.9560 | LR: 2.87e-04 | 1387.9ms/step | 14.4 samples/s +Step 20350/44000 | Loss: 2.9754 | LR: 2.87e-04 | 1386.5ms/step | 14.4 samples/s +Step 20360/44000 | Loss: 2.9569 | LR: 2.86e-04 | 1386.3ms/step | 14.4 samples/s +Step 20370/44000 | Loss: 2.9759 | LR: 2.86e-04 | 1387.2ms/step | 14.4 samples/s +Step 20380/44000 | Loss: 2.9690 | LR: 2.86e-04 | 1386.7ms/step | 14.4 samples/s +Step 20390/44000 | Loss: 2.9912 | LR: 2.86e-04 | 1389.2ms/step | 14.4 samples/s +Step 20400/44000 | Loss: 2.9879 | LR: 2.86e-04 | 1386.6ms/step | 14.4 samples/s +Step 20410/44000 | Loss: 2.9359 | LR: 2.86e-04 | 1386.0ms/step | 14.4 samples/s +Step 20420/44000 | Loss: 2.9693 | LR: 2.85e-04 | 1386.1ms/step | 14.4 samples/s +Step 20430/44000 | Loss: 2.9875 | LR: 2.85e-04 | 1384.7ms/step | 14.4 samples/s +Step 20440/44000 | Loss: 2.9416 | LR: 2.85e-04 | 1384.9ms/step | 14.4 samples/s +Step 20450/44000 | Loss: 2.9654 | LR: 2.85e-04 | 1386.1ms/step | 14.4 samples/s +Step 20460/44000 | Loss: 2.9440 | LR: 2.85e-04 | 1385.4ms/step | 14.4 samples/s +Step 20470/44000 | Loss: 2.9698 | LR: 2.84e-04 | 1388.7ms/step | 14.4 samples/s +Step 20480/44000 | Loss: 2.9659 | LR: 2.84e-04 | 1384.6ms/step | 14.4 samples/s +Step 20490/44000 | Loss: 2.9774 | LR: 2.84e-04 | 1384.0ms/step | 14.5 samples/s +Step 20500/44000 | Loss: 2.9320 | LR: 2.84e-04 | 1385.6ms/step | 14.4 samples/s +Step 20510/44000 | Loss: 2.9388 | LR: 2.84e-04 | 1384.0ms/step | 14.5 samples/s +Step 20520/44000 | Loss: 2.9607 | LR: 2.84e-04 | 1381.9ms/step | 14.5 samples/s +Step 20530/44000 | Loss: 2.9621 | LR: 2.83e-04 | 1384.1ms/step | 14.5 samples/s +Step 20540/44000 | Loss: 2.9526 | LR: 2.83e-04 | 1381.9ms/step | 14.5 samples/s +Step 20550/44000 | Loss: 2.9487 | LR: 2.83e-04 | 1383.4ms/step | 14.5 samples/s +Step 20560/44000 | Loss: 2.9743 | LR: 2.83e-04 | 1387.8ms/step | 14.4 samples/s +Step 20570/44000 | Loss: 2.9821 | LR: 2.83e-04 | 1385.9ms/step | 14.4 samples/s +Step 20580/44000 | Loss: 2.9452 | LR: 2.83e-04 | 1383.6ms/step | 14.5 samples/s +Step 20590/44000 | Loss: 2.9787 | LR: 2.82e-04 | 1387.0ms/step | 14.4 samples/s +Step 20600/44000 | Loss: 2.9480 | LR: 2.82e-04 | 1386.1ms/step | 14.4 samples/s +Step 20610/44000 | Loss: 2.9399 | LR: 2.82e-04 | 1388.2ms/step | 14.4 samples/s +Step 20620/44000 | Loss: 2.9196 | LR: 2.82e-04 | 1387.3ms/step | 14.4 samples/s +Step 20630/44000 | Loss: 2.9525 | LR: 2.82e-04 | 1386.8ms/step | 14.4 samples/s +Step 20640/44000 | Loss: 2.9843 | LR: 2.81e-04 | 1387.6ms/step | 14.4 samples/s +Step 20650/44000 | Loss: 2.9575 | LR: 2.81e-04 | 1385.0ms/step | 14.4 samples/s +Step 20660/44000 | Loss: 2.9487 | LR: 2.81e-04 | 1383.8ms/step | 14.5 samples/s +Step 20670/44000 | Loss: 2.9839 | LR: 2.81e-04 | 1388.0ms/step | 14.4 samples/s +Step 20680/44000 | Loss: 2.9511 | LR: 2.81e-04 | 1387.5ms/step | 14.4 samples/s +Step 20690/44000 | Loss: 2.9719 | LR: 2.81e-04 | 1385.4ms/step | 14.4 samples/s +Step 20700/44000 | Loss: 2.9499 | LR: 2.80e-04 | 1387.9ms/step | 14.4 samples/s +Step 20710/44000 | Loss: 2.9573 | LR: 2.80e-04 | 1385.6ms/step | 14.4 samples/s +Step 20720/44000 | Loss: 2.9374 | LR: 2.80e-04 | 1389.2ms/step | 14.4 samples/s +Step 20730/44000 | Loss: 2.9295 | LR: 2.80e-04 | 1386.2ms/step | 14.4 samples/s +Step 20740/44000 | Loss: 2.9911 | LR: 2.80e-04 | 1385.8ms/step | 14.4 samples/s +Step 20750/44000 | Loss: 2.9696 | LR: 2.79e-04 | 1386.2ms/step | 14.4 samples/s +Step 20760/44000 | Loss: 2.9564 | LR: 2.79e-04 | 1385.9ms/step | 14.4 samples/s +Step 20770/44000 | Loss: 2.9847 | LR: 2.79e-04 | 1387.9ms/step | 14.4 samples/s +Step 20780/44000 | Loss: 2.9602 | LR: 2.79e-04 | 1384.7ms/step | 14.4 samples/s +Step 20790/44000 | Loss: 2.9758 | LR: 2.79e-04 | 1385.3ms/step | 14.4 samples/s +Step 20800/44000 | Loss: 2.9695 | LR: 2.79e-04 | 1386.2ms/step | 14.4 samples/s +Step 20810/44000 | Loss: 2.9743 | LR: 2.78e-04 | 1385.0ms/step | 14.4 samples/s +Step 20820/44000 | Loss: 2.9569 | LR: 2.78e-04 | 1387.0ms/step | 14.4 samples/s +Step 20830/44000 | Loss: 2.9554 | LR: 2.78e-04 | 1384.9ms/step | 14.4 samples/s +Step 20840/44000 | Loss: 2.9299 | LR: 2.78e-04 | 1387.4ms/step | 14.4 samples/s +Step 20850/44000 | Loss: 2.9499 | LR: 2.78e-04 | 1390.1ms/step | 14.4 samples/s +Step 20860/44000 | Loss: 2.9728 | LR: 2.77e-04 | 1387.5ms/step | 14.4 samples/s +Step 20870/44000 | Loss: 2.9637 | LR: 2.77e-04 | 1387.8ms/step | 14.4 samples/s +Step 20880/44000 | Loss: 2.9692 | LR: 2.77e-04 | 1389.3ms/step | 14.4 samples/s +Step 20890/44000 | Loss: 2.9667 | LR: 2.77e-04 | 1387.1ms/step | 14.4 samples/s +Step 20900/44000 | Loss: 2.9444 | LR: 2.77e-04 | 1387.7ms/step | 14.4 samples/s +Step 20910/44000 | Loss: 2.9100 | LR: 2.77e-04 | 1386.7ms/step | 14.4 samples/s +Step 20920/44000 | Loss: 2.9777 | LR: 2.76e-04 | 1387.8ms/step | 14.4 samples/s +Step 20930/44000 | Loss: 2.9538 | LR: 2.76e-04 | 1388.3ms/step | 14.4 samples/s +Step 20940/44000 | Loss: 2.9945 | LR: 2.76e-04 | 1383.9ms/step | 14.5 samples/s +Step 20950/44000 | Loss: 2.9676 | LR: 2.76e-04 | 1386.7ms/step | 14.4 samples/s +Step 20960/44000 | Loss: 2.9542 | LR: 2.76e-04 | 1386.7ms/step | 14.4 samples/s +Step 20970/44000 | Loss: 2.9191 | LR: 2.75e-04 | 1388.2ms/step | 14.4 samples/s +Step 20980/44000 | Loss: 2.9403 | LR: 2.75e-04 | 1387.9ms/step | 14.4 samples/s +Step 20990/44000 | Loss: 2.9507 | LR: 2.75e-04 | 1388.6ms/step | 14.4 samples/s +Step 21000/44000 | Loss: 2.9650 | LR: 2.75e-04 | 1390.8ms/step | 14.4 samples/s +Validation at step 21000: Loss = 2.9587, Perplexity = 19.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-21000 +Step 21010/44000 | Loss: 2.9743 | LR: 2.75e-04 | 3887.9ms/step | 5.1 samples/s +Step 21020/44000 | Loss: 2.9537 | LR: 2.75e-04 | 1386.9ms/step | 14.4 samples/s +Step 21030/44000 | Loss: 2.9766 | LR: 2.74e-04 | 1386.7ms/step | 14.4 samples/s +Step 21040/44000 | Loss: 2.9925 | LR: 2.74e-04 | 1387.3ms/step | 14.4 samples/s +Step 21050/44000 | Loss: 2.9669 | LR: 2.74e-04 | 1388.1ms/step | 14.4 samples/s +Step 21060/44000 | Loss: 2.9492 | LR: 2.74e-04 | 1386.0ms/step | 14.4 samples/s +Step 21070/44000 | Loss: 2.9404 | LR: 2.74e-04 | 1390.1ms/step | 14.4 samples/s +Step 21080/44000 | Loss: 2.9459 | LR: 2.73e-04 | 1386.7ms/step | 14.4 samples/s +Step 21090/44000 | Loss: 2.9557 | LR: 2.73e-04 | 1387.2ms/step | 14.4 samples/s +Step 21100/44000 | Loss: 2.9286 | LR: 2.73e-04 | 1387.8ms/step | 14.4 samples/s +Step 21110/44000 | Loss: 2.9565 | LR: 2.73e-04 | 1388.0ms/step | 14.4 samples/s +Step 21120/44000 | Loss: 2.9104 | LR: 2.73e-04 | 1388.3ms/step | 14.4 samples/s +Step 21130/44000 | Loss: 2.9450 | LR: 2.73e-04 | 1385.9ms/step | 14.4 samples/s +Step 21140/44000 | Loss: 2.9730 | LR: 2.72e-04 | 1389.2ms/step | 14.4 samples/s +Step 21150/44000 | Loss: 2.9654 | LR: 2.72e-04 | 1387.1ms/step | 14.4 samples/s +Step 21160/44000 | Loss: 2.9679 | LR: 2.72e-04 | 1389.6ms/step | 14.4 samples/s +Step 21170/44000 | Loss: 2.9439 | LR: 2.72e-04 | 1388.5ms/step | 14.4 samples/s +Step 21180/44000 | Loss: 2.9380 | LR: 2.72e-04 | 1388.8ms/step | 14.4 samples/s +Step 21190/44000 | Loss: 2.9673 | LR: 2.71e-04 | 1388.5ms/step | 14.4 samples/s +Step 21200/44000 | Loss: 2.9355 | LR: 2.71e-04 | 1386.3ms/step | 14.4 samples/s +Step 21210/44000 | Loss: 2.9543 | LR: 2.71e-04 | 1388.4ms/step | 14.4 samples/s +Step 21220/44000 | Loss: 2.9613 | LR: 2.71e-04 | 1389.9ms/step | 14.4 samples/s +Step 21230/44000 | Loss: 2.9395 | LR: 2.71e-04 | 1388.6ms/step | 14.4 samples/s +Step 21240/44000 | Loss: 2.9581 | LR: 2.71e-04 | 1387.8ms/step | 14.4 samples/s +Step 21250/44000 | Loss: 2.9548 | LR: 2.70e-04 | 1387.8ms/step | 14.4 samples/s +Step 21260/44000 | Loss: 2.9778 | LR: 2.70e-04 | 1388.3ms/step | 14.4 samples/s +Step 21270/44000 | Loss: 2.9404 | LR: 2.70e-04 | 1387.3ms/step | 14.4 samples/s +Step 21280/44000 | Loss: 2.9617 | LR: 2.70e-04 | 1386.2ms/step | 14.4 samples/s +Step 21290/44000 | Loss: 2.9563 | LR: 2.70e-04 | 1387.6ms/step | 14.4 samples/s +Step 21300/44000 | Loss: 2.9531 | LR: 2.70e-04 | 1385.4ms/step | 14.4 samples/s +Step 21310/44000 | Loss: 2.9499 | LR: 2.69e-04 | 1387.5ms/step | 14.4 samples/s +Step 21320/44000 | Loss: 2.9258 | LR: 2.69e-04 | 1386.9ms/step | 14.4 samples/s +Step 21330/44000 | Loss: 2.9433 | LR: 2.69e-04 | 1386.5ms/step | 14.4 samples/s +Step 21340/44000 | Loss: 2.9712 | LR: 2.69e-04 | 1387.5ms/step | 14.4 samples/s +Step 21350/44000 | Loss: 2.9501 | LR: 2.69e-04 | 1387.5ms/step | 14.4 samples/s +Step 21360/44000 | Loss: 2.9651 | LR: 2.68e-04 | 1386.8ms/step | 14.4 samples/s +Step 21370/44000 | Loss: 2.9753 | LR: 2.68e-04 | 1388.0ms/step | 14.4 samples/s +Step 21380/44000 | Loss: 2.9513 | LR: 2.68e-04 | 1385.5ms/step | 14.4 samples/s +Step 21390/44000 | Loss: 2.9935 | LR: 2.68e-04 | 1386.7ms/step | 14.4 samples/s +Step 21400/44000 | Loss: 2.9955 | LR: 2.68e-04 | 1387.5ms/step | 14.4 samples/s +Step 21410/44000 | Loss: 2.9445 | LR: 2.68e-04 | 1388.3ms/step | 14.4 samples/s +Step 21420/44000 | Loss: 2.9565 | LR: 2.67e-04 | 1389.0ms/step | 14.4 samples/s +Step 21430/44000 | Loss: 2.9525 | LR: 2.67e-04 | 1389.0ms/step | 14.4 samples/s +Step 21440/44000 | Loss: 2.9461 | LR: 2.67e-04 | 1388.5ms/step | 14.4 samples/s +Step 21450/44000 | Loss: 2.9666 | LR: 2.67e-04 | 1387.6ms/step | 14.4 samples/s +Step 21460/44000 | Loss: 2.9631 | LR: 2.67e-04 | 1388.3ms/step | 14.4 samples/s +Step 21470/44000 | Loss: 2.9367 | LR: 2.66e-04 | 1388.0ms/step | 14.4 samples/s +Step 21480/44000 | Loss: 2.9428 | LR: 2.66e-04 | 1388.0ms/step | 14.4 samples/s +Step 21490/44000 | Loss: 2.9437 | LR: 2.66e-04 | 1387.1ms/step | 14.4 samples/s +Step 21500/44000 | Loss: 2.9623 | LR: 2.66e-04 | 1387.7ms/step | 14.4 samples/s +Step 21510/44000 | Loss: 2.9311 | LR: 2.66e-04 | 1389.9ms/step | 14.4 samples/s +Step 21520/44000 | Loss: 2.9437 | LR: 2.66e-04 | 1389.6ms/step | 14.4 samples/s +Step 21530/44000 | Loss: 2.9589 | LR: 2.65e-04 | 1385.1ms/step | 14.4 samples/s +Step 21540/44000 | Loss: 2.9547 | LR: 2.65e-04 | 1388.3ms/step | 14.4 samples/s +Step 21550/44000 | Loss: 2.9624 | LR: 2.65e-04 | 1385.6ms/step | 14.4 samples/s +Step 21560/44000 | Loss: 2.9522 | LR: 2.65e-04 | 1386.0ms/step | 14.4 samples/s +Step 21570/44000 | Loss: 2.9640 | LR: 2.65e-04 | 1386.3ms/step | 14.4 samples/s +Step 21580/44000 | Loss: 2.9661 | LR: 2.64e-04 | 1385.5ms/step | 14.4 samples/s +Step 21590/44000 | Loss: 2.9698 | LR: 2.64e-04 | 1387.5ms/step | 14.4 samples/s +Step 21600/44000 | Loss: 2.9405 | LR: 2.64e-04 | 1387.6ms/step | 14.4 samples/s +Step 21610/44000 | Loss: 2.9556 | LR: 2.64e-04 | 1388.8ms/step | 14.4 samples/s +Step 21620/44000 | Loss: 2.9351 | LR: 2.64e-04 | 1388.8ms/step | 14.4 samples/s +Step 21630/44000 | Loss: 2.9845 | LR: 2.64e-04 | 1389.8ms/step | 14.4 samples/s +Step 21640/44000 | Loss: 2.9410 | LR: 2.63e-04 | 1388.3ms/step | 14.4 samples/s +Step 21650/44000 | Loss: 2.9653 | LR: 2.63e-04 | 1389.4ms/step | 14.4 samples/s +Step 21660/44000 | Loss: 2.9610 | LR: 2.63e-04 | 1386.6ms/step | 14.4 samples/s +Step 21670/44000 | Loss: 2.9366 | LR: 2.63e-04 | 1387.4ms/step | 14.4 samples/s +Step 21680/44000 | Loss: 2.9217 | LR: 2.63e-04 | 1388.2ms/step | 14.4 samples/s +Step 21690/44000 | Loss: 2.9392 | LR: 2.62e-04 | 1387.3ms/step | 14.4 samples/s +Step 21700/44000 | Loss: 2.9475 | LR: 2.62e-04 | 1388.8ms/step | 14.4 samples/s +Step 21710/44000 | Loss: 2.9459 | LR: 2.62e-04 | 1387.8ms/step | 14.4 samples/s +Step 21720/44000 | Loss: 2.9461 | LR: 2.62e-04 | 1387.9ms/step | 14.4 samples/s +Step 21730/44000 | Loss: 2.9389 | LR: 2.62e-04 | 1386.7ms/step | 14.4 samples/s +Step 21740/44000 | Loss: 2.9386 | LR: 2.62e-04 | 1387.0ms/step | 14.4 samples/s +Step 21750/44000 | Loss: 2.9409 | LR: 2.61e-04 | 1387.2ms/step | 14.4 samples/s +Step 21760/44000 | Loss: 2.9642 | LR: 2.61e-04 | 1386.3ms/step | 14.4 samples/s +Step 21770/44000 | Loss: 2.9484 | LR: 2.61e-04 | 1387.5ms/step | 14.4 samples/s +Step 21780/44000 | Loss: 2.9524 | LR: 2.61e-04 | 1387.2ms/step | 14.4 samples/s +Step 21790/44000 | Loss: 2.9617 | LR: 2.61e-04 | 1387.1ms/step | 14.4 samples/s +Step 21800/44000 | Loss: 2.9201 | LR: 2.60e-04 | 1388.3ms/step | 14.4 samples/s +Step 21810/44000 | Loss: 2.9576 | LR: 2.60e-04 | 1389.2ms/step | 14.4 samples/s +Step 21820/44000 | Loss: 2.9551 | LR: 2.60e-04 | 1388.3ms/step | 14.4 samples/s +Step 21830/44000 | Loss: 2.9163 | LR: 2.60e-04 | 1387.8ms/step | 14.4 samples/s +Step 21840/44000 | Loss: 2.9760 | LR: 2.60e-04 | 1386.4ms/step | 14.4 samples/s +Step 21850/44000 | Loss: 2.9705 | LR: 2.60e-04 | 1385.5ms/step | 14.4 samples/s +Step 21860/44000 | Loss: 2.9263 | LR: 2.59e-04 | 1387.5ms/step | 14.4 samples/s +Step 21870/44000 | Loss: 2.9625 | LR: 2.59e-04 | 1387.5ms/step | 14.4 samples/s +Step 21880/44000 | Loss: 2.9545 | LR: 2.59e-04 | 1387.6ms/step | 14.4 samples/s +Step 21890/44000 | Loss: 2.9215 | LR: 2.59e-04 | 1388.8ms/step | 14.4 samples/s +Step 21900/44000 | Loss: 2.9005 | LR: 2.59e-04 | 1388.4ms/step | 14.4 samples/s +Step 21910/44000 | Loss: 2.9070 | LR: 2.58e-04 | 1388.7ms/step | 14.4 samples/s +Step 21920/44000 | Loss: 2.9470 | LR: 2.58e-04 | 1391.5ms/step | 14.4 samples/s +Step 21930/44000 | Loss: 2.9303 | LR: 2.58e-04 | 1388.2ms/step | 14.4 samples/s +Step 21940/44000 | Loss: 2.9609 | LR: 2.58e-04 | 1388.0ms/step | 14.4 samples/s +Step 21950/44000 | Loss: 2.9466 | LR: 2.58e-04 | 1386.0ms/step | 14.4 samples/s +Step 21960/44000 | Loss: 2.9388 | LR: 2.58e-04 | 1386.1ms/step | 14.4 samples/s +Step 21970/44000 | Loss: 2.9747 | LR: 2.57e-04 | 1389.9ms/step | 14.4 samples/s +Step 21980/44000 | Loss: 2.9432 | LR: 2.57e-04 | 1388.1ms/step | 14.4 samples/s +Step 21990/44000 | Loss: 2.9323 | LR: 2.57e-04 | 1387.8ms/step | 14.4 samples/s +Step 22000/44000 | Loss: 2.9291 | LR: 2.57e-04 | 1384.6ms/step | 14.4 samples/s +Validation at step 22000: Loss = 2.9481, Perplexity = 19.07 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-22000 +Step 22010/44000 | Loss: 2.9496 | LR: 2.57e-04 | 3905.3ms/step | 5.1 samples/s +Step 22020/44000 | Loss: 2.9629 | LR: 2.56e-04 | 1386.2ms/step | 14.4 samples/s +Step 22030/44000 | Loss: 2.9486 | LR: 2.56e-04 | 1390.1ms/step | 14.4 samples/s +Step 22040/44000 | Loss: 2.9196 | LR: 2.56e-04 | 1386.9ms/step | 14.4 samples/s +Step 22050/44000 | Loss: 2.9163 | LR: 2.56e-04 | 1385.8ms/step | 14.4 samples/s +Step 22060/44000 | Loss: 2.9438 | LR: 2.56e-04 | 1387.6ms/step | 14.4 samples/s +Step 22070/44000 | Loss: 2.9684 | LR: 2.56e-04 | 1386.9ms/step | 14.4 samples/s +Step 22080/44000 | Loss: 2.9456 | LR: 2.55e-04 | 1388.5ms/step | 14.4 samples/s +Step 22090/44000 | Loss: 2.9324 | LR: 2.55e-04 | 1388.4ms/step | 14.4 samples/s +Step 22100/44000 | Loss: 2.9363 | LR: 2.55e-04 | 1389.8ms/step | 14.4 samples/s +Step 22110/44000 | Loss: 2.9374 | LR: 2.55e-04 | 1388.4ms/step | 14.4 samples/s +Step 22120/44000 | Loss: 2.9485 | LR: 2.55e-04 | 1387.9ms/step | 14.4 samples/s +Step 22130/44000 | Loss: 2.9284 | LR: 2.54e-04 | 1386.1ms/step | 14.4 samples/s +Step 22140/44000 | Loss: 2.9446 | LR: 2.54e-04 | 1387.8ms/step | 14.4 samples/s +Step 22150/44000 | Loss: 2.9326 | LR: 2.54e-04 | 1386.1ms/step | 14.4 samples/s +Step 22160/44000 | Loss: 2.9136 | LR: 2.54e-04 | 1388.0ms/step | 14.4 samples/s +Step 22170/44000 | Loss: 2.9403 | LR: 2.54e-04 | 1387.3ms/step | 14.4 samples/s +Step 22180/44000 | Loss: 2.9431 | LR: 2.54e-04 | 1389.3ms/step | 14.4 samples/s +Step 22190/44000 | Loss: 2.9712 | LR: 2.53e-04 | 1387.2ms/step | 14.4 samples/s +Step 22200/44000 | Loss: 2.9393 | LR: 2.53e-04 | 1385.5ms/step | 14.4 samples/s +Step 22210/44000 | Loss: 2.9148 | LR: 2.53e-04 | 1389.3ms/step | 14.4 samples/s +Step 22220/44000 | Loss: 2.9651 | LR: 2.53e-04 | 1389.0ms/step | 14.4 samples/s +Step 22230/44000 | Loss: 2.9375 | LR: 2.53e-04 | 1389.8ms/step | 14.4 samples/s +Step 22240/44000 | Loss: 2.9508 | LR: 2.52e-04 | 1388.7ms/step | 14.4 samples/s +Step 22250/44000 | Loss: 2.9103 | LR: 2.52e-04 | 1388.2ms/step | 14.4 samples/s +Step 22260/44000 | Loss: 2.9437 | LR: 2.52e-04 | 1389.4ms/step | 14.4 samples/s +Step 22270/44000 | Loss: 2.9801 | LR: 2.52e-04 | 1388.8ms/step | 14.4 samples/s +Step 22280/44000 | Loss: 2.9752 | LR: 2.52e-04 | 1388.6ms/step | 14.4 samples/s +Step 22290/44000 | Loss: 2.9571 | LR: 2.52e-04 | 1389.3ms/step | 14.4 samples/s +Step 22300/44000 | Loss: 2.9521 | LR: 2.51e-04 | 1385.6ms/step | 14.4 samples/s +Step 22310/44000 | Loss: 2.9528 | LR: 2.51e-04 | 1386.3ms/step | 14.4 samples/s +Step 22320/44000 | Loss: 2.9466 | LR: 2.51e-04 | 1386.0ms/step | 14.4 samples/s +Step 22330/44000 | Loss: 2.9170 | LR: 2.51e-04 | 1385.7ms/step | 14.4 samples/s +Step 22340/44000 | Loss: 2.9448 | LR: 2.51e-04 | 1388.4ms/step | 14.4 samples/s +Step 22350/44000 | Loss: 2.9386 | LR: 2.50e-04 | 1387.2ms/step | 14.4 samples/s +Step 22360/44000 | Loss: 2.9305 | LR: 2.50e-04 | 1388.6ms/step | 14.4 samples/s +Step 22370/44000 | Loss: 2.9170 | LR: 2.50e-04 | 1387.6ms/step | 14.4 samples/s +Step 22380/44000 | Loss: 2.9418 | LR: 2.50e-04 | 1387.7ms/step | 14.4 samples/s +Step 22390/44000 | Loss: 2.9403 | LR: 2.50e-04 | 1388.1ms/step | 14.4 samples/s +Step 22400/44000 | Loss: 2.9468 | LR: 2.50e-04 | 1388.5ms/step | 14.4 samples/s +Step 22410/44000 | Loss: 2.9669 | LR: 2.49e-04 | 1390.9ms/step | 14.4 samples/s +Step 22420/44000 | Loss: 2.9469 | LR: 2.49e-04 | 1389.2ms/step | 14.4 samples/s +Step 22430/44000 | Loss: 2.9463 | LR: 2.49e-04 | 1387.5ms/step | 14.4 samples/s +Step 22440/44000 | Loss: 2.9499 | LR: 2.49e-04 | 1386.6ms/step | 14.4 samples/s +Step 22450/44000 | Loss: 2.9314 | LR: 2.49e-04 | 1387.5ms/step | 14.4 samples/s +Step 22460/44000 | Loss: 2.9326 | LR: 2.48e-04 | 1388.0ms/step | 14.4 samples/s +Step 22470/44000 | Loss: 2.9230 | LR: 2.48e-04 | 1388.2ms/step | 14.4 samples/s +Step 22480/44000 | Loss: 2.9011 | LR: 2.48e-04 | 1386.7ms/step | 14.4 samples/s +Step 22490/44000 | Loss: 2.9080 | LR: 2.48e-04 | 1386.6ms/step | 14.4 samples/s +Step 22500/44000 | Loss: 2.9346 | LR: 2.48e-04 | 1389.8ms/step | 14.4 samples/s +Step 22510/44000 | Loss: 2.9344 | LR: 2.48e-04 | 1387.8ms/step | 14.4 samples/s +Step 22520/44000 | Loss: 2.9348 | LR: 2.47e-04 | 1388.3ms/step | 14.4 samples/s +Step 22530/44000 | Loss: 2.9830 | LR: 2.47e-04 | 1389.8ms/step | 14.4 samples/s +Step 22540/44000 | Loss: 2.9463 | LR: 2.47e-04 | 1389.3ms/step | 14.4 samples/s +Step 22550/44000 | Loss: 2.9475 | LR: 2.47e-04 | 1384.8ms/step | 14.4 samples/s +Step 22560/44000 | Loss: 2.9274 | LR: 2.47e-04 | 1385.2ms/step | 14.4 samples/s +Step 22570/44000 | Loss: 2.9304 | LR: 2.46e-04 | 1386.6ms/step | 14.4 samples/s +Step 22580/44000 | Loss: 2.9393 | LR: 2.46e-04 | 1383.4ms/step | 14.5 samples/s +Step 22590/44000 | Loss: 2.9260 | LR: 2.46e-04 | 1383.6ms/step | 14.5 samples/s +Step 22600/44000 | Loss: 2.9469 | LR: 2.46e-04 | 1382.9ms/step | 14.5 samples/s +Step 22610/44000 | Loss: 2.9269 | LR: 2.46e-04 | 1382.4ms/step | 14.5 samples/s +Step 22620/44000 | Loss: 2.9547 | LR: 2.46e-04 | 1384.1ms/step | 14.5 samples/s +Step 22630/44000 | Loss: 2.9483 | LR: 2.45e-04 | 1383.3ms/step | 14.5 samples/s +Step 22640/44000 | Loss: 2.9647 | LR: 2.45e-04 | 1384.2ms/step | 14.4 samples/s +Step 22650/44000 | Loss: 2.9588 | LR: 2.45e-04 | 1387.0ms/step | 14.4 samples/s +Step 22660/44000 | Loss: 2.9209 | LR: 2.45e-04 | 1386.8ms/step | 14.4 samples/s +Step 22670/44000 | Loss: 2.9291 | LR: 2.45e-04 | 1385.8ms/step | 14.4 samples/s +Step 22680/44000 | Loss: 2.9197 | LR: 2.44e-04 | 1386.2ms/step | 14.4 samples/s +Step 22690/44000 | Loss: 2.9371 | LR: 2.44e-04 | 1388.5ms/step | 14.4 samples/s +Step 22700/44000 | Loss: 2.9719 | LR: 2.44e-04 | 1387.6ms/step | 14.4 samples/s +Step 22710/44000 | Loss: 2.9519 | LR: 2.44e-04 | 1385.7ms/step | 14.4 samples/s +Step 22720/44000 | Loss: 2.9458 | LR: 2.44e-04 | 1386.8ms/step | 14.4 samples/s +Step 22730/44000 | Loss: 2.9547 | LR: 2.44e-04 | 1385.1ms/step | 14.4 samples/s +Step 22740/44000 | Loss: 2.9423 | LR: 2.43e-04 | 1388.2ms/step | 14.4 samples/s +Step 22750/44000 | Loss: 2.9309 | LR: 2.43e-04 | 1386.1ms/step | 14.4 samples/s +Step 22760/44000 | Loss: 2.9407 | LR: 2.43e-04 | 1384.8ms/step | 14.4 samples/s +Step 22770/44000 | Loss: 2.9350 | LR: 2.43e-04 | 1385.0ms/step | 14.4 samples/s +Step 22780/44000 | Loss: 2.9275 | LR: 2.43e-04 | 1383.6ms/step | 14.5 samples/s +Step 22790/44000 | Loss: 2.9677 | LR: 2.42e-04 | 1383.8ms/step | 14.5 samples/s +Step 22800/44000 | Loss: 2.9491 | LR: 2.42e-04 | 1386.0ms/step | 14.4 samples/s +Step 22810/44000 | Loss: 2.9212 | LR: 2.42e-04 | 1384.7ms/step | 14.4 samples/s +Step 22820/44000 | Loss: 2.9380 | LR: 2.42e-04 | 1384.3ms/step | 14.4 samples/s +Step 22830/44000 | Loss: 2.9732 | LR: 2.42e-04 | 1386.0ms/step | 14.4 samples/s +Step 22840/44000 | Loss: 2.9391 | LR: 2.42e-04 | 1387.3ms/step | 14.4 samples/s +Step 22850/44000 | Loss: 2.9302 | LR: 2.41e-04 | 1386.3ms/step | 14.4 samples/s +Step 22860/44000 | Loss: 2.9171 | LR: 2.41e-04 | 1388.0ms/step | 14.4 samples/s +Step 22870/44000 | Loss: 2.9281 | LR: 2.41e-04 | 1387.9ms/step | 14.4 samples/s +Step 22880/44000 | Loss: 2.9510 | LR: 2.41e-04 | 1385.6ms/step | 14.4 samples/s +Step 22890/44000 | Loss: 2.9458 | LR: 2.41e-04 | 1387.0ms/step | 14.4 samples/s +Step 22900/44000 | Loss: 2.9458 | LR: 2.40e-04 | 1386.7ms/step | 14.4 samples/s +Step 22910/44000 | Loss: 2.9483 | LR: 2.40e-04 | 1386.3ms/step | 14.4 samples/s +Step 22920/44000 | Loss: 2.9165 | LR: 2.40e-04 | 1385.3ms/step | 14.4 samples/s +Step 22930/44000 | Loss: 2.9249 | LR: 2.40e-04 | 1384.8ms/step | 14.4 samples/s +Step 22940/44000 | Loss: 2.9421 | LR: 2.40e-04 | 1386.4ms/step | 14.4 samples/s +Step 22950/44000 | Loss: 2.9212 | LR: 2.40e-04 | 1386.2ms/step | 14.4 samples/s +Step 22960/44000 | Loss: 2.9432 | LR: 2.39e-04 | 1385.9ms/step | 14.4 samples/s +Step 22970/44000 | Loss: 2.9322 | LR: 2.39e-04 | 1384.0ms/step | 14.5 samples/s +Step 22980/44000 | Loss: 2.9432 | LR: 2.39e-04 | 1384.8ms/step | 14.4 samples/s +Step 22990/44000 | Loss: 2.9387 | LR: 2.39e-04 | 1385.4ms/step | 14.4 samples/s +Step 23000/44000 | Loss: 2.9375 | LR: 2.39e-04 | 1379.9ms/step | 14.5 samples/s +Validation at step 23000: Loss = 2.9391, Perplexity = 18.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-23000 +Step 23010/44000 | Loss: 2.9332 | LR: 2.38e-04 | 3880.2ms/step | 5.2 samples/s +Step 23020/44000 | Loss: 2.9275 | LR: 2.38e-04 | 1380.2ms/step | 14.5 samples/s +Step 23030/44000 | Loss: 2.9634 | LR: 2.38e-04 | 1380.8ms/step | 14.5 samples/s +Step 23040/44000 | Loss: 2.9218 | LR: 2.38e-04 | 1384.3ms/step | 14.4 samples/s +Step 23050/44000 | Loss: 2.9382 | LR: 2.38e-04 | 1384.6ms/step | 14.4 samples/s +Step 23060/44000 | Loss: 2.9260 | LR: 2.38e-04 | 1384.9ms/step | 14.4 samples/s +Step 23070/44000 | Loss: 2.9504 | LR: 2.37e-04 | 1385.9ms/step | 14.4 samples/s +Step 23080/44000 | Loss: 2.9145 | LR: 2.37e-04 | 1385.1ms/step | 14.4 samples/s +Step 23090/44000 | Loss: 2.9467 | LR: 2.37e-04 | 1385.4ms/step | 14.4 samples/s +Step 23100/44000 | Loss: 2.9377 | LR: 2.37e-04 | 1386.5ms/step | 14.4 samples/s +Step 23110/44000 | Loss: 2.9137 | LR: 2.37e-04 | 1386.0ms/step | 14.4 samples/s +Step 23120/44000 | Loss: 2.9240 | LR: 2.36e-04 | 1385.4ms/step | 14.4 samples/s +Step 23130/44000 | Loss: 2.9350 | LR: 2.36e-04 | 1387.1ms/step | 14.4 samples/s +Step 23140/44000 | Loss: 2.9322 | LR: 2.36e-04 | 1385.9ms/step | 14.4 samples/s +Step 23150/44000 | Loss: 2.9424 | LR: 2.36e-04 | 1387.1ms/step | 14.4 samples/s +Step 23160/44000 | Loss: 2.9629 | LR: 2.36e-04 | 1384.0ms/step | 14.5 samples/s +Step 23170/44000 | Loss: 2.9193 | LR: 2.36e-04 | 1388.4ms/step | 14.4 samples/s +Step 23180/44000 | Loss: 2.9516 | LR: 2.35e-04 | 1385.3ms/step | 14.4 samples/s +Step 23190/44000 | Loss: 2.9537 | LR: 2.35e-04 | 1386.9ms/step | 14.4 samples/s +Step 23200/44000 | Loss: 2.9049 | LR: 2.35e-04 | 1385.0ms/step | 14.4 samples/s +Step 23210/44000 | Loss: 2.9553 | LR: 2.35e-04 | 1385.4ms/step | 14.4 samples/s +Step 23220/44000 | Loss: 2.8991 | LR: 2.35e-04 | 1384.3ms/step | 14.4 samples/s +Step 23230/44000 | Loss: 2.9223 | LR: 2.34e-04 | 1385.7ms/step | 14.4 samples/s +Step 23240/44000 | Loss: 2.9539 | LR: 2.34e-04 | 1386.9ms/step | 14.4 samples/s +Step 23250/44000 | Loss: 2.9484 | LR: 2.34e-04 | 1385.1ms/step | 14.4 samples/s +Step 23260/44000 | Loss: 2.9361 | LR: 2.34e-04 | 1386.2ms/step | 14.4 samples/s +Step 23270/44000 | Loss: 2.9528 | LR: 2.34e-04 | 1387.7ms/step | 14.4 samples/s +Step 23280/44000 | Loss: 2.9307 | LR: 2.34e-04 | 1386.1ms/step | 14.4 samples/s +Step 23290/44000 | Loss: 2.9260 | LR: 2.33e-04 | 1385.5ms/step | 14.4 samples/s +Step 23300/44000 | Loss: 2.9374 | LR: 2.33e-04 | 1383.0ms/step | 14.5 samples/s +Step 23310/44000 | Loss: 2.9322 | LR: 2.33e-04 | 1384.8ms/step | 14.4 samples/s +Step 23320/44000 | Loss: 2.9287 | LR: 2.33e-04 | 1384.6ms/step | 14.4 samples/s +Step 23330/44000 | Loss: 2.9159 | LR: 2.33e-04 | 1385.8ms/step | 14.4 samples/s +Step 23340/44000 | Loss: 2.9524 | LR: 2.32e-04 | 1384.5ms/step | 14.4 samples/s +Step 23350/44000 | Loss: 2.9257 | LR: 2.32e-04 | 1385.9ms/step | 14.4 samples/s +Step 23360/44000 | Loss: 2.9249 | LR: 2.32e-04 | 1385.5ms/step | 14.4 samples/s +Step 23370/44000 | Loss: 2.9170 | LR: 2.32e-04 | 1385.8ms/step | 14.4 samples/s +Step 23380/44000 | Loss: 2.9345 | LR: 2.32e-04 | 1385.0ms/step | 14.4 samples/s +Step 23390/44000 | Loss: 2.9274 | LR: 2.32e-04 | 1384.0ms/step | 14.5 samples/s +Step 23400/44000 | Loss: 2.9651 | LR: 2.31e-04 | 1386.0ms/step | 14.4 samples/s +Step 23410/44000 | Loss: 2.9021 | LR: 2.31e-04 | 1384.3ms/step | 14.4 samples/s +Step 23420/44000 | Loss: 2.9235 | LR: 2.31e-04 | 1385.4ms/step | 14.4 samples/s +Step 23430/44000 | Loss: 2.9531 | LR: 2.31e-04 | 1384.4ms/step | 14.4 samples/s +Step 23440/44000 | Loss: 2.9086 | LR: 2.31e-04 | 1385.4ms/step | 14.4 samples/s +Step 23450/44000 | Loss: 2.9245 | LR: 2.30e-04 | 1385.7ms/step | 14.4 samples/s +Step 23460/44000 | Loss: 2.8867 | LR: 2.30e-04 | 1386.1ms/step | 14.4 samples/s +Step 23470/44000 | Loss: 2.9497 | LR: 2.30e-04 | 1384.8ms/step | 14.4 samples/s +Step 23480/44000 | Loss: 2.9416 | LR: 2.30e-04 | 1384.0ms/step | 14.5 samples/s +Step 23490/44000 | Loss: 2.9554 | LR: 2.30e-04 | 1385.0ms/step | 14.4 samples/s +Step 23500/44000 | Loss: 2.9052 | LR: 2.30e-04 | 1384.6ms/step | 14.4 samples/s +Step 23510/44000 | Loss: 2.9390 | LR: 2.29e-04 | 1385.8ms/step | 14.4 samples/s +Step 23520/44000 | Loss: 2.9274 | LR: 2.29e-04 | 1386.0ms/step | 14.4 samples/s +Step 23530/44000 | Loss: 2.9351 | LR: 2.29e-04 | 1386.0ms/step | 14.4 samples/s +Step 23540/44000 | Loss: 2.9362 | LR: 2.29e-04 | 1385.8ms/step | 14.4 samples/s +Step 23550/44000 | Loss: 2.9679 | LR: 2.29e-04 | 1385.6ms/step | 14.4 samples/s +Step 23560/44000 | Loss: 2.9360 | LR: 2.29e-04 | 1385.0ms/step | 14.4 samples/s +Step 23570/44000 | Loss: 2.9016 | LR: 2.28e-04 | 1389.8ms/step | 14.4 samples/s +Step 23580/44000 | Loss: 2.8946 | LR: 2.28e-04 | 1384.8ms/step | 14.4 samples/s +Step 23590/44000 | Loss: 2.9715 | LR: 2.28e-04 | 1385.7ms/step | 14.4 samples/s +Step 23600/44000 | Loss: 2.9244 | LR: 2.28e-04 | 1385.5ms/step | 14.4 samples/s +Step 23610/44000 | Loss: 2.9110 | LR: 2.28e-04 | 1384.5ms/step | 14.4 samples/s +Step 23620/44000 | Loss: 2.9300 | LR: 2.27e-04 | 1383.4ms/step | 14.5 samples/s +Step 23630/44000 | Loss: 2.8991 | LR: 2.27e-04 | 1383.9ms/step | 14.5 samples/s +Step 23640/44000 | Loss: 2.9382 | LR: 2.27e-04 | 1384.1ms/step | 14.4 samples/s +Step 23650/44000 | Loss: 2.9354 | LR: 2.27e-04 | 1385.8ms/step | 14.4 samples/s +Step 23660/44000 | Loss: 2.9176 | LR: 2.27e-04 | 1384.2ms/step | 14.4 samples/s +Step 23670/44000 | Loss: 2.9272 | LR: 2.27e-04 | 1384.0ms/step | 14.5 samples/s +Step 23680/44000 | Loss: 2.9352 | LR: 2.26e-04 | 1383.9ms/step | 14.5 samples/s +Step 23690/44000 | Loss: 2.9236 | LR: 2.26e-04 | 1385.0ms/step | 14.4 samples/s +Step 23700/44000 | Loss: 2.9366 | LR: 2.26e-04 | 1384.7ms/step | 14.4 samples/s +Step 23710/44000 | Loss: 2.9492 | LR: 2.26e-04 | 1384.9ms/step | 14.4 samples/s +Step 23720/44000 | Loss: 2.9191 | LR: 2.26e-04 | 1385.5ms/step | 14.4 samples/s +Step 23730/44000 | Loss: 2.9265 | LR: 2.25e-04 | 1383.4ms/step | 14.5 samples/s +Step 23740/44000 | Loss: 2.9226 | LR: 2.25e-04 | 1384.5ms/step | 14.4 samples/s +Step 23750/44000 | Loss: 2.9252 | LR: 2.25e-04 | 1384.9ms/step | 14.4 samples/s +Step 23760/44000 | Loss: 2.9359 | LR: 2.25e-04 | 1385.1ms/step | 14.4 samples/s +Step 23770/44000 | Loss: 2.9509 | LR: 2.25e-04 | 1384.7ms/step | 14.4 samples/s +Step 23780/44000 | Loss: 2.9716 | LR: 2.25e-04 | 1385.8ms/step | 14.4 samples/s +Step 23790/44000 | Loss: 2.9317 | LR: 2.24e-04 | 1383.9ms/step | 14.5 samples/s +Step 23800/44000 | Loss: 2.9674 | LR: 2.24e-04 | 1383.9ms/step | 14.5 samples/s +Step 23810/44000 | Loss: 2.9234 | LR: 2.24e-04 | 1383.2ms/step | 14.5 samples/s +Step 23820/44000 | Loss: 2.9354 | LR: 2.24e-04 | 1385.4ms/step | 14.4 samples/s +Step 23830/44000 | Loss: 2.9127 | LR: 2.24e-04 | 1382.1ms/step | 14.5 samples/s +Step 23840/44000 | Loss: 2.9228 | LR: 2.23e-04 | 1384.8ms/step | 14.4 samples/s +Step 23850/44000 | Loss: 2.9367 | LR: 2.23e-04 | 1383.6ms/step | 14.5 samples/s +Step 23860/44000 | Loss: 2.9214 | LR: 2.23e-04 | 1386.0ms/step | 14.4 samples/s +Step 23870/44000 | Loss: 2.9294 | LR: 2.23e-04 | 1386.3ms/step | 14.4 samples/s +Step 23880/44000 | Loss: 2.9105 | LR: 2.23e-04 | 1385.5ms/step | 14.4 samples/s +Step 23890/44000 | Loss: 2.9248 | LR: 2.23e-04 | 1386.9ms/step | 14.4 samples/s +Step 23900/44000 | Loss: 2.9668 | LR: 2.22e-04 | 1385.3ms/step | 14.4 samples/s +Step 23910/44000 | Loss: 2.9444 | LR: 2.22e-04 | 1384.8ms/step | 14.4 samples/s +Step 23920/44000 | Loss: 2.9385 | LR: 2.22e-04 | 1385.1ms/step | 14.4 samples/s +Step 23930/44000 | Loss: 2.9247 | LR: 2.22e-04 | 1384.5ms/step | 14.4 samples/s +Step 23940/44000 | Loss: 2.9376 | LR: 2.22e-04 | 1384.7ms/step | 14.4 samples/s +Step 23950/44000 | Loss: 2.9386 | LR: 2.21e-04 | 1385.4ms/step | 14.4 samples/s +Step 23960/44000 | Loss: 2.9432 | LR: 2.21e-04 | 1384.6ms/step | 14.4 samples/s +Step 23970/44000 | Loss: 2.9288 | LR: 2.21e-04 | 1385.2ms/step | 14.4 samples/s +Step 23980/44000 | Loss: 2.9214 | LR: 2.21e-04 | 1387.0ms/step | 14.4 samples/s +Step 23990/44000 | Loss: 2.9261 | LR: 2.21e-04 | 1388.2ms/step | 14.4 samples/s +Step 24000/44000 | Loss: 2.9176 | LR: 2.21e-04 | 1384.9ms/step | 14.4 samples/s +Validation at step 24000: Loss = 2.9302, Perplexity = 18.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-24000 +Step 24010/44000 | Loss: 2.9412 | LR: 2.20e-04 | 3890.1ms/step | 5.1 samples/s +Step 24020/44000 | Loss: 2.9265 | LR: 2.20e-04 | 1378.7ms/step | 14.5 samples/s +Step 24030/44000 | Loss: 2.9408 | LR: 2.20e-04 | 1382.0ms/step | 14.5 samples/s +Step 24040/44000 | Loss: 2.9523 | LR: 2.20e-04 | 1384.3ms/step | 14.4 samples/s +Step 24050/44000 | Loss: 2.9382 | LR: 2.20e-04 | 1386.7ms/step | 14.4 samples/s +Step 24060/44000 | Loss: 2.9557 | LR: 2.19e-04 | 1384.7ms/step | 14.4 samples/s +Step 24070/44000 | Loss: 2.9329 | LR: 2.19e-04 | 1382.4ms/step | 14.5 samples/s +Step 24080/44000 | Loss: 2.9510 | LR: 2.19e-04 | 1384.4ms/step | 14.4 samples/s +Step 24090/44000 | Loss: 2.9452 | LR: 2.19e-04 | 1386.0ms/step | 14.4 samples/s +Step 24100/44000 | Loss: 2.9340 | LR: 2.19e-04 | 1384.1ms/step | 14.4 samples/s +Step 24110/44000 | Loss: 2.9585 | LR: 2.19e-04 | 1386.9ms/step | 14.4 samples/s +Step 24120/44000 | Loss: 2.9229 | LR: 2.18e-04 | 1384.8ms/step | 14.4 samples/s +Step 24130/44000 | Loss: 2.9250 | LR: 2.18e-04 | 1386.2ms/step | 14.4 samples/s +Step 24140/44000 | Loss: 2.8989 | LR: 2.18e-04 | 1387.5ms/step | 14.4 samples/s +Step 24150/44000 | Loss: 2.9140 | LR: 2.18e-04 | 1384.6ms/step | 14.4 samples/s +Step 24160/44000 | Loss: 2.9185 | LR: 2.18e-04 | 1384.4ms/step | 14.4 samples/s +Step 24170/44000 | Loss: 2.8895 | LR: 2.17e-04 | 1386.1ms/step | 14.4 samples/s +Step 24180/44000 | Loss: 2.9426 | LR: 2.17e-04 | 1387.2ms/step | 14.4 samples/s +Step 24190/44000 | Loss: 2.9283 | LR: 2.17e-04 | 1386.9ms/step | 14.4 samples/s +Step 24200/44000 | Loss: 2.9441 | LR: 2.17e-04 | 1386.4ms/step | 14.4 samples/s +Step 24210/44000 | Loss: 2.9358 | LR: 2.17e-04 | 1386.1ms/step | 14.4 samples/s +Step 24220/44000 | Loss: 2.9207 | LR: 2.17e-04 | 1384.4ms/step | 14.4 samples/s +Step 24230/44000 | Loss: 2.8998 | LR: 2.16e-04 | 1382.8ms/step | 14.5 samples/s +Step 24240/44000 | Loss: 2.9242 | LR: 2.16e-04 | 1385.0ms/step | 14.4 samples/s +Step 24250/44000 | Loss: 2.9256 | LR: 2.16e-04 | 1385.1ms/step | 14.4 samples/s +Step 24260/44000 | Loss: 2.8977 | LR: 2.16e-04 | 1386.5ms/step | 14.4 samples/s +Step 24270/44000 | Loss: 2.9191 | LR: 2.16e-04 | 1385.3ms/step | 14.4 samples/s +Step 24280/44000 | Loss: 2.9278 | LR: 2.16e-04 | 1387.9ms/step | 14.4 samples/s +Step 24290/44000 | Loss: 2.9066 | LR: 2.15e-04 | 1385.1ms/step | 14.4 samples/s +Step 24300/44000 | Loss: 2.9072 | LR: 2.15e-04 | 1384.8ms/step | 14.4 samples/s +Step 24310/44000 | Loss: 2.9265 | LR: 2.15e-04 | 1381.6ms/step | 14.5 samples/s +Step 24320/44000 | Loss: 2.9172 | LR: 2.15e-04 | 1383.1ms/step | 14.5 samples/s +Step 24330/44000 | Loss: 2.9400 | LR: 2.15e-04 | 1385.0ms/step | 14.4 samples/s +Step 24340/44000 | Loss: 2.9078 | LR: 2.14e-04 | 1384.7ms/step | 14.4 samples/s +Step 24350/44000 | Loss: 2.9316 | LR: 2.14e-04 | 1385.9ms/step | 14.4 samples/s +Step 24360/44000 | Loss: 2.9062 | LR: 2.14e-04 | 1383.9ms/step | 14.5 samples/s +Step 24370/44000 | Loss: 2.9234 | LR: 2.14e-04 | 1385.6ms/step | 14.4 samples/s +Step 24380/44000 | Loss: 2.8903 | LR: 2.14e-04 | 1387.4ms/step | 14.4 samples/s +Step 24390/44000 | Loss: 2.9130 | LR: 2.14e-04 | 1386.1ms/step | 14.4 samples/s +Step 24400/44000 | Loss: 2.9255 | LR: 2.13e-04 | 1385.3ms/step | 14.4 samples/s +Step 24410/44000 | Loss: 2.9258 | LR: 2.13e-04 | 1386.4ms/step | 14.4 samples/s +Step 24420/44000 | Loss: 2.9364 | LR: 2.13e-04 | 1385.9ms/step | 14.4 samples/s +Step 24430/44000 | Loss: 2.9086 | LR: 2.13e-04 | 1385.0ms/step | 14.4 samples/s +Step 24440/44000 | Loss: 2.9035 | LR: 2.13e-04 | 1386.1ms/step | 14.4 samples/s +Step 24450/44000 | Loss: 2.9128 | LR: 2.12e-04 | 1386.0ms/step | 14.4 samples/s +Step 24460/44000 | Loss: 2.9468 | LR: 2.12e-04 | 1386.7ms/step | 14.4 samples/s +Step 24470/44000 | Loss: 2.9326 | LR: 2.12e-04 | 1385.0ms/step | 14.4 samples/s +Step 24480/44000 | Loss: 2.9037 | LR: 2.12e-04 | 1385.5ms/step | 14.4 samples/s +Step 24490/44000 | Loss: 2.9089 | LR: 2.12e-04 | 1386.3ms/step | 14.4 samples/s +Step 24500/44000 | Loss: 2.9547 | LR: 2.12e-04 | 1387.1ms/step | 14.4 samples/s +Step 24510/44000 | Loss: 2.9018 | LR: 2.11e-04 | 1385.0ms/step | 14.4 samples/s +Step 24520/44000 | Loss: 2.9027 | LR: 2.11e-04 | 1385.4ms/step | 14.4 samples/s +Step 24530/44000 | Loss: 2.9380 | LR: 2.11e-04 | 1385.3ms/step | 14.4 samples/s +Step 24540/44000 | Loss: 2.9505 | LR: 2.11e-04 | 1385.4ms/step | 14.4 samples/s +Step 24550/44000 | Loss: 2.8978 | LR: 2.11e-04 | 1386.9ms/step | 14.4 samples/s +Step 24560/44000 | Loss: 2.9232 | LR: 2.10e-04 | 1386.0ms/step | 14.4 samples/s +Step 24570/44000 | Loss: 2.8983 | LR: 2.10e-04 | 1386.4ms/step | 14.4 samples/s +Step 24580/44000 | Loss: 2.9258 | LR: 2.10e-04 | 1386.1ms/step | 14.4 samples/s +Step 24590/44000 | Loss: 2.9250 | LR: 2.10e-04 | 1385.8ms/step | 14.4 samples/s +Step 24600/44000 | Loss: 2.9088 | LR: 2.10e-04 | 1386.7ms/step | 14.4 samples/s +Step 24610/44000 | Loss: 2.9376 | LR: 2.10e-04 | 1386.2ms/step | 14.4 samples/s +Step 24620/44000 | Loss: 2.8973 | LR: 2.09e-04 | 1387.3ms/step | 14.4 samples/s +Step 24630/44000 | Loss: 2.9440 | LR: 2.09e-04 | 1386.2ms/step | 14.4 samples/s +Step 24640/44000 | Loss: 2.9282 | LR: 2.09e-04 | 1382.8ms/step | 14.5 samples/s +Step 24650/44000 | Loss: 2.9231 | LR: 2.09e-04 | 1383.7ms/step | 14.5 samples/s +Step 24660/44000 | Loss: 2.9320 | LR: 2.09e-04 | 1383.7ms/step | 14.5 samples/s +Step 24670/44000 | Loss: 2.9342 | LR: 2.09e-04 | 1384.9ms/step | 14.4 samples/s +Step 24680/44000 | Loss: 2.9517 | LR: 2.08e-04 | 1385.4ms/step | 14.4 samples/s +Step 24690/44000 | Loss: 2.9073 | LR: 2.08e-04 | 1385.0ms/step | 14.4 samples/s +Step 24700/44000 | Loss: 2.9283 | LR: 2.08e-04 | 1384.7ms/step | 14.4 samples/s +Step 24710/44000 | Loss: 2.9253 | LR: 2.08e-04 | 1385.7ms/step | 14.4 samples/s +Step 24720/44000 | Loss: 2.9257 | LR: 2.08e-04 | 1386.2ms/step | 14.4 samples/s +Step 24730/44000 | Loss: 2.9223 | LR: 2.07e-04 | 1386.3ms/step | 14.4 samples/s +Step 24740/44000 | Loss: 2.9300 | LR: 2.07e-04 | 1387.3ms/step | 14.4 samples/s +Step 24750/44000 | Loss: 2.9360 | LR: 2.07e-04 | 1386.4ms/step | 14.4 samples/s +Step 24760/44000 | Loss: 2.9115 | LR: 2.07e-04 | 1385.5ms/step | 14.4 samples/s +Step 24770/44000 | Loss: 2.9050 | LR: 2.07e-04 | 1385.8ms/step | 14.4 samples/s +Step 24780/44000 | Loss: 2.8743 | LR: 2.07e-04 | 1385.2ms/step | 14.4 samples/s +Step 24790/44000 | Loss: 2.9399 | LR: 2.06e-04 | 1384.7ms/step | 14.4 samples/s +Step 24800/44000 | Loss: 2.9058 | LR: 2.06e-04 | 1385.7ms/step | 14.4 samples/s +Step 24810/44000 | Loss: 2.9239 | LR: 2.06e-04 | 1386.4ms/step | 14.4 samples/s +Step 24820/44000 | Loss: 2.9290 | LR: 2.06e-04 | 1386.4ms/step | 14.4 samples/s +Step 24830/44000 | Loss: 2.9233 | LR: 2.06e-04 | 1388.7ms/step | 14.4 samples/s +Step 24840/44000 | Loss: 2.9283 | LR: 2.05e-04 | 1387.6ms/step | 14.4 samples/s +Step 24850/44000 | Loss: 2.9185 | LR: 2.05e-04 | 1386.2ms/step | 14.4 samples/s +Step 24860/44000 | Loss: 2.9120 | LR: 2.05e-04 | 1385.2ms/step | 14.4 samples/s +Step 24870/44000 | Loss: 2.9350 | LR: 2.05e-04 | 1384.8ms/step | 14.4 samples/s +Step 24880/44000 | Loss: 2.9278 | LR: 2.05e-04 | 1386.6ms/step | 14.4 samples/s +Step 24890/44000 | Loss: 2.9112 | LR: 2.05e-04 | 1386.6ms/step | 14.4 samples/s +Step 24900/44000 | Loss: 2.9194 | LR: 2.04e-04 | 1385.2ms/step | 14.4 samples/s +Step 24910/44000 | Loss: 2.9034 | LR: 2.04e-04 | 1387.8ms/step | 14.4 samples/s +Step 24920/44000 | Loss: 2.9210 | LR: 2.04e-04 | 1391.8ms/step | 14.4 samples/s +Step 24930/44000 | Loss: 2.8980 | LR: 2.04e-04 | 1388.1ms/step | 14.4 samples/s +Step 24940/44000 | Loss: 2.9457 | LR: 2.04e-04 | 1386.6ms/step | 14.4 samples/s +Step 24950/44000 | Loss: 2.9317 | LR: 2.04e-04 | 1385.3ms/step | 14.4 samples/s +Step 24960/44000 | Loss: 2.9185 | LR: 2.03e-04 | 1386.2ms/step | 14.4 samples/s +Step 24970/44000 | Loss: 2.9067 | LR: 2.03e-04 | 1386.2ms/step | 14.4 samples/s +Step 24980/44000 | Loss: 2.9227 | LR: 2.03e-04 | 1385.8ms/step | 14.4 samples/s +Step 24990/44000 | Loss: 2.9566 | LR: 2.03e-04 | 1386.3ms/step | 14.4 samples/s +Step 25000/44000 | Loss: 2.9166 | LR: 2.03e-04 | 1386.8ms/step | 14.4 samples/s +Validation at step 25000: Loss = 2.9225, Perplexity = 18.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-25000 +Step 25010/44000 | Loss: 2.9224 | LR: 2.02e-04 | 3868.9ms/step | 5.2 samples/s +Step 25020/44000 | Loss: 2.9296 | LR: 2.02e-04 | 1380.1ms/step | 14.5 samples/s +Step 25030/44000 | Loss: 2.9339 | LR: 2.02e-04 | 1378.4ms/step | 14.5 samples/s +Step 25040/44000 | Loss: 2.9087 | LR: 2.02e-04 | 1381.4ms/step | 14.5 samples/s +Step 25050/44000 | Loss: 2.9314 | LR: 2.02e-04 | 1380.4ms/step | 14.5 samples/s +Step 25060/44000 | Loss: 2.9463 | LR: 2.02e-04 | 1380.5ms/step | 14.5 samples/s +Step 25070/44000 | Loss: 2.9464 | LR: 2.01e-04 | 1381.4ms/step | 14.5 samples/s +Step 25080/44000 | Loss: 2.9324 | LR: 2.01e-04 | 1384.0ms/step | 14.5 samples/s +Step 25090/44000 | Loss: 2.9552 | LR: 2.01e-04 | 1385.3ms/step | 14.4 samples/s +Step 25100/44000 | Loss: 2.9110 | LR: 2.01e-04 | 1384.7ms/step | 14.4 samples/s +Step 25110/44000 | Loss: 2.9488 | LR: 2.01e-04 | 1385.8ms/step | 14.4 samples/s +Step 25120/44000 | Loss: 2.9199 | LR: 2.00e-04 | 1385.8ms/step | 14.4 samples/s +Step 25130/44000 | Loss: 2.9198 | LR: 2.00e-04 | 1384.3ms/step | 14.4 samples/s +Step 25140/44000 | Loss: 2.9427 | LR: 2.00e-04 | 1385.6ms/step | 14.4 samples/s +Step 25150/44000 | Loss: 2.9185 | LR: 2.00e-04 | 1383.5ms/step | 14.5 samples/s +Step 25160/44000 | Loss: 2.9105 | LR: 2.00e-04 | 1382.8ms/step | 14.5 samples/s +Step 25170/44000 | Loss: 2.9368 | LR: 2.00e-04 | 1386.1ms/step | 14.4 samples/s +Step 25180/44000 | Loss: 2.9260 | LR: 1.99e-04 | 1385.0ms/step | 14.4 samples/s +Step 25190/44000 | Loss: 2.9100 | LR: 1.99e-04 | 1385.7ms/step | 14.4 samples/s +Step 25200/44000 | Loss: 2.8942 | LR: 1.99e-04 | 1387.3ms/step | 14.4 samples/s +Step 25210/44000 | Loss: 2.9235 | LR: 1.99e-04 | 1388.4ms/step | 14.4 samples/s +Step 25220/44000 | Loss: 2.9157 | LR: 1.99e-04 | 1386.7ms/step | 14.4 samples/s +Step 25230/44000 | Loss: 2.9026 | LR: 1.99e-04 | 1385.1ms/step | 14.4 samples/s +Step 25240/44000 | Loss: 2.9589 | LR: 1.98e-04 | 1389.7ms/step | 14.4 samples/s +Step 25250/44000 | Loss: 2.8979 | LR: 1.98e-04 | 1387.7ms/step | 14.4 samples/s +Step 25260/44000 | Loss: 2.9058 | LR: 1.98e-04 | 1385.3ms/step | 14.4 samples/s +Step 25270/44000 | Loss: 2.9124 | LR: 1.98e-04 | 1386.1ms/step | 14.4 samples/s +Step 25280/44000 | Loss: 2.9166 | LR: 1.98e-04 | 1385.9ms/step | 14.4 samples/s +Step 25290/44000 | Loss: 2.9054 | LR: 1.97e-04 | 1385.7ms/step | 14.4 samples/s +Step 25300/44000 | Loss: 2.8855 | LR: 1.97e-04 | 1387.1ms/step | 14.4 samples/s +Step 25310/44000 | Loss: 2.9100 | LR: 1.97e-04 | 1385.0ms/step | 14.4 samples/s +Step 25320/44000 | Loss: 2.9086 | LR: 1.97e-04 | 1386.8ms/step | 14.4 samples/s +Step 25330/44000 | Loss: 2.9201 | LR: 1.97e-04 | 1387.8ms/step | 14.4 samples/s +Step 25340/44000 | Loss: 2.8988 | LR: 1.97e-04 | 1386.8ms/step | 14.4 samples/s +Step 25350/44000 | Loss: 2.9100 | LR: 1.96e-04 | 1386.4ms/step | 14.4 samples/s +Step 25360/44000 | Loss: 2.9261 | LR: 1.96e-04 | 1387.6ms/step | 14.4 samples/s +Step 25370/44000 | Loss: 2.9330 | LR: 1.96e-04 | 1388.1ms/step | 14.4 samples/s +Step 25380/44000 | Loss: 2.9458 | LR: 1.96e-04 | 1385.3ms/step | 14.4 samples/s +Step 25390/44000 | Loss: 2.9204 | LR: 1.96e-04 | 1385.6ms/step | 14.4 samples/s +Step 25400/44000 | Loss: 2.9169 | LR: 1.96e-04 | 1385.1ms/step | 14.4 samples/s +Step 25410/44000 | Loss: 2.9116 | LR: 1.95e-04 | 1385.9ms/step | 14.4 samples/s +Step 25420/44000 | Loss: 2.9238 | LR: 1.95e-04 | 1385.9ms/step | 14.4 samples/s +Step 25430/44000 | Loss: 2.9248 | LR: 1.95e-04 | 1386.2ms/step | 14.4 samples/s +Step 25440/44000 | Loss: 2.9158 | LR: 1.95e-04 | 1386.7ms/step | 14.4 samples/s +Step 25450/44000 | Loss: 2.9289 | LR: 1.95e-04 | 1387.2ms/step | 14.4 samples/s +Step 25460/44000 | Loss: 2.9158 | LR: 1.94e-04 | 1386.3ms/step | 14.4 samples/s +Step 25470/44000 | Loss: 2.9281 | LR: 1.94e-04 | 1384.4ms/step | 14.4 samples/s +Step 25480/44000 | Loss: 2.8903 | LR: 1.94e-04 | 1383.8ms/step | 14.5 samples/s +Step 25490/44000 | Loss: 2.9325 | LR: 1.94e-04 | 1385.6ms/step | 14.4 samples/s +Step 25500/44000 | Loss: 2.9125 | LR: 1.94e-04 | 1384.7ms/step | 14.4 samples/s +Step 25510/44000 | Loss: 2.8929 | LR: 1.94e-04 | 1384.6ms/step | 14.4 samples/s +Step 25520/44000 | Loss: 2.9354 | LR: 1.93e-04 | 1384.7ms/step | 14.4 samples/s +Step 25530/44000 | Loss: 2.8762 | LR: 1.93e-04 | 1386.1ms/step | 14.4 samples/s +Step 25540/44000 | Loss: 2.9402 | LR: 1.93e-04 | 1384.4ms/step | 14.4 samples/s +Step 25550/44000 | Loss: 2.9437 | LR: 1.93e-04 | 1382.5ms/step | 14.5 samples/s +Step 25560/44000 | Loss: 2.9130 | LR: 1.93e-04 | 1385.2ms/step | 14.4 samples/s +Step 25570/44000 | Loss: 2.9129 | LR: 1.92e-04 | 1385.2ms/step | 14.4 samples/s +Step 25580/44000 | Loss: 2.9244 | LR: 1.92e-04 | 1387.1ms/step | 14.4 samples/s +Step 25590/44000 | Loss: 2.9279 | LR: 1.92e-04 | 1384.8ms/step | 14.4 samples/s +Step 25600/44000 | Loss: 2.9301 | LR: 1.92e-04 | 1381.8ms/step | 14.5 samples/s +Step 25610/44000 | Loss: 2.9200 | LR: 1.92e-04 | 1383.5ms/step | 14.5 samples/s +Step 25620/44000 | Loss: 2.9137 | LR: 1.92e-04 | 1384.3ms/step | 14.4 samples/s +Step 25630/44000 | Loss: 2.9280 | LR: 1.91e-04 | 1386.0ms/step | 14.4 samples/s +Step 25640/44000 | Loss: 2.9027 | LR: 1.91e-04 | 1387.4ms/step | 14.4 samples/s +Step 25650/44000 | Loss: 2.9334 | LR: 1.91e-04 | 1385.6ms/step | 14.4 samples/s +Step 25660/44000 | Loss: 2.9385 | LR: 1.91e-04 | 1385.0ms/step | 14.4 samples/s +Step 25670/44000 | Loss: 2.9254 | LR: 1.91e-04 | 1386.7ms/step | 14.4 samples/s +Step 25680/44000 | Loss: 2.9362 | LR: 1.91e-04 | 1382.8ms/step | 14.5 samples/s +Step 25690/44000 | Loss: 2.9214 | LR: 1.90e-04 | 1383.5ms/step | 14.5 samples/s +Step 25700/44000 | Loss: 2.9369 | LR: 1.90e-04 | 1383.8ms/step | 14.5 samples/s +Step 25710/44000 | Loss: 2.9073 | LR: 1.90e-04 | 1381.4ms/step | 14.5 samples/s +Step 25720/44000 | Loss: 2.9111 | LR: 1.90e-04 | 1381.6ms/step | 14.5 samples/s +Step 25730/44000 | Loss: 2.9345 | LR: 1.90e-04 | 1380.7ms/step | 14.5 samples/s +Step 25740/44000 | Loss: 2.9402 | LR: 1.89e-04 | 1381.4ms/step | 14.5 samples/s +Step 25750/44000 | Loss: 2.9116 | LR: 1.89e-04 | 1383.0ms/step | 14.5 samples/s +Step 25760/44000 | Loss: 2.9256 | LR: 1.89e-04 | 1383.6ms/step | 14.5 samples/s +Step 25770/44000 | Loss: 2.9234 | LR: 1.89e-04 | 1384.7ms/step | 14.4 samples/s +Step 25780/44000 | Loss: 2.9124 | LR: 1.89e-04 | 1384.5ms/step | 14.4 samples/s +Step 25790/44000 | Loss: 2.8970 | LR: 1.89e-04 | 1386.2ms/step | 14.4 samples/s +Step 25800/44000 | Loss: 2.9388 | LR: 1.88e-04 | 1383.9ms/step | 14.5 samples/s +Step 25810/44000 | Loss: 2.9370 | LR: 1.88e-04 | 1385.7ms/step | 14.4 samples/s +Step 25820/44000 | Loss: 2.9244 | LR: 1.88e-04 | 1386.2ms/step | 14.4 samples/s +Step 25830/44000 | Loss: 2.9136 | LR: 1.88e-04 | 1386.1ms/step | 14.4 samples/s +Step 25840/44000 | Loss: 2.8965 | LR: 1.88e-04 | 1384.5ms/step | 14.4 samples/s +Step 25850/44000 | Loss: 2.9279 | LR: 1.88e-04 | 1385.3ms/step | 14.4 samples/s +Step 25860/44000 | Loss: 2.9521 | LR: 1.87e-04 | 1385.3ms/step | 14.4 samples/s +Step 25870/44000 | Loss: 2.9310 | LR: 1.87e-04 | 1386.4ms/step | 14.4 samples/s +Step 25880/44000 | Loss: 2.8866 | LR: 1.87e-04 | 1386.2ms/step | 14.4 samples/s +Step 25890/44000 | Loss: 2.8860 | LR: 1.87e-04 | 1385.7ms/step | 14.4 samples/s +Step 25900/44000 | Loss: 2.9269 | LR: 1.87e-04 | 1385.4ms/step | 14.4 samples/s +Step 25910/44000 | Loss: 2.9094 | LR: 1.87e-04 | 1386.9ms/step | 14.4 samples/s +Step 25920/44000 | Loss: 2.9134 | LR: 1.86e-04 | 1383.9ms/step | 14.5 samples/s +Step 25930/44000 | Loss: 2.9232 | LR: 1.86e-04 | 1385.7ms/step | 14.4 samples/s +Step 25940/44000 | Loss: 2.9114 | LR: 1.86e-04 | 1386.5ms/step | 14.4 samples/s +Step 25950/44000 | Loss: 2.9404 | LR: 1.86e-04 | 1386.0ms/step | 14.4 samples/s +Step 25960/44000 | Loss: 2.9049 | LR: 1.86e-04 | 1386.0ms/step | 14.4 samples/s +Step 25970/44000 | Loss: 2.9268 | LR: 1.85e-04 | 1386.4ms/step | 14.4 samples/s +Step 25980/44000 | Loss: 2.9102 | LR: 1.85e-04 | 1389.0ms/step | 14.4 samples/s +Step 25990/44000 | Loss: 2.9086 | LR: 1.85e-04 | 1386.8ms/step | 14.4 samples/s +Step 26000/44000 | Loss: 2.8802 | LR: 1.85e-04 | 1385.8ms/step | 14.4 samples/s +Validation at step 26000: Loss = 2.9160, Perplexity = 18.47 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-26000 +Step 26010/44000 | Loss: 2.9195 | LR: 1.85e-04 | 3851.8ms/step | 5.2 samples/s +Step 26020/44000 | Loss: 2.9140 | LR: 1.85e-04 | 1384.4ms/step | 14.4 samples/s +Step 26030/44000 | Loss: 2.9012 | LR: 1.84e-04 | 1382.7ms/step | 14.5 samples/s +Step 26040/44000 | Loss: 2.8935 | LR: 1.84e-04 | 1385.4ms/step | 14.4 samples/s +Step 26050/44000 | Loss: 2.9223 | LR: 1.84e-04 | 1387.0ms/step | 14.4 samples/s +Step 26060/44000 | Loss: 2.8912 | LR: 1.84e-04 | 1388.5ms/step | 14.4 samples/s +Step 26070/44000 | Loss: 2.9247 | LR: 1.84e-04 | 1387.7ms/step | 14.4 samples/s +Step 26080/44000 | Loss: 2.8878 | LR: 1.84e-04 | 1384.8ms/step | 14.4 samples/s +Step 26090/44000 | Loss: 2.8779 | LR: 1.83e-04 | 1385.0ms/step | 14.4 samples/s +Step 26100/44000 | Loss: 2.9326 | LR: 1.83e-04 | 1386.1ms/step | 14.4 samples/s +Step 26110/44000 | Loss: 2.9276 | LR: 1.83e-04 | 1389.0ms/step | 14.4 samples/s +Step 26120/44000 | Loss: 2.8937 | LR: 1.83e-04 | 1386.0ms/step | 14.4 samples/s +Step 26130/44000 | Loss: 2.9422 | LR: 1.83e-04 | 1387.8ms/step | 14.4 samples/s +Step 26140/44000 | Loss: 2.9319 | LR: 1.82e-04 | 1387.8ms/step | 14.4 samples/s +Step 26150/44000 | Loss: 2.9061 | LR: 1.82e-04 | 1386.7ms/step | 14.4 samples/s +Step 26160/44000 | Loss: 2.9060 | LR: 1.82e-04 | 1387.8ms/step | 14.4 samples/s +Step 26170/44000 | Loss: 2.9101 | LR: 1.82e-04 | 1387.9ms/step | 14.4 samples/s +Step 26180/44000 | Loss: 2.8929 | LR: 1.82e-04 | 1389.3ms/step | 14.4 samples/s +Step 26190/44000 | Loss: 2.9340 | LR: 1.82e-04 | 1386.9ms/step | 14.4 samples/s +Step 26200/44000 | Loss: 2.9282 | LR: 1.81e-04 | 1381.6ms/step | 14.5 samples/s +Step 26210/44000 | Loss: 2.9508 | LR: 1.81e-04 | 1383.5ms/step | 14.5 samples/s +Step 26220/44000 | Loss: 2.9409 | LR: 1.81e-04 | 1384.4ms/step | 14.4 samples/s +Step 26230/44000 | Loss: 2.8938 | LR: 1.81e-04 | 1383.7ms/step | 14.5 samples/s +Step 26240/44000 | Loss: 2.8818 | LR: 1.81e-04 | 1385.5ms/step | 14.4 samples/s +Step 26250/44000 | Loss: 2.8940 | LR: 1.81e-04 | 1384.0ms/step | 14.5 samples/s +Step 26260/44000 | Loss: 2.9020 | LR: 1.80e-04 | 1383.4ms/step | 14.5 samples/s +Step 26270/44000 | Loss: 2.9147 | LR: 1.80e-04 | 1383.7ms/step | 14.5 samples/s +Step 26280/44000 | Loss: 2.9400 | LR: 1.80e-04 | 1383.9ms/step | 14.5 samples/s +Step 26290/44000 | Loss: 2.9038 | LR: 1.80e-04 | 1384.8ms/step | 14.4 samples/s +Step 26300/44000 | Loss: 2.9201 | LR: 1.80e-04 | 1386.8ms/step | 14.4 samples/s +Step 26310/44000 | Loss: 2.8997 | LR: 1.80e-04 | 1386.2ms/step | 14.4 samples/s +Step 26320/44000 | Loss: 2.9296 | LR: 1.79e-04 | 1386.6ms/step | 14.4 samples/s +Step 26330/44000 | Loss: 2.9137 | LR: 1.79e-04 | 1387.5ms/step | 14.4 samples/s +Step 26340/44000 | Loss: 2.8920 | LR: 1.79e-04 | 1386.3ms/step | 14.4 samples/s +Step 26350/44000 | Loss: 2.9160 | LR: 1.79e-04 | 1389.2ms/step | 14.4 samples/s +Step 26360/44000 | Loss: 2.9086 | LR: 1.79e-04 | 1388.9ms/step | 14.4 samples/s +Step 26370/44000 | Loss: 2.9252 | LR: 1.78e-04 | 1388.1ms/step | 14.4 samples/s +Step 26380/44000 | Loss: 2.9268 | LR: 1.78e-04 | 1387.8ms/step | 14.4 samples/s +Step 26390/44000 | Loss: 2.8919 | LR: 1.78e-04 | 1388.5ms/step | 14.4 samples/s +Step 26400/44000 | Loss: 2.9430 | LR: 1.78e-04 | 1387.4ms/step | 14.4 samples/s +Step 26410/44000 | Loss: 2.9193 | LR: 1.78e-04 | 1386.3ms/step | 14.4 samples/s +Step 26420/44000 | Loss: 2.8896 | LR: 1.78e-04 | 1387.8ms/step | 14.4 samples/s +Step 26430/44000 | Loss: 2.9140 | LR: 1.77e-04 | 1387.4ms/step | 14.4 samples/s +Step 26440/44000 | Loss: 2.8970 | LR: 1.77e-04 | 1388.0ms/step | 14.4 samples/s +Step 26450/44000 | Loss: 2.8949 | LR: 1.77e-04 | 1386.8ms/step | 14.4 samples/s +Step 26460/44000 | Loss: 2.8901 | LR: 1.77e-04 | 1389.5ms/step | 14.4 samples/s +Step 26470/44000 | Loss: 2.9006 | LR: 1.77e-04 | 1389.1ms/step | 14.4 samples/s +Step 26480/44000 | Loss: 2.9370 | LR: 1.77e-04 | 1387.6ms/step | 14.4 samples/s +Step 26490/44000 | Loss: 2.9036 | LR: 1.76e-04 | 1387.3ms/step | 14.4 samples/s +Step 26500/44000 | Loss: 2.9113 | LR: 1.76e-04 | 1385.8ms/step | 14.4 samples/s +Step 26510/44000 | Loss: 2.8918 | LR: 1.76e-04 | 1384.5ms/step | 14.4 samples/s +Step 26520/44000 | Loss: 2.9643 | LR: 1.76e-04 | 1386.2ms/step | 14.4 samples/s +Step 26530/44000 | Loss: 2.8960 | LR: 1.76e-04 | 1385.8ms/step | 14.4 samples/s +Step 26540/44000 | Loss: 2.9304 | LR: 1.76e-04 | 1387.9ms/step | 14.4 samples/s +Step 26550/44000 | Loss: 2.9119 | LR: 1.75e-04 | 1387.9ms/step | 14.4 samples/s +Step 26560/44000 | Loss: 2.9234 | LR: 1.75e-04 | 1385.7ms/step | 14.4 samples/s +Step 26570/44000 | Loss: 2.9072 | LR: 1.75e-04 | 1388.6ms/step | 14.4 samples/s +Step 26580/44000 | Loss: 2.9106 | LR: 1.75e-04 | 1388.0ms/step | 14.4 samples/s +Step 26590/44000 | Loss: 2.8919 | LR: 1.75e-04 | 1388.6ms/step | 14.4 samples/s +Step 26600/44000 | Loss: 2.8924 | LR: 1.74e-04 | 1388.6ms/step | 14.4 samples/s +Step 26610/44000 | Loss: 2.9012 | LR: 1.74e-04 | 1387.3ms/step | 14.4 samples/s +Step 26620/44000 | Loss: 2.8904 | LR: 1.74e-04 | 1389.2ms/step | 14.4 samples/s +Step 26630/44000 | Loss: 2.8982 | LR: 1.74e-04 | 1388.6ms/step | 14.4 samples/s +Step 26640/44000 | Loss: 2.9166 | LR: 1.74e-04 | 1387.1ms/step | 14.4 samples/s +Step 26650/44000 | Loss: 2.9251 | LR: 1.74e-04 | 1390.0ms/step | 14.4 samples/s +Step 26660/44000 | Loss: 2.8942 | LR: 1.73e-04 | 1386.6ms/step | 14.4 samples/s +Step 26670/44000 | Loss: 2.9038 | LR: 1.73e-04 | 1386.8ms/step | 14.4 samples/s +Step 26680/44000 | Loss: 2.9450 | LR: 1.73e-04 | 1387.5ms/step | 14.4 samples/s +Step 26690/44000 | Loss: 2.9129 | LR: 1.73e-04 | 1387.8ms/step | 14.4 samples/s +Step 26700/44000 | Loss: 2.9102 | LR: 1.73e-04 | 1387.6ms/step | 14.4 samples/s +Step 26710/44000 | Loss: 2.8911 | LR: 1.73e-04 | 1385.4ms/step | 14.4 samples/s +Step 26720/44000 | Loss: 2.9372 | LR: 1.72e-04 | 1385.8ms/step | 14.4 samples/s +Step 26730/44000 | Loss: 2.9220 | LR: 1.72e-04 | 1384.8ms/step | 14.4 samples/s +Step 26740/44000 | Loss: 2.9231 | LR: 1.72e-04 | 1385.6ms/step | 14.4 samples/s +Step 26750/44000 | Loss: 2.8791 | LR: 1.72e-04 | 1386.2ms/step | 14.4 samples/s +Step 26760/44000 | Loss: 2.9200 | LR: 1.72e-04 | 1387.6ms/step | 14.4 samples/s +Step 26770/44000 | Loss: 2.9143 | LR: 1.72e-04 | 1386.3ms/step | 14.4 samples/s +Step 26780/44000 | Loss: 2.9164 | LR: 1.71e-04 | 1386.0ms/step | 14.4 samples/s +Step 26790/44000 | Loss: 2.9102 | LR: 1.71e-04 | 1386.6ms/step | 14.4 samples/s +Step 26800/44000 | Loss: 2.9129 | LR: 1.71e-04 | 1386.0ms/step | 14.4 samples/s +Step 26810/44000 | Loss: 2.9143 | LR: 1.71e-04 | 1388.0ms/step | 14.4 samples/s +Step 26820/44000 | Loss: 2.9207 | LR: 1.71e-04 | 1383.6ms/step | 14.5 samples/s +Step 26830/44000 | Loss: 2.9083 | LR: 1.71e-04 | 1384.2ms/step | 14.4 samples/s +Step 26840/44000 | Loss: 2.9095 | LR: 1.70e-04 | 1385.2ms/step | 14.4 samples/s +Step 26850/44000 | Loss: 2.9062 | LR: 1.70e-04 | 1382.5ms/step | 14.5 samples/s +Step 26860/44000 | Loss: 2.9570 | LR: 1.70e-04 | 1384.0ms/step | 14.5 samples/s +Step 26870/44000 | Loss: 2.9026 | LR: 1.70e-04 | 1383.0ms/step | 14.5 samples/s +Step 26880/44000 | Loss: 2.9321 | LR: 1.70e-04 | 1385.2ms/step | 14.4 samples/s +Step 26890/44000 | Loss: 2.9007 | LR: 1.69e-04 | 1388.2ms/step | 14.4 samples/s +Step 26900/44000 | Loss: 2.9167 | LR: 1.69e-04 | 1388.6ms/step | 14.4 samples/s +Step 26910/44000 | Loss: 2.8856 | LR: 1.69e-04 | 1386.8ms/step | 14.4 samples/s +Step 26920/44000 | Loss: 2.9110 | LR: 1.69e-04 | 1384.5ms/step | 14.4 samples/s +Step 26930/44000 | Loss: 2.8983 | LR: 1.69e-04 | 1386.0ms/step | 14.4 samples/s +Step 26940/44000 | Loss: 2.9152 | LR: 1.69e-04 | 1386.1ms/step | 14.4 samples/s +Step 26950/44000 | Loss: 2.9011 | LR: 1.68e-04 | 1385.2ms/step | 14.4 samples/s +Step 26960/44000 | Loss: 2.9190 | LR: 1.68e-04 | 1386.4ms/step | 14.4 samples/s +Step 26970/44000 | Loss: 2.9062 | LR: 1.68e-04 | 1385.7ms/step | 14.4 samples/s +Step 26980/44000 | Loss: 2.8987 | LR: 1.68e-04 | 1388.5ms/step | 14.4 samples/s +Step 26990/44000 | Loss: 2.8972 | LR: 1.68e-04 | 1387.2ms/step | 14.4 samples/s +Step 27000/44000 | Loss: 2.8785 | LR: 1.68e-04 | 1388.9ms/step | 14.4 samples/s +Validation at step 27000: Loss = 2.9100, Perplexity = 18.36 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-27000 +Step 27010/44000 | Loss: 2.9059 | LR: 1.67e-04 | 3880.1ms/step | 5.2 samples/s +Step 27020/44000 | Loss: 2.9022 | LR: 1.67e-04 | 1384.1ms/step | 14.4 samples/s +Step 27030/44000 | Loss: 2.8894 | LR: 1.67e-04 | 1386.3ms/step | 14.4 samples/s +Step 27040/44000 | Loss: 2.9242 | LR: 1.67e-04 | 1388.6ms/step | 14.4 samples/s +Step 27050/44000 | Loss: 2.9037 | LR: 1.67e-04 | 1389.0ms/step | 14.4 samples/s +Step 27060/44000 | Loss: 2.9248 | LR: 1.67e-04 | 1387.3ms/step | 14.4 samples/s +Step 27070/44000 | Loss: 2.8653 | LR: 1.66e-04 | 1386.3ms/step | 14.4 samples/s +Step 27080/44000 | Loss: 2.8982 | LR: 1.66e-04 | 1388.5ms/step | 14.4 samples/s +Step 27090/44000 | Loss: 2.8877 | LR: 1.66e-04 | 1388.0ms/step | 14.4 samples/s +Step 27100/44000 | Loss: 2.9170 | LR: 1.66e-04 | 1386.5ms/step | 14.4 samples/s +Step 27110/44000 | Loss: 2.8877 | LR: 1.66e-04 | 1388.9ms/step | 14.4 samples/s +Step 27120/44000 | Loss: 2.9143 | LR: 1.66e-04 | 1388.1ms/step | 14.4 samples/s +Step 27130/44000 | Loss: 2.9083 | LR: 1.65e-04 | 1388.4ms/step | 14.4 samples/s +Step 27140/44000 | Loss: 2.9122 | LR: 1.65e-04 | 1387.0ms/step | 14.4 samples/s +Step 27150/44000 | Loss: 2.9125 | LR: 1.65e-04 | 1389.2ms/step | 14.4 samples/s +Step 27160/44000 | Loss: 2.9329 | LR: 1.65e-04 | 1386.8ms/step | 14.4 samples/s +Step 27170/44000 | Loss: 2.8899 | LR: 1.65e-04 | 1387.3ms/step | 14.4 samples/s +Step 27180/44000 | Loss: 2.9019 | LR: 1.65e-04 | 1388.0ms/step | 14.4 samples/s +Step 27190/44000 | Loss: 2.8784 | LR: 1.64e-04 | 1386.0ms/step | 14.4 samples/s +Step 27200/44000 | Loss: 2.8698 | LR: 1.64e-04 | 1387.7ms/step | 14.4 samples/s +Step 27210/44000 | Loss: 2.8818 | LR: 1.64e-04 | 1387.6ms/step | 14.4 samples/s +Step 27220/44000 | Loss: 2.9108 | LR: 1.64e-04 | 1386.5ms/step | 14.4 samples/s +Step 27230/44000 | Loss: 2.9095 | LR: 1.64e-04 | 1385.7ms/step | 14.4 samples/s +Step 27240/44000 | Loss: 2.9137 | LR: 1.63e-04 | 1386.1ms/step | 14.4 samples/s +Step 27250/44000 | Loss: 2.9136 | LR: 1.63e-04 | 1385.0ms/step | 14.4 samples/s +Step 27260/44000 | Loss: 2.8987 | LR: 1.63e-04 | 1387.2ms/step | 14.4 samples/s +Step 27270/44000 | Loss: 2.9315 | LR: 1.63e-04 | 1389.6ms/step | 14.4 samples/s +Step 27280/44000 | Loss: 2.9489 | LR: 1.63e-04 | 1388.1ms/step | 14.4 samples/s +Step 27290/44000 | Loss: 2.8971 | LR: 1.63e-04 | 1387.1ms/step | 14.4 samples/s +Step 27300/44000 | Loss: 2.9163 | LR: 1.62e-04 | 1386.5ms/step | 14.4 samples/s +Step 27310/44000 | Loss: 2.9013 | LR: 1.62e-04 | 1388.7ms/step | 14.4 samples/s +Step 27320/44000 | Loss: 2.9068 | LR: 1.62e-04 | 1386.2ms/step | 14.4 samples/s +Step 27330/44000 | Loss: 2.9255 | LR: 1.62e-04 | 1384.7ms/step | 14.4 samples/s +Step 27340/44000 | Loss: 2.9184 | LR: 1.62e-04 | 1386.2ms/step | 14.4 samples/s +Step 27350/44000 | Loss: 2.8831 | LR: 1.62e-04 | 1385.0ms/step | 14.4 samples/s +Step 27360/44000 | Loss: 2.8737 | LR: 1.61e-04 | 1387.7ms/step | 14.4 samples/s +Step 27370/44000 | Loss: 2.8974 | LR: 1.61e-04 | 1385.8ms/step | 14.4 samples/s +Step 27380/44000 | Loss: 2.8854 | LR: 1.61e-04 | 1388.1ms/step | 14.4 samples/s +Step 27390/44000 | Loss: 2.9144 | LR: 1.61e-04 | 1384.5ms/step | 14.4 samples/s +Step 27400/44000 | Loss: 2.8977 | LR: 1.61e-04 | 1387.3ms/step | 14.4 samples/s +Step 27410/44000 | Loss: 2.9052 | LR: 1.61e-04 | 1388.7ms/step | 14.4 samples/s +Step 27420/44000 | Loss: 2.9016 | LR: 1.60e-04 | 1391.2ms/step | 14.4 samples/s +Step 27430/44000 | Loss: 2.9214 | LR: 1.60e-04 | 1387.0ms/step | 14.4 samples/s +Step 27440/44000 | Loss: 2.9142 | LR: 1.60e-04 | 1385.6ms/step | 14.4 samples/s +Step 27450/44000 | Loss: 2.9091 | LR: 1.60e-04 | 1386.0ms/step | 14.4 samples/s +Step 27460/44000 | Loss: 2.9164 | LR: 1.60e-04 | 1384.3ms/step | 14.4 samples/s +Step 27470/44000 | Loss: 2.8936 | LR: 1.60e-04 | 1385.8ms/step | 14.4 samples/s +Step 27480/44000 | Loss: 2.8760 | LR: 1.59e-04 | 1385.5ms/step | 14.4 samples/s +Step 27490/44000 | Loss: 2.9225 | LR: 1.59e-04 | 1386.1ms/step | 14.4 samples/s +Step 27500/44000 | Loss: 2.9053 | LR: 1.59e-04 | 1385.9ms/step | 14.4 samples/s +Step 27510/44000 | Loss: 2.9057 | LR: 1.59e-04 | 1386.4ms/step | 14.4 samples/s +Step 27520/44000 | Loss: 2.8877 | LR: 1.59e-04 | 1387.9ms/step | 14.4 samples/s +Step 27530/44000 | Loss: 2.8832 | LR: 1.59e-04 | 1385.1ms/step | 14.4 samples/s +Step 27540/44000 | Loss: 2.9184 | LR: 1.58e-04 | 1385.2ms/step | 14.4 samples/s +Step 27550/44000 | Loss: 2.9058 | LR: 1.58e-04 | 1385.9ms/step | 14.4 samples/s +Step 27560/44000 | Loss: 2.8974 | LR: 1.58e-04 | 1386.5ms/step | 14.4 samples/s +Step 27570/44000 | Loss: 2.8981 | LR: 1.58e-04 | 1386.9ms/step | 14.4 samples/s +Step 27580/44000 | Loss: 2.8914 | LR: 1.58e-04 | 1386.7ms/step | 14.4 samples/s +Step 27590/44000 | Loss: 2.8805 | LR: 1.58e-04 | 1387.5ms/step | 14.4 samples/s +Step 27600/44000 | Loss: 2.8832 | LR: 1.57e-04 | 1387.7ms/step | 14.4 samples/s +Step 27610/44000 | Loss: 2.8990 | LR: 1.57e-04 | 1385.5ms/step | 14.4 samples/s +Step 27620/44000 | Loss: 2.9021 | LR: 1.57e-04 | 1387.0ms/step | 14.4 samples/s +Step 27630/44000 | Loss: 2.9155 | LR: 1.57e-04 | 1386.3ms/step | 14.4 samples/s +Step 27640/44000 | Loss: 2.9041 | LR: 1.57e-04 | 1387.6ms/step | 14.4 samples/s +Step 27650/44000 | Loss: 2.8940 | LR: 1.57e-04 | 1386.6ms/step | 14.4 samples/s +Step 27660/44000 | Loss: 2.8890 | LR: 1.56e-04 | 1384.9ms/step | 14.4 samples/s +Step 27670/44000 | Loss: 2.8943 | LR: 1.56e-04 | 1385.9ms/step | 14.4 samples/s +Step 27680/44000 | Loss: 2.8966 | LR: 1.56e-04 | 1385.9ms/step | 14.4 samples/s +Step 27690/44000 | Loss: 2.8900 | LR: 1.56e-04 | 1386.2ms/step | 14.4 samples/s +Step 27700/44000 | Loss: 2.8764 | LR: 1.56e-04 | 1385.0ms/step | 14.4 samples/s +Step 27710/44000 | Loss: 2.9054 | LR: 1.56e-04 | 1386.2ms/step | 14.4 samples/s +Step 27720/44000 | Loss: 2.8920 | LR: 1.55e-04 | 1386.1ms/step | 14.4 samples/s +Step 27730/44000 | Loss: 2.9215 | LR: 1.55e-04 | 1385.8ms/step | 14.4 samples/s +Step 27740/44000 | Loss: 2.9098 | LR: 1.55e-04 | 1385.8ms/step | 14.4 samples/s +Step 27750/44000 | Loss: 2.9067 | LR: 1.55e-04 | 1385.9ms/step | 14.4 samples/s +Step 27760/44000 | Loss: 2.8922 | LR: 1.55e-04 | 1384.5ms/step | 14.4 samples/s +Step 27770/44000 | Loss: 2.9006 | LR: 1.55e-04 | 1384.3ms/step | 14.4 samples/s +Step 27780/44000 | Loss: 2.9159 | LR: 1.54e-04 | 1381.8ms/step | 14.5 samples/s +Step 27790/44000 | Loss: 2.9019 | LR: 1.54e-04 | 1383.5ms/step | 14.5 samples/s +Step 27800/44000 | Loss: 2.9100 | LR: 1.54e-04 | 1384.3ms/step | 14.4 samples/s +Step 27810/44000 | Loss: 2.9235 | LR: 1.54e-04 | 1386.0ms/step | 14.4 samples/s +Step 27820/44000 | Loss: 2.8869 | LR: 1.54e-04 | 1385.9ms/step | 14.4 samples/s +Step 27830/44000 | Loss: 2.9058 | LR: 1.54e-04 | 1385.9ms/step | 14.4 samples/s +Step 27840/44000 | Loss: 2.9333 | LR: 1.53e-04 | 1387.0ms/step | 14.4 samples/s +Step 27850/44000 | Loss: 2.9194 | LR: 1.53e-04 | 1386.8ms/step | 14.4 samples/s +Step 27860/44000 | Loss: 2.8908 | LR: 1.53e-04 | 1387.8ms/step | 14.4 samples/s +Step 27870/44000 | Loss: 2.9043 | LR: 1.53e-04 | 1386.5ms/step | 14.4 samples/s +Step 27880/44000 | Loss: 2.8689 | LR: 1.53e-04 | 1384.8ms/step | 14.4 samples/s +Step 27890/44000 | Loss: 2.8754 | LR: 1.53e-04 | 1386.9ms/step | 14.4 samples/s +Step 27900/44000 | Loss: 2.8881 | LR: 1.52e-04 | 1385.5ms/step | 14.4 samples/s +Step 27910/44000 | Loss: 2.9081 | LR: 1.52e-04 | 1387.3ms/step | 14.4 samples/s +Step 27920/44000 | Loss: 2.8773 | LR: 1.52e-04 | 1389.3ms/step | 14.4 samples/s +Step 27930/44000 | Loss: 2.9001 | LR: 1.52e-04 | 1388.3ms/step | 14.4 samples/s +Step 27940/44000 | Loss: 2.8945 | LR: 1.52e-04 | 1384.1ms/step | 14.4 samples/s +Step 27950/44000 | Loss: 2.8902 | LR: 1.52e-04 | 1382.1ms/step | 14.5 samples/s +Step 27960/44000 | Loss: 2.9241 | LR: 1.51e-04 | 1383.3ms/step | 14.5 samples/s +Step 27970/44000 | Loss: 2.8999 | LR: 1.51e-04 | 1383.2ms/step | 14.5 samples/s +Step 27980/44000 | Loss: 2.8830 | LR: 1.51e-04 | 1384.1ms/step | 14.4 samples/s +Step 27990/44000 | Loss: 2.9009 | LR: 1.51e-04 | 1384.4ms/step | 14.4 samples/s +Step 28000/44000 | Loss: 2.9206 | LR: 1.51e-04 | 1384.5ms/step | 14.4 samples/s +Validation at step 28000: Loss = 2.9049, Perplexity = 18.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-28000 +Step 28010/44000 | Loss: 2.9171 | LR: 1.51e-04 | 3882.9ms/step | 5.2 samples/s +Step 28020/44000 | Loss: 2.8954 | LR: 1.50e-04 | 1379.2ms/step | 14.5 samples/s +Step 28030/44000 | Loss: 2.9170 | LR: 1.50e-04 | 1378.0ms/step | 14.5 samples/s +Step 28040/44000 | Loss: 2.9023 | LR: 1.50e-04 | 1380.7ms/step | 14.5 samples/s +Step 28050/44000 | Loss: 2.8991 | LR: 1.50e-04 | 1381.9ms/step | 14.5 samples/s +Step 28060/44000 | Loss: 2.8925 | LR: 1.50e-04 | 1382.1ms/step | 14.5 samples/s +Step 28070/44000 | Loss: 2.9002 | LR: 1.50e-04 | 1382.8ms/step | 14.5 samples/s +Step 28080/44000 | Loss: 2.9100 | LR: 1.49e-04 | 1383.4ms/step | 14.5 samples/s +Step 28090/44000 | Loss: 2.9213 | LR: 1.49e-04 | 1384.4ms/step | 14.4 samples/s +Step 28100/44000 | Loss: 2.9226 | LR: 1.49e-04 | 1384.8ms/step | 14.4 samples/s +Step 28110/44000 | Loss: 2.8948 | LR: 1.49e-04 | 1384.1ms/step | 14.5 samples/s +Step 28120/44000 | Loss: 2.8748 | LR: 1.49e-04 | 1384.8ms/step | 14.4 samples/s +Step 28130/44000 | Loss: 2.8858 | LR: 1.49e-04 | 1385.3ms/step | 14.4 samples/s +Step 28140/44000 | Loss: 2.8817 | LR: 1.48e-04 | 1384.7ms/step | 14.4 samples/s +Step 28150/44000 | Loss: 2.9042 | LR: 1.48e-04 | 1385.2ms/step | 14.4 samples/s +Step 28160/44000 | Loss: 2.8706 | LR: 1.48e-04 | 1384.4ms/step | 14.4 samples/s +Step 28170/44000 | Loss: 2.9044 | LR: 1.48e-04 | 1382.0ms/step | 14.5 samples/s +Step 28180/44000 | Loss: 2.9339 | LR: 1.48e-04 | 1380.5ms/step | 14.5 samples/s +Step 28190/44000 | Loss: 2.9104 | LR: 1.48e-04 | 1383.1ms/step | 14.5 samples/s +Step 28200/44000 | Loss: 2.9266 | LR: 1.47e-04 | 1383.7ms/step | 14.5 samples/s +Step 28210/44000 | Loss: 2.9091 | LR: 1.47e-04 | 1383.6ms/step | 14.5 samples/s +Step 28220/44000 | Loss: 2.9098 | LR: 1.47e-04 | 1385.1ms/step | 14.4 samples/s +Step 28230/44000 | Loss: 2.9124 | LR: 1.47e-04 | 1383.8ms/step | 14.5 samples/s +Step 28240/44000 | Loss: 2.9262 | LR: 1.47e-04 | 1384.5ms/step | 14.4 samples/s +Step 28250/44000 | Loss: 2.8970 | LR: 1.47e-04 | 1384.1ms/step | 14.5 samples/s +Step 28260/44000 | Loss: 2.9299 | LR: 1.46e-04 | 1383.3ms/step | 14.5 samples/s +Step 28270/44000 | Loss: 2.9154 | LR: 1.46e-04 | 1384.1ms/step | 14.4 samples/s +Step 28280/44000 | Loss: 2.8812 | LR: 1.46e-04 | 1384.3ms/step | 14.4 samples/s +Step 28290/44000 | Loss: 2.9074 | LR: 1.46e-04 | 1384.7ms/step | 14.4 samples/s +Step 28300/44000 | Loss: 2.9070 | LR: 1.46e-04 | 1385.1ms/step | 14.4 samples/s +Step 28310/44000 | Loss: 2.8963 | LR: 1.46e-04 | 1384.4ms/step | 14.4 samples/s +Step 28320/44000 | Loss: 2.8910 | LR: 1.45e-04 | 1385.6ms/step | 14.4 samples/s +Step 28330/44000 | Loss: 2.9144 | LR: 1.45e-04 | 1384.4ms/step | 14.4 samples/s +Step 28340/44000 | Loss: 2.8999 | LR: 1.45e-04 | 1383.4ms/step | 14.5 samples/s +Step 28350/44000 | Loss: 2.8894 | LR: 1.45e-04 | 1384.5ms/step | 14.4 samples/s +Step 28360/44000 | Loss: 2.8761 | LR: 1.45e-04 | 1382.7ms/step | 14.5 samples/s +Step 28370/44000 | Loss: 2.8933 | LR: 1.45e-04 | 1382.0ms/step | 14.5 samples/s +Step 28380/44000 | Loss: 2.8937 | LR: 1.44e-04 | 1382.9ms/step | 14.5 samples/s +Step 28390/44000 | Loss: 2.9093 | LR: 1.44e-04 | 1382.8ms/step | 14.5 samples/s +Step 28400/44000 | Loss: 2.8764 | LR: 1.44e-04 | 1384.9ms/step | 14.4 samples/s +Step 28410/44000 | Loss: 2.9049 | LR: 1.44e-04 | 1385.1ms/step | 14.4 samples/s +Step 28420/44000 | Loss: 2.8946 | LR: 1.44e-04 | 1385.7ms/step | 14.4 samples/s +Step 28430/44000 | Loss: 2.9412 | LR: 1.44e-04 | 1387.6ms/step | 14.4 samples/s +Step 28440/44000 | Loss: 2.8774 | LR: 1.43e-04 | 1386.4ms/step | 14.4 samples/s +Step 28450/44000 | Loss: 2.9000 | LR: 1.43e-04 | 1386.2ms/step | 14.4 samples/s +Step 28460/44000 | Loss: 2.9247 | LR: 1.43e-04 | 1383.2ms/step | 14.5 samples/s +Step 28470/44000 | Loss: 2.8743 | LR: 1.43e-04 | 1382.8ms/step | 14.5 samples/s +Step 28480/44000 | Loss: 2.8791 | LR: 1.43e-04 | 1385.0ms/step | 14.4 samples/s +Step 28490/44000 | Loss: 2.9329 | LR: 1.43e-04 | 1382.1ms/step | 14.5 samples/s +Step 28500/44000 | Loss: 2.9213 | LR: 1.42e-04 | 1383.5ms/step | 14.5 samples/s +Step 28510/44000 | Loss: 2.9075 | LR: 1.42e-04 | 1384.9ms/step | 14.4 samples/s +Step 28520/44000 | Loss: 2.9082 | LR: 1.42e-04 | 1384.5ms/step | 14.4 samples/s +Step 28530/44000 | Loss: 2.8968 | LR: 1.42e-04 | 1385.5ms/step | 14.4 samples/s +Step 28540/44000 | Loss: 2.9237 | LR: 1.42e-04 | 1385.4ms/step | 14.4 samples/s +Step 28550/44000 | Loss: 2.8940 | LR: 1.42e-04 | 1384.8ms/step | 14.4 samples/s +Step 28560/44000 | Loss: 2.8981 | LR: 1.41e-04 | 1383.9ms/step | 14.5 samples/s +Step 28570/44000 | Loss: 2.8857 | LR: 1.41e-04 | 1383.4ms/step | 14.5 samples/s +Step 28580/44000 | Loss: 2.8966 | LR: 1.41e-04 | 1382.8ms/step | 14.5 samples/s +Step 28590/44000 | Loss: 2.9095 | LR: 1.41e-04 | 1386.0ms/step | 14.4 samples/s +Step 28600/44000 | Loss: 2.8888 | LR: 1.41e-04 | 1385.7ms/step | 14.4 samples/s +Step 28610/44000 | Loss: 2.9179 | LR: 1.41e-04 | 1386.0ms/step | 14.4 samples/s +Step 28620/44000 | Loss: 2.9228 | LR: 1.40e-04 | 1384.2ms/step | 14.4 samples/s +Step 28630/44000 | Loss: 2.8979 | LR: 1.40e-04 | 1383.9ms/step | 14.5 samples/s +Step 28640/44000 | Loss: 2.9213 | LR: 1.40e-04 | 1386.0ms/step | 14.4 samples/s +Step 28650/44000 | Loss: 2.9119 | LR: 1.40e-04 | 1384.9ms/step | 14.4 samples/s +Step 28660/44000 | Loss: 2.8845 | LR: 1.40e-04 | 1384.0ms/step | 14.5 samples/s +Step 28670/44000 | Loss: 2.8879 | LR: 1.40e-04 | 1384.1ms/step | 14.4 samples/s +Step 28680/44000 | Loss: 2.9099 | LR: 1.39e-04 | 1383.3ms/step | 14.5 samples/s +Step 28690/44000 | Loss: 2.9055 | LR: 1.39e-04 | 1383.1ms/step | 14.5 samples/s +Step 28700/44000 | Loss: 2.9030 | LR: 1.39e-04 | 1383.5ms/step | 14.5 samples/s +Step 28710/44000 | Loss: 2.8706 | LR: 1.39e-04 | 1386.6ms/step | 14.4 samples/s +Step 28720/44000 | Loss: 2.9295 | LR: 1.39e-04 | 1382.7ms/step | 14.5 samples/s +Step 28730/44000 | Loss: 2.9498 | LR: 1.39e-04 | 1385.2ms/step | 14.4 samples/s +Step 28740/44000 | Loss: 2.9198 | LR: 1.38e-04 | 1386.2ms/step | 14.4 samples/s +Step 28750/44000 | Loss: 2.9013 | LR: 1.38e-04 | 1387.1ms/step | 14.4 samples/s +Step 28760/44000 | Loss: 2.8998 | LR: 1.38e-04 | 1384.5ms/step | 14.4 samples/s +Step 28770/44000 | Loss: 2.8759 | LR: 1.38e-04 | 1384.0ms/step | 14.5 samples/s +Step 28780/44000 | Loss: 2.8984 | LR: 1.38e-04 | 1384.1ms/step | 14.5 samples/s +Step 28790/44000 | Loss: 2.9075 | LR: 1.38e-04 | 1382.9ms/step | 14.5 samples/s +Step 28800/44000 | Loss: 2.9171 | LR: 1.38e-04 | 1381.1ms/step | 14.5 samples/s +Step 28810/44000 | Loss: 2.8931 | LR: 1.37e-04 | 1382.5ms/step | 14.5 samples/s +Step 28820/44000 | Loss: 2.9056 | LR: 1.37e-04 | 1382.9ms/step | 14.5 samples/s +Step 28830/44000 | Loss: 2.9054 | LR: 1.37e-04 | 1385.4ms/step | 14.4 samples/s +Step 28840/44000 | Loss: 2.8787 | LR: 1.37e-04 | 1383.4ms/step | 14.5 samples/s +Step 28850/44000 | Loss: 2.8708 | LR: 1.37e-04 | 1382.5ms/step | 14.5 samples/s +Step 28860/44000 | Loss: 2.8776 | LR: 1.37e-04 | 1385.8ms/step | 14.4 samples/s +Step 28870/44000 | Loss: 2.8999 | LR: 1.36e-04 | 1384.5ms/step | 14.4 samples/s +Step 28880/44000 | Loss: 2.9050 | LR: 1.36e-04 | 1384.5ms/step | 14.4 samples/s +Step 28890/44000 | Loss: 2.8902 | LR: 1.36e-04 | 1385.3ms/step | 14.4 samples/s +Step 28900/44000 | Loss: 2.8741 | LR: 1.36e-04 | 1385.5ms/step | 14.4 samples/s +Step 28910/44000 | Loss: 2.8874 | LR: 1.36e-04 | 1385.2ms/step | 14.4 samples/s +Step 28920/44000 | Loss: 2.9020 | LR: 1.36e-04 | 1384.4ms/step | 14.4 samples/s +Step 28930/44000 | Loss: 2.9291 | LR: 1.35e-04 | 1386.4ms/step | 14.4 samples/s +Step 28940/44000 | Loss: 2.8933 | LR: 1.35e-04 | 1381.8ms/step | 14.5 samples/s +Step 28950/44000 | Loss: 2.8988 | LR: 1.35e-04 | 1385.1ms/step | 14.4 samples/s +Step 28960/44000 | Loss: 2.8900 | LR: 1.35e-04 | 1384.0ms/step | 14.5 samples/s +Step 28970/44000 | Loss: 2.8781 | LR: 1.35e-04 | 1385.2ms/step | 14.4 samples/s +Step 28980/44000 | Loss: 2.9106 | LR: 1.35e-04 | 1384.8ms/step | 14.4 samples/s +Step 28990/44000 | Loss: 2.9152 | LR: 1.34e-04 | 1382.1ms/step | 14.5 samples/s +Step 29000/44000 | Loss: 2.8887 | LR: 1.34e-04 | 1380.6ms/step | 14.5 samples/s +Validation at step 29000: Loss = 2.9005, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-29000 +Step 29010/44000 | Loss: 2.8910 | LR: 1.34e-04 | 3856.9ms/step | 5.2 samples/s +Step 29020/44000 | Loss: 2.8895 | LR: 1.34e-04 | 1383.0ms/step | 14.5 samples/s +Step 29030/44000 | Loss: 2.9055 | LR: 1.34e-04 | 1382.8ms/step | 14.5 samples/s +Step 29040/44000 | Loss: 2.9207 | LR: 1.34e-04 | 1383.0ms/step | 14.5 samples/s +Step 29050/44000 | Loss: 2.9098 | LR: 1.33e-04 | 1383.5ms/step | 14.5 samples/s +Step 29060/44000 | Loss: 2.8850 | LR: 1.33e-04 | 1384.0ms/step | 14.5 samples/s +Step 29070/44000 | Loss: 2.8955 | LR: 1.33e-04 | 1381.8ms/step | 14.5 samples/s +Step 29080/44000 | Loss: 2.8793 | LR: 1.33e-04 | 1381.3ms/step | 14.5 samples/s +Step 29090/44000 | Loss: 2.8960 | LR: 1.33e-04 | 1380.3ms/step | 14.5 samples/s +Step 29100/44000 | Loss: 2.9120 | LR: 1.33e-04 | 1382.6ms/step | 14.5 samples/s +Step 29110/44000 | Loss: 2.9208 | LR: 1.33e-04 | 1382.3ms/step | 14.5 samples/s +Step 29120/44000 | Loss: 2.8809 | LR: 1.32e-04 | 1382.8ms/step | 14.5 samples/s +Step 29130/44000 | Loss: 2.9065 | LR: 1.32e-04 | 1382.7ms/step | 14.5 samples/s +Step 29140/44000 | Loss: 2.8905 | LR: 1.32e-04 | 1383.7ms/step | 14.5 samples/s +Step 29150/44000 | Loss: 2.9192 | LR: 1.32e-04 | 1385.8ms/step | 14.4 samples/s +Step 29160/44000 | Loss: 2.8634 | LR: 1.32e-04 | 1384.9ms/step | 14.4 samples/s +Step 29170/44000 | Loss: 2.8846 | LR: 1.32e-04 | 1383.8ms/step | 14.5 samples/s +Step 29180/44000 | Loss: 2.9043 | LR: 1.31e-04 | 1385.6ms/step | 14.4 samples/s +Step 29190/44000 | Loss: 2.8845 | LR: 1.31e-04 | 1384.2ms/step | 14.4 samples/s +Step 29200/44000 | Loss: 2.9010 | LR: 1.31e-04 | 1385.1ms/step | 14.4 samples/s +Step 29210/44000 | Loss: 2.9251 | LR: 1.31e-04 | 1384.9ms/step | 14.4 samples/s +Step 29220/44000 | Loss: 2.8991 | LR: 1.31e-04 | 1382.9ms/step | 14.5 samples/s +Step 29230/44000 | Loss: 2.8889 | LR: 1.31e-04 | 1383.4ms/step | 14.5 samples/s +Step 29240/44000 | Loss: 2.8850 | LR: 1.30e-04 | 1382.3ms/step | 14.5 samples/s +Step 29250/44000 | Loss: 2.9303 | LR: 1.30e-04 | 1380.1ms/step | 14.5 samples/s +Step 29260/44000 | Loss: 2.8788 | LR: 1.30e-04 | 1382.1ms/step | 14.5 samples/s +Step 29270/44000 | Loss: 2.8956 | LR: 1.30e-04 | 1382.4ms/step | 14.5 samples/s +Step 29280/44000 | Loss: 2.8955 | LR: 1.30e-04 | 1384.8ms/step | 14.4 samples/s +Step 29290/44000 | Loss: 2.8878 | LR: 1.30e-04 | 1383.9ms/step | 14.5 samples/s +Step 29300/44000 | Loss: 2.9102 | LR: 1.29e-04 | 1383.8ms/step | 14.5 samples/s +Step 29310/44000 | Loss: 2.8889 | LR: 1.29e-04 | 1382.0ms/step | 14.5 samples/s +Step 29320/44000 | Loss: 2.8785 | LR: 1.29e-04 | 1380.9ms/step | 14.5 samples/s +Step 29330/44000 | Loss: 2.8626 | LR: 1.29e-04 | 1383.8ms/step | 14.5 samples/s +Step 29340/44000 | Loss: 2.8971 | LR: 1.29e-04 | 1384.2ms/step | 14.4 samples/s +Step 29350/44000 | Loss: 2.8582 | LR: 1.29e-04 | 1382.9ms/step | 14.5 samples/s +Step 29360/44000 | Loss: 2.9053 | LR: 1.29e-04 | 1382.3ms/step | 14.5 samples/s +Step 29370/44000 | Loss: 2.8990 | LR: 1.28e-04 | 1381.8ms/step | 14.5 samples/s +Step 29380/44000 | Loss: 2.9097 | LR: 1.28e-04 | 1383.1ms/step | 14.5 samples/s +Step 29390/44000 | Loss: 2.8930 | LR: 1.28e-04 | 1383.1ms/step | 14.5 samples/s +Step 29400/44000 | Loss: 2.8931 | LR: 1.28e-04 | 1383.8ms/step | 14.5 samples/s +Step 29410/44000 | Loss: 2.9039 | LR: 1.28e-04 | 1382.6ms/step | 14.5 samples/s +Step 29420/44000 | Loss: 2.8638 | LR: 1.28e-04 | 1383.6ms/step | 14.5 samples/s +Step 29430/44000 | Loss: 2.9035 | LR: 1.27e-04 | 1381.5ms/step | 14.5 samples/s +Step 29440/44000 | Loss: 2.8834 | LR: 1.27e-04 | 1383.4ms/step | 14.5 samples/s +Step 29450/44000 | Loss: 2.8924 | LR: 1.27e-04 | 1382.7ms/step | 14.5 samples/s +Step 29460/44000 | Loss: 2.9186 | LR: 1.27e-04 | 1381.4ms/step | 14.5 samples/s +Step 29470/44000 | Loss: 2.8932 | LR: 1.27e-04 | 1380.2ms/step | 14.5 samples/s +Step 29480/44000 | Loss: 2.8580 | LR: 1.27e-04 | 1381.9ms/step | 14.5 samples/s +Step 29490/44000 | Loss: 2.9058 | LR: 1.26e-04 | 1382.9ms/step | 14.5 samples/s +Step 29500/44000 | Loss: 2.8894 | LR: 1.26e-04 | 1382.6ms/step | 14.5 samples/s +Step 29510/44000 | Loss: 2.8507 | LR: 1.26e-04 | 1382.8ms/step | 14.5 samples/s +Step 29520/44000 | Loss: 2.8981 | LR: 1.26e-04 | 1383.0ms/step | 14.5 samples/s +Step 29530/44000 | Loss: 2.9102 | LR: 1.26e-04 | 1381.8ms/step | 14.5 samples/s +Step 29540/44000 | Loss: 2.8848 | LR: 1.26e-04 | 1384.8ms/step | 14.4 samples/s +Step 29550/44000 | Loss: 2.8964 | LR: 1.26e-04 | 1385.3ms/step | 14.4 samples/s +Step 29560/44000 | Loss: 2.8930 | LR: 1.25e-04 | 1384.5ms/step | 14.4 samples/s +Step 29570/44000 | Loss: 2.9129 | LR: 1.25e-04 | 1383.1ms/step | 14.5 samples/s +Step 29580/44000 | Loss: 2.8845 | LR: 1.25e-04 | 1382.9ms/step | 14.5 samples/s +Step 29590/44000 | Loss: 2.8790 | LR: 1.25e-04 | 1381.1ms/step | 14.5 samples/s +Step 29600/44000 | Loss: 2.8615 | LR: 1.25e-04 | 1381.9ms/step | 14.5 samples/s +Step 29610/44000 | Loss: 2.8928 | LR: 1.25e-04 | 1383.3ms/step | 14.5 samples/s +Step 29620/44000 | Loss: 2.8752 | LR: 1.24e-04 | 1385.0ms/step | 14.4 samples/s +Step 29630/44000 | Loss: 2.9118 | LR: 1.24e-04 | 1383.5ms/step | 14.5 samples/s +Step 29640/44000 | Loss: 2.9001 | LR: 1.24e-04 | 1384.5ms/step | 14.4 samples/s +Step 29650/44000 | Loss: 2.9220 | LR: 1.24e-04 | 1385.9ms/step | 14.4 samples/s +Step 29660/44000 | Loss: 2.9095 | LR: 1.24e-04 | 1385.1ms/step | 14.4 samples/s +Step 29670/44000 | Loss: 2.8880 | LR: 1.24e-04 | 1382.5ms/step | 14.5 samples/s +Step 29680/44000 | Loss: 2.9018 | LR: 1.23e-04 | 1381.3ms/step | 14.5 samples/s +Step 29690/44000 | Loss: 2.8951 | LR: 1.23e-04 | 1381.4ms/step | 14.5 samples/s +Step 29700/44000 | Loss: 2.8872 | LR: 1.23e-04 | 1380.5ms/step | 14.5 samples/s +Step 29710/44000 | Loss: 2.9080 | LR: 1.23e-04 | 1383.6ms/step | 14.5 samples/s +Step 29720/44000 | Loss: 2.9080 | LR: 1.23e-04 | 1386.0ms/step | 14.4 samples/s +Step 29730/44000 | Loss: 2.9058 | LR: 1.23e-04 | 1382.4ms/step | 14.5 samples/s +Step 29740/44000 | Loss: 2.8757 | LR: 1.23e-04 | 1383.1ms/step | 14.5 samples/s +Step 29750/44000 | Loss: 2.8844 | LR: 1.22e-04 | 1383.6ms/step | 14.5 samples/s +Step 29760/44000 | Loss: 2.9396 | LR: 1.22e-04 | 1382.8ms/step | 14.5 samples/s +Step 29770/44000 | Loss: 2.8929 | LR: 1.22e-04 | 1383.0ms/step | 14.5 samples/s +Step 29780/44000 | Loss: 2.9138 | LR: 1.22e-04 | 1383.8ms/step | 14.5 samples/s +Step 29790/44000 | Loss: 2.9015 | LR: 1.22e-04 | 1385.6ms/step | 14.4 samples/s +Step 29800/44000 | Loss: 2.8995 | LR: 1.22e-04 | 1386.0ms/step | 14.4 samples/s +Step 29810/44000 | Loss: 2.9156 | LR: 1.21e-04 | 1383.9ms/step | 14.5 samples/s +Step 29820/44000 | Loss: 2.9024 | LR: 1.21e-04 | 1383.6ms/step | 14.5 samples/s +Step 29830/44000 | Loss: 2.8698 | LR: 1.21e-04 | 1385.5ms/step | 14.4 samples/s +Step 29840/44000 | Loss: 2.9187 | LR: 1.21e-04 | 1380.7ms/step | 14.5 samples/s +Step 29850/44000 | Loss: 2.8746 | LR: 1.21e-04 | 1380.8ms/step | 14.5 samples/s +Step 29860/44000 | Loss: 2.9304 | LR: 1.21e-04 | 1384.6ms/step | 14.4 samples/s +Step 29870/44000 | Loss: 2.8896 | LR: 1.21e-04 | 1380.5ms/step | 14.5 samples/s +Step 29880/44000 | Loss: 2.8809 | LR: 1.20e-04 | 1382.5ms/step | 14.5 samples/s +Step 29890/44000 | Loss: 2.8891 | LR: 1.20e-04 | 1385.6ms/step | 14.4 samples/s +Step 29900/44000 | Loss: 2.8877 | LR: 1.20e-04 | 1384.4ms/step | 14.4 samples/s +Step 29910/44000 | Loss: 2.8852 | LR: 1.20e-04 | 1383.7ms/step | 14.5 samples/s +Step 29920/44000 | Loss: 2.8612 | LR: 1.20e-04 | 1386.2ms/step | 14.4 samples/s +Step 29930/44000 | Loss: 2.8887 | LR: 1.20e-04 | 1385.2ms/step | 14.4 samples/s +Step 29940/44000 | Loss: 2.8794 | LR: 1.19e-04 | 1384.7ms/step | 14.4 samples/s +Step 29950/44000 | Loss: 2.9034 | LR: 1.19e-04 | 1386.8ms/step | 14.4 samples/s +Step 29960/44000 | Loss: 2.9133 | LR: 1.19e-04 | 1384.8ms/step | 14.4 samples/s +Step 29970/44000 | Loss: 2.8576 | LR: 1.19e-04 | 1382.8ms/step | 14.5 samples/s +Step 29980/44000 | Loss: 2.8789 | LR: 1.19e-04 | 1386.9ms/step | 14.4 samples/s +Step 29990/44000 | Loss: 2.8761 | LR: 1.19e-04 | 1387.6ms/step | 14.4 samples/s +Step 30000/44000 | Loss: 2.8827 | LR: 1.19e-04 | 1385.0ms/step | 14.4 samples/s +Validation at step 30000: Loss = 2.8965, Perplexity = 18.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-30000 +Step 30010/44000 | Loss: 2.9399 | LR: 1.18e-04 | 3905.3ms/step | 5.1 samples/s +Step 30020/44000 | Loss: 2.9056 | LR: 1.18e-04 | 1377.3ms/step | 14.5 samples/s +Step 30030/44000 | Loss: 2.8953 | LR: 1.18e-04 | 1375.1ms/step | 14.5 samples/s +Step 30040/44000 | Loss: 2.8881 | LR: 1.18e-04 | 1379.0ms/step | 14.5 samples/s +Step 30050/44000 | Loss: 2.8882 | LR: 1.18e-04 | 1379.4ms/step | 14.5 samples/s +Step 30060/44000 | Loss: 2.8625 | LR: 1.18e-04 | 1381.6ms/step | 14.5 samples/s +Step 30070/44000 | Loss: 2.9050 | LR: 1.17e-04 | 1384.0ms/step | 14.5 samples/s +Step 30080/44000 | Loss: 2.8602 | LR: 1.17e-04 | 1382.3ms/step | 14.5 samples/s +Step 30090/44000 | Loss: 2.8910 | LR: 1.17e-04 | 1381.4ms/step | 14.5 samples/s +Step 30100/44000 | Loss: 2.9042 | LR: 1.17e-04 | 1380.8ms/step | 14.5 samples/s +Step 30110/44000 | Loss: 2.8750 | LR: 1.17e-04 | 1379.9ms/step | 14.5 samples/s +Step 30120/44000 | Loss: 2.9012 | LR: 1.17e-04 | 1381.4ms/step | 14.5 samples/s +Step 30130/44000 | Loss: 2.8650 | LR: 1.17e-04 | 1387.0ms/step | 14.4 samples/s +Step 30140/44000 | Loss: 2.8926 | LR: 1.16e-04 | 1383.4ms/step | 14.5 samples/s +Step 30150/44000 | Loss: 2.9043 | LR: 1.16e-04 | 1382.0ms/step | 14.5 samples/s +Step 30160/44000 | Loss: 2.8823 | LR: 1.16e-04 | 1383.3ms/step | 14.5 samples/s +Step 30170/44000 | Loss: 2.9098 | LR: 1.16e-04 | 1383.6ms/step | 14.5 samples/s +Step 30180/44000 | Loss: 2.8770 | LR: 1.16e-04 | 1382.8ms/step | 14.5 samples/s +Step 30190/44000 | Loss: 2.9038 | LR: 1.16e-04 | 1382.4ms/step | 14.5 samples/s +Step 30200/44000 | Loss: 2.8956 | LR: 1.15e-04 | 1387.1ms/step | 14.4 samples/s +Step 30210/44000 | Loss: 2.8824 | LR: 1.15e-04 | 1381.5ms/step | 14.5 samples/s +Step 30220/44000 | Loss: 2.8895 | LR: 1.15e-04 | 1381.9ms/step | 14.5 samples/s +Step 30230/44000 | Loss: 2.8984 | LR: 1.15e-04 | 1385.5ms/step | 14.4 samples/s +Step 30240/44000 | Loss: 2.8820 | LR: 1.15e-04 | 1381.0ms/step | 14.5 samples/s +Step 30250/44000 | Loss: 2.9157 | LR: 1.15e-04 | 1381.3ms/step | 14.5 samples/s +Step 30260/44000 | Loss: 2.8709 | LR: 1.15e-04 | 1384.0ms/step | 14.5 samples/s +Step 30270/44000 | Loss: 2.8791 | LR: 1.14e-04 | 1383.3ms/step | 14.5 samples/s +Step 30280/44000 | Loss: 2.9038 | LR: 1.14e-04 | 1382.7ms/step | 14.5 samples/s +Step 30290/44000 | Loss: 2.8998 | LR: 1.14e-04 | 1383.5ms/step | 14.5 samples/s +Step 30300/44000 | Loss: 2.8993 | LR: 1.14e-04 | 1382.1ms/step | 14.5 samples/s +Step 30310/44000 | Loss: 2.9178 | LR: 1.14e-04 | 1382.4ms/step | 14.5 samples/s +Step 30320/44000 | Loss: 2.9045 | LR: 1.14e-04 | 1381.2ms/step | 14.5 samples/s +Step 30330/44000 | Loss: 2.8900 | LR: 1.13e-04 | 1380.6ms/step | 14.5 samples/s +Step 30340/44000 | Loss: 2.9128 | LR: 1.13e-04 | 1380.4ms/step | 14.5 samples/s +Step 30350/44000 | Loss: 2.8601 | LR: 1.13e-04 | 1384.0ms/step | 14.5 samples/s +Step 30360/44000 | Loss: 2.8994 | LR: 1.13e-04 | 1383.6ms/step | 14.5 samples/s +Step 30370/44000 | Loss: 2.8978 | LR: 1.13e-04 | 1384.8ms/step | 14.4 samples/s +Step 30380/44000 | Loss: 2.9313 | LR: 1.13e-04 | 1383.2ms/step | 14.5 samples/s +Step 30390/44000 | Loss: 2.9016 | LR: 1.13e-04 | 1382.0ms/step | 14.5 samples/s +Step 30400/44000 | Loss: 2.9061 | LR: 1.12e-04 | 1382.7ms/step | 14.5 samples/s +Step 30410/44000 | Loss: 2.8511 | LR: 1.12e-04 | 1382.1ms/step | 14.5 samples/s +Step 30420/44000 | Loss: 2.8645 | LR: 1.12e-04 | 1382.9ms/step | 14.5 samples/s +Step 30430/44000 | Loss: 2.9141 | LR: 1.12e-04 | 1382.3ms/step | 14.5 samples/s +Step 30440/44000 | Loss: 2.8869 | LR: 1.12e-04 | 1381.6ms/step | 14.5 samples/s +Step 30450/44000 | Loss: 2.8756 | LR: 1.12e-04 | 1380.3ms/step | 14.5 samples/s +Step 30460/44000 | Loss: 2.8782 | LR: 1.11e-04 | 1380.1ms/step | 14.5 samples/s +Step 30470/44000 | Loss: 2.8984 | LR: 1.11e-04 | 1381.9ms/step | 14.5 samples/s +Step 30480/44000 | Loss: 2.9155 | LR: 1.11e-04 | 1383.0ms/step | 14.5 samples/s +Step 30490/44000 | Loss: 2.8935 | LR: 1.11e-04 | 1381.4ms/step | 14.5 samples/s +Step 30500/44000 | Loss: 2.9002 | LR: 1.11e-04 | 1383.5ms/step | 14.5 samples/s +Step 30510/44000 | Loss: 2.8763 | LR: 1.11e-04 | 1383.8ms/step | 14.5 samples/s +Step 30520/44000 | Loss: 2.8843 | LR: 1.11e-04 | 1381.7ms/step | 14.5 samples/s +Step 30530/44000 | Loss: 2.9029 | LR: 1.10e-04 | 1381.0ms/step | 14.5 samples/s +Step 30540/44000 | Loss: 2.9065 | LR: 1.10e-04 | 1382.1ms/step | 14.5 samples/s +Step 30550/44000 | Loss: 2.8857 | LR: 1.10e-04 | 1381.8ms/step | 14.5 samples/s +Step 30560/44000 | Loss: 2.8741 | LR: 1.10e-04 | 1381.7ms/step | 14.5 samples/s +Step 30570/44000 | Loss: 2.9072 | LR: 1.10e-04 | 1382.0ms/step | 14.5 samples/s +Step 30580/44000 | Loss: 2.8750 | LR: 1.10e-04 | 1381.4ms/step | 14.5 samples/s +Step 30590/44000 | Loss: 2.9127 | LR: 1.10e-04 | 1381.2ms/step | 14.5 samples/s +Step 30600/44000 | Loss: 2.8615 | LR: 1.09e-04 | 1377.6ms/step | 14.5 samples/s +Step 30610/44000 | Loss: 2.8733 | LR: 1.09e-04 | 1380.7ms/step | 14.5 samples/s +Step 30620/44000 | Loss: 2.8764 | LR: 1.09e-04 | 1379.3ms/step | 14.5 samples/s +Step 30630/44000 | Loss: 2.8897 | LR: 1.09e-04 | 1382.3ms/step | 14.5 samples/s +Step 30640/44000 | Loss: 2.8872 | LR: 1.09e-04 | 1381.8ms/step | 14.5 samples/s +Step 30650/44000 | Loss: 2.8641 | LR: 1.09e-04 | 1381.8ms/step | 14.5 samples/s +Step 30660/44000 | Loss: 2.8973 | LR: 1.08e-04 | 1382.2ms/step | 14.5 samples/s +Step 30670/44000 | Loss: 2.8698 | LR: 1.08e-04 | 1380.3ms/step | 14.5 samples/s +Step 30680/44000 | Loss: 2.8821 | LR: 1.08e-04 | 1381.4ms/step | 14.5 samples/s +Step 30690/44000 | Loss: 2.8914 | LR: 1.08e-04 | 1383.5ms/step | 14.5 samples/s +Step 30700/44000 | Loss: 2.9131 | LR: 1.08e-04 | 1381.1ms/step | 14.5 samples/s +Step 30710/44000 | Loss: 2.8994 | LR: 1.08e-04 | 1383.7ms/step | 14.5 samples/s +Step 30720/44000 | Loss: 2.8775 | LR: 1.08e-04 | 1381.2ms/step | 14.5 samples/s +Step 30730/44000 | Loss: 2.9006 | LR: 1.07e-04 | 1383.6ms/step | 14.5 samples/s +Step 30740/44000 | Loss: 2.8885 | LR: 1.07e-04 | 1382.1ms/step | 14.5 samples/s +Step 30750/44000 | Loss: 2.8675 | LR: 1.07e-04 | 1382.3ms/step | 14.5 samples/s +Step 30760/44000 | Loss: 2.8698 | LR: 1.07e-04 | 1383.1ms/step | 14.5 samples/s +Step 30770/44000 | Loss: 2.8795 | LR: 1.07e-04 | 1383.9ms/step | 14.5 samples/s +Step 30780/44000 | Loss: 2.8894 | LR: 1.07e-04 | 1383.9ms/step | 14.5 samples/s +Step 30790/44000 | Loss: 2.8981 | LR: 1.07e-04 | 1382.0ms/step | 14.5 samples/s +Step 30800/44000 | Loss: 2.9460 | LR: 1.06e-04 | 1381.7ms/step | 14.5 samples/s +Step 30810/44000 | Loss: 2.9386 | LR: 1.06e-04 | 1378.8ms/step | 14.5 samples/s +Step 30820/44000 | Loss: 2.8988 | LR: 1.06e-04 | 1381.1ms/step | 14.5 samples/s +Step 30830/44000 | Loss: 2.8870 | LR: 1.06e-04 | 1381.0ms/step | 14.5 samples/s +Step 30840/44000 | Loss: 2.8949 | LR: 1.06e-04 | 1378.1ms/step | 14.5 samples/s +Step 30850/44000 | Loss: 2.9252 | LR: 1.06e-04 | 1378.8ms/step | 14.5 samples/s +Step 30860/44000 | Loss: 2.8956 | LR: 1.05e-04 | 1381.7ms/step | 14.5 samples/s +Step 30870/44000 | Loss: 2.8779 | LR: 1.05e-04 | 1380.7ms/step | 14.5 samples/s +Step 30880/44000 | Loss: 2.8881 | LR: 1.05e-04 | 1382.0ms/step | 14.5 samples/s +Step 30890/44000 | Loss: 2.9089 | LR: 1.05e-04 | 1383.9ms/step | 14.5 samples/s +Step 30900/44000 | Loss: 2.8697 | LR: 1.05e-04 | 1379.5ms/step | 14.5 samples/s +Step 30910/44000 | Loss: 2.8832 | LR: 1.05e-04 | 1378.1ms/step | 14.5 samples/s +Step 30920/44000 | Loss: 2.8817 | LR: 1.05e-04 | 1381.3ms/step | 14.5 samples/s +Step 30930/44000 | Loss: 2.8853 | LR: 1.04e-04 | 1381.5ms/step | 14.5 samples/s +Step 30940/44000 | Loss: 2.8871 | LR: 1.04e-04 | 1381.4ms/step | 14.5 samples/s +Step 30950/44000 | Loss: 2.8960 | LR: 1.04e-04 | 1383.5ms/step | 14.5 samples/s +Step 30960/44000 | Loss: 2.9131 | LR: 1.04e-04 | 1382.8ms/step | 14.5 samples/s +Step 30970/44000 | Loss: 2.8621 | LR: 1.04e-04 | 1382.6ms/step | 14.5 samples/s +Step 30980/44000 | Loss: 2.9016 | LR: 1.04e-04 | 1383.1ms/step | 14.5 samples/s +Step 30990/44000 | Loss: 2.9056 | LR: 1.04e-04 | 1380.9ms/step | 14.5 samples/s +Step 31000/44000 | Loss: 2.8930 | LR: 1.03e-04 | 1380.2ms/step | 14.5 samples/s +Validation at step 31000: Loss = 2.8933, Perplexity = 18.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-31000 +Step 31010/44000 | Loss: 2.8819 | LR: 1.03e-04 | 3874.9ms/step | 5.2 samples/s +Step 31020/44000 | Loss: 2.9023 | LR: 1.03e-04 | 1381.2ms/step | 14.5 samples/s +Step 31030/44000 | Loss: 2.8753 | LR: 1.03e-04 | 1381.0ms/step | 14.5 samples/s +Step 31040/44000 | Loss: 2.8811 | LR: 1.03e-04 | 1382.2ms/step | 14.5 samples/s +Step 31050/44000 | Loss: 2.8636 | LR: 1.03e-04 | 1383.5ms/step | 14.5 samples/s +Step 31060/44000 | Loss: 2.8919 | LR: 1.03e-04 | 1383.2ms/step | 14.5 samples/s +Step 31070/44000 | Loss: 2.9063 | LR: 1.02e-04 | 1382.8ms/step | 14.5 samples/s +Step 31080/44000 | Loss: 2.8968 | LR: 1.02e-04 | 1384.1ms/step | 14.4 samples/s +Step 31090/44000 | Loss: 2.8681 | LR: 1.02e-04 | 1385.8ms/step | 14.4 samples/s +Step 31100/44000 | Loss: 2.8906 | LR: 1.02e-04 | 1386.0ms/step | 14.4 samples/s +Step 31110/44000 | Loss: 2.8917 | LR: 1.02e-04 | 1385.2ms/step | 14.4 samples/s +Step 31120/44000 | Loss: 2.8909 | LR: 1.02e-04 | 1382.6ms/step | 14.5 samples/s +Step 31130/44000 | Loss: 2.8850 | LR: 1.02e-04 | 1384.7ms/step | 14.4 samples/s +Step 31140/44000 | Loss: 2.8800 | LR: 1.01e-04 | 1383.9ms/step | 14.5 samples/s +Step 31150/44000 | Loss: 2.8823 | LR: 1.01e-04 | 1384.3ms/step | 14.4 samples/s +Step 31160/44000 | Loss: 2.9179 | LR: 1.01e-04 | 1386.2ms/step | 14.4 samples/s +Step 31170/44000 | Loss: 2.9059 | LR: 1.01e-04 | 1383.5ms/step | 14.5 samples/s +Step 31180/44000 | Loss: 2.9084 | LR: 1.01e-04 | 1382.3ms/step | 14.5 samples/s +Step 31190/44000 | Loss: 2.8617 | LR: 1.01e-04 | 1380.9ms/step | 14.5 samples/s +Step 31200/44000 | Loss: 2.8722 | LR: 1.00e-04 | 1383.1ms/step | 14.5 samples/s +Step 31210/44000 | Loss: 2.8772 | LR: 1.00e-04 | 1385.5ms/step | 14.4 samples/s +Step 31220/44000 | Loss: 2.8709 | LR: 1.00e-04 | 1384.4ms/step | 14.4 samples/s +Step 31230/44000 | Loss: 2.9071 | LR: 1.00e-04 | 1385.9ms/step | 14.4 samples/s +Step 31240/44000 | Loss: 2.8635 | LR: 9.99e-05 | 1387.3ms/step | 14.4 samples/s +Step 31250/44000 | Loss: 2.9039 | LR: 9.98e-05 | 1384.8ms/step | 14.4 samples/s +Step 31260/44000 | Loss: 2.9000 | LR: 9.96e-05 | 1385.8ms/step | 14.4 samples/s +Step 31270/44000 | Loss: 2.9030 | LR: 9.95e-05 | 1383.6ms/step | 14.5 samples/s +Step 31280/44000 | Loss: 2.8911 | LR: 9.93e-05 | 1381.1ms/step | 14.5 samples/s +Step 31290/44000 | Loss: 2.8823 | LR: 9.92e-05 | 1383.9ms/step | 14.5 samples/s +Step 31300/44000 | Loss: 2.9308 | LR: 9.90e-05 | 1386.0ms/step | 14.4 samples/s +Step 31310/44000 | Loss: 2.9173 | LR: 9.89e-05 | 1384.5ms/step | 14.4 samples/s +Step 31320/44000 | Loss: 2.8845 | LR: 9.88e-05 | 1383.7ms/step | 14.5 samples/s +Step 31330/44000 | Loss: 2.9142 | LR: 9.86e-05 | 1383.9ms/step | 14.5 samples/s +Step 31340/44000 | Loss: 2.8891 | LR: 9.85e-05 | 1385.7ms/step | 14.4 samples/s +Step 31350/44000 | Loss: 2.8932 | LR: 9.83e-05 | 1385.5ms/step | 14.4 samples/s +Step 31360/44000 | Loss: 2.9014 | LR: 9.82e-05 | 1385.4ms/step | 14.4 samples/s +Step 31370/44000 | Loss: 2.8706 | LR: 9.80e-05 | 1384.9ms/step | 14.4 samples/s +Step 31380/44000 | Loss: 2.8726 | LR: 9.79e-05 | 1385.6ms/step | 14.4 samples/s +Step 31390/44000 | Loss: 2.8949 | LR: 9.77e-05 | 1384.1ms/step | 14.4 samples/s +Step 31400/44000 | Loss: 2.8757 | LR: 9.76e-05 | 1384.2ms/step | 14.4 samples/s +Step 31410/44000 | Loss: 2.9077 | LR: 9.75e-05 | 1384.3ms/step | 14.4 samples/s +Step 31420/44000 | Loss: 2.9091 | LR: 9.73e-05 | 1381.9ms/step | 14.5 samples/s +Step 31430/44000 | Loss: 2.8888 | LR: 9.72e-05 | 1381.1ms/step | 14.5 samples/s +Step 31440/44000 | Loss: 2.9019 | LR: 9.70e-05 | 1381.0ms/step | 14.5 samples/s +Step 31450/44000 | Loss: 2.9126 | LR: 9.69e-05 | 1380.7ms/step | 14.5 samples/s +Step 31460/44000 | Loss: 2.8777 | LR: 9.67e-05 | 1380.5ms/step | 14.5 samples/s +Step 31470/44000 | Loss: 2.9418 | LR: 9.66e-05 | 1382.9ms/step | 14.5 samples/s +Step 31480/44000 | Loss: 2.9029 | LR: 9.65e-05 | 1381.8ms/step | 14.5 samples/s +Step 31490/44000 | Loss: 2.9130 | LR: 9.63e-05 | 1382.7ms/step | 14.5 samples/s +Step 31500/44000 | Loss: 2.8896 | LR: 9.62e-05 | 1382.5ms/step | 14.5 samples/s +Step 31510/44000 | Loss: 2.8852 | LR: 9.60e-05 | 1383.7ms/step | 14.5 samples/s +Step 31520/44000 | Loss: 2.8794 | LR: 9.59e-05 | 1380.9ms/step | 14.5 samples/s +Step 31530/44000 | Loss: 2.9062 | LR: 9.57e-05 | 1382.4ms/step | 14.5 samples/s +Step 31540/44000 | Loss: 2.8709 | LR: 9.56e-05 | 1382.1ms/step | 14.5 samples/s +Step 31550/44000 | Loss: 2.8841 | LR: 9.54e-05 | 1383.6ms/step | 14.5 samples/s +Step 31560/44000 | Loss: 2.8707 | LR: 9.53e-05 | 1383.4ms/step | 14.5 samples/s +Step 31570/44000 | Loss: 2.8973 | LR: 9.52e-05 | 1382.3ms/step | 14.5 samples/s +Step 31580/44000 | Loss: 2.8976 | LR: 9.50e-05 | 1381.4ms/step | 14.5 samples/s +Step 31590/44000 | Loss: 2.8742 | LR: 9.49e-05 | 1383.1ms/step | 14.5 samples/s +Step 31600/44000 | Loss: 2.8711 | LR: 9.47e-05 | 1382.4ms/step | 14.5 samples/s +Step 31610/44000 | Loss: 2.8451 | LR: 9.46e-05 | 1383.9ms/step | 14.5 samples/s +Step 31620/44000 | Loss: 2.8366 | LR: 9.45e-05 | 1382.7ms/step | 14.5 samples/s +Step 31630/44000 | Loss: 2.8928 | LR: 9.43e-05 | 1383.4ms/step | 14.5 samples/s +Step 31640/44000 | Loss: 2.9058 | LR: 9.42e-05 | 1385.2ms/step | 14.4 samples/s +Step 31650/44000 | Loss: 2.8713 | LR: 9.40e-05 | 1384.2ms/step | 14.4 samples/s +Step 31660/44000 | Loss: 2.8660 | LR: 9.39e-05 | 1382.1ms/step | 14.5 samples/s +Step 31670/44000 | Loss: 2.8681 | LR: 9.37e-05 | 1382.7ms/step | 14.5 samples/s +Step 31680/44000 | Loss: 2.8706 | LR: 9.36e-05 | 1381.9ms/step | 14.5 samples/s +Step 31690/44000 | Loss: 2.8579 | LR: 9.35e-05 | 1381.8ms/step | 14.5 samples/s +Step 31700/44000 | Loss: 2.9042 | LR: 9.33e-05 | 1381.7ms/step | 14.5 samples/s +Step 31710/44000 | Loss: 2.9058 | LR: 9.32e-05 | 1382.2ms/step | 14.5 samples/s +Step 31720/44000 | Loss: 2.8868 | LR: 9.30e-05 | 1382.8ms/step | 14.5 samples/s +Step 31730/44000 | Loss: 2.8930 | LR: 9.29e-05 | 1384.1ms/step | 14.4 samples/s +Step 31740/44000 | Loss: 2.8938 | LR: 9.28e-05 | 1382.4ms/step | 14.5 samples/s +Step 31750/44000 | Loss: 2.8960 | LR: 9.26e-05 | 1382.4ms/step | 14.5 samples/s +Step 31760/44000 | Loss: 2.8706 | LR: 9.25e-05 | 1382.1ms/step | 14.5 samples/s +Step 31770/44000 | Loss: 2.8951 | LR: 9.23e-05 | 1380.2ms/step | 14.5 samples/s +Step 31780/44000 | Loss: 2.8919 | LR: 9.22e-05 | 1382.6ms/step | 14.5 samples/s +Step 31790/44000 | Loss: 2.8946 | LR: 9.20e-05 | 1384.3ms/step | 14.4 samples/s +Step 31800/44000 | Loss: 2.9005 | LR: 9.19e-05 | 1383.1ms/step | 14.5 samples/s +Step 31810/44000 | Loss: 2.8869 | LR: 9.18e-05 | 1385.1ms/step | 14.4 samples/s +Step 31820/44000 | Loss: 2.8806 | LR: 9.16e-05 | 1386.7ms/step | 14.4 samples/s +Step 31830/44000 | Loss: 2.8881 | LR: 9.15e-05 | 1384.9ms/step | 14.4 samples/s +Step 31840/44000 | Loss: 2.8750 | LR: 9.13e-05 | 1383.5ms/step | 14.5 samples/s +Step 31850/44000 | Loss: 2.9207 | LR: 9.12e-05 | 1384.1ms/step | 14.5 samples/s +Step 31860/44000 | Loss: 2.9070 | LR: 9.11e-05 | 1384.1ms/step | 14.4 samples/s +Step 31870/44000 | Loss: 2.9062 | LR: 9.09e-05 | 1386.4ms/step | 14.4 samples/s +Step 31880/44000 | Loss: 2.9266 | LR: 9.08e-05 | 1383.0ms/step | 14.5 samples/s +Step 31890/44000 | Loss: 2.8623 | LR: 9.06e-05 | 1380.4ms/step | 14.5 samples/s +Step 31900/44000 | Loss: 2.8828 | LR: 9.05e-05 | 1381.8ms/step | 14.5 samples/s +Step 31910/44000 | Loss: 2.9150 | LR: 9.04e-05 | 1380.0ms/step | 14.5 samples/s +Step 31920/44000 | Loss: 2.8659 | LR: 9.02e-05 | 1381.5ms/step | 14.5 samples/s +Step 31930/44000 | Loss: 2.8968 | LR: 9.01e-05 | 1382.4ms/step | 14.5 samples/s +Step 31940/44000 | Loss: 2.9226 | LR: 8.99e-05 | 1381.0ms/step | 14.5 samples/s +Step 31950/44000 | Loss: 2.8571 | LR: 8.98e-05 | 1381.7ms/step | 14.5 samples/s +Step 31960/44000 | Loss: 2.8824 | LR: 8.97e-05 | 1382.6ms/step | 14.5 samples/s +Step 31970/44000 | Loss: 2.9186 | LR: 8.95e-05 | 1381.7ms/step | 14.5 samples/s +Step 31980/44000 | Loss: 2.8874 | LR: 8.94e-05 | 1382.5ms/step | 14.5 samples/s +Step 31990/44000 | Loss: 2.9139 | LR: 8.92e-05 | 1382.5ms/step | 14.5 samples/s +Step 32000/44000 | Loss: 2.8991 | LR: 8.91e-05 | 1380.2ms/step | 14.5 samples/s +Validation at step 32000: Loss = 2.8907, Perplexity = 18.01 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-32000 +Step 32010/44000 | Loss: 2.8546 | LR: 8.90e-05 | 3871.8ms/step | 5.2 samples/s +Step 32020/44000 | Loss: 2.9128 | LR: 8.88e-05 | 1380.7ms/step | 14.5 samples/s +Step 32030/44000 | Loss: 2.8933 | LR: 8.87e-05 | 1383.0ms/step | 14.5 samples/s +Step 32040/44000 | Loss: 2.8913 | LR: 8.86e-05 | 1381.0ms/step | 14.5 samples/s +Step 32050/44000 | Loss: 2.9110 | LR: 8.84e-05 | 1383.4ms/step | 14.5 samples/s +Step 32060/44000 | Loss: 2.9174 | LR: 8.83e-05 | 1383.8ms/step | 14.5 samples/s +Step 32070/44000 | Loss: 2.9186 | LR: 8.81e-05 | 1382.3ms/step | 14.5 samples/s +Step 32080/44000 | Loss: 2.8801 | LR: 8.80e-05 | 1382.8ms/step | 14.5 samples/s +Step 32090/44000 | Loss: 2.9072 | LR: 8.79e-05 | 1382.7ms/step | 14.5 samples/s +Step 32100/44000 | Loss: 2.8665 | LR: 8.77e-05 | 1380.9ms/step | 14.5 samples/s +Step 32110/44000 | Loss: 2.8715 | LR: 8.76e-05 | 1380.2ms/step | 14.5 samples/s +Step 32120/44000 | Loss: 2.8822 | LR: 8.74e-05 | 1382.2ms/step | 14.5 samples/s +Step 32130/44000 | Loss: 2.8932 | LR: 8.73e-05 | 1381.0ms/step | 14.5 samples/s +Step 32140/44000 | Loss: 2.8962 | LR: 8.72e-05 | 1381.1ms/step | 14.5 samples/s +Step 32150/44000 | Loss: 2.8949 | LR: 8.70e-05 | 1380.4ms/step | 14.5 samples/s +Step 32160/44000 | Loss: 2.9008 | LR: 8.69e-05 | 1380.7ms/step | 14.5 samples/s +Step 32170/44000 | Loss: 2.9058 | LR: 8.68e-05 | 1385.2ms/step | 14.4 samples/s +Step 32180/44000 | Loss: 2.9036 | LR: 8.66e-05 | 1383.5ms/step | 14.5 samples/s +Step 32190/44000 | Loss: 2.8870 | LR: 8.65e-05 | 1383.7ms/step | 14.5 samples/s +Step 32200/44000 | Loss: 2.8976 | LR: 8.63e-05 | 1383.6ms/step | 14.5 samples/s +Step 32210/44000 | Loss: 2.8957 | LR: 8.62e-05 | 1382.3ms/step | 14.5 samples/s +Step 32220/44000 | Loss: 2.9015 | LR: 8.61e-05 | 1384.0ms/step | 14.5 samples/s +Step 32230/44000 | Loss: 2.8813 | LR: 8.59e-05 | 1382.9ms/step | 14.5 samples/s +Step 32240/44000 | Loss: 2.9147 | LR: 8.58e-05 | 1383.9ms/step | 14.5 samples/s +Step 32250/44000 | Loss: 2.8726 | LR: 8.57e-05 | 1384.5ms/step | 14.4 samples/s +Step 32260/44000 | Loss: 2.8865 | LR: 8.55e-05 | 1382.5ms/step | 14.5 samples/s +Step 32270/44000 | Loss: 2.9098 | LR: 8.54e-05 | 1385.4ms/step | 14.4 samples/s +Step 32280/44000 | Loss: 2.9241 | LR: 8.53e-05 | 1383.8ms/step | 14.5 samples/s +Step 32290/44000 | Loss: 2.9046 | LR: 8.51e-05 | 1383.2ms/step | 14.5 samples/s +Step 32300/44000 | Loss: 2.9062 | LR: 8.50e-05 | 1380.9ms/step | 14.5 samples/s +Step 32310/44000 | Loss: 2.9052 | LR: 8.48e-05 | 1381.5ms/step | 14.5 samples/s +Step 32320/44000 | Loss: 2.8649 | LR: 8.47e-05 | 1380.8ms/step | 14.5 samples/s +Step 32330/44000 | Loss: 2.9047 | LR: 8.46e-05 | 1380.0ms/step | 14.5 samples/s +Step 32340/44000 | Loss: 2.8925 | LR: 8.44e-05 | 1380.5ms/step | 14.5 samples/s +Step 32350/44000 | Loss: 2.8922 | LR: 8.43e-05 | 1380.6ms/step | 14.5 samples/s +Step 32360/44000 | Loss: 2.8860 | LR: 8.42e-05 | 1378.3ms/step | 14.5 samples/s +Step 32370/44000 | Loss: 2.8759 | LR: 8.40e-05 | 1380.9ms/step | 14.5 samples/s +Step 32380/44000 | Loss: 2.8496 | LR: 8.39e-05 | 1383.9ms/step | 14.5 samples/s +Step 32390/44000 | Loss: 2.8802 | LR: 8.38e-05 | 1383.8ms/step | 14.5 samples/s +Step 32400/44000 | Loss: 2.8968 | LR: 8.36e-05 | 1382.0ms/step | 14.5 samples/s +Step 32410/44000 | Loss: 2.8800 | LR: 8.35e-05 | 1381.4ms/step | 14.5 samples/s +Step 32420/44000 | Loss: 2.8888 | LR: 8.33e-05 | 1382.5ms/step | 14.5 samples/s +Step 32430/44000 | Loss: 2.8681 | LR: 8.32e-05 | 1380.2ms/step | 14.5 samples/s +Step 32440/44000 | Loss: 2.9006 | LR: 8.31e-05 | 1380.6ms/step | 14.5 samples/s +Step 32450/44000 | Loss: 2.8837 | LR: 8.29e-05 | 1380.5ms/step | 14.5 samples/s +Step 32460/44000 | Loss: 2.9077 | LR: 8.28e-05 | 1380.9ms/step | 14.5 samples/s +Step 32470/44000 | Loss: 2.8805 | LR: 8.27e-05 | 1382.6ms/step | 14.5 samples/s +Step 32480/44000 | Loss: 2.8907 | LR: 8.25e-05 | 1383.3ms/step | 14.5 samples/s +Step 32490/44000 | Loss: 2.9023 | LR: 8.24e-05 | 1383.1ms/step | 14.5 samples/s +Step 32500/44000 | Loss: 2.9263 | LR: 8.23e-05 | 1382.7ms/step | 14.5 samples/s +Step 32510/44000 | Loss: 2.8852 | LR: 8.21e-05 | 1381.3ms/step | 14.5 samples/s +Step 32520/44000 | Loss: 2.8719 | LR: 8.20e-05 | 1379.3ms/step | 14.5 samples/s +Step 32530/44000 | Loss: 2.9019 | LR: 8.19e-05 | 1382.1ms/step | 14.5 samples/s +Step 32540/44000 | Loss: 2.8976 | LR: 8.17e-05 | 1381.1ms/step | 14.5 samples/s +Step 32550/44000 | Loss: 2.8859 | LR: 8.16e-05 | 1382.1ms/step | 14.5 samples/s +Step 32560/44000 | Loss: 2.8758 | LR: 8.15e-05 | 1380.9ms/step | 14.5 samples/s +Step 32570/44000 | Loss: 2.8932 | LR: 8.13e-05 | 1380.7ms/step | 14.5 samples/s +Step 32580/44000 | Loss: 2.8864 | LR: 8.12e-05 | 1384.0ms/step | 14.5 samples/s +Step 32590/44000 | Loss: 2.8846 | LR: 8.11e-05 | 1383.2ms/step | 14.5 samples/s +Step 32600/44000 | Loss: 2.8846 | LR: 8.09e-05 | 1383.3ms/step | 14.5 samples/s +Step 32610/44000 | Loss: 2.8662 | LR: 8.08e-05 | 1388.8ms/step | 14.4 samples/s +Step 32620/44000 | Loss: 2.9138 | LR: 8.07e-05 | 1383.6ms/step | 14.5 samples/s +Step 32630/44000 | Loss: 2.8719 | LR: 8.05e-05 | 1384.0ms/step | 14.5 samples/s +Step 32640/44000 | Loss: 2.8876 | LR: 8.04e-05 | 1381.9ms/step | 14.5 samples/s +Step 32650/44000 | Loss: 2.8989 | LR: 8.03e-05 | 1383.8ms/step | 14.5 samples/s +Step 32660/44000 | Loss: 2.9028 | LR: 8.01e-05 | 1382.7ms/step | 14.5 samples/s +Step 32670/44000 | Loss: 2.8781 | LR: 8.00e-05 | 1385.3ms/step | 14.4 samples/s +Step 32680/44000 | Loss: 2.8481 | LR: 7.99e-05 | 1383.2ms/step | 14.5 samples/s +Step 32690/44000 | Loss: 2.8775 | LR: 7.97e-05 | 1380.7ms/step | 14.5 samples/s +Step 32700/44000 | Loss: 2.8928 | LR: 7.96e-05 | 1378.3ms/step | 14.5 samples/s +Step 32710/44000 | Loss: 2.8712 | LR: 7.95e-05 | 1382.4ms/step | 14.5 samples/s +Step 32720/44000 | Loss: 2.8831 | LR: 7.93e-05 | 1388.3ms/step | 14.4 samples/s +Step 32730/44000 | Loss: 2.8799 | LR: 7.92e-05 | 1382.7ms/step | 14.5 samples/s +Step 32740/44000 | Loss: 2.8566 | LR: 7.91e-05 | 1382.3ms/step | 14.5 samples/s +Step 32750/44000 | Loss: 2.8600 | LR: 7.89e-05 | 1381.3ms/step | 14.5 samples/s +Step 32760/44000 | Loss: 2.8852 | LR: 7.88e-05 | 1383.6ms/step | 14.5 samples/s +Step 32770/44000 | Loss: 2.8645 | LR: 7.87e-05 | 1384.3ms/step | 14.4 samples/s +Step 32780/44000 | Loss: 2.9101 | LR: 7.85e-05 | 1382.9ms/step | 14.5 samples/s +Step 32790/44000 | Loss: 2.9057 | LR: 7.84e-05 | 1382.3ms/step | 14.5 samples/s +Step 32800/44000 | Loss: 2.8980 | LR: 7.83e-05 | 1383.4ms/step | 14.5 samples/s +Step 32810/44000 | Loss: 2.8568 | LR: 7.81e-05 | 1383.7ms/step | 14.5 samples/s +Step 32820/44000 | Loss: 2.8698 | LR: 7.80e-05 | 1383.4ms/step | 14.5 samples/s +Step 32830/44000 | Loss: 2.9015 | LR: 7.79e-05 | 1382.6ms/step | 14.5 samples/s +Step 32840/44000 | Loss: 2.8811 | LR: 7.77e-05 | 1381.0ms/step | 14.5 samples/s +Step 32850/44000 | Loss: 2.8931 | LR: 7.76e-05 | 1383.6ms/step | 14.5 samples/s +Step 32860/44000 | Loss: 2.8885 | LR: 7.75e-05 | 1382.2ms/step | 14.5 samples/s +Step 32870/44000 | Loss: 2.8629 | LR: 7.73e-05 | 1384.1ms/step | 14.5 samples/s +Step 32880/44000 | Loss: 2.8712 | LR: 7.72e-05 | 1383.1ms/step | 14.5 samples/s +Step 32890/44000 | Loss: 2.9028 | LR: 7.71e-05 | 1384.9ms/step | 14.4 samples/s +Step 32900/44000 | Loss: 2.8830 | LR: 7.70e-05 | 1385.6ms/step | 14.4 samples/s +Step 32910/44000 | Loss: 2.8884 | LR: 7.68e-05 | 1383.3ms/step | 14.5 samples/s +Step 32920/44000 | Loss: 2.9211 | LR: 7.67e-05 | 1380.8ms/step | 14.5 samples/s +Step 32930/44000 | Loss: 2.8840 | LR: 7.66e-05 | 1376.4ms/step | 14.5 samples/s +Step 32940/44000 | Loss: 2.9038 | LR: 7.64e-05 | 1378.7ms/step | 14.5 samples/s +Step 32950/44000 | Loss: 2.8868 | LR: 7.63e-05 | 1378.9ms/step | 14.5 samples/s +Step 32960/44000 | Loss: 2.8988 | LR: 7.62e-05 | 1379.5ms/step | 14.5 samples/s +Step 32970/44000 | Loss: 2.9098 | LR: 7.60e-05 | 1380.8ms/step | 14.5 samples/s +Step 32980/44000 | Loss: 2.8794 | LR: 7.59e-05 | 1381.8ms/step | 14.5 samples/s +Step 32990/44000 | Loss: 2.8877 | LR: 7.58e-05 | 1380.5ms/step | 14.5 samples/s +Step 33000/44000 | Loss: 2.8747 | LR: 7.56e-05 | 1381.4ms/step | 14.5 samples/s +Validation at step 33000: Loss = 2.8889, Perplexity = 17.97 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-33000 +Step 33010/44000 | Loss: 2.8840 | LR: 7.55e-05 | 3868.2ms/step | 5.2 samples/s +Step 33020/44000 | Loss: 2.9040 | LR: 7.54e-05 | 1380.4ms/step | 14.5 samples/s +Step 33030/44000 | Loss: 2.8636 | LR: 7.53e-05 | 1381.1ms/step | 14.5 samples/s +Step 33040/44000 | Loss: 2.8967 | LR: 7.51e-05 | 1380.9ms/step | 14.5 samples/s +Step 33050/44000 | Loss: 2.8579 | LR: 7.50e-05 | 1381.5ms/step | 14.5 samples/s +Step 33060/44000 | Loss: 2.8703 | LR: 7.49e-05 | 1381.9ms/step | 14.5 samples/s +Step 33070/44000 | Loss: 2.8837 | LR: 7.47e-05 | 1383.4ms/step | 14.5 samples/s +Step 33080/44000 | Loss: 2.9073 | LR: 7.46e-05 | 1383.6ms/step | 14.5 samples/s +Step 33090/44000 | Loss: 2.9070 | LR: 7.45e-05 | 1383.0ms/step | 14.5 samples/s +Step 33100/44000 | Loss: 2.8797 | LR: 7.44e-05 | 1382.7ms/step | 14.5 samples/s +Step 33110/44000 | Loss: 2.9099 | LR: 7.42e-05 | 1383.4ms/step | 14.5 samples/s +Step 33120/44000 | Loss: 2.9158 | LR: 7.41e-05 | 1384.0ms/step | 14.5 samples/s +Step 33130/44000 | Loss: 2.8964 | LR: 7.40e-05 | 1382.0ms/step | 14.5 samples/s +Step 33140/44000 | Loss: 2.8948 | LR: 7.38e-05 | 1381.2ms/step | 14.5 samples/s +Step 33150/44000 | Loss: 2.8687 | LR: 7.37e-05 | 1381.1ms/step | 14.5 samples/s +Step 33160/44000 | Loss: 2.8901 | LR: 7.36e-05 | 1381.0ms/step | 14.5 samples/s +Step 33170/44000 | Loss: 2.8790 | LR: 7.34e-05 | 1383.0ms/step | 14.5 samples/s +Step 33180/44000 | Loss: 2.9096 | LR: 7.33e-05 | 1382.3ms/step | 14.5 samples/s +Step 33190/44000 | Loss: 2.8998 | LR: 7.32e-05 | 1383.3ms/step | 14.5 samples/s +Step 33200/44000 | Loss: 2.8816 | LR: 7.31e-05 | 1381.6ms/step | 14.5 samples/s +Step 33210/44000 | Loss: 2.8890 | LR: 7.29e-05 | 1382.7ms/step | 14.5 samples/s +Step 33220/44000 | Loss: 2.8747 | LR: 7.28e-05 | 1384.0ms/step | 14.5 samples/s +Step 33230/44000 | Loss: 2.8947 | LR: 7.27e-05 | 1383.9ms/step | 14.5 samples/s +Step 33240/44000 | Loss: 2.8997 | LR: 7.26e-05 | 1382.5ms/step | 14.5 samples/s +Step 33250/44000 | Loss: 2.8680 | LR: 7.24e-05 | 1381.7ms/step | 14.5 samples/s +Step 33260/44000 | Loss: 2.8966 | LR: 7.23e-05 | 1383.7ms/step | 14.5 samples/s +Step 33270/44000 | Loss: 2.8925 | LR: 7.22e-05 | 1384.3ms/step | 14.4 samples/s +Step 33280/44000 | Loss: 2.8641 | LR: 7.20e-05 | 1382.9ms/step | 14.5 samples/s +Step 33290/44000 | Loss: 2.9137 | LR: 7.19e-05 | 1382.9ms/step | 14.5 samples/s +Step 33300/44000 | Loss: 2.8765 | LR: 7.18e-05 | 1385.0ms/step | 14.4 samples/s +Step 33310/44000 | Loss: 2.8862 | LR: 7.17e-05 | 1385.4ms/step | 14.4 samples/s +Step 33320/44000 | Loss: 2.8509 | LR: 7.15e-05 | 1385.4ms/step | 14.4 samples/s +Step 33330/44000 | Loss: 2.8805 | LR: 7.14e-05 | 1391.1ms/step | 14.4 samples/s +Step 33340/44000 | Loss: 2.8848 | LR: 7.13e-05 | 1387.4ms/step | 14.4 samples/s +Step 33350/44000 | Loss: 2.8923 | LR: 7.11e-05 | 1387.4ms/step | 14.4 samples/s +Step 33360/44000 | Loss: 2.8811 | LR: 7.10e-05 | 1386.4ms/step | 14.4 samples/s +Step 33370/44000 | Loss: 2.8948 | LR: 7.09e-05 | 1387.2ms/step | 14.4 samples/s +Step 33380/44000 | Loss: 2.8806 | LR: 7.08e-05 | 1386.9ms/step | 14.4 samples/s +Step 33390/44000 | Loss: 2.8752 | LR: 7.06e-05 | 1388.2ms/step | 14.4 samples/s +Step 33400/44000 | Loss: 2.8589 | LR: 7.05e-05 | 1387.2ms/step | 14.4 samples/s +Step 33410/44000 | Loss: 2.9110 | LR: 7.04e-05 | 1389.5ms/step | 14.4 samples/s +Step 33420/44000 | Loss: 2.8885 | LR: 7.03e-05 | 1387.8ms/step | 14.4 samples/s +Step 33430/44000 | Loss: 2.8841 | LR: 7.01e-05 | 1387.8ms/step | 14.4 samples/s +Step 33440/44000 | Loss: 2.8802 | LR: 7.00e-05 | 1387.3ms/step | 14.4 samples/s +Step 33450/44000 | Loss: 2.8982 | LR: 6.99e-05 | 1387.0ms/step | 14.4 samples/s +Step 33460/44000 | Loss: 2.8981 | LR: 6.98e-05 | 1386.1ms/step | 14.4 samples/s +Step 33470/44000 | Loss: 2.8932 | LR: 6.96e-05 | 1385.7ms/step | 14.4 samples/s +Step 33480/44000 | Loss: 2.8770 | LR: 6.95e-05 | 1385.8ms/step | 14.4 samples/s +Step 33490/44000 | Loss: 2.8635 | LR: 6.94e-05 | 1385.4ms/step | 14.4 samples/s +Step 33500/44000 | Loss: 2.8941 | LR: 6.93e-05 | 1386.3ms/step | 14.4 samples/s +Step 33510/44000 | Loss: 2.8818 | LR: 6.91e-05 | 1387.9ms/step | 14.4 samples/s +Step 33520/44000 | Loss: 2.8756 | LR: 6.90e-05 | 1388.5ms/step | 14.4 samples/s +Step 33530/44000 | Loss: 2.8961 | LR: 6.89e-05 | 1386.7ms/step | 14.4 samples/s +Step 33540/44000 | Loss: 2.8768 | LR: 6.88e-05 | 1387.0ms/step | 14.4 samples/s +Step 33550/44000 | Loss: 2.8856 | LR: 6.86e-05 | 1385.3ms/step | 14.4 samples/s +Step 33560/44000 | Loss: 2.9081 | LR: 6.85e-05 | 1386.9ms/step | 14.4 samples/s +Step 33570/44000 | Loss: 2.8824 | LR: 6.84e-05 | 1387.7ms/step | 14.4 samples/s +Step 33580/44000 | Loss: 2.8653 | LR: 6.83e-05 | 1385.3ms/step | 14.4 samples/s +Step 33590/44000 | Loss: 2.8902 | LR: 6.81e-05 | 1387.0ms/step | 14.4 samples/s +Step 33600/44000 | Loss: 2.8989 | LR: 6.80e-05 | 1389.8ms/step | 14.4 samples/s +Step 33610/44000 | Loss: 2.8831 | LR: 6.79e-05 | 1387.6ms/step | 14.4 samples/s +Step 33620/44000 | Loss: 2.8812 | LR: 6.78e-05 | 1387.2ms/step | 14.4 samples/s +Step 33630/44000 | Loss: 2.8997 | LR: 6.76e-05 | 1388.0ms/step | 14.4 samples/s +Step 33640/44000 | Loss: 2.8926 | LR: 6.75e-05 | 1387.0ms/step | 14.4 samples/s +Step 33650/44000 | Loss: 2.8824 | LR: 6.74e-05 | 1386.4ms/step | 14.4 samples/s +Step 33660/44000 | Loss: 2.9100 | LR: 6.73e-05 | 1386.0ms/step | 14.4 samples/s +Step 33670/44000 | Loss: 2.8702 | LR: 6.71e-05 | 1390.0ms/step | 14.4 samples/s +Step 33680/44000 | Loss: 2.8873 | LR: 6.70e-05 | 1390.0ms/step | 14.4 samples/s +Step 33690/44000 | Loss: 2.8762 | LR: 6.69e-05 | 1387.8ms/step | 14.4 samples/s +Step 33700/44000 | Loss: 2.8597 | LR: 6.68e-05 | 1388.0ms/step | 14.4 samples/s +Step 33710/44000 | Loss: 2.8878 | LR: 6.66e-05 | 1387.5ms/step | 14.4 samples/s +Step 33720/44000 | Loss: 2.8827 | LR: 6.65e-05 | 1388.8ms/step | 14.4 samples/s +Step 33730/44000 | Loss: 2.8678 | LR: 6.64e-05 | 1387.9ms/step | 14.4 samples/s +Step 33740/44000 | Loss: 2.9005 | LR: 6.63e-05 | 1384.9ms/step | 14.4 samples/s +Step 33750/44000 | Loss: 2.8926 | LR: 6.61e-05 | 1388.8ms/step | 14.4 samples/s +Step 33760/44000 | Loss: 2.8474 | LR: 6.60e-05 | 1386.6ms/step | 14.4 samples/s +Step 33770/44000 | Loss: 2.8865 | LR: 6.59e-05 | 1387.6ms/step | 14.4 samples/s +Step 33780/44000 | Loss: 2.8848 | LR: 6.58e-05 | 1388.3ms/step | 14.4 samples/s +Step 33790/44000 | Loss: 2.8834 | LR: 6.57e-05 | 1388.3ms/step | 14.4 samples/s +Step 33800/44000 | Loss: 2.8448 | LR: 6.55e-05 | 1389.7ms/step | 14.4 samples/s +Step 33810/44000 | Loss: 2.9200 | LR: 6.54e-05 | 1387.9ms/step | 14.4 samples/s +Step 33820/44000 | Loss: 2.8744 | LR: 6.53e-05 | 1385.9ms/step | 14.4 samples/s +Step 33830/44000 | Loss: 2.9004 | LR: 6.52e-05 | 1388.0ms/step | 14.4 samples/s +Step 33840/44000 | Loss: 2.8866 | LR: 6.50e-05 | 1388.4ms/step | 14.4 samples/s +Step 33850/44000 | Loss: 2.9020 | LR: 6.49e-05 | 1387.9ms/step | 14.4 samples/s +Step 33860/44000 | Loss: 2.8908 | LR: 6.48e-05 | 1388.5ms/step | 14.4 samples/s +Step 33870/44000 | Loss: 2.8790 | LR: 6.47e-05 | 1389.5ms/step | 14.4 samples/s +Step 33880/44000 | Loss: 2.8816 | LR: 6.46e-05 | 1386.6ms/step | 14.4 samples/s +Step 33890/44000 | Loss: 2.8937 | LR: 6.44e-05 | 1387.2ms/step | 14.4 samples/s +Step 33900/44000 | Loss: 2.8757 | LR: 6.43e-05 | 1388.1ms/step | 14.4 samples/s +Step 33910/44000 | Loss: 2.8978 | LR: 6.42e-05 | 1387.7ms/step | 14.4 samples/s +Step 33920/44000 | Loss: 2.8969 | LR: 6.41e-05 | 1387.6ms/step | 14.4 samples/s +Step 33930/44000 | Loss: 2.8845 | LR: 6.40e-05 | 1390.3ms/step | 14.4 samples/s +Step 33940/44000 | Loss: 2.8810 | LR: 6.38e-05 | 1384.9ms/step | 14.4 samples/s +Step 33950/44000 | Loss: 2.8875 | LR: 6.37e-05 | 1384.1ms/step | 14.4 samples/s +Step 33960/44000 | Loss: 2.8959 | LR: 6.36e-05 | 1383.5ms/step | 14.5 samples/s +Step 33970/44000 | Loss: 2.8790 | LR: 6.35e-05 | 1382.5ms/step | 14.5 samples/s +Step 33980/44000 | Loss: 2.9147 | LR: 6.33e-05 | 1382.0ms/step | 14.5 samples/s +Step 33990/44000 | Loss: 2.8941 | LR: 6.32e-05 | 1382.5ms/step | 14.5 samples/s +Step 34000/44000 | Loss: 2.8838 | LR: 6.31e-05 | 1383.9ms/step | 14.5 samples/s +Validation at step 34000: Loss = 2.8874, Perplexity = 17.95 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-34000 +Step 34010/44000 | Loss: 2.8789 | LR: 6.30e-05 | 3868.0ms/step | 5.2 samples/s +Step 34020/44000 | Loss: 2.9063 | LR: 6.29e-05 | 1386.8ms/step | 14.4 samples/s +Step 34030/44000 | Loss: 2.8989 | LR: 6.27e-05 | 1386.1ms/step | 14.4 samples/s +Step 34040/44000 | Loss: 2.9060 | LR: 6.26e-05 | 1388.3ms/step | 14.4 samples/s +Step 34050/44000 | Loss: 2.8538 | LR: 6.25e-05 | 1387.4ms/step | 14.4 samples/s +Step 34060/44000 | Loss: 2.8793 | LR: 6.24e-05 | 1388.6ms/step | 14.4 samples/s +Step 34070/44000 | Loss: 2.8705 | LR: 6.23e-05 | 1386.7ms/step | 14.4 samples/s +Step 34080/44000 | Loss: 2.8882 | LR: 6.21e-05 | 1386.7ms/step | 14.4 samples/s +Step 34090/44000 | Loss: 2.8520 | LR: 6.20e-05 | 1387.4ms/step | 14.4 samples/s +Step 34100/44000 | Loss: 2.9032 | LR: 6.19e-05 | 1387.3ms/step | 14.4 samples/s +Step 34110/44000 | Loss: 2.9145 | LR: 6.18e-05 | 1389.2ms/step | 14.4 samples/s +Step 34120/44000 | Loss: 2.9073 | LR: 6.17e-05 | 1387.8ms/step | 14.4 samples/s +Step 34130/44000 | Loss: 2.9011 | LR: 6.15e-05 | 1386.9ms/step | 14.4 samples/s +Step 34140/44000 | Loss: 2.8990 | LR: 6.14e-05 | 1387.7ms/step | 14.4 samples/s +Step 34150/44000 | Loss: 2.8685 | LR: 6.13e-05 | 1389.0ms/step | 14.4 samples/s +Step 34160/44000 | Loss: 2.8645 | LR: 6.12e-05 | 1389.4ms/step | 14.4 samples/s +Step 34170/44000 | Loss: 2.8778 | LR: 6.11e-05 | 1386.7ms/step | 14.4 samples/s +Step 34180/44000 | Loss: 2.8858 | LR: 6.09e-05 | 1389.0ms/step | 14.4 samples/s +Step 34190/44000 | Loss: 2.8734 | LR: 6.08e-05 | 1384.6ms/step | 14.4 samples/s +Step 34200/44000 | Loss: 2.8776 | LR: 6.07e-05 | 1384.4ms/step | 14.4 samples/s +Step 34210/44000 | Loss: 2.8821 | LR: 6.06e-05 | 1385.0ms/step | 14.4 samples/s +Step 34220/44000 | Loss: 2.8781 | LR: 6.05e-05 | 1386.6ms/step | 14.4 samples/s +Step 34230/44000 | Loss: 2.8710 | LR: 6.04e-05 | 1387.4ms/step | 14.4 samples/s +Step 34240/44000 | Loss: 2.9052 | LR: 6.02e-05 | 1386.7ms/step | 14.4 samples/s +Step 34250/44000 | Loss: 2.9124 | LR: 6.01e-05 | 1386.6ms/step | 14.4 samples/s +Step 34260/44000 | Loss: 2.9143 | LR: 6.00e-05 | 1386.8ms/step | 14.4 samples/s +Step 34270/44000 | Loss: 2.8564 | LR: 5.99e-05 | 1386.0ms/step | 14.4 samples/s +Step 34280/44000 | Loss: 2.8962 | LR: 5.98e-05 | 1386.2ms/step | 14.4 samples/s +Step 34290/44000 | Loss: 2.8972 | LR: 5.96e-05 | 1384.8ms/step | 14.4 samples/s +Step 34300/44000 | Loss: 2.8924 | LR: 5.95e-05 | 1386.2ms/step | 14.4 samples/s +Step 34310/44000 | Loss: 2.8999 | LR: 5.94e-05 | 1385.6ms/step | 14.4 samples/s +Step 34320/44000 | Loss: 2.8887 | LR: 5.93e-05 | 1387.3ms/step | 14.4 samples/s +Step 34330/44000 | Loss: 2.8678 | LR: 5.92e-05 | 1388.0ms/step | 14.4 samples/s +Step 34340/44000 | Loss: 2.8582 | LR: 5.91e-05 | 1386.3ms/step | 14.4 samples/s +Step 34350/44000 | Loss: 2.8896 | LR: 5.89e-05 | 1385.3ms/step | 14.4 samples/s +Step 34360/44000 | Loss: 2.8981 | LR: 5.88e-05 | 1384.4ms/step | 14.4 samples/s +Step 34370/44000 | Loss: 2.8771 | LR: 5.87e-05 | 1384.2ms/step | 14.4 samples/s +Step 34380/44000 | Loss: 2.8908 | LR: 5.86e-05 | 1383.8ms/step | 14.5 samples/s +Step 34390/44000 | Loss: 2.8786 | LR: 5.85e-05 | 1386.4ms/step | 14.4 samples/s +Step 34400/44000 | Loss: 2.8844 | LR: 5.84e-05 | 1386.3ms/step | 14.4 samples/s +Step 34410/44000 | Loss: 2.8775 | LR: 5.82e-05 | 1386.9ms/step | 14.4 samples/s +Step 34420/44000 | Loss: 2.8761 | LR: 5.81e-05 | 1384.7ms/step | 14.4 samples/s +Step 34430/44000 | Loss: 2.8728 | LR: 5.80e-05 | 1385.4ms/step | 14.4 samples/s +Step 34440/44000 | Loss: 2.8582 | LR: 5.79e-05 | 1385.1ms/step | 14.4 samples/s +Step 34450/44000 | Loss: 2.8690 | LR: 5.78e-05 | 1386.0ms/step | 14.4 samples/s +Step 34460/44000 | Loss: 2.8881 | LR: 5.77e-05 | 1387.2ms/step | 14.4 samples/s +Step 34470/44000 | Loss: 2.8875 | LR: 5.75e-05 | 1386.7ms/step | 14.4 samples/s +Step 34480/44000 | Loss: 2.8707 | LR: 5.74e-05 | 1387.9ms/step | 14.4 samples/s +Step 34490/44000 | Loss: 2.9020 | LR: 5.73e-05 | 1386.2ms/step | 14.4 samples/s +Step 34500/44000 | Loss: 2.8798 | LR: 5.72e-05 | 1387.0ms/step | 14.4 samples/s +Step 34510/44000 | Loss: 2.8737 | LR: 5.71e-05 | 1386.0ms/step | 14.4 samples/s +Step 34520/44000 | Loss: 2.8964 | LR: 5.70e-05 | 1386.2ms/step | 14.4 samples/s +Step 34530/44000 | Loss: 2.8706 | LR: 5.69e-05 | 1388.0ms/step | 14.4 samples/s +Step 34540/44000 | Loss: 2.8805 | LR: 5.67e-05 | 1389.0ms/step | 14.4 samples/s +Step 34550/44000 | Loss: 2.8511 | LR: 5.66e-05 | 1385.8ms/step | 14.4 samples/s +Step 34560/44000 | Loss: 2.8623 | LR: 5.65e-05 | 1388.1ms/step | 14.4 samples/s +Step 34570/44000 | Loss: 2.8658 | LR: 5.64e-05 | 1386.2ms/step | 14.4 samples/s +Step 34580/44000 | Loss: 2.8469 | LR: 5.63e-05 | 1385.5ms/step | 14.4 samples/s +Step 34590/44000 | Loss: 2.9022 | LR: 5.62e-05 | 1389.4ms/step | 14.4 samples/s +Step 34600/44000 | Loss: 2.8847 | LR: 5.60e-05 | 1386.2ms/step | 14.4 samples/s +Step 34610/44000 | Loss: 2.8888 | LR: 5.59e-05 | 1387.2ms/step | 14.4 samples/s +Step 34620/44000 | Loss: 2.8991 | LR: 5.58e-05 | 1388.4ms/step | 14.4 samples/s +Step 34630/44000 | Loss: 2.8982 | LR: 5.57e-05 | 1386.5ms/step | 14.4 samples/s +Step 34640/44000 | Loss: 2.8819 | LR: 5.56e-05 | 1387.2ms/step | 14.4 samples/s +Step 34650/44000 | Loss: 2.8434 | LR: 5.55e-05 | 1388.2ms/step | 14.4 samples/s +Step 34660/44000 | Loss: 2.8864 | LR: 5.54e-05 | 1387.2ms/step | 14.4 samples/s +Step 34670/44000 | Loss: 2.8945 | LR: 5.52e-05 | 1386.4ms/step | 14.4 samples/s +Step 34680/44000 | Loss: 2.8970 | LR: 5.51e-05 | 1386.6ms/step | 14.4 samples/s +Step 34690/44000 | Loss: 2.9206 | LR: 5.50e-05 | 1387.1ms/step | 14.4 samples/s +Step 34700/44000 | Loss: 2.8870 | LR: 5.49e-05 | 1387.0ms/step | 14.4 samples/s +Step 34710/44000 | Loss: 2.8921 | LR: 5.48e-05 | 1384.8ms/step | 14.4 samples/s +Step 34720/44000 | Loss: 2.8578 | LR: 5.47e-05 | 1387.2ms/step | 14.4 samples/s +Step 34730/44000 | Loss: 2.8772 | LR: 5.46e-05 | 1388.3ms/step | 14.4 samples/s +Step 34740/44000 | Loss: 2.8828 | LR: 5.45e-05 | 1387.3ms/step | 14.4 samples/s +Step 34750/44000 | Loss: 2.8945 | LR: 5.43e-05 | 1387.2ms/step | 14.4 samples/s +Step 34760/44000 | Loss: 2.8771 | LR: 5.42e-05 | 1386.9ms/step | 14.4 samples/s +Step 34770/44000 | Loss: 2.8968 | LR: 5.41e-05 | 1386.9ms/step | 14.4 samples/s +Step 34780/44000 | Loss: 2.8801 | LR: 5.40e-05 | 1386.7ms/step | 14.4 samples/s +Step 34790/44000 | Loss: 2.8643 | LR: 5.39e-05 | 1387.6ms/step | 14.4 samples/s +Step 34800/44000 | Loss: 2.8782 | LR: 5.38e-05 | 1386.7ms/step | 14.4 samples/s +Step 34810/44000 | Loss: 2.8898 | LR: 5.37e-05 | 1386.2ms/step | 14.4 samples/s +Step 34820/44000 | Loss: 2.9012 | LR: 5.36e-05 | 1386.4ms/step | 14.4 samples/s +Step 34830/44000 | Loss: 2.8805 | LR: 5.34e-05 | 1388.1ms/step | 14.4 samples/s +Step 34840/44000 | Loss: 2.8535 | LR: 5.33e-05 | 1385.8ms/step | 14.4 samples/s +Step 34850/44000 | Loss: 2.8618 | LR: 5.32e-05 | 1387.8ms/step | 14.4 samples/s +Step 34860/44000 | Loss: 2.8723 | LR: 5.31e-05 | 1386.5ms/step | 14.4 samples/s +Step 34870/44000 | Loss: 2.8951 | LR: 5.30e-05 | 1386.3ms/step | 14.4 samples/s +Step 34880/44000 | Loss: 2.8903 | LR: 5.29e-05 | 1389.5ms/step | 14.4 samples/s +Step 34890/44000 | Loss: 2.8795 | LR: 5.28e-05 | 1386.2ms/step | 14.4 samples/s +Step 34900/44000 | Loss: 2.8710 | LR: 5.27e-05 | 1387.0ms/step | 14.4 samples/s +Step 34910/44000 | Loss: 2.9161 | LR: 5.25e-05 | 1387.6ms/step | 14.4 samples/s +Step 34920/44000 | Loss: 2.8952 | LR: 5.24e-05 | 1387.1ms/step | 14.4 samples/s +Step 34930/44000 | Loss: 2.8811 | LR: 5.23e-05 | 1385.3ms/step | 14.4 samples/s +Step 34940/44000 | Loss: 2.8901 | LR: 5.22e-05 | 1386.9ms/step | 14.4 samples/s +Step 34950/44000 | Loss: 2.9053 | LR: 5.21e-05 | 1388.2ms/step | 14.4 samples/s +Step 34960/44000 | Loss: 2.9000 | LR: 5.20e-05 | 1387.5ms/step | 14.4 samples/s +Step 34970/44000 | Loss: 2.8958 | LR: 5.19e-05 | 1386.1ms/step | 14.4 samples/s +Step 34980/44000 | Loss: 2.9109 | LR: 5.18e-05 | 1384.1ms/step | 14.5 samples/s +Step 34990/44000 | Loss: 2.8645 | LR: 5.17e-05 | 1384.1ms/step | 14.4 samples/s +Step 35000/44000 | Loss: 2.8765 | LR: 5.15e-05 | 1384.0ms/step | 14.5 samples/s +Validation at step 35000: Loss = 2.8862, Perplexity = 17.92 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-35000 +Step 35010/44000 | Loss: 2.8459 | LR: 5.14e-05 | 3888.3ms/step | 5.1 samples/s +Step 35020/44000 | Loss: 2.8697 | LR: 5.13e-05 | 1386.8ms/step | 14.4 samples/s +Step 35030/44000 | Loss: 2.8512 | LR: 5.12e-05 | 1389.1ms/step | 14.4 samples/s +Step 35040/44000 | Loss: 2.8869 | LR: 5.11e-05 | 1387.6ms/step | 14.4 samples/s +Step 35050/44000 | Loss: 2.8806 | LR: 5.10e-05 | 1388.1ms/step | 14.4 samples/s +Step 35060/44000 | Loss: 2.8616 | LR: 5.09e-05 | 1387.0ms/step | 14.4 samples/s +Step 35070/44000 | Loss: 2.8711 | LR: 5.08e-05 | 1388.6ms/step | 14.4 samples/s +Step 35080/44000 | Loss: 2.8474 | LR: 5.07e-05 | 1387.9ms/step | 14.4 samples/s +Step 35090/44000 | Loss: 2.8818 | LR: 5.06e-05 | 1386.9ms/step | 14.4 samples/s +Step 35100/44000 | Loss: 2.8760 | LR: 5.04e-05 | 1387.5ms/step | 14.4 samples/s +Step 35110/44000 | Loss: 2.8881 | LR: 5.03e-05 | 1387.4ms/step | 14.4 samples/s +Step 35120/44000 | Loss: 2.8892 | LR: 5.02e-05 | 1386.8ms/step | 14.4 samples/s +Step 35130/44000 | Loss: 2.9032 | LR: 5.01e-05 | 1387.2ms/step | 14.4 samples/s +Step 35140/44000 | Loss: 2.8805 | LR: 5.00e-05 | 1388.7ms/step | 14.4 samples/s +Step 35150/44000 | Loss: 2.8858 | LR: 4.99e-05 | 1387.4ms/step | 14.4 samples/s +Step 35160/44000 | Loss: 2.8789 | LR: 4.98e-05 | 1389.2ms/step | 14.4 samples/s +Step 35170/44000 | Loss: 2.8689 | LR: 4.97e-05 | 1387.8ms/step | 14.4 samples/s +Step 35180/44000 | Loss: 2.8977 | LR: 4.96e-05 | 1387.9ms/step | 14.4 samples/s +Step 35190/44000 | Loss: 2.8691 | LR: 4.95e-05 | 1389.1ms/step | 14.4 samples/s +Step 35200/44000 | Loss: 2.9019 | LR: 4.94e-05 | 1388.1ms/step | 14.4 samples/s +Step 35210/44000 | Loss: 2.8808 | LR: 4.93e-05 | 1388.7ms/step | 14.4 samples/s +Step 35220/44000 | Loss: 2.8746 | LR: 4.91e-05 | 1389.0ms/step | 14.4 samples/s +Step 35230/44000 | Loss: 2.8571 | LR: 4.90e-05 | 1386.9ms/step | 14.4 samples/s +Step 35240/44000 | Loss: 2.8888 | LR: 4.89e-05 | 1386.3ms/step | 14.4 samples/s +Step 35250/44000 | Loss: 2.8948 | LR: 4.88e-05 | 1381.0ms/step | 14.5 samples/s +Step 35260/44000 | Loss: 2.8537 | LR: 4.87e-05 | 1383.5ms/step | 14.5 samples/s +Step 35270/44000 | Loss: 2.8999 | LR: 4.86e-05 | 1384.7ms/step | 14.4 samples/s +Step 35280/44000 | Loss: 2.9047 | LR: 4.85e-05 | 1385.3ms/step | 14.4 samples/s +Step 35290/44000 | Loss: 2.8601 | LR: 4.84e-05 | 1385.6ms/step | 14.4 samples/s +Step 35300/44000 | Loss: 2.8943 | LR: 4.83e-05 | 1385.0ms/step | 14.4 samples/s +Step 35310/44000 | Loss: 2.8630 | LR: 4.82e-05 | 1385.6ms/step | 14.4 samples/s +Step 35320/44000 | Loss: 2.8793 | LR: 4.81e-05 | 1387.0ms/step | 14.4 samples/s +Step 35330/44000 | Loss: 2.8846 | LR: 4.80e-05 | 1386.1ms/step | 14.4 samples/s +Step 35340/44000 | Loss: 2.8722 | LR: 4.79e-05 | 1385.4ms/step | 14.4 samples/s +Step 35350/44000 | Loss: 2.8932 | LR: 4.77e-05 | 1385.9ms/step | 14.4 samples/s +Step 35360/44000 | Loss: 2.8828 | LR: 4.76e-05 | 1387.1ms/step | 14.4 samples/s +Step 35370/44000 | Loss: 2.8771 | LR: 4.75e-05 | 1386.6ms/step | 14.4 samples/s +Step 35380/44000 | Loss: 2.8757 | LR: 4.74e-05 | 1386.0ms/step | 14.4 samples/s +Step 35390/44000 | Loss: 2.8742 | LR: 4.73e-05 | 1386.2ms/step | 14.4 samples/s +Step 35400/44000 | Loss: 2.8630 | LR: 4.72e-05 | 1382.1ms/step | 14.5 samples/s +Step 35410/44000 | Loss: 2.8860 | LR: 4.71e-05 | 1383.5ms/step | 14.5 samples/s +Step 35420/44000 | Loss: 2.8704 | LR: 4.70e-05 | 1383.6ms/step | 14.5 samples/s +Step 35430/44000 | Loss: 2.8676 | LR: 4.69e-05 | 1385.8ms/step | 14.4 samples/s +Step 35440/44000 | Loss: 2.8568 | LR: 4.68e-05 | 1384.3ms/step | 14.4 samples/s +Step 35450/44000 | Loss: 2.8785 | LR: 4.67e-05 | 1384.2ms/step | 14.4 samples/s +Step 35460/44000 | Loss: 2.8985 | LR: 4.66e-05 | 1385.1ms/step | 14.4 samples/s +Step 35470/44000 | Loss: 2.9109 | LR: 4.65e-05 | 1386.3ms/step | 14.4 samples/s +Step 35480/44000 | Loss: 2.8845 | LR: 4.64e-05 | 1387.1ms/step | 14.4 samples/s +Step 35490/44000 | Loss: 2.8951 | LR: 4.63e-05 | 1386.6ms/step | 14.4 samples/s +Step 35500/44000 | Loss: 2.8865 | LR: 4.62e-05 | 1384.9ms/step | 14.4 samples/s +Step 35510/44000 | Loss: 2.8725 | LR: 4.61e-05 | 1383.6ms/step | 14.5 samples/s +Step 35520/44000 | Loss: 2.8937 | LR: 4.59e-05 | 1385.1ms/step | 14.4 samples/s +Step 35530/44000 | Loss: 2.8483 | LR: 4.58e-05 | 1384.9ms/step | 14.4 samples/s +Step 35540/44000 | Loss: 2.8878 | LR: 4.57e-05 | 1385.6ms/step | 14.4 samples/s +Step 35550/44000 | Loss: 2.9161 | LR: 4.56e-05 | 1385.6ms/step | 14.4 samples/s +Step 35560/44000 | Loss: 2.8827 | LR: 4.55e-05 | 1385.1ms/step | 14.4 samples/s +Step 35570/44000 | Loss: 2.8480 | LR: 4.54e-05 | 1385.7ms/step | 14.4 samples/s +Step 35580/44000 | Loss: 2.8718 | LR: 4.53e-05 | 1386.1ms/step | 14.4 samples/s +Step 35590/44000 | Loss: 2.9033 | LR: 4.52e-05 | 1385.6ms/step | 14.4 samples/s +Step 35600/44000 | Loss: 2.8805 | LR: 4.51e-05 | 1384.7ms/step | 14.4 samples/s +Step 35610/44000 | Loss: 2.8761 | LR: 4.50e-05 | 1387.4ms/step | 14.4 samples/s +Step 35620/44000 | Loss: 2.8681 | LR: 4.49e-05 | 1386.2ms/step | 14.4 samples/s +Step 35630/44000 | Loss: 2.9134 | LR: 4.48e-05 | 1385.4ms/step | 14.4 samples/s +Step 35640/44000 | Loss: 2.8775 | LR: 4.47e-05 | 1385.4ms/step | 14.4 samples/s +Step 35650/44000 | Loss: 2.9158 | LR: 4.46e-05 | 1388.2ms/step | 14.4 samples/s +Step 35660/44000 | Loss: 2.8426 | LR: 4.45e-05 | 1386.9ms/step | 14.4 samples/s +Step 35670/44000 | Loss: 2.9055 | LR: 4.44e-05 | 1386.3ms/step | 14.4 samples/s +Step 35680/44000 | Loss: 2.9095 | LR: 4.43e-05 | 1386.4ms/step | 14.4 samples/s +Step 35690/44000 | Loss: 2.8772 | LR: 4.42e-05 | 1386.7ms/step | 14.4 samples/s +Step 35700/44000 | Loss: 2.8886 | LR: 4.41e-05 | 1387.0ms/step | 14.4 samples/s +Step 35710/44000 | Loss: 2.8771 | LR: 4.40e-05 | 1385.7ms/step | 14.4 samples/s +Step 35720/44000 | Loss: 2.8905 | LR: 4.39e-05 | 1388.1ms/step | 14.4 samples/s +Step 35730/44000 | Loss: 2.8990 | LR: 4.38e-05 | 1388.2ms/step | 14.4 samples/s +Step 35740/44000 | Loss: 2.8861 | LR: 4.37e-05 | 1387.8ms/step | 14.4 samples/s +Step 35750/44000 | Loss: 2.8955 | LR: 4.36e-05 | 1388.0ms/step | 14.4 samples/s +Step 35760/44000 | Loss: 2.8620 | LR: 4.35e-05 | 1387.7ms/step | 14.4 samples/s +Step 35770/44000 | Loss: 2.8683 | LR: 4.34e-05 | 1388.4ms/step | 14.4 samples/s +Step 35780/44000 | Loss: 2.8918 | LR: 4.33e-05 | 1387.6ms/step | 14.4 samples/s +Step 35790/44000 | Loss: 2.9291 | LR: 4.32e-05 | 1387.9ms/step | 14.4 samples/s +Step 35800/44000 | Loss: 2.8525 | LR: 4.31e-05 | 1386.6ms/step | 14.4 samples/s +Step 35810/44000 | Loss: 2.9066 | LR: 4.29e-05 | 1387.3ms/step | 14.4 samples/s +Step 35820/44000 | Loss: 2.8554 | LR: 4.28e-05 | 1385.9ms/step | 14.4 samples/s +Step 35830/44000 | Loss: 2.8807 | LR: 4.27e-05 | 1386.8ms/step | 14.4 samples/s +Step 35840/44000 | Loss: 2.8644 | LR: 4.26e-05 | 1386.8ms/step | 14.4 samples/s +Step 35850/44000 | Loss: 2.8868 | LR: 4.25e-05 | 1385.1ms/step | 14.4 samples/s +Step 35860/44000 | Loss: 2.8793 | LR: 4.24e-05 | 1388.4ms/step | 14.4 samples/s +Step 35870/44000 | Loss: 2.8705 | LR: 4.23e-05 | 1387.3ms/step | 14.4 samples/s +Step 35880/44000 | Loss: 2.8798 | LR: 4.22e-05 | 1386.2ms/step | 14.4 samples/s +Step 35890/44000 | Loss: 2.8576 | LR: 4.21e-05 | 1387.8ms/step | 14.4 samples/s +Step 35900/44000 | Loss: 2.8800 | LR: 4.20e-05 | 1384.6ms/step | 14.4 samples/s +Step 35910/44000 | Loss: 2.8529 | LR: 4.19e-05 | 1386.1ms/step | 14.4 samples/s +Step 35920/44000 | Loss: 2.8672 | LR: 4.18e-05 | 1386.3ms/step | 14.4 samples/s +Step 35930/44000 | Loss: 2.8903 | LR: 4.17e-05 | 1385.3ms/step | 14.4 samples/s +Step 35940/44000 | Loss: 2.8897 | LR: 4.16e-05 | 1387.7ms/step | 14.4 samples/s +Step 35950/44000 | Loss: 2.8485 | LR: 4.15e-05 | 1386.9ms/step | 14.4 samples/s +Step 35960/44000 | Loss: 2.8857 | LR: 4.14e-05 | 1387.4ms/step | 14.4 samples/s +Step 35970/44000 | Loss: 2.9042 | LR: 4.13e-05 | 1386.9ms/step | 14.4 samples/s +Step 35980/44000 | Loss: 2.8584 | LR: 4.12e-05 | 1387.3ms/step | 14.4 samples/s +Step 35990/44000 | Loss: 2.8635 | LR: 4.11e-05 | 1386.0ms/step | 14.4 samples/s +Step 36000/44000 | Loss: 2.8718 | LR: 4.10e-05 | 1385.0ms/step | 14.4 samples/s +Validation at step 36000: Loss = 2.8853, Perplexity = 17.91 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-36000 +Step 36010/44000 | Loss: 2.8835 | LR: 4.09e-05 | 3861.0ms/step | 5.2 samples/s +Step 36020/44000 | Loss: 2.8928 | LR: 4.08e-05 | 1380.5ms/step | 14.5 samples/s +Step 36030/44000 | Loss: 2.8653 | LR: 4.07e-05 | 1382.0ms/step | 14.5 samples/s +Step 36040/44000 | Loss: 2.8827 | LR: 4.06e-05 | 1382.4ms/step | 14.5 samples/s +Step 36050/44000 | Loss: 2.9098 | LR: 4.05e-05 | 1382.7ms/step | 14.5 samples/s +Step 36060/44000 | Loss: 2.8954 | LR: 4.04e-05 | 1384.8ms/step | 14.4 samples/s +Step 36070/44000 | Loss: 2.9094 | LR: 4.03e-05 | 1387.0ms/step | 14.4 samples/s +Step 36080/44000 | Loss: 2.8706 | LR: 4.02e-05 | 1386.3ms/step | 14.4 samples/s +Step 36090/44000 | Loss: 2.8780 | LR: 4.01e-05 | 1384.9ms/step | 14.4 samples/s +Step 36100/44000 | Loss: 2.9010 | LR: 4.00e-05 | 1386.0ms/step | 14.4 samples/s +Step 36110/44000 | Loss: 2.8863 | LR: 3.99e-05 | 1386.6ms/step | 14.4 samples/s +Step 36120/44000 | Loss: 2.8712 | LR: 3.98e-05 | 1386.5ms/step | 14.4 samples/s +Step 36130/44000 | Loss: 2.8926 | LR: 3.97e-05 | 1386.9ms/step | 14.4 samples/s +Step 36140/44000 | Loss: 2.8512 | LR: 3.97e-05 | 1385.1ms/step | 14.4 samples/s +Step 36150/44000 | Loss: 2.8860 | LR: 3.96e-05 | 1386.5ms/step | 14.4 samples/s +Step 36160/44000 | Loss: 2.8737 | LR: 3.95e-05 | 1387.5ms/step | 14.4 samples/s +Step 36170/44000 | Loss: 2.8877 | LR: 3.94e-05 | 1388.4ms/step | 14.4 samples/s +Step 36180/44000 | Loss: 2.8878 | LR: 3.93e-05 | 1387.6ms/step | 14.4 samples/s +Step 36190/44000 | Loss: 2.8254 | LR: 3.92e-05 | 1386.2ms/step | 14.4 samples/s +Step 36200/44000 | Loss: 2.8825 | LR: 3.91e-05 | 1389.9ms/step | 14.4 samples/s +Step 36210/44000 | Loss: 2.8906 | LR: 3.90e-05 | 1387.2ms/step | 14.4 samples/s +Step 36220/44000 | Loss: 2.8620 | LR: 3.89e-05 | 1386.8ms/step | 14.4 samples/s +Step 36230/44000 | Loss: 2.8750 | LR: 3.88e-05 | 1385.8ms/step | 14.4 samples/s +Step 36240/44000 | Loss: 2.8920 | LR: 3.87e-05 | 1384.5ms/step | 14.4 samples/s +Step 36250/44000 | Loss: 2.8814 | LR: 3.86e-05 | 1387.7ms/step | 14.4 samples/s +Step 36260/44000 | Loss: 2.8944 | LR: 3.85e-05 | 1387.2ms/step | 14.4 samples/s +Step 36270/44000 | Loss: 2.9032 | LR: 3.84e-05 | 1386.3ms/step | 14.4 samples/s +Step 36280/44000 | Loss: 2.9088 | LR: 3.83e-05 | 1387.5ms/step | 14.4 samples/s +Step 36290/44000 | Loss: 2.8930 | LR: 3.82e-05 | 1387.2ms/step | 14.4 samples/s +Step 36300/44000 | Loss: 2.8924 | LR: 3.81e-05 | 1386.3ms/step | 14.4 samples/s +Step 36310/44000 | Loss: 2.8558 | LR: 3.80e-05 | 1385.2ms/step | 14.4 samples/s +Step 36320/44000 | Loss: 2.8750 | LR: 3.79e-05 | 1385.3ms/step | 14.4 samples/s +Step 36330/44000 | Loss: 2.8606 | LR: 3.78e-05 | 1385.1ms/step | 14.4 samples/s +Step 36340/44000 | Loss: 2.8702 | LR: 3.77e-05 | 1387.8ms/step | 14.4 samples/s +Step 36350/44000 | Loss: 2.8908 | LR: 3.76e-05 | 1385.0ms/step | 14.4 samples/s +Step 36360/44000 | Loss: 2.9062 | LR: 3.75e-05 | 1384.9ms/step | 14.4 samples/s +Step 36370/44000 | Loss: 2.8536 | LR: 3.74e-05 | 1386.2ms/step | 14.4 samples/s +Step 36380/44000 | Loss: 2.8923 | LR: 3.73e-05 | 1384.8ms/step | 14.4 samples/s +Step 36390/44000 | Loss: 2.8981 | LR: 3.72e-05 | 1385.2ms/step | 14.4 samples/s +Step 36400/44000 | Loss: 2.8678 | LR: 3.71e-05 | 1385.9ms/step | 14.4 samples/s +Step 36410/44000 | Loss: 2.9049 | LR: 3.70e-05 | 1386.7ms/step | 14.4 samples/s +Step 36420/44000 | Loss: 2.8893 | LR: 3.69e-05 | 1388.4ms/step | 14.4 samples/s +Step 36430/44000 | Loss: 2.8679 | LR: 3.69e-05 | 1388.6ms/step | 14.4 samples/s +Step 36440/44000 | Loss: 2.8943 | LR: 3.68e-05 | 1386.0ms/step | 14.4 samples/s +Step 36450/44000 | Loss: 2.8639 | LR: 3.67e-05 | 1385.2ms/step | 14.4 samples/s +Step 36460/44000 | Loss: 2.8810 | LR: 3.66e-05 | 1386.1ms/step | 14.4 samples/s +Step 36470/44000 | Loss: 2.8919 | LR: 3.65e-05 | 1385.3ms/step | 14.4 samples/s +Step 36480/44000 | Loss: 2.8772 | LR: 3.64e-05 | 1386.1ms/step | 14.4 samples/s +Step 36490/44000 | Loss: 2.8739 | LR: 3.63e-05 | 1385.2ms/step | 14.4 samples/s +Step 36500/44000 | Loss: 2.8833 | LR: 3.62e-05 | 1384.8ms/step | 14.4 samples/s +Step 36510/44000 | Loss: 2.8478 | LR: 3.61e-05 | 1385.8ms/step | 14.4 samples/s +Step 36520/44000 | Loss: 2.8693 | LR: 3.60e-05 | 1385.7ms/step | 14.4 samples/s +Step 36530/44000 | Loss: 2.8749 | LR: 3.59e-05 | 1385.3ms/step | 14.4 samples/s +Step 36540/44000 | Loss: 2.9106 | LR: 3.58e-05 | 1384.5ms/step | 14.4 samples/s +Step 36550/44000 | Loss: 2.8732 | LR: 3.57e-05 | 1386.0ms/step | 14.4 samples/s +Step 36560/44000 | Loss: 2.8693 | LR: 3.56e-05 | 1386.7ms/step | 14.4 samples/s +Step 36570/44000 | Loss: 2.8881 | LR: 3.55e-05 | 1386.4ms/step | 14.4 samples/s +Step 36580/44000 | Loss: 2.8701 | LR: 3.54e-05 | 1386.1ms/step | 14.4 samples/s +Step 36590/44000 | Loss: 2.9144 | LR: 3.53e-05 | 1387.8ms/step | 14.4 samples/s +Step 36600/44000 | Loss: 2.8814 | LR: 3.53e-05 | 1388.9ms/step | 14.4 samples/s +Step 36610/44000 | Loss: 2.8688 | LR: 3.52e-05 | 1387.4ms/step | 14.4 samples/s +Step 36620/44000 | Loss: 2.9125 | LR: 3.51e-05 | 1387.3ms/step | 14.4 samples/s +Step 36630/44000 | Loss: 2.9098 | LR: 3.50e-05 | 1385.8ms/step | 14.4 samples/s +Step 36640/44000 | Loss: 2.9090 | LR: 3.49e-05 | 1385.9ms/step | 14.4 samples/s +Step 36650/44000 | Loss: 2.8957 | LR: 3.48e-05 | 1387.7ms/step | 14.4 samples/s +Step 36660/44000 | Loss: 2.8677 | LR: 3.47e-05 | 1388.8ms/step | 14.4 samples/s +Step 36670/44000 | Loss: 2.8795 | LR: 3.46e-05 | 1386.3ms/step | 14.4 samples/s +Step 36680/44000 | Loss: 2.8728 | LR: 3.45e-05 | 1384.2ms/step | 14.4 samples/s +Step 36690/44000 | Loss: 2.8705 | LR: 3.44e-05 | 1387.6ms/step | 14.4 samples/s +Step 36700/44000 | Loss: 2.8950 | LR: 3.43e-05 | 1385.6ms/step | 14.4 samples/s +Step 36710/44000 | Loss: 2.8751 | LR: 3.42e-05 | 1385.8ms/step | 14.4 samples/s +Step 36720/44000 | Loss: 2.8612 | LR: 3.41e-05 | 1386.7ms/step | 14.4 samples/s +Step 36730/44000 | Loss: 2.8896 | LR: 3.41e-05 | 1385.7ms/step | 14.4 samples/s +Step 36740/44000 | Loss: 2.8770 | LR: 3.40e-05 | 1386.2ms/step | 14.4 samples/s +Step 36750/44000 | Loss: 2.8971 | LR: 3.39e-05 | 1386.0ms/step | 14.4 samples/s +Step 36760/44000 | Loss: 2.8743 | LR: 3.38e-05 | 1386.6ms/step | 14.4 samples/s +Step 36770/44000 | Loss: 2.8784 | LR: 3.37e-05 | 1386.7ms/step | 14.4 samples/s +Step 36780/44000 | Loss: 2.8895 | LR: 3.36e-05 | 1386.2ms/step | 14.4 samples/s +Step 36790/44000 | Loss: 2.8558 | LR: 3.35e-05 | 1387.5ms/step | 14.4 samples/s +Step 36800/44000 | Loss: 2.8866 | LR: 3.34e-05 | 1388.7ms/step | 14.4 samples/s +Step 36810/44000 | Loss: 2.8847 | LR: 3.33e-05 | 1387.4ms/step | 14.4 samples/s +Step 36820/44000 | Loss: 2.8898 | LR: 3.32e-05 | 1386.9ms/step | 14.4 samples/s +Step 36830/44000 | Loss: 2.8640 | LR: 3.31e-05 | 1385.9ms/step | 14.4 samples/s +Step 36840/44000 | Loss: 2.8889 | LR: 3.31e-05 | 1385.2ms/step | 14.4 samples/s +Step 36850/44000 | Loss: 2.8899 | LR: 3.30e-05 | 1387.1ms/step | 14.4 samples/s +Step 36860/44000 | Loss: 2.8885 | LR: 3.29e-05 | 1387.7ms/step | 14.4 samples/s +Step 36870/44000 | Loss: 2.8596 | LR: 3.28e-05 | 1387.5ms/step | 14.4 samples/s +Step 36880/44000 | Loss: 2.8592 | LR: 3.27e-05 | 1386.6ms/step | 14.4 samples/s +Step 36890/44000 | Loss: 2.9091 | LR: 3.26e-05 | 1387.6ms/step | 14.4 samples/s +Step 36900/44000 | Loss: 2.8762 | LR: 3.25e-05 | 1386.9ms/step | 14.4 samples/s +Step 36910/44000 | Loss: 2.8724 | LR: 3.24e-05 | 1389.2ms/step | 14.4 samples/s +Step 36920/44000 | Loss: 2.8903 | LR: 3.23e-05 | 1386.2ms/step | 14.4 samples/s +Step 36930/44000 | Loss: 2.8602 | LR: 3.22e-05 | 1385.9ms/step | 14.4 samples/s +Step 36940/44000 | Loss: 2.8794 | LR: 3.22e-05 | 1386.2ms/step | 14.4 samples/s +Step 36950/44000 | Loss: 2.8933 | LR: 3.21e-05 | 1388.0ms/step | 14.4 samples/s +Step 36960/44000 | Loss: 2.8736 | LR: 3.20e-05 | 1387.1ms/step | 14.4 samples/s +Step 36970/44000 | Loss: 2.9054 | LR: 3.19e-05 | 1385.1ms/step | 14.4 samples/s +Step 36980/44000 | Loss: 2.8906 | LR: 3.18e-05 | 1387.8ms/step | 14.4 samples/s +Step 36990/44000 | Loss: 2.8857 | LR: 3.17e-05 | 1386.0ms/step | 14.4 samples/s +Step 37000/44000 | Loss: 2.8944 | LR: 3.16e-05 | 1387.3ms/step | 14.4 samples/s +Validation at step 37000: Loss = 2.8848, Perplexity = 17.90 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-37000 +Step 37010/44000 | Loss: 2.8832 | LR: 3.15e-05 | 3871.3ms/step | 5.2 samples/s +Step 37020/44000 | Loss: 2.8963 | LR: 3.15e-05 | 1382.8ms/step | 14.5 samples/s +Step 37030/44000 | Loss: 2.8777 | LR: 3.14e-05 | 1384.3ms/step | 14.4 samples/s +Step 37040/44000 | Loss: 2.8589 | LR: 3.13e-05 | 1387.8ms/step | 14.4 samples/s +Step 37050/44000 | Loss: 2.8870 | LR: 3.12e-05 | 1387.1ms/step | 14.4 samples/s +Step 37060/44000 | Loss: 2.8443 | LR: 3.11e-05 | 1385.7ms/step | 14.4 samples/s +Step 37070/44000 | Loss: 2.9023 | LR: 3.10e-05 | 1386.9ms/step | 14.4 samples/s +Step 37080/44000 | Loss: 2.8494 | LR: 3.09e-05 | 1386.7ms/step | 14.4 samples/s +Step 37090/44000 | Loss: 2.8747 | LR: 3.08e-05 | 1388.9ms/step | 14.4 samples/s +Step 37100/44000 | Loss: 2.9059 | LR: 3.07e-05 | 1390.7ms/step | 14.4 samples/s +Step 37110/44000 | Loss: 2.8353 | LR: 3.07e-05 | 1396.7ms/step | 14.3 samples/s +Step 37120/44000 | Loss: 2.8948 | LR: 3.06e-05 | 1388.1ms/step | 14.4 samples/s +Step 37130/44000 | Loss: 2.8489 | LR: 3.05e-05 | 1390.0ms/step | 14.4 samples/s +Step 37140/44000 | Loss: 2.8870 | LR: 3.04e-05 | 1387.8ms/step | 14.4 samples/s +Step 37150/44000 | Loss: 2.8835 | LR: 3.03e-05 | 1389.7ms/step | 14.4 samples/s +Step 37160/44000 | Loss: 2.9086 | LR: 3.02e-05 | 1388.2ms/step | 14.4 samples/s +Step 37170/44000 | Loss: 2.8783 | LR: 3.01e-05 | 1389.5ms/step | 14.4 samples/s +Step 37180/44000 | Loss: 2.8839 | LR: 3.01e-05 | 1387.4ms/step | 14.4 samples/s +Step 37190/44000 | Loss: 2.8879 | LR: 3.00e-05 | 1389.3ms/step | 14.4 samples/s +Step 37200/44000 | Loss: 2.9002 | LR: 2.99e-05 | 1389.8ms/step | 14.4 samples/s +Step 37210/44000 | Loss: 2.8557 | LR: 2.98e-05 | 1388.3ms/step | 14.4 samples/s +Step 37220/44000 | Loss: 2.8683 | LR: 2.97e-05 | 1387.7ms/step | 14.4 samples/s +Step 37230/44000 | Loss: 2.8813 | LR: 2.96e-05 | 1388.1ms/step | 14.4 samples/s +Step 37240/44000 | Loss: 2.8958 | LR: 2.95e-05 | 1387.9ms/step | 14.4 samples/s +Step 37250/44000 | Loss: 2.9019 | LR: 2.95e-05 | 1383.9ms/step | 14.5 samples/s +Step 37260/44000 | Loss: 2.8710 | LR: 2.94e-05 | 1383.9ms/step | 14.5 samples/s +Step 37270/44000 | Loss: 2.9097 | LR: 2.93e-05 | 1384.6ms/step | 14.4 samples/s +Step 37280/44000 | Loss: 2.8702 | LR: 2.92e-05 | 1386.7ms/step | 14.4 samples/s +Step 37290/44000 | Loss: 2.8732 | LR: 2.91e-05 | 1386.4ms/step | 14.4 samples/s +Step 37300/44000 | Loss: 2.8841 | LR: 2.90e-05 | 1386.4ms/step | 14.4 samples/s +Step 37310/44000 | Loss: 2.8963 | LR: 2.89e-05 | 1384.4ms/step | 14.4 samples/s +Step 37320/44000 | Loss: 2.8881 | LR: 2.89e-05 | 1385.3ms/step | 14.4 samples/s +Step 37330/44000 | Loss: 2.8633 | LR: 2.88e-05 | 1386.5ms/step | 14.4 samples/s +Step 37340/44000 | Loss: 2.8662 | LR: 2.87e-05 | 1386.7ms/step | 14.4 samples/s +Step 37350/44000 | Loss: 2.8985 | LR: 2.86e-05 | 1386.6ms/step | 14.4 samples/s +Step 37360/44000 | Loss: 2.8685 | LR: 2.85e-05 | 1389.0ms/step | 14.4 samples/s +Step 37370/44000 | Loss: 2.8852 | LR: 2.84e-05 | 1387.8ms/step | 14.4 samples/s +Step 37380/44000 | Loss: 2.8738 | LR: 2.84e-05 | 1389.9ms/step | 14.4 samples/s +Step 37390/44000 | Loss: 2.8577 | LR: 2.83e-05 | 1387.9ms/step | 14.4 samples/s +Step 37400/44000 | Loss: 2.8664 | LR: 2.82e-05 | 1387.5ms/step | 14.4 samples/s +Step 37410/44000 | Loss: 2.8488 | LR: 2.81e-05 | 1387.3ms/step | 14.4 samples/s +Step 37420/44000 | Loss: 2.8691 | LR: 2.80e-05 | 1387.7ms/step | 14.4 samples/s +Step 37430/44000 | Loss: 2.9053 | LR: 2.79e-05 | 1389.7ms/step | 14.4 samples/s +Step 37440/44000 | Loss: 2.8965 | LR: 2.78e-05 | 1388.2ms/step | 14.4 samples/s +Step 37450/44000 | Loss: 2.8826 | LR: 2.78e-05 | 1390.1ms/step | 14.4 samples/s +Step 37460/44000 | Loss: 2.8748 | LR: 2.77e-05 | 1390.0ms/step | 14.4 samples/s +Step 37470/44000 | Loss: 2.8924 | LR: 2.76e-05 | 1389.9ms/step | 14.4 samples/s +Step 37480/44000 | Loss: 2.8794 | LR: 2.75e-05 | 1388.8ms/step | 14.4 samples/s +Step 37490/44000 | Loss: 2.8658 | LR: 2.74e-05 | 1387.1ms/step | 14.4 samples/s +Step 37500/44000 | Loss: 2.8848 | LR: 2.74e-05 | 1389.6ms/step | 14.4 samples/s +Step 37510/44000 | Loss: 2.8586 | LR: 2.73e-05 | 1388.5ms/step | 14.4 samples/s +Step 37520/44000 | Loss: 2.8938 | LR: 2.72e-05 | 1388.8ms/step | 14.4 samples/s +Step 37530/44000 | Loss: 2.8736 | LR: 2.71e-05 | 1387.1ms/step | 14.4 samples/s +Step 37540/44000 | Loss: 2.8618 | LR: 2.70e-05 | 1386.5ms/step | 14.4 samples/s +Step 37550/44000 | Loss: 2.8259 | LR: 2.69e-05 | 1386.0ms/step | 14.4 samples/s +Step 37560/44000 | Loss: 2.8767 | LR: 2.69e-05 | 1387.0ms/step | 14.4 samples/s +Step 37570/44000 | Loss: 2.8675 | LR: 2.68e-05 | 1387.5ms/step | 14.4 samples/s +Step 37580/44000 | Loss: 2.8737 | LR: 2.67e-05 | 1386.3ms/step | 14.4 samples/s +Step 37590/44000 | Loss: 2.8760 | LR: 2.66e-05 | 1386.8ms/step | 14.4 samples/s +Step 37600/44000 | Loss: 2.8669 | LR: 2.65e-05 | 1386.2ms/step | 14.4 samples/s +Step 37610/44000 | Loss: 2.8682 | LR: 2.65e-05 | 1386.1ms/step | 14.4 samples/s +Step 37620/44000 | Loss: 2.8842 | LR: 2.64e-05 | 1387.8ms/step | 14.4 samples/s +Step 37630/44000 | Loss: 2.8832 | LR: 2.63e-05 | 1386.6ms/step | 14.4 samples/s +Step 37640/44000 | Loss: 2.9099 | LR: 2.62e-05 | 1387.4ms/step | 14.4 samples/s +Step 37650/44000 | Loss: 2.8560 | LR: 2.61e-05 | 1385.6ms/step | 14.4 samples/s +Step 37660/44000 | Loss: 2.8978 | LR: 2.60e-05 | 1387.2ms/step | 14.4 samples/s +Step 37670/44000 | Loss: 2.8608 | LR: 2.60e-05 | 1386.6ms/step | 14.4 samples/s +Step 37680/44000 | Loss: 2.8988 | LR: 2.59e-05 | 1385.9ms/step | 14.4 samples/s +Step 37690/44000 | Loss: 2.8879 | LR: 2.58e-05 | 1387.0ms/step | 14.4 samples/s +Step 37700/44000 | Loss: 2.8984 | LR: 2.57e-05 | 1387.8ms/step | 14.4 samples/s +Step 37710/44000 | Loss: 2.8994 | LR: 2.56e-05 | 1386.0ms/step | 14.4 samples/s +Step 37720/44000 | Loss: 2.8790 | LR: 2.56e-05 | 1386.2ms/step | 14.4 samples/s +Step 37730/44000 | Loss: 2.8752 | LR: 2.55e-05 | 1386.1ms/step | 14.4 samples/s +Step 37740/44000 | Loss: 2.8621 | LR: 2.54e-05 | 1386.3ms/step | 14.4 samples/s +Step 37750/44000 | Loss: 2.8658 | LR: 2.53e-05 | 1386.8ms/step | 14.4 samples/s +Step 37760/44000 | Loss: 2.8634 | LR: 2.52e-05 | 1385.5ms/step | 14.4 samples/s +Step 37770/44000 | Loss: 2.9042 | LR: 2.52e-05 | 1384.6ms/step | 14.4 samples/s +Step 37780/44000 | Loss: 2.8841 | LR: 2.51e-05 | 1384.5ms/step | 14.4 samples/s +Step 37790/44000 | Loss: 2.8811 | LR: 2.50e-05 | 1384.0ms/step | 14.5 samples/s +Step 37800/44000 | Loss: 2.8981 | LR: 2.49e-05 | 1385.4ms/step | 14.4 samples/s +Step 37810/44000 | Loss: 2.8842 | LR: 2.48e-05 | 1386.8ms/step | 14.4 samples/s +Step 37820/44000 | Loss: 2.8923 | LR: 2.48e-05 | 1385.8ms/step | 14.4 samples/s +Step 37830/44000 | Loss: 2.9044 | LR: 2.47e-05 | 1385.9ms/step | 14.4 samples/s +Step 37840/44000 | Loss: 2.8770 | LR: 2.46e-05 | 1386.9ms/step | 14.4 samples/s +Step 37850/44000 | Loss: 2.8700 | LR: 2.45e-05 | 1385.7ms/step | 14.4 samples/s +Step 37860/44000 | Loss: 2.8627 | LR: 2.45e-05 | 1386.8ms/step | 14.4 samples/s +Step 37870/44000 | Loss: 2.8596 | LR: 2.44e-05 | 1385.2ms/step | 14.4 samples/s +Step 37880/44000 | Loss: 2.8825 | LR: 2.43e-05 | 1386.4ms/step | 14.4 samples/s +Step 37890/44000 | Loss: 2.9026 | LR: 2.42e-05 | 1386.1ms/step | 14.4 samples/s +Step 37900/44000 | Loss: 2.8880 | LR: 2.41e-05 | 1384.8ms/step | 14.4 samples/s +Step 37910/44000 | Loss: 2.8923 | LR: 2.41e-05 | 1385.2ms/step | 14.4 samples/s +Step 37920/44000 | Loss: 2.8762 | LR: 2.40e-05 | 1385.3ms/step | 14.4 samples/s +Step 37930/44000 | Loss: 2.8576 | LR: 2.39e-05 | 1384.9ms/step | 14.4 samples/s +Step 37940/44000 | Loss: 2.8581 | LR: 2.38e-05 | 1385.0ms/step | 14.4 samples/s +Step 37950/44000 | Loss: 2.8798 | LR: 2.38e-05 | 1383.3ms/step | 14.5 samples/s +Step 37960/44000 | Loss: 2.8734 | LR: 2.37e-05 | 1384.5ms/step | 14.4 samples/s +Step 37970/44000 | Loss: 2.8790 | LR: 2.36e-05 | 1383.3ms/step | 14.5 samples/s +Step 37980/44000 | Loss: 2.8981 | LR: 2.35e-05 | 1385.3ms/step | 14.4 samples/s +Step 37990/44000 | Loss: 2.8799 | LR: 2.34e-05 | 1385.3ms/step | 14.4 samples/s +Step 38000/44000 | Loss: 2.8936 | LR: 2.34e-05 | 1385.3ms/step | 14.4 samples/s +Validation at step 38000: Loss = 2.8845, Perplexity = 17.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-38000 +Step 38010/44000 | Loss: 2.8732 | LR: 2.33e-05 | 3885.4ms/step | 5.1 samples/s +Step 38020/44000 | Loss: 2.8903 | LR: 2.32e-05 | 1387.0ms/step | 14.4 samples/s +Step 38030/44000 | Loss: 2.8827 | LR: 2.31e-05 | 1384.3ms/step | 14.4 samples/s +Step 38040/44000 | Loss: 2.8812 | LR: 2.31e-05 | 1384.3ms/step | 14.4 samples/s +Step 38050/44000 | Loss: 2.8530 | LR: 2.30e-05 | 1386.5ms/step | 14.4 samples/s +Step 38060/44000 | Loss: 2.8944 | LR: 2.29e-05 | 1381.8ms/step | 14.5 samples/s +Step 38070/44000 | Loss: 2.8735 | LR: 2.28e-05 | 1383.0ms/step | 14.5 samples/s +Step 38080/44000 | Loss: 2.8914 | LR: 2.28e-05 | 1382.2ms/step | 14.5 samples/s +Step 38090/44000 | Loss: 2.9088 | LR: 2.27e-05 | 1383.1ms/step | 14.5 samples/s +Step 38100/44000 | Loss: 2.8753 | LR: 2.26e-05 | 1384.3ms/step | 14.4 samples/s +Step 38110/44000 | Loss: 2.8646 | LR: 2.25e-05 | 1384.2ms/step | 14.4 samples/s +Step 38120/44000 | Loss: 2.8962 | LR: 2.25e-05 | 1386.0ms/step | 14.4 samples/s +Step 38130/44000 | Loss: 2.8995 | LR: 2.24e-05 | 1386.8ms/step | 14.4 samples/s +Step 38140/44000 | Loss: 2.9090 | LR: 2.23e-05 | 1386.4ms/step | 14.4 samples/s +Step 38150/44000 | Loss: 2.9317 | LR: 2.22e-05 | 1385.7ms/step | 14.4 samples/s +Step 38160/44000 | Loss: 2.8850 | LR: 2.22e-05 | 1385.8ms/step | 14.4 samples/s +Step 38170/44000 | Loss: 2.8936 | LR: 2.21e-05 | 1385.6ms/step | 14.4 samples/s +Step 38180/44000 | Loss: 2.8827 | LR: 2.20e-05 | 1385.4ms/step | 14.4 samples/s +Step 38190/44000 | Loss: 2.8600 | LR: 2.19e-05 | 1385.4ms/step | 14.4 samples/s +Step 38200/44000 | Loss: 2.8746 | LR: 2.19e-05 | 1386.0ms/step | 14.4 samples/s +Step 38210/44000 | Loss: 2.8600 | LR: 2.18e-05 | 1384.6ms/step | 14.4 samples/s +Step 38220/44000 | Loss: 2.8829 | LR: 2.17e-05 | 1386.2ms/step | 14.4 samples/s +Step 38230/44000 | Loss: 2.8883 | LR: 2.16e-05 | 1385.1ms/step | 14.4 samples/s +Step 38240/44000 | Loss: 2.8790 | LR: 2.16e-05 | 1386.0ms/step | 14.4 samples/s +Step 38250/44000 | Loss: 2.8805 | LR: 2.15e-05 | 1386.3ms/step | 14.4 samples/s +Step 38260/44000 | Loss: 2.8730 | LR: 2.14e-05 | 1385.9ms/step | 14.4 samples/s +Step 38270/44000 | Loss: 2.8767 | LR: 2.13e-05 | 1385.7ms/step | 14.4 samples/s +Step 38280/44000 | Loss: 2.8939 | LR: 2.13e-05 | 1388.7ms/step | 14.4 samples/s +Step 38290/44000 | Loss: 2.8776 | LR: 2.12e-05 | 1387.4ms/step | 14.4 samples/s +Step 38300/44000 | Loss: 2.8422 | LR: 2.11e-05 | 1389.6ms/step | 14.4 samples/s +Step 38310/44000 | Loss: 2.8842 | LR: 2.11e-05 | 1386.2ms/step | 14.4 samples/s +Step 38320/44000 | Loss: 2.8899 | LR: 2.10e-05 | 1386.7ms/step | 14.4 samples/s +Step 38330/44000 | Loss: 2.8881 | LR: 2.09e-05 | 1386.1ms/step | 14.4 samples/s +Step 38340/44000 | Loss: 2.8654 | LR: 2.08e-05 | 1387.4ms/step | 14.4 samples/s +Step 38350/44000 | Loss: 2.8989 | LR: 2.08e-05 | 1385.8ms/step | 14.4 samples/s +Step 38360/44000 | Loss: 2.8822 | LR: 2.07e-05 | 1384.2ms/step | 14.4 samples/s +Step 38370/44000 | Loss: 2.8879 | LR: 2.06e-05 | 1385.8ms/step | 14.4 samples/s +Step 38380/44000 | Loss: 2.9039 | LR: 2.05e-05 | 1385.9ms/step | 14.4 samples/s +Step 38390/44000 | Loss: 2.8577 | LR: 2.05e-05 | 1385.7ms/step | 14.4 samples/s +Step 38400/44000 | Loss: 2.8660 | LR: 2.04e-05 | 1385.0ms/step | 14.4 samples/s +Step 38410/44000 | Loss: 2.8989 | LR: 2.03e-05 | 1385.2ms/step | 14.4 samples/s +Step 38420/44000 | Loss: 2.8675 | LR: 2.03e-05 | 1385.7ms/step | 14.4 samples/s +Step 38430/44000 | Loss: 2.8771 | LR: 2.02e-05 | 1388.0ms/step | 14.4 samples/s +Step 38440/44000 | Loss: 2.8848 | LR: 2.01e-05 | 1386.7ms/step | 14.4 samples/s +Step 38450/44000 | Loss: 2.8866 | LR: 2.00e-05 | 1384.6ms/step | 14.4 samples/s +Step 38460/44000 | Loss: 2.9069 | LR: 2.00e-05 | 1386.2ms/step | 14.4 samples/s +Step 38470/44000 | Loss: 2.8753 | LR: 1.99e-05 | 1386.1ms/step | 14.4 samples/s +Step 38480/44000 | Loss: 2.8964 | LR: 1.98e-05 | 1386.6ms/step | 14.4 samples/s +Step 38490/44000 | Loss: 2.8717 | LR: 1.98e-05 | 1385.3ms/step | 14.4 samples/s +Step 38500/44000 | Loss: 2.9005 | LR: 1.97e-05 | 1385.8ms/step | 14.4 samples/s +Step 38510/44000 | Loss: 2.8470 | LR: 1.96e-05 | 1386.3ms/step | 14.4 samples/s +Step 38520/44000 | Loss: 2.8752 | LR: 1.95e-05 | 1384.0ms/step | 14.5 samples/s +Step 38530/44000 | Loss: 2.8843 | LR: 1.95e-05 | 1383.5ms/step | 14.5 samples/s +Step 38540/44000 | Loss: 2.8862 | LR: 1.94e-05 | 1385.4ms/step | 14.4 samples/s +Step 38550/44000 | Loss: 2.8689 | LR: 1.93e-05 | 1387.9ms/step | 14.4 samples/s +Step 38560/44000 | Loss: 2.8820 | LR: 1.93e-05 | 1385.1ms/step | 14.4 samples/s +Step 38570/44000 | Loss: 2.8794 | LR: 1.92e-05 | 1385.0ms/step | 14.4 samples/s +Step 38580/44000 | Loss: 2.8846 | LR: 1.91e-05 | 1386.5ms/step | 14.4 samples/s +Step 38590/44000 | Loss: 2.8638 | LR: 1.91e-05 | 1385.3ms/step | 14.4 samples/s +Step 38600/44000 | Loss: 2.8790 | LR: 1.90e-05 | 1384.6ms/step | 14.4 samples/s +Step 38610/44000 | Loss: 2.8725 | LR: 1.89e-05 | 1386.3ms/step | 14.4 samples/s +Step 38620/44000 | Loss: 2.8718 | LR: 1.88e-05 | 1384.0ms/step | 14.5 samples/s +Step 38630/44000 | Loss: 2.9002 | LR: 1.88e-05 | 1383.9ms/step | 14.5 samples/s +Step 38640/44000 | Loss: 2.8794 | LR: 1.87e-05 | 1384.2ms/step | 14.4 samples/s +Step 38650/44000 | Loss: 2.9012 | LR: 1.86e-05 | 1387.3ms/step | 14.4 samples/s +Step 38660/44000 | Loss: 2.8794 | LR: 1.86e-05 | 1385.1ms/step | 14.4 samples/s +Step 38670/44000 | Loss: 2.9171 | LR: 1.85e-05 | 1385.2ms/step | 14.4 samples/s +Step 38680/44000 | Loss: 2.8931 | LR: 1.84e-05 | 1385.2ms/step | 14.4 samples/s +Step 38690/44000 | Loss: 2.8784 | LR: 1.84e-05 | 1385.9ms/step | 14.4 samples/s +Step 38700/44000 | Loss: 2.8878 | LR: 1.83e-05 | 1384.9ms/step | 14.4 samples/s +Step 38710/44000 | Loss: 2.8651 | LR: 1.82e-05 | 1384.3ms/step | 14.4 samples/s +Step 38720/44000 | Loss: 2.8743 | LR: 1.82e-05 | 1386.0ms/step | 14.4 samples/s +Step 38730/44000 | Loss: 2.8705 | LR: 1.81e-05 | 1385.6ms/step | 14.4 samples/s +Step 38740/44000 | Loss: 2.8740 | LR: 1.80e-05 | 1384.6ms/step | 14.4 samples/s +Step 38750/44000 | Loss: 2.9020 | LR: 1.80e-05 | 1385.4ms/step | 14.4 samples/s +Step 38760/44000 | Loss: 2.8867 | LR: 1.79e-05 | 1392.2ms/step | 14.4 samples/s +Step 38770/44000 | Loss: 2.8825 | LR: 1.78e-05 | 1386.8ms/step | 14.4 samples/s +Step 38780/44000 | Loss: 2.8816 | LR: 1.78e-05 | 1383.4ms/step | 14.5 samples/s +Step 38790/44000 | Loss: 2.8936 | LR: 1.77e-05 | 1385.8ms/step | 14.4 samples/s +Step 38800/44000 | Loss: 2.8733 | LR: 1.76e-05 | 1385.0ms/step | 14.4 samples/s +Step 38810/44000 | Loss: 2.8794 | LR: 1.76e-05 | 1387.0ms/step | 14.4 samples/s +Step 38820/44000 | Loss: 2.8898 | LR: 1.75e-05 | 1385.7ms/step | 14.4 samples/s +Step 38830/44000 | Loss: 2.8975 | LR: 1.74e-05 | 1386.0ms/step | 14.4 samples/s +Step 38840/44000 | Loss: 2.8746 | LR: 1.74e-05 | 1387.1ms/step | 14.4 samples/s +Step 38850/44000 | Loss: 2.9103 | LR: 1.73e-05 | 1386.9ms/step | 14.4 samples/s +Step 38860/44000 | Loss: 2.9016 | LR: 1.72e-05 | 1390.5ms/step | 14.4 samples/s +Step 38870/44000 | Loss: 2.8882 | LR: 1.72e-05 | 1392.2ms/step | 14.4 samples/s +Step 38880/44000 | Loss: 2.8545 | LR: 1.71e-05 | 1386.7ms/step | 14.4 samples/s +Step 38890/44000 | Loss: 2.8641 | LR: 1.70e-05 | 1386.7ms/step | 14.4 samples/s +Step 38900/44000 | Loss: 2.8842 | LR: 1.70e-05 | 1385.9ms/step | 14.4 samples/s +Step 38910/44000 | Loss: 2.8747 | LR: 1.69e-05 | 1386.1ms/step | 14.4 samples/s +Step 38920/44000 | Loss: 2.8758 | LR: 1.68e-05 | 1387.4ms/step | 14.4 samples/s +Step 38930/44000 | Loss: 2.8717 | LR: 1.68e-05 | 1384.6ms/step | 14.4 samples/s +Step 38940/44000 | Loss: 2.8933 | LR: 1.67e-05 | 1387.9ms/step | 14.4 samples/s +Step 38950/44000 | Loss: 2.8859 | LR: 1.66e-05 | 1386.4ms/step | 14.4 samples/s +Step 38960/44000 | Loss: 2.8792 | LR: 1.66e-05 | 1385.8ms/step | 14.4 samples/s +Step 38970/44000 | Loss: 2.8561 | LR: 1.65e-05 | 1386.8ms/step | 14.4 samples/s +Step 38980/44000 | Loss: 2.8835 | LR: 1.64e-05 | 1387.7ms/step | 14.4 samples/s +Step 38990/44000 | Loss: 2.9049 | LR: 1.64e-05 | 1386.0ms/step | 14.4 samples/s +Step 39000/44000 | Loss: 2.8801 | LR: 1.63e-05 | 1385.8ms/step | 14.4 samples/s +Validation at step 39000: Loss = 2.8843, Perplexity = 17.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-39000 +Step 39010/44000 | Loss: 2.8802 | LR: 1.62e-05 | 3894.6ms/step | 5.1 samples/s +Step 39020/44000 | Loss: 2.8670 | LR: 1.62e-05 | 1385.2ms/step | 14.4 samples/s +Step 39030/44000 | Loss: 2.8959 | LR: 1.61e-05 | 1386.9ms/step | 14.4 samples/s +Step 39040/44000 | Loss: 2.8626 | LR: 1.61e-05 | 1387.3ms/step | 14.4 samples/s +Step 39050/44000 | Loss: 2.8934 | LR: 1.60e-05 | 1386.5ms/step | 14.4 samples/s +Step 39060/44000 | Loss: 2.8872 | LR: 1.59e-05 | 1391.3ms/step | 14.4 samples/s +Step 39070/44000 | Loss: 2.9018 | LR: 1.59e-05 | 1387.3ms/step | 14.4 samples/s +Step 39080/44000 | Loss: 2.8902 | LR: 1.58e-05 | 1386.9ms/step | 14.4 samples/s +Step 39090/44000 | Loss: 2.8790 | LR: 1.57e-05 | 1383.9ms/step | 14.5 samples/s +Step 39100/44000 | Loss: 2.8791 | LR: 1.57e-05 | 1384.5ms/step | 14.4 samples/s +Step 39110/44000 | Loss: 2.8701 | LR: 1.56e-05 | 1387.8ms/step | 14.4 samples/s +Step 39120/44000 | Loss: 2.8807 | LR: 1.55e-05 | 1382.1ms/step | 14.5 samples/s +Step 39130/44000 | Loss: 2.8947 | LR: 1.55e-05 | 1384.0ms/step | 14.5 samples/s +Step 39140/44000 | Loss: 2.8784 | LR: 1.54e-05 | 1383.9ms/step | 14.5 samples/s +Step 39150/44000 | Loss: 2.9049 | LR: 1.54e-05 | 1385.2ms/step | 14.4 samples/s +Step 39160/44000 | Loss: 2.9044 | LR: 1.53e-05 | 1384.6ms/step | 14.4 samples/s +Step 39170/44000 | Loss: 2.8986 | LR: 1.52e-05 | 1384.6ms/step | 14.4 samples/s +Step 39180/44000 | Loss: 2.8839 | LR: 1.52e-05 | 1385.9ms/step | 14.4 samples/s +Step 39190/44000 | Loss: 2.8680 | LR: 1.51e-05 | 1386.5ms/step | 14.4 samples/s +Step 39200/44000 | Loss: 2.9056 | LR: 1.50e-05 | 1385.4ms/step | 14.4 samples/s +Step 39210/44000 | Loss: 2.8571 | LR: 1.50e-05 | 1384.1ms/step | 14.5 samples/s +Step 39220/44000 | Loss: 2.8619 | LR: 1.49e-05 | 1385.4ms/step | 14.4 samples/s +Step 39230/44000 | Loss: 2.9069 | LR: 1.49e-05 | 1385.0ms/step | 14.4 samples/s +Step 39240/44000 | Loss: 2.8730 | LR: 1.48e-05 | 1387.2ms/step | 14.4 samples/s +Step 39250/44000 | Loss: 2.9022 | LR: 1.47e-05 | 1386.2ms/step | 14.4 samples/s +Step 39260/44000 | Loss: 2.8724 | LR: 1.47e-05 | 1386.1ms/step | 14.4 samples/s +Step 39270/44000 | Loss: 2.8759 | LR: 1.46e-05 | 1387.1ms/step | 14.4 samples/s +Step 39280/44000 | Loss: 2.8601 | LR: 1.46e-05 | 1387.0ms/step | 14.4 samples/s +Step 39290/44000 | Loss: 2.8857 | LR: 1.45e-05 | 1387.7ms/step | 14.4 samples/s +Step 39300/44000 | Loss: 2.8951 | LR: 1.44e-05 | 1386.2ms/step | 14.4 samples/s +Step 39310/44000 | Loss: 2.8736 | LR: 1.44e-05 | 1386.1ms/step | 14.4 samples/s +Step 39320/44000 | Loss: 2.8768 | LR: 1.43e-05 | 1385.4ms/step | 14.4 samples/s +Step 39330/44000 | Loss: 2.8745 | LR: 1.42e-05 | 1383.5ms/step | 14.5 samples/s +Step 39340/44000 | Loss: 2.8865 | LR: 1.42e-05 | 1384.8ms/step | 14.4 samples/s +Step 39350/44000 | Loss: 2.8833 | LR: 1.41e-05 | 1384.4ms/step | 14.4 samples/s +Step 39360/44000 | Loss: 2.8737 | LR: 1.41e-05 | 1383.9ms/step | 14.5 samples/s +Step 39370/44000 | Loss: 2.9029 | LR: 1.40e-05 | 1385.7ms/step | 14.4 samples/s +Step 39380/44000 | Loss: 2.8899 | LR: 1.39e-05 | 1386.1ms/step | 14.4 samples/s +Step 39390/44000 | Loss: 2.8851 | LR: 1.39e-05 | 1387.9ms/step | 14.4 samples/s +Step 39400/44000 | Loss: 2.8939 | LR: 1.38e-05 | 1386.1ms/step | 14.4 samples/s +Step 39410/44000 | Loss: 2.8695 | LR: 1.38e-05 | 1386.8ms/step | 14.4 samples/s +Step 39420/44000 | Loss: 2.8839 | LR: 1.37e-05 | 1385.3ms/step | 14.4 samples/s +Step 39430/44000 | Loss: 2.8659 | LR: 1.36e-05 | 1386.5ms/step | 14.4 samples/s +Step 39440/44000 | Loss: 2.9015 | LR: 1.36e-05 | 1387.2ms/step | 14.4 samples/s +Step 39450/44000 | Loss: 2.8961 | LR: 1.35e-05 | 1386.7ms/step | 14.4 samples/s +Step 39460/44000 | Loss: 2.8776 | LR: 1.35e-05 | 1386.5ms/step | 14.4 samples/s +Step 39470/44000 | Loss: 2.8765 | LR: 1.34e-05 | 1388.9ms/step | 14.4 samples/s +Step 39480/44000 | Loss: 2.8612 | LR: 1.34e-05 | 1386.7ms/step | 14.4 samples/s +Step 39490/44000 | Loss: 2.8518 | LR: 1.33e-05 | 1387.2ms/step | 14.4 samples/s +Step 39500/44000 | Loss: 2.8750 | LR: 1.32e-05 | 1385.5ms/step | 14.4 samples/s +Step 39510/44000 | Loss: 2.8630 | LR: 1.32e-05 | 1387.5ms/step | 14.4 samples/s +Step 39520/44000 | Loss: 2.9010 | LR: 1.31e-05 | 1387.5ms/step | 14.4 samples/s +Step 39530/44000 | Loss: 2.9012 | LR: 1.31e-05 | 1386.3ms/step | 14.4 samples/s +Step 39540/44000 | Loss: 2.8814 | LR: 1.30e-05 | 1387.7ms/step | 14.4 samples/s +Step 39550/44000 | Loss: 2.9129 | LR: 1.29e-05 | 1389.7ms/step | 14.4 samples/s +Step 39560/44000 | Loss: 2.8687 | LR: 1.29e-05 | 1387.5ms/step | 14.4 samples/s +Step 39570/44000 | Loss: 2.8802 | LR: 1.28e-05 | 1385.4ms/step | 14.4 samples/s +Step 39580/44000 | Loss: 2.8807 | LR: 1.28e-05 | 1385.7ms/step | 14.4 samples/s +Step 39590/44000 | Loss: 2.8990 | LR: 1.27e-05 | 1387.5ms/step | 14.4 samples/s +Step 39600/44000 | Loss: 2.8828 | LR: 1.27e-05 | 1386.6ms/step | 14.4 samples/s +Step 39610/44000 | Loss: 2.8755 | LR: 1.26e-05 | 1383.8ms/step | 14.5 samples/s +Step 39620/44000 | Loss: 2.9223 | LR: 1.25e-05 | 1387.5ms/step | 14.4 samples/s +Step 39630/44000 | Loss: 2.8893 | LR: 1.25e-05 | 1385.5ms/step | 14.4 samples/s +Step 39640/44000 | Loss: 2.8534 | LR: 1.24e-05 | 1387.7ms/step | 14.4 samples/s +Step 39650/44000 | Loss: 2.9096 | LR: 1.24e-05 | 1385.6ms/step | 14.4 samples/s +Step 39660/44000 | Loss: 2.9114 | LR: 1.23e-05 | 1386.1ms/step | 14.4 samples/s +Step 39670/44000 | Loss: 2.8675 | LR: 1.23e-05 | 1386.1ms/step | 14.4 samples/s +Step 39680/44000 | Loss: 2.8875 | LR: 1.22e-05 | 1387.4ms/step | 14.4 samples/s +Step 39690/44000 | Loss: 2.8873 | LR: 1.22e-05 | 1385.1ms/step | 14.4 samples/s +Step 39700/44000 | Loss: 2.8953 | LR: 1.21e-05 | 1386.8ms/step | 14.4 samples/s +Step 39710/44000 | Loss: 2.8942 | LR: 1.20e-05 | 1389.2ms/step | 14.4 samples/s +Step 39720/44000 | Loss: 2.8877 | LR: 1.20e-05 | 1384.7ms/step | 14.4 samples/s +Step 39730/44000 | Loss: 2.8617 | LR: 1.19e-05 | 1386.3ms/step | 14.4 samples/s +Step 39740/44000 | Loss: 2.8634 | LR: 1.19e-05 | 1386.4ms/step | 14.4 samples/s +Step 39750/44000 | Loss: 2.8848 | LR: 1.18e-05 | 1386.5ms/step | 14.4 samples/s +Step 39760/44000 | Loss: 2.8902 | LR: 1.18e-05 | 1387.4ms/step | 14.4 samples/s +Step 39770/44000 | Loss: 2.8870 | LR: 1.17e-05 | 1387.9ms/step | 14.4 samples/s +Step 39780/44000 | Loss: 2.8857 | LR: 1.17e-05 | 1387.8ms/step | 14.4 samples/s +Step 39790/44000 | Loss: 2.8659 | LR: 1.16e-05 | 1384.5ms/step | 14.4 samples/s +Step 39800/44000 | Loss: 2.9040 | LR: 1.15e-05 | 1386.5ms/step | 14.4 samples/s +Step 39810/44000 | Loss: 2.8938 | LR: 1.15e-05 | 1385.6ms/step | 14.4 samples/s +Step 39820/44000 | Loss: 2.8737 | LR: 1.14e-05 | 1386.2ms/step | 14.4 samples/s +Step 39830/44000 | Loss: 2.8937 | LR: 1.14e-05 | 1385.4ms/step | 14.4 samples/s +Step 39840/44000 | Loss: 2.8869 | LR: 1.13e-05 | 1387.3ms/step | 14.4 samples/s +Step 39850/44000 | Loss: 2.9141 | LR: 1.13e-05 | 1387.2ms/step | 14.4 samples/s +Step 39860/44000 | Loss: 2.8661 | LR: 1.12e-05 | 1386.4ms/step | 14.4 samples/s +Step 39870/44000 | Loss: 2.9095 | LR: 1.12e-05 | 1386.4ms/step | 14.4 samples/s +Step 39880/44000 | Loss: 2.8992 | LR: 1.11e-05 | 1385.0ms/step | 14.4 samples/s +Step 39890/44000 | Loss: 2.8913 | LR: 1.11e-05 | 1385.1ms/step | 14.4 samples/s +Step 39900/44000 | Loss: 2.8902 | LR: 1.10e-05 | 1387.3ms/step | 14.4 samples/s +Step 39910/44000 | Loss: 2.8506 | LR: 1.10e-05 | 1387.3ms/step | 14.4 samples/s +Step 39920/44000 | Loss: 2.8729 | LR: 1.09e-05 | 1386.1ms/step | 14.4 samples/s +Step 39930/44000 | Loss: 2.8647 | LR: 1.08e-05 | 1385.7ms/step | 14.4 samples/s +Step 39940/44000 | Loss: 2.8803 | LR: 1.08e-05 | 1385.4ms/step | 14.4 samples/s +Step 39950/44000 | Loss: 2.8961 | LR: 1.07e-05 | 1385.8ms/step | 14.4 samples/s +Step 39960/44000 | Loss: 2.8800 | LR: 1.07e-05 | 1386.3ms/step | 14.4 samples/s +Step 39970/44000 | Loss: 2.8477 | LR: 1.06e-05 | 1386.8ms/step | 14.4 samples/s +Step 39980/44000 | Loss: 2.9133 | LR: 1.06e-05 | 1387.8ms/step | 14.4 samples/s +Step 39990/44000 | Loss: 2.8689 | LR: 1.05e-05 | 1387.5ms/step | 14.4 samples/s +Step 40000/44000 | Loss: 2.9024 | LR: 1.05e-05 | 1385.3ms/step | 14.4 samples/s +Validation at step 40000: Loss = 2.8842, Perplexity = 17.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-40000 +Step 40010/44000 | Loss: 2.8741 | LR: 1.04e-05 | 3859.8ms/step | 5.2 samples/s +Step 40020/44000 | Loss: 2.9021 | LR: 1.04e-05 | 1387.6ms/step | 14.4 samples/s +Step 40030/44000 | Loss: 2.8701 | LR: 1.03e-05 | 1388.0ms/step | 14.4 samples/s +Step 40040/44000 | Loss: 2.9019 | LR: 1.03e-05 | 1386.3ms/step | 14.4 samples/s +Step 40050/44000 | Loss: 2.8688 | LR: 1.02e-05 | 1389.3ms/step | 14.4 samples/s +Step 40060/44000 | Loss: 2.8924 | LR: 1.02e-05 | 1388.5ms/step | 14.4 samples/s +Step 40070/44000 | Loss: 2.8712 | LR: 1.01e-05 | 1389.0ms/step | 14.4 samples/s +Step 40080/44000 | Loss: 2.9008 | LR: 1.01e-05 | 1389.1ms/step | 14.4 samples/s +Step 40090/44000 | Loss: 2.8754 | LR: 1.00e-05 | 1389.7ms/step | 14.4 samples/s +Step 40100/44000 | Loss: 2.8617 | LR: 9.96e-06 | 1386.4ms/step | 14.4 samples/s +Step 40110/44000 | Loss: 2.8834 | LR: 9.91e-06 | 1387.9ms/step | 14.4 samples/s +Step 40120/44000 | Loss: 2.8952 | LR: 9.86e-06 | 1386.0ms/step | 14.4 samples/s +Step 40130/44000 | Loss: 2.8969 | LR: 9.81e-06 | 1388.0ms/step | 14.4 samples/s +Step 40140/44000 | Loss: 2.8569 | LR: 9.76e-06 | 1384.7ms/step | 14.4 samples/s +Step 40150/44000 | Loss: 2.8815 | LR: 9.71e-06 | 1383.4ms/step | 14.5 samples/s +Step 40160/44000 | Loss: 2.8717 | LR: 9.66e-06 | 1385.0ms/step | 14.4 samples/s +Step 40170/44000 | Loss: 2.8883 | LR: 9.61e-06 | 1385.2ms/step | 14.4 samples/s +Step 40180/44000 | Loss: 2.8804 | LR: 9.56e-06 | 1386.3ms/step | 14.4 samples/s +Step 40190/44000 | Loss: 2.8495 | LR: 9.51e-06 | 1387.3ms/step | 14.4 samples/s +Step 40200/44000 | Loss: 2.8784 | LR: 9.46e-06 | 1388.1ms/step | 14.4 samples/s +Step 40210/44000 | Loss: 2.8972 | LR: 9.41e-06 | 1386.7ms/step | 14.4 samples/s +Step 40220/44000 | Loss: 2.9041 | LR: 9.36e-06 | 1386.6ms/step | 14.4 samples/s +Step 40230/44000 | Loss: 2.8653 | LR: 9.32e-06 | 1385.7ms/step | 14.4 samples/s +Step 40240/44000 | Loss: 2.9003 | LR: 9.27e-06 | 1387.9ms/step | 14.4 samples/s +Step 40250/44000 | Loss: 2.8898 | LR: 9.22e-06 | 1385.8ms/step | 14.4 samples/s +Step 40260/44000 | Loss: 2.8726 | LR: 9.17e-06 | 1386.8ms/step | 14.4 samples/s +Step 40270/44000 | Loss: 2.8733 | LR: 9.12e-06 | 1386.4ms/step | 14.4 samples/s +Step 40280/44000 | Loss: 2.8780 | LR: 9.07e-06 | 1387.7ms/step | 14.4 samples/s +Step 40290/44000 | Loss: 2.9013 | LR: 9.02e-06 | 1386.9ms/step | 14.4 samples/s +Step 40300/44000 | Loss: 2.8949 | LR: 8.97e-06 | 1386.1ms/step | 14.4 samples/s +Step 40310/44000 | Loss: 2.8917 | LR: 8.93e-06 | 1387.4ms/step | 14.4 samples/s +Step 40320/44000 | Loss: 2.8912 | LR: 8.88e-06 | 1388.0ms/step | 14.4 samples/s +Step 40330/44000 | Loss: 2.8850 | LR: 8.83e-06 | 1389.0ms/step | 14.4 samples/s +Step 40340/44000 | Loss: 2.8827 | LR: 8.78e-06 | 1388.6ms/step | 14.4 samples/s +Step 40350/44000 | Loss: 2.8651 | LR: 8.74e-06 | 1388.4ms/step | 14.4 samples/s +Step 40360/44000 | Loss: 2.8887 | LR: 8.69e-06 | 1386.6ms/step | 14.4 samples/s +Step 40370/44000 | Loss: 2.8759 | LR: 8.64e-06 | 1386.7ms/step | 14.4 samples/s +Step 40380/44000 | Loss: 2.8933 | LR: 8.59e-06 | 1387.0ms/step | 14.4 samples/s +Step 40390/44000 | Loss: 2.8868 | LR: 8.55e-06 | 1387.4ms/step | 14.4 samples/s +Step 40400/44000 | Loss: 2.8926 | LR: 8.50e-06 | 1388.4ms/step | 14.4 samples/s +Step 40410/44000 | Loss: 2.8563 | LR: 8.45e-06 | 1389.2ms/step | 14.4 samples/s +Step 40420/44000 | Loss: 2.8521 | LR: 8.41e-06 | 1387.6ms/step | 14.4 samples/s +Step 40430/44000 | Loss: 2.8789 | LR: 8.36e-06 | 1387.1ms/step | 14.4 samples/s +Step 40440/44000 | Loss: 2.9021 | LR: 8.31e-06 | 1387.1ms/step | 14.4 samples/s +Step 40450/44000 | Loss: 2.8748 | LR: 8.27e-06 | 1387.0ms/step | 14.4 samples/s +Step 40460/44000 | Loss: 2.8938 | LR: 8.22e-06 | 1385.8ms/step | 14.4 samples/s +Step 40470/44000 | Loss: 2.8613 | LR: 8.17e-06 | 1387.5ms/step | 14.4 samples/s +Step 40480/44000 | Loss: 2.8787 | LR: 8.13e-06 | 1387.0ms/step | 14.4 samples/s +Step 40490/44000 | Loss: 2.9002 | LR: 8.08e-06 | 1388.3ms/step | 14.4 samples/s +Step 40500/44000 | Loss: 2.8841 | LR: 8.04e-06 | 1387.1ms/step | 14.4 samples/s +Step 40510/44000 | Loss: 2.9048 | LR: 7.99e-06 | 1386.4ms/step | 14.4 samples/s +Step 40520/44000 | Loss: 2.8589 | LR: 7.94e-06 | 1387.5ms/step | 14.4 samples/s +Step 40530/44000 | Loss: 2.9161 | LR: 7.90e-06 | 1386.2ms/step | 14.4 samples/s +Step 40540/44000 | Loss: 2.8829 | LR: 7.85e-06 | 1386.9ms/step | 14.4 samples/s +Step 40550/44000 | Loss: 2.8543 | LR: 7.81e-06 | 1386.1ms/step | 14.4 samples/s +Step 40560/44000 | Loss: 2.8901 | LR: 7.76e-06 | 1386.0ms/step | 14.4 samples/s +Step 40570/44000 | Loss: 2.8719 | LR: 7.72e-06 | 1387.8ms/step | 14.4 samples/s +Step 40580/44000 | Loss: 2.8832 | LR: 7.67e-06 | 1387.5ms/step | 14.4 samples/s +Step 40590/44000 | Loss: 2.8716 | LR: 7.63e-06 | 1387.8ms/step | 14.4 samples/s +Step 40600/44000 | Loss: 2.8678 | LR: 7.59e-06 | 1389.0ms/step | 14.4 samples/s +Step 40610/44000 | Loss: 2.8797 | LR: 7.54e-06 | 1388.7ms/step | 14.4 samples/s +Step 40620/44000 | Loss: 2.8689 | LR: 7.50e-06 | 1388.4ms/step | 14.4 samples/s +Step 40630/44000 | Loss: 2.8778 | LR: 7.45e-06 | 1387.7ms/step | 14.4 samples/s +Step 40640/44000 | Loss: 2.8867 | LR: 7.41e-06 | 1386.4ms/step | 14.4 samples/s +Step 40650/44000 | Loss: 2.8716 | LR: 7.37e-06 | 1386.0ms/step | 14.4 samples/s +Step 40660/44000 | Loss: 2.8676 | LR: 7.32e-06 | 1388.0ms/step | 14.4 samples/s +Step 40670/44000 | Loss: 2.8490 | LR: 7.28e-06 | 1389.5ms/step | 14.4 samples/s +Step 40680/44000 | Loss: 2.9116 | LR: 7.23e-06 | 1388.6ms/step | 14.4 samples/s +Step 40690/44000 | Loss: 2.9083 | LR: 7.19e-06 | 1388.0ms/step | 14.4 samples/s +Step 40700/44000 | Loss: 2.8364 | LR: 7.15e-06 | 1389.3ms/step | 14.4 samples/s +Step 40710/44000 | Loss: 2.8907 | LR: 7.10e-06 | 1389.8ms/step | 14.4 samples/s +Step 40720/44000 | Loss: 2.8559 | LR: 7.06e-06 | 1390.3ms/step | 14.4 samples/s +Step 40730/44000 | Loss: 2.8910 | LR: 7.02e-06 | 1388.3ms/step | 14.4 samples/s +Step 40740/44000 | Loss: 2.8960 | LR: 6.98e-06 | 1385.6ms/step | 14.4 samples/s +Step 40750/44000 | Loss: 2.8923 | LR: 6.93e-06 | 1386.1ms/step | 14.4 samples/s +Step 40760/44000 | Loss: 2.8657 | LR: 6.89e-06 | 1386.5ms/step | 14.4 samples/s +Step 40770/44000 | Loss: 2.8634 | LR: 6.85e-06 | 1387.9ms/step | 14.4 samples/s +Step 40780/44000 | Loss: 2.8705 | LR: 6.81e-06 | 1390.0ms/step | 14.4 samples/s +Step 40790/44000 | Loss: 2.8901 | LR: 6.77e-06 | 1388.5ms/step | 14.4 samples/s +Step 40800/44000 | Loss: 2.8497 | LR: 6.72e-06 | 1389.7ms/step | 14.4 samples/s +Step 40810/44000 | Loss: 2.9009 | LR: 6.68e-06 | 1387.9ms/step | 14.4 samples/s +Step 40820/44000 | Loss: 2.9046 | LR: 6.64e-06 | 1388.2ms/step | 14.4 samples/s +Step 40830/44000 | Loss: 2.8713 | LR: 6.60e-06 | 1388.2ms/step | 14.4 samples/s +Step 40840/44000 | Loss: 2.8916 | LR: 6.56e-06 | 1388.8ms/step | 14.4 samples/s +Step 40850/44000 | Loss: 2.8842 | LR: 6.52e-06 | 1388.8ms/step | 14.4 samples/s +Step 40860/44000 | Loss: 2.8914 | LR: 6.47e-06 | 1388.9ms/step | 14.4 samples/s +Step 40870/44000 | Loss: 2.8962 | LR: 6.43e-06 | 1385.1ms/step | 14.4 samples/s +Step 40880/44000 | Loss: 2.8940 | LR: 6.39e-06 | 1386.0ms/step | 14.4 samples/s +Step 40890/44000 | Loss: 2.9284 | LR: 6.35e-06 | 1389.9ms/step | 14.4 samples/s +Step 40900/44000 | Loss: 2.9097 | LR: 6.31e-06 | 1389.5ms/step | 14.4 samples/s +Step 40910/44000 | Loss: 2.8767 | LR: 6.27e-06 | 1387.0ms/step | 14.4 samples/s +Step 40920/44000 | Loss: 2.8637 | LR: 6.23e-06 | 1387.6ms/step | 14.4 samples/s +Step 40930/44000 | Loss: 2.9176 | LR: 6.19e-06 | 1386.5ms/step | 14.4 samples/s +Step 40940/44000 | Loss: 2.8528 | LR: 6.15e-06 | 1387.5ms/step | 14.4 samples/s +Step 40950/44000 | Loss: 2.9039 | LR: 6.11e-06 | 1385.9ms/step | 14.4 samples/s +Step 40960/44000 | Loss: 2.9002 | LR: 6.07e-06 | 1388.4ms/step | 14.4 samples/s +Step 40970/44000 | Loss: 2.8753 | LR: 6.03e-06 | 1389.4ms/step | 14.4 samples/s +Step 40980/44000 | Loss: 2.8828 | LR: 5.99e-06 | 1388.4ms/step | 14.4 samples/s +Step 40990/44000 | Loss: 2.9095 | LR: 5.95e-06 | 1387.8ms/step | 14.4 samples/s +Step 41000/44000 | Loss: 2.8660 | LR: 5.91e-06 | 1387.4ms/step | 14.4 samples/s +Validation at step 41000: Loss = 2.8841, Perplexity = 17.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-41000 +Step 41010/44000 | Loss: 2.8742 | LR: 5.87e-06 | 3871.6ms/step | 5.2 samples/s +Step 41020/44000 | Loss: 2.9096 | LR: 5.83e-06 | 1386.6ms/step | 14.4 samples/s +Step 41030/44000 | Loss: 2.8840 | LR: 5.80e-06 | 1386.3ms/step | 14.4 samples/s +Step 41040/44000 | Loss: 2.8655 | LR: 5.76e-06 | 1386.5ms/step | 14.4 samples/s +Step 41050/44000 | Loss: 2.8582 | LR: 5.72e-06 | 1387.7ms/step | 14.4 samples/s +Step 41060/44000 | Loss: 2.8807 | LR: 5.68e-06 | 1387.0ms/step | 14.4 samples/s +Step 41070/44000 | Loss: 2.8820 | LR: 5.64e-06 | 1387.1ms/step | 14.4 samples/s +Step 41080/44000 | Loss: 2.8794 | LR: 5.60e-06 | 1388.8ms/step | 14.4 samples/s +Step 41090/44000 | Loss: 2.8733 | LR: 5.56e-06 | 1389.1ms/step | 14.4 samples/s +Step 41100/44000 | Loss: 2.8806 | LR: 5.53e-06 | 1388.5ms/step | 14.4 samples/s +Step 41110/44000 | Loss: 2.9053 | LR: 5.49e-06 | 1389.3ms/step | 14.4 samples/s +Step 41120/44000 | Loss: 2.8946 | LR: 5.45e-06 | 1392.7ms/step | 14.4 samples/s +Step 41130/44000 | Loss: 2.8849 | LR: 5.41e-06 | 1389.8ms/step | 14.4 samples/s +Step 41140/44000 | Loss: 2.8875 | LR: 5.38e-06 | 1385.0ms/step | 14.4 samples/s +Step 41150/44000 | Loss: 2.8729 | LR: 5.34e-06 | 1388.3ms/step | 14.4 samples/s +Step 41160/44000 | Loss: 2.8811 | LR: 5.30e-06 | 1382.6ms/step | 14.5 samples/s +Step 41170/44000 | Loss: 2.9020 | LR: 5.26e-06 | 1384.4ms/step | 14.4 samples/s +Step 41180/44000 | Loss: 2.8548 | LR: 5.23e-06 | 1388.1ms/step | 14.4 samples/s +Step 41190/44000 | Loss: 2.8862 | LR: 5.19e-06 | 1382.4ms/step | 14.5 samples/s +Step 41200/44000 | Loss: 2.8771 | LR: 5.15e-06 | 1383.5ms/step | 14.5 samples/s +Step 41210/44000 | Loss: 2.8710 | LR: 5.12e-06 | 1389.3ms/step | 14.4 samples/s +Step 41220/44000 | Loss: 2.8888 | LR: 5.08e-06 | 1386.5ms/step | 14.4 samples/s +Step 41230/44000 | Loss: 2.8975 | LR: 5.04e-06 | 1386.3ms/step | 14.4 samples/s +Step 41240/44000 | Loss: 2.8649 | LR: 5.01e-06 | 1393.4ms/step | 14.4 samples/s +Step 41250/44000 | Loss: 2.9097 | LR: 4.97e-06 | 1386.3ms/step | 14.4 samples/s +Step 41260/44000 | Loss: 2.9162 | LR: 4.94e-06 | 1385.7ms/step | 14.4 samples/s +Step 41270/44000 | Loss: 2.8529 | LR: 4.90e-06 | 1388.2ms/step | 14.4 samples/s +Step 41280/44000 | Loss: 2.8946 | LR: 4.86e-06 | 1384.3ms/step | 14.4 samples/s +Step 41290/44000 | Loss: 2.9194 | LR: 4.83e-06 | 1383.8ms/step | 14.5 samples/s +Step 41300/44000 | Loss: 2.8882 | LR: 4.79e-06 | 1388.1ms/step | 14.4 samples/s +Step 41310/44000 | Loss: 2.8842 | LR: 4.76e-06 | 1384.4ms/step | 14.4 samples/s +Step 41320/44000 | Loss: 2.8846 | LR: 4.72e-06 | 1385.7ms/step | 14.4 samples/s +Step 41330/44000 | Loss: 2.8756 | LR: 4.69e-06 | 1386.5ms/step | 14.4 samples/s +Step 41340/44000 | Loss: 2.8921 | LR: 4.65e-06 | 1384.5ms/step | 14.4 samples/s +Step 41350/44000 | Loss: 2.8871 | LR: 4.62e-06 | 1385.3ms/step | 14.4 samples/s +Step 41360/44000 | Loss: 2.8795 | LR: 4.58e-06 | 1389.9ms/step | 14.4 samples/s +Step 41370/44000 | Loss: 2.8768 | LR: 4.55e-06 | 1385.5ms/step | 14.4 samples/s +Step 41380/44000 | Loss: 2.9346 | LR: 4.51e-06 | 1386.0ms/step | 14.4 samples/s +Step 41390/44000 | Loss: 2.8717 | LR: 4.48e-06 | 1386.7ms/step | 14.4 samples/s +Step 41400/44000 | Loss: 2.8639 | LR: 4.45e-06 | 1385.7ms/step | 14.4 samples/s +Step 41410/44000 | Loss: 2.8711 | LR: 4.41e-06 | 1385.0ms/step | 14.4 samples/s +Step 41420/44000 | Loss: 2.8723 | LR: 4.38e-06 | 1389.1ms/step | 14.4 samples/s +Step 41430/44000 | Loss: 2.8777 | LR: 4.34e-06 | 1390.2ms/step | 14.4 samples/s +Step 41440/44000 | Loss: 2.8875 | LR: 4.31e-06 | 1385.9ms/step | 14.4 samples/s +Step 41450/44000 | Loss: 2.8879 | LR: 4.28e-06 | 1385.8ms/step | 14.4 samples/s +Step 41460/44000 | Loss: 2.8626 | LR: 4.24e-06 | 1389.9ms/step | 14.4 samples/s +Step 41470/44000 | Loss: 2.9012 | LR: 4.21e-06 | 1385.7ms/step | 14.4 samples/s +Step 41480/44000 | Loss: 2.8996 | LR: 4.18e-06 | 1386.5ms/step | 14.4 samples/s +Step 41490/44000 | Loss: 2.8875 | LR: 4.14e-06 | 1387.7ms/step | 14.4 samples/s +Step 41500/44000 | Loss: 2.8734 | LR: 4.11e-06 | 1383.8ms/step | 14.5 samples/s +Step 41510/44000 | Loss: 2.9020 | LR: 4.08e-06 | 1386.0ms/step | 14.4 samples/s +Step 41520/44000 | Loss: 2.8463 | LR: 4.05e-06 | 1386.7ms/step | 14.4 samples/s +Step 41530/44000 | Loss: 2.8760 | LR: 4.01e-06 | 1385.3ms/step | 14.4 samples/s +Step 41540/44000 | Loss: 2.8802 | LR: 3.98e-06 | 1388.1ms/step | 14.4 samples/s +Step 41550/44000 | Loss: 2.8710 | LR: 3.95e-06 | 1394.8ms/step | 14.3 samples/s +Step 41560/44000 | Loss: 2.8921 | LR: 3.92e-06 | 1388.0ms/step | 14.4 samples/s +Step 41570/44000 | Loss: 2.8470 | LR: 3.88e-06 | 1385.9ms/step | 14.4 samples/s +Step 41580/44000 | Loss: 2.8717 | LR: 3.85e-06 | 1389.4ms/step | 14.4 samples/s +Step 41590/44000 | Loss: 2.8526 | LR: 3.82e-06 | 1386.3ms/step | 14.4 samples/s +Step 41600/44000 | Loss: 2.8869 | LR: 3.79e-06 | 1390.1ms/step | 14.4 samples/s +Step 41610/44000 | Loss: 2.8609 | LR: 3.76e-06 | 1390.0ms/step | 14.4 samples/s +Step 41620/44000 | Loss: 2.8682 | LR: 3.73e-06 | 1384.2ms/step | 14.4 samples/s +Step 41630/44000 | Loss: 2.9098 | LR: 3.70e-06 | 1387.8ms/step | 14.4 samples/s +Step 41640/44000 | Loss: 2.8838 | LR: 3.66e-06 | 1388.3ms/step | 14.4 samples/s +Step 41650/44000 | Loss: 2.8530 | LR: 3.63e-06 | 1387.9ms/step | 14.4 samples/s +Step 41660/44000 | Loss: 2.9075 | LR: 3.60e-06 | 1386.3ms/step | 14.4 samples/s +Step 41670/44000 | Loss: 2.8719 | LR: 3.57e-06 | 1386.4ms/step | 14.4 samples/s +Step 41680/44000 | Loss: 2.8809 | LR: 3.54e-06 | 1385.2ms/step | 14.4 samples/s +Step 41690/44000 | Loss: 2.8962 | LR: 3.51e-06 | 1385.1ms/step | 14.4 samples/s +Step 41700/44000 | Loss: 2.8696 | LR: 3.48e-06 | 1385.8ms/step | 14.4 samples/s +Step 41710/44000 | Loss: 2.8620 | LR: 3.45e-06 | 1386.8ms/step | 14.4 samples/s +Step 41720/44000 | Loss: 2.8928 | LR: 3.42e-06 | 1383.5ms/step | 14.5 samples/s +Step 41730/44000 | Loss: 2.8964 | LR: 3.39e-06 | 1385.3ms/step | 14.4 samples/s +Step 41740/44000 | Loss: 2.8728 | LR: 3.36e-06 | 1386.2ms/step | 14.4 samples/s +Step 41750/44000 | Loss: 2.8962 | LR: 3.33e-06 | 1387.0ms/step | 14.4 samples/s +Step 41760/44000 | Loss: 2.8929 | LR: 3.30e-06 | 1388.5ms/step | 14.4 samples/s +Step 41770/44000 | Loss: 2.8788 | LR: 3.27e-06 | 1386.3ms/step | 14.4 samples/s +Step 41780/44000 | Loss: 2.8713 | LR: 3.24e-06 | 1385.7ms/step | 14.4 samples/s +Step 41790/44000 | Loss: 2.8624 | LR: 3.21e-06 | 1385.1ms/step | 14.4 samples/s +Step 41800/44000 | Loss: 2.9075 | LR: 3.19e-06 | 1384.9ms/step | 14.4 samples/s +Step 41810/44000 | Loss: 2.8712 | LR: 3.16e-06 | 1385.1ms/step | 14.4 samples/s +Step 41820/44000 | Loss: 2.8807 | LR: 3.13e-06 | 1386.9ms/step | 14.4 samples/s +Step 41830/44000 | Loss: 2.8919 | LR: 3.10e-06 | 1384.2ms/step | 14.4 samples/s +Step 41840/44000 | Loss: 2.8934 | LR: 3.07e-06 | 1387.4ms/step | 14.4 samples/s +Step 41850/44000 | Loss: 2.8794 | LR: 3.04e-06 | 1386.2ms/step | 14.4 samples/s +Step 41860/44000 | Loss: 2.9018 | LR: 3.01e-06 | 1388.1ms/step | 14.4 samples/s +Step 41870/44000 | Loss: 2.8869 | LR: 2.99e-06 | 1384.7ms/step | 14.4 samples/s +Step 41880/44000 | Loss: 2.8660 | LR: 2.96e-06 | 1387.7ms/step | 14.4 samples/s +Step 41890/44000 | Loss: 2.8749 | LR: 2.93e-06 | 1387.6ms/step | 14.4 samples/s +Step 41900/44000 | Loss: 2.8715 | LR: 2.90e-06 | 1384.3ms/step | 14.4 samples/s +Step 41910/44000 | Loss: 2.8830 | LR: 2.88e-06 | 1385.3ms/step | 14.4 samples/s +Step 41920/44000 | Loss: 2.9256 | LR: 2.85e-06 | 1386.3ms/step | 14.4 samples/s +Step 41930/44000 | Loss: 2.8613 | LR: 2.82e-06 | 1386.8ms/step | 14.4 samples/s +Step 41940/44000 | Loss: 2.8726 | LR: 2.79e-06 | 1385.6ms/step | 14.4 samples/s +Step 41950/44000 | Loss: 2.8873 | LR: 2.77e-06 | 1385.5ms/step | 14.4 samples/s +Step 41960/44000 | Loss: 2.8725 | LR: 2.74e-06 | 1385.7ms/step | 14.4 samples/s +Step 41970/44000 | Loss: 2.8870 | LR: 2.71e-06 | 1386.3ms/step | 14.4 samples/s +Step 41980/44000 | Loss: 2.8638 | LR: 2.69e-06 | 1386.7ms/step | 14.4 samples/s +Step 41990/44000 | Loss: 2.8649 | LR: 2.66e-06 | 1387.8ms/step | 14.4 samples/s +Step 42000/44000 | Loss: 2.8922 | LR: 2.63e-06 | 1385.8ms/step | 14.4 samples/s +Validation at step 42000: Loss = 2.8841, Perplexity = 17.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-42000 +Step 42010/44000 | Loss: 2.8562 | LR: 2.61e-06 | 3888.2ms/step | 5.1 samples/s +Step 42020/44000 | Loss: 2.8788 | LR: 2.58e-06 | 1388.5ms/step | 14.4 samples/s +Step 42030/44000 | Loss: 2.8586 | LR: 2.56e-06 | 1391.2ms/step | 14.4 samples/s +Step 42040/44000 | Loss: 2.8885 | LR: 2.53e-06 | 1390.1ms/step | 14.4 samples/s +Step 42050/44000 | Loss: 2.8640 | LR: 2.50e-06 | 1389.3ms/step | 14.4 samples/s +Step 42060/44000 | Loss: 2.8720 | LR: 2.48e-06 | 1388.8ms/step | 14.4 samples/s +Step 42070/44000 | Loss: 2.8903 | LR: 2.45e-06 | 1388.6ms/step | 14.4 samples/s +Step 42080/44000 | Loss: 2.8826 | LR: 2.43e-06 | 1387.2ms/step | 14.4 samples/s +Step 42090/44000 | Loss: 2.9023 | LR: 2.40e-06 | 1388.0ms/step | 14.4 samples/s +Step 42100/44000 | Loss: 2.8829 | LR: 2.38e-06 | 1386.4ms/step | 14.4 samples/s +Step 42110/44000 | Loss: 2.8943 | LR: 2.35e-06 | 1388.8ms/step | 14.4 samples/s +Step 42120/44000 | Loss: 2.8776 | LR: 2.33e-06 | 1389.3ms/step | 14.4 samples/s +Step 42130/44000 | Loss: 2.8785 | LR: 2.30e-06 | 1391.1ms/step | 14.4 samples/s +Step 42140/44000 | Loss: 2.8801 | LR: 2.28e-06 | 1388.6ms/step | 14.4 samples/s +Step 42150/44000 | Loss: 2.8747 | LR: 2.25e-06 | 1388.4ms/step | 14.4 samples/s +Step 42160/44000 | Loss: 2.8854 | LR: 2.23e-06 | 1389.5ms/step | 14.4 samples/s +Step 42170/44000 | Loss: 2.8680 | LR: 2.21e-06 | 1388.2ms/step | 14.4 samples/s +Step 42180/44000 | Loss: 2.8868 | LR: 2.18e-06 | 1386.1ms/step | 14.4 samples/s +Step 42190/44000 | Loss: 2.8633 | LR: 2.16e-06 | 1389.7ms/step | 14.4 samples/s +Step 42200/44000 | Loss: 2.8729 | LR: 2.13e-06 | 1385.0ms/step | 14.4 samples/s +Step 42210/44000 | Loss: 2.8999 | LR: 2.11e-06 | 1385.2ms/step | 14.4 samples/s +Step 42220/44000 | Loss: 2.8987 | LR: 2.09e-06 | 1386.1ms/step | 14.4 samples/s +Step 42230/44000 | Loss: 2.9005 | LR: 2.06e-06 | 1386.0ms/step | 14.4 samples/s +Step 42240/44000 | Loss: 2.8929 | LR: 2.04e-06 | 1387.7ms/step | 14.4 samples/s +Step 42250/44000 | Loss: 2.8801 | LR: 2.02e-06 | 1386.3ms/step | 14.4 samples/s +Step 42260/44000 | Loss: 2.8660 | LR: 1.99e-06 | 1386.2ms/step | 14.4 samples/s +Step 42270/44000 | Loss: 2.8984 | LR: 1.97e-06 | 1387.8ms/step | 14.4 samples/s +Step 42280/44000 | Loss: 2.8833 | LR: 1.95e-06 | 1390.0ms/step | 14.4 samples/s +Step 42290/44000 | Loss: 2.8826 | LR: 1.93e-06 | 1390.0ms/step | 14.4 samples/s +Step 42300/44000 | Loss: 2.8684 | LR: 1.90e-06 | 1388.2ms/step | 14.4 samples/s +Step 42310/44000 | Loss: 2.8898 | LR: 1.88e-06 | 1387.3ms/step | 14.4 samples/s +Step 42320/44000 | Loss: 2.8854 | LR: 1.86e-06 | 1387.4ms/step | 14.4 samples/s +Step 42330/44000 | Loss: 2.8773 | LR: 1.84e-06 | 1388.3ms/step | 14.4 samples/s +Step 42340/44000 | Loss: 2.8710 | LR: 1.82e-06 | 1386.8ms/step | 14.4 samples/s +Step 42350/44000 | Loss: 2.8556 | LR: 1.79e-06 | 1386.8ms/step | 14.4 samples/s +Step 42360/44000 | Loss: 2.8950 | LR: 1.77e-06 | 1387.2ms/step | 14.4 samples/s +Step 42370/44000 | Loss: 2.8795 | LR: 1.75e-06 | 1390.8ms/step | 14.4 samples/s +Step 42380/44000 | Loss: 2.8525 | LR: 1.73e-06 | 1387.3ms/step | 14.4 samples/s +Step 42390/44000 | Loss: 2.8753 | LR: 1.71e-06 | 1388.0ms/step | 14.4 samples/s +Step 42400/44000 | Loss: 2.8663 | LR: 1.69e-06 | 1388.0ms/step | 14.4 samples/s +Step 42410/44000 | Loss: 2.9046 | LR: 1.67e-06 | 1386.7ms/step | 14.4 samples/s +Step 42420/44000 | Loss: 2.8780 | LR: 1.64e-06 | 1389.2ms/step | 14.4 samples/s +Step 42430/44000 | Loss: 2.8785 | LR: 1.62e-06 | 1388.8ms/step | 14.4 samples/s +Step 42440/44000 | Loss: 2.8888 | LR: 1.60e-06 | 1387.6ms/step | 14.4 samples/s +Step 42450/44000 | Loss: 2.8987 | LR: 1.58e-06 | 1391.1ms/step | 14.4 samples/s +Step 42460/44000 | Loss: 2.8645 | LR: 1.56e-06 | 1390.4ms/step | 14.4 samples/s +Step 42470/44000 | Loss: 2.8750 | LR: 1.54e-06 | 1389.8ms/step | 14.4 samples/s +Step 42480/44000 | Loss: 2.8799 | LR: 1.52e-06 | 1386.6ms/step | 14.4 samples/s +Step 42490/44000 | Loss: 2.8737 | LR: 1.50e-06 | 1389.3ms/step | 14.4 samples/s +Step 42500/44000 | Loss: 2.8911 | LR: 1.48e-06 | 1388.7ms/step | 14.4 samples/s +Step 42510/44000 | Loss: 2.8740 | LR: 1.46e-06 | 1388.0ms/step | 14.4 samples/s +Step 42520/44000 | Loss: 2.9128 | LR: 1.44e-06 | 1388.9ms/step | 14.4 samples/s +Step 42530/44000 | Loss: 2.8719 | LR: 1.42e-06 | 1387.2ms/step | 14.4 samples/s +Step 42540/44000 | Loss: 2.8820 | LR: 1.40e-06 | 1387.6ms/step | 14.4 samples/s +Step 42550/44000 | Loss: 2.8958 | LR: 1.39e-06 | 1389.6ms/step | 14.4 samples/s +Step 42560/44000 | Loss: 2.8974 | LR: 1.37e-06 | 1387.7ms/step | 14.4 samples/s +Step 42570/44000 | Loss: 2.8554 | LR: 1.35e-06 | 1388.2ms/step | 14.4 samples/s +Step 42580/44000 | Loss: 2.8868 | LR: 1.33e-06 | 1388.6ms/step | 14.4 samples/s +Step 42590/44000 | Loss: 2.8837 | LR: 1.31e-06 | 1387.5ms/step | 14.4 samples/s +Step 42600/44000 | Loss: 2.8757 | LR: 1.29e-06 | 1388.6ms/step | 14.4 samples/s +Step 42610/44000 | Loss: 2.8542 | LR: 1.27e-06 | 1389.5ms/step | 14.4 samples/s +Step 42620/44000 | Loss: 2.9059 | LR: 1.25e-06 | 1386.2ms/step | 14.4 samples/s +Step 42630/44000 | Loss: 2.9174 | LR: 1.24e-06 | 1388.1ms/step | 14.4 samples/s +Step 42640/44000 | Loss: 2.8803 | LR: 1.22e-06 | 1386.9ms/step | 14.4 samples/s +Step 42650/44000 | Loss: 2.8589 | LR: 1.20e-06 | 1391.0ms/step | 14.4 samples/s +Step 42660/44000 | Loss: 2.8708 | LR: 1.18e-06 | 1390.2ms/step | 14.4 samples/s +Step 42670/44000 | Loss: 2.8793 | LR: 1.17e-06 | 1390.2ms/step | 14.4 samples/s +Step 42680/44000 | Loss: 2.8800 | LR: 1.15e-06 | 1388.2ms/step | 14.4 samples/s +Step 42690/44000 | Loss: 2.8791 | LR: 1.13e-06 | 1389.6ms/step | 14.4 samples/s +Step 42700/44000 | Loss: 2.8635 | LR: 1.11e-06 | 1388.2ms/step | 14.4 samples/s +Step 42710/44000 | Loss: 2.8720 | LR: 1.10e-06 | 1389.3ms/step | 14.4 samples/s +Step 42720/44000 | Loss: 2.8766 | LR: 1.08e-06 | 1386.1ms/step | 14.4 samples/s +Step 42730/44000 | Loss: 2.8677 | LR: 1.06e-06 | 1390.4ms/step | 14.4 samples/s +Step 42740/44000 | Loss: 2.8857 | LR: 1.05e-06 | 1389.1ms/step | 14.4 samples/s +Step 42750/44000 | Loss: 2.8889 | LR: 1.03e-06 | 1389.2ms/step | 14.4 samples/s +Step 42760/44000 | Loss: 2.8780 | LR: 1.01e-06 | 1388.0ms/step | 14.4 samples/s +Step 42770/44000 | Loss: 2.8978 | LR: 9.97e-07 | 1390.1ms/step | 14.4 samples/s +Step 42780/44000 | Loss: 2.8568 | LR: 9.81e-07 | 1388.6ms/step | 14.4 samples/s +Step 42790/44000 | Loss: 2.8851 | LR: 9.65e-07 | 1391.2ms/step | 14.4 samples/s +Step 42800/44000 | Loss: 2.9091 | LR: 9.49e-07 | 1388.9ms/step | 14.4 samples/s +Step 42810/44000 | Loss: 2.8809 | LR: 9.33e-07 | 1387.0ms/step | 14.4 samples/s +Step 42820/44000 | Loss: 2.8690 | LR: 9.18e-07 | 1383.4ms/step | 14.5 samples/s +Step 42830/44000 | Loss: 2.8832 | LR: 9.02e-07 | 1383.0ms/step | 14.5 samples/s +Step 42840/44000 | Loss: 2.8584 | LR: 8.87e-07 | 1383.2ms/step | 14.5 samples/s +Step 42850/44000 | Loss: 2.8673 | LR: 8.72e-07 | 1385.1ms/step | 14.4 samples/s +Step 42860/44000 | Loss: 2.9021 | LR: 8.57e-07 | 1386.3ms/step | 14.4 samples/s +Step 42870/44000 | Loss: 2.8948 | LR: 8.42e-07 | 1385.6ms/step | 14.4 samples/s +Step 42880/44000 | Loss: 2.8937 | LR: 8.27e-07 | 1385.3ms/step | 14.4 samples/s +Step 42890/44000 | Loss: 2.8815 | LR: 8.12e-07 | 1386.5ms/step | 14.4 samples/s +Step 42900/44000 | Loss: 2.8585 | LR: 7.98e-07 | 1385.3ms/step | 14.4 samples/s +Step 42910/44000 | Loss: 2.8815 | LR: 7.83e-07 | 1385.3ms/step | 14.4 samples/s +Step 42920/44000 | Loss: 2.8682 | LR: 7.69e-07 | 1383.9ms/step | 14.5 samples/s +Step 42930/44000 | Loss: 2.8821 | LR: 7.55e-07 | 1384.3ms/step | 14.4 samples/s +Step 42940/44000 | Loss: 2.9021 | LR: 7.41e-07 | 1386.3ms/step | 14.4 samples/s +Step 42950/44000 | Loss: 2.8767 | LR: 7.27e-07 | 1386.2ms/step | 14.4 samples/s +Step 42960/44000 | Loss: 2.9157 | LR: 7.13e-07 | 1387.6ms/step | 14.4 samples/s +Step 42970/44000 | Loss: 2.8679 | LR: 6.99e-07 | 1386.7ms/step | 14.4 samples/s +Step 42980/44000 | Loss: 2.8726 | LR: 6.86e-07 | 1386.3ms/step | 14.4 samples/s +Step 42990/44000 | Loss: 2.8750 | LR: 6.72e-07 | 1386.7ms/step | 14.4 samples/s +Step 43000/44000 | Loss: 2.8923 | LR: 6.59e-07 | 1387.2ms/step | 14.4 samples/s +Validation at step 43000: Loss = 2.8841, Perplexity = 17.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-43000 +Step 43010/44000 | Loss: 2.8599 | LR: 6.46e-07 | 3857.7ms/step | 5.2 samples/s +Step 43020/44000 | Loss: 2.8905 | LR: 6.33e-07 | 1385.0ms/step | 14.4 samples/s +Step 43030/44000 | Loss: 2.8832 | LR: 6.20e-07 | 1387.0ms/step | 14.4 samples/s +Step 43040/44000 | Loss: 2.8977 | LR: 6.08e-07 | 1387.1ms/step | 14.4 samples/s +Step 43050/44000 | Loss: 2.8804 | LR: 5.95e-07 | 1387.5ms/step | 14.4 samples/s +Step 43060/44000 | Loss: 2.8786 | LR: 5.83e-07 | 1385.3ms/step | 14.4 samples/s +Step 43070/44000 | Loss: 2.8951 | LR: 5.70e-07 | 1386.6ms/step | 14.4 samples/s +Step 43080/44000 | Loss: 2.8585 | LR: 5.58e-07 | 1389.7ms/step | 14.4 samples/s +Step 43090/44000 | Loss: 2.8445 | LR: 5.46e-07 | 1388.3ms/step | 14.4 samples/s +Step 43100/44000 | Loss: 2.9250 | LR: 5.34e-07 | 1387.1ms/step | 14.4 samples/s +Step 43110/44000 | Loss: 2.8864 | LR: 5.22e-07 | 1386.9ms/step | 14.4 samples/s +Step 43120/44000 | Loss: 2.8471 | LR: 5.11e-07 | 1384.2ms/step | 14.4 samples/s +Step 43130/44000 | Loss: 2.8950 | LR: 4.99e-07 | 1384.4ms/step | 14.4 samples/s +Step 43140/44000 | Loss: 2.8797 | LR: 4.88e-07 | 1383.3ms/step | 14.5 samples/s +Step 43150/44000 | Loss: 2.8720 | LR: 4.76e-07 | 1384.9ms/step | 14.4 samples/s +Step 43160/44000 | Loss: 2.8940 | LR: 4.65e-07 | 1385.0ms/step | 14.4 samples/s +Step 43170/44000 | Loss: 2.8550 | LR: 4.54e-07 | 1385.6ms/step | 14.4 samples/s +Step 43180/44000 | Loss: 2.8702 | LR: 4.43e-07 | 1386.7ms/step | 14.4 samples/s +Step 43190/44000 | Loss: 2.8819 | LR: 4.33e-07 | 1386.5ms/step | 14.4 samples/s +Step 43200/44000 | Loss: 2.9005 | LR: 4.22e-07 | 1384.7ms/step | 14.4 samples/s +Step 43210/44000 | Loss: 2.8633 | LR: 4.12e-07 | 1386.0ms/step | 14.4 samples/s +Step 43220/44000 | Loss: 2.8711 | LR: 4.01e-07 | 1382.8ms/step | 14.5 samples/s +Step 43230/44000 | Loss: 2.8892 | LR: 3.91e-07 | 1384.4ms/step | 14.4 samples/s +Step 43240/44000 | Loss: 2.8794 | LR: 3.81e-07 | 1384.0ms/step | 14.5 samples/s +Step 43250/44000 | Loss: 2.8604 | LR: 3.71e-07 | 1383.6ms/step | 14.5 samples/s +Step 43260/44000 | Loss: 2.8974 | LR: 3.61e-07 | 1383.8ms/step | 14.5 samples/s +Step 43270/44000 | Loss: 2.8671 | LR: 3.51e-07 | 1385.1ms/step | 14.4 samples/s +Step 43280/44000 | Loss: 2.8643 | LR: 3.42e-07 | 1384.3ms/step | 14.4 samples/s +Step 43290/44000 | Loss: 2.8757 | LR: 3.32e-07 | 1385.7ms/step | 14.4 samples/s +Step 43300/44000 | Loss: 2.8798 | LR: 3.23e-07 | 1385.2ms/step | 14.4 samples/s +Step 43310/44000 | Loss: 2.8951 | LR: 3.14e-07 | 1385.8ms/step | 14.4 samples/s +Step 43320/44000 | Loss: 2.8826 | LR: 3.05e-07 | 1387.7ms/step | 14.4 samples/s +Step 43330/44000 | Loss: 2.8788 | LR: 2.96e-07 | 1386.5ms/step | 14.4 samples/s +Step 43340/44000 | Loss: 2.8991 | LR: 2.87e-07 | 1386.3ms/step | 14.4 samples/s +Step 43350/44000 | Loss: 2.8785 | LR: 2.79e-07 | 1387.0ms/step | 14.4 samples/s +Step 43360/44000 | Loss: 2.8989 | LR: 2.70e-07 | 1385.6ms/step | 14.4 samples/s +Step 43370/44000 | Loss: 2.8922 | LR: 2.62e-07 | 1385.4ms/step | 14.4 samples/s +Step 43380/44000 | Loss: 2.8534 | LR: 2.53e-07 | 1387.2ms/step | 14.4 samples/s +Step 43390/44000 | Loss: 2.9130 | LR: 2.45e-07 | 1386.3ms/step | 14.4 samples/s +Step 43400/44000 | Loss: 2.9090 | LR: 2.37e-07 | 1386.4ms/step | 14.4 samples/s +Step 43410/44000 | Loss: 2.8819 | LR: 2.30e-07 | 1387.4ms/step | 14.4 samples/s +Step 43420/44000 | Loss: 2.8749 | LR: 2.22e-07 | 1384.4ms/step | 14.4 samples/s +Step 43430/44000 | Loss: 2.8479 | LR: 2.14e-07 | 1387.6ms/step | 14.4 samples/s +Step 43440/44000 | Loss: 2.8899 | LR: 2.07e-07 | 1386.4ms/step | 14.4 samples/s +Step 43450/44000 | Loss: 2.8859 | LR: 1.99e-07 | 1385.9ms/step | 14.4 samples/s +Step 43460/44000 | Loss: 2.8653 | LR: 1.92e-07 | 1386.2ms/step | 14.4 samples/s +Step 43470/44000 | Loss: 2.8981 | LR: 1.85e-07 | 1388.6ms/step | 14.4 samples/s +Step 43480/44000 | Loss: 2.8722 | LR: 1.78e-07 | 1385.3ms/step | 14.4 samples/s +Step 43490/44000 | Loss: 2.8739 | LR: 1.72e-07 | 1387.9ms/step | 14.4 samples/s +Step 43500/44000 | Loss: 2.8939 | LR: 1.65e-07 | 1385.9ms/step | 14.4 samples/s +Step 43510/44000 | Loss: 2.8697 | LR: 1.58e-07 | 1387.4ms/step | 14.4 samples/s +Step 43520/44000 | Loss: 2.8786 | LR: 1.52e-07 | 1386.4ms/step | 14.4 samples/s +Step 43530/44000 | Loss: 2.8701 | LR: 1.46e-07 | 1386.4ms/step | 14.4 samples/s +Step 43540/44000 | Loss: 2.9040 | LR: 1.40e-07 | 1383.6ms/step | 14.5 samples/s +Step 43550/44000 | Loss: 2.8995 | LR: 1.34e-07 | 1384.5ms/step | 14.4 samples/s +Step 43560/44000 | Loss: 2.8619 | LR: 1.28e-07 | 1386.1ms/step | 14.4 samples/s +Step 43570/44000 | Loss: 2.8728 | LR: 1.22e-07 | 1387.5ms/step | 14.4 samples/s +Step 43580/44000 | Loss: 2.8699 | LR: 1.16e-07 | 1386.2ms/step | 14.4 samples/s +Step 43590/44000 | Loss: 2.8608 | LR: 1.11e-07 | 1386.6ms/step | 14.4 samples/s +Step 43600/44000 | Loss: 2.8921 | LR: 1.06e-07 | 1385.7ms/step | 14.4 samples/s +Step 43610/44000 | Loss: 2.8799 | LR: 1.00e-07 | 1384.9ms/step | 14.4 samples/s +Step 43620/44000 | Loss: 2.8610 | LR: 9.52e-08 | 1384.2ms/step | 14.4 samples/s +Step 43630/44000 | Loss: 2.8784 | LR: 9.03e-08 | 1385.6ms/step | 14.4 samples/s +Step 43640/44000 | Loss: 2.8943 | LR: 8.55e-08 | 1385.4ms/step | 14.4 samples/s +Step 43650/44000 | Loss: 2.9106 | LR: 8.08e-08 | 1385.3ms/step | 14.4 samples/s +Step 43660/44000 | Loss: 2.8864 | LR: 7.62e-08 | 1386.8ms/step | 14.4 samples/s +Step 43670/44000 | Loss: 2.8694 | LR: 7.18e-08 | 1385.4ms/step | 14.4 samples/s +Step 43680/44000 | Loss: 2.8847 | LR: 6.75e-08 | 1386.4ms/step | 14.4 samples/s +Step 43690/44000 | Loss: 2.8608 | LR: 6.34e-08 | 1387.1ms/step | 14.4 samples/s +Step 43700/44000 | Loss: 2.9090 | LR: 5.94e-08 | 1385.9ms/step | 14.4 samples/s +Step 43710/44000 | Loss: 2.9060 | LR: 5.55e-08 | 1386.5ms/step | 14.4 samples/s +Step 43720/44000 | Loss: 2.8702 | LR: 5.17e-08 | 1385.7ms/step | 14.4 samples/s +Step 43730/44000 | Loss: 2.9131 | LR: 4.81e-08 | 1392.8ms/step | 14.4 samples/s +Step 43740/44000 | Loss: 2.8733 | LR: 4.46e-08 | 1396.4ms/step | 14.3 samples/s +Step 43750/44000 | Loss: 2.8877 | LR: 4.12e-08 | 1386.4ms/step | 14.4 samples/s +Step 43760/44000 | Loss: 2.8721 | LR: 3.80e-08 | 1393.6ms/step | 14.4 samples/s +Step 43770/44000 | Loss: 2.8654 | LR: 3.49e-08 | 1389.4ms/step | 14.4 samples/s +Step 43780/44000 | Loss: 2.8770 | LR: 3.19e-08 | 1383.5ms/step | 14.5 samples/s +Step 43790/44000 | Loss: 2.9103 | LR: 2.91e-08 | 1384.4ms/step | 14.4 samples/s +Step 43800/44000 | Loss: 2.8981 | LR: 2.64e-08 | 1385.3ms/step | 14.4 samples/s +Step 43810/44000 | Loss: 2.8874 | LR: 2.38e-08 | 1386.9ms/step | 14.4 samples/s +Step 43820/44000 | Loss: 2.8852 | LR: 2.14e-08 | 1388.3ms/step | 14.4 samples/s +Step 43830/44000 | Loss: 2.9064 | LR: 1.91e-08 | 1385.5ms/step | 14.4 samples/s +Step 43840/44000 | Loss: 2.8546 | LR: 1.69e-08 | 1386.4ms/step | 14.4 samples/s +Step 43850/44000 | Loss: 2.8773 | LR: 1.48e-08 | 1387.1ms/step | 14.4 samples/s +Step 43860/44000 | Loss: 2.8968 | LR: 1.29e-08 | 1385.2ms/step | 14.4 samples/s +Step 43870/44000 | Loss: 2.8962 | LR: 1.11e-08 | 1383.4ms/step | 14.5 samples/s +Step 43880/44000 | Loss: 2.9242 | LR: 9.50e-09 | 1386.1ms/step | 14.4 samples/s +Step 43890/44000 | Loss: 2.8969 | LR: 7.98e-09 | 1386.4ms/step | 14.4 samples/s +Step 43900/44000 | Loss: 2.8648 | LR: 6.60e-09 | 1384.9ms/step | 14.4 samples/s +Step 43910/44000 | Loss: 2.9104 | LR: 5.34e-09 | 1385.4ms/step | 14.4 samples/s +Step 43920/44000 | Loss: 2.8815 | LR: 4.22e-09 | 1386.0ms/step | 14.4 samples/s +Step 43930/44000 | Loss: 2.8685 | LR: 3.23e-09 | 1385.1ms/step | 14.4 samples/s +Step 43940/44000 | Loss: 2.8601 | LR: 2.37e-09 | 1384.5ms/step | 14.4 samples/s +Step 43950/44000 | Loss: 2.8677 | LR: 1.65e-09 | 1385.6ms/step | 14.4 samples/s +Step 43960/44000 | Loss: 2.8915 | LR: 1.06e-09 | 1384.5ms/step | 14.4 samples/s +Step 43970/44000 | Loss: 2.8810 | LR: 5.94e-10 | 1385.7ms/step | 14.4 samples/s +Step 43980/44000 | Loss: 2.8546 | LR: 2.64e-10 | 1385.5ms/step | 14.4 samples/s +Step 43990/44000 | Loss: 2.9087 | LR: 6.60e-11 | 1385.0ms/step | 14.4 samples/s +Step 44000/44000 | Loss: 2.8718 | LR: 0.00e+00 | 1385.7ms/step | 14.4 samples/s +Validation at step 44000: Loss = 2.8841, Perplexity = 17.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-44000 +wandb: updating run metadata +wandb: uploading config.yaml; uploading output.log; uploading wandb-summary.json +wandb: uploading config.yaml; uploading output.log +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▇▆▅▅▄▃▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▆▆▅▄▄▃▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▇█▅▃▇▅▇▅▅▅▇▆▅▅▅▂▂▁▅▂▁▂▂▃▁▃▆▅▅▆▅▆▄▅▇▅▅▆▆▄ +wandb: speed/samples_per_sec ▇▆▆▂▃▃▂▂▇▆▅▅▅▃▃▅▇▅█▇█▆▄█▂▄▆▅█▇▃▆▄▄▅▁▅▄▅▅ +wandb: speed/steps_per_sec █████████████████████████████████▁██████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.88411 +wandb: eval/perplexity 17.88759 +wandb: eval/uses_dense_attention 1 +wandb: eval/uses_sparse_attention 0 +wandb: speed/ms_per_step 1385.71849 +wandb: speed/samples_per_sec 14.43295 +wandb: speed/steps_per_sec 0.72165 +wandb: +4 ... +wandb: +wandb: 🚀 View run l8k365m_dense_ropectl_a1p0_seed46 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/zovy63wq +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260820_144758-zovy63wq/logs +Training complete! diff --git a/epfl_final/train_logs/seed47_dense_ddp5/dense.20260820T235808Z.log b/epfl_final/train_logs/seed47_dense_ddp5/dense.20260820T235808Z.log new file mode 100644 index 0000000000000000000000000000000000000000..2a62fab391a512908e528c03bb3ac1ae10ff1c15 --- /dev/null +++ b/epfl_final/train_logs/seed47_dense_ddp5/dense.20260820T235808Z.log @@ -0,0 +1,1394 @@ +W0820 23:58:10.860000 56707 site-packages/torch/distributed/run.py:851] +W0820 23:58:10.860000 56707 site-packages/torch/distributed/run.py:851] ***************************************** +W0820 23:58:10.860000 56707 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0820 23:58:10.860000 56707 site-packages/torch/distributed/run.py:851] ***************************************** +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-33000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-33000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 2, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 47, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m dense 8k seed47 ddp5", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "dense", + "multiseed", + "seed47", + "ddp5" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-33000 +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 5 +Gradient accumulation steps: 2 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 302776 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-33000... +Preserving checkpoint LR state for exact resume: current=[7.564725718274601e-05], schedule_base=[0.0005] +Resumed from checkpoint at step 33000 (epoch 0, micro_step 165000) +Resuming training from step 33000 +Resuming wandb run 89kzpaot +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_235903-89kzpaot +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 365m dense 8k seed47 ddp5 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/89kzpaot +Starting training... +Validation at step 33000: Loss = 2.8718, Perplexity = 17.67 +wandb: WARNING Tried to log to step 33000 that is less than the current step 33021. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 33001: dense (dense step interval) +Step 33010/44000 | Loss: 2.8901 | LR: 7.55e-05 | 45585.2ms/step | 0.4 samples/s +wandb: WARNING Tried to log to step 33010 that is less than the current step 33021. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 33020/44000 | Loss: 2.9065 | LR: 7.54e-05 | 1198.6ms/step | 16.7 samples/s +wandb: WARNING Tried to log to step 33020 that is less than the current step 33021. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 33030/44000 | Loss: 2.8921 | LR: 7.53e-05 | 1208.8ms/step | 16.5 samples/s +Step 33040/44000 | Loss: 2.8898 | LR: 7.51e-05 | 1188.3ms/step | 16.8 samples/s +Step 33050/44000 | Loss: 2.8832 | LR: 7.50e-05 | 1152.4ms/step | 17.4 samples/s +Step 33060/44000 | Loss: 2.9673 | LR: 7.49e-05 | 1172.0ms/step | 17.1 samples/s +Step 33070/44000 | Loss: 2.8608 | LR: 7.47e-05 | 1250.0ms/step | 16.0 samples/s +Step 33080/44000 | Loss: 2.8688 | LR: 7.46e-05 | 1214.5ms/step | 16.5 samples/s +Step 33090/44000 | Loss: 2.8920 | LR: 7.45e-05 | 1193.1ms/step | 16.8 samples/s +Step 33100/44000 | Loss: 2.8769 | LR: 7.44e-05 | 1130.9ms/step | 17.7 samples/s +Step 33110/44000 | Loss: 2.8225 | LR: 7.42e-05 | 1219.3ms/step | 16.4 samples/s +Step 33120/44000 | Loss: 2.9086 | LR: 7.41e-05 | 1208.6ms/step | 16.5 samples/s +Step 33130/44000 | Loss: 2.8441 | LR: 7.40e-05 | 1245.1ms/step | 16.1 samples/s +Step 33140/44000 | Loss: 2.9014 | LR: 7.38e-05 | 1213.9ms/step | 16.5 samples/s +Step 33150/44000 | Loss: 2.9336 | LR: 7.37e-05 | 1191.5ms/step | 16.8 samples/s +Step 33160/44000 | Loss: 2.8470 | LR: 7.36e-05 | 1248.0ms/step | 16.0 samples/s +Step 33170/44000 | Loss: 2.9122 | LR: 7.34e-05 | 1207.7ms/step | 16.6 samples/s +Step 33180/44000 | Loss: 2.8164 | LR: 7.33e-05 | 1266.3ms/step | 15.8 samples/s +Step 33190/44000 | Loss: 2.7727 | LR: 7.32e-05 | 1255.4ms/step | 15.9 samples/s +Step 33200/44000 | Loss: 2.8890 | LR: 7.31e-05 | 1224.6ms/step | 16.3 samples/s +Step 33210/44000 | Loss: 2.9031 | LR: 7.29e-05 | 1172.4ms/step | 17.1 samples/s +Step 33220/44000 | Loss: 2.9009 | LR: 7.28e-05 | 1211.1ms/step | 16.5 samples/s +Step 33230/44000 | Loss: 2.9633 | LR: 7.27e-05 | 1123.1ms/step | 17.8 samples/s +Step 33240/44000 | Loss: 2.8298 | LR: 7.26e-05 | 1193.2ms/step | 16.8 samples/s +Step 33250/44000 | Loss: 2.8398 | LR: 7.24e-05 | 1156.3ms/step | 17.3 samples/s +Validation at step 33250: Loss = 2.8716, Perplexity = 17.67 +Step 33260/44000 | Loss: 2.8834 | LR: 7.23e-05 | 1630.8ms/step | 12.3 samples/s +Step 33270/44000 | Loss: 2.9957 | LR: 7.22e-05 | 1198.1ms/step | 16.7 samples/s +Step 33280/44000 | Loss: 2.8760 | LR: 7.20e-05 | 1167.6ms/step | 17.1 samples/s +Step 33290/44000 | Loss: 2.8536 | LR: 7.19e-05 | 1191.8ms/step | 16.8 samples/s +Step 33300/44000 | Loss: 2.8785 | LR: 7.18e-05 | 1177.6ms/step | 17.0 samples/s +Step 33310/44000 | Loss: 2.8816 | LR: 7.17e-05 | 1224.1ms/step | 16.3 samples/s +Step 33320/44000 | Loss: 2.8931 | LR: 7.15e-05 | 1193.2ms/step | 16.8 samples/s +Step 33330/44000 | Loss: 2.8830 | LR: 7.14e-05 | 1099.4ms/step | 18.2 samples/s +Step 33340/44000 | Loss: 2.8949 | LR: 7.13e-05 | 1130.5ms/step | 17.7 samples/s +Step 33350/44000 | Loss: 2.8830 | LR: 7.11e-05 | 1111.2ms/step | 18.0 samples/s +Step 33360/44000 | Loss: 2.8969 | LR: 7.10e-05 | 1176.1ms/step | 17.0 samples/s +Step 33370/44000 | Loss: 2.8556 | LR: 7.09e-05 | 1225.5ms/step | 16.3 samples/s +Step 33380/44000 | Loss: 2.9224 | LR: 7.08e-05 | 1243.4ms/step | 16.1 samples/s +Step 33390/44000 | Loss: 2.7726 | LR: 7.06e-05 | 1240.3ms/step | 16.1 samples/s +Step 33400/44000 | Loss: 2.9757 | LR: 7.05e-05 | 1239.8ms/step | 16.1 samples/s +Step 33410/44000 | Loss: 2.8786 | LR: 7.04e-05 | 1187.8ms/step | 16.8 samples/s +Step 33420/44000 | Loss: 2.8852 | LR: 7.03e-05 | 1195.5ms/step | 16.7 samples/s +Step 33430/44000 | Loss: 2.8835 | LR: 7.01e-05 | 1159.4ms/step | 17.3 samples/s +Step 33440/44000 | Loss: 2.9075 | LR: 7.00e-05 | 1172.4ms/step | 17.1 samples/s +Step 33450/44000 | Loss: 2.8694 | LR: 6.99e-05 | 1146.0ms/step | 17.5 samples/s +Step 33460/44000 | Loss: 2.9338 | LR: 6.98e-05 | 1141.5ms/step | 17.5 samples/s +Step 33470/44000 | Loss: 2.8997 | LR: 6.96e-05 | 1130.4ms/step | 17.7 samples/s +Step 33480/44000 | Loss: 2.8223 | LR: 6.95e-05 | 1213.3ms/step | 16.5 samples/s +Step 33490/44000 | Loss: 2.8639 | LR: 6.94e-05 | 1177.6ms/step | 17.0 samples/s +Step 33500/44000 | Loss: 2.8454 | LR: 6.93e-05 | 1150.6ms/step | 17.4 samples/s +Validation at step 33500: Loss = 2.8712, Perplexity = 17.66 +Step 33510/44000 | Loss: 2.8871 | LR: 6.91e-05 | 1546.8ms/step | 12.9 samples/s +Step 33520/44000 | Loss: 2.9204 | LR: 6.90e-05 | 1287.0ms/step | 15.5 samples/s +Step 33530/44000 | Loss: 2.9654 | LR: 6.89e-05 | 1275.3ms/step | 15.7 samples/s +Step 33540/44000 | Loss: 2.8817 | LR: 6.88e-05 | 1245.6ms/step | 16.1 samples/s +Step 33550/44000 | Loss: 2.9550 | LR: 6.86e-05 | 1271.9ms/step | 15.7 samples/s +Step 33560/44000 | Loss: 2.9123 | LR: 6.85e-05 | 1313.0ms/step | 15.2 samples/s +Step 33570/44000 | Loss: 2.9191 | LR: 6.84e-05 | 1209.0ms/step | 16.5 samples/s +Step 33580/44000 | Loss: 2.8683 | LR: 6.83e-05 | 1161.8ms/step | 17.2 samples/s +Step 33590/44000 | Loss: 2.9797 | LR: 6.81e-05 | 1198.5ms/step | 16.7 samples/s +Step 33600/44000 | Loss: 2.8421 | LR: 6.80e-05 | 1228.7ms/step | 16.3 samples/s +Step 33610/44000 | Loss: 2.8901 | LR: 6.79e-05 | 1214.5ms/step | 16.5 samples/s +Step 33620/44000 | Loss: 2.9140 | LR: 6.78e-05 | 1172.0ms/step | 17.1 samples/s +Step 33630/44000 | Loss: 2.8423 | LR: 6.76e-05 | 1182.8ms/step | 16.9 samples/s +Step 33640/44000 | Loss: 2.8338 | LR: 6.75e-05 | 1234.2ms/step | 16.2 samples/s +Step 33650/44000 | Loss: 2.8796 | LR: 6.74e-05 | 1265.7ms/step | 15.8 samples/s +Step 33660/44000 | Loss: 2.8872 | LR: 6.73e-05 | 1255.9ms/step | 15.9 samples/s +Step 33670/44000 | Loss: 2.8492 | LR: 6.71e-05 | 1172.5ms/step | 17.1 samples/s +Step 33680/44000 | Loss: 2.9288 | LR: 6.70e-05 | 1187.9ms/step | 16.8 samples/s +Step 33690/44000 | Loss: 2.8652 | LR: 6.69e-05 | 1167.2ms/step | 17.1 samples/s +Step 33700/44000 | Loss: 2.8006 | LR: 6.68e-05 | 1129.9ms/step | 17.7 samples/s +Step 33710/44000 | Loss: 2.8842 | LR: 6.66e-05 | 1141.5ms/step | 17.5 samples/s +Step 33720/44000 | Loss: 2.8872 | LR: 6.65e-05 | 1156.3ms/step | 17.3 samples/s +Step 33730/44000 | Loss: 2.8552 | LR: 6.64e-05 | 1214.4ms/step | 16.5 samples/s +Step 33740/44000 | Loss: 2.9286 | LR: 6.63e-05 | 1172.5ms/step | 17.1 samples/s +Step 33750/44000 | Loss: 2.8694 | LR: 6.61e-05 | 1250.4ms/step | 16.0 samples/s +Validation at step 33750: Loss = 2.8707, Perplexity = 17.65 +Step 33760/44000 | Loss: 2.8498 | LR: 6.60e-05 | 1667.2ms/step | 12.0 samples/s +Step 33770/44000 | Loss: 2.9064 | LR: 6.59e-05 | 1261.1ms/step | 15.9 samples/s +Step 33780/44000 | Loss: 2.9689 | LR: 6.58e-05 | 1271.5ms/step | 15.7 samples/s +Step 33790/44000 | Loss: 2.8921 | LR: 6.57e-05 | 1250.3ms/step | 16.0 samples/s +Step 33800/44000 | Loss: 2.8879 | LR: 6.55e-05 | 1260.9ms/step | 15.9 samples/s +Step 33810/44000 | Loss: 2.8697 | LR: 6.54e-05 | 1203.1ms/step | 16.6 samples/s +Step 33820/44000 | Loss: 2.8417 | LR: 6.53e-05 | 1224.6ms/step | 16.3 samples/s +Step 33830/44000 | Loss: 2.8653 | LR: 6.52e-05 | 1211.9ms/step | 16.5 samples/s +Step 33840/44000 | Loss: 2.8817 | LR: 6.50e-05 | 1262.8ms/step | 15.8 samples/s +Step 33850/44000 | Loss: 2.8983 | LR: 6.49e-05 | 1255.3ms/step | 15.9 samples/s +Step 33860/44000 | Loss: 2.7842 | LR: 6.48e-05 | 1229.4ms/step | 16.3 samples/s +Step 33870/44000 | Loss: 2.8685 | LR: 6.47e-05 | 1203.2ms/step | 16.6 samples/s +Step 33880/44000 | Loss: 2.9163 | LR: 6.46e-05 | 1209.0ms/step | 16.5 samples/s +Step 33890/44000 | Loss: 2.8827 | LR: 6.44e-05 | 1202.6ms/step | 16.6 samples/s +Step 33900/44000 | Loss: 2.8613 | LR: 6.43e-05 | 1235.4ms/step | 16.2 samples/s +Step 33910/44000 | Loss: 2.9232 | LR: 6.42e-05 | 1229.1ms/step | 16.3 samples/s +Step 33920/44000 | Loss: 2.8079 | LR: 6.41e-05 | 1213.6ms/step | 16.5 samples/s +Step 33930/44000 | Loss: 2.8997 | LR: 6.40e-05 | 1248.1ms/step | 16.0 samples/s +Step 33940/44000 | Loss: 2.8741 | LR: 6.38e-05 | 1257.6ms/step | 15.9 samples/s +Step 33950/44000 | Loss: 2.8476 | LR: 6.37e-05 | 1213.8ms/step | 16.5 samples/s +Step 33960/44000 | Loss: 2.8121 | LR: 6.36e-05 | 1224.1ms/step | 16.3 samples/s +Step 33970/44000 | Loss: 2.8853 | LR: 6.35e-05 | 1234.5ms/step | 16.2 samples/s +Step 33980/44000 | Loss: 2.9301 | LR: 6.33e-05 | 1218.5ms/step | 16.4 samples/s +Step 33990/44000 | Loss: 2.8645 | LR: 6.32e-05 | 1213.3ms/step | 16.5 samples/s +Step 34000/44000 | Loss: 2.9056 | LR: 6.31e-05 | 1192.9ms/step | 16.8 samples/s +Validation at step 34000: Loss = 2.8704, Perplexity = 17.64 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-34000 +Step 34010/44000 | Loss: 2.7919 | LR: 6.30e-05 | 1987.4ms/step | 10.1 samples/s +Step 34020/44000 | Loss: 2.8792 | LR: 6.29e-05 | 1163.9ms/step | 17.2 samples/s +Step 34030/44000 | Loss: 2.8983 | LR: 6.27e-05 | 1197.8ms/step | 16.7 samples/s +Step 34040/44000 | Loss: 2.8291 | LR: 6.26e-05 | 1234.0ms/step | 16.2 samples/s +Step 34050/44000 | Loss: 2.9293 | LR: 6.25e-05 | 1161.7ms/step | 17.2 samples/s +Step 34060/44000 | Loss: 2.9649 | LR: 6.24e-05 | 1114.4ms/step | 17.9 samples/s +Step 34070/44000 | Loss: 2.8736 | LR: 6.23e-05 | 1249.7ms/step | 16.0 samples/s +Step 34080/44000 | Loss: 2.9025 | LR: 6.21e-05 | 1256.0ms/step | 15.9 samples/s +Step 34090/44000 | Loss: 2.9351 | LR: 6.20e-05 | 1312.1ms/step | 15.2 samples/s +Step 34100/44000 | Loss: 2.8549 | LR: 6.19e-05 | 1250.8ms/step | 16.0 samples/s +Step 34110/44000 | Loss: 2.8439 | LR: 6.18e-05 | 1244.7ms/step | 16.1 samples/s +Step 34120/44000 | Loss: 2.9094 | LR: 6.17e-05 | 1266.0ms/step | 15.8 samples/s +Step 34130/44000 | Loss: 2.9407 | LR: 6.15e-05 | 1224.4ms/step | 16.3 samples/s +Step 34140/44000 | Loss: 2.9075 | LR: 6.14e-05 | 1208.1ms/step | 16.6 samples/s +Step 34150/44000 | Loss: 2.8813 | LR: 6.13e-05 | 1156.3ms/step | 17.3 samples/s +Step 34160/44000 | Loss: 2.9016 | LR: 6.12e-05 | 1223.6ms/step | 16.3 samples/s +Step 34170/44000 | Loss: 2.8511 | LR: 6.11e-05 | 1213.1ms/step | 16.5 samples/s +Step 34180/44000 | Loss: 2.9201 | LR: 6.09e-05 | 1291.4ms/step | 15.5 samples/s +Step 34190/44000 | Loss: 2.8612 | LR: 6.08e-05 | 1203.7ms/step | 16.6 samples/s +Step 34200/44000 | Loss: 2.8927 | LR: 6.07e-05 | 1212.7ms/step | 16.5 samples/s +Step 34210/44000 | Loss: 2.9213 | LR: 6.06e-05 | 1150.8ms/step | 17.4 samples/s +Step 34220/44000 | Loss: 2.8711 | LR: 6.05e-05 | 1223.8ms/step | 16.3 samples/s +Step 34230/44000 | Loss: 2.8826 | LR: 6.04e-05 | 1312.8ms/step | 15.2 samples/s +Step 34240/44000 | Loss: 2.9158 | LR: 6.02e-05 | 1260.2ms/step | 15.9 samples/s +Step 34250/44000 | Loss: 2.8678 | LR: 6.01e-05 | 1260.1ms/step | 15.9 samples/s +Validation at step 34250: Loss = 2.8702, Perplexity = 17.64 +Step 34260/44000 | Loss: 2.9301 | LR: 6.00e-05 | 1702.9ms/step | 11.7 samples/s +Step 34270/44000 | Loss: 2.8291 | LR: 5.99e-05 | 1239.8ms/step | 16.1 samples/s +Step 34280/44000 | Loss: 2.8724 | LR: 5.98e-05 | 1187.2ms/step | 16.8 samples/s +Step 34290/44000 | Loss: 2.9208 | LR: 5.96e-05 | 1145.5ms/step | 17.5 samples/s +Step 34300/44000 | Loss: 2.8379 | LR: 5.95e-05 | 1096.2ms/step | 18.2 samples/s +Step 34310/44000 | Loss: 2.9033 | LR: 5.94e-05 | 1191.0ms/step | 16.8 samples/s +Step 34320/44000 | Loss: 2.8772 | LR: 5.93e-05 | 1160.0ms/step | 17.2 samples/s +Step 34330/44000 | Loss: 2.8822 | LR: 5.92e-05 | 1182.8ms/step | 16.9 samples/s +Step 34340/44000 | Loss: 2.9201 | LR: 5.91e-05 | 1213.1ms/step | 16.5 samples/s +Step 34350/44000 | Loss: 2.8370 | LR: 5.89e-05 | 1167.1ms/step | 17.1 samples/s +Step 34360/44000 | Loss: 2.8849 | LR: 5.88e-05 | 1254.9ms/step | 15.9 samples/s +Step 34370/44000 | Loss: 2.9414 | LR: 5.87e-05 | 1224.2ms/step | 16.3 samples/s +Step 34380/44000 | Loss: 2.8387 | LR: 5.86e-05 | 1182.9ms/step | 16.9 samples/s +Step 34390/44000 | Loss: 2.9123 | LR: 5.85e-05 | 1228.9ms/step | 16.3 samples/s +Step 34400/44000 | Loss: 2.9505 | LR: 5.84e-05 | 1265.3ms/step | 15.8 samples/s +Step 34410/44000 | Loss: 2.9604 | LR: 5.82e-05 | 1260.2ms/step | 15.9 samples/s +Step 34420/44000 | Loss: 2.8947 | LR: 5.81e-05 | 1140.7ms/step | 17.5 samples/s +Step 34430/44000 | Loss: 2.8887 | LR: 5.80e-05 | 1192.3ms/step | 16.8 samples/s +Step 34440/44000 | Loss: 2.7924 | LR: 5.79e-05 | 1161.6ms/step | 17.2 samples/s +Step 34450/44000 | Loss: 2.8869 | LR: 5.78e-05 | 1223.7ms/step | 16.3 samples/s +Step 34460/44000 | Loss: 2.8155 | LR: 5.77e-05 | 1187.6ms/step | 16.8 samples/s +Step 34470/44000 | Loss: 2.8640 | LR: 5.75e-05 | 1181.6ms/step | 16.9 samples/s +Step 34480/44000 | Loss: 2.9339 | LR: 5.74e-05 | 1167.2ms/step | 17.1 samples/s +Step 34490/44000 | Loss: 2.8884 | LR: 5.73e-05 | 1255.2ms/step | 15.9 samples/s +Step 34500/44000 | Loss: 2.8760 | LR: 5.72e-05 | 1203.6ms/step | 16.6 samples/s +Validation at step 34500: Loss = 2.8699, Perplexity = 17.64 +Step 34510/44000 | Loss: 2.8633 | LR: 5.71e-05 | 1630.7ms/step | 12.3 samples/s +Step 34520/44000 | Loss: 2.8325 | LR: 5.70e-05 | 1192.8ms/step | 16.8 samples/s +Step 34530/44000 | Loss: 2.8970 | LR: 5.69e-05 | 1172.0ms/step | 17.1 samples/s +Step 34540/44000 | Loss: 2.8728 | LR: 5.67e-05 | 1152.9ms/step | 17.3 samples/s +Step 34550/44000 | Loss: 2.8617 | LR: 5.66e-05 | 1133.4ms/step | 17.6 samples/s +Step 34560/44000 | Loss: 2.9208 | LR: 5.65e-05 | 1213.0ms/step | 16.5 samples/s +Step 34570/44000 | Loss: 2.8224 | LR: 5.64e-05 | 1205.1ms/step | 16.6 samples/s +Step 34580/44000 | Loss: 2.8889 | LR: 5.63e-05 | 1175.4ms/step | 17.0 samples/s +Step 34590/44000 | Loss: 2.8932 | LR: 5.62e-05 | 1270.6ms/step | 15.7 samples/s +Step 34600/44000 | Loss: 2.9484 | LR: 5.60e-05 | 1187.5ms/step | 16.8 samples/s +Step 34610/44000 | Loss: 2.8316 | LR: 5.59e-05 | 1202.8ms/step | 16.6 samples/s +Step 34620/44000 | Loss: 2.9160 | LR: 5.58e-05 | 1239.4ms/step | 16.1 samples/s +Step 34630/44000 | Loss: 2.8662 | LR: 5.57e-05 | 1282.3ms/step | 15.6 samples/s +Step 34640/44000 | Loss: 2.8983 | LR: 5.56e-05 | 1285.3ms/step | 15.6 samples/s +Step 34650/44000 | Loss: 2.8990 | LR: 5.55e-05 | 1203.2ms/step | 16.6 samples/s +Step 34660/44000 | Loss: 2.8626 | LR: 5.54e-05 | 1171.2ms/step | 17.1 samples/s +Step 34670/44000 | Loss: 2.8326 | LR: 5.52e-05 | 1235.1ms/step | 16.2 samples/s +Step 34680/44000 | Loss: 2.8637 | LR: 5.51e-05 | 1202.8ms/step | 16.6 samples/s +Step 34690/44000 | Loss: 2.9251 | LR: 5.50e-05 | 1244.3ms/step | 16.1 samples/s +Step 34700/44000 | Loss: 2.8873 | LR: 5.49e-05 | 1234.5ms/step | 16.2 samples/s +Step 34710/44000 | Loss: 2.9480 | LR: 5.48e-05 | 1213.9ms/step | 16.5 samples/s +Step 34720/44000 | Loss: 2.8337 | LR: 5.47e-05 | 1219.4ms/step | 16.4 samples/s +Step 34730/44000 | Loss: 2.8423 | LR: 5.46e-05 | 1088.4ms/step | 18.4 samples/s +Step 34740/44000 | Loss: 2.9110 | LR: 5.45e-05 | 1146.1ms/step | 17.5 samples/s +Step 34750/44000 | Loss: 2.9196 | LR: 5.43e-05 | 1202.6ms/step | 16.6 samples/s +Validation at step 34750: Loss = 2.8696, Perplexity = 17.63 +Step 34760/44000 | Loss: 2.8006 | LR: 5.42e-05 | 1584.3ms/step | 12.6 samples/s +Step 34770/44000 | Loss: 2.9754 | LR: 5.41e-05 | 1281.0ms/step | 15.6 samples/s +Step 34780/44000 | Loss: 2.8995 | LR: 5.40e-05 | 1240.1ms/step | 16.1 samples/s +Step 34790/44000 | Loss: 2.9917 | LR: 5.39e-05 | 1135.4ms/step | 17.6 samples/s +Step 34800/44000 | Loss: 2.8492 | LR: 5.38e-05 | 1099.2ms/step | 18.2 samples/s +Step 34810/44000 | Loss: 2.8759 | LR: 5.37e-05 | 1135.9ms/step | 17.6 samples/s +Step 34820/44000 | Loss: 2.9234 | LR: 5.36e-05 | 1161.2ms/step | 17.2 samples/s +Step 34830/44000 | Loss: 2.9049 | LR: 5.34e-05 | 1169.2ms/step | 17.1 samples/s +Step 34840/44000 | Loss: 2.9113 | LR: 5.33e-05 | 1190.3ms/step | 16.8 samples/s +Step 34850/44000 | Loss: 2.8206 | LR: 5.32e-05 | 1182.3ms/step | 16.9 samples/s +Step 34860/44000 | Loss: 2.8432 | LR: 5.31e-05 | 1234.7ms/step | 16.2 samples/s +Step 34870/44000 | Loss: 2.8563 | LR: 5.30e-05 | 1177.2ms/step | 17.0 samples/s +Step 34880/44000 | Loss: 2.8784 | LR: 5.29e-05 | 1135.6ms/step | 17.6 samples/s +Step 34890/44000 | Loss: 2.9286 | LR: 5.28e-05 | 1192.6ms/step | 16.8 samples/s +Step 34900/44000 | Loss: 2.8150 | LR: 5.27e-05 | 1229.7ms/step | 16.3 samples/s +Step 34910/44000 | Loss: 2.8699 | LR: 5.25e-05 | 1229.7ms/step | 16.3 samples/s +Step 34920/44000 | Loss: 2.9110 | LR: 5.24e-05 | 1166.5ms/step | 17.1 samples/s +Step 34930/44000 | Loss: 2.8946 | LR: 5.23e-05 | 1110.0ms/step | 18.0 samples/s +Step 34940/44000 | Loss: 2.8601 | LR: 5.22e-05 | 1151.1ms/step | 17.4 samples/s +Step 34950/44000 | Loss: 2.8702 | LR: 5.21e-05 | 1234.4ms/step | 16.2 samples/s +Step 34960/44000 | Loss: 2.8700 | LR: 5.20e-05 | 1265.2ms/step | 15.8 samples/s +Step 34970/44000 | Loss: 2.8793 | LR: 5.19e-05 | 1235.0ms/step | 16.2 samples/s +Step 34980/44000 | Loss: 2.8696 | LR: 5.18e-05 | 1114.7ms/step | 17.9 samples/s +Step 34990/44000 | Loss: 2.9471 | LR: 5.17e-05 | 1109.1ms/step | 18.0 samples/s +Step 35000/44000 | Loss: 2.8637 | LR: 5.15e-05 | 1234.2ms/step | 16.2 samples/s +Validation at step 35000: Loss = 2.8693, Perplexity = 17.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-35000 +Step 35010/44000 | Loss: 2.8954 | LR: 5.14e-05 | 2062.5ms/step | 9.7 samples/s +Step 35020/44000 | Loss: 2.8657 | LR: 5.13e-05 | 1264.9ms/step | 15.8 samples/s +Step 35030/44000 | Loss: 2.9311 | LR: 5.12e-05 | 1270.6ms/step | 15.7 samples/s +Step 35040/44000 | Loss: 2.8671 | LR: 5.11e-05 | 1233.5ms/step | 16.2 samples/s +Step 35050/44000 | Loss: 2.8778 | LR: 5.10e-05 | 1222.3ms/step | 16.4 samples/s +Step 35060/44000 | Loss: 2.8690 | LR: 5.09e-05 | 1205.7ms/step | 16.6 samples/s +Step 35070/44000 | Loss: 2.8576 | LR: 5.08e-05 | 1109.0ms/step | 18.0 samples/s +Step 35080/44000 | Loss: 2.8663 | LR: 5.07e-05 | 1177.3ms/step | 17.0 samples/s +Step 35090/44000 | Loss: 2.9352 | LR: 5.06e-05 | 1116.1ms/step | 17.9 samples/s +Step 35100/44000 | Loss: 2.9015 | LR: 5.04e-05 | 1175.6ms/step | 17.0 samples/s +Step 35110/44000 | Loss: 2.8536 | LR: 5.03e-05 | 1130.5ms/step | 17.7 samples/s +Step 35120/44000 | Loss: 2.8387 | LR: 5.02e-05 | 1161.6ms/step | 17.2 samples/s +Step 35130/44000 | Loss: 2.9110 | LR: 5.01e-05 | 1229.1ms/step | 16.3 samples/s +Step 35140/44000 | Loss: 2.8732 | LR: 5.00e-05 | 1213.5ms/step | 16.5 samples/s +Step 35150/44000 | Loss: 2.8741 | LR: 4.99e-05 | 1245.3ms/step | 16.1 samples/s +Step 35160/44000 | Loss: 2.8496 | LR: 4.98e-05 | 1218.6ms/step | 16.4 samples/s +Step 35170/44000 | Loss: 2.9018 | LR: 4.97e-05 | 1286.4ms/step | 15.5 samples/s +Step 35180/44000 | Loss: 2.8854 | LR: 4.96e-05 | 1255.3ms/step | 15.9 samples/s +Step 35190/44000 | Loss: 2.8593 | LR: 4.95e-05 | 1250.0ms/step | 16.0 samples/s +Step 35200/44000 | Loss: 2.9883 | LR: 4.94e-05 | 1220.6ms/step | 16.4 samples/s +Step 35210/44000 | Loss: 2.8451 | LR: 4.93e-05 | 1228.6ms/step | 16.3 samples/s +Step 35220/44000 | Loss: 2.9162 | LR: 4.91e-05 | 1265.5ms/step | 15.8 samples/s +Step 35230/44000 | Loss: 2.9014 | LR: 4.90e-05 | 1214.5ms/step | 16.5 samples/s +Step 35240/44000 | Loss: 2.9622 | LR: 4.89e-05 | 1217.9ms/step | 16.4 samples/s +Step 35250/44000 | Loss: 2.8697 | LR: 4.88e-05 | 1182.4ms/step | 16.9 samples/s +Validation at step 35250: Loss = 2.8692, Perplexity = 17.62 +Step 35260/44000 | Loss: 2.8878 | LR: 4.87e-05 | 1559.5ms/step | 12.8 samples/s +Step 35270/44000 | Loss: 2.9434 | LR: 4.86e-05 | 1195.9ms/step | 16.7 samples/s +Step 35280/44000 | Loss: 2.8478 | LR: 4.85e-05 | 1187.8ms/step | 16.8 samples/s +Step 35290/44000 | Loss: 2.8901 | LR: 4.84e-05 | 1250.2ms/step | 16.0 samples/s +Step 35300/44000 | Loss: 2.8745 | LR: 4.83e-05 | 1219.3ms/step | 16.4 samples/s +Step 35310/44000 | Loss: 2.8916 | LR: 4.82e-05 | 1234.7ms/step | 16.2 samples/s +Step 35320/44000 | Loss: 2.8477 | LR: 4.81e-05 | 1244.4ms/step | 16.1 samples/s +Step 35330/44000 | Loss: 2.8586 | LR: 4.80e-05 | 1221.6ms/step | 16.4 samples/s +Step 35340/44000 | Loss: 2.8400 | LR: 4.79e-05 | 1221.4ms/step | 16.4 samples/s +Step 35350/44000 | Loss: 2.8951 | LR: 4.77e-05 | 1156.3ms/step | 17.3 samples/s +Step 35360/44000 | Loss: 2.9019 | LR: 4.76e-05 | 1266.0ms/step | 15.8 samples/s +Step 35370/44000 | Loss: 2.8906 | LR: 4.75e-05 | 1208.2ms/step | 16.6 samples/s +Step 35380/44000 | Loss: 2.9137 | LR: 4.74e-05 | 1208.5ms/step | 16.5 samples/s +Step 35390/44000 | Loss: 2.9833 | LR: 4.73e-05 | 1193.2ms/step | 16.8 samples/s +Step 35400/44000 | Loss: 2.9042 | LR: 4.72e-05 | 1182.7ms/step | 16.9 samples/s +Step 35410/44000 | Loss: 2.8898 | LR: 4.71e-05 | 1130.3ms/step | 17.7 samples/s +Step 35420/44000 | Loss: 2.8733 | LR: 4.70e-05 | 1104.5ms/step | 18.1 samples/s +Step 35430/44000 | Loss: 2.8045 | LR: 4.69e-05 | 1078.5ms/step | 18.5 samples/s +Step 35440/44000 | Loss: 2.9125 | LR: 4.68e-05 | 1156.7ms/step | 17.3 samples/s +Step 35450/44000 | Loss: 2.9155 | LR: 4.67e-05 | 1208.5ms/step | 16.5 samples/s +Step 35460/44000 | Loss: 2.9274 | LR: 4.66e-05 | 1172.1ms/step | 17.1 samples/s +Step 35470/44000 | Loss: 2.8408 | LR: 4.65e-05 | 1187.8ms/step | 16.8 samples/s +Step 35480/44000 | Loss: 2.8819 | LR: 4.64e-05 | 1270.6ms/step | 15.7 samples/s +Step 35490/44000 | Loss: 2.9141 | LR: 4.63e-05 | 1229.7ms/step | 16.3 samples/s +Step 35500/44000 | Loss: 2.8513 | LR: 4.62e-05 | 1250.8ms/step | 16.0 samples/s +Validation at step 35500: Loss = 2.8689, Perplexity = 17.62 +Step 35510/44000 | Loss: 2.8660 | LR: 4.61e-05 | 1750.2ms/step | 11.4 samples/s +Step 35520/44000 | Loss: 2.8831 | LR: 4.59e-05 | 1333.4ms/step | 15.0 samples/s +Step 35530/44000 | Loss: 2.9105 | LR: 4.58e-05 | 1240.6ms/step | 16.1 samples/s +Step 35540/44000 | Loss: 2.8648 | LR: 4.57e-05 | 1151.9ms/step | 17.4 samples/s +Step 35550/44000 | Loss: 2.9154 | LR: 4.56e-05 | 1119.9ms/step | 17.9 samples/s +Step 35560/44000 | Loss: 2.7834 | LR: 4.55e-05 | 1145.7ms/step | 17.5 samples/s +Step 35570/44000 | Loss: 2.9365 | LR: 4.54e-05 | 1198.4ms/step | 16.7 samples/s +Step 35580/44000 | Loss: 2.8519 | LR: 4.53e-05 | 1171.7ms/step | 17.1 samples/s +Step 35590/44000 | Loss: 2.8967 | LR: 4.52e-05 | 1167.5ms/step | 17.1 samples/s +Step 35600/44000 | Loss: 2.9091 | LR: 4.51e-05 | 1203.5ms/step | 16.6 samples/s +Step 35610/44000 | Loss: 2.9106 | LR: 4.50e-05 | 1172.3ms/step | 17.1 samples/s +Step 35620/44000 | Loss: 2.8761 | LR: 4.49e-05 | 1150.8ms/step | 17.4 samples/s +Step 35630/44000 | Loss: 2.9273 | LR: 4.48e-05 | 1197.7ms/step | 16.7 samples/s +Step 35640/44000 | Loss: 2.9085 | LR: 4.47e-05 | 1233.0ms/step | 16.2 samples/s +Step 35650/44000 | Loss: 2.9285 | LR: 4.46e-05 | 1262.3ms/step | 15.8 samples/s +Step 35660/44000 | Loss: 2.8626 | LR: 4.45e-05 | 1225.0ms/step | 16.3 samples/s +Step 35670/44000 | Loss: 2.8530 | LR: 4.44e-05 | 1271.2ms/step | 15.7 samples/s +Step 35680/44000 | Loss: 2.8642 | LR: 4.43e-05 | 1213.8ms/step | 16.5 samples/s +Step 35690/44000 | Loss: 2.8508 | LR: 4.42e-05 | 1208.0ms/step | 16.6 samples/s +Step 35700/44000 | Loss: 2.8945 | LR: 4.41e-05 | 1252.8ms/step | 16.0 samples/s +Step 35710/44000 | Loss: 2.8777 | LR: 4.40e-05 | 1206.2ms/step | 16.6 samples/s +Step 35720/44000 | Loss: 2.8629 | LR: 4.39e-05 | 1157.3ms/step | 17.3 samples/s +Step 35730/44000 | Loss: 2.9572 | LR: 4.38e-05 | 1218.2ms/step | 16.4 samples/s +Step 35740/44000 | Loss: 2.9083 | LR: 4.37e-05 | 1260.7ms/step | 15.9 samples/s +Step 35750/44000 | Loss: 2.9138 | LR: 4.36e-05 | 1255.4ms/step | 15.9 samples/s +Validation at step 35750: Loss = 2.8688, Perplexity = 17.62 +Step 35760/44000 | Loss: 2.8137 | LR: 4.35e-05 | 1651.3ms/step | 12.1 samples/s +Step 35770/44000 | Loss: 2.8566 | LR: 4.34e-05 | 1151.3ms/step | 17.4 samples/s +Step 35780/44000 | Loss: 2.9019 | LR: 4.33e-05 | 1177.6ms/step | 17.0 samples/s +Step 35790/44000 | Loss: 2.9136 | LR: 4.32e-05 | 1168.9ms/step | 17.1 samples/s +Step 35800/44000 | Loss: 2.8961 | LR: 4.31e-05 | 1153.4ms/step | 17.3 samples/s +Step 35810/44000 | Loss: 2.9199 | LR: 4.29e-05 | 1147.2ms/step | 17.4 samples/s +Step 35820/44000 | Loss: 2.9205 | LR: 4.28e-05 | 1124.4ms/step | 17.8 samples/s +Step 35830/44000 | Loss: 2.8484 | LR: 4.27e-05 | 1115.1ms/step | 17.9 samples/s +Step 35840/44000 | Loss: 2.9365 | LR: 4.26e-05 | 1198.0ms/step | 16.7 samples/s +Step 35850/44000 | Loss: 2.9068 | LR: 4.25e-05 | 1177.6ms/step | 17.0 samples/s +Step 35860/44000 | Loss: 2.9831 | LR: 4.24e-05 | 1188.0ms/step | 16.8 samples/s +Step 35870/44000 | Loss: 2.8459 | LR: 4.23e-05 | 1229.3ms/step | 16.3 samples/s +Step 35880/44000 | Loss: 2.8938 | LR: 4.22e-05 | 1221.5ms/step | 16.4 samples/s +Step 35890/44000 | Loss: 2.8743 | LR: 4.21e-05 | 1274.2ms/step | 15.7 samples/s +Step 35900/44000 | Loss: 2.8441 | LR: 4.20e-05 | 1224.4ms/step | 16.3 samples/s +Step 35910/44000 | Loss: 2.7833 | LR: 4.19e-05 | 1214.1ms/step | 16.5 samples/s +Step 35920/44000 | Loss: 2.8622 | LR: 4.18e-05 | 1177.4ms/step | 17.0 samples/s +Step 35930/44000 | Loss: 2.9701 | LR: 4.17e-05 | 1208.7ms/step | 16.5 samples/s +Step 35940/44000 | Loss: 2.9656 | LR: 4.16e-05 | 1161.9ms/step | 17.2 samples/s +Step 35950/44000 | Loss: 2.8926 | LR: 4.15e-05 | 1203.6ms/step | 16.6 samples/s +Step 35960/44000 | Loss: 2.8838 | LR: 4.14e-05 | 1153.6ms/step | 17.3 samples/s +Step 35970/44000 | Loss: 2.8743 | LR: 4.13e-05 | 1144.3ms/step | 17.5 samples/s +Step 35980/44000 | Loss: 2.9152 | LR: 4.12e-05 | 1166.8ms/step | 17.1 samples/s +Step 35990/44000 | Loss: 2.8313 | LR: 4.11e-05 | 1135.4ms/step | 17.6 samples/s +Step 36000/44000 | Loss: 2.8793 | LR: 4.10e-05 | 1172.3ms/step | 17.1 samples/s +Validation at step 36000: Loss = 2.8688, Perplexity = 17.62 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-36000 +Step 36010/44000 | Loss: 2.8726 | LR: 4.09e-05 | 2058.0ms/step | 9.7 samples/s +Step 36020/44000 | Loss: 2.8855 | LR: 4.08e-05 | 1146.5ms/step | 17.4 samples/s +Step 36030/44000 | Loss: 2.8977 | LR: 4.07e-05 | 1140.7ms/step | 17.5 samples/s +Step 36040/44000 | Loss: 2.8906 | LR: 4.06e-05 | 1135.9ms/step | 17.6 samples/s +Step 36050/44000 | Loss: 2.8968 | LR: 4.05e-05 | 1178.0ms/step | 17.0 samples/s +Step 36060/44000 | Loss: 2.8890 | LR: 4.04e-05 | 1202.9ms/step | 16.6 samples/s +Step 36070/44000 | Loss: 2.8061 | LR: 4.03e-05 | 1266.6ms/step | 15.8 samples/s +Step 36080/44000 | Loss: 2.8319 | LR: 4.02e-05 | 1217.2ms/step | 16.4 samples/s +Step 36090/44000 | Loss: 2.9970 | LR: 4.01e-05 | 1179.6ms/step | 17.0 samples/s +Step 36100/44000 | Loss: 2.8838 | LR: 4.00e-05 | 1229.2ms/step | 16.3 samples/s +Step 36110/44000 | Loss: 2.8070 | LR: 3.99e-05 | 1184.3ms/step | 16.9 samples/s +Step 36120/44000 | Loss: 2.8751 | LR: 3.98e-05 | 1170.6ms/step | 17.1 samples/s +Step 36130/44000 | Loss: 2.8590 | LR: 3.97e-05 | 1130.6ms/step | 17.7 samples/s +Step 36140/44000 | Loss: 2.9218 | LR: 3.97e-05 | 1187.5ms/step | 16.8 samples/s +Step 36150/44000 | Loss: 2.8424 | LR: 3.96e-05 | 1182.8ms/step | 16.9 samples/s +Step 36160/44000 | Loss: 2.8568 | LR: 3.95e-05 | 1161.8ms/step | 17.2 samples/s +Step 36170/44000 | Loss: 2.8680 | LR: 3.94e-05 | 1156.7ms/step | 17.3 samples/s +Step 36180/44000 | Loss: 2.9111 | LR: 3.93e-05 | 1224.4ms/step | 16.3 samples/s +Step 36190/44000 | Loss: 2.8879 | LR: 3.92e-05 | 1208.2ms/step | 16.6 samples/s +Step 36200/44000 | Loss: 2.9051 | LR: 3.91e-05 | 1279.1ms/step | 15.6 samples/s +Step 36210/44000 | Loss: 2.9228 | LR: 3.90e-05 | 1216.6ms/step | 16.4 samples/s +Step 36220/44000 | Loss: 2.9183 | LR: 3.89e-05 | 1208.0ms/step | 16.6 samples/s +Step 36230/44000 | Loss: 2.9337 | LR: 3.88e-05 | 1121.2ms/step | 17.8 samples/s +Step 36240/44000 | Loss: 2.8224 | LR: 3.87e-05 | 1227.8ms/step | 16.3 samples/s +Step 36250/44000 | Loss: 2.8865 | LR: 3.86e-05 | 1291.4ms/step | 15.5 samples/s +Validation at step 36250: Loss = 2.8686, Perplexity = 17.61 +Step 36260/44000 | Loss: 2.8846 | LR: 3.85e-05 | 1714.8ms/step | 11.7 samples/s +Step 36270/44000 | Loss: 2.9455 | LR: 3.84e-05 | 1166.5ms/step | 17.1 samples/s +Step 36280/44000 | Loss: 2.8622 | LR: 3.83e-05 | 1245.6ms/step | 16.1 samples/s +Step 36290/44000 | Loss: 2.9297 | LR: 3.82e-05 | 1250.7ms/step | 16.0 samples/s +Step 36300/44000 | Loss: 2.8817 | LR: 3.81e-05 | 1235.0ms/step | 16.2 samples/s +Step 36310/44000 | Loss: 2.8518 | LR: 3.80e-05 | 1193.7ms/step | 16.8 samples/s +Step 36320/44000 | Loss: 2.9008 | LR: 3.79e-05 | 1166.7ms/step | 17.1 samples/s +Step 36330/44000 | Loss: 2.9207 | LR: 3.78e-05 | 1214.1ms/step | 16.5 samples/s +Step 36340/44000 | Loss: 2.8767 | LR: 3.77e-05 | 1177.3ms/step | 17.0 samples/s +Step 36350/44000 | Loss: 2.9093 | LR: 3.76e-05 | 1206.5ms/step | 16.6 samples/s +Step 36360/44000 | Loss: 2.9049 | LR: 3.75e-05 | 1197.8ms/step | 16.7 samples/s +Step 36370/44000 | Loss: 2.8831 | LR: 3.74e-05 | 1179.9ms/step | 17.0 samples/s +Step 36380/44000 | Loss: 2.9115 | LR: 3.73e-05 | 1255.2ms/step | 15.9 samples/s +Step 36390/44000 | Loss: 2.9214 | LR: 3.72e-05 | 1193.8ms/step | 16.8 samples/s +Step 36400/44000 | Loss: 2.9160 | LR: 3.71e-05 | 1150.6ms/step | 17.4 samples/s +Step 36410/44000 | Loss: 2.8966 | LR: 3.70e-05 | 1146.5ms/step | 17.4 samples/s +Step 36420/44000 | Loss: 2.9001 | LR: 3.69e-05 | 1135.9ms/step | 17.6 samples/s +Step 36430/44000 | Loss: 2.9686 | LR: 3.69e-05 | 1143.8ms/step | 17.5 samples/s +Step 36440/44000 | Loss: 2.9050 | LR: 3.68e-05 | 1241.7ms/step | 16.1 samples/s +Step 36450/44000 | Loss: 2.8694 | LR: 3.67e-05 | 1195.9ms/step | 16.7 samples/s +Step 36460/44000 | Loss: 2.9287 | LR: 3.66e-05 | 1159.0ms/step | 17.3 samples/s +Step 36470/44000 | Loss: 2.8901 | LR: 3.65e-05 | 1135.9ms/step | 17.6 samples/s +Step 36480/44000 | Loss: 2.9756 | LR: 3.64e-05 | 1182.6ms/step | 16.9 samples/s +Step 36490/44000 | Loss: 2.8486 | LR: 3.63e-05 | 1161.8ms/step | 17.2 samples/s +Step 36500/44000 | Loss: 2.9230 | LR: 3.62e-05 | 1187.3ms/step | 16.8 samples/s +Validation at step 36500: Loss = 2.8684, Perplexity = 17.61 +Step 36510/44000 | Loss: 2.9019 | LR: 3.61e-05 | 1582.0ms/step | 12.6 samples/s +Step 36520/44000 | Loss: 2.8433 | LR: 3.60e-05 | 1194.0ms/step | 16.8 samples/s +Step 36530/44000 | Loss: 2.8814 | LR: 3.59e-05 | 1156.8ms/step | 17.3 samples/s +Step 36540/44000 | Loss: 2.8796 | LR: 3.58e-05 | 1213.7ms/step | 16.5 samples/s +Step 36550/44000 | Loss: 2.8865 | LR: 3.57e-05 | 1146.2ms/step | 17.4 samples/s +Step 36560/44000 | Loss: 2.8749 | LR: 3.56e-05 | 1172.7ms/step | 17.1 samples/s +Step 36570/44000 | Loss: 2.9264 | LR: 3.55e-05 | 1193.2ms/step | 16.8 samples/s +Step 36580/44000 | Loss: 2.8940 | LR: 3.54e-05 | 1208.2ms/step | 16.6 samples/s +Step 36590/44000 | Loss: 2.9320 | LR: 3.53e-05 | 1178.9ms/step | 17.0 samples/s +Step 36600/44000 | Loss: 2.8499 | LR: 3.53e-05 | 1176.0ms/step | 17.0 samples/s +Step 36610/44000 | Loss: 2.8902 | LR: 3.52e-05 | 1203.3ms/step | 16.6 samples/s +Step 36620/44000 | Loss: 2.8155 | LR: 3.51e-05 | 1224.0ms/step | 16.3 samples/s +Step 36630/44000 | Loss: 2.8817 | LR: 3.50e-05 | 1339.2ms/step | 14.9 samples/s +Step 36640/44000 | Loss: 2.9056 | LR: 3.49e-05 | 1250.0ms/step | 16.0 samples/s +Step 36650/44000 | Loss: 2.9058 | LR: 3.48e-05 | 1245.2ms/step | 16.1 samples/s +Step 36660/44000 | Loss: 2.9490 | LR: 3.47e-05 | 1198.0ms/step | 16.7 samples/s +Step 36670/44000 | Loss: 2.9613 | LR: 3.46e-05 | 1198.5ms/step | 16.7 samples/s +Step 36680/44000 | Loss: 2.8980 | LR: 3.45e-05 | 1167.3ms/step | 17.1 samples/s +Step 36690/44000 | Loss: 2.9225 | LR: 3.44e-05 | 1208.6ms/step | 16.5 samples/s +Step 36700/44000 | Loss: 2.9236 | LR: 3.43e-05 | 1208.5ms/step | 16.5 samples/s +Step 36710/44000 | Loss: 2.9081 | LR: 3.42e-05 | 1202.8ms/step | 16.6 samples/s +Step 36720/44000 | Loss: 2.8682 | LR: 3.41e-05 | 1161.1ms/step | 17.2 samples/s +Step 36730/44000 | Loss: 2.8849 | LR: 3.41e-05 | 1182.3ms/step | 16.9 samples/s +Step 36740/44000 | Loss: 2.8779 | LR: 3.40e-05 | 1156.7ms/step | 17.3 samples/s +Step 36750/44000 | Loss: 2.8673 | LR: 3.39e-05 | 1182.5ms/step | 16.9 samples/s +Validation at step 36750: Loss = 2.8682, Perplexity = 17.61 +Step 36760/44000 | Loss: 2.9273 | LR: 3.38e-05 | 1619.5ms/step | 12.3 samples/s +Step 36770/44000 | Loss: 2.9052 | LR: 3.37e-05 | 1146.5ms/step | 17.4 samples/s +Step 36780/44000 | Loss: 2.8640 | LR: 3.36e-05 | 1210.1ms/step | 16.5 samples/s +Step 36790/44000 | Loss: 2.8102 | LR: 3.35e-05 | 1222.7ms/step | 16.4 samples/s +Step 36800/44000 | Loss: 2.8539 | LR: 3.34e-05 | 1166.5ms/step | 17.1 samples/s +Step 36810/44000 | Loss: 2.8310 | LR: 3.33e-05 | 1271.6ms/step | 15.7 samples/s +Step 36820/44000 | Loss: 2.9012 | LR: 3.32e-05 | 1193.2ms/step | 16.8 samples/s +Step 36830/44000 | Loss: 2.8649 | LR: 3.31e-05 | 1208.6ms/step | 16.5 samples/s +Step 36840/44000 | Loss: 2.8613 | LR: 3.31e-05 | 1234.8ms/step | 16.2 samples/s +Step 36850/44000 | Loss: 2.8924 | LR: 3.30e-05 | 1245.1ms/step | 16.1 samples/s +Step 36860/44000 | Loss: 2.9228 | LR: 3.29e-05 | 1244.8ms/step | 16.1 samples/s +Step 36870/44000 | Loss: 2.9186 | LR: 3.28e-05 | 1192.4ms/step | 16.8 samples/s +Step 36880/44000 | Loss: 2.8855 | LR: 3.27e-05 | 1182.8ms/step | 16.9 samples/s +Step 36890/44000 | Loss: 2.7807 | LR: 3.26e-05 | 1163.9ms/step | 17.2 samples/s +Step 36900/44000 | Loss: 2.8585 | LR: 3.25e-05 | 1164.8ms/step | 17.2 samples/s +Step 36910/44000 | Loss: 2.9025 | LR: 3.24e-05 | 1163.0ms/step | 17.2 samples/s +Step 36920/44000 | Loss: 2.8149 | LR: 3.23e-05 | 1223.0ms/step | 16.4 samples/s +Step 36930/44000 | Loss: 2.8627 | LR: 3.22e-05 | 1229.7ms/step | 16.3 samples/s +Step 36940/44000 | Loss: 2.8824 | LR: 3.22e-05 | 1245.0ms/step | 16.1 samples/s +Step 36950/44000 | Loss: 2.9239 | LR: 3.21e-05 | 1229.5ms/step | 16.3 samples/s +Step 36960/44000 | Loss: 2.8827 | LR: 3.20e-05 | 1217.2ms/step | 16.4 samples/s +Step 36970/44000 | Loss: 2.8470 | LR: 3.19e-05 | 1257.4ms/step | 15.9 samples/s +Step 36980/44000 | Loss: 2.8830 | LR: 3.18e-05 | 1255.5ms/step | 15.9 samples/s +Step 36990/44000 | Loss: 2.8334 | LR: 3.17e-05 | 1390.3ms/step | 14.4 samples/s +Step 37000/44000 | Loss: 2.9582 | LR: 3.16e-05 | 1297.4ms/step | 15.4 samples/s +Validation at step 37000: Loss = 2.8682, Perplexity = 17.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-37000 +Step 37010/44000 | Loss: 2.9237 | LR: 3.15e-05 | 2016.4ms/step | 9.9 samples/s +Step 37020/44000 | Loss: 2.8873 | LR: 3.15e-05 | 1218.8ms/step | 16.4 samples/s +Step 37030/44000 | Loss: 2.9559 | LR: 3.14e-05 | 1167.2ms/step | 17.1 samples/s +Step 37040/44000 | Loss: 2.8605 | LR: 3.13e-05 | 1203.3ms/step | 16.6 samples/s +Step 37050/44000 | Loss: 2.9339 | LR: 3.12e-05 | 1187.7ms/step | 16.8 samples/s +Step 37060/44000 | Loss: 2.9166 | LR: 3.11e-05 | 1190.0ms/step | 16.8 samples/s +Step 37070/44000 | Loss: 2.9347 | LR: 3.10e-05 | 1175.2ms/step | 17.0 samples/s +Step 37080/44000 | Loss: 2.8642 | LR: 3.09e-05 | 1213.7ms/step | 16.5 samples/s +Step 37090/44000 | Loss: 2.9085 | LR: 3.08e-05 | 1234.5ms/step | 16.2 samples/s +Step 37100/44000 | Loss: 2.8815 | LR: 3.07e-05 | 1224.3ms/step | 16.3 samples/s +Step 37110/44000 | Loss: 2.8868 | LR: 3.07e-05 | 1193.1ms/step | 16.8 samples/s +Step 37120/44000 | Loss: 2.8678 | LR: 3.06e-05 | 1244.9ms/step | 16.1 samples/s +Step 37130/44000 | Loss: 2.8915 | LR: 3.05e-05 | 1203.6ms/step | 16.6 samples/s +Step 37140/44000 | Loss: 2.8758 | LR: 3.04e-05 | 1223.9ms/step | 16.3 samples/s +Step 37150/44000 | Loss: 2.8921 | LR: 3.03e-05 | 1255.3ms/step | 15.9 samples/s +Step 37160/44000 | Loss: 2.9242 | LR: 3.02e-05 | 1146.1ms/step | 17.4 samples/s +Step 37170/44000 | Loss: 2.8706 | LR: 3.01e-05 | 1270.3ms/step | 15.7 samples/s +Step 37180/44000 | Loss: 2.8270 | LR: 3.01e-05 | 1235.1ms/step | 16.2 samples/s +Step 37190/44000 | Loss: 2.7995 | LR: 3.00e-05 | 1106.7ms/step | 18.1 samples/s +Step 37200/44000 | Loss: 2.8611 | LR: 2.99e-05 | 1145.7ms/step | 17.5 samples/s +Step 37210/44000 | Loss: 2.8351 | LR: 2.98e-05 | 1133.1ms/step | 17.7 samples/s +Step 37220/44000 | Loss: 2.8576 | LR: 2.97e-05 | 1166.5ms/step | 17.1 samples/s +Step 37230/44000 | Loss: 2.8565 | LR: 2.96e-05 | 1187.7ms/step | 16.8 samples/s +Step 37240/44000 | Loss: 2.8509 | LR: 2.95e-05 | 1172.3ms/step | 17.1 samples/s +Step 37250/44000 | Loss: 2.8539 | LR: 2.95e-05 | 1182.5ms/step | 16.9 samples/s +Validation at step 37250: Loss = 2.8679, Perplexity = 17.60 +Step 37260/44000 | Loss: 2.9127 | LR: 2.94e-05 | 1578.5ms/step | 12.7 samples/s +Step 37270/44000 | Loss: 2.8872 | LR: 2.93e-05 | 1224.3ms/step | 16.3 samples/s +Step 37280/44000 | Loss: 2.8591 | LR: 2.92e-05 | 1188.4ms/step | 16.8 samples/s +Step 37290/44000 | Loss: 2.8777 | LR: 2.91e-05 | 1176.2ms/step | 17.0 samples/s +Step 37300/44000 | Loss: 2.9087 | LR: 2.90e-05 | 1178.8ms/step | 17.0 samples/s +Step 37310/44000 | Loss: 2.9062 | LR: 2.89e-05 | 1245.4ms/step | 16.1 samples/s +Step 37320/44000 | Loss: 2.8641 | LR: 2.89e-05 | 1187.6ms/step | 16.8 samples/s +Step 37330/44000 | Loss: 2.8964 | LR: 2.88e-05 | 1193.0ms/step | 16.8 samples/s +Step 37340/44000 | Loss: 2.9492 | LR: 2.87e-05 | 1214.2ms/step | 16.5 samples/s +Step 37350/44000 | Loss: 2.8882 | LR: 2.86e-05 | 1207.0ms/step | 16.6 samples/s +Step 37360/44000 | Loss: 2.8610 | LR: 2.85e-05 | 1205.7ms/step | 16.6 samples/s +Step 37370/44000 | Loss: 2.8919 | LR: 2.84e-05 | 1239.9ms/step | 16.1 samples/s +Step 37380/44000 | Loss: 2.8570 | LR: 2.84e-05 | 1204.5ms/step | 16.6 samples/s +Step 37390/44000 | Loss: 2.9041 | LR: 2.83e-05 | 1213.8ms/step | 16.5 samples/s +Step 37400/44000 | Loss: 2.8177 | LR: 2.82e-05 | 1183.1ms/step | 16.9 samples/s +Step 37410/44000 | Loss: 2.8118 | LR: 2.81e-05 | 1179.0ms/step | 17.0 samples/s +Step 37420/44000 | Loss: 2.7828 | LR: 2.80e-05 | 1161.0ms/step | 17.2 samples/s +Step 37430/44000 | Loss: 2.8546 | LR: 2.79e-05 | 1177.6ms/step | 17.0 samples/s +Step 37440/44000 | Loss: 2.8901 | LR: 2.78e-05 | 1146.7ms/step | 17.4 samples/s +Step 37450/44000 | Loss: 2.9257 | LR: 2.78e-05 | 1173.1ms/step | 17.0 samples/s +Step 37460/44000 | Loss: 2.8906 | LR: 2.77e-05 | 1151.4ms/step | 17.4 samples/s +Step 37470/44000 | Loss: 2.8721 | LR: 2.76e-05 | 1219.5ms/step | 16.4 samples/s +Step 37480/44000 | Loss: 2.8388 | LR: 2.75e-05 | 1136.7ms/step | 17.6 samples/s +Step 37490/44000 | Loss: 2.9193 | LR: 2.74e-05 | 1182.6ms/step | 16.9 samples/s +Step 37500/44000 | Loss: 2.8853 | LR: 2.74e-05 | 1240.3ms/step | 16.1 samples/s +Validation at step 37500: Loss = 2.8679, Perplexity = 17.60 +Step 37510/44000 | Loss: 2.9582 | LR: 2.73e-05 | 1637.2ms/step | 12.2 samples/s +Step 37520/44000 | Loss: 2.8941 | LR: 2.72e-05 | 1224.6ms/step | 16.3 samples/s +Step 37530/44000 | Loss: 2.8974 | LR: 2.71e-05 | 1219.8ms/step | 16.4 samples/s +Step 37540/44000 | Loss: 2.9659 | LR: 2.70e-05 | 1250.5ms/step | 16.0 samples/s +Step 37550/44000 | Loss: 2.8597 | LR: 2.69e-05 | 1235.3ms/step | 16.2 samples/s +Step 37560/44000 | Loss: 2.8293 | LR: 2.69e-05 | 1198.6ms/step | 16.7 samples/s +Step 37570/44000 | Loss: 2.8553 | LR: 2.68e-05 | 1214.1ms/step | 16.5 samples/s +Step 37580/44000 | Loss: 2.8981 | LR: 2.67e-05 | 1256.1ms/step | 15.9 samples/s +Step 37590/44000 | Loss: 2.9575 | LR: 2.66e-05 | 1214.1ms/step | 16.5 samples/s +Step 37600/44000 | Loss: 2.9281 | LR: 2.65e-05 | 1193.8ms/step | 16.8 samples/s +Step 37610/44000 | Loss: 2.9083 | LR: 2.65e-05 | 1168.1ms/step | 17.1 samples/s +Step 37620/44000 | Loss: 2.9283 | LR: 2.64e-05 | 1213.9ms/step | 16.5 samples/s +Step 37630/44000 | Loss: 2.9559 | LR: 2.63e-05 | 1141.4ms/step | 17.5 samples/s +Step 37640/44000 | Loss: 2.8744 | LR: 2.62e-05 | 1204.0ms/step | 16.6 samples/s +Step 37650/44000 | Loss: 2.8562 | LR: 2.61e-05 | 1177.6ms/step | 17.0 samples/s +Step 37660/44000 | Loss: 2.9031 | LR: 2.60e-05 | 1198.5ms/step | 16.7 samples/s +Step 37670/44000 | Loss: 2.8871 | LR: 2.60e-05 | 1229.2ms/step | 16.3 samples/s +Step 37680/44000 | Loss: 2.8237 | LR: 2.59e-05 | 1162.6ms/step | 17.2 samples/s +Step 37690/44000 | Loss: 2.9256 | LR: 2.58e-05 | 1132.5ms/step | 17.7 samples/s +Step 37700/44000 | Loss: 2.8928 | LR: 2.57e-05 | 1114.6ms/step | 17.9 samples/s +Step 37710/44000 | Loss: 2.8724 | LR: 2.56e-05 | 1109.0ms/step | 18.0 samples/s +Step 37720/44000 | Loss: 2.9427 | LR: 2.56e-05 | 1150.0ms/step | 17.4 samples/s +Step 37730/44000 | Loss: 2.9136 | LR: 2.55e-05 | 1211.7ms/step | 16.5 samples/s +Step 37740/44000 | Loss: 2.9044 | LR: 2.54e-05 | 1209.3ms/step | 16.5 samples/s +Step 37750/44000 | Loss: 2.8960 | LR: 2.53e-05 | 1188.3ms/step | 16.8 samples/s +Validation at step 37750: Loss = 2.8679, Perplexity = 17.60 +Step 37760/44000 | Loss: 2.8212 | LR: 2.52e-05 | 1595.4ms/step | 12.5 samples/s +Step 37770/44000 | Loss: 2.8591 | LR: 2.52e-05 | 1203.9ms/step | 16.6 samples/s +Step 37780/44000 | Loss: 2.8443 | LR: 2.51e-05 | 1287.6ms/step | 15.5 samples/s +Step 37790/44000 | Loss: 2.9168 | LR: 2.50e-05 | 1204.1ms/step | 16.6 samples/s +Step 37800/44000 | Loss: 2.8953 | LR: 2.49e-05 | 1183.0ms/step | 16.9 samples/s +Step 37810/44000 | Loss: 2.8924 | LR: 2.48e-05 | 1167.2ms/step | 17.1 samples/s +Step 37820/44000 | Loss: 2.8582 | LR: 2.48e-05 | 1163.1ms/step | 17.2 samples/s +Step 37830/44000 | Loss: 2.8795 | LR: 2.47e-05 | 1167.8ms/step | 17.1 samples/s +Step 37840/44000 | Loss: 2.8902 | LR: 2.46e-05 | 1162.8ms/step | 17.2 samples/s +Step 37850/44000 | Loss: 2.8571 | LR: 2.45e-05 | 1183.2ms/step | 16.9 samples/s +Step 37860/44000 | Loss: 2.9099 | LR: 2.45e-05 | 1177.8ms/step | 17.0 samples/s +Step 37870/44000 | Loss: 2.9160 | LR: 2.44e-05 | 1167.5ms/step | 17.1 samples/s +Step 37880/44000 | Loss: 2.8857 | LR: 2.43e-05 | 1135.6ms/step | 17.6 samples/s +Step 37890/44000 | Loss: 2.8570 | LR: 2.42e-05 | 1209.6ms/step | 16.5 samples/s +Step 37900/44000 | Loss: 2.7908 | LR: 2.41e-05 | 1230.1ms/step | 16.3 samples/s +Step 37910/44000 | Loss: 2.9430 | LR: 2.41e-05 | 1251.0ms/step | 16.0 samples/s +Step 37920/44000 | Loss: 2.9139 | LR: 2.40e-05 | 1261.1ms/step | 15.9 samples/s +Step 37930/44000 | Loss: 2.9330 | LR: 2.39e-05 | 1209.3ms/step | 16.5 samples/s +Step 37940/44000 | Loss: 2.8949 | LR: 2.38e-05 | 1173.0ms/step | 17.1 samples/s +Step 37950/44000 | Loss: 2.8726 | LR: 2.38e-05 | 1182.9ms/step | 16.9 samples/s +Step 37960/44000 | Loss: 2.8482 | LR: 2.37e-05 | 1151.6ms/step | 17.4 samples/s +Step 37970/44000 | Loss: 2.8441 | LR: 2.36e-05 | 1068.5ms/step | 18.7 samples/s +Step 37980/44000 | Loss: 2.9074 | LR: 2.35e-05 | 1155.6ms/step | 17.3 samples/s +Step 37990/44000 | Loss: 2.8648 | LR: 2.34e-05 | 1168.5ms/step | 17.1 samples/s +Step 38000/44000 | Loss: 2.9364 | LR: 2.34e-05 | 1158.5ms/step | 17.3 samples/s +Validation at step 38000: Loss = 2.8677, Perplexity = 17.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-38000 +Step 38010/44000 | Loss: 2.9112 | LR: 2.33e-05 | 2048.7ms/step | 9.8 samples/s +Step 38020/44000 | Loss: 2.9261 | LR: 2.32e-05 | 1166.4ms/step | 17.1 samples/s +Step 38030/44000 | Loss: 2.9147 | LR: 2.31e-05 | 1219.6ms/step | 16.4 samples/s +Step 38040/44000 | Loss: 2.8900 | LR: 2.31e-05 | 1220.4ms/step | 16.4 samples/s +Step 38050/44000 | Loss: 2.8939 | LR: 2.30e-05 | 1276.3ms/step | 15.7 samples/s +Step 38060/44000 | Loss: 2.8258 | LR: 2.29e-05 | 1210.1ms/step | 16.5 samples/s +Step 38070/44000 | Loss: 2.8323 | LR: 2.28e-05 | 1147.3ms/step | 17.4 samples/s +Step 38080/44000 | Loss: 2.8585 | LR: 2.28e-05 | 1209.1ms/step | 16.5 samples/s +Step 38090/44000 | Loss: 2.8520 | LR: 2.27e-05 | 1209.6ms/step | 16.5 samples/s +Step 38100/44000 | Loss: 2.9156 | LR: 2.26e-05 | 1303.2ms/step | 15.3 samples/s +Step 38110/44000 | Loss: 2.8680 | LR: 2.25e-05 | 1224.9ms/step | 16.3 samples/s +Step 38120/44000 | Loss: 2.9146 | LR: 2.25e-05 | 1219.9ms/step | 16.4 samples/s +Step 38130/44000 | Loss: 2.8383 | LR: 2.24e-05 | 1214.8ms/step | 16.5 samples/s +Step 38140/44000 | Loss: 2.8868 | LR: 2.23e-05 | 1172.6ms/step | 17.1 samples/s +Step 38150/44000 | Loss: 2.8784 | LR: 2.22e-05 | 1219.8ms/step | 16.4 samples/s +Step 38160/44000 | Loss: 2.9038 | LR: 2.22e-05 | 1209.3ms/step | 16.5 samples/s +Step 38170/44000 | Loss: 2.8656 | LR: 2.21e-05 | 1240.6ms/step | 16.1 samples/s +Step 38180/44000 | Loss: 2.8306 | LR: 2.20e-05 | 1230.0ms/step | 16.3 samples/s +Step 38190/44000 | Loss: 2.9216 | LR: 2.19e-05 | 1172.7ms/step | 17.1 samples/s +Step 38200/44000 | Loss: 2.8576 | LR: 2.19e-05 | 1167.5ms/step | 17.1 samples/s +Step 38210/44000 | Loss: 2.9015 | LR: 2.18e-05 | 1204.0ms/step | 16.6 samples/s +Step 38220/44000 | Loss: 2.8937 | LR: 2.17e-05 | 1214.4ms/step | 16.5 samples/s +Step 38230/44000 | Loss: 2.9782 | LR: 2.16e-05 | 1219.2ms/step | 16.4 samples/s +Step 38240/44000 | Loss: 2.9537 | LR: 2.16e-05 | 1177.8ms/step | 17.0 samples/s +Step 38250/44000 | Loss: 2.9227 | LR: 2.15e-05 | 1235.9ms/step | 16.2 samples/s +Validation at step 38250: Loss = 2.8676, Perplexity = 17.59 +Step 38260/44000 | Loss: 2.8959 | LR: 2.14e-05 | 1590.0ms/step | 12.6 samples/s +Step 38270/44000 | Loss: 2.8954 | LR: 2.13e-05 | 1224.7ms/step | 16.3 samples/s +Step 38280/44000 | Loss: 2.8920 | LR: 2.13e-05 | 1266.9ms/step | 15.8 samples/s +Step 38290/44000 | Loss: 2.8431 | LR: 2.12e-05 | 1261.6ms/step | 15.9 samples/s +Step 38300/44000 | Loss: 2.8856 | LR: 2.11e-05 | 1188.3ms/step | 16.8 samples/s +Step 38310/44000 | Loss: 2.8633 | LR: 2.11e-05 | 1225.2ms/step | 16.3 samples/s +Step 38320/44000 | Loss: 2.9528 | LR: 2.10e-05 | 1156.7ms/step | 17.3 samples/s +Step 38330/44000 | Loss: 2.9116 | LR: 2.09e-05 | 1230.3ms/step | 16.3 samples/s +Step 38340/44000 | Loss: 2.8714 | LR: 2.08e-05 | 1162.7ms/step | 17.2 samples/s +Step 38350/44000 | Loss: 2.8958 | LR: 2.08e-05 | 1230.1ms/step | 16.3 samples/s +Step 38360/44000 | Loss: 2.9417 | LR: 2.07e-05 | 1230.4ms/step | 16.3 samples/s +Step 38370/44000 | Loss: 2.9289 | LR: 2.06e-05 | 1178.0ms/step | 17.0 samples/s +Step 38380/44000 | Loss: 2.9361 | LR: 2.05e-05 | 1199.3ms/step | 16.7 samples/s +Step 38390/44000 | Loss: 2.8878 | LR: 2.05e-05 | 1286.9ms/step | 15.5 samples/s +Step 38400/44000 | Loss: 2.8486 | LR: 2.04e-05 | 1229.2ms/step | 16.3 samples/s +Step 38410/44000 | Loss: 2.8614 | LR: 2.03e-05 | 1221.5ms/step | 16.4 samples/s +Step 38420/44000 | Loss: 2.8840 | LR: 2.03e-05 | 1240.6ms/step | 16.1 samples/s +Step 38430/44000 | Loss: 2.8886 | LR: 2.02e-05 | 1199.5ms/step | 16.7 samples/s +Step 38440/44000 | Loss: 2.9306 | LR: 2.01e-05 | 1193.3ms/step | 16.8 samples/s +Step 38450/44000 | Loss: 2.8405 | LR: 2.00e-05 | 1241.1ms/step | 16.1 samples/s +Step 38460/44000 | Loss: 2.8575 | LR: 2.00e-05 | 1276.7ms/step | 15.7 samples/s +Step 38470/44000 | Loss: 2.8377 | LR: 1.99e-05 | 1288.4ms/step | 15.5 samples/s +Step 38480/44000 | Loss: 2.9236 | LR: 1.98e-05 | 1199.4ms/step | 16.7 samples/s +Step 38490/44000 | Loss: 2.8463 | LR: 1.98e-05 | 1162.8ms/step | 17.2 samples/s +Step 38500/44000 | Loss: 2.9434 | LR: 1.97e-05 | 1173.2ms/step | 17.0 samples/s +Validation at step 38500: Loss = 2.8676, Perplexity = 17.59 +Step 38510/44000 | Loss: 2.8767 | LR: 1.96e-05 | 1585.3ms/step | 12.6 samples/s +Step 38520/44000 | Loss: 2.8775 | LR: 1.95e-05 | 1159.5ms/step | 17.2 samples/s +Step 38530/44000 | Loss: 2.8552 | LR: 1.95e-05 | 1189.6ms/step | 16.8 samples/s +Step 38540/44000 | Loss: 2.9116 | LR: 1.94e-05 | 1186.0ms/step | 16.9 samples/s +Step 38550/44000 | Loss: 2.8590 | LR: 1.93e-05 | 1178.1ms/step | 17.0 samples/s +Step 38560/44000 | Loss: 2.9427 | LR: 1.93e-05 | 1178.2ms/step | 17.0 samples/s +Step 38570/44000 | Loss: 2.8746 | LR: 1.92e-05 | 1209.6ms/step | 16.5 samples/s +Step 38580/44000 | Loss: 2.9234 | LR: 1.91e-05 | 1229.6ms/step | 16.3 samples/s +Step 38590/44000 | Loss: 2.8809 | LR: 1.91e-05 | 1319.4ms/step | 15.2 samples/s +Step 38600/44000 | Loss: 2.9415 | LR: 1.90e-05 | 1230.3ms/step | 16.3 samples/s +Step 38610/44000 | Loss: 2.9044 | LR: 1.89e-05 | 1229.9ms/step | 16.3 samples/s +Step 38620/44000 | Loss: 2.8889 | LR: 1.88e-05 | 1271.7ms/step | 15.7 samples/s +Step 38630/44000 | Loss: 2.8882 | LR: 1.88e-05 | 1204.5ms/step | 16.6 samples/s +Step 38640/44000 | Loss: 2.8949 | LR: 1.87e-05 | 1198.2ms/step | 16.7 samples/s +Step 38650/44000 | Loss: 2.8613 | LR: 1.86e-05 | 1256.7ms/step | 15.9 samples/s +Step 38660/44000 | Loss: 2.8800 | LR: 1.86e-05 | 1182.9ms/step | 16.9 samples/s +Step 38670/44000 | Loss: 2.8818 | LR: 1.85e-05 | 1193.8ms/step | 16.8 samples/s +Step 38680/44000 | Loss: 2.8407 | LR: 1.84e-05 | 1214.5ms/step | 16.5 samples/s +Step 38690/44000 | Loss: 2.8420 | LR: 1.84e-05 | 1204.0ms/step | 16.6 samples/s +Step 38700/44000 | Loss: 2.8698 | LR: 1.83e-05 | 1211.4ms/step | 16.5 samples/s +Step 38710/44000 | Loss: 2.8877 | LR: 1.82e-05 | 1232.3ms/step | 16.2 samples/s +Step 38720/44000 | Loss: 2.8948 | LR: 1.82e-05 | 1203.6ms/step | 16.6 samples/s +Step 38730/44000 | Loss: 2.9230 | LR: 1.81e-05 | 1089.8ms/step | 18.4 samples/s +Step 38740/44000 | Loss: 2.9059 | LR: 1.80e-05 | 1156.6ms/step | 17.3 samples/s +Step 38750/44000 | Loss: 2.8939 | LR: 1.80e-05 | 1182.3ms/step | 16.9 samples/s +Validation at step 38750: Loss = 2.8676, Perplexity = 17.59 +Step 38760/44000 | Loss: 2.9212 | LR: 1.79e-05 | 1678.5ms/step | 11.9 samples/s +Step 38770/44000 | Loss: 2.9366 | LR: 1.78e-05 | 1229.9ms/step | 16.3 samples/s +Step 38780/44000 | Loss: 2.8745 | LR: 1.78e-05 | 1230.3ms/step | 16.3 samples/s +Step 38790/44000 | Loss: 2.9109 | LR: 1.77e-05 | 1292.8ms/step | 15.5 samples/s +Step 38800/44000 | Loss: 2.8758 | LR: 1.76e-05 | 1187.9ms/step | 16.8 samples/s +Step 38810/44000 | Loss: 2.8496 | LR: 1.76e-05 | 1199.0ms/step | 16.7 samples/s +Step 38820/44000 | Loss: 2.8452 | LR: 1.75e-05 | 1209.2ms/step | 16.5 samples/s +Step 38830/44000 | Loss: 2.9189 | LR: 1.74e-05 | 1235.8ms/step | 16.2 samples/s +Step 38840/44000 | Loss: 2.8900 | LR: 1.74e-05 | 1214.5ms/step | 16.5 samples/s +Step 38850/44000 | Loss: 2.8478 | LR: 1.73e-05 | 1194.0ms/step | 16.8 samples/s +Step 38860/44000 | Loss: 2.9199 | LR: 1.72e-05 | 1188.5ms/step | 16.8 samples/s +Step 38870/44000 | Loss: 2.9180 | LR: 1.72e-05 | 1204.0ms/step | 16.6 samples/s +Step 38880/44000 | Loss: 2.8710 | LR: 1.71e-05 | 1176.2ms/step | 17.0 samples/s +Step 38890/44000 | Loss: 2.9342 | LR: 1.70e-05 | 1096.4ms/step | 18.2 samples/s +Step 38900/44000 | Loss: 2.7717 | LR: 1.70e-05 | 1151.3ms/step | 17.4 samples/s +Step 38910/44000 | Loss: 2.7799 | LR: 1.69e-05 | 1219.2ms/step | 16.4 samples/s +Step 38920/44000 | Loss: 2.8399 | LR: 1.68e-05 | 1188.7ms/step | 16.8 samples/s +Step 38930/44000 | Loss: 2.9021 | LR: 1.68e-05 | 1230.0ms/step | 16.3 samples/s +Step 38940/44000 | Loss: 2.9006 | LR: 1.67e-05 | 1208.9ms/step | 16.5 samples/s +Step 38950/44000 | Loss: 2.9582 | LR: 1.66e-05 | 1208.6ms/step | 16.5 samples/s +Step 38960/44000 | Loss: 2.8525 | LR: 1.66e-05 | 1183.7ms/step | 16.9 samples/s +Step 38970/44000 | Loss: 2.8378 | LR: 1.65e-05 | 1161.0ms/step | 17.2 samples/s +Step 38980/44000 | Loss: 2.9234 | LR: 1.64e-05 | 1167.5ms/step | 17.1 samples/s +Step 38990/44000 | Loss: 3.0044 | LR: 1.64e-05 | 1203.7ms/step | 16.6 samples/s +Step 39000/44000 | Loss: 2.8797 | LR: 1.63e-05 | 1230.1ms/step | 16.3 samples/s +Validation at step 39000: Loss = 2.8676, Perplexity = 17.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-39000 +Step 39010/44000 | Loss: 2.8429 | LR: 1.62e-05 | 2110.5ms/step | 9.5 samples/s +Step 39020/44000 | Loss: 2.8989 | LR: 1.62e-05 | 1224.5ms/step | 16.3 samples/s +Step 39030/44000 | Loss: 2.8591 | LR: 1.61e-05 | 1193.6ms/step | 16.8 samples/s +Step 39040/44000 | Loss: 2.8880 | LR: 1.61e-05 | 1229.9ms/step | 16.3 samples/s +Step 39050/44000 | Loss: 2.9340 | LR: 1.60e-05 | 1251.5ms/step | 16.0 samples/s +Step 39060/44000 | Loss: 2.9284 | LR: 1.59e-05 | 1229.7ms/step | 16.3 samples/s +Step 39070/44000 | Loss: 2.8935 | LR: 1.59e-05 | 1215.2ms/step | 16.5 samples/s +Step 39080/44000 | Loss: 2.8148 | LR: 1.58e-05 | 1234.3ms/step | 16.2 samples/s +Step 39090/44000 | Loss: 2.9122 | LR: 1.57e-05 | 1194.6ms/step | 16.7 samples/s +Step 39100/44000 | Loss: 2.8448 | LR: 1.57e-05 | 1183.4ms/step | 16.9 samples/s +Step 39110/44000 | Loss: 2.9010 | LR: 1.56e-05 | 1198.8ms/step | 16.7 samples/s +Step 39120/44000 | Loss: 2.9608 | LR: 1.55e-05 | 1187.7ms/step | 16.8 samples/s +Step 39130/44000 | Loss: 2.8606 | LR: 1.55e-05 | 1240.7ms/step | 16.1 samples/s +Step 39140/44000 | Loss: 2.8847 | LR: 1.54e-05 | 1177.6ms/step | 17.0 samples/s +Step 39150/44000 | Loss: 2.9449 | LR: 1.54e-05 | 1115.5ms/step | 17.9 samples/s +Step 39160/44000 | Loss: 2.8940 | LR: 1.53e-05 | 1152.4ms/step | 17.4 samples/s +Step 39170/44000 | Loss: 2.9157 | LR: 1.52e-05 | 1201.6ms/step | 16.6 samples/s +Step 39180/44000 | Loss: 2.8646 | LR: 1.52e-05 | 1221.6ms/step | 16.4 samples/s +Step 39190/44000 | Loss: 2.9076 | LR: 1.51e-05 | 1198.4ms/step | 16.7 samples/s +Step 39200/44000 | Loss: 2.8480 | LR: 1.50e-05 | 1213.5ms/step | 16.5 samples/s +Step 39210/44000 | Loss: 2.8495 | LR: 1.50e-05 | 1194.4ms/step | 16.7 samples/s +Step 39220/44000 | Loss: 2.8992 | LR: 1.49e-05 | 1193.7ms/step | 16.8 samples/s +Step 39230/44000 | Loss: 2.9354 | LR: 1.49e-05 | 1250.5ms/step | 16.0 samples/s +Step 39240/44000 | Loss: 2.8861 | LR: 1.48e-05 | 1203.5ms/step | 16.6 samples/s +Step 39250/44000 | Loss: 2.9278 | LR: 1.47e-05 | 1156.9ms/step | 17.3 samples/s +Validation at step 39250: Loss = 2.8675, Perplexity = 17.59 +Step 39260/44000 | Loss: 2.9166 | LR: 1.47e-05 | 1516.4ms/step | 13.2 samples/s +Step 39270/44000 | Loss: 2.8513 | LR: 1.46e-05 | 1135.8ms/step | 17.6 samples/s +Step 39280/44000 | Loss: 2.8416 | LR: 1.46e-05 | 1157.6ms/step | 17.3 samples/s +Step 39290/44000 | Loss: 2.9243 | LR: 1.45e-05 | 1088.3ms/step | 18.4 samples/s +Step 39300/44000 | Loss: 2.9603 | LR: 1.44e-05 | 1162.1ms/step | 17.2 samples/s +Step 39310/44000 | Loss: 2.9480 | LR: 1.44e-05 | 1136.4ms/step | 17.6 samples/s +Step 39320/44000 | Loss: 2.9236 | LR: 1.43e-05 | 1156.7ms/step | 17.3 samples/s +Step 39330/44000 | Loss: 2.8062 | LR: 1.42e-05 | 1183.0ms/step | 16.9 samples/s +Step 39340/44000 | Loss: 2.8853 | LR: 1.42e-05 | 1204.1ms/step | 16.6 samples/s +Step 39350/44000 | Loss: 2.8967 | LR: 1.41e-05 | 1198.6ms/step | 16.7 samples/s +Step 39360/44000 | Loss: 2.8102 | LR: 1.41e-05 | 1224.5ms/step | 16.3 samples/s +Step 39370/44000 | Loss: 2.8252 | LR: 1.40e-05 | 1193.8ms/step | 16.8 samples/s +Step 39380/44000 | Loss: 2.9035 | LR: 1.39e-05 | 1240.3ms/step | 16.1 samples/s +Step 39390/44000 | Loss: 2.8441 | LR: 1.39e-05 | 1282.1ms/step | 15.6 samples/s +Step 39400/44000 | Loss: 2.8390 | LR: 1.38e-05 | 1183.7ms/step | 16.9 samples/s +Step 39410/44000 | Loss: 2.8960 | LR: 1.38e-05 | 1159.0ms/step | 17.3 samples/s +Step 39420/44000 | Loss: 2.8399 | LR: 1.37e-05 | 1195.6ms/step | 16.7 samples/s +Step 39430/44000 | Loss: 2.9081 | LR: 1.36e-05 | 1198.9ms/step | 16.7 samples/s +Step 39440/44000 | Loss: 2.8980 | LR: 1.36e-05 | 1141.1ms/step | 17.5 samples/s +Step 39450/44000 | Loss: 2.8583 | LR: 1.35e-05 | 1183.1ms/step | 16.9 samples/s +Step 39460/44000 | Loss: 2.8670 | LR: 1.35e-05 | 1130.4ms/step | 17.7 samples/s +Step 39470/44000 | Loss: 2.8450 | LR: 1.34e-05 | 1148.3ms/step | 17.4 samples/s +Step 39480/44000 | Loss: 2.8026 | LR: 1.34e-05 | 1131.3ms/step | 17.7 samples/s +Step 39490/44000 | Loss: 2.9121 | LR: 1.33e-05 | 1134.4ms/step | 17.6 samples/s +Step 39500/44000 | Loss: 2.8750 | LR: 1.32e-05 | 1172.0ms/step | 17.1 samples/s +Validation at step 39500: Loss = 2.8675, Perplexity = 17.59 +Step 39510/44000 | Loss: 2.9092 | LR: 1.32e-05 | 1485.5ms/step | 13.5 samples/s +Step 39520/44000 | Loss: 2.8714 | LR: 1.31e-05 | 1146.8ms/step | 17.4 samples/s +Step 39530/44000 | Loss: 2.9072 | LR: 1.31e-05 | 1214.2ms/step | 16.5 samples/s +Step 39540/44000 | Loss: 2.8657 | LR: 1.30e-05 | 1194.1ms/step | 16.7 samples/s +Step 39550/44000 | Loss: 2.8726 | LR: 1.29e-05 | 1209.1ms/step | 16.5 samples/s +Step 39560/44000 | Loss: 2.8936 | LR: 1.29e-05 | 1204.3ms/step | 16.6 samples/s +Step 39570/44000 | Loss: 2.9094 | LR: 1.28e-05 | 1266.5ms/step | 15.8 samples/s +Step 39580/44000 | Loss: 2.8743 | LR: 1.28e-05 | 1245.2ms/step | 16.1 samples/s +Step 39590/44000 | Loss: 2.8829 | LR: 1.27e-05 | 1188.7ms/step | 16.8 samples/s +Step 39600/44000 | Loss: 2.9396 | LR: 1.27e-05 | 1193.6ms/step | 16.8 samples/s +Step 39610/44000 | Loss: 2.8615 | LR: 1.26e-05 | 1177.2ms/step | 17.0 samples/s +Step 39620/44000 | Loss: 2.8932 | LR: 1.25e-05 | 1198.9ms/step | 16.7 samples/s +Step 39630/44000 | Loss: 2.8958 | LR: 1.25e-05 | 1120.6ms/step | 17.8 samples/s +Step 39640/44000 | Loss: 2.7460 | LR: 1.24e-05 | 1148.9ms/step | 17.4 samples/s +Step 39650/44000 | Loss: 2.9160 | LR: 1.24e-05 | 1237.9ms/step | 16.2 samples/s +Step 39660/44000 | Loss: 2.9516 | LR: 1.23e-05 | 1229.7ms/step | 16.3 samples/s +Step 39670/44000 | Loss: 2.8702 | LR: 1.23e-05 | 1230.0ms/step | 16.3 samples/s +Step 39680/44000 | Loss: 2.9399 | LR: 1.22e-05 | 1198.6ms/step | 16.7 samples/s +Step 39690/44000 | Loss: 2.9282 | LR: 1.22e-05 | 1203.4ms/step | 16.6 samples/s +Step 39700/44000 | Loss: 2.9123 | LR: 1.21e-05 | 1234.4ms/step | 16.2 samples/s +Step 39710/44000 | Loss: 2.8718 | LR: 1.20e-05 | 1266.1ms/step | 15.8 samples/s +Step 39720/44000 | Loss: 2.9177 | LR: 1.20e-05 | 1215.8ms/step | 16.4 samples/s +Step 39730/44000 | Loss: 2.8662 | LR: 1.19e-05 | 1180.8ms/step | 16.9 samples/s +Step 39740/44000 | Loss: 2.9325 | LR: 1.19e-05 | 1188.1ms/step | 16.8 samples/s +Step 39750/44000 | Loss: 2.9157 | LR: 1.18e-05 | 1198.7ms/step | 16.7 samples/s +Validation at step 39750: Loss = 2.8675, Perplexity = 17.59 +Step 39760/44000 | Loss: 2.9172 | LR: 1.18e-05 | 1610.0ms/step | 12.4 samples/s +Step 39770/44000 | Loss: 2.8842 | LR: 1.17e-05 | 1183.1ms/step | 16.9 samples/s +Step 39780/44000 | Loss: 2.8811 | LR: 1.17e-05 | 1182.8ms/step | 16.9 samples/s +Step 39790/44000 | Loss: 2.9034 | LR: 1.16e-05 | 1261.1ms/step | 15.9 samples/s +Step 39800/44000 | Loss: 2.8777 | LR: 1.15e-05 | 1244.6ms/step | 16.1 samples/s +Step 39810/44000 | Loss: 2.8786 | LR: 1.15e-05 | 1182.8ms/step | 16.9 samples/s +Step 39820/44000 | Loss: 2.8940 | LR: 1.14e-05 | 1141.3ms/step | 17.5 samples/s +Step 39830/44000 | Loss: 2.8756 | LR: 1.14e-05 | 1193.4ms/step | 16.8 samples/s +Step 39840/44000 | Loss: 2.8662 | LR: 1.13e-05 | 1120.2ms/step | 17.9 samples/s +Step 39850/44000 | Loss: 2.9138 | LR: 1.13e-05 | 1130.5ms/step | 17.7 samples/s +Step 39860/44000 | Loss: 2.9034 | LR: 1.12e-05 | 1125.3ms/step | 17.8 samples/s +Step 39870/44000 | Loss: 2.8469 | LR: 1.12e-05 | 1136.4ms/step | 17.6 samples/s +Step 39880/44000 | Loss: 2.9063 | LR: 1.11e-05 | 1141.5ms/step | 17.5 samples/s +Step 39890/44000 | Loss: 2.9318 | LR: 1.11e-05 | 1250.3ms/step | 16.0 samples/s +Step 39900/44000 | Loss: 2.9460 | LR: 1.10e-05 | 1188.1ms/step | 16.8 samples/s +Step 39910/44000 | Loss: 2.9045 | LR: 1.10e-05 | 1177.2ms/step | 17.0 samples/s +Step 39920/44000 | Loss: 2.8896 | LR: 1.09e-05 | 1182.7ms/step | 16.9 samples/s +Step 39930/44000 | Loss: 2.8676 | LR: 1.08e-05 | 1193.1ms/step | 16.8 samples/s +Step 39940/44000 | Loss: 2.8352 | LR: 1.08e-05 | 1193.5ms/step | 16.8 samples/s +Step 39950/44000 | Loss: 2.8728 | LR: 1.07e-05 | 1250.7ms/step | 16.0 samples/s +Step 39960/44000 | Loss: 2.8960 | LR: 1.07e-05 | 1166.2ms/step | 17.2 samples/s +Step 39970/44000 | Loss: 2.8625 | LR: 1.06e-05 | 1103.9ms/step | 18.1 samples/s +Step 39980/44000 | Loss: 2.8704 | LR: 1.06e-05 | 1164.1ms/step | 17.2 samples/s +Step 39990/44000 | Loss: 2.9394 | LR: 1.05e-05 | 1193.3ms/step | 16.8 samples/s +Step 40000/44000 | Loss: 2.9034 | LR: 1.05e-05 | 1204.5ms/step | 16.6 samples/s +Validation at step 40000: Loss = 2.8675, Perplexity = 17.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-40000 +Step 40010/44000 | Loss: 2.8667 | LR: 1.04e-05 | 2025.0ms/step | 9.9 samples/s +Step 40020/44000 | Loss: 2.8412 | LR: 1.04e-05 | 1150.9ms/step | 17.4 samples/s +Step 40030/44000 | Loss: 2.8943 | LR: 1.03e-05 | 1135.1ms/step | 17.6 samples/s +Step 40040/44000 | Loss: 2.8325 | LR: 1.03e-05 | 1212.8ms/step | 16.5 samples/s +Step 40050/44000 | Loss: 2.9049 | LR: 1.02e-05 | 1218.9ms/step | 16.4 samples/s +Step 40060/44000 | Loss: 2.8642 | LR: 1.02e-05 | 1167.8ms/step | 17.1 samples/s +Step 40070/44000 | Loss: 2.8833 | LR: 1.01e-05 | 1219.0ms/step | 16.4 samples/s +Step 40080/44000 | Loss: 2.8689 | LR: 1.01e-05 | 1203.8ms/step | 16.6 samples/s +Step 40090/44000 | Loss: 2.9039 | LR: 1.00e-05 | 1245.6ms/step | 16.1 samples/s +Step 40100/44000 | Loss: 2.9219 | LR: 9.96e-06 | 1136.3ms/step | 17.6 samples/s +Step 40110/44000 | Loss: 2.8505 | LR: 9.91e-06 | 1239.5ms/step | 16.1 samples/s +Step 40120/44000 | Loss: 2.8300 | LR: 9.86e-06 | 1187.9ms/step | 16.8 samples/s +Step 40130/44000 | Loss: 2.8893 | LR: 9.81e-06 | 1298.2ms/step | 15.4 samples/s +Step 40140/44000 | Loss: 2.9794 | LR: 9.76e-06 | 1202.0ms/step | 16.6 samples/s +Step 40150/44000 | Loss: 2.8519 | LR: 9.71e-06 | 1078.5ms/step | 18.5 samples/s +Step 40160/44000 | Loss: 2.8409 | LR: 9.66e-06 | 1163.8ms/step | 17.2 samples/s +Step 40170/44000 | Loss: 2.7198 | LR: 9.61e-06 | 1126.0ms/step | 17.8 samples/s +Step 40180/44000 | Loss: 2.9231 | LR: 9.56e-06 | 1094.4ms/step | 18.3 samples/s +Step 40190/44000 | Loss: 2.8989 | LR: 9.51e-06 | 1146.2ms/step | 17.4 samples/s +Step 40200/44000 | Loss: 2.9671 | LR: 9.46e-06 | 1148.9ms/step | 17.4 samples/s +Step 40210/44000 | Loss: 2.8466 | LR: 9.41e-06 | 1123.4ms/step | 17.8 samples/s +Step 40220/44000 | Loss: 2.8905 | LR: 9.36e-06 | 1130.8ms/step | 17.7 samples/s +Step 40230/44000 | Loss: 2.8875 | LR: 9.32e-06 | 1208.3ms/step | 16.6 samples/s +Step 40240/44000 | Loss: 2.8474 | LR: 9.27e-06 | 1214.3ms/step | 16.5 samples/s +Step 40250/44000 | Loss: 2.9013 | LR: 9.22e-06 | 1167.2ms/step | 17.1 samples/s +Validation at step 40250: Loss = 2.8674, Perplexity = 17.59 +Step 40260/44000 | Loss: 2.8038 | LR: 9.17e-06 | 1563.3ms/step | 12.8 samples/s +Step 40270/44000 | Loss: 2.9149 | LR: 9.12e-06 | 1213.7ms/step | 16.5 samples/s +Step 40280/44000 | Loss: 2.8724 | LR: 9.07e-06 | 1203.6ms/step | 16.6 samples/s +Step 40290/44000 | Loss: 3.0167 | LR: 9.02e-06 | 1177.6ms/step | 17.0 samples/s +Step 40300/44000 | Loss: 2.8299 | LR: 8.97e-06 | 1156.4ms/step | 17.3 samples/s +Step 40310/44000 | Loss: 2.8776 | LR: 8.93e-06 | 1124.9ms/step | 17.8 samples/s +Step 40320/44000 | Loss: 2.9246 | LR: 8.88e-06 | 1263.6ms/step | 15.8 samples/s +Step 40330/44000 | Loss: 2.8813 | LR: 8.83e-06 | 1241.2ms/step | 16.1 samples/s +Step 40340/44000 | Loss: 2.8852 | LR: 8.78e-06 | 1229.9ms/step | 16.3 samples/s +Step 40350/44000 | Loss: 2.8994 | LR: 8.74e-06 | 1239.4ms/step | 16.1 samples/s +Step 40360/44000 | Loss: 2.8834 | LR: 8.69e-06 | 1213.8ms/step | 16.5 samples/s +Step 40370/44000 | Loss: 2.8903 | LR: 8.64e-06 | 1208.4ms/step | 16.6 samples/s +Step 40380/44000 | Loss: 2.8652 | LR: 8.59e-06 | 1192.8ms/step | 16.8 samples/s +Step 40390/44000 | Loss: 2.8773 | LR: 8.55e-06 | 1203.4ms/step | 16.6 samples/s +Step 40400/44000 | Loss: 2.8416 | LR: 8.50e-06 | 1138.0ms/step | 17.6 samples/s +Step 40410/44000 | Loss: 2.8611 | LR: 8.45e-06 | 1160.5ms/step | 17.2 samples/s +Step 40420/44000 | Loss: 2.9069 | LR: 8.41e-06 | 1102.8ms/step | 18.1 samples/s +Step 40430/44000 | Loss: 2.9214 | LR: 8.36e-06 | 1110.2ms/step | 18.0 samples/s +Step 40440/44000 | Loss: 2.8952 | LR: 8.31e-06 | 1177.4ms/step | 17.0 samples/s +Step 40450/44000 | Loss: 2.9249 | LR: 8.27e-06 | 1245.4ms/step | 16.1 samples/s +Step 40460/44000 | Loss: 2.8045 | LR: 8.22e-06 | 1182.4ms/step | 16.9 samples/s +Step 40470/44000 | Loss: 2.8680 | LR: 8.17e-06 | 1195.4ms/step | 16.7 samples/s +Step 40480/44000 | Loss: 2.8649 | LR: 8.13e-06 | 1234.0ms/step | 16.2 samples/s +Step 40490/44000 | Loss: 2.8659 | LR: 8.08e-06 | 1217.7ms/step | 16.4 samples/s +Step 40500/44000 | Loss: 2.8351 | LR: 8.04e-06 | 1251.6ms/step | 16.0 samples/s +Validation at step 40500: Loss = 2.8675, Perplexity = 17.59 +Step 40510/44000 | Loss: 2.9555 | LR: 7.99e-06 | 1619.0ms/step | 12.4 samples/s +Step 40520/44000 | Loss: 2.8894 | LR: 7.94e-06 | 1156.6ms/step | 17.3 samples/s +Step 40530/44000 | Loss: 2.8965 | LR: 7.90e-06 | 1177.5ms/step | 17.0 samples/s +Step 40540/44000 | Loss: 2.8507 | LR: 7.85e-06 | 1271.1ms/step | 15.7 samples/s +Step 40550/44000 | Loss: 2.8787 | LR: 7.81e-06 | 1176.5ms/step | 17.0 samples/s +Step 40560/44000 | Loss: 2.8891 | LR: 7.76e-06 | 1219.3ms/step | 16.4 samples/s +Step 40570/44000 | Loss: 2.8751 | LR: 7.72e-06 | 1098.7ms/step | 18.2 samples/s +Step 40580/44000 | Loss: 2.8567 | LR: 7.67e-06 | 1146.2ms/step | 17.4 samples/s +Step 40590/44000 | Loss: 2.9185 | LR: 7.63e-06 | 1130.4ms/step | 17.7 samples/s +Step 40600/44000 | Loss: 2.8403 | LR: 7.59e-06 | 1161.3ms/step | 17.2 samples/s +Step 40610/44000 | Loss: 2.8658 | LR: 7.54e-06 | 1114.7ms/step | 17.9 samples/s +Step 40620/44000 | Loss: 2.7961 | LR: 7.50e-06 | 1136.0ms/step | 17.6 samples/s +Step 40630/44000 | Loss: 2.9529 | LR: 7.45e-06 | 1103.8ms/step | 18.1 samples/s +Step 40640/44000 | Loss: 2.8985 | LR: 7.41e-06 | 1152.9ms/step | 17.3 samples/s +Step 40650/44000 | Loss: 2.9084 | LR: 7.37e-06 | 1169.9ms/step | 17.1 samples/s +Step 40660/44000 | Loss: 2.9089 | LR: 7.32e-06 | 1194.2ms/step | 16.7 samples/s +Step 40670/44000 | Loss: 2.8992 | LR: 7.28e-06 | 1207.8ms/step | 16.6 samples/s +Step 40680/44000 | Loss: 2.8787 | LR: 7.23e-06 | 1219.4ms/step | 16.4 samples/s +Step 40690/44000 | Loss: 2.8913 | LR: 7.19e-06 | 1229.2ms/step | 16.3 samples/s +Step 40700/44000 | Loss: 2.8527 | LR: 7.15e-06 | 1240.0ms/step | 16.1 samples/s +Step 40710/44000 | Loss: 2.8934 | LR: 7.10e-06 | 1197.5ms/step | 16.7 samples/s +Step 40720/44000 | Loss: 2.9017 | LR: 7.06e-06 | 1203.7ms/step | 16.6 samples/s +Step 40730/44000 | Loss: 2.9929 | LR: 7.02e-06 | 1177.2ms/step | 17.0 samples/s +Step 40740/44000 | Loss: 2.8622 | LR: 6.98e-06 | 1181.2ms/step | 16.9 samples/s +Step 40750/44000 | Loss: 2.8402 | LR: 6.93e-06 | 1195.0ms/step | 16.7 samples/s +Validation at step 40750: Loss = 2.8674, Perplexity = 17.59 +Step 40760/44000 | Loss: 2.8580 | LR: 6.89e-06 | 1557.5ms/step | 12.8 samples/s +Step 40770/44000 | Loss: 2.8204 | LR: 6.85e-06 | 1156.7ms/step | 17.3 samples/s +Step 40780/44000 | Loss: 2.8970 | LR: 6.81e-06 | 1228.6ms/step | 16.3 samples/s +Step 40790/44000 | Loss: 2.9115 | LR: 6.77e-06 | 1255.4ms/step | 15.9 samples/s +Step 40800/44000 | Loss: 2.8408 | LR: 6.72e-06 | 1234.9ms/step | 16.2 samples/s +Step 40810/44000 | Loss: 2.9158 | LR: 6.68e-06 | 1229.3ms/step | 16.3 samples/s +Step 40820/44000 | Loss: 2.8546 | LR: 6.64e-06 | 1250.4ms/step | 16.0 samples/s +Step 40830/44000 | Loss: 2.8738 | LR: 6.60e-06 | 1234.2ms/step | 16.2 samples/s +Step 40840/44000 | Loss: 2.8921 | LR: 6.56e-06 | 1220.0ms/step | 16.4 samples/s +Step 40850/44000 | Loss: 2.8978 | LR: 6.52e-06 | 1149.6ms/step | 17.4 samples/s +Step 40860/44000 | Loss: 2.8922 | LR: 6.47e-06 | 1135.9ms/step | 17.6 samples/s +Step 40870/44000 | Loss: 2.8897 | LR: 6.43e-06 | 1111.8ms/step | 18.0 samples/s +Step 40880/44000 | Loss: 2.8763 | LR: 6.39e-06 | 1186.3ms/step | 16.9 samples/s +Step 40890/44000 | Loss: 2.9232 | LR: 6.35e-06 | 1156.4ms/step | 17.3 samples/s +Step 40900/44000 | Loss: 2.8559 | LR: 6.31e-06 | 1133.0ms/step | 17.7 samples/s +Step 40910/44000 | Loss: 2.8842 | LR: 6.27e-06 | 1180.4ms/step | 16.9 samples/s +Step 40920/44000 | Loss: 2.8797 | LR: 6.23e-06 | 1169.7ms/step | 17.1 samples/s +Step 40930/44000 | Loss: 2.9184 | LR: 6.19e-06 | 1154.4ms/step | 17.3 samples/s +Step 40940/44000 | Loss: 2.9125 | LR: 6.15e-06 | 1187.7ms/step | 16.8 samples/s +Step 40950/44000 | Loss: 2.8580 | LR: 6.11e-06 | 1204.4ms/step | 16.6 samples/s +Step 40960/44000 | Loss: 2.9139 | LR: 6.07e-06 | 1203.6ms/step | 16.6 samples/s +Step 40970/44000 | Loss: 2.8726 | LR: 6.03e-06 | 1240.4ms/step | 16.1 samples/s +Step 40980/44000 | Loss: 2.9437 | LR: 5.99e-06 | 1213.9ms/step | 16.5 samples/s +Step 40990/44000 | Loss: 2.8803 | LR: 5.95e-06 | 1240.2ms/step | 16.1 samples/s +Step 41000/44000 | Loss: 2.8197 | LR: 5.91e-06 | 1250.1ms/step | 16.0 samples/s +Validation at step 41000: Loss = 2.8675, Perplexity = 17.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-41000 +Step 41010/44000 | Loss: 2.9122 | LR: 5.87e-06 | 2047.7ms/step | 9.8 samples/s +Step 41020/44000 | Loss: 2.9193 | LR: 5.83e-06 | 1219.3ms/step | 16.4 samples/s +Step 41030/44000 | Loss: 2.8969 | LR: 5.80e-06 | 1193.4ms/step | 16.8 samples/s +Step 41040/44000 | Loss: 2.8328 | LR: 5.76e-06 | 1266.2ms/step | 15.8 samples/s +Step 41050/44000 | Loss: 2.8156 | LR: 5.72e-06 | 1256.0ms/step | 15.9 samples/s +Step 41060/44000 | Loss: 2.8918 | LR: 5.68e-06 | 1224.6ms/step | 16.3 samples/s +Step 41070/44000 | Loss: 2.8439 | LR: 5.64e-06 | 1188.3ms/step | 16.8 samples/s +Step 41080/44000 | Loss: 2.9090 | LR: 5.60e-06 | 1167.7ms/step | 17.1 samples/s +Step 41090/44000 | Loss: 2.8347 | LR: 5.56e-06 | 1208.8ms/step | 16.5 samples/s +Step 41100/44000 | Loss: 2.9406 | LR: 5.53e-06 | 1203.6ms/step | 16.6 samples/s +Step 41110/44000 | Loss: 2.8885 | LR: 5.49e-06 | 1219.6ms/step | 16.4 samples/s +Step 41120/44000 | Loss: 2.8210 | LR: 5.45e-06 | 1161.7ms/step | 17.2 samples/s +Step 41130/44000 | Loss: 2.8963 | LR: 5.41e-06 | 1209.0ms/step | 16.5 samples/s +Step 41140/44000 | Loss: 2.8647 | LR: 5.38e-06 | 1185.1ms/step | 16.9 samples/s +Step 41150/44000 | Loss: 2.8979 | LR: 5.34e-06 | 1175.1ms/step | 17.0 samples/s +Step 41160/44000 | Loss: 2.8360 | LR: 5.30e-06 | 1167.4ms/step | 17.1 samples/s +Step 41170/44000 | Loss: 2.8358 | LR: 5.26e-06 | 1235.0ms/step | 16.2 samples/s +Step 41180/44000 | Loss: 2.9057 | LR: 5.23e-06 | 1188.3ms/step | 16.8 samples/s +Step 41190/44000 | Loss: 2.8526 | LR: 5.19e-06 | 1209.0ms/step | 16.5 samples/s +Step 41200/44000 | Loss: 2.9322 | LR: 5.15e-06 | 1224.6ms/step | 16.3 samples/s +Step 41210/44000 | Loss: 2.8934 | LR: 5.12e-06 | 1212.6ms/step | 16.5 samples/s +Step 41220/44000 | Loss: 2.9701 | LR: 5.08e-06 | 1252.8ms/step | 16.0 samples/s +Step 41230/44000 | Loss: 2.8803 | LR: 5.04e-06 | 1199.1ms/step | 16.7 samples/s +Step 41240/44000 | Loss: 2.8245 | LR: 5.01e-06 | 1115.3ms/step | 17.9 samples/s +Step 41250/44000 | Loss: 2.8115 | LR: 4.97e-06 | 1188.3ms/step | 16.8 samples/s +Validation at step 41250: Loss = 2.8674, Perplexity = 17.59 +Step 41260/44000 | Loss: 2.8550 | LR: 4.94e-06 | 1487.9ms/step | 13.4 samples/s +Step 41270/44000 | Loss: 2.9173 | LR: 4.90e-06 | 1169.9ms/step | 17.1 samples/s +Step 41280/44000 | Loss: 2.9324 | LR: 4.86e-06 | 1172.8ms/step | 17.1 samples/s +Step 41290/44000 | Loss: 2.8684 | LR: 4.83e-06 | 1147.7ms/step | 17.4 samples/s +Step 41300/44000 | Loss: 2.8879 | LR: 4.79e-06 | 1124.6ms/step | 17.8 samples/s +Step 41310/44000 | Loss: 2.8716 | LR: 4.76e-06 | 1104.6ms/step | 18.1 samples/s +Step 41320/44000 | Loss: 2.8974 | LR: 4.72e-06 | 1082.3ms/step | 18.5 samples/s +Step 41330/44000 | Loss: 2.8732 | LR: 4.69e-06 | 1097.0ms/step | 18.2 samples/s +Step 41340/44000 | Loss: 2.8360 | LR: 4.65e-06 | 1157.5ms/step | 17.3 samples/s +Step 41350/44000 | Loss: 2.8764 | LR: 4.62e-06 | 1124.2ms/step | 17.8 samples/s +Step 41360/44000 | Loss: 2.9106 | LR: 4.58e-06 | 1166.6ms/step | 17.1 samples/s +Step 41370/44000 | Loss: 2.9408 | LR: 4.55e-06 | 1125.4ms/step | 17.8 samples/s +Step 41380/44000 | Loss: 2.8984 | LR: 4.51e-06 | 1229.6ms/step | 16.3 samples/s +Step 41390/44000 | Loss: 2.9236 | LR: 4.48e-06 | 1213.9ms/step | 16.5 samples/s +Step 41400/44000 | Loss: 2.8617 | LR: 4.45e-06 | 1218.9ms/step | 16.4 samples/s +Step 41410/44000 | Loss: 2.8399 | LR: 4.41e-06 | 1162.1ms/step | 17.2 samples/s +Step 41420/44000 | Loss: 2.9739 | LR: 4.38e-06 | 1168.4ms/step | 17.1 samples/s +Step 41430/44000 | Loss: 2.9401 | LR: 4.34e-06 | 1196.6ms/step | 16.7 samples/s +Step 41440/44000 | Loss: 2.8819 | LR: 4.31e-06 | 1194.5ms/step | 16.7 samples/s +Step 41450/44000 | Loss: 2.8954 | LR: 4.28e-06 | 1187.0ms/step | 16.8 samples/s +Step 41460/44000 | Loss: 2.9127 | LR: 4.24e-06 | 1140.2ms/step | 17.5 samples/s +Step 41470/44000 | Loss: 2.9358 | LR: 4.21e-06 | 1162.7ms/step | 17.2 samples/s +Step 41480/44000 | Loss: 2.8932 | LR: 4.18e-06 | 1115.0ms/step | 17.9 samples/s +Step 41490/44000 | Loss: 2.9475 | LR: 4.14e-06 | 1082.1ms/step | 18.5 samples/s +Step 41500/44000 | Loss: 2.9353 | LR: 4.11e-06 | 1114.5ms/step | 17.9 samples/s +Validation at step 41500: Loss = 2.8674, Perplexity = 17.59 +Step 41510/44000 | Loss: 2.8567 | LR: 4.08e-06 | 1502.3ms/step | 13.3 samples/s +Step 41520/44000 | Loss: 2.9415 | LR: 4.05e-06 | 1120.1ms/step | 17.9 samples/s +Step 41530/44000 | Loss: 2.9162 | LR: 4.01e-06 | 1121.0ms/step | 17.8 samples/s +Step 41540/44000 | Loss: 2.8712 | LR: 3.98e-06 | 1219.0ms/step | 16.4 samples/s +Step 41550/44000 | Loss: 2.8670 | LR: 3.95e-06 | 1219.6ms/step | 16.4 samples/s +Step 41560/44000 | Loss: 2.9787 | LR: 3.92e-06 | 1167.6ms/step | 17.1 samples/s +Step 41570/44000 | Loss: 2.8562 | LR: 3.88e-06 | 1177.6ms/step | 17.0 samples/s +Step 41580/44000 | Loss: 2.9031 | LR: 3.85e-06 | 1167.1ms/step | 17.1 samples/s +Step 41590/44000 | Loss: 2.9406 | LR: 3.82e-06 | 1167.5ms/step | 17.1 samples/s +Step 41600/44000 | Loss: 2.9524 | LR: 3.79e-06 | 1140.7ms/step | 17.5 samples/s +Step 41610/44000 | Loss: 2.8659 | LR: 3.76e-06 | 1133.4ms/step | 17.6 samples/s +Step 41620/44000 | Loss: 2.8213 | LR: 3.73e-06 | 1217.3ms/step | 16.4 samples/s +Step 41630/44000 | Loss: 2.8659 | LR: 3.70e-06 | 1203.5ms/step | 16.6 samples/s +Step 41640/44000 | Loss: 2.8799 | LR: 3.66e-06 | 1146.4ms/step | 17.4 samples/s +Step 41650/44000 | Loss: 2.8765 | LR: 3.63e-06 | 1193.4ms/step | 16.8 samples/s +Step 41660/44000 | Loss: 2.8400 | LR: 3.60e-06 | 1193.1ms/step | 16.8 samples/s +Step 41670/44000 | Loss: 2.8694 | LR: 3.57e-06 | 1167.0ms/step | 17.1 samples/s +Step 41680/44000 | Loss: 2.8637 | LR: 3.54e-06 | 1156.4ms/step | 17.3 samples/s +Step 41690/44000 | Loss: 2.8758 | LR: 3.51e-06 | 1177.4ms/step | 17.0 samples/s +Step 41700/44000 | Loss: 2.8707 | LR: 3.48e-06 | 1192.9ms/step | 16.8 samples/s +Step 41710/44000 | Loss: 2.8482 | LR: 3.45e-06 | 1206.6ms/step | 16.6 samples/s +Step 41720/44000 | Loss: 2.8760 | LR: 3.42e-06 | 1227.0ms/step | 16.3 samples/s +Step 41730/44000 | Loss: 2.9015 | LR: 3.39e-06 | 1239.5ms/step | 16.1 samples/s +Step 41740/44000 | Loss: 2.8565 | LR: 3.36e-06 | 1224.1ms/step | 16.3 samples/s +Step 41750/44000 | Loss: 2.9458 | LR: 3.33e-06 | 1205.9ms/step | 16.6 samples/s +Validation at step 41750: Loss = 2.8674, Perplexity = 17.59 +Step 41760/44000 | Loss: 2.8678 | LR: 3.30e-06 | 1581.7ms/step | 12.6 samples/s +Step 41770/44000 | Loss: 2.8527 | LR: 3.27e-06 | 1167.3ms/step | 17.1 samples/s +Step 41780/44000 | Loss: 2.9582 | LR: 3.24e-06 | 1192.9ms/step | 16.8 samples/s +Step 41790/44000 | Loss: 2.8645 | LR: 3.21e-06 | 1177.8ms/step | 17.0 samples/s +Step 41800/44000 | Loss: 2.9045 | LR: 3.19e-06 | 1140.4ms/step | 17.5 samples/s +Step 41810/44000 | Loss: 2.8425 | LR: 3.16e-06 | 1255.3ms/step | 15.9 samples/s +Step 41820/44000 | Loss: 2.8653 | LR: 3.13e-06 | 1250.4ms/step | 16.0 samples/s +Step 41830/44000 | Loss: 2.9146 | LR: 3.10e-06 | 1182.3ms/step | 16.9 samples/s +Step 41840/44000 | Loss: 2.9407 | LR: 3.07e-06 | 1198.4ms/step | 16.7 samples/s +Step 41850/44000 | Loss: 2.8389 | LR: 3.04e-06 | 1172.3ms/step | 17.1 samples/s +Step 41860/44000 | Loss: 2.9130 | LR: 3.01e-06 | 1182.6ms/step | 16.9 samples/s +Step 41870/44000 | Loss: 2.8920 | LR: 2.99e-06 | 1173.5ms/step | 17.0 samples/s +Step 41880/44000 | Loss: 2.9038 | LR: 2.96e-06 | 1160.6ms/step | 17.2 samples/s +Step 41890/44000 | Loss: 2.8743 | LR: 2.93e-06 | 1172.9ms/step | 17.1 samples/s +Step 41900/44000 | Loss: 2.8347 | LR: 2.90e-06 | 1177.3ms/step | 17.0 samples/s +Step 41910/44000 | Loss: 2.8558 | LR: 2.88e-06 | 1094.6ms/step | 18.3 samples/s +Step 41920/44000 | Loss: 2.8605 | LR: 2.85e-06 | 1105.4ms/step | 18.1 samples/s +Step 41930/44000 | Loss: 2.8952 | LR: 2.82e-06 | 1173.1ms/step | 17.0 samples/s +Step 41940/44000 | Loss: 2.8673 | LR: 2.79e-06 | 1170.9ms/step | 17.1 samples/s +Step 41950/44000 | Loss: 2.8772 | LR: 2.77e-06 | 1156.9ms/step | 17.3 samples/s +Step 41960/44000 | Loss: 2.9311 | LR: 2.74e-06 | 1152.0ms/step | 17.4 samples/s +Step 41970/44000 | Loss: 2.8440 | LR: 2.71e-06 | 1151.5ms/step | 17.4 samples/s +Step 41980/44000 | Loss: 2.9347 | LR: 2.69e-06 | 1151.8ms/step | 17.4 samples/s +Step 41990/44000 | Loss: 2.9253 | LR: 2.66e-06 | 1177.5ms/step | 17.0 samples/s +Step 42000/44000 | Loss: 2.8153 | LR: 2.63e-06 | 1141.9ms/step | 17.5 samples/s +Validation at step 42000: Loss = 2.8674, Perplexity = 17.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-42000 +Step 42010/44000 | Loss: 2.8889 | LR: 2.61e-06 | 2020.2ms/step | 9.9 samples/s +Step 42020/44000 | Loss: 2.8710 | LR: 2.58e-06 | 1138.4ms/step | 17.6 samples/s +Step 42030/44000 | Loss: 2.9161 | LR: 2.56e-06 | 1136.9ms/step | 17.6 samples/s +Step 42040/44000 | Loss: 2.8957 | LR: 2.53e-06 | 1208.8ms/step | 16.5 samples/s +Step 42050/44000 | Loss: 2.9747 | LR: 2.50e-06 | 1229.7ms/step | 16.3 samples/s +Step 42060/44000 | Loss: 2.8515 | LR: 2.48e-06 | 1229.7ms/step | 16.3 samples/s +Step 42070/44000 | Loss: 2.8805 | LR: 2.45e-06 | 1229.7ms/step | 16.3 samples/s +Step 42080/44000 | Loss: 2.9007 | LR: 2.43e-06 | 1224.8ms/step | 16.3 samples/s +Step 42090/44000 | Loss: 2.9011 | LR: 2.40e-06 | 1208.8ms/step | 16.5 samples/s +Step 42100/44000 | Loss: 2.8980 | LR: 2.38e-06 | 1183.4ms/step | 16.9 samples/s +Step 42110/44000 | Loss: 2.9048 | LR: 2.35e-06 | 1208.9ms/step | 16.5 samples/s +Step 42120/44000 | Loss: 2.8414 | LR: 2.33e-06 | 1203.6ms/step | 16.6 samples/s +Step 42130/44000 | Loss: 2.8768 | LR: 2.30e-06 | 1146.5ms/step | 17.4 samples/s +Step 42140/44000 | Loss: 2.8752 | LR: 2.28e-06 | 1146.4ms/step | 17.4 samples/s +Step 42150/44000 | Loss: 2.7769 | LR: 2.25e-06 | 1188.4ms/step | 16.8 samples/s +Step 42160/44000 | Loss: 2.9104 | LR: 2.23e-06 | 1208.9ms/step | 16.5 samples/s +Step 42170/44000 | Loss: 2.8726 | LR: 2.21e-06 | 1198.4ms/step | 16.7 samples/s +Step 42180/44000 | Loss: 2.9445 | LR: 2.18e-06 | 1229.8ms/step | 16.3 samples/s +Step 42190/44000 | Loss: 2.9264 | LR: 2.16e-06 | 1188.3ms/step | 16.8 samples/s +Step 42200/44000 | Loss: 2.8744 | LR: 2.13e-06 | 1235.0ms/step | 16.2 samples/s +Step 42210/44000 | Loss: 2.8446 | LR: 2.11e-06 | 1170.0ms/step | 17.1 samples/s +Step 42220/44000 | Loss: 2.9407 | LR: 2.09e-06 | 1196.5ms/step | 16.7 samples/s +Step 42230/44000 | Loss: 2.8551 | LR: 2.06e-06 | 1178.4ms/step | 17.0 samples/s +Step 42240/44000 | Loss: 2.9131 | LR: 2.04e-06 | 1185.1ms/step | 16.9 samples/s +Step 42250/44000 | Loss: 2.8833 | LR: 2.02e-06 | 1159.7ms/step | 17.2 samples/s +Validation at step 42250: Loss = 2.8675, Perplexity = 17.59 +Step 42260/44000 | Loss: 2.8526 | LR: 1.99e-06 | 1547.9ms/step | 12.9 samples/s +Step 42270/44000 | Loss: 2.8655 | LR: 1.97e-06 | 1146.6ms/step | 17.4 samples/s +Step 42280/44000 | Loss: 2.8557 | LR: 1.95e-06 | 1167.5ms/step | 17.1 samples/s +Step 42290/44000 | Loss: 2.8584 | LR: 1.93e-06 | 1198.6ms/step | 16.7 samples/s +Step 42300/44000 | Loss: 2.8366 | LR: 1.90e-06 | 1188.5ms/step | 16.8 samples/s +Step 42310/44000 | Loss: 2.9573 | LR: 1.88e-06 | 1193.3ms/step | 16.8 samples/s +Step 42320/44000 | Loss: 2.8794 | LR: 1.86e-06 | 1172.7ms/step | 17.1 samples/s +Step 42330/44000 | Loss: 2.8392 | LR: 1.84e-06 | 1151.9ms/step | 17.4 samples/s +Step 42340/44000 | Loss: 2.9433 | LR: 1.82e-06 | 1172.7ms/step | 17.1 samples/s +Step 42350/44000 | Loss: 2.8157 | LR: 1.79e-06 | 1178.0ms/step | 17.0 samples/s +Step 42360/44000 | Loss: 2.9187 | LR: 1.77e-06 | 1188.6ms/step | 16.8 samples/s +Step 42370/44000 | Loss: 2.9076 | LR: 1.75e-06 | 1229.7ms/step | 16.3 samples/s +Step 42380/44000 | Loss: 2.8699 | LR: 1.73e-06 | 1209.3ms/step | 16.5 samples/s +Step 42390/44000 | Loss: 2.8965 | LR: 1.71e-06 | 1224.6ms/step | 16.3 samples/s +Step 42400/44000 | Loss: 2.9124 | LR: 1.69e-06 | 1214.2ms/step | 16.5 samples/s +Step 42410/44000 | Loss: 2.8574 | LR: 1.67e-06 | 1173.2ms/step | 17.0 samples/s +Step 42420/44000 | Loss: 2.8519 | LR: 1.64e-06 | 1214.1ms/step | 16.5 samples/s +Step 42430/44000 | Loss: 2.8858 | LR: 1.62e-06 | 1224.9ms/step | 16.3 samples/s +Step 42440/44000 | Loss: 2.8462 | LR: 1.60e-06 | 1219.3ms/step | 16.4 samples/s +Step 42450/44000 | Loss: 2.9189 | LR: 1.58e-06 | 1204.5ms/step | 16.6 samples/s +Step 42460/44000 | Loss: 2.8524 | LR: 1.56e-06 | 1204.5ms/step | 16.6 samples/s +Step 42470/44000 | Loss: 2.9095 | LR: 1.54e-06 | 1240.1ms/step | 16.1 samples/s +Step 42480/44000 | Loss: 2.8392 | LR: 1.52e-06 | 1199.2ms/step | 16.7 samples/s +Step 42490/44000 | Loss: 2.9067 | LR: 1.50e-06 | 1229.8ms/step | 16.3 samples/s +Step 42500/44000 | Loss: 2.8936 | LR: 1.48e-06 | 1218.8ms/step | 16.4 samples/s +Validation at step 42500: Loss = 2.8674, Perplexity = 17.59 +Step 42510/44000 | Loss: 2.8796 | LR: 1.46e-06 | 1537.0ms/step | 13.0 samples/s +Step 42520/44000 | Loss: 2.7564 | LR: 1.44e-06 | 1165.6ms/step | 17.2 samples/s +Step 42530/44000 | Loss: 2.8438 | LR: 1.42e-06 | 1190.1ms/step | 16.8 samples/s +Step 42540/44000 | Loss: 2.8786 | LR: 1.40e-06 | 1136.5ms/step | 17.6 samples/s +Step 42550/44000 | Loss: 2.9340 | LR: 1.39e-06 | 1109.7ms/step | 18.0 samples/s +Step 42560/44000 | Loss: 2.9385 | LR: 1.37e-06 | 1188.6ms/step | 16.8 samples/s +Step 42570/44000 | Loss: 2.9259 | LR: 1.35e-06 | 1193.8ms/step | 16.8 samples/s +Step 42580/44000 | Loss: 2.8574 | LR: 1.33e-06 | 1131.3ms/step | 17.7 samples/s +Step 42590/44000 | Loss: 2.8222 | LR: 1.31e-06 | 1152.1ms/step | 17.4 samples/s +Step 42600/44000 | Loss: 2.9231 | LR: 1.29e-06 | 1146.4ms/step | 17.4 samples/s +Step 42610/44000 | Loss: 2.8608 | LR: 1.27e-06 | 1141.4ms/step | 17.5 samples/s +Step 42620/44000 | Loss: 2.8609 | LR: 1.25e-06 | 1178.1ms/step | 17.0 samples/s +Step 42630/44000 | Loss: 2.9611 | LR: 1.24e-06 | 1167.5ms/step | 17.1 samples/s +Step 42640/44000 | Loss: 2.9132 | LR: 1.22e-06 | 1105.0ms/step | 18.1 samples/s +Step 42650/44000 | Loss: 2.9398 | LR: 1.20e-06 | 1136.8ms/step | 17.6 samples/s +Step 42660/44000 | Loss: 2.7990 | LR: 1.18e-06 | 1183.2ms/step | 16.9 samples/s +Step 42670/44000 | Loss: 2.9082 | LR: 1.17e-06 | 1162.2ms/step | 17.2 samples/s +Step 42680/44000 | Loss: 2.8806 | LR: 1.15e-06 | 1176.9ms/step | 17.0 samples/s +Step 42690/44000 | Loss: 2.8625 | LR: 1.13e-06 | 1220.9ms/step | 16.4 samples/s +Step 42700/44000 | Loss: 2.9215 | LR: 1.11e-06 | 1240.4ms/step | 16.1 samples/s +Step 42710/44000 | Loss: 2.8751 | LR: 1.10e-06 | 1203.7ms/step | 16.6 samples/s +Step 42720/44000 | Loss: 2.8540 | LR: 1.08e-06 | 1196.4ms/step | 16.7 samples/s +Step 42730/44000 | Loss: 2.9111 | LR: 1.06e-06 | 1189.2ms/step | 16.8 samples/s +Step 42740/44000 | Loss: 2.8507 | LR: 1.05e-06 | 1174.0ms/step | 17.0 samples/s +Step 42750/44000 | Loss: 2.9237 | LR: 1.03e-06 | 1219.7ms/step | 16.4 samples/s +Validation at step 42750: Loss = 2.8674, Perplexity = 17.59 +Step 42760/44000 | Loss: 2.9126 | LR: 1.01e-06 | 1584.4ms/step | 12.6 samples/s +Step 42770/44000 | Loss: 2.8761 | LR: 9.97e-07 | 1140.9ms/step | 17.5 samples/s +Step 42780/44000 | Loss: 2.8745 | LR: 9.81e-07 | 1136.2ms/step | 17.6 samples/s +Step 42790/44000 | Loss: 2.8746 | LR: 9.65e-07 | 1099.9ms/step | 18.2 samples/s +Step 42800/44000 | Loss: 2.8291 | LR: 9.49e-07 | 1158.1ms/step | 17.3 samples/s +Step 42810/44000 | Loss: 2.9011 | LR: 9.33e-07 | 1108.1ms/step | 18.0 samples/s +Step 42820/44000 | Loss: 2.9316 | LR: 9.18e-07 | 1110.3ms/step | 18.0 samples/s +Step 42830/44000 | Loss: 2.9071 | LR: 9.02e-07 | 1093.8ms/step | 18.3 samples/s +Step 42840/44000 | Loss: 2.8602 | LR: 8.87e-07 | 1104.4ms/step | 18.1 samples/s +Step 42850/44000 | Loss: 2.9436 | LR: 8.72e-07 | 1100.3ms/step | 18.2 samples/s +Step 42860/44000 | Loss: 2.9840 | LR: 8.57e-07 | 1103.9ms/step | 18.1 samples/s +Step 42870/44000 | Loss: 2.9601 | LR: 8.42e-07 | 1099.6ms/step | 18.2 samples/s +Step 42880/44000 | Loss: 2.8911 | LR: 8.27e-07 | 1062.7ms/step | 18.8 samples/s +Step 42890/44000 | Loss: 2.8961 | LR: 8.12e-07 | 1119.9ms/step | 17.9 samples/s +Step 42900/44000 | Loss: 2.8961 | LR: 7.98e-07 | 1052.8ms/step | 19.0 samples/s +Step 42910/44000 | Loss: 2.8592 | LR: 7.83e-07 | 1114.8ms/step | 17.9 samples/s +Step 42920/44000 | Loss: 2.8521 | LR: 7.69e-07 | 1114.6ms/step | 17.9 samples/s +Step 42930/44000 | Loss: 2.8633 | LR: 7.55e-07 | 1156.8ms/step | 17.3 samples/s +Step 42940/44000 | Loss: 2.9380 | LR: 7.41e-07 | 1172.5ms/step | 17.1 samples/s +Step 42950/44000 | Loss: 2.8766 | LR: 7.27e-07 | 1167.0ms/step | 17.1 samples/s +Step 42960/44000 | Loss: 2.8785 | LR: 7.13e-07 | 1120.1ms/step | 17.9 samples/s +Step 42970/44000 | Loss: 2.8493 | LR: 6.99e-07 | 1104.3ms/step | 18.1 samples/s +Step 42980/44000 | Loss: 2.8880 | LR: 6.86e-07 | 1093.1ms/step | 18.3 samples/s +Step 42990/44000 | Loss: 2.8903 | LR: 6.72e-07 | 1166.9ms/step | 17.1 samples/s +Step 43000/44000 | Loss: 2.8270 | LR: 6.59e-07 | 1101.9ms/step | 18.2 samples/s +Validation at step 43000: Loss = 2.8675, Perplexity = 17.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-43000 +Step 43010/44000 | Loss: 2.8852 | LR: 6.46e-07 | 2013.4ms/step | 9.9 samples/s +Step 43020/44000 | Loss: 2.8746 | LR: 6.33e-07 | 1208.1ms/step | 16.6 samples/s +Step 43030/44000 | Loss: 2.8692 | LR: 6.20e-07 | 1234.4ms/step | 16.2 samples/s +Step 43040/44000 | Loss: 2.8459 | LR: 6.08e-07 | 1208.6ms/step | 16.5 samples/s +Step 43050/44000 | Loss: 2.8655 | LR: 5.95e-07 | 1182.7ms/step | 16.9 samples/s +Step 43060/44000 | Loss: 2.8912 | LR: 5.83e-07 | 1177.3ms/step | 17.0 samples/s +Step 43070/44000 | Loss: 2.8658 | LR: 5.70e-07 | 1162.7ms/step | 17.2 samples/s +Step 43080/44000 | Loss: 2.8680 | LR: 5.58e-07 | 1156.0ms/step | 17.3 samples/s +Step 43090/44000 | Loss: 2.9161 | LR: 5.46e-07 | 1167.3ms/step | 17.1 samples/s +Step 43100/44000 | Loss: 2.9429 | LR: 5.34e-07 | 1172.1ms/step | 17.1 samples/s +Step 43110/44000 | Loss: 2.9262 | LR: 5.22e-07 | 1125.6ms/step | 17.8 samples/s +Step 43120/44000 | Loss: 2.8623 | LR: 5.11e-07 | 1125.4ms/step | 17.8 samples/s +Step 43130/44000 | Loss: 2.8304 | LR: 4.99e-07 | 1177.0ms/step | 17.0 samples/s +Step 43140/44000 | Loss: 2.8886 | LR: 4.88e-07 | 1214.3ms/step | 16.5 samples/s +Step 43150/44000 | Loss: 2.9259 | LR: 4.76e-07 | 1187.7ms/step | 16.8 samples/s +Step 43160/44000 | Loss: 2.8664 | LR: 4.65e-07 | 1250.5ms/step | 16.0 samples/s +Step 43170/44000 | Loss: 2.9129 | LR: 4.54e-07 | 1123.0ms/step | 17.8 samples/s +Step 43180/44000 | Loss: 2.9052 | LR: 4.43e-07 | 1211.2ms/step | 16.5 samples/s +Step 43190/44000 | Loss: 2.8759 | LR: 4.33e-07 | 1210.6ms/step | 16.5 samples/s +Step 43200/44000 | Loss: 2.8616 | LR: 4.22e-07 | 1144.4ms/step | 17.5 samples/s +Step 43210/44000 | Loss: 2.9443 | LR: 4.12e-07 | 1184.9ms/step | 16.9 samples/s +Step 43220/44000 | Loss: 2.8446 | LR: 4.01e-07 | 1180.3ms/step | 16.9 samples/s +Step 43230/44000 | Loss: 2.9207 | LR: 3.91e-07 | 1167.0ms/step | 17.1 samples/s +Step 43240/44000 | Loss: 2.9035 | LR: 3.81e-07 | 1166.8ms/step | 17.1 samples/s +Step 43250/44000 | Loss: 2.8928 | LR: 3.71e-07 | 1178.2ms/step | 17.0 samples/s +Validation at step 43250: Loss = 2.8674, Perplexity = 17.59 +Step 43260/44000 | Loss: 2.9006 | LR: 3.61e-07 | 1507.0ms/step | 13.3 samples/s +Step 43270/44000 | Loss: 2.8700 | LR: 3.51e-07 | 1150.5ms/step | 17.4 samples/s +Step 43280/44000 | Loss: 2.8648 | LR: 3.42e-07 | 1105.1ms/step | 18.1 samples/s +Step 43290/44000 | Loss: 2.8854 | LR: 3.32e-07 | 1078.7ms/step | 18.5 samples/s +Step 43300/44000 | Loss: 2.8944 | LR: 3.23e-07 | 1130.9ms/step | 17.7 samples/s +Step 43310/44000 | Loss: 2.8282 | LR: 3.14e-07 | 1109.7ms/step | 18.0 samples/s +Step 43320/44000 | Loss: 2.8961 | LR: 3.05e-07 | 1119.6ms/step | 17.9 samples/s +Step 43330/44000 | Loss: 2.9395 | LR: 2.96e-07 | 1136.0ms/step | 17.6 samples/s +Step 43340/44000 | Loss: 2.8596 | LR: 2.87e-07 | 1156.5ms/step | 17.3 samples/s +Step 43350/44000 | Loss: 2.8996 | LR: 2.79e-07 | 1172.5ms/step | 17.1 samples/s +Step 43360/44000 | Loss: 2.8601 | LR: 2.70e-07 | 1140.8ms/step | 17.5 samples/s +Step 43370/44000 | Loss: 2.9174 | LR: 2.62e-07 | 1094.3ms/step | 18.3 samples/s +Step 43380/44000 | Loss: 2.8903 | LR: 2.53e-07 | 1172.2ms/step | 17.1 samples/s +Step 43390/44000 | Loss: 2.8618 | LR: 2.45e-07 | 1175.0ms/step | 17.0 samples/s +Step 43400/44000 | Loss: 2.8576 | LR: 2.37e-07 | 1232.1ms/step | 16.2 samples/s +Step 43410/44000 | Loss: 2.8524 | LR: 2.30e-07 | 1253.1ms/step | 16.0 samples/s +Step 43420/44000 | Loss: 2.9292 | LR: 2.22e-07 | 1221.9ms/step | 16.4 samples/s +Step 43430/44000 | Loss: 2.8871 | LR: 2.14e-07 | 1220.3ms/step | 16.4 samples/s +Step 43440/44000 | Loss: 2.8817 | LR: 2.07e-07 | 1223.4ms/step | 16.3 samples/s +Step 43450/44000 | Loss: 2.9339 | LR: 1.99e-07 | 1193.3ms/step | 16.8 samples/s +Step 43460/44000 | Loss: 2.8930 | LR: 1.92e-07 | 1213.7ms/step | 16.5 samples/s +Step 43470/44000 | Loss: 2.8601 | LR: 1.85e-07 | 1099.5ms/step | 18.2 samples/s +Step 43480/44000 | Loss: 2.8364 | LR: 1.78e-07 | 1078.6ms/step | 18.5 samples/s +Step 43490/44000 | Loss: 2.9254 | LR: 1.72e-07 | 1099.1ms/step | 18.2 samples/s +Step 43500/44000 | Loss: 2.8697 | LR: 1.65e-07 | 1136.6ms/step | 17.6 samples/s +Validation at step 43500: Loss = 2.8674, Perplexity = 17.59 +Step 43510/44000 | Loss: 2.9341 | LR: 1.58e-07 | 1557.1ms/step | 12.8 samples/s +Step 43520/44000 | Loss: 2.9587 | LR: 1.52e-07 | 1297.1ms/step | 15.4 samples/s +Step 43530/44000 | Loss: 2.8796 | LR: 1.46e-07 | 1194.8ms/step | 16.7 samples/s +Step 43540/44000 | Loss: 2.8633 | LR: 1.40e-07 | 1135.3ms/step | 17.6 samples/s +Step 43550/44000 | Loss: 2.9582 | LR: 1.34e-07 | 1195.1ms/step | 16.7 samples/s +Step 43560/44000 | Loss: 2.9002 | LR: 1.28e-07 | 1175.5ms/step | 17.0 samples/s +Step 43570/44000 | Loss: 2.8789 | LR: 1.22e-07 | 1198.1ms/step | 16.7 samples/s +Step 43580/44000 | Loss: 2.8340 | LR: 1.16e-07 | 1167.5ms/step | 17.1 samples/s +Step 43590/44000 | Loss: 2.9683 | LR: 1.11e-07 | 1172.3ms/step | 17.1 samples/s +Step 43600/44000 | Loss: 2.8664 | LR: 1.06e-07 | 1178.9ms/step | 17.0 samples/s +Step 43610/44000 | Loss: 2.9615 | LR: 1.00e-07 | 1186.3ms/step | 16.9 samples/s +Step 43620/44000 | Loss: 2.8919 | LR: 9.52e-08 | 1152.0ms/step | 17.4 samples/s +Step 43630/44000 | Loss: 2.8873 | LR: 9.03e-08 | 1198.5ms/step | 16.7 samples/s +Step 43640/44000 | Loss: 2.8590 | LR: 8.55e-08 | 1104.7ms/step | 18.1 samples/s +Step 43650/44000 | Loss: 2.8527 | LR: 8.08e-08 | 1120.7ms/step | 17.8 samples/s +Step 43660/44000 | Loss: 2.8745 | LR: 7.62e-08 | 1094.4ms/step | 18.3 samples/s +Step 43670/44000 | Loss: 2.9292 | LR: 7.18e-08 | 1159.4ms/step | 17.3 samples/s +Step 43680/44000 | Loss: 2.8839 | LR: 6.75e-08 | 1154.5ms/step | 17.3 samples/s +Step 43690/44000 | Loss: 2.8763 | LR: 6.34e-08 | 1157.1ms/step | 17.3 samples/s +Step 43700/44000 | Loss: 2.9049 | LR: 5.94e-08 | 1146.5ms/step | 17.4 samples/s +Step 43710/44000 | Loss: 2.8988 | LR: 5.55e-08 | 1177.4ms/step | 17.0 samples/s +Step 43720/44000 | Loss: 2.8901 | LR: 5.17e-08 | 1178.1ms/step | 17.0 samples/s +Step 43730/44000 | Loss: 2.8555 | LR: 4.81e-08 | 1172.6ms/step | 17.1 samples/s +Step 43740/44000 | Loss: 2.9061 | LR: 4.46e-08 | 1127.1ms/step | 17.7 samples/s +Step 43750/44000 | Loss: 2.9130 | LR: 4.12e-08 | 1129.2ms/step | 17.7 samples/s +Validation at step 43750: Loss = 2.8674, Perplexity = 17.59 +Step 43760/44000 | Loss: 2.8658 | LR: 3.80e-08 | 1568.5ms/step | 12.8 samples/s +Step 43770/44000 | Loss: 2.8445 | LR: 3.49e-08 | 1183.3ms/step | 16.9 samples/s +Step 43780/44000 | Loss: 2.8563 | LR: 3.19e-08 | 1255.7ms/step | 15.9 samples/s +Step 43790/44000 | Loss: 2.8951 | LR: 2.91e-08 | 1236.4ms/step | 16.2 samples/s +Step 43800/44000 | Loss: 2.9002 | LR: 2.64e-08 | 1229.3ms/step | 16.3 samples/s +Step 43810/44000 | Loss: 2.9127 | LR: 2.38e-08 | 1214.5ms/step | 16.5 samples/s +Step 43820/44000 | Loss: 2.8743 | LR: 2.14e-08 | 1214.0ms/step | 16.5 samples/s +Step 43830/44000 | Loss: 2.8550 | LR: 1.91e-08 | 1198.6ms/step | 16.7 samples/s +Step 43840/44000 | Loss: 2.8883 | LR: 1.69e-08 | 1183.3ms/step | 16.9 samples/s +Step 43850/44000 | Loss: 2.8539 | LR: 1.48e-08 | 1166.9ms/step | 17.1 samples/s +Step 43860/44000 | Loss: 2.8280 | LR: 1.29e-08 | 1167.9ms/step | 17.1 samples/s +Step 43870/44000 | Loss: 2.8790 | LR: 1.11e-08 | 1156.7ms/step | 17.3 samples/s +Step 43880/44000 | Loss: 2.8705 | LR: 9.50e-09 | 1188.3ms/step | 16.8 samples/s +Step 43890/44000 | Loss: 2.8843 | LR: 7.98e-09 | 1193.0ms/step | 16.8 samples/s +Step 43900/44000 | Loss: 2.8766 | LR: 6.60e-09 | 1240.1ms/step | 16.1 samples/s +Step 43910/44000 | Loss: 2.8639 | LR: 5.34e-09 | 1193.4ms/step | 16.8 samples/s +Step 43920/44000 | Loss: 2.8832 | LR: 4.22e-09 | 1167.9ms/step | 17.1 samples/s +Step 43930/44000 | Loss: 2.8811 | LR: 3.23e-09 | 1151.6ms/step | 17.4 samples/s +Step 43940/44000 | Loss: 2.8640 | LR: 2.37e-09 | 1224.9ms/step | 16.3 samples/s +Step 43950/44000 | Loss: 2.9305 | LR: 1.65e-09 | 1240.1ms/step | 16.1 samples/s +Step 43960/44000 | Loss: 2.9169 | LR: 1.06e-09 | 1183.7ms/step | 16.9 samples/s +Step 43970/44000 | Loss: 2.9076 | LR: 5.94e-10 | 1208.6ms/step | 16.5 samples/s +Step 43980/44000 | Loss: 2.9348 | LR: 2.64e-10 | 1203.6ms/step | 16.6 samples/s +Step 43990/44000 | Loss: 2.9495 | LR: 6.60e-11 | 1167.6ms/step | 17.1 samples/s +Step 44000/44000 | Loss: 2.9452 | LR: 0.00e+00 | 1168.6ms/step | 17.1 samples/s +Validation at step 44000: Loss = 2.8675, Perplexity = 17.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-44000 +wandb: updating run metadata +wandb: uploading wandb-summary.json; uploading config.yaml; uploading output.log +wandb: uploading wandb-summary.json; uploading config.yaml +wandb: uploading history steps 4399-4399, summary, console lines 1159-1161 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▇▆▆▅▅▄▄▄▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▇▇▆▆▅▅▄▄▄▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▃▂█▂▃▂▂▁▂▂▂▇▁▂▂▁▂▁▂▂▁▁▁▁▂▂▂▂▂▂▂▁▂▁▂▂▂▂ +wandb: speed/samples_per_sec ▃▃▂▅▄▄▂█▄▁▂▃▅▆▂▂▄▂▃▅▆▄▁█▄▂▆▆▃▅▂▃▄▄▆▄▂▃▆▂ +wandb: speed/steps_per_sec ▁▆▆▇▇▇▆▇▆▅▇▇▇▇▄▆▇▆▅▆▆▇▆▆▇▆▅▇▇▇█▆▆▇█▆█▇█▆ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.86746 +wandb: eval/perplexity 17.59221 +wandb: eval/uses_dense_attention 1 +wandb: eval/uses_sparse_attention 0 +wandb: speed/ms_per_step 1168.63854 +wandb: speed/samples_per_sec 17.11393 +wandb: speed/steps_per_sec 0.8557 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m dense 8k seed47 ddp5 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/89kzpaot +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260820_235903-89kzpaot/logs +Training complete! diff --git a/epfl_final/train_logs/seed47_mixed_h100_ddp2/mixed.20260820T235114Z.log b/epfl_final/train_logs/seed47_mixed_h100_ddp2/mixed.20260820T235114Z.log new file mode 100644 index 0000000000000000000000000000000000000000..f48caae310b239b77d8374f6b8561181488c3cc6 --- /dev/null +++ b/epfl_final/train_logs/seed47_mixed_h100_ddp2/mixed.20260820T235114Z.log @@ -0,0 +1,2153 @@ +W0820 23:51:17.800000 43318 site-packages/torch/distributed/run.py:851] +W0820 23:51:17.800000 43318 site-packages/torch/distributed/run.py:851] ***************************************** +W0820 23:51:17.800000 43318 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0820 23:51:17.800000 43318 site-packages/torch/distributed/run.py:851] ***************************************** +Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-26000 +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-26000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 47, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m mixed 8k seed47 ddp2 h100-resume", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "mixed", + "multiseed", + "seed47", + "ddp2", + "h100-resume" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": [ + 63, + 47, + 31, + 21, + 11 + ], + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-26000 +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-26000... +Preserving checkpoint LR state for exact resume: current=[0.00018492988117498478], schedule_base=[0.0005] +Resumed from checkpoint at step 26000 (epoch 0, micro_step 130000) +Resuming training from step 26000 +Resuming wandb run 93ukqj9s +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 93ukqj9s +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_235426-93ukqj9s +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 365m mixed 8k seed47 ddp2 h100-resume +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/93ukqj9s +Starting training... +Validation at step 26000: Loss = 2.9799, Perplexity = 19.69 +wandb: WARNING Tried to log to step 26000 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 26001: sparse_stochastic (configured sparse attention) +Step 26010/44000 | Loss: 2.9677 | LR: 1.85e-04 | 27713.0ms/step | 0.7 samples/s +wandb: WARNING Tried to log to step 26010 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26020/44000 | Loss: 2.9250 | LR: 1.85e-04 | 699.3ms/step | 28.6 samples/s +wandb: WARNING Tried to log to step 26020 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26030/44000 | Loss: 2.9238 | LR: 1.84e-04 | 698.3ms/step | 28.6 samples/s +Step 26040/44000 | Loss: 2.9750 | LR: 1.84e-04 | 699.8ms/step | 28.6 samples/s +wandb: WARNING Tried to log to step 26030 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 26040 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26050/44000 | Loss: 2.9683 | LR: 1.84e-04 | 699.1ms/step | 28.6 samples/s +wandb: WARNING Tried to log to step 26050 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26060/44000 | Loss: 2.9867 | LR: 1.84e-04 | 701.3ms/step | 28.5 samples/s +Step 26070/44000 | Loss: 2.9906 | LR: 1.84e-04 | 704.1ms/step | 28.4 samples/s +wandb: WARNING Tried to log to step 26060 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 26070 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26080/44000 | Loss: 2.9751 | LR: 1.84e-04 | 703.4ms/step | 28.4 samples/s +wandb: WARNING Tried to log to step 26080 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26090/44000 | Loss: 2.9413 | LR: 1.83e-04 | 700.9ms/step | 28.5 samples/s +wandb: WARNING Tried to log to step 26090 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26100/44000 | Loss: 2.9612 | LR: 1.83e-04 | 701.9ms/step | 28.5 samples/s +Step 26110/44000 | Loss: 2.9435 | LR: 1.83e-04 | 703.6ms/step | 28.4 samples/s +wandb: WARNING Tried to log to step 26100 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 26110 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26120/44000 | Loss: 3.0050 | LR: 1.83e-04 | 701.3ms/step | 28.5 samples/s +wandb: WARNING Tried to log to step 26120 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26130/44000 | Loss: 2.9613 | LR: 1.83e-04 | 701.7ms/step | 28.5 samples/s +Step 26140/44000 | Loss: 2.9551 | LR: 1.82e-04 | 699.6ms/step | 28.6 samples/s +wandb: WARNING Tried to log to step 26130 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 26140 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26150/44000 | Loss: 2.9981 | LR: 1.82e-04 | 702.9ms/step | 28.5 samples/s +wandb: WARNING Tried to log to step 26150 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26160/44000 | Loss: 2.9946 | LR: 1.82e-04 | 700.1ms/step | 28.6 samples/s +Step 26170/44000 | Loss: 2.9491 | LR: 1.82e-04 | 698.4ms/step | 28.6 samples/s +wandb: WARNING Tried to log to step 26160 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 26170 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26180/44000 | Loss: 2.9140 | LR: 1.82e-04 | 699.5ms/step | 28.6 samples/s +wandb: WARNING Tried to log to step 26180 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26190/44000 | Loss: 2.9722 | LR: 1.82e-04 | 699.3ms/step | 28.6 samples/s +wandb: WARNING Tried to log to step 26190 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26200/44000 | Loss: 2.9114 | LR: 1.81e-04 | 702.7ms/step | 28.5 samples/s +Step 26210/44000 | Loss: 2.9624 | LR: 1.81e-04 | 698.5ms/step | 28.6 samples/s +wandb: WARNING Tried to log to step 26200 that is less than the current step 26201. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 26220/44000 | Loss: 2.9324 | LR: 1.81e-04 | 702.1ms/step | 28.5 samples/s +Step 26230/44000 | Loss: 2.9895 | LR: 1.81e-04 | 701.4ms/step | 28.5 samples/s +Step 26240/44000 | Loss: 2.9943 | LR: 1.81e-04 | 703.7ms/step | 28.4 samples/s +Step 26250/44000 | Loss: 2.9858 | LR: 1.81e-04 | 701.5ms/step | 28.5 samples/s +Validation at step 26250: Loss = 2.9788, Perplexity = 19.66 +Step 26260/44000 | Loss: 2.9615 | LR: 1.80e-04 | 1273.2ms/step | 15.7 samples/s +Step 26270/44000 | Loss: 2.9584 | LR: 1.80e-04 | 707.0ms/step | 28.3 samples/s +Step 26280/44000 | Loss: 2.9803 | LR: 1.80e-04 | 710.5ms/step | 28.1 samples/s +Step 26290/44000 | Loss: 2.9938 | LR: 1.80e-04 | 711.0ms/step | 28.1 samples/s +Step 26300/44000 | Loss: 2.9343 | LR: 1.80e-04 | 725.1ms/step | 27.6 samples/s +Step 26310/44000 | Loss: 2.9435 | LR: 1.80e-04 | 730.5ms/step | 27.4 samples/s +Step 26320/44000 | Loss: 2.9393 | LR: 1.79e-04 | 724.6ms/step | 27.6 samples/s +Step 26330/44000 | Loss: 2.9348 | LR: 1.79e-04 | 724.7ms/step | 27.6 samples/s +Step 26340/44000 | Loss: 2.9117 | LR: 1.79e-04 | 724.3ms/step | 27.6 samples/s +Step 26350/44000 | Loss: 2.9620 | LR: 1.79e-04 | 706.9ms/step | 28.3 samples/s +Step 26360/44000 | Loss: 2.9741 | LR: 1.79e-04 | 706.3ms/step | 28.3 samples/s +Step 26370/44000 | Loss: 2.9645 | LR: 1.78e-04 | 705.7ms/step | 28.3 samples/s +Step 26380/44000 | Loss: 2.9542 | LR: 1.78e-04 | 710.6ms/step | 28.1 samples/s +Step 26390/44000 | Loss: 2.9800 | LR: 1.78e-04 | 707.8ms/step | 28.3 samples/s +Step 26400/44000 | Loss: 3.0002 | LR: 1.78e-04 | 707.4ms/step | 28.3 samples/s +Step 26410/44000 | Loss: 2.9852 | LR: 1.78e-04 | 705.9ms/step | 28.3 samples/s +Step 26420/44000 | Loss: 2.9566 | LR: 1.78e-04 | 705.1ms/step | 28.4 samples/s +Step 26430/44000 | Loss: 2.9405 | LR: 1.77e-04 | 705.0ms/step | 28.4 samples/s +Step 26440/44000 | Loss: 2.9474 | LR: 1.77e-04 | 704.4ms/step | 28.4 samples/s +Step 26450/44000 | Loss: 2.9941 | LR: 1.77e-04 | 705.9ms/step | 28.3 samples/s +Step 26460/44000 | Loss: 2.9903 | LR: 1.77e-04 | 709.6ms/step | 28.2 samples/s +Step 26470/44000 | Loss: 2.9523 | LR: 1.77e-04 | 708.5ms/step | 28.2 samples/s +Step 26480/44000 | Loss: 2.9859 | LR: 1.77e-04 | 702.2ms/step | 28.5 samples/s +Step 26490/44000 | Loss: 2.9739 | LR: 1.76e-04 | 704.8ms/step | 28.4 samples/s +Step 26500/44000 | Loss: 2.9646 | LR: 1.76e-04 | 704.1ms/step | 28.4 samples/s +Validation at step 26500: Loss = 2.9769, Perplexity = 19.63 +Step 26510/44000 | Loss: 3.0004 | LR: 1.76e-04 | 1275.0ms/step | 15.7 samples/s +Step 26520/44000 | Loss: 2.9114 | LR: 1.76e-04 | 704.6ms/step | 28.4 samples/s +Step 26530/44000 | Loss: 2.9255 | LR: 1.76e-04 | 701.9ms/step | 28.5 samples/s +Step 26540/44000 | Loss: 2.9567 | LR: 1.76e-04 | 704.0ms/step | 28.4 samples/s +Step 26550/44000 | Loss: 2.9233 | LR: 1.75e-04 | 707.0ms/step | 28.3 samples/s +Step 26560/44000 | Loss: 2.9538 | LR: 1.75e-04 | 703.1ms/step | 28.4 samples/s +Step 26570/44000 | Loss: 2.9764 | LR: 1.75e-04 | 703.7ms/step | 28.4 samples/s +Step 26580/44000 | Loss: 2.9611 | LR: 1.75e-04 | 703.1ms/step | 28.4 samples/s +Step 26590/44000 | Loss: 2.9756 | LR: 1.75e-04 | 703.2ms/step | 28.4 samples/s +Step 26600/44000 | Loss: 2.9545 | LR: 1.74e-04 | 703.5ms/step | 28.4 samples/s +Step 26610/44000 | Loss: 2.9855 | LR: 1.74e-04 | 700.0ms/step | 28.6 samples/s +Step 26620/44000 | Loss: 2.9786 | LR: 1.74e-04 | 702.8ms/step | 28.5 samples/s +Step 26630/44000 | Loss: 2.9343 | LR: 1.74e-04 | 705.1ms/step | 28.4 samples/s +Step 26640/44000 | Loss: 2.9343 | LR: 1.74e-04 | 711.3ms/step | 28.1 samples/s +Step 26650/44000 | Loss: 2.9925 | LR: 1.74e-04 | 712.1ms/step | 28.1 samples/s +Step 26660/44000 | Loss: 2.9637 | LR: 1.73e-04 | 711.3ms/step | 28.1 samples/s +Step 26670/44000 | Loss: 2.9330 | LR: 1.73e-04 | 708.8ms/step | 28.2 samples/s +Step 26680/44000 | Loss: 2.9376 | LR: 1.73e-04 | 707.4ms/step | 28.3 samples/s +Step 26690/44000 | Loss: 2.9747 | LR: 1.73e-04 | 702.5ms/step | 28.5 samples/s +Step 26700/44000 | Loss: 2.9778 | LR: 1.73e-04 | 704.3ms/step | 28.4 samples/s +Step 26710/44000 | Loss: 2.9418 | LR: 1.73e-04 | 703.9ms/step | 28.4 samples/s +Step 26720/44000 | Loss: 2.9530 | LR: 1.72e-04 | 702.8ms/step | 28.5 samples/s +Step 26730/44000 | Loss: 2.9215 | LR: 1.72e-04 | 705.6ms/step | 28.3 samples/s +Step 26740/44000 | Loss: 2.9851 | LR: 1.72e-04 | 704.3ms/step | 28.4 samples/s +Step 26750/44000 | Loss: 2.9496 | LR: 1.72e-04 | 702.1ms/step | 28.5 samples/s +Validation at step 26750: Loss = 2.9753, Perplexity = 19.60 +Step 26760/44000 | Loss: 2.9554 | LR: 1.72e-04 | 1277.2ms/step | 15.7 samples/s +Step 26770/44000 | Loss: 2.9431 | LR: 1.72e-04 | 708.0ms/step | 28.2 samples/s +Step 26780/44000 | Loss: 2.9746 | LR: 1.71e-04 | 699.8ms/step | 28.6 samples/s +Step 26790/44000 | Loss: 2.9774 | LR: 1.71e-04 | 700.1ms/step | 28.6 samples/s +Step 26800/44000 | Loss: 2.9821 | LR: 1.71e-04 | 705.7ms/step | 28.3 samples/s +Step 26810/44000 | Loss: 2.9867 | LR: 1.71e-04 | 704.2ms/step | 28.4 samples/s +Step 26820/44000 | Loss: 2.9403 | LR: 1.71e-04 | 703.1ms/step | 28.4 samples/s +Step 26830/44000 | Loss: 2.9282 | LR: 1.71e-04 | 702.5ms/step | 28.5 samples/s +Step 26840/44000 | Loss: 2.8960 | LR: 1.70e-04 | 704.8ms/step | 28.4 samples/s +Step 26850/44000 | Loss: 2.9710 | LR: 1.70e-04 | 706.6ms/step | 28.3 samples/s +Step 26860/44000 | Loss: 2.9959 | LR: 1.70e-04 | 701.7ms/step | 28.5 samples/s +Step 26870/44000 | Loss: 2.9714 | LR: 1.70e-04 | 706.1ms/step | 28.3 samples/s +Step 26880/44000 | Loss: 2.9590 | LR: 1.70e-04 | 705.8ms/step | 28.3 samples/s +Step 26890/44000 | Loss: 2.9746 | LR: 1.69e-04 | 704.1ms/step | 28.4 samples/s +Step 26900/44000 | Loss: 2.9750 | LR: 1.69e-04 | 700.4ms/step | 28.6 samples/s +Step 26910/44000 | Loss: 2.9764 | LR: 1.69e-04 | 704.2ms/step | 28.4 samples/s +Step 26920/44000 | Loss: 2.9614 | LR: 1.69e-04 | 703.1ms/step | 28.4 samples/s +Step 26930/44000 | Loss: 2.9819 | LR: 1.69e-04 | 704.8ms/step | 28.4 samples/s +Step 26940/44000 | Loss: 2.9528 | LR: 1.69e-04 | 704.2ms/step | 28.4 samples/s +Step 26950/44000 | Loss: 2.9691 | LR: 1.68e-04 | 709.1ms/step | 28.2 samples/s +Step 26960/44000 | Loss: 2.9950 | LR: 1.68e-04 | 706.8ms/step | 28.3 samples/s +Step 26970/44000 | Loss: 2.9274 | LR: 1.68e-04 | 703.5ms/step | 28.4 samples/s +Step 26980/44000 | Loss: 2.9839 | LR: 1.68e-04 | 702.7ms/step | 28.5 samples/s +Step 26990/44000 | Loss: 2.9642 | LR: 1.68e-04 | 700.1ms/step | 28.6 samples/s +Step 27000/44000 | Loss: 2.9849 | LR: 1.68e-04 | 703.0ms/step | 28.4 samples/s +Validation at step 27000: Loss = 2.9737, Perplexity = 19.56 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-27000 +Step 27010/44000 | Loss: 2.9132 | LR: 1.67e-04 | 1576.3ms/step | 12.7 samples/s +Step 27020/44000 | Loss: 2.9350 | LR: 1.67e-04 | 701.5ms/step | 28.5 samples/s +Step 27030/44000 | Loss: 2.9532 | LR: 1.67e-04 | 699.3ms/step | 28.6 samples/s +Step 27040/44000 | Loss: 2.9786 | LR: 1.67e-04 | 702.4ms/step | 28.5 samples/s +Step 27050/44000 | Loss: 2.9175 | LR: 1.67e-04 | 699.8ms/step | 28.6 samples/s +Step 27060/44000 | Loss: 2.9367 | LR: 1.67e-04 | 699.7ms/step | 28.6 samples/s +Step 27070/44000 | Loss: 2.9947 | LR: 1.66e-04 | 701.1ms/step | 28.5 samples/s +Step 27080/44000 | Loss: 2.9880 | LR: 1.66e-04 | 699.6ms/step | 28.6 samples/s +Step 27090/44000 | Loss: 2.9757 | LR: 1.66e-04 | 700.5ms/step | 28.5 samples/s +Step 27100/44000 | Loss: 2.9790 | LR: 1.66e-04 | 701.7ms/step | 28.5 samples/s +Step 27110/44000 | Loss: 2.9709 | LR: 1.66e-04 | 701.3ms/step | 28.5 samples/s +Step 27120/44000 | Loss: 2.9892 | LR: 1.66e-04 | 702.1ms/step | 28.5 samples/s +Step 27130/44000 | Loss: 2.9531 | LR: 1.65e-04 | 705.1ms/step | 28.4 samples/s +Step 27140/44000 | Loss: 2.9526 | LR: 1.65e-04 | 703.9ms/step | 28.4 samples/s +Step 27150/44000 | Loss: 2.9623 | LR: 1.65e-04 | 702.2ms/step | 28.5 samples/s +Step 27160/44000 | Loss: 3.0086 | LR: 1.65e-04 | 703.4ms/step | 28.4 samples/s +Step 27170/44000 | Loss: 2.9259 | LR: 1.65e-04 | 701.1ms/step | 28.5 samples/s +Step 27180/44000 | Loss: 2.9541 | LR: 1.65e-04 | 700.8ms/step | 28.5 samples/s +Step 27190/44000 | Loss: 2.9820 | LR: 1.64e-04 | 700.4ms/step | 28.6 samples/s +Step 27200/44000 | Loss: 2.9102 | LR: 1.64e-04 | 703.5ms/step | 28.4 samples/s +Step 27210/44000 | Loss: 2.9519 | LR: 1.64e-04 | 702.3ms/step | 28.5 samples/s +Step 27220/44000 | Loss: 2.9778 | LR: 1.64e-04 | 702.4ms/step | 28.5 samples/s +Step 27230/44000 | Loss: 2.9167 | LR: 1.64e-04 | 699.1ms/step | 28.6 samples/s +Step 27240/44000 | Loss: 2.9333 | LR: 1.63e-04 | 701.8ms/step | 28.5 samples/s +Step 27250/44000 | Loss: 2.9923 | LR: 1.63e-04 | 702.8ms/step | 28.5 samples/s +Validation at step 27250: Loss = 2.9735, Perplexity = 19.56 +Step 27260/44000 | Loss: 2.9462 | LR: 1.63e-04 | 1270.5ms/step | 15.7 samples/s +Step 27270/44000 | Loss: 3.0014 | LR: 1.63e-04 | 702.0ms/step | 28.5 samples/s +Step 27280/44000 | Loss: 2.9489 | LR: 1.63e-04 | 702.4ms/step | 28.5 samples/s +Step 27290/44000 | Loss: 2.9356 | LR: 1.63e-04 | 703.4ms/step | 28.4 samples/s +Step 27300/44000 | Loss: 2.9902 | LR: 1.62e-04 | 703.1ms/step | 28.4 samples/s +Step 27310/44000 | Loss: 2.9884 | LR: 1.62e-04 | 702.4ms/step | 28.5 samples/s +Step 27320/44000 | Loss: 3.0124 | LR: 1.62e-04 | 702.0ms/step | 28.5 samples/s +Step 27330/44000 | Loss: 2.9738 | LR: 1.62e-04 | 702.0ms/step | 28.5 samples/s +Step 27340/44000 | Loss: 2.9437 | LR: 1.62e-04 | 700.6ms/step | 28.5 samples/s +Step 27350/44000 | Loss: 2.9314 | LR: 1.62e-04 | 702.3ms/step | 28.5 samples/s +Step 27360/44000 | Loss: 2.9585 | LR: 1.61e-04 | 701.2ms/step | 28.5 samples/s +Step 27370/44000 | Loss: 2.9472 | LR: 1.61e-04 | 704.0ms/step | 28.4 samples/s +Step 27380/44000 | Loss: 2.9288 | LR: 1.61e-04 | 703.0ms/step | 28.4 samples/s +Step 27390/44000 | Loss: 2.9895 | LR: 1.61e-04 | 703.1ms/step | 28.4 samples/s +Step 27400/44000 | Loss: 2.9513 | LR: 1.61e-04 | 705.7ms/step | 28.3 samples/s +Step 27410/44000 | Loss: 2.9755 | LR: 1.61e-04 | 704.0ms/step | 28.4 samples/s +Step 27420/44000 | Loss: 2.9748 | LR: 1.60e-04 | 701.2ms/step | 28.5 samples/s +Step 27430/44000 | Loss: 2.9557 | LR: 1.60e-04 | 704.0ms/step | 28.4 samples/s +Step 27440/44000 | Loss: 3.0057 | LR: 1.60e-04 | 704.3ms/step | 28.4 samples/s +Step 27450/44000 | Loss: 2.9625 | LR: 1.60e-04 | 702.0ms/step | 28.5 samples/s +Step 27460/44000 | Loss: 2.9891 | LR: 1.60e-04 | 699.9ms/step | 28.6 samples/s +Step 27470/44000 | Loss: 2.9907 | LR: 1.60e-04 | 706.9ms/step | 28.3 samples/s +Step 27480/44000 | Loss: 2.9944 | LR: 1.59e-04 | 701.8ms/step | 28.5 samples/s +Step 27490/44000 | Loss: 2.9129 | LR: 1.59e-04 | 700.4ms/step | 28.6 samples/s +Step 27500/44000 | Loss: 2.9300 | LR: 1.59e-04 | 706.5ms/step | 28.3 samples/s +Validation at step 27500: Loss = 2.9721, Perplexity = 19.53 +Step 27510/44000 | Loss: 2.9731 | LR: 1.59e-04 | 1273.3ms/step | 15.7 samples/s +Step 27520/44000 | Loss: 2.9861 | LR: 1.59e-04 | 701.8ms/step | 28.5 samples/s +Step 27530/44000 | Loss: 2.9669 | LR: 1.59e-04 | 703.7ms/step | 28.4 samples/s +Step 27540/44000 | Loss: 2.9776 | LR: 1.58e-04 | 700.5ms/step | 28.5 samples/s +Step 27550/44000 | Loss: 2.9776 | LR: 1.58e-04 | 705.4ms/step | 28.4 samples/s +Step 27560/44000 | Loss: 2.9570 | LR: 1.58e-04 | 699.8ms/step | 28.6 samples/s +Step 27570/44000 | Loss: 2.9599 | LR: 1.58e-04 | 700.3ms/step | 28.6 samples/s +Step 27580/44000 | Loss: 2.9939 | LR: 1.58e-04 | 704.4ms/step | 28.4 samples/s +Step 27590/44000 | Loss: 2.9670 | LR: 1.58e-04 | 703.5ms/step | 28.4 samples/s +Step 27600/44000 | Loss: 2.9579 | LR: 1.57e-04 | 700.1ms/step | 28.6 samples/s +Step 27610/44000 | Loss: 2.9492 | LR: 1.57e-04 | 705.7ms/step | 28.3 samples/s +Step 27620/44000 | Loss: 2.9419 | LR: 1.57e-04 | 705.1ms/step | 28.4 samples/s +Step 27630/44000 | Loss: 2.9414 | LR: 1.57e-04 | 701.7ms/step | 28.5 samples/s +Step 27640/44000 | Loss: 2.9656 | LR: 1.57e-04 | 702.4ms/step | 28.5 samples/s +Step 27650/44000 | Loss: 2.9144 | LR: 1.57e-04 | 707.4ms/step | 28.3 samples/s +Step 27660/44000 | Loss: 2.9837 | LR: 1.56e-04 | 703.3ms/step | 28.4 samples/s +Step 27670/44000 | Loss: 2.9461 | LR: 1.56e-04 | 705.7ms/step | 28.3 samples/s +Step 27680/44000 | Loss: 2.9456 | LR: 1.56e-04 | 702.9ms/step | 28.5 samples/s +Step 27690/44000 | Loss: 2.9761 | LR: 1.56e-04 | 704.7ms/step | 28.4 samples/s +Step 27700/44000 | Loss: 2.9551 | LR: 1.56e-04 | 703.9ms/step | 28.4 samples/s +Step 27710/44000 | Loss: 2.9525 | LR: 1.56e-04 | 699.4ms/step | 28.6 samples/s +Step 27720/44000 | Loss: 2.9193 | LR: 1.55e-04 | 704.0ms/step | 28.4 samples/s +Step 27730/44000 | Loss: 2.9196 | LR: 1.55e-04 | 702.1ms/step | 28.5 samples/s +Step 27740/44000 | Loss: 2.9674 | LR: 1.55e-04 | 702.1ms/step | 28.5 samples/s +Step 27750/44000 | Loss: 2.9728 | LR: 1.55e-04 | 704.2ms/step | 28.4 samples/s +Validation at step 27750: Loss = 2.9689, Perplexity = 19.47 +Step 27760/44000 | Loss: 2.9469 | LR: 1.55e-04 | 1273.3ms/step | 15.7 samples/s +Step 27770/44000 | Loss: 2.9619 | LR: 1.55e-04 | 702.5ms/step | 28.5 samples/s +Step 27780/44000 | Loss: 2.9488 | LR: 1.54e-04 | 700.0ms/step | 28.6 samples/s +Step 27790/44000 | Loss: 2.9393 | LR: 1.54e-04 | 698.6ms/step | 28.6 samples/s +Step 27800/44000 | Loss: 2.9266 | LR: 1.54e-04 | 700.6ms/step | 28.5 samples/s +Step 27810/44000 | Loss: 2.9717 | LR: 1.54e-04 | 702.9ms/step | 28.5 samples/s +Step 27820/44000 | Loss: 2.9743 | LR: 1.54e-04 | 702.3ms/step | 28.5 samples/s +Step 27830/44000 | Loss: 2.9397 | LR: 1.54e-04 | 702.1ms/step | 28.5 samples/s +Step 27840/44000 | Loss: 2.9698 | LR: 1.53e-04 | 703.1ms/step | 28.4 samples/s +Step 27850/44000 | Loss: 2.9745 | LR: 1.53e-04 | 702.2ms/step | 28.5 samples/s +Step 27860/44000 | Loss: 2.9506 | LR: 1.53e-04 | 700.1ms/step | 28.6 samples/s +Step 27870/44000 | Loss: 2.9137 | LR: 1.53e-04 | 701.7ms/step | 28.5 samples/s +Step 27880/44000 | Loss: 2.9613 | LR: 1.53e-04 | 704.2ms/step | 28.4 samples/s +Step 27890/44000 | Loss: 2.9626 | LR: 1.53e-04 | 705.4ms/step | 28.4 samples/s +Step 27900/44000 | Loss: 2.9617 | LR: 1.52e-04 | 700.5ms/step | 28.5 samples/s +Step 27910/44000 | Loss: 2.9777 | LR: 1.52e-04 | 702.5ms/step | 28.5 samples/s +Step 27920/44000 | Loss: 2.9912 | LR: 1.52e-04 | 701.8ms/step | 28.5 samples/s +Step 27930/44000 | Loss: 2.9806 | LR: 1.52e-04 | 701.6ms/step | 28.5 samples/s +Step 27940/44000 | Loss: 2.9484 | LR: 1.52e-04 | 702.5ms/step | 28.5 samples/s +Step 27950/44000 | Loss: 2.9437 | LR: 1.52e-04 | 701.6ms/step | 28.5 samples/s +Step 27960/44000 | Loss: 2.9636 | LR: 1.51e-04 | 702.9ms/step | 28.5 samples/s +Step 27970/44000 | Loss: 2.9533 | LR: 1.51e-04 | 703.7ms/step | 28.4 samples/s +Step 27980/44000 | Loss: 2.9802 | LR: 1.51e-04 | 699.7ms/step | 28.6 samples/s +Step 27990/44000 | Loss: 2.9721 | LR: 1.51e-04 | 704.7ms/step | 28.4 samples/s +Step 28000/44000 | Loss: 2.9775 | LR: 1.51e-04 | 701.9ms/step | 28.5 samples/s +Validation at step 28000: Loss = 2.9684, Perplexity = 19.46 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-28000 +Step 28010/44000 | Loss: 2.9678 | LR: 1.51e-04 | 1567.1ms/step | 12.8 samples/s +Step 28020/44000 | Loss: 2.9391 | LR: 1.50e-04 | 700.3ms/step | 28.6 samples/s +Step 28030/44000 | Loss: 2.9833 | LR: 1.50e-04 | 700.8ms/step | 28.5 samples/s +Step 28040/44000 | Loss: 2.9448 | LR: 1.50e-04 | 703.5ms/step | 28.4 samples/s +Step 28050/44000 | Loss: 2.8868 | LR: 1.50e-04 | 706.2ms/step | 28.3 samples/s +Step 28060/44000 | Loss: 2.9495 | LR: 1.50e-04 | 699.9ms/step | 28.6 samples/s +Step 28070/44000 | Loss: 2.9876 | LR: 1.50e-04 | 702.3ms/step | 28.5 samples/s +Step 28080/44000 | Loss: 2.9846 | LR: 1.49e-04 | 703.1ms/step | 28.4 samples/s +Step 28090/44000 | Loss: 2.9686 | LR: 1.49e-04 | 702.9ms/step | 28.5 samples/s +Step 28100/44000 | Loss: 2.9812 | LR: 1.49e-04 | 700.2ms/step | 28.6 samples/s +Step 28110/44000 | Loss: 2.9089 | LR: 1.49e-04 | 702.5ms/step | 28.5 samples/s +Step 28120/44000 | Loss: 2.9574 | LR: 1.49e-04 | 699.5ms/step | 28.6 samples/s +Step 28130/44000 | Loss: 2.9868 | LR: 1.49e-04 | 699.0ms/step | 28.6 samples/s +Step 28140/44000 | Loss: 2.9372 | LR: 1.48e-04 | 700.2ms/step | 28.6 samples/s +Step 28150/44000 | Loss: 2.9600 | LR: 1.48e-04 | 700.7ms/step | 28.5 samples/s +Step 28160/44000 | Loss: 2.9541 | LR: 1.48e-04 | 701.4ms/step | 28.5 samples/s +Step 28170/44000 | Loss: 2.9495 | LR: 1.48e-04 | 700.5ms/step | 28.6 samples/s +Step 28180/44000 | Loss: 2.9020 | LR: 1.48e-04 | 701.0ms/step | 28.5 samples/s +Step 28190/44000 | Loss: 2.9509 | LR: 1.48e-04 | 702.3ms/step | 28.5 samples/s +Step 28200/44000 | Loss: 2.9137 | LR: 1.47e-04 | 700.0ms/step | 28.6 samples/s +Step 28210/44000 | Loss: 2.9732 | LR: 1.47e-04 | 702.6ms/step | 28.5 samples/s +Step 28220/44000 | Loss: 2.9477 | LR: 1.47e-04 | 701.5ms/step | 28.5 samples/s +Step 28230/44000 | Loss: 2.9407 | LR: 1.47e-04 | 701.8ms/step | 28.5 samples/s +Step 28240/44000 | Loss: 2.9711 | LR: 1.47e-04 | 699.9ms/step | 28.6 samples/s +Step 28250/44000 | Loss: 2.9558 | LR: 1.47e-04 | 706.9ms/step | 28.3 samples/s +Validation at step 28250: Loss = 2.9673, Perplexity = 19.44 +Step 28260/44000 | Loss: 2.9676 | LR: 1.46e-04 | 1273.4ms/step | 15.7 samples/s +Step 28270/44000 | Loss: 2.9527 | LR: 1.46e-04 | 702.3ms/step | 28.5 samples/s +Step 28280/44000 | Loss: 2.9434 | LR: 1.46e-04 | 699.3ms/step | 28.6 samples/s +Step 28290/44000 | Loss: 2.9376 | LR: 1.46e-04 | 703.3ms/step | 28.4 samples/s +Step 28300/44000 | Loss: 2.9794 | LR: 1.46e-04 | 705.1ms/step | 28.4 samples/s +Step 28310/44000 | Loss: 2.9460 | LR: 1.46e-04 | 700.9ms/step | 28.5 samples/s +Step 28320/44000 | Loss: 2.9461 | LR: 1.45e-04 | 701.8ms/step | 28.5 samples/s +Step 28330/44000 | Loss: 2.9634 | LR: 1.45e-04 | 701.0ms/step | 28.5 samples/s +Step 28340/44000 | Loss: 2.9458 | LR: 1.45e-04 | 702.2ms/step | 28.5 samples/s +Step 28350/44000 | Loss: 3.0132 | LR: 1.45e-04 | 703.5ms/step | 28.4 samples/s +Step 28360/44000 | Loss: 2.9913 | LR: 1.45e-04 | 697.9ms/step | 28.7 samples/s +Step 28370/44000 | Loss: 2.9522 | LR: 1.45e-04 | 701.7ms/step | 28.5 samples/s +Step 28380/44000 | Loss: 2.9760 | LR: 1.44e-04 | 702.8ms/step | 28.5 samples/s +Step 28390/44000 | Loss: 2.9738 | LR: 1.44e-04 | 702.4ms/step | 28.5 samples/s +Step 28400/44000 | Loss: 2.9699 | LR: 1.44e-04 | 700.1ms/step | 28.6 samples/s +Step 28410/44000 | Loss: 2.9783 | LR: 1.44e-04 | 704.4ms/step | 28.4 samples/s +Step 28420/44000 | Loss: 2.9312 | LR: 1.44e-04 | 703.1ms/step | 28.4 samples/s +Step 28430/44000 | Loss: 2.9415 | LR: 1.44e-04 | 700.5ms/step | 28.5 samples/s +Step 28440/44000 | Loss: 2.9596 | LR: 1.43e-04 | 699.5ms/step | 28.6 samples/s +Step 28450/44000 | Loss: 2.9145 | LR: 1.43e-04 | 700.3ms/step | 28.6 samples/s +Step 28460/44000 | Loss: 3.0103 | LR: 1.43e-04 | 699.6ms/step | 28.6 samples/s +Step 28470/44000 | Loss: 2.9517 | LR: 1.43e-04 | 702.2ms/step | 28.5 samples/s +Step 28480/44000 | Loss: 2.9861 | LR: 1.43e-04 | 701.3ms/step | 28.5 samples/s +Step 28490/44000 | Loss: 2.9853 | LR: 1.43e-04 | 700.4ms/step | 28.6 samples/s +Step 28500/44000 | Loss: 2.9975 | LR: 1.42e-04 | 702.4ms/step | 28.5 samples/s +Validation at step 28500: Loss = 2.9682, Perplexity = 19.46 +Step 28510/44000 | Loss: 2.9607 | LR: 1.42e-04 | 1268.4ms/step | 15.8 samples/s +Step 28520/44000 | Loss: 2.9771 | LR: 1.42e-04 | 704.6ms/step | 28.4 samples/s +Step 28530/44000 | Loss: 2.9705 | LR: 1.42e-04 | 703.1ms/step | 28.4 samples/s +Step 28540/44000 | Loss: 2.9512 | LR: 1.42e-04 | 702.0ms/step | 28.5 samples/s +Step 28550/44000 | Loss: 2.9767 | LR: 1.42e-04 | 699.8ms/step | 28.6 samples/s +Step 28560/44000 | Loss: 2.9771 | LR: 1.41e-04 | 702.5ms/step | 28.5 samples/s +Step 28570/44000 | Loss: 2.9183 | LR: 1.41e-04 | 702.0ms/step | 28.5 samples/s +Step 28580/44000 | Loss: 2.9276 | LR: 1.41e-04 | 701.5ms/step | 28.5 samples/s +Step 28590/44000 | Loss: 2.9728 | LR: 1.41e-04 | 702.8ms/step | 28.5 samples/s +Step 28600/44000 | Loss: 2.9473 | LR: 1.41e-04 | 701.9ms/step | 28.5 samples/s +Step 28610/44000 | Loss: 2.9204 | LR: 1.41e-04 | 703.5ms/step | 28.4 samples/s +Step 28620/44000 | Loss: 2.9500 | LR: 1.40e-04 | 702.8ms/step | 28.5 samples/s +Step 28630/44000 | Loss: 2.9546 | LR: 1.40e-04 | 701.6ms/step | 28.5 samples/s +Step 28640/44000 | Loss: 2.9405 | LR: 1.40e-04 | 701.7ms/step | 28.5 samples/s +Step 28650/44000 | Loss: 2.9052 | LR: 1.40e-04 | 701.1ms/step | 28.5 samples/s +Step 28660/44000 | Loss: 2.9808 | LR: 1.40e-04 | 702.0ms/step | 28.5 samples/s +Step 28670/44000 | Loss: 2.9855 | LR: 1.40e-04 | 702.8ms/step | 28.5 samples/s +Step 28680/44000 | Loss: 2.9604 | LR: 1.39e-04 | 703.4ms/step | 28.4 samples/s +Step 28690/44000 | Loss: 2.9640 | LR: 1.39e-04 | 703.1ms/step | 28.4 samples/s +Step 28700/44000 | Loss: 2.9469 | LR: 1.39e-04 | 698.8ms/step | 28.6 samples/s +Step 28710/44000 | Loss: 2.9807 | LR: 1.39e-04 | 701.3ms/step | 28.5 samples/s +Step 28720/44000 | Loss: 2.9590 | LR: 1.39e-04 | 702.2ms/step | 28.5 samples/s +Step 28730/44000 | Loss: 2.9799 | LR: 1.39e-04 | 701.3ms/step | 28.5 samples/s +Step 28740/44000 | Loss: 2.9949 | LR: 1.38e-04 | 699.5ms/step | 28.6 samples/s +Step 28750/44000 | Loss: 2.9444 | LR: 1.38e-04 | 703.9ms/step | 28.4 samples/s +Validation at step 28750: Loss = 2.9652, Perplexity = 19.40 +Step 28760/44000 | Loss: 2.9637 | LR: 1.38e-04 | 1271.5ms/step | 15.7 samples/s +Step 28770/44000 | Loss: 2.9536 | LR: 1.38e-04 | 704.4ms/step | 28.4 samples/s +Step 28780/44000 | Loss: 2.9593 | LR: 1.38e-04 | 702.5ms/step | 28.5 samples/s +Step 28790/44000 | Loss: 2.9804 | LR: 1.38e-04 | 699.9ms/step | 28.6 samples/s +Step 28800/44000 | Loss: 2.9617 | LR: 1.38e-04 | 700.5ms/step | 28.6 samples/s +Step 28810/44000 | Loss: 2.9438 | LR: 1.37e-04 | 704.2ms/step | 28.4 samples/s +Step 28820/44000 | Loss: 2.9941 | LR: 1.37e-04 | 702.9ms/step | 28.5 samples/s +Step 28830/44000 | Loss: 2.9320 | LR: 1.37e-04 | 699.6ms/step | 28.6 samples/s +Step 28840/44000 | Loss: 2.9592 | LR: 1.37e-04 | 701.9ms/step | 28.5 samples/s +Step 28850/44000 | Loss: 2.9175 | LR: 1.37e-04 | 702.8ms/step | 28.5 samples/s +Step 28860/44000 | Loss: 2.9342 | LR: 1.37e-04 | 699.4ms/step | 28.6 samples/s +Step 28870/44000 | Loss: 2.9747 | LR: 1.36e-04 | 701.4ms/step | 28.5 samples/s +Step 28880/44000 | Loss: 2.9745 | LR: 1.36e-04 | 701.9ms/step | 28.5 samples/s +Step 28890/44000 | Loss: 2.9307 | LR: 1.36e-04 | 698.1ms/step | 28.6 samples/s +Step 28900/44000 | Loss: 2.9346 | LR: 1.36e-04 | 699.5ms/step | 28.6 samples/s +Step 28910/44000 | Loss: 2.9980 | LR: 1.36e-04 | 701.0ms/step | 28.5 samples/s +Step 28920/44000 | Loss: 2.9428 | LR: 1.36e-04 | 701.8ms/step | 28.5 samples/s +Step 28930/44000 | Loss: 2.9146 | LR: 1.35e-04 | 702.9ms/step | 28.5 samples/s +Step 28940/44000 | Loss: 2.9564 | LR: 1.35e-04 | 703.9ms/step | 28.4 samples/s +Step 28950/44000 | Loss: 2.9789 | LR: 1.35e-04 | 704.0ms/step | 28.4 samples/s +Step 28960/44000 | Loss: 2.9356 | LR: 1.35e-04 | 702.2ms/step | 28.5 samples/s +Step 28970/44000 | Loss: 2.9586 | LR: 1.35e-04 | 703.8ms/step | 28.4 samples/s +Step 28980/44000 | Loss: 2.9679 | LR: 1.35e-04 | 701.3ms/step | 28.5 samples/s +Step 28990/44000 | Loss: 2.9980 | LR: 1.34e-04 | 703.6ms/step | 28.4 samples/s +Step 29000/44000 | Loss: 2.9144 | LR: 1.34e-04 | 700.5ms/step | 28.6 samples/s +Validation at step 29000: Loss = 2.9651, Perplexity = 19.40 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-29000 +Step 29010/44000 | Loss: 2.9331 | LR: 1.34e-04 | 1562.6ms/step | 12.8 samples/s +Step 29020/44000 | Loss: 2.9495 | LR: 1.34e-04 | 700.9ms/step | 28.5 samples/s +Step 29030/44000 | Loss: 2.9667 | LR: 1.34e-04 | 702.2ms/step | 28.5 samples/s +Step 29040/44000 | Loss: 2.9922 | LR: 1.34e-04 | 698.3ms/step | 28.6 samples/s +Step 29050/44000 | Loss: 2.9545 | LR: 1.33e-04 | 699.7ms/step | 28.6 samples/s +Step 29060/44000 | Loss: 2.9592 | LR: 1.33e-04 | 702.9ms/step | 28.5 samples/s +Step 29070/44000 | Loss: 2.9202 | LR: 1.33e-04 | 700.5ms/step | 28.5 samples/s +Step 29080/44000 | Loss: 2.9625 | LR: 1.33e-04 | 698.9ms/step | 28.6 samples/s +Step 29090/44000 | Loss: 2.9414 | LR: 1.33e-04 | 701.8ms/step | 28.5 samples/s +Step 29100/44000 | Loss: 2.9314 | LR: 1.33e-04 | 698.9ms/step | 28.6 samples/s +Step 29110/44000 | Loss: 2.9583 | LR: 1.33e-04 | 701.6ms/step | 28.5 samples/s +Step 29120/44000 | Loss: 2.9677 | LR: 1.32e-04 | 701.1ms/step | 28.5 samples/s +Step 29130/44000 | Loss: 2.9325 | LR: 1.32e-04 | 699.5ms/step | 28.6 samples/s +Step 29140/44000 | Loss: 2.9480 | LR: 1.32e-04 | 705.8ms/step | 28.3 samples/s +Step 29150/44000 | Loss: 2.9570 | LR: 1.32e-04 | 701.1ms/step | 28.5 samples/s +Step 29160/44000 | Loss: 2.9469 | LR: 1.32e-04 | 700.9ms/step | 28.5 samples/s +Step 29170/44000 | Loss: 2.9514 | LR: 1.32e-04 | 705.7ms/step | 28.3 samples/s +Step 29180/44000 | Loss: 2.9454 | LR: 1.31e-04 | 702.1ms/step | 28.5 samples/s +Step 29190/44000 | Loss: 2.9324 | LR: 1.31e-04 | 701.2ms/step | 28.5 samples/s +Step 29200/44000 | Loss: 2.9110 | LR: 1.31e-04 | 702.9ms/step | 28.5 samples/s +Step 29210/44000 | Loss: 2.9154 | LR: 1.31e-04 | 702.7ms/step | 28.5 samples/s +Step 29220/44000 | Loss: 2.9076 | LR: 1.31e-04 | 704.4ms/step | 28.4 samples/s +Step 29230/44000 | Loss: 2.9660 | LR: 1.31e-04 | 703.2ms/step | 28.4 samples/s +Step 29240/44000 | Loss: 2.9006 | LR: 1.30e-04 | 704.8ms/step | 28.4 samples/s +Step 29250/44000 | Loss: 2.9454 | LR: 1.30e-04 | 700.1ms/step | 28.6 samples/s +Validation at step 29250: Loss = 2.9638, Perplexity = 19.37 +Step 29260/44000 | Loss: 2.9457 | LR: 1.30e-04 | 1269.9ms/step | 15.7 samples/s +Step 29270/44000 | Loss: 2.9142 | LR: 1.30e-04 | 705.6ms/step | 28.3 samples/s +Step 29280/44000 | Loss: 2.9563 | LR: 1.30e-04 | 701.8ms/step | 28.5 samples/s +Step 29290/44000 | Loss: 2.9296 | LR: 1.30e-04 | 704.4ms/step | 28.4 samples/s +Step 29300/44000 | Loss: 2.9146 | LR: 1.29e-04 | 701.0ms/step | 28.5 samples/s +Step 29310/44000 | Loss: 2.9055 | LR: 1.29e-04 | 702.3ms/step | 28.5 samples/s +Step 29320/44000 | Loss: 2.8931 | LR: 1.29e-04 | 700.5ms/step | 28.6 samples/s +Step 29330/44000 | Loss: 2.9363 | LR: 1.29e-04 | 703.5ms/step | 28.4 samples/s +Step 29340/44000 | Loss: 3.0111 | LR: 1.29e-04 | 703.3ms/step | 28.4 samples/s +Step 29350/44000 | Loss: 2.9174 | LR: 1.29e-04 | 703.2ms/step | 28.4 samples/s +Step 29360/44000 | Loss: 2.9651 | LR: 1.29e-04 | 702.2ms/step | 28.5 samples/s +Step 29370/44000 | Loss: 2.9821 | LR: 1.28e-04 | 703.7ms/step | 28.4 samples/s +Step 29380/44000 | Loss: 2.9692 | LR: 1.28e-04 | 703.3ms/step | 28.4 samples/s +Step 29390/44000 | Loss: 2.9396 | LR: 1.28e-04 | 702.3ms/step | 28.5 samples/s +Step 29400/44000 | Loss: 2.9194 | LR: 1.28e-04 | 702.1ms/step | 28.5 samples/s +Step 29410/44000 | Loss: 2.9859 | LR: 1.28e-04 | 705.1ms/step | 28.4 samples/s +Step 29420/44000 | Loss: 2.9491 | LR: 1.28e-04 | 702.4ms/step | 28.5 samples/s +Step 29430/44000 | Loss: 2.9103 | LR: 1.27e-04 | 703.4ms/step | 28.4 samples/s +Step 29440/44000 | Loss: 2.9298 | LR: 1.27e-04 | 703.4ms/step | 28.4 samples/s +Step 29450/44000 | Loss: 2.9373 | LR: 1.27e-04 | 702.8ms/step | 28.5 samples/s +Step 29460/44000 | Loss: 2.9220 | LR: 1.27e-04 | 704.3ms/step | 28.4 samples/s +Step 29470/44000 | Loss: 2.9488 | LR: 1.27e-04 | 705.6ms/step | 28.3 samples/s +Step 29480/44000 | Loss: 2.9391 | LR: 1.27e-04 | 702.8ms/step | 28.5 samples/s +Step 29490/44000 | Loss: 2.9454 | LR: 1.26e-04 | 702.0ms/step | 28.5 samples/s +Step 29500/44000 | Loss: 2.9421 | LR: 1.26e-04 | 703.7ms/step | 28.4 samples/s +Validation at step 29500: Loss = 2.9628, Perplexity = 19.35 +Step 29510/44000 | Loss: 2.9162 | LR: 1.26e-04 | 1272.3ms/step | 15.7 samples/s +Step 29520/44000 | Loss: 2.9461 | LR: 1.26e-04 | 701.4ms/step | 28.5 samples/s +Step 29530/44000 | Loss: 2.9271 | LR: 1.26e-04 | 702.0ms/step | 28.5 samples/s +Step 29540/44000 | Loss: 2.9530 | LR: 1.26e-04 | 700.6ms/step | 28.5 samples/s +Step 29550/44000 | Loss: 2.9445 | LR: 1.26e-04 | 704.2ms/step | 28.4 samples/s +Step 29560/44000 | Loss: 2.9370 | LR: 1.25e-04 | 702.4ms/step | 28.5 samples/s +Step 29570/44000 | Loss: 2.9416 | LR: 1.25e-04 | 700.9ms/step | 28.5 samples/s +Step 29580/44000 | Loss: 2.9233 | LR: 1.25e-04 | 700.7ms/step | 28.5 samples/s +Step 29590/44000 | Loss: 2.9839 | LR: 1.25e-04 | 699.7ms/step | 28.6 samples/s +Step 29600/44000 | Loss: 2.9566 | LR: 1.25e-04 | 700.6ms/step | 28.5 samples/s +Step 29610/44000 | Loss: 2.9148 | LR: 1.25e-04 | 703.6ms/step | 28.4 samples/s +Step 29620/44000 | Loss: 2.9337 | LR: 1.24e-04 | 701.1ms/step | 28.5 samples/s +Step 29630/44000 | Loss: 2.9313 | LR: 1.24e-04 | 700.3ms/step | 28.6 samples/s +Step 29640/44000 | Loss: 2.9510 | LR: 1.24e-04 | 702.8ms/step | 28.5 samples/s +Step 29650/44000 | Loss: 2.9756 | LR: 1.24e-04 | 701.5ms/step | 28.5 samples/s +Step 29660/44000 | Loss: 2.9578 | LR: 1.24e-04 | 701.2ms/step | 28.5 samples/s +Step 29670/44000 | Loss: 2.9571 | LR: 1.24e-04 | 700.1ms/step | 28.6 samples/s +Step 29680/44000 | Loss: 2.9353 | LR: 1.23e-04 | 699.3ms/step | 28.6 samples/s +Step 29690/44000 | Loss: 2.9582 | LR: 1.23e-04 | 700.4ms/step | 28.6 samples/s +Step 29700/44000 | Loss: 2.9409 | LR: 1.23e-04 | 714.6ms/step | 28.0 samples/s +Step 29710/44000 | Loss: 2.9851 | LR: 1.23e-04 | 721.7ms/step | 27.7 samples/s +Step 29720/44000 | Loss: 2.9244 | LR: 1.23e-04 | 711.3ms/step | 28.1 samples/s +Step 29730/44000 | Loss: 2.9108 | LR: 1.23e-04 | 707.7ms/step | 28.3 samples/s +Step 29740/44000 | Loss: 2.9571 | LR: 1.23e-04 | 709.9ms/step | 28.2 samples/s +Step 29750/44000 | Loss: 2.9627 | LR: 1.22e-04 | 708.1ms/step | 28.2 samples/s +Validation at step 29750: Loss = 2.9617, Perplexity = 19.33 +Step 29760/44000 | Loss: 2.9969 | LR: 1.22e-04 | 1278.1ms/step | 15.6 samples/s +Step 29770/44000 | Loss: 2.9311 | LR: 1.22e-04 | 702.2ms/step | 28.5 samples/s +Step 29780/44000 | Loss: 2.9502 | LR: 1.22e-04 | 704.4ms/step | 28.4 samples/s +Step 29790/44000 | Loss: 2.9723 | LR: 1.22e-04 | 701.9ms/step | 28.5 samples/s +Step 29800/44000 | Loss: 2.9592 | LR: 1.22e-04 | 699.6ms/step | 28.6 samples/s +Step 29810/44000 | Loss: 2.9414 | LR: 1.21e-04 | 701.7ms/step | 28.5 samples/s +Step 29820/44000 | Loss: 2.9278 | LR: 1.21e-04 | 705.7ms/step | 28.3 samples/s +Step 29830/44000 | Loss: 2.9847 | LR: 1.21e-04 | 701.8ms/step | 28.5 samples/s +Step 29840/44000 | Loss: 2.9295 | LR: 1.21e-04 | 702.2ms/step | 28.5 samples/s +Step 29850/44000 | Loss: 2.9745 | LR: 1.21e-04 | 701.3ms/step | 28.5 samples/s +Step 29860/44000 | Loss: 2.9669 | LR: 1.21e-04 | 716.8ms/step | 27.9 samples/s +Step 29870/44000 | Loss: 2.9569 | LR: 1.21e-04 | 727.5ms/step | 27.5 samples/s +Step 29880/44000 | Loss: 2.9341 | LR: 1.20e-04 | 725.5ms/step | 27.6 samples/s +Step 29890/44000 | Loss: 2.9973 | LR: 1.20e-04 | 723.3ms/step | 27.7 samples/s +Step 29900/44000 | Loss: 2.9296 | LR: 1.20e-04 | 713.7ms/step | 28.0 samples/s +Step 29910/44000 | Loss: 2.9098 | LR: 1.20e-04 | 703.8ms/step | 28.4 samples/s +Step 29920/44000 | Loss: 2.9285 | LR: 1.20e-04 | 705.2ms/step | 28.4 samples/s +Step 29930/44000 | Loss: 2.9175 | LR: 1.20e-04 | 708.7ms/step | 28.2 samples/s +Step 29940/44000 | Loss: 2.9255 | LR: 1.19e-04 | 702.2ms/step | 28.5 samples/s +Step 29950/44000 | Loss: 2.9309 | LR: 1.19e-04 | 704.9ms/step | 28.4 samples/s +Step 29960/44000 | Loss: 2.9439 | LR: 1.19e-04 | 705.5ms/step | 28.3 samples/s +Step 29970/44000 | Loss: 2.8744 | LR: 1.19e-04 | 702.5ms/step | 28.5 samples/s +Step 29980/44000 | Loss: 2.9462 | LR: 1.19e-04 | 705.8ms/step | 28.3 samples/s +Step 29990/44000 | Loss: 2.9404 | LR: 1.19e-04 | 703.9ms/step | 28.4 samples/s +Step 30000/44000 | Loss: 2.9641 | LR: 1.19e-04 | 704.7ms/step | 28.4 samples/s +Validation at step 30000: Loss = 2.9601, Perplexity = 19.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-30000 +Step 30010/44000 | Loss: 2.9560 | LR: 1.18e-04 | 1564.3ms/step | 12.8 samples/s +Step 30020/44000 | Loss: 2.9389 | LR: 1.18e-04 | 702.0ms/step | 28.5 samples/s +Step 30030/44000 | Loss: 2.9134 | LR: 1.18e-04 | 701.3ms/step | 28.5 samples/s +Step 30040/44000 | Loss: 2.9498 | LR: 1.18e-04 | 700.6ms/step | 28.5 samples/s +Step 30050/44000 | Loss: 2.9532 | LR: 1.18e-04 | 699.2ms/step | 28.6 samples/s +Step 30060/44000 | Loss: 2.9571 | LR: 1.18e-04 | 699.5ms/step | 28.6 samples/s +Step 30070/44000 | Loss: 2.9177 | LR: 1.17e-04 | 701.7ms/step | 28.5 samples/s +Step 30080/44000 | Loss: 2.8888 | LR: 1.17e-04 | 707.4ms/step | 28.3 samples/s +Step 30090/44000 | Loss: 2.9360 | LR: 1.17e-04 | 700.7ms/step | 28.5 samples/s +Step 30100/44000 | Loss: 2.9596 | LR: 1.17e-04 | 700.4ms/step | 28.6 samples/s +Step 30110/44000 | Loss: 2.9211 | LR: 1.17e-04 | 702.0ms/step | 28.5 samples/s +Step 30120/44000 | Loss: 2.9463 | LR: 1.17e-04 | 703.7ms/step | 28.4 samples/s +Step 30130/44000 | Loss: 2.9343 | LR: 1.17e-04 | 702.2ms/step | 28.5 samples/s +Step 30140/44000 | Loss: 2.9401 | LR: 1.16e-04 | 703.9ms/step | 28.4 samples/s +Step 30150/44000 | Loss: 2.9335 | LR: 1.16e-04 | 702.2ms/step | 28.5 samples/s +Step 30160/44000 | Loss: 2.9293 | LR: 1.16e-04 | 704.9ms/step | 28.4 samples/s +Step 30170/44000 | Loss: 2.9488 | LR: 1.16e-04 | 700.7ms/step | 28.5 samples/s +Step 30180/44000 | Loss: 2.8576 | LR: 1.16e-04 | 703.3ms/step | 28.4 samples/s +Step 30190/44000 | Loss: 2.9926 | LR: 1.16e-04 | 706.1ms/step | 28.3 samples/s +Step 30200/44000 | Loss: 2.9356 | LR: 1.15e-04 | 702.6ms/step | 28.5 samples/s +Step 30210/44000 | Loss: 2.9242 | LR: 1.15e-04 | 703.1ms/step | 28.4 samples/s +Step 30220/44000 | Loss: 2.9592 | LR: 1.15e-04 | 700.1ms/step | 28.6 samples/s +Step 30230/44000 | Loss: 2.9006 | LR: 1.15e-04 | 701.3ms/step | 28.5 samples/s +Step 30240/44000 | Loss: 2.9532 | LR: 1.15e-04 | 702.8ms/step | 28.5 samples/s +Step 30250/44000 | Loss: 2.9764 | LR: 1.15e-04 | 700.3ms/step | 28.6 samples/s +Validation at step 30250: Loss = 2.9604, Perplexity = 19.31 +Step 30260/44000 | Loss: 2.9280 | LR: 1.15e-04 | 1271.9ms/step | 15.7 samples/s +Step 30270/44000 | Loss: 2.9308 | LR: 1.14e-04 | 700.0ms/step | 28.6 samples/s +Step 30280/44000 | Loss: 2.9401 | LR: 1.14e-04 | 703.0ms/step | 28.5 samples/s +Step 30290/44000 | Loss: 2.9449 | LR: 1.14e-04 | 702.7ms/step | 28.5 samples/s +Step 30300/44000 | Loss: 2.9683 | LR: 1.14e-04 | 705.6ms/step | 28.3 samples/s +Step 30310/44000 | Loss: 2.9595 | LR: 1.14e-04 | 702.0ms/step | 28.5 samples/s +Step 30320/44000 | Loss: 2.9442 | LR: 1.14e-04 | 703.0ms/step | 28.4 samples/s +Step 30330/44000 | Loss: 2.9110 | LR: 1.13e-04 | 701.3ms/step | 28.5 samples/s +Step 30340/44000 | Loss: 2.9421 | LR: 1.13e-04 | 705.3ms/step | 28.4 samples/s +Step 30350/44000 | Loss: 2.8883 | LR: 1.13e-04 | 701.9ms/step | 28.5 samples/s +Step 30360/44000 | Loss: 2.9201 | LR: 1.13e-04 | 703.1ms/step | 28.4 samples/s +Step 30370/44000 | Loss: 2.9384 | LR: 1.13e-04 | 703.7ms/step | 28.4 samples/s +Step 30380/44000 | Loss: 2.9661 | LR: 1.13e-04 | 706.9ms/step | 28.3 samples/s +Step 30390/44000 | Loss: 2.9056 | LR: 1.13e-04 | 703.2ms/step | 28.4 samples/s +Step 30400/44000 | Loss: 2.9289 | LR: 1.12e-04 | 703.6ms/step | 28.4 samples/s +Step 30410/44000 | Loss: 2.9539 | LR: 1.12e-04 | 704.9ms/step | 28.4 samples/s +Step 30420/44000 | Loss: 2.9420 | LR: 1.12e-04 | 705.0ms/step | 28.4 samples/s +Step 30430/44000 | Loss: 2.9377 | LR: 1.12e-04 | 701.6ms/step | 28.5 samples/s +Step 30440/44000 | Loss: 2.9234 | LR: 1.12e-04 | 703.3ms/step | 28.4 samples/s +Step 30450/44000 | Loss: 2.9274 | LR: 1.12e-04 | 699.6ms/step | 28.6 samples/s +Step 30460/44000 | Loss: 2.9285 | LR: 1.11e-04 | 704.4ms/step | 28.4 samples/s +Step 30470/44000 | Loss: 2.9314 | LR: 1.11e-04 | 702.7ms/step | 28.5 samples/s +Step 30480/44000 | Loss: 2.9112 | LR: 1.11e-04 | 703.3ms/step | 28.4 samples/s +Step 30490/44000 | Loss: 2.9271 | LR: 1.11e-04 | 703.5ms/step | 28.4 samples/s +Step 30500/44000 | Loss: 2.9429 | LR: 1.11e-04 | 702.5ms/step | 28.5 samples/s +Validation at step 30500: Loss = 2.9590, Perplexity = 19.28 +Step 30510/44000 | Loss: 2.9478 | LR: 1.11e-04 | 1268.6ms/step | 15.8 samples/s +Step 30520/44000 | Loss: 2.9196 | LR: 1.11e-04 | 701.6ms/step | 28.5 samples/s +Step 30530/44000 | Loss: 2.9422 | LR: 1.10e-04 | 701.3ms/step | 28.5 samples/s +Step 30540/44000 | Loss: 2.9771 | LR: 1.10e-04 | 704.1ms/step | 28.4 samples/s +Step 30550/44000 | Loss: 2.8928 | LR: 1.10e-04 | 701.5ms/step | 28.5 samples/s +Step 30560/44000 | Loss: 2.9783 | LR: 1.10e-04 | 702.6ms/step | 28.5 samples/s +Step 30570/44000 | Loss: 2.9044 | LR: 1.10e-04 | 702.9ms/step | 28.5 samples/s +Step 30580/44000 | Loss: 2.9252 | LR: 1.10e-04 | 705.7ms/step | 28.3 samples/s +Step 30590/44000 | Loss: 2.9321 | LR: 1.10e-04 | 703.4ms/step | 28.4 samples/s +Step 30600/44000 | Loss: 2.9840 | LR: 1.09e-04 | 705.2ms/step | 28.4 samples/s +Step 30610/44000 | Loss: 2.9128 | LR: 1.09e-04 | 703.3ms/step | 28.4 samples/s +Step 30620/44000 | Loss: 2.8946 | LR: 1.09e-04 | 700.1ms/step | 28.6 samples/s +Step 30630/44000 | Loss: 2.9273 | LR: 1.09e-04 | 704.6ms/step | 28.4 samples/s +Step 30640/44000 | Loss: 2.9647 | LR: 1.09e-04 | 703.8ms/step | 28.4 samples/s +Step 30650/44000 | Loss: 2.8920 | LR: 1.09e-04 | 701.2ms/step | 28.5 samples/s +Step 30660/44000 | Loss: 2.9766 | LR: 1.08e-04 | 701.8ms/step | 28.5 samples/s +Step 30670/44000 | Loss: 2.9200 | LR: 1.08e-04 | 702.6ms/step | 28.5 samples/s +Step 30680/44000 | Loss: 2.9586 | LR: 1.08e-04 | 701.7ms/step | 28.5 samples/s +Step 30690/44000 | Loss: 2.9118 | LR: 1.08e-04 | 700.7ms/step | 28.5 samples/s +Step 30700/44000 | Loss: 2.9898 | LR: 1.08e-04 | 702.2ms/step | 28.5 samples/s +Step 30710/44000 | Loss: 2.9263 | LR: 1.08e-04 | 701.0ms/step | 28.5 samples/s +Step 30720/44000 | Loss: 2.9091 | LR: 1.08e-04 | 703.3ms/step | 28.4 samples/s +Step 30730/44000 | Loss: 2.9755 | LR: 1.07e-04 | 701.7ms/step | 28.5 samples/s +Step 30740/44000 | Loss: 2.9682 | LR: 1.07e-04 | 699.8ms/step | 28.6 samples/s +Step 30750/44000 | Loss: 2.9023 | LR: 1.07e-04 | 698.9ms/step | 28.6 samples/s +Validation at step 30750: Loss = 2.9586, Perplexity = 19.27 +Step 30760/44000 | Loss: 2.9819 | LR: 1.07e-04 | 1269.7ms/step | 15.8 samples/s +Step 30770/44000 | Loss: 2.9361 | LR: 1.07e-04 | 705.6ms/step | 28.3 samples/s +Step 30780/44000 | Loss: 2.9641 | LR: 1.07e-04 | 704.4ms/step | 28.4 samples/s +Step 30790/44000 | Loss: 2.9502 | LR: 1.07e-04 | 703.3ms/step | 28.4 samples/s +Step 30800/44000 | Loss: 2.9566 | LR: 1.06e-04 | 703.8ms/step | 28.4 samples/s +Step 30810/44000 | Loss: 2.9003 | LR: 1.06e-04 | 702.2ms/step | 28.5 samples/s +Step 30820/44000 | Loss: 2.9265 | LR: 1.06e-04 | 701.1ms/step | 28.5 samples/s +Step 30830/44000 | Loss: 2.9519 | LR: 1.06e-04 | 702.0ms/step | 28.5 samples/s +Step 30840/44000 | Loss: 2.9341 | LR: 1.06e-04 | 700.0ms/step | 28.6 samples/s +Step 30850/44000 | Loss: 2.9458 | LR: 1.06e-04 | 702.0ms/step | 28.5 samples/s +Step 30860/44000 | Loss: 2.9414 | LR: 1.05e-04 | 703.7ms/step | 28.4 samples/s +Step 30870/44000 | Loss: 2.8903 | LR: 1.05e-04 | 706.9ms/step | 28.3 samples/s +Step 30880/44000 | Loss: 2.9800 | LR: 1.05e-04 | 699.8ms/step | 28.6 samples/s +Step 30890/44000 | Loss: 2.9543 | LR: 1.05e-04 | 700.0ms/step | 28.6 samples/s +Step 30900/44000 | Loss: 2.9663 | LR: 1.05e-04 | 701.4ms/step | 28.5 samples/s +Step 30910/44000 | Loss: 2.9277 | LR: 1.05e-04 | 702.2ms/step | 28.5 samples/s +Step 30920/44000 | Loss: 2.9260 | LR: 1.05e-04 | 701.2ms/step | 28.5 samples/s +Step 30930/44000 | Loss: 2.9617 | LR: 1.04e-04 | 703.5ms/step | 28.4 samples/s +Step 30940/44000 | Loss: 2.9291 | LR: 1.04e-04 | 705.9ms/step | 28.3 samples/s +Step 30950/44000 | Loss: 2.9705 | LR: 1.04e-04 | 699.0ms/step | 28.6 samples/s +Step 30960/44000 | Loss: 2.8819 | LR: 1.04e-04 | 702.0ms/step | 28.5 samples/s +Step 30970/44000 | Loss: 2.9249 | LR: 1.04e-04 | 700.7ms/step | 28.5 samples/s +Step 30980/44000 | Loss: 2.9434 | LR: 1.04e-04 | 704.4ms/step | 28.4 samples/s +Step 30990/44000 | Loss: 2.9390 | LR: 1.04e-04 | 701.9ms/step | 28.5 samples/s +Step 31000/44000 | Loss: 2.9466 | LR: 1.03e-04 | 702.1ms/step | 28.5 samples/s +Validation at step 31000: Loss = 2.9570, Perplexity = 19.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-31000 +Step 31010/44000 | Loss: 2.9259 | LR: 1.03e-04 | 1569.7ms/step | 12.7 samples/s +Step 31020/44000 | Loss: 2.9583 | LR: 1.03e-04 | 701.7ms/step | 28.5 samples/s +Step 31030/44000 | Loss: 2.9388 | LR: 1.03e-04 | 704.5ms/step | 28.4 samples/s +Step 31040/44000 | Loss: 2.9322 | LR: 1.03e-04 | 702.8ms/step | 28.5 samples/s +Step 31050/44000 | Loss: 2.9033 | LR: 1.03e-04 | 701.6ms/step | 28.5 samples/s +Step 31060/44000 | Loss: 2.9192 | LR: 1.03e-04 | 700.7ms/step | 28.5 samples/s +Step 31070/44000 | Loss: 2.9271 | LR: 1.02e-04 | 704.9ms/step | 28.4 samples/s +Step 31080/44000 | Loss: 2.9384 | LR: 1.02e-04 | 702.0ms/step | 28.5 samples/s +Step 31090/44000 | Loss: 2.8979 | LR: 1.02e-04 | 703.8ms/step | 28.4 samples/s +Step 31100/44000 | Loss: 2.9821 | LR: 1.02e-04 | 703.9ms/step | 28.4 samples/s +Step 31110/44000 | Loss: 2.9054 | LR: 1.02e-04 | 700.0ms/step | 28.6 samples/s +Step 31120/44000 | Loss: 2.9446 | LR: 1.02e-04 | 701.0ms/step | 28.5 samples/s +Step 31130/44000 | Loss: 2.9746 | LR: 1.02e-04 | 700.0ms/step | 28.6 samples/s +Step 31140/44000 | Loss: 2.9626 | LR: 1.01e-04 | 703.3ms/step | 28.4 samples/s +Step 31150/44000 | Loss: 2.9513 | LR: 1.01e-04 | 704.2ms/step | 28.4 samples/s +Step 31160/44000 | Loss: 2.9431 | LR: 1.01e-04 | 699.3ms/step | 28.6 samples/s +Step 31170/44000 | Loss: 2.9196 | LR: 1.01e-04 | 702.0ms/step | 28.5 samples/s +Step 31180/44000 | Loss: 2.9895 | LR: 1.01e-04 | 701.8ms/step | 28.5 samples/s +Step 31190/44000 | Loss: 2.9336 | LR: 1.01e-04 | 704.8ms/step | 28.4 samples/s +Step 31200/44000 | Loss: 2.9316 | LR: 1.00e-04 | 701.3ms/step | 28.5 samples/s +Step 31210/44000 | Loss: 2.9844 | LR: 1.00e-04 | 699.7ms/step | 28.6 samples/s +Step 31220/44000 | Loss: 2.9261 | LR: 1.00e-04 | 702.9ms/step | 28.5 samples/s +Step 31230/44000 | Loss: 2.9402 | LR: 1.00e-04 | 702.1ms/step | 28.5 samples/s +Step 31240/44000 | Loss: 2.9670 | LR: 9.99e-05 | 701.9ms/step | 28.5 samples/s +Step 31250/44000 | Loss: 2.9365 | LR: 9.98e-05 | 704.2ms/step | 28.4 samples/s +Validation at step 31250: Loss = 2.9566, Perplexity = 19.23 +Step 31260/44000 | Loss: 2.9301 | LR: 9.96e-05 | 1269.6ms/step | 15.8 samples/s +Step 31270/44000 | Loss: 2.9492 | LR: 9.95e-05 | 701.9ms/step | 28.5 samples/s +Step 31280/44000 | Loss: 2.9025 | LR: 9.93e-05 | 702.4ms/step | 28.5 samples/s +Step 31290/44000 | Loss: 2.9603 | LR: 9.92e-05 | 701.1ms/step | 28.5 samples/s +Step 31300/44000 | Loss: 2.9246 | LR: 9.90e-05 | 702.9ms/step | 28.5 samples/s +Step 31310/44000 | Loss: 2.9605 | LR: 9.89e-05 | 702.4ms/step | 28.5 samples/s +Step 31320/44000 | Loss: 2.9290 | LR: 9.88e-05 | 701.4ms/step | 28.5 samples/s +Step 31330/44000 | Loss: 2.9350 | LR: 9.86e-05 | 701.9ms/step | 28.5 samples/s +Step 31340/44000 | Loss: 2.9289 | LR: 9.85e-05 | 700.0ms/step | 28.6 samples/s +Step 31350/44000 | Loss: 2.9416 | LR: 9.83e-05 | 700.1ms/step | 28.6 samples/s +Step 31360/44000 | Loss: 2.9484 | LR: 9.82e-05 | 699.3ms/step | 28.6 samples/s +Step 31370/44000 | Loss: 2.9282 | LR: 9.80e-05 | 703.7ms/step | 28.4 samples/s +Step 31380/44000 | Loss: 2.9425 | LR: 9.79e-05 | 701.3ms/step | 28.5 samples/s +Step 31390/44000 | Loss: 2.8922 | LR: 9.77e-05 | 701.7ms/step | 28.5 samples/s +Step 31400/44000 | Loss: 2.9597 | LR: 9.76e-05 | 702.2ms/step | 28.5 samples/s +Step 31410/44000 | Loss: 2.8979 | LR: 9.75e-05 | 702.4ms/step | 28.5 samples/s +Step 31420/44000 | Loss: 2.9456 | LR: 9.73e-05 | 701.2ms/step | 28.5 samples/s +Step 31430/44000 | Loss: 2.9258 | LR: 9.72e-05 | 698.3ms/step | 28.6 samples/s +Step 31440/44000 | Loss: 2.9480 | LR: 9.70e-05 | 702.3ms/step | 28.5 samples/s +Step 31450/44000 | Loss: 2.9388 | LR: 9.69e-05 | 701.7ms/step | 28.5 samples/s +Step 31460/44000 | Loss: 2.9150 | LR: 9.67e-05 | 700.8ms/step | 28.5 samples/s +Step 31470/44000 | Loss: 2.9593 | LR: 9.66e-05 | 701.0ms/step | 28.5 samples/s +Step 31480/44000 | Loss: 2.9184 | LR: 9.65e-05 | 702.5ms/step | 28.5 samples/s +Step 31490/44000 | Loss: 2.9466 | LR: 9.63e-05 | 701.1ms/step | 28.5 samples/s +Step 31500/44000 | Loss: 2.9491 | LR: 9.62e-05 | 700.1ms/step | 28.6 samples/s +Validation at step 31500: Loss = 2.9559, Perplexity = 19.22 +Step 31510/44000 | Loss: 2.9120 | LR: 9.60e-05 | 1268.5ms/step | 15.8 samples/s +Step 31520/44000 | Loss: 2.9374 | LR: 9.59e-05 | 701.8ms/step | 28.5 samples/s +Step 31530/44000 | Loss: 2.9324 | LR: 9.57e-05 | 701.7ms/step | 28.5 samples/s +Step 31540/44000 | Loss: 2.9469 | LR: 9.56e-05 | 700.2ms/step | 28.6 samples/s +Step 31550/44000 | Loss: 2.9229 | LR: 9.54e-05 | 704.7ms/step | 28.4 samples/s +Step 31560/44000 | Loss: 2.9911 | LR: 9.53e-05 | 701.6ms/step | 28.5 samples/s +Step 31570/44000 | Loss: 2.9409 | LR: 9.52e-05 | 703.7ms/step | 28.4 samples/s +Step 31580/44000 | Loss: 2.9443 | LR: 9.50e-05 | 700.4ms/step | 28.6 samples/s +Step 31590/44000 | Loss: 2.9476 | LR: 9.49e-05 | 699.8ms/step | 28.6 samples/s +Step 31600/44000 | Loss: 2.9234 | LR: 9.47e-05 | 699.9ms/step | 28.6 samples/s +Step 31610/44000 | Loss: 2.9312 | LR: 9.46e-05 | 699.5ms/step | 28.6 samples/s +Step 31620/44000 | Loss: 2.8984 | LR: 9.45e-05 | 699.1ms/step | 28.6 samples/s +Step 31630/44000 | Loss: 2.9350 | LR: 9.43e-05 | 701.4ms/step | 28.5 samples/s +Step 31640/44000 | Loss: 2.9565 | LR: 9.42e-05 | 701.6ms/step | 28.5 samples/s +Step 31650/44000 | Loss: 2.9217 | LR: 9.40e-05 | 704.7ms/step | 28.4 samples/s +Step 31660/44000 | Loss: 2.9489 | LR: 9.39e-05 | 697.8ms/step | 28.7 samples/s +Step 31670/44000 | Loss: 2.9451 | LR: 9.37e-05 | 702.3ms/step | 28.5 samples/s +Step 31680/44000 | Loss: 2.8894 | LR: 9.36e-05 | 699.1ms/step | 28.6 samples/s +Step 31690/44000 | Loss: 2.8823 | LR: 9.35e-05 | 700.6ms/step | 28.5 samples/s +Step 31700/44000 | Loss: 2.9603 | LR: 9.33e-05 | 700.5ms/step | 28.6 samples/s +Step 31710/44000 | Loss: 2.9224 | LR: 9.32e-05 | 699.9ms/step | 28.6 samples/s +Step 31720/44000 | Loss: 2.9624 | LR: 9.30e-05 | 701.8ms/step | 28.5 samples/s +Step 31730/44000 | Loss: 2.9371 | LR: 9.29e-05 | 698.7ms/step | 28.6 samples/s +Step 31740/44000 | Loss: 2.9344 | LR: 9.28e-05 | 702.4ms/step | 28.5 samples/s +Step 31750/44000 | Loss: 2.9711 | LR: 9.26e-05 | 703.0ms/step | 28.4 samples/s +Validation at step 31750: Loss = 2.9540, Perplexity = 19.18 +Step 31760/44000 | Loss: 2.9624 | LR: 9.25e-05 | 1268.6ms/step | 15.8 samples/s +Step 31770/44000 | Loss: 2.9875 | LR: 9.23e-05 | 703.0ms/step | 28.4 samples/s +Step 31780/44000 | Loss: 2.9769 | LR: 9.22e-05 | 701.0ms/step | 28.5 samples/s +Step 31790/44000 | Loss: 2.9222 | LR: 9.20e-05 | 702.4ms/step | 28.5 samples/s +Step 31800/44000 | Loss: 2.9209 | LR: 9.19e-05 | 700.8ms/step | 28.5 samples/s +Step 31810/44000 | Loss: 2.9318 | LR: 9.18e-05 | 700.2ms/step | 28.6 samples/s +Step 31820/44000 | Loss: 2.9337 | LR: 9.16e-05 | 702.4ms/step | 28.5 samples/s +Step 31830/44000 | Loss: 2.9593 | LR: 9.15e-05 | 701.5ms/step | 28.5 samples/s +Step 31840/44000 | Loss: 2.9407 | LR: 9.13e-05 | 701.1ms/step | 28.5 samples/s +Step 31850/44000 | Loss: 2.8954 | LR: 9.12e-05 | 702.9ms/step | 28.5 samples/s +Step 31860/44000 | Loss: 2.9245 | LR: 9.11e-05 | 703.6ms/step | 28.4 samples/s +Step 31870/44000 | Loss: 2.9301 | LR: 9.09e-05 | 702.1ms/step | 28.5 samples/s +Step 31880/44000 | Loss: 2.9312 | LR: 9.08e-05 | 698.2ms/step | 28.6 samples/s +Step 31890/44000 | Loss: 2.9269 | LR: 9.06e-05 | 702.7ms/step | 28.5 samples/s +Step 31900/44000 | Loss: 2.9066 | LR: 9.05e-05 | 701.4ms/step | 28.5 samples/s +Step 31910/44000 | Loss: 2.9716 | LR: 9.04e-05 | 702.8ms/step | 28.5 samples/s +Step 31920/44000 | Loss: 2.9416 | LR: 9.02e-05 | 697.8ms/step | 28.7 samples/s +Step 31930/44000 | Loss: 2.9084 | LR: 9.01e-05 | 699.8ms/step | 28.6 samples/s +Step 31940/44000 | Loss: 2.9385 | LR: 8.99e-05 | 703.8ms/step | 28.4 samples/s +Step 31950/44000 | Loss: 2.9365 | LR: 8.98e-05 | 699.3ms/step | 28.6 samples/s +Step 31960/44000 | Loss: 2.8975 | LR: 8.97e-05 | 701.1ms/step | 28.5 samples/s +Step 31970/44000 | Loss: 2.9063 | LR: 8.95e-05 | 704.6ms/step | 28.4 samples/s +Step 31980/44000 | Loss: 2.9317 | LR: 8.94e-05 | 703.1ms/step | 28.4 samples/s +Step 31990/44000 | Loss: 2.9399 | LR: 8.92e-05 | 699.6ms/step | 28.6 samples/s +Step 32000/44000 | Loss: 2.8966 | LR: 8.91e-05 | 700.2ms/step | 28.6 samples/s +Validation at step 32000: Loss = 2.9551, Perplexity = 19.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-32000 +Step 32010/44000 | Loss: 2.9452 | LR: 8.90e-05 | 1566.1ms/step | 12.8 samples/s +Step 32020/44000 | Loss: 2.9434 | LR: 8.88e-05 | 699.7ms/step | 28.6 samples/s +Step 32030/44000 | Loss: 3.0128 | LR: 8.87e-05 | 699.5ms/step | 28.6 samples/s +Step 32040/44000 | Loss: 2.9422 | LR: 8.86e-05 | 702.3ms/step | 28.5 samples/s +Step 32050/44000 | Loss: 2.9484 | LR: 8.84e-05 | 700.2ms/step | 28.6 samples/s +Step 32060/44000 | Loss: 2.9613 | LR: 8.83e-05 | 698.0ms/step | 28.7 samples/s +Step 32070/44000 | Loss: 2.9470 | LR: 8.81e-05 | 702.4ms/step | 28.5 samples/s +Step 32080/44000 | Loss: 2.9196 | LR: 8.80e-05 | 700.2ms/step | 28.6 samples/s +Step 32090/44000 | Loss: 2.9696 | LR: 8.79e-05 | 699.6ms/step | 28.6 samples/s +Step 32100/44000 | Loss: 2.9644 | LR: 8.77e-05 | 700.2ms/step | 28.6 samples/s +Step 32110/44000 | Loss: 2.9079 | LR: 8.76e-05 | 699.8ms/step | 28.6 samples/s +Step 32120/44000 | Loss: 2.9635 | LR: 8.74e-05 | 700.8ms/step | 28.5 samples/s +Step 32130/44000 | Loss: 2.9203 | LR: 8.73e-05 | 702.7ms/step | 28.5 samples/s +Step 32140/44000 | Loss: 2.9043 | LR: 8.72e-05 | 701.1ms/step | 28.5 samples/s +Step 32150/44000 | Loss: 2.9256 | LR: 8.70e-05 | 702.2ms/step | 28.5 samples/s +Step 32160/44000 | Loss: 2.9168 | LR: 8.69e-05 | 704.4ms/step | 28.4 samples/s +Step 32170/44000 | Loss: 2.9213 | LR: 8.68e-05 | 703.0ms/step | 28.4 samples/s +Step 32180/44000 | Loss: 2.9378 | LR: 8.66e-05 | 700.5ms/step | 28.6 samples/s +Step 32190/44000 | Loss: 2.9548 | LR: 8.65e-05 | 701.8ms/step | 28.5 samples/s +Step 32200/44000 | Loss: 2.9000 | LR: 8.63e-05 | 701.6ms/step | 28.5 samples/s +Step 32210/44000 | Loss: 2.9668 | LR: 8.62e-05 | 700.1ms/step | 28.6 samples/s +Step 32220/44000 | Loss: 2.9388 | LR: 8.61e-05 | 703.0ms/step | 28.4 samples/s +Step 32230/44000 | Loss: 2.9172 | LR: 8.59e-05 | 703.3ms/step | 28.4 samples/s +Step 32240/44000 | Loss: 2.9580 | LR: 8.58e-05 | 702.7ms/step | 28.5 samples/s +Step 32250/44000 | Loss: 2.9434 | LR: 8.57e-05 | 702.9ms/step | 28.5 samples/s +Validation at step 32250: Loss = 2.9532, Perplexity = 19.17 +Step 32260/44000 | Loss: 2.9571 | LR: 8.55e-05 | 1272.3ms/step | 15.7 samples/s +Step 32270/44000 | Loss: 2.9039 | LR: 8.54e-05 | 701.7ms/step | 28.5 samples/s +Step 32280/44000 | Loss: 2.9637 | LR: 8.53e-05 | 701.5ms/step | 28.5 samples/s +Step 32290/44000 | Loss: 2.9492 | LR: 8.51e-05 | 698.4ms/step | 28.6 samples/s +Step 32300/44000 | Loss: 2.9671 | LR: 8.50e-05 | 699.8ms/step | 28.6 samples/s +Step 32310/44000 | Loss: 2.9623 | LR: 8.48e-05 | 700.2ms/step | 28.6 samples/s +Step 32320/44000 | Loss: 2.9687 | LR: 8.47e-05 | 702.3ms/step | 28.5 samples/s +Step 32330/44000 | Loss: 2.9344 | LR: 8.46e-05 | 704.9ms/step | 28.4 samples/s +Step 32340/44000 | Loss: 2.9304 | LR: 8.44e-05 | 699.3ms/step | 28.6 samples/s +Step 32350/44000 | Loss: 2.9058 | LR: 8.43e-05 | 703.3ms/step | 28.4 samples/s +Step 32360/44000 | Loss: 2.9353 | LR: 8.42e-05 | 702.9ms/step | 28.5 samples/s +Step 32370/44000 | Loss: 2.9823 | LR: 8.40e-05 | 702.0ms/step | 28.5 samples/s +Step 32380/44000 | Loss: 2.9117 | LR: 8.39e-05 | 699.5ms/step | 28.6 samples/s +Step 32390/44000 | Loss: 2.9493 | LR: 8.38e-05 | 700.6ms/step | 28.5 samples/s +Step 32400/44000 | Loss: 2.9531 | LR: 8.36e-05 | 702.2ms/step | 28.5 samples/s +Step 32410/44000 | Loss: 2.9802 | LR: 8.35e-05 | 702.7ms/step | 28.5 samples/s +Step 32420/44000 | Loss: 2.9553 | LR: 8.33e-05 | 698.6ms/step | 28.6 samples/s +Step 32430/44000 | Loss: 2.9202 | LR: 8.32e-05 | 702.3ms/step | 28.5 samples/s +Step 32440/44000 | Loss: 2.9118 | LR: 8.31e-05 | 702.4ms/step | 28.5 samples/s +Step 32450/44000 | Loss: 2.9896 | LR: 8.29e-05 | 700.4ms/step | 28.6 samples/s +Step 32460/44000 | Loss: 2.9879 | LR: 8.28e-05 | 702.0ms/step | 28.5 samples/s +Step 32470/44000 | Loss: 2.9206 | LR: 8.27e-05 | 701.5ms/step | 28.5 samples/s +Step 32480/44000 | Loss: 2.9738 | LR: 8.25e-05 | 703.5ms/step | 28.4 samples/s +Step 32490/44000 | Loss: 2.9028 | LR: 8.24e-05 | 703.8ms/step | 28.4 samples/s +Step 32500/44000 | Loss: 2.9406 | LR: 8.23e-05 | 700.5ms/step | 28.6 samples/s +Validation at step 32500: Loss = 2.9547, Perplexity = 19.20 +Step 32510/44000 | Loss: 2.9184 | LR: 8.21e-05 | 1269.4ms/step | 15.8 samples/s +Step 32520/44000 | Loss: 2.9481 | LR: 8.20e-05 | 702.6ms/step | 28.5 samples/s +Step 32530/44000 | Loss: 2.9213 | LR: 8.19e-05 | 706.5ms/step | 28.3 samples/s +Step 32540/44000 | Loss: 2.9246 | LR: 8.17e-05 | 704.4ms/step | 28.4 samples/s +Step 32550/44000 | Loss: 2.9399 | LR: 8.16e-05 | 702.6ms/step | 28.5 samples/s +Step 32560/44000 | Loss: 2.9577 | LR: 8.15e-05 | 702.4ms/step | 28.5 samples/s +Step 32570/44000 | Loss: 2.9440 | LR: 8.13e-05 | 700.8ms/step | 28.5 samples/s +Step 32580/44000 | Loss: 2.9546 | LR: 8.12e-05 | 702.7ms/step | 28.5 samples/s +Step 32590/44000 | Loss: 2.9546 | LR: 8.11e-05 | 698.6ms/step | 28.6 samples/s +Step 32600/44000 | Loss: 2.9196 | LR: 8.09e-05 | 703.6ms/step | 28.4 samples/s +Step 32610/44000 | Loss: 2.9391 | LR: 8.08e-05 | 700.3ms/step | 28.6 samples/s +Step 32620/44000 | Loss: 2.9530 | LR: 8.07e-05 | 699.7ms/step | 28.6 samples/s +Step 32630/44000 | Loss: 2.9334 | LR: 8.05e-05 | 699.8ms/step | 28.6 samples/s +Step 32640/44000 | Loss: 2.9138 | LR: 8.04e-05 | 702.9ms/step | 28.5 samples/s +Step 32650/44000 | Loss: 2.9353 | LR: 8.03e-05 | 699.4ms/step | 28.6 samples/s +Step 32660/44000 | Loss: 2.9285 | LR: 8.01e-05 | 704.2ms/step | 28.4 samples/s +Step 32670/44000 | Loss: 2.9699 | LR: 8.00e-05 | 698.5ms/step | 28.6 samples/s +Step 32680/44000 | Loss: 2.9293 | LR: 7.99e-05 | 700.9ms/step | 28.5 samples/s +Step 32690/44000 | Loss: 2.9556 | LR: 7.97e-05 | 700.3ms/step | 28.6 samples/s +Step 32700/44000 | Loss: 2.9599 | LR: 7.96e-05 | 698.3ms/step | 28.6 samples/s +Step 32710/44000 | Loss: 2.9271 | LR: 7.95e-05 | 698.2ms/step | 28.6 samples/s +Step 32720/44000 | Loss: 2.9516 | LR: 7.93e-05 | 701.6ms/step | 28.5 samples/s +Step 32730/44000 | Loss: 2.9261 | LR: 7.92e-05 | 701.6ms/step | 28.5 samples/s +Step 32740/44000 | Loss: 2.9456 | LR: 7.91e-05 | 700.5ms/step | 28.6 samples/s +Step 32750/44000 | Loss: 2.9437 | LR: 7.89e-05 | 699.2ms/step | 28.6 samples/s +Validation at step 32750: Loss = 2.9533, Perplexity = 19.17 +Step 32760/44000 | Loss: 2.9323 | LR: 7.88e-05 | 1271.7ms/step | 15.7 samples/s +Step 32770/44000 | Loss: 2.9442 | LR: 7.87e-05 | 700.4ms/step | 28.6 samples/s +Step 32780/44000 | Loss: 2.9086 | LR: 7.85e-05 | 700.6ms/step | 28.5 samples/s +Step 32790/44000 | Loss: 2.9530 | LR: 7.84e-05 | 701.5ms/step | 28.5 samples/s +Step 32800/44000 | Loss: 2.9128 | LR: 7.83e-05 | 701.0ms/step | 28.5 samples/s +Step 32810/44000 | Loss: 2.9240 | LR: 7.81e-05 | 697.6ms/step | 28.7 samples/s +Step 32820/44000 | Loss: 2.9150 | LR: 7.80e-05 | 696.4ms/step | 28.7 samples/s +Step 32830/44000 | Loss: 2.9642 | LR: 7.79e-05 | 699.3ms/step | 28.6 samples/s +Step 32840/44000 | Loss: 2.9556 | LR: 7.77e-05 | 702.4ms/step | 28.5 samples/s +Step 32850/44000 | Loss: 2.9604 | LR: 7.76e-05 | 703.8ms/step | 28.4 samples/s +Step 32860/44000 | Loss: 2.9224 | LR: 7.75e-05 | 703.2ms/step | 28.4 samples/s +Step 32870/44000 | Loss: 2.9141 | LR: 7.73e-05 | 703.4ms/step | 28.4 samples/s +Step 32880/44000 | Loss: 2.9416 | LR: 7.72e-05 | 700.3ms/step | 28.6 samples/s +Step 32890/44000 | Loss: 2.9646 | LR: 7.71e-05 | 702.7ms/step | 28.5 samples/s +Step 32900/44000 | Loss: 2.9145 | LR: 7.70e-05 | 702.4ms/step | 28.5 samples/s +Step 32910/44000 | Loss: 2.9754 | LR: 7.68e-05 | 702.3ms/step | 28.5 samples/s +Step 32920/44000 | Loss: 2.9650 | LR: 7.67e-05 | 699.7ms/step | 28.6 samples/s +Step 32930/44000 | Loss: 2.9395 | LR: 7.66e-05 | 701.3ms/step | 28.5 samples/s +Step 32940/44000 | Loss: 2.9424 | LR: 7.64e-05 | 702.7ms/step | 28.5 samples/s +Step 32950/44000 | Loss: 2.9072 | LR: 7.63e-05 | 702.9ms/step | 28.5 samples/s +Step 32960/44000 | Loss: 2.9169 | LR: 7.62e-05 | 702.6ms/step | 28.5 samples/s +Step 32970/44000 | Loss: 2.9068 | LR: 7.60e-05 | 702.0ms/step | 28.5 samples/s +Step 32980/44000 | Loss: 2.9433 | LR: 7.59e-05 | 702.5ms/step | 28.5 samples/s +Step 32990/44000 | Loss: 2.9641 | LR: 7.58e-05 | 702.2ms/step | 28.5 samples/s +Step 33000/44000 | Loss: 2.9236 | LR: 7.56e-05 | 699.6ms/step | 28.6 samples/s +Validation at step 33000: Loss = 2.9537, Perplexity = 19.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-33000 +Step 33010/44000 | Loss: 2.9660 | LR: 7.55e-05 | 1564.7ms/step | 12.8 samples/s +Step 33020/44000 | Loss: 2.9554 | LR: 7.54e-05 | 700.9ms/step | 28.5 samples/s +Step 33030/44000 | Loss: 2.9503 | LR: 7.53e-05 | 701.4ms/step | 28.5 samples/s +Step 33040/44000 | Loss: 2.9480 | LR: 7.51e-05 | 703.6ms/step | 28.4 samples/s +Step 33050/44000 | Loss: 2.9988 | LR: 7.50e-05 | 702.0ms/step | 28.5 samples/s +Step 33060/44000 | Loss: 2.9227 | LR: 7.49e-05 | 703.0ms/step | 28.5 samples/s +Step 33070/44000 | Loss: 2.9328 | LR: 7.47e-05 | 697.3ms/step | 28.7 samples/s +Step 33080/44000 | Loss: 2.9539 | LR: 7.46e-05 | 698.7ms/step | 28.6 samples/s +Step 33090/44000 | Loss: 2.9578 | LR: 7.45e-05 | 700.4ms/step | 28.6 samples/s +Step 33100/44000 | Loss: 2.9203 | LR: 7.44e-05 | 701.9ms/step | 28.5 samples/s +Step 33110/44000 | Loss: 2.9622 | LR: 7.42e-05 | 702.5ms/step | 28.5 samples/s +Step 33120/44000 | Loss: 2.9413 | LR: 7.41e-05 | 703.1ms/step | 28.4 samples/s +Step 33130/44000 | Loss: 2.9322 | LR: 7.40e-05 | 700.4ms/step | 28.6 samples/s +Step 33140/44000 | Loss: 2.8988 | LR: 7.38e-05 | 701.5ms/step | 28.5 samples/s +Step 33150/44000 | Loss: 2.9608 | LR: 7.37e-05 | 699.2ms/step | 28.6 samples/s +Step 33160/44000 | Loss: 2.9195 | LR: 7.36e-05 | 704.0ms/step | 28.4 samples/s +Step 33170/44000 | Loss: 2.9116 | LR: 7.34e-05 | 702.5ms/step | 28.5 samples/s +Step 33180/44000 | Loss: 2.8999 | LR: 7.33e-05 | 701.2ms/step | 28.5 samples/s +Step 33190/44000 | Loss: 2.9289 | LR: 7.32e-05 | 700.4ms/step | 28.6 samples/s +Step 33200/44000 | Loss: 2.9240 | LR: 7.31e-05 | 703.0ms/step | 28.4 samples/s +Step 33210/44000 | Loss: 2.9346 | LR: 7.29e-05 | 704.4ms/step | 28.4 samples/s +Step 33220/44000 | Loss: 2.9615 | LR: 7.28e-05 | 702.4ms/step | 28.5 samples/s +Step 33230/44000 | Loss: 2.9666 | LR: 7.27e-05 | 703.4ms/step | 28.4 samples/s +Step 33240/44000 | Loss: 2.9775 | LR: 7.26e-05 | 701.7ms/step | 28.5 samples/s +Step 33250/44000 | Loss: 2.9336 | LR: 7.24e-05 | 700.9ms/step | 28.5 samples/s +Validation at step 33250: Loss = 2.9522, Perplexity = 19.15 +Step 33260/44000 | Loss: 2.9141 | LR: 7.23e-05 | 1271.7ms/step | 15.7 samples/s +Step 33270/44000 | Loss: 2.9364 | LR: 7.22e-05 | 697.7ms/step | 28.7 samples/s +Step 33280/44000 | Loss: 2.9576 | LR: 7.20e-05 | 701.0ms/step | 28.5 samples/s +Step 33290/44000 | Loss: 2.9320 | LR: 7.19e-05 | 703.3ms/step | 28.4 samples/s +Step 33300/44000 | Loss: 2.9196 | LR: 7.18e-05 | 698.6ms/step | 28.6 samples/s +Step 33310/44000 | Loss: 2.9754 | LR: 7.17e-05 | 704.9ms/step | 28.4 samples/s +Step 33320/44000 | Loss: 2.8972 | LR: 7.15e-05 | 704.4ms/step | 28.4 samples/s +Step 33330/44000 | Loss: 2.9692 | LR: 7.14e-05 | 701.3ms/step | 28.5 samples/s +Step 33340/44000 | Loss: 2.9118 | LR: 7.13e-05 | 700.8ms/step | 28.5 samples/s +Step 33350/44000 | Loss: 2.9729 | LR: 7.11e-05 | 703.3ms/step | 28.4 samples/s +Step 33360/44000 | Loss: 2.9554 | LR: 7.10e-05 | 699.1ms/step | 28.6 samples/s +Step 33370/44000 | Loss: 2.9246 | LR: 7.09e-05 | 704.0ms/step | 28.4 samples/s +Step 33380/44000 | Loss: 2.9581 | LR: 7.08e-05 | 702.1ms/step | 28.5 samples/s +Step 33390/44000 | Loss: 2.8937 | LR: 7.06e-05 | 700.2ms/step | 28.6 samples/s +Step 33400/44000 | Loss: 2.9044 | LR: 7.05e-05 | 701.9ms/step | 28.5 samples/s +Step 33410/44000 | Loss: 2.8982 | LR: 7.04e-05 | 700.7ms/step | 28.5 samples/s +Step 33420/44000 | Loss: 2.8843 | LR: 7.03e-05 | 702.1ms/step | 28.5 samples/s +Step 33430/44000 | Loss: 2.9352 | LR: 7.01e-05 | 701.6ms/step | 28.5 samples/s +Step 33440/44000 | Loss: 2.9370 | LR: 7.00e-05 | 702.8ms/step | 28.5 samples/s +Step 33450/44000 | Loss: 2.9315 | LR: 6.99e-05 | 701.2ms/step | 28.5 samples/s +Step 33460/44000 | Loss: 2.9312 | LR: 6.98e-05 | 696.1ms/step | 28.7 samples/s +Step 33470/44000 | Loss: 2.9495 | LR: 6.96e-05 | 702.5ms/step | 28.5 samples/s +Step 33480/44000 | Loss: 2.9016 | LR: 6.95e-05 | 702.4ms/step | 28.5 samples/s +Step 33490/44000 | Loss: 2.9286 | LR: 6.94e-05 | 705.3ms/step | 28.4 samples/s +Step 33500/44000 | Loss: 2.9301 | LR: 6.93e-05 | 702.2ms/step | 28.5 samples/s +Validation at step 33500: Loss = 2.9511, Perplexity = 19.13 +Step 33510/44000 | Loss: 2.9305 | LR: 6.91e-05 | 1270.8ms/step | 15.7 samples/s +Step 33520/44000 | Loss: 2.9173 | LR: 6.90e-05 | 703.1ms/step | 28.4 samples/s +Step 33530/44000 | Loss: 2.9005 | LR: 6.89e-05 | 698.9ms/step | 28.6 samples/s +Step 33540/44000 | Loss: 2.9432 | LR: 6.88e-05 | 699.7ms/step | 28.6 samples/s +Step 33550/44000 | Loss: 2.9117 | LR: 6.86e-05 | 698.0ms/step | 28.7 samples/s +Step 33560/44000 | Loss: 2.9020 | LR: 6.85e-05 | 703.1ms/step | 28.4 samples/s +Step 33570/44000 | Loss: 2.9329 | LR: 6.84e-05 | 704.9ms/step | 28.4 samples/s +Step 33580/44000 | Loss: 2.9156 | LR: 6.83e-05 | 700.9ms/step | 28.5 samples/s +Step 33590/44000 | Loss: 2.9546 | LR: 6.81e-05 | 702.4ms/step | 28.5 samples/s +Step 33600/44000 | Loss: 2.9776 | LR: 6.80e-05 | 700.8ms/step | 28.5 samples/s +Step 33610/44000 | Loss: 2.9316 | LR: 6.79e-05 | 704.7ms/step | 28.4 samples/s +Step 33620/44000 | Loss: 2.9265 | LR: 6.78e-05 | 700.4ms/step | 28.6 samples/s +Step 33630/44000 | Loss: 2.9178 | LR: 6.76e-05 | 702.3ms/step | 28.5 samples/s +Step 33640/44000 | Loss: 2.9540 | LR: 6.75e-05 | 702.9ms/step | 28.5 samples/s +Step 33650/44000 | Loss: 2.9285 | LR: 6.74e-05 | 700.4ms/step | 28.6 samples/s +Step 33660/44000 | Loss: 2.9448 | LR: 6.73e-05 | 700.9ms/step | 28.5 samples/s +Step 33670/44000 | Loss: 2.9510 | LR: 6.71e-05 | 697.5ms/step | 28.7 samples/s +Step 33680/44000 | Loss: 2.9512 | LR: 6.70e-05 | 699.3ms/step | 28.6 samples/s +Step 33690/44000 | Loss: 2.9355 | LR: 6.69e-05 | 702.5ms/step | 28.5 samples/s +Step 33700/44000 | Loss: 2.9300 | LR: 6.68e-05 | 697.9ms/step | 28.7 samples/s +Step 33710/44000 | Loss: 2.8995 | LR: 6.66e-05 | 701.2ms/step | 28.5 samples/s +Step 33720/44000 | Loss: 2.9043 | LR: 6.65e-05 | 697.4ms/step | 28.7 samples/s +Step 33730/44000 | Loss: 2.9433 | LR: 6.64e-05 | 700.9ms/step | 28.5 samples/s +Step 33740/44000 | Loss: 2.9230 | LR: 6.63e-05 | 701.8ms/step | 28.5 samples/s +Step 33750/44000 | Loss: 2.9714 | LR: 6.61e-05 | 700.3ms/step | 28.6 samples/s +Validation at step 33750: Loss = 2.9515, Perplexity = 19.13 +Step 33760/44000 | Loss: 2.9808 | LR: 6.60e-05 | 1306.3ms/step | 15.3 samples/s +Step 33770/44000 | Loss: 2.9182 | LR: 6.59e-05 | 721.8ms/step | 27.7 samples/s +Step 33780/44000 | Loss: 2.9703 | LR: 6.58e-05 | 716.5ms/step | 27.9 samples/s +Step 33790/44000 | Loss: 2.9196 | LR: 6.57e-05 | 700.4ms/step | 28.6 samples/s +Step 33800/44000 | Loss: 2.9581 | LR: 6.55e-05 | 704.9ms/step | 28.4 samples/s +Step 33810/44000 | Loss: 2.9396 | LR: 6.54e-05 | 702.5ms/step | 28.5 samples/s +Step 33820/44000 | Loss: 2.9678 | LR: 6.53e-05 | 704.6ms/step | 28.4 samples/s +Step 33830/44000 | Loss: 2.9039 | LR: 6.52e-05 | 708.2ms/step | 28.2 samples/s +Step 33840/44000 | Loss: 2.9470 | LR: 6.50e-05 | 709.6ms/step | 28.2 samples/s +Step 33850/44000 | Loss: 2.9089 | LR: 6.49e-05 | 701.4ms/step | 28.5 samples/s +Step 33860/44000 | Loss: 2.9467 | LR: 6.48e-05 | 702.1ms/step | 28.5 samples/s +Step 33870/44000 | Loss: 2.9284 | LR: 6.47e-05 | 701.7ms/step | 28.5 samples/s +Step 33880/44000 | Loss: 2.9351 | LR: 6.46e-05 | 701.5ms/step | 28.5 samples/s +Step 33890/44000 | Loss: 2.9356 | LR: 6.44e-05 | 701.6ms/step | 28.5 samples/s +Step 33900/44000 | Loss: 2.9219 | LR: 6.43e-05 | 699.3ms/step | 28.6 samples/s +Step 33910/44000 | Loss: 2.9532 | LR: 6.42e-05 | 703.1ms/step | 28.4 samples/s +Step 33920/44000 | Loss: 2.9423 | LR: 6.41e-05 | 703.0ms/step | 28.5 samples/s +Step 33930/44000 | Loss: 2.9609 | LR: 6.40e-05 | 700.3ms/step | 28.6 samples/s +Step 33940/44000 | Loss: 2.9101 | LR: 6.38e-05 | 705.3ms/step | 28.4 samples/s +Step 33950/44000 | Loss: 2.9389 | LR: 6.37e-05 | 705.5ms/step | 28.3 samples/s +Step 33960/44000 | Loss: 2.9726 | LR: 6.36e-05 | 704.5ms/step | 28.4 samples/s +Step 33970/44000 | Loss: 2.9449 | LR: 6.35e-05 | 703.5ms/step | 28.4 samples/s +Step 33980/44000 | Loss: 2.9457 | LR: 6.33e-05 | 704.0ms/step | 28.4 samples/s +Step 33990/44000 | Loss: 2.9174 | LR: 6.32e-05 | 703.0ms/step | 28.4 samples/s +Step 34000/44000 | Loss: 2.9516 | LR: 6.31e-05 | 698.6ms/step | 28.6 samples/s +Validation at step 34000: Loss = 2.9507, Perplexity = 19.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-34000 +Step 34010/44000 | Loss: 2.9906 | LR: 6.30e-05 | 1629.1ms/step | 12.3 samples/s +Step 34020/44000 | Loss: 2.9279 | LR: 6.29e-05 | 701.4ms/step | 28.5 samples/s +Step 34030/44000 | Loss: 2.9287 | LR: 6.27e-05 | 700.5ms/step | 28.6 samples/s +Step 34040/44000 | Loss: 2.9493 | LR: 6.26e-05 | 699.8ms/step | 28.6 samples/s +Step 34050/44000 | Loss: 2.9329 | LR: 6.25e-05 | 700.5ms/step | 28.5 samples/s +Step 34060/44000 | Loss: 2.9129 | LR: 6.24e-05 | 699.7ms/step | 28.6 samples/s +Step 34070/44000 | Loss: 2.9719 | LR: 6.23e-05 | 699.0ms/step | 28.6 samples/s +Step 34080/44000 | Loss: 2.9106 | LR: 6.21e-05 | 703.3ms/step | 28.4 samples/s +Step 34090/44000 | Loss: 2.9403 | LR: 6.20e-05 | 700.3ms/step | 28.6 samples/s +Step 34100/44000 | Loss: 2.9853 | LR: 6.19e-05 | 701.9ms/step | 28.5 samples/s +Step 34110/44000 | Loss: 2.9451 | LR: 6.18e-05 | 701.7ms/step | 28.5 samples/s +Step 34120/44000 | Loss: 2.9163 | LR: 6.17e-05 | 705.7ms/step | 28.3 samples/s +Step 34130/44000 | Loss: 2.9076 | LR: 6.15e-05 | 702.2ms/step | 28.5 samples/s +Step 34140/44000 | Loss: 2.9764 | LR: 6.14e-05 | 698.5ms/step | 28.6 samples/s +Step 34150/44000 | Loss: 2.9511 | LR: 6.13e-05 | 703.2ms/step | 28.4 samples/s +Step 34160/44000 | Loss: 2.9473 | LR: 6.12e-05 | 701.1ms/step | 28.5 samples/s +Step 34170/44000 | Loss: 2.9282 | LR: 6.11e-05 | 700.4ms/step | 28.6 samples/s +Step 34180/44000 | Loss: 2.9355 | LR: 6.09e-05 | 702.5ms/step | 28.5 samples/s +Step 34190/44000 | Loss: 2.8879 | LR: 6.08e-05 | 701.2ms/step | 28.5 samples/s +Step 34200/44000 | Loss: 2.9418 | LR: 6.07e-05 | 702.9ms/step | 28.5 samples/s +Step 34210/44000 | Loss: 2.9864 | LR: 6.06e-05 | 701.0ms/step | 28.5 samples/s +Step 34220/44000 | Loss: 2.9606 | LR: 6.05e-05 | 701.7ms/step | 28.5 samples/s +Step 34230/44000 | Loss: 2.9084 | LR: 6.04e-05 | 701.9ms/step | 28.5 samples/s +Step 34240/44000 | Loss: 2.9446 | LR: 6.02e-05 | 703.5ms/step | 28.4 samples/s +Step 34250/44000 | Loss: 2.9169 | LR: 6.01e-05 | 700.4ms/step | 28.6 samples/s +Validation at step 34250: Loss = 2.9494, Perplexity = 19.09 +Step 34260/44000 | Loss: 2.9833 | LR: 6.00e-05 | 1272.3ms/step | 15.7 samples/s +Step 34270/44000 | Loss: 2.9166 | LR: 5.99e-05 | 701.1ms/step | 28.5 samples/s +Step 34280/44000 | Loss: 2.9508 | LR: 5.98e-05 | 698.9ms/step | 28.6 samples/s +Step 34290/44000 | Loss: 2.9731 | LR: 5.96e-05 | 702.8ms/step | 28.5 samples/s +Step 34300/44000 | Loss: 2.9218 | LR: 5.95e-05 | 700.9ms/step | 28.5 samples/s +Step 34310/44000 | Loss: 2.9542 | LR: 5.94e-05 | 699.7ms/step | 28.6 samples/s +Step 34320/44000 | Loss: 2.9256 | LR: 5.93e-05 | 701.5ms/step | 28.5 samples/s +Step 34330/44000 | Loss: 3.0094 | LR: 5.92e-05 | 697.1ms/step | 28.7 samples/s +Step 34340/44000 | Loss: 2.9325 | LR: 5.91e-05 | 700.3ms/step | 28.6 samples/s +Step 34350/44000 | Loss: 2.9336 | LR: 5.89e-05 | 703.4ms/step | 28.4 samples/s +Step 34360/44000 | Loss: 2.8938 | LR: 5.88e-05 | 702.9ms/step | 28.5 samples/s +Step 34370/44000 | Loss: 2.9240 | LR: 5.87e-05 | 703.6ms/step | 28.4 samples/s +Step 34380/44000 | Loss: 2.8888 | LR: 5.86e-05 | 701.5ms/step | 28.5 samples/s +Step 34390/44000 | Loss: 2.9175 | LR: 5.85e-05 | 702.2ms/step | 28.5 samples/s +Step 34400/44000 | Loss: 2.8952 | LR: 5.84e-05 | 704.5ms/step | 28.4 samples/s +Step 34410/44000 | Loss: 2.9484 | LR: 5.82e-05 | 702.0ms/step | 28.5 samples/s +Step 34420/44000 | Loss: 2.9076 | LR: 5.81e-05 | 702.0ms/step | 28.5 samples/s +Step 34430/44000 | Loss: 2.9691 | LR: 5.80e-05 | 699.1ms/step | 28.6 samples/s +Step 34440/44000 | Loss: 2.9535 | LR: 5.79e-05 | 701.1ms/step | 28.5 samples/s +Step 34450/44000 | Loss: 2.9154 | LR: 5.78e-05 | 701.9ms/step | 28.5 samples/s +Step 34460/44000 | Loss: 2.9453 | LR: 5.77e-05 | 706.7ms/step | 28.3 samples/s +Step 34470/44000 | Loss: 2.9508 | LR: 5.75e-05 | 703.2ms/step | 28.4 samples/s +Step 34480/44000 | Loss: 2.9610 | LR: 5.74e-05 | 703.3ms/step | 28.4 samples/s +Step 34490/44000 | Loss: 2.9274 | LR: 5.73e-05 | 700.3ms/step | 28.6 samples/s +Step 34500/44000 | Loss: 2.9190 | LR: 5.72e-05 | 701.8ms/step | 28.5 samples/s +Validation at step 34500: Loss = 2.9492, Perplexity = 19.09 +Step 34510/44000 | Loss: 2.9249 | LR: 5.71e-05 | 1276.4ms/step | 15.7 samples/s +Step 34520/44000 | Loss: 2.9240 | LR: 5.70e-05 | 700.3ms/step | 28.6 samples/s +Step 34530/44000 | Loss: 2.9826 | LR: 5.69e-05 | 702.5ms/step | 28.5 samples/s +Step 34540/44000 | Loss: 2.9116 | LR: 5.67e-05 | 702.1ms/step | 28.5 samples/s +Step 34550/44000 | Loss: 2.9152 | LR: 5.66e-05 | 702.7ms/step | 28.5 samples/s +Step 34560/44000 | Loss: 2.9530 | LR: 5.65e-05 | 701.8ms/step | 28.5 samples/s +Step 34570/44000 | Loss: 2.9447 | LR: 5.64e-05 | 700.2ms/step | 28.6 samples/s +Step 34580/44000 | Loss: 2.9687 | LR: 5.63e-05 | 702.4ms/step | 28.5 samples/s +Step 34590/44000 | Loss: 2.9328 | LR: 5.62e-05 | 704.8ms/step | 28.4 samples/s +Step 34600/44000 | Loss: 2.8802 | LR: 5.60e-05 | 702.3ms/step | 28.5 samples/s +Step 34610/44000 | Loss: 2.9640 | LR: 5.59e-05 | 702.4ms/step | 28.5 samples/s +Step 34620/44000 | Loss: 2.9630 | LR: 5.58e-05 | 700.3ms/step | 28.6 samples/s +Step 34630/44000 | Loss: 2.8892 | LR: 5.57e-05 | 702.5ms/step | 28.5 samples/s +Step 34640/44000 | Loss: 2.9952 | LR: 5.56e-05 | 703.0ms/step | 28.5 samples/s +Step 34650/44000 | Loss: 2.9274 | LR: 5.55e-05 | 699.3ms/step | 28.6 samples/s +Step 34660/44000 | Loss: 2.9564 | LR: 5.54e-05 | 700.7ms/step | 28.5 samples/s +Step 34670/44000 | Loss: 2.9219 | LR: 5.52e-05 | 703.2ms/step | 28.4 samples/s +Step 34680/44000 | Loss: 2.9368 | LR: 5.51e-05 | 702.7ms/step | 28.5 samples/s +Step 34690/44000 | Loss: 2.9076 | LR: 5.50e-05 | 704.1ms/step | 28.4 samples/s +Step 34700/44000 | Loss: 2.9112 | LR: 5.49e-05 | 698.6ms/step | 28.6 samples/s +Step 34710/44000 | Loss: 2.9289 | LR: 5.48e-05 | 697.7ms/step | 28.7 samples/s +Step 34720/44000 | Loss: 2.9357 | LR: 5.47e-05 | 701.9ms/step | 28.5 samples/s +Step 34730/44000 | Loss: 2.9480 | LR: 5.46e-05 | 702.8ms/step | 28.5 samples/s +Step 34740/44000 | Loss: 2.9184 | LR: 5.45e-05 | 702.3ms/step | 28.5 samples/s +Step 34750/44000 | Loss: 2.9663 | LR: 5.43e-05 | 701.7ms/step | 28.5 samples/s +Validation at step 34750: Loss = 2.9502, Perplexity = 19.11 +Step 34760/44000 | Loss: 2.9364 | LR: 5.42e-05 | 1269.3ms/step | 15.8 samples/s +Step 34770/44000 | Loss: 2.9591 | LR: 5.41e-05 | 699.3ms/step | 28.6 samples/s +Step 34780/44000 | Loss: 2.9404 | LR: 5.40e-05 | 699.0ms/step | 28.6 samples/s +Step 34790/44000 | Loss: 2.9495 | LR: 5.39e-05 | 699.5ms/step | 28.6 samples/s +Step 34800/44000 | Loss: 2.9467 | LR: 5.38e-05 | 704.6ms/step | 28.4 samples/s +Step 34810/44000 | Loss: 2.9328 | LR: 5.37e-05 | 701.4ms/step | 28.5 samples/s +Step 34820/44000 | Loss: 2.8965 | LR: 5.36e-05 | 698.6ms/step | 28.6 samples/s +Step 34830/44000 | Loss: 2.9262 | LR: 5.34e-05 | 702.2ms/step | 28.5 samples/s +Step 34840/44000 | Loss: 2.9644 | LR: 5.33e-05 | 699.0ms/step | 28.6 samples/s +Step 34850/44000 | Loss: 2.9825 | LR: 5.32e-05 | 699.9ms/step | 28.6 samples/s +Step 34860/44000 | Loss: 2.9447 | LR: 5.31e-05 | 702.1ms/step | 28.5 samples/s +Step 34870/44000 | Loss: 2.9435 | LR: 5.30e-05 | 702.5ms/step | 28.5 samples/s +Step 34880/44000 | Loss: 2.9135 | LR: 5.29e-05 | 700.7ms/step | 28.5 samples/s +Step 34890/44000 | Loss: 2.9229 | LR: 5.28e-05 | 699.1ms/step | 28.6 samples/s +Step 34900/44000 | Loss: 2.9198 | LR: 5.27e-05 | 699.8ms/step | 28.6 samples/s +Step 34910/44000 | Loss: 2.9203 | LR: 5.25e-05 | 701.6ms/step | 28.5 samples/s +Step 34920/44000 | Loss: 2.9206 | LR: 5.24e-05 | 702.2ms/step | 28.5 samples/s +Step 34930/44000 | Loss: 2.9320 | LR: 5.23e-05 | 700.4ms/step | 28.6 samples/s +Step 34940/44000 | Loss: 2.9532 | LR: 5.22e-05 | 699.6ms/step | 28.6 samples/s +Step 34950/44000 | Loss: 2.9348 | LR: 5.21e-05 | 699.4ms/step | 28.6 samples/s +Step 34960/44000 | Loss: 2.9494 | LR: 5.20e-05 | 703.1ms/step | 28.4 samples/s +Step 34970/44000 | Loss: 2.9713 | LR: 5.19e-05 | 700.0ms/step | 28.6 samples/s +Step 34980/44000 | Loss: 2.9463 | LR: 5.18e-05 | 699.9ms/step | 28.6 samples/s +Step 34990/44000 | Loss: 2.9320 | LR: 5.17e-05 | 702.0ms/step | 28.5 samples/s +Step 35000/44000 | Loss: 2.9025 | LR: 5.15e-05 | 701.2ms/step | 28.5 samples/s +Validation at step 35000: Loss = 2.9507, Perplexity = 19.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-35000 +Step 35010/44000 | Loss: 2.9452 | LR: 5.14e-05 | 1559.7ms/step | 12.8 samples/s +Step 35020/44000 | Loss: 2.9233 | LR: 5.13e-05 | 700.6ms/step | 28.5 samples/s +Step 35030/44000 | Loss: 2.9359 | LR: 5.12e-05 | 698.0ms/step | 28.7 samples/s +Step 35040/44000 | Loss: 2.9338 | LR: 5.11e-05 | 698.2ms/step | 28.6 samples/s +Step 35050/44000 | Loss: 2.9481 | LR: 5.10e-05 | 698.2ms/step | 28.6 samples/s +Step 35060/44000 | Loss: 2.9359 | LR: 5.09e-05 | 698.0ms/step | 28.7 samples/s +Step 35070/44000 | Loss: 2.9133 | LR: 5.08e-05 | 702.2ms/step | 28.5 samples/s +Step 35080/44000 | Loss: 2.9246 | LR: 5.07e-05 | 699.1ms/step | 28.6 samples/s +Step 35090/44000 | Loss: 2.9437 | LR: 5.06e-05 | 702.2ms/step | 28.5 samples/s +Step 35100/44000 | Loss: 2.8899 | LR: 5.04e-05 | 701.3ms/step | 28.5 samples/s +Step 35110/44000 | Loss: 2.9800 | LR: 5.03e-05 | 704.7ms/step | 28.4 samples/s +Step 35120/44000 | Loss: 2.9315 | LR: 5.02e-05 | 703.5ms/step | 28.4 samples/s +Step 35130/44000 | Loss: 2.9087 | LR: 5.01e-05 | 703.1ms/step | 28.4 samples/s +Step 35140/44000 | Loss: 2.9332 | LR: 5.00e-05 | 699.1ms/step | 28.6 samples/s +Step 35150/44000 | Loss: 2.9009 | LR: 4.99e-05 | 702.8ms/step | 28.5 samples/s +Step 35160/44000 | Loss: 2.9409 | LR: 4.98e-05 | 701.0ms/step | 28.5 samples/s +Step 35170/44000 | Loss: 2.9550 | LR: 4.97e-05 | 698.5ms/step | 28.6 samples/s +Step 35180/44000 | Loss: 2.9430 | LR: 4.96e-05 | 700.1ms/step | 28.6 samples/s +Step 35190/44000 | Loss: 2.9225 | LR: 4.95e-05 | 706.2ms/step | 28.3 samples/s +Step 35200/44000 | Loss: 2.9287 | LR: 4.94e-05 | 700.1ms/step | 28.6 samples/s +Step 35210/44000 | Loss: 2.9483 | LR: 4.93e-05 | 699.4ms/step | 28.6 samples/s +Step 35220/44000 | Loss: 2.9338 | LR: 4.91e-05 | 700.8ms/step | 28.5 samples/s +Step 35230/44000 | Loss: 2.9211 | LR: 4.90e-05 | 700.3ms/step | 28.6 samples/s +Step 35240/44000 | Loss: 2.9414 | LR: 4.89e-05 | 702.0ms/step | 28.5 samples/s +Step 35250/44000 | Loss: 2.9089 | LR: 4.88e-05 | 698.5ms/step | 28.6 samples/s +Validation at step 35250: Loss = 2.9488, Perplexity = 19.08 +Step 35260/44000 | Loss: 2.9301 | LR: 4.87e-05 | 1272.0ms/step | 15.7 samples/s +Step 35270/44000 | Loss: 2.9427 | LR: 4.86e-05 | 704.0ms/step | 28.4 samples/s +Step 35280/44000 | Loss: 2.9622 | LR: 4.85e-05 | 701.7ms/step | 28.5 samples/s +Step 35290/44000 | Loss: 2.9422 | LR: 4.84e-05 | 701.9ms/step | 28.5 samples/s +Step 35300/44000 | Loss: 2.9398 | LR: 4.83e-05 | 703.0ms/step | 28.5 samples/s +Step 35310/44000 | Loss: 2.8959 | LR: 4.82e-05 | 702.0ms/step | 28.5 samples/s +Step 35320/44000 | Loss: 2.8909 | LR: 4.81e-05 | 701.5ms/step | 28.5 samples/s +Step 35330/44000 | Loss: 2.9233 | LR: 4.80e-05 | 696.6ms/step | 28.7 samples/s +Step 35340/44000 | Loss: 2.9380 | LR: 4.79e-05 | 701.5ms/step | 28.5 samples/s +Step 35350/44000 | Loss: 2.8806 | LR: 4.77e-05 | 700.5ms/step | 28.6 samples/s +Step 35360/44000 | Loss: 2.9171 | LR: 4.76e-05 | 700.1ms/step | 28.6 samples/s +Step 35370/44000 | Loss: 2.9163 | LR: 4.75e-05 | 704.1ms/step | 28.4 samples/s +Step 35380/44000 | Loss: 2.9265 | LR: 4.74e-05 | 700.5ms/step | 28.6 samples/s +Step 35390/44000 | Loss: 2.9316 | LR: 4.73e-05 | 703.7ms/step | 28.4 samples/s +Step 35400/44000 | Loss: 2.9253 | LR: 4.72e-05 | 703.7ms/step | 28.4 samples/s +Step 35410/44000 | Loss: 2.9490 | LR: 4.71e-05 | 701.1ms/step | 28.5 samples/s +Step 35420/44000 | Loss: 2.9868 | LR: 4.70e-05 | 705.0ms/step | 28.4 samples/s +Step 35430/44000 | Loss: 2.9444 | LR: 4.69e-05 | 701.4ms/step | 28.5 samples/s +Step 35440/44000 | Loss: 2.9268 | LR: 4.68e-05 | 702.4ms/step | 28.5 samples/s +Step 35450/44000 | Loss: 2.9161 | LR: 4.67e-05 | 700.7ms/step | 28.5 samples/s +Step 35460/44000 | Loss: 2.9412 | LR: 4.66e-05 | 702.5ms/step | 28.5 samples/s +Step 35470/44000 | Loss: 2.9032 | LR: 4.65e-05 | 701.2ms/step | 28.5 samples/s +Step 35480/44000 | Loss: 2.9532 | LR: 4.64e-05 | 700.7ms/step | 28.5 samples/s +Step 35490/44000 | Loss: 2.9334 | LR: 4.63e-05 | 702.9ms/step | 28.5 samples/s +Step 35500/44000 | Loss: 2.9568 | LR: 4.62e-05 | 702.1ms/step | 28.5 samples/s +Validation at step 35500: Loss = 2.9489, Perplexity = 19.09 +Step 35510/44000 | Loss: 2.9367 | LR: 4.61e-05 | 1271.5ms/step | 15.7 samples/s +Step 35520/44000 | Loss: 2.9454 | LR: 4.59e-05 | 701.4ms/step | 28.5 samples/s +Step 35530/44000 | Loss: 2.9429 | LR: 4.58e-05 | 702.2ms/step | 28.5 samples/s +Step 35540/44000 | Loss: 2.8937 | LR: 4.57e-05 | 699.4ms/step | 28.6 samples/s +Step 35550/44000 | Loss: 2.9349 | LR: 4.56e-05 | 701.3ms/step | 28.5 samples/s +Step 35560/44000 | Loss: 2.8802 | LR: 4.55e-05 | 696.2ms/step | 28.7 samples/s +Step 35570/44000 | Loss: 2.9393 | LR: 4.54e-05 | 703.9ms/step | 28.4 samples/s +Step 35580/44000 | Loss: 2.9399 | LR: 4.53e-05 | 702.4ms/step | 28.5 samples/s +Step 35590/44000 | Loss: 2.9632 | LR: 4.52e-05 | 701.2ms/step | 28.5 samples/s +Step 35600/44000 | Loss: 2.9325 | LR: 4.51e-05 | 704.4ms/step | 28.4 samples/s +Step 35610/44000 | Loss: 2.9858 | LR: 4.50e-05 | 699.9ms/step | 28.6 samples/s +Step 35620/44000 | Loss: 2.9042 | LR: 4.49e-05 | 700.5ms/step | 28.6 samples/s +Step 35630/44000 | Loss: 2.9497 | LR: 4.48e-05 | 699.8ms/step | 28.6 samples/s +Step 35640/44000 | Loss: 2.9333 | LR: 4.47e-05 | 699.1ms/step | 28.6 samples/s +Step 35650/44000 | Loss: 2.9299 | LR: 4.46e-05 | 700.3ms/step | 28.6 samples/s +Step 35660/44000 | Loss: 2.9024 | LR: 4.45e-05 | 699.3ms/step | 28.6 samples/s +Step 35670/44000 | Loss: 2.9091 | LR: 4.44e-05 | 701.6ms/step | 28.5 samples/s +Step 35680/44000 | Loss: 2.9099 | LR: 4.43e-05 | 699.3ms/step | 28.6 samples/s +Step 35690/44000 | Loss: 2.9373 | LR: 4.42e-05 | 703.1ms/step | 28.4 samples/s +Step 35700/44000 | Loss: 2.9166 | LR: 4.41e-05 | 701.3ms/step | 28.5 samples/s +Step 35710/44000 | Loss: 2.9123 | LR: 4.40e-05 | 700.0ms/step | 28.6 samples/s +Step 35720/44000 | Loss: 2.9001 | LR: 4.39e-05 | 701.0ms/step | 28.5 samples/s +Step 35730/44000 | Loss: 2.8903 | LR: 4.38e-05 | 696.5ms/step | 28.7 samples/s +Step 35740/44000 | Loss: 2.9663 | LR: 4.37e-05 | 700.3ms/step | 28.6 samples/s +Step 35750/44000 | Loss: 2.9218 | LR: 4.36e-05 | 699.4ms/step | 28.6 samples/s +Validation at step 35750: Loss = 2.9490, Perplexity = 19.09 +Step 35760/44000 | Loss: 2.9281 | LR: 4.35e-05 | 1266.5ms/step | 15.8 samples/s +Step 35770/44000 | Loss: 2.9021 | LR: 4.34e-05 | 700.6ms/step | 28.5 samples/s +Step 35780/44000 | Loss: 2.9491 | LR: 4.33e-05 | 698.9ms/step | 28.6 samples/s +Step 35790/44000 | Loss: 2.9032 | LR: 4.32e-05 | 703.0ms/step | 28.5 samples/s +Step 35800/44000 | Loss: 2.9483 | LR: 4.31e-05 | 704.0ms/step | 28.4 samples/s +Step 35810/44000 | Loss: 2.9529 | LR: 4.29e-05 | 704.7ms/step | 28.4 samples/s +Step 35820/44000 | Loss: 2.9378 | LR: 4.28e-05 | 704.3ms/step | 28.4 samples/s +Step 35830/44000 | Loss: 2.8945 | LR: 4.27e-05 | 703.0ms/step | 28.5 samples/s +Step 35840/44000 | Loss: 2.9501 | LR: 4.26e-05 | 698.7ms/step | 28.6 samples/s +Step 35850/44000 | Loss: 2.9614 | LR: 4.25e-05 | 704.3ms/step | 28.4 samples/s +Step 35860/44000 | Loss: 2.8698 | LR: 4.24e-05 | 704.2ms/step | 28.4 samples/s +Step 35870/44000 | Loss: 2.9147 | LR: 4.23e-05 | 704.1ms/step | 28.4 samples/s +Step 35880/44000 | Loss: 2.9166 | LR: 4.22e-05 | 702.0ms/step | 28.5 samples/s +Step 35890/44000 | Loss: 2.9325 | LR: 4.21e-05 | 702.7ms/step | 28.5 samples/s +Step 35900/44000 | Loss: 2.9370 | LR: 4.20e-05 | 702.3ms/step | 28.5 samples/s +Step 35910/44000 | Loss: 2.9330 | LR: 4.19e-05 | 703.3ms/step | 28.4 samples/s +Step 35920/44000 | Loss: 2.9344 | LR: 4.18e-05 | 704.2ms/step | 28.4 samples/s +Step 35930/44000 | Loss: 2.8826 | LR: 4.17e-05 | 700.6ms/step | 28.5 samples/s +Step 35940/44000 | Loss: 2.9540 | LR: 4.16e-05 | 699.7ms/step | 28.6 samples/s +Step 35950/44000 | Loss: 2.9521 | LR: 4.15e-05 | 702.3ms/step | 28.5 samples/s +Step 35960/44000 | Loss: 2.9433 | LR: 4.14e-05 | 703.6ms/step | 28.4 samples/s +Step 35970/44000 | Loss: 2.9836 | LR: 4.13e-05 | 701.1ms/step | 28.5 samples/s +Step 35980/44000 | Loss: 2.9297 | LR: 4.12e-05 | 699.7ms/step | 28.6 samples/s +Step 35990/44000 | Loss: 2.9400 | LR: 4.11e-05 | 699.6ms/step | 28.6 samples/s +Step 36000/44000 | Loss: 2.9447 | LR: 4.10e-05 | 699.1ms/step | 28.6 samples/s +Validation at step 36000: Loss = 2.9484, Perplexity = 19.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-36000 +Step 36010/44000 | Loss: 2.9284 | LR: 4.09e-05 | 1562.4ms/step | 12.8 samples/s +Step 36020/44000 | Loss: 2.8992 | LR: 4.08e-05 | 701.7ms/step | 28.5 samples/s +Step 36030/44000 | Loss: 2.9417 | LR: 4.07e-05 | 702.9ms/step | 28.5 samples/s +Step 36040/44000 | Loss: 2.9247 | LR: 4.06e-05 | 699.8ms/step | 28.6 samples/s +Step 36050/44000 | Loss: 2.9545 | LR: 4.05e-05 | 703.9ms/step | 28.4 samples/s +Step 36060/44000 | Loss: 2.9460 | LR: 4.04e-05 | 696.2ms/step | 28.7 samples/s +Step 36070/44000 | Loss: 2.9438 | LR: 4.03e-05 | 701.5ms/step | 28.5 samples/s +Step 36080/44000 | Loss: 2.9635 | LR: 4.02e-05 | 699.6ms/step | 28.6 samples/s +Step 36090/44000 | Loss: 2.9126 | LR: 4.01e-05 | 700.9ms/step | 28.5 samples/s +Step 36100/44000 | Loss: 2.9459 | LR: 4.00e-05 | 703.4ms/step | 28.4 samples/s +Step 36110/44000 | Loss: 2.9082 | LR: 3.99e-05 | 699.7ms/step | 28.6 samples/s +Step 36120/44000 | Loss: 2.9390 | LR: 3.98e-05 | 702.6ms/step | 28.5 samples/s +Step 36130/44000 | Loss: 2.9794 | LR: 3.97e-05 | 701.3ms/step | 28.5 samples/s +Step 36140/44000 | Loss: 2.9739 | LR: 3.97e-05 | 702.0ms/step | 28.5 samples/s +Step 36150/44000 | Loss: 2.9151 | LR: 3.96e-05 | 703.1ms/step | 28.4 samples/s +Step 36160/44000 | Loss: 2.9150 | LR: 3.95e-05 | 703.6ms/step | 28.4 samples/s +Step 36170/44000 | Loss: 2.9646 | LR: 3.94e-05 | 700.5ms/step | 28.6 samples/s +Step 36180/44000 | Loss: 2.9330 | LR: 3.93e-05 | 701.8ms/step | 28.5 samples/s +Step 36190/44000 | Loss: 2.9172 | LR: 3.92e-05 | 701.7ms/step | 28.5 samples/s +Step 36200/44000 | Loss: 2.9318 | LR: 3.91e-05 | 703.4ms/step | 28.4 samples/s +Step 36210/44000 | Loss: 2.9220 | LR: 3.90e-05 | 701.5ms/step | 28.5 samples/s +Step 36220/44000 | Loss: 2.9680 | LR: 3.89e-05 | 701.1ms/step | 28.5 samples/s +Step 36230/44000 | Loss: 2.9292 | LR: 3.88e-05 | 702.3ms/step | 28.5 samples/s +Step 36240/44000 | Loss: 2.9291 | LR: 3.87e-05 | 699.1ms/step | 28.6 samples/s +Step 36250/44000 | Loss: 2.9080 | LR: 3.86e-05 | 699.9ms/step | 28.6 samples/s +Validation at step 36250: Loss = 2.9482, Perplexity = 19.07 +Step 36260/44000 | Loss: 2.9307 | LR: 3.85e-05 | 1271.0ms/step | 15.7 samples/s +Step 36270/44000 | Loss: 2.9193 | LR: 3.84e-05 | 703.8ms/step | 28.4 samples/s +Step 36280/44000 | Loss: 2.9478 | LR: 3.83e-05 | 702.0ms/step | 28.5 samples/s +Step 36290/44000 | Loss: 2.8922 | LR: 3.82e-05 | 701.4ms/step | 28.5 samples/s +Step 36300/44000 | Loss: 2.8744 | LR: 3.81e-05 | 704.2ms/step | 28.4 samples/s +Step 36310/44000 | Loss: 2.9103 | LR: 3.80e-05 | 699.2ms/step | 28.6 samples/s +Step 36320/44000 | Loss: 2.9161 | LR: 3.79e-05 | 701.2ms/step | 28.5 samples/s +Step 36330/44000 | Loss: 2.9205 | LR: 3.78e-05 | 698.6ms/step | 28.6 samples/s +Step 36340/44000 | Loss: 2.9574 | LR: 3.77e-05 | 698.7ms/step | 28.6 samples/s +Step 36350/44000 | Loss: 2.8946 | LR: 3.76e-05 | 701.4ms/step | 28.5 samples/s +Step 36360/44000 | Loss: 2.9057 | LR: 3.75e-05 | 701.5ms/step | 28.5 samples/s +Step 36370/44000 | Loss: 2.9274 | LR: 3.74e-05 | 702.5ms/step | 28.5 samples/s +Step 36380/44000 | Loss: 2.9646 | LR: 3.73e-05 | 700.0ms/step | 28.6 samples/s +Step 36390/44000 | Loss: 2.9034 | LR: 3.72e-05 | 701.9ms/step | 28.5 samples/s +Step 36400/44000 | Loss: 2.9215 | LR: 3.71e-05 | 701.2ms/step | 28.5 samples/s +Step 36410/44000 | Loss: 2.9334 | LR: 3.70e-05 | 702.4ms/step | 28.5 samples/s +Step 36420/44000 | Loss: 2.9554 | LR: 3.69e-05 | 701.6ms/step | 28.5 samples/s +Step 36430/44000 | Loss: 2.9522 | LR: 3.69e-05 | 702.6ms/step | 28.5 samples/s +Step 36440/44000 | Loss: 2.9014 | LR: 3.68e-05 | 700.4ms/step | 28.6 samples/s +Step 36450/44000 | Loss: 2.9198 | LR: 3.67e-05 | 702.2ms/step | 28.5 samples/s +Step 36460/44000 | Loss: 2.9077 | LR: 3.66e-05 | 703.3ms/step | 28.4 samples/s +Step 36470/44000 | Loss: 2.9392 | LR: 3.65e-05 | 703.3ms/step | 28.4 samples/s +Step 36480/44000 | Loss: 2.9728 | LR: 3.64e-05 | 702.1ms/step | 28.5 samples/s +Step 36490/44000 | Loss: 2.9337 | LR: 3.63e-05 | 698.6ms/step | 28.6 samples/s +Step 36500/44000 | Loss: 2.9378 | LR: 3.62e-05 | 699.9ms/step | 28.6 samples/s +Validation at step 36500: Loss = 2.9487, Perplexity = 19.08 +Step 36510/44000 | Loss: 2.9134 | LR: 3.61e-05 | 1269.7ms/step | 15.8 samples/s +Step 36520/44000 | Loss: 2.9294 | LR: 3.60e-05 | 700.5ms/step | 28.5 samples/s +Step 36530/44000 | Loss: 2.9452 | LR: 3.59e-05 | 699.8ms/step | 28.6 samples/s +Step 36540/44000 | Loss: 2.9212 | LR: 3.58e-05 | 698.9ms/step | 28.6 samples/s +Step 36550/44000 | Loss: 2.9132 | LR: 3.57e-05 | 698.7ms/step | 28.6 samples/s +Step 36560/44000 | Loss: 2.9498 | LR: 3.56e-05 | 697.4ms/step | 28.7 samples/s +Step 36570/44000 | Loss: 2.9288 | LR: 3.55e-05 | 699.0ms/step | 28.6 samples/s +Step 36580/44000 | Loss: 2.9386 | LR: 3.54e-05 | 700.7ms/step | 28.5 samples/s +Step 36590/44000 | Loss: 2.9326 | LR: 3.53e-05 | 700.6ms/step | 28.5 samples/s +Step 36600/44000 | Loss: 2.9744 | LR: 3.53e-05 | 698.6ms/step | 28.6 samples/s +Step 36610/44000 | Loss: 2.9657 | LR: 3.52e-05 | 699.7ms/step | 28.6 samples/s +Step 36620/44000 | Loss: 2.9477 | LR: 3.51e-05 | 697.2ms/step | 28.7 samples/s +Step 36630/44000 | Loss: 2.8890 | LR: 3.50e-05 | 702.4ms/step | 28.5 samples/s +Step 36640/44000 | Loss: 2.8825 | LR: 3.49e-05 | 701.2ms/step | 28.5 samples/s +Step 36650/44000 | Loss: 2.9049 | LR: 3.48e-05 | 700.7ms/step | 28.5 samples/s +Step 36660/44000 | Loss: 2.9466 | LR: 3.47e-05 | 702.7ms/step | 28.5 samples/s +Step 36670/44000 | Loss: 2.9580 | LR: 3.46e-05 | 700.8ms/step | 28.5 samples/s +Step 36680/44000 | Loss: 2.9256 | LR: 3.45e-05 | 702.9ms/step | 28.5 samples/s +Step 36690/44000 | Loss: 2.9290 | LR: 3.44e-05 | 703.7ms/step | 28.4 samples/s +Step 36700/44000 | Loss: 2.9507 | LR: 3.43e-05 | 704.5ms/step | 28.4 samples/s +Step 36710/44000 | Loss: 2.9387 | LR: 3.42e-05 | 701.5ms/step | 28.5 samples/s +Step 36720/44000 | Loss: 2.9657 | LR: 3.41e-05 | 699.7ms/step | 28.6 samples/s +Step 36730/44000 | Loss: 2.9067 | LR: 3.41e-05 | 701.6ms/step | 28.5 samples/s +Step 36740/44000 | Loss: 2.9151 | LR: 3.40e-05 | 701.8ms/step | 28.5 samples/s +Step 36750/44000 | Loss: 2.9612 | LR: 3.39e-05 | 703.6ms/step | 28.4 samples/s +Validation at step 36750: Loss = 2.9492, Perplexity = 19.09 +Step 36760/44000 | Loss: 2.9821 | LR: 3.38e-05 | 1272.5ms/step | 15.7 samples/s +Step 36770/44000 | Loss: 2.9641 | LR: 3.37e-05 | 700.7ms/step | 28.5 samples/s +Step 36780/44000 | Loss: 2.9285 | LR: 3.36e-05 | 701.7ms/step | 28.5 samples/s +Step 36790/44000 | Loss: 2.8984 | LR: 3.35e-05 | 700.1ms/step | 28.6 samples/s +Step 36800/44000 | Loss: 2.9011 | LR: 3.34e-05 | 700.5ms/step | 28.5 samples/s +Step 36810/44000 | Loss: 2.9280 | LR: 3.33e-05 | 698.8ms/step | 28.6 samples/s +Step 36820/44000 | Loss: 2.9106 | LR: 3.32e-05 | 697.4ms/step | 28.7 samples/s +Step 36830/44000 | Loss: 2.9146 | LR: 3.31e-05 | 700.6ms/step | 28.5 samples/s +Step 36840/44000 | Loss: 2.9850 | LR: 3.31e-05 | 701.8ms/step | 28.5 samples/s +Step 36850/44000 | Loss: 2.9234 | LR: 3.30e-05 | 699.9ms/step | 28.6 samples/s +Step 36860/44000 | Loss: 2.9233 | LR: 3.29e-05 | 701.9ms/step | 28.5 samples/s +Step 36870/44000 | Loss: 2.9398 | LR: 3.28e-05 | 702.0ms/step | 28.5 samples/s +Step 36880/44000 | Loss: 2.8990 | LR: 3.27e-05 | 701.9ms/step | 28.5 samples/s +Step 36890/44000 | Loss: 2.9482 | LR: 3.26e-05 | 697.6ms/step | 28.7 samples/s +Step 36900/44000 | Loss: 2.9079 | LR: 3.25e-05 | 697.8ms/step | 28.7 samples/s +Step 36910/44000 | Loss: 2.9254 | LR: 3.24e-05 | 700.2ms/step | 28.6 samples/s +Step 36920/44000 | Loss: 2.8919 | LR: 3.23e-05 | 703.3ms/step | 28.4 samples/s +Step 36930/44000 | Loss: 2.9352 | LR: 3.22e-05 | 699.9ms/step | 28.6 samples/s +Step 36940/44000 | Loss: 2.9443 | LR: 3.22e-05 | 700.2ms/step | 28.6 samples/s +Step 36950/44000 | Loss: 2.9391 | LR: 3.21e-05 | 700.2ms/step | 28.6 samples/s +Step 36960/44000 | Loss: 2.9400 | LR: 3.20e-05 | 701.8ms/step | 28.5 samples/s +Step 36970/44000 | Loss: 2.8920 | LR: 3.19e-05 | 702.1ms/step | 28.5 samples/s +Step 36980/44000 | Loss: 2.9240 | LR: 3.18e-05 | 702.6ms/step | 28.5 samples/s +Step 36990/44000 | Loss: 2.9354 | LR: 3.17e-05 | 703.1ms/step | 28.4 samples/s +Step 37000/44000 | Loss: 2.9709 | LR: 3.16e-05 | 700.6ms/step | 28.5 samples/s +Validation at step 37000: Loss = 2.9475, Perplexity = 19.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-37000 +Step 37010/44000 | Loss: 2.9241 | LR: 3.15e-05 | 1555.7ms/step | 12.9 samples/s +Step 37020/44000 | Loss: 2.8995 | LR: 3.15e-05 | 698.0ms/step | 28.7 samples/s +Step 37030/44000 | Loss: 2.8995 | LR: 3.14e-05 | 699.9ms/step | 28.6 samples/s +Step 37040/44000 | Loss: 2.9412 | LR: 3.13e-05 | 702.2ms/step | 28.5 samples/s +Step 37050/44000 | Loss: 2.9377 | LR: 3.12e-05 | 698.4ms/step | 28.6 samples/s +Step 37060/44000 | Loss: 2.9111 | LR: 3.11e-05 | 699.1ms/step | 28.6 samples/s +Step 37070/44000 | Loss: 2.9551 | LR: 3.10e-05 | 699.3ms/step | 28.6 samples/s +Step 37080/44000 | Loss: 2.9122 | LR: 3.09e-05 | 699.9ms/step | 28.6 samples/s +Step 37090/44000 | Loss: 2.9786 | LR: 3.08e-05 | 702.4ms/step | 28.5 samples/s +Step 37100/44000 | Loss: 2.9292 | LR: 3.07e-05 | 700.6ms/step | 28.5 samples/s +Step 37110/44000 | Loss: 2.8991 | LR: 3.07e-05 | 701.3ms/step | 28.5 samples/s +Step 37120/44000 | Loss: 2.9279 | LR: 3.06e-05 | 703.9ms/step | 28.4 samples/s +Step 37130/44000 | Loss: 2.9487 | LR: 3.05e-05 | 699.2ms/step | 28.6 samples/s +Step 37140/44000 | Loss: 2.9161 | LR: 3.04e-05 | 699.6ms/step | 28.6 samples/s +Step 37150/44000 | Loss: 2.9305 | LR: 3.03e-05 | 699.9ms/step | 28.6 samples/s +Step 37160/44000 | Loss: 2.9445 | LR: 3.02e-05 | 700.2ms/step | 28.6 samples/s +Step 37170/44000 | Loss: 2.9164 | LR: 3.01e-05 | 701.0ms/step | 28.5 samples/s +Step 37180/44000 | Loss: 2.9812 | LR: 3.01e-05 | 700.9ms/step | 28.5 samples/s +Step 37190/44000 | Loss: 2.8958 | LR: 3.00e-05 | 702.2ms/step | 28.5 samples/s +Step 37200/44000 | Loss: 2.9194 | LR: 2.99e-05 | 701.1ms/step | 28.5 samples/s +Step 37210/44000 | Loss: 2.9636 | LR: 2.98e-05 | 703.5ms/step | 28.4 samples/s +Step 37220/44000 | Loss: 2.9626 | LR: 2.97e-05 | 703.2ms/step | 28.4 samples/s +Step 37230/44000 | Loss: 2.9179 | LR: 2.96e-05 | 702.5ms/step | 28.5 samples/s +Step 37240/44000 | Loss: 2.9602 | LR: 2.95e-05 | 699.7ms/step | 28.6 samples/s +Step 37250/44000 | Loss: 2.9362 | LR: 2.95e-05 | 702.4ms/step | 28.5 samples/s +Validation at step 37250: Loss = 2.9486, Perplexity = 19.08 +Step 37260/44000 | Loss: 2.8667 | LR: 2.94e-05 | 1268.5ms/step | 15.8 samples/s +Step 37270/44000 | Loss: 2.8979 | LR: 2.93e-05 | 700.4ms/step | 28.6 samples/s +Step 37280/44000 | Loss: 2.9510 | LR: 2.92e-05 | 703.3ms/step | 28.4 samples/s +Step 37290/44000 | Loss: 2.9164 | LR: 2.91e-05 | 701.7ms/step | 28.5 samples/s +Step 37300/44000 | Loss: 2.9199 | LR: 2.90e-05 | 703.5ms/step | 28.4 samples/s +Step 37310/44000 | Loss: 2.9125 | LR: 2.89e-05 | 702.2ms/step | 28.5 samples/s +Step 37320/44000 | Loss: 2.8919 | LR: 2.89e-05 | 701.4ms/step | 28.5 samples/s +Step 37330/44000 | Loss: 2.9124 | LR: 2.88e-05 | 703.5ms/step | 28.4 samples/s +Step 37340/44000 | Loss: 2.9106 | LR: 2.87e-05 | 700.1ms/step | 28.6 samples/s +Step 37350/44000 | Loss: 2.9816 | LR: 2.86e-05 | 700.4ms/step | 28.6 samples/s +Step 37360/44000 | Loss: 2.9216 | LR: 2.85e-05 | 701.6ms/step | 28.5 samples/s +Step 37370/44000 | Loss: 2.9298 | LR: 2.84e-05 | 702.6ms/step | 28.5 samples/s +Step 37380/44000 | Loss: 2.9449 | LR: 2.84e-05 | 701.0ms/step | 28.5 samples/s +Step 37390/44000 | Loss: 2.9421 | LR: 2.83e-05 | 699.9ms/step | 28.6 samples/s +Step 37400/44000 | Loss: 2.8908 | LR: 2.82e-05 | 702.2ms/step | 28.5 samples/s +Step 37410/44000 | Loss: 2.9436 | LR: 2.81e-05 | 702.3ms/step | 28.5 samples/s +Step 37420/44000 | Loss: 2.8984 | LR: 2.80e-05 | 700.5ms/step | 28.5 samples/s +Step 37430/44000 | Loss: 2.8953 | LR: 2.79e-05 | 700.2ms/step | 28.6 samples/s +Step 37440/44000 | Loss: 2.9162 | LR: 2.78e-05 | 702.0ms/step | 28.5 samples/s +Step 37450/44000 | Loss: 2.9084 | LR: 2.78e-05 | 700.0ms/step | 28.6 samples/s +Step 37460/44000 | Loss: 2.9248 | LR: 2.77e-05 | 700.2ms/step | 28.6 samples/s +Step 37470/44000 | Loss: 2.9092 | LR: 2.76e-05 | 703.0ms/step | 28.5 samples/s +Step 37480/44000 | Loss: 2.9464 | LR: 2.75e-05 | 702.3ms/step | 28.5 samples/s +Step 37490/44000 | Loss: 2.9020 | LR: 2.74e-05 | 703.1ms/step | 28.4 samples/s +Step 37500/44000 | Loss: 2.9231 | LR: 2.74e-05 | 703.3ms/step | 28.4 samples/s +Validation at step 37500: Loss = 2.9489, Perplexity = 19.09 +Step 37510/44000 | Loss: 2.9279 | LR: 2.73e-05 | 1272.5ms/step | 15.7 samples/s +Step 37520/44000 | Loss: 2.9146 | LR: 2.72e-05 | 700.8ms/step | 28.5 samples/s +Step 37530/44000 | Loss: 2.9184 | LR: 2.71e-05 | 703.1ms/step | 28.4 samples/s +Step 37540/44000 | Loss: 2.9194 | LR: 2.70e-05 | 699.8ms/step | 28.6 samples/s +Step 37550/44000 | Loss: 2.8923 | LR: 2.69e-05 | 699.6ms/step | 28.6 samples/s +Step 37560/44000 | Loss: 2.9521 | LR: 2.69e-05 | 699.9ms/step | 28.6 samples/s +Step 37570/44000 | Loss: 2.9385 | LR: 2.68e-05 | 702.3ms/step | 28.5 samples/s +Step 37580/44000 | Loss: 2.9041 | LR: 2.67e-05 | 700.1ms/step | 28.6 samples/s +Step 37590/44000 | Loss: 2.9413 | LR: 2.66e-05 | 702.8ms/step | 28.5 samples/s +Step 37600/44000 | Loss: 2.9589 | LR: 2.65e-05 | 702.9ms/step | 28.5 samples/s +Step 37610/44000 | Loss: 2.9304 | LR: 2.65e-05 | 700.5ms/step | 28.6 samples/s +Step 37620/44000 | Loss: 2.9623 | LR: 2.64e-05 | 698.5ms/step | 28.6 samples/s +Step 37630/44000 | Loss: 2.9503 | LR: 2.63e-05 | 699.4ms/step | 28.6 samples/s +Step 37640/44000 | Loss: 2.8845 | LR: 2.62e-05 | 705.5ms/step | 28.3 samples/s +Step 37650/44000 | Loss: 2.9335 | LR: 2.61e-05 | 700.9ms/step | 28.5 samples/s +Step 37660/44000 | Loss: 2.8907 | LR: 2.60e-05 | 703.9ms/step | 28.4 samples/s +Step 37670/44000 | Loss: 2.9339 | LR: 2.60e-05 | 705.6ms/step | 28.3 samples/s +Step 37680/44000 | Loss: 2.9218 | LR: 2.59e-05 | 697.7ms/step | 28.7 samples/s +Step 37690/44000 | Loss: 2.8869 | LR: 2.58e-05 | 702.9ms/step | 28.5 samples/s +Step 37700/44000 | Loss: 2.9361 | LR: 2.57e-05 | 699.3ms/step | 28.6 samples/s +Step 37710/44000 | Loss: 2.9775 | LR: 2.56e-05 | 702.5ms/step | 28.5 samples/s +Step 37720/44000 | Loss: 2.9211 | LR: 2.56e-05 | 703.4ms/step | 28.4 samples/s +Step 37730/44000 | Loss: 2.9547 | LR: 2.55e-05 | 700.3ms/step | 28.6 samples/s +Step 37740/44000 | Loss: 2.9326 | LR: 2.54e-05 | 701.4ms/step | 28.5 samples/s +Step 37750/44000 | Loss: 2.9211 | LR: 2.53e-05 | 700.9ms/step | 28.5 samples/s +Validation at step 37750: Loss = 2.9476, Perplexity = 19.06 +Step 37760/44000 | Loss: 2.9532 | LR: 2.52e-05 | 1267.0ms/step | 15.8 samples/s +Step 37770/44000 | Loss: 2.9365 | LR: 2.52e-05 | 701.7ms/step | 28.5 samples/s +Step 37780/44000 | Loss: 2.9615 | LR: 2.51e-05 | 700.9ms/step | 28.5 samples/s +Step 37790/44000 | Loss: 2.8916 | LR: 2.50e-05 | 700.4ms/step | 28.6 samples/s +Step 37800/44000 | Loss: 2.9437 | LR: 2.49e-05 | 701.5ms/step | 28.5 samples/s +Step 37810/44000 | Loss: 2.9528 | LR: 2.48e-05 | 703.7ms/step | 28.4 samples/s +Step 37820/44000 | Loss: 2.9269 | LR: 2.48e-05 | 704.3ms/step | 28.4 samples/s +Step 37830/44000 | Loss: 2.9248 | LR: 2.47e-05 | 697.6ms/step | 28.7 samples/s +Step 37840/44000 | Loss: 2.9363 | LR: 2.46e-05 | 701.3ms/step | 28.5 samples/s +Step 37850/44000 | Loss: 2.9294 | LR: 2.45e-05 | 702.1ms/step | 28.5 samples/s +Step 37860/44000 | Loss: 2.9098 | LR: 2.45e-05 | 698.7ms/step | 28.6 samples/s +Step 37870/44000 | Loss: 2.9969 | LR: 2.44e-05 | 699.9ms/step | 28.6 samples/s +Step 37880/44000 | Loss: 2.9532 | LR: 2.43e-05 | 700.7ms/step | 28.5 samples/s +Step 37890/44000 | Loss: 2.9111 | LR: 2.42e-05 | 702.0ms/step | 28.5 samples/s +Step 37900/44000 | Loss: 2.9236 | LR: 2.41e-05 | 703.0ms/step | 28.5 samples/s +Step 37910/44000 | Loss: 2.8822 | LR: 2.41e-05 | 702.0ms/step | 28.5 samples/s +Step 37920/44000 | Loss: 2.9264 | LR: 2.40e-05 | 703.3ms/step | 28.4 samples/s +Step 37930/44000 | Loss: 2.9445 | LR: 2.39e-05 | 702.4ms/step | 28.5 samples/s +Step 37940/44000 | Loss: 2.9480 | LR: 2.38e-05 | 702.1ms/step | 28.5 samples/s +Step 37950/44000 | Loss: 2.9579 | LR: 2.38e-05 | 696.5ms/step | 28.7 samples/s +Step 37960/44000 | Loss: 2.8949 | LR: 2.37e-05 | 699.9ms/step | 28.6 samples/s +Step 37970/44000 | Loss: 2.9238 | LR: 2.36e-05 | 702.4ms/step | 28.5 samples/s +Step 37980/44000 | Loss: 2.9013 | LR: 2.35e-05 | 703.4ms/step | 28.4 samples/s +Step 37990/44000 | Loss: 2.9538 | LR: 2.34e-05 | 700.5ms/step | 28.6 samples/s +Step 38000/44000 | Loss: 2.9587 | LR: 2.34e-05 | 702.4ms/step | 28.5 samples/s +Validation at step 38000: Loss = 2.9479, Perplexity = 19.07 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-38000 +Step 38010/44000 | Loss: 2.9392 | LR: 2.33e-05 | 1555.9ms/step | 12.9 samples/s +Step 38020/44000 | Loss: 2.9510 | LR: 2.32e-05 | 699.0ms/step | 28.6 samples/s +Step 38030/44000 | Loss: 2.9468 | LR: 2.31e-05 | 702.8ms/step | 28.5 samples/s +Step 38040/44000 | Loss: 2.9617 | LR: 2.31e-05 | 699.9ms/step | 28.6 samples/s +Step 38050/44000 | Loss: 2.9248 | LR: 2.30e-05 | 706.8ms/step | 28.3 samples/s +Step 38060/44000 | Loss: 2.9635 | LR: 2.29e-05 | 699.1ms/step | 28.6 samples/s +Step 38070/44000 | Loss: 2.8867 | LR: 2.28e-05 | 698.8ms/step | 28.6 samples/s +Step 38080/44000 | Loss: 2.9618 | LR: 2.28e-05 | 703.7ms/step | 28.4 samples/s +Step 38090/44000 | Loss: 2.9071 | LR: 2.27e-05 | 700.7ms/step | 28.5 samples/s +Step 38100/44000 | Loss: 2.8946 | LR: 2.26e-05 | 700.5ms/step | 28.6 samples/s +Step 38110/44000 | Loss: 2.8898 | LR: 2.25e-05 | 702.3ms/step | 28.5 samples/s +Step 38120/44000 | Loss: 2.9161 | LR: 2.25e-05 | 701.5ms/step | 28.5 samples/s +Step 38130/44000 | Loss: 2.9492 | LR: 2.24e-05 | 699.8ms/step | 28.6 samples/s +Step 38140/44000 | Loss: 2.9019 | LR: 2.23e-05 | 702.7ms/step | 28.5 samples/s +Step 38150/44000 | Loss: 2.9436 | LR: 2.22e-05 | 706.4ms/step | 28.3 samples/s +Step 38160/44000 | Loss: 2.9276 | LR: 2.22e-05 | 703.7ms/step | 28.4 samples/s +Step 38170/44000 | Loss: 2.9006 | LR: 2.21e-05 | 700.8ms/step | 28.5 samples/s +Step 38180/44000 | Loss: 2.9338 | LR: 2.20e-05 | 702.0ms/step | 28.5 samples/s +Step 38190/44000 | Loss: 2.9280 | LR: 2.19e-05 | 701.4ms/step | 28.5 samples/s +Step 38200/44000 | Loss: 2.9405 | LR: 2.19e-05 | 701.8ms/step | 28.5 samples/s +Step 38210/44000 | Loss: 2.9576 | LR: 2.18e-05 | 700.6ms/step | 28.5 samples/s +Step 38220/44000 | Loss: 2.9563 | LR: 2.17e-05 | 702.0ms/step | 28.5 samples/s +Step 38230/44000 | Loss: 2.9216 | LR: 2.16e-05 | 698.8ms/step | 28.6 samples/s +Step 38240/44000 | Loss: 2.9825 | LR: 2.16e-05 | 699.4ms/step | 28.6 samples/s +Step 38250/44000 | Loss: 2.9339 | LR: 2.15e-05 | 699.9ms/step | 28.6 samples/s +Validation at step 38250: Loss = 2.9483, Perplexity = 19.07 +Step 38260/44000 | Loss: 2.9088 | LR: 2.14e-05 | 1267.0ms/step | 15.8 samples/s +Step 38270/44000 | Loss: 2.9366 | LR: 2.13e-05 | 702.7ms/step | 28.5 samples/s +Step 38280/44000 | Loss: 2.9756 | LR: 2.13e-05 | 698.8ms/step | 28.6 samples/s +Step 38290/44000 | Loss: 2.9099 | LR: 2.12e-05 | 702.8ms/step | 28.5 samples/s +Step 38300/44000 | Loss: 2.9809 | LR: 2.11e-05 | 699.8ms/step | 28.6 samples/s +Step 38310/44000 | Loss: 2.9332 | LR: 2.11e-05 | 700.3ms/step | 28.6 samples/s +Step 38320/44000 | Loss: 2.9241 | LR: 2.10e-05 | 704.4ms/step | 28.4 samples/s +Step 38330/44000 | Loss: 2.9606 | LR: 2.09e-05 | 702.2ms/step | 28.5 samples/s +Step 38340/44000 | Loss: 2.9485 | LR: 2.08e-05 | 697.9ms/step | 28.7 samples/s +Step 38350/44000 | Loss: 2.9650 | LR: 2.08e-05 | 699.3ms/step | 28.6 samples/s +Step 38360/44000 | Loss: 2.8718 | LR: 2.07e-05 | 702.3ms/step | 28.5 samples/s +Step 38370/44000 | Loss: 2.9456 | LR: 2.06e-05 | 701.0ms/step | 28.5 samples/s +Step 38380/44000 | Loss: 2.9773 | LR: 2.05e-05 | 702.1ms/step | 28.5 samples/s +Step 38390/44000 | Loss: 2.8997 | LR: 2.05e-05 | 701.9ms/step | 28.5 samples/s +Step 38400/44000 | Loss: 2.9553 | LR: 2.04e-05 | 702.8ms/step | 28.5 samples/s +Step 38410/44000 | Loss: 2.9239 | LR: 2.03e-05 | 702.4ms/step | 28.5 samples/s +Step 38420/44000 | Loss: 2.8795 | LR: 2.03e-05 | 699.6ms/step | 28.6 samples/s +Step 38430/44000 | Loss: 2.9154 | LR: 2.02e-05 | 699.8ms/step | 28.6 samples/s +Step 38440/44000 | Loss: 2.9354 | LR: 2.01e-05 | 700.6ms/step | 28.5 samples/s +Step 38450/44000 | Loss: 2.9382 | LR: 2.00e-05 | 700.0ms/step | 28.6 samples/s +Step 38460/44000 | Loss: 2.9723 | LR: 2.00e-05 | 698.8ms/step | 28.6 samples/s +Step 38470/44000 | Loss: 2.9277 | LR: 1.99e-05 | 701.1ms/step | 28.5 samples/s +Step 38480/44000 | Loss: 2.9187 | LR: 1.98e-05 | 699.6ms/step | 28.6 samples/s +Step 38490/44000 | Loss: 2.9470 | LR: 1.98e-05 | 697.2ms/step | 28.7 samples/s +Step 38500/44000 | Loss: 2.9692 | LR: 1.97e-05 | 701.8ms/step | 28.5 samples/s +Validation at step 38500: Loss = 2.9475, Perplexity = 19.06 +Step 38510/44000 | Loss: 2.9473 | LR: 1.96e-05 | 1266.3ms/step | 15.8 samples/s +Step 38520/44000 | Loss: 2.9234 | LR: 1.95e-05 | 700.8ms/step | 28.5 samples/s +Step 38530/44000 | Loss: 2.9316 | LR: 1.95e-05 | 699.3ms/step | 28.6 samples/s +Step 38540/44000 | Loss: 2.9253 | LR: 1.94e-05 | 700.7ms/step | 28.5 samples/s +Step 38550/44000 | Loss: 2.9441 | LR: 1.93e-05 | 699.9ms/step | 28.6 samples/s +Step 38560/44000 | Loss: 2.9190 | LR: 1.93e-05 | 702.0ms/step | 28.5 samples/s +Step 38570/44000 | Loss: 2.9330 | LR: 1.92e-05 | 702.0ms/step | 28.5 samples/s +Step 38580/44000 | Loss: 2.9398 | LR: 1.91e-05 | 700.0ms/step | 28.6 samples/s +Step 38590/44000 | Loss: 2.8787 | LR: 1.91e-05 | 700.4ms/step | 28.6 samples/s +Step 38600/44000 | Loss: 2.9293 | LR: 1.90e-05 | 701.9ms/step | 28.5 samples/s +Step 38610/44000 | Loss: 2.9301 | LR: 1.89e-05 | 702.2ms/step | 28.5 samples/s +Step 38620/44000 | Loss: 2.9736 | LR: 1.88e-05 | 703.9ms/step | 28.4 samples/s +Step 38630/44000 | Loss: 2.9562 | LR: 1.88e-05 | 699.3ms/step | 28.6 samples/s +Step 38640/44000 | Loss: 2.9597 | LR: 1.87e-05 | 699.9ms/step | 28.6 samples/s +Step 38650/44000 | Loss: 2.9380 | LR: 1.86e-05 | 700.2ms/step | 28.6 samples/s +Step 38660/44000 | Loss: 2.9332 | LR: 1.86e-05 | 699.8ms/step | 28.6 samples/s +Step 38670/44000 | Loss: 2.8945 | LR: 1.85e-05 | 699.6ms/step | 28.6 samples/s +Step 38680/44000 | Loss: 2.9483 | LR: 1.84e-05 | 699.1ms/step | 28.6 samples/s +Step 38690/44000 | Loss: 2.9579 | LR: 1.84e-05 | 701.4ms/step | 28.5 samples/s +Step 38700/44000 | Loss: 2.9706 | LR: 1.83e-05 | 701.0ms/step | 28.5 samples/s +Step 38710/44000 | Loss: 2.9367 | LR: 1.82e-05 | 699.3ms/step | 28.6 samples/s +Step 38720/44000 | Loss: 2.9376 | LR: 1.82e-05 | 701.0ms/step | 28.5 samples/s +Step 38730/44000 | Loss: 2.8907 | LR: 1.81e-05 | 701.2ms/step | 28.5 samples/s +Step 38740/44000 | Loss: 2.8577 | LR: 1.80e-05 | 702.0ms/step | 28.5 samples/s +Step 38750/44000 | Loss: 2.9242 | LR: 1.80e-05 | 698.6ms/step | 28.6 samples/s +Validation at step 38750: Loss = 2.9479, Perplexity = 19.07 +Step 38760/44000 | Loss: 2.9397 | LR: 1.79e-05 | 1265.2ms/step | 15.8 samples/s +Step 38770/44000 | Loss: 2.8993 | LR: 1.78e-05 | 698.6ms/step | 28.6 samples/s +Step 38780/44000 | Loss: 2.9705 | LR: 1.78e-05 | 699.5ms/step | 28.6 samples/s +Step 38790/44000 | Loss: 2.9832 | LR: 1.77e-05 | 700.1ms/step | 28.6 samples/s +Step 38800/44000 | Loss: 2.9551 | LR: 1.76e-05 | 700.4ms/step | 28.6 samples/s +Step 38810/44000 | Loss: 2.9387 | LR: 1.76e-05 | 699.7ms/step | 28.6 samples/s +Step 38820/44000 | Loss: 2.8941 | LR: 1.75e-05 | 700.0ms/step | 28.6 samples/s +Step 38830/44000 | Loss: 2.9056 | LR: 1.74e-05 | 702.5ms/step | 28.5 samples/s +Step 38840/44000 | Loss: 2.9143 | LR: 1.74e-05 | 703.1ms/step | 28.4 samples/s +Step 38850/44000 | Loss: 2.9115 | LR: 1.73e-05 | 701.8ms/step | 28.5 samples/s +Step 38860/44000 | Loss: 2.9193 | LR: 1.72e-05 | 701.7ms/step | 28.5 samples/s +Step 38870/44000 | Loss: 2.9193 | LR: 1.72e-05 | 702.6ms/step | 28.5 samples/s +Step 38880/44000 | Loss: 2.9486 | LR: 1.71e-05 | 703.0ms/step | 28.5 samples/s +Step 38890/44000 | Loss: 2.9613 | LR: 1.70e-05 | 703.2ms/step | 28.4 samples/s +Step 38900/44000 | Loss: 2.9603 | LR: 1.70e-05 | 705.5ms/step | 28.3 samples/s +Step 38910/44000 | Loss: 2.9519 | LR: 1.69e-05 | 700.0ms/step | 28.6 samples/s +Step 38920/44000 | Loss: 2.9328 | LR: 1.68e-05 | 704.4ms/step | 28.4 samples/s +Step 38930/44000 | Loss: 2.9556 | LR: 1.68e-05 | 700.3ms/step | 28.6 samples/s +Step 38940/44000 | Loss: 2.9146 | LR: 1.67e-05 | 703.5ms/step | 28.4 samples/s +Step 38950/44000 | Loss: 2.9410 | LR: 1.66e-05 | 701.3ms/step | 28.5 samples/s +Step 38960/44000 | Loss: 2.9332 | LR: 1.66e-05 | 701.2ms/step | 28.5 samples/s +Step 38970/44000 | Loss: 2.9665 | LR: 1.65e-05 | 700.8ms/step | 28.5 samples/s +Step 38980/44000 | Loss: 2.9220 | LR: 1.64e-05 | 703.3ms/step | 28.4 samples/s +Step 38990/44000 | Loss: 2.9241 | LR: 1.64e-05 | 704.4ms/step | 28.4 samples/s +Step 39000/44000 | Loss: 2.9200 | LR: 1.63e-05 | 700.9ms/step | 28.5 samples/s +Validation at step 39000: Loss = 2.9479, Perplexity = 19.07 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-39000 +Step 39010/44000 | Loss: 2.9013 | LR: 1.62e-05 | 1574.2ms/step | 12.7 samples/s +Step 39020/44000 | Loss: 2.9600 | LR: 1.62e-05 | 699.8ms/step | 28.6 samples/s +Step 39030/44000 | Loss: 2.9358 | LR: 1.61e-05 | 700.5ms/step | 28.5 samples/s +Step 39040/44000 | Loss: 2.9123 | LR: 1.61e-05 | 702.6ms/step | 28.5 samples/s +Step 39050/44000 | Loss: 2.9534 | LR: 1.60e-05 | 703.2ms/step | 28.4 samples/s +Step 39060/44000 | Loss: 2.9386 | LR: 1.59e-05 | 697.3ms/step | 28.7 samples/s +Step 39070/44000 | Loss: 2.9539 | LR: 1.59e-05 | 698.3ms/step | 28.6 samples/s +Step 39080/44000 | Loss: 2.9119 | LR: 1.58e-05 | 704.6ms/step | 28.4 samples/s +Step 39090/44000 | Loss: 2.9435 | LR: 1.57e-05 | 704.0ms/step | 28.4 samples/s +Step 39100/44000 | Loss: 2.9198 | LR: 1.57e-05 | 700.4ms/step | 28.6 samples/s +Step 39110/44000 | Loss: 2.9381 | LR: 1.56e-05 | 703.6ms/step | 28.4 samples/s +Step 39120/44000 | Loss: 2.9474 | LR: 1.55e-05 | 699.3ms/step | 28.6 samples/s +Step 39130/44000 | Loss: 2.9164 | LR: 1.55e-05 | 700.2ms/step | 28.6 samples/s +Step 39140/44000 | Loss: 2.9563 | LR: 1.54e-05 | 702.0ms/step | 28.5 samples/s +Step 39150/44000 | Loss: 2.9070 | LR: 1.54e-05 | 704.1ms/step | 28.4 samples/s +Step 39160/44000 | Loss: 2.9260 | LR: 1.53e-05 | 700.6ms/step | 28.5 samples/s +Step 39170/44000 | Loss: 2.9204 | LR: 1.52e-05 | 704.9ms/step | 28.4 samples/s +Step 39180/44000 | Loss: 2.9662 | LR: 1.52e-05 | 701.8ms/step | 28.5 samples/s +Step 39190/44000 | Loss: 2.8846 | LR: 1.51e-05 | 702.8ms/step | 28.5 samples/s +Step 39200/44000 | Loss: 2.9131 | LR: 1.50e-05 | 703.6ms/step | 28.4 samples/s +Step 39210/44000 | Loss: 2.9243 | LR: 1.50e-05 | 702.7ms/step | 28.5 samples/s +Step 39220/44000 | Loss: 2.9212 | LR: 1.49e-05 | 701.6ms/step | 28.5 samples/s +Step 39230/44000 | Loss: 2.8898 | LR: 1.49e-05 | 699.6ms/step | 28.6 samples/s +Step 39240/44000 | Loss: 2.9109 | LR: 1.48e-05 | 701.2ms/step | 28.5 samples/s +Step 39250/44000 | Loss: 2.8987 | LR: 1.47e-05 | 705.1ms/step | 28.4 samples/s +Validation at step 39250: Loss = 2.9471, Perplexity = 19.05 +Step 39260/44000 | Loss: 2.9405 | LR: 1.47e-05 | 1269.1ms/step | 15.8 samples/s +Step 39270/44000 | Loss: 2.9317 | LR: 1.46e-05 | 701.1ms/step | 28.5 samples/s +Step 39280/44000 | Loss: 2.8844 | LR: 1.46e-05 | 702.0ms/step | 28.5 samples/s +Step 39290/44000 | Loss: 2.9622 | LR: 1.45e-05 | 699.3ms/step | 28.6 samples/s +Step 39300/44000 | Loss: 2.9006 | LR: 1.44e-05 | 699.3ms/step | 28.6 samples/s +Step 39310/44000 | Loss: 2.9431 | LR: 1.44e-05 | 701.2ms/step | 28.5 samples/s +Step 39320/44000 | Loss: 2.9604 | LR: 1.43e-05 | 703.7ms/step | 28.4 samples/s +Step 39330/44000 | Loss: 2.9994 | LR: 1.42e-05 | 700.4ms/step | 28.6 samples/s +Step 39340/44000 | Loss: 2.9853 | LR: 1.42e-05 | 704.6ms/step | 28.4 samples/s +Step 39350/44000 | Loss: 2.9198 | LR: 1.41e-05 | 704.2ms/step | 28.4 samples/s +Step 39360/44000 | Loss: 2.9107 | LR: 1.41e-05 | 699.9ms/step | 28.6 samples/s +Step 39370/44000 | Loss: 2.9266 | LR: 1.40e-05 | 702.9ms/step | 28.5 samples/s +Step 39380/44000 | Loss: 2.9592 | LR: 1.39e-05 | 705.1ms/step | 28.4 samples/s +Step 39390/44000 | Loss: 2.9466 | LR: 1.39e-05 | 701.6ms/step | 28.5 samples/s +Step 39400/44000 | Loss: 2.9432 | LR: 1.38e-05 | 702.0ms/step | 28.5 samples/s +Step 39410/44000 | Loss: 2.9456 | LR: 1.38e-05 | 705.8ms/step | 28.3 samples/s +Step 39420/44000 | Loss: 2.9274 | LR: 1.37e-05 | 702.9ms/step | 28.5 samples/s +Step 39430/44000 | Loss: 2.9217 | LR: 1.36e-05 | 696.7ms/step | 28.7 samples/s +Step 39440/44000 | Loss: 2.9083 | LR: 1.36e-05 | 699.2ms/step | 28.6 samples/s +Step 39450/44000 | Loss: 2.9090 | LR: 1.35e-05 | 704.1ms/step | 28.4 samples/s +Step 39460/44000 | Loss: 2.9391 | LR: 1.35e-05 | 701.6ms/step | 28.5 samples/s +Step 39470/44000 | Loss: 2.9482 | LR: 1.34e-05 | 701.8ms/step | 28.5 samples/s +Step 39480/44000 | Loss: 2.9311 | LR: 1.34e-05 | 699.7ms/step | 28.6 samples/s +Step 39490/44000 | Loss: 2.9042 | LR: 1.33e-05 | 702.5ms/step | 28.5 samples/s +Step 39500/44000 | Loss: 2.9606 | LR: 1.32e-05 | 701.9ms/step | 28.5 samples/s +Validation at step 39500: Loss = 2.9486, Perplexity = 19.08 +Step 39510/44000 | Loss: 2.9305 | LR: 1.32e-05 | 1270.4ms/step | 15.7 samples/s +Step 39520/44000 | Loss: 2.9334 | LR: 1.31e-05 | 700.2ms/step | 28.6 samples/s +Step 39530/44000 | Loss: 2.9667 | LR: 1.31e-05 | 700.8ms/step | 28.5 samples/s +Step 39540/44000 | Loss: 2.9265 | LR: 1.30e-05 | 700.6ms/step | 28.5 samples/s +Step 39550/44000 | Loss: 2.9327 | LR: 1.29e-05 | 703.2ms/step | 28.4 samples/s +Step 39560/44000 | Loss: 2.9108 | LR: 1.29e-05 | 700.1ms/step | 28.6 samples/s +Step 39570/44000 | Loss: 2.9111 | LR: 1.28e-05 | 704.4ms/step | 28.4 samples/s +Step 39580/44000 | Loss: 2.8959 | LR: 1.28e-05 | 704.1ms/step | 28.4 samples/s +Step 39590/44000 | Loss: 2.9376 | LR: 1.27e-05 | 701.8ms/step | 28.5 samples/s +Step 39600/44000 | Loss: 2.9497 | LR: 1.27e-05 | 700.8ms/step | 28.5 samples/s +Step 39610/44000 | Loss: 2.8917 | LR: 1.26e-05 | 702.0ms/step | 28.5 samples/s +Step 39620/44000 | Loss: 2.9068 | LR: 1.25e-05 | 702.3ms/step | 28.5 samples/s +Step 39630/44000 | Loss: 2.9314 | LR: 1.25e-05 | 700.6ms/step | 28.5 samples/s +Step 39640/44000 | Loss: 2.9462 | LR: 1.24e-05 | 700.3ms/step | 28.6 samples/s +Step 39650/44000 | Loss: 2.9432 | LR: 1.24e-05 | 700.0ms/step | 28.6 samples/s +Step 39660/44000 | Loss: 2.9193 | LR: 1.23e-05 | 700.1ms/step | 28.6 samples/s +Step 39670/44000 | Loss: 2.9790 | LR: 1.23e-05 | 702.3ms/step | 28.5 samples/s +Step 39680/44000 | Loss: 2.9269 | LR: 1.22e-05 | 698.5ms/step | 28.6 samples/s +Step 39690/44000 | Loss: 2.9480 | LR: 1.22e-05 | 702.8ms/step | 28.5 samples/s +Step 39700/44000 | Loss: 2.9577 | LR: 1.21e-05 | 701.9ms/step | 28.5 samples/s +Step 39710/44000 | Loss: 2.9664 | LR: 1.20e-05 | 701.6ms/step | 28.5 samples/s +Step 39720/44000 | Loss: 2.9713 | LR: 1.20e-05 | 701.2ms/step | 28.5 samples/s +Step 39730/44000 | Loss: 2.9456 | LR: 1.19e-05 | 701.0ms/step | 28.5 samples/s +Step 39740/44000 | Loss: 2.9012 | LR: 1.19e-05 | 700.2ms/step | 28.6 samples/s +Step 39750/44000 | Loss: 2.8999 | LR: 1.18e-05 | 701.2ms/step | 28.5 samples/s +Validation at step 39750: Loss = 2.9481, Perplexity = 19.07 +Step 39760/44000 | Loss: 2.9480 | LR: 1.18e-05 | 1272.4ms/step | 15.7 samples/s +Step 39770/44000 | Loss: 2.9100 | LR: 1.17e-05 | 699.2ms/step | 28.6 samples/s +Step 39780/44000 | Loss: 2.9518 | LR: 1.17e-05 | 704.2ms/step | 28.4 samples/s +Step 39790/44000 | Loss: 2.9340 | LR: 1.16e-05 | 700.9ms/step | 28.5 samples/s +Step 39800/44000 | Loss: 2.9463 | LR: 1.15e-05 | 700.9ms/step | 28.5 samples/s +Step 39810/44000 | Loss: 2.9306 | LR: 1.15e-05 | 699.9ms/step | 28.6 samples/s +Step 39820/44000 | Loss: 2.9137 | LR: 1.14e-05 | 701.7ms/step | 28.5 samples/s +Step 39830/44000 | Loss: 2.9566 | LR: 1.14e-05 | 699.8ms/step | 28.6 samples/s +Step 39840/44000 | Loss: 2.9529 | LR: 1.13e-05 | 699.9ms/step | 28.6 samples/s +Step 39850/44000 | Loss: 2.9090 | LR: 1.13e-05 | 699.3ms/step | 28.6 samples/s +Step 39860/44000 | Loss: 2.9159 | LR: 1.12e-05 | 700.5ms/step | 28.5 samples/s +Step 39870/44000 | Loss: 2.9483 | LR: 1.12e-05 | 697.8ms/step | 28.7 samples/s +Step 39880/44000 | Loss: 2.9129 | LR: 1.11e-05 | 702.5ms/step | 28.5 samples/s +Step 39890/44000 | Loss: 2.8814 | LR: 1.11e-05 | 702.7ms/step | 28.5 samples/s +Step 39900/44000 | Loss: 2.9331 | LR: 1.10e-05 | 697.9ms/step | 28.7 samples/s +Step 39910/44000 | Loss: 2.9507 | LR: 1.10e-05 | 700.8ms/step | 28.5 samples/s +Step 39920/44000 | Loss: 2.9230 | LR: 1.09e-05 | 701.6ms/step | 28.5 samples/s +Step 39930/44000 | Loss: 2.9641 | LR: 1.08e-05 | 701.6ms/step | 28.5 samples/s +Step 39940/44000 | Loss: 2.9291 | LR: 1.08e-05 | 701.4ms/step | 28.5 samples/s +Step 39950/44000 | Loss: 2.9316 | LR: 1.07e-05 | 703.8ms/step | 28.4 samples/s +Step 39960/44000 | Loss: 2.9172 | LR: 1.07e-05 | 702.0ms/step | 28.5 samples/s +Step 39970/44000 | Loss: 2.9586 | LR: 1.06e-05 | 699.8ms/step | 28.6 samples/s +Step 39980/44000 | Loss: 2.9335 | LR: 1.06e-05 | 698.4ms/step | 28.6 samples/s +Step 39990/44000 | Loss: 2.9178 | LR: 1.05e-05 | 702.9ms/step | 28.5 samples/s +Step 40000/44000 | Loss: 2.9447 | LR: 1.05e-05 | 699.7ms/step | 28.6 samples/s +Validation at step 40000: Loss = 2.9471, Perplexity = 19.05 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-40000 +Step 40010/44000 | Loss: 2.9354 | LR: 1.04e-05 | 1559.0ms/step | 12.8 samples/s +Step 40020/44000 | Loss: 2.8864 | LR: 1.04e-05 | 695.3ms/step | 28.8 samples/s +Step 40030/44000 | Loss: 2.8958 | LR: 1.03e-05 | 702.4ms/step | 28.5 samples/s +Step 40040/44000 | Loss: 2.8933 | LR: 1.03e-05 | 702.1ms/step | 28.5 samples/s +Step 40050/44000 | Loss: 2.9306 | LR: 1.02e-05 | 699.2ms/step | 28.6 samples/s +Step 40060/44000 | Loss: 2.9117 | LR: 1.02e-05 | 700.8ms/step | 28.5 samples/s +Step 40070/44000 | Loss: 2.9325 | LR: 1.01e-05 | 703.1ms/step | 28.4 samples/s +Step 40080/44000 | Loss: 2.9943 | LR: 1.01e-05 | 704.3ms/step | 28.4 samples/s +Step 40090/44000 | Loss: 2.9139 | LR: 1.00e-05 | 700.1ms/step | 28.6 samples/s +Step 40100/44000 | Loss: 2.9244 | LR: 9.96e-06 | 701.8ms/step | 28.5 samples/s +Step 40110/44000 | Loss: 2.9468 | LR: 9.91e-06 | 700.3ms/step | 28.6 samples/s +Step 40120/44000 | Loss: 2.8948 | LR: 9.86e-06 | 703.9ms/step | 28.4 samples/s +Step 40130/44000 | Loss: 2.9179 | LR: 9.81e-06 | 703.2ms/step | 28.4 samples/s +Step 40140/44000 | Loss: 2.9228 | LR: 9.76e-06 | 701.1ms/step | 28.5 samples/s +Step 40150/44000 | Loss: 2.9111 | LR: 9.71e-06 | 699.0ms/step | 28.6 samples/s +Step 40160/44000 | Loss: 2.9485 | LR: 9.66e-06 | 701.5ms/step | 28.5 samples/s +Step 40170/44000 | Loss: 2.9428 | LR: 9.61e-06 | 699.2ms/step | 28.6 samples/s +Step 40180/44000 | Loss: 2.9138 | LR: 9.56e-06 | 700.6ms/step | 28.5 samples/s +Step 40190/44000 | Loss: 2.9325 | LR: 9.51e-06 | 700.2ms/step | 28.6 samples/s +Step 40200/44000 | Loss: 2.9284 | LR: 9.46e-06 | 702.8ms/step | 28.5 samples/s +Step 40210/44000 | Loss: 2.9588 | LR: 9.41e-06 | 699.8ms/step | 28.6 samples/s +Step 40220/44000 | Loss: 2.9402 | LR: 9.36e-06 | 700.7ms/step | 28.5 samples/s +Step 40230/44000 | Loss: 2.9708 | LR: 9.32e-06 | 701.6ms/step | 28.5 samples/s +Step 40240/44000 | Loss: 2.9031 | LR: 9.27e-06 | 702.5ms/step | 28.5 samples/s +Step 40250/44000 | Loss: 2.9471 | LR: 9.22e-06 | 703.2ms/step | 28.4 samples/s +Validation at step 40250: Loss = 2.9478, Perplexity = 19.06 +Step 40260/44000 | Loss: 2.9600 | LR: 9.17e-06 | 1270.4ms/step | 15.7 samples/s +Step 40270/44000 | Loss: 2.9207 | LR: 9.12e-06 | 698.9ms/step | 28.6 samples/s +Step 40280/44000 | Loss: 2.9283 | LR: 9.07e-06 | 702.5ms/step | 28.5 samples/s +Step 40290/44000 | Loss: 2.9269 | LR: 9.02e-06 | 699.8ms/step | 28.6 samples/s +Step 40300/44000 | Loss: 2.9081 | LR: 8.97e-06 | 700.3ms/step | 28.6 samples/s +Step 40310/44000 | Loss: 2.9336 | LR: 8.93e-06 | 702.8ms/step | 28.5 samples/s +Step 40320/44000 | Loss: 2.9550 | LR: 8.88e-06 | 699.7ms/step | 28.6 samples/s +Step 40330/44000 | Loss: 2.9138 | LR: 8.83e-06 | 704.8ms/step | 28.4 samples/s +Step 40340/44000 | Loss: 2.9521 | LR: 8.78e-06 | 701.0ms/step | 28.5 samples/s +Step 40350/44000 | Loss: 2.9342 | LR: 8.74e-06 | 702.4ms/step | 28.5 samples/s +Step 40360/44000 | Loss: 2.9666 | LR: 8.69e-06 | 698.4ms/step | 28.6 samples/s +Step 40370/44000 | Loss: 2.8863 | LR: 8.64e-06 | 702.4ms/step | 28.5 samples/s +Step 40380/44000 | Loss: 2.8993 | LR: 8.59e-06 | 701.7ms/step | 28.5 samples/s +Step 40390/44000 | Loss: 2.9405 | LR: 8.55e-06 | 698.8ms/step | 28.6 samples/s +Step 40400/44000 | Loss: 2.9152 | LR: 8.50e-06 | 700.1ms/step | 28.6 samples/s +Step 40410/44000 | Loss: 2.9486 | LR: 8.45e-06 | 703.4ms/step | 28.4 samples/s +Step 40420/44000 | Loss: 2.9452 | LR: 8.41e-06 | 701.5ms/step | 28.5 samples/s +Step 40430/44000 | Loss: 2.9375 | LR: 8.36e-06 | 701.7ms/step | 28.5 samples/s +Step 40440/44000 | Loss: 2.9518 | LR: 8.31e-06 | 700.8ms/step | 28.5 samples/s +Step 40450/44000 | Loss: 2.9259 | LR: 8.27e-06 | 701.6ms/step | 28.5 samples/s +Step 40460/44000 | Loss: 2.9578 | LR: 8.22e-06 | 699.9ms/step | 28.6 samples/s +Step 40470/44000 | Loss: 2.9021 | LR: 8.17e-06 | 697.8ms/step | 28.7 samples/s +Step 40480/44000 | Loss: 2.9344 | LR: 8.13e-06 | 697.3ms/step | 28.7 samples/s +Step 40490/44000 | Loss: 2.9489 | LR: 8.08e-06 | 701.2ms/step | 28.5 samples/s +Step 40500/44000 | Loss: 2.9486 | LR: 8.04e-06 | 699.6ms/step | 28.6 samples/s +Validation at step 40500: Loss = 2.9481, Perplexity = 19.07 +Step 40510/44000 | Loss: 2.9315 | LR: 7.99e-06 | 1267.2ms/step | 15.8 samples/s +Step 40520/44000 | Loss: 2.9200 | LR: 7.94e-06 | 701.1ms/step | 28.5 samples/s +Step 40530/44000 | Loss: 2.9364 | LR: 7.90e-06 | 696.3ms/step | 28.7 samples/s +Step 40540/44000 | Loss: 2.8939 | LR: 7.85e-06 | 704.2ms/step | 28.4 samples/s +Step 40550/44000 | Loss: 2.9672 | LR: 7.81e-06 | 701.4ms/step | 28.5 samples/s +Step 40560/44000 | Loss: 2.9212 | LR: 7.76e-06 | 700.6ms/step | 28.5 samples/s +Step 40570/44000 | Loss: 2.9314 | LR: 7.72e-06 | 699.2ms/step | 28.6 samples/s +Step 40580/44000 | Loss: 2.9702 | LR: 7.67e-06 | 699.1ms/step | 28.6 samples/s +Step 40590/44000 | Loss: 2.9124 | LR: 7.63e-06 | 701.5ms/step | 28.5 samples/s +Step 40600/44000 | Loss: 2.9233 | LR: 7.59e-06 | 699.2ms/step | 28.6 samples/s +Step 40610/44000 | Loss: 2.9311 | LR: 7.54e-06 | 700.0ms/step | 28.6 samples/s +Step 40620/44000 | Loss: 2.9489 | LR: 7.50e-06 | 699.9ms/step | 28.6 samples/s +Step 40630/44000 | Loss: 2.8965 | LR: 7.45e-06 | 702.3ms/step | 28.5 samples/s +Step 40640/44000 | Loss: 2.9336 | LR: 7.41e-06 | 703.9ms/step | 28.4 samples/s +Step 40650/44000 | Loss: 2.9062 | LR: 7.37e-06 | 700.8ms/step | 28.5 samples/s +Step 40660/44000 | Loss: 2.9063 | LR: 7.32e-06 | 700.8ms/step | 28.5 samples/s +Step 40670/44000 | Loss: 2.8774 | LR: 7.28e-06 | 703.5ms/step | 28.4 samples/s +Step 40680/44000 | Loss: 2.9286 | LR: 7.23e-06 | 699.7ms/step | 28.6 samples/s +Step 40690/44000 | Loss: 2.9412 | LR: 7.19e-06 | 704.5ms/step | 28.4 samples/s +Step 40700/44000 | Loss: 2.9540 | LR: 7.15e-06 | 700.7ms/step | 28.5 samples/s +Step 40710/44000 | Loss: 2.9012 | LR: 7.10e-06 | 702.7ms/step | 28.5 samples/s +Step 40720/44000 | Loss: 2.9229 | LR: 7.06e-06 | 718.5ms/step | 27.8 samples/s +Step 40730/44000 | Loss: 2.8923 | LR: 7.02e-06 | 703.3ms/step | 28.4 samples/s +Step 40740/44000 | Loss: 2.9492 | LR: 6.98e-06 | 702.8ms/step | 28.5 samples/s +Step 40750/44000 | Loss: 2.9560 | LR: 6.93e-06 | 703.9ms/step | 28.4 samples/s +Validation at step 40750: Loss = 2.9483, Perplexity = 19.07 +Step 40760/44000 | Loss: 2.9228 | LR: 6.89e-06 | 1275.0ms/step | 15.7 samples/s +Step 40770/44000 | Loss: 2.9395 | LR: 6.85e-06 | 701.8ms/step | 28.5 samples/s +Step 40780/44000 | Loss: 2.9317 | LR: 6.81e-06 | 699.8ms/step | 28.6 samples/s +Step 40790/44000 | Loss: 2.9281 | LR: 6.77e-06 | 703.3ms/step | 28.4 samples/s +Step 40800/44000 | Loss: 2.9457 | LR: 6.72e-06 | 705.8ms/step | 28.3 samples/s +Step 40810/44000 | Loss: 2.9372 | LR: 6.68e-06 | 700.0ms/step | 28.6 samples/s +Step 40820/44000 | Loss: 2.9493 | LR: 6.64e-06 | 700.3ms/step | 28.6 samples/s +Step 40830/44000 | Loss: 2.8969 | LR: 6.60e-06 | 702.1ms/step | 28.5 samples/s +Step 40840/44000 | Loss: 2.8814 | LR: 6.56e-06 | 702.1ms/step | 28.5 samples/s +Step 40850/44000 | Loss: 2.9286 | LR: 6.52e-06 | 700.0ms/step | 28.6 samples/s +Step 40860/44000 | Loss: 2.9371 | LR: 6.47e-06 | 701.7ms/step | 28.5 samples/s +Step 40870/44000 | Loss: 2.8919 | LR: 6.43e-06 | 699.7ms/step | 28.6 samples/s +Step 40880/44000 | Loss: 2.9258 | LR: 6.39e-06 | 702.5ms/step | 28.5 samples/s +Step 40890/44000 | Loss: 2.9327 | LR: 6.35e-06 | 700.4ms/step | 28.6 samples/s +Step 40900/44000 | Loss: 2.9201 | LR: 6.31e-06 | 704.5ms/step | 28.4 samples/s +Step 40910/44000 | Loss: 2.9177 | LR: 6.27e-06 | 701.3ms/step | 28.5 samples/s +Step 40920/44000 | Loss: 2.9633 | LR: 6.23e-06 | 703.7ms/step | 28.4 samples/s +Step 40930/44000 | Loss: 2.9173 | LR: 6.19e-06 | 700.0ms/step | 28.6 samples/s +Step 40940/44000 | Loss: 2.9235 | LR: 6.15e-06 | 703.7ms/step | 28.4 samples/s +Step 40950/44000 | Loss: 2.9543 | LR: 6.11e-06 | 706.0ms/step | 28.3 samples/s +Step 40960/44000 | Loss: 2.9481 | LR: 6.07e-06 | 702.2ms/step | 28.5 samples/s +Step 40970/44000 | Loss: 2.9347 | LR: 6.03e-06 | 699.4ms/step | 28.6 samples/s +Step 40980/44000 | Loss: 2.9270 | LR: 5.99e-06 | 700.3ms/step | 28.6 samples/s +Step 40990/44000 | Loss: 2.9409 | LR: 5.95e-06 | 701.6ms/step | 28.5 samples/s +Step 41000/44000 | Loss: 2.9100 | LR: 5.91e-06 | 701.4ms/step | 28.5 samples/s +Validation at step 41000: Loss = 2.9487, Perplexity = 19.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-41000 +Step 41010/44000 | Loss: 2.9524 | LR: 5.87e-06 | 1558.7ms/step | 12.8 samples/s +Step 41020/44000 | Loss: 2.9216 | LR: 5.83e-06 | 701.6ms/step | 28.5 samples/s +Step 41030/44000 | Loss: 2.9162 | LR: 5.80e-06 | 698.8ms/step | 28.6 samples/s +Step 41040/44000 | Loss: 2.9067 | LR: 5.76e-06 | 701.9ms/step | 28.5 samples/s +Step 41050/44000 | Loss: 2.9354 | LR: 5.72e-06 | 697.5ms/step | 28.7 samples/s +Step 41060/44000 | Loss: 2.9370 | LR: 5.68e-06 | 702.0ms/step | 28.5 samples/s +Step 41070/44000 | Loss: 2.9287 | LR: 5.64e-06 | 700.6ms/step | 28.5 samples/s +Step 41080/44000 | Loss: 2.9488 | LR: 5.60e-06 | 701.6ms/step | 28.5 samples/s +Step 41090/44000 | Loss: 2.8771 | LR: 5.56e-06 | 701.0ms/step | 28.5 samples/s +Step 41100/44000 | Loss: 2.8919 | LR: 5.53e-06 | 701.6ms/step | 28.5 samples/s +Step 41110/44000 | Loss: 2.9363 | LR: 5.49e-06 | 699.3ms/step | 28.6 samples/s +Step 41120/44000 | Loss: 2.9210 | LR: 5.45e-06 | 699.3ms/step | 28.6 samples/s +Step 41130/44000 | Loss: 2.9203 | LR: 5.41e-06 | 701.3ms/step | 28.5 samples/s +Step 41140/44000 | Loss: 2.9429 | LR: 5.38e-06 | 705.9ms/step | 28.3 samples/s +Step 41150/44000 | Loss: 2.9362 | LR: 5.34e-06 | 700.3ms/step | 28.6 samples/s +Step 41160/44000 | Loss: 2.9012 | LR: 5.30e-06 | 699.8ms/step | 28.6 samples/s +Step 41170/44000 | Loss: 2.9694 | LR: 5.26e-06 | 700.4ms/step | 28.6 samples/s +Step 41180/44000 | Loss: 2.9228 | LR: 5.23e-06 | 700.2ms/step | 28.6 samples/s +Step 41190/44000 | Loss: 2.9611 | LR: 5.19e-06 | 704.5ms/step | 28.4 samples/s +Step 41200/44000 | Loss: 2.9586 | LR: 5.15e-06 | 705.0ms/step | 28.4 samples/s +Step 41210/44000 | Loss: 2.9548 | LR: 5.12e-06 | 698.9ms/step | 28.6 samples/s +Step 41220/44000 | Loss: 2.9169 | LR: 5.08e-06 | 702.5ms/step | 28.5 samples/s +Step 41230/44000 | Loss: 2.9272 | LR: 5.04e-06 | 702.0ms/step | 28.5 samples/s +Step 41240/44000 | Loss: 2.9708 | LR: 5.01e-06 | 700.4ms/step | 28.6 samples/s +Step 41250/44000 | Loss: 2.9140 | LR: 4.97e-06 | 699.3ms/step | 28.6 samples/s +Validation at step 41250: Loss = 2.9470, Perplexity = 19.05 +Step 41260/44000 | Loss: 2.9057 | LR: 4.94e-06 | 1272.2ms/step | 15.7 samples/s +Step 41270/44000 | Loss: 2.9220 | LR: 4.90e-06 | 701.3ms/step | 28.5 samples/s +Step 41280/44000 | Loss: 2.9530 | LR: 4.86e-06 | 699.9ms/step | 28.6 samples/s +Step 41290/44000 | Loss: 2.9409 | LR: 4.83e-06 | 700.6ms/step | 28.5 samples/s +Step 41300/44000 | Loss: 2.9373 | LR: 4.79e-06 | 703.7ms/step | 28.4 samples/s +Step 41310/44000 | Loss: 2.9183 | LR: 4.76e-06 | 699.1ms/step | 28.6 samples/s +Step 41320/44000 | Loss: 2.8779 | LR: 4.72e-06 | 701.0ms/step | 28.5 samples/s +Step 41330/44000 | Loss: 2.9627 | LR: 4.69e-06 | 703.3ms/step | 28.4 samples/s +Step 41340/44000 | Loss: 2.9685 | LR: 4.65e-06 | 699.3ms/step | 28.6 samples/s +Step 41350/44000 | Loss: 2.9265 | LR: 4.62e-06 | 701.1ms/step | 28.5 samples/s +Step 41360/44000 | Loss: 2.8892 | LR: 4.58e-06 | 703.3ms/step | 28.4 samples/s +Step 41370/44000 | Loss: 2.9608 | LR: 4.55e-06 | 699.5ms/step | 28.6 samples/s +Step 41380/44000 | Loss: 2.9636 | LR: 4.51e-06 | 699.9ms/step | 28.6 samples/s +Step 41390/44000 | Loss: 2.9359 | LR: 4.48e-06 | 702.3ms/step | 28.5 samples/s +Step 41400/44000 | Loss: 2.9447 | LR: 4.45e-06 | 700.9ms/step | 28.5 samples/s +Step 41410/44000 | Loss: 2.9294 | LR: 4.41e-06 | 704.2ms/step | 28.4 samples/s +Step 41420/44000 | Loss: 2.9533 | LR: 4.38e-06 | 703.4ms/step | 28.4 samples/s +Step 41430/44000 | Loss: 2.9170 | LR: 4.34e-06 | 703.4ms/step | 28.4 samples/s +Step 41440/44000 | Loss: 2.8982 | LR: 4.31e-06 | 701.8ms/step | 28.5 samples/s +Step 41450/44000 | Loss: 2.9533 | LR: 4.28e-06 | 701.0ms/step | 28.5 samples/s +Step 41460/44000 | Loss: 2.9310 | LR: 4.24e-06 | 703.8ms/step | 28.4 samples/s +Step 41470/44000 | Loss: 2.9385 | LR: 4.21e-06 | 714.7ms/step | 28.0 samples/s +Step 41480/44000 | Loss: 2.9355 | LR: 4.18e-06 | 715.8ms/step | 27.9 samples/s +Step 41490/44000 | Loss: 2.9187 | LR: 4.14e-06 | 717.4ms/step | 27.9 samples/s +Step 41500/44000 | Loss: 2.9553 | LR: 4.11e-06 | 720.2ms/step | 27.8 samples/s +Validation at step 41500: Loss = 2.9484, Perplexity = 19.08 +Step 41510/44000 | Loss: 2.9306 | LR: 4.08e-06 | 1310.8ms/step | 15.3 samples/s +Step 41520/44000 | Loss: 2.8952 | LR: 4.05e-06 | 714.6ms/step | 28.0 samples/s +Step 41530/44000 | Loss: 2.9135 | LR: 4.01e-06 | 701.9ms/step | 28.5 samples/s +Step 41540/44000 | Loss: 2.9791 | LR: 3.98e-06 | 702.2ms/step | 28.5 samples/s +Step 41550/44000 | Loss: 2.9299 | LR: 3.95e-06 | 702.5ms/step | 28.5 samples/s +Step 41560/44000 | Loss: 2.9234 | LR: 3.92e-06 | 699.5ms/step | 28.6 samples/s +Step 41570/44000 | Loss: 2.9008 | LR: 3.88e-06 | 703.8ms/step | 28.4 samples/s +Step 41580/44000 | Loss: 2.9590 | LR: 3.85e-06 | 702.2ms/step | 28.5 samples/s +Step 41590/44000 | Loss: 2.9340 | LR: 3.82e-06 | 699.9ms/step | 28.6 samples/s +Step 41600/44000 | Loss: 2.9495 | LR: 3.79e-06 | 703.7ms/step | 28.4 samples/s +Step 41610/44000 | Loss: 2.9345 | LR: 3.76e-06 | 702.1ms/step | 28.5 samples/s +Step 41620/44000 | Loss: 2.9600 | LR: 3.73e-06 | 699.8ms/step | 28.6 samples/s +Step 41630/44000 | Loss: 2.9478 | LR: 3.70e-06 | 703.1ms/step | 28.4 samples/s +Step 41640/44000 | Loss: 2.9304 | LR: 3.66e-06 | 701.4ms/step | 28.5 samples/s +Step 41650/44000 | Loss: 2.9038 | LR: 3.63e-06 | 699.5ms/step | 28.6 samples/s +Step 41660/44000 | Loss: 2.9227 | LR: 3.60e-06 | 704.9ms/step | 28.4 samples/s +Step 41670/44000 | Loss: 2.9285 | LR: 3.57e-06 | 703.9ms/step | 28.4 samples/s +Step 41680/44000 | Loss: 2.9302 | LR: 3.54e-06 | 699.1ms/step | 28.6 samples/s +Step 41690/44000 | Loss: 2.9719 | LR: 3.51e-06 | 706.8ms/step | 28.3 samples/s +Step 41700/44000 | Loss: 2.9358 | LR: 3.48e-06 | 701.3ms/step | 28.5 samples/s +Step 41710/44000 | Loss: 2.9280 | LR: 3.45e-06 | 702.7ms/step | 28.5 samples/s +Step 41720/44000 | Loss: 2.9346 | LR: 3.42e-06 | 702.3ms/step | 28.5 samples/s +Step 41730/44000 | Loss: 2.9347 | LR: 3.39e-06 | 705.2ms/step | 28.4 samples/s +Step 41740/44000 | Loss: 2.9640 | LR: 3.36e-06 | 700.6ms/step | 28.5 samples/s +Step 41750/44000 | Loss: 2.9037 | LR: 3.33e-06 | 701.6ms/step | 28.5 samples/s +Validation at step 41750: Loss = 2.9465, Perplexity = 19.04 +Step 41760/44000 | Loss: 2.9319 | LR: 3.30e-06 | 1275.0ms/step | 15.7 samples/s +Step 41770/44000 | Loss: 2.9256 | LR: 3.27e-06 | 703.5ms/step | 28.4 samples/s +Step 41780/44000 | Loss: 2.9078 | LR: 3.24e-06 | 699.0ms/step | 28.6 samples/s +Step 41790/44000 | Loss: 2.9632 | LR: 3.21e-06 | 703.4ms/step | 28.4 samples/s +Step 41800/44000 | Loss: 2.8909 | LR: 3.19e-06 | 701.1ms/step | 28.5 samples/s +Step 41810/44000 | Loss: 2.9328 | LR: 3.16e-06 | 702.8ms/step | 28.5 samples/s +Step 41820/44000 | Loss: 2.8924 | LR: 3.13e-06 | 701.8ms/step | 28.5 samples/s +Step 41830/44000 | Loss: 2.8826 | LR: 3.10e-06 | 705.2ms/step | 28.4 samples/s +Step 41840/44000 | Loss: 2.9699 | LR: 3.07e-06 | 703.2ms/step | 28.4 samples/s +Step 41850/44000 | Loss: 2.9111 | LR: 3.04e-06 | 703.2ms/step | 28.4 samples/s +Step 41860/44000 | Loss: 2.9349 | LR: 3.01e-06 | 700.5ms/step | 28.6 samples/s +Step 41870/44000 | Loss: 2.9112 | LR: 2.99e-06 | 703.4ms/step | 28.4 samples/s +Step 41880/44000 | Loss: 2.9725 | LR: 2.96e-06 | 700.0ms/step | 28.6 samples/s +Step 41890/44000 | Loss: 2.9557 | LR: 2.93e-06 | 700.1ms/step | 28.6 samples/s +Step 41900/44000 | Loss: 2.9136 | LR: 2.90e-06 | 700.3ms/step | 28.6 samples/s +Step 41910/44000 | Loss: 2.9287 | LR: 2.88e-06 | 698.5ms/step | 28.6 samples/s +Step 41920/44000 | Loss: 2.8813 | LR: 2.85e-06 | 701.2ms/step | 28.5 samples/s +Step 41930/44000 | Loss: 2.9298 | LR: 2.82e-06 | 700.0ms/step | 28.6 samples/s +Step 41940/44000 | Loss: 2.8909 | LR: 2.79e-06 | 699.7ms/step | 28.6 samples/s +Step 41950/44000 | Loss: 2.9280 | LR: 2.77e-06 | 700.6ms/step | 28.5 samples/s +Step 41960/44000 | Loss: 2.9180 | LR: 2.74e-06 | 703.2ms/step | 28.4 samples/s +Step 41970/44000 | Loss: 2.9419 | LR: 2.71e-06 | 703.4ms/step | 28.4 samples/s +Step 41980/44000 | Loss: 2.9344 | LR: 2.69e-06 | 703.3ms/step | 28.4 samples/s +Step 41990/44000 | Loss: 2.9228 | LR: 2.66e-06 | 702.1ms/step | 28.5 samples/s +Step 42000/44000 | Loss: 2.8882 | LR: 2.63e-06 | 701.4ms/step | 28.5 samples/s +Validation at step 42000: Loss = 2.9463, Perplexity = 19.04 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-42000 +Step 42010/44000 | Loss: 2.9343 | LR: 2.61e-06 | 1555.2ms/step | 12.9 samples/s +Step 42020/44000 | Loss: 2.9206 | LR: 2.58e-06 | 703.0ms/step | 28.4 samples/s +Step 42030/44000 | Loss: 2.9239 | LR: 2.56e-06 | 701.0ms/step | 28.5 samples/s +Step 42040/44000 | Loss: 2.9225 | LR: 2.53e-06 | 698.9ms/step | 28.6 samples/s +Step 42050/44000 | Loss: 2.9334 | LR: 2.50e-06 | 698.0ms/step | 28.7 samples/s +Step 42060/44000 | Loss: 2.9578 | LR: 2.48e-06 | 699.7ms/step | 28.6 samples/s +Step 42070/44000 | Loss: 2.9394 | LR: 2.45e-06 | 700.2ms/step | 28.6 samples/s +Step 42080/44000 | Loss: 2.9185 | LR: 2.43e-06 | 700.3ms/step | 28.6 samples/s +Step 42090/44000 | Loss: 2.9888 | LR: 2.40e-06 | 704.5ms/step | 28.4 samples/s +Step 42100/44000 | Loss: 2.9032 | LR: 2.38e-06 | 700.5ms/step | 28.6 samples/s +Step 42110/44000 | Loss: 2.9233 | LR: 2.35e-06 | 701.2ms/step | 28.5 samples/s +Step 42120/44000 | Loss: 2.9724 | LR: 2.33e-06 | 701.9ms/step | 28.5 samples/s +Step 42130/44000 | Loss: 2.9300 | LR: 2.30e-06 | 701.9ms/step | 28.5 samples/s +Step 42140/44000 | Loss: 2.9275 | LR: 2.28e-06 | 702.4ms/step | 28.5 samples/s +Step 42150/44000 | Loss: 2.9381 | LR: 2.25e-06 | 699.9ms/step | 28.6 samples/s +Step 42160/44000 | Loss: 2.9283 | LR: 2.23e-06 | 700.1ms/step | 28.6 samples/s +Step 42170/44000 | Loss: 2.9523 | LR: 2.21e-06 | 700.2ms/step | 28.6 samples/s +Step 42180/44000 | Loss: 2.9660 | LR: 2.18e-06 | 701.4ms/step | 28.5 samples/s +Step 42190/44000 | Loss: 2.9310 | LR: 2.16e-06 | 703.9ms/step | 28.4 samples/s +Step 42200/44000 | Loss: 2.9124 | LR: 2.13e-06 | 703.2ms/step | 28.4 samples/s +Step 42210/44000 | Loss: 2.9413 | LR: 2.11e-06 | 702.3ms/step | 28.5 samples/s +Step 42220/44000 | Loss: 2.9466 | LR: 2.09e-06 | 700.2ms/step | 28.6 samples/s +Step 42230/44000 | Loss: 2.9431 | LR: 2.06e-06 | 704.8ms/step | 28.4 samples/s +Step 42240/44000 | Loss: 2.9029 | LR: 2.04e-06 | 700.7ms/step | 28.5 samples/s +Step 42250/44000 | Loss: 2.9473 | LR: 2.02e-06 | 701.8ms/step | 28.5 samples/s +Validation at step 42250: Loss = 2.9482, Perplexity = 19.07 +Step 42260/44000 | Loss: 3.0104 | LR: 1.99e-06 | 1270.3ms/step | 15.7 samples/s +Step 42270/44000 | Loss: 2.9326 | LR: 1.97e-06 | 699.4ms/step | 28.6 samples/s +Step 42280/44000 | Loss: 2.9335 | LR: 1.95e-06 | 702.4ms/step | 28.5 samples/s +Step 42290/44000 | Loss: 2.9413 | LR: 1.93e-06 | 698.4ms/step | 28.6 samples/s +Step 42300/44000 | Loss: 2.9490 | LR: 1.90e-06 | 704.9ms/step | 28.4 samples/s +Step 42310/44000 | Loss: 2.9419 | LR: 1.88e-06 | 698.1ms/step | 28.7 samples/s +Step 42320/44000 | Loss: 2.9263 | LR: 1.86e-06 | 702.7ms/step | 28.5 samples/s +Step 42330/44000 | Loss: 2.9134 | LR: 1.84e-06 | 700.7ms/step | 28.5 samples/s +Step 42340/44000 | Loss: 2.9451 | LR: 1.82e-06 | 702.1ms/step | 28.5 samples/s +Step 42350/44000 | Loss: 2.9505 | LR: 1.79e-06 | 702.0ms/step | 28.5 samples/s +Step 42360/44000 | Loss: 2.9208 | LR: 1.77e-06 | 700.5ms/step | 28.6 samples/s +Step 42370/44000 | Loss: 2.9327 | LR: 1.75e-06 | 702.3ms/step | 28.5 samples/s +Step 42380/44000 | Loss: 2.9630 | LR: 1.73e-06 | 703.3ms/step | 28.4 samples/s +Step 42390/44000 | Loss: 2.9562 | LR: 1.71e-06 | 700.4ms/step | 28.6 samples/s +Step 42400/44000 | Loss: 2.9220 | LR: 1.69e-06 | 701.9ms/step | 28.5 samples/s +Step 42410/44000 | Loss: 2.9528 | LR: 1.67e-06 | 700.4ms/step | 28.6 samples/s +Step 42420/44000 | Loss: 2.9412 | LR: 1.64e-06 | 703.7ms/step | 28.4 samples/s +Step 42430/44000 | Loss: 2.8973 | LR: 1.62e-06 | 702.5ms/step | 28.5 samples/s +Step 42440/44000 | Loss: 2.9621 | LR: 1.60e-06 | 699.3ms/step | 28.6 samples/s +Step 42450/44000 | Loss: 2.9480 | LR: 1.58e-06 | 702.2ms/step | 28.5 samples/s +Step 42460/44000 | Loss: 2.9009 | LR: 1.56e-06 | 703.5ms/step | 28.4 samples/s +Step 42470/44000 | Loss: 2.8932 | LR: 1.54e-06 | 703.3ms/step | 28.4 samples/s +Step 42480/44000 | Loss: 2.9210 | LR: 1.52e-06 | 704.0ms/step | 28.4 samples/s +Step 42490/44000 | Loss: 2.9523 | LR: 1.50e-06 | 699.1ms/step | 28.6 samples/s +Step 42500/44000 | Loss: 2.9060 | LR: 1.48e-06 | 701.8ms/step | 28.5 samples/s +Validation at step 42500: Loss = 2.9463, Perplexity = 19.03 +Step 42510/44000 | Loss: 2.9492 | LR: 1.46e-06 | 1276.2ms/step | 15.7 samples/s +Step 42520/44000 | Loss: 2.9761 | LR: 1.44e-06 | 700.5ms/step | 28.6 samples/s +Step 42530/44000 | Loss: 2.9066 | LR: 1.42e-06 | 699.8ms/step | 28.6 samples/s +Step 42540/44000 | Loss: 2.9027 | LR: 1.40e-06 | 702.8ms/step | 28.5 samples/s +Step 42550/44000 | Loss: 2.9166 | LR: 1.39e-06 | 701.2ms/step | 28.5 samples/s +Step 42560/44000 | Loss: 2.9459 | LR: 1.37e-06 | 698.9ms/step | 28.6 samples/s +Step 42570/44000 | Loss: 2.9118 | LR: 1.35e-06 | 702.8ms/step | 28.5 samples/s +Step 42580/44000 | Loss: 2.9367 | LR: 1.33e-06 | 703.8ms/step | 28.4 samples/s +Step 42590/44000 | Loss: 2.9467 | LR: 1.31e-06 | 706.4ms/step | 28.3 samples/s +Step 42600/44000 | Loss: 2.9457 | LR: 1.29e-06 | 697.8ms/step | 28.7 samples/s +Step 42610/44000 | Loss: 2.9319 | LR: 1.27e-06 | 702.0ms/step | 28.5 samples/s +Step 42620/44000 | Loss: 2.9708 | LR: 1.25e-06 | 699.9ms/step | 28.6 samples/s +Step 42630/44000 | Loss: 2.9626 | LR: 1.24e-06 | 702.5ms/step | 28.5 samples/s +Step 42640/44000 | Loss: 2.9129 | LR: 1.22e-06 | 699.8ms/step | 28.6 samples/s +Step 42650/44000 | Loss: 2.9069 | LR: 1.20e-06 | 701.5ms/step | 28.5 samples/s +Step 42660/44000 | Loss: 2.9379 | LR: 1.18e-06 | 700.0ms/step | 28.6 samples/s +Step 42670/44000 | Loss: 2.9198 | LR: 1.17e-06 | 701.7ms/step | 28.5 samples/s +Step 42680/44000 | Loss: 2.9532 | LR: 1.15e-06 | 700.6ms/step | 28.5 samples/s +Step 42690/44000 | Loss: 2.9350 | LR: 1.13e-06 | 702.9ms/step | 28.5 samples/s +Step 42700/44000 | Loss: 2.9154 | LR: 1.11e-06 | 702.4ms/step | 28.5 samples/s +Step 42710/44000 | Loss: 2.9108 | LR: 1.10e-06 | 700.8ms/step | 28.5 samples/s +Step 42720/44000 | Loss: 2.9438 | LR: 1.08e-06 | 702.3ms/step | 28.5 samples/s +Step 42730/44000 | Loss: 2.9774 | LR: 1.06e-06 | 699.8ms/step | 28.6 samples/s +Step 42740/44000 | Loss: 2.9560 | LR: 1.05e-06 | 704.1ms/step | 28.4 samples/s +Step 42750/44000 | Loss: 2.9303 | LR: 1.03e-06 | 699.6ms/step | 28.6 samples/s +Validation at step 42750: Loss = 2.9481, Perplexity = 19.07 +Step 42760/44000 | Loss: 2.9524 | LR: 1.01e-06 | 1272.7ms/step | 15.7 samples/s +Step 42770/44000 | Loss: 2.9467 | LR: 9.97e-07 | 699.8ms/step | 28.6 samples/s +Step 42780/44000 | Loss: 2.8811 | LR: 9.81e-07 | 704.0ms/step | 28.4 samples/s +Step 42790/44000 | Loss: 2.9610 | LR: 9.65e-07 | 702.4ms/step | 28.5 samples/s +Step 42800/44000 | Loss: 2.9607 | LR: 9.49e-07 | 701.3ms/step | 28.5 samples/s +Step 42810/44000 | Loss: 2.9159 | LR: 9.33e-07 | 703.8ms/step | 28.4 samples/s +Step 42820/44000 | Loss: 2.9325 | LR: 9.18e-07 | 704.4ms/step | 28.4 samples/s +Step 42830/44000 | Loss: 2.9174 | LR: 9.02e-07 | 702.2ms/step | 28.5 samples/s +Step 42840/44000 | Loss: 2.9068 | LR: 8.87e-07 | 703.3ms/step | 28.4 samples/s +Step 42850/44000 | Loss: 2.9348 | LR: 8.72e-07 | 700.4ms/step | 28.6 samples/s +Step 42860/44000 | Loss: 2.8940 | LR: 8.57e-07 | 704.0ms/step | 28.4 samples/s +Step 42870/44000 | Loss: 2.9359 | LR: 8.42e-07 | 701.7ms/step | 28.5 samples/s +Step 42880/44000 | Loss: 2.9218 | LR: 8.27e-07 | 699.9ms/step | 28.6 samples/s +Step 42890/44000 | Loss: 2.9492 | LR: 8.12e-07 | 713.9ms/step | 28.0 samples/s +Step 42900/44000 | Loss: 2.9380 | LR: 7.98e-07 | 701.9ms/step | 28.5 samples/s +Step 42910/44000 | Loss: 2.9514 | LR: 7.83e-07 | 702.6ms/step | 28.5 samples/s +Step 42920/44000 | Loss: 2.9553 | LR: 7.69e-07 | 703.1ms/step | 28.4 samples/s +Step 42930/44000 | Loss: 2.9452 | LR: 7.55e-07 | 702.2ms/step | 28.5 samples/s +Step 42940/44000 | Loss: 2.9625 | LR: 7.41e-07 | 700.5ms/step | 28.6 samples/s +Step 42950/44000 | Loss: 2.9390 | LR: 7.27e-07 | 700.1ms/step | 28.6 samples/s +Step 42960/44000 | Loss: 2.8943 | LR: 7.13e-07 | 700.6ms/step | 28.5 samples/s +Step 42970/44000 | Loss: 2.9190 | LR: 6.99e-07 | 701.5ms/step | 28.5 samples/s +Step 42980/44000 | Loss: 2.9271 | LR: 6.86e-07 | 700.2ms/step | 28.6 samples/s +Step 42990/44000 | Loss: 2.9026 | LR: 6.72e-07 | 701.2ms/step | 28.5 samples/s +Step 43000/44000 | Loss: 2.9144 | LR: 6.59e-07 | 705.4ms/step | 28.4 samples/s +Validation at step 43000: Loss = 2.9493, Perplexity = 19.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-43000 +Step 43010/44000 | Loss: 2.9522 | LR: 6.46e-07 | 1574.7ms/step | 12.7 samples/s +Step 43020/44000 | Loss: 2.9235 | LR: 6.33e-07 | 701.2ms/step | 28.5 samples/s +Step 43030/44000 | Loss: 2.9609 | LR: 6.20e-07 | 703.7ms/step | 28.4 samples/s +Step 43040/44000 | Loss: 2.9112 | LR: 6.08e-07 | 700.8ms/step | 28.5 samples/s +Step 43050/44000 | Loss: 2.9516 | LR: 5.95e-07 | 698.9ms/step | 28.6 samples/s +Step 43060/44000 | Loss: 2.9426 | LR: 5.83e-07 | 701.4ms/step | 28.5 samples/s +Step 43070/44000 | Loss: 2.8865 | LR: 5.70e-07 | 703.1ms/step | 28.4 samples/s +Step 43080/44000 | Loss: 2.9586 | LR: 5.58e-07 | 698.8ms/step | 28.6 samples/s +Step 43090/44000 | Loss: 2.9480 | LR: 5.46e-07 | 701.4ms/step | 28.5 samples/s +Step 43100/44000 | Loss: 2.9081 | LR: 5.34e-07 | 704.7ms/step | 28.4 samples/s +Step 43110/44000 | Loss: 2.9564 | LR: 5.22e-07 | 702.4ms/step | 28.5 samples/s +Step 43120/44000 | Loss: 2.9172 | LR: 5.11e-07 | 702.4ms/step | 28.5 samples/s +Step 43130/44000 | Loss: 2.9691 | LR: 4.99e-07 | 701.9ms/step | 28.5 samples/s +Step 43140/44000 | Loss: 2.9724 | LR: 4.88e-07 | 702.9ms/step | 28.5 samples/s +Step 43150/44000 | Loss: 2.9536 | LR: 4.76e-07 | 701.7ms/step | 28.5 samples/s +Step 43160/44000 | Loss: 2.9053 | LR: 4.65e-07 | 707.8ms/step | 28.3 samples/s +Step 43170/44000 | Loss: 2.9341 | LR: 4.54e-07 | 699.6ms/step | 28.6 samples/s +Step 43180/44000 | Loss: 2.9095 | LR: 4.43e-07 | 699.8ms/step | 28.6 samples/s +Step 43190/44000 | Loss: 2.9575 | LR: 4.33e-07 | 701.1ms/step | 28.5 samples/s +Step 43200/44000 | Loss: 2.9366 | LR: 4.22e-07 | 702.8ms/step | 28.5 samples/s +Step 43210/44000 | Loss: 2.9436 | LR: 4.12e-07 | 700.0ms/step | 28.6 samples/s +Step 43220/44000 | Loss: 2.9374 | LR: 4.01e-07 | 699.9ms/step | 28.6 samples/s +Step 43230/44000 | Loss: 2.9615 | LR: 3.91e-07 | 701.8ms/step | 28.5 samples/s +Step 43240/44000 | Loss: 2.9021 | LR: 3.81e-07 | 698.1ms/step | 28.6 samples/s +Step 43250/44000 | Loss: 2.9461 | LR: 3.71e-07 | 698.3ms/step | 28.6 samples/s +Validation at step 43250: Loss = 2.9475, Perplexity = 19.06 +Step 43260/44000 | Loss: 2.9196 | LR: 3.61e-07 | 1271.5ms/step | 15.7 samples/s +Step 43270/44000 | Loss: 2.9472 | LR: 3.51e-07 | 703.9ms/step | 28.4 samples/s +Step 43280/44000 | Loss: 2.8993 | LR: 3.42e-07 | 703.9ms/step | 28.4 samples/s +Step 43290/44000 | Loss: 2.9064 | LR: 3.32e-07 | 701.0ms/step | 28.5 samples/s +Step 43300/44000 | Loss: 2.9384 | LR: 3.23e-07 | 705.4ms/step | 28.4 samples/s +Step 43310/44000 | Loss: 2.9093 | LR: 3.14e-07 | 700.6ms/step | 28.5 samples/s +Step 43320/44000 | Loss: 2.9384 | LR: 3.05e-07 | 703.4ms/step | 28.4 samples/s +Step 43330/44000 | Loss: 2.9564 | LR: 2.96e-07 | 703.5ms/step | 28.4 samples/s +Step 43340/44000 | Loss: 2.9286 | LR: 2.87e-07 | 700.1ms/step | 28.6 samples/s +Step 43350/44000 | Loss: 2.9178 | LR: 2.79e-07 | 704.0ms/step | 28.4 samples/s +Step 43360/44000 | Loss: 2.9321 | LR: 2.70e-07 | 700.7ms/step | 28.5 samples/s +Step 43370/44000 | Loss: 2.9607 | LR: 2.62e-07 | 703.4ms/step | 28.4 samples/s +Step 43380/44000 | Loss: 2.9251 | LR: 2.53e-07 | 702.6ms/step | 28.5 samples/s +Step 43390/44000 | Loss: 2.8868 | LR: 2.45e-07 | 704.1ms/step | 28.4 samples/s +Step 43400/44000 | Loss: 2.9451 | LR: 2.37e-07 | 704.6ms/step | 28.4 samples/s +Step 43410/44000 | Loss: 2.9676 | LR: 2.30e-07 | 704.0ms/step | 28.4 samples/s +Step 43420/44000 | Loss: 2.9433 | LR: 2.22e-07 | 701.2ms/step | 28.5 samples/s +Step 43430/44000 | Loss: 2.9482 | LR: 2.14e-07 | 703.5ms/step | 28.4 samples/s +Step 43440/44000 | Loss: 2.9610 | LR: 2.07e-07 | 704.9ms/step | 28.4 samples/s +Step 43450/44000 | Loss: 2.9530 | LR: 1.99e-07 | 699.6ms/step | 28.6 samples/s +Step 43460/44000 | Loss: 2.9614 | LR: 1.92e-07 | 698.1ms/step | 28.6 samples/s +Step 43470/44000 | Loss: 2.9181 | LR: 1.85e-07 | 702.2ms/step | 28.5 samples/s +Step 43480/44000 | Loss: 2.8899 | LR: 1.78e-07 | 700.9ms/step | 28.5 samples/s +Step 43490/44000 | Loss: 2.9513 | LR: 1.72e-07 | 702.3ms/step | 28.5 samples/s +Step 43500/44000 | Loss: 2.9534 | LR: 1.65e-07 | 703.0ms/step | 28.5 samples/s +Validation at step 43500: Loss = 2.9480, Perplexity = 19.07 +Step 43510/44000 | Loss: 2.9095 | LR: 1.58e-07 | 1271.5ms/step | 15.7 samples/s +Step 43520/44000 | Loss: 2.9056 | LR: 1.52e-07 | 700.4ms/step | 28.6 samples/s +Step 43530/44000 | Loss: 2.9236 | LR: 1.46e-07 | 702.3ms/step | 28.5 samples/s +Step 43540/44000 | Loss: 2.9567 | LR: 1.40e-07 | 701.5ms/step | 28.5 samples/s +Step 43550/44000 | Loss: 2.9211 | LR: 1.34e-07 | 700.2ms/step | 28.6 samples/s +Step 43560/44000 | Loss: 2.9144 | LR: 1.28e-07 | 702.3ms/step | 28.5 samples/s +Step 43570/44000 | Loss: 2.9051 | LR: 1.22e-07 | 700.9ms/step | 28.5 samples/s +Step 43580/44000 | Loss: 2.9052 | LR: 1.16e-07 | 704.3ms/step | 28.4 samples/s +Step 43590/44000 | Loss: 2.9294 | LR: 1.11e-07 | 702.3ms/step | 28.5 samples/s +Step 43600/44000 | Loss: 2.9125 | LR: 1.06e-07 | 703.8ms/step | 28.4 samples/s +Step 43610/44000 | Loss: 2.9485 | LR: 1.00e-07 | 702.4ms/step | 28.5 samples/s +Step 43620/44000 | Loss: 2.9456 | LR: 9.52e-08 | 699.2ms/step | 28.6 samples/s +Step 43630/44000 | Loss: 2.9108 | LR: 9.03e-08 | 701.3ms/step | 28.5 samples/s +Step 43640/44000 | Loss: 2.9402 | LR: 8.55e-08 | 701.5ms/step | 28.5 samples/s +Step 43650/44000 | Loss: 2.9512 | LR: 8.08e-08 | 702.9ms/step | 28.5 samples/s +Step 43660/44000 | Loss: 2.9632 | LR: 7.62e-08 | 702.5ms/step | 28.5 samples/s +Step 43670/44000 | Loss: 2.8882 | LR: 7.18e-08 | 702.3ms/step | 28.5 samples/s +Step 43680/44000 | Loss: 2.9318 | LR: 6.75e-08 | 701.5ms/step | 28.5 samples/s +Step 43690/44000 | Loss: 2.9417 | LR: 6.34e-08 | 702.9ms/step | 28.5 samples/s +Step 43700/44000 | Loss: 2.8944 | LR: 5.94e-08 | 704.4ms/step | 28.4 samples/s +Step 43710/44000 | Loss: 2.9467 | LR: 5.55e-08 | 704.2ms/step | 28.4 samples/s +Step 43720/44000 | Loss: 2.9161 | LR: 5.17e-08 | 701.6ms/step | 28.5 samples/s +Step 43730/44000 | Loss: 2.9481 | LR: 4.81e-08 | 699.2ms/step | 28.6 samples/s +Step 43740/44000 | Loss: 2.8944 | LR: 4.46e-08 | 701.5ms/step | 28.5 samples/s +Step 43750/44000 | Loss: 2.9366 | LR: 4.12e-08 | 700.3ms/step | 28.6 samples/s +Validation at step 43750: Loss = 2.9475, Perplexity = 19.06 +Step 43760/44000 | Loss: 2.9884 | LR: 3.80e-08 | 1270.6ms/step | 15.7 samples/s +Step 43770/44000 | Loss: 2.8954 | LR: 3.49e-08 | 701.7ms/step | 28.5 samples/s +Step 43780/44000 | Loss: 2.9045 | LR: 3.19e-08 | 700.2ms/step | 28.6 samples/s +Step 43790/44000 | Loss: 2.9786 | LR: 2.91e-08 | 700.3ms/step | 28.6 samples/s +Step 43800/44000 | Loss: 2.9187 | LR: 2.64e-08 | 703.6ms/step | 28.4 samples/s +Step 43810/44000 | Loss: 2.9455 | LR: 2.38e-08 | 702.1ms/step | 28.5 samples/s +Step 43820/44000 | Loss: 2.9517 | LR: 2.14e-08 | 704.1ms/step | 28.4 samples/s +Step 43830/44000 | Loss: 2.9746 | LR: 1.91e-08 | 699.7ms/step | 28.6 samples/s +Step 43840/44000 | Loss: 2.9238 | LR: 1.69e-08 | 702.5ms/step | 28.5 samples/s +Step 43850/44000 | Loss: 2.9086 | LR: 1.48e-08 | 699.2ms/step | 28.6 samples/s +Step 43860/44000 | Loss: 2.9335 | LR: 1.29e-08 | 702.0ms/step | 28.5 samples/s +Step 43870/44000 | Loss: 2.9502 | LR: 1.11e-08 | 700.7ms/step | 28.5 samples/s +Step 43880/44000 | Loss: 2.9727 | LR: 9.50e-09 | 699.3ms/step | 28.6 samples/s +Step 43890/44000 | Loss: 2.9088 | LR: 7.98e-09 | 700.9ms/step | 28.5 samples/s +Step 43900/44000 | Loss: 2.9239 | LR: 6.60e-09 | 698.7ms/step | 28.6 samples/s +Step 43910/44000 | Loss: 2.9143 | LR: 5.34e-09 | 701.3ms/step | 28.5 samples/s +Step 43920/44000 | Loss: 2.9564 | LR: 4.22e-09 | 700.0ms/step | 28.6 samples/s +Step 43930/44000 | Loss: 2.9285 | LR: 3.23e-09 | 703.8ms/step | 28.4 samples/s +Step 43940/44000 | Loss: 2.9517 | LR: 2.37e-09 | 701.3ms/step | 28.5 samples/s +Step 43950/44000 | Loss: 2.9154 | LR: 1.65e-09 | 702.0ms/step | 28.5 samples/s +Step 43960/44000 | Loss: 2.9595 | LR: 1.06e-09 | 700.2ms/step | 28.6 samples/s +Step 43970/44000 | Loss: 2.9390 | LR: 5.94e-10 | 699.6ms/step | 28.6 samples/s +Step 43980/44000 | Loss: 2.9157 | LR: 2.64e-10 | 698.0ms/step | 28.7 samples/s +Step 43990/44000 | Loss: 2.8652 | LR: 6.60e-11 | 697.7ms/step | 28.7 samples/s +Step 44000/44000 | Loss: 2.9857 | LR: 0.00e+00 | 702.8ms/step | 28.5 samples/s +Validation at step 44000: Loss = 2.9477, Perplexity = 19.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json +wandb: uploading output.log +wandb: uploading history steps 4399-4399, summary, console lines 1912-1914 +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▇▇▆▅▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▂▁▂▁▁▁▁▁▁▂▁▁▁▁▁▁▁ +wandb: eval/perplexity ██▇▇▇▆▅▅▅▅▄▄▄▃▃▃▂▃▂▂▂▂▂▂▂▁▁▁▁▂▁▁▁▁▂▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▃▃▃▃▂▂█▁▂▂▁▂▁▂▂▂▂▂▂▂▁▂▁▂▂▁▁▂▃▃▂▂▃▂▃▁▂▂▂▂ +wandb: speed/samples_per_sec █████▁█████████████████▂████████████████ +wandb: speed/steps_per_sec ██████████████▁█▂████▂█████████████████▂ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.94767 +wandb: eval/perplexity 19.06154 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 702.82359 +wandb: speed/samples_per_sec 28.45664 +wandb: speed/steps_per_sec 1.42283 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m mixed 8k seed47 ddp2 h100-resume at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/93ukqj9s +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20260820_235426-93ukqj9s/logs +Training complete! diff --git a/epfl_final/train_logs/seed47_pair_ddp2/dense.log b/epfl_final/train_logs/seed47_pair_ddp2/dense.log new file mode 100644 index 0000000000000000000000000000000000000000..7a5ba413bb7f0343566221a5214a56e5d39c9640 --- /dev/null +++ b/epfl_final/train_logs/seed47_pair_ddp2/dense.log @@ -0,0 +1,3687 @@ +W0820 03:12:36.635000 38825 site-packages/torch/distributed/run.py:851] +W0820 03:12:36.635000 38825 site-packages/torch/distributed/run.py:851] ***************************************** +W0820 03:12:36.635000 38825 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0820 03:12:36.635000 38825 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 47, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m dense 8k seed47 ddp2", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "dense", + "multiseed", + "seed47", + "ddp2" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)...Checkpoint activations: False + +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 89kzpaot +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_031315-89kzpaot +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m dense 8k seed47 ddp2 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/89kzpaot +Starting training... +Switching attention mode for step 1: dense (dense step interval) +Step 10/44000 | Loss: 12.1136 | LR: 6.67e-06 | 10502.1ms/step | 1.9 samples/s +Step 20/44000 | Loss: 12.0222 | LR: 1.33e-05 | 1880.0ms/step | 10.6 samples/s +Step 30/44000 | Loss: 11.7165 | LR: 2.00e-05 | 1893.7ms/step | 10.6 samples/s +Step 40/44000 | Loss: 11.1559 | LR: 2.67e-05 | 1883.6ms/step | 10.6 samples/s +Step 50/44000 | Loss: 10.8121 | LR: 3.33e-05 | 1886.0ms/step | 10.6 samples/s +Step 60/44000 | Loss: 10.5712 | LR: 4.00e-05 | 1885.0ms/step | 10.6 samples/s +Step 70/44000 | Loss: 10.3259 | LR: 4.67e-05 | 1885.6ms/step | 10.6 samples/s +Step 80/44000 | Loss: 10.0207 | LR: 5.33e-05 | 1885.2ms/step | 10.6 samples/s +Step 90/44000 | Loss: 9.6658 | LR: 6.00e-05 | 1888.1ms/step | 10.6 samples/s +Step 100/44000 | Loss: 9.1631 | LR: 6.67e-05 | 1889.5ms/step | 10.6 samples/s +Step 110/44000 | Loss: 8.6647 | LR: 7.33e-05 | 1889.6ms/step | 10.6 samples/s +Step 120/44000 | Loss: 8.1658 | LR: 8.00e-05 | 1890.4ms/step | 10.6 samples/s +Step 130/44000 | Loss: 7.7892 | LR: 8.67e-05 | 1891.2ms/step | 10.6 samples/s +Step 140/44000 | Loss: 7.5348 | LR: 9.33e-05 | 1891.4ms/step | 10.6 samples/s +Step 150/44000 | Loss: 7.3523 | LR: 1.00e-04 | 1896.5ms/step | 10.5 samples/s +Step 160/44000 | Loss: 7.2347 | LR: 1.07e-04 | 1897.0ms/step | 10.5 samples/s +Step 170/44000 | Loss: 7.0993 | LR: 1.13e-04 | 1898.0ms/step | 10.5 samples/s +Step 180/44000 | Loss: 7.0152 | LR: 1.20e-04 | 1897.4ms/step | 10.5 samples/s +Step 190/44000 | Loss: 6.8841 | LR: 1.27e-04 | 1896.5ms/step | 10.5 samples/s +Step 200/44000 | Loss: 6.8038 | LR: 1.33e-04 | 1896.6ms/step | 10.5 samples/s +Step 210/44000 | Loss: 6.7463 | LR: 1.40e-04 | 1897.3ms/step | 10.5 samples/s +Step 220/44000 | Loss: 6.6690 | LR: 1.47e-04 | 1897.8ms/step | 10.5 samples/s +Step 230/44000 | Loss: 6.5850 | LR: 1.53e-04 | 1901.7ms/step | 10.5 samples/s +Step 240/44000 | Loss: 6.5022 | LR: 1.60e-04 | 1905.2ms/step | 10.5 samples/s +Step 250/44000 | Loss: 6.5164 | LR: 1.67e-04 | 1898.9ms/step | 10.5 samples/s +Validation at step 250: Loss = 6.4494, Perplexity = 632.34 +Step 260/44000 | Loss: 6.3813 | LR: 1.73e-04 | 7451.5ms/step | 2.7 samples/s +Step 270/44000 | Loss: 6.3885 | LR: 1.80e-04 | 1979.9ms/step | 10.1 samples/s +Step 280/44000 | Loss: 6.3057 | LR: 1.87e-04 | 1980.5ms/step | 10.1 samples/s +Step 290/44000 | Loss: 6.2816 | LR: 1.93e-04 | 1981.6ms/step | 10.1 samples/s +Step 300/44000 | Loss: 6.1669 | LR: 2.00e-04 | 1979.4ms/step | 10.1 samples/s +Step 310/44000 | Loss: 6.1925 | LR: 2.07e-04 | 1986.3ms/step | 10.1 samples/s +Step 320/44000 | Loss: 6.1009 | LR: 2.13e-04 | 1991.1ms/step | 10.0 samples/s +Step 330/44000 | Loss: 6.1003 | LR: 2.20e-04 | 1984.4ms/step | 10.1 samples/s +Step 340/44000 | Loss: 5.9902 | LR: 2.27e-04 | 1996.8ms/step | 10.0 samples/s +Step 350/44000 | Loss: 5.9901 | LR: 2.33e-04 | 1995.9ms/step | 10.0 samples/s +Step 360/44000 | Loss: 5.9625 | LR: 2.40e-04 | 2248.3ms/step | 8.9 samples/s +Step 370/44000 | Loss: 5.9370 | LR: 2.47e-04 | 2108.9ms/step | 9.5 samples/s +Step 380/44000 | Loss: 5.8736 | LR: 2.53e-04 | 1954.4ms/step | 10.2 samples/s +Step 390/44000 | Loss: 5.8622 | LR: 2.60e-04 | 2007.0ms/step | 10.0 samples/s +Step 400/44000 | Loss: 5.8278 | LR: 2.67e-04 | 2119.7ms/step | 9.4 samples/s +Step 410/44000 | Loss: 5.7702 | LR: 2.73e-04 | 2065.9ms/step | 9.7 samples/s +Step 420/44000 | Loss: 5.7540 | LR: 2.80e-04 | 2102.2ms/step | 9.5 samples/s +Step 430/44000 | Loss: 5.7292 | LR: 2.87e-04 | 2129.2ms/step | 9.4 samples/s +Step 440/44000 | Loss: 5.6705 | LR: 2.93e-04 | 1990.5ms/step | 10.0 samples/s +Step 450/44000 | Loss: 5.6962 | LR: 3.00e-04 | 2190.9ms/step | 9.1 samples/s +Step 460/44000 | Loss: 5.6392 | LR: 3.07e-04 | 2015.8ms/step | 9.9 samples/s +Step 470/44000 | Loss: 5.5728 | LR: 3.13e-04 | 2096.2ms/step | 9.5 samples/s +Step 480/44000 | Loss: 5.5495 | LR: 3.20e-04 | 1954.5ms/step | 10.2 samples/s +Step 490/44000 | Loss: 5.5168 | LR: 3.27e-04 | 2178.4ms/step | 9.2 samples/s +Step 500/44000 | Loss: 5.4691 | LR: 3.33e-04 | 2032.2ms/step | 9.8 samples/s +Validation at step 500: Loss = 5.4878, Perplexity = 241.72 +Step 510/44000 | Loss: 5.4580 | LR: 3.40e-04 | 3837.0ms/step | 5.2 samples/s +Step 520/44000 | Loss: 5.4437 | LR: 3.47e-04 | 2016.9ms/step | 9.9 samples/s +Step 530/44000 | Loss: 5.3839 | LR: 3.53e-04 | 1934.1ms/step | 10.3 samples/s +Step 540/44000 | Loss: 5.3504 | LR: 3.60e-04 | 1903.3ms/step | 10.5 samples/s +Step 550/44000 | Loss: 5.3023 | LR: 3.67e-04 | 1937.8ms/step | 10.3 samples/s +Step 560/44000 | Loss: 5.2891 | LR: 3.73e-04 | 2148.6ms/step | 9.3 samples/s +Step 570/44000 | Loss: 5.2507 | LR: 3.80e-04 | 2161.1ms/step | 9.3 samples/s +Step 580/44000 | Loss: 5.2478 | LR: 3.87e-04 | 2030.4ms/step | 9.9 samples/s +Step 590/44000 | Loss: 5.2034 | LR: 3.93e-04 | 2360.4ms/step | 8.5 samples/s +Step 600/44000 | Loss: 5.1882 | LR: 4.00e-04 | 2011.4ms/step | 9.9 samples/s +Step 610/44000 | Loss: 5.1384 | LR: 4.07e-04 | 2142.3ms/step | 9.3 samples/s +Step 620/44000 | Loss: 5.1160 | LR: 4.13e-04 | 2246.1ms/step | 8.9 samples/s +Step 630/44000 | Loss: 5.0912 | LR: 4.20e-04 | 2001.5ms/step | 10.0 samples/s +Step 640/44000 | Loss: 5.0530 | LR: 4.27e-04 | 2303.4ms/step | 8.7 samples/s +Step 650/44000 | Loss: 5.0768 | LR: 4.33e-04 | 2028.0ms/step | 9.9 samples/s +Step 660/44000 | Loss: 4.9974 | LR: 4.40e-04 | 2047.2ms/step | 9.8 samples/s +Step 670/44000 | Loss: 4.9970 | LR: 4.47e-04 | 2341.1ms/step | 8.5 samples/s +Step 680/44000 | Loss: 4.9147 | LR: 4.53e-04 | 2012.3ms/step | 9.9 samples/s +Step 690/44000 | Loss: 4.9141 | LR: 4.60e-04 | 2272.2ms/step | 8.8 samples/s +Step 700/44000 | Loss: 4.8543 | LR: 4.67e-04 | 2059.3ms/step | 9.7 samples/s +Step 710/44000 | Loss: 4.8532 | LR: 4.73e-04 | 2022.8ms/step | 9.9 samples/s +Step 720/44000 | Loss: 4.8204 | LR: 4.80e-04 | 2329.5ms/step | 8.6 samples/s +Step 730/44000 | Loss: 4.8319 | LR: 4.87e-04 | 1996.1ms/step | 10.0 samples/s +Step 740/44000 | Loss: 4.7697 | LR: 4.93e-04 | 2267.0ms/step | 8.8 samples/s +Step 750/44000 | Loss: 4.7646 | LR: 5.00e-04 | 2073.6ms/step | 9.6 samples/s +Validation at step 750: Loss = 4.7505, Perplexity = 115.64 +Step 760/44000 | Loss: 4.7084 | LR: 5.00e-04 | 3643.3ms/step | 5.5 samples/s +Step 770/44000 | Loss: 4.6092 | LR: 5.00e-04 | 1991.0ms/step | 10.0 samples/s +Step 780/44000 | Loss: 4.6500 | LR: 5.00e-04 | 2183.9ms/step | 9.2 samples/s +Step 790/44000 | Loss: 4.6011 | LR: 5.00e-04 | 2106.6ms/step | 9.5 samples/s +Step 800/44000 | Loss: 4.5785 | LR: 5.00e-04 | 1985.9ms/step | 10.1 samples/s +Step 810/44000 | Loss: 4.6121 | LR: 5.00e-04 | 1986.1ms/step | 10.1 samples/s +Step 820/44000 | Loss: 4.5758 | LR: 5.00e-04 | 2001.7ms/step | 10.0 samples/s +Step 830/44000 | Loss: 4.5334 | LR: 5.00e-04 | 2006.7ms/step | 10.0 samples/s +Step 840/44000 | Loss: 4.5053 | LR: 5.00e-04 | 2001.0ms/step | 10.0 samples/s +Step 850/44000 | Loss: 4.4689 | LR: 5.00e-04 | 2012.6ms/step | 9.9 samples/s +Step 860/44000 | Loss: 4.4568 | LR: 5.00e-04 | 2013.1ms/step | 9.9 samples/s +Step 870/44000 | Loss: 4.4189 | LR: 5.00e-04 | 1989.5ms/step | 10.1 samples/s +Step 880/44000 | Loss: 4.4130 | LR: 5.00e-04 | 1986.8ms/step | 10.1 samples/s +Step 890/44000 | Loss: 4.4291 | LR: 5.00e-04 | 2001.9ms/step | 10.0 samples/s +Step 900/44000 | Loss: 4.3631 | LR: 5.00e-04 | 2005.4ms/step | 10.0 samples/s +Step 910/44000 | Loss: 4.3613 | LR: 5.00e-04 | 1997.8ms/step | 10.0 samples/s +Step 920/44000 | Loss: 4.3600 | LR: 5.00e-04 | 1991.3ms/step | 10.0 samples/s +Step 930/44000 | Loss: 4.3745 | LR: 5.00e-04 | 2011.1ms/step | 9.9 samples/s +Step 940/44000 | Loss: 4.3284 | LR: 5.00e-04 | 2002.8ms/step | 10.0 samples/s +Step 950/44000 | Loss: 4.3235 | LR: 5.00e-04 | 1990.5ms/step | 10.0 samples/s +Step 960/44000 | Loss: 4.2613 | LR: 5.00e-04 | 1990.9ms/step | 10.0 samples/s +Step 970/44000 | Loss: 4.2988 | LR: 5.00e-04 | 2002.0ms/step | 10.0 samples/s +Step 980/44000 | Loss: 4.2558 | LR: 5.00e-04 | 2006.5ms/step | 10.0 samples/s +Step 990/44000 | Loss: 4.2514 | LR: 5.00e-04 | 2017.4ms/step | 9.9 samples/s +Step 1000/44000 | Loss: 4.2331 | LR: 5.00e-04 | 2022.4ms/step | 9.9 samples/s +Validation at step 1000: Loss = 4.2645, Perplexity = 71.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-1000 +Step 1010/44000 | Loss: 4.2865 | LR: 5.00e-04 | 4086.3ms/step | 4.9 samples/s +Step 1020/44000 | Loss: 4.1899 | LR: 5.00e-04 | 2032.8ms/step | 9.8 samples/s +Step 1030/44000 | Loss: 4.2330 | LR: 5.00e-04 | 2040.6ms/step | 9.8 samples/s +Step 1040/44000 | Loss: 4.2216 | LR: 5.00e-04 | 2025.2ms/step | 9.9 samples/s +Step 1050/44000 | Loss: 4.2004 | LR: 5.00e-04 | 2048.4ms/step | 9.8 samples/s +Step 1060/44000 | Loss: 4.1803 | LR: 5.00e-04 | 2042.4ms/step | 9.8 samples/s +Step 1070/44000 | Loss: 4.1970 | LR: 5.00e-04 | 2039.0ms/step | 9.8 samples/s +Step 1080/44000 | Loss: 4.1532 | LR: 5.00e-04 | 2043.8ms/step | 9.8 samples/s +Step 1090/44000 | Loss: 4.0904 | LR: 5.00e-04 | 2044.1ms/step | 9.8 samples/s +Step 1100/44000 | Loss: 4.0873 | LR: 5.00e-04 | 2022.5ms/step | 9.9 samples/s +Step 1110/44000 | Loss: 4.1017 | LR: 5.00e-04 | 2018.2ms/step | 9.9 samples/s +Step 1120/44000 | Loss: 4.1328 | LR: 5.00e-04 | 2012.7ms/step | 9.9 samples/s +Step 1130/44000 | Loss: 4.0907 | LR: 5.00e-04 | 2011.8ms/step | 9.9 samples/s +Step 1140/44000 | Loss: 4.0758 | LR: 5.00e-04 | 1986.5ms/step | 10.1 samples/s +Step 1150/44000 | Loss: 4.1182 | LR: 5.00e-04 | 1996.4ms/step | 10.0 samples/s +Step 1160/44000 | Loss: 4.1058 | LR: 5.00e-04 | 2002.5ms/step | 10.0 samples/s +Step 1170/44000 | Loss: 4.0691 | LR: 5.00e-04 | 2001.3ms/step | 10.0 samples/s +Step 1180/44000 | Loss: 4.0561 | LR: 5.00e-04 | 1985.8ms/step | 10.1 samples/s +Step 1190/44000 | Loss: 4.0748 | LR: 5.00e-04 | 1981.1ms/step | 10.1 samples/s +Step 1200/44000 | Loss: 4.0274 | LR: 5.00e-04 | 2002.8ms/step | 10.0 samples/s +Step 1210/44000 | Loss: 4.0396 | LR: 5.00e-04 | 1996.3ms/step | 10.0 samples/s +Step 1220/44000 | Loss: 4.0061 | LR: 5.00e-04 | 2011.7ms/step | 9.9 samples/s +Step 1230/44000 | Loss: 4.0359 | LR: 5.00e-04 | 2011.7ms/step | 9.9 samples/s +Step 1240/44000 | Loss: 3.9949 | LR: 5.00e-04 | 2011.2ms/step | 9.9 samples/s +Step 1250/44000 | Loss: 4.0019 | LR: 5.00e-04 | 2006.5ms/step | 10.0 samples/s +Validation at step 1250: Loss = 4.0254, Perplexity = 56.00 +Step 1260/44000 | Loss: 4.0115 | LR: 5.00e-04 | 3695.8ms/step | 5.4 samples/s +Step 1270/44000 | Loss: 3.9967 | LR: 5.00e-04 | 2006.4ms/step | 10.0 samples/s +Step 1280/44000 | Loss: 4.0969 | LR: 5.00e-04 | 2006.6ms/step | 10.0 samples/s +Step 1290/44000 | Loss: 3.9950 | LR: 5.00e-04 | 2007.0ms/step | 10.0 samples/s +Step 1300/44000 | Loss: 3.9317 | LR: 5.00e-04 | 2010.5ms/step | 9.9 samples/s +Step 1310/44000 | Loss: 3.9500 | LR: 5.00e-04 | 2011.3ms/step | 9.9 samples/s +Step 1320/44000 | Loss: 3.9988 | LR: 5.00e-04 | 2004.0ms/step | 10.0 samples/s +Step 1330/44000 | Loss: 3.9470 | LR: 5.00e-04 | 2017.8ms/step | 9.9 samples/s +Step 1340/44000 | Loss: 3.9540 | LR: 5.00e-04 | 1995.7ms/step | 10.0 samples/s +Step 1350/44000 | Loss: 3.9419 | LR: 5.00e-04 | 2023.2ms/step | 9.9 samples/s +Step 1360/44000 | Loss: 3.9277 | LR: 5.00e-04 | 2012.3ms/step | 9.9 samples/s +Step 1370/44000 | Loss: 3.9261 | LR: 5.00e-04 | 2006.9ms/step | 10.0 samples/s +Step 1380/44000 | Loss: 3.9429 | LR: 5.00e-04 | 2005.3ms/step | 10.0 samples/s +Step 1390/44000 | Loss: 3.9372 | LR: 5.00e-04 | 2013.1ms/step | 9.9 samples/s +Step 1400/44000 | Loss: 3.8746 | LR: 5.00e-04 | 2027.5ms/step | 9.9 samples/s +Step 1410/44000 | Loss: 3.9244 | LR: 5.00e-04 | 1996.8ms/step | 10.0 samples/s +Step 1420/44000 | Loss: 3.9463 | LR: 5.00e-04 | 2001.3ms/step | 10.0 samples/s +Step 1430/44000 | Loss: 3.8801 | LR: 5.00e-04 | 2017.6ms/step | 9.9 samples/s +Step 1440/44000 | Loss: 3.9105 | LR: 5.00e-04 | 2013.0ms/step | 9.9 samples/s +Step 1450/44000 | Loss: 3.8525 | LR: 5.00e-04 | 2028.5ms/step | 9.9 samples/s +Step 1460/44000 | Loss: 3.8650 | LR: 5.00e-04 | 2012.0ms/step | 9.9 samples/s +Step 1470/44000 | Loss: 3.8660 | LR: 5.00e-04 | 2023.1ms/step | 9.9 samples/s +Step 1480/44000 | Loss: 3.8459 | LR: 5.00e-04 | 2017.1ms/step | 9.9 samples/s +Step 1490/44000 | Loss: 3.8447 | LR: 5.00e-04 | 2012.2ms/step | 9.9 samples/s +Step 1500/44000 | Loss: 3.8765 | LR: 5.00e-04 | 1990.4ms/step | 10.0 samples/s +Validation at step 1500: Loss = 3.8757, Perplexity = 48.22 +Step 1510/44000 | Loss: 3.8351 | LR: 5.00e-04 | 3499.2ms/step | 5.7 samples/s +Step 1520/44000 | Loss: 3.8648 | LR: 5.00e-04 | 2006.7ms/step | 10.0 samples/s +Step 1530/44000 | Loss: 3.7968 | LR: 5.00e-04 | 2011.6ms/step | 9.9 samples/s +Step 1540/44000 | Loss: 3.8489 | LR: 5.00e-04 | 2005.9ms/step | 10.0 samples/s +Step 1550/44000 | Loss: 3.7865 | LR: 5.00e-04 | 2021.3ms/step | 9.9 samples/s +Step 1560/44000 | Loss: 3.8335 | LR: 5.00e-04 | 2003.1ms/step | 10.0 samples/s +Step 1570/44000 | Loss: 3.8377 | LR: 5.00e-04 | 2004.9ms/step | 10.0 samples/s +Step 1580/44000 | Loss: 3.8117 | LR: 5.00e-04 | 2017.1ms/step | 9.9 samples/s +Step 1590/44000 | Loss: 3.8104 | LR: 5.00e-04 | 2001.0ms/step | 10.0 samples/s +Step 1600/44000 | Loss: 3.8218 | LR: 5.00e-04 | 2015.4ms/step | 9.9 samples/s +Step 1610/44000 | Loss: 3.8095 | LR: 5.00e-04 | 1997.1ms/step | 10.0 samples/s +Step 1620/44000 | Loss: 3.8182 | LR: 5.00e-04 | 2006.6ms/step | 10.0 samples/s +Step 1630/44000 | Loss: 3.7623 | LR: 4.99e-04 | 2012.1ms/step | 9.9 samples/s +Step 1640/44000 | Loss: 3.8214 | LR: 4.99e-04 | 2011.1ms/step | 9.9 samples/s +Step 1650/44000 | Loss: 3.8117 | LR: 4.99e-04 | 2012.2ms/step | 9.9 samples/s +Step 1660/44000 | Loss: 3.7539 | LR: 4.99e-04 | 2032.2ms/step | 9.8 samples/s +Step 1670/44000 | Loss: 3.7845 | LR: 4.99e-04 | 2014.3ms/step | 9.9 samples/s +Step 1680/44000 | Loss: 3.7361 | LR: 4.99e-04 | 2002.6ms/step | 10.0 samples/s +Step 1690/44000 | Loss: 3.8115 | LR: 4.99e-04 | 2011.5ms/step | 9.9 samples/s +Step 1700/44000 | Loss: 3.7656 | LR: 4.99e-04 | 2021.1ms/step | 9.9 samples/s +Step 1710/44000 | Loss: 3.7409 | LR: 4.99e-04 | 2006.7ms/step | 10.0 samples/s +Step 1720/44000 | Loss: 3.7203 | LR: 4.99e-04 | 1995.3ms/step | 10.0 samples/s +Step 1730/44000 | Loss: 3.7352 | LR: 4.99e-04 | 2026.7ms/step | 9.9 samples/s +Step 1740/44000 | Loss: 3.7373 | LR: 4.99e-04 | 2013.7ms/step | 9.9 samples/s +Step 1750/44000 | Loss: 3.7673 | LR: 4.99e-04 | 2004.3ms/step | 10.0 samples/s +Validation at step 1750: Loss = 3.7707, Perplexity = 43.41 +Step 1760/44000 | Loss: 3.7213 | LR: 4.99e-04 | 3695.1ms/step | 5.4 samples/s +Step 1770/44000 | Loss: 3.7385 | LR: 4.99e-04 | 2017.0ms/step | 9.9 samples/s +Step 1780/44000 | Loss: 3.6983 | LR: 4.99e-04 | 2017.3ms/step | 9.9 samples/s +Step 1790/44000 | Loss: 3.7753 | LR: 4.99e-04 | 2006.5ms/step | 10.0 samples/s +Step 1800/44000 | Loss: 3.7324 | LR: 4.99e-04 | 1990.4ms/step | 10.0 samples/s +Step 1810/44000 | Loss: 3.7468 | LR: 4.99e-04 | 1977.9ms/step | 10.1 samples/s +Step 1820/44000 | Loss: 3.7253 | LR: 4.99e-04 | 2008.9ms/step | 10.0 samples/s +Step 1830/44000 | Loss: 3.7131 | LR: 4.99e-04 | 2011.2ms/step | 9.9 samples/s +Step 1840/44000 | Loss: 3.7118 | LR: 4.99e-04 | 1996.4ms/step | 10.0 samples/s +Step 1850/44000 | Loss: 3.7678 | LR: 4.99e-04 | 2011.5ms/step | 9.9 samples/s +Step 1860/44000 | Loss: 3.7557 | LR: 4.99e-04 | 2004.6ms/step | 10.0 samples/s +Step 1870/44000 | Loss: 3.7006 | LR: 4.99e-04 | 2030.5ms/step | 9.8 samples/s +Step 1880/44000 | Loss: 3.6538 | LR: 4.99e-04 | 2016.2ms/step | 9.9 samples/s +Step 1890/44000 | Loss: 3.6904 | LR: 4.99e-04 | 1996.6ms/step | 10.0 samples/s +Step 1900/44000 | Loss: 3.6842 | LR: 4.99e-04 | 2016.4ms/step | 9.9 samples/s +Step 1910/44000 | Loss: 3.6801 | LR: 4.99e-04 | 2022.2ms/step | 9.9 samples/s +Step 1920/44000 | Loss: 3.6910 | LR: 4.99e-04 | 2011.1ms/step | 9.9 samples/s +Step 1930/44000 | Loss: 3.6560 | LR: 4.99e-04 | 2038.1ms/step | 9.8 samples/s +Step 1940/44000 | Loss: 3.7055 | LR: 4.99e-04 | 2005.0ms/step | 10.0 samples/s +Step 1950/44000 | Loss: 3.6807 | LR: 4.99e-04 | 1987.6ms/step | 10.1 samples/s +Step 1960/44000 | Loss: 3.6896 | LR: 4.99e-04 | 1985.7ms/step | 10.1 samples/s +Step 1970/44000 | Loss: 3.7047 | LR: 4.99e-04 | 1980.9ms/step | 10.1 samples/s +Step 1980/44000 | Loss: 3.6774 | LR: 4.99e-04 | 1979.9ms/step | 10.1 samples/s +Step 1990/44000 | Loss: 3.6818 | LR: 4.99e-04 | 1995.1ms/step | 10.0 samples/s +Step 2000/44000 | Loss: 3.6615 | LR: 4.99e-04 | 1999.2ms/step | 10.0 samples/s +Validation at step 2000: Loss = 3.6856, Perplexity = 39.87 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-2000 +Step 2010/44000 | Loss: 3.7131 | LR: 4.99e-04 | 4285.6ms/step | 4.7 samples/s +Step 2020/44000 | Loss: 3.6675 | LR: 4.99e-04 | 2042.9ms/step | 9.8 samples/s +Step 2030/44000 | Loss: 3.6482 | LR: 4.99e-04 | 2048.0ms/step | 9.8 samples/s +Step 2040/44000 | Loss: 3.6633 | LR: 4.99e-04 | 2054.0ms/step | 9.7 samples/s +Step 2050/44000 | Loss: 3.7158 | LR: 4.99e-04 | 2053.0ms/step | 9.7 samples/s +Step 2060/44000 | Loss: 3.6719 | LR: 4.99e-04 | 2027.0ms/step | 9.9 samples/s +Step 2070/44000 | Loss: 3.6155 | LR: 4.99e-04 | 2016.3ms/step | 9.9 samples/s +Step 2080/44000 | Loss: 3.6452 | LR: 4.99e-04 | 2017.3ms/step | 9.9 samples/s +Step 2090/44000 | Loss: 3.6707 | LR: 4.99e-04 | 2011.3ms/step | 9.9 samples/s +Step 2100/44000 | Loss: 3.6403 | LR: 4.99e-04 | 2011.7ms/step | 9.9 samples/s +Step 2110/44000 | Loss: 3.6275 | LR: 4.99e-04 | 2012.2ms/step | 9.9 samples/s +Step 2120/44000 | Loss: 3.6116 | LR: 4.99e-04 | 1979.2ms/step | 10.1 samples/s +Step 2130/44000 | Loss: 3.6644 | LR: 4.99e-04 | 1996.5ms/step | 10.0 samples/s +Step 2140/44000 | Loss: 3.6252 | LR: 4.99e-04 | 2001.8ms/step | 10.0 samples/s +Step 2150/44000 | Loss: 3.5822 | LR: 4.99e-04 | 2026.7ms/step | 9.9 samples/s +Step 2160/44000 | Loss: 3.6621 | LR: 4.99e-04 | 1996.4ms/step | 10.0 samples/s +Step 2170/44000 | Loss: 3.6445 | LR: 4.99e-04 | 1996.3ms/step | 10.0 samples/s +Step 2180/44000 | Loss: 3.5918 | LR: 4.99e-04 | 2001.8ms/step | 10.0 samples/s +Step 2190/44000 | Loss: 3.6364 | LR: 4.99e-04 | 2000.8ms/step | 10.0 samples/s +Step 2200/44000 | Loss: 3.6148 | LR: 4.99e-04 | 2007.2ms/step | 10.0 samples/s +Step 2210/44000 | Loss: 3.5663 | LR: 4.99e-04 | 1996.9ms/step | 10.0 samples/s +Step 2220/44000 | Loss: 3.5992 | LR: 4.99e-04 | 2006.4ms/step | 10.0 samples/s +Step 2230/44000 | Loss: 3.5894 | LR: 4.99e-04 | 2006.9ms/step | 10.0 samples/s +Step 2240/44000 | Loss: 3.6141 | LR: 4.99e-04 | 1996.8ms/step | 10.0 samples/s +Step 2250/44000 | Loss: 3.6144 | LR: 4.99e-04 | 1991.2ms/step | 10.0 samples/s +Validation at step 2250: Loss = 3.6216, Perplexity = 37.40 +Step 2260/44000 | Loss: 3.6040 | LR: 4.98e-04 | 3709.6ms/step | 5.4 samples/s +Step 2270/44000 | Loss: 3.6218 | LR: 4.98e-04 | 2000.0ms/step | 10.0 samples/s +Step 2280/44000 | Loss: 3.5805 | LR: 4.98e-04 | 2008.1ms/step | 10.0 samples/s +Step 2290/44000 | Loss: 3.5948 | LR: 4.98e-04 | 2006.0ms/step | 10.0 samples/s +Step 2300/44000 | Loss: 3.5864 | LR: 4.98e-04 | 2015.9ms/step | 9.9 samples/s +Step 2310/44000 | Loss: 3.5803 | LR: 4.98e-04 | 2001.9ms/step | 10.0 samples/s +Step 2320/44000 | Loss: 3.6115 | LR: 4.98e-04 | 2007.2ms/step | 10.0 samples/s +Step 2330/44000 | Loss: 3.5609 | LR: 4.98e-04 | 2011.9ms/step | 9.9 samples/s +Step 2340/44000 | Loss: 3.5791 | LR: 4.98e-04 | 2007.1ms/step | 10.0 samples/s +Step 2350/44000 | Loss: 3.5544 | LR: 4.98e-04 | 2007.2ms/step | 10.0 samples/s +Step 2360/44000 | Loss: 3.5679 | LR: 4.98e-04 | 2017.2ms/step | 9.9 samples/s +Step 2370/44000 | Loss: 3.5911 | LR: 4.98e-04 | 2003.1ms/step | 10.0 samples/s +Step 2380/44000 | Loss: 3.5979 | LR: 4.98e-04 | 2031.8ms/step | 9.8 samples/s +Step 2390/44000 | Loss: 3.6096 | LR: 4.98e-04 | 2054.2ms/step | 9.7 samples/s +Step 2400/44000 | Loss: 3.6137 | LR: 4.98e-04 | 2033.5ms/step | 9.8 samples/s +Step 2410/44000 | Loss: 3.5733 | LR: 4.98e-04 | 2048.5ms/step | 9.8 samples/s +Step 2420/44000 | Loss: 3.5510 | LR: 4.98e-04 | 2037.9ms/step | 9.8 samples/s +Step 2430/44000 | Loss: 3.5939 | LR: 4.98e-04 | 2034.5ms/step | 9.8 samples/s +Step 2440/44000 | Loss: 3.5750 | LR: 4.98e-04 | 2004.7ms/step | 10.0 samples/s +Step 2450/44000 | Loss: 3.5836 | LR: 4.98e-04 | 2043.4ms/step | 9.8 samples/s +Step 2460/44000 | Loss: 3.5400 | LR: 4.98e-04 | 2021.8ms/step | 9.9 samples/s +Step 2470/44000 | Loss: 3.5377 | LR: 4.98e-04 | 2002.0ms/step | 10.0 samples/s +Step 2480/44000 | Loss: 3.6161 | LR: 4.98e-04 | 2011.3ms/step | 9.9 samples/s +Step 2490/44000 | Loss: 3.5189 | LR: 4.98e-04 | 2017.6ms/step | 9.9 samples/s +Step 2500/44000 | Loss: 3.5718 | LR: 4.98e-04 | 2001.6ms/step | 10.0 samples/s +Validation at step 2500: Loss = 3.5723, Perplexity = 35.60 +Step 2510/44000 | Loss: 3.5495 | LR: 4.98e-04 | 3704.8ms/step | 5.4 samples/s +Step 2520/44000 | Loss: 3.5549 | LR: 4.98e-04 | 2016.2ms/step | 9.9 samples/s +Step 2530/44000 | Loss: 3.5570 | LR: 4.98e-04 | 2011.4ms/step | 9.9 samples/s +Step 2540/44000 | Loss: 3.5239 | LR: 4.98e-04 | 2006.6ms/step | 10.0 samples/s +Step 2550/44000 | Loss: 3.5374 | LR: 4.98e-04 | 2007.9ms/step | 10.0 samples/s +Step 2560/44000 | Loss: 3.5369 | LR: 4.98e-04 | 2014.1ms/step | 9.9 samples/s +Step 2570/44000 | Loss: 3.5303 | LR: 4.98e-04 | 2008.7ms/step | 10.0 samples/s +Step 2580/44000 | Loss: 3.5355 | LR: 4.98e-04 | 2002.3ms/step | 10.0 samples/s +Step 2590/44000 | Loss: 3.4989 | LR: 4.98e-04 | 2001.0ms/step | 10.0 samples/s +Step 2600/44000 | Loss: 3.5424 | LR: 4.98e-04 | 2006.4ms/step | 10.0 samples/s +Step 2610/44000 | Loss: 3.4990 | LR: 4.98e-04 | 2011.3ms/step | 9.9 samples/s +Step 2620/44000 | Loss: 3.5129 | LR: 4.98e-04 | 2011.4ms/step | 9.9 samples/s +Step 2630/44000 | Loss: 3.5005 | LR: 4.98e-04 | 2041.7ms/step | 9.8 samples/s +Step 2640/44000 | Loss: 3.5677 | LR: 4.98e-04 | 2027.7ms/step | 9.9 samples/s +Step 2650/44000 | Loss: 3.5490 | LR: 4.98e-04 | 2047.4ms/step | 9.8 samples/s +Step 2660/44000 | Loss: 3.5234 | LR: 4.98e-04 | 2052.9ms/step | 9.7 samples/s +Step 2670/44000 | Loss: 3.4863 | LR: 4.98e-04 | 2051.9ms/step | 9.7 samples/s +Step 2680/44000 | Loss: 3.4892 | LR: 4.98e-04 | 2043.6ms/step | 9.8 samples/s +Step 2690/44000 | Loss: 3.4847 | LR: 4.98e-04 | 2062.4ms/step | 9.7 samples/s +Step 2700/44000 | Loss: 3.5305 | LR: 4.97e-04 | 2052.7ms/step | 9.7 samples/s +Step 2710/44000 | Loss: 3.5421 | LR: 4.97e-04 | 2053.2ms/step | 9.7 samples/s +Step 2720/44000 | Loss: 3.5106 | LR: 4.97e-04 | 2029.9ms/step | 9.9 samples/s +Step 2730/44000 | Loss: 3.5398 | LR: 4.97e-04 | 2001.7ms/step | 10.0 samples/s +Step 2740/44000 | Loss: 3.5143 | LR: 4.97e-04 | 2015.4ms/step | 9.9 samples/s +Step 2750/44000 | Loss: 3.4811 | LR: 4.97e-04 | 1989.3ms/step | 10.1 samples/s +Validation at step 2750: Loss = 3.5278, Perplexity = 34.05 +Step 2760/44000 | Loss: 3.5133 | LR: 4.97e-04 | 3531.2ms/step | 5.7 samples/s +Step 2770/44000 | Loss: 3.5292 | LR: 4.97e-04 | 1907.3ms/step | 10.5 samples/s +Step 2780/44000 | Loss: 3.4861 | LR: 4.97e-04 | 1973.7ms/step | 10.1 samples/s +Step 2790/44000 | Loss: 3.5296 | LR: 4.97e-04 | 2018.4ms/step | 9.9 samples/s +Step 2800/44000 | Loss: 3.4999 | LR: 4.97e-04 | 2018.9ms/step | 9.9 samples/s +Step 2810/44000 | Loss: 3.5340 | LR: 4.97e-04 | 2001.1ms/step | 10.0 samples/s +Step 2820/44000 | Loss: 3.5274 | LR: 4.97e-04 | 2010.7ms/step | 9.9 samples/s +Step 2830/44000 | Loss: 3.5089 | LR: 4.97e-04 | 2017.4ms/step | 9.9 samples/s +Step 2840/44000 | Loss: 3.4672 | LR: 4.97e-04 | 2016.8ms/step | 9.9 samples/s +Step 2850/44000 | Loss: 3.4162 | LR: 4.97e-04 | 2006.3ms/step | 10.0 samples/s +Step 2860/44000 | Loss: 3.4837 | LR: 4.97e-04 | 2005.4ms/step | 10.0 samples/s +Step 2870/44000 | Loss: 3.4860 | LR: 4.97e-04 | 2012.6ms/step | 9.9 samples/s +Step 2880/44000 | Loss: 3.4862 | LR: 4.97e-04 | 2019.8ms/step | 9.9 samples/s +Step 2890/44000 | Loss: 3.5345 | LR: 4.97e-04 | 2021.6ms/step | 9.9 samples/s +Step 2900/44000 | Loss: 3.5133 | LR: 4.97e-04 | 2000.9ms/step | 10.0 samples/s +Step 2910/44000 | Loss: 3.5095 | LR: 4.97e-04 | 2010.4ms/step | 9.9 samples/s +Step 2920/44000 | Loss: 3.4556 | LR: 4.97e-04 | 2011.6ms/step | 9.9 samples/s +Step 2930/44000 | Loss: 3.4864 | LR: 4.97e-04 | 2021.8ms/step | 9.9 samples/s +Step 2940/44000 | Loss: 3.4474 | LR: 4.97e-04 | 2007.3ms/step | 10.0 samples/s +Step 2950/44000 | Loss: 3.4558 | LR: 4.97e-04 | 2020.8ms/step | 9.9 samples/s +Step 2960/44000 | Loss: 3.4771 | LR: 4.97e-04 | 2016.0ms/step | 9.9 samples/s +Step 2970/44000 | Loss: 3.4915 | LR: 4.97e-04 | 2032.5ms/step | 9.8 samples/s +Step 2980/44000 | Loss: 3.4557 | LR: 4.97e-04 | 2027.8ms/step | 9.9 samples/s +Step 2990/44000 | Loss: 3.4491 | LR: 4.97e-04 | 2016.4ms/step | 9.9 samples/s +Step 3000/44000 | Loss: 3.4292 | LR: 4.97e-04 | 2006.4ms/step | 10.0 samples/s +Validation at step 3000: Loss = 3.4937, Perplexity = 32.91 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-3000 +Step 3010/44000 | Loss: 3.4539 | LR: 4.97e-04 | 4127.7ms/step | 4.8 samples/s +Step 3020/44000 | Loss: 3.4314 | LR: 4.97e-04 | 2021.3ms/step | 9.9 samples/s +Step 3030/44000 | Loss: 3.4366 | LR: 4.97e-04 | 2012.3ms/step | 9.9 samples/s +Step 3040/44000 | Loss: 3.4272 | LR: 4.97e-04 | 2007.2ms/step | 10.0 samples/s +Step 3050/44000 | Loss: 3.4288 | LR: 4.97e-04 | 1985.2ms/step | 10.1 samples/s +Step 3060/44000 | Loss: 3.4544 | LR: 4.96e-04 | 1980.5ms/step | 10.1 samples/s +Step 3070/44000 | Loss: 3.4467 | LR: 4.96e-04 | 1985.8ms/step | 10.1 samples/s +Step 3080/44000 | Loss: 3.4794 | LR: 4.96e-04 | 1998.6ms/step | 10.0 samples/s +Step 3090/44000 | Loss: 3.4737 | LR: 4.96e-04 | 2104.1ms/step | 9.5 samples/s +Step 3100/44000 | Loss: 3.4524 | LR: 4.96e-04 | 2054.8ms/step | 9.7 samples/s +Step 3110/44000 | Loss: 3.4811 | LR: 4.96e-04 | 2298.1ms/step | 8.7 samples/s +Step 3120/44000 | Loss: 3.4632 | LR: 4.96e-04 | 2121.4ms/step | 9.4 samples/s +Step 3130/44000 | Loss: 3.4283 | LR: 4.96e-04 | 1969.1ms/step | 10.2 samples/s +Step 3140/44000 | Loss: 3.4304 | LR: 4.96e-04 | 1980.8ms/step | 10.1 samples/s +Step 3150/44000 | Loss: 3.4464 | LR: 4.96e-04 | 1982.1ms/step | 10.1 samples/s +Step 3160/44000 | Loss: 3.4550 | LR: 4.96e-04 | 1995.3ms/step | 10.0 samples/s +Step 3170/44000 | Loss: 3.4486 | LR: 4.96e-04 | 1994.5ms/step | 10.0 samples/s +Step 3180/44000 | Loss: 3.4735 | LR: 4.96e-04 | 2000.3ms/step | 10.0 samples/s +Step 3190/44000 | Loss: 3.4288 | LR: 4.96e-04 | 2025.8ms/step | 9.9 samples/s +Step 3200/44000 | Loss: 3.4519 | LR: 4.96e-04 | 2037.5ms/step | 9.8 samples/s +Step 3210/44000 | Loss: 3.4572 | LR: 4.96e-04 | 2048.1ms/step | 9.8 samples/s +Step 3220/44000 | Loss: 3.4858 | LR: 4.96e-04 | 2038.4ms/step | 9.8 samples/s +Step 3230/44000 | Loss: 3.4195 | LR: 4.96e-04 | 2048.3ms/step | 9.8 samples/s +Step 3240/44000 | Loss: 3.4358 | LR: 4.96e-04 | 2047.7ms/step | 9.8 samples/s +Step 3250/44000 | Loss: 3.3969 | LR: 4.96e-04 | 2033.0ms/step | 9.8 samples/s +Validation at step 3250: Loss = 3.4594, Perplexity = 31.80 +Step 3260/44000 | Loss: 3.4194 | LR: 4.96e-04 | 3762.7ms/step | 5.3 samples/s +Step 3270/44000 | Loss: 3.4366 | LR: 4.96e-04 | 2034.7ms/step | 9.8 samples/s +Step 3280/44000 | Loss: 3.4284 | LR: 4.96e-04 | 2015.3ms/step | 9.9 samples/s +Step 3290/44000 | Loss: 3.4561 | LR: 4.96e-04 | 2016.8ms/step | 9.9 samples/s +Step 3300/44000 | Loss: 3.4866 | LR: 4.96e-04 | 2001.7ms/step | 10.0 samples/s +Step 3310/44000 | Loss: 3.4389 | LR: 4.96e-04 | 2011.1ms/step | 9.9 samples/s +Step 3320/44000 | Loss: 3.4740 | LR: 4.96e-04 | 2001.9ms/step | 10.0 samples/s +Step 3330/44000 | Loss: 3.3821 | LR: 4.96e-04 | 2017.0ms/step | 9.9 samples/s +Step 3340/44000 | Loss: 3.4243 | LR: 4.96e-04 | 2026.6ms/step | 9.9 samples/s +Step 3350/44000 | Loss: 3.4268 | LR: 4.96e-04 | 2006.6ms/step | 10.0 samples/s +Step 3360/44000 | Loss: 3.4460 | LR: 4.96e-04 | 2001.9ms/step | 10.0 samples/s +Step 3370/44000 | Loss: 3.4154 | LR: 4.95e-04 | 1975.4ms/step | 10.1 samples/s +Step 3380/44000 | Loss: 3.4550 | LR: 4.95e-04 | 2007.1ms/step | 10.0 samples/s +Step 3390/44000 | Loss: 3.4668 | LR: 4.95e-04 | 2001.1ms/step | 10.0 samples/s +Step 3400/44000 | Loss: 3.4254 | LR: 4.95e-04 | 2022.8ms/step | 9.9 samples/s +Step 3410/44000 | Loss: 3.4438 | LR: 4.95e-04 | 2005.8ms/step | 10.0 samples/s +Step 3420/44000 | Loss: 3.4176 | LR: 4.95e-04 | 2012.1ms/step | 9.9 samples/s +Step 3430/44000 | Loss: 3.3975 | LR: 4.95e-04 | 2016.7ms/step | 9.9 samples/s +Step 3440/44000 | Loss: 3.4349 | LR: 4.95e-04 | 2022.3ms/step | 9.9 samples/s +Step 3450/44000 | Loss: 3.4359 | LR: 4.95e-04 | 2016.6ms/step | 9.9 samples/s +Step 3460/44000 | Loss: 3.3848 | LR: 4.95e-04 | 2022.4ms/step | 9.9 samples/s +Step 3470/44000 | Loss: 3.4038 | LR: 4.95e-04 | 2012.1ms/step | 9.9 samples/s +Step 3480/44000 | Loss: 3.4142 | LR: 4.95e-04 | 2032.4ms/step | 9.8 samples/s +Step 3490/44000 | Loss: 3.4238 | LR: 4.95e-04 | 2022.3ms/step | 9.9 samples/s +Step 3500/44000 | Loss: 3.4064 | LR: 4.95e-04 | 2017.3ms/step | 9.9 samples/s +Validation at step 3500: Loss = 3.4305, Perplexity = 30.89 +Step 3510/44000 | Loss: 3.4288 | LR: 4.95e-04 | 3705.0ms/step | 5.4 samples/s +Step 3520/44000 | Loss: 3.3817 | LR: 4.95e-04 | 2021.7ms/step | 9.9 samples/s +Step 3530/44000 | Loss: 3.4560 | LR: 4.95e-04 | 2032.8ms/step | 9.8 samples/s +Step 3540/44000 | Loss: 3.3862 | LR: 4.95e-04 | 2010.8ms/step | 9.9 samples/s +Step 3550/44000 | Loss: 3.4221 | LR: 4.95e-04 | 2011.2ms/step | 9.9 samples/s +Step 3560/44000 | Loss: 3.3612 | LR: 4.95e-04 | 2012.0ms/step | 9.9 samples/s +Step 3570/44000 | Loss: 3.4193 | LR: 4.95e-04 | 2022.9ms/step | 9.9 samples/s +Step 3580/44000 | Loss: 3.4677 | LR: 4.95e-04 | 2021.0ms/step | 9.9 samples/s +Step 3590/44000 | Loss: 3.3813 | LR: 4.95e-04 | 2053.5ms/step | 9.7 samples/s +Step 3600/44000 | Loss: 3.3486 | LR: 4.95e-04 | 2047.7ms/step | 9.8 samples/s +Step 3610/44000 | Loss: 3.3771 | LR: 4.95e-04 | 2038.0ms/step | 9.8 samples/s +Step 3620/44000 | Loss: 3.3781 | LR: 4.95e-04 | 2042.0ms/step | 9.8 samples/s +Step 3630/44000 | Loss: 3.4060 | LR: 4.95e-04 | 2037.4ms/step | 9.8 samples/s +Step 3640/44000 | Loss: 3.4443 | LR: 4.95e-04 | 2042.7ms/step | 9.8 samples/s +Step 3650/44000 | Loss: 3.4188 | LR: 4.94e-04 | 2063.2ms/step | 9.7 samples/s +Step 3660/44000 | Loss: 3.4036 | LR: 4.94e-04 | 2050.0ms/step | 9.8 samples/s +Step 3670/44000 | Loss: 3.3825 | LR: 4.94e-04 | 2045.9ms/step | 9.8 samples/s +Step 3680/44000 | Loss: 3.3639 | LR: 4.94e-04 | 1988.1ms/step | 10.1 samples/s +Step 3690/44000 | Loss: 3.3751 | LR: 4.94e-04 | 2001.9ms/step | 10.0 samples/s +Step 3700/44000 | Loss: 3.3649 | LR: 4.94e-04 | 2011.7ms/step | 9.9 samples/s +Step 3710/44000 | Loss: 3.3835 | LR: 4.94e-04 | 2017.2ms/step | 9.9 samples/s +Step 3720/44000 | Loss: 3.3884 | LR: 4.94e-04 | 2001.9ms/step | 10.0 samples/s +Step 3730/44000 | Loss: 3.3720 | LR: 4.94e-04 | 2008.2ms/step | 10.0 samples/s +Step 3740/44000 | Loss: 3.3754 | LR: 4.94e-04 | 2015.0ms/step | 9.9 samples/s +Step 3750/44000 | Loss: 3.3940 | LR: 4.94e-04 | 2010.3ms/step | 9.9 samples/s +Validation at step 3750: Loss = 3.4036, Perplexity = 30.07 +Step 3760/44000 | Loss: 3.4389 | LR: 4.94e-04 | 3707.4ms/step | 5.4 samples/s +Step 3770/44000 | Loss: 3.3665 | LR: 4.94e-04 | 2022.7ms/step | 9.9 samples/s +Step 3780/44000 | Loss: 3.3795 | LR: 4.94e-04 | 2021.3ms/step | 9.9 samples/s +Step 3790/44000 | Loss: 3.3797 | LR: 4.94e-04 | 2017.4ms/step | 9.9 samples/s +Step 3800/44000 | Loss: 3.3853 | LR: 4.94e-04 | 2017.3ms/step | 9.9 samples/s +Step 3810/44000 | Loss: 3.3900 | LR: 4.94e-04 | 2011.5ms/step | 9.9 samples/s +Step 3820/44000 | Loss: 3.3743 | LR: 4.94e-04 | 2006.7ms/step | 10.0 samples/s +Step 3830/44000 | Loss: 3.3793 | LR: 4.94e-04 | 2020.9ms/step | 9.9 samples/s +Step 3840/44000 | Loss: 3.3935 | LR: 4.94e-04 | 2003.3ms/step | 10.0 samples/s +Step 3850/44000 | Loss: 3.3947 | LR: 4.94e-04 | 2001.3ms/step | 10.0 samples/s +Step 3860/44000 | Loss: 3.3804 | LR: 4.94e-04 | 2027.2ms/step | 9.9 samples/s +Step 3870/44000 | Loss: 3.3866 | LR: 4.94e-04 | 2012.3ms/step | 9.9 samples/s +Step 3880/44000 | Loss: 3.3667 | LR: 4.94e-04 | 2016.9ms/step | 9.9 samples/s +Step 3890/44000 | Loss: 3.3878 | LR: 4.94e-04 | 2022.2ms/step | 9.9 samples/s +Step 3900/44000 | Loss: 3.4087 | LR: 4.93e-04 | 2021.5ms/step | 9.9 samples/s +Step 3910/44000 | Loss: 3.3091 | LR: 4.93e-04 | 2012.7ms/step | 9.9 samples/s +Step 3920/44000 | Loss: 3.3604 | LR: 4.93e-04 | 2026.7ms/step | 9.9 samples/s +Step 3930/44000 | Loss: 3.4003 | LR: 4.93e-04 | 2064.3ms/step | 9.7 samples/s +Step 3940/44000 | Loss: 3.3379 | LR: 4.93e-04 | 2052.8ms/step | 9.7 samples/s +Step 3950/44000 | Loss: 3.3554 | LR: 4.93e-04 | 2048.2ms/step | 9.8 samples/s +Step 3960/44000 | Loss: 3.3441 | LR: 4.93e-04 | 2037.6ms/step | 9.8 samples/s +Step 3970/44000 | Loss: 3.4025 | LR: 4.93e-04 | 2059.1ms/step | 9.7 samples/s +Step 3980/44000 | Loss: 3.3793 | LR: 4.93e-04 | 2073.7ms/step | 9.6 samples/s +Step 3990/44000 | Loss: 3.3536 | LR: 4.93e-04 | 2029.1ms/step | 9.9 samples/s +Step 4000/44000 | Loss: 3.3816 | LR: 4.93e-04 | 1980.3ms/step | 10.1 samples/s +Validation at step 4000: Loss = 3.3821, Perplexity = 29.43 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-4000 +Step 4010/44000 | Loss: 3.3574 | LR: 4.93e-04 | 4100.8ms/step | 4.9 samples/s +Step 4020/44000 | Loss: 3.3760 | LR: 4.93e-04 | 1986.6ms/step | 10.1 samples/s +Step 4030/44000 | Loss: 3.3628 | LR: 4.93e-04 | 2021.4ms/step | 9.9 samples/s +Step 4040/44000 | Loss: 3.3765 | LR: 4.93e-04 | 2018.0ms/step | 9.9 samples/s +Step 4050/44000 | Loss: 3.3994 | LR: 4.93e-04 | 2012.4ms/step | 9.9 samples/s +Step 4060/44000 | Loss: 3.3525 | LR: 4.93e-04 | 1996.4ms/step | 10.0 samples/s +Step 4070/44000 | Loss: 3.3579 | LR: 4.93e-04 | 2007.0ms/step | 10.0 samples/s +Step 4080/44000 | Loss: 3.3627 | LR: 4.93e-04 | 2023.9ms/step | 9.9 samples/s +Step 4090/44000 | Loss: 3.2978 | LR: 4.93e-04 | 2020.4ms/step | 9.9 samples/s +Step 4100/44000 | Loss: 3.3984 | LR: 4.93e-04 | 2012.9ms/step | 9.9 samples/s +Step 4110/44000 | Loss: 3.3736 | LR: 4.93e-04 | 2010.7ms/step | 9.9 samples/s +Step 4120/44000 | Loss: 3.3751 | LR: 4.93e-04 | 2011.8ms/step | 9.9 samples/s +Step 4130/44000 | Loss: 3.3679 | LR: 4.93e-04 | 2021.4ms/step | 9.9 samples/s +Step 4140/44000 | Loss: 3.3638 | LR: 4.92e-04 | 2022.9ms/step | 9.9 samples/s +Step 4150/44000 | Loss: 3.3473 | LR: 4.92e-04 | 1996.8ms/step | 10.0 samples/s +Step 4160/44000 | Loss: 3.3406 | LR: 4.92e-04 | 2017.3ms/step | 9.9 samples/s +Step 4170/44000 | Loss: 3.3547 | LR: 4.92e-04 | 2022.3ms/step | 9.9 samples/s +Step 4180/44000 | Loss: 3.3239 | LR: 4.92e-04 | 2016.9ms/step | 9.9 samples/s +Step 4190/44000 | Loss: 3.3766 | LR: 4.92e-04 | 2011.5ms/step | 9.9 samples/s +Step 4200/44000 | Loss: 3.3700 | LR: 4.92e-04 | 2014.2ms/step | 9.9 samples/s +Step 4210/44000 | Loss: 3.3587 | LR: 4.92e-04 | 2003.3ms/step | 10.0 samples/s +Step 4220/44000 | Loss: 3.3371 | LR: 4.92e-04 | 2032.9ms/step | 9.8 samples/s +Step 4230/44000 | Loss: 3.3107 | LR: 4.92e-04 | 2042.9ms/step | 9.8 samples/s +Step 4240/44000 | Loss: 3.3661 | LR: 4.92e-04 | 2053.2ms/step | 9.7 samples/s +Step 4250/44000 | Loss: 3.3589 | LR: 4.92e-04 | 2033.2ms/step | 9.8 samples/s +Validation at step 4250: Loss = 3.3617, Perplexity = 28.84 +Step 4260/44000 | Loss: 3.3333 | LR: 4.92e-04 | 3789.1ms/step | 5.3 samples/s +Step 4270/44000 | Loss: 3.3362 | LR: 4.92e-04 | 2042.5ms/step | 9.8 samples/s +Step 4280/44000 | Loss: 3.3053 | LR: 4.92e-04 | 2070.1ms/step | 9.7 samples/s +Step 4290/44000 | Loss: 3.2718 | LR: 4.92e-04 | 2040.9ms/step | 9.8 samples/s +Step 4300/44000 | Loss: 3.3463 | LR: 4.92e-04 | 2099.8ms/step | 9.5 samples/s +Step 4310/44000 | Loss: 3.3089 | LR: 4.92e-04 | 2017.3ms/step | 9.9 samples/s +Step 4320/44000 | Loss: 3.3569 | LR: 4.92e-04 | 2317.5ms/step | 8.6 samples/s +Step 4330/44000 | Loss: 3.3394 | LR: 4.92e-04 | 2315.5ms/step | 8.6 samples/s +Step 4340/44000 | Loss: 3.3367 | LR: 4.92e-04 | 1985.4ms/step | 10.1 samples/s +Step 4350/44000 | Loss: 3.3430 | LR: 4.92e-04 | 1994.4ms/step | 10.0 samples/s +Step 4360/44000 | Loss: 3.3512 | LR: 4.91e-04 | 1991.6ms/step | 10.0 samples/s +Step 4370/44000 | Loss: 3.2946 | LR: 4.91e-04 | 2011.9ms/step | 9.9 samples/s +Step 4380/44000 | Loss: 3.3440 | LR: 4.91e-04 | 2016.6ms/step | 9.9 samples/s +Step 4390/44000 | Loss: 3.3662 | LR: 4.91e-04 | 2006.0ms/step | 10.0 samples/s +Step 4400/44000 | Loss: 3.3278 | LR: 4.91e-04 | 2011.3ms/step | 9.9 samples/s +Step 4410/44000 | Loss: 3.3563 | LR: 4.91e-04 | 2027.1ms/step | 9.9 samples/s +Step 4420/44000 | Loss: 3.3193 | LR: 4.91e-04 | 2027.2ms/step | 9.9 samples/s +Step 4430/44000 | Loss: 3.2660 | LR: 4.91e-04 | 2014.8ms/step | 9.9 samples/s +Step 4440/44000 | Loss: 3.3294 | LR: 4.91e-04 | 2022.7ms/step | 9.9 samples/s +Step 4450/44000 | Loss: 3.3011 | LR: 4.91e-04 | 2027.6ms/step | 9.9 samples/s +Step 4460/44000 | Loss: 3.3123 | LR: 4.91e-04 | 2016.8ms/step | 9.9 samples/s +Step 4470/44000 | Loss: 3.3413 | LR: 4.91e-04 | 2012.0ms/step | 9.9 samples/s +Step 4480/44000 | Loss: 3.3891 | LR: 4.91e-04 | 2016.6ms/step | 9.9 samples/s +Step 4490/44000 | Loss: 3.3216 | LR: 4.91e-04 | 2017.1ms/step | 9.9 samples/s +Step 4500/44000 | Loss: 3.3128 | LR: 4.91e-04 | 2026.9ms/step | 9.9 samples/s +Validation at step 4500: Loss = 3.3450, Perplexity = 28.36 +Step 4510/44000 | Loss: 3.3508 | LR: 4.91e-04 | 3714.7ms/step | 5.4 samples/s +Step 4520/44000 | Loss: 3.2778 | LR: 4.91e-04 | 2048.1ms/step | 9.8 samples/s +Step 4530/44000 | Loss: 3.3570 | LR: 4.91e-04 | 2078.8ms/step | 9.6 samples/s +Step 4540/44000 | Loss: 3.3444 | LR: 4.91e-04 | 2048.1ms/step | 9.8 samples/s +Step 4550/44000 | Loss: 3.3065 | LR: 4.91e-04 | 2032.4ms/step | 9.8 samples/s +Step 4560/44000 | Loss: 3.3567 | LR: 4.90e-04 | 2063.0ms/step | 9.7 samples/s +Step 4570/44000 | Loss: 3.3338 | LR: 4.90e-04 | 2053.1ms/step | 9.7 samples/s +Step 4580/44000 | Loss: 3.3513 | LR: 4.90e-04 | 2052.5ms/step | 9.7 samples/s +Step 4590/44000 | Loss: 3.3460 | LR: 4.90e-04 | 2052.9ms/step | 9.7 samples/s +Step 4600/44000 | Loss: 3.3627 | LR: 4.90e-04 | 2047.2ms/step | 9.8 samples/s +Step 4610/44000 | Loss: 3.3497 | LR: 4.90e-04 | 2001.3ms/step | 10.0 samples/s +Step 4620/44000 | Loss: 3.2983 | LR: 4.90e-04 | 1990.1ms/step | 10.0 samples/s +Step 4630/44000 | Loss: 3.3188 | LR: 4.90e-04 | 2010.8ms/step | 9.9 samples/s +Step 4640/44000 | Loss: 3.3138 | LR: 4.90e-04 | 2021.4ms/step | 9.9 samples/s +Step 4650/44000 | Loss: 3.3247 | LR: 4.90e-04 | 2021.0ms/step | 9.9 samples/s +Step 4660/44000 | Loss: 3.3679 | LR: 4.90e-04 | 2023.2ms/step | 9.9 samples/s +Step 4670/44000 | Loss: 3.3177 | LR: 4.90e-04 | 2022.7ms/step | 9.9 samples/s +Step 4680/44000 | Loss: 3.2877 | LR: 4.90e-04 | 2022.2ms/step | 9.9 samples/s +Step 4690/44000 | Loss: 3.2963 | LR: 4.90e-04 | 2019.7ms/step | 9.9 samples/s +Step 4700/44000 | Loss: 3.3510 | LR: 4.90e-04 | 2032.8ms/step | 9.8 samples/s +Step 4710/44000 | Loss: 3.3020 | LR: 4.90e-04 | 2013.4ms/step | 9.9 samples/s +Step 4720/44000 | Loss: 3.3357 | LR: 4.90e-04 | 2026.0ms/step | 9.9 samples/s +Step 4730/44000 | Loss: 3.3001 | LR: 4.90e-04 | 2030.4ms/step | 9.9 samples/s +Step 4740/44000 | Loss: 3.3047 | LR: 4.90e-04 | 2034.9ms/step | 9.8 samples/s +Step 4750/44000 | Loss: 3.2851 | LR: 4.90e-04 | 2005.5ms/step | 10.0 samples/s +Validation at step 4750: Loss = 3.3282, Perplexity = 27.89 +Step 4760/44000 | Loss: 3.3102 | LR: 4.89e-04 | 3707.8ms/step | 5.4 samples/s +Step 4770/44000 | Loss: 3.3156 | LR: 4.89e-04 | 2022.0ms/step | 9.9 samples/s +Step 4780/44000 | Loss: 3.3194 | LR: 4.89e-04 | 2033.0ms/step | 9.8 samples/s +Step 4790/44000 | Loss: 3.2912 | LR: 4.89e-04 | 2037.2ms/step | 9.8 samples/s +Step 4800/44000 | Loss: 3.3105 | LR: 4.89e-04 | 2032.9ms/step | 9.8 samples/s +Step 4810/44000 | Loss: 3.2827 | LR: 4.89e-04 | 2011.5ms/step | 9.9 samples/s +Step 4820/44000 | Loss: 3.3222 | LR: 4.89e-04 | 2011.7ms/step | 9.9 samples/s +Step 4830/44000 | Loss: 3.3331 | LR: 4.89e-04 | 2006.0ms/step | 10.0 samples/s +Step 4840/44000 | Loss: 3.2809 | LR: 4.89e-04 | 2022.5ms/step | 9.9 samples/s +Step 4850/44000 | Loss: 3.2731 | LR: 4.89e-04 | 2026.7ms/step | 9.9 samples/s +Step 4860/44000 | Loss: 3.2948 | LR: 4.89e-04 | 2037.5ms/step | 9.8 samples/s +Step 4870/44000 | Loss: 3.3156 | LR: 4.89e-04 | 2005.5ms/step | 10.0 samples/s +Step 4880/44000 | Loss: 3.2967 | LR: 4.89e-04 | 2022.0ms/step | 9.9 samples/s +Step 4890/44000 | Loss: 3.3046 | LR: 4.89e-04 | 2042.4ms/step | 9.8 samples/s +Step 4900/44000 | Loss: 3.2813 | LR: 4.89e-04 | 2021.6ms/step | 9.9 samples/s +Step 4910/44000 | Loss: 3.3084 | LR: 4.89e-04 | 2011.3ms/step | 9.9 samples/s +Step 4920/44000 | Loss: 3.2989 | LR: 4.89e-04 | 1996.3ms/step | 10.0 samples/s +Step 4930/44000 | Loss: 3.2615 | LR: 4.89e-04 | 1981.6ms/step | 10.1 samples/s +Step 4940/44000 | Loss: 3.2869 | LR: 4.89e-04 | 2017.2ms/step | 9.9 samples/s +Step 4950/44000 | Loss: 3.3365 | LR: 4.88e-04 | 2006.5ms/step | 10.0 samples/s +Step 4960/44000 | Loss: 3.3100 | LR: 4.88e-04 | 2011.7ms/step | 9.9 samples/s +Step 4970/44000 | Loss: 3.2935 | LR: 4.88e-04 | 2002.0ms/step | 10.0 samples/s +Step 4980/44000 | Loss: 3.3417 | LR: 4.88e-04 | 2005.7ms/step | 10.0 samples/s +Step 4990/44000 | Loss: 3.2407 | LR: 4.88e-04 | 2010.4ms/step | 9.9 samples/s +Step 5000/44000 | Loss: 3.2907 | LR: 4.88e-04 | 1992.9ms/step | 10.0 samples/s +Validation at step 5000: Loss = 3.3138, Perplexity = 27.49 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-5000 +Step 5010/44000 | Loss: 3.3203 | LR: 4.88e-04 | 4064.9ms/step | 4.9 samples/s +Step 5020/44000 | Loss: 3.3111 | LR: 4.88e-04 | 1900.8ms/step | 10.5 samples/s +Step 5030/44000 | Loss: 3.2642 | LR: 4.88e-04 | 1901.6ms/step | 10.5 samples/s +Step 5040/44000 | Loss: 3.2395 | LR: 4.88e-04 | 2021.6ms/step | 9.9 samples/s +Step 5050/44000 | Loss: 3.2711 | LR: 4.88e-04 | 2038.6ms/step | 9.8 samples/s +Step 5060/44000 | Loss: 3.2604 | LR: 4.88e-04 | 2048.3ms/step | 9.8 samples/s +Step 5070/44000 | Loss: 3.2845 | LR: 4.88e-04 | 2074.9ms/step | 9.6 samples/s +Step 5080/44000 | Loss: 3.2868 | LR: 4.88e-04 | 2057.7ms/step | 9.7 samples/s +Step 5090/44000 | Loss: 3.3310 | LR: 4.88e-04 | 2042.2ms/step | 9.8 samples/s +Step 5100/44000 | Loss: 3.3018 | LR: 4.88e-04 | 2054.1ms/step | 9.7 samples/s +Step 5110/44000 | Loss: 3.2549 | LR: 4.88e-04 | 2058.2ms/step | 9.7 samples/s +Step 5120/44000 | Loss: 3.2882 | LR: 4.88e-04 | 2032.4ms/step | 9.8 samples/s +Step 5130/44000 | Loss: 3.2220 | LR: 4.87e-04 | 2019.2ms/step | 9.9 samples/s +Step 5140/44000 | Loss: 3.3103 | LR: 4.87e-04 | 2005.1ms/step | 10.0 samples/s +Step 5150/44000 | Loss: 3.2542 | LR: 4.87e-04 | 2000.6ms/step | 10.0 samples/s +Step 5160/44000 | Loss: 3.2789 | LR: 4.87e-04 | 2017.4ms/step | 9.9 samples/s +Step 5170/44000 | Loss: 3.2924 | LR: 4.87e-04 | 2017.4ms/step | 9.9 samples/s +Step 5180/44000 | Loss: 3.3091 | LR: 4.87e-04 | 2016.8ms/step | 9.9 samples/s +Step 5190/44000 | Loss: 3.2921 | LR: 4.87e-04 | 2016.6ms/step | 9.9 samples/s +Step 5200/44000 | Loss: 3.2574 | LR: 4.87e-04 | 2006.8ms/step | 10.0 samples/s +Step 5210/44000 | Loss: 3.2987 | LR: 4.87e-04 | 1997.2ms/step | 10.0 samples/s +Step 5220/44000 | Loss: 3.2749 | LR: 4.87e-04 | 2000.7ms/step | 10.0 samples/s +Step 5230/44000 | Loss: 3.2449 | LR: 4.87e-04 | 1961.3ms/step | 10.2 samples/s +Step 5240/44000 | Loss: 3.2545 | LR: 4.87e-04 | 1932.8ms/step | 10.3 samples/s +Step 5250/44000 | Loss: 3.2843 | LR: 4.87e-04 | 2019.0ms/step | 9.9 samples/s +Validation at step 5250: Loss = 3.3012, Perplexity = 27.15 +Step 5260/44000 | Loss: 3.2869 | LR: 4.87e-04 | 3717.2ms/step | 5.4 samples/s +Step 5270/44000 | Loss: 3.2644 | LR: 4.87e-04 | 2021.5ms/step | 9.9 samples/s +Step 5280/44000 | Loss: 3.3001 | LR: 4.87e-04 | 2013.2ms/step | 9.9 samples/s +Step 5290/44000 | Loss: 3.2867 | LR: 4.87e-04 | 2022.4ms/step | 9.9 samples/s +Step 5300/44000 | Loss: 3.2883 | LR: 4.86e-04 | 2032.4ms/step | 9.8 samples/s +Step 5310/44000 | Loss: 3.2827 | LR: 4.86e-04 | 2023.0ms/step | 9.9 samples/s +Step 5320/44000 | Loss: 3.2861 | LR: 4.86e-04 | 2022.7ms/step | 9.9 samples/s +Step 5330/44000 | Loss: 3.2984 | LR: 4.86e-04 | 2012.6ms/step | 9.9 samples/s +Step 5340/44000 | Loss: 3.2789 | LR: 4.86e-04 | 2022.2ms/step | 9.9 samples/s +Step 5350/44000 | Loss: 3.2736 | LR: 4.86e-04 | 2018.0ms/step | 9.9 samples/s +Step 5360/44000 | Loss: 3.2952 | LR: 4.86e-04 | 2038.2ms/step | 9.8 samples/s +Step 5370/44000 | Loss: 3.2712 | LR: 4.86e-04 | 2027.3ms/step | 9.9 samples/s +Step 5380/44000 | Loss: 3.2724 | LR: 4.86e-04 | 2012.6ms/step | 9.9 samples/s +Step 5390/44000 | Loss: 3.2505 | LR: 4.86e-04 | 2027.6ms/step | 9.9 samples/s +Step 5400/44000 | Loss: 3.2805 | LR: 4.86e-04 | 2012.3ms/step | 9.9 samples/s +Step 5410/44000 | Loss: 3.2970 | LR: 4.86e-04 | 2027.0ms/step | 9.9 samples/s +Step 5420/44000 | Loss: 3.2734 | LR: 4.86e-04 | 2049.2ms/step | 9.8 samples/s +Step 5430/44000 | Loss: 3.2813 | LR: 4.86e-04 | 2059.2ms/step | 9.7 samples/s +Step 5440/44000 | Loss: 3.3116 | LR: 4.86e-04 | 2038.1ms/step | 9.8 samples/s +Step 5450/44000 | Loss: 3.2561 | LR: 4.86e-04 | 2074.9ms/step | 9.6 samples/s +Step 5460/44000 | Loss: 3.2925 | LR: 4.86e-04 | 2059.5ms/step | 9.7 samples/s +Step 5470/44000 | Loss: 3.2646 | LR: 4.85e-04 | 2060.5ms/step | 9.7 samples/s +Step 5480/44000 | Loss: 3.2469 | LR: 4.85e-04 | 2037.3ms/step | 9.8 samples/s +Step 5490/44000 | Loss: 3.2652 | LR: 4.85e-04 | 2043.2ms/step | 9.8 samples/s +Step 5500/44000 | Loss: 3.2574 | LR: 4.85e-04 | 2142.7ms/step | 9.3 samples/s +Validation at step 5500: Loss = 3.2855, Perplexity = 26.72 +Step 5510/44000 | Loss: 3.2590 | LR: 4.85e-04 | 3926.8ms/step | 5.1 samples/s +Step 5520/44000 | Loss: 3.2962 | LR: 4.85e-04 | 2391.4ms/step | 8.4 samples/s +Step 5530/44000 | Loss: 3.2694 | LR: 4.85e-04 | 2032.0ms/step | 9.8 samples/s +Step 5540/44000 | Loss: 3.2984 | LR: 4.85e-04 | 1965.2ms/step | 10.2 samples/s +Step 5550/44000 | Loss: 3.3088 | LR: 4.85e-04 | 1989.6ms/step | 10.1 samples/s +Step 5560/44000 | Loss: 3.2415 | LR: 4.85e-04 | 1993.3ms/step | 10.0 samples/s +Step 5570/44000 | Loss: 3.2859 | LR: 4.85e-04 | 2016.8ms/step | 9.9 samples/s +Step 5580/44000 | Loss: 3.2587 | LR: 4.85e-04 | 2017.1ms/step | 9.9 samples/s +Step 5590/44000 | Loss: 3.2648 | LR: 4.85e-04 | 2017.2ms/step | 9.9 samples/s +Step 5600/44000 | Loss: 3.2887 | LR: 4.85e-04 | 2027.8ms/step | 9.9 samples/s +Step 5610/44000 | Loss: 3.2992 | LR: 4.85e-04 | 2022.0ms/step | 9.9 samples/s +Step 5620/44000 | Loss: 3.2572 | LR: 4.85e-04 | 2020.1ms/step | 9.9 samples/s +Step 5630/44000 | Loss: 3.2463 | LR: 4.84e-04 | 2010.4ms/step | 9.9 samples/s +Step 5640/44000 | Loss: 3.2453 | LR: 4.84e-04 | 2015.7ms/step | 9.9 samples/s +Step 5650/44000 | Loss: 3.2918 | LR: 4.84e-04 | 2022.1ms/step | 9.9 samples/s +Step 5660/44000 | Loss: 3.2470 | LR: 4.84e-04 | 2007.3ms/step | 10.0 samples/s +Step 5670/44000 | Loss: 3.2708 | LR: 4.84e-04 | 2027.7ms/step | 9.9 samples/s +Step 5680/44000 | Loss: 3.2848 | LR: 4.84e-04 | 2000.6ms/step | 10.0 samples/s +Step 5690/44000 | Loss: 3.2760 | LR: 4.84e-04 | 2002.2ms/step | 10.0 samples/s +Step 5700/44000 | Loss: 3.2503 | LR: 4.84e-04 | 2012.1ms/step | 9.9 samples/s +Step 5710/44000 | Loss: 3.2335 | LR: 4.84e-04 | 2022.1ms/step | 9.9 samples/s +Step 5720/44000 | Loss: 3.2409 | LR: 4.84e-04 | 2017.4ms/step | 9.9 samples/s +Step 5730/44000 | Loss: 3.2166 | LR: 4.84e-04 | 2011.4ms/step | 9.9 samples/s +Step 5740/44000 | Loss: 3.2454 | LR: 4.84e-04 | 2011.7ms/step | 9.9 samples/s +Step 5750/44000 | Loss: 3.2477 | LR: 4.84e-04 | 2068.5ms/step | 9.7 samples/s +Validation at step 5750: Loss = 3.2760, Perplexity = 26.47 +Step 5760/44000 | Loss: 3.2822 | LR: 4.84e-04 | 3799.7ms/step | 5.3 samples/s +Step 5770/44000 | Loss: 3.2470 | LR: 4.84e-04 | 2032.1ms/step | 9.8 samples/s +Step 5780/44000 | Loss: 3.2510 | LR: 4.83e-04 | 2037.7ms/step | 9.8 samples/s +Step 5790/44000 | Loss: 3.2265 | LR: 4.83e-04 | 2144.3ms/step | 9.3 samples/s +Step 5800/44000 | Loss: 3.2797 | LR: 4.83e-04 | 2155.3ms/step | 9.3 samples/s +Step 5810/44000 | Loss: 3.2271 | LR: 4.83e-04 | 2429.2ms/step | 8.2 samples/s +Step 5820/44000 | Loss: 3.2665 | LR: 4.83e-04 | 2105.0ms/step | 9.5 samples/s +Step 5830/44000 | Loss: 3.2776 | LR: 4.83e-04 | 2032.0ms/step | 9.8 samples/s +Step 5840/44000 | Loss: 3.2281 | LR: 4.83e-04 | 1995.4ms/step | 10.0 samples/s +Step 5850/44000 | Loss: 3.2840 | LR: 4.83e-04 | 1986.6ms/step | 10.1 samples/s +Step 5860/44000 | Loss: 3.3038 | LR: 4.83e-04 | 2047.6ms/step | 9.8 samples/s +Step 5870/44000 | Loss: 3.2562 | LR: 4.83e-04 | 2053.8ms/step | 9.7 samples/s +Step 5880/44000 | Loss: 3.2338 | LR: 4.83e-04 | 2037.9ms/step | 9.8 samples/s +Step 5890/44000 | Loss: 3.2199 | LR: 4.83e-04 | 2024.9ms/step | 9.9 samples/s +Step 5900/44000 | Loss: 3.2429 | LR: 4.83e-04 | 2023.6ms/step | 9.9 samples/s +Step 5910/44000 | Loss: 3.2415 | LR: 4.83e-04 | 2037.4ms/step | 9.8 samples/s +Step 5920/44000 | Loss: 3.2241 | LR: 4.83e-04 | 2016.3ms/step | 9.9 samples/s +Step 5930/44000 | Loss: 3.2754 | LR: 4.83e-04 | 2018.5ms/step | 9.9 samples/s +Step 5940/44000 | Loss: 3.2739 | LR: 4.82e-04 | 2025.6ms/step | 9.9 samples/s +Step 5950/44000 | Loss: 3.2451 | LR: 4.82e-04 | 2042.1ms/step | 9.8 samples/s +Step 5960/44000 | Loss: 3.2510 | LR: 4.82e-04 | 2022.4ms/step | 9.9 samples/s +Step 5970/44000 | Loss: 3.2260 | LR: 4.82e-04 | 2007.0ms/step | 10.0 samples/s +Step 5980/44000 | Loss: 3.2608 | LR: 4.82e-04 | 2031.0ms/step | 9.8 samples/s +Step 5990/44000 | Loss: 3.2287 | LR: 4.82e-04 | 2022.3ms/step | 9.9 samples/s +Step 6000/44000 | Loss: 3.2443 | LR: 4.82e-04 | 2021.9ms/step | 9.9 samples/s +Validation at step 6000: Loss = 3.2625, Perplexity = 26.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-6000 +Step 6010/44000 | Loss: 3.1962 | LR: 4.82e-04 | 4314.6ms/step | 4.6 samples/s +Step 6020/44000 | Loss: 3.2573 | LR: 4.82e-04 | 2021.6ms/step | 9.9 samples/s +Step 6030/44000 | Loss: 3.2616 | LR: 4.82e-04 | 2049.4ms/step | 9.8 samples/s +Step 6040/44000 | Loss: 3.2050 | LR: 4.82e-04 | 2046.6ms/step | 9.8 samples/s +Step 6050/44000 | Loss: 3.2568 | LR: 4.82e-04 | 2147.7ms/step | 9.3 samples/s +Step 6060/44000 | Loss: 3.2517 | LR: 4.82e-04 | 2084.2ms/step | 9.6 samples/s +Step 6070/44000 | Loss: 3.1810 | LR: 4.82e-04 | 2255.4ms/step | 8.9 samples/s +Step 6080/44000 | Loss: 3.2799 | LR: 4.81e-04 | 2267.8ms/step | 8.8 samples/s +Step 6090/44000 | Loss: 3.2230 | LR: 4.81e-04 | 2000.9ms/step | 10.0 samples/s +Step 6100/44000 | Loss: 3.2144 | LR: 4.81e-04 | 1979.6ms/step | 10.1 samples/s +Step 6110/44000 | Loss: 3.2628 | LR: 4.81e-04 | 1985.9ms/step | 10.1 samples/s +Step 6120/44000 | Loss: 3.1904 | LR: 4.81e-04 | 2021.5ms/step | 9.9 samples/s +Step 6130/44000 | Loss: 3.2057 | LR: 4.81e-04 | 2026.7ms/step | 9.9 samples/s +Step 6140/44000 | Loss: 3.2368 | LR: 4.81e-04 | 2011.1ms/step | 9.9 samples/s +Step 6150/44000 | Loss: 3.2385 | LR: 4.81e-04 | 1979.5ms/step | 10.1 samples/s +Step 6160/44000 | Loss: 3.2119 | LR: 4.81e-04 | 1975.7ms/step | 10.1 samples/s +Step 6170/44000 | Loss: 3.2230 | LR: 4.81e-04 | 2021.5ms/step | 9.9 samples/s +Step 6180/44000 | Loss: 3.1866 | LR: 4.81e-04 | 1995.9ms/step | 10.0 samples/s +Step 6190/44000 | Loss: 3.2228 | LR: 4.81e-04 | 2006.2ms/step | 10.0 samples/s +Step 6200/44000 | Loss: 3.2761 | LR: 4.81e-04 | 2027.2ms/step | 9.9 samples/s +Step 6210/44000 | Loss: 3.2202 | LR: 4.81e-04 | 2034.2ms/step | 9.8 samples/s +Step 6220/44000 | Loss: 3.2752 | LR: 4.81e-04 | 2009.7ms/step | 10.0 samples/s +Step 6230/44000 | Loss: 3.1993 | LR: 4.80e-04 | 2011.3ms/step | 9.9 samples/s +Step 6240/44000 | Loss: 3.2068 | LR: 4.80e-04 | 2012.1ms/step | 9.9 samples/s +Step 6250/44000 | Loss: 3.2079 | LR: 4.80e-04 | 2006.3ms/step | 10.0 samples/s +Validation at step 6250: Loss = 3.2510, Perplexity = 25.82 +Step 6260/44000 | Loss: 3.1820 | LR: 4.80e-04 | 3720.3ms/step | 5.4 samples/s +Step 6270/44000 | Loss: 3.2402 | LR: 4.80e-04 | 2016.8ms/step | 9.9 samples/s +Step 6280/44000 | Loss: 3.2190 | LR: 4.80e-04 | 2016.0ms/step | 9.9 samples/s +Step 6290/44000 | Loss: 3.1970 | LR: 4.80e-04 | 2022.7ms/step | 9.9 samples/s +Step 6300/44000 | Loss: 3.2118 | LR: 4.80e-04 | 2026.6ms/step | 9.9 samples/s +Step 6310/44000 | Loss: 3.2071 | LR: 4.80e-04 | 2016.8ms/step | 9.9 samples/s +Step 6320/44000 | Loss: 3.2263 | LR: 4.80e-04 | 2146.6ms/step | 9.3 samples/s +Step 6330/44000 | Loss: 3.2700 | LR: 4.80e-04 | 2010.8ms/step | 9.9 samples/s +Step 6340/44000 | Loss: 3.2596 | LR: 4.80e-04 | 2304.0ms/step | 8.7 samples/s +Step 6350/44000 | Loss: 3.2140 | LR: 4.80e-04 | 2224.2ms/step | 9.0 samples/s +Step 6360/44000 | Loss: 3.2239 | LR: 4.80e-04 | 2016.2ms/step | 9.9 samples/s +Step 6370/44000 | Loss: 3.2186 | LR: 4.79e-04 | 2017.3ms/step | 9.9 samples/s +Step 6380/44000 | Loss: 3.2537 | LR: 4.79e-04 | 2016.5ms/step | 9.9 samples/s +Step 6390/44000 | Loss: 3.2474 | LR: 4.79e-04 | 2027.8ms/step | 9.9 samples/s +Step 6400/44000 | Loss: 3.1972 | LR: 4.79e-04 | 2046.4ms/step | 9.8 samples/s +Step 6410/44000 | Loss: 3.2326 | LR: 4.79e-04 | 2053.4ms/step | 9.7 samples/s +Step 6420/44000 | Loss: 3.2260 | LR: 4.79e-04 | 2048.2ms/step | 9.8 samples/s +Step 6430/44000 | Loss: 3.2232 | LR: 4.79e-04 | 2068.3ms/step | 9.7 samples/s +Step 6440/44000 | Loss: 3.1952 | LR: 4.79e-04 | 2053.0ms/step | 9.7 samples/s +Step 6450/44000 | Loss: 3.2524 | LR: 4.79e-04 | 2032.1ms/step | 9.8 samples/s +Step 6460/44000 | Loss: 3.2448 | LR: 4.79e-04 | 2015.3ms/step | 9.9 samples/s +Step 6470/44000 | Loss: 3.2393 | LR: 4.79e-04 | 1958.7ms/step | 10.2 samples/s +Step 6480/44000 | Loss: 3.2195 | LR: 4.79e-04 | 1905.0ms/step | 10.5 samples/s +Step 6490/44000 | Loss: 3.2020 | LR: 4.79e-04 | 1905.4ms/step | 10.5 samples/s +Step 6500/44000 | Loss: 3.2622 | LR: 4.79e-04 | 1989.1ms/step | 10.1 samples/s +Validation at step 6500: Loss = 3.2395, Perplexity = 25.52 +Step 6510/44000 | Loss: 3.2500 | LR: 4.78e-04 | 3710.0ms/step | 5.4 samples/s +Step 6520/44000 | Loss: 3.2247 | LR: 4.78e-04 | 2009.2ms/step | 10.0 samples/s +Step 6530/44000 | Loss: 3.2037 | LR: 4.78e-04 | 2012.6ms/step | 9.9 samples/s +Step 6540/44000 | Loss: 3.1765 | LR: 4.78e-04 | 2137.0ms/step | 9.4 samples/s +Step 6550/44000 | Loss: 3.2148 | LR: 4.78e-04 | 2058.1ms/step | 9.7 samples/s +Step 6560/44000 | Loss: 3.2270 | LR: 4.78e-04 | 2344.8ms/step | 8.5 samples/s +Step 6570/44000 | Loss: 3.2068 | LR: 4.78e-04 | 2213.7ms/step | 9.0 samples/s +Step 6580/44000 | Loss: 3.2518 | LR: 4.78e-04 | 2000.7ms/step | 10.0 samples/s +Step 6590/44000 | Loss: 3.1956 | LR: 4.78e-04 | 2000.1ms/step | 10.0 samples/s +Step 6600/44000 | Loss: 3.1986 | LR: 4.78e-04 | 1990.1ms/step | 10.0 samples/s +Step 6610/44000 | Loss: 3.2065 | LR: 4.78e-04 | 2031.5ms/step | 9.8 samples/s +Step 6620/44000 | Loss: 3.1985 | LR: 4.78e-04 | 2021.6ms/step | 9.9 samples/s +Step 6630/44000 | Loss: 3.2298 | LR: 4.78e-04 | 2011.1ms/step | 9.9 samples/s +Step 6640/44000 | Loss: 3.2663 | LR: 4.77e-04 | 2015.9ms/step | 9.9 samples/s +Step 6650/44000 | Loss: 3.2109 | LR: 4.77e-04 | 2017.2ms/step | 9.9 samples/s +Step 6660/44000 | Loss: 3.2132 | LR: 4.77e-04 | 2026.9ms/step | 9.9 samples/s +Step 6670/44000 | Loss: 3.1933 | LR: 4.77e-04 | 2037.1ms/step | 9.8 samples/s +Step 6680/44000 | Loss: 3.2391 | LR: 4.77e-04 | 2015.7ms/step | 9.9 samples/s +Step 6690/44000 | Loss: 3.2108 | LR: 4.77e-04 | 2022.3ms/step | 9.9 samples/s +Step 6700/44000 | Loss: 3.2507 | LR: 4.77e-04 | 2027.5ms/step | 9.9 samples/s +Step 6710/44000 | Loss: 3.1698 | LR: 4.77e-04 | 2041.3ms/step | 9.8 samples/s +Step 6720/44000 | Loss: 3.1619 | LR: 4.77e-04 | 2022.1ms/step | 9.9 samples/s +Step 6730/44000 | Loss: 3.2040 | LR: 4.77e-04 | 2026.8ms/step | 9.9 samples/s +Step 6740/44000 | Loss: 3.2526 | LR: 4.77e-04 | 2026.4ms/step | 9.9 samples/s +Step 6750/44000 | Loss: 3.2356 | LR: 4.77e-04 | 2000.6ms/step | 10.0 samples/s +Validation at step 6750: Loss = 3.2297, Perplexity = 25.27 +Step 6760/44000 | Loss: 3.1898 | LR: 4.77e-04 | 3683.7ms/step | 5.4 samples/s +Step 6770/44000 | Loss: 3.2392 | LR: 4.76e-04 | 1984.6ms/step | 10.1 samples/s +Step 6780/44000 | Loss: 3.2157 | LR: 4.76e-04 | 1984.7ms/step | 10.1 samples/s +Step 6790/44000 | Loss: 3.1957 | LR: 4.76e-04 | 2000.4ms/step | 10.0 samples/s +Step 6800/44000 | Loss: 3.1929 | LR: 4.76e-04 | 2007.4ms/step | 10.0 samples/s +Step 6810/44000 | Loss: 3.2237 | LR: 4.76e-04 | 2005.7ms/step | 10.0 samples/s +Step 6820/44000 | Loss: 3.1980 | LR: 4.76e-04 | 2085.0ms/step | 9.6 samples/s +Step 6830/44000 | Loss: 3.2309 | LR: 4.76e-04 | 2047.4ms/step | 9.8 samples/s +Step 6840/44000 | Loss: 3.1880 | LR: 4.76e-04 | 2301.1ms/step | 8.7 samples/s +Step 6850/44000 | Loss: 3.1836 | LR: 4.76e-04 | 2148.0ms/step | 9.3 samples/s +Step 6860/44000 | Loss: 3.2171 | LR: 4.76e-04 | 1990.1ms/step | 10.0 samples/s +Step 6870/44000 | Loss: 3.2084 | LR: 4.76e-04 | 1979.7ms/step | 10.1 samples/s +Step 6880/44000 | Loss: 3.1982 | LR: 4.76e-04 | 1975.5ms/step | 10.1 samples/s +Step 6890/44000 | Loss: 3.2515 | LR: 4.76e-04 | 2000.7ms/step | 10.0 samples/s +Step 6900/44000 | Loss: 3.2095 | LR: 4.75e-04 | 1996.6ms/step | 10.0 samples/s +Step 6910/44000 | Loss: 3.2355 | LR: 4.75e-04 | 2005.6ms/step | 10.0 samples/s +Step 6920/44000 | Loss: 3.2039 | LR: 4.75e-04 | 1995.5ms/step | 10.0 samples/s +Step 6930/44000 | Loss: 3.1726 | LR: 4.75e-04 | 1996.2ms/step | 10.0 samples/s +Step 6940/44000 | Loss: 3.2057 | LR: 4.75e-04 | 2000.0ms/step | 10.0 samples/s +Step 6950/44000 | Loss: 3.2093 | LR: 4.75e-04 | 1992.5ms/step | 10.0 samples/s +Step 6960/44000 | Loss: 3.2468 | LR: 4.75e-04 | 1989.1ms/step | 10.1 samples/s +Step 6970/44000 | Loss: 3.2238 | LR: 4.75e-04 | 2000.0ms/step | 10.0 samples/s +Step 6980/44000 | Loss: 3.1946 | LR: 4.75e-04 | 1980.3ms/step | 10.1 samples/s +Step 6990/44000 | Loss: 3.1723 | LR: 4.75e-04 | 1996.3ms/step | 10.0 samples/s +Step 7000/44000 | Loss: 3.1953 | LR: 4.75e-04 | 1985.7ms/step | 10.1 samples/s +Validation at step 7000: Loss = 3.2214, Perplexity = 25.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-7000 +Step 7010/44000 | Loss: 3.2159 | LR: 4.75e-04 | 4052.3ms/step | 4.9 samples/s +Step 7020/44000 | Loss: 3.1754 | LR: 4.75e-04 | 2023.8ms/step | 9.9 samples/s +Step 7030/44000 | Loss: 3.1866 | LR: 4.74e-04 | 2033.1ms/step | 9.8 samples/s +Step 7040/44000 | Loss: 3.2206 | LR: 4.74e-04 | 2333.4ms/step | 8.6 samples/s +Step 7050/44000 | Loss: 3.2191 | LR: 4.74e-04 | 2120.5ms/step | 9.4 samples/s +Step 7060/44000 | Loss: 3.2634 | LR: 4.74e-04 | 1958.8ms/step | 10.2 samples/s +Step 7070/44000 | Loss: 3.1727 | LR: 4.74e-04 | 1968.8ms/step | 10.2 samples/s +Step 7080/44000 | Loss: 3.1813 | LR: 4.74e-04 | 1974.3ms/step | 10.1 samples/s +Step 7090/44000 | Loss: 3.2282 | LR: 4.74e-04 | 1990.6ms/step | 10.0 samples/s +Step 7100/44000 | Loss: 3.2260 | LR: 4.74e-04 | 1990.3ms/step | 10.0 samples/s +Step 7110/44000 | Loss: 3.2353 | LR: 4.74e-04 | 1997.2ms/step | 10.0 samples/s +Step 7120/44000 | Loss: 3.2054 | LR: 4.74e-04 | 1997.3ms/step | 10.0 samples/s +Step 7130/44000 | Loss: 3.2105 | LR: 4.74e-04 | 2003.7ms/step | 10.0 samples/s +Step 7140/44000 | Loss: 3.1849 | LR: 4.74e-04 | 2004.7ms/step | 10.0 samples/s +Step 7150/44000 | Loss: 3.2086 | LR: 4.73e-04 | 2005.0ms/step | 10.0 samples/s +Step 7160/44000 | Loss: 3.1725 | LR: 4.73e-04 | 1996.8ms/step | 10.0 samples/s +Step 7170/44000 | Loss: 3.2019 | LR: 4.73e-04 | 2010.5ms/step | 9.9 samples/s +Step 7180/44000 | Loss: 3.1838 | LR: 4.73e-04 | 1994.6ms/step | 10.0 samples/s +Step 7190/44000 | Loss: 3.2006 | LR: 4.73e-04 | 1993.1ms/step | 10.0 samples/s +Step 7200/44000 | Loss: 3.1940 | LR: 4.73e-04 | 2010.1ms/step | 9.9 samples/s +Step 7210/44000 | Loss: 3.2264 | LR: 4.73e-04 | 2004.5ms/step | 10.0 samples/s +Step 7220/44000 | Loss: 3.2240 | LR: 4.73e-04 | 2004.7ms/step | 10.0 samples/s +Step 7230/44000 | Loss: 3.2439 | LR: 4.73e-04 | 2004.4ms/step | 10.0 samples/s +Step 7240/44000 | Loss: 3.1807 | LR: 4.73e-04 | 2016.2ms/step | 9.9 samples/s +Step 7250/44000 | Loss: 3.2353 | LR: 4.73e-04 | 2015.0ms/step | 9.9 samples/s +Validation at step 7250: Loss = 3.2121, Perplexity = 24.83 +Step 7260/44000 | Loss: 3.1788 | LR: 4.73e-04 | 3701.7ms/step | 5.4 samples/s +Step 7270/44000 | Loss: 3.1845 | LR: 4.72e-04 | 2006.3ms/step | 10.0 samples/s +Step 7280/44000 | Loss: 3.1717 | LR: 4.72e-04 | 2011.4ms/step | 9.9 samples/s +Step 7290/44000 | Loss: 3.2020 | LR: 4.72e-04 | 2006.6ms/step | 10.0 samples/s +Step 7300/44000 | Loss: 3.2278 | LR: 4.72e-04 | 2005.3ms/step | 10.0 samples/s +Step 7310/44000 | Loss: 3.1648 | LR: 4.72e-04 | 2011.9ms/step | 9.9 samples/s +Step 7320/44000 | Loss: 3.1985 | LR: 4.72e-04 | 2016.7ms/step | 9.9 samples/s +Step 7330/44000 | Loss: 3.1693 | LR: 4.72e-04 | 2006.3ms/step | 10.0 samples/s +Step 7340/44000 | Loss: 3.1985 | LR: 4.72e-04 | 2007.0ms/step | 10.0 samples/s +Step 7350/44000 | Loss: 3.2172 | LR: 4.72e-04 | 2010.7ms/step | 9.9 samples/s +Step 7360/44000 | Loss: 3.1730 | LR: 4.72e-04 | 1992.8ms/step | 10.0 samples/s +Step 7370/44000 | Loss: 3.2116 | LR: 4.72e-04 | 1990.2ms/step | 10.0 samples/s +Step 7380/44000 | Loss: 3.2147 | LR: 4.72e-04 | 2011.9ms/step | 9.9 samples/s +Step 7390/44000 | Loss: 3.1301 | LR: 4.71e-04 | 1996.8ms/step | 10.0 samples/s +Step 7400/44000 | Loss: 3.1329 | LR: 4.71e-04 | 1991.2ms/step | 10.0 samples/s +Step 7410/44000 | Loss: 3.1961 | LR: 4.71e-04 | 2012.5ms/step | 9.9 samples/s +Step 7420/44000 | Loss: 3.1935 | LR: 4.71e-04 | 2002.1ms/step | 10.0 samples/s +Step 7430/44000 | Loss: 3.1886 | LR: 4.71e-04 | 2002.6ms/step | 10.0 samples/s +Step 7440/44000 | Loss: 3.1923 | LR: 4.71e-04 | 2011.1ms/step | 9.9 samples/s +Step 7450/44000 | Loss: 3.2015 | LR: 4.71e-04 | 2022.3ms/step | 9.9 samples/s +Step 7460/44000 | Loss: 3.1311 | LR: 4.71e-04 | 2013.9ms/step | 9.9 samples/s +Step 7470/44000 | Loss: 3.2285 | LR: 4.71e-04 | 2027.1ms/step | 9.9 samples/s +Step 7480/44000 | Loss: 3.1775 | LR: 4.71e-04 | 2021.1ms/step | 9.9 samples/s +Step 7490/44000 | Loss: 3.1914 | LR: 4.71e-04 | 2016.6ms/step | 9.9 samples/s +Step 7500/44000 | Loss: 3.1957 | LR: 4.71e-04 | 2006.8ms/step | 10.0 samples/s +Validation at step 7500: Loss = 3.2025, Perplexity = 24.59 +Step 7510/44000 | Loss: 3.2295 | LR: 4.70e-04 | 3704.6ms/step | 5.4 samples/s +Step 7520/44000 | Loss: 3.2128 | LR: 4.70e-04 | 2006.9ms/step | 10.0 samples/s +Step 7530/44000 | Loss: 3.1989 | LR: 4.70e-04 | 2022.7ms/step | 9.9 samples/s +Step 7540/44000 | Loss: 3.2232 | LR: 4.70e-04 | 2016.5ms/step | 9.9 samples/s +Step 7550/44000 | Loss: 3.1277 | LR: 4.70e-04 | 2006.9ms/step | 10.0 samples/s +Step 7560/44000 | Loss: 3.2105 | LR: 4.70e-04 | 1996.2ms/step | 10.0 samples/s +Step 7570/44000 | Loss: 3.2049 | LR: 4.70e-04 | 2002.0ms/step | 10.0 samples/s +Step 7580/44000 | Loss: 3.1578 | LR: 4.70e-04 | 1997.0ms/step | 10.0 samples/s +Step 7590/44000 | Loss: 3.1516 | LR: 4.70e-04 | 2012.4ms/step | 9.9 samples/s +Step 7600/44000 | Loss: 3.1954 | LR: 4.70e-04 | 2006.0ms/step | 10.0 samples/s +Step 7610/44000 | Loss: 3.1558 | LR: 4.70e-04 | 2005.5ms/step | 10.0 samples/s +Step 7620/44000 | Loss: 3.1967 | LR: 4.70e-04 | 2007.4ms/step | 10.0 samples/s +Step 7630/44000 | Loss: 3.1534 | LR: 4.69e-04 | 2011.6ms/step | 9.9 samples/s +Step 7640/44000 | Loss: 3.1634 | LR: 4.69e-04 | 2000.7ms/step | 10.0 samples/s +Step 7650/44000 | Loss: 3.1939 | LR: 4.69e-04 | 1996.9ms/step | 10.0 samples/s +Step 7660/44000 | Loss: 3.1797 | LR: 4.69e-04 | 2000.6ms/step | 10.0 samples/s +Step 7670/44000 | Loss: 3.1437 | LR: 4.69e-04 | 1941.9ms/step | 10.3 samples/s +Step 7680/44000 | Loss: 3.1701 | LR: 4.69e-04 | 1909.2ms/step | 10.5 samples/s +Step 7690/44000 | Loss: 3.2145 | LR: 4.69e-04 | 1913.2ms/step | 10.5 samples/s +Step 7700/44000 | Loss: 3.1868 | LR: 4.69e-04 | 2009.1ms/step | 10.0 samples/s +Step 7710/44000 | Loss: 3.1668 | LR: 4.69e-04 | 1992.1ms/step | 10.0 samples/s +Step 7720/44000 | Loss: 3.1488 | LR: 4.69e-04 | 2001.6ms/step | 10.0 samples/s +Step 7730/44000 | Loss: 3.1572 | LR: 4.69e-04 | 2005.0ms/step | 10.0 samples/s +Step 7740/44000 | Loss: 3.1836 | LR: 4.68e-04 | 2003.7ms/step | 10.0 samples/s +Step 7750/44000 | Loss: 3.1561 | LR: 4.68e-04 | 2016.1ms/step | 9.9 samples/s +Validation at step 7750: Loss = 3.1949, Perplexity = 24.41 +Step 7760/44000 | Loss: 3.2204 | LR: 4.68e-04 | 3669.4ms/step | 5.5 samples/s +Step 7770/44000 | Loss: 3.1275 | LR: 4.68e-04 | 1996.4ms/step | 10.0 samples/s +Step 7780/44000 | Loss: 3.1695 | LR: 4.68e-04 | 1995.8ms/step | 10.0 samples/s +Step 7790/44000 | Loss: 3.1938 | LR: 4.68e-04 | 2006.8ms/step | 10.0 samples/s +Step 7800/44000 | Loss: 3.1855 | LR: 4.68e-04 | 1996.3ms/step | 10.0 samples/s +Step 7810/44000 | Loss: 3.1899 | LR: 4.68e-04 | 1981.3ms/step | 10.1 samples/s +Step 7820/44000 | Loss: 3.1692 | LR: 4.68e-04 | 1990.8ms/step | 10.0 samples/s +Step 7830/44000 | Loss: 3.1843 | LR: 4.68e-04 | 1996.4ms/step | 10.0 samples/s +Step 7840/44000 | Loss: 3.1596 | LR: 4.68e-04 | 2001.8ms/step | 10.0 samples/s +Step 7850/44000 | Loss: 3.2166 | LR: 4.67e-04 | 2011.4ms/step | 9.9 samples/s +Step 7860/44000 | Loss: 3.2067 | LR: 4.67e-04 | 2012.1ms/step | 9.9 samples/s +Step 7870/44000 | Loss: 3.1860 | LR: 4.67e-04 | 2011.8ms/step | 9.9 samples/s +Step 7880/44000 | Loss: 3.2129 | LR: 4.67e-04 | 1996.6ms/step | 10.0 samples/s +Step 7890/44000 | Loss: 3.1445 | LR: 4.67e-04 | 2014.7ms/step | 9.9 samples/s +Step 7900/44000 | Loss: 3.1660 | LR: 4.67e-04 | 2004.3ms/step | 10.0 samples/s +Step 7910/44000 | Loss: 3.1468 | LR: 4.67e-04 | 2002.7ms/step | 10.0 samples/s +Step 7920/44000 | Loss: 3.1742 | LR: 4.67e-04 | 2007.5ms/step | 10.0 samples/s +Step 7930/44000 | Loss: 3.2046 | LR: 4.67e-04 | 2007.0ms/step | 10.0 samples/s +Step 7940/44000 | Loss: 3.1827 | LR: 4.67e-04 | 1996.8ms/step | 10.0 samples/s +Step 7950/44000 | Loss: 3.1718 | LR: 4.67e-04 | 2012.1ms/step | 9.9 samples/s +Step 7960/44000 | Loss: 3.1729 | LR: 4.66e-04 | 1992.2ms/step | 10.0 samples/s +Step 7970/44000 | Loss: 3.1774 | LR: 4.66e-04 | 1980.9ms/step | 10.1 samples/s +Step 7980/44000 | Loss: 3.2008 | LR: 4.66e-04 | 2007.3ms/step | 10.0 samples/s +Step 7990/44000 | Loss: 3.1835 | LR: 4.66e-04 | 2007.3ms/step | 10.0 samples/s +Step 8000/44000 | Loss: 3.2055 | LR: 4.66e-04 | 2011.7ms/step | 9.9 samples/s +Validation at step 8000: Loss = 3.1864, Perplexity = 24.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-8000 +Step 8010/44000 | Loss: 3.1472 | LR: 4.66e-04 | 4082.2ms/step | 4.9 samples/s +Step 8020/44000 | Loss: 3.1714 | LR: 4.66e-04 | 1933.2ms/step | 10.3 samples/s +Step 8030/44000 | Loss: 3.1813 | LR: 4.66e-04 | 2036.9ms/step | 9.8 samples/s +Step 8040/44000 | Loss: 3.1493 | LR: 4.66e-04 | 2084.2ms/step | 9.6 samples/s +Step 8050/44000 | Loss: 3.1457 | LR: 4.66e-04 | 2183.6ms/step | 9.2 samples/s +Step 8060/44000 | Loss: 3.1218 | LR: 4.66e-04 | 2351.5ms/step | 8.5 samples/s +Step 8070/44000 | Loss: 3.1651 | LR: 4.65e-04 | 2042.1ms/step | 9.8 samples/s +Step 8080/44000 | Loss: 3.1831 | LR: 4.65e-04 | 1980.9ms/step | 10.1 samples/s +Step 8090/44000 | Loss: 3.1698 | LR: 4.65e-04 | 1981.8ms/step | 10.1 samples/s +Step 8100/44000 | Loss: 3.1951 | LR: 4.65e-04 | 2012.1ms/step | 9.9 samples/s +Step 8110/44000 | Loss: 3.1900 | LR: 4.65e-04 | 2169.4ms/step | 9.2 samples/s +Step 8120/44000 | Loss: 3.1826 | LR: 4.65e-04 | 2312.9ms/step | 8.6 samples/s +Step 8130/44000 | Loss: 3.1708 | LR: 4.65e-04 | 2271.9ms/step | 8.8 samples/s +Step 8140/44000 | Loss: 3.1606 | LR: 4.65e-04 | 2229.9ms/step | 9.0 samples/s +Step 8150/44000 | Loss: 3.2051 | LR: 4.65e-04 | 2241.6ms/step | 8.9 samples/s +Step 8160/44000 | Loss: 3.1748 | LR: 4.65e-04 | 2245.3ms/step | 8.9 samples/s +Step 8170/44000 | Loss: 3.1252 | LR: 4.65e-04 | 2283.5ms/step | 8.8 samples/s +Step 8180/44000 | Loss: 3.1331 | LR: 4.64e-04 | 2283.3ms/step | 8.8 samples/s +Step 8190/44000 | Loss: 3.1318 | LR: 4.64e-04 | 2240.5ms/step | 8.9 samples/s +Step 8200/44000 | Loss: 3.1852 | LR: 4.64e-04 | 2299.2ms/step | 8.7 samples/s +Step 8210/44000 | Loss: 3.1506 | LR: 4.64e-04 | 2251.6ms/step | 8.9 samples/s +Step 8220/44000 | Loss: 3.1569 | LR: 4.64e-04 | 2293.7ms/step | 8.7 samples/s +Step 8230/44000 | Loss: 3.1824 | LR: 4.64e-04 | 2283.1ms/step | 8.8 samples/s +Step 8240/44000 | Loss: 3.1965 | LR: 4.64e-04 | 2264.6ms/step | 8.8 samples/s +Step 8250/44000 | Loss: 3.1434 | LR: 4.64e-04 | 2265.4ms/step | 8.8 samples/s +Validation at step 8250: Loss = 3.1801, Perplexity = 24.05 +Step 8260/44000 | Loss: 3.1177 | LR: 4.64e-04 | 4247.6ms/step | 4.7 samples/s +Step 8270/44000 | Loss: 3.1369 | LR: 4.64e-04 | 2162.5ms/step | 9.2 samples/s +Step 8280/44000 | Loss: 3.1439 | LR: 4.64e-04 | 2233.4ms/step | 9.0 samples/s +Step 8290/44000 | Loss: 3.1703 | LR: 4.63e-04 | 2284.4ms/step | 8.8 samples/s +Step 8300/44000 | Loss: 3.1884 | LR: 4.63e-04 | 2304.1ms/step | 8.7 samples/s +Step 8310/44000 | Loss: 3.1606 | LR: 4.63e-04 | 2278.9ms/step | 8.8 samples/s +Step 8320/44000 | Loss: 3.1520 | LR: 4.63e-04 | 2247.2ms/step | 8.9 samples/s +Step 8330/44000 | Loss: 3.1591 | LR: 4.63e-04 | 2263.0ms/step | 8.8 samples/s +Step 8340/44000 | Loss: 3.1283 | LR: 4.63e-04 | 2284.9ms/step | 8.8 samples/s +Step 8350/44000 | Loss: 3.1709 | LR: 4.63e-04 | 2249.6ms/step | 8.9 samples/s +Step 8360/44000 | Loss: 3.1754 | LR: 4.63e-04 | 2257.6ms/step | 8.9 samples/s +Step 8370/44000 | Loss: 3.1740 | LR: 4.63e-04 | 2278.7ms/step | 8.8 samples/s +Step 8380/44000 | Loss: 3.1427 | LR: 4.63e-04 | 2235.7ms/step | 8.9 samples/s +Step 8390/44000 | Loss: 3.1670 | LR: 4.62e-04 | 2283.4ms/step | 8.8 samples/s +Step 8400/44000 | Loss: 3.1552 | LR: 4.62e-04 | 2272.9ms/step | 8.8 samples/s +Step 8410/44000 | Loss: 3.1633 | LR: 4.62e-04 | 2257.5ms/step | 8.9 samples/s +Step 8420/44000 | Loss: 3.1548 | LR: 4.62e-04 | 2272.9ms/step | 8.8 samples/s +Step 8430/44000 | Loss: 3.1553 | LR: 4.62e-04 | 2253.1ms/step | 8.9 samples/s +Step 8440/44000 | Loss: 3.1546 | LR: 4.62e-04 | 2307.3ms/step | 8.7 samples/s +Step 8450/44000 | Loss: 3.1816 | LR: 4.62e-04 | 2257.2ms/step | 8.9 samples/s +Step 8460/44000 | Loss: 3.1457 | LR: 4.62e-04 | 2236.5ms/step | 8.9 samples/s +Step 8470/44000 | Loss: 3.1348 | LR: 4.62e-04 | 2235.3ms/step | 8.9 samples/s +Step 8480/44000 | Loss: 3.1039 | LR: 4.62e-04 | 2231.8ms/step | 9.0 samples/s +Step 8490/44000 | Loss: 3.1307 | LR: 4.62e-04 | 2261.8ms/step | 8.8 samples/s +Step 8500/44000 | Loss: 3.2257 | LR: 4.61e-04 | 2256.8ms/step | 8.9 samples/s +Validation at step 8500: Loss = 3.1732, Perplexity = 23.88 +Step 8510/44000 | Loss: 3.1538 | LR: 4.61e-04 | 4201.4ms/step | 4.8 samples/s +Step 8520/44000 | Loss: 3.1878 | LR: 4.61e-04 | 2288.3ms/step | 8.7 samples/s +Step 8530/44000 | Loss: 3.1828 | LR: 4.61e-04 | 2283.4ms/step | 8.8 samples/s +Step 8540/44000 | Loss: 3.1521 | LR: 4.61e-04 | 2245.7ms/step | 8.9 samples/s +Step 8550/44000 | Loss: 3.1258 | LR: 4.61e-04 | 2254.2ms/step | 8.9 samples/s +Step 8560/44000 | Loss: 3.1696 | LR: 4.61e-04 | 2282.2ms/step | 8.8 samples/s +Step 8570/44000 | Loss: 3.1432 | LR: 4.61e-04 | 2275.2ms/step | 8.8 samples/s +Step 8580/44000 | Loss: 3.1852 | LR: 4.61e-04 | 2310.0ms/step | 8.7 samples/s +Step 8590/44000 | Loss: 3.1843 | LR: 4.61e-04 | 2243.8ms/step | 8.9 samples/s +Step 8600/44000 | Loss: 3.1366 | LR: 4.60e-04 | 2251.6ms/step | 8.9 samples/s +Step 8610/44000 | Loss: 3.2221 | LR: 4.60e-04 | 2140.5ms/step | 9.3 samples/s +Step 8620/44000 | Loss: 3.1438 | LR: 4.60e-04 | 2186.5ms/step | 9.1 samples/s +Step 8630/44000 | Loss: 3.1777 | LR: 4.60e-04 | 2291.0ms/step | 8.7 samples/s +Step 8640/44000 | Loss: 3.1912 | LR: 4.60e-04 | 2255.5ms/step | 8.9 samples/s +Step 8650/44000 | Loss: 3.1442 | LR: 4.60e-04 | 2267.6ms/step | 8.8 samples/s +Step 8660/44000 | Loss: 3.1647 | LR: 4.60e-04 | 2256.3ms/step | 8.9 samples/s +Step 8670/44000 | Loss: 3.1102 | LR: 4.60e-04 | 2316.5ms/step | 8.6 samples/s +Step 8680/44000 | Loss: 3.1497 | LR: 4.60e-04 | 2278.4ms/step | 8.8 samples/s +Step 8690/44000 | Loss: 3.1509 | LR: 4.60e-04 | 2241.0ms/step | 8.9 samples/s +Step 8700/44000 | Loss: 3.1680 | LR: 4.59e-04 | 2278.0ms/step | 8.8 samples/s +Step 8710/44000 | Loss: 3.1923 | LR: 4.59e-04 | 2238.6ms/step | 8.9 samples/s +Step 8720/44000 | Loss: 3.1575 | LR: 4.59e-04 | 2275.3ms/step | 8.8 samples/s +Step 8730/44000 | Loss: 3.1477 | LR: 4.59e-04 | 2277.7ms/step | 8.8 samples/s +Step 8740/44000 | Loss: 3.1255 | LR: 4.59e-04 | 2234.2ms/step | 9.0 samples/s +Step 8750/44000 | Loss: 3.1694 | LR: 4.59e-04 | 2223.4ms/step | 9.0 samples/s +Validation at step 8750: Loss = 3.1669, Perplexity = 23.73 +Step 8760/44000 | Loss: 3.1282 | LR: 4.59e-04 | 4206.3ms/step | 4.8 samples/s +Step 8770/44000 | Loss: 3.1963 | LR: 4.59e-04 | 2184.5ms/step | 9.2 samples/s +Step 8780/44000 | Loss: 3.1618 | LR: 4.59e-04 | 2255.5ms/step | 8.9 samples/s +Step 8790/44000 | Loss: 3.1556 | LR: 4.59e-04 | 2238.0ms/step | 8.9 samples/s +Step 8800/44000 | Loss: 3.1534 | LR: 4.58e-04 | 2273.6ms/step | 8.8 samples/s +Step 8810/44000 | Loss: 3.1483 | LR: 4.58e-04 | 2248.8ms/step | 8.9 samples/s +Step 8820/44000 | Loss: 3.1227 | LR: 4.58e-04 | 2261.4ms/step | 8.8 samples/s +Step 8830/44000 | Loss: 3.1875 | LR: 4.58e-04 | 2303.8ms/step | 8.7 samples/s +Step 8840/44000 | Loss: 3.1121 | LR: 4.58e-04 | 2282.8ms/step | 8.8 samples/s +Step 8850/44000 | Loss: 3.1101 | LR: 4.58e-04 | 2173.8ms/step | 9.2 samples/s +Step 8860/44000 | Loss: 3.1845 | LR: 4.58e-04 | 2147.8ms/step | 9.3 samples/s +Step 8870/44000 | Loss: 3.1899 | LR: 4.58e-04 | 2199.8ms/step | 9.1 samples/s +Step 8880/44000 | Loss: 3.1763 | LR: 4.58e-04 | 2189.3ms/step | 9.1 samples/s +Step 8890/44000 | Loss: 3.1798 | LR: 4.58e-04 | 2215.1ms/step | 9.0 samples/s +Step 8900/44000 | Loss: 3.1431 | LR: 4.57e-04 | 2230.4ms/step | 9.0 samples/s +Step 8910/44000 | Loss: 3.1123 | LR: 4.57e-04 | 2201.0ms/step | 9.1 samples/s +Step 8920/44000 | Loss: 3.1633 | LR: 4.57e-04 | 2230.9ms/step | 9.0 samples/s +Step 8930/44000 | Loss: 3.1490 | LR: 4.57e-04 | 2157.5ms/step | 9.3 samples/s +Step 8940/44000 | Loss: 3.1672 | LR: 4.57e-04 | 2196.2ms/step | 9.1 samples/s +Step 8950/44000 | Loss: 3.1133 | LR: 4.57e-04 | 2100.7ms/step | 9.5 samples/s +Step 8960/44000 | Loss: 3.1737 | LR: 4.57e-04 | 2000.1ms/step | 10.0 samples/s +Step 8970/44000 | Loss: 3.1495 | LR: 4.57e-04 | 2123.3ms/step | 9.4 samples/s +Step 8980/44000 | Loss: 3.1346 | LR: 4.57e-04 | 2188.8ms/step | 9.1 samples/s +Step 8990/44000 | Loss: 3.1382 | LR: 4.57e-04 | 2162.9ms/step | 9.2 samples/s +Step 9000/44000 | Loss: 3.1203 | LR: 4.56e-04 | 2220.2ms/step | 9.0 samples/s +Validation at step 9000: Loss = 3.1583, Perplexity = 23.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-9000 +Step 9010/44000 | Loss: 3.1351 | LR: 4.56e-04 | 4851.9ms/step | 4.1 samples/s +Step 9020/44000 | Loss: 3.1442 | LR: 4.56e-04 | 2314.4ms/step | 8.6 samples/s +Step 9030/44000 | Loss: 3.1193 | LR: 4.56e-04 | 2292.4ms/step | 8.7 samples/s +Step 9040/44000 | Loss: 3.1355 | LR: 4.56e-04 | 2209.1ms/step | 9.1 samples/s +Step 9050/44000 | Loss: 3.1358 | LR: 4.56e-04 | 2223.7ms/step | 9.0 samples/s +Step 9060/44000 | Loss: 3.1375 | LR: 4.56e-04 | 2173.8ms/step | 9.2 samples/s +Step 9070/44000 | Loss: 3.1193 | LR: 4.56e-04 | 2235.8ms/step | 8.9 samples/s +Step 9080/44000 | Loss: 3.1168 | LR: 4.56e-04 | 2251.6ms/step | 8.9 samples/s +Step 9090/44000 | Loss: 3.1500 | LR: 4.56e-04 | 2220.3ms/step | 9.0 samples/s +Step 9100/44000 | Loss: 3.1572 | LR: 4.55e-04 | 2262.4ms/step | 8.8 samples/s +Step 9110/44000 | Loss: 3.1377 | LR: 4.55e-04 | 2250.8ms/step | 8.9 samples/s +Step 9120/44000 | Loss: 3.1185 | LR: 4.55e-04 | 2273.8ms/step | 8.8 samples/s +Step 9130/44000 | Loss: 3.1685 | LR: 4.55e-04 | 2251.9ms/step | 8.9 samples/s +Step 9140/44000 | Loss: 3.1009 | LR: 4.55e-04 | 2311.3ms/step | 8.7 samples/s +Step 9150/44000 | Loss: 3.1477 | LR: 4.55e-04 | 2191.9ms/step | 9.1 samples/s +Step 9160/44000 | Loss: 3.1058 | LR: 4.55e-04 | 2220.3ms/step | 9.0 samples/s +Step 9170/44000 | Loss: 3.1484 | LR: 4.55e-04 | 2225.7ms/step | 9.0 samples/s +Step 9180/44000 | Loss: 3.1104 | LR: 4.55e-04 | 2209.9ms/step | 9.1 samples/s +Step 9190/44000 | Loss: 3.1380 | LR: 4.54e-04 | 2236.7ms/step | 8.9 samples/s +Step 9200/44000 | Loss: 3.0851 | LR: 4.54e-04 | 2257.5ms/step | 8.9 samples/s +Step 9210/44000 | Loss: 3.1357 | LR: 4.54e-04 | 2250.5ms/step | 8.9 samples/s +Step 9220/44000 | Loss: 3.1705 | LR: 4.54e-04 | 2252.0ms/step | 8.9 samples/s +Step 9230/44000 | Loss: 3.1159 | LR: 4.54e-04 | 2252.0ms/step | 8.9 samples/s +Step 9240/44000 | Loss: 3.1556 | LR: 4.54e-04 | 2259.3ms/step | 8.9 samples/s +Step 9250/44000 | Loss: 3.1307 | LR: 4.54e-04 | 2228.5ms/step | 9.0 samples/s +Validation at step 9250: Loss = 3.1538, Perplexity = 23.42 +Step 9260/44000 | Loss: 3.1636 | LR: 4.54e-04 | 4124.5ms/step | 4.8 samples/s +Step 9270/44000 | Loss: 3.1281 | LR: 4.54e-04 | 2227.2ms/step | 9.0 samples/s +Step 9280/44000 | Loss: 3.1264 | LR: 4.54e-04 | 2145.9ms/step | 9.3 samples/s +Step 9290/44000 | Loss: 3.1657 | LR: 4.53e-04 | 2214.5ms/step | 9.0 samples/s +Step 9300/44000 | Loss: 3.1214 | LR: 4.53e-04 | 2217.9ms/step | 9.0 samples/s +Step 9310/44000 | Loss: 3.1023 | LR: 4.53e-04 | 2246.8ms/step | 8.9 samples/s +Step 9320/44000 | Loss: 3.1585 | LR: 4.53e-04 | 2257.8ms/step | 8.9 samples/s +Step 9330/44000 | Loss: 3.1035 | LR: 4.53e-04 | 2205.4ms/step | 9.1 samples/s +Step 9340/44000 | Loss: 3.1561 | LR: 4.53e-04 | 2205.3ms/step | 9.1 samples/s +Step 9350/44000 | Loss: 3.1702 | LR: 4.53e-04 | 2185.6ms/step | 9.2 samples/s +Step 9360/44000 | Loss: 3.1296 | LR: 4.53e-04 | 2238.6ms/step | 8.9 samples/s +Step 9370/44000 | Loss: 3.1323 | LR: 4.53e-04 | 2251.6ms/step | 8.9 samples/s +Step 9380/44000 | Loss: 3.1740 | LR: 4.52e-04 | 2232.1ms/step | 9.0 samples/s +Step 9390/44000 | Loss: 3.1359 | LR: 4.52e-04 | 2299.1ms/step | 8.7 samples/s +Step 9400/44000 | Loss: 3.1203 | LR: 4.52e-04 | 2206.5ms/step | 9.1 samples/s +Step 9410/44000 | Loss: 3.1363 | LR: 4.52e-04 | 2199.3ms/step | 9.1 samples/s +Step 9420/44000 | Loss: 3.1578 | LR: 4.52e-04 | 2158.0ms/step | 9.3 samples/s +Step 9430/44000 | Loss: 3.1259 | LR: 4.52e-04 | 2231.5ms/step | 9.0 samples/s +Step 9440/44000 | Loss: 3.1094 | LR: 4.52e-04 | 2257.1ms/step | 8.9 samples/s +Step 9450/44000 | Loss: 3.1352 | LR: 4.52e-04 | 2258.7ms/step | 8.9 samples/s +Step 9460/44000 | Loss: 3.1382 | LR: 4.52e-04 | 2252.5ms/step | 8.9 samples/s +Step 9470/44000 | Loss: 3.1401 | LR: 4.52e-04 | 2226.8ms/step | 9.0 samples/s +Step 9480/44000 | Loss: 3.1417 | LR: 4.51e-04 | 2226.5ms/step | 9.0 samples/s +Step 9490/44000 | Loss: 3.1118 | LR: 4.51e-04 | 2195.6ms/step | 9.1 samples/s +Step 9500/44000 | Loss: 3.1578 | LR: 4.51e-04 | 2201.3ms/step | 9.1 samples/s +Validation at step 9500: Loss = 3.1481, Perplexity = 23.29 +Step 9510/44000 | Loss: 3.1131 | LR: 4.51e-04 | 4200.4ms/step | 4.8 samples/s +Step 9520/44000 | Loss: 3.1617 | LR: 4.51e-04 | 2361.5ms/step | 8.5 samples/s +Step 9530/44000 | Loss: 3.1558 | LR: 4.51e-04 | 2214.0ms/step | 9.0 samples/s +Step 9540/44000 | Loss: 3.1296 | LR: 4.51e-04 | 2273.4ms/step | 8.8 samples/s +Step 9550/44000 | Loss: 3.1101 | LR: 4.51e-04 | 2232.2ms/step | 9.0 samples/s +Step 9560/44000 | Loss: 3.1224 | LR: 4.51e-04 | 2268.2ms/step | 8.8 samples/s +Step 9570/44000 | Loss: 3.1624 | LR: 4.50e-04 | 2143.1ms/step | 9.3 samples/s +Step 9580/44000 | Loss: 3.1105 | LR: 4.50e-04 | 2168.3ms/step | 9.2 samples/s +Step 9590/44000 | Loss: 3.1180 | LR: 4.50e-04 | 2173.1ms/step | 9.2 samples/s +Step 9600/44000 | Loss: 3.1156 | LR: 4.50e-04 | 2212.3ms/step | 9.0 samples/s +Step 9610/44000 | Loss: 3.1306 | LR: 4.50e-04 | 2146.6ms/step | 9.3 samples/s +Step 9620/44000 | Loss: 3.0747 | LR: 4.50e-04 | 2059.6ms/step | 9.7 samples/s +Step 9630/44000 | Loss: 3.1164 | LR: 4.50e-04 | 2143.2ms/step | 9.3 samples/s +Step 9640/44000 | Loss: 3.0972 | LR: 4.50e-04 | 2156.0ms/step | 9.3 samples/s +Step 9650/44000 | Loss: 3.1647 | LR: 4.50e-04 | 2189.8ms/step | 9.1 samples/s +Step 9660/44000 | Loss: 3.1119 | LR: 4.49e-04 | 2170.0ms/step | 9.2 samples/s +Step 9670/44000 | Loss: 3.1584 | LR: 4.49e-04 | 2267.4ms/step | 8.8 samples/s +Step 9680/44000 | Loss: 3.0852 | LR: 4.49e-04 | 2190.4ms/step | 9.1 samples/s +Step 9690/44000 | Loss: 3.1614 | LR: 4.49e-04 | 2192.2ms/step | 9.1 samples/s +Step 9700/44000 | Loss: 3.1314 | LR: 4.49e-04 | 2156.3ms/step | 9.3 samples/s +Step 9710/44000 | Loss: 3.1351 | LR: 4.49e-04 | 2132.8ms/step | 9.4 samples/s +Step 9720/44000 | Loss: 3.0863 | LR: 4.49e-04 | 2137.9ms/step | 9.4 samples/s +Step 9730/44000 | Loss: 3.1075 | LR: 4.49e-04 | 2143.5ms/step | 9.3 samples/s +Step 9740/44000 | Loss: 3.1424 | LR: 4.49e-04 | 2187.6ms/step | 9.1 samples/s +Step 9750/44000 | Loss: 3.1197 | LR: 4.48e-04 | 2208.2ms/step | 9.1 samples/s +Validation at step 9750: Loss = 3.1408, Perplexity = 23.12 +Step 9760/44000 | Loss: 3.1539 | LR: 4.48e-04 | 4202.8ms/step | 4.8 samples/s +Step 9770/44000 | Loss: 3.1066 | LR: 4.48e-04 | 2151.7ms/step | 9.3 samples/s +Step 9780/44000 | Loss: 3.1284 | LR: 4.48e-04 | 2378.5ms/step | 8.4 samples/s +Step 9790/44000 | Loss: 3.0903 | LR: 4.48e-04 | 2428.4ms/step | 8.2 samples/s +Step 9800/44000 | Loss: 3.0956 | LR: 4.48e-04 | 2086.3ms/step | 9.6 samples/s +Step 9810/44000 | Loss: 3.1372 | LR: 4.48e-04 | 2133.7ms/step | 9.4 samples/s +Step 9820/44000 | Loss: 3.0823 | LR: 4.48e-04 | 2100.3ms/step | 9.5 samples/s +Step 9830/44000 | Loss: 3.1791 | LR: 4.48e-04 | 2106.4ms/step | 9.5 samples/s +Step 9840/44000 | Loss: 3.1405 | LR: 4.47e-04 | 2167.3ms/step | 9.2 samples/s +Step 9850/44000 | Loss: 3.1268 | LR: 4.47e-04 | 2154.2ms/step | 9.3 samples/s +Step 9860/44000 | Loss: 3.1358 | LR: 4.47e-04 | 2111.2ms/step | 9.5 samples/s +Step 9870/44000 | Loss: 3.0883 | LR: 4.47e-04 | 2161.3ms/step | 9.3 samples/s +Step 9880/44000 | Loss: 3.1112 | LR: 4.47e-04 | 2175.2ms/step | 9.2 samples/s +Step 9890/44000 | Loss: 3.1213 | LR: 4.47e-04 | 2180.2ms/step | 9.2 samples/s +Step 9900/44000 | Loss: 3.1760 | LR: 4.47e-04 | 2148.0ms/step | 9.3 samples/s +Step 9910/44000 | Loss: 3.0924 | LR: 4.47e-04 | 2148.1ms/step | 9.3 samples/s +Step 9920/44000 | Loss: 3.1233 | LR: 4.47e-04 | 2178.2ms/step | 9.2 samples/s +Step 9930/44000 | Loss: 3.0738 | LR: 4.46e-04 | 2141.2ms/step | 9.3 samples/s +Step 9940/44000 | Loss: 3.1228 | LR: 4.46e-04 | 2151.4ms/step | 9.3 samples/s +Step 9950/44000 | Loss: 3.0823 | LR: 4.46e-04 | 2043.3ms/step | 9.8 samples/s +Step 9960/44000 | Loss: 3.0983 | LR: 4.46e-04 | 2204.9ms/step | 9.1 samples/s +Step 9970/44000 | Loss: 3.1133 | LR: 4.46e-04 | 2231.1ms/step | 9.0 samples/s +Step 9980/44000 | Loss: 3.0852 | LR: 4.46e-04 | 2237.1ms/step | 8.9 samples/s +Step 9990/44000 | Loss: 3.1353 | LR: 4.46e-04 | 2221.7ms/step | 9.0 samples/s +Step 10000/44000 | Loss: 3.1320 | LR: 4.46e-04 | 2193.7ms/step | 9.1 samples/s +Validation at step 10000: Loss = 3.1350, Perplexity = 22.99 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-10000 +Step 10010/44000 | Loss: 3.1322 | LR: 4.46e-04 | 4435.0ms/step | 4.5 samples/s +Step 10020/44000 | Loss: 3.1253 | LR: 4.45e-04 | 2186.7ms/step | 9.1 samples/s +Step 10030/44000 | Loss: 3.0992 | LR: 4.45e-04 | 2194.6ms/step | 9.1 samples/s +Step 10040/44000 | Loss: 3.1006 | LR: 4.45e-04 | 2205.2ms/step | 9.1 samples/s +Step 10050/44000 | Loss: 3.0484 | LR: 4.45e-04 | 2196.5ms/step | 9.1 samples/s +Step 10060/44000 | Loss: 3.1644 | LR: 4.45e-04 | 2167.1ms/step | 9.2 samples/s +Step 10070/44000 | Loss: 3.1205 | LR: 4.45e-04 | 2163.8ms/step | 9.2 samples/s +Step 10080/44000 | Loss: 3.1311 | LR: 4.45e-04 | 2148.6ms/step | 9.3 samples/s +Step 10090/44000 | Loss: 3.1272 | LR: 4.45e-04 | 2168.6ms/step | 9.2 samples/s +Step 10100/44000 | Loss: 3.1090 | LR: 4.45e-04 | 2215.1ms/step | 9.0 samples/s +Step 10110/44000 | Loss: 3.1225 | LR: 4.44e-04 | 2201.3ms/step | 9.1 samples/s +Step 10120/44000 | Loss: 3.1229 | LR: 4.44e-04 | 2223.5ms/step | 9.0 samples/s +Step 10130/44000 | Loss: 3.1169 | LR: 4.44e-04 | 2198.3ms/step | 9.1 samples/s +Step 10140/44000 | Loss: 3.1155 | LR: 4.44e-04 | 2183.8ms/step | 9.2 samples/s +Step 10150/44000 | Loss: 3.1044 | LR: 4.44e-04 | 2158.9ms/step | 9.3 samples/s +Step 10160/44000 | Loss: 3.1497 | LR: 4.44e-04 | 2168.1ms/step | 9.2 samples/s +Step 10170/44000 | Loss: 3.1332 | LR: 4.44e-04 | 2096.4ms/step | 9.5 samples/s +Step 10180/44000 | Loss: 3.1008 | LR: 4.44e-04 | 2143.2ms/step | 9.3 samples/s +Step 10190/44000 | Loss: 3.0891 | LR: 4.43e-04 | 2203.5ms/step | 9.1 samples/s +Step 10200/44000 | Loss: 3.0861 | LR: 4.43e-04 | 2165.4ms/step | 9.2 samples/s +Step 10210/44000 | Loss: 3.1167 | LR: 4.43e-04 | 2209.3ms/step | 9.1 samples/s +Step 10220/44000 | Loss: 3.0748 | LR: 4.43e-04 | 2101.3ms/step | 9.5 samples/s +Step 10230/44000 | Loss: 3.1151 | LR: 4.43e-04 | 2130.2ms/step | 9.4 samples/s +Step 10240/44000 | Loss: 3.1086 | LR: 4.43e-04 | 2154.6ms/step | 9.3 samples/s +Step 10250/44000 | Loss: 3.0739 | LR: 4.43e-04 | 2192.8ms/step | 9.1 samples/s +Validation at step 10250: Loss = 3.1322, Perplexity = 22.92 +Step 10260/44000 | Loss: 3.1478 | LR: 4.43e-04 | 3979.8ms/step | 5.0 samples/s +Step 10270/44000 | Loss: 3.1017 | LR: 4.43e-04 | 2033.1ms/step | 9.8 samples/s +Step 10280/44000 | Loss: 3.0962 | LR: 4.42e-04 | 2125.1ms/step | 9.4 samples/s +Step 10290/44000 | Loss: 3.0998 | LR: 4.42e-04 | 2123.7ms/step | 9.4 samples/s +Step 10300/44000 | Loss: 3.0814 | LR: 4.42e-04 | 2169.9ms/step | 9.2 samples/s +Step 10310/44000 | Loss: 3.1090 | LR: 4.42e-04 | 2219.6ms/step | 9.0 samples/s +Step 10320/44000 | Loss: 3.1325 | LR: 4.42e-04 | 2168.9ms/step | 9.2 samples/s +Step 10330/44000 | Loss: 3.1058 | LR: 4.42e-04 | 2193.7ms/step | 9.1 samples/s +Step 10340/44000 | Loss: 3.1086 | LR: 4.42e-04 | 2217.1ms/step | 9.0 samples/s +Step 10350/44000 | Loss: 3.1185 | LR: 4.42e-04 | 2247.5ms/step | 8.9 samples/s +Step 10360/44000 | Loss: 3.1336 | LR: 4.42e-04 | 2247.2ms/step | 8.9 samples/s +Step 10370/44000 | Loss: 3.1113 | LR: 4.41e-04 | 2226.2ms/step | 9.0 samples/s +Step 10380/44000 | Loss: 3.0500 | LR: 4.41e-04 | 2257.5ms/step | 8.9 samples/s +Step 10390/44000 | Loss: 3.0743 | LR: 4.41e-04 | 2221.6ms/step | 9.0 samples/s +Step 10400/44000 | Loss: 3.1246 | LR: 4.41e-04 | 2242.5ms/step | 8.9 samples/s +Step 10410/44000 | Loss: 3.1043 | LR: 4.41e-04 | 2210.5ms/step | 9.0 samples/s +Step 10420/44000 | Loss: 3.0847 | LR: 4.41e-04 | 2200.9ms/step | 9.1 samples/s +Step 10430/44000 | Loss: 3.1151 | LR: 4.41e-04 | 2278.4ms/step | 8.8 samples/s +Step 10440/44000 | Loss: 3.1061 | LR: 4.41e-04 | 2253.6ms/step | 8.9 samples/s +Step 10450/44000 | Loss: 3.0913 | LR: 4.40e-04 | 2298.6ms/step | 8.7 samples/s +Step 10460/44000 | Loss: 3.1489 | LR: 4.40e-04 | 2250.0ms/step | 8.9 samples/s +Step 10470/44000 | Loss: 3.0974 | LR: 4.40e-04 | 2187.7ms/step | 9.1 samples/s +Step 10480/44000 | Loss: 3.0913 | LR: 4.40e-04 | 2222.7ms/step | 9.0 samples/s +Step 10490/44000 | Loss: 3.1167 | LR: 4.40e-04 | 2284.6ms/step | 8.8 samples/s +Step 10500/44000 | Loss: 3.0540 | LR: 4.40e-04 | 2225.1ms/step | 9.0 samples/s +Validation at step 10500: Loss = 3.1243, Perplexity = 22.74 +Step 10510/44000 | Loss: 3.1324 | LR: 4.40e-04 | 4172.1ms/step | 4.8 samples/s +Step 10520/44000 | Loss: 3.1280 | LR: 4.40e-04 | 2210.8ms/step | 9.0 samples/s +Step 10530/44000 | Loss: 3.1054 | LR: 4.40e-04 | 2279.6ms/step | 8.8 samples/s +Step 10540/44000 | Loss: 3.0939 | LR: 4.39e-04 | 2288.7ms/step | 8.7 samples/s +Step 10550/44000 | Loss: 3.1609 | LR: 4.39e-04 | 2236.9ms/step | 8.9 samples/s +Step 10560/44000 | Loss: 3.0995 | LR: 4.39e-04 | 2286.6ms/step | 8.7 samples/s +Step 10570/44000 | Loss: 3.0927 | LR: 4.39e-04 | 2218.8ms/step | 9.0 samples/s +Step 10580/44000 | Loss: 3.1144 | LR: 4.39e-04 | 2236.4ms/step | 8.9 samples/s +Step 10590/44000 | Loss: 3.1134 | LR: 4.39e-04 | 2117.0ms/step | 9.4 samples/s +Step 10600/44000 | Loss: 3.2003 | LR: 4.39e-04 | 2236.7ms/step | 8.9 samples/s +Step 10610/44000 | Loss: 3.0525 | LR: 4.39e-04 | 2257.4ms/step | 8.9 samples/s +Step 10620/44000 | Loss: 3.0931 | LR: 4.38e-04 | 2252.6ms/step | 8.9 samples/s +Step 10630/44000 | Loss: 3.1062 | LR: 4.38e-04 | 2189.0ms/step | 9.1 samples/s +Step 10640/44000 | Loss: 3.1128 | LR: 4.38e-04 | 2200.4ms/step | 9.1 samples/s +Step 10650/44000 | Loss: 3.1395 | LR: 4.38e-04 | 2272.9ms/step | 8.8 samples/s +Step 10660/44000 | Loss: 3.1613 | LR: 4.38e-04 | 2310.3ms/step | 8.7 samples/s +Step 10670/44000 | Loss: 3.1030 | LR: 4.38e-04 | 2335.4ms/step | 8.6 samples/s +Step 10680/44000 | Loss: 3.0816 | LR: 4.38e-04 | 2290.7ms/step | 8.7 samples/s +Step 10690/44000 | Loss: 3.1112 | LR: 4.38e-04 | 2382.5ms/step | 8.4 samples/s +Step 10700/44000 | Loss: 3.0993 | LR: 4.37e-04 | 2407.8ms/step | 8.3 samples/s +Step 10710/44000 | Loss: 3.1071 | LR: 4.37e-04 | 2737.4ms/step | 7.3 samples/s +Step 10720/44000 | Loss: 3.1051 | LR: 4.37e-04 | 2683.3ms/step | 7.5 samples/s +Step 10730/44000 | Loss: 3.0895 | LR: 4.37e-04 | 2791.4ms/step | 7.2 samples/s +Step 10740/44000 | Loss: 3.0870 | LR: 4.37e-04 | 2719.9ms/step | 7.4 samples/s +Step 10750/44000 | Loss: 3.0910 | LR: 4.37e-04 | 2707.5ms/step | 7.4 samples/s +Validation at step 10750: Loss = 3.1181, Perplexity = 22.60 +Step 10760/44000 | Loss: 3.0945 | LR: 4.37e-04 | 5125.1ms/step | 3.9 samples/s +Step 10770/44000 | Loss: 3.1029 | LR: 4.37e-04 | 2747.3ms/step | 7.3 samples/s +Step 10780/44000 | Loss: 3.1152 | LR: 4.37e-04 | 2254.3ms/step | 8.9 samples/s +Step 10790/44000 | Loss: 3.0760 | LR: 4.36e-04 | 2207.5ms/step | 9.1 samples/s +Step 10800/44000 | Loss: 3.1219 | LR: 4.36e-04 | 2237.1ms/step | 8.9 samples/s +Step 10810/44000 | Loss: 3.1096 | LR: 4.36e-04 | 2189.8ms/step | 9.1 samples/s +Step 10820/44000 | Loss: 3.0783 | LR: 4.36e-04 | 2204.9ms/step | 9.1 samples/s +Step 10830/44000 | Loss: 3.0722 | LR: 4.36e-04 | 2189.7ms/step | 9.1 samples/s +Step 10840/44000 | Loss: 3.0953 | LR: 4.36e-04 | 2176.1ms/step | 9.2 samples/s +Step 10850/44000 | Loss: 3.1100 | LR: 4.36e-04 | 2219.2ms/step | 9.0 samples/s +Step 10860/44000 | Loss: 3.0747 | LR: 4.36e-04 | 2202.1ms/step | 9.1 samples/s +Step 10870/44000 | Loss: 3.0812 | LR: 4.35e-04 | 2194.1ms/step | 9.1 samples/s +Step 10880/44000 | Loss: 3.0888 | LR: 4.35e-04 | 2570.7ms/step | 7.8 samples/s +Step 10890/44000 | Loss: 3.1020 | LR: 4.35e-04 | 2910.8ms/step | 6.9 samples/s +Step 10900/44000 | Loss: 3.0963 | LR: 4.35e-04 | 2917.8ms/step | 6.9 samples/s +Step 10910/44000 | Loss: 3.0572 | LR: 4.35e-04 | 2773.9ms/step | 7.2 samples/s +Step 10920/44000 | Loss: 3.0855 | LR: 4.35e-04 | 2884.3ms/step | 6.9 samples/s +Step 10930/44000 | Loss: 3.1237 | LR: 4.35e-04 | 2948.6ms/step | 6.8 samples/s +Step 10940/44000 | Loss: 3.1019 | LR: 4.35e-04 | 2900.8ms/step | 6.9 samples/s +Step 10950/44000 | Loss: 3.0856 | LR: 4.34e-04 | 2941.6ms/step | 6.8 samples/s +Step 10960/44000 | Loss: 3.1000 | LR: 4.34e-04 | 2967.8ms/step | 6.7 samples/s +Step 10970/44000 | Loss: 3.0823 | LR: 4.34e-04 | 2883.5ms/step | 6.9 samples/s +Step 10980/44000 | Loss: 3.0905 | LR: 4.34e-04 | 2486.5ms/step | 8.0 samples/s +Step 10990/44000 | Loss: 3.1006 | LR: 4.34e-04 | 2316.1ms/step | 8.6 samples/s +Step 11000/44000 | Loss: 3.1639 | LR: 4.34e-04 | 2250.8ms/step | 8.9 samples/s +Validation at step 11000: Loss = 3.1150, Perplexity = 22.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-11000 +Step 11010/44000 | Loss: 3.1165 | LR: 4.34e-04 | 4347.3ms/step | 4.6 samples/s +Step 11020/44000 | Loss: 3.0968 | LR: 4.34e-04 | 2127.7ms/step | 9.4 samples/s +Step 11030/44000 | Loss: 3.0932 | LR: 4.33e-04 | 2172.4ms/step | 9.2 samples/s +Step 11040/44000 | Loss: 3.0905 | LR: 4.33e-04 | 2134.4ms/step | 9.4 samples/s +Step 11050/44000 | Loss: 3.1187 | LR: 4.33e-04 | 2192.8ms/step | 9.1 samples/s +Step 11060/44000 | Loss: 3.0982 | LR: 4.33e-04 | 2083.5ms/step | 9.6 samples/s +Step 11070/44000 | Loss: 3.1071 | LR: 4.33e-04 | 2127.8ms/step | 9.4 samples/s +Step 11080/44000 | Loss: 3.1040 | LR: 4.33e-04 | 2091.5ms/step | 9.6 samples/s +Step 11090/44000 | Loss: 3.0723 | LR: 4.33e-04 | 2180.1ms/step | 9.2 samples/s +Step 11100/44000 | Loss: 3.0444 | LR: 4.33e-04 | 2153.5ms/step | 9.3 samples/s +Step 11110/44000 | Loss: 3.0949 | LR: 4.32e-04 | 2136.0ms/step | 9.4 samples/s +Step 11120/44000 | Loss: 3.1278 | LR: 4.32e-04 | 2129.1ms/step | 9.4 samples/s +Step 11130/44000 | Loss: 3.0822 | LR: 4.32e-04 | 2132.6ms/step | 9.4 samples/s +Step 11140/44000 | Loss: 3.1224 | LR: 4.32e-04 | 2093.7ms/step | 9.6 samples/s +Step 11150/44000 | Loss: 3.0737 | LR: 4.32e-04 | 2192.7ms/step | 9.1 samples/s +Step 11160/44000 | Loss: 3.0932 | LR: 4.32e-04 | 2154.7ms/step | 9.3 samples/s +Step 11170/44000 | Loss: 3.1034 | LR: 4.32e-04 | 2157.1ms/step | 9.3 samples/s +Step 11180/44000 | Loss: 3.0796 | LR: 4.32e-04 | 2216.0ms/step | 9.0 samples/s +Step 11190/44000 | Loss: 3.1196 | LR: 4.31e-04 | 2207.7ms/step | 9.1 samples/s +Step 11200/44000 | Loss: 3.0557 | LR: 4.31e-04 | 2224.2ms/step | 9.0 samples/s +Step 11210/44000 | Loss: 3.0793 | LR: 4.31e-04 | 2189.5ms/step | 9.1 samples/s +Step 11220/44000 | Loss: 3.1299 | LR: 4.31e-04 | 2127.0ms/step | 9.4 samples/s +Step 11230/44000 | Loss: 3.1016 | LR: 4.31e-04 | 2053.9ms/step | 9.7 samples/s +Step 11240/44000 | Loss: 3.0797 | LR: 4.31e-04 | 2148.4ms/step | 9.3 samples/s +Step 11250/44000 | Loss: 3.0818 | LR: 4.31e-04 | 2169.8ms/step | 9.2 samples/s +Validation at step 11250: Loss = 3.1099, Perplexity = 22.42 +Step 11260/44000 | Loss: 3.0732 | LR: 4.31e-04 | 4179.8ms/step | 4.8 samples/s +Step 11270/44000 | Loss: 3.0874 | LR: 4.30e-04 | 2215.8ms/step | 9.0 samples/s +Step 11280/44000 | Loss: 3.0647 | LR: 4.30e-04 | 2225.9ms/step | 9.0 samples/s +Step 11290/44000 | Loss: 3.1105 | LR: 4.30e-04 | 2221.3ms/step | 9.0 samples/s +Step 11300/44000 | Loss: 3.0960 | LR: 4.30e-04 | 2174.4ms/step | 9.2 samples/s +Step 11310/44000 | Loss: 3.0552 | LR: 4.30e-04 | 2194.4ms/step | 9.1 samples/s +Step 11320/44000 | Loss: 3.0603 | LR: 4.30e-04 | 2168.9ms/step | 9.2 samples/s +Step 11330/44000 | Loss: 3.1377 | LR: 4.30e-04 | 2164.9ms/step | 9.2 samples/s +Step 11340/44000 | Loss: 3.0857 | LR: 4.30e-04 | 2241.3ms/step | 8.9 samples/s +Step 11350/44000 | Loss: 3.1080 | LR: 4.29e-04 | 2231.1ms/step | 9.0 samples/s +Step 11360/44000 | Loss: 3.1149 | LR: 4.29e-04 | 2246.3ms/step | 8.9 samples/s +Step 11370/44000 | Loss: 3.0756 | LR: 4.29e-04 | 2147.0ms/step | 9.3 samples/s +Step 11380/44000 | Loss: 3.0918 | LR: 4.29e-04 | 2147.7ms/step | 9.3 samples/s +Step 11390/44000 | Loss: 3.0924 | LR: 4.29e-04 | 2084.6ms/step | 9.6 samples/s +Step 11400/44000 | Loss: 3.0809 | LR: 4.29e-04 | 2143.2ms/step | 9.3 samples/s +Step 11410/44000 | Loss: 3.0882 | LR: 4.29e-04 | 2079.3ms/step | 9.6 samples/s +Step 11420/44000 | Loss: 3.0687 | LR: 4.29e-04 | 2153.0ms/step | 9.3 samples/s +Step 11430/44000 | Loss: 3.0465 | LR: 4.28e-04 | 2246.4ms/step | 8.9 samples/s +Step 11440/44000 | Loss: 3.0995 | LR: 4.28e-04 | 2137.8ms/step | 9.4 samples/s +Step 11450/44000 | Loss: 3.1077 | LR: 4.28e-04 | 2137.1ms/step | 9.4 samples/s +Step 11460/44000 | Loss: 3.0834 | LR: 4.28e-04 | 2139.7ms/step | 9.3 samples/s +Step 11470/44000 | Loss: 3.1129 | LR: 4.28e-04 | 2144.9ms/step | 9.3 samples/s +Step 11480/44000 | Loss: 3.0627 | LR: 4.28e-04 | 2168.6ms/step | 9.2 samples/s +Step 11490/44000 | Loss: 3.0434 | LR: 4.28e-04 | 2121.4ms/step | 9.4 samples/s +Step 11500/44000 | Loss: 3.1032 | LR: 4.28e-04 | 2106.7ms/step | 9.5 samples/s +Validation at step 11500: Loss = 3.1042, Perplexity = 22.29 +Step 11510/44000 | Loss: 3.0940 | LR: 4.27e-04 | 4189.8ms/step | 4.8 samples/s +Step 11520/44000 | Loss: 3.1017 | LR: 4.27e-04 | 2274.0ms/step | 8.8 samples/s +Step 11530/44000 | Loss: 3.0912 | LR: 4.27e-04 | 2238.2ms/step | 8.9 samples/s +Step 11540/44000 | Loss: 3.0725 | LR: 4.27e-04 | 2279.1ms/step | 8.8 samples/s +Step 11550/44000 | Loss: 3.1308 | LR: 4.27e-04 | 2090.7ms/step | 9.6 samples/s +Step 11560/44000 | Loss: 3.0985 | LR: 4.27e-04 | 2116.5ms/step | 9.4 samples/s +Step 11570/44000 | Loss: 3.0654 | LR: 4.27e-04 | 2183.5ms/step | 9.2 samples/s +Step 11580/44000 | Loss: 3.1223 | LR: 4.27e-04 | 2267.3ms/step | 8.8 samples/s +Step 11590/44000 | Loss: 3.0589 | LR: 4.26e-04 | 2194.8ms/step | 9.1 samples/s +Step 11600/44000 | Loss: 3.0982 | LR: 4.26e-04 | 2239.3ms/step | 8.9 samples/s +Step 11610/44000 | Loss: 3.1075 | LR: 4.26e-04 | 2339.6ms/step | 8.5 samples/s +Step 11620/44000 | Loss: 3.0731 | LR: 4.26e-04 | 2284.4ms/step | 8.8 samples/s +Step 11630/44000 | Loss: 3.1064 | LR: 4.26e-04 | 2317.5ms/step | 8.6 samples/s +Step 11640/44000 | Loss: 3.0746 | LR: 4.26e-04 | 2222.3ms/step | 9.0 samples/s +Step 11650/44000 | Loss: 3.1162 | LR: 4.26e-04 | 2328.8ms/step | 8.6 samples/s +Step 11660/44000 | Loss: 3.0627 | LR: 4.26e-04 | 2330.5ms/step | 8.6 samples/s +Step 11670/44000 | Loss: 3.0922 | LR: 4.25e-04 | 2292.2ms/step | 8.7 samples/s +Step 11680/44000 | Loss: 3.0732 | LR: 4.25e-04 | 2194.8ms/step | 9.1 samples/s +Step 11690/44000 | Loss: 3.0813 | LR: 4.25e-04 | 2097.0ms/step | 9.5 samples/s +Step 11700/44000 | Loss: 3.0911 | LR: 4.25e-04 | 2099.7ms/step | 9.5 samples/s +Step 11710/44000 | Loss: 3.0569 | LR: 4.25e-04 | 2106.0ms/step | 9.5 samples/s +Step 11720/44000 | Loss: 3.0814 | LR: 4.25e-04 | 2200.5ms/step | 9.1 samples/s +Step 11730/44000 | Loss: 3.0938 | LR: 4.25e-04 | 2199.5ms/step | 9.1 samples/s +Step 11740/44000 | Loss: 3.0528 | LR: 4.24e-04 | 2163.6ms/step | 9.2 samples/s +Step 11750/44000 | Loss: 3.0951 | LR: 4.24e-04 | 2141.9ms/step | 9.3 samples/s +Validation at step 11750: Loss = 3.0989, Perplexity = 22.17 +Step 11760/44000 | Loss: 3.0844 | LR: 4.24e-04 | 3885.4ms/step | 5.1 samples/s +Step 11770/44000 | Loss: 3.0475 | LR: 4.24e-04 | 2229.9ms/step | 9.0 samples/s +Step 11780/44000 | Loss: 3.0286 | LR: 4.24e-04 | 2125.5ms/step | 9.4 samples/s +Step 11790/44000 | Loss: 3.0439 | LR: 4.24e-04 | 2095.3ms/step | 9.5 samples/s +Step 11800/44000 | Loss: 3.1007 | LR: 4.24e-04 | 2168.4ms/step | 9.2 samples/s +Step 11810/44000 | Loss: 3.0658 | LR: 4.24e-04 | 2169.2ms/step | 9.2 samples/s +Step 11820/44000 | Loss: 3.0845 | LR: 4.23e-04 | 2188.0ms/step | 9.1 samples/s +Step 11830/44000 | Loss: 3.0839 | LR: 4.23e-04 | 2292.1ms/step | 8.7 samples/s +Step 11840/44000 | Loss: 3.0579 | LR: 4.23e-04 | 2275.5ms/step | 8.8 samples/s +Step 11850/44000 | Loss: 3.1108 | LR: 4.23e-04 | 2104.7ms/step | 9.5 samples/s +Step 11860/44000 | Loss: 3.0621 | LR: 4.23e-04 | 2118.2ms/step | 9.4 samples/s +Step 11870/44000 | Loss: 3.0998 | LR: 4.23e-04 | 2454.4ms/step | 8.1 samples/s +Step 11880/44000 | Loss: 3.0988 | LR: 4.23e-04 | 2356.1ms/step | 8.5 samples/s +Step 11890/44000 | Loss: 3.0556 | LR: 4.23e-04 | 2616.0ms/step | 7.6 samples/s +Step 11900/44000 | Loss: 3.0584 | LR: 4.22e-04 | 2291.1ms/step | 8.7 samples/s +Step 11910/44000 | Loss: 3.0531 | LR: 4.22e-04 | 2155.0ms/step | 9.3 samples/s +Step 11920/44000 | Loss: 3.0670 | LR: 4.22e-04 | 2074.1ms/step | 9.6 samples/s +Step 11930/44000 | Loss: 3.0694 | LR: 4.22e-04 | 2095.7ms/step | 9.5 samples/s +Step 11940/44000 | Loss: 3.0720 | LR: 4.22e-04 | 2117.2ms/step | 9.4 samples/s +Step 11950/44000 | Loss: 3.0391 | LR: 4.22e-04 | 2120.9ms/step | 9.4 samples/s +Step 11960/44000 | Loss: 3.0642 | LR: 4.22e-04 | 2163.4ms/step | 9.2 samples/s +Step 11970/44000 | Loss: 3.0668 | LR: 4.21e-04 | 2152.5ms/step | 9.3 samples/s +Step 11980/44000 | Loss: 3.0961 | LR: 4.21e-04 | 2147.7ms/step | 9.3 samples/s +Step 11990/44000 | Loss: 3.1167 | LR: 4.21e-04 | 2142.4ms/step | 9.3 samples/s +Step 12000/44000 | Loss: 3.0998 | LR: 4.21e-04 | 2115.6ms/step | 9.5 samples/s +Validation at step 12000: Loss = 3.0936, Perplexity = 22.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-12000 +Step 12010/44000 | Loss: 3.0739 | LR: 4.21e-04 | 4402.6ms/step | 4.5 samples/s +Step 12020/44000 | Loss: 3.0500 | LR: 4.21e-04 | 2217.8ms/step | 9.0 samples/s +Step 12030/44000 | Loss: 3.0806 | LR: 4.21e-04 | 2149.2ms/step | 9.3 samples/s +Step 12040/44000 | Loss: 3.0987 | LR: 4.21e-04 | 2164.0ms/step | 9.2 samples/s +Step 12050/44000 | Loss: 3.0787 | LR: 4.20e-04 | 2163.5ms/step | 9.2 samples/s +Step 12060/44000 | Loss: 3.1043 | LR: 4.20e-04 | 2133.6ms/step | 9.4 samples/s +Step 12070/44000 | Loss: 3.0682 | LR: 4.20e-04 | 2141.5ms/step | 9.3 samples/s +Step 12080/44000 | Loss: 3.0997 | LR: 4.20e-04 | 2127.2ms/step | 9.4 samples/s +Step 12090/44000 | Loss: 3.0527 | LR: 4.20e-04 | 2117.0ms/step | 9.4 samples/s +Step 12100/44000 | Loss: 3.1289 | LR: 4.20e-04 | 2081.1ms/step | 9.6 samples/s +Step 12110/44000 | Loss: 3.0772 | LR: 4.20e-04 | 2116.6ms/step | 9.4 samples/s +Step 12120/44000 | Loss: 3.1080 | LR: 4.19e-04 | 2101.3ms/step | 9.5 samples/s +Step 12130/44000 | Loss: 3.0495 | LR: 4.19e-04 | 2044.3ms/step | 9.8 samples/s +Step 12140/44000 | Loss: 3.1049 | LR: 4.19e-04 | 2038.8ms/step | 9.8 samples/s +Step 12150/44000 | Loss: 3.0410 | LR: 4.19e-04 | 2033.5ms/step | 9.8 samples/s +Step 12160/44000 | Loss: 3.0895 | LR: 4.19e-04 | 2033.9ms/step | 9.8 samples/s +Step 12170/44000 | Loss: 3.0870 | LR: 4.19e-04 | 2028.9ms/step | 9.9 samples/s +Step 12180/44000 | Loss: 3.0877 | LR: 4.19e-04 | 2121.5ms/step | 9.4 samples/s +Step 12190/44000 | Loss: 3.0765 | LR: 4.19e-04 | 2147.9ms/step | 9.3 samples/s +Step 12200/44000 | Loss: 3.0380 | LR: 4.18e-04 | 2212.7ms/step | 9.0 samples/s +Step 12210/44000 | Loss: 3.0833 | LR: 4.18e-04 | 2223.5ms/step | 9.0 samples/s +Step 12220/44000 | Loss: 3.0580 | LR: 4.18e-04 | 2429.3ms/step | 8.2 samples/s +Step 12230/44000 | Loss: 3.0475 | LR: 4.18e-04 | 2047.8ms/step | 9.8 samples/s +Step 12240/44000 | Loss: 3.0803 | LR: 4.18e-04 | 2028.3ms/step | 9.9 samples/s +Step 12250/44000 | Loss: 3.1100 | LR: 4.18e-04 | 2063.7ms/step | 9.7 samples/s +Validation at step 12250: Loss = 3.0918, Perplexity = 22.02 +Step 12260/44000 | Loss: 3.0551 | LR: 4.18e-04 | 4013.3ms/step | 5.0 samples/s +Step 12270/44000 | Loss: 3.1028 | LR: 4.17e-04 | 2178.4ms/step | 9.2 samples/s +Step 12280/44000 | Loss: 3.0890 | LR: 4.17e-04 | 2178.8ms/step | 9.2 samples/s +Step 12290/44000 | Loss: 3.0945 | LR: 4.17e-04 | 2121.7ms/step | 9.4 samples/s +Step 12300/44000 | Loss: 3.1198 | LR: 4.17e-04 | 2133.3ms/step | 9.4 samples/s +Step 12310/44000 | Loss: 3.0637 | LR: 4.17e-04 | 2151.0ms/step | 9.3 samples/s +Step 12320/44000 | Loss: 3.0603 | LR: 4.17e-04 | 2126.3ms/step | 9.4 samples/s +Step 12330/44000 | Loss: 3.0872 | LR: 4.17e-04 | 2178.3ms/step | 9.2 samples/s +Step 12340/44000 | Loss: 3.0677 | LR: 4.17e-04 | 2209.9ms/step | 9.1 samples/s +Step 12350/44000 | Loss: 3.0976 | LR: 4.16e-04 | 2175.5ms/step | 9.2 samples/s +Step 12360/44000 | Loss: 3.0951 | LR: 4.16e-04 | 2156.2ms/step | 9.3 samples/s +Step 12370/44000 | Loss: 3.1029 | LR: 4.16e-04 | 2120.0ms/step | 9.4 samples/s +Step 12380/44000 | Loss: 3.1230 | LR: 4.16e-04 | 2111.4ms/step | 9.5 samples/s +Step 12390/44000 | Loss: 3.0764 | LR: 4.16e-04 | 2095.8ms/step | 9.5 samples/s +Step 12400/44000 | Loss: 3.0870 | LR: 4.16e-04 | 2131.4ms/step | 9.4 samples/s +Step 12410/44000 | Loss: 3.0491 | LR: 4.16e-04 | 2143.6ms/step | 9.3 samples/s +Step 12420/44000 | Loss: 3.1008 | LR: 4.15e-04 | 2100.8ms/step | 9.5 samples/s +Step 12430/44000 | Loss: 3.0429 | LR: 4.15e-04 | 2107.2ms/step | 9.5 samples/s +Step 12440/44000 | Loss: 3.0504 | LR: 4.15e-04 | 2131.0ms/step | 9.4 samples/s +Step 12450/44000 | Loss: 3.0962 | LR: 4.15e-04 | 2059.0ms/step | 9.7 samples/s +Step 12460/44000 | Loss: 3.0828 | LR: 4.15e-04 | 2070.5ms/step | 9.7 samples/s +Step 12470/44000 | Loss: 3.0304 | LR: 4.15e-04 | 2053.1ms/step | 9.7 samples/s +Step 12480/44000 | Loss: 3.0892 | LR: 4.15e-04 | 2038.4ms/step | 9.8 samples/s +Step 12490/44000 | Loss: 3.0692 | LR: 4.14e-04 | 2075.2ms/step | 9.6 samples/s +Step 12500/44000 | Loss: 3.0408 | LR: 4.14e-04 | 2111.1ms/step | 9.5 samples/s +Validation at step 12500: Loss = 3.0872, Perplexity = 21.92 +Step 12510/44000 | Loss: 3.0080 | LR: 4.14e-04 | 3920.6ms/step | 5.1 samples/s +Step 12520/44000 | Loss: 3.0853 | LR: 4.14e-04 | 2158.8ms/step | 9.3 samples/s +Step 12530/44000 | Loss: 3.0327 | LR: 4.14e-04 | 2147.1ms/step | 9.3 samples/s +Step 12540/44000 | Loss: 3.0732 | LR: 4.14e-04 | 2446.7ms/step | 8.2 samples/s +Step 12550/44000 | Loss: 3.0287 | LR: 4.14e-04 | 2288.5ms/step | 8.7 samples/s +Step 12560/44000 | Loss: 3.1012 | LR: 4.14e-04 | 2104.4ms/step | 9.5 samples/s +Step 12570/44000 | Loss: 3.0799 | LR: 4.13e-04 | 2127.4ms/step | 9.4 samples/s +Step 12580/44000 | Loss: 3.0716 | LR: 4.13e-04 | 2148.0ms/step | 9.3 samples/s +Step 12590/44000 | Loss: 3.0963 | LR: 4.13e-04 | 2166.2ms/step | 9.2 samples/s +Step 12600/44000 | Loss: 3.0618 | LR: 4.13e-04 | 2154.4ms/step | 9.3 samples/s +Step 12610/44000 | Loss: 3.1099 | LR: 4.13e-04 | 2157.6ms/step | 9.3 samples/s +Step 12620/44000 | Loss: 3.0831 | LR: 4.13e-04 | 2158.7ms/step | 9.3 samples/s +Step 12630/44000 | Loss: 3.0485 | LR: 4.13e-04 | 2178.5ms/step | 9.2 samples/s +Step 12640/44000 | Loss: 3.0670 | LR: 4.12e-04 | 2178.8ms/step | 9.2 samples/s +Step 12650/44000 | Loss: 3.0753 | LR: 4.12e-04 | 2185.5ms/step | 9.2 samples/s +Step 12660/44000 | Loss: 3.0630 | LR: 4.12e-04 | 2182.4ms/step | 9.2 samples/s +Step 12670/44000 | Loss: 3.0487 | LR: 4.12e-04 | 2147.0ms/step | 9.3 samples/s +Step 12680/44000 | Loss: 3.0764 | LR: 4.12e-04 | 2048.3ms/step | 9.8 samples/s +Step 12690/44000 | Loss: 3.0778 | LR: 4.12e-04 | 2075.5ms/step | 9.6 samples/s +Step 12700/44000 | Loss: 3.0453 | LR: 4.12e-04 | 2135.9ms/step | 9.4 samples/s +Step 12710/44000 | Loss: 3.0575 | LR: 4.11e-04 | 2149.6ms/step | 9.3 samples/s +Step 12720/44000 | Loss: 3.0324 | LR: 4.11e-04 | 2137.1ms/step | 9.4 samples/s +Step 12730/44000 | Loss: 3.0769 | LR: 4.11e-04 | 2090.7ms/step | 9.6 samples/s +Step 12740/44000 | Loss: 3.0752 | LR: 4.11e-04 | 2104.9ms/step | 9.5 samples/s +Step 12750/44000 | Loss: 3.0833 | LR: 4.11e-04 | 2043.5ms/step | 9.8 samples/s +Validation at step 12750: Loss = 3.0831, Perplexity = 21.83 +Step 12760/44000 | Loss: 3.0432 | LR: 4.11e-04 | 3801.7ms/step | 5.3 samples/s +Step 12770/44000 | Loss: 3.1004 | LR: 4.11e-04 | 2212.6ms/step | 9.0 samples/s +Step 12780/44000 | Loss: 3.0756 | LR: 4.10e-04 | 2032.6ms/step | 9.8 samples/s +Step 12790/44000 | Loss: 3.0470 | LR: 4.10e-04 | 1975.6ms/step | 10.1 samples/s +Step 12800/44000 | Loss: 3.0388 | LR: 4.10e-04 | 2095.0ms/step | 9.5 samples/s +Step 12810/44000 | Loss: 3.0654 | LR: 4.10e-04 | 2090.0ms/step | 9.6 samples/s +Step 12820/44000 | Loss: 3.1056 | LR: 4.10e-04 | 2126.7ms/step | 9.4 samples/s +Step 12830/44000 | Loss: 3.0267 | LR: 4.10e-04 | 2135.9ms/step | 9.4 samples/s +Step 12840/44000 | Loss: 3.0662 | LR: 4.10e-04 | 2209.6ms/step | 9.1 samples/s +Step 12850/44000 | Loss: 3.0511 | LR: 4.09e-04 | 2124.4ms/step | 9.4 samples/s +Step 12860/44000 | Loss: 3.0489 | LR: 4.09e-04 | 2424.9ms/step | 8.2 samples/s +Step 12870/44000 | Loss: 3.1095 | LR: 4.09e-04 | 2349.9ms/step | 8.5 samples/s +Step 12880/44000 | Loss: 3.0555 | LR: 4.09e-04 | 2151.9ms/step | 9.3 samples/s +Step 12890/44000 | Loss: 3.1017 | LR: 4.09e-04 | 2069.2ms/step | 9.7 samples/s +Step 12900/44000 | Loss: 3.0906 | LR: 4.09e-04 | 2053.1ms/step | 9.7 samples/s +Step 12910/44000 | Loss: 3.0625 | LR: 4.09e-04 | 2043.6ms/step | 9.8 samples/s +Step 12920/44000 | Loss: 3.1142 | LR: 4.09e-04 | 2108.1ms/step | 9.5 samples/s +Step 12930/44000 | Loss: 3.0151 | LR: 4.08e-04 | 2105.7ms/step | 9.5 samples/s +Step 12940/44000 | Loss: 3.0717 | LR: 4.08e-04 | 2108.4ms/step | 9.5 samples/s +Step 12950/44000 | Loss: 3.0438 | LR: 4.08e-04 | 2096.3ms/step | 9.5 samples/s +Step 12960/44000 | Loss: 3.0348 | LR: 4.08e-04 | 2075.1ms/step | 9.6 samples/s +Step 12970/44000 | Loss: 3.0210 | LR: 4.08e-04 | 2058.2ms/step | 9.7 samples/s +Step 12980/44000 | Loss: 3.0589 | LR: 4.08e-04 | 2053.3ms/step | 9.7 samples/s +Step 12990/44000 | Loss: 3.0885 | LR: 4.08e-04 | 2110.2ms/step | 9.5 samples/s +Step 13000/44000 | Loss: 3.0471 | LR: 4.07e-04 | 2117.7ms/step | 9.4 samples/s +Validation at step 13000: Loss = 3.0793, Perplexity = 21.74 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-13000 +Step 13010/44000 | Loss: 3.0228 | LR: 4.07e-04 | 4329.3ms/step | 4.6 samples/s +Step 13020/44000 | Loss: 3.0968 | LR: 4.07e-04 | 2038.0ms/step | 9.8 samples/s +Step 13030/44000 | Loss: 3.0621 | LR: 4.07e-04 | 2105.6ms/step | 9.5 samples/s +Step 13040/44000 | Loss: 3.0463 | LR: 4.07e-04 | 2105.4ms/step | 9.5 samples/s +Step 13050/44000 | Loss: 3.0636 | LR: 4.07e-04 | 2126.4ms/step | 9.4 samples/s +Step 13060/44000 | Loss: 3.0203 | LR: 4.07e-04 | 2117.8ms/step | 9.4 samples/s +Step 13070/44000 | Loss: 3.0281 | LR: 4.06e-04 | 2161.5ms/step | 9.3 samples/s +Step 13080/44000 | Loss: 3.0628 | LR: 4.06e-04 | 2146.2ms/step | 9.3 samples/s +Step 13090/44000 | Loss: 3.0158 | LR: 4.06e-04 | 2143.5ms/step | 9.3 samples/s +Step 13100/44000 | Loss: 3.0441 | LR: 4.06e-04 | 2016.0ms/step | 9.9 samples/s +Step 13110/44000 | Loss: 3.0764 | LR: 4.06e-04 | 2142.4ms/step | 9.3 samples/s +Step 13120/44000 | Loss: 3.0757 | LR: 4.06e-04 | 2152.1ms/step | 9.3 samples/s +Step 13130/44000 | Loss: 3.0460 | LR: 4.06e-04 | 2199.0ms/step | 9.1 samples/s +Step 13140/44000 | Loss: 3.0599 | LR: 4.05e-04 | 2153.5ms/step | 9.3 samples/s +Step 13150/44000 | Loss: 3.0702 | LR: 4.05e-04 | 2173.1ms/step | 9.2 samples/s +Step 13160/44000 | Loss: 3.0579 | LR: 4.05e-04 | 2153.7ms/step | 9.3 samples/s +Step 13170/44000 | Loss: 3.0652 | LR: 4.05e-04 | 2318.4ms/step | 8.6 samples/s +Step 13180/44000 | Loss: 3.0384 | LR: 4.05e-04 | 2105.9ms/step | 9.5 samples/s +Step 13190/44000 | Loss: 3.0513 | LR: 4.05e-04 | 2361.4ms/step | 8.5 samples/s +Step 13200/44000 | Loss: 3.0812 | LR: 4.05e-04 | 2533.1ms/step | 7.9 samples/s +Step 13210/44000 | Loss: 3.0451 | LR: 4.04e-04 | 2126.5ms/step | 9.4 samples/s +Step 13220/44000 | Loss: 3.0366 | LR: 4.04e-04 | 2084.6ms/step | 9.6 samples/s +Step 13230/44000 | Loss: 3.0945 | LR: 4.04e-04 | 2118.6ms/step | 9.4 samples/s +Step 13240/44000 | Loss: 3.0894 | LR: 4.04e-04 | 2095.8ms/step | 9.5 samples/s +Step 13250/44000 | Loss: 3.0794 | LR: 4.04e-04 | 2137.1ms/step | 9.4 samples/s +Validation at step 13250: Loss = 3.0746, Perplexity = 21.64 +Step 13260/44000 | Loss: 3.0775 | LR: 4.04e-04 | 3897.0ms/step | 5.1 samples/s +Step 13270/44000 | Loss: 3.0691 | LR: 4.04e-04 | 2082.7ms/step | 9.6 samples/s +Step 13280/44000 | Loss: 3.0433 | LR: 4.03e-04 | 2106.9ms/step | 9.5 samples/s +Step 13290/44000 | Loss: 3.0348 | LR: 4.03e-04 | 2094.7ms/step | 9.5 samples/s +Step 13300/44000 | Loss: 3.0625 | LR: 4.03e-04 | 2141.7ms/step | 9.3 samples/s +Step 13310/44000 | Loss: 3.0459 | LR: 4.03e-04 | 2107.0ms/step | 9.5 samples/s +Step 13320/44000 | Loss: 3.0239 | LR: 4.03e-04 | 2109.3ms/step | 9.5 samples/s +Step 13330/44000 | Loss: 3.0382 | LR: 4.03e-04 | 2065.5ms/step | 9.7 samples/s +Step 13340/44000 | Loss: 3.0689 | LR: 4.03e-04 | 2043.1ms/step | 9.8 samples/s +Step 13350/44000 | Loss: 3.0652 | LR: 4.02e-04 | 2100.9ms/step | 9.5 samples/s +Step 13360/44000 | Loss: 3.0921 | LR: 4.02e-04 | 2106.1ms/step | 9.5 samples/s +Step 13370/44000 | Loss: 3.0498 | LR: 4.02e-04 | 2090.2ms/step | 9.6 samples/s +Step 13380/44000 | Loss: 3.0595 | LR: 4.02e-04 | 2031.0ms/step | 9.8 samples/s +Step 13390/44000 | Loss: 3.0454 | LR: 4.02e-04 | 2023.8ms/step | 9.9 samples/s +Step 13400/44000 | Loss: 3.0769 | LR: 4.02e-04 | 2024.9ms/step | 9.9 samples/s +Step 13410/44000 | Loss: 3.0077 | LR: 4.02e-04 | 2009.8ms/step | 10.0 samples/s +Step 13420/44000 | Loss: 3.0586 | LR: 4.01e-04 | 2063.9ms/step | 9.7 samples/s +Step 13430/44000 | Loss: 3.0456 | LR: 4.01e-04 | 2090.2ms/step | 9.6 samples/s +Step 13440/44000 | Loss: 3.0942 | LR: 4.01e-04 | 2106.1ms/step | 9.5 samples/s +Step 13450/44000 | Loss: 3.0591 | LR: 4.01e-04 | 2099.2ms/step | 9.5 samples/s +Step 13460/44000 | Loss: 3.0256 | LR: 4.01e-04 | 2112.2ms/step | 9.5 samples/s +Step 13470/44000 | Loss: 3.0454 | LR: 4.01e-04 | 2107.0ms/step | 9.5 samples/s +Step 13480/44000 | Loss: 3.0076 | LR: 4.01e-04 | 2094.4ms/step | 9.5 samples/s +Step 13490/44000 | Loss: 3.0461 | LR: 4.00e-04 | 2105.5ms/step | 9.5 samples/s +Step 13500/44000 | Loss: 3.0613 | LR: 4.00e-04 | 2150.9ms/step | 9.3 samples/s +Validation at step 13500: Loss = 3.0715, Perplexity = 21.57 +Step 13510/44000 | Loss: 3.0716 | LR: 4.00e-04 | 4168.3ms/step | 4.8 samples/s +Step 13520/44000 | Loss: 3.0415 | LR: 4.00e-04 | 2403.0ms/step | 8.3 samples/s +Step 13530/44000 | Loss: 3.0503 | LR: 4.00e-04 | 2011.6ms/step | 9.9 samples/s +Step 13540/44000 | Loss: 3.0377 | LR: 4.00e-04 | 1997.0ms/step | 10.0 samples/s +Step 13550/44000 | Loss: 3.0321 | LR: 4.00e-04 | 2007.3ms/step | 10.0 samples/s +Step 13560/44000 | Loss: 3.0394 | LR: 3.99e-04 | 2041.1ms/step | 9.8 samples/s +Step 13570/44000 | Loss: 3.0538 | LR: 3.99e-04 | 2030.3ms/step | 9.9 samples/s +Step 13580/44000 | Loss: 3.0313 | LR: 3.99e-04 | 2043.8ms/step | 9.8 samples/s +Step 13590/44000 | Loss: 3.0566 | LR: 3.99e-04 | 2073.3ms/step | 9.6 samples/s +Step 13600/44000 | Loss: 3.0948 | LR: 3.99e-04 | 2118.9ms/step | 9.4 samples/s +Step 13610/44000 | Loss: 3.0882 | LR: 3.99e-04 | 2095.2ms/step | 9.5 samples/s +Step 13620/44000 | Loss: 3.0532 | LR: 3.98e-04 | 2063.4ms/step | 9.7 samples/s +Step 13630/44000 | Loss: 3.0744 | LR: 3.98e-04 | 2064.7ms/step | 9.7 samples/s +Step 13640/44000 | Loss: 3.0388 | LR: 3.98e-04 | 2110.5ms/step | 9.5 samples/s +Step 13650/44000 | Loss: 3.0719 | LR: 3.98e-04 | 2110.6ms/step | 9.5 samples/s +Step 13660/44000 | Loss: 3.0939 | LR: 3.98e-04 | 2146.9ms/step | 9.3 samples/s +Step 13670/44000 | Loss: 3.0501 | LR: 3.98e-04 | 2209.1ms/step | 9.1 samples/s +Step 13680/44000 | Loss: 3.0307 | LR: 3.98e-04 | 2182.9ms/step | 9.2 samples/s +Step 13690/44000 | Loss: 3.0071 | LR: 3.97e-04 | 2181.7ms/step | 9.2 samples/s +Step 13700/44000 | Loss: 3.0346 | LR: 3.97e-04 | 2295.9ms/step | 8.7 samples/s +Step 13710/44000 | Loss: 3.0207 | LR: 3.97e-04 | 2110.0ms/step | 9.5 samples/s +Step 13720/44000 | Loss: 3.0604 | LR: 3.97e-04 | 2038.1ms/step | 9.8 samples/s +Step 13730/44000 | Loss: 3.0292 | LR: 3.97e-04 | 2197.8ms/step | 9.1 samples/s +Step 13740/44000 | Loss: 3.0364 | LR: 3.97e-04 | 2079.8ms/step | 9.6 samples/s +Step 13750/44000 | Loss: 3.0361 | LR: 3.97e-04 | 2116.6ms/step | 9.4 samples/s +Validation at step 13750: Loss = 3.0665, Perplexity = 21.47 +Step 13760/44000 | Loss: 3.0741 | LR: 3.96e-04 | 4012.5ms/step | 5.0 samples/s +Step 13770/44000 | Loss: 3.0394 | LR: 3.96e-04 | 2150.1ms/step | 9.3 samples/s +Step 13780/44000 | Loss: 3.0509 | LR: 3.96e-04 | 2101.8ms/step | 9.5 samples/s +Step 13790/44000 | Loss: 3.0421 | LR: 3.96e-04 | 2192.2ms/step | 9.1 samples/s +Step 13800/44000 | Loss: 3.0794 | LR: 3.96e-04 | 2174.4ms/step | 9.2 samples/s +Step 13810/44000 | Loss: 3.0558 | LR: 3.96e-04 | 2152.2ms/step | 9.3 samples/s +Step 13820/44000 | Loss: 3.0650 | LR: 3.96e-04 | 2256.7ms/step | 8.9 samples/s +Step 13830/44000 | Loss: 3.0767 | LR: 3.95e-04 | 2225.3ms/step | 9.0 samples/s +Step 13840/44000 | Loss: 3.0575 | LR: 3.95e-04 | 2162.3ms/step | 9.2 samples/s +Step 13850/44000 | Loss: 3.0215 | LR: 3.95e-04 | 2163.1ms/step | 9.2 samples/s +Step 13860/44000 | Loss: 3.0717 | LR: 3.95e-04 | 2201.0ms/step | 9.1 samples/s +Step 13870/44000 | Loss: 3.0640 | LR: 3.95e-04 | 2245.3ms/step | 8.9 samples/s +Step 13880/44000 | Loss: 3.0442 | LR: 3.95e-04 | 2246.2ms/step | 8.9 samples/s +Step 13890/44000 | Loss: 3.0004 | LR: 3.95e-04 | 2231.0ms/step | 9.0 samples/s +Step 13900/44000 | Loss: 3.0503 | LR: 3.94e-04 | 2377.4ms/step | 8.4 samples/s +Step 13910/44000 | Loss: 3.0391 | LR: 3.94e-04 | 2279.1ms/step | 8.8 samples/s +Step 13920/44000 | Loss: 3.0276 | LR: 3.94e-04 | 2270.6ms/step | 8.8 samples/s +Step 13930/44000 | Loss: 3.0765 | LR: 3.94e-04 | 2314.0ms/step | 8.6 samples/s +Step 13940/44000 | Loss: 3.0505 | LR: 3.94e-04 | 2278.4ms/step | 8.8 samples/s +Step 13950/44000 | Loss: 3.0710 | LR: 3.94e-04 | 2192.9ms/step | 9.1 samples/s +Step 13960/44000 | Loss: 3.0497 | LR: 3.93e-04 | 2147.1ms/step | 9.3 samples/s +Step 13970/44000 | Loss: 3.0203 | LR: 3.93e-04 | 2209.3ms/step | 9.1 samples/s +Step 13980/44000 | Loss: 3.0541 | LR: 3.93e-04 | 2219.8ms/step | 9.0 samples/s +Step 13990/44000 | Loss: 3.0488 | LR: 3.93e-04 | 2283.0ms/step | 8.8 samples/s +Step 14000/44000 | Loss: 3.0575 | LR: 3.93e-04 | 2354.8ms/step | 8.5 samples/s +Validation at step 14000: Loss = 3.0624, Perplexity = 21.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-14000 +Step 14010/44000 | Loss: 3.0835 | LR: 3.93e-04 | 4951.6ms/step | 4.0 samples/s +Step 14020/44000 | Loss: 3.0608 | LR: 3.93e-04 | 2180.4ms/step | 9.2 samples/s +Step 14030/44000 | Loss: 3.0472 | LR: 3.92e-04 | 2052.0ms/step | 9.7 samples/s +Step 14040/44000 | Loss: 3.0664 | LR: 3.92e-04 | 2223.1ms/step | 9.0 samples/s +Step 14050/44000 | Loss: 3.0405 | LR: 3.92e-04 | 2326.5ms/step | 8.6 samples/s +Step 14060/44000 | Loss: 3.0432 | LR: 3.92e-04 | 2217.5ms/step | 9.0 samples/s +Step 14070/44000 | Loss: 3.0479 | LR: 3.92e-04 | 2222.7ms/step | 9.0 samples/s +Step 14080/44000 | Loss: 3.0422 | LR: 3.92e-04 | 2324.8ms/step | 8.6 samples/s +Step 14090/44000 | Loss: 3.0471 | LR: 3.92e-04 | 2261.6ms/step | 8.8 samples/s +Step 14100/44000 | Loss: 3.0122 | LR: 3.91e-04 | 2251.4ms/step | 8.9 samples/s +Step 14110/44000 | Loss: 3.0315 | LR: 3.91e-04 | 2234.8ms/step | 8.9 samples/s +Step 14120/44000 | Loss: 3.0055 | LR: 3.91e-04 | 2229.8ms/step | 9.0 samples/s +Step 14130/44000 | Loss: 3.0369 | LR: 3.91e-04 | 2279.2ms/step | 8.8 samples/s +Step 14140/44000 | Loss: 3.0422 | LR: 3.91e-04 | 2271.7ms/step | 8.8 samples/s +Step 14150/44000 | Loss: 3.0602 | LR: 3.91e-04 | 2234.2ms/step | 9.0 samples/s +Step 14160/44000 | Loss: 3.0087 | LR: 3.90e-04 | 2389.0ms/step | 8.4 samples/s +Step 14170/44000 | Loss: 3.0604 | LR: 3.90e-04 | 2374.9ms/step | 8.4 samples/s +Step 14180/44000 | Loss: 3.0664 | LR: 3.90e-04 | 2326.6ms/step | 8.6 samples/s +Step 14190/44000 | Loss: 3.0504 | LR: 3.90e-04 | 2261.2ms/step | 8.8 samples/s +Step 14200/44000 | Loss: 3.0428 | LR: 3.90e-04 | 2271.5ms/step | 8.8 samples/s +Step 14210/44000 | Loss: 3.0513 | LR: 3.90e-04 | 2184.9ms/step | 9.2 samples/s +Step 14220/44000 | Loss: 3.0065 | LR: 3.90e-04 | 2234.7ms/step | 8.9 samples/s +Step 14230/44000 | Loss: 3.0072 | LR: 3.89e-04 | 2283.3ms/step | 8.8 samples/s +Step 14240/44000 | Loss: 3.0358 | LR: 3.89e-04 | 2290.8ms/step | 8.7 samples/s +Step 14250/44000 | Loss: 3.0557 | LR: 3.89e-04 | 2252.8ms/step | 8.9 samples/s +Validation at step 14250: Loss = 3.0580, Perplexity = 21.29 +Step 14260/44000 | Loss: 3.0414 | LR: 3.89e-04 | 4221.2ms/step | 4.7 samples/s +Step 14270/44000 | Loss: 3.0638 | LR: 3.89e-04 | 2234.6ms/step | 9.0 samples/s +Step 14280/44000 | Loss: 3.0647 | LR: 3.89e-04 | 2253.2ms/step | 8.9 samples/s +Step 14290/44000 | Loss: 3.0496 | LR: 3.89e-04 | 2240.5ms/step | 8.9 samples/s +Step 14300/44000 | Loss: 3.0259 | LR: 3.88e-04 | 2286.8ms/step | 8.7 samples/s +Step 14310/44000 | Loss: 3.0602 | LR: 3.88e-04 | 2266.0ms/step | 8.8 samples/s +Step 14320/44000 | Loss: 3.0648 | LR: 3.88e-04 | 2308.7ms/step | 8.7 samples/s +Step 14330/44000 | Loss: 3.0228 | LR: 3.88e-04 | 2179.5ms/step | 9.2 samples/s +Step 14340/44000 | Loss: 3.0001 | LR: 3.88e-04 | 2204.9ms/step | 9.1 samples/s +Step 14350/44000 | Loss: 3.0186 | LR: 3.88e-04 | 2320.0ms/step | 8.6 samples/s +Step 14360/44000 | Loss: 3.0636 | LR: 3.87e-04 | 2272.5ms/step | 8.8 samples/s +Step 14370/44000 | Loss: 3.0034 | LR: 3.87e-04 | 2183.5ms/step | 9.2 samples/s +Step 14380/44000 | Loss: 3.0537 | LR: 3.87e-04 | 2295.5ms/step | 8.7 samples/s +Step 14390/44000 | Loss: 3.0546 | LR: 3.87e-04 | 2259.6ms/step | 8.9 samples/s +Step 14400/44000 | Loss: 2.9941 | LR: 3.87e-04 | 2340.6ms/step | 8.5 samples/s +Step 14410/44000 | Loss: 3.0587 | LR: 3.87e-04 | 2321.8ms/step | 8.6 samples/s +Step 14420/44000 | Loss: 3.0153 | LR: 3.87e-04 | 2348.2ms/step | 8.5 samples/s +Step 14430/44000 | Loss: 3.0561 | LR: 3.86e-04 | 2215.2ms/step | 9.0 samples/s +Step 14440/44000 | Loss: 3.0406 | LR: 3.86e-04 | 2277.5ms/step | 8.8 samples/s +Step 14450/44000 | Loss: 3.0703 | LR: 3.86e-04 | 2240.7ms/step | 8.9 samples/s +Step 14460/44000 | Loss: 3.0667 | LR: 3.86e-04 | 2251.6ms/step | 8.9 samples/s +Step 14470/44000 | Loss: 3.0428 | LR: 3.86e-04 | 2262.2ms/step | 8.8 samples/s +Step 14480/44000 | Loss: 3.0258 | LR: 3.86e-04 | 2304.0ms/step | 8.7 samples/s +Step 14490/44000 | Loss: 3.0247 | LR: 3.85e-04 | 2349.8ms/step | 8.5 samples/s +Step 14500/44000 | Loss: 3.0068 | LR: 3.85e-04 | 2342.7ms/step | 8.5 samples/s +Validation at step 14500: Loss = 3.0551, Perplexity = 21.22 +Step 14510/44000 | Loss: 2.9897 | LR: 3.85e-04 | 4208.9ms/step | 4.8 samples/s +Step 14520/44000 | Loss: 3.0068 | LR: 3.85e-04 | 2207.8ms/step | 9.1 samples/s +Step 14530/44000 | Loss: 3.0351 | LR: 3.85e-04 | 2204.1ms/step | 9.1 samples/s +Step 14540/44000 | Loss: 3.0516 | LR: 3.85e-04 | 2145.0ms/step | 9.3 samples/s +Step 14550/44000 | Loss: 3.0529 | LR: 3.85e-04 | 2227.3ms/step | 9.0 samples/s +Step 14560/44000 | Loss: 3.0445 | LR: 3.84e-04 | 2344.4ms/step | 8.5 samples/s +Step 14570/44000 | Loss: 3.0398 | LR: 3.84e-04 | 2381.0ms/step | 8.4 samples/s +Step 14580/44000 | Loss: 3.0716 | LR: 3.84e-04 | 2324.4ms/step | 8.6 samples/s +Step 14590/44000 | Loss: 2.9947 | LR: 3.84e-04 | 2303.3ms/step | 8.7 samples/s +Step 14600/44000 | Loss: 3.0765 | LR: 3.84e-04 | 2234.1ms/step | 9.0 samples/s +Step 14610/44000 | Loss: 3.0636 | LR: 3.84e-04 | 2260.1ms/step | 8.8 samples/s +Step 14620/44000 | Loss: 3.0093 | LR: 3.83e-04 | 2153.2ms/step | 9.3 samples/s +Step 14630/44000 | Loss: 3.0260 | LR: 3.83e-04 | 2252.9ms/step | 8.9 samples/s +Step 14640/44000 | Loss: 3.0364 | LR: 3.83e-04 | 2294.5ms/step | 8.7 samples/s +Step 14650/44000 | Loss: 3.0018 | LR: 3.83e-04 | 2203.8ms/step | 9.1 samples/s +Step 14660/44000 | Loss: 3.0452 | LR: 3.83e-04 | 2193.2ms/step | 9.1 samples/s +Step 14670/44000 | Loss: 3.0678 | LR: 3.83e-04 | 2328.7ms/step | 8.6 samples/s +Step 14680/44000 | Loss: 3.0340 | LR: 3.83e-04 | 2220.8ms/step | 9.0 samples/s +Step 14690/44000 | Loss: 3.0110 | LR: 3.82e-04 | 2275.3ms/step | 8.8 samples/s +Step 14700/44000 | Loss: 3.0029 | LR: 3.82e-04 | 2191.7ms/step | 9.1 samples/s +Step 14710/44000 | Loss: 2.9996 | LR: 3.82e-04 | 2348.9ms/step | 8.5 samples/s +Step 14720/44000 | Loss: 3.0565 | LR: 3.82e-04 | 2331.7ms/step | 8.6 samples/s +Step 14730/44000 | Loss: 2.9984 | LR: 3.82e-04 | 2262.9ms/step | 8.8 samples/s +Step 14740/44000 | Loss: 3.0247 | LR: 3.82e-04 | 2293.1ms/step | 8.7 samples/s +Step 14750/44000 | Loss: 3.0295 | LR: 3.81e-04 | 2366.3ms/step | 8.5 samples/s +Validation at step 14750: Loss = 3.0516, Perplexity = 21.15 +Step 14760/44000 | Loss: 3.0190 | LR: 3.81e-04 | 4122.4ms/step | 4.9 samples/s +Step 14770/44000 | Loss: 3.0611 | LR: 3.81e-04 | 2195.7ms/step | 9.1 samples/s +Step 14780/44000 | Loss: 3.0411 | LR: 3.81e-04 | 2190.8ms/step | 9.1 samples/s +Step 14790/44000 | Loss: 3.0261 | LR: 3.81e-04 | 2330.5ms/step | 8.6 samples/s +Step 14800/44000 | Loss: 3.0475 | LR: 3.81e-04 | 2282.0ms/step | 8.8 samples/s +Step 14810/44000 | Loss: 3.0385 | LR: 3.81e-04 | 2308.8ms/step | 8.7 samples/s +Step 14820/44000 | Loss: 3.0259 | LR: 3.80e-04 | 2228.0ms/step | 9.0 samples/s +Step 14830/44000 | Loss: 3.0558 | LR: 3.80e-04 | 2218.4ms/step | 9.0 samples/s +Step 14840/44000 | Loss: 3.0330 | LR: 3.80e-04 | 2158.1ms/step | 9.3 samples/s +Step 14850/44000 | Loss: 3.0367 | LR: 3.80e-04 | 2226.3ms/step | 9.0 samples/s +Step 14860/44000 | Loss: 3.0224 | LR: 3.80e-04 | 2230.6ms/step | 9.0 samples/s +Step 14870/44000 | Loss: 3.0487 | LR: 3.80e-04 | 2299.8ms/step | 8.7 samples/s +Step 14880/44000 | Loss: 3.0305 | LR: 3.79e-04 | 2335.4ms/step | 8.6 samples/s +Step 14890/44000 | Loss: 3.0358 | LR: 3.79e-04 | 2293.0ms/step | 8.7 samples/s +Step 14900/44000 | Loss: 3.0362 | LR: 3.79e-04 | 2299.1ms/step | 8.7 samples/s +Step 14910/44000 | Loss: 2.9811 | LR: 3.79e-04 | 2240.8ms/step | 8.9 samples/s +Step 14920/44000 | Loss: 3.0259 | LR: 3.79e-04 | 2162.7ms/step | 9.2 samples/s +Step 14930/44000 | Loss: 3.0213 | LR: 3.79e-04 | 2179.2ms/step | 9.2 samples/s +Step 14940/44000 | Loss: 3.0440 | LR: 3.79e-04 | 2189.3ms/step | 9.1 samples/s +Step 14950/44000 | Loss: 3.0046 | LR: 3.78e-04 | 2407.9ms/step | 8.3 samples/s +Step 14960/44000 | Loss: 3.0576 | LR: 3.78e-04 | 2203.1ms/step | 9.1 samples/s +Step 14970/44000 | Loss: 3.0350 | LR: 3.78e-04 | 2116.9ms/step | 9.4 samples/s +Step 14980/44000 | Loss: 3.0191 | LR: 3.78e-04 | 2258.2ms/step | 8.9 samples/s +Step 14990/44000 | Loss: 3.0083 | LR: 3.78e-04 | 2212.9ms/step | 9.0 samples/s +Step 15000/44000 | Loss: 2.9903 | LR: 3.78e-04 | 2115.7ms/step | 9.5 samples/s +Validation at step 15000: Loss = 3.0491, Perplexity = 21.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-15000 +Step 15010/44000 | Loss: 3.0785 | LR: 3.77e-04 | 4475.2ms/step | 4.5 samples/s +Step 15020/44000 | Loss: 3.0007 | LR: 3.77e-04 | 2198.8ms/step | 9.1 samples/s +Step 15030/44000 | Loss: 3.0211 | LR: 3.77e-04 | 2210.1ms/step | 9.0 samples/s +Step 15040/44000 | Loss: 3.0195 | LR: 3.77e-04 | 2178.8ms/step | 9.2 samples/s +Step 15050/44000 | Loss: 3.0178 | LR: 3.77e-04 | 2184.8ms/step | 9.2 samples/s +Step 15060/44000 | Loss: 3.0446 | LR: 3.77e-04 | 2173.1ms/step | 9.2 samples/s +Step 15070/44000 | Loss: 2.9887 | LR: 3.77e-04 | 2298.4ms/step | 8.7 samples/s +Step 15080/44000 | Loss: 3.0251 | LR: 3.76e-04 | 2223.6ms/step | 9.0 samples/s +Step 15090/44000 | Loss: 3.0118 | LR: 3.76e-04 | 2305.8ms/step | 8.7 samples/s +Step 15100/44000 | Loss: 3.0496 | LR: 3.76e-04 | 2206.6ms/step | 9.1 samples/s +Step 15110/44000 | Loss: 3.0084 | LR: 3.76e-04 | 2242.8ms/step | 8.9 samples/s +Step 15120/44000 | Loss: 3.0737 | LR: 3.76e-04 | 2202.3ms/step | 9.1 samples/s +Step 15130/44000 | Loss: 3.0321 | LR: 3.76e-04 | 2194.5ms/step | 9.1 samples/s +Step 15140/44000 | Loss: 3.0232 | LR: 3.75e-04 | 2371.7ms/step | 8.4 samples/s +Step 15150/44000 | Loss: 3.0423 | LR: 3.75e-04 | 2192.5ms/step | 9.1 samples/s +Step 15160/44000 | Loss: 3.0347 | LR: 3.75e-04 | 2471.4ms/step | 8.1 samples/s +Step 15170/44000 | Loss: 3.0828 | LR: 3.75e-04 | 2520.9ms/step | 7.9 samples/s +Step 15180/44000 | Loss: 3.0374 | LR: 3.75e-04 | 2211.2ms/step | 9.0 samples/s +Step 15190/44000 | Loss: 3.0240 | LR: 3.75e-04 | 2193.3ms/step | 9.1 samples/s +Step 15200/44000 | Loss: 3.0231 | LR: 3.74e-04 | 2263.2ms/step | 8.8 samples/s +Step 15210/44000 | Loss: 3.0513 | LR: 3.74e-04 | 2178.4ms/step | 9.2 samples/s +Step 15220/44000 | Loss: 3.0136 | LR: 3.74e-04 | 2162.9ms/step | 9.2 samples/s +Step 15230/44000 | Loss: 3.0146 | LR: 3.74e-04 | 2341.2ms/step | 8.5 samples/s +Step 15240/44000 | Loss: 3.0228 | LR: 3.74e-04 | 2171.4ms/step | 9.2 samples/s +Step 15250/44000 | Loss: 3.0390 | LR: 3.74e-04 | 2198.7ms/step | 9.1 samples/s +Validation at step 15250: Loss = 3.0445, Perplexity = 21.00 +Step 15260/44000 | Loss: 3.0029 | LR: 3.74e-04 | 4225.9ms/step | 4.7 samples/s +Step 15270/44000 | Loss: 3.0322 | LR: 3.73e-04 | 2115.2ms/step | 9.5 samples/s +Step 15280/44000 | Loss: 3.0675 | LR: 3.73e-04 | 2093.9ms/step | 9.6 samples/s +Step 15290/44000 | Loss: 3.0221 | LR: 3.73e-04 | 2231.0ms/step | 9.0 samples/s +Step 15300/44000 | Loss: 3.0105 | LR: 3.73e-04 | 2177.9ms/step | 9.2 samples/s +Step 15310/44000 | Loss: 2.9902 | LR: 3.73e-04 | 2178.3ms/step | 9.2 samples/s +Step 15320/44000 | Loss: 3.0219 | LR: 3.73e-04 | 2214.3ms/step | 9.0 samples/s +Step 15330/44000 | Loss: 3.0419 | LR: 3.72e-04 | 2251.1ms/step | 8.9 samples/s +Step 15340/44000 | Loss: 3.0477 | LR: 3.72e-04 | 2250.5ms/step | 8.9 samples/s +Step 15350/44000 | Loss: 3.0284 | LR: 3.72e-04 | 2242.5ms/step | 8.9 samples/s +Step 15360/44000 | Loss: 3.0458 | LR: 3.72e-04 | 2197.1ms/step | 9.1 samples/s +Step 15370/44000 | Loss: 3.0054 | LR: 3.72e-04 | 2224.0ms/step | 9.0 samples/s +Step 15380/44000 | Loss: 3.0335 | LR: 3.72e-04 | 2247.2ms/step | 8.9 samples/s +Step 15390/44000 | Loss: 3.0103 | LR: 3.71e-04 | 2151.2ms/step | 9.3 samples/s +Step 15400/44000 | Loss: 3.0616 | LR: 3.71e-04 | 2292.6ms/step | 8.7 samples/s +Step 15410/44000 | Loss: 3.0344 | LR: 3.71e-04 | 2196.6ms/step | 9.1 samples/s +Step 15420/44000 | Loss: 3.0492 | LR: 3.71e-04 | 2180.0ms/step | 9.2 samples/s +Step 15430/44000 | Loss: 3.0411 | LR: 3.71e-04 | 2168.1ms/step | 9.2 samples/s +Step 15440/44000 | Loss: 2.9944 | LR: 3.71e-04 | 2279.4ms/step | 8.8 samples/s +Step 15450/44000 | Loss: 3.0245 | LR: 3.71e-04 | 2191.0ms/step | 9.1 samples/s +Step 15460/44000 | Loss: 3.0357 | LR: 3.70e-04 | 2255.7ms/step | 8.9 samples/s +Step 15470/44000 | Loss: 3.0103 | LR: 3.70e-04 | 2162.2ms/step | 9.3 samples/s +Step 15480/44000 | Loss: 3.0265 | LR: 3.70e-04 | 2246.3ms/step | 8.9 samples/s +Step 15490/44000 | Loss: 3.0206 | LR: 3.70e-04 | 2260.7ms/step | 8.8 samples/s +Step 15500/44000 | Loss: 3.0424 | LR: 3.70e-04 | 2215.6ms/step | 9.0 samples/s +Validation at step 15500: Loss = 3.0419, Perplexity = 20.94 +Step 15510/44000 | Loss: 3.0273 | LR: 3.70e-04 | 4215.8ms/step | 4.7 samples/s +Step 15520/44000 | Loss: 3.0267 | LR: 3.69e-04 | 2215.0ms/step | 9.0 samples/s +Step 15530/44000 | Loss: 3.0048 | LR: 3.69e-04 | 2198.6ms/step | 9.1 samples/s +Step 15540/44000 | Loss: 3.0151 | LR: 3.69e-04 | 2225.1ms/step | 9.0 samples/s +Step 15550/44000 | Loss: 3.0542 | LR: 3.69e-04 | 2223.2ms/step | 9.0 samples/s +Step 15560/44000 | Loss: 3.0249 | LR: 3.69e-04 | 2243.7ms/step | 8.9 samples/s +Step 15570/44000 | Loss: 3.0501 | LR: 3.69e-04 | 2269.9ms/step | 8.8 samples/s +Step 15580/44000 | Loss: 2.9994 | LR: 3.68e-04 | 2173.0ms/step | 9.2 samples/s +Step 15590/44000 | Loss: 3.0854 | LR: 3.68e-04 | 2185.1ms/step | 9.2 samples/s +Step 15600/44000 | Loss: 2.9983 | LR: 3.68e-04 | 2267.0ms/step | 8.8 samples/s +Step 15610/44000 | Loss: 3.0403 | LR: 3.68e-04 | 2251.4ms/step | 8.9 samples/s +Step 15620/44000 | Loss: 2.9725 | LR: 3.68e-04 | 2293.5ms/step | 8.7 samples/s +Step 15630/44000 | Loss: 3.0080 | LR: 3.68e-04 | 2302.7ms/step | 8.7 samples/s +Step 15640/44000 | Loss: 3.0318 | LR: 3.67e-04 | 2153.5ms/step | 9.3 samples/s +Step 15650/44000 | Loss: 3.0185 | LR: 3.67e-04 | 2266.8ms/step | 8.8 samples/s +Step 15660/44000 | Loss: 3.0285 | LR: 3.67e-04 | 2235.6ms/step | 8.9 samples/s +Step 15670/44000 | Loss: 3.0188 | LR: 3.67e-04 | 2322.7ms/step | 8.6 samples/s +Step 15680/44000 | Loss: 3.0098 | LR: 3.67e-04 | 2373.3ms/step | 8.4 samples/s +Step 15690/44000 | Loss: 3.0463 | LR: 3.67e-04 | 2294.2ms/step | 8.7 samples/s +Step 15700/44000 | Loss: 3.0214 | LR: 3.67e-04 | 2239.4ms/step | 8.9 samples/s +Step 15710/44000 | Loss: 3.0270 | LR: 3.66e-04 | 2293.6ms/step | 8.7 samples/s +Step 15720/44000 | Loss: 3.0513 | LR: 3.66e-04 | 2270.7ms/step | 8.8 samples/s +Step 15730/44000 | Loss: 3.0808 | LR: 3.66e-04 | 2146.9ms/step | 9.3 samples/s +Step 15740/44000 | Loss: 3.0331 | LR: 3.66e-04 | 2296.8ms/step | 8.7 samples/s +Step 15750/44000 | Loss: 3.0207 | LR: 3.66e-04 | 2283.6ms/step | 8.8 samples/s +Validation at step 15750: Loss = 3.0384, Perplexity = 20.87 +Step 15760/44000 | Loss: 2.9941 | LR: 3.66e-04 | 4309.8ms/step | 4.6 samples/s +Step 15770/44000 | Loss: 3.0190 | LR: 3.65e-04 | 2319.7ms/step | 8.6 samples/s +Step 15780/44000 | Loss: 3.0294 | LR: 3.65e-04 | 2314.8ms/step | 8.6 samples/s +Step 15790/44000 | Loss: 3.0493 | LR: 3.65e-04 | 2324.2ms/step | 8.6 samples/s +Step 15800/44000 | Loss: 2.9950 | LR: 3.65e-04 | 2649.5ms/step | 7.5 samples/s +Step 15810/44000 | Loss: 3.0150 | LR: 3.65e-04 | 2870.4ms/step | 7.0 samples/s +Step 15820/44000 | Loss: 3.0411 | LR: 3.65e-04 | 3007.3ms/step | 6.7 samples/s +Step 15830/44000 | Loss: 2.9764 | LR: 3.64e-04 | 2899.3ms/step | 6.9 samples/s +Step 15840/44000 | Loss: 3.0274 | LR: 3.64e-04 | 3016.4ms/step | 6.6 samples/s +Step 15850/44000 | Loss: 2.9932 | LR: 3.64e-04 | 3004.8ms/step | 6.7 samples/s +Step 15860/44000 | Loss: 3.0302 | LR: 3.64e-04 | 3006.5ms/step | 6.7 samples/s +Step 15870/44000 | Loss: 2.9755 | LR: 3.64e-04 | 2957.0ms/step | 6.8 samples/s +Step 15880/44000 | Loss: 3.0222 | LR: 3.64e-04 | 2926.5ms/step | 6.8 samples/s +Step 15890/44000 | Loss: 3.0259 | LR: 3.63e-04 | 2902.2ms/step | 6.9 samples/s +Step 15900/44000 | Loss: 2.9981 | LR: 3.63e-04 | 3055.7ms/step | 6.5 samples/s +Step 15910/44000 | Loss: 3.0301 | LR: 3.63e-04 | 3081.8ms/step | 6.5 samples/s +Step 15920/44000 | Loss: 3.0026 | LR: 3.63e-04 | 3000.2ms/step | 6.7 samples/s +Step 15930/44000 | Loss: 3.0316 | LR: 3.63e-04 | 2997.7ms/step | 6.7 samples/s +Step 15940/44000 | Loss: 3.0256 | LR: 3.63e-04 | 3132.2ms/step | 6.4 samples/s +Step 15950/44000 | Loss: 3.0390 | LR: 3.62e-04 | 3006.3ms/step | 6.7 samples/s +Step 15960/44000 | Loss: 3.0361 | LR: 3.62e-04 | 3164.1ms/step | 6.3 samples/s +Step 15970/44000 | Loss: 3.0084 | LR: 3.62e-04 | 3023.1ms/step | 6.6 samples/s +Step 15980/44000 | Loss: 3.0096 | LR: 3.62e-04 | 2749.3ms/step | 7.3 samples/s +Step 15990/44000 | Loss: 2.9986 | LR: 3.62e-04 | 2247.7ms/step | 8.9 samples/s +Step 16000/44000 | Loss: 3.0314 | LR: 3.62e-04 | 2261.6ms/step | 8.8 samples/s +Validation at step 16000: Loss = 3.0342, Perplexity = 20.79 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-16000 +Step 16010/44000 | Loss: 2.9962 | LR: 3.62e-04 | 4889.2ms/step | 4.1 samples/s +Step 16020/44000 | Loss: 3.0408 | LR: 3.61e-04 | 2359.7ms/step | 8.5 samples/s +Step 16030/44000 | Loss: 3.0176 | LR: 3.61e-04 | 2407.9ms/step | 8.3 samples/s +Step 16040/44000 | Loss: 2.9885 | LR: 3.61e-04 | 2258.1ms/step | 8.9 samples/s +Step 16050/44000 | Loss: 3.0015 | LR: 3.61e-04 | 2253.4ms/step | 8.9 samples/s +Step 16060/44000 | Loss: 3.0237 | LR: 3.61e-04 | 2410.9ms/step | 8.3 samples/s +Step 16070/44000 | Loss: 3.0082 | LR: 3.61e-04 | 2287.5ms/step | 8.7 samples/s +Step 16080/44000 | Loss: 3.0247 | LR: 3.60e-04 | 2350.6ms/step | 8.5 samples/s +Step 16090/44000 | Loss: 3.0181 | LR: 3.60e-04 | 2293.0ms/step | 8.7 samples/s +Step 16100/44000 | Loss: 3.0359 | LR: 3.60e-04 | 2313.4ms/step | 8.6 samples/s +Step 16110/44000 | Loss: 2.9882 | LR: 3.60e-04 | 2330.0ms/step | 8.6 samples/s +Step 16120/44000 | Loss: 3.0326 | LR: 3.60e-04 | 2350.2ms/step | 8.5 samples/s +Step 16130/44000 | Loss: 3.0247 | LR: 3.60e-04 | 2282.4ms/step | 8.8 samples/s +Step 16140/44000 | Loss: 2.9920 | LR: 3.59e-04 | 2340.3ms/step | 8.5 samples/s +Step 16150/44000 | Loss: 3.0389 | LR: 3.59e-04 | 2460.1ms/step | 8.1 samples/s +Step 16160/44000 | Loss: 3.0218 | LR: 3.59e-04 | 2268.3ms/step | 8.8 samples/s +Step 16170/44000 | Loss: 3.0074 | LR: 3.59e-04 | 2340.3ms/step | 8.5 samples/s +Step 16180/44000 | Loss: 3.0197 | LR: 3.59e-04 | 2335.1ms/step | 8.6 samples/s +Step 16190/44000 | Loss: 3.0151 | LR: 3.59e-04 | 2249.6ms/step | 8.9 samples/s +Step 16200/44000 | Loss: 2.9995 | LR: 3.58e-04 | 2223.2ms/step | 9.0 samples/s +Step 16210/44000 | Loss: 3.0139 | LR: 3.58e-04 | 2336.2ms/step | 8.6 samples/s +Step 16220/44000 | Loss: 3.0389 | LR: 3.58e-04 | 2386.8ms/step | 8.4 samples/s +Step 16230/44000 | Loss: 3.0064 | LR: 3.58e-04 | 2311.5ms/step | 8.7 samples/s +Step 16240/44000 | Loss: 3.0028 | LR: 3.58e-04 | 2302.1ms/step | 8.7 samples/s +Step 16250/44000 | Loss: 2.9884 | LR: 3.58e-04 | 2246.6ms/step | 8.9 samples/s +Validation at step 16250: Loss = 3.0316, Perplexity = 20.73 +Step 16260/44000 | Loss: 3.0044 | LR: 3.57e-04 | 4347.2ms/step | 4.6 samples/s +Step 16270/44000 | Loss: 2.9820 | LR: 3.57e-04 | 2273.0ms/step | 8.8 samples/s +Step 16280/44000 | Loss: 3.0848 | LR: 3.57e-04 | 2262.2ms/step | 8.8 samples/s +Step 16290/44000 | Loss: 2.9738 | LR: 3.57e-04 | 2323.5ms/step | 8.6 samples/s +Step 16300/44000 | Loss: 3.0080 | LR: 3.57e-04 | 2432.3ms/step | 8.2 samples/s +Step 16310/44000 | Loss: 3.0058 | LR: 3.57e-04 | 2273.2ms/step | 8.8 samples/s +Step 16320/44000 | Loss: 3.0175 | LR: 3.56e-04 | 2284.1ms/step | 8.8 samples/s +Step 16330/44000 | Loss: 3.0235 | LR: 3.56e-04 | 2225.3ms/step | 9.0 samples/s +Step 16340/44000 | Loss: 3.0586 | LR: 3.56e-04 | 2335.4ms/step | 8.6 samples/s +Step 16350/44000 | Loss: 3.0062 | LR: 3.56e-04 | 2319.0ms/step | 8.6 samples/s +Step 16360/44000 | Loss: 3.0275 | LR: 3.56e-04 | 2287.9ms/step | 8.7 samples/s +Step 16370/44000 | Loss: 3.0498 | LR: 3.56e-04 | 2294.8ms/step | 8.7 samples/s +Step 16380/44000 | Loss: 3.0439 | LR: 3.55e-04 | 2296.6ms/step | 8.7 samples/s +Step 16390/44000 | Loss: 3.0085 | LR: 3.55e-04 | 2211.2ms/step | 9.0 samples/s +Step 16400/44000 | Loss: 2.9814 | LR: 3.55e-04 | 2292.3ms/step | 8.7 samples/s +Step 16410/44000 | Loss: 3.0220 | LR: 3.55e-04 | 2254.8ms/step | 8.9 samples/s +Step 16420/44000 | Loss: 2.9665 | LR: 3.55e-04 | 2153.9ms/step | 9.3 samples/s +Step 16430/44000 | Loss: 3.0479 | LR: 3.55e-04 | 2302.7ms/step | 8.7 samples/s +Step 16440/44000 | Loss: 3.0015 | LR: 3.54e-04 | 2319.4ms/step | 8.6 samples/s +Step 16450/44000 | Loss: 2.9871 | LR: 3.54e-04 | 2370.9ms/step | 8.4 samples/s +Step 16460/44000 | Loss: 2.9850 | LR: 3.54e-04 | 2230.9ms/step | 9.0 samples/s +Step 16470/44000 | Loss: 3.0151 | LR: 3.54e-04 | 2314.2ms/step | 8.6 samples/s +Step 16480/44000 | Loss: 3.0598 | LR: 3.54e-04 | 2188.7ms/step | 9.1 samples/s +Step 16490/44000 | Loss: 3.0051 | LR: 3.54e-04 | 2221.8ms/step | 9.0 samples/s +Step 16500/44000 | Loss: 3.0551 | LR: 3.53e-04 | 2265.9ms/step | 8.8 samples/s +Validation at step 16500: Loss = 3.0277, Perplexity = 20.65 +Step 16510/44000 | Loss: 3.0326 | LR: 3.53e-04 | 4576.0ms/step | 4.4 samples/s +Step 16520/44000 | Loss: 3.0381 | LR: 3.53e-04 | 2335.5ms/step | 8.6 samples/s +Step 16530/44000 | Loss: 3.0245 | LR: 3.53e-04 | 2315.0ms/step | 8.6 samples/s +Step 16540/44000 | Loss: 2.9943 | LR: 3.53e-04 | 2267.5ms/step | 8.8 samples/s +Step 16550/44000 | Loss: 3.0380 | LR: 3.53e-04 | 2210.1ms/step | 9.0 samples/s +Step 16560/44000 | Loss: 2.9879 | LR: 3.52e-04 | 2153.8ms/step | 9.3 samples/s +Step 16570/44000 | Loss: 3.0296 | LR: 3.52e-04 | 2200.9ms/step | 9.1 samples/s +Step 16580/44000 | Loss: 3.0595 | LR: 3.52e-04 | 2203.2ms/step | 9.1 samples/s +Step 16590/44000 | Loss: 2.9825 | LR: 3.52e-04 | 2201.4ms/step | 9.1 samples/s +Step 16600/44000 | Loss: 3.0118 | LR: 3.52e-04 | 2176.4ms/step | 9.2 samples/s +Step 16610/44000 | Loss: 3.0434 | LR: 3.52e-04 | 2272.1ms/step | 8.8 samples/s +Step 16620/44000 | Loss: 3.0601 | LR: 3.51e-04 | 2257.8ms/step | 8.9 samples/s +Step 16630/44000 | Loss: 3.0404 | LR: 3.51e-04 | 2242.7ms/step | 8.9 samples/s +Step 16640/44000 | Loss: 3.0560 | LR: 3.51e-04 | 2273.9ms/step | 8.8 samples/s +Step 16650/44000 | Loss: 3.0023 | LR: 3.51e-04 | 2228.6ms/step | 9.0 samples/s +Step 16660/44000 | Loss: 3.0044 | LR: 3.51e-04 | 2249.2ms/step | 8.9 samples/s +Step 16670/44000 | Loss: 2.9829 | LR: 3.51e-04 | 2303.1ms/step | 8.7 samples/s +Step 16680/44000 | Loss: 2.9331 | LR: 3.50e-04 | 2326.0ms/step | 8.6 samples/s +Step 16690/44000 | Loss: 3.0037 | LR: 3.50e-04 | 2235.9ms/step | 8.9 samples/s +Step 16700/44000 | Loss: 3.0340 | LR: 3.50e-04 | 2260.8ms/step | 8.8 samples/s +Step 16710/44000 | Loss: 2.9830 | LR: 3.50e-04 | 2240.9ms/step | 8.9 samples/s +Step 16720/44000 | Loss: 3.0068 | LR: 3.50e-04 | 2261.6ms/step | 8.8 samples/s +Step 16730/44000 | Loss: 3.0349 | LR: 3.50e-04 | 2215.5ms/step | 9.0 samples/s +Step 16740/44000 | Loss: 2.9946 | LR: 3.49e-04 | 2266.7ms/step | 8.8 samples/s +Step 16750/44000 | Loss: 3.0008 | LR: 3.49e-04 | 2287.6ms/step | 8.7 samples/s +Validation at step 16750: Loss = 3.0245, Perplexity = 20.58 +Step 16760/44000 | Loss: 2.9824 | LR: 3.49e-04 | 4262.4ms/step | 4.7 samples/s +Step 16770/44000 | Loss: 3.0245 | LR: 3.49e-04 | 2307.9ms/step | 8.7 samples/s +Step 16780/44000 | Loss: 3.0256 | LR: 3.49e-04 | 2355.1ms/step | 8.5 samples/s +Step 16790/44000 | Loss: 3.0181 | LR: 3.49e-04 | 2350.1ms/step | 8.5 samples/s +Step 16800/44000 | Loss: 2.9507 | LR: 3.48e-04 | 2626.0ms/step | 7.6 samples/s +Step 16810/44000 | Loss: 2.9995 | LR: 3.48e-04 | 2799.0ms/step | 7.1 samples/s +Step 16820/44000 | Loss: 3.0212 | LR: 3.48e-04 | 2945.6ms/step | 6.8 samples/s +Step 16830/44000 | Loss: 3.0104 | LR: 3.48e-04 | 2852.0ms/step | 7.0 samples/s +Step 16840/44000 | Loss: 2.9667 | LR: 3.48e-04 | 2975.8ms/step | 6.7 samples/s +Step 16850/44000 | Loss: 3.0133 | LR: 3.48e-04 | 2909.1ms/step | 6.9 samples/s +Step 16860/44000 | Loss: 2.9791 | LR: 3.47e-04 | 2917.3ms/step | 6.9 samples/s +Step 16870/44000 | Loss: 3.0201 | LR: 3.47e-04 | 2973.3ms/step | 6.7 samples/s +Step 16880/44000 | Loss: 2.9764 | LR: 3.47e-04 | 2947.4ms/step | 6.8 samples/s +Step 16890/44000 | Loss: 3.0121 | LR: 3.47e-04 | 3157.5ms/step | 6.3 samples/s +Step 16900/44000 | Loss: 3.0028 | LR: 3.47e-04 | 2966.4ms/step | 6.7 samples/s +Step 16910/44000 | Loss: 2.9641 | LR: 3.47e-04 | 2472.5ms/step | 8.1 samples/s +Step 16920/44000 | Loss: 3.0313 | LR: 3.46e-04 | 2463.4ms/step | 8.1 samples/s +Step 16930/44000 | Loss: 3.0112 | LR: 3.46e-04 | 2270.1ms/step | 8.8 samples/s +Step 16940/44000 | Loss: 2.9861 | LR: 3.46e-04 | 2394.5ms/step | 8.4 samples/s +Step 16950/44000 | Loss: 3.0231 | LR: 3.46e-04 | 2410.8ms/step | 8.3 samples/s +Step 16960/44000 | Loss: 2.9950 | LR: 3.46e-04 | 2313.2ms/step | 8.6 samples/s +Step 16970/44000 | Loss: 2.9947 | LR: 3.46e-04 | 2408.8ms/step | 8.3 samples/s +Step 16980/44000 | Loss: 2.9823 | LR: 3.45e-04 | 2319.9ms/step | 8.6 samples/s +Step 16990/44000 | Loss: 2.9948 | LR: 3.45e-04 | 2388.3ms/step | 8.4 samples/s +Step 17000/44000 | Loss: 3.0432 | LR: 3.45e-04 | 2720.0ms/step | 7.4 samples/s +Validation at step 17000: Loss = 3.0217, Perplexity = 20.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-17000 +Step 17010/44000 | Loss: 3.0260 | LR: 3.45e-04 | 4888.9ms/step | 4.1 samples/s +Step 17020/44000 | Loss: 3.0374 | LR: 3.45e-04 | 2319.6ms/step | 8.6 samples/s +Step 17030/44000 | Loss: 2.9885 | LR: 3.45e-04 | 2199.5ms/step | 9.1 samples/s +Step 17040/44000 | Loss: 3.0191 | LR: 3.44e-04 | 2330.0ms/step | 8.6 samples/s +Step 17050/44000 | Loss: 2.9932 | LR: 3.44e-04 | 2304.5ms/step | 8.7 samples/s +Step 17060/44000 | Loss: 2.9991 | LR: 3.44e-04 | 2318.5ms/step | 8.6 samples/s +Step 17070/44000 | Loss: 2.9893 | LR: 3.44e-04 | 2294.8ms/step | 8.7 samples/s +Step 17080/44000 | Loss: 3.0163 | LR: 3.44e-04 | 2184.4ms/step | 9.2 samples/s +Step 17090/44000 | Loss: 2.9889 | LR: 3.44e-04 | 2285.3ms/step | 8.8 samples/s +Step 17100/44000 | Loss: 2.9964 | LR: 3.43e-04 | 2326.3ms/step | 8.6 samples/s +Step 17110/44000 | Loss: 2.9896 | LR: 3.43e-04 | 2169.7ms/step | 9.2 samples/s +Step 17120/44000 | Loss: 2.9900 | LR: 3.43e-04 | 2153.0ms/step | 9.3 samples/s +Step 17130/44000 | Loss: 3.0029 | LR: 3.43e-04 | 2272.5ms/step | 8.8 samples/s +Step 17140/44000 | Loss: 2.9748 | LR: 3.43e-04 | 2251.5ms/step | 8.9 samples/s +Step 17150/44000 | Loss: 2.9716 | LR: 3.43e-04 | 2260.5ms/step | 8.8 samples/s +Step 17160/44000 | Loss: 2.9938 | LR: 3.42e-04 | 2249.9ms/step | 8.9 samples/s +Step 17170/44000 | Loss: 2.9909 | LR: 3.42e-04 | 2263.7ms/step | 8.8 samples/s +Step 17180/44000 | Loss: 2.9931 | LR: 3.42e-04 | 2268.4ms/step | 8.8 samples/s +Step 17190/44000 | Loss: 2.9993 | LR: 3.42e-04 | 2245.5ms/step | 8.9 samples/s +Step 17200/44000 | Loss: 3.0107 | LR: 3.42e-04 | 2304.8ms/step | 8.7 samples/s +Step 17210/44000 | Loss: 3.0375 | LR: 3.42e-04 | 2285.2ms/step | 8.8 samples/s +Step 17220/44000 | Loss: 2.9712 | LR: 3.41e-04 | 2233.8ms/step | 9.0 samples/s +Step 17230/44000 | Loss: 3.0210 | LR: 3.41e-04 | 2230.9ms/step | 9.0 samples/s +Step 17240/44000 | Loss: 2.9814 | LR: 3.41e-04 | 2238.2ms/step | 8.9 samples/s +Step 17250/44000 | Loss: 2.9629 | LR: 3.41e-04 | 2252.3ms/step | 8.9 samples/s +Validation at step 17250: Loss = 3.0186, Perplexity = 20.46 +Step 17260/44000 | Loss: 3.0053 | LR: 3.41e-04 | 4203.8ms/step | 4.8 samples/s +Step 17270/44000 | Loss: 2.9750 | LR: 3.41e-04 | 2257.6ms/step | 8.9 samples/s +Step 17280/44000 | Loss: 3.0072 | LR: 3.40e-04 | 2286.0ms/step | 8.7 samples/s +Step 17290/44000 | Loss: 3.0206 | LR: 3.40e-04 | 2285.4ms/step | 8.8 samples/s +Step 17300/44000 | Loss: 2.9578 | LR: 3.40e-04 | 2209.1ms/step | 9.1 samples/s +Step 17310/44000 | Loss: 2.9819 | LR: 3.40e-04 | 2283.2ms/step | 8.8 samples/s +Step 17320/44000 | Loss: 2.9915 | LR: 3.40e-04 | 2179.3ms/step | 9.2 samples/s +Step 17330/44000 | Loss: 2.9787 | LR: 3.40e-04 | 2236.3ms/step | 8.9 samples/s +Step 17340/44000 | Loss: 3.0283 | LR: 3.39e-04 | 2218.7ms/step | 9.0 samples/s +Step 17350/44000 | Loss: 2.9851 | LR: 3.39e-04 | 2278.1ms/step | 8.8 samples/s +Step 17360/44000 | Loss: 2.9890 | LR: 3.39e-04 | 2272.7ms/step | 8.8 samples/s +Step 17370/44000 | Loss: 3.0006 | LR: 3.39e-04 | 2341.2ms/step | 8.5 samples/s +Step 17380/44000 | Loss: 2.9947 | LR: 3.39e-04 | 2238.3ms/step | 8.9 samples/s +Step 17390/44000 | Loss: 3.0175 | LR: 3.39e-04 | 2213.1ms/step | 9.0 samples/s +Step 17400/44000 | Loss: 2.9722 | LR: 3.38e-04 | 2236.0ms/step | 8.9 samples/s +Step 17410/44000 | Loss: 3.0182 | LR: 3.38e-04 | 2192.5ms/step | 9.1 samples/s +Step 17420/44000 | Loss: 2.9266 | LR: 3.38e-04 | 2222.5ms/step | 9.0 samples/s +Step 17430/44000 | Loss: 3.0397 | LR: 3.38e-04 | 2164.8ms/step | 9.2 samples/s +Step 17440/44000 | Loss: 3.0084 | LR: 3.38e-04 | 2276.1ms/step | 8.8 samples/s +Step 17450/44000 | Loss: 3.0185 | LR: 3.38e-04 | 2248.7ms/step | 8.9 samples/s +Step 17460/44000 | Loss: 2.9666 | LR: 3.37e-04 | 2366.6ms/step | 8.5 samples/s +Step 17470/44000 | Loss: 3.0097 | LR: 3.37e-04 | 2207.0ms/step | 9.1 samples/s +Step 17480/44000 | Loss: 3.0283 | LR: 3.37e-04 | 2249.5ms/step | 8.9 samples/s +Step 17490/44000 | Loss: 2.9862 | LR: 3.37e-04 | 2215.3ms/step | 9.0 samples/s +Step 17500/44000 | Loss: 3.0163 | LR: 3.37e-04 | 2324.8ms/step | 8.6 samples/s +Validation at step 17500: Loss = 3.0160, Perplexity = 20.41 +Step 17510/44000 | Loss: 3.0268 | LR: 3.37e-04 | 4338.8ms/step | 4.6 samples/s +Step 17520/44000 | Loss: 3.0632 | LR: 3.36e-04 | 2231.1ms/step | 9.0 samples/s +Step 17530/44000 | Loss: 2.9757 | LR: 3.36e-04 | 2278.3ms/step | 8.8 samples/s +Step 17540/44000 | Loss: 3.0078 | LR: 3.36e-04 | 2408.4ms/step | 8.3 samples/s +Step 17550/44000 | Loss: 3.0090 | LR: 3.36e-04 | 2283.0ms/step | 8.8 samples/s +Step 17560/44000 | Loss: 3.0064 | LR: 3.36e-04 | 2299.3ms/step | 8.7 samples/s +Step 17570/44000 | Loss: 2.9899 | LR: 3.35e-04 | 2335.2ms/step | 8.6 samples/s +Step 17580/44000 | Loss: 3.0011 | LR: 3.35e-04 | 2293.8ms/step | 8.7 samples/s +Step 17590/44000 | Loss: 3.0077 | LR: 3.35e-04 | 2248.4ms/step | 8.9 samples/s +Step 17600/44000 | Loss: 2.9777 | LR: 3.35e-04 | 2292.9ms/step | 8.7 samples/s +Step 17610/44000 | Loss: 2.9976 | LR: 3.35e-04 | 2343.6ms/step | 8.5 samples/s +Step 17620/44000 | Loss: 2.9758 | LR: 3.35e-04 | 2299.0ms/step | 8.7 samples/s +Step 17630/44000 | Loss: 2.9813 | LR: 3.34e-04 | 2363.7ms/step | 8.5 samples/s +Step 17640/44000 | Loss: 2.9733 | LR: 3.34e-04 | 2259.5ms/step | 8.9 samples/s +Step 17650/44000 | Loss: 2.9374 | LR: 3.34e-04 | 2380.7ms/step | 8.4 samples/s +Step 17660/44000 | Loss: 2.9743 | LR: 3.34e-04 | 2190.6ms/step | 9.1 samples/s +Step 17670/44000 | Loss: 2.9791 | LR: 3.34e-04 | 2220.2ms/step | 9.0 samples/s +Step 17680/44000 | Loss: 2.9950 | LR: 3.34e-04 | 2319.5ms/step | 8.6 samples/s +Step 17690/44000 | Loss: 2.9902 | LR: 3.33e-04 | 2273.3ms/step | 8.8 samples/s +Step 17700/44000 | Loss: 2.9671 | LR: 3.33e-04 | 2225.0ms/step | 9.0 samples/s +Step 17710/44000 | Loss: 3.0115 | LR: 3.33e-04 | 2284.8ms/step | 8.8 samples/s +Step 17720/44000 | Loss: 3.0155 | LR: 3.33e-04 | 2296.0ms/step | 8.7 samples/s +Step 17730/44000 | Loss: 3.0133 | LR: 3.33e-04 | 2267.1ms/step | 8.8 samples/s +Step 17740/44000 | Loss: 2.9933 | LR: 3.33e-04 | 2127.4ms/step | 9.4 samples/s +Step 17750/44000 | Loss: 2.9909 | LR: 3.32e-04 | 2158.2ms/step | 9.3 samples/s +Validation at step 17750: Loss = 3.0135, Perplexity = 20.36 +Step 17760/44000 | Loss: 2.9920 | LR: 3.32e-04 | 4160.2ms/step | 4.8 samples/s +Step 17770/44000 | Loss: 2.9999 | LR: 3.32e-04 | 2293.7ms/step | 8.7 samples/s +Step 17780/44000 | Loss: 2.9975 | LR: 3.32e-04 | 2361.3ms/step | 8.5 samples/s +Step 17790/44000 | Loss: 2.9996 | LR: 3.32e-04 | 2231.5ms/step | 9.0 samples/s +Step 17800/44000 | Loss: 3.0138 | LR: 3.32e-04 | 2339.5ms/step | 8.5 samples/s +Step 17810/44000 | Loss: 3.0084 | LR: 3.31e-04 | 2273.2ms/step | 8.8 samples/s +Step 17820/44000 | Loss: 3.0160 | LR: 3.31e-04 | 2272.5ms/step | 8.8 samples/s +Step 17830/44000 | Loss: 2.9768 | LR: 3.31e-04 | 2267.6ms/step | 8.8 samples/s +Step 17840/44000 | Loss: 2.9957 | LR: 3.31e-04 | 2272.5ms/step | 8.8 samples/s +Step 17850/44000 | Loss: 2.9539 | LR: 3.31e-04 | 2332.5ms/step | 8.6 samples/s +Step 17860/44000 | Loss: 3.0085 | LR: 3.31e-04 | 2421.1ms/step | 8.3 samples/s +Step 17870/44000 | Loss: 3.0158 | LR: 3.30e-04 | 2241.2ms/step | 8.9 samples/s +Step 17880/44000 | Loss: 2.9871 | LR: 3.30e-04 | 2283.2ms/step | 8.8 samples/s +Step 17890/44000 | Loss: 2.9959 | LR: 3.30e-04 | 2314.8ms/step | 8.6 samples/s +Step 17900/44000 | Loss: 3.0272 | LR: 3.30e-04 | 2231.1ms/step | 9.0 samples/s +Step 17910/44000 | Loss: 2.9949 | LR: 3.30e-04 | 2267.3ms/step | 8.8 samples/s +Step 17920/44000 | Loss: 3.0299 | LR: 3.29e-04 | 2257.2ms/step | 8.9 samples/s +Step 17930/44000 | Loss: 3.0723 | LR: 3.29e-04 | 2236.5ms/step | 8.9 samples/s +Step 17940/44000 | Loss: 2.9865 | LR: 3.29e-04 | 2319.0ms/step | 8.6 samples/s +Step 17950/44000 | Loss: 2.9785 | LR: 3.29e-04 | 2273.0ms/step | 8.8 samples/s +Step 17960/44000 | Loss: 3.0473 | LR: 3.29e-04 | 2283.9ms/step | 8.8 samples/s +Step 17970/44000 | Loss: 3.0227 | LR: 3.29e-04 | 2309.3ms/step | 8.7 samples/s +Step 17980/44000 | Loss: 3.0186 | LR: 3.28e-04 | 2338.0ms/step | 8.6 samples/s +Step 17990/44000 | Loss: 3.0048 | LR: 3.28e-04 | 2231.8ms/step | 9.0 samples/s +Step 18000/44000 | Loss: 2.9859 | LR: 3.28e-04 | 2262.7ms/step | 8.8 samples/s +Validation at step 18000: Loss = 3.0109, Perplexity = 20.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-18000 +Step 18010/44000 | Loss: 2.9698 | LR: 3.28e-04 | 4817.4ms/step | 4.2 samples/s +Step 18020/44000 | Loss: 3.0008 | LR: 3.28e-04 | 2300.9ms/step | 8.7 samples/s +Step 18030/44000 | Loss: 3.0034 | LR: 3.28e-04 | 2166.6ms/step | 9.2 samples/s +Step 18040/44000 | Loss: 3.0900 | LR: 3.27e-04 | 2178.7ms/step | 9.2 samples/s +Step 18050/44000 | Loss: 3.0258 | LR: 3.27e-04 | 2221.4ms/step | 9.0 samples/s +Step 18060/44000 | Loss: 2.9877 | LR: 3.27e-04 | 2206.1ms/step | 9.1 samples/s +Step 18070/44000 | Loss: 2.9895 | LR: 3.27e-04 | 2365.9ms/step | 8.5 samples/s +Step 18080/44000 | Loss: 2.9855 | LR: 3.27e-04 | 2196.6ms/step | 9.1 samples/s +Step 18090/44000 | Loss: 2.9824 | LR: 3.27e-04 | 2261.4ms/step | 8.8 samples/s +Step 18100/44000 | Loss: 3.0008 | LR: 3.26e-04 | 2294.0ms/step | 8.7 samples/s +Step 18110/44000 | Loss: 2.9972 | LR: 3.26e-04 | 2320.0ms/step | 8.6 samples/s +Step 18120/44000 | Loss: 3.0254 | LR: 3.26e-04 | 2319.4ms/step | 8.6 samples/s +Step 18130/44000 | Loss: 3.0391 | LR: 3.26e-04 | 2273.0ms/step | 8.8 samples/s +Step 18140/44000 | Loss: 2.9735 | LR: 3.26e-04 | 2271.0ms/step | 8.8 samples/s +Step 18150/44000 | Loss: 3.0083 | LR: 3.26e-04 | 2367.8ms/step | 8.4 samples/s +Step 18160/44000 | Loss: 2.9950 | LR: 3.25e-04 | 2298.2ms/step | 8.7 samples/s +Step 18170/44000 | Loss: 2.9731 | LR: 3.25e-04 | 2241.7ms/step | 8.9 samples/s +Step 18180/44000 | Loss: 3.0175 | LR: 3.25e-04 | 2272.6ms/step | 8.8 samples/s +Step 18190/44000 | Loss: 2.9702 | LR: 3.25e-04 | 2341.1ms/step | 8.5 samples/s +Step 18200/44000 | Loss: 2.9804 | LR: 3.25e-04 | 2252.1ms/step | 8.9 samples/s +Step 18210/44000 | Loss: 2.9719 | LR: 3.24e-04 | 2273.0ms/step | 8.8 samples/s +Step 18220/44000 | Loss: 3.0372 | LR: 3.24e-04 | 2241.3ms/step | 8.9 samples/s +Step 18230/44000 | Loss: 2.9709 | LR: 3.24e-04 | 2206.3ms/step | 9.1 samples/s +Step 18240/44000 | Loss: 2.9719 | LR: 3.24e-04 | 2256.3ms/step | 8.9 samples/s +Step 18250/44000 | Loss: 3.0265 | LR: 3.24e-04 | 2224.5ms/step | 9.0 samples/s +Validation at step 18250: Loss = 3.0078, Perplexity = 20.24 +Step 18260/44000 | Loss: 2.9849 | LR: 3.24e-04 | 4175.0ms/step | 4.8 samples/s +Step 18270/44000 | Loss: 2.9714 | LR: 3.23e-04 | 2279.4ms/step | 8.8 samples/s +Step 18280/44000 | Loss: 2.9975 | LR: 3.23e-04 | 2179.3ms/step | 9.2 samples/s +Step 18290/44000 | Loss: 3.0209 | LR: 3.23e-04 | 2227.4ms/step | 9.0 samples/s +Step 18300/44000 | Loss: 3.0039 | LR: 3.23e-04 | 2253.7ms/step | 8.9 samples/s +Step 18310/44000 | Loss: 2.9829 | LR: 3.23e-04 | 2235.3ms/step | 8.9 samples/s +Step 18320/44000 | Loss: 2.9800 | LR: 3.23e-04 | 2325.4ms/step | 8.6 samples/s +Step 18330/44000 | Loss: 2.9604 | LR: 3.22e-04 | 2291.6ms/step | 8.7 samples/s +Step 18340/44000 | Loss: 2.9626 | LR: 3.22e-04 | 2237.0ms/step | 8.9 samples/s +Step 18350/44000 | Loss: 2.9834 | LR: 3.22e-04 | 2210.4ms/step | 9.0 samples/s +Step 18360/44000 | Loss: 3.0033 | LR: 3.22e-04 | 2246.0ms/step | 8.9 samples/s +Step 18370/44000 | Loss: 2.9949 | LR: 3.22e-04 | 2225.0ms/step | 9.0 samples/s +Step 18380/44000 | Loss: 2.9761 | LR: 3.22e-04 | 2251.8ms/step | 8.9 samples/s +Step 18390/44000 | Loss: 2.9948 | LR: 3.21e-04 | 2176.1ms/step | 9.2 samples/s +Step 18400/44000 | Loss: 2.9630 | LR: 3.21e-04 | 2166.5ms/step | 9.2 samples/s +Step 18410/44000 | Loss: 2.9673 | LR: 3.21e-04 | 2264.1ms/step | 8.8 samples/s +Step 18420/44000 | Loss: 2.9887 | LR: 3.21e-04 | 2300.2ms/step | 8.7 samples/s +Step 18430/44000 | Loss: 2.9846 | LR: 3.21e-04 | 2221.6ms/step | 9.0 samples/s +Step 18440/44000 | Loss: 2.9582 | LR: 3.20e-04 | 2268.1ms/step | 8.8 samples/s +Step 18450/44000 | Loss: 2.9719 | LR: 3.20e-04 | 2328.7ms/step | 8.6 samples/s +Step 18460/44000 | Loss: 3.0388 | LR: 3.20e-04 | 2249.8ms/step | 8.9 samples/s +Step 18470/44000 | Loss: 2.9755 | LR: 3.20e-04 | 2261.9ms/step | 8.8 samples/s +Step 18480/44000 | Loss: 3.0251 | LR: 3.20e-04 | 2345.7ms/step | 8.5 samples/s +Step 18490/44000 | Loss: 2.9849 | LR: 3.20e-04 | 2382.3ms/step | 8.4 samples/s +Step 18500/44000 | Loss: 3.0159 | LR: 3.19e-04 | 2382.8ms/step | 8.4 samples/s +Validation at step 18500: Loss = 3.0050, Perplexity = 20.19 +Step 18510/44000 | Loss: 2.9746 | LR: 3.19e-04 | 4199.2ms/step | 4.8 samples/s +Step 18520/44000 | Loss: 3.0074 | LR: 3.19e-04 | 2263.1ms/step | 8.8 samples/s +Step 18530/44000 | Loss: 2.9672 | LR: 3.19e-04 | 2280.7ms/step | 8.8 samples/s +Step 18540/44000 | Loss: 3.0035 | LR: 3.19e-04 | 2263.8ms/step | 8.8 samples/s +Step 18550/44000 | Loss: 2.9716 | LR: 3.19e-04 | 2158.6ms/step | 9.3 samples/s +Step 18560/44000 | Loss: 3.0133 | LR: 3.18e-04 | 2222.1ms/step | 9.0 samples/s +Step 18570/44000 | Loss: 2.9752 | LR: 3.18e-04 | 2184.1ms/step | 9.2 samples/s +Step 18580/44000 | Loss: 2.9573 | LR: 3.18e-04 | 2279.3ms/step | 8.8 samples/s +Step 18590/44000 | Loss: 3.0056 | LR: 3.18e-04 | 2261.0ms/step | 8.8 samples/s +Step 18600/44000 | Loss: 2.9779 | LR: 3.18e-04 | 2263.6ms/step | 8.8 samples/s +Step 18610/44000 | Loss: 2.9611 | LR: 3.18e-04 | 2408.0ms/step | 8.3 samples/s +Step 18620/44000 | Loss: 3.0128 | LR: 3.17e-04 | 2200.4ms/step | 9.1 samples/s +Step 18630/44000 | Loss: 2.9671 | LR: 3.17e-04 | 2178.4ms/step | 9.2 samples/s +Step 18640/44000 | Loss: 2.9813 | LR: 3.17e-04 | 2195.3ms/step | 9.1 samples/s +Step 18650/44000 | Loss: 3.0120 | LR: 3.17e-04 | 2127.2ms/step | 9.4 samples/s +Step 18660/44000 | Loss: 2.9473 | LR: 3.17e-04 | 2122.0ms/step | 9.4 samples/s +Step 18670/44000 | Loss: 3.0087 | LR: 3.16e-04 | 2263.2ms/step | 8.8 samples/s +Step 18680/44000 | Loss: 2.9475 | LR: 3.16e-04 | 2257.1ms/step | 8.9 samples/s +Step 18690/44000 | Loss: 2.9606 | LR: 3.16e-04 | 2325.0ms/step | 8.6 samples/s +Step 18700/44000 | Loss: 2.9447 | LR: 3.16e-04 | 2450.4ms/step | 8.2 samples/s +Step 18710/44000 | Loss: 2.9982 | LR: 3.16e-04 | 2137.8ms/step | 9.4 samples/s +Step 18720/44000 | Loss: 2.9681 | LR: 3.16e-04 | 2390.7ms/step | 8.4 samples/s +Step 18730/44000 | Loss: 2.9966 | LR: 3.15e-04 | 2579.0ms/step | 7.8 samples/s +Step 18740/44000 | Loss: 2.9942 | LR: 3.15e-04 | 2240.8ms/step | 8.9 samples/s +Step 18750/44000 | Loss: 2.9786 | LR: 3.15e-04 | 2195.0ms/step | 9.1 samples/s +Validation at step 18750: Loss = 3.0028, Perplexity = 20.14 +Step 18760/44000 | Loss: 3.0078 | LR: 3.15e-04 | 4219.6ms/step | 4.7 samples/s +Step 18770/44000 | Loss: 2.9812 | LR: 3.15e-04 | 2282.3ms/step | 8.8 samples/s +Step 18780/44000 | Loss: 2.9938 | LR: 3.15e-04 | 2194.3ms/step | 9.1 samples/s +Step 18790/44000 | Loss: 3.0091 | LR: 3.14e-04 | 2149.5ms/step | 9.3 samples/s +Step 18800/44000 | Loss: 2.9535 | LR: 3.14e-04 | 2319.4ms/step | 8.6 samples/s +Step 18810/44000 | Loss: 2.9778 | LR: 3.14e-04 | 2190.5ms/step | 9.1 samples/s +Step 18820/44000 | Loss: 2.9807 | LR: 3.14e-04 | 2232.3ms/step | 9.0 samples/s +Step 18830/44000 | Loss: 2.9466 | LR: 3.14e-04 | 2319.8ms/step | 8.6 samples/s +Step 18840/44000 | Loss: 2.9770 | LR: 3.13e-04 | 2298.2ms/step | 8.7 samples/s +Step 18850/44000 | Loss: 3.0308 | LR: 3.13e-04 | 2304.2ms/step | 8.7 samples/s +Step 18860/44000 | Loss: 2.9861 | LR: 3.13e-04 | 2310.0ms/step | 8.7 samples/s +Step 18870/44000 | Loss: 3.0114 | LR: 3.13e-04 | 2309.7ms/step | 8.7 samples/s +Step 18880/44000 | Loss: 3.0183 | LR: 3.13e-04 | 2236.6ms/step | 8.9 samples/s +Step 18890/44000 | Loss: 2.9633 | LR: 3.13e-04 | 2325.3ms/step | 8.6 samples/s +Step 18900/44000 | Loss: 2.9979 | LR: 3.12e-04 | 2220.5ms/step | 9.0 samples/s +Step 18910/44000 | Loss: 2.9560 | LR: 3.12e-04 | 2319.2ms/step | 8.6 samples/s +Step 18920/44000 | Loss: 3.0282 | LR: 3.12e-04 | 2297.2ms/step | 8.7 samples/s +Step 18930/44000 | Loss: 3.0050 | LR: 3.12e-04 | 2286.1ms/step | 8.7 samples/s +Step 18940/44000 | Loss: 2.9510 | LR: 3.12e-04 | 2276.2ms/step | 8.8 samples/s +Step 18950/44000 | Loss: 2.9789 | LR: 3.12e-04 | 2368.2ms/step | 8.4 samples/s +Step 18960/44000 | Loss: 2.9700 | LR: 3.11e-04 | 2206.0ms/step | 9.1 samples/s +Step 18970/44000 | Loss: 3.0005 | LR: 3.11e-04 | 2065.2ms/step | 9.7 samples/s +Step 18980/44000 | Loss: 2.9656 | LR: 3.11e-04 | 2073.7ms/step | 9.6 samples/s +Step 18990/44000 | Loss: 2.9905 | LR: 3.11e-04 | 2263.4ms/step | 8.8 samples/s +Step 19000/44000 | Loss: 2.9921 | LR: 3.11e-04 | 2350.5ms/step | 8.5 samples/s +Validation at step 19000: Loss = 2.9988, Perplexity = 20.06 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-19000 +Step 19010/44000 | Loss: 2.9659 | LR: 3.11e-04 | 4860.1ms/step | 4.1 samples/s +Step 19020/44000 | Loss: 2.9801 | LR: 3.10e-04 | 2162.8ms/step | 9.2 samples/s +Step 19030/44000 | Loss: 3.0117 | LR: 3.10e-04 | 2257.6ms/step | 8.9 samples/s +Step 19040/44000 | Loss: 3.0002 | LR: 3.10e-04 | 2126.6ms/step | 9.4 samples/s +Step 19050/44000 | Loss: 2.9615 | LR: 3.10e-04 | 2320.7ms/step | 8.6 samples/s +Step 19060/44000 | Loss: 2.9557 | LR: 3.10e-04 | 2393.4ms/step | 8.4 samples/s +Step 19070/44000 | Loss: 2.9829 | LR: 3.09e-04 | 2240.6ms/step | 8.9 samples/s +Step 19080/44000 | Loss: 2.9515 | LR: 3.09e-04 | 2663.5ms/step | 7.5 samples/s +Step 19090/44000 | Loss: 2.9568 | LR: 3.09e-04 | 2556.1ms/step | 7.8 samples/s +Step 19100/44000 | Loss: 2.9866 | LR: 3.09e-04 | 2270.1ms/step | 8.8 samples/s +Step 19110/44000 | Loss: 2.9954 | LR: 3.09e-04 | 2131.5ms/step | 9.4 samples/s +Step 19120/44000 | Loss: 2.9872 | LR: 3.09e-04 | 2205.8ms/step | 9.1 samples/s +Step 19130/44000 | Loss: 2.9625 | LR: 3.08e-04 | 2198.4ms/step | 9.1 samples/s +Step 19140/44000 | Loss: 3.0107 | LR: 3.08e-04 | 2284.2ms/step | 8.8 samples/s +Step 19150/44000 | Loss: 2.9708 | LR: 3.08e-04 | 2256.4ms/step | 8.9 samples/s +Step 19160/44000 | Loss: 2.9951 | LR: 3.08e-04 | 2281.7ms/step | 8.8 samples/s +Step 19170/44000 | Loss: 2.9849 | LR: 3.08e-04 | 2350.8ms/step | 8.5 samples/s +Step 19180/44000 | Loss: 3.0032 | LR: 3.08e-04 | 2251.9ms/step | 8.9 samples/s +Step 19190/44000 | Loss: 2.9721 | LR: 3.07e-04 | 2171.0ms/step | 9.2 samples/s +Step 19200/44000 | Loss: 2.9935 | LR: 3.07e-04 | 2254.7ms/step | 8.9 samples/s +Step 19210/44000 | Loss: 2.9739 | LR: 3.07e-04 | 2214.8ms/step | 9.0 samples/s +Step 19220/44000 | Loss: 2.9881 | LR: 3.07e-04 | 2186.1ms/step | 9.1 samples/s +Step 19230/44000 | Loss: 2.9973 | LR: 3.07e-04 | 2249.9ms/step | 8.9 samples/s +Step 19240/44000 | Loss: 2.9629 | LR: 3.06e-04 | 2249.5ms/step | 8.9 samples/s +Step 19250/44000 | Loss: 2.9397 | LR: 3.06e-04 | 2233.0ms/step | 9.0 samples/s +Validation at step 19250: Loss = 2.9966, Perplexity = 20.02 +Step 19260/44000 | Loss: 2.9920 | LR: 3.06e-04 | 4185.3ms/step | 4.8 samples/s +Step 19270/44000 | Loss: 2.9994 | LR: 3.06e-04 | 2178.8ms/step | 9.2 samples/s +Step 19280/44000 | Loss: 2.9729 | LR: 3.06e-04 | 2121.0ms/step | 9.4 samples/s +Step 19290/44000 | Loss: 3.0284 | LR: 3.06e-04 | 2168.8ms/step | 9.2 samples/s +Step 19300/44000 | Loss: 2.9460 | LR: 3.05e-04 | 2308.9ms/step | 8.7 samples/s +Step 19310/44000 | Loss: 3.0010 | LR: 3.05e-04 | 2293.5ms/step | 8.7 samples/s +Step 19320/44000 | Loss: 3.0129 | LR: 3.05e-04 | 2335.6ms/step | 8.6 samples/s +Step 19330/44000 | Loss: 2.9990 | LR: 3.05e-04 | 2194.3ms/step | 9.1 samples/s +Step 19340/44000 | Loss: 3.0052 | LR: 3.05e-04 | 2317.5ms/step | 8.6 samples/s +Step 19350/44000 | Loss: 2.9510 | LR: 3.04e-04 | 2149.0ms/step | 9.3 samples/s +Step 19360/44000 | Loss: 2.9953 | LR: 3.04e-04 | 2194.0ms/step | 9.1 samples/s +Step 19370/44000 | Loss: 2.9590 | LR: 3.04e-04 | 2197.2ms/step | 9.1 samples/s +Step 19380/44000 | Loss: 3.0081 | LR: 3.04e-04 | 2352.6ms/step | 8.5 samples/s +Step 19390/44000 | Loss: 2.9869 | LR: 3.04e-04 | 2282.7ms/step | 8.8 samples/s +Step 19400/44000 | Loss: 2.9879 | LR: 3.04e-04 | 2269.8ms/step | 8.8 samples/s +Step 19410/44000 | Loss: 2.9882 | LR: 3.03e-04 | 2284.1ms/step | 8.8 samples/s +Step 19420/44000 | Loss: 2.9765 | LR: 3.03e-04 | 2209.8ms/step | 9.1 samples/s +Step 19430/44000 | Loss: 2.9542 | LR: 3.03e-04 | 2147.0ms/step | 9.3 samples/s +Step 19440/44000 | Loss: 3.0130 | LR: 3.03e-04 | 2210.2ms/step | 9.0 samples/s +Step 19450/44000 | Loss: 2.9866 | LR: 3.03e-04 | 2319.2ms/step | 8.6 samples/s +Step 19460/44000 | Loss: 2.9532 | LR: 3.03e-04 | 2225.9ms/step | 9.0 samples/s +Step 19470/44000 | Loss: 2.9934 | LR: 3.02e-04 | 2638.4ms/step | 7.6 samples/s +Step 19480/44000 | Loss: 3.0127 | LR: 3.02e-04 | 2475.7ms/step | 8.1 samples/s +Step 19490/44000 | Loss: 2.9301 | LR: 3.02e-04 | 2208.0ms/step | 9.1 samples/s +Step 19500/44000 | Loss: 2.9378 | LR: 3.02e-04 | 2218.0ms/step | 9.0 samples/s +Validation at step 19500: Loss = 2.9943, Perplexity = 19.97 +Step 19510/44000 | Loss: 2.9640 | LR: 3.02e-04 | 4054.1ms/step | 4.9 samples/s +Step 19520/44000 | Loss: 2.9824 | LR: 3.01e-04 | 2205.5ms/step | 9.1 samples/s +Step 19530/44000 | Loss: 2.9826 | LR: 3.01e-04 | 2159.3ms/step | 9.3 samples/s +Step 19540/44000 | Loss: 3.0118 | LR: 3.01e-04 | 2303.8ms/step | 8.7 samples/s +Step 19550/44000 | Loss: 3.0161 | LR: 3.01e-04 | 2273.7ms/step | 8.8 samples/s +Step 19560/44000 | Loss: 2.9881 | LR: 3.01e-04 | 2300.3ms/step | 8.7 samples/s +Step 19570/44000 | Loss: 3.0030 | LR: 3.01e-04 | 2327.9ms/step | 8.6 samples/s +Step 19580/44000 | Loss: 2.9901 | LR: 3.00e-04 | 2361.9ms/step | 8.5 samples/s +Step 19590/44000 | Loss: 2.9718 | LR: 3.00e-04 | 2089.7ms/step | 9.6 samples/s +Step 19600/44000 | Loss: 2.9459 | LR: 3.00e-04 | 2064.7ms/step | 9.7 samples/s +Step 19610/44000 | Loss: 2.9517 | LR: 3.00e-04 | 2185.9ms/step | 9.1 samples/s +Step 19620/44000 | Loss: 2.9728 | LR: 3.00e-04 | 2287.2ms/step | 8.7 samples/s +Step 19630/44000 | Loss: 2.9988 | LR: 3.00e-04 | 2293.9ms/step | 8.7 samples/s +Step 19640/44000 | Loss: 2.9631 | LR: 2.99e-04 | 2288.8ms/step | 8.7 samples/s +Step 19650/44000 | Loss: 2.9239 | LR: 2.99e-04 | 2249.6ms/step | 8.9 samples/s +Step 19660/44000 | Loss: 2.9725 | LR: 2.99e-04 | 2285.6ms/step | 8.8 samples/s +Step 19670/44000 | Loss: 2.9684 | LR: 2.99e-04 | 2184.7ms/step | 9.2 samples/s +Step 19680/44000 | Loss: 2.9720 | LR: 2.99e-04 | 2169.8ms/step | 9.2 samples/s +Step 19690/44000 | Loss: 2.9729 | LR: 2.98e-04 | 2178.5ms/step | 9.2 samples/s +Step 19700/44000 | Loss: 2.9306 | LR: 2.98e-04 | 2168.6ms/step | 9.2 samples/s +Step 19710/44000 | Loss: 2.9685 | LR: 2.98e-04 | 2247.0ms/step | 8.9 samples/s +Step 19720/44000 | Loss: 2.9806 | LR: 2.98e-04 | 2304.7ms/step | 8.7 samples/s +Step 19730/44000 | Loss: 2.9901 | LR: 2.98e-04 | 2273.5ms/step | 8.8 samples/s +Step 19740/44000 | Loss: 2.9835 | LR: 2.98e-04 | 2254.1ms/step | 8.9 samples/s +Step 19750/44000 | Loss: 2.9715 | LR: 2.97e-04 | 2317.1ms/step | 8.6 samples/s +Validation at step 19750: Loss = 2.9915, Perplexity = 19.92 +Step 19760/44000 | Loss: 3.0083 | LR: 2.97e-04 | 4074.3ms/step | 4.9 samples/s +Step 19770/44000 | Loss: 2.9609 | LR: 2.97e-04 | 2193.8ms/step | 9.1 samples/s +Step 19780/44000 | Loss: 2.9623 | LR: 2.97e-04 | 2293.6ms/step | 8.7 samples/s +Step 19790/44000 | Loss: 2.9688 | LR: 2.97e-04 | 2206.3ms/step | 9.1 samples/s +Step 19800/44000 | Loss: 2.9368 | LR: 2.96e-04 | 2205.6ms/step | 9.1 samples/s +Step 19810/44000 | Loss: 2.9867 | LR: 2.96e-04 | 2268.0ms/step | 8.8 samples/s +Step 19820/44000 | Loss: 2.9856 | LR: 2.96e-04 | 2330.4ms/step | 8.6 samples/s +Step 19830/44000 | Loss: 2.9507 | LR: 2.96e-04 | 2254.6ms/step | 8.9 samples/s +Step 19840/44000 | Loss: 2.9435 | LR: 2.96e-04 | 2331.7ms/step | 8.6 samples/s +Step 19850/44000 | Loss: 2.9571 | LR: 2.96e-04 | 2491.9ms/step | 8.0 samples/s +Step 19860/44000 | Loss: 3.0181 | LR: 2.95e-04 | 2273.8ms/step | 8.8 samples/s +Step 19870/44000 | Loss: 2.9975 | LR: 2.95e-04 | 2166.1ms/step | 9.2 samples/s +Step 19880/44000 | Loss: 2.9705 | LR: 2.95e-04 | 2188.6ms/step | 9.1 samples/s +Step 19890/44000 | Loss: 2.9816 | LR: 2.95e-04 | 2256.2ms/step | 8.9 samples/s +Step 19900/44000 | Loss: 2.9564 | LR: 2.95e-04 | 2308.9ms/step | 8.7 samples/s +Step 19910/44000 | Loss: 2.9788 | LR: 2.95e-04 | 2283.8ms/step | 8.8 samples/s +Step 19920/44000 | Loss: 2.9927 | LR: 2.94e-04 | 2079.0ms/step | 9.6 samples/s +Step 19930/44000 | Loss: 3.0001 | LR: 2.94e-04 | 2255.7ms/step | 8.9 samples/s +Step 19940/44000 | Loss: 3.0080 | LR: 2.94e-04 | 2170.7ms/step | 9.2 samples/s +Step 19950/44000 | Loss: 2.9725 | LR: 2.94e-04 | 2281.7ms/step | 8.8 samples/s +Step 19960/44000 | Loss: 2.9964 | LR: 2.94e-04 | 2310.7ms/step | 8.7 samples/s +Step 19970/44000 | Loss: 2.9606 | LR: 2.93e-04 | 2288.9ms/step | 8.7 samples/s +Step 19980/44000 | Loss: 2.9805 | LR: 2.93e-04 | 2288.5ms/step | 8.7 samples/s +Step 19990/44000 | Loss: 2.9807 | LR: 2.93e-04 | 2314.6ms/step | 8.6 samples/s +Step 20000/44000 | Loss: 2.9039 | LR: 2.93e-04 | 2354.7ms/step | 8.5 samples/s +Validation at step 20000: Loss = 2.9889, Perplexity = 19.86 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-20000 +Step 20010/44000 | Loss: 2.9816 | LR: 2.93e-04 | 4689.3ms/step | 4.3 samples/s +Step 20020/44000 | Loss: 2.9797 | LR: 2.93e-04 | 2223.8ms/step | 9.0 samples/s +Step 20030/44000 | Loss: 2.9963 | LR: 2.92e-04 | 2273.5ms/step | 8.8 samples/s +Step 20040/44000 | Loss: 2.9536 | LR: 2.92e-04 | 2366.2ms/step | 8.5 samples/s +Step 20050/44000 | Loss: 2.9349 | LR: 2.92e-04 | 2304.8ms/step | 8.7 samples/s +Step 20060/44000 | Loss: 2.9516 | LR: 2.92e-04 | 2296.9ms/step | 8.7 samples/s +Step 20070/44000 | Loss: 2.9483 | LR: 2.92e-04 | 2257.8ms/step | 8.9 samples/s +Step 20080/44000 | Loss: 2.9598 | LR: 2.91e-04 | 2173.0ms/step | 9.2 samples/s +Step 20090/44000 | Loss: 3.0009 | LR: 2.91e-04 | 2224.4ms/step | 9.0 samples/s +Step 20100/44000 | Loss: 3.0159 | LR: 2.91e-04 | 2246.7ms/step | 8.9 samples/s +Step 20110/44000 | Loss: 2.9557 | LR: 2.91e-04 | 2248.2ms/step | 8.9 samples/s +Step 20120/44000 | Loss: 2.9712 | LR: 2.91e-04 | 2247.1ms/step | 8.9 samples/s +Step 20130/44000 | Loss: 2.9643 | LR: 2.91e-04 | 2190.2ms/step | 9.1 samples/s +Step 20140/44000 | Loss: 3.0288 | LR: 2.90e-04 | 2134.6ms/step | 9.4 samples/s +Step 20150/44000 | Loss: 2.9972 | LR: 2.90e-04 | 2221.5ms/step | 9.0 samples/s +Step 20160/44000 | Loss: 2.9663 | LR: 2.90e-04 | 2210.1ms/step | 9.0 samples/s +Step 20170/44000 | Loss: 2.9368 | LR: 2.90e-04 | 2159.6ms/step | 9.3 samples/s +Step 20180/44000 | Loss: 2.9708 | LR: 2.90e-04 | 2366.0ms/step | 8.5 samples/s +Step 20190/44000 | Loss: 2.9435 | LR: 2.90e-04 | 2195.3ms/step | 9.1 samples/s +Step 20200/44000 | Loss: 2.9744 | LR: 2.89e-04 | 2404.3ms/step | 8.3 samples/s +Step 20210/44000 | Loss: 2.9759 | LR: 2.89e-04 | 2410.9ms/step | 8.3 samples/s +Step 20220/44000 | Loss: 2.9877 | LR: 2.89e-04 | 2186.5ms/step | 9.1 samples/s +Step 20230/44000 | Loss: 2.9894 | LR: 2.89e-04 | 2074.8ms/step | 9.6 samples/s +Step 20240/44000 | Loss: 2.9791 | LR: 2.89e-04 | 2013.7ms/step | 9.9 samples/s +Step 20250/44000 | Loss: 2.9562 | LR: 2.88e-04 | 2120.4ms/step | 9.4 samples/s +Validation at step 20250: Loss = 2.9848, Perplexity = 19.78 +Step 20260/44000 | Loss: 2.9420 | LR: 2.88e-04 | 4183.5ms/step | 4.8 samples/s +Step 20270/44000 | Loss: 2.9973 | LR: 2.88e-04 | 2125.2ms/step | 9.4 samples/s +Step 20280/44000 | Loss: 2.9965 | LR: 2.88e-04 | 2179.6ms/step | 9.2 samples/s +Step 20290/44000 | Loss: 2.9881 | LR: 2.88e-04 | 2136.0ms/step | 9.4 samples/s +Step 20300/44000 | Loss: 2.9617 | LR: 2.88e-04 | 2282.6ms/step | 8.8 samples/s +Step 20310/44000 | Loss: 2.9433 | LR: 2.87e-04 | 2245.6ms/step | 8.9 samples/s +Step 20320/44000 | Loss: 2.9618 | LR: 2.87e-04 | 2214.2ms/step | 9.0 samples/s +Step 20330/44000 | Loss: 2.9467 | LR: 2.87e-04 | 2295.6ms/step | 8.7 samples/s +Step 20340/44000 | Loss: 2.9440 | LR: 2.87e-04 | 2271.0ms/step | 8.8 samples/s +Step 20350/44000 | Loss: 2.9691 | LR: 2.87e-04 | 2244.6ms/step | 8.9 samples/s +Step 20360/44000 | Loss: 2.9561 | LR: 2.86e-04 | 2221.0ms/step | 9.0 samples/s +Step 20370/44000 | Loss: 2.9737 | LR: 2.86e-04 | 2257.4ms/step | 8.9 samples/s +Step 20380/44000 | Loss: 2.9740 | LR: 2.86e-04 | 2241.8ms/step | 8.9 samples/s +Step 20390/44000 | Loss: 2.9790 | LR: 2.86e-04 | 2365.8ms/step | 8.5 samples/s +Step 20400/44000 | Loss: 2.9814 | LR: 2.86e-04 | 2356.1ms/step | 8.5 samples/s +Step 20410/44000 | Loss: 3.0174 | LR: 2.86e-04 | 2350.6ms/step | 8.5 samples/s +Step 20420/44000 | Loss: 3.0192 | LR: 2.85e-04 | 2303.9ms/step | 8.7 samples/s +Step 20430/44000 | Loss: 2.9834 | LR: 2.85e-04 | 2251.9ms/step | 8.9 samples/s +Step 20440/44000 | Loss: 2.9771 | LR: 2.85e-04 | 2304.3ms/step | 8.7 samples/s +Step 20450/44000 | Loss: 2.9289 | LR: 2.85e-04 | 2221.8ms/step | 9.0 samples/s +Step 20460/44000 | Loss: 2.9664 | LR: 2.85e-04 | 2282.0ms/step | 8.8 samples/s +Step 20470/44000 | Loss: 2.9406 | LR: 2.84e-04 | 2231.6ms/step | 9.0 samples/s +Step 20480/44000 | Loss: 2.9295 | LR: 2.84e-04 | 2323.1ms/step | 8.6 samples/s +Step 20490/44000 | Loss: 2.9865 | LR: 2.84e-04 | 2304.2ms/step | 8.7 samples/s +Step 20500/44000 | Loss: 2.9768 | LR: 2.84e-04 | 2262.8ms/step | 8.8 samples/s +Validation at step 20500: Loss = 2.9827, Perplexity = 19.74 +Step 20510/44000 | Loss: 2.9489 | LR: 2.84e-04 | 4168.5ms/step | 4.8 samples/s +Step 20520/44000 | Loss: 2.9694 | LR: 2.84e-04 | 2162.6ms/step | 9.2 samples/s +Step 20530/44000 | Loss: 2.9836 | LR: 2.83e-04 | 2490.1ms/step | 8.0 samples/s +Step 20540/44000 | Loss: 2.9646 | LR: 2.83e-04 | 2129.5ms/step | 9.4 samples/s +Step 20550/44000 | Loss: 2.9805 | LR: 2.83e-04 | 2585.5ms/step | 7.7 samples/s +Step 20560/44000 | Loss: 2.9869 | LR: 2.83e-04 | 2385.0ms/step | 8.4 samples/s +Step 20570/44000 | Loss: 2.9537 | LR: 2.83e-04 | 2216.7ms/step | 9.0 samples/s +Step 20580/44000 | Loss: 2.9504 | LR: 2.83e-04 | 2162.3ms/step | 9.2 samples/s +Step 20590/44000 | Loss: 2.9783 | LR: 2.82e-04 | 2163.5ms/step | 9.2 samples/s +Step 20600/44000 | Loss: 2.9596 | LR: 2.82e-04 | 2137.9ms/step | 9.4 samples/s +Step 20610/44000 | Loss: 2.9549 | LR: 2.82e-04 | 2253.7ms/step | 8.9 samples/s +Step 20620/44000 | Loss: 3.0034 | LR: 2.82e-04 | 2214.9ms/step | 9.0 samples/s +Step 20630/44000 | Loss: 2.9699 | LR: 2.82e-04 | 2328.4ms/step | 8.6 samples/s +Step 20640/44000 | Loss: 2.9724 | LR: 2.81e-04 | 2261.7ms/step | 8.8 samples/s +Step 20650/44000 | Loss: 2.9796 | LR: 2.81e-04 | 2255.5ms/step | 8.9 samples/s +Step 20660/44000 | Loss: 2.9428 | LR: 2.81e-04 | 2212.6ms/step | 9.0 samples/s +Step 20670/44000 | Loss: 2.9384 | LR: 2.81e-04 | 2159.5ms/step | 9.3 samples/s +Step 20680/44000 | Loss: 2.9471 | LR: 2.81e-04 | 2136.9ms/step | 9.4 samples/s +Step 20690/44000 | Loss: 2.9577 | LR: 2.81e-04 | 2185.3ms/step | 9.2 samples/s +Step 20700/44000 | Loss: 2.9272 | LR: 2.80e-04 | 2244.2ms/step | 8.9 samples/s +Step 20710/44000 | Loss: 2.9164 | LR: 2.80e-04 | 2255.1ms/step | 8.9 samples/s +Step 20720/44000 | Loss: 2.9638 | LR: 2.80e-04 | 2278.5ms/step | 8.8 samples/s +Step 20730/44000 | Loss: 2.9572 | LR: 2.80e-04 | 2267.4ms/step | 8.8 samples/s +Step 20740/44000 | Loss: 2.9465 | LR: 2.80e-04 | 2249.9ms/step | 8.9 samples/s +Step 20750/44000 | Loss: 2.9766 | LR: 2.79e-04 | 2222.6ms/step | 9.0 samples/s +Validation at step 20750: Loss = 2.9798, Perplexity = 19.68 +Step 20760/44000 | Loss: 2.9467 | LR: 2.79e-04 | 4049.4ms/step | 4.9 samples/s +Step 20770/44000 | Loss: 2.9583 | LR: 2.79e-04 | 2246.4ms/step | 8.9 samples/s +Step 20780/44000 | Loss: 3.0029 | LR: 2.79e-04 | 2345.8ms/step | 8.5 samples/s +Step 20790/44000 | Loss: 2.9471 | LR: 2.79e-04 | 2244.5ms/step | 8.9 samples/s +Step 20800/44000 | Loss: 2.9416 | LR: 2.79e-04 | 2258.2ms/step | 8.9 samples/s +Step 20810/44000 | Loss: 2.9585 | LR: 2.78e-04 | 2200.0ms/step | 9.1 samples/s +Step 20820/44000 | Loss: 2.9442 | LR: 2.78e-04 | 2226.1ms/step | 9.0 samples/s +Step 20830/44000 | Loss: 2.9976 | LR: 2.78e-04 | 2184.7ms/step | 9.2 samples/s +Step 20840/44000 | Loss: 2.9694 | LR: 2.78e-04 | 2199.4ms/step | 9.1 samples/s +Step 20850/44000 | Loss: 2.9640 | LR: 2.78e-04 | 2220.4ms/step | 9.0 samples/s +Step 20860/44000 | Loss: 2.9846 | LR: 2.77e-04 | 2100.6ms/step | 9.5 samples/s +Step 20870/44000 | Loss: 2.9546 | LR: 2.77e-04 | 2189.8ms/step | 9.1 samples/s +Step 20880/44000 | Loss: 2.9781 | LR: 2.77e-04 | 2245.4ms/step | 8.9 samples/s +Step 20890/44000 | Loss: 2.9842 | LR: 2.77e-04 | 2268.6ms/step | 8.8 samples/s +Step 20900/44000 | Loss: 2.9764 | LR: 2.77e-04 | 2272.6ms/step | 8.8 samples/s +Step 20910/44000 | Loss: 2.9435 | LR: 2.77e-04 | 2200.9ms/step | 9.1 samples/s +Step 20920/44000 | Loss: 2.9647 | LR: 2.76e-04 | 2281.4ms/step | 8.8 samples/s +Step 20930/44000 | Loss: 2.9364 | LR: 2.76e-04 | 2383.0ms/step | 8.4 samples/s +Step 20940/44000 | Loss: 2.9507 | LR: 2.76e-04 | 2694.3ms/step | 7.4 samples/s +Step 20950/44000 | Loss: 2.9260 | LR: 2.76e-04 | 2313.9ms/step | 8.6 samples/s +Step 20960/44000 | Loss: 2.9608 | LR: 2.76e-04 | 2207.8ms/step | 9.1 samples/s +Step 20970/44000 | Loss: 2.9870 | LR: 2.75e-04 | 2228.0ms/step | 9.0 samples/s +Step 20980/44000 | Loss: 2.9416 | LR: 2.75e-04 | 2202.6ms/step | 9.1 samples/s +Step 20990/44000 | Loss: 2.9684 | LR: 2.75e-04 | 2195.1ms/step | 9.1 samples/s +Step 21000/44000 | Loss: 2.9438 | LR: 2.75e-04 | 2219.2ms/step | 9.0 samples/s +Validation at step 21000: Loss = 2.9772, Perplexity = 19.63 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-21000 +Step 21010/44000 | Loss: 3.0149 | LR: 2.75e-04 | 4581.0ms/step | 4.4 samples/s +Step 21020/44000 | Loss: 2.9495 | LR: 2.75e-04 | 2277.4ms/step | 8.8 samples/s +Step 21030/44000 | Loss: 2.9623 | LR: 2.74e-04 | 2250.8ms/step | 8.9 samples/s +Step 21040/44000 | Loss: 3.0058 | LR: 2.74e-04 | 2203.3ms/step | 9.1 samples/s +Step 21050/44000 | Loss: 2.9069 | LR: 2.74e-04 | 2206.0ms/step | 9.1 samples/s +Step 21060/44000 | Loss: 2.9432 | LR: 2.74e-04 | 2209.9ms/step | 9.0 samples/s +Step 21070/44000 | Loss: 2.9223 | LR: 2.74e-04 | 2155.2ms/step | 9.3 samples/s +Step 21080/44000 | Loss: 2.9455 | LR: 2.73e-04 | 2259.4ms/step | 8.9 samples/s +Step 21090/44000 | Loss: 2.9706 | LR: 2.73e-04 | 2280.7ms/step | 8.8 samples/s +Step 21100/44000 | Loss: 2.9788 | LR: 2.73e-04 | 2253.1ms/step | 8.9 samples/s +Step 21110/44000 | Loss: 2.9777 | LR: 2.73e-04 | 2256.6ms/step | 8.9 samples/s +Step 21120/44000 | Loss: 2.9481 | LR: 2.73e-04 | 2225.5ms/step | 9.0 samples/s +Step 21130/44000 | Loss: 2.9422 | LR: 2.73e-04 | 2261.1ms/step | 8.8 samples/s +Step 21140/44000 | Loss: 2.9543 | LR: 2.72e-04 | 2194.8ms/step | 9.1 samples/s +Step 21150/44000 | Loss: 2.9591 | LR: 2.72e-04 | 2148.9ms/step | 9.3 samples/s +Step 21160/44000 | Loss: 2.9622 | LR: 2.72e-04 | 2224.8ms/step | 9.0 samples/s +Step 21170/44000 | Loss: 2.9705 | LR: 2.72e-04 | 2100.6ms/step | 9.5 samples/s +Step 21180/44000 | Loss: 2.9202 | LR: 2.72e-04 | 2229.9ms/step | 9.0 samples/s +Step 21190/44000 | Loss: 2.9587 | LR: 2.71e-04 | 2235.1ms/step | 8.9 samples/s +Step 21200/44000 | Loss: 2.9829 | LR: 2.71e-04 | 2206.8ms/step | 9.1 samples/s +Step 21210/44000 | Loss: 2.9412 | LR: 2.71e-04 | 2251.2ms/step | 8.9 samples/s +Step 21220/44000 | Loss: 2.9493 | LR: 2.71e-04 | 2216.1ms/step | 9.0 samples/s +Step 21230/44000 | Loss: 3.0068 | LR: 2.71e-04 | 2199.7ms/step | 9.1 samples/s +Step 21240/44000 | Loss: 2.9314 | LR: 2.71e-04 | 2131.7ms/step | 9.4 samples/s +Step 21250/44000 | Loss: 2.9376 | LR: 2.70e-04 | 2173.4ms/step | 9.2 samples/s +Validation at step 21250: Loss = 2.9749, Perplexity = 19.59 +Step 21260/44000 | Loss: 2.9263 | LR: 2.70e-04 | 4121.1ms/step | 4.9 samples/s +Step 21270/44000 | Loss: 2.9849 | LR: 2.70e-04 | 2264.1ms/step | 8.8 samples/s +Step 21280/44000 | Loss: 2.9830 | LR: 2.70e-04 | 2265.4ms/step | 8.8 samples/s +Step 21290/44000 | Loss: 2.9964 | LR: 2.70e-04 | 2300.4ms/step | 8.7 samples/s +Step 21300/44000 | Loss: 2.9267 | LR: 2.70e-04 | 2205.3ms/step | 9.1 samples/s +Step 21310/44000 | Loss: 2.9825 | LR: 2.69e-04 | 2204.9ms/step | 9.1 samples/s +Step 21320/44000 | Loss: 2.9600 | LR: 2.69e-04 | 2185.6ms/step | 9.2 samples/s +Step 21330/44000 | Loss: 2.9682 | LR: 2.69e-04 | 2152.3ms/step | 9.3 samples/s +Step 21340/44000 | Loss: 3.0074 | LR: 2.69e-04 | 2233.4ms/step | 9.0 samples/s +Step 21350/44000 | Loss: 2.9637 | LR: 2.69e-04 | 2294.3ms/step | 8.7 samples/s +Step 21360/44000 | Loss: 2.9169 | LR: 2.68e-04 | 2274.3ms/step | 8.8 samples/s +Step 21370/44000 | Loss: 2.9540 | LR: 2.68e-04 | 2247.2ms/step | 8.9 samples/s +Step 21380/44000 | Loss: 2.9808 | LR: 2.68e-04 | 2235.7ms/step | 8.9 samples/s +Step 21390/44000 | Loss: 2.9904 | LR: 2.68e-04 | 2234.2ms/step | 9.0 samples/s +Step 21400/44000 | Loss: 2.9476 | LR: 2.68e-04 | 2159.5ms/step | 9.3 samples/s +Step 21410/44000 | Loss: 2.9229 | LR: 2.68e-04 | 2235.7ms/step | 8.9 samples/s +Step 21420/44000 | Loss: 2.9884 | LR: 2.67e-04 | 2226.2ms/step | 9.0 samples/s +Step 21430/44000 | Loss: 2.9763 | LR: 2.67e-04 | 2241.3ms/step | 8.9 samples/s +Step 21440/44000 | Loss: 2.9395 | LR: 2.67e-04 | 2277.8ms/step | 8.8 samples/s +Step 21450/44000 | Loss: 2.9420 | LR: 2.67e-04 | 2238.7ms/step | 8.9 samples/s +Step 21460/44000 | Loss: 2.9203 | LR: 2.67e-04 | 2265.7ms/step | 8.8 samples/s +Step 21470/44000 | Loss: 2.9488 | LR: 2.66e-04 | 2225.7ms/step | 9.0 samples/s +Step 21480/44000 | Loss: 2.9973 | LR: 2.66e-04 | 2199.9ms/step | 9.1 samples/s +Step 21490/44000 | Loss: 2.9281 | LR: 2.66e-04 | 2022.0ms/step | 9.9 samples/s +Step 21500/44000 | Loss: 2.9694 | LR: 2.66e-04 | 2084.9ms/step | 9.6 samples/s +Validation at step 21500: Loss = 2.9726, Perplexity = 19.54 +Step 21510/44000 | Loss: 2.9903 | LR: 2.66e-04 | 4163.0ms/step | 4.8 samples/s +Step 21520/44000 | Loss: 2.9384 | LR: 2.66e-04 | 2272.0ms/step | 8.8 samples/s +Step 21530/44000 | Loss: 2.9618 | LR: 2.65e-04 | 2235.3ms/step | 8.9 samples/s +Step 21540/44000 | Loss: 2.9908 | LR: 2.65e-04 | 2245.2ms/step | 8.9 samples/s +Step 21550/44000 | Loss: 2.9661 | LR: 2.65e-04 | 2211.9ms/step | 9.0 samples/s +Step 21560/44000 | Loss: 2.9182 | LR: 2.65e-04 | 2190.0ms/step | 9.1 samples/s +Step 21570/44000 | Loss: 2.9440 | LR: 2.65e-04 | 2185.9ms/step | 9.1 samples/s +Step 21580/44000 | Loss: 2.9705 | LR: 2.64e-04 | 2184.8ms/step | 9.2 samples/s +Step 21590/44000 | Loss: 2.9570 | LR: 2.64e-04 | 2234.6ms/step | 9.0 samples/s +Step 21600/44000 | Loss: 2.9482 | LR: 2.64e-04 | 2319.3ms/step | 8.6 samples/s +Step 21610/44000 | Loss: 2.9258 | LR: 2.64e-04 | 2230.0ms/step | 9.0 samples/s +Step 21620/44000 | Loss: 2.9664 | LR: 2.64e-04 | 2249.7ms/step | 8.9 samples/s +Step 21630/44000 | Loss: 2.8978 | LR: 2.64e-04 | 2213.9ms/step | 9.0 samples/s +Step 21640/44000 | Loss: 3.0167 | LR: 2.63e-04 | 2164.1ms/step | 9.2 samples/s +Step 21650/44000 | Loss: 2.9516 | LR: 2.63e-04 | 2145.1ms/step | 9.3 samples/s +Step 21660/44000 | Loss: 2.9711 | LR: 2.63e-04 | 2204.4ms/step | 9.1 samples/s +Step 21670/44000 | Loss: 2.9722 | LR: 2.63e-04 | 2189.0ms/step | 9.1 samples/s +Step 21680/44000 | Loss: 2.9300 | LR: 2.63e-04 | 2278.9ms/step | 8.8 samples/s +Step 21690/44000 | Loss: 2.9635 | LR: 2.62e-04 | 2271.0ms/step | 8.8 samples/s +Step 21700/44000 | Loss: 2.9222 | LR: 2.62e-04 | 2234.9ms/step | 8.9 samples/s +Step 21710/44000 | Loss: 2.9649 | LR: 2.62e-04 | 2246.3ms/step | 8.9 samples/s +Step 21720/44000 | Loss: 2.9892 | LR: 2.62e-04 | 2173.0ms/step | 9.2 samples/s +Step 21730/44000 | Loss: 2.9723 | LR: 2.62e-04 | 2170.0ms/step | 9.2 samples/s +Step 21740/44000 | Loss: 2.9745 | LR: 2.62e-04 | 2186.8ms/step | 9.1 samples/s +Step 21750/44000 | Loss: 2.9252 | LR: 2.61e-04 | 2199.0ms/step | 9.1 samples/s +Validation at step 21750: Loss = 2.9694, Perplexity = 19.48 +Step 21760/44000 | Loss: 2.9281 | LR: 2.61e-04 | 4151.5ms/step | 4.8 samples/s +Step 21770/44000 | Loss: 2.9798 | LR: 2.61e-04 | 2287.5ms/step | 8.7 samples/s +Step 21780/44000 | Loss: 2.9132 | LR: 2.61e-04 | 2208.8ms/step | 9.1 samples/s +Step 21790/44000 | Loss: 2.9406 | LR: 2.61e-04 | 2252.8ms/step | 8.9 samples/s +Step 21800/44000 | Loss: 2.9320 | LR: 2.60e-04 | 2111.7ms/step | 9.5 samples/s +Step 21810/44000 | Loss: 2.9397 | LR: 2.60e-04 | 2147.6ms/step | 9.3 samples/s +Step 21820/44000 | Loss: 2.9255 | LR: 2.60e-04 | 2214.4ms/step | 9.0 samples/s +Step 21830/44000 | Loss: 2.9376 | LR: 2.60e-04 | 2200.3ms/step | 9.1 samples/s +Step 21840/44000 | Loss: 2.9496 | LR: 2.60e-04 | 2241.1ms/step | 8.9 samples/s +Step 21850/44000 | Loss: 2.9269 | LR: 2.60e-04 | 2222.4ms/step | 9.0 samples/s +Step 21860/44000 | Loss: 2.9935 | LR: 2.59e-04 | 2269.0ms/step | 8.8 samples/s +Step 21870/44000 | Loss: 2.9472 | LR: 2.59e-04 | 2273.9ms/step | 8.8 samples/s +Step 21880/44000 | Loss: 2.9397 | LR: 2.59e-04 | 2225.0ms/step | 9.0 samples/s +Step 21890/44000 | Loss: 2.8878 | LR: 2.59e-04 | 2168.8ms/step | 9.2 samples/s +Step 21900/44000 | Loss: 2.9779 | LR: 2.59e-04 | 2203.9ms/step | 9.1 samples/s +Step 21910/44000 | Loss: 2.9746 | LR: 2.58e-04 | 2221.4ms/step | 9.0 samples/s +Step 21920/44000 | Loss: 2.9662 | LR: 2.58e-04 | 2217.3ms/step | 9.0 samples/s +Step 21930/44000 | Loss: 2.9819 | LR: 2.58e-04 | 2257.4ms/step | 8.9 samples/s +Step 21940/44000 | Loss: 2.9942 | LR: 2.58e-04 | 2213.3ms/step | 9.0 samples/s +Step 21950/44000 | Loss: 2.9473 | LR: 2.58e-04 | 2234.7ms/step | 8.9 samples/s +Step 21960/44000 | Loss: 2.9833 | LR: 2.58e-04 | 2244.7ms/step | 8.9 samples/s +Step 21970/44000 | Loss: 2.8903 | LR: 2.57e-04 | 2293.8ms/step | 8.7 samples/s +Step 21980/44000 | Loss: 2.9447 | LR: 2.57e-04 | 2164.4ms/step | 9.2 samples/s +Step 21990/44000 | Loss: 2.9347 | LR: 2.57e-04 | 2173.7ms/step | 9.2 samples/s +Step 22000/44000 | Loss: 2.9771 | LR: 2.57e-04 | 2145.3ms/step | 9.3 samples/s +Validation at step 22000: Loss = 2.9671, Perplexity = 19.44 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-22000 +Step 22010/44000 | Loss: 2.9541 | LR: 2.57e-04 | 4781.5ms/step | 4.2 samples/s +Step 22020/44000 | Loss: 2.9585 | LR: 2.56e-04 | 2286.2ms/step | 8.7 samples/s +Step 22030/44000 | Loss: 2.9811 | LR: 2.56e-04 | 2269.9ms/step | 8.8 samples/s +Step 22040/44000 | Loss: 2.9504 | LR: 2.56e-04 | 2255.0ms/step | 8.9 samples/s +Step 22050/44000 | Loss: 2.9621 | LR: 2.56e-04 | 2136.9ms/step | 9.4 samples/s +Step 22060/44000 | Loss: 2.9564 | LR: 2.56e-04 | 2196.5ms/step | 9.1 samples/s +Step 22070/44000 | Loss: 2.9730 | LR: 2.56e-04 | 2131.6ms/step | 9.4 samples/s +Step 22080/44000 | Loss: 2.9258 | LR: 2.55e-04 | 2225.1ms/step | 9.0 samples/s +Step 22090/44000 | Loss: 2.9488 | LR: 2.55e-04 | 2277.6ms/step | 8.8 samples/s +Step 22100/44000 | Loss: 2.9494 | LR: 2.55e-04 | 2229.7ms/step | 9.0 samples/s +Step 22110/44000 | Loss: 2.9361 | LR: 2.55e-04 | 2121.4ms/step | 9.4 samples/s +Step 22120/44000 | Loss: 2.9461 | LR: 2.55e-04 | 2180.7ms/step | 9.2 samples/s +Step 22130/44000 | Loss: 2.9440 | LR: 2.54e-04 | 2221.1ms/step | 9.0 samples/s +Step 22140/44000 | Loss: 2.9590 | LR: 2.54e-04 | 2137.2ms/step | 9.4 samples/s +Step 22150/44000 | Loss: 2.9808 | LR: 2.54e-04 | 2225.7ms/step | 9.0 samples/s +Step 22160/44000 | Loss: 2.9451 | LR: 2.54e-04 | 2121.9ms/step | 9.4 samples/s +Step 22170/44000 | Loss: 2.9678 | LR: 2.54e-04 | 2288.7ms/step | 8.7 samples/s +Step 22180/44000 | Loss: 2.9486 | LR: 2.54e-04 | 2281.5ms/step | 8.8 samples/s +Step 22190/44000 | Loss: 2.9198 | LR: 2.53e-04 | 2266.6ms/step | 8.8 samples/s +Step 22200/44000 | Loss: 2.9443 | LR: 2.53e-04 | 2272.4ms/step | 8.8 samples/s +Step 22210/44000 | Loss: 2.9686 | LR: 2.53e-04 | 2251.6ms/step | 8.9 samples/s +Step 22220/44000 | Loss: 2.9636 | LR: 2.53e-04 | 2184.7ms/step | 9.2 samples/s +Step 22230/44000 | Loss: 2.9552 | LR: 2.53e-04 | 2125.7ms/step | 9.4 samples/s +Step 22240/44000 | Loss: 2.9785 | LR: 2.52e-04 | 2221.5ms/step | 9.0 samples/s +Step 22250/44000 | Loss: 2.9234 | LR: 2.52e-04 | 2141.4ms/step | 9.3 samples/s +Validation at step 22250: Loss = 2.9645, Perplexity = 19.38 +Step 22260/44000 | Loss: 2.8997 | LR: 2.52e-04 | 4118.5ms/step | 4.9 samples/s +Step 22270/44000 | Loss: 2.9532 | LR: 2.52e-04 | 2267.1ms/step | 8.8 samples/s +Step 22280/44000 | Loss: 2.9838 | LR: 2.52e-04 | 2222.5ms/step | 9.0 samples/s +Step 22290/44000 | Loss: 2.9299 | LR: 2.52e-04 | 2149.7ms/step | 9.3 samples/s +Step 22300/44000 | Loss: 2.9109 | LR: 2.51e-04 | 2163.5ms/step | 9.2 samples/s +Step 22310/44000 | Loss: 2.9626 | LR: 2.51e-04 | 2166.8ms/step | 9.2 samples/s +Step 22320/44000 | Loss: 2.9386 | LR: 2.51e-04 | 2231.1ms/step | 9.0 samples/s +Step 22330/44000 | Loss: 2.9454 | LR: 2.51e-04 | 2201.7ms/step | 9.1 samples/s +Step 22340/44000 | Loss: 2.9269 | LR: 2.51e-04 | 2250.8ms/step | 8.9 samples/s +Step 22350/44000 | Loss: 2.9671 | LR: 2.50e-04 | 2238.1ms/step | 8.9 samples/s +Step 22360/44000 | Loss: 2.9496 | LR: 2.50e-04 | 2265.2ms/step | 8.8 samples/s +Step 22370/44000 | Loss: 2.9021 | LR: 2.50e-04 | 2256.9ms/step | 8.9 samples/s +Step 22380/44000 | Loss: 2.9460 | LR: 2.50e-04 | 2183.8ms/step | 9.2 samples/s +Step 22390/44000 | Loss: 2.9560 | LR: 2.50e-04 | 2227.1ms/step | 9.0 samples/s +Step 22400/44000 | Loss: 2.9131 | LR: 2.50e-04 | 2167.7ms/step | 9.2 samples/s +Step 22410/44000 | Loss: 2.9471 | LR: 2.49e-04 | 2176.7ms/step | 9.2 samples/s +Step 22420/44000 | Loss: 2.9637 | LR: 2.49e-04 | 2118.3ms/step | 9.4 samples/s +Step 22430/44000 | Loss: 2.9601 | LR: 2.49e-04 | 2069.0ms/step | 9.7 samples/s +Step 22440/44000 | Loss: 2.9662 | LR: 2.49e-04 | 2133.0ms/step | 9.4 samples/s +Step 22450/44000 | Loss: 2.9438 | LR: 2.49e-04 | 2194.9ms/step | 9.1 samples/s +Step 22460/44000 | Loss: 2.9335 | LR: 2.48e-04 | 2225.8ms/step | 9.0 samples/s +Step 22470/44000 | Loss: 2.9652 | LR: 2.48e-04 | 2159.1ms/step | 9.3 samples/s +Step 22480/44000 | Loss: 3.0060 | LR: 2.48e-04 | 2162.4ms/step | 9.2 samples/s +Step 22490/44000 | Loss: 2.9250 | LR: 2.48e-04 | 2202.6ms/step | 9.1 samples/s +Step 22500/44000 | Loss: 2.9690 | LR: 2.48e-04 | 2233.1ms/step | 9.0 samples/s +Validation at step 22500: Loss = 2.9624, Perplexity = 19.35 +Step 22510/44000 | Loss: 2.9680 | LR: 2.48e-04 | 4178.9ms/step | 4.8 samples/s +Step 22520/44000 | Loss: 2.9328 | LR: 2.47e-04 | 2238.0ms/step | 8.9 samples/s +Step 22530/44000 | Loss: 2.9447 | LR: 2.47e-04 | 2243.1ms/step | 8.9 samples/s +Step 22540/44000 | Loss: 2.9224 | LR: 2.47e-04 | 2224.8ms/step | 9.0 samples/s +Step 22550/44000 | Loss: 2.9275 | LR: 2.47e-04 | 2126.9ms/step | 9.4 samples/s +Step 22560/44000 | Loss: 2.9092 | LR: 2.47e-04 | 2158.4ms/step | 9.3 samples/s +Step 22570/44000 | Loss: 2.9431 | LR: 2.46e-04 | 2194.6ms/step | 9.1 samples/s +Step 22580/44000 | Loss: 2.9347 | LR: 2.46e-04 | 2215.6ms/step | 9.0 samples/s +Step 22590/44000 | Loss: 2.8946 | LR: 2.46e-04 | 2246.9ms/step | 8.9 samples/s +Step 22600/44000 | Loss: 2.9380 | LR: 2.46e-04 | 2278.4ms/step | 8.8 samples/s +Step 22610/44000 | Loss: 2.9276 | LR: 2.46e-04 | 2247.3ms/step | 8.9 samples/s +Step 22620/44000 | Loss: 2.9539 | LR: 2.46e-04 | 2234.8ms/step | 8.9 samples/s +Step 22630/44000 | Loss: 2.9153 | LR: 2.45e-04 | 2196.5ms/step | 9.1 samples/s +Step 22640/44000 | Loss: 2.9241 | LR: 2.45e-04 | 2176.6ms/step | 9.2 samples/s +Step 22650/44000 | Loss: 2.9108 | LR: 2.45e-04 | 2301.3ms/step | 8.7 samples/s +Step 22660/44000 | Loss: 2.9600 | LR: 2.45e-04 | 2206.7ms/step | 9.1 samples/s +Step 22670/44000 | Loss: 2.9829 | LR: 2.45e-04 | 2450.0ms/step | 8.2 samples/s +Step 22680/44000 | Loss: 2.8907 | LR: 2.44e-04 | 2436.8ms/step | 8.2 samples/s +Step 22690/44000 | Loss: 2.9535 | LR: 2.44e-04 | 2255.0ms/step | 8.9 samples/s +Step 22700/44000 | Loss: 2.9586 | LR: 2.44e-04 | 2231.2ms/step | 9.0 samples/s +Step 22710/44000 | Loss: 3.0007 | LR: 2.44e-04 | 2230.6ms/step | 9.0 samples/s +Step 22720/44000 | Loss: 2.9740 | LR: 2.44e-04 | 2192.5ms/step | 9.1 samples/s +Step 22730/44000 | Loss: 2.9963 | LR: 2.44e-04 | 2190.6ms/step | 9.1 samples/s +Step 22740/44000 | Loss: 2.9621 | LR: 2.43e-04 | 2071.7ms/step | 9.7 samples/s +Step 22750/44000 | Loss: 2.9428 | LR: 2.43e-04 | 2026.9ms/step | 9.9 samples/s +Validation at step 22750: Loss = 2.9595, Perplexity = 19.29 +Step 22760/44000 | Loss: 2.9396 | LR: 2.43e-04 | 4071.5ms/step | 4.9 samples/s +Step 22770/44000 | Loss: 2.9197 | LR: 2.43e-04 | 2247.1ms/step | 8.9 samples/s +Step 22780/44000 | Loss: 2.9325 | LR: 2.43e-04 | 2283.1ms/step | 8.8 samples/s +Step 22790/44000 | Loss: 2.9339 | LR: 2.42e-04 | 2262.7ms/step | 8.8 samples/s +Step 22800/44000 | Loss: 2.9023 | LR: 2.42e-04 | 2180.6ms/step | 9.2 samples/s +Step 22810/44000 | Loss: 2.9370 | LR: 2.42e-04 | 2118.4ms/step | 9.4 samples/s +Step 22820/44000 | Loss: 2.9474 | LR: 2.42e-04 | 2217.6ms/step | 9.0 samples/s +Step 22830/44000 | Loss: 2.9457 | LR: 2.42e-04 | 2204.9ms/step | 9.1 samples/s +Step 22840/44000 | Loss: 2.9345 | LR: 2.42e-04 | 2268.0ms/step | 8.8 samples/s +Step 22850/44000 | Loss: 3.0042 | LR: 2.41e-04 | 2270.1ms/step | 8.8 samples/s +Step 22860/44000 | Loss: 2.9401 | LR: 2.41e-04 | 2215.7ms/step | 9.0 samples/s +Step 22870/44000 | Loss: 2.9246 | LR: 2.41e-04 | 2248.7ms/step | 8.9 samples/s +Step 22880/44000 | Loss: 2.9554 | LR: 2.41e-04 | 2253.7ms/step | 8.9 samples/s +Step 22890/44000 | Loss: 2.9707 | LR: 2.41e-04 | 2163.3ms/step | 9.2 samples/s +Step 22900/44000 | Loss: 2.9427 | LR: 2.40e-04 | 2177.6ms/step | 9.2 samples/s +Step 22910/44000 | Loss: 2.9272 | LR: 2.40e-04 | 2205.4ms/step | 9.1 samples/s +Step 22920/44000 | Loss: 2.9185 | LR: 2.40e-04 | 2176.5ms/step | 9.2 samples/s +Step 22930/44000 | Loss: 2.9591 | LR: 2.40e-04 | 2296.9ms/step | 8.7 samples/s +Step 22940/44000 | Loss: 2.9324 | LR: 2.40e-04 | 2255.3ms/step | 8.9 samples/s +Step 22950/44000 | Loss: 2.9616 | LR: 2.40e-04 | 2269.0ms/step | 8.8 samples/s +Step 22960/44000 | Loss: 2.9331 | LR: 2.39e-04 | 2237.1ms/step | 8.9 samples/s +Step 22970/44000 | Loss: 2.9645 | LR: 2.39e-04 | 2243.2ms/step | 8.9 samples/s +Step 22980/44000 | Loss: 2.9396 | LR: 2.39e-04 | 2195.7ms/step | 9.1 samples/s +Step 22990/44000 | Loss: 2.9275 | LR: 2.39e-04 | 2142.4ms/step | 9.3 samples/s +Step 23000/44000 | Loss: 2.9314 | LR: 2.39e-04 | 2230.3ms/step | 9.0 samples/s +Validation at step 23000: Loss = 2.9572, Perplexity = 19.24 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-23000 +Step 23010/44000 | Loss: 2.9602 | LR: 2.38e-04 | 4609.2ms/step | 4.3 samples/s +Step 23020/44000 | Loss: 2.9457 | LR: 2.38e-04 | 2254.3ms/step | 8.9 samples/s +Step 23030/44000 | Loss: 2.9782 | LR: 2.38e-04 | 2268.5ms/step | 8.8 samples/s +Step 23040/44000 | Loss: 2.9323 | LR: 2.38e-04 | 2292.4ms/step | 8.7 samples/s +Step 23050/44000 | Loss: 2.9476 | LR: 2.38e-04 | 2118.9ms/step | 9.4 samples/s +Step 23060/44000 | Loss: 2.9371 | LR: 2.38e-04 | 2103.1ms/step | 9.5 samples/s +Step 23070/44000 | Loss: 2.9165 | LR: 2.37e-04 | 2236.1ms/step | 8.9 samples/s +Step 23080/44000 | Loss: 2.9479 | LR: 2.37e-04 | 2221.0ms/step | 9.0 samples/s +Step 23090/44000 | Loss: 2.9823 | LR: 2.37e-04 | 2339.9ms/step | 8.5 samples/s +Step 23100/44000 | Loss: 2.9498 | LR: 2.37e-04 | 2310.7ms/step | 8.7 samples/s +Step 23110/44000 | Loss: 2.9445 | LR: 2.37e-04 | 2292.2ms/step | 8.7 samples/s +Step 23120/44000 | Loss: 2.9583 | LR: 2.36e-04 | 2337.1ms/step | 8.6 samples/s +Step 23130/44000 | Loss: 2.9425 | LR: 2.36e-04 | 2287.7ms/step | 8.7 samples/s +Step 23140/44000 | Loss: 2.9409 | LR: 2.36e-04 | 2169.9ms/step | 9.2 samples/s +Step 23150/44000 | Loss: 2.9377 | LR: 2.36e-04 | 2197.7ms/step | 9.1 samples/s +Step 23160/44000 | Loss: 2.9431 | LR: 2.36e-04 | 2217.6ms/step | 9.0 samples/s +Step 23170/44000 | Loss: 2.9362 | LR: 2.36e-04 | 2184.1ms/step | 9.2 samples/s +Step 23180/44000 | Loss: 2.9056 | LR: 2.35e-04 | 2333.0ms/step | 8.6 samples/s +Step 23190/44000 | Loss: 2.9340 | LR: 2.35e-04 | 2330.3ms/step | 8.6 samples/s +Step 23200/44000 | Loss: 2.9328 | LR: 2.35e-04 | 2257.8ms/step | 8.9 samples/s +Step 23210/44000 | Loss: 2.9399 | LR: 2.35e-04 | 2266.9ms/step | 8.8 samples/s +Step 23220/44000 | Loss: 2.9456 | LR: 2.35e-04 | 2273.1ms/step | 8.8 samples/s +Step 23230/44000 | Loss: 2.9269 | LR: 2.34e-04 | 2142.9ms/step | 9.3 samples/s +Step 23240/44000 | Loss: 2.9588 | LR: 2.34e-04 | 2226.8ms/step | 9.0 samples/s +Step 23250/44000 | Loss: 2.9435 | LR: 2.34e-04 | 2201.2ms/step | 9.1 samples/s +Validation at step 23250: Loss = 2.9546, Perplexity = 19.19 +Step 23260/44000 | Loss: 2.9334 | LR: 2.34e-04 | 4378.8ms/step | 4.6 samples/s +Step 23270/44000 | Loss: 2.9756 | LR: 2.34e-04 | 2309.2ms/step | 8.7 samples/s +Step 23280/44000 | Loss: 2.9720 | LR: 2.34e-04 | 2304.1ms/step | 8.7 samples/s +Step 23290/44000 | Loss: 2.9295 | LR: 2.33e-04 | 2236.8ms/step | 8.9 samples/s +Step 23300/44000 | Loss: 2.9181 | LR: 2.33e-04 | 2245.0ms/step | 8.9 samples/s +Step 23310/44000 | Loss: 2.9317 | LR: 2.33e-04 | 2181.1ms/step | 9.2 samples/s +Step 23320/44000 | Loss: 2.9661 | LR: 2.33e-04 | 2210.4ms/step | 9.0 samples/s +Step 23330/44000 | Loss: 2.9388 | LR: 2.33e-04 | 2200.0ms/step | 9.1 samples/s +Step 23340/44000 | Loss: 2.9235 | LR: 2.32e-04 | 2168.8ms/step | 9.2 samples/s +Step 23350/44000 | Loss: 2.9972 | LR: 2.32e-04 | 2250.5ms/step | 8.9 samples/s +Step 23360/44000 | Loss: 2.9439 | LR: 2.32e-04 | 2185.9ms/step | 9.1 samples/s +Step 23370/44000 | Loss: 2.9289 | LR: 2.32e-04 | 2180.1ms/step | 9.2 samples/s +Step 23380/44000 | Loss: 2.9378 | LR: 2.32e-04 | 2243.8ms/step | 8.9 samples/s +Step 23390/44000 | Loss: 2.9752 | LR: 2.32e-04 | 2211.4ms/step | 9.0 samples/s +Step 23400/44000 | Loss: 2.8817 | LR: 2.31e-04 | 2163.6ms/step | 9.2 samples/s +Step 23410/44000 | Loss: 2.9214 | LR: 2.31e-04 | 2221.1ms/step | 9.0 samples/s +Step 23420/44000 | Loss: 2.9650 | LR: 2.31e-04 | 2134.6ms/step | 9.4 samples/s +Step 23430/44000 | Loss: 2.9702 | LR: 2.31e-04 | 2216.4ms/step | 9.0 samples/s +Step 23440/44000 | Loss: 2.9921 | LR: 2.31e-04 | 2285.1ms/step | 8.8 samples/s +Step 23450/44000 | Loss: 2.9260 | LR: 2.30e-04 | 2246.5ms/step | 8.9 samples/s +Step 23460/44000 | Loss: 2.9555 | LR: 2.30e-04 | 2258.0ms/step | 8.9 samples/s +Step 23470/44000 | Loss: 2.9678 | LR: 2.30e-04 | 2229.5ms/step | 9.0 samples/s +Step 23480/44000 | Loss: 2.9630 | LR: 2.30e-04 | 2265.3ms/step | 8.8 samples/s +Step 23490/44000 | Loss: 2.9387 | LR: 2.30e-04 | 2138.0ms/step | 9.4 samples/s +Step 23500/44000 | Loss: 2.9241 | LR: 2.30e-04 | 2188.7ms/step | 9.1 samples/s +Validation at step 23500: Loss = 2.9528, Perplexity = 19.16 +Step 23510/44000 | Loss: 2.9521 | LR: 2.29e-04 | 4107.7ms/step | 4.9 samples/s +Step 23520/44000 | Loss: 2.9630 | LR: 2.29e-04 | 2243.2ms/step | 8.9 samples/s +Step 23530/44000 | Loss: 2.9459 | LR: 2.29e-04 | 2260.8ms/step | 8.8 samples/s +Step 23540/44000 | Loss: 2.9212 | LR: 2.29e-04 | 2264.5ms/step | 8.8 samples/s +Step 23550/44000 | Loss: 2.9400 | LR: 2.29e-04 | 2272.0ms/step | 8.8 samples/s +Step 23560/44000 | Loss: 2.9261 | LR: 2.29e-04 | 2213.1ms/step | 9.0 samples/s +Step 23570/44000 | Loss: 2.9436 | LR: 2.28e-04 | 2160.6ms/step | 9.3 samples/s +Step 23580/44000 | Loss: 2.9363 | LR: 2.28e-04 | 2185.7ms/step | 9.2 samples/s +Step 23590/44000 | Loss: 2.9337 | LR: 2.28e-04 | 2182.6ms/step | 9.2 samples/s +Step 23600/44000 | Loss: 2.9271 | LR: 2.28e-04 | 2231.0ms/step | 9.0 samples/s +Step 23610/44000 | Loss: 2.9363 | LR: 2.28e-04 | 2323.0ms/step | 8.6 samples/s +Step 23620/44000 | Loss: 2.9443 | LR: 2.27e-04 | 2377.6ms/step | 8.4 samples/s +Step 23630/44000 | Loss: 2.9792 | LR: 2.27e-04 | 2253.4ms/step | 8.9 samples/s +Step 23640/44000 | Loss: 2.9054 | LR: 2.27e-04 | 2246.8ms/step | 8.9 samples/s +Step 23650/44000 | Loss: 2.8709 | LR: 2.27e-04 | 2208.8ms/step | 9.1 samples/s +Step 23660/44000 | Loss: 2.9504 | LR: 2.27e-04 | 2160.0ms/step | 9.3 samples/s +Step 23670/44000 | Loss: 2.9225 | LR: 2.27e-04 | 2184.3ms/step | 9.2 samples/s +Step 23680/44000 | Loss: 2.9334 | LR: 2.26e-04 | 2063.3ms/step | 9.7 samples/s +Step 23690/44000 | Loss: 2.9504 | LR: 2.26e-04 | 2181.0ms/step | 9.2 samples/s +Step 23700/44000 | Loss: 2.9350 | LR: 2.26e-04 | 2270.8ms/step | 8.8 samples/s +Step 23710/44000 | Loss: 2.9105 | LR: 2.26e-04 | 2274.9ms/step | 8.8 samples/s +Step 23720/44000 | Loss: 2.9489 | LR: 2.26e-04 | 2266.3ms/step | 8.8 samples/s +Step 23730/44000 | Loss: 2.9129 | LR: 2.25e-04 | 2156.5ms/step | 9.3 samples/s +Step 23740/44000 | Loss: 2.9294 | LR: 2.25e-04 | 2191.9ms/step | 9.1 samples/s +Step 23750/44000 | Loss: 2.9459 | LR: 2.25e-04 | 2183.9ms/step | 9.2 samples/s +Validation at step 23750: Loss = 2.9502, Perplexity = 19.11 +Step 23760/44000 | Loss: 2.9365 | LR: 2.25e-04 | 4232.1ms/step | 4.7 samples/s +Step 23770/44000 | Loss: 2.9150 | LR: 2.25e-04 | 2267.2ms/step | 8.8 samples/s +Step 23780/44000 | Loss: 2.9449 | LR: 2.25e-04 | 2269.6ms/step | 8.8 samples/s +Step 23790/44000 | Loss: 2.9208 | LR: 2.24e-04 | 2234.2ms/step | 9.0 samples/s +Step 23800/44000 | Loss: 2.9465 | LR: 2.24e-04 | 2272.4ms/step | 8.8 samples/s +Step 23810/44000 | Loss: 2.9617 | LR: 2.24e-04 | 2193.7ms/step | 9.1 samples/s +Step 23820/44000 | Loss: 2.9524 | LR: 2.24e-04 | 2182.5ms/step | 9.2 samples/s +Step 23830/44000 | Loss: 2.9308 | LR: 2.24e-04 | 2212.4ms/step | 9.0 samples/s +Step 23840/44000 | Loss: 2.9009 | LR: 2.23e-04 | 2183.2ms/step | 9.2 samples/s +Step 23850/44000 | Loss: 2.9513 | LR: 2.23e-04 | 2340.7ms/step | 8.5 samples/s +Step 23860/44000 | Loss: 2.9058 | LR: 2.23e-04 | 2283.1ms/step | 8.8 samples/s +Step 23870/44000 | Loss: 2.9790 | LR: 2.23e-04 | 2236.2ms/step | 8.9 samples/s +Step 23880/44000 | Loss: 2.9907 | LR: 2.23e-04 | 2223.8ms/step | 9.0 samples/s +Step 23890/44000 | Loss: 2.8998 | LR: 2.23e-04 | 2268.8ms/step | 8.8 samples/s +Step 23900/44000 | Loss: 2.9463 | LR: 2.22e-04 | 2162.4ms/step | 9.2 samples/s +Step 23910/44000 | Loss: 2.9464 | LR: 2.22e-04 | 2180.0ms/step | 9.2 samples/s +Step 23920/44000 | Loss: 2.9380 | LR: 2.22e-04 | 2166.4ms/step | 9.2 samples/s +Step 23930/44000 | Loss: 2.9265 | LR: 2.22e-04 | 2128.4ms/step | 9.4 samples/s +Step 23940/44000 | Loss: 2.9128 | LR: 2.22e-04 | 2188.6ms/step | 9.1 samples/s +Step 23950/44000 | Loss: 2.8999 | LR: 2.21e-04 | 2189.5ms/step | 9.1 samples/s +Step 23960/44000 | Loss: 2.9394 | LR: 2.21e-04 | 2247.7ms/step | 8.9 samples/s +Step 23970/44000 | Loss: 2.9165 | LR: 2.21e-04 | 2198.4ms/step | 9.1 samples/s +Step 23980/44000 | Loss: 2.9693 | LR: 2.21e-04 | 2226.1ms/step | 9.0 samples/s +Step 23990/44000 | Loss: 2.9197 | LR: 2.21e-04 | 2059.2ms/step | 9.7 samples/s +Step 24000/44000 | Loss: 2.8909 | LR: 2.21e-04 | 2022.9ms/step | 9.9 samples/s +Validation at step 24000: Loss = 2.9485, Perplexity = 19.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-24000 +Step 24010/44000 | Loss: 2.9450 | LR: 2.20e-04 | 4617.4ms/step | 4.3 samples/s +Step 24020/44000 | Loss: 2.9132 | LR: 2.20e-04 | 2174.5ms/step | 9.2 samples/s +Step 24030/44000 | Loss: 2.9095 | LR: 2.20e-04 | 2168.5ms/step | 9.2 samples/s +Step 24040/44000 | Loss: 2.9216 | LR: 2.20e-04 | 2147.8ms/step | 9.3 samples/s +Step 24050/44000 | Loss: 2.9026 | LR: 2.20e-04 | 2156.4ms/step | 9.3 samples/s +Step 24060/44000 | Loss: 2.9127 | LR: 2.19e-04 | 2167.2ms/step | 9.2 samples/s +Step 24070/44000 | Loss: 2.9528 | LR: 2.19e-04 | 2166.7ms/step | 9.2 samples/s +Step 24080/44000 | Loss: 2.9389 | LR: 2.19e-04 | 2261.2ms/step | 8.8 samples/s +Step 24090/44000 | Loss: 2.9387 | LR: 2.19e-04 | 2304.3ms/step | 8.7 samples/s +Step 24100/44000 | Loss: 2.9267 | LR: 2.19e-04 | 2197.5ms/step | 9.1 samples/s +Step 24110/44000 | Loss: 2.9345 | LR: 2.19e-04 | 2197.8ms/step | 9.1 samples/s +Step 24120/44000 | Loss: 2.9195 | LR: 2.18e-04 | 2194.1ms/step | 9.1 samples/s +Step 24130/44000 | Loss: 2.9405 | LR: 2.18e-04 | 2197.2ms/step | 9.1 samples/s +Step 24140/44000 | Loss: 2.9393 | LR: 2.18e-04 | 2187.1ms/step | 9.1 samples/s +Step 24150/44000 | Loss: 2.8997 | LR: 2.18e-04 | 2231.0ms/step | 9.0 samples/s +Step 24160/44000 | Loss: 2.9459 | LR: 2.18e-04 | 2252.0ms/step | 8.9 samples/s +Step 24170/44000 | Loss: 2.9809 | LR: 2.17e-04 | 2267.6ms/step | 8.8 samples/s +Step 24180/44000 | Loss: 2.9242 | LR: 2.17e-04 | 2252.4ms/step | 8.9 samples/s +Step 24190/44000 | Loss: 2.9202 | LR: 2.17e-04 | 2192.7ms/step | 9.1 samples/s +Step 24200/44000 | Loss: 2.9051 | LR: 2.17e-04 | 2166.9ms/step | 9.2 samples/s +Step 24210/44000 | Loss: 2.9212 | LR: 2.17e-04 | 2224.4ms/step | 9.0 samples/s +Step 24220/44000 | Loss: 2.8969 | LR: 2.17e-04 | 2185.4ms/step | 9.2 samples/s +Step 24230/44000 | Loss: 2.9558 | LR: 2.16e-04 | 2230.7ms/step | 9.0 samples/s +Step 24240/44000 | Loss: 2.9152 | LR: 2.16e-04 | 2221.2ms/step | 9.0 samples/s +Step 24250/44000 | Loss: 2.9169 | LR: 2.16e-04 | 2270.4ms/step | 8.8 samples/s +Validation at step 24250: Loss = 2.9458, Perplexity = 19.03 +Step 24260/44000 | Loss: 2.9246 | LR: 2.16e-04 | 4276.4ms/step | 4.7 samples/s +Step 24270/44000 | Loss: 2.9644 | LR: 2.16e-04 | 2274.1ms/step | 8.8 samples/s +Step 24280/44000 | Loss: 2.9145 | LR: 2.16e-04 | 2184.2ms/step | 9.2 samples/s +Step 24290/44000 | Loss: 2.9355 | LR: 2.15e-04 | 2246.6ms/step | 8.9 samples/s +Step 24300/44000 | Loss: 2.9079 | LR: 2.15e-04 | 2089.7ms/step | 9.6 samples/s +Step 24310/44000 | Loss: 2.9155 | LR: 2.15e-04 | 2153.3ms/step | 9.3 samples/s +Step 24320/44000 | Loss: 2.9547 | LR: 2.15e-04 | 2256.6ms/step | 8.9 samples/s +Step 24330/44000 | Loss: 2.9297 | LR: 2.15e-04 | 2221.8ms/step | 9.0 samples/s +Step 24340/44000 | Loss: 2.9345 | LR: 2.14e-04 | 2249.7ms/step | 8.9 samples/s +Step 24350/44000 | Loss: 2.9161 | LR: 2.14e-04 | 2246.2ms/step | 8.9 samples/s +Step 24360/44000 | Loss: 2.9482 | LR: 2.14e-04 | 2267.2ms/step | 8.8 samples/s +Step 24370/44000 | Loss: 2.9214 | LR: 2.14e-04 | 2225.4ms/step | 9.0 samples/s +Step 24380/44000 | Loss: 2.9184 | LR: 2.14e-04 | 2173.9ms/step | 9.2 samples/s +Step 24390/44000 | Loss: 2.9295 | LR: 2.14e-04 | 2349.0ms/step | 8.5 samples/s +Step 24400/44000 | Loss: 2.9376 | LR: 2.13e-04 | 2229.1ms/step | 9.0 samples/s +Step 24410/44000 | Loss: 2.9359 | LR: 2.13e-04 | 2468.0ms/step | 8.1 samples/s +Step 24420/44000 | Loss: 2.9101 | LR: 2.13e-04 | 2501.9ms/step | 8.0 samples/s +Step 24430/44000 | Loss: 2.9292 | LR: 2.13e-04 | 2261.9ms/step | 8.8 samples/s +Step 24440/44000 | Loss: 2.8949 | LR: 2.13e-04 | 2204.6ms/step | 9.1 samples/s +Step 24450/44000 | Loss: 2.9372 | LR: 2.12e-04 | 2167.5ms/step | 9.2 samples/s +Step 24460/44000 | Loss: 2.9016 | LR: 2.12e-04 | 2142.0ms/step | 9.3 samples/s +Step 24470/44000 | Loss: 2.9710 | LR: 2.12e-04 | 2101.4ms/step | 9.5 samples/s +Step 24480/44000 | Loss: 2.9321 | LR: 2.12e-04 | 2213.6ms/step | 9.0 samples/s +Step 24490/44000 | Loss: 2.9340 | LR: 2.12e-04 | 2225.6ms/step | 9.0 samples/s +Step 24500/44000 | Loss: 2.9643 | LR: 2.12e-04 | 2251.8ms/step | 8.9 samples/s +Validation at step 24500: Loss = 2.9442, Perplexity = 19.00 +Step 24510/44000 | Loss: 2.8601 | LR: 2.11e-04 | 4156.6ms/step | 4.8 samples/s +Step 24520/44000 | Loss: 2.9183 | LR: 2.11e-04 | 2251.0ms/step | 8.9 samples/s +Step 24530/44000 | Loss: 2.8945 | LR: 2.11e-04 | 2220.8ms/step | 9.0 samples/s +Step 24540/44000 | Loss: 2.9066 | LR: 2.11e-04 | 2170.9ms/step | 9.2 samples/s +Step 24550/44000 | Loss: 2.9060 | LR: 2.11e-04 | 2212.2ms/step | 9.0 samples/s +Step 24560/44000 | Loss: 2.9278 | LR: 2.10e-04 | 2303.2ms/step | 8.7 samples/s +Step 24570/44000 | Loss: 2.9509 | LR: 2.10e-04 | 2238.5ms/step | 8.9 samples/s +Step 24580/44000 | Loss: 2.9488 | LR: 2.10e-04 | 2243.6ms/step | 8.9 samples/s +Step 24590/44000 | Loss: 2.9295 | LR: 2.10e-04 | 2288.6ms/step | 8.7 samples/s +Step 24600/44000 | Loss: 2.9318 | LR: 2.10e-04 | 2271.4ms/step | 8.8 samples/s +Step 24610/44000 | Loss: 2.9281 | LR: 2.10e-04 | 2157.6ms/step | 9.3 samples/s +Step 24620/44000 | Loss: 2.9038 | LR: 2.09e-04 | 2082.1ms/step | 9.6 samples/s +Step 24630/44000 | Loss: 2.8779 | LR: 2.09e-04 | 2181.1ms/step | 9.2 samples/s +Step 24640/44000 | Loss: 2.9356 | LR: 2.09e-04 | 2225.0ms/step | 9.0 samples/s +Step 24650/44000 | Loss: 2.9464 | LR: 2.09e-04 | 2282.7ms/step | 8.8 samples/s +Step 24660/44000 | Loss: 2.8921 | LR: 2.09e-04 | 2269.9ms/step | 8.8 samples/s +Step 24670/44000 | Loss: 2.9314 | LR: 2.09e-04 | 2248.7ms/step | 8.9 samples/s +Step 24680/44000 | Loss: 2.9521 | LR: 2.08e-04 | 2215.1ms/step | 9.0 samples/s +Step 24690/44000 | Loss: 2.9242 | LR: 2.08e-04 | 2167.4ms/step | 9.2 samples/s +Step 24700/44000 | Loss: 2.8886 | LR: 2.08e-04 | 2219.5ms/step | 9.0 samples/s +Step 24710/44000 | Loss: 2.9246 | LR: 2.08e-04 | 2213.8ms/step | 9.0 samples/s +Step 24720/44000 | Loss: 2.9338 | LR: 2.08e-04 | 2266.7ms/step | 8.8 samples/s +Step 24730/44000 | Loss: 2.9384 | LR: 2.07e-04 | 2219.8ms/step | 9.0 samples/s +Step 24740/44000 | Loss: 2.9527 | LR: 2.07e-04 | 2258.1ms/step | 8.9 samples/s +Step 24750/44000 | Loss: 2.9421 | LR: 2.07e-04 | 2214.4ms/step | 9.0 samples/s +Validation at step 24750: Loss = 2.9423, Perplexity = 18.96 +Step 24760/44000 | Loss: 2.9050 | LR: 2.07e-04 | 4134.8ms/step | 4.8 samples/s +Step 24770/44000 | Loss: 2.9141 | LR: 2.07e-04 | 2241.4ms/step | 8.9 samples/s +Step 24780/44000 | Loss: 2.9265 | LR: 2.07e-04 | 2240.3ms/step | 8.9 samples/s +Step 24790/44000 | Loss: 2.8798 | LR: 2.06e-04 | 2203.9ms/step | 9.1 samples/s +Step 24800/44000 | Loss: 2.9590 | LR: 2.06e-04 | 2230.1ms/step | 9.0 samples/s +Step 24810/44000 | Loss: 2.9489 | LR: 2.06e-04 | 2129.5ms/step | 9.4 samples/s +Step 24820/44000 | Loss: 2.9246 | LR: 2.06e-04 | 2087.7ms/step | 9.6 samples/s +Step 24830/44000 | Loss: 2.9522 | LR: 2.06e-04 | 2099.3ms/step | 9.5 samples/s +Step 24840/44000 | Loss: 2.8904 | LR: 2.05e-04 | 2069.3ms/step | 9.7 samples/s +Step 24850/44000 | Loss: 2.9083 | LR: 2.05e-04 | 2078.3ms/step | 9.6 samples/s +Step 24860/44000 | Loss: 2.9085 | LR: 2.05e-04 | 2042.6ms/step | 9.8 samples/s +Step 24870/44000 | Loss: 2.9551 | LR: 2.05e-04 | 2066.5ms/step | 9.7 samples/s +Step 24880/44000 | Loss: 2.9268 | LR: 2.05e-04 | 2090.9ms/step | 9.6 samples/s +Step 24890/44000 | Loss: 2.9215 | LR: 2.05e-04 | 2073.1ms/step | 9.6 samples/s +Step 24900/44000 | Loss: 2.9195 | LR: 2.04e-04 | 2084.1ms/step | 9.6 samples/s +Step 24910/44000 | Loss: 2.9241 | LR: 2.04e-04 | 2078.4ms/step | 9.6 samples/s +Step 24920/44000 | Loss: 2.9274 | LR: 2.04e-04 | 2046.5ms/step | 9.8 samples/s +Step 24930/44000 | Loss: 2.8895 | LR: 2.04e-04 | 2037.2ms/step | 9.8 samples/s +Step 24940/44000 | Loss: 2.9510 | LR: 2.04e-04 | 2073.8ms/step | 9.6 samples/s +Step 24950/44000 | Loss: 2.9207 | LR: 2.04e-04 | 2110.0ms/step | 9.5 samples/s +Step 24960/44000 | Loss: 2.8923 | LR: 2.03e-04 | 2068.3ms/step | 9.7 samples/s +Step 24970/44000 | Loss: 2.9335 | LR: 2.03e-04 | 2050.2ms/step | 9.8 samples/s +Step 24980/44000 | Loss: 2.9449 | LR: 2.03e-04 | 2044.6ms/step | 9.8 samples/s +Step 24990/44000 | Loss: 2.9152 | LR: 2.03e-04 | 2058.1ms/step | 9.7 samples/s +Step 25000/44000 | Loss: 2.9211 | LR: 2.03e-04 | 2068.1ms/step | 9.7 samples/s +Validation at step 25000: Loss = 2.9404, Perplexity = 18.92 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-25000 +Step 25010/44000 | Loss: 2.9256 | LR: 2.02e-04 | 4329.3ms/step | 4.6 samples/s +Step 25020/44000 | Loss: 2.9518 | LR: 2.02e-04 | 2057.6ms/step | 9.7 samples/s +Step 25030/44000 | Loss: 2.9082 | LR: 2.02e-04 | 2037.3ms/step | 9.8 samples/s +Step 25040/44000 | Loss: 2.9420 | LR: 2.02e-04 | 2078.5ms/step | 9.6 samples/s +Step 25050/44000 | Loss: 2.9314 | LR: 2.02e-04 | 2125.8ms/step | 9.4 samples/s +Step 25060/44000 | Loss: 2.9781 | LR: 2.02e-04 | 2109.9ms/step | 9.5 samples/s +Step 25070/44000 | Loss: 2.9403 | LR: 2.01e-04 | 2125.6ms/step | 9.4 samples/s +Step 25080/44000 | Loss: 2.9149 | LR: 2.01e-04 | 2062.7ms/step | 9.7 samples/s +Step 25090/44000 | Loss: 2.9317 | LR: 2.01e-04 | 2041.6ms/step | 9.8 samples/s +Step 25100/44000 | Loss: 2.8956 | LR: 2.01e-04 | 2058.3ms/step | 9.7 samples/s +Step 25110/44000 | Loss: 2.9133 | LR: 2.01e-04 | 2064.0ms/step | 9.7 samples/s +Step 25120/44000 | Loss: 2.8853 | LR: 2.00e-04 | 2125.3ms/step | 9.4 samples/s +Step 25130/44000 | Loss: 2.9242 | LR: 2.00e-04 | 2105.0ms/step | 9.5 samples/s +Step 25140/44000 | Loss: 2.9288 | LR: 2.00e-04 | 2069.3ms/step | 9.7 samples/s +Step 25150/44000 | Loss: 2.8981 | LR: 2.00e-04 | 2047.9ms/step | 9.8 samples/s +Step 25160/44000 | Loss: 2.8848 | LR: 2.00e-04 | 2042.7ms/step | 9.8 samples/s +Step 25170/44000 | Loss: 2.9428 | LR: 2.00e-04 | 2080.1ms/step | 9.6 samples/s +Step 25180/44000 | Loss: 2.9538 | LR: 1.99e-04 | 2064.0ms/step | 9.7 samples/s +Step 25190/44000 | Loss: 2.9440 | LR: 1.99e-04 | 2110.3ms/step | 9.5 samples/s +Step 25200/44000 | Loss: 2.9240 | LR: 1.99e-04 | 2092.3ms/step | 9.6 samples/s +Step 25210/44000 | Loss: 2.8951 | LR: 1.99e-04 | 2103.0ms/step | 9.5 samples/s +Step 25220/44000 | Loss: 2.8741 | LR: 1.99e-04 | 2053.0ms/step | 9.7 samples/s +Step 25230/44000 | Loss: 2.9033 | LR: 1.99e-04 | 2027.5ms/step | 9.9 samples/s +Step 25240/44000 | Loss: 2.8785 | LR: 1.98e-04 | 2026.8ms/step | 9.9 samples/s +Step 25250/44000 | Loss: 2.8733 | LR: 1.98e-04 | 2058.3ms/step | 9.7 samples/s +Validation at step 25250: Loss = 2.9392, Perplexity = 18.90 +Step 25260/44000 | Loss: 2.9377 | LR: 1.98e-04 | 3902.6ms/step | 5.1 samples/s +Step 25270/44000 | Loss: 2.9640 | LR: 1.98e-04 | 2052.3ms/step | 9.7 samples/s +Step 25280/44000 | Loss: 2.9083 | LR: 1.98e-04 | 2028.8ms/step | 9.9 samples/s +Step 25290/44000 | Loss: 2.9428 | LR: 1.97e-04 | 2064.3ms/step | 9.7 samples/s +Step 25300/44000 | Loss: 2.9059 | LR: 1.97e-04 | 2089.9ms/step | 9.6 samples/s +Step 25310/44000 | Loss: 2.9351 | LR: 1.97e-04 | 2069.2ms/step | 9.7 samples/s +Step 25320/44000 | Loss: 2.9687 | LR: 1.97e-04 | 2114.1ms/step | 9.5 samples/s +Step 25330/44000 | Loss: 2.9183 | LR: 1.97e-04 | 2065.4ms/step | 9.7 samples/s +Step 25340/44000 | Loss: 2.9098 | LR: 1.97e-04 | 2048.4ms/step | 9.8 samples/s +Step 25350/44000 | Loss: 2.9111 | LR: 1.96e-04 | 2048.0ms/step | 9.8 samples/s +Step 25360/44000 | Loss: 2.9466 | LR: 1.96e-04 | 2042.8ms/step | 9.8 samples/s +Step 25370/44000 | Loss: 2.9426 | LR: 1.96e-04 | 2064.2ms/step | 9.7 samples/s +Step 25380/44000 | Loss: 2.8939 | LR: 1.96e-04 | 2052.6ms/step | 9.7 samples/s +Step 25390/44000 | Loss: 2.8786 | LR: 1.96e-04 | 2042.9ms/step | 9.8 samples/s +Step 25400/44000 | Loss: 2.9152 | LR: 1.96e-04 | 2016.8ms/step | 9.9 samples/s +Step 25410/44000 | Loss: 2.9210 | LR: 1.95e-04 | 2058.7ms/step | 9.7 samples/s +Step 25420/44000 | Loss: 2.8893 | LR: 1.95e-04 | 2110.4ms/step | 9.5 samples/s +Step 25430/44000 | Loss: 2.8837 | LR: 1.95e-04 | 2089.9ms/step | 9.6 samples/s +Step 25440/44000 | Loss: 2.9566 | LR: 1.95e-04 | 2083.7ms/step | 9.6 samples/s +Step 25450/44000 | Loss: 2.9092 | LR: 1.95e-04 | 2098.0ms/step | 9.5 samples/s +Step 25460/44000 | Loss: 2.9670 | LR: 1.94e-04 | 2027.7ms/step | 9.9 samples/s +Step 25470/44000 | Loss: 2.9292 | LR: 1.94e-04 | 2029.1ms/step | 9.9 samples/s +Step 25480/44000 | Loss: 2.9104 | LR: 1.94e-04 | 2043.1ms/step | 9.8 samples/s +Step 25490/44000 | Loss: 2.9477 | LR: 1.94e-04 | 2106.4ms/step | 9.5 samples/s +Step 25500/44000 | Loss: 2.9642 | LR: 1.94e-04 | 2087.8ms/step | 9.6 samples/s +Validation at step 25500: Loss = 2.9371, Perplexity = 18.86 +Step 25510/44000 | Loss: 2.9124 | LR: 1.94e-04 | 3902.5ms/step | 5.1 samples/s +Step 25520/44000 | Loss: 2.8856 | LR: 1.93e-04 | 2069.0ms/step | 9.7 samples/s +Step 25530/44000 | Loss: 2.9709 | LR: 1.93e-04 | 2027.9ms/step | 9.9 samples/s +Step 25540/44000 | Loss: 2.9287 | LR: 1.93e-04 | 2005.6ms/step | 10.0 samples/s +Step 25550/44000 | Loss: 2.9350 | LR: 1.93e-04 | 2099.5ms/step | 9.5 samples/s +Step 25560/44000 | Loss: 2.8900 | LR: 1.93e-04 | 2094.6ms/step | 9.5 samples/s +Step 25570/44000 | Loss: 2.8936 | LR: 1.92e-04 | 2094.7ms/step | 9.5 samples/s +Step 25580/44000 | Loss: 2.9131 | LR: 1.92e-04 | 2058.2ms/step | 9.7 samples/s +Step 25590/44000 | Loss: 2.9379 | LR: 1.92e-04 | 2027.0ms/step | 9.9 samples/s +Step 25600/44000 | Loss: 2.9266 | LR: 1.92e-04 | 2027.5ms/step | 9.9 samples/s +Step 25610/44000 | Loss: 2.9072 | LR: 1.92e-04 | 2074.0ms/step | 9.6 samples/s +Step 25620/44000 | Loss: 2.9151 | LR: 1.92e-04 | 2104.7ms/step | 9.5 samples/s +Step 25630/44000 | Loss: 2.9380 | LR: 1.91e-04 | 2083.5ms/step | 9.6 samples/s +Step 25640/44000 | Loss: 2.9212 | LR: 1.91e-04 | 2089.1ms/step | 9.6 samples/s +Step 25650/44000 | Loss: 2.8793 | LR: 1.91e-04 | 2085.0ms/step | 9.6 samples/s +Step 25660/44000 | Loss: 2.8683 | LR: 1.91e-04 | 2024.0ms/step | 9.9 samples/s +Step 25670/44000 | Loss: 2.9354 | LR: 1.91e-04 | 2024.1ms/step | 9.9 samples/s +Step 25680/44000 | Loss: 2.9535 | LR: 1.91e-04 | 2089.7ms/step | 9.6 samples/s +Step 25690/44000 | Loss: 2.9097 | LR: 1.90e-04 | 2125.7ms/step | 9.4 samples/s +Step 25700/44000 | Loss: 2.8954 | LR: 1.90e-04 | 2099.4ms/step | 9.5 samples/s +Step 25710/44000 | Loss: 2.8803 | LR: 1.90e-04 | 2079.8ms/step | 9.6 samples/s +Step 25720/44000 | Loss: 2.9179 | LR: 1.90e-04 | 2058.3ms/step | 9.7 samples/s +Step 25730/44000 | Loss: 2.9144 | LR: 1.90e-04 | 2022.2ms/step | 9.9 samples/s +Step 25740/44000 | Loss: 2.8690 | LR: 1.89e-04 | 2032.1ms/step | 9.8 samples/s +Step 25750/44000 | Loss: 2.9290 | LR: 1.89e-04 | 2084.1ms/step | 9.6 samples/s +Validation at step 25750: Loss = 2.9351, Perplexity = 18.82 +Step 25760/44000 | Loss: 2.9208 | LR: 1.89e-04 | 3918.8ms/step | 5.1 samples/s +Step 25770/44000 | Loss: 2.8864 | LR: 1.89e-04 | 2109.3ms/step | 9.5 samples/s +Step 25780/44000 | Loss: 2.9147 | LR: 1.89e-04 | 2063.8ms/step | 9.7 samples/s +Step 25790/44000 | Loss: 2.9904 | LR: 1.89e-04 | 2016.9ms/step | 9.9 samples/s +Step 25800/44000 | Loss: 2.9023 | LR: 1.88e-04 | 2041.0ms/step | 9.8 samples/s +Step 25810/44000 | Loss: 2.9027 | LR: 1.88e-04 | 2115.4ms/step | 9.5 samples/s +Step 25820/44000 | Loss: 2.9377 | LR: 1.88e-04 | 2100.3ms/step | 9.5 samples/s +Step 25830/44000 | Loss: 2.9489 | LR: 1.88e-04 | 2099.5ms/step | 9.5 samples/s +Step 25840/44000 | Loss: 2.9500 | LR: 1.88e-04 | 2099.2ms/step | 9.5 samples/s +Step 25850/44000 | Loss: 2.8980 | LR: 1.88e-04 | 2078.4ms/step | 9.6 samples/s +Step 25860/44000 | Loss: 2.9337 | LR: 1.87e-04 | 2252.3ms/step | 8.9 samples/s +Step 25870/44000 | Loss: 2.9469 | LR: 1.87e-04 | 2291.4ms/step | 8.7 samples/s +Step 25880/44000 | Loss: 2.9302 | LR: 1.87e-04 | 2057.3ms/step | 9.7 samples/s +Step 25890/44000 | Loss: 2.9192 | LR: 1.87e-04 | 2017.5ms/step | 9.9 samples/s +Step 25900/44000 | Loss: 2.8999 | LR: 1.87e-04 | 1994.6ms/step | 10.0 samples/s +Step 25910/44000 | Loss: 2.9309 | LR: 1.87e-04 | 2020.8ms/step | 9.9 samples/s +Step 25920/44000 | Loss: 2.9095 | LR: 1.86e-04 | 2037.9ms/step | 9.8 samples/s +Step 25930/44000 | Loss: 2.8911 | LR: 1.86e-04 | 2069.0ms/step | 9.7 samples/s +Step 25940/44000 | Loss: 2.9286 | LR: 1.86e-04 | 2062.6ms/step | 9.7 samples/s +Step 25950/44000 | Loss: 2.9289 | LR: 1.86e-04 | 2120.3ms/step | 9.4 samples/s +Step 25960/44000 | Loss: 2.9262 | LR: 1.86e-04 | 2051.2ms/step | 9.8 samples/s +Step 25970/44000 | Loss: 2.9149 | LR: 1.85e-04 | 2034.0ms/step | 9.8 samples/s +Step 25980/44000 | Loss: 2.8907 | LR: 1.85e-04 | 2032.5ms/step | 9.8 samples/s +Step 25990/44000 | Loss: 2.9062 | LR: 1.85e-04 | 2020.8ms/step | 9.9 samples/s +Step 26000/44000 | Loss: 2.9299 | LR: 1.85e-04 | 2053.4ms/step | 9.7 samples/s +Validation at step 26000: Loss = 2.9342, Perplexity = 18.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-26000 +Step 26010/44000 | Loss: 2.9236 | LR: 1.85e-04 | 4402.4ms/step | 4.5 samples/s +Step 26020/44000 | Loss: 2.8805 | LR: 1.85e-04 | 2073.8ms/step | 9.6 samples/s +Step 26030/44000 | Loss: 2.8765 | LR: 1.84e-04 | 2038.1ms/step | 9.8 samples/s +Step 26040/44000 | Loss: 2.9274 | LR: 1.84e-04 | 2020.9ms/step | 9.9 samples/s +Step 26050/44000 | Loss: 2.9230 | LR: 1.84e-04 | 2060.7ms/step | 9.7 samples/s +Step 26060/44000 | Loss: 2.9409 | LR: 1.84e-04 | 2091.7ms/step | 9.6 samples/s +Step 26070/44000 | Loss: 2.9439 | LR: 1.84e-04 | 2099.6ms/step | 9.5 samples/s +Step 26080/44000 | Loss: 2.9286 | LR: 1.84e-04 | 2099.6ms/step | 9.5 samples/s +Step 26090/44000 | Loss: 2.8953 | LR: 1.83e-04 | 2068.5ms/step | 9.7 samples/s +Step 26100/44000 | Loss: 2.9151 | LR: 1.83e-04 | 2025.3ms/step | 9.9 samples/s +Step 26110/44000 | Loss: 2.8968 | LR: 1.83e-04 | 2050.9ms/step | 9.8 samples/s +Step 26120/44000 | Loss: 2.9602 | LR: 1.83e-04 | 2070.4ms/step | 9.7 samples/s +Step 26130/44000 | Loss: 2.9170 | LR: 1.83e-04 | 2130.2ms/step | 9.4 samples/s +Step 26140/44000 | Loss: 2.9072 | LR: 1.82e-04 | 2166.8ms/step | 9.2 samples/s +Step 26150/44000 | Loss: 2.9497 | LR: 1.82e-04 | 2124.4ms/step | 9.4 samples/s +Step 26160/44000 | Loss: 2.9475 | LR: 1.82e-04 | 2167.8ms/step | 9.2 samples/s +Step 26170/44000 | Loss: 2.9030 | LR: 1.82e-04 | 2198.6ms/step | 9.1 samples/s +Step 26180/44000 | Loss: 2.8665 | LR: 1.82e-04 | 2036.9ms/step | 9.8 samples/s +Step 26190/44000 | Loss: 2.9258 | LR: 1.82e-04 | 2033.1ms/step | 9.8 samples/s +Step 26200/44000 | Loss: 2.8651 | LR: 1.81e-04 | 2084.5ms/step | 9.6 samples/s +Step 26210/44000 | Loss: 2.9152 | LR: 1.81e-04 | 2085.6ms/step | 9.6 samples/s +Step 26220/44000 | Loss: 2.8848 | LR: 1.81e-04 | 2061.6ms/step | 9.7 samples/s +Step 26230/44000 | Loss: 2.9437 | LR: 1.81e-04 | 2053.9ms/step | 9.7 samples/s +Step 26240/44000 | Loss: 2.9492 | LR: 1.81e-04 | 2036.1ms/step | 9.8 samples/s +Step 26250/44000 | Loss: 2.9407 | LR: 1.81e-04 | 2043.1ms/step | 9.8 samples/s +Validation at step 26250: Loss = 2.9324, Perplexity = 18.77 +Step 26260/44000 | Loss: 2.9153 | LR: 1.80e-04 | 3795.8ms/step | 5.3 samples/s +Step 26270/44000 | Loss: 2.9134 | LR: 1.80e-04 | 2057.7ms/step | 9.7 samples/s +Step 26280/44000 | Loss: 2.9356 | LR: 1.80e-04 | 2001.6ms/step | 10.0 samples/s +Step 26290/44000 | Loss: 2.9477 | LR: 1.80e-04 | 2026.4ms/step | 9.9 samples/s +Step 26300/44000 | Loss: 2.8889 | LR: 1.80e-04 | 2078.3ms/step | 9.6 samples/s +Step 26310/44000 | Loss: 2.8977 | LR: 1.80e-04 | 2105.6ms/step | 9.5 samples/s +Step 26320/44000 | Loss: 2.8919 | LR: 1.79e-04 | 2084.2ms/step | 9.6 samples/s +Step 26330/44000 | Loss: 2.8881 | LR: 1.79e-04 | 2083.4ms/step | 9.6 samples/s +Step 26340/44000 | Loss: 2.8662 | LR: 1.79e-04 | 2026.3ms/step | 9.9 samples/s +Step 26350/44000 | Loss: 2.9147 | LR: 1.79e-04 | 2021.8ms/step | 9.9 samples/s +Step 26360/44000 | Loss: 2.9288 | LR: 1.79e-04 | 2037.8ms/step | 9.8 samples/s +Step 26370/44000 | Loss: 2.9194 | LR: 1.78e-04 | 2089.5ms/step | 9.6 samples/s +Step 26380/44000 | Loss: 2.9106 | LR: 1.78e-04 | 2114.9ms/step | 9.5 samples/s +Step 26390/44000 | Loss: 2.9338 | LR: 1.78e-04 | 2094.6ms/step | 9.5 samples/s +Step 26400/44000 | Loss: 2.9526 | LR: 1.78e-04 | 2072.8ms/step | 9.6 samples/s +Step 26410/44000 | Loss: 2.9387 | LR: 1.78e-04 | 2033.4ms/step | 9.8 samples/s +Step 26420/44000 | Loss: 2.9096 | LR: 1.78e-04 | 2037.8ms/step | 9.8 samples/s +Step 26430/44000 | Loss: 2.8955 | LR: 1.77e-04 | 2030.9ms/step | 9.8 samples/s +Step 26440/44000 | Loss: 2.9019 | LR: 1.77e-04 | 2144.5ms/step | 9.3 samples/s +Step 26450/44000 | Loss: 2.9480 | LR: 1.77e-04 | 2232.9ms/step | 9.0 samples/s +Step 26460/44000 | Loss: 2.9455 | LR: 1.77e-04 | 2106.9ms/step | 9.5 samples/s +Step 26470/44000 | Loss: 2.9081 | LR: 1.77e-04 | 2072.8ms/step | 9.6 samples/s +Step 26480/44000 | Loss: 2.9389 | LR: 1.77e-04 | 1982.1ms/step | 10.1 samples/s +Step 26490/44000 | Loss: 2.9292 | LR: 1.76e-04 | 2034.1ms/step | 9.8 samples/s +Step 26500/44000 | Loss: 2.9191 | LR: 1.76e-04 | 2125.5ms/step | 9.4 samples/s +Validation at step 26500: Loss = 2.9308, Perplexity = 18.74 +Step 26510/44000 | Loss: 2.9535 | LR: 1.76e-04 | 3798.1ms/step | 5.3 samples/s +Step 26520/44000 | Loss: 2.8656 | LR: 1.76e-04 | 2031.2ms/step | 9.8 samples/s +Step 26530/44000 | Loss: 2.8789 | LR: 1.76e-04 | 2032.6ms/step | 9.8 samples/s +Step 26540/44000 | Loss: 2.9083 | LR: 1.76e-04 | 2031.8ms/step | 9.8 samples/s +Step 26550/44000 | Loss: 2.8796 | LR: 1.75e-04 | 2079.0ms/step | 9.6 samples/s +Step 26560/44000 | Loss: 2.9114 | LR: 1.75e-04 | 2069.1ms/step | 9.7 samples/s +Step 26570/44000 | Loss: 2.9294 | LR: 1.75e-04 | 2069.4ms/step | 9.7 samples/s +Step 26580/44000 | Loss: 2.9147 | LR: 1.75e-04 | 2062.7ms/step | 9.7 samples/s +Step 26590/44000 | Loss: 2.9284 | LR: 1.75e-04 | 2021.2ms/step | 9.9 samples/s +Step 26600/44000 | Loss: 2.9077 | LR: 1.74e-04 | 2011.0ms/step | 9.9 samples/s +Step 26610/44000 | Loss: 2.9386 | LR: 1.74e-04 | 2025.6ms/step | 9.9 samples/s +Step 26620/44000 | Loss: 2.9300 | LR: 1.74e-04 | 2047.9ms/step | 9.8 samples/s +Step 26630/44000 | Loss: 2.8877 | LR: 1.74e-04 | 2053.0ms/step | 9.7 samples/s +Step 26640/44000 | Loss: 2.8884 | LR: 1.74e-04 | 2022.1ms/step | 9.9 samples/s +Step 26650/44000 | Loss: 2.9442 | LR: 1.74e-04 | 2026.9ms/step | 9.9 samples/s +Step 26660/44000 | Loss: 2.9173 | LR: 1.73e-04 | 2032.9ms/step | 9.8 samples/s +Step 26670/44000 | Loss: 2.8881 | LR: 1.73e-04 | 2026.7ms/step | 9.9 samples/s +Step 26680/44000 | Loss: 2.8923 | LR: 1.73e-04 | 2084.0ms/step | 9.6 samples/s +Step 26690/44000 | Loss: 2.9297 | LR: 1.73e-04 | 2059.0ms/step | 9.7 samples/s +Step 26700/44000 | Loss: 2.9334 | LR: 1.73e-04 | 2021.4ms/step | 9.9 samples/s +Step 26710/44000 | Loss: 2.8947 | LR: 1.73e-04 | 2032.3ms/step | 9.8 samples/s +Step 26720/44000 | Loss: 2.9071 | LR: 1.72e-04 | 2031.5ms/step | 9.8 samples/s +Step 26730/44000 | Loss: 2.8774 | LR: 1.72e-04 | 2022.1ms/step | 9.9 samples/s +Step 26740/44000 | Loss: 2.9398 | LR: 1.72e-04 | 2052.9ms/step | 9.7 samples/s +Step 26750/44000 | Loss: 2.9036 | LR: 1.72e-04 | 2073.3ms/step | 9.6 samples/s +Validation at step 26750: Loss = 2.9299, Perplexity = 18.73 +Step 26760/44000 | Loss: 2.9124 | LR: 1.72e-04 | 3762.6ms/step | 5.3 samples/s +Step 26770/44000 | Loss: 2.8973 | LR: 1.72e-04 | 1990.2ms/step | 10.0 samples/s +Step 26780/44000 | Loss: 2.9267 | LR: 1.71e-04 | 2032.4ms/step | 9.8 samples/s +Step 26790/44000 | Loss: 2.9296 | LR: 1.71e-04 | 2058.3ms/step | 9.7 samples/s +Step 26800/44000 | Loss: 2.9402 | LR: 1.71e-04 | 2047.9ms/step | 9.8 samples/s +Step 26810/44000 | Loss: 2.9408 | LR: 1.71e-04 | 2010.5ms/step | 9.9 samples/s +Step 26820/44000 | Loss: 2.8952 | LR: 1.71e-04 | 2038.0ms/step | 9.8 samples/s +Step 26830/44000 | Loss: 2.8825 | LR: 1.71e-04 | 2027.0ms/step | 9.9 samples/s +Step 26840/44000 | Loss: 2.8524 | LR: 1.70e-04 | 2048.2ms/step | 9.8 samples/s +Step 26850/44000 | Loss: 2.9287 | LR: 1.70e-04 | 2062.0ms/step | 9.7 samples/s +Step 26860/44000 | Loss: 2.9494 | LR: 1.70e-04 | 2079.0ms/step | 9.6 samples/s +Step 26870/44000 | Loss: 2.9247 | LR: 1.70e-04 | 2031.6ms/step | 9.8 samples/s +Step 26880/44000 | Loss: 2.9129 | LR: 1.70e-04 | 2036.5ms/step | 9.8 samples/s +Step 26890/44000 | Loss: 2.9278 | LR: 1.69e-04 | 2016.0ms/step | 9.9 samples/s +Step 26900/44000 | Loss: 2.9262 | LR: 1.69e-04 | 2046.9ms/step | 9.8 samples/s +Step 26910/44000 | Loss: 2.9333 | LR: 1.69e-04 | 2037.9ms/step | 9.8 samples/s +Step 26920/44000 | Loss: 2.9134 | LR: 1.69e-04 | 2061.5ms/step | 9.7 samples/s +Step 26930/44000 | Loss: 2.9368 | LR: 1.69e-04 | 2043.1ms/step | 9.8 samples/s +Step 26940/44000 | Loss: 2.9070 | LR: 1.69e-04 | 2041.7ms/step | 9.8 samples/s +Step 26950/44000 | Loss: 2.9245 | LR: 1.68e-04 | 2036.6ms/step | 9.8 samples/s +Step 26960/44000 | Loss: 2.9502 | LR: 1.68e-04 | 2056.5ms/step | 9.7 samples/s +Step 26970/44000 | Loss: 2.8810 | LR: 1.68e-04 | 2046.4ms/step | 9.8 samples/s +Step 26980/44000 | Loss: 2.9376 | LR: 1.68e-04 | 2062.5ms/step | 9.7 samples/s +Step 26990/44000 | Loss: 2.9177 | LR: 1.68e-04 | 2026.3ms/step | 9.9 samples/s +Step 27000/44000 | Loss: 2.9378 | LR: 1.68e-04 | 2026.5ms/step | 9.9 samples/s +Validation at step 27000: Loss = 2.9283, Perplexity = 18.70 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-27000 +Step 27010/44000 | Loss: 2.8658 | LR: 1.67e-04 | 4151.8ms/step | 4.8 samples/s +Step 27020/44000 | Loss: 2.8892 | LR: 1.67e-04 | 1990.8ms/step | 10.0 samples/s +Step 27030/44000 | Loss: 2.9066 | LR: 1.67e-04 | 2039.4ms/step | 9.8 samples/s +Step 27040/44000 | Loss: 2.9321 | LR: 1.67e-04 | 2031.4ms/step | 9.8 samples/s +Step 27050/44000 | Loss: 2.8697 | LR: 1.67e-04 | 2086.8ms/step | 9.6 samples/s +Step 27060/44000 | Loss: 2.8897 | LR: 1.67e-04 | 2041.3ms/step | 9.8 samples/s +Step 27070/44000 | Loss: 2.9485 | LR: 1.66e-04 | 2032.2ms/step | 9.8 samples/s +Step 27080/44000 | Loss: 2.9407 | LR: 1.66e-04 | 1985.0ms/step | 10.1 samples/s +Step 27090/44000 | Loss: 2.9297 | LR: 1.66e-04 | 2010.4ms/step | 9.9 samples/s +Step 27100/44000 | Loss: 2.9324 | LR: 1.66e-04 | 2020.9ms/step | 9.9 samples/s +Step 27110/44000 | Loss: 2.9292 | LR: 1.66e-04 | 2048.7ms/step | 9.8 samples/s +Step 27120/44000 | Loss: 2.9425 | LR: 1.66e-04 | 2016.1ms/step | 9.9 samples/s +Step 27130/44000 | Loss: 2.9065 | LR: 1.65e-04 | 2037.5ms/step | 9.8 samples/s +Step 27140/44000 | Loss: 2.9074 | LR: 1.65e-04 | 2074.2ms/step | 9.6 samples/s +Step 27150/44000 | Loss: 2.9178 | LR: 1.65e-04 | 2047.6ms/step | 9.8 samples/s +Step 27160/44000 | Loss: 2.9637 | LR: 1.65e-04 | 2038.0ms/step | 9.8 samples/s +Step 27170/44000 | Loss: 2.8809 | LR: 1.65e-04 | 2037.1ms/step | 9.8 samples/s +Step 27180/44000 | Loss: 2.9096 | LR: 1.65e-04 | 2067.6ms/step | 9.7 samples/s +Step 27190/44000 | Loss: 2.9359 | LR: 1.64e-04 | 2052.8ms/step | 9.7 samples/s +Step 27200/44000 | Loss: 2.8640 | LR: 1.64e-04 | 2041.9ms/step | 9.8 samples/s +Step 27210/44000 | Loss: 2.9048 | LR: 1.64e-04 | 2048.6ms/step | 9.8 samples/s +Step 27220/44000 | Loss: 2.9290 | LR: 1.64e-04 | 2042.1ms/step | 9.8 samples/s +Step 27230/44000 | Loss: 2.8712 | LR: 1.64e-04 | 2027.0ms/step | 9.9 samples/s +Step 27240/44000 | Loss: 2.8867 | LR: 1.63e-04 | 2099.5ms/step | 9.5 samples/s +Step 27250/44000 | Loss: 2.9474 | LR: 1.63e-04 | 2029.4ms/step | 9.9 samples/s +Validation at step 27250: Loss = 2.9270, Perplexity = 18.67 +Step 27260/44000 | Loss: 2.8989 | LR: 1.63e-04 | 3758.7ms/step | 5.3 samples/s +Step 27270/44000 | Loss: 2.9534 | LR: 1.63e-04 | 2005.4ms/step | 10.0 samples/s +Step 27280/44000 | Loss: 2.9016 | LR: 1.63e-04 | 2026.6ms/step | 9.9 samples/s +Step 27290/44000 | Loss: 2.8907 | LR: 1.63e-04 | 2038.6ms/step | 9.8 samples/s +Step 27300/44000 | Loss: 2.9446 | LR: 1.62e-04 | 2048.9ms/step | 9.8 samples/s +Step 27310/44000 | Loss: 2.9415 | LR: 1.62e-04 | 2039.5ms/step | 9.8 samples/s +Step 27320/44000 | Loss: 2.9683 | LR: 1.62e-04 | 2052.3ms/step | 9.7 samples/s +Step 27330/44000 | Loss: 2.9286 | LR: 1.62e-04 | 2031.9ms/step | 9.8 samples/s +Step 27340/44000 | Loss: 2.8960 | LR: 1.62e-04 | 2032.6ms/step | 9.8 samples/s +Step 27350/44000 | Loss: 2.8850 | LR: 1.62e-04 | 2025.6ms/step | 9.9 samples/s +Step 27360/44000 | Loss: 2.9121 | LR: 1.61e-04 | 2015.8ms/step | 9.9 samples/s +Step 27370/44000 | Loss: 2.8983 | LR: 1.61e-04 | 2046.7ms/step | 9.8 samples/s +Step 27380/44000 | Loss: 2.8849 | LR: 1.61e-04 | 2010.3ms/step | 9.9 samples/s +Step 27390/44000 | Loss: 2.9428 | LR: 1.61e-04 | 2002.3ms/step | 10.0 samples/s +Step 27400/44000 | Loss: 2.9077 | LR: 1.61e-04 | 2023.0ms/step | 9.9 samples/s +Step 27410/44000 | Loss: 2.9283 | LR: 1.61e-04 | 2066.5ms/step | 9.7 samples/s +Step 27420/44000 | Loss: 2.9282 | LR: 1.60e-04 | 2054.0ms/step | 9.7 samples/s +Step 27430/44000 | Loss: 2.9089 | LR: 1.60e-04 | 2035.6ms/step | 9.8 samples/s +Step 27440/44000 | Loss: 2.9593 | LR: 1.60e-04 | 2042.4ms/step | 9.8 samples/s +Step 27450/44000 | Loss: 2.9173 | LR: 1.60e-04 | 2041.7ms/step | 9.8 samples/s +Step 27460/44000 | Loss: 2.9421 | LR: 1.60e-04 | 2078.1ms/step | 9.6 samples/s +Step 27470/44000 | Loss: 2.9478 | LR: 1.60e-04 | 2078.1ms/step | 9.6 samples/s +Step 27480/44000 | Loss: 2.9469 | LR: 1.59e-04 | 2131.1ms/step | 9.4 samples/s +Step 27490/44000 | Loss: 2.8648 | LR: 1.59e-04 | 2052.8ms/step | 9.7 samples/s +Step 27500/44000 | Loss: 2.8885 | LR: 1.59e-04 | 2075.6ms/step | 9.6 samples/s +Validation at step 27500: Loss = 2.9255, Perplexity = 18.64 +Step 27510/44000 | Loss: 2.9278 | LR: 1.59e-04 | 3855.4ms/step | 5.2 samples/s +Step 27520/44000 | Loss: 2.9421 | LR: 1.59e-04 | 2073.3ms/step | 9.6 samples/s +Step 27530/44000 | Loss: 2.9205 | LR: 1.59e-04 | 2047.4ms/step | 9.8 samples/s +Step 27540/44000 | Loss: 2.9321 | LR: 1.58e-04 | 2099.0ms/step | 9.5 samples/s +Step 27550/44000 | Loss: 2.9335 | LR: 1.58e-04 | 2056.9ms/step | 9.7 samples/s +Step 27560/44000 | Loss: 2.9091 | LR: 1.58e-04 | 2094.8ms/step | 9.5 samples/s +Step 27570/44000 | Loss: 2.9107 | LR: 1.58e-04 | 2090.5ms/step | 9.6 samples/s +Step 27580/44000 | Loss: 2.9523 | LR: 1.58e-04 | 2065.4ms/step | 9.7 samples/s +Step 27590/44000 | Loss: 2.9241 | LR: 1.58e-04 | 2094.4ms/step | 9.5 samples/s +Step 27600/44000 | Loss: 2.9110 | LR: 1.57e-04 | 2563.2ms/step | 7.8 samples/s +Step 27610/44000 | Loss: 2.9041 | LR: 1.57e-04 | 2683.1ms/step | 7.5 samples/s +Step 27620/44000 | Loss: 2.8936 | LR: 1.57e-04 | 2792.5ms/step | 7.2 samples/s +Step 27630/44000 | Loss: 2.8951 | LR: 1.57e-04 | 2720.0ms/step | 7.4 samples/s +Step 27640/44000 | Loss: 2.9195 | LR: 1.57e-04 | 2730.1ms/step | 7.3 samples/s +Step 27650/44000 | Loss: 2.8670 | LR: 1.57e-04 | 2766.3ms/step | 7.2 samples/s +Step 27660/44000 | Loss: 2.9371 | LR: 1.56e-04 | 2750.7ms/step | 7.3 samples/s +Step 27670/44000 | Loss: 2.9012 | LR: 1.56e-04 | 2696.1ms/step | 7.4 samples/s +Step 27680/44000 | Loss: 2.8995 | LR: 1.56e-04 | 2712.7ms/step | 7.4 samples/s +Step 27690/44000 | Loss: 2.9278 | LR: 1.56e-04 | 2598.2ms/step | 7.7 samples/s +Step 27700/44000 | Loss: 2.9073 | LR: 1.56e-04 | 2341.1ms/step | 8.5 samples/s +Step 27710/44000 | Loss: 2.9041 | LR: 1.56e-04 | 2885.1ms/step | 6.9 samples/s +Step 27720/44000 | Loss: 2.8713 | LR: 1.55e-04 | 2659.2ms/step | 7.5 samples/s +Step 27730/44000 | Loss: 2.8740 | LR: 1.55e-04 | 2757.1ms/step | 7.3 samples/s +Step 27740/44000 | Loss: 2.9224 | LR: 1.55e-04 | 2813.6ms/step | 7.1 samples/s +Step 27750/44000 | Loss: 2.9251 | LR: 1.55e-04 | 2225.8ms/step | 9.0 samples/s +Validation at step 27750: Loss = 2.9243, Perplexity = 18.62 +Step 27760/44000 | Loss: 2.9012 | LR: 1.55e-04 | 3808.8ms/step | 5.3 samples/s +Step 27770/44000 | Loss: 2.9170 | LR: 1.55e-04 | 2136.9ms/step | 9.4 samples/s +Step 27780/44000 | Loss: 2.9034 | LR: 1.54e-04 | 2090.1ms/step | 9.6 samples/s +Step 27790/44000 | Loss: 2.8960 | LR: 1.54e-04 | 2358.3ms/step | 8.5 samples/s +Step 27800/44000 | Loss: 2.8798 | LR: 1.54e-04 | 2191.3ms/step | 9.1 samples/s +Step 27810/44000 | Loss: 2.9267 | LR: 1.54e-04 | 2084.6ms/step | 9.6 samples/s +Step 27820/44000 | Loss: 2.9306 | LR: 1.54e-04 | 2036.8ms/step | 9.8 samples/s +Step 27830/44000 | Loss: 2.8954 | LR: 1.54e-04 | 2037.4ms/step | 9.8 samples/s +Step 27840/44000 | Loss: 2.9232 | LR: 1.53e-04 | 2043.0ms/step | 9.8 samples/s +Step 27850/44000 | Loss: 2.9292 | LR: 1.53e-04 | 2052.5ms/step | 9.7 samples/s +Step 27860/44000 | Loss: 2.9033 | LR: 1.53e-04 | 2095.6ms/step | 9.5 samples/s +Step 27870/44000 | Loss: 2.8677 | LR: 1.53e-04 | 2079.6ms/step | 9.6 samples/s +Step 27880/44000 | Loss: 2.9146 | LR: 1.53e-04 | 2108.7ms/step | 9.5 samples/s +Step 27890/44000 | Loss: 2.9165 | LR: 1.53e-04 | 2060.2ms/step | 9.7 samples/s +Step 27900/44000 | Loss: 2.9133 | LR: 1.52e-04 | 2068.7ms/step | 9.7 samples/s +Step 27910/44000 | Loss: 2.9306 | LR: 1.52e-04 | 2042.4ms/step | 9.8 samples/s +Step 27920/44000 | Loss: 2.9448 | LR: 1.52e-04 | 2059.4ms/step | 9.7 samples/s +Step 27930/44000 | Loss: 2.9316 | LR: 1.52e-04 | 2110.9ms/step | 9.5 samples/s +Step 27940/44000 | Loss: 2.9014 | LR: 1.52e-04 | 2121.0ms/step | 9.4 samples/s +Step 27950/44000 | Loss: 2.8976 | LR: 1.52e-04 | 2115.8ms/step | 9.5 samples/s +Step 27960/44000 | Loss: 2.9183 | LR: 1.51e-04 | 2054.0ms/step | 9.7 samples/s +Step 27970/44000 | Loss: 2.9071 | LR: 1.51e-04 | 2048.1ms/step | 9.8 samples/s +Step 27980/44000 | Loss: 2.9370 | LR: 1.51e-04 | 2053.8ms/step | 9.7 samples/s +Step 27990/44000 | Loss: 2.9276 | LR: 1.51e-04 | 2272.7ms/step | 8.8 samples/s +Step 28000/44000 | Loss: 2.9300 | LR: 1.51e-04 | 2631.3ms/step | 7.6 samples/s +Validation at step 28000: Loss = 2.9231, Perplexity = 18.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-28000 +Step 28010/44000 | Loss: 2.9234 | LR: 1.51e-04 | 5723.6ms/step | 3.5 samples/s +Step 28020/44000 | Loss: 2.8927 | LR: 1.50e-04 | 2773.3ms/step | 7.2 samples/s +Step 28030/44000 | Loss: 2.9310 | LR: 1.50e-04 | 2704.3ms/step | 7.4 samples/s +Step 28040/44000 | Loss: 2.8989 | LR: 1.50e-04 | 2676.4ms/step | 7.5 samples/s +Step 28050/44000 | Loss: 2.8422 | LR: 1.50e-04 | 2716.8ms/step | 7.4 samples/s +Step 28060/44000 | Loss: 2.9052 | LR: 1.50e-04 | 2812.3ms/step | 7.1 samples/s +Step 28070/44000 | Loss: 2.9418 | LR: 1.50e-04 | 2773.1ms/step | 7.2 samples/s +Step 28080/44000 | Loss: 2.9379 | LR: 1.49e-04 | 2792.6ms/step | 7.2 samples/s +Step 28090/44000 | Loss: 2.9223 | LR: 1.49e-04 | 2726.7ms/step | 7.3 samples/s +Step 28100/44000 | Loss: 2.9370 | LR: 1.49e-04 | 2708.1ms/step | 7.4 samples/s +Step 28110/44000 | Loss: 2.8628 | LR: 1.49e-04 | 2735.2ms/step | 7.3 samples/s +Step 28120/44000 | Loss: 2.9100 | LR: 1.49e-04 | 2641.3ms/step | 7.6 samples/s +Step 28130/44000 | Loss: 2.9413 | LR: 1.49e-04 | 2695.0ms/step | 7.4 samples/s +Step 28140/44000 | Loss: 2.8872 | LR: 1.48e-04 | 2696.9ms/step | 7.4 samples/s +Step 28150/44000 | Loss: 2.9137 | LR: 1.48e-04 | 2552.6ms/step | 7.8 samples/s +Step 28160/44000 | Loss: 2.9076 | LR: 1.48e-04 | 2793.8ms/step | 7.2 samples/s +Step 28170/44000 | Loss: 2.9021 | LR: 1.48e-04 | 2827.0ms/step | 7.1 samples/s +Step 28180/44000 | Loss: 2.8546 | LR: 1.48e-04 | 2334.0ms/step | 8.6 samples/s +Step 28190/44000 | Loss: 2.9085 | LR: 1.48e-04 | 2119.9ms/step | 9.4 samples/s +Step 28200/44000 | Loss: 2.8679 | LR: 1.47e-04 | 2118.2ms/step | 9.4 samples/s +Step 28210/44000 | Loss: 2.9279 | LR: 1.47e-04 | 2007.3ms/step | 10.0 samples/s +Step 28220/44000 | Loss: 2.9043 | LR: 1.47e-04 | 2020.7ms/step | 9.9 samples/s +Step 28230/44000 | Loss: 2.8927 | LR: 1.47e-04 | 2089.7ms/step | 9.6 samples/s +Step 28240/44000 | Loss: 2.9231 | LR: 1.47e-04 | 2068.2ms/step | 9.7 samples/s +Step 28250/44000 | Loss: 2.9119 | LR: 1.47e-04 | 2057.7ms/step | 9.7 samples/s +Validation at step 28250: Loss = 2.9223, Perplexity = 18.58 +Step 28260/44000 | Loss: 2.9218 | LR: 1.46e-04 | 3771.6ms/step | 5.3 samples/s +Step 28270/44000 | Loss: 2.9072 | LR: 1.46e-04 | 2010.8ms/step | 9.9 samples/s +Step 28280/44000 | Loss: 2.8978 | LR: 1.46e-04 | 2090.0ms/step | 9.6 samples/s +Step 28290/44000 | Loss: 2.8906 | LR: 1.46e-04 | 2026.6ms/step | 9.9 samples/s +Step 28300/44000 | Loss: 2.9342 | LR: 1.46e-04 | 2042.5ms/step | 9.8 samples/s +Step 28310/44000 | Loss: 2.9012 | LR: 1.46e-04 | 2035.9ms/step | 9.8 samples/s +Step 28320/44000 | Loss: 2.8991 | LR: 1.45e-04 | 2013.0ms/step | 9.9 samples/s +Step 28330/44000 | Loss: 2.9176 | LR: 1.45e-04 | 2065.8ms/step | 9.7 samples/s +Step 28340/44000 | Loss: 2.8992 | LR: 1.45e-04 | 2099.8ms/step | 9.5 samples/s +Step 28350/44000 | Loss: 2.9683 | LR: 1.45e-04 | 2087.3ms/step | 9.6 samples/s +Step 28360/44000 | Loss: 2.9432 | LR: 1.45e-04 | 2125.2ms/step | 9.4 samples/s +Step 28370/44000 | Loss: 2.9078 | LR: 1.45e-04 | 2073.9ms/step | 9.6 samples/s +Step 28380/44000 | Loss: 2.9313 | LR: 1.44e-04 | 2021.0ms/step | 9.9 samples/s +Step 28390/44000 | Loss: 2.9249 | LR: 1.44e-04 | 2057.7ms/step | 9.7 samples/s +Step 28400/44000 | Loss: 2.9248 | LR: 1.44e-04 | 2046.8ms/step | 9.8 samples/s +Step 28410/44000 | Loss: 2.9338 | LR: 1.44e-04 | 2120.3ms/step | 9.4 samples/s +Step 28420/44000 | Loss: 2.8839 | LR: 1.44e-04 | 2125.2ms/step | 9.4 samples/s +Step 28430/44000 | Loss: 2.8937 | LR: 1.44e-04 | 2115.1ms/step | 9.5 samples/s +Step 28440/44000 | Loss: 2.9139 | LR: 1.43e-04 | 2089.0ms/step | 9.6 samples/s +Step 28450/44000 | Loss: 2.8684 | LR: 1.43e-04 | 2031.9ms/step | 9.8 samples/s +Step 28460/44000 | Loss: 2.9641 | LR: 1.43e-04 | 2046.5ms/step | 9.8 samples/s +Step 28470/44000 | Loss: 2.9089 | LR: 1.43e-04 | 2052.1ms/step | 9.7 samples/s +Step 28480/44000 | Loss: 2.9407 | LR: 1.43e-04 | 2068.2ms/step | 9.7 samples/s +Step 28490/44000 | Loss: 2.9401 | LR: 1.43e-04 | 2104.4ms/step | 9.5 samples/s +Step 28500/44000 | Loss: 2.9525 | LR: 1.42e-04 | 2057.3ms/step | 9.7 samples/s +Validation at step 28500: Loss = 2.9210, Perplexity = 18.56 +Step 28510/44000 | Loss: 2.9151 | LR: 1.42e-04 | 3776.4ms/step | 5.3 samples/s +Step 28520/44000 | Loss: 2.9309 | LR: 1.42e-04 | 2099.4ms/step | 9.5 samples/s +Step 28530/44000 | Loss: 2.9221 | LR: 1.42e-04 | 2130.4ms/step | 9.4 samples/s +Step 28540/44000 | Loss: 2.9087 | LR: 1.42e-04 | 2109.4ms/step | 9.5 samples/s +Step 28550/44000 | Loss: 2.9315 | LR: 1.42e-04 | 2037.8ms/step | 9.8 samples/s +Step 28560/44000 | Loss: 2.9307 | LR: 1.41e-04 | 2020.3ms/step | 9.9 samples/s +Step 28570/44000 | Loss: 2.8739 | LR: 1.41e-04 | 2062.6ms/step | 9.7 samples/s +Step 28580/44000 | Loss: 2.8809 | LR: 1.41e-04 | 2099.5ms/step | 9.5 samples/s +Step 28590/44000 | Loss: 2.9258 | LR: 1.41e-04 | 2135.4ms/step | 9.4 samples/s +Step 28600/44000 | Loss: 2.9020 | LR: 1.41e-04 | 2036.8ms/step | 9.8 samples/s +Step 28610/44000 | Loss: 2.8772 | LR: 1.41e-04 | 2026.3ms/step | 9.9 samples/s +Step 28620/44000 | Loss: 2.9051 | LR: 1.40e-04 | 2036.6ms/step | 9.8 samples/s +Step 28630/44000 | Loss: 2.9076 | LR: 1.40e-04 | 2016.0ms/step | 9.9 samples/s +Step 28640/44000 | Loss: 2.8951 | LR: 1.40e-04 | 2089.4ms/step | 9.6 samples/s +Step 28650/44000 | Loss: 2.8559 | LR: 1.40e-04 | 2105.5ms/step | 9.5 samples/s +Step 28660/44000 | Loss: 2.9346 | LR: 1.40e-04 | 2105.3ms/step | 9.5 samples/s +Step 28670/44000 | Loss: 2.9392 | LR: 1.40e-04 | 2083.9ms/step | 9.6 samples/s +Step 28680/44000 | Loss: 2.9160 | LR: 1.39e-04 | 2006.0ms/step | 10.0 samples/s +Step 28690/44000 | Loss: 2.9210 | LR: 1.39e-04 | 2026.4ms/step | 9.9 samples/s +Step 28700/44000 | Loss: 2.8989 | LR: 1.39e-04 | 2103.7ms/step | 9.5 samples/s +Step 28710/44000 | Loss: 2.9362 | LR: 1.39e-04 | 2105.1ms/step | 9.5 samples/s +Step 28720/44000 | Loss: 2.9097 | LR: 1.39e-04 | 2072.7ms/step | 9.6 samples/s +Step 28730/44000 | Loss: 2.9339 | LR: 1.39e-04 | 2117.8ms/step | 9.4 samples/s +Step 28740/44000 | Loss: 2.9453 | LR: 1.38e-04 | 2026.9ms/step | 9.9 samples/s +Step 28750/44000 | Loss: 2.8949 | LR: 1.38e-04 | 2031.4ms/step | 9.8 samples/s +Validation at step 28750: Loss = 2.9198, Perplexity = 18.54 +Step 28760/44000 | Loss: 2.9180 | LR: 1.38e-04 | 3892.4ms/step | 5.1 samples/s +Step 28770/44000 | Loss: 2.9091 | LR: 1.38e-04 | 2091.2ms/step | 9.6 samples/s +Step 28780/44000 | Loss: 2.9142 | LR: 1.38e-04 | 2092.2ms/step | 9.6 samples/s +Step 28790/44000 | Loss: 2.9351 | LR: 1.38e-04 | 2030.1ms/step | 9.9 samples/s +Step 28800/44000 | Loss: 2.9173 | LR: 1.38e-04 | 2210.8ms/step | 9.0 samples/s +Step 28810/44000 | Loss: 2.8980 | LR: 1.37e-04 | 2016.7ms/step | 9.9 samples/s +Step 28820/44000 | Loss: 2.9476 | LR: 1.37e-04 | 2115.3ms/step | 9.5 samples/s +Step 28830/44000 | Loss: 2.8855 | LR: 1.37e-04 | 2069.4ms/step | 9.7 samples/s +Step 28840/44000 | Loss: 2.9144 | LR: 1.37e-04 | 2110.2ms/step | 9.5 samples/s +Step 28850/44000 | Loss: 2.8731 | LR: 1.37e-04 | 2089.6ms/step | 9.6 samples/s +Step 28860/44000 | Loss: 2.8891 | LR: 1.37e-04 | 2027.1ms/step | 9.9 samples/s +Step 28870/44000 | Loss: 2.9324 | LR: 1.36e-04 | 2037.3ms/step | 9.8 samples/s +Step 28880/44000 | Loss: 2.9299 | LR: 1.36e-04 | 2074.5ms/step | 9.6 samples/s +Step 28890/44000 | Loss: 2.8859 | LR: 1.36e-04 | 2277.2ms/step | 8.8 samples/s +Step 28900/44000 | Loss: 2.8868 | LR: 1.36e-04 | 2767.9ms/step | 7.2 samples/s +Step 28910/44000 | Loss: 2.9501 | LR: 1.36e-04 | 2805.3ms/step | 7.1 samples/s +Step 28920/44000 | Loss: 2.8980 | LR: 1.36e-04 | 2473.3ms/step | 8.1 samples/s +Step 28930/44000 | Loss: 2.8687 | LR: 1.35e-04 | 2715.0ms/step | 7.4 samples/s +Step 28940/44000 | Loss: 2.9121 | LR: 1.35e-04 | 2880.1ms/step | 6.9 samples/s +Step 28950/44000 | Loss: 2.9340 | LR: 1.35e-04 | 2858.1ms/step | 7.0 samples/s +Step 28960/44000 | Loss: 2.8886 | LR: 1.35e-04 | 3147.2ms/step | 6.4 samples/s +Step 28970/44000 | Loss: 2.9125 | LR: 1.35e-04 | 2900.7ms/step | 6.9 samples/s +Step 28980/44000 | Loss: 2.9223 | LR: 1.35e-04 | 3098.2ms/step | 6.5 samples/s +Step 28990/44000 | Loss: 2.9502 | LR: 1.34e-04 | 2948.8ms/step | 6.8 samples/s +Step 29000/44000 | Loss: 2.8688 | LR: 1.34e-04 | 2984.5ms/step | 6.7 samples/s +Validation at step 29000: Loss = 2.9185, Perplexity = 18.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-29000 +Step 29010/44000 | Loss: 2.8856 | LR: 1.34e-04 | 6061.0ms/step | 3.3 samples/s +Step 29020/44000 | Loss: 2.9032 | LR: 1.34e-04 | 2904.2ms/step | 6.9 samples/s +Step 29030/44000 | Loss: 2.9200 | LR: 1.34e-04 | 3061.2ms/step | 6.5 samples/s +Step 29040/44000 | Loss: 2.9448 | LR: 1.34e-04 | 2975.9ms/step | 6.7 samples/s +Step 29050/44000 | Loss: 2.9087 | LR: 1.33e-04 | 2925.1ms/step | 6.8 samples/s +Step 29060/44000 | Loss: 2.9112 | LR: 1.33e-04 | 2973.2ms/step | 6.7 samples/s +Step 29070/44000 | Loss: 2.8749 | LR: 1.33e-04 | 2437.6ms/step | 8.2 samples/s +Step 29080/44000 | Loss: 2.9152 | LR: 1.33e-04 | 2239.1ms/step | 8.9 samples/s +Step 29090/44000 | Loss: 2.8939 | LR: 1.33e-04 | 2186.0ms/step | 9.1 samples/s +Step 29100/44000 | Loss: 2.8856 | LR: 1.33e-04 | 2148.0ms/step | 9.3 samples/s +Step 29110/44000 | Loss: 2.9128 | LR: 1.33e-04 | 2105.8ms/step | 9.5 samples/s +Step 29120/44000 | Loss: 2.9234 | LR: 1.32e-04 | 2227.1ms/step | 9.0 samples/s +Step 29130/44000 | Loss: 2.8820 | LR: 1.32e-04 | 2224.3ms/step | 9.0 samples/s +Step 29140/44000 | Loss: 2.9040 | LR: 1.32e-04 | 2194.8ms/step | 9.1 samples/s +Step 29150/44000 | Loss: 2.9129 | LR: 1.32e-04 | 2246.4ms/step | 8.9 samples/s +Step 29160/44000 | Loss: 2.9021 | LR: 1.32e-04 | 2224.4ms/step | 9.0 samples/s +Step 29170/44000 | Loss: 2.9046 | LR: 1.32e-04 | 2143.4ms/step | 9.3 samples/s +Step 29180/44000 | Loss: 2.8958 | LR: 1.31e-04 | 2163.0ms/step | 9.2 samples/s +Step 29190/44000 | Loss: 2.8830 | LR: 1.31e-04 | 2193.9ms/step | 9.1 samples/s +Step 29200/44000 | Loss: 2.8671 | LR: 1.31e-04 | 2221.7ms/step | 9.0 samples/s +Step 29210/44000 | Loss: 2.8698 | LR: 1.31e-04 | 2244.5ms/step | 8.9 samples/s +Step 29220/44000 | Loss: 2.8632 | LR: 1.31e-04 | 2138.6ms/step | 9.4 samples/s +Step 29230/44000 | Loss: 2.9215 | LR: 1.31e-04 | 2189.5ms/step | 9.1 samples/s +Step 29240/44000 | Loss: 2.8521 | LR: 1.30e-04 | 2220.2ms/step | 9.0 samples/s +Step 29250/44000 | Loss: 2.8996 | LR: 1.30e-04 | 2205.0ms/step | 9.1 samples/s +Validation at step 29250: Loss = 2.9176, Perplexity = 18.50 +Step 29260/44000 | Loss: 2.8971 | LR: 1.30e-04 | 4140.4ms/step | 4.8 samples/s +Step 29270/44000 | Loss: 2.8679 | LR: 1.30e-04 | 2623.3ms/step | 7.6 samples/s +Step 29280/44000 | Loss: 2.9079 | LR: 1.30e-04 | 2426.2ms/step | 8.2 samples/s +Step 29290/44000 | Loss: 2.8863 | LR: 1.30e-04 | 2236.4ms/step | 8.9 samples/s +Step 29300/44000 | Loss: 2.8694 | LR: 1.29e-04 | 2224.1ms/step | 9.0 samples/s +Step 29310/44000 | Loss: 2.8590 | LR: 1.29e-04 | 2250.2ms/step | 8.9 samples/s +Step 29320/44000 | Loss: 2.8470 | LR: 1.29e-04 | 2250.5ms/step | 8.9 samples/s +Step 29330/44000 | Loss: 2.8902 | LR: 1.29e-04 | 2805.6ms/step | 7.1 samples/s +Step 29340/44000 | Loss: 2.9663 | LR: 1.29e-04 | 2852.8ms/step | 7.0 samples/s +Step 29350/44000 | Loss: 2.8720 | LR: 1.29e-04 | 2628.4ms/step | 7.6 samples/s +Step 29360/44000 | Loss: 2.9212 | LR: 1.29e-04 | 2731.7ms/step | 7.3 samples/s +Step 29370/44000 | Loss: 2.9377 | LR: 1.28e-04 | 2673.7ms/step | 7.5 samples/s +Step 29380/44000 | Loss: 2.9237 | LR: 1.28e-04 | 2646.9ms/step | 7.6 samples/s +Step 29390/44000 | Loss: 2.8939 | LR: 1.28e-04 | 2689.5ms/step | 7.4 samples/s +Step 29400/44000 | Loss: 2.8728 | LR: 1.28e-04 | 2713.8ms/step | 7.4 samples/s +Step 29410/44000 | Loss: 2.9403 | LR: 1.28e-04 | 2744.0ms/step | 7.3 samples/s +Step 29420/44000 | Loss: 2.9002 | LR: 1.28e-04 | 2596.5ms/step | 7.7 samples/s +Step 29430/44000 | Loss: 2.8649 | LR: 1.27e-04 | 2682.3ms/step | 7.5 samples/s +Step 29440/44000 | Loss: 2.8822 | LR: 1.27e-04 | 2702.8ms/step | 7.4 samples/s +Step 29450/44000 | Loss: 2.8916 | LR: 1.27e-04 | 2695.9ms/step | 7.4 samples/s +Step 29460/44000 | Loss: 2.8786 | LR: 1.27e-04 | 2684.9ms/step | 7.4 samples/s +Step 29470/44000 | Loss: 2.9041 | LR: 1.27e-04 | 2704.1ms/step | 7.4 samples/s +Step 29480/44000 | Loss: 2.8932 | LR: 1.27e-04 | 2677.0ms/step | 7.5 samples/s +Step 29490/44000 | Loss: 2.8970 | LR: 1.26e-04 | 2748.8ms/step | 7.3 samples/s +Step 29500/44000 | Loss: 2.8987 | LR: 1.26e-04 | 2621.1ms/step | 7.6 samples/s +Validation at step 29500: Loss = 2.9166, Perplexity = 18.48 +Step 29510/44000 | Loss: 2.8701 | LR: 1.26e-04 | 4855.2ms/step | 4.1 samples/s +Step 29520/44000 | Loss: 2.8985 | LR: 1.26e-04 | 1987.7ms/step | 10.1 samples/s +Step 29530/44000 | Loss: 2.8817 | LR: 1.26e-04 | 2001.3ms/step | 10.0 samples/s +Step 29540/44000 | Loss: 2.9059 | LR: 1.26e-04 | 1996.4ms/step | 10.0 samples/s +Step 29550/44000 | Loss: 2.8996 | LR: 1.26e-04 | 2000.1ms/step | 10.0 samples/s +Step 29560/44000 | Loss: 2.8928 | LR: 1.25e-04 | 1976.8ms/step | 10.1 samples/s +Step 29570/44000 | Loss: 2.8968 | LR: 1.25e-04 | 2040.7ms/step | 9.8 samples/s +Step 29580/44000 | Loss: 2.8762 | LR: 1.25e-04 | 2044.1ms/step | 9.8 samples/s +Step 29590/44000 | Loss: 2.9383 | LR: 1.25e-04 | 2048.1ms/step | 9.8 samples/s +Step 29600/44000 | Loss: 2.9109 | LR: 1.25e-04 | 2011.1ms/step | 9.9 samples/s +Step 29610/44000 | Loss: 2.8685 | LR: 1.25e-04 | 2013.5ms/step | 9.9 samples/s +Step 29620/44000 | Loss: 2.8881 | LR: 1.24e-04 | 2032.6ms/step | 9.8 samples/s +Step 29630/44000 | Loss: 2.8850 | LR: 1.24e-04 | 2058.4ms/step | 9.7 samples/s +Step 29640/44000 | Loss: 2.9035 | LR: 1.24e-04 | 2048.8ms/step | 9.8 samples/s +Step 29650/44000 | Loss: 2.9295 | LR: 1.24e-04 | 2069.5ms/step | 9.7 samples/s +Step 29660/44000 | Loss: 2.9115 | LR: 1.24e-04 | 2022.0ms/step | 9.9 samples/s +Step 29670/44000 | Loss: 2.9088 | LR: 1.24e-04 | 2012.1ms/step | 9.9 samples/s +Step 29680/44000 | Loss: 2.8891 | LR: 1.23e-04 | 2064.1ms/step | 9.7 samples/s +Step 29690/44000 | Loss: 2.9116 | LR: 1.23e-04 | 2682.8ms/step | 7.5 samples/s +Step 29700/44000 | Loss: 2.8957 | LR: 1.23e-04 | 2701.1ms/step | 7.4 samples/s +Step 29710/44000 | Loss: 2.9419 | LR: 1.23e-04 | 2605.9ms/step | 7.7 samples/s +Step 29720/44000 | Loss: 2.8783 | LR: 1.23e-04 | 2718.7ms/step | 7.4 samples/s +Step 29730/44000 | Loss: 2.8651 | LR: 1.23e-04 | 2691.9ms/step | 7.4 samples/s +Step 29740/44000 | Loss: 2.9105 | LR: 1.23e-04 | 2726.4ms/step | 7.3 samples/s +Step 29750/44000 | Loss: 2.9164 | LR: 1.22e-04 | 2567.5ms/step | 7.8 samples/s +Validation at step 29750: Loss = 2.9157, Perplexity = 18.46 +Step 29760/44000 | Loss: 2.9487 | LR: 1.22e-04 | 5241.8ms/step | 3.8 samples/s +Step 29770/44000 | Loss: 2.8860 | LR: 1.22e-04 | 2800.2ms/step | 7.1 samples/s +Step 29780/44000 | Loss: 2.9044 | LR: 1.22e-04 | 2657.8ms/step | 7.5 samples/s +Step 29790/44000 | Loss: 2.9251 | LR: 1.22e-04 | 2829.2ms/step | 7.1 samples/s +Step 29800/44000 | Loss: 2.9146 | LR: 1.22e-04 | 2696.9ms/step | 7.4 samples/s +Step 29810/44000 | Loss: 2.8980 | LR: 1.21e-04 | 2582.5ms/step | 7.7 samples/s +Step 29820/44000 | Loss: 2.8822 | LR: 1.21e-04 | 2668.2ms/step | 7.5 samples/s +Step 29830/44000 | Loss: 2.9357 | LR: 1.21e-04 | 2652.7ms/step | 7.5 samples/s +Step 29840/44000 | Loss: 2.8843 | LR: 1.21e-04 | 2761.4ms/step | 7.2 samples/s +Step 29850/44000 | Loss: 2.9290 | LR: 1.21e-04 | 2653.6ms/step | 7.5 samples/s +Step 29860/44000 | Loss: 2.9228 | LR: 1.21e-04 | 2645.8ms/step | 7.6 samples/s +Step 29870/44000 | Loss: 2.9130 | LR: 1.21e-04 | 2112.3ms/step | 9.5 samples/s +Step 29880/44000 | Loss: 2.8840 | LR: 1.20e-04 | 2032.5ms/step | 9.8 samples/s +Step 29890/44000 | Loss: 2.9511 | LR: 1.20e-04 | 1997.0ms/step | 10.0 samples/s +Step 29900/44000 | Loss: 2.8837 | LR: 1.20e-04 | 2006.5ms/step | 10.0 samples/s +Step 29910/44000 | Loss: 2.8627 | LR: 1.20e-04 | 2054.4ms/step | 9.7 samples/s +Step 29920/44000 | Loss: 2.8828 | LR: 1.20e-04 | 2073.2ms/step | 9.6 samples/s +Step 29930/44000 | Loss: 2.8726 | LR: 1.20e-04 | 2032.7ms/step | 9.8 samples/s +Step 29940/44000 | Loss: 2.8778 | LR: 1.19e-04 | 1995.5ms/step | 10.0 samples/s +Step 29950/44000 | Loss: 2.8843 | LR: 1.19e-04 | 2012.7ms/step | 9.9 samples/s +Step 29960/44000 | Loss: 2.8973 | LR: 1.19e-04 | 2079.3ms/step | 9.6 samples/s +Step 29970/44000 | Loss: 2.8288 | LR: 1.19e-04 | 2074.3ms/step | 9.6 samples/s +Step 29980/44000 | Loss: 2.8996 | LR: 1.19e-04 | 2089.8ms/step | 9.6 samples/s +Step 29990/44000 | Loss: 2.8944 | LR: 1.19e-04 | 2062.7ms/step | 9.7 samples/s +Step 30000/44000 | Loss: 2.9146 | LR: 1.19e-04 | 2022.8ms/step | 9.9 samples/s +Validation at step 30000: Loss = 2.9148, Perplexity = 18.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-30000 +Step 30010/44000 | Loss: 2.9089 | LR: 1.18e-04 | 4601.3ms/step | 4.3 samples/s +Step 30020/44000 | Loss: 2.8915 | LR: 1.18e-04 | 2650.7ms/step | 7.5 samples/s +Step 30030/44000 | Loss: 2.8672 | LR: 1.18e-04 | 2575.4ms/step | 7.8 samples/s +Step 30040/44000 | Loss: 2.9047 | LR: 1.18e-04 | 2590.9ms/step | 7.7 samples/s +Step 30050/44000 | Loss: 2.9043 | LR: 1.18e-04 | 2585.2ms/step | 7.7 samples/s +Step 30060/44000 | Loss: 2.9093 | LR: 1.18e-04 | 2632.3ms/step | 7.6 samples/s +Step 30070/44000 | Loss: 2.8717 | LR: 1.17e-04 | 2679.2ms/step | 7.5 samples/s +Step 30080/44000 | Loss: 2.8450 | LR: 1.17e-04 | 2740.5ms/step | 7.3 samples/s +Step 30090/44000 | Loss: 2.8892 | LR: 1.17e-04 | 2613.1ms/step | 7.7 samples/s +Step 30100/44000 | Loss: 2.9146 | LR: 1.17e-04 | 2710.7ms/step | 7.4 samples/s +Step 30110/44000 | Loss: 2.8781 | LR: 1.17e-04 | 2498.8ms/step | 8.0 samples/s +Step 30120/44000 | Loss: 2.9011 | LR: 1.17e-04 | 2295.1ms/step | 8.7 samples/s +Step 30130/44000 | Loss: 2.8879 | LR: 1.17e-04 | 2805.9ms/step | 7.1 samples/s +Step 30140/44000 | Loss: 2.8943 | LR: 1.16e-04 | 2659.5ms/step | 7.5 samples/s +Step 30150/44000 | Loss: 2.8871 | LR: 1.16e-04 | 2746.8ms/step | 7.3 samples/s +Step 30160/44000 | Loss: 2.8828 | LR: 1.16e-04 | 2777.7ms/step | 7.2 samples/s +Step 30170/44000 | Loss: 2.9029 | LR: 1.16e-04 | 2671.0ms/step | 7.5 samples/s +Step 30180/44000 | Loss: 2.8121 | LR: 1.16e-04 | 2680.9ms/step | 7.5 samples/s +Step 30190/44000 | Loss: 2.9491 | LR: 1.16e-04 | 2708.3ms/step | 7.4 samples/s +Step 30200/44000 | Loss: 2.8911 | LR: 1.15e-04 | 2585.4ms/step | 7.7 samples/s +Step 30210/44000 | Loss: 2.8767 | LR: 1.15e-04 | 2617.0ms/step | 7.6 samples/s +Step 30220/44000 | Loss: 2.9114 | LR: 1.15e-04 | 2009.8ms/step | 10.0 samples/s +Step 30230/44000 | Loss: 2.8573 | LR: 1.15e-04 | 2008.1ms/step | 10.0 samples/s +Step 30240/44000 | Loss: 2.9062 | LR: 1.15e-04 | 2023.6ms/step | 9.9 samples/s +Step 30250/44000 | Loss: 2.9296 | LR: 1.15e-04 | 2011.6ms/step | 9.9 samples/s +Validation at step 30250: Loss = 2.9139, Perplexity = 18.43 +Step 30260/44000 | Loss: 2.8799 | LR: 1.15e-04 | 3671.3ms/step | 5.4 samples/s +Step 30270/44000 | Loss: 2.8858 | LR: 1.14e-04 | 2027.8ms/step | 9.9 samples/s +Step 30280/44000 | Loss: 2.8931 | LR: 1.14e-04 | 2033.9ms/step | 9.8 samples/s +Step 30290/44000 | Loss: 2.8995 | LR: 1.14e-04 | 2068.2ms/step | 9.7 samples/s +Step 30300/44000 | Loss: 2.9230 | LR: 1.14e-04 | 2052.8ms/step | 9.7 samples/s +Step 30310/44000 | Loss: 2.9142 | LR: 1.14e-04 | 1983.1ms/step | 10.1 samples/s +Step 30320/44000 | Loss: 2.8978 | LR: 1.14e-04 | 2015.6ms/step | 9.9 samples/s +Step 30330/44000 | Loss: 2.8665 | LR: 1.13e-04 | 2048.6ms/step | 9.8 samples/s +Step 30340/44000 | Loss: 2.8963 | LR: 1.13e-04 | 2054.3ms/step | 9.7 samples/s +Step 30350/44000 | Loss: 2.8442 | LR: 1.13e-04 | 2017.0ms/step | 9.9 samples/s +Step 30360/44000 | Loss: 2.8753 | LR: 1.13e-04 | 2017.5ms/step | 9.9 samples/s +Step 30370/44000 | Loss: 2.8940 | LR: 1.13e-04 | 2000.8ms/step | 10.0 samples/s +Step 30380/44000 | Loss: 2.9206 | LR: 1.13e-04 | 2043.1ms/step | 9.8 samples/s +Step 30390/44000 | Loss: 2.8563 | LR: 1.13e-04 | 2057.1ms/step | 9.7 samples/s +Step 30400/44000 | Loss: 2.8839 | LR: 1.12e-04 | 2034.2ms/step | 9.8 samples/s +Step 30410/44000 | Loss: 2.9075 | LR: 1.12e-04 | 1996.6ms/step | 10.0 samples/s +Step 30420/44000 | Loss: 2.8943 | LR: 1.12e-04 | 1995.3ms/step | 10.0 samples/s +Step 30430/44000 | Loss: 2.8908 | LR: 1.12e-04 | 2016.6ms/step | 9.9 samples/s +Step 30440/44000 | Loss: 2.8792 | LR: 1.12e-04 | 2048.6ms/step | 9.8 samples/s +Step 30450/44000 | Loss: 2.8821 | LR: 1.12e-04 | 2032.8ms/step | 9.8 samples/s +Step 30460/44000 | Loss: 2.8801 | LR: 1.11e-04 | 2021.0ms/step | 9.9 samples/s +Step 30470/44000 | Loss: 2.8853 | LR: 1.11e-04 | 2007.2ms/step | 10.0 samples/s +Step 30480/44000 | Loss: 2.8665 | LR: 1.11e-04 | 2024.1ms/step | 9.9 samples/s +Step 30490/44000 | Loss: 2.8817 | LR: 1.11e-04 | 2035.9ms/step | 9.8 samples/s +Step 30500/44000 | Loss: 2.8959 | LR: 1.11e-04 | 2026.2ms/step | 9.9 samples/s +Validation at step 30500: Loss = 2.9133, Perplexity = 18.42 +Step 30510/44000 | Loss: 2.9018 | LR: 1.11e-04 | 3707.1ms/step | 5.4 samples/s +Step 30520/44000 | Loss: 2.8739 | LR: 1.11e-04 | 2010.3ms/step | 9.9 samples/s +Step 30530/44000 | Loss: 2.8951 | LR: 1.10e-04 | 2049.1ms/step | 9.8 samples/s +Step 30540/44000 | Loss: 2.9316 | LR: 1.10e-04 | 2090.0ms/step | 9.6 samples/s +Step 30550/44000 | Loss: 2.8471 | LR: 1.10e-04 | 2037.7ms/step | 9.8 samples/s +Step 30560/44000 | Loss: 2.9293 | LR: 1.10e-04 | 2059.2ms/step | 9.7 samples/s +Step 30570/44000 | Loss: 2.8560 | LR: 1.10e-04 | 2053.2ms/step | 9.7 samples/s +Step 30580/44000 | Loss: 2.8832 | LR: 1.10e-04 | 2033.1ms/step | 9.8 samples/s +Step 30590/44000 | Loss: 2.8850 | LR: 1.10e-04 | 2054.1ms/step | 9.7 samples/s +Step 30600/44000 | Loss: 2.9383 | LR: 1.09e-04 | 2067.8ms/step | 9.7 samples/s +Step 30610/44000 | Loss: 2.8665 | LR: 1.09e-04 | 2033.3ms/step | 9.8 samples/s +Step 30620/44000 | Loss: 2.8481 | LR: 1.09e-04 | 2033.1ms/step | 9.8 samples/s +Step 30630/44000 | Loss: 2.8814 | LR: 1.09e-04 | 2022.6ms/step | 9.9 samples/s +Step 30640/44000 | Loss: 2.9202 | LR: 1.09e-04 | 2668.6ms/step | 7.5 samples/s +Step 30650/44000 | Loss: 2.8456 | LR: 1.09e-04 | 2841.3ms/step | 7.0 samples/s +Step 30660/44000 | Loss: 2.9298 | LR: 1.08e-04 | 2904.7ms/step | 6.9 samples/s +Step 30670/44000 | Loss: 2.8747 | LR: 1.08e-04 | 3096.3ms/step | 6.5 samples/s +Step 30680/44000 | Loss: 2.9115 | LR: 1.08e-04 | 2827.3ms/step | 7.1 samples/s +Step 30690/44000 | Loss: 2.8652 | LR: 1.08e-04 | 2909.1ms/step | 6.9 samples/s +Step 30700/44000 | Loss: 2.9449 | LR: 1.08e-04 | 3026.7ms/step | 6.6 samples/s +Step 30710/44000 | Loss: 2.8818 | LR: 1.08e-04 | 2807.4ms/step | 7.1 samples/s +Step 30720/44000 | Loss: 2.8618 | LR: 1.08e-04 | 2826.0ms/step | 7.1 samples/s +Step 30730/44000 | Loss: 2.9283 | LR: 1.07e-04 | 2917.2ms/step | 6.9 samples/s +Step 30740/44000 | Loss: 2.9215 | LR: 1.07e-04 | 2970.3ms/step | 6.7 samples/s +Step 30750/44000 | Loss: 2.8578 | LR: 1.07e-04 | 2976.4ms/step | 6.7 samples/s +Validation at step 30750: Loss = 2.9123, Perplexity = 18.40 +Step 30760/44000 | Loss: 2.9363 | LR: 1.07e-04 | 5386.5ms/step | 3.7 samples/s +Step 30770/44000 | Loss: 2.8893 | LR: 1.07e-04 | 2835.3ms/step | 7.1 samples/s +Step 30780/44000 | Loss: 2.9196 | LR: 1.07e-04 | 2900.8ms/step | 6.9 samples/s +Step 30790/44000 | Loss: 2.9046 | LR: 1.07e-04 | 3032.1ms/step | 6.6 samples/s +Step 30800/44000 | Loss: 2.9119 | LR: 1.06e-04 | 2679.6ms/step | 7.5 samples/s +Step 30810/44000 | Loss: 2.8560 | LR: 1.06e-04 | 2178.8ms/step | 9.2 samples/s +Step 30820/44000 | Loss: 2.8787 | LR: 1.06e-04 | 2206.7ms/step | 9.1 samples/s +Step 30830/44000 | Loss: 2.9082 | LR: 1.06e-04 | 2186.2ms/step | 9.1 samples/s +Step 30840/44000 | Loss: 2.8902 | LR: 1.06e-04 | 2189.5ms/step | 9.1 samples/s +Step 30850/44000 | Loss: 2.8992 | LR: 1.06e-04 | 2106.5ms/step | 9.5 samples/s +Step 30860/44000 | Loss: 2.8960 | LR: 1.05e-04 | 2123.2ms/step | 9.4 samples/s +Step 30870/44000 | Loss: 2.8441 | LR: 1.05e-04 | 2138.5ms/step | 9.4 samples/s +Step 30880/44000 | Loss: 2.9310 | LR: 1.05e-04 | 2081.9ms/step | 9.6 samples/s +Step 30890/44000 | Loss: 2.9059 | LR: 1.05e-04 | 2152.4ms/step | 9.3 samples/s +Step 30900/44000 | Loss: 2.9196 | LR: 1.05e-04 | 2126.1ms/step | 9.4 samples/s +Step 30910/44000 | Loss: 2.8817 | LR: 1.05e-04 | 2173.3ms/step | 9.2 samples/s +Step 30920/44000 | Loss: 2.8814 | LR: 1.05e-04 | 2136.6ms/step | 9.4 samples/s +Step 30930/44000 | Loss: 2.9163 | LR: 1.04e-04 | 2121.1ms/step | 9.4 samples/s +Step 30940/44000 | Loss: 2.8858 | LR: 1.04e-04 | 2122.2ms/step | 9.4 samples/s +Step 30950/44000 | Loss: 2.9220 | LR: 1.04e-04 | 2193.5ms/step | 9.1 samples/s +Step 30960/44000 | Loss: 2.8363 | LR: 1.04e-04 | 2169.0ms/step | 9.2 samples/s +Step 30970/44000 | Loss: 2.8778 | LR: 1.04e-04 | 2203.7ms/step | 9.1 samples/s +Step 30980/44000 | Loss: 2.8957 | LR: 1.04e-04 | 2209.4ms/step | 9.1 samples/s +Step 30990/44000 | Loss: 2.8925 | LR: 1.04e-04 | 2263.9ms/step | 8.8 samples/s +Step 31000/44000 | Loss: 2.8993 | LR: 1.03e-04 | 2196.6ms/step | 9.1 samples/s +Validation at step 31000: Loss = 2.9116, Perplexity = 18.39 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-31000 +Step 31010/44000 | Loss: 2.8791 | LR: 1.03e-04 | 4454.9ms/step | 4.5 samples/s +Step 31020/44000 | Loss: 2.9118 | LR: 1.03e-04 | 2203.8ms/step | 9.1 samples/s +Step 31030/44000 | Loss: 2.8933 | LR: 1.03e-04 | 2205.1ms/step | 9.1 samples/s +Step 31040/44000 | Loss: 2.8887 | LR: 1.03e-04 | 2162.5ms/step | 9.2 samples/s +Step 31050/44000 | Loss: 2.8562 | LR: 1.03e-04 | 2167.5ms/step | 9.2 samples/s +Step 31060/44000 | Loss: 2.8728 | LR: 1.03e-04 | 2192.4ms/step | 9.1 samples/s +Step 31070/44000 | Loss: 2.8807 | LR: 1.02e-04 | 2142.9ms/step | 9.3 samples/s +Step 31080/44000 | Loss: 2.8911 | LR: 1.02e-04 | 2208.3ms/step | 9.1 samples/s +Step 31090/44000 | Loss: 2.8516 | LR: 1.02e-04 | 2175.1ms/step | 9.2 samples/s +Step 31100/44000 | Loss: 2.9352 | LR: 1.02e-04 | 2186.6ms/step | 9.1 samples/s +Step 31110/44000 | Loss: 2.8580 | LR: 1.02e-04 | 2266.5ms/step | 8.8 samples/s +Step 31120/44000 | Loss: 2.8966 | LR: 1.02e-04 | 2214.4ms/step | 9.0 samples/s +Step 31130/44000 | Loss: 2.9264 | LR: 1.02e-04 | 2174.8ms/step | 9.2 samples/s +Step 31140/44000 | Loss: 2.9197 | LR: 1.01e-04 | 2150.3ms/step | 9.3 samples/s +Step 31150/44000 | Loss: 2.9054 | LR: 1.01e-04 | 2140.7ms/step | 9.3 samples/s +Step 31160/44000 | Loss: 2.8941 | LR: 1.01e-04 | 2214.9ms/step | 9.0 samples/s +Step 31170/44000 | Loss: 2.8745 | LR: 1.01e-04 | 2194.1ms/step | 9.1 samples/s +Step 31180/44000 | Loss: 2.9450 | LR: 1.01e-04 | 2156.9ms/step | 9.3 samples/s +Step 31190/44000 | Loss: 2.8877 | LR: 1.01e-04 | 2179.6ms/step | 9.2 samples/s +Step 31200/44000 | Loss: 2.8889 | LR: 1.00e-04 | 2111.4ms/step | 9.5 samples/s +Step 31210/44000 | Loss: 2.9369 | LR: 1.00e-04 | 2180.2ms/step | 9.2 samples/s +Step 31220/44000 | Loss: 2.8815 | LR: 1.00e-04 | 2135.8ms/step | 9.4 samples/s +Step 31230/44000 | Loss: 2.8943 | LR: 1.00e-04 | 2126.3ms/step | 9.4 samples/s +Step 31240/44000 | Loss: 2.9202 | LR: 9.99e-05 | 2142.2ms/step | 9.3 samples/s +Step 31250/44000 | Loss: 2.8894 | LR: 9.98e-05 | 2135.0ms/step | 9.4 samples/s +Validation at step 31250: Loss = 2.9111, Perplexity = 18.38 +Step 31260/44000 | Loss: 2.8812 | LR: 9.96e-05 | 3940.9ms/step | 5.1 samples/s +Step 31270/44000 | Loss: 2.9061 | LR: 9.95e-05 | 2134.5ms/step | 9.4 samples/s +Step 31280/44000 | Loss: 2.8559 | LR: 9.93e-05 | 2166.9ms/step | 9.2 samples/s +Step 31290/44000 | Loss: 2.9150 | LR: 9.92e-05 | 2224.9ms/step | 9.0 samples/s +Step 31300/44000 | Loss: 2.8796 | LR: 9.90e-05 | 2221.5ms/step | 9.0 samples/s +Step 31310/44000 | Loss: 2.9139 | LR: 9.89e-05 | 2147.9ms/step | 9.3 samples/s +Step 31320/44000 | Loss: 2.8821 | LR: 9.88e-05 | 2040.8ms/step | 9.8 samples/s +Step 31330/44000 | Loss: 2.8891 | LR: 9.86e-05 | 2040.9ms/step | 9.8 samples/s +Step 31340/44000 | Loss: 2.8812 | LR: 9.85e-05 | 2207.8ms/step | 9.1 samples/s +Step 31350/44000 | Loss: 2.8944 | LR: 9.83e-05 | 2152.3ms/step | 9.3 samples/s +Step 31360/44000 | Loss: 2.9028 | LR: 9.82e-05 | 2144.1ms/step | 9.3 samples/s +Step 31370/44000 | Loss: 2.8809 | LR: 9.80e-05 | 2178.2ms/step | 9.2 samples/s +Step 31380/44000 | Loss: 2.8957 | LR: 9.79e-05 | 2170.5ms/step | 9.2 samples/s +Step 31390/44000 | Loss: 2.8469 | LR: 9.77e-05 | 2158.2ms/step | 9.3 samples/s +Step 31400/44000 | Loss: 2.9127 | LR: 9.76e-05 | 2199.1ms/step | 9.1 samples/s +Step 31410/44000 | Loss: 2.8523 | LR: 9.75e-05 | 2151.8ms/step | 9.3 samples/s +Step 31420/44000 | Loss: 2.8982 | LR: 9.73e-05 | 2177.1ms/step | 9.2 samples/s +Step 31430/44000 | Loss: 2.8806 | LR: 9.72e-05 | 2132.4ms/step | 9.4 samples/s +Step 31440/44000 | Loss: 2.8997 | LR: 9.70e-05 | 2209.5ms/step | 9.1 samples/s +Step 31450/44000 | Loss: 2.8928 | LR: 9.69e-05 | 2255.3ms/step | 8.9 samples/s +Step 31460/44000 | Loss: 2.8695 | LR: 9.67e-05 | 2228.3ms/step | 9.0 samples/s +Step 31470/44000 | Loss: 2.9125 | LR: 9.66e-05 | 2203.5ms/step | 9.1 samples/s +Step 31480/44000 | Loss: 2.8742 | LR: 9.65e-05 | 2169.4ms/step | 9.2 samples/s +Step 31490/44000 | Loss: 2.9021 | LR: 9.63e-05 | 2185.7ms/step | 9.2 samples/s +Step 31500/44000 | Loss: 2.9032 | LR: 9.62e-05 | 2184.2ms/step | 9.2 samples/s +Validation at step 31500: Loss = 2.9105, Perplexity = 18.37 +Step 31510/44000 | Loss: 2.8645 | LR: 9.60e-05 | 4088.5ms/step | 4.9 samples/s +Step 31520/44000 | Loss: 2.8917 | LR: 9.59e-05 | 2203.6ms/step | 9.1 samples/s +Step 31530/44000 | Loss: 2.8845 | LR: 9.57e-05 | 2183.3ms/step | 9.2 samples/s +Step 31540/44000 | Loss: 2.8982 | LR: 9.56e-05 | 2182.8ms/step | 9.2 samples/s +Step 31550/44000 | Loss: 2.8748 | LR: 9.54e-05 | 2187.9ms/step | 9.1 samples/s +Step 31560/44000 | Loss: 2.9469 | LR: 9.53e-05 | 2161.2ms/step | 9.3 samples/s +Step 31570/44000 | Loss: 2.8979 | LR: 9.52e-05 | 2184.3ms/step | 9.2 samples/s +Step 31580/44000 | Loss: 2.8996 | LR: 9.50e-05 | 2151.8ms/step | 9.3 samples/s +Step 31590/44000 | Loss: 2.9018 | LR: 9.49e-05 | 2251.5ms/step | 8.9 samples/s +Step 31600/44000 | Loss: 2.8793 | LR: 9.47e-05 | 2145.0ms/step | 9.3 samples/s +Step 31610/44000 | Loss: 2.8838 | LR: 9.46e-05 | 2179.3ms/step | 9.2 samples/s +Step 31620/44000 | Loss: 2.8513 | LR: 9.45e-05 | 2157.3ms/step | 9.3 samples/s +Step 31630/44000 | Loss: 2.8882 | LR: 9.43e-05 | 2134.4ms/step | 9.4 samples/s +Step 31640/44000 | Loss: 2.9093 | LR: 9.42e-05 | 2099.7ms/step | 9.5 samples/s +Step 31650/44000 | Loss: 2.8764 | LR: 9.40e-05 | 2198.1ms/step | 9.1 samples/s +Step 31660/44000 | Loss: 2.9030 | LR: 9.39e-05 | 2151.0ms/step | 9.3 samples/s +Step 31670/44000 | Loss: 2.8981 | LR: 9.37e-05 | 2188.2ms/step | 9.1 samples/s +Step 31680/44000 | Loss: 2.8437 | LR: 9.36e-05 | 2165.4ms/step | 9.2 samples/s +Step 31690/44000 | Loss: 2.8364 | LR: 9.35e-05 | 2198.7ms/step | 9.1 samples/s +Step 31700/44000 | Loss: 2.9150 | LR: 9.33e-05 | 2177.1ms/step | 9.2 samples/s +Step 31710/44000 | Loss: 2.8780 | LR: 9.32e-05 | 2186.0ms/step | 9.1 samples/s +Step 31720/44000 | Loss: 2.9149 | LR: 9.30e-05 | 2189.7ms/step | 9.1 samples/s +Step 31730/44000 | Loss: 2.8897 | LR: 9.29e-05 | 2166.9ms/step | 9.2 samples/s +Step 31740/44000 | Loss: 2.8891 | LR: 9.28e-05 | 2188.0ms/step | 9.1 samples/s +Step 31750/44000 | Loss: 2.9238 | LR: 9.26e-05 | 2135.8ms/step | 9.4 samples/s +Validation at step 31750: Loss = 2.9098, Perplexity = 18.35 +Step 31760/44000 | Loss: 2.9163 | LR: 9.25e-05 | 4032.1ms/step | 5.0 samples/s +Step 31770/44000 | Loss: 2.9412 | LR: 9.23e-05 | 2250.7ms/step | 8.9 samples/s +Step 31780/44000 | Loss: 2.9303 | LR: 9.22e-05 | 2177.6ms/step | 9.2 samples/s +Step 31790/44000 | Loss: 2.8770 | LR: 9.20e-05 | 2142.6ms/step | 9.3 samples/s +Step 31800/44000 | Loss: 2.8751 | LR: 9.19e-05 | 2165.5ms/step | 9.2 samples/s +Step 31810/44000 | Loss: 2.8848 | LR: 9.18e-05 | 2235.1ms/step | 8.9 samples/s +Step 31820/44000 | Loss: 2.8882 | LR: 9.16e-05 | 2141.5ms/step | 9.3 samples/s +Step 31830/44000 | Loss: 2.9099 | LR: 9.15e-05 | 2198.4ms/step | 9.1 samples/s +Step 31840/44000 | Loss: 2.8932 | LR: 9.13e-05 | 2178.4ms/step | 9.2 samples/s +Step 31850/44000 | Loss: 2.8508 | LR: 9.12e-05 | 2180.8ms/step | 9.2 samples/s +Step 31860/44000 | Loss: 2.8779 | LR: 9.11e-05 | 2184.1ms/step | 9.2 samples/s +Step 31870/44000 | Loss: 2.8848 | LR: 9.09e-05 | 2177.7ms/step | 9.2 samples/s +Step 31880/44000 | Loss: 2.8861 | LR: 9.08e-05 | 2142.1ms/step | 9.3 samples/s +Step 31890/44000 | Loss: 2.8812 | LR: 9.06e-05 | 2219.3ms/step | 9.0 samples/s +Step 31900/44000 | Loss: 2.8610 | LR: 9.05e-05 | 2136.4ms/step | 9.4 samples/s +Step 31910/44000 | Loss: 2.9264 | LR: 9.04e-05 | 2214.9ms/step | 9.0 samples/s +Step 31920/44000 | Loss: 2.8983 | LR: 9.02e-05 | 2198.3ms/step | 9.1 samples/s +Step 31930/44000 | Loss: 2.8592 | LR: 9.01e-05 | 2239.4ms/step | 8.9 samples/s +Step 31940/44000 | Loss: 2.8932 | LR: 8.99e-05 | 2072.9ms/step | 9.6 samples/s +Step 31950/44000 | Loss: 2.8923 | LR: 8.98e-05 | 2157.5ms/step | 9.3 samples/s +Step 31960/44000 | Loss: 2.8535 | LR: 8.97e-05 | 2151.1ms/step | 9.3 samples/s +Step 31970/44000 | Loss: 2.8604 | LR: 8.95e-05 | 2178.1ms/step | 9.2 samples/s +Step 31980/44000 | Loss: 2.8859 | LR: 8.94e-05 | 2162.1ms/step | 9.3 samples/s +Step 31990/44000 | Loss: 2.8949 | LR: 8.92e-05 | 2183.0ms/step | 9.2 samples/s +Step 32000/44000 | Loss: 2.8479 | LR: 8.91e-05 | 2130.6ms/step | 9.4 samples/s +Validation at step 32000: Loss = 2.9091, Perplexity = 18.34 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-32000 +Step 32010/44000 | Loss: 2.9002 | LR: 8.90e-05 | 4522.5ms/step | 4.4 samples/s +Step 32020/44000 | Loss: 2.8991 | LR: 8.88e-05 | 2182.5ms/step | 9.2 samples/s +Step 32030/44000 | Loss: 2.9702 | LR: 8.87e-05 | 2154.4ms/step | 9.3 samples/s +Step 32040/44000 | Loss: 2.8965 | LR: 8.86e-05 | 2165.7ms/step | 9.2 samples/s +Step 32050/44000 | Loss: 2.9004 | LR: 8.84e-05 | 2121.1ms/step | 9.4 samples/s +Step 32060/44000 | Loss: 2.9113 | LR: 8.83e-05 | 2167.6ms/step | 9.2 samples/s +Step 32070/44000 | Loss: 2.9024 | LR: 8.81e-05 | 2178.2ms/step | 9.2 samples/s +Step 32080/44000 | Loss: 2.8721 | LR: 8.80e-05 | 2168.7ms/step | 9.2 samples/s +Step 32090/44000 | Loss: 2.9249 | LR: 8.79e-05 | 2151.7ms/step | 9.3 samples/s +Step 32100/44000 | Loss: 2.9202 | LR: 8.77e-05 | 2204.7ms/step | 9.1 samples/s +Step 32110/44000 | Loss: 2.8630 | LR: 8.76e-05 | 2236.0ms/step | 8.9 samples/s +Step 32120/44000 | Loss: 2.9194 | LR: 8.74e-05 | 2172.3ms/step | 9.2 samples/s +Step 32130/44000 | Loss: 2.8748 | LR: 8.73e-05 | 2136.6ms/step | 9.4 samples/s +Step 32140/44000 | Loss: 2.8590 | LR: 8.72e-05 | 2189.2ms/step | 9.1 samples/s +Step 32150/44000 | Loss: 2.8800 | LR: 8.70e-05 | 2161.7ms/step | 9.3 samples/s +Step 32160/44000 | Loss: 2.8721 | LR: 8.69e-05 | 2173.9ms/step | 9.2 samples/s +Step 32170/44000 | Loss: 2.8763 | LR: 8.68e-05 | 2193.0ms/step | 9.1 samples/s +Step 32180/44000 | Loss: 2.8915 | LR: 8.66e-05 | 2194.8ms/step | 9.1 samples/s +Step 32190/44000 | Loss: 2.9070 | LR: 8.65e-05 | 2161.7ms/step | 9.3 samples/s +Step 32200/44000 | Loss: 2.8535 | LR: 8.63e-05 | 2152.0ms/step | 9.3 samples/s +Step 32210/44000 | Loss: 2.9204 | LR: 8.62e-05 | 2179.3ms/step | 9.2 samples/s +Step 32220/44000 | Loss: 2.8958 | LR: 8.61e-05 | 2239.8ms/step | 8.9 samples/s +Step 32230/44000 | Loss: 2.8733 | LR: 8.59e-05 | 2194.5ms/step | 9.1 samples/s +Step 32240/44000 | Loss: 2.9144 | LR: 8.58e-05 | 2152.7ms/step | 9.3 samples/s +Step 32250/44000 | Loss: 2.8998 | LR: 8.57e-05 | 2109.9ms/step | 9.5 samples/s +Validation at step 32250: Loss = 2.9086, Perplexity = 18.33 +Step 32260/44000 | Loss: 2.9113 | LR: 8.55e-05 | 3929.7ms/step | 5.1 samples/s +Step 32270/44000 | Loss: 2.8594 | LR: 8.54e-05 | 2207.9ms/step | 9.1 samples/s +Step 32280/44000 | Loss: 2.9208 | LR: 8.53e-05 | 2123.0ms/step | 9.4 samples/s +Step 32290/44000 | Loss: 2.9044 | LR: 8.51e-05 | 2194.7ms/step | 9.1 samples/s +Step 32300/44000 | Loss: 2.9190 | LR: 8.50e-05 | 2125.9ms/step | 9.4 samples/s +Step 32310/44000 | Loss: 2.9184 | LR: 8.48e-05 | 2166.7ms/step | 9.2 samples/s +Step 32320/44000 | Loss: 2.9214 | LR: 8.47e-05 | 2063.1ms/step | 9.7 samples/s +Step 32330/44000 | Loss: 2.8911 | LR: 8.46e-05 | 2168.9ms/step | 9.2 samples/s +Step 32340/44000 | Loss: 2.8852 | LR: 8.44e-05 | 2194.5ms/step | 9.1 samples/s +Step 32350/44000 | Loss: 2.8587 | LR: 8.43e-05 | 2251.4ms/step | 8.9 samples/s +Step 32360/44000 | Loss: 2.8851 | LR: 8.42e-05 | 2202.4ms/step | 9.1 samples/s +Step 32370/44000 | Loss: 2.9340 | LR: 8.40e-05 | 2246.0ms/step | 8.9 samples/s +Step 32380/44000 | Loss: 2.8657 | LR: 8.39e-05 | 2172.6ms/step | 9.2 samples/s +Step 32390/44000 | Loss: 2.9030 | LR: 8.38e-05 | 2141.8ms/step | 9.3 samples/s +Step 32400/44000 | Loss: 2.9102 | LR: 8.36e-05 | 2110.8ms/step | 9.5 samples/s +Step 32410/44000 | Loss: 2.9328 | LR: 8.35e-05 | 2156.9ms/step | 9.3 samples/s +Step 32420/44000 | Loss: 2.9092 | LR: 8.33e-05 | 2198.5ms/step | 9.1 samples/s +Step 32430/44000 | Loss: 2.8757 | LR: 8.32e-05 | 2183.1ms/step | 9.2 samples/s +Step 32440/44000 | Loss: 2.8636 | LR: 8.31e-05 | 2188.2ms/step | 9.1 samples/s +Step 32450/44000 | Loss: 2.9456 | LR: 8.29e-05 | 2142.9ms/step | 9.3 samples/s +Step 32460/44000 | Loss: 2.9402 | LR: 8.28e-05 | 2175.6ms/step | 9.2 samples/s +Step 32470/44000 | Loss: 2.8778 | LR: 8.27e-05 | 2179.1ms/step | 9.2 samples/s +Step 32480/44000 | Loss: 2.9270 | LR: 8.25e-05 | 2142.2ms/step | 9.3 samples/s +Step 32490/44000 | Loss: 2.8574 | LR: 8.24e-05 | 2212.9ms/step | 9.0 samples/s +Step 32500/44000 | Loss: 2.8938 | LR: 8.23e-05 | 2149.3ms/step | 9.3 samples/s +Validation at step 32500: Loss = 2.9081, Perplexity = 18.32 +Step 32510/44000 | Loss: 2.8717 | LR: 8.21e-05 | 4079.1ms/step | 4.9 samples/s +Step 32520/44000 | Loss: 2.9005 | LR: 8.20e-05 | 2145.8ms/step | 9.3 samples/s +Step 32530/44000 | Loss: 2.8776 | LR: 8.19e-05 | 2151.1ms/step | 9.3 samples/s +Step 32540/44000 | Loss: 2.8796 | LR: 8.17e-05 | 2167.1ms/step | 9.2 samples/s +Step 32550/44000 | Loss: 2.8951 | LR: 8.16e-05 | 2145.9ms/step | 9.3 samples/s +Step 32560/44000 | Loss: 2.9147 | LR: 8.15e-05 | 2062.0ms/step | 9.7 samples/s +Step 32570/44000 | Loss: 2.8953 | LR: 8.13e-05 | 2077.9ms/step | 9.6 samples/s +Step 32580/44000 | Loss: 2.9077 | LR: 8.12e-05 | 2210.9ms/step | 9.0 samples/s +Step 32590/44000 | Loss: 2.9108 | LR: 8.11e-05 | 2112.7ms/step | 9.5 samples/s +Step 32600/44000 | Loss: 2.8732 | LR: 8.09e-05 | 2135.2ms/step | 9.4 samples/s +Step 32610/44000 | Loss: 2.8923 | LR: 8.08e-05 | 2104.4ms/step | 9.5 samples/s +Step 32620/44000 | Loss: 2.9061 | LR: 8.07e-05 | 2181.3ms/step | 9.2 samples/s +Step 32630/44000 | Loss: 2.8863 | LR: 8.05e-05 | 2140.7ms/step | 9.3 samples/s +Step 32640/44000 | Loss: 2.8683 | LR: 8.04e-05 | 2220.6ms/step | 9.0 samples/s +Step 32650/44000 | Loss: 2.8908 | LR: 8.03e-05 | 2224.6ms/step | 9.0 samples/s +Step 32660/44000 | Loss: 2.8832 | LR: 8.01e-05 | 2187.2ms/step | 9.1 samples/s +Step 32670/44000 | Loss: 2.9239 | LR: 8.00e-05 | 2139.7ms/step | 9.3 samples/s +Step 32680/44000 | Loss: 2.8826 | LR: 7.99e-05 | 2164.4ms/step | 9.2 samples/s +Step 32690/44000 | Loss: 2.9080 | LR: 7.97e-05 | 2093.7ms/step | 9.6 samples/s +Step 32700/44000 | Loss: 2.9096 | LR: 7.96e-05 | 2193.4ms/step | 9.1 samples/s +Step 32710/44000 | Loss: 2.8803 | LR: 7.95e-05 | 2172.1ms/step | 9.2 samples/s +Step 32720/44000 | Loss: 2.9041 | LR: 7.93e-05 | 2201.3ms/step | 9.1 samples/s +Step 32730/44000 | Loss: 2.8829 | LR: 7.92e-05 | 2206.2ms/step | 9.1 samples/s +Step 32740/44000 | Loss: 2.8996 | LR: 7.91e-05 | 2177.5ms/step | 9.2 samples/s +Step 32750/44000 | Loss: 2.8959 | LR: 7.89e-05 | 2104.1ms/step | 9.5 samples/s +Validation at step 32750: Loss = 2.9076, Perplexity = 18.31 +Step 32760/44000 | Loss: 2.8883 | LR: 7.88e-05 | 4026.2ms/step | 5.0 samples/s +Step 32770/44000 | Loss: 2.8978 | LR: 7.87e-05 | 2144.9ms/step | 9.3 samples/s +Step 32780/44000 | Loss: 2.8618 | LR: 7.85e-05 | 2161.6ms/step | 9.3 samples/s +Step 32790/44000 | Loss: 2.9043 | LR: 7.84e-05 | 2208.0ms/step | 9.1 samples/s +Step 32800/44000 | Loss: 2.8665 | LR: 7.83e-05 | 2213.3ms/step | 9.0 samples/s +Step 32810/44000 | Loss: 2.8771 | LR: 7.81e-05 | 2213.8ms/step | 9.0 samples/s +Step 32820/44000 | Loss: 2.8664 | LR: 7.80e-05 | 2144.2ms/step | 9.3 samples/s +Step 32830/44000 | Loss: 2.9157 | LR: 7.79e-05 | 2204.9ms/step | 9.1 samples/s +Step 32840/44000 | Loss: 2.9107 | LR: 7.77e-05 | 1904.7ms/step | 10.5 samples/s +Step 32850/44000 | Loss: 2.9123 | LR: 7.76e-05 | 1901.5ms/step | 10.5 samples/s +Step 32860/44000 | Loss: 2.8755 | LR: 7.75e-05 | 1901.9ms/step | 10.5 samples/s +Step 32870/44000 | Loss: 2.8648 | LR: 7.73e-05 | 1904.2ms/step | 10.5 samples/s +Step 32880/44000 | Loss: 2.8935 | LR: 7.72e-05 | 1903.5ms/step | 10.5 samples/s +Step 32890/44000 | Loss: 2.9171 | LR: 7.71e-05 | 1902.5ms/step | 10.5 samples/s +Step 32900/44000 | Loss: 2.8679 | LR: 7.70e-05 | 1902.3ms/step | 10.5 samples/s +Step 32910/44000 | Loss: 2.9322 | LR: 7.68e-05 | 1903.0ms/step | 10.5 samples/s +Step 32920/44000 | Loss: 2.9190 | LR: 7.67e-05 | 1903.7ms/step | 10.5 samples/s +Step 32930/44000 | Loss: 2.8921 | LR: 7.66e-05 | 1903.4ms/step | 10.5 samples/s +Step 32940/44000 | Loss: 2.8974 | LR: 7.64e-05 | 1904.3ms/step | 10.5 samples/s +Step 32950/44000 | Loss: 2.8599 | LR: 7.63e-05 | 1904.7ms/step | 10.5 samples/s +Step 32960/44000 | Loss: 2.8714 | LR: 7.62e-05 | 1903.9ms/step | 10.5 samples/s +Step 32970/44000 | Loss: 2.8627 | LR: 7.60e-05 | 1903.4ms/step | 10.5 samples/s +Step 32980/44000 | Loss: 2.8994 | LR: 7.59e-05 | 1901.7ms/step | 10.5 samples/s +Step 32990/44000 | Loss: 2.9177 | LR: 7.58e-05 | 1903.5ms/step | 10.5 samples/s +Step 33000/44000 | Loss: 2.8766 | LR: 7.56e-05 | 1903.7ms/step | 10.5 samples/s +Validation at step 33000: Loss = 2.9072, Perplexity = 18.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247/checkpoint-33000 +Step 33010/44000 | Loss: 2.9182 | LR: 7.55e-05 | 3841.5ms/step | 5.2 samples/s +Step 33020/44000 | Loss: 2.9083 | LR: 7.54e-05 | 1902.0ms/step | 10.5 samples/s +Step 33030/44000 | Loss: 2.9051 | LR: 7.53e-05 | 1903.6ms/step | 10.5 samples/s +W0820 23:57:38.542000 38825 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers +W0820 23:57:38.546000 38825 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 38961 closing signal SIGTERM +W0820 23:57:38.552000 38825 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 38962 closing signal SIGTERM +Traceback (most recent call last): + File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in + sys.exit(main()) + ~~~~^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper + return f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main + run(args) + ~~~^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run + elastic_launch( + ~~~~~~~~~~~~~~~ + config=config, + ~~~~~~~~~~~~~~ + entrypoint=cmd, + ~~~~~~~~~~~~~~~ + )(*cmd_args) + ~^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent + result = agent.run() + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run + result = self._invoke_run(role) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run + time.sleep(monitor_interval) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 38825 got signal: 15 diff --git a/epfl_final/train_logs/seed47_pair_ddp2/mixed.log b/epfl_final/train_logs/seed47_pair_ddp2/mixed.log new file mode 100644 index 0000000000000000000000000000000000000000..4d7e43eb302a1f83df4034d6f1b21eaad17d3573 --- /dev/null +++ b/epfl_final/train_logs/seed47_pair_ddp2/mixed.log @@ -0,0 +1,2984 @@ +W0820 03:18:04.783000 40373 site-packages/torch/distributed/run.py:851] +W0820 03:18:04.783000 40373 site-packages/torch/distributed/run.py:851] ***************************************** +W0820 03:18:04.783000 40373 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0820 03:18:04.783000 40373 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 47, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m mixed 8k seed47 ddp2", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "mixed", + "multiseed", + "seed47", + "ddp2" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": [ + 63, + 47, + 31, + 21, + 11 + ], + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run 93ukqj9s +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_032150-93ukqj9s +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m mixed 8k seed47 ddp2 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/93ukqj9s +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.1152 | LR: 6.67e-06 | 8556.0ms/step | 2.3 samples/s +Step 20/44000 | Loss: 12.0130 | LR: 1.33e-05 | 2184.5ms/step | 9.2 samples/s +Step 30/44000 | Loss: 11.6767 | LR: 2.00e-05 | 2418.6ms/step | 8.3 samples/s +Step 40/44000 | Loss: 11.1197 | LR: 2.67e-05 | 2428.8ms/step | 8.2 samples/s +Step 50/44000 | Loss: 10.7999 | LR: 3.33e-05 | 2387.5ms/step | 8.4 samples/s +Step 60/44000 | Loss: 10.5632 | LR: 4.00e-05 | 2386.9ms/step | 8.4 samples/s +Step 70/44000 | Loss: 10.3149 | LR: 4.67e-05 | 2423.4ms/step | 8.3 samples/s +Step 80/44000 | Loss: 10.0000 | LR: 5.33e-05 | 2485.3ms/step | 8.0 samples/s +Step 90/44000 | Loss: 9.6390 | LR: 6.00e-05 | 2455.2ms/step | 8.1 samples/s +Step 100/44000 | Loss: 9.1376 | LR: 6.67e-05 | 2486.0ms/step | 8.0 samples/s +Step 110/44000 | Loss: 8.6417 | LR: 7.33e-05 | 2934.3ms/step | 6.8 samples/s +Step 120/44000 | Loss: 8.1488 | LR: 8.00e-05 | 2281.8ms/step | 8.8 samples/s +Step 130/44000 | Loss: 7.7734 | LR: 8.67e-05 | 2153.9ms/step | 9.3 samples/s +Step 140/44000 | Loss: 7.5224 | LR: 9.33e-05 | 2620.3ms/step | 7.6 samples/s +Step 150/44000 | Loss: 7.3472 | LR: 1.00e-04 | 2472.0ms/step | 8.1 samples/s +Step 160/44000 | Loss: 7.2297 | LR: 1.07e-04 | 2371.5ms/step | 8.4 samples/s +Step 170/44000 | Loss: 7.0948 | LR: 1.13e-04 | 2594.0ms/step | 7.7 samples/s +Step 180/44000 | Loss: 7.0127 | LR: 1.20e-04 | 2416.5ms/step | 8.3 samples/s +Step 190/44000 | Loss: 6.8924 | LR: 1.27e-04 | 2457.3ms/step | 8.1 samples/s +Step 200/44000 | Loss: 6.8107 | LR: 1.33e-04 | 2516.8ms/step | 7.9 samples/s +Step 210/44000 | Loss: 6.7505 | LR: 1.40e-04 | 2279.0ms/step | 8.8 samples/s +Step 220/44000 | Loss: 6.6633 | LR: 1.47e-04 | 2453.8ms/step | 8.2 samples/s +Step 230/44000 | Loss: 6.5715 | LR: 1.53e-04 | 2515.1ms/step | 8.0 samples/s +Step 240/44000 | Loss: 6.4913 | LR: 1.60e-04 | 2419.4ms/step | 8.3 samples/s +Step 250/44000 | Loss: 6.5078 | LR: 1.67e-04 | 2362.5ms/step | 8.5 samples/s +Validation at step 250: Loss = 6.4488, Perplexity = 631.93 +Step 260/44000 | Loss: 6.3795 | LR: 1.73e-04 | 9847.2ms/step | 2.0 samples/s +Step 270/44000 | Loss: 6.3921 | LR: 1.80e-04 | 2710.6ms/step | 7.4 samples/s +Step 280/44000 | Loss: 6.3110 | LR: 1.87e-04 | 2789.6ms/step | 7.2 samples/s +Step 290/44000 | Loss: 6.2773 | LR: 1.93e-04 | 2564.4ms/step | 7.8 samples/s +Step 300/44000 | Loss: 6.1712 | LR: 2.00e-04 | 2898.5ms/step | 6.9 samples/s +Step 310/44000 | Loss: 6.2062 | LR: 2.07e-04 | 2658.0ms/step | 7.5 samples/s +Step 320/44000 | Loss: 6.1060 | LR: 2.13e-04 | 2840.6ms/step | 7.0 samples/s +Step 330/44000 | Loss: 6.0995 | LR: 2.20e-04 | 2522.8ms/step | 7.9 samples/s +Step 340/44000 | Loss: 5.9995 | LR: 2.27e-04 | 2918.3ms/step | 6.9 samples/s +Step 350/44000 | Loss: 6.0061 | LR: 2.33e-04 | 2587.0ms/step | 7.7 samples/s +Step 360/44000 | Loss: 5.9744 | LR: 2.40e-04 | 2870.6ms/step | 7.0 samples/s +Step 370/44000 | Loss: 5.9413 | LR: 2.47e-04 | 2517.1ms/step | 7.9 samples/s +Step 380/44000 | Loss: 5.8809 | LR: 2.53e-04 | 2908.0ms/step | 6.9 samples/s +Step 390/44000 | Loss: 5.8764 | LR: 2.60e-04 | 2606.5ms/step | 7.7 samples/s +Step 400/44000 | Loss: 5.8327 | LR: 2.67e-04 | 2813.5ms/step | 7.1 samples/s +Step 410/44000 | Loss: 5.7756 | LR: 2.73e-04 | 2381.2ms/step | 8.4 samples/s +Step 420/44000 | Loss: 5.7428 | LR: 2.80e-04 | 2444.8ms/step | 8.2 samples/s +Step 430/44000 | Loss: 5.7307 | LR: 2.87e-04 | 2406.4ms/step | 8.3 samples/s +Step 440/44000 | Loss: 5.6667 | LR: 2.93e-04 | 2405.2ms/step | 8.3 samples/s +Step 450/44000 | Loss: 5.6919 | LR: 3.00e-04 | 2439.5ms/step | 8.2 samples/s +Step 460/44000 | Loss: 5.6364 | LR: 3.07e-04 | 2435.7ms/step | 8.2 samples/s +Step 470/44000 | Loss: 5.5693 | LR: 3.13e-04 | 2542.2ms/step | 7.9 samples/s +Step 480/44000 | Loss: 5.5475 | LR: 3.20e-04 | 2507.7ms/step | 8.0 samples/s +Step 490/44000 | Loss: 5.5191 | LR: 3.27e-04 | 2533.3ms/step | 7.9 samples/s +Step 500/44000 | Loss: 5.4711 | LR: 3.33e-04 | 2501.9ms/step | 8.0 samples/s +Validation at step 500: Loss = 5.5048, Perplexity = 245.86 +Step 510/44000 | Loss: 5.4648 | LR: 3.40e-04 | 4743.1ms/step | 4.2 samples/s +Step 520/44000 | Loss: 5.4457 | LR: 3.47e-04 | 2533.4ms/step | 7.9 samples/s +Step 530/44000 | Loss: 5.3875 | LR: 3.53e-04 | 2523.4ms/step | 7.9 samples/s +Step 540/44000 | Loss: 5.3525 | LR: 3.60e-04 | 2536.0ms/step | 7.9 samples/s +Step 550/44000 | Loss: 5.3090 | LR: 3.67e-04 | 2561.9ms/step | 7.8 samples/s +Step 560/44000 | Loss: 5.2978 | LR: 3.73e-04 | 2523.1ms/step | 7.9 samples/s +Step 570/44000 | Loss: 5.2500 | LR: 3.80e-04 | 2517.2ms/step | 7.9 samples/s +Step 580/44000 | Loss: 5.2646 | LR: 3.87e-04 | 2527.8ms/step | 7.9 samples/s +Step 590/44000 | Loss: 5.2121 | LR: 3.93e-04 | 2538.6ms/step | 7.9 samples/s +Step 600/44000 | Loss: 5.1915 | LR: 4.00e-04 | 2523.7ms/step | 7.9 samples/s +Step 610/44000 | Loss: 5.1525 | LR: 4.07e-04 | 2346.3ms/step | 8.5 samples/s +Step 620/44000 | Loss: 5.1211 | LR: 4.13e-04 | 2046.7ms/step | 9.8 samples/s +Step 630/44000 | Loss: 5.0983 | LR: 4.20e-04 | 2546.1ms/step | 7.9 samples/s +Step 640/44000 | Loss: 5.0609 | LR: 4.27e-04 | 2587.3ms/step | 7.7 samples/s +Step 650/44000 | Loss: 5.0836 | LR: 4.33e-04 | 2674.4ms/step | 7.5 samples/s +Step 660/44000 | Loss: 5.0064 | LR: 4.40e-04 | 2637.4ms/step | 7.6 samples/s +Step 670/44000 | Loss: 5.0166 | LR: 4.47e-04 | 2601.5ms/step | 7.7 samples/s +Step 680/44000 | Loss: 4.9245 | LR: 4.53e-04 | 2620.9ms/step | 7.6 samples/s +Step 690/44000 | Loss: 4.9045 | LR: 4.60e-04 | 2551.5ms/step | 7.8 samples/s +Step 700/44000 | Loss: 4.8849 | LR: 4.67e-04 | 2523.8ms/step | 7.9 samples/s +Step 710/44000 | Loss: 4.8668 | LR: 4.73e-04 | 2528.5ms/step | 7.9 samples/s +Step 720/44000 | Loss: 4.8368 | LR: 4.80e-04 | 2507.4ms/step | 8.0 samples/s +Step 730/44000 | Loss: 4.8706 | LR: 4.87e-04 | 2528.6ms/step | 7.9 samples/s +Step 740/44000 | Loss: 4.8009 | LR: 4.93e-04 | 2548.9ms/step | 7.8 samples/s +Step 750/44000 | Loss: 4.7807 | LR: 5.00e-04 | 2486.4ms/step | 8.0 samples/s +Validation at step 750: Loss = 4.7719, Perplexity = 118.14 +Step 760/44000 | Loss: 4.7397 | LR: 5.00e-04 | 4676.8ms/step | 4.3 samples/s +Step 770/44000 | Loss: 4.6343 | LR: 5.00e-04 | 2531.7ms/step | 7.9 samples/s +Step 780/44000 | Loss: 4.6920 | LR: 5.00e-04 | 2518.8ms/step | 7.9 samples/s +Step 790/44000 | Loss: 4.6511 | LR: 5.00e-04 | 2553.2ms/step | 7.8 samples/s +Step 800/44000 | Loss: 4.6333 | LR: 5.00e-04 | 2522.3ms/step | 7.9 samples/s +Step 810/44000 | Loss: 4.6505 | LR: 5.00e-04 | 2455.6ms/step | 8.1 samples/s +Step 820/44000 | Loss: 4.6258 | LR: 5.00e-04 | 2526.8ms/step | 7.9 samples/s +Step 830/44000 | Loss: 4.5854 | LR: 5.00e-04 | 2493.4ms/step | 8.0 samples/s +Step 840/44000 | Loss: 4.5620 | LR: 5.00e-04 | 2547.2ms/step | 7.9 samples/s +Step 850/44000 | Loss: 4.5303 | LR: 5.00e-04 | 2518.0ms/step | 7.9 samples/s +Step 860/44000 | Loss: 4.5187 | LR: 5.00e-04 | 2485.8ms/step | 8.0 samples/s +Step 870/44000 | Loss: 4.4783 | LR: 5.00e-04 | 2523.1ms/step | 7.9 samples/s +Step 880/44000 | Loss: 4.4630 | LR: 5.00e-04 | 2466.8ms/step | 8.1 samples/s +Step 890/44000 | Loss: 4.4883 | LR: 5.00e-04 | 2528.2ms/step | 7.9 samples/s +Step 900/44000 | Loss: 4.4258 | LR: 5.00e-04 | 2522.9ms/step | 7.9 samples/s +Step 910/44000 | Loss: 4.4270 | LR: 5.00e-04 | 2475.8ms/step | 8.1 samples/s +Step 920/44000 | Loss: 4.4161 | LR: 5.00e-04 | 2527.7ms/step | 7.9 samples/s +Step 930/44000 | Loss: 4.4374 | LR: 5.00e-04 | 2559.5ms/step | 7.8 samples/s +Step 940/44000 | Loss: 4.3948 | LR: 5.00e-04 | 2506.9ms/step | 8.0 samples/s +Step 950/44000 | Loss: 4.3828 | LR: 5.00e-04 | 2482.0ms/step | 8.1 samples/s +Step 960/44000 | Loss: 4.3286 | LR: 5.00e-04 | 2554.8ms/step | 7.8 samples/s +Step 970/44000 | Loss: 4.3647 | LR: 5.00e-04 | 2559.6ms/step | 7.8 samples/s +Step 980/44000 | Loss: 4.3233 | LR: 5.00e-04 | 2481.6ms/step | 8.1 samples/s +Step 990/44000 | Loss: 4.3162 | LR: 5.00e-04 | 2518.5ms/step | 7.9 samples/s +Step 1000/44000 | Loss: 4.3012 | LR: 5.00e-04 | 2543.8ms/step | 7.9 samples/s +Validation at step 1000: Loss = 4.3375, Perplexity = 76.51 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-1000 +Step 1010/44000 | Loss: 4.3567 | LR: 5.00e-04 | 4572.5ms/step | 4.4 samples/s +Step 1020/44000 | Loss: 4.2582 | LR: 5.00e-04 | 2370.5ms/step | 8.4 samples/s +Step 1030/44000 | Loss: 4.3029 | LR: 5.00e-04 | 2522.2ms/step | 7.9 samples/s +Step 1040/44000 | Loss: 4.2887 | LR: 5.00e-04 | 2548.1ms/step | 7.8 samples/s +Step 1050/44000 | Loss: 4.2658 | LR: 5.00e-04 | 2527.7ms/step | 7.9 samples/s +Step 1060/44000 | Loss: 4.2485 | LR: 5.00e-04 | 2485.5ms/step | 8.0 samples/s +Step 1070/44000 | Loss: 4.2668 | LR: 5.00e-04 | 2497.0ms/step | 8.0 samples/s +Step 1080/44000 | Loss: 4.2193 | LR: 5.00e-04 | 2512.1ms/step | 8.0 samples/s +Step 1090/44000 | Loss: 4.1602 | LR: 5.00e-04 | 2521.7ms/step | 7.9 samples/s +Step 1100/44000 | Loss: 4.1640 | LR: 5.00e-04 | 2507.3ms/step | 8.0 samples/s +Step 1110/44000 | Loss: 4.1705 | LR: 5.00e-04 | 2527.0ms/step | 7.9 samples/s +Step 1120/44000 | Loss: 4.1991 | LR: 5.00e-04 | 2512.0ms/step | 8.0 samples/s +Step 1130/44000 | Loss: 4.1539 | LR: 5.00e-04 | 2559.0ms/step | 7.8 samples/s +Step 1140/44000 | Loss: 4.1459 | LR: 5.00e-04 | 2506.6ms/step | 8.0 samples/s +Step 1150/44000 | Loss: 4.1805 | LR: 5.00e-04 | 2521.0ms/step | 7.9 samples/s +Step 1160/44000 | Loss: 4.1743 | LR: 5.00e-04 | 2506.2ms/step | 8.0 samples/s +Step 1170/44000 | Loss: 4.1352 | LR: 5.00e-04 | 2521.9ms/step | 7.9 samples/s +Step 1180/44000 | Loss: 4.1315 | LR: 5.00e-04 | 2511.8ms/step | 8.0 samples/s +Step 1190/44000 | Loss: 4.1459 | LR: 5.00e-04 | 2511.9ms/step | 8.0 samples/s +Step 1200/44000 | Loss: 4.0945 | LR: 5.00e-04 | 2532.3ms/step | 7.9 samples/s +Step 1210/44000 | Loss: 4.1062 | LR: 5.00e-04 | 2464.6ms/step | 8.1 samples/s +Step 1220/44000 | Loss: 4.0771 | LR: 5.00e-04 | 2502.8ms/step | 8.0 samples/s +Step 1230/44000 | Loss: 4.1092 | LR: 5.00e-04 | 2552.8ms/step | 7.8 samples/s +Step 1240/44000 | Loss: 4.0700 | LR: 5.00e-04 | 2532.7ms/step | 7.9 samples/s +Step 1250/44000 | Loss: 4.0712 | LR: 5.00e-04 | 2491.6ms/step | 8.0 samples/s +Validation at step 1250: Loss = 4.0921, Perplexity = 59.87 +Step 1260/44000 | Loss: 4.0830 | LR: 5.00e-04 | 4778.5ms/step | 4.2 samples/s +Step 1270/44000 | Loss: 4.0647 | LR: 5.00e-04 | 2532.8ms/step | 7.9 samples/s +Step 1280/44000 | Loss: 4.1629 | LR: 5.00e-04 | 2514.9ms/step | 8.0 samples/s +Step 1290/44000 | Loss: 4.0667 | LR: 5.00e-04 | 2472.5ms/step | 8.1 samples/s +Step 1300/44000 | Loss: 4.0000 | LR: 5.00e-04 | 2544.2ms/step | 7.9 samples/s +Step 1310/44000 | Loss: 4.0181 | LR: 5.00e-04 | 2538.1ms/step | 7.9 samples/s +Step 1320/44000 | Loss: 4.0572 | LR: 5.00e-04 | 2486.0ms/step | 8.0 samples/s +Step 1330/44000 | Loss: 4.0164 | LR: 5.00e-04 | 2532.1ms/step | 7.9 samples/s +Step 1340/44000 | Loss: 4.0268 | LR: 5.00e-04 | 2491.5ms/step | 8.0 samples/s +Step 1350/44000 | Loss: 4.0100 | LR: 5.00e-04 | 2563.7ms/step | 7.8 samples/s +Step 1360/44000 | Loss: 3.9949 | LR: 5.00e-04 | 2428.7ms/step | 8.2 samples/s +Step 1370/44000 | Loss: 3.9870 | LR: 5.00e-04 | 2390.2ms/step | 8.4 samples/s +Step 1380/44000 | Loss: 4.0070 | LR: 5.00e-04 | 2420.6ms/step | 8.3 samples/s +Step 1390/44000 | Loss: 4.0021 | LR: 5.00e-04 | 2403.2ms/step | 8.3 samples/s +Step 1400/44000 | Loss: 3.9443 | LR: 5.00e-04 | 2402.3ms/step | 8.3 samples/s +Step 1410/44000 | Loss: 3.9955 | LR: 5.00e-04 | 2516.6ms/step | 7.9 samples/s +Step 1420/44000 | Loss: 4.0165 | LR: 5.00e-04 | 2373.4ms/step | 8.4 samples/s +Step 1430/44000 | Loss: 3.9473 | LR: 5.00e-04 | 2598.3ms/step | 7.7 samples/s +Step 1440/44000 | Loss: 3.9776 | LR: 5.00e-04 | 2636.8ms/step | 7.6 samples/s +Step 1450/44000 | Loss: 3.9253 | LR: 5.00e-04 | 2632.3ms/step | 7.6 samples/s +Step 1460/44000 | Loss: 3.9311 | LR: 5.00e-04 | 2553.1ms/step | 7.8 samples/s +Step 1470/44000 | Loss: 3.9328 | LR: 5.00e-04 | 2501.3ms/step | 8.0 samples/s +Step 1480/44000 | Loss: 3.9132 | LR: 5.00e-04 | 2537.6ms/step | 7.9 samples/s +Step 1490/44000 | Loss: 3.9113 | LR: 5.00e-04 | 2496.6ms/step | 8.0 samples/s +Step 1500/44000 | Loss: 3.9408 | LR: 5.00e-04 | 2559.1ms/step | 7.8 samples/s +Validation at step 1500: Loss = 3.9451, Perplexity = 51.68 +Step 1510/44000 | Loss: 3.9000 | LR: 5.00e-04 | 4721.9ms/step | 4.2 samples/s +Step 1520/44000 | Loss: 3.9255 | LR: 5.00e-04 | 2537.6ms/step | 7.9 samples/s +Step 1530/44000 | Loss: 3.8613 | LR: 5.00e-04 | 2542.9ms/step | 7.9 samples/s +Step 1540/44000 | Loss: 3.9199 | LR: 5.00e-04 | 2491.7ms/step | 8.0 samples/s +Step 1550/44000 | Loss: 3.8559 | LR: 5.00e-04 | 2512.7ms/step | 8.0 samples/s +Step 1560/44000 | Loss: 3.8988 | LR: 5.00e-04 | 2522.9ms/step | 7.9 samples/s +Step 1570/44000 | Loss: 3.9029 | LR: 5.00e-04 | 2539.2ms/step | 7.9 samples/s +Step 1580/44000 | Loss: 3.8734 | LR: 5.00e-04 | 2511.5ms/step | 8.0 samples/s +Step 1590/44000 | Loss: 3.8796 | LR: 5.00e-04 | 2519.2ms/step | 7.9 samples/s +Step 1600/44000 | Loss: 3.8857 | LR: 5.00e-04 | 2521.2ms/step | 7.9 samples/s +Step 1610/44000 | Loss: 3.8721 | LR: 5.00e-04 | 2521.8ms/step | 7.9 samples/s +Step 1620/44000 | Loss: 3.8830 | LR: 5.00e-04 | 2525.2ms/step | 7.9 samples/s +Step 1630/44000 | Loss: 3.8327 | LR: 4.99e-04 | 2478.7ms/step | 8.1 samples/s +Step 1640/44000 | Loss: 3.8868 | LR: 4.99e-04 | 2501.1ms/step | 8.0 samples/s +Step 1650/44000 | Loss: 3.8754 | LR: 4.99e-04 | 2532.2ms/step | 7.9 samples/s +Step 1660/44000 | Loss: 3.8168 | LR: 4.99e-04 | 2523.2ms/step | 7.9 samples/s +Step 1670/44000 | Loss: 3.8502 | LR: 4.99e-04 | 2522.5ms/step | 7.9 samples/s +Step 1680/44000 | Loss: 3.8138 | LR: 4.99e-04 | 2497.1ms/step | 8.0 samples/s +Step 1690/44000 | Loss: 3.8840 | LR: 4.99e-04 | 2486.3ms/step | 8.0 samples/s +Step 1700/44000 | Loss: 3.8306 | LR: 4.99e-04 | 2512.8ms/step | 8.0 samples/s +Step 1710/44000 | Loss: 3.8075 | LR: 4.99e-04 | 2507.5ms/step | 8.0 samples/s +Step 1720/44000 | Loss: 3.7882 | LR: 4.99e-04 | 2544.6ms/step | 7.9 samples/s +Step 1730/44000 | Loss: 3.8014 | LR: 4.99e-04 | 2599.9ms/step | 7.7 samples/s +Step 1740/44000 | Loss: 3.7978 | LR: 4.99e-04 | 2613.8ms/step | 7.7 samples/s +Step 1750/44000 | Loss: 3.8324 | LR: 4.99e-04 | 2616.3ms/step | 7.6 samples/s +Validation at step 1750: Loss = 3.8320, Perplexity = 46.15 +Step 1760/44000 | Loss: 3.7842 | LR: 4.99e-04 | 5087.1ms/step | 3.9 samples/s +Step 1770/44000 | Loss: 3.8069 | LR: 4.99e-04 | 2537.6ms/step | 7.9 samples/s +Step 1780/44000 | Loss: 3.7610 | LR: 4.99e-04 | 2548.7ms/step | 7.8 samples/s +Step 1790/44000 | Loss: 3.8393 | LR: 4.99e-04 | 2543.6ms/step | 7.9 samples/s +Step 1800/44000 | Loss: 3.7945 | LR: 4.99e-04 | 2522.2ms/step | 7.9 samples/s +Step 1810/44000 | Loss: 3.8164 | LR: 4.99e-04 | 2470.7ms/step | 8.1 samples/s +Step 1820/44000 | Loss: 3.7909 | LR: 4.99e-04 | 2542.2ms/step | 7.9 samples/s +Step 1830/44000 | Loss: 3.7766 | LR: 4.99e-04 | 2532.4ms/step | 7.9 samples/s +Step 1840/44000 | Loss: 3.7740 | LR: 4.99e-04 | 2531.8ms/step | 7.9 samples/s +Step 1850/44000 | Loss: 3.8279 | LR: 4.99e-04 | 2495.8ms/step | 8.0 samples/s +Step 1860/44000 | Loss: 3.8158 | LR: 4.99e-04 | 2511.3ms/step | 8.0 samples/s +Step 1870/44000 | Loss: 3.7612 | LR: 4.99e-04 | 2490.6ms/step | 8.0 samples/s +Step 1880/44000 | Loss: 3.7240 | LR: 4.99e-04 | 2508.5ms/step | 8.0 samples/s +Step 1890/44000 | Loss: 3.7587 | LR: 4.99e-04 | 2525.3ms/step | 7.9 samples/s +Step 1900/44000 | Loss: 3.7457 | LR: 4.99e-04 | 2505.7ms/step | 8.0 samples/s +Step 1910/44000 | Loss: 3.7423 | LR: 4.99e-04 | 2532.6ms/step | 7.9 samples/s +Step 1920/44000 | Loss: 3.7577 | LR: 4.99e-04 | 2588.0ms/step | 7.7 samples/s +Step 1930/44000 | Loss: 3.7200 | LR: 4.99e-04 | 2595.7ms/step | 7.7 samples/s +Step 1940/44000 | Loss: 3.7671 | LR: 4.99e-04 | 2601.2ms/step | 7.7 samples/s +Step 1950/44000 | Loss: 3.7427 | LR: 4.99e-04 | 2567.9ms/step | 7.8 samples/s +Step 1960/44000 | Loss: 3.7543 | LR: 4.99e-04 | 2599.8ms/step | 7.7 samples/s +Step 1970/44000 | Loss: 3.7663 | LR: 4.99e-04 | 2595.6ms/step | 7.7 samples/s +Step 1980/44000 | Loss: 3.7430 | LR: 4.99e-04 | 2561.7ms/step | 7.8 samples/s +Step 1990/44000 | Loss: 3.7471 | LR: 4.99e-04 | 2463.6ms/step | 8.1 samples/s +Step 2000/44000 | Loss: 3.7206 | LR: 4.99e-04 | 2532.1ms/step | 7.9 samples/s +Validation at step 2000: Loss = 3.7481, Perplexity = 42.44 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-2000 +Step 2010/44000 | Loss: 3.7726 | LR: 4.99e-04 | 4657.3ms/step | 4.3 samples/s +Step 2020/44000 | Loss: 3.7298 | LR: 4.99e-04 | 1904.0ms/step | 10.5 samples/s +Step 2030/44000 | Loss: 3.7114 | LR: 4.99e-04 | 2297.3ms/step | 8.7 samples/s +Step 2040/44000 | Loss: 3.7263 | LR: 4.99e-04 | 2521.6ms/step | 7.9 samples/s +Step 2050/44000 | Loss: 3.7795 | LR: 4.99e-04 | 2485.2ms/step | 8.0 samples/s +Step 2060/44000 | Loss: 3.7298 | LR: 4.99e-04 | 2525.9ms/step | 7.9 samples/s +Step 2070/44000 | Loss: 3.6776 | LR: 4.99e-04 | 2513.3ms/step | 8.0 samples/s +Step 2080/44000 | Loss: 3.7066 | LR: 4.99e-04 | 2517.5ms/step | 7.9 samples/s +Step 2090/44000 | Loss: 3.7363 | LR: 4.99e-04 | 2453.7ms/step | 8.2 samples/s +Step 2100/44000 | Loss: 3.7063 | LR: 4.99e-04 | 2521.5ms/step | 7.9 samples/s +Step 2110/44000 | Loss: 3.6918 | LR: 4.99e-04 | 2516.1ms/step | 7.9 samples/s +Step 2120/44000 | Loss: 3.6744 | LR: 4.99e-04 | 2531.6ms/step | 7.9 samples/s +Step 2130/44000 | Loss: 3.7254 | LR: 4.99e-04 | 2480.2ms/step | 8.1 samples/s +Step 2140/44000 | Loss: 3.6828 | LR: 4.99e-04 | 2521.9ms/step | 7.9 samples/s +Step 2150/44000 | Loss: 3.6420 | LR: 4.99e-04 | 2521.7ms/step | 7.9 samples/s +Step 2160/44000 | Loss: 3.7262 | LR: 4.99e-04 | 2480.7ms/step | 8.1 samples/s +Step 2170/44000 | Loss: 3.7058 | LR: 4.99e-04 | 2506.7ms/step | 8.0 samples/s +Step 2180/44000 | Loss: 3.6527 | LR: 4.99e-04 | 2495.8ms/step | 8.0 samples/s +Step 2190/44000 | Loss: 3.6989 | LR: 4.99e-04 | 2527.6ms/step | 7.9 samples/s +Step 2200/44000 | Loss: 3.6796 | LR: 4.99e-04 | 2506.7ms/step | 8.0 samples/s +Step 2210/44000 | Loss: 3.6262 | LR: 4.99e-04 | 2464.9ms/step | 8.1 samples/s +Step 2220/44000 | Loss: 3.6517 | LR: 4.99e-04 | 2204.4ms/step | 9.1 samples/s +Step 2230/44000 | Loss: 3.6466 | LR: 4.99e-04 | 2511.5ms/step | 8.0 samples/s +Step 2240/44000 | Loss: 3.6723 | LR: 4.99e-04 | 2523.0ms/step | 7.9 samples/s +Step 2250/44000 | Loss: 3.6771 | LR: 4.99e-04 | 2496.8ms/step | 8.0 samples/s +Validation at step 2250: Loss = 3.6829, Perplexity = 39.76 +Step 2260/44000 | Loss: 3.6649 | LR: 4.98e-04 | 4648.4ms/step | 4.3 samples/s +Step 2270/44000 | Loss: 3.6824 | LR: 4.98e-04 | 2408.7ms/step | 8.3 samples/s +Step 2280/44000 | Loss: 3.6416 | LR: 4.98e-04 | 2585.4ms/step | 7.7 samples/s +Step 2290/44000 | Loss: 3.6572 | LR: 4.98e-04 | 2653.3ms/step | 7.5 samples/s +Step 2300/44000 | Loss: 3.6531 | LR: 4.98e-04 | 2927.5ms/step | 6.8 samples/s +Step 2310/44000 | Loss: 3.6445 | LR: 4.98e-04 | 2331.2ms/step | 8.6 samples/s +Step 2320/44000 | Loss: 3.6712 | LR: 4.98e-04 | 2330.1ms/step | 8.6 samples/s +Step 2330/44000 | Loss: 3.6223 | LR: 4.98e-04 | 2402.5ms/step | 8.3 samples/s +Step 2340/44000 | Loss: 3.6395 | LR: 4.98e-04 | 2408.7ms/step | 8.3 samples/s +Step 2350/44000 | Loss: 3.6160 | LR: 4.98e-04 | 2424.1ms/step | 8.3 samples/s +Step 2360/44000 | Loss: 3.6273 | LR: 4.98e-04 | 2507.0ms/step | 8.0 samples/s +Step 2370/44000 | Loss: 3.6538 | LR: 4.98e-04 | 2551.5ms/step | 7.8 samples/s +Step 2380/44000 | Loss: 3.6572 | LR: 4.98e-04 | 2558.8ms/step | 7.8 samples/s +Step 2390/44000 | Loss: 3.6684 | LR: 4.98e-04 | 2593.3ms/step | 7.7 samples/s +Step 2400/44000 | Loss: 3.6747 | LR: 4.98e-04 | 2584.9ms/step | 7.7 samples/s +Step 2410/44000 | Loss: 3.6339 | LR: 4.98e-04 | 2563.6ms/step | 7.8 samples/s +Step 2420/44000 | Loss: 3.6107 | LR: 4.98e-04 | 2575.0ms/step | 7.8 samples/s +Step 2430/44000 | Loss: 3.6498 | LR: 4.98e-04 | 2595.6ms/step | 7.7 samples/s +Step 2440/44000 | Loss: 3.6367 | LR: 4.98e-04 | 2483.8ms/step | 8.1 samples/s +Step 2450/44000 | Loss: 3.6438 | LR: 4.98e-04 | 2504.0ms/step | 8.0 samples/s +Step 2460/44000 | Loss: 3.6008 | LR: 4.98e-04 | 2487.6ms/step | 8.0 samples/s +Step 2470/44000 | Loss: 3.5983 | LR: 4.98e-04 | 2489.3ms/step | 8.0 samples/s +Step 2480/44000 | Loss: 3.6764 | LR: 4.98e-04 | 2518.4ms/step | 7.9 samples/s +Step 2490/44000 | Loss: 3.5848 | LR: 4.98e-04 | 2563.8ms/step | 7.8 samples/s +Step 2500/44000 | Loss: 3.6304 | LR: 4.98e-04 | 2533.0ms/step | 7.9 samples/s +Validation at step 2500: Loss = 3.6333, Perplexity = 37.84 +Step 2510/44000 | Loss: 3.6068 | LR: 4.98e-04 | 4670.3ms/step | 4.3 samples/s +Step 2520/44000 | Loss: 3.6115 | LR: 4.98e-04 | 2480.4ms/step | 8.1 samples/s +Step 2530/44000 | Loss: 3.6159 | LR: 4.98e-04 | 2502.1ms/step | 8.0 samples/s +Step 2540/44000 | Loss: 3.5845 | LR: 4.98e-04 | 2515.0ms/step | 8.0 samples/s +Step 2550/44000 | Loss: 3.5976 | LR: 4.98e-04 | 2503.8ms/step | 8.0 samples/s +Step 2560/44000 | Loss: 3.5933 | LR: 4.98e-04 | 2522.5ms/step | 7.9 samples/s +Step 2570/44000 | Loss: 3.5880 | LR: 4.98e-04 | 2517.5ms/step | 7.9 samples/s +Step 2580/44000 | Loss: 3.5942 | LR: 4.98e-04 | 2470.4ms/step | 8.1 samples/s +Step 2590/44000 | Loss: 3.5598 | LR: 4.98e-04 | 2507.2ms/step | 8.0 samples/s +Step 2600/44000 | Loss: 3.5979 | LR: 4.98e-04 | 2517.1ms/step | 7.9 samples/s +Step 2610/44000 | Loss: 3.5537 | LR: 4.98e-04 | 2485.9ms/step | 8.0 samples/s +Step 2620/44000 | Loss: 3.5698 | LR: 4.98e-04 | 2480.5ms/step | 8.1 samples/s +Step 2630/44000 | Loss: 3.5581 | LR: 4.98e-04 | 2480.7ms/step | 8.1 samples/s +Step 2640/44000 | Loss: 3.6289 | LR: 4.98e-04 | 2552.5ms/step | 7.8 samples/s +Step 2650/44000 | Loss: 3.6052 | LR: 4.98e-04 | 2495.4ms/step | 8.0 samples/s +Step 2660/44000 | Loss: 3.5830 | LR: 4.98e-04 | 2518.1ms/step | 7.9 samples/s +Step 2670/44000 | Loss: 3.5426 | LR: 4.98e-04 | 2516.9ms/step | 7.9 samples/s +Step 2680/44000 | Loss: 3.5503 | LR: 4.98e-04 | 2481.0ms/step | 8.1 samples/s +Step 2690/44000 | Loss: 3.5423 | LR: 4.98e-04 | 2638.1ms/step | 7.6 samples/s +Step 2700/44000 | Loss: 3.5880 | LR: 4.97e-04 | 2577.8ms/step | 7.8 samples/s +Step 2710/44000 | Loss: 3.6001 | LR: 4.97e-04 | 2574.3ms/step | 7.8 samples/s +Step 2720/44000 | Loss: 3.5693 | LR: 4.97e-04 | 2611.3ms/step | 7.7 samples/s +Step 2730/44000 | Loss: 3.6006 | LR: 4.97e-04 | 2589.2ms/step | 7.7 samples/s +Step 2740/44000 | Loss: 3.5733 | LR: 4.97e-04 | 2591.7ms/step | 7.7 samples/s +Step 2750/44000 | Loss: 3.5389 | LR: 4.97e-04 | 2631.6ms/step | 7.6 samples/s +Validation at step 2750: Loss = 3.5882, Perplexity = 36.17 +Step 2760/44000 | Loss: 3.5736 | LR: 4.97e-04 | 4774.8ms/step | 4.2 samples/s +Step 2770/44000 | Loss: 3.5881 | LR: 4.97e-04 | 2491.6ms/step | 8.0 samples/s +Step 2780/44000 | Loss: 3.5423 | LR: 4.97e-04 | 2517.6ms/step | 7.9 samples/s +Step 2790/44000 | Loss: 3.5869 | LR: 4.97e-04 | 2470.6ms/step | 8.1 samples/s +Step 2800/44000 | Loss: 3.5591 | LR: 4.97e-04 | 2481.1ms/step | 8.1 samples/s +Step 2810/44000 | Loss: 3.5860 | LR: 4.97e-04 | 2515.2ms/step | 8.0 samples/s +Step 2820/44000 | Loss: 3.5853 | LR: 4.97e-04 | 2488.1ms/step | 8.0 samples/s +Step 2830/44000 | Loss: 3.5660 | LR: 4.97e-04 | 2496.3ms/step | 8.0 samples/s +Step 2840/44000 | Loss: 3.5192 | LR: 4.97e-04 | 2527.6ms/step | 7.9 samples/s +Step 2850/44000 | Loss: 3.4746 | LR: 4.97e-04 | 2527.6ms/step | 7.9 samples/s +Step 2860/44000 | Loss: 3.5331 | LR: 4.97e-04 | 2532.9ms/step | 7.9 samples/s +Step 2870/44000 | Loss: 3.5391 | LR: 4.97e-04 | 2497.2ms/step | 8.0 samples/s +Step 2880/44000 | Loss: 3.5419 | LR: 4.97e-04 | 2486.4ms/step | 8.0 samples/s +Step 2890/44000 | Loss: 3.5856 | LR: 4.97e-04 | 2500.9ms/step | 8.0 samples/s +Step 2900/44000 | Loss: 3.5668 | LR: 4.97e-04 | 2538.6ms/step | 7.9 samples/s +Step 2910/44000 | Loss: 3.5654 | LR: 4.97e-04 | 2517.9ms/step | 7.9 samples/s +Step 2920/44000 | Loss: 3.5098 | LR: 4.97e-04 | 2511.7ms/step | 8.0 samples/s +Step 2930/44000 | Loss: 3.5448 | LR: 4.97e-04 | 2502.3ms/step | 8.0 samples/s +Step 2940/44000 | Loss: 3.5042 | LR: 4.97e-04 | 2533.3ms/step | 7.9 samples/s +Step 2950/44000 | Loss: 3.5118 | LR: 4.97e-04 | 2516.9ms/step | 7.9 samples/s +Step 2960/44000 | Loss: 3.5339 | LR: 4.97e-04 | 2537.3ms/step | 7.9 samples/s +Step 2970/44000 | Loss: 3.5533 | LR: 4.97e-04 | 2585.3ms/step | 7.7 samples/s +Step 2980/44000 | Loss: 3.5101 | LR: 4.97e-04 | 2583.2ms/step | 7.7 samples/s +Step 2990/44000 | Loss: 3.5029 | LR: 4.97e-04 | 2612.6ms/step | 7.7 samples/s +Step 3000/44000 | Loss: 3.4873 | LR: 4.97e-04 | 2574.4ms/step | 7.8 samples/s +Validation at step 3000: Loss = 3.5502, Perplexity = 34.82 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-3000 +Step 3010/44000 | Loss: 3.5058 | LR: 4.97e-04 | 5179.3ms/step | 3.9 samples/s +Step 3020/44000 | Loss: 3.4894 | LR: 4.97e-04 | 1903.0ms/step | 10.5 samples/s +Step 3030/44000 | Loss: 3.4922 | LR: 4.97e-04 | 2415.0ms/step | 8.3 samples/s +Step 3040/44000 | Loss: 3.4874 | LR: 4.97e-04 | 2530.9ms/step | 7.9 samples/s +Step 3050/44000 | Loss: 3.4882 | LR: 4.97e-04 | 2483.6ms/step | 8.1 samples/s +Step 3060/44000 | Loss: 3.5106 | LR: 4.96e-04 | 2496.8ms/step | 8.0 samples/s +Step 3070/44000 | Loss: 3.5035 | LR: 4.96e-04 | 2492.5ms/step | 8.0 samples/s +Step 3080/44000 | Loss: 3.5348 | LR: 4.96e-04 | 2506.4ms/step | 8.0 samples/s +Step 3090/44000 | Loss: 3.5339 | LR: 4.96e-04 | 2496.3ms/step | 8.0 samples/s +Step 3100/44000 | Loss: 3.5121 | LR: 4.96e-04 | 2470.3ms/step | 8.1 samples/s +Step 3110/44000 | Loss: 3.5373 | LR: 4.96e-04 | 2481.3ms/step | 8.1 samples/s +Step 3120/44000 | Loss: 3.5261 | LR: 4.96e-04 | 2501.8ms/step | 8.0 samples/s +Step 3130/44000 | Loss: 3.4858 | LR: 4.96e-04 | 2490.1ms/step | 8.0 samples/s +Step 3140/44000 | Loss: 3.4857 | LR: 4.96e-04 | 2471.9ms/step | 8.1 samples/s +Step 3150/44000 | Loss: 3.5034 | LR: 4.96e-04 | 2475.6ms/step | 8.1 samples/s +Step 3160/44000 | Loss: 3.5097 | LR: 4.96e-04 | 2522.5ms/step | 7.9 samples/s +Step 3170/44000 | Loss: 3.5034 | LR: 4.96e-04 | 2518.4ms/step | 7.9 samples/s +Step 3180/44000 | Loss: 3.5265 | LR: 4.96e-04 | 2505.5ms/step | 8.0 samples/s +Step 3190/44000 | Loss: 3.4861 | LR: 4.96e-04 | 2547.7ms/step | 7.9 samples/s +Step 3200/44000 | Loss: 3.5117 | LR: 4.96e-04 | 2576.3ms/step | 7.8 samples/s +Step 3210/44000 | Loss: 3.5118 | LR: 4.96e-04 | 2599.7ms/step | 7.7 samples/s +Step 3220/44000 | Loss: 3.5374 | LR: 4.96e-04 | 2522.7ms/step | 7.9 samples/s +Step 3230/44000 | Loss: 3.4746 | LR: 4.96e-04 | 2606.0ms/step | 7.7 samples/s +Step 3240/44000 | Loss: 3.4900 | LR: 4.96e-04 | 2568.4ms/step | 7.8 samples/s +Step 3250/44000 | Loss: 3.4510 | LR: 4.96e-04 | 2573.8ms/step | 7.8 samples/s +Validation at step 3250: Loss = 3.5155, Perplexity = 33.63 +Step 3260/44000 | Loss: 3.4758 | LR: 4.96e-04 | 5066.5ms/step | 3.9 samples/s +Step 3270/44000 | Loss: 3.4887 | LR: 4.96e-04 | 3011.6ms/step | 6.6 samples/s +Step 3280/44000 | Loss: 3.4850 | LR: 4.96e-04 | 2594.9ms/step | 7.7 samples/s +Step 3290/44000 | Loss: 3.5132 | LR: 4.96e-04 | 2401.3ms/step | 8.3 samples/s +Step 3300/44000 | Loss: 3.5452 | LR: 4.96e-04 | 2403.4ms/step | 8.3 samples/s +Step 3310/44000 | Loss: 3.4983 | LR: 4.96e-04 | 2511.5ms/step | 8.0 samples/s +Step 3320/44000 | Loss: 3.5272 | LR: 4.96e-04 | 2464.8ms/step | 8.1 samples/s +Step 3330/44000 | Loss: 3.4372 | LR: 4.96e-04 | 2511.5ms/step | 8.0 samples/s +Step 3340/44000 | Loss: 3.4754 | LR: 4.96e-04 | 2465.2ms/step | 8.1 samples/s +Step 3350/44000 | Loss: 3.4875 | LR: 4.96e-04 | 2526.7ms/step | 7.9 samples/s +Step 3360/44000 | Loss: 3.5029 | LR: 4.96e-04 | 2491.0ms/step | 8.0 samples/s +Step 3370/44000 | Loss: 3.4742 | LR: 4.95e-04 | 2519.1ms/step | 7.9 samples/s +Step 3380/44000 | Loss: 3.5090 | LR: 4.95e-04 | 2520.0ms/step | 7.9 samples/s +Step 3390/44000 | Loss: 3.5172 | LR: 4.95e-04 | 2480.9ms/step | 8.1 samples/s +Step 3400/44000 | Loss: 3.4804 | LR: 4.95e-04 | 2491.3ms/step | 8.0 samples/s +Step 3410/44000 | Loss: 3.4977 | LR: 4.95e-04 | 2500.7ms/step | 8.0 samples/s +Step 3420/44000 | Loss: 3.4741 | LR: 4.95e-04 | 2470.0ms/step | 8.1 samples/s +Step 3430/44000 | Loss: 3.4511 | LR: 4.95e-04 | 2484.9ms/step | 8.0 samples/s +Step 3440/44000 | Loss: 3.4931 | LR: 4.95e-04 | 2575.0ms/step | 7.8 samples/s +Step 3450/44000 | Loss: 3.4938 | LR: 4.95e-04 | 2553.2ms/step | 7.8 samples/s +Step 3460/44000 | Loss: 3.4395 | LR: 4.95e-04 | 2580.2ms/step | 7.8 samples/s +Step 3470/44000 | Loss: 3.4584 | LR: 4.95e-04 | 2583.4ms/step | 7.7 samples/s +Step 3480/44000 | Loss: 3.4701 | LR: 4.95e-04 | 2594.7ms/step | 7.7 samples/s +Step 3490/44000 | Loss: 3.4789 | LR: 4.95e-04 | 2579.1ms/step | 7.8 samples/s +Step 3500/44000 | Loss: 3.4589 | LR: 4.95e-04 | 2605.2ms/step | 7.7 samples/s +Validation at step 3500: Loss = 3.4849, Perplexity = 32.62 +Step 3510/44000 | Loss: 3.4851 | LR: 4.95e-04 | 4699.6ms/step | 4.3 samples/s +Step 3520/44000 | Loss: 3.4384 | LR: 4.95e-04 | 2506.5ms/step | 8.0 samples/s +Step 3530/44000 | Loss: 3.5075 | LR: 4.95e-04 | 2516.4ms/step | 7.9 samples/s +Step 3540/44000 | Loss: 3.4376 | LR: 4.95e-04 | 2517.2ms/step | 7.9 samples/s +Step 3550/44000 | Loss: 3.4725 | LR: 4.95e-04 | 2532.6ms/step | 7.9 samples/s +Step 3560/44000 | Loss: 3.4120 | LR: 4.95e-04 | 2517.7ms/step | 7.9 samples/s +Step 3570/44000 | Loss: 3.4776 | LR: 4.95e-04 | 2500.7ms/step | 8.0 samples/s +Step 3580/44000 | Loss: 3.5226 | LR: 4.95e-04 | 2507.5ms/step | 8.0 samples/s +Step 3590/44000 | Loss: 3.4358 | LR: 4.95e-04 | 2510.7ms/step | 8.0 samples/s +Step 3600/44000 | Loss: 3.4026 | LR: 4.95e-04 | 2532.8ms/step | 7.9 samples/s +Step 3610/44000 | Loss: 3.4304 | LR: 4.95e-04 | 2512.6ms/step | 8.0 samples/s +Step 3620/44000 | Loss: 3.4336 | LR: 4.95e-04 | 2413.5ms/step | 8.3 samples/s +Step 3630/44000 | Loss: 3.4581 | LR: 4.95e-04 | 2491.2ms/step | 8.0 samples/s +Step 3640/44000 | Loss: 3.4984 | LR: 4.95e-04 | 2522.4ms/step | 7.9 samples/s +Step 3650/44000 | Loss: 3.4736 | LR: 4.94e-04 | 2497.0ms/step | 8.0 samples/s +Step 3660/44000 | Loss: 3.4570 | LR: 4.94e-04 | 2506.2ms/step | 8.0 samples/s +Step 3670/44000 | Loss: 3.4361 | LR: 4.94e-04 | 2463.8ms/step | 8.1 samples/s +Step 3680/44000 | Loss: 3.4191 | LR: 4.94e-04 | 2476.9ms/step | 8.1 samples/s +Step 3690/44000 | Loss: 3.4328 | LR: 4.94e-04 | 2491.1ms/step | 8.0 samples/s +Step 3700/44000 | Loss: 3.4182 | LR: 4.94e-04 | 2511.8ms/step | 8.0 samples/s +Step 3710/44000 | Loss: 3.4399 | LR: 4.94e-04 | 2506.0ms/step | 8.0 samples/s +Step 3720/44000 | Loss: 3.4409 | LR: 4.94e-04 | 2496.2ms/step | 8.0 samples/s +Step 3730/44000 | Loss: 3.4243 | LR: 4.94e-04 | 2552.9ms/step | 7.8 samples/s +Step 3740/44000 | Loss: 3.4318 | LR: 4.94e-04 | 2522.3ms/step | 7.9 samples/s +Step 3750/44000 | Loss: 3.4477 | LR: 4.94e-04 | 2490.7ms/step | 8.0 samples/s +Validation at step 3750: Loss = 3.4583, Perplexity = 31.76 +Step 3760/44000 | Loss: 3.4934 | LR: 4.94e-04 | 4693.0ms/step | 4.3 samples/s +Step 3770/44000 | Loss: 3.4216 | LR: 4.94e-04 | 2452.6ms/step | 8.2 samples/s +Step 3780/44000 | Loss: 3.4302 | LR: 4.94e-04 | 2460.7ms/step | 8.1 samples/s +Step 3790/44000 | Loss: 3.4360 | LR: 4.94e-04 | 2475.9ms/step | 8.1 samples/s +Step 3800/44000 | Loss: 3.4362 | LR: 4.94e-04 | 2501.3ms/step | 8.0 samples/s +Step 3810/44000 | Loss: 3.4433 | LR: 4.94e-04 | 2439.6ms/step | 8.2 samples/s +Step 3820/44000 | Loss: 3.4310 | LR: 4.94e-04 | 2256.9ms/step | 8.9 samples/s +Step 3830/44000 | Loss: 3.4346 | LR: 4.94e-04 | 1902.5ms/step | 10.5 samples/s +Step 3840/44000 | Loss: 3.4482 | LR: 4.94e-04 | 1906.0ms/step | 10.5 samples/s +Step 3850/44000 | Loss: 3.4473 | LR: 4.94e-04 | 1909.5ms/step | 10.5 samples/s +Step 3860/44000 | Loss: 3.4362 | LR: 4.94e-04 | 2398.2ms/step | 8.3 samples/s +Step 3870/44000 | Loss: 3.4404 | LR: 4.94e-04 | 2523.3ms/step | 7.9 samples/s +Step 3880/44000 | Loss: 3.4192 | LR: 4.94e-04 | 2585.5ms/step | 7.7 samples/s +Step 3890/44000 | Loss: 3.4435 | LR: 4.94e-04 | 2589.9ms/step | 7.7 samples/s +Step 3900/44000 | Loss: 3.4605 | LR: 4.93e-04 | 2568.8ms/step | 7.8 samples/s +Step 3910/44000 | Loss: 3.3595 | LR: 4.93e-04 | 2590.3ms/step | 7.7 samples/s +Step 3920/44000 | Loss: 3.4127 | LR: 4.93e-04 | 2538.8ms/step | 7.9 samples/s +Step 3930/44000 | Loss: 3.4523 | LR: 4.93e-04 | 2500.4ms/step | 8.0 samples/s +Step 3940/44000 | Loss: 3.3914 | LR: 4.93e-04 | 2475.9ms/step | 8.1 samples/s +Step 3950/44000 | Loss: 3.4069 | LR: 4.93e-04 | 2496.7ms/step | 8.0 samples/s +Step 3960/44000 | Loss: 3.3962 | LR: 4.93e-04 | 2504.3ms/step | 8.0 samples/s +Step 3970/44000 | Loss: 3.4561 | LR: 4.93e-04 | 2472.9ms/step | 8.1 samples/s +Step 3980/44000 | Loss: 3.4341 | LR: 4.93e-04 | 2486.4ms/step | 8.0 samples/s +Step 3990/44000 | Loss: 3.4048 | LR: 4.93e-04 | 2528.2ms/step | 7.9 samples/s +Step 4000/44000 | Loss: 3.4377 | LR: 4.93e-04 | 2538.7ms/step | 7.9 samples/s +Validation at step 4000: Loss = 3.4392, Perplexity = 31.16 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-4000 +Step 4010/44000 | Loss: 3.4096 | LR: 4.93e-04 | 4911.9ms/step | 4.1 samples/s +Step 4020/44000 | Loss: 3.4314 | LR: 4.93e-04 | 2512.6ms/step | 8.0 samples/s +Step 4030/44000 | Loss: 3.4181 | LR: 4.93e-04 | 2470.6ms/step | 8.1 samples/s +Step 4040/44000 | Loss: 3.4276 | LR: 4.93e-04 | 2502.2ms/step | 8.0 samples/s +Step 4050/44000 | Loss: 3.4522 | LR: 4.93e-04 | 2482.4ms/step | 8.1 samples/s +Step 4060/44000 | Loss: 3.4070 | LR: 4.93e-04 | 2506.4ms/step | 8.0 samples/s +Step 4070/44000 | Loss: 3.4114 | LR: 4.93e-04 | 2487.6ms/step | 8.0 samples/s +Step 4080/44000 | Loss: 3.4148 | LR: 4.93e-04 | 2495.9ms/step | 8.0 samples/s +Step 4090/44000 | Loss: 3.3521 | LR: 4.93e-04 | 2513.1ms/step | 8.0 samples/s +Step 4100/44000 | Loss: 3.4527 | LR: 4.93e-04 | 2486.4ms/step | 8.0 samples/s +Step 4110/44000 | Loss: 3.4267 | LR: 4.93e-04 | 2512.2ms/step | 8.0 samples/s +Step 4120/44000 | Loss: 3.4310 | LR: 4.93e-04 | 2489.5ms/step | 8.0 samples/s +Step 4130/44000 | Loss: 3.4202 | LR: 4.93e-04 | 2500.3ms/step | 8.0 samples/s +Step 4140/44000 | Loss: 3.4178 | LR: 4.92e-04 | 2495.1ms/step | 8.0 samples/s +Step 4150/44000 | Loss: 3.4007 | LR: 4.92e-04 | 2503.0ms/step | 8.0 samples/s +Step 4160/44000 | Loss: 3.3962 | LR: 4.92e-04 | 2497.2ms/step | 8.0 samples/s +Step 4170/44000 | Loss: 3.4074 | LR: 4.92e-04 | 2575.7ms/step | 7.8 samples/s +Step 4180/44000 | Loss: 3.3725 | LR: 4.92e-04 | 2579.4ms/step | 7.8 samples/s +Step 4190/44000 | Loss: 3.4274 | LR: 4.92e-04 | 2576.0ms/step | 7.8 samples/s +Step 4200/44000 | Loss: 3.4204 | LR: 4.92e-04 | 2591.2ms/step | 7.7 samples/s +Step 4210/44000 | Loss: 3.4118 | LR: 4.92e-04 | 2554.0ms/step | 7.8 samples/s +Step 4220/44000 | Loss: 3.3888 | LR: 4.92e-04 | 2570.3ms/step | 7.8 samples/s +Step 4230/44000 | Loss: 3.3653 | LR: 4.92e-04 | 2694.9ms/step | 7.4 samples/s +Step 4240/44000 | Loss: 3.4222 | LR: 4.92e-04 | 2695.3ms/step | 7.4 samples/s +Step 4250/44000 | Loss: 3.4092 | LR: 4.92e-04 | 2981.4ms/step | 6.7 samples/s +Validation at step 4250: Loss = 3.4146, Perplexity = 30.40 +Step 4260/44000 | Loss: 3.3830 | LR: 4.92e-04 | 4508.3ms/step | 4.4 samples/s +Step 4270/44000 | Loss: 3.3887 | LR: 4.92e-04 | 2433.9ms/step | 8.2 samples/s +Step 4280/44000 | Loss: 3.3588 | LR: 4.92e-04 | 2460.7ms/step | 8.1 samples/s +Step 4290/44000 | Loss: 3.3205 | LR: 4.92e-04 | 2518.0ms/step | 7.9 samples/s +Step 4300/44000 | Loss: 3.3986 | LR: 4.92e-04 | 2470.3ms/step | 8.1 samples/s +Step 4310/44000 | Loss: 3.3593 | LR: 4.92e-04 | 2496.1ms/step | 8.0 samples/s +Step 4320/44000 | Loss: 3.4067 | LR: 4.92e-04 | 2511.9ms/step | 8.0 samples/s +Step 4330/44000 | Loss: 3.3912 | LR: 4.92e-04 | 2548.4ms/step | 7.8 samples/s +Step 4340/44000 | Loss: 3.3884 | LR: 4.92e-04 | 2486.7ms/step | 8.0 samples/s +Step 4350/44000 | Loss: 3.3962 | LR: 4.92e-04 | 2486.7ms/step | 8.0 samples/s +Step 4360/44000 | Loss: 3.4015 | LR: 4.91e-04 | 2523.3ms/step | 7.9 samples/s +Step 4370/44000 | Loss: 3.3463 | LR: 4.91e-04 | 2454.9ms/step | 8.1 samples/s +Step 4380/44000 | Loss: 3.3932 | LR: 4.91e-04 | 2459.3ms/step | 8.1 samples/s +Step 4390/44000 | Loss: 3.4189 | LR: 4.91e-04 | 2508.1ms/step | 8.0 samples/s +Step 4400/44000 | Loss: 3.3812 | LR: 4.91e-04 | 2506.7ms/step | 8.0 samples/s +Step 4410/44000 | Loss: 3.4111 | LR: 4.91e-04 | 2460.1ms/step | 8.1 samples/s +Step 4420/44000 | Loss: 3.3717 | LR: 4.91e-04 | 2501.1ms/step | 8.0 samples/s +Step 4430/44000 | Loss: 3.3187 | LR: 4.91e-04 | 2533.3ms/step | 7.9 samples/s +Step 4440/44000 | Loss: 3.3806 | LR: 4.91e-04 | 2574.4ms/step | 7.8 samples/s +Step 4450/44000 | Loss: 3.3583 | LR: 4.91e-04 | 2542.9ms/step | 7.9 samples/s +Step 4460/44000 | Loss: 3.3660 | LR: 4.91e-04 | 2639.9ms/step | 7.6 samples/s +Step 4470/44000 | Loss: 3.3949 | LR: 4.91e-04 | 2653.3ms/step | 7.5 samples/s +Step 4480/44000 | Loss: 3.4458 | LR: 4.91e-04 | 3087.0ms/step | 6.5 samples/s +Step 4490/44000 | Loss: 3.3758 | LR: 4.91e-04 | 2579.7ms/step | 7.8 samples/s +Step 4500/44000 | Loss: 3.3658 | LR: 4.91e-04 | 2397.2ms/step | 8.3 samples/s +Validation at step 4500: Loss = 3.3955, Perplexity = 29.83 +Step 4510/44000 | Loss: 3.4001 | LR: 4.91e-04 | 4633.1ms/step | 4.3 samples/s +Step 4520/44000 | Loss: 3.3279 | LR: 4.91e-04 | 2523.3ms/step | 7.9 samples/s +Step 4530/44000 | Loss: 3.4090 | LR: 4.91e-04 | 2484.4ms/step | 8.1 samples/s +Step 4540/44000 | Loss: 3.3969 | LR: 4.91e-04 | 2459.8ms/step | 8.1 samples/s +Step 4550/44000 | Loss: 3.3582 | LR: 4.91e-04 | 2470.4ms/step | 8.1 samples/s +Step 4560/44000 | Loss: 3.4060 | LR: 4.90e-04 | 2479.9ms/step | 8.1 samples/s +Step 4570/44000 | Loss: 3.3834 | LR: 4.90e-04 | 2470.0ms/step | 8.1 samples/s +Step 4580/44000 | Loss: 3.4021 | LR: 4.90e-04 | 2469.7ms/step | 8.1 samples/s +Step 4590/44000 | Loss: 3.3886 | LR: 4.90e-04 | 2511.7ms/step | 8.0 samples/s +Step 4600/44000 | Loss: 3.4138 | LR: 4.90e-04 | 2456.2ms/step | 8.1 samples/s +Step 4610/44000 | Loss: 3.4025 | LR: 4.90e-04 | 2452.0ms/step | 8.2 samples/s +Step 4620/44000 | Loss: 3.3482 | LR: 4.90e-04 | 2470.7ms/step | 8.1 samples/s +Step 4630/44000 | Loss: 3.3583 | LR: 4.90e-04 | 2464.6ms/step | 8.1 samples/s +Step 4640/44000 | Loss: 3.3683 | LR: 4.90e-04 | 2293.1ms/step | 8.7 samples/s +Step 4650/44000 | Loss: 3.3779 | LR: 4.90e-04 | 2563.5ms/step | 7.8 samples/s +Step 4660/44000 | Loss: 3.4197 | LR: 4.90e-04 | 2584.3ms/step | 7.7 samples/s +Step 4670/44000 | Loss: 3.3700 | LR: 4.90e-04 | 2569.9ms/step | 7.8 samples/s +Step 4680/44000 | Loss: 3.3408 | LR: 4.90e-04 | 2756.4ms/step | 7.3 samples/s +Step 4690/44000 | Loss: 3.3520 | LR: 4.90e-04 | 2793.7ms/step | 7.2 samples/s +Step 4700/44000 | Loss: 3.4039 | LR: 4.90e-04 | 2964.8ms/step | 6.7 samples/s +Step 4710/44000 | Loss: 3.3530 | LR: 4.90e-04 | 2433.1ms/step | 8.2 samples/s +Step 4720/44000 | Loss: 3.3875 | LR: 4.90e-04 | 2407.3ms/step | 8.3 samples/s +Step 4730/44000 | Loss: 3.3552 | LR: 4.90e-04 | 2408.0ms/step | 8.3 samples/s +Step 4740/44000 | Loss: 3.3592 | LR: 4.90e-04 | 2505.8ms/step | 8.0 samples/s +Step 4750/44000 | Loss: 3.3341 | LR: 4.90e-04 | 2490.1ms/step | 8.0 samples/s +Validation at step 4750: Loss = 3.3795, Perplexity = 29.36 +Step 4760/44000 | Loss: 3.3639 | LR: 4.89e-04 | 4742.3ms/step | 4.2 samples/s +Step 4770/44000 | Loss: 3.3648 | LR: 4.89e-04 | 2521.4ms/step | 7.9 samples/s +Step 4780/44000 | Loss: 3.3712 | LR: 4.89e-04 | 2533.0ms/step | 7.9 samples/s +Step 4790/44000 | Loss: 3.3417 | LR: 4.89e-04 | 2527.6ms/step | 7.9 samples/s +Step 4800/44000 | Loss: 3.3651 | LR: 4.89e-04 | 2522.3ms/step | 7.9 samples/s +Step 4810/44000 | Loss: 3.3335 | LR: 4.89e-04 | 2452.7ms/step | 8.2 samples/s +Step 4820/44000 | Loss: 3.3723 | LR: 4.89e-04 | 2544.6ms/step | 7.9 samples/s +Step 4830/44000 | Loss: 3.3887 | LR: 4.89e-04 | 2480.6ms/step | 8.1 samples/s +Step 4840/44000 | Loss: 3.3350 | LR: 4.89e-04 | 2516.3ms/step | 7.9 samples/s +Step 4850/44000 | Loss: 3.3289 | LR: 4.89e-04 | 2512.0ms/step | 8.0 samples/s +Step 4860/44000 | Loss: 3.3457 | LR: 4.89e-04 | 2507.1ms/step | 8.0 samples/s +Step 4870/44000 | Loss: 3.3652 | LR: 4.89e-04 | 2485.6ms/step | 8.0 samples/s +Step 4880/44000 | Loss: 3.3494 | LR: 4.89e-04 | 2531.6ms/step | 7.9 samples/s +Step 4890/44000 | Loss: 3.3526 | LR: 4.89e-04 | 2617.3ms/step | 7.6 samples/s +Step 4900/44000 | Loss: 3.3319 | LR: 4.89e-04 | 2546.9ms/step | 7.9 samples/s +Step 4910/44000 | Loss: 3.3602 | LR: 4.89e-04 | 2926.9ms/step | 6.8 samples/s +Step 4920/44000 | Loss: 3.3517 | LR: 4.89e-04 | 2611.4ms/step | 7.7 samples/s +Step 4930/44000 | Loss: 3.3143 | LR: 4.89e-04 | 2464.7ms/step | 8.1 samples/s +Step 4940/44000 | Loss: 3.3404 | LR: 4.89e-04 | 2532.5ms/step | 7.9 samples/s +Step 4950/44000 | Loss: 3.3876 | LR: 4.88e-04 | 2600.4ms/step | 7.7 samples/s +Step 4960/44000 | Loss: 3.3600 | LR: 4.88e-04 | 2571.8ms/step | 7.8 samples/s +Step 4970/44000 | Loss: 3.3408 | LR: 4.88e-04 | 2571.4ms/step | 7.8 samples/s +Step 4980/44000 | Loss: 3.3932 | LR: 4.88e-04 | 2589.8ms/step | 7.7 samples/s +Step 4990/44000 | Loss: 3.2949 | LR: 4.88e-04 | 2568.5ms/step | 7.8 samples/s +Step 5000/44000 | Loss: 3.3428 | LR: 4.88e-04 | 2553.1ms/step | 7.8 samples/s +Validation at step 5000: Loss = 3.3684, Perplexity = 29.03 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-5000 +Step 5010/44000 | Loss: 3.3710 | LR: 4.88e-04 | 4556.5ms/step | 4.4 samples/s +Step 5020/44000 | Loss: 3.3623 | LR: 4.88e-04 | 1907.4ms/step | 10.5 samples/s +Step 5030/44000 | Loss: 3.3155 | LR: 4.88e-04 | 2350.9ms/step | 8.5 samples/s +Step 5040/44000 | Loss: 3.2893 | LR: 4.88e-04 | 2459.5ms/step | 8.1 samples/s +Step 5050/44000 | Loss: 3.3211 | LR: 4.88e-04 | 2485.7ms/step | 8.0 samples/s +Step 5060/44000 | Loss: 3.3129 | LR: 4.88e-04 | 2469.4ms/step | 8.1 samples/s +Step 5070/44000 | Loss: 3.3352 | LR: 4.88e-04 | 2657.1ms/step | 7.5 samples/s +Step 5080/44000 | Loss: 3.3400 | LR: 4.88e-04 | 2740.9ms/step | 7.3 samples/s +Step 5090/44000 | Loss: 3.3784 | LR: 4.88e-04 | 2886.5ms/step | 6.9 samples/s +Step 5100/44000 | Loss: 3.3524 | LR: 4.88e-04 | 2447.8ms/step | 8.2 samples/s +Step 5110/44000 | Loss: 3.3102 | LR: 4.88e-04 | 2411.6ms/step | 8.3 samples/s +Step 5120/44000 | Loss: 3.3375 | LR: 4.88e-04 | 2418.8ms/step | 8.3 samples/s +Step 5130/44000 | Loss: 3.2738 | LR: 4.87e-04 | 2478.8ms/step | 8.1 samples/s +Step 5140/44000 | Loss: 3.3593 | LR: 4.87e-04 | 2469.9ms/step | 8.1 samples/s +Step 5150/44000 | Loss: 3.3063 | LR: 4.87e-04 | 2490.8ms/step | 8.0 samples/s +Step 5160/44000 | Loss: 3.3316 | LR: 4.87e-04 | 2495.5ms/step | 8.0 samples/s +Step 5170/44000 | Loss: 3.3437 | LR: 4.87e-04 | 2516.2ms/step | 7.9 samples/s +Step 5180/44000 | Loss: 3.3600 | LR: 4.87e-04 | 2491.3ms/step | 8.0 samples/s +Step 5190/44000 | Loss: 3.3424 | LR: 4.87e-04 | 2469.5ms/step | 8.1 samples/s +Step 5200/44000 | Loss: 3.3065 | LR: 4.87e-04 | 2469.3ms/step | 8.1 samples/s +Step 5210/44000 | Loss: 3.3491 | LR: 4.87e-04 | 2506.2ms/step | 8.0 samples/s +Step 5220/44000 | Loss: 3.3293 | LR: 4.87e-04 | 2495.8ms/step | 8.0 samples/s +Step 5230/44000 | Loss: 3.2969 | LR: 4.87e-04 | 2469.4ms/step | 8.1 samples/s +Step 5240/44000 | Loss: 3.3077 | LR: 4.87e-04 | 2438.6ms/step | 8.2 samples/s +Step 5250/44000 | Loss: 3.3330 | LR: 4.87e-04 | 2464.5ms/step | 8.1 samples/s +Validation at step 5250: Loss = 3.3510, Perplexity = 28.53 +Step 5260/44000 | Loss: 3.3385 | LR: 4.87e-04 | 4583.9ms/step | 4.4 samples/s +Step 5270/44000 | Loss: 3.3149 | LR: 4.87e-04 | 2386.9ms/step | 8.4 samples/s +Step 5280/44000 | Loss: 3.3514 | LR: 4.87e-04 | 2380.7ms/step | 8.4 samples/s +Step 5290/44000 | Loss: 3.3370 | LR: 4.87e-04 | 2402.7ms/step | 8.3 samples/s +Step 5300/44000 | Loss: 3.3404 | LR: 4.86e-04 | 2516.1ms/step | 7.9 samples/s +Step 5310/44000 | Loss: 3.3303 | LR: 4.86e-04 | 2796.7ms/step | 7.2 samples/s +Step 5320/44000 | Loss: 3.3374 | LR: 4.86e-04 | 2616.8ms/step | 7.6 samples/s +Step 5330/44000 | Loss: 3.3477 | LR: 4.86e-04 | 2323.8ms/step | 8.6 samples/s +Step 5340/44000 | Loss: 3.3281 | LR: 4.86e-04 | 2302.5ms/step | 8.7 samples/s +Step 5350/44000 | Loss: 3.3235 | LR: 4.86e-04 | 2367.3ms/step | 8.4 samples/s +Step 5360/44000 | Loss: 3.3440 | LR: 4.86e-04 | 2412.2ms/step | 8.3 samples/s +Step 5370/44000 | Loss: 3.3196 | LR: 4.86e-04 | 2406.7ms/step | 8.3 samples/s +Step 5380/44000 | Loss: 3.3224 | LR: 4.86e-04 | 2397.4ms/step | 8.3 samples/s +Step 5390/44000 | Loss: 3.3003 | LR: 4.86e-04 | 2386.3ms/step | 8.4 samples/s +Step 5400/44000 | Loss: 3.3300 | LR: 4.86e-04 | 2381.4ms/step | 8.4 samples/s +Step 5410/44000 | Loss: 3.3487 | LR: 4.86e-04 | 2380.5ms/step | 8.4 samples/s +Step 5420/44000 | Loss: 3.3259 | LR: 4.86e-04 | 2412.2ms/step | 8.3 samples/s +Step 5430/44000 | Loss: 3.3369 | LR: 4.86e-04 | 2407.9ms/step | 8.3 samples/s +Step 5440/44000 | Loss: 3.3623 | LR: 4.86e-04 | 2397.1ms/step | 8.3 samples/s +Step 5450/44000 | Loss: 3.3051 | LR: 4.86e-04 | 2376.8ms/step | 8.4 samples/s +Step 5460/44000 | Loss: 3.3466 | LR: 4.86e-04 | 1952.6ms/step | 10.2 samples/s +Step 5470/44000 | Loss: 3.3145 | LR: 4.85e-04 | 1902.2ms/step | 10.5 samples/s +Step 5480/44000 | Loss: 3.2935 | LR: 4.85e-04 | 2497.2ms/step | 8.0 samples/s +Step 5490/44000 | Loss: 3.3110 | LR: 4.85e-04 | 2824.4ms/step | 7.1 samples/s +Step 5500/44000 | Loss: 3.3066 | LR: 4.85e-04 | 2521.0ms/step | 7.9 samples/s +Validation at step 5500: Loss = 3.3374, Perplexity = 28.15 +Step 5510/44000 | Loss: 3.3093 | LR: 4.85e-04 | 4209.2ms/step | 4.8 samples/s +Step 5520/44000 | Loss: 3.3456 | LR: 4.85e-04 | 2313.6ms/step | 8.6 samples/s +Step 5530/44000 | Loss: 3.3192 | LR: 4.85e-04 | 2397.1ms/step | 8.3 samples/s +Step 5540/44000 | Loss: 3.3496 | LR: 4.85e-04 | 2402.3ms/step | 8.3 samples/s +Step 5550/44000 | Loss: 3.3586 | LR: 4.85e-04 | 2411.9ms/step | 8.3 samples/s +Step 5560/44000 | Loss: 3.2951 | LR: 4.85e-04 | 2365.8ms/step | 8.5 samples/s +Step 5570/44000 | Loss: 3.3370 | LR: 4.85e-04 | 2369.8ms/step | 8.4 samples/s +Step 5580/44000 | Loss: 3.3083 | LR: 4.85e-04 | 2381.1ms/step | 8.4 samples/s +Step 5590/44000 | Loss: 3.3137 | LR: 4.85e-04 | 2407.9ms/step | 8.3 samples/s +Step 5600/44000 | Loss: 3.3396 | LR: 4.85e-04 | 2378.8ms/step | 8.4 samples/s +Step 5610/44000 | Loss: 3.3470 | LR: 4.85e-04 | 2415.2ms/step | 8.3 samples/s +Step 5620/44000 | Loss: 3.3071 | LR: 4.85e-04 | 2417.7ms/step | 8.3 samples/s +Step 5630/44000 | Loss: 3.2933 | LR: 4.84e-04 | 2412.3ms/step | 8.3 samples/s +Step 5640/44000 | Loss: 3.2981 | LR: 4.84e-04 | 2407.0ms/step | 8.3 samples/s +Step 5650/44000 | Loss: 3.3396 | LR: 4.84e-04 | 2408.4ms/step | 8.3 samples/s +Step 5660/44000 | Loss: 3.3021 | LR: 4.84e-04 | 2412.2ms/step | 8.3 samples/s +Step 5670/44000 | Loss: 3.3227 | LR: 4.84e-04 | 2370.9ms/step | 8.4 samples/s +Step 5680/44000 | Loss: 3.3333 | LR: 4.84e-04 | 2402.7ms/step | 8.3 samples/s +Step 5690/44000 | Loss: 3.3271 | LR: 4.84e-04 | 2417.7ms/step | 8.3 samples/s +Step 5700/44000 | Loss: 3.3043 | LR: 4.84e-04 | 2406.9ms/step | 8.3 samples/s +Step 5710/44000 | Loss: 3.2853 | LR: 4.84e-04 | 2427.1ms/step | 8.2 samples/s +Step 5720/44000 | Loss: 3.2914 | LR: 4.84e-04 | 2403.4ms/step | 8.3 samples/s +Step 5730/44000 | Loss: 3.2686 | LR: 4.84e-04 | 2392.5ms/step | 8.4 samples/s +Step 5740/44000 | Loss: 3.2952 | LR: 4.84e-04 | 2397.8ms/step | 8.3 samples/s +Step 5750/44000 | Loss: 3.3010 | LR: 4.84e-04 | 2418.1ms/step | 8.3 samples/s +Validation at step 5750: Loss = 3.3259, Perplexity = 27.83 +Step 5760/44000 | Loss: 3.3327 | LR: 4.84e-04 | 4514.2ms/step | 4.4 samples/s +Step 5770/44000 | Loss: 3.2979 | LR: 4.84e-04 | 2381.8ms/step | 8.4 samples/s +Step 5780/44000 | Loss: 3.3012 | LR: 4.83e-04 | 2367.0ms/step | 8.4 samples/s +Step 5790/44000 | Loss: 3.2769 | LR: 4.83e-04 | 2403.6ms/step | 8.3 samples/s +Step 5800/44000 | Loss: 3.3344 | LR: 4.83e-04 | 2397.7ms/step | 8.3 samples/s +Step 5810/44000 | Loss: 3.2765 | LR: 4.83e-04 | 2403.2ms/step | 8.3 samples/s +Step 5820/44000 | Loss: 3.3182 | LR: 4.83e-04 | 2392.7ms/step | 8.4 samples/s +Step 5830/44000 | Loss: 3.3278 | LR: 4.83e-04 | 2382.2ms/step | 8.4 samples/s +Step 5840/44000 | Loss: 3.2767 | LR: 4.83e-04 | 2400.8ms/step | 8.3 samples/s +Step 5850/44000 | Loss: 3.3338 | LR: 4.83e-04 | 2399.4ms/step | 8.3 samples/s +Step 5860/44000 | Loss: 3.3523 | LR: 4.83e-04 | 2417.4ms/step | 8.3 samples/s +Step 5870/44000 | Loss: 3.3062 | LR: 4.83e-04 | 2396.3ms/step | 8.3 samples/s +Step 5880/44000 | Loss: 3.2872 | LR: 4.83e-04 | 2414.1ms/step | 8.3 samples/s +Step 5890/44000 | Loss: 3.2691 | LR: 4.83e-04 | 2402.5ms/step | 8.3 samples/s +Step 5900/44000 | Loss: 3.2925 | LR: 4.83e-04 | 2450.1ms/step | 8.2 samples/s +Step 5910/44000 | Loss: 3.2916 | LR: 4.83e-04 | 2392.6ms/step | 8.4 samples/s +Step 5920/44000 | Loss: 3.2741 | LR: 4.83e-04 | 2380.9ms/step | 8.4 samples/s +Step 5930/44000 | Loss: 3.3239 | LR: 4.83e-04 | 2382.6ms/step | 8.4 samples/s +Step 5940/44000 | Loss: 3.3240 | LR: 4.82e-04 | 2371.8ms/step | 8.4 samples/s +Step 5950/44000 | Loss: 3.2954 | LR: 4.82e-04 | 2435.5ms/step | 8.2 samples/s +Step 5960/44000 | Loss: 3.3008 | LR: 4.82e-04 | 2406.3ms/step | 8.3 samples/s +Step 5970/44000 | Loss: 3.2767 | LR: 4.82e-04 | 2376.7ms/step | 8.4 samples/s +Step 5980/44000 | Loss: 3.3094 | LR: 4.82e-04 | 2412.7ms/step | 8.3 samples/s +Step 5990/44000 | Loss: 3.2812 | LR: 4.82e-04 | 2382.3ms/step | 8.4 samples/s +Step 6000/44000 | Loss: 3.2910 | LR: 4.82e-04 | 2413.0ms/step | 8.3 samples/s +Validation at step 6000: Loss = 3.3106, Perplexity = 27.40 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-6000 +Step 6010/44000 | Loss: 3.2477 | LR: 4.82e-04 | 4440.5ms/step | 4.5 samples/s +Step 6020/44000 | Loss: 3.3096 | LR: 4.82e-04 | 1900.1ms/step | 10.5 samples/s +Step 6030/44000 | Loss: 3.3125 | LR: 4.82e-04 | 2289.4ms/step | 8.7 samples/s +Step 6040/44000 | Loss: 3.2539 | LR: 4.82e-04 | 2370.5ms/step | 8.4 samples/s +Step 6050/44000 | Loss: 3.3085 | LR: 4.82e-04 | 2407.7ms/step | 8.3 samples/s +Step 6060/44000 | Loss: 3.2998 | LR: 4.82e-04 | 2418.0ms/step | 8.3 samples/s +Step 6070/44000 | Loss: 3.2323 | LR: 4.82e-04 | 2407.8ms/step | 8.3 samples/s +Step 6080/44000 | Loss: 3.3318 | LR: 4.81e-04 | 2365.9ms/step | 8.5 samples/s +Step 6090/44000 | Loss: 3.2755 | LR: 4.81e-04 | 2377.8ms/step | 8.4 samples/s +Step 6100/44000 | Loss: 3.2631 | LR: 4.81e-04 | 2387.1ms/step | 8.4 samples/s +Step 6110/44000 | Loss: 3.3135 | LR: 4.81e-04 | 2355.0ms/step | 8.5 samples/s +Step 6120/44000 | Loss: 3.2406 | LR: 4.81e-04 | 2382.3ms/step | 8.4 samples/s +Step 6130/44000 | Loss: 3.2575 | LR: 4.81e-04 | 2392.7ms/step | 8.4 samples/s +Step 6140/44000 | Loss: 3.2866 | LR: 4.81e-04 | 2371.4ms/step | 8.4 samples/s +Step 6150/44000 | Loss: 3.2893 | LR: 4.81e-04 | 2372.2ms/step | 8.4 samples/s +Step 6160/44000 | Loss: 3.2637 | LR: 4.81e-04 | 2418.1ms/step | 8.3 samples/s +Step 6170/44000 | Loss: 3.2725 | LR: 4.81e-04 | 2402.7ms/step | 8.3 samples/s +Step 6180/44000 | Loss: 3.2384 | LR: 4.81e-04 | 2439.9ms/step | 8.2 samples/s +Step 6190/44000 | Loss: 3.2749 | LR: 4.81e-04 | 2392.5ms/step | 8.4 samples/s +Step 6200/44000 | Loss: 3.3253 | LR: 4.81e-04 | 2470.7ms/step | 8.1 samples/s +Step 6210/44000 | Loss: 3.2703 | LR: 4.81e-04 | 2430.1ms/step | 8.2 samples/s +Step 6220/44000 | Loss: 3.3234 | LR: 4.81e-04 | 2435.0ms/step | 8.2 samples/s +Step 6230/44000 | Loss: 3.2474 | LR: 4.80e-04 | 2471.4ms/step | 8.1 samples/s +Step 6240/44000 | Loss: 3.2591 | LR: 4.80e-04 | 2413.2ms/step | 8.3 samples/s +Step 6250/44000 | Loss: 3.2569 | LR: 4.80e-04 | 2456.2ms/step | 8.1 samples/s +Validation at step 6250: Loss = 3.3010, Perplexity = 27.14 +Step 6260/44000 | Loss: 3.2307 | LR: 4.80e-04 | 4504.5ms/step | 4.4 samples/s +Step 6270/44000 | Loss: 3.2901 | LR: 4.80e-04 | 2418.8ms/step | 8.3 samples/s +Step 6280/44000 | Loss: 3.2664 | LR: 4.80e-04 | 2413.3ms/step | 8.3 samples/s +Step 6290/44000 | Loss: 3.2451 | LR: 4.80e-04 | 2208.2ms/step | 9.1 samples/s +Step 6300/44000 | Loss: 3.2623 | LR: 4.80e-04 | 1891.7ms/step | 10.6 samples/s +Step 6310/44000 | Loss: 3.2552 | LR: 4.80e-04 | 2094.7ms/step | 9.5 samples/s +Step 6320/44000 | Loss: 3.2726 | LR: 4.80e-04 | 2606.7ms/step | 7.7 samples/s +Step 6330/44000 | Loss: 3.3164 | LR: 4.80e-04 | 2528.6ms/step | 7.9 samples/s +Step 6340/44000 | Loss: 3.3113 | LR: 4.80e-04 | 2970.8ms/step | 6.7 samples/s +Step 6350/44000 | Loss: 3.2607 | LR: 4.80e-04 | 2517.5ms/step | 7.9 samples/s +Step 6360/44000 | Loss: 3.2778 | LR: 4.80e-04 | 2356.2ms/step | 8.5 samples/s +Step 6370/44000 | Loss: 3.2703 | LR: 4.79e-04 | 2363.4ms/step | 8.5 samples/s +Step 6380/44000 | Loss: 3.3024 | LR: 4.79e-04 | 2817.9ms/step | 7.1 samples/s +Step 6390/44000 | Loss: 3.2971 | LR: 4.79e-04 | 3210.9ms/step | 6.2 samples/s +Step 6400/44000 | Loss: 3.2466 | LR: 4.79e-04 | 2980.9ms/step | 6.7 samples/s +Step 6410/44000 | Loss: 3.2823 | LR: 4.79e-04 | 2970.6ms/step | 6.7 samples/s +Step 6420/44000 | Loss: 3.2747 | LR: 4.79e-04 | 2995.2ms/step | 6.7 samples/s +Step 6430/44000 | Loss: 3.2733 | LR: 4.79e-04 | 3071.7ms/step | 6.5 samples/s +Step 6440/44000 | Loss: 3.2471 | LR: 4.79e-04 | 3002.0ms/step | 6.7 samples/s +Step 6450/44000 | Loss: 3.3008 | LR: 4.79e-04 | 3059.2ms/step | 6.5 samples/s +Step 6460/44000 | Loss: 3.2944 | LR: 4.79e-04 | 3028.8ms/step | 6.6 samples/s +Step 6470/44000 | Loss: 3.2890 | LR: 4.79e-04 | 3065.9ms/step | 6.5 samples/s +Step 6480/44000 | Loss: 3.2680 | LR: 4.79e-04 | 3043.4ms/step | 6.6 samples/s +Step 6490/44000 | Loss: 3.2516 | LR: 4.79e-04 | 2996.0ms/step | 6.7 samples/s +Step 6500/44000 | Loss: 3.3139 | LR: 4.79e-04 | 3048.4ms/step | 6.6 samples/s +Validation at step 6500: Loss = 3.2886, Perplexity = 26.81 +Step 6510/44000 | Loss: 3.2986 | LR: 4.78e-04 | 5994.0ms/step | 3.3 samples/s +Step 6520/44000 | Loss: 3.2768 | LR: 4.78e-04 | 3004.5ms/step | 6.7 samples/s +Step 6530/44000 | Loss: 3.2552 | LR: 4.78e-04 | 3076.6ms/step | 6.5 samples/s +Step 6540/44000 | Loss: 3.2254 | LR: 4.78e-04 | 3013.9ms/step | 6.6 samples/s +Step 6550/44000 | Loss: 3.2646 | LR: 4.78e-04 | 3036.6ms/step | 6.6 samples/s +Step 6560/44000 | Loss: 3.2756 | LR: 4.78e-04 | 3013.7ms/step | 6.6 samples/s +Step 6570/44000 | Loss: 3.2534 | LR: 4.78e-04 | 3075.8ms/step | 6.5 samples/s +Step 6580/44000 | Loss: 3.3026 | LR: 4.78e-04 | 2998.7ms/step | 6.7 samples/s +Step 6590/44000 | Loss: 3.2434 | LR: 4.78e-04 | 3058.3ms/step | 6.5 samples/s +Step 6600/44000 | Loss: 3.2453 | LR: 4.78e-04 | 3075.3ms/step | 6.5 samples/s +Step 6610/44000 | Loss: 3.2540 | LR: 4.78e-04 | 3086.9ms/step | 6.5 samples/s +Step 6620/44000 | Loss: 3.2463 | LR: 4.78e-04 | 2980.8ms/step | 6.7 samples/s +Step 6630/44000 | Loss: 3.2791 | LR: 4.78e-04 | 3050.1ms/step | 6.6 samples/s +Step 6640/44000 | Loss: 3.3158 | LR: 4.77e-04 | 3089.0ms/step | 6.5 samples/s +Step 6650/44000 | Loss: 3.2609 | LR: 4.77e-04 | 3076.4ms/step | 6.5 samples/s +Step 6660/44000 | Loss: 3.2632 | LR: 4.77e-04 | 3018.1ms/step | 6.6 samples/s +Step 6670/44000 | Loss: 3.2435 | LR: 4.77e-04 | 3024.5ms/step | 6.6 samples/s +Step 6680/44000 | Loss: 3.2857 | LR: 4.77e-04 | 2985.5ms/step | 6.7 samples/s +Step 6690/44000 | Loss: 3.2581 | LR: 4.77e-04 | 3080.6ms/step | 6.5 samples/s +Step 6700/44000 | Loss: 3.3002 | LR: 4.77e-04 | 2992.6ms/step | 6.7 samples/s +Step 6710/44000 | Loss: 3.2162 | LR: 4.77e-04 | 3033.2ms/step | 6.6 samples/s +Step 6720/44000 | Loss: 3.2160 | LR: 4.77e-04 | 3088.5ms/step | 6.5 samples/s +Step 6730/44000 | Loss: 3.2551 | LR: 4.77e-04 | 3012.1ms/step | 6.6 samples/s +Step 6740/44000 | Loss: 3.3004 | LR: 4.77e-04 | 3059.9ms/step | 6.5 samples/s +Step 6750/44000 | Loss: 3.2866 | LR: 4.77e-04 | 2955.7ms/step | 6.8 samples/s +Validation at step 6750: Loss = 3.2781, Perplexity = 26.53 +Step 6760/44000 | Loss: 3.2397 | LR: 4.77e-04 | 6152.7ms/step | 3.3 samples/s +Step 6770/44000 | Loss: 3.2902 | LR: 4.76e-04 | 3044.6ms/step | 6.6 samples/s +Step 6780/44000 | Loss: 3.2645 | LR: 4.76e-04 | 3079.8ms/step | 6.5 samples/s +Step 6790/44000 | Loss: 3.2476 | LR: 4.76e-04 | 3018.6ms/step | 6.6 samples/s +Step 6800/44000 | Loss: 3.2422 | LR: 4.76e-04 | 3007.6ms/step | 6.6 samples/s +Step 6810/44000 | Loss: 3.2769 | LR: 4.76e-04 | 2939.6ms/step | 6.8 samples/s +Step 6820/44000 | Loss: 3.2454 | LR: 4.76e-04 | 2935.6ms/step | 6.8 samples/s +Step 6830/44000 | Loss: 3.2771 | LR: 4.76e-04 | 3105.8ms/step | 6.4 samples/s +Step 6840/44000 | Loss: 3.2365 | LR: 4.76e-04 | 3018.2ms/step | 6.6 samples/s +Step 6850/44000 | Loss: 3.2323 | LR: 4.76e-04 | 3117.7ms/step | 6.4 samples/s +Step 6860/44000 | Loss: 3.2647 | LR: 4.76e-04 | 2933.9ms/step | 6.8 samples/s +Step 6870/44000 | Loss: 3.2551 | LR: 4.76e-04 | 3071.4ms/step | 6.5 samples/s +Step 6880/44000 | Loss: 3.2485 | LR: 4.76e-04 | 2992.2ms/step | 6.7 samples/s +Step 6890/44000 | Loss: 3.3025 | LR: 4.76e-04 | 2978.5ms/step | 6.7 samples/s +Step 6900/44000 | Loss: 3.2618 | LR: 4.75e-04 | 3033.3ms/step | 6.6 samples/s +Step 6910/44000 | Loss: 3.2856 | LR: 4.75e-04 | 3007.9ms/step | 6.6 samples/s +Step 6920/44000 | Loss: 3.2522 | LR: 4.75e-04 | 3154.4ms/step | 6.3 samples/s +Step 6930/44000 | Loss: 3.2216 | LR: 4.75e-04 | 3007.7ms/step | 6.6 samples/s +Step 6940/44000 | Loss: 3.2527 | LR: 4.75e-04 | 2928.3ms/step | 6.8 samples/s +Step 6950/44000 | Loss: 3.2576 | LR: 4.75e-04 | 2919.8ms/step | 6.8 samples/s +Step 6960/44000 | Loss: 3.2942 | LR: 4.75e-04 | 2894.6ms/step | 6.9 samples/s +Step 6970/44000 | Loss: 3.2730 | LR: 4.75e-04 | 3058.7ms/step | 6.5 samples/s +Step 6980/44000 | Loss: 3.2454 | LR: 4.75e-04 | 3008.2ms/step | 6.6 samples/s +Step 6990/44000 | Loss: 3.2208 | LR: 4.75e-04 | 2977.0ms/step | 6.7 samples/s +Step 7000/44000 | Loss: 3.2473 | LR: 4.75e-04 | 2882.6ms/step | 6.9 samples/s +Validation at step 7000: Loss = 3.2697, Perplexity = 26.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-7000 +Step 7010/44000 | Loss: 3.2658 | LR: 4.75e-04 | 6338.9ms/step | 3.2 samples/s +Step 7020/44000 | Loss: 3.2230 | LR: 4.75e-04 | 2953.5ms/step | 6.8 samples/s +Step 7030/44000 | Loss: 3.2347 | LR: 4.74e-04 | 2925.3ms/step | 6.8 samples/s +Step 7040/44000 | Loss: 3.2664 | LR: 4.74e-04 | 2627.7ms/step | 7.6 samples/s +Step 7050/44000 | Loss: 3.2675 | LR: 4.74e-04 | 2834.1ms/step | 7.1 samples/s +Step 7060/44000 | Loss: 3.3084 | LR: 4.74e-04 | 2983.9ms/step | 6.7 samples/s +Step 7070/44000 | Loss: 3.2240 | LR: 4.74e-04 | 3054.8ms/step | 6.5 samples/s +Step 7080/44000 | Loss: 3.2295 | LR: 4.74e-04 | 2912.5ms/step | 6.9 samples/s +Step 7090/44000 | Loss: 3.2792 | LR: 4.74e-04 | 2909.1ms/step | 6.9 samples/s +Step 7100/44000 | Loss: 3.2729 | LR: 4.74e-04 | 3043.2ms/step | 6.6 samples/s +Step 7110/44000 | Loss: 3.2848 | LR: 4.74e-04 | 3003.8ms/step | 6.7 samples/s +Step 7120/44000 | Loss: 3.2556 | LR: 4.74e-04 | 3037.5ms/step | 6.6 samples/s +Step 7130/44000 | Loss: 3.2578 | LR: 4.74e-04 | 3015.0ms/step | 6.6 samples/s +Step 7140/44000 | Loss: 3.2386 | LR: 4.74e-04 | 3066.7ms/step | 6.5 samples/s +Step 7150/44000 | Loss: 3.2617 | LR: 4.73e-04 | 2982.0ms/step | 6.7 samples/s +Step 7160/44000 | Loss: 3.2209 | LR: 4.73e-04 | 2919.9ms/step | 6.8 samples/s +Step 7170/44000 | Loss: 3.2540 | LR: 4.73e-04 | 2876.4ms/step | 7.0 samples/s +Step 7180/44000 | Loss: 3.2392 | LR: 4.73e-04 | 2991.7ms/step | 6.7 samples/s +Step 7190/44000 | Loss: 3.2516 | LR: 4.73e-04 | 2966.7ms/step | 6.7 samples/s +Step 7200/44000 | Loss: 3.2447 | LR: 4.73e-04 | 3001.2ms/step | 6.7 samples/s +Step 7210/44000 | Loss: 3.2761 | LR: 4.73e-04 | 2977.1ms/step | 6.7 samples/s +Step 7220/44000 | Loss: 3.2716 | LR: 4.73e-04 | 2929.5ms/step | 6.8 samples/s +Step 7230/44000 | Loss: 3.2953 | LR: 4.73e-04 | 2891.6ms/step | 6.9 samples/s +Step 7240/44000 | Loss: 3.2260 | LR: 4.73e-04 | 2952.1ms/step | 6.8 samples/s +Step 7250/44000 | Loss: 3.2843 | LR: 4.73e-04 | 2960.2ms/step | 6.8 samples/s +Validation at step 7250: Loss = 3.2606, Perplexity = 26.06 +Step 7260/44000 | Loss: 3.2283 | LR: 4.73e-04 | 5818.9ms/step | 3.4 samples/s +Step 7270/44000 | Loss: 3.2324 | LR: 4.72e-04 | 2918.9ms/step | 6.9 samples/s +Step 7280/44000 | Loss: 3.2188 | LR: 4.72e-04 | 2946.5ms/step | 6.8 samples/s +Step 7290/44000 | Loss: 3.2526 | LR: 4.72e-04 | 2988.6ms/step | 6.7 samples/s +Step 7300/44000 | Loss: 3.2771 | LR: 4.72e-04 | 3008.8ms/step | 6.6 samples/s +Step 7310/44000 | Loss: 3.2138 | LR: 4.72e-04 | 2997.8ms/step | 6.7 samples/s +Step 7320/44000 | Loss: 3.2463 | LR: 4.72e-04 | 2939.1ms/step | 6.8 samples/s +Step 7330/44000 | Loss: 3.2203 | LR: 4.72e-04 | 2954.5ms/step | 6.8 samples/s +Step 7340/44000 | Loss: 3.2461 | LR: 4.72e-04 | 2936.6ms/step | 6.8 samples/s +Step 7350/44000 | Loss: 3.2674 | LR: 4.72e-04 | 3006.9ms/step | 6.7 samples/s +Step 7360/44000 | Loss: 3.2231 | LR: 4.72e-04 | 3056.7ms/step | 6.5 samples/s +Step 7370/44000 | Loss: 3.2603 | LR: 4.72e-04 | 3067.3ms/step | 6.5 samples/s +Step 7380/44000 | Loss: 3.2627 | LR: 4.72e-04 | 2984.4ms/step | 6.7 samples/s +Step 7390/44000 | Loss: 3.1781 | LR: 4.71e-04 | 2925.9ms/step | 6.8 samples/s +Step 7400/44000 | Loss: 3.1844 | LR: 4.71e-04 | 2899.0ms/step | 6.9 samples/s +Step 7410/44000 | Loss: 3.2448 | LR: 4.71e-04 | 2967.0ms/step | 6.7 samples/s +Step 7420/44000 | Loss: 3.2419 | LR: 4.71e-04 | 2945.5ms/step | 6.8 samples/s +Step 7430/44000 | Loss: 3.2380 | LR: 4.71e-04 | 2960.5ms/step | 6.8 samples/s +Step 7440/44000 | Loss: 3.2415 | LR: 4.71e-04 | 2998.0ms/step | 6.7 samples/s +Step 7450/44000 | Loss: 3.2486 | LR: 4.71e-04 | 2924.8ms/step | 6.8 samples/s +Step 7460/44000 | Loss: 3.1815 | LR: 4.71e-04 | 2903.2ms/step | 6.9 samples/s +Step 7470/44000 | Loss: 3.2766 | LR: 4.71e-04 | 2937.0ms/step | 6.8 samples/s +Step 7480/44000 | Loss: 3.2257 | LR: 4.71e-04 | 2944.8ms/step | 6.8 samples/s +Step 7490/44000 | Loss: 3.2440 | LR: 4.71e-04 | 2809.5ms/step | 7.1 samples/s +Step 7500/44000 | Loss: 3.2450 | LR: 4.71e-04 | 2892.9ms/step | 6.9 samples/s +Validation at step 7500: Loss = 3.2508, Perplexity = 25.81 +Step 7510/44000 | Loss: 3.2778 | LR: 4.70e-04 | 5562.0ms/step | 3.6 samples/s +Step 7520/44000 | Loss: 3.2654 | LR: 4.70e-04 | 2763.7ms/step | 7.2 samples/s +Step 7530/44000 | Loss: 3.2470 | LR: 4.70e-04 | 2898.2ms/step | 6.9 samples/s +Step 7540/44000 | Loss: 3.2705 | LR: 4.70e-04 | 2846.9ms/step | 7.0 samples/s +Step 7550/44000 | Loss: 3.1766 | LR: 4.70e-04 | 2833.2ms/step | 7.1 samples/s +Step 7560/44000 | Loss: 3.2610 | LR: 4.70e-04 | 2798.0ms/step | 7.1 samples/s +Step 7570/44000 | Loss: 3.2493 | LR: 4.70e-04 | 2827.3ms/step | 7.1 samples/s +Step 7580/44000 | Loss: 3.2073 | LR: 4.70e-04 | 2804.5ms/step | 7.1 samples/s +Step 7590/44000 | Loss: 3.1999 | LR: 4.70e-04 | 2814.3ms/step | 7.1 samples/s +Step 7600/44000 | Loss: 3.2443 | LR: 4.70e-04 | 2853.1ms/step | 7.0 samples/s +Step 7610/44000 | Loss: 3.2079 | LR: 4.70e-04 | 2837.4ms/step | 7.0 samples/s +Step 7620/44000 | Loss: 3.2463 | LR: 4.70e-04 | 2977.6ms/step | 6.7 samples/s +Step 7630/44000 | Loss: 3.2015 | LR: 4.69e-04 | 3059.8ms/step | 6.5 samples/s +Step 7640/44000 | Loss: 3.2136 | LR: 4.69e-04 | 3132.8ms/step | 6.4 samples/s +Step 7650/44000 | Loss: 3.2442 | LR: 4.69e-04 | 2831.7ms/step | 7.1 samples/s +Step 7660/44000 | Loss: 3.2318 | LR: 4.69e-04 | 2865.8ms/step | 7.0 samples/s +Step 7670/44000 | Loss: 3.1926 | LR: 4.69e-04 | 2817.1ms/step | 7.1 samples/s +Step 7680/44000 | Loss: 3.2158 | LR: 4.69e-04 | 2950.3ms/step | 6.8 samples/s +Step 7690/44000 | Loss: 3.2624 | LR: 4.69e-04 | 2781.8ms/step | 7.2 samples/s +Step 7700/44000 | Loss: 3.2350 | LR: 4.69e-04 | 2788.7ms/step | 7.2 samples/s +Step 7710/44000 | Loss: 3.2170 | LR: 4.69e-04 | 2802.2ms/step | 7.1 samples/s +Step 7720/44000 | Loss: 3.1999 | LR: 4.69e-04 | 2908.5ms/step | 6.9 samples/s +Step 7730/44000 | Loss: 3.2057 | LR: 4.69e-04 | 2962.2ms/step | 6.8 samples/s +Step 7740/44000 | Loss: 3.2354 | LR: 4.68e-04 | 2899.0ms/step | 6.9 samples/s +Step 7750/44000 | Loss: 3.2020 | LR: 4.68e-04 | 2799.2ms/step | 7.1 samples/s +Validation at step 7750: Loss = 3.2436, Perplexity = 25.63 +Step 7760/44000 | Loss: 3.2680 | LR: 4.68e-04 | 5553.0ms/step | 3.6 samples/s +Step 7770/44000 | Loss: 3.1769 | LR: 4.68e-04 | 2840.6ms/step | 7.0 samples/s +Step 7780/44000 | Loss: 3.2183 | LR: 4.68e-04 | 2841.2ms/step | 7.0 samples/s +Step 7790/44000 | Loss: 3.2412 | LR: 4.68e-04 | 2773.7ms/step | 7.2 samples/s +Step 7800/44000 | Loss: 3.2342 | LR: 4.68e-04 | 2475.2ms/step | 8.1 samples/s +Step 7810/44000 | Loss: 3.2402 | LR: 4.68e-04 | 2801.7ms/step | 7.1 samples/s +Step 7820/44000 | Loss: 3.2194 | LR: 4.68e-04 | 2866.6ms/step | 7.0 samples/s +Step 7830/44000 | Loss: 3.2329 | LR: 4.68e-04 | 2820.7ms/step | 7.1 samples/s +Step 7840/44000 | Loss: 3.2082 | LR: 4.68e-04 | 2840.2ms/step | 7.0 samples/s +Step 7850/44000 | Loss: 3.2643 | LR: 4.67e-04 | 2788.8ms/step | 7.2 samples/s +Step 7860/44000 | Loss: 3.2535 | LR: 4.67e-04 | 2728.1ms/step | 7.3 samples/s +Step 7870/44000 | Loss: 3.2347 | LR: 4.67e-04 | 2837.1ms/step | 7.0 samples/s +Step 7880/44000 | Loss: 3.2618 | LR: 4.67e-04 | 2846.8ms/step | 7.0 samples/s +Step 7890/44000 | Loss: 3.1949 | LR: 4.67e-04 | 2834.4ms/step | 7.1 samples/s +Step 7900/44000 | Loss: 3.2159 | LR: 4.67e-04 | 2822.7ms/step | 7.1 samples/s +Step 7910/44000 | Loss: 3.1921 | LR: 4.67e-04 | 2765.3ms/step | 7.2 samples/s +Step 7920/44000 | Loss: 3.2232 | LR: 4.67e-04 | 2776.8ms/step | 7.2 samples/s +Step 7930/44000 | Loss: 3.2543 | LR: 4.67e-04 | 2711.3ms/step | 7.4 samples/s +Step 7940/44000 | Loss: 3.2280 | LR: 4.67e-04 | 2753.2ms/step | 7.3 samples/s +Step 7950/44000 | Loss: 3.2192 | LR: 4.67e-04 | 2804.5ms/step | 7.1 samples/s +Step 7960/44000 | Loss: 3.2227 | LR: 4.66e-04 | 2757.8ms/step | 7.3 samples/s +Step 7970/44000 | Loss: 3.2250 | LR: 4.66e-04 | 2752.2ms/step | 7.3 samples/s +Step 7980/44000 | Loss: 3.2520 | LR: 4.66e-04 | 2773.0ms/step | 7.2 samples/s +Step 7990/44000 | Loss: 3.2338 | LR: 4.66e-04 | 2797.9ms/step | 7.1 samples/s +Step 8000/44000 | Loss: 3.2558 | LR: 4.66e-04 | 2729.1ms/step | 7.3 samples/s +Validation at step 8000: Loss = 3.2364, Perplexity = 25.44 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-8000 +Step 8010/44000 | Loss: 3.1951 | LR: 4.66e-04 | 6020.8ms/step | 3.3 samples/s +Step 8020/44000 | Loss: 3.2208 | LR: 4.66e-04 | 2772.2ms/step | 7.2 samples/s +Step 8030/44000 | Loss: 3.2275 | LR: 4.66e-04 | 2836.5ms/step | 7.1 samples/s +Step 8040/44000 | Loss: 3.1928 | LR: 4.66e-04 | 2820.4ms/step | 7.1 samples/s +Step 8050/44000 | Loss: 3.1931 | LR: 4.66e-04 | 2835.6ms/step | 7.1 samples/s +Step 8060/44000 | Loss: 3.1724 | LR: 4.66e-04 | 2821.8ms/step | 7.1 samples/s +Step 8070/44000 | Loss: 3.2137 | LR: 4.65e-04 | 2902.6ms/step | 6.9 samples/s +Step 8080/44000 | Loss: 3.2334 | LR: 4.65e-04 | 2919.3ms/step | 6.9 samples/s +Step 8090/44000 | Loss: 3.2153 | LR: 4.65e-04 | 2903.9ms/step | 6.9 samples/s +Step 8100/44000 | Loss: 3.2412 | LR: 4.65e-04 | 2875.5ms/step | 7.0 samples/s +Step 8110/44000 | Loss: 3.2411 | LR: 4.65e-04 | 2790.6ms/step | 7.2 samples/s +Step 8120/44000 | Loss: 3.2351 | LR: 4.65e-04 | 2857.1ms/step | 7.0 samples/s +Step 8130/44000 | Loss: 3.2213 | LR: 4.65e-04 | 2889.6ms/step | 6.9 samples/s +Step 8140/44000 | Loss: 3.2084 | LR: 4.65e-04 | 2881.7ms/step | 6.9 samples/s +Step 8150/44000 | Loss: 3.2533 | LR: 4.65e-04 | 2864.4ms/step | 7.0 samples/s +Step 8160/44000 | Loss: 3.2204 | LR: 4.65e-04 | 2851.3ms/step | 7.0 samples/s +Step 8170/44000 | Loss: 3.1725 | LR: 4.65e-04 | 2899.8ms/step | 6.9 samples/s +Step 8180/44000 | Loss: 3.1822 | LR: 4.64e-04 | 2888.8ms/step | 6.9 samples/s +Step 8190/44000 | Loss: 3.1794 | LR: 4.64e-04 | 2826.4ms/step | 7.1 samples/s +Step 8200/44000 | Loss: 3.2325 | LR: 4.64e-04 | 2921.4ms/step | 6.8 samples/s +Step 8210/44000 | Loss: 3.2014 | LR: 4.64e-04 | 2922.9ms/step | 6.8 samples/s +Step 8220/44000 | Loss: 3.2060 | LR: 4.64e-04 | 2916.1ms/step | 6.9 samples/s +Step 8230/44000 | Loss: 3.2318 | LR: 4.64e-04 | 2886.3ms/step | 6.9 samples/s +Step 8240/44000 | Loss: 3.2460 | LR: 4.64e-04 | 2896.3ms/step | 6.9 samples/s +Step 8250/44000 | Loss: 3.1948 | LR: 4.64e-04 | 2853.1ms/step | 7.0 samples/s +Validation at step 8250: Loss = 3.2292, Perplexity = 25.26 +Step 8260/44000 | Loss: 3.1729 | LR: 4.64e-04 | 5828.9ms/step | 3.4 samples/s +Step 8270/44000 | Loss: 3.1827 | LR: 4.64e-04 | 2904.1ms/step | 6.9 samples/s +Step 8280/44000 | Loss: 3.1892 | LR: 4.64e-04 | 2856.8ms/step | 7.0 samples/s +Step 8290/44000 | Loss: 3.2154 | LR: 4.63e-04 | 2888.2ms/step | 6.9 samples/s +Step 8300/44000 | Loss: 3.2374 | LR: 4.63e-04 | 2961.6ms/step | 6.8 samples/s +Step 8310/44000 | Loss: 3.2088 | LR: 4.63e-04 | 2921.8ms/step | 6.8 samples/s +Step 8320/44000 | Loss: 3.2000 | LR: 4.63e-04 | 2966.6ms/step | 6.7 samples/s +Step 8330/44000 | Loss: 3.2087 | LR: 4.63e-04 | 2889.9ms/step | 6.9 samples/s +Step 8340/44000 | Loss: 3.1740 | LR: 4.63e-04 | 3255.3ms/step | 6.1 samples/s +Step 8350/44000 | Loss: 3.2225 | LR: 4.63e-04 | 3503.9ms/step | 5.7 samples/s +Step 8360/44000 | Loss: 3.2247 | LR: 4.63e-04 | 3385.5ms/step | 5.9 samples/s +Step 8370/44000 | Loss: 3.2204 | LR: 4.63e-04 | 3469.5ms/step | 5.8 samples/s +Step 8380/44000 | Loss: 3.1890 | LR: 4.63e-04 | 3499.8ms/step | 5.7 samples/s +Step 8390/44000 | Loss: 3.2120 | LR: 4.62e-04 | 3503.3ms/step | 5.7 samples/s +Step 8400/44000 | Loss: 3.2048 | LR: 4.62e-04 | 3221.0ms/step | 6.2 samples/s +Step 8410/44000 | Loss: 3.2081 | LR: 4.62e-04 | 2909.7ms/step | 6.9 samples/s +Step 8420/44000 | Loss: 3.2026 | LR: 4.62e-04 | 2832.9ms/step | 7.1 samples/s +Step 8430/44000 | Loss: 3.2047 | LR: 4.62e-04 | 2772.9ms/step | 7.2 samples/s +Step 8440/44000 | Loss: 3.2034 | LR: 4.62e-04 | 2824.2ms/step | 7.1 samples/s +Step 8450/44000 | Loss: 3.2305 | LR: 4.62e-04 | 2744.8ms/step | 7.3 samples/s +Step 8460/44000 | Loss: 3.1894 | LR: 4.62e-04 | 2833.4ms/step | 7.1 samples/s +Step 8470/44000 | Loss: 3.1808 | LR: 4.62e-04 | 2737.4ms/step | 7.3 samples/s +Step 8480/44000 | Loss: 3.1539 | LR: 4.62e-04 | 3009.7ms/step | 6.6 samples/s +Step 8490/44000 | Loss: 3.1795 | LR: 4.62e-04 | 3406.4ms/step | 5.9 samples/s +Step 8500/44000 | Loss: 3.2709 | LR: 4.61e-04 | 3479.6ms/step | 5.7 samples/s +Validation at step 8500: Loss = 3.2216, Perplexity = 25.07 +Step 8510/44000 | Loss: 3.2013 | LR: 4.61e-04 | 6707.8ms/step | 3.0 samples/s +Step 8520/44000 | Loss: 3.2358 | LR: 4.61e-04 | 3534.6ms/step | 5.7 samples/s +Step 8530/44000 | Loss: 3.2329 | LR: 4.61e-04 | 3569.9ms/step | 5.6 samples/s +Step 8540/44000 | Loss: 3.2002 | LR: 4.61e-04 | 3563.8ms/step | 5.6 samples/s +Step 8550/44000 | Loss: 3.1782 | LR: 4.61e-04 | 3431.8ms/step | 5.8 samples/s +Step 8560/44000 | Loss: 3.2160 | LR: 4.61e-04 | 2872.8ms/step | 7.0 samples/s +Step 8570/44000 | Loss: 3.1921 | LR: 4.61e-04 | 2849.0ms/step | 7.0 samples/s +Step 8580/44000 | Loss: 3.2337 | LR: 4.61e-04 | 2482.9ms/step | 8.1 samples/s +Step 8590/44000 | Loss: 3.2326 | LR: 4.61e-04 | 2497.1ms/step | 8.0 samples/s +Step 8600/44000 | Loss: 3.1836 | LR: 4.60e-04 | 2724.3ms/step | 7.3 samples/s +Step 8610/44000 | Loss: 3.2723 | LR: 4.60e-04 | 2729.8ms/step | 7.3 samples/s +Step 8620/44000 | Loss: 3.1893 | LR: 4.60e-04 | 2766.7ms/step | 7.2 samples/s +Step 8630/44000 | Loss: 3.2280 | LR: 4.60e-04 | 2630.7ms/step | 7.6 samples/s +Step 8640/44000 | Loss: 3.2410 | LR: 4.60e-04 | 2727.5ms/step | 7.3 samples/s +Step 8650/44000 | Loss: 3.1898 | LR: 4.60e-04 | 2688.3ms/step | 7.4 samples/s +Step 8660/44000 | Loss: 3.2115 | LR: 4.60e-04 | 2796.5ms/step | 7.2 samples/s +Step 8670/44000 | Loss: 3.1583 | LR: 4.60e-04 | 2709.5ms/step | 7.4 samples/s +Step 8680/44000 | Loss: 3.1987 | LR: 4.60e-04 | 2699.2ms/step | 7.4 samples/s +Step 8690/44000 | Loss: 3.2013 | LR: 4.60e-04 | 2674.5ms/step | 7.5 samples/s +Step 8700/44000 | Loss: 3.2164 | LR: 4.59e-04 | 2761.9ms/step | 7.2 samples/s +Step 8710/44000 | Loss: 3.2423 | LR: 4.59e-04 | 2710.4ms/step | 7.4 samples/s +Step 8720/44000 | Loss: 3.2061 | LR: 4.59e-04 | 2760.9ms/step | 7.2 samples/s +Step 8730/44000 | Loss: 3.1961 | LR: 4.59e-04 | 2813.9ms/step | 7.1 samples/s +Step 8740/44000 | Loss: 3.1747 | LR: 4.59e-04 | 2781.1ms/step | 7.2 samples/s +Step 8750/44000 | Loss: 3.2168 | LR: 4.59e-04 | 2803.0ms/step | 7.1 samples/s +Validation at step 8750: Loss = 3.2142, Perplexity = 24.88 +Step 8760/44000 | Loss: 3.1749 | LR: 4.59e-04 | 5387.2ms/step | 3.7 samples/s +Step 8770/44000 | Loss: 3.2472 | LR: 4.59e-04 | 2799.3ms/step | 7.1 samples/s +Step 8780/44000 | Loss: 3.2094 | LR: 4.59e-04 | 2804.1ms/step | 7.1 samples/s +Step 8790/44000 | Loss: 3.2046 | LR: 4.59e-04 | 2765.2ms/step | 7.2 samples/s +Step 8800/44000 | Loss: 3.1998 | LR: 4.58e-04 | 2803.0ms/step | 7.1 samples/s +Step 8810/44000 | Loss: 3.1975 | LR: 4.58e-04 | 2823.6ms/step | 7.1 samples/s +Step 8820/44000 | Loss: 3.1696 | LR: 4.58e-04 | 2766.8ms/step | 7.2 samples/s +Step 8830/44000 | Loss: 3.2337 | LR: 4.58e-04 | 2798.3ms/step | 7.1 samples/s +Step 8840/44000 | Loss: 3.1600 | LR: 4.58e-04 | 2763.0ms/step | 7.2 samples/s +Step 8850/44000 | Loss: 3.1581 | LR: 4.58e-04 | 2790.5ms/step | 7.2 samples/s +Step 8860/44000 | Loss: 3.2352 | LR: 4.58e-04 | 2835.4ms/step | 7.1 samples/s +Step 8870/44000 | Loss: 3.2375 | LR: 4.58e-04 | 2738.9ms/step | 7.3 samples/s +Step 8880/44000 | Loss: 3.2241 | LR: 4.58e-04 | 2682.1ms/step | 7.5 samples/s +Step 8890/44000 | Loss: 3.2275 | LR: 4.58e-04 | 2673.0ms/step | 7.5 samples/s +Step 8900/44000 | Loss: 3.1888 | LR: 4.57e-04 | 2702.1ms/step | 7.4 samples/s +Step 8910/44000 | Loss: 3.1602 | LR: 4.57e-04 | 2673.4ms/step | 7.5 samples/s +Step 8920/44000 | Loss: 3.2094 | LR: 4.57e-04 | 2708.4ms/step | 7.4 samples/s +Step 8930/44000 | Loss: 3.1952 | LR: 4.57e-04 | 2717.7ms/step | 7.4 samples/s +Step 8940/44000 | Loss: 3.2157 | LR: 4.57e-04 | 2694.7ms/step | 7.4 samples/s +Step 8950/44000 | Loss: 3.1625 | LR: 4.57e-04 | 2674.6ms/step | 7.5 samples/s +Step 8960/44000 | Loss: 3.2230 | LR: 4.57e-04 | 2730.0ms/step | 7.3 samples/s +Step 8970/44000 | Loss: 3.1971 | LR: 4.57e-04 | 2647.2ms/step | 7.6 samples/s +Step 8980/44000 | Loss: 3.1794 | LR: 4.57e-04 | 2645.3ms/step | 7.6 samples/s +Step 8990/44000 | Loss: 3.1854 | LR: 4.57e-04 | 2831.1ms/step | 7.1 samples/s +Step 9000/44000 | Loss: 3.1686 | LR: 4.56e-04 | 2771.1ms/step | 7.2 samples/s +Validation at step 9000: Loss = 3.2077, Perplexity = 24.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-9000 +Step 9010/44000 | Loss: 3.1881 | LR: 4.56e-04 | 6542.0ms/step | 3.1 samples/s +Step 9020/44000 | Loss: 3.1906 | LR: 4.56e-04 | 2614.2ms/step | 7.7 samples/s +Step 9030/44000 | Loss: 3.1667 | LR: 4.56e-04 | 2691.6ms/step | 7.4 samples/s +Step 9040/44000 | Loss: 3.1838 | LR: 4.56e-04 | 2763.8ms/step | 7.2 samples/s +Step 9050/44000 | Loss: 3.1818 | LR: 4.56e-04 | 2721.6ms/step | 7.3 samples/s +Step 9060/44000 | Loss: 3.1858 | LR: 4.56e-04 | 2684.0ms/step | 7.5 samples/s +Step 9070/44000 | Loss: 3.1684 | LR: 4.56e-04 | 2664.1ms/step | 7.5 samples/s +Step 9080/44000 | Loss: 3.1660 | LR: 4.56e-04 | 2726.8ms/step | 7.3 samples/s +Step 9090/44000 | Loss: 3.1998 | LR: 4.56e-04 | 2779.7ms/step | 7.2 samples/s +Step 9100/44000 | Loss: 3.2065 | LR: 4.55e-04 | 2688.1ms/step | 7.4 samples/s +Step 9110/44000 | Loss: 3.1872 | LR: 4.55e-04 | 2720.4ms/step | 7.4 samples/s +Step 9120/44000 | Loss: 3.1687 | LR: 4.55e-04 | 2664.1ms/step | 7.5 samples/s +Step 9130/44000 | Loss: 3.2190 | LR: 4.55e-04 | 2610.6ms/step | 7.7 samples/s +Step 9140/44000 | Loss: 3.1489 | LR: 4.55e-04 | 2701.4ms/step | 7.4 samples/s +Step 9150/44000 | Loss: 3.1961 | LR: 4.55e-04 | 2711.8ms/step | 7.4 samples/s +Step 9160/44000 | Loss: 3.1543 | LR: 4.55e-04 | 2721.6ms/step | 7.3 samples/s +Step 9170/44000 | Loss: 3.1988 | LR: 4.55e-04 | 2761.9ms/step | 7.2 samples/s +Step 9180/44000 | Loss: 3.1566 | LR: 4.55e-04 | 2684.9ms/step | 7.4 samples/s +Step 9190/44000 | Loss: 3.1860 | LR: 4.54e-04 | 2667.7ms/step | 7.5 samples/s +Step 9200/44000 | Loss: 3.1316 | LR: 4.54e-04 | 2733.6ms/step | 7.3 samples/s +Step 9210/44000 | Loss: 3.1809 | LR: 4.54e-04 | 2635.2ms/step | 7.6 samples/s +Step 9220/44000 | Loss: 3.2164 | LR: 4.54e-04 | 2695.1ms/step | 7.4 samples/s +Step 9230/44000 | Loss: 3.1595 | LR: 4.54e-04 | 2759.0ms/step | 7.2 samples/s +Step 9240/44000 | Loss: 3.2024 | LR: 4.54e-04 | 2863.4ms/step | 7.0 samples/s +Step 9250/44000 | Loss: 3.1755 | LR: 4.54e-04 | 3125.5ms/step | 6.4 samples/s +Validation at step 9250: Loss = 3.2018, Perplexity = 24.58 +Step 9260/44000 | Loss: 3.2152 | LR: 4.54e-04 | 5498.7ms/step | 3.6 samples/s +Step 9270/44000 | Loss: 3.1786 | LR: 4.54e-04 | 2747.7ms/step | 7.3 samples/s +Step 9280/44000 | Loss: 3.1761 | LR: 4.54e-04 | 3118.8ms/step | 6.4 samples/s +Step 9290/44000 | Loss: 3.2138 | LR: 4.53e-04 | 2779.8ms/step | 7.2 samples/s +Step 9300/44000 | Loss: 3.1717 | LR: 4.53e-04 | 2752.1ms/step | 7.3 samples/s +Step 9310/44000 | Loss: 3.1518 | LR: 4.53e-04 | 2593.0ms/step | 7.7 samples/s +Step 9320/44000 | Loss: 3.2052 | LR: 4.53e-04 | 2648.2ms/step | 7.6 samples/s +Step 9330/44000 | Loss: 3.1557 | LR: 4.53e-04 | 2671.3ms/step | 7.5 samples/s +Step 9340/44000 | Loss: 3.2046 | LR: 4.53e-04 | 2710.8ms/step | 7.4 samples/s +Step 9350/44000 | Loss: 3.2171 | LR: 4.53e-04 | 2793.2ms/step | 7.2 samples/s +Step 9360/44000 | Loss: 3.1781 | LR: 4.53e-04 | 2748.7ms/step | 7.3 samples/s +Step 9370/44000 | Loss: 3.1800 | LR: 4.53e-04 | 2647.3ms/step | 7.6 samples/s +Step 9380/44000 | Loss: 3.2200 | LR: 4.52e-04 | 2396.6ms/step | 8.3 samples/s +Step 9390/44000 | Loss: 3.1856 | LR: 4.52e-04 | 2707.9ms/step | 7.4 samples/s +Step 9400/44000 | Loss: 3.1683 | LR: 4.52e-04 | 2699.6ms/step | 7.4 samples/s +Step 9410/44000 | Loss: 3.1852 | LR: 4.52e-04 | 2722.5ms/step | 7.3 samples/s +Step 9420/44000 | Loss: 3.2058 | LR: 4.52e-04 | 2703.8ms/step | 7.4 samples/s +Step 9430/44000 | Loss: 3.1763 | LR: 4.52e-04 | 2650.7ms/step | 7.5 samples/s +Step 9440/44000 | Loss: 3.1601 | LR: 4.52e-04 | 2684.3ms/step | 7.5 samples/s +Step 9450/44000 | Loss: 3.1836 | LR: 4.52e-04 | 2634.3ms/step | 7.6 samples/s +Step 9460/44000 | Loss: 3.1892 | LR: 4.52e-04 | 2607.4ms/step | 7.7 samples/s +Step 9470/44000 | Loss: 3.1919 | LR: 4.52e-04 | 2633.8ms/step | 7.6 samples/s +Step 9480/44000 | Loss: 3.1914 | LR: 4.51e-04 | 2543.9ms/step | 7.9 samples/s +Step 9490/44000 | Loss: 3.1584 | LR: 4.51e-04 | 2508.0ms/step | 8.0 samples/s +Step 9500/44000 | Loss: 3.2044 | LR: 4.51e-04 | 2534.1ms/step | 7.9 samples/s +Validation at step 9500: Loss = 3.1950, Perplexity = 24.41 +Step 9510/44000 | Loss: 3.1595 | LR: 4.51e-04 | 5083.0ms/step | 3.9 samples/s +Step 9520/44000 | Loss: 3.2114 | LR: 4.51e-04 | 2718.6ms/step | 7.4 samples/s +Step 9530/44000 | Loss: 3.2035 | LR: 4.51e-04 | 2636.2ms/step | 7.6 samples/s +Step 9540/44000 | Loss: 3.1753 | LR: 4.51e-04 | 2967.8ms/step | 6.7 samples/s +Step 9550/44000 | Loss: 3.1581 | LR: 4.51e-04 | 2651.2ms/step | 7.5 samples/s +Step 9560/44000 | Loss: 3.1698 | LR: 4.51e-04 | 2496.8ms/step | 8.0 samples/s +Step 9570/44000 | Loss: 3.2102 | LR: 4.50e-04 | 2522.5ms/step | 7.9 samples/s +Step 9580/44000 | Loss: 3.1605 | LR: 4.50e-04 | 2647.8ms/step | 7.6 samples/s +Step 9590/44000 | Loss: 3.1674 | LR: 4.50e-04 | 2704.9ms/step | 7.4 samples/s +Step 9600/44000 | Loss: 3.1618 | LR: 4.50e-04 | 2689.6ms/step | 7.4 samples/s +Step 9610/44000 | Loss: 3.1805 | LR: 4.50e-04 | 2648.1ms/step | 7.6 samples/s +Step 9620/44000 | Loss: 3.1269 | LR: 4.50e-04 | 2650.2ms/step | 7.5 samples/s +Step 9630/44000 | Loss: 3.1639 | LR: 4.50e-04 | 2676.1ms/step | 7.5 samples/s +Step 9640/44000 | Loss: 3.1468 | LR: 4.50e-04 | 2715.0ms/step | 7.4 samples/s +Step 9650/44000 | Loss: 3.2117 | LR: 4.50e-04 | 2720.7ms/step | 7.4 samples/s +Step 9660/44000 | Loss: 3.1586 | LR: 4.49e-04 | 2715.7ms/step | 7.4 samples/s +Step 9670/44000 | Loss: 3.2083 | LR: 4.49e-04 | 2657.9ms/step | 7.5 samples/s +Step 9680/44000 | Loss: 3.1337 | LR: 4.49e-04 | 2606.3ms/step | 7.7 samples/s +Step 9690/44000 | Loss: 3.2122 | LR: 4.49e-04 | 2633.0ms/step | 7.6 samples/s +Step 9700/44000 | Loss: 3.1785 | LR: 4.49e-04 | 2684.7ms/step | 7.4 samples/s +Step 9710/44000 | Loss: 3.1846 | LR: 4.49e-04 | 2604.2ms/step | 7.7 samples/s +Step 9720/44000 | Loss: 3.1318 | LR: 4.49e-04 | 2606.1ms/step | 7.7 samples/s +Step 9730/44000 | Loss: 3.1524 | LR: 4.49e-04 | 2639.7ms/step | 7.6 samples/s +Step 9740/44000 | Loss: 3.1878 | LR: 4.49e-04 | 2555.0ms/step | 7.8 samples/s +Step 9750/44000 | Loss: 3.1647 | LR: 4.48e-04 | 2596.8ms/step | 7.7 samples/s +Validation at step 9750: Loss = 3.1875, Perplexity = 24.23 +Step 9760/44000 | Loss: 3.2023 | LR: 4.48e-04 | 5061.5ms/step | 4.0 samples/s +Step 9770/44000 | Loss: 3.1526 | LR: 4.48e-04 | 2690.2ms/step | 7.4 samples/s +Step 9780/44000 | Loss: 3.1768 | LR: 4.48e-04 | 2586.0ms/step | 7.7 samples/s +Step 9790/44000 | Loss: 3.1411 | LR: 4.48e-04 | 2674.7ms/step | 7.5 samples/s +Step 9800/44000 | Loss: 3.1452 | LR: 4.48e-04 | 2808.2ms/step | 7.1 samples/s +Step 9810/44000 | Loss: 3.1821 | LR: 4.48e-04 | 2947.7ms/step | 6.8 samples/s +Step 9820/44000 | Loss: 3.1292 | LR: 4.48e-04 | 2638.0ms/step | 7.6 samples/s +Step 9830/44000 | Loss: 3.2235 | LR: 4.48e-04 | 2633.4ms/step | 7.6 samples/s +Step 9840/44000 | Loss: 3.1873 | LR: 4.47e-04 | 2620.1ms/step | 7.6 samples/s +Step 9850/44000 | Loss: 3.1787 | LR: 4.47e-04 | 2584.8ms/step | 7.7 samples/s +Step 9860/44000 | Loss: 3.1853 | LR: 4.47e-04 | 2684.7ms/step | 7.4 samples/s +Step 9870/44000 | Loss: 3.1359 | LR: 4.47e-04 | 2686.7ms/step | 7.4 samples/s +Step 9880/44000 | Loss: 3.1605 | LR: 4.47e-04 | 2708.6ms/step | 7.4 samples/s +Step 9890/44000 | Loss: 3.1685 | LR: 4.47e-04 | 2660.2ms/step | 7.5 samples/s +Step 9900/44000 | Loss: 3.2235 | LR: 4.47e-04 | 2676.9ms/step | 7.5 samples/s +Step 9910/44000 | Loss: 3.1394 | LR: 4.47e-04 | 2653.5ms/step | 7.5 samples/s +Step 9920/44000 | Loss: 3.1701 | LR: 4.47e-04 | 2528.9ms/step | 7.9 samples/s +Step 9930/44000 | Loss: 3.1222 | LR: 4.46e-04 | 2572.9ms/step | 7.8 samples/s +Step 9940/44000 | Loss: 3.1675 | LR: 4.46e-04 | 2602.2ms/step | 7.7 samples/s +Step 9950/44000 | Loss: 3.1304 | LR: 4.46e-04 | 2590.4ms/step | 7.7 samples/s +Step 9960/44000 | Loss: 3.1468 | LR: 4.46e-04 | 2652.9ms/step | 7.5 samples/s +Step 9970/44000 | Loss: 3.1609 | LR: 4.46e-04 | 2612.3ms/step | 7.7 samples/s +Step 9980/44000 | Loss: 3.1343 | LR: 4.46e-04 | 2517.5ms/step | 7.9 samples/s +Step 9990/44000 | Loss: 3.1815 | LR: 4.46e-04 | 2543.3ms/step | 7.9 samples/s +Step 10000/44000 | Loss: 3.1835 | LR: 4.46e-04 | 2624.1ms/step | 7.6 samples/s +Validation at step 10000: Loss = 3.1833, Perplexity = 24.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-10000 +Step 10010/44000 | Loss: 3.1807 | LR: 4.46e-04 | 5538.0ms/step | 3.6 samples/s +Step 10020/44000 | Loss: 3.1727 | LR: 4.45e-04 | 2663.2ms/step | 7.5 samples/s +Step 10030/44000 | Loss: 3.1456 | LR: 4.45e-04 | 2673.2ms/step | 7.5 samples/s +Step 10040/44000 | Loss: 3.1462 | LR: 4.45e-04 | 2781.6ms/step | 7.2 samples/s +Step 10050/44000 | Loss: 3.0978 | LR: 4.45e-04 | 2715.6ms/step | 7.4 samples/s +Step 10060/44000 | Loss: 3.2150 | LR: 4.45e-04 | 3090.3ms/step | 6.5 samples/s +Step 10070/44000 | Loss: 3.1653 | LR: 4.45e-04 | 2777.1ms/step | 7.2 samples/s +Step 10080/44000 | Loss: 3.1754 | LR: 4.45e-04 | 2601.0ms/step | 7.7 samples/s +Step 10090/44000 | Loss: 3.1734 | LR: 4.45e-04 | 2563.5ms/step | 7.8 samples/s +Step 10100/44000 | Loss: 3.1558 | LR: 4.45e-04 | 2547.2ms/step | 7.9 samples/s +Step 10110/44000 | Loss: 3.1701 | LR: 4.44e-04 | 2626.2ms/step | 7.6 samples/s +Step 10120/44000 | Loss: 3.1717 | LR: 4.44e-04 | 2608.7ms/step | 7.7 samples/s +Step 10130/44000 | Loss: 3.1654 | LR: 4.44e-04 | 2635.5ms/step | 7.6 samples/s +Step 10140/44000 | Loss: 3.1639 | LR: 4.44e-04 | 2547.0ms/step | 7.9 samples/s +Step 10150/44000 | Loss: 3.1516 | LR: 4.44e-04 | 2594.8ms/step | 7.7 samples/s +Step 10160/44000 | Loss: 3.1923 | LR: 4.44e-04 | 2579.6ms/step | 7.8 samples/s +Step 10170/44000 | Loss: 3.1795 | LR: 4.44e-04 | 2641.6ms/step | 7.6 samples/s +Step 10180/44000 | Loss: 3.1505 | LR: 4.44e-04 | 2391.1ms/step | 8.4 samples/s +Step 10190/44000 | Loss: 3.1371 | LR: 4.43e-04 | 2542.8ms/step | 7.9 samples/s +Step 10200/44000 | Loss: 3.1318 | LR: 4.43e-04 | 2572.9ms/step | 7.8 samples/s +Step 10210/44000 | Loss: 3.1632 | LR: 4.43e-04 | 2646.7ms/step | 7.6 samples/s +Step 10220/44000 | Loss: 3.1193 | LR: 4.43e-04 | 2652.6ms/step | 7.5 samples/s +Step 10230/44000 | Loss: 3.1601 | LR: 4.43e-04 | 2668.6ms/step | 7.5 samples/s +Step 10240/44000 | Loss: 3.1571 | LR: 4.43e-04 | 2653.0ms/step | 7.5 samples/s +Step 10250/44000 | Loss: 3.1200 | LR: 4.43e-04 | 2692.8ms/step | 7.4 samples/s +Validation at step 10250: Loss = 3.1787, Perplexity = 24.02 +Step 10260/44000 | Loss: 3.1916 | LR: 4.43e-04 | 5161.4ms/step | 3.9 samples/s +Step 10270/44000 | Loss: 3.1504 | LR: 4.43e-04 | 2704.6ms/step | 7.4 samples/s +Step 10280/44000 | Loss: 3.1434 | LR: 4.42e-04 | 2728.1ms/step | 7.3 samples/s +Step 10290/44000 | Loss: 3.1449 | LR: 4.42e-04 | 2663.9ms/step | 7.5 samples/s +Step 10300/44000 | Loss: 3.1280 | LR: 4.42e-04 | 2680.6ms/step | 7.5 samples/s +Step 10310/44000 | Loss: 3.1552 | LR: 4.42e-04 | 2768.7ms/step | 7.2 samples/s +Step 10320/44000 | Loss: 3.1801 | LR: 4.42e-04 | 2789.0ms/step | 7.2 samples/s +Step 10330/44000 | Loss: 3.1515 | LR: 4.42e-04 | 2955.3ms/step | 6.8 samples/s +Step 10340/44000 | Loss: 3.1547 | LR: 4.42e-04 | 2621.6ms/step | 7.6 samples/s +Step 10350/44000 | Loss: 3.1669 | LR: 4.42e-04 | 2623.8ms/step | 7.6 samples/s +Step 10360/44000 | Loss: 3.1802 | LR: 4.42e-04 | 2605.4ms/step | 7.7 samples/s +Step 10370/44000 | Loss: 3.1578 | LR: 4.41e-04 | 2624.5ms/step | 7.6 samples/s +Step 10380/44000 | Loss: 3.0978 | LR: 4.41e-04 | 2585.2ms/step | 7.7 samples/s +Step 10390/44000 | Loss: 3.1244 | LR: 4.41e-04 | 2616.5ms/step | 7.6 samples/s +Step 10400/44000 | Loss: 3.1709 | LR: 4.41e-04 | 2559.6ms/step | 7.8 samples/s +Step 10410/44000 | Loss: 3.1536 | LR: 4.41e-04 | 2604.7ms/step | 7.7 samples/s +Step 10420/44000 | Loss: 3.1313 | LR: 4.41e-04 | 2600.5ms/step | 7.7 samples/s +Step 10430/44000 | Loss: 3.1629 | LR: 4.41e-04 | 2650.3ms/step | 7.5 samples/s +Step 10440/44000 | Loss: 3.1563 | LR: 4.41e-04 | 2549.9ms/step | 7.8 samples/s +Step 10450/44000 | Loss: 3.1375 | LR: 4.40e-04 | 2547.6ms/step | 7.9 samples/s +Step 10460/44000 | Loss: 3.1951 | LR: 4.40e-04 | 2604.0ms/step | 7.7 samples/s +Step 10470/44000 | Loss: 3.1445 | LR: 4.40e-04 | 2582.6ms/step | 7.7 samples/s +Step 10480/44000 | Loss: 3.1377 | LR: 4.40e-04 | 2559.7ms/step | 7.8 samples/s +Step 10490/44000 | Loss: 3.1632 | LR: 4.40e-04 | 2507.4ms/step | 8.0 samples/s +Step 10500/44000 | Loss: 3.1034 | LR: 4.40e-04 | 2462.3ms/step | 8.1 samples/s +Validation at step 10500: Loss = 3.1715, Perplexity = 23.84 +Step 10510/44000 | Loss: 3.1801 | LR: 4.40e-04 | 4883.1ms/step | 4.1 samples/s +Step 10520/44000 | Loss: 3.1791 | LR: 4.40e-04 | 2582.1ms/step | 7.7 samples/s +Step 10530/44000 | Loss: 3.1516 | LR: 4.40e-04 | 2626.4ms/step | 7.6 samples/s +Step 10540/44000 | Loss: 3.1441 | LR: 4.39e-04 | 2640.7ms/step | 7.6 samples/s +Step 10550/44000 | Loss: 3.2082 | LR: 4.39e-04 | 2577.8ms/step | 7.8 samples/s +Step 10560/44000 | Loss: 3.1453 | LR: 4.39e-04 | 2618.5ms/step | 7.6 samples/s +Step 10570/44000 | Loss: 3.1422 | LR: 4.39e-04 | 2588.1ms/step | 7.7 samples/s +Step 10580/44000 | Loss: 3.1585 | LR: 4.39e-04 | 2609.6ms/step | 7.7 samples/s +Step 10590/44000 | Loss: 3.1639 | LR: 4.39e-04 | 2877.7ms/step | 7.0 samples/s +Step 10600/44000 | Loss: 3.2485 | LR: 4.39e-04 | 2626.7ms/step | 7.6 samples/s +Step 10610/44000 | Loss: 3.1000 | LR: 4.39e-04 | 2387.4ms/step | 8.4 samples/s +Step 10620/44000 | Loss: 3.1428 | LR: 4.38e-04 | 2392.9ms/step | 8.4 samples/s +Step 10630/44000 | Loss: 3.1590 | LR: 4.38e-04 | 2487.1ms/step | 8.0 samples/s +Step 10640/44000 | Loss: 3.1616 | LR: 4.38e-04 | 2492.0ms/step | 8.0 samples/s +Step 10650/44000 | Loss: 3.1886 | LR: 4.38e-04 | 2461.2ms/step | 8.1 samples/s +Step 10660/44000 | Loss: 3.2087 | LR: 4.38e-04 | 2596.1ms/step | 7.7 samples/s +Step 10670/44000 | Loss: 3.1516 | LR: 4.38e-04 | 2568.2ms/step | 7.8 samples/s +Step 10680/44000 | Loss: 3.1311 | LR: 4.38e-04 | 2503.8ms/step | 8.0 samples/s +Step 10690/44000 | Loss: 3.1600 | LR: 4.38e-04 | 2512.2ms/step | 8.0 samples/s +Step 10700/44000 | Loss: 3.1485 | LR: 4.37e-04 | 2542.9ms/step | 7.9 samples/s +Step 10710/44000 | Loss: 3.1544 | LR: 4.37e-04 | 2631.7ms/step | 7.6 samples/s +Step 10720/44000 | Loss: 3.1508 | LR: 4.37e-04 | 2724.8ms/step | 7.3 samples/s +Step 10730/44000 | Loss: 3.1360 | LR: 4.37e-04 | 2641.7ms/step | 7.6 samples/s +Step 10740/44000 | Loss: 3.1334 | LR: 4.37e-04 | 2668.6ms/step | 7.5 samples/s +Step 10750/44000 | Loss: 3.1370 | LR: 4.37e-04 | 2647.7ms/step | 7.6 samples/s +Validation at step 10750: Loss = 3.1665, Perplexity = 23.73 +Step 10760/44000 | Loss: 3.1431 | LR: 4.37e-04 | 5115.9ms/step | 3.9 samples/s +Step 10770/44000 | Loss: 3.1498 | LR: 4.37e-04 | 2658.6ms/step | 7.5 samples/s +Step 10780/44000 | Loss: 3.1614 | LR: 4.37e-04 | 2617.6ms/step | 7.6 samples/s +Step 10790/44000 | Loss: 3.1265 | LR: 4.36e-04 | 2615.0ms/step | 7.6 samples/s +Step 10800/44000 | Loss: 3.1677 | LR: 4.36e-04 | 2694.6ms/step | 7.4 samples/s +Step 10810/44000 | Loss: 3.1580 | LR: 4.36e-04 | 2683.0ms/step | 7.5 samples/s +Step 10820/44000 | Loss: 3.1264 | LR: 4.36e-04 | 2756.3ms/step | 7.3 samples/s +Step 10830/44000 | Loss: 3.1232 | LR: 4.36e-04 | 2683.6ms/step | 7.5 samples/s +Step 10840/44000 | Loss: 3.1435 | LR: 4.36e-04 | 2739.7ms/step | 7.3 samples/s +Step 10850/44000 | Loss: 3.1579 | LR: 4.36e-04 | 2697.1ms/step | 7.4 samples/s +Step 10860/44000 | Loss: 3.1191 | LR: 4.36e-04 | 2672.3ms/step | 7.5 samples/s +Step 10870/44000 | Loss: 3.1254 | LR: 4.35e-04 | 2720.5ms/step | 7.4 samples/s +Step 10880/44000 | Loss: 3.1350 | LR: 4.35e-04 | 2757.5ms/step | 7.3 samples/s +Step 10890/44000 | Loss: 3.1525 | LR: 4.35e-04 | 2805.2ms/step | 7.1 samples/s +Step 10900/44000 | Loss: 3.1464 | LR: 4.35e-04 | 2782.9ms/step | 7.2 samples/s +Step 10910/44000 | Loss: 3.1076 | LR: 4.35e-04 | 2814.0ms/step | 7.1 samples/s +Step 10920/44000 | Loss: 3.1327 | LR: 4.35e-04 | 2777.5ms/step | 7.2 samples/s +Step 10930/44000 | Loss: 3.1679 | LR: 4.35e-04 | 2796.9ms/step | 7.2 samples/s +Step 10940/44000 | Loss: 3.1493 | LR: 4.35e-04 | 2771.7ms/step | 7.2 samples/s +Step 10950/44000 | Loss: 3.1334 | LR: 4.34e-04 | 2728.5ms/step | 7.3 samples/s +Step 10960/44000 | Loss: 3.1515 | LR: 4.34e-04 | 2805.0ms/step | 7.1 samples/s +Step 10970/44000 | Loss: 3.1314 | LR: 4.34e-04 | 2697.9ms/step | 7.4 samples/s +Step 10980/44000 | Loss: 3.1396 | LR: 4.34e-04 | 2833.2ms/step | 7.1 samples/s +Step 10990/44000 | Loss: 3.1508 | LR: 4.34e-04 | 2951.9ms/step | 6.8 samples/s +Step 11000/44000 | Loss: 3.2108 | LR: 4.34e-04 | 2574.2ms/step | 7.8 samples/s +Validation at step 11000: Loss = 3.1633, Perplexity = 23.65 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-11000 +Step 11010/44000 | Loss: 3.1644 | LR: 4.34e-04 | 6039.6ms/step | 3.3 samples/s +Step 11020/44000 | Loss: 3.1431 | LR: 4.34e-04 | 2857.1ms/step | 7.0 samples/s +Step 11030/44000 | Loss: 3.1410 | LR: 4.33e-04 | 2777.7ms/step | 7.2 samples/s +Step 11040/44000 | Loss: 3.1362 | LR: 4.33e-04 | 2831.5ms/step | 7.1 samples/s +Step 11050/44000 | Loss: 3.1640 | LR: 4.33e-04 | 2844.7ms/step | 7.0 samples/s +Step 11060/44000 | Loss: 3.1480 | LR: 4.33e-04 | 2777.6ms/step | 7.2 samples/s +Step 11070/44000 | Loss: 3.1576 | LR: 4.33e-04 | 2772.8ms/step | 7.2 samples/s +Step 11080/44000 | Loss: 3.1498 | LR: 4.33e-04 | 2788.9ms/step | 7.2 samples/s +Step 11090/44000 | Loss: 3.1194 | LR: 4.33e-04 | 2869.8ms/step | 7.0 samples/s +Step 11100/44000 | Loss: 3.0920 | LR: 4.33e-04 | 2871.9ms/step | 7.0 samples/s +Step 11110/44000 | Loss: 3.1437 | LR: 4.32e-04 | 2949.7ms/step | 6.8 samples/s +Step 11120/44000 | Loss: 3.1729 | LR: 4.32e-04 | 2996.8ms/step | 6.7 samples/s +Step 11130/44000 | Loss: 3.1298 | LR: 4.32e-04 | 2819.8ms/step | 7.1 samples/s +Step 11140/44000 | Loss: 3.1713 | LR: 4.32e-04 | 2803.0ms/step | 7.1 samples/s +Step 11150/44000 | Loss: 3.1212 | LR: 4.32e-04 | 2777.9ms/step | 7.2 samples/s +Step 11160/44000 | Loss: 3.1429 | LR: 4.32e-04 | 2748.9ms/step | 7.3 samples/s +Step 11170/44000 | Loss: 3.1514 | LR: 4.32e-04 | 2873.2ms/step | 7.0 samples/s +Step 11180/44000 | Loss: 3.1274 | LR: 4.32e-04 | 2774.8ms/step | 7.2 samples/s +Step 11190/44000 | Loss: 3.1671 | LR: 4.31e-04 | 2729.2ms/step | 7.3 samples/s +Step 11200/44000 | Loss: 3.1020 | LR: 4.31e-04 | 2809.3ms/step | 7.1 samples/s +Step 11210/44000 | Loss: 3.1259 | LR: 4.31e-04 | 2830.3ms/step | 7.1 samples/s +Step 11220/44000 | Loss: 3.1774 | LR: 4.31e-04 | 2798.4ms/step | 7.1 samples/s +Step 11230/44000 | Loss: 3.1488 | LR: 4.31e-04 | 2896.7ms/step | 6.9 samples/s +Step 11240/44000 | Loss: 3.1256 | LR: 4.31e-04 | 2886.6ms/step | 6.9 samples/s +Step 11250/44000 | Loss: 3.1319 | LR: 4.31e-04 | 2855.9ms/step | 7.0 samples/s +Validation at step 11250: Loss = 3.1567, Perplexity = 23.49 +Step 11260/44000 | Loss: 3.1177 | LR: 4.31e-04 | 5626.3ms/step | 3.6 samples/s +Step 11270/44000 | Loss: 3.1332 | LR: 4.30e-04 | 2745.3ms/step | 7.3 samples/s +Step 11280/44000 | Loss: 3.1125 | LR: 4.30e-04 | 2798.0ms/step | 7.1 samples/s +Step 11290/44000 | Loss: 3.1584 | LR: 4.30e-04 | 2786.7ms/step | 7.2 samples/s +Step 11300/44000 | Loss: 3.1467 | LR: 4.30e-04 | 2866.1ms/step | 7.0 samples/s +Step 11310/44000 | Loss: 3.1009 | LR: 4.30e-04 | 2853.4ms/step | 7.0 samples/s +Step 11320/44000 | Loss: 3.1099 | LR: 4.30e-04 | 2926.0ms/step | 6.8 samples/s +Step 11330/44000 | Loss: 3.1857 | LR: 4.30e-04 | 2857.7ms/step | 7.0 samples/s +Step 11340/44000 | Loss: 3.1313 | LR: 4.30e-04 | 2753.7ms/step | 7.3 samples/s +Step 11350/44000 | Loss: 3.1526 | LR: 4.29e-04 | 2882.2ms/step | 6.9 samples/s +Step 11360/44000 | Loss: 3.1603 | LR: 4.29e-04 | 2874.0ms/step | 7.0 samples/s +Step 11370/44000 | Loss: 3.1232 | LR: 4.29e-04 | 2754.7ms/step | 7.3 samples/s +Step 11380/44000 | Loss: 3.1390 | LR: 4.29e-04 | 2860.3ms/step | 7.0 samples/s +Step 11390/44000 | Loss: 3.1391 | LR: 4.29e-04 | 2794.0ms/step | 7.2 samples/s +Step 11400/44000 | Loss: 3.1284 | LR: 4.29e-04 | 2755.8ms/step | 7.3 samples/s +Step 11410/44000 | Loss: 3.1375 | LR: 4.29e-04 | 2869.8ms/step | 7.0 samples/s +Step 11420/44000 | Loss: 3.1146 | LR: 4.29e-04 | 2862.4ms/step | 7.0 samples/s +Step 11430/44000 | Loss: 3.0940 | LR: 4.28e-04 | 2944.2ms/step | 6.8 samples/s +Step 11440/44000 | Loss: 3.1453 | LR: 4.28e-04 | 2834.0ms/step | 7.1 samples/s +Step 11450/44000 | Loss: 3.1554 | LR: 4.28e-04 | 2800.8ms/step | 7.1 samples/s +Step 11460/44000 | Loss: 3.1292 | LR: 4.28e-04 | 2849.3ms/step | 7.0 samples/s +Step 11470/44000 | Loss: 3.1605 | LR: 4.28e-04 | 2838.2ms/step | 7.0 samples/s +Step 11480/44000 | Loss: 3.1117 | LR: 4.28e-04 | 2784.4ms/step | 7.2 samples/s +Step 11490/44000 | Loss: 3.0922 | LR: 4.28e-04 | 2827.6ms/step | 7.1 samples/s +Step 11500/44000 | Loss: 3.1499 | LR: 4.28e-04 | 2908.2ms/step | 6.9 samples/s +Validation at step 11500: Loss = 3.1525, Perplexity = 23.39 +Step 11510/44000 | Loss: 3.1374 | LR: 4.27e-04 | 5601.5ms/step | 3.6 samples/s +Step 11520/44000 | Loss: 3.1506 | LR: 4.27e-04 | 2722.5ms/step | 7.3 samples/s +Step 11530/44000 | Loss: 3.1387 | LR: 4.27e-04 | 2756.7ms/step | 7.3 samples/s +Step 11540/44000 | Loss: 3.1188 | LR: 4.27e-04 | 2805.7ms/step | 7.1 samples/s +Step 11550/44000 | Loss: 3.1762 | LR: 4.27e-04 | 2865.1ms/step | 7.0 samples/s +Step 11560/44000 | Loss: 3.1458 | LR: 4.27e-04 | 2797.0ms/step | 7.2 samples/s +Step 11570/44000 | Loss: 3.1133 | LR: 4.27e-04 | 2821.9ms/step | 7.1 samples/s +Step 11580/44000 | Loss: 3.1691 | LR: 4.27e-04 | 2728.2ms/step | 7.3 samples/s +Step 11590/44000 | Loss: 3.1067 | LR: 4.26e-04 | 2689.4ms/step | 7.4 samples/s +Step 11600/44000 | Loss: 3.1445 | LR: 4.26e-04 | 2775.4ms/step | 7.2 samples/s +Step 11610/44000 | Loss: 3.1534 | LR: 4.26e-04 | 2881.6ms/step | 6.9 samples/s +Step 11620/44000 | Loss: 3.1204 | LR: 4.26e-04 | 2912.7ms/step | 6.9 samples/s +Step 11630/44000 | Loss: 3.1510 | LR: 4.26e-04 | 2828.2ms/step | 7.1 samples/s +Step 11640/44000 | Loss: 3.1200 | LR: 4.26e-04 | 2789.4ms/step | 7.2 samples/s +Step 11650/44000 | Loss: 3.1666 | LR: 4.26e-04 | 2734.4ms/step | 7.3 samples/s +Step 11660/44000 | Loss: 3.1114 | LR: 4.26e-04 | 2735.7ms/step | 7.3 samples/s +Step 11670/44000 | Loss: 3.1381 | LR: 4.25e-04 | 2760.4ms/step | 7.2 samples/s +Step 11680/44000 | Loss: 3.1191 | LR: 4.25e-04 | 2853.3ms/step | 7.0 samples/s +Step 11690/44000 | Loss: 3.1288 | LR: 4.25e-04 | 2820.2ms/step | 7.1 samples/s +Step 11700/44000 | Loss: 3.1374 | LR: 4.25e-04 | 2835.4ms/step | 7.1 samples/s +Step 11710/44000 | Loss: 3.1028 | LR: 4.25e-04 | 2835.2ms/step | 7.1 samples/s +Step 11720/44000 | Loss: 3.1304 | LR: 4.25e-04 | 2740.6ms/step | 7.3 samples/s +Step 11730/44000 | Loss: 3.1381 | LR: 4.25e-04 | 2766.6ms/step | 7.2 samples/s +Step 11740/44000 | Loss: 3.1008 | LR: 4.24e-04 | 2816.0ms/step | 7.1 samples/s +Step 11750/44000 | Loss: 3.1459 | LR: 4.24e-04 | 2773.0ms/step | 7.2 samples/s +Validation at step 11750: Loss = 3.1464, Perplexity = 23.25 +Step 11760/44000 | Loss: 3.1333 | LR: 4.24e-04 | 5456.7ms/step | 3.7 samples/s +Step 11770/44000 | Loss: 3.0949 | LR: 4.24e-04 | 2689.2ms/step | 7.4 samples/s +Step 11780/44000 | Loss: 3.0775 | LR: 4.24e-04 | 2654.5ms/step | 7.5 samples/s +Step 11790/44000 | Loss: 3.0891 | LR: 4.24e-04 | 2435.8ms/step | 8.2 samples/s +Step 11800/44000 | Loss: 3.1482 | LR: 4.24e-04 | 2744.8ms/step | 7.3 samples/s +Step 11810/44000 | Loss: 3.1115 | LR: 4.24e-04 | 2744.5ms/step | 7.3 samples/s +Step 11820/44000 | Loss: 3.1344 | LR: 4.23e-04 | 2718.5ms/step | 7.4 samples/s +Step 11830/44000 | Loss: 3.1311 | LR: 4.23e-04 | 2762.7ms/step | 7.2 samples/s +Step 11840/44000 | Loss: 3.1053 | LR: 4.23e-04 | 2788.9ms/step | 7.2 samples/s +Step 11850/44000 | Loss: 3.1578 | LR: 4.23e-04 | 2749.5ms/step | 7.3 samples/s +Step 11860/44000 | Loss: 3.1088 | LR: 4.23e-04 | 2709.0ms/step | 7.4 samples/s +Step 11870/44000 | Loss: 3.1459 | LR: 4.23e-04 | 2767.6ms/step | 7.2 samples/s +Step 11880/44000 | Loss: 3.1434 | LR: 4.23e-04 | 2778.1ms/step | 7.2 samples/s +Step 11890/44000 | Loss: 3.1054 | LR: 4.23e-04 | 2699.9ms/step | 7.4 samples/s +Step 11900/44000 | Loss: 3.1036 | LR: 4.22e-04 | 2982.1ms/step | 6.7 samples/s +Step 11910/44000 | Loss: 3.1030 | LR: 4.22e-04 | 2818.4ms/step | 7.1 samples/s +Step 11920/44000 | Loss: 3.1119 | LR: 4.22e-04 | 3168.9ms/step | 6.3 samples/s +Step 11930/44000 | Loss: 3.1156 | LR: 4.22e-04 | 2845.0ms/step | 7.0 samples/s +Step 11940/44000 | Loss: 3.1160 | LR: 4.22e-04 | 2733.4ms/step | 7.3 samples/s +Step 11950/44000 | Loss: 3.0844 | LR: 4.22e-04 | 2729.0ms/step | 7.3 samples/s +Step 11960/44000 | Loss: 3.1130 | LR: 4.22e-04 | 2648.7ms/step | 7.6 samples/s +Step 11970/44000 | Loss: 3.1126 | LR: 4.21e-04 | 2767.3ms/step | 7.2 samples/s +Step 11980/44000 | Loss: 3.1438 | LR: 4.21e-04 | 2713.4ms/step | 7.4 samples/s +Step 11990/44000 | Loss: 3.1630 | LR: 4.21e-04 | 2750.3ms/step | 7.3 samples/s +Step 12000/44000 | Loss: 3.1443 | LR: 4.21e-04 | 2801.3ms/step | 7.1 samples/s +Validation at step 12000: Loss = 3.1426, Perplexity = 23.16 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-12000 +Step 12010/44000 | Loss: 3.1238 | LR: 4.21e-04 | 5932.6ms/step | 3.4 samples/s +Step 12020/44000 | Loss: 3.0953 | LR: 4.21e-04 | 2735.1ms/step | 7.3 samples/s +Step 12030/44000 | Loss: 3.1244 | LR: 4.21e-04 | 2814.5ms/step | 7.1 samples/s +Step 12040/44000 | Loss: 3.1424 | LR: 4.21e-04 | 2813.1ms/step | 7.1 samples/s +Step 12050/44000 | Loss: 3.1251 | LR: 4.20e-04 | 2751.2ms/step | 7.3 samples/s +Step 12060/44000 | Loss: 3.1518 | LR: 4.20e-04 | 2829.9ms/step | 7.1 samples/s +Step 12070/44000 | Loss: 3.1143 | LR: 4.20e-04 | 2801.1ms/step | 7.1 samples/s +Step 12080/44000 | Loss: 3.1499 | LR: 4.20e-04 | 2849.2ms/step | 7.0 samples/s +Step 12090/44000 | Loss: 3.1009 | LR: 4.20e-04 | 2746.9ms/step | 7.3 samples/s +Step 12100/44000 | Loss: 3.1788 | LR: 4.20e-04 | 2780.6ms/step | 7.2 samples/s +Step 12110/44000 | Loss: 3.1265 | LR: 4.20e-04 | 2749.8ms/step | 7.3 samples/s +Step 12120/44000 | Loss: 3.1577 | LR: 4.19e-04 | 2823.8ms/step | 7.1 samples/s +Step 12130/44000 | Loss: 3.0977 | LR: 4.19e-04 | 2886.3ms/step | 6.9 samples/s +Step 12140/44000 | Loss: 3.1538 | LR: 4.19e-04 | 2841.6ms/step | 7.0 samples/s +Step 12150/44000 | Loss: 3.0880 | LR: 4.19e-04 | 2839.3ms/step | 7.0 samples/s +Step 12160/44000 | Loss: 3.1345 | LR: 4.19e-04 | 2759.8ms/step | 7.2 samples/s +Step 12170/44000 | Loss: 3.1345 | LR: 4.19e-04 | 2806.2ms/step | 7.1 samples/s +Step 12180/44000 | Loss: 3.1362 | LR: 4.19e-04 | 2769.4ms/step | 7.2 samples/s +Step 12190/44000 | Loss: 3.1230 | LR: 4.19e-04 | 2742.3ms/step | 7.3 samples/s +Step 12200/44000 | Loss: 3.0824 | LR: 4.18e-04 | 2808.1ms/step | 7.1 samples/s +Step 12210/44000 | Loss: 3.1303 | LR: 4.18e-04 | 2694.4ms/step | 7.4 samples/s +Step 12220/44000 | Loss: 3.1074 | LR: 4.18e-04 | 2741.3ms/step | 7.3 samples/s +Step 12230/44000 | Loss: 3.0958 | LR: 4.18e-04 | 2747.2ms/step | 7.3 samples/s +Step 12240/44000 | Loss: 3.1269 | LR: 4.18e-04 | 2759.2ms/step | 7.2 samples/s +Step 12250/44000 | Loss: 3.1573 | LR: 4.18e-04 | 2741.3ms/step | 7.3 samples/s +Validation at step 12250: Loss = 3.1387, Perplexity = 23.07 +Step 12260/44000 | Loss: 3.1007 | LR: 4.18e-04 | 5588.5ms/step | 3.6 samples/s +Step 12270/44000 | Loss: 3.1533 | LR: 4.17e-04 | 2823.4ms/step | 7.1 samples/s +Step 12280/44000 | Loss: 3.1394 | LR: 4.17e-04 | 2853.9ms/step | 7.0 samples/s +Step 12290/44000 | Loss: 3.1402 | LR: 4.17e-04 | 2794.5ms/step | 7.2 samples/s +Step 12300/44000 | Loss: 3.1677 | LR: 4.17e-04 | 2854.1ms/step | 7.0 samples/s +Step 12310/44000 | Loss: 3.1104 | LR: 4.17e-04 | 2854.5ms/step | 7.0 samples/s +Step 12320/44000 | Loss: 3.1080 | LR: 4.17e-04 | 2918.5ms/step | 6.9 samples/s +Step 12330/44000 | Loss: 3.1321 | LR: 4.17e-04 | 2855.6ms/step | 7.0 samples/s +Step 12340/44000 | Loss: 3.1147 | LR: 4.17e-04 | 2757.0ms/step | 7.3 samples/s +Step 12350/44000 | Loss: 3.1461 | LR: 4.16e-04 | 2818.4ms/step | 7.1 samples/s +Step 12360/44000 | Loss: 3.1416 | LR: 4.16e-04 | 2768.1ms/step | 7.2 samples/s +Step 12370/44000 | Loss: 3.1505 | LR: 4.16e-04 | 2838.0ms/step | 7.0 samples/s +Step 12380/44000 | Loss: 3.1710 | LR: 4.16e-04 | 2847.6ms/step | 7.0 samples/s +Step 12390/44000 | Loss: 3.1223 | LR: 4.16e-04 | 2778.4ms/step | 7.2 samples/s +Step 12400/44000 | Loss: 3.1325 | LR: 4.16e-04 | 2827.8ms/step | 7.1 samples/s +Step 12410/44000 | Loss: 3.0963 | LR: 4.16e-04 | 2778.2ms/step | 7.2 samples/s +Step 12420/44000 | Loss: 3.1472 | LR: 4.15e-04 | 2804.2ms/step | 7.1 samples/s +Step 12430/44000 | Loss: 3.0886 | LR: 4.15e-04 | 3535.4ms/step | 5.7 samples/s +Step 12440/44000 | Loss: 3.0965 | LR: 4.15e-04 | 3538.2ms/step | 5.7 samples/s +Step 12450/44000 | Loss: 3.1430 | LR: 4.15e-04 | 3441.3ms/step | 5.8 samples/s +Step 12460/44000 | Loss: 3.1292 | LR: 4.15e-04 | 3526.9ms/step | 5.7 samples/s +Step 12470/44000 | Loss: 3.0798 | LR: 4.15e-04 | 3519.2ms/step | 5.7 samples/s +Step 12480/44000 | Loss: 3.1349 | LR: 4.15e-04 | 3409.5ms/step | 5.9 samples/s +Step 12490/44000 | Loss: 3.1148 | LR: 4.14e-04 | 3417.7ms/step | 5.9 samples/s +Step 12500/44000 | Loss: 3.0873 | LR: 4.14e-04 | 3330.3ms/step | 6.0 samples/s +Validation at step 12500: Loss = 3.1328, Perplexity = 22.94 +Step 12510/44000 | Loss: 3.0527 | LR: 4.14e-04 | 6664.1ms/step | 3.0 samples/s +Step 12520/44000 | Loss: 3.1330 | LR: 4.14e-04 | 3494.9ms/step | 5.7 samples/s +Step 12530/44000 | Loss: 3.0803 | LR: 4.14e-04 | 3440.7ms/step | 5.8 samples/s +Step 12540/44000 | Loss: 3.1215 | LR: 4.14e-04 | 3451.5ms/step | 5.8 samples/s +Step 12550/44000 | Loss: 3.0766 | LR: 4.14e-04 | 3385.9ms/step | 5.9 samples/s +Step 12560/44000 | Loss: 3.1486 | LR: 4.14e-04 | 3372.4ms/step | 5.9 samples/s +Step 12570/44000 | Loss: 3.1264 | LR: 4.13e-04 | 3409.5ms/step | 5.9 samples/s +Step 12580/44000 | Loss: 3.1187 | LR: 4.13e-04 | 2881.2ms/step | 6.9 samples/s +Step 12590/44000 | Loss: 3.1455 | LR: 4.13e-04 | 2830.6ms/step | 7.1 samples/s +Step 12600/44000 | Loss: 3.1098 | LR: 4.13e-04 | 2532.3ms/step | 7.9 samples/s +Step 12610/44000 | Loss: 3.1572 | LR: 4.13e-04 | 2831.0ms/step | 7.1 samples/s +Step 12620/44000 | Loss: 3.1291 | LR: 4.13e-04 | 2931.2ms/step | 6.8 samples/s +Step 12630/44000 | Loss: 3.0962 | LR: 4.13e-04 | 2788.3ms/step | 7.2 samples/s +Step 12640/44000 | Loss: 3.1120 | LR: 4.12e-04 | 2919.1ms/step | 6.9 samples/s +Step 12650/44000 | Loss: 3.1236 | LR: 4.12e-04 | 2825.2ms/step | 7.1 samples/s +Step 12660/44000 | Loss: 3.1098 | LR: 4.12e-04 | 2845.6ms/step | 7.0 samples/s +Step 12670/44000 | Loss: 3.0981 | LR: 4.12e-04 | 2897.5ms/step | 6.9 samples/s +Step 12680/44000 | Loss: 3.1232 | LR: 4.12e-04 | 2840.0ms/step | 7.0 samples/s +Step 12690/44000 | Loss: 3.1234 | LR: 4.12e-04 | 2866.8ms/step | 7.0 samples/s +Step 12700/44000 | Loss: 3.0943 | LR: 4.12e-04 | 2850.0ms/step | 7.0 samples/s +Step 12710/44000 | Loss: 3.1061 | LR: 4.11e-04 | 2838.9ms/step | 7.0 samples/s +Step 12720/44000 | Loss: 3.0815 | LR: 4.11e-04 | 2834.7ms/step | 7.1 samples/s +Step 12730/44000 | Loss: 3.1214 | LR: 4.11e-04 | 2724.0ms/step | 7.3 samples/s +Step 12740/44000 | Loss: 3.1203 | LR: 4.11e-04 | 2819.9ms/step | 7.1 samples/s +Step 12750/44000 | Loss: 3.1313 | LR: 4.11e-04 | 2876.9ms/step | 7.0 samples/s +Validation at step 12750: Loss = 3.1303, Perplexity = 22.88 +Step 12760/44000 | Loss: 3.0877 | LR: 4.11e-04 | 5668.4ms/step | 3.5 samples/s +Step 12770/44000 | Loss: 3.1486 | LR: 4.11e-04 | 2931.6ms/step | 6.8 samples/s +Step 12780/44000 | Loss: 3.1224 | LR: 4.10e-04 | 2859.8ms/step | 7.0 samples/s +Step 12790/44000 | Loss: 3.0947 | LR: 4.10e-04 | 2849.5ms/step | 7.0 samples/s +Step 12800/44000 | Loss: 3.0848 | LR: 4.10e-04 | 2819.2ms/step | 7.1 samples/s +Step 12810/44000 | Loss: 3.1138 | LR: 4.10e-04 | 2879.3ms/step | 6.9 samples/s +Step 12820/44000 | Loss: 3.1513 | LR: 4.10e-04 | 2878.5ms/step | 6.9 samples/s +Step 12830/44000 | Loss: 3.0757 | LR: 4.10e-04 | 2917.3ms/step | 6.9 samples/s +Step 12840/44000 | Loss: 3.1128 | LR: 4.10e-04 | 2850.3ms/step | 7.0 samples/s +Step 12850/44000 | Loss: 3.0972 | LR: 4.09e-04 | 2762.5ms/step | 7.2 samples/s +Step 12860/44000 | Loss: 3.0972 | LR: 4.09e-04 | 2770.3ms/step | 7.2 samples/s +Step 12870/44000 | Loss: 3.1597 | LR: 4.09e-04 | 2817.3ms/step | 7.1 samples/s +Step 12880/44000 | Loss: 3.1001 | LR: 4.09e-04 | 2782.1ms/step | 7.2 samples/s +Step 12890/44000 | Loss: 3.1480 | LR: 4.09e-04 | 2888.2ms/step | 6.9 samples/s +Step 12900/44000 | Loss: 3.1370 | LR: 4.09e-04 | 2883.2ms/step | 6.9 samples/s +Step 12910/44000 | Loss: 3.1072 | LR: 4.09e-04 | 2812.5ms/step | 7.1 samples/s +Step 12920/44000 | Loss: 3.1609 | LR: 4.09e-04 | 2830.3ms/step | 7.1 samples/s +Step 12930/44000 | Loss: 3.0625 | LR: 4.08e-04 | 2766.7ms/step | 7.2 samples/s +Step 12940/44000 | Loss: 3.1193 | LR: 4.08e-04 | 2748.0ms/step | 7.3 samples/s +Step 12950/44000 | Loss: 3.0929 | LR: 4.08e-04 | 2794.0ms/step | 7.2 samples/s +Step 12960/44000 | Loss: 3.0792 | LR: 4.08e-04 | 2727.0ms/step | 7.3 samples/s +Step 12970/44000 | Loss: 3.0689 | LR: 4.08e-04 | 2816.5ms/step | 7.1 samples/s +Step 12980/44000 | Loss: 3.1012 | LR: 4.08e-04 | 2757.4ms/step | 7.3 samples/s +Step 12990/44000 | Loss: 3.1330 | LR: 4.08e-04 | 2699.6ms/step | 7.4 samples/s +Step 13000/44000 | Loss: 3.0924 | LR: 4.07e-04 | 2762.1ms/step | 7.2 samples/s +Validation at step 13000: Loss = 3.1251, Perplexity = 22.76 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-13000 +Step 13010/44000 | Loss: 3.0699 | LR: 4.07e-04 | 7195.3ms/step | 2.8 samples/s +Step 13020/44000 | Loss: 3.1447 | LR: 4.07e-04 | 2871.6ms/step | 7.0 samples/s +Step 13030/44000 | Loss: 3.1077 | LR: 4.07e-04 | 2816.5ms/step | 7.1 samples/s +Step 13040/44000 | Loss: 3.0934 | LR: 4.07e-04 | 2754.1ms/step | 7.3 samples/s +Step 13050/44000 | Loss: 3.1118 | LR: 4.07e-04 | 2728.6ms/step | 7.3 samples/s +Step 13060/44000 | Loss: 3.0689 | LR: 4.07e-04 | 2757.4ms/step | 7.3 samples/s +Step 13070/44000 | Loss: 3.0721 | LR: 4.06e-04 | 2742.3ms/step | 7.3 samples/s +Step 13080/44000 | Loss: 3.1077 | LR: 4.06e-04 | 2866.0ms/step | 7.0 samples/s +Step 13090/44000 | Loss: 3.0604 | LR: 4.06e-04 | 2857.0ms/step | 7.0 samples/s +Step 13100/44000 | Loss: 3.0933 | LR: 4.06e-04 | 2798.9ms/step | 7.1 samples/s +Step 13110/44000 | Loss: 3.1220 | LR: 4.06e-04 | 2788.8ms/step | 7.2 samples/s +Step 13120/44000 | Loss: 3.1216 | LR: 4.06e-04 | 2819.3ms/step | 7.1 samples/s +Step 13130/44000 | Loss: 3.0938 | LR: 4.06e-04 | 2811.8ms/step | 7.1 samples/s +Step 13140/44000 | Loss: 3.1093 | LR: 4.05e-04 | 2850.8ms/step | 7.0 samples/s +Step 13150/44000 | Loss: 3.1200 | LR: 4.05e-04 | 2818.8ms/step | 7.1 samples/s +Step 13160/44000 | Loss: 3.1029 | LR: 4.05e-04 | 3012.6ms/step | 6.6 samples/s +Step 13170/44000 | Loss: 3.1122 | LR: 4.05e-04 | 2902.7ms/step | 6.9 samples/s +Step 13180/44000 | Loss: 3.0839 | LR: 4.05e-04 | 2768.8ms/step | 7.2 samples/s +Step 13190/44000 | Loss: 3.0980 | LR: 4.05e-04 | 2807.0ms/step | 7.1 samples/s +Step 13200/44000 | Loss: 3.1300 | LR: 4.05e-04 | 2774.0ms/step | 7.2 samples/s +Step 13210/44000 | Loss: 3.0903 | LR: 4.04e-04 | 2779.0ms/step | 7.2 samples/s +Step 13220/44000 | Loss: 3.0830 | LR: 4.04e-04 | 2813.9ms/step | 7.1 samples/s +Step 13230/44000 | Loss: 3.1429 | LR: 4.04e-04 | 2877.7ms/step | 6.9 samples/s +Step 13240/44000 | Loss: 3.1345 | LR: 4.04e-04 | 2861.5ms/step | 7.0 samples/s +Step 13250/44000 | Loss: 3.1270 | LR: 4.04e-04 | 3372.8ms/step | 5.9 samples/s +Validation at step 13250: Loss = 3.1222, Perplexity = 22.70 +Step 13260/44000 | Loss: 3.1249 | LR: 4.04e-04 | 6550.7ms/step | 3.1 samples/s +Step 13270/44000 | Loss: 3.1149 | LR: 4.04e-04 | 3350.0ms/step | 6.0 samples/s +Step 13280/44000 | Loss: 3.0916 | LR: 4.03e-04 | 3379.4ms/step | 5.9 samples/s +Step 13290/44000 | Loss: 3.0815 | LR: 4.03e-04 | 3383.0ms/step | 5.9 samples/s +Step 13300/44000 | Loss: 3.1089 | LR: 4.03e-04 | 3362.8ms/step | 5.9 samples/s +Step 13310/44000 | Loss: 3.0922 | LR: 4.03e-04 | 3378.6ms/step | 5.9 samples/s +Step 13320/44000 | Loss: 3.0695 | LR: 4.03e-04 | 3357.4ms/step | 6.0 samples/s +Step 13330/44000 | Loss: 3.0834 | LR: 4.03e-04 | 2992.2ms/step | 6.7 samples/s +Step 13340/44000 | Loss: 3.1155 | LR: 4.03e-04 | 2751.4ms/step | 7.3 samples/s +Step 13350/44000 | Loss: 3.1114 | LR: 4.02e-04 | 2861.6ms/step | 7.0 samples/s +Step 13360/44000 | Loss: 3.1399 | LR: 4.02e-04 | 2773.5ms/step | 7.2 samples/s +Step 13370/44000 | Loss: 3.0954 | LR: 4.02e-04 | 2882.7ms/step | 6.9 samples/s +Step 13380/44000 | Loss: 3.1072 | LR: 4.02e-04 | 2940.9ms/step | 6.8 samples/s +Step 13390/44000 | Loss: 3.0904 | LR: 4.02e-04 | 2851.3ms/step | 7.0 samples/s +Step 13400/44000 | Loss: 3.1243 | LR: 4.02e-04 | 3003.4ms/step | 6.7 samples/s +Step 13410/44000 | Loss: 3.0549 | LR: 4.02e-04 | 3160.9ms/step | 6.3 samples/s +Step 13420/44000 | Loss: 3.1060 | LR: 4.01e-04 | 2513.0ms/step | 8.0 samples/s +Step 13430/44000 | Loss: 3.0913 | LR: 4.01e-04 | 2783.6ms/step | 7.2 samples/s +Step 13440/44000 | Loss: 3.1420 | LR: 4.01e-04 | 2744.0ms/step | 7.3 samples/s +Step 13450/44000 | Loss: 3.1051 | LR: 4.01e-04 | 2825.2ms/step | 7.1 samples/s +Step 13460/44000 | Loss: 3.0702 | LR: 4.01e-04 | 2851.6ms/step | 7.0 samples/s +Step 13470/44000 | Loss: 3.0939 | LR: 4.01e-04 | 2838.2ms/step | 7.0 samples/s +Step 13480/44000 | Loss: 3.0542 | LR: 4.01e-04 | 2817.8ms/step | 7.1 samples/s +Step 13490/44000 | Loss: 3.0900 | LR: 4.00e-04 | 2789.4ms/step | 7.2 samples/s +Step 13500/44000 | Loss: 3.1092 | LR: 4.00e-04 | 2777.5ms/step | 7.2 samples/s +Validation at step 13500: Loss = 3.1173, Perplexity = 22.59 +Step 13510/44000 | Loss: 3.1164 | LR: 4.00e-04 | 5660.4ms/step | 3.5 samples/s +Step 13520/44000 | Loss: 3.0875 | LR: 4.00e-04 | 2786.7ms/step | 7.2 samples/s +Step 13530/44000 | Loss: 3.1001 | LR: 4.00e-04 | 2763.4ms/step | 7.2 samples/s +Step 13540/44000 | Loss: 3.0857 | LR: 4.00e-04 | 2893.4ms/step | 6.9 samples/s +Step 13550/44000 | Loss: 3.0759 | LR: 4.00e-04 | 2781.7ms/step | 7.2 samples/s +Step 13560/44000 | Loss: 3.0851 | LR: 3.99e-04 | 2779.7ms/step | 7.2 samples/s +Step 13570/44000 | Loss: 3.1038 | LR: 3.99e-04 | 2930.0ms/step | 6.8 samples/s +Step 13580/44000 | Loss: 3.0753 | LR: 3.99e-04 | 2819.4ms/step | 7.1 samples/s +Step 13590/44000 | Loss: 3.1024 | LR: 3.99e-04 | 2799.0ms/step | 7.1 samples/s +Step 13600/44000 | Loss: 3.1436 | LR: 3.99e-04 | 2777.4ms/step | 7.2 samples/s +Step 13610/44000 | Loss: 3.1325 | LR: 3.99e-04 | 2805.2ms/step | 7.1 samples/s +Step 13620/44000 | Loss: 3.1016 | LR: 3.98e-04 | 2756.5ms/step | 7.3 samples/s +Step 13630/44000 | Loss: 3.1220 | LR: 3.98e-04 | 2875.9ms/step | 7.0 samples/s +Step 13640/44000 | Loss: 3.0872 | LR: 3.98e-04 | 2919.7ms/step | 6.8 samples/s +Step 13650/44000 | Loss: 3.1217 | LR: 3.98e-04 | 2753.7ms/step | 7.3 samples/s +Step 13660/44000 | Loss: 3.1432 | LR: 3.98e-04 | 2668.0ms/step | 7.5 samples/s +Step 13670/44000 | Loss: 3.0967 | LR: 3.98e-04 | 2762.7ms/step | 7.2 samples/s +Step 13680/44000 | Loss: 3.0786 | LR: 3.98e-04 | 2777.6ms/step | 7.2 samples/s +Step 13690/44000 | Loss: 3.0558 | LR: 3.97e-04 | 2766.4ms/step | 7.2 samples/s +Step 13700/44000 | Loss: 3.0808 | LR: 3.97e-04 | 2774.8ms/step | 7.2 samples/s +Step 13710/44000 | Loss: 3.0691 | LR: 3.97e-04 | 2846.7ms/step | 7.0 samples/s +Step 13720/44000 | Loss: 3.1078 | LR: 3.97e-04 | 2762.6ms/step | 7.2 samples/s +Step 13730/44000 | Loss: 3.0727 | LR: 3.97e-04 | 2763.0ms/step | 7.2 samples/s +Step 13740/44000 | Loss: 3.0850 | LR: 3.97e-04 | 2725.3ms/step | 7.3 samples/s +Step 13750/44000 | Loss: 3.0827 | LR: 3.97e-04 | 2799.3ms/step | 7.1 samples/s +Validation at step 13750: Loss = 3.1112, Perplexity = 22.45 +Step 13760/44000 | Loss: 3.1219 | LR: 3.96e-04 | 5537.3ms/step | 3.6 samples/s +Step 13770/44000 | Loss: 3.0886 | LR: 3.96e-04 | 2847.9ms/step | 7.0 samples/s +Step 13780/44000 | Loss: 3.0996 | LR: 3.96e-04 | 2766.9ms/step | 7.2 samples/s +Step 13790/44000 | Loss: 3.0895 | LR: 3.96e-04 | 2736.8ms/step | 7.3 samples/s +Step 13800/44000 | Loss: 3.1215 | LR: 3.96e-04 | 2784.4ms/step | 7.2 samples/s +Step 13810/44000 | Loss: 3.1024 | LR: 3.96e-04 | 2762.5ms/step | 7.2 samples/s +Step 13820/44000 | Loss: 3.1112 | LR: 3.96e-04 | 2766.2ms/step | 7.2 samples/s +Step 13830/44000 | Loss: 3.1218 | LR: 3.95e-04 | 2801.5ms/step | 7.1 samples/s +Step 13840/44000 | Loss: 3.1065 | LR: 3.95e-04 | 2870.8ms/step | 7.0 samples/s +Step 13850/44000 | Loss: 3.0660 | LR: 3.95e-04 | 2820.7ms/step | 7.1 samples/s +Step 13860/44000 | Loss: 3.1183 | LR: 3.95e-04 | 2736.1ms/step | 7.3 samples/s +Step 13870/44000 | Loss: 3.1122 | LR: 3.95e-04 | 2819.3ms/step | 7.1 samples/s +Step 13880/44000 | Loss: 3.0949 | LR: 3.95e-04 | 2806.6ms/step | 7.1 samples/s +Step 13890/44000 | Loss: 3.0490 | LR: 3.95e-04 | 2762.4ms/step | 7.2 samples/s +Step 13900/44000 | Loss: 3.0986 | LR: 3.94e-04 | 2829.9ms/step | 7.1 samples/s +Step 13910/44000 | Loss: 3.0882 | LR: 3.94e-04 | 2820.3ms/step | 7.1 samples/s +Step 13920/44000 | Loss: 3.0728 | LR: 3.94e-04 | 2799.5ms/step | 7.1 samples/s +Step 13930/44000 | Loss: 3.1214 | LR: 3.94e-04 | 2759.7ms/step | 7.2 samples/s +Step 13940/44000 | Loss: 3.0961 | LR: 3.94e-04 | 2817.6ms/step | 7.1 samples/s +Step 13950/44000 | Loss: 3.1167 | LR: 3.94e-04 | 2798.1ms/step | 7.1 samples/s +Step 13960/44000 | Loss: 3.0996 | LR: 3.93e-04 | 2751.1ms/step | 7.3 samples/s +Step 13970/44000 | Loss: 3.0662 | LR: 3.93e-04 | 2789.4ms/step | 7.2 samples/s +Step 13980/44000 | Loss: 3.1012 | LR: 3.93e-04 | 2844.6ms/step | 7.0 samples/s +Step 13990/44000 | Loss: 3.0952 | LR: 3.93e-04 | 2836.2ms/step | 7.1 samples/s +Step 14000/44000 | Loss: 3.1007 | LR: 3.93e-04 | 2879.7ms/step | 6.9 samples/s +Validation at step 14000: Loss = 3.1096, Perplexity = 22.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-14000 +Step 14010/44000 | Loss: 3.1331 | LR: 3.93e-04 | 5868.6ms/step | 3.4 samples/s +Step 14020/44000 | Loss: 3.1078 | LR: 3.93e-04 | 2730.9ms/step | 7.3 samples/s +Step 14030/44000 | Loss: 3.0932 | LR: 3.92e-04 | 2781.6ms/step | 7.2 samples/s +Step 14040/44000 | Loss: 3.1092 | LR: 3.92e-04 | 2736.8ms/step | 7.3 samples/s +Step 14050/44000 | Loss: 3.0879 | LR: 3.92e-04 | 2732.9ms/step | 7.3 samples/s +Step 14060/44000 | Loss: 3.0919 | LR: 3.92e-04 | 2819.7ms/step | 7.1 samples/s +Step 14070/44000 | Loss: 3.0967 | LR: 3.92e-04 | 2769.2ms/step | 7.2 samples/s +Step 14080/44000 | Loss: 3.0900 | LR: 3.92e-04 | 2767.3ms/step | 7.2 samples/s +Step 14090/44000 | Loss: 3.0951 | LR: 3.92e-04 | 2773.2ms/step | 7.2 samples/s +Step 14100/44000 | Loss: 3.0591 | LR: 3.91e-04 | 2767.0ms/step | 7.2 samples/s +Step 14110/44000 | Loss: 3.0768 | LR: 3.91e-04 | 2738.2ms/step | 7.3 samples/s +Step 14120/44000 | Loss: 3.0520 | LR: 3.91e-04 | 2744.8ms/step | 7.3 samples/s +Step 14130/44000 | Loss: 3.0808 | LR: 3.91e-04 | 2784.1ms/step | 7.2 samples/s +Step 14140/44000 | Loss: 3.0860 | LR: 3.91e-04 | 2795.3ms/step | 7.2 samples/s +Step 14150/44000 | Loss: 3.1030 | LR: 3.91e-04 | 2751.6ms/step | 7.3 samples/s +Step 14160/44000 | Loss: 3.0557 | LR: 3.90e-04 | 2779.3ms/step | 7.2 samples/s +Step 14170/44000 | Loss: 3.1030 | LR: 3.90e-04 | 2749.4ms/step | 7.3 samples/s +Step 14180/44000 | Loss: 3.1146 | LR: 3.90e-04 | 2790.8ms/step | 7.2 samples/s +Step 14190/44000 | Loss: 3.0953 | LR: 3.90e-04 | 2758.5ms/step | 7.3 samples/s +Step 14200/44000 | Loss: 3.0889 | LR: 3.90e-04 | 2771.7ms/step | 7.2 samples/s +Step 14210/44000 | Loss: 3.0960 | LR: 3.90e-04 | 2740.9ms/step | 7.3 samples/s +Step 14220/44000 | Loss: 3.0540 | LR: 3.90e-04 | 2739.7ms/step | 7.3 samples/s +Step 14230/44000 | Loss: 3.0531 | LR: 3.89e-04 | 2461.8ms/step | 8.1 samples/s +Step 14240/44000 | Loss: 3.0818 | LR: 3.89e-04 | 2796.4ms/step | 7.2 samples/s +Step 14250/44000 | Loss: 3.1011 | LR: 3.89e-04 | 2779.3ms/step | 7.2 samples/s +Validation at step 14250: Loss = 3.1054, Perplexity = 22.32 +Step 14260/44000 | Loss: 3.0896 | LR: 3.89e-04 | 5486.5ms/step | 3.6 samples/s +Step 14270/44000 | Loss: 3.1081 | LR: 3.89e-04 | 2804.7ms/step | 7.1 samples/s +Step 14280/44000 | Loss: 3.1118 | LR: 3.89e-04 | 2796.7ms/step | 7.2 samples/s +Step 14290/44000 | Loss: 3.0932 | LR: 3.89e-04 | 2764.3ms/step | 7.2 samples/s +Step 14300/44000 | Loss: 3.0705 | LR: 3.88e-04 | 2830.9ms/step | 7.1 samples/s +Step 14310/44000 | Loss: 3.1082 | LR: 3.88e-04 | 2822.8ms/step | 7.1 samples/s +Step 14320/44000 | Loss: 3.1104 | LR: 3.88e-04 | 2707.5ms/step | 7.4 samples/s +Step 14330/44000 | Loss: 3.0683 | LR: 3.88e-04 | 2773.0ms/step | 7.2 samples/s +Step 14340/44000 | Loss: 3.0445 | LR: 3.88e-04 | 2804.6ms/step | 7.1 samples/s +Step 14350/44000 | Loss: 3.0639 | LR: 3.88e-04 | 2733.1ms/step | 7.3 samples/s +Step 14360/44000 | Loss: 3.1102 | LR: 3.87e-04 | 2823.5ms/step | 7.1 samples/s +Step 14370/44000 | Loss: 3.0541 | LR: 3.87e-04 | 2850.9ms/step | 7.0 samples/s +Step 14380/44000 | Loss: 3.1018 | LR: 3.87e-04 | 2867.2ms/step | 7.0 samples/s +Step 14390/44000 | Loss: 3.1026 | LR: 3.87e-04 | 2714.8ms/step | 7.4 samples/s +Step 14400/44000 | Loss: 3.0426 | LR: 3.87e-04 | 2696.4ms/step | 7.4 samples/s +Step 14410/44000 | Loss: 3.1041 | LR: 3.87e-04 | 2860.1ms/step | 7.0 samples/s +Step 14420/44000 | Loss: 3.0633 | LR: 3.87e-04 | 2778.9ms/step | 7.2 samples/s +Step 14430/44000 | Loss: 3.1033 | LR: 3.86e-04 | 2696.9ms/step | 7.4 samples/s +Step 14440/44000 | Loss: 3.0879 | LR: 3.86e-04 | 2720.4ms/step | 7.4 samples/s +Step 14450/44000 | Loss: 3.1176 | LR: 3.86e-04 | 2722.5ms/step | 7.3 samples/s +Step 14460/44000 | Loss: 3.1162 | LR: 3.86e-04 | 2716.9ms/step | 7.4 samples/s +Step 14470/44000 | Loss: 3.0886 | LR: 3.86e-04 | 2793.9ms/step | 7.2 samples/s +Step 14480/44000 | Loss: 3.0718 | LR: 3.86e-04 | 2862.0ms/step | 7.0 samples/s +Step 14490/44000 | Loss: 3.0688 | LR: 3.85e-04 | 2828.0ms/step | 7.1 samples/s +Step 14500/44000 | Loss: 3.0537 | LR: 3.85e-04 | 2830.8ms/step | 7.1 samples/s +Validation at step 14500: Loss = 3.1012, Perplexity = 22.22 +Step 14510/44000 | Loss: 3.0368 | LR: 3.85e-04 | 5402.1ms/step | 3.7 samples/s +Step 14520/44000 | Loss: 3.0509 | LR: 3.85e-04 | 2737.1ms/step | 7.3 samples/s +Step 14530/44000 | Loss: 3.0799 | LR: 3.85e-04 | 2727.6ms/step | 7.3 samples/s +Step 14540/44000 | Loss: 3.1019 | LR: 3.85e-04 | 2718.3ms/step | 7.4 samples/s +Step 14550/44000 | Loss: 3.1020 | LR: 3.85e-04 | 2827.9ms/step | 7.1 samples/s +Step 14560/44000 | Loss: 3.0933 | LR: 3.84e-04 | 2841.2ms/step | 7.0 samples/s +Step 14570/44000 | Loss: 3.0853 | LR: 3.84e-04 | 2732.2ms/step | 7.3 samples/s +Step 14580/44000 | Loss: 3.1206 | LR: 3.84e-04 | 2783.9ms/step | 7.2 samples/s +Step 14590/44000 | Loss: 3.0408 | LR: 3.84e-04 | 2883.6ms/step | 6.9 samples/s +Step 14600/44000 | Loss: 3.1241 | LR: 3.84e-04 | 2805.5ms/step | 7.1 samples/s +Step 14610/44000 | Loss: 3.1109 | LR: 3.84e-04 | 2785.1ms/step | 7.2 samples/s +Step 14620/44000 | Loss: 3.0537 | LR: 3.83e-04 | 2849.1ms/step | 7.0 samples/s +Step 14630/44000 | Loss: 3.0746 | LR: 3.83e-04 | 2836.9ms/step | 7.0 samples/s +Step 14640/44000 | Loss: 3.0776 | LR: 3.83e-04 | 2726.7ms/step | 7.3 samples/s +Step 14650/44000 | Loss: 3.0462 | LR: 3.83e-04 | 2780.6ms/step | 7.2 samples/s +Step 14660/44000 | Loss: 3.0896 | LR: 3.83e-04 | 2773.3ms/step | 7.2 samples/s +Step 14670/44000 | Loss: 3.1116 | LR: 3.83e-04 | 2695.3ms/step | 7.4 samples/s +Step 14680/44000 | Loss: 3.0827 | LR: 3.83e-04 | 2737.6ms/step | 7.3 samples/s +Step 14690/44000 | Loss: 3.0557 | LR: 3.82e-04 | 2760.4ms/step | 7.2 samples/s +Step 14700/44000 | Loss: 3.0526 | LR: 3.82e-04 | 2865.6ms/step | 7.0 samples/s +Step 14710/44000 | Loss: 3.0464 | LR: 3.82e-04 | 2762.9ms/step | 7.2 samples/s +Step 14720/44000 | Loss: 3.1048 | LR: 3.82e-04 | 2763.8ms/step | 7.2 samples/s +Step 14730/44000 | Loss: 3.0466 | LR: 3.82e-04 | 2699.8ms/step | 7.4 samples/s +Step 14740/44000 | Loss: 3.0715 | LR: 3.82e-04 | 2668.6ms/step | 7.5 samples/s +Step 14750/44000 | Loss: 3.0757 | LR: 3.81e-04 | 2726.0ms/step | 7.3 samples/s +Validation at step 14750: Loss = 3.0978, Perplexity = 22.15 +Step 14760/44000 | Loss: 3.0656 | LR: 3.81e-04 | 5558.9ms/step | 3.6 samples/s +Step 14770/44000 | Loss: 3.1062 | LR: 3.81e-04 | 2817.3ms/step | 7.1 samples/s +Step 14780/44000 | Loss: 3.0892 | LR: 3.81e-04 | 2889.1ms/step | 6.9 samples/s +Step 14790/44000 | Loss: 3.0715 | LR: 3.81e-04 | 2750.9ms/step | 7.3 samples/s +Step 14800/44000 | Loss: 3.0939 | LR: 3.81e-04 | 2906.9ms/step | 6.9 samples/s +Step 14810/44000 | Loss: 3.0840 | LR: 3.81e-04 | 3099.9ms/step | 6.5 samples/s +Step 14820/44000 | Loss: 3.0735 | LR: 3.80e-04 | 2726.2ms/step | 7.3 samples/s +Step 14830/44000 | Loss: 3.1024 | LR: 3.80e-04 | 2731.0ms/step | 7.3 samples/s +Step 14840/44000 | Loss: 3.0791 | LR: 3.80e-04 | 2755.8ms/step | 7.3 samples/s +Step 14850/44000 | Loss: 3.0879 | LR: 3.80e-04 | 2811.7ms/step | 7.1 samples/s +Step 14860/44000 | Loss: 3.0671 | LR: 3.80e-04 | 2724.3ms/step | 7.3 samples/s +Step 14870/44000 | Loss: 3.0967 | LR: 3.80e-04 | 2771.9ms/step | 7.2 samples/s +Step 14880/44000 | Loss: 3.0786 | LR: 3.79e-04 | 2811.3ms/step | 7.1 samples/s +Step 14890/44000 | Loss: 3.0843 | LR: 3.79e-04 | 2928.6ms/step | 6.8 samples/s +Step 14900/44000 | Loss: 3.0849 | LR: 3.79e-04 | 2827.4ms/step | 7.1 samples/s +Step 14910/44000 | Loss: 3.0297 | LR: 3.79e-04 | 2804.4ms/step | 7.1 samples/s +Step 14920/44000 | Loss: 3.0766 | LR: 3.79e-04 | 2825.4ms/step | 7.1 samples/s +Step 14930/44000 | Loss: 3.0673 | LR: 3.79e-04 | 2851.7ms/step | 7.0 samples/s +Step 14940/44000 | Loss: 3.0900 | LR: 3.79e-04 | 2769.5ms/step | 7.2 samples/s +Step 14950/44000 | Loss: 3.0511 | LR: 3.78e-04 | 2830.2ms/step | 7.1 samples/s +Step 14960/44000 | Loss: 3.1030 | LR: 3.78e-04 | 2809.3ms/step | 7.1 samples/s +Step 14970/44000 | Loss: 3.0805 | LR: 3.78e-04 | 2910.0ms/step | 6.9 samples/s +Step 14980/44000 | Loss: 3.0664 | LR: 3.78e-04 | 2831.5ms/step | 7.1 samples/s +Step 14990/44000 | Loss: 3.0522 | LR: 3.78e-04 | 2804.8ms/step | 7.1 samples/s +Step 15000/44000 | Loss: 3.0358 | LR: 3.78e-04 | 2785.5ms/step | 7.2 samples/s +Validation at step 15000: Loss = 3.0950, Perplexity = 22.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-15000 +Step 15010/44000 | Loss: 3.1162 | LR: 3.77e-04 | 6034.0ms/step | 3.3 samples/s +Step 15020/44000 | Loss: 3.0472 | LR: 3.77e-04 | 2822.1ms/step | 7.1 samples/s +Step 15030/44000 | Loss: 3.0664 | LR: 3.77e-04 | 2548.4ms/step | 7.8 samples/s +Step 15040/44000 | Loss: 3.0672 | LR: 3.77e-04 | 2748.4ms/step | 7.3 samples/s +Step 15050/44000 | Loss: 3.0634 | LR: 3.77e-04 | 2715.8ms/step | 7.4 samples/s +Step 15060/44000 | Loss: 3.0905 | LR: 3.77e-04 | 2823.0ms/step | 7.1 samples/s +Step 15070/44000 | Loss: 3.0364 | LR: 3.77e-04 | 2791.3ms/step | 7.2 samples/s +Step 15080/44000 | Loss: 3.0714 | LR: 3.76e-04 | 2925.3ms/step | 6.8 samples/s +Step 15090/44000 | Loss: 3.0579 | LR: 3.76e-04 | 2856.0ms/step | 7.0 samples/s +Step 15100/44000 | Loss: 3.0988 | LR: 3.76e-04 | 3012.0ms/step | 6.6 samples/s +Step 15110/44000 | Loss: 3.0568 | LR: 3.76e-04 | 2726.6ms/step | 7.3 samples/s +Step 15120/44000 | Loss: 3.1207 | LR: 3.76e-04 | 2647.9ms/step | 7.6 samples/s +Step 15130/44000 | Loss: 3.0780 | LR: 3.76e-04 | 2711.0ms/step | 7.4 samples/s +Step 15140/44000 | Loss: 3.0699 | LR: 3.75e-04 | 2918.6ms/step | 6.9 samples/s +Step 15150/44000 | Loss: 3.0868 | LR: 3.75e-04 | 2794.1ms/step | 7.2 samples/s +Step 15160/44000 | Loss: 3.0832 | LR: 3.75e-04 | 2748.5ms/step | 7.3 samples/s +Step 15170/44000 | Loss: 3.1312 | LR: 3.75e-04 | 2837.2ms/step | 7.0 samples/s +Step 15180/44000 | Loss: 3.0840 | LR: 3.75e-04 | 2778.8ms/step | 7.2 samples/s +Step 15190/44000 | Loss: 3.0730 | LR: 3.75e-04 | 2814.5ms/step | 7.1 samples/s +Step 15200/44000 | Loss: 3.0715 | LR: 3.74e-04 | 2934.4ms/step | 6.8 samples/s +Step 15210/44000 | Loss: 3.0992 | LR: 3.74e-04 | 2872.3ms/step | 7.0 samples/s +Step 15220/44000 | Loss: 3.0587 | LR: 3.74e-04 | 2902.8ms/step | 6.9 samples/s +Step 15230/44000 | Loss: 3.0609 | LR: 3.74e-04 | 2776.5ms/step | 7.2 samples/s +Step 15240/44000 | Loss: 3.0702 | LR: 3.74e-04 | 2851.9ms/step | 7.0 samples/s +Step 15250/44000 | Loss: 3.0865 | LR: 3.74e-04 | 2825.7ms/step | 7.1 samples/s +Validation at step 15250: Loss = 3.0911, Perplexity = 22.00 +Step 15260/44000 | Loss: 3.0488 | LR: 3.74e-04 | 5603.3ms/step | 3.6 samples/s +Step 15270/44000 | Loss: 3.0780 | LR: 3.73e-04 | 2887.1ms/step | 6.9 samples/s +Step 15280/44000 | Loss: 3.1125 | LR: 3.73e-04 | 2810.7ms/step | 7.1 samples/s +Step 15290/44000 | Loss: 3.0721 | LR: 3.73e-04 | 2896.1ms/step | 6.9 samples/s +Step 15300/44000 | Loss: 3.0569 | LR: 3.73e-04 | 2883.7ms/step | 6.9 samples/s +Step 15310/44000 | Loss: 3.0394 | LR: 3.73e-04 | 2732.3ms/step | 7.3 samples/s +Step 15320/44000 | Loss: 3.0717 | LR: 3.73e-04 | 2818.2ms/step | 7.1 samples/s +Step 15330/44000 | Loss: 3.0880 | LR: 3.72e-04 | 2883.3ms/step | 6.9 samples/s +Step 15340/44000 | Loss: 3.0921 | LR: 3.72e-04 | 2859.1ms/step | 7.0 samples/s +Step 15350/44000 | Loss: 3.0787 | LR: 3.72e-04 | 2904.1ms/step | 6.9 samples/s +Step 15360/44000 | Loss: 3.0946 | LR: 3.72e-04 | 2886.3ms/step | 6.9 samples/s +Step 15370/44000 | Loss: 3.0504 | LR: 3.72e-04 | 2783.0ms/step | 7.2 samples/s +Step 15380/44000 | Loss: 3.0818 | LR: 3.72e-04 | 2844.9ms/step | 7.0 samples/s +Step 15390/44000 | Loss: 3.0573 | LR: 3.71e-04 | 2899.6ms/step | 6.9 samples/s +Step 15400/44000 | Loss: 3.1067 | LR: 3.71e-04 | 3190.9ms/step | 6.3 samples/s +Step 15410/44000 | Loss: 3.0796 | LR: 3.71e-04 | 3044.0ms/step | 6.6 samples/s +Step 15420/44000 | Loss: 3.0959 | LR: 3.71e-04 | 2793.9ms/step | 7.2 samples/s +Step 15430/44000 | Loss: 3.0884 | LR: 3.71e-04 | 2762.6ms/step | 7.2 samples/s +Step 15440/44000 | Loss: 3.0400 | LR: 3.71e-04 | 2796.4ms/step | 7.2 samples/s +Step 15450/44000 | Loss: 3.0678 | LR: 3.71e-04 | 2781.5ms/step | 7.2 samples/s +Step 15460/44000 | Loss: 3.0837 | LR: 3.70e-04 | 2875.0ms/step | 7.0 samples/s +Step 15470/44000 | Loss: 3.0548 | LR: 3.70e-04 | 2884.6ms/step | 6.9 samples/s +Step 15480/44000 | Loss: 3.0771 | LR: 3.70e-04 | 2866.2ms/step | 7.0 samples/s +Step 15490/44000 | Loss: 3.0657 | LR: 3.70e-04 | 2830.6ms/step | 7.1 samples/s +Step 15500/44000 | Loss: 3.0899 | LR: 3.70e-04 | 2806.3ms/step | 7.1 samples/s +Validation at step 15500: Loss = 3.0892, Perplexity = 21.96 +Step 15510/44000 | Loss: 3.0744 | LR: 3.70e-04 | 5593.4ms/step | 3.6 samples/s +Step 15520/44000 | Loss: 3.0763 | LR: 3.69e-04 | 2857.0ms/step | 7.0 samples/s +Step 15530/44000 | Loss: 3.0507 | LR: 3.69e-04 | 2909.6ms/step | 6.9 samples/s +Step 15540/44000 | Loss: 3.0623 | LR: 3.69e-04 | 2826.5ms/step | 7.1 samples/s +Step 15550/44000 | Loss: 3.1008 | LR: 3.69e-04 | 2761.8ms/step | 7.2 samples/s +Step 15560/44000 | Loss: 3.0723 | LR: 3.69e-04 | 2820.6ms/step | 7.1 samples/s +Step 15570/44000 | Loss: 3.0955 | LR: 3.69e-04 | 2831.3ms/step | 7.1 samples/s +Step 15580/44000 | Loss: 3.0490 | LR: 3.68e-04 | 2773.8ms/step | 7.2 samples/s +Step 15590/44000 | Loss: 3.1318 | LR: 3.68e-04 | 2904.0ms/step | 6.9 samples/s +Step 15600/44000 | Loss: 3.0404 | LR: 3.68e-04 | 2861.9ms/step | 7.0 samples/s +Step 15610/44000 | Loss: 3.0861 | LR: 3.68e-04 | 2874.3ms/step | 7.0 samples/s +Step 15620/44000 | Loss: 3.0188 | LR: 3.68e-04 | 2840.3ms/step | 7.0 samples/s +Step 15630/44000 | Loss: 3.0553 | LR: 3.68e-04 | 2693.9ms/step | 7.4 samples/s +Step 15640/44000 | Loss: 3.0817 | LR: 3.67e-04 | 2827.6ms/step | 7.1 samples/s +Step 15650/44000 | Loss: 3.0651 | LR: 3.67e-04 | 2950.9ms/step | 6.8 samples/s +Step 15660/44000 | Loss: 3.0779 | LR: 3.67e-04 | 2862.8ms/step | 7.0 samples/s +Step 15670/44000 | Loss: 3.0632 | LR: 3.67e-04 | 2867.2ms/step | 7.0 samples/s +Step 15680/44000 | Loss: 3.0569 | LR: 3.67e-04 | 2904.0ms/step | 6.9 samples/s +Step 15690/44000 | Loss: 3.0913 | LR: 3.67e-04 | 2853.2ms/step | 7.0 samples/s +Step 15700/44000 | Loss: 3.0678 | LR: 3.67e-04 | 3017.7ms/step | 6.6 samples/s +Step 15710/44000 | Loss: 3.0720 | LR: 3.66e-04 | 3003.5ms/step | 6.7 samples/s +Step 15720/44000 | Loss: 3.0965 | LR: 3.66e-04 | 2778.1ms/step | 7.2 samples/s +Step 15730/44000 | Loss: 3.1287 | LR: 3.66e-04 | 2945.2ms/step | 6.8 samples/s +Step 15740/44000 | Loss: 3.0783 | LR: 3.66e-04 | 2924.1ms/step | 6.8 samples/s +Step 15750/44000 | Loss: 3.0675 | LR: 3.66e-04 | 2844.1ms/step | 7.0 samples/s +Validation at step 15750: Loss = 3.0843, Perplexity = 21.85 +Step 15760/44000 | Loss: 3.0429 | LR: 3.66e-04 | 5527.9ms/step | 3.6 samples/s +Step 15770/44000 | Loss: 3.0646 | LR: 3.65e-04 | 2908.7ms/step | 6.9 samples/s +Step 15780/44000 | Loss: 3.0757 | LR: 3.65e-04 | 2924.5ms/step | 6.8 samples/s +Step 15790/44000 | Loss: 3.0944 | LR: 3.65e-04 | 2863.3ms/step | 7.0 samples/s +Step 15800/44000 | Loss: 3.0387 | LR: 3.65e-04 | 2885.6ms/step | 6.9 samples/s +Step 15810/44000 | Loss: 3.0629 | LR: 3.65e-04 | 2858.6ms/step | 7.0 samples/s +Step 15820/44000 | Loss: 3.0913 | LR: 3.65e-04 | 2752.1ms/step | 7.3 samples/s +Step 15830/44000 | Loss: 3.0252 | LR: 3.64e-04 | 2533.5ms/step | 7.9 samples/s +Step 15840/44000 | Loss: 3.0738 | LR: 3.64e-04 | 2886.2ms/step | 6.9 samples/s +Step 15850/44000 | Loss: 3.0395 | LR: 3.64e-04 | 2903.8ms/step | 6.9 samples/s +Step 15860/44000 | Loss: 3.0766 | LR: 3.64e-04 | 2777.9ms/step | 7.2 samples/s +Step 15870/44000 | Loss: 3.0220 | LR: 3.64e-04 | 2742.4ms/step | 7.3 samples/s +Step 15880/44000 | Loss: 3.0685 | LR: 3.64e-04 | 2712.0ms/step | 7.4 samples/s +Step 15890/44000 | Loss: 3.0736 | LR: 3.63e-04 | 2714.1ms/step | 7.4 samples/s +Step 15900/44000 | Loss: 3.0444 | LR: 3.63e-04 | 2735.8ms/step | 7.3 samples/s +Step 15910/44000 | Loss: 3.0769 | LR: 3.63e-04 | 2827.9ms/step | 7.1 samples/s +Step 15920/44000 | Loss: 3.0473 | LR: 3.63e-04 | 2798.2ms/step | 7.1 samples/s +Step 15930/44000 | Loss: 3.0805 | LR: 3.63e-04 | 2831.5ms/step | 7.1 samples/s +Step 15940/44000 | Loss: 3.0738 | LR: 3.63e-04 | 2879.3ms/step | 6.9 samples/s +Step 15950/44000 | Loss: 3.0823 | LR: 3.62e-04 | 2878.8ms/step | 6.9 samples/s +Step 15960/44000 | Loss: 3.0817 | LR: 3.62e-04 | 2992.6ms/step | 6.7 samples/s +Step 15970/44000 | Loss: 3.0558 | LR: 3.62e-04 | 3050.7ms/step | 6.6 samples/s +Step 15980/44000 | Loss: 3.0585 | LR: 3.62e-04 | 2971.8ms/step | 6.7 samples/s +Step 15990/44000 | Loss: 3.0434 | LR: 3.62e-04 | 3269.3ms/step | 6.1 samples/s +Step 16000/44000 | Loss: 3.0796 | LR: 3.62e-04 | 2782.7ms/step | 7.2 samples/s +Validation at step 16000: Loss = 3.0808, Perplexity = 21.78 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-16000 +Step 16010/44000 | Loss: 3.0396 | LR: 3.62e-04 | 5941.7ms/step | 3.4 samples/s +Step 16020/44000 | Loss: 3.0885 | LR: 3.61e-04 | 2831.2ms/step | 7.1 samples/s +Step 16030/44000 | Loss: 3.0653 | LR: 3.61e-04 | 2697.7ms/step | 7.4 samples/s +Step 16040/44000 | Loss: 3.0323 | LR: 3.61e-04 | 2778.0ms/step | 7.2 samples/s +Step 16050/44000 | Loss: 3.0515 | LR: 3.61e-04 | 2827.2ms/step | 7.1 samples/s +Step 16060/44000 | Loss: 3.0698 | LR: 3.61e-04 | 2931.6ms/step | 6.8 samples/s +Step 16070/44000 | Loss: 3.0551 | LR: 3.61e-04 | 2943.7ms/step | 6.8 samples/s +Step 16080/44000 | Loss: 3.0710 | LR: 3.60e-04 | 2861.0ms/step | 7.0 samples/s +Step 16090/44000 | Loss: 3.0634 | LR: 3.60e-04 | 2675.3ms/step | 7.5 samples/s +Step 16100/44000 | Loss: 3.0840 | LR: 3.60e-04 | 2656.2ms/step | 7.5 samples/s +Step 16110/44000 | Loss: 3.0346 | LR: 3.60e-04 | 2777.8ms/step | 7.2 samples/s +Step 16120/44000 | Loss: 3.0814 | LR: 3.60e-04 | 2770.6ms/step | 7.2 samples/s +Step 16130/44000 | Loss: 3.0710 | LR: 3.60e-04 | 2773.9ms/step | 7.2 samples/s +Step 16140/44000 | Loss: 3.0366 | LR: 3.59e-04 | 2802.7ms/step | 7.1 samples/s +Step 16150/44000 | Loss: 3.0839 | LR: 3.59e-04 | 2765.4ms/step | 7.2 samples/s +Step 16160/44000 | Loss: 3.0632 | LR: 3.59e-04 | 2770.5ms/step | 7.2 samples/s +Step 16170/44000 | Loss: 3.0500 | LR: 3.59e-04 | 2738.3ms/step | 7.3 samples/s +Step 16180/44000 | Loss: 3.0682 | LR: 3.59e-04 | 2767.1ms/step | 7.2 samples/s +Step 16190/44000 | Loss: 3.0621 | LR: 3.59e-04 | 2796.0ms/step | 7.2 samples/s +Step 16200/44000 | Loss: 3.0428 | LR: 3.58e-04 | 2833.9ms/step | 7.1 samples/s +Step 16210/44000 | Loss: 3.0578 | LR: 3.58e-04 | 2792.5ms/step | 7.2 samples/s +Step 16220/44000 | Loss: 3.0820 | LR: 3.58e-04 | 2792.6ms/step | 7.2 samples/s +Step 16230/44000 | Loss: 3.0504 | LR: 3.58e-04 | 2821.2ms/step | 7.1 samples/s +Step 16240/44000 | Loss: 3.0507 | LR: 3.58e-04 | 2782.7ms/step | 7.2 samples/s +Step 16250/44000 | Loss: 3.0345 | LR: 3.58e-04 | 2908.0ms/step | 6.9 samples/s +Validation at step 16250: Loss = 3.0795, Perplexity = 21.75 +Step 16260/44000 | Loss: 3.0526 | LR: 3.57e-04 | 6075.6ms/step | 3.3 samples/s +Step 16270/44000 | Loss: 3.0277 | LR: 3.57e-04 | 2799.7ms/step | 7.1 samples/s +Step 16280/44000 | Loss: 3.1304 | LR: 3.57e-04 | 2761.3ms/step | 7.2 samples/s +Step 16290/44000 | Loss: 3.0188 | LR: 3.57e-04 | 2673.1ms/step | 7.5 samples/s +Step 16300/44000 | Loss: 3.0551 | LR: 3.57e-04 | 2784.9ms/step | 7.2 samples/s +Step 16310/44000 | Loss: 3.0522 | LR: 3.57e-04 | 2811.3ms/step | 7.1 samples/s +Step 16320/44000 | Loss: 3.0643 | LR: 3.56e-04 | 2896.3ms/step | 6.9 samples/s +Step 16330/44000 | Loss: 3.0689 | LR: 3.56e-04 | 2908.0ms/step | 6.9 samples/s +Step 16340/44000 | Loss: 3.1068 | LR: 3.56e-04 | 2761.6ms/step | 7.2 samples/s +Step 16350/44000 | Loss: 3.0513 | LR: 3.56e-04 | 2803.1ms/step | 7.1 samples/s +Step 16360/44000 | Loss: 3.0749 | LR: 3.56e-04 | 2737.8ms/step | 7.3 samples/s +Step 16370/44000 | Loss: 3.0963 | LR: 3.56e-04 | 2766.1ms/step | 7.2 samples/s +Step 16380/44000 | Loss: 3.0910 | LR: 3.55e-04 | 2891.5ms/step | 6.9 samples/s +Step 16390/44000 | Loss: 3.0570 | LR: 3.55e-04 | 2830.2ms/step | 7.1 samples/s +Step 16400/44000 | Loss: 3.0318 | LR: 3.55e-04 | 2761.8ms/step | 7.2 samples/s +Step 16410/44000 | Loss: 3.0703 | LR: 3.55e-04 | 2799.8ms/step | 7.1 samples/s +Step 16420/44000 | Loss: 3.0134 | LR: 3.55e-04 | 2663.3ms/step | 7.5 samples/s +Step 16430/44000 | Loss: 3.0956 | LR: 3.55e-04 | 2788.3ms/step | 7.2 samples/s +Step 16440/44000 | Loss: 3.0513 | LR: 3.54e-04 | 2813.7ms/step | 7.1 samples/s +Step 16450/44000 | Loss: 3.0347 | LR: 3.54e-04 | 2805.3ms/step | 7.1 samples/s +Step 16460/44000 | Loss: 3.0305 | LR: 3.54e-04 | 2840.3ms/step | 7.0 samples/s +Step 16470/44000 | Loss: 3.0615 | LR: 3.54e-04 | 2778.4ms/step | 7.2 samples/s +Step 16480/44000 | Loss: 3.1038 | LR: 3.54e-04 | 2804.6ms/step | 7.1 samples/s +Step 16490/44000 | Loss: 3.0497 | LR: 3.54e-04 | 2734.7ms/step | 7.3 samples/s +Step 16500/44000 | Loss: 3.1006 | LR: 3.53e-04 | 2741.1ms/step | 7.3 samples/s +Validation at step 16500: Loss = 3.0740, Perplexity = 21.63 +Step 16510/44000 | Loss: 3.0817 | LR: 3.53e-04 | 5565.3ms/step | 3.6 samples/s +Step 16520/44000 | Loss: 3.0839 | LR: 3.53e-04 | 2793.1ms/step | 7.2 samples/s +Step 16530/44000 | Loss: 3.0698 | LR: 3.53e-04 | 2822.2ms/step | 7.1 samples/s +Step 16540/44000 | Loss: 3.0436 | LR: 3.53e-04 | 2818.1ms/step | 7.1 samples/s +Step 16550/44000 | Loss: 3.0848 | LR: 3.53e-04 | 2755.7ms/step | 7.3 samples/s +Step 16560/44000 | Loss: 3.0342 | LR: 3.52e-04 | 3066.8ms/step | 6.5 samples/s +Step 16570/44000 | Loss: 3.0752 | LR: 3.52e-04 | 2895.0ms/step | 6.9 samples/s +Step 16580/44000 | Loss: 3.1057 | LR: 3.52e-04 | 2717.7ms/step | 7.4 samples/s +Step 16590/44000 | Loss: 3.0269 | LR: 3.52e-04 | 2750.7ms/step | 7.3 samples/s +Step 16600/44000 | Loss: 3.0571 | LR: 3.52e-04 | 2710.3ms/step | 7.4 samples/s +Step 16610/44000 | Loss: 3.0882 | LR: 3.52e-04 | 2749.7ms/step | 7.3 samples/s +Step 16620/44000 | Loss: 3.1086 | LR: 3.51e-04 | 2467.7ms/step | 8.1 samples/s +Step 16630/44000 | Loss: 3.0889 | LR: 3.51e-04 | 2688.4ms/step | 7.4 samples/s +Step 16640/44000 | Loss: 3.1032 | LR: 3.51e-04 | 2792.6ms/step | 7.2 samples/s +Step 16650/44000 | Loss: 3.0509 | LR: 3.51e-04 | 2804.0ms/step | 7.1 samples/s +Step 16660/44000 | Loss: 3.0520 | LR: 3.51e-04 | 2746.4ms/step | 7.3 samples/s +Step 16670/44000 | Loss: 3.0286 | LR: 3.51e-04 | 2770.8ms/step | 7.2 samples/s +Step 16680/44000 | Loss: 2.9808 | LR: 3.50e-04 | 2680.7ms/step | 7.5 samples/s +Step 16690/44000 | Loss: 3.0525 | LR: 3.50e-04 | 2752.5ms/step | 7.3 samples/s +Step 16700/44000 | Loss: 3.0808 | LR: 3.50e-04 | 2805.7ms/step | 7.1 samples/s +Step 16710/44000 | Loss: 3.0289 | LR: 3.50e-04 | 2843.2ms/step | 7.0 samples/s +Step 16720/44000 | Loss: 3.0532 | LR: 3.50e-04 | 2834.1ms/step | 7.1 samples/s +Step 16730/44000 | Loss: 3.0805 | LR: 3.50e-04 | 2784.3ms/step | 7.2 samples/s +Step 16740/44000 | Loss: 3.0443 | LR: 3.49e-04 | 2735.3ms/step | 7.3 samples/s +Step 16750/44000 | Loss: 3.0496 | LR: 3.49e-04 | 2725.6ms/step | 7.3 samples/s +Validation at step 16750: Loss = 3.0723, Perplexity = 21.59 +Step 16760/44000 | Loss: 3.0249 | LR: 3.49e-04 | 5515.8ms/step | 3.6 samples/s +Step 16770/44000 | Loss: 3.0714 | LR: 3.49e-04 | 2876.3ms/step | 7.0 samples/s +Step 16780/44000 | Loss: 3.0704 | LR: 3.49e-04 | 2830.3ms/step | 7.1 samples/s +Step 16790/44000 | Loss: 3.0635 | LR: 3.49e-04 | 2818.8ms/step | 7.1 samples/s +Step 16800/44000 | Loss: 2.9974 | LR: 3.48e-04 | 2734.3ms/step | 7.3 samples/s +Step 16810/44000 | Loss: 3.0455 | LR: 3.48e-04 | 2746.8ms/step | 7.3 samples/s +Step 16820/44000 | Loss: 3.0666 | LR: 3.48e-04 | 2757.3ms/step | 7.3 samples/s +Step 16830/44000 | Loss: 3.0547 | LR: 3.48e-04 | 2819.5ms/step | 7.1 samples/s +Step 16840/44000 | Loss: 3.0086 | LR: 3.48e-04 | 2846.5ms/step | 7.0 samples/s +Step 16850/44000 | Loss: 3.0579 | LR: 3.48e-04 | 2835.1ms/step | 7.1 samples/s +Step 16860/44000 | Loss: 3.0253 | LR: 3.47e-04 | 2745.3ms/step | 7.3 samples/s +Step 16870/44000 | Loss: 3.0657 | LR: 3.47e-04 | 2744.6ms/step | 7.3 samples/s +Step 16880/44000 | Loss: 3.0224 | LR: 3.47e-04 | 2727.5ms/step | 7.3 samples/s +Step 16890/44000 | Loss: 3.0596 | LR: 3.47e-04 | 2751.4ms/step | 7.3 samples/s +Step 16900/44000 | Loss: 3.0494 | LR: 3.47e-04 | 2799.2ms/step | 7.1 samples/s +Step 16910/44000 | Loss: 3.0089 | LR: 3.47e-04 | 2829.2ms/step | 7.1 samples/s +Step 16920/44000 | Loss: 3.0771 | LR: 3.46e-04 | 2840.2ms/step | 7.0 samples/s +Step 16930/44000 | Loss: 3.0584 | LR: 3.46e-04 | 2745.3ms/step | 7.3 samples/s +Step 16940/44000 | Loss: 3.0311 | LR: 3.46e-04 | 2729.6ms/step | 7.3 samples/s +Step 16950/44000 | Loss: 3.0694 | LR: 3.46e-04 | 2740.9ms/step | 7.3 samples/s +Step 16960/44000 | Loss: 3.0412 | LR: 3.46e-04 | 2752.6ms/step | 7.3 samples/s +Step 16970/44000 | Loss: 3.0383 | LR: 3.46e-04 | 2836.0ms/step | 7.1 samples/s +Step 16980/44000 | Loss: 3.0276 | LR: 3.45e-04 | 2787.1ms/step | 7.2 samples/s +Step 16990/44000 | Loss: 3.0424 | LR: 3.45e-04 | 2774.5ms/step | 7.2 samples/s +Step 17000/44000 | Loss: 3.0887 | LR: 3.45e-04 | 2793.7ms/step | 7.2 samples/s +Validation at step 17000: Loss = 3.0672, Perplexity = 21.48 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-17000 +Step 17010/44000 | Loss: 3.0756 | LR: 3.45e-04 | 5736.3ms/step | 3.5 samples/s +Step 17020/44000 | Loss: 3.0842 | LR: 3.45e-04 | 2774.6ms/step | 7.2 samples/s +Step 17030/44000 | Loss: 3.0343 | LR: 3.45e-04 | 2818.9ms/step | 7.1 samples/s +Step 17040/44000 | Loss: 3.0670 | LR: 3.44e-04 | 2820.1ms/step | 7.1 samples/s +Step 17050/44000 | Loss: 3.0407 | LR: 3.44e-04 | 2766.7ms/step | 7.2 samples/s +Step 17060/44000 | Loss: 3.0457 | LR: 3.44e-04 | 2725.5ms/step | 7.3 samples/s +Step 17070/44000 | Loss: 3.0368 | LR: 3.44e-04 | 2736.8ms/step | 7.3 samples/s +Step 17080/44000 | Loss: 3.0631 | LR: 3.44e-04 | 2740.7ms/step | 7.3 samples/s +Step 17090/44000 | Loss: 3.0362 | LR: 3.44e-04 | 2828.5ms/step | 7.1 samples/s +Step 17100/44000 | Loss: 3.0417 | LR: 3.43e-04 | 2811.3ms/step | 7.1 samples/s +Step 17110/44000 | Loss: 3.0369 | LR: 3.43e-04 | 2772.0ms/step | 7.2 samples/s +Step 17120/44000 | Loss: 3.0372 | LR: 3.43e-04 | 2826.4ms/step | 7.1 samples/s +Step 17130/44000 | Loss: 3.0474 | LR: 3.43e-04 | 2651.9ms/step | 7.5 samples/s +Step 17140/44000 | Loss: 3.0217 | LR: 3.43e-04 | 2759.2ms/step | 7.2 samples/s +Step 17150/44000 | Loss: 3.0173 | LR: 3.43e-04 | 2739.1ms/step | 7.3 samples/s +Step 17160/44000 | Loss: 3.0397 | LR: 3.42e-04 | 2865.1ms/step | 7.0 samples/s +Step 17170/44000 | Loss: 3.0337 | LR: 3.42e-04 | 2824.1ms/step | 7.1 samples/s +Step 17180/44000 | Loss: 3.0411 | LR: 3.42e-04 | 2865.8ms/step | 7.0 samples/s +Step 17190/44000 | Loss: 3.0466 | LR: 3.42e-04 | 2753.4ms/step | 7.3 samples/s +Step 17200/44000 | Loss: 3.0564 | LR: 3.42e-04 | 2778.6ms/step | 7.2 samples/s +Step 17210/44000 | Loss: 3.0871 | LR: 3.42e-04 | 2728.6ms/step | 7.3 samples/s +Step 17220/44000 | Loss: 3.0223 | LR: 3.41e-04 | 2726.4ms/step | 7.3 samples/s +Step 17230/44000 | Loss: 3.0695 | LR: 3.41e-04 | 2799.9ms/step | 7.1 samples/s +Step 17240/44000 | Loss: 3.0282 | LR: 3.41e-04 | 2908.7ms/step | 6.9 samples/s +Step 17250/44000 | Loss: 3.0103 | LR: 3.41e-04 | 2871.9ms/step | 7.0 samples/s +Validation at step 17250: Loss = 3.0660, Perplexity = 21.46 +Step 17260/44000 | Loss: 3.0517 | LR: 3.41e-04 | 5351.4ms/step | 3.7 samples/s +Step 17270/44000 | Loss: 3.0202 | LR: 3.41e-04 | 2749.4ms/step | 7.3 samples/s +Step 17280/44000 | Loss: 3.0523 | LR: 3.40e-04 | 2824.6ms/step | 7.1 samples/s +Step 17290/44000 | Loss: 3.0657 | LR: 3.40e-04 | 2817.9ms/step | 7.1 samples/s +Step 17300/44000 | Loss: 3.0024 | LR: 3.40e-04 | 2803.5ms/step | 7.1 samples/s +Step 17310/44000 | Loss: 3.0287 | LR: 3.40e-04 | 2827.4ms/step | 7.1 samples/s +Step 17320/44000 | Loss: 3.0378 | LR: 3.40e-04 | 2735.3ms/step | 7.3 samples/s +Step 17330/44000 | Loss: 3.0280 | LR: 3.40e-04 | 2727.7ms/step | 7.3 samples/s +Step 17340/44000 | Loss: 3.0780 | LR: 3.39e-04 | 2783.0ms/step | 7.2 samples/s +Step 17350/44000 | Loss: 3.0315 | LR: 3.39e-04 | 2766.8ms/step | 7.2 samples/s +Step 17360/44000 | Loss: 3.0334 | LR: 3.39e-04 | 2830.1ms/step | 7.1 samples/s +Step 17370/44000 | Loss: 3.0451 | LR: 3.39e-04 | 2804.6ms/step | 7.1 samples/s +Step 17380/44000 | Loss: 3.0399 | LR: 3.39e-04 | 2798.7ms/step | 7.1 samples/s +Step 17390/44000 | Loss: 3.0651 | LR: 3.39e-04 | 2753.5ms/step | 7.3 samples/s +Step 17400/44000 | Loss: 3.0158 | LR: 3.38e-04 | 2766.4ms/step | 7.2 samples/s +Step 17410/44000 | Loss: 3.0619 | LR: 3.38e-04 | 2715.5ms/step | 7.4 samples/s +Step 17420/44000 | Loss: 2.9734 | LR: 3.38e-04 | 2465.0ms/step | 8.1 samples/s +Step 17430/44000 | Loss: 3.0876 | LR: 3.38e-04 | 2762.5ms/step | 7.2 samples/s +Step 17440/44000 | Loss: 3.0544 | LR: 3.38e-04 | 2767.1ms/step | 7.2 samples/s +Step 17450/44000 | Loss: 3.0638 | LR: 3.38e-04 | 2769.0ms/step | 7.2 samples/s +Step 17460/44000 | Loss: 3.0134 | LR: 3.37e-04 | 2698.2ms/step | 7.4 samples/s +Step 17470/44000 | Loss: 3.0543 | LR: 3.37e-04 | 2767.5ms/step | 7.2 samples/s +Step 17480/44000 | Loss: 3.0756 | LR: 3.37e-04 | 2758.2ms/step | 7.3 samples/s +Step 17490/44000 | Loss: 3.0347 | LR: 3.37e-04 | 2803.2ms/step | 7.1 samples/s +Step 17500/44000 | Loss: 3.0639 | LR: 3.37e-04 | 2818.6ms/step | 7.1 samples/s +Validation at step 17500: Loss = 3.0626, Perplexity = 21.38 +Step 17510/44000 | Loss: 3.0725 | LR: 3.37e-04 | 5550.6ms/step | 3.6 samples/s +Step 17520/44000 | Loss: 3.1077 | LR: 3.36e-04 | 2720.7ms/step | 7.4 samples/s +Step 17530/44000 | Loss: 3.0255 | LR: 3.36e-04 | 2748.8ms/step | 7.3 samples/s +Step 17540/44000 | Loss: 3.0543 | LR: 3.36e-04 | 2776.0ms/step | 7.2 samples/s +Step 17550/44000 | Loss: 3.0562 | LR: 3.36e-04 | 2781.5ms/step | 7.2 samples/s +Step 17560/44000 | Loss: 3.0538 | LR: 3.36e-04 | 2763.5ms/step | 7.2 samples/s +Step 17570/44000 | Loss: 3.0341 | LR: 3.35e-04 | 2829.5ms/step | 7.1 samples/s +Step 17580/44000 | Loss: 3.0502 | LR: 3.35e-04 | 2783.5ms/step | 7.2 samples/s +Step 17590/44000 | Loss: 3.0535 | LR: 3.35e-04 | 2700.2ms/step | 7.4 samples/s +Step 17600/44000 | Loss: 3.0216 | LR: 3.35e-04 | 2805.3ms/step | 7.1 samples/s +Step 17610/44000 | Loss: 3.0426 | LR: 3.35e-04 | 2668.3ms/step | 7.5 samples/s +Step 17620/44000 | Loss: 3.0225 | LR: 3.35e-04 | 2705.4ms/step | 7.4 samples/s +Step 17630/44000 | Loss: 3.0263 | LR: 3.34e-04 | 2791.8ms/step | 7.2 samples/s +Step 17640/44000 | Loss: 3.0225 | LR: 3.34e-04 | 2764.2ms/step | 7.2 samples/s +Step 17650/44000 | Loss: 2.9819 | LR: 3.34e-04 | 2716.1ms/step | 7.4 samples/s +Step 17660/44000 | Loss: 3.0210 | LR: 3.34e-04 | 2717.6ms/step | 7.4 samples/s +Step 17670/44000 | Loss: 3.0264 | LR: 3.34e-04 | 2723.9ms/step | 7.3 samples/s +Step 17680/44000 | Loss: 3.0423 | LR: 3.34e-04 | 2803.0ms/step | 7.1 samples/s +Step 17690/44000 | Loss: 3.0369 | LR: 3.33e-04 | 2770.1ms/step | 7.2 samples/s +Step 17700/44000 | Loss: 3.0126 | LR: 3.33e-04 | 2783.7ms/step | 7.2 samples/s +Step 17710/44000 | Loss: 3.0562 | LR: 3.33e-04 | 2751.4ms/step | 7.3 samples/s +Step 17720/44000 | Loss: 3.0611 | LR: 3.33e-04 | 2733.4ms/step | 7.3 samples/s +Step 17730/44000 | Loss: 3.0580 | LR: 3.33e-04 | 2737.9ms/step | 7.3 samples/s +Step 17740/44000 | Loss: 3.0394 | LR: 3.33e-04 | 2694.9ms/step | 7.4 samples/s +Step 17750/44000 | Loss: 3.0345 | LR: 3.32e-04 | 2768.0ms/step | 7.2 samples/s +Validation at step 17750: Loss = 3.0574, Perplexity = 21.27 +Step 17760/44000 | Loss: 3.0402 | LR: 3.32e-04 | 5374.7ms/step | 3.7 samples/s +Step 17770/44000 | Loss: 3.0487 | LR: 3.32e-04 | 2752.8ms/step | 7.3 samples/s +Step 17780/44000 | Loss: 3.0431 | LR: 3.32e-04 | 2752.5ms/step | 7.3 samples/s +Step 17790/44000 | Loss: 3.0419 | LR: 3.32e-04 | 2688.0ms/step | 7.4 samples/s +Step 17800/44000 | Loss: 3.0605 | LR: 3.32e-04 | 2769.0ms/step | 7.2 samples/s +Step 17810/44000 | Loss: 3.0547 | LR: 3.31e-04 | 2783.7ms/step | 7.2 samples/s +Step 17820/44000 | Loss: 3.0635 | LR: 3.31e-04 | 2782.9ms/step | 7.2 samples/s +Step 17830/44000 | Loss: 3.0224 | LR: 3.31e-04 | 2779.5ms/step | 7.2 samples/s +Step 17840/44000 | Loss: 3.0413 | LR: 3.31e-04 | 2804.6ms/step | 7.1 samples/s +Step 17850/44000 | Loss: 3.0025 | LR: 3.31e-04 | 2758.2ms/step | 7.3 samples/s +Step 17860/44000 | Loss: 3.0544 | LR: 3.31e-04 | 2694.7ms/step | 7.4 samples/s +Step 17870/44000 | Loss: 3.0608 | LR: 3.30e-04 | 2731.9ms/step | 7.3 samples/s +Step 17880/44000 | Loss: 3.0333 | LR: 3.30e-04 | 2786.9ms/step | 7.2 samples/s +Step 17890/44000 | Loss: 3.0411 | LR: 3.30e-04 | 2790.8ms/step | 7.2 samples/s +Step 17900/44000 | Loss: 3.0733 | LR: 3.30e-04 | 2805.7ms/step | 7.1 samples/s +Step 17910/44000 | Loss: 3.0434 | LR: 3.30e-04 | 2825.9ms/step | 7.1 samples/s +Step 17920/44000 | Loss: 3.0749 | LR: 3.29e-04 | 2800.2ms/step | 7.1 samples/s +Step 17930/44000 | Loss: 3.1187 | LR: 3.29e-04 | 2886.2ms/step | 6.9 samples/s +Step 17940/44000 | Loss: 3.0308 | LR: 3.29e-04 | 2730.4ms/step | 7.3 samples/s +Step 17950/44000 | Loss: 3.0233 | LR: 3.29e-04 | 3183.4ms/step | 6.3 samples/s +Step 17960/44000 | Loss: 3.0927 | LR: 3.29e-04 | 2987.0ms/step | 6.7 samples/s +Step 17970/44000 | Loss: 3.0686 | LR: 3.29e-04 | 2747.4ms/step | 7.3 samples/s +Step 17980/44000 | Loss: 3.0626 | LR: 3.28e-04 | 2783.5ms/step | 7.2 samples/s +Step 17990/44000 | Loss: 3.0499 | LR: 3.28e-04 | 2742.0ms/step | 7.3 samples/s +Step 18000/44000 | Loss: 3.0290 | LR: 3.28e-04 | 2801.4ms/step | 7.1 samples/s +Validation at step 18000: Loss = 3.0582, Perplexity = 21.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-18000 +Step 18010/44000 | Loss: 3.0117 | LR: 3.28e-04 | 5905.6ms/step | 3.4 samples/s +Step 18020/44000 | Loss: 3.0480 | LR: 3.28e-04 | 2824.5ms/step | 7.1 samples/s +Step 18030/44000 | Loss: 3.0507 | LR: 3.28e-04 | 2825.4ms/step | 7.1 samples/s +Step 18040/44000 | Loss: 3.1343 | LR: 3.27e-04 | 2810.3ms/step | 7.1 samples/s +Step 18050/44000 | Loss: 3.0725 | LR: 3.27e-04 | 2695.6ms/step | 7.4 samples/s +Step 18060/44000 | Loss: 3.0307 | LR: 3.27e-04 | 2723.6ms/step | 7.3 samples/s +Step 18070/44000 | Loss: 3.0378 | LR: 3.27e-04 | 2724.8ms/step | 7.3 samples/s +Step 18080/44000 | Loss: 3.0363 | LR: 3.27e-04 | 2793.5ms/step | 7.2 samples/s +Step 18090/44000 | Loss: 3.0305 | LR: 3.27e-04 | 2966.6ms/step | 6.7 samples/s +Step 18100/44000 | Loss: 3.0498 | LR: 3.26e-04 | 2788.8ms/step | 7.2 samples/s +Step 18110/44000 | Loss: 3.0434 | LR: 3.26e-04 | 2829.2ms/step | 7.1 samples/s +Step 18120/44000 | Loss: 3.0722 | LR: 3.26e-04 | 2732.3ms/step | 7.3 samples/s +Step 18130/44000 | Loss: 3.0878 | LR: 3.26e-04 | 2732.8ms/step | 7.3 samples/s +Step 18140/44000 | Loss: 3.0196 | LR: 3.26e-04 | 2744.9ms/step | 7.3 samples/s +Step 18150/44000 | Loss: 3.0544 | LR: 3.26e-04 | 2816.8ms/step | 7.1 samples/s +Step 18160/44000 | Loss: 3.0439 | LR: 3.25e-04 | 2788.6ms/step | 7.2 samples/s +Step 18170/44000 | Loss: 3.0199 | LR: 3.25e-04 | 2819.0ms/step | 7.1 samples/s +Step 18180/44000 | Loss: 3.0650 | LR: 3.25e-04 | 2846.1ms/step | 7.0 samples/s +Step 18190/44000 | Loss: 3.0186 | LR: 3.25e-04 | 2747.3ms/step | 7.3 samples/s +Step 18200/44000 | Loss: 3.0283 | LR: 3.25e-04 | 2765.1ms/step | 7.2 samples/s +Step 18210/44000 | Loss: 3.0197 | LR: 3.24e-04 | 2715.8ms/step | 7.4 samples/s +Step 18220/44000 | Loss: 3.0818 | LR: 3.24e-04 | 2503.6ms/step | 8.0 samples/s +Step 18230/44000 | Loss: 3.0169 | LR: 3.24e-04 | 2869.3ms/step | 7.0 samples/s +Step 18240/44000 | Loss: 3.0142 | LR: 3.24e-04 | 2802.3ms/step | 7.1 samples/s +Step 18250/44000 | Loss: 3.0698 | LR: 3.24e-04 | 2831.7ms/step | 7.1 samples/s +Validation at step 18250: Loss = 3.0545, Perplexity = 21.21 +Step 18260/44000 | Loss: 3.0314 | LR: 3.24e-04 | 5276.9ms/step | 3.8 samples/s +Step 18270/44000 | Loss: 3.0179 | LR: 3.23e-04 | 2780.1ms/step | 7.2 samples/s +Step 18280/44000 | Loss: 3.0445 | LR: 3.23e-04 | 2804.7ms/step | 7.1 samples/s +Step 18290/44000 | Loss: 3.0668 | LR: 3.23e-04 | 2784.4ms/step | 7.2 samples/s +Step 18300/44000 | Loss: 3.0528 | LR: 3.23e-04 | 2770.2ms/step | 7.2 samples/s +Step 18310/44000 | Loss: 3.0286 | LR: 3.23e-04 | 2794.0ms/step | 7.2 samples/s +Step 18320/44000 | Loss: 3.0248 | LR: 3.23e-04 | 2661.6ms/step | 7.5 samples/s +Step 18330/44000 | Loss: 3.0050 | LR: 3.22e-04 | 2742.0ms/step | 7.3 samples/s +Step 18340/44000 | Loss: 3.0080 | LR: 3.22e-04 | 2749.1ms/step | 7.3 samples/s +Step 18350/44000 | Loss: 3.0320 | LR: 3.22e-04 | 2768.2ms/step | 7.2 samples/s +Step 18360/44000 | Loss: 3.0504 | LR: 3.22e-04 | 2777.3ms/step | 7.2 samples/s +Step 18370/44000 | Loss: 3.0455 | LR: 3.22e-04 | 2790.4ms/step | 7.2 samples/s +Step 18380/44000 | Loss: 3.0228 | LR: 3.22e-04 | 2784.1ms/step | 7.2 samples/s +Step 18390/44000 | Loss: 3.0395 | LR: 3.21e-04 | 2768.2ms/step | 7.2 samples/s +Step 18400/44000 | Loss: 3.0073 | LR: 3.21e-04 | 2727.0ms/step | 7.3 samples/s +Step 18410/44000 | Loss: 3.0148 | LR: 3.21e-04 | 2726.6ms/step | 7.3 samples/s +Step 18420/44000 | Loss: 3.0344 | LR: 3.21e-04 | 2680.7ms/step | 7.5 samples/s +Step 18430/44000 | Loss: 3.0296 | LR: 3.21e-04 | 2775.6ms/step | 7.2 samples/s +Step 18440/44000 | Loss: 3.0033 | LR: 3.20e-04 | 2797.0ms/step | 7.2 samples/s +Step 18450/44000 | Loss: 3.0190 | LR: 3.20e-04 | 2798.1ms/step | 7.1 samples/s +Step 18460/44000 | Loss: 3.0851 | LR: 3.20e-04 | 2821.7ms/step | 7.1 samples/s +Step 18470/44000 | Loss: 3.0217 | LR: 3.20e-04 | 2695.6ms/step | 7.4 samples/s +Step 18480/44000 | Loss: 3.0727 | LR: 3.20e-04 | 2742.3ms/step | 7.3 samples/s +Step 18490/44000 | Loss: 3.0318 | LR: 3.20e-04 | 2731.7ms/step | 7.3 samples/s +Step 18500/44000 | Loss: 3.0609 | LR: 3.19e-04 | 2831.2ms/step | 7.1 samples/s +Validation at step 18500: Loss = 3.0489, Perplexity = 21.09 +Step 18510/44000 | Loss: 3.0195 | LR: 3.19e-04 | 5516.8ms/step | 3.6 samples/s +Step 18520/44000 | Loss: 3.0537 | LR: 3.19e-04 | 2818.4ms/step | 7.1 samples/s +Step 18530/44000 | Loss: 3.0118 | LR: 3.19e-04 | 2682.7ms/step | 7.5 samples/s +Step 18540/44000 | Loss: 3.0497 | LR: 3.19e-04 | 2758.4ms/step | 7.3 samples/s +Step 18550/44000 | Loss: 3.0145 | LR: 3.19e-04 | 2803.4ms/step | 7.1 samples/s +Step 18560/44000 | Loss: 3.0535 | LR: 3.18e-04 | 2779.8ms/step | 7.2 samples/s +Step 18570/44000 | Loss: 3.0215 | LR: 3.18e-04 | 2809.4ms/step | 7.1 samples/s +Step 18580/44000 | Loss: 3.0031 | LR: 3.18e-04 | 2793.4ms/step | 7.2 samples/s +Step 18590/44000 | Loss: 3.0523 | LR: 3.18e-04 | 2818.3ms/step | 7.1 samples/s +Step 18600/44000 | Loss: 3.0262 | LR: 3.18e-04 | 2751.5ms/step | 7.3 samples/s +Step 18610/44000 | Loss: 3.0067 | LR: 3.18e-04 | 2793.9ms/step | 7.2 samples/s +Step 18620/44000 | Loss: 3.0573 | LR: 3.17e-04 | 2728.5ms/step | 7.3 samples/s +Step 18630/44000 | Loss: 3.0121 | LR: 3.17e-04 | 2843.2ms/step | 7.0 samples/s +Step 18640/44000 | Loss: 3.0269 | LR: 3.17e-04 | 2825.1ms/step | 7.1 samples/s +Step 18650/44000 | Loss: 3.0587 | LR: 3.17e-04 | 2818.8ms/step | 7.1 samples/s +Step 18660/44000 | Loss: 2.9928 | LR: 3.17e-04 | 2778.5ms/step | 7.2 samples/s +Step 18670/44000 | Loss: 3.0530 | LR: 3.16e-04 | 2724.7ms/step | 7.3 samples/s +Step 18680/44000 | Loss: 2.9934 | LR: 3.16e-04 | 2738.3ms/step | 7.3 samples/s +Step 18690/44000 | Loss: 3.0075 | LR: 3.16e-04 | 2737.5ms/step | 7.3 samples/s +Step 18700/44000 | Loss: 2.9928 | LR: 3.16e-04 | 2835.3ms/step | 7.1 samples/s +Step 18710/44000 | Loss: 3.0457 | LR: 3.16e-04 | 2777.9ms/step | 7.2 samples/s +Step 18720/44000 | Loss: 3.0110 | LR: 3.16e-04 | 2799.5ms/step | 7.1 samples/s +Step 18730/44000 | Loss: 3.0435 | LR: 3.15e-04 | 2700.7ms/step | 7.4 samples/s +Step 18740/44000 | Loss: 3.0420 | LR: 3.15e-04 | 2719.1ms/step | 7.4 samples/s +Step 18750/44000 | Loss: 3.0262 | LR: 3.15e-04 | 2736.8ms/step | 7.3 samples/s +Validation at step 18750: Loss = 3.0482, Perplexity = 21.08 +Step 18760/44000 | Loss: 3.0560 | LR: 3.15e-04 | 5444.0ms/step | 3.7 samples/s +Step 18770/44000 | Loss: 3.0253 | LR: 3.15e-04 | 2826.2ms/step | 7.1 samples/s +Step 18780/44000 | Loss: 3.0388 | LR: 3.15e-04 | 2800.1ms/step | 7.1 samples/s +Step 18790/44000 | Loss: 3.0548 | LR: 3.14e-04 | 2788.3ms/step | 7.2 samples/s +Step 18800/44000 | Loss: 3.0006 | LR: 3.14e-04 | 2799.0ms/step | 7.1 samples/s +Step 18810/44000 | Loss: 3.0244 | LR: 3.14e-04 | 2710.1ms/step | 7.4 samples/s +Step 18820/44000 | Loss: 3.0284 | LR: 3.14e-04 | 2793.1ms/step | 7.2 samples/s +Step 18830/44000 | Loss: 2.9919 | LR: 3.14e-04 | 2815.7ms/step | 7.1 samples/s +Step 18840/44000 | Loss: 3.0255 | LR: 3.13e-04 | 2780.2ms/step | 7.2 samples/s +Step 18850/44000 | Loss: 3.0754 | LR: 3.13e-04 | 2833.3ms/step | 7.1 samples/s +Step 18860/44000 | Loss: 3.0316 | LR: 3.13e-04 | 2744.8ms/step | 7.3 samples/s +Step 18870/44000 | Loss: 3.0583 | LR: 3.13e-04 | 2776.8ms/step | 7.2 samples/s +Step 18880/44000 | Loss: 3.0651 | LR: 3.13e-04 | 2734.6ms/step | 7.3 samples/s +Step 18890/44000 | Loss: 3.0110 | LR: 3.13e-04 | 2836.0ms/step | 7.1 samples/s +Step 18900/44000 | Loss: 3.0449 | LR: 3.12e-04 | 2840.3ms/step | 7.0 samples/s +Step 18910/44000 | Loss: 3.0025 | LR: 3.12e-04 | 2812.5ms/step | 7.1 samples/s +Step 18920/44000 | Loss: 3.0738 | LR: 3.12e-04 | 2806.2ms/step | 7.1 samples/s +Step 18930/44000 | Loss: 3.0478 | LR: 3.12e-04 | 2747.4ms/step | 7.3 samples/s +Step 18940/44000 | Loss: 3.0003 | LR: 3.12e-04 | 2706.2ms/step | 7.4 samples/s +Step 18950/44000 | Loss: 3.0251 | LR: 3.12e-04 | 2688.6ms/step | 7.4 samples/s +Step 18960/44000 | Loss: 3.0160 | LR: 3.11e-04 | 2721.5ms/step | 7.3 samples/s +Step 18970/44000 | Loss: 3.0502 | LR: 3.11e-04 | 2735.3ms/step | 7.3 samples/s +Step 18980/44000 | Loss: 3.0125 | LR: 3.11e-04 | 2776.5ms/step | 7.2 samples/s +Step 18990/44000 | Loss: 3.0386 | LR: 3.11e-04 | 2738.4ms/step | 7.3 samples/s +Step 19000/44000 | Loss: 3.0346 | LR: 3.11e-04 | 2722.0ms/step | 7.3 samples/s +Validation at step 19000: Loss = 3.0448, Perplexity = 21.01 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-19000 +Step 19010/44000 | Loss: 3.0125 | LR: 3.11e-04 | 5546.6ms/step | 3.6 samples/s +Step 19020/44000 | Loss: 3.0247 | LR: 3.10e-04 | 2829.5ms/step | 7.1 samples/s +Step 19030/44000 | Loss: 3.0553 | LR: 3.10e-04 | 2810.0ms/step | 7.1 samples/s +Step 19040/44000 | Loss: 3.0426 | LR: 3.10e-04 | 2670.3ms/step | 7.5 samples/s +Step 19050/44000 | Loss: 3.0087 | LR: 3.10e-04 | 2722.8ms/step | 7.3 samples/s +Step 19060/44000 | Loss: 3.0014 | LR: 3.10e-04 | 2717.5ms/step | 7.4 samples/s +Step 19070/44000 | Loss: 3.0292 | LR: 3.09e-04 | 2752.2ms/step | 7.3 samples/s +Step 19080/44000 | Loss: 3.0017 | LR: 3.09e-04 | 2778.1ms/step | 7.2 samples/s +Step 19090/44000 | Loss: 3.0054 | LR: 3.09e-04 | 2760.5ms/step | 7.2 samples/s +Step 19100/44000 | Loss: 3.0330 | LR: 3.09e-04 | 2739.4ms/step | 7.3 samples/s +Step 19110/44000 | Loss: 3.0400 | LR: 3.09e-04 | 2798.6ms/step | 7.1 samples/s +Step 19120/44000 | Loss: 3.0334 | LR: 3.09e-04 | 2741.6ms/step | 7.3 samples/s +Step 19130/44000 | Loss: 3.0076 | LR: 3.08e-04 | 2796.2ms/step | 7.2 samples/s +Step 19140/44000 | Loss: 3.0579 | LR: 3.08e-04 | 2809.4ms/step | 7.1 samples/s +Step 19150/44000 | Loss: 3.0204 | LR: 3.08e-04 | 2805.0ms/step | 7.1 samples/s +Step 19160/44000 | Loss: 3.0420 | LR: 3.08e-04 | 2739.1ms/step | 7.3 samples/s +Step 19170/44000 | Loss: 3.0350 | LR: 3.08e-04 | 2742.7ms/step | 7.3 samples/s +Step 19180/44000 | Loss: 3.0491 | LR: 3.08e-04 | 2798.8ms/step | 7.1 samples/s +Step 19190/44000 | Loss: 3.0212 | LR: 3.07e-04 | 2756.1ms/step | 7.3 samples/s +Step 19200/44000 | Loss: 3.0399 | LR: 3.07e-04 | 2763.3ms/step | 7.2 samples/s +Step 19210/44000 | Loss: 3.0213 | LR: 3.07e-04 | 2784.9ms/step | 7.2 samples/s +Step 19220/44000 | Loss: 3.0334 | LR: 3.07e-04 | 2767.2ms/step | 7.2 samples/s +Step 19230/44000 | Loss: 3.0443 | LR: 3.07e-04 | 2815.3ms/step | 7.1 samples/s +Step 19240/44000 | Loss: 3.0088 | LR: 3.06e-04 | 2768.0ms/step | 7.2 samples/s +Step 19250/44000 | Loss: 2.9869 | LR: 3.06e-04 | 2790.8ms/step | 7.2 samples/s +Validation at step 19250: Loss = 3.0418, Perplexity = 20.94 +Step 19260/44000 | Loss: 3.0384 | LR: 3.06e-04 | 5381.6ms/step | 3.7 samples/s +Step 19270/44000 | Loss: 3.0415 | LR: 3.06e-04 | 2908.9ms/step | 6.9 samples/s +Step 19280/44000 | Loss: 3.0186 | LR: 3.06e-04 | 2881.3ms/step | 6.9 samples/s +Step 19290/44000 | Loss: 3.0768 | LR: 3.06e-04 | 2819.7ms/step | 7.1 samples/s +Step 19300/44000 | Loss: 2.9927 | LR: 3.05e-04 | 2802.3ms/step | 7.1 samples/s +Step 19310/44000 | Loss: 3.0468 | LR: 3.05e-04 | 2746.1ms/step | 7.3 samples/s +Step 19320/44000 | Loss: 3.0566 | LR: 3.05e-04 | 2990.5ms/step | 6.7 samples/s +Step 19330/44000 | Loss: 3.0450 | LR: 3.05e-04 | 2860.2ms/step | 7.0 samples/s +Step 19340/44000 | Loss: 3.0489 | LR: 3.05e-04 | 3116.7ms/step | 6.4 samples/s +Step 19350/44000 | Loss: 2.9973 | LR: 3.04e-04 | 2838.8ms/step | 7.0 samples/s +Step 19360/44000 | Loss: 3.0427 | LR: 3.04e-04 | 2695.5ms/step | 7.4 samples/s +Step 19370/44000 | Loss: 3.0058 | LR: 3.04e-04 | 2709.8ms/step | 7.4 samples/s +Step 19380/44000 | Loss: 3.0542 | LR: 3.04e-04 | 2664.4ms/step | 7.5 samples/s +Step 19390/44000 | Loss: 3.0348 | LR: 3.04e-04 | 2764.5ms/step | 7.2 samples/s +Step 19400/44000 | Loss: 3.0343 | LR: 3.04e-04 | 2789.9ms/step | 7.2 samples/s +Step 19410/44000 | Loss: 3.0375 | LR: 3.03e-04 | 2766.3ms/step | 7.2 samples/s +Step 19420/44000 | Loss: 3.0230 | LR: 3.03e-04 | 2779.6ms/step | 7.2 samples/s +Step 19430/44000 | Loss: 3.0026 | LR: 3.03e-04 | 2787.6ms/step | 7.2 samples/s +Step 19440/44000 | Loss: 3.0612 | LR: 3.03e-04 | 2744.0ms/step | 7.3 samples/s +Step 19450/44000 | Loss: 3.0303 | LR: 3.03e-04 | 2743.2ms/step | 7.3 samples/s +Step 19460/44000 | Loss: 2.9996 | LR: 3.03e-04 | 2800.8ms/step | 7.1 samples/s +Step 19470/44000 | Loss: 3.0379 | LR: 3.02e-04 | 2759.2ms/step | 7.2 samples/s +Step 19480/44000 | Loss: 3.0560 | LR: 3.02e-04 | 2766.1ms/step | 7.2 samples/s +Step 19490/44000 | Loss: 2.9737 | LR: 3.02e-04 | 2785.3ms/step | 7.2 samples/s +Step 19500/44000 | Loss: 2.9845 | LR: 3.02e-04 | 2751.0ms/step | 7.3 samples/s +Validation at step 19500: Loss = 3.0406, Perplexity = 20.92 +Step 19510/44000 | Loss: 3.0100 | LR: 3.02e-04 | 5361.5ms/step | 3.7 samples/s +Step 19520/44000 | Loss: 3.0277 | LR: 3.01e-04 | 2781.4ms/step | 7.2 samples/s +Step 19530/44000 | Loss: 3.0278 | LR: 3.01e-04 | 2819.6ms/step | 7.1 samples/s +Step 19540/44000 | Loss: 3.0567 | LR: 3.01e-04 | 2816.9ms/step | 7.1 samples/s +Step 19550/44000 | Loss: 3.0612 | LR: 3.01e-04 | 2772.1ms/step | 7.2 samples/s +Step 19560/44000 | Loss: 3.0329 | LR: 3.01e-04 | 2724.8ms/step | 7.3 samples/s +Step 19570/44000 | Loss: 3.0495 | LR: 3.01e-04 | 2724.6ms/step | 7.3 samples/s +Step 19580/44000 | Loss: 3.0332 | LR: 3.00e-04 | 2783.1ms/step | 7.2 samples/s +Step 19590/44000 | Loss: 3.0187 | LR: 3.00e-04 | 2710.5ms/step | 7.4 samples/s +Step 19600/44000 | Loss: 2.9918 | LR: 3.00e-04 | 2832.4ms/step | 7.1 samples/s +Step 19610/44000 | Loss: 2.9986 | LR: 3.00e-04 | 2782.3ms/step | 7.2 samples/s +Step 19620/44000 | Loss: 3.0185 | LR: 3.00e-04 | 2777.9ms/step | 7.2 samples/s +Step 19630/44000 | Loss: 3.0425 | LR: 3.00e-04 | 2808.7ms/step | 7.1 samples/s +Step 19640/44000 | Loss: 3.0118 | LR: 2.99e-04 | 2730.9ms/step | 7.3 samples/s +Step 19650/44000 | Loss: 2.9699 | LR: 2.99e-04 | 2788.4ms/step | 7.2 samples/s +Step 19660/44000 | Loss: 3.0182 | LR: 2.99e-04 | 2691.3ms/step | 7.4 samples/s +Step 19670/44000 | Loss: 3.0129 | LR: 2.99e-04 | 2578.6ms/step | 7.8 samples/s +Step 19680/44000 | Loss: 3.0199 | LR: 2.99e-04 | 2562.8ms/step | 7.8 samples/s +Step 19690/44000 | Loss: 3.0183 | LR: 2.98e-04 | 2513.5ms/step | 8.0 samples/s +Step 19700/44000 | Loss: 2.9778 | LR: 2.98e-04 | 2440.1ms/step | 8.2 samples/s +Step 19710/44000 | Loss: 3.0149 | LR: 2.98e-04 | 2547.4ms/step | 7.9 samples/s +Step 19720/44000 | Loss: 3.0246 | LR: 2.98e-04 | 2599.9ms/step | 7.7 samples/s +Step 19730/44000 | Loss: 3.0323 | LR: 2.98e-04 | 2568.0ms/step | 7.8 samples/s +Step 19740/44000 | Loss: 3.0269 | LR: 2.98e-04 | 2556.3ms/step | 7.8 samples/s +Step 19750/44000 | Loss: 3.0174 | LR: 2.97e-04 | 2507.5ms/step | 8.0 samples/s +Validation at step 19750: Loss = 3.0371, Perplexity = 20.84 +Step 19760/44000 | Loss: 3.0545 | LR: 2.97e-04 | 4898.0ms/step | 4.1 samples/s +Step 19770/44000 | Loss: 3.0082 | LR: 2.97e-04 | 2609.3ms/step | 7.7 samples/s +Step 19780/44000 | Loss: 3.0121 | LR: 2.97e-04 | 2527.1ms/step | 7.9 samples/s +Step 19790/44000 | Loss: 3.0134 | LR: 2.97e-04 | 2479.0ms/step | 8.1 samples/s +Step 19800/44000 | Loss: 2.9828 | LR: 2.96e-04 | 2570.6ms/step | 7.8 samples/s +Step 19810/44000 | Loss: 3.0354 | LR: 2.96e-04 | 2545.5ms/step | 7.9 samples/s +Step 19820/44000 | Loss: 3.0302 | LR: 2.96e-04 | 2386.2ms/step | 8.4 samples/s +Step 19830/44000 | Loss: 2.9975 | LR: 2.96e-04 | 2536.3ms/step | 7.9 samples/s +Step 19840/44000 | Loss: 2.9906 | LR: 2.96e-04 | 2501.8ms/step | 8.0 samples/s +Step 19850/44000 | Loss: 3.0029 | LR: 2.96e-04 | 2542.6ms/step | 7.9 samples/s +Step 19860/44000 | Loss: 3.0644 | LR: 2.95e-04 | 2589.2ms/step | 7.7 samples/s +Step 19870/44000 | Loss: 3.0448 | LR: 2.95e-04 | 2588.9ms/step | 7.7 samples/s +Step 19880/44000 | Loss: 3.0179 | LR: 2.95e-04 | 2516.2ms/step | 7.9 samples/s +Step 19890/44000 | Loss: 3.0293 | LR: 2.95e-04 | 2500.3ms/step | 8.0 samples/s +Step 19900/44000 | Loss: 3.0050 | LR: 2.95e-04 | 2500.9ms/step | 8.0 samples/s +Step 19910/44000 | Loss: 3.0241 | LR: 2.95e-04 | 2548.5ms/step | 7.8 samples/s +Step 19920/44000 | Loss: 3.0411 | LR: 2.94e-04 | 2574.1ms/step | 7.8 samples/s +Step 19930/44000 | Loss: 3.0468 | LR: 2.94e-04 | 2537.9ms/step | 7.9 samples/s +Step 19940/44000 | Loss: 3.0542 | LR: 2.94e-04 | 2538.3ms/step | 7.9 samples/s +Step 19950/44000 | Loss: 3.0172 | LR: 2.94e-04 | 2512.6ms/step | 8.0 samples/s +Step 19960/44000 | Loss: 3.0425 | LR: 2.94e-04 | 2542.8ms/step | 7.9 samples/s +Step 19970/44000 | Loss: 3.0078 | LR: 2.93e-04 | 2581.9ms/step | 7.7 samples/s +Step 19980/44000 | Loss: 3.0230 | LR: 2.93e-04 | 2584.9ms/step | 7.7 samples/s +Step 19990/44000 | Loss: 3.0262 | LR: 2.93e-04 | 2530.7ms/step | 7.9 samples/s +Step 20000/44000 | Loss: 2.9531 | LR: 2.93e-04 | 2526.6ms/step | 7.9 samples/s +Validation at step 20000: Loss = 3.0351, Perplexity = 20.80 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-20000 +Step 20010/44000 | Loss: 3.0310 | LR: 2.93e-04 | 5309.3ms/step | 3.8 samples/s +Step 20020/44000 | Loss: 3.0249 | LR: 2.93e-04 | 2564.1ms/step | 7.8 samples/s +Step 20030/44000 | Loss: 3.0403 | LR: 2.92e-04 | 2511.6ms/step | 8.0 samples/s +Step 20040/44000 | Loss: 3.0002 | LR: 2.92e-04 | 2518.4ms/step | 7.9 samples/s +Step 20050/44000 | Loss: 2.9852 | LR: 2.92e-04 | 2538.4ms/step | 7.9 samples/s +Step 20060/44000 | Loss: 2.9987 | LR: 2.92e-04 | 2574.3ms/step | 7.8 samples/s +Step 20070/44000 | Loss: 2.9974 | LR: 2.92e-04 | 2563.7ms/step | 7.8 samples/s +Step 20080/44000 | Loss: 3.0093 | LR: 2.91e-04 | 2548.9ms/step | 7.8 samples/s +Step 20090/44000 | Loss: 3.0473 | LR: 2.91e-04 | 2480.6ms/step | 8.1 samples/s +Step 20100/44000 | Loss: 3.0652 | LR: 2.91e-04 | 2496.8ms/step | 8.0 samples/s +Step 20110/44000 | Loss: 3.0004 | LR: 2.91e-04 | 2490.7ms/step | 8.0 samples/s +Step 20120/44000 | Loss: 3.0162 | LR: 2.91e-04 | 2548.7ms/step | 7.8 samples/s +Step 20130/44000 | Loss: 3.0101 | LR: 2.91e-04 | 2491.0ms/step | 8.0 samples/s +Step 20140/44000 | Loss: 3.0750 | LR: 2.90e-04 | 2433.3ms/step | 8.2 samples/s +Step 20150/44000 | Loss: 3.0397 | LR: 2.90e-04 | 2583.5ms/step | 7.7 samples/s +Step 20160/44000 | Loss: 3.0096 | LR: 2.90e-04 | 2571.0ms/step | 7.8 samples/s +Step 20170/44000 | Loss: 2.9824 | LR: 2.90e-04 | 2588.0ms/step | 7.7 samples/s +Step 20180/44000 | Loss: 3.0145 | LR: 2.90e-04 | 2564.8ms/step | 7.8 samples/s +Step 20190/44000 | Loss: 2.9895 | LR: 2.90e-04 | 2407.3ms/step | 8.3 samples/s +Step 20200/44000 | Loss: 3.0223 | LR: 2.89e-04 | 2500.6ms/step | 8.0 samples/s +Step 20210/44000 | Loss: 3.0253 | LR: 2.89e-04 | 2555.1ms/step | 7.8 samples/s +Step 20220/44000 | Loss: 3.0321 | LR: 2.89e-04 | 2552.8ms/step | 7.8 samples/s +Step 20230/44000 | Loss: 3.0328 | LR: 2.89e-04 | 2564.0ms/step | 7.8 samples/s +Step 20240/44000 | Loss: 3.0235 | LR: 2.89e-04 | 2532.5ms/step | 7.9 samples/s +Step 20250/44000 | Loss: 3.0007 | LR: 2.88e-04 | 2490.9ms/step | 8.0 samples/s +Validation at step 20250: Loss = 3.0303, Perplexity = 20.70 +Step 20260/44000 | Loss: 2.9903 | LR: 2.88e-04 | 4924.9ms/step | 4.1 samples/s +Step 20270/44000 | Loss: 3.0457 | LR: 2.88e-04 | 2588.0ms/step | 7.7 samples/s +Step 20280/44000 | Loss: 3.0464 | LR: 2.88e-04 | 2589.9ms/step | 7.7 samples/s +Step 20290/44000 | Loss: 3.0345 | LR: 2.88e-04 | 2491.3ms/step | 8.0 samples/s +Step 20300/44000 | Loss: 3.0086 | LR: 2.88e-04 | 2496.3ms/step | 8.0 samples/s +Step 20310/44000 | Loss: 2.9909 | LR: 2.87e-04 | 2574.0ms/step | 7.8 samples/s +Step 20320/44000 | Loss: 3.0099 | LR: 2.87e-04 | 2597.9ms/step | 7.7 samples/s +Step 20330/44000 | Loss: 2.9897 | LR: 2.87e-04 | 2575.7ms/step | 7.8 samples/s +Step 20340/44000 | Loss: 2.9914 | LR: 2.87e-04 | 2552.7ms/step | 7.8 samples/s +Step 20350/44000 | Loss: 3.0174 | LR: 2.87e-04 | 2454.6ms/step | 8.1 samples/s +Step 20360/44000 | Loss: 3.0050 | LR: 2.86e-04 | 2542.9ms/step | 7.9 samples/s +Step 20370/44000 | Loss: 3.0215 | LR: 2.86e-04 | 2594.9ms/step | 7.7 samples/s +Step 20380/44000 | Loss: 3.0200 | LR: 2.86e-04 | 2561.2ms/step | 7.8 samples/s +Step 20390/44000 | Loss: 3.0242 | LR: 2.86e-04 | 2597.1ms/step | 7.7 samples/s +Step 20400/44000 | Loss: 3.0271 | LR: 2.86e-04 | 2465.1ms/step | 8.1 samples/s +Step 20410/44000 | Loss: 3.0580 | LR: 2.86e-04 | 2490.2ms/step | 8.0 samples/s +Step 20420/44000 | Loss: 3.0673 | LR: 2.85e-04 | 2538.4ms/step | 7.9 samples/s +Step 20430/44000 | Loss: 3.0305 | LR: 2.85e-04 | 2584.0ms/step | 7.7 samples/s +Step 20440/44000 | Loss: 3.0236 | LR: 2.85e-04 | 2553.5ms/step | 7.8 samples/s +Step 20450/44000 | Loss: 2.9759 | LR: 2.85e-04 | 2527.1ms/step | 7.9 samples/s +Step 20460/44000 | Loss: 3.0113 | LR: 2.85e-04 | 2468.8ms/step | 8.1 samples/s +Step 20470/44000 | Loss: 2.9896 | LR: 2.84e-04 | 2563.3ms/step | 7.8 samples/s +Step 20480/44000 | Loss: 2.9757 | LR: 2.84e-04 | 2589.6ms/step | 7.7 samples/s +Step 20490/44000 | Loss: 3.0353 | LR: 2.84e-04 | 2605.0ms/step | 7.7 samples/s +Step 20500/44000 | Loss: 3.0213 | LR: 2.84e-04 | 2624.0ms/step | 7.6 samples/s +Validation at step 20500: Loss = 3.0286, Perplexity = 20.67 +Step 20510/44000 | Loss: 2.9974 | LR: 2.84e-04 | 5352.6ms/step | 3.7 samples/s +Step 20520/44000 | Loss: 3.0174 | LR: 2.84e-04 | 2599.3ms/step | 7.7 samples/s +Step 20530/44000 | Loss: 3.0292 | LR: 2.83e-04 | 2453.7ms/step | 8.2 samples/s +Step 20540/44000 | Loss: 3.0116 | LR: 2.83e-04 | 2423.0ms/step | 8.3 samples/s +Step 20550/44000 | Loss: 3.0289 | LR: 2.83e-04 | 2438.2ms/step | 8.2 samples/s +Step 20560/44000 | Loss: 3.0340 | LR: 2.83e-04 | 2546.7ms/step | 7.9 samples/s +Step 20570/44000 | Loss: 3.0007 | LR: 2.83e-04 | 2525.9ms/step | 7.9 samples/s +Step 20580/44000 | Loss: 2.9983 | LR: 2.83e-04 | 2581.6ms/step | 7.7 samples/s +Step 20590/44000 | Loss: 3.0217 | LR: 2.82e-04 | 2506.0ms/step | 8.0 samples/s +Step 20600/44000 | Loss: 3.0027 | LR: 2.82e-04 | 2480.2ms/step | 8.1 samples/s +Step 20610/44000 | Loss: 3.0032 | LR: 2.82e-04 | 2469.6ms/step | 8.1 samples/s +Step 20620/44000 | Loss: 3.0478 | LR: 2.82e-04 | 2495.4ms/step | 8.0 samples/s +Step 20630/44000 | Loss: 3.0189 | LR: 2.82e-04 | 2413.0ms/step | 8.3 samples/s +Step 20640/44000 | Loss: 3.0188 | LR: 2.81e-04 | 2511.0ms/step | 8.0 samples/s +Step 20650/44000 | Loss: 3.0238 | LR: 2.81e-04 | 2542.3ms/step | 7.9 samples/s +Step 20660/44000 | Loss: 2.9874 | LR: 2.81e-04 | 2485.4ms/step | 8.0 samples/s +Step 20670/44000 | Loss: 2.9860 | LR: 2.81e-04 | 2491.1ms/step | 8.0 samples/s +Step 20680/44000 | Loss: 2.9916 | LR: 2.81e-04 | 2557.3ms/step | 7.8 samples/s +Step 20690/44000 | Loss: 3.0030 | LR: 2.81e-04 | 2584.0ms/step | 7.7 samples/s +Step 20700/44000 | Loss: 2.9751 | LR: 2.80e-04 | 2577.1ms/step | 7.8 samples/s +Step 20710/44000 | Loss: 2.9652 | LR: 2.80e-04 | 2461.9ms/step | 8.1 samples/s +Step 20720/44000 | Loss: 3.0108 | LR: 2.80e-04 | 2511.1ms/step | 8.0 samples/s +Step 20730/44000 | Loss: 3.0012 | LR: 2.80e-04 | 2552.3ms/step | 7.8 samples/s +Step 20740/44000 | Loss: 2.9938 | LR: 2.80e-04 | 2575.3ms/step | 7.8 samples/s +Step 20750/44000 | Loss: 3.0226 | LR: 2.79e-04 | 2694.3ms/step | 7.4 samples/s +Validation at step 20750: Loss = 3.0255, Perplexity = 20.60 +Step 20760/44000 | Loss: 2.9934 | LR: 2.79e-04 | 5398.0ms/step | 3.7 samples/s +Step 20770/44000 | Loss: 3.0072 | LR: 2.79e-04 | 2458.8ms/step | 8.1 samples/s +Step 20780/44000 | Loss: 3.0484 | LR: 2.79e-04 | 2528.1ms/step | 7.9 samples/s +Step 20790/44000 | Loss: 2.9926 | LR: 2.79e-04 | 2578.4ms/step | 7.8 samples/s +Step 20800/44000 | Loss: 2.9870 | LR: 2.79e-04 | 2548.5ms/step | 7.8 samples/s +Step 20810/44000 | Loss: 3.0030 | LR: 2.78e-04 | 2519.8ms/step | 7.9 samples/s +Step 20820/44000 | Loss: 2.9905 | LR: 2.78e-04 | 2516.9ms/step | 7.9 samples/s +Step 20830/44000 | Loss: 3.0446 | LR: 2.78e-04 | 2470.7ms/step | 8.1 samples/s +Step 20840/44000 | Loss: 3.0178 | LR: 2.78e-04 | 2532.2ms/step | 7.9 samples/s +Step 20850/44000 | Loss: 3.0130 | LR: 2.78e-04 | 2479.1ms/step | 8.1 samples/s +Step 20860/44000 | Loss: 3.0326 | LR: 2.77e-04 | 2448.8ms/step | 8.2 samples/s +Step 20870/44000 | Loss: 3.0016 | LR: 2.77e-04 | 2490.1ms/step | 8.0 samples/s +Step 20880/44000 | Loss: 3.0231 | LR: 2.77e-04 | 2589.2ms/step | 7.7 samples/s +Step 20890/44000 | Loss: 3.0266 | LR: 2.77e-04 | 2574.1ms/step | 7.8 samples/s +Step 20900/44000 | Loss: 3.0200 | LR: 2.77e-04 | 2569.6ms/step | 7.8 samples/s +Step 20910/44000 | Loss: 2.9903 | LR: 2.77e-04 | 2405.5ms/step | 8.3 samples/s +Step 20920/44000 | Loss: 3.0120 | LR: 2.76e-04 | 2475.0ms/step | 8.1 samples/s +Step 20930/44000 | Loss: 2.9796 | LR: 2.76e-04 | 2574.0ms/step | 7.8 samples/s +Step 20940/44000 | Loss: 2.9950 | LR: 2.76e-04 | 2576.2ms/step | 7.8 samples/s +Step 20950/44000 | Loss: 2.9708 | LR: 2.76e-04 | 2550.7ms/step | 7.8 samples/s +Step 20960/44000 | Loss: 3.0082 | LR: 2.76e-04 | 2532.2ms/step | 7.9 samples/s +Step 20970/44000 | Loss: 3.0353 | LR: 2.75e-04 | 2487.4ms/step | 8.0 samples/s +Step 20980/44000 | Loss: 2.9865 | LR: 2.75e-04 | 2504.9ms/step | 8.0 samples/s +Step 20990/44000 | Loss: 3.0116 | LR: 2.75e-04 | 2641.8ms/step | 7.6 samples/s +Step 21000/44000 | Loss: 2.9905 | LR: 2.75e-04 | 2699.3ms/step | 7.4 samples/s +Validation at step 21000: Loss = 3.0234, Perplexity = 20.56 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-21000 +Step 21010/44000 | Loss: 3.0611 | LR: 2.75e-04 | 5277.5ms/step | 3.8 samples/s +Step 21020/44000 | Loss: 3.0005 | LR: 2.75e-04 | 2511.9ms/step | 8.0 samples/s +Step 21030/44000 | Loss: 3.0116 | LR: 2.74e-04 | 2523.6ms/step | 7.9 samples/s +Step 21040/44000 | Loss: 3.0509 | LR: 2.74e-04 | 2527.9ms/step | 7.9 samples/s +Step 21050/44000 | Loss: 2.9493 | LR: 2.74e-04 | 2505.2ms/step | 8.0 samples/s +Step 21060/44000 | Loss: 2.9899 | LR: 2.74e-04 | 2394.7ms/step | 8.4 samples/s +Step 21070/44000 | Loss: 2.9670 | LR: 2.74e-04 | 2500.1ms/step | 8.0 samples/s +Step 21080/44000 | Loss: 2.9913 | LR: 2.73e-04 | 2512.2ms/step | 8.0 samples/s +Step 21090/44000 | Loss: 3.0157 | LR: 2.73e-04 | 2512.0ms/step | 8.0 samples/s +Step 21100/44000 | Loss: 3.0216 | LR: 2.73e-04 | 2476.3ms/step | 8.1 samples/s +Step 21110/44000 | Loss: 3.0238 | LR: 2.73e-04 | 2399.5ms/step | 8.3 samples/s +Step 21120/44000 | Loss: 2.9951 | LR: 2.73e-04 | 2645.5ms/step | 7.6 samples/s +Step 21130/44000 | Loss: 2.9890 | LR: 2.73e-04 | 2715.1ms/step | 7.4 samples/s +Step 21140/44000 | Loss: 3.0021 | LR: 2.72e-04 | 2579.2ms/step | 7.8 samples/s +Step 21150/44000 | Loss: 3.0029 | LR: 2.72e-04 | 2538.7ms/step | 7.9 samples/s +Step 21160/44000 | Loss: 3.0118 | LR: 2.72e-04 | 2471.1ms/step | 8.1 samples/s +Step 21170/44000 | Loss: 3.0165 | LR: 2.72e-04 | 2562.4ms/step | 7.8 samples/s +Step 21180/44000 | Loss: 2.9679 | LR: 2.72e-04 | 2567.1ms/step | 7.8 samples/s +Step 21190/44000 | Loss: 3.0054 | LR: 2.71e-04 | 2481.6ms/step | 8.1 samples/s +Step 21200/44000 | Loss: 3.0278 | LR: 2.71e-04 | 2500.6ms/step | 8.0 samples/s +Step 21210/44000 | Loss: 2.9888 | LR: 2.71e-04 | 2459.7ms/step | 8.1 samples/s +Step 21220/44000 | Loss: 2.9970 | LR: 2.71e-04 | 2569.3ms/step | 7.8 samples/s +Step 21230/44000 | Loss: 3.0521 | LR: 2.71e-04 | 2594.2ms/step | 7.7 samples/s +Step 21240/44000 | Loss: 2.9758 | LR: 2.71e-04 | 2489.3ms/step | 8.0 samples/s +Step 21250/44000 | Loss: 2.9853 | LR: 2.70e-04 | 2507.4ms/step | 8.0 samples/s +Validation at step 21250: Loss = 3.0215, Perplexity = 20.52 +Step 21260/44000 | Loss: 2.9721 | LR: 2.70e-04 | 4669.6ms/step | 4.3 samples/s +Step 21270/44000 | Loss: 3.0298 | LR: 2.70e-04 | 2586.3ms/step | 7.7 samples/s +Step 21280/44000 | Loss: 3.0301 | LR: 2.70e-04 | 2498.9ms/step | 8.0 samples/s +Step 21290/44000 | Loss: 3.0433 | LR: 2.70e-04 | 2485.5ms/step | 8.0 samples/s +Step 21300/44000 | Loss: 2.9718 | LR: 2.70e-04 | 2490.8ms/step | 8.0 samples/s +Step 21310/44000 | Loss: 3.0295 | LR: 2.69e-04 | 2528.2ms/step | 7.9 samples/s +Step 21320/44000 | Loss: 3.0037 | LR: 2.69e-04 | 2509.8ms/step | 8.0 samples/s +Step 21330/44000 | Loss: 3.0136 | LR: 2.69e-04 | 2474.4ms/step | 8.1 samples/s +Step 21340/44000 | Loss: 3.0601 | LR: 2.69e-04 | 2511.3ms/step | 8.0 samples/s +Step 21350/44000 | Loss: 3.0082 | LR: 2.69e-04 | 2568.1ms/step | 7.8 samples/s +Step 21360/44000 | Loss: 2.9628 | LR: 2.68e-04 | 2505.2ms/step | 8.0 samples/s +Step 21370/44000 | Loss: 2.9994 | LR: 2.68e-04 | 2552.4ms/step | 7.8 samples/s +Step 21380/44000 | Loss: 3.0275 | LR: 2.68e-04 | 2495.8ms/step | 8.0 samples/s +Step 21390/44000 | Loss: 3.0357 | LR: 2.68e-04 | 2608.8ms/step | 7.7 samples/s +Step 21400/44000 | Loss: 2.9930 | LR: 2.68e-04 | 2531.3ms/step | 7.9 samples/s +Step 21410/44000 | Loss: 2.9706 | LR: 2.68e-04 | 2494.2ms/step | 8.0 samples/s +Step 21420/44000 | Loss: 3.0339 | LR: 2.67e-04 | 2546.9ms/step | 7.9 samples/s +Step 21430/44000 | Loss: 3.0235 | LR: 2.67e-04 | 2505.1ms/step | 8.0 samples/s +Step 21440/44000 | Loss: 2.9873 | LR: 2.67e-04 | 2450.6ms/step | 8.2 samples/s +Step 21450/44000 | Loss: 2.9878 | LR: 2.67e-04 | 2284.7ms/step | 8.8 samples/s +Step 21460/44000 | Loss: 2.9656 | LR: 2.67e-04 | 2375.4ms/step | 8.4 samples/s +Step 21470/44000 | Loss: 2.9983 | LR: 2.66e-04 | 2485.1ms/step | 8.0 samples/s +Step 21480/44000 | Loss: 3.0423 | LR: 2.66e-04 | 2500.6ms/step | 8.0 samples/s +Step 21490/44000 | Loss: 2.9738 | LR: 2.66e-04 | 2554.4ms/step | 7.8 samples/s +Step 21500/44000 | Loss: 3.0169 | LR: 2.66e-04 | 2497.9ms/step | 8.0 samples/s +Validation at step 21500: Loss = 3.0188, Perplexity = 20.47 +Step 21510/44000 | Loss: 3.0361 | LR: 2.66e-04 | 4643.0ms/step | 4.3 samples/s +Step 21520/44000 | Loss: 2.9856 | LR: 2.66e-04 | 2463.5ms/step | 8.1 samples/s +Step 21530/44000 | Loss: 3.0086 | LR: 2.65e-04 | 2511.4ms/step | 8.0 samples/s +Step 21540/44000 | Loss: 3.0376 | LR: 2.65e-04 | 2480.3ms/step | 8.1 samples/s +Step 21550/44000 | Loss: 3.0126 | LR: 2.65e-04 | 2513.0ms/step | 8.0 samples/s +Step 21560/44000 | Loss: 2.9655 | LR: 2.65e-04 | 2501.0ms/step | 8.0 samples/s +Step 21570/44000 | Loss: 2.9879 | LR: 2.65e-04 | 2523.6ms/step | 7.9 samples/s +Step 21580/44000 | Loss: 3.0180 | LR: 2.64e-04 | 2498.5ms/step | 8.0 samples/s +Step 21590/44000 | Loss: 3.0025 | LR: 2.64e-04 | 2458.9ms/step | 8.1 samples/s +Step 21600/44000 | Loss: 2.9938 | LR: 2.64e-04 | 2480.3ms/step | 8.1 samples/s +Step 21610/44000 | Loss: 2.9726 | LR: 2.64e-04 | 2495.9ms/step | 8.0 samples/s +Step 21620/44000 | Loss: 3.0108 | LR: 2.64e-04 | 2495.8ms/step | 8.0 samples/s +Step 21630/44000 | Loss: 2.9464 | LR: 2.64e-04 | 2506.3ms/step | 8.0 samples/s +Step 21640/44000 | Loss: 3.0601 | LR: 2.63e-04 | 2453.2ms/step | 8.2 samples/s +Step 21650/44000 | Loss: 3.0004 | LR: 2.63e-04 | 2442.5ms/step | 8.2 samples/s +Step 21660/44000 | Loss: 3.0166 | LR: 2.63e-04 | 2417.9ms/step | 8.3 samples/s +Step 21670/44000 | Loss: 3.0193 | LR: 2.63e-04 | 2453.7ms/step | 8.2 samples/s +Step 21680/44000 | Loss: 2.9764 | LR: 2.63e-04 | 2446.9ms/step | 8.2 samples/s +Step 21690/44000 | Loss: 3.0075 | LR: 2.62e-04 | 2481.8ms/step | 8.1 samples/s +Step 21700/44000 | Loss: 2.9697 | LR: 2.62e-04 | 2550.1ms/step | 7.8 samples/s +Step 21710/44000 | Loss: 3.0120 | LR: 2.62e-04 | 2513.8ms/step | 8.0 samples/s +Step 21720/44000 | Loss: 3.0356 | LR: 2.62e-04 | 2485.2ms/step | 8.0 samples/s +Step 21730/44000 | Loss: 3.0171 | LR: 2.62e-04 | 2474.1ms/step | 8.1 samples/s +Step 21740/44000 | Loss: 3.0208 | LR: 2.62e-04 | 2463.6ms/step | 8.1 samples/s +Step 21750/44000 | Loss: 2.9710 | LR: 2.61e-04 | 2507.0ms/step | 8.0 samples/s +Validation at step 21750: Loss = 3.0161, Perplexity = 20.41 +Step 21760/44000 | Loss: 2.9733 | LR: 2.61e-04 | 4771.3ms/step | 4.2 samples/s +Step 21770/44000 | Loss: 3.0263 | LR: 2.61e-04 | 2550.2ms/step | 7.8 samples/s +Step 21780/44000 | Loss: 2.9588 | LR: 2.61e-04 | 2482.8ms/step | 8.1 samples/s +Step 21790/44000 | Loss: 2.9866 | LR: 2.61e-04 | 2469.2ms/step | 8.1 samples/s +Step 21800/44000 | Loss: 2.9795 | LR: 2.60e-04 | 2495.6ms/step | 8.0 samples/s +Step 21810/44000 | Loss: 2.9882 | LR: 2.60e-04 | 2542.5ms/step | 7.9 samples/s +Step 21820/44000 | Loss: 2.9685 | LR: 2.60e-04 | 2595.0ms/step | 7.7 samples/s +Step 21830/44000 | Loss: 2.9847 | LR: 2.60e-04 | 2535.3ms/step | 7.9 samples/s +Step 21840/44000 | Loss: 2.9968 | LR: 2.60e-04 | 2565.7ms/step | 7.8 samples/s +Step 21850/44000 | Loss: 2.9761 | LR: 2.60e-04 | 2544.0ms/step | 7.9 samples/s +Step 21860/44000 | Loss: 3.0412 | LR: 2.59e-04 | 2539.1ms/step | 7.9 samples/s +Step 21870/44000 | Loss: 2.9929 | LR: 2.59e-04 | 2538.1ms/step | 7.9 samples/s +Step 21880/44000 | Loss: 2.9865 | LR: 2.59e-04 | 2568.5ms/step | 7.8 samples/s +Step 21890/44000 | Loss: 2.9313 | LR: 2.59e-04 | 2524.2ms/step | 7.9 samples/s +Step 21900/44000 | Loss: 3.0244 | LR: 2.59e-04 | 2548.8ms/step | 7.8 samples/s +Step 21910/44000 | Loss: 3.0207 | LR: 2.58e-04 | 2568.6ms/step | 7.8 samples/s +Step 21920/44000 | Loss: 3.0154 | LR: 2.58e-04 | 2532.3ms/step | 7.9 samples/s +Step 21930/44000 | Loss: 3.0307 | LR: 2.58e-04 | 2947.1ms/step | 6.8 samples/s +Step 21940/44000 | Loss: 3.0391 | LR: 2.58e-04 | 3193.5ms/step | 6.3 samples/s +Step 21950/44000 | Loss: 2.9951 | LR: 2.58e-04 | 3131.4ms/step | 6.4 samples/s +Step 21960/44000 | Loss: 3.0285 | LR: 2.58e-04 | 3220.3ms/step | 6.2 samples/s +Step 21970/44000 | Loss: 2.9371 | LR: 2.57e-04 | 3312.1ms/step | 6.0 samples/s +Step 21980/44000 | Loss: 2.9926 | LR: 2.57e-04 | 3351.3ms/step | 6.0 samples/s +Step 21990/44000 | Loss: 2.9824 | LR: 2.57e-04 | 3222.6ms/step | 6.2 samples/s +Step 22000/44000 | Loss: 3.0221 | LR: 2.57e-04 | 3170.0ms/step | 6.3 samples/s +Validation at step 22000: Loss = 3.0146, Perplexity = 20.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-22000 +Step 22010/44000 | Loss: 3.0014 | LR: 2.57e-04 | 6804.1ms/step | 2.9 samples/s +Step 22020/44000 | Loss: 3.0060 | LR: 2.56e-04 | 3225.6ms/step | 6.2 samples/s +Step 22030/44000 | Loss: 3.0269 | LR: 2.56e-04 | 3253.3ms/step | 6.1 samples/s +Step 22040/44000 | Loss: 2.9994 | LR: 2.56e-04 | 3158.2ms/step | 6.3 samples/s +Step 22050/44000 | Loss: 3.0097 | LR: 2.56e-04 | 2538.3ms/step | 7.9 samples/s +Step 22060/44000 | Loss: 3.0035 | LR: 2.56e-04 | 2547.5ms/step | 7.9 samples/s +Step 22070/44000 | Loss: 3.0199 | LR: 2.56e-04 | 2579.2ms/step | 7.8 samples/s +Step 22080/44000 | Loss: 2.9735 | LR: 2.55e-04 | 2718.1ms/step | 7.4 samples/s +Step 22090/44000 | Loss: 2.9948 | LR: 2.55e-04 | 2869.1ms/step | 7.0 samples/s +Step 22100/44000 | Loss: 2.9957 | LR: 2.55e-04 | 2532.6ms/step | 7.9 samples/s +Step 22110/44000 | Loss: 2.9842 | LR: 2.55e-04 | 2475.3ms/step | 8.1 samples/s +Step 22120/44000 | Loss: 2.9928 | LR: 2.55e-04 | 2475.3ms/step | 8.1 samples/s +Step 22130/44000 | Loss: 2.9918 | LR: 2.54e-04 | 2517.2ms/step | 7.9 samples/s +Step 22140/44000 | Loss: 3.0034 | LR: 2.54e-04 | 2589.4ms/step | 7.7 samples/s +Step 22150/44000 | Loss: 3.0270 | LR: 2.54e-04 | 2621.5ms/step | 7.6 samples/s +Step 22160/44000 | Loss: 2.9934 | LR: 2.54e-04 | 2588.4ms/step | 7.7 samples/s +Step 22170/44000 | Loss: 3.0161 | LR: 2.54e-04 | 2518.6ms/step | 7.9 samples/s +Step 22180/44000 | Loss: 2.9960 | LR: 2.54e-04 | 2485.4ms/step | 8.0 samples/s +Step 22190/44000 | Loss: 2.9694 | LR: 2.53e-04 | 2508.8ms/step | 8.0 samples/s +Step 22200/44000 | Loss: 2.9896 | LR: 2.53e-04 | 2584.5ms/step | 7.7 samples/s +Step 22210/44000 | Loss: 3.0154 | LR: 2.53e-04 | 2606.2ms/step | 7.7 samples/s +Step 22220/44000 | Loss: 3.0089 | LR: 2.53e-04 | 2563.9ms/step | 7.8 samples/s +Step 22230/44000 | Loss: 3.0002 | LR: 2.53e-04 | 2517.0ms/step | 7.9 samples/s +Step 22240/44000 | Loss: 3.0259 | LR: 2.52e-04 | 2512.8ms/step | 8.0 samples/s +Step 22250/44000 | Loss: 2.9686 | LR: 2.52e-04 | 2868.9ms/step | 7.0 samples/s +Validation at step 22250: Loss = 3.0092, Perplexity = 20.27 +Step 22260/44000 | Loss: 2.9443 | LR: 2.52e-04 | 5834.8ms/step | 3.4 samples/s +Step 22270/44000 | Loss: 3.0014 | LR: 2.52e-04 | 3173.7ms/step | 6.3 samples/s +Step 22280/44000 | Loss: 3.0299 | LR: 2.52e-04 | 3195.4ms/step | 6.3 samples/s +Step 22290/44000 | Loss: 2.9794 | LR: 2.52e-04 | 3188.9ms/step | 6.3 samples/s +Step 22300/44000 | Loss: 2.9554 | LR: 2.51e-04 | 3241.0ms/step | 6.2 samples/s +Step 22310/44000 | Loss: 3.0080 | LR: 2.51e-04 | 3214.9ms/step | 6.2 samples/s +Step 22320/44000 | Loss: 2.9848 | LR: 2.51e-04 | 3338.3ms/step | 6.0 samples/s +Step 22330/44000 | Loss: 2.9922 | LR: 2.51e-04 | 3255.0ms/step | 6.1 samples/s +Step 22340/44000 | Loss: 2.9722 | LR: 2.51e-04 | 3259.7ms/step | 6.1 samples/s +Step 22350/44000 | Loss: 3.0124 | LR: 2.50e-04 | 3247.6ms/step | 6.2 samples/s +Step 22360/44000 | Loss: 2.9931 | LR: 2.50e-04 | 3214.8ms/step | 6.2 samples/s +Step 22370/44000 | Loss: 2.9472 | LR: 2.50e-04 | 3158.3ms/step | 6.3 samples/s +Step 22380/44000 | Loss: 2.9906 | LR: 2.50e-04 | 3403.6ms/step | 5.9 samples/s +Step 22390/44000 | Loss: 2.9992 | LR: 2.50e-04 | 3300.7ms/step | 6.1 samples/s +Step 22400/44000 | Loss: 2.9565 | LR: 2.50e-04 | 3200.8ms/step | 6.2 samples/s +Step 22410/44000 | Loss: 2.9977 | LR: 2.49e-04 | 2760.9ms/step | 7.2 samples/s +Step 22420/44000 | Loss: 3.0121 | LR: 2.49e-04 | 2572.9ms/step | 7.8 samples/s +Step 22430/44000 | Loss: 3.0068 | LR: 2.49e-04 | 2473.9ms/step | 8.1 samples/s +Step 22440/44000 | Loss: 3.0111 | LR: 2.49e-04 | 2432.8ms/step | 8.2 samples/s +Step 22450/44000 | Loss: 2.9884 | LR: 2.49e-04 | 2496.1ms/step | 8.0 samples/s +Step 22460/44000 | Loss: 2.9798 | LR: 2.48e-04 | 2536.8ms/step | 7.9 samples/s +Step 22470/44000 | Loss: 3.0121 | LR: 2.48e-04 | 2516.6ms/step | 7.9 samples/s +Step 22480/44000 | Loss: 3.0503 | LR: 2.48e-04 | 2422.5ms/step | 8.3 samples/s +Step 22490/44000 | Loss: 2.9716 | LR: 2.48e-04 | 2427.3ms/step | 8.2 samples/s +Step 22500/44000 | Loss: 3.0142 | LR: 2.48e-04 | 2538.2ms/step | 7.9 samples/s +Validation at step 22500: Loss = 3.0099, Perplexity = 20.29 +Step 22510/44000 | Loss: 3.0136 | LR: 2.48e-04 | 4882.1ms/step | 4.1 samples/s +Step 22520/44000 | Loss: 2.9800 | LR: 2.47e-04 | 2451.7ms/step | 8.2 samples/s +Step 22530/44000 | Loss: 2.9911 | LR: 2.47e-04 | 2529.9ms/step | 7.9 samples/s +Step 22540/44000 | Loss: 2.9675 | LR: 2.47e-04 | 2595.0ms/step | 7.7 samples/s +Step 22550/44000 | Loss: 2.9724 | LR: 2.47e-04 | 2600.9ms/step | 7.7 samples/s +Step 22560/44000 | Loss: 2.9552 | LR: 2.47e-04 | 2588.3ms/step | 7.7 samples/s +Step 22570/44000 | Loss: 2.9894 | LR: 2.46e-04 | 2432.9ms/step | 8.2 samples/s +Step 22580/44000 | Loss: 2.9810 | LR: 2.46e-04 | 2495.4ms/step | 8.0 samples/s +Step 22590/44000 | Loss: 2.9426 | LR: 2.46e-04 | 2539.9ms/step | 7.9 samples/s +Step 22600/44000 | Loss: 2.9822 | LR: 2.46e-04 | 2631.1ms/step | 7.6 samples/s +Step 22610/44000 | Loss: 2.9732 | LR: 2.46e-04 | 2595.8ms/step | 7.7 samples/s +Step 22620/44000 | Loss: 3.0031 | LR: 2.46e-04 | 2553.4ms/step | 7.8 samples/s +Step 22630/44000 | Loss: 2.9635 | LR: 2.45e-04 | 2504.5ms/step | 8.0 samples/s +Step 22640/44000 | Loss: 2.9684 | LR: 2.45e-04 | 2474.0ms/step | 8.1 samples/s +Step 22650/44000 | Loss: 2.9585 | LR: 2.45e-04 | 2583.5ms/step | 7.7 samples/s +Step 22660/44000 | Loss: 3.0057 | LR: 2.45e-04 | 2589.1ms/step | 7.7 samples/s +Step 22670/44000 | Loss: 3.0329 | LR: 2.45e-04 | 2447.6ms/step | 8.2 samples/s +Step 22680/44000 | Loss: 2.9364 | LR: 2.44e-04 | 2474.3ms/step | 8.1 samples/s +Step 22690/44000 | Loss: 3.0002 | LR: 2.44e-04 | 2558.9ms/step | 7.8 samples/s +Step 22700/44000 | Loss: 3.0044 | LR: 2.44e-04 | 2603.3ms/step | 7.7 samples/s +Step 22710/44000 | Loss: 3.0453 | LR: 2.44e-04 | 2563.1ms/step | 7.8 samples/s +Step 22720/44000 | Loss: 3.0207 | LR: 2.44e-04 | 2458.7ms/step | 8.1 samples/s +Step 22730/44000 | Loss: 3.0452 | LR: 2.44e-04 | 2505.0ms/step | 8.0 samples/s +Step 22740/44000 | Loss: 3.0107 | LR: 2.43e-04 | 2583.7ms/step | 7.7 samples/s +Step 22750/44000 | Loss: 2.9899 | LR: 2.43e-04 | 2583.7ms/step | 7.7 samples/s +Validation at step 22750: Loss = 3.0063, Perplexity = 20.21 +Step 22760/44000 | Loss: 2.9847 | LR: 2.43e-04 | 4941.5ms/step | 4.0 samples/s +Step 22770/44000 | Loss: 2.9663 | LR: 2.43e-04 | 2513.5ms/step | 8.0 samples/s +Step 22780/44000 | Loss: 2.9814 | LR: 2.43e-04 | 2616.4ms/step | 7.6 samples/s +Step 22790/44000 | Loss: 2.9790 | LR: 2.42e-04 | 2603.9ms/step | 7.7 samples/s +Step 22800/44000 | Loss: 2.9492 | LR: 2.42e-04 | 2662.4ms/step | 7.5 samples/s +Step 22810/44000 | Loss: 2.9850 | LR: 2.42e-04 | 2537.2ms/step | 7.9 samples/s +Step 22820/44000 | Loss: 2.9939 | LR: 2.42e-04 | 2583.8ms/step | 7.7 samples/s +Step 22830/44000 | Loss: 2.9924 | LR: 2.42e-04 | 2693.7ms/step | 7.4 samples/s +Step 22840/44000 | Loss: 2.9817 | LR: 2.42e-04 | 2688.8ms/step | 7.4 samples/s +Step 22850/44000 | Loss: 3.0521 | LR: 2.41e-04 | 2600.2ms/step | 7.7 samples/s +Step 22860/44000 | Loss: 2.9846 | LR: 2.41e-04 | 2465.0ms/step | 8.1 samples/s +Step 22870/44000 | Loss: 2.9705 | LR: 2.41e-04 | 2563.0ms/step | 7.8 samples/s +Step 22880/44000 | Loss: 3.0031 | LR: 2.41e-04 | 2636.7ms/step | 7.6 samples/s +Step 22890/44000 | Loss: 3.0162 | LR: 2.41e-04 | 2592.9ms/step | 7.7 samples/s +Step 22900/44000 | Loss: 2.9869 | LR: 2.40e-04 | 2585.0ms/step | 7.7 samples/s +Step 22910/44000 | Loss: 2.9758 | LR: 2.40e-04 | 2589.8ms/step | 7.7 samples/s +Step 22920/44000 | Loss: 2.9668 | LR: 2.40e-04 | 2557.9ms/step | 7.8 samples/s +Step 22930/44000 | Loss: 3.0062 | LR: 2.40e-04 | 2605.7ms/step | 7.7 samples/s +Step 22940/44000 | Loss: 2.9794 | LR: 2.40e-04 | 2599.8ms/step | 7.7 samples/s +Step 22950/44000 | Loss: 3.0071 | LR: 2.40e-04 | 2632.3ms/step | 7.6 samples/s +Step 22960/44000 | Loss: 2.9769 | LR: 2.39e-04 | 2501.2ms/step | 8.0 samples/s +Step 22970/44000 | Loss: 3.0130 | LR: 2.39e-04 | 2491.0ms/step | 8.0 samples/s +Step 22980/44000 | Loss: 2.9898 | LR: 2.39e-04 | 2527.3ms/step | 7.9 samples/s +Step 22990/44000 | Loss: 2.9740 | LR: 2.39e-04 | 2844.9ms/step | 7.0 samples/s +Step 23000/44000 | Loss: 2.9807 | LR: 2.39e-04 | 3223.9ms/step | 6.2 samples/s +Validation at step 23000: Loss = 3.0030, Perplexity = 20.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-23000 +Step 23010/44000 | Loss: 3.0078 | LR: 2.38e-04 | 7645.5ms/step | 2.6 samples/s +Step 23020/44000 | Loss: 2.9919 | LR: 2.38e-04 | 3552.2ms/step | 5.6 samples/s +Step 23030/44000 | Loss: 3.0261 | LR: 2.38e-04 | 3518.5ms/step | 5.7 samples/s +Step 23040/44000 | Loss: 2.9787 | LR: 2.38e-04 | 3383.8ms/step | 5.9 samples/s +Step 23050/44000 | Loss: 2.9924 | LR: 2.38e-04 | 3383.1ms/step | 5.9 samples/s +Step 23060/44000 | Loss: 2.9845 | LR: 2.38e-04 | 3401.7ms/step | 5.9 samples/s +Step 23070/44000 | Loss: 2.9653 | LR: 2.37e-04 | 3313.9ms/step | 6.0 samples/s +Step 23080/44000 | Loss: 2.9952 | LR: 2.37e-04 | 2887.8ms/step | 6.9 samples/s +Step 23090/44000 | Loss: 3.0311 | LR: 2.37e-04 | 3366.8ms/step | 5.9 samples/s +Step 23100/44000 | Loss: 2.9959 | LR: 2.37e-04 | 3664.1ms/step | 5.5 samples/s +Step 23110/44000 | Loss: 2.9929 | LR: 2.37e-04 | 3392.2ms/step | 5.9 samples/s +Step 23120/44000 | Loss: 3.0073 | LR: 2.36e-04 | 3409.7ms/step | 5.9 samples/s +Step 23130/44000 | Loss: 2.9853 | LR: 2.36e-04 | 3401.9ms/step | 5.9 samples/s +Step 23140/44000 | Loss: 2.9854 | LR: 2.36e-04 | 2918.6ms/step | 6.9 samples/s +Step 23150/44000 | Loss: 2.9857 | LR: 2.36e-04 | 2732.2ms/step | 7.3 samples/s +Step 23160/44000 | Loss: 2.9878 | LR: 2.36e-04 | 2689.7ms/step | 7.4 samples/s +Step 23170/44000 | Loss: 2.9855 | LR: 2.36e-04 | 2714.9ms/step | 7.4 samples/s +Step 23180/44000 | Loss: 2.9528 | LR: 2.35e-04 | 2780.3ms/step | 7.2 samples/s +Step 23190/44000 | Loss: 2.9811 | LR: 2.35e-04 | 2792.9ms/step | 7.2 samples/s +Step 23200/44000 | Loss: 2.9800 | LR: 2.35e-04 | 2699.9ms/step | 7.4 samples/s +Step 23210/44000 | Loss: 2.9872 | LR: 2.35e-04 | 2760.1ms/step | 7.2 samples/s +Step 23220/44000 | Loss: 2.9933 | LR: 2.35e-04 | 2650.3ms/step | 7.5 samples/s +Step 23230/44000 | Loss: 2.9731 | LR: 2.34e-04 | 2684.1ms/step | 7.5 samples/s +Step 23240/44000 | Loss: 3.0060 | LR: 2.34e-04 | 2882.8ms/step | 6.9 samples/s +Step 23250/44000 | Loss: 2.9890 | LR: 2.34e-04 | 2866.6ms/step | 7.0 samples/s +Validation at step 23250: Loss = 3.0010, Perplexity = 20.11 +Step 23260/44000 | Loss: 2.9817 | LR: 2.34e-04 | 5671.1ms/step | 3.5 samples/s +Step 23270/44000 | Loss: 3.0225 | LR: 2.34e-04 | 2725.6ms/step | 7.3 samples/s +Step 23280/44000 | Loss: 3.0156 | LR: 2.34e-04 | 2841.4ms/step | 7.0 samples/s +Step 23290/44000 | Loss: 2.9757 | LR: 2.33e-04 | 2915.7ms/step | 6.9 samples/s +Step 23300/44000 | Loss: 2.9639 | LR: 2.33e-04 | 3116.8ms/step | 6.4 samples/s +Step 23310/44000 | Loss: 2.9795 | LR: 2.33e-04 | 2781.4ms/step | 7.2 samples/s +Step 23320/44000 | Loss: 3.0131 | LR: 2.33e-04 | 2721.9ms/step | 7.3 samples/s +Step 23330/44000 | Loss: 2.9884 | LR: 2.33e-04 | 2777.5ms/step | 7.2 samples/s +Step 23340/44000 | Loss: 2.9725 | LR: 2.32e-04 | 3073.1ms/step | 6.5 samples/s +Step 23350/44000 | Loss: 3.0434 | LR: 2.32e-04 | 3268.3ms/step | 6.1 samples/s +Step 23360/44000 | Loss: 2.9892 | LR: 2.32e-04 | 3322.1ms/step | 6.0 samples/s +Step 23370/44000 | Loss: 2.9754 | LR: 2.32e-04 | 3174.3ms/step | 6.3 samples/s +Step 23380/44000 | Loss: 2.9837 | LR: 2.32e-04 | 3238.5ms/step | 6.2 samples/s +Step 23390/44000 | Loss: 3.0204 | LR: 2.32e-04 | 3075.2ms/step | 6.5 samples/s +Step 23400/44000 | Loss: 2.9306 | LR: 2.31e-04 | 3110.2ms/step | 6.4 samples/s +Step 23410/44000 | Loss: 2.9664 | LR: 2.31e-04 | 3137.8ms/step | 6.4 samples/s +Step 23420/44000 | Loss: 3.0107 | LR: 2.31e-04 | 3142.8ms/step | 6.4 samples/s +Step 23430/44000 | Loss: 3.0150 | LR: 2.31e-04 | 3093.7ms/step | 6.5 samples/s +Step 23440/44000 | Loss: 3.0388 | LR: 2.31e-04 | 3112.9ms/step | 6.4 samples/s +Step 23450/44000 | Loss: 2.9734 | LR: 2.30e-04 | 3200.8ms/step | 6.2 samples/s +Step 23460/44000 | Loss: 3.0013 | LR: 2.30e-04 | 3241.1ms/step | 6.2 samples/s +Step 23470/44000 | Loss: 3.0150 | LR: 2.30e-04 | 3153.0ms/step | 6.3 samples/s +Step 23480/44000 | Loss: 3.0093 | LR: 2.30e-04 | 3206.4ms/step | 6.2 samples/s +Step 23490/44000 | Loss: 2.9881 | LR: 2.30e-04 | 3086.4ms/step | 6.5 samples/s +Step 23500/44000 | Loss: 2.9713 | LR: 2.30e-04 | 3088.0ms/step | 6.5 samples/s +Validation at step 23500: Loss = 2.9989, Perplexity = 20.06 +Step 23510/44000 | Loss: 3.0011 | LR: 2.29e-04 | 4909.0ms/step | 4.1 samples/s +Step 23520/44000 | Loss: 3.0096 | LR: 2.29e-04 | 2381.9ms/step | 8.4 samples/s +Step 23530/44000 | Loss: 2.9945 | LR: 2.29e-04 | 2355.7ms/step | 8.5 samples/s +Step 23540/44000 | Loss: 2.9688 | LR: 2.29e-04 | 2245.5ms/step | 8.9 samples/s +Step 23550/44000 | Loss: 2.9872 | LR: 2.29e-04 | 2465.4ms/step | 8.1 samples/s +Step 23560/44000 | Loss: 2.9718 | LR: 2.29e-04 | 2481.4ms/step | 8.1 samples/s +Step 23570/44000 | Loss: 2.9875 | LR: 2.28e-04 | 2380.8ms/step | 8.4 samples/s +Step 23580/44000 | Loss: 2.9817 | LR: 2.28e-04 | 2388.7ms/step | 8.4 samples/s +Step 23590/44000 | Loss: 2.9784 | LR: 2.28e-04 | 2480.9ms/step | 8.1 samples/s +Step 23600/44000 | Loss: 2.9706 | LR: 2.28e-04 | 2522.6ms/step | 7.9 samples/s +Step 23610/44000 | Loss: 2.9798 | LR: 2.28e-04 | 2570.3ms/step | 7.8 samples/s +Step 23620/44000 | Loss: 2.9908 | LR: 2.27e-04 | 2433.6ms/step | 8.2 samples/s +Step 23630/44000 | Loss: 3.0250 | LR: 2.27e-04 | 2423.6ms/step | 8.3 samples/s +Step 23640/44000 | Loss: 2.9543 | LR: 2.27e-04 | 2694.7ms/step | 7.4 samples/s +Step 23650/44000 | Loss: 2.9167 | LR: 2.27e-04 | 3126.3ms/step | 6.4 samples/s +Step 23660/44000 | Loss: 2.9962 | LR: 2.27e-04 | 3143.1ms/step | 6.4 samples/s +Step 23670/44000 | Loss: 2.9689 | LR: 2.27e-04 | 3121.0ms/step | 6.4 samples/s +Step 23680/44000 | Loss: 2.9820 | LR: 2.26e-04 | 3162.2ms/step | 6.3 samples/s +Step 23690/44000 | Loss: 2.9979 | LR: 2.26e-04 | 3077.5ms/step | 6.5 samples/s +Step 23700/44000 | Loss: 2.9836 | LR: 2.26e-04 | 3048.2ms/step | 6.6 samples/s +Step 23710/44000 | Loss: 2.9607 | LR: 2.26e-04 | 2970.7ms/step | 6.7 samples/s +Step 23720/44000 | Loss: 2.9949 | LR: 2.26e-04 | 3124.9ms/step | 6.4 samples/s +Step 23730/44000 | Loss: 2.9624 | LR: 2.25e-04 | 3171.8ms/step | 6.3 samples/s +Step 23740/44000 | Loss: 2.9793 | LR: 2.25e-04 | 3068.5ms/step | 6.5 samples/s +Step 23750/44000 | Loss: 2.9943 | LR: 2.25e-04 | 3113.9ms/step | 6.4 samples/s +Validation at step 23750: Loss = 2.9953, Perplexity = 19.99 +Step 23760/44000 | Loss: 2.9821 | LR: 2.25e-04 | 6058.3ms/step | 3.3 samples/s +Step 23770/44000 | Loss: 2.9641 | LR: 2.25e-04 | 3132.4ms/step | 6.4 samples/s +Step 23780/44000 | Loss: 2.9910 | LR: 2.25e-04 | 3169.8ms/step | 6.3 samples/s +Step 23790/44000 | Loss: 2.9666 | LR: 2.24e-04 | 3157.6ms/step | 6.3 samples/s +Step 23800/44000 | Loss: 2.9930 | LR: 2.24e-04 | 2580.0ms/step | 7.8 samples/s +Step 23810/44000 | Loss: 3.0088 | LR: 2.24e-04 | 2469.9ms/step | 8.1 samples/s +Step 23820/44000 | Loss: 2.9968 | LR: 2.24e-04 | 2356.1ms/step | 8.5 samples/s +Step 23830/44000 | Loss: 2.9792 | LR: 2.24e-04 | 2433.4ms/step | 8.2 samples/s +Step 23840/44000 | Loss: 2.9491 | LR: 2.23e-04 | 2508.2ms/step | 8.0 samples/s +Step 23850/44000 | Loss: 2.9962 | LR: 2.23e-04 | 2474.8ms/step | 8.1 samples/s +Step 23860/44000 | Loss: 2.9537 | LR: 2.23e-04 | 2371.3ms/step | 8.4 samples/s +Step 23870/44000 | Loss: 3.0282 | LR: 2.23e-04 | 2470.4ms/step | 8.1 samples/s +Step 23880/44000 | Loss: 3.0397 | LR: 2.23e-04 | 2532.3ms/step | 7.9 samples/s +Step 23890/44000 | Loss: 2.9466 | LR: 2.23e-04 | 2504.3ms/step | 8.0 samples/s +Step 23900/44000 | Loss: 2.9890 | LR: 2.22e-04 | 2468.2ms/step | 8.1 samples/s +Step 23910/44000 | Loss: 2.9953 | LR: 2.22e-04 | 2407.5ms/step | 8.3 samples/s +Step 23920/44000 | Loss: 2.9850 | LR: 2.22e-04 | 2273.9ms/step | 8.8 samples/s +Step 23930/44000 | Loss: 2.9697 | LR: 2.22e-04 | 2994.3ms/step | 6.7 samples/s +Step 23940/44000 | Loss: 2.9615 | LR: 2.22e-04 | 3173.0ms/step | 6.3 samples/s +Step 23950/44000 | Loss: 2.9470 | LR: 2.21e-04 | 3273.7ms/step | 6.1 samples/s +Step 23960/44000 | Loss: 2.9893 | LR: 2.21e-04 | 3102.3ms/step | 6.4 samples/s +Step 23970/44000 | Loss: 2.9607 | LR: 2.21e-04 | 3183.7ms/step | 6.3 samples/s +Step 23980/44000 | Loss: 3.0163 | LR: 2.21e-04 | 3294.5ms/step | 6.1 samples/s +Step 23990/44000 | Loss: 2.9657 | LR: 2.21e-04 | 3140.3ms/step | 6.4 samples/s +Step 24000/44000 | Loss: 2.9377 | LR: 2.21e-04 | 3099.5ms/step | 6.5 samples/s +Validation at step 24000: Loss = 2.9949, Perplexity = 19.98 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-24000 +Step 24010/44000 | Loss: 2.9900 | LR: 2.20e-04 | 6762.3ms/step | 3.0 samples/s +Step 24020/44000 | Loss: 2.9602 | LR: 2.20e-04 | 3116.6ms/step | 6.4 samples/s +Step 24030/44000 | Loss: 2.9564 | LR: 2.20e-04 | 3174.2ms/step | 6.3 samples/s +Step 24040/44000 | Loss: 2.9673 | LR: 2.20e-04 | 3170.1ms/step | 6.3 samples/s +Step 24050/44000 | Loss: 2.9473 | LR: 2.20e-04 | 3172.8ms/step | 6.3 samples/s +Step 24060/44000 | Loss: 2.9620 | LR: 2.19e-04 | 3186.3ms/step | 6.3 samples/s +Step 24070/44000 | Loss: 2.9986 | LR: 2.19e-04 | 3195.2ms/step | 6.3 samples/s +Step 24080/44000 | Loss: 2.9863 | LR: 2.19e-04 | 3154.1ms/step | 6.3 samples/s +Step 24090/44000 | Loss: 2.9854 | LR: 2.19e-04 | 2423.3ms/step | 8.3 samples/s +Step 24100/44000 | Loss: 2.9746 | LR: 2.19e-04 | 2366.6ms/step | 8.5 samples/s +Step 24110/44000 | Loss: 2.9795 | LR: 2.19e-04 | 2429.5ms/step | 8.2 samples/s +Step 24120/44000 | Loss: 2.9633 | LR: 2.18e-04 | 2382.5ms/step | 8.4 samples/s +Step 24130/44000 | Loss: 2.9869 | LR: 2.18e-04 | 2304.5ms/step | 8.7 samples/s +Step 24140/44000 | Loss: 2.9880 | LR: 2.18e-04 | 2466.5ms/step | 8.1 samples/s +Step 24150/44000 | Loss: 2.9457 | LR: 2.18e-04 | 2480.9ms/step | 8.1 samples/s +Step 24160/44000 | Loss: 2.9919 | LR: 2.18e-04 | 2512.9ms/step | 8.0 samples/s +Step 24170/44000 | Loss: 3.0258 | LR: 2.17e-04 | 2360.3ms/step | 8.5 samples/s +Step 24180/44000 | Loss: 2.9701 | LR: 2.17e-04 | 2418.7ms/step | 8.3 samples/s +Step 24190/44000 | Loss: 2.9644 | LR: 2.17e-04 | 2440.8ms/step | 8.2 samples/s +Step 24200/44000 | Loss: 2.9521 | LR: 2.17e-04 | 2490.8ms/step | 8.0 samples/s +Step 24210/44000 | Loss: 2.9665 | LR: 2.17e-04 | 2429.2ms/step | 8.2 samples/s +Step 24220/44000 | Loss: 2.9409 | LR: 2.17e-04 | 2401.9ms/step | 8.3 samples/s +Step 24230/44000 | Loss: 3.0002 | LR: 2.16e-04 | 2413.4ms/step | 8.3 samples/s +Step 24240/44000 | Loss: 2.9629 | LR: 2.16e-04 | 2461.3ms/step | 8.1 samples/s +Step 24250/44000 | Loss: 2.9636 | LR: 2.16e-04 | 2411.9ms/step | 8.3 samples/s +Validation at step 24250: Loss = 2.9911, Perplexity = 19.91 +Step 24260/44000 | Loss: 2.9701 | LR: 2.16e-04 | 4623.0ms/step | 4.3 samples/s +Step 24270/44000 | Loss: 3.0077 | LR: 2.16e-04 | 2448.9ms/step | 8.2 samples/s +Step 24280/44000 | Loss: 2.9574 | LR: 2.16e-04 | 2453.7ms/step | 8.2 samples/s +Step 24290/44000 | Loss: 2.9834 | LR: 2.15e-04 | 2402.8ms/step | 8.3 samples/s +Step 24300/44000 | Loss: 2.9551 | LR: 2.15e-04 | 2402.6ms/step | 8.3 samples/s +Step 24310/44000 | Loss: 2.9626 | LR: 2.15e-04 | 2444.7ms/step | 8.2 samples/s +Step 24320/44000 | Loss: 3.0038 | LR: 2.15e-04 | 2459.5ms/step | 8.1 samples/s +Step 24330/44000 | Loss: 2.9766 | LR: 2.15e-04 | 2365.9ms/step | 8.5 samples/s +Step 24340/44000 | Loss: 2.9821 | LR: 2.14e-04 | 2423.6ms/step | 8.3 samples/s +Step 24350/44000 | Loss: 2.9642 | LR: 2.14e-04 | 2496.5ms/step | 8.0 samples/s +Step 24360/44000 | Loss: 2.9952 | LR: 2.14e-04 | 2496.7ms/step | 8.0 samples/s +Step 24370/44000 | Loss: 2.9698 | LR: 2.14e-04 | 2434.4ms/step | 8.2 samples/s +Step 24380/44000 | Loss: 2.9637 | LR: 2.14e-04 | 2512.6ms/step | 8.0 samples/s +Step 24390/44000 | Loss: 2.9773 | LR: 2.14e-04 | 2460.5ms/step | 8.1 samples/s +Step 24400/44000 | Loss: 2.9809 | LR: 2.13e-04 | 2527.9ms/step | 7.9 samples/s +Step 24410/44000 | Loss: 2.9833 | LR: 2.13e-04 | 2547.9ms/step | 7.8 samples/s +Step 24420/44000 | Loss: 2.9579 | LR: 2.13e-04 | 2444.7ms/step | 8.2 samples/s +Step 24430/44000 | Loss: 2.9772 | LR: 2.13e-04 | 2470.8ms/step | 8.1 samples/s +Step 24440/44000 | Loss: 2.9403 | LR: 2.13e-04 | 2734.9ms/step | 7.3 samples/s +Step 24450/44000 | Loss: 2.9848 | LR: 2.12e-04 | 3161.0ms/step | 6.3 samples/s +Step 24460/44000 | Loss: 2.9506 | LR: 2.12e-04 | 3294.0ms/step | 6.1 samples/s +Step 24470/44000 | Loss: 3.0166 | LR: 2.12e-04 | 3272.8ms/step | 6.1 samples/s +Step 24480/44000 | Loss: 2.9784 | LR: 2.12e-04 | 3247.0ms/step | 6.2 samples/s +Step 24490/44000 | Loss: 2.9803 | LR: 2.12e-04 | 3173.1ms/step | 6.3 samples/s +Step 24500/44000 | Loss: 3.0091 | LR: 2.12e-04 | 3252.2ms/step | 6.1 samples/s +Validation at step 24500: Loss = 2.9903, Perplexity = 19.89 +Step 24510/44000 | Loss: 2.9049 | LR: 2.11e-04 | 6451.2ms/step | 3.1 samples/s +Step 24520/44000 | Loss: 2.9678 | LR: 2.11e-04 | 3285.1ms/step | 6.1 samples/s +Step 24530/44000 | Loss: 2.9401 | LR: 2.11e-04 | 3436.5ms/step | 5.8 samples/s +Step 24540/44000 | Loss: 2.9513 | LR: 2.11e-04 | 3272.2ms/step | 6.1 samples/s +Step 24550/44000 | Loss: 2.9566 | LR: 2.11e-04 | 3320.6ms/step | 6.0 samples/s +Step 24560/44000 | Loss: 2.9713 | LR: 2.10e-04 | 3295.2ms/step | 6.1 samples/s +Step 24570/44000 | Loss: 2.9962 | LR: 2.10e-04 | 3273.9ms/step | 6.1 samples/s +Step 24580/44000 | Loss: 2.9929 | LR: 2.10e-04 | 3381.3ms/step | 5.9 samples/s +Step 24590/44000 | Loss: 2.9754 | LR: 2.10e-04 | 2777.5ms/step | 7.2 samples/s +Step 24600/44000 | Loss: 2.9761 | LR: 2.10e-04 | 2683.4ms/step | 7.5 samples/s +Step 24610/44000 | Loss: 2.9737 | LR: 2.10e-04 | 2708.2ms/step | 7.4 samples/s +Step 24620/44000 | Loss: 2.9488 | LR: 2.09e-04 | 2659.8ms/step | 7.5 samples/s +Step 24630/44000 | Loss: 2.9237 | LR: 2.09e-04 | 2600.0ms/step | 7.7 samples/s +Step 24640/44000 | Loss: 2.9793 | LR: 2.09e-04 | 2698.9ms/step | 7.4 samples/s +Step 24650/44000 | Loss: 2.9918 | LR: 2.09e-04 | 2594.9ms/step | 7.7 samples/s +Step 24660/44000 | Loss: 2.9396 | LR: 2.09e-04 | 2629.7ms/step | 7.6 samples/s +Step 24670/44000 | Loss: 2.9801 | LR: 2.09e-04 | 2680.4ms/step | 7.5 samples/s +Step 24680/44000 | Loss: 2.9965 | LR: 2.08e-04 | 2643.9ms/step | 7.6 samples/s +Step 24690/44000 | Loss: 2.9689 | LR: 2.08e-04 | 2656.8ms/step | 7.5 samples/s +Step 24700/44000 | Loss: 2.9317 | LR: 2.08e-04 | 2689.0ms/step | 7.4 samples/s +Step 24710/44000 | Loss: 2.9726 | LR: 2.08e-04 | 2694.9ms/step | 7.4 samples/s +Step 24720/44000 | Loss: 2.9797 | LR: 2.08e-04 | 2730.5ms/step | 7.3 samples/s +Step 24730/44000 | Loss: 2.9813 | LR: 2.07e-04 | 2731.5ms/step | 7.3 samples/s +Step 24740/44000 | Loss: 2.9998 | LR: 2.07e-04 | 2678.6ms/step | 7.5 samples/s +Step 24750/44000 | Loss: 2.9886 | LR: 2.07e-04 | 2712.7ms/step | 7.4 samples/s +Validation at step 24750: Loss = 2.9876, Perplexity = 19.84 +Step 24760/44000 | Loss: 2.9535 | LR: 2.07e-04 | 4687.9ms/step | 4.3 samples/s +Step 24770/44000 | Loss: 2.9643 | LR: 2.07e-04 | 2702.4ms/step | 7.4 samples/s +Step 24780/44000 | Loss: 2.9758 | LR: 2.07e-04 | 2696.2ms/step | 7.4 samples/s +Step 24790/44000 | Loss: 2.9248 | LR: 2.06e-04 | 2721.1ms/step | 7.3 samples/s +Step 24800/44000 | Loss: 3.0050 | LR: 2.06e-04 | 2690.4ms/step | 7.4 samples/s +Step 24810/44000 | Loss: 2.9939 | LR: 2.06e-04 | 2674.2ms/step | 7.5 samples/s +Step 24820/44000 | Loss: 2.9708 | LR: 2.06e-04 | 2729.0ms/step | 7.3 samples/s +Step 24830/44000 | Loss: 2.9996 | LR: 2.06e-04 | 2667.8ms/step | 7.5 samples/s +Step 24840/44000 | Loss: 2.9361 | LR: 2.05e-04 | 2659.7ms/step | 7.5 samples/s +Step 24850/44000 | Loss: 2.9526 | LR: 2.05e-04 | 2705.7ms/step | 7.4 samples/s +Step 24860/44000 | Loss: 2.9558 | LR: 2.05e-04 | 2669.9ms/step | 7.5 samples/s +Step 24870/44000 | Loss: 3.0008 | LR: 2.05e-04 | 2683.6ms/step | 7.5 samples/s +Step 24880/44000 | Loss: 2.9724 | LR: 2.05e-04 | 2734.6ms/step | 7.3 samples/s +Step 24890/44000 | Loss: 2.9658 | LR: 2.05e-04 | 2637.5ms/step | 7.6 samples/s +Step 24900/44000 | Loss: 2.9640 | LR: 2.04e-04 | 2720.3ms/step | 7.4 samples/s +Step 24910/44000 | Loss: 2.9705 | LR: 2.04e-04 | 2691.3ms/step | 7.4 samples/s +Step 24920/44000 | Loss: 2.9747 | LR: 2.04e-04 | 2777.1ms/step | 7.2 samples/s +Step 24930/44000 | Loss: 2.9375 | LR: 2.04e-04 | 2690.9ms/step | 7.4 samples/s +Step 24940/44000 | Loss: 2.9958 | LR: 2.04e-04 | 2706.4ms/step | 7.4 samples/s +Step 24950/44000 | Loss: 2.9667 | LR: 2.04e-04 | 2649.4ms/step | 7.5 samples/s +Step 24960/44000 | Loss: 2.9409 | LR: 2.03e-04 | 2584.5ms/step | 7.7 samples/s +Step 24970/44000 | Loss: 2.9787 | LR: 2.03e-04 | 2671.0ms/step | 7.5 samples/s +Step 24980/44000 | Loss: 2.9925 | LR: 2.03e-04 | 2750.6ms/step | 7.3 samples/s +Step 24990/44000 | Loss: 2.9586 | LR: 2.03e-04 | 2791.8ms/step | 7.2 samples/s +Step 25000/44000 | Loss: 2.9681 | LR: 2.03e-04 | 2738.4ms/step | 7.3 samples/s +Validation at step 25000: Loss = 2.9869, Perplexity = 19.82 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-25000 +Step 25010/44000 | Loss: 2.9749 | LR: 2.02e-04 | 5571.9ms/step | 3.6 samples/s +Step 25020/44000 | Loss: 2.9996 | LR: 2.02e-04 | 2698.8ms/step | 7.4 samples/s +Step 25030/44000 | Loss: 2.9540 | LR: 2.02e-04 | 2751.9ms/step | 7.3 samples/s +Step 25040/44000 | Loss: 2.9864 | LR: 2.02e-04 | 2679.0ms/step | 7.5 samples/s +Step 25050/44000 | Loss: 2.9786 | LR: 2.02e-04 | 2710.5ms/step | 7.4 samples/s +Step 25060/44000 | Loss: 3.0273 | LR: 2.02e-04 | 2685.1ms/step | 7.4 samples/s +Step 25070/44000 | Loss: 2.9846 | LR: 2.01e-04 | 2692.8ms/step | 7.4 samples/s +Step 25080/44000 | Loss: 2.9614 | LR: 2.01e-04 | 2687.9ms/step | 7.4 samples/s +Step 25090/44000 | Loss: 2.9777 | LR: 2.01e-04 | 2719.3ms/step | 7.4 samples/s +Step 25100/44000 | Loss: 2.9422 | LR: 2.01e-04 | 2713.9ms/step | 7.4 samples/s +Step 25110/44000 | Loss: 2.9583 | LR: 2.01e-04 | 2668.2ms/step | 7.5 samples/s +Step 25120/44000 | Loss: 2.9309 | LR: 2.00e-04 | 2707.1ms/step | 7.4 samples/s +Step 25130/44000 | Loss: 2.9686 | LR: 2.00e-04 | 2631.3ms/step | 7.6 samples/s +Step 25140/44000 | Loss: 2.9736 | LR: 2.00e-04 | 2687.5ms/step | 7.4 samples/s +Step 25150/44000 | Loss: 2.9423 | LR: 2.00e-04 | 2668.2ms/step | 7.5 samples/s +Step 25160/44000 | Loss: 2.9322 | LR: 2.00e-04 | 2672.6ms/step | 7.5 samples/s +Step 25170/44000 | Loss: 2.9897 | LR: 2.00e-04 | 2719.4ms/step | 7.4 samples/s +Step 25180/44000 | Loss: 2.9959 | LR: 1.99e-04 | 2693.9ms/step | 7.4 samples/s +Step 25190/44000 | Loss: 2.9896 | LR: 1.99e-04 | 2697.7ms/step | 7.4 samples/s +Step 25200/44000 | Loss: 2.9697 | LR: 1.99e-04 | 2674.8ms/step | 7.5 samples/s +Step 25210/44000 | Loss: 2.9427 | LR: 1.99e-04 | 2697.9ms/step | 7.4 samples/s +Step 25220/44000 | Loss: 2.9201 | LR: 1.99e-04 | 2671.4ms/step | 7.5 samples/s +Step 25230/44000 | Loss: 2.9509 | LR: 1.99e-04 | 2705.8ms/step | 7.4 samples/s +Step 25240/44000 | Loss: 2.9243 | LR: 1.98e-04 | 2672.6ms/step | 7.5 samples/s +Step 25250/44000 | Loss: 2.9182 | LR: 1.98e-04 | 2767.3ms/step | 7.2 samples/s +Validation at step 25250: Loss = 2.9851, Perplexity = 19.79 +Step 25260/44000 | Loss: 2.9833 | LR: 1.98e-04 | 5265.4ms/step | 3.8 samples/s +Step 25270/44000 | Loss: 3.0111 | LR: 1.98e-04 | 2650.7ms/step | 7.5 samples/s +Step 25280/44000 | Loss: 2.9552 | LR: 1.98e-04 | 2713.8ms/step | 7.4 samples/s +Step 25290/44000 | Loss: 2.9911 | LR: 1.97e-04 | 2724.5ms/step | 7.3 samples/s +Step 25300/44000 | Loss: 2.9499 | LR: 1.97e-04 | 2684.7ms/step | 7.4 samples/s +Step 25310/44000 | Loss: 2.9825 | LR: 1.97e-04 | 2678.0ms/step | 7.5 samples/s +Step 25320/44000 | Loss: 3.0155 | LR: 1.97e-04 | 2685.2ms/step | 7.4 samples/s +Step 25330/44000 | Loss: 2.9652 | LR: 1.97e-04 | 2689.0ms/step | 7.4 samples/s +Step 25340/44000 | Loss: 2.9592 | LR: 1.97e-04 | 2717.0ms/step | 7.4 samples/s +Step 25350/44000 | Loss: 2.9587 | LR: 1.96e-04 | 2680.4ms/step | 7.5 samples/s +Step 25360/44000 | Loss: 2.9894 | LR: 1.96e-04 | 2649.6ms/step | 7.5 samples/s +Step 25370/44000 | Loss: 2.9903 | LR: 1.96e-04 | 2729.7ms/step | 7.3 samples/s +Step 25380/44000 | Loss: 2.9382 | LR: 1.96e-04 | 2647.4ms/step | 7.6 samples/s +Step 25390/44000 | Loss: 2.9262 | LR: 1.96e-04 | 2673.7ms/step | 7.5 samples/s +Step 25400/44000 | Loss: 2.9609 | LR: 1.96e-04 | 2655.0ms/step | 7.5 samples/s +Step 25410/44000 | Loss: 2.9658 | LR: 1.95e-04 | 2697.9ms/step | 7.4 samples/s +Step 25420/44000 | Loss: 2.9350 | LR: 1.95e-04 | 2697.7ms/step | 7.4 samples/s +Step 25430/44000 | Loss: 2.9300 | LR: 1.95e-04 | 2695.5ms/step | 7.4 samples/s +Step 25440/44000 | Loss: 3.0006 | LR: 1.95e-04 | 2713.4ms/step | 7.4 samples/s +Step 25450/44000 | Loss: 2.9557 | LR: 1.95e-04 | 2714.8ms/step | 7.4 samples/s +Step 25460/44000 | Loss: 3.0160 | LR: 1.94e-04 | 2694.7ms/step | 7.4 samples/s +Step 25470/44000 | Loss: 2.9757 | LR: 1.94e-04 | 2703.7ms/step | 7.4 samples/s +Step 25480/44000 | Loss: 2.9579 | LR: 1.94e-04 | 2657.0ms/step | 7.5 samples/s +Step 25490/44000 | Loss: 2.9933 | LR: 1.94e-04 | 2674.4ms/step | 7.5 samples/s +Step 25500/44000 | Loss: 3.0127 | LR: 1.94e-04 | 2679.0ms/step | 7.5 samples/s +Validation at step 25500: Loss = 2.9832, Perplexity = 19.75 +Step 25510/44000 | Loss: 2.9607 | LR: 1.94e-04 | 5237.8ms/step | 3.8 samples/s +Step 25520/44000 | Loss: 2.9299 | LR: 1.93e-04 | 2688.2ms/step | 7.4 samples/s +Step 25530/44000 | Loss: 3.0191 | LR: 1.93e-04 | 2696.7ms/step | 7.4 samples/s +Step 25540/44000 | Loss: 2.9766 | LR: 1.93e-04 | 2740.5ms/step | 7.3 samples/s +Step 25550/44000 | Loss: 2.9823 | LR: 1.93e-04 | 2747.3ms/step | 7.3 samples/s +Step 25560/44000 | Loss: 2.9372 | LR: 1.93e-04 | 2459.0ms/step | 8.1 samples/s +Step 25570/44000 | Loss: 2.9434 | LR: 1.92e-04 | 2751.1ms/step | 7.3 samples/s +Step 25580/44000 | Loss: 2.9588 | LR: 1.92e-04 | 2703.0ms/step | 7.4 samples/s +Step 25590/44000 | Loss: 2.9854 | LR: 1.92e-04 | 2690.5ms/step | 7.4 samples/s +Step 25600/44000 | Loss: 2.9745 | LR: 1.92e-04 | 2671.2ms/step | 7.5 samples/s +Step 25610/44000 | Loss: 2.9569 | LR: 1.92e-04 | 2701.7ms/step | 7.4 samples/s +Step 25620/44000 | Loss: 2.9608 | LR: 1.92e-04 | 2652.8ms/step | 7.5 samples/s +Step 25630/44000 | Loss: 2.9832 | LR: 1.91e-04 | 2687.0ms/step | 7.4 samples/s +Step 25640/44000 | Loss: 2.9688 | LR: 1.91e-04 | 2721.9ms/step | 7.3 samples/s +Step 25650/44000 | Loss: 2.9280 | LR: 1.91e-04 | 2731.3ms/step | 7.3 samples/s +Step 25660/44000 | Loss: 2.9156 | LR: 1.91e-04 | 2735.3ms/step | 7.3 samples/s +Step 25670/44000 | Loss: 2.9803 | LR: 1.91e-04 | 2720.9ms/step | 7.4 samples/s +Step 25680/44000 | Loss: 2.9961 | LR: 1.91e-04 | 2696.5ms/step | 7.4 samples/s +Step 25690/44000 | Loss: 2.9557 | LR: 1.90e-04 | 2743.9ms/step | 7.3 samples/s +Step 25700/44000 | Loss: 2.9415 | LR: 1.90e-04 | 2716.3ms/step | 7.4 samples/s +Step 25710/44000 | Loss: 2.9274 | LR: 1.90e-04 | 2674.0ms/step | 7.5 samples/s +Step 25720/44000 | Loss: 2.9643 | LR: 1.90e-04 | 2672.9ms/step | 7.5 samples/s +Step 25730/44000 | Loss: 2.9590 | LR: 1.90e-04 | 2718.2ms/step | 7.4 samples/s +Step 25740/44000 | Loss: 2.9148 | LR: 1.89e-04 | 2660.1ms/step | 7.5 samples/s +Step 25750/44000 | Loss: 2.9773 | LR: 1.89e-04 | 2717.3ms/step | 7.4 samples/s +Validation at step 25750: Loss = 2.9816, Perplexity = 19.72 +Step 25760/44000 | Loss: 2.9655 | LR: 1.89e-04 | 5220.0ms/step | 3.8 samples/s +Step 25770/44000 | Loss: 2.9339 | LR: 1.89e-04 | 2640.1ms/step | 7.6 samples/s +Step 25780/44000 | Loss: 2.9607 | LR: 1.89e-04 | 2648.9ms/step | 7.6 samples/s +Step 25790/44000 | Loss: 3.0400 | LR: 1.89e-04 | 2634.2ms/step | 7.6 samples/s +Step 25800/44000 | Loss: 2.9488 | LR: 1.88e-04 | 2649.7ms/step | 7.5 samples/s +Step 25810/44000 | Loss: 2.9495 | LR: 1.88e-04 | 2555.9ms/step | 7.8 samples/s +Step 25820/44000 | Loss: 2.9852 | LR: 1.88e-04 | 2661.6ms/step | 7.5 samples/s +Step 25830/44000 | Loss: 2.9946 | LR: 1.88e-04 | 2684.3ms/step | 7.5 samples/s +Step 25840/44000 | Loss: 2.9983 | LR: 1.88e-04 | 2736.0ms/step | 7.3 samples/s +Step 25850/44000 | Loss: 2.9458 | LR: 1.88e-04 | 2678.5ms/step | 7.5 samples/s +Step 25860/44000 | Loss: 2.9795 | LR: 1.87e-04 | 2683.9ms/step | 7.5 samples/s +Step 25870/44000 | Loss: 2.9960 | LR: 1.87e-04 | 2641.5ms/step | 7.6 samples/s +Step 25880/44000 | Loss: 2.9745 | LR: 1.87e-04 | 2756.8ms/step | 7.3 samples/s +Step 25890/44000 | Loss: 2.9632 | LR: 1.87e-04 | 2697.7ms/step | 7.4 samples/s +Step 25900/44000 | Loss: 2.9464 | LR: 1.87e-04 | 2736.2ms/step | 7.3 samples/s +Step 25910/44000 | Loss: 2.9787 | LR: 1.87e-04 | 2731.0ms/step | 7.3 samples/s +Step 25920/44000 | Loss: 2.9581 | LR: 1.86e-04 | 2692.0ms/step | 7.4 samples/s +Step 25930/44000 | Loss: 2.9360 | LR: 1.86e-04 | 2732.6ms/step | 7.3 samples/s +Step 25940/44000 | Loss: 2.9762 | LR: 1.86e-04 | 2690.3ms/step | 7.4 samples/s +Step 25950/44000 | Loss: 2.9733 | LR: 1.86e-04 | 2682.5ms/step | 7.5 samples/s +Step 25960/44000 | Loss: 2.9750 | LR: 1.86e-04 | 2651.2ms/step | 7.5 samples/s +Step 25970/44000 | Loss: 2.9619 | LR: 1.85e-04 | 2698.6ms/step | 7.4 samples/s +Step 25980/44000 | Loss: 2.9374 | LR: 1.85e-04 | 2687.8ms/step | 7.4 samples/s +Step 25990/44000 | Loss: 2.9558 | LR: 1.85e-04 | 2693.0ms/step | 7.4 samples/s +Step 26000/44000 | Loss: 2.9746 | LR: 1.85e-04 | 2651.1ms/step | 7.5 samples/s +Validation at step 26000: Loss = 2.9795, Perplexity = 19.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812/checkpoint-26000 +Step 26010/44000 | Loss: 2.9690 | LR: 1.85e-04 | 5765.3ms/step | 3.5 samples/s +Step 26020/44000 | Loss: 2.9236 | LR: 1.85e-04 | 2682.7ms/step | 7.5 samples/s +Step 26030/44000 | Loss: 2.9243 | LR: 1.84e-04 | 2678.4ms/step | 7.5 samples/s +Step 26040/44000 | Loss: 2.9732 | LR: 1.84e-04 | 2669.7ms/step | 7.5 samples/s +Step 26050/44000 | Loss: 2.9711 | LR: 1.84e-04 | 2683.3ms/step | 7.5 samples/s +Step 26060/44000 | Loss: 2.9893 | LR: 1.84e-04 | 2676.7ms/step | 7.5 samples/s +Step 26070/44000 | Loss: 2.9889 | LR: 1.84e-04 | 2726.7ms/step | 7.3 samples/s +Step 26080/44000 | Loss: 2.9723 | LR: 1.84e-04 | 2734.2ms/step | 7.3 samples/s +Step 26090/44000 | Loss: 2.9431 | LR: 1.83e-04 | 2646.6ms/step | 7.6 samples/s +Step 26100/44000 | Loss: 2.9640 | LR: 1.83e-04 | 2672.4ms/step | 7.5 samples/s +Step 26110/44000 | Loss: 2.9445 | LR: 1.83e-04 | 2677.6ms/step | 7.5 samples/s +Step 26120/44000 | Loss: 3.0054 | LR: 1.83e-04 | 2687.2ms/step | 7.4 samples/s +Step 26130/44000 | Loss: 2.9628 | LR: 1.83e-04 | 2736.0ms/step | 7.3 samples/s +Step 26140/44000 | Loss: 2.9540 | LR: 1.82e-04 | 2735.9ms/step | 7.3 samples/s +Step 26150/44000 | Loss: 2.9972 | LR: 1.82e-04 | 2686.4ms/step | 7.4 samples/s +Step 26160/44000 | Loss: 2.9946 | LR: 1.82e-04 | 2758.6ms/step | 7.3 samples/s +Step 26170/44000 | Loss: 2.9512 | LR: 1.82e-04 | 2704.1ms/step | 7.4 samples/s +Step 26180/44000 | Loss: 2.9129 | LR: 1.82e-04 | 2677.5ms/step | 7.5 samples/s +Step 26190/44000 | Loss: 2.9734 | LR: 1.82e-04 | 2651.3ms/step | 7.5 samples/s +Step 26200/44000 | Loss: 2.9133 | LR: 1.81e-04 | 2686.8ms/step | 7.4 samples/s +Step 26210/44000 | Loss: 2.9605 | LR: 1.81e-04 | 2703.5ms/step | 7.4 samples/s +Step 26220/44000 | Loss: 2.9311 | LR: 1.81e-04 | 2724.3ms/step | 7.3 samples/s +W0820 23:50:48.004000 40373 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers +W0820 23:50:48.007000 40373 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 40442 closing signal SIGTERM +W0820 23:50:48.009000 40373 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 40443 closing signal SIGTERM +Traceback (most recent call last): + File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in + sys.exit(main()) + ~~~~^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper + return f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main + run(args) + ~~~^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run + elastic_launch( + ~~~~~~~~~~~~~~~ + config=config, + ~~~~~~~~~~~~~~ + entrypoint=cmd, + ~~~~~~~~~~~~~~~ + )(*cmd_args) + ~^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent + result = agent.run() + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run + result = self._invoke_run(role) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run + time.sleep(monitor_interval) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 40373 got signal: 15 diff --git a/epfl_final/train_logs/seed48_dense_ddp5/dense.20260821T043813Z.log b/epfl_final/train_logs/seed48_dense_ddp5/dense.20260821T043813Z.log new file mode 100644 index 0000000000000000000000000000000000000000..c62f471caba4f19999817df3cef2f5321b1931bf --- /dev/null +++ b/epfl_final/train_logs/seed48_dense_ddp5/dense.20260821T043813Z.log @@ -0,0 +1,2947 @@ +W0821 04:38:16.105000 64391 site-packages/torch/distributed/run.py:851] +W0821 04:38:16.105000 64391 site-packages/torch/distributed/run.py:851] ***************************************** +W0821 04:38:16.105000 64391 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0821 04:38:16.105000 64391 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 2, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 48, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m dense 8k seed48 ddp5", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "scaleladder", + "dense", + "multiseed", + "seed48", + "ddp5" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +Initializing model from scratch (random weights)... +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 5 +Gradient accumulation steps: 2 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 302776 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run zrelwtt2 +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_043849-zrelwtt2 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m dense 8k seed48 ddp5 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/zrelwtt2 +Starting training... +Switching attention mode for step 1: dense (dense step interval) +Step 10/44000 | Loss: 12.1023 | LR: 6.67e-06 | 3818.4ms/step | 5.2 samples/s +Step 20/44000 | Loss: 12.0008 | LR: 1.33e-05 | 1063.7ms/step | 18.8 samples/s +Step 30/44000 | Loss: 11.7118 | LR: 2.00e-05 | 1161.8ms/step | 17.2 samples/s +Step 40/44000 | Loss: 11.1092 | LR: 2.67e-05 | 1145.5ms/step | 17.5 samples/s +Step 50/44000 | Loss: 10.7767 | LR: 3.33e-05 | 1090.2ms/step | 18.3 samples/s +Step 60/44000 | Loss: 10.5434 | LR: 4.00e-05 | 1207.7ms/step | 16.6 samples/s +Step 70/44000 | Loss: 10.3021 | LR: 4.67e-05 | 1116.6ms/step | 17.9 samples/s +Step 80/44000 | Loss: 9.9955 | LR: 5.33e-05 | 1063.0ms/step | 18.8 samples/s +Step 90/44000 | Loss: 9.6754 | LR: 6.00e-05 | 1129.5ms/step | 17.7 samples/s +Step 100/44000 | Loss: 9.1801 | LR: 6.67e-05 | 1095.7ms/step | 18.3 samples/s +Step 110/44000 | Loss: 8.6510 | LR: 7.33e-05 | 1104.9ms/step | 18.1 samples/s +Step 120/44000 | Loss: 8.1818 | LR: 8.00e-05 | 1167.0ms/step | 17.1 samples/s +Step 130/44000 | Loss: 7.8071 | LR: 8.67e-05 | 1068.4ms/step | 18.7 samples/s +Step 140/44000 | Loss: 7.5639 | LR: 9.33e-05 | 1058.2ms/step | 18.9 samples/s +Step 150/44000 | Loss: 7.3086 | LR: 1.00e-04 | 1114.8ms/step | 17.9 samples/s +Step 160/44000 | Loss: 7.2513 | LR: 1.07e-04 | 1099.8ms/step | 18.2 samples/s +Step 170/44000 | Loss: 7.1011 | LR: 1.13e-04 | 1094.5ms/step | 18.3 samples/s +Step 180/44000 | Loss: 6.9791 | LR: 1.20e-04 | 1177.0ms/step | 17.0 samples/s +Step 190/44000 | Loss: 6.9944 | LR: 1.27e-04 | 1116.7ms/step | 17.9 samples/s +Step 200/44000 | Loss: 6.8579 | LR: 1.33e-04 | 1124.9ms/step | 17.8 samples/s +Step 210/44000 | Loss: 6.7385 | LR: 1.40e-04 | 1062.6ms/step | 18.8 samples/s +Step 220/44000 | Loss: 6.7211 | LR: 1.47e-04 | 1094.1ms/step | 18.3 samples/s +Step 230/44000 | Loss: 6.6122 | LR: 1.53e-04 | 1069.6ms/step | 18.7 samples/s +Step 240/44000 | Loss: 6.5061 | LR: 1.60e-04 | 1123.4ms/step | 17.8 samples/s +Step 250/44000 | Loss: 6.4419 | LR: 1.67e-04 | 1126.2ms/step | 17.8 samples/s +Validation at step 250: Loss = 6.4231, Perplexity = 615.88 +Step 260/44000 | Loss: 6.4221 | LR: 1.73e-04 | 2912.5ms/step | 6.9 samples/s +Step 270/44000 | Loss: 6.3547 | LR: 1.80e-04 | 1091.2ms/step | 18.3 samples/s +Step 280/44000 | Loss: 6.3246 | LR: 1.87e-04 | 1108.7ms/step | 18.0 samples/s +Step 290/44000 | Loss: 6.2571 | LR: 1.93e-04 | 1130.1ms/step | 17.7 samples/s +Step 300/44000 | Loss: 6.1956 | LR: 2.00e-04 | 1115.9ms/step | 17.9 samples/s +Step 310/44000 | Loss: 6.1859 | LR: 2.07e-04 | 1140.8ms/step | 17.5 samples/s +Step 320/44000 | Loss: 6.1381 | LR: 2.13e-04 | 1110.6ms/step | 18.0 samples/s +Step 330/44000 | Loss: 6.1015 | LR: 2.20e-04 | 1069.9ms/step | 18.7 samples/s +Step 340/44000 | Loss: 6.0757 | LR: 2.27e-04 | 1134.9ms/step | 17.6 samples/s +Step 350/44000 | Loss: 6.0827 | LR: 2.33e-04 | 1120.7ms/step | 17.8 samples/s +Step 360/44000 | Loss: 6.0112 | LR: 2.40e-04 | 1115.5ms/step | 17.9 samples/s +Step 370/44000 | Loss: 5.9340 | LR: 2.47e-04 | 1214.1ms/step | 16.5 samples/s +Step 380/44000 | Loss: 5.9166 | LR: 2.53e-04 | 1167.8ms/step | 17.1 samples/s +Step 390/44000 | Loss: 5.9003 | LR: 2.60e-04 | 1121.7ms/step | 17.8 samples/s +Step 400/44000 | Loss: 5.8345 | LR: 2.67e-04 | 1150.4ms/step | 17.4 samples/s +Step 410/44000 | Loss: 5.8125 | LR: 2.73e-04 | 1147.2ms/step | 17.4 samples/s +Step 420/44000 | Loss: 5.8063 | LR: 2.80e-04 | 1099.7ms/step | 18.2 samples/s +Step 430/44000 | Loss: 5.7426 | LR: 2.87e-04 | 1104.5ms/step | 18.1 samples/s +Step 440/44000 | Loss: 5.7153 | LR: 2.93e-04 | 1094.2ms/step | 18.3 samples/s +Step 450/44000 | Loss: 5.6934 | LR: 3.00e-04 | 1099.3ms/step | 18.2 samples/s +Step 460/44000 | Loss: 5.6876 | LR: 3.07e-04 | 1105.5ms/step | 18.1 samples/s +Step 470/44000 | Loss: 5.5678 | LR: 3.13e-04 | 1084.5ms/step | 18.4 samples/s +Step 480/44000 | Loss: 5.5892 | LR: 3.20e-04 | 1068.4ms/step | 18.7 samples/s +Step 490/44000 | Loss: 5.5327 | LR: 3.27e-04 | 1073.5ms/step | 18.6 samples/s +Step 500/44000 | Loss: 5.4651 | LR: 3.33e-04 | 1167.7ms/step | 17.1 samples/s +Validation at step 500: Loss = 5.4457, Perplexity = 231.75 +Step 510/44000 | Loss: 5.4764 | LR: 3.40e-04 | 1483.9ms/step | 13.5 samples/s +Step 520/44000 | Loss: 5.4173 | LR: 3.47e-04 | 1142.4ms/step | 17.5 samples/s +Step 530/44000 | Loss: 5.3466 | LR: 3.53e-04 | 1142.5ms/step | 17.5 samples/s +Step 540/44000 | Loss: 5.3906 | LR: 3.60e-04 | 1123.1ms/step | 17.8 samples/s +Step 550/44000 | Loss: 5.3071 | LR: 3.67e-04 | 1127.8ms/step | 17.7 samples/s +Step 560/44000 | Loss: 5.2725 | LR: 3.73e-04 | 1099.4ms/step | 18.2 samples/s +Step 570/44000 | Loss: 5.2357 | LR: 3.80e-04 | 1149.1ms/step | 17.4 samples/s +Step 580/44000 | Loss: 5.2557 | LR: 3.87e-04 | 1097.5ms/step | 18.2 samples/s +Step 590/44000 | Loss: 5.1600 | LR: 3.93e-04 | 1124.7ms/step | 17.8 samples/s +Step 600/44000 | Loss: 5.1466 | LR: 4.00e-04 | 1126.4ms/step | 17.8 samples/s +Step 610/44000 | Loss: 5.1518 | LR: 4.07e-04 | 1126.0ms/step | 17.8 samples/s +Step 620/44000 | Loss: 5.1533 | LR: 4.13e-04 | 1146.1ms/step | 17.5 samples/s +Step 630/44000 | Loss: 5.0765 | LR: 4.20e-04 | 1104.9ms/step | 18.1 samples/s +Step 640/44000 | Loss: 5.0571 | LR: 4.27e-04 | 1105.1ms/step | 18.1 samples/s +Step 650/44000 | Loss: 5.0751 | LR: 4.33e-04 | 1110.5ms/step | 18.0 samples/s +Step 660/44000 | Loss: 5.0789 | LR: 4.40e-04 | 1167.4ms/step | 17.1 samples/s +Step 670/44000 | Loss: 4.9268 | LR: 4.47e-04 | 1128.5ms/step | 17.7 samples/s +Step 680/44000 | Loss: 5.0261 | LR: 4.53e-04 | 1096.7ms/step | 18.2 samples/s +Step 690/44000 | Loss: 4.9404 | LR: 4.60e-04 | 1064.2ms/step | 18.8 samples/s +Step 700/44000 | Loss: 4.8903 | LR: 4.67e-04 | 1156.2ms/step | 17.3 samples/s +Step 710/44000 | Loss: 4.8553 | LR: 4.73e-04 | 1146.3ms/step | 17.4 samples/s +Step 720/44000 | Loss: 4.8290 | LR: 4.80e-04 | 1098.8ms/step | 18.2 samples/s +Step 730/44000 | Loss: 4.8038 | LR: 4.87e-04 | 1099.1ms/step | 18.2 samples/s +Step 740/44000 | Loss: 4.7632 | LR: 4.93e-04 | 1105.8ms/step | 18.1 samples/s +Step 750/44000 | Loss: 4.7872 | LR: 5.00e-04 | 1089.1ms/step | 18.4 samples/s +Validation at step 750: Loss = 4.7299, Perplexity = 113.28 +Step 760/44000 | Loss: 4.7236 | LR: 5.00e-04 | 1469.2ms/step | 13.6 samples/s +Step 770/44000 | Loss: 4.6681 | LR: 5.00e-04 | 1162.7ms/step | 17.2 samples/s +Step 780/44000 | Loss: 4.6622 | LR: 5.00e-04 | 1143.5ms/step | 17.5 samples/s +Step 790/44000 | Loss: 4.6386 | LR: 5.00e-04 | 1097.2ms/step | 18.2 samples/s +Step 800/44000 | Loss: 4.6797 | LR: 5.00e-04 | 1146.3ms/step | 17.4 samples/s +Step 810/44000 | Loss: 4.6349 | LR: 5.00e-04 | 1130.9ms/step | 17.7 samples/s +Step 820/44000 | Loss: 4.5741 | LR: 5.00e-04 | 1141.3ms/step | 17.5 samples/s +Step 830/44000 | Loss: 4.5607 | LR: 5.00e-04 | 1113.0ms/step | 18.0 samples/s +Step 840/44000 | Loss: 4.5426 | LR: 5.00e-04 | 1085.9ms/step | 18.4 samples/s +Step 850/44000 | Loss: 4.5046 | LR: 5.00e-04 | 1119.9ms/step | 17.9 samples/s +Step 860/44000 | Loss: 4.3974 | LR: 5.00e-04 | 1125.3ms/step | 17.8 samples/s +Step 870/44000 | Loss: 4.4404 | LR: 5.00e-04 | 1094.1ms/step | 18.3 samples/s +Step 880/44000 | Loss: 4.3944 | LR: 5.00e-04 | 1114.6ms/step | 17.9 samples/s +Step 890/44000 | Loss: 4.4531 | LR: 5.00e-04 | 1089.0ms/step | 18.4 samples/s +Step 900/44000 | Loss: 4.4472 | LR: 5.00e-04 | 1145.7ms/step | 17.5 samples/s +Step 910/44000 | Loss: 4.3831 | LR: 5.00e-04 | 1120.1ms/step | 17.9 samples/s +Step 920/44000 | Loss: 4.4247 | LR: 5.00e-04 | 1109.8ms/step | 18.0 samples/s +Step 930/44000 | Loss: 4.3631 | LR: 5.00e-04 | 1115.1ms/step | 17.9 samples/s +Step 940/44000 | Loss: 4.3006 | LR: 5.00e-04 | 1073.5ms/step | 18.6 samples/s +Step 950/44000 | Loss: 4.3208 | LR: 5.00e-04 | 1110.3ms/step | 18.0 samples/s +Step 960/44000 | Loss: 4.3199 | LR: 5.00e-04 | 1171.2ms/step | 17.1 samples/s +Step 970/44000 | Loss: 4.2648 | LR: 5.00e-04 | 1137.1ms/step | 17.6 samples/s +Step 980/44000 | Loss: 4.2912 | LR: 5.00e-04 | 1092.2ms/step | 18.3 samples/s +Step 990/44000 | Loss: 4.3805 | LR: 5.00e-04 | 1075.1ms/step | 18.6 samples/s +Step 1000/44000 | Loss: 4.2367 | LR: 5.00e-04 | 1119.7ms/step | 17.9 samples/s +Validation at step 1000: Loss = 4.2425, Perplexity = 69.58 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-1000 +Step 1010/44000 | Loss: 4.2941 | LR: 5.00e-04 | 1868.1ms/step | 10.7 samples/s +Step 1020/44000 | Loss: 4.1990 | LR: 5.00e-04 | 1145.0ms/step | 17.5 samples/s +Step 1030/44000 | Loss: 4.1869 | LR: 5.00e-04 | 1109.7ms/step | 18.0 samples/s +Step 1040/44000 | Loss: 4.1762 | LR: 5.00e-04 | 1099.8ms/step | 18.2 samples/s +Step 1050/44000 | Loss: 4.2096 | LR: 5.00e-04 | 1100.4ms/step | 18.2 samples/s +Step 1060/44000 | Loss: 4.1865 | LR: 5.00e-04 | 1087.9ms/step | 18.4 samples/s +Step 1070/44000 | Loss: 4.1753 | LR: 5.00e-04 | 1161.7ms/step | 17.2 samples/s +Step 1080/44000 | Loss: 4.1464 | LR: 5.00e-04 | 1099.0ms/step | 18.2 samples/s +Step 1090/44000 | Loss: 4.1667 | LR: 5.00e-04 | 1085.1ms/step | 18.4 samples/s +Step 1100/44000 | Loss: 4.0718 | LR: 5.00e-04 | 1129.7ms/step | 17.7 samples/s +Step 1110/44000 | Loss: 4.1672 | LR: 5.00e-04 | 1078.8ms/step | 18.5 samples/s +Step 1120/44000 | Loss: 4.1686 | LR: 5.00e-04 | 1100.9ms/step | 18.2 samples/s +Step 1130/44000 | Loss: 4.0843 | LR: 5.00e-04 | 1077.8ms/step | 18.6 samples/s +Step 1140/44000 | Loss: 4.1826 | LR: 5.00e-04 | 1089.2ms/step | 18.4 samples/s +Step 1150/44000 | Loss: 4.1625 | LR: 5.00e-04 | 1088.9ms/step | 18.4 samples/s +Step 1160/44000 | Loss: 4.1040 | LR: 5.00e-04 | 1131.0ms/step | 17.7 samples/s +Step 1170/44000 | Loss: 4.1122 | LR: 5.00e-04 | 1172.5ms/step | 17.1 samples/s +Step 1180/44000 | Loss: 4.0386 | LR: 5.00e-04 | 1042.9ms/step | 19.2 samples/s +Step 1190/44000 | Loss: 4.0258 | LR: 5.00e-04 | 1109.8ms/step | 18.0 samples/s +Step 1200/44000 | Loss: 3.9979 | LR: 5.00e-04 | 1083.3ms/step | 18.5 samples/s +Step 1210/44000 | Loss: 4.0175 | LR: 5.00e-04 | 1095.3ms/step | 18.3 samples/s +Step 1220/44000 | Loss: 4.0255 | LR: 5.00e-04 | 1017.0ms/step | 19.7 samples/s +Step 1230/44000 | Loss: 4.0486 | LR: 5.00e-04 | 1079.0ms/step | 18.5 samples/s +Step 1240/44000 | Loss: 3.9803 | LR: 5.00e-04 | 1085.0ms/step | 18.4 samples/s +Step 1250/44000 | Loss: 4.0278 | LR: 5.00e-04 | 1057.2ms/step | 18.9 samples/s +Validation at step 1250: Loss = 4.0031, Perplexity = 54.77 +Step 1260/44000 | Loss: 4.0208 | LR: 5.00e-04 | 1444.0ms/step | 13.9 samples/s +Step 1270/44000 | Loss: 4.1419 | LR: 5.00e-04 | 1089.6ms/step | 18.4 samples/s +Step 1280/44000 | Loss: 3.9761 | LR: 5.00e-04 | 1111.4ms/step | 18.0 samples/s +Step 1290/44000 | Loss: 3.9793 | LR: 5.00e-04 | 1108.8ms/step | 18.0 samples/s +Step 1300/44000 | Loss: 3.9967 | LR: 5.00e-04 | 1065.0ms/step | 18.8 samples/s +Step 1310/44000 | Loss: 4.0008 | LR: 5.00e-04 | 1064.7ms/step | 18.8 samples/s +Step 1320/44000 | Loss: 3.9332 | LR: 5.00e-04 | 1097.2ms/step | 18.2 samples/s +Step 1330/44000 | Loss: 3.9543 | LR: 5.00e-04 | 1073.8ms/step | 18.6 samples/s +Step 1340/44000 | Loss: 3.9515 | LR: 5.00e-04 | 1089.1ms/step | 18.4 samples/s +Step 1350/44000 | Loss: 3.8913 | LR: 5.00e-04 | 1063.9ms/step | 18.8 samples/s +Step 1360/44000 | Loss: 3.9117 | LR: 5.00e-04 | 1077.9ms/step | 18.6 samples/s +Step 1370/44000 | Loss: 3.9227 | LR: 5.00e-04 | 1158.6ms/step | 17.3 samples/s +Step 1380/44000 | Loss: 3.8839 | LR: 5.00e-04 | 1114.2ms/step | 17.9 samples/s +Step 1390/44000 | Loss: 3.9718 | LR: 5.00e-04 | 1079.0ms/step | 18.5 samples/s +Step 1400/44000 | Loss: 3.8573 | LR: 5.00e-04 | 1062.8ms/step | 18.8 samples/s +Step 1410/44000 | Loss: 3.9226 | LR: 5.00e-04 | 1088.3ms/step | 18.4 samples/s +Step 1420/44000 | Loss: 3.9163 | LR: 5.00e-04 | 1080.0ms/step | 18.5 samples/s +Step 1430/44000 | Loss: 3.8564 | LR: 5.00e-04 | 1070.1ms/step | 18.7 samples/s +Step 1440/44000 | Loss: 3.9012 | LR: 5.00e-04 | 1080.6ms/step | 18.5 samples/s +Step 1450/44000 | Loss: 3.9055 | LR: 5.00e-04 | 1095.8ms/step | 18.3 samples/s +Step 1460/44000 | Loss: 3.8850 | LR: 5.00e-04 | 1042.4ms/step | 19.2 samples/s +Step 1470/44000 | Loss: 3.8506 | LR: 5.00e-04 | 1068.5ms/step | 18.7 samples/s +Step 1480/44000 | Loss: 3.8546 | LR: 5.00e-04 | 1042.1ms/step | 19.2 samples/s +Step 1490/44000 | Loss: 3.8209 | LR: 5.00e-04 | 1088.9ms/step | 18.4 samples/s +Step 1500/44000 | Loss: 3.9187 | LR: 5.00e-04 | 1079.5ms/step | 18.5 samples/s +Validation at step 1500: Loss = 3.8525, Perplexity = 47.11 +Step 1510/44000 | Loss: 3.8380 | LR: 5.00e-04 | 1484.5ms/step | 13.5 samples/s +Step 1520/44000 | Loss: 3.7830 | LR: 5.00e-04 | 1048.4ms/step | 19.1 samples/s +Step 1530/44000 | Loss: 3.8674 | LR: 5.00e-04 | 1094.0ms/step | 18.3 samples/s +Step 1540/44000 | Loss: 3.7948 | LR: 5.00e-04 | 1110.4ms/step | 18.0 samples/s +Step 1550/44000 | Loss: 3.9009 | LR: 5.00e-04 | 1068.4ms/step | 18.7 samples/s +Step 1560/44000 | Loss: 3.8592 | LR: 5.00e-04 | 1094.2ms/step | 18.3 samples/s +Step 1570/44000 | Loss: 3.7663 | LR: 5.00e-04 | 1125.5ms/step | 17.8 samples/s +Step 1580/44000 | Loss: 3.8173 | LR: 5.00e-04 | 1099.1ms/step | 18.2 samples/s +Step 1590/44000 | Loss: 3.8484 | LR: 5.00e-04 | 1078.6ms/step | 18.5 samples/s +Step 1600/44000 | Loss: 3.7586 | LR: 5.00e-04 | 1064.4ms/step | 18.8 samples/s +Step 1610/44000 | Loss: 3.8482 | LR: 5.00e-04 | 1135.9ms/step | 17.6 samples/s +Step 1620/44000 | Loss: 3.7618 | LR: 5.00e-04 | 1130.1ms/step | 17.7 samples/s +Step 1630/44000 | Loss: 3.7900 | LR: 4.99e-04 | 1116.5ms/step | 17.9 samples/s +Step 1640/44000 | Loss: 3.8275 | LR: 4.99e-04 | 1056.8ms/step | 18.9 samples/s +Step 1650/44000 | Loss: 3.7499 | LR: 4.99e-04 | 1067.9ms/step | 18.7 samples/s +Step 1660/44000 | Loss: 3.7566 | LR: 4.99e-04 | 1099.5ms/step | 18.2 samples/s +Step 1670/44000 | Loss: 3.7862 | LR: 4.99e-04 | 1085.1ms/step | 18.4 samples/s +Step 1680/44000 | Loss: 3.8033 | LR: 4.99e-04 | 1101.5ms/step | 18.2 samples/s +Step 1690/44000 | Loss: 3.7526 | LR: 4.99e-04 | 1091.0ms/step | 18.3 samples/s +Step 1700/44000 | Loss: 3.7199 | LR: 4.99e-04 | 1089.3ms/step | 18.4 samples/s +Step 1710/44000 | Loss: 3.8164 | LR: 4.99e-04 | 1063.3ms/step | 18.8 samples/s +Step 1720/44000 | Loss: 3.7794 | LR: 4.99e-04 | 1104.7ms/step | 18.1 samples/s +Step 1730/44000 | Loss: 3.8053 | LR: 4.99e-04 | 1103.9ms/step | 18.1 samples/s +Step 1740/44000 | Loss: 3.7530 | LR: 4.99e-04 | 1105.4ms/step | 18.1 samples/s +Step 1750/44000 | Loss: 3.7574 | LR: 4.99e-04 | 1027.2ms/step | 19.5 samples/s +Validation at step 1750: Loss = 3.7441, Perplexity = 42.27 +Step 1760/44000 | Loss: 3.7265 | LR: 4.99e-04 | 1470.0ms/step | 13.6 samples/s +Step 1770/44000 | Loss: 3.7085 | LR: 4.99e-04 | 1150.9ms/step | 17.4 samples/s +Step 1780/44000 | Loss: 3.6857 | LR: 4.99e-04 | 1105.6ms/step | 18.1 samples/s +Step 1790/44000 | Loss: 3.8022 | LR: 4.99e-04 | 1120.7ms/step | 17.8 samples/s +Step 1800/44000 | Loss: 3.7602 | LR: 4.99e-04 | 1089.0ms/step | 18.4 samples/s +Step 1810/44000 | Loss: 3.7470 | LR: 4.99e-04 | 1100.5ms/step | 18.2 samples/s +Step 1820/44000 | Loss: 3.7728 | LR: 4.99e-04 | 1077.7ms/step | 18.6 samples/s +Step 1830/44000 | Loss: 3.7002 | LR: 4.99e-04 | 1125.5ms/step | 17.8 samples/s +Step 1840/44000 | Loss: 3.7126 | LR: 4.99e-04 | 1047.9ms/step | 19.1 samples/s +Step 1850/44000 | Loss: 3.8246 | LR: 4.99e-04 | 1079.0ms/step | 18.5 samples/s +Step 1860/44000 | Loss: 3.6706 | LR: 4.99e-04 | 1104.6ms/step | 18.1 samples/s +Step 1870/44000 | Loss: 3.7251 | LR: 4.99e-04 | 1064.5ms/step | 18.8 samples/s +Step 1880/44000 | Loss: 3.7298 | LR: 4.99e-04 | 1071.5ms/step | 18.7 samples/s +Step 1890/44000 | Loss: 3.6845 | LR: 4.99e-04 | 1121.6ms/step | 17.8 samples/s +Step 1900/44000 | Loss: 3.6933 | LR: 4.99e-04 | 1052.8ms/step | 19.0 samples/s +Step 1910/44000 | Loss: 3.7133 | LR: 4.99e-04 | 1063.5ms/step | 18.8 samples/s +Step 1920/44000 | Loss: 3.6417 | LR: 4.99e-04 | 1145.8ms/step | 17.5 samples/s +Step 1930/44000 | Loss: 3.6765 | LR: 4.99e-04 | 1066.4ms/step | 18.8 samples/s +Step 1940/44000 | Loss: 3.6542 | LR: 4.99e-04 | 1091.4ms/step | 18.3 samples/s +Step 1950/44000 | Loss: 3.7115 | LR: 4.99e-04 | 1105.0ms/step | 18.1 samples/s +Step 1960/44000 | Loss: 3.6827 | LR: 4.99e-04 | 1031.8ms/step | 19.4 samples/s +Step 1970/44000 | Loss: 3.7396 | LR: 4.99e-04 | 1031.9ms/step | 19.4 samples/s +Step 1980/44000 | Loss: 3.6282 | LR: 4.99e-04 | 1150.8ms/step | 17.4 samples/s +Step 1990/44000 | Loss: 3.6362 | LR: 4.99e-04 | 1120.8ms/step | 17.8 samples/s +Step 2000/44000 | Loss: 3.6326 | LR: 4.99e-04 | 1068.1ms/step | 18.7 samples/s +Validation at step 2000: Loss = 3.6689, Perplexity = 39.21 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-2000 +Step 2010/44000 | Loss: 3.6153 | LR: 4.99e-04 | 1916.5ms/step | 10.4 samples/s +Step 2020/44000 | Loss: 3.6534 | LR: 4.99e-04 | 1074.1ms/step | 18.6 samples/s +Step 2030/44000 | Loss: 3.6784 | LR: 4.99e-04 | 1083.1ms/step | 18.5 samples/s +Step 2040/44000 | Loss: 3.6862 | LR: 4.99e-04 | 1048.3ms/step | 19.1 samples/s +Step 2050/44000 | Loss: 3.6723 | LR: 4.99e-04 | 1072.8ms/step | 18.6 samples/s +Step 2060/44000 | Loss: 3.6511 | LR: 4.99e-04 | 1068.8ms/step | 18.7 samples/s +Step 2070/44000 | Loss: 3.6310 | LR: 4.99e-04 | 1081.6ms/step | 18.5 samples/s +Step 2080/44000 | Loss: 3.5856 | LR: 4.99e-04 | 1085.5ms/step | 18.4 samples/s +Step 2090/44000 | Loss: 3.6590 | LR: 4.99e-04 | 1110.0ms/step | 18.0 samples/s +Step 2100/44000 | Loss: 3.6293 | LR: 4.99e-04 | 1104.8ms/step | 18.1 samples/s +Step 2110/44000 | Loss: 3.6972 | LR: 4.99e-04 | 1071.8ms/step | 18.7 samples/s +Step 2120/44000 | Loss: 3.7107 | LR: 4.99e-04 | 1048.8ms/step | 19.1 samples/s +Step 2130/44000 | Loss: 3.6154 | LR: 4.99e-04 | 1130.6ms/step | 17.7 samples/s +Step 2140/44000 | Loss: 3.6851 | LR: 4.99e-04 | 1094.1ms/step | 18.3 samples/s +Step 2150/44000 | Loss: 3.6408 | LR: 4.99e-04 | 1099.6ms/step | 18.2 samples/s +Step 2160/44000 | Loss: 3.5980 | LR: 4.99e-04 | 1093.9ms/step | 18.3 samples/s +Step 2170/44000 | Loss: 3.6328 | LR: 4.99e-04 | 1037.0ms/step | 19.3 samples/s +Step 2180/44000 | Loss: 3.5319 | LR: 4.99e-04 | 1148.0ms/step | 17.4 samples/s +Step 2190/44000 | Loss: 3.6265 | LR: 4.99e-04 | 1114.0ms/step | 18.0 samples/s +Step 2200/44000 | Loss: 3.5712 | LR: 4.99e-04 | 1061.6ms/step | 18.8 samples/s +Step 2210/44000 | Loss: 3.6332 | LR: 4.99e-04 | 1093.6ms/step | 18.3 samples/s +Step 2220/44000 | Loss: 3.6246 | LR: 4.99e-04 | 1086.3ms/step | 18.4 samples/s +Step 2230/44000 | Loss: 3.6382 | LR: 4.99e-04 | 1079.1ms/step | 18.5 samples/s +Step 2240/44000 | Loss: 3.6575 | LR: 4.99e-04 | 1058.4ms/step | 18.9 samples/s +Step 2250/44000 | Loss: 3.6825 | LR: 4.99e-04 | 1084.8ms/step | 18.4 samples/s +Validation at step 2250: Loss = 3.6085, Perplexity = 36.91 +Step 2260/44000 | Loss: 3.6309 | LR: 4.98e-04 | 1437.3ms/step | 13.9 samples/s +Step 2270/44000 | Loss: 3.5777 | LR: 4.98e-04 | 1063.8ms/step | 18.8 samples/s +Step 2280/44000 | Loss: 3.6534 | LR: 4.98e-04 | 1058.1ms/step | 18.9 samples/s +Step 2290/44000 | Loss: 3.5631 | LR: 4.98e-04 | 1084.3ms/step | 18.4 samples/s +Step 2300/44000 | Loss: 3.6349 | LR: 4.98e-04 | 1084.1ms/step | 18.4 samples/s +Step 2310/44000 | Loss: 3.6057 | LR: 4.98e-04 | 1058.1ms/step | 18.9 samples/s +Step 2320/44000 | Loss: 3.5824 | LR: 4.98e-04 | 1109.5ms/step | 18.0 samples/s +Step 2330/44000 | Loss: 3.5950 | LR: 4.98e-04 | 1080.5ms/step | 18.5 samples/s +Step 2340/44000 | Loss: 3.5862 | LR: 4.98e-04 | 1072.2ms/step | 18.7 samples/s +Step 2350/44000 | Loss: 3.5493 | LR: 4.98e-04 | 1063.4ms/step | 18.8 samples/s +Step 2360/44000 | Loss: 3.5147 | LR: 4.98e-04 | 1073.6ms/step | 18.6 samples/s +Step 2370/44000 | Loss: 3.6056 | LR: 4.98e-04 | 1099.4ms/step | 18.2 samples/s +Step 2380/44000 | Loss: 3.6013 | LR: 4.98e-04 | 1146.5ms/step | 17.4 samples/s +Step 2390/44000 | Loss: 3.5882 | LR: 4.98e-04 | 1099.4ms/step | 18.2 samples/s +Step 2400/44000 | Loss: 3.5830 | LR: 4.98e-04 | 1094.7ms/step | 18.3 samples/s +Step 2410/44000 | Loss: 3.5444 | LR: 4.98e-04 | 1057.5ms/step | 18.9 samples/s +Step 2420/44000 | Loss: 3.5869 | LR: 4.98e-04 | 1104.2ms/step | 18.1 samples/s +Step 2430/44000 | Loss: 3.4856 | LR: 4.98e-04 | 1042.4ms/step | 19.2 samples/s +Step 2440/44000 | Loss: 3.5431 | LR: 4.98e-04 | 1135.5ms/step | 17.6 samples/s +Step 2450/44000 | Loss: 3.6237 | LR: 4.98e-04 | 1063.7ms/step | 18.8 samples/s +Step 2460/44000 | Loss: 3.5652 | LR: 4.98e-04 | 1031.8ms/step | 19.4 samples/s +Step 2470/44000 | Loss: 3.5660 | LR: 4.98e-04 | 1084.3ms/step | 18.4 samples/s +Step 2480/44000 | Loss: 3.5745 | LR: 4.98e-04 | 1072.1ms/step | 18.7 samples/s +Step 2490/44000 | Loss: 3.5438 | LR: 4.98e-04 | 1075.3ms/step | 18.6 samples/s +Step 2500/44000 | Loss: 3.4654 | LR: 4.98e-04 | 1069.7ms/step | 18.7 samples/s +Validation at step 2500: Loss = 3.5624, Perplexity = 35.25 +Step 2510/44000 | Loss: 3.6300 | LR: 4.98e-04 | 1436.9ms/step | 13.9 samples/s +Step 2520/44000 | Loss: 3.5445 | LR: 4.98e-04 | 1099.7ms/step | 18.2 samples/s +Step 2530/44000 | Loss: 3.4988 | LR: 4.98e-04 | 1063.0ms/step | 18.8 samples/s +Step 2540/44000 | Loss: 3.5154 | LR: 4.98e-04 | 1083.6ms/step | 18.5 samples/s +Step 2550/44000 | Loss: 3.5417 | LR: 4.98e-04 | 1093.7ms/step | 18.3 samples/s +Step 2560/44000 | Loss: 3.5439 | LR: 4.98e-04 | 1057.7ms/step | 18.9 samples/s +Step 2570/44000 | Loss: 3.5709 | LR: 4.98e-04 | 1059.5ms/step | 18.9 samples/s +Step 2580/44000 | Loss: 3.5335 | LR: 4.98e-04 | 1106.3ms/step | 18.1 samples/s +Step 2590/44000 | Loss: 3.5508 | LR: 4.98e-04 | 1143.8ms/step | 17.5 samples/s +Step 2600/44000 | Loss: 3.5174 | LR: 4.98e-04 | 1104.7ms/step | 18.1 samples/s +Step 2610/44000 | Loss: 3.5190 | LR: 4.98e-04 | 1064.6ms/step | 18.8 samples/s +Step 2620/44000 | Loss: 3.5501 | LR: 4.98e-04 | 1103.2ms/step | 18.1 samples/s +Step 2630/44000 | Loss: 3.5538 | LR: 4.98e-04 | 1087.8ms/step | 18.4 samples/s +Step 2640/44000 | Loss: 3.5102 | LR: 4.98e-04 | 1079.8ms/step | 18.5 samples/s +Step 2650/44000 | Loss: 3.5576 | LR: 4.98e-04 | 1095.1ms/step | 18.3 samples/s +Step 2660/44000 | Loss: 3.4308 | LR: 4.98e-04 | 1051.7ms/step | 19.0 samples/s +Step 2670/44000 | Loss: 3.4578 | LR: 4.98e-04 | 1041.6ms/step | 19.2 samples/s +Step 2680/44000 | Loss: 3.4902 | LR: 4.98e-04 | 1073.3ms/step | 18.6 samples/s +Step 2690/44000 | Loss: 3.5190 | LR: 4.98e-04 | 1052.2ms/step | 19.0 samples/s +Step 2700/44000 | Loss: 3.4991 | LR: 4.97e-04 | 1088.3ms/step | 18.4 samples/s +Step 2710/44000 | Loss: 3.4990 | LR: 4.97e-04 | 1110.1ms/step | 18.0 samples/s +Step 2720/44000 | Loss: 3.4485 | LR: 4.97e-04 | 1066.5ms/step | 18.8 samples/s +Step 2730/44000 | Loss: 3.4966 | LR: 4.97e-04 | 1095.4ms/step | 18.3 samples/s +Step 2740/44000 | Loss: 3.4760 | LR: 4.97e-04 | 1119.3ms/step | 17.9 samples/s +Step 2750/44000 | Loss: 3.5029 | LR: 4.97e-04 | 1064.0ms/step | 18.8 samples/s +Validation at step 2750: Loss = 3.5144, Perplexity = 33.59 +Step 2760/44000 | Loss: 3.5302 | LR: 4.97e-04 | 1438.3ms/step | 13.9 samples/s +Step 2770/44000 | Loss: 3.5752 | LR: 4.97e-04 | 1068.0ms/step | 18.7 samples/s +Step 2780/44000 | Loss: 3.4666 | LR: 4.97e-04 | 1068.5ms/step | 18.7 samples/s +Step 2790/44000 | Loss: 3.5131 | LR: 4.97e-04 | 1125.6ms/step | 17.8 samples/s +Step 2800/44000 | Loss: 3.5286 | LR: 4.97e-04 | 1162.2ms/step | 17.2 samples/s +Step 2810/44000 | Loss: 3.5421 | LR: 4.97e-04 | 1115.6ms/step | 17.9 samples/s +Step 2820/44000 | Loss: 3.4824 | LR: 4.97e-04 | 1095.1ms/step | 18.3 samples/s +Step 2830/44000 | Loss: 3.5398 | LR: 4.97e-04 | 1077.8ms/step | 18.6 samples/s +Step 2840/44000 | Loss: 3.5521 | LR: 4.97e-04 | 1137.7ms/step | 17.6 samples/s +Step 2850/44000 | Loss: 3.5175 | LR: 4.97e-04 | 1061.7ms/step | 18.8 samples/s +Step 2860/44000 | Loss: 3.4658 | LR: 4.97e-04 | 1084.9ms/step | 18.4 samples/s +Step 2870/44000 | Loss: 3.4793 | LR: 4.97e-04 | 1071.1ms/step | 18.7 samples/s +Step 2880/44000 | Loss: 3.4854 | LR: 4.97e-04 | 1069.6ms/step | 18.7 samples/s +Step 2890/44000 | Loss: 3.4363 | LR: 4.97e-04 | 1048.7ms/step | 19.1 samples/s +Step 2900/44000 | Loss: 3.4621 | LR: 4.97e-04 | 1095.8ms/step | 18.3 samples/s +Step 2910/44000 | Loss: 3.5084 | LR: 4.97e-04 | 1070.5ms/step | 18.7 samples/s +Step 2920/44000 | Loss: 3.5024 | LR: 4.97e-04 | 1047.9ms/step | 19.1 samples/s +Step 2930/44000 | Loss: 3.4581 | LR: 4.97e-04 | 1075.2ms/step | 18.6 samples/s +Step 2940/44000 | Loss: 3.4813 | LR: 4.97e-04 | 1077.5ms/step | 18.6 samples/s +Step 2950/44000 | Loss: 3.4689 | LR: 4.97e-04 | 1089.4ms/step | 18.4 samples/s +Step 2960/44000 | Loss: 3.5184 | LR: 4.97e-04 | 1120.1ms/step | 17.9 samples/s +Step 2970/44000 | Loss: 3.4158 | LR: 4.97e-04 | 1121.5ms/step | 17.8 samples/s +Step 2980/44000 | Loss: 3.4669 | LR: 4.97e-04 | 1088.1ms/step | 18.4 samples/s +Step 2990/44000 | Loss: 3.4770 | LR: 4.97e-04 | 1073.5ms/step | 18.6 samples/s +Step 3000/44000 | Loss: 3.4693 | LR: 4.97e-04 | 1173.8ms/step | 17.0 samples/s +Validation at step 3000: Loss = 3.4779, Perplexity = 32.39 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-3000 +Step 3010/44000 | Loss: 3.5010 | LR: 4.97e-04 | 1888.6ms/step | 10.6 samples/s +Step 3020/44000 | Loss: 3.4722 | LR: 4.97e-04 | 1080.8ms/step | 18.5 samples/s +Step 3030/44000 | Loss: 3.4819 | LR: 4.97e-04 | 1042.3ms/step | 19.2 samples/s +Step 3040/44000 | Loss: 3.4707 | LR: 4.97e-04 | 1052.7ms/step | 19.0 samples/s +Step 3050/44000 | Loss: 3.4476 | LR: 4.97e-04 | 1085.4ms/step | 18.4 samples/s +Step 3060/44000 | Loss: 3.4956 | LR: 4.96e-04 | 1077.5ms/step | 18.6 samples/s +Step 3070/44000 | Loss: 3.4660 | LR: 4.96e-04 | 1088.4ms/step | 18.4 samples/s +Step 3080/44000 | Loss: 3.4319 | LR: 4.96e-04 | 1058.5ms/step | 18.9 samples/s +Step 3090/44000 | Loss: 3.5447 | LR: 4.96e-04 | 1028.4ms/step | 19.4 samples/s +Step 3100/44000 | Loss: 3.4599 | LR: 4.96e-04 | 1050.7ms/step | 19.0 samples/s +Step 3110/44000 | Loss: 3.5079 | LR: 4.96e-04 | 1056.5ms/step | 18.9 samples/s +Step 3120/44000 | Loss: 3.4733 | LR: 4.96e-04 | 1063.0ms/step | 18.8 samples/s +Step 3130/44000 | Loss: 3.4273 | LR: 4.96e-04 | 1099.5ms/step | 18.2 samples/s +Step 3140/44000 | Loss: 3.4309 | LR: 4.96e-04 | 1073.0ms/step | 18.6 samples/s +Step 3150/44000 | Loss: 3.4282 | LR: 4.96e-04 | 1052.6ms/step | 19.0 samples/s +Step 3160/44000 | Loss: 3.4732 | LR: 4.96e-04 | 1057.2ms/step | 18.9 samples/s +Step 3170/44000 | Loss: 3.4866 | LR: 4.96e-04 | 1057.4ms/step | 18.9 samples/s +Step 3180/44000 | Loss: 3.4697 | LR: 4.96e-04 | 1057.3ms/step | 18.9 samples/s +Step 3190/44000 | Loss: 3.3753 | LR: 4.96e-04 | 1068.3ms/step | 18.7 samples/s +Step 3200/44000 | Loss: 3.4443 | LR: 4.96e-04 | 1160.9ms/step | 17.2 samples/s +Step 3210/44000 | Loss: 3.4534 | LR: 4.96e-04 | 1084.1ms/step | 18.4 samples/s +Step 3220/44000 | Loss: 3.4616 | LR: 4.96e-04 | 1099.3ms/step | 18.2 samples/s +Step 3230/44000 | Loss: 3.4096 | LR: 4.96e-04 | 1068.5ms/step | 18.7 samples/s +Step 3240/44000 | Loss: 3.4220 | LR: 4.96e-04 | 1124.9ms/step | 17.8 samples/s +Step 3250/44000 | Loss: 3.5011 | LR: 4.96e-04 | 1103.7ms/step | 18.1 samples/s +Validation at step 3250: Loss = 3.4474, Perplexity = 31.42 +Step 3260/44000 | Loss: 3.4971 | LR: 4.96e-04 | 1438.1ms/step | 13.9 samples/s +Step 3270/44000 | Loss: 3.4056 | LR: 4.96e-04 | 1063.2ms/step | 18.8 samples/s +Step 3280/44000 | Loss: 3.4196 | LR: 4.96e-04 | 1067.6ms/step | 18.7 samples/s +Step 3290/44000 | Loss: 3.3884 | LR: 4.96e-04 | 1067.4ms/step | 18.7 samples/s +Step 3300/44000 | Loss: 3.3623 | LR: 4.96e-04 | 1042.0ms/step | 19.2 samples/s +Step 3310/44000 | Loss: 3.4086 | LR: 4.96e-04 | 1068.1ms/step | 18.7 samples/s +Step 3320/44000 | Loss: 3.4203 | LR: 4.96e-04 | 1097.5ms/step | 18.2 samples/s +Step 3330/44000 | Loss: 3.4356 | LR: 4.96e-04 | 1097.7ms/step | 18.2 samples/s +Step 3340/44000 | Loss: 3.4536 | LR: 4.96e-04 | 1108.9ms/step | 18.0 samples/s +Step 3350/44000 | Loss: 3.4533 | LR: 4.96e-04 | 1088.4ms/step | 18.4 samples/s +Step 3360/44000 | Loss: 3.4625 | LR: 4.96e-04 | 1073.9ms/step | 18.6 samples/s +Step 3370/44000 | Loss: 3.4146 | LR: 4.95e-04 | 1087.9ms/step | 18.4 samples/s +Step 3380/44000 | Loss: 3.3691 | LR: 4.95e-04 | 1057.1ms/step | 18.9 samples/s +Step 3390/44000 | Loss: 3.4194 | LR: 4.95e-04 | 1073.4ms/step | 18.6 samples/s +Step 3400/44000 | Loss: 3.4222 | LR: 4.95e-04 | 1099.8ms/step | 18.2 samples/s +Step 3410/44000 | Loss: 3.3990 | LR: 4.95e-04 | 1192.6ms/step | 16.8 samples/s +Step 3420/44000 | Loss: 3.3711 | LR: 4.95e-04 | 1109.9ms/step | 18.0 samples/s +Step 3430/44000 | Loss: 3.4968 | LR: 4.95e-04 | 1068.4ms/step | 18.7 samples/s +Step 3440/44000 | Loss: 3.4650 | LR: 4.95e-04 | 1072.7ms/step | 18.6 samples/s +Step 3450/44000 | Loss: 3.4103 | LR: 4.95e-04 | 1070.7ms/step | 18.7 samples/s +Step 3460/44000 | Loss: 3.3907 | LR: 4.95e-04 | 1085.8ms/step | 18.4 samples/s +Step 3470/44000 | Loss: 3.4983 | LR: 4.95e-04 | 1077.9ms/step | 18.6 samples/s +Step 3480/44000 | Loss: 3.4424 | LR: 4.95e-04 | 1043.0ms/step | 19.2 samples/s +Step 3490/44000 | Loss: 3.4136 | LR: 4.95e-04 | 1047.4ms/step | 19.1 samples/s +Step 3500/44000 | Loss: 3.3340 | LR: 4.95e-04 | 1057.9ms/step | 18.9 samples/s +Validation at step 3500: Loss = 3.4212, Perplexity = 30.61 +Step 3510/44000 | Loss: 3.3625 | LR: 4.95e-04 | 1438.0ms/step | 13.9 samples/s +Step 3520/44000 | Loss: 3.3533 | LR: 4.95e-04 | 1103.9ms/step | 18.1 samples/s +Step 3530/44000 | Loss: 3.4081 | LR: 4.95e-04 | 1073.7ms/step | 18.6 samples/s +Step 3540/44000 | Loss: 3.4653 | LR: 4.95e-04 | 1099.4ms/step | 18.2 samples/s +Step 3550/44000 | Loss: 3.3634 | LR: 4.95e-04 | 1125.4ms/step | 17.8 samples/s +Step 3560/44000 | Loss: 3.3230 | LR: 4.95e-04 | 1063.3ms/step | 18.8 samples/s +Step 3570/44000 | Loss: 3.4338 | LR: 4.95e-04 | 1062.7ms/step | 18.8 samples/s +Step 3580/44000 | Loss: 3.3839 | LR: 4.95e-04 | 1083.3ms/step | 18.5 samples/s +Step 3590/44000 | Loss: 3.3744 | LR: 4.95e-04 | 1083.7ms/step | 18.5 samples/s +Step 3600/44000 | Loss: 3.4054 | LR: 4.95e-04 | 1088.7ms/step | 18.4 samples/s +Step 3610/44000 | Loss: 3.3773 | LR: 4.95e-04 | 1166.9ms/step | 17.1 samples/s +Step 3620/44000 | Loss: 3.3926 | LR: 4.95e-04 | 1120.2ms/step | 17.9 samples/s +Step 3630/44000 | Loss: 3.4388 | LR: 4.95e-04 | 1095.0ms/step | 18.3 samples/s +Step 3640/44000 | Loss: 3.4182 | LR: 4.95e-04 | 1100.6ms/step | 18.2 samples/s +Step 3650/44000 | Loss: 3.4318 | LR: 4.94e-04 | 1098.0ms/step | 18.2 samples/s +Step 3660/44000 | Loss: 3.4612 | LR: 4.94e-04 | 1144.7ms/step | 17.5 samples/s +Step 3670/44000 | Loss: 3.4157 | LR: 4.94e-04 | 1089.1ms/step | 18.4 samples/s +Step 3680/44000 | Loss: 3.4550 | LR: 4.94e-04 | 1130.2ms/step | 17.7 samples/s +Step 3690/44000 | Loss: 3.3804 | LR: 4.94e-04 | 1109.5ms/step | 18.0 samples/s +Step 3700/44000 | Loss: 3.3879 | LR: 4.94e-04 | 1109.0ms/step | 18.0 samples/s +Step 3710/44000 | Loss: 3.3596 | LR: 4.94e-04 | 1079.4ms/step | 18.5 samples/s +Step 3720/44000 | Loss: 3.3633 | LR: 4.94e-04 | 1083.0ms/step | 18.5 samples/s +Step 3730/44000 | Loss: 3.3929 | LR: 4.94e-04 | 1093.2ms/step | 18.3 samples/s +Step 3740/44000 | Loss: 3.4019 | LR: 4.94e-04 | 1078.7ms/step | 18.5 samples/s +Step 3750/44000 | Loss: 3.4330 | LR: 4.94e-04 | 1041.8ms/step | 19.2 samples/s +Validation at step 3750: Loss = 3.3990, Perplexity = 29.93 +Step 3760/44000 | Loss: 3.3787 | LR: 4.94e-04 | 1458.8ms/step | 13.7 samples/s +Step 3770/44000 | Loss: 3.3989 | LR: 4.94e-04 | 1124.8ms/step | 17.8 samples/s +Step 3780/44000 | Loss: 3.3295 | LR: 4.94e-04 | 1083.5ms/step | 18.5 samples/s +Step 3790/44000 | Loss: 3.4597 | LR: 4.94e-04 | 1073.7ms/step | 18.6 samples/s +Step 3800/44000 | Loss: 3.3148 | LR: 4.94e-04 | 1125.0ms/step | 17.8 samples/s +Step 3810/44000 | Loss: 3.4319 | LR: 4.94e-04 | 1150.8ms/step | 17.4 samples/s +Step 3820/44000 | Loss: 3.3512 | LR: 4.94e-04 | 1074.0ms/step | 18.6 samples/s +Step 3830/44000 | Loss: 3.3953 | LR: 4.94e-04 | 1031.6ms/step | 19.4 samples/s +Step 3840/44000 | Loss: 3.4056 | LR: 4.94e-04 | 1073.5ms/step | 18.6 samples/s +Step 3850/44000 | Loss: 3.3630 | LR: 4.94e-04 | 1060.7ms/step | 18.9 samples/s +Step 3860/44000 | Loss: 3.4212 | LR: 4.94e-04 | 1059.7ms/step | 18.9 samples/s +Step 3870/44000 | Loss: 3.4252 | LR: 4.94e-04 | 1110.7ms/step | 18.0 samples/s +Step 3880/44000 | Loss: 3.3608 | LR: 4.94e-04 | 1010.5ms/step | 19.8 samples/s +Step 3890/44000 | Loss: 3.3834 | LR: 4.94e-04 | 1068.3ms/step | 18.7 samples/s +Step 3900/44000 | Loss: 3.4191 | LR: 4.93e-04 | 1062.6ms/step | 18.8 samples/s +Step 3910/44000 | Loss: 3.3033 | LR: 4.93e-04 | 1078.1ms/step | 18.6 samples/s +Step 3920/44000 | Loss: 3.3145 | LR: 4.93e-04 | 1088.0ms/step | 18.4 samples/s +Step 3930/44000 | Loss: 3.4278 | LR: 4.93e-04 | 1075.1ms/step | 18.6 samples/s +Step 3940/44000 | Loss: 3.3663 | LR: 4.93e-04 | 1076.8ms/step | 18.6 samples/s +Step 3950/44000 | Loss: 3.3822 | LR: 4.93e-04 | 1096.0ms/step | 18.2 samples/s +Step 3960/44000 | Loss: 3.3734 | LR: 4.93e-04 | 1078.2ms/step | 18.5 samples/s +Step 3970/44000 | Loss: 3.3734 | LR: 4.93e-04 | 1027.4ms/step | 19.5 samples/s +Step 3980/44000 | Loss: 3.3458 | LR: 4.93e-04 | 1045.1ms/step | 19.1 samples/s +Step 3990/44000 | Loss: 3.3039 | LR: 4.93e-04 | 1091.0ms/step | 18.3 samples/s +Step 4000/44000 | Loss: 3.3724 | LR: 4.93e-04 | 1062.8ms/step | 18.8 samples/s +Validation at step 4000: Loss = 3.3743, Perplexity = 29.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-4000 +Step 4010/44000 | Loss: 3.2877 | LR: 4.93e-04 | 1870.0ms/step | 10.7 samples/s +Step 4020/44000 | Loss: 3.3148 | LR: 4.93e-04 | 1110.6ms/step | 18.0 samples/s +Step 4030/44000 | Loss: 3.4183 | LR: 4.93e-04 | 1109.8ms/step | 18.0 samples/s +Step 4040/44000 | Loss: 3.4262 | LR: 4.93e-04 | 1057.3ms/step | 18.9 samples/s +Step 4050/44000 | Loss: 3.3349 | LR: 4.93e-04 | 1125.6ms/step | 17.8 samples/s +Step 4060/44000 | Loss: 3.3423 | LR: 4.93e-04 | 1084.7ms/step | 18.4 samples/s +Step 4070/44000 | Loss: 3.3925 | LR: 4.93e-04 | 1101.0ms/step | 18.2 samples/s +Step 4080/44000 | Loss: 3.3275 | LR: 4.93e-04 | 1013.2ms/step | 19.7 samples/s +Step 4090/44000 | Loss: 3.3685 | LR: 4.93e-04 | 1068.7ms/step | 18.7 samples/s +Step 4100/44000 | Loss: 3.3572 | LR: 4.93e-04 | 1059.8ms/step | 18.9 samples/s +Step 4110/44000 | Loss: 3.3495 | LR: 4.93e-04 | 1106.7ms/step | 18.1 samples/s +Step 4120/44000 | Loss: 3.3638 | LR: 4.93e-04 | 1063.5ms/step | 18.8 samples/s +Step 4130/44000 | Loss: 3.3375 | LR: 4.93e-04 | 1071.2ms/step | 18.7 samples/s +Step 4140/44000 | Loss: 3.2975 | LR: 4.92e-04 | 1068.9ms/step | 18.7 samples/s +Step 4150/44000 | Loss: 3.2933 | LR: 4.92e-04 | 1089.9ms/step | 18.4 samples/s +Step 4160/44000 | Loss: 3.3542 | LR: 4.92e-04 | 1105.0ms/step | 18.1 samples/s +Step 4170/44000 | Loss: 3.3785 | LR: 4.92e-04 | 1068.1ms/step | 18.7 samples/s +Step 4180/44000 | Loss: 3.3175 | LR: 4.92e-04 | 1067.1ms/step | 18.7 samples/s +Step 4190/44000 | Loss: 3.4087 | LR: 4.92e-04 | 1107.3ms/step | 18.1 samples/s +Step 4200/44000 | Loss: 3.3857 | LR: 4.92e-04 | 1090.4ms/step | 18.3 samples/s +Step 4210/44000 | Loss: 3.3869 | LR: 4.92e-04 | 1143.9ms/step | 17.5 samples/s +Step 4220/44000 | Loss: 3.3387 | LR: 4.92e-04 | 1219.3ms/step | 16.4 samples/s +Step 4230/44000 | Loss: 3.3375 | LR: 4.92e-04 | 1087.8ms/step | 18.4 samples/s +Step 4240/44000 | Loss: 3.3744 | LR: 4.92e-04 | 1078.4ms/step | 18.5 samples/s +Step 4250/44000 | Loss: 3.3053 | LR: 4.92e-04 | 1076.0ms/step | 18.6 samples/s +Validation at step 4250: Loss = 3.3547, Perplexity = 28.64 +Step 4260/44000 | Loss: 3.3222 | LR: 4.92e-04 | 1448.5ms/step | 13.8 samples/s +Step 4270/44000 | Loss: 3.3034 | LR: 4.92e-04 | 1047.3ms/step | 19.1 samples/s +Step 4280/44000 | Loss: 3.3080 | LR: 4.92e-04 | 1089.0ms/step | 18.4 samples/s +Step 4290/44000 | Loss: 3.3236 | LR: 4.92e-04 | 1079.3ms/step | 18.5 samples/s +Step 4300/44000 | Loss: 3.3392 | LR: 4.92e-04 | 1028.6ms/step | 19.4 samples/s +Step 4310/44000 | Loss: 3.3587 | LR: 4.92e-04 | 1081.7ms/step | 18.5 samples/s +Step 4320/44000 | Loss: 3.2693 | LR: 4.92e-04 | 1100.2ms/step | 18.2 samples/s +Step 4330/44000 | Loss: 3.3063 | LR: 4.92e-04 | 1094.4ms/step | 18.3 samples/s +Step 4340/44000 | Loss: 3.3209 | LR: 4.92e-04 | 1036.8ms/step | 19.3 samples/s +Step 4350/44000 | Loss: 3.3569 | LR: 4.92e-04 | 1049.5ms/step | 19.1 samples/s +Step 4360/44000 | Loss: 3.3001 | LR: 4.91e-04 | 1087.0ms/step | 18.4 samples/s +Step 4370/44000 | Loss: 3.3108 | LR: 4.91e-04 | 1115.4ms/step | 17.9 samples/s +Step 4380/44000 | Loss: 3.3489 | LR: 4.91e-04 | 1058.2ms/step | 18.9 samples/s +Step 4390/44000 | Loss: 3.3667 | LR: 4.91e-04 | 1072.8ms/step | 18.6 samples/s +Step 4400/44000 | Loss: 3.3220 | LR: 4.91e-04 | 1090.3ms/step | 18.3 samples/s +Step 4410/44000 | Loss: 3.2989 | LR: 4.91e-04 | 1089.3ms/step | 18.4 samples/s +Step 4420/44000 | Loss: 3.3503 | LR: 4.91e-04 | 1120.6ms/step | 17.8 samples/s +Step 4430/44000 | Loss: 3.3269 | LR: 4.91e-04 | 1156.4ms/step | 17.3 samples/s +Step 4440/44000 | Loss: 3.2956 | LR: 4.91e-04 | 1095.2ms/step | 18.3 samples/s +Step 4450/44000 | Loss: 3.3167 | LR: 4.91e-04 | 1042.7ms/step | 19.2 samples/s +Step 4460/44000 | Loss: 3.3472 | LR: 4.91e-04 | 1094.2ms/step | 18.3 samples/s +Step 4470/44000 | Loss: 3.3544 | LR: 4.91e-04 | 1104.0ms/step | 18.1 samples/s +Step 4480/44000 | Loss: 3.3309 | LR: 4.91e-04 | 1049.1ms/step | 19.1 samples/s +Step 4490/44000 | Loss: 3.3424 | LR: 4.91e-04 | 1093.7ms/step | 18.3 samples/s +Step 4500/44000 | Loss: 3.2976 | LR: 4.91e-04 | 1059.8ms/step | 18.9 samples/s +Validation at step 4500: Loss = 3.3364, Perplexity = 28.12 +Step 4510/44000 | Loss: 3.3177 | LR: 4.91e-04 | 1431.8ms/step | 14.0 samples/s +Step 4520/44000 | Loss: 3.3098 | LR: 4.91e-04 | 1119.3ms/step | 17.9 samples/s +Step 4530/44000 | Loss: 3.3287 | LR: 4.91e-04 | 1085.5ms/step | 18.4 samples/s +Step 4540/44000 | Loss: 3.3670 | LR: 4.91e-04 | 1089.7ms/step | 18.4 samples/s +Step 4550/44000 | Loss: 3.3444 | LR: 4.91e-04 | 1073.5ms/step | 18.6 samples/s +Step 4560/44000 | Loss: 3.3707 | LR: 4.90e-04 | 1074.1ms/step | 18.6 samples/s +Step 4570/44000 | Loss: 3.2583 | LR: 4.90e-04 | 1065.2ms/step | 18.8 samples/s +Step 4580/44000 | Loss: 3.3507 | LR: 4.90e-04 | 1087.0ms/step | 18.4 samples/s +Step 4590/44000 | Loss: 3.3136 | LR: 4.90e-04 | 1104.9ms/step | 18.1 samples/s +Step 4600/44000 | Loss: 3.3029 | LR: 4.90e-04 | 1099.7ms/step | 18.2 samples/s +Step 4610/44000 | Loss: 3.3721 | LR: 4.90e-04 | 1115.1ms/step | 17.9 samples/s +Step 4620/44000 | Loss: 3.3259 | LR: 4.90e-04 | 1094.5ms/step | 18.3 samples/s +Step 4630/44000 | Loss: 3.2955 | LR: 4.90e-04 | 1192.1ms/step | 16.8 samples/s +Step 4640/44000 | Loss: 3.2727 | LR: 4.90e-04 | 1096.3ms/step | 18.2 samples/s +Step 4650/44000 | Loss: 3.3138 | LR: 4.90e-04 | 1087.3ms/step | 18.4 samples/s +Step 4660/44000 | Loss: 3.3639 | LR: 4.90e-04 | 1107.0ms/step | 18.1 samples/s +Step 4670/44000 | Loss: 3.2901 | LR: 4.90e-04 | 1069.8ms/step | 18.7 samples/s +Step 4680/44000 | Loss: 3.3284 | LR: 4.90e-04 | 1104.5ms/step | 18.1 samples/s +Step 4690/44000 | Loss: 3.2334 | LR: 4.90e-04 | 1063.8ms/step | 18.8 samples/s +Step 4700/44000 | Loss: 3.3632 | LR: 4.90e-04 | 1073.3ms/step | 18.6 samples/s +Step 4710/44000 | Loss: 3.3410 | LR: 4.90e-04 | 1074.2ms/step | 18.6 samples/s +Step 4720/44000 | Loss: 3.2787 | LR: 4.90e-04 | 1067.6ms/step | 18.7 samples/s +Step 4730/44000 | Loss: 3.2888 | LR: 4.90e-04 | 1095.7ms/step | 18.3 samples/s +Step 4740/44000 | Loss: 3.2899 | LR: 4.90e-04 | 1079.7ms/step | 18.5 samples/s +Step 4750/44000 | Loss: 3.3086 | LR: 4.90e-04 | 1067.4ms/step | 18.7 samples/s +Validation at step 4750: Loss = 3.3159, Perplexity = 27.55 +Step 4760/44000 | Loss: 3.3508 | LR: 4.89e-04 | 1428.0ms/step | 14.0 samples/s +Step 4770/44000 | Loss: 3.3515 | LR: 4.89e-04 | 1095.6ms/step | 18.3 samples/s +Step 4780/44000 | Loss: 3.2869 | LR: 4.89e-04 | 1088.4ms/step | 18.4 samples/s +Step 4790/44000 | Loss: 3.3058 | LR: 4.89e-04 | 1052.9ms/step | 19.0 samples/s +Step 4800/44000 | Loss: 3.2947 | LR: 4.89e-04 | 1026.6ms/step | 19.5 samples/s +Step 4810/44000 | Loss: 3.2789 | LR: 4.89e-04 | 1109.8ms/step | 18.0 samples/s +Step 4820/44000 | Loss: 3.3006 | LR: 4.89e-04 | 1068.0ms/step | 18.7 samples/s +Step 4830/44000 | Loss: 3.2712 | LR: 4.89e-04 | 1152.1ms/step | 17.4 samples/s +Step 4840/44000 | Loss: 3.3050 | LR: 4.89e-04 | 1131.3ms/step | 17.7 samples/s +Step 4850/44000 | Loss: 3.2681 | LR: 4.89e-04 | 1089.9ms/step | 18.4 samples/s +Step 4860/44000 | Loss: 3.3068 | LR: 4.89e-04 | 1079.9ms/step | 18.5 samples/s +Step 4870/44000 | Loss: 3.3166 | LR: 4.89e-04 | 1118.4ms/step | 17.9 samples/s +Step 4880/44000 | Loss: 3.2650 | LR: 4.89e-04 | 1063.8ms/step | 18.8 samples/s +Step 4890/44000 | Loss: 3.2688 | LR: 4.89e-04 | 1110.0ms/step | 18.0 samples/s +Step 4900/44000 | Loss: 3.2708 | LR: 4.89e-04 | 1052.7ms/step | 19.0 samples/s +Step 4910/44000 | Loss: 3.3193 | LR: 4.89e-04 | 1078.8ms/step | 18.5 samples/s +Step 4920/44000 | Loss: 3.3864 | LR: 4.89e-04 | 1073.0ms/step | 18.6 samples/s +Step 4930/44000 | Loss: 3.3426 | LR: 4.89e-04 | 1056.1ms/step | 18.9 samples/s +Step 4940/44000 | Loss: 3.3290 | LR: 4.89e-04 | 1096.5ms/step | 18.2 samples/s +Step 4950/44000 | Loss: 3.2938 | LR: 4.88e-04 | 1063.3ms/step | 18.8 samples/s +Step 4960/44000 | Loss: 3.3428 | LR: 4.88e-04 | 1088.8ms/step | 18.4 samples/s +Step 4970/44000 | Loss: 3.3209 | LR: 4.88e-04 | 1082.7ms/step | 18.5 samples/s +Step 4980/44000 | Loss: 3.2688 | LR: 4.88e-04 | 1106.2ms/step | 18.1 samples/s +Step 4990/44000 | Loss: 3.2775 | LR: 4.88e-04 | 1056.4ms/step | 18.9 samples/s +Step 5000/44000 | Loss: 3.2328 | LR: 4.88e-04 | 1090.4ms/step | 18.3 samples/s +Validation at step 5000: Loss = 3.3032, Perplexity = 27.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-5000 +Step 5010/44000 | Loss: 3.3145 | LR: 4.88e-04 | 1907.7ms/step | 10.5 samples/s +Step 5020/44000 | Loss: 3.2898 | LR: 4.88e-04 | 1109.7ms/step | 18.0 samples/s +Step 5030/44000 | Loss: 3.2831 | LR: 4.88e-04 | 1150.8ms/step | 17.4 samples/s +Step 5040/44000 | Loss: 3.2705 | LR: 4.88e-04 | 1101.3ms/step | 18.2 samples/s +Step 5050/44000 | Loss: 3.2387 | LR: 4.88e-04 | 1093.7ms/step | 18.3 samples/s +Step 5060/44000 | Loss: 3.1663 | LR: 4.88e-04 | 1065.8ms/step | 18.8 samples/s +Step 5070/44000 | Loss: 3.2966 | LR: 4.88e-04 | 1097.3ms/step | 18.2 samples/s +Step 5080/44000 | Loss: 3.2704 | LR: 4.88e-04 | 1073.7ms/step | 18.6 samples/s +Step 5090/44000 | Loss: 3.3297 | LR: 4.88e-04 | 1063.2ms/step | 18.8 samples/s +Step 5100/44000 | Loss: 3.3072 | LR: 4.88e-04 | 1073.3ms/step | 18.6 samples/s +Step 5110/44000 | Loss: 3.2739 | LR: 4.88e-04 | 1069.2ms/step | 18.7 samples/s +Step 5120/44000 | Loss: 3.3341 | LR: 4.88e-04 | 1072.8ms/step | 18.6 samples/s +Step 5130/44000 | Loss: 3.2650 | LR: 4.87e-04 | 1058.0ms/step | 18.9 samples/s +Step 5140/44000 | Loss: 3.3091 | LR: 4.87e-04 | 1062.6ms/step | 18.8 samples/s +Step 5150/44000 | Loss: 3.2804 | LR: 4.87e-04 | 1090.5ms/step | 18.3 samples/s +Step 5160/44000 | Loss: 3.3202 | LR: 4.87e-04 | 1057.8ms/step | 18.9 samples/s +Step 5170/44000 | Loss: 3.2190 | LR: 4.87e-04 | 1087.8ms/step | 18.4 samples/s +Step 5180/44000 | Loss: 3.2480 | LR: 4.87e-04 | 1120.5ms/step | 17.8 samples/s +Step 5190/44000 | Loss: 3.2670 | LR: 4.87e-04 | 1047.0ms/step | 19.1 samples/s +Step 5200/44000 | Loss: 3.2785 | LR: 4.87e-04 | 1063.8ms/step | 18.8 samples/s +Step 5210/44000 | Loss: 3.2426 | LR: 4.87e-04 | 1116.1ms/step | 17.9 samples/s +Step 5220/44000 | Loss: 3.2861 | LR: 4.87e-04 | 1052.6ms/step | 19.0 samples/s +Step 5230/44000 | Loss: 3.2197 | LR: 4.87e-04 | 1089.3ms/step | 18.4 samples/s +Step 5240/44000 | Loss: 3.2256 | LR: 4.87e-04 | 1155.9ms/step | 17.3 samples/s +Step 5250/44000 | Loss: 3.2024 | LR: 4.87e-04 | 1126.2ms/step | 17.8 samples/s +Validation at step 5250: Loss = 3.2890, Perplexity = 26.82 +Step 5260/44000 | Loss: 3.3515 | LR: 4.87e-04 | 1437.8ms/step | 13.9 samples/s +Step 5270/44000 | Loss: 3.2698 | LR: 4.87e-04 | 1085.9ms/step | 18.4 samples/s +Step 5280/44000 | Loss: 3.3015 | LR: 4.87e-04 | 1132.0ms/step | 17.7 samples/s +Step 5290/44000 | Loss: 3.2291 | LR: 4.87e-04 | 1095.1ms/step | 18.3 samples/s +Step 5300/44000 | Loss: 3.3075 | LR: 4.86e-04 | 1072.3ms/step | 18.7 samples/s +Step 5310/44000 | Loss: 3.3005 | LR: 4.86e-04 | 1079.1ms/step | 18.5 samples/s +Step 5320/44000 | Loss: 3.3145 | LR: 4.86e-04 | 1068.5ms/step | 18.7 samples/s +Step 5330/44000 | Loss: 3.2701 | LR: 4.86e-04 | 1042.2ms/step | 19.2 samples/s +Step 5340/44000 | Loss: 3.2827 | LR: 4.86e-04 | 1074.0ms/step | 18.6 samples/s +Step 5350/44000 | Loss: 3.2839 | LR: 4.86e-04 | 1047.9ms/step | 19.1 samples/s +Step 5360/44000 | Loss: 3.2160 | LR: 4.86e-04 | 1043.8ms/step | 19.2 samples/s +Step 5370/44000 | Loss: 3.2729 | LR: 4.86e-04 | 1088.1ms/step | 18.4 samples/s +Step 5380/44000 | Loss: 3.3234 | LR: 4.86e-04 | 1079.1ms/step | 18.5 samples/s +Step 5390/44000 | Loss: 3.1952 | LR: 4.86e-04 | 1053.2ms/step | 19.0 samples/s +Step 5400/44000 | Loss: 3.2954 | LR: 4.86e-04 | 1099.5ms/step | 18.2 samples/s +Step 5410/44000 | Loss: 3.2564 | LR: 4.86e-04 | 1079.4ms/step | 18.5 samples/s +Step 5420/44000 | Loss: 3.2039 | LR: 4.86e-04 | 1055.9ms/step | 18.9 samples/s +Step 5430/44000 | Loss: 3.2622 | LR: 4.86e-04 | 1102.5ms/step | 18.1 samples/s +Step 5440/44000 | Loss: 3.2594 | LR: 4.86e-04 | 1047.6ms/step | 19.1 samples/s +Step 5450/44000 | Loss: 3.2652 | LR: 4.86e-04 | 1058.6ms/step | 18.9 samples/s +Step 5460/44000 | Loss: 3.1891 | LR: 4.86e-04 | 1047.9ms/step | 19.1 samples/s +Step 5470/44000 | Loss: 3.2540 | LR: 4.85e-04 | 1099.3ms/step | 18.2 samples/s +Step 5480/44000 | Loss: 3.2896 | LR: 4.85e-04 | 1074.0ms/step | 18.6 samples/s +Step 5490/44000 | Loss: 3.3401 | LR: 4.85e-04 | 1079.3ms/step | 18.5 samples/s +Step 5500/44000 | Loss: 3.2636 | LR: 4.85e-04 | 1104.0ms/step | 18.1 samples/s +Validation at step 5500: Loss = 3.2774, Perplexity = 26.51 +Step 5510/44000 | Loss: 3.2673 | LR: 4.85e-04 | 1408.2ms/step | 14.2 samples/s +Step 5520/44000 | Loss: 3.3065 | LR: 4.85e-04 | 1097.2ms/step | 18.2 samples/s +Step 5530/44000 | Loss: 3.2042 | LR: 4.85e-04 | 1092.1ms/step | 18.3 samples/s +Step 5540/44000 | Loss: 3.2905 | LR: 4.85e-04 | 1100.0ms/step | 18.2 samples/s +Step 5550/44000 | Loss: 3.2589 | LR: 4.85e-04 | 1068.6ms/step | 18.7 samples/s +Step 5560/44000 | Loss: 3.2476 | LR: 4.85e-04 | 1073.6ms/step | 18.6 samples/s +Step 5570/44000 | Loss: 3.3120 | LR: 4.85e-04 | 1073.8ms/step | 18.6 samples/s +Step 5580/44000 | Loss: 3.2730 | LR: 4.85e-04 | 1089.5ms/step | 18.4 samples/s +Step 5590/44000 | Loss: 3.2843 | LR: 4.85e-04 | 1053.1ms/step | 19.0 samples/s +Step 5600/44000 | Loss: 3.2457 | LR: 4.85e-04 | 1084.1ms/step | 18.4 samples/s +Step 5610/44000 | Loss: 3.3103 | LR: 4.85e-04 | 1058.6ms/step | 18.9 samples/s +Step 5620/44000 | Loss: 3.2490 | LR: 4.85e-04 | 1078.5ms/step | 18.5 samples/s +Step 5630/44000 | Loss: 3.1721 | LR: 4.84e-04 | 1061.6ms/step | 18.8 samples/s +Step 5640/44000 | Loss: 3.2495 | LR: 4.84e-04 | 1065.2ms/step | 18.8 samples/s +Step 5650/44000 | Loss: 3.3618 | LR: 4.84e-04 | 1152.0ms/step | 17.4 samples/s +Step 5660/44000 | Loss: 3.2622 | LR: 4.84e-04 | 1094.4ms/step | 18.3 samples/s +Step 5670/44000 | Loss: 3.2997 | LR: 4.84e-04 | 1063.7ms/step | 18.8 samples/s +Step 5680/44000 | Loss: 3.2308 | LR: 4.84e-04 | 1068.5ms/step | 18.7 samples/s +Step 5690/44000 | Loss: 3.2357 | LR: 4.84e-04 | 1110.8ms/step | 18.0 samples/s +Step 5700/44000 | Loss: 3.2325 | LR: 4.84e-04 | 1089.9ms/step | 18.4 samples/s +Step 5710/44000 | Loss: 3.2320 | LR: 4.84e-04 | 1079.2ms/step | 18.5 samples/s +Step 5720/44000 | Loss: 3.2487 | LR: 4.84e-04 | 1085.7ms/step | 18.4 samples/s +Step 5730/44000 | Loss: 3.2723 | LR: 4.84e-04 | 1050.8ms/step | 19.0 samples/s +Step 5740/44000 | Loss: 3.2437 | LR: 4.84e-04 | 1070.4ms/step | 18.7 samples/s +Step 5750/44000 | Loss: 3.2195 | LR: 4.84e-04 | 1061.9ms/step | 18.8 samples/s +Validation at step 5750: Loss = 3.2626, Perplexity = 26.12 +Step 5760/44000 | Loss: 3.3022 | LR: 4.84e-04 | 1475.8ms/step | 13.6 samples/s +Step 5770/44000 | Loss: 3.2303 | LR: 4.84e-04 | 1047.3ms/step | 19.1 samples/s +Step 5780/44000 | Loss: 3.2318 | LR: 4.83e-04 | 1092.3ms/step | 18.3 samples/s +Step 5790/44000 | Loss: 3.2507 | LR: 4.83e-04 | 1093.4ms/step | 18.3 samples/s +Step 5800/44000 | Loss: 3.2933 | LR: 4.83e-04 | 1078.5ms/step | 18.5 samples/s +Step 5810/44000 | Loss: 3.3023 | LR: 4.83e-04 | 1062.3ms/step | 18.8 samples/s +Step 5820/44000 | Loss: 3.2273 | LR: 4.83e-04 | 1089.7ms/step | 18.4 samples/s +Step 5830/44000 | Loss: 3.2058 | LR: 4.83e-04 | 1093.6ms/step | 18.3 samples/s +Step 5840/44000 | Loss: 3.2655 | LR: 4.83e-04 | 1074.5ms/step | 18.6 samples/s +Step 5850/44000 | Loss: 3.1885 | LR: 4.83e-04 | 1120.4ms/step | 17.9 samples/s +Step 5860/44000 | Loss: 3.2086 | LR: 4.83e-04 | 1157.4ms/step | 17.3 samples/s +Step 5870/44000 | Loss: 3.2246 | LR: 4.83e-04 | 1063.1ms/step | 18.8 samples/s +Step 5880/44000 | Loss: 3.1812 | LR: 4.83e-04 | 1068.2ms/step | 18.7 samples/s +Step 5890/44000 | Loss: 3.2886 | LR: 4.83e-04 | 1125.8ms/step | 17.8 samples/s +Step 5900/44000 | Loss: 3.2579 | LR: 4.83e-04 | 1079.9ms/step | 18.5 samples/s +Step 5910/44000 | Loss: 3.2888 | LR: 4.83e-04 | 1062.8ms/step | 18.8 samples/s +Step 5920/44000 | Loss: 3.2054 | LR: 4.83e-04 | 1057.9ms/step | 18.9 samples/s +Step 5930/44000 | Loss: 3.2523 | LR: 4.83e-04 | 1052.4ms/step | 19.0 samples/s +Step 5940/44000 | Loss: 3.2264 | LR: 4.82e-04 | 1110.7ms/step | 18.0 samples/s +Step 5950/44000 | Loss: 3.2198 | LR: 4.82e-04 | 1037.0ms/step | 19.3 samples/s +Step 5960/44000 | Loss: 3.2455 | LR: 4.82e-04 | 1089.4ms/step | 18.4 samples/s +Step 5970/44000 | Loss: 3.1942 | LR: 4.82e-04 | 1084.3ms/step | 18.4 samples/s +Step 5980/44000 | Loss: 3.2540 | LR: 4.82e-04 | 1073.1ms/step | 18.6 samples/s +Step 5990/44000 | Loss: 3.2634 | LR: 4.82e-04 | 1069.5ms/step | 18.7 samples/s +Step 6000/44000 | Loss: 3.2869 | LR: 4.82e-04 | 1077.3ms/step | 18.6 samples/s +Validation at step 6000: Loss = 3.2516, Perplexity = 25.83 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-6000 +Step 6010/44000 | Loss: 3.2042 | LR: 4.82e-04 | 1876.3ms/step | 10.7 samples/s +Step 6020/44000 | Loss: 3.2056 | LR: 4.82e-04 | 1052.6ms/step | 19.0 samples/s +Step 6030/44000 | Loss: 3.2973 | LR: 4.82e-04 | 1076.1ms/step | 18.6 samples/s +Step 6040/44000 | Loss: 3.2715 | LR: 4.82e-04 | 1076.1ms/step | 18.6 samples/s +Step 6050/44000 | Loss: 3.1793 | LR: 4.82e-04 | 1109.7ms/step | 18.0 samples/s +Step 6060/44000 | Loss: 3.2283 | LR: 4.82e-04 | 1156.1ms/step | 17.3 samples/s +Step 6070/44000 | Loss: 3.2225 | LR: 4.82e-04 | 1115.6ms/step | 17.9 samples/s +Step 6080/44000 | Loss: 3.2229 | LR: 4.81e-04 | 1047.2ms/step | 19.1 samples/s +Step 6090/44000 | Loss: 3.2433 | LR: 4.81e-04 | 1065.8ms/step | 18.8 samples/s +Step 6100/44000 | Loss: 3.2259 | LR: 4.81e-04 | 1083.3ms/step | 18.5 samples/s +Step 6110/44000 | Loss: 3.2916 | LR: 4.81e-04 | 1057.6ms/step | 18.9 samples/s +Step 6120/44000 | Loss: 3.2874 | LR: 4.81e-04 | 1060.7ms/step | 18.9 samples/s +Step 6130/44000 | Loss: 3.2776 | LR: 4.81e-04 | 1099.0ms/step | 18.2 samples/s +Step 6140/44000 | Loss: 3.3082 | LR: 4.81e-04 | 1047.6ms/step | 19.1 samples/s +Step 6150/44000 | Loss: 3.2377 | LR: 4.81e-04 | 1057.1ms/step | 18.9 samples/s +Step 6160/44000 | Loss: 3.1962 | LR: 4.81e-04 | 1060.9ms/step | 18.9 samples/s +Step 6170/44000 | Loss: 3.2800 | LR: 4.81e-04 | 1059.9ms/step | 18.9 samples/s +Step 6180/44000 | Loss: 3.1620 | LR: 4.81e-04 | 1102.0ms/step | 18.1 samples/s +Step 6190/44000 | Loss: 3.2348 | LR: 4.81e-04 | 1087.7ms/step | 18.4 samples/s +Step 6200/44000 | Loss: 3.2517 | LR: 4.81e-04 | 1062.2ms/step | 18.8 samples/s +Step 6210/44000 | Loss: 3.2375 | LR: 4.81e-04 | 1088.8ms/step | 18.4 samples/s +Step 6220/44000 | Loss: 3.2247 | LR: 4.81e-04 | 1073.4ms/step | 18.6 samples/s +Step 6230/44000 | Loss: 3.2163 | LR: 4.80e-04 | 1083.1ms/step | 18.5 samples/s +Step 6240/44000 | Loss: 3.1584 | LR: 4.80e-04 | 1100.0ms/step | 18.2 samples/s +Step 6250/44000 | Loss: 3.1464 | LR: 4.80e-04 | 1084.0ms/step | 18.5 samples/s +Validation at step 6250: Loss = 3.2406, Perplexity = 25.55 +Step 6260/44000 | Loss: 3.2379 | LR: 4.80e-04 | 1512.6ms/step | 13.2 samples/s +Step 6270/44000 | Loss: 3.2312 | LR: 4.80e-04 | 1083.6ms/step | 18.5 samples/s +Step 6280/44000 | Loss: 3.2631 | LR: 4.80e-04 | 1072.2ms/step | 18.7 samples/s +Step 6290/44000 | Loss: 3.1320 | LR: 4.80e-04 | 1109.7ms/step | 18.0 samples/s +Step 6300/44000 | Loss: 3.2325 | LR: 4.80e-04 | 1104.2ms/step | 18.1 samples/s +Step 6310/44000 | Loss: 3.1601 | LR: 4.80e-04 | 1063.5ms/step | 18.8 samples/s +Step 6320/44000 | Loss: 3.2413 | LR: 4.80e-04 | 1101.4ms/step | 18.2 samples/s +Step 6330/44000 | Loss: 3.2485 | LR: 4.80e-04 | 1076.2ms/step | 18.6 samples/s +Step 6340/44000 | Loss: 3.1501 | LR: 4.80e-04 | 1109.6ms/step | 18.0 samples/s +Step 6350/44000 | Loss: 3.2260 | LR: 4.80e-04 | 1102.7ms/step | 18.1 samples/s +Step 6360/44000 | Loss: 3.2437 | LR: 4.80e-04 | 1060.4ms/step | 18.9 samples/s +Step 6370/44000 | Loss: 3.1533 | LR: 4.79e-04 | 1110.3ms/step | 18.0 samples/s +Step 6380/44000 | Loss: 3.2289 | LR: 4.79e-04 | 1073.6ms/step | 18.6 samples/s +Step 6390/44000 | Loss: 3.2569 | LR: 4.79e-04 | 1060.0ms/step | 18.9 samples/s +Step 6400/44000 | Loss: 3.2682 | LR: 4.79e-04 | 1057.1ms/step | 18.9 samples/s +Step 6410/44000 | Loss: 3.2294 | LR: 4.79e-04 | 1108.9ms/step | 18.0 samples/s +Step 6420/44000 | Loss: 3.2677 | LR: 4.79e-04 | 1083.6ms/step | 18.5 samples/s +Step 6430/44000 | Loss: 3.2539 | LR: 4.79e-04 | 1113.2ms/step | 18.0 samples/s +Step 6440/44000 | Loss: 3.2211 | LR: 4.79e-04 | 1058.0ms/step | 18.9 samples/s +Step 6450/44000 | Loss: 3.1982 | LR: 4.79e-04 | 1059.3ms/step | 18.9 samples/s +Step 6460/44000 | Loss: 3.1961 | LR: 4.79e-04 | 1096.8ms/step | 18.2 samples/s +Step 6470/44000 | Loss: 3.2632 | LR: 4.79e-04 | 1128.3ms/step | 17.7 samples/s +Step 6480/44000 | Loss: 3.2133 | LR: 4.79e-04 | 1104.5ms/step | 18.1 samples/s +Step 6490/44000 | Loss: 3.2638 | LR: 4.79e-04 | 1090.3ms/step | 18.3 samples/s +Step 6500/44000 | Loss: 3.1655 | LR: 4.79e-04 | 1129.6ms/step | 17.7 samples/s +Validation at step 6500: Loss = 3.2307, Perplexity = 25.30 +Step 6510/44000 | Loss: 3.2386 | LR: 4.78e-04 | 1453.6ms/step | 13.8 samples/s +Step 6520/44000 | Loss: 3.2506 | LR: 4.78e-04 | 1141.5ms/step | 17.5 samples/s +Step 6530/44000 | Loss: 3.2188 | LR: 4.78e-04 | 1047.8ms/step | 19.1 samples/s +Step 6540/44000 | Loss: 3.2193 | LR: 4.78e-04 | 1093.8ms/step | 18.3 samples/s +Step 6550/44000 | Loss: 3.2890 | LR: 4.78e-04 | 1078.2ms/step | 18.5 samples/s +Step 6560/44000 | Loss: 3.2207 | LR: 4.78e-04 | 1081.4ms/step | 18.5 samples/s +Step 6570/44000 | Loss: 3.2865 | LR: 4.78e-04 | 1061.3ms/step | 18.8 samples/s +Step 6580/44000 | Loss: 3.2691 | LR: 4.78e-04 | 1055.6ms/step | 18.9 samples/s +Step 6590/44000 | Loss: 3.2039 | LR: 4.78e-04 | 1126.7ms/step | 17.8 samples/s +Step 6600/44000 | Loss: 3.1874 | LR: 4.78e-04 | 1068.5ms/step | 18.7 samples/s +Step 6610/44000 | Loss: 3.2376 | LR: 4.78e-04 | 1094.5ms/step | 18.3 samples/s +Step 6620/44000 | Loss: 3.2206 | LR: 4.78e-04 | 1068.4ms/step | 18.7 samples/s +Step 6630/44000 | Loss: 3.1961 | LR: 4.78e-04 | 1114.6ms/step | 17.9 samples/s +Step 6640/44000 | Loss: 3.2417 | LR: 4.77e-04 | 1089.7ms/step | 18.4 samples/s +Step 6650/44000 | Loss: 3.2563 | LR: 4.77e-04 | 1068.3ms/step | 18.7 samples/s +Step 6660/44000 | Loss: 3.1955 | LR: 4.77e-04 | 1078.5ms/step | 18.5 samples/s +Step 6670/44000 | Loss: 3.2369 | LR: 4.77e-04 | 1099.8ms/step | 18.2 samples/s +Step 6680/44000 | Loss: 3.1876 | LR: 4.77e-04 | 1073.1ms/step | 18.6 samples/s +Step 6690/44000 | Loss: 3.1611 | LR: 4.77e-04 | 1074.0ms/step | 18.6 samples/s +Step 6700/44000 | Loss: 3.2187 | LR: 4.77e-04 | 1076.2ms/step | 18.6 samples/s +Step 6710/44000 | Loss: 3.2510 | LR: 4.77e-04 | 1102.1ms/step | 18.1 samples/s +Step 6720/44000 | Loss: 3.2675 | LR: 4.77e-04 | 1104.6ms/step | 18.1 samples/s +Step 6730/44000 | Loss: 3.2789 | LR: 4.77e-04 | 1079.7ms/step | 18.5 samples/s +Step 6740/44000 | Loss: 3.2354 | LR: 4.77e-04 | 1029.3ms/step | 19.4 samples/s +Step 6750/44000 | Loss: 3.1774 | LR: 4.77e-04 | 1059.5ms/step | 18.9 samples/s +Validation at step 6750: Loss = 3.2216, Perplexity = 25.07 +Step 6760/44000 | Loss: 3.2251 | LR: 4.77e-04 | 1427.2ms/step | 14.0 samples/s +Step 6770/44000 | Loss: 3.2221 | LR: 4.76e-04 | 1065.6ms/step | 18.8 samples/s +Step 6780/44000 | Loss: 3.2217 | LR: 4.76e-04 | 1073.3ms/step | 18.6 samples/s +Step 6790/44000 | Loss: 3.2280 | LR: 4.76e-04 | 1084.4ms/step | 18.4 samples/s +Step 6800/44000 | Loss: 3.2129 | LR: 4.76e-04 | 1029.5ms/step | 19.4 samples/s +Step 6810/44000 | Loss: 3.2594 | LR: 4.76e-04 | 1107.1ms/step | 18.1 samples/s +Step 6820/44000 | Loss: 3.1757 | LR: 4.76e-04 | 1069.0ms/step | 18.7 samples/s +Step 6830/44000 | Loss: 3.1683 | LR: 4.76e-04 | 1042.5ms/step | 19.2 samples/s +Step 6840/44000 | Loss: 3.1847 | LR: 4.76e-04 | 1073.8ms/step | 18.6 samples/s +Step 6850/44000 | Loss: 3.1990 | LR: 4.76e-04 | 1073.5ms/step | 18.6 samples/s +Step 6860/44000 | Loss: 3.1581 | LR: 4.76e-04 | 1078.9ms/step | 18.5 samples/s +Step 6870/44000 | Loss: 3.2284 | LR: 4.76e-04 | 1088.9ms/step | 18.4 samples/s +Step 6880/44000 | Loss: 3.2308 | LR: 4.76e-04 | 1115.9ms/step | 17.9 samples/s +Step 6890/44000 | Loss: 3.1817 | LR: 4.76e-04 | 1088.9ms/step | 18.4 samples/s +Step 6900/44000 | Loss: 3.1794 | LR: 4.75e-04 | 1036.9ms/step | 19.3 samples/s +Step 6910/44000 | Loss: 3.2355 | LR: 4.75e-04 | 1063.6ms/step | 18.8 samples/s +Step 6920/44000 | Loss: 3.1964 | LR: 4.75e-04 | 1063.2ms/step | 18.8 samples/s +Step 6930/44000 | Loss: 3.2226 | LR: 4.75e-04 | 1071.2ms/step | 18.7 samples/s +Step 6940/44000 | Loss: 3.2013 | LR: 4.75e-04 | 1054.7ms/step | 19.0 samples/s +Step 6950/44000 | Loss: 3.2775 | LR: 4.75e-04 | 1068.3ms/step | 18.7 samples/s +Step 6960/44000 | Loss: 3.1867 | LR: 4.75e-04 | 1068.4ms/step | 18.7 samples/s +Step 6970/44000 | Loss: 3.0994 | LR: 4.75e-04 | 1038.2ms/step | 19.3 samples/s +Step 6980/44000 | Loss: 3.1371 | LR: 4.75e-04 | 1096.4ms/step | 18.2 samples/s +Step 6990/44000 | Loss: 3.2317 | LR: 4.75e-04 | 1033.4ms/step | 19.4 samples/s +Step 7000/44000 | Loss: 3.1638 | LR: 4.75e-04 | 1047.3ms/step | 19.1 samples/s +Validation at step 7000: Loss = 3.2111, Perplexity = 24.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-7000 +Step 7010/44000 | Loss: 3.2463 | LR: 4.75e-04 | 1849.7ms/step | 10.8 samples/s +Step 7020/44000 | Loss: 3.2096 | LR: 4.75e-04 | 1070.7ms/step | 18.7 samples/s +Step 7030/44000 | Loss: 3.2486 | LR: 4.74e-04 | 1091.1ms/step | 18.3 samples/s +Step 7040/44000 | Loss: 3.2492 | LR: 4.74e-04 | 1063.5ms/step | 18.8 samples/s +Step 7050/44000 | Loss: 3.1801 | LR: 4.74e-04 | 1057.7ms/step | 18.9 samples/s +Step 7060/44000 | Loss: 3.1118 | LR: 4.74e-04 | 1078.1ms/step | 18.6 samples/s +Step 7070/44000 | Loss: 3.1853 | LR: 4.74e-04 | 1078.1ms/step | 18.6 samples/s +Step 7080/44000 | Loss: 3.1886 | LR: 4.74e-04 | 1105.0ms/step | 18.1 samples/s +Step 7090/44000 | Loss: 3.1184 | LR: 4.74e-04 | 1078.1ms/step | 18.6 samples/s +Step 7100/44000 | Loss: 3.1599 | LR: 4.74e-04 | 1052.9ms/step | 19.0 samples/s +Step 7110/44000 | Loss: 3.1704 | LR: 4.74e-04 | 1078.3ms/step | 18.5 samples/s +Step 7120/44000 | Loss: 3.1998 | LR: 4.74e-04 | 1043.3ms/step | 19.2 samples/s +Step 7130/44000 | Loss: 3.1109 | LR: 4.74e-04 | 1067.0ms/step | 18.7 samples/s +Step 7140/44000 | Loss: 3.1852 | LR: 4.74e-04 | 1047.5ms/step | 19.1 samples/s +Step 7150/44000 | Loss: 3.2006 | LR: 4.73e-04 | 1042.0ms/step | 19.2 samples/s +Step 7160/44000 | Loss: 3.1765 | LR: 4.73e-04 | 1059.8ms/step | 18.9 samples/s +Step 7170/44000 | Loss: 3.1890 | LR: 4.73e-04 | 1052.1ms/step | 19.0 samples/s +Step 7180/44000 | Loss: 3.1667 | LR: 4.73e-04 | 1034.6ms/step | 19.3 samples/s +Step 7190/44000 | Loss: 3.2227 | LR: 4.73e-04 | 1047.1ms/step | 19.1 samples/s +Step 7200/44000 | Loss: 3.2077 | LR: 4.73e-04 | 1036.9ms/step | 19.3 samples/s +Step 7210/44000 | Loss: 3.2096 | LR: 4.73e-04 | 1020.2ms/step | 19.6 samples/s +Step 7220/44000 | Loss: 3.1174 | LR: 4.73e-04 | 1038.9ms/step | 19.3 samples/s +Step 7230/44000 | Loss: 3.1986 | LR: 4.73e-04 | 1031.2ms/step | 19.4 samples/s +Step 7240/44000 | Loss: 3.1363 | LR: 4.73e-04 | 1044.9ms/step | 19.1 samples/s +Step 7250/44000 | Loss: 3.2172 | LR: 4.73e-04 | 1043.1ms/step | 19.2 samples/s +Validation at step 7250: Loss = 3.2056, Perplexity = 24.67 +Step 7260/44000 | Loss: 3.1661 | LR: 4.73e-04 | 1389.5ms/step | 14.4 samples/s +Step 7270/44000 | Loss: 3.0980 | LR: 4.72e-04 | 1077.6ms/step | 18.6 samples/s +Step 7280/44000 | Loss: 3.1819 | LR: 4.72e-04 | 1099.5ms/step | 18.2 samples/s +Step 7290/44000 | Loss: 3.2099 | LR: 4.72e-04 | 1110.2ms/step | 18.0 samples/s +Step 7300/44000 | Loss: 3.1420 | LR: 4.72e-04 | 1063.4ms/step | 18.8 samples/s +Step 7310/44000 | Loss: 3.1681 | LR: 4.72e-04 | 1073.1ms/step | 18.6 samples/s +Step 7320/44000 | Loss: 3.1611 | LR: 4.72e-04 | 1058.4ms/step | 18.9 samples/s +Step 7330/44000 | Loss: 3.1609 | LR: 4.72e-04 | 1104.4ms/step | 18.1 samples/s +Step 7340/44000 | Loss: 3.1296 | LR: 4.72e-04 | 1068.5ms/step | 18.7 samples/s +Step 7350/44000 | Loss: 3.1779 | LR: 4.72e-04 | 1047.4ms/step | 19.1 samples/s +Step 7360/44000 | Loss: 3.2095 | LR: 4.72e-04 | 1094.1ms/step | 18.3 samples/s +Step 7370/44000 | Loss: 3.1882 | LR: 4.72e-04 | 1115.8ms/step | 17.9 samples/s +Step 7380/44000 | Loss: 3.1764 | LR: 4.72e-04 | 1071.1ms/step | 18.7 samples/s +Step 7390/44000 | Loss: 3.1977 | LR: 4.71e-04 | 1076.5ms/step | 18.6 samples/s +Step 7400/44000 | Loss: 3.0830 | LR: 4.71e-04 | 1044.2ms/step | 19.2 samples/s +Step 7410/44000 | Loss: 3.0977 | LR: 4.71e-04 | 1051.0ms/step | 19.0 samples/s +Step 7420/44000 | Loss: 3.1948 | LR: 4.71e-04 | 1042.3ms/step | 19.2 samples/s +Step 7430/44000 | Loss: 3.2275 | LR: 4.71e-04 | 1052.6ms/step | 19.0 samples/s +Step 7440/44000 | Loss: 3.1868 | LR: 4.71e-04 | 1078.9ms/step | 18.5 samples/s +Step 7450/44000 | Loss: 3.1124 | LR: 4.71e-04 | 1053.1ms/step | 19.0 samples/s +Step 7460/44000 | Loss: 3.1685 | LR: 4.71e-04 | 1047.6ms/step | 19.1 samples/s +Step 7470/44000 | Loss: 3.1646 | LR: 4.71e-04 | 1089.2ms/step | 18.4 samples/s +Step 7480/44000 | Loss: 3.1406 | LR: 4.71e-04 | 1033.6ms/step | 19.4 samples/s +Step 7490/44000 | Loss: 3.1176 | LR: 4.71e-04 | 1056.8ms/step | 18.9 samples/s +Step 7500/44000 | Loss: 3.1755 | LR: 4.71e-04 | 1105.0ms/step | 18.1 samples/s +Validation at step 7500: Loss = 3.1973, Perplexity = 24.47 +Step 7510/44000 | Loss: 3.1864 | LR: 4.70e-04 | 1425.7ms/step | 14.0 samples/s +Step 7520/44000 | Loss: 3.1175 | LR: 4.70e-04 | 1060.9ms/step | 18.9 samples/s +Step 7530/44000 | Loss: 3.1920 | LR: 4.70e-04 | 1065.5ms/step | 18.8 samples/s +Step 7540/44000 | Loss: 3.1664 | LR: 4.70e-04 | 1066.7ms/step | 18.7 samples/s +Step 7550/44000 | Loss: 3.2102 | LR: 4.70e-04 | 1073.4ms/step | 18.6 samples/s +Step 7560/44000 | Loss: 3.2073 | LR: 4.70e-04 | 1075.4ms/step | 18.6 samples/s +Step 7570/44000 | Loss: 3.1289 | LR: 4.70e-04 | 1064.5ms/step | 18.8 samples/s +Step 7580/44000 | Loss: 3.2447 | LR: 4.70e-04 | 1059.7ms/step | 18.9 samples/s +Step 7590/44000 | Loss: 3.2793 | LR: 4.70e-04 | 1034.1ms/step | 19.3 samples/s +Step 7600/44000 | Loss: 3.0936 | LR: 4.70e-04 | 1077.0ms/step | 18.6 samples/s +Step 7610/44000 | Loss: 3.1619 | LR: 4.70e-04 | 1094.2ms/step | 18.3 samples/s +Step 7620/44000 | Loss: 3.1478 | LR: 4.70e-04 | 1088.7ms/step | 18.4 samples/s +Step 7630/44000 | Loss: 3.2221 | LR: 4.69e-04 | 1043.4ms/step | 19.2 samples/s +Step 7640/44000 | Loss: 3.2282 | LR: 4.69e-04 | 1068.5ms/step | 18.7 samples/s +Step 7650/44000 | Loss: 3.0640 | LR: 4.69e-04 | 1082.5ms/step | 18.5 samples/s +Step 7660/44000 | Loss: 3.2006 | LR: 4.69e-04 | 1064.8ms/step | 18.8 samples/s +Step 7670/44000 | Loss: 3.2071 | LR: 4.69e-04 | 1073.8ms/step | 18.6 samples/s +Step 7680/44000 | Loss: 3.2445 | LR: 4.69e-04 | 1060.2ms/step | 18.9 samples/s +Step 7690/44000 | Loss: 3.1717 | LR: 4.69e-04 | 1061.5ms/step | 18.8 samples/s +Step 7700/44000 | Loss: 3.2489 | LR: 4.69e-04 | 1033.5ms/step | 19.4 samples/s +Step 7710/44000 | Loss: 3.2907 | LR: 4.69e-04 | 1076.9ms/step | 18.6 samples/s +Step 7720/44000 | Loss: 3.1422 | LR: 4.69e-04 | 1094.9ms/step | 18.3 samples/s +Step 7730/44000 | Loss: 3.2186 | LR: 4.69e-04 | 1052.8ms/step | 19.0 samples/s +Step 7740/44000 | Loss: 3.1710 | LR: 4.68e-04 | 1063.0ms/step | 18.8 samples/s +Step 7750/44000 | Loss: 3.1643 | LR: 4.68e-04 | 1048.2ms/step | 19.1 samples/s +Validation at step 7750: Loss = 3.1907, Perplexity = 24.31 +Step 7760/44000 | Loss: 3.1835 | LR: 4.68e-04 | 1441.5ms/step | 13.9 samples/s +Step 7770/44000 | Loss: 3.1295 | LR: 4.68e-04 | 1059.1ms/step | 18.9 samples/s +Step 7780/44000 | Loss: 3.1966 | LR: 4.68e-04 | 1053.9ms/step | 19.0 samples/s +Step 7790/44000 | Loss: 3.1544 | LR: 4.68e-04 | 1026.9ms/step | 19.5 samples/s +Step 7800/44000 | Loss: 3.1931 | LR: 4.68e-04 | 1082.0ms/step | 18.5 samples/s +Step 7810/44000 | Loss: 3.1355 | LR: 4.68e-04 | 1060.1ms/step | 18.9 samples/s +Step 7820/44000 | Loss: 3.1621 | LR: 4.68e-04 | 1046.8ms/step | 19.1 samples/s +Step 7830/44000 | Loss: 3.1796 | LR: 4.68e-04 | 1074.1ms/step | 18.6 samples/s +Step 7840/44000 | Loss: 3.1491 | LR: 4.68e-04 | 1073.4ms/step | 18.6 samples/s +Step 7850/44000 | Loss: 3.1713 | LR: 4.67e-04 | 1030.1ms/step | 19.4 samples/s +Step 7860/44000 | Loss: 3.2691 | LR: 4.67e-04 | 1054.2ms/step | 19.0 samples/s +Step 7870/44000 | Loss: 3.1754 | LR: 4.67e-04 | 1073.5ms/step | 18.6 samples/s +Step 7880/44000 | Loss: 3.1658 | LR: 4.67e-04 | 1052.9ms/step | 19.0 samples/s +Step 7890/44000 | Loss: 3.2062 | LR: 4.67e-04 | 1037.0ms/step | 19.3 samples/s +Step 7900/44000 | Loss: 3.1153 | LR: 4.67e-04 | 1058.5ms/step | 18.9 samples/s +Step 7910/44000 | Loss: 3.2128 | LR: 4.67e-04 | 1053.0ms/step | 19.0 samples/s +Step 7920/44000 | Loss: 3.1518 | LR: 4.67e-04 | 1073.9ms/step | 18.6 samples/s +Step 7930/44000 | Loss: 3.1922 | LR: 4.67e-04 | 1052.7ms/step | 19.0 samples/s +Step 7940/44000 | Loss: 3.2268 | LR: 4.67e-04 | 1073.3ms/step | 18.6 samples/s +Step 7950/44000 | Loss: 3.0999 | LR: 4.67e-04 | 1068.2ms/step | 18.7 samples/s +Step 7960/44000 | Loss: 3.1170 | LR: 4.66e-04 | 1035.0ms/step | 19.3 samples/s +Step 7970/44000 | Loss: 3.1099 | LR: 4.66e-04 | 1031.5ms/step | 19.4 samples/s +Step 7980/44000 | Loss: 3.0881 | LR: 4.66e-04 | 1066.6ms/step | 18.8 samples/s +Step 7990/44000 | Loss: 3.1631 | LR: 4.66e-04 | 1035.0ms/step | 19.3 samples/s +Step 8000/44000 | Loss: 3.2316 | LR: 4.66e-04 | 1064.5ms/step | 18.8 samples/s +Validation at step 8000: Loss = 3.1823, Perplexity = 24.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-8000 +Step 8010/44000 | Loss: 3.1221 | LR: 4.66e-04 | 1828.6ms/step | 10.9 samples/s +Step 8020/44000 | Loss: 3.1267 | LR: 4.66e-04 | 1099.5ms/step | 18.2 samples/s +Step 8030/44000 | Loss: 3.1221 | LR: 4.66e-04 | 1061.6ms/step | 18.8 samples/s +Step 8040/44000 | Loss: 3.1899 | LR: 4.66e-04 | 1029.8ms/step | 19.4 samples/s +Step 8050/44000 | Loss: 3.1914 | LR: 4.66e-04 | 1036.4ms/step | 19.3 samples/s +Step 8060/44000 | Loss: 3.1766 | LR: 4.66e-04 | 1073.2ms/step | 18.6 samples/s +Step 8070/44000 | Loss: 3.1729 | LR: 4.65e-04 | 1068.6ms/step | 18.7 samples/s +Step 8080/44000 | Loss: 3.1679 | LR: 4.65e-04 | 1067.6ms/step | 18.7 samples/s +Step 8090/44000 | Loss: 3.0918 | LR: 4.65e-04 | 1044.7ms/step | 19.1 samples/s +Step 8100/44000 | Loss: 3.1856 | LR: 4.65e-04 | 1046.3ms/step | 19.1 samples/s +Step 8110/44000 | Loss: 3.1793 | LR: 4.65e-04 | 1084.1ms/step | 18.4 samples/s +Step 8120/44000 | Loss: 3.1573 | LR: 4.65e-04 | 1125.2ms/step | 17.8 samples/s +Step 8130/44000 | Loss: 3.1711 | LR: 4.65e-04 | 1069.2ms/step | 18.7 samples/s +Step 8140/44000 | Loss: 3.1845 | LR: 4.65e-04 | 1079.4ms/step | 18.5 samples/s +Step 8150/44000 | Loss: 3.1005 | LR: 4.65e-04 | 1067.2ms/step | 18.7 samples/s +Step 8160/44000 | Loss: 3.1814 | LR: 4.65e-04 | 1064.4ms/step | 18.8 samples/s +Step 8170/44000 | Loss: 3.1679 | LR: 4.65e-04 | 1099.8ms/step | 18.2 samples/s +Step 8180/44000 | Loss: 3.1805 | LR: 4.64e-04 | 1097.4ms/step | 18.2 samples/s +Step 8190/44000 | Loss: 3.1311 | LR: 4.64e-04 | 1018.2ms/step | 19.6 samples/s +Step 8200/44000 | Loss: 3.1815 | LR: 4.64e-04 | 1052.4ms/step | 19.0 samples/s +Step 8210/44000 | Loss: 3.2301 | LR: 4.64e-04 | 1042.6ms/step | 19.2 samples/s +Step 8220/44000 | Loss: 3.1688 | LR: 4.64e-04 | 1070.5ms/step | 18.7 samples/s +Step 8230/44000 | Loss: 3.1778 | LR: 4.64e-04 | 1087.4ms/step | 18.4 samples/s +Step 8240/44000 | Loss: 3.2258 | LR: 4.64e-04 | 1047.1ms/step | 19.1 samples/s +Step 8250/44000 | Loss: 3.1240 | LR: 4.64e-04 | 1069.6ms/step | 18.7 samples/s +Validation at step 8250: Loss = 3.1737, Perplexity = 23.89 +Step 8260/44000 | Loss: 3.1518 | LR: 4.64e-04 | 1438.0ms/step | 13.9 samples/s +Step 8270/44000 | Loss: 3.0779 | LR: 4.64e-04 | 1083.6ms/step | 18.5 samples/s +Step 8280/44000 | Loss: 3.1834 | LR: 4.64e-04 | 1043.7ms/step | 19.2 samples/s +Step 8290/44000 | Loss: 3.1256 | LR: 4.63e-04 | 1072.8ms/step | 18.6 samples/s +Step 8300/44000 | Loss: 3.2280 | LR: 4.63e-04 | 1073.0ms/step | 18.6 samples/s +Step 8310/44000 | Loss: 3.1869 | LR: 4.63e-04 | 1078.5ms/step | 18.5 samples/s +Step 8320/44000 | Loss: 3.1807 | LR: 4.63e-04 | 1052.8ms/step | 19.0 samples/s +Step 8330/44000 | Loss: 3.1167 | LR: 4.63e-04 | 1063.0ms/step | 18.8 samples/s +Step 8340/44000 | Loss: 3.1409 | LR: 4.63e-04 | 1052.8ms/step | 19.0 samples/s +Step 8350/44000 | Loss: 3.1968 | LR: 4.63e-04 | 1078.8ms/step | 18.5 samples/s +Step 8360/44000 | Loss: 3.2457 | LR: 4.63e-04 | 1012.5ms/step | 19.8 samples/s +Step 8370/44000 | Loss: 3.2183 | LR: 4.63e-04 | 1078.1ms/step | 18.6 samples/s +Step 8380/44000 | Loss: 3.1685 | LR: 4.63e-04 | 1073.2ms/step | 18.6 samples/s +Step 8390/44000 | Loss: 3.1472 | LR: 4.62e-04 | 1084.7ms/step | 18.4 samples/s +Step 8400/44000 | Loss: 3.1207 | LR: 4.62e-04 | 1032.2ms/step | 19.4 samples/s +Step 8410/44000 | Loss: 3.0998 | LR: 4.62e-04 | 1036.9ms/step | 19.3 samples/s +Step 8420/44000 | Loss: 3.1064 | LR: 4.62e-04 | 1037.4ms/step | 19.3 samples/s +Step 8430/44000 | Loss: 3.1583 | LR: 4.62e-04 | 1083.1ms/step | 18.5 samples/s +Step 8440/44000 | Loss: 3.2111 | LR: 4.62e-04 | 1043.3ms/step | 19.2 samples/s +Step 8450/44000 | Loss: 3.0744 | LR: 4.62e-04 | 1089.7ms/step | 18.4 samples/s +Step 8460/44000 | Loss: 3.1471 | LR: 4.62e-04 | 1068.0ms/step | 18.7 samples/s +Step 8470/44000 | Loss: 3.2347 | LR: 4.62e-04 | 1036.9ms/step | 19.3 samples/s +Step 8480/44000 | Loss: 3.1815 | LR: 4.62e-04 | 1084.4ms/step | 18.4 samples/s +Step 8490/44000 | Loss: 3.2496 | LR: 4.62e-04 | 1062.7ms/step | 18.8 samples/s +Step 8500/44000 | Loss: 3.1885 | LR: 4.61e-04 | 1073.8ms/step | 18.6 samples/s +Validation at step 8500: Loss = 3.1664, Perplexity = 23.72 +Step 8510/44000 | Loss: 3.2008 | LR: 4.61e-04 | 1422.7ms/step | 14.1 samples/s +Step 8520/44000 | Loss: 3.1525 | LR: 4.61e-04 | 1052.3ms/step | 19.0 samples/s +Step 8530/44000 | Loss: 3.1526 | LR: 4.61e-04 | 1046.3ms/step | 19.1 samples/s +Step 8540/44000 | Loss: 3.1233 | LR: 4.61e-04 | 1093.5ms/step | 18.3 samples/s +Step 8550/44000 | Loss: 3.2045 | LR: 4.61e-04 | 1071.9ms/step | 18.7 samples/s +Step 8560/44000 | Loss: 3.2401 | LR: 4.61e-04 | 1058.2ms/step | 18.9 samples/s +Step 8570/44000 | Loss: 3.1697 | LR: 4.61e-04 | 1052.2ms/step | 19.0 samples/s +Step 8580/44000 | Loss: 3.1383 | LR: 4.61e-04 | 1078.5ms/step | 18.5 samples/s +Step 8590/44000 | Loss: 3.1647 | LR: 4.61e-04 | 1073.5ms/step | 18.6 samples/s +Step 8600/44000 | Loss: 3.0912 | LR: 4.60e-04 | 1038.3ms/step | 19.3 samples/s +Step 8610/44000 | Loss: 3.1635 | LR: 4.60e-04 | 1057.7ms/step | 18.9 samples/s +Step 8620/44000 | Loss: 3.1656 | LR: 4.60e-04 | 1054.0ms/step | 19.0 samples/s +Step 8630/44000 | Loss: 3.1689 | LR: 4.60e-04 | 1030.0ms/step | 19.4 samples/s +Step 8640/44000 | Loss: 3.1609 | LR: 4.60e-04 | 1074.5ms/step | 18.6 samples/s +Step 8650/44000 | Loss: 3.1334 | LR: 4.60e-04 | 1051.8ms/step | 19.0 samples/s +Step 8660/44000 | Loss: 3.1265 | LR: 4.60e-04 | 1042.3ms/step | 19.2 samples/s +Step 8670/44000 | Loss: 3.1337 | LR: 4.60e-04 | 1032.0ms/step | 19.4 samples/s +Step 8680/44000 | Loss: 3.2014 | LR: 4.60e-04 | 1058.1ms/step | 18.9 samples/s +Step 8690/44000 | Loss: 3.1528 | LR: 4.60e-04 | 1069.2ms/step | 18.7 samples/s +Step 8700/44000 | Loss: 3.1181 | LR: 4.59e-04 | 1053.0ms/step | 19.0 samples/s +Step 8710/44000 | Loss: 3.1459 | LR: 4.59e-04 | 1043.1ms/step | 19.2 samples/s +Step 8720/44000 | Loss: 3.1221 | LR: 4.59e-04 | 1057.7ms/step | 18.9 samples/s +Step 8730/44000 | Loss: 3.1852 | LR: 4.59e-04 | 1062.9ms/step | 18.8 samples/s +Step 8740/44000 | Loss: 3.1026 | LR: 4.59e-04 | 1084.0ms/step | 18.4 samples/s +Step 8750/44000 | Loss: 3.1675 | LR: 4.59e-04 | 1121.0ms/step | 17.8 samples/s +Validation at step 8750: Loss = 3.1598, Perplexity = 23.57 +Step 8760/44000 | Loss: 3.1121 | LR: 4.59e-04 | 1397.5ms/step | 14.3 samples/s +Step 8770/44000 | Loss: 3.1533 | LR: 4.59e-04 | 1094.7ms/step | 18.3 samples/s +Step 8780/44000 | Loss: 3.1296 | LR: 4.59e-04 | 1078.4ms/step | 18.5 samples/s +Step 8790/44000 | Loss: 3.1188 | LR: 4.59e-04 | 1057.5ms/step | 18.9 samples/s +Step 8800/44000 | Loss: 3.1548 | LR: 4.58e-04 | 1071.3ms/step | 18.7 samples/s +Step 8810/44000 | Loss: 3.1996 | LR: 4.58e-04 | 1062.6ms/step | 18.8 samples/s +Step 8820/44000 | Loss: 3.1506 | LR: 4.58e-04 | 1092.1ms/step | 18.3 samples/s +Step 8830/44000 | Loss: 3.1966 | LR: 4.58e-04 | 1079.1ms/step | 18.5 samples/s +Step 8840/44000 | Loss: 3.1723 | LR: 4.58e-04 | 1053.7ms/step | 19.0 samples/s +Step 8850/44000 | Loss: 3.2174 | LR: 4.58e-04 | 1057.7ms/step | 18.9 samples/s +Step 8860/44000 | Loss: 3.1510 | LR: 4.58e-04 | 1073.1ms/step | 18.6 samples/s +Step 8870/44000 | Loss: 3.1960 | LR: 4.58e-04 | 1037.3ms/step | 19.3 samples/s +Step 8880/44000 | Loss: 3.1681 | LR: 4.58e-04 | 1042.3ms/step | 19.2 samples/s +Step 8890/44000 | Loss: 3.1694 | LR: 4.58e-04 | 1037.4ms/step | 19.3 samples/s +Step 8900/44000 | Loss: 3.2912 | LR: 4.57e-04 | 1073.6ms/step | 18.6 samples/s +Step 8910/44000 | Loss: 3.1861 | LR: 4.57e-04 | 995.6ms/step | 20.1 samples/s +Step 8920/44000 | Loss: 3.1923 | LR: 4.57e-04 | 1062.7ms/step | 18.8 samples/s +Step 8930/44000 | Loss: 3.1326 | LR: 4.57e-04 | 1047.8ms/step | 19.1 samples/s +Step 8940/44000 | Loss: 3.2016 | LR: 4.57e-04 | 1063.0ms/step | 18.8 samples/s +Step 8950/44000 | Loss: 3.1665 | LR: 4.57e-04 | 1057.6ms/step | 18.9 samples/s +Step 8960/44000 | Loss: 3.2208 | LR: 4.57e-04 | 1125.6ms/step | 17.8 samples/s +Step 8970/44000 | Loss: 3.0593 | LR: 4.57e-04 | 1078.6ms/step | 18.5 samples/s +Step 8980/44000 | Loss: 3.2033 | LR: 4.57e-04 | 1074.1ms/step | 18.6 samples/s +Step 8990/44000 | Loss: 3.1291 | LR: 4.57e-04 | 1068.1ms/step | 18.7 samples/s +Step 9000/44000 | Loss: 3.2279 | LR: 4.56e-04 | 1073.4ms/step | 18.6 samples/s +Validation at step 9000: Loss = 3.1549, Perplexity = 23.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-9000 +Step 9010/44000 | Loss: 3.1162 | LR: 4.56e-04 | 1876.2ms/step | 10.7 samples/s +Step 9020/44000 | Loss: 3.1124 | LR: 4.56e-04 | 1036.7ms/step | 19.3 samples/s +Step 9030/44000 | Loss: 3.1400 | LR: 4.56e-04 | 1047.2ms/step | 19.1 samples/s +Step 9040/44000 | Loss: 3.1732 | LR: 4.56e-04 | 1026.8ms/step | 19.5 samples/s +Step 9050/44000 | Loss: 3.1635 | LR: 4.56e-04 | 1041.8ms/step | 19.2 samples/s +Step 9060/44000 | Loss: 3.1422 | LR: 4.56e-04 | 1023.5ms/step | 19.5 samples/s +Step 9070/44000 | Loss: 3.1163 | LR: 4.56e-04 | 1045.2ms/step | 19.1 samples/s +Step 9080/44000 | Loss: 3.1559 | LR: 4.56e-04 | 1116.7ms/step | 17.9 samples/s +Step 9090/44000 | Loss: 3.1549 | LR: 4.56e-04 | 1093.3ms/step | 18.3 samples/s +Step 9100/44000 | Loss: 3.1407 | LR: 4.55e-04 | 1112.5ms/step | 18.0 samples/s +Step 9110/44000 | Loss: 3.1852 | LR: 4.55e-04 | 1061.0ms/step | 18.8 samples/s +Step 9120/44000 | Loss: 3.2060 | LR: 4.55e-04 | 1051.7ms/step | 19.0 samples/s +Step 9130/44000 | Loss: 3.1464 | LR: 4.55e-04 | 1089.7ms/step | 18.4 samples/s +Step 9140/44000 | Loss: 3.1538 | LR: 4.55e-04 | 1041.8ms/step | 19.2 samples/s +Step 9150/44000 | Loss: 3.1159 | LR: 4.55e-04 | 1090.8ms/step | 18.3 samples/s +Step 9160/44000 | Loss: 3.1136 | LR: 4.55e-04 | 1093.3ms/step | 18.3 samples/s +Step 9170/44000 | Loss: 3.1072 | LR: 4.55e-04 | 1068.1ms/step | 18.7 samples/s +Step 9180/44000 | Loss: 3.1042 | LR: 4.55e-04 | 1074.9ms/step | 18.6 samples/s +Step 9190/44000 | Loss: 3.1354 | LR: 4.54e-04 | 1088.4ms/step | 18.4 samples/s +Step 9200/44000 | Loss: 3.1791 | LR: 4.54e-04 | 1078.6ms/step | 18.5 samples/s +Step 9210/44000 | Loss: 3.1386 | LR: 4.54e-04 | 1050.8ms/step | 19.0 samples/s +Step 9220/44000 | Loss: 3.1283 | LR: 4.54e-04 | 1016.9ms/step | 19.7 samples/s +Step 9230/44000 | Loss: 3.1711 | LR: 4.54e-04 | 1079.9ms/step | 18.5 samples/s +Step 9240/44000 | Loss: 3.0459 | LR: 4.54e-04 | 1021.5ms/step | 19.6 samples/s +Step 9250/44000 | Loss: 3.1255 | LR: 4.54e-04 | 1045.6ms/step | 19.1 samples/s +Validation at step 9250: Loss = 3.1472, Perplexity = 23.27 +Step 9260/44000 | Loss: 3.1033 | LR: 4.54e-04 | 1404.1ms/step | 14.2 samples/s +Step 9270/44000 | Loss: 3.1337 | LR: 4.54e-04 | 1027.7ms/step | 19.5 samples/s +Step 9280/44000 | Loss: 3.0986 | LR: 4.54e-04 | 1077.8ms/step | 18.6 samples/s +Step 9290/44000 | Loss: 3.1442 | LR: 4.53e-04 | 1011.4ms/step | 19.8 samples/s +Step 9300/44000 | Loss: 3.0791 | LR: 4.53e-04 | 1099.4ms/step | 18.2 samples/s +Step 9310/44000 | Loss: 3.1316 | LR: 4.53e-04 | 1079.2ms/step | 18.5 samples/s +Step 9320/44000 | Loss: 3.1186 | LR: 4.53e-04 | 1084.9ms/step | 18.4 samples/s +Step 9330/44000 | Loss: 3.2164 | LR: 4.53e-04 | 1062.2ms/step | 18.8 samples/s +Step 9340/44000 | Loss: 3.1312 | LR: 4.53e-04 | 1057.8ms/step | 18.9 samples/s +Step 9350/44000 | Loss: 3.1382 | LR: 4.53e-04 | 1016.7ms/step | 19.7 samples/s +Step 9360/44000 | Loss: 3.0837 | LR: 4.53e-04 | 1049.9ms/step | 19.0 samples/s +Step 9370/44000 | Loss: 3.1500 | LR: 4.53e-04 | 1071.2ms/step | 18.7 samples/s +Step 9380/44000 | Loss: 3.1236 | LR: 4.52e-04 | 1075.3ms/step | 18.6 samples/s +Step 9390/44000 | Loss: 3.1271 | LR: 4.52e-04 | 1040.9ms/step | 19.2 samples/s +Step 9400/44000 | Loss: 3.1319 | LR: 4.52e-04 | 1032.1ms/step | 19.4 samples/s +Step 9410/44000 | Loss: 3.1507 | LR: 4.52e-04 | 1089.4ms/step | 18.4 samples/s +Step 9420/44000 | Loss: 3.2233 | LR: 4.52e-04 | 1104.7ms/step | 18.1 samples/s +Step 9430/44000 | Loss: 3.2177 | LR: 4.52e-04 | 1078.9ms/step | 18.5 samples/s +Step 9440/44000 | Loss: 3.0926 | LR: 4.52e-04 | 1089.0ms/step | 18.4 samples/s +Step 9450/44000 | Loss: 3.0759 | LR: 4.52e-04 | 1059.2ms/step | 18.9 samples/s +Step 9460/44000 | Loss: 3.1328 | LR: 4.52e-04 | 1088.1ms/step | 18.4 samples/s +Step 9470/44000 | Loss: 3.1213 | LR: 4.52e-04 | 1074.0ms/step | 18.6 samples/s +Step 9480/44000 | Loss: 3.1645 | LR: 4.51e-04 | 1026.5ms/step | 19.5 samples/s +Step 9490/44000 | Loss: 3.1675 | LR: 4.51e-04 | 1040.0ms/step | 19.2 samples/s +Step 9500/44000 | Loss: 3.1307 | LR: 4.51e-04 | 1065.4ms/step | 18.8 samples/s +Validation at step 9500: Loss = 3.1434, Perplexity = 23.18 +Step 9510/44000 | Loss: 3.1368 | LR: 4.51e-04 | 1428.0ms/step | 14.0 samples/s +Step 9520/44000 | Loss: 3.1471 | LR: 4.51e-04 | 1057.5ms/step | 18.9 samples/s +Step 9530/44000 | Loss: 3.1141 | LR: 4.51e-04 | 1068.3ms/step | 18.7 samples/s +Step 9540/44000 | Loss: 3.1514 | LR: 4.51e-04 | 1048.7ms/step | 19.1 samples/s +Step 9550/44000 | Loss: 3.1426 | LR: 4.51e-04 | 1082.4ms/step | 18.5 samples/s +Step 9560/44000 | Loss: 3.1169 | LR: 4.51e-04 | 1084.0ms/step | 18.5 samples/s +Step 9570/44000 | Loss: 3.1264 | LR: 4.50e-04 | 1029.4ms/step | 19.4 samples/s +Step 9580/44000 | Loss: 3.1007 | LR: 4.50e-04 | 1101.1ms/step | 18.2 samples/s +Step 9590/44000 | Loss: 3.0935 | LR: 4.50e-04 | 1089.8ms/step | 18.4 samples/s +Step 9600/44000 | Loss: 3.1387 | LR: 4.50e-04 | 1052.7ms/step | 19.0 samples/s +Step 9610/44000 | Loss: 3.0709 | LR: 4.50e-04 | 1054.8ms/step | 19.0 samples/s +Step 9620/44000 | Loss: 3.1116 | LR: 4.50e-04 | 1085.0ms/step | 18.4 samples/s +Step 9630/44000 | Loss: 3.0870 | LR: 4.50e-04 | 1075.5ms/step | 18.6 samples/s +Step 9640/44000 | Loss: 3.1534 | LR: 4.50e-04 | 1064.3ms/step | 18.8 samples/s +Step 9650/44000 | Loss: 3.2020 | LR: 4.50e-04 | 1066.5ms/step | 18.8 samples/s +Step 9660/44000 | Loss: 3.1645 | LR: 4.49e-04 | 1063.2ms/step | 18.8 samples/s +Step 9670/44000 | Loss: 3.1331 | LR: 4.49e-04 | 1034.9ms/step | 19.3 samples/s +Step 9680/44000 | Loss: 3.1506 | LR: 4.49e-04 | 1039.5ms/step | 19.2 samples/s +Step 9690/44000 | Loss: 3.1064 | LR: 4.49e-04 | 1078.4ms/step | 18.5 samples/s +Step 9700/44000 | Loss: 3.1860 | LR: 4.49e-04 | 1053.5ms/step | 19.0 samples/s +Step 9710/44000 | Loss: 3.1001 | LR: 4.49e-04 | 1036.9ms/step | 19.3 samples/s +Step 9720/44000 | Loss: 3.0645 | LR: 4.49e-04 | 1057.8ms/step | 18.9 samples/s +Step 9730/44000 | Loss: 3.0613 | LR: 4.49e-04 | 1047.0ms/step | 19.1 samples/s +Step 9740/44000 | Loss: 3.1171 | LR: 4.49e-04 | 1059.4ms/step | 18.9 samples/s +Step 9750/44000 | Loss: 3.1566 | LR: 4.48e-04 | 1044.8ms/step | 19.1 samples/s +Validation at step 9750: Loss = 3.1387, Perplexity = 23.07 +Step 9760/44000 | Loss: 3.1207 | LR: 4.48e-04 | 1429.5ms/step | 14.0 samples/s +Step 9770/44000 | Loss: 3.0742 | LR: 4.48e-04 | 1063.1ms/step | 18.8 samples/s +Step 9780/44000 | Loss: 3.1049 | LR: 4.48e-04 | 1063.7ms/step | 18.8 samples/s +Step 9790/44000 | Loss: 3.1477 | LR: 4.48e-04 | 1078.9ms/step | 18.5 samples/s +Step 9800/44000 | Loss: 3.1515 | LR: 4.48e-04 | 1089.1ms/step | 18.4 samples/s +Step 9810/44000 | Loss: 3.1402 | LR: 4.48e-04 | 1022.0ms/step | 19.6 samples/s +Step 9820/44000 | Loss: 3.1070 | LR: 4.48e-04 | 1016.5ms/step | 19.7 samples/s +Step 9830/44000 | Loss: 3.2102 | LR: 4.48e-04 | 1094.3ms/step | 18.3 samples/s +Step 9840/44000 | Loss: 3.0318 | LR: 4.47e-04 | 1068.2ms/step | 18.7 samples/s +Step 9850/44000 | Loss: 3.0910 | LR: 4.47e-04 | 1110.1ms/step | 18.0 samples/s +Step 9860/44000 | Loss: 3.2071 | LR: 4.47e-04 | 1061.5ms/step | 18.8 samples/s +Step 9870/44000 | Loss: 3.0848 | LR: 4.47e-04 | 1083.4ms/step | 18.5 samples/s +Step 9880/44000 | Loss: 3.1111 | LR: 4.47e-04 | 1081.1ms/step | 18.5 samples/s +Step 9890/44000 | Loss: 3.1439 | LR: 4.47e-04 | 1031.9ms/step | 19.4 samples/s +Step 9900/44000 | Loss: 3.1100 | LR: 4.47e-04 | 1021.2ms/step | 19.6 samples/s +Step 9910/44000 | Loss: 3.1729 | LR: 4.47e-04 | 1079.1ms/step | 18.5 samples/s +Step 9920/44000 | Loss: 3.0925 | LR: 4.47e-04 | 1047.0ms/step | 19.1 samples/s +Step 9930/44000 | Loss: 3.0865 | LR: 4.46e-04 | 1068.7ms/step | 18.7 samples/s +Step 9940/44000 | Loss: 3.1093 | LR: 4.46e-04 | 1093.1ms/step | 18.3 samples/s +Step 9950/44000 | Loss: 3.0663 | LR: 4.46e-04 | 1049.3ms/step | 19.1 samples/s +Step 9960/44000 | Loss: 3.1493 | LR: 4.46e-04 | 1052.9ms/step | 19.0 samples/s +Step 9970/44000 | Loss: 3.1188 | LR: 4.46e-04 | 1069.2ms/step | 18.7 samples/s +Step 9980/44000 | Loss: 3.1081 | LR: 4.46e-04 | 1094.5ms/step | 18.3 samples/s +Step 9990/44000 | Loss: 3.0938 | LR: 4.46e-04 | 1036.5ms/step | 19.3 samples/s +Step 10000/44000 | Loss: 3.1256 | LR: 4.46e-04 | 1116.1ms/step | 17.9 samples/s +Validation at step 10000: Loss = 3.1306, Perplexity = 22.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-10000 +Step 10010/44000 | Loss: 3.1676 | LR: 4.46e-04 | 1860.9ms/step | 10.7 samples/s +Step 10020/44000 | Loss: 3.0742 | LR: 4.45e-04 | 1053.0ms/step | 19.0 samples/s +Step 10030/44000 | Loss: 3.0801 | LR: 4.45e-04 | 1104.8ms/step | 18.1 samples/s +Step 10040/44000 | Loss: 3.1150 | LR: 4.45e-04 | 1056.2ms/step | 18.9 samples/s +Step 10050/44000 | Loss: 3.1338 | LR: 4.45e-04 | 1042.2ms/step | 19.2 samples/s +Step 10060/44000 | Loss: 3.1751 | LR: 4.45e-04 | 1067.7ms/step | 18.7 samples/s +Step 10070/44000 | Loss: 3.0949 | LR: 4.45e-04 | 1050.8ms/step | 19.0 samples/s +Step 10080/44000 | Loss: 3.1096 | LR: 4.45e-04 | 1068.6ms/step | 18.7 samples/s +Step 10090/44000 | Loss: 3.1096 | LR: 4.45e-04 | 1058.1ms/step | 18.9 samples/s +Step 10100/44000 | Loss: 3.0569 | LR: 4.45e-04 | 1073.7ms/step | 18.6 samples/s +Step 10110/44000 | Loss: 3.1331 | LR: 4.44e-04 | 1099.1ms/step | 18.2 samples/s +Step 10120/44000 | Loss: 3.1004 | LR: 4.44e-04 | 1063.7ms/step | 18.8 samples/s +Step 10130/44000 | Loss: 3.1468 | LR: 4.44e-04 | 1084.7ms/step | 18.4 samples/s +Step 10140/44000 | Loss: 3.0955 | LR: 4.44e-04 | 1078.6ms/step | 18.5 samples/s +Step 10150/44000 | Loss: 3.0934 | LR: 4.44e-04 | 995.8ms/step | 20.1 samples/s +Step 10160/44000 | Loss: 3.1706 | LR: 4.44e-04 | 1045.7ms/step | 19.1 samples/s +Step 10170/44000 | Loss: 3.1193 | LR: 4.44e-04 | 1075.1ms/step | 18.6 samples/s +Step 10180/44000 | Loss: 3.0950 | LR: 4.44e-04 | 1068.4ms/step | 18.7 samples/s +Step 10190/44000 | Loss: 3.1195 | LR: 4.43e-04 | 1058.2ms/step | 18.9 samples/s +Step 10200/44000 | Loss: 3.0893 | LR: 4.43e-04 | 1073.9ms/step | 18.6 samples/s +Step 10210/44000 | Loss: 3.0686 | LR: 4.43e-04 | 1074.7ms/step | 18.6 samples/s +Step 10220/44000 | Loss: 3.0558 | LR: 4.43e-04 | 1062.6ms/step | 18.8 samples/s +Step 10230/44000 | Loss: 3.1186 | LR: 4.43e-04 | 1006.3ms/step | 19.9 samples/s +Step 10240/44000 | Loss: 3.1240 | LR: 4.43e-04 | 1048.1ms/step | 19.1 samples/s +Step 10250/44000 | Loss: 3.1192 | LR: 4.43e-04 | 1068.6ms/step | 18.7 samples/s +Validation at step 10250: Loss = 3.1265, Perplexity = 22.79 +Step 10260/44000 | Loss: 3.1911 | LR: 4.43e-04 | 1443.0ms/step | 13.9 samples/s +Step 10270/44000 | Loss: 3.0441 | LR: 4.43e-04 | 1063.3ms/step | 18.8 samples/s +Step 10280/44000 | Loss: 3.1951 | LR: 4.42e-04 | 1084.2ms/step | 18.4 samples/s +Step 10290/44000 | Loss: 3.0797 | LR: 4.42e-04 | 1089.7ms/step | 18.4 samples/s +Step 10300/44000 | Loss: 3.1202 | LR: 4.42e-04 | 1049.3ms/step | 19.1 samples/s +Step 10310/44000 | Loss: 3.0656 | LR: 4.42e-04 | 1073.5ms/step | 18.6 samples/s +Step 10320/44000 | Loss: 3.0906 | LR: 4.42e-04 | 1046.4ms/step | 19.1 samples/s +Step 10330/44000 | Loss: 3.1038 | LR: 4.42e-04 | 1079.1ms/step | 18.5 samples/s +Step 10340/44000 | Loss: 3.0594 | LR: 4.42e-04 | 1063.8ms/step | 18.8 samples/s +Step 10350/44000 | Loss: 3.0421 | LR: 4.42e-04 | 1047.0ms/step | 19.1 samples/s +Step 10360/44000 | Loss: 3.1127 | LR: 4.42e-04 | 1060.8ms/step | 18.9 samples/s +Step 10370/44000 | Loss: 3.1306 | LR: 4.41e-04 | 1052.8ms/step | 19.0 samples/s +Step 10380/44000 | Loss: 3.0825 | LR: 4.41e-04 | 1077.2ms/step | 18.6 samples/s +Step 10390/44000 | Loss: 3.1610 | LR: 4.41e-04 | 1052.6ms/step | 19.0 samples/s +Step 10400/44000 | Loss: 3.1543 | LR: 4.41e-04 | 1038.4ms/step | 19.3 samples/s +Step 10410/44000 | Loss: 3.0573 | LR: 4.41e-04 | 1102.7ms/step | 18.1 samples/s +Step 10420/44000 | Loss: 3.0855 | LR: 4.41e-04 | 1069.7ms/step | 18.7 samples/s +Step 10430/44000 | Loss: 3.1494 | LR: 4.41e-04 | 1087.5ms/step | 18.4 samples/s +Step 10440/44000 | Loss: 3.2015 | LR: 4.41e-04 | 1076.8ms/step | 18.6 samples/s +Step 10450/44000 | Loss: 3.1274 | LR: 4.40e-04 | 1066.8ms/step | 18.7 samples/s +Step 10460/44000 | Loss: 3.1173 | LR: 4.40e-04 | 1060.9ms/step | 18.9 samples/s +Step 10470/44000 | Loss: 3.0599 | LR: 4.40e-04 | 1070.2ms/step | 18.7 samples/s +Step 10480/44000 | Loss: 3.1177 | LR: 4.40e-04 | 1089.9ms/step | 18.3 samples/s +Step 10490/44000 | Loss: 3.0927 | LR: 4.40e-04 | 1052.7ms/step | 19.0 samples/s +Step 10500/44000 | Loss: 3.0425 | LR: 4.40e-04 | 1073.9ms/step | 18.6 samples/s +Validation at step 10500: Loss = 3.1230, Perplexity = 22.71 +Step 10510/44000 | Loss: 3.0982 | LR: 4.40e-04 | 1385.2ms/step | 14.4 samples/s +Step 10520/44000 | Loss: 3.1196 | LR: 4.40e-04 | 1054.5ms/step | 19.0 samples/s +Step 10530/44000 | Loss: 3.0820 | LR: 4.40e-04 | 1047.4ms/step | 19.1 samples/s +Step 10540/44000 | Loss: 3.0778 | LR: 4.39e-04 | 1038.1ms/step | 19.3 samples/s +Step 10550/44000 | Loss: 3.0300 | LR: 4.39e-04 | 1066.3ms/step | 18.8 samples/s +Step 10560/44000 | Loss: 3.0788 | LR: 4.39e-04 | 1071.4ms/step | 18.7 samples/s +Step 10570/44000 | Loss: 3.0436 | LR: 4.39e-04 | 1042.3ms/step | 19.2 samples/s +Step 10580/44000 | Loss: 3.0604 | LR: 4.39e-04 | 1059.1ms/step | 18.9 samples/s +Step 10590/44000 | Loss: 3.1896 | LR: 4.39e-04 | 1078.4ms/step | 18.5 samples/s +Step 10600/44000 | Loss: 3.0885 | LR: 4.39e-04 | 1066.3ms/step | 18.8 samples/s +Step 10610/44000 | Loss: 3.1230 | LR: 4.39e-04 | 1115.6ms/step | 17.9 samples/s +Step 10620/44000 | Loss: 3.1577 | LR: 4.38e-04 | 1081.5ms/step | 18.5 samples/s +Step 10630/44000 | Loss: 3.0868 | LR: 4.38e-04 | 1048.7ms/step | 19.1 samples/s +Step 10640/44000 | Loss: 3.1337 | LR: 4.38e-04 | 1036.1ms/step | 19.3 samples/s +Step 10650/44000 | Loss: 3.0465 | LR: 4.38e-04 | 1069.4ms/step | 18.7 samples/s +Step 10660/44000 | Loss: 3.0897 | LR: 4.38e-04 | 1096.1ms/step | 18.2 samples/s +Step 10670/44000 | Loss: 3.0757 | LR: 4.38e-04 | 1066.7ms/step | 18.7 samples/s +Step 10680/44000 | Loss: 3.1418 | LR: 4.38e-04 | 1079.2ms/step | 18.5 samples/s +Step 10690/44000 | Loss: 3.0281 | LR: 4.38e-04 | 1073.3ms/step | 18.6 samples/s +Step 10700/44000 | Loss: 3.1599 | LR: 4.37e-04 | 1052.9ms/step | 19.0 samples/s +Step 10710/44000 | Loss: 3.1067 | LR: 4.37e-04 | 1063.2ms/step | 18.8 samples/s +Step 10720/44000 | Loss: 3.0850 | LR: 4.37e-04 | 1052.6ms/step | 19.0 samples/s +Step 10730/44000 | Loss: 3.1057 | LR: 4.37e-04 | 1058.2ms/step | 18.9 samples/s +Step 10740/44000 | Loss: 3.1434 | LR: 4.37e-04 | 1073.6ms/step | 18.6 samples/s +Step 10750/44000 | Loss: 3.0870 | LR: 4.37e-04 | 1062.4ms/step | 18.8 samples/s +Validation at step 10750: Loss = 3.1198, Perplexity = 22.64 +Step 10760/44000 | Loss: 3.1353 | LR: 4.37e-04 | 1418.5ms/step | 14.1 samples/s +Step 10770/44000 | Loss: 3.1085 | LR: 4.37e-04 | 1067.6ms/step | 18.7 samples/s +Step 10780/44000 | Loss: 3.0512 | LR: 4.37e-04 | 1058.2ms/step | 18.9 samples/s +Step 10790/44000 | Loss: 3.0975 | LR: 4.36e-04 | 1081.3ms/step | 18.5 samples/s +Step 10800/44000 | Loss: 3.1370 | LR: 4.36e-04 | 1019.3ms/step | 19.6 samples/s +Step 10810/44000 | Loss: 3.0996 | LR: 4.36e-04 | 1073.7ms/step | 18.6 samples/s +Step 10820/44000 | Loss: 3.0921 | LR: 4.36e-04 | 1050.9ms/step | 19.0 samples/s +Step 10830/44000 | Loss: 3.0991 | LR: 4.36e-04 | 1106.8ms/step | 18.1 samples/s +Step 10840/44000 | Loss: 3.0586 | LR: 4.36e-04 | 1031.3ms/step | 19.4 samples/s +Step 10850/44000 | Loss: 3.1563 | LR: 4.36e-04 | 1066.2ms/step | 18.8 samples/s +Step 10860/44000 | Loss: 3.1906 | LR: 4.36e-04 | 1049.7ms/step | 19.1 samples/s +Step 10870/44000 | Loss: 3.1043 | LR: 4.35e-04 | 1031.3ms/step | 19.4 samples/s +Step 10880/44000 | Loss: 3.0928 | LR: 4.35e-04 | 1068.5ms/step | 18.7 samples/s +Step 10890/44000 | Loss: 3.1406 | LR: 4.35e-04 | 1068.1ms/step | 18.7 samples/s +Step 10900/44000 | Loss: 3.0981 | LR: 4.35e-04 | 1032.1ms/step | 19.4 samples/s +Step 10910/44000 | Loss: 3.1352 | LR: 4.35e-04 | 1057.7ms/step | 18.9 samples/s +Step 10920/44000 | Loss: 3.1655 | LR: 4.35e-04 | 1036.3ms/step | 19.3 samples/s +Step 10930/44000 | Loss: 3.1176 | LR: 4.35e-04 | 1058.4ms/step | 18.9 samples/s +Step 10940/44000 | Loss: 3.1182 | LR: 4.35e-04 | 1057.6ms/step | 18.9 samples/s +Step 10950/44000 | Loss: 3.1178 | LR: 4.34e-04 | 1042.0ms/step | 19.2 samples/s +Step 10960/44000 | Loss: 3.2090 | LR: 4.34e-04 | 1003.1ms/step | 19.9 samples/s +Step 10970/44000 | Loss: 3.0508 | LR: 4.34e-04 | 1060.7ms/step | 18.9 samples/s +Step 10980/44000 | Loss: 3.0754 | LR: 4.34e-04 | 1063.7ms/step | 18.8 samples/s +Step 10990/44000 | Loss: 3.1061 | LR: 4.34e-04 | 1073.9ms/step | 18.6 samples/s +Step 11000/44000 | Loss: 3.0557 | LR: 4.34e-04 | 1052.6ms/step | 19.0 samples/s +Validation at step 11000: Loss = 3.1128, Perplexity = 22.48 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-11000 +Step 11010/44000 | Loss: 3.1036 | LR: 4.34e-04 | 1809.4ms/step | 11.1 samples/s +Step 11020/44000 | Loss: 3.0636 | LR: 4.34e-04 | 1053.0ms/step | 19.0 samples/s +Step 11030/44000 | Loss: 3.0948 | LR: 4.33e-04 | 1062.9ms/step | 18.8 samples/s +Step 11040/44000 | Loss: 3.1278 | LR: 4.33e-04 | 1089.0ms/step | 18.4 samples/s +Step 11050/44000 | Loss: 3.1380 | LR: 4.33e-04 | 1021.3ms/step | 19.6 samples/s +Step 11060/44000 | Loss: 3.0920 | LR: 4.33e-04 | 1017.1ms/step | 19.7 samples/s +Step 11070/44000 | Loss: 3.0917 | LR: 4.33e-04 | 1026.2ms/step | 19.5 samples/s +Step 11080/44000 | Loss: 3.0929 | LR: 4.33e-04 | 1052.9ms/step | 19.0 samples/s +Step 11090/44000 | Loss: 3.0840 | LR: 4.33e-04 | 1029.9ms/step | 19.4 samples/s +Step 11100/44000 | Loss: 3.1445 | LR: 4.33e-04 | 1062.0ms/step | 18.8 samples/s +Step 11110/44000 | Loss: 3.0826 | LR: 4.32e-04 | 1056.2ms/step | 18.9 samples/s +Step 11120/44000 | Loss: 3.1826 | LR: 4.32e-04 | 1031.7ms/step | 19.4 samples/s +Step 11130/44000 | Loss: 3.1032 | LR: 4.32e-04 | 1060.2ms/step | 18.9 samples/s +Step 11140/44000 | Loss: 3.0951 | LR: 4.32e-04 | 1035.2ms/step | 19.3 samples/s +Step 11150/44000 | Loss: 3.0883 | LR: 4.32e-04 | 1063.3ms/step | 18.8 samples/s +Step 11160/44000 | Loss: 3.1263 | LR: 4.32e-04 | 1021.5ms/step | 19.6 samples/s +Step 11170/44000 | Loss: 3.1278 | LR: 4.32e-04 | 1058.2ms/step | 18.9 samples/s +Step 11180/44000 | Loss: 3.0783 | LR: 4.32e-04 | 1042.2ms/step | 19.2 samples/s +Step 11190/44000 | Loss: 3.0577 | LR: 4.31e-04 | 1073.6ms/step | 18.6 samples/s +Step 11200/44000 | Loss: 3.1156 | LR: 4.31e-04 | 1063.1ms/step | 18.8 samples/s +Step 11210/44000 | Loss: 3.1293 | LR: 4.31e-04 | 1067.7ms/step | 18.7 samples/s +Step 11220/44000 | Loss: 3.1304 | LR: 4.31e-04 | 1058.6ms/step | 18.9 samples/s +Step 11230/44000 | Loss: 3.0681 | LR: 4.31e-04 | 1067.5ms/step | 18.7 samples/s +Step 11240/44000 | Loss: 3.0883 | LR: 4.31e-04 | 1065.5ms/step | 18.8 samples/s +Step 11250/44000 | Loss: 3.1291 | LR: 4.31e-04 | 1098.0ms/step | 18.2 samples/s +Validation at step 11250: Loss = 3.1082, Perplexity = 22.38 +Step 11260/44000 | Loss: 3.1172 | LR: 4.31e-04 | 1371.0ms/step | 14.6 samples/s +Step 11270/44000 | Loss: 3.1097 | LR: 4.30e-04 | 1027.0ms/step | 19.5 samples/s +Step 11280/44000 | Loss: 3.0768 | LR: 4.30e-04 | 1089.3ms/step | 18.4 samples/s +Step 11290/44000 | Loss: 3.1018 | LR: 4.30e-04 | 1131.1ms/step | 17.7 samples/s +Step 11300/44000 | Loss: 3.0593 | LR: 4.30e-04 | 1168.5ms/step | 17.1 samples/s +Step 11310/44000 | Loss: 3.1100 | LR: 4.30e-04 | 1193.0ms/step | 16.8 samples/s +Step 11320/44000 | Loss: 3.1139 | LR: 4.30e-04 | 1157.4ms/step | 17.3 samples/s +Step 11330/44000 | Loss: 3.0919 | LR: 4.30e-04 | 1182.9ms/step | 16.9 samples/s +Step 11340/44000 | Loss: 3.0268 | LR: 4.30e-04 | 1167.4ms/step | 17.1 samples/s +Step 11350/44000 | Loss: 3.0964 | LR: 4.29e-04 | 1107.7ms/step | 18.1 samples/s +Step 11360/44000 | Loss: 3.0754 | LR: 4.29e-04 | 1070.3ms/step | 18.7 samples/s +Step 11370/44000 | Loss: 3.0934 | LR: 4.29e-04 | 1135.7ms/step | 17.6 samples/s +Step 11380/44000 | Loss: 3.0612 | LR: 4.29e-04 | 1026.8ms/step | 19.5 samples/s +Step 11390/44000 | Loss: 3.1203 | LR: 4.29e-04 | 1046.9ms/step | 19.1 samples/s +Step 11400/44000 | Loss: 3.0492 | LR: 4.29e-04 | 1016.6ms/step | 19.7 samples/s +Step 11410/44000 | Loss: 3.1115 | LR: 4.29e-04 | 1017.3ms/step | 19.7 samples/s +Step 11420/44000 | Loss: 3.1031 | LR: 4.29e-04 | 1140.4ms/step | 17.5 samples/s +Step 11430/44000 | Loss: 3.1229 | LR: 4.28e-04 | 1094.3ms/step | 18.3 samples/s +Step 11440/44000 | Loss: 3.1050 | LR: 4.28e-04 | 1089.1ms/step | 18.4 samples/s +Step 11450/44000 | Loss: 3.1148 | LR: 4.28e-04 | 1140.1ms/step | 17.5 samples/s +Step 11460/44000 | Loss: 3.1290 | LR: 4.28e-04 | 1116.4ms/step | 17.9 samples/s +Step 11470/44000 | Loss: 3.1080 | LR: 4.28e-04 | 1089.1ms/step | 18.4 samples/s +Step 11480/44000 | Loss: 3.0489 | LR: 4.28e-04 | 1146.2ms/step | 17.4 samples/s +Step 11490/44000 | Loss: 3.0218 | LR: 4.28e-04 | 1151.7ms/step | 17.4 samples/s +Step 11500/44000 | Loss: 3.1293 | LR: 4.28e-04 | 1078.6ms/step | 18.5 samples/s +Validation at step 11500: Loss = 3.1043, Perplexity = 22.29 +Step 11510/44000 | Loss: 3.0449 | LR: 4.27e-04 | 1334.1ms/step | 15.0 samples/s +Step 11520/44000 | Loss: 3.1001 | LR: 4.27e-04 | 1048.6ms/step | 19.1 samples/s +Step 11530/44000 | Loss: 3.1531 | LR: 4.27e-04 | 1124.1ms/step | 17.8 samples/s +Step 11540/44000 | Loss: 3.1038 | LR: 4.27e-04 | 1121.4ms/step | 17.8 samples/s +Step 11550/44000 | Loss: 3.1045 | LR: 4.27e-04 | 1147.4ms/step | 17.4 samples/s +Step 11560/44000 | Loss: 3.1373 | LR: 4.27e-04 | 1082.2ms/step | 18.5 samples/s +Step 11570/44000 | Loss: 3.1564 | LR: 4.27e-04 | 1117.9ms/step | 17.9 samples/s +Step 11580/44000 | Loss: 3.1026 | LR: 4.27e-04 | 1145.2ms/step | 17.5 samples/s +Step 11590/44000 | Loss: 3.1157 | LR: 4.26e-04 | 1103.6ms/step | 18.1 samples/s +Step 11600/44000 | Loss: 3.1067 | LR: 4.26e-04 | 1141.4ms/step | 17.5 samples/s +Step 11610/44000 | Loss: 3.0330 | LR: 4.26e-04 | 1083.9ms/step | 18.5 samples/s +Step 11620/44000 | Loss: 3.1136 | LR: 4.26e-04 | 1015.9ms/step | 19.7 samples/s +Step 11630/44000 | Loss: 3.1024 | LR: 4.26e-04 | 1022.0ms/step | 19.6 samples/s +Step 11640/44000 | Loss: 2.9615 | LR: 4.26e-04 | 1031.9ms/step | 19.4 samples/s +Step 11650/44000 | Loss: 3.1329 | LR: 4.26e-04 | 1141.4ms/step | 17.5 samples/s +Step 11660/44000 | Loss: 3.0981 | LR: 4.26e-04 | 1105.2ms/step | 18.1 samples/s +Step 11670/44000 | Loss: 3.0922 | LR: 4.25e-04 | 1084.6ms/step | 18.4 samples/s +Step 11680/44000 | Loss: 3.1162 | LR: 4.25e-04 | 1125.5ms/step | 17.8 samples/s +Step 11690/44000 | Loss: 3.1292 | LR: 4.25e-04 | 1129.8ms/step | 17.7 samples/s +Step 11700/44000 | Loss: 3.1412 | LR: 4.25e-04 | 1122.3ms/step | 17.8 samples/s +Step 11710/44000 | Loss: 3.0241 | LR: 4.25e-04 | 1166.4ms/step | 17.1 samples/s +Step 11720/44000 | Loss: 3.0704 | LR: 4.25e-04 | 1127.0ms/step | 17.7 samples/s +Step 11730/44000 | Loss: 3.0195 | LR: 4.25e-04 | 1115.2ms/step | 17.9 samples/s +Step 11740/44000 | Loss: 3.0543 | LR: 4.24e-04 | 1025.5ms/step | 19.5 samples/s +Step 11750/44000 | Loss: 3.0921 | LR: 4.24e-04 | 1002.4ms/step | 20.0 samples/s +Validation at step 11750: Loss = 3.1007, Perplexity = 22.21 +Step 11760/44000 | Loss: 3.0167 | LR: 4.24e-04 | 1464.8ms/step | 13.7 samples/s +Step 11770/44000 | Loss: 3.0526 | LR: 4.24e-04 | 1152.1ms/step | 17.4 samples/s +Step 11780/44000 | Loss: 3.0392 | LR: 4.24e-04 | 1116.7ms/step | 17.9 samples/s +Step 11790/44000 | Loss: 3.1330 | LR: 4.24e-04 | 1171.7ms/step | 17.1 samples/s +Step 11800/44000 | Loss: 3.0765 | LR: 4.24e-04 | 1151.9ms/step | 17.4 samples/s +Step 11810/44000 | Loss: 3.0858 | LR: 4.24e-04 | 1190.2ms/step | 16.8 samples/s +Step 11820/44000 | Loss: 3.1050 | LR: 4.23e-04 | 1186.6ms/step | 16.9 samples/s +Step 11830/44000 | Loss: 3.1062 | LR: 4.23e-04 | 1182.8ms/step | 16.9 samples/s +Step 11840/44000 | Loss: 3.0985 | LR: 4.23e-04 | 1183.1ms/step | 16.9 samples/s +Step 11850/44000 | Loss: 3.0103 | LR: 4.23e-04 | 1052.9ms/step | 19.0 samples/s +Step 11860/44000 | Loss: 3.2660 | LR: 4.23e-04 | 1031.8ms/step | 19.4 samples/s +Step 11870/44000 | Loss: 3.0996 | LR: 4.23e-04 | 1105.4ms/step | 18.1 samples/s +Step 11880/44000 | Loss: 3.1005 | LR: 4.23e-04 | 1104.3ms/step | 18.1 samples/s +Step 11890/44000 | Loss: 3.0582 | LR: 4.23e-04 | 1153.0ms/step | 17.3 samples/s +Step 11900/44000 | Loss: 3.0768 | LR: 4.22e-04 | 1146.1ms/step | 17.5 samples/s +Step 11910/44000 | Loss: 3.0512 | LR: 4.22e-04 | 1152.5ms/step | 17.4 samples/s +Step 11920/44000 | Loss: 3.1015 | LR: 4.22e-04 | 1119.1ms/step | 17.9 samples/s +Step 11930/44000 | Loss: 3.1225 | LR: 4.22e-04 | 1153.2ms/step | 17.3 samples/s +Step 11940/44000 | Loss: 3.0933 | LR: 4.22e-04 | 1135.6ms/step | 17.6 samples/s +Step 11950/44000 | Loss: 3.0280 | LR: 4.22e-04 | 1135.9ms/step | 17.6 samples/s +Step 11960/44000 | Loss: 3.1051 | LR: 4.22e-04 | 1041.5ms/step | 19.2 samples/s +Step 11970/44000 | Loss: 3.1022 | LR: 4.21e-04 | 1037.1ms/step | 19.3 samples/s +Step 11980/44000 | Loss: 3.1542 | LR: 4.21e-04 | 1068.6ms/step | 18.7 samples/s +Step 11990/44000 | Loss: 3.1099 | LR: 4.21e-04 | 1167.6ms/step | 17.1 samples/s +Step 12000/44000 | Loss: 3.1371 | LR: 4.21e-04 | 1166.8ms/step | 17.1 samples/s +Validation at step 12000: Loss = 3.0949, Perplexity = 22.08 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-12000 +Step 12010/44000 | Loss: 3.0928 | LR: 4.21e-04 | 2037.4ms/step | 9.8 samples/s +Step 12020/44000 | Loss: 3.0903 | LR: 4.21e-04 | 1121.9ms/step | 17.8 samples/s +Step 12030/44000 | Loss: 3.0376 | LR: 4.21e-04 | 1165.7ms/step | 17.2 samples/s +Step 12040/44000 | Loss: 3.1119 | LR: 4.21e-04 | 1193.4ms/step | 16.8 samples/s +Step 12050/44000 | Loss: 3.1390 | LR: 4.20e-04 | 1193.7ms/step | 16.8 samples/s +Step 12060/44000 | Loss: 3.0449 | LR: 4.20e-04 | 1062.5ms/step | 18.8 samples/s +Step 12070/44000 | Loss: 3.1009 | LR: 4.20e-04 | 1027.0ms/step | 19.5 samples/s +Step 12080/44000 | Loss: 3.1208 | LR: 4.20e-04 | 1047.2ms/step | 19.1 samples/s +Step 12090/44000 | Loss: 3.0998 | LR: 4.20e-04 | 985.4ms/step | 20.3 samples/s +Step 12100/44000 | Loss: 3.0820 | LR: 4.20e-04 | 1130.8ms/step | 17.7 samples/s +Step 12110/44000 | Loss: 3.0669 | LR: 4.20e-04 | 1114.5ms/step | 17.9 samples/s +Step 12120/44000 | Loss: 3.0660 | LR: 4.19e-04 | 1079.2ms/step | 18.5 samples/s +Step 12130/44000 | Loss: 3.0649 | LR: 4.19e-04 | 1144.5ms/step | 17.5 samples/s +Step 12140/44000 | Loss: 3.0211 | LR: 4.19e-04 | 1095.9ms/step | 18.3 samples/s +Step 12150/44000 | Loss: 3.1135 | LR: 4.19e-04 | 1125.2ms/step | 17.8 samples/s +Step 12160/44000 | Loss: 3.1020 | LR: 4.19e-04 | 1126.3ms/step | 17.8 samples/s +Step 12170/44000 | Loss: 3.0392 | LR: 4.19e-04 | 1115.1ms/step | 17.9 samples/s +Step 12180/44000 | Loss: 3.0685 | LR: 4.19e-04 | 1101.2ms/step | 18.2 samples/s +Step 12190/44000 | Loss: 3.1662 | LR: 4.19e-04 | 1033.8ms/step | 19.3 samples/s +Step 12200/44000 | Loss: 3.0739 | LR: 4.18e-04 | 996.6ms/step | 20.1 samples/s +Step 12210/44000 | Loss: 3.0761 | LR: 4.18e-04 | 1136.0ms/step | 17.6 samples/s +Step 12220/44000 | Loss: 3.0670 | LR: 4.18e-04 | 1109.4ms/step | 18.0 samples/s +Step 12230/44000 | Loss: 3.0497 | LR: 4.18e-04 | 1125.9ms/step | 17.8 samples/s +Step 12240/44000 | Loss: 3.0759 | LR: 4.18e-04 | 1156.9ms/step | 17.3 samples/s +Step 12250/44000 | Loss: 3.0490 | LR: 4.18e-04 | 1151.4ms/step | 17.4 samples/s +Validation at step 12250: Loss = 3.0913, Perplexity = 22.00 +Step 12260/44000 | Loss: 3.1258 | LR: 4.18e-04 | 1512.5ms/step | 13.2 samples/s +Step 12270/44000 | Loss: 3.1434 | LR: 4.17e-04 | 1119.1ms/step | 17.9 samples/s +Step 12280/44000 | Loss: 3.0956 | LR: 4.17e-04 | 1156.3ms/step | 17.3 samples/s +Step 12290/44000 | Loss: 3.0601 | LR: 4.17e-04 | 1022.0ms/step | 19.6 samples/s +Step 12300/44000 | Loss: 3.1021 | LR: 4.17e-04 | 1026.5ms/step | 19.5 samples/s +Step 12310/44000 | Loss: 3.1503 | LR: 4.17e-04 | 1032.2ms/step | 19.4 samples/s +Step 12320/44000 | Loss: 3.0908 | LR: 4.17e-04 | 1099.2ms/step | 18.2 samples/s +Step 12330/44000 | Loss: 3.0921 | LR: 4.17e-04 | 1104.9ms/step | 18.1 samples/s +Step 12340/44000 | Loss: 3.0048 | LR: 4.17e-04 | 1136.4ms/step | 17.6 samples/s +Step 12350/44000 | Loss: 3.0620 | LR: 4.16e-04 | 1146.8ms/step | 17.4 samples/s +Step 12360/44000 | Loss: 3.0685 | LR: 4.16e-04 | 1152.0ms/step | 17.4 samples/s +Step 12370/44000 | Loss: 3.0771 | LR: 4.16e-04 | 1139.5ms/step | 17.6 samples/s +Step 12380/44000 | Loss: 3.0822 | LR: 4.16e-04 | 1153.6ms/step | 17.3 samples/s +Step 12390/44000 | Loss: 3.0975 | LR: 4.16e-04 | 1118.9ms/step | 17.9 samples/s +Step 12400/44000 | Loss: 3.0514 | LR: 4.16e-04 | 1106.3ms/step | 18.1 samples/s +Step 12410/44000 | Loss: 3.0864 | LR: 4.16e-04 | 1018.8ms/step | 19.6 samples/s +Step 12420/44000 | Loss: 2.9987 | LR: 4.15e-04 | 1029.6ms/step | 19.4 samples/s +Step 12430/44000 | Loss: 3.1523 | LR: 4.15e-04 | 1052.8ms/step | 19.0 samples/s +Step 12440/44000 | Loss: 3.0359 | LR: 4.15e-04 | 1104.6ms/step | 18.1 samples/s +Step 12450/44000 | Loss: 3.1146 | LR: 4.15e-04 | 1131.7ms/step | 17.7 samples/s +Step 12460/44000 | Loss: 3.0170 | LR: 4.15e-04 | 1094.2ms/step | 18.3 samples/s +Step 12470/44000 | Loss: 3.0777 | LR: 4.15e-04 | 1114.8ms/step | 17.9 samples/s +Step 12480/44000 | Loss: 3.0576 | LR: 4.15e-04 | 1093.9ms/step | 18.3 samples/s +Step 12490/44000 | Loss: 3.0453 | LR: 4.14e-04 | 1130.9ms/step | 17.7 samples/s +Step 12500/44000 | Loss: 3.0845 | LR: 4.14e-04 | 1172.7ms/step | 17.1 samples/s +Validation at step 12500: Loss = 3.0867, Perplexity = 21.91 +Step 12510/44000 | Loss: 3.0585 | LR: 4.14e-04 | 1475.2ms/step | 13.6 samples/s +Step 12520/44000 | Loss: 3.0514 | LR: 4.14e-04 | 1005.8ms/step | 19.9 samples/s +Step 12530/44000 | Loss: 3.0743 | LR: 4.14e-04 | 1022.1ms/step | 19.6 samples/s +Step 12540/44000 | Loss: 3.0680 | LR: 4.14e-04 | 1051.9ms/step | 19.0 samples/s +Step 12550/44000 | Loss: 3.0583 | LR: 4.14e-04 | 1094.9ms/step | 18.3 samples/s +Step 12560/44000 | Loss: 3.0577 | LR: 4.14e-04 | 1131.5ms/step | 17.7 samples/s +Step 12570/44000 | Loss: 3.0392 | LR: 4.13e-04 | 1110.7ms/step | 18.0 samples/s +Step 12580/44000 | Loss: 3.0724 | LR: 4.13e-04 | 1141.1ms/step | 17.5 samples/s +Step 12590/44000 | Loss: 3.0338 | LR: 4.13e-04 | 1120.3ms/step | 17.9 samples/s +Step 12600/44000 | Loss: 3.0908 | LR: 4.13e-04 | 1137.3ms/step | 17.6 samples/s +Step 12610/44000 | Loss: 3.0719 | LR: 4.13e-04 | 1130.8ms/step | 17.7 samples/s +Step 12620/44000 | Loss: 3.0963 | LR: 4.13e-04 | 1099.8ms/step | 18.2 samples/s +Step 12630/44000 | Loss: 3.0000 | LR: 4.13e-04 | 1027.2ms/step | 19.5 samples/s +Step 12640/44000 | Loss: 3.0767 | LR: 4.12e-04 | 996.3ms/step | 20.1 samples/s +Step 12650/44000 | Loss: 3.0626 | LR: 4.12e-04 | 1053.3ms/step | 19.0 samples/s +Step 12660/44000 | Loss: 3.0393 | LR: 4.12e-04 | 1021.2ms/step | 19.6 samples/s +Step 12670/44000 | Loss: 3.0561 | LR: 4.12e-04 | 1115.0ms/step | 17.9 samples/s +Step 12680/44000 | Loss: 3.1265 | LR: 4.12e-04 | 1121.4ms/step | 17.8 samples/s +Step 12690/44000 | Loss: 3.0459 | LR: 4.12e-04 | 1157.1ms/step | 17.3 samples/s +Step 12700/44000 | Loss: 3.0751 | LR: 4.12e-04 | 1060.6ms/step | 18.9 samples/s +Step 12710/44000 | Loss: 3.0809 | LR: 4.11e-04 | 1097.8ms/step | 18.2 samples/s +Step 12720/44000 | Loss: 3.0442 | LR: 4.11e-04 | 1173.2ms/step | 17.0 samples/s +Step 12730/44000 | Loss: 2.9999 | LR: 4.11e-04 | 1115.8ms/step | 17.9 samples/s +Step 12740/44000 | Loss: 3.1237 | LR: 4.11e-04 | 1162.5ms/step | 17.2 samples/s +Step 12750/44000 | Loss: 3.0819 | LR: 4.11e-04 | 1177.8ms/step | 17.0 samples/s +Validation at step 12750: Loss = 3.0824, Perplexity = 21.81 +Step 12760/44000 | Loss: 3.0221 | LR: 4.11e-04 | 1377.3ms/step | 14.5 samples/s +Step 12770/44000 | Loss: 3.0395 | LR: 4.11e-04 | 1077.6ms/step | 18.6 samples/s +Step 12780/44000 | Loss: 3.0769 | LR: 4.10e-04 | 1115.2ms/step | 17.9 samples/s +Step 12790/44000 | Loss: 3.0343 | LR: 4.10e-04 | 1149.2ms/step | 17.4 samples/s +Step 12800/44000 | Loss: 3.0681 | LR: 4.10e-04 | 1181.6ms/step | 16.9 samples/s +Step 12810/44000 | Loss: 3.1493 | LR: 4.10e-04 | 1116.4ms/step | 17.9 samples/s +Step 12820/44000 | Loss: 3.1340 | LR: 4.10e-04 | 1187.4ms/step | 16.8 samples/s +Step 12830/44000 | Loss: 3.0449 | LR: 4.10e-04 | 1119.4ms/step | 17.9 samples/s +Step 12840/44000 | Loss: 3.0675 | LR: 4.10e-04 | 1174.2ms/step | 17.0 samples/s +Step 12850/44000 | Loss: 3.1245 | LR: 4.09e-04 | 1183.2ms/step | 16.9 samples/s +Step 12860/44000 | Loss: 3.0818 | LR: 4.09e-04 | 1068.1ms/step | 18.7 samples/s +Step 12870/44000 | Loss: 3.0426 | LR: 4.09e-04 | 1048.4ms/step | 19.1 samples/s +Step 12880/44000 | Loss: 3.1908 | LR: 4.09e-04 | 1099.3ms/step | 18.2 samples/s +Step 12890/44000 | Loss: 3.1026 | LR: 4.09e-04 | 1163.3ms/step | 17.2 samples/s +Step 12900/44000 | Loss: 3.0555 | LR: 4.09e-04 | 1182.7ms/step | 16.9 samples/s +Step 12910/44000 | Loss: 3.0768 | LR: 4.09e-04 | 1147.4ms/step | 17.4 samples/s +Step 12920/44000 | Loss: 2.9885 | LR: 4.09e-04 | 1127.6ms/step | 17.7 samples/s +Step 12930/44000 | Loss: 3.0697 | LR: 4.08e-04 | 1195.7ms/step | 16.7 samples/s +Step 12940/44000 | Loss: 3.0640 | LR: 4.08e-04 | 1207.1ms/step | 16.6 samples/s +Step 12950/44000 | Loss: 3.0492 | LR: 4.08e-04 | 1219.8ms/step | 16.4 samples/s +Step 12960/44000 | Loss: 3.0086 | LR: 4.08e-04 | 1121.5ms/step | 17.8 samples/s +Step 12970/44000 | Loss: 3.0866 | LR: 4.08e-04 | 1074.2ms/step | 18.6 samples/s +Step 12980/44000 | Loss: 3.0408 | LR: 4.08e-04 | 1089.8ms/step | 18.4 samples/s +Step 12990/44000 | Loss: 3.0992 | LR: 4.08e-04 | 1115.3ms/step | 17.9 samples/s +Step 13000/44000 | Loss: 3.0893 | LR: 4.07e-04 | 1179.2ms/step | 17.0 samples/s +Validation at step 13000: Loss = 3.0785, Perplexity = 21.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-13000 +Step 13010/44000 | Loss: 3.0798 | LR: 4.07e-04 | 2067.6ms/step | 9.7 samples/s +Step 13020/44000 | Loss: 3.0564 | LR: 4.07e-04 | 1247.2ms/step | 16.0 samples/s +Step 13030/44000 | Loss: 3.0527 | LR: 4.07e-04 | 1183.2ms/step | 16.9 samples/s +Step 13040/44000 | Loss: 3.0379 | LR: 4.07e-04 | 1266.7ms/step | 15.8 samples/s +Step 13050/44000 | Loss: 3.0065 | LR: 4.07e-04 | 1194.3ms/step | 16.7 samples/s +Step 13060/44000 | Loss: 3.0626 | LR: 4.07e-04 | 1136.6ms/step | 17.6 samples/s +Step 13070/44000 | Loss: 3.0490 | LR: 4.06e-04 | 1058.6ms/step | 18.9 samples/s +Step 13080/44000 | Loss: 3.1515 | LR: 4.06e-04 | 1079.4ms/step | 18.5 samples/s +Step 13090/44000 | Loss: 3.0954 | LR: 4.06e-04 | 1135.8ms/step | 17.6 samples/s +Step 13100/44000 | Loss: 3.0939 | LR: 4.06e-04 | 1136.5ms/step | 17.6 samples/s +Step 13110/44000 | Loss: 3.0823 | LR: 4.06e-04 | 1193.7ms/step | 16.8 samples/s +Step 13120/44000 | Loss: 3.0979 | LR: 4.06e-04 | 1179.5ms/step | 17.0 samples/s +Step 13130/44000 | Loss: 3.1319 | LR: 4.06e-04 | 1192.9ms/step | 16.8 samples/s +Step 13140/44000 | Loss: 3.0865 | LR: 4.05e-04 | 1188.2ms/step | 16.8 samples/s +Step 13150/44000 | Loss: 3.0800 | LR: 4.05e-04 | 1204.1ms/step | 16.6 samples/s +Step 13160/44000 | Loss: 3.1486 | LR: 4.05e-04 | 1209.2ms/step | 16.5 samples/s +Step 13170/44000 | Loss: 3.0146 | LR: 4.05e-04 | 1162.9ms/step | 17.2 samples/s +Step 13180/44000 | Loss: 3.0086 | LR: 4.05e-04 | 1027.2ms/step | 19.5 samples/s +Step 13190/44000 | Loss: 3.0997 | LR: 4.05e-04 | 1058.4ms/step | 18.9 samples/s +Step 13200/44000 | Loss: 3.0781 | LR: 4.05e-04 | 1089.4ms/step | 18.4 samples/s +Step 13210/44000 | Loss: 3.0527 | LR: 4.04e-04 | 1157.7ms/step | 17.3 samples/s +Step 13220/44000 | Loss: 3.1292 | LR: 4.04e-04 | 1172.0ms/step | 17.1 samples/s +Step 13230/44000 | Loss: 3.0714 | LR: 4.04e-04 | 1193.5ms/step | 16.8 samples/s +Step 13240/44000 | Loss: 3.0518 | LR: 4.04e-04 | 1251.1ms/step | 16.0 samples/s +Step 13250/44000 | Loss: 3.0593 | LR: 4.04e-04 | 1150.5ms/step | 17.4 samples/s +Validation at step 13250: Loss = 3.0743, Perplexity = 21.63 +Step 13260/44000 | Loss: 3.1038 | LR: 4.04e-04 | 1585.7ms/step | 12.6 samples/s +Step 13270/44000 | Loss: 3.0441 | LR: 4.04e-04 | 1214.1ms/step | 16.5 samples/s +Step 13280/44000 | Loss: 3.1270 | LR: 4.03e-04 | 1170.7ms/step | 17.1 samples/s +Step 13290/44000 | Loss: 3.1048 | LR: 4.03e-04 | 1132.3ms/step | 17.7 samples/s +Step 13300/44000 | Loss: 3.0071 | LR: 4.03e-04 | 1079.4ms/step | 18.5 samples/s +Step 13310/44000 | Loss: 3.0303 | LR: 4.03e-04 | 1077.8ms/step | 18.6 samples/s +Step 13320/44000 | Loss: 3.0386 | LR: 4.03e-04 | 1130.7ms/step | 17.7 samples/s +Step 13330/44000 | Loss: 2.9961 | LR: 4.03e-04 | 1120.3ms/step | 17.9 samples/s +Step 13340/44000 | Loss: 3.0083 | LR: 4.03e-04 | 1167.3ms/step | 17.1 samples/s +Step 13350/44000 | Loss: 3.0455 | LR: 4.02e-04 | 1203.8ms/step | 16.6 samples/s +Step 13360/44000 | Loss: 3.0502 | LR: 4.02e-04 | 1201.8ms/step | 16.6 samples/s +Step 13370/44000 | Loss: 3.0972 | LR: 4.02e-04 | 1242.4ms/step | 16.1 samples/s +Step 13380/44000 | Loss: 3.0273 | LR: 4.02e-04 | 1219.2ms/step | 16.4 samples/s +Step 13390/44000 | Loss: 3.0666 | LR: 4.02e-04 | 1264.3ms/step | 15.8 samples/s +Step 13400/44000 | Loss: 3.0690 | LR: 4.02e-04 | 1201.7ms/step | 16.6 samples/s +Step 13410/44000 | Loss: 3.0383 | LR: 4.02e-04 | 1103.3ms/step | 18.1 samples/s +Step 13420/44000 | Loss: 3.0173 | LR: 4.01e-04 | 1138.2ms/step | 17.6 samples/s +Step 13430/44000 | Loss: 3.1070 | LR: 4.01e-04 | 1130.5ms/step | 17.7 samples/s +Step 13440/44000 | Loss: 3.0464 | LR: 4.01e-04 | 1150.7ms/step | 17.4 samples/s +Step 13450/44000 | Loss: 3.0428 | LR: 4.01e-04 | 1146.5ms/step | 17.4 samples/s +Step 13460/44000 | Loss: 3.0931 | LR: 4.01e-04 | 1206.9ms/step | 16.6 samples/s +Step 13470/44000 | Loss: 3.0272 | LR: 4.01e-04 | 1211.3ms/step | 16.5 samples/s +Step 13480/44000 | Loss: 3.0458 | LR: 4.01e-04 | 1208.7ms/step | 16.5 samples/s +Step 13490/44000 | Loss: 3.0492 | LR: 4.00e-04 | 1248.2ms/step | 16.0 samples/s +Step 13500/44000 | Loss: 3.1362 | LR: 4.00e-04 | 1233.0ms/step | 16.2 samples/s +Validation at step 13500: Loss = 3.0692, Perplexity = 21.53 +Step 13510/44000 | Loss: 3.0879 | LR: 4.00e-04 | 1579.3ms/step | 12.7 samples/s +Step 13520/44000 | Loss: 3.0525 | LR: 4.00e-04 | 1107.2ms/step | 18.1 samples/s +Step 13530/44000 | Loss: 3.0417 | LR: 4.00e-04 | 1097.1ms/step | 18.2 samples/s +Step 13540/44000 | Loss: 3.0853 | LR: 4.00e-04 | 1108.4ms/step | 18.0 samples/s +Step 13550/44000 | Loss: 3.0344 | LR: 4.00e-04 | 1111.2ms/step | 18.0 samples/s +Step 13560/44000 | Loss: 3.0144 | LR: 3.99e-04 | 1126.0ms/step | 17.8 samples/s +Step 13570/44000 | Loss: 2.9813 | LR: 3.99e-04 | 1193.2ms/step | 16.8 samples/s +Step 13580/44000 | Loss: 3.0172 | LR: 3.99e-04 | 1239.7ms/step | 16.1 samples/s +Step 13590/44000 | Loss: 3.0542 | LR: 3.99e-04 | 1208.7ms/step | 16.5 samples/s +Step 13600/44000 | Loss: 3.0491 | LR: 3.99e-04 | 1252.3ms/step | 16.0 samples/s +Step 13610/44000 | Loss: 3.0085 | LR: 3.99e-04 | 1276.2ms/step | 15.7 samples/s +Step 13620/44000 | Loss: 3.0660 | LR: 3.98e-04 | 1260.8ms/step | 15.9 samples/s +Step 13630/44000 | Loss: 3.0633 | LR: 3.98e-04 | 1177.3ms/step | 17.0 samples/s +Step 13640/44000 | Loss: 3.0640 | LR: 3.98e-04 | 1079.6ms/step | 18.5 samples/s +Step 13650/44000 | Loss: 3.0586 | LR: 3.98e-04 | 1130.2ms/step | 17.7 samples/s +Step 13660/44000 | Loss: 3.0572 | LR: 3.98e-04 | 1157.0ms/step | 17.3 samples/s +Step 13670/44000 | Loss: 3.0484 | LR: 3.98e-04 | 1151.6ms/step | 17.4 samples/s +Step 13680/44000 | Loss: 3.0831 | LR: 3.98e-04 | 1140.7ms/step | 17.5 samples/s +Step 13690/44000 | Loss: 3.0871 | LR: 3.97e-04 | 1210.6ms/step | 16.5 samples/s +Step 13700/44000 | Loss: 3.0800 | LR: 3.97e-04 | 1256.6ms/step | 15.9 samples/s +Step 13710/44000 | Loss: 3.0738 | LR: 3.97e-04 | 1261.7ms/step | 15.9 samples/s +Step 13720/44000 | Loss: 3.0553 | LR: 3.97e-04 | 1217.5ms/step | 16.4 samples/s +Step 13730/44000 | Loss: 3.0343 | LR: 3.97e-04 | 1268.3ms/step | 15.8 samples/s +Step 13740/44000 | Loss: 3.0296 | LR: 3.97e-04 | 1271.8ms/step | 15.7 samples/s +Step 13750/44000 | Loss: 3.0370 | LR: 3.97e-04 | 1126.5ms/step | 17.8 samples/s +Validation at step 13750: Loss = 3.0661, Perplexity = 21.46 +Step 13760/44000 | Loss: 3.0350 | LR: 3.96e-04 | 1495.8ms/step | 13.4 samples/s +Step 13770/44000 | Loss: 3.0497 | LR: 3.96e-04 | 1095.1ms/step | 18.3 samples/s +Step 13780/44000 | Loss: 3.0714 | LR: 3.96e-04 | 1157.2ms/step | 17.3 samples/s +Step 13790/44000 | Loss: 3.0840 | LR: 3.96e-04 | 1172.5ms/step | 17.1 samples/s +Step 13800/44000 | Loss: 3.0244 | LR: 3.96e-04 | 1194.2ms/step | 16.7 samples/s +Step 13810/44000 | Loss: 3.0679 | LR: 3.96e-04 | 1303.6ms/step | 15.3 samples/s +Step 13820/44000 | Loss: 3.0924 | LR: 3.96e-04 | 1282.2ms/step | 15.6 samples/s +Step 13830/44000 | Loss: 3.0515 | LR: 3.95e-04 | 1303.6ms/step | 15.3 samples/s +Step 13840/44000 | Loss: 3.0496 | LR: 3.95e-04 | 1329.9ms/step | 15.0 samples/s +Step 13850/44000 | Loss: 3.0571 | LR: 3.95e-04 | 1303.4ms/step | 15.3 samples/s +Step 13860/44000 | Loss: 3.0585 | LR: 3.95e-04 | 1115.9ms/step | 17.9 samples/s +Step 13870/44000 | Loss: 3.0257 | LR: 3.95e-04 | 1006.1ms/step | 19.9 samples/s +Step 13880/44000 | Loss: 3.0329 | LR: 3.95e-04 | 1038.3ms/step | 19.3 samples/s +Step 13890/44000 | Loss: 3.0438 | LR: 3.95e-04 | 1084.5ms/step | 18.4 samples/s +Step 13900/44000 | Loss: 3.0129 | LR: 3.94e-04 | 1168.4ms/step | 17.1 samples/s +Step 13910/44000 | Loss: 2.9867 | LR: 3.94e-04 | 1178.4ms/step | 17.0 samples/s +Step 13920/44000 | Loss: 3.0347 | LR: 3.94e-04 | 1171.7ms/step | 17.1 samples/s +Step 13930/44000 | Loss: 3.0496 | LR: 3.94e-04 | 1183.4ms/step | 16.9 samples/s +Step 13940/44000 | Loss: 3.0963 | LR: 3.94e-04 | 1159.9ms/step | 17.2 samples/s +Step 13950/44000 | Loss: 3.0587 | LR: 3.94e-04 | 1237.9ms/step | 16.2 samples/s +Step 13960/44000 | Loss: 3.1654 | LR: 3.93e-04 | 1205.5ms/step | 16.6 samples/s +Step 13970/44000 | Loss: 3.0732 | LR: 3.93e-04 | 1213.3ms/step | 16.5 samples/s +Step 13980/44000 | Loss: 3.0434 | LR: 3.93e-04 | 1079.9ms/step | 18.5 samples/s +Step 13990/44000 | Loss: 3.0125 | LR: 3.93e-04 | 1063.3ms/step | 18.8 samples/s +Step 14000/44000 | Loss: 3.0477 | LR: 3.93e-04 | 1073.8ms/step | 18.6 samples/s +Validation at step 14000: Loss = 3.0612, Perplexity = 21.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-14000 +Step 14010/44000 | Loss: 3.0410 | LR: 3.93e-04 | 2043.4ms/step | 9.8 samples/s +Step 14020/44000 | Loss: 2.9618 | LR: 3.93e-04 | 1110.8ms/step | 18.0 samples/s +Step 14030/44000 | Loss: 3.0335 | LR: 3.92e-04 | 1167.4ms/step | 17.1 samples/s +Step 14040/44000 | Loss: 3.0660 | LR: 3.92e-04 | 1165.9ms/step | 17.2 samples/s +Step 14050/44000 | Loss: 3.0990 | LR: 3.92e-04 | 1179.5ms/step | 17.0 samples/s +Step 14060/44000 | Loss: 3.0182 | LR: 3.92e-04 | 1183.5ms/step | 16.9 samples/s +Step 14070/44000 | Loss: 3.0419 | LR: 3.92e-04 | 1064.2ms/step | 18.8 samples/s +Step 14080/44000 | Loss: 3.0352 | LR: 3.92e-04 | 1061.9ms/step | 18.8 samples/s +Step 14090/44000 | Loss: 3.0737 | LR: 3.92e-04 | 1130.8ms/step | 17.7 samples/s +Step 14100/44000 | Loss: 3.0425 | LR: 3.91e-04 | 1157.3ms/step | 17.3 samples/s +Step 14110/44000 | Loss: 3.1026 | LR: 3.91e-04 | 1141.1ms/step | 17.5 samples/s +Step 14120/44000 | Loss: 3.0515 | LR: 3.91e-04 | 1133.6ms/step | 17.6 samples/s +Step 14130/44000 | Loss: 3.0591 | LR: 3.91e-04 | 1169.4ms/step | 17.1 samples/s +Step 14140/44000 | Loss: 3.0982 | LR: 3.91e-04 | 1177.8ms/step | 17.0 samples/s +Step 14150/44000 | Loss: 3.0551 | LR: 3.91e-04 | 1143.7ms/step | 17.5 samples/s +Step 14160/44000 | Loss: 3.0659 | LR: 3.90e-04 | 1202.2ms/step | 16.6 samples/s +Step 14170/44000 | Loss: 3.0702 | LR: 3.90e-04 | 1172.1ms/step | 17.1 samples/s +Step 14180/44000 | Loss: 3.0257 | LR: 3.90e-04 | 1088.7ms/step | 18.4 samples/s +Step 14190/44000 | Loss: 3.0502 | LR: 3.90e-04 | 1073.3ms/step | 18.6 samples/s +Step 14200/44000 | Loss: 3.0994 | LR: 3.90e-04 | 1172.8ms/step | 17.1 samples/s +Step 14210/44000 | Loss: 3.0711 | LR: 3.90e-04 | 1121.1ms/step | 17.8 samples/s +Step 14220/44000 | Loss: 3.0102 | LR: 3.90e-04 | 1193.4ms/step | 16.8 samples/s +Step 14230/44000 | Loss: 3.0571 | LR: 3.89e-04 | 1148.2ms/step | 17.4 samples/s +Step 14240/44000 | Loss: 3.0071 | LR: 3.89e-04 | 1151.1ms/step | 17.4 samples/s +Step 14250/44000 | Loss: 3.0160 | LR: 3.89e-04 | 1146.2ms/step | 17.4 samples/s +Validation at step 14250: Loss = 3.0585, Perplexity = 21.30 +Step 14260/44000 | Loss: 2.9839 | LR: 3.89e-04 | 1526.8ms/step | 13.1 samples/s +Step 14270/44000 | Loss: 3.0067 | LR: 3.89e-04 | 1136.5ms/step | 17.6 samples/s +Step 14280/44000 | Loss: 2.9758 | LR: 3.89e-04 | 1005.6ms/step | 19.9 samples/s +Step 14290/44000 | Loss: 3.0548 | LR: 3.89e-04 | 1052.5ms/step | 19.0 samples/s +Step 14300/44000 | Loss: 3.0887 | LR: 3.88e-04 | 1063.6ms/step | 18.8 samples/s +Step 14310/44000 | Loss: 3.0799 | LR: 3.88e-04 | 1094.2ms/step | 18.3 samples/s +Step 14320/44000 | Loss: 3.0347 | LR: 3.88e-04 | 1136.1ms/step | 17.6 samples/s +Step 14330/44000 | Loss: 3.0243 | LR: 3.88e-04 | 1134.1ms/step | 17.6 samples/s +Step 14340/44000 | Loss: 2.9933 | LR: 3.88e-04 | 1177.5ms/step | 17.0 samples/s +Step 14350/44000 | Loss: 3.0217 | LR: 3.88e-04 | 1154.0ms/step | 17.3 samples/s +Step 14360/44000 | Loss: 3.0499 | LR: 3.87e-04 | 1120.9ms/step | 17.8 samples/s +Step 14370/44000 | Loss: 3.0252 | LR: 3.87e-04 | 1245.1ms/step | 16.1 samples/s +Step 14380/44000 | Loss: 3.0091 | LR: 3.87e-04 | 1278.1ms/step | 15.6 samples/s +Step 14390/44000 | Loss: 3.0191 | LR: 3.87e-04 | 1230.3ms/step | 16.3 samples/s +Step 14400/44000 | Loss: 3.0453 | LR: 3.87e-04 | 1152.0ms/step | 17.4 samples/s +Step 14410/44000 | Loss: 3.0003 | LR: 3.87e-04 | 1162.8ms/step | 17.2 samples/s +Step 14420/44000 | Loss: 3.0539 | LR: 3.87e-04 | 1240.4ms/step | 16.1 samples/s +Step 14430/44000 | Loss: 3.0053 | LR: 3.86e-04 | 1230.4ms/step | 16.3 samples/s +Step 14440/44000 | Loss: 3.0451 | LR: 3.86e-04 | 1282.4ms/step | 15.6 samples/s +Step 14450/44000 | Loss: 3.0010 | LR: 3.86e-04 | 1226.9ms/step | 16.3 samples/s +Step 14460/44000 | Loss: 3.0744 | LR: 3.86e-04 | 1237.3ms/step | 16.2 samples/s +Step 14470/44000 | Loss: 3.0086 | LR: 3.86e-04 | 1238.6ms/step | 16.1 samples/s +Step 14480/44000 | Loss: 2.9769 | LR: 3.86e-04 | 1274.3ms/step | 15.7 samples/s +Step 14490/44000 | Loss: 3.0090 | LR: 3.85e-04 | 1242.4ms/step | 16.1 samples/s +Step 14500/44000 | Loss: 3.0561 | LR: 3.85e-04 | 1157.0ms/step | 17.3 samples/s +Validation at step 14500: Loss = 3.0544, Perplexity = 21.21 +Step 14510/44000 | Loss: 2.9920 | LR: 3.85e-04 | 1501.2ms/step | 13.3 samples/s +Step 14520/44000 | Loss: 3.0507 | LR: 3.85e-04 | 1298.0ms/step | 15.4 samples/s +Step 14530/44000 | Loss: 2.9650 | LR: 3.85e-04 | 1214.6ms/step | 16.5 samples/s +Step 14540/44000 | Loss: 3.0917 | LR: 3.85e-04 | 1235.1ms/step | 16.2 samples/s +Step 14550/44000 | Loss: 3.0657 | LR: 3.85e-04 | 1319.6ms/step | 15.2 samples/s +Step 14560/44000 | Loss: 3.0747 | LR: 3.84e-04 | 1251.4ms/step | 16.0 samples/s +Step 14570/44000 | Loss: 2.9728 | LR: 3.84e-04 | 1235.5ms/step | 16.2 samples/s +Step 14580/44000 | Loss: 3.0617 | LR: 3.84e-04 | 1287.5ms/step | 15.5 samples/s +Step 14590/44000 | Loss: 2.9815 | LR: 3.84e-04 | 1266.5ms/step | 15.8 samples/s +Step 14600/44000 | Loss: 2.9932 | LR: 3.84e-04 | 1151.9ms/step | 17.4 samples/s +Step 14610/44000 | Loss: 2.9853 | LR: 3.84e-04 | 1136.8ms/step | 17.6 samples/s +Step 14620/44000 | Loss: 3.0612 | LR: 3.83e-04 | 1183.5ms/step | 16.9 samples/s +Step 14630/44000 | Loss: 3.0468 | LR: 3.83e-04 | 1125.9ms/step | 17.8 samples/s +Step 14640/44000 | Loss: 3.0097 | LR: 3.83e-04 | 1076.6ms/step | 18.6 samples/s +Step 14650/44000 | Loss: 3.0656 | LR: 3.83e-04 | 1088.8ms/step | 18.4 samples/s +Step 14660/44000 | Loss: 2.9439 | LR: 3.83e-04 | 1092.5ms/step | 18.3 samples/s +Step 14670/44000 | Loss: 3.0232 | LR: 3.83e-04 | 1105.3ms/step | 18.1 samples/s +Step 14680/44000 | Loss: 3.0346 | LR: 3.83e-04 | 1079.2ms/step | 18.5 samples/s +Step 14690/44000 | Loss: 3.0400 | LR: 3.82e-04 | 1042.3ms/step | 19.2 samples/s +Step 14700/44000 | Loss: 3.0546 | LR: 3.82e-04 | 1079.8ms/step | 18.5 samples/s +Step 14710/44000 | Loss: 3.0309 | LR: 3.82e-04 | 1136.1ms/step | 17.6 samples/s +Step 14720/44000 | Loss: 3.0256 | LR: 3.82e-04 | 1173.4ms/step | 17.0 samples/s +Step 14730/44000 | Loss: 3.0828 | LR: 3.82e-04 | 1074.5ms/step | 18.6 samples/s +Step 14740/44000 | Loss: 3.0748 | LR: 3.82e-04 | 1131.2ms/step | 17.7 samples/s +Step 14750/44000 | Loss: 3.0470 | LR: 3.81e-04 | 1094.5ms/step | 18.3 samples/s +Validation at step 14750: Loss = 3.0528, Perplexity = 21.17 +Step 14760/44000 | Loss: 3.0273 | LR: 3.81e-04 | 1501.5ms/step | 13.3 samples/s +Step 14770/44000 | Loss: 3.1007 | LR: 3.81e-04 | 1089.4ms/step | 18.4 samples/s +Step 14780/44000 | Loss: 2.9660 | LR: 3.81e-04 | 1059.0ms/step | 18.9 samples/s +Step 14790/44000 | Loss: 3.1252 | LR: 3.81e-04 | 1074.1ms/step | 18.6 samples/s +Step 14800/44000 | Loss: 3.0809 | LR: 3.81e-04 | 1115.9ms/step | 17.9 samples/s +Step 14810/44000 | Loss: 3.0263 | LR: 3.81e-04 | 1121.3ms/step | 17.8 samples/s +Step 14820/44000 | Loss: 3.1007 | LR: 3.80e-04 | 1108.3ms/step | 18.0 samples/s +Step 14830/44000 | Loss: 3.1005 | LR: 3.80e-04 | 1123.4ms/step | 17.8 samples/s +Step 14840/44000 | Loss: 3.0041 | LR: 3.80e-04 | 1115.7ms/step | 17.9 samples/s +Step 14850/44000 | Loss: 3.0023 | LR: 3.80e-04 | 1173.2ms/step | 17.0 samples/s +Step 14860/44000 | Loss: 2.9830 | LR: 3.80e-04 | 1089.3ms/step | 18.4 samples/s +Step 14870/44000 | Loss: 2.9904 | LR: 3.80e-04 | 1094.3ms/step | 18.3 samples/s +Step 14880/44000 | Loss: 2.9776 | LR: 3.79e-04 | 1138.0ms/step | 17.6 samples/s +Step 14890/44000 | Loss: 3.0463 | LR: 3.79e-04 | 1158.0ms/step | 17.3 samples/s +Step 14900/44000 | Loss: 3.0683 | LR: 3.79e-04 | 1121.4ms/step | 17.8 samples/s +Step 14910/44000 | Loss: 3.0352 | LR: 3.79e-04 | 1085.1ms/step | 18.4 samples/s +Step 14920/44000 | Loss: 2.9705 | LR: 3.79e-04 | 1129.4ms/step | 17.7 samples/s +Step 14930/44000 | Loss: 3.0565 | LR: 3.79e-04 | 1143.3ms/step | 17.5 samples/s +Step 14940/44000 | Loss: 3.0137 | LR: 3.79e-04 | 1125.9ms/step | 17.8 samples/s +Step 14950/44000 | Loss: 3.0220 | LR: 3.78e-04 | 1171.4ms/step | 17.1 samples/s +Step 14960/44000 | Loss: 2.9947 | LR: 3.78e-04 | 1136.5ms/step | 17.6 samples/s +Step 14970/44000 | Loss: 3.0235 | LR: 3.78e-04 | 1125.2ms/step | 17.8 samples/s +Step 14980/44000 | Loss: 3.0426 | LR: 3.78e-04 | 1182.5ms/step | 16.9 samples/s +Step 14990/44000 | Loss: 2.9994 | LR: 3.78e-04 | 1167.2ms/step | 17.1 samples/s +Step 15000/44000 | Loss: 3.0276 | LR: 3.78e-04 | 1101.4ms/step | 18.2 samples/s +Validation at step 15000: Loss = 3.0499, Perplexity = 21.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-15000 +Step 15010/44000 | Loss: 3.0885 | LR: 3.77e-04 | 1942.9ms/step | 10.3 samples/s +Step 15020/44000 | Loss: 3.0842 | LR: 3.77e-04 | 1116.5ms/step | 17.9 samples/s +Step 15030/44000 | Loss: 3.0566 | LR: 3.77e-04 | 1126.1ms/step | 17.8 samples/s +Step 15040/44000 | Loss: 3.0284 | LR: 3.77e-04 | 1079.2ms/step | 18.5 samples/s +Step 15050/44000 | Loss: 3.0903 | LR: 3.77e-04 | 1053.3ms/step | 19.0 samples/s +Step 15060/44000 | Loss: 2.9800 | LR: 3.77e-04 | 1079.2ms/step | 18.5 samples/s +Step 15070/44000 | Loss: 3.0683 | LR: 3.77e-04 | 1073.7ms/step | 18.6 samples/s +Step 15080/44000 | Loss: 3.0791 | LR: 3.76e-04 | 1095.0ms/step | 18.3 samples/s +Step 15090/44000 | Loss: 2.9763 | LR: 3.76e-04 | 1136.1ms/step | 17.6 samples/s +Step 15100/44000 | Loss: 3.0589 | LR: 3.76e-04 | 1141.9ms/step | 17.5 samples/s +Step 15110/44000 | Loss: 2.9685 | LR: 3.76e-04 | 1105.7ms/step | 18.1 samples/s +Step 15120/44000 | Loss: 3.0504 | LR: 3.76e-04 | 1130.9ms/step | 17.7 samples/s +Step 15130/44000 | Loss: 3.0261 | LR: 3.76e-04 | 1262.1ms/step | 15.8 samples/s +Step 15140/44000 | Loss: 2.9681 | LR: 3.75e-04 | 1254.4ms/step | 15.9 samples/s +Step 15150/44000 | Loss: 3.0840 | LR: 3.75e-04 | 1237.6ms/step | 16.2 samples/s +Step 15160/44000 | Loss: 3.0981 | LR: 3.75e-04 | 1110.1ms/step | 18.0 samples/s +Step 15170/44000 | Loss: 3.0481 | LR: 3.75e-04 | 1095.4ms/step | 18.3 samples/s +Step 15180/44000 | Loss: 3.0031 | LR: 3.75e-04 | 1126.3ms/step | 17.8 samples/s +Step 15190/44000 | Loss: 2.9974 | LR: 3.75e-04 | 1151.5ms/step | 17.4 samples/s +Step 15200/44000 | Loss: 2.9826 | LR: 3.74e-04 | 1168.3ms/step | 17.1 samples/s +Step 15210/44000 | Loss: 3.0017 | LR: 3.74e-04 | 1136.8ms/step | 17.6 samples/s +Step 15220/44000 | Loss: 3.0863 | LR: 3.74e-04 | 1156.0ms/step | 17.3 samples/s +Step 15230/44000 | Loss: 3.0708 | LR: 3.74e-04 | 1086.8ms/step | 18.4 samples/s +Step 15240/44000 | Loss: 2.9725 | LR: 3.74e-04 | 1168.1ms/step | 17.1 samples/s +Step 15250/44000 | Loss: 2.9789 | LR: 3.74e-04 | 1141.6ms/step | 17.5 samples/s +Validation at step 15250: Loss = 3.0455, Perplexity = 21.02 +Step 15260/44000 | Loss: 3.1035 | LR: 3.74e-04 | 1523.4ms/step | 13.1 samples/s +Step 15270/44000 | Loss: 3.0688 | LR: 3.73e-04 | 1213.3ms/step | 16.5 samples/s +Step 15280/44000 | Loss: 3.0102 | LR: 3.73e-04 | 1153.3ms/step | 17.3 samples/s +Step 15290/44000 | Loss: 3.0157 | LR: 3.73e-04 | 1172.7ms/step | 17.1 samples/s +Step 15300/44000 | Loss: 3.0138 | LR: 3.73e-04 | 1159.3ms/step | 17.3 samples/s +Step 15310/44000 | Loss: 3.0027 | LR: 3.73e-04 | 1167.2ms/step | 17.1 samples/s +Step 15320/44000 | Loss: 3.0491 | LR: 3.73e-04 | 1136.8ms/step | 17.6 samples/s +Step 15330/44000 | Loss: 2.9636 | LR: 3.72e-04 | 1220.3ms/step | 16.4 samples/s +Step 15340/44000 | Loss: 3.0294 | LR: 3.72e-04 | 1231.0ms/step | 16.2 samples/s +Step 15350/44000 | Loss: 3.0387 | LR: 3.72e-04 | 1220.3ms/step | 16.4 samples/s +Step 15360/44000 | Loss: 3.0147 | LR: 3.72e-04 | 1232.8ms/step | 16.2 samples/s +Step 15370/44000 | Loss: 3.0540 | LR: 3.72e-04 | 1191.8ms/step | 16.8 samples/s +Step 15380/44000 | Loss: 3.0147 | LR: 3.72e-04 | 1225.3ms/step | 16.3 samples/s +Step 15390/44000 | Loss: 3.0470 | LR: 3.71e-04 | 1178.2ms/step | 17.0 samples/s +Step 15400/44000 | Loss: 3.0189 | LR: 3.71e-04 | 1214.9ms/step | 16.5 samples/s +Step 15410/44000 | Loss: 3.0744 | LR: 3.71e-04 | 1172.8ms/step | 17.1 samples/s +Step 15420/44000 | Loss: 3.0466 | LR: 3.71e-04 | 1150.7ms/step | 17.4 samples/s +Step 15430/44000 | Loss: 3.1261 | LR: 3.71e-04 | 1170.0ms/step | 17.1 samples/s +Step 15440/44000 | Loss: 3.0069 | LR: 3.71e-04 | 1147.1ms/step | 17.4 samples/s +Step 15450/44000 | Loss: 3.0256 | LR: 3.71e-04 | 1129.8ms/step | 17.7 samples/s +Step 15460/44000 | Loss: 3.1042 | LR: 3.70e-04 | 1153.6ms/step | 17.3 samples/s +Step 15470/44000 | Loss: 3.0501 | LR: 3.70e-04 | 1131.7ms/step | 17.7 samples/s +Step 15480/44000 | Loss: 3.0069 | LR: 3.70e-04 | 1142.6ms/step | 17.5 samples/s +Step 15490/44000 | Loss: 3.0171 | LR: 3.70e-04 | 1130.6ms/step | 17.7 samples/s +Step 15500/44000 | Loss: 3.0622 | LR: 3.70e-04 | 1141.1ms/step | 17.5 samples/s +Validation at step 15500: Loss = 3.0427, Perplexity = 20.96 +Step 15510/44000 | Loss: 3.0368 | LR: 3.70e-04 | 1510.7ms/step | 13.2 samples/s +Step 15520/44000 | Loss: 3.0744 | LR: 3.69e-04 | 1138.2ms/step | 17.6 samples/s +Step 15530/44000 | Loss: 3.0986 | LR: 3.69e-04 | 1157.5ms/step | 17.3 samples/s +Step 15540/44000 | Loss: 3.0634 | LR: 3.69e-04 | 1166.6ms/step | 17.1 samples/s +Step 15550/44000 | Loss: 3.0367 | LR: 3.69e-04 | 1148.1ms/step | 17.4 samples/s +Step 15560/44000 | Loss: 3.0809 | LR: 3.69e-04 | 1136.2ms/step | 17.6 samples/s +Step 15570/44000 | Loss: 3.0140 | LR: 3.69e-04 | 1172.9ms/step | 17.1 samples/s +Step 15580/44000 | Loss: 3.0952 | LR: 3.68e-04 | 1186.0ms/step | 16.9 samples/s +Step 15590/44000 | Loss: 3.0134 | LR: 3.68e-04 | 1146.8ms/step | 17.4 samples/s +Step 15600/44000 | Loss: 3.0404 | LR: 3.68e-04 | 1166.2ms/step | 17.1 samples/s +Step 15610/44000 | Loss: 2.9896 | LR: 3.68e-04 | 1136.3ms/step | 17.6 samples/s +Step 15620/44000 | Loss: 3.0743 | LR: 3.68e-04 | 1110.5ms/step | 18.0 samples/s +Step 15630/44000 | Loss: 3.0226 | LR: 3.68e-04 | 1146.8ms/step | 17.4 samples/s +Step 15640/44000 | Loss: 2.9675 | LR: 3.67e-04 | 1106.7ms/step | 18.1 samples/s +Step 15650/44000 | Loss: 3.1364 | LR: 3.67e-04 | 1145.9ms/step | 17.5 samples/s +Step 15660/44000 | Loss: 2.9753 | LR: 3.67e-04 | 1162.7ms/step | 17.2 samples/s +Step 15670/44000 | Loss: 3.0041 | LR: 3.67e-04 | 1152.3ms/step | 17.4 samples/s +Step 15680/44000 | Loss: 2.9257 | LR: 3.67e-04 | 1155.9ms/step | 17.3 samples/s +Step 15690/44000 | Loss: 2.9932 | LR: 3.67e-04 | 1179.6ms/step | 17.0 samples/s +Step 15700/44000 | Loss: 3.0854 | LR: 3.67e-04 | 1162.7ms/step | 17.2 samples/s +Step 15710/44000 | Loss: 2.9618 | LR: 3.66e-04 | 1231.1ms/step | 16.2 samples/s +Step 15720/44000 | Loss: 3.0350 | LR: 3.66e-04 | 1193.3ms/step | 16.8 samples/s +Step 15730/44000 | Loss: 3.0491 | LR: 3.66e-04 | 1203.2ms/step | 16.6 samples/s +Step 15740/44000 | Loss: 2.9678 | LR: 3.66e-04 | 1257.6ms/step | 15.9 samples/s +Step 15750/44000 | Loss: 3.0173 | LR: 3.66e-04 | 1199.0ms/step | 16.7 samples/s +Validation at step 15750: Loss = 3.0402, Perplexity = 20.91 +Step 15760/44000 | Loss: 3.0135 | LR: 3.66e-04 | 1600.3ms/step | 12.5 samples/s +Step 15770/44000 | Loss: 3.0050 | LR: 3.65e-04 | 1157.3ms/step | 17.3 samples/s +Step 15780/44000 | Loss: 3.0109 | LR: 3.65e-04 | 1199.0ms/step | 16.7 samples/s +Step 15790/44000 | Loss: 3.0162 | LR: 3.65e-04 | 1195.3ms/step | 16.7 samples/s +Step 15800/44000 | Loss: 3.0170 | LR: 3.65e-04 | 1197.8ms/step | 16.7 samples/s +Step 15810/44000 | Loss: 3.0636 | LR: 3.65e-04 | 1204.9ms/step | 16.6 samples/s +Step 15820/44000 | Loss: 3.0292 | LR: 3.65e-04 | 1245.8ms/step | 16.1 samples/s +Step 15830/44000 | Loss: 3.0514 | LR: 3.64e-04 | 1245.4ms/step | 16.1 samples/s +Step 15840/44000 | Loss: 2.9595 | LR: 3.64e-04 | 1204.7ms/step | 16.6 samples/s +Step 15850/44000 | Loss: 3.0156 | LR: 3.64e-04 | 1256.7ms/step | 15.9 samples/s +Step 15860/44000 | Loss: 2.9977 | LR: 3.64e-04 | 1256.2ms/step | 15.9 samples/s +Step 15870/44000 | Loss: 3.0376 | LR: 3.64e-04 | 1228.6ms/step | 16.3 samples/s +Step 15880/44000 | Loss: 3.0447 | LR: 3.64e-04 | 1185.1ms/step | 16.9 samples/s +Step 15890/44000 | Loss: 3.0569 | LR: 3.63e-04 | 1146.2ms/step | 17.4 samples/s +Step 15900/44000 | Loss: 3.0379 | LR: 3.63e-04 | 1141.7ms/step | 17.5 samples/s +Step 15910/44000 | Loss: 2.9623 | LR: 3.63e-04 | 1157.7ms/step | 17.3 samples/s +Step 15920/44000 | Loss: 2.9820 | LR: 3.63e-04 | 1145.6ms/step | 17.5 samples/s +Step 15930/44000 | Loss: 3.0400 | LR: 3.63e-04 | 1188.9ms/step | 16.8 samples/s +Step 15940/44000 | Loss: 3.0663 | LR: 3.63e-04 | 1189.2ms/step | 16.8 samples/s +Step 15950/44000 | Loss: 3.0141 | LR: 3.62e-04 | 1191.5ms/step | 16.8 samples/s +Step 15960/44000 | Loss: 3.0354 | LR: 3.62e-04 | 1152.0ms/step | 17.4 samples/s +Step 15970/44000 | Loss: 2.9657 | LR: 3.62e-04 | 1188.0ms/step | 16.8 samples/s +Step 15980/44000 | Loss: 3.0348 | LR: 3.62e-04 | 1157.3ms/step | 17.3 samples/s +Step 15990/44000 | Loss: 3.0706 | LR: 3.62e-04 | 1167.8ms/step | 17.1 samples/s +Step 16000/44000 | Loss: 2.9802 | LR: 3.62e-04 | 1176.9ms/step | 17.0 samples/s +Validation at step 16000: Loss = 3.0356, Perplexity = 20.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-16000 +Step 16010/44000 | Loss: 3.0014 | LR: 3.62e-04 | 2055.2ms/step | 9.7 samples/s +Step 16020/44000 | Loss: 2.9837 | LR: 3.61e-04 | 1173.4ms/step | 17.0 samples/s +Step 16030/44000 | Loss: 3.0717 | LR: 3.61e-04 | 1162.3ms/step | 17.2 samples/s +Step 16040/44000 | Loss: 2.9970 | LR: 3.61e-04 | 1139.8ms/step | 17.5 samples/s +Step 16050/44000 | Loss: 3.0533 | LR: 3.61e-04 | 1182.2ms/step | 16.9 samples/s +Step 16060/44000 | Loss: 3.0063 | LR: 3.61e-04 | 1150.4ms/step | 17.4 samples/s +Step 16070/44000 | Loss: 3.0434 | LR: 3.61e-04 | 1152.6ms/step | 17.4 samples/s +Step 16080/44000 | Loss: 3.0245 | LR: 3.60e-04 | 1188.5ms/step | 16.8 samples/s +Step 16090/44000 | Loss: 3.0252 | LR: 3.60e-04 | 1219.4ms/step | 16.4 samples/s +Step 16100/44000 | Loss: 3.0647 | LR: 3.60e-04 | 1134.5ms/step | 17.6 samples/s +Step 16110/44000 | Loss: 3.0376 | LR: 3.60e-04 | 1196.5ms/step | 16.7 samples/s +Step 16120/44000 | Loss: 3.0136 | LR: 3.60e-04 | 1188.8ms/step | 16.8 samples/s +Step 16130/44000 | Loss: 2.9574 | LR: 3.60e-04 | 1224.7ms/step | 16.3 samples/s +Step 16140/44000 | Loss: 2.9962 | LR: 3.59e-04 | 1190.3ms/step | 16.8 samples/s +Step 16150/44000 | Loss: 2.9920 | LR: 3.59e-04 | 1168.8ms/step | 17.1 samples/s +Step 16160/44000 | Loss: 2.9786 | LR: 3.59e-04 | 1227.2ms/step | 16.3 samples/s +Step 16170/44000 | Loss: 3.0141 | LR: 3.59e-04 | 1261.6ms/step | 15.9 samples/s +Step 16180/44000 | Loss: 3.0528 | LR: 3.59e-04 | 1214.3ms/step | 16.5 samples/s +Step 16190/44000 | Loss: 3.0181 | LR: 3.59e-04 | 1124.9ms/step | 17.8 samples/s +Step 16200/44000 | Loss: 3.0216 | LR: 3.58e-04 | 1114.6ms/step | 17.9 samples/s +Step 16210/44000 | Loss: 3.0350 | LR: 3.58e-04 | 1075.4ms/step | 18.6 samples/s +Step 16220/44000 | Loss: 2.9997 | LR: 3.58e-04 | 1141.5ms/step | 17.5 samples/s +Step 16230/44000 | Loss: 3.0041 | LR: 3.58e-04 | 1099.3ms/step | 18.2 samples/s +Step 16240/44000 | Loss: 2.9945 | LR: 3.58e-04 | 1120.3ms/step | 17.9 samples/s +Step 16250/44000 | Loss: 2.9860 | LR: 3.58e-04 | 1058.5ms/step | 18.9 samples/s +Validation at step 16250: Loss = 3.0319, Perplexity = 20.74 +Step 16260/44000 | Loss: 3.0205 | LR: 3.57e-04 | 1448.1ms/step | 13.8 samples/s +Step 16270/44000 | Loss: 3.0606 | LR: 3.57e-04 | 1141.6ms/step | 17.5 samples/s +Step 16280/44000 | Loss: 2.9481 | LR: 3.57e-04 | 1141.1ms/step | 17.5 samples/s +Step 16290/44000 | Loss: 2.9683 | LR: 3.57e-04 | 1110.3ms/step | 18.0 samples/s +Step 16300/44000 | Loss: 3.0363 | LR: 3.57e-04 | 1125.3ms/step | 17.8 samples/s +Step 16310/44000 | Loss: 3.0248 | LR: 3.57e-04 | 1146.4ms/step | 17.4 samples/s +Step 16320/44000 | Loss: 3.0110 | LR: 3.56e-04 | 1108.1ms/step | 18.0 samples/s +Step 16330/44000 | Loss: 2.9416 | LR: 3.56e-04 | 1087.5ms/step | 18.4 samples/s +Step 16340/44000 | Loss: 2.9653 | LR: 3.56e-04 | 1121.8ms/step | 17.8 samples/s +Step 16350/44000 | Loss: 3.0748 | LR: 3.56e-04 | 1102.8ms/step | 18.1 samples/s +Step 16360/44000 | Loss: 3.0076 | LR: 3.56e-04 | 1103.5ms/step | 18.1 samples/s +Step 16370/44000 | Loss: 2.9895 | LR: 3.56e-04 | 1136.6ms/step | 17.6 samples/s +Step 16380/44000 | Loss: 3.0353 | LR: 3.55e-04 | 1126.1ms/step | 17.8 samples/s +Step 16390/44000 | Loss: 3.0842 | LR: 3.55e-04 | 1127.4ms/step | 17.7 samples/s +Step 16400/44000 | Loss: 3.0037 | LR: 3.55e-04 | 1159.3ms/step | 17.3 samples/s +Step 16410/44000 | Loss: 2.9695 | LR: 3.55e-04 | 1152.8ms/step | 17.3 samples/s +Step 16420/44000 | Loss: 2.9945 | LR: 3.55e-04 | 1121.3ms/step | 17.8 samples/s +Step 16430/44000 | Loss: 2.9992 | LR: 3.55e-04 | 1151.4ms/step | 17.4 samples/s +Step 16440/44000 | Loss: 3.0102 | LR: 3.54e-04 | 1115.5ms/step | 17.9 samples/s +Step 16450/44000 | Loss: 2.9373 | LR: 3.54e-04 | 1151.6ms/step | 17.4 samples/s +Step 16460/44000 | Loss: 2.9555 | LR: 3.54e-04 | 1125.8ms/step | 17.8 samples/s +Step 16470/44000 | Loss: 2.9849 | LR: 3.54e-04 | 1156.6ms/step | 17.3 samples/s +Step 16480/44000 | Loss: 3.0602 | LR: 3.54e-04 | 1146.2ms/step | 17.4 samples/s +Step 16490/44000 | Loss: 2.9722 | LR: 3.54e-04 | 1110.3ms/step | 18.0 samples/s +Step 16500/44000 | Loss: 2.9374 | LR: 3.53e-04 | 1068.5ms/step | 18.7 samples/s +Validation at step 16500: Loss = 3.0288, Perplexity = 20.67 +Step 16510/44000 | Loss: 3.0072 | LR: 3.53e-04 | 1479.5ms/step | 13.5 samples/s +Step 16520/44000 | Loss: 3.0766 | LR: 3.53e-04 | 1120.9ms/step | 17.8 samples/s +Step 16530/44000 | Loss: 3.0576 | LR: 3.53e-04 | 1130.8ms/step | 17.7 samples/s +Step 16540/44000 | Loss: 2.9710 | LR: 3.53e-04 | 1094.3ms/step | 18.3 samples/s +Step 16550/44000 | Loss: 2.9953 | LR: 3.53e-04 | 1126.0ms/step | 17.8 samples/s +Step 16560/44000 | Loss: 3.0474 | LR: 3.52e-04 | 1096.5ms/step | 18.2 samples/s +Step 16570/44000 | Loss: 2.9412 | LR: 3.52e-04 | 1097.7ms/step | 18.2 samples/s +Step 16580/44000 | Loss: 2.9785 | LR: 3.52e-04 | 1110.2ms/step | 18.0 samples/s +Step 16590/44000 | Loss: 2.9819 | LR: 3.52e-04 | 1052.8ms/step | 19.0 samples/s +Step 16600/44000 | Loss: 2.9645 | LR: 3.52e-04 | 1103.3ms/step | 18.1 samples/s +Step 16610/44000 | Loss: 3.0223 | LR: 3.52e-04 | 1106.7ms/step | 18.1 samples/s +Step 16620/44000 | Loss: 2.9881 | LR: 3.51e-04 | 1126.1ms/step | 17.8 samples/s +Step 16630/44000 | Loss: 2.9960 | LR: 3.51e-04 | 1099.6ms/step | 18.2 samples/s +Step 16640/44000 | Loss: 2.9577 | LR: 3.51e-04 | 1083.7ms/step | 18.5 samples/s +Step 16650/44000 | Loss: 2.9655 | LR: 3.51e-04 | 1111.0ms/step | 18.0 samples/s +Step 16660/44000 | Loss: 3.0414 | LR: 3.51e-04 | 1108.1ms/step | 18.0 samples/s +Step 16670/44000 | Loss: 3.0014 | LR: 3.51e-04 | 1136.5ms/step | 17.6 samples/s +Step 16680/44000 | Loss: 3.0539 | LR: 3.50e-04 | 1115.9ms/step | 17.9 samples/s +Step 16690/44000 | Loss: 3.0600 | LR: 3.50e-04 | 1105.7ms/step | 18.1 samples/s +Step 16700/44000 | Loss: 3.0405 | LR: 3.50e-04 | 1083.6ms/step | 18.5 samples/s +Step 16710/44000 | Loss: 3.0103 | LR: 3.50e-04 | 1131.3ms/step | 17.7 samples/s +Step 16720/44000 | Loss: 3.0180 | LR: 3.50e-04 | 1084.8ms/step | 18.4 samples/s +Step 16730/44000 | Loss: 3.1008 | LR: 3.50e-04 | 1084.5ms/step | 18.4 samples/s +Step 16740/44000 | Loss: 2.9559 | LR: 3.49e-04 | 1149.5ms/step | 17.4 samples/s +Step 16750/44000 | Loss: 2.9803 | LR: 3.49e-04 | 1113.1ms/step | 18.0 samples/s +Validation at step 16750: Loss = 3.0253, Perplexity = 20.60 +Step 16760/44000 | Loss: 2.9791 | LR: 3.49e-04 | 1458.2ms/step | 13.7 samples/s +Step 16770/44000 | Loss: 3.0298 | LR: 3.49e-04 | 1094.9ms/step | 18.3 samples/s +Step 16780/44000 | Loss: 3.0591 | LR: 3.49e-04 | 1079.4ms/step | 18.5 samples/s +Step 16790/44000 | Loss: 2.9998 | LR: 3.49e-04 | 1125.0ms/step | 17.8 samples/s +Step 16800/44000 | Loss: 2.8951 | LR: 3.48e-04 | 1116.2ms/step | 17.9 samples/s +Step 16810/44000 | Loss: 2.9911 | LR: 3.48e-04 | 1125.7ms/step | 17.8 samples/s +Step 16820/44000 | Loss: 3.0078 | LR: 3.48e-04 | 1146.7ms/step | 17.4 samples/s +Step 16830/44000 | Loss: 2.9883 | LR: 3.48e-04 | 1136.7ms/step | 17.6 samples/s +Step 16840/44000 | Loss: 3.0432 | LR: 3.48e-04 | 1100.0ms/step | 18.2 samples/s +Step 16850/44000 | Loss: 2.9739 | LR: 3.48e-04 | 1151.9ms/step | 17.4 samples/s +Step 16860/44000 | Loss: 3.0206 | LR: 3.47e-04 | 1136.0ms/step | 17.6 samples/s +Step 16870/44000 | Loss: 2.9803 | LR: 3.47e-04 | 1142.3ms/step | 17.5 samples/s +Step 16880/44000 | Loss: 3.0073 | LR: 3.47e-04 | 1157.7ms/step | 17.3 samples/s +Step 16890/44000 | Loss: 2.9890 | LR: 3.47e-04 | 1127.0ms/step | 17.7 samples/s +Step 16900/44000 | Loss: 2.9710 | LR: 3.47e-04 | 1166.5ms/step | 17.1 samples/s +Step 16910/44000 | Loss: 2.9323 | LR: 3.47e-04 | 1160.1ms/step | 17.2 samples/s +Step 16920/44000 | Loss: 2.9959 | LR: 3.46e-04 | 1180.4ms/step | 16.9 samples/s +Step 16930/44000 | Loss: 3.0563 | LR: 3.46e-04 | 1178.4ms/step | 17.0 samples/s +Step 16940/44000 | Loss: 2.9581 | LR: 3.46e-04 | 1245.3ms/step | 16.1 samples/s +Step 16950/44000 | Loss: 2.9628 | LR: 3.46e-04 | 1267.2ms/step | 15.8 samples/s +Step 16960/44000 | Loss: 3.0524 | LR: 3.46e-04 | 1235.2ms/step | 16.2 samples/s +Step 16970/44000 | Loss: 2.9641 | LR: 3.46e-04 | 1102.4ms/step | 18.1 samples/s +Step 16980/44000 | Loss: 3.0803 | LR: 3.45e-04 | 1133.2ms/step | 17.6 samples/s +Step 16990/44000 | Loss: 3.0418 | LR: 3.45e-04 | 1186.1ms/step | 16.9 samples/s +Step 17000/44000 | Loss: 2.9728 | LR: 3.45e-04 | 1196.1ms/step | 16.7 samples/s +Validation at step 17000: Loss = 3.0217, Perplexity = 20.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-17000 +Step 17010/44000 | Loss: 3.0470 | LR: 3.45e-04 | 2027.5ms/step | 9.9 samples/s +Step 17020/44000 | Loss: 2.9493 | LR: 3.45e-04 | 1199.5ms/step | 16.7 samples/s +Step 17030/44000 | Loss: 2.9862 | LR: 3.45e-04 | 1182.7ms/step | 16.9 samples/s +Step 17040/44000 | Loss: 3.0679 | LR: 3.44e-04 | 1162.5ms/step | 17.2 samples/s +Step 17050/44000 | Loss: 3.0985 | LR: 3.44e-04 | 1173.1ms/step | 17.0 samples/s +Step 17060/44000 | Loss: 2.9584 | LR: 3.44e-04 | 1214.2ms/step | 16.5 samples/s +Step 17070/44000 | Loss: 2.9756 | LR: 3.44e-04 | 1200.7ms/step | 16.7 samples/s +Step 17080/44000 | Loss: 3.0481 | LR: 3.44e-04 | 1193.6ms/step | 16.8 samples/s +Step 17090/44000 | Loss: 2.9680 | LR: 3.44e-04 | 1188.3ms/step | 16.8 samples/s +Step 17100/44000 | Loss: 3.0010 | LR: 3.43e-04 | 1180.8ms/step | 16.9 samples/s +Step 17110/44000 | Loss: 2.9987 | LR: 3.43e-04 | 1248.1ms/step | 16.0 samples/s +Step 17120/44000 | Loss: 3.0418 | LR: 3.43e-04 | 1204.2ms/step | 16.6 samples/s +Step 17130/44000 | Loss: 3.0114 | LR: 3.43e-04 | 1198.5ms/step | 16.7 samples/s +Step 17140/44000 | Loss: 3.0012 | LR: 3.43e-04 | 1225.0ms/step | 16.3 samples/s +Step 17150/44000 | Loss: 2.9681 | LR: 3.43e-04 | 1209.1ms/step | 16.5 samples/s +Step 17160/44000 | Loss: 3.0125 | LR: 3.42e-04 | 1209.8ms/step | 16.5 samples/s +Step 17170/44000 | Loss: 3.0361 | LR: 3.42e-04 | 1214.7ms/step | 16.5 samples/s +Step 17180/44000 | Loss: 2.9250 | LR: 3.42e-04 | 1198.2ms/step | 16.7 samples/s +Step 17190/44000 | Loss: 2.9934 | LR: 3.42e-04 | 1246.0ms/step | 16.1 samples/s +Step 17200/44000 | Loss: 2.9394 | LR: 3.42e-04 | 1183.0ms/step | 16.9 samples/s +Step 17210/44000 | Loss: 3.0223 | LR: 3.42e-04 | 1203.9ms/step | 16.6 samples/s +Step 17220/44000 | Loss: 3.0546 | LR: 3.41e-04 | 1178.0ms/step | 17.0 samples/s +Step 17230/44000 | Loss: 3.0194 | LR: 3.41e-04 | 1188.6ms/step | 16.8 samples/s +Step 17240/44000 | Loss: 2.9824 | LR: 3.41e-04 | 1224.3ms/step | 16.3 samples/s +Step 17250/44000 | Loss: 3.0245 | LR: 3.41e-04 | 1278.1ms/step | 15.6 samples/s +Validation at step 17250: Loss = 3.0197, Perplexity = 20.49 +Step 17260/44000 | Loss: 3.0511 | LR: 3.41e-04 | 1541.0ms/step | 13.0 samples/s +Step 17270/44000 | Loss: 3.0804 | LR: 3.41e-04 | 1145.8ms/step | 17.5 samples/s +Step 17280/44000 | Loss: 2.9401 | LR: 3.40e-04 | 1164.5ms/step | 17.2 samples/s +Step 17290/44000 | Loss: 3.0246 | LR: 3.40e-04 | 1146.4ms/step | 17.4 samples/s +Step 17300/44000 | Loss: 2.9968 | LR: 3.40e-04 | 1155.2ms/step | 17.3 samples/s +Step 17310/44000 | Loss: 2.9926 | LR: 3.40e-04 | 1129.1ms/step | 17.7 samples/s +Step 17320/44000 | Loss: 3.0150 | LR: 3.40e-04 | 1122.3ms/step | 17.8 samples/s +Step 17330/44000 | Loss: 3.0046 | LR: 3.40e-04 | 1151.5ms/step | 17.4 samples/s +Step 17340/44000 | Loss: 3.0377 | LR: 3.39e-04 | 1110.3ms/step | 18.0 samples/s +Step 17350/44000 | Loss: 2.9822 | LR: 3.39e-04 | 1100.6ms/step | 18.2 samples/s +Step 17360/44000 | Loss: 2.9839 | LR: 3.39e-04 | 1103.9ms/step | 18.1 samples/s +Step 17370/44000 | Loss: 2.9877 | LR: 3.39e-04 | 1078.4ms/step | 18.5 samples/s +Step 17380/44000 | Loss: 3.0625 | LR: 3.39e-04 | 1137.0ms/step | 17.6 samples/s +Step 17390/44000 | Loss: 2.9953 | LR: 3.39e-04 | 1146.8ms/step | 17.4 samples/s +Step 17400/44000 | Loss: 2.9890 | LR: 3.38e-04 | 1146.4ms/step | 17.4 samples/s +Step 17410/44000 | Loss: 3.0257 | LR: 3.38e-04 | 1115.3ms/step | 17.9 samples/s +Step 17420/44000 | Loss: 2.9362 | LR: 3.38e-04 | 1147.4ms/step | 17.4 samples/s +Step 17430/44000 | Loss: 2.9242 | LR: 3.38e-04 | 1177.0ms/step | 17.0 samples/s +Step 17440/44000 | Loss: 3.0592 | LR: 3.38e-04 | 1189.3ms/step | 16.8 samples/s +Step 17450/44000 | Loss: 3.0919 | LR: 3.38e-04 | 1119.1ms/step | 17.9 samples/s +Step 17460/44000 | Loss: 3.0357 | LR: 3.37e-04 | 1171.8ms/step | 17.1 samples/s +Step 17470/44000 | Loss: 3.0025 | LR: 3.37e-04 | 1128.2ms/step | 17.7 samples/s +Step 17480/44000 | Loss: 2.9545 | LR: 3.37e-04 | 1157.3ms/step | 17.3 samples/s +Step 17490/44000 | Loss: 3.0160 | LR: 3.37e-04 | 1181.5ms/step | 16.9 samples/s +Step 17500/44000 | Loss: 2.9887 | LR: 3.37e-04 | 1101.3ms/step | 18.2 samples/s +Validation at step 17500: Loss = 3.0151, Perplexity = 20.39 +Step 17510/44000 | Loss: 2.9500 | LR: 3.37e-04 | 1552.7ms/step | 12.9 samples/s +Step 17520/44000 | Loss: 3.0169 | LR: 3.36e-04 | 1162.1ms/step | 17.2 samples/s +Step 17530/44000 | Loss: 2.9414 | LR: 3.36e-04 | 1150.0ms/step | 17.4 samples/s +Step 17540/44000 | Loss: 2.9663 | LR: 3.36e-04 | 1120.7ms/step | 17.8 samples/s +Step 17550/44000 | Loss: 3.0254 | LR: 3.36e-04 | 1138.4ms/step | 17.6 samples/s +Step 17560/44000 | Loss: 3.0520 | LR: 3.36e-04 | 1094.9ms/step | 18.3 samples/s +Step 17570/44000 | Loss: 2.9650 | LR: 3.35e-04 | 1110.8ms/step | 18.0 samples/s +Step 17580/44000 | Loss: 2.9461 | LR: 3.35e-04 | 1105.9ms/step | 18.1 samples/s +Step 17590/44000 | Loss: 2.9855 | LR: 3.35e-04 | 1119.0ms/step | 17.9 samples/s +Step 17600/44000 | Loss: 2.9997 | LR: 3.35e-04 | 1245.3ms/step | 16.1 samples/s +Step 17610/44000 | Loss: 2.9961 | LR: 3.35e-04 | 1152.4ms/step | 17.4 samples/s +Step 17620/44000 | Loss: 3.0993 | LR: 3.35e-04 | 1142.9ms/step | 17.5 samples/s +Step 17630/44000 | Loss: 3.0280 | LR: 3.34e-04 | 1166.6ms/step | 17.1 samples/s +Step 17640/44000 | Loss: 2.9111 | LR: 3.34e-04 | 1177.3ms/step | 17.0 samples/s +Step 17650/44000 | Loss: 3.0304 | LR: 3.34e-04 | 1173.7ms/step | 17.0 samples/s +Step 17660/44000 | Loss: 3.0045 | LR: 3.34e-04 | 1169.1ms/step | 17.1 samples/s +Step 17670/44000 | Loss: 3.0069 | LR: 3.34e-04 | 1223.1ms/step | 16.4 samples/s +Step 17680/44000 | Loss: 3.0003 | LR: 3.34e-04 | 1162.4ms/step | 17.2 samples/s +Step 17690/44000 | Loss: 3.0328 | LR: 3.33e-04 | 1125.7ms/step | 17.8 samples/s +Step 17700/44000 | Loss: 3.0430 | LR: 3.33e-04 | 1131.0ms/step | 17.7 samples/s +Step 17710/44000 | Loss: 2.9814 | LR: 3.33e-04 | 1158.3ms/step | 17.3 samples/s +Step 17720/44000 | Loss: 2.9255 | LR: 3.33e-04 | 1110.0ms/step | 18.0 samples/s +Step 17730/44000 | Loss: 3.0672 | LR: 3.33e-04 | 1131.3ms/step | 17.7 samples/s +Step 17740/44000 | Loss: 3.0062 | LR: 3.33e-04 | 1149.9ms/step | 17.4 samples/s +Step 17750/44000 | Loss: 3.0063 | LR: 3.32e-04 | 1170.2ms/step | 17.1 samples/s +Validation at step 17750: Loss = 3.0141, Perplexity = 20.37 +Step 17760/44000 | Loss: 3.0228 | LR: 3.32e-04 | 1552.0ms/step | 12.9 samples/s +Step 17770/44000 | Loss: 3.1028 | LR: 3.32e-04 | 1155.8ms/step | 17.3 samples/s +Step 17780/44000 | Loss: 3.0224 | LR: 3.32e-04 | 1132.1ms/step | 17.7 samples/s +Step 17790/44000 | Loss: 3.0304 | LR: 3.32e-04 | 1156.8ms/step | 17.3 samples/s +Step 17800/44000 | Loss: 2.9466 | LR: 3.32e-04 | 1172.6ms/step | 17.1 samples/s +Step 17810/44000 | Loss: 2.9829 | LR: 3.31e-04 | 1203.6ms/step | 16.6 samples/s +Step 17820/44000 | Loss: 3.0051 | LR: 3.31e-04 | 1215.1ms/step | 16.5 samples/s +Step 17830/44000 | Loss: 2.9623 | LR: 3.31e-04 | 1172.4ms/step | 17.1 samples/s +Step 17840/44000 | Loss: 3.0111 | LR: 3.31e-04 | 1135.7ms/step | 17.6 samples/s +Step 17850/44000 | Loss: 3.0209 | LR: 3.31e-04 | 1146.8ms/step | 17.4 samples/s +Step 17860/44000 | Loss: 3.0279 | LR: 3.31e-04 | 1167.1ms/step | 17.1 samples/s +Step 17870/44000 | Loss: 3.0312 | LR: 3.30e-04 | 1146.5ms/step | 17.4 samples/s +Step 17880/44000 | Loss: 2.9571 | LR: 3.30e-04 | 1136.3ms/step | 17.6 samples/s +Step 17890/44000 | Loss: 3.0403 | LR: 3.30e-04 | 1141.5ms/step | 17.5 samples/s +Step 17900/44000 | Loss: 2.9847 | LR: 3.30e-04 | 1151.2ms/step | 17.4 samples/s +Step 17910/44000 | Loss: 2.9599 | LR: 3.30e-04 | 1109.3ms/step | 18.0 samples/s +Step 17920/44000 | Loss: 3.0167 | LR: 3.29e-04 | 1121.9ms/step | 17.8 samples/s +Step 17930/44000 | Loss: 2.9808 | LR: 3.29e-04 | 1182.3ms/step | 16.9 samples/s +Step 17940/44000 | Loss: 3.0182 | LR: 3.29e-04 | 1177.8ms/step | 17.0 samples/s +Step 17950/44000 | Loss: 2.9152 | LR: 3.29e-04 | 1162.0ms/step | 17.2 samples/s +Step 17960/44000 | Loss: 3.0341 | LR: 3.29e-04 | 1156.7ms/step | 17.3 samples/s +Step 17970/44000 | Loss: 2.9995 | LR: 3.29e-04 | 1131.7ms/step | 17.7 samples/s +Step 17980/44000 | Loss: 3.0020 | LR: 3.28e-04 | 1165.7ms/step | 17.2 samples/s +Step 17990/44000 | Loss: 2.8959 | LR: 3.28e-04 | 1138.3ms/step | 17.6 samples/s +Step 18000/44000 | Loss: 3.0265 | LR: 3.28e-04 | 1165.5ms/step | 17.2 samples/s +Validation at step 18000: Loss = 3.0112, Perplexity = 20.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-18000 +Step 18010/44000 | Loss: 3.0437 | LR: 3.28e-04 | 2166.3ms/step | 9.2 samples/s +Step 18020/44000 | Loss: 3.0170 | LR: 3.28e-04 | 1130.8ms/step | 17.7 samples/s +Step 18030/44000 | Loss: 2.9535 | LR: 3.28e-04 | 1134.5ms/step | 17.6 samples/s +Step 18040/44000 | Loss: 2.9994 | LR: 3.27e-04 | 1154.8ms/step | 17.3 samples/s +Step 18050/44000 | Loss: 2.9921 | LR: 3.27e-04 | 1152.1ms/step | 17.4 samples/s +Step 18060/44000 | Loss: 2.9985 | LR: 3.27e-04 | 1174.7ms/step | 17.0 samples/s +Step 18070/44000 | Loss: 2.9246 | LR: 3.27e-04 | 1151.1ms/step | 17.4 samples/s +Step 18080/44000 | Loss: 3.0028 | LR: 3.27e-04 | 1176.7ms/step | 17.0 samples/s +Step 18090/44000 | Loss: 2.9466 | LR: 3.27e-04 | 1125.8ms/step | 17.8 samples/s +Step 18100/44000 | Loss: 3.0200 | LR: 3.26e-04 | 1135.7ms/step | 17.6 samples/s +Step 18110/44000 | Loss: 2.9148 | LR: 3.26e-04 | 1173.7ms/step | 17.0 samples/s +Step 18120/44000 | Loss: 2.9189 | LR: 3.26e-04 | 1171.3ms/step | 17.1 samples/s +Step 18130/44000 | Loss: 3.0320 | LR: 3.26e-04 | 1151.5ms/step | 17.4 samples/s +Step 18140/44000 | Loss: 2.9557 | LR: 3.26e-04 | 1126.7ms/step | 17.8 samples/s +Step 18150/44000 | Loss: 2.9930 | LR: 3.26e-04 | 1191.3ms/step | 16.8 samples/s +Step 18160/44000 | Loss: 2.9438 | LR: 3.25e-04 | 1141.9ms/step | 17.5 samples/s +Step 18170/44000 | Loss: 3.0065 | LR: 3.25e-04 | 1146.6ms/step | 17.4 samples/s +Step 18180/44000 | Loss: 2.9735 | LR: 3.25e-04 | 1177.4ms/step | 17.0 samples/s +Step 18190/44000 | Loss: 3.0151 | LR: 3.25e-04 | 1135.7ms/step | 17.6 samples/s +Step 18200/44000 | Loss: 3.0006 | LR: 3.25e-04 | 1182.5ms/step | 16.9 samples/s +Step 18210/44000 | Loss: 3.0014 | LR: 3.24e-04 | 1149.2ms/step | 17.4 samples/s +Step 18220/44000 | Loss: 2.9969 | LR: 3.24e-04 | 1190.5ms/step | 16.8 samples/s +Step 18230/44000 | Loss: 2.9959 | LR: 3.24e-04 | 1170.1ms/step | 17.1 samples/s +Step 18240/44000 | Loss: 2.9669 | LR: 3.24e-04 | 1177.6ms/step | 17.0 samples/s +Step 18250/44000 | Loss: 3.0037 | LR: 3.24e-04 | 1158.8ms/step | 17.3 samples/s +Validation at step 18250: Loss = 3.0069, Perplexity = 20.22 +Step 18260/44000 | Loss: 2.9648 | LR: 3.24e-04 | 1604.4ms/step | 12.5 samples/s +Step 18270/44000 | Loss: 2.9385 | LR: 3.23e-04 | 1183.7ms/step | 16.9 samples/s +Step 18280/44000 | Loss: 2.9720 | LR: 3.23e-04 | 1178.2ms/step | 17.0 samples/s +Step 18290/44000 | Loss: 2.9584 | LR: 3.23e-04 | 1180.8ms/step | 16.9 samples/s +Step 18300/44000 | Loss: 2.9875 | LR: 3.23e-04 | 1153.8ms/step | 17.3 samples/s +Step 18310/44000 | Loss: 2.9877 | LR: 3.23e-04 | 1141.3ms/step | 17.5 samples/s +Step 18320/44000 | Loss: 2.9975 | LR: 3.23e-04 | 1182.8ms/step | 16.9 samples/s +Step 18330/44000 | Loss: 2.9418 | LR: 3.22e-04 | 1187.4ms/step | 16.8 samples/s +Step 18340/44000 | Loss: 2.9393 | LR: 3.22e-04 | 1210.5ms/step | 16.5 samples/s +Step 18350/44000 | Loss: 2.9957 | LR: 3.22e-04 | 1173.1ms/step | 17.0 samples/s +Step 18360/44000 | Loss: 3.0523 | LR: 3.22e-04 | 1178.0ms/step | 17.0 samples/s +Step 18370/44000 | Loss: 3.0579 | LR: 3.22e-04 | 1178.4ms/step | 17.0 samples/s +Step 18380/44000 | Loss: 3.0182 | LR: 3.22e-04 | 1162.3ms/step | 17.2 samples/s +Step 18390/44000 | Loss: 2.9594 | LR: 3.21e-04 | 1204.2ms/step | 16.6 samples/s +Step 18400/44000 | Loss: 3.0128 | LR: 3.21e-04 | 1175.5ms/step | 17.0 samples/s +Step 18410/44000 | Loss: 2.9382 | LR: 3.21e-04 | 1175.2ms/step | 17.0 samples/s +Step 18420/44000 | Loss: 2.8820 | LR: 3.21e-04 | 1188.6ms/step | 16.8 samples/s +Step 18430/44000 | Loss: 3.0149 | LR: 3.21e-04 | 1167.9ms/step | 17.1 samples/s +Step 18440/44000 | Loss: 2.9888 | LR: 3.20e-04 | 1172.3ms/step | 17.1 samples/s +Step 18450/44000 | Loss: 3.0108 | LR: 3.20e-04 | 1147.3ms/step | 17.4 samples/s +Step 18460/44000 | Loss: 3.0126 | LR: 3.20e-04 | 1172.8ms/step | 17.1 samples/s +Step 18470/44000 | Loss: 2.9948 | LR: 3.20e-04 | 1204.3ms/step | 16.6 samples/s +Step 18480/44000 | Loss: 2.9477 | LR: 3.20e-04 | 1177.4ms/step | 17.0 samples/s +Step 18490/44000 | Loss: 2.9982 | LR: 3.20e-04 | 1157.6ms/step | 17.3 samples/s +Step 18500/44000 | Loss: 3.0615 | LR: 3.19e-04 | 1187.8ms/step | 16.8 samples/s +Validation at step 18500: Loss = 3.0055, Perplexity = 20.20 +Step 18510/44000 | Loss: 3.0031 | LR: 3.19e-04 | 1605.7ms/step | 12.5 samples/s +Step 18520/44000 | Loss: 2.9883 | LR: 3.19e-04 | 1203.8ms/step | 16.6 samples/s +Step 18530/44000 | Loss: 2.9760 | LR: 3.19e-04 | 1157.0ms/step | 17.3 samples/s +Step 18540/44000 | Loss: 3.0402 | LR: 3.19e-04 | 1188.6ms/step | 16.8 samples/s +Step 18550/44000 | Loss: 3.0172 | LR: 3.19e-04 | 1174.5ms/step | 17.0 samples/s +Step 18560/44000 | Loss: 3.0245 | LR: 3.18e-04 | 1170.9ms/step | 17.1 samples/s +Step 18570/44000 | Loss: 2.9850 | LR: 3.18e-04 | 1224.4ms/step | 16.3 samples/s +Step 18580/44000 | Loss: 3.0359 | LR: 3.18e-04 | 1189.2ms/step | 16.8 samples/s +Step 18590/44000 | Loss: 2.9425 | LR: 3.18e-04 | 1157.4ms/step | 17.3 samples/s +Step 18600/44000 | Loss: 2.9672 | LR: 3.18e-04 | 1161.4ms/step | 17.2 samples/s +Step 18610/44000 | Loss: 2.9504 | LR: 3.18e-04 | 1148.9ms/step | 17.4 samples/s +Step 18620/44000 | Loss: 2.9989 | LR: 3.17e-04 | 1197.9ms/step | 16.7 samples/s +Step 18630/44000 | Loss: 3.0327 | LR: 3.17e-04 | 1219.5ms/step | 16.4 samples/s +Step 18640/44000 | Loss: 2.9646 | LR: 3.17e-04 | 1178.3ms/step | 17.0 samples/s +Step 18650/44000 | Loss: 3.0913 | LR: 3.17e-04 | 1177.9ms/step | 17.0 samples/s +Step 18660/44000 | Loss: 2.9643 | LR: 3.17e-04 | 1203.9ms/step | 16.6 samples/s +Step 18670/44000 | Loss: 3.0189 | LR: 3.16e-04 | 1198.5ms/step | 16.7 samples/s +Step 18680/44000 | Loss: 3.0262 | LR: 3.16e-04 | 1183.3ms/step | 16.9 samples/s +Step 18690/44000 | Loss: 2.9435 | LR: 3.16e-04 | 1214.6ms/step | 16.5 samples/s +Step 18700/44000 | Loss: 2.9781 | LR: 3.16e-04 | 1208.3ms/step | 16.6 samples/s +Step 18710/44000 | Loss: 2.9159 | LR: 3.16e-04 | 1184.0ms/step | 16.9 samples/s +Step 18720/44000 | Loss: 3.0316 | LR: 3.16e-04 | 1182.4ms/step | 16.9 samples/s +Step 18730/44000 | Loss: 3.0325 | LR: 3.15e-04 | 1180.5ms/step | 16.9 samples/s +Step 18740/44000 | Loss: 2.9910 | LR: 3.15e-04 | 1149.6ms/step | 17.4 samples/s +Step 18750/44000 | Loss: 3.0114 | LR: 3.15e-04 | 1178.3ms/step | 17.0 samples/s +Validation at step 18750: Loss = 3.0036, Perplexity = 20.16 +Step 18760/44000 | Loss: 2.9486 | LR: 3.15e-04 | 1598.2ms/step | 12.5 samples/s +Step 18770/44000 | Loss: 2.9387 | LR: 3.15e-04 | 1216.1ms/step | 16.4 samples/s +Step 18780/44000 | Loss: 2.9079 | LR: 3.15e-04 | 1240.3ms/step | 16.1 samples/s +Step 18790/44000 | Loss: 3.0741 | LR: 3.14e-04 | 1215.2ms/step | 16.5 samples/s +Step 18800/44000 | Loss: 3.0084 | LR: 3.14e-04 | 1193.8ms/step | 16.8 samples/s +Step 18810/44000 | Loss: 3.0077 | LR: 3.14e-04 | 1138.9ms/step | 17.6 samples/s +Step 18820/44000 | Loss: 2.9879 | LR: 3.14e-04 | 1170.4ms/step | 17.1 samples/s +Step 18830/44000 | Loss: 2.9670 | LR: 3.14e-04 | 1172.7ms/step | 17.1 samples/s +Step 18840/44000 | Loss: 2.9585 | LR: 3.13e-04 | 1250.9ms/step | 16.0 samples/s +Step 18850/44000 | Loss: 2.9695 | LR: 3.13e-04 | 1299.8ms/step | 15.4 samples/s +Step 18860/44000 | Loss: 3.0043 | LR: 3.13e-04 | 1191.0ms/step | 16.8 samples/s +Step 18870/44000 | Loss: 2.9858 | LR: 3.13e-04 | 1073.8ms/step | 18.6 samples/s +Step 18880/44000 | Loss: 2.9988 | LR: 3.13e-04 | 1089.7ms/step | 18.4 samples/s +Step 18890/44000 | Loss: 2.9727 | LR: 3.13e-04 | 1115.2ms/step | 17.9 samples/s +Step 18900/44000 | Loss: 2.9792 | LR: 3.12e-04 | 1089.6ms/step | 18.4 samples/s +Step 18910/44000 | Loss: 3.0075 | LR: 3.12e-04 | 1099.6ms/step | 18.2 samples/s +Step 18920/44000 | Loss: 3.0161 | LR: 3.12e-04 | 1084.3ms/step | 18.4 samples/s +Step 18930/44000 | Loss: 3.0477 | LR: 3.12e-04 | 1094.8ms/step | 18.3 samples/s +Step 18940/44000 | Loss: 2.9564 | LR: 3.12e-04 | 1084.0ms/step | 18.5 samples/s +Step 18950/44000 | Loss: 2.8981 | LR: 3.12e-04 | 1125.4ms/step | 17.8 samples/s +Step 18960/44000 | Loss: 2.9464 | LR: 3.11e-04 | 1147.0ms/step | 17.4 samples/s +Step 18970/44000 | Loss: 3.0924 | LR: 3.11e-04 | 1104.5ms/step | 18.1 samples/s +Step 18980/44000 | Loss: 2.9512 | LR: 3.11e-04 | 1115.3ms/step | 17.9 samples/s +Step 18990/44000 | Loss: 3.0059 | LR: 3.11e-04 | 1073.2ms/step | 18.6 samples/s +Step 19000/44000 | Loss: 2.9858 | LR: 3.11e-04 | 1095.5ms/step | 18.3 samples/s +Validation at step 19000: Loss = 2.9993, Perplexity = 20.07 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-19000 +Step 19010/44000 | Loss: 2.9817 | LR: 3.11e-04 | 2048.1ms/step | 9.8 samples/s +Step 19020/44000 | Loss: 3.0016 | LR: 3.10e-04 | 1105.0ms/step | 18.1 samples/s +Step 19030/44000 | Loss: 2.9921 | LR: 3.10e-04 | 1080.0ms/step | 18.5 samples/s +Step 19040/44000 | Loss: 2.9767 | LR: 3.10e-04 | 1083.3ms/step | 18.5 samples/s +Step 19050/44000 | Loss: 2.9524 | LR: 3.10e-04 | 1105.1ms/step | 18.1 samples/s +Step 19060/44000 | Loss: 2.9465 | LR: 3.10e-04 | 1089.6ms/step | 18.4 samples/s +Step 19070/44000 | Loss: 2.9176 | LR: 3.09e-04 | 1057.3ms/step | 18.9 samples/s +Step 19080/44000 | Loss: 3.0455 | LR: 3.09e-04 | 1109.4ms/step | 18.0 samples/s +Step 19090/44000 | Loss: 3.0315 | LR: 3.09e-04 | 1065.5ms/step | 18.8 samples/s +Step 19100/44000 | Loss: 3.0403 | LR: 3.09e-04 | 1090.3ms/step | 18.3 samples/s +Step 19110/44000 | Loss: 3.0508 | LR: 3.09e-04 | 1089.4ms/step | 18.4 samples/s +Step 19120/44000 | Loss: 2.9486 | LR: 3.09e-04 | 1146.8ms/step | 17.4 samples/s +Step 19130/44000 | Loss: 3.0127 | LR: 3.08e-04 | 1099.7ms/step | 18.2 samples/s +Step 19140/44000 | Loss: 3.0816 | LR: 3.08e-04 | 1110.1ms/step | 18.0 samples/s +Step 19150/44000 | Loss: 2.9244 | LR: 3.08e-04 | 1136.5ms/step | 17.6 samples/s +Step 19160/44000 | Loss: 3.0445 | LR: 3.08e-04 | 1110.2ms/step | 18.0 samples/s +Step 19170/44000 | Loss: 3.0270 | LR: 3.08e-04 | 1057.6ms/step | 18.9 samples/s +Step 19180/44000 | Loss: 2.9956 | LR: 3.08e-04 | 1121.4ms/step | 17.8 samples/s +Step 19190/44000 | Loss: 2.9592 | LR: 3.07e-04 | 1141.9ms/step | 17.5 samples/s +Step 19200/44000 | Loss: 3.0182 | LR: 3.07e-04 | 1104.9ms/step | 18.1 samples/s +Step 19210/44000 | Loss: 2.9671 | LR: 3.07e-04 | 1115.9ms/step | 17.9 samples/s +Step 19220/44000 | Loss: 2.9857 | LR: 3.07e-04 | 1090.1ms/step | 18.3 samples/s +Step 19230/44000 | Loss: 3.0268 | LR: 3.07e-04 | 1089.0ms/step | 18.4 samples/s +Step 19240/44000 | Loss: 3.0288 | LR: 3.06e-04 | 1105.3ms/step | 18.1 samples/s +Step 19250/44000 | Loss: 2.9720 | LR: 3.06e-04 | 1068.4ms/step | 18.7 samples/s +Validation at step 19250: Loss = 2.9970, Perplexity = 20.03 +Step 19260/44000 | Loss: 2.9745 | LR: 3.06e-04 | 1465.4ms/step | 13.6 samples/s +Step 19270/44000 | Loss: 2.9578 | LR: 3.06e-04 | 1086.9ms/step | 18.4 samples/s +Step 19280/44000 | Loss: 2.9762 | LR: 3.06e-04 | 1082.1ms/step | 18.5 samples/s +Step 19290/44000 | Loss: 2.9904 | LR: 3.06e-04 | 1125.9ms/step | 17.8 samples/s +Step 19300/44000 | Loss: 2.8722 | LR: 3.05e-04 | 1145.9ms/step | 17.5 samples/s +Step 19310/44000 | Loss: 3.0479 | LR: 3.05e-04 | 1106.0ms/step | 18.1 samples/s +Step 19320/44000 | Loss: 2.9716 | LR: 3.05e-04 | 1115.4ms/step | 17.9 samples/s +Step 19330/44000 | Loss: 2.9855 | LR: 3.05e-04 | 1107.0ms/step | 18.1 samples/s +Step 19340/44000 | Loss: 3.0456 | LR: 3.05e-04 | 1192.8ms/step | 16.8 samples/s +Step 19350/44000 | Loss: 2.9191 | LR: 3.04e-04 | 1172.9ms/step | 17.1 samples/s +Step 19360/44000 | Loss: 2.9978 | LR: 3.04e-04 | 1152.8ms/step | 17.3 samples/s +Step 19370/44000 | Loss: 2.9714 | LR: 3.04e-04 | 1074.4ms/step | 18.6 samples/s +Step 19380/44000 | Loss: 3.0241 | LR: 3.04e-04 | 1110.5ms/step | 18.0 samples/s +Step 19390/44000 | Loss: 2.9212 | LR: 3.04e-04 | 1125.7ms/step | 17.8 samples/s +Step 19400/44000 | Loss: 2.9242 | LR: 3.04e-04 | 1142.0ms/step | 17.5 samples/s +Step 19410/44000 | Loss: 2.9588 | LR: 3.03e-04 | 1110.1ms/step | 18.0 samples/s +Step 19420/44000 | Loss: 3.0164 | LR: 3.03e-04 | 1048.8ms/step | 19.1 samples/s +Step 19430/44000 | Loss: 2.9828 | LR: 3.03e-04 | 1110.1ms/step | 18.0 samples/s +Step 19440/44000 | Loss: 2.9415 | LR: 3.03e-04 | 1094.3ms/step | 18.3 samples/s +Step 19450/44000 | Loss: 2.9728 | LR: 3.03e-04 | 1105.7ms/step | 18.1 samples/s +Step 19460/44000 | Loss: 2.8931 | LR: 3.03e-04 | 1131.3ms/step | 17.7 samples/s +Step 19470/44000 | Loss: 2.9401 | LR: 3.02e-04 | 1103.5ms/step | 18.1 samples/s +Step 19480/44000 | Loss: 3.0067 | LR: 3.02e-04 | 1109.7ms/step | 18.0 samples/s +Step 19490/44000 | Loss: 2.9781 | LR: 3.02e-04 | 1121.0ms/step | 17.8 samples/s +Step 19500/44000 | Loss: 2.9851 | LR: 3.02e-04 | 1160.0ms/step | 17.2 samples/s +Validation at step 19500: Loss = 2.9948, Perplexity = 19.98 +Step 19510/44000 | Loss: 2.9929 | LR: 3.02e-04 | 1496.8ms/step | 13.4 samples/s +Step 19520/44000 | Loss: 2.9852 | LR: 3.01e-04 | 1079.1ms/step | 18.5 samples/s +Step 19530/44000 | Loss: 2.9797 | LR: 3.01e-04 | 1141.0ms/step | 17.5 samples/s +Step 19540/44000 | Loss: 2.9911 | LR: 3.01e-04 | 1149.5ms/step | 17.4 samples/s +Step 19550/44000 | Loss: 3.0174 | LR: 3.01e-04 | 1188.0ms/step | 16.8 samples/s +Step 19560/44000 | Loss: 2.9247 | LR: 3.01e-04 | 1126.0ms/step | 17.8 samples/s +Step 19570/44000 | Loss: 3.0109 | LR: 3.01e-04 | 1046.2ms/step | 19.1 samples/s +Step 19580/44000 | Loss: 3.0950 | LR: 3.00e-04 | 1126.0ms/step | 17.8 samples/s +Step 19590/44000 | Loss: 3.0220 | LR: 3.00e-04 | 1169.8ms/step | 17.1 samples/s +Step 19600/44000 | Loss: 3.0046 | LR: 3.00e-04 | 1105.3ms/step | 18.1 samples/s +Step 19610/44000 | Loss: 2.9712 | LR: 3.00e-04 | 1105.7ms/step | 18.1 samples/s +Step 19620/44000 | Loss: 2.9870 | LR: 3.00e-04 | 1100.0ms/step | 18.2 samples/s +Step 19630/44000 | Loss: 3.0052 | LR: 3.00e-04 | 1095.1ms/step | 18.3 samples/s +Step 19640/44000 | Loss: 2.9769 | LR: 2.99e-04 | 1094.6ms/step | 18.3 samples/s +Step 19650/44000 | Loss: 2.9104 | LR: 2.99e-04 | 1074.4ms/step | 18.6 samples/s +Step 19660/44000 | Loss: 3.0082 | LR: 2.99e-04 | 1094.7ms/step | 18.3 samples/s +Step 19670/44000 | Loss: 2.9784 | LR: 2.99e-04 | 1081.0ms/step | 18.5 samples/s +Step 19680/44000 | Loss: 2.9887 | LR: 2.99e-04 | 1119.8ms/step | 17.9 samples/s +Step 19690/44000 | Loss: 2.9740 | LR: 2.98e-04 | 1115.7ms/step | 17.9 samples/s +Step 19700/44000 | Loss: 2.9574 | LR: 2.98e-04 | 1121.2ms/step | 17.8 samples/s +Step 19710/44000 | Loss: 2.9860 | LR: 2.98e-04 | 1058.5ms/step | 18.9 samples/s +Step 19720/44000 | Loss: 2.9589 | LR: 2.98e-04 | 1110.1ms/step | 18.0 samples/s +Step 19730/44000 | Loss: 2.9621 | LR: 2.98e-04 | 1116.2ms/step | 17.9 samples/s +Step 19740/44000 | Loss: 2.9722 | LR: 2.98e-04 | 1084.1ms/step | 18.4 samples/s +Step 19750/44000 | Loss: 2.8675 | LR: 2.97e-04 | 1127.4ms/step | 17.7 samples/s +Validation at step 19750: Loss = 2.9923, Perplexity = 19.93 +Step 19760/44000 | Loss: 2.9427 | LR: 2.97e-04 | 1480.6ms/step | 13.5 samples/s +Step 19770/44000 | Loss: 2.9857 | LR: 2.97e-04 | 1120.4ms/step | 17.9 samples/s +Step 19780/44000 | Loss: 2.9539 | LR: 2.97e-04 | 1152.2ms/step | 17.4 samples/s +Step 19790/44000 | Loss: 2.9804 | LR: 2.97e-04 | 1168.1ms/step | 17.1 samples/s +Step 19800/44000 | Loss: 2.9797 | LR: 2.96e-04 | 1167.6ms/step | 17.1 samples/s +Step 19810/44000 | Loss: 2.9764 | LR: 2.96e-04 | 1137.9ms/step | 17.6 samples/s +Step 19820/44000 | Loss: 2.9467 | LR: 2.96e-04 | 1093.9ms/step | 18.3 samples/s +Step 19830/44000 | Loss: 2.9339 | LR: 2.96e-04 | 1120.9ms/step | 17.8 samples/s +Step 19840/44000 | Loss: 3.0300 | LR: 2.96e-04 | 1038.3ms/step | 19.3 samples/s +Step 19850/44000 | Loss: 3.0132 | LR: 2.96e-04 | 1074.1ms/step | 18.6 samples/s +Step 19860/44000 | Loss: 3.0428 | LR: 2.95e-04 | 1106.5ms/step | 18.1 samples/s +Step 19870/44000 | Loss: 2.9284 | LR: 2.95e-04 | 1117.2ms/step | 17.9 samples/s +Step 19880/44000 | Loss: 2.9773 | LR: 2.95e-04 | 1077.2ms/step | 18.6 samples/s +Step 19890/44000 | Loss: 2.9538 | LR: 2.95e-04 | 1126.6ms/step | 17.8 samples/s +Step 19900/44000 | Loss: 2.9801 | LR: 2.95e-04 | 1080.7ms/step | 18.5 samples/s +Step 19910/44000 | Loss: 2.9567 | LR: 2.95e-04 | 1096.8ms/step | 18.2 samples/s +Step 19920/44000 | Loss: 2.9799 | LR: 2.94e-04 | 1081.7ms/step | 18.5 samples/s +Step 19930/44000 | Loss: 2.9868 | LR: 2.94e-04 | 1141.0ms/step | 17.5 samples/s +Step 19940/44000 | Loss: 3.0331 | LR: 2.94e-04 | 1095.0ms/step | 18.3 samples/s +Step 19950/44000 | Loss: 2.9284 | LR: 2.94e-04 | 1137.4ms/step | 17.6 samples/s +Step 19960/44000 | Loss: 2.9341 | LR: 2.94e-04 | 1122.0ms/step | 17.8 samples/s +Step 19970/44000 | Loss: 3.0336 | LR: 2.93e-04 | 1116.2ms/step | 17.9 samples/s +Step 19980/44000 | Loss: 2.9421 | LR: 2.93e-04 | 1115.6ms/step | 17.9 samples/s +Step 19990/44000 | Loss: 2.9884 | LR: 2.93e-04 | 1135.9ms/step | 17.6 samples/s +Step 20000/44000 | Loss: 2.9683 | LR: 2.93e-04 | 1116.2ms/step | 17.9 samples/s +Validation at step 20000: Loss = 2.9881, Perplexity = 19.85 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-20000 +Step 20010/44000 | Loss: 2.9771 | LR: 2.93e-04 | 1971.4ms/step | 10.1 samples/s +Step 20020/44000 | Loss: 2.9534 | LR: 2.93e-04 | 1106.0ms/step | 18.1 samples/s +Step 20030/44000 | Loss: 2.9374 | LR: 2.92e-04 | 1100.0ms/step | 18.2 samples/s +Step 20040/44000 | Loss: 2.9794 | LR: 2.92e-04 | 1115.6ms/step | 17.9 samples/s +Step 20050/44000 | Loss: 3.0183 | LR: 2.92e-04 | 1075.3ms/step | 18.6 samples/s +Step 20060/44000 | Loss: 3.0192 | LR: 2.92e-04 | 1099.4ms/step | 18.2 samples/s +Step 20070/44000 | Loss: 2.9985 | LR: 2.92e-04 | 1100.9ms/step | 18.2 samples/s +Step 20080/44000 | Loss: 2.9568 | LR: 2.91e-04 | 1115.9ms/step | 17.9 samples/s +Step 20090/44000 | Loss: 2.9744 | LR: 2.91e-04 | 1106.0ms/step | 18.1 samples/s +Step 20100/44000 | Loss: 2.9585 | LR: 2.91e-04 | 1099.9ms/step | 18.2 samples/s +Step 20110/44000 | Loss: 2.9237 | LR: 2.91e-04 | 1105.5ms/step | 18.1 samples/s +Step 20120/44000 | Loss: 2.9345 | LR: 2.91e-04 | 1090.3ms/step | 18.3 samples/s +Step 20130/44000 | Loss: 2.9377 | LR: 2.91e-04 | 1080.5ms/step | 18.5 samples/s +Step 20140/44000 | Loss: 2.8879 | LR: 2.90e-04 | 1123.1ms/step | 17.8 samples/s +Step 20150/44000 | Loss: 2.8939 | LR: 2.90e-04 | 1155.2ms/step | 17.3 samples/s +Step 20160/44000 | Loss: 2.9955 | LR: 2.90e-04 | 1121.3ms/step | 17.8 samples/s +Step 20170/44000 | Loss: 2.8918 | LR: 2.90e-04 | 1105.1ms/step | 18.1 samples/s +Step 20180/44000 | Loss: 2.9716 | LR: 2.90e-04 | 1131.2ms/step | 17.7 samples/s +Step 20190/44000 | Loss: 2.9564 | LR: 2.90e-04 | 1061.1ms/step | 18.8 samples/s +Step 20200/44000 | Loss: 2.9820 | LR: 2.89e-04 | 1129.2ms/step | 17.7 samples/s +Step 20210/44000 | Loss: 2.9930 | LR: 2.89e-04 | 1110.8ms/step | 18.0 samples/s +Step 20220/44000 | Loss: 2.9471 | LR: 2.89e-04 | 1131.8ms/step | 17.7 samples/s +Step 20230/44000 | Loss: 2.9841 | LR: 2.89e-04 | 1136.6ms/step | 17.6 samples/s +Step 20240/44000 | Loss: 2.9121 | LR: 2.89e-04 | 1131.5ms/step | 17.7 samples/s +Step 20250/44000 | Loss: 2.9022 | LR: 2.88e-04 | 1168.5ms/step | 17.1 samples/s +Validation at step 20250: Loss = 2.9865, Perplexity = 19.82 +Step 20260/44000 | Loss: 2.9773 | LR: 2.88e-04 | 1579.6ms/step | 12.7 samples/s +Step 20270/44000 | Loss: 2.9487 | LR: 2.88e-04 | 1155.7ms/step | 17.3 samples/s +Step 20280/44000 | Loss: 3.0170 | LR: 2.88e-04 | 1149.1ms/step | 17.4 samples/s +Step 20290/44000 | Loss: 2.9855 | LR: 2.88e-04 | 1100.1ms/step | 18.2 samples/s +Step 20300/44000 | Loss: 2.9379 | LR: 2.88e-04 | 1162.9ms/step | 17.2 samples/s +Step 20310/44000 | Loss: 2.9836 | LR: 2.87e-04 | 1127.3ms/step | 17.7 samples/s +Step 20320/44000 | Loss: 2.9560 | LR: 2.87e-04 | 1095.4ms/step | 18.3 samples/s +Step 20330/44000 | Loss: 2.9465 | LR: 2.87e-04 | 1108.6ms/step | 18.0 samples/s +Step 20340/44000 | Loss: 2.9703 | LR: 2.87e-04 | 1142.1ms/step | 17.5 samples/s +Step 20350/44000 | Loss: 2.9843 | LR: 2.87e-04 | 1100.4ms/step | 18.2 samples/s +Step 20360/44000 | Loss: 3.0073 | LR: 2.86e-04 | 1074.4ms/step | 18.6 samples/s +Step 20370/44000 | Loss: 2.9510 | LR: 2.86e-04 | 1053.5ms/step | 19.0 samples/s +Step 20380/44000 | Loss: 2.9648 | LR: 2.86e-04 | 1142.2ms/step | 17.5 samples/s +Step 20390/44000 | Loss: 2.9053 | LR: 2.86e-04 | 1079.4ms/step | 18.5 samples/s +Step 20400/44000 | Loss: 2.9646 | LR: 2.86e-04 | 1115.5ms/step | 17.9 samples/s +Step 20410/44000 | Loss: 3.0242 | LR: 2.86e-04 | 1141.2ms/step | 17.5 samples/s +Step 20420/44000 | Loss: 2.9892 | LR: 2.85e-04 | 1116.1ms/step | 17.9 samples/s +Step 20430/44000 | Loss: 2.9140 | LR: 2.85e-04 | 1081.8ms/step | 18.5 samples/s +Step 20440/44000 | Loss: 2.9249 | LR: 2.85e-04 | 1101.9ms/step | 18.2 samples/s +Step 20450/44000 | Loss: 2.9311 | LR: 2.85e-04 | 1126.5ms/step | 17.8 samples/s +Step 20460/44000 | Loss: 2.9760 | LR: 2.85e-04 | 1089.6ms/step | 18.4 samples/s +Step 20470/44000 | Loss: 2.9301 | LR: 2.84e-04 | 1100.2ms/step | 18.2 samples/s +Step 20480/44000 | Loss: 2.9236 | LR: 2.84e-04 | 1120.9ms/step | 17.8 samples/s +Step 20490/44000 | Loss: 2.9889 | LR: 2.84e-04 | 1099.6ms/step | 18.2 samples/s +Step 20500/44000 | Loss: 2.9590 | LR: 2.84e-04 | 1084.6ms/step | 18.4 samples/s +Validation at step 20500: Loss = 2.9856, Perplexity = 19.80 +Step 20510/44000 | Loss: 2.9649 | LR: 2.84e-04 | 1507.0ms/step | 13.3 samples/s +Step 20520/44000 | Loss: 3.0143 | LR: 2.84e-04 | 1126.1ms/step | 17.8 samples/s +Step 20530/44000 | Loss: 2.9409 | LR: 2.83e-04 | 1136.5ms/step | 17.6 samples/s +Step 20540/44000 | Loss: 3.0004 | LR: 2.83e-04 | 1126.0ms/step | 17.8 samples/s +Step 20550/44000 | Loss: 2.9603 | LR: 2.83e-04 | 1146.8ms/step | 17.4 samples/s +Step 20560/44000 | Loss: 2.9398 | LR: 2.83e-04 | 1094.9ms/step | 18.3 samples/s +Step 20570/44000 | Loss: 3.0415 | LR: 2.83e-04 | 1120.9ms/step | 17.8 samples/s +Step 20580/44000 | Loss: 2.9303 | LR: 2.83e-04 | 1110.5ms/step | 18.0 samples/s +Step 20590/44000 | Loss: 2.9290 | LR: 2.82e-04 | 1100.3ms/step | 18.2 samples/s +Step 20600/44000 | Loss: 3.0336 | LR: 2.82e-04 | 1147.3ms/step | 17.4 samples/s +Step 20610/44000 | Loss: 2.9710 | LR: 2.82e-04 | 1110.7ms/step | 18.0 samples/s +Step 20620/44000 | Loss: 2.9668 | LR: 2.82e-04 | 1063.8ms/step | 18.8 samples/s +Step 20630/44000 | Loss: 2.9618 | LR: 2.82e-04 | 1079.1ms/step | 18.5 samples/s +Step 20640/44000 | Loss: 3.0040 | LR: 2.81e-04 | 1085.2ms/step | 18.4 samples/s +Step 20650/44000 | Loss: 2.9322 | LR: 2.81e-04 | 1062.7ms/step | 18.8 samples/s +Step 20660/44000 | Loss: 2.9116 | LR: 2.81e-04 | 1110.0ms/step | 18.0 samples/s +Step 20670/44000 | Loss: 2.9380 | LR: 2.81e-04 | 1083.9ms/step | 18.5 samples/s +Step 20680/44000 | Loss: 2.9584 | LR: 2.81e-04 | 1100.6ms/step | 18.2 samples/s +Step 20690/44000 | Loss: 2.9056 | LR: 2.81e-04 | 1109.7ms/step | 18.0 samples/s +Step 20700/44000 | Loss: 2.9717 | LR: 2.80e-04 | 1089.9ms/step | 18.4 samples/s +Step 20710/44000 | Loss: 2.9350 | LR: 2.80e-04 | 1084.4ms/step | 18.4 samples/s +Step 20720/44000 | Loss: 2.9469 | LR: 2.80e-04 | 1126.0ms/step | 17.8 samples/s +Step 20730/44000 | Loss: 2.9526 | LR: 2.80e-04 | 1110.2ms/step | 18.0 samples/s +Step 20740/44000 | Loss: 2.9980 | LR: 2.80e-04 | 1105.2ms/step | 18.1 samples/s +Step 20750/44000 | Loss: 2.9409 | LR: 2.79e-04 | 1138.9ms/step | 17.6 samples/s +Validation at step 20750: Loss = 2.9825, Perplexity = 19.74 +Step 20760/44000 | Loss: 2.9967 | LR: 2.79e-04 | 1512.6ms/step | 13.2 samples/s +Step 20770/44000 | Loss: 2.9924 | LR: 2.79e-04 | 1062.4ms/step | 18.8 samples/s +Step 20780/44000 | Loss: 2.9550 | LR: 2.79e-04 | 1110.4ms/step | 18.0 samples/s +Step 20790/44000 | Loss: 2.9688 | LR: 2.79e-04 | 1100.4ms/step | 18.2 samples/s +Step 20800/44000 | Loss: 2.9841 | LR: 2.79e-04 | 1071.9ms/step | 18.7 samples/s +Step 20810/44000 | Loss: 2.9539 | LR: 2.78e-04 | 1082.6ms/step | 18.5 samples/s +Step 20820/44000 | Loss: 2.9715 | LR: 2.78e-04 | 1111.0ms/step | 18.0 samples/s +Step 20830/44000 | Loss: 2.9674 | LR: 2.78e-04 | 1101.8ms/step | 18.2 samples/s +Step 20840/44000 | Loss: 2.9936 | LR: 2.78e-04 | 1063.5ms/step | 18.8 samples/s +Step 20850/44000 | Loss: 2.9175 | LR: 2.78e-04 | 1084.1ms/step | 18.4 samples/s +Step 20860/44000 | Loss: 2.9762 | LR: 2.77e-04 | 1111.2ms/step | 18.0 samples/s +Step 20870/44000 | Loss: 2.9390 | LR: 2.77e-04 | 1121.0ms/step | 17.8 samples/s +Step 20880/44000 | Loss: 2.9748 | LR: 2.77e-04 | 1103.8ms/step | 18.1 samples/s +Step 20890/44000 | Loss: 2.8843 | LR: 2.77e-04 | 1086.4ms/step | 18.4 samples/s +Step 20900/44000 | Loss: 2.9529 | LR: 2.77e-04 | 1078.4ms/step | 18.5 samples/s +Step 20910/44000 | Loss: 2.9474 | LR: 2.77e-04 | 1121.2ms/step | 17.8 samples/s +Step 20920/44000 | Loss: 2.9316 | LR: 2.76e-04 | 1084.8ms/step | 18.4 samples/s +Step 20930/44000 | Loss: 2.9979 | LR: 2.76e-04 | 1110.7ms/step | 18.0 samples/s +Step 20940/44000 | Loss: 2.9240 | LR: 2.76e-04 | 1166.6ms/step | 17.1 samples/s +Step 20950/44000 | Loss: 2.9359 | LR: 2.76e-04 | 1090.6ms/step | 18.3 samples/s +Step 20960/44000 | Loss: 3.0223 | LR: 2.76e-04 | 1071.1ms/step | 18.7 samples/s +Step 20970/44000 | Loss: 2.8853 | LR: 2.75e-04 | 1091.1ms/step | 18.3 samples/s +Step 20980/44000 | Loss: 2.9810 | LR: 2.75e-04 | 1092.0ms/step | 18.3 samples/s +Step 20990/44000 | Loss: 2.9116 | LR: 2.75e-04 | 1045.5ms/step | 19.1 samples/s +Step 21000/44000 | Loss: 2.9713 | LR: 2.75e-04 | 1128.2ms/step | 17.7 samples/s +Validation at step 21000: Loss = 2.9796, Perplexity = 19.68 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-21000 +Step 21010/44000 | Loss: 2.9756 | LR: 2.75e-04 | 1973.6ms/step | 10.1 samples/s +Step 21020/44000 | Loss: 2.9430 | LR: 2.75e-04 | 1102.5ms/step | 18.1 samples/s +Step 21030/44000 | Loss: 2.9331 | LR: 2.74e-04 | 1112.7ms/step | 18.0 samples/s +Step 21040/44000 | Loss: 2.9235 | LR: 2.74e-04 | 1098.9ms/step | 18.2 samples/s +Step 21050/44000 | Loss: 2.9455 | LR: 2.74e-04 | 1110.8ms/step | 18.0 samples/s +Step 21060/44000 | Loss: 2.9382 | LR: 2.74e-04 | 1051.2ms/step | 19.0 samples/s +Step 21070/44000 | Loss: 2.9857 | LR: 2.74e-04 | 1086.8ms/step | 18.4 samples/s +Step 21080/44000 | Loss: 3.0009 | LR: 2.73e-04 | 1084.9ms/step | 18.4 samples/s +Step 21090/44000 | Loss: 2.9320 | LR: 2.73e-04 | 1058.1ms/step | 18.9 samples/s +Step 21100/44000 | Loss: 2.9210 | LR: 2.73e-04 | 1079.4ms/step | 18.5 samples/s +Step 21110/44000 | Loss: 2.8866 | LR: 2.73e-04 | 1115.9ms/step | 17.9 samples/s +Step 21120/44000 | Loss: 2.9500 | LR: 2.73e-04 | 1105.9ms/step | 18.1 samples/s +Step 21130/44000 | Loss: 2.9515 | LR: 2.73e-04 | 1093.5ms/step | 18.3 samples/s +Step 21140/44000 | Loss: 2.9459 | LR: 2.72e-04 | 1125.3ms/step | 17.8 samples/s +Step 21150/44000 | Loss: 2.9198 | LR: 2.72e-04 | 1114.7ms/step | 17.9 samples/s +Step 21160/44000 | Loss: 2.9565 | LR: 2.72e-04 | 1070.2ms/step | 18.7 samples/s +Step 21170/44000 | Loss: 2.9942 | LR: 2.72e-04 | 1086.6ms/step | 18.4 samples/s +Step 21180/44000 | Loss: 2.9802 | LR: 2.72e-04 | 1089.4ms/step | 18.4 samples/s +Step 21190/44000 | Loss: 3.0249 | LR: 2.71e-04 | 1105.6ms/step | 18.1 samples/s +Step 21200/44000 | Loss: 2.9358 | LR: 2.71e-04 | 1082.8ms/step | 18.5 samples/s +Step 21210/44000 | Loss: 3.0021 | LR: 2.71e-04 | 1107.5ms/step | 18.1 samples/s +Step 21220/44000 | Loss: 2.9649 | LR: 2.71e-04 | 1110.9ms/step | 18.0 samples/s +Step 21230/44000 | Loss: 2.9186 | LR: 2.71e-04 | 1090.9ms/step | 18.3 samples/s +Step 21240/44000 | Loss: 2.9639 | LR: 2.71e-04 | 1078.4ms/step | 18.5 samples/s +Step 21250/44000 | Loss: 2.9082 | LR: 2.70e-04 | 1089.6ms/step | 18.4 samples/s +Validation at step 21250: Loss = 2.9769, Perplexity = 19.63 +Step 21260/44000 | Loss: 2.9632 | LR: 2.70e-04 | 1475.8ms/step | 13.6 samples/s +Step 21270/44000 | Loss: 2.9815 | LR: 2.70e-04 | 1080.1ms/step | 18.5 samples/s +Step 21280/44000 | Loss: 2.9806 | LR: 2.70e-04 | 1064.2ms/step | 18.8 samples/s +Step 21290/44000 | Loss: 2.9349 | LR: 2.70e-04 | 1069.4ms/step | 18.7 samples/s +Step 21300/44000 | Loss: 3.0275 | LR: 2.70e-04 | 1126.3ms/step | 17.8 samples/s +Step 21310/44000 | Loss: 2.9595 | LR: 2.69e-04 | 1110.7ms/step | 18.0 samples/s +Step 21320/44000 | Loss: 2.9595 | LR: 2.69e-04 | 1079.6ms/step | 18.5 samples/s +Step 21330/44000 | Loss: 2.9398 | LR: 2.69e-04 | 1068.7ms/step | 18.7 samples/s +Step 21340/44000 | Loss: 2.9101 | LR: 2.69e-04 | 1100.6ms/step | 18.2 samples/s +Step 21350/44000 | Loss: 2.9515 | LR: 2.69e-04 | 1110.8ms/step | 18.0 samples/s +Step 21360/44000 | Loss: 2.9455 | LR: 2.68e-04 | 1092.5ms/step | 18.3 samples/s +Step 21370/44000 | Loss: 2.9981 | LR: 2.68e-04 | 1118.0ms/step | 17.9 samples/s +Step 21380/44000 | Loss: 2.9310 | LR: 2.68e-04 | 1079.7ms/step | 18.5 samples/s +Step 21390/44000 | Loss: 2.9049 | LR: 2.68e-04 | 1146.8ms/step | 17.4 samples/s +Step 21400/44000 | Loss: 2.9322 | LR: 2.68e-04 | 1071.7ms/step | 18.7 samples/s +Step 21410/44000 | Loss: 2.9012 | LR: 2.68e-04 | 1086.9ms/step | 18.4 samples/s +Step 21420/44000 | Loss: 2.8982 | LR: 2.67e-04 | 1087.8ms/step | 18.4 samples/s +Step 21430/44000 | Loss: 2.9815 | LR: 2.67e-04 | 1097.4ms/step | 18.2 samples/s +Step 21440/44000 | Loss: 2.9369 | LR: 2.67e-04 | 1089.2ms/step | 18.4 samples/s +Step 21450/44000 | Loss: 2.9884 | LR: 2.67e-04 | 1120.6ms/step | 17.8 samples/s +Step 21460/44000 | Loss: 2.9636 | LR: 2.67e-04 | 1074.3ms/step | 18.6 samples/s +Step 21470/44000 | Loss: 2.9442 | LR: 2.66e-04 | 1064.2ms/step | 18.8 samples/s +Step 21480/44000 | Loss: 2.9590 | LR: 2.66e-04 | 1109.3ms/step | 18.0 samples/s +Step 21490/44000 | Loss: 2.9867 | LR: 2.66e-04 | 1094.4ms/step | 18.3 samples/s +Step 21500/44000 | Loss: 2.9243 | LR: 2.66e-04 | 1089.7ms/step | 18.4 samples/s +Validation at step 21500: Loss = 2.9750, Perplexity = 19.59 +Step 21510/44000 | Loss: 2.8701 | LR: 2.66e-04 | 1449.4ms/step | 13.8 samples/s +Step 21520/44000 | Loss: 2.8925 | LR: 2.66e-04 | 1103.8ms/step | 18.1 samples/s +Step 21530/44000 | Loss: 2.9579 | LR: 2.65e-04 | 1104.7ms/step | 18.1 samples/s +Step 21540/44000 | Loss: 2.9379 | LR: 2.65e-04 | 1180.4ms/step | 16.9 samples/s +Step 21550/44000 | Loss: 2.9092 | LR: 2.65e-04 | 1282.2ms/step | 15.6 samples/s +Step 21560/44000 | Loss: 2.9847 | LR: 2.65e-04 | 1318.4ms/step | 15.2 samples/s +Step 21570/44000 | Loss: 2.8825 | LR: 2.65e-04 | 1276.2ms/step | 15.7 samples/s +Step 21580/44000 | Loss: 2.9622 | LR: 2.64e-04 | 1204.5ms/step | 16.6 samples/s +Step 21590/44000 | Loss: 2.9371 | LR: 2.64e-04 | 1094.8ms/step | 18.3 samples/s +Step 21600/44000 | Loss: 2.9627 | LR: 2.64e-04 | 1098.9ms/step | 18.2 samples/s +Step 21610/44000 | Loss: 3.0159 | LR: 2.64e-04 | 1111.2ms/step | 18.0 samples/s +Step 21620/44000 | Loss: 2.9638 | LR: 2.64e-04 | 1184.6ms/step | 16.9 samples/s +Step 21630/44000 | Loss: 2.9373 | LR: 2.64e-04 | 1153.5ms/step | 17.3 samples/s +Step 21640/44000 | Loss: 2.9373 | LR: 2.63e-04 | 1143.5ms/step | 17.5 samples/s +Step 21650/44000 | Loss: 2.9611 | LR: 2.63e-04 | 1120.6ms/step | 17.8 samples/s +Step 21660/44000 | Loss: 2.9235 | LR: 2.63e-04 | 1162.2ms/step | 17.2 samples/s +Step 21670/44000 | Loss: 2.9122 | LR: 2.63e-04 | 1125.9ms/step | 17.8 samples/s +Step 21680/44000 | Loss: 2.9000 | LR: 2.63e-04 | 1198.4ms/step | 16.7 samples/s +Step 21690/44000 | Loss: 2.9266 | LR: 2.62e-04 | 1193.9ms/step | 16.8 samples/s +Step 21700/44000 | Loss: 2.9526 | LR: 2.62e-04 | 1253.6ms/step | 16.0 samples/s +Step 21710/44000 | Loss: 3.0242 | LR: 2.62e-04 | 1197.4ms/step | 16.7 samples/s +Step 21720/44000 | Loss: 2.9240 | LR: 2.62e-04 | 1208.6ms/step | 16.5 samples/s +Step 21730/44000 | Loss: 2.9468 | LR: 2.62e-04 | 1172.9ms/step | 17.1 samples/s +Step 21740/44000 | Loss: 2.9318 | LR: 2.62e-04 | 1162.4ms/step | 17.2 samples/s +Step 21750/44000 | Loss: 2.8748 | LR: 2.61e-04 | 1229.7ms/step | 16.3 samples/s +Validation at step 21750: Loss = 2.9728, Perplexity = 19.55 +Step 21760/44000 | Loss: 2.9794 | LR: 2.61e-04 | 1466.7ms/step | 13.6 samples/s +Step 21770/44000 | Loss: 2.9499 | LR: 2.61e-04 | 1129.5ms/step | 17.7 samples/s +Step 21780/44000 | Loss: 3.0119 | LR: 2.61e-04 | 1113.4ms/step | 18.0 samples/s +Step 21790/44000 | Loss: 2.9701 | LR: 2.61e-04 | 1077.5ms/step | 18.6 samples/s +Step 21800/44000 | Loss: 2.9262 | LR: 2.60e-04 | 1089.3ms/step | 18.4 samples/s +Step 21810/44000 | Loss: 2.9207 | LR: 2.60e-04 | 1100.1ms/step | 18.2 samples/s +Step 21820/44000 | Loss: 2.9167 | LR: 2.60e-04 | 1213.3ms/step | 16.5 samples/s +Step 21830/44000 | Loss: 2.9300 | LR: 2.60e-04 | 1075.3ms/step | 18.6 samples/s +Step 21840/44000 | Loss: 2.9182 | LR: 2.60e-04 | 1136.6ms/step | 17.6 samples/s +Step 21850/44000 | Loss: 2.9440 | LR: 2.60e-04 | 1137.1ms/step | 17.6 samples/s +Step 21860/44000 | Loss: 2.9235 | LR: 2.59e-04 | 1167.8ms/step | 17.1 samples/s +Step 21870/44000 | Loss: 2.9328 | LR: 2.59e-04 | 1162.8ms/step | 17.2 samples/s +Step 21880/44000 | Loss: 2.9145 | LR: 2.59e-04 | 1168.0ms/step | 17.1 samples/s +Step 21890/44000 | Loss: 2.9910 | LR: 2.59e-04 | 1152.6ms/step | 17.4 samples/s +Step 21900/44000 | Loss: 2.9406 | LR: 2.59e-04 | 1173.4ms/step | 17.0 samples/s +Step 21910/44000 | Loss: 2.9463 | LR: 2.58e-04 | 1157.1ms/step | 17.3 samples/s +Step 21920/44000 | Loss: 3.0248 | LR: 2.58e-04 | 1131.3ms/step | 17.7 samples/s +Step 21930/44000 | Loss: 2.9805 | LR: 2.58e-04 | 1157.1ms/step | 17.3 samples/s +Step 21940/44000 | Loss: 2.9503 | LR: 2.58e-04 | 1178.9ms/step | 17.0 samples/s +Step 21950/44000 | Loss: 2.9520 | LR: 2.58e-04 | 1186.1ms/step | 16.9 samples/s +Step 21960/44000 | Loss: 2.8922 | LR: 2.58e-04 | 1165.1ms/step | 17.2 samples/s +Step 21970/44000 | Loss: 2.9917 | LR: 2.57e-04 | 1167.7ms/step | 17.1 samples/s +Step 21980/44000 | Loss: 2.9521 | LR: 2.57e-04 | 1112.2ms/step | 18.0 samples/s +Step 21990/44000 | Loss: 2.9509 | LR: 2.57e-04 | 1154.8ms/step | 17.3 samples/s +Step 22000/44000 | Loss: 2.9988 | LR: 2.57e-04 | 1136.6ms/step | 17.6 samples/s +Validation at step 22000: Loss = 2.9696, Perplexity = 19.48 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-22000 +Step 22010/44000 | Loss: 2.9866 | LR: 2.57e-04 | 2106.7ms/step | 9.5 samples/s +Step 22020/44000 | Loss: 2.9647 | LR: 2.56e-04 | 1167.6ms/step | 17.1 samples/s +Step 22030/44000 | Loss: 2.9295 | LR: 2.56e-04 | 1141.7ms/step | 17.5 samples/s +Step 22040/44000 | Loss: 2.9150 | LR: 2.56e-04 | 1173.2ms/step | 17.0 samples/s +Step 22050/44000 | Loss: 2.9058 | LR: 2.56e-04 | 1136.0ms/step | 17.6 samples/s +Step 22060/44000 | Loss: 2.9279 | LR: 2.56e-04 | 1141.7ms/step | 17.5 samples/s +Step 22070/44000 | Loss: 2.9074 | LR: 2.56e-04 | 1143.6ms/step | 17.5 samples/s +Step 22080/44000 | Loss: 2.9364 | LR: 2.55e-04 | 1150.4ms/step | 17.4 samples/s +Step 22090/44000 | Loss: 2.9463 | LR: 2.55e-04 | 1136.6ms/step | 17.6 samples/s +Step 22100/44000 | Loss: 2.9599 | LR: 2.55e-04 | 1167.6ms/step | 17.1 samples/s +Step 22110/44000 | Loss: 2.9844 | LR: 2.55e-04 | 1157.1ms/step | 17.3 samples/s +Step 22120/44000 | Loss: 2.9696 | LR: 2.55e-04 | 1209.6ms/step | 16.5 samples/s +Step 22130/44000 | Loss: 3.0009 | LR: 2.54e-04 | 1102.5ms/step | 18.1 samples/s +Step 22140/44000 | Loss: 2.9798 | LR: 2.54e-04 | 1146.8ms/step | 17.4 samples/s +Step 22150/44000 | Loss: 2.9627 | LR: 2.54e-04 | 1133.6ms/step | 17.6 samples/s +Step 22160/44000 | Loss: 2.9719 | LR: 2.54e-04 | 1124.5ms/step | 17.8 samples/s +Step 22170/44000 | Loss: 2.8989 | LR: 2.54e-04 | 1184.4ms/step | 16.9 samples/s +Step 22180/44000 | Loss: 2.9575 | LR: 2.54e-04 | 1146.7ms/step | 17.4 samples/s +Step 22190/44000 | Loss: 3.0363 | LR: 2.53e-04 | 1162.6ms/step | 17.2 samples/s +Step 22200/44000 | Loss: 2.8856 | LR: 2.53e-04 | 1141.8ms/step | 17.5 samples/s +Step 22210/44000 | Loss: 2.9394 | LR: 2.53e-04 | 1144.8ms/step | 17.5 samples/s +Step 22220/44000 | Loss: 2.9990 | LR: 2.53e-04 | 1136.3ms/step | 17.6 samples/s +Step 22230/44000 | Loss: 2.9445 | LR: 2.53e-04 | 1153.0ms/step | 17.3 samples/s +Step 22240/44000 | Loss: 2.9775 | LR: 2.52e-04 | 1122.3ms/step | 17.8 samples/s +Step 22250/44000 | Loss: 3.0111 | LR: 2.52e-04 | 1152.0ms/step | 17.4 samples/s +Validation at step 22250: Loss = 2.9681, Perplexity = 19.45 +Step 22260/44000 | Loss: 2.9256 | LR: 2.52e-04 | 1546.7ms/step | 12.9 samples/s +Step 22270/44000 | Loss: 2.9776 | LR: 2.52e-04 | 1142.6ms/step | 17.5 samples/s +Step 22280/44000 | Loss: 2.9597 | LR: 2.52e-04 | 1142.0ms/step | 17.5 samples/s +Step 22290/44000 | Loss: 2.9715 | LR: 2.52e-04 | 1157.6ms/step | 17.3 samples/s +Step 22300/44000 | Loss: 2.9225 | LR: 2.51e-04 | 1177.9ms/step | 17.0 samples/s +Step 22310/44000 | Loss: 2.9306 | LR: 2.51e-04 | 1147.3ms/step | 17.4 samples/s +Step 22320/44000 | Loss: 2.9308 | LR: 2.51e-04 | 1193.5ms/step | 16.8 samples/s +Step 22330/44000 | Loss: 2.9423 | LR: 2.51e-04 | 1120.9ms/step | 17.8 samples/s +Step 22340/44000 | Loss: 2.9557 | LR: 2.51e-04 | 1165.1ms/step | 17.2 samples/s +Step 22350/44000 | Loss: 3.0042 | LR: 2.50e-04 | 1154.3ms/step | 17.3 samples/s +Step 22360/44000 | Loss: 2.9448 | LR: 2.50e-04 | 1184.6ms/step | 16.9 samples/s +Step 22370/44000 | Loss: 2.9137 | LR: 2.50e-04 | 1193.9ms/step | 16.8 samples/s +Step 22380/44000 | Loss: 2.9096 | LR: 2.50e-04 | 1184.0ms/step | 16.9 samples/s +Step 22390/44000 | Loss: 2.9781 | LR: 2.50e-04 | 1182.0ms/step | 16.9 samples/s +Step 22400/44000 | Loss: 2.9703 | LR: 2.50e-04 | 1157.2ms/step | 17.3 samples/s +Step 22410/44000 | Loss: 2.9804 | LR: 2.49e-04 | 1167.7ms/step | 17.1 samples/s +Step 22420/44000 | Loss: 2.9621 | LR: 2.49e-04 | 1152.4ms/step | 17.4 samples/s +Step 22430/44000 | Loss: 2.9396 | LR: 2.49e-04 | 1178.5ms/step | 17.0 samples/s +Step 22440/44000 | Loss: 2.9974 | LR: 2.49e-04 | 1177.8ms/step | 17.0 samples/s +Step 22450/44000 | Loss: 2.8564 | LR: 2.49e-04 | 1194.6ms/step | 16.7 samples/s +Step 22460/44000 | Loss: 2.9740 | LR: 2.48e-04 | 1176.9ms/step | 17.0 samples/s +Step 22470/44000 | Loss: 2.8837 | LR: 2.48e-04 | 1145.0ms/step | 17.5 samples/s +Step 22480/44000 | Loss: 2.8891 | LR: 2.48e-04 | 1175.1ms/step | 17.0 samples/s +Step 22490/44000 | Loss: 2.9158 | LR: 2.48e-04 | 1136.8ms/step | 17.6 samples/s +Step 22500/44000 | Loss: 3.0007 | LR: 2.48e-04 | 1202.3ms/step | 16.6 samples/s +Validation at step 22500: Loss = 2.9646, Perplexity = 19.39 +Step 22510/44000 | Loss: 2.9679 | LR: 2.48e-04 | 1534.4ms/step | 13.0 samples/s +Step 22520/44000 | Loss: 2.9239 | LR: 2.47e-04 | 1198.6ms/step | 16.7 samples/s +Step 22530/44000 | Loss: 2.9125 | LR: 2.47e-04 | 1194.1ms/step | 16.7 samples/s +Step 22540/44000 | Loss: 3.0000 | LR: 2.47e-04 | 1146.5ms/step | 17.4 samples/s +Step 22550/44000 | Loss: 2.9614 | LR: 2.47e-04 | 1162.2ms/step | 17.2 samples/s +Step 22560/44000 | Loss: 2.9218 | LR: 2.47e-04 | 1151.0ms/step | 17.4 samples/s +Step 22570/44000 | Loss: 2.9306 | LR: 2.46e-04 | 1147.5ms/step | 17.4 samples/s +Step 22580/44000 | Loss: 3.0005 | LR: 2.46e-04 | 1089.6ms/step | 18.4 samples/s +Step 22590/44000 | Loss: 2.9767 | LR: 2.46e-04 | 1126.8ms/step | 17.7 samples/s +Step 22600/44000 | Loss: 2.9246 | LR: 2.46e-04 | 1188.0ms/step | 16.8 samples/s +Step 22610/44000 | Loss: 2.8970 | LR: 2.46e-04 | 1170.5ms/step | 17.1 samples/s +Step 22620/44000 | Loss: 3.0036 | LR: 2.46e-04 | 1165.6ms/step | 17.2 samples/s +Step 22630/44000 | Loss: 2.9037 | LR: 2.45e-04 | 1188.6ms/step | 16.8 samples/s +Step 22640/44000 | Loss: 2.9858 | LR: 2.45e-04 | 1188.7ms/step | 16.8 samples/s +Step 22650/44000 | Loss: 2.8669 | LR: 2.45e-04 | 1183.0ms/step | 16.9 samples/s +Step 22660/44000 | Loss: 2.9296 | LR: 2.45e-04 | 1146.5ms/step | 17.4 samples/s +Step 22670/44000 | Loss: 2.9382 | LR: 2.45e-04 | 1194.8ms/step | 16.7 samples/s +Step 22680/44000 | Loss: 3.0045 | LR: 2.44e-04 | 1199.4ms/step | 16.7 samples/s +Step 22690/44000 | Loss: 2.9545 | LR: 2.44e-04 | 1173.0ms/step | 17.1 samples/s +Step 22700/44000 | Loss: 2.9431 | LR: 2.44e-04 | 1181.9ms/step | 16.9 samples/s +Step 22710/44000 | Loss: 2.9616 | LR: 2.44e-04 | 1148.6ms/step | 17.4 samples/s +Step 22720/44000 | Loss: 2.9847 | LR: 2.44e-04 | 1166.4ms/step | 17.1 samples/s +Step 22730/44000 | Loss: 2.9011 | LR: 2.44e-04 | 1139.7ms/step | 17.5 samples/s +Step 22740/44000 | Loss: 2.9446 | LR: 2.43e-04 | 1181.6ms/step | 16.9 samples/s +Step 22750/44000 | Loss: 2.9658 | LR: 2.43e-04 | 1173.5ms/step | 17.0 samples/s +Validation at step 22750: Loss = 2.9618, Perplexity = 19.33 +Step 22760/44000 | Loss: 2.9593 | LR: 2.43e-04 | 1585.9ms/step | 12.6 samples/s +Step 22770/44000 | Loss: 2.9473 | LR: 2.43e-04 | 1182.9ms/step | 16.9 samples/s +Step 22780/44000 | Loss: 2.9873 | LR: 2.43e-04 | 1168.0ms/step | 17.1 samples/s +Step 22790/44000 | Loss: 2.9488 | LR: 2.42e-04 | 1136.7ms/step | 17.6 samples/s +Step 22800/44000 | Loss: 2.9256 | LR: 2.42e-04 | 1152.2ms/step | 17.4 samples/s +Step 22810/44000 | Loss: 2.9211 | LR: 2.42e-04 | 1172.7ms/step | 17.1 samples/s +Step 22820/44000 | Loss: 2.8823 | LR: 2.42e-04 | 1199.4ms/step | 16.7 samples/s +Step 22830/44000 | Loss: 2.9297 | LR: 2.42e-04 | 1162.8ms/step | 17.2 samples/s +Step 22840/44000 | Loss: 2.9533 | LR: 2.42e-04 | 1199.4ms/step | 16.7 samples/s +Step 22850/44000 | Loss: 2.9713 | LR: 2.41e-04 | 1225.6ms/step | 16.3 samples/s +Step 22860/44000 | Loss: 2.9879 | LR: 2.41e-04 | 1245.8ms/step | 16.1 samples/s +Step 22870/44000 | Loss: 2.8957 | LR: 2.41e-04 | 1177.9ms/step | 17.0 samples/s +Step 22880/44000 | Loss: 2.9933 | LR: 2.41e-04 | 1105.5ms/step | 18.1 samples/s +Step 22890/44000 | Loss: 2.8498 | LR: 2.41e-04 | 1103.6ms/step | 18.1 samples/s +Step 22900/44000 | Loss: 2.9454 | LR: 2.40e-04 | 1133.9ms/step | 17.6 samples/s +Step 22910/44000 | Loss: 2.9453 | LR: 2.40e-04 | 1084.4ms/step | 18.4 samples/s +Step 22920/44000 | Loss: 2.8797 | LR: 2.40e-04 | 1053.0ms/step | 19.0 samples/s +Step 22930/44000 | Loss: 2.9951 | LR: 2.40e-04 | 1084.4ms/step | 18.4 samples/s +Step 22940/44000 | Loss: 2.9348 | LR: 2.40e-04 | 1077.6ms/step | 18.6 samples/s +Step 22950/44000 | Loss: 2.8864 | LR: 2.40e-04 | 1081.3ms/step | 18.5 samples/s +Step 22960/44000 | Loss: 2.9394 | LR: 2.39e-04 | 1092.9ms/step | 18.3 samples/s +Step 22970/44000 | Loss: 2.9269 | LR: 2.39e-04 | 1102.5ms/step | 18.1 samples/s +Step 22980/44000 | Loss: 2.9646 | LR: 2.39e-04 | 1084.6ms/step | 18.4 samples/s +Step 22990/44000 | Loss: 2.9528 | LR: 2.39e-04 | 1105.1ms/step | 18.1 samples/s +Step 23000/44000 | Loss: 2.9259 | LR: 2.39e-04 | 1062.9ms/step | 18.8 samples/s +Validation at step 23000: Loss = 2.9601, Perplexity = 19.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-23000 +Step 23010/44000 | Loss: 2.9516 | LR: 2.38e-04 | 1858.4ms/step | 10.8 samples/s +Step 23020/44000 | Loss: 2.9005 | LR: 2.38e-04 | 1086.6ms/step | 18.4 samples/s +Step 23030/44000 | Loss: 2.9655 | LR: 2.38e-04 | 1087.0ms/step | 18.4 samples/s +Step 23040/44000 | Loss: 2.9508 | LR: 2.38e-04 | 1086.5ms/step | 18.4 samples/s +Step 23050/44000 | Loss: 2.9175 | LR: 2.38e-04 | 1058.7ms/step | 18.9 samples/s +Step 23060/44000 | Loss: 2.8633 | LR: 2.38e-04 | 1062.5ms/step | 18.8 samples/s +Step 23070/44000 | Loss: 2.9784 | LR: 2.37e-04 | 1079.9ms/step | 18.5 samples/s +Step 23080/44000 | Loss: 2.9545 | LR: 2.37e-04 | 1125.3ms/step | 17.8 samples/s +Step 23090/44000 | Loss: 2.9583 | LR: 2.37e-04 | 1069.8ms/step | 18.7 samples/s +Step 23100/44000 | Loss: 2.9023 | LR: 2.37e-04 | 1063.3ms/step | 18.8 samples/s +Step 23110/44000 | Loss: 2.9089 | LR: 2.37e-04 | 1115.2ms/step | 17.9 samples/s +Step 23120/44000 | Loss: 2.9009 | LR: 2.36e-04 | 1078.9ms/step | 18.5 samples/s +Step 23130/44000 | Loss: 2.9381 | LR: 2.36e-04 | 1083.7ms/step | 18.5 samples/s +Step 23140/44000 | Loss: 2.9039 | LR: 2.36e-04 | 1078.0ms/step | 18.6 samples/s +Step 23150/44000 | Loss: 2.9190 | LR: 2.36e-04 | 1043.7ms/step | 19.2 samples/s +Step 23160/44000 | Loss: 2.9862 | LR: 2.36e-04 | 1068.7ms/step | 18.7 samples/s +Step 23170/44000 | Loss: 2.9288 | LR: 2.36e-04 | 1080.7ms/step | 18.5 samples/s +Step 23180/44000 | Loss: 2.9502 | LR: 2.35e-04 | 1105.5ms/step | 18.1 samples/s +Step 23190/44000 | Loss: 2.9676 | LR: 2.35e-04 | 1102.4ms/step | 18.1 samples/s +Step 23200/44000 | Loss: 2.8909 | LR: 2.35e-04 | 1052.3ms/step | 19.0 samples/s +Step 23210/44000 | Loss: 2.9691 | LR: 2.35e-04 | 1037.5ms/step | 19.3 samples/s +Step 23220/44000 | Loss: 2.8905 | LR: 2.35e-04 | 1125.7ms/step | 17.8 samples/s +Step 23230/44000 | Loss: 2.9829 | LR: 2.34e-04 | 1094.6ms/step | 18.3 samples/s +Step 23240/44000 | Loss: 2.9742 | LR: 2.34e-04 | 1056.3ms/step | 18.9 samples/s +Step 23250/44000 | Loss: 2.8740 | LR: 2.34e-04 | 1049.5ms/step | 19.1 samples/s +Validation at step 23250: Loss = 2.9574, Perplexity = 19.25 +Step 23260/44000 | Loss: 2.9637 | LR: 2.34e-04 | 1439.3ms/step | 13.9 samples/s +Step 23270/44000 | Loss: 2.9674 | LR: 2.34e-04 | 1052.7ms/step | 19.0 samples/s +Step 23280/44000 | Loss: 2.9737 | LR: 2.34e-04 | 1089.4ms/step | 18.4 samples/s +Step 23290/44000 | Loss: 2.9093 | LR: 2.33e-04 | 1074.1ms/step | 18.6 samples/s +Step 23300/44000 | Loss: 2.9360 | LR: 2.33e-04 | 1084.2ms/step | 18.4 samples/s +Step 23310/44000 | Loss: 2.8920 | LR: 2.33e-04 | 1052.8ms/step | 19.0 samples/s +Step 23320/44000 | Loss: 2.9019 | LR: 2.33e-04 | 1075.4ms/step | 18.6 samples/s +Step 23330/44000 | Loss: 2.9279 | LR: 2.33e-04 | 1041.3ms/step | 19.2 samples/s +Step 23340/44000 | Loss: 2.9989 | LR: 2.32e-04 | 1089.3ms/step | 18.4 samples/s +Step 23350/44000 | Loss: 2.9289 | LR: 2.32e-04 | 1042.6ms/step | 19.2 samples/s +Step 23360/44000 | Loss: 2.9451 | LR: 2.32e-04 | 1031.4ms/step | 19.4 samples/s +Step 23370/44000 | Loss: 3.0076 | LR: 2.32e-04 | 1064.4ms/step | 18.8 samples/s +Step 23380/44000 | Loss: 2.9308 | LR: 2.32e-04 | 1073.0ms/step | 18.6 samples/s +Step 23390/44000 | Loss: 2.9609 | LR: 2.32e-04 | 1079.7ms/step | 18.5 samples/s +Step 23400/44000 | Loss: 2.9292 | LR: 2.31e-04 | 1073.9ms/step | 18.6 samples/s +Step 23410/44000 | Loss: 2.8755 | LR: 2.31e-04 | 1066.2ms/step | 18.8 samples/s +Step 23420/44000 | Loss: 2.9148 | LR: 2.31e-04 | 1070.9ms/step | 18.7 samples/s +Step 23430/44000 | Loss: 2.9108 | LR: 2.31e-04 | 1094.7ms/step | 18.3 samples/s +Step 23440/44000 | Loss: 2.9039 | LR: 2.31e-04 | 1076.9ms/step | 18.6 samples/s +Step 23450/44000 | Loss: 2.9464 | LR: 2.30e-04 | 1066.1ms/step | 18.8 samples/s +Step 23460/44000 | Loss: 3.0104 | LR: 2.30e-04 | 1058.3ms/step | 18.9 samples/s +Step 23470/44000 | Loss: 2.9770 | LR: 2.30e-04 | 1066.3ms/step | 18.8 samples/s +Step 23480/44000 | Loss: 2.9447 | LR: 2.30e-04 | 1237.8ms/step | 16.2 samples/s +Step 23490/44000 | Loss: 2.9830 | LR: 2.30e-04 | 1240.9ms/step | 16.1 samples/s +Step 23500/44000 | Loss: 2.9699 | LR: 2.30e-04 | 1245.8ms/step | 16.1 samples/s +Validation at step 23500: Loss = 2.9570, Perplexity = 19.24 +Step 23510/44000 | Loss: 2.9754 | LR: 2.29e-04 | 1505.8ms/step | 13.3 samples/s +Step 23520/44000 | Loss: 2.9171 | LR: 2.29e-04 | 1092.9ms/step | 18.3 samples/s +Step 23530/44000 | Loss: 2.9292 | LR: 2.29e-04 | 1057.8ms/step | 18.9 samples/s +Step 23540/44000 | Loss: 2.9684 | LR: 2.29e-04 | 1043.6ms/step | 19.2 samples/s +Step 23550/44000 | Loss: 2.9397 | LR: 2.29e-04 | 1111.4ms/step | 18.0 samples/s +Step 23560/44000 | Loss: 2.8237 | LR: 2.29e-04 | 1094.9ms/step | 18.3 samples/s +Step 23570/44000 | Loss: 2.8614 | LR: 2.28e-04 | 1050.1ms/step | 19.0 samples/s +Step 23580/44000 | Loss: 2.9958 | LR: 2.28e-04 | 1051.5ms/step | 19.0 samples/s +Step 23590/44000 | Loss: 2.9467 | LR: 2.28e-04 | 1084.2ms/step | 18.4 samples/s +Step 23600/44000 | Loss: 2.9369 | LR: 2.28e-04 | 1084.0ms/step | 18.4 samples/s +Step 23610/44000 | Loss: 2.9368 | LR: 2.28e-04 | 1037.9ms/step | 19.3 samples/s +Step 23620/44000 | Loss: 2.9025 | LR: 2.27e-04 | 1041.0ms/step | 19.2 samples/s +Step 23630/44000 | Loss: 2.9321 | LR: 2.27e-04 | 1058.5ms/step | 18.9 samples/s +Step 23640/44000 | Loss: 2.9696 | LR: 2.27e-04 | 1048.1ms/step | 19.1 samples/s +Step 23650/44000 | Loss: 2.9220 | LR: 2.27e-04 | 1079.0ms/step | 18.5 samples/s +Step 23660/44000 | Loss: 2.9074 | LR: 2.27e-04 | 1052.2ms/step | 19.0 samples/s +Step 23670/44000 | Loss: 2.9496 | LR: 2.27e-04 | 1068.9ms/step | 18.7 samples/s +Step 23680/44000 | Loss: 2.9203 | LR: 2.26e-04 | 1070.5ms/step | 18.7 samples/s +Step 23690/44000 | Loss: 3.0036 | LR: 2.26e-04 | 1087.4ms/step | 18.4 samples/s +Step 23700/44000 | Loss: 2.9227 | LR: 2.26e-04 | 1089.3ms/step | 18.4 samples/s +Step 23710/44000 | Loss: 2.9866 | LR: 2.26e-04 | 1089.4ms/step | 18.4 samples/s +Step 23720/44000 | Loss: 2.9850 | LR: 2.26e-04 | 1073.5ms/step | 18.6 samples/s +Step 23730/44000 | Loss: 3.0267 | LR: 2.25e-04 | 1042.9ms/step | 19.2 samples/s +Step 23740/44000 | Loss: 2.9256 | LR: 2.25e-04 | 1034.9ms/step | 19.3 samples/s +Step 23750/44000 | Loss: 2.9525 | LR: 2.25e-04 | 1055.2ms/step | 19.0 samples/s +Validation at step 23750: Loss = 2.9533, Perplexity = 19.17 +Step 23760/44000 | Loss: 2.9169 | LR: 2.25e-04 | 1438.7ms/step | 13.9 samples/s +Step 23770/44000 | Loss: 2.9356 | LR: 2.25e-04 | 1079.2ms/step | 18.5 samples/s +Step 23780/44000 | Loss: 2.9950 | LR: 2.25e-04 | 1045.5ms/step | 19.1 samples/s +Step 23790/44000 | Loss: 2.9068 | LR: 2.24e-04 | 1043.7ms/step | 19.2 samples/s +Step 23800/44000 | Loss: 2.9050 | LR: 2.24e-04 | 1092.4ms/step | 18.3 samples/s +Step 23810/44000 | Loss: 2.9978 | LR: 2.24e-04 | 1017.2ms/step | 19.7 samples/s +Step 23820/44000 | Loss: 2.9409 | LR: 2.24e-04 | 1061.3ms/step | 18.8 samples/s +Step 23830/44000 | Loss: 3.0347 | LR: 2.24e-04 | 1060.5ms/step | 18.9 samples/s +Step 23840/44000 | Loss: 2.9688 | LR: 2.23e-04 | 1053.4ms/step | 19.0 samples/s +Step 23850/44000 | Loss: 2.9412 | LR: 2.23e-04 | 1032.4ms/step | 19.4 samples/s +Step 23860/44000 | Loss: 2.8949 | LR: 2.23e-04 | 1069.9ms/step | 18.7 samples/s +Step 23870/44000 | Loss: 2.9408 | LR: 2.23e-04 | 1018.4ms/step | 19.6 samples/s +Step 23880/44000 | Loss: 2.9530 | LR: 2.23e-04 | 1029.6ms/step | 19.4 samples/s +Step 23890/44000 | Loss: 2.9969 | LR: 2.23e-04 | 1058.2ms/step | 18.9 samples/s +Step 23900/44000 | Loss: 2.9556 | LR: 2.22e-04 | 1058.2ms/step | 18.9 samples/s +Step 23910/44000 | Loss: 2.9522 | LR: 2.22e-04 | 1099.9ms/step | 18.2 samples/s +Step 23920/44000 | Loss: 2.9396 | LR: 2.22e-04 | 1058.4ms/step | 18.9 samples/s +Step 23930/44000 | Loss: 2.9075 | LR: 2.22e-04 | 1032.2ms/step | 19.4 samples/s +Step 23940/44000 | Loss: 2.9372 | LR: 2.22e-04 | 1058.3ms/step | 18.9 samples/s +Step 23950/44000 | Loss: 2.9179 | LR: 2.21e-04 | 1095.1ms/step | 18.3 samples/s +Step 23960/44000 | Loss: 2.9892 | LR: 2.21e-04 | 1071.8ms/step | 18.7 samples/s +Step 23970/44000 | Loss: 2.9137 | LR: 2.21e-04 | 1070.5ms/step | 18.7 samples/s +Step 23980/44000 | Loss: 2.9037 | LR: 2.21e-04 | 1067.6ms/step | 18.7 samples/s +Step 23990/44000 | Loss: 2.9422 | LR: 2.21e-04 | 1075.4ms/step | 18.6 samples/s +Step 24000/44000 | Loss: 2.8667 | LR: 2.21e-04 | 1062.6ms/step | 18.8 samples/s +Validation at step 24000: Loss = 2.9525, Perplexity = 19.15 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-24000 +Step 24010/44000 | Loss: 2.9652 | LR: 2.20e-04 | 1960.8ms/step | 10.2 samples/s +Step 24020/44000 | Loss: 2.9488 | LR: 2.20e-04 | 1097.2ms/step | 18.2 samples/s +Step 24030/44000 | Loss: 2.9699 | LR: 2.20e-04 | 1071.2ms/step | 18.7 samples/s +Step 24040/44000 | Loss: 2.9076 | LR: 2.20e-04 | 1082.6ms/step | 18.5 samples/s +Step 24050/44000 | Loss: 2.9420 | LR: 2.20e-04 | 1099.6ms/step | 18.2 samples/s +Step 24060/44000 | Loss: 2.9105 | LR: 2.19e-04 | 1016.3ms/step | 19.7 samples/s +Step 24070/44000 | Loss: 2.9927 | LR: 2.19e-04 | 1079.4ms/step | 18.5 samples/s +Step 24080/44000 | Loss: 2.9077 | LR: 2.19e-04 | 1052.5ms/step | 19.0 samples/s +Step 24090/44000 | Loss: 2.9043 | LR: 2.19e-04 | 1073.5ms/step | 18.6 samples/s +Step 24100/44000 | Loss: 2.8984 | LR: 2.19e-04 | 1053.2ms/step | 19.0 samples/s +Step 24110/44000 | Loss: 2.9151 | LR: 2.19e-04 | 1130.9ms/step | 17.7 samples/s +Step 24120/44000 | Loss: 2.8891 | LR: 2.18e-04 | 1048.1ms/step | 19.1 samples/s +Step 24130/44000 | Loss: 2.9440 | LR: 2.18e-04 | 1034.4ms/step | 19.3 samples/s +Step 24140/44000 | Loss: 2.9157 | LR: 2.18e-04 | 1071.4ms/step | 18.7 samples/s +Step 24150/44000 | Loss: 2.9656 | LR: 2.18e-04 | 1063.8ms/step | 18.8 samples/s +Step 24160/44000 | Loss: 2.9604 | LR: 2.18e-04 | 1072.6ms/step | 18.6 samples/s +Step 24170/44000 | Loss: 2.8822 | LR: 2.17e-04 | 1043.9ms/step | 19.2 samples/s +Step 24180/44000 | Loss: 2.8396 | LR: 2.17e-04 | 1016.7ms/step | 19.7 samples/s +Step 24190/44000 | Loss: 2.9175 | LR: 2.17e-04 | 1056.8ms/step | 18.9 samples/s +Step 24200/44000 | Loss: 2.9798 | LR: 2.17e-04 | 1057.5ms/step | 18.9 samples/s +Step 24210/44000 | Loss: 2.9626 | LR: 2.17e-04 | 1044.1ms/step | 19.2 samples/s +Step 24220/44000 | Loss: 2.9098 | LR: 2.17e-04 | 1063.6ms/step | 18.8 samples/s +Step 24230/44000 | Loss: 2.8897 | LR: 2.16e-04 | 1063.2ms/step | 18.8 samples/s +Step 24240/44000 | Loss: 2.9190 | LR: 2.16e-04 | 1093.5ms/step | 18.3 samples/s +Step 24250/44000 | Loss: 2.8472 | LR: 2.16e-04 | 1080.6ms/step | 18.5 samples/s +Validation at step 24250: Loss = 2.9500, Perplexity = 19.11 +Step 24260/44000 | Loss: 2.9397 | LR: 2.16e-04 | 1404.1ms/step | 14.2 samples/s +Step 24270/44000 | Loss: 2.8629 | LR: 2.16e-04 | 1061.6ms/step | 18.8 samples/s +Step 24280/44000 | Loss: 2.9532 | LR: 2.16e-04 | 1079.3ms/step | 18.5 samples/s +Step 24290/44000 | Loss: 2.9422 | LR: 2.15e-04 | 1068.5ms/step | 18.7 samples/s +Step 24300/44000 | Loss: 2.9887 | LR: 2.15e-04 | 1052.8ms/step | 19.0 samples/s +Step 24310/44000 | Loss: 2.9373 | LR: 2.15e-04 | 1095.1ms/step | 18.3 samples/s +Step 24320/44000 | Loss: 2.9460 | LR: 2.15e-04 | 1120.6ms/step | 17.8 samples/s +Step 24330/44000 | Loss: 2.9299 | LR: 2.15e-04 | 1120.9ms/step | 17.8 samples/s +Step 24340/44000 | Loss: 2.9133 | LR: 2.14e-04 | 1074.1ms/step | 18.6 samples/s +Step 24350/44000 | Loss: 2.9085 | LR: 2.14e-04 | 1089.2ms/step | 18.4 samples/s +Step 24360/44000 | Loss: 2.9007 | LR: 2.14e-04 | 1073.4ms/step | 18.6 samples/s +Step 24370/44000 | Loss: 2.9159 | LR: 2.14e-04 | 1069.6ms/step | 18.7 samples/s +Step 24380/44000 | Loss: 2.8293 | LR: 2.14e-04 | 1064.4ms/step | 18.8 samples/s +Step 24390/44000 | Loss: 2.9766 | LR: 2.14e-04 | 1056.8ms/step | 18.9 samples/s +Step 24400/44000 | Loss: 2.8930 | LR: 2.13e-04 | 1046.5ms/step | 19.1 samples/s +Step 24410/44000 | Loss: 2.8889 | LR: 2.13e-04 | 1092.0ms/step | 18.3 samples/s +Step 24420/44000 | Loss: 2.9258 | LR: 2.13e-04 | 1053.1ms/step | 19.0 samples/s +Step 24430/44000 | Loss: 2.9155 | LR: 2.13e-04 | 1062.3ms/step | 18.8 samples/s +Step 24440/44000 | Loss: 2.9326 | LR: 2.13e-04 | 1063.5ms/step | 18.8 samples/s +Step 24450/44000 | Loss: 2.9671 | LR: 2.12e-04 | 1079.1ms/step | 18.5 samples/s +Step 24460/44000 | Loss: 2.8748 | LR: 2.12e-04 | 1084.1ms/step | 18.4 samples/s +Step 24470/44000 | Loss: 2.8635 | LR: 2.12e-04 | 1073.3ms/step | 18.6 samples/s +Step 24480/44000 | Loss: 2.9500 | LR: 2.12e-04 | 1053.3ms/step | 19.0 samples/s +Step 24490/44000 | Loss: 2.9109 | LR: 2.12e-04 | 1101.0ms/step | 18.2 samples/s +Step 24500/44000 | Loss: 2.9051 | LR: 2.12e-04 | 1094.3ms/step | 18.3 samples/s +Validation at step 24500: Loss = 2.9490, Perplexity = 19.09 +Step 24510/44000 | Loss: 2.9440 | LR: 2.11e-04 | 1441.1ms/step | 13.9 samples/s +Step 24520/44000 | Loss: 2.8931 | LR: 2.11e-04 | 1123.3ms/step | 17.8 samples/s +Step 24530/44000 | Loss: 2.8796 | LR: 2.11e-04 | 1089.1ms/step | 18.4 samples/s +Step 24540/44000 | Loss: 2.9121 | LR: 2.11e-04 | 1078.8ms/step | 18.5 samples/s +Step 24550/44000 | Loss: 2.9806 | LR: 2.11e-04 | 1095.1ms/step | 18.3 samples/s +Step 24560/44000 | Loss: 2.9104 | LR: 2.10e-04 | 1048.5ms/step | 19.1 samples/s +Step 24570/44000 | Loss: 2.9123 | LR: 2.10e-04 | 1083.8ms/step | 18.5 samples/s +Step 24580/44000 | Loss: 2.8794 | LR: 2.10e-04 | 1100.4ms/step | 18.2 samples/s +Step 24590/44000 | Loss: 2.9215 | LR: 2.10e-04 | 1100.0ms/step | 18.2 samples/s +Step 24600/44000 | Loss: 2.9242 | LR: 2.10e-04 | 1074.2ms/step | 18.6 samples/s +Step 24610/44000 | Loss: 2.9439 | LR: 2.10e-04 | 1089.3ms/step | 18.4 samples/s +Step 24620/44000 | Loss: 2.9296 | LR: 2.09e-04 | 1063.6ms/step | 18.8 samples/s +Step 24630/44000 | Loss: 2.8716 | LR: 2.09e-04 | 1063.9ms/step | 18.8 samples/s +Step 24640/44000 | Loss: 2.9593 | LR: 2.09e-04 | 1058.3ms/step | 18.9 samples/s +Step 24650/44000 | Loss: 2.9667 | LR: 2.09e-04 | 1042.7ms/step | 19.2 samples/s +Step 24660/44000 | Loss: 2.9910 | LR: 2.09e-04 | 1121.6ms/step | 17.8 samples/s +Step 24670/44000 | Loss: 3.0204 | LR: 2.09e-04 | 1094.1ms/step | 18.3 samples/s +Step 24680/44000 | Loss: 2.8879 | LR: 2.08e-04 | 1068.0ms/step | 18.7 samples/s +Step 24690/44000 | Loss: 2.9608 | LR: 2.08e-04 | 1079.6ms/step | 18.5 samples/s +Step 24700/44000 | Loss: 2.9661 | LR: 2.08e-04 | 1069.8ms/step | 18.7 samples/s +Step 24710/44000 | Loss: 3.0125 | LR: 2.08e-04 | 1078.1ms/step | 18.6 samples/s +Step 24720/44000 | Loss: 2.9681 | LR: 2.08e-04 | 1089.9ms/step | 18.4 samples/s +Step 24730/44000 | Loss: 2.9580 | LR: 2.07e-04 | 1125.5ms/step | 17.8 samples/s +Step 24740/44000 | Loss: 2.9272 | LR: 2.07e-04 | 1111.1ms/step | 18.0 samples/s +Step 24750/44000 | Loss: 2.9615 | LR: 2.07e-04 | 1100.3ms/step | 18.2 samples/s +Validation at step 24750: Loss = 2.9455, Perplexity = 19.02 +Step 24760/44000 | Loss: 2.9329 | LR: 2.07e-04 | 1434.0ms/step | 13.9 samples/s +Step 24770/44000 | Loss: 2.9024 | LR: 2.07e-04 | 1099.8ms/step | 18.2 samples/s +Step 24780/44000 | Loss: 2.9498 | LR: 2.07e-04 | 1074.2ms/step | 18.6 samples/s +Step 24790/44000 | Loss: 2.9043 | LR: 2.06e-04 | 1071.2ms/step | 18.7 samples/s +Step 24800/44000 | Loss: 2.9345 | LR: 2.06e-04 | 1045.1ms/step | 19.1 samples/s +Step 24810/44000 | Loss: 2.8608 | LR: 2.06e-04 | 1054.0ms/step | 19.0 samples/s +Step 24820/44000 | Loss: 2.9028 | LR: 2.06e-04 | 1063.7ms/step | 18.8 samples/s +Step 24830/44000 | Loss: 2.9226 | LR: 2.06e-04 | 1074.0ms/step | 18.6 samples/s +Step 24840/44000 | Loss: 2.8997 | LR: 2.05e-04 | 1070.9ms/step | 18.7 samples/s +Step 24850/44000 | Loss: 2.9342 | LR: 2.05e-04 | 1108.2ms/step | 18.0 samples/s +Step 24860/44000 | Loss: 2.9529 | LR: 2.05e-04 | 1077.1ms/step | 18.6 samples/s +Step 24870/44000 | Loss: 2.8612 | LR: 2.05e-04 | 1066.0ms/step | 18.8 samples/s +Step 24880/44000 | Loss: 2.8643 | LR: 2.05e-04 | 1084.7ms/step | 18.4 samples/s +Step 24890/44000 | Loss: 2.9135 | LR: 2.05e-04 | 1058.3ms/step | 18.9 samples/s +Step 24900/44000 | Loss: 2.8634 | LR: 2.04e-04 | 1079.4ms/step | 18.5 samples/s +Step 24910/44000 | Loss: 2.9606 | LR: 2.04e-04 | 1043.1ms/step | 19.2 samples/s +Step 24920/44000 | Loss: 2.9423 | LR: 2.04e-04 | 1045.2ms/step | 19.1 samples/s +Step 24930/44000 | Loss: 2.9009 | LR: 2.04e-04 | 1065.8ms/step | 18.8 samples/s +Step 24940/44000 | Loss: 2.8589 | LR: 2.04e-04 | 1084.1ms/step | 18.4 samples/s +Step 24950/44000 | Loss: 2.9002 | LR: 2.04e-04 | 1084.5ms/step | 18.4 samples/s +Step 24960/44000 | Loss: 2.9588 | LR: 2.03e-04 | 1074.5ms/step | 18.6 samples/s +Step 24970/44000 | Loss: 2.9911 | LR: 2.03e-04 | 1078.8ms/step | 18.5 samples/s +Step 24980/44000 | Loss: 2.9429 | LR: 2.03e-04 | 1042.7ms/step | 19.2 samples/s +Step 24990/44000 | Loss: 2.9209 | LR: 2.03e-04 | 1075.1ms/step | 18.6 samples/s +Step 25000/44000 | Loss: 2.9365 | LR: 2.03e-04 | 1056.9ms/step | 18.9 samples/s +Validation at step 25000: Loss = 2.9452, Perplexity = 19.01 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-25000 +Step 25010/44000 | Loss: 3.0124 | LR: 2.02e-04 | 1970.9ms/step | 10.1 samples/s +Step 25020/44000 | Loss: 2.9190 | LR: 2.02e-04 | 1042.4ms/step | 19.2 samples/s +Step 25030/44000 | Loss: 2.8891 | LR: 2.02e-04 | 1063.2ms/step | 18.8 samples/s +Step 25040/44000 | Loss: 2.9559 | LR: 2.02e-04 | 1095.0ms/step | 18.3 samples/s +Step 25050/44000 | Loss: 2.9858 | LR: 2.02e-04 | 1078.6ms/step | 18.5 samples/s +Step 25060/44000 | Loss: 2.8626 | LR: 2.02e-04 | 1064.7ms/step | 18.8 samples/s +Step 25070/44000 | Loss: 2.9538 | LR: 2.01e-04 | 1079.6ms/step | 18.5 samples/s +Step 25080/44000 | Loss: 2.9138 | LR: 2.01e-04 | 1046.2ms/step | 19.1 samples/s +Step 25090/44000 | Loss: 2.8868 | LR: 2.01e-04 | 1063.9ms/step | 18.8 samples/s +Step 25100/44000 | Loss: 2.9299 | LR: 2.01e-04 | 1042.2ms/step | 19.2 samples/s +Step 25110/44000 | Loss: 2.9110 | LR: 2.01e-04 | 1088.6ms/step | 18.4 samples/s +Step 25120/44000 | Loss: 2.9482 | LR: 2.00e-04 | 1026.9ms/step | 19.5 samples/s +Step 25130/44000 | Loss: 2.9883 | LR: 2.00e-04 | 1052.7ms/step | 19.0 samples/s +Step 25140/44000 | Loss: 2.9079 | LR: 2.00e-04 | 1106.0ms/step | 18.1 samples/s +Step 25150/44000 | Loss: 2.8661 | LR: 2.00e-04 | 1088.4ms/step | 18.4 samples/s +Step 25160/44000 | Loss: 2.9004 | LR: 2.00e-04 | 1054.0ms/step | 19.0 samples/s +Step 25170/44000 | Loss: 3.0217 | LR: 2.00e-04 | 1031.4ms/step | 19.4 samples/s +Step 25180/44000 | Loss: 2.9315 | LR: 1.99e-04 | 1052.9ms/step | 19.0 samples/s +Step 25190/44000 | Loss: 2.9591 | LR: 1.99e-04 | 1068.3ms/step | 18.7 samples/s +Step 25200/44000 | Loss: 2.9476 | LR: 1.99e-04 | 1062.8ms/step | 18.8 samples/s +Step 25210/44000 | Loss: 2.9135 | LR: 1.99e-04 | 1084.7ms/step | 18.4 samples/s +Step 25220/44000 | Loss: 2.9767 | LR: 1.99e-04 | 1000.8ms/step | 20.0 samples/s +Step 25230/44000 | Loss: 2.9111 | LR: 1.99e-04 | 1054.4ms/step | 19.0 samples/s +Step 25240/44000 | Loss: 2.9341 | LR: 1.98e-04 | 1115.7ms/step | 17.9 samples/s +Step 25250/44000 | Loss: 2.9137 | LR: 1.98e-04 | 1061.5ms/step | 18.8 samples/s +Validation at step 25250: Loss = 2.9437, Perplexity = 18.99 +Step 25260/44000 | Loss: 2.8661 | LR: 1.98e-04 | 1448.8ms/step | 13.8 samples/s +Step 25270/44000 | Loss: 3.0118 | LR: 1.98e-04 | 1107.7ms/step | 18.1 samples/s +Step 25280/44000 | Loss: 2.9061 | LR: 1.98e-04 | 1029.1ms/step | 19.4 samples/s +Step 25290/44000 | Loss: 2.9687 | LR: 1.97e-04 | 1067.4ms/step | 18.7 samples/s +Step 25300/44000 | Loss: 2.9155 | LR: 1.97e-04 | 1079.6ms/step | 18.5 samples/s +Step 25310/44000 | Loss: 2.8811 | LR: 1.97e-04 | 1083.7ms/step | 18.5 samples/s +Step 25320/44000 | Loss: 2.9015 | LR: 1.97e-04 | 1062.9ms/step | 18.8 samples/s +Step 25330/44000 | Loss: 2.8874 | LR: 1.97e-04 | 1047.5ms/step | 19.1 samples/s +Step 25340/44000 | Loss: 2.9199 | LR: 1.97e-04 | 1064.5ms/step | 18.8 samples/s +Step 25350/44000 | Loss: 2.9290 | LR: 1.96e-04 | 1120.5ms/step | 17.8 samples/s +Step 25360/44000 | Loss: 2.8764 | LR: 1.96e-04 | 1063.5ms/step | 18.8 samples/s +Step 25370/44000 | Loss: 2.8824 | LR: 1.96e-04 | 1083.4ms/step | 18.5 samples/s +Step 25380/44000 | Loss: 2.8725 | LR: 1.96e-04 | 1120.4ms/step | 17.9 samples/s +Step 25390/44000 | Loss: 2.8838 | LR: 1.96e-04 | 1054.0ms/step | 19.0 samples/s +Step 25400/44000 | Loss: 3.0795 | LR: 1.96e-04 | 1074.2ms/step | 18.6 samples/s +Step 25410/44000 | Loss: 2.8931 | LR: 1.95e-04 | 1051.8ms/step | 19.0 samples/s +Step 25420/44000 | Loss: 2.9015 | LR: 1.95e-04 | 1084.5ms/step | 18.4 samples/s +Step 25430/44000 | Loss: 2.8648 | LR: 1.95e-04 | 1089.4ms/step | 18.4 samples/s +Step 25440/44000 | Loss: 2.9349 | LR: 1.95e-04 | 1063.8ms/step | 18.8 samples/s +Step 25450/44000 | Loss: 2.9408 | LR: 1.95e-04 | 1084.4ms/step | 18.4 samples/s +Step 25460/44000 | Loss: 2.9156 | LR: 1.94e-04 | 1057.2ms/step | 18.9 samples/s +Step 25470/44000 | Loss: 2.9279 | LR: 1.94e-04 | 1059.2ms/step | 18.9 samples/s +Step 25480/44000 | Loss: 2.9182 | LR: 1.94e-04 | 1058.4ms/step | 18.9 samples/s +Step 25490/44000 | Loss: 2.9261 | LR: 1.94e-04 | 1068.8ms/step | 18.7 samples/s +Step 25500/44000 | Loss: 2.9303 | LR: 1.94e-04 | 1047.9ms/step | 19.1 samples/s +Validation at step 25500: Loss = 2.9420, Perplexity = 18.95 +Step 25510/44000 | Loss: 2.9339 | LR: 1.94e-04 | 1380.8ms/step | 14.5 samples/s +Step 25520/44000 | Loss: 2.9742 | LR: 1.93e-04 | 1027.6ms/step | 19.5 samples/s +Step 25530/44000 | Loss: 2.9289 | LR: 1.93e-04 | 1058.7ms/step | 18.9 samples/s +Step 25540/44000 | Loss: 2.9920 | LR: 1.93e-04 | 1060.4ms/step | 18.9 samples/s +Step 25550/44000 | Loss: 2.9616 | LR: 1.93e-04 | 1067.4ms/step | 18.7 samples/s +Step 25560/44000 | Loss: 2.9464 | LR: 1.93e-04 | 1084.8ms/step | 18.4 samples/s +Step 25570/44000 | Loss: 2.8724 | LR: 1.92e-04 | 1067.6ms/step | 18.7 samples/s +Step 25580/44000 | Loss: 2.9046 | LR: 1.92e-04 | 1090.7ms/step | 18.3 samples/s +Step 25590/44000 | Loss: 2.9892 | LR: 1.92e-04 | 1066.9ms/step | 18.7 samples/s +Step 25600/44000 | Loss: 2.8889 | LR: 1.92e-04 | 1071.4ms/step | 18.7 samples/s +Step 25610/44000 | Loss: 2.9754 | LR: 1.92e-04 | 1089.5ms/step | 18.4 samples/s +Step 25620/44000 | Loss: 2.9215 | LR: 1.92e-04 | 1082.9ms/step | 18.5 samples/s +Step 25630/44000 | Loss: 2.9192 | LR: 1.91e-04 | 1099.0ms/step | 18.2 samples/s +Step 25640/44000 | Loss: 2.9363 | LR: 1.91e-04 | 1077.1ms/step | 18.6 samples/s +Step 25650/44000 | Loss: 2.9570 | LR: 1.91e-04 | 1053.4ms/step | 19.0 samples/s +Step 25660/44000 | Loss: 2.9803 | LR: 1.91e-04 | 1063.7ms/step | 18.8 samples/s +Step 25670/44000 | Loss: 2.9155 | LR: 1.91e-04 | 1087.6ms/step | 18.4 samples/s +Step 25680/44000 | Loss: 2.9133 | LR: 1.91e-04 | 1087.1ms/step | 18.4 samples/s +Step 25690/44000 | Loss: 2.9645 | LR: 1.90e-04 | 1085.7ms/step | 18.4 samples/s +Step 25700/44000 | Loss: 2.8874 | LR: 1.90e-04 | 1068.3ms/step | 18.7 samples/s +Step 25710/44000 | Loss: 2.9110 | LR: 1.90e-04 | 1094.9ms/step | 18.3 samples/s +Step 25720/44000 | Loss: 2.9145 | LR: 1.90e-04 | 1068.6ms/step | 18.7 samples/s +Step 25730/44000 | Loss: 2.8813 | LR: 1.90e-04 | 1100.5ms/step | 18.2 samples/s +Step 25740/44000 | Loss: 2.9065 | LR: 1.89e-04 | 1094.9ms/step | 18.3 samples/s +Step 25750/44000 | Loss: 2.9589 | LR: 1.89e-04 | 1064.1ms/step | 18.8 samples/s +Validation at step 25750: Loss = 2.9401, Perplexity = 18.92 +Step 25760/44000 | Loss: 2.8925 | LR: 1.89e-04 | 1441.8ms/step | 13.9 samples/s +Step 25770/44000 | Loss: 2.9240 | LR: 1.89e-04 | 1107.8ms/step | 18.1 samples/s +Step 25780/44000 | Loss: 2.9125 | LR: 1.89e-04 | 1118.2ms/step | 17.9 samples/s +Step 25790/44000 | Loss: 2.9330 | LR: 1.89e-04 | 1063.5ms/step | 18.8 samples/s +Step 25800/44000 | Loss: 2.9568 | LR: 1.88e-04 | 1057.5ms/step | 18.9 samples/s +Step 25810/44000 | Loss: 2.9183 | LR: 1.88e-04 | 1063.5ms/step | 18.8 samples/s +Step 25820/44000 | Loss: 2.8929 | LR: 1.88e-04 | 1047.8ms/step | 19.1 samples/s +Step 25830/44000 | Loss: 2.9754 | LR: 1.88e-04 | 1115.8ms/step | 17.9 samples/s +Step 25840/44000 | Loss: 2.9222 | LR: 1.88e-04 | 1120.5ms/step | 17.8 samples/s +Step 25850/44000 | Loss: 2.8770 | LR: 1.88e-04 | 1070.2ms/step | 18.7 samples/s +Step 25860/44000 | Loss: 2.9103 | LR: 1.87e-04 | 1078.0ms/step | 18.6 samples/s +Step 25870/44000 | Loss: 2.8880 | LR: 1.87e-04 | 1041.9ms/step | 19.2 samples/s +Step 25880/44000 | Loss: 2.9521 | LR: 1.87e-04 | 1064.0ms/step | 18.8 samples/s +W0821 12:44:59.481000 64391 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers +W0821 12:44:59.501000 64391 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 64459 closing signal SIGTERM +W0821 12:44:59.503000 64391 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 64460 closing signal SIGTERM +W0821 12:44:59.504000 64391 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 64461 closing signal SIGTERM +W0821 12:44:59.517000 64391 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 64462 closing signal SIGTERM +W0821 12:44:59.528000 64391 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 64463 closing signal SIGTERM +Traceback (most recent call last): + File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in + sys.exit(main()) + ~~~~^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper + return f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main + run(args) + ~~~^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run + elastic_launch( + ~~~~~~~~~~~~~~~ + config=config, + ~~~~~~~~~~~~~~ + entrypoint=cmd, + ~~~~~~~~~~~~~~~ + )(*cmd_args) + ~^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent + result = agent.run() + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run + result = self._invoke_run(role) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run + time.sleep(monitor_interval) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 64391 got signal: 15 diff --git a/epfl_final/train_logs/seed48_dense_h100_ddp4/dense.20260821T124000Z.log b/epfl_final/train_logs/seed48_dense_h100_ddp4/dense.20260821T124000Z.log new file mode 100644 index 0000000000000000000000000000000000000000..dadf2d3cbe12e8354b429ace575c61ab236695f6 --- /dev/null +++ b/epfl_final/train_logs/seed48_dense_h100_ddp4/dense.20260821T124000Z.log @@ -0,0 +1,2316 @@ +W0821 12:45:33.514000 154 site-packages/torch/distributed/run.py:851] +W0821 12:45:33.514000 154 site-packages/torch/distributed/run.py:851] ***************************************** +W0821 12:45:33.514000 154 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0821 12:45:33.514000 154 site-packages/torch/distributed/run.py:851] ***************************************** +Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824", + "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-25000", + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 48, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m dense 8k seed48 h100 ddp4 resume25k", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "dense", + "multiseed", + "seed48", + "ddp4", + "h100", + "resume25k" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 1, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": null, + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": null, + "dense_mode": true, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-25000 +Checkpoint activations: FalseCheckpoint activations: FalseCheckpoint activations: False + + +Checkpoint activations: False +dense_mode=True: sparse attention layers will use standard causal SDPA. + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 1 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 8192] -- -- True +├─Qwen3Model: 1-1 -- [1, 8192, 768] -- True +│ └─Embedding: 2-1 [1, 8192] [1, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [1, 8192, 768] [1, 8192, 768] 768 True +├─Linear: 1-2 [1, 8192, 768] [1, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 475.65 +================================================================================================================================================================ +Input size (MB): 0.07 +Forward/backward pass size (MB): 15598.62 +Params size (MB): 951.29 +Estimated Total Size (MB): 16549.97 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) + +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) + +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 1 +World size (GPUs): 4 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-25000... +Preserving checkpoint LR state for exact resume: current=[0.00020261964778734296], schedule_base=[0.0005] +Resumed from checkpoint at step 25000 (epoch 0, micro_step 50000) +Resuming training from step 25000 +Resuming wandb run zrelwtt2 +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: setting up run zrelwtt2 +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/wandb/wandb/run-20260821_124634-zrelwtt2 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run 365m dense 8k seed48 h100 ddp4 resume25k +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/zrelwtt2 +Starting training... +Validation at step 25000: Loss = 2.9324, Perplexity = 18.77 +wandb: WARNING Tried to log to step 25000 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Switching attention mode for step 25001: dense (dense step interval) +Step 25010/44000 | Loss: 2.8851 | LR: 2.02e-04 | 15614.3ms/step | 1.3 samples/s +Step 25020/44000 | Loss: 2.8916 | LR: 2.02e-04 | 400.1ms/step | 50.0 samples/s +wandb: WARNING Tried to log to step 25010 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25020 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25030/44000 | Loss: 2.8882 | LR: 2.02e-04 | 401.0ms/step | 49.9 samples/s +Step 25040/44000 | Loss: 2.9067 | LR: 2.02e-04 | 399.8ms/step | 50.0 samples/s +wandb: WARNING Tried to log to step 25030 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25040 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25050/44000 | Loss: 2.9526 | LR: 2.02e-04 | 400.1ms/step | 50.0 samples/s +Step 25060/44000 | Loss: 2.9119 | LR: 2.02e-04 | 399.8ms/step | 50.0 samples/s +Step 25070/44000 | Loss: 2.9574 | LR: 2.01e-04 | 402.2ms/step | 49.7 samples/s +wandb: WARNING Tried to log to step 25050 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25060 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25070 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25080/44000 | Loss: 2.8537 | LR: 2.01e-04 | 400.0ms/step | 50.0 samples/s +Step 25090/44000 | Loss: 2.8627 | LR: 2.01e-04 | 400.3ms/step | 50.0 samples/s +wandb: WARNING Tried to log to step 25080 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25090 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25100/44000 | Loss: 2.8937 | LR: 2.01e-04 | 400.8ms/step | 49.9 samples/s +Step 25110/44000 | Loss: 2.8802 | LR: 2.01e-04 | 402.3ms/step | 49.7 samples/s +Step 25120/44000 | Loss: 2.8872 | LR: 2.00e-04 | 402.2ms/step | 49.7 samples/s +wandb: WARNING Tried to log to step 25100 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25110 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25120 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25130/44000 | Loss: 2.9163 | LR: 2.00e-04 | 403.6ms/step | 49.6 samples/s +Step 25140/44000 | Loss: 2.9155 | LR: 2.00e-04 | 402.4ms/step | 49.7 samples/s +wandb: WARNING Tried to log to step 25130 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25140 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25150/44000 | Loss: 2.9241 | LR: 2.00e-04 | 406.7ms/step | 49.2 samples/s +Step 25160/44000 | Loss: 2.8955 | LR: 2.00e-04 | 403.8ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25150 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25160 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25170/44000 | Loss: 2.9168 | LR: 2.00e-04 | 401.7ms/step | 49.8 samples/s +Step 25180/44000 | Loss: 2.9036 | LR: 1.99e-04 | 400.2ms/step | 50.0 samples/s +Step 25190/44000 | Loss: 2.9300 | LR: 1.99e-04 | 401.8ms/step | 49.8 samples/s +wandb: WARNING Tried to log to step 25170 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25180 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25190 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25200/44000 | Loss: 2.8626 | LR: 1.99e-04 | 401.1ms/step | 49.9 samples/s +Step 25210/44000 | Loss: 2.9046 | LR: 1.99e-04 | 403.0ms/step | 49.6 samples/s +wandb: WARNING Tried to log to step 25200 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25210 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25220/44000 | Loss: 2.8733 | LR: 1.99e-04 | 402.0ms/step | 49.7 samples/s +Step 25230/44000 | Loss: 2.8195 | LR: 1.99e-04 | 401.9ms/step | 49.8 samples/s +Step 25240/44000 | Loss: 2.8798 | LR: 1.98e-04 | 402.1ms/step | 49.7 samples/s +wandb: WARNING Tried to log to step 25220 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25230 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25240 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25250/44000 | Loss: 2.8637 | LR: 1.98e-04 | 403.5ms/step | 49.6 samples/s +Validation at step 25250: Loss = 2.9307, Perplexity = 18.74 +Step 25260/44000 | Loss: 2.9021 | LR: 1.98e-04 | 564.8ms/step | 35.4 samples/s +wandb: WARNING Tried to log to step 25250 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25260 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25270/44000 | Loss: 2.8904 | LR: 1.98e-04 | 404.2ms/step | 49.5 samples/s +Step 25280/44000 | Loss: 2.9450 | LR: 1.98e-04 | 403.8ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25270 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25280 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25290/44000 | Loss: 2.8906 | LR: 1.97e-04 | 403.6ms/step | 49.5 samples/s +Step 25300/44000 | Loss: 2.8424 | LR: 1.97e-04 | 404.5ms/step | 49.4 samples/s +Step 25310/44000 | Loss: 2.9188 | LR: 1.97e-04 | 403.7ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25290 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25300 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25310 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25320/44000 | Loss: 2.8405 | LR: 1.97e-04 | 404.4ms/step | 49.5 samples/s +Step 25330/44000 | Loss: 2.9479 | LR: 1.97e-04 | 403.9ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25320 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25330 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25340/44000 | Loss: 2.8964 | LR: 1.97e-04 | 403.1ms/step | 49.6 samples/s +Step 25350/44000 | Loss: 2.8282 | LR: 1.96e-04 | 402.7ms/step | 49.7 samples/s +Step 25360/44000 | Loss: 2.9348 | LR: 1.96e-04 | 405.7ms/step | 49.3 samples/s +wandb: WARNING Tried to log to step 25340 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25350 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25360 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25370/44000 | Loss: 2.9145 | LR: 1.96e-04 | 406.0ms/step | 49.3 samples/s +Step 25380/44000 | Loss: 2.8359 | LR: 1.96e-04 | 404.8ms/step | 49.4 samples/s +wandb: WARNING Tried to log to step 25370 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25380 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25390/44000 | Loss: 2.9545 | LR: 1.96e-04 | 402.9ms/step | 49.6 samples/s +Step 25400/44000 | Loss: 2.9116 | LR: 1.96e-04 | 403.5ms/step | 49.6 samples/s +Step 25410/44000 | Loss: 2.8963 | LR: 1.95e-04 | 404.6ms/step | 49.4 samples/s +wandb: WARNING Tried to log to step 25390 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25400 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25410 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25420/44000 | Loss: 2.9343 | LR: 1.95e-04 | 403.3ms/step | 49.6 samples/s +Step 25430/44000 | Loss: 2.8856 | LR: 1.95e-04 | 403.0ms/step | 49.6 samples/s +wandb: WARNING Tried to log to step 25420 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25430 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25440/44000 | Loss: 2.9082 | LR: 1.95e-04 | 402.4ms/step | 49.7 samples/s +Step 25450/44000 | Loss: 2.9006 | LR: 1.95e-04 | 403.2ms/step | 49.6 samples/s +Step 25460/44000 | Loss: 2.9579 | LR: 1.94e-04 | 404.3ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25440 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25450 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25460 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25470/44000 | Loss: 2.8825 | LR: 1.94e-04 | 407.9ms/step | 49.0 samples/s +Step 25480/44000 | Loss: 2.9102 | LR: 1.94e-04 | 407.3ms/step | 49.1 samples/s +wandb: WARNING Tried to log to step 25470 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25480 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25490/44000 | Loss: 2.8771 | LR: 1.94e-04 | 403.9ms/step | 49.5 samples/s +Step 25500/44000 | Loss: 2.8828 | LR: 1.94e-04 | 403.1ms/step | 49.6 samples/s +Validation at step 25500: Loss = 2.9293, Perplexity = 18.71 +wandb: WARNING Tried to log to step 25490 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25500 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25510/44000 | Loss: 2.8820 | LR: 1.94e-04 | 558.2ms/step | 35.8 samples/s +Step 25520/44000 | Loss: 2.9179 | LR: 1.93e-04 | 403.9ms/step | 49.5 samples/s +Step 25530/44000 | Loss: 2.9449 | LR: 1.93e-04 | 403.2ms/step | 49.6 samples/s +wandb: WARNING Tried to log to step 25510 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25520 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25530 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25540/44000 | Loss: 2.9260 | LR: 1.93e-04 | 402.8ms/step | 49.7 samples/s +Step 25550/44000 | Loss: 2.8972 | LR: 1.93e-04 | 403.0ms/step | 49.6 samples/s +wandb: WARNING Tried to log to step 25540 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25550 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25560/44000 | Loss: 2.8899 | LR: 1.93e-04 | 402.6ms/step | 49.7 samples/s +Step 25570/44000 | Loss: 2.9461 | LR: 1.92e-04 | 406.1ms/step | 49.3 samples/s +Step 25580/44000 | Loss: 2.9197 | LR: 1.92e-04 | 408.9ms/step | 48.9 samples/s +wandb: WARNING Tried to log to step 25560 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25570 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25580 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25590/44000 | Loss: 2.9393 | LR: 1.92e-04 | 406.2ms/step | 49.2 samples/s +Step 25600/44000 | Loss: 2.9785 | LR: 1.92e-04 | 404.0ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25590 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25600 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25610/44000 | Loss: 2.8932 | LR: 1.92e-04 | 404.3ms/step | 49.5 samples/s +Step 25620/44000 | Loss: 2.9024 | LR: 1.92e-04 | 405.2ms/step | 49.4 samples/s +Step 25630/44000 | Loss: 2.9079 | LR: 1.91e-04 | 404.4ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25610 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25620 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25630 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25640/44000 | Loss: 2.8558 | LR: 1.91e-04 | 404.0ms/step | 49.5 samples/s +Step 25650/44000 | Loss: 2.8823 | LR: 1.91e-04 | 403.7ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25640 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25650 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25660/44000 | Loss: 2.8821 | LR: 1.91e-04 | 405.2ms/step | 49.4 samples/s +Step 25670/44000 | Loss: 2.9310 | LR: 1.91e-04 | 407.5ms/step | 49.1 samples/s +Step 25680/44000 | Loss: 2.9284 | LR: 1.91e-04 | 405.7ms/step | 49.3 samples/s +wandb: WARNING Tried to log to step 25660 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25670 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25680 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25690/44000 | Loss: 2.8974 | LR: 1.90e-04 | 408.0ms/step | 49.0 samples/s +Step 25700/44000 | Loss: 2.8760 | LR: 1.90e-04 | 404.4ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25690 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25700 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25710/44000 | Loss: 2.9539 | LR: 1.90e-04 | 403.9ms/step | 49.5 samples/s +Step 25720/44000 | Loss: 2.9302 | LR: 1.90e-04 | 405.0ms/step | 49.4 samples/s +Step 25730/44000 | Loss: 2.8558 | LR: 1.90e-04 | 404.1ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25710 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25720 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25730 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25740/44000 | Loss: 2.9280 | LR: 1.89e-04 | 404.0ms/step | 49.5 samples/s +Step 25750/44000 | Loss: 2.8849 | LR: 1.89e-04 | 404.9ms/step | 49.4 samples/s +Validation at step 25750: Loss = 2.9280, Perplexity = 18.69 +wandb: WARNING Tried to log to step 25740 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25750 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25760/44000 | Loss: 2.8928 | LR: 1.89e-04 | 558.2ms/step | 35.8 samples/s +Step 25770/44000 | Loss: 2.8899 | LR: 1.89e-04 | 403.8ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25760 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25770 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25780/44000 | Loss: 2.8508 | LR: 1.89e-04 | 406.2ms/step | 49.2 samples/s +Step 25790/44000 | Loss: 2.8651 | LR: 1.89e-04 | 407.8ms/step | 49.0 samples/s +Step 25800/44000 | Loss: 2.8999 | LR: 1.88e-04 | 407.2ms/step | 49.1 samples/s +wandb: WARNING Tried to log to step 25780 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25790 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25800 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25810/44000 | Loss: 2.9160 | LR: 1.88e-04 | 403.4ms/step | 49.6 samples/s +Step 25820/44000 | Loss: 2.8997 | LR: 1.88e-04 | 404.2ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25810 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25820 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25830/44000 | Loss: 2.8794 | LR: 1.88e-04 | 403.7ms/step | 49.5 samples/s +Step 25840/44000 | Loss: 2.9549 | LR: 1.88e-04 | 404.0ms/step | 49.5 samples/s +Step 25850/44000 | Loss: 2.9262 | LR: 1.88e-04 | 402.7ms/step | 49.7 samples/s +wandb: WARNING Tried to log to step 25830 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25840 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25850 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25860/44000 | Loss: 2.9125 | LR: 1.87e-04 | 404.2ms/step | 49.5 samples/s +Step 25870/44000 | Loss: 2.9230 | LR: 1.87e-04 | 404.4ms/step | 49.5 samples/s +wandb: WARNING Tried to log to step 25860 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 25870 that is less than the current step 25871. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Step 25880/44000 | Loss: 2.9214 | LR: 1.87e-04 | 404.1ms/step | 49.5 samples/s +Step 25890/44000 | Loss: 2.8860 | LR: 1.87e-04 | 405.4ms/step | 49.3 samples/s +Step 25900/44000 | Loss: 2.8479 | LR: 1.87e-04 | 406.1ms/step | 49.3 samples/s +Step 25910/44000 | Loss: 2.9358 | LR: 1.87e-04 | 403.9ms/step | 49.5 samples/s +Step 25920/44000 | Loss: 2.8944 | LR: 1.86e-04 | 403.6ms/step | 49.5 samples/s +Step 25930/44000 | Loss: 2.8835 | LR: 1.86e-04 | 403.7ms/step | 49.5 samples/s +Step 25940/44000 | Loss: 2.8822 | LR: 1.86e-04 | 403.2ms/step | 49.6 samples/s +Step 25950/44000 | Loss: 2.9570 | LR: 1.86e-04 | 403.5ms/step | 49.6 samples/s +Step 25960/44000 | Loss: 2.8862 | LR: 1.86e-04 | 404.2ms/step | 49.5 samples/s +Step 25970/44000 | Loss: 2.8830 | LR: 1.85e-04 | 404.4ms/step | 49.5 samples/s +Step 25980/44000 | Loss: 2.8235 | LR: 1.85e-04 | 404.4ms/step | 49.5 samples/s +Step 25990/44000 | Loss: 2.9142 | LR: 1.85e-04 | 403.3ms/step | 49.6 samples/s +Step 26000/44000 | Loss: 2.9340 | LR: 1.85e-04 | 404.9ms/step | 49.4 samples/s +Validation at step 26000: Loss = 2.9265, Perplexity = 18.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-26000 +Step 26010/44000 | Loss: 2.9848 | LR: 1.85e-04 | 910.7ms/step | 22.0 samples/s +Step 26020/44000 | Loss: 2.8612 | LR: 1.85e-04 | 402.9ms/step | 49.6 samples/s +Step 26030/44000 | Loss: 2.8721 | LR: 1.84e-04 | 404.4ms/step | 49.5 samples/s +Step 26040/44000 | Loss: 2.9433 | LR: 1.84e-04 | 403.7ms/step | 49.5 samples/s +Step 26050/44000 | Loss: 2.9167 | LR: 1.84e-04 | 403.1ms/step | 49.6 samples/s +Step 26060/44000 | Loss: 2.8831 | LR: 1.84e-04 | 404.4ms/step | 49.5 samples/s +Step 26070/44000 | Loss: 2.9386 | LR: 1.84e-04 | 404.5ms/step | 49.4 samples/s +Step 26080/44000 | Loss: 2.8656 | LR: 1.84e-04 | 403.7ms/step | 49.5 samples/s +Step 26090/44000 | Loss: 2.9205 | LR: 1.83e-04 | 404.2ms/step | 49.5 samples/s +Step 26100/44000 | Loss: 2.9326 | LR: 1.83e-04 | 406.2ms/step | 49.2 samples/s +Step 26110/44000 | Loss: 2.9156 | LR: 1.83e-04 | 406.8ms/step | 49.2 samples/s +Step 26120/44000 | Loss: 2.9156 | LR: 1.83e-04 | 404.2ms/step | 49.5 samples/s +Step 26130/44000 | Loss: 2.8771 | LR: 1.83e-04 | 403.8ms/step | 49.5 samples/s +Step 26140/44000 | Loss: 2.8443 | LR: 1.82e-04 | 403.2ms/step | 49.6 samples/s +Step 26150/44000 | Loss: 2.8695 | LR: 1.82e-04 | 404.1ms/step | 49.5 samples/s +Step 26160/44000 | Loss: 2.8889 | LR: 1.82e-04 | 403.3ms/step | 49.6 samples/s +Step 26170/44000 | Loss: 2.9529 | LR: 1.82e-04 | 403.5ms/step | 49.6 samples/s +Step 26180/44000 | Loss: 2.8674 | LR: 1.82e-04 | 404.0ms/step | 49.5 samples/s +Step 26190/44000 | Loss: 2.8703 | LR: 1.82e-04 | 404.2ms/step | 49.5 samples/s +Step 26200/44000 | Loss: 2.8608 | LR: 1.81e-04 | 406.0ms/step | 49.3 samples/s +Step 26210/44000 | Loss: 2.8841 | LR: 1.81e-04 | 407.3ms/step | 49.1 samples/s +Step 26220/44000 | Loss: 2.8729 | LR: 1.81e-04 | 406.2ms/step | 49.2 samples/s +Step 26230/44000 | Loss: 2.8874 | LR: 1.81e-04 | 404.5ms/step | 49.4 samples/s +Step 26240/44000 | Loss: 2.8906 | LR: 1.81e-04 | 404.2ms/step | 49.5 samples/s +Step 26250/44000 | Loss: 2.8657 | LR: 1.81e-04 | 403.5ms/step | 49.6 samples/s +Validation at step 26250: Loss = 2.9247, Perplexity = 18.63 +Step 26260/44000 | Loss: 2.8993 | LR: 1.80e-04 | 557.0ms/step | 35.9 samples/s +Step 26270/44000 | Loss: 2.8904 | LR: 1.80e-04 | 404.4ms/step | 49.5 samples/s +Step 26280/44000 | Loss: 2.8886 | LR: 1.80e-04 | 402.8ms/step | 49.7 samples/s +Step 26290/44000 | Loss: 2.8851 | LR: 1.80e-04 | 404.1ms/step | 49.5 samples/s +Step 26300/44000 | Loss: 2.8944 | LR: 1.80e-04 | 403.6ms/step | 49.5 samples/s +Step 26310/44000 | Loss: 2.8690 | LR: 1.80e-04 | 404.6ms/step | 49.4 samples/s +Step 26320/44000 | Loss: 2.9201 | LR: 1.79e-04 | 407.3ms/step | 49.1 samples/s +Step 26330/44000 | Loss: 2.9013 | LR: 1.79e-04 | 407.4ms/step | 49.1 samples/s +Step 26340/44000 | Loss: 2.8600 | LR: 1.79e-04 | 403.1ms/step | 49.6 samples/s +Step 26350/44000 | Loss: 2.9160 | LR: 1.79e-04 | 403.0ms/step | 49.6 samples/s +Step 26360/44000 | Loss: 2.8448 | LR: 1.79e-04 | 403.9ms/step | 49.5 samples/s +Step 26370/44000 | Loss: 2.8882 | LR: 1.78e-04 | 404.3ms/step | 49.5 samples/s +Step 26380/44000 | Loss: 2.8599 | LR: 1.78e-04 | 403.6ms/step | 49.6 samples/s +Step 26390/44000 | Loss: 2.9685 | LR: 1.78e-04 | 403.9ms/step | 49.5 samples/s +Step 26400/44000 | Loss: 2.8870 | LR: 1.78e-04 | 402.1ms/step | 49.7 samples/s +Step 26410/44000 | Loss: 2.8975 | LR: 1.78e-04 | 404.5ms/step | 49.4 samples/s +Step 26420/44000 | Loss: 2.8783 | LR: 1.78e-04 | 405.9ms/step | 49.3 samples/s +Step 26430/44000 | Loss: 2.8649 | LR: 1.77e-04 | 407.4ms/step | 49.1 samples/s +Step 26440/44000 | Loss: 2.8831 | LR: 1.77e-04 | 405.2ms/step | 49.4 samples/s +Step 26450/44000 | Loss: 2.8784 | LR: 1.77e-04 | 402.7ms/step | 49.7 samples/s +Step 26460/44000 | Loss: 2.9425 | LR: 1.77e-04 | 403.7ms/step | 49.5 samples/s +Step 26470/44000 | Loss: 2.8682 | LR: 1.77e-04 | 403.4ms/step | 49.6 samples/s +Step 26480/44000 | Loss: 2.8696 | LR: 1.77e-04 | 403.3ms/step | 49.6 samples/s +Step 26490/44000 | Loss: 2.8978 | LR: 1.76e-04 | 403.5ms/step | 49.6 samples/s +Step 26500/44000 | Loss: 2.8852 | LR: 1.76e-04 | 403.8ms/step | 49.5 samples/s +Validation at step 26500: Loss = 2.9234, Perplexity = 18.60 +Step 26510/44000 | Loss: 2.8586 | LR: 1.76e-04 | 555.2ms/step | 36.0 samples/s +Step 26520/44000 | Loss: 2.8743 | LR: 1.76e-04 | 405.7ms/step | 49.3 samples/s +Step 26530/44000 | Loss: 2.8914 | LR: 1.76e-04 | 405.5ms/step | 49.3 samples/s +Step 26540/44000 | Loss: 2.8535 | LR: 1.76e-04 | 405.0ms/step | 49.4 samples/s +Step 26550/44000 | Loss: 2.8779 | LR: 1.75e-04 | 402.7ms/step | 49.7 samples/s +Step 26560/44000 | Loss: 2.8750 | LR: 1.75e-04 | 403.5ms/step | 49.6 samples/s +Step 26570/44000 | Loss: 2.9532 | LR: 1.75e-04 | 403.3ms/step | 49.6 samples/s +Step 26580/44000 | Loss: 2.8403 | LR: 1.75e-04 | 404.5ms/step | 49.4 samples/s +Step 26590/44000 | Loss: 2.8543 | LR: 1.75e-04 | 403.0ms/step | 49.6 samples/s +Step 26600/44000 | Loss: 2.9480 | LR: 1.74e-04 | 404.9ms/step | 49.4 samples/s +Step 26610/44000 | Loss: 2.8772 | LR: 1.74e-04 | 404.0ms/step | 49.5 samples/s +Step 26620/44000 | Loss: 2.9150 | LR: 1.74e-04 | 403.4ms/step | 49.6 samples/s +Step 26630/44000 | Loss: 2.9276 | LR: 1.74e-04 | 406.0ms/step | 49.3 samples/s +Step 26640/44000 | Loss: 2.7685 | LR: 1.74e-04 | 406.1ms/step | 49.2 samples/s +Step 26650/44000 | Loss: 2.8741 | LR: 1.74e-04 | 407.7ms/step | 49.1 samples/s +Step 26660/44000 | Loss: 2.9317 | LR: 1.73e-04 | 404.5ms/step | 49.4 samples/s +Step 26670/44000 | Loss: 2.8193 | LR: 1.73e-04 | 404.1ms/step | 49.5 samples/s +Step 26680/44000 | Loss: 2.8798 | LR: 1.73e-04 | 404.5ms/step | 49.4 samples/s +Step 26690/44000 | Loss: 2.9553 | LR: 1.73e-04 | 403.4ms/step | 49.6 samples/s +Step 26700/44000 | Loss: 2.9912 | LR: 1.73e-04 | 404.6ms/step | 49.4 samples/s +Step 26710/44000 | Loss: 2.8513 | LR: 1.73e-04 | 404.0ms/step | 49.5 samples/s +Step 26720/44000 | Loss: 2.8610 | LR: 1.72e-04 | 404.2ms/step | 49.5 samples/s +Step 26730/44000 | Loss: 2.8538 | LR: 1.72e-04 | 403.8ms/step | 49.5 samples/s +Step 26740/44000 | Loss: 2.9046 | LR: 1.72e-04 | 405.7ms/step | 49.3 samples/s +Step 26750/44000 | Loss: 2.9170 | LR: 1.72e-04 | 407.8ms/step | 49.0 samples/s +Validation at step 26750: Loss = 2.9217, Perplexity = 18.57 +Step 26760/44000 | Loss: 2.8262 | LR: 1.72e-04 | 560.7ms/step | 35.7 samples/s +Step 26770/44000 | Loss: 2.8328 | LR: 1.72e-04 | 404.5ms/step | 49.4 samples/s +Step 26780/44000 | Loss: 2.8649 | LR: 1.71e-04 | 406.2ms/step | 49.2 samples/s +Step 26790/44000 | Loss: 2.8948 | LR: 1.71e-04 | 404.2ms/step | 49.5 samples/s +Step 26800/44000 | Loss: 2.9117 | LR: 1.71e-04 | 404.8ms/step | 49.4 samples/s +Step 26810/44000 | Loss: 2.8760 | LR: 1.71e-04 | 403.5ms/step | 49.6 samples/s +Step 26820/44000 | Loss: 2.9024 | LR: 1.71e-04 | 404.5ms/step | 49.4 samples/s +Step 26830/44000 | Loss: 2.8376 | LR: 1.71e-04 | 404.3ms/step | 49.5 samples/s +Step 26840/44000 | Loss: 2.8573 | LR: 1.70e-04 | 404.5ms/step | 49.4 samples/s +Step 26850/44000 | Loss: 2.8825 | LR: 1.70e-04 | 405.8ms/step | 49.3 samples/s +Step 26860/44000 | Loss: 2.9472 | LR: 1.70e-04 | 405.7ms/step | 49.3 samples/s +Step 26870/44000 | Loss: 2.9140 | LR: 1.70e-04 | 405.4ms/step | 49.3 samples/s +Step 26880/44000 | Loss: 2.8843 | LR: 1.70e-04 | 404.3ms/step | 49.5 samples/s +Step 26890/44000 | Loss: 2.9223 | LR: 1.69e-04 | 404.3ms/step | 49.5 samples/s +Step 26900/44000 | Loss: 2.9158 | LR: 1.69e-04 | 404.1ms/step | 49.5 samples/s +Step 26910/44000 | Loss: 2.8709 | LR: 1.69e-04 | 403.4ms/step | 49.6 samples/s +Step 26920/44000 | Loss: 2.9095 | LR: 1.69e-04 | 403.4ms/step | 49.6 samples/s +Step 26930/44000 | Loss: 2.9020 | LR: 1.69e-04 | 404.6ms/step | 49.4 samples/s +Step 26940/44000 | Loss: 2.9236 | LR: 1.69e-04 | 405.6ms/step | 49.3 samples/s +Step 26950/44000 | Loss: 2.8382 | LR: 1.68e-04 | 405.9ms/step | 49.3 samples/s +Step 26960/44000 | Loss: 2.9336 | LR: 1.68e-04 | 406.8ms/step | 49.2 samples/s +Step 26970/44000 | Loss: 2.9533 | LR: 1.68e-04 | 407.2ms/step | 49.1 samples/s +Step 26980/44000 | Loss: 2.8245 | LR: 1.68e-04 | 404.9ms/step | 49.4 samples/s +Step 26990/44000 | Loss: 2.8990 | LR: 1.68e-04 | 405.3ms/step | 49.3 samples/s +Step 27000/44000 | Loss: 2.9254 | LR: 1.68e-04 | 405.4ms/step | 49.3 samples/s +Validation at step 27000: Loss = 2.9205, Perplexity = 18.55 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-27000 +Step 27010/44000 | Loss: 2.9061 | LR: 1.67e-04 | 867.7ms/step | 23.0 samples/s +Step 27020/44000 | Loss: 2.9418 | LR: 1.67e-04 | 403.6ms/step | 49.6 samples/s +Step 27030/44000 | Loss: 2.8659 | LR: 1.67e-04 | 403.1ms/step | 49.6 samples/s +Step 27040/44000 | Loss: 2.9485 | LR: 1.67e-04 | 405.1ms/step | 49.4 samples/s +Step 27050/44000 | Loss: 2.8734 | LR: 1.67e-04 | 404.2ms/step | 49.5 samples/s +Step 27060/44000 | Loss: 2.8864 | LR: 1.67e-04 | 405.1ms/step | 49.4 samples/s +Step 27070/44000 | Loss: 2.8536 | LR: 1.66e-04 | 405.0ms/step | 49.4 samples/s +Step 27080/44000 | Loss: 2.9234 | LR: 1.66e-04 | 404.3ms/step | 49.5 samples/s +Step 27090/44000 | Loss: 2.9000 | LR: 1.66e-04 | 403.4ms/step | 49.6 samples/s +Step 27100/44000 | Loss: 2.8881 | LR: 1.66e-04 | 403.6ms/step | 49.6 samples/s +Step 27110/44000 | Loss: 2.9034 | LR: 1.66e-04 | 404.0ms/step | 49.5 samples/s +Step 27120/44000 | Loss: 2.8755 | LR: 1.66e-04 | 402.4ms/step | 49.7 samples/s +Step 27130/44000 | Loss: 2.8680 | LR: 1.65e-04 | 403.6ms/step | 49.5 samples/s +Step 27140/44000 | Loss: 2.8573 | LR: 1.65e-04 | 403.5ms/step | 49.6 samples/s +Step 27150/44000 | Loss: 2.9112 | LR: 1.65e-04 | 402.9ms/step | 49.6 samples/s +Step 27160/44000 | Loss: 2.8621 | LR: 1.65e-04 | 405.0ms/step | 49.4 samples/s +Step 27170/44000 | Loss: 2.8723 | LR: 1.65e-04 | 406.2ms/step | 49.2 samples/s +Step 27180/44000 | Loss: 2.8780 | LR: 1.65e-04 | 406.1ms/step | 49.2 samples/s +Step 27190/44000 | Loss: 2.8830 | LR: 1.64e-04 | 403.9ms/step | 49.5 samples/s +Step 27200/44000 | Loss: 2.9445 | LR: 1.64e-04 | 404.2ms/step | 49.5 samples/s +Step 27210/44000 | Loss: 2.8868 | LR: 1.64e-04 | 403.6ms/step | 49.6 samples/s +Step 27220/44000 | Loss: 2.9589 | LR: 1.64e-04 | 403.3ms/step | 49.6 samples/s +Step 27230/44000 | Loss: 2.8505 | LR: 1.64e-04 | 402.2ms/step | 49.7 samples/s +Step 27240/44000 | Loss: 2.9057 | LR: 1.63e-04 | 403.3ms/step | 49.6 samples/s +Step 27250/44000 | Loss: 2.8717 | LR: 1.63e-04 | 403.0ms/step | 49.6 samples/s +Validation at step 27250: Loss = 2.9193, Perplexity = 18.53 +Step 27260/44000 | Loss: 2.9108 | LR: 1.63e-04 | 556.1ms/step | 36.0 samples/s +Step 27270/44000 | Loss: 2.9125 | LR: 1.63e-04 | 407.4ms/step | 49.1 samples/s +Step 27280/44000 | Loss: 2.8858 | LR: 1.63e-04 | 412.6ms/step | 48.5 samples/s +Step 27290/44000 | Loss: 2.9030 | LR: 1.63e-04 | 404.1ms/step | 49.5 samples/s +Step 27300/44000 | Loss: 2.8009 | LR: 1.62e-04 | 404.5ms/step | 49.4 samples/s +Step 27310/44000 | Loss: 2.8891 | LR: 1.62e-04 | 404.1ms/step | 49.5 samples/s +Step 27320/44000 | Loss: 2.9039 | LR: 1.62e-04 | 404.3ms/step | 49.5 samples/s +Step 27330/44000 | Loss: 2.9291 | LR: 1.62e-04 | 405.9ms/step | 49.3 samples/s +Step 27340/44000 | Loss: 2.9209 | LR: 1.62e-04 | 405.6ms/step | 49.3 samples/s +Step 27350/44000 | Loss: 2.8706 | LR: 1.62e-04 | 404.5ms/step | 49.4 samples/s +Step 27360/44000 | Loss: 2.8852 | LR: 1.61e-04 | 405.4ms/step | 49.3 samples/s +Step 27370/44000 | Loss: 2.9020 | LR: 1.61e-04 | 404.2ms/step | 49.5 samples/s +Step 27380/44000 | Loss: 2.8435 | LR: 1.61e-04 | 404.6ms/step | 49.4 samples/s +Step 27390/44000 | Loss: 2.8735 | LR: 1.61e-04 | 405.4ms/step | 49.3 samples/s +Step 27400/44000 | Loss: 2.8466 | LR: 1.61e-04 | 403.3ms/step | 49.6 samples/s +Step 27410/44000 | Loss: 2.8740 | LR: 1.61e-04 | 403.4ms/step | 49.6 samples/s +Step 27420/44000 | Loss: 2.8885 | LR: 1.60e-04 | 404.7ms/step | 49.4 samples/s +Step 27430/44000 | Loss: 2.9258 | LR: 1.60e-04 | 404.0ms/step | 49.5 samples/s +Step 27440/44000 | Loss: 2.8926 | LR: 1.60e-04 | 404.5ms/step | 49.4 samples/s +Step 27450/44000 | Loss: 2.8841 | LR: 1.60e-04 | 404.6ms/step | 49.4 samples/s +Step 27460/44000 | Loss: 2.8464 | LR: 1.60e-04 | 405.0ms/step | 49.4 samples/s +Step 27470/44000 | Loss: 2.8877 | LR: 1.60e-04 | 405.2ms/step | 49.4 samples/s +Step 27480/44000 | Loss: 2.8791 | LR: 1.59e-04 | 405.4ms/step | 49.3 samples/s +Step 27490/44000 | Loss: 2.8601 | LR: 1.59e-04 | 405.7ms/step | 49.3 samples/s +Step 27500/44000 | Loss: 2.8942 | LR: 1.59e-04 | 407.4ms/step | 49.1 samples/s +Validation at step 27500: Loss = 2.9185, Perplexity = 18.51 +Step 27510/44000 | Loss: 2.8492 | LR: 1.59e-04 | 555.5ms/step | 36.0 samples/s +Step 27520/44000 | Loss: 2.8426 | LR: 1.59e-04 | 405.4ms/step | 49.3 samples/s +Step 27530/44000 | Loss: 2.9018 | LR: 1.59e-04 | 403.5ms/step | 49.6 samples/s +Step 27540/44000 | Loss: 2.9252 | LR: 1.58e-04 | 403.2ms/step | 49.6 samples/s +Step 27550/44000 | Loss: 2.9191 | LR: 1.58e-04 | 404.9ms/step | 49.4 samples/s +Step 27560/44000 | Loss: 2.8961 | LR: 1.58e-04 | 404.3ms/step | 49.5 samples/s +Step 27570/44000 | Loss: 2.8724 | LR: 1.58e-04 | 404.3ms/step | 49.5 samples/s +Step 27580/44000 | Loss: 2.7861 | LR: 1.58e-04 | 408.2ms/step | 49.0 samples/s +Step 27590/44000 | Loss: 2.9105 | LR: 1.58e-04 | 408.0ms/step | 49.0 samples/s +Step 27600/44000 | Loss: 2.8747 | LR: 1.57e-04 | 407.5ms/step | 49.1 samples/s +Step 27610/44000 | Loss: 2.8611 | LR: 1.57e-04 | 413.1ms/step | 48.4 samples/s +Step 27620/44000 | Loss: 2.8586 | LR: 1.57e-04 | 411.7ms/step | 48.6 samples/s +Step 27630/44000 | Loss: 2.8363 | LR: 1.57e-04 | 409.2ms/step | 48.9 samples/s +Step 27640/44000 | Loss: 2.9056 | LR: 1.57e-04 | 409.0ms/step | 48.9 samples/s +Step 27650/44000 | Loss: 2.8860 | LR: 1.57e-04 | 409.7ms/step | 48.8 samples/s +Step 27660/44000 | Loss: 2.8506 | LR: 1.56e-04 | 405.0ms/step | 49.4 samples/s +Step 27670/44000 | Loss: 2.8769 | LR: 1.56e-04 | 404.0ms/step | 49.5 samples/s +Step 27680/44000 | Loss: 2.8950 | LR: 1.56e-04 | 404.4ms/step | 49.5 samples/s +Step 27690/44000 | Loss: 2.8991 | LR: 1.56e-04 | 404.7ms/step | 49.4 samples/s +Step 27700/44000 | Loss: 2.8568 | LR: 1.56e-04 | 406.1ms/step | 49.2 samples/s +Step 27710/44000 | Loss: 2.8839 | LR: 1.56e-04 | 405.6ms/step | 49.3 samples/s +Step 27720/44000 | Loss: 2.9247 | LR: 1.55e-04 | 403.7ms/step | 49.5 samples/s +Step 27730/44000 | Loss: 2.7734 | LR: 1.55e-04 | 404.4ms/step | 49.5 samples/s +Step 27740/44000 | Loss: 2.9016 | LR: 1.55e-04 | 404.6ms/step | 49.4 samples/s +Step 27750/44000 | Loss: 2.8898 | LR: 1.55e-04 | 404.1ms/step | 49.5 samples/s +Validation at step 27750: Loss = 2.9180, Perplexity = 18.50 +Step 27760/44000 | Loss: 2.8489 | LR: 1.55e-04 | 556.2ms/step | 36.0 samples/s +Step 27770/44000 | Loss: 2.8273 | LR: 1.55e-04 | 405.1ms/step | 49.4 samples/s +Step 27780/44000 | Loss: 2.9016 | LR: 1.54e-04 | 404.2ms/step | 49.5 samples/s +Step 27790/44000 | Loss: 2.8405 | LR: 1.54e-04 | 404.3ms/step | 49.5 samples/s +Step 27800/44000 | Loss: 2.8366 | LR: 1.54e-04 | 406.5ms/step | 49.2 samples/s +Step 27810/44000 | Loss: 2.8633 | LR: 1.54e-04 | 408.0ms/step | 49.0 samples/s +Step 27820/44000 | Loss: 2.8606 | LR: 1.54e-04 | 404.8ms/step | 49.4 samples/s +Step 27830/44000 | Loss: 2.8639 | LR: 1.54e-04 | 404.2ms/step | 49.5 samples/s +Step 27840/44000 | Loss: 2.9206 | LR: 1.53e-04 | 404.1ms/step | 49.5 samples/s +Step 27850/44000 | Loss: 2.9252 | LR: 1.53e-04 | 405.2ms/step | 49.4 samples/s +Step 27860/44000 | Loss: 2.8714 | LR: 1.53e-04 | 404.5ms/step | 49.4 samples/s +Step 27870/44000 | Loss: 2.9224 | LR: 1.53e-04 | 404.2ms/step | 49.5 samples/s +Step 27880/44000 | Loss: 2.9069 | LR: 1.53e-04 | 407.1ms/step | 49.1 samples/s +Step 27890/44000 | Loss: 2.8788 | LR: 1.53e-04 | 405.0ms/step | 49.4 samples/s +Step 27900/44000 | Loss: 2.8847 | LR: 1.52e-04 | 404.2ms/step | 49.5 samples/s +Step 27910/44000 | Loss: 2.9148 | LR: 1.52e-04 | 407.3ms/step | 49.1 samples/s +Step 27920/44000 | Loss: 2.8628 | LR: 1.52e-04 | 407.9ms/step | 49.0 samples/s +Step 27930/44000 | Loss: 2.8393 | LR: 1.52e-04 | 405.0ms/step | 49.4 samples/s +Step 27940/44000 | Loss: 2.8371 | LR: 1.52e-04 | 405.2ms/step | 49.4 samples/s +Step 27950/44000 | Loss: 2.8763 | LR: 1.52e-04 | 405.3ms/step | 49.3 samples/s +Step 27960/44000 | Loss: 2.8704 | LR: 1.51e-04 | 406.5ms/step | 49.2 samples/s +Step 27970/44000 | Loss: 2.8859 | LR: 1.51e-04 | 405.6ms/step | 49.3 samples/s +Step 27980/44000 | Loss: 2.8207 | LR: 1.51e-04 | 405.2ms/step | 49.4 samples/s +Step 27990/44000 | Loss: 2.8656 | LR: 1.51e-04 | 405.9ms/step | 49.3 samples/s +Step 28000/44000 | Loss: 2.8942 | LR: 1.51e-04 | 405.1ms/step | 49.4 samples/s +Validation at step 28000: Loss = 2.9166, Perplexity = 18.48 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-28000 +Step 28010/44000 | Loss: 2.8911 | LR: 1.51e-04 | 879.3ms/step | 22.7 samples/s +Step 28020/44000 | Loss: 2.8959 | LR: 1.50e-04 | 408.1ms/step | 49.0 samples/s +Step 28030/44000 | Loss: 2.8804 | LR: 1.50e-04 | 404.8ms/step | 49.4 samples/s +Step 28040/44000 | Loss: 2.8383 | LR: 1.50e-04 | 404.8ms/step | 49.4 samples/s +Step 28050/44000 | Loss: 2.8568 | LR: 1.50e-04 | 406.8ms/step | 49.2 samples/s +Step 28060/44000 | Loss: 2.8696 | LR: 1.50e-04 | 404.5ms/step | 49.4 samples/s +Step 28070/44000 | Loss: 2.8463 | LR: 1.50e-04 | 404.3ms/step | 49.5 samples/s +Step 28080/44000 | Loss: 2.9731 | LR: 1.49e-04 | 404.6ms/step | 49.4 samples/s +Step 28090/44000 | Loss: 2.9127 | LR: 1.49e-04 | 405.4ms/step | 49.3 samples/s +Step 28100/44000 | Loss: 2.8824 | LR: 1.49e-04 | 404.2ms/step | 49.5 samples/s +Step 28110/44000 | Loss: 2.8442 | LR: 1.49e-04 | 405.5ms/step | 49.3 samples/s +Step 28120/44000 | Loss: 2.8648 | LR: 1.49e-04 | 406.3ms/step | 49.2 samples/s +Step 28130/44000 | Loss: 2.9142 | LR: 1.49e-04 | 405.8ms/step | 49.3 samples/s +Step 28140/44000 | Loss: 2.8844 | LR: 1.48e-04 | 404.8ms/step | 49.4 samples/s +Step 28150/44000 | Loss: 2.9345 | LR: 1.48e-04 | 405.5ms/step | 49.3 samples/s +Step 28160/44000 | Loss: 2.9194 | LR: 1.48e-04 | 403.7ms/step | 49.5 samples/s +Step 28170/44000 | Loss: 2.8601 | LR: 1.48e-04 | 404.6ms/step | 49.4 samples/s +Step 28180/44000 | Loss: 2.8307 | LR: 1.48e-04 | 404.2ms/step | 49.5 samples/s +Step 28190/44000 | Loss: 2.9101 | LR: 1.48e-04 | 404.6ms/step | 49.4 samples/s +Step 28200/44000 | Loss: 2.9482 | LR: 1.47e-04 | 413.2ms/step | 48.4 samples/s +Step 28210/44000 | Loss: 2.8885 | LR: 1.47e-04 | 408.6ms/step | 48.9 samples/s +Step 28220/44000 | Loss: 2.9055 | LR: 1.47e-04 | 407.4ms/step | 49.1 samples/s +Step 28230/44000 | Loss: 2.9057 | LR: 1.47e-04 | 407.1ms/step | 49.1 samples/s +Step 28240/44000 | Loss: 2.8140 | LR: 1.47e-04 | 408.7ms/step | 48.9 samples/s +Step 28250/44000 | Loss: 2.8315 | LR: 1.47e-04 | 405.0ms/step | 49.4 samples/s +Validation at step 28250: Loss = 2.9148, Perplexity = 18.44 +Step 28260/44000 | Loss: 2.9105 | LR: 1.46e-04 | 559.3ms/step | 35.8 samples/s +Step 28270/44000 | Loss: 2.8696 | LR: 1.46e-04 | 405.8ms/step | 49.3 samples/s +Step 28280/44000 | Loss: 2.9018 | LR: 1.46e-04 | 404.3ms/step | 49.5 samples/s +Step 28290/44000 | Loss: 2.8959 | LR: 1.46e-04 | 405.1ms/step | 49.4 samples/s +Step 28300/44000 | Loss: 2.8535 | LR: 1.46e-04 | 404.6ms/step | 49.4 samples/s +Step 28310/44000 | Loss: 2.9106 | LR: 1.46e-04 | 405.9ms/step | 49.3 samples/s +Step 28320/44000 | Loss: 2.8804 | LR: 1.45e-04 | 406.8ms/step | 49.2 samples/s +Step 28330/44000 | Loss: 2.8937 | LR: 1.45e-04 | 407.1ms/step | 49.1 samples/s +Step 28340/44000 | Loss: 2.8434 | LR: 1.45e-04 | 407.6ms/step | 49.1 samples/s +Step 28350/44000 | Loss: 2.8946 | LR: 1.45e-04 | 405.7ms/step | 49.3 samples/s +Step 28360/44000 | Loss: 2.8405 | LR: 1.45e-04 | 404.3ms/step | 49.5 samples/s +Step 28370/44000 | Loss: 2.9162 | LR: 1.45e-04 | 404.7ms/step | 49.4 samples/s +Step 28380/44000 | Loss: 2.9214 | LR: 1.44e-04 | 405.3ms/step | 49.3 samples/s +Step 28390/44000 | Loss: 2.8761 | LR: 1.44e-04 | 404.2ms/step | 49.5 samples/s +Step 28400/44000 | Loss: 2.8608 | LR: 1.44e-04 | 404.8ms/step | 49.4 samples/s +Step 28410/44000 | Loss: 2.8477 | LR: 1.44e-04 | 405.3ms/step | 49.3 samples/s +Step 28420/44000 | Loss: 2.8739 | LR: 1.44e-04 | 404.3ms/step | 49.5 samples/s +Step 28430/44000 | Loss: 2.8380 | LR: 1.44e-04 | 405.5ms/step | 49.3 samples/s +Step 28440/44000 | Loss: 2.9345 | LR: 1.43e-04 | 406.4ms/step | 49.2 samples/s +Step 28450/44000 | Loss: 2.8340 | LR: 1.43e-04 | 406.6ms/step | 49.2 samples/s +Step 28460/44000 | Loss: 2.8807 | LR: 1.43e-04 | 405.0ms/step | 49.4 samples/s +Step 28470/44000 | Loss: 2.8969 | LR: 1.43e-04 | 404.1ms/step | 49.5 samples/s +Step 28480/44000 | Loss: 2.8804 | LR: 1.43e-04 | 403.5ms/step | 49.6 samples/s +Step 28490/44000 | Loss: 2.8325 | LR: 1.43e-04 | 404.0ms/step | 49.5 samples/s +Step 28500/44000 | Loss: 2.9132 | LR: 1.42e-04 | 404.2ms/step | 49.5 samples/s +Validation at step 28500: Loss = 2.9140, Perplexity = 18.43 +Step 28510/44000 | Loss: 2.8441 | LR: 1.42e-04 | 556.9ms/step | 35.9 samples/s +Step 28520/44000 | Loss: 2.8676 | LR: 1.42e-04 | 403.8ms/step | 49.5 samples/s +Step 28530/44000 | Loss: 2.8899 | LR: 1.42e-04 | 405.3ms/step | 49.3 samples/s +Step 28540/44000 | Loss: 2.8176 | LR: 1.42e-04 | 406.0ms/step | 49.3 samples/s +Step 28550/44000 | Loss: 2.8389 | LR: 1.42e-04 | 406.0ms/step | 49.3 samples/s +Step 28560/44000 | Loss: 2.8483 | LR: 1.41e-04 | 405.7ms/step | 49.3 samples/s +Step 28570/44000 | Loss: 2.8922 | LR: 1.41e-04 | 405.0ms/step | 49.4 samples/s +Step 28580/44000 | Loss: 2.8554 | LR: 1.41e-04 | 403.4ms/step | 49.6 samples/s +Step 28590/44000 | Loss: 2.8521 | LR: 1.41e-04 | 403.9ms/step | 49.5 samples/s +Step 28600/44000 | Loss: 2.9166 | LR: 1.41e-04 | 404.9ms/step | 49.4 samples/s +Step 28610/44000 | Loss: 2.7781 | LR: 1.41e-04 | 405.1ms/step | 49.4 samples/s +Step 28620/44000 | Loss: 2.8628 | LR: 1.40e-04 | 404.3ms/step | 49.5 samples/s +Step 28630/44000 | Loss: 2.9130 | LR: 1.40e-04 | 405.4ms/step | 49.3 samples/s +Step 28640/44000 | Loss: 2.8695 | LR: 1.40e-04 | 404.6ms/step | 49.4 samples/s +Step 28650/44000 | Loss: 2.8702 | LR: 1.40e-04 | 406.5ms/step | 49.2 samples/s +Step 28660/44000 | Loss: 2.8455 | LR: 1.40e-04 | 406.5ms/step | 49.2 samples/s +Step 28670/44000 | Loss: 2.9087 | LR: 1.40e-04 | 405.6ms/step | 49.3 samples/s +Step 28680/44000 | Loss: 2.8754 | LR: 1.39e-04 | 405.5ms/step | 49.3 samples/s +Step 28690/44000 | Loss: 2.9009 | LR: 1.39e-04 | 405.2ms/step | 49.4 samples/s +Step 28700/44000 | Loss: 2.8981 | LR: 1.39e-04 | 405.0ms/step | 49.4 samples/s +Step 28710/44000 | Loss: 2.9134 | LR: 1.39e-04 | 404.1ms/step | 49.5 samples/s +Step 28720/44000 | Loss: 2.8948 | LR: 1.39e-04 | 404.7ms/step | 49.4 samples/s +Step 28730/44000 | Loss: 2.9030 | LR: 1.39e-04 | 405.8ms/step | 49.3 samples/s +Step 28740/44000 | Loss: 2.8468 | LR: 1.38e-04 | 405.4ms/step | 49.3 samples/s +Step 28750/44000 | Loss: 2.8995 | LR: 1.38e-04 | 405.4ms/step | 49.3 samples/s +Validation at step 28750: Loss = 2.9130, Perplexity = 18.41 +Step 28760/44000 | Loss: 2.8891 | LR: 1.38e-04 | 561.5ms/step | 35.6 samples/s +Step 28770/44000 | Loss: 2.8955 | LR: 1.38e-04 | 407.9ms/step | 49.0 samples/s +Step 28780/44000 | Loss: 2.8792 | LR: 1.38e-04 | 405.2ms/step | 49.4 samples/s +Step 28790/44000 | Loss: 2.8698 | LR: 1.38e-04 | 404.3ms/step | 49.5 samples/s +Step 28800/44000 | Loss: 2.8900 | LR: 1.38e-04 | 403.9ms/step | 49.5 samples/s +Step 28810/44000 | Loss: 2.8768 | LR: 1.37e-04 | 405.7ms/step | 49.3 samples/s +Step 28820/44000 | Loss: 2.9088 | LR: 1.37e-04 | 404.8ms/step | 49.4 samples/s +Step 28830/44000 | Loss: 2.8888 | LR: 1.37e-04 | 406.1ms/step | 49.2 samples/s +Step 28840/44000 | Loss: 2.9134 | LR: 1.37e-04 | 403.7ms/step | 49.5 samples/s +Step 28850/44000 | Loss: 2.8482 | LR: 1.37e-04 | 405.2ms/step | 49.4 samples/s +Step 28860/44000 | Loss: 2.8277 | LR: 1.37e-04 | 410.7ms/step | 48.7 samples/s +Step 28870/44000 | Loss: 2.8591 | LR: 1.36e-04 | 416.4ms/step | 48.0 samples/s +Step 28880/44000 | Loss: 2.9028 | LR: 1.36e-04 | 416.5ms/step | 48.0 samples/s +Step 28890/44000 | Loss: 2.9137 | LR: 1.36e-04 | 409.4ms/step | 48.8 samples/s +Step 28900/44000 | Loss: 2.7537 | LR: 1.36e-04 | 409.9ms/step | 48.8 samples/s +Step 28910/44000 | Loss: 2.8452 | LR: 1.36e-04 | 408.5ms/step | 49.0 samples/s +Step 28920/44000 | Loss: 2.8828 | LR: 1.36e-04 | 408.2ms/step | 49.0 samples/s +Step 28930/44000 | Loss: 2.8185 | LR: 1.35e-04 | 407.9ms/step | 49.0 samples/s +Step 28940/44000 | Loss: 2.8710 | LR: 1.35e-04 | 408.3ms/step | 49.0 samples/s +Step 28950/44000 | Loss: 2.8606 | LR: 1.35e-04 | 408.5ms/step | 49.0 samples/s +Step 28960/44000 | Loss: 2.8612 | LR: 1.35e-04 | 408.2ms/step | 49.0 samples/s +Step 28970/44000 | Loss: 2.8769 | LR: 1.35e-04 | 414.5ms/step | 48.2 samples/s +Step 28980/44000 | Loss: 2.8573 | LR: 1.35e-04 | 415.6ms/step | 48.1 samples/s +Step 28990/44000 | Loss: 2.9420 | LR: 1.34e-04 | 412.8ms/step | 48.4 samples/s +Step 29000/44000 | Loss: 2.8630 | LR: 1.34e-04 | 407.8ms/step | 49.0 samples/s +Validation at step 29000: Loss = 2.9110, Perplexity = 18.38 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-29000 +Step 29010/44000 | Loss: 2.8920 | LR: 1.34e-04 | 877.2ms/step | 22.8 samples/s +Step 29020/44000 | Loss: 2.8099 | LR: 1.34e-04 | 402.0ms/step | 49.8 samples/s +Step 29030/44000 | Loss: 2.9126 | LR: 1.34e-04 | 403.1ms/step | 49.6 samples/s +Step 29040/44000 | Loss: 2.9546 | LR: 1.34e-04 | 402.2ms/step | 49.7 samples/s +Step 29050/44000 | Loss: 2.8606 | LR: 1.33e-04 | 402.1ms/step | 49.7 samples/s +Step 29060/44000 | Loss: 2.8963 | LR: 1.33e-04 | 404.2ms/step | 49.5 samples/s +Step 29070/44000 | Loss: 2.8587 | LR: 1.33e-04 | 407.7ms/step | 49.1 samples/s +Step 29080/44000 | Loss: 2.8220 | LR: 1.33e-04 | 408.5ms/step | 49.0 samples/s +Step 29090/44000 | Loss: 2.9046 | LR: 1.33e-04 | 404.0ms/step | 49.5 samples/s +Step 29100/44000 | Loss: 2.8686 | LR: 1.33e-04 | 402.6ms/step | 49.7 samples/s +Step 29110/44000 | Loss: 2.9091 | LR: 1.33e-04 | 405.0ms/step | 49.4 samples/s +Step 29120/44000 | Loss: 2.9083 | LR: 1.32e-04 | 404.1ms/step | 49.5 samples/s +Step 29130/44000 | Loss: 2.8847 | LR: 1.32e-04 | 403.8ms/step | 49.5 samples/s +Step 29140/44000 | Loss: 2.8650 | LR: 1.32e-04 | 403.2ms/step | 49.6 samples/s +Step 29150/44000 | Loss: 2.8985 | LR: 1.32e-04 | 403.5ms/step | 49.6 samples/s +Step 29160/44000 | Loss: 2.8550 | LR: 1.32e-04 | 403.9ms/step | 49.5 samples/s +Step 29170/44000 | Loss: 2.8613 | LR: 1.32e-04 | 404.1ms/step | 49.5 samples/s +Step 29180/44000 | Loss: 2.8723 | LR: 1.31e-04 | 408.5ms/step | 49.0 samples/s +Step 29190/44000 | Loss: 2.9335 | LR: 1.31e-04 | 406.9ms/step | 49.1 samples/s +Step 29200/44000 | Loss: 2.8025 | LR: 1.31e-04 | 402.9ms/step | 49.6 samples/s +Step 29210/44000 | Loss: 2.8608 | LR: 1.31e-04 | 403.8ms/step | 49.5 samples/s +Step 29220/44000 | Loss: 2.8806 | LR: 1.31e-04 | 404.6ms/step | 49.4 samples/s +Step 29230/44000 | Loss: 2.8552 | LR: 1.31e-04 | 405.2ms/step | 49.4 samples/s +Step 29240/44000 | Loss: 2.8876 | LR: 1.30e-04 | 403.2ms/step | 49.6 samples/s +Step 29250/44000 | Loss: 2.8738 | LR: 1.30e-04 | 404.4ms/step | 49.5 samples/s +Validation at step 29250: Loss = 2.9105, Perplexity = 18.37 +Step 29260/44000 | Loss: 2.8587 | LR: 1.30e-04 | 564.8ms/step | 35.4 samples/s +Step 29270/44000 | Loss: 2.8893 | LR: 1.30e-04 | 411.5ms/step | 48.6 samples/s +Step 29280/44000 | Loss: 2.8366 | LR: 1.30e-04 | 417.9ms/step | 47.9 samples/s +Step 29290/44000 | Loss: 2.8945 | LR: 1.30e-04 | 412.4ms/step | 48.5 samples/s +Step 29300/44000 | Loss: 2.9062 | LR: 1.29e-04 | 410.9ms/step | 48.7 samples/s +Step 29310/44000 | Loss: 2.8144 | LR: 1.29e-04 | 405.6ms/step | 49.3 samples/s +Step 29320/44000 | Loss: 2.8129 | LR: 1.29e-04 | 406.2ms/step | 49.2 samples/s +Step 29330/44000 | Loss: 2.8543 | LR: 1.29e-04 | 406.0ms/step | 49.3 samples/s +Step 29340/44000 | Loss: 2.9259 | LR: 1.29e-04 | 406.9ms/step | 49.2 samples/s +Step 29350/44000 | Loss: 2.8977 | LR: 1.29e-04 | 405.0ms/step | 49.4 samples/s +Step 29360/44000 | Loss: 2.8895 | LR: 1.29e-04 | 407.4ms/step | 49.1 samples/s +Step 29370/44000 | Loss: 2.8721 | LR: 1.28e-04 | 406.0ms/step | 49.3 samples/s +Step 29380/44000 | Loss: 2.8609 | LR: 1.28e-04 | 406.6ms/step | 49.2 samples/s +Step 29390/44000 | Loss: 2.8063 | LR: 1.28e-04 | 412.6ms/step | 48.5 samples/s +Step 29400/44000 | Loss: 2.8804 | LR: 1.28e-04 | 415.6ms/step | 48.1 samples/s +Step 29410/44000 | Loss: 2.8305 | LR: 1.28e-04 | 407.3ms/step | 49.1 samples/s +Step 29420/44000 | Loss: 2.8575 | LR: 1.28e-04 | 406.1ms/step | 49.3 samples/s +Step 29430/44000 | Loss: 2.8714 | LR: 1.27e-04 | 408.2ms/step | 49.0 samples/s +Step 29440/44000 | Loss: 2.8552 | LR: 1.27e-04 | 406.6ms/step | 49.2 samples/s +Step 29450/44000 | Loss: 2.8752 | LR: 1.27e-04 | 404.9ms/step | 49.4 samples/s +Step 29460/44000 | Loss: 2.9336 | LR: 1.27e-04 | 404.6ms/step | 49.4 samples/s +Step 29470/44000 | Loss: 2.8882 | LR: 1.27e-04 | 405.2ms/step | 49.4 samples/s +Step 29480/44000 | Loss: 2.8564 | LR: 1.27e-04 | 405.5ms/step | 49.3 samples/s +Step 29490/44000 | Loss: 2.8415 | LR: 1.26e-04 | 407.1ms/step | 49.1 samples/s +Step 29500/44000 | Loss: 2.8562 | LR: 1.26e-04 | 413.6ms/step | 48.4 samples/s +Validation at step 29500: Loss = 2.9092, Perplexity = 18.34 +Step 29510/44000 | Loss: 2.8690 | LR: 1.26e-04 | 570.2ms/step | 35.1 samples/s +Step 29520/44000 | Loss: 2.8371 | LR: 1.26e-04 | 406.3ms/step | 49.2 samples/s +Step 29530/44000 | Loss: 2.8905 | LR: 1.26e-04 | 407.1ms/step | 49.1 samples/s +Step 29540/44000 | Loss: 2.8379 | LR: 1.26e-04 | 407.3ms/step | 49.1 samples/s +Step 29550/44000 | Loss: 2.8383 | LR: 1.26e-04 | 406.9ms/step | 49.2 samples/s +Step 29560/44000 | Loss: 2.8825 | LR: 1.25e-04 | 405.5ms/step | 49.3 samples/s +Step 29570/44000 | Loss: 2.8964 | LR: 1.25e-04 | 407.0ms/step | 49.1 samples/s +Step 29580/44000 | Loss: 2.8571 | LR: 1.25e-04 | 406.7ms/step | 49.2 samples/s +Step 29590/44000 | Loss: 2.9048 | LR: 1.25e-04 | 406.6ms/step | 49.2 samples/s +Step 29600/44000 | Loss: 2.8827 | LR: 1.25e-04 | 410.4ms/step | 48.7 samples/s +Step 29610/44000 | Loss: 2.8687 | LR: 1.25e-04 | 410.5ms/step | 48.7 samples/s +Step 29620/44000 | Loss: 2.8347 | LR: 1.24e-04 | 409.7ms/step | 48.8 samples/s +Step 29630/44000 | Loss: 2.9043 | LR: 1.24e-04 | 406.5ms/step | 49.2 samples/s +Step 29640/44000 | Loss: 2.8608 | LR: 1.24e-04 | 406.4ms/step | 49.2 samples/s +Step 29650/44000 | Loss: 2.8626 | LR: 1.24e-04 | 407.7ms/step | 49.1 samples/s +Step 29660/44000 | Loss: 2.8183 | LR: 1.24e-04 | 406.3ms/step | 49.2 samples/s +Step 29670/44000 | Loss: 2.8795 | LR: 1.24e-04 | 406.5ms/step | 49.2 samples/s +Step 29680/44000 | Loss: 2.8834 | LR: 1.23e-04 | 407.1ms/step | 49.1 samples/s +Step 29690/44000 | Loss: 2.9023 | LR: 1.23e-04 | 406.3ms/step | 49.2 samples/s +Step 29700/44000 | Loss: 2.8503 | LR: 1.23e-04 | 406.4ms/step | 49.2 samples/s +Step 29710/44000 | Loss: 2.8550 | LR: 1.23e-04 | 409.9ms/step | 48.8 samples/s +Step 29720/44000 | Loss: 2.8869 | LR: 1.23e-04 | 410.3ms/step | 48.7 samples/s +Step 29730/44000 | Loss: 2.8573 | LR: 1.23e-04 | 412.5ms/step | 48.5 samples/s +Step 29740/44000 | Loss: 2.8954 | LR: 1.23e-04 | 405.9ms/step | 49.3 samples/s +Step 29750/44000 | Loss: 2.9117 | LR: 1.22e-04 | 404.5ms/step | 49.4 samples/s +Validation at step 29750: Loss = 2.9086, Perplexity = 18.33 +Step 29760/44000 | Loss: 2.9034 | LR: 1.22e-04 | 558.9ms/step | 35.8 samples/s +Step 29770/44000 | Loss: 2.8876 | LR: 1.22e-04 | 407.3ms/step | 49.1 samples/s +Step 29780/44000 | Loss: 2.8204 | LR: 1.22e-04 | 406.0ms/step | 49.3 samples/s +Step 29790/44000 | Loss: 2.9182 | LR: 1.22e-04 | 405.9ms/step | 49.3 samples/s +Step 29800/44000 | Loss: 2.9657 | LR: 1.22e-04 | 406.0ms/step | 49.3 samples/s +Step 29810/44000 | Loss: 2.8145 | LR: 1.21e-04 | 405.5ms/step | 49.3 samples/s +Step 29820/44000 | Loss: 2.8747 | LR: 1.21e-04 | 411.0ms/step | 48.7 samples/s +Step 29830/44000 | Loss: 2.9297 | LR: 1.21e-04 | 410.4ms/step | 48.7 samples/s +Step 29840/44000 | Loss: 2.9023 | LR: 1.21e-04 | 414.3ms/step | 48.3 samples/s +Step 29850/44000 | Loss: 2.8715 | LR: 1.21e-04 | 406.2ms/step | 49.2 samples/s +Step 29860/44000 | Loss: 2.8552 | LR: 1.21e-04 | 405.5ms/step | 49.3 samples/s +Step 29870/44000 | Loss: 2.8659 | LR: 1.21e-04 | 406.9ms/step | 49.1 samples/s +Step 29880/44000 | Loss: 2.8497 | LR: 1.20e-04 | 407.5ms/step | 49.1 samples/s +Step 29890/44000 | Loss: 2.8779 | LR: 1.20e-04 | 405.6ms/step | 49.3 samples/s +Step 29900/44000 | Loss: 2.8603 | LR: 1.20e-04 | 407.7ms/step | 49.1 samples/s +Step 29910/44000 | Loss: 2.8702 | LR: 1.20e-04 | 406.9ms/step | 49.2 samples/s +Step 29920/44000 | Loss: 2.8545 | LR: 1.20e-04 | 406.3ms/step | 49.2 samples/s +Step 29930/44000 | Loss: 2.9262 | LR: 1.20e-04 | 411.5ms/step | 48.6 samples/s +Step 29940/44000 | Loss: 2.8249 | LR: 1.19e-04 | 411.7ms/step | 48.6 samples/s +Step 29950/44000 | Loss: 2.8591 | LR: 1.19e-04 | 409.5ms/step | 48.8 samples/s +Step 29960/44000 | Loss: 2.8512 | LR: 1.19e-04 | 404.9ms/step | 49.4 samples/s +Step 29970/44000 | Loss: 2.7954 | LR: 1.19e-04 | 406.0ms/step | 49.3 samples/s +Step 29980/44000 | Loss: 2.8303 | LR: 1.19e-04 | 404.8ms/step | 49.4 samples/s +Step 29990/44000 | Loss: 2.8571 | LR: 1.19e-04 | 406.4ms/step | 49.2 samples/s +Step 30000/44000 | Loss: 2.8938 | LR: 1.19e-04 | 405.1ms/step | 49.4 samples/s +Validation at step 30000: Loss = 2.9076, Perplexity = 18.31 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-30000 +Step 30010/44000 | Loss: 2.8391 | LR: 1.18e-04 | 867.9ms/step | 23.0 samples/s +Step 30020/44000 | Loss: 2.9255 | LR: 1.18e-04 | 406.3ms/step | 49.2 samples/s +Step 30030/44000 | Loss: 2.8516 | LR: 1.18e-04 | 409.2ms/step | 48.9 samples/s +Step 30040/44000 | Loss: 2.9335 | LR: 1.18e-04 | 408.2ms/step | 49.0 samples/s +Step 30050/44000 | Loss: 2.8500 | LR: 1.18e-04 | 408.3ms/step | 49.0 samples/s +Step 30060/44000 | Loss: 2.8346 | LR: 1.18e-04 | 404.4ms/step | 49.5 samples/s +Step 30070/44000 | Loss: 2.8836 | LR: 1.17e-04 | 404.7ms/step | 49.4 samples/s +Step 30080/44000 | Loss: 2.8692 | LR: 1.17e-04 | 407.2ms/step | 49.1 samples/s +Step 30090/44000 | Loss: 2.8297 | LR: 1.17e-04 | 405.3ms/step | 49.4 samples/s +Step 30100/44000 | Loss: 2.8876 | LR: 1.17e-04 | 406.0ms/step | 49.3 samples/s +Step 30110/44000 | Loss: 2.8410 | LR: 1.17e-04 | 407.9ms/step | 49.0 samples/s +Step 30120/44000 | Loss: 2.8996 | LR: 1.17e-04 | 406.6ms/step | 49.2 samples/s +Step 30130/44000 | Loss: 2.8439 | LR: 1.17e-04 | 408.4ms/step | 49.0 samples/s +Step 30140/44000 | Loss: 2.9014 | LR: 1.16e-04 | 412.0ms/step | 48.5 samples/s +Step 30150/44000 | Loss: 2.8649 | LR: 1.16e-04 | 411.0ms/step | 48.7 samples/s +Step 30160/44000 | Loss: 2.8710 | LR: 1.16e-04 | 404.4ms/step | 49.5 samples/s +Step 30170/44000 | Loss: 2.8298 | LR: 1.16e-04 | 406.0ms/step | 49.3 samples/s +Step 30180/44000 | Loss: 2.8528 | LR: 1.16e-04 | 405.3ms/step | 49.3 samples/s +Step 30190/44000 | Loss: 2.8255 | LR: 1.16e-04 | 405.4ms/step | 49.3 samples/s +Step 30200/44000 | Loss: 2.9226 | LR: 1.15e-04 | 405.7ms/step | 49.3 samples/s +Step 30210/44000 | Loss: 2.8782 | LR: 1.15e-04 | 406.7ms/step | 49.2 samples/s +Step 30220/44000 | Loss: 2.8726 | LR: 1.15e-04 | 407.2ms/step | 49.1 samples/s +Step 30230/44000 | Loss: 2.9070 | LR: 1.15e-04 | 406.8ms/step | 49.2 samples/s +Step 30240/44000 | Loss: 2.8258 | LR: 1.15e-04 | 411.3ms/step | 48.6 samples/s +Step 30250/44000 | Loss: 2.8534 | LR: 1.15e-04 | 413.7ms/step | 48.3 samples/s +Validation at step 30250: Loss = 2.9068, Perplexity = 18.30 +Step 30260/44000 | Loss: 2.8878 | LR: 1.15e-04 | 567.0ms/step | 35.3 samples/s +Step 30270/44000 | Loss: 2.8621 | LR: 1.14e-04 | 405.4ms/step | 49.3 samples/s +Step 30280/44000 | Loss: 2.8631 | LR: 1.14e-04 | 405.8ms/step | 49.3 samples/s +Step 30290/44000 | Loss: 2.8268 | LR: 1.14e-04 | 405.3ms/step | 49.3 samples/s +Step 30300/44000 | Loss: 2.8606 | LR: 1.14e-04 | 405.6ms/step | 49.3 samples/s +Step 30310/44000 | Loss: 2.8806 | LR: 1.14e-04 | 407.2ms/step | 49.1 samples/s +Step 30320/44000 | Loss: 2.8682 | LR: 1.14e-04 | 407.3ms/step | 49.1 samples/s +Step 30330/44000 | Loss: 2.8695 | LR: 1.13e-04 | 405.3ms/step | 49.3 samples/s +Step 30340/44000 | Loss: 2.9469 | LR: 1.13e-04 | 407.7ms/step | 49.1 samples/s +Step 30350/44000 | Loss: 2.8649 | LR: 1.13e-04 | 410.1ms/step | 48.8 samples/s +Step 30360/44000 | Loss: 2.8897 | LR: 1.13e-04 | 411.6ms/step | 48.6 samples/s +Step 30370/44000 | Loss: 2.8675 | LR: 1.13e-04 | 405.8ms/step | 49.3 samples/s +Step 30380/44000 | Loss: 2.8916 | LR: 1.13e-04 | 405.3ms/step | 49.3 samples/s +Step 30390/44000 | Loss: 2.8601 | LR: 1.13e-04 | 407.6ms/step | 49.1 samples/s +Step 30400/44000 | Loss: 2.8556 | LR: 1.12e-04 | 405.3ms/step | 49.4 samples/s +Step 30410/44000 | Loss: 2.9283 | LR: 1.12e-04 | 404.9ms/step | 49.4 samples/s +Step 30420/44000 | Loss: 2.9126 | LR: 1.12e-04 | 404.9ms/step | 49.4 samples/s +Step 30430/44000 | Loss: 2.8742 | LR: 1.12e-04 | 405.8ms/step | 49.3 samples/s +Step 30440/44000 | Loss: 2.8716 | LR: 1.12e-04 | 406.3ms/step | 49.2 samples/s +Step 30450/44000 | Loss: 2.8525 | LR: 1.12e-04 | 412.3ms/step | 48.5 samples/s +Step 30460/44000 | Loss: 2.8840 | LR: 1.11e-04 | 409.2ms/step | 48.9 samples/s +Step 30470/44000 | Loss: 2.8518 | LR: 1.11e-04 | 406.8ms/step | 49.2 samples/s +Step 30480/44000 | Loss: 2.9148 | LR: 1.11e-04 | 406.0ms/step | 49.3 samples/s +Step 30490/44000 | Loss: 2.8363 | LR: 1.11e-04 | 406.0ms/step | 49.3 samples/s +Step 30500/44000 | Loss: 2.8848 | LR: 1.11e-04 | 405.7ms/step | 49.3 samples/s +Validation at step 30500: Loss = 2.9065, Perplexity = 18.29 +Step 30510/44000 | Loss: 2.9106 | LR: 1.11e-04 | 562.1ms/step | 35.6 samples/s +Step 30520/44000 | Loss: 2.8671 | LR: 1.11e-04 | 405.3ms/step | 49.3 samples/s +Step 30530/44000 | Loss: 2.8370 | LR: 1.10e-04 | 405.7ms/step | 49.3 samples/s +Step 30540/44000 | Loss: 2.8921 | LR: 1.10e-04 | 406.5ms/step | 49.2 samples/s +Step 30550/44000 | Loss: 2.8588 | LR: 1.10e-04 | 411.8ms/step | 48.6 samples/s +Step 30560/44000 | Loss: 2.8758 | LR: 1.10e-04 | 410.3ms/step | 48.7 samples/s +Step 30570/44000 | Loss: 2.8724 | LR: 1.10e-04 | 410.7ms/step | 48.7 samples/s +Step 30580/44000 | Loss: 2.8595 | LR: 1.10e-04 | 406.3ms/step | 49.2 samples/s +Step 30590/44000 | Loss: 2.8573 | LR: 1.10e-04 | 407.9ms/step | 49.0 samples/s +Step 30600/44000 | Loss: 2.9119 | LR: 1.09e-04 | 405.5ms/step | 49.3 samples/s +Step 30610/44000 | Loss: 2.8392 | LR: 1.09e-04 | 406.2ms/step | 49.2 samples/s +Step 30620/44000 | Loss: 2.8488 | LR: 1.09e-04 | 404.0ms/step | 49.5 samples/s +Step 30630/44000 | Loss: 2.8214 | LR: 1.09e-04 | 405.9ms/step | 49.3 samples/s +Step 30640/44000 | Loss: 2.8375 | LR: 1.09e-04 | 406.2ms/step | 49.2 samples/s +Step 30650/44000 | Loss: 2.8629 | LR: 1.09e-04 | 405.9ms/step | 49.3 samples/s +Step 30660/44000 | Loss: 2.8482 | LR: 1.08e-04 | 407.7ms/step | 49.1 samples/s +Step 30670/44000 | Loss: 2.8838 | LR: 1.08e-04 | 410.6ms/step | 48.7 samples/s +Step 30680/44000 | Loss: 2.7914 | LR: 1.08e-04 | 408.6ms/step | 48.9 samples/s +Step 30690/44000 | Loss: 2.8103 | LR: 1.08e-04 | 405.8ms/step | 49.3 samples/s +Step 30700/44000 | Loss: 2.8388 | LR: 1.08e-04 | 405.4ms/step | 49.3 samples/s +Step 30710/44000 | Loss: 2.8330 | LR: 1.08e-04 | 405.1ms/step | 49.4 samples/s +Step 30720/44000 | Loss: 2.8188 | LR: 1.08e-04 | 403.4ms/step | 49.6 samples/s +Step 30730/44000 | Loss: 2.8107 | LR: 1.07e-04 | 406.2ms/step | 49.2 samples/s +Step 30740/44000 | Loss: 2.8520 | LR: 1.07e-04 | 405.8ms/step | 49.3 samples/s +Step 30750/44000 | Loss: 2.8351 | LR: 1.07e-04 | 405.4ms/step | 49.3 samples/s +Validation at step 30750: Loss = 2.9056, Perplexity = 18.28 +Step 30760/44000 | Loss: 2.8344 | LR: 1.07e-04 | 560.8ms/step | 35.7 samples/s +Step 30770/44000 | Loss: 2.8629 | LR: 1.07e-04 | 411.3ms/step | 48.6 samples/s +Step 30780/44000 | Loss: 2.9265 | LR: 1.07e-04 | 409.2ms/step | 48.9 samples/s +Step 30790/44000 | Loss: 2.8581 | LR: 1.07e-04 | 402.9ms/step | 49.6 samples/s +Step 30800/44000 | Loss: 2.8261 | LR: 1.06e-04 | 406.7ms/step | 49.2 samples/s +Step 30810/44000 | Loss: 2.8510 | LR: 1.06e-04 | 405.1ms/step | 49.4 samples/s +Step 30820/44000 | Loss: 2.8407 | LR: 1.06e-04 | 406.1ms/step | 49.2 samples/s +Step 30830/44000 | Loss: 2.8898 | LR: 1.06e-04 | 406.0ms/step | 49.3 samples/s +Step 30840/44000 | Loss: 2.9249 | LR: 1.06e-04 | 404.9ms/step | 49.4 samples/s +Step 30850/44000 | Loss: 2.8025 | LR: 1.06e-04 | 408.7ms/step | 48.9 samples/s +Step 30860/44000 | Loss: 2.8098 | LR: 1.05e-04 | 409.7ms/step | 48.8 samples/s +Step 30870/44000 | Loss: 2.8570 | LR: 1.05e-04 | 413.3ms/step | 48.4 samples/s +Step 30880/44000 | Loss: 2.8945 | LR: 1.05e-04 | 411.7ms/step | 48.6 samples/s +Step 30890/44000 | Loss: 2.8164 | LR: 1.05e-04 | 407.6ms/step | 49.1 samples/s +Step 30900/44000 | Loss: 2.8403 | LR: 1.05e-04 | 406.4ms/step | 49.2 samples/s +Step 30910/44000 | Loss: 2.8629 | LR: 1.05e-04 | 406.9ms/step | 49.2 samples/s +Step 30920/44000 | Loss: 2.8023 | LR: 1.05e-04 | 410.0ms/step | 48.8 samples/s +Step 30930/44000 | Loss: 2.7848 | LR: 1.04e-04 | 405.4ms/step | 49.3 samples/s +Step 30940/44000 | Loss: 2.8301 | LR: 1.04e-04 | 406.6ms/step | 49.2 samples/s +Step 30950/44000 | Loss: 2.8528 | LR: 1.04e-04 | 406.1ms/step | 49.2 samples/s +Step 30960/44000 | Loss: 2.8934 | LR: 1.04e-04 | 406.7ms/step | 49.2 samples/s +Step 30970/44000 | Loss: 2.8472 | LR: 1.04e-04 | 410.8ms/step | 48.7 samples/s +Step 30980/44000 | Loss: 2.9066 | LR: 1.04e-04 | 411.9ms/step | 48.6 samples/s +Step 30990/44000 | Loss: 2.8823 | LR: 1.04e-04 | 409.1ms/step | 48.9 samples/s +Step 31000/44000 | Loss: 2.9290 | LR: 1.03e-04 | 405.8ms/step | 49.3 samples/s +Validation at step 31000: Loss = 2.9054, Perplexity = 18.27 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-31000 +Step 31010/44000 | Loss: 2.9122 | LR: 1.03e-04 | 873.0ms/step | 22.9 samples/s +Step 31020/44000 | Loss: 2.8603 | LR: 1.03e-04 | 406.4ms/step | 49.2 samples/s +Step 31030/44000 | Loss: 2.9112 | LR: 1.03e-04 | 406.1ms/step | 49.2 samples/s +Step 31040/44000 | Loss: 2.8015 | LR: 1.03e-04 | 406.7ms/step | 49.2 samples/s +Step 31050/44000 | Loss: 2.8780 | LR: 1.03e-04 | 404.9ms/step | 49.4 samples/s +Step 31060/44000 | Loss: 2.8767 | LR: 1.03e-04 | 409.2ms/step | 48.9 samples/s +Step 31070/44000 | Loss: 2.8444 | LR: 1.02e-04 | 412.0ms/step | 48.5 samples/s +Step 31080/44000 | Loss: 2.8793 | LR: 1.02e-04 | 409.5ms/step | 48.8 samples/s +Step 31090/44000 | Loss: 2.8733 | LR: 1.02e-04 | 406.4ms/step | 49.2 samples/s +Step 31100/44000 | Loss: 2.9107 | LR: 1.02e-04 | 407.4ms/step | 49.1 samples/s +Step 31110/44000 | Loss: 2.8863 | LR: 1.02e-04 | 406.2ms/step | 49.2 samples/s +Step 31120/44000 | Loss: 2.8641 | LR: 1.02e-04 | 405.7ms/step | 49.3 samples/s +Step 31130/44000 | Loss: 2.8497 | LR: 1.02e-04 | 405.8ms/step | 49.3 samples/s +Step 31140/44000 | Loss: 2.8439 | LR: 1.01e-04 | 406.5ms/step | 49.2 samples/s +Step 31150/44000 | Loss: 2.8403 | LR: 1.01e-04 | 406.9ms/step | 49.2 samples/s +Step 31160/44000 | Loss: 2.7930 | LR: 1.01e-04 | 407.3ms/step | 49.1 samples/s +Step 31170/44000 | Loss: 2.8420 | LR: 1.01e-04 | 412.9ms/step | 48.4 samples/s +Step 31180/44000 | Loss: 2.8624 | LR: 1.01e-04 | 411.2ms/step | 48.6 samples/s +Step 31190/44000 | Loss: 2.8241 | LR: 1.01e-04 | 408.2ms/step | 49.0 samples/s +Step 31200/44000 | Loss: 2.8965 | LR: 1.00e-04 | 407.7ms/step | 49.1 samples/s +Step 31210/44000 | Loss: 2.8476 | LR: 1.00e-04 | 406.1ms/step | 49.3 samples/s +Step 31220/44000 | Loss: 2.8274 | LR: 1.00e-04 | 409.1ms/step | 48.9 samples/s +Step 31230/44000 | Loss: 2.8650 | LR: 1.00e-04 | 406.9ms/step | 49.2 samples/s +Step 31240/44000 | Loss: 2.8590 | LR: 9.99e-05 | 405.6ms/step | 49.3 samples/s +Step 31250/44000 | Loss: 2.8290 | LR: 9.98e-05 | 407.8ms/step | 49.0 samples/s +Validation at step 31250: Loss = 2.9043, Perplexity = 18.25 +Step 31260/44000 | Loss: 2.9098 | LR: 9.96e-05 | 561.1ms/step | 35.6 samples/s +Step 31270/44000 | Loss: 2.8951 | LR: 9.95e-05 | 412.5ms/step | 48.5 samples/s +Step 31280/44000 | Loss: 2.8595 | LR: 9.93e-05 | 411.5ms/step | 48.6 samples/s +Step 31290/44000 | Loss: 2.8187 | LR: 9.92e-05 | 410.1ms/step | 48.8 samples/s +Step 31300/44000 | Loss: 2.8274 | LR: 9.90e-05 | 406.1ms/step | 49.2 samples/s +Step 31310/44000 | Loss: 2.8411 | LR: 9.89e-05 | 406.5ms/step | 49.2 samples/s +Step 31320/44000 | Loss: 2.8521 | LR: 9.88e-05 | 406.3ms/step | 49.2 samples/s +Step 31330/44000 | Loss: 2.8252 | LR: 9.86e-05 | 407.0ms/step | 49.1 samples/s +Step 31340/44000 | Loss: 2.8623 | LR: 9.85e-05 | 406.7ms/step | 49.2 samples/s +Step 31350/44000 | Loss: 2.8716 | LR: 9.83e-05 | 407.1ms/step | 49.1 samples/s +Step 31360/44000 | Loss: 2.8685 | LR: 9.82e-05 | 407.4ms/step | 49.1 samples/s +Step 31370/44000 | Loss: 2.9229 | LR: 9.80e-05 | 408.9ms/step | 48.9 samples/s +Step 31380/44000 | Loss: 2.8313 | LR: 9.79e-05 | 414.1ms/step | 48.3 samples/s +Step 31390/44000 | Loss: 2.9023 | LR: 9.77e-05 | 413.1ms/step | 48.4 samples/s +Step 31400/44000 | Loss: 2.8804 | LR: 9.76e-05 | 406.6ms/step | 49.2 samples/s +Step 31410/44000 | Loss: 2.8507 | LR: 9.75e-05 | 408.0ms/step | 49.0 samples/s +Step 31420/44000 | Loss: 2.8700 | LR: 9.73e-05 | 406.6ms/step | 49.2 samples/s +Step 31430/44000 | Loss: 2.8063 | LR: 9.72e-05 | 406.3ms/step | 49.2 samples/s +Step 31440/44000 | Loss: 2.8649 | LR: 9.70e-05 | 407.1ms/step | 49.1 samples/s +Step 31450/44000 | Loss: 2.8216 | LR: 9.69e-05 | 406.8ms/step | 49.2 samples/s +Step 31460/44000 | Loss: 2.8506 | LR: 9.67e-05 | 406.7ms/step | 49.2 samples/s +Step 31470/44000 | Loss: 2.8389 | LR: 9.66e-05 | 407.5ms/step | 49.1 samples/s +Step 31480/44000 | Loss: 2.8780 | LR: 9.65e-05 | 411.1ms/step | 48.7 samples/s +Step 31490/44000 | Loss: 2.7917 | LR: 9.63e-05 | 412.1ms/step | 48.5 samples/s +Step 31500/44000 | Loss: 2.8192 | LR: 9.62e-05 | 407.6ms/step | 49.1 samples/s +Validation at step 31500: Loss = 2.9036, Perplexity = 18.24 +Step 31510/44000 | Loss: 2.8370 | LR: 9.60e-05 | 560.2ms/step | 35.7 samples/s +Step 31520/44000 | Loss: 2.8986 | LR: 9.59e-05 | 407.1ms/step | 49.1 samples/s +Step 31530/44000 | Loss: 2.9103 | LR: 9.57e-05 | 409.3ms/step | 48.9 samples/s +Step 31540/44000 | Loss: 2.8739 | LR: 9.56e-05 | 407.9ms/step | 49.0 samples/s +Step 31550/44000 | Loss: 2.8896 | LR: 9.54e-05 | 407.3ms/step | 49.1 samples/s +Step 31560/44000 | Loss: 2.8591 | LR: 9.53e-05 | 407.5ms/step | 49.1 samples/s +Step 31570/44000 | Loss: 2.8314 | LR: 9.52e-05 | 408.0ms/step | 49.0 samples/s +Step 31580/44000 | Loss: 2.8877 | LR: 9.50e-05 | 416.5ms/step | 48.0 samples/s +Step 31590/44000 | Loss: 2.8537 | LR: 9.49e-05 | 415.8ms/step | 48.1 samples/s +Step 31600/44000 | Loss: 2.8567 | LR: 9.47e-05 | 408.8ms/step | 48.9 samples/s +Step 31610/44000 | Loss: 2.8603 | LR: 9.46e-05 | 407.0ms/step | 49.1 samples/s +Step 31620/44000 | Loss: 2.8491 | LR: 9.45e-05 | 406.8ms/step | 49.2 samples/s +Step 31630/44000 | Loss: 2.8729 | LR: 9.43e-05 | 407.3ms/step | 49.1 samples/s +Step 31640/44000 | Loss: 2.8810 | LR: 9.42e-05 | 406.5ms/step | 49.2 samples/s +Step 31650/44000 | Loss: 2.8418 | LR: 9.40e-05 | 408.0ms/step | 49.0 samples/s +Step 31660/44000 | Loss: 2.8145 | LR: 9.39e-05 | 407.1ms/step | 49.1 samples/s +Step 31670/44000 | Loss: 2.8559 | LR: 9.37e-05 | 408.0ms/step | 49.0 samples/s +Step 31680/44000 | Loss: 2.8886 | LR: 9.36e-05 | 409.3ms/step | 48.9 samples/s +Step 31690/44000 | Loss: 2.9447 | LR: 9.35e-05 | 411.1ms/step | 48.7 samples/s +Step 31700/44000 | Loss: 2.8510 | LR: 9.33e-05 | 411.1ms/step | 48.6 samples/s +Step 31710/44000 | Loss: 2.8679 | LR: 9.32e-05 | 407.5ms/step | 49.1 samples/s +Step 31720/44000 | Loss: 2.8296 | LR: 9.30e-05 | 407.6ms/step | 49.1 samples/s +Step 31730/44000 | Loss: 2.8844 | LR: 9.29e-05 | 406.9ms/step | 49.2 samples/s +Step 31740/44000 | Loss: 2.8490 | LR: 9.28e-05 | 407.4ms/step | 49.1 samples/s +Step 31750/44000 | Loss: 2.8300 | LR: 9.26e-05 | 406.3ms/step | 49.2 samples/s +Validation at step 31750: Loss = 2.9030, Perplexity = 18.23 +Step 31760/44000 | Loss: 2.8630 | LR: 9.25e-05 | 564.2ms/step | 35.4 samples/s +Step 31770/44000 | Loss: 2.8715 | LR: 9.23e-05 | 407.7ms/step | 49.1 samples/s +Step 31780/44000 | Loss: 2.8821 | LR: 9.22e-05 | 409.9ms/step | 48.8 samples/s +Step 31790/44000 | Loss: 2.8549 | LR: 9.20e-05 | 411.1ms/step | 48.6 samples/s +Step 31800/44000 | Loss: 2.8413 | LR: 9.19e-05 | 410.4ms/step | 48.7 samples/s +Step 31810/44000 | Loss: 2.8649 | LR: 9.18e-05 | 407.2ms/step | 49.1 samples/s +Step 31820/44000 | Loss: 2.8108 | LR: 9.16e-05 | 404.6ms/step | 49.4 samples/s +Step 31830/44000 | Loss: 2.8798 | LR: 9.15e-05 | 406.8ms/step | 49.2 samples/s +Step 31840/44000 | Loss: 2.9100 | LR: 9.13e-05 | 407.2ms/step | 49.1 samples/s +Step 31850/44000 | Loss: 2.8178 | LR: 9.12e-05 | 404.7ms/step | 49.4 samples/s +Step 31860/44000 | Loss: 2.8262 | LR: 9.11e-05 | 404.6ms/step | 49.4 samples/s +Step 31870/44000 | Loss: 2.8262 | LR: 9.09e-05 | 406.2ms/step | 49.2 samples/s +Step 31880/44000 | Loss: 2.8404 | LR: 9.08e-05 | 408.2ms/step | 49.0 samples/s +Step 31890/44000 | Loss: 2.8453 | LR: 9.06e-05 | 411.9ms/step | 48.6 samples/s +Step 31900/44000 | Loss: 2.8721 | LR: 9.05e-05 | 408.1ms/step | 49.0 samples/s +Step 31910/44000 | Loss: 2.8435 | LR: 9.04e-05 | 407.4ms/step | 49.1 samples/s +Step 31920/44000 | Loss: 2.8921 | LR: 9.02e-05 | 406.9ms/step | 49.1 samples/s +Step 31930/44000 | Loss: 2.8233 | LR: 9.01e-05 | 407.2ms/step | 49.1 samples/s +Step 31940/44000 | Loss: 2.8229 | LR: 8.99e-05 | 407.1ms/step | 49.1 samples/s +Step 31950/44000 | Loss: 2.8274 | LR: 8.98e-05 | 407.3ms/step | 49.1 samples/s +Step 31960/44000 | Loss: 2.8764 | LR: 8.97e-05 | 406.6ms/step | 49.2 samples/s +Step 31970/44000 | Loss: 2.8764 | LR: 8.95e-05 | 406.5ms/step | 49.2 samples/s +Step 31980/44000 | Loss: 2.8771 | LR: 8.94e-05 | 406.6ms/step | 49.2 samples/s +Step 31990/44000 | Loss: 2.8512 | LR: 8.92e-05 | 408.3ms/step | 49.0 samples/s +Step 32000/44000 | Loss: 2.8132 | LR: 8.91e-05 | 409.3ms/step | 48.9 samples/s +Validation at step 32000: Loss = 2.9025, Perplexity = 18.22 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-32000 +Step 32010/44000 | Loss: 2.8254 | LR: 8.90e-05 | 885.4ms/step | 22.6 samples/s +Step 32020/44000 | Loss: 2.8770 | LR: 8.88e-05 | 406.0ms/step | 49.3 samples/s +Step 32030/44000 | Loss: 2.9258 | LR: 8.87e-05 | 405.6ms/step | 49.3 samples/s +Step 32040/44000 | Loss: 2.9022 | LR: 8.86e-05 | 406.3ms/step | 49.2 samples/s +Step 32050/44000 | Loss: 2.8630 | LR: 8.84e-05 | 406.6ms/step | 49.2 samples/s +Step 32060/44000 | Loss: 2.8479 | LR: 8.83e-05 | 404.8ms/step | 49.4 samples/s +Step 32070/44000 | Loss: 2.8304 | LR: 8.81e-05 | 405.2ms/step | 49.4 samples/s +Step 32080/44000 | Loss: 2.8724 | LR: 8.80e-05 | 406.6ms/step | 49.2 samples/s +Step 32090/44000 | Loss: 2.8747 | LR: 8.79e-05 | 411.6ms/step | 48.6 samples/s +Step 32100/44000 | Loss: 2.8460 | LR: 8.77e-05 | 409.6ms/step | 48.8 samples/s +Step 32110/44000 | Loss: 2.8866 | LR: 8.76e-05 | 406.3ms/step | 49.2 samples/s +Step 32120/44000 | Loss: 2.8168 | LR: 8.74e-05 | 407.6ms/step | 49.1 samples/s +Step 32130/44000 | Loss: 2.9082 | LR: 8.73e-05 | 406.3ms/step | 49.2 samples/s +Step 32140/44000 | Loss: 2.8531 | LR: 8.72e-05 | 405.6ms/step | 49.3 samples/s +Step 32150/44000 | Loss: 2.8277 | LR: 8.70e-05 | 406.0ms/step | 49.3 samples/s +Step 32160/44000 | Loss: 2.8378 | LR: 8.69e-05 | 406.3ms/step | 49.2 samples/s +Step 32170/44000 | Loss: 2.8910 | LR: 8.68e-05 | 405.5ms/step | 49.3 samples/s +Step 32180/44000 | Loss: 2.7959 | LR: 8.66e-05 | 406.3ms/step | 49.2 samples/s +Step 32190/44000 | Loss: 2.8322 | LR: 8.65e-05 | 411.8ms/step | 48.6 samples/s +Step 32200/44000 | Loss: 2.8979 | LR: 8.63e-05 | 410.6ms/step | 48.7 samples/s +Step 32210/44000 | Loss: 2.8493 | LR: 8.62e-05 | 405.9ms/step | 49.3 samples/s +Step 32220/44000 | Loss: 2.8871 | LR: 8.61e-05 | 406.0ms/step | 49.3 samples/s +Step 32230/44000 | Loss: 2.8876 | LR: 8.59e-05 | 405.0ms/step | 49.4 samples/s +Step 32240/44000 | Loss: 2.9273 | LR: 8.58e-05 | 404.4ms/step | 49.5 samples/s +Step 32250/44000 | Loss: 2.8876 | LR: 8.57e-05 | 404.2ms/step | 49.5 samples/s +Validation at step 32250: Loss = 2.9019, Perplexity = 18.21 +Step 32260/44000 | Loss: 2.8593 | LR: 8.55e-05 | 562.2ms/step | 35.6 samples/s +Step 32270/44000 | Loss: 2.8065 | LR: 8.54e-05 | 405.0ms/step | 49.4 samples/s +Step 32280/44000 | Loss: 2.8746 | LR: 8.53e-05 | 406.1ms/step | 49.3 samples/s +Step 32290/44000 | Loss: 2.8491 | LR: 8.51e-05 | 411.4ms/step | 48.6 samples/s +Step 32300/44000 | Loss: 2.8502 | LR: 8.50e-05 | 410.0ms/step | 48.8 samples/s +Step 32310/44000 | Loss: 2.8383 | LR: 8.48e-05 | 407.8ms/step | 49.0 samples/s +Step 32320/44000 | Loss: 2.8923 | LR: 8.47e-05 | 404.5ms/step | 49.4 samples/s +Step 32330/44000 | Loss: 2.9045 | LR: 8.46e-05 | 405.4ms/step | 49.3 samples/s +Step 32340/44000 | Loss: 2.7989 | LR: 8.44e-05 | 404.5ms/step | 49.4 samples/s +Step 32350/44000 | Loss: 2.9276 | LR: 8.43e-05 | 405.4ms/step | 49.3 samples/s +Step 32360/44000 | Loss: 2.7921 | LR: 8.42e-05 | 404.5ms/step | 49.4 samples/s +Step 32370/44000 | Loss: 2.8590 | LR: 8.40e-05 | 405.6ms/step | 49.3 samples/s +Step 32380/44000 | Loss: 2.8458 | LR: 8.39e-05 | 403.9ms/step | 49.5 samples/s +Step 32390/44000 | Loss: 2.8686 | LR: 8.38e-05 | 405.1ms/step | 49.4 samples/s +Step 32400/44000 | Loss: 2.8471 | LR: 8.36e-05 | 406.9ms/step | 49.2 samples/s +Step 32410/44000 | Loss: 2.8482 | LR: 8.35e-05 | 406.6ms/step | 49.2 samples/s +Step 32420/44000 | Loss: 2.8482 | LR: 8.33e-05 | 405.4ms/step | 49.3 samples/s +Step 32430/44000 | Loss: 2.8239 | LR: 8.32e-05 | 405.5ms/step | 49.3 samples/s +Step 32440/44000 | Loss: 2.9024 | LR: 8.31e-05 | 404.6ms/step | 49.4 samples/s +Step 32450/44000 | Loss: 2.9075 | LR: 8.29e-05 | 404.1ms/step | 49.5 samples/s +Step 32460/44000 | Loss: 2.8855 | LR: 8.28e-05 | 404.3ms/step | 49.5 samples/s +Step 32470/44000 | Loss: 2.8689 | LR: 8.27e-05 | 405.4ms/step | 49.3 samples/s +Step 32480/44000 | Loss: 2.9277 | LR: 8.25e-05 | 404.6ms/step | 49.4 samples/s +Step 32490/44000 | Loss: 2.8673 | LR: 8.24e-05 | 403.7ms/step | 49.5 samples/s +Step 32500/44000 | Loss: 2.8536 | LR: 8.23e-05 | 407.0ms/step | 49.1 samples/s +Validation at step 32500: Loss = 2.9012, Perplexity = 18.20 +Step 32510/44000 | Loss: 2.7577 | LR: 8.21e-05 | 561.2ms/step | 35.6 samples/s +Step 32520/44000 | Loss: 2.9290 | LR: 8.20e-05 | 405.7ms/step | 49.3 samples/s +Step 32530/44000 | Loss: 2.8719 | LR: 8.19e-05 | 404.1ms/step | 49.5 samples/s +Step 32540/44000 | Loss: 2.8320 | LR: 8.17e-05 | 405.5ms/step | 49.3 samples/s +Step 32550/44000 | Loss: 2.8291 | LR: 8.16e-05 | 406.2ms/step | 49.2 samples/s +Step 32560/44000 | Loss: 2.8981 | LR: 8.15e-05 | 405.3ms/step | 49.3 samples/s +Step 32570/44000 | Loss: 2.8959 | LR: 8.13e-05 | 405.0ms/step | 49.4 samples/s +Step 32580/44000 | Loss: 2.8383 | LR: 8.12e-05 | 405.1ms/step | 49.4 samples/s +Step 32590/44000 | Loss: 2.8536 | LR: 8.11e-05 | 404.5ms/step | 49.4 samples/s +Step 32600/44000 | Loss: 2.9079 | LR: 8.09e-05 | 410.0ms/step | 48.8 samples/s +Step 32610/44000 | Loss: 2.8081 | LR: 8.08e-05 | 410.8ms/step | 48.7 samples/s +Step 32620/44000 | Loss: 2.8492 | LR: 8.07e-05 | 407.9ms/step | 49.0 samples/s +Step 32630/44000 | Loss: 2.8795 | LR: 8.05e-05 | 405.8ms/step | 49.3 samples/s +Step 32640/44000 | Loss: 2.8195 | LR: 8.04e-05 | 404.2ms/step | 49.5 samples/s +Step 32650/44000 | Loss: 2.7999 | LR: 8.03e-05 | 405.6ms/step | 49.3 samples/s +Step 32660/44000 | Loss: 2.8114 | LR: 8.01e-05 | 404.7ms/step | 49.4 samples/s +Step 32670/44000 | Loss: 2.8031 | LR: 8.00e-05 | 404.6ms/step | 49.4 samples/s +Step 32680/44000 | Loss: 2.8797 | LR: 7.99e-05 | 404.8ms/step | 49.4 samples/s +Step 32690/44000 | Loss: 2.7905 | LR: 7.97e-05 | 406.1ms/step | 49.3 samples/s +Step 32700/44000 | Loss: 2.7961 | LR: 7.96e-05 | 407.4ms/step | 49.1 samples/s +Step 32710/44000 | Loss: 2.8876 | LR: 7.95e-05 | 408.5ms/step | 49.0 samples/s +Step 32720/44000 | Loss: 2.8499 | LR: 7.93e-05 | 406.9ms/step | 49.2 samples/s +Step 32730/44000 | Loss: 2.8976 | LR: 7.92e-05 | 404.1ms/step | 49.5 samples/s +Step 32740/44000 | Loss: 2.8943 | LR: 7.91e-05 | 404.6ms/step | 49.4 samples/s +Step 32750/44000 | Loss: 2.8470 | LR: 7.89e-05 | 405.9ms/step | 49.3 samples/s +Validation at step 32750: Loss = 2.9011, Perplexity = 18.19 +Step 32760/44000 | Loss: 2.8444 | LR: 7.88e-05 | 558.3ms/step | 35.8 samples/s +Step 32770/44000 | Loss: 2.8426 | LR: 7.87e-05 | 405.8ms/step | 49.3 samples/s +Step 32780/44000 | Loss: 2.8880 | LR: 7.85e-05 | 405.4ms/step | 49.3 samples/s +Step 32790/44000 | Loss: 2.8526 | LR: 7.84e-05 | 404.9ms/step | 49.4 samples/s +Step 32800/44000 | Loss: 2.8428 | LR: 7.83e-05 | 406.3ms/step | 49.2 samples/s +Step 32810/44000 | Loss: 2.9057 | LR: 7.81e-05 | 411.7ms/step | 48.6 samples/s +Step 32820/44000 | Loss: 2.8546 | LR: 7.80e-05 | 409.0ms/step | 48.9 samples/s +Step 32830/44000 | Loss: 2.8903 | LR: 7.79e-05 | 405.3ms/step | 49.3 samples/s +Step 32840/44000 | Loss: 2.8139 | LR: 7.77e-05 | 405.9ms/step | 49.3 samples/s +Step 32850/44000 | Loss: 2.9045 | LR: 7.76e-05 | 404.5ms/step | 49.4 samples/s +Step 32860/44000 | Loss: 2.8387 | LR: 7.75e-05 | 406.7ms/step | 49.2 samples/s +Step 32870/44000 | Loss: 2.9016 | LR: 7.73e-05 | 404.4ms/step | 49.5 samples/s +Step 32880/44000 | Loss: 2.8525 | LR: 7.72e-05 | 403.7ms/step | 49.5 samples/s +Step 32890/44000 | Loss: 2.8558 | LR: 7.71e-05 | 404.6ms/step | 49.4 samples/s +Step 32900/44000 | Loss: 2.8252 | LR: 7.70e-05 | 404.6ms/step | 49.4 samples/s +Step 32910/44000 | Loss: 2.9191 | LR: 7.68e-05 | 409.6ms/step | 48.8 samples/s +Step 32920/44000 | Loss: 2.8620 | LR: 7.67e-05 | 409.3ms/step | 48.9 samples/s +Step 32930/44000 | Loss: 2.8289 | LR: 7.66e-05 | 404.3ms/step | 49.5 samples/s +Step 32940/44000 | Loss: 2.8680 | LR: 7.64e-05 | 403.9ms/step | 49.5 samples/s +Step 32950/44000 | Loss: 2.8274 | LR: 7.63e-05 | 403.8ms/step | 49.5 samples/s +Step 32960/44000 | Loss: 2.8334 | LR: 7.62e-05 | 404.6ms/step | 49.4 samples/s +Step 32970/44000 | Loss: 2.8772 | LR: 7.60e-05 | 404.4ms/step | 49.5 samples/s +Step 32980/44000 | Loss: 2.8792 | LR: 7.59e-05 | 404.4ms/step | 49.5 samples/s +Step 32990/44000 | Loss: 2.8411 | LR: 7.58e-05 | 405.0ms/step | 49.4 samples/s +Step 33000/44000 | Loss: 2.8762 | LR: 7.56e-05 | 405.3ms/step | 49.4 samples/s +Validation at step 33000: Loss = 2.9006, Perplexity = 18.18 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-33000 +Step 33010/44000 | Loss: 2.9347 | LR: 7.55e-05 | 933.5ms/step | 21.4 samples/s +Step 33020/44000 | Loss: 2.8922 | LR: 7.54e-05 | 404.5ms/step | 49.4 samples/s +Step 33030/44000 | Loss: 2.8234 | LR: 7.53e-05 | 404.2ms/step | 49.5 samples/s +Step 33040/44000 | Loss: 2.8930 | LR: 7.51e-05 | 404.6ms/step | 49.4 samples/s +Step 33050/44000 | Loss: 2.8460 | LR: 7.50e-05 | 405.6ms/step | 49.3 samples/s +Step 33060/44000 | Loss: 2.7976 | LR: 7.49e-05 | 404.1ms/step | 49.5 samples/s +Step 33070/44000 | Loss: 2.9410 | LR: 7.47e-05 | 402.8ms/step | 49.6 samples/s +Step 33080/44000 | Loss: 2.8198 | LR: 7.46e-05 | 403.7ms/step | 49.5 samples/s +Step 33090/44000 | Loss: 2.8743 | LR: 7.45e-05 | 404.7ms/step | 49.4 samples/s +Step 33100/44000 | Loss: 2.8380 | LR: 7.44e-05 | 407.0ms/step | 49.1 samples/s +Step 33110/44000 | Loss: 2.8442 | LR: 7.42e-05 | 407.3ms/step | 49.1 samples/s +Step 33120/44000 | Loss: 2.8762 | LR: 7.41e-05 | 407.0ms/step | 49.1 samples/s +Step 33130/44000 | Loss: 2.8762 | LR: 7.40e-05 | 404.6ms/step | 49.4 samples/s +Step 33140/44000 | Loss: 2.8456 | LR: 7.38e-05 | 404.7ms/step | 49.4 samples/s +Step 33150/44000 | Loss: 2.8518 | LR: 7.37e-05 | 403.8ms/step | 49.5 samples/s +Step 33160/44000 | Loss: 2.8426 | LR: 7.36e-05 | 405.3ms/step | 49.3 samples/s +Step 33170/44000 | Loss: 2.8857 | LR: 7.34e-05 | 404.4ms/step | 49.5 samples/s +Step 33180/44000 | Loss: 2.8412 | LR: 7.33e-05 | 405.1ms/step | 49.4 samples/s +Step 33190/44000 | Loss: 2.8821 | LR: 7.32e-05 | 404.5ms/step | 49.4 samples/s +Step 33200/44000 | Loss: 2.8311 | LR: 7.31e-05 | 406.9ms/step | 49.2 samples/s +Step 33210/44000 | Loss: 2.8549 | LR: 7.29e-05 | 408.6ms/step | 48.9 samples/s +Step 33220/44000 | Loss: 2.8390 | LR: 7.28e-05 | 409.1ms/step | 48.9 samples/s +Step 33230/44000 | Loss: 2.8406 | LR: 7.27e-05 | 405.2ms/step | 49.4 samples/s +Step 33240/44000 | Loss: 2.8255 | LR: 7.26e-05 | 404.6ms/step | 49.4 samples/s +Step 33250/44000 | Loss: 2.8850 | LR: 7.24e-05 | 404.8ms/step | 49.4 samples/s +Validation at step 33250: Loss = 2.9004, Perplexity = 18.18 +Step 33260/44000 | Loss: 2.8156 | LR: 7.23e-05 | 556.9ms/step | 35.9 samples/s +Step 33270/44000 | Loss: 2.8298 | LR: 7.22e-05 | 407.2ms/step | 49.1 samples/s +Step 33280/44000 | Loss: 2.8432 | LR: 7.20e-05 | 404.8ms/step | 49.4 samples/s +Step 33290/44000 | Loss: 2.7823 | LR: 7.19e-05 | 405.2ms/step | 49.4 samples/s +Step 33300/44000 | Loss: 2.8660 | LR: 7.18e-05 | 404.1ms/step | 49.5 samples/s +Step 33310/44000 | Loss: 2.8392 | LR: 7.17e-05 | 409.5ms/step | 48.8 samples/s +Step 33320/44000 | Loss: 2.8473 | LR: 7.15e-05 | 408.6ms/step | 49.0 samples/s +Step 33330/44000 | Loss: 2.8424 | LR: 7.14e-05 | 407.4ms/step | 49.1 samples/s +Step 33340/44000 | Loss: 2.7581 | LR: 7.13e-05 | 404.7ms/step | 49.4 samples/s +Step 33350/44000 | Loss: 2.7793 | LR: 7.11e-05 | 404.6ms/step | 49.4 samples/s +Step 33360/44000 | Loss: 2.8339 | LR: 7.10e-05 | 405.5ms/step | 49.3 samples/s +Step 33370/44000 | Loss: 2.8835 | LR: 7.09e-05 | 405.5ms/step | 49.3 samples/s +Step 33380/44000 | Loss: 2.9302 | LR: 7.08e-05 | 406.1ms/step | 49.2 samples/s +Step 33390/44000 | Loss: 2.8299 | LR: 7.06e-05 | 405.5ms/step | 49.3 samples/s +Step 33400/44000 | Loss: 2.8791 | LR: 7.05e-05 | 404.6ms/step | 49.4 samples/s +Step 33410/44000 | Loss: 2.8619 | LR: 7.04e-05 | 408.9ms/step | 48.9 samples/s +Step 33420/44000 | Loss: 2.8846 | LR: 7.03e-05 | 410.8ms/step | 48.7 samples/s +Step 33430/44000 | Loss: 2.8596 | LR: 7.01e-05 | 407.5ms/step | 49.1 samples/s +Step 33440/44000 | Loss: 2.8937 | LR: 7.00e-05 | 404.7ms/step | 49.4 samples/s +Step 33450/44000 | Loss: 2.8353 | LR: 6.99e-05 | 405.6ms/step | 49.3 samples/s +Step 33460/44000 | Loss: 2.8544 | LR: 6.98e-05 | 404.9ms/step | 49.4 samples/s +Step 33470/44000 | Loss: 2.8113 | LR: 6.96e-05 | 405.5ms/step | 49.3 samples/s +Step 33480/44000 | Loss: 2.8302 | LR: 6.95e-05 | 405.7ms/step | 49.3 samples/s +Step 33490/44000 | Loss: 2.8469 | LR: 6.94e-05 | 406.0ms/step | 49.3 samples/s +Step 33500/44000 | Loss: 2.8107 | LR: 6.93e-05 | 405.1ms/step | 49.4 samples/s +Validation at step 33500: Loss = 2.8998, Perplexity = 18.17 +Step 33510/44000 | Loss: 2.8314 | LR: 6.91e-05 | 561.5ms/step | 35.6 samples/s +Step 33520/44000 | Loss: 2.9088 | LR: 6.90e-05 | 409.4ms/step | 48.9 samples/s +Step 33530/44000 | Loss: 2.8359 | LR: 6.89e-05 | 407.6ms/step | 49.1 samples/s +Step 33540/44000 | Loss: 2.9449 | LR: 6.88e-05 | 404.8ms/step | 49.4 samples/s +Step 33550/44000 | Loss: 2.8171 | LR: 6.86e-05 | 404.9ms/step | 49.4 samples/s +Step 33560/44000 | Loss: 2.8852 | LR: 6.85e-05 | 404.1ms/step | 49.5 samples/s +Step 33570/44000 | Loss: 2.8514 | LR: 6.84e-05 | 404.2ms/step | 49.5 samples/s +Step 33580/44000 | Loss: 2.8889 | LR: 6.83e-05 | 405.2ms/step | 49.4 samples/s +Step 33590/44000 | Loss: 2.8964 | LR: 6.81e-05 | 406.1ms/step | 49.2 samples/s +Step 33600/44000 | Loss: 2.8467 | LR: 6.80e-05 | 404.3ms/step | 49.5 samples/s +Step 33610/44000 | Loss: 2.8572 | LR: 6.79e-05 | 406.1ms/step | 49.2 samples/s +Step 33620/44000 | Loss: 2.8326 | LR: 6.78e-05 | 409.8ms/step | 48.8 samples/s +Step 33630/44000 | Loss: 2.8718 | LR: 6.76e-05 | 407.5ms/step | 49.1 samples/s +Step 33640/44000 | Loss: 2.8599 | LR: 6.75e-05 | 405.3ms/step | 49.3 samples/s +Step 33650/44000 | Loss: 2.8700 | LR: 6.74e-05 | 404.4ms/step | 49.5 samples/s +Step 33660/44000 | Loss: 2.8325 | LR: 6.73e-05 | 405.1ms/step | 49.4 samples/s +Step 33670/44000 | Loss: 2.8289 | LR: 6.71e-05 | 404.6ms/step | 49.4 samples/s +Step 33680/44000 | Loss: 2.8978 | LR: 6.70e-05 | 405.2ms/step | 49.4 samples/s +Step 33690/44000 | Loss: 2.8336 | LR: 6.69e-05 | 405.2ms/step | 49.4 samples/s +Step 33700/44000 | Loss: 2.8504 | LR: 6.68e-05 | 404.6ms/step | 49.4 samples/s +Step 33710/44000 | Loss: 2.8589 | LR: 6.66e-05 | 403.8ms/step | 49.5 samples/s +Step 33720/44000 | Loss: 2.8410 | LR: 6.65e-05 | 410.6ms/step | 48.7 samples/s +Step 33730/44000 | Loss: 2.8313 | LR: 6.64e-05 | 409.6ms/step | 48.8 samples/s +Step 33740/44000 | Loss: 2.8203 | LR: 6.63e-05 | 405.2ms/step | 49.4 samples/s +Step 33750/44000 | Loss: 2.8151 | LR: 6.61e-05 | 404.2ms/step | 49.5 samples/s +Validation at step 33750: Loss = 2.8997, Perplexity = 18.17 +Step 33760/44000 | Loss: 2.8550 | LR: 6.60e-05 | 556.8ms/step | 35.9 samples/s +Step 33770/44000 | Loss: 2.8409 | LR: 6.59e-05 | 403.8ms/step | 49.5 samples/s +Step 33780/44000 | Loss: 2.8930 | LR: 6.58e-05 | 404.3ms/step | 49.5 samples/s +Step 33790/44000 | Loss: 2.8662 | LR: 6.57e-05 | 405.6ms/step | 49.3 samples/s +Step 33800/44000 | Loss: 2.8630 | LR: 6.55e-05 | 404.5ms/step | 49.4 samples/s +Step 33810/44000 | Loss: 2.8953 | LR: 6.54e-05 | 405.0ms/step | 49.4 samples/s +Step 33820/44000 | Loss: 2.8764 | LR: 6.53e-05 | 407.9ms/step | 49.0 samples/s +Step 33830/44000 | Loss: 2.9192 | LR: 6.52e-05 | 408.3ms/step | 49.0 samples/s +Step 33840/44000 | Loss: 2.8233 | LR: 6.50e-05 | 407.7ms/step | 49.1 samples/s +Step 33850/44000 | Loss: 2.8884 | LR: 6.49e-05 | 407.6ms/step | 49.1 samples/s +Step 33860/44000 | Loss: 2.8462 | LR: 6.48e-05 | 407.9ms/step | 49.0 samples/s +Step 33870/44000 | Loss: 2.8216 | LR: 6.47e-05 | 404.6ms/step | 49.4 samples/s +Step 33880/44000 | Loss: 2.8580 | LR: 6.46e-05 | 403.9ms/step | 49.5 samples/s +Step 33890/44000 | Loss: 2.8569 | LR: 6.44e-05 | 404.0ms/step | 49.5 samples/s +Step 33900/44000 | Loss: 2.8366 | LR: 6.43e-05 | 405.2ms/step | 49.4 samples/s +Step 33910/44000 | Loss: 2.8469 | LR: 6.42e-05 | 404.8ms/step | 49.4 samples/s +Step 33920/44000 | Loss: 2.7954 | LR: 6.41e-05 | 405.6ms/step | 49.3 samples/s +Step 33930/44000 | Loss: 2.8611 | LR: 6.40e-05 | 403.9ms/step | 49.5 samples/s +Step 33940/44000 | Loss: 2.8201 | LR: 6.38e-05 | 405.7ms/step | 49.3 samples/s +Step 33950/44000 | Loss: 2.8439 | LR: 6.37e-05 | 409.9ms/step | 48.8 samples/s +Step 33960/44000 | Loss: 2.8084 | LR: 6.36e-05 | 408.2ms/step | 49.0 samples/s +Step 33970/44000 | Loss: 2.8897 | LR: 6.35e-05 | 407.8ms/step | 49.0 samples/s +Step 33980/44000 | Loss: 2.7611 | LR: 6.33e-05 | 405.0ms/step | 49.4 samples/s +Step 33990/44000 | Loss: 2.8162 | LR: 6.32e-05 | 405.0ms/step | 49.4 samples/s +Step 34000/44000 | Loss: 2.8500 | LR: 6.31e-05 | 405.3ms/step | 49.3 samples/s +Validation at step 34000: Loss = 2.8992, Perplexity = 18.16 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-34000 +Step 34010/44000 | Loss: 2.8309 | LR: 6.30e-05 | 871.8ms/step | 22.9 samples/s +Step 34020/44000 | Loss: 2.8667 | LR: 6.29e-05 | 404.8ms/step | 49.4 samples/s +Step 34030/44000 | Loss: 2.8767 | LR: 6.27e-05 | 406.5ms/step | 49.2 samples/s +Step 34040/44000 | Loss: 2.8854 | LR: 6.26e-05 | 409.7ms/step | 48.8 samples/s +Step 34050/44000 | Loss: 2.8442 | LR: 6.25e-05 | 407.0ms/step | 49.1 samples/s +Step 34060/44000 | Loss: 2.8509 | LR: 6.24e-05 | 404.1ms/step | 49.5 samples/s +Step 34070/44000 | Loss: 2.8313 | LR: 6.23e-05 | 404.3ms/step | 49.5 samples/s +Step 34080/44000 | Loss: 2.8533 | LR: 6.21e-05 | 405.6ms/step | 49.3 samples/s +Step 34090/44000 | Loss: 2.7712 | LR: 6.20e-05 | 404.7ms/step | 49.4 samples/s +Step 34100/44000 | Loss: 2.8648 | LR: 6.19e-05 | 405.8ms/step | 49.3 samples/s +Step 34110/44000 | Loss: 2.8799 | LR: 6.18e-05 | 404.4ms/step | 49.5 samples/s +Step 34120/44000 | Loss: 2.8587 | LR: 6.17e-05 | 405.2ms/step | 49.4 samples/s +Step 34130/44000 | Loss: 2.8208 | LR: 6.15e-05 | 405.2ms/step | 49.4 samples/s +Step 34140/44000 | Loss: 2.8012 | LR: 6.14e-05 | 411.3ms/step | 48.6 samples/s +Step 34150/44000 | Loss: 2.7791 | LR: 6.13e-05 | 409.6ms/step | 48.8 samples/s +Step 34160/44000 | Loss: 2.8895 | LR: 6.12e-05 | 405.0ms/step | 49.4 samples/s +Step 34170/44000 | Loss: 2.8550 | LR: 6.11e-05 | 404.8ms/step | 49.4 samples/s +Step 34180/44000 | Loss: 2.8241 | LR: 6.09e-05 | 405.2ms/step | 49.4 samples/s +Step 34190/44000 | Loss: 2.8829 | LR: 6.08e-05 | 403.7ms/step | 49.5 samples/s +Step 34200/44000 | Loss: 2.8726 | LR: 6.07e-05 | 405.1ms/step | 49.4 samples/s +Step 34210/44000 | Loss: 2.8373 | LR: 6.06e-05 | 404.2ms/step | 49.5 samples/s +Step 34220/44000 | Loss: 2.8432 | LR: 6.05e-05 | 405.4ms/step | 49.3 samples/s +Step 34230/44000 | Loss: 2.9117 | LR: 6.04e-05 | 404.7ms/step | 49.4 samples/s +Step 34240/44000 | Loss: 2.8321 | LR: 6.02e-05 | 410.6ms/step | 48.7 samples/s +Step 34250/44000 | Loss: 2.8258 | LR: 6.01e-05 | 410.4ms/step | 48.7 samples/s +Validation at step 34250: Loss = 2.8992, Perplexity = 18.16 +Step 34260/44000 | Loss: 2.8499 | LR: 6.00e-05 | 563.4ms/step | 35.5 samples/s +Step 34270/44000 | Loss: 2.8771 | LR: 5.99e-05 | 410.1ms/step | 48.8 samples/s +Step 34280/44000 | Loss: 2.8769 | LR: 5.98e-05 | 407.1ms/step | 49.1 samples/s +Step 34290/44000 | Loss: 2.8787 | LR: 5.96e-05 | 404.9ms/step | 49.4 samples/s +Step 34300/44000 | Loss: 2.8098 | LR: 5.95e-05 | 407.1ms/step | 49.1 samples/s +Step 34310/44000 | Loss: 2.8727 | LR: 5.94e-05 | 405.8ms/step | 49.3 samples/s +Step 34320/44000 | Loss: 2.8360 | LR: 5.93e-05 | 406.1ms/step | 49.3 samples/s +Step 34330/44000 | Loss: 2.8300 | LR: 5.92e-05 | 404.5ms/step | 49.4 samples/s +Step 34340/44000 | Loss: 2.8380 | LR: 5.91e-05 | 405.0ms/step | 49.4 samples/s +Step 34350/44000 | Loss: 2.8339 | LR: 5.89e-05 | 406.0ms/step | 49.3 samples/s +Step 34360/44000 | Loss: 2.7991 | LR: 5.88e-05 | 407.7ms/step | 49.1 samples/s +Step 34370/44000 | Loss: 2.8355 | LR: 5.87e-05 | 409.9ms/step | 48.8 samples/s +Step 34380/44000 | Loss: 2.8680 | LR: 5.86e-05 | 408.2ms/step | 49.0 samples/s +Step 34390/44000 | Loss: 2.8583 | LR: 5.85e-05 | 405.1ms/step | 49.4 samples/s +Step 34400/44000 | Loss: 2.8873 | LR: 5.84e-05 | 405.5ms/step | 49.3 samples/s +Step 34410/44000 | Loss: 2.8586 | LR: 5.82e-05 | 404.9ms/step | 49.4 samples/s +Step 34420/44000 | Loss: 2.8543 | LR: 5.81e-05 | 405.7ms/step | 49.3 samples/s +Step 34430/44000 | Loss: 2.8064 | LR: 5.80e-05 | 405.6ms/step | 49.3 samples/s +Step 34440/44000 | Loss: 2.8508 | LR: 5.79e-05 | 404.5ms/step | 49.4 samples/s +Step 34450/44000 | Loss: 2.8407 | LR: 5.78e-05 | 405.8ms/step | 49.3 samples/s +Step 34460/44000 | Loss: 2.8080 | LR: 5.77e-05 | 406.4ms/step | 49.2 samples/s +Step 34470/44000 | Loss: 2.8060 | LR: 5.75e-05 | 409.7ms/step | 48.8 samples/s +Step 34480/44000 | Loss: 2.8259 | LR: 5.74e-05 | 410.4ms/step | 48.7 samples/s +Step 34490/44000 | Loss: 2.8268 | LR: 5.73e-05 | 405.8ms/step | 49.3 samples/s +Step 34500/44000 | Loss: 2.8080 | LR: 5.72e-05 | 403.6ms/step | 49.6 samples/s +Validation at step 34500: Loss = 2.8987, Perplexity = 18.15 +Step 34510/44000 | Loss: 2.8333 | LR: 5.71e-05 | 559.8ms/step | 35.7 samples/s +Step 34520/44000 | Loss: 2.8739 | LR: 5.70e-05 | 406.2ms/step | 49.2 samples/s +Step 34530/44000 | Loss: 2.9485 | LR: 5.69e-05 | 405.3ms/step | 49.3 samples/s +Step 34540/44000 | Loss: 2.8985 | LR: 5.67e-05 | 405.4ms/step | 49.3 samples/s +Step 34550/44000 | Loss: 2.8665 | LR: 5.66e-05 | 405.9ms/step | 49.3 samples/s +Step 34560/44000 | Loss: 2.8606 | LR: 5.65e-05 | 408.0ms/step | 49.0 samples/s +Step 34570/44000 | Loss: 2.8183 | LR: 5.64e-05 | 411.7ms/step | 48.6 samples/s +Step 34580/44000 | Loss: 2.8603 | LR: 5.63e-05 | 410.4ms/step | 48.7 samples/s +Step 34590/44000 | Loss: 2.8515 | LR: 5.62e-05 | 404.7ms/step | 49.4 samples/s +Step 34600/44000 | Loss: 2.8781 | LR: 5.60e-05 | 405.0ms/step | 49.4 samples/s +Step 34610/44000 | Loss: 2.9000 | LR: 5.59e-05 | 405.5ms/step | 49.3 samples/s +Step 34620/44000 | Loss: 2.8095 | LR: 5.58e-05 | 404.2ms/step | 49.5 samples/s +Step 34630/44000 | Loss: 2.8610 | LR: 5.57e-05 | 404.6ms/step | 49.4 samples/s +Step 34640/44000 | Loss: 2.8319 | LR: 5.56e-05 | 405.5ms/step | 49.3 samples/s +Step 34650/44000 | Loss: 2.8727 | LR: 5.55e-05 | 404.6ms/step | 49.4 samples/s +Step 34660/44000 | Loss: 2.8349 | LR: 5.54e-05 | 405.2ms/step | 49.4 samples/s +Step 34670/44000 | Loss: 2.8434 | LR: 5.52e-05 | 407.9ms/step | 49.0 samples/s +Step 34680/44000 | Loss: 2.8650 | LR: 5.51e-05 | 409.5ms/step | 48.8 samples/s +Step 34690/44000 | Loss: 2.8419 | LR: 5.50e-05 | 404.4ms/step | 49.5 samples/s +Step 34700/44000 | Loss: 2.8630 | LR: 5.49e-05 | 405.9ms/step | 49.3 samples/s +Step 34710/44000 | Loss: 2.8374 | LR: 5.48e-05 | 405.7ms/step | 49.3 samples/s +Step 34720/44000 | Loss: 2.8397 | LR: 5.47e-05 | 404.4ms/step | 49.5 samples/s +Step 34730/44000 | Loss: 2.8376 | LR: 5.46e-05 | 406.3ms/step | 49.2 samples/s +Step 34740/44000 | Loss: 2.7997 | LR: 5.45e-05 | 405.6ms/step | 49.3 samples/s +Step 34750/44000 | Loss: 2.8123 | LR: 5.43e-05 | 405.3ms/step | 49.4 samples/s +Validation at step 34750: Loss = 2.8986, Perplexity = 18.15 +Step 34760/44000 | Loss: 2.8128 | LR: 5.42e-05 | 557.5ms/step | 35.9 samples/s +Step 34770/44000 | Loss: 2.9096 | LR: 5.41e-05 | 411.5ms/step | 48.6 samples/s +Step 34780/44000 | Loss: 2.8286 | LR: 5.40e-05 | 411.0ms/step | 48.7 samples/s +Step 34790/44000 | Loss: 2.8531 | LR: 5.39e-05 | 404.7ms/step | 49.4 samples/s +Step 34800/44000 | Loss: 2.8256 | LR: 5.38e-05 | 404.5ms/step | 49.4 samples/s +Step 34810/44000 | Loss: 2.8417 | LR: 5.37e-05 | 406.6ms/step | 49.2 samples/s +Step 34820/44000 | Loss: 2.7859 | LR: 5.36e-05 | 405.2ms/step | 49.4 samples/s +Step 34830/44000 | Loss: 2.8253 | LR: 5.34e-05 | 405.7ms/step | 49.3 samples/s +Step 34840/44000 | Loss: 2.8519 | LR: 5.33e-05 | 408.6ms/step | 49.0 samples/s +Step 34850/44000 | Loss: 2.8181 | LR: 5.32e-05 | 404.5ms/step | 49.4 samples/s +Step 34860/44000 | Loss: 2.8637 | LR: 5.31e-05 | 405.0ms/step | 49.4 samples/s +Step 34870/44000 | Loss: 2.8542 | LR: 5.30e-05 | 408.1ms/step | 49.0 samples/s +Step 34880/44000 | Loss: 2.8476 | LR: 5.29e-05 | 408.8ms/step | 48.9 samples/s +Step 34890/44000 | Loss: 2.8504 | LR: 5.28e-05 | 405.5ms/step | 49.3 samples/s +Step 34900/44000 | Loss: 2.8602 | LR: 5.27e-05 | 404.0ms/step | 49.5 samples/s +Step 34910/44000 | Loss: 2.8240 | LR: 5.25e-05 | 403.8ms/step | 49.5 samples/s +Step 34920/44000 | Loss: 2.8589 | LR: 5.24e-05 | 405.4ms/step | 49.3 samples/s +Step 34930/44000 | Loss: 2.8351 | LR: 5.23e-05 | 404.6ms/step | 49.4 samples/s +Step 34940/44000 | Loss: 2.8871 | LR: 5.22e-05 | 404.2ms/step | 49.5 samples/s +Step 34950/44000 | Loss: 2.7843 | LR: 5.21e-05 | 404.6ms/step | 49.4 samples/s +Step 34960/44000 | Loss: 2.8292 | LR: 5.20e-05 | 404.5ms/step | 49.4 samples/s +Step 34970/44000 | Loss: 2.8375 | LR: 5.19e-05 | 405.8ms/step | 49.3 samples/s +Step 34980/44000 | Loss: 2.8796 | LR: 5.18e-05 | 410.0ms/step | 48.8 samples/s +Step 34990/44000 | Loss: 2.9290 | LR: 5.17e-05 | 407.9ms/step | 49.0 samples/s +Step 35000/44000 | Loss: 2.8259 | LR: 5.15e-05 | 403.6ms/step | 49.6 samples/s +Validation at step 35000: Loss = 2.8983, Perplexity = 18.14 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-35000 +Step 35010/44000 | Loss: 2.8628 | LR: 5.14e-05 | 868.1ms/step | 23.0 samples/s +Step 35020/44000 | Loss: 2.7979 | LR: 5.13e-05 | 405.4ms/step | 49.3 samples/s +Step 35030/44000 | Loss: 2.8381 | LR: 5.12e-05 | 405.1ms/step | 49.4 samples/s +Step 35040/44000 | Loss: 2.8440 | LR: 5.11e-05 | 405.4ms/step | 49.3 samples/s +Step 35050/44000 | Loss: 2.8637 | LR: 5.10e-05 | 405.3ms/step | 49.3 samples/s +Step 35060/44000 | Loss: 2.9233 | LR: 5.09e-05 | 406.8ms/step | 49.2 samples/s +Step 35070/44000 | Loss: 2.8434 | LR: 5.08e-05 | 411.6ms/step | 48.6 samples/s +Step 35080/44000 | Loss: 2.8268 | LR: 5.07e-05 | 409.1ms/step | 48.9 samples/s +Step 35090/44000 | Loss: 2.8443 | LR: 5.06e-05 | 405.6ms/step | 49.3 samples/s +Step 35100/44000 | Loss: 2.8985 | LR: 5.04e-05 | 405.6ms/step | 49.3 samples/s +Step 35110/44000 | Loss: 2.8686 | LR: 5.03e-05 | 405.4ms/step | 49.3 samples/s +Step 35120/44000 | Loss: 2.8679 | LR: 5.02e-05 | 404.9ms/step | 49.4 samples/s +Step 35130/44000 | Loss: 2.8471 | LR: 5.01e-05 | 405.5ms/step | 49.3 samples/s +Step 35140/44000 | Loss: 2.8043 | LR: 5.00e-05 | 408.4ms/step | 49.0 samples/s +Step 35150/44000 | Loss: 2.8509 | LR: 4.99e-05 | 406.8ms/step | 49.2 samples/s +Step 35160/44000 | Loss: 2.8858 | LR: 4.98e-05 | 405.5ms/step | 49.3 samples/s +Step 35170/44000 | Loss: 2.8250 | LR: 4.97e-05 | 411.6ms/step | 48.6 samples/s +Step 35180/44000 | Loss: 2.8306 | LR: 4.96e-05 | 409.5ms/step | 48.8 samples/s +Step 35190/44000 | Loss: 2.8309 | LR: 4.95e-05 | 406.0ms/step | 49.3 samples/s +Step 35200/44000 | Loss: 2.8484 | LR: 4.94e-05 | 406.2ms/step | 49.2 samples/s +Step 35210/44000 | Loss: 2.8674 | LR: 4.93e-05 | 406.6ms/step | 49.2 samples/s +Step 35220/44000 | Loss: 2.8932 | LR: 4.91e-05 | 405.1ms/step | 49.4 samples/s +Step 35230/44000 | Loss: 2.8781 | LR: 4.90e-05 | 406.4ms/step | 49.2 samples/s +Step 35240/44000 | Loss: 2.9275 | LR: 4.89e-05 | 406.7ms/step | 49.2 samples/s +Step 35250/44000 | Loss: 2.8675 | LR: 4.88e-05 | 405.1ms/step | 49.4 samples/s +Validation at step 35250: Loss = 2.8981, Perplexity = 18.14 +Step 35260/44000 | Loss: 2.8711 | LR: 4.87e-05 | 558.8ms/step | 35.8 samples/s +Step 35270/44000 | Loss: 2.8757 | LR: 4.86e-05 | 409.9ms/step | 48.8 samples/s +Step 35280/44000 | Loss: 2.8176 | LR: 4.85e-05 | 409.9ms/step | 48.8 samples/s +Step 35290/44000 | Loss: 2.8640 | LR: 4.84e-05 | 405.4ms/step | 49.3 samples/s +Step 35300/44000 | Loss: 2.9222 | LR: 4.83e-05 | 405.3ms/step | 49.4 samples/s +Step 35310/44000 | Loss: 2.7957 | LR: 4.82e-05 | 405.1ms/step | 49.4 samples/s +Step 35320/44000 | Loss: 2.9108 | LR: 4.81e-05 | 405.2ms/step | 49.4 samples/s +Step 35330/44000 | Loss: 2.8771 | LR: 4.80e-05 | 407.0ms/step | 49.1 samples/s +Step 35340/44000 | Loss: 2.8393 | LR: 4.79e-05 | 405.8ms/step | 49.3 samples/s +Step 35350/44000 | Loss: 2.8529 | LR: 4.77e-05 | 405.7ms/step | 49.3 samples/s +Step 35360/44000 | Loss: 2.8710 | LR: 4.76e-05 | 405.0ms/step | 49.4 samples/s +Step 35370/44000 | Loss: 2.8206 | LR: 4.75e-05 | 410.8ms/step | 48.7 samples/s +Step 35380/44000 | Loss: 2.8813 | LR: 4.74e-05 | 410.4ms/step | 48.7 samples/s +Step 35390/44000 | Loss: 2.9323 | LR: 4.73e-05 | 405.8ms/step | 49.3 samples/s +Step 35400/44000 | Loss: 2.8649 | LR: 4.72e-05 | 405.9ms/step | 49.3 samples/s +Step 35410/44000 | Loss: 2.8693 | LR: 4.71e-05 | 405.7ms/step | 49.3 samples/s +Step 35420/44000 | Loss: 2.8906 | LR: 4.70e-05 | 406.1ms/step | 49.2 samples/s +Step 35430/44000 | Loss: 2.8101 | LR: 4.69e-05 | 405.9ms/step | 49.3 samples/s +Step 35440/44000 | Loss: 2.8326 | LR: 4.68e-05 | 406.3ms/step | 49.2 samples/s +Step 35450/44000 | Loss: 2.8085 | LR: 4.67e-05 | 406.6ms/step | 49.2 samples/s +Step 35460/44000 | Loss: 2.9148 | LR: 4.66e-05 | 405.8ms/step | 49.3 samples/s +Step 35470/44000 | Loss: 2.7985 | LR: 4.65e-05 | 410.0ms/step | 48.8 samples/s +Step 35480/44000 | Loss: 2.8449 | LR: 4.64e-05 | 410.4ms/step | 48.7 samples/s +Step 35490/44000 | Loss: 2.8471 | LR: 4.63e-05 | 406.5ms/step | 49.2 samples/s +Step 35500/44000 | Loss: 2.8701 | LR: 4.62e-05 | 404.9ms/step | 49.4 samples/s +Validation at step 35500: Loss = 2.8981, Perplexity = 18.14 +Step 35510/44000 | Loss: 2.8393 | LR: 4.61e-05 | 562.1ms/step | 35.6 samples/s +Step 35520/44000 | Loss: 2.8426 | LR: 4.59e-05 | 405.3ms/step | 49.3 samples/s +Step 35530/44000 | Loss: 2.8379 | LR: 4.58e-05 | 406.7ms/step | 49.2 samples/s +Step 35540/44000 | Loss: 2.8801 | LR: 4.57e-05 | 405.7ms/step | 49.3 samples/s +Step 35550/44000 | Loss: 2.7986 | LR: 4.56e-05 | 406.5ms/step | 49.2 samples/s +Step 35560/44000 | Loss: 2.8682 | LR: 4.55e-05 | 405.7ms/step | 49.3 samples/s +Step 35570/44000 | Loss: 2.8700 | LR: 4.54e-05 | 408.7ms/step | 48.9 samples/s +Step 35580/44000 | Loss: 2.8621 | LR: 4.53e-05 | 407.8ms/step | 49.0 samples/s +Step 35590/44000 | Loss: 2.8483 | LR: 4.52e-05 | 406.6ms/step | 49.2 samples/s +Step 35600/44000 | Loss: 2.8174 | LR: 4.51e-05 | 406.2ms/step | 49.2 samples/s +Step 35610/44000 | Loss: 2.8448 | LR: 4.50e-05 | 404.6ms/step | 49.4 samples/s +Step 35620/44000 | Loss: 2.8271 | LR: 4.49e-05 | 404.4ms/step | 49.5 samples/s +Step 35630/44000 | Loss: 2.8424 | LR: 4.48e-05 | 405.8ms/step | 49.3 samples/s +Step 35640/44000 | Loss: 2.8414 | LR: 4.47e-05 | 404.5ms/step | 49.4 samples/s +Step 35650/44000 | Loss: 2.8221 | LR: 4.46e-05 | 404.8ms/step | 49.4 samples/s +Step 35660/44000 | Loss: 2.8004 | LR: 4.45e-05 | 404.3ms/step | 49.5 samples/s +Step 35670/44000 | Loss: 2.8542 | LR: 4.44e-05 | 406.8ms/step | 49.2 samples/s +Step 35680/44000 | Loss: 2.8038 | LR: 4.43e-05 | 409.2ms/step | 48.9 samples/s +Step 35690/44000 | Loss: 2.8058 | LR: 4.42e-05 | 406.7ms/step | 49.2 samples/s +Step 35700/44000 | Loss: 2.8137 | LR: 4.41e-05 | 404.8ms/step | 49.4 samples/s +Step 35710/44000 | Loss: 2.8221 | LR: 4.40e-05 | 406.7ms/step | 49.2 samples/s +Step 35720/44000 | Loss: 2.8560 | LR: 4.39e-05 | 404.3ms/step | 49.5 samples/s +Step 35730/44000 | Loss: 2.9031 | LR: 4.38e-05 | 406.8ms/step | 49.2 samples/s +Step 35740/44000 | Loss: 2.8693 | LR: 4.37e-05 | 403.9ms/step | 49.5 samples/s +Step 35750/44000 | Loss: 2.9238 | LR: 4.36e-05 | 404.7ms/step | 49.4 samples/s +Validation at step 35750: Loss = 2.8978, Perplexity = 18.13 +Step 35760/44000 | Loss: 2.8357 | LR: 4.35e-05 | 555.8ms/step | 36.0 samples/s +Step 35770/44000 | Loss: 2.8588 | LR: 4.34e-05 | 405.6ms/step | 49.3 samples/s +Step 35780/44000 | Loss: 2.8284 | LR: 4.33e-05 | 408.3ms/step | 49.0 samples/s +Step 35790/44000 | Loss: 2.8876 | LR: 4.32e-05 | 408.1ms/step | 49.0 samples/s +Step 35800/44000 | Loss: 2.8256 | LR: 4.31e-05 | 404.9ms/step | 49.4 samples/s +Step 35810/44000 | Loss: 2.8140 | LR: 4.29e-05 | 403.8ms/step | 49.5 samples/s +Step 35820/44000 | Loss: 2.8576 | LR: 4.28e-05 | 405.0ms/step | 49.4 samples/s +Step 35830/44000 | Loss: 2.8604 | LR: 4.27e-05 | 403.5ms/step | 49.6 samples/s +Step 35840/44000 | Loss: 2.8616 | LR: 4.26e-05 | 404.6ms/step | 49.4 samples/s +Step 35850/44000 | Loss: 2.8478 | LR: 4.25e-05 | 404.9ms/step | 49.4 samples/s +Step 35860/44000 | Loss: 2.8021 | LR: 4.24e-05 | 404.8ms/step | 49.4 samples/s +Step 35870/44000 | Loss: 2.8528 | LR: 4.23e-05 | 405.0ms/step | 49.4 samples/s +Step 35880/44000 | Loss: 2.8178 | LR: 4.22e-05 | 407.4ms/step | 49.1 samples/s +Step 35890/44000 | Loss: 2.8256 | LR: 4.21e-05 | 408.6ms/step | 48.9 samples/s +Step 35900/44000 | Loss: 2.8618 | LR: 4.20e-05 | 403.5ms/step | 49.6 samples/s +Step 35910/44000 | Loss: 2.8076 | LR: 4.19e-05 | 403.9ms/step | 49.5 samples/s +Step 35920/44000 | Loss: 2.8533 | LR: 4.18e-05 | 408.2ms/step | 49.0 samples/s +Step 35930/44000 | Loss: 2.8224 | LR: 4.17e-05 | 404.7ms/step | 49.4 samples/s +Step 35940/44000 | Loss: 2.7880 | LR: 4.16e-05 | 403.9ms/step | 49.5 samples/s +Step 35950/44000 | Loss: 2.8126 | LR: 4.15e-05 | 404.6ms/step | 49.4 samples/s +Step 35960/44000 | Loss: 2.8837 | LR: 4.14e-05 | 404.9ms/step | 49.4 samples/s +Step 35970/44000 | Loss: 2.8399 | LR: 4.13e-05 | 403.8ms/step | 49.5 samples/s +Step 35980/44000 | Loss: 2.8171 | LR: 4.12e-05 | 410.2ms/step | 48.8 samples/s +Step 35990/44000 | Loss: 2.8426 | LR: 4.11e-05 | 409.8ms/step | 48.8 samples/s +Step 36000/44000 | Loss: 2.8164 | LR: 4.10e-05 | 405.6ms/step | 49.3 samples/s +Validation at step 36000: Loss = 2.8977, Perplexity = 18.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-36000 +Step 36010/44000 | Loss: 2.8415 | LR: 4.09e-05 | 876.1ms/step | 22.8 samples/s +Step 36020/44000 | Loss: 2.8232 | LR: 4.08e-05 | 404.3ms/step | 49.5 samples/s +Step 36030/44000 | Loss: 2.8894 | LR: 4.07e-05 | 405.3ms/step | 49.3 samples/s +Step 36040/44000 | Loss: 2.8335 | LR: 4.06e-05 | 403.5ms/step | 49.6 samples/s +Step 36050/44000 | Loss: 2.8531 | LR: 4.05e-05 | 403.8ms/step | 49.5 samples/s +Step 36060/44000 | Loss: 2.8563 | LR: 4.04e-05 | 403.2ms/step | 49.6 samples/s +Step 36070/44000 | Loss: 2.8642 | LR: 4.03e-05 | 408.6ms/step | 49.0 samples/s +Step 36080/44000 | Loss: 2.7740 | LR: 4.02e-05 | 409.6ms/step | 48.8 samples/s +Step 36090/44000 | Loss: 2.8103 | LR: 4.01e-05 | 404.9ms/step | 49.4 samples/s +Step 36100/44000 | Loss: 2.8320 | LR: 4.00e-05 | 404.6ms/step | 49.4 samples/s +Step 36110/44000 | Loss: 2.8998 | LR: 3.99e-05 | 406.1ms/step | 49.3 samples/s +Step 36120/44000 | Loss: 2.8696 | LR: 3.98e-05 | 405.4ms/step | 49.3 samples/s +Step 36130/44000 | Loss: 2.7840 | LR: 3.97e-05 | 405.3ms/step | 49.3 samples/s +Step 36140/44000 | Loss: 2.8112 | LR: 3.97e-05 | 404.3ms/step | 49.5 samples/s +Step 36150/44000 | Loss: 2.8433 | LR: 3.96e-05 | 403.0ms/step | 49.6 samples/s +Step 36160/44000 | Loss: 2.8114 | LR: 3.95e-05 | 404.3ms/step | 49.5 samples/s +Step 36170/44000 | Loss: 2.8264 | LR: 3.94e-05 | 408.8ms/step | 48.9 samples/s +Step 36180/44000 | Loss: 2.8471 | LR: 3.93e-05 | 411.3ms/step | 48.6 samples/s +Step 36190/44000 | Loss: 2.9054 | LR: 3.92e-05 | 405.4ms/step | 49.3 samples/s +Step 36200/44000 | Loss: 2.8776 | LR: 3.91e-05 | 404.7ms/step | 49.4 samples/s +Step 36210/44000 | Loss: 2.8093 | LR: 3.90e-05 | 404.5ms/step | 49.4 samples/s +Step 36220/44000 | Loss: 2.7612 | LR: 3.89e-05 | 404.7ms/step | 49.4 samples/s +Step 36230/44000 | Loss: 2.8572 | LR: 3.88e-05 | 403.9ms/step | 49.5 samples/s +Step 36240/44000 | Loss: 2.8804 | LR: 3.87e-05 | 403.6ms/step | 49.6 samples/s +Step 36250/44000 | Loss: 2.8205 | LR: 3.86e-05 | 405.3ms/step | 49.3 samples/s +Validation at step 36250: Loss = 2.8974, Perplexity = 18.13 +Step 36260/44000 | Loss: 2.8780 | LR: 3.85e-05 | 557.7ms/step | 35.9 samples/s +Step 36270/44000 | Loss: 2.8138 | LR: 3.84e-05 | 408.3ms/step | 49.0 samples/s +Step 36280/44000 | Loss: 2.8956 | LR: 3.83e-05 | 410.6ms/step | 48.7 samples/s +Step 36290/44000 | Loss: 2.8723 | LR: 3.82e-05 | 404.3ms/step | 49.5 samples/s +Step 36300/44000 | Loss: 2.8376 | LR: 3.81e-05 | 404.8ms/step | 49.4 samples/s +Step 36310/44000 | Loss: 2.9127 | LR: 3.80e-05 | 403.3ms/step | 49.6 samples/s +Step 36320/44000 | Loss: 2.8997 | LR: 3.79e-05 | 404.8ms/step | 49.4 samples/s +Step 36330/44000 | Loss: 2.7900 | LR: 3.78e-05 | 406.3ms/step | 49.2 samples/s +Step 36340/44000 | Loss: 2.8666 | LR: 3.77e-05 | 404.5ms/step | 49.4 samples/s +Step 36350/44000 | Loss: 2.8058 | LR: 3.76e-05 | 406.0ms/step | 49.3 samples/s +Step 36360/44000 | Loss: 2.8626 | LR: 3.75e-05 | 403.8ms/step | 49.5 samples/s +Step 36370/44000 | Loss: 2.8156 | LR: 3.74e-05 | 409.7ms/step | 48.8 samples/s +Step 36380/44000 | Loss: 2.8417 | LR: 3.73e-05 | 410.9ms/step | 48.7 samples/s +Step 36390/44000 | Loss: 2.8048 | LR: 3.72e-05 | 406.0ms/step | 49.3 samples/s +Step 36400/44000 | Loss: 2.7778 | LR: 3.71e-05 | 403.8ms/step | 49.5 samples/s +Step 36410/44000 | Loss: 2.7794 | LR: 3.70e-05 | 404.0ms/step | 49.5 samples/s +Step 36420/44000 | Loss: 2.8005 | LR: 3.69e-05 | 402.5ms/step | 49.7 samples/s +Step 36430/44000 | Loss: 2.8412 | LR: 3.69e-05 | 403.5ms/step | 49.6 samples/s +Step 36440/44000 | Loss: 2.8681 | LR: 3.68e-05 | 404.4ms/step | 49.5 samples/s +Step 36450/44000 | Loss: 2.7997 | LR: 3.67e-05 | 404.7ms/step | 49.4 samples/s +Step 36460/44000 | Loss: 2.8808 | LR: 3.66e-05 | 404.8ms/step | 49.4 samples/s +Step 36470/44000 | Loss: 2.8815 | LR: 3.65e-05 | 407.4ms/step | 49.1 samples/s +Step 36480/44000 | Loss: 2.8393 | LR: 3.64e-05 | 409.8ms/step | 48.8 samples/s +Step 36490/44000 | Loss: 2.8831 | LR: 3.63e-05 | 405.8ms/step | 49.3 samples/s +Step 36500/44000 | Loss: 2.7811 | LR: 3.62e-05 | 403.7ms/step | 49.5 samples/s +Validation at step 36500: Loss = 2.8974, Perplexity = 18.13 +Step 36510/44000 | Loss: 2.7946 | LR: 3.61e-05 | 556.2ms/step | 36.0 samples/s +Step 36520/44000 | Loss: 2.8361 | LR: 3.60e-05 | 404.4ms/step | 49.5 samples/s +Step 36530/44000 | Loss: 2.8596 | LR: 3.59e-05 | 404.3ms/step | 49.5 samples/s +Step 36540/44000 | Loss: 2.8254 | LR: 3.58e-05 | 404.1ms/step | 49.5 samples/s +Step 36550/44000 | Loss: 2.8075 | LR: 3.57e-05 | 406.1ms/step | 49.3 samples/s +Step 36560/44000 | Loss: 2.8711 | LR: 3.56e-05 | 403.9ms/step | 49.5 samples/s +Step 36570/44000 | Loss: 2.8711 | LR: 3.55e-05 | 408.7ms/step | 48.9 samples/s +Step 36580/44000 | Loss: 2.8501 | LR: 3.54e-05 | 410.9ms/step | 48.7 samples/s +Step 36590/44000 | Loss: 2.8140 | LR: 3.53e-05 | 406.1ms/step | 49.3 samples/s +Step 36600/44000 | Loss: 2.8752 | LR: 3.53e-05 | 404.4ms/step | 49.5 samples/s +Step 36610/44000 | Loss: 2.8415 | LR: 3.52e-05 | 405.7ms/step | 49.3 samples/s +Step 36620/44000 | Loss: 2.8510 | LR: 3.51e-05 | 405.4ms/step | 49.3 samples/s +Step 36630/44000 | Loss: 2.8331 | LR: 3.50e-05 | 405.2ms/step | 49.4 samples/s +Step 36640/44000 | Loss: 2.8589 | LR: 3.49e-05 | 404.1ms/step | 49.5 samples/s +Step 36650/44000 | Loss: 2.8361 | LR: 3.48e-05 | 405.2ms/step | 49.4 samples/s +Step 36660/44000 | Loss: 2.8504 | LR: 3.47e-05 | 403.8ms/step | 49.5 samples/s +Step 36670/44000 | Loss: 2.8679 | LR: 3.46e-05 | 406.3ms/step | 49.2 samples/s +Step 36680/44000 | Loss: 2.8331 | LR: 3.45e-05 | 408.2ms/step | 49.0 samples/s +Step 36690/44000 | Loss: 2.8460 | LR: 3.44e-05 | 407.6ms/step | 49.1 samples/s +Step 36700/44000 | Loss: 2.8860 | LR: 3.43e-05 | 404.3ms/step | 49.5 samples/s +Step 36710/44000 | Loss: 2.8866 | LR: 3.42e-05 | 405.1ms/step | 49.4 samples/s +Step 36720/44000 | Loss: 2.8188 | LR: 3.41e-05 | 404.0ms/step | 49.5 samples/s +Step 36730/44000 | Loss: 2.8229 | LR: 3.41e-05 | 404.4ms/step | 49.5 samples/s +Step 36740/44000 | Loss: 2.8185 | LR: 3.40e-05 | 404.1ms/step | 49.5 samples/s +Step 36750/44000 | Loss: 2.8772 | LR: 3.39e-05 | 403.6ms/step | 49.5 samples/s +Validation at step 36750: Loss = 2.8972, Perplexity = 18.12 +Step 36760/44000 | Loss: 2.8403 | LR: 3.38e-05 | 556.4ms/step | 35.9 samples/s +Step 36770/44000 | Loss: 2.8756 | LR: 3.37e-05 | 406.0ms/step | 49.3 samples/s +Step 36780/44000 | Loss: 2.8665 | LR: 3.36e-05 | 410.7ms/step | 48.7 samples/s +Step 36790/44000 | Loss: 2.7792 | LR: 3.35e-05 | 406.2ms/step | 49.2 samples/s +Step 36800/44000 | Loss: 2.8580 | LR: 3.34e-05 | 404.2ms/step | 49.5 samples/s +Step 36810/44000 | Loss: 2.8862 | LR: 3.33e-05 | 404.7ms/step | 49.4 samples/s +Step 36820/44000 | Loss: 2.8448 | LR: 3.32e-05 | 405.1ms/step | 49.4 samples/s +Step 36830/44000 | Loss: 2.8354 | LR: 3.31e-05 | 404.3ms/step | 49.5 samples/s +Step 36840/44000 | Loss: 2.8983 | LR: 3.31e-05 | 405.6ms/step | 49.3 samples/s +Step 36850/44000 | Loss: 2.8670 | LR: 3.30e-05 | 405.0ms/step | 49.4 samples/s +Step 36860/44000 | Loss: 2.8344 | LR: 3.29e-05 | 405.8ms/step | 49.3 samples/s +Step 36870/44000 | Loss: 2.8486 | LR: 3.28e-05 | 407.7ms/step | 49.1 samples/s +Step 36880/44000 | Loss: 2.9032 | LR: 3.27e-05 | 409.7ms/step | 48.8 samples/s +Step 36890/44000 | Loss: 2.8419 | LR: 3.26e-05 | 407.9ms/step | 49.0 samples/s +Step 36900/44000 | Loss: 2.8215 | LR: 3.25e-05 | 404.5ms/step | 49.4 samples/s +Step 36910/44000 | Loss: 2.7806 | LR: 3.24e-05 | 404.5ms/step | 49.4 samples/s +Step 36920/44000 | Loss: 2.8992 | LR: 3.23e-05 | 403.1ms/step | 49.6 samples/s +Step 36930/44000 | Loss: 2.9017 | LR: 3.22e-05 | 404.3ms/step | 49.5 samples/s +Step 36940/44000 | Loss: 2.8044 | LR: 3.22e-05 | 405.0ms/step | 49.4 samples/s +Step 36950/44000 | Loss: 2.8626 | LR: 3.21e-05 | 405.4ms/step | 49.3 samples/s +Step 36960/44000 | Loss: 2.8043 | LR: 3.20e-05 | 405.7ms/step | 49.3 samples/s +Step 36970/44000 | Loss: 2.7869 | LR: 3.19e-05 | 406.0ms/step | 49.3 samples/s +Step 36980/44000 | Loss: 2.8602 | LR: 3.18e-05 | 408.0ms/step | 49.0 samples/s +Step 36990/44000 | Loss: 2.8647 | LR: 3.17e-05 | 405.7ms/step | 49.3 samples/s +Step 37000/44000 | Loss: 2.8971 | LR: 3.16e-05 | 405.3ms/step | 49.3 samples/s +Validation at step 37000: Loss = 2.8970, Perplexity = 18.12 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-37000 +Step 37010/44000 | Loss: 2.8179 | LR: 3.15e-05 | 869.9ms/step | 23.0 samples/s +Step 37020/44000 | Loss: 2.8501 | LR: 3.15e-05 | 405.5ms/step | 49.3 samples/s +Step 37030/44000 | Loss: 2.8794 | LR: 3.14e-05 | 403.4ms/step | 49.6 samples/s +Step 37040/44000 | Loss: 2.8456 | LR: 3.13e-05 | 402.4ms/step | 49.7 samples/s +Step 37050/44000 | Loss: 2.8289 | LR: 3.12e-05 | 405.7ms/step | 49.3 samples/s +Step 37060/44000 | Loss: 2.8019 | LR: 3.11e-05 | 407.1ms/step | 49.1 samples/s +Step 37070/44000 | Loss: 2.8233 | LR: 3.10e-05 | 409.6ms/step | 48.8 samples/s +Step 37080/44000 | Loss: 2.8919 | LR: 3.09e-05 | 406.4ms/step | 49.2 samples/s +Step 37090/44000 | Loss: 2.8702 | LR: 3.08e-05 | 405.5ms/step | 49.3 samples/s +Step 37100/44000 | Loss: 2.8116 | LR: 3.07e-05 | 405.7ms/step | 49.3 samples/s +Step 37110/44000 | Loss: 2.9220 | LR: 3.07e-05 | 405.6ms/step | 49.3 samples/s +Step 37120/44000 | Loss: 2.8440 | LR: 3.06e-05 | 404.3ms/step | 49.5 samples/s +Step 37130/44000 | Loss: 2.8856 | LR: 3.05e-05 | 404.9ms/step | 49.4 samples/s +Step 37140/44000 | Loss: 2.8680 | LR: 3.04e-05 | 405.9ms/step | 49.3 samples/s +Step 37150/44000 | Loss: 2.8193 | LR: 3.03e-05 | 404.4ms/step | 49.5 samples/s +Step 37160/44000 | Loss: 2.8234 | LR: 3.02e-05 | 408.3ms/step | 49.0 samples/s +Step 37170/44000 | Loss: 2.8278 | LR: 3.01e-05 | 410.3ms/step | 48.7 samples/s +Step 37180/44000 | Loss: 2.8473 | LR: 3.01e-05 | 407.3ms/step | 49.1 samples/s +Step 37190/44000 | Loss: 2.9136 | LR: 3.00e-05 | 405.6ms/step | 49.3 samples/s +Step 37200/44000 | Loss: 2.8628 | LR: 2.99e-05 | 404.8ms/step | 49.4 samples/s +Step 37210/44000 | Loss: 2.8330 | LR: 2.98e-05 | 404.5ms/step | 49.4 samples/s +Step 37220/44000 | Loss: 2.8909 | LR: 2.97e-05 | 404.1ms/step | 49.5 samples/s +Step 37230/44000 | Loss: 2.7966 | LR: 2.96e-05 | 405.5ms/step | 49.3 samples/s +Step 37240/44000 | Loss: 2.8786 | LR: 2.95e-05 | 404.2ms/step | 49.5 samples/s +Step 37250/44000 | Loss: 2.8850 | LR: 2.95e-05 | 403.9ms/step | 49.5 samples/s +Validation at step 37250: Loss = 2.8968, Perplexity = 18.12 +Step 37260/44000 | Loss: 2.8309 | LR: 2.94e-05 | 559.8ms/step | 35.7 samples/s +Step 37270/44000 | Loss: 2.8582 | LR: 2.93e-05 | 410.1ms/step | 48.8 samples/s +Step 37280/44000 | Loss: 2.8860 | LR: 2.92e-05 | 407.0ms/step | 49.1 samples/s +Step 37290/44000 | Loss: 2.9044 | LR: 2.91e-05 | 404.2ms/step | 49.5 samples/s +Step 37300/44000 | Loss: 2.8633 | LR: 2.90e-05 | 404.1ms/step | 49.5 samples/s +Step 37310/44000 | Loss: 2.8340 | LR: 2.89e-05 | 403.9ms/step | 49.5 samples/s +Step 37320/44000 | Loss: 2.8438 | LR: 2.89e-05 | 404.8ms/step | 49.4 samples/s +Step 37330/44000 | Loss: 2.7974 | LR: 2.88e-05 | 404.0ms/step | 49.5 samples/s +Step 37340/44000 | Loss: 2.8465 | LR: 2.87e-05 | 404.2ms/step | 49.5 samples/s +Step 37350/44000 | Loss: 2.8615 | LR: 2.86e-05 | 404.4ms/step | 49.5 samples/s +Step 37360/44000 | Loss: 2.8770 | LR: 2.85e-05 | 406.9ms/step | 49.2 samples/s +Step 37370/44000 | Loss: 2.8185 | LR: 2.84e-05 | 408.5ms/step | 49.0 samples/s +Step 37380/44000 | Loss: 2.8167 | LR: 2.84e-05 | 408.0ms/step | 49.0 samples/s +Step 37390/44000 | Loss: 2.7781 | LR: 2.83e-05 | 404.4ms/step | 49.5 samples/s +Step 37400/44000 | Loss: 2.8999 | LR: 2.82e-05 | 404.5ms/step | 49.4 samples/s +Step 37410/44000 | Loss: 2.8694 | LR: 2.81e-05 | 405.5ms/step | 49.3 samples/s +Step 37420/44000 | Loss: 2.8302 | LR: 2.80e-05 | 403.9ms/step | 49.5 samples/s +Step 37430/44000 | Loss: 2.8494 | LR: 2.79e-05 | 405.0ms/step | 49.4 samples/s +Step 37440/44000 | Loss: 2.8570 | LR: 2.78e-05 | 405.4ms/step | 49.3 samples/s +Step 37450/44000 | Loss: 2.8169 | LR: 2.78e-05 | 406.3ms/step | 49.2 samples/s +Step 37460/44000 | Loss: 2.8598 | LR: 2.77e-05 | 406.6ms/step | 49.2 samples/s +Step 37470/44000 | Loss: 2.7903 | LR: 2.76e-05 | 407.2ms/step | 49.1 samples/s +Step 37480/44000 | Loss: 2.8992 | LR: 2.75e-05 | 406.7ms/step | 49.2 samples/s +Step 37490/44000 | Loss: 2.8330 | LR: 2.74e-05 | 405.4ms/step | 49.3 samples/s +Step 37500/44000 | Loss: 2.8466 | LR: 2.74e-05 | 404.4ms/step | 49.5 samples/s +Validation at step 37500: Loss = 2.8967, Perplexity = 18.12 +Step 37510/44000 | Loss: 2.9008 | LR: 2.73e-05 | 561.2ms/step | 35.6 samples/s +Step 37520/44000 | Loss: 2.8359 | LR: 2.72e-05 | 403.9ms/step | 49.5 samples/s +Step 37530/44000 | Loss: 2.7941 | LR: 2.71e-05 | 405.8ms/step | 49.3 samples/s +Step 37540/44000 | Loss: 2.8201 | LR: 2.70e-05 | 405.6ms/step | 49.3 samples/s +Step 37550/44000 | Loss: 2.8199 | LR: 2.69e-05 | 405.0ms/step | 49.4 samples/s +Step 37560/44000 | Loss: 2.8337 | LR: 2.69e-05 | 409.0ms/step | 48.9 samples/s +Step 37570/44000 | Loss: 2.8013 | LR: 2.68e-05 | 409.6ms/step | 48.8 samples/s +Step 37580/44000 | Loss: 2.8246 | LR: 2.67e-05 | 407.6ms/step | 49.1 samples/s +Step 37590/44000 | Loss: 2.8565 | LR: 2.66e-05 | 405.9ms/step | 49.3 samples/s +Step 37600/44000 | Loss: 2.8474 | LR: 2.65e-05 | 404.4ms/step | 49.5 samples/s +Step 37610/44000 | Loss: 2.8752 | LR: 2.65e-05 | 405.7ms/step | 49.3 samples/s +Step 37620/44000 | Loss: 2.8391 | LR: 2.64e-05 | 405.2ms/step | 49.4 samples/s +Step 37630/44000 | Loss: 2.8158 | LR: 2.63e-05 | 404.1ms/step | 49.5 samples/s +Step 37640/44000 | Loss: 2.9721 | LR: 2.62e-05 | 404.3ms/step | 49.5 samples/s +Step 37650/44000 | Loss: 2.8401 | LR: 2.61e-05 | 405.4ms/step | 49.3 samples/s +Step 37660/44000 | Loss: 2.8732 | LR: 2.60e-05 | 405.8ms/step | 49.3 samples/s +Step 37670/44000 | Loss: 2.8791 | LR: 2.60e-05 | 410.5ms/step | 48.7 samples/s +Step 37680/44000 | Loss: 2.8466 | LR: 2.59e-05 | 408.8ms/step | 48.9 samples/s +Step 37690/44000 | Loss: 2.8738 | LR: 2.58e-05 | 405.8ms/step | 49.3 samples/s +Step 37700/44000 | Loss: 2.8025 | LR: 2.57e-05 | 406.3ms/step | 49.2 samples/s +Step 37710/44000 | Loss: 2.8482 | LR: 2.56e-05 | 405.2ms/step | 49.4 samples/s +Step 37720/44000 | Loss: 2.8440 | LR: 2.56e-05 | 405.8ms/step | 49.3 samples/s +Step 37730/44000 | Loss: 2.8092 | LR: 2.55e-05 | 404.6ms/step | 49.4 samples/s +Step 37740/44000 | Loss: 2.7987 | LR: 2.54e-05 | 405.2ms/step | 49.4 samples/s +Step 37750/44000 | Loss: 2.8764 | LR: 2.53e-05 | 404.9ms/step | 49.4 samples/s +Validation at step 37750: Loss = 2.8967, Perplexity = 18.11 +Step 37760/44000 | Loss: 2.9176 | LR: 2.52e-05 | 560.8ms/step | 35.7 samples/s +Step 37770/44000 | Loss: 2.8359 | LR: 2.52e-05 | 408.6ms/step | 49.0 samples/s +Step 37780/44000 | Loss: 2.8408 | LR: 2.51e-05 | 406.3ms/step | 49.2 samples/s +Step 37790/44000 | Loss: 2.8537 | LR: 2.50e-05 | 405.6ms/step | 49.3 samples/s +Step 37800/44000 | Loss: 2.8529 | LR: 2.49e-05 | 406.6ms/step | 49.2 samples/s +Step 37810/44000 | Loss: 2.8482 | LR: 2.48e-05 | 404.7ms/step | 49.4 samples/s +Step 37820/44000 | Loss: 2.7983 | LR: 2.48e-05 | 405.1ms/step | 49.4 samples/s +Step 37830/44000 | Loss: 2.8008 | LR: 2.47e-05 | 406.0ms/step | 49.3 samples/s +Step 37840/44000 | Loss: 2.8919 | LR: 2.46e-05 | 404.5ms/step | 49.4 samples/s +Step 37850/44000 | Loss: 2.7883 | LR: 2.45e-05 | 404.4ms/step | 49.5 samples/s +Step 37860/44000 | Loss: 2.9137 | LR: 2.45e-05 | 406.4ms/step | 49.2 samples/s +Step 37870/44000 | Loss: 2.8572 | LR: 2.44e-05 | 409.4ms/step | 48.9 samples/s +Step 37880/44000 | Loss: 2.8911 | LR: 2.43e-05 | 407.7ms/step | 49.1 samples/s +Step 37890/44000 | Loss: 2.8493 | LR: 2.42e-05 | 405.5ms/step | 49.3 samples/s +Step 37900/44000 | Loss: 2.8127 | LR: 2.41e-05 | 404.6ms/step | 49.4 samples/s +Step 37910/44000 | Loss: 2.8452 | LR: 2.41e-05 | 405.8ms/step | 49.3 samples/s +Step 37920/44000 | Loss: 2.8240 | LR: 2.40e-05 | 405.7ms/step | 49.3 samples/s +Step 37930/44000 | Loss: 2.7728 | LR: 2.39e-05 | 405.3ms/step | 49.3 samples/s +Step 37940/44000 | Loss: 2.8440 | LR: 2.38e-05 | 404.8ms/step | 49.4 samples/s +Step 37950/44000 | Loss: 2.8015 | LR: 2.38e-05 | 404.3ms/step | 49.5 samples/s +Step 37960/44000 | Loss: 2.9129 | LR: 2.37e-05 | 405.6ms/step | 49.3 samples/s +Step 37970/44000 | Loss: 2.8413 | LR: 2.36e-05 | 408.9ms/step | 48.9 samples/s +Step 37980/44000 | Loss: 2.8341 | LR: 2.35e-05 | 407.5ms/step | 49.1 samples/s +Step 37990/44000 | Loss: 2.8101 | LR: 2.34e-05 | 406.3ms/step | 49.2 samples/s +Step 38000/44000 | Loss: 2.8524 | LR: 2.34e-05 | 405.6ms/step | 49.3 samples/s +Validation at step 38000: Loss = 2.8965, Perplexity = 18.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-38000 +Step 38010/44000 | Loss: 2.7968 | LR: 2.33e-05 | 891.7ms/step | 22.4 samples/s +Step 38020/44000 | Loss: 2.8827 | LR: 2.32e-05 | 405.7ms/step | 49.3 samples/s +Step 38030/44000 | Loss: 2.8907 | LR: 2.31e-05 | 406.7ms/step | 49.2 samples/s +Step 38040/44000 | Loss: 2.8184 | LR: 2.31e-05 | 406.2ms/step | 49.2 samples/s +Step 38050/44000 | Loss: 2.8393 | LR: 2.30e-05 | 406.8ms/step | 49.2 samples/s +Step 38060/44000 | Loss: 2.9054 | LR: 2.29e-05 | 409.1ms/step | 48.9 samples/s +Step 38070/44000 | Loss: 2.8665 | LR: 2.28e-05 | 407.6ms/step | 49.1 samples/s +Step 38080/44000 | Loss: 2.8265 | LR: 2.28e-05 | 405.0ms/step | 49.4 samples/s +Step 38090/44000 | Loss: 2.7967 | LR: 2.27e-05 | 403.9ms/step | 49.5 samples/s +Step 38100/44000 | Loss: 2.8094 | LR: 2.26e-05 | 405.1ms/step | 49.4 samples/s +Step 38110/44000 | Loss: 2.8605 | LR: 2.25e-05 | 405.2ms/step | 49.4 samples/s +Step 38120/44000 | Loss: 2.8543 | LR: 2.25e-05 | 405.2ms/step | 49.4 samples/s +Step 38130/44000 | Loss: 2.8211 | LR: 2.24e-05 | 404.3ms/step | 49.5 samples/s +Step 38140/44000 | Loss: 2.8417 | LR: 2.23e-05 | 405.9ms/step | 49.3 samples/s +Step 38150/44000 | Loss: 2.8129 | LR: 2.22e-05 | 405.1ms/step | 49.4 samples/s +Step 38160/44000 | Loss: 2.9252 | LR: 2.22e-05 | 411.3ms/step | 48.6 samples/s +Step 38170/44000 | Loss: 2.8400 | LR: 2.21e-05 | 407.3ms/step | 49.1 samples/s +Step 38180/44000 | Loss: 2.8592 | LR: 2.20e-05 | 404.1ms/step | 49.5 samples/s +Step 38190/44000 | Loss: 2.8679 | LR: 2.19e-05 | 406.6ms/step | 49.2 samples/s +Step 38200/44000 | Loss: 2.8991 | LR: 2.19e-05 | 405.2ms/step | 49.4 samples/s +Step 38210/44000 | Loss: 2.8881 | LR: 2.18e-05 | 403.7ms/step | 49.5 samples/s +Step 38220/44000 | Loss: 2.8044 | LR: 2.17e-05 | 405.4ms/step | 49.3 samples/s +Step 38230/44000 | Loss: 2.8660 | LR: 2.16e-05 | 406.3ms/step | 49.2 samples/s +Step 38240/44000 | Loss: 2.8596 | LR: 2.16e-05 | 404.3ms/step | 49.5 samples/s +Step 38250/44000 | Loss: 2.8148 | LR: 2.15e-05 | 406.0ms/step | 49.3 samples/s +Validation at step 38250: Loss = 2.8965, Perplexity = 18.11 +Step 38260/44000 | Loss: 2.8024 | LR: 2.14e-05 | 566.2ms/step | 35.3 samples/s +Step 38270/44000 | Loss: 2.8914 | LR: 2.13e-05 | 407.0ms/step | 49.1 samples/s +Step 38280/44000 | Loss: 2.7981 | LR: 2.13e-05 | 406.2ms/step | 49.2 samples/s +Step 38290/44000 | Loss: 2.8412 | LR: 2.12e-05 | 404.5ms/step | 49.4 samples/s +Step 38300/44000 | Loss: 2.8491 | LR: 2.11e-05 | 406.7ms/step | 49.2 samples/s +Step 38310/44000 | Loss: 2.8450 | LR: 2.11e-05 | 405.2ms/step | 49.4 samples/s +Step 38320/44000 | Loss: 2.8298 | LR: 2.10e-05 | 405.6ms/step | 49.3 samples/s +Step 38330/44000 | Loss: 2.8845 | LR: 2.09e-05 | 406.2ms/step | 49.2 samples/s +Step 38340/44000 | Loss: 2.9476 | LR: 2.08e-05 | 407.2ms/step | 49.1 samples/s +Step 38350/44000 | Loss: 2.8652 | LR: 2.08e-05 | 407.1ms/step | 49.1 samples/s +Step 38360/44000 | Loss: 2.9207 | LR: 2.07e-05 | 409.2ms/step | 48.9 samples/s +Step 38370/44000 | Loss: 2.8927 | LR: 2.06e-05 | 406.6ms/step | 49.2 samples/s +Step 38380/44000 | Loss: 2.8694 | LR: 2.05e-05 | 404.2ms/step | 49.5 samples/s +Step 38390/44000 | Loss: 2.8545 | LR: 2.05e-05 | 406.4ms/step | 49.2 samples/s +Step 38400/44000 | Loss: 2.8782 | LR: 2.04e-05 | 405.5ms/step | 49.3 samples/s +Step 38410/44000 | Loss: 2.8896 | LR: 2.03e-05 | 406.7ms/step | 49.2 samples/s +Step 38420/44000 | Loss: 2.8323 | LR: 2.03e-05 | 407.3ms/step | 49.1 samples/s +Step 38430/44000 | Loss: 2.8168 | LR: 2.02e-05 | 406.4ms/step | 49.2 samples/s +Step 38440/44000 | Loss: 2.7997 | LR: 2.01e-05 | 405.5ms/step | 49.3 samples/s +Step 38450/44000 | Loss: 2.8001 | LR: 2.00e-05 | 406.3ms/step | 49.2 samples/s +Step 38460/44000 | Loss: 2.8677 | LR: 2.00e-05 | 408.6ms/step | 48.9 samples/s +Step 38470/44000 | Loss: 2.9151 | LR: 1.99e-05 | 407.6ms/step | 49.1 samples/s +Step 38480/44000 | Loss: 2.8882 | LR: 1.98e-05 | 404.3ms/step | 49.5 samples/s +Step 38490/44000 | Loss: 2.8713 | LR: 1.98e-05 | 408.7ms/step | 48.9 samples/s +Step 38500/44000 | Loss: 2.8475 | LR: 1.97e-05 | 406.5ms/step | 49.2 samples/s +Validation at step 38500: Loss = 2.8964, Perplexity = 18.11 +Step 38510/44000 | Loss: 2.8295 | LR: 1.96e-05 | 559.7ms/step | 35.7 samples/s +Step 38520/44000 | Loss: 2.8661 | LR: 1.95e-05 | 405.4ms/step | 49.3 samples/s +Step 38530/44000 | Loss: 2.8496 | LR: 1.95e-05 | 407.0ms/step | 49.1 samples/s +Step 38540/44000 | Loss: 2.9182 | LR: 1.94e-05 | 405.5ms/step | 49.3 samples/s +Step 38550/44000 | Loss: 2.8504 | LR: 1.93e-05 | 408.4ms/step | 49.0 samples/s +Step 38560/44000 | Loss: 2.7977 | LR: 1.93e-05 | 411.8ms/step | 48.6 samples/s +Step 38570/44000 | Loss: 2.8849 | LR: 1.92e-05 | 408.4ms/step | 49.0 samples/s +Step 38580/44000 | Loss: 2.8629 | LR: 1.91e-05 | 406.0ms/step | 49.3 samples/s +Step 38590/44000 | Loss: 2.8822 | LR: 1.91e-05 | 404.2ms/step | 49.5 samples/s +Step 38600/44000 | Loss: 2.8622 | LR: 1.90e-05 | 403.6ms/step | 49.6 samples/s +Step 38610/44000 | Loss: 2.8608 | LR: 1.89e-05 | 405.5ms/step | 49.3 samples/s +Step 38620/44000 | Loss: 2.8638 | LR: 1.88e-05 | 406.0ms/step | 49.3 samples/s +Step 38630/44000 | Loss: 2.8286 | LR: 1.88e-05 | 406.5ms/step | 49.2 samples/s +Step 38640/44000 | Loss: 2.8794 | LR: 1.87e-05 | 405.1ms/step | 49.4 samples/s +Step 38650/44000 | Loss: 2.7900 | LR: 1.86e-05 | 409.0ms/step | 48.9 samples/s +Step 38660/44000 | Loss: 2.8729 | LR: 1.86e-05 | 409.1ms/step | 48.9 samples/s +Step 38670/44000 | Loss: 2.7967 | LR: 1.85e-05 | 407.0ms/step | 49.1 samples/s +Step 38680/44000 | Loss: 2.8673 | LR: 1.84e-05 | 405.9ms/step | 49.3 samples/s +Step 38690/44000 | Loss: 2.7988 | LR: 1.84e-05 | 404.3ms/step | 49.5 samples/s +Step 38700/44000 | Loss: 2.8328 | LR: 1.83e-05 | 405.5ms/step | 49.3 samples/s +Step 38710/44000 | Loss: 2.8779 | LR: 1.82e-05 | 405.5ms/step | 49.3 samples/s +Step 38720/44000 | Loss: 2.8272 | LR: 1.82e-05 | 405.4ms/step | 49.3 samples/s +Step 38730/44000 | Loss: 2.7989 | LR: 1.81e-05 | 404.9ms/step | 49.4 samples/s +Step 38740/44000 | Loss: 2.8225 | LR: 1.80e-05 | 404.7ms/step | 49.4 samples/s +Step 38750/44000 | Loss: 2.8667 | LR: 1.80e-05 | 406.5ms/step | 49.2 samples/s +Validation at step 38750: Loss = 2.8964, Perplexity = 18.11 +Step 38760/44000 | Loss: 2.8443 | LR: 1.79e-05 | 564.5ms/step | 35.4 samples/s +Step 38770/44000 | Loss: 2.8834 | LR: 1.78e-05 | 406.2ms/step | 49.2 samples/s +Step 38780/44000 | Loss: 2.9593 | LR: 1.78e-05 | 405.3ms/step | 49.3 samples/s +Step 38790/44000 | Loss: 2.8058 | LR: 1.77e-05 | 405.4ms/step | 49.3 samples/s +Step 38800/44000 | Loss: 2.8532 | LR: 1.76e-05 | 407.0ms/step | 49.1 samples/s +Step 38810/44000 | Loss: 2.8608 | LR: 1.76e-05 | 406.6ms/step | 49.2 samples/s +Step 38820/44000 | Loss: 2.7980 | LR: 1.75e-05 | 404.3ms/step | 49.5 samples/s +Step 38830/44000 | Loss: 2.8872 | LR: 1.74e-05 | 405.8ms/step | 49.3 samples/s +Step 38840/44000 | Loss: 2.8444 | LR: 1.74e-05 | 403.9ms/step | 49.5 samples/s +Step 38850/44000 | Loss: 2.8172 | LR: 1.73e-05 | 408.1ms/step | 49.0 samples/s +Step 38860/44000 | Loss: 2.8505 | LR: 1.72e-05 | 410.2ms/step | 48.8 samples/s +Step 38870/44000 | Loss: 2.8825 | LR: 1.72e-05 | 405.9ms/step | 49.3 samples/s +Step 38880/44000 | Loss: 2.7875 | LR: 1.71e-05 | 406.9ms/step | 49.2 samples/s +Step 38890/44000 | Loss: 2.8569 | LR: 1.70e-05 | 406.3ms/step | 49.2 samples/s +Step 38900/44000 | Loss: 2.9168 | LR: 1.70e-05 | 405.5ms/step | 49.3 samples/s +Step 38910/44000 | Loss: 2.8731 | LR: 1.69e-05 | 405.0ms/step | 49.4 samples/s +Step 38920/44000 | Loss: 2.8329 | LR: 1.68e-05 | 404.3ms/step | 49.5 samples/s +Step 38930/44000 | Loss: 2.8610 | LR: 1.68e-05 | 405.9ms/step | 49.3 samples/s +Step 38940/44000 | Loss: 2.8410 | LR: 1.67e-05 | 404.4ms/step | 49.5 samples/s +Step 38950/44000 | Loss: 2.8320 | LR: 1.66e-05 | 408.7ms/step | 48.9 samples/s +Step 38960/44000 | Loss: 2.8301 | LR: 1.66e-05 | 408.1ms/step | 49.0 samples/s +Step 38970/44000 | Loss: 2.8475 | LR: 1.65e-05 | 406.5ms/step | 49.2 samples/s +Step 38980/44000 | Loss: 2.8246 | LR: 1.64e-05 | 404.5ms/step | 49.4 samples/s +Step 38990/44000 | Loss: 2.8525 | LR: 1.64e-05 | 404.5ms/step | 49.4 samples/s +Step 39000/44000 | Loss: 2.9063 | LR: 1.63e-05 | 403.7ms/step | 49.5 samples/s +Validation at step 39000: Loss = 2.8963, Perplexity = 18.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-39000 +Step 39010/44000 | Loss: 2.8512 | LR: 1.62e-05 | 861.4ms/step | 23.2 samples/s +Step 39020/44000 | Loss: 2.9109 | LR: 1.62e-05 | 405.4ms/step | 49.3 samples/s +Step 39030/44000 | Loss: 2.8711 | LR: 1.61e-05 | 404.9ms/step | 49.4 samples/s +Step 39040/44000 | Loss: 2.8562 | LR: 1.61e-05 | 408.1ms/step | 49.0 samples/s +Step 39050/44000 | Loss: 2.9056 | LR: 1.60e-05 | 409.2ms/step | 48.9 samples/s +Step 39060/44000 | Loss: 2.8949 | LR: 1.59e-05 | 405.7ms/step | 49.3 samples/s +Step 39070/44000 | Loss: 2.8239 | LR: 1.59e-05 | 406.2ms/step | 49.2 samples/s +Step 39080/44000 | Loss: 2.9438 | LR: 1.58e-05 | 404.8ms/step | 49.4 samples/s +Step 39090/44000 | Loss: 2.8463 | LR: 1.57e-05 | 404.1ms/step | 49.5 samples/s +Step 39100/44000 | Loss: 2.8710 | LR: 1.57e-05 | 403.9ms/step | 49.5 samples/s +Step 39110/44000 | Loss: 2.8321 | LR: 1.56e-05 | 404.9ms/step | 49.4 samples/s +Step 39120/44000 | Loss: 2.7636 | LR: 1.55e-05 | 405.7ms/step | 49.3 samples/s +Step 39130/44000 | Loss: 2.8607 | LR: 1.55e-05 | 404.7ms/step | 49.4 samples/s +Step 39140/44000 | Loss: 2.8151 | LR: 1.54e-05 | 407.1ms/step | 49.1 samples/s +Step 39150/44000 | Loss: 2.8827 | LR: 1.54e-05 | 409.7ms/step | 48.8 samples/s +Step 39160/44000 | Loss: 2.8241 | LR: 1.53e-05 | 406.6ms/step | 49.2 samples/s +Step 39170/44000 | Loss: 2.8103 | LR: 1.52e-05 | 405.7ms/step | 49.3 samples/s +Step 39180/44000 | Loss: 2.8588 | LR: 1.52e-05 | 405.5ms/step | 49.3 samples/s +Step 39190/44000 | Loss: 2.8397 | LR: 1.51e-05 | 404.5ms/step | 49.4 samples/s +Step 39200/44000 | Loss: 2.8747 | LR: 1.50e-05 | 404.7ms/step | 49.4 samples/s +Step 39210/44000 | Loss: 2.8367 | LR: 1.50e-05 | 405.9ms/step | 49.3 samples/s +Step 39220/44000 | Loss: 2.8431 | LR: 1.49e-05 | 405.3ms/step | 49.3 samples/s +Step 39230/44000 | Loss: 2.8264 | LR: 1.49e-05 | 404.5ms/step | 49.4 samples/s +Step 39240/44000 | Loss: 2.8004 | LR: 1.48e-05 | 405.1ms/step | 49.4 samples/s +Step 39250/44000 | Loss: 2.7947 | LR: 1.47e-05 | 410.5ms/step | 48.7 samples/s +Validation at step 39250: Loss = 2.8963, Perplexity = 18.11 +Step 39260/44000 | Loss: 2.8450 | LR: 1.47e-05 | 561.5ms/step | 35.6 samples/s +Step 39270/44000 | Loss: 2.8246 | LR: 1.46e-05 | 407.3ms/step | 49.1 samples/s +Step 39280/44000 | Loss: 2.8959 | LR: 1.46e-05 | 406.1ms/step | 49.2 samples/s +Step 39290/44000 | Loss: 2.8567 | LR: 1.45e-05 | 407.0ms/step | 49.1 samples/s +Step 39300/44000 | Loss: 2.7973 | LR: 1.44e-05 | 406.9ms/step | 49.2 samples/s +Step 39310/44000 | Loss: 2.9074 | LR: 1.44e-05 | 405.4ms/step | 49.3 samples/s +Step 39320/44000 | Loss: 2.8201 | LR: 1.43e-05 | 404.2ms/step | 49.5 samples/s +Step 39330/44000 | Loss: 2.8853 | LR: 1.42e-05 | 404.6ms/step | 49.4 samples/s +Step 39340/44000 | Loss: 2.8355 | LR: 1.42e-05 | 406.6ms/step | 49.2 samples/s +Step 39350/44000 | Loss: 2.8479 | LR: 1.41e-05 | 410.2ms/step | 48.8 samples/s +Step 39360/44000 | Loss: 2.8352 | LR: 1.41e-05 | 406.0ms/step | 49.3 samples/s +Step 39370/44000 | Loss: 2.8261 | LR: 1.40e-05 | 406.3ms/step | 49.2 samples/s +Step 39380/44000 | Loss: 2.8285 | LR: 1.39e-05 | 404.9ms/step | 49.4 samples/s +Step 39390/44000 | Loss: 2.8452 | LR: 1.39e-05 | 404.6ms/step | 49.4 samples/s +Step 39400/44000 | Loss: 2.7699 | LR: 1.38e-05 | 404.2ms/step | 49.5 samples/s +Step 39410/44000 | Loss: 2.7888 | LR: 1.38e-05 | 404.4ms/step | 49.5 samples/s +Step 39420/44000 | Loss: 2.8578 | LR: 1.37e-05 | 404.1ms/step | 49.5 samples/s +Step 39430/44000 | Loss: 2.8129 | LR: 1.36e-05 | 405.3ms/step | 49.3 samples/s +Step 39440/44000 | Loss: 2.8950 | LR: 1.36e-05 | 406.9ms/step | 49.2 samples/s +Step 39450/44000 | Loss: 2.8345 | LR: 1.35e-05 | 409.6ms/step | 48.8 samples/s +Step 39460/44000 | Loss: 2.8584 | LR: 1.35e-05 | 406.9ms/step | 49.2 samples/s +Step 39470/44000 | Loss: 2.8201 | LR: 1.34e-05 | 404.4ms/step | 49.5 samples/s +Step 39480/44000 | Loss: 2.8774 | LR: 1.34e-05 | 405.7ms/step | 49.3 samples/s +Step 39490/44000 | Loss: 2.8817 | LR: 1.33e-05 | 402.5ms/step | 49.7 samples/s +Step 39500/44000 | Loss: 2.8186 | LR: 1.32e-05 | 404.6ms/step | 49.4 samples/s +Validation at step 39500: Loss = 2.8963, Perplexity = 18.11 +Step 39510/44000 | Loss: 2.8605 | LR: 1.32e-05 | 555.3ms/step | 36.0 samples/s +Step 39520/44000 | Loss: 2.8311 | LR: 1.31e-05 | 403.2ms/step | 49.6 samples/s +Step 39530/44000 | Loss: 2.8310 | LR: 1.31e-05 | 403.3ms/step | 49.6 samples/s +Step 39540/44000 | Loss: 2.8213 | LR: 1.30e-05 | 407.0ms/step | 49.1 samples/s +Step 39550/44000 | Loss: 2.8792 | LR: 1.29e-05 | 410.4ms/step | 48.7 samples/s +Step 39560/44000 | Loss: 2.8744 | LR: 1.29e-05 | 402.8ms/step | 49.7 samples/s +Step 39570/44000 | Loss: 2.8599 | LR: 1.28e-05 | 404.2ms/step | 49.5 samples/s +Step 39580/44000 | Loss: 2.8052 | LR: 1.28e-05 | 404.0ms/step | 49.5 samples/s +Step 39590/44000 | Loss: 2.9084 | LR: 1.27e-05 | 404.5ms/step | 49.4 samples/s +Step 39600/44000 | Loss: 2.8473 | LR: 1.27e-05 | 405.1ms/step | 49.4 samples/s +Step 39610/44000 | Loss: 2.8503 | LR: 1.26e-05 | 403.8ms/step | 49.5 samples/s +Step 39620/44000 | Loss: 2.7774 | LR: 1.25e-05 | 406.0ms/step | 49.3 samples/s +Step 39630/44000 | Loss: 2.8859 | LR: 1.25e-05 | 403.8ms/step | 49.5 samples/s +Step 39640/44000 | Loss: 2.8661 | LR: 1.24e-05 | 407.2ms/step | 49.1 samples/s +Step 39650/44000 | Loss: 2.7483 | LR: 1.24e-05 | 408.1ms/step | 49.0 samples/s +Step 39660/44000 | Loss: 2.8613 | LR: 1.23e-05 | 404.8ms/step | 49.4 samples/s +Step 39670/44000 | Loss: 2.8356 | LR: 1.23e-05 | 404.1ms/step | 49.5 samples/s +Step 39680/44000 | Loss: 2.8446 | LR: 1.22e-05 | 404.5ms/step | 49.4 samples/s +Step 39690/44000 | Loss: 2.8212 | LR: 1.22e-05 | 403.7ms/step | 49.5 samples/s +Step 39700/44000 | Loss: 2.8333 | LR: 1.21e-05 | 404.8ms/step | 49.4 samples/s +Step 39710/44000 | Loss: 2.9264 | LR: 1.20e-05 | 404.0ms/step | 49.5 samples/s +Step 39720/44000 | Loss: 2.8378 | LR: 1.20e-05 | 405.6ms/step | 49.3 samples/s +Step 39730/44000 | Loss: 2.7970 | LR: 1.19e-05 | 404.0ms/step | 49.5 samples/s +Step 39740/44000 | Loss: 2.9059 | LR: 1.19e-05 | 406.7ms/step | 49.2 samples/s +Step 39750/44000 | Loss: 2.8165 | LR: 1.18e-05 | 408.7ms/step | 48.9 samples/s +Validation at step 39750: Loss = 2.8963, Perplexity = 18.11 +Step 39760/44000 | Loss: 2.8858 | LR: 1.18e-05 | 557.9ms/step | 35.8 samples/s +Step 39770/44000 | Loss: 2.8607 | LR: 1.17e-05 | 403.9ms/step | 49.5 samples/s +Step 39780/44000 | Loss: 2.8453 | LR: 1.17e-05 | 404.9ms/step | 49.4 samples/s +Step 39790/44000 | Loss: 2.8735 | LR: 1.16e-05 | 404.1ms/step | 49.5 samples/s +Step 39800/44000 | Loss: 2.8590 | LR: 1.15e-05 | 405.2ms/step | 49.4 samples/s +Step 39810/44000 | Loss: 2.8183 | LR: 1.15e-05 | 403.7ms/step | 49.5 samples/s +Step 39820/44000 | Loss: 2.8513 | LR: 1.14e-05 | 406.2ms/step | 49.2 samples/s +Step 39830/44000 | Loss: 2.8592 | LR: 1.14e-05 | 404.0ms/step | 49.5 samples/s +Step 39840/44000 | Loss: 2.8876 | LR: 1.13e-05 | 408.9ms/step | 48.9 samples/s +Step 39850/44000 | Loss: 2.8370 | LR: 1.13e-05 | 410.7ms/step | 48.7 samples/s +Step 39860/44000 | Loss: 2.8923 | LR: 1.12e-05 | 406.1ms/step | 49.2 samples/s +Step 39870/44000 | Loss: 2.9296 | LR: 1.12e-05 | 404.9ms/step | 49.4 samples/s +Step 39880/44000 | Loss: 2.8152 | LR: 1.11e-05 | 406.6ms/step | 49.2 samples/s +Step 39890/44000 | Loss: 2.8689 | LR: 1.11e-05 | 405.2ms/step | 49.4 samples/s +Step 39900/44000 | Loss: 2.8128 | LR: 1.10e-05 | 404.2ms/step | 49.5 samples/s +Step 39910/44000 | Loss: 2.8674 | LR: 1.10e-05 | 402.5ms/step | 49.7 samples/s +Step 39920/44000 | Loss: 2.8429 | LR: 1.09e-05 | 405.5ms/step | 49.3 samples/s +Step 39930/44000 | Loss: 2.8488 | LR: 1.08e-05 | 404.3ms/step | 49.5 samples/s +Step 39940/44000 | Loss: 2.8712 | LR: 1.08e-05 | 407.3ms/step | 49.1 samples/s +Step 39950/44000 | Loss: 2.8508 | LR: 1.07e-05 | 409.7ms/step | 48.8 samples/s +Step 39960/44000 | Loss: 2.8597 | LR: 1.07e-05 | 404.5ms/step | 49.4 samples/s +Step 39970/44000 | Loss: 2.8658 | LR: 1.06e-05 | 406.0ms/step | 49.3 samples/s +Step 39980/44000 | Loss: 2.8545 | LR: 1.06e-05 | 405.8ms/step | 49.3 samples/s +Step 39990/44000 | Loss: 2.9345 | LR: 1.05e-05 | 407.0ms/step | 49.1 samples/s +Step 40000/44000 | Loss: 2.8949 | LR: 1.05e-05 | 403.8ms/step | 49.5 samples/s +Validation at step 40000: Loss = 2.8962, Perplexity = 18.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-40000 +Step 40010/44000 | Loss: 2.8904 | LR: 1.04e-05 | 863.8ms/step | 23.2 samples/s +Step 40020/44000 | Loss: 2.9201 | LR: 1.04e-05 | 403.3ms/step | 49.6 samples/s +Step 40030/44000 | Loss: 2.8465 | LR: 1.03e-05 | 407.6ms/step | 49.1 samples/s +Step 40040/44000 | Loss: 2.9385 | LR: 1.03e-05 | 407.2ms/step | 49.1 samples/s +Step 40050/44000 | Loss: 2.9384 | LR: 1.02e-05 | 405.4ms/step | 49.3 samples/s +Step 40060/44000 | Loss: 2.8410 | LR: 1.02e-05 | 403.5ms/step | 49.6 samples/s +Step 40070/44000 | Loss: 2.8964 | LR: 1.01e-05 | 405.1ms/step | 49.4 samples/s +Step 40080/44000 | Loss: 2.9102 | LR: 1.01e-05 | 404.4ms/step | 49.5 samples/s +Step 40090/44000 | Loss: 2.9237 | LR: 1.00e-05 | 405.9ms/step | 49.3 samples/s +Step 40100/44000 | Loss: 2.8663 | LR: 9.96e-06 | 404.8ms/step | 49.4 samples/s +Step 40110/44000 | Loss: 2.9099 | LR: 9.91e-06 | 403.6ms/step | 49.6 samples/s +Step 40120/44000 | Loss: 2.9242 | LR: 9.86e-06 | 403.2ms/step | 49.6 samples/s +Step 40130/44000 | Loss: 2.9112 | LR: 9.81e-06 | 404.0ms/step | 49.5 samples/s +Step 40140/44000 | Loss: 2.9234 | LR: 9.76e-06 | 406.3ms/step | 49.2 samples/s +Step 40150/44000 | Loss: 2.8651 | LR: 9.71e-06 | 405.1ms/step | 49.4 samples/s +Step 40160/44000 | Loss: 2.8868 | LR: 9.66e-06 | 402.9ms/step | 49.6 samples/s +Step 40170/44000 | Loss: 2.9710 | LR: 9.61e-06 | 403.7ms/step | 49.5 samples/s +Step 40180/44000 | Loss: 2.8823 | LR: 9.56e-06 | 403.4ms/step | 49.6 samples/s +Step 40190/44000 | Loss: 2.9538 | LR: 9.51e-06 | 404.1ms/step | 49.5 samples/s +Step 40200/44000 | Loss: 2.9343 | LR: 9.46e-06 | 403.1ms/step | 49.6 samples/s +Step 40210/44000 | Loss: 2.9305 | LR: 9.41e-06 | 402.8ms/step | 49.6 samples/s +Step 40220/44000 | Loss: 2.8794 | LR: 9.36e-06 | 402.1ms/step | 49.7 samples/s +Step 40230/44000 | Loss: 2.8252 | LR: 9.32e-06 | 403.7ms/step | 49.5 samples/s +Step 40240/44000 | Loss: 2.8724 | LR: 9.27e-06 | 408.7ms/step | 48.9 samples/s +Step 40250/44000 | Loss: 2.9056 | LR: 9.22e-06 | 405.1ms/step | 49.4 samples/s +Validation at step 40250: Loss = 2.8957, Perplexity = 18.10 +Step 40260/44000 | Loss: 2.8885 | LR: 9.17e-06 | 558.0ms/step | 35.8 samples/s +Step 40270/44000 | Loss: 2.8642 | LR: 9.12e-06 | 402.3ms/step | 49.7 samples/s +Step 40280/44000 | Loss: 2.8607 | LR: 9.07e-06 | 404.1ms/step | 49.5 samples/s +Step 40290/44000 | Loss: 2.8369 | LR: 9.02e-06 | 405.6ms/step | 49.3 samples/s +Step 40300/44000 | Loss: 2.8887 | LR: 8.97e-06 | 404.3ms/step | 49.5 samples/s +Step 40310/44000 | Loss: 2.8773 | LR: 8.93e-06 | 403.9ms/step | 49.5 samples/s +Step 40320/44000 | Loss: 2.8463 | LR: 8.88e-06 | 404.5ms/step | 49.4 samples/s +Step 40330/44000 | Loss: 2.9146 | LR: 8.83e-06 | 406.0ms/step | 49.3 samples/s +Step 40340/44000 | Loss: 2.8678 | LR: 8.78e-06 | 409.6ms/step | 48.8 samples/s +Step 40350/44000 | Loss: 2.8912 | LR: 8.74e-06 | 405.5ms/step | 49.3 samples/s +Step 40360/44000 | Loss: 2.8701 | LR: 8.69e-06 | 403.2ms/step | 49.6 samples/s +Step 40370/44000 | Loss: 2.8708 | LR: 8.64e-06 | 403.5ms/step | 49.6 samples/s +Step 40380/44000 | Loss: 2.8719 | LR: 8.59e-06 | 403.8ms/step | 49.5 samples/s +Step 40390/44000 | Loss: 2.8752 | LR: 8.55e-06 | 405.9ms/step | 49.3 samples/s +Step 40400/44000 | Loss: 2.9017 | LR: 8.50e-06 | 404.4ms/step | 49.5 samples/s +Step 40410/44000 | Loss: 2.9032 | LR: 8.45e-06 | 405.9ms/step | 49.3 samples/s +Step 40420/44000 | Loss: 2.8153 | LR: 8.41e-06 | 406.7ms/step | 49.2 samples/s +Step 40430/44000 | Loss: 2.8254 | LR: 8.36e-06 | 406.7ms/step | 49.2 samples/s +Step 40440/44000 | Loss: 2.8776 | LR: 8.31e-06 | 408.4ms/step | 49.0 samples/s +Step 40450/44000 | Loss: 2.8106 | LR: 8.27e-06 | 406.1ms/step | 49.2 samples/s +Step 40460/44000 | Loss: 2.8794 | LR: 8.22e-06 | 405.4ms/step | 49.3 samples/s +Step 40470/44000 | Loss: 2.8287 | LR: 8.17e-06 | 405.1ms/step | 49.4 samples/s +Step 40480/44000 | Loss: 2.8788 | LR: 8.13e-06 | 404.3ms/step | 49.5 samples/s +Step 40490/44000 | Loss: 2.8411 | LR: 8.08e-06 | 403.2ms/step | 49.6 samples/s +Step 40500/44000 | Loss: 2.9008 | LR: 8.04e-06 | 403.9ms/step | 49.5 samples/s +Validation at step 40500: Loss = 2.8956, Perplexity = 18.09 +Step 40510/44000 | Loss: 2.8879 | LR: 7.99e-06 | 554.2ms/step | 36.1 samples/s +Step 40520/44000 | Loss: 2.8830 | LR: 7.94e-06 | 402.6ms/step | 49.7 samples/s +Step 40530/44000 | Loss: 2.8799 | LR: 7.90e-06 | 409.7ms/step | 48.8 samples/s +Step 40540/44000 | Loss: 2.9273 | LR: 7.85e-06 | 408.2ms/step | 49.0 samples/s +Step 40550/44000 | Loss: 2.8479 | LR: 7.81e-06 | 406.0ms/step | 49.3 samples/s +Step 40560/44000 | Loss: 2.8724 | LR: 7.76e-06 | 405.6ms/step | 49.3 samples/s +Step 40570/44000 | Loss: 2.8071 | LR: 7.72e-06 | 403.8ms/step | 49.5 samples/s +Step 40580/44000 | Loss: 2.8371 | LR: 7.67e-06 | 403.9ms/step | 49.5 samples/s +Step 40590/44000 | Loss: 2.9814 | LR: 7.63e-06 | 405.3ms/step | 49.3 samples/s +Step 40600/44000 | Loss: 2.9105 | LR: 7.59e-06 | 403.0ms/step | 49.6 samples/s +Step 40610/44000 | Loss: 2.9275 | LR: 7.54e-06 | 402.3ms/step | 49.7 samples/s +Step 40620/44000 | Loss: 2.8912 | LR: 7.50e-06 | 403.5ms/step | 49.6 samples/s +Step 40630/44000 | Loss: 2.8637 | LR: 7.45e-06 | 407.4ms/step | 49.1 samples/s +Step 40640/44000 | Loss: 2.8940 | LR: 7.41e-06 | 404.9ms/step | 49.4 samples/s +Step 40650/44000 | Loss: 2.8256 | LR: 7.37e-06 | 402.2ms/step | 49.7 samples/s +Step 40660/44000 | Loss: 2.8682 | LR: 7.32e-06 | 402.5ms/step | 49.7 samples/s +Step 40670/44000 | Loss: 2.9061 | LR: 7.28e-06 | 403.7ms/step | 49.5 samples/s +Step 40680/44000 | Loss: 2.8601 | LR: 7.23e-06 | 403.4ms/step | 49.6 samples/s +Step 40690/44000 | Loss: 2.9093 | LR: 7.19e-06 | 402.7ms/step | 49.7 samples/s +Step 40700/44000 | Loss: 2.8246 | LR: 7.15e-06 | 403.9ms/step | 49.5 samples/s +Step 40710/44000 | Loss: 2.8938 | LR: 7.10e-06 | 404.9ms/step | 49.4 samples/s +Step 40720/44000 | Loss: 2.8808 | LR: 7.06e-06 | 404.2ms/step | 49.5 samples/s +Step 40730/44000 | Loss: 2.9117 | LR: 7.02e-06 | 407.5ms/step | 49.1 samples/s +Step 40740/44000 | Loss: 2.8415 | LR: 6.98e-06 | 408.3ms/step | 49.0 samples/s +Step 40750/44000 | Loss: 2.9549 | LR: 6.93e-06 | 404.5ms/step | 49.4 samples/s +Validation at step 40750: Loss = 2.8956, Perplexity = 18.09 +Step 40760/44000 | Loss: 2.8544 | LR: 6.89e-06 | 556.4ms/step | 35.9 samples/s +Step 40770/44000 | Loss: 2.9131 | LR: 6.85e-06 | 403.6ms/step | 49.6 samples/s +Step 40780/44000 | Loss: 2.9063 | LR: 6.81e-06 | 405.1ms/step | 49.4 samples/s +Step 40790/44000 | Loss: 2.8883 | LR: 6.77e-06 | 403.2ms/step | 49.6 samples/s +Step 40800/44000 | Loss: 2.9218 | LR: 6.72e-06 | 402.3ms/step | 49.7 samples/s +Step 40810/44000 | Loss: 2.9065 | LR: 6.68e-06 | 404.7ms/step | 49.4 samples/s +Step 40820/44000 | Loss: 2.9016 | LR: 6.64e-06 | 408.7ms/step | 48.9 samples/s +Step 40830/44000 | Loss: 2.9247 | LR: 6.60e-06 | 408.0ms/step | 49.0 samples/s +Step 40840/44000 | Loss: 2.8939 | LR: 6.56e-06 | 404.0ms/step | 49.5 samples/s +Step 40850/44000 | Loss: 2.8828 | LR: 6.52e-06 | 404.3ms/step | 49.5 samples/s +Step 40860/44000 | Loss: 2.9191 | LR: 6.47e-06 | 403.3ms/step | 49.6 samples/s +Step 40870/44000 | Loss: 2.8491 | LR: 6.43e-06 | 402.8ms/step | 49.7 samples/s +Step 40880/44000 | Loss: 2.9234 | LR: 6.39e-06 | 404.0ms/step | 49.5 samples/s +Step 40890/44000 | Loss: 2.8601 | LR: 6.35e-06 | 403.8ms/step | 49.5 samples/s +Step 40900/44000 | Loss: 2.9264 | LR: 6.31e-06 | 402.8ms/step | 49.7 samples/s +Step 40910/44000 | Loss: 2.8944 | LR: 6.27e-06 | 401.7ms/step | 49.8 samples/s +Step 40920/44000 | Loss: 2.8955 | LR: 6.23e-06 | 405.3ms/step | 49.3 samples/s +Step 40930/44000 | Loss: 2.8616 | LR: 6.19e-06 | 406.2ms/step | 49.2 samples/s +Step 40940/44000 | Loss: 2.8776 | LR: 6.15e-06 | 402.4ms/step | 49.7 samples/s +Step 40950/44000 | Loss: 2.9572 | LR: 6.11e-06 | 404.3ms/step | 49.5 samples/s +Step 40960/44000 | Loss: 2.8876 | LR: 6.07e-06 | 404.2ms/step | 49.5 samples/s +Step 40970/44000 | Loss: 2.9243 | LR: 6.03e-06 | 405.1ms/step | 49.4 samples/s +Step 40980/44000 | Loss: 2.8653 | LR: 5.99e-06 | 404.3ms/step | 49.5 samples/s +Step 40990/44000 | Loss: 2.9069 | LR: 5.95e-06 | 403.8ms/step | 49.5 samples/s +Step 41000/44000 | Loss: 2.9278 | LR: 5.91e-06 | 403.5ms/step | 49.6 samples/s +Validation at step 41000: Loss = 2.8955, Perplexity = 18.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-41000 +Step 41010/44000 | Loss: 2.9176 | LR: 5.87e-06 | 897.7ms/step | 22.3 samples/s +Step 41020/44000 | Loss: 2.8299 | LR: 5.83e-06 | 408.4ms/step | 49.0 samples/s +Step 41030/44000 | Loss: 2.8935 | LR: 5.80e-06 | 403.2ms/step | 49.6 samples/s +Step 41040/44000 | Loss: 2.8424 | LR: 5.76e-06 | 403.7ms/step | 49.5 samples/s +Step 41050/44000 | Loss: 2.8245 | LR: 5.72e-06 | 403.6ms/step | 49.6 samples/s +Step 41060/44000 | Loss: 2.9120 | LR: 5.68e-06 | 403.2ms/step | 49.6 samples/s +Step 41070/44000 | Loss: 2.8913 | LR: 5.64e-06 | 402.6ms/step | 49.7 samples/s +Step 41080/44000 | Loss: 2.8547 | LR: 5.60e-06 | 402.0ms/step | 49.7 samples/s +Step 41090/44000 | Loss: 2.8883 | LR: 5.56e-06 | 403.8ms/step | 49.5 samples/s +Step 41100/44000 | Loss: 2.8691 | LR: 5.53e-06 | 404.5ms/step | 49.4 samples/s +Step 41110/44000 | Loss: 2.9035 | LR: 5.49e-06 | 408.5ms/step | 49.0 samples/s +Step 41120/44000 | Loss: 2.8679 | LR: 5.45e-06 | 405.8ms/step | 49.3 samples/s +Step 41130/44000 | Loss: 2.8977 | LR: 5.41e-06 | 403.8ms/step | 49.5 samples/s +Step 41140/44000 | Loss: 2.8541 | LR: 5.38e-06 | 405.3ms/step | 49.3 samples/s +Step 41150/44000 | Loss: 2.8925 | LR: 5.34e-06 | 404.6ms/step | 49.4 samples/s +Step 41160/44000 | Loss: 2.9316 | LR: 5.30e-06 | 405.1ms/step | 49.4 samples/s +Step 41170/44000 | Loss: 2.8618 | LR: 5.26e-06 | 404.0ms/step | 49.5 samples/s +Step 41180/44000 | Loss: 2.8532 | LR: 5.23e-06 | 406.4ms/step | 49.2 samples/s +Step 41190/44000 | Loss: 2.8773 | LR: 5.19e-06 | 404.0ms/step | 49.5 samples/s +Step 41200/44000 | Loss: 2.8673 | LR: 5.15e-06 | 404.6ms/step | 49.4 samples/s +Step 41210/44000 | Loss: 2.8860 | LR: 5.12e-06 | 411.1ms/step | 48.6 samples/s +Step 41220/44000 | Loss: 2.8499 | LR: 5.08e-06 | 406.3ms/step | 49.2 samples/s +Step 41230/44000 | Loss: 2.8586 | LR: 5.04e-06 | 403.7ms/step | 49.5 samples/s +Step 41240/44000 | Loss: 2.9349 | LR: 5.01e-06 | 404.8ms/step | 49.4 samples/s +Step 41250/44000 | Loss: 2.8428 | LR: 4.97e-06 | 405.6ms/step | 49.3 samples/s +Validation at step 41250: Loss = 2.8955, Perplexity = 18.09 +Step 41260/44000 | Loss: 2.8960 | LR: 4.94e-06 | 558.7ms/step | 35.8 samples/s +Step 41270/44000 | Loss: 2.8429 | LR: 4.90e-06 | 405.4ms/step | 49.3 samples/s +Step 41280/44000 | Loss: 2.8283 | LR: 4.86e-06 | 405.6ms/step | 49.3 samples/s +Step 41290/44000 | Loss: 2.8525 | LR: 4.83e-06 | 404.6ms/step | 49.4 samples/s +Step 41300/44000 | Loss: 2.9141 | LR: 4.79e-06 | 408.8ms/step | 48.9 samples/s +Step 41310/44000 | Loss: 2.9213 | LR: 4.76e-06 | 408.2ms/step | 49.0 samples/s +Step 41320/44000 | Loss: 2.8800 | LR: 4.72e-06 | 405.0ms/step | 49.4 samples/s +Step 41330/44000 | Loss: 2.8881 | LR: 4.69e-06 | 405.8ms/step | 49.3 samples/s +Step 41340/44000 | Loss: 2.9023 | LR: 4.65e-06 | 406.3ms/step | 49.2 samples/s +Step 41350/44000 | Loss: 2.9093 | LR: 4.62e-06 | 405.2ms/step | 49.4 samples/s +Step 41360/44000 | Loss: 2.8680 | LR: 4.58e-06 | 404.7ms/step | 49.4 samples/s +Step 41370/44000 | Loss: 2.8400 | LR: 4.55e-06 | 404.1ms/step | 49.5 samples/s +Step 41380/44000 | Loss: 2.8696 | LR: 4.51e-06 | 405.3ms/step | 49.3 samples/s +Step 41390/44000 | Loss: 2.9438 | LR: 4.48e-06 | 405.3ms/step | 49.4 samples/s +Step 41400/44000 | Loss: 2.9211 | LR: 4.45e-06 | 409.5ms/step | 48.8 samples/s +Step 41410/44000 | Loss: 2.9175 | LR: 4.41e-06 | 407.0ms/step | 49.1 samples/s +Step 41420/44000 | Loss: 2.9060 | LR: 4.38e-06 | 405.9ms/step | 49.3 samples/s +Step 41430/44000 | Loss: 2.8880 | LR: 4.34e-06 | 403.4ms/step | 49.6 samples/s +Step 41440/44000 | Loss: 2.9185 | LR: 4.31e-06 | 405.4ms/step | 49.3 samples/s +Step 41450/44000 | Loss: 2.9222 | LR: 4.28e-06 | 406.6ms/step | 49.2 samples/s +Step 41460/44000 | Loss: 2.9010 | LR: 4.24e-06 | 404.6ms/step | 49.4 samples/s +Step 41470/44000 | Loss: 2.8998 | LR: 4.21e-06 | 405.3ms/step | 49.4 samples/s +Step 41480/44000 | Loss: 2.8312 | LR: 4.18e-06 | 404.6ms/step | 49.4 samples/s +Step 41490/44000 | Loss: 2.9217 | LR: 4.14e-06 | 404.9ms/step | 49.4 samples/s +Step 41500/44000 | Loss: 2.8679 | LR: 4.11e-06 | 410.6ms/step | 48.7 samples/s +Validation at step 41500: Loss = 2.8955, Perplexity = 18.09 +Step 41510/44000 | Loss: 2.9405 | LR: 4.08e-06 | 559.0ms/step | 35.8 samples/s +Step 41520/44000 | Loss: 2.8810 | LR: 4.05e-06 | 405.7ms/step | 49.3 samples/s +Step 41530/44000 | Loss: 2.8965 | LR: 4.01e-06 | 405.7ms/step | 49.3 samples/s +Step 41540/44000 | Loss: 2.8994 | LR: 3.98e-06 | 405.6ms/step | 49.3 samples/s +Step 41550/44000 | Loss: 2.8981 | LR: 3.95e-06 | 406.1ms/step | 49.3 samples/s +Step 41560/44000 | Loss: 2.9436 | LR: 3.92e-06 | 405.2ms/step | 49.4 samples/s +Step 41570/44000 | Loss: 2.8839 | LR: 3.88e-06 | 405.7ms/step | 49.3 samples/s +Step 41580/44000 | Loss: 2.8878 | LR: 3.85e-06 | 405.1ms/step | 49.4 samples/s +Step 41590/44000 | Loss: 2.9250 | LR: 3.82e-06 | 410.0ms/step | 48.8 samples/s +Step 41600/44000 | Loss: 2.8251 | LR: 3.79e-06 | 407.6ms/step | 49.1 samples/s +Step 41610/44000 | Loss: 2.8467 | LR: 3.76e-06 | 404.9ms/step | 49.4 samples/s +Step 41620/44000 | Loss: 2.9582 | LR: 3.73e-06 | 406.0ms/step | 49.3 samples/s +Step 41630/44000 | Loss: 2.9283 | LR: 3.70e-06 | 405.8ms/step | 49.3 samples/s +Step 41640/44000 | Loss: 2.8023 | LR: 3.66e-06 | 407.2ms/step | 49.1 samples/s +Step 41650/44000 | Loss: 2.9056 | LR: 3.63e-06 | 406.4ms/step | 49.2 samples/s +Step 41660/44000 | Loss: 2.8754 | LR: 3.60e-06 | 406.1ms/step | 49.3 samples/s +Step 41670/44000 | Loss: 2.8765 | LR: 3.57e-06 | 404.9ms/step | 49.4 samples/s +Step 41680/44000 | Loss: 2.9259 | LR: 3.54e-06 | 407.5ms/step | 49.1 samples/s +Step 41690/44000 | Loss: 2.9216 | LR: 3.51e-06 | 409.5ms/step | 48.8 samples/s +Step 41700/44000 | Loss: 2.8729 | LR: 3.48e-06 | 408.2ms/step | 49.0 samples/s +Step 41710/44000 | Loss: 2.9214 | LR: 3.45e-06 | 406.7ms/step | 49.2 samples/s +Step 41720/44000 | Loss: 2.8469 | LR: 3.42e-06 | 404.9ms/step | 49.4 samples/s +Step 41730/44000 | Loss: 2.9242 | LR: 3.39e-06 | 405.6ms/step | 49.3 samples/s +Step 41740/44000 | Loss: 2.9112 | LR: 3.36e-06 | 404.7ms/step | 49.4 samples/s +Step 41750/44000 | Loss: 2.8445 | LR: 3.33e-06 | 407.4ms/step | 49.1 samples/s +Validation at step 41750: Loss = 2.8955, Perplexity = 18.09 +Step 41760/44000 | Loss: 2.8853 | LR: 3.30e-06 | 558.9ms/step | 35.8 samples/s +Step 41770/44000 | Loss: 2.8919 | LR: 3.27e-06 | 405.6ms/step | 49.3 samples/s +Step 41780/44000 | Loss: 2.8783 | LR: 3.24e-06 | 408.0ms/step | 49.0 samples/s +Step 41790/44000 | Loss: 2.9112 | LR: 3.21e-06 | 412.0ms/step | 48.5 samples/s +Step 41800/44000 | Loss: 2.8984 | LR: 3.19e-06 | 405.5ms/step | 49.3 samples/s +Step 41810/44000 | Loss: 2.8656 | LR: 3.16e-06 | 404.6ms/step | 49.4 samples/s +Step 41820/44000 | Loss: 2.8575 | LR: 3.13e-06 | 405.3ms/step | 49.3 samples/s +Step 41830/44000 | Loss: 2.8863 | LR: 3.10e-06 | 404.3ms/step | 49.5 samples/s +Step 41840/44000 | Loss: 2.8835 | LR: 3.07e-06 | 405.1ms/step | 49.4 samples/s +Step 41850/44000 | Loss: 2.9050 | LR: 3.04e-06 | 405.6ms/step | 49.3 samples/s +Step 41860/44000 | Loss: 2.8966 | LR: 3.01e-06 | 405.5ms/step | 49.3 samples/s +Step 41870/44000 | Loss: 2.9144 | LR: 2.99e-06 | 405.5ms/step | 49.3 samples/s +Step 41880/44000 | Loss: 2.9500 | LR: 2.96e-06 | 410.2ms/step | 48.8 samples/s +Step 41890/44000 | Loss: 2.8964 | LR: 2.93e-06 | 411.6ms/step | 48.6 samples/s +Step 41900/44000 | Loss: 2.8326 | LR: 2.90e-06 | 405.1ms/step | 49.4 samples/s +Step 41910/44000 | Loss: 2.8505 | LR: 2.88e-06 | 405.8ms/step | 49.3 samples/s +Step 41920/44000 | Loss: 2.8573 | LR: 2.85e-06 | 404.8ms/step | 49.4 samples/s +Step 41930/44000 | Loss: 2.9347 | LR: 2.82e-06 | 404.4ms/step | 49.5 samples/s +Step 41940/44000 | Loss: 2.8788 | LR: 2.79e-06 | 405.9ms/step | 49.3 samples/s +Step 41950/44000 | Loss: 2.8701 | LR: 2.77e-06 | 404.8ms/step | 49.4 samples/s +Step 41960/44000 | Loss: 2.8830 | LR: 2.74e-06 | 405.7ms/step | 49.3 samples/s +Step 41970/44000 | Loss: 2.9305 | LR: 2.71e-06 | 405.7ms/step | 49.3 samples/s +Step 41980/44000 | Loss: 2.8799 | LR: 2.69e-06 | 409.9ms/step | 48.8 samples/s +Step 41990/44000 | Loss: 2.8676 | LR: 2.66e-06 | 408.0ms/step | 49.0 samples/s +Step 42000/44000 | Loss: 2.8509 | LR: 2.63e-06 | 404.4ms/step | 49.5 samples/s +Validation at step 42000: Loss = 2.8955, Perplexity = 18.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-42000 +Step 42010/44000 | Loss: 2.9303 | LR: 2.61e-06 | 880.0ms/step | 22.7 samples/s +Step 42020/44000 | Loss: 2.8723 | LR: 2.58e-06 | 404.7ms/step | 49.4 samples/s +Step 42030/44000 | Loss: 2.8955 | LR: 2.56e-06 | 404.4ms/step | 49.5 samples/s +Step 42040/44000 | Loss: 2.9005 | LR: 2.53e-06 | 404.3ms/step | 49.5 samples/s +Step 42050/44000 | Loss: 2.8880 | LR: 2.50e-06 | 405.6ms/step | 49.3 samples/s +Step 42060/44000 | Loss: 2.8812 | LR: 2.48e-06 | 407.1ms/step | 49.1 samples/s +Step 42070/44000 | Loss: 2.8465 | LR: 2.45e-06 | 409.1ms/step | 48.9 samples/s +Step 42080/44000 | Loss: 2.8626 | LR: 2.43e-06 | 404.9ms/step | 49.4 samples/s +Step 42090/44000 | Loss: 2.9027 | LR: 2.40e-06 | 402.6ms/step | 49.7 samples/s +Step 42100/44000 | Loss: 2.9013 | LR: 2.38e-06 | 403.9ms/step | 49.5 samples/s +Step 42110/44000 | Loss: 2.8945 | LR: 2.35e-06 | 405.4ms/step | 49.3 samples/s +Step 42120/44000 | Loss: 2.8915 | LR: 2.33e-06 | 404.5ms/step | 49.4 samples/s +Step 42130/44000 | Loss: 2.8724 | LR: 2.30e-06 | 405.2ms/step | 49.4 samples/s +Step 42140/44000 | Loss: 2.9306 | LR: 2.28e-06 | 404.3ms/step | 49.5 samples/s +Step 42150/44000 | Loss: 2.8829 | LR: 2.25e-06 | 405.4ms/step | 49.3 samples/s +Step 42160/44000 | Loss: 2.9042 | LR: 2.23e-06 | 407.1ms/step | 49.1 samples/s +Step 42170/44000 | Loss: 2.8709 | LR: 2.21e-06 | 410.1ms/step | 48.8 samples/s +Step 42180/44000 | Loss: 2.8437 | LR: 2.18e-06 | 404.9ms/step | 49.4 samples/s +Step 42190/44000 | Loss: 2.8297 | LR: 2.16e-06 | 404.9ms/step | 49.4 samples/s +Step 42200/44000 | Loss: 2.8581 | LR: 2.13e-06 | 405.8ms/step | 49.3 samples/s +Step 42210/44000 | Loss: 2.9081 | LR: 2.11e-06 | 405.2ms/step | 49.4 samples/s +Step 42220/44000 | Loss: 2.8920 | LR: 2.09e-06 | 404.8ms/step | 49.4 samples/s +Step 42230/44000 | Loss: 2.9127 | LR: 2.06e-06 | 406.0ms/step | 49.3 samples/s +Step 42240/44000 | Loss: 2.8474 | LR: 2.04e-06 | 404.1ms/step | 49.5 samples/s +Step 42250/44000 | Loss: 2.8787 | LR: 2.02e-06 | 405.1ms/step | 49.4 samples/s +Validation at step 42250: Loss = 2.8955, Perplexity = 18.09 +Step 42260/44000 | Loss: 2.9143 | LR: 1.99e-06 | 562.0ms/step | 35.6 samples/s +Step 42270/44000 | Loss: 2.8965 | LR: 1.97e-06 | 404.7ms/step | 49.4 samples/s +Step 42280/44000 | Loss: 2.8636 | LR: 1.95e-06 | 405.6ms/step | 49.3 samples/s +Step 42290/44000 | Loss: 2.9204 | LR: 1.93e-06 | 404.7ms/step | 49.4 samples/s +Step 42300/44000 | Loss: 2.8990 | LR: 1.90e-06 | 404.1ms/step | 49.5 samples/s +Step 42310/44000 | Loss: 2.8941 | LR: 1.88e-06 | 404.4ms/step | 49.5 samples/s +Step 42320/44000 | Loss: 2.9503 | LR: 1.86e-06 | 403.6ms/step | 49.6 samples/s +Step 42330/44000 | Loss: 2.9184 | LR: 1.84e-06 | 404.2ms/step | 49.5 samples/s +Step 42340/44000 | Loss: 2.9259 | LR: 1.82e-06 | 404.6ms/step | 49.4 samples/s +Step 42350/44000 | Loss: 2.9360 | LR: 1.79e-06 | 407.0ms/step | 49.1 samples/s +Step 42360/44000 | Loss: 2.8817 | LR: 1.77e-06 | 409.0ms/step | 48.9 samples/s +Step 42370/44000 | Loss: 2.8806 | LR: 1.75e-06 | 406.7ms/step | 49.2 samples/s +Step 42380/44000 | Loss: 2.8250 | LR: 1.73e-06 | 406.1ms/step | 49.3 samples/s +Step 42390/44000 | Loss: 2.8766 | LR: 1.71e-06 | 404.7ms/step | 49.4 samples/s +Step 42400/44000 | Loss: 2.8721 | LR: 1.69e-06 | 405.5ms/step | 49.3 samples/s +Step 42410/44000 | Loss: 2.9218 | LR: 1.67e-06 | 405.8ms/step | 49.3 samples/s +Step 42420/44000 | Loss: 2.8951 | LR: 1.64e-06 | 404.1ms/step | 49.5 samples/s +Step 42430/44000 | Loss: 2.8997 | LR: 1.62e-06 | 404.2ms/step | 49.5 samples/s +Step 42440/44000 | Loss: 2.8732 | LR: 1.60e-06 | 405.0ms/step | 49.4 samples/s +Step 42450/44000 | Loss: 2.8485 | LR: 1.58e-06 | 405.1ms/step | 49.4 samples/s +Step 42460/44000 | Loss: 2.8797 | LR: 1.56e-06 | 407.4ms/step | 49.1 samples/s +Step 42470/44000 | Loss: 2.8425 | LR: 1.54e-06 | 402.9ms/step | 49.6 samples/s +Step 42480/44000 | Loss: 2.8945 | LR: 1.52e-06 | 403.9ms/step | 49.5 samples/s +Step 42490/44000 | Loss: 2.8949 | LR: 1.50e-06 | 404.4ms/step | 49.4 samples/s +Step 42500/44000 | Loss: 2.8732 | LR: 1.48e-06 | 402.7ms/step | 49.7 samples/s +Validation at step 42500: Loss = 2.8955, Perplexity = 18.09 +Step 42510/44000 | Loss: 2.8969 | LR: 1.46e-06 | 558.8ms/step | 35.8 samples/s +Step 42520/44000 | Loss: 2.8647 | LR: 1.44e-06 | 404.1ms/step | 49.5 samples/s +Step 42530/44000 | Loss: 2.9532 | LR: 1.42e-06 | 403.7ms/step | 49.5 samples/s +Step 42540/44000 | Loss: 2.8824 | LR: 1.40e-06 | 403.9ms/step | 49.5 samples/s +Step 42550/44000 | Loss: 2.9040 | LR: 1.39e-06 | 407.9ms/step | 49.0 samples/s +Step 42560/44000 | Loss: 2.8830 | LR: 1.37e-06 | 408.2ms/step | 49.0 samples/s +Step 42570/44000 | Loss: 2.8998 | LR: 1.35e-06 | 404.4ms/step | 49.5 samples/s +Step 42580/44000 | Loss: 2.9300 | LR: 1.33e-06 | 403.5ms/step | 49.6 samples/s +Step 42590/44000 | Loss: 2.9196 | LR: 1.31e-06 | 404.1ms/step | 49.5 samples/s +Step 42600/44000 | Loss: 2.8965 | LR: 1.29e-06 | 404.0ms/step | 49.5 samples/s +Step 42610/44000 | Loss: 2.9360 | LR: 1.27e-06 | 404.2ms/step | 49.5 samples/s +Step 42620/44000 | Loss: 2.8892 | LR: 1.25e-06 | 404.2ms/step | 49.5 samples/s +Step 42630/44000 | Loss: 2.8836 | LR: 1.24e-06 | 404.2ms/step | 49.5 samples/s +Step 42640/44000 | Loss: 2.8985 | LR: 1.22e-06 | 403.9ms/step | 49.5 samples/s +Step 42650/44000 | Loss: 2.8997 | LR: 1.20e-06 | 408.3ms/step | 49.0 samples/s +Step 42660/44000 | Loss: 2.8755 | LR: 1.18e-06 | 407.4ms/step | 49.1 samples/s +Step 42670/44000 | Loss: 2.9286 | LR: 1.17e-06 | 403.9ms/step | 49.5 samples/s +Step 42680/44000 | Loss: 2.9078 | LR: 1.15e-06 | 404.0ms/step | 49.5 samples/s +Step 42690/44000 | Loss: 2.9032 | LR: 1.13e-06 | 404.0ms/step | 49.5 samples/s +Step 42700/44000 | Loss: 2.8460 | LR: 1.11e-06 | 403.4ms/step | 49.6 samples/s +Step 42710/44000 | Loss: 2.9139 | LR: 1.10e-06 | 404.8ms/step | 49.4 samples/s +Step 42720/44000 | Loss: 2.9222 | LR: 1.08e-06 | 403.9ms/step | 49.5 samples/s +Step 42730/44000 | Loss: 2.8590 | LR: 1.06e-06 | 404.7ms/step | 49.4 samples/s +Step 42740/44000 | Loss: 2.9074 | LR: 1.05e-06 | 404.9ms/step | 49.4 samples/s +Step 42750/44000 | Loss: 2.8399 | LR: 1.03e-06 | 408.5ms/step | 49.0 samples/s +Validation at step 42750: Loss = 2.8955, Perplexity = 18.09 +Step 42760/44000 | Loss: 2.9042 | LR: 1.01e-06 | 559.2ms/step | 35.8 samples/s +Step 42770/44000 | Loss: 2.8756 | LR: 9.97e-07 | 402.5ms/step | 49.7 samples/s +Step 42780/44000 | Loss: 2.8429 | LR: 9.81e-07 | 403.4ms/step | 49.6 samples/s +Step 42790/44000 | Loss: 2.8713 | LR: 9.65e-07 | 405.3ms/step | 49.3 samples/s +Step 42800/44000 | Loss: 2.8973 | LR: 9.49e-07 | 404.8ms/step | 49.4 samples/s +Step 42810/44000 | Loss: 2.9019 | LR: 9.33e-07 | 405.5ms/step | 49.3 samples/s +Step 42820/44000 | Loss: 2.9172 | LR: 9.18e-07 | 404.1ms/step | 49.5 samples/s +Step 42830/44000 | Loss: 2.9333 | LR: 9.02e-07 | 404.3ms/step | 49.5 samples/s +Step 42840/44000 | Loss: 2.9523 | LR: 8.87e-07 | 406.6ms/step | 49.2 samples/s +Step 42850/44000 | Loss: 2.8590 | LR: 8.72e-07 | 406.2ms/step | 49.2 samples/s +Step 42860/44000 | Loss: 2.9184 | LR: 8.57e-07 | 403.0ms/step | 49.6 samples/s +Step 42870/44000 | Loss: 2.8737 | LR: 8.42e-07 | 403.3ms/step | 49.6 samples/s +Step 42880/44000 | Loss: 2.9690 | LR: 8.27e-07 | 402.5ms/step | 49.7 samples/s +Step 42890/44000 | Loss: 2.8450 | LR: 8.12e-07 | 403.2ms/step | 49.6 samples/s +Step 42900/44000 | Loss: 2.8712 | LR: 7.98e-07 | 404.6ms/step | 49.4 samples/s +Step 42910/44000 | Loss: 2.9522 | LR: 7.83e-07 | 403.5ms/step | 49.6 samples/s +Step 42920/44000 | Loss: 2.9086 | LR: 7.69e-07 | 404.1ms/step | 49.5 samples/s +Step 42930/44000 | Loss: 2.8310 | LR: 7.55e-07 | 403.7ms/step | 49.5 samples/s +Step 42940/44000 | Loss: 2.8830 | LR: 7.41e-07 | 407.3ms/step | 49.1 samples/s +Step 42950/44000 | Loss: 2.9558 | LR: 7.27e-07 | 406.7ms/step | 49.2 samples/s +Step 42960/44000 | Loss: 2.9079 | LR: 7.13e-07 | 404.2ms/step | 49.5 samples/s +Step 42970/44000 | Loss: 2.8910 | LR: 6.99e-07 | 404.2ms/step | 49.5 samples/s +Step 42980/44000 | Loss: 2.8426 | LR: 6.86e-07 | 404.2ms/step | 49.5 samples/s +Step 42990/44000 | Loss: 2.9253 | LR: 6.72e-07 | 404.7ms/step | 49.4 samples/s +Step 43000/44000 | Loss: 2.8640 | LR: 6.59e-07 | 403.5ms/step | 49.6 samples/s +Validation at step 43000: Loss = 2.8955, Perplexity = 18.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-43000 +Step 43010/44000 | Loss: 2.8600 | LR: 6.46e-07 | 868.5ms/step | 23.0 samples/s +Step 43020/44000 | Loss: 2.8598 | LR: 6.33e-07 | 403.8ms/step | 49.5 samples/s +Step 43030/44000 | Loss: 2.9001 | LR: 6.20e-07 | 410.2ms/step | 48.8 samples/s +Step 43040/44000 | Loss: 2.8691 | LR: 6.08e-07 | 406.2ms/step | 49.2 samples/s +Step 43050/44000 | Loss: 2.8284 | LR: 5.95e-07 | 403.0ms/step | 49.6 samples/s +Step 43060/44000 | Loss: 2.8590 | LR: 5.83e-07 | 402.2ms/step | 49.7 samples/s +Step 43070/44000 | Loss: 2.8934 | LR: 5.70e-07 | 403.9ms/step | 49.5 samples/s +Step 43080/44000 | Loss: 2.8812 | LR: 5.58e-07 | 404.3ms/step | 49.5 samples/s +Step 43090/44000 | Loss: 2.8799 | LR: 5.46e-07 | 404.3ms/step | 49.5 samples/s +Step 43100/44000 | Loss: 2.9089 | LR: 5.34e-07 | 401.7ms/step | 49.8 samples/s +Step 43110/44000 | Loss: 2.8760 | LR: 5.22e-07 | 404.6ms/step | 49.4 samples/s +Step 43120/44000 | Loss: 2.8424 | LR: 5.11e-07 | 405.5ms/step | 49.3 samples/s +Step 43130/44000 | Loss: 2.8670 | LR: 4.99e-07 | 409.6ms/step | 48.8 samples/s +Step 43140/44000 | Loss: 2.8662 | LR: 4.88e-07 | 405.7ms/step | 49.3 samples/s +Step 43150/44000 | Loss: 2.8884 | LR: 4.76e-07 | 403.6ms/step | 49.6 samples/s +Step 43160/44000 | Loss: 2.8644 | LR: 4.65e-07 | 404.9ms/step | 49.4 samples/s +Step 43170/44000 | Loss: 2.9186 | LR: 4.54e-07 | 405.4ms/step | 49.3 samples/s +Step 43180/44000 | Loss: 2.8723 | LR: 4.43e-07 | 404.6ms/step | 49.4 samples/s +Step 43190/44000 | Loss: 2.9336 | LR: 4.33e-07 | 406.2ms/step | 49.2 samples/s +Step 43200/44000 | Loss: 2.8370 | LR: 4.22e-07 | 405.2ms/step | 49.4 samples/s +Step 43210/44000 | Loss: 2.9337 | LR: 4.12e-07 | 404.8ms/step | 49.4 samples/s +Step 43220/44000 | Loss: 2.9160 | LR: 4.01e-07 | 405.8ms/step | 49.3 samples/s +Step 43230/44000 | Loss: 2.8658 | LR: 3.91e-07 | 409.1ms/step | 48.9 samples/s +Step 43240/44000 | Loss: 2.8957 | LR: 3.81e-07 | 403.4ms/step | 49.6 samples/s +Step 43250/44000 | Loss: 2.9086 | LR: 3.71e-07 | 403.2ms/step | 49.6 samples/s +Validation at step 43250: Loss = 2.8955, Perplexity = 18.09 +Step 43260/44000 | Loss: 2.8954 | LR: 3.61e-07 | 556.6ms/step | 35.9 samples/s +Step 43270/44000 | Loss: 2.8681 | LR: 3.51e-07 | 401.6ms/step | 49.8 samples/s +Step 43280/44000 | Loss: 2.8582 | LR: 3.42e-07 | 403.0ms/step | 49.6 samples/s +Step 43290/44000 | Loss: 2.9080 | LR: 3.32e-07 | 402.9ms/step | 49.6 samples/s +Step 43300/44000 | Loss: 2.8669 | LR: 3.23e-07 | 403.7ms/step | 49.5 samples/s +Step 43310/44000 | Loss: 2.8786 | LR: 3.14e-07 | 404.2ms/step | 49.5 samples/s +Step 43320/44000 | Loss: 2.8242 | LR: 3.05e-07 | 407.9ms/step | 49.0 samples/s +Step 43330/44000 | Loss: 2.8919 | LR: 2.96e-07 | 407.3ms/step | 49.1 samples/s +Step 43340/44000 | Loss: 2.9446 | LR: 2.87e-07 | 403.9ms/step | 49.5 samples/s +Step 43350/44000 | Loss: 2.9037 | LR: 2.79e-07 | 404.2ms/step | 49.5 samples/s +Step 43360/44000 | Loss: 2.8843 | LR: 2.70e-07 | 403.7ms/step | 49.5 samples/s +Step 43370/44000 | Loss: 2.8793 | LR: 2.62e-07 | 404.8ms/step | 49.4 samples/s +Step 43380/44000 | Loss: 2.8810 | LR: 2.53e-07 | 404.0ms/step | 49.5 samples/s +Step 43390/44000 | Loss: 2.9056 | LR: 2.45e-07 | 404.1ms/step | 49.5 samples/s +Step 43400/44000 | Loss: 2.8974 | LR: 2.37e-07 | 403.3ms/step | 49.6 samples/s +Step 43410/44000 | Loss: 2.8808 | LR: 2.30e-07 | 405.7ms/step | 49.3 samples/s +Step 43420/44000 | Loss: 2.9787 | LR: 2.22e-07 | 408.8ms/step | 48.9 samples/s +Step 43430/44000 | Loss: 2.8548 | LR: 2.14e-07 | 405.4ms/step | 49.3 samples/s +Step 43440/44000 | Loss: 2.8287 | LR: 2.07e-07 | 403.9ms/step | 49.5 samples/s +Step 43450/44000 | Loss: 2.8701 | LR: 1.99e-07 | 403.3ms/step | 49.6 samples/s +Step 43460/44000 | Loss: 2.9116 | LR: 1.92e-07 | 405.7ms/step | 49.3 samples/s +Step 43470/44000 | Loss: 2.8417 | LR: 1.85e-07 | 404.0ms/step | 49.5 samples/s +Step 43480/44000 | Loss: 2.9634 | LR: 1.78e-07 | 403.3ms/step | 49.6 samples/s +Step 43490/44000 | Loss: 2.9178 | LR: 1.72e-07 | 402.2ms/step | 49.7 samples/s +Step 43500/44000 | Loss: 2.8989 | LR: 1.65e-07 | 404.1ms/step | 49.5 samples/s +Validation at step 43500: Loss = 2.8955, Perplexity = 18.09 +Step 43510/44000 | Loss: 2.8462 | LR: 1.58e-07 | 560.3ms/step | 35.7 samples/s +Step 43520/44000 | Loss: 2.9342 | LR: 1.52e-07 | 405.0ms/step | 49.4 samples/s +Step 43530/44000 | Loss: 2.9040 | LR: 1.46e-07 | 404.4ms/step | 49.5 samples/s +Step 43540/44000 | Loss: 2.8691 | LR: 1.40e-07 | 403.4ms/step | 49.6 samples/s +Step 43550/44000 | Loss: 2.8783 | LR: 1.34e-07 | 403.6ms/step | 49.6 samples/s +Step 43560/44000 | Loss: 2.9113 | LR: 1.28e-07 | 403.6ms/step | 49.6 samples/s +Step 43570/44000 | Loss: 2.8904 | LR: 1.22e-07 | 404.4ms/step | 49.5 samples/s +Step 43580/44000 | Loss: 2.8927 | LR: 1.16e-07 | 403.8ms/step | 49.5 samples/s +Step 43590/44000 | Loss: 2.8096 | LR: 1.11e-07 | 404.7ms/step | 49.4 samples/s +Step 43600/44000 | Loss: 2.9072 | LR: 1.06e-07 | 404.7ms/step | 49.4 samples/s +Step 43610/44000 | Loss: 2.9834 | LR: 1.00e-07 | 408.2ms/step | 49.0 samples/s +Step 43620/44000 | Loss: 2.9048 | LR: 9.52e-08 | 405.7ms/step | 49.3 samples/s +Step 43630/44000 | Loss: 2.8882 | LR: 9.03e-08 | 402.8ms/step | 49.7 samples/s +Step 43640/44000 | Loss: 2.8175 | LR: 8.55e-08 | 403.7ms/step | 49.5 samples/s +Step 43650/44000 | Loss: 2.8811 | LR: 8.08e-08 | 402.4ms/step | 49.7 samples/s +Step 43660/44000 | Loss: 2.9045 | LR: 7.62e-08 | 402.2ms/step | 49.7 samples/s +Step 43670/44000 | Loss: 2.9273 | LR: 7.18e-08 | 404.5ms/step | 49.4 samples/s +Step 43680/44000 | Loss: 2.9118 | LR: 6.75e-08 | 403.3ms/step | 49.6 samples/s +Step 43690/44000 | Loss: 2.8462 | LR: 6.34e-08 | 404.0ms/step | 49.5 samples/s +Step 43700/44000 | Loss: 2.8869 | LR: 5.94e-08 | 404.6ms/step | 49.4 samples/s +Step 43710/44000 | Loss: 2.8658 | LR: 5.55e-08 | 408.0ms/step | 49.0 samples/s +Step 43720/44000 | Loss: 2.8787 | LR: 5.17e-08 | 405.3ms/step | 49.3 samples/s +Step 43730/44000 | Loss: 2.8695 | LR: 4.81e-08 | 405.0ms/step | 49.4 samples/s +Step 43740/44000 | Loss: 2.8340 | LR: 4.46e-08 | 403.9ms/step | 49.5 samples/s +Step 43750/44000 | Loss: 2.8737 | LR: 4.12e-08 | 403.1ms/step | 49.6 samples/s +Validation at step 43750: Loss = 2.8955, Perplexity = 18.09 +Step 43760/44000 | Loss: 2.8371 | LR: 3.80e-08 | 555.4ms/step | 36.0 samples/s +Step 43770/44000 | Loss: 2.8448 | LR: 3.49e-08 | 405.6ms/step | 49.3 samples/s +Step 43780/44000 | Loss: 2.8307 | LR: 3.19e-08 | 404.6ms/step | 49.4 samples/s +Step 43790/44000 | Loss: 2.8498 | LR: 2.91e-08 | 403.8ms/step | 49.5 samples/s +Step 43800/44000 | Loss: 2.9203 | LR: 2.64e-08 | 407.1ms/step | 49.1 samples/s +Step 43810/44000 | Loss: 2.8847 | LR: 2.38e-08 | 407.0ms/step | 49.1 samples/s +Step 43820/44000 | Loss: 2.8870 | LR: 2.14e-08 | 404.6ms/step | 49.4 samples/s +Step 43830/44000 | Loss: 2.9526 | LR: 1.91e-08 | 404.5ms/step | 49.4 samples/s +Step 43840/44000 | Loss: 2.8419 | LR: 1.69e-08 | 404.6ms/step | 49.4 samples/s +Step 43850/44000 | Loss: 2.8339 | LR: 1.48e-08 | 406.7ms/step | 49.2 samples/s +Step 43860/44000 | Loss: 2.9658 | LR: 1.29e-08 | 404.1ms/step | 49.5 samples/s +Step 43870/44000 | Loss: 2.8444 | LR: 1.11e-08 | 404.1ms/step | 49.5 samples/s +Step 43880/44000 | Loss: 2.8956 | LR: 9.50e-09 | 404.0ms/step | 49.5 samples/s +Step 43890/44000 | Loss: 2.8035 | LR: 7.98e-09 | 405.7ms/step | 49.3 samples/s +Step 43900/44000 | Loss: 2.8658 | LR: 6.60e-09 | 408.8ms/step | 48.9 samples/s +Step 43910/44000 | Loss: 2.9170 | LR: 5.34e-09 | 406.7ms/step | 49.2 samples/s +Step 43920/44000 | Loss: 2.8917 | LR: 4.22e-09 | 405.2ms/step | 49.4 samples/s +Step 43930/44000 | Loss: 2.8861 | LR: 3.23e-09 | 405.4ms/step | 49.3 samples/s +Step 43940/44000 | Loss: 2.9117 | LR: 2.37e-09 | 405.3ms/step | 49.3 samples/s +Step 43950/44000 | Loss: 2.8793 | LR: 1.65e-09 | 405.6ms/step | 49.3 samples/s +Step 43960/44000 | Loss: 2.9148 | LR: 1.06e-09 | 405.5ms/step | 49.3 samples/s +Step 43970/44000 | Loss: 2.9035 | LR: 5.94e-10 | 404.8ms/step | 49.4 samples/s +Step 43980/44000 | Loss: 2.8488 | LR: 2.64e-10 | 404.6ms/step | 49.4 samples/s +Step 43990/44000 | Loss: 2.8397 | LR: 6.60e-11 | 408.1ms/step | 49.0 samples/s +Step 44000/44000 | Loss: 2.8868 | LR: 0.00e+00 | 408.9ms/step | 48.9 samples/s +Validation at step 44000: Loss = 2.8955, Perplexity = 18.09 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-44000 +wandb: updating run metadata +wandb: uploading output.log; uploading wandb-summary.json +wandb: uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▇▇▇▆▅▅▅▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity ██▇▇▇▅▅▄▄▃▃▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▁ +wandb: speed/samples_per_sec ██████████▁█████████████████████████████ +wandb: speed/steps_per_sec ▆▇▇▇█▇▆▆▆▆▃▄█▁▆▆▄▆▆▇▂▄▆▄▇▅▇▇▇▂▇█▇█▇▇▇▇▄▄ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.8955 +wandb: eval/perplexity 18.09257 +wandb: eval/uses_dense_attention 1 +wandb: eval/uses_sparse_attention 0 +wandb: speed/ms_per_step 408.86455 +wandb: speed/samples_per_sec 48.91596 +wandb: speed/steps_per_sec 2.4458 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m dense 8k seed48 h100 ddp4 resume25k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/zrelwtt2 +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: /mloscratch/homes/navasard/wandb/wandb/run-20260821_124634-zrelwtt2/logs +Training complete! diff --git a/epfl_final/train_logs/seed48_mixed_h100_ddp2/mixed.20260821T050800Z.log b/epfl_final/train_logs/seed48_mixed_h100_ddp2/mixed.20260821T050800Z.log new file mode 100644 index 0000000000000000000000000000000000000000..81b92609e4cc6aa96f032e1cae3c2da90f56486d --- /dev/null +++ b/epfl_final/train_logs/seed48_mixed_h100_ddp2/mixed.20260821T050800Z.log @@ -0,0 +1,4853 @@ +W0821 05:09:17.652000 274 site-packages/torch/distributed/run.py:851] +W0821 05:09:17.652000 274 site-packages/torch/distributed/run.py:851] ***************************************** +W0821 05:09:17.652000 274 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0821 05:09:17.652000 274 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923", + "resume_from_checkpoint": null, + "force_new_run": false, + "num_epochs": 1, + "max_steps": 44000, + "per_device_batch_size": 2, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 48, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": true, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m mixed 8k seed48 h100 ddp2", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "mixed", + "multiseed", + "seed48", + "ddp2", + "h100" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": [ + 63, + 47, + 31, + 21, + 11 + ], + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Initializing model from scratch (random weights)... +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [2, 8192] -- -- True +├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True +│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True +├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 951.29 +================================================================================================================================================================ +Input size (MB): 0.13 +Forward/backward pass size (MB): 31197.23 +Params size (MB): 951.29 +Estimated Total Size (MB): 32148.66 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 2 +World size (GPUs): 2 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs) +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY. +wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.26.1 +wandb: Run data is saved locally in /mloscratch/homes/navasard/wandb/wandb/run-20260821_051200-noud30xe +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run 365m mixed 8k seed48 h100 ddp2 +wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/noud30xe +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/44000 | Loss: 12.0963 | LR: 6.67e-06 | 6991.3ms/step | 2.9 samples/s +Step 20/44000 | Loss: 11.9922 | LR: 1.33e-05 | 690.6ms/step | 29.0 samples/s +Step 30/44000 | Loss: 11.6640 | LR: 2.00e-05 | 690.6ms/step | 29.0 samples/s +Step 40/44000 | Loss: 11.0914 | LR: 2.67e-05 | 685.6ms/step | 29.2 samples/s +Step 50/44000 | Loss: 10.7680 | LR: 3.33e-05 | 689.9ms/step | 29.0 samples/s +Step 60/44000 | Loss: 10.5422 | LR: 4.00e-05 | 689.2ms/step | 29.0 samples/s +Step 70/44000 | Loss: 10.2853 | LR: 4.67e-05 | 690.1ms/step | 29.0 samples/s +Step 80/44000 | Loss: 9.9974 | LR: 5.33e-05 | 689.1ms/step | 29.0 samples/s +Step 90/44000 | Loss: 9.6380 | LR: 6.00e-05 | 688.2ms/step | 29.1 samples/s +Step 100/44000 | Loss: 9.1597 | LR: 6.67e-05 | 691.1ms/step | 28.9 samples/s +Step 110/44000 | Loss: 8.6398 | LR: 7.33e-05 | 694.6ms/step | 28.8 samples/s +Step 120/44000 | Loss: 8.1386 | LR: 8.00e-05 | 698.0ms/step | 28.7 samples/s +Step 130/44000 | Loss: 7.8045 | LR: 8.67e-05 | 694.6ms/step | 28.8 samples/s +Step 140/44000 | Loss: 7.5424 | LR: 9.33e-05 | 693.5ms/step | 28.8 samples/s +Step 150/44000 | Loss: 7.3541 | LR: 1.00e-04 | 696.6ms/step | 28.7 samples/s +Step 160/44000 | Loss: 7.2434 | LR: 1.07e-04 | 695.4ms/step | 28.8 samples/s +Step 170/44000 | Loss: 7.0841 | LR: 1.13e-04 | 698.6ms/step | 28.6 samples/s +Step 180/44000 | Loss: 6.9701 | LR: 1.20e-04 | 693.3ms/step | 28.8 samples/s +Step 190/44000 | Loss: 6.9393 | LR: 1.27e-04 | 696.7ms/step | 28.7 samples/s +Step 200/44000 | Loss: 6.8594 | LR: 1.33e-04 | 699.0ms/step | 28.6 samples/s +Step 210/44000 | Loss: 6.7787 | LR: 1.40e-04 | 696.4ms/step | 28.7 samples/s +Step 220/44000 | Loss: 6.7004 | LR: 1.47e-04 | 696.8ms/step | 28.7 samples/s +Step 230/44000 | Loss: 6.6101 | LR: 1.53e-04 | 696.4ms/step | 28.7 samples/s +Step 240/44000 | Loss: 6.5091 | LR: 1.60e-04 | 697.7ms/step | 28.7 samples/s +Step 250/44000 | Loss: 6.4545 | LR: 1.67e-04 | 696.0ms/step | 28.7 samples/s +Validation at step 250: Loss = 6.4419, Perplexity = 627.61 +Step 260/44000 | Loss: 6.4244 | LR: 1.73e-04 | 4750.8ms/step | 4.2 samples/s +Step 270/44000 | Loss: 6.3623 | LR: 1.80e-04 | 698.4ms/step | 28.6 samples/s +Step 280/44000 | Loss: 6.3270 | LR: 1.87e-04 | 695.9ms/step | 28.7 samples/s +Step 290/44000 | Loss: 6.2875 | LR: 1.93e-04 | 696.8ms/step | 28.7 samples/s +Step 300/44000 | Loss: 6.1868 | LR: 2.00e-04 | 701.1ms/step | 28.5 samples/s +Step 310/44000 | Loss: 6.1684 | LR: 2.07e-04 | 697.0ms/step | 28.7 samples/s +Step 320/44000 | Loss: 6.1296 | LR: 2.13e-04 | 698.6ms/step | 28.6 samples/s +Step 330/44000 | Loss: 6.1037 | LR: 2.20e-04 | 700.1ms/step | 28.6 samples/s +Step 340/44000 | Loss: 6.0432 | LR: 2.27e-04 | 699.9ms/step | 28.6 samples/s +Step 350/44000 | Loss: 6.0191 | LR: 2.33e-04 | 696.7ms/step | 28.7 samples/s +Step 360/44000 | Loss: 5.9576 | LR: 2.40e-04 | 699.8ms/step | 28.6 samples/s +Step 370/44000 | Loss: 5.9278 | LR: 2.47e-04 | 699.3ms/step | 28.6 samples/s +Step 380/44000 | Loss: 5.8975 | LR: 2.53e-04 | 699.1ms/step | 28.6 samples/s +Step 390/44000 | Loss: 5.8879 | LR: 2.60e-04 | 698.9ms/step | 28.6 samples/s +Step 400/44000 | Loss: 5.8401 | LR: 2.67e-04 | 698.0ms/step | 28.7 samples/s +Step 410/44000 | Loss: 5.7973 | LR: 2.73e-04 | 700.1ms/step | 28.6 samples/s +Step 420/44000 | Loss: 5.7813 | LR: 2.80e-04 | 699.9ms/step | 28.6 samples/s +Step 430/44000 | Loss: 5.7372 | LR: 2.87e-04 | 701.6ms/step | 28.5 samples/s +Step 440/44000 | Loss: 5.7073 | LR: 2.93e-04 | 698.4ms/step | 28.6 samples/s +Step 450/44000 | Loss: 5.6339 | LR: 3.00e-04 | 703.1ms/step | 28.4 samples/s +Step 460/44000 | Loss: 5.6295 | LR: 3.07e-04 | 699.3ms/step | 28.6 samples/s +Step 470/44000 | Loss: 5.5882 | LR: 3.13e-04 | 699.0ms/step | 28.6 samples/s +Step 480/44000 | Loss: 5.5712 | LR: 3.20e-04 | 699.6ms/step | 28.6 samples/s +Step 490/44000 | Loss: 5.5519 | LR: 3.27e-04 | 700.7ms/step | 28.5 samples/s +Step 500/44000 | Loss: 5.4553 | LR: 3.33e-04 | 697.5ms/step | 28.7 samples/s +Validation at step 500: Loss = 5.5034, Perplexity = 245.53 +Step 510/44000 | Loss: 5.4800 | LR: 3.40e-04 | 1268.9ms/step | 15.8 samples/s +Step 520/44000 | Loss: 5.4190 | LR: 3.47e-04 | 702.9ms/step | 28.5 samples/s +Step 530/44000 | Loss: 5.3939 | LR: 3.53e-04 | 696.3ms/step | 28.7 samples/s +Step 540/44000 | Loss: 5.3575 | LR: 3.60e-04 | 700.2ms/step | 28.6 samples/s +Step 550/44000 | Loss: 5.3785 | LR: 3.67e-04 | 696.9ms/step | 28.7 samples/s +Step 560/44000 | Loss: 5.2714 | LR: 3.73e-04 | 700.4ms/step | 28.6 samples/s +Step 570/44000 | Loss: 5.2543 | LR: 3.80e-04 | 703.9ms/step | 28.4 samples/s +Step 580/44000 | Loss: 5.2045 | LR: 3.87e-04 | 701.2ms/step | 28.5 samples/s +Step 590/44000 | Loss: 5.2094 | LR: 3.93e-04 | 696.2ms/step | 28.7 samples/s +Step 600/44000 | Loss: 5.1816 | LR: 4.00e-04 | 698.5ms/step | 28.6 samples/s +Step 610/44000 | Loss: 5.1837 | LR: 4.07e-04 | 701.2ms/step | 28.5 samples/s +Step 620/44000 | Loss: 5.1162 | LR: 4.13e-04 | 700.0ms/step | 28.6 samples/s +Step 630/44000 | Loss: 5.1829 | LR: 4.20e-04 | 698.3ms/step | 28.6 samples/s +Step 640/44000 | Loss: 5.0693 | LR: 4.27e-04 | 698.5ms/step | 28.6 samples/s +Step 650/44000 | Loss: 5.0770 | LR: 4.33e-04 | 701.0ms/step | 28.5 samples/s +Step 660/44000 | Loss: 5.0557 | LR: 4.40e-04 | 697.6ms/step | 28.7 samples/s +Step 670/44000 | Loss: 4.9529 | LR: 4.47e-04 | 700.8ms/step | 28.5 samples/s +Step 680/44000 | Loss: 5.0263 | LR: 4.53e-04 | 698.2ms/step | 28.6 samples/s +Step 690/44000 | Loss: 4.9507 | LR: 4.60e-04 | 697.3ms/step | 28.7 samples/s +Step 700/44000 | Loss: 4.9047 | LR: 4.67e-04 | 698.0ms/step | 28.7 samples/s +Step 710/44000 | Loss: 4.9017 | LR: 4.73e-04 | 702.1ms/step | 28.5 samples/s +Step 720/44000 | Loss: 4.8544 | LR: 4.80e-04 | 699.9ms/step | 28.6 samples/s +Step 730/44000 | Loss: 4.8629 | LR: 4.87e-04 | 698.9ms/step | 28.6 samples/s +Step 740/44000 | Loss: 4.8224 | LR: 4.93e-04 | 702.5ms/step | 28.5 samples/s +Step 750/44000 | Loss: 4.7621 | LR: 5.00e-04 | 697.2ms/step | 28.7 samples/s +Validation at step 750: Loss = 4.8065, Perplexity = 122.30 +Step 760/44000 | Loss: 4.7786 | LR: 5.00e-04 | 1268.0ms/step | 15.8 samples/s +Step 770/44000 | Loss: 4.7420 | LR: 5.00e-04 | 702.6ms/step | 28.5 samples/s +Step 780/44000 | Loss: 4.6996 | LR: 5.00e-04 | 696.7ms/step | 28.7 samples/s +Step 790/44000 | Loss: 4.7048 | LR: 5.00e-04 | 695.6ms/step | 28.8 samples/s +Step 800/44000 | Loss: 4.6441 | LR: 5.00e-04 | 698.3ms/step | 28.6 samples/s +Step 810/44000 | Loss: 4.6468 | LR: 5.00e-04 | 698.6ms/step | 28.6 samples/s +Step 820/44000 | Loss: 4.6185 | LR: 5.00e-04 | 697.8ms/step | 28.7 samples/s +Step 830/44000 | Loss: 4.5799 | LR: 5.00e-04 | 696.3ms/step | 28.7 samples/s +Step 840/44000 | Loss: 4.5612 | LR: 5.00e-04 | 699.6ms/step | 28.6 samples/s +Step 850/44000 | Loss: 4.5360 | LR: 5.00e-04 | 697.3ms/step | 28.7 samples/s +Step 860/44000 | Loss: 4.5021 | LR: 5.00e-04 | 696.0ms/step | 28.7 samples/s +Step 870/44000 | Loss: 4.4960 | LR: 5.00e-04 | 697.2ms/step | 28.7 samples/s +Step 880/44000 | Loss: 4.4506 | LR: 5.00e-04 | 697.2ms/step | 28.7 samples/s +Step 890/44000 | Loss: 4.4812 | LR: 5.00e-04 | 693.5ms/step | 28.8 samples/s +Step 900/44000 | Loss: 4.4662 | LR: 5.00e-04 | 695.8ms/step | 28.7 samples/s +Step 910/44000 | Loss: 4.4534 | LR: 5.00e-04 | 697.0ms/step | 28.7 samples/s +Step 920/44000 | Loss: 4.4670 | LR: 5.00e-04 | 696.0ms/step | 28.7 samples/s +Step 930/44000 | Loss: 4.3942 | LR: 5.00e-04 | 695.0ms/step | 28.8 samples/s +Step 940/44000 | Loss: 4.3849 | LR: 5.00e-04 | 694.4ms/step | 28.8 samples/s +Step 950/44000 | Loss: 4.3936 | LR: 5.00e-04 | 693.6ms/step | 28.8 samples/s +Step 960/44000 | Loss: 4.3578 | LR: 5.00e-04 | 697.7ms/step | 28.7 samples/s +Step 970/44000 | Loss: 4.3614 | LR: 5.00e-04 | 701.6ms/step | 28.5 samples/s +Step 980/44000 | Loss: 4.3722 | LR: 5.00e-04 | 694.4ms/step | 28.8 samples/s +Step 990/44000 | Loss: 4.3406 | LR: 5.00e-04 | 700.4ms/step | 28.6 samples/s +Step 1000/44000 | Loss: 4.3204 | LR: 5.00e-04 | 699.7ms/step | 28.6 samples/s +Validation at step 1000: Loss = 4.3266, Perplexity = 75.69 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-1000 +Step 1010/44000 | Loss: 4.3534 | LR: 5.00e-04 | 1579.8ms/step | 12.7 samples/s +Step 1020/44000 | Loss: 4.2691 | LR: 5.00e-04 | 698.5ms/step | 28.6 samples/s +Step 1030/44000 | Loss: 4.2755 | LR: 5.00e-04 | 699.4ms/step | 28.6 samples/s +Step 1040/44000 | Loss: 4.2101 | LR: 5.00e-04 | 700.4ms/step | 28.6 samples/s +Step 1050/44000 | Loss: 4.2678 | LR: 5.00e-04 | 698.5ms/step | 28.6 samples/s +Step 1060/44000 | Loss: 4.2649 | LR: 5.00e-04 | 696.3ms/step | 28.7 samples/s +Step 1070/44000 | Loss: 4.2128 | LR: 5.00e-04 | 700.7ms/step | 28.5 samples/s +Step 1080/44000 | Loss: 4.1731 | LR: 5.00e-04 | 698.1ms/step | 28.7 samples/s +Step 1090/44000 | Loss: 4.2279 | LR: 5.00e-04 | 702.2ms/step | 28.5 samples/s +Step 1100/44000 | Loss: 4.1752 | LR: 5.00e-04 | 701.4ms/step | 28.5 samples/s +Step 1110/44000 | Loss: 4.1863 | LR: 5.00e-04 | 702.2ms/step | 28.5 samples/s +Step 1120/44000 | Loss: 4.2261 | LR: 5.00e-04 | 697.5ms/step | 28.7 samples/s +Step 1130/44000 | Loss: 4.1743 | LR: 5.00e-04 | 698.7ms/step | 28.6 samples/s +Step 1140/44000 | Loss: 4.1944 | LR: 5.00e-04 | 696.6ms/step | 28.7 samples/s +Step 1150/44000 | Loss: 4.1788 | LR: 5.00e-04 | 700.3ms/step | 28.6 samples/s +Step 1160/44000 | Loss: 4.1477 | LR: 5.00e-04 | 699.7ms/step | 28.6 samples/s +Step 1170/44000 | Loss: 4.1276 | LR: 5.00e-04 | 697.3ms/step | 28.7 samples/s +Step 1180/44000 | Loss: 4.0839 | LR: 5.00e-04 | 698.4ms/step | 28.6 samples/s +Step 1190/44000 | Loss: 4.1425 | LR: 5.00e-04 | 699.8ms/step | 28.6 samples/s +Step 1200/44000 | Loss: 4.1245 | LR: 5.00e-04 | 695.7ms/step | 28.7 samples/s +Step 1210/44000 | Loss: 4.1021 | LR: 5.00e-04 | 698.5ms/step | 28.6 samples/s +Step 1220/44000 | Loss: 4.0782 | LR: 5.00e-04 | 697.5ms/step | 28.7 samples/s +Step 1230/44000 | Loss: 4.1067 | LR: 5.00e-04 | 696.7ms/step | 28.7 samples/s +Step 1240/44000 | Loss: 4.0771 | LR: 5.00e-04 | 699.9ms/step | 28.6 samples/s +Step 1250/44000 | Loss: 4.0294 | LR: 5.00e-04 | 698.6ms/step | 28.6 samples/s +Validation at step 1250: Loss = 4.0965, Perplexity = 60.13 +Step 1260/44000 | Loss: 4.0908 | LR: 5.00e-04 | 1262.1ms/step | 15.8 samples/s +Step 1270/44000 | Loss: 4.0418 | LR: 5.00e-04 | 714.0ms/step | 28.0 samples/s +Step 1280/44000 | Loss: 4.0349 | LR: 5.00e-04 | 715.7ms/step | 27.9 samples/s +Step 1290/44000 | Loss: 4.0402 | LR: 5.00e-04 | 699.6ms/step | 28.6 samples/s +Step 1300/44000 | Loss: 4.0701 | LR: 5.00e-04 | 696.4ms/step | 28.7 samples/s +Step 1310/44000 | Loss: 4.0424 | LR: 5.00e-04 | 700.3ms/step | 28.6 samples/s +Step 1320/44000 | Loss: 4.0743 | LR: 5.00e-04 | 700.4ms/step | 28.6 samples/s +Step 1330/44000 | Loss: 3.9941 | LR: 5.00e-04 | 699.4ms/step | 28.6 samples/s +Step 1340/44000 | Loss: 4.0060 | LR: 5.00e-04 | 697.7ms/step | 28.7 samples/s +Step 1350/44000 | Loss: 4.0320 | LR: 5.00e-04 | 698.1ms/step | 28.7 samples/s +Step 1360/44000 | Loss: 4.0372 | LR: 5.00e-04 | 699.8ms/step | 28.6 samples/s +Step 1370/44000 | Loss: 3.9745 | LR: 5.00e-04 | 696.5ms/step | 28.7 samples/s +Step 1380/44000 | Loss: 4.0005 | LR: 5.00e-04 | 702.3ms/step | 28.5 samples/s +Step 1390/44000 | Loss: 3.9518 | LR: 5.00e-04 | 703.5ms/step | 28.4 samples/s +Step 1400/44000 | Loss: 3.9575 | LR: 5.00e-04 | 700.7ms/step | 28.5 samples/s +Step 1410/44000 | Loss: 3.9535 | LR: 5.00e-04 | 698.1ms/step | 28.6 samples/s +Step 1420/44000 | Loss: 3.9561 | LR: 5.00e-04 | 698.0ms/step | 28.7 samples/s +Step 1430/44000 | Loss: 3.9690 | LR: 5.00e-04 | 706.7ms/step | 28.3 samples/s +Step 1440/44000 | Loss: 3.9473 | LR: 5.00e-04 | 706.4ms/step | 28.3 samples/s +Step 1450/44000 | Loss: 3.9465 | LR: 5.00e-04 | 716.8ms/step | 27.9 samples/s +Step 1460/44000 | Loss: 3.9339 | LR: 5.00e-04 | 723.2ms/step | 27.7 samples/s +Step 1470/44000 | Loss: 3.9482 | LR: 5.00e-04 | 717.1ms/step | 27.9 samples/s +Step 1480/44000 | Loss: 3.8976 | LR: 5.00e-04 | 719.7ms/step | 27.8 samples/s +Step 1490/44000 | Loss: 3.9453 | LR: 5.00e-04 | 716.9ms/step | 27.9 samples/s +Step 1500/44000 | Loss: 3.8954 | LR: 5.00e-04 | 715.3ms/step | 28.0 samples/s +Validation at step 1500: Loss = 3.9382, Perplexity = 51.33 +Step 1510/44000 | Loss: 3.9431 | LR: 5.00e-04 | 1313.8ms/step | 15.2 samples/s +Step 1520/44000 | Loss: 3.8936 | LR: 5.00e-04 | 702.9ms/step | 28.5 samples/s +Step 1530/44000 | Loss: 3.8792 | LR: 5.00e-04 | 701.0ms/step | 28.5 samples/s +Step 1540/44000 | Loss: 3.8802 | LR: 5.00e-04 | 707.8ms/step | 28.3 samples/s +Step 1550/44000 | Loss: 3.9193 | LR: 5.00e-04 | 705.6ms/step | 28.3 samples/s +Step 1560/44000 | Loss: 3.8938 | LR: 5.00e-04 | 706.9ms/step | 28.3 samples/s +Step 1570/44000 | Loss: 3.8632 | LR: 5.00e-04 | 716.1ms/step | 27.9 samples/s +Step 1580/44000 | Loss: 3.8765 | LR: 5.00e-04 | 702.4ms/step | 28.5 samples/s +Step 1590/44000 | Loss: 3.8682 | LR: 5.00e-04 | 700.9ms/step | 28.5 samples/s +Step 1600/44000 | Loss: 3.8952 | LR: 5.00e-04 | 703.2ms/step | 28.4 samples/s +Step 1610/44000 | Loss: 3.8483 | LR: 5.00e-04 | 703.7ms/step | 28.4 samples/s +Step 1620/44000 | Loss: 3.8958 | LR: 5.00e-04 | 703.5ms/step | 28.4 samples/s +Step 1630/44000 | Loss: 3.8314 | LR: 4.99e-04 | 701.8ms/step | 28.5 samples/s +Step 1640/44000 | Loss: 3.8376 | LR: 4.99e-04 | 704.5ms/step | 28.4 samples/s +Step 1650/44000 | Loss: 3.8696 | LR: 4.99e-04 | 702.8ms/step | 28.5 samples/s +Step 1660/44000 | Loss: 3.7944 | LR: 4.99e-04 | 700.5ms/step | 28.6 samples/s +Step 1670/44000 | Loss: 3.8318 | LR: 4.99e-04 | 700.7ms/step | 28.5 samples/s +Step 1680/44000 | Loss: 3.8380 | LR: 4.99e-04 | 701.3ms/step | 28.5 samples/s +Step 1690/44000 | Loss: 3.8242 | LR: 4.99e-04 | 702.2ms/step | 28.5 samples/s +Step 1700/44000 | Loss: 3.8000 | LR: 4.99e-04 | 709.2ms/step | 28.2 samples/s +Step 1710/44000 | Loss: 3.8273 | LR: 4.99e-04 | 714.2ms/step | 28.0 samples/s +Step 1720/44000 | Loss: 3.8500 | LR: 4.99e-04 | 716.3ms/step | 27.9 samples/s +Step 1730/44000 | Loss: 3.8134 | LR: 4.99e-04 | 716.2ms/step | 27.9 samples/s +Step 1740/44000 | Loss: 3.8113 | LR: 4.99e-04 | 707.4ms/step | 28.3 samples/s +Step 1750/44000 | Loss: 3.8476 | LR: 4.99e-04 | 704.3ms/step | 28.4 samples/s +Validation at step 1750: Loss = 3.8320, Perplexity = 46.16 +Step 1760/44000 | Loss: 3.7776 | LR: 4.99e-04 | 1282.1ms/step | 15.6 samples/s +Step 1770/44000 | Loss: 3.7750 | LR: 4.99e-04 | 701.4ms/step | 28.5 samples/s +Step 1780/44000 | Loss: 3.7984 | LR: 4.99e-04 | 706.1ms/step | 28.3 samples/s +Step 1790/44000 | Loss: 3.8052 | LR: 4.99e-04 | 705.0ms/step | 28.4 samples/s +Step 1800/44000 | Loss: 3.8005 | LR: 4.99e-04 | 705.9ms/step | 28.3 samples/s +Step 1810/44000 | Loss: 3.7822 | LR: 4.99e-04 | 702.5ms/step | 28.5 samples/s +Step 1820/44000 | Loss: 3.7373 | LR: 4.99e-04 | 705.4ms/step | 28.4 samples/s +Step 1830/44000 | Loss: 3.7631 | LR: 4.99e-04 | 702.6ms/step | 28.5 samples/s +Step 1840/44000 | Loss: 3.7705 | LR: 4.99e-04 | 709.6ms/step | 28.2 samples/s +Step 1850/44000 | Loss: 3.7401 | LR: 4.99e-04 | 705.1ms/step | 28.4 samples/s +Step 1860/44000 | Loss: 3.7626 | LR: 4.99e-04 | 708.4ms/step | 28.2 samples/s +Step 1870/44000 | Loss: 3.7820 | LR: 4.99e-04 | 703.1ms/step | 28.4 samples/s +Step 1880/44000 | Loss: 3.7665 | LR: 4.99e-04 | 706.1ms/step | 28.3 samples/s +Step 1890/44000 | Loss: 3.7659 | LR: 4.99e-04 | 708.1ms/step | 28.2 samples/s +Step 1900/44000 | Loss: 3.7415 | LR: 4.99e-04 | 703.4ms/step | 28.4 samples/s +Step 1910/44000 | Loss: 3.7959 | LR: 4.99e-04 | 706.1ms/step | 28.3 samples/s +Step 1920/44000 | Loss: 3.7425 | LR: 4.99e-04 | 714.6ms/step | 28.0 samples/s +Step 1930/44000 | Loss: 3.7544 | LR: 4.99e-04 | 717.6ms/step | 27.9 samples/s +Step 1940/44000 | Loss: 3.7382 | LR: 4.99e-04 | 715.2ms/step | 28.0 samples/s +Step 1950/44000 | Loss: 3.7195 | LR: 4.99e-04 | 713.5ms/step | 28.0 samples/s +Step 1960/44000 | Loss: 3.7229 | LR: 4.99e-04 | 722.8ms/step | 27.7 samples/s +Step 1970/44000 | Loss: 3.7378 | LR: 4.99e-04 | 712.4ms/step | 28.1 samples/s +Step 1980/44000 | Loss: 3.7043 | LR: 4.99e-04 | 711.8ms/step | 28.1 samples/s +Step 1990/44000 | Loss: 3.7198 | LR: 4.99e-04 | 708.8ms/step | 28.2 samples/s +Step 2000/44000 | Loss: 3.6761 | LR: 4.99e-04 | 709.3ms/step | 28.2 samples/s +Validation at step 2000: Loss = 3.7503, Perplexity = 42.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-2000 +Step 2010/44000 | Loss: 3.6931 | LR: 4.99e-04 | 1581.2ms/step | 12.6 samples/s +Step 2020/44000 | Loss: 3.7081 | LR: 4.99e-04 | 708.4ms/step | 28.2 samples/s +Step 2030/44000 | Loss: 3.6887 | LR: 4.99e-04 | 713.2ms/step | 28.0 samples/s +Step 2040/44000 | Loss: 3.7232 | LR: 4.99e-04 | 711.2ms/step | 28.1 samples/s +Step 2050/44000 | Loss: 3.7082 | LR: 4.99e-04 | 705.6ms/step | 28.3 samples/s +Step 2060/44000 | Loss: 3.7044 | LR: 4.99e-04 | 711.5ms/step | 28.1 samples/s +Step 2070/44000 | Loss: 3.6807 | LR: 4.99e-04 | 702.7ms/step | 28.5 samples/s +Step 2080/44000 | Loss: 3.7172 | LR: 4.99e-04 | 701.9ms/step | 28.5 samples/s +Step 2090/44000 | Loss: 3.7143 | LR: 4.99e-04 | 701.8ms/step | 28.5 samples/s +Step 2100/44000 | Loss: 3.6965 | LR: 4.99e-04 | 703.0ms/step | 28.4 samples/s +Step 2110/44000 | Loss: 3.6781 | LR: 4.99e-04 | 703.0ms/step | 28.5 samples/s +Step 2120/44000 | Loss: 3.6924 | LR: 4.99e-04 | 705.4ms/step | 28.4 samples/s +Step 2130/44000 | Loss: 3.6640 | LR: 4.99e-04 | 708.1ms/step | 28.2 samples/s +Step 2140/44000 | Loss: 3.6833 | LR: 4.99e-04 | 706.3ms/step | 28.3 samples/s +Step 2150/44000 | Loss: 3.6897 | LR: 4.99e-04 | 706.6ms/step | 28.3 samples/s +Step 2160/44000 | Loss: 3.6945 | LR: 4.99e-04 | 712.4ms/step | 28.1 samples/s +Step 2170/44000 | Loss: 3.6912 | LR: 4.99e-04 | 703.3ms/step | 28.4 samples/s +Step 2180/44000 | Loss: 3.6611 | LR: 4.99e-04 | 699.4ms/step | 28.6 samples/s +Step 2190/44000 | Loss: 3.6986 | LR: 4.99e-04 | 700.2ms/step | 28.6 samples/s +Step 2200/44000 | Loss: 3.6764 | LR: 4.99e-04 | 705.9ms/step | 28.3 samples/s +Step 2210/44000 | Loss: 3.6453 | LR: 4.99e-04 | 702.7ms/step | 28.5 samples/s +Step 2220/44000 | Loss: 3.6984 | LR: 4.99e-04 | 707.0ms/step | 28.3 samples/s +Step 2230/44000 | Loss: 3.7033 | LR: 4.99e-04 | 703.3ms/step | 28.4 samples/s +Step 2240/44000 | Loss: 3.6981 | LR: 4.99e-04 | 699.1ms/step | 28.6 samples/s +Step 2250/44000 | Loss: 3.6457 | LR: 4.99e-04 | 705.0ms/step | 28.4 samples/s +Validation at step 2250: Loss = 3.6904, Perplexity = 40.06 +Step 2260/44000 | Loss: 3.6826 | LR: 4.98e-04 | 1293.5ms/step | 15.5 samples/s +Step 2270/44000 | Loss: 3.6692 | LR: 4.98e-04 | 708.9ms/step | 28.2 samples/s +Step 2280/44000 | Loss: 3.6592 | LR: 4.98e-04 | 706.2ms/step | 28.3 samples/s +Step 2290/44000 | Loss: 3.6363 | LR: 4.98e-04 | 706.8ms/step | 28.3 samples/s +Step 2300/44000 | Loss: 3.6766 | LR: 4.98e-04 | 705.9ms/step | 28.3 samples/s +Step 2310/44000 | Loss: 3.6791 | LR: 4.98e-04 | 709.6ms/step | 28.2 samples/s +Step 2320/44000 | Loss: 3.6368 | LR: 4.98e-04 | 709.7ms/step | 28.2 samples/s +Step 2330/44000 | Loss: 3.6609 | LR: 4.98e-04 | 708.4ms/step | 28.2 samples/s +Step 2340/44000 | Loss: 3.6630 | LR: 4.98e-04 | 703.2ms/step | 28.4 samples/s +Step 2350/44000 | Loss: 3.6327 | LR: 4.98e-04 | 705.9ms/step | 28.3 samples/s +Step 2360/44000 | Loss: 3.6075 | LR: 4.98e-04 | 700.1ms/step | 28.6 samples/s +Step 2370/44000 | Loss: 3.6703 | LR: 4.98e-04 | 705.3ms/step | 28.4 samples/s +Step 2380/44000 | Loss: 3.6525 | LR: 4.98e-04 | 704.6ms/step | 28.4 samples/s +Step 2390/44000 | Loss: 3.6139 | LR: 4.98e-04 | 704.1ms/step | 28.4 samples/s +Step 2400/44000 | Loss: 3.6590 | LR: 4.98e-04 | 705.2ms/step | 28.4 samples/s +Step 2410/44000 | Loss: 3.6137 | LR: 4.98e-04 | 710.9ms/step | 28.1 samples/s +Step 2420/44000 | Loss: 3.6500 | LR: 4.98e-04 | 703.8ms/step | 28.4 samples/s +Step 2430/44000 | Loss: 3.6181 | LR: 4.98e-04 | 706.1ms/step | 28.3 samples/s +Step 2440/44000 | Loss: 3.6739 | LR: 4.98e-04 | 704.7ms/step | 28.4 samples/s +Step 2450/44000 | Loss: 3.6699 | LR: 4.98e-04 | 704.7ms/step | 28.4 samples/s +Step 2460/44000 | Loss: 3.6140 | LR: 4.98e-04 | 704.2ms/step | 28.4 samples/s +Step 2470/44000 | Loss: 3.6051 | LR: 4.98e-04 | 703.2ms/step | 28.4 samples/s +Step 2480/44000 | Loss: 3.6016 | LR: 4.98e-04 | 704.1ms/step | 28.4 samples/s +Step 2490/44000 | Loss: 3.5800 | LR: 4.98e-04 | 703.7ms/step | 28.4 samples/s +Step 2500/44000 | Loss: 3.5819 | LR: 4.98e-04 | 703.6ms/step | 28.4 samples/s +Validation at step 2500: Loss = 3.6363, Perplexity = 37.95 +Step 2510/44000 | Loss: 3.6483 | LR: 4.98e-04 | 1280.6ms/step | 15.6 samples/s +Step 2520/44000 | Loss: 3.5742 | LR: 4.98e-04 | 702.5ms/step | 28.5 samples/s +Step 2530/44000 | Loss: 3.5823 | LR: 4.98e-04 | 706.2ms/step | 28.3 samples/s +Step 2540/44000 | Loss: 3.5545 | LR: 4.98e-04 | 708.8ms/step | 28.2 samples/s +Step 2550/44000 | Loss: 3.5751 | LR: 4.98e-04 | 703.0ms/step | 28.5 samples/s +Step 2560/44000 | Loss: 3.5690 | LR: 4.98e-04 | 704.1ms/step | 28.4 samples/s +Step 2570/44000 | Loss: 3.5920 | LR: 4.98e-04 | 703.9ms/step | 28.4 samples/s +Step 2580/44000 | Loss: 3.6108 | LR: 4.98e-04 | 699.9ms/step | 28.6 samples/s +Step 2590/44000 | Loss: 3.5867 | LR: 4.98e-04 | 703.7ms/step | 28.4 samples/s +Step 2600/44000 | Loss: 3.5684 | LR: 4.98e-04 | 703.9ms/step | 28.4 samples/s +Step 2610/44000 | Loss: 3.5798 | LR: 4.98e-04 | 700.3ms/step | 28.6 samples/s +Step 2620/44000 | Loss: 3.6051 | LR: 4.98e-04 | 701.9ms/step | 28.5 samples/s +Step 2630/44000 | Loss: 3.5749 | LR: 4.98e-04 | 702.4ms/step | 28.5 samples/s +Step 2640/44000 | Loss: 3.5510 | LR: 4.98e-04 | 699.7ms/step | 28.6 samples/s +Step 2650/44000 | Loss: 3.6084 | LR: 4.98e-04 | 703.7ms/step | 28.4 samples/s +Step 2660/44000 | Loss: 3.5537 | LR: 4.98e-04 | 700.3ms/step | 28.6 samples/s +Step 2670/44000 | Loss: 3.5414 | LR: 4.98e-04 | 700.0ms/step | 28.6 samples/s +Step 2680/44000 | Loss: 3.5576 | LR: 4.98e-04 | 699.1ms/step | 28.6 samples/s +Step 2690/44000 | Loss: 3.5880 | LR: 4.98e-04 | 699.9ms/step | 28.6 samples/s +Step 2700/44000 | Loss: 3.5666 | LR: 4.97e-04 | 702.0ms/step | 28.5 samples/s +Step 2710/44000 | Loss: 3.5779 | LR: 4.97e-04 | 707.1ms/step | 28.3 samples/s +Step 2720/44000 | Loss: 3.5286 | LR: 4.97e-04 | 701.5ms/step | 28.5 samples/s +Step 2730/44000 | Loss: 3.5392 | LR: 4.97e-04 | 701.9ms/step | 28.5 samples/s +Step 2740/44000 | Loss: 3.5332 | LR: 4.97e-04 | 699.5ms/step | 28.6 samples/s +Step 2750/44000 | Loss: 3.5835 | LR: 4.97e-04 | 700.6ms/step | 28.5 samples/s +Validation at step 2750: Loss = 3.5869, Perplexity = 36.12 +Step 2760/44000 | Loss: 3.5510 | LR: 4.97e-04 | 1274.0ms/step | 15.7 samples/s +Step 2770/44000 | Loss: 3.6110 | LR: 4.97e-04 | 700.8ms/step | 28.5 samples/s +Step 2780/44000 | Loss: 3.5688 | LR: 4.97e-04 | 700.1ms/step | 28.6 samples/s +Step 2790/44000 | Loss: 3.5263 | LR: 4.97e-04 | 698.8ms/step | 28.6 samples/s +Step 2800/44000 | Loss: 3.5242 | LR: 4.97e-04 | 701.3ms/step | 28.5 samples/s +Step 2810/44000 | Loss: 3.5876 | LR: 4.97e-04 | 703.1ms/step | 28.4 samples/s +Step 2820/44000 | Loss: 3.5477 | LR: 4.97e-04 | 700.7ms/step | 28.5 samples/s +Step 2830/44000 | Loss: 3.5656 | LR: 4.97e-04 | 695.6ms/step | 28.8 samples/s +Step 2840/44000 | Loss: 3.5525 | LR: 4.97e-04 | 699.5ms/step | 28.6 samples/s +Step 2850/44000 | Loss: 3.5584 | LR: 4.97e-04 | 700.1ms/step | 28.6 samples/s +Step 2860/44000 | Loss: 3.5278 | LR: 4.97e-04 | 699.5ms/step | 28.6 samples/s +Step 2870/44000 | Loss: 3.5360 | LR: 4.97e-04 | 704.1ms/step | 28.4 samples/s +Step 2880/44000 | Loss: 3.5744 | LR: 4.97e-04 | 701.1ms/step | 28.5 samples/s +Step 2890/44000 | Loss: 3.5521 | LR: 4.97e-04 | 703.4ms/step | 28.4 samples/s +Step 2900/44000 | Loss: 3.5728 | LR: 4.97e-04 | 699.5ms/step | 28.6 samples/s +Step 2910/44000 | Loss: 3.5165 | LR: 4.97e-04 | 699.3ms/step | 28.6 samples/s +Step 2920/44000 | Loss: 3.5348 | LR: 4.97e-04 | 695.6ms/step | 28.8 samples/s +Step 2930/44000 | Loss: 3.5632 | LR: 4.97e-04 | 696.6ms/step | 28.7 samples/s +Step 2940/44000 | Loss: 3.5087 | LR: 4.97e-04 | 698.9ms/step | 28.6 samples/s +Step 2950/44000 | Loss: 3.5287 | LR: 4.97e-04 | 697.3ms/step | 28.7 samples/s +Step 2960/44000 | Loss: 3.5483 | LR: 4.97e-04 | 702.3ms/step | 28.5 samples/s +Step 2970/44000 | Loss: 3.5302 | LR: 4.97e-04 | 701.4ms/step | 28.5 samples/s +Step 2980/44000 | Loss: 3.5341 | LR: 4.97e-04 | 700.4ms/step | 28.6 samples/s +Step 2990/44000 | Loss: 3.5085 | LR: 4.97e-04 | 700.4ms/step | 28.6 samples/s +Step 3000/44000 | Loss: 3.5162 | LR: 4.97e-04 | 697.6ms/step | 28.7 samples/s +Validation at step 3000: Loss = 3.5504, Perplexity = 34.83 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-3000 +Step 3010/44000 | Loss: 3.5282 | LR: 4.97e-04 | 1592.7ms/step | 12.6 samples/s +Step 3020/44000 | Loss: 3.5092 | LR: 4.97e-04 | 700.6ms/step | 28.5 samples/s +Step 3030/44000 | Loss: 3.5023 | LR: 4.97e-04 | 698.8ms/step | 28.6 samples/s +Step 3040/44000 | Loss: 3.5582 | LR: 4.97e-04 | 702.8ms/step | 28.5 samples/s +Step 3050/44000 | Loss: 3.5064 | LR: 4.97e-04 | 699.2ms/step | 28.6 samples/s +Step 3060/44000 | Loss: 3.5319 | LR: 4.96e-04 | 698.0ms/step | 28.7 samples/s +Step 3070/44000 | Loss: 3.4553 | LR: 4.96e-04 | 698.6ms/step | 28.6 samples/s +Step 3080/44000 | Loss: 3.4979 | LR: 4.96e-04 | 698.5ms/step | 28.6 samples/s +Step 3090/44000 | Loss: 3.4899 | LR: 4.96e-04 | 700.0ms/step | 28.6 samples/s +Step 3100/44000 | Loss: 3.5478 | LR: 4.96e-04 | 699.6ms/step | 28.6 samples/s +Step 3110/44000 | Loss: 3.5416 | LR: 4.96e-04 | 699.3ms/step | 28.6 samples/s +Step 3120/44000 | Loss: 3.4597 | LR: 4.96e-04 | 701.0ms/step | 28.5 samples/s +Step 3130/44000 | Loss: 3.5094 | LR: 4.96e-04 | 699.4ms/step | 28.6 samples/s +Step 3140/44000 | Loss: 3.5024 | LR: 4.96e-04 | 697.0ms/step | 28.7 samples/s +Step 3150/44000 | Loss: 3.4635 | LR: 4.96e-04 | 693.6ms/step | 28.8 samples/s +Step 3160/44000 | Loss: 3.5213 | LR: 4.96e-04 | 695.5ms/step | 28.8 samples/s +Step 3170/44000 | Loss: 3.5423 | LR: 4.96e-04 | 698.0ms/step | 28.7 samples/s +Step 3180/44000 | Loss: 3.5055 | LR: 4.96e-04 | 700.4ms/step | 28.6 samples/s +Step 3190/44000 | Loss: 3.5008 | LR: 4.96e-04 | 700.4ms/step | 28.6 samples/s +Step 3200/44000 | Loss: 3.4690 | LR: 4.96e-04 | 697.2ms/step | 28.7 samples/s +Step 3210/44000 | Loss: 3.5023 | LR: 4.96e-04 | 699.5ms/step | 28.6 samples/s +Step 3220/44000 | Loss: 3.5059 | LR: 4.96e-04 | 699.3ms/step | 28.6 samples/s +Step 3230/44000 | Loss: 3.4799 | LR: 4.96e-04 | 697.0ms/step | 28.7 samples/s +Step 3240/44000 | Loss: 3.4944 | LR: 4.96e-04 | 700.4ms/step | 28.6 samples/s +Step 3250/44000 | Loss: 3.5316 | LR: 4.96e-04 | 697.4ms/step | 28.7 samples/s +Validation at step 3250: Loss = 3.5186, Perplexity = 33.74 +Step 3260/44000 | Loss: 3.5380 | LR: 4.96e-04 | 1268.2ms/step | 15.8 samples/s +Step 3270/44000 | Loss: 3.5822 | LR: 4.96e-04 | 700.5ms/step | 28.5 samples/s +Step 3280/44000 | Loss: 3.5123 | LR: 4.96e-04 | 699.5ms/step | 28.6 samples/s +Step 3290/44000 | Loss: 3.4557 | LR: 4.96e-04 | 699.5ms/step | 28.6 samples/s +Step 3300/44000 | Loss: 3.4822 | LR: 4.96e-04 | 699.1ms/step | 28.6 samples/s +Step 3310/44000 | Loss: 3.4372 | LR: 4.96e-04 | 697.6ms/step | 28.7 samples/s +Step 3320/44000 | Loss: 3.4794 | LR: 4.96e-04 | 697.0ms/step | 28.7 samples/s +Step 3330/44000 | Loss: 3.4498 | LR: 4.96e-04 | 700.9ms/step | 28.5 samples/s +Step 3340/44000 | Loss: 3.4335 | LR: 4.96e-04 | 695.6ms/step | 28.8 samples/s +Step 3350/44000 | Loss: 3.5322 | LR: 4.96e-04 | 698.6ms/step | 28.6 samples/s +Step 3360/44000 | Loss: 3.4723 | LR: 4.96e-04 | 697.5ms/step | 28.7 samples/s +Step 3370/44000 | Loss: 3.4678 | LR: 4.95e-04 | 700.4ms/step | 28.6 samples/s +Step 3380/44000 | Loss: 3.4409 | LR: 4.95e-04 | 699.7ms/step | 28.6 samples/s +Step 3390/44000 | Loss: 3.4747 | LR: 4.95e-04 | 702.4ms/step | 28.5 samples/s +Step 3400/44000 | Loss: 3.4607 | LR: 4.95e-04 | 701.7ms/step | 28.5 samples/s +Step 3410/44000 | Loss: 3.4522 | LR: 4.95e-04 | 699.4ms/step | 28.6 samples/s +Step 3420/44000 | Loss: 3.4622 | LR: 4.95e-04 | 699.3ms/step | 28.6 samples/s +Step 3430/44000 | Loss: 3.5288 | LR: 4.95e-04 | 697.0ms/step | 28.7 samples/s +Step 3440/44000 | Loss: 3.4774 | LR: 4.95e-04 | 700.4ms/step | 28.6 samples/s +Step 3450/44000 | Loss: 3.4496 | LR: 4.95e-04 | 699.3ms/step | 28.6 samples/s +Step 3460/44000 | Loss: 3.4595 | LR: 4.95e-04 | 698.8ms/step | 28.6 samples/s +Step 3470/44000 | Loss: 3.4491 | LR: 4.95e-04 | 697.3ms/step | 28.7 samples/s +Step 3480/44000 | Loss: 3.4418 | LR: 4.95e-04 | 701.3ms/step | 28.5 samples/s +Step 3490/44000 | Loss: 3.4840 | LR: 4.95e-04 | 697.4ms/step | 28.7 samples/s +Step 3500/44000 | Loss: 3.4593 | LR: 4.95e-04 | 702.0ms/step | 28.5 samples/s +Validation at step 3500: Loss = 3.4897, Perplexity = 32.78 +Step 3510/44000 | Loss: 3.4839 | LR: 4.95e-04 | 1271.1ms/step | 15.7 samples/s +Step 3520/44000 | Loss: 3.4645 | LR: 4.95e-04 | 698.7ms/step | 28.6 samples/s +Step 3530/44000 | Loss: 3.4395 | LR: 4.95e-04 | 700.6ms/step | 28.5 samples/s +Step 3540/44000 | Loss: 3.4817 | LR: 4.95e-04 | 697.1ms/step | 28.7 samples/s +Step 3550/44000 | Loss: 3.4639 | LR: 4.95e-04 | 698.2ms/step | 28.6 samples/s +Step 3560/44000 | Loss: 3.4352 | LR: 4.95e-04 | 700.5ms/step | 28.6 samples/s +Step 3570/44000 | Loss: 3.4685 | LR: 4.95e-04 | 697.4ms/step | 28.7 samples/s +Step 3580/44000 | Loss: 3.4505 | LR: 4.95e-04 | 695.9ms/step | 28.7 samples/s +Step 3590/44000 | Loss: 3.4795 | LR: 4.95e-04 | 698.9ms/step | 28.6 samples/s +Step 3600/44000 | Loss: 3.4302 | LR: 4.95e-04 | 702.4ms/step | 28.5 samples/s +Step 3610/44000 | Loss: 3.4260 | LR: 4.95e-04 | 695.3ms/step | 28.8 samples/s +Step 3620/44000 | Loss: 3.4214 | LR: 4.95e-04 | 694.9ms/step | 28.8 samples/s +Step 3630/44000 | Loss: 3.4663 | LR: 4.95e-04 | 692.7ms/step | 28.9 samples/s +Step 3640/44000 | Loss: 3.4404 | LR: 4.95e-04 | 697.1ms/step | 28.7 samples/s +Step 3650/44000 | Loss: 3.4463 | LR: 4.94e-04 | 698.8ms/step | 28.6 samples/s +Step 3660/44000 | Loss: 3.4452 | LR: 4.94e-04 | 699.3ms/step | 28.6 samples/s +Step 3670/44000 | Loss: 3.4359 | LR: 4.94e-04 | 700.5ms/step | 28.6 samples/s +Step 3680/44000 | Loss: 3.4268 | LR: 4.94e-04 | 699.8ms/step | 28.6 samples/s +Step 3690/44000 | Loss: 3.4469 | LR: 4.94e-04 | 700.8ms/step | 28.5 samples/s +Step 3700/44000 | Loss: 3.4430 | LR: 4.94e-04 | 699.5ms/step | 28.6 samples/s +Step 3710/44000 | Loss: 3.4340 | LR: 4.94e-04 | 700.7ms/step | 28.5 samples/s +Step 3720/44000 | Loss: 3.4353 | LR: 4.94e-04 | 699.9ms/step | 28.6 samples/s +Step 3730/44000 | Loss: 3.4774 | LR: 4.94e-04 | 697.6ms/step | 28.7 samples/s +Step 3740/44000 | Loss: 3.4435 | LR: 4.94e-04 | 699.3ms/step | 28.6 samples/s +Step 3750/44000 | Loss: 3.4282 | LR: 4.94e-04 | 694.8ms/step | 28.8 samples/s +Validation at step 3750: Loss = 3.4660, Perplexity = 32.01 +Step 3760/44000 | Loss: 3.4455 | LR: 4.94e-04 | 1269.6ms/step | 15.8 samples/s +Step 3770/44000 | Loss: 3.4306 | LR: 4.94e-04 | 697.3ms/step | 28.7 samples/s +Step 3780/44000 | Loss: 3.4314 | LR: 4.94e-04 | 701.2ms/step | 28.5 samples/s +Step 3790/44000 | Loss: 3.4723 | LR: 4.94e-04 | 698.7ms/step | 28.6 samples/s +Step 3800/44000 | Loss: 3.4087 | LR: 4.94e-04 | 702.6ms/step | 28.5 samples/s +Step 3810/44000 | Loss: 3.4184 | LR: 4.94e-04 | 698.9ms/step | 28.6 samples/s +Step 3820/44000 | Loss: 3.4428 | LR: 4.94e-04 | 700.0ms/step | 28.6 samples/s +Step 3830/44000 | Loss: 3.4499 | LR: 4.94e-04 | 697.0ms/step | 28.7 samples/s +Step 3840/44000 | Loss: 3.4594 | LR: 4.94e-04 | 702.3ms/step | 28.5 samples/s +Step 3850/44000 | Loss: 3.4285 | LR: 4.94e-04 | 702.3ms/step | 28.5 samples/s +Step 3860/44000 | Loss: 3.4485 | LR: 4.94e-04 | 702.7ms/step | 28.5 samples/s +Step 3870/44000 | Loss: 3.4396 | LR: 4.94e-04 | 700.5ms/step | 28.6 samples/s +Step 3880/44000 | Loss: 3.4320 | LR: 4.94e-04 | 698.9ms/step | 28.6 samples/s +Step 3890/44000 | Loss: 3.4291 | LR: 4.94e-04 | 701.3ms/step | 28.5 samples/s +Step 3900/44000 | Loss: 3.4638 | LR: 4.93e-04 | 695.9ms/step | 28.7 samples/s +Step 3910/44000 | Loss: 3.4184 | LR: 4.93e-04 | 697.8ms/step | 28.7 samples/s +Step 3920/44000 | Loss: 3.4237 | LR: 4.93e-04 | 695.7ms/step | 28.7 samples/s +Step 3930/44000 | Loss: 3.4184 | LR: 4.93e-04 | 697.9ms/step | 28.7 samples/s +Step 3940/44000 | Loss: 3.4543 | LR: 4.93e-04 | 699.9ms/step | 28.6 samples/s +Step 3950/44000 | Loss: 3.4195 | LR: 4.93e-04 | 701.2ms/step | 28.5 samples/s +Step 3960/44000 | Loss: 3.4051 | LR: 4.93e-04 | 700.3ms/step | 28.6 samples/s +Step 3970/44000 | Loss: 3.4313 | LR: 4.93e-04 | 700.9ms/step | 28.5 samples/s +Step 3980/44000 | Loss: 3.4180 | LR: 4.93e-04 | 699.2ms/step | 28.6 samples/s +Step 3990/44000 | Loss: 3.3684 | LR: 4.93e-04 | 701.5ms/step | 28.5 samples/s +Step 4000/44000 | Loss: 3.4193 | LR: 4.93e-04 | 700.6ms/step | 28.5 samples/s +Validation at step 4000: Loss = 3.4437, Perplexity = 31.30 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-4000 +Step 4010/44000 | Loss: 3.3825 | LR: 4.93e-04 | 1589.6ms/step | 12.6 samples/s +Step 4020/44000 | Loss: 3.3944 | LR: 4.93e-04 | 697.8ms/step | 28.7 samples/s +Step 4030/44000 | Loss: 3.4724 | LR: 4.93e-04 | 698.9ms/step | 28.6 samples/s +Step 4040/44000 | Loss: 3.4296 | LR: 4.93e-04 | 698.8ms/step | 28.6 samples/s +Step 4050/44000 | Loss: 3.3990 | LR: 4.93e-04 | 700.6ms/step | 28.5 samples/s +Step 4060/44000 | Loss: 3.3908 | LR: 4.93e-04 | 697.1ms/step | 28.7 samples/s +Step 4070/44000 | Loss: 3.3879 | LR: 4.93e-04 | 698.1ms/step | 28.6 samples/s +Step 4080/44000 | Loss: 3.4424 | LR: 4.93e-04 | 701.5ms/step | 28.5 samples/s +Step 4090/44000 | Loss: 3.4050 | LR: 4.93e-04 | 698.8ms/step | 28.6 samples/s +Step 4100/44000 | Loss: 3.3560 | LR: 4.93e-04 | 695.5ms/step | 28.8 samples/s +Step 4110/44000 | Loss: 3.4408 | LR: 4.93e-04 | 695.7ms/step | 28.7 samples/s +Step 4120/44000 | Loss: 3.4368 | LR: 4.93e-04 | 700.7ms/step | 28.5 samples/s +Step 4130/44000 | Loss: 3.3995 | LR: 4.93e-04 | 698.2ms/step | 28.6 samples/s +Step 4140/44000 | Loss: 3.4133 | LR: 4.92e-04 | 699.9ms/step | 28.6 samples/s +Step 4150/44000 | Loss: 3.3936 | LR: 4.92e-04 | 700.9ms/step | 28.5 samples/s +Step 4160/44000 | Loss: 3.3925 | LR: 4.92e-04 | 698.4ms/step | 28.6 samples/s +Step 4170/44000 | Loss: 3.4148 | LR: 4.92e-04 | 696.5ms/step | 28.7 samples/s +Step 4180/44000 | Loss: 3.4629 | LR: 4.92e-04 | 696.0ms/step | 28.7 samples/s +Step 4190/44000 | Loss: 3.4122 | LR: 4.92e-04 | 702.5ms/step | 28.5 samples/s +Step 4200/44000 | Loss: 3.4188 | LR: 4.92e-04 | 698.2ms/step | 28.6 samples/s +Step 4210/44000 | Loss: 3.3977 | LR: 4.92e-04 | 698.5ms/step | 28.6 samples/s +Step 4220/44000 | Loss: 3.3679 | LR: 4.92e-04 | 699.3ms/step | 28.6 samples/s +Step 4230/44000 | Loss: 3.3738 | LR: 4.92e-04 | 697.6ms/step | 28.7 samples/s +Step 4240/44000 | Loss: 3.3943 | LR: 4.92e-04 | 698.1ms/step | 28.6 samples/s +Step 4250/44000 | Loss: 3.3968 | LR: 4.92e-04 | 695.9ms/step | 28.7 samples/s +Validation at step 4250: Loss = 3.4249, Perplexity = 30.72 +Step 4260/44000 | Loss: 3.3440 | LR: 4.92e-04 | 1266.8ms/step | 15.8 samples/s +Step 4270/44000 | Loss: 3.4033 | LR: 4.92e-04 | 699.8ms/step | 28.6 samples/s +Step 4280/44000 | Loss: 3.4073 | LR: 4.92e-04 | 698.6ms/step | 28.6 samples/s +Step 4290/44000 | Loss: 3.3822 | LR: 4.92e-04 | 697.3ms/step | 28.7 samples/s +Step 4300/44000 | Loss: 3.4149 | LR: 4.92e-04 | 696.8ms/step | 28.7 samples/s +Step 4310/44000 | Loss: 3.4031 | LR: 4.92e-04 | 695.0ms/step | 28.8 samples/s +Step 4320/44000 | Loss: 3.4221 | LR: 4.92e-04 | 699.5ms/step | 28.6 samples/s +Step 4330/44000 | Loss: 3.3914 | LR: 4.92e-04 | 701.1ms/step | 28.5 samples/s +Step 4340/44000 | Loss: 3.4033 | LR: 4.92e-04 | 696.9ms/step | 28.7 samples/s +Step 4350/44000 | Loss: 3.4195 | LR: 4.92e-04 | 702.1ms/step | 28.5 samples/s +Step 4360/44000 | Loss: 3.3983 | LR: 4.91e-04 | 696.4ms/step | 28.7 samples/s +Step 4370/44000 | Loss: 3.3845 | LR: 4.91e-04 | 694.1ms/step | 28.8 samples/s +Step 4380/44000 | Loss: 3.3764 | LR: 4.91e-04 | 698.9ms/step | 28.6 samples/s +Step 4390/44000 | Loss: 3.3723 | LR: 4.91e-04 | 697.5ms/step | 28.7 samples/s +Step 4400/44000 | Loss: 3.4037 | LR: 4.91e-04 | 699.0ms/step | 28.6 samples/s +Step 4410/44000 | Loss: 3.4099 | LR: 4.91e-04 | 700.9ms/step | 28.5 samples/s +Step 4420/44000 | Loss: 3.3711 | LR: 4.91e-04 | 700.0ms/step | 28.6 samples/s +Step 4430/44000 | Loss: 3.3838 | LR: 4.91e-04 | 701.0ms/step | 28.5 samples/s +Step 4440/44000 | Loss: 3.3480 | LR: 4.91e-04 | 695.3ms/step | 28.8 samples/s +Step 4450/44000 | Loss: 3.4131 | LR: 4.91e-04 | 696.9ms/step | 28.7 samples/s +Step 4460/44000 | Loss: 3.3573 | LR: 4.91e-04 | 700.3ms/step | 28.6 samples/s +Step 4470/44000 | Loss: 3.3650 | LR: 4.91e-04 | 699.4ms/step | 28.6 samples/s +Step 4480/44000 | Loss: 3.3714 | LR: 4.91e-04 | 701.2ms/step | 28.5 samples/s +Step 4490/44000 | Loss: 3.3697 | LR: 4.91e-04 | 697.6ms/step | 28.7 samples/s +Step 4500/44000 | Loss: 3.3765 | LR: 4.91e-04 | 699.2ms/step | 28.6 samples/s +Validation at step 4500: Loss = 3.4044, Perplexity = 30.10 +Step 4510/44000 | Loss: 3.3726 | LR: 4.91e-04 | 1271.3ms/step | 15.7 samples/s +Step 4520/44000 | Loss: 3.3403 | LR: 4.91e-04 | 700.3ms/step | 28.6 samples/s +Step 4530/44000 | Loss: 3.3663 | LR: 4.91e-04 | 696.9ms/step | 28.7 samples/s +Step 4540/44000 | Loss: 3.3702 | LR: 4.91e-04 | 699.7ms/step | 28.6 samples/s +Step 4550/44000 | Loss: 3.3618 | LR: 4.91e-04 | 696.0ms/step | 28.7 samples/s +Step 4560/44000 | Loss: 3.3489 | LR: 4.90e-04 | 698.7ms/step | 28.6 samples/s +Step 4570/44000 | Loss: 3.3378 | LR: 4.90e-04 | 695.4ms/step | 28.8 samples/s +Step 4580/44000 | Loss: 3.3569 | LR: 4.90e-04 | 698.5ms/step | 28.6 samples/s +Step 4590/44000 | Loss: 3.3461 | LR: 4.90e-04 | 696.7ms/step | 28.7 samples/s +Step 4600/44000 | Loss: 3.3592 | LR: 4.90e-04 | 699.4ms/step | 28.6 samples/s +Step 4610/44000 | Loss: 3.4073 | LR: 4.90e-04 | 694.6ms/step | 28.8 samples/s +Step 4620/44000 | Loss: 3.4050 | LR: 4.90e-04 | 702.0ms/step | 28.5 samples/s +Step 4630/44000 | Loss: 3.3719 | LR: 4.90e-04 | 698.4ms/step | 28.6 samples/s +Step 4640/44000 | Loss: 3.3543 | LR: 4.90e-04 | 698.0ms/step | 28.7 samples/s +Step 4650/44000 | Loss: 3.3390 | LR: 4.90e-04 | 696.2ms/step | 28.7 samples/s +Step 4660/44000 | Loss: 3.3945 | LR: 4.90e-04 | 700.2ms/step | 28.6 samples/s +Step 4670/44000 | Loss: 3.3588 | LR: 4.90e-04 | 698.1ms/step | 28.7 samples/s +Step 4680/44000 | Loss: 3.3455 | LR: 4.90e-04 | 696.4ms/step | 28.7 samples/s +Step 4690/44000 | Loss: 3.3305 | LR: 4.90e-04 | 697.7ms/step | 28.7 samples/s +Step 4700/44000 | Loss: 3.3980 | LR: 4.90e-04 | 697.8ms/step | 28.7 samples/s +Step 4710/44000 | Loss: 3.4169 | LR: 4.90e-04 | 698.1ms/step | 28.6 samples/s +Step 4720/44000 | Loss: 3.3397 | LR: 4.90e-04 | 698.1ms/step | 28.6 samples/s +Step 4730/44000 | Loss: 3.3580 | LR: 4.90e-04 | 700.3ms/step | 28.6 samples/s +Step 4740/44000 | Loss: 3.3259 | LR: 4.90e-04 | 697.3ms/step | 28.7 samples/s +Step 4750/44000 | Loss: 3.3317 | LR: 4.90e-04 | 699.7ms/step | 28.6 samples/s +Validation at step 4750: Loss = 3.3851, Perplexity = 29.52 +Step 4760/44000 | Loss: 3.3451 | LR: 4.89e-04 | 1265.1ms/step | 15.8 samples/s +Step 4770/44000 | Loss: 3.3176 | LR: 4.89e-04 | 698.1ms/step | 28.7 samples/s +Step 4780/44000 | Loss: 3.3517 | LR: 4.89e-04 | 701.1ms/step | 28.5 samples/s +Step 4790/44000 | Loss: 3.3573 | LR: 4.89e-04 | 698.2ms/step | 28.6 samples/s +Step 4800/44000 | Loss: 3.3819 | LR: 4.89e-04 | 698.9ms/step | 28.6 samples/s +Step 4810/44000 | Loss: 3.3477 | LR: 4.89e-04 | 696.8ms/step | 28.7 samples/s +Step 4820/44000 | Loss: 3.3308 | LR: 4.89e-04 | 697.9ms/step | 28.7 samples/s +Step 4830/44000 | Loss: 3.3129 | LR: 4.89e-04 | 698.2ms/step | 28.6 samples/s +Step 4840/44000 | Loss: 3.3746 | LR: 4.89e-04 | 699.7ms/step | 28.6 samples/s +Step 4850/44000 | Loss: 3.3329 | LR: 4.89e-04 | 702.3ms/step | 28.5 samples/s +Step 4860/44000 | Loss: 3.3853 | LR: 4.89e-04 | 697.3ms/step | 28.7 samples/s +Step 4870/44000 | Loss: 3.3547 | LR: 4.89e-04 | 697.5ms/step | 28.7 samples/s +Step 4880/44000 | Loss: 3.3211 | LR: 4.89e-04 | 700.1ms/step | 28.6 samples/s +Step 4890/44000 | Loss: 3.3451 | LR: 4.89e-04 | 698.7ms/step | 28.6 samples/s +Step 4900/44000 | Loss: 3.3867 | LR: 4.89e-04 | 699.1ms/step | 28.6 samples/s +Step 4910/44000 | Loss: 3.3620 | LR: 4.89e-04 | 700.0ms/step | 28.6 samples/s +Step 4920/44000 | Loss: 3.3996 | LR: 4.89e-04 | 696.7ms/step | 28.7 samples/s +Step 4930/44000 | Loss: 3.3611 | LR: 4.89e-04 | 695.5ms/step | 28.8 samples/s +Step 4940/44000 | Loss: 3.3158 | LR: 4.89e-04 | 698.6ms/step | 28.6 samples/s +Step 4950/44000 | Loss: 3.3390 | LR: 4.88e-04 | 696.5ms/step | 28.7 samples/s +Step 4960/44000 | Loss: 3.3531 | LR: 4.88e-04 | 695.9ms/step | 28.7 samples/s +Step 4970/44000 | Loss: 3.3354 | LR: 4.88e-04 | 701.4ms/step | 28.5 samples/s +Step 4980/44000 | Loss: 3.3662 | LR: 4.88e-04 | 700.0ms/step | 28.6 samples/s +Step 4990/44000 | Loss: 3.3386 | LR: 4.88e-04 | 698.6ms/step | 28.6 samples/s +Step 5000/44000 | Loss: 3.3257 | LR: 4.88e-04 | 696.6ms/step | 28.7 samples/s +Validation at step 5000: Loss = 3.3718, Perplexity = 29.13 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-5000 +Step 5010/44000 | Loss: 3.3696 | LR: 4.88e-04 | 1621.4ms/step | 12.3 samples/s +Step 5020/44000 | Loss: 3.3552 | LR: 4.88e-04 | 698.8ms/step | 28.6 samples/s +Step 5030/44000 | Loss: 3.3613 | LR: 4.88e-04 | 696.3ms/step | 28.7 samples/s +Step 5040/44000 | Loss: 3.3669 | LR: 4.88e-04 | 698.5ms/step | 28.6 samples/s +Step 5050/44000 | Loss: 3.3252 | LR: 4.88e-04 | 697.8ms/step | 28.7 samples/s +Step 5060/44000 | Loss: 3.2691 | LR: 4.88e-04 | 700.3ms/step | 28.6 samples/s +Step 5070/44000 | Loss: 3.3350 | LR: 4.88e-04 | 700.1ms/step | 28.6 samples/s +Step 5080/44000 | Loss: 3.3515 | LR: 4.88e-04 | 697.8ms/step | 28.7 samples/s +Step 5090/44000 | Loss: 3.3313 | LR: 4.88e-04 | 698.4ms/step | 28.6 samples/s +Step 5100/44000 | Loss: 3.3821 | LR: 4.88e-04 | 698.0ms/step | 28.7 samples/s +Step 5110/44000 | Loss: 3.3186 | LR: 4.88e-04 | 703.7ms/step | 28.4 samples/s +Step 5120/44000 | Loss: 3.3634 | LR: 4.88e-04 | 697.4ms/step | 28.7 samples/s +Step 5130/44000 | Loss: 3.2907 | LR: 4.87e-04 | 695.2ms/step | 28.8 samples/s +Step 5140/44000 | Loss: 3.3629 | LR: 4.87e-04 | 699.7ms/step | 28.6 samples/s +Step 5150/44000 | Loss: 3.3548 | LR: 4.87e-04 | 700.1ms/step | 28.6 samples/s +Step 5160/44000 | Loss: 3.3106 | LR: 4.87e-04 | 698.7ms/step | 28.6 samples/s +Step 5170/44000 | Loss: 3.3111 | LR: 4.87e-04 | 700.8ms/step | 28.5 samples/s +Step 5180/44000 | Loss: 3.3219 | LR: 4.87e-04 | 696.3ms/step | 28.7 samples/s +Step 5190/44000 | Loss: 3.3273 | LR: 4.87e-04 | 699.8ms/step | 28.6 samples/s +Step 5200/44000 | Loss: 3.2996 | LR: 4.87e-04 | 698.8ms/step | 28.6 samples/s +Step 5210/44000 | Loss: 3.3161 | LR: 4.87e-04 | 700.7ms/step | 28.5 samples/s +Step 5220/44000 | Loss: 3.3453 | LR: 4.87e-04 | 701.0ms/step | 28.5 samples/s +Step 5230/44000 | Loss: 3.3390 | LR: 4.87e-04 | 697.1ms/step | 28.7 samples/s +Step 5240/44000 | Loss: 3.2496 | LR: 4.87e-04 | 698.4ms/step | 28.6 samples/s +Step 5250/44000 | Loss: 3.3160 | LR: 4.87e-04 | 702.8ms/step | 28.5 samples/s +Validation at step 5250: Loss = 3.3568, Perplexity = 28.70 +Step 5260/44000 | Loss: 3.3615 | LR: 4.87e-04 | 1268.8ms/step | 15.8 samples/s +Step 5270/44000 | Loss: 3.3602 | LR: 4.87e-04 | 699.7ms/step | 28.6 samples/s +Step 5280/44000 | Loss: 3.3399 | LR: 4.87e-04 | 696.4ms/step | 28.7 samples/s +Step 5290/44000 | Loss: 3.3275 | LR: 4.87e-04 | 699.6ms/step | 28.6 samples/s +Step 5300/44000 | Loss: 3.2939 | LR: 4.86e-04 | 696.8ms/step | 28.7 samples/s +Step 5310/44000 | Loss: 3.3080 | LR: 4.86e-04 | 698.6ms/step | 28.6 samples/s +Step 5320/44000 | Loss: 3.3692 | LR: 4.86e-04 | 698.0ms/step | 28.7 samples/s +Step 5330/44000 | Loss: 3.3267 | LR: 4.86e-04 | 699.6ms/step | 28.6 samples/s +Step 5340/44000 | Loss: 3.3575 | LR: 4.86e-04 | 698.1ms/step | 28.7 samples/s +Step 5350/44000 | Loss: 3.3043 | LR: 4.86e-04 | 699.1ms/step | 28.6 samples/s +Step 5360/44000 | Loss: 3.3217 | LR: 4.86e-04 | 695.9ms/step | 28.7 samples/s +Step 5370/44000 | Loss: 3.2801 | LR: 4.86e-04 | 701.1ms/step | 28.5 samples/s +Step 5380/44000 | Loss: 3.3207 | LR: 4.86e-04 | 698.4ms/step | 28.6 samples/s +Step 5390/44000 | Loss: 3.3229 | LR: 4.86e-04 | 697.2ms/step | 28.7 samples/s +Step 5400/44000 | Loss: 3.3403 | LR: 4.86e-04 | 697.6ms/step | 28.7 samples/s +Step 5410/44000 | Loss: 3.2877 | LR: 4.86e-04 | 697.5ms/step | 28.7 samples/s +Step 5420/44000 | Loss: 3.3115 | LR: 4.86e-04 | 696.5ms/step | 28.7 samples/s +Step 5430/44000 | Loss: 3.3043 | LR: 4.86e-04 | 698.6ms/step | 28.6 samples/s +Step 5440/44000 | Loss: 3.2987 | LR: 4.86e-04 | 696.9ms/step | 28.7 samples/s +Step 5450/44000 | Loss: 3.3367 | LR: 4.86e-04 | 699.9ms/step | 28.6 samples/s +Step 5460/44000 | Loss: 3.3195 | LR: 4.86e-04 | 697.3ms/step | 28.7 samples/s +Step 5470/44000 | Loss: 3.2969 | LR: 4.85e-04 | 701.0ms/step | 28.5 samples/s +Step 5480/44000 | Loss: 3.2872 | LR: 4.85e-04 | 700.3ms/step | 28.6 samples/s +Step 5490/44000 | Loss: 3.3385 | LR: 4.85e-04 | 697.0ms/step | 28.7 samples/s +Step 5500/44000 | Loss: 3.3608 | LR: 4.85e-04 | 698.9ms/step | 28.6 samples/s +Validation at step 5500: Loss = 3.3457, Perplexity = 28.38 +Step 5510/44000 | Loss: 3.3496 | LR: 4.85e-04 | 1265.5ms/step | 15.8 samples/s +Step 5520/44000 | Loss: 3.3681 | LR: 4.85e-04 | 700.3ms/step | 28.6 samples/s +Step 5530/44000 | Loss: 3.3256 | LR: 4.85e-04 | 698.5ms/step | 28.6 samples/s +Step 5540/44000 | Loss: 3.3606 | LR: 4.85e-04 | 700.1ms/step | 28.6 samples/s +Step 5550/44000 | Loss: 3.2765 | LR: 4.85e-04 | 697.1ms/step | 28.7 samples/s +Step 5560/44000 | Loss: 3.3415 | LR: 4.85e-04 | 698.8ms/step | 28.6 samples/s +Step 5570/44000 | Loss: 3.3416 | LR: 4.85e-04 | 699.4ms/step | 28.6 samples/s +Step 5580/44000 | Loss: 3.3466 | LR: 4.85e-04 | 701.0ms/step | 28.5 samples/s +Step 5590/44000 | Loss: 3.2812 | LR: 4.85e-04 | 697.9ms/step | 28.7 samples/s +Step 5600/44000 | Loss: 3.3073 | LR: 4.85e-04 | 699.0ms/step | 28.6 samples/s +Step 5610/44000 | Loss: 3.3278 | LR: 4.85e-04 | 700.7ms/step | 28.5 samples/s +Step 5620/44000 | Loss: 3.3033 | LR: 4.85e-04 | 700.6ms/step | 28.5 samples/s +Step 5630/44000 | Loss: 3.2741 | LR: 4.84e-04 | 699.4ms/step | 28.6 samples/s +Step 5640/44000 | Loss: 3.2779 | LR: 4.84e-04 | 698.5ms/step | 28.6 samples/s +Step 5650/44000 | Loss: 3.3261 | LR: 4.84e-04 | 701.0ms/step | 28.5 samples/s +Step 5660/44000 | Loss: 3.3428 | LR: 4.84e-04 | 702.7ms/step | 28.5 samples/s +Step 5670/44000 | Loss: 3.3248 | LR: 4.84e-04 | 694.2ms/step | 28.8 samples/s +Step 5680/44000 | Loss: 3.3078 | LR: 4.84e-04 | 699.1ms/step | 28.6 samples/s +Step 5690/44000 | Loss: 3.2989 | LR: 4.84e-04 | 700.7ms/step | 28.5 samples/s +Step 5700/44000 | Loss: 3.2689 | LR: 4.84e-04 | 699.3ms/step | 28.6 samples/s +Step 5710/44000 | Loss: 3.2965 | LR: 4.84e-04 | 701.4ms/step | 28.5 samples/s +Step 5720/44000 | Loss: 3.3255 | LR: 4.84e-04 | 699.0ms/step | 28.6 samples/s +Step 5730/44000 | Loss: 3.3334 | LR: 4.84e-04 | 700.5ms/step | 28.6 samples/s +Step 5740/44000 | Loss: 3.2528 | LR: 4.84e-04 | 699.5ms/step | 28.6 samples/s +Step 5750/44000 | Loss: 3.2907 | LR: 4.84e-04 | 700.6ms/step | 28.5 samples/s +Validation at step 5750: Loss = 3.3301, Perplexity = 27.94 +Step 5760/44000 | Loss: 3.3296 | LR: 4.84e-04 | 1266.6ms/step | 15.8 samples/s +Step 5770/44000 | Loss: 3.3508 | LR: 4.84e-04 | 698.3ms/step | 28.6 samples/s +Step 5780/44000 | Loss: 3.2708 | LR: 4.83e-04 | 700.4ms/step | 28.6 samples/s +Step 5790/44000 | Loss: 3.2543 | LR: 4.83e-04 | 696.1ms/step | 28.7 samples/s +Step 5800/44000 | Loss: 3.3093 | LR: 4.83e-04 | 693.9ms/step | 28.8 samples/s +Step 5810/44000 | Loss: 3.3307 | LR: 4.83e-04 | 696.2ms/step | 28.7 samples/s +Step 5820/44000 | Loss: 3.2710 | LR: 4.83e-04 | 695.9ms/step | 28.7 samples/s +Step 5830/44000 | Loss: 3.2924 | LR: 4.83e-04 | 695.8ms/step | 28.7 samples/s +Step 5840/44000 | Loss: 3.3028 | LR: 4.83e-04 | 700.7ms/step | 28.5 samples/s +Step 5850/44000 | Loss: 3.3543 | LR: 4.83e-04 | 698.8ms/step | 28.6 samples/s +Step 5860/44000 | Loss: 3.2834 | LR: 4.83e-04 | 694.8ms/step | 28.8 samples/s +Step 5870/44000 | Loss: 3.2520 | LR: 4.83e-04 | 700.0ms/step | 28.6 samples/s +Step 5880/44000 | Loss: 3.2992 | LR: 4.83e-04 | 697.1ms/step | 28.7 samples/s +Step 5890/44000 | Loss: 3.3594 | LR: 4.83e-04 | 697.2ms/step | 28.7 samples/s +Step 5900/44000 | Loss: 3.2609 | LR: 4.83e-04 | 697.9ms/step | 28.7 samples/s +Step 5910/44000 | Loss: 3.2752 | LR: 4.83e-04 | 697.2ms/step | 28.7 samples/s +Step 5920/44000 | Loss: 3.3132 | LR: 4.83e-04 | 698.0ms/step | 28.7 samples/s +Step 5930/44000 | Loss: 3.3214 | LR: 4.83e-04 | 697.8ms/step | 28.7 samples/s +Step 5940/44000 | Loss: 3.2690 | LR: 4.82e-04 | 699.6ms/step | 28.6 samples/s +Step 5950/44000 | Loss: 3.3047 | LR: 4.82e-04 | 698.6ms/step | 28.6 samples/s +Step 5960/44000 | Loss: 3.3025 | LR: 4.82e-04 | 695.9ms/step | 28.7 samples/s +Step 5970/44000 | Loss: 3.2563 | LR: 4.82e-04 | 696.6ms/step | 28.7 samples/s +Step 5980/44000 | Loss: 3.2985 | LR: 4.82e-04 | 698.9ms/step | 28.6 samples/s +Step 5990/44000 | Loss: 3.2669 | LR: 4.82e-04 | 699.5ms/step | 28.6 samples/s +Step 6000/44000 | Loss: 3.2984 | LR: 4.82e-04 | 700.2ms/step | 28.6 samples/s +Validation at step 6000: Loss = 3.3153, Perplexity = 27.53 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-6000 +Step 6010/44000 | Loss: 3.2699 | LR: 4.82e-04 | 1579.5ms/step | 12.7 samples/s +Step 6020/44000 | Loss: 3.2649 | LR: 4.82e-04 | 699.9ms/step | 28.6 samples/s +Step 6030/44000 | Loss: 3.2608 | LR: 4.82e-04 | 698.7ms/step | 28.6 samples/s +Step 6040/44000 | Loss: 3.2805 | LR: 4.82e-04 | 699.3ms/step | 28.6 samples/s +Step 6050/44000 | Loss: 3.2801 | LR: 4.82e-04 | 697.2ms/step | 28.7 samples/s +Step 6060/44000 | Loss: 3.2498 | LR: 4.82e-04 | 700.1ms/step | 28.6 samples/s +Step 6070/44000 | Loss: 3.2942 | LR: 4.82e-04 | 700.1ms/step | 28.6 samples/s +Step 6080/44000 | Loss: 3.2711 | LR: 4.81e-04 | 697.1ms/step | 28.7 samples/s +Step 6090/44000 | Loss: 3.2772 | LR: 4.81e-04 | 703.3ms/step | 28.4 samples/s +Step 6100/44000 | Loss: 3.2978 | LR: 4.81e-04 | 698.4ms/step | 28.6 samples/s +Step 6110/44000 | Loss: 3.3122 | LR: 4.81e-04 | 700.5ms/step | 28.6 samples/s +Step 6120/44000 | Loss: 3.2898 | LR: 4.81e-04 | 698.5ms/step | 28.6 samples/s +Step 6130/44000 | Loss: 3.2895 | LR: 4.81e-04 | 698.6ms/step | 28.6 samples/s +Step 6140/44000 | Loss: 3.2781 | LR: 4.81e-04 | 700.3ms/step | 28.6 samples/s +Step 6150/44000 | Loss: 3.3103 | LR: 4.81e-04 | 696.5ms/step | 28.7 samples/s +Step 6160/44000 | Loss: 3.2695 | LR: 4.81e-04 | 699.7ms/step | 28.6 samples/s +Step 6170/44000 | Loss: 3.2462 | LR: 4.81e-04 | 703.4ms/step | 28.4 samples/s +Step 6180/44000 | Loss: 3.2477 | LR: 4.81e-04 | 697.9ms/step | 28.7 samples/s +Step 6190/44000 | Loss: 3.2794 | LR: 4.81e-04 | 700.2ms/step | 28.6 samples/s +Step 6200/44000 | Loss: 3.2727 | LR: 4.81e-04 | 699.0ms/step | 28.6 samples/s +Step 6210/44000 | Loss: 3.3156 | LR: 4.81e-04 | 699.7ms/step | 28.6 samples/s +Step 6220/44000 | Loss: 3.3134 | LR: 4.81e-04 | 700.1ms/step | 28.6 samples/s +Step 6230/44000 | Loss: 3.2540 | LR: 4.80e-04 | 699.6ms/step | 28.6 samples/s +Step 6240/44000 | Loss: 3.2462 | LR: 4.80e-04 | 697.7ms/step | 28.7 samples/s +Step 6250/44000 | Loss: 3.2482 | LR: 4.80e-04 | 698.6ms/step | 28.6 samples/s +Validation at step 6250: Loss = 3.3076, Perplexity = 27.32 +Step 6260/44000 | Loss: 3.2365 | LR: 4.80e-04 | 1269.6ms/step | 15.8 samples/s +Step 6270/44000 | Loss: 3.3045 | LR: 4.80e-04 | 700.0ms/step | 28.6 samples/s +Step 6280/44000 | Loss: 3.3047 | LR: 4.80e-04 | 698.8ms/step | 28.6 samples/s +Step 6290/44000 | Loss: 3.2794 | LR: 4.80e-04 | 699.8ms/step | 28.6 samples/s +Step 6300/44000 | Loss: 3.2774 | LR: 4.80e-04 | 701.5ms/step | 28.5 samples/s +Step 6310/44000 | Loss: 3.2184 | LR: 4.80e-04 | 698.8ms/step | 28.6 samples/s +Step 6320/44000 | Loss: 3.2889 | LR: 4.80e-04 | 698.2ms/step | 28.6 samples/s +Step 6330/44000 | Loss: 3.2600 | LR: 4.80e-04 | 697.8ms/step | 28.7 samples/s +Step 6340/44000 | Loss: 3.2707 | LR: 4.80e-04 | 696.0ms/step | 28.7 samples/s +Step 6350/44000 | Loss: 3.2869 | LR: 4.80e-04 | 698.1ms/step | 28.7 samples/s +Step 6360/44000 | Loss: 3.2865 | LR: 4.80e-04 | 700.4ms/step | 28.6 samples/s +Step 6370/44000 | Loss: 3.2643 | LR: 4.79e-04 | 701.0ms/step | 28.5 samples/s +Step 6380/44000 | Loss: 3.3054 | LR: 4.79e-04 | 699.3ms/step | 28.6 samples/s +Step 6390/44000 | Loss: 3.2792 | LR: 4.79e-04 | 700.3ms/step | 28.6 samples/s +Step 6400/44000 | Loss: 3.2663 | LR: 4.79e-04 | 700.6ms/step | 28.5 samples/s +Step 6410/44000 | Loss: 3.2806 | LR: 4.79e-04 | 700.3ms/step | 28.6 samples/s +Step 6420/44000 | Loss: 3.2882 | LR: 4.79e-04 | 699.0ms/step | 28.6 samples/s +Step 6430/44000 | Loss: 3.3163 | LR: 4.79e-04 | 697.8ms/step | 28.7 samples/s +Step 6440/44000 | Loss: 3.2719 | LR: 4.79e-04 | 701.8ms/step | 28.5 samples/s +Step 6450/44000 | Loss: 3.2302 | LR: 4.79e-04 | 698.2ms/step | 28.6 samples/s +Step 6460/44000 | Loss: 3.2847 | LR: 4.79e-04 | 704.0ms/step | 28.4 samples/s +Step 6470/44000 | Loss: 3.3059 | LR: 4.79e-04 | 699.7ms/step | 28.6 samples/s +Step 6480/44000 | Loss: 3.2476 | LR: 4.79e-04 | 700.5ms/step | 28.5 samples/s +Step 6490/44000 | Loss: 3.3179 | LR: 4.79e-04 | 698.5ms/step | 28.6 samples/s +Step 6500/44000 | Loss: 3.2519 | LR: 4.79e-04 | 699.5ms/step | 28.6 samples/s +Validation at step 6500: Loss = 3.2926, Perplexity = 26.91 +Step 6510/44000 | Loss: 3.2530 | LR: 4.78e-04 | 1269.6ms/step | 15.8 samples/s +Step 6520/44000 | Loss: 3.2892 | LR: 4.78e-04 | 700.3ms/step | 28.6 samples/s +Step 6530/44000 | Loss: 3.3066 | LR: 4.78e-04 | 699.8ms/step | 28.6 samples/s +Step 6540/44000 | Loss: 3.2842 | LR: 4.78e-04 | 696.0ms/step | 28.7 samples/s +Step 6550/44000 | Loss: 3.2776 | LR: 4.78e-04 | 696.6ms/step | 28.7 samples/s +Step 6560/44000 | Loss: 3.2971 | LR: 4.78e-04 | 698.5ms/step | 28.6 samples/s +Step 6570/44000 | Loss: 3.2744 | LR: 4.78e-04 | 697.6ms/step | 28.7 samples/s +Step 6580/44000 | Loss: 3.2886 | LR: 4.78e-04 | 696.2ms/step | 28.7 samples/s +Step 6590/44000 | Loss: 3.2857 | LR: 4.78e-04 | 693.2ms/step | 28.9 samples/s +Step 6600/44000 | Loss: 3.2519 | LR: 4.78e-04 | 697.9ms/step | 28.7 samples/s +Step 6610/44000 | Loss: 3.2706 | LR: 4.78e-04 | 700.5ms/step | 28.6 samples/s +Step 6620/44000 | Loss: 3.2857 | LR: 4.78e-04 | 697.0ms/step | 28.7 samples/s +Step 6630/44000 | Loss: 3.2451 | LR: 4.78e-04 | 698.1ms/step | 28.6 samples/s +Step 6640/44000 | Loss: 3.2677 | LR: 4.77e-04 | 699.5ms/step | 28.6 samples/s +Step 6650/44000 | Loss: 3.2779 | LR: 4.77e-04 | 698.9ms/step | 28.6 samples/s +Step 6660/44000 | Loss: 3.2381 | LR: 4.77e-04 | 697.4ms/step | 28.7 samples/s +Step 6670/44000 | Loss: 3.2777 | LR: 4.77e-04 | 694.8ms/step | 28.8 samples/s +Step 6680/44000 | Loss: 3.2672 | LR: 4.77e-04 | 697.5ms/step | 28.7 samples/s +Step 6690/44000 | Loss: 3.2518 | LR: 4.77e-04 | 692.9ms/step | 28.9 samples/s +Step 6700/44000 | Loss: 3.2504 | LR: 4.77e-04 | 697.2ms/step | 28.7 samples/s +Step 6710/44000 | Loss: 3.2595 | LR: 4.77e-04 | 698.2ms/step | 28.6 samples/s +Step 6720/44000 | Loss: 3.2180 | LR: 4.77e-04 | 697.6ms/step | 28.7 samples/s +Step 6730/44000 | Loss: 3.2638 | LR: 4.77e-04 | 697.3ms/step | 28.7 samples/s +Step 6740/44000 | Loss: 3.3051 | LR: 4.77e-04 | 697.1ms/step | 28.7 samples/s +Step 6750/44000 | Loss: 3.2615 | LR: 4.77e-04 | 700.5ms/step | 28.6 samples/s +Validation at step 6750: Loss = 3.2854, Perplexity = 26.72 +Step 6760/44000 | Loss: 3.3406 | LR: 4.77e-04 | 1267.2ms/step | 15.8 samples/s +Step 6770/44000 | Loss: 3.2714 | LR: 4.76e-04 | 701.5ms/step | 28.5 samples/s +Step 6780/44000 | Loss: 3.2632 | LR: 4.76e-04 | 699.9ms/step | 28.6 samples/s +Step 6790/44000 | Loss: 3.2331 | LR: 4.76e-04 | 700.8ms/step | 28.5 samples/s +Step 6800/44000 | Loss: 3.2384 | LR: 4.76e-04 | 696.2ms/step | 28.7 samples/s +Step 6810/44000 | Loss: 3.2576 | LR: 4.76e-04 | 700.0ms/step | 28.6 samples/s +Step 6820/44000 | Loss: 3.2508 | LR: 4.76e-04 | 700.6ms/step | 28.5 samples/s +Step 6830/44000 | Loss: 3.2322 | LR: 4.76e-04 | 697.3ms/step | 28.7 samples/s +Step 6840/44000 | Loss: 3.2367 | LR: 4.76e-04 | 699.3ms/step | 28.6 samples/s +Step 6850/44000 | Loss: 3.2647 | LR: 4.76e-04 | 698.1ms/step | 28.6 samples/s +Step 6860/44000 | Loss: 3.2355 | LR: 4.76e-04 | 698.2ms/step | 28.6 samples/s +Step 6870/44000 | Loss: 3.2468 | LR: 4.76e-04 | 698.3ms/step | 28.6 samples/s +Step 6880/44000 | Loss: 3.2324 | LR: 4.76e-04 | 697.9ms/step | 28.7 samples/s +Step 6890/44000 | Loss: 3.2632 | LR: 4.76e-04 | 699.9ms/step | 28.6 samples/s +Step 6900/44000 | Loss: 3.2842 | LR: 4.75e-04 | 696.1ms/step | 28.7 samples/s +Step 6910/44000 | Loss: 3.2148 | LR: 4.75e-04 | 698.2ms/step | 28.6 samples/s +Step 6920/44000 | Loss: 3.2655 | LR: 4.75e-04 | 700.1ms/step | 28.6 samples/s +Step 6930/44000 | Loss: 3.2515 | LR: 4.75e-04 | 695.5ms/step | 28.8 samples/s +Step 6940/44000 | Loss: 3.2637 | LR: 4.75e-04 | 699.8ms/step | 28.6 samples/s +Step 6950/44000 | Loss: 3.2425 | LR: 4.75e-04 | 698.3ms/step | 28.6 samples/s +Step 6960/44000 | Loss: 3.2662 | LR: 4.75e-04 | 697.9ms/step | 28.7 samples/s +Step 6970/44000 | Loss: 3.2432 | LR: 4.75e-04 | 698.3ms/step | 28.6 samples/s +Step 6980/44000 | Loss: 3.2490 | LR: 4.75e-04 | 700.5ms/step | 28.6 samples/s +Step 6990/44000 | Loss: 3.2576 | LR: 4.75e-04 | 699.4ms/step | 28.6 samples/s +Step 7000/44000 | Loss: 3.2092 | LR: 4.75e-04 | 696.6ms/step | 28.7 samples/s +Validation at step 7000: Loss = 3.2770, Perplexity = 26.50 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-7000 +Step 7010/44000 | Loss: 3.2482 | LR: 4.75e-04 | 1585.6ms/step | 12.6 samples/s +Step 7020/44000 | Loss: 3.2123 | LR: 4.75e-04 | 699.9ms/step | 28.6 samples/s +Step 7030/44000 | Loss: 3.2798 | LR: 4.74e-04 | 698.0ms/step | 28.7 samples/s +Step 7040/44000 | Loss: 3.2800 | LR: 4.74e-04 | 699.6ms/step | 28.6 samples/s +Step 7050/44000 | Loss: 3.2502 | LR: 4.74e-04 | 699.9ms/step | 28.6 samples/s +Step 7060/44000 | Loss: 3.2112 | LR: 4.74e-04 | 698.6ms/step | 28.6 samples/s +Step 7070/44000 | Loss: 3.2151 | LR: 4.74e-04 | 691.8ms/step | 28.9 samples/s +Step 7080/44000 | Loss: 3.2332 | LR: 4.74e-04 | 697.3ms/step | 28.7 samples/s +Step 7090/44000 | Loss: 3.2003 | LR: 4.74e-04 | 698.1ms/step | 28.6 samples/s +Step 7100/44000 | Loss: 3.2498 | LR: 4.74e-04 | 696.8ms/step | 28.7 samples/s +Step 7110/44000 | Loss: 3.2173 | LR: 4.74e-04 | 697.6ms/step | 28.7 samples/s +Step 7120/44000 | Loss: 3.2367 | LR: 4.74e-04 | 695.0ms/step | 28.8 samples/s +Step 7130/44000 | Loss: 3.2476 | LR: 4.74e-04 | 698.8ms/step | 28.6 samples/s +Step 7140/44000 | Loss: 3.2150 | LR: 4.74e-04 | 698.5ms/step | 28.6 samples/s +Step 7150/44000 | Loss: 3.2347 | LR: 4.73e-04 | 695.6ms/step | 28.8 samples/s +Step 7160/44000 | Loss: 3.1919 | LR: 4.73e-04 | 697.8ms/step | 28.7 samples/s +Step 7170/44000 | Loss: 3.2433 | LR: 4.73e-04 | 698.6ms/step | 28.6 samples/s +Step 7180/44000 | Loss: 3.2585 | LR: 4.73e-04 | 697.9ms/step | 28.7 samples/s +Step 7190/44000 | Loss: 3.2839 | LR: 4.73e-04 | 696.8ms/step | 28.7 samples/s +Step 7200/44000 | Loss: 3.2313 | LR: 4.73e-04 | 701.1ms/step | 28.5 samples/s +Step 7210/44000 | Loss: 3.2554 | LR: 4.73e-04 | 694.8ms/step | 28.8 samples/s +Step 7220/44000 | Loss: 3.2138 | LR: 4.73e-04 | 697.0ms/step | 28.7 samples/s +Step 7230/44000 | Loss: 3.2661 | LR: 4.73e-04 | 698.7ms/step | 28.6 samples/s +Step 7240/44000 | Loss: 3.1911 | LR: 4.73e-04 | 700.0ms/step | 28.6 samples/s +Step 7250/44000 | Loss: 3.2734 | LR: 4.73e-04 | 698.4ms/step | 28.6 samples/s +Validation at step 7250: Loss = 3.2674, Perplexity = 26.24 +Step 7260/44000 | Loss: 3.2485 | LR: 4.73e-04 | 1267.3ms/step | 15.8 samples/s +Step 7270/44000 | Loss: 3.2179 | LR: 4.72e-04 | 700.0ms/step | 28.6 samples/s +Step 7280/44000 | Loss: 3.2047 | LR: 4.72e-04 | 697.3ms/step | 28.7 samples/s +Step 7290/44000 | Loss: 3.2398 | LR: 4.72e-04 | 701.7ms/step | 28.5 samples/s +Step 7300/44000 | Loss: 3.2212 | LR: 4.72e-04 | 696.6ms/step | 28.7 samples/s +Step 7310/44000 | Loss: 3.1984 | LR: 4.72e-04 | 699.2ms/step | 28.6 samples/s +Step 7320/44000 | Loss: 3.2484 | LR: 4.72e-04 | 698.5ms/step | 28.6 samples/s +Step 7330/44000 | Loss: 3.2214 | LR: 4.72e-04 | 699.2ms/step | 28.6 samples/s +Step 7340/44000 | Loss: 3.2226 | LR: 4.72e-04 | 698.7ms/step | 28.6 samples/s +Step 7350/44000 | Loss: 3.2193 | LR: 4.72e-04 | 697.9ms/step | 28.7 samples/s +Step 7360/44000 | Loss: 3.2097 | LR: 4.72e-04 | 695.5ms/step | 28.8 samples/s +Step 7370/44000 | Loss: 3.2197 | LR: 4.72e-04 | 699.5ms/step | 28.6 samples/s +Step 7380/44000 | Loss: 3.2028 | LR: 4.72e-04 | 698.3ms/step | 28.6 samples/s +Step 7390/44000 | Loss: 3.2159 | LR: 4.71e-04 | 695.7ms/step | 28.7 samples/s +Step 7400/44000 | Loss: 3.2033 | LR: 4.71e-04 | 697.5ms/step | 28.7 samples/s +Step 7410/44000 | Loss: 3.2163 | LR: 4.71e-04 | 696.3ms/step | 28.7 samples/s +Step 7420/44000 | Loss: 3.2387 | LR: 4.71e-04 | 697.0ms/step | 28.7 samples/s +Step 7430/44000 | Loss: 3.2416 | LR: 4.71e-04 | 695.9ms/step | 28.7 samples/s +Step 7440/44000 | Loss: 3.1715 | LR: 4.71e-04 | 697.7ms/step | 28.7 samples/s +Step 7450/44000 | Loss: 3.2369 | LR: 4.71e-04 | 697.0ms/step | 28.7 samples/s +Step 7460/44000 | Loss: 3.2194 | LR: 4.71e-04 | 696.9ms/step | 28.7 samples/s +Step 7470/44000 | Loss: 3.2493 | LR: 4.71e-04 | 697.1ms/step | 28.7 samples/s +Step 7480/44000 | Loss: 3.1872 | LR: 4.71e-04 | 696.4ms/step | 28.7 samples/s +Step 7490/44000 | Loss: 3.2280 | LR: 4.71e-04 | 698.8ms/step | 28.6 samples/s +Step 7500/44000 | Loss: 3.2328 | LR: 4.71e-04 | 696.9ms/step | 28.7 samples/s +Validation at step 7500: Loss = 3.2598, Perplexity = 26.04 +Step 7510/44000 | Loss: 3.1946 | LR: 4.70e-04 | 1265.2ms/step | 15.8 samples/s +Step 7520/44000 | Loss: 3.2176 | LR: 4.70e-04 | 699.5ms/step | 28.6 samples/s +Step 7530/44000 | Loss: 3.2314 | LR: 4.70e-04 | 695.7ms/step | 28.7 samples/s +Step 7540/44000 | Loss: 3.2040 | LR: 4.70e-04 | 697.1ms/step | 28.7 samples/s +Step 7550/44000 | Loss: 3.2463 | LR: 4.70e-04 | 697.9ms/step | 28.7 samples/s +Step 7560/44000 | Loss: 3.2506 | LR: 4.70e-04 | 696.5ms/step | 28.7 samples/s +Step 7570/44000 | Loss: 3.1832 | LR: 4.70e-04 | 698.3ms/step | 28.6 samples/s +Step 7580/44000 | Loss: 3.2373 | LR: 4.70e-04 | 698.2ms/step | 28.6 samples/s +Step 7590/44000 | Loss: 3.2510 | LR: 4.70e-04 | 701.6ms/step | 28.5 samples/s +Step 7600/44000 | Loss: 3.2236 | LR: 4.70e-04 | 700.7ms/step | 28.5 samples/s +Step 7610/44000 | Loss: 3.2322 | LR: 4.70e-04 | 697.5ms/step | 28.7 samples/s +Step 7620/44000 | Loss: 3.2612 | LR: 4.70e-04 | 698.3ms/step | 28.6 samples/s +Step 7630/44000 | Loss: 3.2501 | LR: 4.69e-04 | 695.3ms/step | 28.8 samples/s +Step 7640/44000 | Loss: 3.2166 | LR: 4.69e-04 | 699.0ms/step | 28.6 samples/s +Step 7650/44000 | Loss: 3.1987 | LR: 4.69e-04 | 699.1ms/step | 28.6 samples/s +Step 7660/44000 | Loss: 3.2505 | LR: 4.69e-04 | 695.2ms/step | 28.8 samples/s +Step 7670/44000 | Loss: 3.2347 | LR: 4.69e-04 | 695.9ms/step | 28.7 samples/s +Step 7680/44000 | Loss: 3.2006 | LR: 4.69e-04 | 695.4ms/step | 28.8 samples/s +Step 7690/44000 | Loss: 3.2369 | LR: 4.69e-04 | 695.1ms/step | 28.8 samples/s +Step 7700/44000 | Loss: 3.2390 | LR: 4.69e-04 | 698.5ms/step | 28.6 samples/s +Step 7710/44000 | Loss: 3.2706 | LR: 4.69e-04 | 696.6ms/step | 28.7 samples/s +Step 7720/44000 | Loss: 3.2483 | LR: 4.69e-04 | 697.8ms/step | 28.7 samples/s +Step 7730/44000 | Loss: 3.2296 | LR: 4.69e-04 | 701.5ms/step | 28.5 samples/s +Step 7740/44000 | Loss: 3.2196 | LR: 4.68e-04 | 696.5ms/step | 28.7 samples/s +Step 7750/44000 | Loss: 3.2314 | LR: 4.68e-04 | 697.8ms/step | 28.7 samples/s +Validation at step 7750: Loss = 3.2510, Perplexity = 25.82 +Step 7760/44000 | Loss: 3.2309 | LR: 4.68e-04 | 1266.7ms/step | 15.8 samples/s +Step 7770/44000 | Loss: 3.2333 | LR: 4.68e-04 | 698.7ms/step | 28.6 samples/s +Step 7780/44000 | Loss: 3.2238 | LR: 4.68e-04 | 697.9ms/step | 28.7 samples/s +Step 7790/44000 | Loss: 3.2747 | LR: 4.68e-04 | 698.3ms/step | 28.6 samples/s +Step 7800/44000 | Loss: 3.2379 | LR: 4.68e-04 | 698.2ms/step | 28.6 samples/s +Step 7810/44000 | Loss: 3.2210 | LR: 4.68e-04 | 698.5ms/step | 28.6 samples/s +Step 7820/44000 | Loss: 3.2504 | LR: 4.68e-04 | 701.0ms/step | 28.5 samples/s +Step 7830/44000 | Loss: 3.2419 | LR: 4.68e-04 | 699.4ms/step | 28.6 samples/s +Step 7840/44000 | Loss: 3.2108 | LR: 4.68e-04 | 699.9ms/step | 28.6 samples/s +Step 7850/44000 | Loss: 3.2041 | LR: 4.67e-04 | 699.7ms/step | 28.6 samples/s +Step 7860/44000 | Loss: 3.2431 | LR: 4.67e-04 | 697.4ms/step | 28.7 samples/s +Step 7870/44000 | Loss: 3.2248 | LR: 4.67e-04 | 698.1ms/step | 28.6 samples/s +Step 7880/44000 | Loss: 3.1659 | LR: 4.67e-04 | 693.7ms/step | 28.8 samples/s +Step 7890/44000 | Loss: 3.2473 | LR: 4.67e-04 | 696.5ms/step | 28.7 samples/s +Step 7900/44000 | Loss: 3.2209 | LR: 4.67e-04 | 697.2ms/step | 28.7 samples/s +Step 7910/44000 | Loss: 3.2229 | LR: 4.67e-04 | 695.6ms/step | 28.8 samples/s +Step 7920/44000 | Loss: 3.2190 | LR: 4.67e-04 | 695.5ms/step | 28.8 samples/s +Step 7930/44000 | Loss: 3.2453 | LR: 4.67e-04 | 693.8ms/step | 28.8 samples/s +Step 7940/44000 | Loss: 3.2392 | LR: 4.67e-04 | 696.6ms/step | 28.7 samples/s +Step 7950/44000 | Loss: 3.1845 | LR: 4.67e-04 | 698.8ms/step | 28.6 samples/s +Step 7960/44000 | Loss: 3.2628 | LR: 4.66e-04 | 698.7ms/step | 28.6 samples/s +Step 7970/44000 | Loss: 3.1859 | LR: 4.66e-04 | 696.1ms/step | 28.7 samples/s +Step 7980/44000 | Loss: 3.1391 | LR: 4.66e-04 | 698.0ms/step | 28.7 samples/s +Step 7990/44000 | Loss: 3.2140 | LR: 4.66e-04 | 698.1ms/step | 28.6 samples/s +Step 8000/44000 | Loss: 3.2466 | LR: 4.66e-04 | 697.4ms/step | 28.7 samples/s +Validation at step 8000: Loss = 3.2415, Perplexity = 25.57 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-8000 +Step 8010/44000 | Loss: 3.2061 | LR: 4.66e-04 | 1588.0ms/step | 12.6 samples/s +Step 8020/44000 | Loss: 3.2106 | LR: 4.66e-04 | 696.5ms/step | 28.7 samples/s +Step 8030/44000 | Loss: 3.1729 | LR: 4.66e-04 | 700.4ms/step | 28.6 samples/s +Step 8040/44000 | Loss: 3.2106 | LR: 4.66e-04 | 699.9ms/step | 28.6 samples/s +Step 8050/44000 | Loss: 3.1850 | LR: 4.66e-04 | 701.4ms/step | 28.5 samples/s +Step 8060/44000 | Loss: 3.2135 | LR: 4.66e-04 | 694.6ms/step | 28.8 samples/s +Step 8070/44000 | Loss: 3.2060 | LR: 4.65e-04 | 699.3ms/step | 28.6 samples/s +Step 8080/44000 | Loss: 3.2299 | LR: 4.65e-04 | 698.1ms/step | 28.6 samples/s +Step 8090/44000 | Loss: 3.2325 | LR: 4.65e-04 | 700.3ms/step | 28.6 samples/s +Step 8100/44000 | Loss: 3.2346 | LR: 4.65e-04 | 699.0ms/step | 28.6 samples/s +Step 8110/44000 | Loss: 3.2127 | LR: 4.65e-04 | 697.0ms/step | 28.7 samples/s +Step 8120/44000 | Loss: 3.2460 | LR: 4.65e-04 | 700.9ms/step | 28.5 samples/s +Step 8130/44000 | Loss: 3.1728 | LR: 4.65e-04 | 709.0ms/step | 28.2 samples/s +Step 8140/44000 | Loss: 3.2158 | LR: 4.65e-04 | 704.6ms/step | 28.4 samples/s +Step 8150/44000 | Loss: 3.2192 | LR: 4.65e-04 | 702.7ms/step | 28.5 samples/s +Step 8160/44000 | Loss: 3.2131 | LR: 4.65e-04 | 702.0ms/step | 28.5 samples/s +Step 8170/44000 | Loss: 3.2194 | LR: 4.65e-04 | 700.1ms/step | 28.6 samples/s +Step 8180/44000 | Loss: 3.1691 | LR: 4.64e-04 | 700.8ms/step | 28.5 samples/s +Step 8190/44000 | Loss: 3.2139 | LR: 4.64e-04 | 700.1ms/step | 28.6 samples/s +Step 8200/44000 | Loss: 3.2306 | LR: 4.64e-04 | 700.2ms/step | 28.6 samples/s +Step 8210/44000 | Loss: 3.2126 | LR: 4.64e-04 | 700.9ms/step | 28.5 samples/s +Step 8220/44000 | Loss: 3.2104 | LR: 4.64e-04 | 699.6ms/step | 28.6 samples/s +Step 8230/44000 | Loss: 3.1891 | LR: 4.64e-04 | 701.6ms/step | 28.5 samples/s +Step 8240/44000 | Loss: 3.1980 | LR: 4.64e-04 | 709.6ms/step | 28.2 samples/s +Step 8250/44000 | Loss: 3.1858 | LR: 4.64e-04 | 704.2ms/step | 28.4 samples/s +Validation at step 8250: Loss = 3.2355, Perplexity = 25.42 +Step 8260/44000 | Loss: 3.2166 | LR: 4.64e-04 | 1271.2ms/step | 15.7 samples/s +Step 8270/44000 | Loss: 3.2102 | LR: 4.64e-04 | 700.2ms/step | 28.6 samples/s +Step 8280/44000 | Loss: 3.2092 | LR: 4.64e-04 | 701.3ms/step | 28.5 samples/s +Step 8290/44000 | Loss: 3.1855 | LR: 4.63e-04 | 697.3ms/step | 28.7 samples/s +Step 8300/44000 | Loss: 3.1946 | LR: 4.63e-04 | 700.1ms/step | 28.6 samples/s +Step 8310/44000 | Loss: 3.2154 | LR: 4.63e-04 | 699.5ms/step | 28.6 samples/s +Step 8320/44000 | Loss: 3.2082 | LR: 4.63e-04 | 701.6ms/step | 28.5 samples/s +Step 8330/44000 | Loss: 3.1894 | LR: 4.63e-04 | 699.0ms/step | 28.6 samples/s +Step 8340/44000 | Loss: 3.2089 | LR: 4.63e-04 | 697.8ms/step | 28.7 samples/s +Step 8350/44000 | Loss: 3.1938 | LR: 4.63e-04 | 701.4ms/step | 28.5 samples/s +Step 8360/44000 | Loss: 3.2274 | LR: 4.63e-04 | 698.1ms/step | 28.7 samples/s +Step 8370/44000 | Loss: 3.1985 | LR: 4.63e-04 | 697.8ms/step | 28.7 samples/s +Step 8380/44000 | Loss: 3.2099 | LR: 4.63e-04 | 698.5ms/step | 28.6 samples/s +Step 8390/44000 | Loss: 3.1844 | LR: 4.62e-04 | 697.7ms/step | 28.7 samples/s +Step 8400/44000 | Loss: 3.1551 | LR: 4.62e-04 | 699.8ms/step | 28.6 samples/s +Step 8410/44000 | Loss: 3.2054 | LR: 4.62e-04 | 698.7ms/step | 28.6 samples/s +Step 8420/44000 | Loss: 3.2123 | LR: 4.62e-04 | 698.7ms/step | 28.6 samples/s +Step 8430/44000 | Loss: 3.1971 | LR: 4.62e-04 | 701.3ms/step | 28.5 samples/s +Step 8440/44000 | Loss: 3.2216 | LR: 4.62e-04 | 699.4ms/step | 28.6 samples/s +Step 8450/44000 | Loss: 3.2259 | LR: 4.62e-04 | 696.5ms/step | 28.7 samples/s +Step 8460/44000 | Loss: 3.1778 | LR: 4.62e-04 | 699.1ms/step | 28.6 samples/s +Step 8470/44000 | Loss: 3.1936 | LR: 4.62e-04 | 696.1ms/step | 28.7 samples/s +Step 8480/44000 | Loss: 3.2530 | LR: 4.62e-04 | 697.0ms/step | 28.7 samples/s +Step 8490/44000 | Loss: 3.1990 | LR: 4.62e-04 | 699.1ms/step | 28.6 samples/s +Step 8500/44000 | Loss: 3.1870 | LR: 4.61e-04 | 697.3ms/step | 28.7 samples/s +Validation at step 8500: Loss = 3.2262, Perplexity = 25.18 +Step 8510/44000 | Loss: 3.2096 | LR: 4.61e-04 | 1270.0ms/step | 15.7 samples/s +Step 8520/44000 | Loss: 3.2318 | LR: 4.61e-04 | 697.8ms/step | 28.7 samples/s +Step 8530/44000 | Loss: 3.2251 | LR: 4.61e-04 | 694.8ms/step | 28.8 samples/s +Step 8540/44000 | Loss: 3.2124 | LR: 4.61e-04 | 694.8ms/step | 28.8 samples/s +Step 8550/44000 | Loss: 3.2268 | LR: 4.61e-04 | 697.8ms/step | 28.7 samples/s +Step 8560/44000 | Loss: 3.1997 | LR: 4.61e-04 | 697.0ms/step | 28.7 samples/s +Step 8570/44000 | Loss: 3.2150 | LR: 4.61e-04 | 698.8ms/step | 28.6 samples/s +Step 8580/44000 | Loss: 3.2317 | LR: 4.61e-04 | 699.6ms/step | 28.6 samples/s +Step 8590/44000 | Loss: 3.2102 | LR: 4.61e-04 | 695.1ms/step | 28.8 samples/s +Step 8600/44000 | Loss: 3.1810 | LR: 4.60e-04 | 699.3ms/step | 28.6 samples/s +Step 8610/44000 | Loss: 3.1812 | LR: 4.60e-04 | 700.2ms/step | 28.6 samples/s +Step 8620/44000 | Loss: 3.2439 | LR: 4.60e-04 | 701.8ms/step | 28.5 samples/s +Step 8630/44000 | Loss: 3.1937 | LR: 4.60e-04 | 700.3ms/step | 28.6 samples/s +Step 8640/44000 | Loss: 3.2033 | LR: 4.60e-04 | 699.5ms/step | 28.6 samples/s +Step 8650/44000 | Loss: 3.1615 | LR: 4.60e-04 | 699.7ms/step | 28.6 samples/s +Step 8660/44000 | Loss: 3.1890 | LR: 4.60e-04 | 700.5ms/step | 28.6 samples/s +Step 8670/44000 | Loss: 3.1785 | LR: 4.60e-04 | 699.8ms/step | 28.6 samples/s +Step 8680/44000 | Loss: 3.2521 | LR: 4.60e-04 | 699.8ms/step | 28.6 samples/s +Step 8690/44000 | Loss: 3.1865 | LR: 4.60e-04 | 698.7ms/step | 28.6 samples/s +Step 8700/44000 | Loss: 3.2213 | LR: 4.59e-04 | 702.4ms/step | 28.5 samples/s +Step 8710/44000 | Loss: 3.1987 | LR: 4.59e-04 | 694.8ms/step | 28.8 samples/s +Step 8720/44000 | Loss: 3.1804 | LR: 4.59e-04 | 698.9ms/step | 28.6 samples/s +Step 8730/44000 | Loss: 3.2013 | LR: 4.59e-04 | 700.9ms/step | 28.5 samples/s +Step 8740/44000 | Loss: 3.1717 | LR: 4.59e-04 | 697.9ms/step | 28.7 samples/s +Step 8750/44000 | Loss: 3.1712 | LR: 4.59e-04 | 700.5ms/step | 28.6 samples/s +Validation at step 8750: Loss = 3.2202, Perplexity = 25.03 +Step 8760/44000 | Loss: 3.1965 | LR: 4.59e-04 | 1268.4ms/step | 15.8 samples/s +Step 8770/44000 | Loss: 3.2194 | LR: 4.59e-04 | 698.1ms/step | 28.6 samples/s +Step 8780/44000 | Loss: 3.2084 | LR: 4.59e-04 | 699.7ms/step | 28.6 samples/s +Step 8790/44000 | Loss: 3.1898 | LR: 4.59e-04 | 699.9ms/step | 28.6 samples/s +Step 8800/44000 | Loss: 3.2548 | LR: 4.58e-04 | 700.3ms/step | 28.6 samples/s +Step 8810/44000 | Loss: 3.1560 | LR: 4.58e-04 | 701.1ms/step | 28.5 samples/s +Step 8820/44000 | Loss: 3.1916 | LR: 4.58e-04 | 700.8ms/step | 28.5 samples/s +Step 8830/44000 | Loss: 3.2022 | LR: 4.58e-04 | 699.4ms/step | 28.6 samples/s +Step 8840/44000 | Loss: 3.2145 | LR: 4.58e-04 | 699.5ms/step | 28.6 samples/s +Step 8850/44000 | Loss: 3.2012 | LR: 4.58e-04 | 702.8ms/step | 28.5 samples/s +Step 8860/44000 | Loss: 3.2015 | LR: 4.58e-04 | 697.7ms/step | 28.7 samples/s +Step 8870/44000 | Loss: 3.2178 | LR: 4.58e-04 | 697.7ms/step | 28.7 samples/s +Step 8880/44000 | Loss: 3.2272 | LR: 4.58e-04 | 700.1ms/step | 28.6 samples/s +Step 8890/44000 | Loss: 3.1815 | LR: 4.58e-04 | 698.9ms/step | 28.6 samples/s +Step 8900/44000 | Loss: 3.2175 | LR: 4.57e-04 | 698.9ms/step | 28.6 samples/s +Step 8910/44000 | Loss: 3.2174 | LR: 4.57e-04 | 696.5ms/step | 28.7 samples/s +Step 8920/44000 | Loss: 3.2211 | LR: 4.57e-04 | 703.6ms/step | 28.4 samples/s +Step 8930/44000 | Loss: 3.1996 | LR: 4.57e-04 | 700.7ms/step | 28.5 samples/s +Step 8940/44000 | Loss: 3.1836 | LR: 4.57e-04 | 698.5ms/step | 28.6 samples/s +Step 8950/44000 | Loss: 3.2311 | LR: 4.57e-04 | 697.9ms/step | 28.7 samples/s +Step 8960/44000 | Loss: 3.1771 | LR: 4.57e-04 | 697.5ms/step | 28.7 samples/s +Step 8970/44000 | Loss: 3.1795 | LR: 4.57e-04 | 696.7ms/step | 28.7 samples/s +Step 8980/44000 | Loss: 3.1902 | LR: 4.57e-04 | 696.7ms/step | 28.7 samples/s +Step 8990/44000 | Loss: 3.1866 | LR: 4.57e-04 | 700.7ms/step | 28.5 samples/s +Step 9000/44000 | Loss: 3.2094 | LR: 4.56e-04 | 700.7ms/step | 28.5 samples/s +Validation at step 9000: Loss = 3.2157, Perplexity = 24.92 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-9000 +Step 9010/44000 | Loss: 3.1972 | LR: 4.56e-04 | 1574.6ms/step | 12.7 samples/s +Step 9020/44000 | Loss: 3.1827 | LR: 4.56e-04 | 699.4ms/step | 28.6 samples/s +Step 9030/44000 | Loss: 3.2199 | LR: 4.56e-04 | 696.0ms/step | 28.7 samples/s +Step 9040/44000 | Loss: 3.2088 | LR: 4.56e-04 | 699.1ms/step | 28.6 samples/s +Step 9050/44000 | Loss: 3.1829 | LR: 4.56e-04 | 698.8ms/step | 28.6 samples/s +Step 9060/44000 | Loss: 3.2262 | LR: 4.56e-04 | 695.2ms/step | 28.8 samples/s +Step 9070/44000 | Loss: 3.2097 | LR: 4.56e-04 | 698.6ms/step | 28.6 samples/s +Step 9080/44000 | Loss: 3.1793 | LR: 4.56e-04 | 700.5ms/step | 28.6 samples/s +Step 9090/44000 | Loss: 3.1573 | LR: 4.56e-04 | 697.2ms/step | 28.7 samples/s +Step 9100/44000 | Loss: 3.1849 | LR: 4.55e-04 | 697.7ms/step | 28.7 samples/s +Step 9110/44000 | Loss: 3.2210 | LR: 4.55e-04 | 699.6ms/step | 28.6 samples/s +Step 9120/44000 | Loss: 3.1450 | LR: 4.55e-04 | 699.7ms/step | 28.6 samples/s +Step 9130/44000 | Loss: 3.2195 | LR: 4.55e-04 | 697.0ms/step | 28.7 samples/s +Step 9140/44000 | Loss: 3.1912 | LR: 4.55e-04 | 697.2ms/step | 28.7 samples/s +Step 9150/44000 | Loss: 3.1680 | LR: 4.55e-04 | 701.2ms/step | 28.5 samples/s +Step 9160/44000 | Loss: 3.2302 | LR: 4.55e-04 | 697.3ms/step | 28.7 samples/s +Step 9170/44000 | Loss: 3.1979 | LR: 4.55e-04 | 694.1ms/step | 28.8 samples/s +Step 9180/44000 | Loss: 3.1885 | LR: 4.55e-04 | 696.6ms/step | 28.7 samples/s +Step 9190/44000 | Loss: 3.1990 | LR: 4.54e-04 | 696.4ms/step | 28.7 samples/s +Step 9200/44000 | Loss: 3.2105 | LR: 4.54e-04 | 701.5ms/step | 28.5 samples/s +Step 9210/44000 | Loss: 3.2055 | LR: 4.54e-04 | 696.3ms/step | 28.7 samples/s +Step 9220/44000 | Loss: 3.1833 | LR: 4.54e-04 | 693.5ms/step | 28.8 samples/s +Step 9230/44000 | Loss: 3.1774 | LR: 4.54e-04 | 697.8ms/step | 28.7 samples/s +Step 9240/44000 | Loss: 3.1479 | LR: 4.54e-04 | 697.5ms/step | 28.7 samples/s +Step 9250/44000 | Loss: 3.1660 | LR: 4.54e-04 | 694.7ms/step | 28.8 samples/s +Validation at step 9250: Loss = 3.2049, Perplexity = 24.65 +Step 9260/44000 | Loss: 3.1626 | LR: 4.54e-04 | 1267.9ms/step | 15.8 samples/s +Step 9270/44000 | Loss: 3.1502 | LR: 4.54e-04 | 697.4ms/step | 28.7 samples/s +Step 9280/44000 | Loss: 3.1562 | LR: 4.54e-04 | 696.7ms/step | 28.7 samples/s +Step 9290/44000 | Loss: 3.1156 | LR: 4.53e-04 | 697.6ms/step | 28.7 samples/s +Step 9300/44000 | Loss: 3.1888 | LR: 4.53e-04 | 693.7ms/step | 28.8 samples/s +Step 9310/44000 | Loss: 3.2082 | LR: 4.53e-04 | 698.6ms/step | 28.6 samples/s +Step 9320/44000 | Loss: 3.1909 | LR: 4.53e-04 | 695.9ms/step | 28.7 samples/s +Step 9330/44000 | Loss: 3.1892 | LR: 4.53e-04 | 695.8ms/step | 28.7 samples/s +Step 9340/44000 | Loss: 3.1724 | LR: 4.53e-04 | 699.6ms/step | 28.6 samples/s +Step 9350/44000 | Loss: 3.1804 | LR: 4.53e-04 | 698.7ms/step | 28.6 samples/s +Step 9360/44000 | Loss: 3.1515 | LR: 4.53e-04 | 697.8ms/step | 28.7 samples/s +Step 9370/44000 | Loss: 3.1788 | LR: 4.53e-04 | 697.4ms/step | 28.7 samples/s +Step 9380/44000 | Loss: 3.1820 | LR: 4.52e-04 | 698.3ms/step | 28.6 samples/s +Step 9390/44000 | Loss: 3.2179 | LR: 4.52e-04 | 698.8ms/step | 28.6 samples/s +Step 9400/44000 | Loss: 3.1963 | LR: 4.52e-04 | 700.0ms/step | 28.6 samples/s +Step 9410/44000 | Loss: 3.1891 | LR: 4.52e-04 | 698.0ms/step | 28.7 samples/s +Step 9420/44000 | Loss: 3.1802 | LR: 4.52e-04 | 694.0ms/step | 28.8 samples/s +Step 9430/44000 | Loss: 3.2381 | LR: 4.52e-04 | 697.1ms/step | 28.7 samples/s +Step 9440/44000 | Loss: 3.1419 | LR: 4.52e-04 | 696.3ms/step | 28.7 samples/s +Step 9450/44000 | Loss: 3.1573 | LR: 4.52e-04 | 697.9ms/step | 28.7 samples/s +Step 9460/44000 | Loss: 3.1598 | LR: 4.52e-04 | 698.2ms/step | 28.6 samples/s +Step 9470/44000 | Loss: 3.1611 | LR: 4.52e-04 | 696.7ms/step | 28.7 samples/s +Step 9480/44000 | Loss: 3.2091 | LR: 4.51e-04 | 696.7ms/step | 28.7 samples/s +Step 9490/44000 | Loss: 3.1597 | LR: 4.51e-04 | 698.8ms/step | 28.6 samples/s +Step 9500/44000 | Loss: 3.1589 | LR: 4.51e-04 | 696.6ms/step | 28.7 samples/s +Validation at step 9500: Loss = 3.2013, Perplexity = 24.56 +Step 9510/44000 | Loss: 3.1840 | LR: 4.51e-04 | 1266.3ms/step | 15.8 samples/s +Step 9520/44000 | Loss: 3.1487 | LR: 4.51e-04 | 696.8ms/step | 28.7 samples/s +Step 9530/44000 | Loss: 3.1191 | LR: 4.51e-04 | 697.4ms/step | 28.7 samples/s +Step 9540/44000 | Loss: 3.1581 | LR: 4.51e-04 | 698.5ms/step | 28.6 samples/s +Step 9550/44000 | Loss: 3.2531 | LR: 4.51e-04 | 698.4ms/step | 28.6 samples/s +Step 9560/44000 | Loss: 3.1962 | LR: 4.51e-04 | 698.5ms/step | 28.6 samples/s +Step 9570/44000 | Loss: 3.1960 | LR: 4.50e-04 | 700.2ms/step | 28.6 samples/s +Step 9580/44000 | Loss: 3.1711 | LR: 4.50e-04 | 697.5ms/step | 28.7 samples/s +Step 9590/44000 | Loss: 3.2009 | LR: 4.50e-04 | 697.7ms/step | 28.7 samples/s +Step 9600/44000 | Loss: 3.1786 | LR: 4.50e-04 | 697.0ms/step | 28.7 samples/s +Step 9610/44000 | Loss: 3.1609 | LR: 4.50e-04 | 699.7ms/step | 28.6 samples/s +Step 9620/44000 | Loss: 3.1757 | LR: 4.50e-04 | 696.6ms/step | 28.7 samples/s +Step 9630/44000 | Loss: 3.1841 | LR: 4.50e-04 | 697.7ms/step | 28.7 samples/s +Step 9640/44000 | Loss: 3.2037 | LR: 4.50e-04 | 701.2ms/step | 28.5 samples/s +Step 9650/44000 | Loss: 3.2067 | LR: 4.50e-04 | 701.1ms/step | 28.5 samples/s +Step 9660/44000 | Loss: 3.2009 | LR: 4.49e-04 | 698.6ms/step | 28.6 samples/s +Step 9670/44000 | Loss: 3.1425 | LR: 4.49e-04 | 703.1ms/step | 28.4 samples/s +Step 9680/44000 | Loss: 3.1854 | LR: 4.49e-04 | 698.4ms/step | 28.6 samples/s +Step 9690/44000 | Loss: 3.1799 | LR: 4.49e-04 | 698.0ms/step | 28.7 samples/s +Step 9700/44000 | Loss: 3.1528 | LR: 4.49e-04 | 696.9ms/step | 28.7 samples/s +Step 9710/44000 | Loss: 3.1515 | LR: 4.49e-04 | 698.7ms/step | 28.6 samples/s +Step 9720/44000 | Loss: 3.1842 | LR: 4.49e-04 | 698.7ms/step | 28.6 samples/s +Step 9730/44000 | Loss: 3.1325 | LR: 4.49e-04 | 697.3ms/step | 28.7 samples/s +Step 9740/44000 | Loss: 3.1388 | LR: 4.49e-04 | 697.2ms/step | 28.7 samples/s +Step 9750/44000 | Loss: 3.1903 | LR: 4.48e-04 | 698.6ms/step | 28.6 samples/s +Validation at step 9750: Loss = 3.1983, Perplexity = 24.49 +Step 9760/44000 | Loss: 3.1555 | LR: 4.48e-04 | 1270.2ms/step | 15.7 samples/s +Step 9770/44000 | Loss: 3.1874 | LR: 4.48e-04 | 702.8ms/step | 28.5 samples/s +Step 9780/44000 | Loss: 3.1265 | LR: 4.48e-04 | 697.2ms/step | 28.7 samples/s +Step 9790/44000 | Loss: 3.1758 | LR: 4.48e-04 | 697.9ms/step | 28.7 samples/s +Step 9800/44000 | Loss: 3.2136 | LR: 4.48e-04 | 695.8ms/step | 28.7 samples/s +Step 9810/44000 | Loss: 3.1809 | LR: 4.48e-04 | 698.7ms/step | 28.6 samples/s +Step 9820/44000 | Loss: 3.1643 | LR: 4.48e-04 | 699.1ms/step | 28.6 samples/s +Step 9830/44000 | Loss: 3.2510 | LR: 4.48e-04 | 696.9ms/step | 28.7 samples/s +Step 9840/44000 | Loss: 3.1319 | LR: 4.47e-04 | 696.1ms/step | 28.7 samples/s +Step 9850/44000 | Loss: 3.1517 | LR: 4.47e-04 | 699.9ms/step | 28.6 samples/s +Step 9860/44000 | Loss: 3.2568 | LR: 4.47e-04 | 699.4ms/step | 28.6 samples/s +Step 9870/44000 | Loss: 3.1189 | LR: 4.47e-04 | 697.2ms/step | 28.7 samples/s +Step 9880/44000 | Loss: 3.1593 | LR: 4.47e-04 | 698.9ms/step | 28.6 samples/s +Step 9890/44000 | Loss: 3.1557 | LR: 4.47e-04 | 696.6ms/step | 28.7 samples/s +Step 9900/44000 | Loss: 3.1700 | LR: 4.47e-04 | 697.9ms/step | 28.7 samples/s +Step 9910/44000 | Loss: 3.1964 | LR: 4.47e-04 | 698.7ms/step | 28.6 samples/s +Step 9920/44000 | Loss: 3.1378 | LR: 4.47e-04 | 699.3ms/step | 28.6 samples/s +Step 9930/44000 | Loss: 3.1477 | LR: 4.46e-04 | 695.2ms/step | 28.8 samples/s +Step 9940/44000 | Loss: 3.1654 | LR: 4.46e-04 | 699.5ms/step | 28.6 samples/s +Step 9950/44000 | Loss: 3.1802 | LR: 4.46e-04 | 698.4ms/step | 28.6 samples/s +Step 9960/44000 | Loss: 3.1823 | LR: 4.46e-04 | 701.5ms/step | 28.5 samples/s +Step 9970/44000 | Loss: 3.1777 | LR: 4.46e-04 | 697.0ms/step | 28.7 samples/s +Step 9980/44000 | Loss: 3.1492 | LR: 4.46e-04 | 698.7ms/step | 28.6 samples/s +Step 9990/44000 | Loss: 3.1401 | LR: 4.46e-04 | 700.5ms/step | 28.6 samples/s +Step 10000/44000 | Loss: 3.2065 | LR: 4.46e-04 | 695.8ms/step | 28.7 samples/s +Validation at step 10000: Loss = 3.1916, Perplexity = 24.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-10000 +Step 10010/44000 | Loss: 3.1693 | LR: 4.46e-04 | 1592.2ms/step | 12.6 samples/s +Step 10020/44000 | Loss: 3.1421 | LR: 4.45e-04 | 698.0ms/step | 28.7 samples/s +Step 10030/44000 | Loss: 3.1623 | LR: 4.45e-04 | 701.1ms/step | 28.5 samples/s +Step 10040/44000 | Loss: 3.1726 | LR: 4.45e-04 | 700.1ms/step | 28.6 samples/s +Step 10050/44000 | Loss: 3.1697 | LR: 4.45e-04 | 692.9ms/step | 28.9 samples/s +Step 10060/44000 | Loss: 3.1859 | LR: 4.45e-04 | 698.0ms/step | 28.7 samples/s +Step 10070/44000 | Loss: 3.1927 | LR: 4.45e-04 | 697.4ms/step | 28.7 samples/s +Step 10080/44000 | Loss: 3.1678 | LR: 4.45e-04 | 698.9ms/step | 28.6 samples/s +Step 10090/44000 | Loss: 3.1146 | LR: 4.45e-04 | 694.8ms/step | 28.8 samples/s +Step 10100/44000 | Loss: 3.1846 | LR: 4.45e-04 | 698.9ms/step | 28.6 samples/s +Step 10110/44000 | Loss: 3.1668 | LR: 4.44e-04 | 697.2ms/step | 28.7 samples/s +Step 10120/44000 | Loss: 3.1540 | LR: 4.44e-04 | 696.8ms/step | 28.7 samples/s +Step 10130/44000 | Loss: 3.1611 | LR: 4.44e-04 | 697.2ms/step | 28.7 samples/s +Step 10140/44000 | Loss: 3.1614 | LR: 4.44e-04 | 696.1ms/step | 28.7 samples/s +Step 10150/44000 | Loss: 3.1489 | LR: 4.44e-04 | 697.9ms/step | 28.7 samples/s +Step 10160/44000 | Loss: 3.1685 | LR: 4.44e-04 | 698.4ms/step | 28.6 samples/s +Step 10170/44000 | Loss: 3.1599 | LR: 4.44e-04 | 697.3ms/step | 28.7 samples/s +Step 10180/44000 | Loss: 3.1457 | LR: 4.44e-04 | 698.1ms/step | 28.6 samples/s +Step 10190/44000 | Loss: 3.1717 | LR: 4.43e-04 | 699.1ms/step | 28.6 samples/s +Step 10200/44000 | Loss: 3.1086 | LR: 4.43e-04 | 698.2ms/step | 28.6 samples/s +Step 10210/44000 | Loss: 3.1735 | LR: 4.43e-04 | 697.2ms/step | 28.7 samples/s +Step 10220/44000 | Loss: 3.1654 | LR: 4.43e-04 | 701.1ms/step | 28.5 samples/s +Step 10230/44000 | Loss: 3.1331 | LR: 4.43e-04 | 700.1ms/step | 28.6 samples/s +Step 10240/44000 | Loss: 3.1336 | LR: 4.43e-04 | 700.6ms/step | 28.5 samples/s +Step 10250/44000 | Loss: 3.1276 | LR: 4.43e-04 | 703.6ms/step | 28.4 samples/s +Validation at step 10250: Loss = 3.1827, Perplexity = 24.11 +Step 10260/44000 | Loss: 3.1884 | LR: 4.43e-04 | 1270.2ms/step | 15.7 samples/s +Step 10270/44000 | Loss: 3.1378 | LR: 4.43e-04 | 701.9ms/step | 28.5 samples/s +Step 10280/44000 | Loss: 3.1588 | LR: 4.42e-04 | 701.2ms/step | 28.5 samples/s +Step 10290/44000 | Loss: 3.1557 | LR: 4.42e-04 | 701.1ms/step | 28.5 samples/s +Step 10300/44000 | Loss: 3.1172 | LR: 4.42e-04 | 698.7ms/step | 28.6 samples/s +Step 10310/44000 | Loss: 3.1531 | LR: 4.42e-04 | 699.6ms/step | 28.6 samples/s +Step 10320/44000 | Loss: 3.1407 | LR: 4.42e-04 | 700.9ms/step | 28.5 samples/s +Step 10330/44000 | Loss: 3.1771 | LR: 4.42e-04 | 698.1ms/step | 28.7 samples/s +Step 10340/44000 | Loss: 3.1155 | LR: 4.42e-04 | 700.4ms/step | 28.6 samples/s +Step 10350/44000 | Loss: 3.0942 | LR: 4.42e-04 | 699.7ms/step | 28.6 samples/s +Step 10360/44000 | Loss: 3.2061 | LR: 4.42e-04 | 699.4ms/step | 28.6 samples/s +Step 10370/44000 | Loss: 3.2131 | LR: 4.41e-04 | 700.5ms/step | 28.5 samples/s +Step 10380/44000 | Loss: 3.0716 | LR: 4.41e-04 | 699.8ms/step | 28.6 samples/s +Step 10390/44000 | Loss: 3.2207 | LR: 4.41e-04 | 699.0ms/step | 28.6 samples/s +Step 10400/44000 | Loss: 3.1625 | LR: 4.41e-04 | 699.2ms/step | 28.6 samples/s +Step 10410/44000 | Loss: 3.1161 | LR: 4.41e-04 | 699.2ms/step | 28.6 samples/s +Step 10420/44000 | Loss: 3.1867 | LR: 4.41e-04 | 697.2ms/step | 28.7 samples/s +Step 10430/44000 | Loss: 3.1479 | LR: 4.41e-04 | 699.6ms/step | 28.6 samples/s +Step 10440/44000 | Loss: 3.1784 | LR: 4.41e-04 | 698.7ms/step | 28.6 samples/s +Step 10450/44000 | Loss: 3.1343 | LR: 4.40e-04 | 699.0ms/step | 28.6 samples/s +Step 10460/44000 | Loss: 3.1818 | LR: 4.40e-04 | 699.9ms/step | 28.6 samples/s +Step 10470/44000 | Loss: 3.1370 | LR: 4.40e-04 | 701.6ms/step | 28.5 samples/s +Step 10480/44000 | Loss: 3.1397 | LR: 4.40e-04 | 699.3ms/step | 28.6 samples/s +Step 10490/44000 | Loss: 3.1605 | LR: 4.40e-04 | 698.4ms/step | 28.6 samples/s +Step 10500/44000 | Loss: 3.1238 | LR: 4.40e-04 | 698.2ms/step | 28.6 samples/s +Validation at step 10500: Loss = 3.1776, Perplexity = 23.99 +Step 10510/44000 | Loss: 3.1167 | LR: 4.40e-04 | 1266.7ms/step | 15.8 samples/s +Step 10520/44000 | Loss: 3.1541 | LR: 4.40e-04 | 699.7ms/step | 28.6 samples/s +Step 10530/44000 | Loss: 3.1651 | LR: 4.40e-04 | 706.9ms/step | 28.3 samples/s +Step 10540/44000 | Loss: 3.1764 | LR: 4.39e-04 | 699.7ms/step | 28.6 samples/s +Step 10550/44000 | Loss: 3.1199 | LR: 4.39e-04 | 698.1ms/step | 28.6 samples/s +Step 10560/44000 | Loss: 3.1469 | LR: 4.39e-04 | 699.3ms/step | 28.6 samples/s +Step 10570/44000 | Loss: 3.1480 | LR: 4.39e-04 | 701.4ms/step | 28.5 samples/s +Step 10580/44000 | Loss: 3.1460 | LR: 4.39e-04 | 701.4ms/step | 28.5 samples/s +Step 10590/44000 | Loss: 3.1444 | LR: 4.39e-04 | 698.2ms/step | 28.6 samples/s +Step 10600/44000 | Loss: 3.1798 | LR: 4.39e-04 | 700.4ms/step | 28.6 samples/s +Step 10610/44000 | Loss: 3.1280 | LR: 4.39e-04 | 698.2ms/step | 28.6 samples/s +Step 10620/44000 | Loss: 3.1682 | LR: 4.38e-04 | 698.7ms/step | 28.6 samples/s +Step 10630/44000 | Loss: 3.1402 | LR: 4.38e-04 | 701.8ms/step | 28.5 samples/s +Step 10640/44000 | Loss: 3.1464 | LR: 4.38e-04 | 701.4ms/step | 28.5 samples/s +Step 10650/44000 | Loss: 3.1464 | LR: 4.38e-04 | 697.0ms/step | 28.7 samples/s +Step 10660/44000 | Loss: 3.1190 | LR: 4.38e-04 | 696.7ms/step | 28.7 samples/s +Step 10670/44000 | Loss: 3.1578 | LR: 4.38e-04 | 699.6ms/step | 28.6 samples/s +Step 10680/44000 | Loss: 3.1499 | LR: 4.38e-04 | 699.4ms/step | 28.6 samples/s +Step 10690/44000 | Loss: 3.1666 | LR: 4.38e-04 | 702.5ms/step | 28.5 samples/s +Step 10700/44000 | Loss: 3.1276 | LR: 4.37e-04 | 701.4ms/step | 28.5 samples/s +Step 10710/44000 | Loss: 3.1820 | LR: 4.37e-04 | 698.7ms/step | 28.6 samples/s +Step 10720/44000 | Loss: 3.1710 | LR: 4.37e-04 | 700.3ms/step | 28.6 samples/s +Step 10730/44000 | Loss: 3.1406 | LR: 4.37e-04 | 698.4ms/step | 28.6 samples/s +Step 10740/44000 | Loss: 3.1644 | LR: 4.37e-04 | 701.8ms/step | 28.5 samples/s +Step 10750/44000 | Loss: 3.1525 | LR: 4.37e-04 | 698.1ms/step | 28.7 samples/s +Validation at step 10750: Loss = 3.1726, Perplexity = 23.87 +Step 10760/44000 | Loss: 3.1147 | LR: 4.37e-04 | 1269.1ms/step | 15.8 samples/s +Step 10770/44000 | Loss: 3.1322 | LR: 4.37e-04 | 698.2ms/step | 28.6 samples/s +Step 10780/44000 | Loss: 3.1022 | LR: 4.37e-04 | 704.1ms/step | 28.4 samples/s +Step 10790/44000 | Loss: 3.1519 | LR: 4.36e-04 | 700.7ms/step | 28.5 samples/s +Step 10800/44000 | Loss: 3.1724 | LR: 4.36e-04 | 702.0ms/step | 28.5 samples/s +Step 10810/44000 | Loss: 3.1536 | LR: 4.36e-04 | 702.2ms/step | 28.5 samples/s +Step 10820/44000 | Loss: 3.1484 | LR: 4.36e-04 | 698.0ms/step | 28.7 samples/s +Step 10830/44000 | Loss: 3.1521 | LR: 4.36e-04 | 699.1ms/step | 28.6 samples/s +Step 10840/44000 | Loss: 3.1555 | LR: 4.36e-04 | 696.9ms/step | 28.7 samples/s +Step 10850/44000 | Loss: 3.1727 | LR: 4.36e-04 | 698.9ms/step | 28.6 samples/s +Step 10860/44000 | Loss: 3.1543 | LR: 4.36e-04 | 696.6ms/step | 28.7 samples/s +Step 10870/44000 | Loss: 3.1481 | LR: 4.35e-04 | 702.4ms/step | 28.5 samples/s +Step 10880/44000 | Loss: 3.1717 | LR: 4.35e-04 | 698.1ms/step | 28.6 samples/s +Step 10890/44000 | Loss: 3.1564 | LR: 4.35e-04 | 705.8ms/step | 28.3 samples/s +Step 10900/44000 | Loss: 3.1113 | LR: 4.35e-04 | 699.4ms/step | 28.6 samples/s +Step 10910/44000 | Loss: 3.1477 | LR: 4.35e-04 | 695.6ms/step | 28.8 samples/s +Step 10920/44000 | Loss: 3.1359 | LR: 4.35e-04 | 696.9ms/step | 28.7 samples/s +Step 10930/44000 | Loss: 3.1470 | LR: 4.35e-04 | 700.3ms/step | 28.6 samples/s +Step 10940/44000 | Loss: 3.1325 | LR: 4.35e-04 | 702.8ms/step | 28.5 samples/s +Step 10950/44000 | Loss: 3.1892 | LR: 4.34e-04 | 700.5ms/step | 28.6 samples/s +Step 10960/44000 | Loss: 3.1284 | LR: 4.34e-04 | 701.6ms/step | 28.5 samples/s +Step 10970/44000 | Loss: 3.1158 | LR: 4.34e-04 | 699.6ms/step | 28.6 samples/s +Step 10980/44000 | Loss: 3.1126 | LR: 4.34e-04 | 694.0ms/step | 28.8 samples/s +Step 10990/44000 | Loss: 3.1474 | LR: 4.34e-04 | 698.6ms/step | 28.6 samples/s +Step 11000/44000 | Loss: 3.1598 | LR: 4.34e-04 | 703.1ms/step | 28.4 samples/s +Validation at step 11000: Loss = 3.1662, Perplexity = 23.72 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-11000 +Step 11010/44000 | Loss: 3.1912 | LR: 4.34e-04 | 1600.6ms/step | 12.5 samples/s +Step 11020/44000 | Loss: 3.1300 | LR: 4.34e-04 | 700.6ms/step | 28.5 samples/s +Step 11030/44000 | Loss: 3.1475 | LR: 4.33e-04 | 701.8ms/step | 28.5 samples/s +Step 11040/44000 | Loss: 3.1460 | LR: 4.33e-04 | 697.7ms/step | 28.7 samples/s +Step 11050/44000 | Loss: 3.1316 | LR: 4.33e-04 | 703.4ms/step | 28.4 samples/s +Step 11060/44000 | Loss: 3.1461 | LR: 4.33e-04 | 703.9ms/step | 28.4 samples/s +Step 11070/44000 | Loss: 3.1443 | LR: 4.33e-04 | 699.2ms/step | 28.6 samples/s +Step 11080/44000 | Loss: 3.1708 | LR: 4.33e-04 | 698.5ms/step | 28.6 samples/s +Step 11090/44000 | Loss: 3.1713 | LR: 4.33e-04 | 696.4ms/step | 28.7 samples/s +Step 11100/44000 | Loss: 3.1668 | LR: 4.33e-04 | 698.2ms/step | 28.6 samples/s +Step 11110/44000 | Loss: 3.1225 | LR: 4.32e-04 | 699.6ms/step | 28.6 samples/s +Step 11120/44000 | Loss: 3.1687 | LR: 4.32e-04 | 694.5ms/step | 28.8 samples/s +Step 11130/44000 | Loss: 3.1277 | LR: 4.32e-04 | 700.0ms/step | 28.6 samples/s +Step 11140/44000 | Loss: 3.1218 | LR: 4.32e-04 | 699.4ms/step | 28.6 samples/s +Step 11150/44000 | Loss: 3.1021 | LR: 4.32e-04 | 699.8ms/step | 28.6 samples/s +Step 11160/44000 | Loss: 3.1295 | LR: 4.32e-04 | 697.0ms/step | 28.7 samples/s +Step 11170/44000 | Loss: 3.1695 | LR: 4.32e-04 | 700.3ms/step | 28.6 samples/s +Step 11180/44000 | Loss: 3.1133 | LR: 4.32e-04 | 698.0ms/step | 28.7 samples/s +Step 11190/44000 | Loss: 3.1111 | LR: 4.31e-04 | 699.7ms/step | 28.6 samples/s +Step 11200/44000 | Loss: 3.1104 | LR: 4.31e-04 | 698.4ms/step | 28.6 samples/s +Step 11210/44000 | Loss: 3.1679 | LR: 4.31e-04 | 697.1ms/step | 28.7 samples/s +Step 11220/44000 | Loss: 3.1363 | LR: 4.31e-04 | 699.6ms/step | 28.6 samples/s +Step 11230/44000 | Loss: 3.1386 | LR: 4.31e-04 | 699.4ms/step | 28.6 samples/s +Step 11240/44000 | Loss: 3.1587 | LR: 4.31e-04 | 702.8ms/step | 28.5 samples/s +Step 11250/44000 | Loss: 3.1312 | LR: 4.31e-04 | 699.4ms/step | 28.6 samples/s +Validation at step 11250: Loss = 3.1616, Perplexity = 23.61 +Step 11260/44000 | Loss: 3.1504 | LR: 4.31e-04 | 1268.3ms/step | 15.8 samples/s +Step 11270/44000 | Loss: 3.1391 | LR: 4.30e-04 | 697.9ms/step | 28.7 samples/s +Step 11280/44000 | Loss: 3.1496 | LR: 4.30e-04 | 698.2ms/step | 28.6 samples/s +Step 11290/44000 | Loss: 3.0981 | LR: 4.30e-04 | 695.7ms/step | 28.7 samples/s +Step 11300/44000 | Loss: 3.1429 | LR: 4.30e-04 | 701.5ms/step | 28.5 samples/s +Step 11310/44000 | Loss: 3.1150 | LR: 4.30e-04 | 702.3ms/step | 28.5 samples/s +Step 11320/44000 | Loss: 3.1746 | LR: 4.30e-04 | 700.1ms/step | 28.6 samples/s +Step 11330/44000 | Loss: 3.1399 | LR: 4.30e-04 | 699.9ms/step | 28.6 samples/s +Step 11340/44000 | Loss: 3.1184 | LR: 4.30e-04 | 700.1ms/step | 28.6 samples/s +Step 11350/44000 | Loss: 3.1338 | LR: 4.29e-04 | 699.1ms/step | 28.6 samples/s +Step 11360/44000 | Loss: 3.1277 | LR: 4.29e-04 | 699.2ms/step | 28.6 samples/s +Step 11370/44000 | Loss: 3.1231 | LR: 4.29e-04 | 699.6ms/step | 28.6 samples/s +Step 11380/44000 | Loss: 3.1217 | LR: 4.29e-04 | 697.8ms/step | 28.7 samples/s +Step 11390/44000 | Loss: 3.1989 | LR: 4.29e-04 | 698.3ms/step | 28.6 samples/s +Step 11400/44000 | Loss: 3.1450 | LR: 4.29e-04 | 697.3ms/step | 28.7 samples/s +Step 11410/44000 | Loss: 3.1415 | LR: 4.29e-04 | 702.0ms/step | 28.5 samples/s +Step 11420/44000 | Loss: 3.1109 | LR: 4.29e-04 | 701.2ms/step | 28.5 samples/s +Step 11430/44000 | Loss: 3.1395 | LR: 4.28e-04 | 696.8ms/step | 28.7 samples/s +Step 11440/44000 | Loss: 3.1370 | LR: 4.28e-04 | 698.5ms/step | 28.6 samples/s +Step 11450/44000 | Loss: 3.1637 | LR: 4.28e-04 | 697.1ms/step | 28.7 samples/s +Step 11460/44000 | Loss: 3.1384 | LR: 4.28e-04 | 699.5ms/step | 28.6 samples/s +Step 11470/44000 | Loss: 3.1176 | LR: 4.28e-04 | 697.8ms/step | 28.7 samples/s +Step 11480/44000 | Loss: 3.0988 | LR: 4.28e-04 | 700.0ms/step | 28.6 samples/s +Step 11490/44000 | Loss: 3.1302 | LR: 4.28e-04 | 699.5ms/step | 28.6 samples/s +Step 11500/44000 | Loss: 3.1504 | LR: 4.28e-04 | 697.8ms/step | 28.7 samples/s +Validation at step 11500: Loss = 3.1569, Perplexity = 23.50 +Step 11510/44000 | Loss: 3.1165 | LR: 4.27e-04 | 1265.1ms/step | 15.8 samples/s +Step 11520/44000 | Loss: 3.1036 | LR: 4.27e-04 | 697.6ms/step | 28.7 samples/s +Step 11530/44000 | Loss: 3.1160 | LR: 4.27e-04 | 702.0ms/step | 28.5 samples/s +Step 11540/44000 | Loss: 3.1193 | LR: 4.27e-04 | 698.4ms/step | 28.6 samples/s +Step 11550/44000 | Loss: 3.1087 | LR: 4.27e-04 | 698.9ms/step | 28.6 samples/s +Step 11560/44000 | Loss: 3.1205 | LR: 4.27e-04 | 701.1ms/step | 28.5 samples/s +Step 11570/44000 | Loss: 3.1425 | LR: 4.27e-04 | 697.6ms/step | 28.7 samples/s +Step 11580/44000 | Loss: 3.1045 | LR: 4.27e-04 | 697.7ms/step | 28.7 samples/s +Step 11590/44000 | Loss: 3.1222 | LR: 4.26e-04 | 699.4ms/step | 28.6 samples/s +Step 11600/44000 | Loss: 3.1565 | LR: 4.26e-04 | 702.6ms/step | 28.5 samples/s +Step 11610/44000 | Loss: 3.1243 | LR: 4.26e-04 | 699.3ms/step | 28.6 samples/s +Step 11620/44000 | Loss: 3.1747 | LR: 4.26e-04 | 698.5ms/step | 28.6 samples/s +Step 11630/44000 | Loss: 3.1462 | LR: 4.26e-04 | 696.7ms/step | 28.7 samples/s +Step 11640/44000 | Loss: 3.0262 | LR: 4.26e-04 | 699.8ms/step | 28.6 samples/s +Step 11650/44000 | Loss: 3.0881 | LR: 4.26e-04 | 697.9ms/step | 28.7 samples/s +Step 11660/44000 | Loss: 3.1169 | LR: 4.26e-04 | 701.0ms/step | 28.5 samples/s +Step 11670/44000 | Loss: 3.1320 | LR: 4.25e-04 | 696.8ms/step | 28.7 samples/s +Step 11680/44000 | Loss: 3.1115 | LR: 4.25e-04 | 695.4ms/step | 28.8 samples/s +Step 11690/44000 | Loss: 3.1530 | LR: 4.25e-04 | 699.4ms/step | 28.6 samples/s +Step 11700/44000 | Loss: 3.1411 | LR: 4.25e-04 | 698.1ms/step | 28.6 samples/s +Step 11710/44000 | Loss: 3.1136 | LR: 4.25e-04 | 700.2ms/step | 28.6 samples/s +Step 11720/44000 | Loss: 3.1002 | LR: 4.25e-04 | 700.9ms/step | 28.5 samples/s +Step 11730/44000 | Loss: 3.1004 | LR: 4.25e-04 | 700.7ms/step | 28.5 samples/s +Step 11740/44000 | Loss: 3.1528 | LR: 4.24e-04 | 699.3ms/step | 28.6 samples/s +Step 11750/44000 | Loss: 3.1388 | LR: 4.24e-04 | 697.8ms/step | 28.7 samples/s +Validation at step 11750: Loss = 3.1532, Perplexity = 23.41 +Step 11760/44000 | Loss: 3.0424 | LR: 4.24e-04 | 1266.8ms/step | 15.8 samples/s +Step 11770/44000 | Loss: 3.0840 | LR: 4.24e-04 | 700.9ms/step | 28.5 samples/s +Step 11780/44000 | Loss: 3.0984 | LR: 4.24e-04 | 695.1ms/step | 28.8 samples/s +Step 11790/44000 | Loss: 3.1541 | LR: 4.24e-04 | 696.1ms/step | 28.7 samples/s +Step 11800/44000 | Loss: 3.1512 | LR: 4.24e-04 | 700.8ms/step | 28.5 samples/s +Step 11810/44000 | Loss: 3.1186 | LR: 4.24e-04 | 700.9ms/step | 28.5 samples/s +Step 11820/44000 | Loss: 3.1264 | LR: 4.23e-04 | 698.3ms/step | 28.6 samples/s +Step 11830/44000 | Loss: 3.0868 | LR: 4.23e-04 | 698.6ms/step | 28.6 samples/s +Step 11840/44000 | Loss: 3.0884 | LR: 4.23e-04 | 696.3ms/step | 28.7 samples/s +Step 11850/44000 | Loss: 3.1144 | LR: 4.23e-04 | 698.1ms/step | 28.7 samples/s +Step 11860/44000 | Loss: 3.1958 | LR: 4.23e-04 | 697.9ms/step | 28.7 samples/s +Step 11870/44000 | Loss: 3.1624 | LR: 4.23e-04 | 700.5ms/step | 28.6 samples/s +Step 11880/44000 | Loss: 3.1158 | LR: 4.23e-04 | 699.5ms/step | 28.6 samples/s +Step 11890/44000 | Loss: 3.1308 | LR: 4.23e-04 | 697.1ms/step | 28.7 samples/s +Step 11900/44000 | Loss: 3.1565 | LR: 4.22e-04 | 699.2ms/step | 28.6 samples/s +Step 11910/44000 | Loss: 3.1116 | LR: 4.22e-04 | 699.5ms/step | 28.6 samples/s +Step 11920/44000 | Loss: 3.1194 | LR: 4.22e-04 | 697.9ms/step | 28.7 samples/s +Step 11930/44000 | Loss: 3.1478 | LR: 4.22e-04 | 697.7ms/step | 28.7 samples/s +Step 11940/44000 | Loss: 3.1426 | LR: 4.22e-04 | 701.7ms/step | 28.5 samples/s +Step 11950/44000 | Loss: 3.1093 | LR: 4.22e-04 | 701.4ms/step | 28.5 samples/s +Step 11960/44000 | Loss: 3.1533 | LR: 4.22e-04 | 697.5ms/step | 28.7 samples/s +Step 11970/44000 | Loss: 3.1723 | LR: 4.21e-04 | 702.1ms/step | 28.5 samples/s +Step 11980/44000 | Loss: 3.0835 | LR: 4.21e-04 | 697.9ms/step | 28.7 samples/s +Step 11990/44000 | Loss: 3.1295 | LR: 4.21e-04 | 697.9ms/step | 28.7 samples/s +Step 12000/44000 | Loss: 3.1509 | LR: 4.21e-04 | 701.0ms/step | 28.5 samples/s +Validation at step 12000: Loss = 3.1481, Perplexity = 23.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-12000 +Step 12010/44000 | Loss: 3.1374 | LR: 4.21e-04 | 1577.8ms/step | 12.7 samples/s +Step 12020/44000 | Loss: 3.1680 | LR: 4.21e-04 | 696.7ms/step | 28.7 samples/s +Step 12030/44000 | Loss: 3.0721 | LR: 4.21e-04 | 699.8ms/step | 28.6 samples/s +Step 12040/44000 | Loss: 3.1783 | LR: 4.21e-04 | 696.0ms/step | 28.7 samples/s +Step 12050/44000 | Loss: 3.1173 | LR: 4.20e-04 | 700.3ms/step | 28.6 samples/s +Step 12060/44000 | Loss: 3.1176 | LR: 4.20e-04 | 698.5ms/step | 28.6 samples/s +Step 12070/44000 | Loss: 3.1117 | LR: 4.20e-04 | 697.4ms/step | 28.7 samples/s +Step 12080/44000 | Loss: 3.1363 | LR: 4.20e-04 | 702.0ms/step | 28.5 samples/s +Step 12090/44000 | Loss: 3.1509 | LR: 4.20e-04 | 700.4ms/step | 28.6 samples/s +Step 12100/44000 | Loss: 3.1164 | LR: 4.20e-04 | 697.5ms/step | 28.7 samples/s +Step 12110/44000 | Loss: 3.1164 | LR: 4.20e-04 | 699.6ms/step | 28.6 samples/s +Step 12120/44000 | Loss: 3.0965 | LR: 4.19e-04 | 699.2ms/step | 28.6 samples/s +Step 12130/44000 | Loss: 3.1145 | LR: 4.19e-04 | 701.1ms/step | 28.5 samples/s +Step 12140/44000 | Loss: 3.0805 | LR: 4.19e-04 | 694.5ms/step | 28.8 samples/s +Step 12150/44000 | Loss: 3.1315 | LR: 4.19e-04 | 700.4ms/step | 28.6 samples/s +Step 12160/44000 | Loss: 3.1177 | LR: 4.19e-04 | 696.7ms/step | 28.7 samples/s +Step 12170/44000 | Loss: 3.0803 | LR: 4.19e-04 | 696.4ms/step | 28.7 samples/s +Step 12180/44000 | Loss: 3.1316 | LR: 4.19e-04 | 696.1ms/step | 28.7 samples/s +Step 12190/44000 | Loss: 3.1205 | LR: 4.19e-04 | 697.6ms/step | 28.7 samples/s +Step 12200/44000 | Loss: 3.1244 | LR: 4.18e-04 | 699.7ms/step | 28.6 samples/s +Step 12210/44000 | Loss: 3.1254 | LR: 4.18e-04 | 695.0ms/step | 28.8 samples/s +Step 12220/44000 | Loss: 3.1423 | LR: 4.18e-04 | 698.2ms/step | 28.6 samples/s +Step 12230/44000 | Loss: 3.0779 | LR: 4.18e-04 | 695.6ms/step | 28.8 samples/s +Step 12240/44000 | Loss: 3.1360 | LR: 4.18e-04 | 694.9ms/step | 28.8 samples/s +Step 12250/44000 | Loss: 3.1157 | LR: 4.18e-04 | 696.4ms/step | 28.7 samples/s +Validation at step 12250: Loss = 3.1443, Perplexity = 23.20 +Step 12260/44000 | Loss: 3.1251 | LR: 4.18e-04 | 1263.8ms/step | 15.8 samples/s +Step 12270/44000 | Loss: 3.1146 | LR: 4.17e-04 | 695.2ms/step | 28.8 samples/s +Step 12280/44000 | Loss: 3.1310 | LR: 4.17e-04 | 695.0ms/step | 28.8 samples/s +Step 12290/44000 | Loss: 3.1015 | LR: 4.17e-04 | 694.7ms/step | 28.8 samples/s +Step 12300/44000 | Loss: 3.0819 | LR: 4.17e-04 | 695.1ms/step | 28.8 samples/s +Step 12310/44000 | Loss: 3.1240 | LR: 4.17e-04 | 695.8ms/step | 28.7 samples/s +Step 12320/44000 | Loss: 3.1531 | LR: 4.17e-04 | 696.7ms/step | 28.7 samples/s +Step 12330/44000 | Loss: 3.1389 | LR: 4.17e-04 | 695.0ms/step | 28.8 samples/s +Step 12340/44000 | Loss: 3.1230 | LR: 4.17e-04 | 699.2ms/step | 28.6 samples/s +Step 12350/44000 | Loss: 3.1162 | LR: 4.16e-04 | 694.8ms/step | 28.8 samples/s +Step 12360/44000 | Loss: 3.1082 | LR: 4.16e-04 | 693.3ms/step | 28.8 samples/s +Step 12370/44000 | Loss: 3.1262 | LR: 4.16e-04 | 694.3ms/step | 28.8 samples/s +Step 12380/44000 | Loss: 3.1115 | LR: 4.16e-04 | 693.3ms/step | 28.8 samples/s +Step 12390/44000 | Loss: 3.1134 | LR: 4.16e-04 | 696.8ms/step | 28.7 samples/s +Step 12400/44000 | Loss: 3.0922 | LR: 4.16e-04 | 697.3ms/step | 28.7 samples/s +Step 12410/44000 | Loss: 3.1225 | LR: 4.16e-04 | 696.0ms/step | 28.7 samples/s +Step 12420/44000 | Loss: 3.0922 | LR: 4.15e-04 | 696.8ms/step | 28.7 samples/s +Step 12430/44000 | Loss: 3.1564 | LR: 4.15e-04 | 697.4ms/step | 28.7 samples/s +Step 12440/44000 | Loss: 3.1064 | LR: 4.15e-04 | 694.5ms/step | 28.8 samples/s +Step 12450/44000 | Loss: 3.1271 | LR: 4.15e-04 | 695.7ms/step | 28.7 samples/s +Step 12460/44000 | Loss: 3.0750 | LR: 4.15e-04 | 697.1ms/step | 28.7 samples/s +Step 12470/44000 | Loss: 3.0969 | LR: 4.15e-04 | 698.1ms/step | 28.6 samples/s +Step 12480/44000 | Loss: 3.1134 | LR: 4.15e-04 | 697.0ms/step | 28.7 samples/s +Step 12490/44000 | Loss: 3.1055 | LR: 4.14e-04 | 697.2ms/step | 28.7 samples/s +Step 12500/44000 | Loss: 3.0899 | LR: 4.14e-04 | 701.0ms/step | 28.5 samples/s +Validation at step 12500: Loss = 3.1409, Perplexity = 23.12 +Step 12510/44000 | Loss: 3.1083 | LR: 4.14e-04 | 1265.1ms/step | 15.8 samples/s +Step 12520/44000 | Loss: 3.0956 | LR: 4.14e-04 | 700.4ms/step | 28.6 samples/s +Step 12530/44000 | Loss: 3.1222 | LR: 4.14e-04 | 699.5ms/step | 28.6 samples/s +Step 12540/44000 | Loss: 3.1614 | LR: 4.14e-04 | 698.4ms/step | 28.6 samples/s +Step 12550/44000 | Loss: 3.1293 | LR: 4.14e-04 | 695.9ms/step | 28.7 samples/s +Step 12560/44000 | Loss: 3.1143 | LR: 4.14e-04 | 694.1ms/step | 28.8 samples/s +Step 12570/44000 | Loss: 3.1002 | LR: 4.13e-04 | 697.6ms/step | 28.7 samples/s +Step 12580/44000 | Loss: 3.0829 | LR: 4.13e-04 | 696.5ms/step | 28.7 samples/s +Step 12590/44000 | Loss: 3.1319 | LR: 4.13e-04 | 697.0ms/step | 28.7 samples/s +Step 12600/44000 | Loss: 3.1405 | LR: 4.13e-04 | 697.3ms/step | 28.7 samples/s +Step 12610/44000 | Loss: 3.1119 | LR: 4.13e-04 | 695.6ms/step | 28.8 samples/s +Step 12620/44000 | Loss: 3.0981 | LR: 4.13e-04 | 697.2ms/step | 28.7 samples/s +Step 12630/44000 | Loss: 3.0793 | LR: 4.13e-04 | 695.9ms/step | 28.7 samples/s +Step 12640/44000 | Loss: 3.1679 | LR: 4.12e-04 | 694.0ms/step | 28.8 samples/s +Step 12650/44000 | Loss: 3.1185 | LR: 4.12e-04 | 694.7ms/step | 28.8 samples/s +Step 12660/44000 | Loss: 3.0736 | LR: 4.12e-04 | 698.7ms/step | 28.6 samples/s +Step 12670/44000 | Loss: 3.1116 | LR: 4.12e-04 | 699.7ms/step | 28.6 samples/s +Step 12680/44000 | Loss: 3.1012 | LR: 4.12e-04 | 697.2ms/step | 28.7 samples/s +Step 12690/44000 | Loss: 3.0890 | LR: 4.12e-04 | 697.5ms/step | 28.7 samples/s +Step 12700/44000 | Loss: 3.1138 | LR: 4.12e-04 | 695.3ms/step | 28.8 samples/s +Step 12710/44000 | Loss: 3.1151 | LR: 4.11e-04 | 699.5ms/step | 28.6 samples/s +Step 12720/44000 | Loss: 3.1144 | LR: 4.11e-04 | 697.6ms/step | 28.7 samples/s +Step 12730/44000 | Loss: 3.0420 | LR: 4.11e-04 | 695.6ms/step | 28.8 samples/s +Step 12740/44000 | Loss: 3.1163 | LR: 4.11e-04 | 697.5ms/step | 28.7 samples/s +Step 12750/44000 | Loss: 3.0975 | LR: 4.11e-04 | 699.2ms/step | 28.6 samples/s +Validation at step 12750: Loss = 3.1360, Perplexity = 23.01 +Step 12760/44000 | Loss: 3.0941 | LR: 4.11e-04 | 1266.9ms/step | 15.8 samples/s +Step 12770/44000 | Loss: 3.0872 | LR: 4.11e-04 | 696.7ms/step | 28.7 samples/s +Step 12780/44000 | Loss: 3.1231 | LR: 4.10e-04 | 693.2ms/step | 28.9 samples/s +Step 12790/44000 | Loss: 3.0789 | LR: 4.10e-04 | 700.0ms/step | 28.6 samples/s +Step 12800/44000 | Loss: 3.1004 | LR: 4.10e-04 | 697.5ms/step | 28.7 samples/s +Step 12810/44000 | Loss: 3.1081 | LR: 4.10e-04 | 701.8ms/step | 28.5 samples/s +Step 12820/44000 | Loss: 3.1004 | LR: 4.10e-04 | 695.8ms/step | 28.7 samples/s +Step 12830/44000 | Loss: 3.0870 | LR: 4.10e-04 | 696.0ms/step | 28.7 samples/s +Step 12840/44000 | Loss: 3.1203 | LR: 4.10e-04 | 699.6ms/step | 28.6 samples/s +Step 12850/44000 | Loss: 3.1252 | LR: 4.09e-04 | 698.7ms/step | 28.6 samples/s +Step 12860/44000 | Loss: 3.1256 | LR: 4.09e-04 | 695.9ms/step | 28.7 samples/s +Step 12870/44000 | Loss: 3.1293 | LR: 4.09e-04 | 696.2ms/step | 28.7 samples/s +Step 12880/44000 | Loss: 3.1381 | LR: 4.09e-04 | 700.9ms/step | 28.5 samples/s +Step 12890/44000 | Loss: 3.0989 | LR: 4.09e-04 | 699.3ms/step | 28.6 samples/s +Step 12900/44000 | Loss: 3.0924 | LR: 4.09e-04 | 696.3ms/step | 28.7 samples/s +Step 12910/44000 | Loss: 3.1108 | LR: 4.09e-04 | 696.2ms/step | 28.7 samples/s +Step 12920/44000 | Loss: 3.0668 | LR: 4.09e-04 | 698.0ms/step | 28.7 samples/s +Step 12930/44000 | Loss: 3.1049 | LR: 4.08e-04 | 699.0ms/step | 28.6 samples/s +Step 12940/44000 | Loss: 3.0726 | LR: 4.08e-04 | 699.2ms/step | 28.6 samples/s +Step 12950/44000 | Loss: 3.1012 | LR: 4.08e-04 | 697.6ms/step | 28.7 samples/s +Step 12960/44000 | Loss: 3.0991 | LR: 4.08e-04 | 699.3ms/step | 28.6 samples/s +Step 12970/44000 | Loss: 3.0984 | LR: 4.08e-04 | 700.9ms/step | 28.5 samples/s +Step 12980/44000 | Loss: 3.0961 | LR: 4.08e-04 | 699.2ms/step | 28.6 samples/s +Step 12990/44000 | Loss: 3.0853 | LR: 4.08e-04 | 699.6ms/step | 28.6 samples/s +Step 13000/44000 | Loss: 3.1025 | LR: 4.07e-04 | 700.9ms/step | 28.5 samples/s +Validation at step 13000: Loss = 3.1308, Perplexity = 22.89 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-13000 +Step 13010/44000 | Loss: 3.1083 | LR: 4.07e-04 | 1645.6ms/step | 12.2 samples/s +Step 13020/44000 | Loss: 3.1239 | LR: 4.07e-04 | 699.6ms/step | 28.6 samples/s +Step 13030/44000 | Loss: 3.0830 | LR: 4.07e-04 | 698.0ms/step | 28.7 samples/s +Step 13040/44000 | Loss: 3.1022 | LR: 4.07e-04 | 699.6ms/step | 28.6 samples/s +Step 13050/44000 | Loss: 3.0913 | LR: 4.07e-04 | 701.8ms/step | 28.5 samples/s +Step 13060/44000 | Loss: 3.1121 | LR: 4.07e-04 | 700.7ms/step | 28.5 samples/s +Step 13070/44000 | Loss: 3.0908 | LR: 4.06e-04 | 702.5ms/step | 28.5 samples/s +Step 13080/44000 | Loss: 3.1872 | LR: 4.06e-04 | 697.7ms/step | 28.7 samples/s +Step 13090/44000 | Loss: 3.0843 | LR: 4.06e-04 | 698.8ms/step | 28.6 samples/s +Step 13100/44000 | Loss: 3.1112 | LR: 4.06e-04 | 702.5ms/step | 28.5 samples/s +Step 13110/44000 | Loss: 3.1006 | LR: 4.06e-04 | 696.3ms/step | 28.7 samples/s +Step 13120/44000 | Loss: 3.1001 | LR: 4.06e-04 | 694.4ms/step | 28.8 samples/s +Step 13130/44000 | Loss: 3.1309 | LR: 4.06e-04 | 697.1ms/step | 28.7 samples/s +Step 13140/44000 | Loss: 3.1208 | LR: 4.05e-04 | 699.5ms/step | 28.6 samples/s +Step 13150/44000 | Loss: 3.1373 | LR: 4.05e-04 | 700.6ms/step | 28.5 samples/s +Step 13160/44000 | Loss: 3.1629 | LR: 4.05e-04 | 698.9ms/step | 28.6 samples/s +Step 13170/44000 | Loss: 3.0739 | LR: 4.05e-04 | 698.2ms/step | 28.6 samples/s +Step 13180/44000 | Loss: 3.0741 | LR: 4.05e-04 | 695.4ms/step | 28.8 samples/s +Step 13190/44000 | Loss: 3.1284 | LR: 4.05e-04 | 697.6ms/step | 28.7 samples/s +Step 13200/44000 | Loss: 3.1760 | LR: 4.05e-04 | 700.4ms/step | 28.6 samples/s +Step 13210/44000 | Loss: 3.1102 | LR: 4.04e-04 | 700.4ms/step | 28.6 samples/s +Step 13220/44000 | Loss: 3.1474 | LR: 4.04e-04 | 695.6ms/step | 28.8 samples/s +Step 13230/44000 | Loss: 3.1300 | LR: 4.04e-04 | 696.3ms/step | 28.7 samples/s +Step 13240/44000 | Loss: 3.0763 | LR: 4.04e-04 | 699.1ms/step | 28.6 samples/s +Step 13250/44000 | Loss: 3.1110 | LR: 4.04e-04 | 708.7ms/step | 28.2 samples/s +Validation at step 13250: Loss = 3.1285, Perplexity = 22.84 +Step 13260/44000 | Loss: 3.1396 | LR: 4.04e-04 | 1276.9ms/step | 15.7 samples/s +Step 13270/44000 | Loss: 3.0830 | LR: 4.04e-04 | 698.9ms/step | 28.6 samples/s +Step 13280/44000 | Loss: 3.1414 | LR: 4.03e-04 | 703.7ms/step | 28.4 samples/s +Step 13290/44000 | Loss: 3.1288 | LR: 4.03e-04 | 701.1ms/step | 28.5 samples/s +Step 13300/44000 | Loss: 3.0759 | LR: 4.03e-04 | 699.1ms/step | 28.6 samples/s +Step 13310/44000 | Loss: 3.1503 | LR: 4.03e-04 | 699.9ms/step | 28.6 samples/s +Step 13320/44000 | Loss: 3.1009 | LR: 4.03e-04 | 700.0ms/step | 28.6 samples/s +Step 13330/44000 | Loss: 3.0643 | LR: 4.03e-04 | 699.6ms/step | 28.6 samples/s +Step 13340/44000 | Loss: 3.1130 | LR: 4.03e-04 | 698.5ms/step | 28.6 samples/s +Step 13350/44000 | Loss: 3.1242 | LR: 4.02e-04 | 698.7ms/step | 28.6 samples/s +Step 13360/44000 | Loss: 3.0781 | LR: 4.02e-04 | 700.0ms/step | 28.6 samples/s +Step 13370/44000 | Loss: 3.1119 | LR: 4.02e-04 | 702.6ms/step | 28.5 samples/s +Step 13380/44000 | Loss: 3.1138 | LR: 4.02e-04 | 698.3ms/step | 28.6 samples/s +Step 13390/44000 | Loss: 3.1040 | LR: 4.02e-04 | 699.2ms/step | 28.6 samples/s +Step 13400/44000 | Loss: 3.1163 | LR: 4.02e-04 | 697.9ms/step | 28.7 samples/s +Step 13410/44000 | Loss: 3.0998 | LR: 4.02e-04 | 701.7ms/step | 28.5 samples/s +Step 13420/44000 | Loss: 3.0983 | LR: 4.01e-04 | 698.5ms/step | 28.6 samples/s +Step 13430/44000 | Loss: 3.0828 | LR: 4.01e-04 | 701.9ms/step | 28.5 samples/s +Step 13440/44000 | Loss: 3.1591 | LR: 4.01e-04 | 699.8ms/step | 28.6 samples/s +Step 13450/44000 | Loss: 3.0544 | LR: 4.01e-04 | 698.3ms/step | 28.6 samples/s +Step 13460/44000 | Loss: 3.1063 | LR: 4.01e-04 | 701.6ms/step | 28.5 samples/s +Step 13470/44000 | Loss: 3.1113 | LR: 4.01e-04 | 702.2ms/step | 28.5 samples/s +Step 13480/44000 | Loss: 3.1217 | LR: 4.01e-04 | 699.1ms/step | 28.6 samples/s +Step 13490/44000 | Loss: 3.0897 | LR: 4.00e-04 | 701.0ms/step | 28.5 samples/s +Step 13500/44000 | Loss: 3.1187 | LR: 4.00e-04 | 697.5ms/step | 28.7 samples/s +Validation at step 13500: Loss = 3.1229, Perplexity = 22.71 +Step 13510/44000 | Loss: 3.1107 | LR: 4.00e-04 | 1269.3ms/step | 15.8 samples/s +Step 13520/44000 | Loss: 3.0971 | LR: 4.00e-04 | 698.8ms/step | 28.6 samples/s +Step 13530/44000 | Loss: 3.0960 | LR: 4.00e-04 | 698.6ms/step | 28.6 samples/s +Step 13540/44000 | Loss: 3.0717 | LR: 4.00e-04 | 702.9ms/step | 28.5 samples/s +Step 13550/44000 | Loss: 3.0646 | LR: 4.00e-04 | 702.8ms/step | 28.5 samples/s +Step 13560/44000 | Loss: 3.0774 | LR: 3.99e-04 | 701.8ms/step | 28.5 samples/s +Step 13570/44000 | Loss: 3.1026 | LR: 3.99e-04 | 700.0ms/step | 28.6 samples/s +Step 13580/44000 | Loss: 3.0875 | LR: 3.99e-04 | 699.8ms/step | 28.6 samples/s +Step 13590/44000 | Loss: 3.0898 | LR: 3.99e-04 | 698.7ms/step | 28.6 samples/s +Step 13600/44000 | Loss: 3.1077 | LR: 3.99e-04 | 699.0ms/step | 28.6 samples/s +Step 13610/44000 | Loss: 3.0777 | LR: 3.99e-04 | 701.9ms/step | 28.5 samples/s +Step 13620/44000 | Loss: 3.1209 | LR: 3.98e-04 | 699.8ms/step | 28.6 samples/s +Step 13630/44000 | Loss: 3.1189 | LR: 3.98e-04 | 701.3ms/step | 28.5 samples/s +Step 13640/44000 | Loss: 3.1037 | LR: 3.98e-04 | 700.9ms/step | 28.5 samples/s +Step 13650/44000 | Loss: 3.0919 | LR: 3.98e-04 | 700.2ms/step | 28.6 samples/s +Step 13660/44000 | Loss: 3.0809 | LR: 3.98e-04 | 700.8ms/step | 28.5 samples/s +Step 13670/44000 | Loss: 3.1077 | LR: 3.98e-04 | 700.3ms/step | 28.6 samples/s +Step 13680/44000 | Loss: 3.1090 | LR: 3.98e-04 | 698.5ms/step | 28.6 samples/s +Step 13690/44000 | Loss: 3.1362 | LR: 3.97e-04 | 700.6ms/step | 28.5 samples/s +Step 13700/44000 | Loss: 3.1069 | LR: 3.97e-04 | 697.6ms/step | 28.7 samples/s +Step 13710/44000 | Loss: 3.1179 | LR: 3.97e-04 | 700.2ms/step | 28.6 samples/s +Step 13720/44000 | Loss: 3.1225 | LR: 3.97e-04 | 702.7ms/step | 28.5 samples/s +Step 13730/44000 | Loss: 3.1157 | LR: 3.97e-04 | 699.9ms/step | 28.6 samples/s +Step 13740/44000 | Loss: 3.0608 | LR: 3.97e-04 | 696.8ms/step | 28.7 samples/s +Step 13750/44000 | Loss: 3.0835 | LR: 3.97e-04 | 698.3ms/step | 28.6 samples/s +Validation at step 13750: Loss = 3.1190, Perplexity = 22.62 +Step 13760/44000 | Loss: 3.0869 | LR: 3.96e-04 | 1270.4ms/step | 15.7 samples/s +Step 13770/44000 | Loss: 3.0947 | LR: 3.96e-04 | 701.2ms/step | 28.5 samples/s +Step 13780/44000 | Loss: 3.1158 | LR: 3.96e-04 | 702.5ms/step | 28.5 samples/s +Step 13790/44000 | Loss: 3.1173 | LR: 3.96e-04 | 697.3ms/step | 28.7 samples/s +Step 13800/44000 | Loss: 3.0993 | LR: 3.96e-04 | 700.4ms/step | 28.6 samples/s +Step 13810/44000 | Loss: 3.1192 | LR: 3.96e-04 | 695.2ms/step | 28.8 samples/s +Step 13820/44000 | Loss: 3.0993 | LR: 3.96e-04 | 699.5ms/step | 28.6 samples/s +Step 13830/44000 | Loss: 3.1230 | LR: 3.95e-04 | 700.1ms/step | 28.6 samples/s +Step 13840/44000 | Loss: 3.1140 | LR: 3.95e-04 | 699.5ms/step | 28.6 samples/s +Step 13850/44000 | Loss: 3.0936 | LR: 3.95e-04 | 697.8ms/step | 28.7 samples/s +Step 13860/44000 | Loss: 3.0937 | LR: 3.95e-04 | 701.3ms/step | 28.5 samples/s +Step 13870/44000 | Loss: 3.0837 | LR: 3.95e-04 | 699.7ms/step | 28.6 samples/s +Step 13880/44000 | Loss: 3.1171 | LR: 3.95e-04 | 701.3ms/step | 28.5 samples/s +Step 13890/44000 | Loss: 3.1161 | LR: 3.95e-04 | 694.9ms/step | 28.8 samples/s +Step 13900/44000 | Loss: 3.0000 | LR: 3.94e-04 | 699.1ms/step | 28.6 samples/s +Step 13910/44000 | Loss: 3.0813 | LR: 3.94e-04 | 697.8ms/step | 28.7 samples/s +Step 13920/44000 | Loss: 3.0830 | LR: 3.94e-04 | 696.8ms/step | 28.7 samples/s +Step 13930/44000 | Loss: 3.0458 | LR: 3.94e-04 | 699.7ms/step | 28.6 samples/s +Step 13940/44000 | Loss: 3.0861 | LR: 3.94e-04 | 698.1ms/step | 28.6 samples/s +Step 13950/44000 | Loss: 3.0801 | LR: 3.94e-04 | 699.6ms/step | 28.6 samples/s +Step 13960/44000 | Loss: 3.0937 | LR: 3.93e-04 | 698.0ms/step | 28.7 samples/s +Step 13970/44000 | Loss: 3.1028 | LR: 3.93e-04 | 699.5ms/step | 28.6 samples/s +Step 13980/44000 | Loss: 3.0926 | LR: 3.93e-04 | 699.4ms/step | 28.6 samples/s +Step 13990/44000 | Loss: 3.1273 | LR: 3.93e-04 | 696.4ms/step | 28.7 samples/s +Step 14000/44000 | Loss: 3.0892 | LR: 3.93e-04 | 697.4ms/step | 28.7 samples/s +Validation at step 14000: Loss = 3.1173, Perplexity = 22.59 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-14000 +Step 14010/44000 | Loss: 3.1042 | LR: 3.93e-04 | 1597.2ms/step | 12.5 samples/s +Step 14020/44000 | Loss: 3.0244 | LR: 3.93e-04 | 697.9ms/step | 28.7 samples/s +Step 14030/44000 | Loss: 3.1054 | LR: 3.92e-04 | 696.5ms/step | 28.7 samples/s +Step 14040/44000 | Loss: 3.1494 | LR: 3.92e-04 | 700.3ms/step | 28.6 samples/s +Step 14050/44000 | Loss: 3.0707 | LR: 3.92e-04 | 697.2ms/step | 28.7 samples/s +Step 14060/44000 | Loss: 3.1145 | LR: 3.92e-04 | 698.5ms/step | 28.6 samples/s +Step 14070/44000 | Loss: 3.0756 | LR: 3.92e-04 | 700.0ms/step | 28.6 samples/s +Step 14080/44000 | Loss: 3.0994 | LR: 3.92e-04 | 699.9ms/step | 28.6 samples/s +Step 14090/44000 | Loss: 3.0983 | LR: 3.92e-04 | 699.3ms/step | 28.6 samples/s +Step 14100/44000 | Loss: 3.0925 | LR: 3.91e-04 | 701.0ms/step | 28.5 samples/s +Step 14110/44000 | Loss: 3.1121 | LR: 3.91e-04 | 697.7ms/step | 28.7 samples/s +Step 14120/44000 | Loss: 3.0823 | LR: 3.91e-04 | 699.9ms/step | 28.6 samples/s +Step 14130/44000 | Loss: 3.0978 | LR: 3.91e-04 | 697.9ms/step | 28.7 samples/s +Step 14140/44000 | Loss: 3.1227 | LR: 3.91e-04 | 700.5ms/step | 28.6 samples/s +Step 14150/44000 | Loss: 3.0729 | LR: 3.91e-04 | 697.4ms/step | 28.7 samples/s +Step 14160/44000 | Loss: 3.0895 | LR: 3.90e-04 | 695.1ms/step | 28.8 samples/s +Step 14170/44000 | Loss: 3.0489 | LR: 3.90e-04 | 698.4ms/step | 28.6 samples/s +Step 14180/44000 | Loss: 3.0462 | LR: 3.90e-04 | 700.4ms/step | 28.6 samples/s +Step 14190/44000 | Loss: 3.1137 | LR: 3.90e-04 | 701.5ms/step | 28.5 samples/s +Step 14200/44000 | Loss: 3.0473 | LR: 3.90e-04 | 702.4ms/step | 28.5 samples/s +Step 14210/44000 | Loss: 3.0903 | LR: 3.90e-04 | 699.1ms/step | 28.6 samples/s +Step 14220/44000 | Loss: 3.0653 | LR: 3.90e-04 | 711.8ms/step | 28.1 samples/s +Step 14230/44000 | Loss: 3.0760 | LR: 3.89e-04 | 703.4ms/step | 28.4 samples/s +Step 14240/44000 | Loss: 3.1011 | LR: 3.89e-04 | 700.3ms/step | 28.6 samples/s +Step 14250/44000 | Loss: 3.0796 | LR: 3.89e-04 | 700.0ms/step | 28.6 samples/s +Validation at step 14250: Loss = 3.1115, Perplexity = 22.46 +Step 14260/44000 | Loss: 3.0524 | LR: 3.89e-04 | 1271.1ms/step | 15.7 samples/s +Step 14270/44000 | Loss: 3.0689 | LR: 3.89e-04 | 699.9ms/step | 28.6 samples/s +Step 14280/44000 | Loss: 3.0446 | LR: 3.89e-04 | 698.5ms/step | 28.6 samples/s +Step 14290/44000 | Loss: 3.1297 | LR: 3.89e-04 | 700.0ms/step | 28.6 samples/s +Step 14300/44000 | Loss: 3.1248 | LR: 3.88e-04 | 700.1ms/step | 28.6 samples/s +Step 14310/44000 | Loss: 3.0632 | LR: 3.88e-04 | 700.8ms/step | 28.5 samples/s +Step 14320/44000 | Loss: 3.0548 | LR: 3.88e-04 | 700.8ms/step | 28.5 samples/s +Step 14330/44000 | Loss: 3.0650 | LR: 3.88e-04 | 699.3ms/step | 28.6 samples/s +Step 14340/44000 | Loss: 3.0994 | LR: 3.88e-04 | 699.1ms/step | 28.6 samples/s +Step 14350/44000 | Loss: 3.0984 | LR: 3.88e-04 | 697.8ms/step | 28.7 samples/s +Step 14360/44000 | Loss: 3.1221 | LR: 3.87e-04 | 701.3ms/step | 28.5 samples/s +Step 14370/44000 | Loss: 3.1097 | LR: 3.87e-04 | 698.7ms/step | 28.6 samples/s +Step 14380/44000 | Loss: 3.0702 | LR: 3.87e-04 | 699.3ms/step | 28.6 samples/s +Step 14390/44000 | Loss: 3.0487 | LR: 3.87e-04 | 700.7ms/step | 28.5 samples/s +Step 14400/44000 | Loss: 3.1034 | LR: 3.87e-04 | 702.1ms/step | 28.5 samples/s +Step 14410/44000 | Loss: 3.0496 | LR: 3.87e-04 | 702.0ms/step | 28.5 samples/s +Step 14420/44000 | Loss: 3.0614 | LR: 3.87e-04 | 702.5ms/step | 28.5 samples/s +Step 14430/44000 | Loss: 3.0547 | LR: 3.86e-04 | 706.3ms/step | 28.3 samples/s +Step 14440/44000 | Loss: 3.0670 | LR: 3.86e-04 | 703.1ms/step | 28.4 samples/s +Step 14450/44000 | Loss: 3.0764 | LR: 3.86e-04 | 700.4ms/step | 28.6 samples/s +Step 14460/44000 | Loss: 3.1038 | LR: 3.86e-04 | 707.0ms/step | 28.3 samples/s +Step 14470/44000 | Loss: 3.0743 | LR: 3.86e-04 | 704.2ms/step | 28.4 samples/s +Step 14480/44000 | Loss: 3.0676 | LR: 3.86e-04 | 706.9ms/step | 28.3 samples/s +Step 14490/44000 | Loss: 3.0596 | LR: 3.85e-04 | 704.3ms/step | 28.4 samples/s +Step 14500/44000 | Loss: 3.0730 | LR: 3.85e-04 | 702.1ms/step | 28.5 samples/s +Validation at step 14500: Loss = 3.1066, Perplexity = 22.35 +Step 14510/44000 | Loss: 3.0854 | LR: 3.85e-04 | 1287.3ms/step | 15.5 samples/s +Step 14520/44000 | Loss: 3.0560 | LR: 3.85e-04 | 703.5ms/step | 28.4 samples/s +Step 14530/44000 | Loss: 3.1070 | LR: 3.85e-04 | 703.1ms/step | 28.4 samples/s +Step 14540/44000 | Loss: 3.0765 | LR: 3.85e-04 | 702.6ms/step | 28.5 samples/s +Step 14550/44000 | Loss: 3.0367 | LR: 3.85e-04 | 703.1ms/step | 28.4 samples/s +Step 14560/44000 | Loss: 3.1137 | LR: 3.84e-04 | 699.8ms/step | 28.6 samples/s +Step 14570/44000 | Loss: 3.0517 | LR: 3.84e-04 | 701.4ms/step | 28.5 samples/s +Step 14580/44000 | Loss: 3.0909 | LR: 3.84e-04 | 700.8ms/step | 28.5 samples/s +Step 14590/44000 | Loss: 3.0868 | LR: 3.84e-04 | 703.8ms/step | 28.4 samples/s +Step 14600/44000 | Loss: 3.0759 | LR: 3.84e-04 | 700.6ms/step | 28.5 samples/s +Step 14610/44000 | Loss: 3.0894 | LR: 3.84e-04 | 699.0ms/step | 28.6 samples/s +Step 14620/44000 | Loss: 3.0886 | LR: 3.83e-04 | 700.3ms/step | 28.6 samples/s +Step 14630/44000 | Loss: 3.0944 | LR: 3.83e-04 | 701.9ms/step | 28.5 samples/s +Step 14640/44000 | Loss: 3.0883 | LR: 3.83e-04 | 701.2ms/step | 28.5 samples/s +Step 14650/44000 | Loss: 3.0783 | LR: 3.83e-04 | 700.0ms/step | 28.6 samples/s +Step 14660/44000 | Loss: 3.0688 | LR: 3.83e-04 | 700.4ms/step | 28.6 samples/s +Step 14670/44000 | Loss: 3.0801 | LR: 3.83e-04 | 699.8ms/step | 28.6 samples/s +Step 14680/44000 | Loss: 3.0792 | LR: 3.83e-04 | 700.3ms/step | 28.6 samples/s +Step 14690/44000 | Loss: 3.1251 | LR: 3.82e-04 | 703.1ms/step | 28.4 samples/s +Step 14700/44000 | Loss: 3.0747 | LR: 3.82e-04 | 701.7ms/step | 28.5 samples/s +Step 14710/44000 | Loss: 3.0608 | LR: 3.82e-04 | 699.3ms/step | 28.6 samples/s +Step 14720/44000 | Loss: 3.0742 | LR: 3.82e-04 | 705.4ms/step | 28.4 samples/s +Step 14730/44000 | Loss: 3.0621 | LR: 3.82e-04 | 700.3ms/step | 28.6 samples/s +Step 14740/44000 | Loss: 3.0963 | LR: 3.82e-04 | 703.8ms/step | 28.4 samples/s +Step 14750/44000 | Loss: 3.0530 | LR: 3.81e-04 | 706.3ms/step | 28.3 samples/s +Validation at step 14750: Loss = 3.1036, Perplexity = 22.28 +Step 14760/44000 | Loss: 3.0864 | LR: 3.81e-04 | 1288.1ms/step | 15.5 samples/s +Step 14770/44000 | Loss: 3.1207 | LR: 3.81e-04 | 703.7ms/step | 28.4 samples/s +Step 14780/44000 | Loss: 3.0441 | LR: 3.81e-04 | 701.8ms/step | 28.5 samples/s +Step 14790/44000 | Loss: 3.0982 | LR: 3.81e-04 | 702.9ms/step | 28.5 samples/s +Step 14800/44000 | Loss: 3.1202 | LR: 3.81e-04 | 706.0ms/step | 28.3 samples/s +Step 14810/44000 | Loss: 3.0643 | LR: 3.81e-04 | 707.7ms/step | 28.3 samples/s +Step 14820/44000 | Loss: 3.1373 | LR: 3.80e-04 | 708.2ms/step | 28.2 samples/s +Step 14830/44000 | Loss: 3.0962 | LR: 3.80e-04 | 705.9ms/step | 28.3 samples/s +Step 14840/44000 | Loss: 3.1399 | LR: 3.80e-04 | 701.2ms/step | 28.5 samples/s +Step 14850/44000 | Loss: 3.0595 | LR: 3.80e-04 | 701.8ms/step | 28.5 samples/s +Step 14860/44000 | Loss: 3.0655 | LR: 3.80e-04 | 702.5ms/step | 28.5 samples/s +Step 14870/44000 | Loss: 3.0925 | LR: 3.80e-04 | 701.6ms/step | 28.5 samples/s +Step 14880/44000 | Loss: 3.0704 | LR: 3.79e-04 | 701.0ms/step | 28.5 samples/s +Step 14890/44000 | Loss: 3.1002 | LR: 3.79e-04 | 699.4ms/step | 28.6 samples/s +Step 14900/44000 | Loss: 3.0592 | LR: 3.79e-04 | 698.3ms/step | 28.6 samples/s +Step 14910/44000 | Loss: 3.0835 | LR: 3.79e-04 | 704.7ms/step | 28.4 samples/s +Step 14920/44000 | Loss: 3.0700 | LR: 3.79e-04 | 708.1ms/step | 28.2 samples/s +Step 14930/44000 | Loss: 3.0926 | LR: 3.79e-04 | 703.3ms/step | 28.4 samples/s +Step 14940/44000 | Loss: 3.0767 | LR: 3.79e-04 | 699.4ms/step | 28.6 samples/s +Step 14950/44000 | Loss: 3.0703 | LR: 3.78e-04 | 706.8ms/step | 28.3 samples/s +Step 14960/44000 | Loss: 3.0463 | LR: 3.78e-04 | 700.3ms/step | 28.6 samples/s +Step 14970/44000 | Loss: 3.0608 | LR: 3.78e-04 | 703.8ms/step | 28.4 samples/s +Step 14980/44000 | Loss: 3.0471 | LR: 3.78e-04 | 705.3ms/step | 28.4 samples/s +Step 14990/44000 | Loss: 3.0618 | LR: 3.78e-04 | 704.2ms/step | 28.4 samples/s +Step 15000/44000 | Loss: 3.0871 | LR: 3.78e-04 | 701.8ms/step | 28.5 samples/s +Validation at step 15000: Loss = 3.1013, Perplexity = 22.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-15000 +Step 15010/44000 | Loss: 3.0612 | LR: 3.77e-04 | 1588.2ms/step | 12.6 samples/s +Step 15020/44000 | Loss: 3.1045 | LR: 3.77e-04 | 698.3ms/step | 28.6 samples/s +Step 15030/44000 | Loss: 3.0748 | LR: 3.77e-04 | 701.7ms/step | 28.5 samples/s +Step 15040/44000 | Loss: 3.1178 | LR: 3.77e-04 | 700.1ms/step | 28.6 samples/s +Step 15050/44000 | Loss: 3.0455 | LR: 3.77e-04 | 697.9ms/step | 28.7 samples/s +Step 15060/44000 | Loss: 3.0480 | LR: 3.77e-04 | 700.0ms/step | 28.6 samples/s +Step 15070/44000 | Loss: 3.0856 | LR: 3.77e-04 | 700.3ms/step | 28.6 samples/s +Step 15080/44000 | Loss: 3.0797 | LR: 3.76e-04 | 699.8ms/step | 28.6 samples/s +Step 15090/44000 | Loss: 3.0735 | LR: 3.76e-04 | 700.7ms/step | 28.5 samples/s +Step 15100/44000 | Loss: 3.0654 | LR: 3.76e-04 | 701.2ms/step | 28.5 samples/s +Step 15110/44000 | Loss: 3.0370 | LR: 3.76e-04 | 699.9ms/step | 28.6 samples/s +Step 15120/44000 | Loss: 3.1008 | LR: 3.76e-04 | 702.2ms/step | 28.5 samples/s +Step 15130/44000 | Loss: 3.0862 | LR: 3.76e-04 | 700.1ms/step | 28.6 samples/s +Step 15140/44000 | Loss: 3.0996 | LR: 3.75e-04 | 702.3ms/step | 28.5 samples/s +Step 15150/44000 | Loss: 3.0925 | LR: 3.75e-04 | 704.0ms/step | 28.4 samples/s +Step 15160/44000 | Loss: 3.0723 | LR: 3.75e-04 | 706.6ms/step | 28.3 samples/s +Step 15170/44000 | Loss: 3.0667 | LR: 3.75e-04 | 708.6ms/step | 28.2 samples/s +Step 15180/44000 | Loss: 3.0286 | LR: 3.75e-04 | 706.3ms/step | 28.3 samples/s +Step 15190/44000 | Loss: 3.0752 | LR: 3.75e-04 | 705.8ms/step | 28.3 samples/s +Step 15200/44000 | Loss: 3.1144 | LR: 3.74e-04 | 707.2ms/step | 28.3 samples/s +Step 15210/44000 | Loss: 3.0371 | LR: 3.74e-04 | 706.4ms/step | 28.3 samples/s +Step 15220/44000 | Loss: 3.0688 | LR: 3.74e-04 | 706.6ms/step | 28.3 samples/s +Step 15230/44000 | Loss: 3.0850 | LR: 3.74e-04 | 708.7ms/step | 28.2 samples/s +Step 15240/44000 | Loss: 3.0444 | LR: 3.74e-04 | 704.8ms/step | 28.4 samples/s +Step 15250/44000 | Loss: 3.0665 | LR: 3.74e-04 | 706.6ms/step | 28.3 samples/s +Validation at step 15250: Loss = 3.0992, Perplexity = 22.18 +Step 15260/44000 | Loss: 3.0992 | LR: 3.74e-04 | 1297.9ms/step | 15.4 samples/s +Step 15270/44000 | Loss: 3.0849 | LR: 3.73e-04 | 704.6ms/step | 28.4 samples/s +Step 15280/44000 | Loss: 3.0740 | LR: 3.73e-04 | 704.7ms/step | 28.4 samples/s +Step 15290/44000 | Loss: 3.0677 | LR: 3.73e-04 | 704.9ms/step | 28.4 samples/s +Step 15300/44000 | Loss: 3.0669 | LR: 3.73e-04 | 704.1ms/step | 28.4 samples/s +Step 15310/44000 | Loss: 3.0749 | LR: 3.73e-04 | 701.9ms/step | 28.5 samples/s +Step 15320/44000 | Loss: 3.1020 | LR: 3.73e-04 | 705.1ms/step | 28.4 samples/s +Step 15330/44000 | Loss: 3.0405 | LR: 3.72e-04 | 703.1ms/step | 28.4 samples/s +Step 15340/44000 | Loss: 3.0895 | LR: 3.72e-04 | 704.8ms/step | 28.4 samples/s +Step 15350/44000 | Loss: 3.0509 | LR: 3.72e-04 | 702.5ms/step | 28.5 samples/s +Step 15360/44000 | Loss: 3.0880 | LR: 3.72e-04 | 703.6ms/step | 28.4 samples/s +Step 15370/44000 | Loss: 3.0796 | LR: 3.72e-04 | 704.8ms/step | 28.4 samples/s +Step 15380/44000 | Loss: 3.1110 | LR: 3.72e-04 | 704.5ms/step | 28.4 samples/s +Step 15390/44000 | Loss: 3.0691 | LR: 3.71e-04 | 705.5ms/step | 28.3 samples/s +Step 15400/44000 | Loss: 3.0707 | LR: 3.71e-04 | 707.8ms/step | 28.3 samples/s +Step 15410/44000 | Loss: 3.0967 | LR: 3.71e-04 | 703.3ms/step | 28.4 samples/s +Step 15420/44000 | Loss: 3.0781 | LR: 3.71e-04 | 702.6ms/step | 28.5 samples/s +Step 15430/44000 | Loss: 3.0749 | LR: 3.71e-04 | 706.2ms/step | 28.3 samples/s +Step 15440/44000 | Loss: 3.0613 | LR: 3.71e-04 | 703.7ms/step | 28.4 samples/s +Step 15450/44000 | Loss: 3.0672 | LR: 3.71e-04 | 707.1ms/step | 28.3 samples/s +Step 15460/44000 | Loss: 3.0765 | LR: 3.70e-04 | 709.2ms/step | 28.2 samples/s +Step 15470/44000 | Loss: 3.0773 | LR: 3.70e-04 | 708.0ms/step | 28.2 samples/s +Step 15480/44000 | Loss: 3.0723 | LR: 3.70e-04 | 707.5ms/step | 28.3 samples/s +Step 15490/44000 | Loss: 3.0695 | LR: 3.70e-04 | 707.6ms/step | 28.3 samples/s +Step 15500/44000 | Loss: 3.0832 | LR: 3.70e-04 | 705.2ms/step | 28.4 samples/s +Validation at step 15500: Loss = 3.0951, Perplexity = 22.09 +Step 15510/44000 | Loss: 3.1038 | LR: 3.70e-04 | 1291.3ms/step | 15.5 samples/s +Step 15520/44000 | Loss: 3.0993 | LR: 3.69e-04 | 706.2ms/step | 28.3 samples/s +Step 15530/44000 | Loss: 3.0690 | LR: 3.69e-04 | 705.7ms/step | 28.3 samples/s +Step 15540/44000 | Loss: 3.1028 | LR: 3.69e-04 | 708.0ms/step | 28.2 samples/s +Step 15550/44000 | Loss: 3.0351 | LR: 3.69e-04 | 707.8ms/step | 28.3 samples/s +Step 15560/44000 | Loss: 3.1052 | LR: 3.69e-04 | 706.5ms/step | 28.3 samples/s +Step 15570/44000 | Loss: 3.0787 | LR: 3.69e-04 | 704.2ms/step | 28.4 samples/s +Step 15580/44000 | Loss: 3.1034 | LR: 3.68e-04 | 708.5ms/step | 28.2 samples/s +Step 15590/44000 | Loss: 3.0454 | LR: 3.68e-04 | 700.5ms/step | 28.6 samples/s +Step 15600/44000 | Loss: 3.0976 | LR: 3.68e-04 | 704.7ms/step | 28.4 samples/s +Step 15610/44000 | Loss: 3.0376 | LR: 3.68e-04 | 705.1ms/step | 28.4 samples/s +Step 15620/44000 | Loss: 3.0801 | LR: 3.68e-04 | 705.3ms/step | 28.4 samples/s +Step 15630/44000 | Loss: 3.0285 | LR: 3.68e-04 | 703.0ms/step | 28.4 samples/s +Step 15640/44000 | Loss: 3.0423 | LR: 3.67e-04 | 703.8ms/step | 28.4 samples/s +Step 15650/44000 | Loss: 3.0737 | LR: 3.67e-04 | 701.3ms/step | 28.5 samples/s +Step 15660/44000 | Loss: 3.0427 | LR: 3.67e-04 | 701.1ms/step | 28.5 samples/s +Step 15670/44000 | Loss: 3.0848 | LR: 3.67e-04 | 707.4ms/step | 28.3 samples/s +Step 15680/44000 | Loss: 3.0459 | LR: 3.67e-04 | 707.2ms/step | 28.3 samples/s +Step 15690/44000 | Loss: 3.0568 | LR: 3.67e-04 | 707.8ms/step | 28.3 samples/s +Step 15700/44000 | Loss: 3.0605 | LR: 3.67e-04 | 710.3ms/step | 28.2 samples/s +Step 15710/44000 | Loss: 3.0427 | LR: 3.66e-04 | 710.2ms/step | 28.2 samples/s +Step 15720/44000 | Loss: 3.0454 | LR: 3.66e-04 | 708.3ms/step | 28.2 samples/s +Step 15730/44000 | Loss: 3.0306 | LR: 3.66e-04 | 703.5ms/step | 28.4 samples/s +Step 15740/44000 | Loss: 3.0814 | LR: 3.66e-04 | 701.5ms/step | 28.5 samples/s +Step 15750/44000 | Loss: 3.0373 | LR: 3.66e-04 | 704.2ms/step | 28.4 samples/s +Validation at step 15750: Loss = 3.0904, Perplexity = 21.99 +Step 15760/44000 | Loss: 3.0562 | LR: 3.66e-04 | 1287.0ms/step | 15.5 samples/s +Step 15770/44000 | Loss: 3.0693 | LR: 3.65e-04 | 708.2ms/step | 28.2 samples/s +Step 15780/44000 | Loss: 3.1104 | LR: 3.65e-04 | 703.8ms/step | 28.4 samples/s +Step 15790/44000 | Loss: 3.0707 | LR: 3.65e-04 | 706.8ms/step | 28.3 samples/s +Step 15800/44000 | Loss: 3.0252 | LR: 3.65e-04 | 704.9ms/step | 28.4 samples/s +Step 15810/44000 | Loss: 3.0517 | LR: 3.65e-04 | 707.0ms/step | 28.3 samples/s +Step 15820/44000 | Loss: 3.0590 | LR: 3.65e-04 | 705.3ms/step | 28.4 samples/s +Step 15830/44000 | Loss: 3.0722 | LR: 3.64e-04 | 702.6ms/step | 28.5 samples/s +Step 15840/44000 | Loss: 3.1012 | LR: 3.64e-04 | 705.2ms/step | 28.4 samples/s +Step 15850/44000 | Loss: 3.0208 | LR: 3.64e-04 | 705.1ms/step | 28.4 samples/s +Step 15860/44000 | Loss: 3.0382 | LR: 3.64e-04 | 703.5ms/step | 28.4 samples/s +Step 15870/44000 | Loss: 3.0617 | LR: 3.64e-04 | 702.6ms/step | 28.5 samples/s +Step 15880/44000 | Loss: 3.1103 | LR: 3.64e-04 | 705.8ms/step | 28.3 samples/s +Step 15890/44000 | Loss: 3.0332 | LR: 3.63e-04 | 703.7ms/step | 28.4 samples/s +Step 15900/44000 | Loss: 3.0584 | LR: 3.63e-04 | 703.4ms/step | 28.4 samples/s +Step 15910/44000 | Loss: 3.0547 | LR: 3.63e-04 | 703.1ms/step | 28.4 samples/s +Step 15920/44000 | Loss: 3.0441 | LR: 3.63e-04 | 704.8ms/step | 28.4 samples/s +Step 15930/44000 | Loss: 3.0117 | LR: 3.63e-04 | 702.4ms/step | 28.5 samples/s +Step 15940/44000 | Loss: 3.0415 | LR: 3.63e-04 | 708.8ms/step | 28.2 samples/s +Step 15950/44000 | Loss: 3.0792 | LR: 3.62e-04 | 707.3ms/step | 28.3 samples/s +Step 15960/44000 | Loss: 3.0838 | LR: 3.62e-04 | 704.8ms/step | 28.4 samples/s +Step 15970/44000 | Loss: 3.0135 | LR: 3.62e-04 | 705.8ms/step | 28.3 samples/s +Step 15980/44000 | Loss: 3.1073 | LR: 3.62e-04 | 708.4ms/step | 28.2 samples/s +Step 15990/44000 | Loss: 3.0917 | LR: 3.62e-04 | 706.4ms/step | 28.3 samples/s +Step 16000/44000 | Loss: 3.1118 | LR: 3.62e-04 | 709.1ms/step | 28.2 samples/s +Validation at step 16000: Loss = 3.0895, Perplexity = 21.97 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-16000 +Step 16010/44000 | Loss: 3.0782 | LR: 3.62e-04 | 1617.8ms/step | 12.4 samples/s +Step 16020/44000 | Loss: 3.0632 | LR: 3.61e-04 | 706.6ms/step | 28.3 samples/s +Step 16030/44000 | Loss: 3.1022 | LR: 3.61e-04 | 703.6ms/step | 28.4 samples/s +Step 16040/44000 | Loss: 3.0433 | LR: 3.61e-04 | 704.2ms/step | 28.4 samples/s +Step 16050/44000 | Loss: 3.0549 | LR: 3.61e-04 | 702.1ms/step | 28.5 samples/s +Step 16060/44000 | Loss: 3.0734 | LR: 3.61e-04 | 699.7ms/step | 28.6 samples/s +Step 16070/44000 | Loss: 3.0673 | LR: 3.61e-04 | 703.9ms/step | 28.4 samples/s +Step 16080/44000 | Loss: 3.0764 | LR: 3.60e-04 | 701.8ms/step | 28.5 samples/s +Step 16090/44000 | Loss: 3.1005 | LR: 3.60e-04 | 703.8ms/step | 28.4 samples/s +Step 16100/44000 | Loss: 3.0752 | LR: 3.60e-04 | 705.7ms/step | 28.3 samples/s +Step 16110/44000 | Loss: 3.0426 | LR: 3.60e-04 | 705.0ms/step | 28.4 samples/s +Step 16120/44000 | Loss: 3.0641 | LR: 3.60e-04 | 704.5ms/step | 28.4 samples/s +Step 16130/44000 | Loss: 3.0303 | LR: 3.60e-04 | 705.8ms/step | 28.3 samples/s +Step 16140/44000 | Loss: 3.0724 | LR: 3.59e-04 | 703.6ms/step | 28.4 samples/s +Step 16150/44000 | Loss: 3.0394 | LR: 3.59e-04 | 703.7ms/step | 28.4 samples/s +Step 16160/44000 | Loss: 3.0446 | LR: 3.59e-04 | 702.7ms/step | 28.5 samples/s +Step 16170/44000 | Loss: 3.0337 | LR: 3.59e-04 | 703.9ms/step | 28.4 samples/s +Step 16180/44000 | Loss: 3.0546 | LR: 3.59e-04 | 702.6ms/step | 28.5 samples/s +Step 16190/44000 | Loss: 3.0558 | LR: 3.59e-04 | 705.1ms/step | 28.4 samples/s +Step 16200/44000 | Loss: 3.1054 | LR: 3.58e-04 | 703.9ms/step | 28.4 samples/s +Step 16210/44000 | Loss: 3.0587 | LR: 3.58e-04 | 702.2ms/step | 28.5 samples/s +Step 16220/44000 | Loss: 3.0853 | LR: 3.58e-04 | 700.8ms/step | 28.5 samples/s +Step 16230/44000 | Loss: 3.0307 | LR: 3.58e-04 | 700.4ms/step | 28.6 samples/s +Step 16240/44000 | Loss: 3.0779 | LR: 3.58e-04 | 702.8ms/step | 28.5 samples/s +Step 16250/44000 | Loss: 3.0256 | LR: 3.58e-04 | 701.7ms/step | 28.5 samples/s +Validation at step 16250: Loss = 3.0838, Perplexity = 21.84 +Step 16260/44000 | Loss: 3.0903 | LR: 3.57e-04 | 1286.0ms/step | 15.6 samples/s +Step 16270/44000 | Loss: 3.0891 | LR: 3.57e-04 | 704.1ms/step | 28.4 samples/s +Step 16280/44000 | Loss: 3.0363 | LR: 3.57e-04 | 701.4ms/step | 28.5 samples/s +Step 16290/44000 | Loss: 3.0362 | LR: 3.57e-04 | 702.4ms/step | 28.5 samples/s +Step 16300/44000 | Loss: 3.0287 | LR: 3.57e-04 | 702.3ms/step | 28.5 samples/s +Step 16310/44000 | Loss: 3.0449 | LR: 3.57e-04 | 706.2ms/step | 28.3 samples/s +Step 16320/44000 | Loss: 3.0483 | LR: 3.56e-04 | 701.5ms/step | 28.5 samples/s +Step 16330/44000 | Loss: 3.0393 | LR: 3.56e-04 | 698.2ms/step | 28.6 samples/s +Step 16340/44000 | Loss: 3.0435 | LR: 3.56e-04 | 702.3ms/step | 28.5 samples/s +Step 16350/44000 | Loss: 3.0737 | LR: 3.56e-04 | 704.4ms/step | 28.4 samples/s +Step 16360/44000 | Loss: 3.0533 | LR: 3.56e-04 | 703.2ms/step | 28.4 samples/s +Step 16370/44000 | Loss: 3.0887 | LR: 3.56e-04 | 703.9ms/step | 28.4 samples/s +Step 16380/44000 | Loss: 3.0251 | LR: 3.55e-04 | 702.0ms/step | 28.5 samples/s +Step 16390/44000 | Loss: 3.0865 | LR: 3.55e-04 | 700.8ms/step | 28.5 samples/s +Step 16400/44000 | Loss: 3.0697 | LR: 3.55e-04 | 705.3ms/step | 28.4 samples/s +Step 16410/44000 | Loss: 3.0506 | LR: 3.55e-04 | 705.3ms/step | 28.4 samples/s +Step 16420/44000 | Loss: 3.1010 | LR: 3.55e-04 | 704.6ms/step | 28.4 samples/s +Step 16430/44000 | Loss: 3.0139 | LR: 3.55e-04 | 704.0ms/step | 28.4 samples/s +Step 16440/44000 | Loss: 3.0684 | LR: 3.54e-04 | 701.8ms/step | 28.5 samples/s +Step 16450/44000 | Loss: 3.0244 | LR: 3.54e-04 | 701.0ms/step | 28.5 samples/s +Step 16460/44000 | Loss: 3.0189 | LR: 3.54e-04 | 701.9ms/step | 28.5 samples/s +Step 16470/44000 | Loss: 3.0391 | LR: 3.54e-04 | 703.1ms/step | 28.4 samples/s +Step 16480/44000 | Loss: 3.0861 | LR: 3.54e-04 | 701.4ms/step | 28.5 samples/s +Step 16490/44000 | Loss: 3.0091 | LR: 3.54e-04 | 701.4ms/step | 28.5 samples/s +Step 16500/44000 | Loss: 2.9995 | LR: 3.53e-04 | 699.5ms/step | 28.6 samples/s +Validation at step 16500: Loss = 3.0805, Perplexity = 21.77 +Step 16510/44000 | Loss: 3.0345 | LR: 3.53e-04 | 1290.0ms/step | 15.5 samples/s +Step 16520/44000 | Loss: 3.0942 | LR: 3.53e-04 | 701.5ms/step | 28.5 samples/s +Step 16530/44000 | Loss: 3.0575 | LR: 3.53e-04 | 703.2ms/step | 28.4 samples/s +Step 16540/44000 | Loss: 3.0301 | LR: 3.53e-04 | 700.8ms/step | 28.5 samples/s +Step 16550/44000 | Loss: 3.0885 | LR: 3.53e-04 | 700.7ms/step | 28.5 samples/s +Step 16560/44000 | Loss: 3.0668 | LR: 3.52e-04 | 701.3ms/step | 28.5 samples/s +Step 16570/44000 | Loss: 3.0416 | LR: 3.52e-04 | 701.0ms/step | 28.5 samples/s +Step 16580/44000 | Loss: 3.0719 | LR: 3.52e-04 | 700.6ms/step | 28.5 samples/s +Step 16590/44000 | Loss: 3.0521 | LR: 3.52e-04 | 701.4ms/step | 28.5 samples/s +Step 16600/44000 | Loss: 3.0493 | LR: 3.52e-04 | 698.1ms/step | 28.6 samples/s +Step 16610/44000 | Loss: 3.0589 | LR: 3.52e-04 | 701.1ms/step | 28.5 samples/s +Step 16620/44000 | Loss: 3.0536 | LR: 3.51e-04 | 700.8ms/step | 28.5 samples/s +Step 16630/44000 | Loss: 3.0696 | LR: 3.51e-04 | 701.6ms/step | 28.5 samples/s +Step 16640/44000 | Loss: 3.0630 | LR: 3.51e-04 | 702.3ms/step | 28.5 samples/s +Step 16650/44000 | Loss: 3.0832 | LR: 3.51e-04 | 702.9ms/step | 28.5 samples/s +Step 16660/44000 | Loss: 3.0299 | LR: 3.51e-04 | 702.8ms/step | 28.5 samples/s +Step 16670/44000 | Loss: 3.0393 | LR: 3.51e-04 | 702.0ms/step | 28.5 samples/s +Step 16680/44000 | Loss: 3.0689 | LR: 3.50e-04 | 701.6ms/step | 28.5 samples/s +Step 16690/44000 | Loss: 3.0885 | LR: 3.50e-04 | 702.2ms/step | 28.5 samples/s +Step 16700/44000 | Loss: 3.0227 | LR: 3.50e-04 | 697.8ms/step | 28.7 samples/s +Step 16710/44000 | Loss: 3.0250 | LR: 3.50e-04 | 699.2ms/step | 28.6 samples/s +Step 16720/44000 | Loss: 3.0440 | LR: 3.50e-04 | 696.5ms/step | 28.7 samples/s +Step 16730/44000 | Loss: 3.0796 | LR: 3.50e-04 | 698.1ms/step | 28.6 samples/s +Step 16740/44000 | Loss: 3.0574 | LR: 3.49e-04 | 698.1ms/step | 28.6 samples/s +Step 16750/44000 | Loss: 3.0469 | LR: 3.49e-04 | 702.2ms/step | 28.5 samples/s +Validation at step 16750: Loss = 3.0776, Perplexity = 21.71 +Step 16760/44000 | Loss: 3.0525 | LR: 3.49e-04 | 1286.1ms/step | 15.6 samples/s +Step 16770/44000 | Loss: 3.0563 | LR: 3.49e-04 | 697.9ms/step | 28.7 samples/s +Step 16780/44000 | Loss: 3.1033 | LR: 3.49e-04 | 701.2ms/step | 28.5 samples/s +Step 16790/44000 | Loss: 3.0375 | LR: 3.49e-04 | 706.9ms/step | 28.3 samples/s +Step 16800/44000 | Loss: 3.0579 | LR: 3.48e-04 | 702.7ms/step | 28.5 samples/s +Step 16810/44000 | Loss: 3.0126 | LR: 3.48e-04 | 703.6ms/step | 28.4 samples/s +Step 16820/44000 | Loss: 3.0421 | LR: 3.48e-04 | 701.5ms/step | 28.5 samples/s +Step 16830/44000 | Loss: 3.0683 | LR: 3.48e-04 | 702.4ms/step | 28.5 samples/s +Step 16840/44000 | Loss: 3.0940 | LR: 3.48e-04 | 704.8ms/step | 28.4 samples/s +Step 16850/44000 | Loss: 3.0543 | LR: 3.48e-04 | 699.1ms/step | 28.6 samples/s +Step 16860/44000 | Loss: 3.0335 | LR: 3.47e-04 | 705.0ms/step | 28.4 samples/s +Step 16870/44000 | Loss: 3.0411 | LR: 3.47e-04 | 702.3ms/step | 28.5 samples/s +Step 16880/44000 | Loss: 3.0798 | LR: 3.47e-04 | 700.5ms/step | 28.6 samples/s +Step 16890/44000 | Loss: 3.0378 | LR: 3.47e-04 | 703.0ms/step | 28.4 samples/s +Step 16900/44000 | Loss: 3.0506 | LR: 3.47e-04 | 702.2ms/step | 28.5 samples/s +Step 16910/44000 | Loss: 3.0493 | LR: 3.47e-04 | 701.6ms/step | 28.5 samples/s +Step 16920/44000 | Loss: 3.0476 | LR: 3.46e-04 | 704.6ms/step | 28.4 samples/s +Step 16930/44000 | Loss: 3.0352 | LR: 3.46e-04 | 703.4ms/step | 28.4 samples/s +Step 16940/44000 | Loss: 3.0150 | LR: 3.46e-04 | 705.7ms/step | 28.3 samples/s +Step 16950/44000 | Loss: 3.0332 | LR: 3.46e-04 | 707.7ms/step | 28.3 samples/s +Step 16960/44000 | Loss: 3.0750 | LR: 3.46e-04 | 705.9ms/step | 28.3 samples/s +Step 16970/44000 | Loss: 3.0370 | LR: 3.46e-04 | 702.9ms/step | 28.5 samples/s +Step 16980/44000 | Loss: 3.0709 | LR: 3.45e-04 | 701.4ms/step | 28.5 samples/s +Step 16990/44000 | Loss: 3.0291 | LR: 3.45e-04 | 707.3ms/step | 28.3 samples/s +Step 17000/44000 | Loss: 3.0566 | LR: 3.45e-04 | 701.8ms/step | 28.5 samples/s +Validation at step 17000: Loss = 3.0756, Perplexity = 21.66 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-17000 +Step 17010/44000 | Loss: 3.0542 | LR: 3.45e-04 | 1582.5ms/step | 12.6 samples/s +Step 17020/44000 | Loss: 3.0609 | LR: 3.45e-04 | 703.8ms/step | 28.4 samples/s +Step 17030/44000 | Loss: 3.0555 | LR: 3.45e-04 | 700.1ms/step | 28.6 samples/s +Step 17040/44000 | Loss: 3.0906 | LR: 3.44e-04 | 706.1ms/step | 28.3 samples/s +Step 17050/44000 | Loss: 3.0813 | LR: 3.44e-04 | 706.5ms/step | 28.3 samples/s +Step 17060/44000 | Loss: 3.0517 | LR: 3.44e-04 | 704.7ms/step | 28.4 samples/s +Step 17070/44000 | Loss: 3.0228 | LR: 3.44e-04 | 702.5ms/step | 28.5 samples/s +Step 17080/44000 | Loss: 3.0364 | LR: 3.44e-04 | 703.2ms/step | 28.4 samples/s +Step 17090/44000 | Loss: 3.0479 | LR: 3.44e-04 | 702.6ms/step | 28.5 samples/s +Step 17100/44000 | Loss: 3.0379 | LR: 3.43e-04 | 701.7ms/step | 28.5 samples/s +Step 17110/44000 | Loss: 3.0597 | LR: 3.43e-04 | 702.2ms/step | 28.5 samples/s +Step 17120/44000 | Loss: 3.0040 | LR: 3.43e-04 | 702.2ms/step | 28.5 samples/s +Step 17130/44000 | Loss: 3.0660 | LR: 3.43e-04 | 703.3ms/step | 28.4 samples/s +Step 17140/44000 | Loss: 3.0512 | LR: 3.43e-04 | 705.1ms/step | 28.4 samples/s +Step 17150/44000 | Loss: 3.0231 | LR: 3.43e-04 | 698.8ms/step | 28.6 samples/s +Step 17160/44000 | Loss: 3.0318 | LR: 3.42e-04 | 700.1ms/step | 28.6 samples/s +Step 17170/44000 | Loss: 3.0516 | LR: 3.42e-04 | 696.6ms/step | 28.7 samples/s +Step 17180/44000 | Loss: 3.0169 | LR: 3.42e-04 | 701.1ms/step | 28.5 samples/s +Step 17190/44000 | Loss: 3.0425 | LR: 3.42e-04 | 701.3ms/step | 28.5 samples/s +Step 17200/44000 | Loss: 3.0724 | LR: 3.42e-04 | 699.0ms/step | 28.6 samples/s +Step 17210/44000 | Loss: 3.0381 | LR: 3.42e-04 | 700.7ms/step | 28.5 samples/s +Step 17220/44000 | Loss: 3.0666 | LR: 3.41e-04 | 700.5ms/step | 28.6 samples/s +Step 17230/44000 | Loss: 3.0638 | LR: 3.41e-04 | 701.6ms/step | 28.5 samples/s +Step 17240/44000 | Loss: 3.0930 | LR: 3.41e-04 | 698.3ms/step | 28.6 samples/s +Step 17250/44000 | Loss: 3.0575 | LR: 3.41e-04 | 700.3ms/step | 28.6 samples/s +Validation at step 17250: Loss = 3.0713, Perplexity = 21.57 +Step 17260/44000 | Loss: 3.0579 | LR: 3.41e-04 | 1287.8ms/step | 15.5 samples/s +Step 17270/44000 | Loss: 3.0166 | LR: 3.41e-04 | 709.9ms/step | 28.2 samples/s +Step 17280/44000 | Loss: 3.0404 | LR: 3.40e-04 | 705.6ms/step | 28.3 samples/s +Step 17290/44000 | Loss: 3.0650 | LR: 3.40e-04 | 703.4ms/step | 28.4 samples/s +Step 17300/44000 | Loss: 3.0649 | LR: 3.40e-04 | 706.2ms/step | 28.3 samples/s +Step 17310/44000 | Loss: 3.0270 | LR: 3.40e-04 | 701.4ms/step | 28.5 samples/s +Step 17320/44000 | Loss: 3.0754 | LR: 3.40e-04 | 702.8ms/step | 28.5 samples/s +Step 17330/44000 | Loss: 3.0823 | LR: 3.40e-04 | 704.2ms/step | 28.4 samples/s +Step 17340/44000 | Loss: 2.9937 | LR: 3.39e-04 | 699.6ms/step | 28.6 samples/s +Step 17350/44000 | Loss: 3.0973 | LR: 3.39e-04 | 701.9ms/step | 28.5 samples/s +Step 17360/44000 | Loss: 3.0323 | LR: 3.39e-04 | 703.1ms/step | 28.4 samples/s +Step 17370/44000 | Loss: 3.0070 | LR: 3.39e-04 | 702.8ms/step | 28.5 samples/s +Step 17380/44000 | Loss: 3.0580 | LR: 3.39e-04 | 704.5ms/step | 28.4 samples/s +Step 17390/44000 | Loss: 3.0736 | LR: 3.39e-04 | 701.4ms/step | 28.5 samples/s +Step 17400/44000 | Loss: 3.0604 | LR: 3.38e-04 | 703.7ms/step | 28.4 samples/s +Step 17410/44000 | Loss: 3.0454 | LR: 3.38e-04 | 702.1ms/step | 28.5 samples/s +Step 17420/44000 | Loss: 3.0416 | LR: 3.38e-04 | 700.7ms/step | 28.5 samples/s +Step 17430/44000 | Loss: 3.0508 | LR: 3.38e-04 | 699.9ms/step | 28.6 samples/s +Step 17440/44000 | Loss: 3.0647 | LR: 3.38e-04 | 700.4ms/step | 28.6 samples/s +Step 17450/44000 | Loss: 3.0652 | LR: 3.38e-04 | 700.7ms/step | 28.5 samples/s +Step 17460/44000 | Loss: 3.0464 | LR: 3.37e-04 | 701.2ms/step | 28.5 samples/s +Step 17470/44000 | Loss: 3.0435 | LR: 3.37e-04 | 704.1ms/step | 28.4 samples/s +Step 17480/44000 | Loss: 3.0793 | LR: 3.37e-04 | 702.0ms/step | 28.5 samples/s +Step 17490/44000 | Loss: 3.0447 | LR: 3.37e-04 | 699.5ms/step | 28.6 samples/s +Step 17500/44000 | Loss: 3.0554 | LR: 3.37e-04 | 700.7ms/step | 28.5 samples/s +Validation at step 17500: Loss = 3.0684, Perplexity = 21.51 +Step 17510/44000 | Loss: 2.9680 | LR: 3.37e-04 | 1285.3ms/step | 15.6 samples/s +Step 17520/44000 | Loss: 3.0753 | LR: 3.36e-04 | 702.4ms/step | 28.5 samples/s +Step 17530/44000 | Loss: 3.0398 | LR: 3.36e-04 | 704.6ms/step | 28.4 samples/s +Step 17540/44000 | Loss: 3.0540 | LR: 3.36e-04 | 707.5ms/step | 28.3 samples/s +Step 17550/44000 | Loss: 3.0436 | LR: 3.36e-04 | 699.8ms/step | 28.6 samples/s +Step 17560/44000 | Loss: 3.0616 | LR: 3.36e-04 | 702.1ms/step | 28.5 samples/s +Step 17570/44000 | Loss: 3.0391 | LR: 3.35e-04 | 701.9ms/step | 28.5 samples/s +Step 17580/44000 | Loss: 3.0432 | LR: 3.35e-04 | 705.9ms/step | 28.3 samples/s +Step 17590/44000 | Loss: 3.0510 | LR: 3.35e-04 | 700.3ms/step | 28.6 samples/s +Step 17600/44000 | Loss: 3.0759 | LR: 3.35e-04 | 700.8ms/step | 28.5 samples/s +Step 17610/44000 | Loss: 3.0386 | LR: 3.35e-04 | 700.5ms/step | 28.5 samples/s +Step 17620/44000 | Loss: 3.0450 | LR: 3.35e-04 | 701.1ms/step | 28.5 samples/s +Step 17630/44000 | Loss: 3.0682 | LR: 3.34e-04 | 705.4ms/step | 28.4 samples/s +Step 17640/44000 | Loss: 3.0324 | LR: 3.34e-04 | 699.6ms/step | 28.6 samples/s +Step 17650/44000 | Loss: 3.0190 | LR: 3.34e-04 | 700.4ms/step | 28.6 samples/s +Step 17660/44000 | Loss: 3.0225 | LR: 3.34e-04 | 699.8ms/step | 28.6 samples/s +Step 17670/44000 | Loss: 2.9880 | LR: 3.34e-04 | 703.8ms/step | 28.4 samples/s +Step 17680/44000 | Loss: 3.0706 | LR: 3.34e-04 | 702.2ms/step | 28.5 samples/s +Step 17690/44000 | Loss: 3.0182 | LR: 3.33e-04 | 698.8ms/step | 28.6 samples/s +Step 17700/44000 | Loss: 3.0202 | LR: 3.33e-04 | 701.4ms/step | 28.5 samples/s +Step 17710/44000 | Loss: 3.0408 | LR: 3.33e-04 | 700.7ms/step | 28.5 samples/s +Step 17720/44000 | Loss: 3.0178 | LR: 3.33e-04 | 699.2ms/step | 28.6 samples/s +Step 17730/44000 | Loss: 3.0745 | LR: 3.33e-04 | 701.8ms/step | 28.5 samples/s +Step 17740/44000 | Loss: 3.0626 | LR: 3.33e-04 | 702.6ms/step | 28.5 samples/s +Step 17750/44000 | Loss: 3.0717 | LR: 3.32e-04 | 701.9ms/step | 28.5 samples/s +Validation at step 17750: Loss = 3.0654, Perplexity = 21.44 +Step 17760/44000 | Loss: 3.0650 | LR: 3.32e-04 | 1285.3ms/step | 15.6 samples/s +Step 17770/44000 | Loss: 3.0436 | LR: 3.32e-04 | 702.3ms/step | 28.5 samples/s +Step 17780/44000 | Loss: 3.0766 | LR: 3.32e-04 | 698.2ms/step | 28.6 samples/s +Step 17790/44000 | Loss: 3.0340 | LR: 3.32e-04 | 700.9ms/step | 28.5 samples/s +Step 17800/44000 | Loss: 3.0394 | LR: 3.32e-04 | 702.3ms/step | 28.5 samples/s +Step 17810/44000 | Loss: 3.0460 | LR: 3.31e-04 | 702.5ms/step | 28.5 samples/s +Step 17820/44000 | Loss: 3.0247 | LR: 3.31e-04 | 706.8ms/step | 28.3 samples/s +Step 17830/44000 | Loss: 3.0483 | LR: 3.31e-04 | 704.7ms/step | 28.4 samples/s +Step 17840/44000 | Loss: 3.0437 | LR: 3.31e-04 | 700.2ms/step | 28.6 samples/s +Step 17850/44000 | Loss: 3.0215 | LR: 3.31e-04 | 702.8ms/step | 28.5 samples/s +Step 17860/44000 | Loss: 3.0090 | LR: 3.31e-04 | 703.3ms/step | 28.4 samples/s +Step 17870/44000 | Loss: 3.0781 | LR: 3.30e-04 | 704.3ms/step | 28.4 samples/s +Step 17880/44000 | Loss: 3.0467 | LR: 3.30e-04 | 703.6ms/step | 28.4 samples/s +Step 17890/44000 | Loss: 3.0504 | LR: 3.30e-04 | 700.4ms/step | 28.6 samples/s +Step 17900/44000 | Loss: 3.0060 | LR: 3.30e-04 | 701.9ms/step | 28.5 samples/s +Step 17910/44000 | Loss: 3.0669 | LR: 3.30e-04 | 702.2ms/step | 28.5 samples/s +Step 17920/44000 | Loss: 3.0404 | LR: 3.29e-04 | 701.6ms/step | 28.5 samples/s +Step 17930/44000 | Loss: 3.0170 | LR: 3.29e-04 | 697.8ms/step | 28.7 samples/s +Step 17940/44000 | Loss: 3.0375 | LR: 3.29e-04 | 703.0ms/step | 28.4 samples/s +Step 17950/44000 | Loss: 3.0161 | LR: 3.29e-04 | 702.9ms/step | 28.5 samples/s +Step 17960/44000 | Loss: 2.9893 | LR: 3.29e-04 | 702.2ms/step | 28.5 samples/s +Step 17970/44000 | Loss: 3.0511 | LR: 3.29e-04 | 700.7ms/step | 28.5 samples/s +Step 17980/44000 | Loss: 3.0778 | LR: 3.28e-04 | 701.3ms/step | 28.5 samples/s +Step 17990/44000 | Loss: 3.0446 | LR: 3.28e-04 | 703.6ms/step | 28.4 samples/s +Step 18000/44000 | Loss: 3.0603 | LR: 3.28e-04 | 702.8ms/step | 28.5 samples/s +Validation at step 18000: Loss = 3.0609, Perplexity = 21.35 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-18000 +Step 18010/44000 | Loss: 3.0771 | LR: 3.28e-04 | 1614.8ms/step | 12.4 samples/s +Step 18020/44000 | Loss: 3.0459 | LR: 3.28e-04 | 700.8ms/step | 28.5 samples/s +Step 18030/44000 | Loss: 3.0313 | LR: 3.28e-04 | 701.3ms/step | 28.5 samples/s +Step 18040/44000 | Loss: 3.0657 | LR: 3.27e-04 | 700.6ms/step | 28.5 samples/s +Step 18050/44000 | Loss: 3.0430 | LR: 3.27e-04 | 698.4ms/step | 28.6 samples/s +Step 18060/44000 | Loss: 3.0266 | LR: 3.27e-04 | 697.0ms/step | 28.7 samples/s +Step 18070/44000 | Loss: 3.0872 | LR: 3.27e-04 | 703.0ms/step | 28.4 samples/s +Step 18080/44000 | Loss: 3.0198 | LR: 3.27e-04 | 701.8ms/step | 28.5 samples/s +Step 18090/44000 | Loss: 3.0829 | LR: 3.27e-04 | 700.1ms/step | 28.6 samples/s +Step 18100/44000 | Loss: 3.0058 | LR: 3.26e-04 | 702.7ms/step | 28.5 samples/s +Step 18110/44000 | Loss: 3.0292 | LR: 3.26e-04 | 699.2ms/step | 28.6 samples/s +Step 18120/44000 | Loss: 3.0224 | LR: 3.26e-04 | 697.6ms/step | 28.7 samples/s +Step 18130/44000 | Loss: 3.0626 | LR: 3.26e-04 | 702.4ms/step | 28.5 samples/s +Step 18140/44000 | Loss: 3.0587 | LR: 3.26e-04 | 700.5ms/step | 28.6 samples/s +Step 18150/44000 | Loss: 3.0322 | LR: 3.26e-04 | 699.4ms/step | 28.6 samples/s +Step 18160/44000 | Loss: 3.0448 | LR: 3.25e-04 | 697.7ms/step | 28.7 samples/s +Step 18170/44000 | Loss: 3.0703 | LR: 3.25e-04 | 697.5ms/step | 28.7 samples/s +Step 18180/44000 | Loss: 3.0347 | LR: 3.25e-04 | 702.7ms/step | 28.5 samples/s +Step 18190/44000 | Loss: 3.0446 | LR: 3.25e-04 | 701.3ms/step | 28.5 samples/s +Step 18200/44000 | Loss: 2.9975 | LR: 3.25e-04 | 698.3ms/step | 28.6 samples/s +Step 18210/44000 | Loss: 3.0648 | LR: 3.24e-04 | 700.1ms/step | 28.6 samples/s +Step 18220/44000 | Loss: 3.0291 | LR: 3.24e-04 | 703.8ms/step | 28.4 samples/s +Step 18230/44000 | Loss: 3.0076 | LR: 3.24e-04 | 699.1ms/step | 28.6 samples/s +Step 18240/44000 | Loss: 3.0022 | LR: 3.24e-04 | 701.8ms/step | 28.5 samples/s +Step 18250/44000 | Loss: 3.0168 | LR: 3.24e-04 | 697.2ms/step | 28.7 samples/s +Validation at step 18250: Loss = 3.0575, Perplexity = 21.28 +Step 18260/44000 | Loss: 3.0163 | LR: 3.24e-04 | 1278.8ms/step | 15.6 samples/s +Step 18270/44000 | Loss: 3.0272 | LR: 3.23e-04 | 698.6ms/step | 28.6 samples/s +Step 18280/44000 | Loss: 3.0363 | LR: 3.23e-04 | 697.0ms/step | 28.7 samples/s +Step 18290/44000 | Loss: 3.0099 | LR: 3.23e-04 | 699.6ms/step | 28.6 samples/s +Step 18300/44000 | Loss: 3.0358 | LR: 3.23e-04 | 696.2ms/step | 28.7 samples/s +Step 18310/44000 | Loss: 3.0339 | LR: 3.23e-04 | 697.7ms/step | 28.7 samples/s +Step 18320/44000 | Loss: 3.0371 | LR: 3.23e-04 | 700.4ms/step | 28.6 samples/s +Step 18330/44000 | Loss: 3.0047 | LR: 3.22e-04 | 699.9ms/step | 28.6 samples/s +Step 18340/44000 | Loss: 2.9755 | LR: 3.22e-04 | 698.0ms/step | 28.7 samples/s +Step 18350/44000 | Loss: 3.0138 | LR: 3.22e-04 | 700.4ms/step | 28.6 samples/s +Step 18360/44000 | Loss: 3.0207 | LR: 3.22e-04 | 698.9ms/step | 28.6 samples/s +Step 18370/44000 | Loss: 3.0537 | LR: 3.22e-04 | 700.9ms/step | 28.5 samples/s +Step 18380/44000 | Loss: 3.0618 | LR: 3.22e-04 | 700.1ms/step | 28.6 samples/s +Step 18390/44000 | Loss: 3.0214 | LR: 3.21e-04 | 697.7ms/step | 28.7 samples/s +Step 18400/44000 | Loss: 3.0615 | LR: 3.21e-04 | 698.6ms/step | 28.6 samples/s +Step 18410/44000 | Loss: 3.0111 | LR: 3.21e-04 | 698.8ms/step | 28.6 samples/s +Step 18420/44000 | Loss: 3.0241 | LR: 3.21e-04 | 696.6ms/step | 28.7 samples/s +Step 18430/44000 | Loss: 3.0537 | LR: 3.21e-04 | 697.3ms/step | 28.7 samples/s +Step 18440/44000 | Loss: 3.0469 | LR: 3.20e-04 | 698.3ms/step | 28.6 samples/s +Step 18450/44000 | Loss: 3.0301 | LR: 3.20e-04 | 704.4ms/step | 28.4 samples/s +Step 18460/44000 | Loss: 3.0061 | LR: 3.20e-04 | 703.7ms/step | 28.4 samples/s +Step 18470/44000 | Loss: 3.0179 | LR: 3.20e-04 | 700.6ms/step | 28.5 samples/s +Step 18480/44000 | Loss: 3.0320 | LR: 3.20e-04 | 699.9ms/step | 28.6 samples/s +Step 18490/44000 | Loss: 3.0405 | LR: 3.20e-04 | 702.5ms/step | 28.5 samples/s +Step 18500/44000 | Loss: 3.0272 | LR: 3.19e-04 | 699.3ms/step | 28.6 samples/s +Validation at step 18500: Loss = 3.0563, Perplexity = 21.25 +Step 18510/44000 | Loss: 3.0233 | LR: 3.19e-04 | 1286.7ms/step | 15.5 samples/s +Step 18520/44000 | Loss: 3.0898 | LR: 3.19e-04 | 698.1ms/step | 28.6 samples/s +Step 18530/44000 | Loss: 3.0168 | LR: 3.19e-04 | 701.8ms/step | 28.5 samples/s +Step 18540/44000 | Loss: 3.0766 | LR: 3.19e-04 | 700.8ms/step | 28.5 samples/s +Step 18550/44000 | Loss: 3.0016 | LR: 3.19e-04 | 701.8ms/step | 28.5 samples/s +Step 18560/44000 | Loss: 3.0755 | LR: 3.18e-04 | 701.0ms/step | 28.5 samples/s +Step 18570/44000 | Loss: 3.0230 | LR: 3.18e-04 | 699.9ms/step | 28.6 samples/s +Step 18580/44000 | Loss: 3.0524 | LR: 3.18e-04 | 700.5ms/step | 28.6 samples/s +Step 18590/44000 | Loss: 3.0469 | LR: 3.18e-04 | 700.3ms/step | 28.6 samples/s +Step 18600/44000 | Loss: 3.0498 | LR: 3.18e-04 | 701.7ms/step | 28.5 samples/s +Step 18610/44000 | Loss: 3.0022 | LR: 3.18e-04 | 700.8ms/step | 28.5 samples/s +Step 18620/44000 | Loss: 3.0384 | LR: 3.17e-04 | 698.9ms/step | 28.6 samples/s +Step 18630/44000 | Loss: 3.0249 | LR: 3.17e-04 | 698.3ms/step | 28.6 samples/s +Step 18640/44000 | Loss: 3.0075 | LR: 3.17e-04 | 700.0ms/step | 28.6 samples/s +Step 18650/44000 | Loss: 3.0364 | LR: 3.17e-04 | 699.4ms/step | 28.6 samples/s +Step 18660/44000 | Loss: 3.0245 | LR: 3.17e-04 | 700.0ms/step | 28.6 samples/s +Step 18670/44000 | Loss: 3.0345 | LR: 3.16e-04 | 701.8ms/step | 28.5 samples/s +Step 18680/44000 | Loss: 3.0643 | LR: 3.16e-04 | 701.8ms/step | 28.5 samples/s +Step 18690/44000 | Loss: 3.0172 | LR: 3.16e-04 | 702.7ms/step | 28.5 samples/s +Step 18700/44000 | Loss: 3.0307 | LR: 3.16e-04 | 699.4ms/step | 28.6 samples/s +Step 18710/44000 | Loss: 3.0421 | LR: 3.16e-04 | 699.6ms/step | 28.6 samples/s +Step 18720/44000 | Loss: 3.0175 | LR: 3.16e-04 | 702.1ms/step | 28.5 samples/s +Step 18730/44000 | Loss: 3.0388 | LR: 3.15e-04 | 697.3ms/step | 28.7 samples/s +Step 18740/44000 | Loss: 3.0116 | LR: 3.15e-04 | 702.2ms/step | 28.5 samples/s +Step 18750/44000 | Loss: 2.9890 | LR: 3.15e-04 | 699.8ms/step | 28.6 samples/s +Validation at step 18750: Loss = 3.0516, Perplexity = 21.15 +Step 18760/44000 | Loss: 3.0367 | LR: 3.15e-04 | 1281.6ms/step | 15.6 samples/s +Step 18770/44000 | Loss: 3.0067 | LR: 3.15e-04 | 700.5ms/step | 28.5 samples/s +Step 18780/44000 | Loss: 3.0472 | LR: 3.15e-04 | 701.2ms/step | 28.5 samples/s +Step 18790/44000 | Loss: 3.0487 | LR: 3.14e-04 | 701.1ms/step | 28.5 samples/s +Step 18800/44000 | Loss: 3.0029 | LR: 3.14e-04 | 696.7ms/step | 28.7 samples/s +Step 18810/44000 | Loss: 3.0565 | LR: 3.14e-04 | 702.3ms/step | 28.5 samples/s +Step 18820/44000 | Loss: 3.0700 | LR: 3.14e-04 | 699.9ms/step | 28.6 samples/s +Step 18830/44000 | Loss: 3.0477 | LR: 3.14e-04 | 701.8ms/step | 28.5 samples/s +Step 18840/44000 | Loss: 3.0046 | LR: 3.13e-04 | 702.3ms/step | 28.5 samples/s +Step 18850/44000 | Loss: 3.0248 | LR: 3.13e-04 | 699.9ms/step | 28.6 samples/s +Step 18860/44000 | Loss: 3.0201 | LR: 3.13e-04 | 701.0ms/step | 28.5 samples/s +Step 18870/44000 | Loss: 3.0207 | LR: 3.13e-04 | 697.1ms/step | 28.7 samples/s +Step 18880/44000 | Loss: 3.0368 | LR: 3.13e-04 | 700.6ms/step | 28.5 samples/s +Step 18890/44000 | Loss: 3.0400 | LR: 3.13e-04 | 701.9ms/step | 28.5 samples/s +Step 18900/44000 | Loss: 3.0002 | LR: 3.12e-04 | 701.8ms/step | 28.5 samples/s +Step 18910/44000 | Loss: 3.0425 | LR: 3.12e-04 | 697.7ms/step | 28.7 samples/s +Step 18920/44000 | Loss: 2.9960 | LR: 3.12e-04 | 700.0ms/step | 28.6 samples/s +Step 18930/44000 | Loss: 3.0350 | LR: 3.12e-04 | 700.8ms/step | 28.5 samples/s +Step 18940/44000 | Loss: 2.9957 | LR: 3.12e-04 | 704.5ms/step | 28.4 samples/s +Step 18950/44000 | Loss: 3.0248 | LR: 3.12e-04 | 701.1ms/step | 28.5 samples/s +Step 18960/44000 | Loss: 3.0161 | LR: 3.11e-04 | 701.8ms/step | 28.5 samples/s +Step 18970/44000 | Loss: 3.0813 | LR: 3.11e-04 | 701.3ms/step | 28.5 samples/s +Step 18980/44000 | Loss: 2.9875 | LR: 3.11e-04 | 700.6ms/step | 28.5 samples/s +Step 18990/44000 | Loss: 2.9875 | LR: 3.11e-04 | 702.7ms/step | 28.5 samples/s +Step 19000/44000 | Loss: 3.0295 | LR: 3.11e-04 | 702.8ms/step | 28.5 samples/s +Validation at step 19000: Loss = 3.0499, Perplexity = 21.11 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-19000 +Step 19010/44000 | Loss: 3.0127 | LR: 3.11e-04 | 1594.2ms/step | 12.5 samples/s +Step 19020/44000 | Loss: 3.0530 | LR: 3.10e-04 | 702.3ms/step | 28.5 samples/s +Step 19030/44000 | Loss: 3.0402 | LR: 3.10e-04 | 700.3ms/step | 28.6 samples/s +Step 19040/44000 | Loss: 3.0315 | LR: 3.10e-04 | 701.3ms/step | 28.5 samples/s +Step 19050/44000 | Loss: 3.0242 | LR: 3.10e-04 | 702.7ms/step | 28.5 samples/s +Step 19060/44000 | Loss: 3.0315 | LR: 3.10e-04 | 699.7ms/step | 28.6 samples/s +Step 19070/44000 | Loss: 3.0400 | LR: 3.09e-04 | 700.0ms/step | 28.6 samples/s +Step 19080/44000 | Loss: 3.0433 | LR: 3.09e-04 | 700.3ms/step | 28.6 samples/s +Step 19090/44000 | Loss: 3.0272 | LR: 3.09e-04 | 703.7ms/step | 28.4 samples/s +Step 19100/44000 | Loss: 3.0368 | LR: 3.09e-04 | 702.7ms/step | 28.5 samples/s +Step 19110/44000 | Loss: 3.0192 | LR: 3.09e-04 | 699.6ms/step | 28.6 samples/s +Step 19120/44000 | Loss: 2.9888 | LR: 3.09e-04 | 701.4ms/step | 28.5 samples/s +Step 19130/44000 | Loss: 3.0385 | LR: 3.08e-04 | 699.8ms/step | 28.6 samples/s +Step 19140/44000 | Loss: 3.0062 | LR: 3.08e-04 | 702.6ms/step | 28.5 samples/s +Step 19150/44000 | Loss: 2.9783 | LR: 3.08e-04 | 699.7ms/step | 28.6 samples/s +Step 19160/44000 | Loss: 3.0776 | LR: 3.08e-04 | 697.9ms/step | 28.7 samples/s +Step 19170/44000 | Loss: 3.0087 | LR: 3.08e-04 | 699.9ms/step | 28.6 samples/s +Step 19180/44000 | Loss: 3.0381 | LR: 3.08e-04 | 699.8ms/step | 28.6 samples/s +Step 19190/44000 | Loss: 3.0388 | LR: 3.07e-04 | 700.4ms/step | 28.6 samples/s +Step 19200/44000 | Loss: 3.0456 | LR: 3.07e-04 | 702.2ms/step | 28.5 samples/s +Step 19210/44000 | Loss: 3.0028 | LR: 3.07e-04 | 700.2ms/step | 28.6 samples/s +Step 19220/44000 | Loss: 3.0166 | LR: 3.07e-04 | 699.5ms/step | 28.6 samples/s +Step 19230/44000 | Loss: 3.0330 | LR: 3.07e-04 | 699.6ms/step | 28.6 samples/s +Step 19240/44000 | Loss: 3.0443 | LR: 3.06e-04 | 701.1ms/step | 28.5 samples/s +Step 19250/44000 | Loss: 3.0203 | LR: 3.06e-04 | 698.4ms/step | 28.6 samples/s +Validation at step 19250: Loss = 3.0474, Perplexity = 21.06 +Step 19260/44000 | Loss: 3.0421 | LR: 3.06e-04 | 1290.3ms/step | 15.5 samples/s +Step 19270/44000 | Loss: 3.0236 | LR: 3.06e-04 | 701.7ms/step | 28.5 samples/s +Step 19280/44000 | Loss: 3.0515 | LR: 3.06e-04 | 701.6ms/step | 28.5 samples/s +Step 19290/44000 | Loss: 3.0360 | LR: 3.06e-04 | 701.8ms/step | 28.5 samples/s +Step 19300/44000 | Loss: 2.9789 | LR: 3.05e-04 | 702.6ms/step | 28.5 samples/s +Step 19310/44000 | Loss: 3.0268 | LR: 3.05e-04 | 703.1ms/step | 28.4 samples/s +Step 19320/44000 | Loss: 3.0178 | LR: 3.05e-04 | 700.7ms/step | 28.5 samples/s +Step 19330/44000 | Loss: 2.9880 | LR: 3.05e-04 | 701.1ms/step | 28.5 samples/s +Step 19340/44000 | Loss: 3.0324 | LR: 3.05e-04 | 703.3ms/step | 28.4 samples/s +Step 19350/44000 | Loss: 3.0251 | LR: 3.04e-04 | 703.7ms/step | 28.4 samples/s +Step 19360/44000 | Loss: 3.0270 | LR: 3.04e-04 | 703.1ms/step | 28.4 samples/s +Step 19370/44000 | Loss: 3.0077 | LR: 3.04e-04 | 704.5ms/step | 28.4 samples/s +Step 19380/44000 | Loss: 3.0280 | LR: 3.04e-04 | 702.3ms/step | 28.5 samples/s +Step 19390/44000 | Loss: 3.0558 | LR: 3.04e-04 | 701.4ms/step | 28.5 samples/s +Step 19400/44000 | Loss: 3.0337 | LR: 3.04e-04 | 703.3ms/step | 28.4 samples/s +Step 19410/44000 | Loss: 3.0392 | LR: 3.03e-04 | 702.5ms/step | 28.5 samples/s +Step 19420/44000 | Loss: 3.0458 | LR: 3.03e-04 | 700.8ms/step | 28.5 samples/s +Step 19430/44000 | Loss: 2.9842 | LR: 3.03e-04 | 702.5ms/step | 28.5 samples/s +Step 19440/44000 | Loss: 2.9915 | LR: 3.03e-04 | 702.9ms/step | 28.5 samples/s +Step 19450/44000 | Loss: 3.0257 | LR: 3.03e-04 | 702.2ms/step | 28.5 samples/s +Step 19460/44000 | Loss: 2.9854 | LR: 3.03e-04 | 699.0ms/step | 28.6 samples/s +Step 19470/44000 | Loss: 3.0178 | LR: 3.02e-04 | 703.5ms/step | 28.4 samples/s +Step 19480/44000 | Loss: 3.0004 | LR: 3.02e-04 | 701.7ms/step | 28.5 samples/s +Step 19490/44000 | Loss: 3.0020 | LR: 3.02e-04 | 700.2ms/step | 28.6 samples/s +Step 19500/44000 | Loss: 2.9976 | LR: 3.02e-04 | 702.0ms/step | 28.5 samples/s +Validation at step 19500: Loss = 3.0436, Perplexity = 20.98 +Step 19510/44000 | Loss: 3.0199 | LR: 3.02e-04 | 1277.9ms/step | 15.7 samples/s +Step 19520/44000 | Loss: 3.0047 | LR: 3.01e-04 | 699.9ms/step | 28.6 samples/s +Step 19530/44000 | Loss: 3.0960 | LR: 3.01e-04 | 700.8ms/step | 28.5 samples/s +Step 19540/44000 | Loss: 3.0659 | LR: 3.01e-04 | 702.7ms/step | 28.5 samples/s +Step 19550/44000 | Loss: 3.0200 | LR: 3.01e-04 | 699.9ms/step | 28.6 samples/s +Step 19560/44000 | Loss: 3.0204 | LR: 3.01e-04 | 702.7ms/step | 28.5 samples/s +Step 19570/44000 | Loss: 3.0255 | LR: 3.01e-04 | 700.2ms/step | 28.6 samples/s +Step 19580/44000 | Loss: 3.0089 | LR: 3.00e-04 | 699.4ms/step | 28.6 samples/s +Step 19590/44000 | Loss: 3.0154 | LR: 3.00e-04 | 699.1ms/step | 28.6 samples/s +Step 19600/44000 | Loss: 3.0548 | LR: 3.00e-04 | 700.0ms/step | 28.6 samples/s +Step 19610/44000 | Loss: 3.0421 | LR: 3.00e-04 | 698.6ms/step | 28.6 samples/s +Step 19620/44000 | Loss: 2.9996 | LR: 3.00e-04 | 699.0ms/step | 28.6 samples/s +Step 19630/44000 | Loss: 3.0388 | LR: 3.00e-04 | 698.9ms/step | 28.6 samples/s +Step 19640/44000 | Loss: 3.0195 | LR: 2.99e-04 | 702.8ms/step | 28.5 samples/s +Step 19650/44000 | Loss: 3.0242 | LR: 2.99e-04 | 700.0ms/step | 28.6 samples/s +Step 19660/44000 | Loss: 3.0456 | LR: 2.99e-04 | 699.9ms/step | 28.6 samples/s +Step 19670/44000 | Loss: 3.0217 | LR: 2.99e-04 | 699.1ms/step | 28.6 samples/s +Step 19680/44000 | Loss: 3.0438 | LR: 2.99e-04 | 701.7ms/step | 28.5 samples/s +Step 19690/44000 | Loss: 3.0393 | LR: 2.98e-04 | 699.0ms/step | 28.6 samples/s +Step 19700/44000 | Loss: 3.0037 | LR: 2.98e-04 | 700.0ms/step | 28.6 samples/s +Step 19710/44000 | Loss: 3.0076 | LR: 2.98e-04 | 702.0ms/step | 28.5 samples/s +Step 19720/44000 | Loss: 3.0477 | LR: 2.98e-04 | 700.6ms/step | 28.5 samples/s +Step 19730/44000 | Loss: 3.0546 | LR: 2.98e-04 | 697.4ms/step | 28.7 samples/s +Step 19740/44000 | Loss: 2.9943 | LR: 2.98e-04 | 697.0ms/step | 28.7 samples/s +Step 19750/44000 | Loss: 2.9904 | LR: 2.97e-04 | 700.8ms/step | 28.5 samples/s +Validation at step 19750: Loss = 3.0408, Perplexity = 20.92 +Step 19760/44000 | Loss: 2.9998 | LR: 2.97e-04 | 1277.8ms/step | 15.7 samples/s +Step 19770/44000 | Loss: 3.0607 | LR: 2.97e-04 | 701.6ms/step | 28.5 samples/s +Step 19780/44000 | Loss: 2.9605 | LR: 2.97e-04 | 700.7ms/step | 28.5 samples/s +Step 19790/44000 | Loss: 3.0173 | LR: 2.97e-04 | 701.3ms/step | 28.5 samples/s +Step 19800/44000 | Loss: 3.0002 | LR: 2.96e-04 | 698.1ms/step | 28.7 samples/s +Step 19810/44000 | Loss: 3.0115 | LR: 2.96e-04 | 700.9ms/step | 28.5 samples/s +Step 19820/44000 | Loss: 2.9976 | LR: 2.96e-04 | 700.9ms/step | 28.5 samples/s +Step 19830/44000 | Loss: 2.9922 | LR: 2.96e-04 | 704.2ms/step | 28.4 samples/s +Step 19840/44000 | Loss: 3.0123 | LR: 2.96e-04 | 700.8ms/step | 28.5 samples/s +Step 19850/44000 | Loss: 3.0146 | LR: 2.96e-04 | 699.5ms/step | 28.6 samples/s +Step 19860/44000 | Loss: 3.0486 | LR: 2.95e-04 | 703.4ms/step | 28.4 samples/s +Step 19870/44000 | Loss: 3.0182 | LR: 2.95e-04 | 701.8ms/step | 28.5 samples/s +Step 19880/44000 | Loss: 3.0479 | LR: 2.95e-04 | 705.3ms/step | 28.4 samples/s +Step 19890/44000 | Loss: 3.0245 | LR: 2.95e-04 | 702.1ms/step | 28.5 samples/s +Step 19900/44000 | Loss: 3.0679 | LR: 2.95e-04 | 699.9ms/step | 28.6 samples/s +Step 19910/44000 | Loss: 2.9977 | LR: 2.95e-04 | 697.2ms/step | 28.7 samples/s +Step 19920/44000 | Loss: 2.9926 | LR: 2.94e-04 | 701.9ms/step | 28.5 samples/s +Step 19930/44000 | Loss: 3.0033 | LR: 2.94e-04 | 700.7ms/step | 28.5 samples/s +Step 19940/44000 | Loss: 3.0366 | LR: 2.94e-04 | 703.4ms/step | 28.4 samples/s +Step 19950/44000 | Loss: 2.9718 | LR: 2.94e-04 | 700.1ms/step | 28.6 samples/s +Step 19960/44000 | Loss: 2.9875 | LR: 2.94e-04 | 699.9ms/step | 28.6 samples/s +Step 19970/44000 | Loss: 2.9953 | LR: 2.93e-04 | 700.8ms/step | 28.5 samples/s +Step 19980/44000 | Loss: 3.0228 | LR: 2.93e-04 | 699.3ms/step | 28.6 samples/s +Step 19990/44000 | Loss: 3.0346 | LR: 2.93e-04 | 701.3ms/step | 28.5 samples/s +Step 20000/44000 | Loss: 2.9834 | LR: 2.93e-04 | 698.6ms/step | 28.6 samples/s +Validation at step 20000: Loss = 3.0401, Perplexity = 20.91 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-20000 +Step 20010/44000 | Loss: 3.0184 | LR: 2.93e-04 | 1570.0ms/step | 12.7 samples/s +Step 20020/44000 | Loss: 2.9981 | LR: 2.93e-04 | 702.7ms/step | 28.5 samples/s +Step 20030/44000 | Loss: 2.9925 | LR: 2.92e-04 | 699.3ms/step | 28.6 samples/s +Step 20040/44000 | Loss: 2.9924 | LR: 2.92e-04 | 701.0ms/step | 28.5 samples/s +Step 20050/44000 | Loss: 3.0103 | LR: 2.92e-04 | 701.2ms/step | 28.5 samples/s +Step 20060/44000 | Loss: 3.0614 | LR: 2.92e-04 | 701.9ms/step | 28.5 samples/s +Step 20070/44000 | Loss: 3.0099 | LR: 2.92e-04 | 702.4ms/step | 28.5 samples/s +Step 20080/44000 | Loss: 2.9850 | LR: 2.91e-04 | 695.7ms/step | 28.7 samples/s +Step 20090/44000 | Loss: 3.0025 | LR: 2.91e-04 | 698.7ms/step | 28.6 samples/s +Step 20100/44000 | Loss: 3.0352 | LR: 2.91e-04 | 697.6ms/step | 28.7 samples/s +Step 20110/44000 | Loss: 3.0214 | LR: 2.91e-04 | 699.9ms/step | 28.6 samples/s +Step 20120/44000 | Loss: 3.0052 | LR: 2.91e-04 | 700.0ms/step | 28.6 samples/s +Step 20130/44000 | Loss: 3.0248 | LR: 2.91e-04 | 700.2ms/step | 28.6 samples/s +Step 20140/44000 | Loss: 3.0075 | LR: 2.90e-04 | 702.6ms/step | 28.5 samples/s +Step 20150/44000 | Loss: 3.0184 | LR: 2.90e-04 | 699.6ms/step | 28.6 samples/s +Step 20160/44000 | Loss: 3.0404 | LR: 2.90e-04 | 700.9ms/step | 28.5 samples/s +Step 20170/44000 | Loss: 2.9985 | LR: 2.90e-04 | 702.3ms/step | 28.5 samples/s +Step 20180/44000 | Loss: 3.0080 | LR: 2.90e-04 | 699.0ms/step | 28.6 samples/s +Step 20190/44000 | Loss: 3.0033 | LR: 2.90e-04 | 697.6ms/step | 28.7 samples/s +Step 20200/44000 | Loss: 3.0181 | LR: 2.89e-04 | 699.0ms/step | 28.6 samples/s +Step 20210/44000 | Loss: 3.0334 | LR: 2.89e-04 | 698.3ms/step | 28.6 samples/s +Step 20220/44000 | Loss: 3.0336 | LR: 2.89e-04 | 701.3ms/step | 28.5 samples/s +Step 20230/44000 | Loss: 3.0399 | LR: 2.89e-04 | 700.2ms/step | 28.6 samples/s +Step 20240/44000 | Loss: 3.0491 | LR: 2.89e-04 | 704.6ms/step | 28.4 samples/s +Step 20250/44000 | Loss: 3.0275 | LR: 2.88e-04 | 698.6ms/step | 28.6 samples/s +Validation at step 20250: Loss = 3.0374, Perplexity = 20.85 +Step 20260/44000 | Loss: 3.0440 | LR: 2.88e-04 | 1273.2ms/step | 15.7 samples/s +Step 20270/44000 | Loss: 3.0175 | LR: 2.88e-04 | 701.9ms/step | 28.5 samples/s +Step 20280/44000 | Loss: 3.0022 | LR: 2.88e-04 | 702.0ms/step | 28.5 samples/s +Step 20290/44000 | Loss: 3.0261 | LR: 2.88e-04 | 699.9ms/step | 28.6 samples/s +Step 20300/44000 | Loss: 3.0412 | LR: 2.88e-04 | 696.9ms/step | 28.7 samples/s +Step 20310/44000 | Loss: 2.9740 | LR: 2.87e-04 | 697.4ms/step | 28.7 samples/s +Step 20320/44000 | Loss: 3.0442 | LR: 2.87e-04 | 697.6ms/step | 28.7 samples/s +Step 20330/44000 | Loss: 3.0184 | LR: 2.87e-04 | 699.9ms/step | 28.6 samples/s +Step 20340/44000 | Loss: 3.0183 | LR: 2.87e-04 | 698.8ms/step | 28.6 samples/s +Step 20350/44000 | Loss: 3.0111 | LR: 2.87e-04 | 699.8ms/step | 28.6 samples/s +Step 20360/44000 | Loss: 3.0176 | LR: 2.86e-04 | 699.1ms/step | 28.6 samples/s +Step 20370/44000 | Loss: 3.0324 | LR: 2.86e-04 | 698.4ms/step | 28.6 samples/s +Step 20380/44000 | Loss: 3.0192 | LR: 2.86e-04 | 700.2ms/step | 28.6 samples/s +Step 20390/44000 | Loss: 3.0491 | LR: 2.86e-04 | 700.8ms/step | 28.5 samples/s +Step 20400/44000 | Loss: 3.0536 | LR: 2.86e-04 | 699.3ms/step | 28.6 samples/s +Step 20410/44000 | Loss: 3.0423 | LR: 2.86e-04 | 702.7ms/step | 28.5 samples/s +Step 20420/44000 | Loss: 3.0554 | LR: 2.85e-04 | 700.5ms/step | 28.6 samples/s +Step 20430/44000 | Loss: 3.0125 | LR: 2.85e-04 | 701.3ms/step | 28.5 samples/s +Step 20440/44000 | Loss: 2.9598 | LR: 2.85e-04 | 700.9ms/step | 28.5 samples/s +Step 20450/44000 | Loss: 2.9862 | LR: 2.85e-04 | 699.1ms/step | 28.6 samples/s +Step 20460/44000 | Loss: 3.0079 | LR: 2.85e-04 | 696.0ms/step | 28.7 samples/s +Step 20470/44000 | Loss: 2.9814 | LR: 2.84e-04 | 698.1ms/step | 28.6 samples/s +Step 20480/44000 | Loss: 3.0029 | LR: 2.84e-04 | 701.0ms/step | 28.5 samples/s +Step 20490/44000 | Loss: 3.0284 | LR: 2.84e-04 | 699.3ms/step | 28.6 samples/s +Step 20500/44000 | Loss: 2.9805 | LR: 2.84e-04 | 697.8ms/step | 28.7 samples/s +Validation at step 20500: Loss = 3.0345, Perplexity = 20.79 +Step 20510/44000 | Loss: 3.0096 | LR: 2.84e-04 | 1273.3ms/step | 15.7 samples/s +Step 20520/44000 | Loss: 3.0309 | LR: 2.84e-04 | 701.4ms/step | 28.5 samples/s +Step 20530/44000 | Loss: 3.0229 | LR: 2.83e-04 | 700.0ms/step | 28.6 samples/s +Step 20540/44000 | Loss: 3.0133 | LR: 2.83e-04 | 703.0ms/step | 28.5 samples/s +Step 20550/44000 | Loss: 2.9737 | LR: 2.83e-04 | 699.7ms/step | 28.6 samples/s +Step 20560/44000 | Loss: 3.0144 | LR: 2.83e-04 | 699.8ms/step | 28.6 samples/s +Step 20570/44000 | Loss: 3.0297 | LR: 2.83e-04 | 695.8ms/step | 28.7 samples/s +Step 20580/44000 | Loss: 2.9981 | LR: 2.83e-04 | 700.6ms/step | 28.5 samples/s +Step 20590/44000 | Loss: 2.9843 | LR: 2.82e-04 | 703.6ms/step | 28.4 samples/s +Step 20600/44000 | Loss: 2.9768 | LR: 2.82e-04 | 702.6ms/step | 28.5 samples/s +Step 20610/44000 | Loss: 3.0011 | LR: 2.82e-04 | 702.9ms/step | 28.5 samples/s +Step 20620/44000 | Loss: 3.0124 | LR: 2.82e-04 | 701.6ms/step | 28.5 samples/s +Step 20630/44000 | Loss: 2.9584 | LR: 2.82e-04 | 702.5ms/step | 28.5 samples/s +Step 20640/44000 | Loss: 3.0178 | LR: 2.81e-04 | 701.5ms/step | 28.5 samples/s +Step 20650/44000 | Loss: 3.0054 | LR: 2.81e-04 | 701.2ms/step | 28.5 samples/s +Step 20660/44000 | Loss: 2.9837 | LR: 2.81e-04 | 699.7ms/step | 28.6 samples/s +Step 20670/44000 | Loss: 2.9853 | LR: 2.81e-04 | 700.4ms/step | 28.6 samples/s +Step 20680/44000 | Loss: 3.0019 | LR: 2.81e-04 | 700.6ms/step | 28.5 samples/s +Step 20690/44000 | Loss: 2.9744 | LR: 2.81e-04 | 699.0ms/step | 28.6 samples/s +Step 20700/44000 | Loss: 3.0123 | LR: 2.80e-04 | 697.2ms/step | 28.7 samples/s +Step 20710/44000 | Loss: 3.0356 | LR: 2.80e-04 | 700.5ms/step | 28.6 samples/s +Step 20720/44000 | Loss: 3.0290 | LR: 2.80e-04 | 700.0ms/step | 28.6 samples/s +Step 20730/44000 | Loss: 3.0233 | LR: 2.80e-04 | 700.1ms/step | 28.6 samples/s +Step 20740/44000 | Loss: 3.0464 | LR: 2.80e-04 | 699.1ms/step | 28.6 samples/s +Step 20750/44000 | Loss: 3.0235 | LR: 2.79e-04 | 698.6ms/step | 28.6 samples/s +Validation at step 20750: Loss = 3.0307, Perplexity = 20.71 +Step 20760/44000 | Loss: 3.0099 | LR: 2.79e-04 | 1279.1ms/step | 15.6 samples/s +Step 20770/44000 | Loss: 3.0254 | LR: 2.79e-04 | 703.3ms/step | 28.4 samples/s +Step 20780/44000 | Loss: 2.9875 | LR: 2.79e-04 | 701.5ms/step | 28.5 samples/s +Step 20790/44000 | Loss: 3.0237 | LR: 2.79e-04 | 704.1ms/step | 28.4 samples/s +Step 20800/44000 | Loss: 2.9603 | LR: 2.79e-04 | 701.9ms/step | 28.5 samples/s +Step 20810/44000 | Loss: 2.9895 | LR: 2.78e-04 | 699.4ms/step | 28.6 samples/s +Step 20820/44000 | Loss: 3.0145 | LR: 2.78e-04 | 699.3ms/step | 28.6 samples/s +Step 20830/44000 | Loss: 3.0342 | LR: 2.78e-04 | 699.3ms/step | 28.6 samples/s +Step 20840/44000 | Loss: 3.0326 | LR: 2.78e-04 | 699.5ms/step | 28.6 samples/s +Step 20850/44000 | Loss: 3.0131 | LR: 2.78e-04 | 701.4ms/step | 28.5 samples/s +Step 20860/44000 | Loss: 2.9881 | LR: 2.77e-04 | 702.9ms/step | 28.5 samples/s +Step 20870/44000 | Loss: 3.0063 | LR: 2.77e-04 | 702.7ms/step | 28.5 samples/s +Step 20880/44000 | Loss: 3.0059 | LR: 2.77e-04 | 699.4ms/step | 28.6 samples/s +Step 20890/44000 | Loss: 3.0280 | LR: 2.77e-04 | 701.3ms/step | 28.5 samples/s +Step 20900/44000 | Loss: 3.0044 | LR: 2.77e-04 | 699.5ms/step | 28.6 samples/s +Step 20910/44000 | Loss: 3.0064 | LR: 2.77e-04 | 700.7ms/step | 28.5 samples/s +Step 20920/44000 | Loss: 3.0328 | LR: 2.76e-04 | 701.0ms/step | 28.5 samples/s +Step 20930/44000 | Loss: 2.9870 | LR: 2.76e-04 | 698.3ms/step | 28.6 samples/s +Step 20940/44000 | Loss: 2.9664 | LR: 2.76e-04 | 702.7ms/step | 28.5 samples/s +Step 20950/44000 | Loss: 2.9673 | LR: 2.76e-04 | 699.0ms/step | 28.6 samples/s +Step 20960/44000 | Loss: 3.0165 | LR: 2.76e-04 | 700.0ms/step | 28.6 samples/s +Step 20970/44000 | Loss: 3.0108 | LR: 2.75e-04 | 700.2ms/step | 28.6 samples/s +Step 20980/44000 | Loss: 2.9764 | LR: 2.75e-04 | 700.2ms/step | 28.6 samples/s +Step 20990/44000 | Loss: 3.0290 | LR: 2.75e-04 | 701.0ms/step | 28.5 samples/s +Step 21000/44000 | Loss: 3.0141 | LR: 2.75e-04 | 701.2ms/step | 28.5 samples/s +Validation at step 21000: Loss = 3.0286, Perplexity = 20.67 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-21000 +Step 21010/44000 | Loss: 3.0348 | LR: 2.75e-04 | 1630.2ms/step | 12.3 samples/s +Step 21020/44000 | Loss: 2.9940 | LR: 2.75e-04 | 701.2ms/step | 28.5 samples/s +Step 21030/44000 | Loss: 3.0052 | LR: 2.74e-04 | 698.7ms/step | 28.6 samples/s +Step 21040/44000 | Loss: 2.9947 | LR: 2.74e-04 | 700.5ms/step | 28.6 samples/s +Step 21050/44000 | Loss: 2.9885 | LR: 2.74e-04 | 699.5ms/step | 28.6 samples/s +Step 21060/44000 | Loss: 3.0174 | LR: 2.74e-04 | 699.7ms/step | 28.6 samples/s +Step 21070/44000 | Loss: 3.0394 | LR: 2.74e-04 | 699.9ms/step | 28.6 samples/s +Step 21080/44000 | Loss: 2.9875 | LR: 2.73e-04 | 703.3ms/step | 28.4 samples/s +Step 21090/44000 | Loss: 3.0091 | LR: 2.73e-04 | 702.2ms/step | 28.5 samples/s +Step 21100/44000 | Loss: 2.9822 | LR: 2.73e-04 | 702.0ms/step | 28.5 samples/s +Step 21110/44000 | Loss: 3.0029 | LR: 2.73e-04 | 699.7ms/step | 28.6 samples/s +Step 21120/44000 | Loss: 3.0088 | LR: 2.73e-04 | 702.0ms/step | 28.5 samples/s +Step 21130/44000 | Loss: 2.9959 | LR: 2.73e-04 | 696.7ms/step | 28.7 samples/s +Step 21140/44000 | Loss: 2.9767 | LR: 2.72e-04 | 699.4ms/step | 28.6 samples/s +Step 21150/44000 | Loss: 3.0158 | LR: 2.72e-04 | 700.0ms/step | 28.6 samples/s +Step 21160/44000 | Loss: 2.9649 | LR: 2.72e-04 | 699.9ms/step | 28.6 samples/s +Step 21170/44000 | Loss: 3.0258 | LR: 2.72e-04 | 698.3ms/step | 28.6 samples/s +Step 21180/44000 | Loss: 3.0232 | LR: 2.72e-04 | 700.6ms/step | 28.5 samples/s +Step 21190/44000 | Loss: 3.0214 | LR: 2.71e-04 | 700.2ms/step | 28.6 samples/s +Step 21200/44000 | Loss: 3.0463 | LR: 2.71e-04 | 698.8ms/step | 28.6 samples/s +Step 21210/44000 | Loss: 2.9533 | LR: 2.71e-04 | 699.8ms/step | 28.6 samples/s +Step 21220/44000 | Loss: 2.9618 | LR: 2.71e-04 | 697.0ms/step | 28.7 samples/s +Step 21230/44000 | Loss: 3.0019 | LR: 2.71e-04 | 700.7ms/step | 28.5 samples/s +Step 21240/44000 | Loss: 3.0086 | LR: 2.71e-04 | 699.0ms/step | 28.6 samples/s +Step 21250/44000 | Loss: 2.9856 | LR: 2.70e-04 | 700.4ms/step | 28.6 samples/s +Validation at step 21250: Loss = 3.0258, Perplexity = 20.61 +Step 21260/44000 | Loss: 3.0283 | LR: 2.70e-04 | 1270.7ms/step | 15.7 samples/s +Step 21270/44000 | Loss: 3.0079 | LR: 2.70e-04 | 697.7ms/step | 28.7 samples/s +Step 21280/44000 | Loss: 3.0345 | LR: 2.70e-04 | 701.7ms/step | 28.5 samples/s +Step 21290/44000 | Loss: 3.0146 | LR: 2.70e-04 | 702.7ms/step | 28.5 samples/s +Step 21300/44000 | Loss: 2.9984 | LR: 2.70e-04 | 699.3ms/step | 28.6 samples/s +Step 21310/44000 | Loss: 3.0122 | LR: 2.69e-04 | 700.9ms/step | 28.5 samples/s +Step 21320/44000 | Loss: 2.9939 | LR: 2.69e-04 | 699.7ms/step | 28.6 samples/s +Step 21330/44000 | Loss: 2.9757 | LR: 2.69e-04 | 699.9ms/step | 28.6 samples/s +Step 21340/44000 | Loss: 3.0192 | LR: 2.69e-04 | 701.2ms/step | 28.5 samples/s +Step 21350/44000 | Loss: 2.9929 | LR: 2.69e-04 | 700.3ms/step | 28.6 samples/s +Step 21360/44000 | Loss: 3.0047 | LR: 2.68e-04 | 701.9ms/step | 28.5 samples/s +Step 21370/44000 | Loss: 2.9945 | LR: 2.68e-04 | 699.6ms/step | 28.6 samples/s +Step 21380/44000 | Loss: 2.9848 | LR: 2.68e-04 | 699.1ms/step | 28.6 samples/s +Step 21390/44000 | Loss: 3.0113 | LR: 2.68e-04 | 699.4ms/step | 28.6 samples/s +Step 21400/44000 | Loss: 2.9754 | LR: 2.68e-04 | 702.3ms/step | 28.5 samples/s +Step 21410/44000 | Loss: 2.9501 | LR: 2.68e-04 | 701.4ms/step | 28.5 samples/s +Step 21420/44000 | Loss: 2.9671 | LR: 2.67e-04 | 699.6ms/step | 28.6 samples/s +Step 21430/44000 | Loss: 2.9963 | LR: 2.67e-04 | 701.8ms/step | 28.5 samples/s +Step 21440/44000 | Loss: 3.0176 | LR: 2.67e-04 | 701.4ms/step | 28.5 samples/s +Step 21450/44000 | Loss: 2.9876 | LR: 2.67e-04 | 702.3ms/step | 28.5 samples/s +Step 21460/44000 | Loss: 3.0366 | LR: 2.67e-04 | 701.8ms/step | 28.5 samples/s +Step 21470/44000 | Loss: 2.9873 | LR: 2.66e-04 | 702.3ms/step | 28.5 samples/s +Step 21480/44000 | Loss: 3.0015 | LR: 2.66e-04 | 700.3ms/step | 28.6 samples/s +Step 21490/44000 | Loss: 3.0240 | LR: 2.66e-04 | 701.9ms/step | 28.5 samples/s +Step 21500/44000 | Loss: 2.9772 | LR: 2.66e-04 | 703.1ms/step | 28.4 samples/s +Validation at step 21500: Loss = 3.0236, Perplexity = 20.57 +Step 21510/44000 | Loss: 2.9655 | LR: 2.66e-04 | 1275.5ms/step | 15.7 samples/s +Step 21520/44000 | Loss: 3.0160 | LR: 2.66e-04 | 704.6ms/step | 28.4 samples/s +Step 21530/44000 | Loss: 3.0361 | LR: 2.65e-04 | 701.0ms/step | 28.5 samples/s +Step 21540/44000 | Loss: 2.9871 | LR: 2.65e-04 | 702.3ms/step | 28.5 samples/s +Step 21550/44000 | Loss: 2.9890 | LR: 2.65e-04 | 704.0ms/step | 28.4 samples/s +Step 21560/44000 | Loss: 3.0347 | LR: 2.65e-04 | 701.7ms/step | 28.5 samples/s +Step 21570/44000 | Loss: 2.9875 | LR: 2.65e-04 | 697.1ms/step | 28.7 samples/s +Step 21580/44000 | Loss: 3.0356 | LR: 2.64e-04 | 701.0ms/step | 28.5 samples/s +Step 21590/44000 | Loss: 2.9680 | LR: 2.64e-04 | 701.5ms/step | 28.5 samples/s +Step 21600/44000 | Loss: 2.9959 | LR: 2.64e-04 | 699.8ms/step | 28.6 samples/s +Step 21610/44000 | Loss: 3.0382 | LR: 2.64e-04 | 700.2ms/step | 28.6 samples/s +Step 21620/44000 | Loss: 3.0117 | LR: 2.64e-04 | 698.6ms/step | 28.6 samples/s +Step 21630/44000 | Loss: 2.9857 | LR: 2.64e-04 | 700.7ms/step | 28.5 samples/s +Step 21640/44000 | Loss: 3.0134 | LR: 2.63e-04 | 698.3ms/step | 28.6 samples/s +Step 21650/44000 | Loss: 3.0009 | LR: 2.63e-04 | 701.5ms/step | 28.5 samples/s +Step 21660/44000 | Loss: 2.9979 | LR: 2.63e-04 | 700.6ms/step | 28.5 samples/s +Step 21670/44000 | Loss: 3.0070 | LR: 2.63e-04 | 702.1ms/step | 28.5 samples/s +Step 21680/44000 | Loss: 3.0091 | LR: 2.63e-04 | 699.1ms/step | 28.6 samples/s +Step 21690/44000 | Loss: 2.9931 | LR: 2.62e-04 | 697.4ms/step | 28.7 samples/s +Step 21700/44000 | Loss: 2.9965 | LR: 2.62e-04 | 698.9ms/step | 28.6 samples/s +Step 21710/44000 | Loss: 3.0495 | LR: 2.62e-04 | 699.1ms/step | 28.6 samples/s +Step 21720/44000 | Loss: 2.9807 | LR: 2.62e-04 | 697.5ms/step | 28.7 samples/s +Step 21730/44000 | Loss: 3.0030 | LR: 2.62e-04 | 703.8ms/step | 28.4 samples/s +Step 21740/44000 | Loss: 2.9996 | LR: 2.62e-04 | 698.7ms/step | 28.6 samples/s +Step 21750/44000 | Loss: 2.9786 | LR: 2.61e-04 | 702.2ms/step | 28.5 samples/s +Validation at step 21750: Loss = 3.0220, Perplexity = 20.53 +Step 21760/44000 | Loss: 3.0216 | LR: 2.61e-04 | 1277.7ms/step | 15.7 samples/s +Step 21770/44000 | Loss: 3.0405 | LR: 2.61e-04 | 698.3ms/step | 28.6 samples/s +Step 21780/44000 | Loss: 3.0331 | LR: 2.61e-04 | 698.6ms/step | 28.6 samples/s +Step 21790/44000 | Loss: 2.9656 | LR: 2.61e-04 | 698.1ms/step | 28.6 samples/s +Step 21800/44000 | Loss: 3.0028 | LR: 2.60e-04 | 701.7ms/step | 28.5 samples/s +Step 21810/44000 | Loss: 3.0089 | LR: 2.60e-04 | 701.9ms/step | 28.5 samples/s +Step 21820/44000 | Loss: 2.9816 | LR: 2.60e-04 | 699.1ms/step | 28.6 samples/s +Step 21830/44000 | Loss: 2.9853 | LR: 2.60e-04 | 697.7ms/step | 28.7 samples/s +Step 21840/44000 | Loss: 3.0150 | LR: 2.60e-04 | 699.4ms/step | 28.6 samples/s +Step 21850/44000 | Loss: 2.9823 | LR: 2.60e-04 | 697.8ms/step | 28.7 samples/s +Step 21860/44000 | Loss: 2.9926 | LR: 2.59e-04 | 701.8ms/step | 28.5 samples/s +Step 21870/44000 | Loss: 2.9657 | LR: 2.59e-04 | 699.8ms/step | 28.6 samples/s +Step 21880/44000 | Loss: 3.0428 | LR: 2.59e-04 | 696.1ms/step | 28.7 samples/s +Step 21890/44000 | Loss: 3.0073 | LR: 2.59e-04 | 698.6ms/step | 28.6 samples/s +Step 21900/44000 | Loss: 2.9626 | LR: 2.59e-04 | 702.6ms/step | 28.5 samples/s +Step 21910/44000 | Loss: 2.9495 | LR: 2.58e-04 | 702.9ms/step | 28.5 samples/s +Step 21920/44000 | Loss: 3.0499 | LR: 2.58e-04 | 697.5ms/step | 28.7 samples/s +Step 21930/44000 | Loss: 3.0427 | LR: 2.58e-04 | 699.6ms/step | 28.6 samples/s +Step 21940/44000 | Loss: 2.9695 | LR: 2.58e-04 | 700.4ms/step | 28.6 samples/s +Step 21950/44000 | Loss: 3.0022 | LR: 2.58e-04 | 698.6ms/step | 28.6 samples/s +Step 21960/44000 | Loss: 2.9704 | LR: 2.58e-04 | 700.7ms/step | 28.5 samples/s +Step 21970/44000 | Loss: 2.9535 | LR: 2.57e-04 | 699.5ms/step | 28.6 samples/s +Step 21980/44000 | Loss: 2.9888 | LR: 2.57e-04 | 700.9ms/step | 28.5 samples/s +Step 21990/44000 | Loss: 3.0041 | LR: 2.57e-04 | 702.2ms/step | 28.5 samples/s +Step 22000/44000 | Loss: 3.0197 | LR: 2.57e-04 | 701.3ms/step | 28.5 samples/s +Validation at step 22000: Loss = 3.0180, Perplexity = 20.45 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-22000 +Step 22010/44000 | Loss: 3.0124 | LR: 2.57e-04 | 1635.9ms/step | 12.2 samples/s +Step 22020/44000 | Loss: 2.9878 | LR: 2.56e-04 | 698.1ms/step | 28.6 samples/s +Step 22030/44000 | Loss: 2.9691 | LR: 2.56e-04 | 703.0ms/step | 28.4 samples/s +Step 22040/44000 | Loss: 3.0025 | LR: 2.56e-04 | 698.9ms/step | 28.6 samples/s +Step 22050/44000 | Loss: 3.0039 | LR: 2.56e-04 | 704.6ms/step | 28.4 samples/s +Step 22060/44000 | Loss: 2.9577 | LR: 2.56e-04 | 701.5ms/step | 28.5 samples/s +Step 22070/44000 | Loss: 2.9731 | LR: 2.56e-04 | 703.9ms/step | 28.4 samples/s +Step 22080/44000 | Loss: 3.0116 | LR: 2.55e-04 | 700.3ms/step | 28.6 samples/s +Step 22090/44000 | Loss: 3.0023 | LR: 2.55e-04 | 700.8ms/step | 28.5 samples/s +Step 22100/44000 | Loss: 2.9448 | LR: 2.55e-04 | 702.2ms/step | 28.5 samples/s +Step 22110/44000 | Loss: 3.0471 | LR: 2.55e-04 | 697.7ms/step | 28.7 samples/s +Step 22120/44000 | Loss: 2.9859 | LR: 2.55e-04 | 700.9ms/step | 28.5 samples/s +Step 22130/44000 | Loss: 3.0092 | LR: 2.54e-04 | 702.1ms/step | 28.5 samples/s +Step 22140/44000 | Loss: 3.0112 | LR: 2.54e-04 | 698.9ms/step | 28.6 samples/s +Step 22150/44000 | Loss: 2.9728 | LR: 2.54e-04 | 698.9ms/step | 28.6 samples/s +Step 22160/44000 | Loss: 2.9745 | LR: 2.54e-04 | 698.6ms/step | 28.6 samples/s +Step 22170/44000 | Loss: 2.9970 | LR: 2.54e-04 | 699.0ms/step | 28.6 samples/s +Step 22180/44000 | Loss: 2.9907 | LR: 2.54e-04 | 697.8ms/step | 28.7 samples/s +Step 22190/44000 | Loss: 3.0254 | LR: 2.53e-04 | 701.6ms/step | 28.5 samples/s +Step 22200/44000 | Loss: 3.0208 | LR: 2.53e-04 | 701.2ms/step | 28.5 samples/s +Step 22210/44000 | Loss: 2.9595 | LR: 2.53e-04 | 699.3ms/step | 28.6 samples/s +Step 22220/44000 | Loss: 3.0078 | LR: 2.53e-04 | 700.9ms/step | 28.5 samples/s +Step 22230/44000 | Loss: 2.9930 | LR: 2.53e-04 | 700.5ms/step | 28.6 samples/s +Step 22240/44000 | Loss: 3.0235 | LR: 2.52e-04 | 702.2ms/step | 28.5 samples/s +Step 22250/44000 | Loss: 3.0212 | LR: 2.52e-04 | 703.3ms/step | 28.4 samples/s +Validation at step 22250: Loss = 3.0166, Perplexity = 20.42 +Step 22260/44000 | Loss: 2.9719 | LR: 2.52e-04 | 1272.7ms/step | 15.7 samples/s +Step 22270/44000 | Loss: 3.0069 | LR: 2.52e-04 | 703.2ms/step | 28.4 samples/s +Step 22280/44000 | Loss: 3.0494 | LR: 2.52e-04 | 698.9ms/step | 28.6 samples/s +Step 22290/44000 | Loss: 3.0043 | LR: 2.52e-04 | 699.6ms/step | 28.6 samples/s +Step 22300/44000 | Loss: 3.0030 | LR: 2.51e-04 | 700.6ms/step | 28.5 samples/s +Step 22310/44000 | Loss: 2.9713 | LR: 2.51e-04 | 701.5ms/step | 28.5 samples/s +Step 22320/44000 | Loss: 2.9776 | LR: 2.51e-04 | 699.1ms/step | 28.6 samples/s +Step 22330/44000 | Loss: 2.9544 | LR: 2.51e-04 | 700.7ms/step | 28.5 samples/s +Step 22340/44000 | Loss: 2.9736 | LR: 2.51e-04 | 700.7ms/step | 28.5 samples/s +Step 22350/44000 | Loss: 3.0038 | LR: 2.50e-04 | 697.1ms/step | 28.7 samples/s +Step 22360/44000 | Loss: 3.0246 | LR: 2.50e-04 | 701.9ms/step | 28.5 samples/s +Step 22370/44000 | Loss: 2.9884 | LR: 2.50e-04 | 700.6ms/step | 28.5 samples/s +Step 22380/44000 | Loss: 2.9675 | LR: 2.50e-04 | 702.3ms/step | 28.5 samples/s +Step 22390/44000 | Loss: 2.9481 | LR: 2.50e-04 | 703.0ms/step | 28.4 samples/s +Step 22400/44000 | Loss: 3.0090 | LR: 2.50e-04 | 703.0ms/step | 28.4 samples/s +Step 22410/44000 | Loss: 3.0127 | LR: 2.49e-04 | 699.8ms/step | 28.6 samples/s +Step 22420/44000 | Loss: 3.0022 | LR: 2.49e-04 | 700.1ms/step | 28.6 samples/s +Step 22430/44000 | Loss: 2.9656 | LR: 2.49e-04 | 700.3ms/step | 28.6 samples/s +Step 22440/44000 | Loss: 2.9992 | LR: 2.49e-04 | 698.4ms/step | 28.6 samples/s +Step 22450/44000 | Loss: 2.9731 | LR: 2.49e-04 | 696.4ms/step | 28.7 samples/s +Step 22460/44000 | Loss: 3.0175 | LR: 2.48e-04 | 699.3ms/step | 28.6 samples/s +Step 22470/44000 | Loss: 2.9383 | LR: 2.48e-04 | 696.7ms/step | 28.7 samples/s +Step 22480/44000 | Loss: 3.0090 | LR: 2.48e-04 | 698.9ms/step | 28.6 samples/s +Step 22490/44000 | Loss: 2.9670 | LR: 2.48e-04 | 701.4ms/step | 28.5 samples/s +Step 22500/44000 | Loss: 2.9810 | LR: 2.48e-04 | 698.6ms/step | 28.6 samples/s +Validation at step 22500: Loss = 3.0124, Perplexity = 20.34 +Step 22510/44000 | Loss: 3.0297 | LR: 2.48e-04 | 1268.9ms/step | 15.8 samples/s +Step 22520/44000 | Loss: 2.9821 | LR: 2.47e-04 | 701.6ms/step | 28.5 samples/s +Step 22530/44000 | Loss: 2.9390 | LR: 2.47e-04 | 698.9ms/step | 28.6 samples/s +Step 22540/44000 | Loss: 2.9615 | LR: 2.47e-04 | 700.3ms/step | 28.6 samples/s +Step 22550/44000 | Loss: 3.0224 | LR: 2.47e-04 | 697.6ms/step | 28.7 samples/s +Step 22560/44000 | Loss: 2.9830 | LR: 2.47e-04 | 701.1ms/step | 28.5 samples/s +Step 22570/44000 | Loss: 2.9455 | LR: 2.46e-04 | 697.5ms/step | 28.7 samples/s +Step 22580/44000 | Loss: 2.9836 | LR: 2.46e-04 | 701.1ms/step | 28.5 samples/s +Step 22590/44000 | Loss: 2.9533 | LR: 2.46e-04 | 701.1ms/step | 28.5 samples/s +Step 22600/44000 | Loss: 2.9941 | LR: 2.46e-04 | 696.0ms/step | 28.7 samples/s +Step 22610/44000 | Loss: 2.9881 | LR: 2.46e-04 | 697.8ms/step | 28.7 samples/s +Step 22620/44000 | Loss: 2.9868 | LR: 2.46e-04 | 699.8ms/step | 28.6 samples/s +Step 22630/44000 | Loss: 2.9775 | LR: 2.45e-04 | 704.0ms/step | 28.4 samples/s +Step 22640/44000 | Loss: 3.0502 | LR: 2.45e-04 | 702.5ms/step | 28.5 samples/s +Step 22650/44000 | Loss: 2.9529 | LR: 2.45e-04 | 700.1ms/step | 28.6 samples/s +Step 22660/44000 | Loss: 3.0277 | LR: 2.45e-04 | 701.9ms/step | 28.5 samples/s +Step 22670/44000 | Loss: 2.9970 | LR: 2.45e-04 | 700.6ms/step | 28.5 samples/s +Step 22680/44000 | Loss: 2.9731 | LR: 2.44e-04 | 702.8ms/step | 28.5 samples/s +Step 22690/44000 | Loss: 2.9889 | LR: 2.44e-04 | 702.2ms/step | 28.5 samples/s +Step 22700/44000 | Loss: 2.9592 | LR: 2.44e-04 | 705.5ms/step | 28.3 samples/s +Step 22710/44000 | Loss: 2.9887 | LR: 2.44e-04 | 700.8ms/step | 28.5 samples/s +Step 22720/44000 | Loss: 2.9569 | LR: 2.44e-04 | 699.8ms/step | 28.6 samples/s +Step 22730/44000 | Loss: 2.9538 | LR: 2.44e-04 | 700.5ms/step | 28.6 samples/s +Step 22740/44000 | Loss: 2.9765 | LR: 2.43e-04 | 700.7ms/step | 28.5 samples/s +Step 22750/44000 | Loss: 3.0195 | LR: 2.43e-04 | 703.8ms/step | 28.4 samples/s +Validation at step 22750: Loss = 3.0125, Perplexity = 20.34 +Step 22760/44000 | Loss: 3.0515 | LR: 2.43e-04 | 1272.3ms/step | 15.7 samples/s +Step 22770/44000 | Loss: 2.9716 | LR: 2.43e-04 | 704.8ms/step | 28.4 samples/s +Step 22780/44000 | Loss: 3.0351 | LR: 2.43e-04 | 703.9ms/step | 28.4 samples/s +Step 22790/44000 | Loss: 2.9880 | LR: 2.42e-04 | 703.5ms/step | 28.4 samples/s +Step 22800/44000 | Loss: 2.9604 | LR: 2.42e-04 | 699.7ms/step | 28.6 samples/s +Step 22810/44000 | Loss: 2.9943 | LR: 2.42e-04 | 701.1ms/step | 28.5 samples/s +Step 22820/44000 | Loss: 2.9348 | LR: 2.42e-04 | 699.3ms/step | 28.6 samples/s +Step 22830/44000 | Loss: 2.9629 | LR: 2.42e-04 | 700.8ms/step | 28.5 samples/s +Step 22840/44000 | Loss: 3.0395 | LR: 2.42e-04 | 700.3ms/step | 28.6 samples/s +Step 22850/44000 | Loss: 2.9360 | LR: 2.41e-04 | 700.8ms/step | 28.5 samples/s +Step 22860/44000 | Loss: 3.0225 | LR: 2.41e-04 | 702.7ms/step | 28.5 samples/s +Step 22870/44000 | Loss: 2.9774 | LR: 2.41e-04 | 699.9ms/step | 28.6 samples/s +Step 22880/44000 | Loss: 3.0259 | LR: 2.41e-04 | 702.5ms/step | 28.5 samples/s +Step 22890/44000 | Loss: 2.9762 | LR: 2.41e-04 | 698.9ms/step | 28.6 samples/s +Step 22900/44000 | Loss: 2.9755 | LR: 2.40e-04 | 699.0ms/step | 28.6 samples/s +Step 22910/44000 | Loss: 2.9704 | LR: 2.40e-04 | 699.3ms/step | 28.6 samples/s +Step 22920/44000 | Loss: 2.9807 | LR: 2.40e-04 | 701.6ms/step | 28.5 samples/s +Step 22930/44000 | Loss: 2.9430 | LR: 2.40e-04 | 698.4ms/step | 28.6 samples/s +Step 22940/44000 | Loss: 2.9546 | LR: 2.40e-04 | 697.2ms/step | 28.7 samples/s +Step 22950/44000 | Loss: 2.9552 | LR: 2.40e-04 | 698.8ms/step | 28.6 samples/s +Step 22960/44000 | Loss: 3.0133 | LR: 2.39e-04 | 701.4ms/step | 28.5 samples/s +Step 22970/44000 | Loss: 2.9580 | LR: 2.39e-04 | 700.3ms/step | 28.6 samples/s +Step 22980/44000 | Loss: 2.9567 | LR: 2.39e-04 | 701.3ms/step | 28.5 samples/s +Step 22990/44000 | Loss: 2.9336 | LR: 2.39e-04 | 703.8ms/step | 28.4 samples/s +Step 23000/44000 | Loss: 2.9709 | LR: 2.39e-04 | 697.6ms/step | 28.7 samples/s +Validation at step 23000: Loss = 3.0102, Perplexity = 20.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-23000 +Step 23010/44000 | Loss: 2.9711 | LR: 2.38e-04 | 1619.6ms/step | 12.3 samples/s +Step 23020/44000 | Loss: 3.0062 | LR: 2.38e-04 | 697.7ms/step | 28.7 samples/s +Step 23030/44000 | Loss: 3.0171 | LR: 2.38e-04 | 697.0ms/step | 28.7 samples/s +Step 23040/44000 | Loss: 2.9868 | LR: 2.38e-04 | 699.6ms/step | 28.6 samples/s +Step 23050/44000 | Loss: 2.9757 | LR: 2.38e-04 | 696.8ms/step | 28.7 samples/s +Step 23060/44000 | Loss: 3.0278 | LR: 2.38e-04 | 699.5ms/step | 28.6 samples/s +Step 23070/44000 | Loss: 3.0016 | LR: 2.37e-04 | 701.2ms/step | 28.5 samples/s +Step 23080/44000 | Loss: 2.9534 | LR: 2.37e-04 | 698.8ms/step | 28.6 samples/s +Step 23090/44000 | Loss: 2.9456 | LR: 2.37e-04 | 701.5ms/step | 28.5 samples/s +Step 23100/44000 | Loss: 2.9667 | LR: 2.37e-04 | 702.9ms/step | 28.5 samples/s +Step 23110/44000 | Loss: 2.9591 | LR: 2.37e-04 | 700.6ms/step | 28.5 samples/s +Step 23120/44000 | Loss: 2.9880 | LR: 2.36e-04 | 698.5ms/step | 28.6 samples/s +Step 23130/44000 | Loss: 2.9788 | LR: 2.36e-04 | 701.7ms/step | 28.5 samples/s +Step 23140/44000 | Loss: 2.9560 | LR: 2.36e-04 | 702.2ms/step | 28.5 samples/s +Step 23150/44000 | Loss: 2.9433 | LR: 2.36e-04 | 698.0ms/step | 28.7 samples/s +Step 23160/44000 | Loss: 3.0425 | LR: 2.36e-04 | 698.7ms/step | 28.6 samples/s +Step 23170/44000 | Loss: 2.9595 | LR: 2.36e-04 | 699.1ms/step | 28.6 samples/s +Step 23180/44000 | Loss: 2.9750 | LR: 2.35e-04 | 698.6ms/step | 28.6 samples/s +Step 23190/44000 | Loss: 2.9755 | LR: 2.35e-04 | 700.8ms/step | 28.5 samples/s +Step 23200/44000 | Loss: 3.0121 | LR: 2.35e-04 | 698.8ms/step | 28.6 samples/s +Step 23210/44000 | Loss: 2.9949 | LR: 2.35e-04 | 701.2ms/step | 28.5 samples/s +Step 23220/44000 | Loss: 2.9491 | LR: 2.35e-04 | 700.5ms/step | 28.6 samples/s +Step 23230/44000 | Loss: 2.9909 | LR: 2.34e-04 | 698.6ms/step | 28.6 samples/s +Step 23240/44000 | Loss: 2.9814 | LR: 2.34e-04 | 700.5ms/step | 28.6 samples/s +Step 23250/44000 | Loss: 2.9427 | LR: 2.34e-04 | 700.6ms/step | 28.5 samples/s +Validation at step 23250: Loss = 3.0082, Perplexity = 20.25 +Step 23260/44000 | Loss: 2.9894 | LR: 2.34e-04 | 1272.0ms/step | 15.7 samples/s +Step 23270/44000 | Loss: 2.9924 | LR: 2.34e-04 | 699.0ms/step | 28.6 samples/s +Step 23280/44000 | Loss: 2.9485 | LR: 2.34e-04 | 699.7ms/step | 28.6 samples/s +Step 23290/44000 | Loss: 2.9896 | LR: 2.33e-04 | 699.8ms/step | 28.6 samples/s +Step 23300/44000 | Loss: 2.9861 | LR: 2.33e-04 | 698.3ms/step | 28.6 samples/s +Step 23310/44000 | Loss: 2.9772 | LR: 2.33e-04 | 701.1ms/step | 28.5 samples/s +Step 23320/44000 | Loss: 2.9904 | LR: 2.33e-04 | 698.0ms/step | 28.7 samples/s +Step 23330/44000 | Loss: 3.0181 | LR: 2.33e-04 | 699.1ms/step | 28.6 samples/s +Step 23340/44000 | Loss: 3.0410 | LR: 2.32e-04 | 698.7ms/step | 28.6 samples/s +Step 23350/44000 | Loss: 2.9840 | LR: 2.32e-04 | 700.5ms/step | 28.6 samples/s +Step 23360/44000 | Loss: 3.0115 | LR: 2.32e-04 | 697.9ms/step | 28.7 samples/s +Step 23370/44000 | Loss: 3.0221 | LR: 2.32e-04 | 700.0ms/step | 28.6 samples/s +Step 23380/44000 | Loss: 2.9883 | LR: 2.32e-04 | 700.3ms/step | 28.6 samples/s +Step 23390/44000 | Loss: 2.9777 | LR: 2.32e-04 | 700.8ms/step | 28.5 samples/s +Step 23400/44000 | Loss: 2.9890 | LR: 2.31e-04 | 695.4ms/step | 28.8 samples/s +Step 23410/44000 | Loss: 2.9813 | LR: 2.31e-04 | 696.6ms/step | 28.7 samples/s +Step 23420/44000 | Loss: 2.9518 | LR: 2.31e-04 | 699.8ms/step | 28.6 samples/s +Step 23430/44000 | Loss: 2.9719 | LR: 2.31e-04 | 698.0ms/step | 28.7 samples/s +Step 23440/44000 | Loss: 2.9844 | LR: 2.31e-04 | 702.7ms/step | 28.5 samples/s +Step 23450/44000 | Loss: 2.9620 | LR: 2.30e-04 | 699.5ms/step | 28.6 samples/s +Step 23460/44000 | Loss: 3.0058 | LR: 2.30e-04 | 701.9ms/step | 28.5 samples/s +Step 23470/44000 | Loss: 3.0055 | LR: 2.30e-04 | 702.3ms/step | 28.5 samples/s +Step 23480/44000 | Loss: 3.0075 | LR: 2.30e-04 | 699.3ms/step | 28.6 samples/s +Step 23490/44000 | Loss: 3.0252 | LR: 2.30e-04 | 699.9ms/step | 28.6 samples/s +Step 23500/44000 | Loss: 2.9925 | LR: 2.30e-04 | 699.0ms/step | 28.6 samples/s +Validation at step 23500: Loss = 3.0064, Perplexity = 20.21 +Step 23510/44000 | Loss: 2.9941 | LR: 2.29e-04 | 1267.9ms/step | 15.8 samples/s +Step 23520/44000 | Loss: 2.9725 | LR: 2.29e-04 | 699.1ms/step | 28.6 samples/s +Step 23530/44000 | Loss: 2.9787 | LR: 2.29e-04 | 698.3ms/step | 28.6 samples/s +Step 23540/44000 | Loss: 3.0202 | LR: 2.29e-04 | 699.3ms/step | 28.6 samples/s +Step 23550/44000 | Loss: 2.9814 | LR: 2.29e-04 | 697.6ms/step | 28.7 samples/s +Step 23560/44000 | Loss: 2.9781 | LR: 2.29e-04 | 697.3ms/step | 28.7 samples/s +Step 23570/44000 | Loss: 2.9865 | LR: 2.28e-04 | 698.8ms/step | 28.6 samples/s +Step 23580/44000 | Loss: 2.9930 | LR: 2.28e-04 | 699.7ms/step | 28.6 samples/s +Step 23590/44000 | Loss: 2.9864 | LR: 2.28e-04 | 700.6ms/step | 28.5 samples/s +Step 23600/44000 | Loss: 2.9646 | LR: 2.28e-04 | 697.8ms/step | 28.7 samples/s +Step 23610/44000 | Loss: 2.9685 | LR: 2.28e-04 | 700.2ms/step | 28.6 samples/s +Step 23620/44000 | Loss: 2.9832 | LR: 2.27e-04 | 698.1ms/step | 28.6 samples/s +Step 23630/44000 | Loss: 2.9754 | LR: 2.27e-04 | 697.1ms/step | 28.7 samples/s +Step 23640/44000 | Loss: 3.0083 | LR: 2.27e-04 | 697.5ms/step | 28.7 samples/s +Step 23650/44000 | Loss: 2.9524 | LR: 2.27e-04 | 697.3ms/step | 28.7 samples/s +Step 23660/44000 | Loss: 3.0179 | LR: 2.27e-04 | 700.7ms/step | 28.5 samples/s +Step 23670/44000 | Loss: 2.9761 | LR: 2.27e-04 | 696.3ms/step | 28.7 samples/s +Step 23680/44000 | Loss: 2.9562 | LR: 2.26e-04 | 700.7ms/step | 28.5 samples/s +Step 23690/44000 | Loss: 2.9862 | LR: 2.26e-04 | 699.1ms/step | 28.6 samples/s +Step 23700/44000 | Loss: 2.9888 | LR: 2.26e-04 | 696.9ms/step | 28.7 samples/s +Step 23710/44000 | Loss: 3.0304 | LR: 2.26e-04 | 696.9ms/step | 28.7 samples/s +Step 23720/44000 | Loss: 2.9654 | LR: 2.26e-04 | 701.2ms/step | 28.5 samples/s +Step 23730/44000 | Loss: 2.9638 | LR: 2.25e-04 | 694.8ms/step | 28.8 samples/s +Step 23740/44000 | Loss: 2.9369 | LR: 2.25e-04 | 697.9ms/step | 28.7 samples/s +Step 23750/44000 | Loss: 2.9995 | LR: 2.25e-04 | 700.1ms/step | 28.6 samples/s +Validation at step 23750: Loss = 2.9997, Perplexity = 20.08 +Step 23760/44000 | Loss: 2.9624 | LR: 2.25e-04 | 1268.4ms/step | 15.8 samples/s +Step 23770/44000 | Loss: 2.9803 | LR: 2.25e-04 | 700.6ms/step | 28.5 samples/s +Step 23780/44000 | Loss: 3.0099 | LR: 2.25e-04 | 697.2ms/step | 28.7 samples/s +Step 23790/44000 | Loss: 2.9540 | LR: 2.24e-04 | 698.5ms/step | 28.6 samples/s +Step 23800/44000 | Loss: 2.9942 | LR: 2.24e-04 | 697.8ms/step | 28.7 samples/s +Step 23810/44000 | Loss: 2.9615 | LR: 2.24e-04 | 701.4ms/step | 28.5 samples/s +Step 23820/44000 | Loss: 2.9673 | LR: 2.24e-04 | 701.5ms/step | 28.5 samples/s +Step 23830/44000 | Loss: 3.0266 | LR: 2.24e-04 | 700.8ms/step | 28.5 samples/s +Step 23840/44000 | Loss: 2.9890 | LR: 2.23e-04 | 697.7ms/step | 28.7 samples/s +Step 23850/44000 | Loss: 2.9723 | LR: 2.23e-04 | 696.6ms/step | 28.7 samples/s +Step 23860/44000 | Loss: 2.9856 | LR: 2.23e-04 | 698.7ms/step | 28.6 samples/s +Step 23870/44000 | Loss: 3.0020 | LR: 2.23e-04 | 698.1ms/step | 28.6 samples/s +Step 23880/44000 | Loss: 3.0067 | LR: 2.23e-04 | 698.4ms/step | 28.6 samples/s +Step 23890/44000 | Loss: 2.9836 | LR: 2.23e-04 | 700.2ms/step | 28.6 samples/s +Step 23900/44000 | Loss: 2.9960 | LR: 2.22e-04 | 696.9ms/step | 28.7 samples/s +Step 23910/44000 | Loss: 2.9836 | LR: 2.22e-04 | 699.6ms/step | 28.6 samples/s +Step 23920/44000 | Loss: 2.9784 | LR: 2.22e-04 | 696.4ms/step | 28.7 samples/s +Step 23930/44000 | Loss: 2.9882 | LR: 2.22e-04 | 700.7ms/step | 28.5 samples/s +Step 23940/44000 | Loss: 2.9789 | LR: 2.22e-04 | 699.0ms/step | 28.6 samples/s +Step 23950/44000 | Loss: 2.9850 | LR: 2.21e-04 | 698.1ms/step | 28.7 samples/s +Step 23960/44000 | Loss: 2.9794 | LR: 2.21e-04 | 699.1ms/step | 28.6 samples/s +Step 23970/44000 | Loss: 2.9485 | LR: 2.21e-04 | 701.7ms/step | 28.5 samples/s +Step 23980/44000 | Loss: 2.9746 | LR: 2.21e-04 | 697.7ms/step | 28.7 samples/s +Step 23990/44000 | Loss: 2.9564 | LR: 2.21e-04 | 699.9ms/step | 28.6 samples/s +Step 24000/44000 | Loss: 2.9753 | LR: 2.21e-04 | 697.6ms/step | 28.7 samples/s +Validation at step 24000: Loss = 3.0008, Perplexity = 20.10 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-24000 +Step 24010/44000 | Loss: 2.9835 | LR: 2.20e-04 | 1573.9ms/step | 12.7 samples/s +Step 24020/44000 | Loss: 3.0138 | LR: 2.20e-04 | 700.3ms/step | 28.6 samples/s +Step 24030/44000 | Loss: 2.9946 | LR: 2.20e-04 | 698.2ms/step | 28.6 samples/s +Step 24040/44000 | Loss: 2.9793 | LR: 2.20e-04 | 698.7ms/step | 28.6 samples/s +Step 24050/44000 | Loss: 3.0184 | LR: 2.20e-04 | 697.6ms/step | 28.7 samples/s +Step 24060/44000 | Loss: 2.9655 | LR: 2.19e-04 | 697.5ms/step | 28.7 samples/s +Step 24070/44000 | Loss: 2.9661 | LR: 2.19e-04 | 700.0ms/step | 28.6 samples/s +Step 24080/44000 | Loss: 3.0168 | LR: 2.19e-04 | 699.8ms/step | 28.6 samples/s +Step 24090/44000 | Loss: 2.9737 | LR: 2.19e-04 | 700.2ms/step | 28.6 samples/s +Step 24100/44000 | Loss: 2.9651 | LR: 2.19e-04 | 695.8ms/step | 28.7 samples/s +Step 24110/44000 | Loss: 2.9419 | LR: 2.19e-04 | 696.1ms/step | 28.7 samples/s +Step 24120/44000 | Loss: 2.9245 | LR: 2.18e-04 | 696.4ms/step | 28.7 samples/s +Step 24130/44000 | Loss: 2.9719 | LR: 2.18e-04 | 695.3ms/step | 28.8 samples/s +Step 24140/44000 | Loss: 2.9390 | LR: 2.18e-04 | 694.9ms/step | 28.8 samples/s +Step 24150/44000 | Loss: 3.0007 | LR: 2.18e-04 | 699.2ms/step | 28.6 samples/s +Step 24160/44000 | Loss: 2.9629 | LR: 2.18e-04 | 696.1ms/step | 28.7 samples/s +Step 24170/44000 | Loss: 2.9621 | LR: 2.17e-04 | 696.5ms/step | 28.7 samples/s +Step 24180/44000 | Loss: 2.9787 | LR: 2.17e-04 | 695.6ms/step | 28.8 samples/s +Step 24190/44000 | Loss: 2.9622 | LR: 2.17e-04 | 697.7ms/step | 28.7 samples/s +Step 24200/44000 | Loss: 2.9729 | LR: 2.17e-04 | 700.0ms/step | 28.6 samples/s +Step 24210/44000 | Loss: 2.9545 | LR: 2.17e-04 | 697.5ms/step | 28.7 samples/s +Step 24220/44000 | Loss: 2.9718 | LR: 2.17e-04 | 697.9ms/step | 28.7 samples/s +Step 24230/44000 | Loss: 2.9397 | LR: 2.16e-04 | 697.0ms/step | 28.7 samples/s +Step 24240/44000 | Loss: 2.9463 | LR: 2.16e-04 | 695.5ms/step | 28.8 samples/s +Step 24250/44000 | Loss: 2.9576 | LR: 2.16e-04 | 694.3ms/step | 28.8 samples/s +Validation at step 24250: Loss = 2.9978, Perplexity = 20.04 +Step 24260/44000 | Loss: 2.9588 | LR: 2.16e-04 | 1265.9ms/step | 15.8 samples/s +Step 24270/44000 | Loss: 2.9403 | LR: 2.16e-04 | 697.8ms/step | 28.7 samples/s +Step 24280/44000 | Loss: 3.0085 | LR: 2.16e-04 | 694.3ms/step | 28.8 samples/s +Step 24290/44000 | Loss: 2.9881 | LR: 2.15e-04 | 697.3ms/step | 28.7 samples/s +Step 24300/44000 | Loss: 2.9514 | LR: 2.15e-04 | 698.8ms/step | 28.6 samples/s +Step 24310/44000 | Loss: 3.0006 | LR: 2.15e-04 | 693.8ms/step | 28.8 samples/s +Step 24320/44000 | Loss: 2.9366 | LR: 2.15e-04 | 699.9ms/step | 28.6 samples/s +Step 24330/44000 | Loss: 3.0186 | LR: 2.15e-04 | 696.2ms/step | 28.7 samples/s +Step 24340/44000 | Loss: 2.9539 | LR: 2.14e-04 | 697.0ms/step | 28.7 samples/s +Step 24350/44000 | Loss: 2.9695 | LR: 2.14e-04 | 696.3ms/step | 28.7 samples/s +Step 24360/44000 | Loss: 2.9497 | LR: 2.14e-04 | 697.7ms/step | 28.7 samples/s +Step 24370/44000 | Loss: 2.9447 | LR: 2.14e-04 | 696.7ms/step | 28.7 samples/s +Step 24380/44000 | Loss: 2.9683 | LR: 2.14e-04 | 697.1ms/step | 28.7 samples/s +Step 24390/44000 | Loss: 2.9844 | LR: 2.14e-04 | 698.2ms/step | 28.6 samples/s +Step 24400/44000 | Loss: 2.9336 | LR: 2.13e-04 | 697.7ms/step | 28.7 samples/s +Step 24410/44000 | Loss: 2.9503 | LR: 2.13e-04 | 701.9ms/step | 28.5 samples/s +Step 24420/44000 | Loss: 2.9631 | LR: 2.13e-04 | 699.5ms/step | 28.6 samples/s +Step 24430/44000 | Loss: 2.9973 | LR: 2.13e-04 | 699.3ms/step | 28.6 samples/s +Step 24440/44000 | Loss: 2.9881 | LR: 2.13e-04 | 694.0ms/step | 28.8 samples/s +Step 24450/44000 | Loss: 2.9808 | LR: 2.12e-04 | 696.3ms/step | 28.7 samples/s +Step 24460/44000 | Loss: 2.9473 | LR: 2.12e-04 | 693.7ms/step | 28.8 samples/s +Step 24470/44000 | Loss: 2.9769 | LR: 2.12e-04 | 696.3ms/step | 28.7 samples/s +Step 24480/44000 | Loss: 3.0132 | LR: 2.12e-04 | 699.7ms/step | 28.6 samples/s +Step 24490/44000 | Loss: 2.9848 | LR: 2.12e-04 | 699.4ms/step | 28.6 samples/s +Step 24500/44000 | Loss: 2.9669 | LR: 2.12e-04 | 697.6ms/step | 28.7 samples/s +Validation at step 24500: Loss = 2.9991, Perplexity = 20.07 +Step 24510/44000 | Loss: 3.0004 | LR: 2.11e-04 | 1268.8ms/step | 15.8 samples/s +Step 24520/44000 | Loss: 2.9651 | LR: 2.11e-04 | 694.1ms/step | 28.8 samples/s +Step 24530/44000 | Loss: 2.9750 | LR: 2.11e-04 | 694.8ms/step | 28.8 samples/s +Step 24540/44000 | Loss: 2.9406 | LR: 2.11e-04 | 701.3ms/step | 28.5 samples/s +Step 24550/44000 | Loss: 2.9912 | LR: 2.11e-04 | 698.6ms/step | 28.6 samples/s +Step 24560/44000 | Loss: 2.9760 | LR: 2.10e-04 | 697.9ms/step | 28.7 samples/s +Step 24570/44000 | Loss: 2.9659 | LR: 2.10e-04 | 700.2ms/step | 28.6 samples/s +Step 24580/44000 | Loss: 2.9319 | LR: 2.10e-04 | 700.1ms/step | 28.6 samples/s +Step 24590/44000 | Loss: 2.9957 | LR: 2.10e-04 | 698.9ms/step | 28.6 samples/s +Step 24600/44000 | Loss: 2.9841 | LR: 2.10e-04 | 698.2ms/step | 28.6 samples/s +Step 24610/44000 | Loss: 2.9986 | LR: 2.10e-04 | 698.2ms/step | 28.6 samples/s +Step 24620/44000 | Loss: 2.9001 | LR: 2.09e-04 | 698.7ms/step | 28.6 samples/s +Step 24630/44000 | Loss: 2.9724 | LR: 2.09e-04 | 698.0ms/step | 28.7 samples/s +Step 24640/44000 | Loss: 2.9750 | LR: 2.09e-04 | 700.1ms/step | 28.6 samples/s +Step 24650/44000 | Loss: 2.9286 | LR: 2.09e-04 | 698.0ms/step | 28.7 samples/s +Step 24660/44000 | Loss: 2.9742 | LR: 2.09e-04 | 695.9ms/step | 28.7 samples/s +Step 24670/44000 | Loss: 2.9776 | LR: 2.09e-04 | 700.1ms/step | 28.6 samples/s +Step 24680/44000 | Loss: 2.9618 | LR: 2.08e-04 | 699.8ms/step | 28.6 samples/s +Step 24690/44000 | Loss: 2.9430 | LR: 2.08e-04 | 697.9ms/step | 28.7 samples/s +Step 24700/44000 | Loss: 2.9632 | LR: 2.08e-04 | 699.6ms/step | 28.6 samples/s +Step 24710/44000 | Loss: 3.0330 | LR: 2.08e-04 | 700.7ms/step | 28.5 samples/s +Step 24720/44000 | Loss: 2.9678 | LR: 2.08e-04 | 697.7ms/step | 28.7 samples/s +Step 24730/44000 | Loss: 2.9455 | LR: 2.07e-04 | 696.3ms/step | 28.7 samples/s +Step 24740/44000 | Loss: 2.9767 | LR: 2.07e-04 | 696.6ms/step | 28.7 samples/s +Step 24750/44000 | Loss: 2.9552 | LR: 2.07e-04 | 695.0ms/step | 28.8 samples/s +Validation at step 24750: Loss = 2.9944, Perplexity = 19.97 +Step 24760/44000 | Loss: 3.0160 | LR: 2.07e-04 | 1269.8ms/step | 15.8 samples/s +Step 24770/44000 | Loss: 2.9606 | LR: 2.07e-04 | 695.7ms/step | 28.7 samples/s +Step 24780/44000 | Loss: 2.9863 | LR: 2.07e-04 | 696.9ms/step | 28.7 samples/s +Step 24790/44000 | Loss: 2.9709 | LR: 2.06e-04 | 695.4ms/step | 28.8 samples/s +Step 24800/44000 | Loss: 2.9311 | LR: 2.06e-04 | 698.6ms/step | 28.6 samples/s +Step 24810/44000 | Loss: 2.9391 | LR: 2.06e-04 | 699.1ms/step | 28.6 samples/s +Step 24820/44000 | Loss: 2.9691 | LR: 2.06e-04 | 700.2ms/step | 28.6 samples/s +Step 24830/44000 | Loss: 2.9814 | LR: 2.06e-04 | 699.8ms/step | 28.6 samples/s +Step 24840/44000 | Loss: 2.9788 | LR: 2.05e-04 | 698.5ms/step | 28.6 samples/s +Step 24850/44000 | Loss: 2.9721 | LR: 2.05e-04 | 701.7ms/step | 28.5 samples/s +Step 24860/44000 | Loss: 3.0118 | LR: 2.05e-04 | 697.4ms/step | 28.7 samples/s +Step 24870/44000 | Loss: 3.0245 | LR: 2.05e-04 | 698.2ms/step | 28.6 samples/s +Step 24880/44000 | Loss: 2.9384 | LR: 2.05e-04 | 700.5ms/step | 28.6 samples/s +Step 24890/44000 | Loss: 2.9695 | LR: 2.05e-04 | 698.7ms/step | 28.6 samples/s +Step 24900/44000 | Loss: 2.9363 | LR: 2.04e-04 | 699.0ms/step | 28.6 samples/s +Step 24910/44000 | Loss: 3.0000 | LR: 2.04e-04 | 699.2ms/step | 28.6 samples/s +Step 24920/44000 | Loss: 2.9514 | LR: 2.04e-04 | 698.0ms/step | 28.7 samples/s +Step 24930/44000 | Loss: 2.9388 | LR: 2.04e-04 | 699.0ms/step | 28.6 samples/s +Step 24940/44000 | Loss: 2.9926 | LR: 2.04e-04 | 698.9ms/step | 28.6 samples/s +Step 24950/44000 | Loss: 2.9631 | LR: 2.04e-04 | 701.8ms/step | 28.5 samples/s +Step 24960/44000 | Loss: 2.9844 | LR: 2.03e-04 | 701.3ms/step | 28.5 samples/s +Step 24970/44000 | Loss: 3.0070 | LR: 2.03e-04 | 698.6ms/step | 28.6 samples/s +Step 24980/44000 | Loss: 2.9721 | LR: 2.03e-04 | 698.2ms/step | 28.6 samples/s +Step 24990/44000 | Loss: 3.0161 | LR: 2.03e-04 | 697.8ms/step | 28.7 samples/s +Step 25000/44000 | Loss: 2.9968 | LR: 2.03e-04 | 700.4ms/step | 28.6 samples/s +Validation at step 25000: Loss = 2.9938, Perplexity = 19.96 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-25000 +Step 25010/44000 | Loss: 2.9750 | LR: 2.02e-04 | 1597.5ms/step | 12.5 samples/s +Step 25020/44000 | Loss: 3.0173 | LR: 2.02e-04 | 699.7ms/step | 28.6 samples/s +Step 25030/44000 | Loss: 2.9498 | LR: 2.02e-04 | 698.9ms/step | 28.6 samples/s +Step 25040/44000 | Loss: 3.0075 | LR: 2.02e-04 | 703.7ms/step | 28.4 samples/s +Step 25050/44000 | Loss: 3.0192 | LR: 2.02e-04 | 697.2ms/step | 28.7 samples/s +Step 25060/44000 | Loss: 2.9503 | LR: 2.02e-04 | 700.5ms/step | 28.6 samples/s +Step 25070/44000 | Loss: 2.9734 | LR: 2.01e-04 | 701.0ms/step | 28.5 samples/s +Step 25080/44000 | Loss: 2.9952 | LR: 2.01e-04 | 700.8ms/step | 28.5 samples/s +Step 25090/44000 | Loss: 3.0032 | LR: 2.01e-04 | 698.7ms/step | 28.6 samples/s +Step 25100/44000 | Loss: 2.9481 | LR: 2.01e-04 | 699.6ms/step | 28.6 samples/s +Step 25110/44000 | Loss: 2.9779 | LR: 2.01e-04 | 700.3ms/step | 28.6 samples/s +Step 25120/44000 | Loss: 2.9954 | LR: 2.00e-04 | 699.9ms/step | 28.6 samples/s +Step 25130/44000 | Loss: 2.9938 | LR: 2.00e-04 | 701.9ms/step | 28.5 samples/s +Step 25140/44000 | Loss: 2.9468 | LR: 2.00e-04 | 702.3ms/step | 28.5 samples/s +Step 25150/44000 | Loss: 2.9710 | LR: 2.00e-04 | 700.0ms/step | 28.6 samples/s +Step 25160/44000 | Loss: 2.9996 | LR: 2.00e-04 | 699.9ms/step | 28.6 samples/s +Step 25170/44000 | Loss: 3.0237 | LR: 2.00e-04 | 703.2ms/step | 28.4 samples/s +Step 25180/44000 | Loss: 2.9489 | LR: 1.99e-04 | 700.4ms/step | 28.6 samples/s +Step 25190/44000 | Loss: 3.0068 | LR: 1.99e-04 | 698.4ms/step | 28.6 samples/s +Step 25200/44000 | Loss: 2.9671 | LR: 1.99e-04 | 700.8ms/step | 28.5 samples/s +Step 25210/44000 | Loss: 2.9803 | LR: 1.99e-04 | 703.6ms/step | 28.4 samples/s +Step 25220/44000 | Loss: 2.9896 | LR: 1.99e-04 | 700.1ms/step | 28.6 samples/s +Step 25230/44000 | Loss: 2.9136 | LR: 1.99e-04 | 698.7ms/step | 28.6 samples/s +Step 25240/44000 | Loss: 2.9445 | LR: 1.98e-04 | 700.5ms/step | 28.6 samples/s +Step 25250/44000 | Loss: 2.9823 | LR: 1.98e-04 | 699.7ms/step | 28.6 samples/s +Validation at step 25250: Loss = 2.9905, Perplexity = 19.90 +Step 25260/44000 | Loss: 2.9520 | LR: 1.98e-04 | 1272.1ms/step | 15.7 samples/s +Step 25270/44000 | Loss: 2.9568 | LR: 1.98e-04 | 700.9ms/step | 28.5 samples/s +Step 25280/44000 | Loss: 2.9512 | LR: 1.98e-04 | 701.0ms/step | 28.5 samples/s +Step 25290/44000 | Loss: 2.9201 | LR: 1.97e-04 | 700.4ms/step | 28.6 samples/s +Step 25300/44000 | Loss: 2.9734 | LR: 1.97e-04 | 700.4ms/step | 28.6 samples/s +Step 25310/44000 | Loss: 2.9587 | LR: 1.97e-04 | 696.9ms/step | 28.7 samples/s +Step 25320/44000 | Loss: 2.9566 | LR: 1.97e-04 | 700.1ms/step | 28.6 samples/s +Step 25330/44000 | Loss: 2.9411 | LR: 1.97e-04 | 700.0ms/step | 28.6 samples/s +Step 25340/44000 | Loss: 2.9643 | LR: 1.97e-04 | 699.6ms/step | 28.6 samples/s +Step 25350/44000 | Loss: 2.9696 | LR: 1.96e-04 | 700.2ms/step | 28.6 samples/s +Step 25360/44000 | Loss: 2.9556 | LR: 1.96e-04 | 699.4ms/step | 28.6 samples/s +Step 25370/44000 | Loss: 2.9339 | LR: 1.96e-04 | 702.3ms/step | 28.5 samples/s +Step 25380/44000 | Loss: 2.9654 | LR: 1.96e-04 | 700.9ms/step | 28.5 samples/s +Step 25390/44000 | Loss: 2.9224 | LR: 1.96e-04 | 697.8ms/step | 28.7 samples/s +Step 25400/44000 | Loss: 2.9847 | LR: 1.96e-04 | 700.8ms/step | 28.5 samples/s +Step 25410/44000 | Loss: 2.9642 | LR: 1.95e-04 | 700.0ms/step | 28.6 samples/s +Step 25420/44000 | Loss: 2.9317 | LR: 1.95e-04 | 703.6ms/step | 28.4 samples/s +Step 25430/44000 | Loss: 2.9149 | LR: 1.95e-04 | 699.6ms/step | 28.6 samples/s +Step 25440/44000 | Loss: 2.9738 | LR: 1.95e-04 | 698.8ms/step | 28.6 samples/s +Step 25450/44000 | Loss: 2.9291 | LR: 1.95e-04 | 698.6ms/step | 28.6 samples/s +Step 25460/44000 | Loss: 2.9554 | LR: 1.94e-04 | 703.1ms/step | 28.4 samples/s +Step 25470/44000 | Loss: 2.9339 | LR: 1.94e-04 | 699.9ms/step | 28.6 samples/s +Step 25480/44000 | Loss: 2.9188 | LR: 1.94e-04 | 701.9ms/step | 28.5 samples/s +Step 25490/44000 | Loss: 2.9298 | LR: 1.94e-04 | 701.9ms/step | 28.5 samples/s +Step 25500/44000 | Loss: 2.9539 | LR: 1.94e-04 | 700.6ms/step | 28.5 samples/s +Validation at step 25500: Loss = 2.9897, Perplexity = 19.88 +Step 25510/44000 | Loss: 2.9737 | LR: 1.94e-04 | 1274.2ms/step | 15.7 samples/s +Step 25520/44000 | Loss: 2.9935 | LR: 1.93e-04 | 700.9ms/step | 28.5 samples/s +Step 25530/44000 | Loss: 2.9623 | LR: 1.93e-04 | 699.2ms/step | 28.6 samples/s +Step 25540/44000 | Loss: 3.0008 | LR: 1.93e-04 | 701.3ms/step | 28.5 samples/s +Step 25550/44000 | Loss: 2.9648 | LR: 1.93e-04 | 699.2ms/step | 28.6 samples/s +Step 25560/44000 | Loss: 2.9625 | LR: 1.93e-04 | 699.3ms/step | 28.6 samples/s +Step 25570/44000 | Loss: 2.9394 | LR: 1.92e-04 | 703.5ms/step | 28.4 samples/s +Step 25580/44000 | Loss: 2.9231 | LR: 1.92e-04 | 699.7ms/step | 28.6 samples/s +Step 25590/44000 | Loss: 3.0093 | LR: 1.92e-04 | 702.1ms/step | 28.5 samples/s +Step 25600/44000 | Loss: 3.0164 | LR: 1.92e-04 | 702.2ms/step | 28.5 samples/s +Step 25610/44000 | Loss: 2.9808 | LR: 1.92e-04 | 698.3ms/step | 28.6 samples/s +Step 25620/44000 | Loss: 2.9895 | LR: 1.92e-04 | 697.8ms/step | 28.7 samples/s +Step 25630/44000 | Loss: 2.9561 | LR: 1.91e-04 | 699.3ms/step | 28.6 samples/s +Step 25640/44000 | Loss: 2.9548 | LR: 1.91e-04 | 700.3ms/step | 28.6 samples/s +Step 25650/44000 | Loss: 2.9293 | LR: 1.91e-04 | 700.7ms/step | 28.5 samples/s +Step 25660/44000 | Loss: 3.0047 | LR: 1.91e-04 | 697.3ms/step | 28.7 samples/s +Step 25670/44000 | Loss: 2.9599 | LR: 1.91e-04 | 698.1ms/step | 28.6 samples/s +Step 25680/44000 | Loss: 2.9534 | LR: 1.91e-04 | 702.2ms/step | 28.5 samples/s +Step 25690/44000 | Loss: 2.9691 | LR: 1.90e-04 | 699.4ms/step | 28.6 samples/s +Step 25700/44000 | Loss: 2.9290 | LR: 1.90e-04 | 700.9ms/step | 28.5 samples/s +Step 25710/44000 | Loss: 2.9494 | LR: 1.90e-04 | 701.3ms/step | 28.5 samples/s +Step 25720/44000 | Loss: 2.9768 | LR: 1.90e-04 | 698.6ms/step | 28.6 samples/s +Step 25730/44000 | Loss: 3.0015 | LR: 1.90e-04 | 699.5ms/step | 28.6 samples/s +Step 25740/44000 | Loss: 2.9342 | LR: 1.89e-04 | 699.2ms/step | 28.6 samples/s +Step 25750/44000 | Loss: 3.0076 | LR: 1.89e-04 | 698.8ms/step | 28.6 samples/s +Validation at step 25750: Loss = 2.9894, Perplexity = 19.87 +Step 25760/44000 | Loss: 2.9511 | LR: 1.89e-04 | 1269.5ms/step | 15.8 samples/s +Step 25770/44000 | Loss: 2.9937 | LR: 1.89e-04 | 701.4ms/step | 28.5 samples/s +Step 25780/44000 | Loss: 2.9939 | LR: 1.89e-04 | 701.2ms/step | 28.5 samples/s +Step 25790/44000 | Loss: 2.9664 | LR: 1.89e-04 | 698.5ms/step | 28.6 samples/s +Step 25800/44000 | Loss: 3.0043 | LR: 1.88e-04 | 697.0ms/step | 28.7 samples/s +Step 25810/44000 | Loss: 2.9956 | LR: 1.88e-04 | 698.8ms/step | 28.6 samples/s +Step 25820/44000 | Loss: 2.9929 | LR: 1.88e-04 | 699.3ms/step | 28.6 samples/s +Step 25830/44000 | Loss: 2.9869 | LR: 1.88e-04 | 700.5ms/step | 28.6 samples/s +Step 25840/44000 | Loss: 2.9702 | LR: 1.88e-04 | 697.7ms/step | 28.7 samples/s +Step 25850/44000 | Loss: 2.9377 | LR: 1.88e-04 | 702.1ms/step | 28.5 samples/s +Step 25860/44000 | Loss: 3.0036 | LR: 1.87e-04 | 696.5ms/step | 28.7 samples/s +Step 25870/44000 | Loss: 2.9594 | LR: 1.87e-04 | 700.7ms/step | 28.5 samples/s +Step 25880/44000 | Loss: 2.9978 | LR: 1.87e-04 | 702.4ms/step | 28.5 samples/s +Step 25890/44000 | Loss: 2.9338 | LR: 1.87e-04 | 700.4ms/step | 28.6 samples/s +Step 25900/44000 | Loss: 3.0128 | LR: 1.87e-04 | 703.9ms/step | 28.4 samples/s +Step 25910/44000 | Loss: 2.9923 | LR: 1.87e-04 | 701.7ms/step | 28.5 samples/s +Step 25920/44000 | Loss: 2.9615 | LR: 1.86e-04 | 700.2ms/step | 28.6 samples/s +Step 25930/44000 | Loss: 2.9452 | LR: 1.86e-04 | 700.1ms/step | 28.6 samples/s +Step 25940/44000 | Loss: 2.9905 | LR: 1.86e-04 | 701.3ms/step | 28.5 samples/s +Step 25950/44000 | Loss: 2.9969 | LR: 1.86e-04 | 697.3ms/step | 28.7 samples/s +Step 25960/44000 | Loss: 2.9693 | LR: 1.86e-04 | 700.0ms/step | 28.6 samples/s +Step 25970/44000 | Loss: 3.0104 | LR: 1.85e-04 | 700.8ms/step | 28.5 samples/s +Step 25980/44000 | Loss: 2.9216 | LR: 1.85e-04 | 700.0ms/step | 28.6 samples/s +Step 25990/44000 | Loss: 2.9538 | LR: 1.85e-04 | 699.0ms/step | 28.6 samples/s +Step 26000/44000 | Loss: 2.9653 | LR: 1.85e-04 | 699.9ms/step | 28.6 samples/s +Validation at step 26000: Loss = 2.9863, Perplexity = 19.81 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-26000 +Step 26010/44000 | Loss: 2.9630 | LR: 1.85e-04 | 1609.7ms/step | 12.4 samples/s +Step 26020/44000 | Loss: 2.9272 | LR: 1.85e-04 | 696.6ms/step | 28.7 samples/s +Step 26030/44000 | Loss: 2.9731 | LR: 1.84e-04 | 695.4ms/step | 28.8 samples/s +Step 26040/44000 | Loss: 2.9225 | LR: 1.84e-04 | 701.7ms/step | 28.5 samples/s +Step 26050/44000 | Loss: 2.9225 | LR: 1.84e-04 | 699.6ms/step | 28.6 samples/s +Step 26060/44000 | Loss: 2.9953 | LR: 1.84e-04 | 700.5ms/step | 28.5 samples/s +Step 26070/44000 | Loss: 2.9569 | LR: 1.84e-04 | 697.7ms/step | 28.7 samples/s +Step 26080/44000 | Loss: 2.9677 | LR: 1.84e-04 | 700.4ms/step | 28.6 samples/s +Step 26090/44000 | Loss: 2.9822 | LR: 1.83e-04 | 698.2ms/step | 28.6 samples/s +Step 26100/44000 | Loss: 2.9625 | LR: 1.83e-04 | 700.1ms/step | 28.6 samples/s +Step 26110/44000 | Loss: 2.9894 | LR: 1.83e-04 | 701.6ms/step | 28.5 samples/s +Step 26120/44000 | Loss: 2.9670 | LR: 1.83e-04 | 699.5ms/step | 28.6 samples/s +Step 26130/44000 | Loss: 2.9353 | LR: 1.83e-04 | 698.7ms/step | 28.6 samples/s +Step 26140/44000 | Loss: 2.9596 | LR: 1.82e-04 | 696.2ms/step | 28.7 samples/s +Step 26150/44000 | Loss: 2.9629 | LR: 1.82e-04 | 699.4ms/step | 28.6 samples/s +Step 26160/44000 | Loss: 2.9613 | LR: 1.82e-04 | 699.3ms/step | 28.6 samples/s +Step 26170/44000 | Loss: 2.9482 | LR: 1.82e-04 | 702.3ms/step | 28.5 samples/s +Step 26180/44000 | Loss: 2.9858 | LR: 1.82e-04 | 699.9ms/step | 28.6 samples/s +Step 26190/44000 | Loss: 2.9618 | LR: 1.82e-04 | 696.8ms/step | 28.7 samples/s +Step 26200/44000 | Loss: 2.9774 | LR: 1.81e-04 | 700.0ms/step | 28.6 samples/s +Step 26210/44000 | Loss: 3.0068 | LR: 1.81e-04 | 698.3ms/step | 28.6 samples/s +Step 26220/44000 | Loss: 2.9588 | LR: 1.81e-04 | 700.9ms/step | 28.5 samples/s +Step 26230/44000 | Loss: 2.9643 | LR: 1.81e-04 | 702.0ms/step | 28.5 samples/s +Step 26240/44000 | Loss: 2.9804 | LR: 1.81e-04 | 702.6ms/step | 28.5 samples/s +Step 26250/44000 | Loss: 2.9270 | LR: 1.81e-04 | 699.3ms/step | 28.6 samples/s +Validation at step 26250: Loss = 2.9863, Perplexity = 19.81 +Step 26260/44000 | Loss: 2.9541 | LR: 1.80e-04 | 1274.2ms/step | 15.7 samples/s +Step 26270/44000 | Loss: 2.9400 | LR: 1.80e-04 | 700.4ms/step | 28.6 samples/s +Step 26280/44000 | Loss: 2.9254 | LR: 1.80e-04 | 700.4ms/step | 28.6 samples/s +Step 26290/44000 | Loss: 2.9486 | LR: 1.80e-04 | 702.6ms/step | 28.5 samples/s +Step 26300/44000 | Loss: 2.9520 | LR: 1.80e-04 | 704.3ms/step | 28.4 samples/s +Step 26310/44000 | Loss: 2.9565 | LR: 1.80e-04 | 699.2ms/step | 28.6 samples/s +Step 26320/44000 | Loss: 2.9444 | LR: 1.79e-04 | 702.9ms/step | 28.5 samples/s +Step 26330/44000 | Loss: 2.9708 | LR: 1.79e-04 | 702.0ms/step | 28.5 samples/s +Step 26340/44000 | Loss: 2.9525 | LR: 1.79e-04 | 700.1ms/step | 28.6 samples/s +Step 26350/44000 | Loss: 2.9950 | LR: 1.79e-04 | 700.6ms/step | 28.5 samples/s +Step 26360/44000 | Loss: 2.9449 | LR: 1.79e-04 | 700.7ms/step | 28.5 samples/s +Step 26370/44000 | Loss: 2.9643 | LR: 1.78e-04 | 699.8ms/step | 28.6 samples/s +Step 26380/44000 | Loss: 2.9531 | LR: 1.78e-04 | 700.0ms/step | 28.6 samples/s +Step 26390/44000 | Loss: 2.9898 | LR: 1.78e-04 | 699.5ms/step | 28.6 samples/s +Step 26400/44000 | Loss: 2.9818 | LR: 1.78e-04 | 700.3ms/step | 28.6 samples/s +Step 26410/44000 | Loss: 2.9676 | LR: 1.78e-04 | 699.7ms/step | 28.6 samples/s +Step 26420/44000 | Loss: 2.9603 | LR: 1.78e-04 | 699.5ms/step | 28.6 samples/s +Step 26430/44000 | Loss: 2.9578 | LR: 1.77e-04 | 701.2ms/step | 28.5 samples/s +Step 26440/44000 | Loss: 2.9455 | LR: 1.77e-04 | 701.4ms/step | 28.5 samples/s +Step 26450/44000 | Loss: 2.9738 | LR: 1.77e-04 | 697.2ms/step | 28.7 samples/s +Step 26460/44000 | Loss: 2.9734 | LR: 1.77e-04 | 700.4ms/step | 28.6 samples/s +Step 26470/44000 | Loss: 2.9563 | LR: 1.77e-04 | 700.2ms/step | 28.6 samples/s +Step 26480/44000 | Loss: 2.9400 | LR: 1.77e-04 | 701.5ms/step | 28.5 samples/s +Step 26490/44000 | Loss: 2.9798 | LR: 1.76e-04 | 701.0ms/step | 28.5 samples/s +Step 26500/44000 | Loss: 2.9658 | LR: 1.76e-04 | 699.6ms/step | 28.6 samples/s +Validation at step 26500: Loss = 2.9826, Perplexity = 19.74 +Step 26510/44000 | Loss: 2.9762 | LR: 1.76e-04 | 1270.1ms/step | 15.7 samples/s +Step 26520/44000 | Loss: 2.9583 | LR: 1.76e-04 | 696.8ms/step | 28.7 samples/s +Step 26530/44000 | Loss: 2.9354 | LR: 1.76e-04 | 702.7ms/step | 28.5 samples/s +Step 26540/44000 | Loss: 2.9420 | LR: 1.76e-04 | 702.7ms/step | 28.5 samples/s +Step 26550/44000 | Loss: 2.9697 | LR: 1.75e-04 | 696.1ms/step | 28.7 samples/s +Step 26560/44000 | Loss: 2.9894 | LR: 1.75e-04 | 699.8ms/step | 28.6 samples/s +Step 26570/44000 | Loss: 2.9799 | LR: 1.75e-04 | 698.8ms/step | 28.6 samples/s +Step 26580/44000 | Loss: 2.9786 | LR: 1.75e-04 | 698.2ms/step | 28.6 samples/s +Step 26590/44000 | Loss: 2.9691 | LR: 1.75e-04 | 699.7ms/step | 28.6 samples/s +Step 26600/44000 | Loss: 2.9204 | LR: 1.74e-04 | 702.5ms/step | 28.5 samples/s +Step 26610/44000 | Loss: 2.9413 | LR: 1.74e-04 | 696.8ms/step | 28.7 samples/s +Step 26620/44000 | Loss: 2.9571 | LR: 1.74e-04 | 699.3ms/step | 28.6 samples/s +Step 26630/44000 | Loss: 2.9629 | LR: 1.74e-04 | 699.6ms/step | 28.6 samples/s +Step 26640/44000 | Loss: 2.9357 | LR: 1.74e-04 | 698.6ms/step | 28.6 samples/s +Step 26650/44000 | Loss: 2.9866 | LR: 1.74e-04 | 701.3ms/step | 28.5 samples/s +Step 26660/44000 | Loss: 2.9581 | LR: 1.73e-04 | 700.8ms/step | 28.5 samples/s +Step 26670/44000 | Loss: 2.9200 | LR: 1.73e-04 | 701.2ms/step | 28.5 samples/s +Step 26680/44000 | Loss: 3.0149 | LR: 1.73e-04 | 699.1ms/step | 28.6 samples/s +Step 26690/44000 | Loss: 2.9786 | LR: 1.73e-04 | 701.3ms/step | 28.5 samples/s +Step 26700/44000 | Loss: 2.9658 | LR: 1.73e-04 | 697.0ms/step | 28.7 samples/s +Step 26710/44000 | Loss: 2.9756 | LR: 1.73e-04 | 696.8ms/step | 28.7 samples/s +Step 26720/44000 | Loss: 2.9512 | LR: 1.72e-04 | 700.8ms/step | 28.5 samples/s +Step 26730/44000 | Loss: 2.9865 | LR: 1.72e-04 | 701.6ms/step | 28.5 samples/s +Step 26740/44000 | Loss: 2.9600 | LR: 1.72e-04 | 698.1ms/step | 28.6 samples/s +Step 26750/44000 | Loss: 2.9020 | LR: 1.72e-04 | 699.5ms/step | 28.6 samples/s +Validation at step 26750: Loss = 2.9831, Perplexity = 19.75 +Step 26760/44000 | Loss: 2.9583 | LR: 1.72e-04 | 1271.4ms/step | 15.7 samples/s +Step 26770/44000 | Loss: 2.9748 | LR: 1.72e-04 | 700.3ms/step | 28.6 samples/s +Step 26780/44000 | Loss: 2.9731 | LR: 1.71e-04 | 696.9ms/step | 28.7 samples/s +Step 26790/44000 | Loss: 2.9786 | LR: 1.71e-04 | 699.4ms/step | 28.6 samples/s +Step 26800/44000 | Loss: 2.9823 | LR: 1.71e-04 | 699.9ms/step | 28.6 samples/s +Step 26810/44000 | Loss: 2.9464 | LR: 1.71e-04 | 699.4ms/step | 28.6 samples/s +Step 26820/44000 | Loss: 2.9339 | LR: 1.71e-04 | 699.4ms/step | 28.6 samples/s +Step 26830/44000 | Loss: 2.9593 | LR: 1.71e-04 | 700.7ms/step | 28.5 samples/s +Step 26840/44000 | Loss: 2.9565 | LR: 1.70e-04 | 700.9ms/step | 28.5 samples/s +Step 26850/44000 | Loss: 2.9519 | LR: 1.70e-04 | 700.2ms/step | 28.6 samples/s +Step 26860/44000 | Loss: 2.9924 | LR: 1.70e-04 | 697.9ms/step | 28.7 samples/s +Step 26870/44000 | Loss: 2.9662 | LR: 1.70e-04 | 701.8ms/step | 28.5 samples/s +Step 26880/44000 | Loss: 2.9929 | LR: 1.70e-04 | 699.5ms/step | 28.6 samples/s +Step 26890/44000 | Loss: 2.9670 | LR: 1.69e-04 | 698.9ms/step | 28.6 samples/s +Step 26900/44000 | Loss: 2.9425 | LR: 1.69e-04 | 699.4ms/step | 28.6 samples/s +Step 26910/44000 | Loss: 2.9517 | LR: 1.69e-04 | 702.6ms/step | 28.5 samples/s +Step 26920/44000 | Loss: 2.9033 | LR: 1.69e-04 | 696.0ms/step | 28.7 samples/s +Step 26930/44000 | Loss: 2.9926 | LR: 1.69e-04 | 698.8ms/step | 28.6 samples/s +Step 26940/44000 | Loss: 2.9379 | LR: 1.69e-04 | 699.2ms/step | 28.6 samples/s +Step 26950/44000 | Loss: 2.9560 | LR: 1.68e-04 | 695.6ms/step | 28.8 samples/s +Step 26960/44000 | Loss: 2.9449 | LR: 1.68e-04 | 696.0ms/step | 28.7 samples/s +Step 26970/44000 | Loss: 2.9855 | LR: 1.68e-04 | 698.0ms/step | 28.7 samples/s +Step 26980/44000 | Loss: 2.9727 | LR: 1.68e-04 | 700.1ms/step | 28.6 samples/s +Step 26990/44000 | Loss: 2.9569 | LR: 1.68e-04 | 700.6ms/step | 28.5 samples/s +Step 27000/44000 | Loss: 2.9481 | LR: 1.68e-04 | 702.9ms/step | 28.5 samples/s +Validation at step 27000: Loss = 2.9821, Perplexity = 19.73 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-27000 +Step 27010/44000 | Loss: 2.9734 | LR: 1.67e-04 | 1609.1ms/step | 12.4 samples/s +Step 27020/44000 | Loss: 2.9342 | LR: 1.67e-04 | 701.7ms/step | 28.5 samples/s +Step 27030/44000 | Loss: 2.9894 | LR: 1.67e-04 | 700.4ms/step | 28.6 samples/s +Step 27040/44000 | Loss: 2.9398 | LR: 1.67e-04 | 696.4ms/step | 28.7 samples/s +Step 27050/44000 | Loss: 2.9719 | LR: 1.67e-04 | 699.3ms/step | 28.6 samples/s +Step 27060/44000 | Loss: 2.9484 | LR: 1.67e-04 | 699.1ms/step | 28.6 samples/s +Step 27070/44000 | Loss: 2.9251 | LR: 1.66e-04 | 699.0ms/step | 28.6 samples/s +Step 27080/44000 | Loss: 2.9293 | LR: 1.66e-04 | 698.0ms/step | 28.7 samples/s +Step 27090/44000 | Loss: 2.9564 | LR: 1.66e-04 | 699.9ms/step | 28.6 samples/s +Step 27100/44000 | Loss: 2.9923 | LR: 1.66e-04 | 698.0ms/step | 28.7 samples/s +Step 27110/44000 | Loss: 2.9549 | LR: 1.66e-04 | 696.3ms/step | 28.7 samples/s +Step 27120/44000 | Loss: 2.9577 | LR: 1.66e-04 | 697.8ms/step | 28.7 samples/s +Step 27130/44000 | Loss: 2.9553 | LR: 1.65e-04 | 699.3ms/step | 28.6 samples/s +Step 27140/44000 | Loss: 2.9768 | LR: 1.65e-04 | 698.9ms/step | 28.6 samples/s +Step 27150/44000 | Loss: 2.9652 | LR: 1.65e-04 | 697.0ms/step | 28.7 samples/s +Step 27160/44000 | Loss: 2.9596 | LR: 1.65e-04 | 696.9ms/step | 28.7 samples/s +Step 27170/44000 | Loss: 2.9452 | LR: 1.65e-04 | 700.9ms/step | 28.5 samples/s +Step 27180/44000 | Loss: 2.9288 | LR: 1.65e-04 | 697.2ms/step | 28.7 samples/s +Step 27190/44000 | Loss: 2.9345 | LR: 1.64e-04 | 698.2ms/step | 28.6 samples/s +Step 27200/44000 | Loss: 2.9570 | LR: 1.64e-04 | 695.8ms/step | 28.7 samples/s +Step 27210/44000 | Loss: 2.9752 | LR: 1.64e-04 | 698.3ms/step | 28.6 samples/s +Step 27220/44000 | Loss: 2.9595 | LR: 1.64e-04 | 699.9ms/step | 28.6 samples/s +Step 27230/44000 | Loss: 2.9559 | LR: 1.64e-04 | 699.4ms/step | 28.6 samples/s +Step 27240/44000 | Loss: 2.9332 | LR: 1.63e-04 | 701.7ms/step | 28.5 samples/s +Step 27250/44000 | Loss: 2.9545 | LR: 1.63e-04 | 698.7ms/step | 28.6 samples/s +Validation at step 27250: Loss = 2.9786, Perplexity = 19.66 +Step 27260/44000 | Loss: 2.9617 | LR: 1.63e-04 | 1276.9ms/step | 15.7 samples/s +Step 27270/44000 | Loss: 2.9567 | LR: 1.63e-04 | 700.1ms/step | 28.6 samples/s +Step 27280/44000 | Loss: 2.9214 | LR: 1.63e-04 | 702.2ms/step | 28.5 samples/s +Step 27290/44000 | Loss: 2.9979 | LR: 1.63e-04 | 698.9ms/step | 28.6 samples/s +Step 27300/44000 | Loss: 2.9560 | LR: 1.62e-04 | 701.2ms/step | 28.5 samples/s +Step 27310/44000 | Loss: 2.9571 | LR: 1.62e-04 | 699.8ms/step | 28.6 samples/s +Step 27320/44000 | Loss: 2.9868 | LR: 1.62e-04 | 699.6ms/step | 28.6 samples/s +Step 27330/44000 | Loss: 2.9760 | LR: 1.62e-04 | 697.1ms/step | 28.7 samples/s +Step 27340/44000 | Loss: 2.9819 | LR: 1.62e-04 | 695.9ms/step | 28.7 samples/s +Step 27350/44000 | Loss: 2.9817 | LR: 1.62e-04 | 696.4ms/step | 28.7 samples/s +Step 27360/44000 | Loss: 2.9373 | LR: 1.61e-04 | 699.5ms/step | 28.6 samples/s +Step 27370/44000 | Loss: 2.9495 | LR: 1.61e-04 | 695.8ms/step | 28.7 samples/s +Step 27380/44000 | Loss: 2.9247 | LR: 1.61e-04 | 700.5ms/step | 28.6 samples/s +Step 27390/44000 | Loss: 2.9508 | LR: 1.61e-04 | 698.5ms/step | 28.6 samples/s +Step 27400/44000 | Loss: 2.9666 | LR: 1.61e-04 | 700.4ms/step | 28.6 samples/s +Step 27410/44000 | Loss: 2.9623 | LR: 1.61e-04 | 695.0ms/step | 28.8 samples/s +Step 27420/44000 | Loss: 2.9536 | LR: 1.60e-04 | 697.6ms/step | 28.7 samples/s +Step 27430/44000 | Loss: 2.9627 | LR: 1.60e-04 | 696.2ms/step | 28.7 samples/s +Step 27440/44000 | Loss: 2.9162 | LR: 1.60e-04 | 698.5ms/step | 28.6 samples/s +Step 27450/44000 | Loss: 2.9177 | LR: 1.60e-04 | 699.3ms/step | 28.6 samples/s +Step 27460/44000 | Loss: 2.9402 | LR: 1.60e-04 | 700.6ms/step | 28.5 samples/s +Step 27470/44000 | Loss: 2.9547 | LR: 1.60e-04 | 697.8ms/step | 28.7 samples/s +Step 27480/44000 | Loss: 2.9381 | LR: 1.59e-04 | 696.4ms/step | 28.7 samples/s +Step 27490/44000 | Loss: 2.9624 | LR: 1.59e-04 | 698.4ms/step | 28.6 samples/s +Step 27500/44000 | Loss: 2.9513 | LR: 1.59e-04 | 698.5ms/step | 28.6 samples/s +Validation at step 27500: Loss = 2.9792, Perplexity = 19.67 +Step 27510/44000 | Loss: 2.9735 | LR: 1.59e-04 | 1270.0ms/step | 15.7 samples/s +Step 27520/44000 | Loss: 2.9642 | LR: 1.59e-04 | 697.9ms/step | 28.7 samples/s +Step 27530/44000 | Loss: 2.9958 | LR: 1.59e-04 | 700.6ms/step | 28.5 samples/s +Step 27540/44000 | Loss: 2.9613 | LR: 1.58e-04 | 698.6ms/step | 28.6 samples/s +Step 27550/44000 | Loss: 2.9617 | LR: 1.58e-04 | 700.2ms/step | 28.6 samples/s +Step 27560/44000 | Loss: 2.9345 | LR: 1.58e-04 | 700.1ms/step | 28.6 samples/s +Step 27570/44000 | Loss: 2.9420 | LR: 1.58e-04 | 700.3ms/step | 28.6 samples/s +Step 27580/44000 | Loss: 2.9909 | LR: 1.58e-04 | 698.5ms/step | 28.6 samples/s +Step 27590/44000 | Loss: 2.9550 | LR: 1.58e-04 | 695.5ms/step | 28.8 samples/s +Step 27600/44000 | Loss: 2.9465 | LR: 1.57e-04 | 697.9ms/step | 28.7 samples/s +Step 27610/44000 | Loss: 2.9978 | LR: 1.57e-04 | 698.3ms/step | 28.6 samples/s +Step 27620/44000 | Loss: 2.9692 | LR: 1.57e-04 | 697.1ms/step | 28.7 samples/s +Step 27630/44000 | Loss: 2.9440 | LR: 1.57e-04 | 699.7ms/step | 28.6 samples/s +Step 27640/44000 | Loss: 2.9458 | LR: 1.57e-04 | 697.8ms/step | 28.7 samples/s +Step 27650/44000 | Loss: 2.9573 | LR: 1.57e-04 | 701.4ms/step | 28.5 samples/s +Step 27660/44000 | Loss: 2.9496 | LR: 1.56e-04 | 698.7ms/step | 28.6 samples/s +Step 27670/44000 | Loss: 2.9667 | LR: 1.56e-04 | 700.6ms/step | 28.5 samples/s +Step 27680/44000 | Loss: 2.9747 | LR: 1.56e-04 | 697.4ms/step | 28.7 samples/s +Step 27690/44000 | Loss: 2.9980 | LR: 1.56e-04 | 699.3ms/step | 28.6 samples/s +Step 27700/44000 | Loss: 2.9690 | LR: 1.56e-04 | 700.5ms/step | 28.6 samples/s +Step 27710/44000 | Loss: 2.9555 | LR: 1.56e-04 | 699.9ms/step | 28.6 samples/s +Step 27720/44000 | Loss: 2.9584 | LR: 1.55e-04 | 700.9ms/step | 28.5 samples/s +Step 27730/44000 | Loss: 2.9192 | LR: 1.55e-04 | 697.8ms/step | 28.7 samples/s +Step 27740/44000 | Loss: 2.9187 | LR: 1.55e-04 | 695.5ms/step | 28.8 samples/s +Step 27750/44000 | Loss: 2.9046 | LR: 1.55e-04 | 700.2ms/step | 28.6 samples/s +Validation at step 27750: Loss = 2.9769, Perplexity = 19.63 +Step 27760/44000 | Loss: 2.9760 | LR: 1.55e-04 | 1268.9ms/step | 15.8 samples/s +Step 27770/44000 | Loss: 2.9538 | LR: 1.55e-04 | 695.6ms/step | 28.8 samples/s +Step 27780/44000 | Loss: 2.9319 | LR: 1.54e-04 | 699.6ms/step | 28.6 samples/s +Step 27790/44000 | Loss: 2.9394 | LR: 1.54e-04 | 696.1ms/step | 28.7 samples/s +Step 27800/44000 | Loss: 2.9897 | LR: 1.54e-04 | 699.4ms/step | 28.6 samples/s +Step 27810/44000 | Loss: 2.9490 | LR: 1.54e-04 | 698.3ms/step | 28.6 samples/s +Step 27820/44000 | Loss: 2.9647 | LR: 1.54e-04 | 698.2ms/step | 28.6 samples/s +Step 27830/44000 | Loss: 2.9709 | LR: 1.54e-04 | 697.5ms/step | 28.7 samples/s +Step 27840/44000 | Loss: 2.9860 | LR: 1.53e-04 | 702.5ms/step | 28.5 samples/s +Step 27850/44000 | Loss: 2.9341 | LR: 1.53e-04 | 699.7ms/step | 28.6 samples/s +Step 27860/44000 | Loss: 2.9625 | LR: 1.53e-04 | 699.6ms/step | 28.6 samples/s +Step 27870/44000 | Loss: 2.9411 | LR: 1.53e-04 | 699.9ms/step | 28.6 samples/s +Step 27880/44000 | Loss: 2.9840 | LR: 1.53e-04 | 698.6ms/step | 28.6 samples/s +Step 27890/44000 | Loss: 2.9292 | LR: 1.53e-04 | 697.6ms/step | 28.7 samples/s +Step 27900/44000 | Loss: 2.9268 | LR: 1.52e-04 | 697.8ms/step | 28.7 samples/s +Step 27910/44000 | Loss: 3.0125 | LR: 1.52e-04 | 699.5ms/step | 28.6 samples/s +Step 27920/44000 | Loss: 2.9843 | LR: 1.52e-04 | 696.5ms/step | 28.7 samples/s +Step 27930/44000 | Loss: 2.9253 | LR: 1.52e-04 | 698.3ms/step | 28.6 samples/s +Step 27940/44000 | Loss: 2.9399 | LR: 1.52e-04 | 698.5ms/step | 28.6 samples/s +Step 27950/44000 | Loss: 2.9882 | LR: 1.52e-04 | 695.8ms/step | 28.7 samples/s +Step 27960/44000 | Loss: 2.9421 | LR: 1.51e-04 | 697.3ms/step | 28.7 samples/s +Step 27970/44000 | Loss: 2.9821 | LR: 1.51e-04 | 694.7ms/step | 28.8 samples/s +Step 27980/44000 | Loss: 2.9495 | LR: 1.51e-04 | 696.6ms/step | 28.7 samples/s +Step 27990/44000 | Loss: 2.9299 | LR: 1.51e-04 | 698.7ms/step | 28.6 samples/s +Step 28000/44000 | Loss: 2.9355 | LR: 1.51e-04 | 697.0ms/step | 28.7 samples/s +Validation at step 28000: Loss = 2.9754, Perplexity = 19.60 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-28000 +Step 28010/44000 | Loss: 2.9347 | LR: 1.51e-04 | 1637.0ms/step | 12.2 samples/s +Step 28020/44000 | Loss: 2.9498 | LR: 1.50e-04 | 703.3ms/step | 28.4 samples/s +Step 28030/44000 | Loss: 2.9598 | LR: 1.50e-04 | 699.2ms/step | 28.6 samples/s +Step 28040/44000 | Loss: 2.9729 | LR: 1.50e-04 | 700.0ms/step | 28.6 samples/s +Step 28050/44000 | Loss: 2.9239 | LR: 1.50e-04 | 699.5ms/step | 28.6 samples/s +Step 28060/44000 | Loss: 2.9424 | LR: 1.50e-04 | 699.6ms/step | 28.6 samples/s +Step 28070/44000 | Loss: 2.9564 | LR: 1.50e-04 | 700.4ms/step | 28.6 samples/s +Step 28080/44000 | Loss: 2.9422 | LR: 1.49e-04 | 700.7ms/step | 28.5 samples/s +Step 28090/44000 | Loss: 2.9286 | LR: 1.49e-04 | 697.9ms/step | 28.7 samples/s +Step 28100/44000 | Loss: 2.9531 | LR: 1.49e-04 | 699.2ms/step | 28.6 samples/s +Step 28110/44000 | Loss: 2.9498 | LR: 1.49e-04 | 699.4ms/step | 28.6 samples/s +Step 28120/44000 | Loss: 2.9139 | LR: 1.49e-04 | 698.6ms/step | 28.6 samples/s +Step 28130/44000 | Loss: 2.9951 | LR: 1.49e-04 | 700.8ms/step | 28.5 samples/s +Step 28140/44000 | Loss: 2.9253 | LR: 1.48e-04 | 701.4ms/step | 28.5 samples/s +Step 28150/44000 | Loss: 2.9524 | LR: 1.48e-04 | 700.7ms/step | 28.5 samples/s +Step 28160/44000 | Loss: 2.9430 | LR: 1.48e-04 | 695.7ms/step | 28.7 samples/s +Step 28170/44000 | Loss: 2.9811 | LR: 1.48e-04 | 699.4ms/step | 28.6 samples/s +Step 28180/44000 | Loss: 2.9500 | LR: 1.48e-04 | 699.3ms/step | 28.6 samples/s +Step 28190/44000 | Loss: 2.9879 | LR: 1.48e-04 | 700.0ms/step | 28.6 samples/s +Step 28200/44000 | Loss: 2.9132 | LR: 1.47e-04 | 695.6ms/step | 28.8 samples/s +Step 28210/44000 | Loss: 2.9423 | LR: 1.47e-04 | 698.3ms/step | 28.6 samples/s +Step 28220/44000 | Loss: 2.9564 | LR: 1.47e-04 | 698.0ms/step | 28.7 samples/s +Step 28230/44000 | Loss: 2.9526 | LR: 1.47e-04 | 697.6ms/step | 28.7 samples/s +Step 28240/44000 | Loss: 2.9528 | LR: 1.47e-04 | 696.9ms/step | 28.7 samples/s +Step 28250/44000 | Loss: 2.9617 | LR: 1.47e-04 | 701.3ms/step | 28.5 samples/s +Validation at step 28250: Loss = 2.9747, Perplexity = 19.58 +Step 28260/44000 | Loss: 2.9416 | LR: 1.46e-04 | 1268.0ms/step | 15.8 samples/s +Step 28270/44000 | Loss: 2.9443 | LR: 1.46e-04 | 699.7ms/step | 28.6 samples/s +Step 28280/44000 | Loss: 2.9498 | LR: 1.46e-04 | 696.5ms/step | 28.7 samples/s +Step 28290/44000 | Loss: 2.9571 | LR: 1.46e-04 | 699.4ms/step | 28.6 samples/s +Step 28300/44000 | Loss: 2.9467 | LR: 1.46e-04 | 697.2ms/step | 28.7 samples/s +Step 28310/44000 | Loss: 2.9517 | LR: 1.46e-04 | 695.0ms/step | 28.8 samples/s +Step 28320/44000 | Loss: 2.9147 | LR: 1.45e-04 | 697.9ms/step | 28.7 samples/s +Step 28330/44000 | Loss: 2.9253 | LR: 1.45e-04 | 700.2ms/step | 28.6 samples/s +Step 28340/44000 | Loss: 2.9867 | LR: 1.45e-04 | 693.7ms/step | 28.8 samples/s +Step 28350/44000 | Loss: 2.9786 | LR: 1.45e-04 | 700.1ms/step | 28.6 samples/s +Step 28360/44000 | Loss: 2.9614 | LR: 1.45e-04 | 698.4ms/step | 28.6 samples/s +Step 28370/44000 | Loss: 2.9321 | LR: 1.45e-04 | 700.1ms/step | 28.6 samples/s +Step 28380/44000 | Loss: 2.9555 | LR: 1.44e-04 | 696.0ms/step | 28.7 samples/s +Step 28390/44000 | Loss: 2.9835 | LR: 1.44e-04 | 697.6ms/step | 28.7 samples/s +Step 28400/44000 | Loss: 2.9511 | LR: 1.44e-04 | 699.2ms/step | 28.6 samples/s +Step 28410/44000 | Loss: 2.9483 | LR: 1.44e-04 | 698.7ms/step | 28.6 samples/s +Step 28420/44000 | Loss: 2.9912 | LR: 1.44e-04 | 698.5ms/step | 28.6 samples/s +Step 28430/44000 | Loss: 2.9291 | LR: 1.44e-04 | 698.7ms/step | 28.6 samples/s +Step 28440/44000 | Loss: 2.9144 | LR: 1.43e-04 | 700.6ms/step | 28.5 samples/s +Step 28450/44000 | Loss: 2.9373 | LR: 1.43e-04 | 698.5ms/step | 28.6 samples/s +Step 28460/44000 | Loss: 2.9559 | LR: 1.43e-04 | 699.3ms/step | 28.6 samples/s +Step 28470/44000 | Loss: 2.9138 | LR: 1.43e-04 | 697.7ms/step | 28.7 samples/s +Step 28480/44000 | Loss: 2.9945 | LR: 1.43e-04 | 699.2ms/step | 28.6 samples/s +Step 28490/44000 | Loss: 2.9473 | LR: 1.43e-04 | 699.6ms/step | 28.6 samples/s +Step 28500/44000 | Loss: 2.9516 | LR: 1.42e-04 | 701.0ms/step | 28.5 samples/s +Validation at step 28500: Loss = 2.9741, Perplexity = 19.57 +Step 28510/44000 | Loss: 2.9766 | LR: 1.42e-04 | 1267.2ms/step | 15.8 samples/s +Step 28520/44000 | Loss: 2.9852 | LR: 1.42e-04 | 694.5ms/step | 28.8 samples/s +Step 28530/44000 | Loss: 2.9766 | LR: 1.42e-04 | 698.9ms/step | 28.6 samples/s +Step 28540/44000 | Loss: 2.9574 | LR: 1.42e-04 | 700.4ms/step | 28.6 samples/s +Step 28550/44000 | Loss: 2.9403 | LR: 1.42e-04 | 699.9ms/step | 28.6 samples/s +Step 28560/44000 | Loss: 2.9777 | LR: 1.41e-04 | 698.1ms/step | 28.6 samples/s +Step 28570/44000 | Loss: 2.9294 | LR: 1.41e-04 | 697.2ms/step | 28.7 samples/s +Step 28580/44000 | Loss: 2.9487 | LR: 1.41e-04 | 700.5ms/step | 28.6 samples/s +Step 28590/44000 | Loss: 2.9003 | LR: 1.41e-04 | 700.8ms/step | 28.5 samples/s +Step 28600/44000 | Loss: 2.9601 | LR: 1.41e-04 | 699.1ms/step | 28.6 samples/s +Step 28610/44000 | Loss: 2.9980 | LR: 1.41e-04 | 700.3ms/step | 28.6 samples/s +Step 28620/44000 | Loss: 2.9684 | LR: 1.40e-04 | 699.4ms/step | 28.6 samples/s +Step 28630/44000 | Loss: 2.9546 | LR: 1.40e-04 | 700.7ms/step | 28.5 samples/s +Step 28640/44000 | Loss: 2.9044 | LR: 1.40e-04 | 698.4ms/step | 28.6 samples/s +Step 28650/44000 | Loss: 2.9671 | LR: 1.40e-04 | 699.2ms/step | 28.6 samples/s +Step 28660/44000 | Loss: 2.9541 | LR: 1.40e-04 | 697.7ms/step | 28.7 samples/s +Step 28670/44000 | Loss: 2.9514 | LR: 1.40e-04 | 699.3ms/step | 28.6 samples/s +Step 28680/44000 | Loss: 2.9620 | LR: 1.39e-04 | 699.4ms/step | 28.6 samples/s +Step 28690/44000 | Loss: 2.9379 | LR: 1.39e-04 | 703.1ms/step | 28.4 samples/s +Step 28700/44000 | Loss: 2.9443 | LR: 1.39e-04 | 699.1ms/step | 28.6 samples/s +Step 28710/44000 | Loss: 2.9362 | LR: 1.39e-04 | 701.4ms/step | 28.5 samples/s +Step 28720/44000 | Loss: 2.9290 | LR: 1.39e-04 | 700.6ms/step | 28.5 samples/s +Step 28730/44000 | Loss: 2.9316 | LR: 1.39e-04 | 699.3ms/step | 28.6 samples/s +Step 28740/44000 | Loss: 2.9547 | LR: 1.38e-04 | 698.0ms/step | 28.7 samples/s +Step 28750/44000 | Loss: 2.9361 | LR: 1.38e-04 | 698.7ms/step | 28.6 samples/s +Validation at step 28750: Loss = 2.9727, Perplexity = 19.54 +Step 28760/44000 | Loss: 2.9238 | LR: 1.38e-04 | 1268.5ms/step | 15.8 samples/s +Step 28770/44000 | Loss: 2.9246 | LR: 1.38e-04 | 699.6ms/step | 28.6 samples/s +Step 28780/44000 | Loss: 2.8695 | LR: 1.38e-04 | 699.1ms/step | 28.6 samples/s +Step 28790/44000 | Loss: 2.9303 | LR: 1.38e-04 | 700.4ms/step | 28.6 samples/s +Step 28800/44000 | Loss: 2.9616 | LR: 1.38e-04 | 701.6ms/step | 28.5 samples/s +Step 28810/44000 | Loss: 2.9410 | LR: 1.37e-04 | 699.9ms/step | 28.6 samples/s +Step 28820/44000 | Loss: 2.9448 | LR: 1.37e-04 | 700.7ms/step | 28.5 samples/s +Step 28830/44000 | Loss: 2.9765 | LR: 1.37e-04 | 698.1ms/step | 28.6 samples/s +Step 28840/44000 | Loss: 2.9121 | LR: 1.37e-04 | 697.7ms/step | 28.7 samples/s +Step 28850/44000 | Loss: 2.9127 | LR: 1.37e-04 | 699.7ms/step | 28.6 samples/s +Step 28860/44000 | Loss: 2.9835 | LR: 1.37e-04 | 696.2ms/step | 28.7 samples/s +Step 28870/44000 | Loss: 2.9344 | LR: 1.36e-04 | 697.3ms/step | 28.7 samples/s +Step 28880/44000 | Loss: 2.9324 | LR: 1.36e-04 | 698.7ms/step | 28.6 samples/s +Step 28890/44000 | Loss: 2.9078 | LR: 1.36e-04 | 700.8ms/step | 28.5 samples/s +Step 28900/44000 | Loss: 2.9415 | LR: 1.36e-04 | 697.6ms/step | 28.7 samples/s +Step 28910/44000 | Loss: 2.9573 | LR: 1.36e-04 | 697.8ms/step | 28.7 samples/s +Step 28920/44000 | Loss: 2.9585 | LR: 1.36e-04 | 699.7ms/step | 28.6 samples/s +Step 28930/44000 | Loss: 2.9446 | LR: 1.35e-04 | 698.3ms/step | 28.6 samples/s +Step 28940/44000 | Loss: 2.9407 | LR: 1.35e-04 | 697.9ms/step | 28.7 samples/s +Step 28950/44000 | Loss: 2.9376 | LR: 1.35e-04 | 700.2ms/step | 28.6 samples/s +Step 28960/44000 | Loss: 2.9552 | LR: 1.35e-04 | 696.1ms/step | 28.7 samples/s +Step 28970/44000 | Loss: 2.9202 | LR: 1.35e-04 | 696.4ms/step | 28.7 samples/s +Step 28980/44000 | Loss: 2.9120 | LR: 1.35e-04 | 701.0ms/step | 28.5 samples/s +Step 28990/44000 | Loss: 2.9161 | LR: 1.34e-04 | 700.6ms/step | 28.5 samples/s +Step 29000/44000 | Loss: 2.9255 | LR: 1.34e-04 | 699.0ms/step | 28.6 samples/s +Validation at step 29000: Loss = 2.9722, Perplexity = 19.54 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-29000 +Step 29010/44000 | Loss: 2.9128 | LR: 1.34e-04 | 1633.3ms/step | 12.2 samples/s +Step 29020/44000 | Loss: 2.9434 | LR: 1.34e-04 | 698.0ms/step | 28.7 samples/s +Step 29030/44000 | Loss: 2.9368 | LR: 1.34e-04 | 697.8ms/step | 28.7 samples/s +Step 29040/44000 | Loss: 2.9098 | LR: 1.34e-04 | 702.2ms/step | 28.5 samples/s +Step 29050/44000 | Loss: 2.9586 | LR: 1.33e-04 | 697.6ms/step | 28.7 samples/s +Step 29060/44000 | Loss: 3.0459 | LR: 1.33e-04 | 697.5ms/step | 28.7 samples/s +Step 29070/44000 | Loss: 2.9581 | LR: 1.33e-04 | 698.2ms/step | 28.6 samples/s +Step 29080/44000 | Loss: 2.9231 | LR: 1.33e-04 | 698.3ms/step | 28.6 samples/s +Step 29090/44000 | Loss: 2.9112 | LR: 1.33e-04 | 698.2ms/step | 28.6 samples/s +Step 29100/44000 | Loss: 2.9500 | LR: 1.33e-04 | 699.6ms/step | 28.6 samples/s +Step 29110/44000 | Loss: 2.9990 | LR: 1.33e-04 | 698.8ms/step | 28.6 samples/s +Step 29120/44000 | Loss: 2.9295 | LR: 1.32e-04 | 697.4ms/step | 28.7 samples/s +Step 29130/44000 | Loss: 2.9614 | LR: 1.32e-04 | 697.8ms/step | 28.7 samples/s +Step 29140/44000 | Loss: 2.9222 | LR: 1.32e-04 | 702.4ms/step | 28.5 samples/s +Step 29150/44000 | Loss: 2.9467 | LR: 1.32e-04 | 699.7ms/step | 28.6 samples/s +Step 29160/44000 | Loss: 2.9763 | LR: 1.32e-04 | 699.4ms/step | 28.6 samples/s +Step 29170/44000 | Loss: 2.9569 | LR: 1.32e-04 | 700.0ms/step | 28.6 samples/s +Step 29180/44000 | Loss: 2.9469 | LR: 1.31e-04 | 700.6ms/step | 28.5 samples/s +Step 29190/44000 | Loss: 2.9465 | LR: 1.31e-04 | 699.2ms/step | 28.6 samples/s +Step 29200/44000 | Loss: 2.9601 | LR: 1.31e-04 | 698.7ms/step | 28.6 samples/s +Step 29210/44000 | Loss: 2.8982 | LR: 1.31e-04 | 701.2ms/step | 28.5 samples/s +Step 29220/44000 | Loss: 2.9483 | LR: 1.31e-04 | 701.5ms/step | 28.5 samples/s +Step 29230/44000 | Loss: 2.9346 | LR: 1.31e-04 | 701.0ms/step | 28.5 samples/s +Step 29240/44000 | Loss: 2.9125 | LR: 1.30e-04 | 698.4ms/step | 28.6 samples/s +Step 29250/44000 | Loss: 2.9476 | LR: 1.30e-04 | 701.8ms/step | 28.5 samples/s +Validation at step 29250: Loss = 2.9716, Perplexity = 19.52 +Step 29260/44000 | Loss: 3.0436 | LR: 1.30e-04 | 1267.0ms/step | 15.8 samples/s +Step 29270/44000 | Loss: 2.9866 | LR: 1.30e-04 | 699.1ms/step | 28.6 samples/s +Step 29280/44000 | Loss: 2.9672 | LR: 1.30e-04 | 699.8ms/step | 28.6 samples/s +Step 29290/44000 | Loss: 2.9200 | LR: 1.30e-04 | 699.7ms/step | 28.6 samples/s +Step 29300/44000 | Loss: 2.9181 | LR: 1.29e-04 | 700.8ms/step | 28.5 samples/s +Step 29310/44000 | Loss: 2.9962 | LR: 1.29e-04 | 698.2ms/step | 28.6 samples/s +Step 29320/44000 | Loss: 2.9733 | LR: 1.29e-04 | 699.1ms/step | 28.6 samples/s +Step 29330/44000 | Loss: 2.8897 | LR: 1.29e-04 | 696.5ms/step | 28.7 samples/s +Step 29340/44000 | Loss: 2.9494 | LR: 1.29e-04 | 700.2ms/step | 28.6 samples/s +Step 29350/44000 | Loss: 2.9671 | LR: 1.29e-04 | 699.3ms/step | 28.6 samples/s +Step 29360/44000 | Loss: 2.9673 | LR: 1.29e-04 | 698.3ms/step | 28.6 samples/s +Step 29370/44000 | Loss: 2.9359 | LR: 1.28e-04 | 698.7ms/step | 28.6 samples/s +Step 29380/44000 | Loss: 2.9415 | LR: 1.28e-04 | 698.0ms/step | 28.7 samples/s +Step 29390/44000 | Loss: 2.9890 | LR: 1.28e-04 | 698.0ms/step | 28.7 samples/s +Step 29400/44000 | Loss: 2.9342 | LR: 1.28e-04 | 698.3ms/step | 28.6 samples/s +Step 29410/44000 | Loss: 2.9651 | LR: 1.28e-04 | 696.7ms/step | 28.7 samples/s +Step 29420/44000 | Loss: 2.9433 | LR: 1.28e-04 | 696.0ms/step | 28.7 samples/s +Step 29430/44000 | Loss: 2.9377 | LR: 1.27e-04 | 698.4ms/step | 28.6 samples/s +Step 29440/44000 | Loss: 2.9368 | LR: 1.27e-04 | 702.0ms/step | 28.5 samples/s +Step 29450/44000 | Loss: 2.9411 | LR: 1.27e-04 | 698.7ms/step | 28.6 samples/s +Step 29460/44000 | Loss: 2.9313 | LR: 1.27e-04 | 697.5ms/step | 28.7 samples/s +Step 29470/44000 | Loss: 2.9334 | LR: 1.27e-04 | 699.1ms/step | 28.6 samples/s +Step 29480/44000 | Loss: 2.9578 | LR: 1.27e-04 | 701.5ms/step | 28.5 samples/s +Step 29490/44000 | Loss: 2.9432 | LR: 1.26e-04 | 701.2ms/step | 28.5 samples/s +Step 29500/44000 | Loss: 2.9591 | LR: 1.26e-04 | 700.2ms/step | 28.6 samples/s +Validation at step 29500: Loss = 2.9683, Perplexity = 19.46 +Step 29510/44000 | Loss: 2.9562 | LR: 1.26e-04 | 1269.9ms/step | 15.7 samples/s +Step 29520/44000 | Loss: 2.9330 | LR: 1.26e-04 | 698.0ms/step | 28.7 samples/s +Step 29530/44000 | Loss: 2.9738 | LR: 1.26e-04 | 699.1ms/step | 28.6 samples/s +Step 29540/44000 | Loss: 2.9714 | LR: 1.26e-04 | 694.7ms/step | 28.8 samples/s +Step 29550/44000 | Loss: 2.9732 | LR: 1.26e-04 | 697.9ms/step | 28.7 samples/s +Step 29560/44000 | Loss: 2.9593 | LR: 1.25e-04 | 695.1ms/step | 28.8 samples/s +Step 29570/44000 | Loss: 2.8950 | LR: 1.25e-04 | 696.9ms/step | 28.7 samples/s +Step 29580/44000 | Loss: 2.9604 | LR: 1.25e-04 | 700.1ms/step | 28.6 samples/s +Step 29590/44000 | Loss: 2.9835 | LR: 1.25e-04 | 696.2ms/step | 28.7 samples/s +Step 29600/44000 | Loss: 2.9797 | LR: 1.25e-04 | 696.0ms/step | 28.7 samples/s +Step 29610/44000 | Loss: 2.9323 | LR: 1.25e-04 | 697.9ms/step | 28.7 samples/s +Step 29620/44000 | Loss: 2.9376 | LR: 1.24e-04 | 699.9ms/step | 28.6 samples/s +Step 29630/44000 | Loss: 2.9582 | LR: 1.24e-04 | 697.6ms/step | 28.7 samples/s +Step 29640/44000 | Loss: 2.9406 | LR: 1.24e-04 | 698.9ms/step | 28.6 samples/s +Step 29650/44000 | Loss: 2.9172 | LR: 1.24e-04 | 695.4ms/step | 28.8 samples/s +Step 29660/44000 | Loss: 2.9568 | LR: 1.24e-04 | 694.7ms/step | 28.8 samples/s +Step 29670/44000 | Loss: 2.9512 | LR: 1.24e-04 | 697.7ms/step | 28.7 samples/s +Step 29680/44000 | Loss: 2.9849 | LR: 1.23e-04 | 700.0ms/step | 28.6 samples/s +Step 29690/44000 | Loss: 2.9438 | LR: 1.23e-04 | 697.9ms/step | 28.7 samples/s +Step 29700/44000 | Loss: 2.9325 | LR: 1.23e-04 | 697.7ms/step | 28.7 samples/s +Step 29710/44000 | Loss: 2.9757 | LR: 1.23e-04 | 695.4ms/step | 28.8 samples/s +Step 29720/44000 | Loss: 2.9337 | LR: 1.23e-04 | 698.4ms/step | 28.6 samples/s +Step 29730/44000 | Loss: 2.8937 | LR: 1.23e-04 | 696.1ms/step | 28.7 samples/s +Step 29740/44000 | Loss: 2.9398 | LR: 1.23e-04 | 699.2ms/step | 28.6 samples/s +Step 29750/44000 | Loss: 2.9769 | LR: 1.22e-04 | 697.0ms/step | 28.7 samples/s +Validation at step 29750: Loss = 2.9681, Perplexity = 19.45 +Step 29760/44000 | Loss: 2.9230 | LR: 1.22e-04 | 1265.9ms/step | 15.8 samples/s +Step 29770/44000 | Loss: 2.9686 | LR: 1.22e-04 | 701.4ms/step | 28.5 samples/s +Step 29780/44000 | Loss: 2.9185 | LR: 1.22e-04 | 701.3ms/step | 28.5 samples/s +Step 29790/44000 | Loss: 2.9281 | LR: 1.22e-04 | 696.1ms/step | 28.7 samples/s +Step 29800/44000 | Loss: 2.9072 | LR: 1.22e-04 | 698.8ms/step | 28.6 samples/s +Step 29810/44000 | Loss: 2.9457 | LR: 1.21e-04 | 699.9ms/step | 28.6 samples/s +Step 29820/44000 | Loss: 2.9265 | LR: 1.21e-04 | 695.5ms/step | 28.8 samples/s +Step 29830/44000 | Loss: 2.9298 | LR: 1.21e-04 | 697.9ms/step | 28.7 samples/s +Step 29840/44000 | Loss: 2.9536 | LR: 1.21e-04 | 700.9ms/step | 28.5 samples/s +Step 29850/44000 | Loss: 2.9191 | LR: 1.21e-04 | 701.0ms/step | 28.5 samples/s +Step 29860/44000 | Loss: 2.9790 | LR: 1.21e-04 | 697.9ms/step | 28.7 samples/s +Step 29870/44000 | Loss: 2.9480 | LR: 1.21e-04 | 698.7ms/step | 28.6 samples/s +Step 29880/44000 | Loss: 2.9431 | LR: 1.20e-04 | 699.5ms/step | 28.6 samples/s +Step 29890/44000 | Loss: 2.9587 | LR: 1.20e-04 | 698.1ms/step | 28.6 samples/s +Step 29900/44000 | Loss: 2.9338 | LR: 1.20e-04 | 700.7ms/step | 28.5 samples/s +Step 29910/44000 | Loss: 2.9689 | LR: 1.20e-04 | 700.0ms/step | 28.6 samples/s +Step 29920/44000 | Loss: 2.9814 | LR: 1.20e-04 | 697.5ms/step | 28.7 samples/s +Step 29930/44000 | Loss: 2.9417 | LR: 1.20e-04 | 694.4ms/step | 28.8 samples/s +Step 29940/44000 | Loss: 2.9850 | LR: 1.19e-04 | 698.9ms/step | 28.6 samples/s +Step 29950/44000 | Loss: 2.8807 | LR: 1.19e-04 | 700.7ms/step | 28.5 samples/s +Step 29960/44000 | Loss: 2.9181 | LR: 1.19e-04 | 700.3ms/step | 28.6 samples/s +Step 29970/44000 | Loss: 2.9308 | LR: 1.19e-04 | 697.4ms/step | 28.7 samples/s +Step 29980/44000 | Loss: 2.9342 | LR: 1.19e-04 | 696.0ms/step | 28.7 samples/s +Step 29990/44000 | Loss: 2.9733 | LR: 1.19e-04 | 698.4ms/step | 28.6 samples/s +Step 30000/44000 | Loss: 2.9136 | LR: 1.19e-04 | 696.6ms/step | 28.7 samples/s +Validation at step 30000: Loss = 2.9672, Perplexity = 19.44 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-30000 +Step 30010/44000 | Loss: 2.9102 | LR: 1.18e-04 | 1584.6ms/step | 12.6 samples/s +Step 30020/44000 | Loss: 2.9643 | LR: 1.18e-04 | 699.2ms/step | 28.6 samples/s +Step 30030/44000 | Loss: 2.9410 | LR: 1.18e-04 | 697.4ms/step | 28.7 samples/s +Step 30040/44000 | Loss: 2.9424 | LR: 1.18e-04 | 696.3ms/step | 28.7 samples/s +Step 30050/44000 | Loss: 2.9387 | LR: 1.18e-04 | 697.9ms/step | 28.7 samples/s +Step 30060/44000 | Loss: 2.9575 | LR: 1.18e-04 | 699.8ms/step | 28.6 samples/s +Step 30070/44000 | Loss: 2.9106 | LR: 1.17e-04 | 699.9ms/step | 28.6 samples/s +Step 30080/44000 | Loss: 2.9403 | LR: 1.17e-04 | 698.9ms/step | 28.6 samples/s +Step 30090/44000 | Loss: 2.9249 | LR: 1.17e-04 | 696.9ms/step | 28.7 samples/s +Step 30100/44000 | Loss: 2.9445 | LR: 1.17e-04 | 694.2ms/step | 28.8 samples/s +Step 30110/44000 | Loss: 2.9518 | LR: 1.17e-04 | 699.6ms/step | 28.6 samples/s +Step 30120/44000 | Loss: 2.9951 | LR: 1.17e-04 | 694.6ms/step | 28.8 samples/s +Step 30130/44000 | Loss: 2.9362 | LR: 1.17e-04 | 694.3ms/step | 28.8 samples/s +Step 30140/44000 | Loss: 2.9487 | LR: 1.16e-04 | 698.7ms/step | 28.6 samples/s +Step 30150/44000 | Loss: 2.9489 | LR: 1.16e-04 | 697.2ms/step | 28.7 samples/s +Step 30160/44000 | Loss: 2.9628 | LR: 1.16e-04 | 697.5ms/step | 28.7 samples/s +Step 30170/44000 | Loss: 2.9411 | LR: 1.16e-04 | 698.0ms/step | 28.7 samples/s +Step 30180/44000 | Loss: 2.9708 | LR: 1.16e-04 | 696.9ms/step | 28.7 samples/s +Step 30190/44000 | Loss: 2.9475 | LR: 1.16e-04 | 696.2ms/step | 28.7 samples/s +Step 30200/44000 | Loss: 2.9765 | LR: 1.15e-04 | 699.2ms/step | 28.6 samples/s +Step 30210/44000 | Loss: 2.9105 | LR: 1.15e-04 | 694.0ms/step | 28.8 samples/s +Step 30220/44000 | Loss: 2.9195 | LR: 1.15e-04 | 695.9ms/step | 28.7 samples/s +Step 30230/44000 | Loss: 2.9328 | LR: 1.15e-04 | 697.8ms/step | 28.7 samples/s +Step 30240/44000 | Loss: 2.9488 | LR: 1.15e-04 | 694.8ms/step | 28.8 samples/s +Step 30250/44000 | Loss: 2.9297 | LR: 1.15e-04 | 694.3ms/step | 28.8 samples/s +Validation at step 30250: Loss = 2.9658, Perplexity = 19.41 +Step 30260/44000 | Loss: 2.9556 | LR: 1.15e-04 | 1264.1ms/step | 15.8 samples/s +Step 30270/44000 | Loss: 2.8875 | LR: 1.14e-04 | 699.1ms/step | 28.6 samples/s +Step 30280/44000 | Loss: 2.9779 | LR: 1.14e-04 | 702.7ms/step | 28.5 samples/s +Step 30290/44000 | Loss: 2.9216 | LR: 1.14e-04 | 698.5ms/step | 28.6 samples/s +Step 30300/44000 | Loss: 2.9353 | LR: 1.14e-04 | 696.6ms/step | 28.7 samples/s +Step 30310/44000 | Loss: 2.9378 | LR: 1.14e-04 | 699.8ms/step | 28.6 samples/s +Step 30320/44000 | Loss: 2.9631 | LR: 1.14e-04 | 698.1ms/step | 28.6 samples/s +Step 30330/44000 | Loss: 2.9008 | LR: 1.13e-04 | 701.8ms/step | 28.5 samples/s +Step 30340/44000 | Loss: 2.9269 | LR: 1.13e-04 | 697.2ms/step | 28.7 samples/s +Step 30350/44000 | Loss: 2.9270 | LR: 1.13e-04 | 700.3ms/step | 28.6 samples/s +Step 30360/44000 | Loss: 2.9680 | LR: 1.13e-04 | 697.0ms/step | 28.7 samples/s +Step 30370/44000 | Loss: 2.9525 | LR: 1.13e-04 | 700.6ms/step | 28.5 samples/s +Step 30380/44000 | Loss: 2.9112 | LR: 1.13e-04 | 697.0ms/step | 28.7 samples/s +Step 30390/44000 | Loss: 2.9007 | LR: 1.13e-04 | 697.3ms/step | 28.7 samples/s +Step 30400/44000 | Loss: 2.9284 | LR: 1.12e-04 | 701.4ms/step | 28.5 samples/s +Step 30410/44000 | Loss: 2.9451 | LR: 1.12e-04 | 700.1ms/step | 28.6 samples/s +Step 30420/44000 | Loss: 2.9468 | LR: 1.12e-04 | 698.9ms/step | 28.6 samples/s +Step 30430/44000 | Loss: 2.9813 | LR: 1.12e-04 | 697.9ms/step | 28.7 samples/s +Step 30440/44000 | Loss: 2.9407 | LR: 1.12e-04 | 699.0ms/step | 28.6 samples/s +Step 30450/44000 | Loss: 2.9267 | LR: 1.12e-04 | 695.7ms/step | 28.7 samples/s +Step 30460/44000 | Loss: 2.9234 | LR: 1.11e-04 | 698.5ms/step | 28.6 samples/s +Step 30470/44000 | Loss: 2.9694 | LR: 1.11e-04 | 695.2ms/step | 28.8 samples/s +Step 30480/44000 | Loss: 2.9466 | LR: 1.11e-04 | 697.6ms/step | 28.7 samples/s +Step 30490/44000 | Loss: 2.9561 | LR: 1.11e-04 | 700.2ms/step | 28.6 samples/s +Step 30500/44000 | Loss: 2.9542 | LR: 1.11e-04 | 698.4ms/step | 28.6 samples/s +Validation at step 30500: Loss = 2.9667, Perplexity = 19.43 +Step 30510/44000 | Loss: 2.9274 | LR: 1.11e-04 | 1266.6ms/step | 15.8 samples/s +Step 30520/44000 | Loss: 2.9619 | LR: 1.11e-04 | 696.4ms/step | 28.7 samples/s +Step 30530/44000 | Loss: 2.8840 | LR: 1.10e-04 | 696.9ms/step | 28.7 samples/s +Step 30540/44000 | Loss: 2.9092 | LR: 1.10e-04 | 693.3ms/step | 28.8 samples/s +Step 30550/44000 | Loss: 2.9482 | LR: 1.10e-04 | 698.4ms/step | 28.6 samples/s +Step 30560/44000 | Loss: 2.9034 | LR: 1.10e-04 | 700.7ms/step | 28.5 samples/s +Step 30570/44000 | Loss: 2.9219 | LR: 1.10e-04 | 698.4ms/step | 28.6 samples/s +Step 30580/44000 | Loss: 2.9692 | LR: 1.10e-04 | 698.4ms/step | 28.6 samples/s +Step 30590/44000 | Loss: 2.9498 | LR: 1.10e-04 | 699.5ms/step | 28.6 samples/s +Step 30600/44000 | Loss: 2.9743 | LR: 1.09e-04 | 702.5ms/step | 28.5 samples/s +Step 30610/44000 | Loss: 2.9241 | LR: 1.09e-04 | 699.3ms/step | 28.6 samples/s +Step 30620/44000 | Loss: 2.9690 | LR: 1.09e-04 | 697.4ms/step | 28.7 samples/s +Step 30630/44000 | Loss: 2.9572 | LR: 1.09e-04 | 698.2ms/step | 28.6 samples/s +Step 30640/44000 | Loss: 2.9174 | LR: 1.09e-04 | 695.5ms/step | 28.8 samples/s +Step 30650/44000 | Loss: 2.8986 | LR: 1.09e-04 | 698.6ms/step | 28.6 samples/s +Step 30660/44000 | Loss: 2.9204 | LR: 1.08e-04 | 698.4ms/step | 28.6 samples/s +Step 30670/44000 | Loss: 2.9686 | LR: 1.08e-04 | 698.1ms/step | 28.7 samples/s +Step 30680/44000 | Loss: 2.9556 | LR: 1.08e-04 | 698.2ms/step | 28.6 samples/s +Step 30690/44000 | Loss: 2.9637 | LR: 1.08e-04 | 696.2ms/step | 28.7 samples/s +Step 30700/44000 | Loss: 2.8982 | LR: 1.08e-04 | 699.0ms/step | 28.6 samples/s +Step 30710/44000 | Loss: 2.9430 | LR: 1.08e-04 | 700.7ms/step | 28.5 samples/s +Step 30720/44000 | Loss: 2.9839 | LR: 1.08e-04 | 695.8ms/step | 28.7 samples/s +Step 30730/44000 | Loss: 2.9486 | LR: 1.07e-04 | 696.3ms/step | 28.7 samples/s +Step 30740/44000 | Loss: 2.9749 | LR: 1.07e-04 | 697.7ms/step | 28.7 samples/s +Step 30750/44000 | Loss: 2.8856 | LR: 1.07e-04 | 696.7ms/step | 28.7 samples/s +Validation at step 30750: Loss = 2.9670, Perplexity = 19.43 +Step 30760/44000 | Loss: 2.9649 | LR: 1.07e-04 | 1267.3ms/step | 15.8 samples/s +Step 30770/44000 | Loss: 2.9161 | LR: 1.07e-04 | 698.7ms/step | 28.6 samples/s +Step 30780/44000 | Loss: 2.9267 | LR: 1.07e-04 | 697.3ms/step | 28.7 samples/s +Step 30790/44000 | Loss: 2.9800 | LR: 1.07e-04 | 697.6ms/step | 28.7 samples/s +Step 30800/44000 | Loss: 2.9236 | LR: 1.06e-04 | 699.8ms/step | 28.6 samples/s +Step 30810/44000 | Loss: 2.9016 | LR: 1.06e-04 | 697.7ms/step | 28.7 samples/s +Step 30820/44000 | Loss: 2.9258 | LR: 1.06e-04 | 698.1ms/step | 28.6 samples/s +Step 30830/44000 | Loss: 2.9744 | LR: 1.06e-04 | 697.3ms/step | 28.7 samples/s +Step 30840/44000 | Loss: 2.9169 | LR: 1.06e-04 | 696.4ms/step | 28.7 samples/s +Step 30850/44000 | Loss: 2.9702 | LR: 1.06e-04 | 696.9ms/step | 28.7 samples/s +Step 30860/44000 | Loss: 2.9404 | LR: 1.05e-04 | 697.5ms/step | 28.7 samples/s +Step 30870/44000 | Loss: 2.9465 | LR: 1.05e-04 | 696.7ms/step | 28.7 samples/s +Step 30880/44000 | Loss: 2.9338 | LR: 1.05e-04 | 695.6ms/step | 28.8 samples/s +Step 30890/44000 | Loss: 2.9371 | LR: 1.05e-04 | 698.8ms/step | 28.6 samples/s +Step 30900/44000 | Loss: 2.8973 | LR: 1.05e-04 | 696.9ms/step | 28.7 samples/s +Step 30910/44000 | Loss: 2.9640 | LR: 1.05e-04 | 697.2ms/step | 28.7 samples/s +Step 30920/44000 | Loss: 2.9446 | LR: 1.05e-04 | 696.1ms/step | 28.7 samples/s +Step 30930/44000 | Loss: 2.9537 | LR: 1.04e-04 | 700.1ms/step | 28.6 samples/s +Step 30940/44000 | Loss: 2.9013 | LR: 1.04e-04 | 701.3ms/step | 28.5 samples/s +Step 30950/44000 | Loss: 2.9154 | LR: 1.04e-04 | 700.0ms/step | 28.6 samples/s +Step 30960/44000 | Loss: 2.9352 | LR: 1.04e-04 | 695.5ms/step | 28.8 samples/s +Step 30970/44000 | Loss: 2.9073 | LR: 1.04e-04 | 696.3ms/step | 28.7 samples/s +Step 30980/44000 | Loss: 2.9123 | LR: 1.04e-04 | 697.7ms/step | 28.7 samples/s +Step 30990/44000 | Loss: 2.9410 | LR: 1.04e-04 | 696.8ms/step | 28.7 samples/s +Step 31000/44000 | Loss: 2.9024 | LR: 1.03e-04 | 699.0ms/step | 28.6 samples/s +Validation at step 31000: Loss = 2.9660, Perplexity = 19.41 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-31000 +Step 31010/44000 | Loss: 2.9589 | LR: 1.03e-04 | 1581.0ms/step | 12.7 samples/s +Step 31020/44000 | Loss: 2.9071 | LR: 1.03e-04 | 696.8ms/step | 28.7 samples/s +Step 31030/44000 | Loss: 2.9472 | LR: 1.03e-04 | 699.6ms/step | 28.6 samples/s +Step 31040/44000 | Loss: 2.9073 | LR: 1.03e-04 | 695.0ms/step | 28.8 samples/s +Step 31050/44000 | Loss: 2.9728 | LR: 1.03e-04 | 695.6ms/step | 28.8 samples/s +Step 31060/44000 | Loss: 2.9635 | LR: 1.03e-04 | 698.4ms/step | 28.6 samples/s +Step 31070/44000 | Loss: 2.9199 | LR: 1.02e-04 | 697.9ms/step | 28.7 samples/s +Step 31080/44000 | Loss: 2.9174 | LR: 1.02e-04 | 697.4ms/step | 28.7 samples/s +Step 31090/44000 | Loss: 2.9290 | LR: 1.02e-04 | 701.8ms/step | 28.5 samples/s +Step 31100/44000 | Loss: 2.9611 | LR: 1.02e-04 | 697.4ms/step | 28.7 samples/s +Step 31110/44000 | Loss: 2.9555 | LR: 1.02e-04 | 698.7ms/step | 28.6 samples/s +Step 31120/44000 | Loss: 2.9304 | LR: 1.02e-04 | 698.7ms/step | 28.6 samples/s +Step 31130/44000 | Loss: 2.9698 | LR: 1.02e-04 | 696.0ms/step | 28.7 samples/s +Step 31140/44000 | Loss: 2.9257 | LR: 1.01e-04 | 694.8ms/step | 28.8 samples/s +Step 31150/44000 | Loss: 2.9558 | LR: 1.01e-04 | 699.1ms/step | 28.6 samples/s +Step 31160/44000 | Loss: 3.0074 | LR: 1.01e-04 | 697.5ms/step | 28.7 samples/s +Step 31170/44000 | Loss: 2.9309 | LR: 1.01e-04 | 699.7ms/step | 28.6 samples/s +Step 31180/44000 | Loss: 2.9173 | LR: 1.01e-04 | 696.9ms/step | 28.7 samples/s +Step 31190/44000 | Loss: 2.9447 | LR: 1.01e-04 | 695.3ms/step | 28.8 samples/s +Step 31200/44000 | Loss: 2.9218 | LR: 1.00e-04 | 695.0ms/step | 28.8 samples/s +Step 31210/44000 | Loss: 2.9787 | LR: 1.00e-04 | 699.2ms/step | 28.6 samples/s +Step 31220/44000 | Loss: 2.9519 | LR: 1.00e-04 | 698.4ms/step | 28.6 samples/s +Step 31230/44000 | Loss: 2.8927 | LR: 1.00e-04 | 699.8ms/step | 28.6 samples/s +Step 31240/44000 | Loss: 2.9477 | LR: 9.99e-05 | 698.2ms/step | 28.6 samples/s +Step 31250/44000 | Loss: 2.9935 | LR: 9.98e-05 | 696.0ms/step | 28.7 samples/s +Validation at step 31250: Loss = 2.9630, Perplexity = 19.36 +Step 31260/44000 | Loss: 2.9460 | LR: 9.96e-05 | 1264.1ms/step | 15.8 samples/s +Step 31270/44000 | Loss: 2.9399 | LR: 9.95e-05 | 697.3ms/step | 28.7 samples/s +Step 31280/44000 | Loss: 2.9542 | LR: 9.93e-05 | 694.8ms/step | 28.8 samples/s +Step 31290/44000 | Loss: 2.9425 | LR: 9.92e-05 | 696.6ms/step | 28.7 samples/s +Step 31300/44000 | Loss: 2.9335 | LR: 9.90e-05 | 701.2ms/step | 28.5 samples/s +Step 31310/44000 | Loss: 2.9556 | LR: 9.89e-05 | 696.3ms/step | 28.7 samples/s +Step 31320/44000 | Loss: 2.9192 | LR: 9.88e-05 | 698.3ms/step | 28.6 samples/s +Step 31330/44000 | Loss: 2.9719 | LR: 9.86e-05 | 697.4ms/step | 28.7 samples/s +Step 31340/44000 | Loss: 2.9205 | LR: 9.85e-05 | 694.2ms/step | 28.8 samples/s +Step 31350/44000 | Loss: 2.9416 | LR: 9.83e-05 | 696.4ms/step | 28.7 samples/s +Step 31360/44000 | Loss: 2.9552 | LR: 9.82e-05 | 697.4ms/step | 28.7 samples/s +Step 31370/44000 | Loss: 2.9379 | LR: 9.80e-05 | 696.6ms/step | 28.7 samples/s +Step 31380/44000 | Loss: 2.9446 | LR: 9.79e-05 | 696.6ms/step | 28.7 samples/s +Step 31390/44000 | Loss: 2.9486 | LR: 9.77e-05 | 695.3ms/step | 28.8 samples/s +Step 31400/44000 | Loss: 2.9470 | LR: 9.76e-05 | 697.8ms/step | 28.7 samples/s +Step 31410/44000 | Loss: 2.9164 | LR: 9.75e-05 | 695.6ms/step | 28.8 samples/s +Step 31420/44000 | Loss: 2.9448 | LR: 9.73e-05 | 698.3ms/step | 28.6 samples/s +Step 31430/44000 | Loss: 2.9251 | LR: 9.72e-05 | 700.3ms/step | 28.6 samples/s +Step 31440/44000 | Loss: 2.9528 | LR: 9.70e-05 | 697.7ms/step | 28.7 samples/s +Step 31450/44000 | Loss: 2.9264 | LR: 9.69e-05 | 695.9ms/step | 28.7 samples/s +Step 31460/44000 | Loss: 2.9526 | LR: 9.67e-05 | 700.0ms/step | 28.6 samples/s +Step 31470/44000 | Loss: 2.9349 | LR: 9.66e-05 | 697.5ms/step | 28.7 samples/s +Step 31480/44000 | Loss: 2.9804 | LR: 9.65e-05 | 699.2ms/step | 28.6 samples/s +Step 31490/44000 | Loss: 2.9378 | LR: 9.63e-05 | 698.4ms/step | 28.6 samples/s +Step 31500/44000 | Loss: 2.9254 | LR: 9.62e-05 | 697.5ms/step | 28.7 samples/s +Validation at step 31500: Loss = 2.9640, Perplexity = 19.37 +Step 31510/44000 | Loss: 2.8981 | LR: 9.60e-05 | 1262.8ms/step | 15.8 samples/s +Step 31520/44000 | Loss: 2.9534 | LR: 9.59e-05 | 702.0ms/step | 28.5 samples/s +Step 31530/44000 | Loss: 2.8861 | LR: 9.57e-05 | 698.6ms/step | 28.6 samples/s +Step 31540/44000 | Loss: 2.9045 | LR: 9.56e-05 | 698.3ms/step | 28.6 samples/s +Step 31550/44000 | Loss: 2.9377 | LR: 9.54e-05 | 697.0ms/step | 28.7 samples/s +Step 31560/44000 | Loss: 2.9669 | LR: 9.53e-05 | 697.8ms/step | 28.7 samples/s +Step 31570/44000 | Loss: 2.9343 | LR: 9.52e-05 | 699.6ms/step | 28.6 samples/s +Step 31580/44000 | Loss: 2.9691 | LR: 9.50e-05 | 697.5ms/step | 28.7 samples/s +Step 31590/44000 | Loss: 2.9753 | LR: 9.49e-05 | 698.2ms/step | 28.6 samples/s +Step 31600/44000 | Loss: 2.9557 | LR: 9.47e-05 | 694.4ms/step | 28.8 samples/s +Step 31610/44000 | Loss: 2.9255 | LR: 9.46e-05 | 696.6ms/step | 28.7 samples/s +Step 31620/44000 | Loss: 2.9604 | LR: 9.45e-05 | 696.8ms/step | 28.7 samples/s +Step 31630/44000 | Loss: 2.9729 | LR: 9.43e-05 | 697.6ms/step | 28.7 samples/s +Step 31640/44000 | Loss: 2.9270 | LR: 9.42e-05 | 699.2ms/step | 28.6 samples/s +Step 31650/44000 | Loss: 2.9371 | LR: 9.40e-05 | 699.4ms/step | 28.6 samples/s +Step 31660/44000 | Loss: 2.9354 | LR: 9.39e-05 | 697.8ms/step | 28.7 samples/s +Step 31670/44000 | Loss: 2.9193 | LR: 9.37e-05 | 697.0ms/step | 28.7 samples/s +Step 31680/44000 | Loss: 2.9110 | LR: 9.36e-05 | 698.6ms/step | 28.6 samples/s +Step 31690/44000 | Loss: 2.9791 | LR: 9.35e-05 | 696.8ms/step | 28.7 samples/s +Step 31700/44000 | Loss: 3.0234 | LR: 9.33e-05 | 699.4ms/step | 28.6 samples/s +Step 31710/44000 | Loss: 2.9459 | LR: 9.32e-05 | 697.9ms/step | 28.7 samples/s +Step 31720/44000 | Loss: 2.9206 | LR: 9.30e-05 | 698.4ms/step | 28.6 samples/s +Step 31730/44000 | Loss: 2.8906 | LR: 9.29e-05 | 699.4ms/step | 28.6 samples/s +Step 31740/44000 | Loss: 2.9530 | LR: 9.28e-05 | 698.2ms/step | 28.6 samples/s +Step 31750/44000 | Loss: 2.9256 | LR: 9.26e-05 | 697.9ms/step | 28.7 samples/s +Validation at step 31750: Loss = 2.9614, Perplexity = 19.33 +Step 31760/44000 | Loss: 2.8836 | LR: 9.25e-05 | 1266.5ms/step | 15.8 samples/s +Step 31770/44000 | Loss: 2.9768 | LR: 9.23e-05 | 698.2ms/step | 28.6 samples/s +Step 31780/44000 | Loss: 2.9622 | LR: 9.22e-05 | 694.4ms/step | 28.8 samples/s +Step 31790/44000 | Loss: 2.9010 | LR: 9.20e-05 | 697.8ms/step | 28.7 samples/s +Step 31800/44000 | Loss: 2.9800 | LR: 9.19e-05 | 698.8ms/step | 28.6 samples/s +Step 31810/44000 | Loss: 2.9219 | LR: 9.18e-05 | 699.7ms/step | 28.6 samples/s +Step 31820/44000 | Loss: 2.9492 | LR: 9.16e-05 | 696.1ms/step | 28.7 samples/s +Step 31830/44000 | Loss: 2.9641 | LR: 9.15e-05 | 698.1ms/step | 28.6 samples/s +Step 31840/44000 | Loss: 2.9268 | LR: 9.13e-05 | 700.5ms/step | 28.5 samples/s +Step 31850/44000 | Loss: 2.8893 | LR: 9.12e-05 | 701.3ms/step | 28.5 samples/s +Step 31860/44000 | Loss: 2.9656 | LR: 9.11e-05 | 697.8ms/step | 28.7 samples/s +Step 31870/44000 | Loss: 2.9648 | LR: 9.09e-05 | 699.4ms/step | 28.6 samples/s +Step 31880/44000 | Loss: 2.9539 | LR: 9.08e-05 | 699.5ms/step | 28.6 samples/s +Step 31890/44000 | Loss: 2.9181 | LR: 9.06e-05 | 699.7ms/step | 28.6 samples/s +Step 31900/44000 | Loss: 2.9627 | LR: 9.05e-05 | 697.4ms/step | 28.7 samples/s +Step 31910/44000 | Loss: 2.9405 | LR: 9.04e-05 | 700.0ms/step | 28.6 samples/s +Step 31920/44000 | Loss: 2.9318 | LR: 9.02e-05 | 698.2ms/step | 28.6 samples/s +Step 31930/44000 | Loss: 2.9888 | LR: 9.01e-05 | 696.6ms/step | 28.7 samples/s +Step 31940/44000 | Loss: 2.9531 | LR: 8.99e-05 | 698.4ms/step | 28.6 samples/s +Step 31950/44000 | Loss: 2.9616 | LR: 8.98e-05 | 696.1ms/step | 28.7 samples/s +Step 31960/44000 | Loss: 2.9837 | LR: 8.97e-05 | 695.9ms/step | 28.7 samples/s +Step 31970/44000 | Loss: 2.9282 | LR: 8.95e-05 | 697.5ms/step | 28.7 samples/s +Step 31980/44000 | Loss: 2.9316 | LR: 8.94e-05 | 697.1ms/step | 28.7 samples/s +Step 31990/44000 | Loss: 2.9330 | LR: 8.92e-05 | 700.2ms/step | 28.6 samples/s +Step 32000/44000 | Loss: 2.9431 | LR: 8.91e-05 | 701.2ms/step | 28.5 samples/s +Validation at step 32000: Loss = 2.9616, Perplexity = 19.33 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-32000 +Step 32010/44000 | Loss: 2.9159 | LR: 8.90e-05 | 1577.8ms/step | 12.7 samples/s +Step 32020/44000 | Loss: 2.9278 | LR: 8.88e-05 | 698.1ms/step | 28.6 samples/s +Step 32030/44000 | Loss: 2.9256 | LR: 8.87e-05 | 699.9ms/step | 28.6 samples/s +Step 32040/44000 | Loss: 2.9454 | LR: 8.86e-05 | 698.2ms/step | 28.6 samples/s +Step 32050/44000 | Loss: 2.9671 | LR: 8.84e-05 | 698.5ms/step | 28.6 samples/s +Step 32060/44000 | Loss: 2.9811 | LR: 8.83e-05 | 701.0ms/step | 28.5 samples/s +Step 32070/44000 | Loss: 2.9313 | LR: 8.81e-05 | 697.8ms/step | 28.7 samples/s +Step 32080/44000 | Loss: 2.9494 | LR: 8.80e-05 | 699.6ms/step | 28.6 samples/s +Step 32090/44000 | Loss: 2.9260 | LR: 8.79e-05 | 699.4ms/step | 28.6 samples/s +Step 32100/44000 | Loss: 2.9567 | LR: 8.77e-05 | 699.8ms/step | 28.6 samples/s +Step 32110/44000 | Loss: 2.9378 | LR: 8.76e-05 | 696.6ms/step | 28.7 samples/s +Step 32120/44000 | Loss: 2.9459 | LR: 8.74e-05 | 696.7ms/step | 28.7 samples/s +Step 32130/44000 | Loss: 2.9369 | LR: 8.73e-05 | 698.1ms/step | 28.6 samples/s +Step 32140/44000 | Loss: 2.9162 | LR: 8.72e-05 | 697.4ms/step | 28.7 samples/s +Step 32150/44000 | Loss: 2.9507 | LR: 8.70e-05 | 699.8ms/step | 28.6 samples/s +Step 32160/44000 | Loss: 2.9412 | LR: 8.69e-05 | 696.3ms/step | 28.7 samples/s +Step 32170/44000 | Loss: 2.9553 | LR: 8.68e-05 | 697.4ms/step | 28.7 samples/s +Step 32180/44000 | Loss: 2.9477 | LR: 8.66e-05 | 700.3ms/step | 28.6 samples/s +Step 32190/44000 | Loss: 2.8915 | LR: 8.65e-05 | 701.1ms/step | 28.5 samples/s +Step 32200/44000 | Loss: 2.9761 | LR: 8.63e-05 | 696.4ms/step | 28.7 samples/s +Step 32210/44000 | Loss: 2.9429 | LR: 8.62e-05 | 700.0ms/step | 28.6 samples/s +Step 32220/44000 | Loss: 2.9671 | LR: 8.61e-05 | 697.8ms/step | 28.7 samples/s +Step 32230/44000 | Loss: 2.9519 | LR: 8.59e-05 | 699.0ms/step | 28.6 samples/s +Step 32240/44000 | Loss: 2.9251 | LR: 8.58e-05 | 700.5ms/step | 28.6 samples/s +Step 32250/44000 | Loss: 2.9490 | LR: 8.57e-05 | 698.1ms/step | 28.6 samples/s +Validation at step 32250: Loss = 2.9598, Perplexity = 19.29 +Step 32260/44000 | Loss: 2.9456 | LR: 8.55e-05 | 1266.7ms/step | 15.8 samples/s +Step 32270/44000 | Loss: 2.9289 | LR: 8.54e-05 | 701.1ms/step | 28.5 samples/s +Step 32280/44000 | Loss: 2.9606 | LR: 8.53e-05 | 698.6ms/step | 28.6 samples/s +Step 32290/44000 | Loss: 2.9906 | LR: 8.51e-05 | 698.8ms/step | 28.6 samples/s +Step 32300/44000 | Loss: 2.9748 | LR: 8.50e-05 | 698.3ms/step | 28.6 samples/s +Step 32310/44000 | Loss: 2.9541 | LR: 8.48e-05 | 698.2ms/step | 28.6 samples/s +Step 32320/44000 | Loss: 2.9417 | LR: 8.47e-05 | 697.3ms/step | 28.7 samples/s +Step 32330/44000 | Loss: 2.9934 | LR: 8.46e-05 | 698.3ms/step | 28.6 samples/s +Step 32340/44000 | Loss: 2.9083 | LR: 8.44e-05 | 699.7ms/step | 28.6 samples/s +Step 32350/44000 | Loss: 2.9653 | LR: 8.43e-05 | 697.0ms/step | 28.7 samples/s +Step 32360/44000 | Loss: 2.9617 | LR: 8.42e-05 | 699.2ms/step | 28.6 samples/s +Step 32370/44000 | Loss: 2.9506 | LR: 8.40e-05 | 699.7ms/step | 28.6 samples/s +Step 32380/44000 | Loss: 2.9384 | LR: 8.39e-05 | 698.4ms/step | 28.6 samples/s +Step 32390/44000 | Loss: 2.9240 | LR: 8.38e-05 | 697.5ms/step | 28.7 samples/s +Step 32400/44000 | Loss: 2.9847 | LR: 8.36e-05 | 698.1ms/step | 28.7 samples/s +Step 32410/44000 | Loss: 2.9351 | LR: 8.35e-05 | 696.2ms/step | 28.7 samples/s +Step 32420/44000 | Loss: 2.9046 | LR: 8.33e-05 | 696.7ms/step | 28.7 samples/s +Step 32430/44000 | Loss: 2.9195 | LR: 8.32e-05 | 700.3ms/step | 28.6 samples/s +Step 32440/44000 | Loss: 2.9718 | LR: 8.31e-05 | 698.3ms/step | 28.6 samples/s +Step 32450/44000 | Loss: 2.9557 | LR: 8.29e-05 | 699.1ms/step | 28.6 samples/s +Step 32460/44000 | Loss: 2.9162 | LR: 8.28e-05 | 699.5ms/step | 28.6 samples/s +Step 32470/44000 | Loss: 2.9407 | LR: 8.27e-05 | 699.4ms/step | 28.6 samples/s +Step 32480/44000 | Loss: 2.9423 | LR: 8.25e-05 | 699.7ms/step | 28.6 samples/s +Step 32490/44000 | Loss: 2.9281 | LR: 8.24e-05 | 700.3ms/step | 28.6 samples/s +Step 32500/44000 | Loss: 2.9645 | LR: 8.23e-05 | 695.1ms/step | 28.8 samples/s +Validation at step 32500: Loss = 2.9604, Perplexity = 19.31 +Step 32510/44000 | Loss: 2.9648 | LR: 8.21e-05 | 1268.0ms/step | 15.8 samples/s +Step 32520/44000 | Loss: 2.9433 | LR: 8.20e-05 | 696.6ms/step | 28.7 samples/s +Step 32530/44000 | Loss: 2.9389 | LR: 8.19e-05 | 698.4ms/step | 28.6 samples/s +Step 32540/44000 | Loss: 2.9355 | LR: 8.17e-05 | 700.8ms/step | 28.5 samples/s +Step 32550/44000 | Loss: 2.9533 | LR: 8.16e-05 | 699.7ms/step | 28.6 samples/s +Step 32560/44000 | Loss: 2.9596 | LR: 8.15e-05 | 697.4ms/step | 28.7 samples/s +Step 32570/44000 | Loss: 2.9393 | LR: 8.13e-05 | 698.2ms/step | 28.6 samples/s +Step 32580/44000 | Loss: 2.9118 | LR: 8.12e-05 | 696.2ms/step | 28.7 samples/s +Step 32590/44000 | Loss: 2.9361 | LR: 8.11e-05 | 697.9ms/step | 28.7 samples/s +Step 32600/44000 | Loss: 2.9276 | LR: 8.09e-05 | 698.1ms/step | 28.6 samples/s +Step 32610/44000 | Loss: 2.9683 | LR: 8.08e-05 | 698.6ms/step | 28.6 samples/s +Step 32620/44000 | Loss: 2.8817 | LR: 8.07e-05 | 698.0ms/step | 28.7 samples/s +Step 32630/44000 | Loss: 2.9740 | LR: 8.05e-05 | 698.8ms/step | 28.6 samples/s +Step 32640/44000 | Loss: 2.9259 | LR: 8.04e-05 | 699.6ms/step | 28.6 samples/s +Step 32650/44000 | Loss: 2.9128 | LR: 8.03e-05 | 695.3ms/step | 28.8 samples/s +Step 32660/44000 | Loss: 2.8950 | LR: 8.01e-05 | 698.2ms/step | 28.6 samples/s +Step 32670/44000 | Loss: 2.9272 | LR: 8.00e-05 | 696.3ms/step | 28.7 samples/s +Step 32680/44000 | Loss: 2.9760 | LR: 7.99e-05 | 694.8ms/step | 28.8 samples/s +Step 32690/44000 | Loss: 2.9708 | LR: 7.97e-05 | 694.8ms/step | 28.8 samples/s +Step 32700/44000 | Loss: 2.9479 | LR: 7.96e-05 | 696.2ms/step | 28.7 samples/s +Step 32710/44000 | Loss: 2.9592 | LR: 7.95e-05 | 695.6ms/step | 28.8 samples/s +Step 32720/44000 | Loss: 2.9358 | LR: 7.93e-05 | 694.2ms/step | 28.8 samples/s +Step 32730/44000 | Loss: 2.9034 | LR: 7.92e-05 | 695.0ms/step | 28.8 samples/s +Step 32740/44000 | Loss: 2.9437 | LR: 7.91e-05 | 695.7ms/step | 28.7 samples/s +Step 32750/44000 | Loss: 2.9559 | LR: 7.89e-05 | 697.1ms/step | 28.7 samples/s +Validation at step 32750: Loss = 2.9618, Perplexity = 19.33 +Step 32760/44000 | Loss: 2.9245 | LR: 7.88e-05 | 1270.2ms/step | 15.7 samples/s +Step 32770/44000 | Loss: 2.9269 | LR: 7.87e-05 | 696.0ms/step | 28.7 samples/s +Step 32780/44000 | Loss: 2.9687 | LR: 7.85e-05 | 697.8ms/step | 28.7 samples/s +Step 32790/44000 | Loss: 2.9101 | LR: 7.84e-05 | 702.2ms/step | 28.5 samples/s +Step 32800/44000 | Loss: 2.9664 | LR: 7.83e-05 | 695.2ms/step | 28.8 samples/s +Step 32810/44000 | Loss: 2.9143 | LR: 7.81e-05 | 699.4ms/step | 28.6 samples/s +Step 32820/44000 | Loss: 2.9385 | LR: 7.80e-05 | 700.5ms/step | 28.6 samples/s +Step 32830/44000 | Loss: 2.9535 | LR: 7.79e-05 | 699.3ms/step | 28.6 samples/s +Step 32840/44000 | Loss: 2.9690 | LR: 7.77e-05 | 697.1ms/step | 28.7 samples/s +Step 32850/44000 | Loss: 2.9667 | LR: 7.76e-05 | 691.3ms/step | 28.9 samples/s +Step 32860/44000 | Loss: 2.9742 | LR: 7.75e-05 | 698.6ms/step | 28.6 samples/s +Step 32870/44000 | Loss: 2.9337 | LR: 7.73e-05 | 696.4ms/step | 28.7 samples/s +Step 32880/44000 | Loss: 2.9486 | LR: 7.72e-05 | 695.8ms/step | 28.7 samples/s +Step 32890/44000 | Loss: 2.9763 | LR: 7.71e-05 | 699.9ms/step | 28.6 samples/s +Step 32900/44000 | Loss: 2.9119 | LR: 7.70e-05 | 696.7ms/step | 28.7 samples/s +Step 32910/44000 | Loss: 2.9542 | LR: 7.68e-05 | 692.8ms/step | 28.9 samples/s +Step 32920/44000 | Loss: 2.9588 | LR: 7.67e-05 | 694.4ms/step | 28.8 samples/s +Step 32930/44000 | Loss: 2.8772 | LR: 7.66e-05 | 698.4ms/step | 28.6 samples/s +Step 32940/44000 | Loss: 2.8767 | LR: 7.64e-05 | 696.0ms/step | 28.7 samples/s +Step 32950/44000 | Loss: 2.9571 | LR: 7.63e-05 | 694.6ms/step | 28.8 samples/s +Step 32960/44000 | Loss: 2.9224 | LR: 7.62e-05 | 697.4ms/step | 28.7 samples/s +Step 32970/44000 | Loss: 2.9240 | LR: 7.60e-05 | 697.8ms/step | 28.7 samples/s +Step 32980/44000 | Loss: 2.9284 | LR: 7.59e-05 | 696.9ms/step | 28.7 samples/s +Step 32990/44000 | Loss: 2.9443 | LR: 7.58e-05 | 699.3ms/step | 28.6 samples/s +Step 33000/44000 | Loss: 2.9239 | LR: 7.56e-05 | 697.1ms/step | 28.7 samples/s +Validation at step 33000: Loss = 2.9593, Perplexity = 19.29 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-33000 +Step 33010/44000 | Loss: 2.9407 | LR: 7.55e-05 | 1626.4ms/step | 12.3 samples/s +Step 33020/44000 | Loss: 2.9271 | LR: 7.54e-05 | 697.8ms/step | 28.7 samples/s +Step 33030/44000 | Loss: 2.9450 | LR: 7.53e-05 | 699.0ms/step | 28.6 samples/s +Step 33040/44000 | Loss: 2.9421 | LR: 7.51e-05 | 696.9ms/step | 28.7 samples/s +Step 33050/44000 | Loss: 2.9388 | LR: 7.50e-05 | 696.2ms/step | 28.7 samples/s +Step 33060/44000 | Loss: 2.9533 | LR: 7.49e-05 | 697.5ms/step | 28.7 samples/s +Step 33070/44000 | Loss: 2.9117 | LR: 7.47e-05 | 696.0ms/step | 28.7 samples/s +Step 33080/44000 | Loss: 2.9183 | LR: 7.46e-05 | 696.6ms/step | 28.7 samples/s +Step 33090/44000 | Loss: 2.9568 | LR: 7.45e-05 | 694.7ms/step | 28.8 samples/s +Step 33100/44000 | Loss: 2.9897 | LR: 7.44e-05 | 694.8ms/step | 28.8 samples/s +Step 33110/44000 | Loss: 2.9515 | LR: 7.42e-05 | 694.5ms/step | 28.8 samples/s +Step 33120/44000 | Loss: 2.9637 | LR: 7.41e-05 | 694.9ms/step | 28.8 samples/s +Step 33130/44000 | Loss: 2.9246 | LR: 7.40e-05 | 698.3ms/step | 28.6 samples/s +Step 33140/44000 | Loss: 2.9317 | LR: 7.38e-05 | 697.1ms/step | 28.7 samples/s +Step 33150/44000 | Loss: 2.9342 | LR: 7.37e-05 | 696.9ms/step | 28.7 samples/s +Step 33160/44000 | Loss: 2.9310 | LR: 7.36e-05 | 696.7ms/step | 28.7 samples/s +Step 33170/44000 | Loss: 2.9748 | LR: 7.34e-05 | 697.6ms/step | 28.7 samples/s +Step 33180/44000 | Loss: 2.9441 | LR: 7.33e-05 | 698.5ms/step | 28.6 samples/s +Step 33190/44000 | Loss: 2.9174 | LR: 7.32e-05 | 696.5ms/step | 28.7 samples/s +Step 33200/44000 | Loss: 2.9759 | LR: 7.31e-05 | 697.2ms/step | 28.7 samples/s +Step 33210/44000 | Loss: 2.9570 | LR: 7.29e-05 | 697.5ms/step | 28.7 samples/s +Step 33220/44000 | Loss: 2.9557 | LR: 7.28e-05 | 697.2ms/step | 28.7 samples/s +Step 33230/44000 | Loss: 2.9349 | LR: 7.27e-05 | 697.1ms/step | 28.7 samples/s +Step 33240/44000 | Loss: 2.9490 | LR: 7.26e-05 | 695.2ms/step | 28.8 samples/s +Step 33250/44000 | Loss: 2.9777 | LR: 7.24e-05 | 697.2ms/step | 28.7 samples/s +Validation at step 33250: Loss = 2.9602, Perplexity = 19.30 +Step 33260/44000 | Loss: 2.8907 | LR: 7.23e-05 | 1272.7ms/step | 15.7 samples/s +Step 33270/44000 | Loss: 2.9126 | LR: 7.22e-05 | 699.9ms/step | 28.6 samples/s +Step 33280/44000 | Loss: 2.9357 | LR: 7.20e-05 | 699.5ms/step | 28.6 samples/s +Step 33290/44000 | Loss: 2.9295 | LR: 7.19e-05 | 699.3ms/step | 28.6 samples/s +Step 33300/44000 | Loss: 2.9520 | LR: 7.18e-05 | 699.5ms/step | 28.6 samples/s +Step 33310/44000 | Loss: 2.9358 | LR: 7.17e-05 | 696.9ms/step | 28.7 samples/s +Step 33320/44000 | Loss: 2.9473 | LR: 7.15e-05 | 696.7ms/step | 28.7 samples/s +Step 33330/44000 | Loss: 2.8970 | LR: 7.14e-05 | 695.2ms/step | 28.8 samples/s +Step 33340/44000 | Loss: 2.9175 | LR: 7.13e-05 | 699.5ms/step | 28.6 samples/s +Step 33350/44000 | Loss: 2.9585 | LR: 7.11e-05 | 698.6ms/step | 28.6 samples/s +Step 33360/44000 | Loss: 2.9634 | LR: 7.10e-05 | 697.6ms/step | 28.7 samples/s +Step 33370/44000 | Loss: 2.9216 | LR: 7.09e-05 | 697.8ms/step | 28.7 samples/s +Step 33380/44000 | Loss: 2.8871 | LR: 7.08e-05 | 701.9ms/step | 28.5 samples/s +Step 33390/44000 | Loss: 2.9611 | LR: 7.06e-05 | 694.2ms/step | 28.8 samples/s +Step 33400/44000 | Loss: 2.9550 | LR: 7.05e-05 | 699.6ms/step | 28.6 samples/s +Step 33410/44000 | Loss: 2.9559 | LR: 7.04e-05 | 698.9ms/step | 28.6 samples/s +Step 33420/44000 | Loss: 2.9854 | LR: 7.03e-05 | 699.2ms/step | 28.6 samples/s +Step 33430/44000 | Loss: 2.9268 | LR: 7.01e-05 | 695.2ms/step | 28.8 samples/s +Step 33440/44000 | Loss: 2.9714 | LR: 7.00e-05 | 696.6ms/step | 28.7 samples/s +Step 33450/44000 | Loss: 2.9582 | LR: 6.99e-05 | 698.8ms/step | 28.6 samples/s +Step 33460/44000 | Loss: 2.9390 | LR: 6.98e-05 | 698.6ms/step | 28.6 samples/s +Step 33470/44000 | Loss: 2.9596 | LR: 6.96e-05 | 695.7ms/step | 28.7 samples/s +Step 33480/44000 | Loss: 2.9538 | LR: 6.95e-05 | 692.7ms/step | 28.9 samples/s +Step 33490/44000 | Loss: 2.9567 | LR: 6.94e-05 | 693.7ms/step | 28.8 samples/s +Step 33500/44000 | Loss: 2.9091 | LR: 6.93e-05 | 695.2ms/step | 28.8 samples/s +Validation at step 33500: Loss = 2.9582, Perplexity = 19.26 +Step 33510/44000 | Loss: 2.9270 | LR: 6.91e-05 | 1271.7ms/step | 15.7 samples/s +Step 33520/44000 | Loss: 2.9348 | LR: 6.90e-05 | 696.4ms/step | 28.7 samples/s +Step 33530/44000 | Loss: 2.9547 | LR: 6.89e-05 | 699.7ms/step | 28.6 samples/s +Step 33540/44000 | Loss: 2.9584 | LR: 6.88e-05 | 698.5ms/step | 28.6 samples/s +Step 33550/44000 | Loss: 2.9723 | LR: 6.86e-05 | 696.9ms/step | 28.7 samples/s +Step 33560/44000 | Loss: 2.9343 | LR: 6.85e-05 | 695.3ms/step | 28.8 samples/s +Step 33570/44000 | Loss: 2.9641 | LR: 6.84e-05 | 698.7ms/step | 28.6 samples/s +Step 33580/44000 | Loss: 2.9393 | LR: 6.83e-05 | 697.5ms/step | 28.7 samples/s +Step 33590/44000 | Loss: 2.9343 | LR: 6.81e-05 | 698.2ms/step | 28.6 samples/s +Step 33600/44000 | Loss: 2.9590 | LR: 6.80e-05 | 697.6ms/step | 28.7 samples/s +Step 33610/44000 | Loss: 2.9247 | LR: 6.79e-05 | 697.7ms/step | 28.7 samples/s +Step 33620/44000 | Loss: 2.9347 | LR: 6.78e-05 | 697.0ms/step | 28.7 samples/s +Step 33630/44000 | Loss: 2.9541 | LR: 6.76e-05 | 698.1ms/step | 28.6 samples/s +Step 33640/44000 | Loss: 2.9133 | LR: 6.75e-05 | 697.6ms/step | 28.7 samples/s +Step 33650/44000 | Loss: 2.9515 | LR: 6.74e-05 | 699.1ms/step | 28.6 samples/s +Step 33660/44000 | Loss: 2.9373 | LR: 6.73e-05 | 698.4ms/step | 28.6 samples/s +Step 33670/44000 | Loss: 2.9600 | LR: 6.71e-05 | 698.7ms/step | 28.6 samples/s +Step 33680/44000 | Loss: 2.9199 | LR: 6.70e-05 | 699.2ms/step | 28.6 samples/s +Step 33690/44000 | Loss: 2.9199 | LR: 6.69e-05 | 697.1ms/step | 28.7 samples/s +Step 33700/44000 | Loss: 2.9258 | LR: 6.68e-05 | 698.1ms/step | 28.6 samples/s +Step 33710/44000 | Loss: 2.9180 | LR: 6.66e-05 | 695.9ms/step | 28.7 samples/s +Step 33720/44000 | Loss: 2.9201 | LR: 6.65e-05 | 697.5ms/step | 28.7 samples/s +Step 33730/44000 | Loss: 2.8765 | LR: 6.64e-05 | 694.9ms/step | 28.8 samples/s +Step 33740/44000 | Loss: 2.8955 | LR: 6.63e-05 | 697.0ms/step | 28.7 samples/s +Step 33750/44000 | Loss: 2.9071 | LR: 6.61e-05 | 695.4ms/step | 28.8 samples/s +Validation at step 33750: Loss = 2.9605, Perplexity = 19.31 +Step 33760/44000 | Loss: 2.9267 | LR: 6.60e-05 | 1273.2ms/step | 15.7 samples/s +Step 33770/44000 | Loss: 2.9515 | LR: 6.59e-05 | 700.5ms/step | 28.6 samples/s +Step 33780/44000 | Loss: 2.9231 | LR: 6.58e-05 | 701.0ms/step | 28.5 samples/s +Step 33790/44000 | Loss: 2.8998 | LR: 6.57e-05 | 700.1ms/step | 28.6 samples/s +Step 33800/44000 | Loss: 2.9456 | LR: 6.55e-05 | 698.3ms/step | 28.6 samples/s +Step 33810/44000 | Loss: 2.9336 | LR: 6.54e-05 | 696.8ms/step | 28.7 samples/s +Step 33820/44000 | Loss: 2.9481 | LR: 6.53e-05 | 698.7ms/step | 28.6 samples/s +Step 33830/44000 | Loss: 2.9083 | LR: 6.52e-05 | 700.5ms/step | 28.6 samples/s +Step 33840/44000 | Loss: 2.9241 | LR: 6.50e-05 | 698.8ms/step | 28.6 samples/s +Step 33850/44000 | Loss: 2.9329 | LR: 6.49e-05 | 700.1ms/step | 28.6 samples/s +Step 33860/44000 | Loss: 2.9214 | LR: 6.48e-05 | 698.7ms/step | 28.6 samples/s +Step 33870/44000 | Loss: 2.8959 | LR: 6.47e-05 | 703.0ms/step | 28.5 samples/s +Step 33880/44000 | Loss: 2.9652 | LR: 6.46e-05 | 697.9ms/step | 28.7 samples/s +Step 33890/44000 | Loss: 2.9270 | LR: 6.44e-05 | 698.2ms/step | 28.6 samples/s +Step 33900/44000 | Loss: 2.9079 | LR: 6.43e-05 | 695.7ms/step | 28.7 samples/s +Step 33910/44000 | Loss: 2.9092 | LR: 6.42e-05 | 700.0ms/step | 28.6 samples/s +Step 33920/44000 | Loss: 2.9218 | LR: 6.41e-05 | 702.2ms/step | 28.5 samples/s +Step 33930/44000 | Loss: 2.9089 | LR: 6.40e-05 | 696.8ms/step | 28.7 samples/s +Step 33940/44000 | Loss: 2.9483 | LR: 6.38e-05 | 697.2ms/step | 28.7 samples/s +Step 33950/44000 | Loss: 2.9220 | LR: 6.37e-05 | 699.9ms/step | 28.6 samples/s +Step 33960/44000 | Loss: 2.9399 | LR: 6.36e-05 | 699.5ms/step | 28.6 samples/s +Step 33970/44000 | Loss: 2.8936 | LR: 6.35e-05 | 699.9ms/step | 28.6 samples/s +Step 33980/44000 | Loss: 2.9380 | LR: 6.33e-05 | 697.8ms/step | 28.7 samples/s +Step 33990/44000 | Loss: 2.9509 | LR: 6.32e-05 | 698.6ms/step | 28.6 samples/s +Step 34000/44000 | Loss: 2.9353 | LR: 6.31e-05 | 701.1ms/step | 28.5 samples/s +Validation at step 34000: Loss = 2.9580, Perplexity = 19.26 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-34000 +Step 34010/44000 | Loss: 2.9352 | LR: 6.30e-05 | 1584.2ms/step | 12.6 samples/s +Step 34020/44000 | Loss: 2.9231 | LR: 6.29e-05 | 698.0ms/step | 28.7 samples/s +Step 34030/44000 | Loss: 2.9564 | LR: 6.27e-05 | 695.2ms/step | 28.8 samples/s +Step 34040/44000 | Loss: 2.8971 | LR: 6.26e-05 | 694.7ms/step | 28.8 samples/s +Step 34050/44000 | Loss: 2.9282 | LR: 6.25e-05 | 697.1ms/step | 28.7 samples/s +Step 34060/44000 | Loss: 2.9314 | LR: 6.24e-05 | 699.1ms/step | 28.6 samples/s +Step 34070/44000 | Loss: 2.9239 | LR: 6.23e-05 | 697.9ms/step | 28.7 samples/s +Step 34080/44000 | Loss: 2.9269 | LR: 6.21e-05 | 696.8ms/step | 28.7 samples/s +Step 34090/44000 | Loss: 2.9091 | LR: 6.20e-05 | 697.0ms/step | 28.7 samples/s +Step 34100/44000 | Loss: 2.9196 | LR: 6.19e-05 | 698.6ms/step | 28.6 samples/s +Step 34110/44000 | Loss: 2.9262 | LR: 6.18e-05 | 696.8ms/step | 28.7 samples/s +Step 34120/44000 | Loss: 2.9503 | LR: 6.17e-05 | 699.9ms/step | 28.6 samples/s +Step 34130/44000 | Loss: 2.9919 | LR: 6.15e-05 | 698.6ms/step | 28.6 samples/s +Step 34140/44000 | Loss: 2.9241 | LR: 6.14e-05 | 697.1ms/step | 28.7 samples/s +Step 34150/44000 | Loss: 2.9483 | LR: 6.13e-05 | 700.3ms/step | 28.6 samples/s +Step 34160/44000 | Loss: 2.9340 | LR: 6.12e-05 | 694.3ms/step | 28.8 samples/s +Step 34170/44000 | Loss: 2.9465 | LR: 6.11e-05 | 696.5ms/step | 28.7 samples/s +Step 34180/44000 | Loss: 2.9515 | LR: 6.09e-05 | 694.4ms/step | 28.8 samples/s +Step 34190/44000 | Loss: 2.9381 | LR: 6.08e-05 | 697.4ms/step | 28.7 samples/s +Step 34200/44000 | Loss: 2.9232 | LR: 6.07e-05 | 697.3ms/step | 28.7 samples/s +Step 34210/44000 | Loss: 2.9141 | LR: 6.06e-05 | 697.2ms/step | 28.7 samples/s +Step 34220/44000 | Loss: 2.9141 | LR: 6.05e-05 | 695.6ms/step | 28.8 samples/s +Step 34230/44000 | Loss: 2.9656 | LR: 6.04e-05 | 697.7ms/step | 28.7 samples/s +Step 34240/44000 | Loss: 2.9625 | LR: 6.02e-05 | 698.2ms/step | 28.6 samples/s +Step 34250/44000 | Loss: 2.9273 | LR: 6.01e-05 | 701.3ms/step | 28.5 samples/s +Validation at step 34250: Loss = 2.9568, Perplexity = 19.24 +Step 34260/44000 | Loss: 2.9200 | LR: 6.00e-05 | 1267.5ms/step | 15.8 samples/s +Step 34270/44000 | Loss: 2.9387 | LR: 5.99e-05 | 700.3ms/step | 28.6 samples/s +Step 34280/44000 | Loss: 2.9505 | LR: 5.98e-05 | 699.1ms/step | 28.6 samples/s +Step 34290/44000 | Loss: 2.9892 | LR: 5.96e-05 | 699.8ms/step | 28.6 samples/s +Step 34300/44000 | Loss: 2.9617 | LR: 5.95e-05 | 698.2ms/step | 28.6 samples/s +Step 34310/44000 | Loss: 2.9745 | LR: 5.94e-05 | 696.8ms/step | 28.7 samples/s +Step 34320/44000 | Loss: 2.9409 | LR: 5.93e-05 | 696.8ms/step | 28.7 samples/s +Step 34330/44000 | Loss: 2.9457 | LR: 5.92e-05 | 695.4ms/step | 28.8 samples/s +Step 34340/44000 | Loss: 2.9459 | LR: 5.91e-05 | 695.2ms/step | 28.8 samples/s +Step 34350/44000 | Loss: 2.9581 | LR: 5.89e-05 | 694.0ms/step | 28.8 samples/s +Step 34360/44000 | Loss: 2.9258 | LR: 5.88e-05 | 695.9ms/step | 28.7 samples/s +Step 34370/44000 | Loss: 2.9234 | LR: 5.87e-05 | 696.5ms/step | 28.7 samples/s +Step 34380/44000 | Loss: 2.9430 | LR: 5.86e-05 | 698.0ms/step | 28.7 samples/s +Step 34390/44000 | Loss: 2.9465 | LR: 5.85e-05 | 697.3ms/step | 28.7 samples/s +Step 34400/44000 | Loss: 2.9041 | LR: 5.84e-05 | 697.6ms/step | 28.7 samples/s +Step 34410/44000 | Loss: 2.9278 | LR: 5.82e-05 | 694.5ms/step | 28.8 samples/s +Step 34420/44000 | Loss: 2.9469 | LR: 5.81e-05 | 698.5ms/step | 28.6 samples/s +Step 34430/44000 | Loss: 2.9109 | LR: 5.80e-05 | 698.8ms/step | 28.6 samples/s +Step 34440/44000 | Loss: 2.9642 | LR: 5.79e-05 | 697.2ms/step | 28.7 samples/s +Step 34450/44000 | Loss: 2.9054 | LR: 5.78e-05 | 693.4ms/step | 28.8 samples/s +Step 34460/44000 | Loss: 2.9423 | LR: 5.77e-05 | 694.1ms/step | 28.8 samples/s +Step 34470/44000 | Loss: 2.8541 | LR: 5.75e-05 | 699.8ms/step | 28.6 samples/s +Step 34480/44000 | Loss: 2.9512 | LR: 5.74e-05 | 699.0ms/step | 28.6 samples/s +Step 34490/44000 | Loss: 2.9630 | LR: 5.73e-05 | 699.4ms/step | 28.6 samples/s +Step 34500/44000 | Loss: 2.9254 | LR: 5.72e-05 | 697.6ms/step | 28.7 samples/s +Validation at step 34500: Loss = 2.9565, Perplexity = 19.23 +Step 34510/44000 | Loss: 2.9606 | LR: 5.71e-05 | 1268.0ms/step | 15.8 samples/s +Step 34520/44000 | Loss: 2.9223 | LR: 5.70e-05 | 694.7ms/step | 28.8 samples/s +Step 34530/44000 | Loss: 2.9005 | LR: 5.69e-05 | 697.5ms/step | 28.7 samples/s +Step 34540/44000 | Loss: 2.9458 | LR: 5.67e-05 | 696.2ms/step | 28.7 samples/s +Step 34550/44000 | Loss: 2.9368 | LR: 5.66e-05 | 694.1ms/step | 28.8 samples/s +Step 34560/44000 | Loss: 2.9736 | LR: 5.65e-05 | 694.8ms/step | 28.8 samples/s +Step 34570/44000 | Loss: 2.9408 | LR: 5.64e-05 | 698.5ms/step | 28.6 samples/s +Step 34580/44000 | Loss: 2.9441 | LR: 5.63e-05 | 693.3ms/step | 28.8 samples/s +Step 34590/44000 | Loss: 2.8822 | LR: 5.62e-05 | 694.1ms/step | 28.8 samples/s +Step 34600/44000 | Loss: 2.9831 | LR: 5.60e-05 | 699.3ms/step | 28.6 samples/s +Step 34610/44000 | Loss: 2.9006 | LR: 5.59e-05 | 698.2ms/step | 28.6 samples/s +Step 34620/44000 | Loss: 2.9541 | LR: 5.58e-05 | 693.8ms/step | 28.8 samples/s +Step 34630/44000 | Loss: 2.9157 | LR: 5.57e-05 | 694.1ms/step | 28.8 samples/s +Step 34640/44000 | Loss: 2.9094 | LR: 5.56e-05 | 696.2ms/step | 28.7 samples/s +Step 34650/44000 | Loss: 2.9546 | LR: 5.55e-05 | 697.5ms/step | 28.7 samples/s +Step 34660/44000 | Loss: 2.8732 | LR: 5.54e-05 | 695.6ms/step | 28.8 samples/s +Step 34670/44000 | Loss: 2.9242 | LR: 5.52e-05 | 698.0ms/step | 28.7 samples/s +Step 34680/44000 | Loss: 2.9243 | LR: 5.51e-05 | 693.8ms/step | 28.8 samples/s +Step 34690/44000 | Loss: 2.9562 | LR: 5.50e-05 | 700.9ms/step | 28.5 samples/s +Step 34700/44000 | Loss: 2.8834 | LR: 5.49e-05 | 697.6ms/step | 28.7 samples/s +Step 34710/44000 | Loss: 2.8848 | LR: 5.48e-05 | 697.8ms/step | 28.7 samples/s +Step 34720/44000 | Loss: 2.9342 | LR: 5.47e-05 | 691.7ms/step | 28.9 samples/s +Step 34730/44000 | Loss: 2.9851 | LR: 5.46e-05 | 697.6ms/step | 28.7 samples/s +Step 34740/44000 | Loss: 2.9653 | LR: 5.45e-05 | 697.1ms/step | 28.7 samples/s +Step 34750/44000 | Loss: 2.9337 | LR: 5.43e-05 | 702.1ms/step | 28.5 samples/s +Validation at step 34750: Loss = 2.9576, Perplexity = 19.25 +Step 34760/44000 | Loss: 2.9117 | LR: 5.42e-05 | 1270.3ms/step | 15.7 samples/s +Step 34770/44000 | Loss: 2.9528 | LR: 5.41e-05 | 694.1ms/step | 28.8 samples/s +Step 34780/44000 | Loss: 2.9028 | LR: 5.40e-05 | 697.5ms/step | 28.7 samples/s +Step 34790/44000 | Loss: 2.9138 | LR: 5.39e-05 | 696.5ms/step | 28.7 samples/s +Step 34800/44000 | Loss: 2.8894 | LR: 5.38e-05 | 695.9ms/step | 28.7 samples/s +Step 34810/44000 | Loss: 2.8957 | LR: 5.37e-05 | 699.3ms/step | 28.6 samples/s +Step 34820/44000 | Loss: 2.9407 | LR: 5.36e-05 | 699.4ms/step | 28.6 samples/s +Step 34830/44000 | Loss: 2.9328 | LR: 5.34e-05 | 712.8ms/step | 28.1 samples/s +Step 34840/44000 | Loss: 2.9485 | LR: 5.33e-05 | 706.3ms/step | 28.3 samples/s +Step 34850/44000 | Loss: 2.9067 | LR: 5.32e-05 | 693.3ms/step | 28.8 samples/s +Step 34860/44000 | Loss: 2.9541 | LR: 5.31e-05 | 695.3ms/step | 28.8 samples/s +Step 34870/44000 | Loss: 2.9198 | LR: 5.30e-05 | 696.6ms/step | 28.7 samples/s +Step 34880/44000 | Loss: 2.9904 | LR: 5.29e-05 | 699.3ms/step | 28.6 samples/s +Step 34890/44000 | Loss: 2.9482 | LR: 5.28e-05 | 698.5ms/step | 28.6 samples/s +Step 34900/44000 | Loss: 2.9101 | LR: 5.27e-05 | 701.2ms/step | 28.5 samples/s +Step 34910/44000 | Loss: 2.9213 | LR: 5.25e-05 | 697.3ms/step | 28.7 samples/s +Step 34920/44000 | Loss: 2.9469 | LR: 5.24e-05 | 698.7ms/step | 28.6 samples/s +Step 34930/44000 | Loss: 2.9372 | LR: 5.23e-05 | 697.1ms/step | 28.7 samples/s +Step 34940/44000 | Loss: 2.9127 | LR: 5.22e-05 | 697.6ms/step | 28.7 samples/s +Step 34950/44000 | Loss: 2.8927 | LR: 5.21e-05 | 695.5ms/step | 28.8 samples/s +Step 34960/44000 | Loss: 2.9425 | LR: 5.20e-05 | 697.7ms/step | 28.7 samples/s +Step 34970/44000 | Loss: 2.9476 | LR: 5.19e-05 | 700.4ms/step | 28.6 samples/s +Step 34980/44000 | Loss: 2.9168 | LR: 5.18e-05 | 698.9ms/step | 28.6 samples/s +Step 34990/44000 | Loss: 2.9548 | LR: 5.17e-05 | 696.0ms/step | 28.7 samples/s +Step 35000/44000 | Loss: 2.9188 | LR: 5.15e-05 | 696.1ms/step | 28.7 samples/s +Validation at step 35000: Loss = 2.9566, Perplexity = 19.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-35000 +Step 35010/44000 | Loss: 2.9304 | LR: 5.14e-05 | 1604.9ms/step | 12.5 samples/s +Step 35020/44000 | Loss: 2.9329 | LR: 5.13e-05 | 699.5ms/step | 28.6 samples/s +Step 35030/44000 | Loss: 2.9011 | LR: 5.12e-05 | 697.0ms/step | 28.7 samples/s +Step 35040/44000 | Loss: 2.8737 | LR: 5.11e-05 | 698.2ms/step | 28.6 samples/s +Step 35050/44000 | Loss: 2.9403 | LR: 5.10e-05 | 696.9ms/step | 28.7 samples/s +Step 35060/44000 | Loss: 2.9364 | LR: 5.09e-05 | 697.5ms/step | 28.7 samples/s +Step 35070/44000 | Loss: 2.9692 | LR: 5.08e-05 | 697.6ms/step | 28.7 samples/s +Step 35080/44000 | Loss: 2.9648 | LR: 5.07e-05 | 698.7ms/step | 28.6 samples/s +Step 35090/44000 | Loss: 2.9049 | LR: 5.06e-05 | 700.0ms/step | 28.6 samples/s +Step 35100/44000 | Loss: 2.8791 | LR: 5.04e-05 | 698.5ms/step | 28.6 samples/s +Step 35110/44000 | Loss: 2.9596 | LR: 5.03e-05 | 691.9ms/step | 28.9 samples/s +Step 35120/44000 | Loss: 2.9437 | LR: 5.02e-05 | 697.0ms/step | 28.7 samples/s +Step 35130/44000 | Loss: 2.9202 | LR: 5.01e-05 | 698.1ms/step | 28.6 samples/s +Step 35140/44000 | Loss: 2.9523 | LR: 5.00e-05 | 697.5ms/step | 28.7 samples/s +Step 35150/44000 | Loss: 2.9156 | LR: 4.99e-05 | 700.0ms/step | 28.6 samples/s +Step 35160/44000 | Loss: 2.9223 | LR: 4.98e-05 | 699.1ms/step | 28.6 samples/s +Step 35170/44000 | Loss: 2.9275 | LR: 4.97e-05 | 697.1ms/step | 28.7 samples/s +Step 35180/44000 | Loss: 2.9154 | LR: 4.96e-05 | 701.3ms/step | 28.5 samples/s +Step 35190/44000 | Loss: 2.9516 | LR: 4.95e-05 | 694.1ms/step | 28.8 samples/s +Step 35200/44000 | Loss: 2.9405 | LR: 4.94e-05 | 693.8ms/step | 28.8 samples/s +Step 35210/44000 | Loss: 2.9250 | LR: 4.93e-05 | 696.8ms/step | 28.7 samples/s +Step 35220/44000 | Loss: 2.9551 | LR: 4.91e-05 | 695.2ms/step | 28.8 samples/s +Step 35230/44000 | Loss: 2.9739 | LR: 4.90e-05 | 698.1ms/step | 28.6 samples/s +Step 35240/44000 | Loss: 2.9660 | LR: 4.89e-05 | 699.2ms/step | 28.6 samples/s +Step 35250/44000 | Loss: 2.9085 | LR: 4.88e-05 | 698.2ms/step | 28.6 samples/s +Validation at step 35250: Loss = 2.9579, Perplexity = 19.26 +Step 35260/44000 | Loss: 2.8926 | LR: 4.87e-05 | 1268.5ms/step | 15.8 samples/s +Step 35270/44000 | Loss: 2.9434 | LR: 4.86e-05 | 697.5ms/step | 28.7 samples/s +Step 35280/44000 | Loss: 2.9498 | LR: 4.85e-05 | 693.6ms/step | 28.8 samples/s +Step 35290/44000 | Loss: 2.9127 | LR: 4.84e-05 | 697.9ms/step | 28.7 samples/s +Step 35300/44000 | Loss: 2.9090 | LR: 4.83e-05 | 696.5ms/step | 28.7 samples/s +Step 35310/44000 | Loss: 2.8983 | LR: 4.82e-05 | 699.5ms/step | 28.6 samples/s +Step 35320/44000 | Loss: 2.9237 | LR: 4.81e-05 | 696.5ms/step | 28.7 samples/s +Step 35330/44000 | Loss: 2.9531 | LR: 4.80e-05 | 696.0ms/step | 28.7 samples/s +Step 35340/44000 | Loss: 2.9748 | LR: 4.79e-05 | 696.4ms/step | 28.7 samples/s +Step 35350/44000 | Loss: 2.9552 | LR: 4.77e-05 | 695.4ms/step | 28.8 samples/s +Step 35360/44000 | Loss: 2.9315 | LR: 4.76e-05 | 694.2ms/step | 28.8 samples/s +Step 35370/44000 | Loss: 2.9449 | LR: 4.75e-05 | 695.0ms/step | 28.8 samples/s +Step 35380/44000 | Loss: 2.9000 | LR: 4.74e-05 | 694.2ms/step | 28.8 samples/s +Step 35390/44000 | Loss: 2.9112 | LR: 4.73e-05 | 694.3ms/step | 28.8 samples/s +Step 35400/44000 | Loss: 2.9093 | LR: 4.72e-05 | 698.0ms/step | 28.7 samples/s +Step 35410/44000 | Loss: 2.8908 | LR: 4.71e-05 | 695.0ms/step | 28.8 samples/s +Step 35420/44000 | Loss: 2.9363 | LR: 4.70e-05 | 697.1ms/step | 28.7 samples/s +Step 35430/44000 | Loss: 2.9533 | LR: 4.69e-05 | 698.1ms/step | 28.6 samples/s +Step 35440/44000 | Loss: 2.9354 | LR: 4.68e-05 | 697.2ms/step | 28.7 samples/s +Step 35450/44000 | Loss: 2.9897 | LR: 4.67e-05 | 697.3ms/step | 28.7 samples/s +Step 35460/44000 | Loss: 2.9590 | LR: 4.66e-05 | 700.1ms/step | 28.6 samples/s +Step 35470/44000 | Loss: 2.9298 | LR: 4.65e-05 | 695.7ms/step | 28.7 samples/s +Step 35480/44000 | Loss: 2.8939 | LR: 4.64e-05 | 698.1ms/step | 28.6 samples/s +Step 35490/44000 | Loss: 2.9347 | LR: 4.63e-05 | 696.7ms/step | 28.7 samples/s +Step 35500/44000 | Loss: 2.9309 | LR: 4.62e-05 | 699.8ms/step | 28.6 samples/s +Validation at step 35500: Loss = 2.9566, Perplexity = 19.23 +Step 35510/44000 | Loss: 2.9440 | LR: 4.61e-05 | 1267.0ms/step | 15.8 samples/s +Step 35520/44000 | Loss: 2.9319 | LR: 4.59e-05 | 696.7ms/step | 28.7 samples/s +Step 35530/44000 | Loss: 2.8747 | LR: 4.58e-05 | 698.6ms/step | 28.6 samples/s +Step 35540/44000 | Loss: 2.9484 | LR: 4.57e-05 | 698.2ms/step | 28.6 samples/s +Step 35550/44000 | Loss: 2.9560 | LR: 4.56e-05 | 697.2ms/step | 28.7 samples/s +Step 35560/44000 | Loss: 2.9710 | LR: 4.55e-05 | 697.7ms/step | 28.7 samples/s +Step 35570/44000 | Loss: 2.9017 | LR: 4.54e-05 | 695.2ms/step | 28.8 samples/s +Step 35580/44000 | Loss: 2.9632 | LR: 4.53e-05 | 694.1ms/step | 28.8 samples/s +Step 35590/44000 | Loss: 2.9632 | LR: 4.52e-05 | 698.5ms/step | 28.6 samples/s +Step 35600/44000 | Loss: 2.9355 | LR: 4.51e-05 | 699.3ms/step | 28.6 samples/s +Step 35610/44000 | Loss: 2.9508 | LR: 4.50e-05 | 699.8ms/step | 28.6 samples/s +Step 35620/44000 | Loss: 2.9234 | LR: 4.49e-05 | 698.0ms/step | 28.7 samples/s +Step 35630/44000 | Loss: 2.9064 | LR: 4.48e-05 | 696.5ms/step | 28.7 samples/s +Step 35640/44000 | Loss: 2.9148 | LR: 4.47e-05 | 697.0ms/step | 28.7 samples/s +Step 35650/44000 | Loss: 2.9194 | LR: 4.46e-05 | 698.4ms/step | 28.6 samples/s +Step 35660/44000 | Loss: 2.9624 | LR: 4.45e-05 | 696.8ms/step | 28.7 samples/s +Step 35670/44000 | Loss: 2.8905 | LR: 4.44e-05 | 697.4ms/step | 28.7 samples/s +Step 35680/44000 | Loss: 2.9753 | LR: 4.43e-05 | 696.4ms/step | 28.7 samples/s +Step 35690/44000 | Loss: 2.9449 | LR: 4.42e-05 | 701.6ms/step | 28.5 samples/s +Step 35700/44000 | Loss: 2.9275 | LR: 4.41e-05 | 696.1ms/step | 28.7 samples/s +Step 35710/44000 | Loss: 2.8971 | LR: 4.40e-05 | 698.1ms/step | 28.6 samples/s +Step 35720/44000 | Loss: 2.9723 | LR: 4.39e-05 | 698.3ms/step | 28.6 samples/s +Step 35730/44000 | Loss: 2.9463 | LR: 4.38e-05 | 697.7ms/step | 28.7 samples/s +Step 35740/44000 | Loss: 2.8944 | LR: 4.37e-05 | 698.1ms/step | 28.6 samples/s +Step 35750/44000 | Loss: 2.9619 | LR: 4.36e-05 | 694.2ms/step | 28.8 samples/s +Validation at step 35750: Loss = 2.9580, Perplexity = 19.26 +Step 35760/44000 | Loss: 2.9496 | LR: 4.35e-05 | 1265.0ms/step | 15.8 samples/s +Step 35770/44000 | Loss: 2.8966 | LR: 4.34e-05 | 696.8ms/step | 28.7 samples/s +Step 35780/44000 | Loss: 2.9780 | LR: 4.33e-05 | 696.8ms/step | 28.7 samples/s +Step 35790/44000 | Loss: 2.9496 | LR: 4.32e-05 | 694.4ms/step | 28.8 samples/s +Step 35800/44000 | Loss: 2.9277 | LR: 4.31e-05 | 696.9ms/step | 28.7 samples/s +Step 35810/44000 | Loss: 2.9796 | LR: 4.29e-05 | 700.4ms/step | 28.6 samples/s +Step 35820/44000 | Loss: 2.9641 | LR: 4.28e-05 | 699.2ms/step | 28.6 samples/s +Step 35830/44000 | Loss: 2.9652 | LR: 4.27e-05 | 698.1ms/step | 28.7 samples/s +Step 35840/44000 | Loss: 2.8885 | LR: 4.26e-05 | 700.2ms/step | 28.6 samples/s +Step 35850/44000 | Loss: 2.9077 | LR: 4.25e-05 | 701.1ms/step | 28.5 samples/s +Step 35860/44000 | Loss: 2.9410 | LR: 4.24e-05 | 697.3ms/step | 28.7 samples/s +Step 35870/44000 | Loss: 2.9246 | LR: 4.23e-05 | 698.2ms/step | 28.6 samples/s +Step 35880/44000 | Loss: 2.9193 | LR: 4.22e-05 | 699.8ms/step | 28.6 samples/s +Step 35890/44000 | Loss: 2.9049 | LR: 4.21e-05 | 699.4ms/step | 28.6 samples/s +Step 35900/44000 | Loss: 2.9121 | LR: 4.20e-05 | 698.7ms/step | 28.6 samples/s +Step 35910/44000 | Loss: 2.9081 | LR: 4.19e-05 | 697.1ms/step | 28.7 samples/s +Step 35920/44000 | Loss: 2.9409 | LR: 4.18e-05 | 696.1ms/step | 28.7 samples/s +Step 35930/44000 | Loss: 2.9519 | LR: 4.17e-05 | 699.4ms/step | 28.6 samples/s +Step 35940/44000 | Loss: 2.9550 | LR: 4.16e-05 | 697.9ms/step | 28.7 samples/s +Step 35950/44000 | Loss: 2.9249 | LR: 4.15e-05 | 698.7ms/step | 28.6 samples/s +Step 35960/44000 | Loss: 2.9410 | LR: 4.14e-05 | 700.7ms/step | 28.5 samples/s +Step 35970/44000 | Loss: 2.9606 | LR: 4.13e-05 | 697.6ms/step | 28.7 samples/s +Step 35980/44000 | Loss: 2.9403 | LR: 4.12e-05 | 699.3ms/step | 28.6 samples/s +Step 35990/44000 | Loss: 2.9187 | LR: 4.11e-05 | 701.1ms/step | 28.5 samples/s +Step 36000/44000 | Loss: 2.9271 | LR: 4.10e-05 | 699.9ms/step | 28.6 samples/s +Validation at step 36000: Loss = 2.9563, Perplexity = 19.23 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-36000 +Step 36010/44000 | Loss: 2.8915 | LR: 4.09e-05 | 1622.7ms/step | 12.3 samples/s +Step 36020/44000 | Loss: 2.9434 | LR: 4.08e-05 | 707.5ms/step | 28.3 samples/s +Step 36030/44000 | Loss: 2.9383 | LR: 4.07e-05 | 707.3ms/step | 28.3 samples/s +Step 36040/44000 | Loss: 2.9495 | LR: 4.06e-05 | 703.2ms/step | 28.4 samples/s +Step 36050/44000 | Loss: 2.9317 | LR: 4.05e-05 | 699.8ms/step | 28.6 samples/s +Step 36060/44000 | Loss: 2.9329 | LR: 4.04e-05 | 701.7ms/step | 28.5 samples/s +Step 36070/44000 | Loss: 2.9325 | LR: 4.03e-05 | 701.5ms/step | 28.5 samples/s +Step 36080/44000 | Loss: 2.8921 | LR: 4.02e-05 | 700.0ms/step | 28.6 samples/s +Step 36090/44000 | Loss: 2.9399 | LR: 4.01e-05 | 698.5ms/step | 28.6 samples/s +Step 36100/44000 | Loss: 2.9417 | LR: 4.00e-05 | 699.6ms/step | 28.6 samples/s +Step 36110/44000 | Loss: 2.9361 | LR: 3.99e-05 | 702.5ms/step | 28.5 samples/s +Step 36120/44000 | Loss: 2.9426 | LR: 3.98e-05 | 699.5ms/step | 28.6 samples/s +Step 36130/44000 | Loss: 2.9519 | LR: 3.97e-05 | 699.4ms/step | 28.6 samples/s +Step 36140/44000 | Loss: 2.9255 | LR: 3.97e-05 | 700.5ms/step | 28.6 samples/s +Step 36150/44000 | Loss: 2.9365 | LR: 3.96e-05 | 698.1ms/step | 28.6 samples/s +Step 36160/44000 | Loss: 2.9409 | LR: 3.95e-05 | 698.4ms/step | 28.6 samples/s +Step 36170/44000 | Loss: 2.9437 | LR: 3.94e-05 | 701.7ms/step | 28.5 samples/s +Step 36180/44000 | Loss: 2.9512 | LR: 3.93e-05 | 698.5ms/step | 28.6 samples/s +Step 36190/44000 | Loss: 2.9849 | LR: 3.92e-05 | 698.9ms/step | 28.6 samples/s +Step 36200/44000 | Loss: 2.9178 | LR: 3.91e-05 | 701.4ms/step | 28.5 samples/s +Step 36210/44000 | Loss: 2.9260 | LR: 3.90e-05 | 699.6ms/step | 28.6 samples/s +Step 36220/44000 | Loss: 2.9379 | LR: 3.89e-05 | 699.6ms/step | 28.6 samples/s +Step 36230/44000 | Loss: 2.9126 | LR: 3.88e-05 | 702.0ms/step | 28.5 samples/s +Step 36240/44000 | Loss: 2.9306 | LR: 3.87e-05 | 700.4ms/step | 28.6 samples/s +Step 36250/44000 | Loss: 2.9166 | LR: 3.86e-05 | 697.5ms/step | 28.7 samples/s +Validation at step 36250: Loss = 2.9573, Perplexity = 19.25 +Step 36260/44000 | Loss: 2.9165 | LR: 3.85e-05 | 1269.6ms/step | 15.8 samples/s +Step 36270/44000 | Loss: 2.9084 | LR: 3.84e-05 | 700.0ms/step | 28.6 samples/s +Step 36280/44000 | Loss: 2.8905 | LR: 3.83e-05 | 696.8ms/step | 28.7 samples/s +Step 36290/44000 | Loss: 2.9501 | LR: 3.82e-05 | 697.4ms/step | 28.7 samples/s +Step 36300/44000 | Loss: 2.9482 | LR: 3.81e-05 | 700.2ms/step | 28.6 samples/s +Step 36310/44000 | Loss: 2.8971 | LR: 3.80e-05 | 696.0ms/step | 28.7 samples/s +Step 36320/44000 | Loss: 2.9022 | LR: 3.79e-05 | 695.3ms/step | 28.8 samples/s +Step 36330/44000 | Loss: 2.9229 | LR: 3.78e-05 | 696.3ms/step | 28.7 samples/s +Step 36340/44000 | Loss: 2.9205 | LR: 3.77e-05 | 697.7ms/step | 28.7 samples/s +Step 36350/44000 | Loss: 2.9058 | LR: 3.76e-05 | 700.5ms/step | 28.6 samples/s +Step 36360/44000 | Loss: 2.9511 | LR: 3.75e-05 | 692.3ms/step | 28.9 samples/s +Step 36370/44000 | Loss: 2.9397 | LR: 3.74e-05 | 697.1ms/step | 28.7 samples/s +Step 36380/44000 | Loss: 2.9126 | LR: 3.73e-05 | 698.7ms/step | 28.6 samples/s +Step 36390/44000 | Loss: 2.9122 | LR: 3.72e-05 | 696.0ms/step | 28.7 samples/s +Step 36400/44000 | Loss: 2.9499 | LR: 3.71e-05 | 695.9ms/step | 28.7 samples/s +Step 36410/44000 | Loss: 2.9182 | LR: 3.70e-05 | 695.5ms/step | 28.8 samples/s +Step 36420/44000 | Loss: 2.9638 | LR: 3.69e-05 | 696.3ms/step | 28.7 samples/s +Step 36430/44000 | Loss: 2.9194 | LR: 3.69e-05 | 697.5ms/step | 28.7 samples/s +Step 36440/44000 | Loss: 2.9187 | LR: 3.68e-05 | 701.2ms/step | 28.5 samples/s +Step 36450/44000 | Loss: 2.9249 | LR: 3.67e-05 | 700.1ms/step | 28.6 samples/s +Step 36460/44000 | Loss: 2.9451 | LR: 3.66e-05 | 699.9ms/step | 28.6 samples/s +Step 36470/44000 | Loss: 2.9101 | LR: 3.65e-05 | 697.1ms/step | 28.7 samples/s +Step 36480/44000 | Loss: 2.9762 | LR: 3.64e-05 | 699.8ms/step | 28.6 samples/s +Step 36490/44000 | Loss: 2.9824 | LR: 3.63e-05 | 697.7ms/step | 28.7 samples/s +Step 36500/44000 | Loss: 2.9592 | LR: 3.62e-05 | 700.7ms/step | 28.5 samples/s +Validation at step 36500: Loss = 2.9574, Perplexity = 19.25 +Step 36510/44000 | Loss: 2.8905 | LR: 3.61e-05 | 1277.5ms/step | 15.7 samples/s +Step 36520/44000 | Loss: 2.9068 | LR: 3.60e-05 | 699.7ms/step | 28.6 samples/s +Step 36530/44000 | Loss: 2.9185 | LR: 3.59e-05 | 703.4ms/step | 28.4 samples/s +Step 36540/44000 | Loss: 2.9092 | LR: 3.58e-05 | 700.8ms/step | 28.5 samples/s +Step 36550/44000 | Loss: 2.9519 | LR: 3.57e-05 | 701.3ms/step | 28.5 samples/s +Step 36560/44000 | Loss: 2.9542 | LR: 3.56e-05 | 697.8ms/step | 28.7 samples/s +Step 36570/44000 | Loss: 2.9271 | LR: 3.55e-05 | 699.9ms/step | 28.6 samples/s +Step 36580/44000 | Loss: 2.9076 | LR: 3.54e-05 | 698.1ms/step | 28.6 samples/s +Step 36590/44000 | Loss: 2.9775 | LR: 3.53e-05 | 695.9ms/step | 28.7 samples/s +Step 36600/44000 | Loss: 2.9400 | LR: 3.53e-05 | 699.5ms/step | 28.6 samples/s +Step 36610/44000 | Loss: 2.9247 | LR: 3.52e-05 | 701.8ms/step | 28.5 samples/s +Step 36620/44000 | Loss: 2.9590 | LR: 3.51e-05 | 698.8ms/step | 28.6 samples/s +Step 36630/44000 | Loss: 2.9364 | LR: 3.50e-05 | 700.3ms/step | 28.6 samples/s +Step 36640/44000 | Loss: 2.9089 | LR: 3.49e-05 | 700.0ms/step | 28.6 samples/s +Step 36650/44000 | Loss: 2.8927 | LR: 3.48e-05 | 698.6ms/step | 28.6 samples/s +Step 36660/44000 | Loss: 2.9503 | LR: 3.47e-05 | 700.8ms/step | 28.5 samples/s +Step 36670/44000 | Loss: 2.9389 | LR: 3.46e-05 | 700.2ms/step | 28.6 samples/s +Step 36680/44000 | Loss: 2.9038 | LR: 3.45e-05 | 700.1ms/step | 28.6 samples/s +Step 36690/44000 | Loss: 2.9434 | LR: 3.44e-05 | 699.8ms/step | 28.6 samples/s +Step 36700/44000 | Loss: 2.9390 | LR: 3.43e-05 | 699.8ms/step | 28.6 samples/s +Step 36710/44000 | Loss: 2.9095 | LR: 3.42e-05 | 700.5ms/step | 28.6 samples/s +Step 36720/44000 | Loss: 2.9067 | LR: 3.41e-05 | 699.9ms/step | 28.6 samples/s +Step 36730/44000 | Loss: 2.9599 | LR: 3.41e-05 | 701.2ms/step | 28.5 samples/s +Step 36740/44000 | Loss: 2.9192 | LR: 3.40e-05 | 700.6ms/step | 28.5 samples/s +Step 36750/44000 | Loss: 2.9759 | LR: 3.39e-05 | 699.3ms/step | 28.6 samples/s +Validation at step 36750: Loss = 2.9563, Perplexity = 19.23 +Step 36760/44000 | Loss: 2.9495 | LR: 3.38e-05 | 1271.8ms/step | 15.7 samples/s +Step 36770/44000 | Loss: 2.9494 | LR: 3.37e-05 | 698.5ms/step | 28.6 samples/s +Step 36780/44000 | Loss: 2.9292 | LR: 3.36e-05 | 700.8ms/step | 28.5 samples/s +Step 36790/44000 | Loss: 2.9678 | LR: 3.35e-05 | 697.2ms/step | 28.7 samples/s +Step 36800/44000 | Loss: 2.9274 | LR: 3.34e-05 | 700.3ms/step | 28.6 samples/s +Step 36810/44000 | Loss: 2.9646 | LR: 3.33e-05 | 703.5ms/step | 28.4 samples/s +Step 36820/44000 | Loss: 2.8934 | LR: 3.32e-05 | 700.4ms/step | 28.6 samples/s +Step 36830/44000 | Loss: 2.9658 | LR: 3.31e-05 | 699.4ms/step | 28.6 samples/s +Step 36840/44000 | Loss: 2.9264 | LR: 3.31e-05 | 698.4ms/step | 28.6 samples/s +Step 36850/44000 | Loss: 2.9162 | LR: 3.30e-05 | 697.6ms/step | 28.7 samples/s +Step 36860/44000 | Loss: 2.9344 | LR: 3.29e-05 | 696.7ms/step | 28.7 samples/s +Step 36870/44000 | Loss: 2.9535 | LR: 3.28e-05 | 695.8ms/step | 28.7 samples/s +Step 36880/44000 | Loss: 2.9285 | LR: 3.27e-05 | 695.6ms/step | 28.8 samples/s +Step 36890/44000 | Loss: 2.9668 | LR: 3.26e-05 | 698.8ms/step | 28.6 samples/s +Step 36900/44000 | Loss: 2.8995 | LR: 3.25e-05 | 696.8ms/step | 28.7 samples/s +Step 36910/44000 | Loss: 2.9106 | LR: 3.24e-05 | 698.4ms/step | 28.6 samples/s +Step 36920/44000 | Loss: 2.9493 | LR: 3.23e-05 | 696.4ms/step | 28.7 samples/s +Step 36930/44000 | Loss: 2.9363 | LR: 3.22e-05 | 696.9ms/step | 28.7 samples/s +Step 36940/44000 | Loss: 2.9381 | LR: 3.22e-05 | 697.3ms/step | 28.7 samples/s +Step 36950/44000 | Loss: 2.8970 | LR: 3.21e-05 | 699.5ms/step | 28.6 samples/s +Step 36960/44000 | Loss: 2.9183 | LR: 3.20e-05 | 697.9ms/step | 28.7 samples/s +Step 36970/44000 | Loss: 2.9282 | LR: 3.19e-05 | 695.9ms/step | 28.7 samples/s +Step 36980/44000 | Loss: 2.9599 | LR: 3.18e-05 | 698.2ms/step | 28.6 samples/s +Step 36990/44000 | Loss: 2.9330 | LR: 3.17e-05 | 695.9ms/step | 28.7 samples/s +Step 37000/44000 | Loss: 2.9273 | LR: 3.16e-05 | 696.6ms/step | 28.7 samples/s +Validation at step 37000: Loss = 2.9561, Perplexity = 19.22 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-37000 +Step 37010/44000 | Loss: 2.9106 | LR: 3.15e-05 | 1627.3ms/step | 12.3 samples/s +Step 37020/44000 | Loss: 2.9399 | LR: 3.15e-05 | 697.8ms/step | 28.7 samples/s +Step 37030/44000 | Loss: 2.9738 | LR: 3.14e-05 | 698.2ms/step | 28.6 samples/s +Step 37040/44000 | Loss: 2.9210 | LR: 3.13e-05 | 698.5ms/step | 28.6 samples/s +Step 37050/44000 | Loss: 2.9525 | LR: 3.12e-05 | 697.4ms/step | 28.7 samples/s +Step 37060/44000 | Loss: 2.9914 | LR: 3.11e-05 | 695.3ms/step | 28.8 samples/s +Step 37070/44000 | Loss: 2.9392 | LR: 3.10e-05 | 699.8ms/step | 28.6 samples/s +Step 37080/44000 | Loss: 2.9101 | LR: 3.09e-05 | 698.8ms/step | 28.6 samples/s +Step 37090/44000 | Loss: 2.9547 | LR: 3.08e-05 | 700.7ms/step | 28.5 samples/s +Step 37100/44000 | Loss: 2.9204 | LR: 3.07e-05 | 700.5ms/step | 28.6 samples/s +Step 37110/44000 | Loss: 2.9419 | LR: 3.07e-05 | 699.6ms/step | 28.6 samples/s +Step 37120/44000 | Loss: 2.9289 | LR: 3.06e-05 | 697.4ms/step | 28.7 samples/s +Step 37130/44000 | Loss: 2.9533 | LR: 3.05e-05 | 697.1ms/step | 28.7 samples/s +Step 37140/44000 | Loss: 2.8928 | LR: 3.04e-05 | 699.0ms/step | 28.6 samples/s +Step 37150/44000 | Loss: 2.8921 | LR: 3.03e-05 | 700.5ms/step | 28.6 samples/s +Step 37160/44000 | Loss: 2.9064 | LR: 3.02e-05 | 697.4ms/step | 28.7 samples/s +Step 37170/44000 | Loss: 2.8845 | LR: 3.01e-05 | 700.2ms/step | 28.6 samples/s +Step 37180/44000 | Loss: 2.8842 | LR: 3.01e-05 | 698.2ms/step | 28.6 samples/s +Step 37190/44000 | Loss: 2.9355 | LR: 3.00e-05 | 696.8ms/step | 28.7 samples/s +Step 37200/44000 | Loss: 2.9489 | LR: 2.99e-05 | 697.4ms/step | 28.7 samples/s +Step 37210/44000 | Loss: 2.9408 | LR: 2.98e-05 | 701.8ms/step | 28.5 samples/s +Step 37220/44000 | Loss: 2.9100 | LR: 2.97e-05 | 699.7ms/step | 28.6 samples/s +Step 37230/44000 | Loss: 2.9542 | LR: 2.96e-05 | 700.1ms/step | 28.6 samples/s +Step 37240/44000 | Loss: 2.9036 | LR: 2.95e-05 | 698.6ms/step | 28.6 samples/s +Step 37250/44000 | Loss: 2.8949 | LR: 2.95e-05 | 697.7ms/step | 28.7 samples/s +Validation at step 37250: Loss = 2.9551, Perplexity = 19.20 +Step 37260/44000 | Loss: 2.9139 | LR: 2.94e-05 | 1267.8ms/step | 15.8 samples/s +Step 37270/44000 | Loss: 2.9045 | LR: 2.93e-05 | 698.3ms/step | 28.6 samples/s +Step 37280/44000 | Loss: 2.9108 | LR: 2.92e-05 | 697.5ms/step | 28.7 samples/s +Step 37290/44000 | Loss: 2.9288 | LR: 2.91e-05 | 698.6ms/step | 28.6 samples/s +Step 37300/44000 | Loss: 2.9487 | LR: 2.90e-05 | 696.9ms/step | 28.7 samples/s +Step 37310/44000 | Loss: 2.9328 | LR: 2.89e-05 | 699.6ms/step | 28.6 samples/s +Step 37320/44000 | Loss: 2.9131 | LR: 2.89e-05 | 699.5ms/step | 28.6 samples/s +Step 37330/44000 | Loss: 2.9496 | LR: 2.88e-05 | 699.7ms/step | 28.6 samples/s +Step 37340/44000 | Loss: 2.8952 | LR: 2.87e-05 | 698.3ms/step | 28.6 samples/s +Step 37350/44000 | Loss: 2.9308 | LR: 2.86e-05 | 700.0ms/step | 28.6 samples/s +Step 37360/44000 | Loss: 2.9159 | LR: 2.85e-05 | 699.3ms/step | 28.6 samples/s +Step 37370/44000 | Loss: 2.9332 | LR: 2.84e-05 | 699.5ms/step | 28.6 samples/s +Step 37380/44000 | Loss: 2.9103 | LR: 2.84e-05 | 697.7ms/step | 28.7 samples/s +Step 37390/44000 | Loss: 2.9349 | LR: 2.83e-05 | 696.1ms/step | 28.7 samples/s +Step 37400/44000 | Loss: 2.8664 | LR: 2.82e-05 | 697.1ms/step | 28.7 samples/s +Step 37410/44000 | Loss: 2.9385 | LR: 2.81e-05 | 698.9ms/step | 28.6 samples/s +Step 37420/44000 | Loss: 2.9596 | LR: 2.80e-05 | 696.6ms/step | 28.7 samples/s +Step 37430/44000 | Loss: 2.9780 | LR: 2.79e-05 | 698.4ms/step | 28.6 samples/s +Step 37440/44000 | Loss: 2.9072 | LR: 2.78e-05 | 699.1ms/step | 28.6 samples/s +Step 37450/44000 | Loss: 2.9066 | LR: 2.78e-05 | 696.7ms/step | 28.7 samples/s +Step 37460/44000 | Loss: 2.9015 | LR: 2.77e-05 | 699.9ms/step | 28.6 samples/s +Step 37470/44000 | Loss: 2.9595 | LR: 2.76e-05 | 699.1ms/step | 28.6 samples/s +Step 37480/44000 | Loss: 2.9238 | LR: 2.75e-05 | 700.7ms/step | 28.5 samples/s +Step 37490/44000 | Loss: 2.9306 | LR: 2.74e-05 | 702.9ms/step | 28.5 samples/s +Step 37500/44000 | Loss: 2.9471 | LR: 2.74e-05 | 699.3ms/step | 28.6 samples/s +Validation at step 37500: Loss = 2.9555, Perplexity = 19.21 +Step 37510/44000 | Loss: 2.9046 | LR: 2.73e-05 | 1267.9ms/step | 15.8 samples/s +Step 37520/44000 | Loss: 2.9477 | LR: 2.72e-05 | 700.2ms/step | 28.6 samples/s +Step 37530/44000 | Loss: 2.9389 | LR: 2.71e-05 | 701.5ms/step | 28.5 samples/s +Step 37540/44000 | Loss: 2.9378 | LR: 2.70e-05 | 697.6ms/step | 28.7 samples/s +Step 37550/44000 | Loss: 2.9282 | LR: 2.69e-05 | 700.3ms/step | 28.6 samples/s +Step 37560/44000 | Loss: 2.9428 | LR: 2.69e-05 | 699.1ms/step | 28.6 samples/s +Step 37570/44000 | Loss: 2.9131 | LR: 2.68e-05 | 698.3ms/step | 28.6 samples/s +Step 37580/44000 | Loss: 2.9605 | LR: 2.67e-05 | 698.9ms/step | 28.6 samples/s +Step 37590/44000 | Loss: 2.9215 | LR: 2.66e-05 | 698.2ms/step | 28.6 samples/s +Step 37600/44000 | Loss: 2.9299 | LR: 2.65e-05 | 701.7ms/step | 28.5 samples/s +Step 37610/44000 | Loss: 2.9029 | LR: 2.65e-05 | 697.8ms/step | 28.7 samples/s +Step 37620/44000 | Loss: 2.9195 | LR: 2.64e-05 | 697.5ms/step | 28.7 samples/s +Step 37630/44000 | Loss: 2.9483 | LR: 2.63e-05 | 699.3ms/step | 28.6 samples/s +Step 37640/44000 | Loss: 2.9683 | LR: 2.62e-05 | 698.7ms/step | 28.6 samples/s +Step 37650/44000 | Loss: 2.9297 | LR: 2.61e-05 | 696.3ms/step | 28.7 samples/s +Step 37660/44000 | Loss: 2.9217 | LR: 2.60e-05 | 696.4ms/step | 28.7 samples/s +Step 37670/44000 | Loss: 2.9695 | LR: 2.60e-05 | 698.3ms/step | 28.6 samples/s +Step 37680/44000 | Loss: 2.9459 | LR: 2.59e-05 | 698.6ms/step | 28.6 samples/s +Step 37690/44000 | Loss: 2.9216 | LR: 2.58e-05 | 699.2ms/step | 28.6 samples/s +Step 37700/44000 | Loss: 2.9706 | LR: 2.57e-05 | 698.7ms/step | 28.6 samples/s +Step 37710/44000 | Loss: 2.9168 | LR: 2.56e-05 | 698.7ms/step | 28.6 samples/s +Step 37720/44000 | Loss: 2.9319 | LR: 2.56e-05 | 698.5ms/step | 28.6 samples/s +Step 37730/44000 | Loss: 2.9584 | LR: 2.55e-05 | 695.7ms/step | 28.7 samples/s +Step 37740/44000 | Loss: 2.9045 | LR: 2.54e-05 | 697.0ms/step | 28.7 samples/s +Step 37750/44000 | Loss: 2.9191 | LR: 2.53e-05 | 696.9ms/step | 28.7 samples/s +Validation at step 37750: Loss = 2.9535, Perplexity = 19.17 +Step 37760/44000 | Loss: 2.9123 | LR: 2.52e-05 | 1268.6ms/step | 15.8 samples/s +Step 37770/44000 | Loss: 2.9106 | LR: 2.52e-05 | 695.5ms/step | 28.8 samples/s +Step 37780/44000 | Loss: 2.9420 | LR: 2.51e-05 | 696.6ms/step | 28.7 samples/s +Step 37790/44000 | Loss: 2.9378 | LR: 2.50e-05 | 700.1ms/step | 28.6 samples/s +Step 37800/44000 | Loss: 2.9332 | LR: 2.49e-05 | 699.3ms/step | 28.6 samples/s +Step 37810/44000 | Loss: 2.9243 | LR: 2.48e-05 | 695.7ms/step | 28.7 samples/s +Step 37820/44000 | Loss: 2.9636 | LR: 2.48e-05 | 696.8ms/step | 28.7 samples/s +Step 37830/44000 | Loss: 2.9647 | LR: 2.47e-05 | 698.2ms/step | 28.6 samples/s +Step 37840/44000 | Loss: 2.9362 | LR: 2.46e-05 | 700.3ms/step | 28.6 samples/s +Step 37850/44000 | Loss: 2.9450 | LR: 2.45e-05 | 696.2ms/step | 28.7 samples/s +Step 37860/44000 | Loss: 2.9539 | LR: 2.45e-05 | 697.3ms/step | 28.7 samples/s +Step 37870/44000 | Loss: 2.9568 | LR: 2.44e-05 | 700.2ms/step | 28.6 samples/s +Step 37880/44000 | Loss: 2.9665 | LR: 2.43e-05 | 698.2ms/step | 28.6 samples/s +Step 37890/44000 | Loss: 2.9379 | LR: 2.42e-05 | 698.5ms/step | 28.6 samples/s +Step 37900/44000 | Loss: 2.9444 | LR: 2.41e-05 | 698.4ms/step | 28.6 samples/s +Step 37910/44000 | Loss: 2.9408 | LR: 2.41e-05 | 698.7ms/step | 28.6 samples/s +Step 37920/44000 | Loss: 2.9040 | LR: 2.40e-05 | 697.5ms/step | 28.7 samples/s +Step 37930/44000 | Loss: 2.9655 | LR: 2.39e-05 | 699.5ms/step | 28.6 samples/s +Step 37940/44000 | Loss: 2.9501 | LR: 2.38e-05 | 697.5ms/step | 28.7 samples/s +Step 37950/44000 | Loss: 2.9789 | LR: 2.38e-05 | 697.9ms/step | 28.7 samples/s +Step 37960/44000 | Loss: 2.9676 | LR: 2.37e-05 | 695.3ms/step | 28.8 samples/s +Step 37970/44000 | Loss: 2.9506 | LR: 2.36e-05 | 695.9ms/step | 28.7 samples/s +Step 37980/44000 | Loss: 2.9252 | LR: 2.35e-05 | 698.2ms/step | 28.6 samples/s +Step 37990/44000 | Loss: 2.9204 | LR: 2.34e-05 | 700.0ms/step | 28.6 samples/s +Step 38000/44000 | Loss: 2.9325 | LR: 2.34e-05 | 695.9ms/step | 28.7 samples/s +Validation at step 38000: Loss = 2.9548, Perplexity = 19.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-38000 +Step 38010/44000 | Loss: 2.9048 | LR: 2.33e-05 | 1621.2ms/step | 12.3 samples/s +Step 38020/44000 | Loss: 2.9195 | LR: 2.32e-05 | 699.4ms/step | 28.6 samples/s +Step 38030/44000 | Loss: 2.9294 | LR: 2.31e-05 | 697.1ms/step | 28.7 samples/s +Step 38040/44000 | Loss: 2.9293 | LR: 2.31e-05 | 698.8ms/step | 28.6 samples/s +Step 38050/44000 | Loss: 2.9234 | LR: 2.30e-05 | 697.6ms/step | 28.7 samples/s +Step 38060/44000 | Loss: 2.9188 | LR: 2.29e-05 | 698.1ms/step | 28.6 samples/s +Step 38070/44000 | Loss: 2.9599 | LR: 2.28e-05 | 699.9ms/step | 28.6 samples/s +Step 38080/44000 | Loss: 2.8840 | LR: 2.28e-05 | 698.2ms/step | 28.6 samples/s +Step 38090/44000 | Loss: 2.9426 | LR: 2.27e-05 | 695.4ms/step | 28.8 samples/s +Step 38100/44000 | Loss: 2.8621 | LR: 2.26e-05 | 696.8ms/step | 28.7 samples/s +Step 38110/44000 | Loss: 2.9282 | LR: 2.25e-05 | 697.3ms/step | 28.7 samples/s +Step 38120/44000 | Loss: 2.9281 | LR: 2.25e-05 | 699.1ms/step | 28.6 samples/s +Step 38130/44000 | Loss: 2.9423 | LR: 2.24e-05 | 697.2ms/step | 28.7 samples/s +Step 38140/44000 | Loss: 2.9022 | LR: 2.23e-05 | 697.2ms/step | 28.7 samples/s +Step 38150/44000 | Loss: 2.9078 | LR: 2.22e-05 | 697.0ms/step | 28.7 samples/s +Step 38160/44000 | Loss: 2.9261 | LR: 2.22e-05 | 698.6ms/step | 28.6 samples/s +Step 38170/44000 | Loss: 2.9485 | LR: 2.21e-05 | 700.2ms/step | 28.6 samples/s +Step 38180/44000 | Loss: 2.9340 | LR: 2.20e-05 | 696.0ms/step | 28.7 samples/s +Step 38190/44000 | Loss: 2.9359 | LR: 2.19e-05 | 697.6ms/step | 28.7 samples/s +Step 38200/44000 | Loss: 2.9179 | LR: 2.19e-05 | 697.2ms/step | 28.7 samples/s +Step 38210/44000 | Loss: 2.9242 | LR: 2.18e-05 | 700.2ms/step | 28.6 samples/s +Step 38220/44000 | Loss: 2.9810 | LR: 2.17e-05 | 698.8ms/step | 28.6 samples/s +Step 38230/44000 | Loss: 2.9571 | LR: 2.16e-05 | 699.2ms/step | 28.6 samples/s +Step 38240/44000 | Loss: 2.9778 | LR: 2.16e-05 | 700.5ms/step | 28.6 samples/s +Step 38250/44000 | Loss: 2.9610 | LR: 2.15e-05 | 698.5ms/step | 28.6 samples/s +Validation at step 38250: Loss = 2.9553, Perplexity = 19.21 +Step 38260/44000 | Loss: 2.9508 | LR: 2.14e-05 | 1264.0ms/step | 15.8 samples/s +Step 38270/44000 | Loss: 2.9836 | LR: 2.13e-05 | 697.4ms/step | 28.7 samples/s +Step 38280/44000 | Loss: 2.8727 | LR: 2.13e-05 | 695.9ms/step | 28.7 samples/s +Step 38290/44000 | Loss: 2.9435 | LR: 2.12e-05 | 699.7ms/step | 28.6 samples/s +Step 38300/44000 | Loss: 2.9595 | LR: 2.11e-05 | 700.0ms/step | 28.6 samples/s +Step 38310/44000 | Loss: 2.9251 | LR: 2.11e-05 | 696.7ms/step | 28.7 samples/s +Step 38320/44000 | Loss: 2.9084 | LR: 2.10e-05 | 699.4ms/step | 28.6 samples/s +Step 38330/44000 | Loss: 2.9361 | LR: 2.09e-05 | 698.6ms/step | 28.6 samples/s +Step 38340/44000 | Loss: 2.9339 | LR: 2.08e-05 | 698.4ms/step | 28.6 samples/s +Step 38350/44000 | Loss: 2.9450 | LR: 2.08e-05 | 699.0ms/step | 28.6 samples/s +Step 38360/44000 | Loss: 2.9200 | LR: 2.07e-05 | 696.3ms/step | 28.7 samples/s +Step 38370/44000 | Loss: 2.9396 | LR: 2.06e-05 | 698.0ms/step | 28.7 samples/s +Step 38380/44000 | Loss: 2.9705 | LR: 2.05e-05 | 698.0ms/step | 28.7 samples/s +Step 38390/44000 | Loss: 2.9892 | LR: 2.05e-05 | 698.6ms/step | 28.6 samples/s +Step 38400/44000 | Loss: 2.8963 | LR: 2.04e-05 | 697.8ms/step | 28.7 samples/s +Step 38410/44000 | Loss: 2.9521 | LR: 2.03e-05 | 699.8ms/step | 28.6 samples/s +Step 38420/44000 | Loss: 2.9636 | LR: 2.03e-05 | 699.3ms/step | 28.6 samples/s +Step 38430/44000 | Loss: 2.9547 | LR: 2.02e-05 | 699.9ms/step | 28.6 samples/s +Step 38440/44000 | Loss: 2.9144 | LR: 2.01e-05 | 700.2ms/step | 28.6 samples/s +Step 38450/44000 | Loss: 2.9583 | LR: 2.00e-05 | 698.3ms/step | 28.6 samples/s +Step 38460/44000 | Loss: 2.9455 | LR: 2.00e-05 | 695.6ms/step | 28.8 samples/s +Step 38470/44000 | Loss: 2.8875 | LR: 1.99e-05 | 698.6ms/step | 28.6 samples/s +Step 38480/44000 | Loss: 2.9581 | LR: 1.98e-05 | 695.9ms/step | 28.7 samples/s +Step 38490/44000 | Loss: 2.9507 | LR: 1.98e-05 | 700.0ms/step | 28.6 samples/s +Step 38500/44000 | Loss: 2.9243 | LR: 1.97e-05 | 700.1ms/step | 28.6 samples/s +Validation at step 38500: Loss = 2.9543, Perplexity = 19.19 +Step 38510/44000 | Loss: 2.9301 | LR: 1.96e-05 | 1266.3ms/step | 15.8 samples/s +Step 38520/44000 | Loss: 2.9602 | LR: 1.95e-05 | 697.7ms/step | 28.7 samples/s +Step 38530/44000 | Loss: 2.9045 | LR: 1.95e-05 | 697.4ms/step | 28.7 samples/s +Step 38540/44000 | Loss: 2.9507 | LR: 1.94e-05 | 696.0ms/step | 28.7 samples/s +Step 38550/44000 | Loss: 2.9408 | LR: 1.93e-05 | 696.1ms/step | 28.7 samples/s +Step 38560/44000 | Loss: 2.8953 | LR: 1.93e-05 | 698.2ms/step | 28.6 samples/s +Step 38570/44000 | Loss: 2.9144 | LR: 1.92e-05 | 696.2ms/step | 28.7 samples/s +Step 38580/44000 | Loss: 2.9143 | LR: 1.91e-05 | 698.0ms/step | 28.7 samples/s +Step 38590/44000 | Loss: 2.9653 | LR: 1.91e-05 | 696.0ms/step | 28.7 samples/s +Step 38600/44000 | Loss: 2.9547 | LR: 1.90e-05 | 695.6ms/step | 28.8 samples/s +Step 38610/44000 | Loss: 2.9548 | LR: 1.89e-05 | 697.4ms/step | 28.7 samples/s +Step 38620/44000 | Loss: 2.9147 | LR: 1.88e-05 | 699.3ms/step | 28.6 samples/s +Step 38630/44000 | Loss: 2.9289 | LR: 1.88e-05 | 693.3ms/step | 28.8 samples/s +Step 38640/44000 | Loss: 2.8981 | LR: 1.87e-05 | 700.0ms/step | 28.6 samples/s +Step 38650/44000 | Loss: 2.9739 | LR: 1.86e-05 | 698.1ms/step | 28.7 samples/s +Step 38660/44000 | Loss: 2.9175 | LR: 1.86e-05 | 696.0ms/step | 28.7 samples/s +Step 38670/44000 | Loss: 2.9149 | LR: 1.85e-05 | 695.6ms/step | 28.8 samples/s +Step 38680/44000 | Loss: 2.9368 | LR: 1.84e-05 | 698.6ms/step | 28.6 samples/s +Step 38690/44000 | Loss: 2.9257 | LR: 1.84e-05 | 699.9ms/step | 28.6 samples/s +Step 38700/44000 | Loss: 2.9346 | LR: 1.83e-05 | 694.8ms/step | 28.8 samples/s +Step 38710/44000 | Loss: 2.9144 | LR: 1.82e-05 | 699.1ms/step | 28.6 samples/s +Step 38720/44000 | Loss: 2.8786 | LR: 1.82e-05 | 700.6ms/step | 28.5 samples/s +Step 38730/44000 | Loss: 2.9502 | LR: 1.81e-05 | 697.3ms/step | 28.7 samples/s +Step 38740/44000 | Loss: 2.9651 | LR: 1.80e-05 | 699.9ms/step | 28.6 samples/s +Step 38750/44000 | Loss: 2.9248 | LR: 1.80e-05 | 696.6ms/step | 28.7 samples/s +Validation at step 38750: Loss = 2.9565, Perplexity = 19.23 +Step 38760/44000 | Loss: 2.9145 | LR: 1.79e-05 | 1267.2ms/step | 15.8 samples/s +Step 38770/44000 | Loss: 2.9140 | LR: 1.78e-05 | 701.3ms/step | 28.5 samples/s +Step 38780/44000 | Loss: 2.9401 | LR: 1.78e-05 | 701.3ms/step | 28.5 samples/s +Step 38790/44000 | Loss: 2.8726 | LR: 1.77e-05 | 697.8ms/step | 28.7 samples/s +Step 38800/44000 | Loss: 2.9333 | LR: 1.76e-05 | 696.7ms/step | 28.7 samples/s +Step 38810/44000 | Loss: 2.9365 | LR: 1.76e-05 | 697.6ms/step | 28.7 samples/s +Step 38820/44000 | Loss: 2.9214 | LR: 1.75e-05 | 699.6ms/step | 28.6 samples/s +Step 38830/44000 | Loss: 2.9577 | LR: 1.74e-05 | 699.2ms/step | 28.6 samples/s +Step 38840/44000 | Loss: 2.9107 | LR: 1.74e-05 | 699.2ms/step | 28.6 samples/s +Step 38850/44000 | Loss: 2.9672 | LR: 1.73e-05 | 695.9ms/step | 28.7 samples/s +Step 38860/44000 | Loss: 2.9841 | LR: 1.72e-05 | 699.7ms/step | 28.6 samples/s +Step 38870/44000 | Loss: 2.9222 | LR: 1.72e-05 | 698.7ms/step | 28.6 samples/s +Step 38880/44000 | Loss: 2.9000 | LR: 1.71e-05 | 697.4ms/step | 28.7 samples/s +Step 38890/44000 | Loss: 2.9416 | LR: 1.70e-05 | 696.8ms/step | 28.7 samples/s +Step 38900/44000 | Loss: 2.9280 | LR: 1.70e-05 | 696.7ms/step | 28.7 samples/s +Step 38910/44000 | Loss: 2.9181 | LR: 1.69e-05 | 702.8ms/step | 28.5 samples/s +Step 38920/44000 | Loss: 2.9454 | LR: 1.68e-05 | 694.7ms/step | 28.8 samples/s +Step 38930/44000 | Loss: 2.9245 | LR: 1.68e-05 | 699.0ms/step | 28.6 samples/s +Step 38940/44000 | Loss: 2.9185 | LR: 1.67e-05 | 699.6ms/step | 28.6 samples/s +Step 38950/44000 | Loss: 2.9423 | LR: 1.66e-05 | 695.9ms/step | 28.7 samples/s +Step 38960/44000 | Loss: 2.9477 | LR: 1.66e-05 | 700.4ms/step | 28.6 samples/s +Step 38970/44000 | Loss: 2.9209 | LR: 1.65e-05 | 700.2ms/step | 28.6 samples/s +Step 38980/44000 | Loss: 2.9279 | LR: 1.64e-05 | 696.6ms/step | 28.7 samples/s +Step 38990/44000 | Loss: 2.9564 | LR: 1.64e-05 | 700.4ms/step | 28.6 samples/s +Step 39000/44000 | Loss: 2.9499 | LR: 1.63e-05 | 700.7ms/step | 28.5 samples/s +Validation at step 39000: Loss = 2.9546, Perplexity = 19.19 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-39000 +Step 39010/44000 | Loss: 2.9323 | LR: 1.62e-05 | 1608.1ms/step | 12.4 samples/s +Step 39020/44000 | Loss: 2.9537 | LR: 1.62e-05 | 698.1ms/step | 28.6 samples/s +Step 39030/44000 | Loss: 2.9556 | LR: 1.61e-05 | 699.6ms/step | 28.6 samples/s +Step 39040/44000 | Loss: 2.9372 | LR: 1.61e-05 | 698.4ms/step | 28.6 samples/s +Step 39050/44000 | Loss: 2.9150 | LR: 1.60e-05 | 698.1ms/step | 28.7 samples/s +Step 39060/44000 | Loss: 2.9803 | LR: 1.59e-05 | 699.6ms/step | 28.6 samples/s +Step 39070/44000 | Loss: 2.9296 | LR: 1.59e-05 | 701.3ms/step | 28.5 samples/s +Step 39080/44000 | Loss: 2.9061 | LR: 1.58e-05 | 698.1ms/step | 28.6 samples/s +Step 39090/44000 | Loss: 2.9179 | LR: 1.57e-05 | 696.9ms/step | 28.7 samples/s +Step 39100/44000 | Loss: 2.9684 | LR: 1.57e-05 | 698.6ms/step | 28.6 samples/s +Step 39110/44000 | Loss: 2.9248 | LR: 1.56e-05 | 699.8ms/step | 28.6 samples/s +Step 39120/44000 | Loss: 2.9197 | LR: 1.55e-05 | 700.8ms/step | 28.5 samples/s +Step 39130/44000 | Loss: 2.9347 | LR: 1.55e-05 | 700.4ms/step | 28.6 samples/s +Step 39140/44000 | Loss: 2.8924 | LR: 1.54e-05 | 696.5ms/step | 28.7 samples/s +Step 39150/44000 | Loss: 2.8735 | LR: 1.54e-05 | 697.8ms/step | 28.7 samples/s +Step 39160/44000 | Loss: 2.9412 | LR: 1.53e-05 | 698.3ms/step | 28.6 samples/s +Step 39170/44000 | Loss: 2.9026 | LR: 1.52e-05 | 700.0ms/step | 28.6 samples/s +Step 39180/44000 | Loss: 2.9206 | LR: 1.52e-05 | 697.5ms/step | 28.7 samples/s +Step 39190/44000 | Loss: 2.9305 | LR: 1.51e-05 | 699.9ms/step | 28.6 samples/s +Step 39200/44000 | Loss: 2.9603 | LR: 1.50e-05 | 700.2ms/step | 28.6 samples/s +Step 39210/44000 | Loss: 2.9115 | LR: 1.50e-05 | 697.7ms/step | 28.7 samples/s +Step 39220/44000 | Loss: 2.9801 | LR: 1.49e-05 | 702.4ms/step | 28.5 samples/s +Step 39230/44000 | Loss: 2.9721 | LR: 1.49e-05 | 702.0ms/step | 28.5 samples/s +Step 39240/44000 | Loss: 2.9825 | LR: 1.48e-05 | 698.7ms/step | 28.6 samples/s +Step 39250/44000 | Loss: 2.9099 | LR: 1.47e-05 | 696.7ms/step | 28.7 samples/s +Validation at step 39250: Loss = 2.9557, Perplexity = 19.21 +Step 39260/44000 | Loss: 2.9160 | LR: 1.47e-05 | 1264.1ms/step | 15.8 samples/s +Step 39270/44000 | Loss: 2.9112 | LR: 1.46e-05 | 700.0ms/step | 28.6 samples/s +Step 39280/44000 | Loss: 2.9207 | LR: 1.46e-05 | 699.5ms/step | 28.6 samples/s +Step 39290/44000 | Loss: 2.9446 | LR: 1.45e-05 | 696.4ms/step | 28.7 samples/s +Step 39300/44000 | Loss: 2.8957 | LR: 1.44e-05 | 699.0ms/step | 28.6 samples/s +Step 39310/44000 | Loss: 2.8950 | LR: 1.44e-05 | 699.7ms/step | 28.6 samples/s +Step 39320/44000 | Loss: 2.9845 | LR: 1.43e-05 | 697.3ms/step | 28.7 samples/s +Step 39330/44000 | Loss: 2.9071 | LR: 1.42e-05 | 696.6ms/step | 28.7 samples/s +Step 39340/44000 | Loss: 2.8792 | LR: 1.42e-05 | 701.8ms/step | 28.5 samples/s +Step 39350/44000 | Loss: 2.9308 | LR: 1.41e-05 | 699.7ms/step | 28.6 samples/s +Step 39360/44000 | Loss: 2.9210 | LR: 1.41e-05 | 697.1ms/step | 28.7 samples/s +Step 39370/44000 | Loss: 2.9360 | LR: 1.40e-05 | 696.5ms/step | 28.7 samples/s +Step 39380/44000 | Loss: 2.9260 | LR: 1.39e-05 | 700.4ms/step | 28.6 samples/s +Step 39390/44000 | Loss: 2.9221 | LR: 1.39e-05 | 698.5ms/step | 28.6 samples/s +Step 39400/44000 | Loss: 2.9392 | LR: 1.38e-05 | 697.9ms/step | 28.7 samples/s +Step 39410/44000 | Loss: 2.9174 | LR: 1.38e-05 | 700.1ms/step | 28.6 samples/s +Step 39420/44000 | Loss: 2.9389 | LR: 1.37e-05 | 697.0ms/step | 28.7 samples/s +Step 39430/44000 | Loss: 2.9137 | LR: 1.36e-05 | 699.3ms/step | 28.6 samples/s +Step 39440/44000 | Loss: 2.9714 | LR: 1.36e-05 | 697.5ms/step | 28.7 samples/s +Step 39450/44000 | Loss: 2.9442 | LR: 1.35e-05 | 694.6ms/step | 28.8 samples/s +Step 39460/44000 | Loss: 2.9361 | LR: 1.35e-05 | 696.3ms/step | 28.7 samples/s +Step 39470/44000 | Loss: 2.9020 | LR: 1.34e-05 | 697.0ms/step | 28.7 samples/s +Step 39480/44000 | Loss: 2.9350 | LR: 1.34e-05 | 699.0ms/step | 28.6 samples/s +Step 39490/44000 | Loss: 2.9829 | LR: 1.33e-05 | 699.4ms/step | 28.6 samples/s +Step 39500/44000 | Loss: 2.9484 | LR: 1.32e-05 | 697.4ms/step | 28.7 samples/s +Validation at step 39500: Loss = 2.9562, Perplexity = 19.22 +Step 39510/44000 | Loss: 2.9370 | LR: 1.32e-05 | 1268.1ms/step | 15.8 samples/s +Step 39520/44000 | Loss: 2.9438 | LR: 1.31e-05 | 699.5ms/step | 28.6 samples/s +Step 39530/44000 | Loss: 2.9416 | LR: 1.31e-05 | 700.6ms/step | 28.5 samples/s +Step 39540/44000 | Loss: 2.9305 | LR: 1.30e-05 | 700.3ms/step | 28.6 samples/s +Step 39550/44000 | Loss: 2.9206 | LR: 1.29e-05 | 699.6ms/step | 28.6 samples/s +Step 39560/44000 | Loss: 2.8749 | LR: 1.29e-05 | 698.9ms/step | 28.6 samples/s +Step 39570/44000 | Loss: 2.9579 | LR: 1.28e-05 | 699.1ms/step | 28.6 samples/s +Step 39580/44000 | Loss: 2.9209 | LR: 1.28e-05 | 697.6ms/step | 28.7 samples/s +Step 39590/44000 | Loss: 2.9759 | LR: 1.27e-05 | 698.4ms/step | 28.6 samples/s +Step 39600/44000 | Loss: 2.9499 | LR: 1.27e-05 | 701.7ms/step | 28.5 samples/s +Step 39610/44000 | Loss: 2.9223 | LR: 1.26e-05 | 698.7ms/step | 28.6 samples/s +Step 39620/44000 | Loss: 2.9095 | LR: 1.25e-05 | 700.1ms/step | 28.6 samples/s +Step 39630/44000 | Loss: 2.9136 | LR: 1.25e-05 | 699.4ms/step | 28.6 samples/s +Step 39640/44000 | Loss: 2.9745 | LR: 1.24e-05 | 700.5ms/step | 28.5 samples/s +Step 39650/44000 | Loss: 2.9447 | LR: 1.24e-05 | 697.0ms/step | 28.7 samples/s +Step 39660/44000 | Loss: 2.9368 | LR: 1.23e-05 | 698.6ms/step | 28.6 samples/s +Step 39670/44000 | Loss: 2.9175 | LR: 1.23e-05 | 696.1ms/step | 28.7 samples/s +Step 39680/44000 | Loss: 2.9310 | LR: 1.22e-05 | 700.1ms/step | 28.6 samples/s +Step 39690/44000 | Loss: 2.9465 | LR: 1.22e-05 | 697.7ms/step | 28.7 samples/s +Step 39700/44000 | Loss: 2.9137 | LR: 1.21e-05 | 699.8ms/step | 28.6 samples/s +Step 39710/44000 | Loss: 2.9474 | LR: 1.20e-05 | 695.8ms/step | 28.7 samples/s +Step 39720/44000 | Loss: 2.9580 | LR: 1.20e-05 | 701.1ms/step | 28.5 samples/s +Step 39730/44000 | Loss: 2.9277 | LR: 1.19e-05 | 699.5ms/step | 28.6 samples/s +Step 39740/44000 | Loss: 2.9115 | LR: 1.19e-05 | 700.0ms/step | 28.6 samples/s +Step 39750/44000 | Loss: 2.9349 | LR: 1.18e-05 | 702.2ms/step | 28.5 samples/s +Validation at step 39750: Loss = 2.9546, Perplexity = 19.19 +Step 39760/44000 | Loss: 2.9476 | LR: 1.18e-05 | 1263.1ms/step | 15.8 samples/s +Step 39770/44000 | Loss: 2.9613 | LR: 1.17e-05 | 699.2ms/step | 28.6 samples/s +Step 39780/44000 | Loss: 2.9303 | LR: 1.17e-05 | 701.6ms/step | 28.5 samples/s +Step 39790/44000 | Loss: 2.9435 | LR: 1.16e-05 | 697.6ms/step | 28.7 samples/s +Step 39800/44000 | Loss: 2.9629 | LR: 1.15e-05 | 699.6ms/step | 28.6 samples/s +Step 39810/44000 | Loss: 2.9152 | LR: 1.15e-05 | 699.3ms/step | 28.6 samples/s +Step 39820/44000 | Loss: 2.9384 | LR: 1.14e-05 | 696.7ms/step | 28.7 samples/s +Step 39830/44000 | Loss: 2.9343 | LR: 1.14e-05 | 697.1ms/step | 28.7 samples/s +Step 39840/44000 | Loss: 2.9320 | LR: 1.13e-05 | 697.0ms/step | 28.7 samples/s +Step 39850/44000 | Loss: 2.9417 | LR: 1.13e-05 | 698.1ms/step | 28.6 samples/s +Step 39860/44000 | Loss: 2.9262 | LR: 1.12e-05 | 695.9ms/step | 28.7 samples/s +Step 39870/44000 | Loss: 2.9652 | LR: 1.12e-05 | 696.6ms/step | 28.7 samples/s +Step 39880/44000 | Loss: 2.9041 | LR: 1.11e-05 | 700.2ms/step | 28.6 samples/s +Step 39890/44000 | Loss: 2.9321 | LR: 1.11e-05 | 697.1ms/step | 28.7 samples/s +Step 39900/44000 | Loss: 2.9203 | LR: 1.10e-05 | 698.7ms/step | 28.6 samples/s +Step 39910/44000 | Loss: 2.9322 | LR: 1.10e-05 | 700.1ms/step | 28.6 samples/s +Step 39920/44000 | Loss: 2.9476 | LR: 1.09e-05 | 697.9ms/step | 28.7 samples/s +Step 39930/44000 | Loss: 2.9626 | LR: 1.08e-05 | 697.9ms/step | 28.7 samples/s +Step 39940/44000 | Loss: 2.9422 | LR: 1.08e-05 | 700.2ms/step | 28.6 samples/s +Step 39950/44000 | Loss: 2.9352 | LR: 1.07e-05 | 698.9ms/step | 28.6 samples/s +Step 39960/44000 | Loss: 2.9575 | LR: 1.07e-05 | 700.6ms/step | 28.5 samples/s +Step 39970/44000 | Loss: 2.9280 | LR: 1.06e-05 | 696.9ms/step | 28.7 samples/s +Step 39980/44000 | Loss: 2.9573 | LR: 1.06e-05 | 699.4ms/step | 28.6 samples/s +Step 39990/44000 | Loss: 2.9382 | LR: 1.05e-05 | 696.4ms/step | 28.7 samples/s +Step 40000/44000 | Loss: 2.9559 | LR: 1.05e-05 | 698.0ms/step | 28.7 samples/s +Validation at step 40000: Loss = 2.9577, Perplexity = 19.25 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-40000 +Step 40010/44000 | Loss: 2.8670 | LR: 1.04e-05 | 1598.8ms/step | 12.5 samples/s +Step 40020/44000 | Loss: 2.9027 | LR: 1.04e-05 | 699.6ms/step | 28.6 samples/s +Step 40030/44000 | Loss: 2.9528 | LR: 1.03e-05 | 697.1ms/step | 28.7 samples/s +Step 40040/44000 | Loss: 2.9316 | LR: 1.03e-05 | 695.9ms/step | 28.7 samples/s +Step 40050/44000 | Loss: 2.9327 | LR: 1.02e-05 | 695.3ms/step | 28.8 samples/s +Step 40060/44000 | Loss: 2.9147 | LR: 1.02e-05 | 695.1ms/step | 28.8 samples/s +Step 40070/44000 | Loss: 2.8996 | LR: 1.01e-05 | 695.1ms/step | 28.8 samples/s +Step 40080/44000 | Loss: 2.9169 | LR: 1.01e-05 | 695.2ms/step | 28.8 samples/s +Step 40090/44000 | Loss: 2.9459 | LR: 1.00e-05 | 700.2ms/step | 28.6 samples/s +Step 40100/44000 | Loss: 2.9172 | LR: 9.96e-06 | 697.1ms/step | 28.7 samples/s +Step 40110/44000 | Loss: 2.8963 | LR: 9.91e-06 | 695.5ms/step | 28.8 samples/s +Step 40120/44000 | Loss: 2.9662 | LR: 9.86e-06 | 696.1ms/step | 28.7 samples/s +Step 40130/44000 | Loss: 2.9289 | LR: 9.81e-06 | 696.5ms/step | 28.7 samples/s +Step 40140/44000 | Loss: 2.9385 | LR: 9.76e-06 | 695.1ms/step | 28.8 samples/s +Step 40150/44000 | Loss: 2.9346 | LR: 9.71e-06 | 696.5ms/step | 28.7 samples/s +Step 40160/44000 | Loss: 2.9094 | LR: 9.66e-06 | 696.8ms/step | 28.7 samples/s +Step 40170/44000 | Loss: 2.9169 | LR: 9.61e-06 | 696.2ms/step | 28.7 samples/s +Step 40180/44000 | Loss: 2.9003 | LR: 9.56e-06 | 699.8ms/step | 28.6 samples/s +Step 40190/44000 | Loss: 2.9295 | LR: 9.51e-06 | 698.4ms/step | 28.6 samples/s +Step 40200/44000 | Loss: 2.9337 | LR: 9.46e-06 | 693.3ms/step | 28.8 samples/s +Step 40210/44000 | Loss: 2.9222 | LR: 9.41e-06 | 694.9ms/step | 28.8 samples/s +Step 40220/44000 | Loss: 2.9164 | LR: 9.36e-06 | 697.8ms/step | 28.7 samples/s +Step 40230/44000 | Loss: 2.9239 | LR: 9.32e-06 | 696.2ms/step | 28.7 samples/s +Step 40240/44000 | Loss: 2.9487 | LR: 9.27e-06 | 696.8ms/step | 28.7 samples/s +Step 40250/44000 | Loss: 2.9927 | LR: 9.22e-06 | 697.3ms/step | 28.7 samples/s +Validation at step 40250: Loss = 2.9558, Perplexity = 19.22 +Step 40260/44000 | Loss: 2.9850 | LR: 9.17e-06 | 1259.8ms/step | 15.9 samples/s +Step 40270/44000 | Loss: 2.8906 | LR: 9.12e-06 | 698.2ms/step | 28.6 samples/s +Step 40280/44000 | Loss: 2.9146 | LR: 9.07e-06 | 697.8ms/step | 28.7 samples/s +Step 40290/44000 | Loss: 2.9464 | LR: 9.02e-06 | 693.4ms/step | 28.8 samples/s +Step 40300/44000 | Loss: 2.9624 | LR: 8.97e-06 | 698.0ms/step | 28.7 samples/s +Step 40310/44000 | Loss: 2.9278 | LR: 8.93e-06 | 694.7ms/step | 28.8 samples/s +Step 40320/44000 | Loss: 2.9547 | LR: 8.88e-06 | 698.4ms/step | 28.6 samples/s +Step 40330/44000 | Loss: 2.9620 | LR: 8.83e-06 | 694.5ms/step | 28.8 samples/s +Step 40340/44000 | Loss: 2.9258 | LR: 8.78e-06 | 697.6ms/step | 28.7 samples/s +Step 40350/44000 | Loss: 2.9238 | LR: 8.74e-06 | 697.2ms/step | 28.7 samples/s +Step 40360/44000 | Loss: 2.9550 | LR: 8.69e-06 | 697.5ms/step | 28.7 samples/s +Step 40370/44000 | Loss: 2.9817 | LR: 8.64e-06 | 695.9ms/step | 28.7 samples/s +Step 40380/44000 | Loss: 2.9538 | LR: 8.59e-06 | 697.3ms/step | 28.7 samples/s +Step 40390/44000 | Loss: 2.9067 | LR: 8.55e-06 | 694.4ms/step | 28.8 samples/s +Step 40400/44000 | Loss: 2.9295 | LR: 8.50e-06 | 697.2ms/step | 28.7 samples/s +Step 40410/44000 | Loss: 2.9222 | LR: 8.45e-06 | 695.2ms/step | 28.8 samples/s +Step 40420/44000 | Loss: 2.9216 | LR: 8.41e-06 | 695.7ms/step | 28.7 samples/s +Step 40430/44000 | Loss: 2.9056 | LR: 8.36e-06 | 698.5ms/step | 28.6 samples/s +Step 40440/44000 | Loss: 2.9105 | LR: 8.31e-06 | 697.9ms/step | 28.7 samples/s +Step 40450/44000 | Loss: 2.9163 | LR: 8.27e-06 | 697.2ms/step | 28.7 samples/s +Step 40460/44000 | Loss: 2.9953 | LR: 8.22e-06 | 695.3ms/step | 28.8 samples/s +Step 40470/44000 | Loss: 2.9376 | LR: 8.17e-06 | 693.8ms/step | 28.8 samples/s +Step 40480/44000 | Loss: 2.9587 | LR: 8.13e-06 | 697.9ms/step | 28.7 samples/s +Step 40490/44000 | Loss: 2.9267 | LR: 8.08e-06 | 692.5ms/step | 28.9 samples/s +Step 40500/44000 | Loss: 2.9613 | LR: 8.04e-06 | 693.2ms/step | 28.9 samples/s +Validation at step 40500: Loss = 2.9546, Perplexity = 19.19 +Step 40510/44000 | Loss: 2.9422 | LR: 7.99e-06 | 1268.4ms/step | 15.8 samples/s +Step 40520/44000 | Loss: 2.9273 | LR: 7.94e-06 | 696.0ms/step | 28.7 samples/s +Step 40530/44000 | Loss: 2.9227 | LR: 7.90e-06 | 696.1ms/step | 28.7 samples/s +Step 40540/44000 | Loss: 2.9303 | LR: 7.85e-06 | 696.5ms/step | 28.7 samples/s +Step 40550/44000 | Loss: 2.9471 | LR: 7.81e-06 | 697.1ms/step | 28.7 samples/s +Step 40560/44000 | Loss: 2.9109 | LR: 7.76e-06 | 698.5ms/step | 28.6 samples/s +Step 40570/44000 | Loss: 2.9420 | LR: 7.72e-06 | 696.4ms/step | 28.7 samples/s +Step 40580/44000 | Loss: 2.9211 | LR: 7.67e-06 | 696.8ms/step | 28.7 samples/s +Step 40590/44000 | Loss: 2.9074 | LR: 7.63e-06 | 697.2ms/step | 28.7 samples/s +Step 40600/44000 | Loss: 2.9403 | LR: 7.59e-06 | 696.7ms/step | 28.7 samples/s +Step 40610/44000 | Loss: 2.8940 | LR: 7.54e-06 | 696.3ms/step | 28.7 samples/s +Step 40620/44000 | Loss: 2.9232 | LR: 7.50e-06 | 695.1ms/step | 28.8 samples/s +Step 40630/44000 | Loss: 2.9255 | LR: 7.45e-06 | 695.3ms/step | 28.8 samples/s +Step 40640/44000 | Loss: 2.9265 | LR: 7.41e-06 | 695.5ms/step | 28.8 samples/s +Step 40650/44000 | Loss: 2.9090 | LR: 7.37e-06 | 695.3ms/step | 28.8 samples/s +Step 40660/44000 | Loss: 2.9580 | LR: 7.32e-06 | 697.9ms/step | 28.7 samples/s +Step 40670/44000 | Loss: 2.9544 | LR: 7.28e-06 | 698.7ms/step | 28.6 samples/s +Step 40680/44000 | Loss: 2.9233 | LR: 7.23e-06 | 696.6ms/step | 28.7 samples/s +Step 40690/44000 | Loss: 2.9572 | LR: 7.19e-06 | 693.7ms/step | 28.8 samples/s +Step 40700/44000 | Loss: 2.9030 | LR: 7.15e-06 | 699.2ms/step | 28.6 samples/s +Step 40710/44000 | Loss: 2.9173 | LR: 7.10e-06 | 697.9ms/step | 28.7 samples/s +Step 40720/44000 | Loss: 2.9403 | LR: 7.06e-06 | 695.1ms/step | 28.8 samples/s +Step 40730/44000 | Loss: 2.8928 | LR: 7.02e-06 | 699.3ms/step | 28.6 samples/s +Step 40740/44000 | Loss: 2.9048 | LR: 6.98e-06 | 698.1ms/step | 28.6 samples/s +Step 40750/44000 | Loss: 2.9507 | LR: 6.93e-06 | 696.3ms/step | 28.7 samples/s +Validation at step 40750: Loss = 2.9544, Perplexity = 19.19 +Step 40760/44000 | Loss: 2.9383 | LR: 6.89e-06 | 1265.6ms/step | 15.8 samples/s +Step 40770/44000 | Loss: 2.9236 | LR: 6.85e-06 | 697.9ms/step | 28.7 samples/s +Step 40780/44000 | Loss: 2.9629 | LR: 6.81e-06 | 699.2ms/step | 28.6 samples/s +Step 40790/44000 | Loss: 2.9299 | LR: 6.77e-06 | 694.9ms/step | 28.8 samples/s +Step 40800/44000 | Loss: 2.9259 | LR: 6.72e-06 | 698.7ms/step | 28.6 samples/s +Step 40810/44000 | Loss: 2.9386 | LR: 6.68e-06 | 698.6ms/step | 28.6 samples/s +Step 40820/44000 | Loss: 2.9276 | LR: 6.64e-06 | 694.7ms/step | 28.8 samples/s +Step 40830/44000 | Loss: 2.9560 | LR: 6.60e-06 | 694.9ms/step | 28.8 samples/s +Step 40840/44000 | Loss: 2.9979 | LR: 6.56e-06 | 697.3ms/step | 28.7 samples/s +Step 40850/44000 | Loss: 2.9424 | LR: 6.52e-06 | 696.6ms/step | 28.7 samples/s +Step 40860/44000 | Loss: 2.9146 | LR: 6.47e-06 | 693.4ms/step | 28.8 samples/s +Step 40870/44000 | Loss: 2.9713 | LR: 6.43e-06 | 697.2ms/step | 28.7 samples/s +Step 40880/44000 | Loss: 2.9230 | LR: 6.39e-06 | 696.8ms/step | 28.7 samples/s +Step 40890/44000 | Loss: 2.9526 | LR: 6.35e-06 | 695.5ms/step | 28.8 samples/s +Step 40900/44000 | Loss: 2.9295 | LR: 6.31e-06 | 698.0ms/step | 28.7 samples/s +Step 40910/44000 | Loss: 2.9472 | LR: 6.27e-06 | 693.9ms/step | 28.8 samples/s +Step 40920/44000 | Loss: 2.9422 | LR: 6.23e-06 | 694.7ms/step | 28.8 samples/s +Step 40930/44000 | Loss: 2.9392 | LR: 6.19e-06 | 697.8ms/step | 28.7 samples/s +Step 40940/44000 | Loss: 2.9340 | LR: 6.15e-06 | 695.6ms/step | 28.8 samples/s +Step 40950/44000 | Loss: 2.8891 | LR: 6.11e-06 | 696.0ms/step | 28.7 samples/s +Step 40960/44000 | Loss: 2.9307 | LR: 6.07e-06 | 698.7ms/step | 28.6 samples/s +Step 40970/44000 | Loss: 2.9259 | LR: 6.03e-06 | 701.2ms/step | 28.5 samples/s +Step 40980/44000 | Loss: 2.9456 | LR: 5.99e-06 | 695.1ms/step | 28.8 samples/s +Step 40990/44000 | Loss: 2.9637 | LR: 5.95e-06 | 693.4ms/step | 28.8 samples/s +Step 41000/44000 | Loss: 2.8759 | LR: 5.91e-06 | 696.2ms/step | 28.7 samples/s +Validation at step 41000: Loss = 2.9547, Perplexity = 19.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-41000 +Step 41010/44000 | Loss: 2.9347 | LR: 5.87e-06 | 1616.9ms/step | 12.4 samples/s +Step 41020/44000 | Loss: 2.9328 | LR: 5.83e-06 | 699.6ms/step | 28.6 samples/s +Step 41030/44000 | Loss: 2.9424 | LR: 5.80e-06 | 699.2ms/step | 28.6 samples/s +Step 41040/44000 | Loss: 2.9387 | LR: 5.76e-06 | 698.7ms/step | 28.6 samples/s +Step 41050/44000 | Loss: 2.9378 | LR: 5.72e-06 | 699.8ms/step | 28.6 samples/s +Step 41060/44000 | Loss: 2.9254 | LR: 5.68e-06 | 698.8ms/step | 28.6 samples/s +Step 41070/44000 | Loss: 2.9354 | LR: 5.64e-06 | 699.3ms/step | 28.6 samples/s +Step 41080/44000 | Loss: 2.9115 | LR: 5.60e-06 | 700.5ms/step | 28.6 samples/s +Step 41090/44000 | Loss: 2.9176 | LR: 5.56e-06 | 699.3ms/step | 28.6 samples/s +Step 41100/44000 | Loss: 2.9380 | LR: 5.53e-06 | 697.8ms/step | 28.7 samples/s +Step 41110/44000 | Loss: 2.9384 | LR: 5.49e-06 | 698.4ms/step | 28.6 samples/s +Step 41120/44000 | Loss: 2.9729 | LR: 5.45e-06 | 698.9ms/step | 28.6 samples/s +Step 41130/44000 | Loss: 2.9326 | LR: 5.41e-06 | 698.6ms/step | 28.6 samples/s +Step 41140/44000 | Loss: 2.9481 | LR: 5.38e-06 | 696.4ms/step | 28.7 samples/s +Step 41150/44000 | Loss: 2.9458 | LR: 5.34e-06 | 698.3ms/step | 28.6 samples/s +Step 41160/44000 | Loss: 2.9450 | LR: 5.30e-06 | 699.7ms/step | 28.6 samples/s +Step 41170/44000 | Loss: 2.8775 | LR: 5.26e-06 | 700.6ms/step | 28.5 samples/s +Step 41180/44000 | Loss: 2.9521 | LR: 5.23e-06 | 698.9ms/step | 28.6 samples/s +Step 41190/44000 | Loss: 2.9345 | LR: 5.19e-06 | 698.0ms/step | 28.7 samples/s +Step 41200/44000 | Loss: 2.9309 | LR: 5.15e-06 | 699.0ms/step | 28.6 samples/s +Step 41210/44000 | Loss: 2.9219 | LR: 5.12e-06 | 697.4ms/step | 28.7 samples/s +Step 41220/44000 | Loss: 2.9564 | LR: 5.08e-06 | 700.4ms/step | 28.6 samples/s +Step 41230/44000 | Loss: 2.9146 | LR: 5.04e-06 | 697.2ms/step | 28.7 samples/s +Step 41240/44000 | Loss: 2.9592 | LR: 5.01e-06 | 697.5ms/step | 28.7 samples/s +Step 41250/44000 | Loss: 2.9553 | LR: 4.97e-06 | 700.4ms/step | 28.6 samples/s +Validation at step 41250: Loss = 2.9543, Perplexity = 19.19 +Step 41260/44000 | Loss: 2.9344 | LR: 4.94e-06 | 1274.8ms/step | 15.7 samples/s +Step 41270/44000 | Loss: 2.9227 | LR: 4.90e-06 | 700.1ms/step | 28.6 samples/s +Step 41280/44000 | Loss: 2.9463 | LR: 4.86e-06 | 697.7ms/step | 28.7 samples/s +Step 41290/44000 | Loss: 2.9588 | LR: 4.83e-06 | 697.8ms/step | 28.7 samples/s +Step 41300/44000 | Loss: 2.8895 | LR: 4.79e-06 | 701.1ms/step | 28.5 samples/s +Step 41310/44000 | Loss: 2.9406 | LR: 4.76e-06 | 698.3ms/step | 28.6 samples/s +Step 41320/44000 | Loss: 2.9379 | LR: 4.72e-06 | 699.0ms/step | 28.6 samples/s +Step 41330/44000 | Loss: 2.9459 | LR: 4.69e-06 | 698.0ms/step | 28.7 samples/s +Step 41340/44000 | Loss: 2.9493 | LR: 4.65e-06 | 700.5ms/step | 28.5 samples/s +Step 41350/44000 | Loss: 2.9455 | LR: 4.62e-06 | 698.5ms/step | 28.6 samples/s +Step 41360/44000 | Loss: 2.9114 | LR: 4.58e-06 | 697.1ms/step | 28.7 samples/s +Step 41370/44000 | Loss: 2.9480 | LR: 4.55e-06 | 700.2ms/step | 28.6 samples/s +Step 41380/44000 | Loss: 2.9580 | LR: 4.51e-06 | 699.3ms/step | 28.6 samples/s +Step 41390/44000 | Loss: 2.9423 | LR: 4.48e-06 | 698.1ms/step | 28.6 samples/s +Step 41400/44000 | Loss: 2.9340 | LR: 4.45e-06 | 700.7ms/step | 28.5 samples/s +Step 41410/44000 | Loss: 2.9027 | LR: 4.41e-06 | 699.5ms/step | 28.6 samples/s +Step 41420/44000 | Loss: 2.9370 | LR: 4.38e-06 | 697.5ms/step | 28.7 samples/s +Step 41430/44000 | Loss: 2.9580 | LR: 4.34e-06 | 700.9ms/step | 28.5 samples/s +Step 41440/44000 | Loss: 2.9281 | LR: 4.31e-06 | 696.9ms/step | 28.7 samples/s +Step 41450/44000 | Loss: 2.9523 | LR: 4.28e-06 | 696.5ms/step | 28.7 samples/s +Step 41460/44000 | Loss: 2.9676 | LR: 4.24e-06 | 697.1ms/step | 28.7 samples/s +Step 41470/44000 | Loss: 2.8934 | LR: 4.21e-06 | 697.1ms/step | 28.7 samples/s +Step 41480/44000 | Loss: 2.9393 | LR: 4.18e-06 | 697.4ms/step | 28.7 samples/s +Step 41490/44000 | Loss: 2.9524 | LR: 4.14e-06 | 695.5ms/step | 28.8 samples/s +Step 41500/44000 | Loss: 2.9609 | LR: 4.11e-06 | 696.3ms/step | 28.7 samples/s +Validation at step 41500: Loss = 2.9539, Perplexity = 19.18 +Step 41510/44000 | Loss: 2.9449 | LR: 4.08e-06 | 1266.4ms/step | 15.8 samples/s +Step 41520/44000 | Loss: 2.9238 | LR: 4.05e-06 | 696.8ms/step | 28.7 samples/s +Step 41530/44000 | Loss: 2.9426 | LR: 4.01e-06 | 698.0ms/step | 28.7 samples/s +Step 41540/44000 | Loss: 2.9565 | LR: 3.98e-06 | 698.0ms/step | 28.7 samples/s +Step 41550/44000 | Loss: 2.8964 | LR: 3.95e-06 | 697.8ms/step | 28.7 samples/s +Step 41560/44000 | Loss: 2.9459 | LR: 3.92e-06 | 696.1ms/step | 28.7 samples/s +Step 41570/44000 | Loss: 2.9257 | LR: 3.88e-06 | 695.8ms/step | 28.7 samples/s +Step 41580/44000 | Loss: 2.9362 | LR: 3.85e-06 | 698.1ms/step | 28.7 samples/s +Step 41590/44000 | Loss: 2.9399 | LR: 3.82e-06 | 699.5ms/step | 28.6 samples/s +Step 41600/44000 | Loss: 2.9274 | LR: 3.79e-06 | 696.9ms/step | 28.7 samples/s +Step 41610/44000 | Loss: 2.9629 | LR: 3.76e-06 | 697.4ms/step | 28.7 samples/s +Step 41620/44000 | Loss: 2.9574 | LR: 3.73e-06 | 698.8ms/step | 28.6 samples/s +Step 41630/44000 | Loss: 2.9768 | LR: 3.70e-06 | 698.6ms/step | 28.6 samples/s +Step 41640/44000 | Loss: 2.9111 | LR: 3.66e-06 | 699.4ms/step | 28.6 samples/s +Step 41650/44000 | Loss: 2.9061 | LR: 3.63e-06 | 697.4ms/step | 28.7 samples/s +Step 41660/44000 | Loss: 2.9369 | LR: 3.60e-06 | 700.4ms/step | 28.6 samples/s +Step 41670/44000 | Loss: 2.8941 | LR: 3.57e-06 | 696.1ms/step | 28.7 samples/s +Step 41680/44000 | Loss: 2.9149 | LR: 3.54e-06 | 699.7ms/step | 28.6 samples/s +Step 41690/44000 | Loss: 2.9354 | LR: 3.51e-06 | 697.7ms/step | 28.7 samples/s +Step 41700/44000 | Loss: 2.9305 | LR: 3.48e-06 | 698.5ms/step | 28.6 samples/s +Step 41710/44000 | Loss: 2.9453 | LR: 3.45e-06 | 697.2ms/step | 28.7 samples/s +Step 41720/44000 | Loss: 2.9542 | LR: 3.42e-06 | 699.4ms/step | 28.6 samples/s +Step 41730/44000 | Loss: 2.9409 | LR: 3.39e-06 | 699.4ms/step | 28.6 samples/s +Step 41740/44000 | Loss: 2.8932 | LR: 3.36e-06 | 696.9ms/step | 28.7 samples/s +Step 41750/44000 | Loss: 2.9773 | LR: 3.33e-06 | 700.2ms/step | 28.6 samples/s +Validation at step 41750: Loss = 2.9544, Perplexity = 19.19 +Step 41760/44000 | Loss: 2.9388 | LR: 3.30e-06 | 1265.3ms/step | 15.8 samples/s +Step 41770/44000 | Loss: 2.9470 | LR: 3.27e-06 | 699.0ms/step | 28.6 samples/s +Step 41780/44000 | Loss: 2.9146 | LR: 3.24e-06 | 697.5ms/step | 28.7 samples/s +Step 41790/44000 | Loss: 2.9057 | LR: 3.21e-06 | 699.2ms/step | 28.6 samples/s +Step 41800/44000 | Loss: 2.9299 | LR: 3.19e-06 | 699.2ms/step | 28.6 samples/s +Step 41810/44000 | Loss: 2.9335 | LR: 3.16e-06 | 697.6ms/step | 28.7 samples/s +Step 41820/44000 | Loss: 2.9046 | LR: 3.13e-06 | 699.3ms/step | 28.6 samples/s +Step 41830/44000 | Loss: 2.9200 | LR: 3.10e-06 | 697.7ms/step | 28.7 samples/s +Step 41840/44000 | Loss: 2.9441 | LR: 3.07e-06 | 697.4ms/step | 28.7 samples/s +Step 41850/44000 | Loss: 2.9036 | LR: 3.04e-06 | 698.4ms/step | 28.6 samples/s +Step 41860/44000 | Loss: 2.9007 | LR: 3.01e-06 | 698.8ms/step | 28.6 samples/s +Step 41870/44000 | Loss: 2.9339 | LR: 2.99e-06 | 699.4ms/step | 28.6 samples/s +Step 41880/44000 | Loss: 2.9185 | LR: 2.96e-06 | 694.9ms/step | 28.8 samples/s +Step 41890/44000 | Loss: 2.9148 | LR: 2.93e-06 | 698.3ms/step | 28.6 samples/s +Step 41900/44000 | Loss: 2.9346 | LR: 2.90e-06 | 698.4ms/step | 28.6 samples/s +Step 41910/44000 | Loss: 2.9302 | LR: 2.88e-06 | 699.9ms/step | 28.6 samples/s +Step 41920/44000 | Loss: 2.9808 | LR: 2.85e-06 | 699.5ms/step | 28.6 samples/s +Step 41930/44000 | Loss: 2.9293 | LR: 2.82e-06 | 697.7ms/step | 28.7 samples/s +Step 41940/44000 | Loss: 2.9257 | LR: 2.79e-06 | 699.1ms/step | 28.6 samples/s +Step 41950/44000 | Loss: 2.9198 | LR: 2.77e-06 | 698.3ms/step | 28.6 samples/s +Step 41960/44000 | Loss: 2.9035 | LR: 2.74e-06 | 701.0ms/step | 28.5 samples/s +Step 41970/44000 | Loss: 2.9092 | LR: 2.71e-06 | 698.6ms/step | 28.6 samples/s +Step 41980/44000 | Loss: 2.9575 | LR: 2.69e-06 | 699.2ms/step | 28.6 samples/s +Step 41990/44000 | Loss: 2.9340 | LR: 2.66e-06 | 698.4ms/step | 28.6 samples/s +Step 42000/44000 | Loss: 2.9246 | LR: 2.63e-06 | 700.2ms/step | 28.6 samples/s +Validation at step 42000: Loss = 2.9546, Perplexity = 19.19 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-42000 +Step 42010/44000 | Loss: 2.9070 | LR: 2.61e-06 | 1593.1ms/step | 12.6 samples/s +Step 42020/44000 | Loss: 2.9197 | LR: 2.58e-06 | 697.2ms/step | 28.7 samples/s +Step 42030/44000 | Loss: 2.9668 | LR: 2.56e-06 | 701.7ms/step | 28.5 samples/s +Step 42040/44000 | Loss: 2.9604 | LR: 2.53e-06 | 699.9ms/step | 28.6 samples/s +Step 42050/44000 | Loss: 2.9089 | LR: 2.50e-06 | 700.7ms/step | 28.5 samples/s +Step 42060/44000 | Loss: 2.8829 | LR: 2.48e-06 | 699.0ms/step | 28.6 samples/s +Step 42070/44000 | Loss: 2.9747 | LR: 2.45e-06 | 698.4ms/step | 28.6 samples/s +Step 42080/44000 | Loss: 2.9470 | LR: 2.43e-06 | 696.0ms/step | 28.7 samples/s +Step 42090/44000 | Loss: 2.9356 | LR: 2.40e-06 | 697.1ms/step | 28.7 samples/s +Step 42100/44000 | Loss: 2.9437 | LR: 2.38e-06 | 697.9ms/step | 28.7 samples/s +Step 42110/44000 | Loss: 2.9599 | LR: 2.35e-06 | 697.2ms/step | 28.7 samples/s +Step 42120/44000 | Loss: 2.8978 | LR: 2.33e-06 | 698.8ms/step | 28.6 samples/s +Step 42130/44000 | Loss: 2.9344 | LR: 2.30e-06 | 698.4ms/step | 28.6 samples/s +Step 42140/44000 | Loss: 2.9451 | LR: 2.28e-06 | 700.0ms/step | 28.6 samples/s +Step 42150/44000 | Loss: 2.8982 | LR: 2.25e-06 | 696.4ms/step | 28.7 samples/s +Step 42160/44000 | Loss: 2.9402 | LR: 2.23e-06 | 701.1ms/step | 28.5 samples/s +Step 42170/44000 | Loss: 2.8825 | LR: 2.21e-06 | 700.8ms/step | 28.5 samples/s +Step 42180/44000 | Loss: 2.9007 | LR: 2.18e-06 | 696.7ms/step | 28.7 samples/s +Step 42190/44000 | Loss: 2.9438 | LR: 2.16e-06 | 699.6ms/step | 28.6 samples/s +Step 42200/44000 | Loss: 2.9426 | LR: 2.13e-06 | 697.2ms/step | 28.7 samples/s +Step 42210/44000 | Loss: 2.9664 | LR: 2.11e-06 | 699.1ms/step | 28.6 samples/s +Step 42220/44000 | Loss: 2.9444 | LR: 2.09e-06 | 697.6ms/step | 28.7 samples/s +Step 42230/44000 | Loss: 2.9331 | LR: 2.06e-06 | 701.4ms/step | 28.5 samples/s +Step 42240/44000 | Loss: 2.9593 | LR: 2.04e-06 | 696.9ms/step | 28.7 samples/s +Step 42250/44000 | Loss: 2.9407 | LR: 2.02e-06 | 698.9ms/step | 28.6 samples/s +Validation at step 42250: Loss = 2.9552, Perplexity = 19.21 +Step 42260/44000 | Loss: 2.9215 | LR: 1.99e-06 | 1264.8ms/step | 15.8 samples/s +Step 42270/44000 | Loss: 2.9526 | LR: 1.97e-06 | 698.9ms/step | 28.6 samples/s +Step 42280/44000 | Loss: 2.9425 | LR: 1.95e-06 | 699.1ms/step | 28.6 samples/s +Step 42290/44000 | Loss: 2.9056 | LR: 1.93e-06 | 696.7ms/step | 28.7 samples/s +Step 42300/44000 | Loss: 2.9002 | LR: 1.90e-06 | 695.7ms/step | 28.8 samples/s +Step 42310/44000 | Loss: 2.8928 | LR: 1.88e-06 | 698.9ms/step | 28.6 samples/s +Step 42320/44000 | Loss: 2.9778 | LR: 1.86e-06 | 698.3ms/step | 28.6 samples/s +Step 42330/44000 | Loss: 2.9765 | LR: 1.84e-06 | 699.1ms/step | 28.6 samples/s +Step 42340/44000 | Loss: 2.9228 | LR: 1.82e-06 | 697.9ms/step | 28.7 samples/s +Step 42350/44000 | Loss: 2.9362 | LR: 1.79e-06 | 698.6ms/step | 28.6 samples/s +Step 42360/44000 | Loss: 2.9780 | LR: 1.77e-06 | 697.2ms/step | 28.7 samples/s +Step 42370/44000 | Loss: 2.9107 | LR: 1.75e-06 | 697.0ms/step | 28.7 samples/s +Step 42380/44000 | Loss: 2.9540 | LR: 1.73e-06 | 696.6ms/step | 28.7 samples/s +Step 42390/44000 | Loss: 2.9184 | LR: 1.71e-06 | 698.1ms/step | 28.7 samples/s +Step 42400/44000 | Loss: 2.9628 | LR: 1.69e-06 | 698.9ms/step | 28.6 samples/s +Step 42410/44000 | Loss: 2.9264 | LR: 1.67e-06 | 695.0ms/step | 28.8 samples/s +Step 42420/44000 | Loss: 2.9020 | LR: 1.64e-06 | 696.2ms/step | 28.7 samples/s +Step 42430/44000 | Loss: 2.8903 | LR: 1.62e-06 | 695.1ms/step | 28.8 samples/s +Step 42440/44000 | Loss: 2.9698 | LR: 1.60e-06 | 697.1ms/step | 28.7 samples/s +Step 42450/44000 | Loss: 2.9176 | LR: 1.58e-06 | 700.0ms/step | 28.6 samples/s +Step 42460/44000 | Loss: 2.9097 | LR: 1.56e-06 | 698.5ms/step | 28.6 samples/s +Step 42470/44000 | Loss: 2.9196 | LR: 1.54e-06 | 697.1ms/step | 28.7 samples/s +Step 42480/44000 | Loss: 2.9007 | LR: 1.52e-06 | 698.0ms/step | 28.7 samples/s +Step 42490/44000 | Loss: 2.9731 | LR: 1.50e-06 | 699.3ms/step | 28.6 samples/s +Step 42500/44000 | Loss: 2.9277 | LR: 1.48e-06 | 700.3ms/step | 28.6 samples/s +Validation at step 42500: Loss = 2.9553, Perplexity = 19.21 +Step 42510/44000 | Loss: 2.9409 | LR: 1.46e-06 | 1264.8ms/step | 15.8 samples/s +Step 42520/44000 | Loss: 2.9381 | LR: 1.44e-06 | 697.7ms/step | 28.7 samples/s +Step 42530/44000 | Loss: 2.9671 | LR: 1.42e-06 | 697.8ms/step | 28.7 samples/s +Step 42540/44000 | Loss: 2.9112 | LR: 1.40e-06 | 699.0ms/step | 28.6 samples/s +Step 42550/44000 | Loss: 2.9117 | LR: 1.39e-06 | 699.1ms/step | 28.6 samples/s +Step 42560/44000 | Loss: 2.9215 | LR: 1.37e-06 | 698.7ms/step | 28.6 samples/s +Step 42570/44000 | Loss: 2.9346 | LR: 1.35e-06 | 696.6ms/step | 28.7 samples/s +Step 42580/44000 | Loss: 2.9114 | LR: 1.33e-06 | 695.5ms/step | 28.8 samples/s +Step 42590/44000 | Loss: 2.9555 | LR: 1.31e-06 | 695.4ms/step | 28.8 samples/s +Step 42600/44000 | Loss: 2.8913 | LR: 1.29e-06 | 696.5ms/step | 28.7 samples/s +Step 42610/44000 | Loss: 2.8738 | LR: 1.27e-06 | 697.2ms/step | 28.7 samples/s +Step 42620/44000 | Loss: 2.9209 | LR: 1.25e-06 | 699.0ms/step | 28.6 samples/s +Step 42630/44000 | Loss: 2.9365 | LR: 1.24e-06 | 697.0ms/step | 28.7 samples/s +Step 42640/44000 | Loss: 2.9015 | LR: 1.22e-06 | 699.2ms/step | 28.6 samples/s +Step 42650/44000 | Loss: 2.9324 | LR: 1.20e-06 | 698.4ms/step | 28.6 samples/s +Step 42660/44000 | Loss: 2.9149 | LR: 1.18e-06 | 696.4ms/step | 28.7 samples/s +Step 42670/44000 | Loss: 2.9184 | LR: 1.17e-06 | 699.2ms/step | 28.6 samples/s +Step 42680/44000 | Loss: 2.9467 | LR: 1.15e-06 | 695.7ms/step | 28.7 samples/s +Step 42690/44000 | Loss: 2.9462 | LR: 1.13e-06 | 696.5ms/step | 28.7 samples/s +Step 42700/44000 | Loss: 2.8825 | LR: 1.11e-06 | 700.6ms/step | 28.5 samples/s +Step 42710/44000 | Loss: 2.9266 | LR: 1.10e-06 | 701.8ms/step | 28.5 samples/s +Step 42720/44000 | Loss: 2.9618 | LR: 1.08e-06 | 700.5ms/step | 28.5 samples/s +Step 42730/44000 | Loss: 2.9391 | LR: 1.06e-06 | 698.9ms/step | 28.6 samples/s +Step 42740/44000 | Loss: 2.9340 | LR: 1.05e-06 | 698.5ms/step | 28.6 samples/s +Step 42750/44000 | Loss: 2.9357 | LR: 1.03e-06 | 698.0ms/step | 28.7 samples/s +Validation at step 42750: Loss = 2.9538, Perplexity = 19.18 +Step 42760/44000 | Loss: 2.9290 | LR: 1.01e-06 | 1266.3ms/step | 15.8 samples/s +Step 42770/44000 | Loss: 2.9468 | LR: 9.97e-07 | 700.8ms/step | 28.5 samples/s +Step 42780/44000 | Loss: 2.9225 | LR: 9.81e-07 | 697.2ms/step | 28.7 samples/s +Step 42790/44000 | Loss: 2.9297 | LR: 9.65e-07 | 700.3ms/step | 28.6 samples/s +Step 42800/44000 | Loss: 2.9086 | LR: 9.49e-07 | 697.3ms/step | 28.7 samples/s +Step 42810/44000 | Loss: 2.9286 | LR: 9.33e-07 | 697.8ms/step | 28.7 samples/s +Step 42820/44000 | Loss: 2.9085 | LR: 9.18e-07 | 700.0ms/step | 28.6 samples/s +Step 42830/44000 | Loss: 2.9401 | LR: 9.02e-07 | 700.8ms/step | 28.5 samples/s +Step 42840/44000 | Loss: 2.9089 | LR: 8.87e-07 | 697.9ms/step | 28.7 samples/s +Step 42850/44000 | Loss: 2.9434 | LR: 8.72e-07 | 699.3ms/step | 28.6 samples/s +Step 42860/44000 | Loss: 2.9374 | LR: 8.57e-07 | 700.2ms/step | 28.6 samples/s +Step 42870/44000 | Loss: 2.8964 | LR: 8.42e-07 | 698.8ms/step | 28.6 samples/s +Step 42880/44000 | Loss: 2.9586 | LR: 8.27e-07 | 697.7ms/step | 28.7 samples/s +Step 42890/44000 | Loss: 2.9207 | LR: 8.12e-07 | 696.8ms/step | 28.7 samples/s +Step 42900/44000 | Loss: 2.9435 | LR: 7.98e-07 | 697.7ms/step | 28.7 samples/s +Step 42910/44000 | Loss: 2.9265 | LR: 7.83e-07 | 697.8ms/step | 28.7 samples/s +Step 42920/44000 | Loss: 2.8864 | LR: 7.69e-07 | 700.1ms/step | 28.6 samples/s +Step 42930/44000 | Loss: 2.9420 | LR: 7.55e-07 | 697.3ms/step | 28.7 samples/s +Step 42940/44000 | Loss: 2.9233 | LR: 7.41e-07 | 698.8ms/step | 28.6 samples/s +Step 42950/44000 | Loss: 2.8895 | LR: 7.27e-07 | 697.0ms/step | 28.7 samples/s +Step 42960/44000 | Loss: 2.9396 | LR: 7.13e-07 | 698.4ms/step | 28.6 samples/s +Step 42970/44000 | Loss: 2.9542 | LR: 6.99e-07 | 697.8ms/step | 28.7 samples/s +Step 42980/44000 | Loss: 2.9610 | LR: 6.86e-07 | 701.3ms/step | 28.5 samples/s +Step 42990/44000 | Loss: 2.9341 | LR: 6.72e-07 | 699.0ms/step | 28.6 samples/s +Step 43000/44000 | Loss: 2.9253 | LR: 6.59e-07 | 700.6ms/step | 28.5 samples/s +Validation at step 43000: Loss = 2.9547, Perplexity = 19.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-43000 +Step 43010/44000 | Loss: 2.9568 | LR: 6.46e-07 | 1576.9ms/step | 12.7 samples/s +Step 43020/44000 | Loss: 2.9387 | LR: 6.33e-07 | 699.4ms/step | 28.6 samples/s +Step 43030/44000 | Loss: 2.9229 | LR: 6.20e-07 | 696.6ms/step | 28.7 samples/s +Step 43040/44000 | Loss: 2.9457 | LR: 6.08e-07 | 693.2ms/step | 28.9 samples/s +Step 43050/44000 | Loss: 2.9351 | LR: 5.95e-07 | 701.5ms/step | 28.5 samples/s +Step 43060/44000 | Loss: 2.9129 | LR: 5.83e-07 | 696.7ms/step | 28.7 samples/s +Step 43070/44000 | Loss: 2.9363 | LR: 5.70e-07 | 695.4ms/step | 28.8 samples/s +Step 43080/44000 | Loss: 2.9179 | LR: 5.58e-07 | 697.3ms/step | 28.7 samples/s +Step 43090/44000 | Loss: 2.9367 | LR: 5.46e-07 | 697.8ms/step | 28.7 samples/s +Step 43100/44000 | Loss: 2.9697 | LR: 5.34e-07 | 698.1ms/step | 28.7 samples/s +Step 43110/44000 | Loss: 2.9805 | LR: 5.22e-07 | 695.7ms/step | 28.7 samples/s +Step 43120/44000 | Loss: 2.9160 | LR: 5.11e-07 | 698.8ms/step | 28.6 samples/s +Step 43130/44000 | Loss: 2.9295 | LR: 4.99e-07 | 694.1ms/step | 28.8 samples/s +Step 43140/44000 | Loss: 2.9714 | LR: 4.88e-07 | 697.0ms/step | 28.7 samples/s +Step 43150/44000 | Loss: 2.9709 | LR: 4.76e-07 | 696.1ms/step | 28.7 samples/s +Step 43160/44000 | Loss: 2.9357 | LR: 4.65e-07 | 699.8ms/step | 28.6 samples/s +Step 43170/44000 | Loss: 2.8967 | LR: 4.54e-07 | 695.0ms/step | 28.8 samples/s +Step 43180/44000 | Loss: 2.9601 | LR: 4.43e-07 | 699.4ms/step | 28.6 samples/s +Step 43190/44000 | Loss: 2.9159 | LR: 4.33e-07 | 698.0ms/step | 28.7 samples/s +Step 43200/44000 | Loss: 2.9337 | LR: 4.22e-07 | 697.9ms/step | 28.7 samples/s +Step 43210/44000 | Loss: 2.8902 | LR: 4.12e-07 | 698.2ms/step | 28.6 samples/s +Step 43220/44000 | Loss: 2.9470 | LR: 4.01e-07 | 694.9ms/step | 28.8 samples/s +Step 43230/44000 | Loss: 2.9548 | LR: 3.91e-07 | 696.6ms/step | 28.7 samples/s +Step 43240/44000 | Loss: 2.9647 | LR: 3.81e-07 | 699.8ms/step | 28.6 samples/s +Step 43250/44000 | Loss: 2.9439 | LR: 3.71e-07 | 694.3ms/step | 28.8 samples/s +Validation at step 43250: Loss = 2.9555, Perplexity = 19.21 +Step 43260/44000 | Loss: 2.9596 | LR: 3.61e-07 | 1262.2ms/step | 15.8 samples/s +Step 43270/44000 | Loss: 2.9543 | LR: 3.51e-07 | 697.8ms/step | 28.7 samples/s +Step 43280/44000 | Loss: 2.9423 | LR: 3.42e-07 | 695.6ms/step | 28.8 samples/s +Step 43290/44000 | Loss: 2.9182 | LR: 3.32e-07 | 694.6ms/step | 28.8 samples/s +Step 43300/44000 | Loss: 2.9246 | LR: 3.23e-07 | 693.7ms/step | 28.8 samples/s +Step 43310/44000 | Loss: 2.9265 | LR: 3.14e-07 | 694.0ms/step | 28.8 samples/s +Step 43320/44000 | Loss: 2.9340 | LR: 3.05e-07 | 694.1ms/step | 28.8 samples/s +Step 43330/44000 | Loss: 2.9623 | LR: 2.96e-07 | 696.3ms/step | 28.7 samples/s +Step 43340/44000 | Loss: 2.9469 | LR: 2.87e-07 | 696.2ms/step | 28.7 samples/s +Step 43350/44000 | Loss: 2.9736 | LR: 2.79e-07 | 697.4ms/step | 28.7 samples/s +Step 43360/44000 | Loss: 2.9627 | LR: 2.70e-07 | 697.2ms/step | 28.7 samples/s +Step 43370/44000 | Loss: 2.9100 | LR: 2.62e-07 | 694.8ms/step | 28.8 samples/s +Step 43380/44000 | Loss: 2.9201 | LR: 2.53e-07 | 694.3ms/step | 28.8 samples/s +Step 43390/44000 | Loss: 2.9401 | LR: 2.45e-07 | 695.9ms/step | 28.7 samples/s +Step 43400/44000 | Loss: 2.9352 | LR: 2.37e-07 | 700.6ms/step | 28.5 samples/s +Step 43410/44000 | Loss: 2.8894 | LR: 2.30e-07 | 696.8ms/step | 28.7 samples/s +Step 43420/44000 | Loss: 2.9475 | LR: 2.22e-07 | 694.0ms/step | 28.8 samples/s +Step 43430/44000 | Loss: 2.9709 | LR: 2.14e-07 | 696.6ms/step | 28.7 samples/s +Step 43440/44000 | Loss: 2.8855 | LR: 2.07e-07 | 697.2ms/step | 28.7 samples/s +Step 43450/44000 | Loss: 2.9045 | LR: 1.99e-07 | 699.3ms/step | 28.6 samples/s +Step 43460/44000 | Loss: 2.9217 | LR: 1.92e-07 | 696.0ms/step | 28.7 samples/s +Step 43470/44000 | Loss: 2.9269 | LR: 1.85e-07 | 697.1ms/step | 28.7 samples/s +Step 43480/44000 | Loss: 2.9178 | LR: 1.78e-07 | 697.0ms/step | 28.7 samples/s +Step 43490/44000 | Loss: 2.9171 | LR: 1.72e-07 | 696.9ms/step | 28.7 samples/s +Step 43500/44000 | Loss: 2.9526 | LR: 1.65e-07 | 695.6ms/step | 28.8 samples/s +Validation at step 43500: Loss = 2.9542, Perplexity = 19.19 +Step 43510/44000 | Loss: 2.9095 | LR: 1.58e-07 | 1261.7ms/step | 15.9 samples/s +Step 43520/44000 | Loss: 2.9176 | LR: 1.52e-07 | 697.0ms/step | 28.7 samples/s +Step 43530/44000 | Loss: 2.9188 | LR: 1.46e-07 | 699.9ms/step | 28.6 samples/s +Step 43540/44000 | Loss: 2.9541 | LR: 1.40e-07 | 697.1ms/step | 28.7 samples/s +Step 43550/44000 | Loss: 2.8985 | LR: 1.34e-07 | 695.2ms/step | 28.8 samples/s +Step 43560/44000 | Loss: 2.9833 | LR: 1.28e-07 | 696.3ms/step | 28.7 samples/s +Step 43570/44000 | Loss: 2.9023 | LR: 1.22e-07 | 695.7ms/step | 28.7 samples/s +Step 43580/44000 | Loss: 2.9110 | LR: 1.16e-07 | 697.8ms/step | 28.7 samples/s +Step 43590/44000 | Loss: 2.9283 | LR: 1.11e-07 | 693.4ms/step | 28.8 samples/s +Step 43600/44000 | Loss: 2.8974 | LR: 1.06e-07 | 697.4ms/step | 28.7 samples/s +Step 43610/44000 | Loss: 2.9501 | LR: 1.00e-07 | 697.3ms/step | 28.7 samples/s +Step 43620/44000 | Loss: 2.8983 | LR: 9.52e-08 | 697.8ms/step | 28.7 samples/s +Step 43630/44000 | Loss: 2.9272 | LR: 9.03e-08 | 697.9ms/step | 28.7 samples/s +Step 43640/44000 | Loss: 2.9239 | LR: 8.55e-08 | 700.8ms/step | 28.5 samples/s +Step 43650/44000 | Loss: 2.9106 | LR: 8.08e-08 | 696.6ms/step | 28.7 samples/s +Step 43660/44000 | Loss: 2.9397 | LR: 7.62e-08 | 696.5ms/step | 28.7 samples/s +Step 43670/44000 | Loss: 2.9075 | LR: 7.18e-08 | 696.5ms/step | 28.7 samples/s +Step 43680/44000 | Loss: 2.9048 | LR: 6.75e-08 | 699.3ms/step | 28.6 samples/s +Step 43690/44000 | Loss: 2.9095 | LR: 6.34e-08 | 698.1ms/step | 28.6 samples/s +Step 43700/44000 | Loss: 2.9733 | LR: 5.94e-08 | 695.5ms/step | 28.8 samples/s +Step 43710/44000 | Loss: 2.9172 | LR: 5.55e-08 | 695.3ms/step | 28.8 samples/s +Step 43720/44000 | Loss: 2.9406 | LR: 5.17e-08 | 695.9ms/step | 28.7 samples/s +Step 43730/44000 | Loss: 2.9367 | LR: 4.81e-08 | 696.6ms/step | 28.7 samples/s +Step 43740/44000 | Loss: 2.9731 | LR: 4.46e-08 | 698.6ms/step | 28.6 samples/s +Step 43750/44000 | Loss: 2.9379 | LR: 4.12e-08 | 697.1ms/step | 28.7 samples/s +Validation at step 43750: Loss = 2.9553, Perplexity = 19.21 +Step 43760/44000 | Loss: 2.9147 | LR: 3.80e-08 | 1267.6ms/step | 15.8 samples/s +Step 43770/44000 | Loss: 2.9257 | LR: 3.49e-08 | 695.7ms/step | 28.7 samples/s +Step 43780/44000 | Loss: 2.9774 | LR: 3.19e-08 | 694.4ms/step | 28.8 samples/s +Step 43790/44000 | Loss: 2.9115 | LR: 2.91e-08 | 698.2ms/step | 28.6 samples/s +Step 43800/44000 | Loss: 2.9386 | LR: 2.64e-08 | 694.3ms/step | 28.8 samples/s +Step 43810/44000 | Loss: 2.9443 | LR: 2.38e-08 | 698.7ms/step | 28.6 samples/s +Step 43820/44000 | Loss: 2.9843 | LR: 2.14e-08 | 698.2ms/step | 28.6 samples/s +Step 43830/44000 | Loss: 2.9580 | LR: 1.91e-08 | 695.3ms/step | 28.8 samples/s +Step 43840/44000 | Loss: 2.9372 | LR: 1.69e-08 | 697.8ms/step | 28.7 samples/s +Step 43850/44000 | Loss: 2.9220 | LR: 1.48e-08 | 695.8ms/step | 28.7 samples/s +Step 43860/44000 | Loss: 2.9128 | LR: 1.29e-08 | 695.8ms/step | 28.7 samples/s +Step 43870/44000 | Loss: 2.9425 | LR: 1.11e-08 | 696.3ms/step | 28.7 samples/s +Step 43880/44000 | Loss: 2.9296 | LR: 9.50e-09 | 696.9ms/step | 28.7 samples/s +Step 43890/44000 | Loss: 2.9346 | LR: 7.98e-09 | 699.1ms/step | 28.6 samples/s +Step 43900/44000 | Loss: 2.9285 | LR: 6.60e-09 | 695.3ms/step | 28.8 samples/s +Step 43910/44000 | Loss: 2.9283 | LR: 5.34e-09 | 695.4ms/step | 28.8 samples/s +Step 43920/44000 | Loss: 2.9457 | LR: 4.22e-09 | 696.8ms/step | 28.7 samples/s +Step 43930/44000 | Loss: 2.9363 | LR: 3.23e-09 | 700.0ms/step | 28.6 samples/s +Step 43940/44000 | Loss: 2.9478 | LR: 2.37e-09 | 696.6ms/step | 28.7 samples/s +Step 43950/44000 | Loss: 2.9273 | LR: 1.65e-09 | 699.1ms/step | 28.6 samples/s +Step 43960/44000 | Loss: 2.9127 | LR: 1.06e-09 | 697.6ms/step | 28.7 samples/s +Step 43970/44000 | Loss: 2.9904 | LR: 5.94e-10 | 696.6ms/step | 28.7 samples/s +Step 43980/44000 | Loss: 2.9035 | LR: 2.64e-10 | 698.6ms/step | 28.6 samples/s +Step 43990/44000 | Loss: 2.9227 | LR: 6.60e-11 | 696.8ms/step | 28.7 samples/s +Step 44000/44000 | Loss: 2.9412 | LR: 0.00e+00 | 696.1ms/step | 28.7 samples/s +Validation at step 44000: Loss = 2.9549, Perplexity = 19.20 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-44000 +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48/run_20260821_050923/checkpoint-44000 +wandb: updating run metadata +wandb: uploading config.yaml +wandb: uploading config.yaml; uploading output.log +wandb: +wandb: Run history: +wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/loss █▇▅▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/perplexity █▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁ +wandb: speed/samples_per_sec ██████████████████▁█████████████████████ +wandb: speed/steps_per_sec ████████▁███████████████████████████████ +wandb: +4 ... +wandb: +wandb: Run summary: +wandb: eval/dense_interval_active 0 +wandb: eval/dense_interval_end -1 +wandb: eval/dense_interval_start -1 +wandb: eval/loss 2.95487 +wandb: eval/perplexity 19.19915 +wandb: eval/uses_dense_attention 0 +wandb: eval/uses_sparse_attention 1 +wandb: speed/ms_per_step 696.08932 +wandb: speed/samples_per_sec 28.73195 +wandb: speed/steps_per_sec 1.4366 +wandb: +4 ... +wandb: +wandb: 🚀 View run 365m mixed 8k seed48 h100 ddp2 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/noud30xe +wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: /mloscratch/homes/navasard/wandb/wandb/run-20260821_051200-noud30xe/logs +Training complete! diff --git a/epfl_final/train_logs/speedprobe_seed48_mixed_ddp4/mixed.20260821T044419Z.log b/epfl_final/train_logs/speedprobe_seed48_mixed_ddp4/mixed.20260821T044419Z.log new file mode 100644 index 0000000000000000000000000000000000000000..e5a6473c496be0fa0f43bf76a19da383f3089a7d --- /dev/null +++ b/epfl_final/train_logs/speedprobe_seed48_mixed_ddp4/mixed.20260821T044419Z.log @@ -0,0 +1,117 @@ +W0821 04:44:22.014000 61278 site-packages/torch/distributed/run.py:851] +W0821 04:44:22.014000 61278 site-packages/torch/distributed/run.py:851] ***************************************** +W0821 04:44:22.014000 61278 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0821 04:44:22.014000 61278 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_speedprobe_seed48/run_20260821_044427 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_speedprobe_seed48/run_20260821_044427", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 50, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 48, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": false, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m mixed 8k speedprobe_seed48 ddp2 h100-resume", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "mixed", + "multiseed", + "speedprobe_seed48", + "ddp2", + "h100-resume" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_speedprobe_seed48_ddp4", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": [ + 63, + 47, + 31, + 21, + 11 + ], + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Initializing model from scratch (random weights)... +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False +W0821 04:48:58.112000 61278 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers +W0821 04:48:58.114000 61278 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 61346 closing signal SIGTERM +W0821 04:48:58.115000 61278 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 61347 closing signal SIGTERM +W0821 04:48:58.115000 61278 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 61348 closing signal SIGTERM +W0821 04:48:58.115000 61278 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 61349 closing signal SIGTERM +Traceback (most recent call last): + File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in + sys.exit(main()) + ~~~~^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper + return f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main + run(args) + ~~~^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run + elastic_launch( + ~~~~~~~~~~~~~~~ + config=config, + ~~~~~~~~~~~~~~ + entrypoint=cmd, + ~~~~~~~~~~~~~~~ + )(*cmd_args) + ~^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent + result = agent.run() + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper + result = f(*args, **kwargs) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run + result = self._invoke_run(role) + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run + time.sleep(monitor_interval) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ + File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 61278 got signal: 15 diff --git a/epfl_final/train_logs/speedprobe_seed48_mixed_ddp4/mixed.20260821T044917Z.log b/epfl_final/train_logs/speedprobe_seed48_mixed_ddp4/mixed.20260821T044917Z.log new file mode 100644 index 0000000000000000000000000000000000000000..fe473f4f75523b19d941ee10d5f14011f6c04a55 --- /dev/null +++ b/epfl_final/train_logs/speedprobe_seed48_mixed_ddp4/mixed.20260821T044917Z.log @@ -0,0 +1,202 @@ +W0821 04:49:20.757000 61575 site-packages/torch/distributed/run.py:851] +W0821 04:49:20.757000 61575 site-packages/torch/distributed/run.py:851] ***************************************** +W0821 04:49:20.757000 61575 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0821 04:49:20.757000 61575 site-packages/torch/distributed/run.py:851] ***************************************** +New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_speedprobe_seed48/run_20260821_044926 +Running config: +{ + "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", + "model_dtype": "bfloat16", + "init_from_scratch": true, + "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", + "val_path": null, + "max_seq_length": 8192, + "num_workers": 8, + "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_speedprobe_seed48/run_20260821_044926", + "resume_from_checkpoint": null, + "force_new_run": true, + "num_epochs": 1, + "max_steps": 50, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 5, + "learning_rate": 0.0005, + "weight_decay": 0.01, + "warmup_steps": 750, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "save_steps": 1000, + "eval_steps": 250, + "eval_batches": 50, + "logging_steps": 10, + "seed": 48, + "verbose": true, + "checkpoint_activations": false, + "strategy": "ddp", + "mixed_precision": "bf16", + "wandb_enabled": false, + "wandb_project": "stochastic_sparsity", + "wandb_name": "365m mixed 8k speedprobe_seed48 ddp2 h100-resume", + "wandb_tags": [ + "qwen3", + "365m", + "8k", + "mixed", + "multiseed", + "speedprobe_seed48", + "ddp2", + "h100-resume" + ], + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 6, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "head_dim": null, + "attention_dropout": null, + "rope_train_position_scale_set": null, + "attention_type": "sparse_stochastic", + "sparse_block_size": 128, + "sparse_num_random_blocks": 31, + "sparse_mask_pool_size": 10000, + "sparse_val_dense_compare": false, + "sparse_layer_indices": null, + "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed", + "sparse_mask_seed": 0, + "sparse_mixed_nrb_set": [ + 63, + 47, + 31, + 21, + 11 + ], + "dense_mode": false, + "dense_step_intervals": null +} +Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... +Checkpoint activations: False +Initializing model from scratch (random weights)... +Checkpoint activations: False +Checkpoint activations: False +Checkpoint activations: False + + +================================================================================ +MODEL ARCHITECTURE +================================================================================ +Model config: + vocab_size: 151936 + hidden_size: 768 + num_hidden_layers: 28 + num_attention_heads: 6 + num_key_value_heads: 2 + head_dim: 128 + intermediate_size: 3072 + max_position_embeddings: 32768 + rope_theta: 1000000 + rope_train_position_scale_set: None + attention_type: sparse_stochastic + sparse_block_size: 128 + sparse_num_random_blocks: 31 + sparse_mask_pool_size: 10000 + sparse_max_seq_length: 8192 + +Parameter count: + Total parameters: 358,958,848 + Trainable parameters: 358,958,848 + Non-trainable parameters: 0 + Model size (BF16): 0.72 GB + +-------------------------------------------------------------------------------- +LAYER-BY-LAYER SUMMARY +-------------------------------------------------------------------------------- +/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + +SOLUTION: Use torch.compile(flex_attention)(...) + +If you want to debug your score_mod/mask_mod, you can set: +torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + +This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( +================================================================================================================================================================ +Layer (type:depth-idx) Input Shape Output Shape Param # Trainable +================================================================================================================================================================ +Qwen3ForCausalLM [1, 8192] -- -- True +├─Qwen3Model: 1-1 -- [1, 8192, 768] -- True +│ └─Embedding: 2-1 [1, 8192] [1, 8192, 768] 116,686,848 True +│ └─ModuleList: 2-2 -- -- -- True +│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 768] [1, 8192, 768] 8,652,544 True +│ └─Qwen3RMSNorm: 2-3 [1, 8192, 768] [1, 8192, 768] 768 True +├─Linear: 1-2 [1, 8192, 768] [1, 8192, 151936] 116,686,848 True +================================================================================================================================================================ +Total params: 475,645,696 +Trainable params: 475,645,696 +Non-trainable params: 0 +Total mult-adds (Units.MEGABYTES): 475.65 +================================================================================================================================================================ +Input size (MB): 0.07 +Forward/backward pass size (MB): 15598.62 +Params size (MB): 951.29 +Estimated Total Size (MB): 16549.97 +================================================================================================================================================================ + NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M) +================================================================================ + +Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Using pre-tokenized dataset (no runtime tokenization) +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) +Dataset split: 3027764 train, 159356 validation +Training examples (packed, train split): 3027764 +Validation examples (packed, val split): 159356 +Per-device batch size: 1 +World size (GPUs): 4 +Gradient accumulation steps: 5 +Effective batch size (seqs/step): 20 +Sequence length: 8192 +Tokens per optimizer step: 163,840 +Micro-batches per epoch (per rank): 756941 +Optimizer steps per epoch: 151388 +Total optimizer steps: 50 (capped by max_steps; 0.00 epochs) +Starting training... +Switching attention mode for step 1: sparse_stochastic (configured sparse attention) +Step 10/50 | Loss: 12.0967 | LR: 6.67e-06 | 6214.9ms/step | 3.2 samples/s +Step 20/50 | Loss: 11.9985 | LR: 1.33e-05 | 402.3ms/step | 49.7 samples/s +Step 30/50 | Loss: 11.6505 | LR: 2.00e-05 | 402.9ms/step | 49.6 samples/s +Step 40/50 | Loss: 11.0928 | LR: 2.67e-05 | 402.2ms/step | 49.7 samples/s +Step 50/50 | Loss: 10.7639 | LR: 3.33e-05 | 403.0ms/step | 49.6 samples/s +Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_speedprobe_seed48/run_20260821_044926/checkpoint-50 +Training complete!