EPFL ALiBi and training provenance 551-600 of 1184

.gitattributes

@@ -259,3 +259,4 @@ epfl_final/wandb/run-20260615_161335-aqqleyxv/run-aqqleyxv.wandb filter=lfs diff
 epfl_final/wandb/run-20260615_224051-7jul6zqc/run-7jul6zqc.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/wandb/run-20260616_012002-apvt4901/run-apvt4901.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/wandb/run-20260618_175952-33yw8345/run-33yw8345.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/log_files/eval_sparse_64.log filter=lfs diff=lfs merge=lfs -text

epfl_final/log_files/1-2b_dense_cont_46k.log

@@ -0,0 +1,351 @@
+nohup: ignoring input
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+New run: Checkpoints will be saved to ./checkpoints/run_20260529_183242
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260529_183242",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000",
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 48k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000
+Checkpoint activations: False
+Checkpoint activations: False
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 1536]           [2, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [2, 8192, 1536]           [2, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (G): 2.87
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 52437.19
+Params size (MB): 2871.45
+Estimated Total Size (MB): 55308.77
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        3
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504627
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000...
+Resumed from checkpoint at step 46000 (epoch 0, micro_step 460000)
+Resuming training from step 46000
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 3yae6xjj
+wandb: Tracking run with wandb version 0.27.0
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260529_185135-3yae6xjj
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense continued 48k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/3yae6xjj
+Starting training...
+Validation at step 46000: Loss = 2.5221, Perplexity = 12.46
+Step 46010/50462 | Loss: 2.5136 | LR: 9.83e-06 | 82727.5ms/step | 0.7 samples/s
+Step 46020/50462 | Loss: 2.4987 | LR: 9.79e-06 | 5073.9ms/step | 11.8 samples/s
+Step 46030/50462 | Loss: 2.5210 | LR: 9.74e-06 | 5090.5ms/step | 11.8 samples/s
+Step 46040/50462 | Loss: 2.5202 | LR: 9.70e-06 | 5078.9ms/step | 11.8 samples/s
+Step 46050/50462 | Loss: 2.5031 | LR: 9.65e-06 | 5081.8ms/step | 11.8 samples/s
+Step 46060/50462 | Loss: 2.4972 | LR: 9.61e-06 | 5097.5ms/step | 11.8 samples/s
+Step 46070/50462 | Loss: 2.5165 | LR: 9.57e-06 | 5084.9ms/step | 11.8 samples/s
+Step 46080/50462 | Loss: 2.5113 | LR: 9.52e-06 | 5091.8ms/step | 11.8 samples/s
+Step 46090/50462 | Loss: 2.4799 | LR: 9.48e-06 | 5234.7ms/step | 11.5 samples/s
+Step 46100/50462 | Loss: 2.5059 | LR: 9.44e-06 | 5098.0ms/step | 11.8 samples/s
+Step 46110/50462 | Loss: 2.5016 | LR: 9.40e-06 | 5101.6ms/step | 11.8 samples/s
+Step 46120/50462 | Loss: 2.4968 | LR: 9.35e-06 | 5074.7ms/step | 11.8 samples/s
+Step 46130/50462 | Loss: 2.4982 | LR: 9.31e-06 | 5087.7ms/step | 11.8 samples/s
+Step 46140/50462 | Loss: 2.4984 | LR: 9.27e-06 | 5089.6ms/step | 11.8 samples/s
+Step 46150/50462 | Loss: 2.4967 | LR: 9.22e-06 | 5108.0ms/step | 11.7 samples/s
+Step 46160/50462 | Loss: 2.5143 | LR: 9.18e-06 | 5084.8ms/step | 11.8 samples/s
+Step 46170/50462 | Loss: 2.5149 | LR: 9.14e-06 | 5064.3ms/step | 11.8 samples/s
+Step 46180/50462 | Loss: 2.4845 | LR: 9.10e-06 | 5102.9ms/step | 11.8 samples/s
+Step 46190/50462 | Loss: 2.4850 | LR: 9.06e-06 | 5080.9ms/step | 11.8 samples/s
+Step 46200/50462 | Loss: 2.4926 | LR: 9.01e-06 | 5067.8ms/step | 11.8 samples/s
+Step 46210/50462 | Loss: 2.5091 | LR: 8.97e-06 | 5069.3ms/step | 11.8 samples/s
+Step 46220/50462 | Loss: 2.5119 | LR: 8.93e-06 | 5073.3ms/step | 11.8 samples/s
+Step 46230/50462 | Loss: 2.5180 | LR: 8.89e-06 | 5082.2ms/step | 11.8 samples/s
+Step 46240/50462 | Loss: 2.5267 | LR: 8.85e-06 | 5088.3ms/step | 11.8 samples/s
+Step 46250/50462 | Loss: 2.5245 | LR: 8.80e-06 | 5070.0ms/step | 11.8 samples/s
+Validation at step 46250: Loss = 2.5160, Perplexity = 12.38
+Step 46260/50462 | Loss: 2.5244 | LR: 8.76e-06 | 7621.7ms/step | 7.9 samples/s
+Step 46270/50462 | Loss: 2.5108 | LR: 8.72e-06 | 5100.0ms/step | 11.8 samples/s
+Step 46280/50462 | Loss: 2.5424 | LR: 8.68e-06 | 5149.3ms/step | 11.7 samples/s
+Step 46290/50462 | Loss: 2.5291 | LR: 8.64e-06 | 5099.2ms/step | 11.8 samples/s
+Step 46300/50462 | Loss: 2.4995 | LR: 8.60e-06 | 5088.5ms/step | 11.8 samples/s
+Step 46310/50462 | Loss: 2.5071 | LR: 8.56e-06 | 5094.1ms/step | 11.8 samples/s
+Step 46320/50462 | Loss: 2.5327 | LR: 8.52e-06 | 5095.4ms/step | 11.8 samples/s
+Step 46330/50462 | Loss: 2.5121 | LR: 8.47e-06 | 5088.7ms/step | 11.8 samples/s
+Step 46340/50462 | Loss: 2.5119 | LR: 8.43e-06 | 5093.9ms/step | 11.8 samples/s
+Step 46350/50462 | Loss: 2.5328 | LR: 8.39e-06 | 5093.1ms/step | 11.8 samples/s
+Step 46360/50462 | Loss: 2.4956 | LR: 8.35e-06 | 5093.4ms/step | 11.8 samples/s
+Step 46370/50462 | Loss: 2.4811 | LR: 8.31e-06 | 5080.5ms/step | 11.8 samples/s
+Step 46380/50462 | Loss: 2.5062 | LR: 8.27e-06 | 5202.4ms/step | 11.5 samples/s
+Step 46390/50462 | Loss: 2.4951 | LR: 8.23e-06 | 5087.3ms/step | 11.8 samples/s
+Step 46400/50462 | Loss: 2.4869 | LR: 8.19e-06 | 5076.9ms/step | 11.8 samples/s
+Step 46410/50462 | Loss: 2.4994 | LR: 8.15e-06 | 5061.7ms/step | 11.9 samples/s
+Step 46420/50462 | Loss: 2.4990 | LR: 8.11e-06 | 5073.7ms/step | 11.8 samples/s
+Step 46430/50462 | Loss: 2.5198 | LR: 8.07e-06 | 5074.7ms/step | 11.8 samples/s
+Step 46440/50462 | Loss: 2.5022 | LR: 8.03e-06 | 5069.0ms/step | 11.8 samples/s
+Step 46450/50462 | Loss: 2.5283 | LR: 7.99e-06 | 5074.0ms/step | 11.8 samples/s
+Step 46460/50462 | Loss: 2.5065 | LR: 7.95e-06 | 5089.8ms/step | 11.8 samples/s
+Step 46470/50462 | Loss: 2.4978 | LR: 7.91e-06 | 5072.7ms/step | 11.8 samples/s
+Step 46480/50462 | Loss: 2.4927 | LR: 7.87e-06 | 5082.2ms/step | 11.8 samples/s
+Step 46490/50462 | Loss: 2.4891 | LR: 7.83e-06 | 5074.3ms/step | 11.8 samples/s
+Step 46500/50462 | Loss: 2.5059 | LR: 7.80e-06 | 5069.6ms/step | 11.8 samples/s
+Validation at step 46500: Loss = 2.5156, Perplexity = 12.37
+Step 46510/50462 | Loss: 2.5005 | LR: 7.76e-06 | 7630.4ms/step | 7.9 samples/s
+Step 46520/50462 | Loss: 2.5238 | LR: 7.72e-06 | 5095.5ms/step | 11.8 samples/s
+Step 46530/50462 | Loss: 2.5039 | LR: 7.68e-06 | 5091.5ms/step | 11.8 samples/s
+Step 46540/50462 | Loss: 2.5166 | LR: 7.64e-06 | 5093.1ms/step | 11.8 samples/s
+Step 46550/50462 | Loss: 2.4782 | LR: 7.60e-06 | 5097.4ms/step | 11.8 samples/s
+Step 46560/50462 | Loss: 2.4951 | LR: 7.56e-06 | 5145.5ms/step | 11.7 samples/s
+Step 46570/50462 | Loss: 2.5057 | LR: 7.52e-06 | 5095.2ms/step | 11.8 samples/s
+Step 46580/50462 | Loss: 2.4974 | LR: 7.49e-06 | 5092.4ms/step | 11.8 samples/s
+Step 46590/50462 | Loss: 2.4902 | LR: 7.45e-06 | 5109.4ms/step | 11.7 samples/s
+Step 46600/50462 | Loss: 2.5052 | LR: 7.41e-06 | 5093.6ms/step | 11.8 samples/s
+Step 46610/50462 | Loss: 2.5036 | LR: 7.37e-06 | 5092.3ms/step | 11.8 samples/s
+Step 46620/50462 | Loss: 2.4863 | LR: 7.33e-06 | 5083.4ms/step | 11.8 samples/s
+Step 46630/50462 | Loss: 2.5004 | LR: 7.29e-06 | 5092.8ms/step | 11.8 samples/s
+Step 46640/50462 | Loss: 2.5125 | LR: 7.26e-06 | 5085.1ms/step | 11.8 samples/s
+Step 46650/50462 | Loss: 2.5046 | LR: 7.22e-06 | 5094.1ms/step | 11.8 samples/s
+Step 46660/50462 | Loss: 2.4945 | LR: 7.18e-06 | 5086.9ms/step | 11.8 samples/s
+Step 46670/50462 | Loss: 2.5093 | LR: 7.14e-06 | 5197.9ms/step | 11.5 samples/s
+Step 46680/50462 | Loss: 2.5178 | LR: 7.11e-06 | 5093.8ms/step | 11.8 samples/s
+Step 46690/50462 | Loss: 2.5347 | LR: 7.07e-06 | 5074.4ms/step | 11.8 samples/s
+Step 46700/50462 | Loss: 2.4950 | LR: 7.03e-06 | 5099.1ms/step | 11.8 samples/s
+Step 46710/50462 | Loss: 2.5279 | LR: 6.99e-06 | 5099.5ms/step | 11.8 samples/s
+Step 46720/50462 | Loss: 2.5116 | LR: 6.96e-06 | 5085.8ms/step | 11.8 samples/s
+Step 46730/50462 | Loss: 2.4914 | LR: 6.92e-06 | 5097.2ms/step | 11.8 samples/s
+Step 46740/50462 | Loss: 2.5180 | LR: 6.88e-06 | 5074.8ms/step | 11.8 samples/s
+Step 46750/50462 | Loss: 2.5009 | LR: 6.85e-06 | 5085.5ms/step | 11.8 samples/s
+Validation at step 46750: Loss = 2.5154, Perplexity = 12.37
+Step 46760/50462 | Loss: 2.5153 | LR: 6.81e-06 | 7626.9ms/step | 7.9 samples/s
+Step 46770/50462 | Loss: 2.5049 | LR: 6.77e-06 | 5095.0ms/step | 11.8 samples/s
+Step 46780/50462 | Loss: 2.5303 | LR: 6.74e-06 | 5084.6ms/step | 11.8 samples/s
+Step 46790/50462 | Loss: 2.4876 | LR: 6.70e-06 | 5082.0ms/step | 11.8 samples/s
+Step 46800/50462 | Loss: 2.5255 | LR: 6.66e-06 | 5090.2ms/step | 11.8 samples/s
+Step 46810/50462 | Loss: 2.5217 | LR: 6.63e-06 | 5101.5ms/step | 11.8 samples/s
+Step 46820/50462 | Loss: 2.4851 | LR: 6.59e-06 | 5087.0ms/step | 11.8 samples/s
+Step 46830/50462 | Loss: 2.4832 | LR: 6.56e-06 | 5085.6ms/step | 11.8 samples/s
+Step 46840/50462 | Loss: 2.5086 | LR: 6.52e-06 | 5138.2ms/step | 11.7 samples/s
+Step 46850/50462 | Loss: 2.5047 | LR: 6.48e-06 | 5085.6ms/step | 11.8 samples/s
+Step 46860/50462 | Loss: 2.5043 | LR: 6.45e-06 | 5081.3ms/step | 11.8 samples/s
+Step 46870/50462 | Loss: 2.4625 | LR: 6.41e-06 | 5097.4ms/step | 11.8 samples/s
+Step 46880/50462 | Loss: 2.4979 | LR: 6.38e-06 | 5093.3ms/step | 11.8 samples/s
+Step 46890/50462 | Loss: 2.5039 | LR: 6.34e-06 | 5102.0ms/step | 11.8 samples/s
+Step 46900/50462 | Loss: 2.4961 | LR: 6.31e-06 | 5105.7ms/step | 11.8 samples/s
+Step 46910/50462 | Loss: 2.4893 | LR: 6.27e-06 | 5087.8ms/step | 11.8 samples/s
+Step 46920/50462 | Loss: 2.5272 | LR: 6.24e-06 | 5093.6ms/step | 11.8 samples/s
+Step 46930/50462 | Loss: 2.5035 | LR: 6.20e-06 | 5082.2ms/step | 11.8 samples/s
+Step 46940/50462 | Loss: 2.5054 | LR: 6.17e-06 | 5093.6ms/step | 11.8 samples/s
+Step 46950/50462 | Loss: 2.4885 | LR: 6.13e-06 | 5082.0ms/step | 11.8 samples/s
+Step 46960/50462 | Loss: 2.5119 | LR: 6.10e-06 | 5136.6ms/step | 11.7 samples/s
+Step 46970/50462 | Loss: 2.5154 | LR: 6.06e-06 | 5155.0ms/step | 11.6 samples/s
+Step 46980/50462 | Loss: 2.5147 | LR: 6.03e-06 | 5107.1ms/step | 11.7 samples/s
+Step 46990/50462 | Loss: 2.5355 | LR: 5.99e-06 | 5113.6ms/step | 11.7 samples/s
+Step 47000/50462 | Loss: 2.5152 | LR: 5.96e-06 | 5107.9ms/step | 11.7 samples/s
+Validation at step 47000: Loss = 2.5153, Perplexity = 12.37
+Saved checkpoint to checkpoints/run_20260529_183242/checkpoint-47000
+Step 47010/50462 | Loss: 2.5271 | LR: 5.93e-06 | 8643.4ms/step | 6.9 samples/s
+Step 47020/50462 | Loss: 2.5038 | LR: 5.89e-06 | 5083.1ms/step | 11.8 samples/s
+Step 47030/50462 | Loss: 2.5237 | LR: 5.86e-06 | 5090.5ms/step | 11.8 samples/s
+Step 47040/50462 | Loss: 2.4690 | LR: 5.82e-06 | 5082.3ms/step | 11.8 samples/s
+Step 47050/50462 | Loss: 2.5055 | LR: 5.79e-06 | 5078.2ms/step | 11.8 samples/s
+Step 47060/50462 | Loss: 2.5038 | LR: 5.76e-06 | 5087.9ms/step | 11.8 samples/s
+Step 47070/50462 | Loss: 2.4966 | LR: 5.72e-06 | 5079.2ms/step | 11.8 samples/s
+Step 47080/50462 | Loss: 2.5046 | LR: 5.69e-06 | 5072.9ms/step | 11.8 samples/s
+Step 47090/50462 | Loss: 2.4994 | LR: 5.65e-06 | 5088.7ms/step | 11.8 samples/s
+Step 47100/50462 | Loss: 2.5033 | LR: 5.62e-06 | 5077.8ms/step | 11.8 samples/s
+Step 47110/50462 | Loss: 2.5071 | LR: 5.59e-06 | 5093.1ms/step | 11.8 samples/s
+Step 47120/50462 | Loss: 2.4575 | LR: 5.56e-06 | 5138.0ms/step | 11.7 samples/s
+Step 47130/50462 | Loss: 2.5043 | LR: 5.52e-06 | 5102.5ms/step | 11.8 samples/s
+Step 47140/50462 | Loss: 2.5167 | LR: 5.49e-06 | 5110.2ms/step | 11.7 samples/s
+Step 47150/50462 | Loss: 2.4729 | LR: 5.46e-06 | 5103.3ms/step | 11.8 samples/s
+Step 47160/50462 | Loss: 2.5147 | LR: 5.42e-06 | 5100.1ms/step | 11.8 samples/s
+Step 47170/50462 | Loss: 2.4977 | LR: 5.39e-06 | 5108.1ms/step | 11.7 samples/s
+Step 47180/50462 | Loss: 2.5073 | LR: 5.36e-06 | 5101.7ms/step | 11.8 samples/s
+Step 47190/50462 | Loss: 2.5018 | LR: 5.33e-06 | 5114.6ms/step | 11.7 samples/s
+Step 47200/50462 | Loss: 2.4998 | LR: 5.29e-06 | 5123.2ms/step | 11.7 samples/s
+Step 47210/50462 | Loss: 2.4737 | LR: 5.26e-06 | 5122.3ms/step | 11.7 samples/s
+Step 47220/50462 | Loss: 2.5144 | LR: 5.23e-06 | 5109.2ms/step | 11.7 samples/s
+Step 47230/50462 | Loss: 2.5125 | LR: 5.20e-06 | 5098.5ms/step | 11.8 samples/s
+Step 47240/50462 | Loss: 2.4808 | LR: 5.16e-06 | 5102.6ms/step | 11.8 samples/s
+Step 47250/50462 | Loss: 2.5115 | LR: 5.13e-06 | 5117.1ms/step | 11.7 samples/s
+Validation at step 47250: Loss = 2.5152, Perplexity = 12.37
+Step 47260/50462 | Loss: 2.5060 | LR: 5.10e-06 | 7731.6ms/step | 7.8 samples/s
+Step 47270/50462 | Loss: 2.5177 | LR: 5.07e-06 | 5077.9ms/step | 11.8 samples/s
+Step 47280/50462 | Loss: 2.4915 | LR: 5.04e-06 | 5083.4ms/step | 11.8 samples/s
+Step 47290/50462 | Loss: 2.5194 | LR: 5.01e-06 | 5090.9ms/step | 11.8 samples/s
+Step 47300/50462 | Loss: 2.5177 | LR: 4.97e-06 | 5064.7ms/step | 11.8 samples/s
+Step 47310/50462 | Loss: 2.4795 | LR: 4.94e-06 | 5069.3ms/step | 11.8 samples/s
+Step 47320/50462 | Loss: 2.4967 | LR: 4.91e-06 | 5072.6ms/step | 11.8 samples/s
+Step 47330/50462 | Loss: 2.4998 | LR: 4.88e-06 | 5065.2ms/step | 11.8 samples/s
+Step 47340/50462 | Loss: 2.4647 | LR: 4.85e-06 | 5068.4ms/step | 11.8 samples/s
+Step 47350/50462 | Loss: 2.5123 | LR: 4.82e-06 | 5072.7ms/step | 11.8 samples/s
+Step 47360/50462 | Loss: 2.4828 | LR: 4.79e-06 | 5087.6ms/step | 11.8 samples/s
+Step 47370/50462 | Loss: 2.5210 | LR: 4.76e-06 | 5065.7ms/step | 11.8 samples/s
+Step 47380/50462 | Loss: 2.5134 | LR: 4.73e-06 | 5062.9ms/step | 11.9 samples/s
+Step 47390/50462 | Loss: 2.4962 | LR: 4.70e-06 | 5072.8ms/step | 11.8 samples/s
+Step 47400/50462 | Loss: 2.4993 | LR: 4.67e-06 | 5133.0ms/step | 11.7 samples/s
+Step 47410/50462 | Loss: 2.5106 | LR: 4.64e-06 | 5110.1ms/step | 11.7 samples/s
+Step 47420/50462 | Loss: 2.4624 | LR: 4.61e-06 | 5112.3ms/step | 11.7 samples/s
+Step 47430/50462 | Loss: 2.4903 | LR: 4.58e-06 | 5106.6ms/step | 11.7 samples/s
+Step 47440/50462 | Loss: 2.4991 | LR: 4.55e-06 | 5109.8ms/step | 11.7 samples/s
+Step 47450/50462 | Loss: 2.4738 | LR: 4.52e-06 | 5118.8ms/step | 11.7 samples/s
+Step 47460/50462 | Loss: 2.5186 | LR: 4.49e-06 | 5111.5ms/step | 11.7 samples/s
+Step 47470/50462 | Loss: 2.5164 | LR: 4.46e-06 | 5107.9ms/step | 11.7 samples/s
+Step 47480/50462 | Loss: 2.5151 | LR: 4.43e-06 | 5114.2ms/step | 11.7 samples/s
+Step 47490/50462 | Loss: 2.5078 | LR: 4.40e-06 | 5105.7ms/step | 11.8 samples/s
+Step 47500/50462 | Loss: 2.4992 | LR: 4.37e-06 | 5118.2ms/step | 11.7 samples/s
+Validation at step 47500: Loss = 2.5152, Perplexity = 12.37
+Step 47510/50462 | Loss: 2.4700 | LR: 4.34e-06 | 7613.1ms/step | 7.9 samples/s
+Step 47520/50462 | Loss: 2.5181 | LR: 4.31e-06 | 5109.3ms/step | 11.7 samples/s
+Step 47530/50462 | Loss: 2.5134 | LR: 4.28e-06 | 5083.8ms/step | 11.8 samples/s
+Step 47540/50462 | Loss: 2.4818 | LR: 4.25e-06 | 5094.2ms/step | 11.8 samples/s
+Step 47550/50462 | Loss: 2.5112 | LR: 4.22e-06 | 5192.0ms/step | 11.6 samples/s
+Step 47560/50462 | Loss: 2.4964 | LR: 4.19e-06 | 5078.0ms/step | 11.8 samples/s
+Step 47570/50462 | Loss: 2.5066 | LR: 4.16e-06 | 5091.1ms/step | 11.8 samples/s
+Step 47580/50462 | Loss: 2.5418 | LR: 4.13e-06 | 5111.1ms/step | 11.7 samples/s
+Step 47590/50462 | Loss: 2.5126 | LR: 4.11e-06 | 5099.3ms/step | 11.8 samples/s
+Step 47600/50462 | Loss: 2.5036 | LR: 4.08e-06 | 5093.2ms/step | 11.8 samples/s
+Step 47610/50462 | Loss: 2.4834 | LR: 4.05e-06 | 5096.8ms/step | 11.8 samples/s
+Step 47620/50462 | Loss: 2.4971 | LR: 4.02e-06 | 5100.0ms/step | 11.8 samples/s
+Step 47630/50462 | Loss: 2.5243 | LR: 3.99e-06 | 5092.0ms/step | 11.8 samples/s
+Step 47640/50462 | Loss: 2.4794 | LR: 3.97e-06 | 5100.3ms/step | 11.8 samples/s

epfl_final/log_files/1-2b_dense_cont_48k.log

@@ -0,0 +1,470 @@
+nohup: ignoring input
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+New run: Checkpoints will be saved to ./checkpoints/run_20260529_182827
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260529_182827",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-48000",
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 48k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-48000
+Checkpoint activations: FalseCheckpoint activations: False
+
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 1536]           [2, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [2, 8192, 1536]           [2, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (G): 2.87
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 52437.19
+Params size (MB): 2871.45
+Estimated Total Size (MB): 55308.77
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        3
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504627
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-48000...
+Resumed from checkpoint at step 48000 (epoch 0, micro_step 480000)
+Resuming training from step 48000
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run wbfo0o07
+wandb: Tracking run with wandb version 0.27.0
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260529_184637-wbfo0o07
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense continued 48k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/wbfo0o07
+Starting training...
+Validation at step 48000: Loss = 2.5221, Perplexity = 12.45
+Step 48010/50462 | Loss: 2.5176 | LR: 3.00e-06 | 79833.7ms/step | 0.8 samples/s
+Step 48020/50462 | Loss: 2.5457 | LR: 2.97e-06 | 5205.0ms/step | 11.5 samples/s
+Step 48030/50462 | Loss: 2.4873 | LR: 2.95e-06 | 5232.3ms/step | 11.5 samples/s
+Step 48040/50462 | Loss: 2.5006 | LR: 2.92e-06 | 5301.5ms/step | 11.3 samples/s
+Step 48050/50462 | Loss: 2.4934 | LR: 2.90e-06 | 5246.8ms/step | 11.4 samples/s
+Step 48060/50462 | Loss: 2.5102 | LR: 2.87e-06 | 5251.1ms/step | 11.4 samples/s
+Step 48070/50462 | Loss: 2.5122 | LR: 2.85e-06 | 5248.6ms/step | 11.4 samples/s
+Step 48080/50462 | Loss: 2.5359 | LR: 2.83e-06 | 5249.9ms/step | 11.4 samples/s
+Step 48090/50462 | Loss: 2.5121 | LR: 2.80e-06 | 5347.1ms/step | 11.2 samples/s
+Step 48100/50462 | Loss: 2.5147 | LR: 2.78e-06 | 5242.9ms/step | 11.4 samples/s
+Step 48110/50462 | Loss: 2.5043 | LR: 2.76e-06 | 5250.5ms/step | 11.4 samples/s
+Step 48120/50462 | Loss: 2.5025 | LR: 2.73e-06 | 5252.1ms/step | 11.4 samples/s
+Step 48130/50462 | Loss: 2.5032 | LR: 2.71e-06 | 5251.2ms/step | 11.4 samples/s
+Step 48140/50462 | Loss: 2.4849 | LR: 2.69e-06 | 5251.6ms/step | 11.4 samples/s
+Step 48150/50462 | Loss: 2.4982 | LR: 2.66e-06 | 5254.7ms/step | 11.4 samples/s
+Step 48160/50462 | Loss: 2.5177 | LR: 2.64e-06 | 5252.6ms/step | 11.4 samples/s
+Step 48170/50462 | Loss: 2.4973 | LR: 2.62e-06 | 5242.5ms/step | 11.4 samples/s
+Step 48180/50462 | Loss: 2.5098 | LR: 2.60e-06 | 5247.1ms/step | 11.4 samples/s
+Step 48190/50462 | Loss: 2.5059 | LR: 2.57e-06 | 5245.1ms/step | 11.4 samples/s
+Step 48200/50462 | Loss: 2.5151 | LR: 2.55e-06 | 5253.5ms/step | 11.4 samples/s
+Step 48210/50462 | Loss: 2.4910 | LR: 2.53e-06 | 5254.3ms/step | 11.4 samples/s
+Step 48220/50462 | Loss: 2.4970 | LR: 2.51e-06 | 5251.0ms/step | 11.4 samples/s
+Step 48230/50462 | Loss: 2.5044 | LR: 2.48e-06 | 5256.5ms/step | 11.4 samples/s
+Step 48240/50462 | Loss: 2.5022 | LR: 2.46e-06 | 5252.8ms/step | 11.4 samples/s
+Step 48250/50462 | Loss: 2.5411 | LR: 2.44e-06 | 5256.3ms/step | 11.4 samples/s
+Validation at step 48250: Loss = 2.5191, Perplexity = 12.42
+Step 48260/50462 | Loss: 2.4783 | LR: 2.42e-06 | 7822.7ms/step | 7.7 samples/s
+Step 48270/50462 | Loss: 2.5044 | LR: 2.39e-06 | 5253.8ms/step | 11.4 samples/s
+Step 48280/50462 | Loss: 2.5126 | LR: 2.37e-06 | 5252.6ms/step | 11.4 samples/s
+Step 48290/50462 | Loss: 2.5001 | LR: 2.35e-06 | 5254.4ms/step | 11.4 samples/s
+Step 48300/50462 | Loss: 2.4932 | LR: 2.33e-06 | 5250.0ms/step | 11.4 samples/s
+Step 48310/50462 | Loss: 2.5132 | LR: 2.31e-06 | 5253.4ms/step | 11.4 samples/s
+Step 48320/50462 | Loss: 2.5122 | LR: 2.29e-06 | 5303.0ms/step | 11.3 samples/s
+Step 48330/50462 | Loss: 2.4952 | LR: 2.27e-06 | 5251.1ms/step | 11.4 samples/s
+Step 48340/50462 | Loss: 2.5184 | LR: 2.24e-06 | 5249.1ms/step | 11.4 samples/s
+Step 48350/50462 | Loss: 2.5319 | LR: 2.22e-06 | 5250.9ms/step | 11.4 samples/s
+Step 48360/50462 | Loss: 2.4888 | LR: 2.20e-06 | 5243.3ms/step | 11.4 samples/s
+Step 48370/50462 | Loss: 2.5259 | LR: 2.18e-06 | 5252.7ms/step | 11.4 samples/s
+Step 48380/50462 | Loss: 2.4924 | LR: 2.16e-06 | 5336.2ms/step | 11.2 samples/s
+Step 48390/50462 | Loss: 2.5401 | LR: 2.14e-06 | 5256.4ms/step | 11.4 samples/s
+Step 48400/50462 | Loss: 2.4950 | LR: 2.12e-06 | 5248.6ms/step | 11.4 samples/s
+Step 48410/50462 | Loss: 2.5232 | LR: 2.10e-06 | 5250.5ms/step | 11.4 samples/s
+Step 48420/50462 | Loss: 2.5001 | LR: 2.08e-06 | 5251.9ms/step | 11.4 samples/s
+Step 48430/50462 | Loss: 2.4958 | LR: 2.06e-06 | 5248.6ms/step | 11.4 samples/s
+Step 48440/50462 | Loss: 2.5134 | LR: 2.04e-06 | 5252.7ms/step | 11.4 samples/s
+Step 48450/50462 | Loss: 2.4793 | LR: 2.02e-06 | 5250.9ms/step | 11.4 samples/s
+Step 48460/50462 | Loss: 2.5265 | LR: 2.00e-06 | 5259.1ms/step | 11.4 samples/s
+Step 48470/50462 | Loss: 2.5147 | LR: 1.98e-06 | 5254.2ms/step | 11.4 samples/s
+Step 48480/50462 | Loss: 2.4980 | LR: 1.96e-06 | 5255.0ms/step | 11.4 samples/s
+Step 48490/50462 | Loss: 2.5062 | LR: 1.94e-06 | 5248.3ms/step | 11.4 samples/s
+Step 48500/50462 | Loss: 2.4746 | LR: 1.92e-06 | 5244.1ms/step | 11.4 samples/s
+Validation at step 48500: Loss = 2.5184, Perplexity = 12.41
+Step 48510/50462 | Loss: 2.5078 | LR: 1.90e-06 | 7816.9ms/step | 7.7 samples/s
+Step 48520/50462 | Loss: 2.5070 | LR: 1.88e-06 | 5253.1ms/step | 11.4 samples/s
+Step 48530/50462 | Loss: 2.5245 | LR: 1.86e-06 | 5251.9ms/step | 11.4 samples/s
+Step 48540/50462 | Loss: 2.4813 | LR: 1.84e-06 | 5252.7ms/step | 11.4 samples/s
+Step 48550/50462 | Loss: 2.4915 | LR: 1.82e-06 | 5254.9ms/step | 11.4 samples/s
+Step 48560/50462 | Loss: 2.5018 | LR: 1.80e-06 | 5250.9ms/step | 11.4 samples/s
+Step 48570/50462 | Loss: 2.5031 | LR: 1.78e-06 | 5254.8ms/step | 11.4 samples/s
+Step 48580/50462 | Loss: 2.4960 | LR: 1.77e-06 | 5247.7ms/step | 11.4 samples/s
+Step 48590/50462 | Loss: 2.5204 | LR: 1.75e-06 | 5248.2ms/step | 11.4 samples/s
+Step 48600/50462 | Loss: 2.5116 | LR: 1.73e-06 | 5243.9ms/step | 11.4 samples/s
+Step 48610/50462 | Loss: 2.4970 | LR: 1.71e-06 | 5295.3ms/step | 11.3 samples/s
+Step 48620/50462 | Loss: 2.5031 | LR: 1.69e-06 | 5250.8ms/step | 11.4 samples/s
+Step 48630/50462 | Loss: 2.5250 | LR: 1.67e-06 | 5243.3ms/step | 11.4 samples/s
+Step 48640/50462 | Loss: 2.5075 | LR: 1.66e-06 | 5253.5ms/step | 11.4 samples/s
+Step 48650/50462 | Loss: 2.5240 | LR: 1.64e-06 | 5249.8ms/step | 11.4 samples/s
+Step 48660/50462 | Loss: 2.5194 | LR: 1.62e-06 | 5237.6ms/step | 11.5 samples/s
+Step 48670/50462 | Loss: 2.5209 | LR: 1.60e-06 | 5251.4ms/step | 11.4 samples/s
+Step 48680/50462 | Loss: 2.5028 | LR: 1.58e-06 | 5330.6ms/step | 11.3 samples/s
+Step 48690/50462 | Loss: 2.5167 | LR: 1.57e-06 | 5244.3ms/step | 11.4 samples/s
+Step 48700/50462 | Loss: 2.4763 | LR: 1.55e-06 | 5253.6ms/step | 11.4 samples/s
+Step 48710/50462 | Loss: 2.4995 | LR: 1.53e-06 | 5244.0ms/step | 11.4 samples/s
+Step 48720/50462 | Loss: 2.5213 | LR: 1.51e-06 | 5243.9ms/step | 11.4 samples/s
+Step 48730/50462 | Loss: 2.5067 | LR: 1.50e-06 | 5247.1ms/step | 11.4 samples/s
+Step 48740/50462 | Loss: 2.4841 | LR: 1.48e-06 | 5242.6ms/step | 11.4 samples/s
+Step 48750/50462 | Loss: 2.5275 | LR: 1.46e-06 | 5243.3ms/step | 11.4 samples/s
+Validation at step 48750: Loss = 2.5183, Perplexity = 12.41
+Step 48760/50462 | Loss: 2.5379 | LR: 1.44e-06 | 7808.8ms/step | 7.7 samples/s
+Step 48770/50462 | Loss: 2.4784 | LR: 1.43e-06 | 5237.1ms/step | 11.5 samples/s
+Step 48780/50462 | Loss: 2.5007 | LR: 1.41e-06 | 5242.7ms/step | 11.4 samples/s
+Step 48790/50462 | Loss: 2.5085 | LR: 1.39e-06 | 5254.2ms/step | 11.4 samples/s
+Step 48800/50462 | Loss: 2.4880 | LR: 1.38e-06 | 5251.7ms/step | 11.4 samples/s
+Step 48810/50462 | Loss: 2.4975 | LR: 1.36e-06 | 5249.1ms/step | 11.4 samples/s
+Step 48820/50462 | Loss: 2.5027 | LR: 1.34e-06 | 5247.9ms/step | 11.4 samples/s
+Step 48830/50462 | Loss: 2.5194 | LR: 1.33e-06 | 5232.1ms/step | 11.5 samples/s
+Step 48840/50462 | Loss: 2.5320 | LR: 1.31e-06 | 5250.8ms/step | 11.4 samples/s
+Step 48850/50462 | Loss: 2.5044 | LR: 1.30e-06 | 5241.8ms/step | 11.4 samples/s
+Step 48860/50462 | Loss: 2.4701 | LR: 1.28e-06 | 5235.5ms/step | 11.5 samples/s
+Step 48870/50462 | Loss: 2.5130 | LR: 1.26e-06 | 5245.6ms/step | 11.4 samples/s
+Step 48880/50462 | Loss: 2.4950 | LR: 1.25e-06 | 5245.0ms/step | 11.4 samples/s
+Step 48890/50462 | Loss: 2.5261 | LR: 1.23e-06 | 5290.0ms/step | 11.3 samples/s
+Step 48900/50462 | Loss: 2.5042 | LR: 1.22e-06 | 5251.0ms/step | 11.4 samples/s
+Step 48910/50462 | Loss: 2.5180 | LR: 1.20e-06 | 5256.7ms/step | 11.4 samples/s
+Step 48920/50462 | Loss: 2.5050 | LR: 1.19e-06 | 5241.5ms/step | 11.4 samples/s
+Step 48930/50462 | Loss: 2.5010 | LR: 1.17e-06 | 5246.4ms/step | 11.4 samples/s
+Step 48940/50462 | Loss: 2.4813 | LR: 1.16e-06 | 5248.6ms/step | 11.4 samples/s
+Step 48950/50462 | Loss: 2.4816 | LR: 1.14e-06 | 5250.7ms/step | 11.4 samples/s
+Step 48960/50462 | Loss: 2.5128 | LR: 1.13e-06 | 5250.3ms/step | 11.4 samples/s
+Step 48970/50462 | Loss: 2.4803 | LR: 1.11e-06 | 5330.7ms/step | 11.3 samples/s
+Step 48980/50462 | Loss: 2.5484 | LR: 1.10e-06 | 5250.8ms/step | 11.4 samples/s
+Step 48990/50462 | Loss: 2.5327 | LR: 1.08e-06 | 5250.3ms/step | 11.4 samples/s
+Step 49000/50462 | Loss: 2.5145 | LR: 1.07e-06 | 5250.4ms/step | 11.4 samples/s
+Validation at step 49000: Loss = 2.5182, Perplexity = 12.41
+Saved checkpoint to checkpoints/run_20260529_182827/checkpoint-49000
+Step 49010/50462 | Loss: 2.5031 | LR: 1.05e-06 | 8666.1ms/step | 6.9 samples/s
+Step 49020/50462 | Loss: 2.5109 | LR: 1.04e-06 | 5243.3ms/step | 11.4 samples/s
+Step 49030/50462 | Loss: 2.5119 | LR: 1.02e-06 | 5247.9ms/step | 11.4 samples/s
+Step 49040/50462 | Loss: 2.5084 | LR: 1.01e-06 | 5249.5ms/step | 11.4 samples/s
+Step 49050/50462 | Loss: 2.5082 | LR: 9.95e-07 | 5249.7ms/step | 11.4 samples/s
+Step 49060/50462 | Loss: 2.5291 | LR: 9.81e-07 | 5251.6ms/step | 11.4 samples/s
+Step 49070/50462 | Loss: 2.5260 | LR: 9.67e-07 | 5253.8ms/step | 11.4 samples/s
+Step 49080/50462 | Loss: 2.5281 | LR: 9.53e-07 | 5240.5ms/step | 11.4 samples/s
+Step 49090/50462 | Loss: 2.4817 | LR: 9.39e-07 | 5250.7ms/step | 11.4 samples/s
+Step 49100/50462 | Loss: 2.5309 | LR: 9.25e-07 | 5246.9ms/step | 11.4 samples/s
+Step 49110/50462 | Loss: 2.5160 | LR: 9.12e-07 | 5252.1ms/step | 11.4 samples/s
+Step 49120/50462 | Loss: 2.4852 | LR: 8.99e-07 | 5252.7ms/step | 11.4 samples/s
+Step 49130/50462 | Loss: 2.5150 | LR: 8.85e-07 | 5252.3ms/step | 11.4 samples/s
+Step 49140/50462 | Loss: 2.4865 | LR: 8.72e-07 | 5249.8ms/step | 11.4 samples/s
+Step 49150/50462 | Loss: 2.5175 | LR: 8.59e-07 | 5250.6ms/step | 11.4 samples/s
+Step 49160/50462 | Loss: 2.5079 | LR: 8.46e-07 | 5285.7ms/step | 11.4 samples/s
+Step 49170/50462 | Loss: 2.5057 | LR: 8.33e-07 | 5247.1ms/step | 11.4 samples/s
+Step 49180/50462 | Loss: 2.4958 | LR: 8.20e-07 | 5253.8ms/step | 11.4 samples/s
+Step 49190/50462 | Loss: 2.4851 | LR: 8.07e-07 | 5244.5ms/step | 11.4 samples/s
+Step 49200/50462 | Loss: 2.5382 | LR: 7.95e-07 | 5250.8ms/step | 11.4 samples/s
+Step 49210/50462 | Loss: 2.5089 | LR: 7.82e-07 | 5239.1ms/step | 11.5 samples/s
+Step 49220/50462 | Loss: 2.5194 | LR: 7.70e-07 | 5245.1ms/step | 11.4 samples/s
+Step 49230/50462 | Loss: 2.5207 | LR: 7.57e-07 | 5252.4ms/step | 11.4 samples/s
+Step 49240/50462 | Loss: 2.5109 | LR: 7.45e-07 | 5250.8ms/step | 11.4 samples/s
+Step 49250/50462 | Loss: 2.5049 | LR: 7.33e-07 | 5242.5ms/step | 11.4 samples/s
+Validation at step 49250: Loss = 2.5182, Perplexity = 12.41
+Step 49260/50462 | Loss: 2.4777 | LR: 7.21e-07 | 7897.9ms/step | 7.6 samples/s
+Step 49270/50462 | Loss: 2.4964 | LR: 7.09e-07 | 5235.6ms/step | 11.5 samples/s
+Step 49280/50462 | Loss: 2.4807 | LR: 6.97e-07 | 5247.2ms/step | 11.4 samples/s
+Step 49290/50462 | Loss: 2.5015 | LR: 6.85e-07 | 5245.5ms/step | 11.4 samples/s
+Step 49300/50462 | Loss: 2.4988 | LR: 6.74e-07 | 5250.1ms/step | 11.4 samples/s
+Step 49310/50462 | Loss: 2.5059 | LR: 6.62e-07 | 5246.0ms/step | 11.4 samples/s
+Step 49320/50462 | Loss: 2.4939 | LR: 6.51e-07 | 5250.4ms/step | 11.4 samples/s
+Step 49330/50462 | Loss: 2.5025 | LR: 6.39e-07 | 5245.8ms/step | 11.4 samples/s
+Step 49340/50462 | Loss: 2.5122 | LR: 6.28e-07 | 5255.0ms/step | 11.4 samples/s
+Step 49350/50462 | Loss: 2.5147 | LR: 6.17e-07 | 5245.9ms/step | 11.4 samples/s
+Step 49360/50462 | Loss: 2.5255 | LR: 6.06e-07 | 5253.5ms/step | 11.4 samples/s
+Step 49370/50462 | Loss: 2.5077 | LR: 5.95e-07 | 5253.4ms/step | 11.4 samples/s
+Step 49380/50462 | Loss: 2.5202 | LR: 5.84e-07 | 5256.5ms/step | 11.4 samples/s
+Step 49390/50462 | Loss: 2.5193 | LR: 5.73e-07 | 5253.0ms/step | 11.4 samples/s
+Step 49400/50462 | Loss: 2.5172 | LR: 5.63e-07 | 5247.2ms/step | 11.4 samples/s
+Step 49410/50462 | Loss: 2.5100 | LR: 5.52e-07 | 5256.5ms/step | 11.4 samples/s
+Step 49420/50462 | Loss: 2.5070 | LR: 5.42e-07 | 5251.2ms/step | 11.4 samples/s
+Step 49430/50462 | Loss: 2.4866 | LR: 5.31e-07 | 5248.1ms/step | 11.4 samples/s
+Step 49440/50462 | Loss: 2.4953 | LR: 5.21e-07 | 5288.2ms/step | 11.3 samples/s
+Step 49450/50462 | Loss: 2.4830 | LR: 5.11e-07 | 5231.1ms/step | 11.5 samples/s
+Step 49460/50462 | Loss: 2.5169 | LR: 5.01e-07 | 5240.5ms/step | 11.4 samples/s
+Step 49470/50462 | Loss: 2.5186 | LR: 4.91e-07 | 5234.9ms/step | 11.5 samples/s
+Step 49480/50462 | Loss: 2.5116 | LR: 4.81e-07 | 5236.3ms/step | 11.5 samples/s
+Step 49490/50462 | Loss: 2.4924 | LR: 4.72e-07 | 5238.8ms/step | 11.5 samples/s
+Step 49500/50462 | Loss: 2.5127 | LR: 4.62e-07 | 5251.6ms/step | 11.4 samples/s
+Validation at step 49500: Loss = 2.5182, Perplexity = 12.41
+Step 49510/50462 | Loss: 2.5222 | LR: 4.52e-07 | 7794.9ms/step | 7.7 samples/s
+Step 49520/50462 | Loss: 2.5022 | LR: 4.43e-07 | 5225.1ms/step | 11.5 samples/s
+Step 49530/50462 | Loss: 2.5061 | LR: 4.34e-07 | 5235.0ms/step | 11.5 samples/s
+Step 49540/50462 | Loss: 2.4980 | LR: 4.24e-07 | 5239.7ms/step | 11.5 samples/s
+Step 49550/50462 | Loss: 2.5049 | LR: 4.15e-07 | 5322.5ms/step | 11.3 samples/s
+Step 49560/50462 | Loss: 2.5199 | LR: 4.06e-07 | 5251.4ms/step | 11.4 samples/s
+Step 49570/50462 | Loss: 2.5226 | LR: 3.97e-07 | 5250.9ms/step | 11.4 samples/s
+Step 49580/50462 | Loss: 2.5221 | LR: 3.88e-07 | 5249.4ms/step | 11.4 samples/s
+Step 49590/50462 | Loss: 2.4956 | LR: 3.79e-07 | 5250.9ms/step | 11.4 samples/s
+Step 49600/50462 | Loss: 2.5266 | LR: 3.71e-07 | 5242.6ms/step | 11.4 samples/s
+Step 49610/50462 | Loss: 2.5066 | LR: 3.62e-07 | 5245.4ms/step | 11.4 samples/s
+Step 49620/50462 | Loss: 2.5053 | LR: 3.54e-07 | 5246.3ms/step | 11.4 samples/s
+Step 49630/50462 | Loss: 2.5050 | LR: 3.45e-07 | 5243.3ms/step | 11.4 samples/s
+Step 49640/50462 | Loss: 2.5081 | LR: 3.37e-07 | 5245.2ms/step | 11.4 samples/s
+Step 49650/50462 | Loss: 2.5114 | LR: 3.29e-07 | 5249.8ms/step | 11.4 samples/s
+Step 49660/50462 | Loss: 2.4942 | LR: 3.21e-07 | 5240.4ms/step | 11.4 samples/s
+Step 49670/50462 | Loss: 2.4816 | LR: 3.13e-07 | 5249.6ms/step | 11.4 samples/s
+Step 49680/50462 | Loss: 2.5262 | LR: 3.05e-07 | 5250.3ms/step | 11.4 samples/s
+Step 49690/50462 | Loss: 2.5438 | LR: 2.97e-07 | 5245.5ms/step | 11.4 samples/s
+Step 49700/50462 | Loss: 2.5157 | LR: 2.90e-07 | 5231.2ms/step | 11.5 samples/s
+Step 49710/50462 | Loss: 2.5097 | LR: 2.82e-07 | 5248.3ms/step | 11.4 samples/s
+Step 49720/50462 | Loss: 2.5002 | LR: 2.75e-07 | 5286.7ms/step | 11.3 samples/s
+Step 49730/50462 | Loss: 2.5158 | LR: 2.67e-07 | 5249.8ms/step | 11.4 samples/s
+Step 49740/50462 | Loss: 2.5361 | LR: 2.60e-07 | 5236.6ms/step | 11.5 samples/s
+Step 49750/50462 | Loss: 2.5083 | LR: 2.53e-07 | 5247.2ms/step | 11.4 samples/s
+Validation at step 49750: Loss = 2.5182, Perplexity = 12.41
+Step 49760/50462 | Loss: 2.5257 | LR: 2.46e-07 | 7802.5ms/step | 7.7 samples/s
+Step 49770/50462 | Loss: 2.5144 | LR: 2.39e-07 | 5239.6ms/step | 11.5 samples/s
+Step 49780/50462 | Loss: 2.4989 | LR: 2.32e-07 | 5250.0ms/step | 11.4 samples/s
+Step 49790/50462 | Loss: 2.4928 | LR: 2.25e-07 | 5246.3ms/step | 11.4 samples/s
+Step 49800/50462 | Loss: 2.5350 | LR: 2.19e-07 | 5246.1ms/step | 11.4 samples/s
+Step 49810/50462 | Loss: 2.5217 | LR: 2.12e-07 | 5250.6ms/step | 11.4 samples/s
+Step 49820/50462 | Loss: 2.5196 | LR: 2.06e-07 | 5249.7ms/step | 11.4 samples/s
+Step 49830/50462 | Loss: 2.5084 | LR: 1.99e-07 | 5242.4ms/step | 11.4 samples/s
+Step 49840/50462 | Loss: 2.5228 | LR: 1.93e-07 | 5312.9ms/step | 11.3 samples/s
+Step 49850/50462 | Loss: 2.4998 | LR: 1.87e-07 | 5235.0ms/step | 11.5 samples/s
+Step 49860/50462 | Loss: 2.4994 | LR: 1.81e-07 | 5246.6ms/step | 11.4 samples/s
+Step 49870/50462 | Loss: 2.5146 | LR: 1.75e-07 | 5251.0ms/step | 11.4 samples/s
+Step 49880/50462 | Loss: 2.4993 | LR: 1.69e-07 | 5249.9ms/step | 11.4 samples/s
+Step 49890/50462 | Loss: 2.5089 | LR: 1.63e-07 | 5249.0ms/step | 11.4 samples/s
+Step 49900/50462 | Loss: 2.5215 | LR: 1.58e-07 | 5250.3ms/step | 11.4 samples/s
+Step 49910/50462 | Loss: 2.5079 | LR: 1.52e-07 | 5250.6ms/step | 11.4 samples/s
+Step 49920/50462 | Loss: 2.5219 | LR: 1.47e-07 | 5249.8ms/step | 11.4 samples/s
+Step 49930/50462 | Loss: 2.4929 | LR: 1.41e-07 | 5247.3ms/step | 11.4 samples/s
+Step 49940/50462 | Loss: 2.5051 | LR: 1.36e-07 | 5248.2ms/step | 11.4 samples/s
+Step 49950/50462 | Loss: 2.4658 | LR: 1.31e-07 | 5246.9ms/step | 11.4 samples/s
+Step 49960/50462 | Loss: 2.4954 | LR: 1.26e-07 | 5250.7ms/step | 11.4 samples/s
+Step 49970/50462 | Loss: 2.4869 | LR: 1.21e-07 | 5250.1ms/step | 11.4 samples/s
+Step 49980/50462 | Loss: 2.5022 | LR: 1.16e-07 | 5246.9ms/step | 11.4 samples/s
+Step 49990/50462 | Loss: 2.4800 | LR: 1.11e-07 | 5244.7ms/step | 11.4 samples/s
+Step 50000/50462 | Loss: 2.4907 | LR: 1.07e-07 | 5287.6ms/step | 11.3 samples/s
+Validation at step 50000: Loss = 2.5182, Perplexity = 12.41
+Saved checkpoint to checkpoints/run_20260529_182827/checkpoint-50000
+Step 50010/50462 | Loss: 2.4861 | LR: 1.02e-07 | 8701.0ms/step | 6.9 samples/s
+Step 50020/50462 | Loss: 2.4939 | LR: 9.75e-08 | 5243.0ms/step | 11.4 samples/s
+Step 50030/50462 | Loss: 2.5242 | LR: 9.32e-08 | 5253.8ms/step | 11.4 samples/s
+Step 50040/50462 | Loss: 2.5542 | LR: 8.89e-08 | 5247.4ms/step | 11.4 samples/s
+Step 50050/50462 | Loss: 2.5080 | LR: 8.47e-08 | 5248.4ms/step | 11.4 samples/s
+Step 50060/50462 | Loss: 2.5007 | LR: 8.07e-08 | 5246.5ms/step | 11.4 samples/s
+Step 50070/50462 | Loss: 2.5212 | LR: 7.67e-08 | 5247.2ms/step | 11.4 samples/s
+Step 50080/50462 | Loss: 2.4925 | LR: 7.28e-08 | 5244.9ms/step | 11.4 samples/s
+Step 50090/50462 | Loss: 2.4856 | LR: 6.91e-08 | 5253.1ms/step | 11.4 samples/s
+Step 50100/50462 | Loss: 2.4915 | LR: 6.54e-08 | 5249.7ms/step | 11.4 samples/s
+Step 50110/50462 | Loss: 2.5314 | LR: 6.19e-08 | 5255.8ms/step | 11.4 samples/s
+Step 50120/50462 | Loss: 2.5123 | LR: 5.84e-08 | 5247.0ms/step | 11.4 samples/s
+Step 50130/50462 | Loss: 2.5125 | LR: 5.50e-08 | 5334.4ms/step | 11.2 samples/s
+Step 50140/50462 | Loss: 2.5308 | LR: 5.18e-08 | 5248.4ms/step | 11.4 samples/s
+Step 50150/50462 | Loss: 2.4860 | LR: 4.86e-08 | 5250.9ms/step | 11.4 samples/s
+Step 50160/50462 | Loss: 2.5018 | LR: 4.55e-08 | 5251.6ms/step | 11.4 samples/s
+Step 50170/50462 | Loss: 2.4697 | LR: 4.26e-08 | 5247.5ms/step | 11.4 samples/s
+Step 50180/50462 | Loss: 2.4745 | LR: 3.97e-08 | 5254.5ms/step | 11.4 samples/s
+Step 50190/50462 | Loss: 2.5220 | LR: 3.69e-08 | 5250.2ms/step | 11.4 samples/s
+Step 50200/50462 | Loss: 2.4912 | LR: 3.43e-08 | 5250.2ms/step | 11.4 samples/s
+Step 50210/50462 | Loss: 2.5166 | LR: 3.17e-08 | 5252.6ms/step | 11.4 samples/s
+Step 50220/50462 | Loss: 2.5125 | LR: 2.92e-08 | 5253.5ms/step | 11.4 samples/s
+Step 50230/50462 | Loss: 2.5299 | LR: 2.69e-08 | 5250.6ms/step | 11.4 samples/s
+Step 50240/50462 | Loss: 2.4942 | LR: 2.46e-08 | 5250.6ms/step | 11.4 samples/s
+Step 50250/50462 | Loss: 2.4882 | LR: 2.24e-08 | 5249.7ms/step | 11.4 samples/s
+Validation at step 50250: Loss = 2.5182, Perplexity = 12.41
+Step 50260/50462 | Loss: 2.5348 | LR: 2.04e-08 | 7818.4ms/step | 7.7 samples/s
+Step 50270/50462 | Loss: 2.5470 | LR: 1.84e-08 | 5249.7ms/step | 11.4 samples/s
+Step 50280/50462 | Loss: 2.5178 | LR: 1.65e-08 | 5295.5ms/step | 11.3 samples/s
+Step 50290/50462 | Loss: 2.5316 | LR: 1.48e-08 | 5254.1ms/step | 11.4 samples/s
+Step 50300/50462 | Loss: 2.4866 | LR: 1.31e-08 | 5252.5ms/step | 11.4 samples/s
+Step 50310/50462 | Loss: 2.4822 | LR: 1.15e-08 | 5252.2ms/step | 11.4 samples/s
+Step 50320/50462 | Loss: 2.5020 | LR: 1.01e-08 | 5247.2ms/step | 11.4 samples/s
+Step 50330/50462 | Loss: 2.5301 | LR: 8.70e-09 | 5251.7ms/step | 11.4 samples/s
+Step 50340/50462 | Loss: 2.5103 | LR: 7.43e-09 | 5249.8ms/step | 11.4 samples/s
+Step 50350/50462 | Loss: 2.4918 | LR: 6.26e-09 | 5252.5ms/step | 11.4 samples/s
+Step 50360/50462 | Loss: 2.5145 | LR: 5.19e-09 | 5254.1ms/step | 11.4 samples/s
+Step 50370/50462 | Loss: 2.4905 | LR: 4.23e-09 | 5247.6ms/step | 11.4 samples/s
+Step 50380/50462 | Loss: 2.4830 | LR: 3.36e-09 | 5247.5ms/step | 11.4 samples/s
+Step 50390/50462 | Loss: 2.5222 | LR: 2.59e-09 | 5247.5ms/step | 11.4 samples/s
+Step 50400/50462 | Loss: 2.5060 | LR: 1.92e-09 | 5248.9ms/step | 11.4 samples/s
+Step 50410/50462 | Loss: 2.4955 | LR: 1.35e-09 | 5252.4ms/step | 11.4 samples/s
+Step 50420/50462 | Loss: 2.5232 | LR: 8.81e-10 | 5285.6ms/step | 11.4 samples/s
+Step 50430/50462 | Loss: 2.5133 | LR: 5.11e-10 | 5287.6ms/step | 11.3 samples/s
+Step 50440/50462 | Loss: 2.4837 | LR: 2.42e-10 | 5250.6ms/step | 11.4 samples/s
+Step 50450/50462 | Loss: 2.5245 | LR: 7.19e-11 | 5250.8ms/step | 11.4 samples/s
+Step 50460/50462 | Loss: 2.4978 | LR: 2.00e-12 | 5247.1ms/step | 11.4 samples/s
+Saved checkpoint to checkpoints/run_20260529_182827/checkpoint-50462
+wandb: updating run metadata
+wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml
+wandb: uploading output.log; uploading config.yaml
+wandb: uploading history steps 246-246, summary, console lines 259-259
+wandb: 
+wandb: Run history:
+wandb:             eval/loss █▃▁▁▁▁▁▁▁▁
+wandb:       eval/perplexity █▃▁▁▁▁▁▁▁▁
+wandb:     speed/ms_per_step ▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb: speed/samples_per_sec ████████████▁███████████████████████████
+wandb:   speed/steps_per_sec ████████████████████████████████▁███████
+wandb:           train/epoch ▁▁▁▁▂▂▂▂▃▃▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇████
+wandb:     train/global_step ▁▁▁▁▁▂▂▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▆▆▆▆▇▇▇▇▇▇█████
+wandb:   train/learning_rate ██▇▇▇▇▇▆▅▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁
+wandb:            train/loss ▇▅▃▁▂▅▄▄▅▄▅▃▄▅▄▁▁▆▅▄▄▁▃▄▅▅▂▃▃▆▁▄▇▄▂▅▂█▆▆
+wandb: 
+wandb: Run summary:
+wandb:             eval/loss 2.51822
+wandb:       eval/perplexity 12.40654
+wandb:     speed/ms_per_step 5247.137
+wandb: speed/samples_per_sec 11.43481
+wandb:   speed/steps_per_sec 0.19058
+wandb:           train/epoch 0.99996
+wandb:     train/global_step 50460
+wandb:   train/learning_rate 0.0
+wandb:            train/loss 2.49778
+wandb: 
+wandb: 🚀 View run dense continued 48k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/wbfo0o07
+wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20260529_184637-wbfo0o07/logs
+Training complete!

epfl_final/log_files/1mask_continued.log

@@ -0,0 +1,425 @@
+nohup: ignoring input
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+New run: Checkpoints will be saved to ./checkpoints/run_20260615_223806
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260615_223806",
+  "resume_from_checkpoint": "checkpoints/run_20260614_032718/checkpoint-40000",
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 40k",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "dense-resume"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: checkpoints/run_20260614_032718/checkpoint-40000
+Checkpoint activations: False
+Checkpoint activations: False
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 1536]           [2, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [2, 8192, 1536]           [2, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (G): 2.87
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 52437.19
+Params size (MB): 2871.45
+Estimated Total Size (MB): 55308.77
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        3
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504627
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+Loading checkpoint from checkpoints/run_20260614_032718/checkpoint-40000...
+Overriding checkpoint LR with config LR: 0.0005
+Resumed from checkpoint at step 40000 (epoch 0, micro_step 400000)
+Resuming training from step 40000
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.27.2
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260615_224051-7jul6zqc
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense continued 40k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/7jul6zqc
+Starting training...
+Validation at step 40000: Loss = 2.5322, Perplexity = 12.58
+Switching attention mode for step 40001: dense (dense step interval)
+Step 40010/50462 | Loss: 2.5463 | LR: 5.26e-05 | 67266.9ms/step | 0.9 samples/s
+Step 40020/50462 | Loss: 2.5609 | LR: 5.25e-05 | 4208.1ms/step | 14.3 samples/s
+Step 40030/50462 | Loss: 2.5036 | LR: 5.24e-05 | 4203.7ms/step | 14.3 samples/s
+Step 40040/50462 | Loss: 2.5354 | LR: 5.23e-05 | 4209.2ms/step | 14.3 samples/s
+Step 40050/50462 | Loss: 2.5220 | LR: 5.22e-05 | 4200.9ms/step | 14.3 samples/s
+Step 40060/50462 | Loss: 2.5152 | LR: 5.21e-05 | 4206.9ms/step | 14.3 samples/s
+Step 40070/50462 | Loss: 2.5253 | LR: 5.20e-05 | 4213.0ms/step | 14.2 samples/s
+Step 40080/50462 | Loss: 2.4990 | LR: 5.19e-05 | 4297.2ms/step | 14.0 samples/s
+Step 40090/50462 | Loss: 2.5332 | LR: 5.18e-05 | 4211.9ms/step | 14.2 samples/s
+Step 40100/50462 | Loss: 2.5294 | LR: 5.17e-05 | 4213.1ms/step | 14.2 samples/s
+Step 40110/50462 | Loss: 2.5173 | LR: 5.16e-05 | 4207.5ms/step | 14.3 samples/s
+Step 40120/50462 | Loss: 2.5305 | LR: 5.15e-05 | 4205.5ms/step | 14.3 samples/s
+Step 40130/50462 | Loss: 2.5134 | LR: 5.14e-05 | 4204.9ms/step | 14.3 samples/s
+Step 40140/50462 | Loss: 2.5095 | LR: 5.13e-05 | 4210.1ms/step | 14.3 samples/s
+Step 40150/50462 | Loss: 2.4877 | LR: 5.12e-05 | 4340.5ms/step | 13.8 samples/s
+Step 40160/50462 | Loss: 2.4844 | LR: 5.11e-05 | 4203.2ms/step | 14.3 samples/s
+Step 40170/50462 | Loss: 2.5237 | LR: 5.10e-05 | 4206.2ms/step | 14.3 samples/s
+Step 40180/50462 | Loss: 2.5255 | LR: 5.09e-05 | 4206.7ms/step | 14.3 samples/s
+Step 40190/50462 | Loss: 2.5225 | LR: 5.09e-05 | 4199.0ms/step | 14.3 samples/s
+Step 40200/50462 | Loss: 2.5299 | LR: 5.08e-05 | 4198.2ms/step | 14.3 samples/s
+Step 40210/50462 | Loss: 2.5353 | LR: 5.07e-05 | 4202.0ms/step | 14.3 samples/s
+Step 40220/50462 | Loss: 2.5356 | LR: 5.06e-05 | 4211.0ms/step | 14.2 samples/s
+Step 40230/50462 | Loss: 2.5268 | LR: 5.05e-05 | 4204.3ms/step | 14.3 samples/s
+Step 40240/50462 | Loss: 2.5581 | LR: 5.04e-05 | 4206.9ms/step | 14.3 samples/s
+Step 40250/50462 | Loss: 2.5169 | LR: 5.03e-05 | 4203.4ms/step | 14.3 samples/s
+Validation at step 40250: Loss = 2.5282, Perplexity = 12.53
+Step 40260/50462 | Loss: 2.5224 | LR: 5.02e-05 | 6201.1ms/step | 9.7 samples/s
+Step 40270/50462 | Loss: 2.4979 | LR: 5.01e-05 | 4201.0ms/step | 14.3 samples/s
+Step 40280/50462 | Loss: 2.5227 | LR: 5.00e-05 | 4192.4ms/step | 14.3 samples/s
+Step 40290/50462 | Loss: 2.5202 | LR: 4.99e-05 | 4194.9ms/step | 14.3 samples/s
+Step 40300/50462 | Loss: 2.5158 | LR: 4.98e-05 | 4205.7ms/step | 14.3 samples/s
+Step 40310/50462 | Loss: 2.5016 | LR: 4.97e-05 | 4205.0ms/step | 14.3 samples/s
+Step 40320/50462 | Loss: 2.5193 | LR: 4.96e-05 | 4197.5ms/step | 14.3 samples/s
+Step 40330/50462 | Loss: 2.5423 | LR: 4.95e-05 | 4201.5ms/step | 14.3 samples/s
+Step 40340/50462 | Loss: 2.5020 | LR: 4.94e-05 | 4204.9ms/step | 14.3 samples/s
+Step 40350/50462 | Loss: 2.4937 | LR: 4.93e-05 | 4196.8ms/step | 14.3 samples/s
+Step 40360/50462 | Loss: 2.5224 | LR: 4.92e-05 | 4205.8ms/step | 14.3 samples/s
+Step 40370/50462 | Loss: 2.5357 | LR: 4.91e-05 | 4205.5ms/step | 14.3 samples/s
+Step 40380/50462 | Loss: 2.5213 | LR: 4.90e-05 | 4254.5ms/step | 14.1 samples/s
+Step 40390/50462 | Loss: 2.5166 | LR: 4.90e-05 | 4206.9ms/step | 14.3 samples/s
+Step 40400/50462 | Loss: 2.5290 | LR: 4.89e-05 | 4204.8ms/step | 14.3 samples/s
+Step 40410/50462 | Loss: 2.5430 | LR: 4.88e-05 | 4203.8ms/step | 14.3 samples/s
+Step 40420/50462 | Loss: 2.5418 | LR: 4.87e-05 | 4206.1ms/step | 14.3 samples/s
+Step 40430/50462 | Loss: 2.5087 | LR: 4.86e-05 | 4206.6ms/step | 14.3 samples/s
+Step 40440/50462 | Loss: 2.5303 | LR: 4.85e-05 | 4202.1ms/step | 14.3 samples/s
+Step 40450/50462 | Loss: 2.5168 | LR: 4.84e-05 | 4321.7ms/step | 13.9 samples/s
+Step 40460/50462 | Loss: 2.5494 | LR: 4.83e-05 | 4205.4ms/step | 14.3 samples/s
+Step 40470/50462 | Loss: 2.5037 | LR: 4.82e-05 | 4196.3ms/step | 14.3 samples/s
+Step 40480/50462 | Loss: 2.5148 | LR: 4.81e-05 | 4202.1ms/step | 14.3 samples/s
+Step 40490/50462 | Loss: 2.5302 | LR: 4.80e-05 | 4203.1ms/step | 14.3 samples/s
+Step 40500/50462 | Loss: 2.5209 | LR: 4.79e-05 | 4202.5ms/step | 14.3 samples/s
+Validation at step 40500: Loss = 2.5276, Perplexity = 12.52
+Step 40510/50462 | Loss: 2.5184 | LR: 4.78e-05 | 6210.0ms/step | 9.7 samples/s
+Step 40520/50462 | Loss: 2.5059 | LR: 4.77e-05 | 4204.5ms/step | 14.3 samples/s
+Step 40530/50462 | Loss: 2.5088 | LR: 4.76e-05 | 4202.2ms/step | 14.3 samples/s
+Step 40540/50462 | Loss: 2.4815 | LR: 4.76e-05 | 4202.0ms/step | 14.3 samples/s
+Step 40550/50462 | Loss: 2.5367 | LR: 4.75e-05 | 4202.5ms/step | 14.3 samples/s
+Step 40560/50462 | Loss: 2.5279 | LR: 4.74e-05 | 4211.4ms/step | 14.2 samples/s
+Step 40570/50462 | Loss: 2.5038 | LR: 4.73e-05 | 4199.2ms/step | 14.3 samples/s
+Step 40580/50462 | Loss: 2.5326 | LR: 4.72e-05 | 4200.6ms/step | 14.3 samples/s
+Step 40590/50462 | Loss: 2.5149 | LR: 4.71e-05 | 4200.2ms/step | 14.3 samples/s
+Step 40600/50462 | Loss: 2.5159 | LR: 4.70e-05 | 4201.7ms/step | 14.3 samples/s
+Step 40610/50462 | Loss: 2.5341 | LR: 4.69e-05 | 4198.0ms/step | 14.3 samples/s
+Step 40620/50462 | Loss: 2.4919 | LR: 4.68e-05 | 4199.5ms/step | 14.3 samples/s
+Step 40630/50462 | Loss: 2.5126 | LR: 4.67e-05 | 4202.7ms/step | 14.3 samples/s
+Step 40640/50462 | Loss: 2.5242 | LR: 4.66e-05 | 4208.7ms/step | 14.3 samples/s
+Step 40650/50462 | Loss: 2.5230 | LR: 4.65e-05 | 4198.2ms/step | 14.3 samples/s
+Step 40660/50462 | Loss: 2.5352 | LR: 4.64e-05 | 4199.3ms/step | 14.3 samples/s
+Step 40670/50462 | Loss: 2.5120 | LR: 4.64e-05 | 4200.4ms/step | 14.3 samples/s
+Step 40680/50462 | Loss: 2.5164 | LR: 4.63e-05 | 4257.1ms/step | 14.1 samples/s
+Step 40690/50462 | Loss: 2.5124 | LR: 4.62e-05 | 4198.2ms/step | 14.3 samples/s
+Step 40700/50462 | Loss: 2.4842 | LR: 4.61e-05 | 4204.3ms/step | 14.3 samples/s
+Step 40710/50462 | Loss: 2.5247 | LR: 4.60e-05 | 4203.9ms/step | 14.3 samples/s
+Step 40720/50462 | Loss: 2.4916 | LR: 4.59e-05 | 4203.5ms/step | 14.3 samples/s
+Step 40730/50462 | Loss: 2.5023 | LR: 4.58e-05 | 4194.3ms/step | 14.3 samples/s
+Step 40740/50462 | Loss: 2.4982 | LR: 4.57e-05 | 4195.7ms/step | 14.3 samples/s
+Step 40750/50462 | Loss: 2.5095 | LR: 4.56e-05 | 4318.3ms/step | 13.9 samples/s
+Validation at step 40750: Loss = 2.5271, Perplexity = 12.52
+Step 40760/50462 | Loss: 2.5332 | LR: 4.55e-05 | 6209.6ms/step | 9.7 samples/s
+Step 40770/50462 | Loss: 2.5152 | LR: 4.54e-05 | 4200.0ms/step | 14.3 samples/s
+Step 40780/50462 | Loss: 2.5297 | LR: 4.54e-05 | 4200.8ms/step | 14.3 samples/s
+Step 40790/50462 | Loss: 2.5147 | LR: 4.53e-05 | 4199.2ms/step | 14.3 samples/s
+Step 40800/50462 | Loss: 2.5241 | LR: 4.52e-05 | 4200.0ms/step | 14.3 samples/s
+Step 40810/50462 | Loss: 2.4814 | LR: 4.51e-05 | 4202.3ms/step | 14.3 samples/s
+Step 40820/50462 | Loss: 2.5553 | LR: 4.50e-05 | 4201.9ms/step | 14.3 samples/s
+Step 40830/50462 | Loss: 2.4696 | LR: 4.49e-05 | 4201.8ms/step | 14.3 samples/s
+Step 40840/50462 | Loss: 2.4939 | LR: 4.48e-05 | 4208.0ms/step | 14.3 samples/s
+Step 40850/50462 | Loss: 2.5362 | LR: 4.47e-05 | 4194.0ms/step | 14.3 samples/s
+Step 40860/50462 | Loss: 2.4831 | LR: 4.46e-05 | 4196.2ms/step | 14.3 samples/s
+Step 40870/50462 | Loss: 2.5381 | LR: 4.45e-05 | 4200.0ms/step | 14.3 samples/s
+Step 40880/50462 | Loss: 2.5449 | LR: 4.45e-05 | 4199.8ms/step | 14.3 samples/s
+Step 40890/50462 | Loss: 2.4778 | LR: 4.44e-05 | 4207.1ms/step | 14.3 samples/s
+Step 40900/50462 | Loss: 2.5016 | LR: 4.43e-05 | 4207.0ms/step | 14.3 samples/s
+Step 40910/50462 | Loss: 2.5231 | LR: 4.42e-05 | 4202.0ms/step | 14.3 samples/s
+Step 40920/50462 | Loss: 2.5317 | LR: 4.41e-05 | 4200.0ms/step | 14.3 samples/s
+Step 40930/50462 | Loss: 2.5232 | LR: 4.40e-05 | 4202.2ms/step | 14.3 samples/s
+Step 40940/50462 | Loss: 2.5401 | LR: 4.39e-05 | 4205.9ms/step | 14.3 samples/s
+Step 40950/50462 | Loss: 2.4941 | LR: 4.38e-05 | 4201.9ms/step | 14.3 samples/s
+Step 40960/50462 | Loss: 2.5237 | LR: 4.37e-05 | 4200.0ms/step | 14.3 samples/s
+Step 40970/50462 | Loss: 2.5137 | LR: 4.36e-05 | 4258.2ms/step | 14.1 samples/s
+Step 40980/50462 | Loss: 2.5148 | LR: 4.36e-05 | 4200.1ms/step | 14.3 samples/s
+Step 40990/50462 | Loss: 2.4829 | LR: 4.35e-05 | 4199.9ms/step | 14.3 samples/s
+Step 41000/50462 | Loss: 2.5280 | LR: 4.34e-05 | 4201.9ms/step | 14.3 samples/s
+Validation at step 41000: Loss = 2.5267, Perplexity = 12.51
+Saved checkpoint to checkpoints/run_20260615_223806/checkpoint-41000
+Step 41010/50462 | Loss: 2.5342 | LR: 4.33e-05 | 7184.0ms/step | 8.4 samples/s
+Step 41020/50462 | Loss: 2.5293 | LR: 4.32e-05 | 4199.8ms/step | 14.3 samples/s
+Step 41030/50462 | Loss: 2.5324 | LR: 4.31e-05 | 4199.9ms/step | 14.3 samples/s
+Step 41040/50462 | Loss: 2.4951 | LR: 4.30e-05 | 4200.2ms/step | 14.3 samples/s
+Step 41050/50462 | Loss: 2.5257 | LR: 4.29e-05 | 4328.5ms/step | 13.9 samples/s
+Step 41060/50462 | Loss: 2.5307 | LR: 4.28e-05 | 4195.5ms/step | 14.3 samples/s
+Step 41070/50462 | Loss: 2.5094 | LR: 4.28e-05 | 4197.7ms/step | 14.3 samples/s
+Step 41080/50462 | Loss: 2.5031 | LR: 4.27e-05 | 4192.2ms/step | 14.3 samples/s
+Step 41090/50462 | Loss: 2.4810 | LR: 4.26e-05 | 4200.0ms/step | 14.3 samples/s
+Step 41100/50462 | Loss: 2.5153 | LR: 4.25e-05 | 4200.1ms/step | 14.3 samples/s
+Step 41110/50462 | Loss: 2.5200 | LR: 4.24e-05 | 4199.9ms/step | 14.3 samples/s
+Step 41120/50462 | Loss: 2.5093 | LR: 4.23e-05 | 4201.0ms/step | 14.3 samples/s
+Step 41130/50462 | Loss: 2.5327 | LR: 4.22e-05 | 4205.1ms/step | 14.3 samples/s
+Step 41140/50462 | Loss: 2.5067 | LR: 4.21e-05 | 4199.9ms/step | 14.3 samples/s
+Step 41150/50462 | Loss: 2.5203 | LR: 4.21e-05 | 4200.2ms/step | 14.3 samples/s
+Step 41160/50462 | Loss: 2.5176 | LR: 4.20e-05 | 4201.9ms/step | 14.3 samples/s
+Step 41170/50462 | Loss: 2.5230 | LR: 4.19e-05 | 4202.0ms/step | 14.3 samples/s
+Step 41180/50462 | Loss: 2.5073 | LR: 4.18e-05 | 4201.5ms/step | 14.3 samples/s
+Step 41190/50462 | Loss: 2.4674 | LR: 4.17e-05 | 4196.3ms/step | 14.3 samples/s
+Step 41200/50462 | Loss: 2.5034 | LR: 4.16e-05 | 4200.6ms/step | 14.3 samples/s
+Step 41210/50462 | Loss: 2.4810 | LR: 4.15e-05 | 4204.4ms/step | 14.3 samples/s
+Step 41220/50462 | Loss: 2.4977 | LR: 4.14e-05 | 4201.1ms/step | 14.3 samples/s
+Step 41230/50462 | Loss: 2.5192 | LR: 4.14e-05 | 4194.0ms/step | 14.3 samples/s
+Step 41240/50462 | Loss: 2.5090 | LR: 4.13e-05 | 4202.0ms/step | 14.3 samples/s
+Step 41250/50462 | Loss: 2.5236 | LR: 4.12e-05 | 4207.1ms/step | 14.3 samples/s
+Validation at step 41250: Loss = 2.5265, Perplexity = 12.51
+Step 41260/50462 | Loss: 2.5193 | LR: 4.11e-05 | 6205.2ms/step | 9.7 samples/s
+Step 41270/50462 | Loss: 2.4936 | LR: 4.10e-05 | 4249.1ms/step | 14.1 samples/s
+Step 41280/50462 | Loss: 2.4717 | LR: 4.09e-05 | 4200.3ms/step | 14.3 samples/s
+Step 41290/50462 | Loss: 2.5333 | LR: 4.08e-05 | 4195.8ms/step | 14.3 samples/s
+Step 41300/50462 | Loss: 2.5240 | LR: 4.07e-05 | 4197.5ms/step | 14.3 samples/s
+Step 41310/50462 | Loss: 2.4848 | LR: 4.07e-05 | 4200.6ms/step | 14.3 samples/s
+Step 41320/50462 | Loss: 2.5681 | LR: 4.06e-05 | 4196.2ms/step | 14.3 samples/s
+Step 41330/50462 | Loss: 2.5030 | LR: 4.05e-05 | 4199.6ms/step | 14.3 samples/s
+Step 41340/50462 | Loss: 2.4963 | LR: 4.04e-05 | 4196.2ms/step | 14.3 samples/s
+Step 41350/50462 | Loss: 2.5176 | LR: 4.03e-05 | 4250.3ms/step | 14.1 samples/s
+Step 41360/50462 | Loss: 2.5449 | LR: 4.02e-05 | 4254.5ms/step | 14.1 samples/s
+Step 41370/50462 | Loss: 2.5158 | LR: 4.01e-05 | 4202.7ms/step | 14.3 samples/s
+Step 41380/50462 | Loss: 2.5086 | LR: 4.01e-05 | 4206.7ms/step | 14.3 samples/s
+Step 41390/50462 | Loss: 2.4851 | LR: 4.00e-05 | 4201.1ms/step | 14.3 samples/s
+Step 41400/50462 | Loss: 2.5247 | LR: 3.99e-05 | 4203.0ms/step | 14.3 samples/s
+Step 41410/50462 | Loss: 2.5306 | LR: 3.98e-05 | 4203.1ms/step | 14.3 samples/s
+Step 41420/50462 | Loss: 2.4950 | LR: 3.97e-05 | 4202.8ms/step | 14.3 samples/s
+Step 41430/50462 | Loss: 2.5258 | LR: 3.96e-05 | 4206.1ms/step | 14.3 samples/s
+Step 41440/50462 | Loss: 2.4946 | LR: 3.95e-05 | 4207.8ms/step | 14.3 samples/s
+Step 41450/50462 | Loss: 2.5137 | LR: 3.95e-05 | 4214.3ms/step | 14.2 samples/s
+Step 41460/50462 | Loss: 2.4960 | LR: 3.94e-05 | 4211.5ms/step | 14.2 samples/s
+Step 41470/50462 | Loss: 2.4761 | LR: 3.93e-05 | 4208.4ms/step | 14.3 samples/s
+Step 41480/50462 | Loss: 2.5337 | LR: 3.92e-05 | 4205.5ms/step | 14.3 samples/s
+Step 41490/50462 | Loss: 2.4873 | LR: 3.91e-05 | 4206.6ms/step | 14.3 samples/s
+Step 41500/50462 | Loss: 2.5293 | LR: 3.90e-05 | 4204.7ms/step | 14.3 samples/s
+Validation at step 41500: Loss = 2.5263, Perplexity = 12.51
+Step 41510/50462 | Loss: 2.4999 | LR: 3.90e-05 | 6211.1ms/step | 9.7 samples/s
+Step 41520/50462 | Loss: 2.5141 | LR: 3.89e-05 | 4200.8ms/step | 14.3 samples/s
+Step 41530/50462 | Loss: 2.5256 | LR: 3.88e-05 | 4195.0ms/step | 14.3 samples/s
+Step 41540/50462 | Loss: 2.4945 | LR: 3.87e-05 | 4201.9ms/step | 14.3 samples/s
+Step 41550/50462 | Loss: 2.5210 | LR: 3.86e-05 | 4218.2ms/step | 14.2 samples/s
+Step 41560/50462 | Loss: 2.5318 | LR: 3.85e-05 | 4207.7ms/step | 14.3 samples/s
+Step 41570/50462 | Loss: 2.5190 | LR: 3.84e-05 | 4258.1ms/step | 14.1 samples/s
+Step 41580/50462 | Loss: 2.5253 | LR: 3.84e-05 | 4202.5ms/step | 14.3 samples/s
+Step 41590/50462 | Loss: 2.5071 | LR: 3.83e-05 | 4205.4ms/step | 14.3 samples/s
+Step 41600/50462 | Loss: 2.5213 | LR: 3.82e-05 | 4200.1ms/step | 14.3 samples/s
+Step 41610/50462 | Loss: 2.5035 | LR: 3.81e-05 | 4201.8ms/step | 14.3 samples/s
+Step 41620/50462 | Loss: 2.5098 | LR: 3.80e-05 | 4200.4ms/step | 14.3 samples/s
+Step 41630/50462 | Loss: 2.4912 | LR: 3.79e-05 | 4201.5ms/step | 14.3 samples/s
+Step 41640/50462 | Loss: 2.5069 | LR: 3.79e-05 | 4200.2ms/step | 14.3 samples/s
+Step 41650/50462 | Loss: 2.4921 | LR: 3.78e-05 | 4253.8ms/step | 14.1 samples/s
+Step 41660/50462 | Loss: 2.5223 | LR: 3.77e-05 | 4264.6ms/step | 14.1 samples/s
+Step 41670/50462 | Loss: 2.5142 | LR: 3.76e-05 | 4210.0ms/step | 14.3 samples/s
+Step 41680/50462 | Loss: 2.4996 | LR: 3.75e-05 | 4201.3ms/step | 14.3 samples/s
+Step 41690/50462 | Loss: 2.4965 | LR: 3.74e-05 | 4202.4ms/step | 14.3 samples/s
+Step 41700/50462 | Loss: 2.5083 | LR: 3.74e-05 | 4205.0ms/step | 14.3 samples/s
+Step 41710/50462 | Loss: 2.5003 | LR: 3.73e-05 | 4203.5ms/step | 14.3 samples/s
+Step 41720/50462 | Loss: 2.4970 | LR: 3.72e-05 | 4203.0ms/step | 14.3 samples/s
+Step 41730/50462 | Loss: 2.5027 | LR: 3.71e-05 | 4200.3ms/step | 14.3 samples/s
+Step 41740/50462 | Loss: 2.4951 | LR: 3.70e-05 | 4204.2ms/step | 14.3 samples/s
+Step 41750/50462 | Loss: 2.5031 | LR: 3.69e-05 | 4203.7ms/step | 14.3 samples/s
+Validation at step 41750: Loss = 2.5260, Perplexity = 12.50
+Step 41760/50462 | Loss: 2.5147 | LR: 3.69e-05 | 6207.6ms/step | 9.7 samples/s
+Step 41770/50462 | Loss: 2.5212 | LR: 3.68e-05 | 4205.2ms/step | 14.3 samples/s
+Step 41780/50462 | Loss: 2.5079 | LR: 3.67e-05 | 4199.0ms/step | 14.3 samples/s
+Step 41790/50462 | Loss: 2.4937 | LR: 3.66e-05 | 4198.0ms/step | 14.3 samples/s
+Step 41800/50462 | Loss: 2.4985 | LR: 3.65e-05 | 4196.3ms/step | 14.3 samples/s
+Step 41810/50462 | Loss: 2.5331 | LR: 3.64e-05 | 4197.8ms/step | 14.3 samples/s
+Step 41820/50462 | Loss: 2.4581 | LR: 3.64e-05 | 4200.0ms/step | 14.3 samples/s
+Step 41830/50462 | Loss: 2.5050 | LR: 3.63e-05 | 4199.7ms/step | 14.3 samples/s
+Step 41840/50462 | Loss: 2.4806 | LR: 3.62e-05 | 4198.2ms/step | 14.3 samples/s
+Step 41850/50462 | Loss: 2.5223 | LR: 3.61e-05 | 4199.8ms/step | 14.3 samples/s
+Step 41860/50462 | Loss: 2.5078 | LR: 3.60e-05 | 4198.4ms/step | 14.3 samples/s
+Step 41870/50462 | Loss: 2.5056 | LR: 3.60e-05 | 4256.1ms/step | 14.1 samples/s
+Step 41880/50462 | Loss: 2.5179 | LR: 3.59e-05 | 4198.2ms/step | 14.3 samples/s
+Step 41890/50462 | Loss: 2.5096 | LR: 3.58e-05 | 4199.6ms/step | 14.3 samples/s
+Step 41900/50462 | Loss: 2.5345 | LR: 3.57e-05 | 4199.5ms/step | 14.3 samples/s
+Step 41910/50462 | Loss: 2.5259 | LR: 3.56e-05 | 4198.5ms/step | 14.3 samples/s
+Step 41920/50462 | Loss: 2.5167 | LR: 3.55e-05 | 4198.0ms/step | 14.3 samples/s
+Step 41930/50462 | Loss: 2.4971 | LR: 3.55e-05 | 4201.9ms/step | 14.3 samples/s
+Step 41940/50462 | Loss: 2.5241 | LR: 3.54e-05 | 4206.6ms/step | 14.3 samples/s
+Step 41950/50462 | Loss: 2.4953 | LR: 3.53e-05 | 4201.5ms/step | 14.3 samples/s
+Step 41960/50462 | Loss: 2.5219 | LR: 3.52e-05 | 4303.8ms/step | 13.9 samples/s
+Step 41970/50462 | Loss: 2.5099 | LR: 3.51e-05 | 4202.1ms/step | 14.3 samples/s
+Step 41980/50462 | Loss: 2.5082 | LR: 3.51e-05 | 4200.2ms/step | 14.3 samples/s
+Step 41990/50462 | Loss: 2.5414 | LR: 3.50e-05 | 4201.5ms/step | 14.3 samples/s
+Step 42000/50462 | Loss: 2.5186 | LR: 3.49e-05 | 4202.3ms/step | 14.3 samples/s
+Validation at step 42000: Loss = 2.5259, Perplexity = 12.50
+Saved checkpoint to checkpoints/run_20260615_223806/checkpoint-42000
+Step 42010/50462 | Loss: 2.4877 | LR: 3.48e-05 | 7057.8ms/step | 8.5 samples/s
+Step 42020/50462 | Loss: 2.5294 | LR: 3.47e-05 | 4200.0ms/step | 14.3 samples/s
+Step 42030/50462 | Loss: 2.5481 | LR: 3.47e-05 | 4200.1ms/step | 14.3 samples/s
+W0616 01:16:46.153000 3487 torch/distributed/elastic/agent/server/api.py:753] Received Signals.SIGTERM death signal, shutting down workers
+W0616 01:16:46.155000 3487 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 3647 closing signal SIGTERM
+W0616 01:16:46.157000 3487 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 3648 closing signal SIGTERM
+W0616 01:16:46.158000 3487 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 3649 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1020, in <module>
+    main()
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 367, in wrapper
+    return f(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1016, in main
+    run(args)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1007, in run
+    elastic_launch(
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 191, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 362, in launch_agent
+    result = agent.run()
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 745, in run
+    result = self._invoke_run(role)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 923, in _invoke_run
+    time.sleep(monitor_interval)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 3487 got signal: 15

epfl_final/log_files/another_sparse_33%.log

@@ -0,0 +1,552 @@
+New run: Checkpoints will be saved to ./checkpoints/run_20260506_171628
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/the_pile_train/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260506_171628",
+  "resume_from_checkpoint": null,
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 2000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": null,
+  "wandb_tags": [
+    "qwen3",
+    "0.6b"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 20,
+  "sparse_mask_pool_size": 1000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run nmzybrc7
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260506_173734-nmzybrc7
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run exalted-wave-16
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/nmzybrc7
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 20
+  sparse_mask_pool_size: 1000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/the_pile_train/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/the_pile_train/tokenized...
+Loaded 5315146 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1328786)
+Dataset split: 1063028 train, 265758 validation
+Training examples (packed, train split):  1063028
+Validation examples (packed, val split):  265758
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       354342
+Optimizer steps per epoch:                44292
+Total optimizer steps:                    20005  (capped by max_steps; 0.45 epochs)
+Starting training...
+Step 10/20005 | Loss: 12.0953 | LR: 2.00e-05 | 9747.4ms/step | 2.5 samples/s
+Step 20/20005 | Loss: 11.5112 | LR: 4.00e-05 | 2455.1ms/step | 9.8 samples/s
+Step 30/20005 | Loss: 10.6615 | LR: 6.00e-05 | 2445.4ms/step | 9.8 samples/s
+Step 40/20005 | Loss: 10.1898 | LR: 8.00e-05 | 2439.2ms/step | 9.8 samples/s
+Step 50/20005 | Loss: 9.3633 | LR: 1.00e-04 | 2437.9ms/step | 9.8 samples/s
+Step 60/20005 | Loss: 8.6099 | LR: 1.20e-04 | 2437.9ms/step | 9.8 samples/s
+Step 70/20005 | Loss: 8.0160 | LR: 1.40e-04 | 2443.9ms/step | 9.8 samples/s
+Step 80/20005 | Loss: 7.6071 | LR: 1.60e-04 | 2450.5ms/step | 9.8 samples/s
+Step 90/20005 | Loss: 7.6196 | LR: 1.80e-04 | 2459.7ms/step | 9.8 samples/s
+Step 100/20005 | Loss: 7.3453 | LR: 2.00e-04 | 2461.2ms/step | 9.8 samples/s
+Step 110/20005 | Loss: 7.2536 | LR: 2.20e-04 | 2465.1ms/step | 9.7 samples/s
+Step 120/20005 | Loss: 7.1495 | LR: 2.40e-04 | 2464.9ms/step | 9.7 samples/s
+Step 130/20005 | Loss: 6.9748 | LR: 2.60e-04 | 2466.5ms/step | 9.7 samples/s
+Step 140/20005 | Loss: 6.8463 | LR: 2.80e-04 | 2472.3ms/step | 9.7 samples/s
+Step 150/20005 | Loss: 6.6894 | LR: 3.00e-04 | 2469.1ms/step | 9.7 samples/s
+Step 160/20005 | Loss: 6.5733 | LR: 3.20e-04 | 2472.4ms/step | 9.7 samples/s
+Step 170/20005 | Loss: 6.5434 | LR: 3.40e-04 | 2471.1ms/step | 9.7 samples/s
+Step 180/20005 | Loss: 6.4437 | LR: 3.60e-04 | 2469.6ms/step | 9.7 samples/s
+Step 190/20005 | Loss: 6.2488 | LR: 3.80e-04 | 2469.1ms/step | 9.7 samples/s
+Step 200/20005 | Loss: 6.3373 | LR: 4.00e-04 | 2470.4ms/step | 9.7 samples/s
+Step 210/20005 | Loss: 6.1415 | LR: 4.20e-04 | 2472.3ms/step | 9.7 samples/s
+Step 220/20005 | Loss: 6.0566 | LR: 4.40e-04 | 2472.8ms/step | 9.7 samples/s
+Step 230/20005 | Loss: 6.0284 | LR: 4.60e-04 | 2472.8ms/step | 9.7 samples/s
+Step 240/20005 | Loss: 6.0281 | LR: 4.80e-04 | 2472.8ms/step | 9.7 samples/s
+Step 250/20005 | Loss: 5.8878 | LR: 5.00e-04 | 2474.1ms/step | 9.7 samples/s
+Validation at step 250: Loss = 5.9092, Perplexity = 368.41
+Step 260/20005 | Loss: 5.8341 | LR: 5.00e-04 | 23166.1ms/step | 1.0 samples/s
+Step 270/20005 | Loss: 5.7961 | LR: 5.00e-04 | 2467.7ms/step | 9.7 samples/s
+Step 280/20005 | Loss: 5.8218 | LR: 5.00e-04 | 2469.0ms/step | 9.7 samples/s
+Step 290/20005 | Loss: 5.7036 | LR: 5.00e-04 | 2469.6ms/step | 9.7 samples/s
+Step 300/20005 | Loss: 5.6739 | LR: 5.00e-04 | 2469.6ms/step | 9.7 samples/s
+Step 310/20005 | Loss: 5.6067 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s
+Step 320/20005 | Loss: 5.5022 | LR: 5.00e-04 | 2472.2ms/step | 9.7 samples/s
+Step 330/20005 | Loss: 5.4496 | LR: 5.00e-04 | 2469.2ms/step | 9.7 samples/s
+Step 340/20005 | Loss: 5.2664 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s
+Step 350/20005 | Loss: 5.2576 | LR: 5.00e-04 | 2469.4ms/step | 9.7 samples/s
+Step 360/20005 | Loss: 5.3474 | LR: 5.00e-04 | 2468.0ms/step | 9.7 samples/s
+Step 370/20005 | Loss: 5.0944 | LR: 5.00e-04 | 2468.5ms/step | 9.7 samples/s
+Step 380/20005 | Loss: 5.1576 | LR: 5.00e-04 | 2473.3ms/step | 9.7 samples/s
+Step 390/20005 | Loss: 5.1447 | LR: 5.00e-04 | 2473.1ms/step | 9.7 samples/s
+Step 400/20005 | Loss: 5.0084 | LR: 5.00e-04 | 2475.2ms/step | 9.7 samples/s
+Step 410/20005 | Loss: 5.0430 | LR: 5.00e-04 | 2472.0ms/step | 9.7 samples/s
+Step 420/20005 | Loss: 4.8453 | LR: 5.00e-04 | 2470.4ms/step | 9.7 samples/s
+Step 430/20005 | Loss: 4.9373 | LR: 5.00e-04 | 2472.6ms/step | 9.7 samples/s
+Step 440/20005 | Loss: 4.7404 | LR: 5.00e-04 | 2473.2ms/step | 9.7 samples/s
+Step 450/20005 | Loss: 4.8040 | LR: 5.00e-04 | 2470.3ms/step | 9.7 samples/s
+Step 460/20005 | Loss: 4.8016 | LR: 5.00e-04 | 2468.5ms/step | 9.7 samples/s
+Step 470/20005 | Loss: 4.7038 | LR: 5.00e-04 | 2470.9ms/step | 9.7 samples/s
+Step 480/20005 | Loss: 4.6288 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s
+Step 490/20005 | Loss: 4.6210 | LR: 5.00e-04 | 2470.4ms/step | 9.7 samples/s
+Step 500/20005 | Loss: 4.5110 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s
+Validation at step 500: Loss = 4.5643, Perplexity = 95.99
+Step 510/20005 | Loss: 4.6094 | LR: 5.00e-04 | 20008.5ms/step | 1.2 samples/s
+Step 520/20005 | Loss: 4.5267 | LR: 5.00e-04 | 2473.6ms/step | 9.7 samples/s
+Step 530/20005 | Loss: 4.4512 | LR: 5.00e-04 | 2471.4ms/step | 9.7 samples/s
+Step 540/20005 | Loss: 4.4637 | LR: 5.00e-04 | 2470.1ms/step | 9.7 samples/s
+Step 550/20005 | Loss: 4.3918 | LR: 5.00e-04 | 2469.5ms/step | 9.7 samples/s
+Step 560/20005 | Loss: 4.4136 | LR: 5.00e-04 | 2471.2ms/step | 9.7 samples/s
+Step 570/20005 | Loss: 4.3111 | LR: 5.00e-04 | 2470.1ms/step | 9.7 samples/s
+Step 580/20005 | Loss: 4.2604 | LR: 5.00e-04 | 2466.7ms/step | 9.7 samples/s
+Step 590/20005 | Loss: 4.2515 | LR: 5.00e-04 | 2471.6ms/step | 9.7 samples/s
+Step 600/20005 | Loss: 4.2258 | LR: 5.00e-04 | 2472.6ms/step | 9.7 samples/s
+Step 610/20005 | Loss: 4.1722 | LR: 5.00e-04 | 2471.3ms/step | 9.7 samples/s
+Step 620/20005 | Loss: 4.1427 | LR: 5.00e-04 | 2471.6ms/step | 9.7 samples/s
+Step 630/20005 | Loss: 4.1107 | LR: 5.00e-04 | 2470.0ms/step | 9.7 samples/s
+Step 640/20005 | Loss: 4.1992 | LR: 5.00e-04 | 2469.4ms/step | 9.7 samples/s
+Step 650/20005 | Loss: 4.1674 | LR: 4.99e-04 | 2472.4ms/step | 9.7 samples/s
+Step 660/20005 | Loss: 4.1263 | LR: 4.99e-04 | 2471.2ms/step | 9.7 samples/s
+Step 670/20005 | Loss: 4.0386 | LR: 4.99e-04 | 2474.5ms/step | 9.7 samples/s
+Step 680/20005 | Loss: 4.0233 | LR: 4.99e-04 | 2472.6ms/step | 9.7 samples/s
+Step 690/20005 | Loss: 3.9074 | LR: 4.99e-04 | 2467.9ms/step | 9.7 samples/s
+Step 700/20005 | Loss: 3.9711 | LR: 4.99e-04 | 2467.3ms/step | 9.7 samples/s
+Step 710/20005 | Loss: 4.0219 | LR: 4.99e-04 | 2472.1ms/step | 9.7 samples/s
+Step 720/20005 | Loss: 3.9824 | LR: 4.99e-04 | 2474.1ms/step | 9.7 samples/s
+Step 730/20005 | Loss: 3.9286 | LR: 4.99e-04 | 2473.0ms/step | 9.7 samples/s
+Step 740/20005 | Loss: 3.8975 | LR: 4.99e-04 | 2475.8ms/step | 9.7 samples/s
+Step 750/20005 | Loss: 3.8868 | LR: 4.99e-04 | 2472.0ms/step | 9.7 samples/s
+Validation at step 750: Loss = 3.9122, Perplexity = 50.01
+Step 760/20005 | Loss: 3.9005 | LR: 4.99e-04 | 19998.9ms/step | 1.2 samples/s
+Step 770/20005 | Loss: 3.8754 | LR: 4.99e-04 | 2471.5ms/step | 9.7 samples/s
+Step 780/20005 | Loss: 3.9505 | LR: 4.99e-04 | 2470.8ms/step | 9.7 samples/s
+Step 790/20005 | Loss: 3.8299 | LR: 4.99e-04 | 2472.9ms/step | 9.7 samples/s
+Step 800/20005 | Loss: 3.7659 | LR: 4.99e-04 | 2471.3ms/step | 9.7 samples/s
+Step 810/20005 | Loss: 3.8095 | LR: 4.99e-04 | 2469.0ms/step | 9.7 samples/s
+Step 820/20005 | Loss: 3.8006 | LR: 4.99e-04 | 2470.9ms/step | 9.7 samples/s
+Step 830/20005 | Loss: 3.8261 | LR: 4.99e-04 | 2472.9ms/step | 9.7 samples/s
+Step 840/20005 | Loss: 3.8026 | LR: 4.99e-04 | 2469.8ms/step | 9.7 samples/s
+Step 850/20005 | Loss: 3.7642 | LR: 4.99e-04 | 2472.0ms/step | 9.7 samples/s
+Step 860/20005 | Loss: 3.7994 | LR: 4.99e-04 | 2472.7ms/step | 9.7 samples/s
+Step 870/20005 | Loss: 3.7685 | LR: 4.99e-04 | 2473.4ms/step | 9.7 samples/s
+Step 880/20005 | Loss: 3.7097 | LR: 4.99e-04 | 2470.1ms/step | 9.7 samples/s
+Step 890/20005 | Loss: 3.6376 | LR: 4.99e-04 | 2468.6ms/step | 9.7 samples/s
+Step 900/20005 | Loss: 3.7063 | LR: 4.99e-04 | 2474.3ms/step | 9.7 samples/s
+Step 910/20005 | Loss: 3.6934 | LR: 4.99e-04 | 2471.7ms/step | 9.7 samples/s
+Step 920/20005 | Loss: 3.6495 | LR: 4.99e-04 | 2470.7ms/step | 9.7 samples/s
+Step 930/20005 | Loss: 3.5923 | LR: 4.99e-04 | 2471.9ms/step | 9.7 samples/s
+Step 940/20005 | Loss: 3.7267 | LR: 4.98e-04 | 2470.2ms/step | 9.7 samples/s
+Step 950/20005 | Loss: 3.5760 | LR: 4.98e-04 | 2468.5ms/step | 9.7 samples/s
+Step 960/20005 | Loss: 3.6459 | LR: 4.98e-04 | 2469.6ms/step | 9.7 samples/s
+Step 970/20005 | Loss: 3.5800 | LR: 4.98e-04 | 2471.9ms/step | 9.7 samples/s
+Step 980/20005 | Loss: 3.5983 | LR: 4.98e-04 | 2475.0ms/step | 9.7 samples/s
+Step 990/20005 | Loss: 3.6347 | LR: 4.98e-04 | 2474.8ms/step | 9.7 samples/s
+Step 1000/20005 | Loss: 3.5700 | LR: 4.98e-04 | 2472.7ms/step | 9.7 samples/s
+Validation at step 1000: Loss = 3.6054, Perplexity = 36.79
+Step 1010/20005 | Loss: 3.5783 | LR: 4.98e-04 | 20042.8ms/step | 1.2 samples/s
+Step 1020/20005 | Loss: 3.5671 | LR: 4.98e-04 | 2473.3ms/step | 9.7 samples/s
+Step 1030/20005 | Loss: 3.5786 | LR: 4.98e-04 | 2470.0ms/step | 9.7 samples/s
+Step 1040/20005 | Loss: 3.6036 | LR: 4.98e-04 | 2572.7ms/step | 9.3 samples/s
+Step 1050/20005 | Loss: 3.4463 | LR: 4.98e-04 | 2481.5ms/step | 9.7 samples/s
+Step 1060/20005 | Loss: 3.5629 | LR: 4.98e-04 | 2484.6ms/step | 9.7 samples/s
+Step 1070/20005 | Loss: 3.5151 | LR: 4.98e-04 | 2482.3ms/step | 9.7 samples/s
+Step 1080/20005 | Loss: 3.6346 | LR: 4.98e-04 | 2482.4ms/step | 9.7 samples/s
+Step 1090/20005 | Loss: 3.5463 | LR: 4.98e-04 | 2483.7ms/step | 9.7 samples/s
+Step 1100/20005 | Loss: 3.4785 | LR: 4.98e-04 | 2485.4ms/step | 9.7 samples/s
+Step 1110/20005 | Loss: 3.5641 | LR: 4.98e-04 | 2482.7ms/step | 9.7 samples/s
+Step 1120/20005 | Loss: 3.5929 | LR: 4.98e-04 | 2481.2ms/step | 9.7 samples/s
+Step 1130/20005 | Loss: 3.5342 | LR: 4.98e-04 | 2480.7ms/step | 9.7 samples/s
+Step 1140/20005 | Loss: 3.5100 | LR: 4.98e-04 | 2483.6ms/step | 9.7 samples/s
+Step 1150/20005 | Loss: 3.5179 | LR: 4.97e-04 | 2488.2ms/step | 9.6 samples/s
+Step 1160/20005 | Loss: 3.4899 | LR: 4.97e-04 | 2485.5ms/step | 9.7 samples/s
+Step 1170/20005 | Loss: 3.5013 | LR: 4.97e-04 | 2483.8ms/step | 9.7 samples/s
+Step 1180/20005 | Loss: 3.4978 | LR: 4.97e-04 | 2482.7ms/step | 9.7 samples/s
+Step 1190/20005 | Loss: 3.5004 | LR: 4.97e-04 | 2483.5ms/step | 9.7 samples/s
+Step 1200/20005 | Loss: 3.4190 | LR: 4.97e-04 | 2484.0ms/step | 9.7 samples/s
+Step 1210/20005 | Loss: 3.3502 | LR: 4.97e-04 | 2484.1ms/step | 9.7 samples/s
+Step 1220/20005 | Loss: 3.5036 | LR: 4.97e-04 | 2483.9ms/step | 9.7 samples/s
+Step 1230/20005 | Loss: 3.4153 | LR: 4.97e-04 | 2481.5ms/step | 9.7 samples/s
+Step 1240/20005 | Loss: 3.5046 | LR: 4.97e-04 | 2480.7ms/step | 9.7 samples/s
+Step 1250/20005 | Loss: 3.3397 | LR: 4.97e-04 | 2482.6ms/step | 9.7 samples/s
+Validation at step 1250: Loss = 3.4208, Perplexity = 30.60
+Step 1260/20005 | Loss: 3.3878 | LR: 4.97e-04 | 20092.1ms/step | 1.2 samples/s
+Step 1270/20005 | Loss: 3.4112 | LR: 4.97e-04 | 2480.6ms/step | 9.7 samples/s
+Step 1280/20005 | Loss: 3.3534 | LR: 4.97e-04 | 2481.9ms/step | 9.7 samples/s
+Step 1290/20005 | Loss: 3.4114 | LR: 4.97e-04 | 2482.4ms/step | 9.7 samples/s
+Step 1300/20005 | Loss: 3.3480 | LR: 4.97e-04 | 2481.8ms/step | 9.7 samples/s
+Step 1310/20005 | Loss: 3.3368 | LR: 4.96e-04 | 2481.3ms/step | 9.7 samples/s
+Step 1320/20005 | Loss: 3.3049 | LR: 4.96e-04 | 2482.6ms/step | 9.7 samples/s
+Step 1330/20005 | Loss: 3.4321 | LR: 4.96e-04 | 2480.4ms/step | 9.7 samples/s
+Step 1340/20005 | Loss: 3.3693 | LR: 4.96e-04 | 2480.2ms/step | 9.7 samples/s
+Step 1350/20005 | Loss: 3.3928 | LR: 4.96e-04 | 2482.5ms/step | 9.7 samples/s
+Step 1360/20005 | Loss: 3.3521 | LR: 4.96e-04 | 2482.5ms/step | 9.7 samples/s
+Step 1370/20005 | Loss: 3.3616 | LR: 4.96e-04 | 2481.4ms/step | 9.7 samples/s
+Step 1380/20005 | Loss: 3.3294 | LR: 4.96e-04 | 2485.7ms/step | 9.7 samples/s
+Step 1390/20005 | Loss: 3.2962 | LR: 4.96e-04 | 2482.7ms/step | 9.7 samples/s
+Step 1400/20005 | Loss: 3.3390 | LR: 4.96e-04 | 2482.3ms/step | 9.7 samples/s
+Step 1410/20005 | Loss: 3.3442 | LR: 4.96e-04 | 2478.7ms/step | 9.7 samples/s
+Step 1420/20005 | Loss: 3.3456 | LR: 4.96e-04 | 2480.5ms/step | 9.7 samples/s
+Step 1430/20005 | Loss: 3.3379 | LR: 4.96e-04 | 2482.0ms/step | 9.7 samples/s
+Step 1440/20005 | Loss: 3.3419 | LR: 4.96e-04 | 2478.3ms/step | 9.7 samples/s
+Step 1450/20005 | Loss: 3.3965 | LR: 4.95e-04 | 2480.7ms/step | 9.7 samples/s
+Step 1460/20005 | Loss: 3.3223 | LR: 4.95e-04 | 2479.9ms/step | 9.7 samples/s
+Step 1470/20005 | Loss: 3.3176 | LR: 4.95e-04 | 2479.8ms/step | 9.7 samples/s
+Step 1480/20005 | Loss: 3.3046 | LR: 4.95e-04 | 2480.6ms/step | 9.7 samples/s
+Step 1490/20005 | Loss: 3.2875 | LR: 4.95e-04 | 2480.2ms/step | 9.7 samples/s
+Step 1500/20005 | Loss: 3.3622 | LR: 4.95e-04 | 2481.7ms/step | 9.7 samples/s
+Validation at step 1500: Loss = 3.3429, Perplexity = 28.30
+Step 1510/20005 | Loss: 3.2782 | LR: 4.95e-04 | 20084.7ms/step | 1.2 samples/s
+Step 1520/20005 | Loss: 3.3071 | LR: 4.95e-04 | 2483.2ms/step | 9.7 samples/s
+Step 1530/20005 | Loss: 3.3001 | LR: 4.95e-04 | 2481.1ms/step | 9.7 samples/s
+Step 1540/20005 | Loss: 3.2394 | LR: 4.95e-04 | 2484.8ms/step | 9.7 samples/s
+Step 1550/20005 | Loss: 3.3707 | LR: 4.95e-04 | 2484.1ms/step | 9.7 samples/s
+Step 1560/20005 | Loss: 3.2867 | LR: 4.95e-04 | 2480.0ms/step | 9.7 samples/s
+Step 1570/20005 | Loss: 3.2819 | LR: 4.95e-04 | 2481.8ms/step | 9.7 samples/s
+Step 1580/20005 | Loss: 3.2988 | LR: 4.94e-04 | 2480.3ms/step | 9.7 samples/s
+Step 1590/20005 | Loss: 3.2332 | LR: 4.94e-04 | 2480.0ms/step | 9.7 samples/s
+Step 1600/20005 | Loss: 3.3096 | LR: 4.94e-04 | 2481.9ms/step | 9.7 samples/s
+Step 1610/20005 | Loss: 3.2179 | LR: 4.94e-04 | 2484.5ms/step | 9.7 samples/s
+Step 1620/20005 | Loss: 3.2699 | LR: 4.94e-04 | 2479.5ms/step | 9.7 samples/s
+Step 1630/20005 | Loss: 3.2273 | LR: 4.94e-04 | 2482.2ms/step | 9.7 samples/s
+Step 1640/20005 | Loss: 3.3416 | LR: 4.94e-04 | 2481.6ms/step | 9.7 samples/s
+Step 1650/20005 | Loss: 3.3088 | LR: 4.94e-04 | 2483.9ms/step | 9.7 samples/s
+Step 1660/20005 | Loss: 3.2767 | LR: 4.94e-04 | 2485.8ms/step | 9.7 samples/s
+Step 1670/20005 | Loss: 3.1671 | LR: 4.94e-04 | 2485.2ms/step | 9.7 samples/s
+Step 1680/20005 | Loss: 3.3102 | LR: 4.94e-04 | 2482.1ms/step | 9.7 samples/s
+Step 1690/20005 | Loss: 3.1658 | LR: 4.93e-04 | 2480.9ms/step | 9.7 samples/s
+Step 1700/20005 | Loss: 3.2453 | LR: 4.93e-04 | 2484.1ms/step | 9.7 samples/s
+Step 1710/20005 | Loss: 3.1070 | LR: 4.93e-04 | 2483.1ms/step | 9.7 samples/s
+Step 1720/20005 | Loss: 3.1972 | LR: 4.93e-04 | 2480.9ms/step | 9.7 samples/s
+Step 1730/20005 | Loss: 3.2333 | LR: 4.93e-04 | 2480.2ms/step | 9.7 samples/s
+Step 1740/20005 | Loss: 3.1906 | LR: 4.93e-04 | 2485.7ms/step | 9.7 samples/s
+Step 1750/20005 | Loss: 3.2600 | LR: 4.93e-04 | 2484.0ms/step | 9.7 samples/s
+Validation at step 1750: Loss = 3.2076, Perplexity = 24.72
+Step 1760/20005 | Loss: 3.2116 | LR: 4.93e-04 | 20073.0ms/step | 1.2 samples/s
+Step 1770/20005 | Loss: 3.2311 | LR: 4.93e-04 | 2482.8ms/step | 9.7 samples/s
+Step 1780/20005 | Loss: 3.2018 | LR: 4.93e-04 | 2480.3ms/step | 9.7 samples/s
+Step 1790/20005 | Loss: 3.1756 | LR: 4.93e-04 | 2480.4ms/step | 9.7 samples/s
+Step 1800/20005 | Loss: 3.1392 | LR: 4.92e-04 | 2482.7ms/step | 9.7 samples/s
+Step 1810/20005 | Loss: 3.2799 | LR: 4.92e-04 | 2482.7ms/step | 9.7 samples/s
+Step 1820/20005 | Loss: 3.1527 | LR: 4.92e-04 | 2483.2ms/step | 9.7 samples/s
+Step 1830/20005 | Loss: 3.2052 | LR: 4.92e-04 | 2582.8ms/step | 9.3 samples/s
+Step 1840/20005 | Loss: 3.1211 | LR: 4.92e-04 | 2482.1ms/step | 9.7 samples/s
+Step 1850/20005 | Loss: 3.1366 | LR: 4.92e-04 | 2483.4ms/step | 9.7 samples/s
+Step 1860/20005 | Loss: 3.1339 | LR: 4.92e-04 | 2485.6ms/step | 9.7 samples/s
+Step 1870/20005 | Loss: 3.1281 | LR: 4.92e-04 | 2481.2ms/step | 9.7 samples/s
+Step 1880/20005 | Loss: 3.1423 | LR: 4.92e-04 | 2480.9ms/step | 9.7 samples/s
+Step 1890/20005 | Loss: 3.1088 | LR: 4.92e-04 | 2484.9ms/step | 9.7 samples/s
+Step 1900/20005 | Loss: 3.2010 | LR: 4.91e-04 | 2480.0ms/step | 9.7 samples/s
+Step 1910/20005 | Loss: 3.1371 | LR: 4.91e-04 | 2481.1ms/step | 9.7 samples/s
+Step 1920/20005 | Loss: 3.1164 | LR: 4.91e-04 | 2482.7ms/step | 9.7 samples/s
+Step 1930/20005 | Loss: 3.1960 | LR: 4.91e-04 | 2482.0ms/step | 9.7 samples/s
+Step 1940/20005 | Loss: 3.0022 | LR: 4.91e-04 | 2482.1ms/step | 9.7 samples/s
+Step 1950/20005 | Loss: 3.1884 | LR: 4.91e-04 | 2484.2ms/step | 9.7 samples/s
+Step 1960/20005 | Loss: 3.1131 | LR: 4.91e-04 | 2481.6ms/step | 9.7 samples/s
+Step 1970/20005 | Loss: 3.1066 | LR: 4.91e-04 | 2480.0ms/step | 9.7 samples/s
+Step 1980/20005 | Loss: 3.0815 | LR: 4.91e-04 | 2481.2ms/step | 9.7 samples/s
+Step 1990/20005 | Loss: 3.0854 | LR: 4.90e-04 | 2485.2ms/step | 9.7 samples/s
+Step 2000/20005 | Loss: 3.1739 | LR: 4.90e-04 | 2483.9ms/step | 9.7 samples/s
+Validation at step 2000: Loss = 3.1330, Perplexity = 22.94
+Saved checkpoint to checkpoints/run_20260506_171628/checkpoint-2000
+Step 2010/20005 | Loss: 3.1899 | LR: 4.90e-04 | 20595.1ms/step | 1.2 samples/s
+Step 2020/20005 | Loss: 3.1322 | LR: 4.90e-04 | 2483.0ms/step | 9.7 samples/s
+Step 2030/20005 | Loss: 3.1557 | LR: 4.90e-04 | 2483.2ms/step | 9.7 samples/s
+Step 2040/20005 | Loss: 3.1301 | LR: 4.90e-04 | 2481.5ms/step | 9.7 samples/s
+Step 2050/20005 | Loss: 3.0754 | LR: 4.90e-04 | 2484.8ms/step | 9.7 samples/s
+Step 2060/20005 | Loss: 3.0958 | LR: 4.90e-04 | 2482.1ms/step | 9.7 samples/s
+Step 2070/20005 | Loss: 3.1173 | LR: 4.90e-04 | 2482.0ms/step | 9.7 samples/s
+Step 2080/20005 | Loss: 3.1208 | LR: 4.89e-04 | 2479.9ms/step | 9.7 samples/s
+Step 2090/20005 | Loss: 3.0896 | LR: 4.89e-04 | 2479.4ms/step | 9.7 samples/s
+Step 2100/20005 | Loss: 3.0643 | LR: 4.89e-04 | 2480.4ms/step | 9.7 samples/s
+Step 2110/20005 | Loss: 3.1077 | LR: 4.89e-04 | 2480.6ms/step | 9.7 samples/s
+Step 2120/20005 | Loss: 3.0934 | LR: 4.89e-04 | 2481.7ms/step | 9.7 samples/s
+Step 2130/20005 | Loss: 3.2026 | LR: 4.89e-04 | 2482.3ms/step | 9.7 samples/s
+Step 2140/20005 | Loss: 2.9999 | LR: 4.89e-04 | 2481.5ms/step | 9.7 samples/s
+Step 2150/20005 | Loss: 3.1597 | LR: 4.89e-04 | 2478.5ms/step | 9.7 samples/s
+Step 2160/20005 | Loss: 3.1149 | LR: 4.89e-04 | 2479.0ms/step | 9.7 samples/s
+Step 2170/20005 | Loss: 3.0599 | LR: 4.88e-04 | 2480.0ms/step | 9.7 samples/s
+Step 2180/20005 | Loss: 3.0751 | LR: 4.88e-04 | 2478.7ms/step | 9.7 samples/s
+Step 2190/20005 | Loss: 3.0508 | LR: 4.88e-04 | 2481.1ms/step | 9.7 samples/s
+Step 2200/20005 | Loss: 3.0565 | LR: 4.88e-04 | 2482.9ms/step | 9.7 samples/s
+Step 2210/20005 | Loss: 3.0679 | LR: 4.88e-04 | 2479.5ms/step | 9.7 samples/s
+Step 2220/20005 | Loss: 3.0740 | LR: 4.88e-04 | 2480.0ms/step | 9.7 samples/s
+Step 2230/20005 | Loss: 3.0267 | LR: 4.88e-04 | 2482.2ms/step | 9.7 samples/s
+Step 2240/20005 | Loss: 3.0273 | LR: 4.88e-04 | 2480.1ms/step | 9.7 samples/s
+Step 2250/20005 | Loss: 3.1010 | LR: 4.87e-04 | 2484.6ms/step | 9.7 samples/s
+Validation at step 2250: Loss = 3.0659, Perplexity = 21.45
+Step 2260/20005 | Loss: 3.0495 | LR: 4.87e-04 | 20211.6ms/step | 1.2 samples/s
+Step 2270/20005 | Loss: 3.0487 | LR: 4.87e-04 | 2485.8ms/step | 9.7 samples/s
+Step 2280/20005 | Loss: 3.0875 | LR: 4.87e-04 | 2485.9ms/step | 9.7 samples/s
+Step 2290/20005 | Loss: 3.0081 | LR: 4.87e-04 | 2482.6ms/step | 9.7 samples/s
+Step 2300/20005 | Loss: 3.0695 | LR: 4.87e-04 | 2485.7ms/step | 9.7 samples/s
+Step 2310/20005 | Loss: 3.0367 | LR: 4.87e-04 | 2482.3ms/step | 9.7 samples/s
+Step 2320/20005 | Loss: 2.9226 | LR: 4.87e-04 | 2481.8ms/step | 9.7 samples/s
+Step 2330/20005 | Loss: 3.0387 | LR: 4.86e-04 | 2482.7ms/step | 9.7 samples/s
+Step 2340/20005 | Loss: 3.0105 | LR: 4.86e-04 | 2484.5ms/step | 9.7 samples/s
+Step 2350/20005 | Loss: 3.0209 | LR: 4.86e-04 | 2486.6ms/step | 9.7 samples/s
+Step 2360/20005 | Loss: 3.0097 | LR: 4.86e-04 | 2490.4ms/step | 9.6 samples/s
+Step 2370/20005 | Loss: 3.0632 | LR: 4.86e-04 | 2485.8ms/step | 9.7 samples/s
+Step 2380/20005 | Loss: 3.0618 | LR: 4.86e-04 | 2485.3ms/step | 9.7 samples/s
+Step 2390/20005 | Loss: 3.0674 | LR: 4.86e-04 | 2488.0ms/step | 9.6 samples/s
+Step 2400/20005 | Loss: 3.0802 | LR: 4.86e-04 | 2485.4ms/step | 9.7 samples/s
+Step 2410/20005 | Loss: 2.9627 | LR: 4.85e-04 | 2487.1ms/step | 9.6 samples/s
+Step 2420/20005 | Loss: 3.0272 | LR: 4.85e-04 | 2485.9ms/step | 9.7 samples/s
+Step 2430/20005 | Loss: 3.0372 | LR: 4.85e-04 | 2486.8ms/step | 9.7 samples/s
+Step 2440/20005 | Loss: 3.1485 | LR: 4.85e-04 | 2483.2ms/step | 9.7 samples/s
+Step 2450/20005 | Loss: 3.0323 | LR: 4.85e-04 | 2483.8ms/step | 9.7 samples/s
+Step 2460/20005 | Loss: 2.9436 | LR: 4.85e-04 | 2484.7ms/step | 9.7 samples/s
+Step 2470/20005 | Loss: 2.9838 | LR: 4.85e-04 | 2485.8ms/step | 9.7 samples/s
+Step 2480/20005 | Loss: 3.0096 | LR: 4.84e-04 | 2485.2ms/step | 9.7 samples/s
+Step 2490/20005 | Loss: 3.0263 | LR: 4.84e-04 | 2483.6ms/step | 9.7 samples/s
+Step 2500/20005 | Loss: 3.0297 | LR: 4.84e-04 | 2486.2ms/step | 9.7 samples/s
+Validation at step 2500: Loss = 3.0174, Perplexity = 20.44
+Step 2510/20005 | Loss: 3.0233 | LR: 4.84e-04 | 20150.6ms/step | 1.2 samples/s
+Step 2520/20005 | Loss: 2.9520 | LR: 4.84e-04 | 2486.3ms/step | 9.7 samples/s
+Step 2530/20005 | Loss: 2.9792 | LR: 4.84e-04 | 2487.3ms/step | 9.6 samples/s
+Step 2540/20005 | Loss: 3.0410 | LR: 4.84e-04 | 2481.8ms/step | 9.7 samples/s
+Step 2550/20005 | Loss: 3.0117 | LR: 4.83e-04 | 2483.7ms/step | 9.7 samples/s
+Step 2560/20005 | Loss: 2.9952 | LR: 4.83e-04 | 2482.3ms/step | 9.7 samples/s
+Step 2570/20005 | Loss: 3.0182 | LR: 4.83e-04 | 2486.8ms/step | 9.7 samples/s
+Step 2580/20005 | Loss: 2.9531 | LR: 4.83e-04 | 2484.5ms/step | 9.7 samples/s
+Step 2590/20005 | Loss: 3.0715 | LR: 4.83e-04 | 2483.8ms/step | 9.7 samples/s
+Step 2600/20005 | Loss: 2.9748 | LR: 4.83e-04 | 2486.1ms/step | 9.7 samples/s
+Step 2610/20005 | Loss: 2.9719 | LR: 4.83e-04 | 2484.1ms/step | 9.7 samples/s
+Step 2620/20005 | Loss: 3.0272 | LR: 4.82e-04 | 2641.3ms/step | 9.1 samples/s
+Step 2630/20005 | Loss: 2.9623 | LR: 4.82e-04 | 2483.6ms/step | 9.7 samples/s
+Step 2640/20005 | Loss: 2.9279 | LR: 4.82e-04 | 2487.8ms/step | 9.6 samples/s
+Step 2650/20005 | Loss: 3.0301 | LR: 4.82e-04 | 2486.4ms/step | 9.7 samples/s
+Step 2660/20005 | Loss: 2.9855 | LR: 4.82e-04 | 2485.6ms/step | 9.7 samples/s
+Step 2670/20005 | Loss: 2.9391 | LR: 4.82e-04 | 2484.7ms/step | 9.7 samples/s
+Step 2680/20005 | Loss: 2.9792 | LR: 4.82e-04 | 2491.4ms/step | 9.6 samples/s
+Step 2690/20005 | Loss: 2.9522 | LR: 4.81e-04 | 2489.8ms/step | 9.6 samples/s
+Step 2700/20005 | Loss: 2.9745 | LR: 4.81e-04 | 2487.5ms/step | 9.6 samples/s
+Step 2710/20005 | Loss: 2.9118 | LR: 4.81e-04 | 2491.0ms/step | 9.6 samples/s
+Step 2720/20005 | Loss: 2.9993 | LR: 4.81e-04 | 2478.8ms/step | 9.7 samples/s
+Step 2730/20005 | Loss: 3.0726 | LR: 4.81e-04 | 2481.5ms/step | 9.7 samples/s
+Step 2740/20005 | Loss: 2.9389 | LR: 4.81e-04 | 2486.7ms/step | 9.7 samples/s
+Step 2750/20005 | Loss: 2.9551 | LR: 4.81e-04 | 2489.4ms/step | 9.6 samples/s
+Validation at step 2750: Loss = 2.9700, Perplexity = 19.49
+Step 2760/20005 | Loss: 2.9470 | LR: 4.80e-04 | 20153.4ms/step | 1.2 samples/s
+Step 2770/20005 | Loss: 3.0599 | LR: 4.80e-04 | 2489.6ms/step | 9.6 samples/s
+Step 2780/20005 | Loss: 2.9620 | LR: 4.80e-04 | 2484.6ms/step | 9.7 samples/s
+Step 2790/20005 | Loss: 3.0053 | LR: 4.80e-04 | 2482.9ms/step | 9.7 samples/s
+Step 2800/20005 | Loss: 2.9547 | LR: 4.80e-04 | 2484.7ms/step | 9.7 samples/s
+Step 2810/20005 | Loss: 2.9998 | LR: 4.80e-04 | 2487.3ms/step | 9.6 samples/s
+Step 2820/20005 | Loss: 3.0059 | LR: 4.79e-04 | 2488.8ms/step | 9.6 samples/s
+Step 2830/20005 | Loss: 2.8953 | LR: 4.79e-04 | 2482.8ms/step | 9.7 samples/s
+Step 2840/20005 | Loss: 2.9341 | LR: 4.79e-04 | 2487.6ms/step | 9.6 samples/s
+Step 2850/20005 | Loss: 2.8997 | LR: 4.79e-04 | 2486.9ms/step | 9.7 samples/s
+Step 2860/20005 | Loss: 2.8518 | LR: 4.79e-04 | 2486.5ms/step | 9.7 samples/s
+Step 2870/20005 | Loss: 3.0818 | LR: 4.79e-04 | 2485.5ms/step | 9.7 samples/s
+Step 2880/20005 | Loss: 2.9127 | LR: 4.78e-04 | 2482.8ms/step | 9.7 samples/s
+Step 2890/20005 | Loss: 2.8781 | LR: 4.78e-04 | 2483.6ms/step | 9.7 samples/s
+Step 2900/20005 | Loss: 2.9468 | LR: 4.78e-04 | 2484.3ms/step | 9.7 samples/s
+Step 2910/20005 | Loss: 2.8944 | LR: 4.78e-04 | 2485.6ms/step | 9.7 samples/s
+Step 2920/20005 | Loss: 2.9003 | LR: 4.78e-04 | 2487.8ms/step | 9.6 samples/s
+Step 2930/20005 | Loss: 2.9508 | LR: 4.78e-04 | 2487.2ms/step | 9.6 samples/s
+Step 2940/20005 | Loss: 2.9581 | LR: 4.77e-04 | 2484.5ms/step | 9.7 samples/s
+Step 2950/20005 | Loss: 2.8649 | LR: 4.77e-04 | 2485.0ms/step | 9.7 samples/s
+Step 2960/20005 | Loss: 2.9527 | LR: 4.77e-04 | 2488.1ms/step | 9.6 samples/s
+Step 2970/20005 | Loss: 2.9560 | LR: 4.77e-04 | 2487.6ms/step | 9.6 samples/s
+Step 2980/20005 | Loss: 2.9454 | LR: 4.77e-04 | 2485.0ms/step | 9.7 samples/s
+Step 2990/20005 | Loss: 2.9172 | LR: 4.77e-04 | 2483.9ms/step | 9.7 samples/s
+Step 3000/20005 | Loss: 2.8768 | LR: 4.76e-04 | 2483.7ms/step | 9.7 samples/s
+Validation at step 3000: Loss = 2.9385, Perplexity = 18.89
+Step 3010/20005 | Loss: 2.9027 | LR: 4.76e-04 | 20163.4ms/step | 1.2 samples/s
+Step 3020/20005 | Loss: 2.8841 | LR: 4.76e-04 | 2490.1ms/step | 9.6 samples/s
+Step 3030/20005 | Loss: 2.8952 | LR: 4.76e-04 | 2490.2ms/step | 9.6 samples/s
+Step 3040/20005 | Loss: 2.9458 | LR: 4.76e-04 | 2486.3ms/step | 9.7 samples/s
+Step 3050/20005 | Loss: 3.0163 | LR: 4.76e-04 | 2488.3ms/step | 9.6 samples/s
+Step 3060/20005 | Loss: 2.8302 | LR: 4.75e-04 | 2483.8ms/step | 9.7 samples/s
+Step 3070/20005 | Loss: 2.9645 | LR: 4.75e-04 | 2485.0ms/step | 9.7 samples/s
+Step 3080/20005 | Loss: 2.8695 | LR: 4.75e-04 | 2486.8ms/step | 9.7 samples/s
+Step 3090/20005 | Loss: 2.9766 | LR: 4.75e-04 | 2487.1ms/step | 9.6 samples/s
+Step 3100/20005 | Loss: 2.9351 | LR: 4.75e-04 | 2488.3ms/step | 9.6 samples/s
+Step 3110/20005 | Loss: 2.8867 | LR: 4.75e-04 | 2489.0ms/step | 9.6 samples/s
+Step 3120/20005 | Loss: 2.9147 | LR: 4.74e-04 | 2486.7ms/step | 9.7 samples/s
+Step 3130/20005 | Loss: 2.9249 | LR: 4.74e-04 | 2489.3ms/step | 9.6 samples/s
+Step 3140/20005 | Loss: 2.9412 | LR: 4.74e-04 | 2487.1ms/step | 9.6 samples/s
+Step 3150/20005 | Loss: 2.9122 | LR: 4.74e-04 | 2490.4ms/step | 9.6 samples/s
+Step 3160/20005 | Loss: 2.9904 | LR: 4.74e-04 | 2489.2ms/step | 9.6 samples/s
+Step 3170/20005 | Loss: 2.8860 | LR: 4.74e-04 | 2483.5ms/step | 9.7 samples/s
+Step 3180/20005 | Loss: 2.8189 | LR: 4.73e-04 | 2489.9ms/step | 9.6 samples/s
+Step 3190/20005 | Loss: 2.8932 | LR: 4.73e-04 | 2487.6ms/step | 9.6 samples/s
+Step 3200/20005 | Loss: 2.9586 | LR: 4.73e-04 | 2482.2ms/step | 9.7 samples/s
+Step 3210/20005 | Loss: 2.7848 | LR: 4.73e-04 | 2484.2ms/step | 9.7 samples/s
+Step 3220/20005 | Loss: 2.8996 | LR: 4.73e-04 | 2485.9ms/step | 9.7 samples/s
+Step 3230/20005 | Loss: 2.9051 | LR: 4.72e-04 | 2486.1ms/step | 9.7 samples/s
+Step 3240/20005 | Loss: 2.8546 | LR: 4.72e-04 | 2484.9ms/step | 9.7 samples/s
+Step 3250/20005 | Loss: 2.8317 | LR: 4.72e-04 | 2481.8ms/step | 9.7 samples/s
+Validation at step 3250: Loss = 2.8979, Perplexity = 18.14
+Step 3260/20005 | Loss: 2.8391 | LR: 4.72e-04 | 20153.4ms/step | 1.2 samples/s
+Step 3270/20005 | Loss: 2.8688 | LR: 4.72e-04 | 2487.9ms/step | 9.6 samples/s
+Step 3280/20005 | Loss: 2.8271 | LR: 4.72e-04 | 2486.2ms/step | 9.7 samples/s
+Step 3290/20005 | Loss: 2.8906 | LR: 4.71e-04 | 2486.1ms/step | 9.7 samples/s
+Step 3300/20005 | Loss: 2.8870 | LR: 4.71e-04 | 2486.2ms/step | 9.7 samples/s
+Step 3310/20005 | Loss: 2.8872 | LR: 4.71e-04 | 2486.1ms/step | 9.7 samples/s
+Step 3320/20005 | Loss: 2.8777 | LR: 4.71e-04 | 2485.0ms/step | 9.7 samples/s
+Step 3330/20005 | Loss: 2.9292 | LR: 4.71e-04 | 2482.9ms/step | 9.7 samples/s
+Step 3340/20005 | Loss: 2.9075 | LR: 4.70e-04 | 2486.3ms/step | 9.7 samples/s
+Step 3350/20005 | Loss: 2.8418 | LR: 4.70e-04 | 2485.6ms/step | 9.7 samples/s
+Step 3360/20005 | Loss: 2.8374 | LR: 4.70e-04 | 2485.8ms/step | 9.7 samples/s
+Step 3370/20005 | Loss: 2.8806 | LR: 4.70e-04 | 2486.6ms/step | 9.7 samples/s
+Step 3380/20005 | Loss: 2.8556 | LR: 4.70e-04 | 2486.1ms/step | 9.7 samples/s
+Step 3390/20005 | Loss: 2.8850 | LR: 4.69e-04 | 2483.9ms/step | 9.7 samples/s
+Step 3400/20005 | Loss: 2.7990 | LR: 4.69e-04 | 2487.9ms/step | 9.6 samples/s
+Step 3410/20005 | Loss: 2.8183 | LR: 4.69e-04 | 2488.4ms/step | 9.6 samples/s
+Step 3420/20005 | Loss: 2.8690 | LR: 4.69e-04 | 2585.4ms/step | 9.3 samples/s
+Step 3430/20005 | Loss: 2.9224 | LR: 4.69e-04 | 2487.0ms/step | 9.7 samples/s
+Step 3440/20005 | Loss: 2.8586 | LR: 4.69e-04 | 2490.0ms/step | 9.6 samples/s
+Step 3450/20005 | Loss: 2.8856 | LR: 4.68e-04 | 2487.9ms/step | 9.6 samples/s
+Step 3460/20005 | Loss: 2.8790 | LR: 4.68e-04 | 2486.1ms/step | 9.7 samples/s
+Step 3470/20005 | Loss: 2.9077 | LR: 4.68e-04 | 2487.2ms/step | 9.6 samples/s
+Step 3480/20005 | Loss: 2.8669 | LR: 4.68e-04 | 2487.2ms/step | 9.6 samples/s
+Step 3490/20005 | Loss: 2.8640 | LR: 4.68e-04 | 2488.6ms/step | 9.6 samples/s
+Step 3500/20005 | Loss: 2.7557 | LR: 4.67e-04 | 2491.2ms/step | 9.6 samples/s
+Validation at step 3500: Loss = 2.8650, Perplexity = 17.55
+Step 3510/20005 | Loss: 2.9039 | LR: 4.67e-04 | 20195.0ms/step | 1.2 samples/s
+Step 3520/20005 | Loss: 2.8394 | LR: 4.67e-04 | 2491.3ms/step | 9.6 samples/s
+Step 3530/20005 | Loss: 2.7960 | LR: 4.67e-04 | 2486.9ms/step | 9.7 samples/s
+Step 3540/20005 | Loss: 2.8816 | LR: 4.67e-04 | 2486.8ms/step | 9.7 samples/s
+Step 3550/20005 | Loss: 2.8719 | LR: 4.66e-04 | 2488.1ms/step | 9.6 samples/s
+Step 3560/20005 | Loss: 2.7991 | LR: 4.66e-04 | 2488.5ms/step | 9.6 samples/s
+Step 3570/20005 | Loss: 2.8148 | LR: 4.66e-04 | 2486.1ms/step | 9.7 samples/s
+Step 3580/20005 | Loss: 2.7765 | LR: 4.66e-04 | 2488.7ms/step | 9.6 samples/s
+Step 3590/20005 | Loss: 2.8594 | LR: 4.66e-04 | 2487.3ms/step | 9.6 samples/s
+Step 3600/20005 | Loss: 2.8868 | LR: 4.65e-04 | 2487.5ms/step | 9.6 samples/s
+Step 3610/20005 | Loss: 2.9076 | LR: 4.65e-04 | 2485.3ms/step | 9.7 samples/s
+Step 3620/20005 | Loss: 2.8390 | LR: 4.65e-04 | 2490.2ms/step | 9.6 samples/s
+Step 3630/20005 | Loss: 2.9418 | LR: 4.65e-04 | 2489.0ms/step | 9.6 samples/s
+Step 3640/20005 | Loss: 2.8458 | LR: 4.65e-04 | 2488.1ms/step | 9.6 samples/s
+Step 3650/20005 | Loss: 2.9035 | LR: 4.64e-04 | 2487.4ms/step | 9.6 samples/s
+Step 3660/20005 | Loss: 2.8621 | LR: 4.64e-04 | 2486.7ms/step | 9.7 samples/s
+Step 3670/20005 | Loss: 2.8240 | LR: 4.64e-04 | 2487.5ms/step | 9.6 samples/s
+Step 3680/20005 | Loss: 2.7992 | LR: 4.64e-04 | 2486.5ms/step | 9.7 samples/s
+Step 3690/20005 | Loss: 2.8206 | LR: 4.64e-04 | 2487.2ms/step | 9.6 samples/s
+Step 3700/20005 | Loss: 2.9252 | LR: 4.63e-04 | 2488.6ms/step | 9.6 samples/s
+Step 3710/20005 | Loss: 2.8458 | LR: 4.63e-04 | 2487.8ms/step | 9.6 samples/s
+Step 3720/20005 | Loss: 2.7925 | LR: 4.63e-04 | 2486.8ms/step | 9.7 samples/s
+Step 3730/20005 | Loss: 2.7895 | LR: 4.63e-04 | 2486.2ms/step | 9.7 samples/s
+Step 3740/20005 | Loss: 2.8958 | LR: 4.62e-04 | 2486.5ms/step | 9.7 samples/s
+Step 3750/20005 | Loss: 2.7859 | LR: 4.62e-04 | 2487.0ms/step | 9.7 samples/s

epfl_final/log_files/build_mask_19.log

@@ -0,0 +1,14 @@
+[build_masks] Config:
+  sparse layers : [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
+  seq_len       : 8192
+  block_size    : 128
+  num_blocks    : 64
+  num_random    : 18
+  mask_pool_size: 2000
+  num_workers   : 192
+  cache_dir     : ./mask_cache_19
+  force_rebuild : False
+
+[build_masks] Building 2000 masks × 28 layers on 1 GPU(s). Est. 140s/layer on GPU.
+
  0%|          | 0/28 [00:00<?, ?layer/s]
  0%|          | 0/28 [01:22<?, ?layer/s, elapsed=83s, layer=0]
  4%|▎         | 1/28 [01:22<37:15, 82.80s/layer, elapsed=83s, layer=0]
  4%|▎         | 1/28 [02:45<37:15, 82.80s/layer, elapsed=165s, layer=1]
  7%|▋         | 2/28 [02:45<35:47, 82.61s/layer, elapsed=165s, layer=1]
  7%|▋         | 2/28 [04:07<35:47, 82.61s/layer, elapsed=248s, layer=2]
 11%|█         | 3/28 [04:07<34:21, 82.47s/layer, elapsed=248s, layer=2]
 11%|█         | 3/28 [05:29<34:21, 82.47s/layer, elapsed=330s, layer=3]
 14%|█▍        | 4/28 [05:29<32:58, 82.42s/layer, elapsed=330s, layer=3]
 14%|█▍        | 4/28 [06:51<32:58, 82.42s/layer, elapsed=412s, layer=4]
 18%|█▊        | 5/28 [06:51<31:30, 82.18s/layer, elapsed=412s, layer=4]
 18%|█▊        | 5/28 [08:13<31:30, 82.18s/layer, elapsed=494s, layer=5]
 21%|██▏       | 6/28 [08:13<30:06, 82.13s/layer, elapsed=494s, layer=5]
 21%|██▏       | 6/28 [09:35<30:06, 82.13s/layer, elapsed=575s, layer=6]
 25%|██▌       | 7/28 [09:35<28:42, 82.01s/layer, elapsed=575s, layer=6]
 25%|██▌       | 7/28 [10:57<28:42, 82.01s/layer, elapsed=657s, layer=7]
 29%|██▊       | 8/28 [10:57<27:19, 81.97s/layer, elapsed=657s, layer=7]
 29%|██▊       | 8/28 [12:19<27:19, 81.97s/layer, elapsed=740s, layer=8]
 32%|███▏      | 9/28 [12:19<25:59, 82.08s/layer, elapsed=740s, layer=8]
 32%|███▏      | 9/28 [13:41<25:59, 82.08s/layer, elapsed=822s, layer=9]
 36%|███▌      | 10/28 [13:41<24:38, 82.14s/layer, elapsed=822s, layer=9]
 36%|███▌      | 10/28 [15:04<24:38, 82.14s/layer, elapsed=905s, layer=10]
 39%|███▉      | 11/28 [15:04<23:19, 82.30s/layer, elapsed=905s, layer=10]
 39%|███▉      | 11/28 [16:26<23:19, 82.30s/layer, elapsed=987s, layer=11]
 43%|████▎     | 12/28 [16:26<21:56, 82.30s/layer, elapsed=987s, layer=11]
 43%|████▎     | 12/28 [17:48<21:56, 82.30s/layer, elapsed=1069s, layer=12]
 46%|████▋     | 13/28 [17:48<20:31, 82.12s/layer, elapsed=1069s, layer=12]
 46%|████▋     | 13/28 [19:10<20:31, 82.12s/layer, elapsed=1150s, layer=13]
 50%|█████     | 14/28 [19:10<19:07, 81.97s/layer, elapsed=1150s, layer=13]
 50%|█████     | 14/28 [20:32<19:07, 81.97s/layer, elapsed=1232s, layer=14]
 54%|█████▎    | 15/28 [20:32<17:45, 81.97s/layer, elapsed=1232s, layer=14]
 54%|█████▎    | 15/28 [21:54<17:45, 81.97s/layer, elapsed=1315s, layer=15]
 57%|█████▋    | 16/28 [21:54<16:24, 82.07s/layer, elapsed=1315s, layer=15]
 57%|█████▋    | 16/28 [23:16<16:24, 82.07s/layer, elapsed=1397s, layer=16]
 61%|██████    | 17/28 [23:16<15:02, 82.06s/layer, elapsed=1397s, layer=16]
 61%|██████    | 17/28 [24:38<15:02, 82.06s/layer, elapsed=1478s, layer=17]
 64%|██████▍   | 18/28 [24:38<13:39, 82.00s/layer, elapsed=1478s, layer=17]
 64%|██████▍   | 18/28 [25:59<13:39, 82.00s/layer, elapsed=1560s, layer=18]
 68%|██████▊   | 19/28 [25:59<12:16, 81.87s/layer, elapsed=1560s, layer=18]
 68%|██████▊   | 19/28 [27:22<12:16, 81.87s/layer, elapsed=1642s, layer=19]
 71%|███████▏  | 20/28 [27:22<10:56, 82.05s/layer, elapsed=1642s, layer=19]
 71%|███████▏  | 20/28 [28:44<10:56, 82.05s/layer, elapsed=1724s, layer=20]
 75%|███████▌  | 21/28 [28:44<09:33, 81.94s/layer, elapsed=1724s, layer=20]
 75%|███████▌  | 21/28 [30:06<09:33, 81.94s/layer, elapsed=1806s, layer=21]
 79%|███████▊  | 22/28 [30:06<08:11, 81.95s/layer, elapsed=1806s, layer=21]
 79%|███████▊  | 22/28 [31:28<08:11, 81.95s/layer, elapsed=1888s, layer=22]
 82%|████████▏ | 23/28 [31:28<06:50, 82.03s/layer, elapsed=1888s, layer=22]
 82%|████████▏ | 23/28 [32:50<06:50, 82.03s/layer, elapsed=1970s, layer=23]
 86%|████████▌ | 24/28 [32:50<05:27, 81.97s/layer, elapsed=1970s, layer=23]
 86%|████████▌ | 24/28 [34:11<05:27, 81.97s/layer, elapsed=2052s, layer=24]
 89%|████████▉ | 25/28 [34:11<04:05, 81.86s/layer, elapsed=2052s, layer=24]
 89%|████████▉ | 25/28 [35:33<04:05, 81.86s/layer, elapsed=2134s, layer=25]
 93%|█████████▎| 26/28 [35:33<02:43, 81.89s/layer, elapsed=2134s, layer=25]
 93%|█████████▎| 26/28 [36:55<02:43, 81.89s/layer, elapsed=2215s, layer=26]
 96%|█████████▋| 27/28 [36:55<01:21, 81.79s/layer, elapsed=2215s, layer=26]
 96%|█████████��| 27/28 [38:17<01:21, 81.79s/layer, elapsed=2297s, layer=27]
+[build_masks] Done. Total wall-time: 2297.4s

epfl_final/log_files/building_masks_16.log

@@ -0,0 +1,41 @@
+[build_masks] Config:
+  sparse layers : [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
+  seq_len       : 8192
+  block_size    : 128
+  num_blocks    : 64
+  num_random    : 15
+  mask_pool_size: 10000
+  num_workers   : 192
+  cache_dir     : /mloscratch/homes/navasard/sparse/mask_chache_16
+  force_rebuild : False
+
+[build_masks] Building 10000 masks × 28 layers on 8 GPU(s). Est. 700s/layer on GPU.
+[build_masks] GPU 7: layer 7 done.
+[build_masks] GPU 5: layer 5 done.
+[build_masks] GPU 3: layer 3 done.
+[build_masks] GPU 4: layer 4 done.
+[build_masks] GPU 6: layer 6 done.
+[build_masks] GPU 2: layer 2 done.
+[build_masks] GPU 1: layer 1 done.
+[build_masks] GPU 0: layer 0 done.
+[build_masks] GPU 7: layer 15 done.
+[build_masks] GPU 5: layer 13 done.
+[build_masks] GPU 3: layer 11 done.
+[build_masks] GPU 4: layer 12 done.
+[build_masks] GPU 6: layer 14 done.
+[build_masks] GPU 1: layer 9 done.
+[build_masks] GPU 2: layer 10 done.
+[build_masks] GPU 0: layer 8 done.
+[build_masks] GPU 7: layer 23 done.
+[build_masks] GPU 5: layer 21 done.
+[build_masks] GPU 4: layer 20 done.
+[build_masks] GPU 3: layer 19 done.
+[build_masks] GPU 6: layer 22 done.
+[build_masks] GPU 1: layer 17 done.
+[build_masks] GPU 2: layer 18 done.
+[build_masks] GPU 0: layer 16 done.
+[build_masks] GPU 3: layer 27 done.
+[build_masks] GPU 1: layer 25 done.
+[build_masks] GPU 2: layer 26 done.
+[build_masks] GPU 0: layer 24 done.
+[build_masks] Done. Total wall-time: 1479.3s

epfl_final/log_files/building_masks_23.log

@@ -0,0 +1,41 @@
+[build_masks] Config:
+  sparse layers : [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
+  seq_len       : 8192
+  block_size    : 128
+  num_blocks    : 64
+  num_random    : 22
+  mask_pool_size: 10000
+  num_workers   : 192
+  cache_dir     : /mloscratch/homes/navasard/sparse/mask_chache_22
+  force_rebuild : False
+
+[build_masks] Building 10000 masks × 28 layers on 8 GPU(s). Est. 700s/layer on GPU.
+[build_masks] GPU 5: layer 5 done.
+[build_masks] GPU 7: layer 7 done.
+[build_masks] GPU 6: layer 6 done.
+[build_masks] GPU 1: layer 1 done.
+[build_masks] GPU 3: layer 3 done.
+[build_masks] GPU 0: layer 0 done.
+[build_masks] GPU 2: layer 2 done.
+[build_masks] GPU 4: layer 4 done.
+[build_masks] GPU 7: layer 15 done.
+[build_masks] GPU 6: layer 14 done.
+[build_masks] GPU 5: layer 13 done.
+[build_masks] GPU 1: layer 9 done.
+[build_masks] GPU 3: layer 11 done.
+[build_masks] GPU 0: layer 8 done.
+[build_masks] GPU 2: layer 10 done.
+[build_masks] GPU 4: layer 12 done.
+[build_masks] GPU 6: layer 22 done.
+[build_masks] GPU 7: layer 23 done.
+[build_masks] GPU 5: layer 21 done.
+[build_masks] GPU 1: layer 17 done.
+[build_masks] GPU 3: layer 19 done.
+[build_masks] GPU 0: layer 16 done.
+[build_masks] GPU 2: layer 18 done.
+[build_masks] GPU 4: layer 20 done.
+[build_masks] GPU 1: layer 25 done.
+[build_masks] GPU 3: layer 27 done.
+[build_masks] GPU 0: layer 24 done.
+[build_masks] GPU 2: layer 26 done.
+[build_masks] Done. Total wall-time: 1946.5s

epfl_final/log_files/cont_pretrain.log

@@ -0,0 +1,276 @@
+nohup: ignoring input
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+New run: Checkpoints will be saved to ./checkpoints/run_20260530_005258
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260530_005258",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000",
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0009,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 48k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000
+Checkpoint activations: False
+Checkpoint activations: False
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 1536]           [2, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [2, 8192, 1536]           [2, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (G): 2.87
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 52437.19
+Params size (MB): 2871.45
+Estimated Total Size (MB): 55308.77
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        3
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504627
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000...
+Resumed from checkpoint at step 46000 (epoch 0, micro_step 460000)
+Resuming training from step 46000
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run z0v8vk8d
+wandb: Tracking run with wandb version 0.27.0
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260530_011057-z0v8vk8d
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense continued 48k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/z0v8vk8d
+Starting training...
+Validation at step 46000: Loss = 2.5221, Perplexity = 12.46
+Step 46010/50462 | Loss: 2.5136 | LR: 9.83e-06 | 66570.4ms/step | 0.9 samples/s
+Step 46020/50462 | Loss: 2.4987 | LR: 9.79e-06 | 5203.9ms/step | 11.5 samples/s
+Step 46030/50462 | Loss: 2.5210 | LR: 9.74e-06 | 5225.9ms/step | 11.5 samples/s
+Step 46040/50462 | Loss: 2.5202 | LR: 9.70e-06 | 5231.0ms/step | 11.5 samples/s
+Step 46050/50462 | Loss: 2.5031 | LR: 9.65e-06 | 5247.4ms/step | 11.4 samples/s
+Step 46060/50462 | Loss: 2.4972 | LR: 9.61e-06 | 5249.1ms/step | 11.4 samples/s
+Step 46070/50462 | Loss: 2.5165 | LR: 9.57e-06 | 5238.6ms/step | 11.5 samples/s
+Step 46080/50462 | Loss: 2.5113 | LR: 9.52e-06 | 5242.0ms/step | 11.4 samples/s
+Step 46090/50462 | Loss: 2.4799 | LR: 9.48e-06 | 5247.8ms/step | 11.4 samples/s
+Step 46100/50462 | Loss: 2.5059 | LR: 9.44e-06 | 5240.8ms/step | 11.4 samples/s
+Step 46110/50462 | Loss: 2.5016 | LR: 9.40e-06 | 5240.6ms/step | 11.4 samples/s
+Step 46120/50462 | Loss: 2.4968 | LR: 9.35e-06 | 5253.1ms/step | 11.4 samples/s
+Step 46130/50462 | Loss: 2.4982 | LR: 9.31e-06 | 5250.1ms/step | 11.4 samples/s
+Step 46140/50462 | Loss: 2.4984 | LR: 9.27e-06 | 5250.1ms/step | 11.4 samples/s
+Step 46150/50462 | Loss: 2.4967 | LR: 9.22e-06 | 5246.1ms/step | 11.4 samples/s
+Step 46160/50462 | Loss: 2.5144 | LR: 9.18e-06 | 5249.0ms/step | 11.4 samples/s
+Step 46170/50462 | Loss: 2.5149 | LR: 9.14e-06 | 5241.9ms/step | 11.4 samples/s
+Step 46180/50462 | Loss: 2.4845 | LR: 9.10e-06 | 5242.3ms/step | 11.4 samples/s
+Step 46190/50462 | Loss: 2.4850 | LR: 9.06e-06 | 5285.4ms/step | 11.4 samples/s
+Step 46200/50462 | Loss: 2.4926 | LR: 9.01e-06 | 5250.8ms/step | 11.4 samples/s
+Step 46210/50462 | Loss: 2.5091 | LR: 8.97e-06 | 5249.7ms/step | 11.4 samples/s
+Step 46220/50462 | Loss: 2.5119 | LR: 8.93e-06 | 5249.8ms/step | 11.4 samples/s
+Step 46230/50462 | Loss: 2.5180 | LR: 8.89e-06 | 5249.3ms/step | 11.4 samples/s
+Step 46240/50462 | Loss: 2.5267 | LR: 8.85e-06 | 5287.2ms/step | 11.3 samples/s
+Step 46250/50462 | Loss: 2.5245 | LR: 8.80e-06 | 5251.4ms/step | 11.4 samples/s
+Validation at step 46250: Loss = 2.5160, Perplexity = 12.38
+Step 46260/50462 | Loss: 2.5244 | LR: 8.76e-06 | 7811.9ms/step | 7.7 samples/s
+Step 46270/50462 | Loss: 2.5108 | LR: 8.72e-06 | 5280.2ms/step | 11.4 samples/s
+Step 46280/50462 | Loss: 2.5424 | LR: 8.68e-06 | 5243.4ms/step | 11.4 samples/s
+Step 46290/50462 | Loss: 2.5291 | LR: 8.64e-06 | 5247.0ms/step | 11.4 samples/s
+Step 46300/50462 | Loss: 2.4995 | LR: 8.60e-06 | 5248.7ms/step | 11.4 samples/s
+Step 46310/50462 | Loss: 2.5071 | LR: 8.56e-06 | 5247.9ms/step | 11.4 samples/s
+Step 46320/50462 | Loss: 2.5327 | LR: 8.52e-06 | 5252.1ms/step | 11.4 samples/s
+Step 46330/50462 | Loss: 2.5121 | LR: 8.47e-06 | 5250.8ms/step | 11.4 samples/s
+Step 46340/50462 | Loss: 2.5119 | LR: 8.43e-06 | 5255.7ms/step | 11.4 samples/s
+Step 46350/50462 | Loss: 2.5328 | LR: 8.39e-06 | 5246.7ms/step | 11.4 samples/s
+Step 46360/50462 | Loss: 2.4956 | LR: 8.35e-06 | 5252.6ms/step | 11.4 samples/s
+Step 46370/50462 | Loss: 2.4811 | LR: 8.31e-06 | 5249.9ms/step | 11.4 samples/s
+Step 46380/50462 | Loss: 2.5062 | LR: 8.27e-06 | 5249.7ms/step | 11.4 samples/s
+Step 46390/50462 | Loss: 2.4951 | LR: 8.23e-06 | 5250.1ms/step | 11.4 samples/s
+Step 46400/50462 | Loss: 2.4869 | LR: 8.19e-06 | 5248.9ms/step | 11.4 samples/s
+Step 46410/50462 | Loss: 2.4993 | LR: 8.15e-06 | 5248.5ms/step | 11.4 samples/s
+Step 46420/50462 | Loss: 2.4990 | LR: 8.11e-06 | 5249.4ms/step | 11.4 samples/s
+Step 46430/50462 | Loss: 2.5198 | LR: 8.07e-06 | 5251.0ms/step | 11.4 samples/s
+Step 46440/50462 | Loss: 2.5022 | LR: 8.03e-06 | 5251.0ms/step | 11.4 samples/s
+Step 46450/50462 | Loss: 2.5282 | LR: 7.99e-06 | 5248.7ms/step | 11.4 samples/s
+Step 46460/50462 | Loss: 2.5065 | LR: 7.95e-06 | 5287.1ms/step | 11.3 samples/s
+Step 46470/50462 | Loss: 2.4978 | LR: 7.91e-06 | 5252.8ms/step | 11.4 samples/s
+Step 46480/50462 | Loss: 2.4927 | LR: 7.87e-06 | 5247.6ms/step | 11.4 samples/s
+E0530 02:03:52.816000 8946 torch/distributed/elastic/multiprocessing/api.py:988] failed (exitcode: -15) local_rank: 0 (pid: 9081) of binary: /usr/bin/python3
+Traceback (most recent call last):
+  File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1020, in <module>
+    main()
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 367, in wrapper
+    return f(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1016, in main
+    run(args)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1007, in run
+    elastic_launch(
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 191, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 371, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+======================================================
+SparseAtt/train.py FAILED
+------------------------------------------------------
+Failures:
+[1]:
+  time      : 2026-05-30_02:03:52
+  host      : h100-wspace-3-gpu-0-0
+  rank      : 1 (local_rank: 1)
+  exitcode  : -15 (pid: 9082)  (SIGTERM)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 9082
+[2]:
+  time      : 2026-05-30_02:03:52
+  host      : h100-wspace-3-gpu-0-0
+  rank      : 2 (local_rank: 2)
+  exitcode  : -15 (pid: 9083)  (SIGTERM)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 9083
+------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2026-05-30_02:03:52
+  host      : h100-wspace-3-gpu-0-0
+  rank      : 0 (local_rank: 0)
+  exitcode  : -15 (pid: 9081)  (SIGTERM)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 9081
+======================================================

epfl_final/log_files/cont_pretrain_16k.log

@@ -0,0 +1,296 @@
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000",
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense-resume-18k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: W&B API key is configured. Use `wandb login --relogin` to force relogin
+wandb: setting up run qudv4nmq
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260513_151725-qudv4nmq
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense-resume-18k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qudv4nmq
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000...
+Warning: no RNG state file found for rank 0; RNG will not be restored.
+Resumed from checkpoint at step 15999 (epoch 0, micro_step 0)
+Resuming training from step 15999
+Starting training...
+Warning: no loader_gen_state file found; DataLoader shuffle order on resume may differ from original run.
+Step 16000/20005 | Loss: 0.2859 | LR: 4.90e-05 | 61450.0ms/step | 0.4 samples/s
+Validation at step 16000: Loss = 2.9095, Perplexity = 18.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000
+Step 16010/20005 | Loss: 2.8805 | LR: 4.88e-05 | 42322.3ms/step | 0.6 samples/s
+Step 16020/20005 | Loss: 2.9052 | LR: 4.85e-05 | 5797.6ms/step | 4.1 samples/s
+Step 16030/20005 | Loss: 2.8774 | LR: 4.83e-05 | 5801.3ms/step | 4.1 samples/s
+Step 16040/20005 | Loss: 2.8951 | LR: 4.80e-05 | 5783.8ms/step | 4.1 samples/s
+Step 16050/20005 | Loss: 2.8999 | LR: 4.78e-05 | 5774.8ms/step | 4.2 samples/s
+Step 16060/20005 | Loss: 2.8975 | LR: 4.76e-05 | 5774.1ms/step | 4.2 samples/s
+Step 16070/20005 | Loss: 2.9103 | LR: 4.73e-05 | 5786.3ms/step | 4.1 samples/s
+Step 16080/20005 | Loss: 2.8558 | LR: 4.71e-05 | 5793.9ms/step | 4.1 samples/s
+Step 16090/20005 | Loss: 2.8936 | LR: 4.69e-05 | 5796.0ms/step | 4.1 samples/s
+Step 16100/20005 | Loss: 2.8871 | LR: 4.67e-05 | 5782.7ms/step | 4.2 samples/s
+Step 16110/20005 | Loss: 2.8781 | LR: 4.64e-05 | 5777.7ms/step | 4.2 samples/s
+Step 16120/20005 | Loss: 2.8904 | LR: 4.62e-05 | 5775.9ms/step | 4.2 samples/s
+Step 16130/20005 | Loss: 2.8697 | LR: 4.60e-05 | 5791.7ms/step | 4.1 samples/s
+Step 16140/20005 | Loss: 2.8868 | LR: 4.57e-05 | 5801.7ms/step | 4.1 samples/s
+Step 16150/20005 | Loss: 2.8816 | LR: 4.55e-05 | 5800.1ms/step | 4.1 samples/s
+Step 16160/20005 | Loss: 2.8591 | LR: 4.53e-05 | 5784.0ms/step | 4.1 samples/s
+Step 16170/20005 | Loss: 2.8515 | LR: 4.50e-05 | 5782.4ms/step | 4.2 samples/s
+Step 16180/20005 | Loss: 2.8859 | LR: 4.48e-05 | 5784.0ms/step | 4.1 samples/s
+Step 16190/20005 | Loss: 2.8966 | LR: 4.46e-05 | 5787.5ms/step | 4.1 samples/s
+Step 16200/20005 | Loss: 2.8766 | LR: 4.44e-05 | 5802.8ms/step | 4.1 samples/s
+Step 16210/20005 | Loss: 2.9043 | LR: 4.41e-05 | 5805.9ms/step | 4.1 samples/s
+Step 16220/20005 | Loss: 2.8939 | LR: 4.39e-05 | 5783.9ms/step | 4.1 samples/s
+Step 16230/20005 | Loss: 2.8999 | LR: 4.37e-05 | 5777.6ms/step | 4.2 samples/s
+Step 16240/20005 | Loss: 2.8962 | LR: 4.35e-05 | 5775.0ms/step | 4.2 samples/s
+Step 16250/20005 | Loss: 2.8544 | LR: 4.32e-05 | 5791.8ms/step | 4.1 samples/s
+Validation at step 16250: Loss = 2.8974, Perplexity = 18.13
+Step 16260/20005 | Loss: 2.8719 | LR: 4.30e-05 | 39967.3ms/step | 0.6 samples/s
+Step 16270/20005 | Loss: 2.8932 | LR: 4.28e-05 | 5794.1ms/step | 4.1 samples/s
+Step 16280/20005 | Loss: 2.8991 | LR: 4.26e-05 | 5778.4ms/step | 4.2 samples/s
+Step 16290/20005 | Loss: 2.8963 | LR: 4.24e-05 | 5782.1ms/step | 4.2 samples/s
+Step 16300/20005 | Loss: 2.8744 | LR: 4.21e-05 | 5779.4ms/step | 4.2 samples/s
+Step 16310/20005 | Loss: 2.8959 | LR: 4.19e-05 | 5794.2ms/step | 4.1 samples/s
+Step 16320/20005 | Loss: 2.8922 | LR: 4.17e-05 | 5805.9ms/step | 4.1 samples/s
+Step 16330/20005 | Loss: 2.8783 | LR: 4.15e-05 | 5792.6ms/step | 4.1 samples/s
+Step 16340/20005 | Loss: 2.8571 | LR: 4.13e-05 | 5780.2ms/step | 4.2 samples/s
+Step 16350/20005 | Loss: 2.8927 | LR: 4.10e-05 | 5782.3ms/step | 4.2 samples/s
+Step 16360/20005 | Loss: 2.8868 | LR: 4.08e-05 | 5774.0ms/step | 4.2 samples/s
+Step 16370/20005 | Loss: 2.8835 | LR: 4.06e-05 | 5793.8ms/step | 4.1 samples/s
+Step 16380/20005 | Loss: 2.8890 | LR: 4.04e-05 | 5805.2ms/step | 4.1 samples/s
+Step 16390/20005 | Loss: 2.8688 | LR: 4.02e-05 | 5792.8ms/step | 4.1 samples/s
+Step 16400/20005 | Loss: 2.8665 | LR: 3.99e-05 | 5775.8ms/step | 4.2 samples/s
+Step 16410/20005 | Loss: 2.9046 | LR: 3.97e-05 | 5798.5ms/step | 4.1 samples/s
+Step 16420/20005 | Loss: 2.8618 | LR: 3.95e-05 | 5777.6ms/step | 4.2 samples/s
+Step 16430/20005 | Loss: 2.8637 | LR: 3.93e-05 | 5798.4ms/step | 4.1 samples/s
+Step 16440/20005 | Loss: 2.8608 | LR: 3.91e-05 | 5808.3ms/step | 4.1 samples/s
+Step 16450/20005 | Loss: 2.8862 | LR: 3.89e-05 | 5787.8ms/step | 4.1 samples/s
+Step 16460/20005 | Loss: 2.8885 | LR: 3.87e-05 | 5775.7ms/step | 4.2 samples/s
+Step 16470/20005 | Loss: 2.8833 | LR: 3.85e-05 | 5771.0ms/step | 4.2 samples/s
+Step 16480/20005 | Loss: 2.8654 | LR: 3.82e-05 | 5777.6ms/step | 4.2 samples/s
+Step 16490/20005 | Loss: 2.8883 | LR: 3.80e-05 | 5805.9ms/step | 4.1 samples/s
+Step 16500/20005 | Loss: 2.8750 | LR: 3.78e-05 | 5847.9ms/step | 4.1 samples/s
+Validation at step 16500: Loss = 2.8964, Perplexity = 18.11
+Step 16510/20005 | Loss: 2.8852 | LR: 3.76e-05 | 39988.2ms/step | 0.6 samples/s
+Step 16520/20005 | Loss: 2.8601 | LR: 3.74e-05 | 5779.8ms/step | 4.2 samples/s
+Step 16530/20005 | Loss: 2.8606 | LR: 3.72e-05 | 5778.4ms/step | 4.2 samples/s
+Step 16540/20005 | Loss: 2.8567 | LR: 3.70e-05 | 5788.9ms/step | 4.1 samples/s
+Step 16550/20005 | Loss: 2.9279 | LR: 3.68e-05 | 5799.2ms/step | 4.1 samples/s
+Step 16560/20005 | Loss: 2.8896 | LR: 3.66e-05 | 5806.0ms/step | 4.1 samples/s
+Step 16570/20005 | Loss: 2.8944 | LR: 3.64e-05 | 5789.4ms/step | 4.1 samples/s
+Step 16580/20005 | Loss: 2.8497 | LR: 3.62e-05 | 5778.2ms/step | 4.2 samples/s
+Step 16590/20005 | Loss: 2.8682 | LR: 3.59e-05 | 5787.9ms/step | 4.1 samples/s
+Step 16600/20005 | Loss: 2.8877 | LR: 3.57e-05 | 5789.2ms/step | 4.1 samples/s
+Step 16610/20005 | Loss: 2.8810 | LR: 3.55e-05 | 5805.3ms/step | 4.1 samples/s
+Step 16620/20005 | Loss: 2.8817 | LR: 3.53e-05 | 5806.6ms/step | 4.1 samples/s
+Step 16630/20005 | Loss: 2.8868 | LR: 3.51e-05 | 5785.3ms/step | 4.1 samples/s
+Step 16640/20005 | Loss: 2.9033 | LR: 3.49e-05 | 5778.0ms/step | 4.2 samples/s
+Step 16650/20005 | Loss: 2.9087 | LR: 3.47e-05 | 5774.1ms/step | 4.2 samples/s
+Step 16660/20005 | Loss: 2.8753 | LR: 3.45e-05 | 5783.8ms/step | 4.1 samples/s
+Step 16670/20005 | Loss: 2.8726 | LR: 3.43e-05 | 5806.0ms/step | 4.1 samples/s
+Step 16680/20005 | Loss: 2.8741 | LR: 3.41e-05 | 5808.2ms/step | 4.1 samples/s
+Step 16690/20005 | Loss: 2.8671 | LR: 3.39e-05 | 5792.3ms/step | 4.1 samples/s
+Step 16700/20005 | Loss: 2.8762 | LR: 3.37e-05 | 5802.2ms/step | 4.1 samples/s
+Step 16710/20005 | Loss: 2.8609 | LR: 3.35e-05 | 5801.5ms/step | 4.1 samples/s
+Step 16720/20005 | Loss: 2.8764 | LR: 3.33e-05 | 5798.4ms/step | 4.1 samples/s
+Step 16730/20005 | Loss: 2.8537 | LR: 3.31e-05 | 5804.1ms/step | 4.1 samples/s
+Step 16740/20005 | Loss: 2.8709 | LR: 3.29e-05 | 5801.8ms/step | 4.1 samples/s
+Step 16750/20005 | Loss: 2.8625 | LR: 3.27e-05 | 5785.3ms/step | 4.1 samples/s
+Validation at step 16750: Loss = 2.8958, Perplexity = 18.10
+Step 16760/20005 | Loss: 2.8645 | LR: 3.25e-05 | 40056.8ms/step | 0.6 samples/s
+Step 16770/20005 | Loss: 2.8947 | LR: 3.23e-05 | 5803.4ms/step | 4.1 samples/s
+Step 16780/20005 | Loss: 2.8879 | LR: 3.21e-05 | 5797.2ms/step | 4.1 samples/s
+Step 16790/20005 | Loss: 2.8779 | LR: 3.20e-05 | 5802.3ms/step | 4.1 samples/s
+Step 16800/20005 | Loss: 2.8738 | LR: 3.18e-05 | 5812.2ms/step | 4.1 samples/s
+Step 16810/20005 | Loss: 2.8759 | LR: 3.16e-05 | 5789.0ms/step | 4.1 samples/s
+Step 16820/20005 | Loss: 2.8653 | LR: 3.14e-05 | 5787.6ms/step | 4.1 samples/s
+Step 16830/20005 | Loss: 2.8641 | LR: 3.12e-05 | 5796.3ms/step | 4.1 samples/s
+Step 16840/20005 | Loss: 2.8667 | LR: 3.10e-05 | 5800.3ms/step | 4.1 samples/s
+Step 16850/20005 | Loss: 2.8774 | LR: 3.08e-05 | 5800.0ms/step | 4.1 samples/s
+Step 16860/20005 | Loss: 2.8557 | LR: 3.06e-05 | 5810.7ms/step | 4.1 samples/s
+Step 16870/20005 | Loss: 2.8982 | LR: 3.04e-05 | 5787.4ms/step | 4.1 samples/s
+Step 16880/20005 | Loss: 2.8782 | LR: 3.02e-05 | 5794.7ms/step | 4.1 samples/s
+Step 16890/20005 | Loss: 2.8829 | LR: 3.00e-05 | 5801.1ms/step | 4.1 samples/s
+Step 16900/20005 | Loss: 2.8573 | LR: 2.98e-05 | 5800.6ms/step | 4.1 samples/s
+Step 16910/20005 | Loss: 2.8912 | LR: 2.97e-05 | 5796.6ms/step | 4.1 samples/s
+Step 16920/20005 | Loss: 2.8813 | LR: 2.95e-05 | 5808.5ms/step | 4.1 samples/s
+Step 16930/20005 | Loss: 2.8525 | LR: 2.93e-05 | 5787.2ms/step | 4.1 samples/s
+Step 16940/20005 | Loss: 2.8718 | LR: 2.91e-05 | 5777.5ms/step | 4.2 samples/s
+Step 16950/20005 | Loss: 2.8829 | LR: 2.89e-05 | 5788.7ms/step | 4.1 samples/s
+Step 16960/20005 | Loss: 2.8935 | LR: 2.87e-05 | 5801.4ms/step | 4.1 samples/s
+Step 16970/20005 | Loss: 2.9022 | LR: 2.85e-05 | 5795.8ms/step | 4.1 samples/s
+Step 16980/20005 | Loss: 2.8955 | LR: 2.84e-05 | 5799.3ms/step | 4.1 samples/s
+Step 16990/20005 | Loss: 2.8606 | LR: 2.82e-05 | 5796.4ms/step | 4.1 samples/s
+Step 17000/20005 | Loss: 2.8556 | LR: 2.80e-05 | 5822.8ms/step | 4.1 samples/s
+Validation at step 17000: Loss = 2.8954, Perplexity = 18.09
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-17000
+Step 17010/20005 | Loss: 2.8932 | LR: 2.78e-05 | 40472.1ms/step | 0.6 samples/s
+Step 17020/20005 | Loss: 2.8715 | LR: 2.76e-05 | 5795.8ms/step | 4.1 samples/s
+Step 17030/20005 | Loss: 2.8630 | LR: 2.74e-05 | 5784.2ms/step | 4.1 samples/s
+Step 17040/20005 | Loss: 2.8649 | LR: 2.73e-05 | 5804.2ms/step | 4.1 samples/s
+Step 17050/20005 | Loss: 2.8631 | LR: 2.71e-05 | 5803.9ms/step | 4.1 samples/s
+Step 17060/20005 | Loss: 2.8951 | LR: 2.69e-05 | 5785.9ms/step | 4.1 samples/s
+Step 17070/20005 | Loss: 2.8709 | LR: 2.67e-05 | 5773.8ms/step | 4.2 samples/s
+Step 17080/20005 | Loss: 2.9004 | LR: 2.65e-05 | 5774.1ms/step | 4.2 samples/s
+Step 17090/20005 | Loss: 2.8492 | LR: 2.64e-05 | 5778.0ms/step | 4.2 samples/s
+Step 17100/20005 | Loss: 2.8993 | LR: 2.62e-05 | 5799.8ms/step | 4.1 samples/s
+Step 17110/20005 | Loss: 2.9000 | LR: 2.60e-05 | 5806.2ms/step | 4.1 samples/s
+Step 17120/20005 | Loss: 2.8823 | LR: 2.58e-05 | 5780.5ms/step | 4.2 samples/s
+Step 17130/20005 | Loss: 2.8930 | LR: 2.57e-05 | 5777.5ms/step | 4.2 samples/s
+Step 17140/20005 | Loss: 2.8838 | LR: 2.55e-05 | 5772.2ms/step | 4.2 samples/s
+Step 17150/20005 | Loss: 2.8710 | LR: 2.53e-05 | 5776.7ms/step | 4.2 samples/s
+Step 17160/20005 | Loss: 2.8906 | LR: 2.51e-05 | 5799.7ms/step | 4.1 samples/s
+Step 17170/20005 | Loss: 2.8886 | LR: 2.50e-05 | 5802.9ms/step | 4.1 samples/s
+Step 17180/20005 | Loss: 2.8920 | LR: 2.48e-05 | 5786.9ms/step | 4.1 samples/s
+Step 17190/20005 | Loss: 2.8665 | LR: 2.46e-05 | 5775.9ms/step | 4.2 samples/s
+Step 17200/20005 | Loss: 2.8719 | LR: 2.44e-05 | 5776.3ms/step | 4.2 samples/s
+Step 17210/20005 | Loss: 2.8837 | LR: 2.43e-05 | 5777.9ms/step | 4.2 samples/s
+Step 17220/20005 | Loss: 2.8843 | LR: 2.41e-05 | 5796.4ms/step | 4.1 samples/s
+Step 17230/20005 | Loss: 2.8992 | LR: 2.39e-05 | 5804.3ms/step | 4.1 samples/s
+Step 17240/20005 | Loss: 2.8704 | LR: 2.38e-05 | 5788.7ms/step | 4.1 samples/s
+Step 17250/20005 | Loss: 2.8868 | LR: 2.36e-05 | 5776.6ms/step | 4.2 samples/s
+Validation at step 17250: Loss = 2.8951, Perplexity = 18.08

epfl_final/log_files/cont_pretrain_19k.log

@@ -0,0 +1,302 @@
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000",
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense-resume-18k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260513_052440-i0f0xwcq
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense-resume-18k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/i0f0xwcq
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000...
+Warning: no RNG state file found for rank 0; RNG will not be restored.
+Resumed from checkpoint at step 18999 (epoch 0, micro_step 0)
+Resuming training from step 18999
+Starting training...
+Warning: no loader_gen_state file found; DataLoader shuffle order on resume may differ from original run.
+Step 19000/20005 | Loss: 0.2856 | LR: 3.18e-06 | 64856.9ms/step | 0.4 samples/s
+Validation at step 19000: Loss = 2.9082, Perplexity = 18.32
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000
+Step 19010/20005 | Loss: 2.8841 | LR: 3.12e-06 | 43506.1ms/step | 0.6 samples/s
+Step 19020/20005 | Loss: 2.9121 | LR: 3.05e-06 | 5798.6ms/step | 4.1 samples/s
+Step 19030/20005 | Loss: 2.8848 | LR: 2.99e-06 | 5799.2ms/step | 4.1 samples/s
+Step 19040/20005 | Loss: 2.9030 | LR: 2.93e-06 | 5796.6ms/step | 4.1 samples/s
+Step 19050/20005 | Loss: 2.9076 | LR: 2.87e-06 | 5801.1ms/step | 4.1 samples/s
+Step 19060/20005 | Loss: 2.9055 | LR: 2.81e-06 | 5802.7ms/step | 4.1 samples/s
+Step 19070/20005 | Loss: 2.9181 | LR: 2.75e-06 | 5800.7ms/step | 4.1 samples/s
+Step 19080/20005 | Loss: 2.8634 | LR: 2.69e-06 | 5803.0ms/step | 4.1 samples/s
+Step 19090/20005 | Loss: 2.9011 | LR: 2.64e-06 | 5803.1ms/step | 4.1 samples/s
+Step 19100/20005 | Loss: 2.8946 | LR: 2.58e-06 | 5801.6ms/step | 4.1 samples/s
+Step 19110/20005 | Loss: 2.8855 | LR: 2.52e-06 | 5846.1ms/step | 4.1 samples/s
+Step 19120/20005 | Loss: 2.8977 | LR: 2.47e-06 | 5802.8ms/step | 4.1 samples/s
+Step 19130/20005 | Loss: 2.8775 | LR: 2.41e-06 | 5799.4ms/step | 4.1 samples/s
+Step 19140/20005 | Loss: 2.8940 | LR: 2.36e-06 | 5801.6ms/step | 4.1 samples/s
+Step 19150/20005 | Loss: 2.8891 | LR: 2.30e-06 | 5801.3ms/step | 4.1 samples/s
+Step 19160/20005 | Loss: 2.8665 | LR: 2.25e-06 | 5803.0ms/step | 4.1 samples/s
+Step 19170/20005 | Loss: 2.8587 | LR: 2.20e-06 | 5802.4ms/step | 4.1 samples/s
+Step 19180/20005 | Loss: 2.8930 | LR: 2.14e-06 | 5798.3ms/step | 4.1 samples/s
+Step 19190/20005 | Loss: 2.9038 | LR: 2.09e-06 | 5801.6ms/step | 4.1 samples/s
+Step 19200/20005 | Loss: 2.8835 | LR: 2.04e-06 | 5799.8ms/step | 4.1 samples/s
+Step 19210/20005 | Loss: 2.9115 | LR: 1.99e-06 | 5801.6ms/step | 4.1 samples/s
+Step 19220/20005 | Loss: 2.9010 | LR: 1.94e-06 | 5800.3ms/step | 4.1 samples/s
+Step 19230/20005 | Loss: 2.9071 | LR: 1.89e-06 | 5800.3ms/step | 4.1 samples/s
+Step 19240/20005 | Loss: 2.9033 | LR: 1.84e-06 | 5802.5ms/step | 4.1 samples/s
+Step 19250/20005 | Loss: 2.8618 | LR: 1.80e-06 | 5799.7ms/step | 4.1 samples/s
+Validation at step 19250: Loss = 2.9042, Perplexity = 18.25
+Step 19260/20005 | Loss: 2.8789 | LR: 1.75e-06 | 40859.3ms/step | 0.6 samples/s
+Step 19270/20005 | Loss: 2.9005 | LR: 1.70e-06 | 5798.0ms/step | 4.1 samples/s
+Step 19280/20005 | Loss: 2.9064 | LR: 1.66e-06 | 5802.0ms/step | 4.1 samples/s
+Step 19290/20005 | Loss: 2.9035 | LR: 1.61e-06 | 5802.4ms/step | 4.1 samples/s
+Step 19300/20005 | Loss: 2.8814 | LR: 1.57e-06 | 5800.4ms/step | 4.1 samples/s
+Step 19310/20005 | Loss: 2.9034 | LR: 1.52e-06 | 5799.9ms/step | 4.1 samples/s
+Step 19320/20005 | Loss: 2.8996 | LR: 1.48e-06 | 5800.9ms/step | 4.1 samples/s
+Step 19330/20005 | Loss: 2.8856 | LR: 1.43e-06 | 5802.4ms/step | 4.1 samples/s
+Step 19340/20005 | Loss: 2.8643 | LR: 1.39e-06 | 5798.2ms/step | 4.1 samples/s
+Step 19350/20005 | Loss: 2.8999 | LR: 1.35e-06 | 5801.1ms/step | 4.1 samples/s
+Step 19360/20005 | Loss: 2.8943 | LR: 1.31e-06 | 5803.3ms/step | 4.1 samples/s
+Step 19370/20005 | Loss: 2.8913 | LR: 1.27e-06 | 5801.3ms/step | 4.1 samples/s
+Step 19380/20005 | Loss: 2.8962 | LR: 1.23e-06 | 5800.5ms/step | 4.1 samples/s
+Step 19390/20005 | Loss: 2.8758 | LR: 1.19e-06 | 5802.7ms/step | 4.1 samples/s
+Step 19400/20005 | Loss: 2.8741 | LR: 1.15e-06 | 5800.3ms/step | 4.1 samples/s
+Step 19410/20005 | Loss: 2.9121 | LR: 1.11e-06 | 5798.2ms/step | 4.1 samples/s
+Step 19420/20005 | Loss: 2.8689 | LR: 1.08e-06 | 5798.5ms/step | 4.1 samples/s
+Step 19430/20005 | Loss: 2.8710 | LR: 1.04e-06 | 5798.4ms/step | 4.1 samples/s
+Step 19440/20005 | Loss: 2.8682 | LR: 1.00e-06 | 5801.2ms/step | 4.1 samples/s
+Step 19450/20005 | Loss: 2.8935 | LR: 9.70e-07 | 5801.5ms/step | 4.1 samples/s
+Step 19460/20005 | Loss: 2.8957 | LR: 9.35e-07 | 5801.1ms/step | 4.1 samples/s
+Step 19470/20005 | Loss: 2.8909 | LR: 9.01e-07 | 5798.9ms/step | 4.1 samples/s
+Step 19480/20005 | Loss: 2.8730 | LR: 8.67e-07 | 5796.8ms/step | 4.1 samples/s
+Step 19490/20005 | Loss: 2.8959 | LR: 8.35e-07 | 5797.7ms/step | 4.1 samples/s
+Step 19500/20005 | Loss: 2.8824 | LR: 8.03e-07 | 5797.8ms/step | 4.1 samples/s
+Validation at step 19500: Loss = 2.9036, Perplexity = 18.24
+Step 19510/20005 | Loss: 2.8928 | LR: 7.71e-07 | 40869.8ms/step | 0.6 samples/s
+Step 19520/20005 | Loss: 2.8678 | LR: 7.40e-07 | 5801.5ms/step | 4.1 samples/s
+Step 19530/20005 | Loss: 2.8681 | LR: 7.10e-07 | 5800.2ms/step | 4.1 samples/s
+Step 19540/20005 | Loss: 2.8642 | LR: 6.80e-07 | 5798.5ms/step | 4.1 samples/s
+Step 19550/20005 | Loss: 2.9356 | LR: 6.51e-07 | 5801.8ms/step | 4.1 samples/s
+Step 19560/20005 | Loss: 2.8972 | LR: 6.23e-07 | 5802.1ms/step | 4.1 samples/s
+Step 19570/20005 | Loss: 2.9023 | LR: 5.95e-07 | 5798.3ms/step | 4.1 samples/s
+Step 19580/20005 | Loss: 2.8574 | LR: 5.68e-07 | 5836.4ms/step | 4.1 samples/s
+Step 19590/20005 | Loss: 2.8756 | LR: 5.42e-07 | 5807.5ms/step | 4.1 samples/s
+Step 19600/20005 | Loss: 2.8952 | LR: 5.16e-07 | 5800.3ms/step | 4.1 samples/s
+Step 19610/20005 | Loss: 2.8886 | LR: 4.91e-07 | 5799.3ms/step | 4.1 samples/s
+Step 19620/20005 | Loss: 2.8894 | LR: 4.66e-07 | 5803.7ms/step | 4.1 samples/s
+Step 19630/20005 | Loss: 2.8946 | LR: 4.42e-07 | 5803.9ms/step | 4.1 samples/s
+Step 19640/20005 | Loss: 2.9110 | LR: 4.19e-07 | 5802.1ms/step | 4.1 samples/s
+Step 19650/20005 | Loss: 2.9165 | LR: 3.96e-07 | 5800.4ms/step | 4.1 samples/s
+Step 19660/20005 | Loss: 2.8830 | LR: 3.74e-07 | 5804.4ms/step | 4.1 samples/s
+Step 19670/20005 | Loss: 2.8803 | LR: 3.53e-07 | 5801.1ms/step | 4.1 samples/s
+Step 19680/20005 | Loss: 2.8821 | LR: 3.32e-07 | 5802.8ms/step | 4.1 samples/s
+Step 19690/20005 | Loss: 2.8748 | LR: 3.12e-07 | 5801.4ms/step | 4.1 samples/s
+Step 19700/20005 | Loss: 2.8839 | LR: 2.92e-07 | 5802.6ms/step | 4.1 samples/s
+Step 19710/20005 | Loss: 2.8690 | LR: 2.73e-07 | 5803.1ms/step | 4.1 samples/s
+Step 19720/20005 | Loss: 2.8843 | LR: 2.55e-07 | 5799.9ms/step | 4.1 samples/s
+Step 19730/20005 | Loss: 2.8619 | LR: 2.37e-07 | 5802.0ms/step | 4.1 samples/s
+Step 19740/20005 | Loss: 2.8787 | LR: 2.20e-07 | 5803.5ms/step | 4.1 samples/s
+Step 19750/20005 | Loss: 2.8705 | LR: 2.04e-07 | 5801.0ms/step | 4.1 samples/s
+Validation at step 19750: Loss = 2.9035, Perplexity = 18.24
+Step 19760/20005 | Loss: 2.8728 | LR: 1.88e-07 | 40861.9ms/step | 0.6 samples/s
+Step 19770/20005 | Loss: 2.9028 | LR: 1.73e-07 | 5803.2ms/step | 4.1 samples/s
+Step 19780/20005 | Loss: 2.8959 | LR: 1.59e-07 | 5801.9ms/step | 4.1 samples/s
+Step 19790/20005 | Loss: 2.8860 | LR: 1.45e-07 | 5800.9ms/step | 4.1 samples/s
+Step 19800/20005 | Loss: 2.8817 | LR: 1.32e-07 | 5799.4ms/step | 4.1 samples/s
+Step 19810/20005 | Loss: 2.8840 | LR: 1.19e-07 | 5798.4ms/step | 4.1 samples/s
+Step 19820/20005 | Loss: 2.8737 | LR: 1.07e-07 | 5797.4ms/step | 4.1 samples/s
+Step 19830/20005 | Loss: 2.8726 | LR: 9.57e-08 | 5800.9ms/step | 4.1 samples/s
+Step 19840/20005 | Loss: 2.8749 | LR: 8.50e-08 | 5800.1ms/step | 4.1 samples/s
+Step 19850/20005 | Loss: 2.8855 | LR: 7.50e-08 | 5798.7ms/step | 4.1 samples/s
+Step 19860/20005 | Loss: 2.8639 | LR: 6.55e-08 | 5798.3ms/step | 4.1 samples/s
+Step 19870/20005 | Loss: 2.9065 | LR: 5.68e-08 | 5798.5ms/step | 4.1 samples/s
+Step 19880/20005 | Loss: 2.8865 | LR: 4.86e-08 | 5799.3ms/step | 4.1 samples/s
+Step 19890/20005 | Loss: 2.8910 | LR: 4.11e-08 | 5801.9ms/step | 4.1 samples/s
+Step 19900/20005 | Loss: 2.8658 | LR: 3.42e-08 | 5806.0ms/step | 4.1 samples/s
+Step 19910/20005 | Loss: 2.8996 | LR: 2.79e-08 | 5799.8ms/step | 4.1 samples/s
+Step 19920/20005 | Loss: 2.8895 | LR: 2.23e-08 | 5799.6ms/step | 4.1 samples/s
+Step 19930/20005 | Loss: 2.8606 | LR: 1.73e-08 | 5802.8ms/step | 4.1 samples/s
+Step 19940/20005 | Loss: 2.8804 | LR: 1.29e-08 | 5800.1ms/step | 4.1 samples/s
+Step 19950/20005 | Loss: 2.8914 | LR: 9.22e-09 | 5803.8ms/step | 4.1 samples/s
+Step 19960/20005 | Loss: 2.9020 | LR: 6.12e-09 | 5804.7ms/step | 4.1 samples/s
+Step 19970/20005 | Loss: 2.9103 | LR: 3.65e-09 | 5801.9ms/step | 4.1 samples/s
+Step 19980/20005 | Loss: 2.9041 | LR: 1.82e-09 | 5802.7ms/step | 4.1 samples/s
+Step 19990/20005 | Loss: 2.8692 | LR: 6.20e-10 | 5801.4ms/step | 4.1 samples/s
+Step 20000/20005 | Loss: 2.8643 | LR: 5.06e-11 | 5798.9ms/step | 4.1 samples/s
+Validation at step 20000: Loss = 2.9035, Perplexity = 18.24
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-20000
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-20005
+wandb: updating run metadata
+wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml
+wandb: uploading output.log; uploading config.yaml
+wandb: uploading history steps 100-100, summary, console lines 210-210
+wandb: 
+wandb: Run history:
+wandb:             eval/loss █▂▁▁▁
+wandb:       eval/perplexity █▂▁▁▁
+wandb:     speed/ms_per_step █▁▁▁▁▁▁▁▁▁▁▅▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb: speed/samples_per_sec ▁███████████���██████████████████▁████████
+wandb:   speed/steps_per_sec ▁██████████████████████▁████████████████
+wandb:           train/epoch ▁▁▁▁▂▂▂▃▃▃▃▃▃▃▃▄▄▄▄▅▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇████
+wandb:     train/global_step ▁▁▂▂▂▂▂▂▃▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇███
+wandb:   train/learning_rate ████▇▇▆▆▆▆▅▅▅▅▅▄▄▄▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁
+wandb:            train/loss ▃▅▆▁▅▂▄▁▄▅▁▅▅▃▄▂▂▄▂▄▂▁█▄▅▄▄▆▃▂▁▃▄▃▁▃▄▅▆▂
+wandb: 
+wandb: Run summary:
+wandb:             eval/loss 2.90353
+wandb:       eval/perplexity 18.23845
+wandb:     speed/ms_per_step 5798.85206
+wandb: speed/samples_per_sec 4.13875
+wandb:   speed/steps_per_sec 0.17245
+wandb:           train/epoch 0.4717
+wandb:     train/global_step 20000
+wandb:   train/learning_rate 0.0
+wandb:            train/loss 2.86429
+wandb: 
+wandb: 🚀 View run dense-resume-18k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/i0f0xwcq
+wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20260513_052440-i0f0xwcq/logs
+Training complete!

epfl_final/log_files/cont_pretrain_20k.log

@@ -0,0 +1,853 @@
+nohup: ignoring input
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-44000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 20k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-44000
+Checkpoint activations: FalseCheckpoint activations: False
+
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 1536]           [2, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [2, 8192, 1536]           [2, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (G): 2.87
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 52437.19
+Params size (MB): 2871.45
+Estimated Total Size (MB): 55308.77
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        3
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504627
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-44000...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Overriding checkpoint LR with config LR: 0.0005
+Resumed from checkpoint at step 44000 (epoch 0, micro_step 440000)
+Resuming training from step 44000
+Resuming wandb run 8igthj1g
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 8igthj1g
+wandb: Tracking run with wandb version 0.27.0
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260604_030818-8igthj1g
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run dense continued 20k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8igthj1g
+Starting training...
+Validation at step 44000: Loss = 2.5048, Perplexity = 12.24
+wandb: WARNING Tried to log to step 44000 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 44001: dense (dense step interval)
+Step 44010/50462 | Loss: 2.5147 | LR: 2.05e-05 | 76343.6ms/step | 0.8 samples/s
+wandb: WARNING Tried to log to step 44010 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44020/50462 | Loss: 2.5009 | LR: 2.04e-05 | 5185.9ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44020 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44030/50462 | Loss: 2.4941 | LR: 2.04e-05 | 5192.2ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44030 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44040/50462 | Loss: 2.4883 | LR: 2.03e-05 | 5203.5ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44040 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44050/50462 | Loss: 2.5206 | LR: 2.02e-05 | 5190.3ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44050 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44060/50462 | Loss: 2.5191 | LR: 2.02e-05 | 5220.3ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44060 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44070/50462 | Loss: 2.5187 | LR: 2.01e-05 | 5193.4ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44070 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44080/50462 | Loss: 2.5006 | LR: 2.01e-05 | 5195.4ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44080 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44090/50462 | Loss: 2.4700 | LR: 2.00e-05 | 5199.3ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44090 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44100/50462 | Loss: 2.4884 | LR: 1.99e-05 | 5205.9ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44100 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44110/50462 | Loss: 2.4903 | LR: 1.99e-05 | 5203.3ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44110 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44120/50462 | Loss: 2.5063 | LR: 1.98e-05 | 5207.8ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44120 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44130/50462 | Loss: 2.5124 | LR: 1.97e-05 | 5256.2ms/step | 11.4 samples/s
+wandb: WARNING Tried to log to step 44130 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44140/50462 | Loss: 2.4806 | LR: 1.97e-05 | 5196.0ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44140 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44150/50462 | Loss: 2.4756 | LR: 1.96e-05 | 5194.8ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44150 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44160/50462 | Loss: 2.5060 | LR: 1.96e-05 | 5217.2ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44160 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44170/50462 | Loss: 2.5094 | LR: 1.95e-05 | 5231.7ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44170 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44180/50462 | Loss: 2.4765 | LR: 1.94e-05 | 5201.0ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44180 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44190/50462 | Loss: 2.4826 | LR: 1.94e-05 | 5194.5ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44190 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44200/50462 | Loss: 2.5043 | LR: 1.93e-05 | 5259.2ms/step | 11.4 samples/s
+wandb: WARNING Tried to log to step 44200 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44210/50462 | Loss: 2.4919 | LR: 1.93e-05 | 5196.4ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44210 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44220/50462 | Loss: 2.4980 | LR: 1.92e-05 | 5192.5ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44220 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44230/50462 | Loss: 2.5093 | LR: 1.91e-05 | 5196.8ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44230 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44240/50462 | Loss: 2.4970 | LR: 1.91e-05 | 5173.2ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44240 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44250/50462 | Loss: 2.5105 | LR: 1.90e-05 | 5175.3ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44250 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 44250: Loss = 2.5056, Perplexity = 12.25
+wandb: WARNING Tried to log to step 44250 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44260/50462 | Loss: 2.5054 | LR: 1.90e-05 | 7741.5ms/step | 7.8 samples/s
+wandb: WARNING Tried to log to step 44260 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44270/50462 | Loss: 2.4754 | LR: 1.89e-05 | 5184.2ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44270 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44280/50462 | Loss: 2.4989 | LR: 1.88e-05 | 5175.8ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44280 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44290/50462 | Loss: 2.4949 | LR: 1.88e-05 | 5171.9ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44290 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44300/50462 | Loss: 2.4461 | LR: 1.87e-05 | 5189.9ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44300 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44310/50462 | Loss: 2.4764 | LR: 1.87e-05 | 5181.6ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44310 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44320/50462 | Loss: 2.5087 | LR: 1.86e-05 | 5192.7ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44320 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44330/50462 | Loss: 2.4992 | LR: 1.85e-05 | 5182.9ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44330 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44340/50462 | Loss: 2.4956 | LR: 1.85e-05 | 5172.1ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44340 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44350/50462 | Loss: 2.5002 | LR: 1.84e-05 | 5184.8ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44350 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44360/50462 | Loss: 2.4735 | LR: 1.84e-05 | 5181.6ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44360 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44370/50462 | Loss: 2.4993 | LR: 1.83e-05 | 5176.3ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44370 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44380/50462 | Loss: 2.4896 | LR: 1.82e-05 | 5203.1ms/step | 11.5 samples/s
+wandb: WARNING Tried to log to step 44380 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44390/50462 | Loss: 2.5099 | LR: 1.82e-05 | 5183.8ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44390 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44400/50462 | Loss: 2.4760 | LR: 1.81e-05 | 5188.7ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44400 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44410/50462 | Loss: 2.4856 | LR: 1.81e-05 | 5189.1ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44410 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44420/50462 | Loss: 2.4958 | LR: 1.80e-05 | 5247.0ms/step | 11.4 samples/s
+wandb: WARNING Tried to log to step 44420 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44430/50462 | Loss: 2.5030 | LR: 1.79e-05 | 5144.2ms/step | 11.7 samples/s
+wandb: WARNING Tried to log to step 44430 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44440/50462 | Loss: 2.4919 | LR: 1.79e-05 | 5158.9ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44440 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44450/50462 | Loss: 2.4807 | LR: 1.78e-05 | 5172.8ms/step | 11.6 samples/s
+wandb: WARNING Tried to log to step 44450 that is less than the current step 44451. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 44460/50462 | Loss: 2.4960 | LR: 1.78e-05 | 5188.2ms/step | 11.6 samples/s
+Step 44470/50462 | Loss: 2.5090 | LR: 1.77e-05 | 5172.5ms/step | 11.6 samples/s
+Step 44480/50462 | Loss: 2.5232 | LR: 1.77e-05 | 5236.6ms/step | 11.5 samples/s
+Step 44490/50462 | Loss: 2.5189 | LR: 1.76e-05 | 5192.7ms/step | 11.6 samples/s
+Step 44500/50462 | Loss: 2.4821 | LR: 1.75e-05 | 5258.1ms/step | 11.4 samples/s
+Validation at step 44500: Loss = 2.5053, Perplexity = 12.25
+Step 44510/50462 | Loss: 2.4710 | LR: 1.75e-05 | 7732.5ms/step | 7.8 samples/s
+Step 44520/50462 | Loss: 2.4697 | LR: 1.74e-05 | 5188.2ms/step | 11.6 samples/s
+Step 44530/50462 | Loss: 2.4962 | LR: 1.74e-05 | 5193.9ms/step | 11.6 samples/s
+Step 44540/50462 | Loss: 2.4847 | LR: 1.73e-05 | 5192.1ms/step | 11.6 samples/s
+Step 44550/50462 | Loss: 2.4967 | LR: 1.72e-05 | 5175.2ms/step | 11.6 samples/s
+Step 44560/50462 | Loss: 2.5024 | LR: 1.72e-05 | 5169.8ms/step | 11.6 samples/s
+Step 44570/50462 | Loss: 2.4888 | LR: 1.71e-05 | 5167.9ms/step | 11.6 samples/s
+Step 44580/50462 | Loss: 2.4674 | LR: 1.71e-05 | 5196.7ms/step | 11.5 samples/s
+Step 44590/50462 | Loss: 2.4856 | LR: 1.70e-05 | 5181.2ms/step | 11.6 samples/s
+Step 44600/50462 | Loss: 2.5096 | LR: 1.70e-05 | 5187.5ms/step | 11.6 samples/s
+Step 44610/50462 | Loss: 2.4910 | LR: 1.69e-05 | 5172.7ms/step | 11.6 samples/s
+Step 44620/50462 | Loss: 2.5085 | LR: 1.68e-05 | 5170.6ms/step | 11.6 samples/s
+Step 44630/50462 | Loss: 2.4936 | LR: 1.68e-05 | 5181.3ms/step | 11.6 samples/s
+Step 44640/50462 | Loss: 2.4850 | LR: 1.67e-05 | 5175.1ms/step | 11.6 samples/s
+Step 44650/50462 | Loss: 2.4734 | LR: 1.67e-05 | 5190.9ms/step | 11.6 samples/s
+Step 44660/50462 | Loss: 2.5113 | LR: 1.66e-05 | 5196.8ms/step | 11.5 samples/s
+Step 44670/50462 | Loss: 2.5205 | LR: 1.66e-05 | 5181.3ms/step | 11.6 samples/s
+Step 44680/50462 | Loss: 2.5065 | LR: 1.65e-05 | 5176.2ms/step | 11.6 samples/s
+Step 44690/50462 | Loss: 2.5030 | LR: 1.64e-05 | 5177.1ms/step | 11.6 samples/s
+Step 44700/50462 | Loss: 2.5082 | LR: 1.64e-05 | 5174.3ms/step | 11.6 samples/s
+Step 44710/50462 | Loss: 2.4924 | LR: 1.63e-05 | 5246.6ms/step | 11.4 samples/s
+Step 44720/50462 | Loss: 2.4936 | LR: 1.63e-05 | 5178.5ms/step | 11.6 samples/s
+Step 44730/50462 | Loss: 2.4819 | LR: 1.62e-05 | 5182.7ms/step | 11.6 samples/s
+Step 44740/50462 | Loss: 2.4776 | LR: 1.62e-05 | 5184.7ms/step | 11.6 samples/s
+Step 44750/50462 | Loss: 2.5021 | LR: 1.61e-05 | 5192.5ms/step | 11.6 samples/s
+Validation at step 44750: Loss = 2.5052, Perplexity = 12.25
+Step 44760/50462 | Loss: 2.5177 | LR: 1.61e-05 | 7733.8ms/step | 7.8 samples/s
+Step 44770/50462 | Loss: 2.4661 | LR: 1.60e-05 | 5187.1ms/step | 11.6 samples/s
+Step 44780/50462 | Loss: 2.4948 | LR: 1.59e-05 | 5222.2ms/step | 11.5 samples/s
+Step 44790/50462 | Loss: 2.4846 | LR: 1.59e-05 | 5191.7ms/step | 11.6 samples/s
+Step 44800/50462 | Loss: 2.4857 | LR: 1.58e-05 | 5183.6ms/step | 11.6 samples/s
+Step 44810/50462 | Loss: 2.5117 | LR: 1.58e-05 | 5224.6ms/step | 11.5 samples/s
+Step 44820/50462 | Loss: 2.5013 | LR: 1.57e-05 | 5174.7ms/step | 11.6 samples/s
+Step 44830/50462 | Loss: 2.5227 | LR: 1.57e-05 | 5169.5ms/step | 11.6 samples/s
+Step 44840/50462 | Loss: 2.4790 | LR: 1.56e-05 | 5160.1ms/step | 11.6 samples/s
+Step 44850/50462 | Loss: 2.4829 | LR: 1.56e-05 | 5169.4ms/step | 11.6 samples/s
+Step 44860/50462 | Loss: 2.4823 | LR: 1.55e-05 | 5169.3ms/step | 11.6 samples/s
+Step 44870/50462 | Loss: 2.4716 | LR: 1.54e-05 | 5178.9ms/step | 11.6 samples/s
+Step 44880/50462 | Loss: 2.4956 | LR: 1.54e-05 | 5177.7ms/step | 11.6 samples/s
+Step 44890/50462 | Loss: 2.4588 | LR: 1.53e-05 | 5170.5ms/step | 11.6 samples/s
+Step 44900/50462 | Loss: 2.4590 | LR: 1.53e-05 | 5182.0ms/step | 11.6 samples/s
+Step 44910/50462 | Loss: 2.4815 | LR: 1.52e-05 | 5182.6ms/step | 11.6 samples/s
+Step 44920/50462 | Loss: 2.4839 | LR: 1.52e-05 | 5190.6ms/step | 11.6 samples/s
+Step 44930/50462 | Loss: 2.4873 | LR: 1.51e-05 | 5184.1ms/step | 11.6 samples/s
+Step 44940/50462 | Loss: 2.4897 | LR: 1.51e-05 | 5181.3ms/step | 11.6 samples/s
+Step 44950/50462 | Loss: 2.4473 | LR: 1.50e-05 | 5190.5ms/step | 11.6 samples/s
+Step 44960/50462 | Loss: 2.5261 | LR: 1.50e-05 | 5190.4ms/step | 11.6 samples/s
+Step 44970/50462 | Loss: 2.4759 | LR: 1.49e-05 | 5179.4ms/step | 11.6 samples/s
+Step 44980/50462 | Loss: 2.4877 | LR: 1.49e-05 | 5194.3ms/step | 11.6 samples/s
+Step 44990/50462 | Loss: 2.5167 | LR: 1.48e-05 | 5185.6ms/step | 11.6 samples/s
+Step 45000/50462 | Loss: 2.5076 | LR: 1.47e-05 | 5236.9ms/step | 11.5 samples/s
+Validation at step 45000: Loss = 2.5051, Perplexity = 12.24
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-45000
+Step 45010/50462 | Loss: 2.4997 | LR: 1.47e-05 | 8739.1ms/step | 6.9 samples/s
+Step 45020/50462 | Loss: 2.4840 | LR: 1.46e-05 | 5203.3ms/step | 11.5 samples/s
+Step 45030/50462 | Loss: 2.4906 | LR: 1.46e-05 | 5188.3ms/step | 11.6 samples/s
+Step 45040/50462 | Loss: 2.5029 | LR: 1.45e-05 | 5191.5ms/step | 11.6 samples/s
+Step 45050/50462 | Loss: 2.4785 | LR: 1.45e-05 | 5200.7ms/step | 11.5 samples/s
+Step 45060/50462 | Loss: 2.5011 | LR: 1.44e-05 | 5203.3ms/step | 11.5 samples/s
+Step 45070/50462 | Loss: 2.4950 | LR: 1.44e-05 | 5193.8ms/step | 11.6 samples/s
+Step 45080/50462 | Loss: 2.4868 | LR: 1.43e-05 | 5203.7ms/step | 11.5 samples/s
+Step 45090/50462 | Loss: 2.4864 | LR: 1.43e-05 | 5250.3ms/step | 11.4 samples/s
+Step 45100/50462 | Loss: 2.5209 | LR: 1.42e-05 | 5204.0ms/step | 11.5 samples/s
+Step 45110/50462 | Loss: 2.5200 | LR: 1.42e-05 | 5244.6ms/step | 11.4 samples/s
+Step 45120/50462 | Loss: 2.5027 | LR: 1.41e-05 | 5177.9ms/step | 11.6 samples/s
+Step 45130/50462 | Loss: 2.5109 | LR: 1.41e-05 | 5186.4ms/step | 11.6 samples/s
+Step 45140/50462 | Loss: 2.4801 | LR: 1.40e-05 | 5187.7ms/step | 11.6 samples/s
+Step 45150/50462 | Loss: 2.5050 | LR: 1.40e-05 | 5187.6ms/step | 11.6 samples/s
+Step 45160/50462 | Loss: 2.5108 | LR: 1.39e-05 | 5185.1ms/step | 11.6 samples/s
+Step 45170/50462 | Loss: 2.4723 | LR: 1.39e-05 | 5201.9ms/step | 11.5 samples/s
+Step 45180/50462 | Loss: 2.5006 | LR: 1.38e-05 | 5186.6ms/step | 11.6 samples/s
+Step 45190/50462 | Loss: 2.4874 | LR: 1.37e-05 | 5197.7ms/step | 11.5 samples/s
+Step 45200/50462 | Loss: 2.5110 | LR: 1.37e-05 | 5201.1ms/step | 11.5 samples/s
+Step 45210/50462 | Loss: 2.4961 | LR: 1.36e-05 | 5189.7ms/step | 11.6 samples/s
+Step 45220/50462 | Loss: 2.4821 | LR: 1.36e-05 | 5181.5ms/step | 11.6 samples/s
+Step 45230/50462 | Loss: 2.4763 | LR: 1.35e-05 | 5186.5ms/step | 11.6 samples/s
+Step 45240/50462 | Loss: 2.5118 | LR: 1.35e-05 | 5184.0ms/step | 11.6 samples/s
+Step 45250/50462 | Loss: 2.4881 | LR: 1.34e-05 | 5178.8ms/step | 11.6 samples/s
+Validation at step 45250: Loss = 2.5050, Perplexity = 12.24
+Step 45260/50462 | Loss: 2.4996 | LR: 1.34e-05 | 7763.9ms/step | 7.7 samples/s
+Step 45270/50462 | Loss: 2.5095 | LR: 1.33e-05 | 5209.7ms/step | 11.5 samples/s
+Step 45280/50462 | Loss: 2.4668 | LR: 1.33e-05 | 5200.2ms/step | 11.5 samples/s
+Step 45290/50462 | Loss: 2.4931 | LR: 1.32e-05 | 5265.0ms/step | 11.4 samples/s
+Step 45300/50462 | Loss: 2.5119 | LR: 1.32e-05 | 5197.1ms/step | 11.5 samples/s
+Step 45310/50462 | Loss: 2.4799 | LR: 1.31e-05 | 5186.7ms/step | 11.6 samples/s
+Step 45320/50462 | Loss: 2.4879 | LR: 1.31e-05 | 5193.1ms/step | 11.6 samples/s
+Step 45330/50462 | Loss: 2.5046 | LR: 1.30e-05 | 5210.3ms/step | 11.5 samples/s
+Step 45340/50462 | Loss: 2.4958 | LR: 1.30e-05 | 5190.2ms/step | 11.6 samples/s
+Step 45350/50462 | Loss: 2.4928 | LR: 1.29e-05 | 5199.7ms/step | 11.5 samples/s
+Step 45360/50462 | Loss: 2.5122 | LR: 1.29e-05 | 5209.8ms/step | 11.5 samples/s
+Step 45370/50462 | Loss: 2.4854 | LR: 1.28e-05 | 5205.9ms/step | 11.5 samples/s
+Step 45380/50462 | Loss: 2.4811 | LR: 1.28e-05 | 5198.9ms/step | 11.5 samples/s
+Step 45390/50462 | Loss: 2.4878 | LR: 1.27e-05 | 5241.0ms/step | 11.4 samples/s
+Step 45400/50462 | Loss: 2.4718 | LR: 1.27e-05 | 5198.8ms/step | 11.5 samples/s
+Step 45410/50462 | Loss: 2.5171 | LR: 1.26e-05 | 5195.2ms/step | 11.5 samples/s
+Step 45420/50462 | Loss: 2.4874 | LR: 1.26e-05 | 5222.4ms/step | 11.5 samples/s
+Step 45430/50462 | Loss: 2.4626 | LR: 1.25e-05 | 5183.6ms/step | 11.6 samples/s
+Step 45440/50462 | Loss: 2.5043 | LR: 1.25e-05 | 5179.3ms/step | 11.6 samples/s
+Step 45450/50462 | Loss: 2.4944 | LR: 1.24e-05 | 5193.3ms/step | 11.6 samples/s
+Step 45460/50462 | Loss: 2.4843 | LR: 1.24e-05 | 5192.8ms/step | 11.6 samples/s
+Step 45470/50462 | Loss: 2.4933 | LR: 1.23e-05 | 5184.8ms/step | 11.6 samples/s
+Step 45480/50462 | Loss: 2.4835 | LR: 1.23e-05 | 5184.9ms/step | 11.6 samples/s
+Step 45490/50462 | Loss: 2.5048 | LR: 1.22e-05 | 5177.5ms/step | 11.6 samples/s
+Step 45500/50462 | Loss: 2.5045 | LR: 1.22e-05 | 5185.4ms/step | 11.6 samples/s
+Validation at step 45500: Loss = 2.5050, Perplexity = 12.24
+Step 45510/50462 | Loss: 2.4787 | LR: 1.21e-05 | 7725.2ms/step | 7.8 samples/s
+Step 45520/50462 | Loss: 2.5250 | LR: 1.21e-05 | 5173.0ms/step | 11.6 samples/s
+Step 45530/50462 | Loss: 2.4709 | LR: 1.20e-05 | 5184.5ms/step | 11.6 samples/s
+Step 45540/50462 | Loss: 2.5193 | LR: 1.20e-05 | 5167.8ms/step | 11.6 samples/s
+Step 45550/50462 | Loss: 2.4792 | LR: 1.19e-05 | 5167.7ms/step | 11.6 samples/s
+Step 45560/50462 | Loss: 2.5038 | LR: 1.19e-05 | 5176.6ms/step | 11.6 samples/s
+Step 45570/50462 | Loss: 2.5085 | LR: 1.19e-05 | 5178.5ms/step | 11.6 samples/s
+Step 45580/50462 | Loss: 2.4852 | LR: 1.18e-05 | 5247.9ms/step | 11.4 samples/s
+Step 45590/50462 | Loss: 2.4742 | LR: 1.18e-05 | 5219.2ms/step | 11.5 samples/s
+Step 45600/50462 | Loss: 2.4828 | LR: 1.17e-05 | 5204.7ms/step | 11.5 samples/s
+Step 45610/50462 | Loss: 2.4611 | LR: 1.17e-05 | 5194.6ms/step | 11.6 samples/s
+Step 45620/50462 | Loss: 2.4929 | LR: 1.16e-05 | 5210.7ms/step | 11.5 samples/s
+Step 45630/50462 | Loss: 2.5094 | LR: 1.16e-05 | 5207.5ms/step | 11.5 samples/s
+Step 45640/50462 | Loss: 2.4878 | LR: 1.15e-05 | 5201.9ms/step | 11.5 samples/s
+Step 45650/50462 | Loss: 2.5127 | LR: 1.15e-05 | 5203.4ms/step | 11.5 samples/s
+Step 45660/50462 | Loss: 2.4982 | LR: 1.14e-05 | 5216.1ms/step | 11.5 samples/s
+Step 45670/50462 | Loss: 2.4642 | LR: 1.14e-05 | 5218.4ms/step | 11.5 samples/s
+Step 45680/50462 | Loss: 2.4839 | LR: 1.13e-05 | 5221.1ms/step | 11.5 samples/s
+Step 45690/50462 | Loss: 2.5045 | LR: 1.13e-05 | 5246.8ms/step | 11.4 samples/s
+Step 45700/50462 | Loss: 2.4966 | LR: 1.12e-05 | 5182.5ms/step | 11.6 samples/s
+Step 45710/50462 | Loss: 2.4997 | LR: 1.12e-05 | 5187.8ms/step | 11.6 samples/s
+Step 45720/50462 | Loss: 2.4908 | LR: 1.11e-05 | 5253.9ms/step | 11.4 samples/s
+Step 45730/50462 | Loss: 2.4725 | LR: 1.11e-05 | 5201.6ms/step | 11.5 samples/s
+Step 45740/50462 | Loss: 2.4864 | LR: 1.10e-05 | 5189.2ms/step | 11.6 samples/s
+Step 45750/50462 | Loss: 2.5044 | LR: 1.10e-05 | 5204.2ms/step | 11.5 samples/s
+Validation at step 45750: Loss = 2.5049, Perplexity = 12.24
+Step 45760/50462 | Loss: 2.4592 | LR: 1.10e-05 | 7774.5ms/step | 7.7 samples/s
+Step 45770/50462 | Loss: 2.4911 | LR: 1.09e-05 | 5211.1ms/step | 11.5 samples/s
+Step 45780/50462 | Loss: 2.4834 | LR: 1.09e-05 | 5198.5ms/step | 11.5 samples/s
+Step 45790/50462 | Loss: 2.5220 | LR: 1.08e-05 | 5202.3ms/step | 11.5 samples/s
+Step 45800/50462 | Loss: 2.4354 | LR: 1.08e-05 | 5204.0ms/step | 11.5 samples/s
+Step 45810/50462 | Loss: 2.4962 | LR: 1.07e-05 | 5191.4ms/step | 11.6 samples/s
+Step 45820/50462 | Loss: 2.4923 | LR: 1.07e-05 | 5195.5ms/step | 11.5 samples/s
+Step 45830/50462 | Loss: 2.4986 | LR: 1.06e-05 | 5212.6ms/step | 11.5 samples/s
+Step 45840/50462 | Loss: 2.5116 | LR: 1.06e-05 | 5201.5ms/step | 11.5 samples/s
+Step 45850/50462 | Loss: 2.4936 | LR: 1.05e-05 | 5211.5ms/step | 11.5 samples/s
+Step 45860/50462 | Loss: 2.5182 | LR: 1.05e-05 | 5206.8ms/step | 11.5 samples/s
+Step 45870/50462 | Loss: 2.5032 | LR: 1.05e-05 | 5242.5ms/step | 11.4 samples/s
+Step 45880/50462 | Loss: 2.5039 | LR: 1.04e-05 | 5184.8ms/step | 11.6 samples/s
+Step 45890/50462 | Loss: 2.4665 | LR: 1.04e-05 | 5190.1ms/step | 11.6 samples/s
+Step 45900/50462 | Loss: 2.4898 | LR: 1.03e-05 | 5178.0ms/step | 11.6 samples/s
+Step 45910/50462 | Loss: 2.5200 | LR: 1.03e-05 | 5191.4ms/step | 11.6 samples/s
+Step 45920/50462 | Loss: 2.4733 | LR: 1.02e-05 | 5184.1ms/step | 11.6 samples/s
+Step 45930/50462 | Loss: 2.4848 | LR: 1.02e-05 | 5181.2ms/step | 11.6 samples/s
+Step 45940/50462 | Loss: 2.5039 | LR: 1.01e-05 | 5173.1ms/step | 11.6 samples/s
+Step 45950/50462 | Loss: 2.4910 | LR: 1.01e-05 | 5175.2ms/step | 11.6 samples/s
+Step 45960/50462 | Loss: 2.4769 | LR: 1.01e-05 | 5181.8ms/step | 11.6 samples/s
+Step 45970/50462 | Loss: 2.4824 | LR: 1.00e-05 | 5163.9ms/step | 11.6 samples/s
+Step 45980/50462 | Loss: 2.5263 | LR: 9.96e-06 | 5187.7ms/step | 11.6 samples/s
+Step 45990/50462 | Loss: 2.4801 | LR: 9.92e-06 | 5190.0ms/step | 11.6 samples/s
+Step 46000/50462 | Loss: 2.4944 | LR: 9.87e-06 | 5252.5ms/step | 11.4 samples/s
+Validation at step 46000: Loss = 2.5049, Perplexity = 12.24
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-46000
+Step 46010/50462 | Loss: 2.4972 | LR: 9.83e-06 | 8699.2ms/step | 6.9 samples/s
+Step 46020/50462 | Loss: 2.4834 | LR: 9.79e-06 | 5214.3ms/step | 11.5 samples/s
+Step 46030/50462 | Loss: 2.5078 | LR: 9.74e-06 | 5192.7ms/step | 11.6 samples/s
+Step 46040/50462 | Loss: 2.5078 | LR: 9.70e-06 | 5201.9ms/step | 11.5 samples/s
+Step 46050/50462 | Loss: 2.4901 | LR: 9.65e-06 | 5202.2ms/step | 11.5 samples/s
+Step 46060/50462 | Loss: 2.4847 | LR: 9.61e-06 | 5204.8ms/step | 11.5 samples/s
+Step 46070/50462 | Loss: 2.5043 | LR: 9.57e-06 | 5186.0ms/step | 11.6 samples/s
+Step 46080/50462 | Loss: 2.4995 | LR: 9.52e-06 | 5212.7ms/step | 11.5 samples/s
+Step 46090/50462 | Loss: 2.4684 | LR: 9.48e-06 | 5211.2ms/step | 11.5 samples/s
+Step 46100/50462 | Loss: 2.4948 | LR: 9.44e-06 | 5192.0ms/step | 11.6 samples/s
+Step 46110/50462 | Loss: 2.4900 | LR: 9.40e-06 | 5213.0ms/step | 11.5 samples/s
+Step 46120/50462 | Loss: 2.4849 | LR: 9.35e-06 | 5202.5ms/step | 11.5 samples/s
+Step 46130/50462 | Loss: 2.4863 | LR: 9.31e-06 | 5197.7ms/step | 11.5 samples/s
+Step 46140/50462 | Loss: 2.4870 | LR: 9.27e-06 | 5204.4ms/step | 11.5 samples/s
+Step 46150/50462 | Loss: 2.4853 | LR: 9.22e-06 | 5185.9ms/step | 11.6 samples/s
+Step 46160/50462 | Loss: 2.5033 | LR: 9.18e-06 | 5251.5ms/step | 11.4 samples/s
+Step 46170/50462 | Loss: 2.5035 | LR: 9.14e-06 | 5172.3ms/step | 11.6 samples/s
+Step 46180/50462 | Loss: 2.4734 | LR: 9.10e-06 | 5176.3ms/step | 11.6 samples/s
+Step 46190/50462 | Loss: 2.4736 | LR: 9.06e-06 | 5157.9ms/step | 11.6 samples/s
+Step 46200/50462 | Loss: 2.4817 | LR: 9.01e-06 | 5154.3ms/step | 11.6 samples/s
+Step 46210/50462 | Loss: 2.4980 | LR: 8.97e-06 | 5191.7ms/step | 11.6 samples/s
+Step 46220/50462 | Loss: 2.5004 | LR: 8.93e-06 | 5176.2ms/step | 11.6 samples/s
+Step 46230/50462 | Loss: 2.5075 | LR: 8.89e-06 | 5206.5ms/step | 11.5 samples/s
+Step 46240/50462 | Loss: 2.5160 | LR: 8.85e-06 | 5202.9ms/step | 11.5 samples/s
+Step 46250/50462 | Loss: 2.5136 | LR: 8.80e-06 | 5196.2ms/step | 11.5 samples/s
+Validation at step 46250: Loss = 2.5049, Perplexity = 12.24
+Step 46260/50462 | Loss: 2.5136 | LR: 8.76e-06 | 7727.5ms/step | 7.8 samples/s
+Step 46270/50462 | Loss: 2.5004 | LR: 8.72e-06 | 5190.9ms/step | 11.6 samples/s
+Step 46280/50462 | Loss: 2.5317 | LR: 8.68e-06 | 5183.3ms/step | 11.6 samples/s
+Step 46290/50462 | Loss: 2.5176 | LR: 8.64e-06 | 5179.9ms/step | 11.6 samples/s
+Step 46300/50462 | Loss: 2.4887 | LR: 8.60e-06 | 5243.9ms/step | 11.4 samples/s
+Step 46310/50462 | Loss: 2.4963 | LR: 8.56e-06 | 5175.4ms/step | 11.6 samples/s
+Step 46320/50462 | Loss: 2.5217 | LR: 8.52e-06 | 5188.3ms/step | 11.6 samples/s
+Step 46330/50462 | Loss: 2.5018 | LR: 8.47e-06 | 5224.0ms/step | 11.5 samples/s
+Step 46340/50462 | Loss: 2.5011 | LR: 8.43e-06 | 5186.2ms/step | 11.6 samples/s
+Step 46350/50462 | Loss: 2.5222 | LR: 8.39e-06 | 5164.1ms/step | 11.6 samples/s
+Step 46360/50462 | Loss: 2.4844 | LR: 8.35e-06 | 5167.3ms/step | 11.6 samples/s
+Step 46370/50462 | Loss: 2.4706 | LR: 8.31e-06 | 5174.8ms/step | 11.6 samples/s
+Step 46380/50462 | Loss: 2.4955 | LR: 8.27e-06 | 5185.8ms/step | 11.6 samples/s
+Step 46390/50462 | Loss: 2.4847 | LR: 8.23e-06 | 5173.9ms/step | 11.6 samples/s
+Step 46400/50462 | Loss: 2.4765 | LR: 8.19e-06 | 5172.2ms/step | 11.6 samples/s
+Step 46410/50462 | Loss: 2.4890 | LR: 8.15e-06 | 5183.7ms/step | 11.6 samples/s
+Step 46420/50462 | Loss: 2.4881 | LR: 8.11e-06 | 5170.6ms/step | 11.6 samples/s
+Step 46430/50462 | Loss: 2.5091 | LR: 8.07e-06 | 5174.4ms/step | 11.6 samples/s
+Step 46440/50462 | Loss: 2.4920 | LR: 8.03e-06 | 5173.6ms/step | 11.6 samples/s
+Step 46450/50462 | Loss: 2.5176 | LR: 7.99e-06 | 5232.9ms/step | 11.5 samples/s
+Step 46460/50462 | Loss: 2.4957 | LR: 7.95e-06 | 5187.6ms/step | 11.6 samples/s
+Step 46470/50462 | Loss: 2.4873 | LR: 7.91e-06 | 5177.5ms/step | 11.6 samples/s
+Step 46480/50462 | Loss: 2.4829 | LR: 7.87e-06 | 5186.0ms/step | 11.6 samples/s
+Step 46490/50462 | Loss: 2.4782 | LR: 7.83e-06 | 5176.1ms/step | 11.6 samples/s
+Step 46500/50462 | Loss: 2.4953 | LR: 7.80e-06 | 5165.8ms/step | 11.6 samples/s
+Validation at step 46500: Loss = 2.5049, Perplexity = 12.24
+Step 46510/50462 | Loss: 2.4902 | LR: 7.76e-06 | 7749.4ms/step | 7.7 samples/s
+Step 46520/50462 | Loss: 2.5135 | LR: 7.72e-06 | 5187.4ms/step | 11.6 samples/s
+Step 46530/50462 | Loss: 2.4931 | LR: 7.68e-06 | 5205.1ms/step | 11.5 samples/s
+Step 46540/50462 | Loss: 2.5066 | LR: 7.64e-06 | 5192.6ms/step | 11.6 samples/s
+Step 46550/50462 | Loss: 2.4674 | LR: 7.60e-06 | 5196.7ms/step | 11.5 samples/s
+Step 46560/50462 | Loss: 2.4845 | LR: 7.56e-06 | 5181.3ms/step | 11.6 samples/s
+Step 46570/50462 | Loss: 2.4955 | LR: 7.52e-06 | 5194.7ms/step | 11.6 samples/s
+Step 46580/50462 | Loss: 2.4876 | LR: 7.49e-06 | 5191.6ms/step | 11.6 samples/s
+Step 46590/50462 | Loss: 2.4797 | LR: 7.45e-06 | 5205.0ms/step | 11.5 samples/s
+Step 46600/50462 | Loss: 2.4952 | LR: 7.41e-06 | 5199.4ms/step | 11.5 samples/s
+Step 46610/50462 | Loss: 2.4932 | LR: 7.37e-06 | 5246.2ms/step | 11.4 samples/s
+Step 46620/50462 | Loss: 2.4753 | LR: 7.33e-06 | 5163.2ms/step | 11.6 samples/s
+Step 46630/50462 | Loss: 2.4900 | LR: 7.29e-06 | 5222.8ms/step | 11.5 samples/s
+Step 46640/50462 | Loss: 2.5022 | LR: 7.26e-06 | 5201.0ms/step | 11.5 samples/s
+Step 46650/50462 | Loss: 2.4942 | LR: 7.22e-06 | 5195.5ms/step | 11.5 samples/s
+Step 46660/50462 | Loss: 2.4840 | LR: 7.18e-06 | 5195.6ms/step | 11.5 samples/s
+Step 46670/50462 | Loss: 2.4988 | LR: 7.14e-06 | 5197.2ms/step | 11.5 samples/s
+Step 46680/50462 | Loss: 2.5076 | LR: 7.11e-06 | 5190.2ms/step | 11.6 samples/s
+Step 46690/50462 | Loss: 2.5246 | LR: 7.07e-06 | 5203.1ms/step | 11.5 samples/s
+Step 46700/50462 | Loss: 2.4851 | LR: 7.03e-06 | 5193.3ms/step | 11.6 samples/s
+Step 46710/50462 | Loss: 2.5175 | LR: 6.99e-06 | 5198.8ms/step | 11.5 samples/s
+Step 46720/50462 | Loss: 2.5011 | LR: 6.96e-06 | 5196.4ms/step | 11.5 samples/s
+Step 46730/50462 | Loss: 2.4808 | LR: 6.92e-06 | 5202.9ms/step | 11.5 samples/s
+Step 46740/50462 | Loss: 2.5074 | LR: 6.88e-06 | 5191.6ms/step | 11.6 samples/s
+Step 46750/50462 | Loss: 2.4906 | LR: 6.85e-06 | 5222.0ms/step | 11.5 samples/s
+Validation at step 46750: Loss = 2.5049, Perplexity = 12.24
+Step 46760/50462 | Loss: 2.5052 | LR: 6.81e-06 | 7730.9ms/step | 7.8 samples/s
+Step 46770/50462 | Loss: 2.4943 | LR: 6.77e-06 | 5159.1ms/step | 11.6 samples/s
+Step 46780/50462 | Loss: 2.5204 | LR: 6.74e-06 | 5184.9ms/step | 11.6 samples/s
+Step 46790/50462 | Loss: 2.4776 | LR: 6.70e-06 | 5170.3ms/step | 11.6 samples/s
+Step 46800/50462 | Loss: 2.5151 | LR: 6.66e-06 | 5160.8ms/step | 11.6 samples/s
+Step 46810/50462 | Loss: 2.5114 | LR: 6.63e-06 | 5181.9ms/step | 11.6 samples/s
+Step 46820/50462 | Loss: 2.4747 | LR: 6.59e-06 | 5169.7ms/step | 11.6 samples/s
+Step 46830/50462 | Loss: 2.4728 | LR: 6.56e-06 | 5176.0ms/step | 11.6 samples/s
+Step 46840/50462 | Loss: 2.4985 | LR: 6.52e-06 | 5184.4ms/step | 11.6 samples/s
+Step 46850/50462 | Loss: 2.4946 | LR: 6.48e-06 | 5174.9ms/step | 11.6 samples/s
+Step 46860/50462 | Loss: 2.4941 | LR: 6.45e-06 | 5167.7ms/step | 11.6 samples/s
+Step 46870/50462 | Loss: 2.4520 | LR: 6.41e-06 | 5172.2ms/step | 11.6 samples/s
+Step 46880/50462 | Loss: 2.4878 | LR: 6.38e-06 | 5170.6ms/step | 11.6 samples/s
+Step 46890/50462 | Loss: 2.4936 | LR: 6.34e-06 | 5169.1ms/step | 11.6 samples/s
+Step 46900/50462 | Loss: 2.4857 | LR: 6.31e-06 | 5173.0ms/step | 11.6 samples/s
+Step 46910/50462 | Loss: 2.4788 | LR: 6.27e-06 | 5219.4ms/step | 11.5 samples/s
+Step 46920/50462 | Loss: 2.5168 | LR: 6.24e-06 | 5186.9ms/step | 11.6 samples/s
+Step 46930/50462 | Loss: 2.4932 | LR: 6.20e-06 | 5234.2ms/step | 11.5 samples/s
+Step 46940/50462 | Loss: 2.4948 | LR: 6.17e-06 | 5198.8ms/step | 11.5 samples/s
+Step 46950/50462 | Loss: 2.4777 | LR: 6.13e-06 | 5164.3ms/step | 11.6 samples/s
+Step 46960/50462 | Loss: 2.5014 | LR: 6.10e-06 | 5171.3ms/step | 11.6 samples/s
+Step 46970/50462 | Loss: 2.5051 | LR: 6.06e-06 | 5153.7ms/step | 11.6 samples/s
+Step 46980/50462 | Loss: 2.5043 | LR: 6.03e-06 | 5184.5ms/step | 11.6 samples/s
+Step 46990/50462 | Loss: 2.5255 | LR: 5.99e-06 | 5179.1ms/step | 11.6 samples/s
+Step 47000/50462 | Loss: 2.5053 | LR: 5.96e-06 | 5176.9ms/step | 11.6 samples/s
+Validation at step 47000: Loss = 2.5048, Perplexity = 12.24
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-47000
+Step 47010/50462 | Loss: 2.5168 | LR: 5.93e-06 | 8707.4ms/step | 6.9 samples/s
+Step 47020/50462 | Loss: 2.4936 | LR: 5.89e-06 | 5186.7ms/step | 11.6 samples/s
+Step 47030/50462 | Loss: 2.5132 | LR: 5.86e-06 | 5240.4ms/step | 11.4 samples/s
+Step 47040/50462 | Loss: 2.4586 | LR: 5.82e-06 | 5206.8ms/step | 11.5 samples/s
+Step 47050/50462 | Loss: 2.4955 | LR: 5.79e-06 | 5205.5ms/step | 11.5 samples/s
+Step 47060/50462 | Loss: 2.4934 | LR: 5.76e-06 | 5200.7ms/step | 11.5 samples/s
+Step 47070/50462 | Loss: 2.4866 | LR: 5.72e-06 | 5200.7ms/step | 11.5 samples/s
+Step 47080/50462 | Loss: 2.4942 | LR: 5.69e-06 | 5174.5ms/step | 11.6 samples/s
+Step 47090/50462 | Loss: 2.4890 | LR: 5.65e-06 | 5190.4ms/step | 11.6 samples/s
+Step 47100/50462 | Loss: 2.4929 | LR: 5.62e-06 | 5185.1ms/step | 11.6 samples/s
+Step 47110/50462 | Loss: 2.4969 | LR: 5.59e-06 | 5213.4ms/step | 11.5 samples/s
+Step 47120/50462 | Loss: 2.4472 | LR: 5.56e-06 | 5197.5ms/step | 11.5 samples/s
+Step 47130/50462 | Loss: 2.4940 | LR: 5.52e-06 | 5202.0ms/step | 11.5 samples/s
+Step 47140/50462 | Loss: 2.5065 | LR: 5.49e-06 | 5208.1ms/step | 11.5 samples/s
+Step 47150/50462 | Loss: 2.4626 | LR: 5.46e-06 | 5173.6ms/step | 11.6 samples/s
+Step 47160/50462 | Loss: 2.5041 | LR: 5.42e-06 | 5200.5ms/step | 11.5 samples/s
+Step 47170/50462 | Loss: 2.4873 | LR: 5.39e-06 | 5184.8ms/step | 11.6 samples/s
+Step 47180/50462 | Loss: 2.4973 | LR: 5.36e-06 | 5173.0ms/step | 11.6 samples/s
+Step 47190/50462 | Loss: 2.4916 | LR: 5.33e-06 | 5193.3ms/step | 11.6 samples/s
+Step 47200/50462 | Loss: 2.4893 | LR: 5.29e-06 | 5188.8ms/step | 11.6 samples/s
+Step 47210/50462 | Loss: 2.4631 | LR: 5.26e-06 | 5203.6ms/step | 11.5 samples/s
+Step 47220/50462 | Loss: 2.5042 | LR: 5.23e-06 | 5233.6ms/step | 11.5 samples/s
+Step 47230/50462 | Loss: 2.5017 | LR: 5.20e-06 | 5182.9ms/step | 11.6 samples/s
+Step 47240/50462 | Loss: 2.4710 | LR: 5.16e-06 | 5224.0ms/step | 11.5 samples/s
+Step 47250/50462 | Loss: 2.5009 | LR: 5.13e-06 | 5167.7ms/step | 11.6 samples/s
+Validation at step 47250: Loss = 2.5048, Perplexity = 12.24
+Step 47260/50462 | Loss: 2.4958 | LR: 5.10e-06 | 7725.9ms/step | 7.8 samples/s
+Step 47270/50462 | Loss: 2.5072 | LR: 5.07e-06 | 5187.3ms/step | 11.6 samples/s
+Step 47280/50462 | Loss: 2.4806 | LR: 5.04e-06 | 5190.9ms/step | 11.6 samples/s
+Step 47290/50462 | Loss: 2.5090 | LR: 5.01e-06 | 5175.9ms/step | 11.6 samples/s
+Step 47300/50462 | Loss: 2.5073 | LR: 4.97e-06 | 5173.6ms/step | 11.6 samples/s
+Step 47310/50462 | Loss: 2.4698 | LR: 4.94e-06 | 5185.4ms/step | 11.6 samples/s
+Step 47320/50462 | Loss: 2.4862 | LR: 4.91e-06 | 5171.5ms/step | 11.6 samples/s
+Step 47330/50462 | Loss: 2.4897 | LR: 4.88e-06 | 5231.8ms/step | 11.5 samples/s
+Step 47340/50462 | Loss: 2.4542 | LR: 4.85e-06 | 5171.5ms/step | 11.6 samples/s
+Step 47350/50462 | Loss: 2.5018 | LR: 4.82e-06 | 5159.6ms/step | 11.6 samples/s
+Step 47360/50462 | Loss: 2.4725 | LR: 4.79e-06 | 5173.0ms/step | 11.6 samples/s
+Step 47370/50462 | Loss: 2.5110 | LR: 4.76e-06 | 5163.0ms/step | 11.6 samples/s
+Step 47380/50462 | Loss: 2.5032 | LR: 4.73e-06 | 5183.0ms/step | 11.6 samples/s
+Step 47390/50462 | Loss: 2.4857 | LR: 4.70e-06 | 5200.3ms/step | 11.5 samples/s
+Step 47400/50462 | Loss: 2.4892 | LR: 4.67e-06 | 5184.6ms/step | 11.6 samples/s
+Step 47410/50462 | Loss: 2.5004 | LR: 4.64e-06 | 5165.8ms/step | 11.6 samples/s
+Step 47420/50462 | Loss: 2.4523 | LR: 4.61e-06 | 5167.7ms/step | 11.6 samples/s
+Step 47430/50462 | Loss: 2.4803 | LR: 4.58e-06 | 5164.2ms/step | 11.6 samples/s
+Step 47440/50462 | Loss: 2.4889 | LR: 4.55e-06 | 5163.8ms/step | 11.6 samples/s
+Step 47450/50462 | Loss: 2.4637 | LR: 4.52e-06 | 5161.8ms/step | 11.6 samples/s
+Step 47460/50462 | Loss: 2.5085 | LR: 4.49e-06 | 5151.5ms/step | 11.6 samples/s
+Step 47470/50462 | Loss: 2.5061 | LR: 4.46e-06 | 5172.4ms/step | 11.6 samples/s
+Step 47480/50462 | Loss: 2.5048 | LR: 4.43e-06 | 5182.2ms/step | 11.6 samples/s
+Step 47490/50462 | Loss: 2.4981 | LR: 4.40e-06 | 5183.1ms/step | 11.6 samples/s
+Step 47500/50462 | Loss: 2.4890 | LR: 4.37e-06 | 5169.8ms/step | 11.6 samples/s
+Validation at step 47500: Loss = 2.5048, Perplexity = 12.24
+Step 47510/50462 | Loss: 2.4600 | LR: 4.34e-06 | 7722.4ms/step | 7.8 samples/s
+Step 47520/50462 | Loss: 2.5080 | LR: 4.31e-06 | 5236.0ms/step | 11.5 samples/s
+Step 47530/50462 | Loss: 2.5029 | LR: 4.28e-06 | 5157.6ms/step | 11.6 samples/s
+Step 47540/50462 | Loss: 2.4708 | LR: 4.25e-06 | 5219.2ms/step | 11.5 samples/s
+Step 47550/50462 | Loss: 2.5007 | LR: 4.22e-06 | 5219.3ms/step | 11.5 samples/s
+Step 47560/50462 | Loss: 2.4864 | LR: 4.19e-06 | 5213.7ms/step | 11.5 samples/s
+Step 47570/50462 | Loss: 2.4968 | LR: 4.16e-06 | 5190.2ms/step | 11.6 samples/s
+Step 47580/50462 | Loss: 2.5322 | LR: 4.13e-06 | 5181.6ms/step | 11.6 samples/s
+Step 47590/50462 | Loss: 2.5026 | LR: 4.11e-06 | 5203.0ms/step | 11.5 samples/s
+Step 47600/50462 | Loss: 2.4937 | LR: 4.08e-06 | 5194.2ms/step | 11.6 samples/s
+Step 47610/50462 | Loss: 2.4731 | LR: 4.05e-06 | 5197.8ms/step | 11.5 samples/s
+Step 47620/50462 | Loss: 2.4871 | LR: 4.02e-06 | 5243.7ms/step | 11.4 samples/s
+Step 47630/50462 | Loss: 2.5144 | LR: 3.99e-06 | 5199.7ms/step | 11.5 samples/s
+Step 47640/50462 | Loss: 2.4698 | LR: 3.97e-06 | 5210.5ms/step | 11.5 samples/s
+Step 47650/50462 | Loss: 2.4882 | LR: 3.94e-06 | 5209.7ms/step | 11.5 samples/s
+Step 47660/50462 | Loss: 2.4879 | LR: 3.91e-06 | 5209.8ms/step | 11.5 samples/s
+Step 47670/50462 | Loss: 2.4799 | LR: 3.88e-06 | 5204.8ms/step | 11.5 samples/s
+Step 47680/50462 | Loss: 2.4980 | LR: 3.85e-06 | 5201.2ms/step | 11.5 samples/s
+Step 47690/50462 | Loss: 2.4934 | LR: 3.83e-06 | 5195.1ms/step | 11.5 samples/s
+Step 47700/50462 | Loss: 2.5015 | LR: 3.80e-06 | 5205.7ms/step | 11.5 samples/s
+Step 47710/50462 | Loss: 2.4681 | LR: 3.77e-06 | 5196.7ms/step | 11.5 samples/s
+Step 47720/50462 | Loss: 2.4946 | LR: 3.74e-06 | 5205.1ms/step | 11.5 samples/s
+Step 47730/50462 | Loss: 2.4959 | LR: 3.72e-06 | 5214.4ms/step | 11.5 samples/s
+Step 47740/50462 | Loss: 2.4806 | LR: 3.69e-06 | 5198.5ms/step | 11.5 samples/s
+Step 47750/50462 | Loss: 2.5072 | LR: 3.66e-06 | 5208.7ms/step | 11.5 samples/s
+Validation at step 47750: Loss = 2.5048, Perplexity = 12.24
+Step 47760/50462 | Loss: 2.4646 | LR: 3.64e-06 | 7766.7ms/step | 7.7 samples/s
+Step 47770/50462 | Loss: 2.5274 | LR: 3.61e-06 | 5202.6ms/step | 11.5 samples/s
+Step 47780/50462 | Loss: 2.4847 | LR: 3.58e-06 | 5200.7ms/step | 11.5 samples/s
+Step 47790/50462 | Loss: 2.4790 | LR: 3.56e-06 | 5215.1ms/step | 11.5 samples/s
+Step 47800/50462 | Loss: 2.4709 | LR: 3.53e-06 | 5206.1ms/step | 11.5 samples/s
+Step 47810/50462 | Loss: 2.5045 | LR: 3.50e-06 | 5205.1ms/step | 11.5 samples/s
+Step 47820/50462 | Loss: 2.4975 | LR: 3.48e-06 | 5204.7ms/step | 11.5 samples/s
+Step 47830/50462 | Loss: 2.4900 | LR: 3.45e-06 | 5265.4ms/step | 11.4 samples/s
+Step 47840/50462 | Loss: 2.5129 | LR: 3.42e-06 | 5202.3ms/step | 11.5 samples/s
+Step 47850/50462 | Loss: 2.5031 | LR: 3.40e-06 | 5237.6ms/step | 11.5 samples/s
+Step 47860/50462 | Loss: 2.5052 | LR: 3.37e-06 | 5183.5ms/step | 11.6 samples/s
+Step 47870/50462 | Loss: 2.4996 | LR: 3.35e-06 | 5163.0ms/step | 11.6 samples/s
+Step 47880/50462 | Loss: 2.4540 | LR: 3.32e-06 | 5172.5ms/step | 11.6 samples/s
+Step 47890/50462 | Loss: 2.5120 | LR: 3.30e-06 | 5178.6ms/step | 11.6 samples/s
+Step 47900/50462 | Loss: 2.4997 | LR: 3.27e-06 | 5170.5ms/step | 11.6 samples/s
+Step 47910/50462 | Loss: 2.4997 | LR: 3.24e-06 | 5252.4ms/step | 11.4 samples/s
+Step 47920/50462 | Loss: 2.4736 | LR: 3.22e-06 | 5212.3ms/step | 11.5 samples/s
+Step 47930/50462 | Loss: 2.5108 | LR: 3.19e-06 | 5182.6ms/step | 11.6 samples/s
+Step 47940/50462 | Loss: 2.4882 | LR: 3.17e-06 | 5194.5ms/step | 11.6 samples/s
+Step 47950/50462 | Loss: 2.4943 | LR: 3.14e-06 | 5192.0ms/step | 11.6 samples/s
+Step 47960/50462 | Loss: 2.4612 | LR: 3.12e-06 | 5213.9ms/step | 11.5 samples/s
+Step 47970/50462 | Loss: 2.4978 | LR: 3.09e-06 | 5208.2ms/step | 11.5 samples/s
+Step 47980/50462 | Loss: 2.4943 | LR: 3.07e-06 | 5158.7ms/step | 11.6 samples/s
+Step 47990/50462 | Loss: 2.4774 | LR: 3.04e-06 | 5176.8ms/step | 11.6 samples/s
+Step 48000/50462 | Loss: 2.4876 | LR: 3.02e-06 | 5183.8ms/step | 11.6 samples/s
+Validation at step 48000: Loss = 2.5048, Perplexity = 12.24
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-48000
+Step 48010/50462 | Loss: 2.5007 | LR: 3.00e-06 | 8752.7ms/step | 6.9 samples/s
+Step 48020/50462 | Loss: 2.5287 | LR: 2.97e-06 | 5155.9ms/step | 11.6 samples/s
+Step 48030/50462 | Loss: 2.4702 | LR: 2.95e-06 | 5177.6ms/step | 11.6 samples/s
+Step 48040/50462 | Loss: 2.4842 | LR: 2.92e-06 | 5172.5ms/step | 11.6 samples/s
+Step 48050/50462 | Loss: 2.4769 | LR: 2.90e-06 | 5171.3ms/step | 11.6 samples/s
+Step 48060/50462 | Loss: 2.4942 | LR: 2.87e-06 | 5193.9ms/step | 11.6 samples/s
+Step 48070/50462 | Loss: 2.4968 | LR: 2.85e-06 | 5161.5ms/step | 11.6 samples/s
+Step 48080/50462 | Loss: 2.5208 | LR: 2.83e-06 | 5177.3ms/step | 11.6 samples/s
+Step 48090/50462 | Loss: 2.4963 | LR: 2.80e-06 | 5172.9ms/step | 11.6 samples/s
+Step 48100/50462 | Loss: 2.4999 | LR: 2.78e-06 | 5170.7ms/step | 11.6 samples/s
+Step 48110/50462 | Loss: 2.4892 | LR: 2.76e-06 | 5164.7ms/step | 11.6 samples/s
+Step 48120/50462 | Loss: 2.4872 | LR: 2.73e-06 | 5173.8ms/step | 11.6 samples/s
+Step 48130/50462 | Loss: 2.4885 | LR: 2.71e-06 | 5173.1ms/step | 11.6 samples/s
+Step 48140/50462 | Loss: 2.4703 | LR: 2.69e-06 | 5220.6ms/step | 11.5 samples/s
+Step 48150/50462 | Loss: 2.4834 | LR: 2.66e-06 | 5224.9ms/step | 11.5 samples/s
+Step 48160/50462 | Loss: 2.5029 | LR: 2.64e-06 | 5215.3ms/step | 11.5 samples/s
+Step 48170/50462 | Loss: 2.4828 | LR: 2.62e-06 | 5201.5ms/step | 11.5 samples/s
+Step 48180/50462 | Loss: 2.4954 | LR: 2.60e-06 | 5208.1ms/step | 11.5 samples/s
+Step 48190/50462 | Loss: 2.4914 | LR: 2.57e-06 | 5210.1ms/step | 11.5 samples/s
+Step 48200/50462 | Loss: 2.5012 | LR: 2.55e-06 | 5246.8ms/step | 11.4 samples/s
+Step 48210/50462 | Loss: 2.4770 | LR: 2.53e-06 | 5202.8ms/step | 11.5 samples/s
+Step 48220/50462 | Loss: 2.4825 | LR: 2.51e-06 | 5209.3ms/step | 11.5 samples/s
+Step 48230/50462 | Loss: 2.4899 | LR: 2.48e-06 | 5213.2ms/step | 11.5 samples/s
+Step 48240/50462 | Loss: 2.4881 | LR: 2.46e-06 | 5211.1ms/step | 11.5 samples/s
+Step 48250/50462 | Loss: 2.5273 | LR: 2.44e-06 | 5213.0ms/step | 11.5 samples/s
+Validation at step 48250: Loss = 2.5048, Perplexity = 12.24
+Step 48260/50462 | Loss: 2.4645 | LR: 2.42e-06 | 7771.1ms/step | 7.7 samples/s
+Step 48270/50462 | Loss: 2.4902 | LR: 2.39e-06 | 5210.3ms/step | 11.5 samples/s
+Step 48280/50462 | Loss: 2.4982 | LR: 2.37e-06 | 5195.8ms/step | 11.5 samples/s
+Step 48290/50462 | Loss: 2.4860 | LR: 2.35e-06 | 5223.5ms/step | 11.5 samples/s
+Step 48300/50462 | Loss: 2.4793 | LR: 2.33e-06 | 5211.3ms/step | 11.5 samples/s
+Step 48310/50462 | Loss: 2.4994 | LR: 2.31e-06 | 5208.5ms/step | 11.5 samples/s
+Step 48320/50462 | Loss: 2.4982 | LR: 2.29e-06 | 5206.0ms/step | 11.5 samples/s
+Step 48330/50462 | Loss: 2.4812 | LR: 2.27e-06 | 5184.7ms/step | 11.6 samples/s
+Step 48340/50462 | Loss: 2.5049 | LR: 2.24e-06 | 5202.1ms/step | 11.5 samples/s
+Step 48350/50462 | Loss: 2.5183 | LR: 2.22e-06 | 5204.0ms/step | 11.5 samples/s
+Step 48360/50462 | Loss: 2.4743 | LR: 2.20e-06 | 5194.3ms/step | 11.6 samples/s
+Step 48370/50462 | Loss: 2.5124 | LR: 2.18e-06 | 5192.6ms/step | 11.6 samples/s
+Step 48380/50462 | Loss: 2.4784 | LR: 2.16e-06 | 5204.3ms/step | 11.5 samples/s
+Step 48390/50462 | Loss: 2.5268 | LR: 2.14e-06 | 5189.8ms/step | 11.6 samples/s
+Step 48400/50462 | Loss: 2.4811 | LR: 2.12e-06 | 5219.6ms/step | 11.5 samples/s
+Step 48410/50462 | Loss: 2.5098 | LR: 2.10e-06 | 5200.8ms/step | 11.5 samples/s
+Step 48420/50462 | Loss: 2.4861 | LR: 2.08e-06 | 5206.8ms/step | 11.5 samples/s
+Step 48430/50462 | Loss: 2.4827 | LR: 2.06e-06 | 5208.2ms/step | 11.5 samples/s
+Step 48440/50462 | Loss: 2.5002 | LR: 2.04e-06 | 5238.1ms/step | 11.5 samples/s
+Step 48450/50462 | Loss: 2.4658 | LR: 2.02e-06 | 5187.7ms/step | 11.6 samples/s
+Step 48460/50462 | Loss: 2.5132 | LR: 2.00e-06 | 5229.7ms/step | 11.5 samples/s
+Step 48470/50462 | Loss: 2.5008 | LR: 1.98e-06 | 5162.1ms/step | 11.6 samples/s
+Step 48480/50462 | Loss: 2.4844 | LR: 1.96e-06 | 5163.8ms/step | 11.6 samples/s
+Step 48490/50462 | Loss: 2.4921 | LR: 1.94e-06 | 5230.6ms/step | 11.5 samples/s
+Step 48500/50462 | Loss: 2.4614 | LR: 1.92e-06 | 5197.7ms/step | 11.5 samples/s
+Validation at step 48500: Loss = 2.5048, Perplexity = 12.24
+Step 48510/50462 | Loss: 2.4943 | LR: 1.90e-06 | 7721.8ms/step | 7.8 samples/s
+Step 48520/50462 | Loss: 2.4938 | LR: 1.88e-06 | 5186.9ms/step | 11.6 samples/s
+Step 48530/50462 | Loss: 2.5110 | LR: 1.86e-06 | 5204.1ms/step | 11.5 samples/s
+Step 48540/50462 | Loss: 2.4682 | LR: 1.84e-06 | 5184.9ms/step | 11.6 samples/s
+Step 48550/50462 | Loss: 2.4775 | LR: 1.82e-06 | 5186.0ms/step | 11.6 samples/s
+Step 48560/50462 | Loss: 2.4882 | LR: 1.80e-06 | 5193.3ms/step | 11.6 samples/s
+Step 48570/50462 | Loss: 2.4897 | LR: 1.78e-06 | 5191.4ms/step | 11.6 samples/s
+Step 48580/50462 | Loss: 2.4826 | LR: 1.77e-06 | 5212.3ms/step | 11.5 samples/s
+Step 48590/50462 | Loss: 2.5070 | LR: 1.75e-06 | 5169.8ms/step | 11.6 samples/s
+Step 48600/50462 | Loss: 2.4984 | LR: 1.73e-06 | 5183.8ms/step | 11.6 samples/s
+Step 48610/50462 | Loss: 2.4841 | LR: 1.71e-06 | 5181.4ms/step | 11.6 samples/s
+Step 48620/50462 | Loss: 2.4898 | LR: 1.69e-06 | 5183.7ms/step | 11.6 samples/s
+Step 48630/50462 | Loss: 2.5119 | LR: 1.67e-06 | 5165.8ms/step | 11.6 samples/s
+Step 48640/50462 | Loss: 2.4940 | LR: 1.66e-06 | 5208.5ms/step | 11.5 samples/s
+Step 48650/50462 | Loss: 2.5105 | LR: 1.64e-06 | 5196.0ms/step | 11.5 samples/s
+Step 48660/50462 | Loss: 2.5061 | LR: 1.62e-06 | 5178.1ms/step | 11.6 samples/s
+Step 48670/50462 | Loss: 2.5075 | LR: 1.60e-06 | 5170.7ms/step | 11.6 samples/s
+Step 48680/50462 | Loss: 2.4898 | LR: 1.58e-06 | 5183.1ms/step | 11.6 samples/s
+Step 48690/50462 | Loss: 2.5035 | LR: 1.57e-06 | 5183.5ms/step | 11.6 samples/s
+Step 48700/50462 | Loss: 2.4631 | LR: 1.55e-06 | 5179.1ms/step | 11.6 samples/s
+Step 48710/50462 | Loss: 2.4862 | LR: 1.53e-06 | 5181.1ms/step | 11.6 samples/s
+Step 48720/50462 | Loss: 2.5078 | LR: 1.51e-06 | 5189.1ms/step | 11.6 samples/s
+Step 48730/50462 | Loss: 2.4936 | LR: 1.50e-06 | 5163.3ms/step | 11.6 samples/s
+Step 48740/50462 | Loss: 2.4703 | LR: 1.48e-06 | 5186.0ms/step | 11.6 samples/s
+Step 48750/50462 | Loss: 2.5140 | LR: 1.46e-06 | 5226.5ms/step | 11.5 samples/s
+Validation at step 48750: Loss = 2.5048, Perplexity = 12.24
+Step 48760/50462 | Loss: 2.5248 | LR: 1.44e-06 | 7776.7ms/step | 7.7 samples/s
+Step 48770/50462 | Loss: 2.4649 | LR: 1.43e-06 | 5180.9ms/step | 11.6 samples/s
+Step 48780/50462 | Loss: 2.4872 | LR: 1.41e-06 | 5180.6ms/step | 11.6 samples/s
+Step 48790/50462 | Loss: 2.4959 | LR: 1.39e-06 | 5256.3ms/step | 11.4 samples/s
+Step 48800/50462 | Loss: 2.4750 | LR: 1.38e-06 | 5207.1ms/step | 11.5 samples/s
+Step 48810/50462 | Loss: 2.4844 | LR: 1.36e-06 | 5178.3ms/step | 11.6 samples/s
+Step 48820/50462 | Loss: 2.4896 | LR: 1.34e-06 | 5201.7ms/step | 11.5 samples/s
+Step 48830/50462 | Loss: 2.5065 | LR: 1.33e-06 | 5206.8ms/step | 11.5 samples/s
+Step 48840/50462 | Loss: 2.5192 | LR: 1.31e-06 | 5204.7ms/step | 11.5 samples/s
+Step 48850/50462 | Loss: 2.4911 | LR: 1.30e-06 | 5200.3ms/step | 11.5 samples/s
+Step 48860/50462 | Loss: 2.4568 | LR: 1.28e-06 | 5221.1ms/step | 11.5 samples/s
+Step 48870/50462 | Loss: 2.4998 | LR: 1.26e-06 | 5204.1ms/step | 11.5 samples/s
+Step 48880/50462 | Loss: 2.4823 | LR: 1.25e-06 | 5208.2ms/step | 11.5 samples/s
+Step 48890/50462 | Loss: 2.5129 | LR: 1.23e-06 | 5196.6ms/step | 11.5 samples/s
+Step 48900/50462 | Loss: 2.4909 | LR: 1.22e-06 | 5211.4ms/step | 11.5 samples/s
+Step 48910/50462 | Loss: 2.5049 | LR: 1.20e-06 | 5209.2ms/step | 11.5 samples/s
+Step 48920/50462 | Loss: 2.4914 | LR: 1.19e-06 | 5200.7ms/step | 11.5 samples/s
+Step 48930/50462 | Loss: 2.4877 | LR: 1.17e-06 | 5207.8ms/step | 11.5 samples/s
+Step 48940/50462 | Loss: 2.4682 | LR: 1.16e-06 | 5204.6ms/step | 11.5 samples/s
+Step 48950/50462 | Loss: 2.4682 | LR: 1.14e-06 | 5180.3ms/step | 11.6 samples/s
+Step 48960/50462 | Loss: 2.4996 | LR: 1.13e-06 | 5185.3ms/step | 11.6 samples/s
+Step 48970/50462 | Loss: 2.4668 | LR: 1.11e-06 | 5171.7ms/step | 11.6 samples/s
+Step 48980/50462 | Loss: 2.5349 | LR: 1.10e-06 | 5168.3ms/step | 11.6 samples/s
+Step 48990/50462 | Loss: 2.5197 | LR: 1.08e-06 | 5179.3ms/step | 11.6 samples/s
+Step 49000/50462 | Loss: 2.5014 | LR: 1.07e-06 | 5173.4ms/step | 11.6 samples/s
+Validation at step 49000: Loss = 2.5048, Perplexity = 12.24
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260602_050252/checkpoint-49000
+Step 49010/50462 | Loss: 2.4900 | LR: 1.05e-06 | 8715.7ms/step | 6.9 samples/s
+Step 49020/50462 | Loss: 2.4978 | LR: 1.04e-06 | 5170.9ms/step | 11.6 samples/s
+Step 49030/50462 | Loss: 2.4990 | LR: 1.02e-06 | 5160.1ms/step | 11.6 samples/s
+Step 49040/50462 | Loss: 2.4951 | LR: 1.01e-06 | 5169.0ms/step | 11.6 samples/s
+Step 49050/50462 | Loss: 2.4951 | LR: 9.95e-07 | 5225.9ms/step | 11.5 samples/s
+Step 49060/50462 | Loss: 2.5158 | LR: 9.81e-07 | 5185.0ms/step | 11.6 samples/s
+Step 49070/50462 | Loss: 2.5128 | LR: 9.67e-07 | 5295.2ms/step | 11.3 samples/s
+Step 49080/50462 | Loss: 2.5147 | LR: 9.53e-07 | 5195.1ms/step | 11.5 samples/s
+Step 49090/50462 | Loss: 2.4686 | LR: 9.39e-07 | 5198.3ms/step | 11.5 samples/s
+Step 49100/50462 | Loss: 2.5177 | LR: 9.25e-07 | 5203.7ms/step | 11.5 samples/s
+Step 49110/50462 | Loss: 2.5028 | LR: 9.12e-07 | 5186.9ms/step | 11.6 samples/s
+Step 49120/50462 | Loss: 2.4722 | LR: 8.99e-07 | 5164.1ms/step | 11.6 samples/s
+Step 49130/50462 | Loss: 2.5013 | LR: 8.85e-07 | 5185.9ms/step | 11.6 samples/s
+Step 49140/50462 | Loss: 2.4733 | LR: 8.72e-07 | 5183.1ms/step | 11.6 samples/s
+Step 49150/50462 | Loss: 2.5046 | LR: 8.59e-07 | 5199.8ms/step | 11.5 samples/s
+Step 49160/50462 | Loss: 2.4944 | LR: 8.46e-07 | 5202.6ms/step | 11.5 samples/s
+Step 49170/50462 | Loss: 2.4924 | LR: 8.33e-07 | 5199.8ms/step | 11.5 samples/s
+Step 49180/50462 | Loss: 2.4823 | LR: 8.20e-07 | 5177.3ms/step | 11.6 samples/s
+Step 49190/50462 | Loss: 2.4718 | LR: 8.07e-07 | 5188.1ms/step | 11.6 samples/s
+Step 49200/50462 | Loss: 2.5251 | LR: 7.95e-07 | 5179.2ms/step | 11.6 samples/s
+Step 49210/50462 | Loss: 2.4958 | LR: 7.82e-07 | 5186.4ms/step | 11.6 samples/s
+Step 49220/50462 | Loss: 2.5059 | LR: 7.70e-07 | 5203.5ms/step | 11.5 samples/s
+Step 49230/50462 | Loss: 2.5070 | LR: 7.57e-07 | 5172.4ms/step | 11.6 samples/s
+Step 49240/50462 | Loss: 2.4977 | LR: 7.45e-07 | 5174.3ms/step | 11.6 samples/s
+Step 49250/50462 | Loss: 2.4913 | LR: 7.33e-07 | 5186.9ms/step | 11.6 samples/s
+Validation at step 49250: Loss = 2.5048, Perplexity = 12.24
+Step 49260/50462 | Loss: 2.4645 | LR: 7.21e-07 | 7761.1ms/step | 7.7 samples/s
+Step 49270/50462 | Loss: 2.4834 | LR: 7.09e-07 | 5207.6ms/step | 11.5 samples/s
+Step 49280/50462 | Loss: 2.4673 | LR: 6.97e-07 | 5211.2ms/step | 11.5 samples/s
+Step 49290/50462 | Loss: 2.4887 | LR: 6.85e-07 | 5219.3ms/step | 11.5 samples/s
+Step 49300/50462 | Loss: 2.4861 | LR: 6.74e-07 | 5200.0ms/step | 11.5 samples/s
+Step 49310/50462 | Loss: 2.4923 | LR: 6.62e-07 | 5217.7ms/step | 11.5 samples/s
+Step 49320/50462 | Loss: 2.4807 | LR: 6.51e-07 | 5205.4ms/step | 11.5 samples/s
+Step 49330/50462 | Loss: 2.4891 | LR: 6.39e-07 | 5201.4ms/step | 11.5 samples/s
+Step 49340/50462 | Loss: 2.4990 | LR: 6.28e-07 | 5198.8ms/step | 11.5 samples/s
+Step 49350/50462 | Loss: 2.5018 | LR: 6.17e-07 | 5202.2ms/step | 11.5 samples/s
+Step 49360/50462 | Loss: 2.5120 | LR: 6.06e-07 | 5254.6ms/step | 11.4 samples/s
+Step 49370/50462 | Loss: 2.4944 | LR: 5.95e-07 | 5281.1ms/step | 11.4 samples/s
+Step 49380/50462 | Loss: 2.5072 | LR: 5.84e-07 | 5204.4ms/step | 11.5 samples/s
+Step 49390/50462 | Loss: 2.5067 | LR: 5.73e-07 | 5189.2ms/step | 11.6 samples/s
+Step 49400/50462 | Loss: 2.5044 | LR: 5.63e-07 | 5201.3ms/step | 11.5 samples/s
+Step 49410/50462 | Loss: 2.4963 | LR: 5.52e-07 | 5205.5ms/step | 11.5 samples/s
+Step 49420/50462 | Loss: 2.4938 | LR: 5.42e-07 | 5201.7ms/step | 11.5 samples/s
+Step 49430/50462 | Loss: 2.4733 | LR: 5.31e-07 | 5212.6ms/step | 11.5 samples/s
+Step 49440/50462 | Loss: 2.4831 | LR: 5.21e-07 | 5194.1ms/step | 11.6 samples/s
+Step 49450/50462 | Loss: 2.4697 | LR: 5.11e-07 | 5198.4ms/step | 11.5 samples/s
+Step 49460/50462 | Loss: 2.5039 | LR: 5.01e-07 | 5198.0ms/step | 11.5 samples/s
+Step 49470/50462 | Loss: 2.5056 | LR: 4.91e-07 | 5208.5ms/step | 11.5 samples/s
+Step 49480/50462 | Loss: 2.4989 | LR: 4.81e-07 | 5201.2ms/step | 11.5 samples/s
+Step 49490/50462 | Loss: 2.4792 | LR: 4.72e-07 | 5228.3ms/step | 11.5 samples/s
+Step 49500/50462 | Loss: 2.5000 | LR: 4.62e-07 | 5213.2ms/step | 11.5 samples/s
+Validation at step 49500: Loss = 2.5048, Perplexity = 12.24
+Step 49510/50462 | Loss: 2.5094 | LR: 4.52e-07 | 7766.6ms/step | 7.7 samples/s
+Step 49520/50462 | Loss: 2.4892 | LR: 4.43e-07 | 5214.1ms/step | 11.5 samples/s
+Step 49530/50462 | Loss: 2.4930 | LR: 4.34e-07 | 5219.1ms/step | 11.5 samples/s
+Step 49540/50462 | Loss: 2.4851 | LR: 4.24e-07 | 5219.1ms/step | 11.5 samples/s
+Step 49550/50462 | Loss: 2.4914 | LR: 4.15e-07 | 5211.8ms/step | 11.5 samples/s
+Step 49560/50462 | Loss: 2.5069 | LR: 4.06e-07 | 5189.7ms/step | 11.6 samples/s
+Step 49570/50462 | Loss: 2.5093 | LR: 3.97e-07 | 5196.5ms/step | 11.5 samples/s
+Step 49580/50462 | Loss: 2.5088 | LR: 3.88e-07 | 5215.3ms/step | 11.5 samples/s
+Step 49590/50462 | Loss: 2.4824 | LR: 3.79e-07 | 5231.3ms/step | 11.5 samples/s
+Step 49600/50462 | Loss: 2.5132 | LR: 3.71e-07 | 5214.2ms/step | 11.5 samples/s
+Step 49610/50462 | Loss: 2.4931 | LR: 3.62e-07 | 5218.9ms/step | 11.5 samples/s
+Step 49620/50462 | Loss: 2.4921 | LR: 3.54e-07 | 5213.7ms/step | 11.5 samples/s
+Step 49630/50462 | Loss: 2.4922 | LR: 3.45e-07 | 5202.0ms/step | 11.5 samples/s
+Step 49640/50462 | Loss: 2.4949 | LR: 3.37e-07 | 5208.8ms/step | 11.5 samples/s
+Step 49650/50462 | Loss: 2.4983 | LR: 3.29e-07 | 5212.5ms/step | 11.5 samples/s
+Step 49660/50462 | Loss: 2.4813 | LR: 3.21e-07 | 5264.4ms/step | 11.4 samples/s
+Step 49670/50462 | Loss: 2.4683 | LR: 3.13e-07 | 5246.9ms/step | 11.4 samples/s
+Step 49680/50462 | Loss: 2.5133 | LR: 3.05e-07 | 5250.4ms/step | 11.4 samples/s
+Step 49690/50462 | Loss: 2.5307 | LR: 2.97e-07 | 5206.3ms/step | 11.5 samples/s
+Step 49700/50462 | Loss: 2.5030 | LR: 2.90e-07 | 5206.5ms/step | 11.5 samples/s
+Step 49710/50462 | Loss: 2.4964 | LR: 2.82e-07 | 5202.6ms/step | 11.5 samples/s
+Step 49720/50462 | Loss: 2.4867 | LR: 2.75e-07 | 5209.3ms/step | 11.5 samples/s
+Step 49730/50462 | Loss: 2.5033 | LR: 2.67e-07 | 5211.5ms/step | 11.5 samples/s
+Step 49740/50462 | Loss: 2.5229 | LR: 2.60e-07 | 5207.6ms/step | 11.5 samples/s
+Step 49750/50462 | Loss: 2.4949 | LR: 2.53e-07 | 5204.6ms/step | 11.5 samples/s
+Validation at step 49750: Loss = 2.5048, Perplexity = 12.24
+Step 49760/50462 | Loss: 2.5122 | LR: 2.46e-07 | 7766.1ms/step | 7.7 samples/s
+Step 49770/50462 | Loss: 2.5014 | LR: 2.39e-07 | 5208.3ms/step | 11.5 samples/s
+Step 49780/50462 | Loss: 2.4859 | LR: 2.32e-07 | 5203.7ms/step | 11.5 samples/s
+Step 49790/50462 | Loss: 2.4796 | LR: 2.25e-07 | 5201.7ms/step | 11.5 samples/s
+Step 49800/50462 | Loss: 2.5217 | LR: 2.19e-07 | 5198.6ms/step | 11.5 samples/s
+Step 49810/50462 | Loss: 2.5085 | LR: 2.12e-07 | 5208.5ms/step | 11.5 samples/s
+Step 49820/50462 | Loss: 2.5062 | LR: 2.06e-07 | 5196.0ms/step | 11.5 samples/s
+Step 49830/50462 | Loss: 2.4951 | LR: 1.99e-07 | 5202.0ms/step | 11.5 samples/s
+Step 49840/50462 | Loss: 2.5099 | LR: 1.93e-07 | 5208.0ms/step | 11.5 samples/s
+Step 49850/50462 | Loss: 2.4864 | LR: 1.87e-07 | 5214.4ms/step | 11.5 samples/s
+Step 49860/50462 | Loss: 2.4863 | LR: 1.81e-07 | 5209.4ms/step | 11.5 samples/s
+Step 49870/50462 | Loss: 2.5005 | LR: 1.75e-07 | 5200.5ms/step | 11.5 samples/s
+Step 49880/50462 | Loss: 2.4865 | LR: 1.69e-07 | 5217.1ms/step | 11.5 samples/s
+Step 49890/50462 | Loss: 2.4959 | LR: 1.63e-07 | 5195.7ms/step | 11.5 samples/s
+Step 49900/50462 | Loss: 2.5084 | LR: 1.58e-07 | 5207.5ms/step | 11.5 samples/s
+Step 49910/50462 | Loss: 2.4943 | LR: 1.52e-07 | 5214.3ms/step | 11.5 samples/s
+Step 49920/50462 | Loss: 2.5087 | LR: 1.47e-07 | 5206.3ms/step | 11.5 samples/s
+Step 49930/50462 | Loss: 2.4796 | LR: 1.41e-07 | 5199.5ms/step | 11.5 samples/s
+Step 49940/50462 | Loss: 2.4918 | LR: 1.36e-07 | 5213.2ms/step | 11.5 samples/s

epfl_final/log_files/cont_pretrain_40k.log

@@ -0,0 +1,1054 @@
+nohup: ignoring input
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+New run: Checkpoints will be saved to ./checkpoints/run_20260601_062617
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260601_062617",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-20000",
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 40k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-20000
+Checkpoint activations: False
+Checkpoint activations: False
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    ���─Qwen3DecoderLayer: 3-6                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 1536]           [2, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [2, 8192, 1536]           [2, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (G): 2.87
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 52437.19
+Params size (MB): 2871.45
+Estimated Total Size (MB): 55308.77
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        3
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504627
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-40000...
+Overriding checkpoint LR with config LR: 0.0005
+Resumed from checkpoint at step 40000 (epoch 0, micro_step 400000)
+Resuming training from step 40000
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run o2l7qkqb
+wandb: Tracking run with wandb version 0.27.0
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260601_064405-o2l7qkqb
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense continued 40k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/o2l7qkqb
+Starting training...
+Validation at step 40000: Loss = 2.5242, Perplexity = 12.48
+Switching attention mode for step 40001: dense (dense step interval)
+Step 40010/50462 | Loss: 2.5457 | LR: 5.26e-05 | 59027.5ms/step | 1.0 samples/s
+Step 40020/50462 | Loss: 2.5522 | LR: 5.25e-05 | 5181.0ms/step | 11.6 samples/s
+Step 40030/50462 | Loss: 2.4929 | LR: 5.24e-05 | 5144.6ms/step | 11.7 samples/s
+Step 40040/50462 | Loss: 2.5234 | LR: 5.23e-05 | 5136.0ms/step | 11.7 samples/s
+Step 40050/50462 | Loss: 2.5101 | LR: 5.22e-05 | 5144.4ms/step | 11.7 samples/s
+Step 40060/50462 | Loss: 2.5036 | LR: 5.21e-05 | 5150.3ms/step | 11.6 samples/s
+Step 40070/50462 | Loss: 2.5129 | LR: 5.20e-05 | 5146.6ms/step | 11.7 samples/s
+Step 40080/50462 | Loss: 2.4868 | LR: 5.19e-05 | 5146.8ms/step | 11.7 samples/s
+Step 40090/50462 | Loss: 2.5215 | LR: 5.18e-05 | 5145.2ms/step | 11.7 samples/s
+Step 40100/50462 | Loss: 2.5162 | LR: 5.17e-05 | 5138.6ms/step | 11.7 samples/s
+Step 40110/50462 | Loss: 2.5050 | LR: 5.16e-05 | 5158.1ms/step | 11.6 samples/s
+Step 40120/50462 | Loss: 2.5181 | LR: 5.15e-05 | 5153.1ms/step | 11.6 samples/s
+Step 40130/50462 | Loss: 2.5002 | LR: 5.14e-05 | 5154.3ms/step | 11.6 samples/s
+Step 40140/50462 | Loss: 2.4977 | LR: 5.13e-05 | 5150.9ms/step | 11.6 samples/s
+Step 40150/50462 | Loss: 2.4753 | LR: 5.12e-05 | 5140.0ms/step | 11.7 samples/s
+Step 40160/50462 | Loss: 2.4735 | LR: 5.11e-05 | 5148.0ms/step | 11.7 samples/s
+Step 40170/50462 | Loss: 2.5115 | LR: 5.10e-05 | 5204.8ms/step | 11.5 samples/s
+Step 40180/50462 | Loss: 2.5118 | LR: 5.09e-05 | 5174.2ms/step | 11.6 samples/s
+Step 40190/50462 | Loss: 2.5083 | LR: 5.09e-05 | 5191.9ms/step | 11.6 samples/s
+Step 40200/50462 | Loss: 2.5179 | LR: 5.08e-05 | 5170.1ms/step | 11.6 samples/s
+Step 40210/50462 | Loss: 2.5223 | LR: 5.07e-05 | 5178.8ms/step | 11.6 samples/s
+Step 40220/50462 | Loss: 2.5235 | LR: 5.06e-05 | 5176.5ms/step | 11.6 samples/s
+Step 40230/50462 | Loss: 2.5141 | LR: 5.05e-05 | 5169.1ms/step | 11.6 samples/s
+Step 40240/50462 | Loss: 2.5448 | LR: 5.04e-05 | 5179.6ms/step | 11.6 samples/s
+Step 40250/50462 | Loss: 2.5040 | LR: 5.03e-05 | 5176.9ms/step | 11.6 samples/s
+Validation at step 40250: Loss = 2.5156, Perplexity = 12.37
+Step 40260/50462 | Loss: 2.5107 | LR: 5.02e-05 | 7743.7ms/step | 7.7 samples/s
+Step 40270/50462 | Loss: 2.4858 | LR: 5.01e-05 | 5152.6ms/step | 11.6 samples/s
+Step 40280/50462 | Loss: 2.5092 | LR: 5.00e-05 | 5156.3ms/step | 11.6 samples/s
+Step 40290/50462 | Loss: 2.5077 | LR: 4.99e-05 | 5168.6ms/step | 11.6 samples/s
+Step 40300/50462 | Loss: 2.5032 | LR: 4.98e-05 | 5199.8ms/step | 11.5 samples/s
+Step 40310/50462 | Loss: 2.4889 | LR: 4.97e-05 | 5169.4ms/step | 11.6 samples/s
+Step 40320/50462 | Loss: 2.5072 | LR: 4.96e-05 | 5158.6ms/step | 11.6 samples/s
+Step 40330/50462 | Loss: 2.5301 | LR: 4.95e-05 | 5166.1ms/step | 11.6 samples/s
+Step 40340/50462 | Loss: 2.4905 | LR: 4.94e-05 | 5164.1ms/step | 11.6 samples/s
+Step 40350/50462 | Loss: 2.4801 | LR: 4.93e-05 | 5175.1ms/step | 11.6 samples/s
+Step 40360/50462 | Loss: 2.5102 | LR: 4.92e-05 | 5160.2ms/step | 11.6 samples/s
+Step 40370/50462 | Loss: 2.5229 | LR: 4.91e-05 | 5163.1ms/step | 11.6 samples/s
+Step 40380/50462 | Loss: 2.5087 | LR: 4.90e-05 | 5160.0ms/step | 11.6 samples/s
+Step 40390/50462 | Loss: 2.5041 | LR: 4.90e-05 | 5165.9ms/step | 11.6 samples/s
+Step 40400/50462 | Loss: 2.5157 | LR: 4.89e-05 | 5176.0ms/step | 11.6 samples/s
+Step 40410/50462 | Loss: 2.5304 | LR: 4.88e-05 | 5175.0ms/step | 11.6 samples/s
+Step 40420/50462 | Loss: 2.5288 | LR: 4.87e-05 | 5160.9ms/step | 11.6 samples/s
+Step 40430/50462 | Loss: 2.4958 | LR: 4.86e-05 | 5199.4ms/step | 11.5 samples/s
+Step 40440/50462 | Loss: 2.5184 | LR: 4.85e-05 | 5159.9ms/step | 11.6 samples/s
+Step 40450/50462 | Loss: 2.5046 | LR: 4.84e-05 | 5145.1ms/step | 11.7 samples/s
+Step 40460/50462 | Loss: 2.5366 | LR: 4.83e-05 | 5149.0ms/step | 11.7 samples/s
+Step 40470/50462 | Loss: 2.4913 | LR: 4.82e-05 | 5145.6ms/step | 11.7 samples/s
+Step 40480/50462 | Loss: 2.5023 | LR: 4.81e-05 | 5149.9ms/step | 11.7 samples/s
+Step 40490/50462 | Loss: 2.5178 | LR: 4.80e-05 | 5140.5ms/step | 11.7 samples/s
+Step 40500/50462 | Loss: 2.5079 | LR: 4.79e-05 | 5158.5ms/step | 11.6 samples/s
+Validation at step 40500: Loss = 2.5150, Perplexity = 12.37
+Step 40510/50462 | Loss: 2.5065 | LR: 4.78e-05 | 7710.7ms/step | 7.8 samples/s
+Step 40520/50462 | Loss: 2.4928 | LR: 4.77e-05 | 5161.4ms/step | 11.6 samples/s
+Step 40530/50462 | Loss: 2.4956 | LR: 4.76e-05 | 5195.4ms/step | 11.5 samples/s
+Step 40540/50462 | Loss: 2.4691 | LR: 4.76e-05 | 5153.7ms/step | 11.6 samples/s
+Step 40550/50462 | Loss: 2.5244 | LR: 4.75e-05 | 5145.9ms/step | 11.7 samples/s
+Step 40560/50462 | Loss: 2.5154 | LR: 4.74e-05 | 5149.2ms/step | 11.7 samples/s
+Step 40570/50462 | Loss: 2.4906 | LR: 4.73e-05 | 5193.1ms/step | 11.6 samples/s
+Step 40580/50462 | Loss: 2.5179 | LR: 4.72e-05 | 5157.6ms/step | 11.6 samples/s
+Step 40590/50462 | Loss: 2.5019 | LR: 4.71e-05 | 5148.9ms/step | 11.7 samples/s
+Step 40600/50462 | Loss: 2.5031 | LR: 4.70e-05 | 5149.1ms/step | 11.7 samples/s
+Step 40610/50462 | Loss: 2.5216 | LR: 4.69e-05 | 5128.4ms/step | 11.7 samples/s
+Step 40620/50462 | Loss: 2.4782 | LR: 4.68e-05 | 5136.6ms/step | 11.7 samples/s
+Step 40630/50462 | Loss: 2.5000 | LR: 4.67e-05 | 5127.1ms/step | 11.7 samples/s
+Step 40640/50462 | Loss: 2.5118 | LR: 4.66e-05 | 5135.5ms/step | 11.7 samples/s
+Step 40650/50462 | Loss: 2.5093 | LR: 4.65e-05 | 5121.0ms/step | 11.7 samples/s
+Step 40660/50462 | Loss: 2.5216 | LR: 4.64e-05 | 5127.8ms/step | 11.7 samples/s
+Step 40670/50462 | Loss: 2.4989 | LR: 4.64e-05 | 5146.8ms/step | 11.7 samples/s
+Step 40680/50462 | Loss: 2.5039 | LR: 4.63e-05 | 5156.4ms/step | 11.6 samples/s
+Step 40690/50462 | Loss: 2.5001 | LR: 4.62e-05 | 5155.7ms/step | 11.6 samples/s
+Step 40700/50462 | Loss: 2.4719 | LR: 4.61e-05 | 5197.2ms/step | 11.5 samples/s
+Step 40710/50462 | Loss: 2.5122 | LR: 4.60e-05 | 5156.6ms/step | 11.6 samples/s
+Step 40720/50462 | Loss: 2.4788 | LR: 4.59e-05 | 5153.1ms/step | 11.6 samples/s
+Step 40730/50462 | Loss: 2.4901 | LR: 4.58e-05 | 5155.2ms/step | 11.6 samples/s
+Step 40740/50462 | Loss: 2.4864 | LR: 4.57e-05 | 5152.4ms/step | 11.6 samples/s
+Step 40750/50462 | Loss: 2.4964 | LR: 4.56e-05 | 5154.2ms/step | 11.6 samples/s
+Validation at step 40750: Loss = 2.5144, Perplexity = 12.36
+Step 40760/50462 | Loss: 2.5211 | LR: 4.55e-05 | 7708.5ms/step | 7.8 samples/s
+Step 40770/50462 | Loss: 2.5020 | LR: 4.54e-05 | 5161.6ms/step | 11.6 samples/s
+Step 40780/50462 | Loss: 2.5177 | LR: 4.54e-05 | 5157.9ms/step | 11.6 samples/s
+Step 40790/50462 | Loss: 2.5024 | LR: 4.53e-05 | 5150.3ms/step | 11.6 samples/s
+Step 40800/50462 | Loss: 2.5103 | LR: 4.52e-05 | 5159.7ms/step | 11.6 samples/s
+Step 40810/50462 | Loss: 2.4689 | LR: 4.51e-05 | 5190.0ms/step | 11.6 samples/s
+Step 40820/50462 | Loss: 2.5425 | LR: 4.50e-05 | 5153.0ms/step | 11.6 samples/s
+Step 40830/50462 | Loss: 2.4560 | LR: 4.49e-05 | 5159.6ms/step | 11.6 samples/s
+Step 40840/50462 | Loss: 2.4798 | LR: 4.48e-05 | 5200.4ms/step | 11.5 samples/s
+Step 40850/50462 | Loss: 2.5234 | LR: 4.47e-05 | 5161.4ms/step | 11.6 samples/s
+Step 40860/50462 | Loss: 2.4707 | LR: 4.46e-05 | 5151.7ms/step | 11.6 samples/s
+Step 40870/50462 | Loss: 2.5245 | LR: 4.45e-05 | 5166.3ms/step | 11.6 samples/s
+Step 40880/50462 | Loss: 2.5316 | LR: 4.45e-05 | 5164.8ms/step | 11.6 samples/s
+Step 40890/50462 | Loss: 2.4653 | LR: 4.44e-05 | 5162.8ms/step | 11.6 samples/s
+Step 40900/50462 | Loss: 2.4903 | LR: 4.43e-05 | 5161.1ms/step | 11.6 samples/s
+Step 40910/50462 | Loss: 2.5100 | LR: 4.42e-05 | 5168.1ms/step | 11.6 samples/s
+Step 40920/50462 | Loss: 2.5193 | LR: 4.41e-05 | 5152.9ms/step | 11.6 samples/s
+Step 40930/50462 | Loss: 2.5103 | LR: 4.40e-05 | 5163.4ms/step | 11.6 samples/s
+Step 40940/50462 | Loss: 2.5274 | LR: 4.39e-05 | 5160.6ms/step | 11.6 samples/s
+Step 40950/50462 | Loss: 2.4808 | LR: 4.38e-05 | 5158.2ms/step | 11.6 samples/s
+Step 40960/50462 | Loss: 2.5118 | LR: 4.37e-05 | 5198.4ms/step | 11.5 samples/s
+Step 40970/50462 | Loss: 2.5009 | LR: 4.36e-05 | 5162.9ms/step | 11.6 samples/s
+Step 40980/50462 | Loss: 2.5012 | LR: 4.36e-05 | 5159.1ms/step | 11.6 samples/s
+Step 40990/50462 | Loss: 2.4704 | LR: 4.35e-05 | 5163.5ms/step | 11.6 samples/s
+Step 41000/50462 | Loss: 2.5149 | LR: 4.34e-05 | 5168.9ms/step | 11.6 samples/s
+Validation at step 41000: Loss = 2.5141, Perplexity = 12.36
+Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-41000
+Step 41010/50462 | Loss: 2.5218 | LR: 4.33e-05 | 8630.8ms/step | 7.0 samples/s
+Step 41020/50462 | Loss: 2.5162 | LR: 4.32e-05 | 5164.6ms/step | 11.6 samples/s
+Step 41030/50462 | Loss: 2.5194 | LR: 4.31e-05 | 5170.4ms/step | 11.6 samples/s
+Step 41040/50462 | Loss: 2.4829 | LR: 4.30e-05 | 5175.8ms/step | 11.6 samples/s
+Step 41050/50462 | Loss: 2.5134 | LR: 4.29e-05 | 5181.1ms/step | 11.6 samples/s
+Step 41060/50462 | Loss: 2.5192 | LR: 4.28e-05 | 5162.8ms/step | 11.6 samples/s
+Step 41070/50462 | Loss: 2.4973 | LR: 4.28e-05 | 5149.7ms/step | 11.7 samples/s
+Step 41080/50462 | Loss: 2.4898 | LR: 4.27e-05 | 5192.4ms/step | 11.6 samples/s
+Step 41090/50462 | Loss: 2.4681 | LR: 4.26e-05 | 5157.7ms/step | 11.6 samples/s
+Step 41100/50462 | Loss: 2.5034 | LR: 4.25e-05 | 5159.6ms/step | 11.6 samples/s
+Step 41110/50462 | Loss: 2.5072 | LR: 4.24e-05 | 5158.6ms/step | 11.6 samples/s
+Step 41120/50462 | Loss: 2.4971 | LR: 4.23e-05 | 5206.1ms/step | 11.5 samples/s
+Step 41130/50462 | Loss: 2.5207 | LR: 4.22e-05 | 5163.8ms/step | 11.6 samples/s
+Step 41140/50462 | Loss: 2.4933 | LR: 4.21e-05 | 5154.7ms/step | 11.6 samples/s
+Step 41150/50462 | Loss: 2.5077 | LR: 4.21e-05 | 5152.6ms/step | 11.6 samples/s
+Step 41160/50462 | Loss: 2.5045 | LR: 4.20e-05 | 5167.9ms/step | 11.6 samples/s
+Step 41170/50462 | Loss: 2.5096 | LR: 4.19e-05 | 5169.6ms/step | 11.6 samples/s
+Step 41180/50462 | Loss: 2.4950 | LR: 4.18e-05 | 5162.6ms/step | 11.6 samples/s
+Step 41190/50462 | Loss: 2.4545 | LR: 4.17e-05 | 5163.8ms/step | 11.6 samples/s
+Step 41200/50462 | Loss: 2.4911 | LR: 4.16e-05 | 5151.3ms/step | 11.6 samples/s
+Step 41210/50462 | Loss: 2.4682 | LR: 4.15e-05 | 5165.6ms/step | 11.6 samples/s
+Step 41220/50462 | Loss: 2.4851 | LR: 4.14e-05 | 5211.2ms/step | 11.5 samples/s
+Step 41230/50462 | Loss: 2.5065 | LR: 4.14e-05 | 5149.2ms/step | 11.7 samples/s
+Step 41240/50462 | Loss: 2.4966 | LR: 4.13e-05 | 5161.5ms/step | 11.6 samples/s
+Step 41250/50462 | Loss: 2.5110 | LR: 4.12e-05 | 5177.9ms/step | 11.6 samples/s
+Validation at step 41250: Loss = 2.5138, Perplexity = 12.35
+Step 41260/50462 | Loss: 2.5067 | LR: 4.11e-05 | 7728.5ms/step | 7.8 samples/s
+Step 41270/50462 | Loss: 2.4809 | LR: 4.10e-05 | 5175.8ms/step | 11.6 samples/s
+Step 41280/50462 | Loss: 2.4587 | LR: 4.09e-05 | 5151.2ms/step | 11.6 samples/s
+Step 41290/50462 | Loss: 2.5209 | LR: 4.08e-05 | 5158.9ms/step | 11.6 samples/s
+Step 41300/50462 | Loss: 2.5103 | LR: 4.07e-05 | 5155.7ms/step | 11.6 samples/s
+Step 41310/50462 | Loss: 2.4724 | LR: 4.07e-05 | 5172.3ms/step | 11.6 samples/s
+Step 41320/50462 | Loss: 2.5552 | LR: 4.06e-05 | 5156.7ms/step | 11.6 samples/s
+Step 41330/50462 | Loss: 2.4914 | LR: 4.05e-05 | 5168.5ms/step | 11.6 samples/s
+Step 41340/50462 | Loss: 2.4843 | LR: 4.04e-05 | 5146.6ms/step | 11.7 samples/s
+Step 41350/50462 | Loss: 2.5042 | LR: 4.03e-05 | 5204.3ms/step | 11.5 samples/s
+Step 41360/50462 | Loss: 2.5318 | LR: 4.02e-05 | 5162.0ms/step | 11.6 samples/s
+Step 41370/50462 | Loss: 2.5036 | LR: 4.01e-05 | 5145.6ms/step | 11.7 samples/s
+Step 41380/50462 | Loss: 2.4951 | LR: 4.01e-05 | 5165.5ms/step | 11.6 samples/s
+Step 41390/50462 | Loss: 2.4720 | LR: 4.00e-05 | 5204.3ms/step | 11.5 samples/s
+Step 41400/50462 | Loss: 2.5110 | LR: 3.99e-05 | 5156.0ms/step | 11.6 samples/s
+Step 41410/50462 | Loss: 2.5178 | LR: 3.98e-05 | 5154.5ms/step | 11.6 samples/s
+Step 41420/50462 | Loss: 2.4816 | LR: 3.97e-05 | 5165.7ms/step | 11.6 samples/s
+Step 41430/50462 | Loss: 2.5131 | LR: 3.96e-05 | 5159.2ms/step | 11.6 samples/s
+Step 41440/50462 | Loss: 2.4809 | LR: 3.95e-05 | 5148.4ms/step | 11.7 samples/s
+Step 41450/50462 | Loss: 2.5018 | LR: 3.95e-05 | 5159.8ms/step | 11.6 samples/s
+Step 41460/50462 | Loss: 2.4831 | LR: 3.94e-05 | 5162.8ms/step | 11.6 samples/s
+Step 41470/50462 | Loss: 2.4636 | LR: 3.93e-05 | 5168.3ms/step | 11.6 samples/s
+Step 41480/50462 | Loss: 2.5206 | LR: 3.92e-05 | 5200.9ms/step | 11.5 samples/s
+Step 41490/50462 | Loss: 2.4754 | LR: 3.91e-05 | 5163.8ms/step | 11.6 samples/s
+Step 41500/50462 | Loss: 2.5153 | LR: 3.90e-05 | 5167.1ms/step | 11.6 samples/s
+Validation at step 41500: Loss = 2.5136, Perplexity = 12.35
+Step 41510/50462 | Loss: 2.4862 | LR: 3.90e-05 | 7719.3ms/step | 7.8 samples/s
+Step 41520/50462 | Loss: 2.5013 | LR: 3.89e-05 | 5170.7ms/step | 11.6 samples/s
+Step 41530/50462 | Loss: 2.5131 | LR: 3.88e-05 | 5175.4ms/step | 11.6 samples/s
+Step 41540/50462 | Loss: 2.4814 | LR: 3.87e-05 | 5161.2ms/step | 11.6 samples/s
+Step 41550/50462 | Loss: 2.5082 | LR: 3.86e-05 | 5180.6ms/step | 11.6 samples/s
+Step 41560/50462 | Loss: 2.5193 | LR: 3.85e-05 | 5168.4ms/step | 11.6 samples/s
+Step 41570/50462 | Loss: 2.5067 | LR: 3.84e-05 | 5151.7ms/step | 11.6 samples/s
+Step 41580/50462 | Loss: 2.5125 | LR: 3.84e-05 | 5171.4ms/step | 11.6 samples/s
+Step 41590/50462 | Loss: 2.4951 | LR: 3.83e-05 | 5169.3ms/step | 11.6 samples/s
+Step 41600/50462 | Loss: 2.5089 | LR: 3.82e-05 | 5174.7ms/step | 11.6 samples/s
+Step 41610/50462 | Loss: 2.4908 | LR: 3.81e-05 | 5156.6ms/step | 11.6 samples/s
+Step 41620/50462 | Loss: 2.4971 | LR: 3.80e-05 | 5183.9ms/step | 11.6 samples/s
+Step 41630/50462 | Loss: 2.4785 | LR: 3.79e-05 | 5208.6ms/step | 11.5 samples/s
+Step 41640/50462 | Loss: 2.4947 | LR: 3.79e-05 | 5162.4ms/step | 11.6 samples/s
+Step 41650/50462 | Loss: 2.4800 | LR: 3.78e-05 | 5168.0ms/step | 11.6 samples/s
+Step 41660/50462 | Loss: 2.5093 | LR: 3.77e-05 | 5216.5ms/step | 11.5 samples/s
+Step 41670/50462 | Loss: 2.5016 | LR: 3.76e-05 | 5170.0ms/step | 11.6 samples/s
+Step 41680/50462 | Loss: 2.4876 | LR: 3.75e-05 | 5180.7ms/step | 11.6 samples/s
+Step 41690/50462 | Loss: 2.4838 | LR: 3.74e-05 | 5169.9ms/step | 11.6 samples/s
+Step 41700/50462 | Loss: 2.4951 | LR: 3.74e-05 | 5178.2ms/step | 11.6 samples/s
+Step 41710/50462 | Loss: 2.4876 | LR: 3.73e-05 | 5165.2ms/step | 11.6 samples/s
+Step 41720/50462 | Loss: 2.4835 | LR: 3.72e-05 | 5166.5ms/step | 11.6 samples/s
+Step 41730/50462 | Loss: 2.4901 | LR: 3.71e-05 | 5177.8ms/step | 11.6 samples/s
+Step 41740/50462 | Loss: 2.4828 | LR: 3.70e-05 | 5216.7ms/step | 11.5 samples/s
+Step 41750/50462 | Loss: 2.4892 | LR: 3.69e-05 | 5159.7ms/step | 11.6 samples/s
+Validation at step 41750: Loss = 2.5134, Perplexity = 12.35
+Step 41760/50462 | Loss: 2.5022 | LR: 3.69e-05 | 7700.7ms/step | 7.8 samples/s
+Step 41770/50462 | Loss: 2.5089 | LR: 3.68e-05 | 5165.7ms/step | 11.6 samples/s
+Step 41780/50462 | Loss: 2.4956 | LR: 3.67e-05 | 5164.8ms/step | 11.6 samples/s
+Step 41790/50462 | Loss: 2.4814 | LR: 3.66e-05 | 5156.2ms/step | 11.6 samples/s
+Step 41800/50462 | Loss: 2.4852 | LR: 3.65e-05 | 5155.4ms/step | 11.6 samples/s
+Step 41810/50462 | Loss: 2.5201 | LR: 3.64e-05 | 5169.0ms/step | 11.6 samples/s
+Step 41820/50462 | Loss: 2.4454 | LR: 3.64e-05 | 5153.5ms/step | 11.6 samples/s
+Step 41830/50462 | Loss: 2.4929 | LR: 3.63e-05 | 5167.6ms/step | 11.6 samples/s
+Step 41840/50462 | Loss: 2.4673 | LR: 3.62e-05 | 5171.4ms/step | 11.6 samples/s
+Step 41850/50462 | Loss: 2.5099 | LR: 3.61e-05 | 5173.1ms/step | 11.6 samples/s
+Step 41860/50462 | Loss: 2.4950 | LR: 3.60e-05 | 5189.0ms/step | 11.6 samples/s
+Step 41870/50462 | Loss: 2.4939 | LR: 3.60e-05 | 5166.1ms/step | 11.6 samples/s
+Step 41880/50462 | Loss: 2.5050 | LR: 3.59e-05 | 5187.6ms/step | 11.6 samples/s
+Step 41890/50462 | Loss: 2.4969 | LR: 3.58e-05 | 5173.5ms/step | 11.6 samples/s
+Step 41900/50462 | Loss: 2.5220 | LR: 3.57e-05 | 5216.8ms/step | 11.5 samples/s
+Step 41910/50462 | Loss: 2.5135 | LR: 3.56e-05 | 5160.4ms/step | 11.6 samples/s
+Step 41920/50462 | Loss: 2.5040 | LR: 3.55e-05 | 5151.6ms/step | 11.6 samples/s
+Step 41930/50462 | Loss: 2.4838 | LR: 3.55e-05 | 5149.4ms/step | 11.7 samples/s
+Step 41940/50462 | Loss: 2.5116 | LR: 3.54e-05 | 5218.6ms/step | 11.5 samples/s
+Step 41950/50462 | Loss: 2.4828 | LR: 3.53e-05 | 5159.9ms/step | 11.6 samples/s
+Step 41960/50462 | Loss: 2.5106 | LR: 3.52e-05 | 5172.8ms/step | 11.6 samples/s
+Step 41970/50462 | Loss: 2.4976 | LR: 3.51e-05 | 5151.7ms/step | 11.6 samples/s
+Step 41980/50462 | Loss: 2.4961 | LR: 3.51e-05 | 5148.7ms/step | 11.7 samples/s
+Step 41990/50462 | Loss: 2.5281 | LR: 3.50e-05 | 5176.2ms/step | 11.6 samples/s
+Step 42000/50462 | Loss: 2.5067 | LR: 3.49e-05 | 5142.5ms/step | 11.7 samples/s
+Validation at step 42000: Loss = 2.5132, Perplexity = 12.34
+Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-42000
+Step 42010/50462 | Loss: 2.4752 | LR: 3.48e-05 | 8679.3ms/step | 6.9 samples/s
+Step 42020/50462 | Loss: 2.5166 | LR: 3.47e-05 | 5136.5ms/step | 11.7 samples/s
+Step 42030/50462 | Loss: 2.5355 | LR: 3.47e-05 | 5138.7ms/step | 11.7 samples/s
+Step 42040/50462 | Loss: 2.5006 | LR: 3.46e-05 | 5156.3ms/step | 11.6 samples/s
+Step 42050/50462 | Loss: 2.5031 | LR: 3.45e-05 | 5144.7ms/step | 11.7 samples/s
+Step 42060/50462 | Loss: 2.5358 | LR: 3.44e-05 | 5156.3ms/step | 11.6 samples/s
+Step 42070/50462 | Loss: 2.5169 | LR: 3.43e-05 | 5147.3ms/step | 11.7 samples/s
+Step 42080/50462 | Loss: 2.4889 | LR: 3.43e-05 | 5144.5ms/step | 11.7 samples/s
+Step 42090/50462 | Loss: 2.5203 | LR: 3.42e-05 | 5147.6ms/step | 11.7 samples/s
+Step 42100/50462 | Loss: 2.5007 | LR: 3.41e-05 | 5152.5ms/step | 11.6 samples/s
+Step 42110/50462 | Loss: 2.4686 | LR: 3.40e-05 | 5161.4ms/step | 11.6 samples/s
+Step 42120/50462 | Loss: 2.5033 | LR: 3.39e-05 | 5154.0ms/step | 11.6 samples/s
+Step 42130/50462 | Loss: 2.5165 | LR: 3.39e-05 | 5157.3ms/step | 11.6 samples/s
+Step 42140/50462 | Loss: 2.5119 | LR: 3.38e-05 | 5144.0ms/step | 11.7 samples/s
+Step 42150/50462 | Loss: 2.5080 | LR: 3.37e-05 | 5139.4ms/step | 11.7 samples/s
+Step 42160/50462 | Loss: 2.4669 | LR: 3.36e-05 | 5149.1ms/step | 11.7 samples/s
+Step 42170/50462 | Loss: 2.4929 | LR: 3.35e-05 | 5184.2ms/step | 11.6 samples/s
+Step 42180/50462 | Loss: 2.4996 | LR: 3.35e-05 | 5151.0ms/step | 11.6 samples/s
+Step 42190/50462 | Loss: 2.5298 | LR: 3.34e-05 | 5149.4ms/step | 11.7 samples/s
+Step 42200/50462 | Loss: 2.4852 | LR: 3.33e-05 | 5153.8ms/step | 11.6 samples/s
+Step 42210/50462 | Loss: 2.4907 | LR: 3.32e-05 | 5217.5ms/step | 11.5 samples/s
+Step 42220/50462 | Loss: 2.5191 | LR: 3.32e-05 | 5148.5ms/step | 11.7 samples/s
+Step 42230/50462 | Loss: 2.5049 | LR: 3.31e-05 | 5170.5ms/step | 11.6 samples/s
+Step 42240/50462 | Loss: 2.4605 | LR: 3.30e-05 | 5147.1ms/step | 11.7 samples/s
+Step 42250/50462 | Loss: 2.5105 | LR: 3.29e-05 | 5147.6ms/step | 11.7 samples/s
+Validation at step 42250: Loss = 2.5130, Perplexity = 12.34
+Step 42260/50462 | Loss: 2.5174 | LR: 3.28e-05 | 7706.1ms/step | 7.8 samples/s
+Step 42270/50462 | Loss: 2.5291 | LR: 3.28e-05 | 5218.1ms/step | 11.5 samples/s
+Step 42280/50462 | Loss: 2.4986 | LR: 3.27e-05 | 5166.7ms/step | 11.6 samples/s
+Step 42290/50462 | Loss: 2.5118 | LR: 3.26e-05 | 5166.1ms/step | 11.6 samples/s
+Step 42300/50462 | Loss: 2.4987 | LR: 3.25e-05 | 5167.4ms/step | 11.6 samples/s
+Step 42310/50462 | Loss: 2.4910 | LR: 3.24e-05 | 5153.6ms/step | 11.6 samples/s
+Step 42320/50462 | Loss: 2.4724 | LR: 3.24e-05 | 5160.3ms/step | 11.6 samples/s
+Step 42330/50462 | Loss: 2.5210 | LR: 3.23e-05 | 5166.3ms/step | 11.6 samples/s
+Step 42340/50462 | Loss: 2.5282 | LR: 3.22e-05 | 5161.3ms/step | 11.6 samples/s
+Step 42350/50462 | Loss: 2.4745 | LR: 3.21e-05 | 5165.0ms/step | 11.6 samples/s
+Step 42360/50462 | Loss: 2.4851 | LR: 3.21e-05 | 5163.1ms/step | 11.6 samples/s
+Step 42370/50462 | Loss: 2.4991 | LR: 3.20e-05 | 5164.2ms/step | 11.6 samples/s
+Step 42380/50462 | Loss: 2.4853 | LR: 3.19e-05 | 5152.9ms/step | 11.6 samples/s
+Step 42390/50462 | Loss: 2.5123 | LR: 3.18e-05 | 5159.2ms/step | 11.6 samples/s
+Step 42400/50462 | Loss: 2.5210 | LR: 3.18e-05 | 5171.0ms/step | 11.6 samples/s
+Step 42410/50462 | Loss: 2.5132 | LR: 3.17e-05 | 5168.4ms/step | 11.6 samples/s
+Step 42420/50462 | Loss: 2.5069 | LR: 3.16e-05 | 5151.9ms/step | 11.6 samples/s
+Step 42430/50462 | Loss: 2.4943 | LR: 3.15e-05 | 5178.7ms/step | 11.6 samples/s
+Step 42440/50462 | Loss: 2.4927 | LR: 3.14e-05 | 5205.3ms/step | 11.5 samples/s
+Step 42450/50462 | Loss: 2.5125 | LR: 3.14e-05 | 5155.1ms/step | 11.6 samples/s
+Step 42460/50462 | Loss: 2.4925 | LR: 3.13e-05 | 5159.3ms/step | 11.6 samples/s
+Step 42470/50462 | Loss: 2.5045 | LR: 3.12e-05 | 5160.3ms/step | 11.6 samples/s
+Step 42480/50462 | Loss: 2.4600 | LR: 3.11e-05 | 5193.2ms/step | 11.6 samples/s
+Step 42490/50462 | Loss: 2.5076 | LR: 3.11e-05 | 5154.0ms/step | 11.6 samples/s
+Step 42500/50462 | Loss: 2.5170 | LR: 3.10e-05 | 5168.4ms/step | 11.6 samples/s
+Validation at step 42500: Loss = 2.5129, Perplexity = 12.34
+Step 42510/50462 | Loss: 2.5126 | LR: 3.09e-05 | 7731.5ms/step | 7.8 samples/s
+Step 42520/50462 | Loss: 2.4983 | LR: 3.08e-05 | 5179.7ms/step | 11.6 samples/s
+Step 42530/50462 | Loss: 2.4910 | LR: 3.08e-05 | 5224.8ms/step | 11.5 samples/s
+Step 42540/50462 | Loss: 2.4972 | LR: 3.07e-05 | 5181.2ms/step | 11.6 samples/s
+Step 42550/50462 | Loss: 2.5209 | LR: 3.06e-05 | 5168.9ms/step | 11.6 samples/s
+Step 42560/50462 | Loss: 2.5787 | LR: 3.05e-05 | 5170.3ms/step | 11.6 samples/s
+Step 42570/50462 | Loss: 2.5091 | LR: 3.05e-05 | 5159.6ms/step | 11.6 samples/s
+Step 42580/50462 | Loss: 2.5029 | LR: 3.04e-05 | 5166.3ms/step | 11.6 samples/s
+Step 42590/50462 | Loss: 2.4970 | LR: 3.03e-05 | 5151.3ms/step | 11.6 samples/s
+Step 42600/50462 | Loss: 2.5453 | LR: 3.02e-05 | 5168.0ms/step | 11.6 samples/s
+Step 42610/50462 | Loss: 2.4944 | LR: 3.02e-05 | 5173.6ms/step | 11.6 samples/s
+Step 42620/50462 | Loss: 2.4871 | LR: 3.01e-05 | 5152.0ms/step | 11.6 samples/s
+Step 42630/50462 | Loss: 2.5191 | LR: 3.00e-05 | 5153.3ms/step | 11.6 samples/s
+Step 42640/50462 | Loss: 2.5439 | LR: 2.99e-05 | 5148.9ms/step | 11.7 samples/s
+Step 42650/50462 | Loss: 2.5139 | LR: 2.99e-05 | 5156.1ms/step | 11.6 samples/s
+Step 42660/50462 | Loss: 2.5160 | LR: 2.98e-05 | 5173.8ms/step | 11.6 samples/s
+Step 42670/50462 | Loss: 2.4854 | LR: 2.97e-05 | 5175.9ms/step | 11.6 samples/s
+Step 42680/50462 | Loss: 2.4854 | LR: 2.96e-05 | 5176.2ms/step | 11.6 samples/s
+Step 42690/50462 | Loss: 2.5066 | LR: 2.96e-05 | 5167.5ms/step | 11.6 samples/s
+Step 42700/50462 | Loss: 2.5136 | LR: 2.95e-05 | 5177.2ms/step | 11.6 samples/s
+Step 42710/50462 | Loss: 2.4962 | LR: 2.94e-05 | 5190.0ms/step | 11.6 samples/s
+Step 42720/50462 | Loss: 2.4969 | LR: 2.93e-05 | 5193.1ms/step | 11.6 samples/s
+Step 42730/50462 | Loss: 2.4975 | LR: 2.93e-05 | 5161.7ms/step | 11.6 samples/s
+Step 42740/50462 | Loss: 2.5020 | LR: 2.92e-05 | 5156.0ms/step | 11.6 samples/s
+Step 42750/50462 | Loss: 2.4985 | LR: 2.91e-05 | 5156.8ms/step | 11.6 samples/s
+Validation at step 42750: Loss = 2.5128, Perplexity = 12.34
+Step 42760/50462 | Loss: 2.5120 | LR: 2.90e-05 | 7753.1ms/step | 7.7 samples/s
+Step 42770/50462 | Loss: 2.5025 | LR: 2.90e-05 | 5168.1ms/step | 11.6 samples/s
+Step 42780/50462 | Loss: 2.5001 | LR: 2.89e-05 | 5168.3ms/step | 11.6 samples/s
+Step 42790/50462 | Loss: 2.5177 | LR: 2.88e-05 | 5199.6ms/step | 11.5 samples/s
+Step 42800/50462 | Loss: 2.5179 | LR: 2.87e-05 | 5182.2ms/step | 11.6 samples/s
+Step 42810/50462 | Loss: 2.5226 | LR: 2.87e-05 | 5195.8ms/step | 11.5 samples/s
+Step 42820/50462 | Loss: 2.4943 | LR: 2.86e-05 | 5188.3ms/step | 11.6 samples/s
+Step 42830/50462 | Loss: 2.4868 | LR: 2.85e-05 | 5183.4ms/step | 11.6 samples/s
+Step 42840/50462 | Loss: 2.5150 | LR: 2.84e-05 | 5183.6ms/step | 11.6 samples/s
+Step 42850/50462 | Loss: 2.5039 | LR: 2.84e-05 | 5187.0ms/step | 11.6 samples/s
+Step 42860/50462 | Loss: 2.5320 | LR: 2.83e-05 | 5170.6ms/step | 11.6 samples/s
+Step 42870/50462 | Loss: 2.5143 | LR: 2.82e-05 | 5194.1ms/step | 11.6 samples/s
+Step 42880/50462 | Loss: 2.5097 | LR: 2.82e-05 | 5184.6ms/step | 11.6 samples/s
+Step 42890/50462 | Loss: 2.4940 | LR: 2.81e-05 | 5167.8ms/step | 11.6 samples/s
+Step 42900/50462 | Loss: 2.5022 | LR: 2.80e-05 | 5182.3ms/step | 11.6 samples/s
+Step 42910/50462 | Loss: 2.5327 | LR: 2.79e-05 | 5185.3ms/step | 11.6 samples/s
+Step 42920/50462 | Loss: 2.4916 | LR: 2.79e-05 | 5194.9ms/step | 11.5 samples/s
+Step 42930/50462 | Loss: 2.4905 | LR: 2.78e-05 | 5178.8ms/step | 11.6 samples/s
+Step 42940/50462 | Loss: 2.4804 | LR: 2.77e-05 | 5197.4ms/step | 11.5 samples/s
+Step 42950/50462 | Loss: 2.5310 | LR: 2.76e-05 | 5192.0ms/step | 11.6 samples/s
+Step 42960/50462 | Loss: 2.5145 | LR: 2.76e-05 | 5187.5ms/step | 11.6 samples/s
+Step 42970/50462 | Loss: 2.4853 | LR: 2.75e-05 | 5180.6ms/step | 11.6 samples/s
+Step 42980/50462 | Loss: 2.4996 | LR: 2.74e-05 | 5176.5ms/step | 11.6 samples/s
+Step 42990/50462 | Loss: 2.5069 | LR: 2.74e-05 | 5211.4ms/step | 11.5 samples/s
+Step 43000/50462 | Loss: 2.4844 | LR: 2.73e-05 | 5191.0ms/step | 11.6 samples/s
+Validation at step 43000: Loss = 2.5127, Perplexity = 12.34
+Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-43000
+Step 43010/50462 | Loss: 2.5256 | LR: 2.72e-05 | 8678.3ms/step | 6.9 samples/s
+Step 43020/50462 | Loss: 2.4983 | LR: 2.71e-05 | 5156.8ms/step | 11.6 samples/s
+Step 43030/50462 | Loss: 2.5038 | LR: 2.71e-05 | 5211.8ms/step | 11.5 samples/s
+Step 43040/50462 | Loss: 2.4917 | LR: 2.70e-05 | 5185.6ms/step | 11.6 samples/s
+Step 43050/50462 | Loss: 2.5126 | LR: 2.69e-05 | 5180.7ms/step | 11.6 samples/s
+Step 43060/50462 | Loss: 2.4868 | LR: 2.69e-05 | 5221.2ms/step | 11.5 samples/s
+Step 43070/50462 | Loss: 2.5046 | LR: 2.68e-05 | 5193.1ms/step | 11.6 samples/s
+Step 43080/50462 | Loss: 2.4839 | LR: 2.67e-05 | 5185.3ms/step | 11.6 samples/s
+Step 43090/50462 | Loss: 2.4793 | LR: 2.66e-05 | 5190.8ms/step | 11.6 samples/s
+Step 43100/50462 | Loss: 2.4791 | LR: 2.66e-05 | 5171.0ms/step | 11.6 samples/s
+Step 43110/50462 | Loss: 2.4869 | LR: 2.65e-05 | 5180.3ms/step | 11.6 samples/s
+Step 43120/50462 | Loss: 2.5018 | LR: 2.64e-05 | 5185.6ms/step | 11.6 samples/s
+Step 43130/50462 | Loss: 2.5119 | LR: 2.64e-05 | 5184.2ms/step | 11.6 samples/s
+Step 43140/50462 | Loss: 2.4820 | LR: 2.63e-05 | 5195.7ms/step | 11.5 samples/s
+Step 43150/50462 | Loss: 2.4840 | LR: 2.62e-05 | 5183.8ms/step | 11.6 samples/s
+Step 43160/50462 | Loss: 2.5231 | LR: 2.61e-05 | 5180.5ms/step | 11.6 samples/s
+Step 43170/50462 | Loss: 2.5147 | LR: 2.61e-05 | 5184.1ms/step | 11.6 samples/s
+Step 43180/50462 | Loss: 2.4685 | LR: 2.60e-05 | 5180.9ms/step | 11.6 samples/s
+Step 43190/50462 | Loss: 2.5239 | LR: 2.59e-05 | 5176.6ms/step | 11.6 samples/s
+Step 43200/50462 | Loss: 2.5007 | LR: 2.59e-05 | 5180.6ms/step | 11.6 samples/s
+Step 43210/50462 | Loss: 2.5074 | LR: 2.58e-05 | 5153.7ms/step | 11.6 samples/s
+Step 43220/50462 | Loss: 2.5150 | LR: 2.57e-05 | 5177.0ms/step | 11.6 samples/s
+Step 43230/50462 | Loss: 2.5001 | LR: 2.57e-05 | 5161.6ms/step | 11.6 samples/s
+Step 43240/50462 | Loss: 2.5207 | LR: 2.56e-05 | 5163.0ms/step | 11.6 samples/s
+Step 43250/50462 | Loss: 2.4714 | LR: 2.55e-05 | 5149.6ms/step | 11.7 samples/s
+Validation at step 43250: Loss = 2.5126, Perplexity = 12.34
+Step 43260/50462 | Loss: 2.4732 | LR: 2.54e-05 | 7757.1ms/step | 7.7 samples/s
+Step 43270/50462 | Loss: 2.4846 | LR: 2.54e-05 | 5158.3ms/step | 11.6 samples/s
+Step 43280/50462 | Loss: 2.5060 | LR: 2.53e-05 | 5167.4ms/step | 11.6 samples/s
+Step 43290/50462 | Loss: 2.4857 | LR: 2.52e-05 | 5171.7ms/step | 11.6 samples/s
+Step 43300/50462 | Loss: 2.5292 | LR: 2.52e-05 | 5196.2ms/step | 11.5 samples/s
+Step 43310/50462 | Loss: 2.4945 | LR: 2.51e-05 | 5162.5ms/step | 11.6 samples/s
+Step 43320/50462 | Loss: 2.4972 | LR: 2.50e-05 | 5201.8ms/step | 11.5 samples/s
+Step 43330/50462 | Loss: 2.4832 | LR: 2.50e-05 | 5166.9ms/step | 11.6 samples/s
+Step 43340/50462 | Loss: 2.4954 | LR: 2.49e-05 | 5158.2ms/step | 11.6 samples/s
+Step 43350/50462 | Loss: 2.4815 | LR: 2.48e-05 | 5185.4ms/step | 11.6 samples/s
+Step 43360/50462 | Loss: 2.5146 | LR: 2.48e-05 | 5161.7ms/step | 11.6 samples/s
+Step 43370/50462 | Loss: 2.5055 | LR: 2.47e-05 | 5171.4ms/step | 11.6 samples/s
+Step 43380/50462 | Loss: 2.4847 | LR: 2.46e-05 | 5157.9ms/step | 11.6 samples/s
+Step 43390/50462 | Loss: 2.5132 | LR: 2.46e-05 | 5164.1ms/step | 11.6 samples/s
+Step 43400/50462 | Loss: 2.4962 | LR: 2.45e-05 | 5159.4ms/step | 11.6 samples/s
+Step 43410/50462 | Loss: 2.5158 | LR: 2.44e-05 | 5165.6ms/step | 11.6 samples/s
+Step 43420/50462 | Loss: 2.4958 | LR: 2.44e-05 | 5173.5ms/step | 11.6 samples/s
+Step 43430/50462 | Loss: 2.5009 | LR: 2.43e-05 | 5175.9ms/step | 11.6 samples/s
+Step 43440/50462 | Loss: 2.4976 | LR: 2.42e-05 | 5150.8ms/step | 11.6 samples/s
+Step 43450/50462 | Loss: 2.5220 | LR: 2.41e-05 | 5147.5ms/step | 11.7 samples/s
+Step 43460/50462 | Loss: 2.5090 | LR: 2.41e-05 | 5155.4ms/step | 11.6 samples/s
+Step 43470/50462 | Loss: 2.5244 | LR: 2.40e-05 | 5173.0ms/step | 11.6 samples/s
+Step 43480/50462 | Loss: 2.5234 | LR: 2.39e-05 | 5170.2ms/step | 11.6 samples/s
+Step 43490/50462 | Loss: 2.4919 | LR: 2.39e-05 | 5160.2ms/step | 11.6 samples/s
+Step 43500/50462 | Loss: 2.5109 | LR: 2.38e-05 | 5153.3ms/step | 11.6 samples/s
+Validation at step 43500: Loss = 2.5125, Perplexity = 12.34
+Step 43510/50462 | Loss: 2.5121 | LR: 2.37e-05 | 7705.5ms/step | 7.8 samples/s
+Step 43520/50462 | Loss: 2.4758 | LR: 2.37e-05 | 5171.0ms/step | 11.6 samples/s
+Step 43530/50462 | Loss: 2.4960 | LR: 2.36e-05 | 5195.5ms/step | 11.5 samples/s
+Step 43540/50462 | Loss: 2.4936 | LR: 2.35e-05 | 5160.9ms/step | 11.6 samples/s
+Step 43550/50462 | Loss: 2.5023 | LR: 2.35e-05 | 5168.2ms/step | 11.6 samples/s
+Step 43560/50462 | Loss: 2.4811 | LR: 2.34e-05 | 5163.3ms/step | 11.6 samples/s
+Step 43570/50462 | Loss: 2.5016 | LR: 2.33e-05 | 5168.3ms/step | 11.6 samples/s
+Step 43580/50462 | Loss: 2.5301 | LR: 2.33e-05 | 5246.1ms/step | 11.4 samples/s
+Step 43590/50462 | Loss: 2.4863 | LR: 2.32e-05 | 5164.2ms/step | 11.6 samples/s
+Step 43600/50462 | Loss: 2.5211 | LR: 2.31e-05 | 5166.4ms/step | 11.6 samples/s
+Step 43610/50462 | Loss: 2.5078 | LR: 2.31e-05 | 5169.4ms/step | 11.6 samples/s
+Step 43620/50462 | Loss: 2.4907 | LR: 2.30e-05 | 5178.0ms/step | 11.6 samples/s
+Step 43630/50462 | Loss: 2.5286 | LR: 2.29e-05 | 5163.2ms/step | 11.6 samples/s
+Step 43640/50462 | Loss: 2.4902 | LR: 2.29e-05 | 5162.9ms/step | 11.6 samples/s
+Step 43650/50462 | Loss: 2.5097 | LR: 2.28e-05 | 5167.1ms/step | 11.6 samples/s
+Step 43660/50462 | Loss: 2.5227 | LR: 2.27e-05 | 5170.2ms/step | 11.6 samples/s
+Step 43670/50462 | Loss: 2.5095 | LR: 2.27e-05 | 5164.6ms/step | 11.6 samples/s
+Step 43680/50462 | Loss: 2.5139 | LR: 2.26e-05 | 5186.5ms/step | 11.6 samples/s
+Step 43690/50462 | Loss: 2.5071 | LR: 2.25e-05 | 5177.8ms/step | 11.6 samples/s
+Step 43700/50462 | Loss: 2.5241 | LR: 2.25e-05 | 5165.4ms/step | 11.6 samples/s
+Step 43710/50462 | Loss: 2.5292 | LR: 2.24e-05 | 5162.2ms/step | 11.6 samples/s
+Step 43720/50462 | Loss: 2.4852 | LR: 2.24e-05 | 5154.4ms/step | 11.6 samples/s
+Step 43730/50462 | Loss: 2.4744 | LR: 2.23e-05 | 5140.4ms/step | 11.7 samples/s
+Step 43740/50462 | Loss: 2.4986 | LR: 2.22e-05 | 5173.9ms/step | 11.6 samples/s
+Step 43750/50462 | Loss: 2.4995 | LR: 2.22e-05 | 5179.9ms/step | 11.6 samples/s
+Validation at step 43750: Loss = 2.5124, Perplexity = 12.34
+Step 43760/50462 | Loss: 2.5051 | LR: 2.21e-05 | 7735.0ms/step | 7.8 samples/s
+Step 43770/50462 | Loss: 2.4901 | LR: 2.20e-05 | 5171.8ms/step | 11.6 samples/s
+Step 43780/50462 | Loss: 2.4985 | LR: 2.20e-05 | 5168.2ms/step | 11.6 samples/s
+Step 43790/50462 | Loss: 2.5069 | LR: 2.19e-05 | 5164.3ms/step | 11.6 samples/s
+Step 43800/50462 | Loss: 2.5193 | LR: 2.18e-05 | 5162.2ms/step | 11.6 samples/s
+Step 43810/50462 | Loss: 2.4630 | LR: 2.18e-05 | 5205.0ms/step | 11.5 samples/s
+Step 43820/50462 | Loss: 2.4672 | LR: 2.17e-05 | 5170.2ms/step | 11.6 samples/s
+Step 43830/50462 | Loss: 2.5112 | LR: 2.16e-05 | 5176.4ms/step | 11.6 samples/s
+Step 43840/50462 | Loss: 2.5255 | LR: 2.16e-05 | 5211.8ms/step | 11.5 samples/s
+Step 43850/50462 | Loss: 2.5218 | LR: 2.15e-05 | 5214.4ms/step | 11.5 samples/s
+Step 43860/50462 | Loss: 2.5043 | LR: 2.14e-05 | 5171.2ms/step | 11.6 samples/s
+Step 43870/50462 | Loss: 2.4938 | LR: 2.14e-05 | 5188.0ms/step | 11.6 samples/s
+Step 43880/50462 | Loss: 2.5138 | LR: 2.13e-05 | 5171.2ms/step | 11.6 samples/s
+Step 43890/50462 | Loss: 2.5056 | LR: 2.13e-05 | 5171.1ms/step | 11.6 samples/s
+Step 43900/50462 | Loss: 2.5575 | LR: 2.12e-05 | 5165.6ms/step | 11.6 samples/s
+Step 43910/50462 | Loss: 2.5240 | LR: 2.11e-05 | 5180.1ms/step | 11.6 samples/s
+Step 43920/50462 | Loss: 2.4996 | LR: 2.11e-05 | 5169.8ms/step | 11.6 samples/s
+Step 43930/50462 | Loss: 2.5250 | LR: 2.10e-05 | 5170.5ms/step | 11.6 samples/s
+Step 43940/50462 | Loss: 2.5002 | LR: 2.09e-05 | 5176.1ms/step | 11.6 samples/s
+Step 43950/50462 | Loss: 2.5125 | LR: 2.09e-05 | 5160.6ms/step | 11.6 samples/s
+Step 43960/50462 | Loss: 2.4953 | LR: 2.08e-05 | 5175.9ms/step | 11.6 samples/s
+Step 43970/50462 | Loss: 2.4909 | LR: 2.07e-05 | 5159.5ms/step | 11.6 samples/s
+Step 43980/50462 | Loss: 2.4818 | LR: 2.07e-05 | 5157.6ms/step | 11.6 samples/s
+Step 43990/50462 | Loss: 2.5192 | LR: 2.06e-05 | 5167.2ms/step | 11.6 samples/s
+Step 44000/50462 | Loss: 2.5244 | LR: 2.06e-05 | 5164.9ms/step | 11.6 samples/s
+Validation at step 44000: Loss = 2.5124, Perplexity = 12.33
+Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-44000
+Step 44010/50462 | Loss: 2.5095 | LR: 2.05e-05 | 8670.6ms/step | 6.9 samples/s
+Step 44020/50462 | Loss: 2.5015 | LR: 2.04e-05 | 5152.2ms/step | 11.6 samples/s
+Step 44030/50462 | Loss: 2.4981 | LR: 2.04e-05 | 5175.1ms/step | 11.6 samples/s
+Step 44040/50462 | Loss: 2.4929 | LR: 2.03e-05 | 5152.4ms/step | 11.6 samples/s
+Step 44050/50462 | Loss: 2.5256 | LR: 2.02e-05 | 5161.2ms/step | 11.6 samples/s
+Step 44060/50462 | Loss: 2.5244 | LR: 2.02e-05 | 5178.8ms/step | 11.6 samples/s
+Step 44070/50462 | Loss: 2.5240 | LR: 2.01e-05 | 5176.1ms/step | 11.6 samples/s
+Step 44080/50462 | Loss: 2.5065 | LR: 2.01e-05 | 5216.6ms/step | 11.5 samples/s
+Step 44090/50462 | Loss: 2.4756 | LR: 2.00e-05 | 5169.4ms/step | 11.6 samples/s
+Step 44100/50462 | Loss: 2.4941 | LR: 1.99e-05 | 5195.9ms/step | 11.5 samples/s
+Step 44110/50462 | Loss: 2.4963 | LR: 1.99e-05 | 5158.8ms/step | 11.6 samples/s
+Step 44120/50462 | Loss: 2.5126 | LR: 1.98e-05 | 5160.9ms/step | 11.6 samples/s
+Step 44130/50462 | Loss: 2.5185 | LR: 1.97e-05 | 5206.2ms/step | 11.5 samples/s
+Step 44140/50462 | Loss: 2.4866 | LR: 1.97e-05 | 5156.2ms/step | 11.6 samples/s
+Step 44150/50462 | Loss: 2.4821 | LR: 1.96e-05 | 5154.9ms/step | 11.6 samples/s
+Step 44160/50462 | Loss: 2.5123 | LR: 1.96e-05 | 5145.1ms/step | 11.7 samples/s
+Step 44170/50462 | Loss: 2.5159 | LR: 1.95e-05 | 5160.7ms/step | 11.6 samples/s
+Step 44180/50462 | Loss: 2.4825 | LR: 1.94e-05 | 5164.1ms/step | 11.6 samples/s
+Step 44190/50462 | Loss: 2.4891 | LR: 1.94e-05 | 5149.2ms/step | 11.7 samples/s
+Step 44200/50462 | Loss: 2.5104 | LR: 1.93e-05 | 5146.7ms/step | 11.7 samples/s
+Step 44210/50462 | Loss: 2.4985 | LR: 1.93e-05 | 5151.3ms/step | 11.6 samples/s
+Step 44220/50462 | Loss: 2.5046 | LR: 1.92e-05 | 5156.1ms/step | 11.6 samples/s
+Step 44230/50462 | Loss: 2.5162 | LR: 1.91e-05 | 5141.8ms/step | 11.7 samples/s
+Step 44240/50462 | Loss: 2.5036 | LR: 1.91e-05 | 5160.0ms/step | 11.6 samples/s
+Step 44250/50462 | Loss: 2.5171 | LR: 1.90e-05 | 5152.9ms/step | 11.6 samples/s
+Validation at step 44250: Loss = 2.5123, Perplexity = 12.33
+Step 44260/50462 | Loss: 2.5115 | LR: 1.90e-05 | 7701.5ms/step | 7.8 samples/s
+Step 44270/50462 | Loss: 2.4823 | LR: 1.89e-05 | 5127.6ms/step | 11.7 samples/s
+Step 44280/50462 | Loss: 2.5057 | LR: 1.88e-05 | 5149.5ms/step | 11.7 samples/s
+Step 44290/50462 | Loss: 2.5020 | LR: 1.88e-05 | 5160.4ms/step | 11.6 samples/s
+Step 44300/50462 | Loss: 2.4532 | LR: 1.87e-05 | 5167.3ms/step | 11.6 samples/s
+Step 44310/50462 | Loss: 2.4831 | LR: 1.87e-05 | 5135.7ms/step | 11.7 samples/s
+Step 44320/50462 | Loss: 2.5155 | LR: 1.86e-05 | 5157.1ms/step | 11.6 samples/s
+Step 44330/50462 | Loss: 2.5060 | LR: 1.85e-05 | 5139.4ms/step | 11.7 samples/s
+Step 44340/50462 | Loss: 2.5022 | LR: 1.85e-05 | 5144.5ms/step | 11.7 samples/s
+Step 44350/50462 | Loss: 2.5071 | LR: 1.84e-05 | 5186.8ms/step | 11.6 samples/s
+Step 44360/50462 | Loss: 2.4802 | LR: 1.84e-05 | 5187.2ms/step | 11.6 samples/s
+Step 44370/50462 | Loss: 2.5056 | LR: 1.83e-05 | 5139.5ms/step | 11.7 samples/s
+Step 44380/50462 | Loss: 2.4965 | LR: 1.82e-05 | 5158.3ms/step | 11.6 samples/s
+Step 44390/50462 | Loss: 2.5167 | LR: 1.82e-05 | 5154.7ms/step | 11.6 samples/s
+Step 44400/50462 | Loss: 2.4830 | LR: 1.81e-05 | 5210.4ms/step | 11.5 samples/s
+Step 44410/50462 | Loss: 2.4924 | LR: 1.81e-05 | 5157.1ms/step | 11.6 samples/s
+Step 44420/50462 | Loss: 2.5024 | LR: 1.80e-05 | 5175.3ms/step | 11.6 samples/s
+Step 44430/50462 | Loss: 2.5098 | LR: 1.79e-05 | 5167.0ms/step | 11.6 samples/s
+Step 44440/50462 | Loss: 2.4990 | LR: 1.79e-05 | 5171.2ms/step | 11.6 samples/s
+Step 44450/50462 | Loss: 2.4877 | LR: 1.78e-05 | 5161.6ms/step | 11.6 samples/s
+Step 44460/50462 | Loss: 2.5033 | LR: 1.78e-05 | 5166.7ms/step | 11.6 samples/s
+Step 44470/50462 | Loss: 2.5160 | LR: 1.77e-05 | 5161.7ms/step | 11.6 samples/s
+Step 44480/50462 | Loss: 2.5302 | LR: 1.77e-05 | 5150.9ms/step | 11.6 samples/s
+Step 44490/50462 | Loss: 2.5257 | LR: 1.76e-05 | 5181.0ms/step | 11.6 samples/s
+Step 44500/50462 | Loss: 2.4886 | LR: 1.75e-05 | 5165.4ms/step | 11.6 samples/s
+Validation at step 44500: Loss = 2.5123, Perplexity = 12.33
+Step 44510/50462 | Loss: 2.4781 | LR: 1.75e-05 | 7720.4ms/step | 7.8 samples/s
+Step 44520/50462 | Loss: 2.4769 | LR: 1.74e-05 | 5174.8ms/step | 11.6 samples/s
+Step 44530/50462 | Loss: 2.5030 | LR: 1.74e-05 | 5168.5ms/step | 11.6 samples/s
+Step 44540/50462 | Loss: 2.4912 | LR: 1.73e-05 | 5175.9ms/step | 11.6 samples/s
+Step 44550/50462 | Loss: 2.5036 | LR: 1.72e-05 | 5170.6ms/step | 11.6 samples/s
+Step 44560/50462 | Loss: 2.5094 | LR: 1.72e-05 | 5179.0ms/step | 11.6 samples/s
+Step 44570/50462 | Loss: 2.4958 | LR: 1.71e-05 | 5156.3ms/step | 11.6 samples/s
+Step 44580/50462 | Loss: 2.4739 | LR: 1.71e-05 | 5185.6ms/step | 11.6 samples/s
+Step 44590/50462 | Loss: 2.4925 | LR: 1.70e-05 | 5171.6ms/step | 11.6 samples/s
+Step 44600/50462 | Loss: 2.5163 | LR: 1.70e-05 | 5163.0ms/step | 11.6 samples/s
+Step 44610/50462 | Loss: 2.4981 | LR: 1.69e-05 | 5165.7ms/step | 11.6 samples/s
+Step 44620/50462 | Loss: 2.5157 | LR: 1.68e-05 | 5165.0ms/step | 11.6 samples/s
+Step 44630/50462 | Loss: 2.5007 | LR: 1.68e-05 | 5250.6ms/step | 11.4 samples/s
+Step 44640/50462 | Loss: 2.4917 | LR: 1.67e-05 | 5166.7ms/step | 11.6 samples/s
+Step 44650/50462 | Loss: 2.4803 | LR: 1.67e-05 | 5167.7ms/step | 11.6 samples/s
+Step 44660/50462 | Loss: 2.5181 | LR: 1.66e-05 | 5180.8ms/step | 11.6 samples/s
+Step 44670/50462 | Loss: 2.5270 | LR: 1.66e-05 | 5214.5ms/step | 11.5 samples/s
+Step 44680/50462 | Loss: 2.5130 | LR: 1.65e-05 | 5177.8ms/step | 11.6 samples/s
+Step 44690/50462 | Loss: 2.5101 | LR: 1.64e-05 | 5166.8ms/step | 11.6 samples/s
+Step 44700/50462 | Loss: 2.5152 | LR: 1.64e-05 | 5178.8ms/step | 11.6 samples/s
+Step 44710/50462 | Loss: 2.4996 | LR: 1.63e-05 | 5178.2ms/step | 11.6 samples/s
+Step 44720/50462 | Loss: 2.5010 | LR: 1.63e-05 | 5161.9ms/step | 11.6 samples/s
+Step 44730/50462 | Loss: 2.4889 | LR: 1.62e-05 | 5153.9ms/step | 11.6 samples/s
+Step 44740/50462 | Loss: 2.4846 | LR: 1.62e-05 | 5171.4ms/step | 11.6 samples/s
+Step 44750/50462 | Loss: 2.5089 | LR: 1.61e-05 | 5172.6ms/step | 11.6 samples/s
+Validation at step 44750: Loss = 2.5122, Perplexity = 12.33
+Step 44760/50462 | Loss: 2.5245 | LR: 1.61e-05 | 7734.9ms/step | 7.8 samples/s
+Step 44770/50462 | Loss: 2.4730 | LR: 1.60e-05 | 5172.9ms/step | 11.6 samples/s
+Step 44780/50462 | Loss: 2.5015 | LR: 1.59e-05 | 5169.2ms/step | 11.6 samples/s
+Step 44790/50462 | Loss: 2.4914 | LR: 1.59e-05 | 5166.6ms/step | 11.6 samples/s
+Step 44800/50462 | Loss: 2.4930 | LR: 1.58e-05 | 5166.7ms/step | 11.6 samples/s
+Step 44810/50462 | Loss: 2.5192 | LR: 1.58e-05 | 5152.5ms/step | 11.6 samples/s
+Step 44820/50462 | Loss: 2.5088 | LR: 1.57e-05 | 5173.6ms/step | 11.6 samples/s
+Step 44830/50462 | Loss: 2.5301 | LR: 1.57e-05 | 5167.2ms/step | 11.6 samples/s
+Step 44840/50462 | Loss: 2.4858 | LR: 1.56e-05 | 5176.4ms/step | 11.6 samples/s
+Step 44850/50462 | Loss: 2.4901 | LR: 1.56e-05 | 5176.0ms/step | 11.6 samples/s
+Step 44860/50462 | Loss: 2.4888 | LR: 1.55e-05 | 5155.2ms/step | 11.6 samples/s
+Step 44870/50462 | Loss: 2.4788 | LR: 1.54e-05 | 5152.3ms/step | 11.6 samples/s
+Step 44880/50462 | Loss: 2.5031 | LR: 1.54e-05 | 5163.2ms/step | 11.6 samples/s
+Step 44890/50462 | Loss: 2.4666 | LR: 1.53e-05 | 5208.8ms/step | 11.5 samples/s
+Step 44900/50462 | Loss: 2.4664 | LR: 1.53e-05 | 5213.6ms/step | 11.5 samples/s
+Step 44910/50462 | Loss: 2.4885 | LR: 1.52e-05 | 5170.2ms/step | 11.6 samples/s
+Step 44920/50462 | Loss: 2.4909 | LR: 1.52e-05 | 5174.0ms/step | 11.6 samples/s
+Step 44930/50462 | Loss: 2.4943 | LR: 1.51e-05 | 5171.5ms/step | 11.6 samples/s
+Step 44940/50462 | Loss: 2.4966 | LR: 1.51e-05 | 5151.0ms/step | 11.6 samples/s
+Step 44950/50462 | Loss: 2.4542 | LR: 1.50e-05 | 5207.3ms/step | 11.5 samples/s
+Step 44960/50462 | Loss: 2.5331 | LR: 1.50e-05 | 5170.8ms/step | 11.6 samples/s
+Step 44970/50462 | Loss: 2.4832 | LR: 1.49e-05 | 5154.2ms/step | 11.6 samples/s
+Step 44980/50462 | Loss: 2.4949 | LR: 1.49e-05 | 5175.7ms/step | 11.6 samples/s
+Step 44990/50462 | Loss: 2.5238 | LR: 1.48e-05 | 5177.0ms/step | 11.6 samples/s
+Step 45000/50462 | Loss: 2.5144 | LR: 1.47e-05 | 5163.8ms/step | 11.6 samples/s
+Validation at step 45000: Loss = 2.5122, Perplexity = 12.33
+Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-45000
+Step 45010/50462 | Loss: 2.5071 | LR: 1.47e-05 | 8679.3ms/step | 6.9 samples/s
+Step 45020/50462 | Loss: 2.4908 | LR: 1.46e-05 | 5178.7ms/step | 11.6 samples/s
+Step 45030/50462 | Loss: 2.4974 | LR: 1.46e-05 | 5161.3ms/step | 11.6 samples/s
+Step 45040/50462 | Loss: 2.5102 | LR: 1.45e-05 | 5150.5ms/step | 11.6 samples/s
+Step 45050/50462 | Loss: 2.4855 | LR: 1.45e-05 | 5167.6ms/step | 11.6 samples/s
+Step 45060/50462 | Loss: 2.5082 | LR: 1.44e-05 | 5166.0ms/step | 11.6 samples/s
+Step 45070/50462 | Loss: 2.5020 | LR: 1.44e-05 | 5171.3ms/step | 11.6 samples/s
+Step 45080/50462 | Loss: 2.4941 | LR: 1.43e-05 | 5177.2ms/step | 11.6 samples/s
+Step 45090/50462 | Loss: 2.4934 | LR: 1.43e-05 | 5165.3ms/step | 11.6 samples/s
+Step 45100/50462 | Loss: 2.5278 | LR: 1.42e-05 | 5173.7ms/step | 11.6 samples/s
+Step 45110/50462 | Loss: 2.5269 | LR: 1.42e-05 | 5155.8ms/step | 11.6 samples/s
+Step 45120/50462 | Loss: 2.5095 | LR: 1.41e-05 | 5157.4ms/step | 11.6 samples/s
+Step 45130/50462 | Loss: 2.5182 | LR: 1.41e-05 | 5160.8ms/step | 11.6 samples/s
+Step 45140/50462 | Loss: 2.4870 | LR: 1.40e-05 | 5169.1ms/step | 11.6 samples/s
+Step 45150/50462 | Loss: 2.5117 | LR: 1.40e-05 | 5204.0ms/step | 11.5 samples/s
+Step 45160/50462 | Loss: 2.5183 | LR: 1.39e-05 | 5179.8ms/step | 11.6 samples/s
+Step 45170/50462 | Loss: 2.4797 | LR: 1.39e-05 | 5212.8ms/step | 11.5 samples/s
+Step 45180/50462 | Loss: 2.5078 | LR: 1.38e-05 | 5155.7ms/step | 11.6 samples/s
+Step 45190/50462 | Loss: 2.4946 | LR: 1.37e-05 | 5152.7ms/step | 11.6 samples/s
+Step 45200/50462 | Loss: 2.5181 | LR: 1.37e-05 | 5173.8ms/step | 11.6 samples/s
+Step 45210/50462 | Loss: 2.5033 | LR: 1.36e-05 | 5157.2ms/step | 11.6 samples/s
+Step 45220/50462 | Loss: 2.4889 | LR: 1.36e-05 | 5207.7ms/step | 11.5 samples/s
+Step 45230/50462 | Loss: 2.4837 | LR: 1.35e-05 | 5166.6ms/step | 11.6 samples/s
+Step 45240/50462 | Loss: 2.5189 | LR: 1.35e-05 | 5156.8ms/step | 11.6 samples/s
+Step 45250/50462 | Loss: 2.4956 | LR: 1.34e-05 | 5164.6ms/step | 11.6 samples/s
+Validation at step 45250: Loss = 2.5122, Perplexity = 12.33
+Step 45260/50462 | Loss: 2.5068 | LR: 1.34e-05 | 7704.6ms/step | 7.8 samples/s
+Step 45270/50462 | Loss: 2.5164 | LR: 1.33e-05 | 5169.5ms/step | 11.6 samples/s
+Step 45280/50462 | Loss: 2.4738 | LR: 1.33e-05 | 5170.7ms/step | 11.6 samples/s
+Step 45290/50462 | Loss: 2.5001 | LR: 1.32e-05 | 5172.2ms/step | 11.6 samples/s
+Step 45300/50462 | Loss: 2.5188 | LR: 1.32e-05 | 5171.7ms/step | 11.6 samples/s
+Step 45310/50462 | Loss: 2.4871 | LR: 1.31e-05 | 5170.4ms/step | 11.6 samples/s
+Step 45320/50462 | Loss: 2.4950 | LR: 1.31e-05 | 5184.4ms/step | 11.6 samples/s
+Step 45330/50462 | Loss: 2.5115 | LR: 1.30e-05 | 5169.6ms/step | 11.6 samples/s
+Step 45340/50462 | Loss: 2.5031 | LR: 1.30e-05 | 5176.2ms/step | 11.6 samples/s
+Step 45350/50462 | Loss: 2.4997 | LR: 1.29e-05 | 5170.3ms/step | 11.6 samples/s
+Step 45360/50462 | Loss: 2.5190 | LR: 1.29e-05 | 5175.3ms/step | 11.6 samples/s
+Step 45370/50462 | Loss: 2.4927 | LR: 1.28e-05 | 5165.0ms/step | 11.6 samples/s
+Step 45380/50462 | Loss: 2.4886 | LR: 1.28e-05 | 5163.1ms/step | 11.6 samples/s
+Step 45390/50462 | Loss: 2.4950 | LR: 1.27e-05 | 5162.0ms/step | 11.6 samples/s
+Step 45400/50462 | Loss: 2.4790 | LR: 1.27e-05 | 5191.3ms/step | 11.6 samples/s
+Step 45410/50462 | Loss: 2.5239 | LR: 1.26e-05 | 5222.8ms/step | 11.5 samples/s
+Step 45420/50462 | Loss: 2.4946 | LR: 1.26e-05 | 5158.9ms/step | 11.6 samples/s
+Step 45430/50462 | Loss: 2.4696 | LR: 1.25e-05 | 5174.2ms/step | 11.6 samples/s
+Step 45440/50462 | Loss: 2.5116 | LR: 1.25e-05 | 5207.5ms/step | 11.5 samples/s
+Step 45450/50462 | Loss: 2.5015 | LR: 1.24e-05 | 5145.4ms/step | 11.7 samples/s
+Step 45460/50462 | Loss: 2.4911 | LR: 1.24e-05 | 5159.3ms/step | 11.6 samples/s
+Step 45470/50462 | Loss: 2.4999 | LR: 1.23e-05 | 5173.4ms/step | 11.6 samples/s
+Step 45480/50462 | Loss: 2.4902 | LR: 1.23e-05 | 5159.5ms/step | 11.6 samples/s
+Step 45490/50462 | Loss: 2.5119 | LR: 1.22e-05 | 5168.5ms/step | 11.6 samples/s
+Step 45500/50462 | Loss: 2.5116 | LR: 1.22e-05 | 5206.0ms/step | 11.5 samples/s
+Validation at step 45500: Loss = 2.5122, Perplexity = 12.33
+Step 45510/50462 | Loss: 2.4855 | LR: 1.21e-05 | 7711.9ms/step | 7.8 samples/s
+Step 45520/50462 | Loss: 2.5318 | LR: 1.21e-05 | 5168.2ms/step | 11.6 samples/s
+Step 45530/50462 | Loss: 2.4780 | LR: 1.20e-05 | 5172.1ms/step | 11.6 samples/s
+Step 45540/50462 | Loss: 2.5264 | LR: 1.20e-05 | 5166.6ms/step | 11.6 samples/s
+Step 45550/50462 | Loss: 2.4863 | LR: 1.19e-05 | 5166.1ms/step | 11.6 samples/s
+Step 45560/50462 | Loss: 2.5110 | LR: 1.19e-05 | 5171.5ms/step | 11.6 samples/s
+Step 45570/50462 | Loss: 2.5157 | LR: 1.19e-05 | 5155.1ms/step | 11.6 samples/s
+Step 45580/50462 | Loss: 2.4926 | LR: 1.18e-05 | 5163.4ms/step | 11.6 samples/s
+Step 45590/50462 | Loss: 2.4813 | LR: 1.18e-05 | 5165.5ms/step | 11.6 samples/s
+Step 45600/50462 | Loss: 2.4902 | LR: 1.17e-05 | 5174.2ms/step | 11.6 samples/s
+Step 45610/50462 | Loss: 2.4683 | LR: 1.17e-05 | 5153.4ms/step | 11.6 samples/s
+Step 45620/50462 | Loss: 2.5001 | LR: 1.16e-05 | 5154.3ms/step | 11.6 samples/s
+Step 45630/50462 | Loss: 2.5168 | LR: 1.16e-05 | 5168.5ms/step | 11.6 samples/s
+Step 45640/50462 | Loss: 2.4951 | LR: 1.15e-05 | 5165.2ms/step | 11.6 samples/s
+Step 45650/50462 | Loss: 2.5196 | LR: 1.15e-05 | 5158.8ms/step | 11.6 samples/s
+Step 45660/50462 | Loss: 2.5050 | LR: 1.14e-05 | 5181.2ms/step | 11.6 samples/s
+Step 45670/50462 | Loss: 2.4715 | LR: 1.14e-05 | 5167.4ms/step | 11.6 samples/s
+Step 45680/50462 | Loss: 2.4913 | LR: 1.13e-05 | 5205.1ms/step | 11.5 samples/s
+Step 45690/50462 | Loss: 2.5117 | LR: 1.13e-05 | 5158.2ms/step | 11.6 samples/s
+Step 45700/50462 | Loss: 2.5041 | LR: 1.12e-05 | 5153.4ms/step | 11.6 samples/s
+Step 45710/50462 | Loss: 2.5068 | LR: 1.12e-05 | 5164.0ms/step | 11.6 samples/s
+Step 45720/50462 | Loss: 2.4981 | LR: 1.11e-05 | 5219.5ms/step | 11.5 samples/s
+Step 45730/50462 | Loss: 2.4800 | LR: 1.11e-05 | 5165.7ms/step | 11.6 samples/s
+Step 45740/50462 | Loss: 2.4934 | LR: 1.10e-05 | 5155.9ms/step | 11.6 samples/s
+Step 45750/50462 | Loss: 2.5114 | LR: 1.10e-05 | 5164.5ms/step | 11.6 samples/s
+Validation at step 45750: Loss = 2.5121, Perplexity = 12.33
+Step 45760/50462 | Loss: 2.4660 | LR: 1.10e-05 | 7716.8ms/step | 7.8 samples/s
+Step 45770/50462 | Loss: 2.4983 | LR: 1.09e-05 | 5214.8ms/step | 11.5 samples/s
+Step 45780/50462 | Loss: 2.4909 | LR: 1.09e-05 | 5154.2ms/step | 11.6 samples/s
+Step 45790/50462 | Loss: 2.5291 | LR: 1.08e-05 | 5175.8ms/step | 11.6 samples/s
+Step 45800/50462 | Loss: 2.4425 | LR: 1.08e-05 | 5178.7ms/step | 11.6 samples/s
+Step 45810/50462 | Loss: 2.5032 | LR: 1.07e-05 | 5170.0ms/step | 11.6 samples/s
+Step 45820/50462 | Loss: 2.4993 | LR: 1.07e-05 | 5165.5ms/step | 11.6 samples/s
+Step 45830/50462 | Loss: 2.5056 | LR: 1.06e-05 | 5180.5ms/step | 11.6 samples/s
+Step 45840/50462 | Loss: 2.5188 | LR: 1.06e-05 | 5169.1ms/step | 11.6 samples/s
+Step 45850/50462 | Loss: 2.5006 | LR: 1.05e-05 | 5168.4ms/step | 11.6 samples/s
+Step 45860/50462 | Loss: 2.5256 | LR: 1.05e-05 | 5157.1ms/step | 11.6 samples/s
+Step 45870/50462 | Loss: 2.5105 | LR: 1.05e-05 | 5170.1ms/step | 11.6 samples/s
+Step 45880/50462 | Loss: 2.5114 | LR: 1.04e-05 | 5159.2ms/step | 11.6 samples/s
+Step 45890/50462 | Loss: 2.4737 | LR: 1.04e-05 | 5188.2ms/step | 11.6 samples/s
+Step 45900/50462 | Loss: 2.4968 | LR: 1.03e-05 | 5173.0ms/step | 11.6 samples/s
+Step 45910/50462 | Loss: 2.5271 | LR: 1.03e-05 | 5174.1ms/step | 11.6 samples/s
+Step 45920/50462 | Loss: 2.4798 | LR: 1.02e-05 | 5170.7ms/step | 11.6 samples/s
+Step 45930/50462 | Loss: 2.4918 | LR: 1.02e-05 | 5175.8ms/step | 11.6 samples/s
+Step 45940/50462 | Loss: 2.5108 | LR: 1.01e-05 | 5204.0ms/step | 11.5 samples/s
+Step 45950/50462 | Loss: 2.4982 | LR: 1.01e-05 | 5175.1ms/step | 11.6 samples/s
+Step 45960/50462 | Loss: 2.4840 | LR: 1.01e-05 | 5173.7ms/step | 11.6 samples/s
+Step 45970/50462 | Loss: 2.4895 | LR: 1.00e-05 | 5173.1ms/step | 11.6 samples/s
+Step 45980/50462 | Loss: 2.5332 | LR: 9.96e-06 | 5171.5ms/step | 11.6 samples/s
+Step 45990/50462 | Loss: 2.4876 | LR: 9.92e-06 | 5201.2ms/step | 11.5 samples/s
+Step 46000/50462 | Loss: 2.5017 | LR: 9.87e-06 | 5176.3ms/step | 11.6 samples/s
+Validation at step 46000: Loss = 2.5121, Perplexity = 12.33
+Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-46000
+Step 46010/50462 | Loss: 2.5043 | LR: 9.83e-06 | 8694.1ms/step | 6.9 samples/s
+Step 46020/50462 | Loss: 2.4909 | LR: 9.79e-06 | 5162.0ms/step | 11.6 samples/s
+Step 46030/50462 | Loss: 2.5147 | LR: 9.74e-06 | 5155.4ms/step | 11.6 samples/s
+Step 46040/50462 | Loss: 2.5145 | LR: 9.70e-06 | 5208.4ms/step | 11.5 samples/s
+Step 46050/50462 | Loss: 2.4978 | LR: 9.65e-06 | 5165.6ms/step | 11.6 samples/s
+Step 46060/50462 | Loss: 2.4923 | LR: 9.61e-06 | 5163.9ms/step | 11.6 samples/s
+Step 46070/50462 | Loss: 2.5114 | LR: 9.57e-06 | 5160.8ms/step | 11.6 samples/s
+Step 46080/50462 | Loss: 2.5065 | LR: 9.52e-06 | 5154.8ms/step | 11.6 samples/s
+Step 46090/50462 | Loss: 2.4753 | LR: 9.48e-06 | 5156.4ms/step | 11.6 samples/s
+Step 46100/50462 | Loss: 2.5017 | LR: 9.44e-06 | 5155.8ms/step | 11.6 samples/s
+Step 46110/50462 | Loss: 2.4972 | LR: 9.40e-06 | 5152.3ms/step | 11.6 samples/s
+Step 46120/50462 | Loss: 2.4924 | LR: 9.35e-06 | 5165.7ms/step | 11.6 samples/s
+Step 46130/50462 | Loss: 2.4939 | LR: 9.31e-06 | 5150.6ms/step | 11.6 samples/s
+Step 46140/50462 | Loss: 2.4942 | LR: 9.27e-06 | 5162.9ms/step | 11.6 samples/s
+Step 46150/50462 | Loss: 2.4926 | LR: 9.22e-06 | 5160.8ms/step | 11.6 samples/s
+Step 46160/50462 | Loss: 2.5103 | LR: 9.18e-06 | 5146.4ms/step | 11.7 samples/s
+Step 46170/50462 | Loss: 2.5108 | LR: 9.14e-06 | 5158.1ms/step | 11.6 samples/s
+Step 46180/50462 | Loss: 2.4806 | LR: 9.10e-06 | 5135.5ms/step | 11.7 samples/s
+Step 46190/50462 | Loss: 2.4809 | LR: 9.06e-06 | 5144.6ms/step | 11.7 samples/s
+Step 46200/50462 | Loss: 2.4887 | LR: 9.01e-06 | 5204.1ms/step | 11.5 samples/s
+Step 46210/50462 | Loss: 2.5053 | LR: 8.97e-06 | 5156.6ms/step | 11.6 samples/s
+Step 46220/50462 | Loss: 2.5080 | LR: 8.93e-06 | 5161.9ms/step | 11.6 samples/s
+Step 46230/50462 | Loss: 2.5142 | LR: 8.89e-06 | 5145.2ms/step | 11.7 samples/s
+Step 46240/50462 | Loss: 2.5229 | LR: 8.85e-06 | 5159.1ms/step | 11.6 samples/s
+Step 46250/50462 | Loss: 2.5206 | LR: 8.80e-06 | 5156.8ms/step | 11.6 samples/s
+Validation at step 46250: Loss = 2.5121, Perplexity = 12.33
+Step 46260/50462 | Loss: 2.5207 | LR: 8.76e-06 | 7757.1ms/step | 7.7 samples/s
+Step 46270/50462 | Loss: 2.5073 | LR: 8.72e-06 | 5188.2ms/step | 11.6 samples/s
+Step 46280/50462 | Loss: 2.5386 | LR: 8.68e-06 | 5176.2ms/step | 11.6 samples/s
+Step 46290/50462 | Loss: 2.5252 | LR: 8.64e-06 | 5168.7ms/step | 11.6 samples/s
+Step 46300/50462 | Loss: 2.4959 | LR: 8.60e-06 | 5174.0ms/step | 11.6 samples/s
+Step 46310/50462 | Loss: 2.5035 | LR: 8.56e-06 | 5168.8ms/step | 11.6 samples/s
+Step 46320/50462 | Loss: 2.5289 | LR: 8.52e-06 | 5195.4ms/step | 11.5 samples/s
+Step 46330/50462 | Loss: 2.5086 | LR: 8.47e-06 | 5155.3ms/step | 11.6 samples/s
+Step 46340/50462 | Loss: 2.5083 | LR: 8.43e-06 | 5143.4ms/step | 11.7 samples/s
+Step 46350/50462 | Loss: 2.5293 | LR: 8.39e-06 | 5147.4ms/step | 11.7 samples/s
+Step 46360/50462 | Loss: 2.4921 | LR: 8.35e-06 | 5137.0ms/step | 11.7 samples/s
+Step 46370/50462 | Loss: 2.4777 | LR: 8.31e-06 | 5134.7ms/step | 11.7 samples/s
+Step 46380/50462 | Loss: 2.5026 | LR: 8.27e-06 | 5156.8ms/step | 11.6 samples/s
+Step 46390/50462 | Loss: 2.4917 | LR: 8.23e-06 | 5148.3ms/step | 11.7 samples/s
+Step 46400/50462 | Loss: 2.4833 | LR: 8.19e-06 | 5155.5ms/step | 11.6 samples/s
+Step 46410/50462 | Loss: 2.4958 | LR: 8.15e-06 | 5139.3ms/step | 11.7 samples/s
+Step 46420/50462 | Loss: 2.4957 | LR: 8.11e-06 | 5151.1ms/step | 11.6 samples/s
+Step 46430/50462 | Loss: 2.5162 | LR: 8.07e-06 | 5155.7ms/step | 11.6 samples/s
+Step 46440/50462 | Loss: 2.4991 | LR: 8.03e-06 | 5162.5ms/step | 11.6 samples/s
+Step 46450/50462 | Loss: 2.5249 | LR: 7.99e-06 | 5141.0ms/step | 11.7 samples/s
+Step 46460/50462 | Loss: 2.5032 | LR: 7.95e-06 | 5192.0ms/step | 11.6 samples/s
+Step 46470/50462 | Loss: 2.4944 | LR: 7.91e-06 | 5157.5ms/step | 11.6 samples/s
+Step 46480/50462 | Loss: 2.4895 | LR: 7.87e-06 | 5151.6ms/step | 11.6 samples/s
+Step 46490/50462 | Loss: 2.4856 | LR: 7.83e-06 | 5151.3ms/step | 11.6 samples/s
+Step 46500/50462 | Loss: 2.5026 | LR: 7.80e-06 | 5141.7ms/step | 11.7 samples/s
+Validation at step 46500: Loss = 2.5121, Perplexity = 12.33
+Step 46510/50462 | Loss: 2.4972 | LR: 7.76e-06 | 7702.3ms/step | 7.8 samples/s
+Step 46520/50462 | Loss: 2.5205 | LR: 7.72e-06 | 5165.0ms/step | 11.6 samples/s
+Step 46530/50462 | Loss: 2.5005 | LR: 7.68e-06 | 5215.7ms/step | 11.5 samples/s
+Step 46540/50462 | Loss: 2.5134 | LR: 7.64e-06 | 5162.7ms/step | 11.6 samples/s
+Step 46550/50462 | Loss: 2.4748 | LR: 7.60e-06 | 5156.6ms/step | 11.6 samples/s
+Step 46560/50462 | Loss: 2.4918 | LR: 7.56e-06 | 5166.6ms/step | 11.6 samples/s
+Step 46570/50462 | Loss: 2.5024 | LR: 7.52e-06 | 5162.7ms/step | 11.6 samples/s
+Step 46580/50462 | Loss: 2.4943 | LR: 7.49e-06 | 5165.8ms/step | 11.6 samples/s
+Step 46590/50462 | Loss: 2.4869 | LR: 7.45e-06 | 5201.1ms/step | 11.5 samples/s
+Step 46600/50462 | Loss: 2.5020 | LR: 7.41e-06 | 5182.2ms/step | 11.6 samples/s
+Step 46610/50462 | Loss: 2.5003 | LR: 7.37e-06 | 5164.7ms/step | 11.6 samples/s
+Step 46620/50462 | Loss: 2.4830 | LR: 7.33e-06 | 5165.9ms/step | 11.6 samples/s
+Step 46630/50462 | Loss: 2.4972 | LR: 7.29e-06 | 5159.5ms/step | 11.6 samples/s
+Step 46640/50462 | Loss: 2.5092 | LR: 7.26e-06 | 5163.1ms/step | 11.6 samples/s
+Step 46650/50462 | Loss: 2.5013 | LR: 7.22e-06 | 5155.6ms/step | 11.6 samples/s
+Step 46660/50462 | Loss: 2.4912 | LR: 7.18e-06 | 5141.4ms/step | 11.7 samples/s
+Step 46670/50462 | Loss: 2.5060 | LR: 7.14e-06 | 5150.0ms/step | 11.7 samples/s
+Step 46680/50462 | Loss: 2.5145 | LR: 7.11e-06 | 5165.3ms/step | 11.6 samples/s
+Step 46690/50462 | Loss: 2.5314 | LR: 7.07e-06 | 5160.2ms/step | 11.6 samples/s
+Step 46700/50462 | Loss: 2.4918 | LR: 7.03e-06 | 5153.5ms/step | 11.6 samples/s
+Step 46710/50462 | Loss: 2.5247 | LR: 6.99e-06 | 5163.8ms/step | 11.6 samples/s
+Step 46720/50462 | Loss: 2.5083 | LR: 6.96e-06 | 5206.7ms/step | 11.5 samples/s
+Step 46730/50462 | Loss: 2.4882 | LR: 6.92e-06 | 5164.7ms/step | 11.6 samples/s
+Step 46740/50462 | Loss: 2.5148 | LR: 6.88e-06 | 5160.4ms/step | 11.6 samples/s
+Step 46750/50462 | Loss: 2.4976 | LR: 6.85e-06 | 5156.4ms/step | 11.6 samples/s
+Validation at step 46750: Loss = 2.5121, Perplexity = 12.33
+Step 46760/50462 | Loss: 2.5122 | LR: 6.81e-06 | 7726.0ms/step | 7.8 samples/s
+Step 46770/50462 | Loss: 2.5018 | LR: 6.77e-06 | 5178.9ms/step | 11.6 samples/s
+Step 46780/50462 | Loss: 2.5271 | LR: 6.74e-06 | 5178.6ms/step | 11.6 samples/s
+Step 46790/50462 | Loss: 2.4844 | LR: 6.70e-06 | 5154.5ms/step | 11.6 samples/s
+Step 46800/50462 | Loss: 2.5224 | LR: 6.66e-06 | 5171.0ms/step | 11.6 samples/s
+Step 46810/50462 | Loss: 2.5185 | LR: 6.63e-06 | 5215.6ms/step | 11.5 samples/s
+Step 46820/50462 | Loss: 2.4818 | LR: 6.59e-06 | 5149.9ms/step | 11.7 samples/s
+Step 46830/50462 | Loss: 2.4800 | LR: 6.56e-06 | 5153.9ms/step | 11.6 samples/s
+Step 46840/50462 | Loss: 2.5056 | LR: 6.52e-06 | 5151.8ms/step | 11.6 samples/s
+Step 46850/50462 | Loss: 2.5016 | LR: 6.48e-06 | 5156.7ms/step | 11.6 samples/s
+Step 46860/50462 | Loss: 2.5011 | LR: 6.45e-06 | 5152.8ms/step | 11.6 samples/s
+Step 46870/50462 | Loss: 2.4593 | LR: 6.41e-06 | 5206.8ms/step | 11.5 samples/s
+Step 46880/50462 | Loss: 2.4948 | LR: 6.38e-06 | 5166.4ms/step | 11.6 samples/s
+Step 46890/50462 | Loss: 2.5007 | LR: 6.34e-06 | 5153.6ms/step | 11.6 samples/s
+Step 46900/50462 | Loss: 2.4931 | LR: 6.31e-06 | 5163.3ms/step | 11.6 samples/s
+Step 46910/50462 | Loss: 2.4861 | LR: 6.27e-06 | 5160.7ms/step | 11.6 samples/s
+Step 46920/50462 | Loss: 2.5241 | LR: 6.24e-06 | 5154.6ms/step | 11.6 samples/s
+Step 46930/50462 | Loss: 2.5003 | LR: 6.20e-06 | 5165.4ms/step | 11.6 samples/s
+Step 46940/50462 | Loss: 2.5022 | LR: 6.17e-06 | 5170.9ms/step | 11.6 samples/s
+Step 46950/50462 | Loss: 2.4854 | LR: 6.13e-06 | 5158.9ms/step | 11.6 samples/s
+Step 46960/50462 | Loss: 2.5086 | LR: 6.10e-06 | 5172.0ms/step | 11.6 samples/s
+Step 46970/50462 | Loss: 2.5124 | LR: 6.06e-06 | 5157.0ms/step | 11.6 samples/s
+Step 46980/50462 | Loss: 2.5115 | LR: 6.03e-06 | 5164.9ms/step | 11.6 samples/s
+Step 46990/50462 | Loss: 2.5326 | LR: 5.99e-06 | 5197.7ms/step | 11.5 samples/s
+Step 47000/50462 | Loss: 2.5122 | LR: 5.96e-06 | 5158.4ms/step | 11.6 samples/s
+Validation at step 47000: Loss = 2.5121, Perplexity = 12.33
+Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-47000
+Step 47010/50462 | Loss: 2.5239 | LR: 5.93e-06 | 8654.8ms/step | 6.9 samples/s
+Step 47020/50462 | Loss: 2.5006 | LR: 5.89e-06 | 5159.7ms/step | 11.6 samples/s
+Step 47030/50462 | Loss: 2.5206 | LR: 5.86e-06 | 5163.1ms/step | 11.6 samples/s
+Step 47040/50462 | Loss: 2.4658 | LR: 5.82e-06 | 5160.4ms/step | 11.6 samples/s
+Step 47050/50462 | Loss: 2.5023 | LR: 5.79e-06 | 5148.0ms/step | 11.7 samples/s
+Step 47060/50462 | Loss: 2.5007 | LR: 5.76e-06 | 5158.2ms/step | 11.6 samples/s
+Step 47070/50462 | Loss: 2.4936 | LR: 5.72e-06 | 5158.2ms/step | 11.6 samples/s
+Step 47080/50462 | Loss: 2.5014 | LR: 5.69e-06 | 5227.3ms/step | 11.5 samples/s
+Step 47090/50462 | Loss: 2.4962 | LR: 5.65e-06 | 5167.7ms/step | 11.6 samples/s
+Step 47100/50462 | Loss: 2.5002 | LR: 5.62e-06 | 5156.2ms/step | 11.6 samples/s
+Step 47110/50462 | Loss: 2.5039 | LR: 5.59e-06 | 5156.0ms/step | 11.6 samples/s
+Step 47120/50462 | Loss: 2.4543 | LR: 5.56e-06 | 5151.7ms/step | 11.6 samples/s
+Step 47130/50462 | Loss: 2.5012 | LR: 5.52e-06 | 5134.2ms/step | 11.7 samples/s
+Step 47140/50462 | Loss: 2.5135 | LR: 5.49e-06 | 5199.2ms/step | 11.5 samples/s
+Step 47150/50462 | Loss: 2.4698 | LR: 5.46e-06 | 5175.1ms/step | 11.6 samples/s
+Step 47160/50462 | Loss: 2.5117 | LR: 5.42e-06 | 5157.5ms/step | 11.6 samples/s
+Step 47170/50462 | Loss: 2.4945 | LR: 5.39e-06 | 5137.3ms/step | 11.7 samples/s
+Step 47180/50462 | Loss: 2.5041 | LR: 5.36e-06 | 5155.3ms/step | 11.6 samples/s
+Step 47190/50462 | Loss: 2.4985 | LR: 5.33e-06 | 5161.5ms/step | 11.6 samples/s
+Step 47200/50462 | Loss: 2.4966 | LR: 5.29e-06 | 5162.1ms/step | 11.6 samples/s
+Step 47210/50462 | Loss: 2.4706 | LR: 5.26e-06 | 5167.6ms/step | 11.6 samples/s
+Step 47220/50462 | Loss: 2.5113 | LR: 5.23e-06 | 5177.5ms/step | 11.6 samples/s
+Step 47230/50462 | Loss: 2.5095 | LR: 5.20e-06 | 5167.2ms/step | 11.6 samples/s
+Step 47240/50462 | Loss: 2.4779 | LR: 5.16e-06 | 5154.2ms/step | 11.6 samples/s
+Step 47250/50462 | Loss: 2.5085 | LR: 5.13e-06 | 5232.1ms/step | 11.5 samples/s
+Validation at step 47250: Loss = 2.5121, Perplexity = 12.33
+Step 47260/50462 | Loss: 2.5029 | LR: 5.10e-06 | 7708.3ms/step | 7.8 samples/s
+Step 47270/50462 | Loss: 2.5147 | LR: 5.07e-06 | 5155.7ms/step | 11.6 samples/s
+Step 47280/50462 | Loss: 2.4883 | LR: 5.04e-06 | 5175.8ms/step | 11.6 samples/s
+Step 47290/50462 | Loss: 2.5163 | LR: 5.01e-06 | 5173.0ms/step | 11.6 samples/s
+Step 47300/50462 | Loss: 2.5146 | LR: 4.97e-06 | 5144.1ms/step | 11.7 samples/s
+Step 47310/50462 | Loss: 2.4767 | LR: 4.94e-06 | 5174.1ms/step | 11.6 samples/s
+Step 47320/50462 | Loss: 2.4937 | LR: 4.91e-06 | 5156.7ms/step | 11.6 samples/s
+Step 47330/50462 | Loss: 2.4967 | LR: 4.88e-06 | 5169.2ms/step | 11.6 samples/s
+Step 47340/50462 | Loss: 2.4616 | LR: 4.85e-06 | 5178.0ms/step | 11.6 samples/s
+Step 47350/50462 | Loss: 2.5092 | LR: 4.82e-06 | 5166.1ms/step | 11.6 samples/s
+Step 47360/50462 | Loss: 2.4798 | LR: 4.79e-06 | 5229.1ms/step | 11.5 samples/s
+Step 47370/50462 | Loss: 2.5180 | LR: 4.76e-06 | 5172.2ms/step | 11.6 samples/s
+Step 47380/50462 | Loss: 2.5103 | LR: 4.73e-06 | 5156.9ms/step | 11.6 samples/s
+Step 47390/50462 | Loss: 2.4930 | LR: 4.70e-06 | 5159.7ms/step | 11.6 samples/s
+Step 47400/50462 | Loss: 2.4963 | LR: 4.67e-06 | 5155.3ms/step | 11.6 samples/s
+Step 47410/50462 | Loss: 2.5076 | LR: 4.64e-06 | 5203.4ms/step | 11.5 samples/s
+Step 47420/50462 | Loss: 2.4593 | LR: 4.61e-06 | 5153.7ms/step | 11.6 samples/s
+Step 47430/50462 | Loss: 2.4873 | LR: 4.58e-06 | 5166.8ms/step | 11.6 samples/s
+Step 47440/50462 | Loss: 2.4960 | LR: 4.55e-06 | 5165.0ms/step | 11.6 samples/s
+Step 47450/50462 | Loss: 2.4708 | LR: 4.52e-06 | 5160.9ms/step | 11.6 samples/s
+Step 47460/50462 | Loss: 2.5154 | LR: 4.49e-06 | 5161.0ms/step | 11.6 samples/s
+Step 47470/50462 | Loss: 2.5133 | LR: 4.46e-06 | 5157.8ms/step | 11.6 samples/s
+Step 47480/50462 | Loss: 2.5120 | LR: 4.43e-06 | 5165.1ms/step | 11.6 samples/s
+Step 47490/50462 | Loss: 2.5050 | LR: 4.40e-06 | 5164.8ms/step | 11.6 samples/s
+Step 47500/50462 | Loss: 2.4962 | LR: 4.37e-06 | 5162.8ms/step | 11.6 samples/s
+Validation at step 47500: Loss = 2.5121, Perplexity = 12.33
+Step 47510/50462 | Loss: 2.4670 | LR: 4.34e-06 | 7753.0ms/step | 7.7 samples/s
+Step 47520/50462 | Loss: 2.5150 | LR: 4.31e-06 | 5169.2ms/step | 11.6 samples/s
+Step 47530/50462 | Loss: 2.5103 | LR: 4.28e-06 | 5163.0ms/step | 11.6 samples/s
+Step 47540/50462 | Loss: 2.4787 | LR: 4.25e-06 | 5181.4ms/step | 11.6 samples/s
+Step 47550/50462 | Loss: 2.5080 | LR: 4.22e-06 | 5166.9ms/step | 11.6 samples/s
+Step 47560/50462 | Loss: 2.4934 | LR: 4.19e-06 | 5159.8ms/step | 11.6 samples/s
+Step 47570/50462 | Loss: 2.5036 | LR: 4.16e-06 | 5169.6ms/step | 11.6 samples/s
+Step 47580/50462 | Loss: 2.5389 | LR: 4.13e-06 | 5158.9ms/step | 11.6 samples/s
+Step 47590/50462 | Loss: 2.5096 | LR: 4.11e-06 | 5141.6ms/step | 11.7 samples/s
+Step 47600/50462 | Loss: 2.5005 | LR: 4.08e-06 | 5161.7ms/step | 11.6 samples/s
+Step 47610/50462 | Loss: 2.4803 | LR: 4.05e-06 | 5154.9ms/step | 11.6 samples/s
+Step 47620/50462 | Loss: 2.4942 | LR: 4.02e-06 | 5167.2ms/step | 11.6 samples/s
+Step 47630/50462 | Loss: 2.5213 | LR: 3.99e-06 | 5207.7ms/step | 11.5 samples/s
+Step 47640/50462 | Loss: 2.4764 | LR: 3.97e-06 | 5159.7ms/step | 11.6 samples/s
+Step 47650/50462 | Loss: 2.4956 | LR: 3.94e-06 | 5153.7ms/step | 11.6 samples/s
+Step 47660/50462 | Loss: 2.4947 | LR: 3.91e-06 | 5162.2ms/step | 11.6 samples/s
+Step 47670/50462 | Loss: 2.4868 | LR: 3.88e-06 | 5164.1ms/step | 11.6 samples/s
+Step 47680/50462 | Loss: 2.5052 | LR: 3.85e-06 | 5163.9ms/step | 11.6 samples/s
+Step 47690/50462 | Loss: 2.5007 | LR: 3.83e-06 | 5197.5ms/step | 11.5 samples/s
+Step 47700/50462 | Loss: 2.5088 | LR: 3.80e-06 | 5157.6ms/step | 11.6 samples/s
+Step 47710/50462 | Loss: 2.4749 | LR: 3.77e-06 | 5158.7ms/step | 11.6 samples/s
+Step 47720/50462 | Loss: 2.5018 | LR: 3.74e-06 | 5147.0ms/step | 11.7 samples/s
+Step 47730/50462 | Loss: 2.5027 | LR: 3.72e-06 | 5164.5ms/step | 11.6 samples/s
+Step 47740/50462 | Loss: 2.4877 | LR: 3.69e-06 | 5172.5ms/step | 11.6 samples/s
+Step 47750/50462 | Loss: 2.5147 | LR: 3.66e-06 | 5163.1ms/step | 11.6 samples/s
+Validation at step 47750: Loss = 2.5121, Perplexity = 12.33
+Step 47760/50462 | Loss: 2.4721 | LR: 3.64e-06 | 7694.0ms/step | 7.8 samples/s
+Step 47770/50462 | Loss: 2.5342 | LR: 3.61e-06 | 5144.6ms/step | 11.7 samples/s
+Step 47780/50462 | Loss: 2.4917 | LR: 3.58e-06 | 5216.3ms/step | 11.5 samples/s
+Step 47790/50462 | Loss: 2.4864 | LR: 3.56e-06 | 5151.4ms/step | 11.6 samples/s
+Step 47800/50462 | Loss: 2.4780 | LR: 3.53e-06 | 5157.5ms/step | 11.6 samples/s
+Step 47810/50462 | Loss: 2.5113 | LR: 3.50e-06 | 5155.8ms/step | 11.6 samples/s
+Step 47820/50462 | Loss: 2.5045 | LR: 3.48e-06 | 5152.0ms/step | 11.6 samples/s
+Step 47830/50462 | Loss: 2.4974 | LR: 3.45e-06 | 5155.1ms/step | 11.6 samples/s
+Step 47840/50462 | Loss: 2.5197 | LR: 3.42e-06 | 5158.4ms/step | 11.6 samples/s
+Step 47850/50462 | Loss: 2.5102 | LR: 3.40e-06 | 5159.5ms/step | 11.6 samples/s
+Step 47860/50462 | Loss: 2.5130 | LR: 3.37e-06 | 5156.5ms/step | 11.6 samples/s
+Step 47870/50462 | Loss: 2.5065 | LR: 3.35e-06 | 5173.1ms/step | 11.6 samples/s
+Step 47880/50462 | Loss: 2.4612 | LR: 3.32e-06 | 5151.7ms/step | 11.6 samples/s
+Step 47890/50462 | Loss: 2.5190 | LR: 3.30e-06 | 5155.2ms/step | 11.6 samples/s
+Step 47900/50462 | Loss: 2.5069 | LR: 3.27e-06 | 5176.8ms/step | 11.6 samples/s
+Step 47910/50462 | Loss: 2.5068 | LR: 3.24e-06 | 5174.2ms/step | 11.6 samples/s
+Step 47920/50462 | Loss: 2.4809 | LR: 3.22e-06 | 5166.4ms/step | 11.6 samples/s
+Step 47930/50462 | Loss: 2.5179 | LR: 3.19e-06 | 5172.6ms/step | 11.6 samples/s
+Step 47940/50462 | Loss: 2.4952 | LR: 3.17e-06 | 5176.6ms/step | 11.6 samples/s
+Step 47950/50462 | Loss: 2.5012 | LR: 3.14e-06 | 5172.3ms/step | 11.6 samples/s
+Step 47960/50462 | Loss: 2.4683 | LR: 3.12e-06 | 5201.4ms/step | 11.5 samples/s
+Step 47970/50462 | Loss: 2.5050 | LR: 3.09e-06 | 5183.8ms/step | 11.6 samples/s
+Step 47980/50462 | Loss: 2.5016 | LR: 3.07e-06 | 5172.0ms/step | 11.6 samples/s
+Step 47990/50462 | Loss: 2.4848 | LR: 3.04e-06 | 5170.3ms/step | 11.6 samples/s
+Step 48000/50462 | Loss: 2.4946 | LR: 3.02e-06 | 5178.2ms/step | 11.6 samples/s
+Validation at step 48000: Loss = 2.5121, Perplexity = 12.33
+Saved checkpoint to checkpoints/run_20260601_062617/checkpoint-48000
+Step 48010/50462 | Loss: 2.5078 | LR: 3.00e-06 | 8664.5ms/step | 6.9 samples/s
+Step 48020/50462 | Loss: 2.5358 | LR: 2.97e-06 | 5166.7ms/step | 11.6 samples/s
+Step 48030/50462 | Loss: 2.4779 | LR: 2.95e-06 | 5196.9ms/step | 11.5 samples/s
+Step 48040/50462 | Loss: 2.4916 | LR: 2.92e-06 | 5163.9ms/step | 11.6 samples/s
+Step 48050/50462 | Loss: 2.4842 | LR: 2.90e-06 | 5166.4ms/step | 11.6 samples/s
+Step 48060/50462 | Loss: 2.5014 | LR: 2.87e-06 | 5165.8ms/step | 11.6 samples/s
+Step 48070/50462 | Loss: 2.5037 | LR: 2.85e-06 | 5159.4ms/step | 11.6 samples/s
+Step 48080/50462 | Loss: 2.5276 | LR: 2.83e-06 | 5160.8ms/step | 11.6 samples/s
+Step 48090/50462 | Loss: 2.5037 | LR: 2.80e-06 | 5167.1ms/step | 11.6 samples/s
+Step 48100/50462 | Loss: 2.5066 | LR: 2.78e-06 | 5155.8ms/step | 11.6 samples/s
+Step 48110/50462 | Loss: 2.4964 | LR: 2.76e-06 | 5158.6ms/step | 11.6 samples/s
+Step 48120/50462 | Loss: 2.4944 | LR: 2.73e-06 | 5166.9ms/step | 11.6 samples/s
+Step 48130/50462 | Loss: 2.4955 | LR: 2.71e-06 | 5162.1ms/step | 11.6 samples/s
+Step 48140/50462 | Loss: 2.4773 | LR: 2.69e-06 | 5148.5ms/step | 11.7 samples/s
+Step 48150/50462 | Loss: 2.4907 | LR: 2.66e-06 | 5151.3ms/step | 11.6 samples/s
+Step 48160/50462 | Loss: 2.5102 | LR: 2.64e-06 | 5146.5ms/step | 11.7 samples/s
+Step 48170/50462 | Loss: 2.4898 | LR: 2.62e-06 | 5152.4ms/step | 11.6 samples/s
+Step 48180/50462 | Loss: 2.5025 | LR: 2.60e-06 | 5207.4ms/step | 11.5 samples/s
+Step 48190/50462 | Loss: 2.4987 | LR: 2.57e-06 | 5148.8ms/step | 11.7 samples/s
+Step 48200/50462 | Loss: 2.5081 | LR: 2.55e-06 | 5141.7ms/step | 11.7 samples/s
+Step 48210/50462 | Loss: 2.4840 | LR: 2.53e-06 | 5151.3ms/step | 11.6 samples/s
+Step 48220/50462 | Loss: 2.4900 | LR: 2.51e-06 | 5145.2ms/step | 11.7 samples/s
+Step 48230/50462 | Loss: 2.4973 | LR: 2.48e-06 | 5148.1ms/step | 11.7 samples/s
+Step 48240/50462 | Loss: 2.4952 | LR: 2.46e-06 | 5224.1ms/step | 11.5 samples/s
+Step 48250/50462 | Loss: 2.5342 | LR: 2.44e-06 | 5171.5ms/step | 11.6 samples/s
+Validation at step 48250: Loss = 2.5121, Perplexity = 12.33
+Step 48260/50462 | Loss: 2.4716 | LR: 2.42e-06 | 7717.7ms/step | 7.8 samples/s
+Step 48270/50462 | Loss: 2.4975 | LR: 2.39e-06 | 5150.2ms/step | 11.6 samples/s
+Step 48280/50462 | Loss: 2.5058 | LR: 2.37e-06 | 5152.5ms/step | 11.6 samples/s
+Step 48290/50462 | Loss: 2.4933 | LR: 2.35e-06 | 5161.0ms/step | 11.6 samples/s
+Step 48300/50462 | Loss: 2.4867 | LR: 2.33e-06 | 5275.4ms/step | 11.4 samples/s

epfl_final/log_files/cont_pretrain_8k.log

@@ -0,0 +1,325 @@
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000",
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 8k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260513_164613-40cb81kv
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense continued 8k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/40cb81kv
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000...
+Warning: no RNG state file found for rank 0; RNG will not be restored.
+Resumed from checkpoint at step 7999 (epoch 0, micro_step 0)
+Resuming training from step 7999
+Starting training...
+Warning: no loader_gen_state file found; DataLoader shuffle order on resume may differ from original run.
+Step 8000/20005 | Loss: 0.3005 | LR: 3.33e-04 | 60738.9ms/step | 0.4 samples/s
+Validation at step 8000: Loss = 3.0463, Perplexity = 21.04
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000
+Step 8010/20005 | Loss: 3.0209 | LR: 3.33e-04 | 43173.7ms/step | 0.6 samples/s
+Step 8020/20005 | Loss: 3.0490 | LR: 3.32e-04 | 5895.2ms/step | 4.1 samples/s
+Step 8030/20005 | Loss: 3.0187 | LR: 3.32e-04 | 5845.3ms/step | 4.1 samples/s
+Step 8040/20005 | Loss: 3.0369 | LR: 3.31e-04 | 5844.8ms/step | 4.1 samples/s
+Step 8050/20005 | Loss: 3.0430 | LR: 3.31e-04 | 5844.2ms/step | 4.1 samples/s
+Step 8060/20005 | Loss: 3.0399 | LR: 3.31e-04 | 5844.5ms/step | 4.1 samples/s
+Step 8070/20005 | Loss: 3.0496 | LR: 3.30e-04 | 5845.8ms/step | 4.1 samples/s
+Step 8080/20005 | Loss: 2.9954 | LR: 3.30e-04 | 5844.4ms/step | 4.1 samples/s
+Step 8090/20005 | Loss: 3.0354 | LR: 3.30e-04 | 5846.4ms/step | 4.1 samples/s
+Step 8100/20005 | Loss: 3.0208 | LR: 3.29e-04 | 5849.0ms/step | 4.1 samples/s
+Step 8110/20005 | Loss: 3.0160 | LR: 3.29e-04 | 5844.7ms/step | 4.1 samples/s
+Step 8120/20005 | Loss: 3.0308 | LR: 3.28e-04 | 5849.4ms/step | 4.1 samples/s
+Step 8130/20005 | Loss: 3.0105 | LR: 3.28e-04 | 5851.7ms/step | 4.1 samples/s
+Step 8140/20005 | Loss: 3.0254 | LR: 3.28e-04 | 5846.8ms/step | 4.1 samples/s
+Step 8150/20005 | Loss: 3.0212 | LR: 3.27e-04 | 5848.3ms/step | 4.1 samples/s
+Step 8160/20005 | Loss: 2.9973 | LR: 3.27e-04 | 5845.1ms/step | 4.1 samples/s
+Step 8170/20005 | Loss: 2.9866 | LR: 3.27e-04 | 5844.8ms/step | 4.1 samples/s
+Step 8180/20005 | Loss: 3.0179 | LR: 3.26e-04 | 5844.1ms/step | 4.1 samples/s
+Step 8190/20005 | Loss: 3.0313 | LR: 3.26e-04 | 5845.2ms/step | 4.1 samples/s
+Step 8200/20005 | Loss: 3.0137 | LR: 3.25e-04 | 5844.7ms/step | 4.1 samples/s
+Step 8210/20005 | Loss: 3.0414 | LR: 3.25e-04 | 5844.0ms/step | 4.1 samples/s
+Step 8220/20005 | Loss: 3.0272 | LR: 3.25e-04 | 5846.0ms/step | 4.1 samples/s
+Step 8230/20005 | Loss: 3.0355 | LR: 3.24e-04 | 5844.2ms/step | 4.1 samples/s
+Step 8240/20005 | Loss: 3.0317 | LR: 3.24e-04 | 5846.5ms/step | 4.1 samples/s
+Step 8250/20005 | Loss: 2.9886 | LR: 3.24e-04 | 5846.2ms/step | 4.1 samples/s
+Validation at step 8250: Loss = 3.0269, Perplexity = 20.63
+Step 8260/20005 | Loss: 3.0043 | LR: 3.23e-04 | 40512.8ms/step | 0.6 samples/s
+Step 8270/20005 | Loss: 3.0274 | LR: 3.23e-04 | 5844.8ms/step | 4.1 samples/s
+Step 8280/20005 | Loss: 3.0327 | LR: 3.22e-04 | 5845.8ms/step | 4.1 samples/s
+Step 8290/20005 | Loss: 3.0293 | LR: 3.22e-04 | 5848.5ms/step | 4.1 samples/s
+Step 8300/20005 | Loss: 3.0040 | LR: 3.22e-04 | 5846.7ms/step | 4.1 samples/s
+Step 8310/20005 | Loss: 3.0314 | LR: 3.21e-04 | 5845.0ms/step | 4.1 samples/s
+Step 8320/20005 | Loss: 3.0232 | LR: 3.21e-04 | 5846.6ms/step | 4.1 samples/s
+Step 8330/20005 | Loss: 3.0138 | LR: 3.20e-04 | 5849.4ms/step | 4.1 samples/s
+Step 8340/20005 | Loss: 2.9915 | LR: 3.20e-04 | 5850.4ms/step | 4.1 samples/s
+Step 8350/20005 | Loss: 3.0238 | LR: 3.20e-04 | 5873.6ms/step | 4.1 samples/s
+Step 8360/20005 | Loss: 3.0174 | LR: 3.19e-04 | 5877.6ms/step | 4.1 samples/s
+Step 8370/20005 | Loss: 3.0191 | LR: 3.19e-04 | 5881.5ms/step | 4.1 samples/s
+Step 8380/20005 | Loss: 3.0175 | LR: 3.19e-04 | 5886.7ms/step | 4.1 samples/s
+Step 8390/20005 | Loss: 2.9975 | LR: 3.18e-04 | 5889.4ms/step | 4.1 samples/s
+Step 8400/20005 | Loss: 3.0001 | LR: 3.18e-04 | 5890.0ms/step | 4.1 samples/s
+Step 8410/20005 | Loss: 3.0340 | LR: 3.17e-04 | 5891.4ms/step | 4.1 samples/s
+Step 8420/20005 | Loss: 2.9922 | LR: 3.17e-04 | 5892.5ms/step | 4.1 samples/s
+Step 8430/20005 | Loss: 2.9943 | LR: 3.17e-04 | 5896.1ms/step | 4.1 samples/s
+Step 8440/20005 | Loss: 2.9912 | LR: 3.16e-04 | 5888.7ms/step | 4.1 samples/s
+Step 8450/20005 | Loss: 3.0143 | LR: 3.16e-04 | 5892.4ms/step | 4.1 samples/s
+Step 8460/20005 | Loss: 3.0145 | LR: 3.15e-04 | 5890.0ms/step | 4.1 samples/s
+Step 8470/20005 | Loss: 3.0116 | LR: 3.15e-04 | 5887.8ms/step | 4.1 samples/s
+Step 8480/20005 | Loss: 2.9898 | LR: 3.15e-04 | 5894.9ms/step | 4.1 samples/s
+Step 8490/20005 | Loss: 3.0156 | LR: 3.14e-04 | 5895.3ms/step | 4.1 samples/s
+Step 8500/20005 | Loss: 3.0061 | LR: 3.14e-04 | 5891.2ms/step | 4.1 samples/s
+Validation at step 8500: Loss = 3.0173, Perplexity = 20.44
+Step 8510/20005 | Loss: 3.0116 | LR: 3.14e-04 | 40904.0ms/step | 0.6 samples/s
+Step 8520/20005 | Loss: 2.9885 | LR: 3.13e-04 | 5926.6ms/step | 4.0 samples/s
+Step 8530/20005 | Loss: 2.9842 | LR: 3.13e-04 | 5884.8ms/step | 4.1 samples/s
+Step 8540/20005 | Loss: 2.9798 | LR: 3.12e-04 | 5889.1ms/step | 4.1 samples/s
+Step 8550/20005 | Loss: 3.0528 | LR: 3.12e-04 | 5889.0ms/step | 4.1 samples/s
+Step 8560/20005 | Loss: 3.0133 | LR: 3.12e-04 | 5890.7ms/step | 4.1 samples/s
+Step 8570/20005 | Loss: 3.0188 | LR: 3.11e-04 | 5886.1ms/step | 4.1 samples/s
+Step 8580/20005 | Loss: 2.9773 | LR: 3.11e-04 | 5889.0ms/step | 4.1 samples/s
+Step 8590/20005 | Loss: 2.9922 | LR: 3.10e-04 | 5892.9ms/step | 4.1 samples/s
+Step 8600/20005 | Loss: 3.0110 | LR: 3.10e-04 | 5891.2ms/step | 4.1 samples/s
+Step 8610/20005 | Loss: 3.0045 | LR: 3.10e-04 | 5892.3ms/step | 4.1 samples/s
+Step 8620/20005 | Loss: 3.0010 | LR: 3.09e-04 | 5896.6ms/step | 4.1 samples/s
+Step 8630/20005 | Loss: 3.0084 | LR: 3.09e-04 | 5891.8ms/step | 4.1 samples/s
+Step 8640/20005 | Loss: 3.0234 | LR: 3.09e-04 | 5885.1ms/step | 4.1 samples/s
+Step 8650/20005 | Loss: 3.0314 | LR: 3.08e-04 | 5891.4ms/step | 4.1 samples/s
+Step 8660/20005 | Loss: 2.9966 | LR: 3.08e-04 | 5881.4ms/step | 4.1 samples/s
+Step 8670/20005 | Loss: 2.9927 | LR: 3.07e-04 | 5885.8ms/step | 4.1 samples/s
+Step 8680/20005 | Loss: 2.9941 | LR: 3.07e-04 | 5893.6ms/step | 4.1 samples/s
+Step 8690/20005 | Loss: 2.9849 | LR: 3.07e-04 | 5890.2ms/step | 4.1 samples/s
+Step 8700/20005 | Loss: 2.9973 | LR: 3.06e-04 | 5872.2ms/step | 4.1 samples/s
+Step 8710/20005 | Loss: 2.9828 | LR: 3.06e-04 | 5873.4ms/step | 4.1 samples/s
+Step 8720/20005 | Loss: 2.9969 | LR: 3.05e-04 | 5872.7ms/step | 4.1 samples/s
+Step 8730/20005 | Loss: 2.9769 | LR: 3.05e-04 | 5878.6ms/step | 4.1 samples/s
+Step 8740/20005 | Loss: 2.9891 | LR: 3.05e-04 | 5876.6ms/step | 4.1 samples/s
+Step 8750/20005 | Loss: 2.9813 | LR: 3.04e-04 | 5881.4ms/step | 4.1 samples/s
+Validation at step 8750: Loss = 3.0081, Perplexity = 20.25
+Step 8760/20005 | Loss: 2.9837 | LR: 3.04e-04 | 40797.1ms/step | 0.6 samples/s
+Step 8770/20005 | Loss: 3.0124 | LR: 3.04e-04 | 5866.5ms/step | 4.1 samples/s
+Step 8780/20005 | Loss: 3.0050 | LR: 3.03e-04 | 5867.2ms/step | 4.1 samples/s
+Step 8790/20005 | Loss: 2.9934 | LR: 3.03e-04 | 5865.9ms/step | 4.1 samples/s
+Step 8800/20005 | Loss: 2.9902 | LR: 3.02e-04 | 5868.6ms/step | 4.1 samples/s
+Step 8810/20005 | Loss: 2.9924 | LR: 3.02e-04 | 5866.2ms/step | 4.1 samples/s
+Step 8820/20005 | Loss: 2.9850 | LR: 3.02e-04 | 5866.9ms/step | 4.1 samples/s
+Step 8830/20005 | Loss: 2.9816 | LR: 3.01e-04 | 5864.8ms/step | 4.1 samples/s
+Step 8840/20005 | Loss: 2.9830 | LR: 3.01e-04 | 5866.6ms/step | 4.1 samples/s
+Step 8850/20005 | Loss: 2.9927 | LR: 3.00e-04 | 5869.9ms/step | 4.1 samples/s
+Step 8860/20005 | Loss: 2.9707 | LR: 3.00e-04 | 5868.1ms/step | 4.1 samples/s
+Step 8870/20005 | Loss: 3.0131 | LR: 3.00e-04 | 5869.0ms/step | 4.1 samples/s
+Step 8880/20005 | Loss: 2.9928 | LR: 2.99e-04 | 5867.9ms/step | 4.1 samples/s
+Step 8890/20005 | Loss: 2.9964 | LR: 2.99e-04 | 5870.2ms/step | 4.1 samples/s
+Step 8900/20005 | Loss: 2.9714 | LR: 2.98e-04 | 5865.4ms/step | 4.1 samples/s
+Step 8910/20005 | Loss: 3.0026 | LR: 2.98e-04 | 5866.9ms/step | 4.1 samples/s
+Step 8920/20005 | Loss: 2.9990 | LR: 2.98e-04 | 5865.3ms/step | 4.1 samples/s
+Step 8930/20005 | Loss: 2.9676 | LR: 2.97e-04 | 5866.0ms/step | 4.1 samples/s
+Step 8940/20005 | Loss: 2.9855 | LR: 2.97e-04 | 5864.0ms/step | 4.1 samples/s
+Step 8950/20005 | Loss: 2.9960 | LR: 2.97e-04 | 5866.3ms/step | 4.1 samples/s
+Step 8960/20005 | Loss: 3.0052 | LR: 2.96e-04 | 5866.2ms/step | 4.1 samples/s
+Step 8970/20005 | Loss: 3.0131 | LR: 2.96e-04 | 5867.7ms/step | 4.1 samples/s
+Step 8980/20005 | Loss: 3.0088 | LR: 2.95e-04 | 5862.7ms/step | 4.1 samples/s
+Step 8990/20005 | Loss: 2.9720 | LR: 2.95e-04 | 5861.5ms/step | 4.1 samples/s
+Step 9000/20005 | Loss: 2.9688 | LR: 2.95e-04 | 5865.9ms/step | 4.1 samples/s
+Validation at step 9000: Loss = 3.0002, Perplexity = 20.09
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-9000
+Step 9010/20005 | Loss: 3.0051 | LR: 2.94e-04 | 41291.0ms/step | 0.6 samples/s
+Step 9020/20005 | Loss: 2.9831 | LR: 2.94e-04 | 5909.6ms/step | 4.1 samples/s
+Step 9030/20005 | Loss: 2.9727 | LR: 2.93e-04 | 5873.1ms/step | 4.1 samples/s
+Step 9040/20005 | Loss: 2.9759 | LR: 2.93e-04 | 5877.0ms/step | 4.1 samples/s
+Step 9050/20005 | Loss: 2.9739 | LR: 2.93e-04 | 5874.8ms/step | 4.1 samples/s
+Step 9060/20005 | Loss: 3.0044 | LR: 2.92e-04 | 5868.5ms/step | 4.1 samples/s
+Step 9070/20005 | Loss: 2.9791 | LR: 2.92e-04 | 5869.0ms/step | 4.1 samples/s
+Step 9080/20005 | Loss: 3.0096 | LR: 2.91e-04 | 5868.7ms/step | 4.1 samples/s
+Step 9090/20005 | Loss: 2.9563 | LR: 2.91e-04 | 5870.9ms/step | 4.1 samples/s
+Step 9100/20005 | Loss: 3.0059 | LR: 2.91e-04 | 5871.2ms/step | 4.1 samples/s
+Step 9110/20005 | Loss: 3.0081 | LR: 2.90e-04 | 5875.1ms/step | 4.1 samples/s
+Step 9120/20005 | Loss: 2.9921 | LR: 2.90e-04 | 5876.5ms/step | 4.1 samples/s
+Step 9130/20005 | Loss: 2.9977 | LR: 2.89e-04 | 5876.8ms/step | 4.1 samples/s
+Step 9140/20005 | Loss: 2.9906 | LR: 2.89e-04 | 5872.4ms/step | 4.1 samples/s
+Step 9150/20005 | Loss: 2.9799 | LR: 2.89e-04 | 5871.3ms/step | 4.1 samples/s
+Step 9160/20005 | Loss: 2.9954 | LR: 2.88e-04 | 5874.4ms/step | 4.1 samples/s
+Step 9170/20005 | Loss: 2.9937 | LR: 2.88e-04 | 5873.1ms/step | 4.1 samples/s
+Step 9180/20005 | Loss: 2.9965 | LR: 2.87e-04 | 5873.8ms/step | 4.1 samples/s
+Step 9190/20005 | Loss: 2.9725 | LR: 2.87e-04 | 5873.2ms/step | 4.1 samples/s
+Step 9200/20005 | Loss: 2.9793 | LR: 2.87e-04 | 5871.9ms/step | 4.1 samples/s
+Step 9210/20005 | Loss: 2.9903 | LR: 2.86e-04 | 5870.8ms/step | 4.1 samples/s
+Step 9220/20005 | Loss: 2.9914 | LR: 2.86e-04 | 5874.5ms/step | 4.1 samples/s
+Step 9230/20005 | Loss: 3.0041 | LR: 2.86e-04 | 5877.4ms/step | 4.1 samples/s
+Step 9240/20005 | Loss: 2.9725 | LR: 2.85e-04 | 5876.6ms/step | 4.1 samples/s
+Step 9250/20005 | Loss: 2.9899 | LR: 2.85e-04 | 5873.9ms/step | 4.1 samples/s
+Validation at step 9250: Loss = 2.9921, Perplexity = 19.93
+Step 9260/20005 | Loss: 2.9607 | LR: 2.84e-04 | 40847.6ms/step | 0.6 samples/s
+Step 9270/20005 | Loss: 2.9754 | LR: 2.84e-04 | 5880.0ms/step | 4.1 samples/s
+Step 9280/20005 | Loss: 2.9653 | LR: 2.84e-04 | 5879.0ms/step | 4.1 samples/s
+Step 9290/20005 | Loss: 2.9909 | LR: 2.83e-04 | 5880.4ms/step | 4.1 samples/s
+Step 9300/20005 | Loss: 2.9755 | LR: 2.83e-04 | 5877.0ms/step | 4.1 samples/s
+Step 9310/20005 | Loss: 2.9887 | LR: 2.82e-04 | 5877.2ms/step | 4.1 samples/s
+Step 9320/20005 | Loss: 3.0098 | LR: 2.82e-04 | 5876.3ms/step | 4.1 samples/s
+Step 9330/20005 | Loss: 2.9954 | LR: 2.82e-04 | 5875.9ms/step | 4.1 samples/s
+Step 9340/20005 | Loss: 2.9673 | LR: 2.81e-04 | 5874.9ms/step | 4.1 samples/s
+Step 9350/20005 | Loss: 2.9928 | LR: 2.81e-04 | 5874.9ms/step | 4.1 samples/s
+Step 9360/20005 | Loss: 2.9668 | LR: 2.80e-04 | 5875.5ms/step | 4.1 samples/s
+Step 9370/20005 | Loss: 2.9698 | LR: 2.80e-04 | 5874.3ms/step | 4.1 samples/s
+Step 9380/20005 | Loss: 2.9749 | LR: 2.80e-04 | 5875.7ms/step | 4.1 samples/s
+Step 9390/20005 | Loss: 2.9713 | LR: 2.79e-04 | 5875.9ms/step | 4.1 samples/s
+Step 9400/20005 | Loss: 2.9855 | LR: 2.79e-04 | 5878.1ms/step | 4.1 samples/s
+Step 9410/20005 | Loss: 2.9873 | LR: 2.78e-04 | 5874.1ms/step | 4.1 samples/s
+Step 9420/20005 | Loss: 2.9552 | LR: 2.78e-04 | 5876.0ms/step | 4.1 samples/s
+Step 9430/20005 | Loss: 3.0031 | LR: 2.78e-04 | 5875.3ms/step | 4.1 samples/s
+Step 9440/20005 | Loss: 3.0040 | LR: 2.77e-04 | 5874.4ms/step | 4.1 samples/s
+Step 9450/20005 | Loss: 2.9816 | LR: 2.77e-04 | 5876.0ms/step | 4.1 samples/s
+Step 9460/20005 | Loss: 2.9469 | LR: 2.76e-04 | 5875.0ms/step | 4.1 samples/s
+Step 9470/20005 | Loss: 2.9661 | LR: 2.76e-04 | 5877.1ms/step | 4.1 samples/s
+Step 9480/20005 | Loss: 2.9778 | LR: 2.76e-04 | 5876.0ms/step | 4.1 samples/s
+Step 9490/20005 | Loss: 2.9720 | LR: 2.75e-04 | 5877.4ms/step | 4.1 samples/s
+Step 9500/20005 | Loss: 2.9646 | LR: 2.75e-04 | 5874.0ms/step | 4.1 samples/s
+Validation at step 9500: Loss = 2.9845, Perplexity = 19.78
+Step 9510/20005 | Loss: 2.9689 | LR: 2.74e-04 | 40859.1ms/step | 0.6 samples/s
+Step 9520/20005 | Loss: 2.9505 | LR: 2.74e-04 | 5914.9ms/step | 4.1 samples/s
+Step 9530/20005 | Loss: 2.9465 | LR: 2.74e-04 | 5875.1ms/step | 4.1 samples/s
+Step 9540/20005 | Loss: 2.9960 | LR: 2.73e-04 | 5875.3ms/step | 4.1 samples/s

epfl_final/log_files/cont_pretrain_diff_lr.log

@@ -0,0 +1,364 @@
+nohup: ignoring input
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+New run: Checkpoints will be saved to ./checkpoints/run_20260530_032950
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260530_032950",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000",
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0009,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 46k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000
+Checkpoint activations: False
+Checkpoint activations: False
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 1536]           [2, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [2, 8192, 1536]           [2, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (G): 2.87
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 52437.19
+Params size (MB): 2871.45
+Estimated Total Size (MB): 55308.77
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        3
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504627
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Overriding checkpoint LR with config LR: 0.0009
+Resumed from checkpoint at step 46000 (epoch 0, micro_step 460000)
+Resuming training from step 46000
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.27.0
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260530_034853-10196kbl
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense continued 46k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/10196kbl
+Starting training...
+Validation at step 46000: Loss = 2.5221, Perplexity = 12.46
+Step 46010/50462 | Loss: 2.8364 | LR: 1.77e-05 | 56608.8ms/step | 1.1 samples/s
+Step 46020/50462 | Loss: 2.6596 | LR: 1.76e-05 | 5187.9ms/step | 11.6 samples/s
+Step 46030/50462 | Loss: 2.6007 | LR: 1.75e-05 | 5205.0ms/step | 11.5 samples/s
+Step 46040/50462 | Loss: 2.5616 | LR: 1.75e-05 | 5240.7ms/step | 11.4 samples/s
+Step 46050/50462 | Loss: 2.5284 | LR: 1.74e-05 | 5239.8ms/step | 11.5 samples/s
+Step 46060/50462 | Loss: 2.5156 | LR: 1.73e-05 | 5244.0ms/step | 11.4 samples/s
+Step 46070/50462 | Loss: 2.5310 | LR: 1.72e-05 | 5219.4ms/step | 11.5 samples/s
+Step 46080/50462 | Loss: 2.5231 | LR: 1.71e-05 | 5235.6ms/step | 11.5 samples/s
+Step 46090/50462 | Loss: 2.4899 | LR: 1.71e-05 | 5236.2ms/step | 11.5 samples/s
+Step 46100/50462 | Loss: 2.5151 | LR: 1.70e-05 | 5255.0ms/step | 11.4 samples/s
+Step 46110/50462 | Loss: 2.5095 | LR: 1.69e-05 | 5254.0ms/step | 11.4 samples/s
+Step 46120/50462 | Loss: 2.5042 | LR: 1.68e-05 | 5246.2ms/step | 11.4 samples/s
+Step 46130/50462 | Loss: 2.5049 | LR: 1.68e-05 | 5250.9ms/step | 11.4 samples/s
+Step 46140/50462 | Loss: 2.5045 | LR: 1.67e-05 | 5239.2ms/step | 11.5 samples/s
+Step 46150/50462 | Loss: 2.5024 | LR: 1.66e-05 | 5238.1ms/step | 11.5 samples/s
+Step 46160/50462 | Loss: 2.5194 | LR: 1.65e-05 | 5241.8ms/step | 11.4 samples/s
+Step 46170/50462 | Loss: 2.5199 | LR: 1.65e-05 | 5240.5ms/step | 11.4 samples/s
+Step 46180/50462 | Loss: 2.4892 | LR: 1.64e-05 | 5284.6ms/step | 11.4 samples/s
+Step 46190/50462 | Loss: 2.4895 | LR: 1.63e-05 | 5285.2ms/step | 11.4 samples/s
+Step 46200/50462 | Loss: 2.4967 | LR: 1.62e-05 | 5258.0ms/step | 11.4 samples/s
+Step 46210/50462 | Loss: 2.5132 | LR: 1.61e-05 | 5253.4ms/step | 11.4 samples/s
+Step 46220/50462 | Loss: 2.5161 | LR: 1.61e-05 | 5265.6ms/step | 11.4 samples/s
+Step 46230/50462 | Loss: 2.5219 | LR: 1.60e-05 | 5297.7ms/step | 11.3 samples/s
+Step 46240/50462 | Loss: 2.5304 | LR: 1.59e-05 | 5229.3ms/step | 11.5 samples/s
+Step 46250/50462 | Loss: 2.5282 | LR: 1.58e-05 | 5252.8ms/step | 11.4 samples/s
+Validation at step 46250: Loss = 2.5195, Perplexity = 12.42
+Step 46260/50462 | Loss: 2.5277 | LR: 1.58e-05 | 7824.6ms/step | 7.7 samples/s
+Step 46270/50462 | Loss: 2.5143 | LR: 1.57e-05 | 5250.4ms/step | 11.4 samples/s
+Step 46280/50462 | Loss: 2.5458 | LR: 1.56e-05 | 5252.6ms/step | 11.4 samples/s
+Step 46290/50462 | Loss: 2.5324 | LR: 1.56e-05 | 5248.0ms/step | 11.4 samples/s
+Step 46300/50462 | Loss: 2.5027 | LR: 1.55e-05 | 5247.2ms/step | 11.4 samples/s
+Step 46310/50462 | Loss: 2.5101 | LR: 1.54e-05 | 5244.6ms/step | 11.4 samples/s
+Step 46320/50462 | Loss: 2.5358 | LR: 1.53e-05 | 5243.2ms/step | 11.4 samples/s
+Step 46330/50462 | Loss: 2.5152 | LR: 1.53e-05 | 5250.3ms/step | 11.4 samples/s
+Step 46340/50462 | Loss: 2.5148 | LR: 1.52e-05 | 5236.8ms/step | 11.5 samples/s
+Step 46350/50462 | Loss: 2.5356 | LR: 1.51e-05 | 5245.1ms/step | 11.4 samples/s
+Step 46360/50462 | Loss: 2.4985 | LR: 1.50e-05 | 5249.4ms/step | 11.4 samples/s
+Step 46370/50462 | Loss: 2.4840 | LR: 1.50e-05 | 5249.6ms/step | 11.4 samples/s
+Step 46380/50462 | Loss: 2.5088 | LR: 1.49e-05 | 5245.3ms/step | 11.4 samples/s
+Step 46390/50462 | Loss: 2.4979 | LR: 1.48e-05 | 5240.0ms/step | 11.5 samples/s
+Step 46400/50462 | Loss: 2.4894 | LR: 1.47e-05 | 5238.0ms/step | 11.5 samples/s
+Step 46410/50462 | Loss: 2.5021 | LR: 1.47e-05 | 5247.4ms/step | 11.4 samples/s
+Step 46420/50462 | Loss: 2.5016 | LR: 1.46e-05 | 5245.3ms/step | 11.4 samples/s
+Step 46430/50462 | Loss: 2.5224 | LR: 1.45e-05 | 5241.4ms/step | 11.4 samples/s
+Step 46440/50462 | Loss: 2.5048 | LR: 1.45e-05 | 5243.5ms/step | 11.4 samples/s
+Step 46450/50462 | Loss: 2.5308 | LR: 1.44e-05 | 5319.2ms/step | 11.3 samples/s
+Step 46460/50462 | Loss: 2.5089 | LR: 1.43e-05 | 5251.3ms/step | 11.4 samples/s
+Step 46470/50462 | Loss: 2.5002 | LR: 1.42e-05 | 5250.7ms/step | 11.4 samples/s
+Step 46480/50462 | Loss: 2.4954 | LR: 1.42e-05 | 5248.2ms/step | 11.4 samples/s
+Step 46490/50462 | Loss: 2.4914 | LR: 1.41e-05 | 5243.4ms/step | 11.4 samples/s
+Step 46500/50462 | Loss: 2.5083 | LR: 1.40e-05 | 5276.5ms/step | 11.4 samples/s
+Validation at step 46500: Loss = 2.5178, Perplexity = 12.40
+Step 46510/50462 | Loss: 2.5026 | LR: 1.40e-05 | 7814.7ms/step | 7.7 samples/s
+Step 46520/50462 | Loss: 2.5260 | LR: 1.39e-05 | 5246.6ms/step | 11.4 samples/s
+Step 46530/50462 | Loss: 2.5060 | LR: 1.38e-05 | 5254.3ms/step | 11.4 samples/s
+Step 46540/50462 | Loss: 2.5187 | LR: 1.38e-05 | 5255.4ms/step | 11.4 samples/s
+Step 46550/50462 | Loss: 2.4805 | LR: 1.37e-05 | 5237.3ms/step | 11.5 samples/s
+Step 46560/50462 | Loss: 2.4973 | LR: 1.36e-05 | 5247.2ms/step | 11.4 samples/s
+Step 46570/50462 | Loss: 2.5079 | LR: 1.35e-05 | 5241.1ms/step | 11.4 samples/s
+Step 46580/50462 | Loss: 2.4997 | LR: 1.35e-05 | 5234.1ms/step | 11.5 samples/s
+Step 46590/50462 | Loss: 2.4922 | LR: 1.34e-05 | 5245.1ms/step | 11.4 samples/s
+Step 46600/50462 | Loss: 2.5074 | LR: 1.33e-05 | 5249.5ms/step | 11.4 samples/s
+Step 46610/50462 | Loss: 2.5054 | LR: 1.33e-05 | 5244.4ms/step | 11.4 samples/s
+Step 46620/50462 | Loss: 2.4882 | LR: 1.32e-05 | 5242.8ms/step | 11.4 samples/s
+Step 46630/50462 | Loss: 2.5024 | LR: 1.31e-05 | 5253.4ms/step | 11.4 samples/s
+Step 46640/50462 | Loss: 2.5145 | LR: 1.31e-05 | 5244.6ms/step | 11.4 samples/s
+Step 46650/50462 | Loss: 2.5065 | LR: 1.30e-05 | 5250.5ms/step | 11.4 samples/s
+Step 46660/50462 | Loss: 2.4965 | LR: 1.29e-05 | 5254.2ms/step | 11.4 samples/s
+Step 46670/50462 | Loss: 2.5113 | LR: 1.29e-05 | 5250.0ms/step | 11.4 samples/s
+Step 46680/50462 | Loss: 2.5199 | LR: 1.28e-05 | 5246.2ms/step | 11.4 samples/s
+Step 46690/50462 | Loss: 2.5366 | LR: 1.27e-05 | 5248.3ms/step | 11.4 samples/s
+Step 46700/50462 | Loss: 2.4968 | LR: 1.27e-05 | 5249.6ms/step | 11.4 samples/s
+Step 46710/50462 | Loss: 2.5298 | LR: 1.26e-05 | 5273.4ms/step | 11.4 samples/s
+Step 46720/50462 | Loss: 2.5134 | LR: 1.25e-05 | 5279.3ms/step | 11.4 samples/s
+Step 46730/50462 | Loss: 2.4932 | LR: 1.25e-05 | 5237.0ms/step | 11.5 samples/s
+Step 46740/50462 | Loss: 2.5198 | LR: 1.24e-05 | 5227.1ms/step | 11.5 samples/s
+Step 46750/50462 | Loss: 2.5026 | LR: 1.23e-05 | 5243.4ms/step | 11.4 samples/s
+Validation at step 46750: Loss = 2.5172, Perplexity = 12.39
+Step 46760/50462 | Loss: 2.5171 | LR: 1.23e-05 | 7815.1ms/step | 7.7 samples/s
+Step 46770/50462 | Loss: 2.5067 | LR: 1.22e-05 | 5292.7ms/step | 11.3 samples/s
+Step 46780/50462 | Loss: 2.5321 | LR: 1.21e-05 | 5237.2ms/step | 11.5 samples/s
+Step 46790/50462 | Loss: 2.4894 | LR: 1.21e-05 | 5240.5ms/step | 11.4 samples/s
+Step 46800/50462 | Loss: 2.5274 | LR: 1.20e-05 | 5245.3ms/step | 11.4 samples/s
+Step 46810/50462 | Loss: 2.5233 | LR: 1.19e-05 | 5250.8ms/step | 11.4 samples/s
+Step 46820/50462 | Loss: 2.4867 | LR: 1.19e-05 | 5246.2ms/step | 11.4 samples/s
+Step 46830/50462 | Loss: 2.4849 | LR: 1.18e-05 | 5246.5ms/step | 11.4 samples/s
+Step 46840/50462 | Loss: 2.5104 | LR: 1.17e-05 | 5244.4ms/step | 11.4 samples/s
+Step 46850/50462 | Loss: 2.5064 | LR: 1.17e-05 | 5237.2ms/step | 11.5 samples/s
+Step 46860/50462 | Loss: 2.5059 | LR: 1.16e-05 | 5243.0ms/step | 11.4 samples/s
+Step 46870/50462 | Loss: 2.4642 | LR: 1.15e-05 | 5248.1ms/step | 11.4 samples/s
+Step 46880/50462 | Loss: 2.4996 | LR: 1.15e-05 | 5249.1ms/step | 11.4 samples/s
+Step 46890/50462 | Loss: 2.5057 | LR: 1.14e-05 | 5249.4ms/step | 11.4 samples/s
+Step 46900/50462 | Loss: 2.4978 | LR: 1.14e-05 | 5254.3ms/step | 11.4 samples/s
+Step 46910/50462 | Loss: 2.4909 | LR: 1.13e-05 | 5249.4ms/step | 11.4 samples/s
+Step 46920/50462 | Loss: 2.5288 | LR: 1.12e-05 | 5249.4ms/step | 11.4 samples/s
+Step 46930/50462 | Loss: 2.5051 | LR: 1.12e-05 | 5250.7ms/step | 11.4 samples/s
+Step 46940/50462 | Loss: 2.5071 | LR: 1.11e-05 | 5240.0ms/step | 11.5 samples/s
+Step 46950/50462 | Loss: 2.4901 | LR: 1.10e-05 | 5248.1ms/step | 11.4 samples/s
+Step 46960/50462 | Loss: 2.5134 | LR: 1.10e-05 | 5287.1ms/step | 11.3 samples/s
+Step 46970/50462 | Loss: 2.5171 | LR: 1.09e-05 | 5229.2ms/step | 11.5 samples/s
+Step 46980/50462 | Loss: 2.5163 | LR: 1.09e-05 | 5229.3ms/step | 11.5 samples/s
+Step 46990/50462 | Loss: 2.5371 | LR: 1.08e-05 | 5270.8ms/step | 11.4 samples/s
+Step 47000/50462 | Loss: 2.5167 | LR: 1.07e-05 | 5239.0ms/step | 11.5 samples/s
+Validation at step 47000: Loss = 2.5168, Perplexity = 12.39
+Saved checkpoint to checkpoints/run_20260530_032950/checkpoint-47000
+Step 47010/50462 | Loss: 2.5287 | LR: 1.07e-05 | 8566.5ms/step | 7.0 samples/s
+Step 47020/50462 | Loss: 2.5053 | LR: 1.06e-05 | 5215.9ms/step | 11.5 samples/s
+Step 47030/50462 | Loss: 2.5252 | LR: 1.05e-05 | 5225.3ms/step | 11.5 samples/s
+Step 47040/50462 | Loss: 2.4702 | LR: 1.05e-05 | 5266.7ms/step | 11.4 samples/s
+Step 47050/50462 | Loss: 2.5070 | LR: 1.04e-05 | 5235.6ms/step | 11.5 samples/s
+Step 47060/50462 | Loss: 2.5053 | LR: 1.04e-05 | 5236.8ms/step | 11.5 samples/s
+Step 47070/50462 | Loss: 2.4982 | LR: 1.03e-05 | 5239.0ms/step | 11.5 samples/s
+Step 47080/50462 | Loss: 2.5060 | LR: 1.02e-05 | 5227.9ms/step | 11.5 samples/s
+Step 47090/50462 | Loss: 2.5009 | LR: 1.02e-05 | 5233.3ms/step | 11.5 samples/s
+Step 47100/50462 | Loss: 2.5048 | LR: 1.01e-05 | 5237.2ms/step | 11.5 samples/s
+Step 47110/50462 | Loss: 2.5085 | LR: 1.01e-05 | 5237.8ms/step | 11.5 samples/s
+Step 47120/50462 | Loss: 2.4590 | LR: 1.00e-05 | 5242.8ms/step | 11.4 samples/s
+Step 47130/50462 | Loss: 2.5058 | LR: 9.94e-06 | 5245.2ms/step | 11.4 samples/s
+Step 47140/50462 | Loss: 2.5182 | LR: 9.88e-06 | 5239.9ms/step | 11.5 samples/s
+Step 47150/50462 | Loss: 2.4744 | LR: 9.82e-06 | 5240.0ms/step | 11.5 samples/s
+Step 47160/50462 | Loss: 2.5162 | LR: 9.76e-06 | 5237.6ms/step | 11.5 samples/s
+Step 47170/50462 | Loss: 2.4992 | LR: 9.70e-06 | 5234.2ms/step | 11.5 samples/s
+Step 47180/50462 | Loss: 2.5086 | LR: 9.64e-06 | 5250.4ms/step | 11.4 samples/s
+Step 47190/50462 | Loss: 2.5033 | LR: 9.59e-06 | 5250.6ms/step | 11.4 samples/s
+Step 47200/50462 | Loss: 2.5014 | LR: 9.53e-06 | 5235.9ms/step | 11.5 samples/s
+Step 47210/50462 | Loss: 2.4751 | LR: 9.47e-06 | 5239.2ms/step | 11.5 samples/s
+Step 47220/50462 | Loss: 2.5158 | LR: 9.41e-06 | 5266.9ms/step | 11.4 samples/s
+Step 47230/50462 | Loss: 2.5139 | LR: 9.35e-06 | 5243.6ms/step | 11.4 samples/s
+Step 47240/50462 | Loss: 2.4825 | LR: 9.30e-06 | 5242.1ms/step | 11.4 samples/s
+Step 47250/50462 | Loss: 2.5128 | LR: 9.24e-06 | 5241.4ms/step | 11.4 samples/s
+Validation at step 47250: Loss = 2.5166, Perplexity = 12.39
+Step 47260/50462 | Loss: 2.5073 | LR: 9.18e-06 | 7860.4ms/step | 7.6 samples/s
+Step 47270/50462 | Loss: 2.5191 | LR: 9.12e-06 | 5244.8ms/step | 11.4 samples/s
+Step 47280/50462 | Loss: 2.4928 | LR: 9.07e-06 | 5242.3ms/step | 11.4 samples/s
+Step 47290/50462 | Loss: 2.5207 | LR: 9.01e-06 | 5247.2ms/step | 11.4 samples/s
+Step 47300/50462 | Loss: 2.5191 | LR: 8.95e-06 | 5241.7ms/step | 11.4 samples/s
+Step 47310/50462 | Loss: 2.4810 | LR: 8.90e-06 | 5283.8ms/step | 11.4 samples/s
+Step 47320/50462 | Loss: 2.4981 | LR: 8.84e-06 | 5244.8ms/step | 11.4 samples/s
+Step 47330/50462 | Loss: 2.5013 | LR: 8.79e-06 | 5250.6ms/step | 11.4 samples/s
+Step 47340/50462 | Loss: 2.4662 | LR: 8.73e-06 | 5257.9ms/step | 11.4 samples/s
+Step 47350/50462 | Loss: 2.5137 | LR: 8.67e-06 | 5252.8ms/step | 11.4 samples/s
+Step 47360/50462 | Loss: 2.4842 | LR: 8.62e-06 | 5239.6ms/step | 11.5 samples/s
+Step 47370/50462 | Loss: 2.5223 | LR: 8.56e-06 | 5234.3ms/step | 11.5 samples/s
+Step 47380/50462 | Loss: 2.5149 | LR: 8.51e-06 | 5236.9ms/step | 11.5 samples/s
+Step 47390/50462 | Loss: 2.4977 | LR: 8.45e-06 | 5229.2ms/step | 11.5 samples/s
+Step 47400/50462 | Loss: 2.5006 | LR: 8.40e-06 | 5240.3ms/step | 11.4 samples/s
+Step 47410/50462 | Loss: 2.5119 | LR: 8.34e-06 | 5236.1ms/step | 11.5 samples/s
+Step 47420/50462 | Loss: 2.4639 | LR: 8.29e-06 | 5236.2ms/step | 11.5 samples/s
+Step 47430/50462 | Loss: 2.4916 | LR: 8.24e-06 | 5240.8ms/step | 11.4 samples/s
+Step 47440/50462 | Loss: 2.5003 | LR: 8.18e-06 | 5233.6ms/step | 11.5 samples/s
+Step 47450/50462 | Loss: 2.4752 | LR: 8.13e-06 | 5230.7ms/step | 11.5 samples/s
+Step 47460/50462 | Loss: 2.5199 | LR: 8.07e-06 | 5228.0ms/step | 11.5 samples/s
+Step 47470/50462 | Loss: 2.5176 | LR: 8.02e-06 | 5239.7ms/step | 11.5 samples/s
+Step 47480/50462 | Loss: 2.5164 | LR: 7.97e-06 | 5273.9ms/step | 11.4 samples/s
+Step 47490/50462 | Loss: 2.5091 | LR: 7.91e-06 | 5210.5ms/step | 11.5 samples/s
+Step 47500/50462 | Loss: 2.5005 | LR: 7.86e-06 | 5232.8ms/step | 11.5 samples/s
+Validation at step 47500: Loss = 2.5165, Perplexity = 12.39
+Step 47510/50462 | Loss: 2.4714 | LR: 7.81e-06 | 7812.7ms/step | 7.7 samples/s
+Step 47520/50462 | Loss: 2.5191 | LR: 7.76e-06 | 5273.9ms/step | 11.4 samples/s
+Step 47530/50462 | Loss: 2.5146 | LR: 7.70e-06 | 5236.1ms/step | 11.5 samples/s
+Step 47540/50462 | Loss: 2.4830 | LR: 7.65e-06 | 5239.3ms/step | 11.5 samples/s
+Step 47550/50462 | Loss: 2.5124 | LR: 7.60e-06 | 5247.5ms/step | 11.4 samples/s
+Step 47560/50462 | Loss: 2.4978 | LR: 7.55e-06 | 5230.9ms/step | 11.5 samples/s
+Step 47570/50462 | Loss: 2.5079 | LR: 7.49e-06 | 5235.6ms/step | 11.5 samples/s
+Step 47580/50462 | Loss: 2.5432 | LR: 7.44e-06 | 5269.0ms/step | 11.4 samples/s
+Step 47590/50462 | Loss: 2.5140 | LR: 7.39e-06 | 5233.3ms/step | 11.5 samples/s
+Step 47600/50462 | Loss: 2.5048 | LR: 7.34e-06 | 5242.6ms/step | 11.4 samples/s
+Step 47610/50462 | Loss: 2.4847 | LR: 7.29e-06 | 5245.5ms/step | 11.4 samples/s
+Step 47620/50462 | Loss: 2.4984 | LR: 7.24e-06 | 5246.2ms/step | 11.4 samples/s
+Step 47630/50462 | Loss: 2.5256 | LR: 7.19e-06 | 5236.9ms/step | 11.5 samples/s
+Step 47640/50462 | Loss: 2.4806 | LR: 7.14e-06 | 5242.7ms/step | 11.4 samples/s
+Step 47650/50462 | Loss: 2.5000 | LR: 7.09e-06 | 5250.1ms/step | 11.4 samples/s
+Step 47660/50462 | Loss: 2.4990 | LR: 7.04e-06 | 5236.7ms/step | 11.5 samples/s
+Step 47670/50462 | Loss: 2.4911 | LR: 6.99e-06 | 5233.8ms/step | 11.5 samples/s
+Step 47680/50462 | Loss: 2.5093 | LR: 6.94e-06 | 5241.7ms/step | 11.4 samples/s
+Step 47690/50462 | Loss: 2.5050 | LR: 6.89e-06 | 5245.5ms/step | 11.4 samples/s
+Step 47700/50462 | Loss: 2.5131 | LR: 6.84e-06 | 5251.1ms/step | 11.4 samples/s
+Step 47710/50462 | Loss: 2.4791 | LR: 6.79e-06 | 5245.5ms/step | 11.4 samples/s
+9ms/step | 11.4 samples/s
+Step 48030/50462 | Loss: 2.4860 | LR: 2.95e-06 | 5289.5ms/step | 11.3 samples/s
+Step 48040/50462 | Loss: 2.4994 | LR: 2.92e-06 | 5251.1ms/step | 11.4 samples/s
+Step 48050/50462 | Loss: 2.4927 | LR: 2.90e-06 | 5245.4ms/step | 11.4 samples/s
+Step 48060/50462 | Loss: 2.5093 | LR: 2.87e-06 | 5250.2ms/step | 11.4 samples/s
+Step 48070/50462 | Loss: 2.5118 | LR: 2.85e-06 | 5249.8ms/step | 11.4 samples/s

epfl_final/log_files/cont_pretrain_lr.log

@@ -0,0 +1,264 @@
+nohup: ignoring input
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+New run: Checkpoints will be saved to ./checkpoints/run_20260530_021500
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260530_021500",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-44000",
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 44k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: False
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-44000
+Checkpoint activations: False
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 1536]           [2, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [2, 8192, 1536]           [2, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (G): 2.87
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 52437.19
+Params size (MB): 2871.45
+Estimated Total Size (MB): 55308.77
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        3
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504627
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-44000...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Resumed from checkpoint at step 44000 (epoch 0, micro_step 440000)
+Resuming training from step 44000
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 408mcc4e
+wandb: Tracking run with wandb version 0.27.0
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260530_023351-408mcc4e
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense continued 44k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/408mcc4e
+Starting training...
+Validation at step 44000: Loss = 2.5224, Perplexity = 12.46
+Step 44010/50462 | Loss: 2.5521 | LR: 2.05e-05 | 59535.6ms/step | 1.0 samples/s
+Step 44020/50462 | Loss: 2.5189 | LR: 2.04e-05 | 5212.1ms/step | 11.5 samples/s
+Step 44030/50462 | Loss: 2.5073 | LR: 2.04e-05 | 5229.4ms/step | 11.5 samples/s
+Step 44040/50462 | Loss: 2.5005 | LR: 2.03e-05 | 5226.5ms/step | 11.5 samples/s
+Step 44050/50462 | Loss: 2.5321 | LR: 2.02e-05 | 5230.2ms/step | 11.5 samples/s
+Step 44060/50462 | Loss: 2.5302 | LR: 2.02e-05 | 5237.1ms/step | 11.5 samples/s
+Step 44070/50462 | Loss: 2.5293 | LR: 2.01e-05 | 5242.2ms/step | 11.4 samples/s
+Step 44080/50462 | Loss: 2.5117 | LR: 2.01e-05 | 5244.5ms/step | 11.4 samples/s
+Step 44090/50462 | Loss: 2.4805 | LR: 2.00e-05 | 5289.2ms/step | 11.3 samples/s
+Step 44100/50462 | Loss: 2.4990 | LR: 1.99e-05 | 5233.3ms/step | 11.5 samples/s
+Step 44110/50462 | Loss: 2.5007 | LR: 1.99e-05 | 5237.0ms/step | 11.5 samples/s
+Step 44120/50462 | Loss: 2.5171 | LR: 1.98e-05 | 5239.0ms/step | 11.5 samples/s
+Step 44130/50462 | Loss: 2.5227 | LR: 1.97e-05 | 5247.7ms/step | 11.4 samples/s
+Step 44140/50462 | Loss: 2.4908 | LR: 1.97e-05 | 5256.8ms/step | 11.4 samples/s
+Step 44150/50462 | Loss: 2.4862 | LR: 1.96e-05 | 5239.4ms/step | 11.5 samples/s
+Step 44160/50462 | Loss: 2.5162 | LR: 1.96e-05 | 5235.0ms/step | 11.5 samples/s
+Step 44170/50462 | Loss: 2.5198 | LR: 1.95e-05 | 5236.4ms/step | 11.5 samples/s
+Step 44180/50462 | Loss: 2.4864 | LR: 1.94e-05 | 5249.8ms/step | 11.4 samples/s
+Step 44190/50462 | Loss: 2.4929 | LR: 1.94e-05 | 5251.5ms/step | 11.4 samples/s
+Step 44200/50462 | Loss: 2.5140 | LR: 1.93e-05 | 5252.4ms/step | 11.4 samples/s
+Step 44210/50462 | Loss: 2.5021 | LR: 1.93e-05 | 5233.8ms/step | 11.5 samples/s
+Step 44220/50462 | Loss: 2.5080 | LR: 1.92e-05 | 5226.6ms/step | 11.5 samples/s
+Step 44230/50462 | Loss: 2.5198 | LR: 1.91e-05 | 5245.4ms/step | 11.4 samples/s
+Step 44240/50462 | Loss: 2.5071 | LR: 1.91e-05 | 5253.7ms/step | 11.4 samples/s
+Step 44250/50462 | Loss: 2.5205 | LR: 1.90e-05 | 5244.2ms/step | 11.4 samples/s
+Validation at step 44250: Loss = 2.5158, Perplexity = 12.38
+Step 44260/50462 | Loss: 2.5148 | LR: 1.90e-05 | 7809.6ms/step | 7.7 samples/s
+Step 44270/50462 | Loss: 2.4856 | LR: 1.89e-05 | 5244.5ms/step | 11.4 samples/s
+Step 44280/50462 | Loss: 2.5090 | LR: 1.88e-05 | 5285.8ms/step | 11.4 samples/s
+Step 44290/50462 | Loss: 2.5052 | LR: 1.88e-05 | 5234.3ms/step | 11.5 samples/s
+Step 44300/50462 | Loss: 2.4564 | LR: 1.87e-05 | 5231.1ms/step | 11.5 samples/s
+Step 44310/50462 | Loss: 2.4863 | LR: 1.87e-05 | 5198.5ms/step | 11.5 samples/s
+Step 44320/50462 | Loss: 2.5187 | LR: 1.86e-05 | 5228.2ms/step | 11.5 samples/s
+Step 44330/50462 | Loss: 2.5091 | LR: 1.85e-05 | 5238.5ms/step | 11.5 samples/s
+Step 44340/50462 | Loss: 2.5053 | LR: 1.85e-05 | 5242.7ms/step | 11.4 samples/s
+Step 44350/50462 | Loss: 2.5100 | LR: 1.84e-05 | 5237.8ms/step | 11.5 samples/s
+Step 44360/50462 | Loss: 2.4832 | LR: 1.84e-05 | 5225.6ms/step | 11.5 samples/s
+Step 44370/50462 | Loss: 2.5084 | LR: 1.83e-05 | 5218.8ms/step | 11.5 samples/s
+Step 44380/50462 | Loss: 2.4995 | LR: 1.82e-05 | 5294.0ms/step | 11.3 samples/s
+Step 44390/50462 | Loss: 2.5195 | LR: 1.82e-05 | 5229.2ms/step | 11.5 samples/s
+Step 44400/50462 | Loss: 2.4860 | LR: 1.81e-05 | 5236.8ms/step | 11.5 samples/s
+Step 44410/50462 | Loss: 2.4953 | LR: 1.81e-05 | 5246.0ms/step | 11.4 samples/s
+Step 44420/50462 | Loss: 2.5053 | LR: 1.80e-05 | 5232.4ms/step | 11.5 samples/s
+Step 44430/50462 | Loss: 2.5128 | LR: 1.79e-05 | 5233.2ms/step | 11.5 samples/s
+Step 44440/50462 | Loss: 2.5019 | LR: 1.79e-05 | 5246.3ms/step | 11.4 samples/s
+Step 44450/50462 | Loss: 2.4906 | LR: 1.78e-05 | 5247.1ms/step | 11.4 samples/s
+Step 44460/50462 | Loss: 2.5063 | LR: 1.78e-05 | 5244.6ms/step | 11.4 samples/s
+Step 44470/50462 | Loss: 2.5189 | LR: 1.77e-05 | 5233.5ms/step | 11.5 samples/s
+Step 44480/50462 | Loss: 2.5330 | LR: 1.77e-05 | 5201.1ms/step | 11.5 samples/s
+Step 44490/50462 | Loss: 2.5285 | LR: 1.76e-05 | 5230.3ms/step | 11.5 samples/s
+Step 44500/50462 | Loss: 2.4912 | LR: 1.75e-05 | 5246.2ms/step | 11.4 samples/s
+Validation at step 44500: Loss = 2.5150, Perplexity = 12.37
+Step 44510/50462 | Loss: 2.4808 | LR: 1.75e-05 | 7822.5ms/step | 7.7 samples/s
+Step 44520/50462 | Loss: 2.4795 | LR: 1.74e-05 | 5244.6ms/step | 11.4 samples/s
+W0530 03:29:22.958000 2737 torch/distributed/elastic/agent/server/api.py:753] Received Signals.SIGTERM death signal, shutting down workers
+W0530 03:29:22.959000 2737 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 2901 closing signal SIGTERM
+W0530 03:29:22.960000 2737 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 2902 closing signal SIGTERM
+W0530 03:29:22.962000 2737 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 2903 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1020, in <module>
+    main()
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 367, in wrapper
+    return f(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1016, in main
+    run(args)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1007, in run
+    elastic_launch(
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 191, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 362, in launch_agent
+    result = agent.run()
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 745, in run
+    result = self._invoke_run(role)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 923, in _invoke_run
+    time.sleep(monitor_interval)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2737 got signal: 15

epfl_final/log_files/eval_sparse_64.log

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bd5473d0f688f6796cec17c573b34f0c4f7b8905670648a53b9b2cce8042111
+size 1247537736

epfl_final/log_files/pretrain_test.log

@@ -0,0 +1,940 @@
+nohup: ignoring input
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+Resuming: Using existing output directory checkpoints/run_20260614_032718
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "checkpoints/run_20260614_032718",
+  "resume_from_checkpoint": "checkpoints/run_20260614_032718/checkpoint-38000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "sparse 1mask",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "sparse_stochastic"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: checkpoints/run_20260614_032718/checkpoint-38000
+Checkpoint activations: False
+Checkpoint activations: False
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 1536]           [2, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [2, 8192, 1536]           [2, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (G): 2.87
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 52437.19
+Params size (MB): 2871.45
+Estimated Total Size (MB): 55308.77
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        3
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504627
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+Loading checkpoint from checkpoints/run_20260614_032718/checkpoint-38000...
+Overriding checkpoint LR with config LR: 0.0005
+Resumed from checkpoint at step 38000 (epoch 0, micro_step 380000)
+Resuming training from step 38000
+Resuming wandb run aqqleyxv
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.27.2
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260615_161335-aqqleyxv
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run sparse 1mask
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/aqqleyxv
+Starting training...
+Validation at step 38000: Loss = 2.6437, Perplexity = 14.07
+wandb: WARNING Tried to log to step 38000 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 38001: sparse_stochastic (configured sparse attention)
+Step 38010/50462 | Loss: 2.6372 | LR: 7.35e-05 | 65177.4ms/step | 0.9 samples/s
+wandb: WARNING Tried to log to step 38010 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38020/50462 | Loss: 2.6248 | LR: 7.34e-05 | 3082.9ms/step | 19.5 samples/s
+wandb: WARNING Tried to log to step 38020 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38030/50462 | Loss: 2.6536 | LR: 7.33e-05 | 3093.2ms/step | 19.4 samples/s
+wandb: WARNING Tried to log to step 38030 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38040/50462 | Loss: 2.6222 | LR: 7.32e-05 | 3180.4ms/step | 18.9 samples/s
+wandb: WARNING Tried to log to step 38040 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38050/50462 | Loss: 2.6359 | LR: 7.30e-05 | 3102.9ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38050 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38060/50462 | Loss: 2.6340 | LR: 7.29e-05 | 3102.1ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38060 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38070/50462 | Loss: 2.6759 | LR: 7.28e-05 | 3182.2ms/step | 18.9 samples/s
+wandb: WARNING Tried to log to step 38070 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38080/50462 | Loss: 2.6323 | LR: 7.27e-05 | 3101.2ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38080 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38090/50462 | Loss: 2.6310 | LR: 7.26e-05 | 3185.9ms/step | 18.8 samples/s
+wandb: WARNING Tried to log to step 38090 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38100/50462 | Loss: 2.6529 | LR: 7.25e-05 | 3102.7ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38100 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38110/50462 | Loss: 2.5917 | LR: 7.24e-05 | 3100.9ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38110 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38120/50462 | Loss: 2.6380 | LR: 7.23e-05 | 3103.8ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38120 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38130/50462 | Loss: 2.6389 | LR: 7.22e-05 | 3105.1ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38130 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38140/50462 | Loss: 2.6455 | LR: 7.20e-05 | 3101.9ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38140 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38150/50462 | Loss: 2.6601 | LR: 7.19e-05 | 3103.2ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38150 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38160/50462 | Loss: 2.6155 | LR: 7.18e-05 | 3105.5ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38160 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38170/50462 | Loss: 2.6468 | LR: 7.17e-05 | 3104.2ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38170 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38180/50462 | Loss: 2.6445 | LR: 7.16e-05 | 3103.3ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38180 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38190/50462 | Loss: 2.6304 | LR: 7.15e-05 | 3106.4ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38190 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38200/50462 | Loss: 2.6668 | LR: 7.14e-05 | 3098.0ms/step | 19.4 samples/s
+wandb: WARNING Tried to log to step 38200 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38210/50462 | Loss: 2.6550 | LR: 7.13e-05 | 3103.9ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38210 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38220/50462 | Loss: 2.6251 | LR: 7.12e-05 | 3101.2ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38220 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38230/50462 | Loss: 2.6414 | LR: 7.10e-05 | 3102.1ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38230 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38240/50462 | Loss: 2.6386 | LR: 7.09e-05 | 3098.8ms/step | 19.4 samples/s
+wandb: WARNING Tried to log to step 38240 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38250/50462 | Loss: 2.6448 | LR: 7.08e-05 | 3102.7ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38250 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 38250: Loss = 2.6435, Perplexity = 14.06
+wandb: WARNING Tried to log to step 38250 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38260/50462 | Loss: 2.6255 | LR: 7.07e-05 | 4745.7ms/step | 12.6 samples/s
+wandb: WARNING Tried to log to step 38260 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38270/50462 | Loss: 2.6637 | LR: 7.06e-05 | 3100.1ms/step | 19.4 samples/s
+wandb: WARNING Tried to log to step 38270 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38280/50462 | Loss: 2.6036 | LR: 7.05e-05 | 3100.2ms/step | 19.4 samples/s
+wandb: WARNING Tried to log to step 38280 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38290/50462 | Loss: 2.6254 | LR: 7.04e-05 | 3095.9ms/step | 19.4 samples/s
+wandb: WARNING Tried to log to step 38290 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38300/50462 | Loss: 2.6355 | LR: 7.03e-05 | 3099.1ms/step | 19.4 samples/s
+wandb: WARNING Tried to log to step 38300 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38310/50462 | Loss: 2.6625 | LR: 7.02e-05 | 3094.0ms/step | 19.4 samples/s
+wandb: WARNING Tried to log to step 38310 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38320/50462 | Loss: 2.6659 | LR: 7.01e-05 | 3098.7ms/step | 19.4 samples/s
+wandb: WARNING Tried to log to step 38320 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38330/50462 | Loss: 2.6248 | LR: 6.99e-05 | 3102.7ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38330 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38340/50462 | Loss: 2.6298 | LR: 6.98e-05 | 3106.0ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38340 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38350/50462 | Loss: 2.6472 | LR: 6.97e-05 | 3104.1ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38350 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38360/50462 | Loss: 2.6444 | LR: 6.96e-05 | 3165.3ms/step | 19.0 samples/s
+wandb: WARNING Tried to log to step 38360 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38370/50462 | Loss: 2.6431 | LR: 6.95e-05 | 3101.6ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38370 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38380/50462 | Loss: 2.6235 | LR: 6.94e-05 | 3102.1ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38380 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38390/50462 | Loss: 2.6393 | LR: 6.93e-05 | 3102.5ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38390 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38400/50462 | Loss: 2.6323 | LR: 6.92e-05 | 3165.0ms/step | 19.0 samples/s
+wandb: WARNING Tried to log to step 38400 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38410/50462 | Loss: 2.6342 | LR: 6.91e-05 | 3108.5ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38410 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38420/50462 | Loss: 2.6500 | LR: 6.90e-05 | 3163.0ms/step | 19.0 samples/s
+wandb: WARNING Tried to log to step 38420 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38430/50462 | Loss: 2.6352 | LR: 6.89e-05 | 3102.7ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38430 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38440/50462 | Loss: 2.6230 | LR: 6.87e-05 | 3104.4ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38440 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38450/50462 | Loss: 2.6454 | LR: 6.86e-05 | 3102.4ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38450 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38460/50462 | Loss: 2.6300 | LR: 6.85e-05 | 3101.1ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38460 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38470/50462 | Loss: 2.6327 | LR: 6.84e-05 | 3104.9ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38470 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38480/50462 | Loss: 2.6224 | LR: 6.83e-05 | 3102.8ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38480 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38490/50462 | Loss: 2.6616 | LR: 6.82e-05 | 3103.8ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38490 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38500/50462 | Loss: 2.6361 | LR: 6.81e-05 | 3103.8ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38500 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 38500: Loss = 2.6431, Perplexity = 14.06
+wandb: WARNING Tried to log to step 38500 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38510/50462 | Loss: 2.6342 | LR: 6.80e-05 | 4750.1ms/step | 12.6 samples/s
+wandb: WARNING Tried to log to step 38510 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38520/50462 | Loss: 2.6211 | LR: 6.79e-05 | 3104.7ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38520 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38530/50462 | Loss: 2.6270 | LR: 6.78e-05 | 3101.4ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38530 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38540/50462 | Loss: 2.6055 | LR: 6.77e-05 | 3104.0ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38540 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38550/50462 | Loss: 2.6426 | LR: 6.76e-05 | 3102.4ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38550 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38560/50462 | Loss: 2.6174 | LR: 6.74e-05 | 3106.2ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38560 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38570/50462 | Loss: 2.6192 | LR: 6.73e-05 | 3103.9ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38570 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38580/50462 | Loss: 2.6287 | LR: 6.72e-05 | 3105.5ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38580 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38590/50462 | Loss: 2.6305 | LR: 6.71e-05 | 3106.1ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38590 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38600/50462 | Loss: 2.6362 | LR: 6.70e-05 | 3105.1ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38600 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38610/50462 | Loss: 2.6111 | LR: 6.69e-05 | 3101.4ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38610 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38620/50462 | Loss: 2.6210 | LR: 6.68e-05 | 3105.1ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38620 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38630/50462 | Loss: 2.6446 | LR: 6.67e-05 | 3102.9ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38630 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38640/50462 | Loss: 2.6216 | LR: 6.66e-05 | 3105.8ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38640 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38650/50462 | Loss: 2.6492 | LR: 6.65e-05 | 3103.1ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38650 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38660/50462 | Loss: 2.6194 | LR: 6.64e-05 | 3102.0ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38660 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38670/50462 | Loss: 2.6255 | LR: 6.63e-05 | 3093.1ms/step | 19.4 samples/s
+wandb: WARNING Tried to log to step 38670 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38680/50462 | Loss: 2.6144 | LR: 6.62e-05 | 3102.9ms/step | 19.3 samples/s
+wandb: WARNING Tried to log to step 38680 that is less than the current step 38681. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 38690/50462 | Loss: 2.5945 | LR: 6.60e-05 | 3160.8ms/step | 19.0 samples/s
+Step 38700/50462 | Loss: 2.6453 | LR: 6.59e-05 | 3107.2ms/step | 19.3 samples/s
+Step 38710/50462 | Loss: 2.6212 | LR: 6.58e-05 | 3108.5ms/step | 19.3 samples/s
+Step 38720/50462 | Loss: 2.6206 | LR: 6.57e-05 | 3093.7ms/step | 19.4 samples/s
+Step 38730/50462 | Loss: 2.6654 | LR: 6.56e-05 | 3161.6ms/step | 19.0 samples/s
+Step 38740/50462 | Loss: 2.6295 | LR: 6.55e-05 | 3104.8ms/step | 19.3 samples/s
+Step 38750/50462 | Loss: 2.6357 | LR: 6.54e-05 | 3153.5ms/step | 19.0 samples/s
+Validation at step 38750: Loss = 2.6427, Perplexity = 14.05
+Step 38760/50462 | Loss: 2.6332 | LR: 6.53e-05 | 4737.8ms/step | 12.7 samples/s
+Step 38770/50462 | Loss: 2.6514 | LR: 6.52e-05 | 3106.1ms/step | 19.3 samples/s
+Step 38780/50462 | Loss: 2.6232 | LR: 6.51e-05 | 3107.4ms/step | 19.3 samples/s
+Step 38790/50462 | Loss: 2.6761 | LR: 6.50e-05 | 3105.6ms/step | 19.3 samples/s
+Step 38800/50462 | Loss: 2.6396 | LR: 6.49e-05 | 3104.2ms/step | 19.3 samples/s
+Step 38810/50462 | Loss: 2.6241 | LR: 6.48e-05 | 3106.3ms/step | 19.3 samples/s
+Step 38820/50462 | Loss: 2.6285 | LR: 6.47e-05 | 3100.0ms/step | 19.4 samples/s
+Step 38830/50462 | Loss: 2.6349 | LR: 6.46e-05 | 3092.5ms/step | 19.4 samples/s
+Step 38840/50462 | Loss: 2.6476 | LR: 6.45e-05 | 3101.2ms/step | 19.3 samples/s
+Step 38850/50462 | Loss: 2.6546 | LR: 6.43e-05 | 3099.4ms/step | 19.4 samples/s
+Step 38860/50462 | Loss: 2.6219 | LR: 6.42e-05 | 3103.6ms/step | 19.3 samples/s
+Step 38870/50462 | Loss: 2.6352 | LR: 6.41e-05 | 3103.7ms/step | 19.3 samples/s
+Step 38880/50462 | Loss: 2.6302 | LR: 6.40e-05 | 3088.0ms/step | 19.4 samples/s
+Step 38890/50462 | Loss: 2.6396 | LR: 6.39e-05 | 3096.0ms/step | 19.4 samples/s
+Step 38900/50462 | Loss: 2.6379 | LR: 6.38e-05 | 3102.8ms/step | 19.3 samples/s
+Step 38910/50462 | Loss: 2.6392 | LR: 6.37e-05 | 3098.2ms/step | 19.4 samples/s
+Step 38920/50462 | Loss: 2.6302 | LR: 6.36e-05 | 3103.9ms/step | 19.3 samples/s
+Step 38930/50462 | Loss: 2.6451 | LR: 6.35e-05 | 3106.4ms/step | 19.3 samples/s
+Step 38940/50462 | Loss: 2.6385 | LR: 6.34e-05 | 3105.8ms/step | 19.3 samples/s
+Step 38950/50462 | Loss: 2.6476 | LR: 6.33e-05 | 3104.1ms/step | 19.3 samples/s
+Step 38960/50462 | Loss: 2.6291 | LR: 6.32e-05 | 3112.1ms/step | 19.3 samples/s
+Step 38970/50462 | Loss: 2.6216 | LR: 6.31e-05 | 3108.4ms/step | 19.3 samples/s
+Step 38980/50462 | Loss: 2.6327 | LR: 6.30e-05 | 3101.7ms/step | 19.3 samples/s
+Step 38990/50462 | Loss: 2.6079 | LR: 6.29e-05 | 3104.3ms/step | 19.3 samples/s
+Step 39000/50462 | Loss: 2.6368 | LR: 6.28e-05 | 3096.0ms/step | 19.4 samples/s
+Validation at step 39000: Loss = 2.6424, Perplexity = 14.05
+Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-39000
+Step 39010/50462 | Loss: 2.6528 | LR: 6.27e-05 | 5767.6ms/step | 10.4 samples/s
+Step 39020/50462 | Loss: 2.6120 | LR: 6.26e-05 | 3098.6ms/step | 19.4 samples/s
+Step 39030/50462 | Loss: 2.6402 | LR: 6.25e-05 | 3105.3ms/step | 19.3 samples/s
+Step 39040/50462 | Loss: 2.6268 | LR: 6.23e-05 | 3112.1ms/step | 19.3 samples/s
+Step 39050/50462 | Loss: 2.6069 | LR: 6.22e-05 | 3176.2ms/step | 18.9 samples/s
+Step 39060/50462 | Loss: 2.6179 | LR: 6.21e-05 | 3109.0ms/step | 19.3 samples/s
+Step 39070/50462 | Loss: 2.6384 | LR: 6.20e-05 | 3166.1ms/step | 19.0 samples/s
+Step 39080/50462 | Loss: 2.6572 | LR: 6.19e-05 | 3103.1ms/step | 19.3 samples/s
+Step 39090/50462 | Loss: 2.6121 | LR: 6.18e-05 | 3107.5ms/step | 19.3 samples/s
+Step 39100/50462 | Loss: 2.6351 | LR: 6.17e-05 | 3105.5ms/step | 19.3 samples/s
+Step 39110/50462 | Loss: 2.6112 | LR: 6.16e-05 | 3105.5ms/step | 19.3 samples/s
+Step 39120/50462 | Loss: 2.6188 | LR: 6.15e-05 | 3108.3ms/step | 19.3 samples/s
+Step 39130/50462 | Loss: 2.6537 | LR: 6.14e-05 | 3106.1ms/step | 19.3 samples/s
+Step 39140/50462 | Loss: 2.6276 | LR: 6.13e-05 | 3109.5ms/step | 19.3 samples/s
+Step 39150/50462 | Loss: 2.6292 | LR: 6.12e-05 | 3108.1ms/step | 19.3 samples/s
+Step 39160/50462 | Loss: 2.6176 | LR: 6.11e-05 | 3110.6ms/step | 19.3 samples/s
+Step 39170/50462 | Loss: 2.6455 | LR: 6.10e-05 | 3109.2ms/step | 19.3 samples/s
+Step 39180/50462 | Loss: 2.6548 | LR: 6.09e-05 | 3111.3ms/step | 19.3 samples/s
+Step 39190/50462 | Loss: 2.6373 | LR: 6.08e-05 | 3093.5ms/step | 19.4 samples/s
+Step 39200/50462 | Loss: 2.6099 | LR: 6.07e-05 | 3103.4ms/step | 19.3 samples/s
+Step 39210/50462 | Loss: 2.6291 | LR: 6.06e-05 | 3103.2ms/step | 19.3 samples/s
+Step 39220/50462 | Loss: 2.6030 | LR: 6.05e-05 | 3100.2ms/step | 19.4 samples/s
+Step 39230/50462 | Loss: 2.6506 | LR: 6.04e-05 | 3105.8ms/step | 19.3 samples/s
+Step 39240/50462 | Loss: 2.6270 | LR: 6.03e-05 | 3110.1ms/step | 19.3 samples/s
+Step 39250/50462 | Loss: 2.6464 | LR: 6.02e-05 | 3105.9ms/step | 19.3 samples/s
+Validation at step 39250: Loss = 2.6422, Perplexity = 14.04
+Step 39260/50462 | Loss: 2.6306 | LR: 6.01e-05 | 4748.9ms/step | 12.6 samples/s
+Step 39270/50462 | Loss: 2.6341 | LR: 6.00e-05 | 3105.2ms/step | 19.3 samples/s
+Step 39280/50462 | Loss: 2.6461 | LR: 5.99e-05 | 3109.1ms/step | 19.3 samples/s
+Step 39290/50462 | Loss: 2.6252 | LR: 5.98e-05 | 3104.6ms/step | 19.3 samples/s
+Step 39300/50462 | Loss: 2.6466 | LR: 5.97e-05 | 3108.2ms/step | 19.3 samples/s
+Step 39310/50462 | Loss: 2.6364 | LR: 5.96e-05 | 3108.1ms/step | 19.3 samples/s
+Step 39320/50462 | Loss: 2.5891 | LR: 5.95e-05 | 3102.4ms/step | 19.3 samples/s
+Step 39330/50462 | Loss: 2.6347 | LR: 5.94e-05 | 3169.6ms/step | 18.9 samples/s
+Step 39340/50462 | Loss: 2.6379 | LR: 5.93e-05 | 3100.1ms/step | 19.4 samples/s
+Step 39350/50462 | Loss: 2.6316 | LR: 5.91e-05 | 3099.8ms/step | 19.4 samples/s
+Step 39360/50462 | Loss: 2.6088 | LR: 5.90e-05 | 3098.5ms/step | 19.4 samples/s
+Step 39370/50462 | Loss: 2.6354 | LR: 5.89e-05 | 3106.0ms/step | 19.3 samples/s
+Step 39380/50462 | Loss: 2.6223 | LR: 5.88e-05 | 3159.4ms/step | 19.0 samples/s
+Step 39390/50462 | Loss: 2.6365 | LR: 5.87e-05 | 3102.1ms/step | 19.3 samples/s
+Step 39400/50462 | Loss: 2.6451 | LR: 5.86e-05 | 3166.4ms/step | 18.9 samples/s
+Step 39410/50462 | Loss: 2.5963 | LR: 5.85e-05 | 3107.7ms/step | 19.3 samples/s
+Step 39420/50462 | Loss: 2.6139 | LR: 5.84e-05 | 3107.6ms/step | 19.3 samples/s
+Step 39430/50462 | Loss: 2.6114 | LR: 5.83e-05 | 3108.3ms/step | 19.3 samples/s
+Step 39440/50462 | Loss: 2.6301 | LR: 5.82e-05 | 3107.6ms/step | 19.3 samples/s
+Step 39450/50462 | Loss: 2.6201 | LR: 5.81e-05 | 3107.5ms/step | 19.3 samples/s
+Step 39460/50462 | Loss: 2.6346 | LR: 5.80e-05 | 3096.2ms/step | 19.4 samples/s
+Step 39470/50462 | Loss: 2.6693 | LR: 5.79e-05 | 3107.1ms/step | 19.3 samples/s
+Step 39480/50462 | Loss: 2.6531 | LR: 5.78e-05 | 3105.1ms/step | 19.3 samples/s
+Step 39490/50462 | Loss: 2.6243 | LR: 5.77e-05 | 3107.5ms/step | 19.3 samples/s
+Step 39500/50462 | Loss: 2.6363 | LR: 5.76e-05 | 3107.6ms/step | 19.3 samples/s
+Validation at step 39500: Loss = 2.6419, Perplexity = 14.04
+Step 39510/50462 | Loss: 2.6362 | LR: 5.75e-05 | 4745.3ms/step | 12.6 samples/s
+Step 39520/50462 | Loss: 2.6455 | LR: 5.74e-05 | 3092.5ms/step | 19.4 samples/s
+Step 39530/50462 | Loss: 2.6347 | LR: 5.73e-05 | 3104.4ms/step | 19.3 samples/s
+Step 39540/50462 | Loss: 2.6335 | LR: 5.72e-05 | 3109.8ms/step | 19.3 samples/s
+Step 39550/50462 | Loss: 2.6375 | LR: 5.71e-05 | 3106.2ms/step | 19.3 samples/s
+Step 39560/50462 | Loss: 2.6295 | LR: 5.70e-05 | 3110.5ms/step | 19.3 samples/s
+Step 39570/50462 | Loss: 2.6386 | LR: 5.69e-05 | 3108.8ms/step | 19.3 samples/s
+Step 39580/50462 | Loss: 2.6195 | LR: 5.68e-05 | 3111.5ms/step | 19.3 samples/s
+Step 39590/50462 | Loss: 2.6433 | LR: 5.67e-05 | 3111.1ms/step | 19.3 samples/s
+Step 39600/50462 | Loss: 2.6273 | LR: 5.66e-05 | 3113.4ms/step | 19.3 samples/s
+Step 39610/50462 | Loss: 2.6192 | LR: 5.65e-05 | 3096.5ms/step | 19.4 samples/s
+Step 39620/50462 | Loss: 2.6145 | LR: 5.64e-05 | 3105.3ms/step | 19.3 samples/s
+Step 39630/50462 | Loss: 2.6042 | LR: 5.63e-05 | 3105.0ms/step | 19.3 samples/s
+Step 39640/50462 | Loss: 2.6600 | LR: 5.62e-05 | 3099.7ms/step | 19.4 samples/s
+Step 39650/50462 | Loss: 2.6344 | LR: 5.61e-05 | 3100.7ms/step | 19.4 samples/s
+Step 39660/50462 | Loss: 2.6342 | LR: 5.60e-05 | 3146.4ms/step | 19.1 samples/s
+Step 39670/50462 | Loss: 2.6355 | LR: 5.59e-05 | 3090.3ms/step | 19.4 samples/s
+Step 39680/50462 | Loss: 2.6455 | LR: 5.58e-05 | 3098.5ms/step | 19.4 samples/s
+Step 39690/50462 | Loss: 2.6268 | LR: 5.57e-05 | 3103.1ms/step | 19.3 samples/s
+Step 39700/50462 | Loss: 2.6238 | LR: 5.56e-05 | 3171.7ms/step | 18.9 samples/s
+Step 39710/50462 | Loss: 2.6153 | LR: 5.55e-05 | 3098.8ms/step | 19.4 samples/s
+Step 39720/50462 | Loss: 2.6079 | LR: 5.54e-05 | 3161.7ms/step | 19.0 samples/s
+Step 39730/50462 | Loss: 2.6075 | LR: 5.53e-05 | 3104.6ms/step | 19.3 samples/s
+Step 39740/50462 | Loss: 2.6299 | LR: 5.52e-05 | 3107.4ms/step | 19.3 samples/s
+Step 39750/50462 | Loss: 2.6408 | LR: 5.51e-05 | 3109.0ms/step | 19.3 samples/s
+Validation at step 39750: Loss = 2.6417, Perplexity = 14.04
+Step 39760/50462 | Loss: 2.6271 | LR: 5.50e-05 | 4737.5ms/step | 12.7 samples/s
+Step 39770/50462 | Loss: 2.6507 | LR: 5.49e-05 | 3099.7ms/step | 19.4 samples/s
+Step 39780/50462 | Loss: 2.6530 | LR: 5.48e-05 | 3105.6ms/step | 19.3 samples/s
+Step 39790/50462 | Loss: 2.6272 | LR: 5.47e-05 | 3105.5ms/step | 19.3 samples/s
+Step 39800/50462 | Loss: 2.6212 | LR: 5.46e-05 | 3107.5ms/step | 19.3 samples/s
+Step 39810/50462 | Loss: 2.6340 | LR: 5.45e-05 | 3086.7ms/step | 19.4 samples/s
+Step 39820/50462 | Loss: 2.6545 | LR: 5.44e-05 | 3096.2ms/step | 19.4 samples/s
+Step 39830/50462 | Loss: 2.6266 | LR: 5.43e-05 | 3098.4ms/step | 19.4 samples/s
+Step 39840/50462 | Loss: 2.6373 | LR: 5.42e-05 | 3098.4ms/step | 19.4 samples/s
+Step 39850/50462 | Loss: 2.6453 | LR: 5.41e-05 | 3100.5ms/step | 19.4 samples/s
+Step 39860/50462 | Loss: 2.6087 | LR: 5.40e-05 | 3107.2ms/step | 19.3 samples/s
+Step 39870/50462 | Loss: 2.6347 | LR: 5.39e-05 | 3100.1ms/step | 19.4 samples/s
+Step 39880/50462 | Loss: 2.6325 | LR: 5.38e-05 | 3095.0ms/step | 19.4 samples/s
+Step 39890/50462 | Loss: 2.6401 | LR: 5.38e-05 | 3095.8ms/step | 19.4 samples/s
+Step 39900/50462 | Loss: 2.6237 | LR: 5.37e-05 | 3096.0ms/step | 19.4 samples/s
+Step 39910/50462 | Loss: 2.6303 | LR: 5.36e-05 | 3098.2ms/step | 19.4 samples/s
+Step 39920/50462 | Loss: 2.6290 | LR: 5.35e-05 | 3104.0ms/step | 19.3 samples/s
+Step 39930/50462 | Loss: 2.6326 | LR: 5.34e-05 | 3110.8ms/step | 19.3 samples/s
+Step 39940/50462 | Loss: 2.6269 | LR: 5.33e-05 | 3106.9ms/step | 19.3 samples/s
+Step 39950/50462 | Loss: 2.6144 | LR: 5.32e-05 | 3096.8ms/step | 19.4 samples/s
+Step 39960/50462 | Loss: 2.6312 | LR: 5.31e-05 | 3103.1ms/step | 19.3 samples/s
+Step 39970/50462 | Loss: 2.6558 | LR: 5.30e-05 | 3098.7ms/step | 19.4 samples/s
+Step 39980/50462 | Loss: 2.6311 | LR: 5.29e-05 | 3102.5ms/step | 19.3 samples/s
+Step 39990/50462 | Loss: 2.6671 | LR: 5.28e-05 | 3156.3ms/step | 19.0 samples/s
+Step 40000/50462 | Loss: 2.6052 | LR: 5.27e-05 | 3097.6ms/step | 19.4 samples/s
+Validation at step 40000: Loss = 2.6415, Perplexity = 14.03
+Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-40000
+Step 40010/50462 | Loss: 2.6490 | LR: 5.26e-05 | 5679.6ms/step | 10.6 samples/s
+Step 40020/50462 | Loss: 2.6703 | LR: 5.25e-05 | 3090.2ms/step | 19.4 samples/s
+Step 40030/50462 | Loss: 2.6162 | LR: 5.24e-05 | 3162.4ms/step | 19.0 samples/s
+Step 40040/50462 | Loss: 2.6445 | LR: 5.23e-05 | 3099.4ms/step | 19.4 samples/s
+Step 40050/50462 | Loss: 2.6335 | LR: 5.22e-05 | 3159.7ms/step | 19.0 samples/s
+Step 40060/50462 | Loss: 2.6271 | LR: 5.21e-05 | 3103.0ms/step | 19.3 samples/s
+Step 40070/50462 | Loss: 2.6395 | LR: 5.20e-05 | 3110.1ms/step | 19.3 samples/s
+Step 40080/50462 | Loss: 2.6135 | LR: 5.19e-05 | 3113.3ms/step | 19.3 samples/s
+Step 40090/50462 | Loss: 2.6480 | LR: 5.18e-05 | 3111.3ms/step | 19.3 samples/s
+Step 40100/50462 | Loss: 2.6454 | LR: 5.17e-05 | 3108.5ms/step | 19.3 samples/s
+Step 40110/50462 | Loss: 2.6325 | LR: 5.16e-05 | 3110.6ms/step | 19.3 samples/s
+Step 40120/50462 | Loss: 2.6450 | LR: 5.15e-05 | 3113.1ms/step | 19.3 samples/s
+Step 40130/50462 | Loss: 2.6260 | LR: 5.14e-05 | 3112.1ms/step | 19.3 samples/s
+Step 40140/50462 | Loss: 2.6237 | LR: 5.13e-05 | 3110.3ms/step | 19.3 samples/s
+Step 40150/50462 | Loss: 2.6010 | LR: 5.12e-05 | 3108.7ms/step | 19.3 samples/s
+Step 40160/50462 | Loss: 2.5960 | LR: 5.11e-05 | 3114.7ms/step | 19.3 samples/s
+Step 40170/50462 | Loss: 2.6377 | LR: 5.10e-05 | 3111.2ms/step | 19.3 samples/s
+Step 40180/50462 | Loss: 2.6411 | LR: 5.09e-05 | 3103.1ms/step | 19.3 samples/s
+Step 40190/50462 | Loss: 2.6388 | LR: 5.09e-05 | 3104.4ms/step | 19.3 samples/s
+Step 40200/50462 | Loss: 2.6457 | LR: 5.08e-05 | 3100.5ms/step | 19.4 samples/s
+Step 40210/50462 | Loss: 2.6515 | LR: 5.07e-05 | 3111.0ms/step | 19.3 samples/s
+Step 40220/50462 | Loss: 2.6501 | LR: 5.06e-05 | 3105.4ms/step | 19.3 samples/s
+Step 40230/50462 | Loss: 2.6434 | LR: 5.05e-05 | 3111.1ms/step | 19.3 samples/s
+Step 40240/50462 | Loss: 2.6716 | LR: 5.04e-05 | 3112.6ms/step | 19.3 samples/s
+Step 40250/50462 | Loss: 2.6310 | LR: 5.03e-05 | 3110.7ms/step | 19.3 samples/s
+Validation at step 40250: Loss = 2.6413, Perplexity = 14.03
+Step 40260/50462 | Loss: 2.6382 | LR: 5.02e-05 | 4753.9ms/step | 12.6 samples/s
+Step 40270/50462 | Loss: 2.6129 | LR: 5.01e-05 | 3110.8ms/step | 19.3 samples/s
+Step 40280/50462 | Loss: 2.6371 | LR: 5.00e-05 | 3097.7ms/step | 19.4 samples/s
+Step 40290/50462 | Loss: 2.6331 | LR: 4.99e-05 | 3102.6ms/step | 19.3 samples/s
+Step 40300/50462 | Loss: 2.6288 | LR: 4.98e-05 | 3109.0ms/step | 19.3 samples/s
+Step 40310/50462 | Loss: 2.6168 | LR: 4.97e-05 | 3169.3ms/step | 18.9 samples/s
+Step 40320/50462 | Loss: 2.6343 | LR: 4.96e-05 | 3110.7ms/step | 19.3 samples/s
+Step 40330/50462 | Loss: 2.6565 | LR: 4.95e-05 | 3112.3ms/step | 19.3 samples/s
+Step 40340/50462 | Loss: 2.6189 | LR: 4.94e-05 | 3110.0ms/step | 19.3 samples/s
+Step 40350/50462 | Loss: 2.6102 | LR: 4.93e-05 | 3107.0ms/step | 19.3 samples/s
+Step 40360/50462 | Loss: 2.6350 | LR: 4.92e-05 | 3169.5ms/step | 18.9 samples/s
+Step 40370/50462 | Loss: 2.6500 | LR: 4.91e-05 | 3101.0ms/step | 19.3 samples/s
+Step 40380/50462 | Loss: 2.6362 | LR: 4.90e-05 | 3161.5ms/step | 19.0 samples/s
+Step 40390/50462 | Loss: 2.6318 | LR: 4.90e-05 | 3106.9ms/step | 19.3 samples/s
+Step 40400/50462 | Loss: 2.6466 | LR: 4.89e-05 | 3104.9ms/step | 19.3 samples/s
+Step 40410/50462 | Loss: 2.6596 | LR: 4.88e-05 | 3107.5ms/step | 19.3 samples/s
+Step 40420/50462 | Loss: 2.6568 | LR: 4.87e-05 | 3104.3ms/step | 19.3 samples/s
+Step 40430/50462 | Loss: 2.6259 | LR: 4.86e-05 | 3105.6ms/step | 19.3 samples/s
+Step 40440/50462 | Loss: 2.6427 | LR: 4.85e-05 | 3096.3ms/step | 19.4 samples/s
+Step 40450/50462 | Loss: 2.6338 | LR: 4.84e-05 | 3103.3ms/step | 19.3 samples/s
+Step 40460/50462 | Loss: 2.6674 | LR: 4.83e-05 | 3105.9ms/step | 19.3 samples/s
+Step 40470/50462 | Loss: 2.6192 | LR: 4.82e-05 | 3100.9ms/step | 19.3 samples/s
+Step 40480/50462 | Loss: 2.6301 | LR: 4.81e-05 | 3103.8ms/step | 19.3 samples/s
+Step 40490/50462 | Loss: 2.6437 | LR: 4.80e-05 | 3104.6ms/step | 19.3 samples/s
+Step 40500/50462 | Loss: 2.6379 | LR: 4.79e-05 | 3113.6ms/step | 19.3 samples/s
+Validation at step 40500: Loss = 2.6412, Perplexity = 14.03
+Step 40510/50462 | Loss: 2.6354 | LR: 4.78e-05 | 4758.1ms/step | 12.6 samples/s
+Step 40520/50462 | Loss: 2.6201 | LR: 4.77e-05 | 3109.1ms/step | 19.3 samples/s
+Step 40530/50462 | Loss: 2.6251 | LR: 4.76e-05 | 3102.2ms/step | 19.3 samples/s
+Step 40540/50462 | Loss: 2.5998 | LR: 4.76e-05 | 3108.0ms/step | 19.3 samples/s
+Step 40550/50462 | Loss: 2.6508 | LR: 4.75e-05 | 3109.8ms/step | 19.3 samples/s
+Step 40560/50462 | Loss: 2.6438 | LR: 4.74e-05 | 3109.7ms/step | 19.3 samples/s
+Step 40570/50462 | Loss: 2.6204 | LR: 4.73e-05 | 3107.7ms/step | 19.3 samples/s
+Step 40580/50462 | Loss: 2.6464 | LR: 4.72e-05 | 3107.6ms/step | 19.3 samples/s
+Step 40590/50462 | Loss: 2.6308 | LR: 4.71e-05 | 3099.8ms/step | 19.4 samples/s
+Step 40600/50462 | Loss: 2.6320 | LR: 4.70e-05 | 3103.7ms/step | 19.3 samples/s
+Step 40610/50462 | Loss: 2.6518 | LR: 4.69e-05 | 3100.9ms/step | 19.3 samples/s
+Step 40620/50462 | Loss: 2.6081 | LR: 4.68e-05 | 3104.7ms/step | 19.3 samples/s
+Step 40630/50462 | Loss: 2.6257 | LR: 4.67e-05 | 3106.7ms/step | 19.3 samples/s
+Step 40640/50462 | Loss: 2.6367 | LR: 4.66e-05 | 3171.5ms/step | 18.9 samples/s
+Step 40650/50462 | Loss: 2.6395 | LR: 4.65e-05 | 3112.4ms/step | 19.3 samples/s
+Step 40660/50462 | Loss: 2.6493 | LR: 4.64e-05 | 3106.2ms/step | 19.3 samples/s
+Step 40670/50462 | Loss: 2.6275 | LR: 4.64e-05 | 3105.5ms/step | 19.3 samples/s
+Step 40680/50462 | Loss: 2.6334 | LR: 4.63e-05 | 3173.7ms/step | 18.9 samples/s
+Step 40690/50462 | Loss: 2.6264 | LR: 4.62e-05 | 3098.4ms/step | 19.4 samples/s
+Step 40700/50462 | Loss: 2.6013 | LR: 4.61e-05 | 3165.6ms/step | 19.0 samples/s
+Step 40710/50462 | Loss: 2.6432 | LR: 4.60e-05 | 3103.9ms/step | 19.3 samples/s
+Step 40720/50462 | Loss: 2.6059 | LR: 4.59e-05 | 3102.3ms/step | 19.3 samples/s
+Step 40730/50462 | Loss: 2.6162 | LR: 4.58e-05 | 3105.9ms/step | 19.3 samples/s
+Step 40740/50462 | Loss: 2.6157 | LR: 4.57e-05 | 3113.2ms/step | 19.3 samples/s
+Step 40750/50462 | Loss: 2.6254 | LR: 4.56e-05 | 3110.3ms/step | 19.3 samples/s
+Validation at step 40750: Loss = 2.6409, Perplexity = 14.03
+Step 40760/50462 | Loss: 2.6512 | LR: 4.55e-05 | 4752.8ms/step | 12.6 samples/s
+Step 40770/50462 | Loss: 2.6311 | LR: 4.54e-05 | 3107.3ms/step | 19.3 samples/s
+Step 40780/50462 | Loss: 2.6454 | LR: 4.54e-05 | 3099.4ms/step | 19.4 samples/s
+Step 40790/50462 | Loss: 2.6283 | LR: 4.53e-05 | 3100.6ms/step | 19.4 samples/s
+Step 40800/50462 | Loss: 2.6387 | LR: 4.52e-05 | 3100.8ms/step | 19.3 samples/s
+Step 40810/50462 | Loss: 2.5951 | LR: 4.51e-05 | 3104.0ms/step | 19.3 samples/s
+Step 40820/50462 | Loss: 2.6696 | LR: 4.50e-05 | 3094.9ms/step | 19.4 samples/s
+Step 40830/50462 | Loss: 2.5875 | LR: 4.49e-05 | 3101.2ms/step | 19.3 samples/s
+Step 40840/50462 | Loss: 2.6108 | LR: 4.48e-05 | 3097.4ms/step | 19.4 samples/s
+Step 40850/50462 | Loss: 2.6535 | LR: 4.47e-05 | 3106.8ms/step | 19.3 samples/s
+Step 40860/50462 | Loss: 2.5977 | LR: 4.46e-05 | 3106.4ms/step | 19.3 samples/s
+Step 40870/50462 | Loss: 2.6558 | LR: 4.45e-05 | 3113.3ms/step | 19.3 samples/s
+Step 40880/50462 | Loss: 2.6613 | LR: 4.45e-05 | 3110.2ms/step | 19.3 samples/s
+Step 40890/50462 | Loss: 2.5926 | LR: 4.44e-05 | 3114.1ms/step | 19.3 samples/s
+Step 40900/50462 | Loss: 2.6177 | LR: 4.43e-05 | 3094.3ms/step | 19.4 samples/s
+Step 40910/50462 | Loss: 2.6397 | LR: 4.42e-05 | 3094.7ms/step | 19.4 samples/s
+Step 40920/50462 | Loss: 2.6467 | LR: 4.41e-05 | 3097.5ms/step | 19.4 samples/s
+Step 40930/50462 | Loss: 2.6377 | LR: 4.40e-05 | 3105.9ms/step | 19.3 samples/s
+Step 40940/50462 | Loss: 2.6555 | LR: 4.39e-05 | 3107.3ms/step | 19.3 samples/s
+Step 40950/50462 | Loss: 2.6094 | LR: 4.38e-05 | 3100.3ms/step | 19.4 samples/s
+Step 40960/50462 | Loss: 2.6384 | LR: 4.37e-05 | 3159.1ms/step | 19.0 samples/s
+Step 40970/50462 | Loss: 2.6288 | LR: 4.36e-05 | 3096.6ms/step | 19.4 samples/s
+Step 40980/50462 | Loss: 2.6306 | LR: 4.36e-05 | 3101.0ms/step | 19.3 samples/s
+Step 40990/50462 | Loss: 2.5982 | LR: 4.35e-05 | 3102.4ms/step | 19.3 samples/s
+Step 41000/50462 | Loss: 2.6444 | LR: 4.34e-05 | 3105.8ms/step | 19.3 samples/s
+Validation at step 41000: Loss = 2.6407, Perplexity = 14.02
+Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-41000
+Step 41010/50462 | Loss: 2.6469 | LR: 4.33e-05 | 5733.0ms/step | 10.5 samples/s
+Step 41020/50462 | Loss: 2.6459 | LR: 4.32e-05 | 3092.3ms/step | 19.4 samples/s
+Step 41030/50462 | Loss: 2.6474 | LR: 4.31e-05 | 3160.7ms/step | 19.0 samples/s
+Step 41040/50462 | Loss: 2.6125 | LR: 4.30e-05 | 3102.9ms/step | 19.3 samples/s
+Step 41050/50462 | Loss: 2.6447 | LR: 4.29e-05 | 3100.8ms/step | 19.3 samples/s
+Step 41060/50462 | Loss: 2.6494 | LR: 4.28e-05 | 3102.2ms/step | 19.3 samples/s
+Step 41070/50462 | Loss: 2.6253 | LR: 4.28e-05 | 3108.5ms/step | 19.3 samples/s
+Step 41080/50462 | Loss: 2.6180 | LR: 4.27e-05 | 3105.8ms/step | 19.3 samples/s
+Step 41090/50462 | Loss: 2.5981 | LR: 4.26e-05 | 3109.2ms/step | 19.3 samples/s
+Step 41100/50462 | Loss: 2.6309 | LR: 4.25e-05 | 3107.1ms/step | 19.3 samples/s
+Step 41110/50462 | Loss: 2.6336 | LR: 4.24e-05 | 3101.8ms/step | 19.3 samples/s
+Step 41120/50462 | Loss: 2.6261 | LR: 4.23e-05 | 3103.2ms/step | 19.3 samples/s
+Step 41130/50462 | Loss: 2.6496 | LR: 4.22e-05 | 3108.4ms/step | 19.3 samples/s
+Step 41140/50462 | Loss: 2.6220 | LR: 4.21e-05 | 3103.7ms/step | 19.3 samples/s
+Step 41150/50462 | Loss: 2.6335 | LR: 4.21e-05 | 3105.2ms/step | 19.3 samples/s
+Step 41160/50462 | Loss: 2.6344 | LR: 4.20e-05 | 3106.3ms/step | 19.3 samples/s
+Step 41170/50462 | Loss: 2.6412 | LR: 4.19e-05 | 3110.5ms/step | 19.3 samples/s
+Step 41180/50462 | Loss: 2.6224 | LR: 4.18e-05 | 3110.1ms/step | 19.3 samples/s
+Step 41190/50462 | Loss: 2.5813 | LR: 4.17e-05 | 3110.3ms/step | 19.3 samples/s
+Step 41200/50462 | Loss: 2.6209 | LR: 4.16e-05 | 3109.8ms/step | 19.3 samples/s
+Step 41210/50462 | Loss: 2.5964 | LR: 4.15e-05 | 3098.0ms/step | 19.4 samples/s
+Step 41220/50462 | Loss: 2.6115 | LR: 4.14e-05 | 3091.7ms/step | 19.4 samples/s
+Step 41230/50462 | Loss: 2.6349 | LR: 4.14e-05 | 3105.1ms/step | 19.3 samples/s
+Step 41240/50462 | Loss: 2.6276 | LR: 4.13e-05 | 3098.6ms/step | 19.4 samples/s
+Step 41250/50462 | Loss: 2.6386 | LR: 4.12e-05 | 3102.5ms/step | 19.3 samples/s
+Validation at step 41250: Loss = 2.6406, Perplexity = 14.02
+Step 41260/50462 | Loss: 2.6347 | LR: 4.11e-05 | 4743.7ms/step | 12.6 samples/s
+Step 41270/50462 | Loss: 2.6102 | LR: 4.10e-05 | 3104.9ms/step | 19.3 samples/s
+Step 41280/50462 | Loss: 2.5885 | LR: 4.09e-05 | 3163.8ms/step | 19.0 samples/s
+Step 41290/50462 | Loss: 2.6491 | LR: 4.08e-05 | 3104.2ms/step | 19.3 samples/s
+Step 41300/50462 | Loss: 2.6392 | LR: 4.07e-05 | 3099.5ms/step | 19.4 samples/s
+Step 41310/50462 | Loss: 2.6026 | LR: 4.07e-05 | 3101.9ms/step | 19.3 samples/s
+Step 41320/50462 | Loss: 2.6830 | LR: 4.06e-05 | 3103.3ms/step | 19.3 samples/s
+Step 41330/50462 | Loss: 2.6195 | LR: 4.05e-05 | 3166.1ms/step | 19.0 samples/s
+Step 41340/50462 | Loss: 2.6133 | LR: 4.04e-05 | 3096.8ms/step | 19.4 samples/s
+Step 41350/50462 | Loss: 2.6335 | LR: 4.03e-05 | 3157.3ms/step | 19.0 samples/s
+Step 41360/50462 | Loss: 2.6618 | LR: 4.02e-05 | 3095.9ms/step | 19.4 samples/s
+Step 41370/50462 | Loss: 2.6304 | LR: 4.01e-05 | 3090.3ms/step | 19.4 samples/s
+Step 41380/50462 | Loss: 2.6278 | LR: 4.01e-05 | 3099.1ms/step | 19.4 samples/s
+Step 41390/50462 | Loss: 2.6008 | LR: 4.00e-05 | 3097.5ms/step | 19.4 samples/s
+Step 41400/50462 | Loss: 2.6418 | LR: 3.99e-05 | 3098.6ms/step | 19.4 samples/s
+Step 41410/50462 | Loss: 2.6462 | LR: 3.98e-05 | 3100.0ms/step | 19.4 samples/s
+Step 41420/50462 | Loss: 2.6076 | LR: 3.97e-05 | 3100.7ms/step | 19.4 samples/s
+Step 41430/50462 | Loss: 2.6408 | LR: 3.96e-05 | 3100.3ms/step | 19.4 samples/s
+Step 41440/50462 | Loss: 2.6108 | LR: 3.95e-05 | 3100.1ms/step | 19.4 samples/s
+Step 41450/50462 | Loss: 2.6291 | LR: 3.95e-05 | 3100.5ms/step | 19.4 samples/s
+Step 41460/50462 | Loss: 2.6136 | LR: 3.94e-05 | 3090.0ms/step | 19.4 samples/s
+Step 41470/50462 | Loss: 2.5906 | LR: 3.93e-05 | 3091.5ms/step | 19.4 samples/s
+Step 41480/50462 | Loss: 2.6483 | LR: 3.92e-05 | 3097.9ms/step | 19.4 samples/s
+Step 41490/50462 | Loss: 2.6016 | LR: 3.91e-05 | 3098.7ms/step | 19.4 samples/s
+Step 41500/50462 | Loss: 2.6471 | LR: 3.90e-05 | 3103.7ms/step | 19.3 samples/s
+Validation at step 41500: Loss = 2.6405, Perplexity = 14.02
+Step 41510/50462 | Loss: 2.6157 | LR: 3.90e-05 | 4749.6ms/step | 12.6 samples/s
+Step 41520/50462 | Loss: 2.6303 | LR: 3.89e-05 | 3107.4ms/step | 19.3 samples/s
+Step 41530/50462 | Loss: 2.6413 | LR: 3.88e-05 | 3110.8ms/step | 19.3 samples/s
+Step 41540/50462 | Loss: 2.6120 | LR: 3.87e-05 | 3106.3ms/step | 19.3 samples/s
+Step 41550/50462 | Loss: 2.6386 | LR: 3.86e-05 | 3100.7ms/step | 19.4 samples/s
+Step 41560/50462 | Loss: 2.6489 | LR: 3.85e-05 | 3102.8ms/step | 19.3 samples/s
+Step 41570/50462 | Loss: 2.6375 | LR: 3.84e-05 | 3100.9ms/step | 19.3 samples/s
+Step 41580/50462 | Loss: 2.6412 | LR: 3.84e-05 | 3110.3ms/step | 19.3 samples/s
+Step 41590/50462 | Loss: 2.6235 | LR: 3.83e-05 | 3108.7ms/step | 19.3 samples/s
+Step 41600/50462 | Loss: 2.6384 | LR: 3.82e-05 | 3098.2ms/step | 19.4 samples/s
+Step 41610/50462 | Loss: 2.6220 | LR: 3.81e-05 | 3159.7ms/step | 19.0 samples/s
+Step 41620/50462 | Loss: 2.6250 | LR: 3.80e-05 | 3096.2ms/step | 19.4 samples/s
+Step 41630/50462 | Loss: 2.6084 | LR: 3.79e-05 | 3104.0ms/step | 19.3 samples/s
+Step 41640/50462 | Loss: 2.6228 | LR: 3.79e-05 | 3103.0ms/step | 19.3 samples/s
+Step 41650/50462 | Loss: 2.6108 | LR: 3.78e-05 | 3109.8ms/step | 19.3 samples/s
+Step 41660/50462 | Loss: 2.6389 | LR: 3.77e-05 | 3155.4ms/step | 19.0 samples/s
+Step 41670/50462 | Loss: 2.6321 | LR: 3.76e-05 | 3102.1ms/step | 19.3 samples/s
+Step 41680/50462 | Loss: 2.6163 | LR: 3.75e-05 | 3152.2ms/step | 19.0 samples/s
+Step 41690/50462 | Loss: 2.6126 | LR: 3.74e-05 | 3098.7ms/step | 19.4 samples/s
+Step 41700/50462 | Loss: 2.6276 | LR: 3.74e-05 | 3107.8ms/step | 19.3 samples/s
+Step 41710/50462 | Loss: 2.6148 | LR: 3.73e-05 | 3103.8ms/step | 19.3 samples/s
+Step 41720/50462 | Loss: 2.6130 | LR: 3.72e-05 | 3098.3ms/step | 19.4 samples/s
+Step 41730/50462 | Loss: 2.6186 | LR: 3.71e-05 | 3102.7ms/step | 19.3 samples/s
+Step 41740/50462 | Loss: 2.6116 | LR: 3.70e-05 | 3104.1ms/step | 19.3 samples/s
+Step 41750/50462 | Loss: 2.6222 | LR: 3.69e-05 | 3103.9ms/step | 19.3 samples/s
+Validation at step 41750: Loss = 2.6403, Perplexity = 14.02
+Step 41760/50462 | Loss: 2.6299 | LR: 3.69e-05 | 4749.4ms/step | 12.6 samples/s
+Step 41770/50462 | Loss: 2.6370 | LR: 3.68e-05 | 3101.9ms/step | 19.3 samples/s
+Step 41780/50462 | Loss: 2.6242 | LR: 3.67e-05 | 3101.0ms/step | 19.3 samples/s
+Step 41790/50462 | Loss: 2.6101 | LR: 3.66e-05 | 3105.7ms/step | 19.3 samples/s
+Step 41800/50462 | Loss: 2.6123 | LR: 3.65e-05 | 3096.9ms/step | 19.4 samples/s
+Step 41810/50462 | Loss: 2.6467 | LR: 3.64e-05 | 3106.4ms/step | 19.3 samples/s
+Step 41820/50462 | Loss: 2.5728 | LR: 3.64e-05 | 3110.6ms/step | 19.3 samples/s
+Step 41830/50462 | Loss: 2.6227 | LR: 3.63e-05 | 3106.2ms/step | 19.3 samples/s
+Step 41840/50462 | Loss: 2.5983 | LR: 3.62e-05 | 3104.4ms/step | 19.3 samples/s
+Step 41850/50462 | Loss: 2.6394 | LR: 3.61e-05 | 3107.3ms/step | 19.3 samples/s
+Step 41860/50462 | Loss: 2.6247 | LR: 3.60e-05 | 3101.2ms/step | 19.3 samples/s
+Step 41870/50462 | Loss: 2.6215 | LR: 3.60e-05 | 3102.7ms/step | 19.3 samples/s
+Step 41880/50462 | Loss: 2.6355 | LR: 3.59e-05 | 3109.6ms/step | 19.3 samples/s
+Step 41890/50462 | Loss: 2.6242 | LR: 3.58e-05 | 3107.2ms/step | 19.3 samples/s
+Step 41900/50462 | Loss: 2.6497 | LR: 3.57e-05 | 3104.4ms/step | 19.3 samples/s
+Step 41910/50462 | Loss: 2.6393 | LR: 3.56e-05 | 3099.5ms/step | 19.4 samples/s
+Step 41920/50462 | Loss: 2.6343 | LR: 3.55e-05 | 3106.3ms/step | 19.3 samples/s
+Step 41930/50462 | Loss: 2.6136 | LR: 3.55e-05 | 3109.8ms/step | 19.3 samples/s
+Step 41940/50462 | Loss: 2.6422 | LR: 3.54e-05 | 3167.9ms/step | 18.9 samples/s
+Step 41950/50462 | Loss: 2.6103 | LR: 3.53e-05 | 3104.7ms/step | 19.3 samples/s
+Step 41960/50462 | Loss: 2.6339 | LR: 3.52e-05 | 3108.1ms/step | 19.3 samples/s
+Step 41970/50462 | Loss: 2.6263 | LR: 3.51e-05 | 3109.8ms/step | 19.3 samples/s
+Step 41980/50462 | Loss: 2.6231 | LR: 3.51e-05 | 3104.2ms/step | 19.3 samples/s
+Step 41990/50462 | Loss: 2.6560 | LR: 3.50e-05 | 3161.1ms/step | 19.0 samples/s
+Step 42000/50462 | Loss: 2.6325 | LR: 3.49e-05 | 3093.5ms/step | 19.4 samples/s
+Validation at step 42000: Loss = 2.6402, Perplexity = 14.02
+Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-42000
+Step 42010/50462 | Loss: 2.6028 | LR: 3.48e-05 | 5728.2ms/step | 10.5 samples/s
+Step 42020/50462 | Loss: 2.6459 | LR: 3.47e-05 | 3097.1ms/step | 19.4 samples/s
+Step 42030/50462 | Loss: 2.6658 | LR: 3.47e-05 | 3110.7ms/step | 19.3 samples/s
+Step 42040/50462 | Loss: 2.6303 | LR: 3.46e-05 | 3104.5ms/step | 19.3 samples/s
+Step 42050/50462 | Loss: 2.6295 | LR: 3.45e-05 | 3108.1ms/step | 19.3 samples/s
+Step 42060/50462 | Loss: 2.6660 | LR: 3.44e-05 | 3105.0ms/step | 19.3 samples/s
+Step 42070/50462 | Loss: 2.6462 | LR: 3.43e-05 | 3108.8ms/step | 19.3 samples/s
+Step 42080/50462 | Loss: 2.6169 | LR: 3.43e-05 | 3104.5ms/step | 19.3 samples/s
+Step 42090/50462 | Loss: 2.6463 | LR: 3.42e-05 | 3104.8ms/step | 19.3 samples/s
+Step 42100/50462 | Loss: 2.6313 | LR: 3.41e-05 | 3107.9ms/step | 19.3 samples/s
+Step 42110/50462 | Loss: 2.5984 | LR: 3.40e-05 | 3112.0ms/step | 19.3 samples/s
+Step 42120/50462 | Loss: 2.6334 | LR: 3.39e-05 | 3107.5ms/step | 19.3 samples/s
+Step 42130/50462 | Loss: 2.6430 | LR: 3.39e-05 | 3104.6ms/step | 19.3 samples/s
+Step 42140/50462 | Loss: 2.6399 | LR: 3.38e-05 | 3100.6ms/step | 19.4 samples/s
+Step 42150/50462 | Loss: 2.6337 | LR: 3.37e-05 | 3100.0ms/step | 19.4 samples/s
+Step 42160/50462 | Loss: 2.5948 | LR: 3.36e-05 | 3100.3ms/step | 19.4 samples/s
+Step 42170/50462 | Loss: 2.6227 | LR: 3.35e-05 | 3100.9ms/step | 19.3 samples/s
+Step 42180/50462 | Loss: 2.6304 | LR: 3.35e-05 | 3104.6ms/step | 19.3 samples/s
+Step 42190/50462 | Loss: 2.6587 | LR: 3.34e-05 | 3099.7ms/step | 19.4 samples/s
+Step 42200/50462 | Loss: 2.6169 | LR: 3.33e-05 | 3094.2ms/step | 19.4 samples/s
+Step 42210/50462 | Loss: 2.6198 | LR: 3.32e-05 | 3104.7ms/step | 19.3 samples/s
+Step 42220/50462 | Loss: 2.6485 | LR: 3.32e-05 | 3098.1ms/step | 19.4 samples/s
+Step 42230/50462 | Loss: 2.6349 | LR: 3.31e-05 | 3103.9ms/step | 19.3 samples/s
+Step 42240/50462 | Loss: 2.5899 | LR: 3.30e-05 | 3105.4ms/step | 19.3 samples/s
+Step 42250/50462 | Loss: 2.6400 | LR: 3.29e-05 | 3098.6ms/step | 19.4 samples/s
+Validation at step 42250: Loss = 2.6401, Perplexity = 14.01
+Step 42260/50462 | Loss: 2.6462 | LR: 3.28e-05 | 4808.6ms/step | 12.5 samples/s
+Step 42270/50462 | Loss: 2.6570 | LR: 3.28e-05 | 3108.3ms/step | 19.3 samples/s
+Step 42280/50462 | Loss: 2.6274 | LR: 3.27e-05 | 3105.7ms/step | 19.3 samples/s
+Step 42290/50462 | Loss: 2.6421 | LR: 3.26e-05 | 3109.0ms/step | 19.3 samples/s
+Step 42300/50462 | Loss: 2.6278 | LR: 3.25e-05 | 3108.8ms/step | 19.3 samples/s
+Step 42310/50462 | Loss: 2.6202 | LR: 3.24e-05 | 3099.3ms/step | 19.4 samples/s
+Step 42320/50462 | Loss: 2.6038 | LR: 3.24e-05 | 3167.9ms/step | 18.9 samples/s
+Step 42330/50462 | Loss: 2.6483 | LR: 3.23e-05 | 3162.9ms/step | 19.0 samples/s
+Step 42340/50462 | Loss: 2.6577 | LR: 3.22e-05 | 3093.3ms/step | 19.4 samples/s
+Step 42350/50462 | Loss: 2.6050 | LR: 3.21e-05 | 3097.9ms/step | 19.4 samples/s
+Step 42360/50462 | Loss: 2.6165 | LR: 3.21e-05 | 3094.1ms/step | 19.4 samples/s
+Step 42370/50462 | Loss: 2.6269 | LR: 3.20e-05 | 3097.6ms/step | 19.4 samples/s
+Step 42380/50462 | Loss: 2.6129 | LR: 3.19e-05 | 3104.4ms/step | 19.3 samples/s
+Step 42390/50462 | Loss: 2.6413 | LR: 3.18e-05 | 3101.9ms/step | 19.3 samples/s
+Step 42400/50462 | Loss: 2.6494 | LR: 3.18e-05 | 3108.6ms/step | 19.3 samples/s
+Step 42410/50462 | Loss: 2.6410 | LR: 3.17e-05 | 3100.4ms/step | 19.4 samples/s
+Step 42420/50462 | Loss: 2.6341 | LR: 3.16e-05 | 3097.8ms/step | 19.4 samples/s
+Step 42430/50462 | Loss: 2.6215 | LR: 3.15e-05 | 3101.7ms/step | 19.3 samples/s
+Step 42440/50462 | Loss: 2.6232 | LR: 3.14e-05 | 3105.0ms/step | 19.3 samples/s
+Step 42450/50462 | Loss: 2.6404 | LR: 3.14e-05 | 3108.8ms/step | 19.3 samples/s
+Step 42460/50462 | Loss: 2.6193 | LR: 3.13e-05 | 3105.4ms/step | 19.3 samples/s
+Step 42470/50462 | Loss: 2.6314 | LR: 3.12e-05 | 3105.2ms/step | 19.3 samples/s
+Step 42480/50462 | Loss: 2.5899 | LR: 3.11e-05 | 3109.1ms/step | 19.3 samples/s
+Step 42490/50462 | Loss: 2.6345 | LR: 3.11e-05 | 3104.4ms/step | 19.3 samples/s
+Step 42500/50462 | Loss: 2.6459 | LR: 3.10e-05 | 3110.7ms/step | 19.3 samples/s
+Validation at step 42500: Loss = 2.6400, Perplexity = 14.01
+Step 42510/50462 | Loss: 2.6430 | LR: 3.09e-05 | 4752.2ms/step | 12.6 samples/s
+Step 42520/50462 | Loss: 2.6256 | LR: 3.08e-05 | 3101.2ms/step | 19.3 samples/s
+Step 42530/50462 | Loss: 2.6180 | LR: 3.08e-05 | 3093.9ms/step | 19.4 samples/s
+Step 42540/50462 | Loss: 2.6226 | LR: 3.07e-05 | 3105.8ms/step | 19.3 samples/s
+Step 42550/50462 | Loss: 2.6512 | LR: 3.06e-05 | 3097.9ms/step | 19.4 samples/s
+Step 42560/50462 | Loss: 2.7099 | LR: 3.05e-05 | 3105.7ms/step | 19.3 samples/s
+Step 42570/50462 | Loss: 2.6386 | LR: 3.05e-05 | 3103.5ms/step | 19.3 samples/s
+Step 42580/50462 | Loss: 2.6321 | LR: 3.04e-05 | 3091.0ms/step | 19.4 samples/s
+Step 42590/50462 | Loss: 2.6229 | LR: 3.03e-05 | 3157.2ms/step | 19.0 samples/s
+Step 42600/50462 | Loss: 2.6743 | LR: 3.02e-05 | 3086.3ms/step | 19.4 samples/s
+Step 42610/50462 | Loss: 2.6238 | LR: 3.02e-05 | 3094.3ms/step | 19.4 samples/s
+Step 42620/50462 | Loss: 2.6174 | LR: 3.01e-05 | 3102.9ms/step | 19.3 samples/s
+Step 42630/50462 | Loss: 2.6473 | LR: 3.00e-05 | 3108.1ms/step | 19.3 samples/s
+Step 42640/50462 | Loss: 2.6753 | LR: 2.99e-05 | 3171.6ms/step | 18.9 samples/s
+Step 42650/50462 | Loss: 2.6417 | LR: 2.99e-05 | 3101.9ms/step | 19.3 samples/s
+Step 42660/50462 | Loss: 2.6465 | LR: 2.98e-05 | 3158.6ms/step | 19.0 samples/s
+Step 42670/50462 | Loss: 2.6182 | LR: 2.97e-05 | 3109.0ms/step | 19.3 samples/s
+Step 42680/50462 | Loss: 2.6167 | LR: 2.96e-05 | 3104.0ms/step | 19.3 samples/s
+Step 42690/50462 | Loss: 2.6355 | LR: 2.96e-05 | 3103.9ms/step | 19.3 samples/s
+Step 42700/50462 | Loss: 2.6435 | LR: 2.95e-05 | 3103.1ms/step | 19.3 samples/s
+Step 42710/50462 | Loss: 2.6237 | LR: 2.94e-05 | 3105.6ms/step | 19.3 samples/s
+Step 42720/50462 | Loss: 2.6235 | LR: 2.93e-05 | 3108.4ms/step | 19.3 samples/s
+Step 42730/50462 | Loss: 2.6237 | LR: 2.93e-05 | 3102.3ms/step | 19.3 samples/s
+Step 42740/50462 | Loss: 2.6299 | LR: 2.92e-05 | 3096.2ms/step | 19.4 samples/s
+Step 42750/50462 | Loss: 2.6242 | LR: 2.91e-05 | 3105.1ms/step | 19.3 samples/s
+Validation at step 42750: Loss = 2.6399, Perplexity = 14.01
+Step 42760/50462 | Loss: 2.6416 | LR: 2.90e-05 | 4733.0ms/step | 12.7 samples/s
+Step 42770/50462 | Loss: 2.6302 | LR: 2.90e-05 | 3103.4ms/step | 19.3 samples/s
+Step 42780/50462 | Loss: 2.6302 | LR: 2.89e-05 | 3090.9ms/step | 19.4 samples/s
+Step 42790/50462 | Loss: 2.6477 | LR: 2.88e-05 | 3103.9ms/step | 19.3 samples/s
+Step 42800/50462 | Loss: 2.6483 | LR: 2.87e-05 | 3100.4ms/step | 19.4 samples/s
+Step 42810/50462 | Loss: 2.6538 | LR: 2.87e-05 | 3102.2ms/step | 19.3 samples/s
+Step 42820/50462 | Loss: 2.6212 | LR: 2.86e-05 | 3106.1ms/step | 19.3 samples/s
+Step 42830/50462 | Loss: 2.6162 | LR: 2.85e-05 | 3108.7ms/step | 19.3 samples/s
+Step 42840/50462 | Loss: 2.6422 | LR: 2.84e-05 | 3109.3ms/step | 19.3 samples/s
+Step 42850/50462 | Loss: 2.6320 | LR: 2.84e-05 | 3102.0ms/step | 19.3 samples/s
+Step 42860/50462 | Loss: 2.6614 | LR: 2.83e-05 | 3104.0ms/step | 19.3 samples/s
+Step 42870/50462 | Loss: 2.6432 | LR: 2.82e-05 | 3105.5ms/step | 19.3 samples/s
+Step 42880/50462 | Loss: 2.6396 | LR: 2.82e-05 | 3097.4ms/step | 19.4 samples/s
+Step 42890/50462 | Loss: 2.6219 | LR: 2.81e-05 | 3093.7ms/step | 19.4 samples/s
+Step 42900/50462 | Loss: 2.6319 | LR: 2.80e-05 | 3098.6ms/step | 19.4 samples/s
+Step 42910/50462 | Loss: 2.6611 | LR: 2.79e-05 | 3161.2ms/step | 19.0 samples/s
+Step 42920/50462 | Loss: 2.6202 | LR: 2.79e-05 | 3101.4ms/step | 19.3 samples/s
+Step 42930/50462 | Loss: 2.6184 | LR: 2.78e-05 | 3101.6ms/step | 19.3 samples/s
+Step 42940/50462 | Loss: 2.6085 | LR: 2.77e-05 | 3101.2ms/step | 19.3 samples/s
+Step 42950/50462 | Loss: 2.6597 | LR: 2.76e-05 | 3104.6ms/step | 19.3 samples/s
+Step 42960/50462 | Loss: 2.6426 | LR: 2.76e-05 | 3172.5ms/step | 18.9 samples/s
+Step 42970/50462 | Loss: 2.6142 | LR: 2.75e-05 | 3099.4ms/step | 19.4 samples/s
+Step 42980/50462 | Loss: 2.6274 | LR: 2.74e-05 | 3166.4ms/step | 18.9 samples/s
+Step 42990/50462 | Loss: 2.6351 | LR: 2.74e-05 | 3104.1ms/step | 19.3 samples/s
+Step 43000/50462 | Loss: 2.6137 | LR: 2.73e-05 | 3102.2ms/step | 19.3 samples/s
+Validation at step 43000: Loss = 2.6398, Perplexity = 14.01
+Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-43000
+Step 43010/50462 | Loss: 2.6536 | LR: 2.72e-05 | 5666.9ms/step | 10.6 samples/s
+Step 43020/50462 | Loss: 2.6259 | LR: 2.71e-05 | 3090.4ms/step | 19.4 samples/s
+Step 43030/50462 | Loss: 2.6330 | LR: 2.71e-05 | 3099.6ms/step | 19.4 samples/s
+Step 43040/50462 | Loss: 2.6176 | LR: 2.70e-05 | 3097.4ms/step | 19.4 samples/s
+Step 43050/50462 | Loss: 2.6389 | LR: 2.69e-05 | 3093.2ms/step | 19.4 samples/s
+Step 43060/50462 | Loss: 2.6145 | LR: 2.69e-05 | 3100.3ms/step | 19.4 samples/s
+Step 43070/50462 | Loss: 2.6337 | LR: 2.68e-05 | 3103.9ms/step | 19.3 samples/s
+Step 43080/50462 | Loss: 2.6110 | LR: 2.67e-05 | 3102.9ms/step | 19.3 samples/s
+Step 43090/50462 | Loss: 2.6125 | LR: 2.66e-05 | 3099.3ms/step | 19.4 samples/s
+Step 43100/50462 | Loss: 2.6059 | LR: 2.66e-05 | 3104.9ms/step | 19.3 samples/s
+Step 43110/50462 | Loss: 2.6154 | LR: 2.65e-05 | 3094.1ms/step | 19.4 samples/s
+Step 43120/50462 | Loss: 2.6289 | LR: 2.64e-05 | 3098.3ms/step | 19.4 samples/s
+Step 43130/50462 | Loss: 2.6407 | LR: 2.64e-05 | 3107.2ms/step | 19.3 samples/s
+Step 43140/50462 | Loss: 2.6105 | LR: 2.63e-05 | 3104.0ms/step | 19.3 samples/s
+Step 43150/50462 | Loss: 2.6128 | LR: 2.62e-05 | 3107.8ms/step | 19.3 samples/s
+Step 43160/50462 | Loss: 2.6500 | LR: 2.61e-05 | 3113.4ms/step | 19.3 samples/s
+Step 43170/50462 | Loss: 2.6424 | LR: 2.61e-05 | 3108.1ms/step | 19.3 samples/s
+Step 43180/50462 | Loss: 2.5981 | LR: 2.60e-05 | 3100.4ms/step | 19.4 samples/s
+Step 43190/50462 | Loss: 2.6505 | LR: 2.59e-05 | 3101.8ms/step | 19.3 samples/s
+Step 43200/50462 | Loss: 2.6305 | LR: 2.59e-05 | 3104.1ms/step | 19.3 samples/s
+Step 43210/50462 | Loss: 2.6396 | LR: 2.58e-05 | 3105.1ms/step | 19.3 samples/s
+Step 43220/50462 | Loss: 2.6427 | LR: 2.57e-05 | 3107.3ms/step | 19.3 samples/s
+Step 43230/50462 | Loss: 2.6279 | LR: 2.57e-05 | 3105.3ms/step | 19.3 samples/s
+Step 43240/50462 | Loss: 2.6531 | LR: 2.56e-05 | 3160.9ms/step | 19.0 samples/s
+Step 43250/50462 | Loss: 2.6026 | LR: 2.55e-05 | 3103.4ms/step | 19.3 samples/s
+Validation at step 43250: Loss = 2.6398, Perplexity = 14.01
+Step 43260/50462 | Loss: 2.6044 | LR: 2.54e-05 | 4752.2ms/step | 12.6 samples/s
+Step 43270/50462 | Loss: 2.6133 | LR: 2.54e-05 | 3104.1ms/step | 19.3 samples/s
+Step 43280/50462 | Loss: 2.6349 | LR: 2.53e-05 | 3105.5ms/step | 19.3 samples/s
+Step 43290/50462 | Loss: 2.6141 | LR: 2.52e-05 | 3166.2ms/step | 19.0 samples/s
+Step 43300/50462 | Loss: 2.6585 | LR: 2.52e-05 | 3164.4ms/step | 19.0 samples/s
+Step 43310/50462 | Loss: 2.6236 | LR: 2.51e-05 | 3107.1ms/step | 19.3 samples/s
+Step 43320/50462 | Loss: 2.6280 | LR: 2.50e-05 | 3108.0ms/step | 19.3 samples/s
+Step 43330/50462 | Loss: 2.6150 | LR: 2.50e-05 | 3105.9ms/step | 19.3 samples/s
+Step 43340/50462 | Loss: 2.6238 | LR: 2.49e-05 | 3104.6ms/step | 19.3 samples/s
+Step 43350/50462 | Loss: 2.6116 | LR: 2.48e-05 | 3106.1ms/step | 19.3 samples/s
+Step 43360/50462 | Loss: 2.6435 | LR: 2.48e-05 | 3105.9ms/step | 19.3 samples/s
+Step 43370/50462 | Loss: 2.6340 | LR: 2.47e-05 | 3106.5ms/step | 19.3 samples/s
+Step 43380/50462 | Loss: 2.6152 | LR: 2.46e-05 | 3106.7ms/step | 19.3 samples/s
+Step 43390/50462 | Loss: 2.6442 | LR: 2.46e-05 | 3105.3ms/step | 19.3 samples/s
+Step 43400/50462 | Loss: 2.6231 | LR: 2.45e-05 | 3104.6ms/step | 19.3 samples/s
+Step 43410/50462 | Loss: 2.6438 | LR: 2.44e-05 | 3103.5ms/step | 19.3 samples/s
+Step 43420/50462 | Loss: 2.6254 | LR: 2.44e-05 | 3104.8ms/step | 19.3 samples/s
+Step 43430/50462 | Loss: 2.6290 | LR: 2.43e-05 | 3103.8ms/step | 19.3 samples/s
+Step 43440/50462 | Loss: 2.6264 | LR: 2.42e-05 | 3107.0ms/step | 19.3 samples/s
+Step 43450/50462 | Loss: 2.6498 | LR: 2.41e-05 | 3104.3ms/step | 19.3 samples/s
+Step 43460/50462 | Loss: 2.6380 | LR: 2.41e-05 | 3105.1ms/step | 19.3 samples/s
+Step 43470/50462 | Loss: 2.6508 | LR: 2.40e-05 | 3107.1ms/step | 19.3 samples/s
+Step 43480/50462 | Loss: 2.6522 | LR: 2.39e-05 | 3107.6ms/step | 19.3 samples/s
+Step 43490/50462 | Loss: 2.6212 | LR: 2.39e-05 | 3105.9ms/step | 19.3 samples/s
+Step 43500/50462 | Loss: 2.6380 | LR: 2.38e-05 | 3105.7ms/step | 19.3 samples/s
+Validation at step 43500: Loss = 2.6397, Perplexity = 14.01
+Step 43510/50462 | Loss: 2.6416 | LR: 2.37e-05 | 4753.2ms/step | 12.6 samples/s
+Step 43520/50462 | Loss: 2.6063 | LR: 2.37e-05 | 3105.4ms/step | 19.3 samples/s
+Step 43530/50462 | Loss: 2.6292 | LR: 2.36e-05 | 3103.0ms/step | 19.3 samples/s
+Step 43540/50462 | Loss: 2.6216 | LR: 2.35e-05 | 3103.8ms/step | 19.3 samples/s
+Step 43550/50462 | Loss: 2.6310 | LR: 2.35e-05 | 3103.0ms/step | 19.3 samples/s
+Step 43560/50462 | Loss: 2.6106 | LR: 2.34e-05 | 3161.7ms/step | 19.0 samples/s
+Step 43570/50462 | Loss: 2.6299 | LR: 2.33e-05 | 3093.0ms/step | 19.4 samples/s
+Step 43580/50462 | Loss: 2.6580 | LR: 2.33e-05 | 3101.5ms/step | 19.3 samples/s
+Step 43590/50462 | Loss: 2.6199 | LR: 2.32e-05 | 3103.5ms/step | 19.3 samples/s
+Step 43600/50462 | Loss: 2.6518 | LR: 2.31e-05 | 3104.4ms/step | 19.3 samples/s
+Step 43610/50462 | Loss: 2.6343 | LR: 2.31e-05 | 3098.1ms/step | 19.4 samples/s
+Step 43620/50462 | Loss: 2.6155 | LR: 2.30e-05 | 3164.6ms/step | 19.0 samples/s
+Step 43630/50462 | Loss: 2.6549 | LR: 2.29e-05 | 3166.9ms/step | 18.9 samples/s
+Step 43640/50462 | Loss: 2.6178 | LR: 2.29e-05 | 3104.2ms/step | 19.3 samples/s
+Step 43650/50462 | Loss: 2.6365 | LR: 2.28e-05 | 3104.9ms/step | 19.3 samples/s
+Step 43660/50462 | Loss: 2.6485 | LR: 2.27e-05 | 3104.8ms/step | 19.3 samples/s
+Step 43670/50462 | Loss: 2.6379 | LR: 2.27e-05 | 3103.0ms/step | 19.3 samples/s
+Step 43680/50462 | Loss: 2.6448 | LR: 2.26e-05 | 3104.6ms/step | 19.3 samples/s
+Step 43690/50462 | Loss: 2.6368 | LR: 2.25e-05 | 3104.5ms/step | 19.3 samples/s
+Step 43700/50462 | Loss: 2.6508 | LR: 2.25e-05 | 3102.7ms/step | 19.3 samples/s
+Step 43710/50462 | Loss: 2.6564 | LR: 2.24e-05 | 3097.6ms/step | 19.4 samples/s
+Step 43720/50462 | Loss: 2.6131 | LR: 2.24e-05 | 3098.2ms/step | 19.4 samples/s
+Step 43730/50462 | Loss: 2.6036 | LR: 2.23e-05 | 3100.5ms/step | 19.4 samples/s
+Step 43740/50462 | Loss: 2.6266 | LR: 2.22e-05 | 3098.2ms/step | 19.4 samples/s
+Step 43750/50462 | Loss: 2.6281 | LR: 2.22e-05 | 3102.2ms/step | 19.3 samples/s
+Validation at step 43750: Loss = 2.6396, Perplexity = 14.01
+Step 43760/50462 | Loss: 2.6321 | LR: 2.21e-05 | 4753.0ms/step | 12.6 samples/s
+Step 43770/50462 | Loss: 2.6191 | LR: 2.20e-05 | 3103.1ms/step | 19.3 samples/s
+Step 43780/50462 | Loss: 2.6260 | LR: 2.20e-05 | 3105.2ms/step | 19.3 samples/s
+Step 43790/50462 | Loss: 2.6309 | LR: 2.19e-05 | 3103.4ms/step | 19.3 samples/s
+Step 43800/50462 | Loss: 2.6495 | LR: 2.18e-05 | 3104.2ms/step | 19.3 samples/s
+Step 43810/50462 | Loss: 2.5939 | LR: 2.18e-05 | 3105.0ms/step | 19.3 samples/s
+Step 43820/50462 | Loss: 2.5984 | LR: 2.17e-05 | 3105.8ms/step | 19.3 samples/s
+Step 43830/50462 | Loss: 2.6382 | LR: 2.16e-05 | 3106.4ms/step | 19.3 samples/s
+Step 43840/50462 | Loss: 2.6584 | LR: 2.16e-05 | 3105.7ms/step | 19.3 samples/s
+Step 43850/50462 | Loss: 2.6500 | LR: 2.15e-05 | 3106.6ms/step | 19.3 samples/s
+Step 43860/50462 | Loss: 2.6336 | LR: 2.14e-05 | 3104.8ms/step | 19.3 samples/s
+Step 43870/50462 | Loss: 2.6184 | LR: 2.14e-05 | 3099.8ms/step | 19.4 samples/s
+Step 43880/50462 | Loss: 2.6424 | LR: 2.13e-05 | 3095.6ms/step | 19.4 samples/s
+Step 43890/50462 | Loss: 2.6326 | LR: 2.13e-05 | 3161.3ms/step | 19.0 samples/s
+Step 43900/50462 | Loss: 2.6873 | LR: 2.12e-05 | 3082.7ms/step | 19.5 samples/s
+Step 43910/50462 | Loss: 2.6524 | LR: 2.11e-05 | 3098.4ms/step | 19.4 samples/s
+Step 43920/50462 | Loss: 2.6287 | LR: 2.11e-05 | 3105.9ms/step | 19.3 samples/s
+Step 43930/50462 | Loss: 2.6554 | LR: 2.10e-05 | 3100.9ms/step | 19.3 samples/s
+Step 43940/50462 | Loss: 2.6294 | LR: 2.09e-05 | 3162.8ms/step | 19.0 samples/s
+Step 43950/50462 | Loss: 2.6434 | LR: 2.09e-05 | 3102.9ms/step | 19.3 samples/s
+Step 43960/50462 | Loss: 2.6222 | LR: 2.08e-05 | 3160.7ms/step | 19.0 samples/s
+Step 43970/50462 | Loss: 2.6198 | LR: 2.07e-05 | 3092.3ms/step | 19.4 samples/s
+Step 43980/50462 | Loss: 2.6110 | LR: 2.07e-05 | 3094.1ms/step | 19.4 samples/s
+Step 43990/50462 | Loss: 2.6470 | LR: 2.06e-05 | 3096.3ms/step | 19.4 samples/s
+Step 44000/50462 | Loss: 2.6527 | LR: 2.06e-05 | 3098.4ms/step | 19.4 samples/s
+Validation at step 44000: Loss = 2.6396, Perplexity = 14.01
+Saved checkpoint to checkpoints/run_20260614_032718/checkpoint-44000
+Step 44010/50462 | Loss: 2.6386 | LR: 2.05e-05 | 5652.4ms/step | 10.6 samples/s
+Step 44020/50462 | Loss: 2.6288 | LR: 2.04e-05 | 3094.8ms/step | 19.4 samples/s
+Step 44030/50462 | Loss: 2.6275 | LR: 2.04e-05 | 3101.2ms/step | 19.3 samples/s
+Step 44040/50462 | Loss: 2.6191 | LR: 2.03e-05 | 3103.1ms/step | 19.3 samples/s
+Step 44050/50462 | Loss: 2.6543 | LR: 2.02e-05 | 3101.4ms/step | 19.3 samples/s
+Step 44060/50462 | Loss: 2.6522 | LR: 2.02e-05 | 3103.9ms/step | 19.3 samples/s
+Step 44070/50462 | Loss: 2.6536 | LR: 2.01e-05 | 3103.0ms/step | 19.3 samples/s
+Step 44080/50462 | Loss: 2.6341 | LR: 2.01e-05 | 3105.4ms/step | 19.3 samples/s
+Step 44090/50462 | Loss: 2.6013 | LR: 2.00e-05 | 3105.4ms/step | 19.3 samples/s
+Step 44100/50462 | Loss: 2.6217 | LR: 1.99e-05 | 3106.4ms/step | 19.3 samples/s
+Step 44110/50462 | Loss: 2.6240 | LR: 1.99e-05 | 3104.4ms/step | 19.3 samples/s
+Step 44120/50462 | Loss: 2.6411 | LR: 1.98e-05 | 3097.7ms/step | 19.4 samples/s
+Step 44130/50462 | Loss: 2.6472 | LR: 1.97e-05 | 3096.4ms/step | 19.4 samples/s
+Step 44140/50462 | Loss: 2.6183 | LR: 1.97e-05 | 3100.6ms/step | 19.4 samples/s
+Step 44150/50462 | Loss: 2.6123 | LR: 1.96e-05 | 3099.9ms/step | 19.4 samples/s
+Step 44160/50462 | Loss: 2.6386 | LR: 1.96e-05 | 3100.9ms/step | 19.3 samples/s
+Step 44170/50462 | Loss: 2.6467 | LR: 1.95e-05 | 3098.0ms/step | 19.4 samples/s
+Step 44180/50462 | Loss: 2.6136 | LR: 1.94e-05 | 3100.6ms/step | 19.4 samples/s
+Step 44190/50462 | Loss: 2.6188 | LR: 1.94e-05 | 3100.9ms/step | 19.3 samples/s
+Step 44200/50462 | Loss: 2.6393 | LR: 1.93e-05 | 3101.5ms/step | 19.3 samples/s
+Step 44210/50462 | Loss: 2.6293 | LR: 1.93e-05 | 3161.4ms/step | 19.0 samples/s
+Step 44220/50462 | Loss: 2.6317 | LR: 1.92e-05 | 3096.4ms/step | 19.4 samples/s
+Step 44230/50462 | Loss: 2.6451 | LR: 1.91e-05 | 3100.4ms/step | 19.4 samples/s
+Step 44240/50462 | Loss: 2.6336 | LR: 1.91e-05 | 3100.1ms/step | 19.4 samples/s
+Step 44250/50462 | Loss: 2.6451 | LR: 1.90e-05 | 3101.7ms/step | 19.3 samples/s
+Validation at step 44250: Loss = 2.6396, Perplexity = 14.01
+Step 44260/50462 | Loss: 2.6393 | LR: 1.90e-05 | 4759.9ms/step | 12.6 samples/s
+Step 44270/50462 | Loss: 2.6144 | LR: 1.89e-05 | 3162.5ms/step | 19.0 samples/s
+Step 44280/50462 | Loss: 2.6330 | LR: 1.88e-05 | 3161.3ms/step | 19.0 samples/s
+Step 44290/50462 | Loss: 2.6317 | LR: 1.88e-05 | 3096.5ms/step | 19.4 samples/s
+Step 44300/50462 | Loss: 2.5841 | LR: 1.87e-05 | 3101.5ms/step | 19.3 samples/s
+Step 44310/50462 | Loss: 2.6100 | LR: 1.87e-05 | 3103.8ms/step | 19.3 samples/s
+Step 44320/50462 | Loss: 2.6443 | LR: 1.86e-05 | 3103.4ms/step | 19.3 samples/s
+Step 44330/50462 | Loss: 2.6365 | LR: 1.85e-05 | 3103.7ms/step | 19.3 samples/s
+Step 44340/50462 | Loss: 2.6298 | LR: 1.85e-05 | 3107.3ms/step | 19.3 samples/s
+Step 44350/50462 | Loss: 2.6370 | LR: 1.84e-05 | 3105.9ms/step | 19.3 samples/s
+Step 44360/50462 | Loss: 2.6084 | LR: 1.84e-05 | 3106.6ms/step | 19.3 samples/s
+Step 44370/50462 | Loss: 2.6344 | LR: 1.83e-05 | 3108.1ms/step | 19.3 samples/s
+Step 44380/50462 | Loss: 2.6256 | LR: 1.82e-05 | 3107.0ms/step | 19.3 samples/s
+Step 44390/50462 | Loss: 2.6452 | LR: 1.82e-05 | 3107.1ms/step | 19.3 samples/s
+Step 44400/50462 | Loss: 2.6140 | LR: 1.81e-05 | 3105.4ms/step | 19.3 samples/s
+Step 44410/50462 | Loss: 2.6197 | LR: 1.81e-05 | 3102.4ms/step | 19.3 samples/s
+Step 44420/50462 | Loss: 2.6305 | LR: 1.80e-05 | 3104.1ms/step | 19.3 samples/s
+Step 44430/50462 | Loss: 2.6395 | LR: 1.79e-05 | 3105.4ms/step | 19.3 samples/s
+Step 44440/50462 | Loss: 2.6276 | LR: 1.79e-05 | 3113.4ms/step | 19.3 samples/s
+Step 44450/50462 | Loss: 2.6148 | LR: 1.78e-05 | 3110.0ms/step | 19.3 samples/s
+Step 44460/50462 | Loss: 2.6332 | LR: 1.78e-05 | 3113.9ms/step | 19.3 samples/s
+Step 44470/50462 | Loss: 2.6444 | LR: 1.77e-05 | 3115.9ms/step | 19.3 samples/s

epfl_final/log_files/sparse_30%.log

@@ -0,0 +1,367 @@
+Auto-resuming from latest checkpoint: checkpoints/run_20260517_023855/checkpoint-18000
+Resuming: Using existing output directory checkpoints/run_20260517_023855
+Resuming wandb run hi2z3jh6
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.27.0
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260518_025234-hi2z3jh6
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run sparse 30%
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/hi2z3jh6
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "checkpoints/run_20260517_023855",
+  "resume_from_checkpoint": "checkpoints/run_20260517_023855/checkpoint-18000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "sparse 30%",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 18,
+  "sparse_mask_pool_size": 2000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_19",
+  "dense_mode": false
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: checkpoints/run_20260517_023855/checkpoint-18000
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 18
+  sparse_mask_pool_size: 2000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from checkpoints/run_20260517_023855/checkpoint-18000...
+Resumed from checkpoint at step 17999 (epoch 0, micro_step 144016)
+Resuming training from step 17999
+Starting training...
+Step 18000/20005 | Loss: 0.2871 | LR: 1.26e-05 | 347868.6ms/step | 0.1 samples/s
+wandb: WARNING Tried to log to step 18000 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 18000: Loss = 2.9449, Perplexity = 19.01
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-18000
+wandb: WARNING Tried to log to step 18000 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18010/20005 | Loss: 2.9262 | LR: 1.24e-05 | 20258.4ms/step | 1.2 samples/s
+wandb: WARNING Tried to log to step 18010 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18020/20005 | Loss: 2.9380 | LR: 1.23e-05 | 2325.3ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18020 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18030/20005 | Loss: 2.9472 | LR: 1.22e-05 | 2322.8ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18030 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18040/20005 | Loss: 2.9313 | LR: 1.21e-05 | 2315.8ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18040 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18050/20005 | Loss: 2.9340 | LR: 1.19e-05 | 2315.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18050 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18060/20005 | Loss: 2.9068 | LR: 1.18e-05 | 2316.8ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18060 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18070/20005 | Loss: 2.8988 | LR: 1.17e-05 | 2315.9ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18070 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18080/20005 | Loss: 2.9077 | LR: 1.16e-05 | 2314.4ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18080 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18090/20005 | Loss: 2.9596 | LR: 1.15e-05 | 2314.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18090 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18100/20005 | Loss: 2.9119 | LR: 1.13e-05 | 2316.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18100 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18110/20005 | Loss: 2.9399 | LR: 1.12e-05 | 2315.4ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18110 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18120/20005 | Loss: 2.9164 | LR: 1.11e-05 | 2315.4ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18120 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18130/20005 | Loss: 2.9187 | LR: 1.10e-05 | 2321.0ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18130 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18140/20005 | Loss: 2.9256 | LR: 1.09e-05 | 2315.5ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18140 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18150/20005 | Loss: 2.9161 | LR: 1.08e-05 | 2318.5ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18150 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18160/20005 | Loss: 2.9449 | LR: 1.06e-05 | 2323.3ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18160 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18170/20005 | Loss: 2.9103 | LR: 1.05e-05 | 2322.9ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18170 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18180/20005 | Loss: 2.9492 | LR: 1.04e-05 | 2322.2ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18180 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18190/20005 | Loss: 2.9149 | LR: 1.03e-05 | 2318.4ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18190 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18200/20005 | Loss: 2.9357 | LR: 1.02e-05 | 2313.1ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18200 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18210/20005 | Loss: 2.9313 | LR: 1.01e-05 | 2313.5ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18210 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18220/20005 | Loss: 2.9328 | LR: 9.97e-06 | 2310.1ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18220 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18230/20005 | Loss: 2.9211 | LR: 9.86e-06 | 2314.0ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18230 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18240/20005 | Loss: 2.9613 | LR: 9.75e-06 | 2312.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18240 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18250/20005 | Loss: 2.9196 | LR: 9.64e-06 | 2310.7ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18250 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 18250: Loss = 2.9449, Perplexity = 19.01
+wandb: WARNING Tried to log to step 18250 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18260/20005 | Loss: 2.9467 | LR: 9.53e-06 | 17425.7ms/step | 1.4 samples/s
+wandb: WARNING Tried to log to step 18260 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18270/20005 | Loss: 2.9209 | LR: 9.42e-06 | 2315.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18270 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18280/20005 | Loss: 2.9144 | LR: 9.32e-06 | 2314.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18280 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18290/20005 | Loss: 2.9284 | LR: 9.21e-06 | 2312.5ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18290 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18300/20005 | Loss: 2.9517 | LR: 9.10e-06 | 2314.0ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18300 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18310/20005 | Loss: 2.9304 | LR: 9.00e-06 | 2315.0ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18310 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18320/20005 | Loss: 2.9358 | LR: 8.89e-06 | 2315.1ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18320 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18330/20005 | Loss: 2.9497 | LR: 8.79e-06 | 2314.7ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18330 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18340/20005 | Loss: 2.9259 | LR: 8.68e-06 | 2313.7ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18340 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18350/20005 | Loss: 2.9214 | LR: 8.58e-06 | 2314.9ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18350 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18360/20005 | Loss: 2.9279 | LR: 8.47e-06 | 2315.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18360 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18370/20005 | Loss: 2.9385 | LR: 8.37e-06 | 2482.2ms/step | 9.7 samples/s
+wandb: WARNING Tried to log to step 18370 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18380/20005 | Loss: 2.9436 | LR: 8.27e-06 | 2321.8ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18380 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18390/20005 | Loss: 2.9214 | LR: 8.17e-06 | 2324.1ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18390 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18400/20005 | Loss: 2.9564 | LR: 8.07e-06 | 2323.5ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18400 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18410/20005 | Loss: 2.9666 | LR: 7.97e-06 | 2326.4ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18410 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18420/20005 | Loss: 2.9280 | LR: 7.87e-06 | 2322.6ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18420 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18430/20005 | Loss: 2.9215 | LR: 7.77e-06 | 2324.1ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18430 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18440/20005 | Loss: 2.9387 | LR: 7.67e-06 | 2321.5ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18440 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18450/20005 | Loss: 2.9111 | LR: 7.58e-06 | 2322.5ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18450 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18460/20005 | Loss: 2.9352 | LR: 7.48e-06 | 2323.8ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18460 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18470/20005 | Loss: 2.9359 | LR: 7.38e-06 | 2322.9ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18470 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18480/20005 | Loss: 2.9074 | LR: 7.29e-06 | 2323.3ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18480 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18490/20005 | Loss: 2.9367 | LR: 7.19e-06 | 2324.8ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18490 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18500/20005 | Loss: 2.9431 | LR: 7.10e-06 | 2324.3ms/step | 10.3 samples/s
+Validation at step 18500: Loss = 2.9448, Perplexity = 19.01
+Step 18510/20005 | Loss: 2.9130 | LR: 7.00e-06 | 17470.7ms/step | 1.4 samples/s
+Step 18520/20005 | Loss: 2.9038 | LR: 6.91e-06 | 2322.2ms/step | 10.3 samples/s
+Step 18530/20005 | Loss: 2.9199 | LR: 6.82e-06 | 2322.1ms/step | 10.3 samples/s
+Step 18540/20005 | Loss: 2.9125 | LR: 6.73e-06 | 2323.1ms/step | 10.3 samples/s
+Step 18550/20005 | Loss: 2.9177 | LR: 6.64e-06 | 2323.3ms/step | 10.3 samples/s
+Step 18560/20005 | Loss: 2.9444 | LR: 6.54e-06 | 2324.7ms/step | 10.3 samples/s
+Step 18570/20005 | Loss: 2.9868 | LR: 6.45e-06 | 2322.0ms/step | 10.3 samples/s
+Step 18580/20005 | Loss: 2.9507 | LR: 6.37e-06 | 2325.7ms/step | 10.3 samples/s
+Step 18590/20005 | Loss: 2.9149 | LR: 6.28e-06 | 2322.9ms/step | 10.3 samples/s
+Step 18600/20005 | Loss: 2.9023 | LR: 6.19e-06 | 2322.2ms/step | 10.3 samples/s
+Step 18610/20005 | Loss: 2.9032 | LR: 6.10e-06 | 2323.1ms/step | 10.3 samples/s
+Step 18620/20005 | Loss: 2.9311 | LR: 6.01e-06 | 2324.1ms/step | 10.3 samples/s
+Step 18630/20005 | Loss: 2.9542 | LR: 5.93e-06 | 2321.4ms/step | 10.3 samples/s
+Step 18640/20005 | Loss: 2.9366 | LR: 5.84e-06 | 2323.6ms/step | 10.3 samples/s
+Step 18650/20005 | Loss: 2.9361 | LR: 5.76e-06 | 2321.7ms/step | 10.3 samples/s
+Step 18660/20005 | Loss: 2.9005 | LR: 5.67e-06 | 2323.5ms/step | 10.3 samples/s
+Step 18670/20005 | Loss: 2.9555 | LR: 5.59e-06 | 2323.3ms/step | 10.3 samples/s
+Step 18680/20005 | Loss: 2.9134 | LR: 5.50e-06 | 2321.8ms/step | 10.3 samples/s
+Step 18690/20005 | Loss: 2.9358 | LR: 5.42e-06 | 2322.4ms/step | 10.3 samples/s
+Step 18700/20005 | Loss: 2.9642 | LR: 5.34e-06 | 2324.8ms/step | 10.3 samples/s
+Step 18710/20005 | Loss: 2.9552 | LR: 5.26e-06 | 2324.6ms/step | 10.3 samples/s
+Step 18720/20005 | Loss: 2.9366 | LR: 5.18e-06 | 2321.9ms/step | 10.3 samples/s
+Step 18730/20005 | Loss: 2.9471 | LR: 5.10e-06 | 2324.7ms/step | 10.3 samples/s
+Step 18740/20005 | Loss: 2.9533 | LR: 5.02e-06 | 2322.0ms/step | 10.3 samples/s
+Step 18750/20005 | Loss: 2.9640 | LR: 4.94e-06 | 2324.2ms/step | 10.3 samples/s
+Validation at step 18750: Loss = 2.9448, Perplexity = 19.01
+Step 18760/20005 | Loss: 2.9110 | LR: 4.86e-06 | 17455.1ms/step | 1.4 samples/s
+Step 18770/20005 | Loss: 2.9430 | LR: 4.78e-06 | 2324.1ms/step | 10.3 samples/s
+Step 18780/20005 | Loss: 2.9269 | LR: 4.71e-06 | 2324.1ms/step | 10.3 samples/s
+Step 18790/20005 | Loss: 2.9330 | LR: 4.63e-06 | 2322.9ms/step | 10.3 samples/s
+Step 18800/20005 | Loss: 2.9265 | LR: 4.55e-06 | 2324.7ms/step | 10.3 samples/s
+Step 18810/20005 | Loss: 2.9540 | LR: 4.48e-06 | 2322.3ms/step | 10.3 samples/s
+Step 18820/20005 | Loss: 2.9309 | LR: 4.40e-06 | 2323.8ms/step | 10.3 samples/s
+Step 18830/20005 | Loss: 2.9221 | LR: 4.33e-06 | 2322.5ms/step | 10.3 samples/s
+Step 18840/20005 | Loss: 2.9307 | LR: 4.26e-06 | 2324.1ms/step | 10.3 samples/s
+Step 18850/20005 | Loss: 2.9446 | LR: 4.18e-06 | 2324.1ms/step | 10.3 samples/s
+Step 18860/20005 | Loss: 2.9341 | LR: 4.11e-06 | 2325.6ms/step | 10.3 samples/s
+Step 18870/20005 | Loss: 2.9326 | LR: 4.04e-06 | 2320.8ms/step | 10.3 samples/s
+Step 18880/20005 | Loss: 2.9186 | LR: 3.97e-06 | 2324.9ms/step | 10.3 samples/s
+Step 18890/20005 | Loss: 2.8829 | LR: 3.90e-06 | 2321.1ms/step | 10.3 samples/s
+Step 18900/20005 | Loss: 2.9331 | LR: 3.83e-06 | 2323.1ms/step | 10.3 samples/s
+Step 18910/20005 | Loss: 2.9285 | LR: 3.76e-06 | 2322.2ms/step | 10.3 samples/s
+Step 18920/20005 | Loss: 2.9191 | LR: 3.69e-06 | 2324.2ms/step | 10.3 samples/s
+Step 18930/20005 | Loss: 2.9445 | LR: 3.62e-06 | 2322.4ms/step | 10.3 samples/s
+Step 18940/20005 | Loss: 2.9198 | LR: 3.56e-06 | 2322.4ms/step | 10.3 samples/s
+Step 18950/20005 | Loss: 2.9121 | LR: 3.49e-06 | 2322.2ms/step | 10.3 samples/s
+Step 18960/20005 | Loss: 2.9489 | LR: 3.42e-06 | 2322.4ms/step | 10.3 samples/s
+Step 18970/20005 | Loss: 2.9495 | LR: 3.36e-06 | 2323.5ms/step | 10.3 samples/s
+Step 18980/20005 | Loss: 2.9574 | LR: 3.29e-06 | 2324.0ms/step | 10.3 samples/s
+Step 18990/20005 | Loss: 2.9411 | LR: 3.23e-06 | 2323.4ms/step | 10.3 samples/s
+Step 19000/20005 | Loss: 2.9165 | LR: 3.17e-06 | 2323.8ms/step | 10.3 samples/s
+Validation at step 19000: Loss = 2.9448, Perplexity = 19.01
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-19000
+Step 19010/20005 | Loss: 2.9485 | LR: 3.10e-06 | 17957.6ms/step | 1.3 samples/s
+Step 19020/20005 | Loss: 2.9212 | LR: 3.04e-06 | 2320.5ms/step | 10.3 samples/s
+Step 19030/20005 | Loss: 2.9169 | LR: 2.98e-06 | 2321.8ms/step | 10.3 samples/s
+Step 19040/20005 | Loss: 2.9269 | LR: 2.92e-06 | 2319.3ms/step | 10.3 samples/s
+Step 19050/20005 | Loss: 2.9127 | LR: 2.86e-06 | 2323.1ms/step | 10.3 samples/s
+Step 19060/20005 | Loss: 2.9314 | LR: 2.80e-06 | 2320.0ms/step | 10.3 samples/s
+Step 19070/20005 | Loss: 2.9063 | LR: 2.74e-06 | 2321.0ms/step | 10.3 samples/s
+Step 19080/20005 | Loss: 2.9503 | LR: 2.68e-06 | 2322.3ms/step | 10.3 samples/s
+Step 19090/20005 | Loss: 2.9136 | LR: 2.62e-06 | 2322.7ms/step | 10.3 samples/s
+Step 19100/20005 | Loss: 2.9468 | LR: 2.57e-06 | 2320.4ms/step | 10.3 samples/s
+Step 19110/20005 | Loss: 2.9347 | LR: 2.51e-06 | 2321.7ms/step | 10.3 samples/s
+Step 19120/20005 | Loss: 2.9539 | LR: 2.46e-06 | 2323.6ms/step | 10.3 samples/s
+Step 19130/20005 | Loss: 2.9157 | LR: 2.40e-06 | 2323.4ms/step | 10.3 samples/s
+Step 19140/20005 | Loss: 2.9245 | LR: 2.35e-06 | 2322.1ms/step | 10.3 samples/s
+Step 19150/20005 | Loss: 2.9305 | LR: 2.29e-06 | 2322.3ms/step | 10.3 samples/s
+Step 19160/20005 | Loss: 2.9189 | LR: 2.24e-06 | 2322.3ms/step | 10.3 samples/s
+Step 19170/20005 | Loss: 2.9229 | LR: 2.19e-06 | 2320.1ms/step | 10.3 samples/s
+Step 19180/20005 | Loss: 2.9198 | LR: 2.13e-06 | 2321.5ms/step | 10.3 samples/s
+Step 19190/20005 | Loss: 2.9404 | LR: 2.08e-06 | 2321.7ms/step | 10.3 samples/s
+Step 19200/20005 | Loss: 2.9275 | LR: 2.03e-06 | 2320.3ms/step | 10.3 samples/s
+Step 19210/20005 | Loss: 2.8980 | LR: 1.98e-06 | 2321.6ms/step | 10.3 samples/s
+Step 19220/20005 | Loss: 2.9544 | LR: 1.93e-06 | 2321.7ms/step | 10.3 samples/s
+Step 19230/20005 | Loss: 2.9006 | LR: 1.88e-06 | 2321.7ms/step | 10.3 samples/s
+Step 19240/20005 | Loss: 2.9618 | LR: 1.83e-06 | 2322.7ms/step | 10.3 samples/s
+Step 19250/20005 | Loss: 2.9312 | LR: 1.79e-06 | 2322.5ms/step | 10.3 samples/s
+Validation at step 19250: Loss = 2.9449, Perplexity = 19.01
+Step 19260/20005 | Loss: 2.9488 | LR: 1.74e-06 | 17457.2ms/step | 1.4 samples/s
+Step 19270/20005 | Loss: 2.9053 | LR: 1.69e-06 | 2323.7ms/step | 10.3 samples/s
+Step 19280/20005 | Loss: 2.9239 | LR: 1.65e-06 | 2322.0ms/step | 10.3 samples/s
+Step 19290/20005 | Loss: 2.9398 | LR: 1.60e-06 | 2324.0ms/step | 10.3 samples/s
+Step 19300/20005 | Loss: 2.9380 | LR: 1.56e-06 | 2320.1ms/step | 10.3 samples/s
+Step 19310/20005 | Loss: 2.9300 | LR: 1.51e-06 | 2324.3ms/step | 10.3 samples/s
+Step 19320/20005 | Loss: 2.9248 | LR: 1.47e-06 | 2323.7ms/step | 10.3 samples/s
+Step 19330/20005 | Loss: 2.9279 | LR: 1.43e-06 | 2323.4ms/step | 10.3 samples/s
+Step 19340/20005 | Loss: 2.9459 | LR: 1.38e-06 | 2325.4ms/step | 10.3 samples/s
+Step 19350/20005 | Loss: 2.9041 | LR: 1.34e-06 | 2322.1ms/step | 10.3 samples/s
+Step 19360/20005 | Loss: 2.9376 | LR: 1.30e-06 | 2322.6ms/step | 10.3 samples/s
+Step 19370/20005 | Loss: 2.9294 | LR: 1.26e-06 | 2322.7ms/step | 10.3 samples/s

epfl_final/log_files/sparse_30%_continued.log

@@ -0,0 +1,226 @@
+New run: Checkpoints will be saved to ./checkpoints/run_20260519_212829
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.27.0
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260519_212833-oeak7lgl
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dense continued 8k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/oeak7lgl
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260519_212829",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000",
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 8k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000...
+Resumed from checkpoint at step 17999 (epoch 0, micro_step 144024)
+Resuming training from step 17999
+Starting training...
+Step 18000/20005 | Loss: 0.2932 | LR: 1.26e-05 | 638407.9ms/step | 0.0 samples/s
+Validation at step 18000: Loss = 2.9421, Perplexity = 18.96
+Saved checkpoint to checkpoints/run_20260519_212829/checkpoint-18000
+Step 18010/20005 | Loss: 2.9429 | LR: 1.24e-05 | 44368.4ms/step | 0.5 samples/s
+Step 18020/20005 | Loss: 2.8996 | LR: 1.23e-05 | 5892.3ms/step | 4.1 samples/s
+Step 18030/20005 | Loss: 2.8949 | LR: 1.22e-05 | 5974.8ms/step | 4.0 samples/s
+Step 18040/20005 | Loss: 2.8822 | LR: 1.21e-05 | 5889.9ms/step | 4.1 samples/s
+Step 18050/20005 | Loss: 2.9279 | LR: 1.19e-05 | 5891.4ms/step | 4.1 samples/s
+Step 18060/20005 | Loss: 2.9004 | LR: 1.18e-05 | 5883.8ms/step | 4.1 samples/s
+Step 18070/20005 | Loss: 2.8831 | LR: 1.17e-05 | 5895.3ms/step | 4.1 samples/s
+Step 18080/20005 | Loss: 2.8618 | LR: 1.16e-05 | 5892.2ms/step | 4.1 samples/s
+Step 18090/20005 | Loss: 2.8818 | LR: 1.15e-05 | 5892.1ms/step | 4.1 samples/s
+Step 18100/20005 | Loss: 2.8697 | LR: 1.13e-05 | 5898.6ms/step | 4.1 samples/s
+Step 18110/20005 | Loss: 2.8987 | LR: 1.12e-05 | 5892.7ms/step | 4.1 samples/s
+Step 18120/20005 | Loss: 2.9018 | LR: 1.11e-05 | 5889.0ms/step | 4.1 samples/s
+Step 18130/20005 | Loss: 2.9144 | LR: 1.10e-05 | 5893.2ms/step | 4.1 samples/s
+Step 18140/20005 | Loss: 2.9089 | LR: 1.09e-05 | 5888.2ms/step | 4.1 samples/s
+Step 18150/20005 | Loss: 2.8999 | LR: 1.08e-05 | 5889.4ms/step | 4.1 samples/s
+Step 18160/20005 | Loss: 2.8862 | LR: 1.06e-05 | 5887.0ms/step | 4.1 samples/s
+Step 18170/20005 | Loss: 2.9011 | LR: 1.05e-05 | 5886.8ms/step | 4.1 samples/s
+Step 18180/20005 | Loss: 2.9096 | LR: 1.04e-05 | 5897.3ms/step | 4.1 samples/s
+Step 18190/20005 | Loss: 2.8956 | LR: 1.03e-05 | 5897.9ms/step | 4.1 samples/s
+Step 18200/20005 | Loss: 2.8945 | LR: 1.02e-05 | 5894.8ms/step | 4.1 samples/s
+Step 18210/20005 | Loss: 2.8873 | LR: 1.01e-05 | 5887.9ms/step | 4.1 samples/s
+Step 18220/20005 | Loss: 2.8870 | LR: 9.96e-06 | 5889.2ms/step | 4.1 samples/s
+Step 18230/20005 | Loss: 2.9008 | LR: 9.85e-06 | 5886.2ms/step | 4.1 samples/s
+Step 18240/20005 | Loss: 2.8994 | LR: 9.74e-06 | 5890.5ms/step | 4.1 samples/s
+Step 18250/20005 | Loss: 2.9039 | LR: 9.63e-06 | 5887.4ms/step | 4.1 samples/s
+Validation at step 18250: Loss = 2.9114, Perplexity = 18.38
+Step 18260/20005 | Loss: 2.8679 | LR: 9.52e-06 | 40989.8ms/step | 0.6 samples/s
+Step 18270/20005 | Loss: 2.8799 | LR: 9.41e-06 | 5890.8ms/step | 4.1 samples/s
+Step 18280/20005 | Loss: 2.8720 | LR: 9.30e-06 | 5890.4ms/step | 4.1 samples/s
+Step 18290/20005 | Loss: 2.8980 | LR: 9.20e-06 | 5890.3ms/step | 4.1 samples/s
+Step 18300/20005 | Loss: 2.9070 | LR: 9.09e-06 | 5890.3ms/step | 4.1 samples/s
+Step 18310/20005 | Loss: 2.8755 | LR: 8.99e-06 | 5888.9ms/step | 4.1 samples/s
+Step 18320/20005 | Loss: 2.9267 | LR: 8.88e-06 | 5891.0ms/step | 4.1 samples/s
+Step 18330/20005 | Loss: 2.9028 | LR: 8.78e-06 | 5887.3ms/step | 4.1 samples/s
+Step 18340/20005 | Loss: 2.8972 | LR: 8.67e-06 | 5883.4ms/step | 4.1 samples/s
+Step 18350/20005 | Loss: 2.9147 | LR: 8.57e-06 | 5882.8ms/step | 4.1 samples/s
+Step 18360/20005 | Loss: 2.8991 | LR: 8.46e-06 | 5897.9ms/step | 4.1 samples/s
+Step 18370/20005 | Loss: 2.9054 | LR: 8.36e-06 | 5898.7ms/step | 4.1 samples/s
+Step 18380/20005 | Loss: 2.8886 | LR: 8.26e-06 | 5897.2ms/step | 4.1 samples/s
+Step 18390/20005 | Loss: 2.8998 | LR: 8.16e-06 | 5885.8ms/step | 4.1 samples/s
+Step 18400/20005 | Loss: 2.9087 | LR: 8.06e-06 | 5890.7ms/step | 4.1 samples/s
+Step 18410/20005 | Loss: 2.8897 | LR: 7.96e-06 | 5889.8ms/step | 4.1 samples/s
+Step 18420/20005 | Loss: 2.8887 | LR: 7.86e-06 | 5887.1ms/step | 4.1 samples/s
+Step 18430/20005 | Loss: 2.8875 | LR: 7.76e-06 | 5885.5ms/step | 4.1 samples/s
+Step 18440/20005 | Loss: 2.8767 | LR: 7.66e-06 | 5888.8ms/step | 4.1 samples/s
+Step 18450/20005 | Loss: 2.9146 | LR: 7.57e-06 | 5886.7ms/step | 4.1 samples/s
+Step 18460/20005 | Loss: 2.8945 | LR: 7.47e-06 | 5884.0ms/step | 4.1 samples/s
+Step 18470/20005 | Loss: 2.9128 | LR: 7.37e-06 | 5895.7ms/step | 4.1 samples/s
+Step 18480/20005 | Loss: 2.9192 | LR: 7.28e-06 | 5897.7ms/step | 4.1 samples/s
+Step 18490/20005 | Loss: 2.8914 | LR: 7.18e-06 | 5895.0ms/step | 4.1 samples/s
+Step 18500/20005 | Loss: 2.8621 | LR: 7.09e-06 | 5954.1ms/step | 4.0 samples/s
+Validation at step 18500: Loss = 2.9103, Perplexity = 18.36
+Step 18510/20005 | Loss: 2.9280 | LR: 6.99e-06 | 41109.7ms/step | 0.6 samples/s
+Step 18520/20005 | Loss: 2.8590 | LR: 6.90e-06 | 5906.6ms/step | 4.1 samples/s
+Step 18530/20005 | Loss: 2.8556 | LR: 6.81e-06 | 5913.0ms/step | 4.1 samples/s
+Step 18540/20005 | Loss: 2.8744 | LR: 6.72e-06 | 5908.0ms/step | 4.1 samples/s
+Step 18550/20005 | Loss: 2.8819 | LR: 6.63e-06 | 5907.4ms/step | 4.1 samples/s
+Step 18560/20005 | Loss: 2.8989 | LR: 6.54e-06 | 5904.5ms/step | 4.1 samples/s
+Step 18570/20005 | Loss: 2.8603 | LR: 6.45e-06 | 5903.6ms/step | 4.1 samples/s
+Step 18580/20005 | Loss: 2.9193 | LR: 6.36e-06 | 5907.4ms/step | 4.1 samples/s
+Step 18590/20005 | Loss: 2.8858 | LR: 6.27e-06 | 5903.6ms/step | 4.1 samples/s
+Step 18600/20005 | Loss: 2.9107 | LR: 6.18e-06 | 5915.9ms/step | 4.1 samples/s
+Step 18610/20005 | Loss: 2.9068 | LR: 6.09e-06 | 5902.7ms/step | 4.1 samples/s

epfl_final/log_files/sparse_pretrain_85%.log

@@ -0,0 +1,734 @@
+jnohup: ignoring input
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+New run: Checkpoints will be saved to ./checkpoints/run_20260613_222732
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260613_222732",
+  "resume_from_checkpoint": null,
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "sparse 85%",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "sparse"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 9,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "mask_cache_10",
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: False
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+Checkpoint activations: False
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 9
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 1536]           [2, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 1536]           [2, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [2, 8192, 1536]           [2, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (G): 2.87
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 52437.19
+Params size (MB): 2871.45
+Estimated Total Size (MB): 55308.77
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        3
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504627
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.27.2
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260613_222959-rp96qdel
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run sparse 85%
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rp96qdel
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 10/50462 | Loss: 12.2625 | LR: 6.67e-06 | 8182.9ms/step | 7.3 samples/s
+Step 20/50462 | Loss: 12.0390 | LR: 1.33e-05 | 2751.4ms/step | 21.8 samples/s
+Step 30/50462 | Loss: 11.2242 | LR: 2.00e-05 | 2756.9ms/step | 21.8 samples/s
+Step 40/50462 | Loss: 10.2579 | LR: 2.67e-05 | 2751.4ms/step | 21.8 samples/s
+Step 50/50462 | Loss: 9.8189 | LR: 3.33e-05 | 2750.1ms/step | 21.8 samples/s
+Step 60/50462 | Loss: 9.3643 | LR: 4.00e-05 | 2749.4ms/step | 21.8 samples/s
+Step 70/50462 | Loss: 8.8529 | LR: 4.67e-05 | 2749.5ms/step | 21.8 samples/s
+Step 80/50462 | Loss: 8.3457 | LR: 5.33e-05 | 2750.8ms/step | 21.8 samples/s
+Step 90/50462 | Loss: 7.9054 | LR: 6.00e-05 | 2753.4ms/step | 21.8 samples/s
+Step 100/50462 | Loss: 7.5376 | LR: 6.67e-05 | 2751.6ms/step | 21.8 samples/s
+Step 110/50462 | Loss: 7.2801 | LR: 7.33e-05 | 2754.5ms/step | 21.8 samples/s
+Step 120/50462 | Loss: 7.1211 | LR: 8.00e-05 | 2758.6ms/step | 21.7 samples/s
+Step 130/50462 | Loss: 7.0000 | LR: 8.67e-05 | 2761.6ms/step | 21.7 samples/s
+Step 140/50462 | Loss: 6.8729 | LR: 9.33e-05 | 2761.3ms/step | 21.7 samples/s
+Step 150/50462 | Loss: 6.7415 | LR: 1.00e-04 | 2760.7ms/step | 21.7 samples/s
+Step 160/50462 | Loss: 6.6354 | LR: 1.07e-04 | 2761.9ms/step | 21.7 samples/s
+Step 170/50462 | Loss: 6.5196 | LR: 1.13e-04 | 2759.5ms/step | 21.7 samples/s
+Step 180/50462 | Loss: 6.4332 | LR: 1.20e-04 | 2759.1ms/step | 21.7 samples/s
+Step 190/50462 | Loss: 6.3651 | LR: 1.27e-04 | 2762.1ms/step | 21.7 samples/s
+Step 200/50462 | Loss: 6.2856 | LR: 1.33e-04 | 2765.8ms/step | 21.7 samples/s
+Step 210/50462 | Loss: 6.2157 | LR: 1.40e-04 | 2763.2ms/step | 21.7 samples/s
+Step 220/50462 | Loss: 6.1514 | LR: 1.47e-04 | 2764.4ms/step | 21.7 samples/s
+Step 230/50462 | Loss: 6.1246 | LR: 1.53e-04 | 2764.8ms/step | 21.7 samples/s
+Step 240/50462 | Loss: 6.0809 | LR: 1.60e-04 | 2765.7ms/step | 21.7 samples/s
+Step 250/50462 | Loss: 6.0134 | LR: 1.67e-04 | 2766.9ms/step | 21.7 samples/s
+Validation at step 250: Loss = 5.9597, Perplexity = 387.50
+Step 260/50462 | Loss: 5.9535 | LR: 1.73e-04 | 6912.4ms/step | 8.7 samples/s
+Step 270/50462 | Loss: 5.9490 | LR: 1.80e-04 | 2761.4ms/step | 21.7 samples/s
+Step 280/50462 | Loss: 5.8735 | LR: 1.87e-04 | 2759.1ms/step | 21.7 samples/s
+Step 290/50462 | Loss: 5.8079 | LR: 1.93e-04 | 2761.0ms/step | 21.7 samples/s
+Step 300/50462 | Loss: 5.7372 | LR: 2.00e-04 | 2762.7ms/step | 21.7 samples/s
+Step 310/50462 | Loss: 5.6924 | LR: 2.07e-04 | 2761.0ms/step | 21.7 samples/s
+Step 320/50462 | Loss: 5.6409 | LR: 2.13e-04 | 2760.7ms/step | 21.7 samples/s
+Step 330/50462 | Loss: 5.6141 | LR: 2.20e-04 | 2760.6ms/step | 21.7 samples/s
+Step 340/50462 | Loss: 5.5656 | LR: 2.27e-04 | 2760.9ms/step | 21.7 samples/s
+Step 350/50462 | Loss: 5.5115 | LR: 2.33e-04 | 2759.4ms/step | 21.7 samples/s
+Step 360/50462 | Loss: 5.4540 | LR: 2.40e-04 | 2760.6ms/step | 21.7 samples/s
+Step 370/50462 | Loss: 5.3852 | LR: 2.47e-04 | 2760.4ms/step | 21.7 samples/s
+Step 380/50462 | Loss: 5.3676 | LR: 2.53e-04 | 2761.4ms/step | 21.7 samples/s
+Step 390/50462 | Loss: 5.3260 | LR: 2.60e-04 | 2759.9ms/step | 21.7 samples/s
+Step 400/50462 | Loss: 5.2715 | LR: 2.67e-04 | 2760.3ms/step | 21.7 samples/s
+Step 410/50462 | Loss: 5.2183 | LR: 2.73e-04 | 2759.8ms/step | 21.7 samples/s
+Step 420/50462 | Loss: 5.1942 | LR: 2.80e-04 | 2761.0ms/step | 21.7 samples/s
+Step 430/50462 | Loss: 5.1278 | LR: 2.87e-04 | 2761.1ms/step | 21.7 samples/s
+Step 440/50462 | Loss: 5.1252 | LR: 2.93e-04 | 2759.0ms/step | 21.7 samples/s
+Step 450/50462 | Loss: 5.0873 | LR: 3.00e-04 | 2758.1ms/step | 21.8 samples/s
+Step 460/50462 | Loss: 5.0026 | LR: 3.07e-04 | 2756.5ms/step | 21.8 samples/s
+Step 470/50462 | Loss: 5.0290 | LR: 3.13e-04 | 2759.6ms/step | 21.7 samples/s
+Step 480/50462 | Loss: 5.0300 | LR: 3.20e-04 | 2760.6ms/step | 21.7 samples/s
+Step 490/50462 | Loss: 4.9552 | LR: 3.27e-04 | 2760.2ms/step | 21.7 samples/s
+Step 500/50462 | Loss: 4.9114 | LR: 3.33e-04 | 2759.2ms/step | 21.7 samples/s
+Validation at step 500: Loss = 4.8836, Perplexity = 132.10
+Step 510/50462 | Loss: 4.9205 | LR: 3.40e-04 | 4253.1ms/step | 14.1 samples/s
+Step 520/50462 | Loss: 4.8721 | LR: 3.47e-04 | 2759.4ms/step | 21.7 samples/s
+Step 530/50462 | Loss: 4.8278 | LR: 3.53e-04 | 2758.7ms/step | 21.7 samples/s
+Step 540/50462 | Loss: 4.7683 | LR: 3.60e-04 | 2756.9ms/step | 21.8 samples/s
+Step 550/50462 | Loss: 4.8274 | LR: 3.67e-04 | 2758.3ms/step | 21.8 samples/s
+Step 560/50462 | Loss: 4.7601 | LR: 3.73e-04 | 2758.6ms/step | 21.7 samples/s
+Step 570/50462 | Loss: 4.6603 | LR: 3.80e-04 | 2757.7ms/step | 21.8 samples/s
+Step 580/50462 | Loss: 4.6139 | LR: 3.87e-04 | 2758.0ms/step | 21.8 samples/s
+Step 590/50462 | Loss: 4.7223 | LR: 3.93e-04 | 2757.1ms/step | 21.8 samples/s
+Step 600/50462 | Loss: 4.7198 | LR: 4.00e-04 | 2757.4ms/step | 21.8 samples/s
+Step 610/50462 | Loss: 4.5945 | LR: 4.07e-04 | 3350.6ms/step | 17.9 samples/s
+Step 620/50462 | Loss: 4.5770 | LR: 4.13e-04 | 2751.4ms/step | 21.8 samples/s
+Step 630/50462 | Loss: 4.5489 | LR: 4.20e-04 | 2755.7ms/step | 21.8 samples/s
+Step 640/50462 | Loss: 4.5989 | LR: 4.27e-04 | 3351.4ms/step | 17.9 samples/s
+Step 650/50462 | Loss: 4.5378 | LR: 4.33e-04 | 2757.0ms/step | 21.8 samples/s
+Step 660/50462 | Loss: 4.4513 | LR: 4.40e-04 | 2759.7ms/step | 21.7 samples/s
+Step 670/50462 | Loss: 4.4261 | LR: 4.47e-04 | 2759.6ms/step | 21.7 samples/s
+Step 680/50462 | Loss: 4.4397 | LR: 4.53e-04 | 2759.4ms/step | 21.7 samples/s
+Step 690/50462 | Loss: 4.4345 | LR: 4.60e-04 | 2758.6ms/step | 21.8 samples/s
+Step 700/50462 | Loss: 4.4282 | LR: 4.67e-04 | 2756.7ms/step | 21.8 samples/s
+Step 710/50462 | Loss: 4.3494 | LR: 4.73e-04 | 2759.1ms/step | 21.7 samples/s
+Step 720/50462 | Loss: 4.3021 | LR: 4.80e-04 | 2758.6ms/step | 21.7 samples/s
+Step 730/50462 | Loss: 4.3007 | LR: 4.87e-04 | 2761.2ms/step | 21.7 samples/s
+Step 740/50462 | Loss: 4.2722 | LR: 4.93e-04 | 2761.0ms/step | 21.7 samples/s
+Step 750/50462 | Loss: 4.2957 | LR: 5.00e-04 | 2760.2ms/step | 21.7 samples/s
+Validation at step 750: Loss = 4.2539, Perplexity = 70.38
+Step 760/50462 | Loss: 4.2268 | LR: 5.00e-04 | 4247.4ms/step | 14.1 samples/s
+Step 770/50462 | Loss: 4.2373 | LR: 5.00e-04 | 2755.4ms/step | 21.8 samples/s
+Step 780/50462 | Loss: 4.1921 | LR: 5.00e-04 | 2756.7ms/step | 21.8 samples/s
+Step 790/50462 | Loss: 4.1957 | LR: 5.00e-04 | 2755.7ms/step | 21.8 samples/s
+Step 800/50462 | Loss: 4.1411 | LR: 5.00e-04 | 2759.5ms/step | 21.7 samples/s
+Step 810/50462 | Loss: 4.1465 | LR: 5.00e-04 | 2758.6ms/step | 21.7 samples/s
+Step 820/50462 | Loss: 4.1310 | LR: 5.00e-04 | 2757.7ms/step | 21.8 samples/s
+Step 830/50462 | Loss: 4.1071 | LR: 5.00e-04 | 2753.5ms/step | 21.8 samples/s
+Step 840/50462 | Loss: 4.0836 | LR: 5.00e-04 | 2758.5ms/step | 21.8 samples/s
+Step 850/50462 | Loss: 4.0653 | LR: 5.00e-04 | 2759.2ms/step | 21.7 samples/s
+Step 860/50462 | Loss: 4.0364 | LR: 5.00e-04 | 2757.6ms/step | 21.8 samples/s
+Step 870/50462 | Loss: 4.0130 | LR: 5.00e-04 | 2759.9ms/step | 21.7 samples/s
+Step 880/50462 | Loss: 3.9941 | LR: 5.00e-04 | 2756.1ms/step | 21.8 samples/s
+Step 890/50462 | Loss: 3.9949 | LR: 5.00e-04 | 2758.7ms/step | 21.7 samples/s
+Step 900/50462 | Loss: 3.9825 | LR: 5.00e-04 | 2759.2ms/step | 21.7 samples/s
+Step 910/50462 | Loss: 3.9619 | LR: 5.00e-04 | 2759.3ms/step | 21.7 samples/s
+Step 920/50462 | Loss: 3.9121 | LR: 5.00e-04 | 2758.2ms/step | 21.8 samples/s
+Step 930/50462 | Loss: 3.9619 | LR: 5.00e-04 | 2760.1ms/step | 21.7 samples/s
+Step 940/50462 | Loss: 3.9100 | LR: 5.00e-04 | 2756.4ms/step | 21.8 samples/s
+Step 950/50462 | Loss: 3.8990 | LR: 5.00e-04 | 2757.9ms/step | 21.8 samples/s
+Step 960/50462 | Loss: 3.8983 | LR: 5.00e-04 | 2756.9ms/step | 21.8 samples/s
+Step 970/50462 | Loss: 3.8782 | LR: 5.00e-04 | 2759.8ms/step | 21.7 samples/s
+Step 980/50462 | Loss: 3.8480 | LR: 5.00e-04 | 2759.5ms/step | 21.7 samples/s
+Step 990/50462 | Loss: 3.8627 | LR: 5.00e-04 | 2759.6ms/step | 21.7 samples/s
+Step 1000/50462 | Loss: 3.8628 | LR: 5.00e-04 | 2758.8ms/step | 21.7 samples/s
+Validation at step 1000: Loss = 3.8567, Perplexity = 47.31
+Saved checkpoint to checkpoints/run_20260613_222732/checkpoint-1000
+Step 1010/50462 | Loss: 3.8364 | LR: 5.00e-04 | 5335.5ms/step | 11.2 samples/s
+Step 1020/50462 | Loss: 3.8440 | LR: 5.00e-04 | 2758.6ms/step | 21.8 samples/s
+Step 1030/50462 | Loss: 3.8335 | LR: 5.00e-04 | 2758.5ms/step | 21.8 samples/s
+Step 1040/50462 | Loss: 3.8327 | LR: 5.00e-04 | 2757.2ms/step | 21.8 samples/s
+Step 1050/50462 | Loss: 3.8008 | LR: 5.00e-04 | 2755.6ms/step | 21.8 samples/s
+Step 1060/50462 | Loss: 3.7957 | LR: 5.00e-04 | 2757.5ms/step | 21.8 samples/s
+Step 1070/50462 | Loss: 3.7822 | LR: 5.00e-04 | 2755.4ms/step | 21.8 samples/s
+Step 1080/50462 | Loss: 3.7829 | LR: 5.00e-04 | 2755.4ms/step | 21.8 samples/s
+Step 1090/50462 | Loss: 3.7583 | LR: 5.00e-04 | 2757.7ms/step | 21.8 samples/s
+Step 1100/50462 | Loss: 3.7709 | LR: 5.00e-04 | 2754.8ms/step | 21.8 samples/s
+Step 1110/50462 | Loss: 3.7790 | LR: 5.00e-04 | 2753.9ms/step | 21.8 samples/s
+Step 1120/50462 | Loss: 3.7341 | LR: 5.00e-04 | 2755.7ms/step | 21.8 samples/s
+Step 1130/50462 | Loss: 3.7454 | LR: 5.00e-04 | 2754.9ms/step | 21.8 samples/s
+Step 1140/50462 | Loss: 3.7167 | LR: 5.00e-04 | 2754.6ms/step | 21.8 samples/s
+Step 1150/50462 | Loss: 3.6859 | LR: 5.00e-04 | 2754.8ms/step | 21.8 samples/s
+Step 1160/50462 | Loss: 3.7077 | LR: 5.00e-04 | 2755.9ms/step | 21.8 samples/s
+Step 1170/50462 | Loss: 3.6990 | LR: 5.00e-04 | 2757.6ms/step | 21.8 samples/s
+Step 1180/50462 | Loss: 3.6600 | LR: 5.00e-04 | 2754.0ms/step | 21.8 samples/s
+Step 1190/50462 | Loss: 3.6831 | LR: 5.00e-04 | 2752.8ms/step | 21.8 samples/s
+Step 1200/50462 | Loss: 3.6980 | LR: 5.00e-04 | 2756.9ms/step | 21.8 samples/s
+Step 1210/50462 | Loss: 3.6939 | LR: 5.00e-04 | 2754.9ms/step | 21.8 samples/s
+Step 1220/50462 | Loss: 3.6551 | LR: 5.00e-04 | 2753.8ms/step | 21.8 samples/s
+Step 1230/50462 | Loss: 3.6261 | LR: 5.00e-04 | 2754.7ms/step | 21.8 samples/s
+Step 1240/50462 | Loss: 3.6812 | LR: 5.00e-04 | 2754.7ms/step | 21.8 samples/s
+Step 1250/50462 | Loss: 3.6441 | LR: 5.00e-04 | 2755.0ms/step | 21.8 samples/s
+Validation at step 1250: Loss = 3.6542, Perplexity = 38.64
+Step 1260/50462 | Loss: 3.6657 | LR: 5.00e-04 | 4239.3ms/step | 14.2 samples/s
+Step 1270/50462 | Loss: 3.6287 | LR: 5.00e-04 | 2756.0ms/step | 21.8 samples/s
+Step 1280/50462 | Loss: 3.6062 | LR: 5.00e-04 | 2758.2ms/step | 21.8 samples/s
+Step 1290/50462 | Loss: 3.6149 | LR: 5.00e-04 | 2756.7ms/step | 21.8 samples/s
+Step 1300/50462 | Loss: 3.6040 | LR: 5.00e-04 | 2757.3ms/step | 21.8 samples/s
+Step 1310/50462 | Loss: 3.6104 | LR: 5.00e-04 | 2758.2ms/step | 21.8 samples/s
+Step 1320/50462 | Loss: 3.6035 | LR: 5.00e-04 | 2755.7ms/step | 21.8 samples/s
+Step 1330/50462 | Loss: 3.5944 | LR: 5.00e-04 | 2757.9ms/step | 21.8 samples/s
+Step 1340/50462 | Loss: 3.5838 | LR: 5.00e-04 | 2753.3ms/step | 21.8 samples/s
+Step 1350/50462 | Loss: 3.5655 | LR: 5.00e-04 | 2752.6ms/step | 21.8 samples/s
+Step 1360/50462 | Loss: 3.5793 | LR: 5.00e-04 | 2759.1ms/step | 21.7 samples/s
+Step 1370/50462 | Loss: 3.5590 | LR: 5.00e-04 | 2760.2ms/step | 21.7 samples/s
+Step 1380/50462 | Loss: 3.5983 | LR: 5.00e-04 | 2757.8ms/step | 21.8 samples/s
+Step 1390/50462 | Loss: 3.5783 | LR: 5.00e-04 | 2750.2ms/step | 21.8 samples/s
+Step 1400/50462 | Loss: 3.5413 | LR: 5.00e-04 | 2751.5ms/step | 21.8 samples/s
+Step 1410/50462 | Loss: 3.5678 | LR: 5.00e-04 | 2750.0ms/step | 21.8 samples/s
+Step 1420/50462 | Loss: 3.5473 | LR: 5.00e-04 | 2751.2ms/step | 21.8 samples/s
+Step 1430/50462 | Loss: 3.5228 | LR: 5.00e-04 | 2751.3ms/step | 21.8 samples/s
+Step 1440/50462 | Loss: 3.5522 | LR: 5.00e-04 | 2750.7ms/step | 21.8 samples/s
+Step 1450/50462 | Loss: 3.5181 | LR: 5.00e-04 | 2750.4ms/step | 21.8 samples/s
+Step 1460/50462 | Loss: 3.5244 | LR: 5.00e-04 | 2752.1ms/step | 21.8 samples/s
+Step 1470/50462 | Loss: 3.4998 | LR: 5.00e-04 | 2749.7ms/step | 21.8 samples/s
+Step 1480/50462 | Loss: 3.5392 | LR: 5.00e-04 | 2749.8ms/step | 21.8 samples/s
+Step 1490/50462 | Loss: 3.5190 | LR: 5.00e-04 | 2754.1ms/step | 21.8 samples/s
+Step 1500/50462 | Loss: 3.4885 | LR: 5.00e-04 | 2752.7ms/step | 21.8 samples/s
+Validation at step 1500: Loss = 3.5274, Perplexity = 34.03
+Step 1510/50462 | Loss: 3.5219 | LR: 5.00e-04 | 4239.9ms/step | 14.2 samples/s
+Step 1520/50462 | Loss: 3.4904 | LR: 5.00e-04 | 2756.5ms/step | 21.8 samples/s
+Step 1530/50462 | Loss: 3.5169 | LR: 5.00e-04 | 2756.5ms/step | 21.8 samples/s
+Step 1540/50462 | Loss: 3.4818 | LR: 5.00e-04 | 2756.4ms/step | 21.8 samples/s
+Step 1550/50462 | Loss: 3.4889 | LR: 5.00e-04 | 2755.1ms/step | 21.8 samples/s
+Step 1560/50462 | Loss: 3.4817 | LR: 5.00e-04 | 2754.7ms/step | 21.8 samples/s
+Step 1570/50462 | Loss: 3.5134 | LR: 5.00e-04 | 2756.2ms/step | 21.8 samples/s
+Step 1580/50462 | Loss: 3.4805 | LR: 5.00e-04 | 2757.0ms/step | 21.8 samples/s
+Step 1590/50462 | Loss: 3.4989 | LR: 5.00e-04 | 2755.5ms/step | 21.8 samples/s
+Step 1600/50462 | Loss: 3.4897 | LR: 5.00e-04 | 2756.5ms/step | 21.8 samples/s
+Step 1610/50462 | Loss: 3.4518 | LR: 5.00e-04 | 2756.2ms/step | 21.8 samples/s
+Step 1620/50462 | Loss: 3.4727 | LR: 5.00e-04 | 2756.0ms/step | 21.8 samples/s
+Step 1630/50462 | Loss: 3.4882 | LR: 5.00e-04 | 2755.3ms/step | 21.8 samples/s
+Step 1640/50462 | Loss: 3.4619 | LR: 5.00e-04 | 2755.1ms/step | 21.8 samples/s
+Step 1650/50462 | Loss: 3.4196 | LR: 5.00e-04 | 2754.2ms/step | 21.8 samples/s
+Step 1660/50462 | Loss: 3.4413 | LR: 5.00e-04 | 2757.2ms/step | 21.8 samples/s
+Step 1670/50462 | Loss: 3.4653 | LR: 5.00e-04 | 2755.3ms/step | 21.8 samples/s
+Step 1680/50462 | Loss: 3.4382 | LR: 5.00e-04 | 2757.2ms/step | 21.8 samples/s
+Step 1690/50462 | Loss: 3.4365 | LR: 5.00e-04 | 2757.5ms/step | 21.8 samples/s
+Step 1700/50462 | Loss: 3.4450 | LR: 5.00e-04 | 2754.2ms/step | 21.8 samples/s
+Step 1710/50462 | Loss: 3.4607 | LR: 5.00e-04 | 2753.4ms/step | 21.8 samples/s
+Step 1720/50462 | Loss: 3.4240 | LR: 5.00e-04 | 2753.0ms/step | 21.8 samples/s
+Step 1730/50462 | Loss: 3.4246 | LR: 5.00e-04 | 2758.0ms/step | 21.8 samples/s
+Step 1740/50462 | Loss: 3.4535 | LR: 5.00e-04 | 2757.1ms/step | 21.8 samples/s
+Step 1750/50462 | Loss: 3.4654 | LR: 5.00e-04 | 2755.9ms/step | 21.8 samples/s
+Validation at step 1750: Loss = 3.4314, Perplexity = 30.92
+Step 1760/50462 | Loss: 3.4416 | LR: 4.99e-04 | 4244.7ms/step | 14.1 samples/s
+Step 1770/50462 | Loss: 3.4231 | LR: 4.99e-04 | 2757.2ms/step | 21.8 samples/s
+Step 1780/50462 | Loss: 3.3943 | LR: 4.99e-04 | 2756.0ms/step | 21.8 samples/s
+Step 1790/50462 | Loss: 3.3960 | LR: 4.99e-04 | 2753.5ms/step | 21.8 samples/s
+Step 1800/50462 | Loss: 3.3913 | LR: 4.99e-04 | 2755.6ms/step | 21.8 samples/s
+Step 1810/50462 | Loss: 3.4023 | LR: 4.99e-04 | 2754.3ms/step | 21.8 samples/s
+Step 1820/50462 | Loss: 3.3828 | LR: 4.99e-04 | 2755.6ms/step | 21.8 samples/s
+Step 1830/50462 | Loss: 3.3855 | LR: 4.99e-04 | 2755.8ms/step | 21.8 samples/s
+Step 1840/50462 | Loss: 3.3854 | LR: 4.99e-04 | 2757.8ms/step | 21.8 samples/s
+Step 1850/50462 | Loss: 3.3936 | LR: 4.99e-04 | 2758.2ms/step | 21.8 samples/s
+Step 1860/50462 | Loss: 3.3723 | LR: 4.99e-04 | 2754.1ms/step | 21.8 samples/s
+Step 1870/50462 | Loss: 3.4110 | LR: 4.99e-04 | 2752.6ms/step | 21.8 samples/s
+Step 1880/50462 | Loss: 3.3946 | LR: 4.99e-04 | 2755.1ms/step | 21.8 samples/s
+Step 1890/50462 | Loss: 3.3591 | LR: 4.99e-04 | 2754.6ms/step | 21.8 samples/s
+Step 1900/50462 | Loss: 3.3619 | LR: 4.99e-04 | 2752.5ms/step | 21.8 samples/s
+Step 1910/50462 | Loss: 3.3959 | LR: 4.99e-04 | 2753.5ms/step | 21.8 samples/s
+Step 1920/50462 | Loss: 3.3929 | LR: 4.99e-04 | 2755.1ms/step | 21.8 samples/s
+Step 1930/50462 | Loss: 3.3972 | LR: 4.99e-04 | 2748.4ms/step | 21.8 samples/s
+Step 1940/50462 | Loss: 3.3899 | LR: 4.99e-04 | 2755.8ms/step | 21.8 samples/s
+Step 1950/50462 | Loss: 3.3772 | LR: 4.99e-04 | 2753.1ms/step | 21.8 samples/s
+Step 1960/50462 | Loss: 3.3742 | LR: 4.99e-04 | 2752.0ms/step | 21.8 samples/s
+Step 1970/50462 | Loss: 3.3719 | LR: 4.99e-04 | 2753.3ms/step | 21.8 samples/s
+Step 1980/50462 | Loss: 3.3642 | LR: 4.99e-04 | 2752.9ms/step | 21.8 samples/s
+Step 1990/50462 | Loss: 3.3920 | LR: 4.99e-04 | 2751.7ms/step | 21.8 samples/s
+Step 2000/50462 | Loss: 3.3910 | LR: 4.99e-04 | 2751.7ms/step | 21.8 samples/s
+Validation at step 2000: Loss = 3.3635, Perplexity = 28.89
+Saved checkpoint to checkpoints/run_20260613_222732/checkpoint-2000
+Step 2010/50462 | Loss: 3.3737 | LR: 4.99e-04 | 5303.6ms/step | 11.3 samples/s
+Step 2020/50462 | Loss: 3.3731 | LR: 4.99e-04 | 2749.0ms/step | 21.8 samples/s
+Step 2030/50462 | Loss: 3.3423 | LR: 4.99e-04 | 2750.6ms/step | 21.8 samples/s
+Step 2040/50462 | Loss: 3.3541 | LR: 4.99e-04 | 2754.8ms/step | 21.8 samples/s
+Step 2050/50462 | Loss: 3.3236 | LR: 4.99e-04 | 2753.4ms/step | 21.8 samples/s
+Step 2060/50462 | Loss: 3.3272 | LR: 4.99e-04 | 2754.1ms/step | 21.8 samples/s
+Step 2070/50462 | Loss: 3.3468 | LR: 4.99e-04 | 2755.6ms/step | 21.8 samples/s
+Step 2080/50462 | Loss: 3.3361 | LR: 4.99e-04 | 2752.7ms/step | 21.8 samples/s
+Step 2090/50462 | Loss: 3.3368 | LR: 4.99e-04 | 2752.5ms/step | 21.8 samples/s
+Step 2100/50462 | Loss: 3.3238 | LR: 4.99e-04 | 2754.0ms/step | 21.8 samples/s
+Step 2110/50462 | Loss: 3.3676 | LR: 4.99e-04 | 2754.5ms/step | 21.8 samples/s
+Step 2120/50462 | Loss: 3.3574 | LR: 4.99e-04 | 2751.1ms/step | 21.8 samples/s
+Step 2130/50462 | Loss: 3.3197 | LR: 4.99e-04 | 2754.1ms/step | 21.8 samples/s
+Step 2140/50462 | Loss: 3.3035 | LR: 4.99e-04 | 2754.4ms/step | 21.8 samples/s
+Step 2150/50462 | Loss: 3.3093 | LR: 4.99e-04 | 2751.5ms/step | 21.8 samples/s
+Step 2160/50462 | Loss: 3.3205 | LR: 4.99e-04 | 2752.5ms/step | 21.8 samples/s
+Step 2170/50462 | Loss: 3.3091 | LR: 4.99e-04 | 2753.6ms/step | 21.8 samples/s
+Step 2180/50462 | Loss: 3.3326 | LR: 4.99e-04 | 2754.3ms/step | 21.8 samples/s
+Step 2190/50462 | Loss: 3.3087 | LR: 4.99e-04 | 2754.1ms/step | 21.8 samples/s
+Step 2200/50462 | Loss: 3.3012 | LR: 4.99e-04 | 2752.7ms/step | 21.8 samples/s
+Step 2210/50462 | Loss: 3.3139 | LR: 4.99e-04 | 2755.2ms/step | 21.8 samples/s
+Step 2220/50462 | Loss: 3.2971 | LR: 4.99e-04 | 2753.7ms/step | 21.8 samples/s
+Step 2230/50462 | Loss: 3.3110 | LR: 4.99e-04 | 2752.6ms/step | 21.8 samples/s
+Step 2240/50462 | Loss: 3.3018 | LR: 4.99e-04 | 2756.0ms/step | 21.8 samples/s
+Step 2250/50462 | Loss: 3.3460 | LR: 4.99e-04 | 2755.3ms/step | 21.8 samples/s
+Validation at step 2250: Loss = 3.3093, Perplexity = 27.37
+Step 2260/50462 | Loss: 3.3055 | LR: 4.99e-04 | 4240.8ms/step | 14.1 samples/s
+Step 2270/50462 | Loss: 3.3194 | LR: 4.99e-04 | 2754.1ms/step | 21.8 samples/s
+Step 2280/50462 | Loss: 3.2712 | LR: 4.99e-04 | 2754.0ms/step | 21.8 samples/s
+Step 2290/50462 | Loss: 3.2688 | LR: 4.99e-04 | 2753.0ms/step | 21.8 samples/s
+Step 2300/50462 | Loss: 3.2727 | LR: 4.99e-04 | 2758.6ms/step | 21.8 samples/s
+Step 2310/50462 | Loss: 3.2953 | LR: 4.99e-04 | 2757.3ms/step | 21.8 samples/s
+Step 2320/50462 | Loss: 3.2905 | LR: 4.99e-04 | 2754.0ms/step | 21.8 samples/s
+Step 2330/50462 | Loss: 3.2547 | LR: 4.99e-04 | 2754.7ms/step | 21.8 samples/s
+Step 2340/50462 | Loss: 3.2857 | LR: 4.99e-04 | 2753.4ms/step | 21.8 samples/s
+Step 2350/50462 | Loss: 3.2591 | LR: 4.99e-04 | 2753.4ms/step | 21.8 samples/s
+Step 2360/50462 | Loss: 3.2575 | LR: 4.99e-04 | 2755.5ms/step | 21.8 samples/s
+Step 2370/50462 | Loss: 3.2791 | LR: 4.99e-04 | 2752.6ms/step | 21.8 samples/s
+Step 2380/50462 | Loss: 3.2312 | LR: 4.99e-04 | 2757.2ms/step | 21.8 samples/s
+Step 2390/50462 | Loss: 3.2538 | LR: 4.99e-04 | 2752.6ms/step | 21.8 samples/s
+Step 2400/50462 | Loss: 3.2761 | LR: 4.99e-04 | 2754.2ms/step | 21.8 samples/s
+Step 2410/50462 | Loss: 3.2397 | LR: 4.99e-04 | 2753.6ms/step | 21.8 samples/s
+Step 2420/50462 | Loss: 3.2826 | LR: 4.99e-04 | 2753.9ms/step | 21.8 samples/s
+Step 2430/50462 | Loss: 3.2701 | LR: 4.99e-04 | 2753.9ms/step | 21.8 samples/s
+Step 2440/50462 | Loss: 3.2825 | LR: 4.99e-04 | 2754.0ms/step | 21.8 samples/s
+Step 2450/50462 | Loss: 3.3077 | LR: 4.99e-04 | 2754.3ms/step | 21.8 samples/s
+Step 2460/50462 | Loss: 3.2552 | LR: 4.99e-04 | 2754.5ms/step | 21.8 samples/s
+Step 2470/50462 | Loss: 3.2625 | LR: 4.99e-04 | 2750.3ms/step | 21.8 samples/s
+Step 2480/50462 | Loss: 3.2772 | LR: 4.99e-04 | 2749.9ms/step | 21.8 samples/s
+Step 2490/50462 | Loss: 3.2593 | LR: 4.98e-04 | 2750.2ms/step | 21.8 samples/s
+Step 2500/50462 | Loss: 3.2519 | LR: 4.98e-04 | 2752.0ms/step | 21.8 samples/s
+Validation at step 2500: Loss = 3.2614, Perplexity = 26.09
+Step 2510/50462 | Loss: 3.2464 | LR: 4.98e-04 | 4242.9ms/step | 14.1 samples/s
+Step 2520/50462 | Loss: 3.2366 | LR: 4.98e-04 | 2755.4ms/step | 21.8 samples/s
+Step 2530/50462 | Loss: 3.2379 | LR: 4.98e-04 | 2752.0ms/step | 21.8 samples/s
+Step 2540/50462 | Loss: 3.2605 | LR: 4.98e-04 | 2752.4ms/step | 21.8 samples/s
+Step 2550/50462 | Loss: 3.2680 | LR: 4.98e-04 | 2750.5ms/step | 21.8 samples/s
+Step 2560/50462 | Loss: 3.2758 | LR: 4.98e-04 | 2752.0ms/step | 21.8 samples/s
+Step 2570/50462 | Loss: 3.2169 | LR: 4.98e-04 | 2752.2ms/step | 21.8 samples/s
+Step 2580/50462 | Loss: 3.2281 | LR: 4.98e-04 | 2751.8ms/step | 21.8 samples/s
+Step 2590/50462 | Loss: 3.2566 | LR: 4.98e-04 | 2750.7ms/step | 21.8 samples/s
+Step 2600/50462 | Loss: 3.2153 | LR: 4.98e-04 | 2752.6ms/step | 21.8 samples/s
+Step 2610/50462 | Loss: 3.2294 | LR: 4.98e-04 | 2752.3ms/step | 21.8 samples/s
+Step 2620/50462 | Loss: 3.2509 | LR: 4.98e-04 | 2754.3ms/step | 21.8 samples/s
+Step 2630/50462 | Loss: 3.2172 | LR: 4.98e-04 | 2751.2ms/step | 21.8 samples/s
+Step 2640/50462 | Loss: 3.2446 | LR: 4.98e-04 | 2750.6ms/step | 21.8 samples/s
+Step 2650/50462 | Loss: 3.2413 | LR: 4.98e-04 | 2753.6ms/step | 21.8 samples/s
+Step 2660/50462 | Loss: 3.2172 | LR: 4.98e-04 | 2749.9ms/step | 21.8 samples/s
+Step 2670/50462 | Loss: 3.2744 | LR: 4.98e-04 | 2748.6ms/step | 21.8 samples/s
+Step 2680/50462 | Loss: 3.2197 | LR: 4.98e-04 | 2751.7ms/step | 21.8 samples/s
+Step 2690/50462 | Loss: 3.2382 | LR: 4.98e-04 | 2752.3ms/step | 21.8 samples/s
+Step 2700/50462 | Loss: 3.2237 | LR: 4.98e-04 | 2751.2ms/step | 21.8 samples/s
+Step 2710/50462 | Loss: 3.2030 | LR: 4.98e-04 | 2753.0ms/step | 21.8 samples/s
+Step 2720/50462 | Loss: 3.2118 | LR: 4.98e-04 | 2754.1ms/step | 21.8 samples/s
+Step 2730/50462 | Loss: 3.2279 | LR: 4.98e-04 | 3265.0ms/step | 18.4 samples/s
+Step 2740/50462 | Loss: 3.1963 | LR: 4.98e-04 | 2748.0ms/step | 21.8 samples/s
+Step 2750/50462 | Loss: 3.2225 | LR: 4.98e-04 | 2750.9ms/step | 21.8 samples/s
+Validation at step 2750: Loss = 3.2229, Perplexity = 25.10
+Step 2760/50462 | Loss: 3.2236 | LR: 4.98e-04 | 4236.9ms/step | 14.2 samples/s
+Step 2770/50462 | Loss: 3.2081 | LR: 4.98e-04 | 2752.8ms/step | 21.8 samples/s
+Step 2780/50462 | Loss: 3.2147 | LR: 4.98e-04 | 3289.5ms/step | 18.2 samples/s
+Step 2790/50462 | Loss: 3.1886 | LR: 4.98e-04 | 2750.6ms/step | 21.8 samples/s
+Step 2800/50462 | Loss: 3.2108 | LR: 4.98e-04 | 2750.3ms/step | 21.8 samples/s
+Step 2810/50462 | Loss: 3.2195 | LR: 4.98e-04 | 2752.1ms/step | 21.8 samples/s
+Step 2820/50462 | Loss: 3.2034 | LR: 4.98e-04 | 2753.2ms/step | 21.8 samples/s
+Step 2830/50462 | Loss: 3.1613 | LR: 4.98e-04 | 2752.7ms/step | 21.8 samples/s
+Step 2840/50462 | Loss: 3.2031 | LR: 4.98e-04 | 2753.9ms/step | 21.8 samples/s
+Step 2850/50462 | Loss: 3.2032 | LR: 4.98e-04 | 2753.0ms/step | 21.8 samples/s
+Step 2860/50462 | Loss: 3.2109 | LR: 4.98e-04 | 2750.1ms/step | 21.8 samples/s
+Step 2870/50462 | Loss: 3.1961 | LR: 4.98e-04 | 2751.8ms/step | 21.8 samples/s
+Step 2880/50462 | Loss: 3.1908 | LR: 4.98e-04 | 2752.1ms/step | 21.8 samples/s
+Step 2890/50462 | Loss: 3.1746 | LR: 4.98e-04 | 2752.3ms/step | 21.8 samples/s
+Step 2900/50462 | Loss: 3.1946 | LR: 4.98e-04 | 2751.9ms/step | 21.8 samples/s
+Step 2910/50462 | Loss: 3.2074 | LR: 4.98e-04 | 2752.4ms/step | 21.8 samples/s
+Step 2920/50462 | Loss: 3.1985 | LR: 4.98e-04 | 2750.9ms/step | 21.8 samples/s
+Step 2930/50462 | Loss: 3.1761 | LR: 4.98e-04 | 2751.8ms/step | 21.8 samples/s
+Step 2940/50462 | Loss: 3.1707 | LR: 4.98e-04 | 2752.2ms/step | 21.8 samples/s
+Step 2950/50462 | Loss: 3.2039 | LR: 4.98e-04 | 2752.7ms/step | 21.8 samples/s
+Step 2960/50462 | Loss: 3.1663 | LR: 4.98e-04 | 2750.3ms/step | 21.8 samples/s
+Step 2970/50462 | Loss: 3.1965 | LR: 4.98e-04 | 2750.3ms/step | 21.8 samples/s
+Step 2980/50462 | Loss: 3.2059 | LR: 4.98e-04 | 2751.1ms/step | 21.8 samples/s
+Step 2990/50462 | Loss: 3.1836 | LR: 4.97e-04 | 2752.6ms/step | 21.8 samples/s
+Step 3000/50462 | Loss: 3.1867 | LR: 4.97e-04 | 2751.1ms/step | 21.8 samples/s
+Validation at step 3000: Loss = 3.1912, Perplexity = 24.32
+Saved checkpoint to checkpoints/run_20260613_222732/checkpoint-3000
+Step 3010/50462 | Loss: 3.1856 | LR: 4.97e-04 | 5336.6ms/step | 11.2 samples/s
+Step 3020/50462 | Loss: 3.1742 | LR: 4.97e-04 | 2747.0ms/step | 21.8 samples/s
+Step 3030/50462 | Loss: 3.1650 | LR: 4.97e-04 | 2749.1ms/step | 21.8 samples/s
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1440, in <module>
+    main()
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1437, in main
+    train(config)
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1301, in train
+    outputs = model(
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1832, in forward
+    else self._run_ddp_forward(*inputs, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1657, in _run_ddp_forward
+    return self.module(*inputs, **kwargs)  # type: ignore[index]
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 473, in __call__
+    return super().__call__(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1047, in compile_wrapper
+    result = fn(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 285, in forward
+    hidden_states, next_cache, all_hidden_states, all_attentions = self.model(
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 139, in forward
+    if has_any_truly_sparse and (attention_mask == 0).any():
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 139, in torch_dynamo_resume_in_forward_at_139
+    if has_any_truly_sparse and (attention_mask == 0).any():
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 473, in __call__
+    return super().__call__(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1298, in _fn
+    return fn(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/fx/graph_module.py", line 949, in call_wrapped
+    return self._wrapped_call(self, *args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/fx/graph_module.py", line 447, in __call__
+    return super(self.cls, obj).__call__(*args, **kwargs)  # type: ignore[misc]
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "<eval_with_key>.720", line 656, in forward
+    submod_16 = self.compiled_submod_16(submod_15, l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_, getitem_15, l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_q_norm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_k_norm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_rotary_emb_buffers_inv_freq_, l_position_ids_, l_sparse_indices_8_kv_num_blocks, l_sparse_indices_8_kv_indices, l_sparse_indices_8_full_kv_num_blocks, l_sparse_indices_8_full_kv_indices, l_sparse_indices_8_q_num_blocks, l_sparse_indices_8_q_indices, l_sparse_indices_8_full_q_num_blocks, l_sparse_indices_8_full_q_indices, l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_);  submod_15 = l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ = getitem_15 = l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_q_norm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_k_norm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_rotary_emb_buffers_inv_freq_ = l_sparse_indices_8_kv_num_blocks = l_sparse_indices_8_kv_indices = l_sparse_indices_8_full_kv_num_blocks = l_sparse_indices_8_full_kv_indices = l_sparse_indices_8_q_num_blocks = l_sparse_indices_8_q_indices = l_sparse_indices_8_full_q_num_blocks = l_sparse_indices_8_full_q_indices = l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ = None
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/backends/distributed.py", line 215, in forward
+    x = self.submod(*args)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1298, in _fn
+    return fn(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1273, in forward
+    return compiled_fn(full_args)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 777, in runtime_wrapper
+    all_outs = compiled_invoker.run(args, on_before_call=exit_prologue)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 506, in run
+    return call_func_at_runtime_with_args(
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args
+    out = normalize_as_list(f(args))
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 93, in g
+    return f(*args)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/function.py", line 596, in apply
+    return super().apply(*args, **kwargs)  # type: ignore[misc]
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2844, in forward
+    fw_outs = call_func_at_runtime_with_args(
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args
+    out = normalize_as_list(f(args))
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 961, in wrapper
+    return compiled_fn(runtime_args)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 850, in wrapper
+    return compiled_fn(runtime_args)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 1055, in inner_fn
+    outs = compiled_fn(args)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 725, in __call__
+    return self.current_callable(inputs)
+  File "/home/navasard/.cache/torch_inductor/oq/coq6impgppmp6dnp26psegq3avd7bsgoow4agrrdejd2tgokhclc.py", line 1390, in call
+    buf19.copy_(primals_12, False)
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1440, in <module>
+[rank0]:     main()
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1437, in main
+[rank0]:     train(config)
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1301, in train
+[rank0]:     outputs = model(
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1832, in forward
+[rank0]:     else self._run_ddp_forward(*inputs, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1657, in _run_ddp_forward
+[rank0]:     return self.module(*inputs, **kwargs)  # type: ignore[index]
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 473, in __call__
+[rank0]:     return super().__call__(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1047, in compile_wrapper
+[rank0]:     result = fn(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 285, in forward
+[rank0]:     hidden_states, next_cache, all_hidden_states, all_attentions = self.model(
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 139, in forward
+[rank0]:     if has_any_truly_sparse and (attention_mask == 0).any():
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 139, in torch_dynamo_resume_in_forward_at_139
+[rank0]:     if has_any_truly_sparse and (attention_mask == 0).any():
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 473, in __call__
+[rank0]:     return super().__call__(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1298, in _fn
+[rank0]:     return fn(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/fx/graph_module.py", line 949, in call_wrapped
+[rank0]:     return self._wrapped_call(self, *args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/fx/graph_module.py", line 447, in __call__
+[rank0]:     return super(self.cls, obj).__call__(*args, **kwargs)  # type: ignore[misc]
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "<eval_with_key>.720", line 656, in forward
+[rank0]:     submod_16 = self.compiled_submod_16(submod_15, l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_, getitem_15, l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_q_norm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_k_norm_parameters_weight_, l_self_modules_layers_modules_8_modules_self_attn_modules_rotary_emb_buffers_inv_freq_, l_position_ids_, l_sparse_indices_8_kv_num_blocks, l_sparse_indices_8_kv_indices, l_sparse_indices_8_full_kv_num_blocks, l_sparse_indices_8_full_kv_indices, l_sparse_indices_8_q_num_blocks, l_sparse_indices_8_q_indices, l_sparse_indices_8_full_q_num_blocks, l_sparse_indices_8_full_q_indices, l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_, l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_);  submod_15 = l_self_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ = getitem_15 = l_self_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_q_norm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_k_norm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_rotary_emb_buffers_inv_freq_ = l_sparse_indices_8_kv_num_blocks = l_sparse_indices_8_kv_indices = l_sparse_indices_8_full_kv_num_blocks = l_sparse_indices_8_full_kv_indices = l_sparse_indices_8_q_num_blocks = l_sparse_indices_8_q_indices = l_sparse_indices_8_full_q_num_blocks = l_sparse_indices_8_full_q_indices = l_self_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ = None
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/backends/distributed.py", line 215, in forward
+[rank0]:     x = self.submod(*args)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1298, in _fn
+[rank0]:     return fn(*args, **kwargs)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1273, in forward
+[rank0]:     return compiled_fn(full_args)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 777, in runtime_wrapper
+[rank0]:     all_outs = compiled_invoker.run(args, on_before_call=exit_prologue)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 506, in run
+[rank0]:     return call_func_at_runtime_with_args(
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args
+[rank0]:     out = normalize_as_list(f(args))
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 93, in g
+[rank0]:     return f(*args)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/function.py", line 596, in apply
+[rank0]:     return super().apply(*args, **kwargs)  # type: ignore[misc]
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2844, in forward
+[rank0]:     fw_outs = call_func_at_runtime_with_args(
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args
+[rank0]:     out = normalize_as_list(f(args))
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 961, in wrapper
+[rank0]:     return compiled_fn(runtime_args)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 850, in wrapper
+[rank0]:     return compiled_fn(runtime_args)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 1055, in inner_fn
+[rank0]:     outs = compiled_fn(args)
+[rank0]:   File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 725, in __call__
+[rank0]:     return self.current_callable(inputs)
+[rank0]:   File "/home/navasard/.cache/torch_inductor/oq/coq6impgppmp6dnp26psegq3avd7bsgoow4agrrdejd2tgokhclc.py", line 1390, in call
+[rank0]:     buf19.copy_(primals_12, False)
+[rank0]: KeyboardInterrupt
+wandb: 
+wandb: 🚀 View run sparse 85% at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rp96qdel
+wandb: Find logs at: wandb/run-20260613_222959-rp96qdel/logs
+W0614 00:56:27.057000 2912 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 3073 closing signal SIGTERM
+W0614 00:56:27.058000 2912 torch/distributed/elastic/multiprocessing/api.py:1014] Sending process 3074 closing signal SIGTERM
+E0614 00:56:27.070000 2912 torch/distributed/elastic/multiprocessing/api.py:988] failed (exitcode: -2) local_rank: 0 (pid: 3072) of binary: /usr/bin/python3
+Traceback (most recent call last):
+  File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1020, in <module>
+    main()
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 367, in wrapper
+    return f(*args, **kwargs)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1016, in main
+    run(args)
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/run.py", line 1007, in run
+    elastic_launch(
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 191, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/home/navasard/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 371, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+======================================================
+SparseAtt/train.py FAILED
+------------------------------------------------------
+Failures:
+[1]:
+  time      : 2026-06-14_00:56:27
+  host      : h100-train-3-gpu-0-9
+  rank      : 1 (local_rank: 1)
+  exitcode  : -15 (pid: 3073)  (SIGTERM)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 3073
+[2]:
+  time      : 2026-06-14_00:56:27
+  host      : h100-train-3-gpu-0-9
+  rank      : 2 (local_rank: 2)
+  exitcode  : -15 (pid: 3074)  (SIGTERM)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 3074
+------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2026-06-14_00:56:27
+  host      : h100-train-3-gpu-0-9
+  rank      : 0 (local_rank: 0)
+  exitcode  : -2 (pid: 3072)  (SIGINT)
+  error_file: <N/A>
+  traceback : Signal 2 (SIGINT) received by PID 3072
+======================================================

epfl_final/logs/autolaunch_resampled_watcher.log

@@ -0,0 +1,380 @@
+=== watcher start Wed Jul  8 08:17:58 AM UTC 2026 pid 17245 ===
+08:17:58 compute_apps=4 orchestrator=alive zerostreak=0
+08:18:28 compute_apps=4 orchestrator=alive zerostreak=0
+08:18:58 compute_apps=4 orchestrator=alive zerostreak=0
+08:19:28 compute_apps=4 orchestrator=alive zerostreak=0
+08:19:58 compute_apps=4 orchestrator=alive zerostreak=0
+08:20:28 compute_apps=4 orchestrator=alive zerostreak=0
+08:20:58 compute_apps=4 orchestrator=alive zerostreak=0
+08:21:28 compute_apps=4 orchestrator=alive zerostreak=0
+08:21:58 compute_apps=4 orchestrator=alive zerostreak=0
+08:22:28 compute_apps=4 orchestrator=alive zerostreak=0
+08:22:58 compute_apps=4 orchestrator=alive zerostreak=0
+08:23:28 compute_apps=4 orchestrator=alive zerostreak=0
+08:23:58 compute_apps=4 orchestrator=alive zerostreak=0
+08:24:29 compute_apps=4 orchestrator=alive zerostreak=0
+08:24:59 compute_apps=4 orchestrator=alive zerostreak=0
+08:25:29 compute_apps=4 orchestrator=alive zerostreak=0
+08:25:59 compute_apps=4 orchestrator=alive zerostreak=0
+08:26:29 compute_apps=4 orchestrator=alive zerostreak=0
+08:26:59 compute_apps=4 orchestrator=alive zerostreak=0
+08:27:29 compute_apps=4 orchestrator=alive zerostreak=0
+08:27:59 compute_apps=4 orchestrator=alive zerostreak=0
+08:28:29 compute_apps=4 orchestrator=alive zerostreak=0
+08:28:59 compute_apps=4 orchestrator=alive zerostreak=0
+08:29:29 compute_apps=4 orchestrator=alive zerostreak=0
+08:29:59 compute_apps=3 orchestrator=alive zerostreak=0
+08:30:29 compute_apps=4 orchestrator=alive zerostreak=0
+08:30:59 compute_apps=4 orchestrator=alive zerostreak=0
+---- hardened watcher swapped in Wed Jul  8 08:31:31 AM UTC 2026 ----
+08:31:31 === hardened watcher start pid 19286 ===
+08:31:31 compute_apps=4 orchestrator=alive zerostreak=0
+08:32:01 compute_apps=4 orchestrator=alive zerostreak=0
+08:32:31 compute_apps=4 orchestrator=alive zerostreak=0
+08:33:01 compute_apps=4 orchestrator=alive zerostreak=0
+08:33:31 compute_apps=4 orchestrator=alive zerostreak=0
+08:34:01 compute_apps=4 orchestrator=alive zerostreak=0
+08:34:31 compute_apps=4 orchestrator=alive zerostreak=0
+08:35:01 compute_apps=4 orchestrator=alive zerostreak=0
+08:35:31 compute_apps=4 orchestrator=alive zerostreak=0
+08:36:01 compute_apps=4 orchestrator=alive zerostreak=0
+08:36:31 compute_apps=4 orchestrator=alive zerostreak=0
+08:37:01 compute_apps=4 orchestrator=alive zerostreak=0
+08:37:31 compute_apps=4 orchestrator=alive zerostreak=0
+08:38:02 compute_apps=4 orchestrator=alive zerostreak=0
+08:38:32 compute_apps=4 orchestrator=alive zerostreak=0
+08:39:02 compute_apps=4 orchestrator=alive zerostreak=0
+08:39:32 compute_apps=4 orchestrator=alive zerostreak=0
+08:40:02 compute_apps=4 orchestrator=alive zerostreak=0
+08:40:32 compute_apps=4 orchestrator=alive zerostreak=0
+08:41:02 compute_apps=4 orchestrator=alive zerostreak=0
+08:41:32 compute_apps=4 orchestrator=alive zerostreak=0
+08:42:02 compute_apps=4 orchestrator=alive zerostreak=0
+08:42:32 compute_apps=4 orchestrator=alive zerostreak=0
+08:43:02 compute_apps=4 orchestrator=alive zerostreak=0
+08:43:32 compute_apps=4 orchestrator=alive zerostreak=0
+08:44:02 compute_apps=4 orchestrator=alive zerostreak=0
+08:44:32 compute_apps=4 orchestrator=alive zerostreak=0
+08:45:03 compute_apps=4 orchestrator=alive zerostreak=0
+08:45:33 compute_apps=4 orchestrator=alive zerostreak=0
+08:46:03 compute_apps=4 orchestrator=alive zerostreak=0
+08:46:33 compute_apps=4 orchestrator=alive zerostreak=0
+08:47:03 compute_apps=4 orchestrator=alive zerostreak=0
+08:47:33 compute_apps=4 orchestrator=alive zerostreak=0
+08:48:03 compute_apps=4 orchestrator=alive zerostreak=0
+08:48:33 compute_apps=4 orchestrator=alive zerostreak=0
+08:49:03 compute_apps=4 orchestrator=alive zerostreak=0
+08:49:33 compute_apps=4 orchestrator=alive zerostreak=0
+08:50:03 compute_apps=4 orchestrator=alive zerostreak=0
+08:50:33 compute_apps=4 orchestrator=alive zerostreak=0
+08:51:03 compute_apps=4 orchestrator=alive zerostreak=0
+08:51:33 compute_apps=4 orchestrator=alive zerostreak=0
+08:52:04 compute_apps=4 orchestrator=alive zerostreak=0
+08:52:34 compute_apps=4 orchestrator=alive zerostreak=0
+08:53:04 compute_apps=4 orchestrator=alive zerostreak=0
+08:53:34 compute_apps=4 orchestrator=alive zerostreak=0
+08:54:04 compute_apps=4 orchestrator=alive zerostreak=0
+08:54:34 compute_apps=4 orchestrator=alive zerostreak=0
+08:55:04 compute_apps=4 orchestrator=alive zerostreak=0
+08:55:34 compute_apps=4 orchestrator=alive zerostreak=0
+08:56:04 compute_apps=4 orchestrator=alive zerostreak=0
+08:56:34 compute_apps=4 orchestrator=alive zerostreak=0
+08:57:04 compute_apps=3 orchestrator=alive zerostreak=0
+08:57:34 compute_apps=4 orchestrator=alive zerostreak=0
+08:58:04 compute_apps=4 orchestrator=alive zerostreak=0
+08:58:34 compute_apps=4 orchestrator=alive zerostreak=0
+08:59:04 compute_apps=4 orchestrator=alive zerostreak=0
+08:59:35 compute_apps=4 orchestrator=alive zerostreak=0
+09:00:05 compute_apps=4 orchestrator=alive zerostreak=0
+09:00:35 compute_apps=4 orchestrator=alive zerostreak=0
+09:01:05 compute_apps=4 orchestrator=alive zerostreak=0
+09:01:35 compute_apps=4 orchestrator=alive zerostreak=0
+09:02:05 compute_apps=4 orchestrator=alive zerostreak=0
+09:02:35 compute_apps=4 orchestrator=alive zerostreak=0
+09:03:05 compute_apps=4 orchestrator=alive zerostreak=0
+09:03:35 compute_apps=4 orchestrator=alive zerostreak=0
+09:04:05 compute_apps=4 orchestrator=alive zerostreak=0
+09:04:35 compute_apps=4 orchestrator=alive zerostreak=0
+09:05:05 compute_apps=4 orchestrator=alive zerostreak=0
+09:05:35 compute_apps=4 orchestrator=alive zerostreak=0
+09:06:05 compute_apps=4 orchestrator=alive zerostreak=0
+09:06:36 compute_apps=3 orchestrator=alive zerostreak=0
+09:07:06 compute_apps=4 orchestrator=alive zerostreak=0
+09:07:36 compute_apps=4 orchestrator=alive zerostreak=0
+09:08:06 compute_apps=4 orchestrator=alive zerostreak=0
+09:08:36 compute_apps=3 orchestrator=alive zerostreak=0
+09:09:06 compute_apps=4 orchestrator=alive zerostreak=0
+09:09:36 compute_apps=4 orchestrator=alive zerostreak=0
+09:10:06 compute_apps=4 orchestrator=alive zerostreak=0
+09:10:36 compute_apps=4 orchestrator=alive zerostreak=0
+09:11:06 compute_apps=4 orchestrator=alive zerostreak=0
+09:11:36 compute_apps=4 orchestrator=alive zerostreak=0
+09:12:06 compute_apps=4 orchestrator=alive zerostreak=0
+09:12:36 compute_apps=4 orchestrator=alive zerostreak=0
+09:13:06 compute_apps=4 orchestrator=alive zerostreak=0
+09:13:37 compute_apps=4 orchestrator=alive zerostreak=0
+09:14:07 compute_apps=4 orchestrator=alive zerostreak=0
+09:14:37 compute_apps=4 orchestrator=alive zerostreak=0
+09:15:07 compute_apps=4 orchestrator=alive zerostreak=0
+09:15:37 compute_apps=4 orchestrator=alive zerostreak=0
+09:16:07 compute_apps=3 orchestrator=alive zerostreak=0
+09:16:37 compute_apps=4 orchestrator=alive zerostreak=0
+09:17:07 compute_apps=4 orchestrator=alive zerostreak=0
+09:17:37 compute_apps=4 orchestrator=alive zerostreak=0
+09:18:07 compute_apps=3 orchestrator=alive zerostreak=0
+09:18:37 compute_apps=4 orchestrator=alive zerostreak=0
+09:19:07 compute_apps=4 orchestrator=alive zerostreak=0
+09:19:37 compute_apps=4 orchestrator=alive zerostreak=0
+09:20:07 compute_apps=4 orchestrator=alive zerostreak=0
+09:20:38 compute_apps=4 orchestrator=alive zerostreak=0
+09:21:08 compute_apps=4 orchestrator=alive zerostreak=0
+09:21:38 compute_apps=4 orchestrator=alive zerostreak=0
+09:22:08 compute_apps=4 orchestrator=alive zerostreak=0
+09:22:38 compute_apps=4 orchestrator=alive zerostreak=0
+09:23:08 compute_apps=4 orchestrator=alive zerostreak=0
+09:23:38 compute_apps=4 orchestrator=alive zerostreak=0
+09:24:08 compute_apps=4 orchestrator=alive zerostreak=0
+09:24:38 compute_apps=4 orchestrator=alive zerostreak=0
+09:25:08 compute_apps=4 orchestrator=alive zerostreak=0
+09:25:38 compute_apps=3 orchestrator=alive zerostreak=0
+09:26:08 compute_apps=4 orchestrator=alive zerostreak=0
+09:26:38 compute_apps=4 orchestrator=alive zerostreak=0
+09:27:08 compute_apps=4 orchestrator=alive zerostreak=0
+09:27:39 compute_apps=4 orchestrator=alive zerostreak=0
+09:28:09 compute_apps=3 orchestrator=alive zerostreak=0
+09:28:39 compute_apps=4 orchestrator=alive zerostreak=0
+09:29:09 compute_apps=4 orchestrator=alive zerostreak=0
+09:29:39 compute_apps=4 orchestrator=alive zerostreak=0
+09:30:09 compute_apps=4 orchestrator=alive zerostreak=0
+09:30:39 compute_apps=3 orchestrator=alive zerostreak=0
+09:31:09 compute_apps=4 orchestrator=alive zerostreak=0
+09:31:39 compute_apps=3 orchestrator=alive zerostreak=0
+09:32:09 compute_apps=3 orchestrator=alive zerostreak=0
+09:32:39 compute_apps=3 orchestrator=alive zerostreak=0
+09:33:09 compute_apps=2 orchestrator=alive zerostreak=0
+09:33:39 compute_apps=2 orchestrator=alive zerostreak=0
+09:34:10 compute_apps=2 orchestrator=alive zerostreak=0
+09:34:40 compute_apps=2 orchestrator=alive zerostreak=0
+09:35:10 compute_apps=2 orchestrator=alive zerostreak=0
+09:35:40 compute_apps=2 orchestrator=alive zerostreak=0
+09:36:10 compute_apps=2 orchestrator=alive zerostreak=0
+09:36:40 compute_apps=2 orchestrator=alive zerostreak=0
+09:37:10 compute_apps=2 orchestrator=alive zerostreak=0
+09:37:40 compute_apps=2 orchestrator=alive zerostreak=0
+09:38:10 compute_apps=2 orchestrator=alive zerostreak=0
+09:38:40 compute_apps=2 orchestrator=alive zerostreak=0
+09:39:10 compute_apps=2 orchestrator=alive zerostreak=0
+09:39:40 compute_apps=2 orchestrator=alive zerostreak=0
+09:40:10 compute_apps=1 orchestrator=gone zerostreak=0
+09:40:41 compute_apps=1 orchestrator=gone zerostreak=0
+09:41:11 compute_apps=1 orchestrator=gone zerostreak=0
+09:41:41 compute_apps=1 orchestrator=gone zerostreak=0
+09:42:11 compute_apps=1 orchestrator=gone zerostreak=0
+09:42:41 compute_apps=1 orchestrator=gone zerostreak=0
+09:43:11 compute_apps=1 orchestrator=gone zerostreak=0
+09:43:41 compute_apps=1 orchestrator=gone zerostreak=0
+09:44:11 compute_apps=1 orchestrator=gone zerostreak=0
+09:44:41 compute_apps=1 orchestrator=gone zerostreak=0
+09:45:11 compute_apps=1 orchestrator=gone zerostreak=0
+09:45:41 compute_apps=1 orchestrator=gone zerostreak=0
+09:46:11 compute_apps=1 orchestrator=gone zerostreak=0
+09:46:42 compute_apps=1 orchestrator=gone zerostreak=0
+09:47:12 compute_apps=1 orchestrator=gone zerostreak=0
+09:47:42 compute_apps=1 orchestrator=gone zerostreak=0
+09:48:12 compute_apps=1 orchestrator=gone zerostreak=0
+09:48:42 compute_apps=1 orchestrator=gone zerostreak=0
+09:49:12 compute_apps=1 orchestrator=gone zerostreak=0
+09:49:42 compute_apps=1 orchestrator=gone zerostreak=0
+09:50:12 compute_apps=1 orchestrator=gone zerostreak=0
+09:50:42 compute_apps=1 orchestrator=gone zerostreak=0
+09:51:12 compute_apps=1 orchestrator=gone zerostreak=0
+09:51:42 compute_apps=1 orchestrator=gone zerostreak=0
+09:52:12 compute_apps=1 orchestrator=gone zerostreak=0
+09:52:42 compute_apps=1 orchestrator=gone zerostreak=0
+09:53:12 compute_apps=1 orchestrator=gone zerostreak=0
+09:53:43 compute_apps=1 orchestrator=gone zerostreak=0
+09:54:13 compute_apps=1 orchestrator=gone zerostreak=0
+09:54:43 compute_apps=1 orchestrator=gone zerostreak=0
+09:55:13 compute_apps=1 orchestrator=gone zerostreak=0
+09:55:43 compute_apps=1 orchestrator=gone zerostreak=0
+09:56:13 compute_apps=1 orchestrator=gone zerostreak=0
+09:56:43 compute_apps=1 orchestrator=gone zerostreak=0
+09:57:13 compute_apps=1 orchestrator=gone zerostreak=0
+09:57:43 compute_apps=1 orchestrator=gone zerostreak=0
+09:58:13 compute_apps=1 orchestrator=gone zerostreak=0
+09:58:43 compute_apps=1 orchestrator=gone zerostreak=0
+09:59:13 compute_apps=1 orchestrator=gone zerostreak=0
+09:59:43 compute_apps=1 orchestrator=gone zerostreak=0
+10:00:14 compute_apps=1 orchestrator=gone zerostreak=0
+10:00:44 compute_apps=1 orchestrator=gone zerostreak=0
+10:01:14 compute_apps=1 orchestrator=gone zerostreak=0
+10:01:44 compute_apps=1 orchestrator=gone zerostreak=0
+10:02:14 compute_apps=1 orchestrator=gone zerostreak=0
+10:02:44 compute_apps=1 orchestrator=gone zerostreak=0
+10:03:14 compute_apps=1 orchestrator=gone zerostreak=0
+10:03:44 compute_apps=1 orchestrator=gone zerostreak=0
+10:04:14 compute_apps=1 orchestrator=gone zerostreak=0
+10:04:44 compute_apps=1 orchestrator=gone zerostreak=0
+10:05:14 compute_apps=1 orchestrator=gone zerostreak=0
+10:05:44 compute_apps=1 orchestrator=gone zerostreak=0
+10:06:14 compute_apps=1 orchestrator=gone zerostreak=0
+10:06:45 compute_apps=1 orchestrator=gone zerostreak=0
+10:07:15 compute_apps=1 orchestrator=gone zerostreak=0
+10:07:45 compute_apps=1 orchestrator=gone zerostreak=0
+10:08:15 compute_apps=1 orchestrator=gone zerostreak=0
+10:08:45 compute_apps=1 orchestrator=gone zerostreak=0
+10:09:15 compute_apps=1 orchestrator=gone zerostreak=0
+10:09:45 compute_apps=1 orchestrator=gone zerostreak=0
+10:10:15 compute_apps=1 orchestrator=gone zerostreak=0
+10:10:45 compute_apps=1 orchestrator=gone zerostreak=0
+10:11:15 compute_apps=1 orchestrator=gone zerostreak=0
+10:11:45 compute_apps=1 orchestrator=gone zerostreak=0
+10:12:15 compute_apps=1 orchestrator=gone zerostreak=0
+10:12:45 compute_apps=1 orchestrator=gone zerostreak=0
+10:13:16 compute_apps=1 orchestrator=gone zerostreak=0
+10:13:46 compute_apps=1 orchestrator=gone zerostreak=0
+10:14:16 compute_apps=1 orchestrator=gone zerostreak=0
+10:14:46 compute_apps=1 orchestrator=gone zerostreak=0
+10:15:16 compute_apps=1 orchestrator=gone zerostreak=0
+10:15:46 compute_apps=1 orchestrator=gone zerostreak=0
+10:16:16 compute_apps=1 orchestrator=gone zerostreak=0
+10:16:46 compute_apps=1 orchestrator=gone zerostreak=0
+10:17:16 compute_apps=1 orchestrator=gone zerostreak=0
+10:17:46 compute_apps=1 orchestrator=gone zerostreak=0
+10:18:16 compute_apps=1 orchestrator=gone zerostreak=0
+10:18:46 compute_apps=1 orchestrator=gone zerostreak=0
+10:19:16 compute_apps=1 orchestrator=gone zerostreak=0
+10:19:47 compute_apps=1 orchestrator=gone zerostreak=0
+10:20:17 compute_apps=1 orchestrator=gone zerostreak=0
+10:20:47 compute_apps=1 orchestrator=gone zerostreak=0
+10:21:17 compute_apps=1 orchestrator=gone zerostreak=0
+10:21:47 compute_apps=1 orchestrator=gone zerostreak=0
+10:22:17 compute_apps=1 orchestrator=gone zerostreak=0
+10:22:47 compute_apps=1 orchestrator=gone zerostreak=0
+10:23:17 compute_apps=1 orchestrator=gone zerostreak=0
+10:23:47 compute_apps=1 orchestrator=gone zerostreak=0
+10:24:17 compute_apps=1 orchestrator=gone zerostreak=0
+10:24:47 compute_apps=1 orchestrator=gone zerostreak=0
+10:25:17 compute_apps=1 orchestrator=gone zerostreak=0
+10:25:47 compute_apps=1 orchestrator=gone zerostreak=0
+10:26:17 compute_apps=1 orchestrator=gone zerostreak=0
+10:26:48 compute_apps=1 orchestrator=gone zerostreak=0
+10:27:18 compute_apps=1 orchestrator=gone zerostreak=0
+10:27:48 compute_apps=1 orchestrator=gone zerostreak=0
+10:28:18 compute_apps=1 orchestrator=gone zerostreak=0
+10:28:48 compute_apps=1 orchestrator=gone zerostreak=0
+10:29:18 compute_apps=1 orchestrator=gone zerostreak=0
+10:29:48 compute_apps=1 orchestrator=gone zerostreak=0
+10:30:18 compute_apps=1 orchestrator=gone zerostreak=0
+10:30:48 compute_apps=1 orchestrator=gone zerostreak=0
+10:31:18 compute_apps=1 orchestrator=gone zerostreak=0
+10:31:48 compute_apps=1 orchestrator=gone zerostreak=0
+10:32:18 compute_apps=1 orchestrator=gone zerostreak=0
+10:32:48 compute_apps=1 orchestrator=gone zerostreak=0
+10:33:19 compute_apps=1 orchestrator=gone zerostreak=0
+10:33:49 compute_apps=1 orchestrator=gone zerostreak=0
+10:34:19 compute_apps=1 orchestrator=gone zerostreak=0
+10:34:49 compute_apps=1 orchestrator=gone zerostreak=0
+10:35:19 compute_apps=1 orchestrator=gone zerostreak=0
+10:35:49 compute_apps=1 orchestrator=gone zerostreak=0
+10:36:19 compute_apps=1 orchestrator=gone zerostreak=0
+10:36:49 compute_apps=1 orchestrator=gone zerostreak=0
+10:37:19 compute_apps=1 orchestrator=gone zerostreak=0
+10:37:49 compute_apps=1 orchestrator=gone zerostreak=0
+10:38:19 compute_apps=1 orchestrator=gone zerostreak=0
+10:38:49 compute_apps=1 orchestrator=gone zerostreak=0
+10:39:19 compute_apps=1 orchestrator=gone zerostreak=0
+10:39:50 compute_apps=1 orchestrator=gone zerostreak=0
+10:40:20 compute_apps=1 orchestrator=gone zerostreak=0
+10:40:50 compute_apps=1 orchestrator=gone zerostreak=0
+10:41:20 compute_apps=1 orchestrator=gone zerostreak=0
+10:41:50 compute_apps=1 orchestrator=gone zerostreak=0
+10:42:20 compute_apps=1 orchestrator=gone zerostreak=0
+10:42:50 compute_apps=1 orchestrator=gone zerostreak=0
+10:43:20 compute_apps=1 orchestrator=gone zerostreak=0
+10:43:50 compute_apps=1 orchestrator=gone zerostreak=0
+10:44:20 compute_apps=1 orchestrator=gone zerostreak=0
+10:44:50 compute_apps=1 orchestrator=gone zerostreak=0
+10:45:20 compute_apps=1 orchestrator=gone zerostreak=0
+10:45:50 compute_apps=1 orchestrator=gone zerostreak=0
+10:46:21 compute_apps=1 orchestrator=gone zerostreak=0
+10:46:51 compute_apps=1 orchestrator=gone zerostreak=0
+10:47:21 compute_apps=1 orchestrator=gone zerostreak=0
+10:47:51 compute_apps=1 orchestrator=gone zerostreak=0
+10:48:21 compute_apps=1 orchestrator=gone zerostreak=0
+10:48:51 compute_apps=1 orchestrator=gone zerostreak=0
+10:49:21 compute_apps=1 orchestrator=gone zerostreak=0
+10:49:51 compute_apps=1 orchestrator=gone zerostreak=0
+10:50:21 compute_apps=1 orchestrator=gone zerostreak=0
+10:50:51 compute_apps=1 orchestrator=gone zerostreak=0
+10:51:21 compute_apps=1 orchestrator=gone zerostreak=0
+10:51:51 compute_apps=1 orchestrator=gone zerostreak=0
+10:52:21 compute_apps=1 orchestrator=gone zerostreak=0
+10:52:52 compute_apps=1 orchestrator=gone zerostreak=0
+10:53:22 compute_apps=1 orchestrator=gone zerostreak=0
+10:53:52 compute_apps=1 orchestrator=gone zerostreak=0
+10:54:22 compute_apps=1 orchestrator=gone zerostreak=0
+10:54:52 compute_apps=1 orchestrator=gone zerostreak=0
+10:55:22 compute_apps=1 orchestrator=gone zerostreak=0
+10:55:52 compute_apps=1 orchestrator=gone zerostreak=0
+10:56:22 compute_apps=1 orchestrator=gone zerostreak=0
+10:56:52 compute_apps=1 orchestrator=gone zerostreak=0
+10:57:22 compute_apps=1 orchestrator=gone zerostreak=0
+10:57:52 compute_apps=1 orchestrator=gone zerostreak=0
+10:58:22 compute_apps=1 orchestrator=gone zerostreak=0
+10:58:52 compute_apps=1 orchestrator=gone zerostreak=0
+10:59:23 compute_apps=1 orchestrator=gone zerostreak=0
+10:59:53 compute_apps=1 orchestrator=gone zerostreak=0
+11:00:23 compute_apps=1 orchestrator=gone zerostreak=0
+11:00:53 compute_apps=1 orchestrator=gone zerostreak=0
+11:01:23 compute_apps=1 orchestrator=gone zerostreak=0
+11:01:53 compute_apps=1 orchestrator=gone zerostreak=0
+11:02:23 compute_apps=1 orchestrator=gone zerostreak=0
+11:02:53 compute_apps=1 orchestrator=gone zerostreak=0
+11:03:23 compute_apps=1 orchestrator=gone zerostreak=0
+11:03:53 compute_apps=1 orchestrator=gone zerostreak=0
+11:04:23 compute_apps=1 orchestrator=gone zerostreak=0
+11:04:53 compute_apps=1 orchestrator=gone zerostreak=0
+11:05:23 compute_apps=1 orchestrator=gone zerostreak=0
+11:05:53 compute_apps=1 orchestrator=gone zerostreak=0
+11:06:24 compute_apps=1 orchestrator=gone zerostreak=0
+11:06:54 compute_apps=1 orchestrator=gone zerostreak=0
+11:07:24 compute_apps=1 orchestrator=gone zerostreak=0
+11:07:54 compute_apps=1 orchestrator=gone zerostreak=0
+11:08:24 compute_apps=1 orchestrator=gone zerostreak=0
+11:08:54 compute_apps=1 orchestrator=gone zerostreak=0
+11:09:24 compute_apps=1 orchestrator=gone zerostreak=0
+11:09:54 compute_apps=1 orchestrator=gone zerostreak=0
+11:10:24 compute_apps=1 orchestrator=gone zerostreak=0
+11:10:54 compute_apps=1 orchestrator=gone zerostreak=0
+11:11:24 compute_apps=1 orchestrator=gone zerostreak=0
+11:11:54 compute_apps=1 orchestrator=gone zerostreak=0
+11:12:24 compute_apps=1 orchestrator=gone zerostreak=0
+11:12:55 compute_apps=1 orchestrator=gone zerostreak=0
+11:13:25 compute_apps=1 orchestrator=gone zerostreak=0
+11:13:55 compute_apps=1 orchestrator=gone zerostreak=0
+11:14:25 compute_apps=1 orchestrator=gone zerostreak=0
+11:14:55 compute_apps=1 orchestrator=gone zerostreak=0
+11:15:25 compute_apps=1 orchestrator=gone zerostreak=0
+11:15:55 compute_apps=1 orchestrator=gone zerostreak=0
+11:16:25 compute_apps=1 orchestrator=gone zerostreak=0
+11:16:55 compute_apps=1 orchestrator=gone zerostreak=0
+11:17:25 compute_apps=1 orchestrator=gone zerostreak=0
+11:17:55 compute_apps=1 orchestrator=gone zerostreak=0
+11:18:25 compute_apps=1 orchestrator=gone zerostreak=0
+11:18:55 compute_apps=1 orchestrator=gone zerostreak=0
+11:19:25 compute_apps=1 orchestrator=gone zerostreak=0
+11:19:56 compute_apps=1 orchestrator=gone zerostreak=0
+11:20:26 compute_apps=1 orchestrator=gone zerostreak=0
+11:20:56 compute_apps=0 orchestrator=gone zerostreak=1
+11:21:26 compute_apps=0 orchestrator=gone zerostreak=2
+11:21:56 compute_apps=0 orchestrator=gone zerostreak=3
+11:21:56 ALL CLEAR (90s sustained)
+11:21:56 launch attempt 1/3
+11:21:56 LAUNCH: torchrun 4-GPU resampled
+11:24:56 VERIFY OK: procs=21 compute_apps=4
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/hn868f80
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+11:24:56 === SUCCESS: resampled launched+verified ===
+11:24:56 === watcher done ===

epfl_final/logs/eval_30k_battery.log

@@ -0,0 +1,28 @@
+[21:18:46] ckpt resampled -> /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-11700
+[21:18:46] ckpt fixed_s1 -> /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-11700
+[21:18:46] ckpt fixed_s2 -> /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-11700
+[21:18:46] ckpt dense -> /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-11700
+[21:18:46] total jobs=104  already-done=0  gpus=[0, 1, 2]  n=100
+[21:18:46] START gpu0 RULER_niah_multikey_resampled_ddense  (~104 remaining)
+[21:18:46] START gpu1 RULER_niah_multikey_resampled_d120  (~104 remaining)
+[21:18:46] START gpu2 RULER_niah_multikey_resampled_d60  (~104 remaining)
+[21:20:46] DONE  gpu0 RULER_niah_multikey_resampled_ddense rc=0 ok
+[21:20:46] START gpu0 RULER_niah_multikey_resampled_d31  (~103 remaining)
+[21:36:31] DONE  gpu0 RULER_niah_multikey_resampled_d31 rc=0 ok
+[21:36:31] START gpu0 RULER_niah_multikey_resampled_d16  (~102 remaining)
+[21:45:07] DONE  gpu2 RULER_niah_multikey_resampled_d60 rc=0 ok
+[21:45:07] START gpu2 RULER_niah_multikey_fixed_s1_ddense  (~101 remaining)
+[21:46:35] DONE  gpu0 RULER_niah_multikey_resampled_d16 rc=0 ok
+[21:46:35] START gpu0 RULER_niah_multikey_fixed_s1_d120  (~100 remaining)
+[21:46:55] DONE  gpu2 RULER_niah_multikey_fixed_s1_ddense rc=0 ok
+[21:46:55] START gpu2 RULER_niah_multikey_fixed_s1_d60  (~99 remaining)
+[22:07:13] DONE  gpu1 RULER_niah_multikey_resampled_d120 rc=0 ok
+[22:07:13] START gpu1 RULER_niah_multikey_fixed_s1_d31  (~98 remaining)
+[22:13:05] DONE  gpu2 RULER_niah_multikey_fixed_s1_d60 rc=0 ok
+[22:13:05] START gpu2 RULER_niah_multikey_fixed_s1_d16  (~97 remaining)
+[22:22:46] DONE  gpu1 RULER_niah_multikey_fixed_s1_d31 rc=0 ok
+[22:22:46] START gpu1 RULER_niah_multikey_fixed_s2_ddense  (~96 remaining)
+[22:23:10] DONE  gpu2 RULER_niah_multikey_fixed_s1_d16 rc=0 ok
+[22:23:10] START gpu2 RULER_niah_multikey_fixed_s2_d120  (~95 remaining)
+[22:24:34] DONE  gpu1 RULER_niah_multikey_fixed_s2_ddense rc=0 ok
+[22:24:34] START gpu1 RULER_niah_multikey_fixed_s2_d60  (~94 remaining)

epfl_final/logs/l30k1_2b_dense.log

@@ -0,0 +1,365 @@
+W0708 17:29:07.301000 6079 site-packages/torch/distributed/run.py:851] 
+W0708 17:29:07.301000 6079 site-packages/torch/distributed/run.py:851] *****************************************
+W0708 17:29:07.301000 6079 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0708 17:29:07.301000 6079 site-packages/torch/distributed/run.py:851] *****************************************
+Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized",
+  "val_path": null,
+  "max_seq_length": 30720,
+  "num_workers": 4,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 5,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 300,
+  "eval_steps": 500,
+  "eval_batches": 20,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b dense 30k longctx",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "dense",
+    "30k",
+    "longctx",
+    "b128",
+    "scaleladder"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 120,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300
+Checkpoint activations: True
+Checkpoint activations: TrueCheckpoint activations: True
+
+Checkpoint activations: True
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 120
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 30720
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 30720]                --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 30720, 1536]          --                        True
+│    └─Embedding: 2-1                                        [1, 30720]                [1, 30720, 1536]          233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 30720, 1536]          [1, 30720, 1536]          1,536                     True
+├─Linear: 1-2                                                [1, 30720, 1536]          [1, 30720, 151936]        233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.25
+Forward/backward pass size (MB): 98319.73
+Params size (MB): 2871.45
+Estimated Total Size (MB): 101191.43
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+
+
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+
+
+
+Dataset split: 749299 train, 39437 validation
+Training examples (packed, train split):  749299
+Validation examples (packed, val split):  39437
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              5
+Effective batch size (seqs/step):         20
+Sequence length:                          30720
+Tokens per optimizer step:                614,400
+Micro-batches per epoch (per rank):       187325
+Optimizer steps per epoch:                37465
+Total optimizer steps:                    37465
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300...
+Overriding checkpoint LR with config LR: 0.0005
+Resumed from checkpoint at step 3300 (epoch 0, micro_step 16500)
+Resuming training from step 3300
+Resuming wandb run cd66e13p
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run cd66e13p
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_173250-cd66e13p
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b dense 30k longctx
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/cd66e13p
+Starting training...
+Validation at step 3300: Loss = 2.5783, Perplexity = 13.17
+wandb: WARNING Tried to log to step 3300 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 3301: dense (dense step interval)
+Step 3310/37465 | Loss: 2.6606 | LR: 4.94e-04 | 16887.6ms/step | 1.2 samples/s
+wandb: WARNING Tried to log to step 3310 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3320/37465 | Loss: 2.3535 | LR: 4.94e-04 | 8333.4ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3320 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3330/37465 | Loss: 2.6802 | LR: 4.94e-04 | 8352.0ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3330 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3340/37465 | Loss: 2.6431 | LR: 4.94e-04 | 8344.2ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3340 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3350/37465 | Loss: 2.9170 | LR: 4.94e-04 | 8354.5ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3350 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3360/37465 | Loss: 2.5131 | LR: 4.94e-04 | 8360.2ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3360 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3370/37465 | Loss: 2.5432 | LR: 4.94e-04 | 8350.8ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3370 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3380/37465 | Loss: 2.5797 | LR: 4.94e-04 | 8388.2ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3380 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3390/37465 | Loss: 2.4650 | LR: 4.94e-04 | 8355.6ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3390 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3400/37465 | Loss: 2.3816 | LR: 4.94e-04 | 8338.4ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3400 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3410/37465 | Loss: 2.4314 | LR: 4.94e-04 | 8332.0ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3410 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3420/37465 | Loss: 2.5678 | LR: 4.94e-04 | 8368.0ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3420 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3430/37465 | Loss: 2.7321 | LR: 4.93e-04 | 8339.8ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3430 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3440/37465 | Loss: 2.6158 | LR: 4.93e-04 | 8360.0ms/step | 2.4 samples/s
+Step 3450/37465 | Loss: 2.5854 | LR: 4.93e-04 | 8369.7ms/step | 2.4 samples/s
+Step 3460/37465 | Loss: 2.4706 | LR: 4.93e-04 | 8346.4ms/step | 2.4 samples/s
+Step 3470/37465 | Loss: 2.3381 | LR: 4.93e-04 | 8365.9ms/step | 2.4 samples/s
+Step 3480/37465 | Loss: 2.4953 | LR: 4.93e-04 | 8343.1ms/step | 2.4 samples/s
+Step 3490/37465 | Loss: 2.6281 | LR: 4.93e-04 | 8368.3ms/step | 2.4 samples/s
+Step 3500/37465 | Loss: 2.5841 | LR: 4.93e-04 | 8351.0ms/step | 2.4 samples/s
+Validation at step 3500: Loss = 2.5524, Perplexity = 12.84
+Step 3510/37465 | Loss: 2.7820 | LR: 4.93e-04 | 9361.4ms/step | 2.1 samples/s
+Step 3520/37465 | Loss: 2.4614 | LR: 4.93e-04 | 8371.8ms/step | 2.4 samples/s
+Step 3530/37465 | Loss: 2.5079 | LR: 4.93e-04 | 8353.9ms/step | 2.4 samples/s
+Step 3540/37465 | Loss: 2.6692 | LR: 4.93e-04 | 8370.6ms/step | 2.4 samples/s
+Step 3550/37465 | Loss: 2.5327 | LR: 4.93e-04 | 8378.5ms/step | 2.4 samples/s
+Step 3560/37465 | Loss: 2.4459 | LR: 4.93e-04 | 8348.7ms/step | 2.4 samples/s
+Step 3570/37465 | Loss: 2.4465 | LR: 4.93e-04 | 8366.9ms/step | 2.4 samples/s
+Step 3580/37465 | Loss: 2.8736 | LR: 4.93e-04 | 8373.4ms/step | 2.4 samples/s
+Step 3590/37465 | Loss: 2.5086 | LR: 4.93e-04 | 8355.8ms/step | 2.4 samples/s
+Step 3600/37465 | Loss: 2.6325 | LR: 4.93e-04 | 8345.8ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3600
+Step 3610/37465 | Loss: 2.5135 | LR: 4.93e-04 | 9375.9ms/step | 2.1 samples/s
+Step 3620/37465 | Loss: 2.4450 | LR: 4.92e-04 | 8396.5ms/step | 2.4 samples/s
+Step 3630/37465 | Loss: 2.3774 | LR: 4.92e-04 | 8368.6ms/step | 2.4 samples/s
+Step 3640/37465 | Loss: 2.4915 | LR: 4.92e-04 | 8383.0ms/step | 2.4 samples/s
+Step 3650/37465 | Loss: 2.5852 | LR: 4.92e-04 | 8368.3ms/step | 2.4 samples/s
+Step 3660/37465 | Loss: 2.4715 | LR: 4.92e-04 | 8369.6ms/step | 2.4 samples/s
+Step 3670/37465 | Loss: 2.3686 | LR: 4.92e-04 | 8378.3ms/step | 2.4 samples/s
+Step 3680/37465 | Loss: 2.6847 | LR: 4.92e-04 | 8372.5ms/step | 2.4 samples/s
+Step 3690/37465 | Loss: 2.8277 | LR: 4.92e-04 | 8361.6ms/step | 2.4 samples/s
+Step 3700/37465 | Loss: 2.4927 | LR: 4.92e-04 | 8381.6ms/step | 2.4 samples/s
+Step 3710/37465 | Loss: 2.7802 | LR: 4.92e-04 | 8385.2ms/step | 2.4 samples/s
+Step 3720/37465 | Loss: 2.5450 | LR: 4.92e-04 | 8361.8ms/step | 2.4 samples/s
+Step 3730/37465 | Loss: 2.7159 | LR: 4.92e-04 | 8368.8ms/step | 2.4 samples/s
+Step 3740/37465 | Loss: 2.3565 | LR: 4.92e-04 | 8356.7ms/step | 2.4 samples/s
+Step 3750/37465 | Loss: 2.5005 | LR: 4.92e-04 | 8359.8ms/step | 2.4 samples/s
+Step 3760/37465 | Loss: 2.7336 | LR: 4.92e-04 | 8391.1ms/step | 2.4 samples/s
+Step 3770/37465 | Loss: 2.7054 | LR: 4.92e-04 | 8368.4ms/step | 2.4 samples/s
+Step 3780/37465 | Loss: 2.5842 | LR: 4.92e-04 | 8387.8ms/step | 2.4 samples/s
+Step 3790/37465 | Loss: 2.4455 | LR: 4.92e-04 | 8385.2ms/step | 2.4 samples/s
+Step 3800/37465 | Loss: 2.5453 | LR: 4.92e-04 | 8380.5ms/step | 2.4 samples/s
+Step 3810/37465 | Loss: 2.6389 | LR: 4.91e-04 | 8374.7ms/step | 2.4 samples/s
+Step 3820/37465 | Loss: 2.5859 | LR: 4.91e-04 | 8377.2ms/step | 2.4 samples/s
+Step 3830/37465 | Loss: 2.7157 | LR: 4.91e-04 | 8396.2ms/step | 2.4 samples/s
+Step 3840/37465 | Loss: 2.2813 | LR: 4.91e-04 | 8359.1ms/step | 2.4 samples/s
+Step 3850/37465 | Loss: 2.6667 | LR: 4.91e-04 | 8374.8ms/step | 2.4 samples/s
+Step 3860/37465 | Loss: 2.3919 | LR: 4.91e-04 | 8379.6ms/step | 2.4 samples/s
+Step 3870/37465 | Loss: 2.7215 | LR: 4.91e-04 | 8365.4ms/step | 2.4 samples/s
+Step 3880/37465 | Loss: 2.3145 | LR: 4.91e-04 | 8378.0ms/step | 2.4 samples/s
+Step 3890/37465 | Loss: 2.3002 | LR: 4.91e-04 | 8357.4ms/step | 2.4 samples/s
+Step 3900/37465 | Loss: 2.3930 | LR: 4.91e-04 | 8363.6ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3900
+Step 3910/37465 | Loss: 2.5740 | LR: 4.91e-04 | 9211.5ms/step | 2.2 samples/s
+Step 3920/37465 | Loss: 2.5662 | LR: 4.91e-04 | 8365.1ms/step | 2.4 samples/s
+Step 3930/37465 | Loss: 2.6290 | LR: 4.91e-04 | 8380.3ms/step | 2.4 samples/s
+Step 3940/37465 | Loss: 2.8159 | LR: 4.91e-04 | 8356.3ms/step | 2.4 samples/s
+Step 3950/37465 | Loss: 2.3214 | LR: 4.91e-04 | 8372.7ms/step | 2.4 samples/s
+Step 3960/37465 | Loss: 2.5481 | LR: 4.91e-04 | 8354.2ms/step | 2.4 samples/s
+Step 3970/37465 | Loss: 2.3466 | LR: 4.91e-04 | 8354.4ms/step | 2.4 samples/s
+Step 3980/37465 | Loss: 2.5501 | LR: 4.91e-04 | 8370.4ms/step | 2.4 samples/s
+Step 3990/37465 | Loss: 2.5272 | LR: 4.90e-04 | 8368.3ms/step | 2.4 samples/s
+Step 4000/37465 | Loss: 2.5115 | LR: 4.90e-04 | 8344.0ms/step | 2.4 samples/s
+Validation at step 4000: Loss = 2.5080, Perplexity = 12.28
+Step 4010/37465 | Loss: 2.1113 | LR: 4.90e-04 | 9366.2ms/step | 2.1 samples/s
+Step 4020/37465 | Loss: 2.2947 | LR: 4.90e-04 | 8339.8ms/step | 2.4 samples/s
+Step 4030/37465 | Loss: 2.4372 | LR: 4.90e-04 | 8352.1ms/step | 2.4 samples/s
+Step 4040/37465 | Loss: 2.4509 | LR: 4.90e-04 | 8351.1ms/step | 2.4 samples/s
+Step 4050/37465 | Loss: 2.6699 | LR: 4.90e-04 | 8366.0ms/step | 2.4 samples/s
+Step 4060/37465 | Loss: 2.5102 | LR: 4.90e-04 | 8357.5ms/step | 2.4 samples/s
+Step 4070/37465 | Loss: 2.4825 | LR: 4.90e-04 | 8367.2ms/step | 2.4 samples/s
+Step 4080/37465 | Loss: 2.4585 | LR: 4.90e-04 | 8340.0ms/step | 2.4 samples/s
+Step 4090/37465 | Loss: 2.5597 | LR: 4.90e-04 | 8359.6ms/step | 2.4 samples/s
+Step 4100/37465 | Loss: 2.3972 | LR: 4.90e-04 | 8364.1ms/step | 2.4 samples/s
+Step 4110/37465 | Loss: 2.5406 | LR: 4.90e-04 | 8352.6ms/step | 2.4 samples/s
+Step 4120/37465 | Loss: 2.3038 | LR: 4.90e-04 | 8352.6ms/step | 2.4 samples/s
+Step 4130/37465 | Loss: 2.2725 | LR: 4.90e-04 | 8359.4ms/step | 2.4 samples/s
+Step 4140/37465 | Loss: 2.5664 | LR: 4.90e-04 | 8358.2ms/step | 2.4 samples/s
+Step 4150/37465 | Loss: 2.5551 | LR: 4.89e-04 | 8347.4ms/step | 2.4 samples/s
+Step 4160/37465 | Loss: 2.5973 | LR: 4.89e-04 | 8338.1ms/step | 2.4 samples/s
+Step 4170/37465 | Loss: 2.4961 | LR: 4.89e-04 | 8352.2ms/step | 2.4 samples/s
+Step 4180/37465 | Loss: 2.2842 | LR: 4.89e-04 | 8351.2ms/step | 2.4 samples/s
+Step 4190/37465 | Loss: 2.5623 | LR: 4.89e-04 | 8356.2ms/step | 2.4 samples/s
+Step 4200/37465 | Loss: 2.5244 | LR: 4.89e-04 | 8359.8ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-4200
+Step 4210/37465 | Loss: 2.1315 | LR: 4.89e-04 | 9181.9ms/step | 2.2 samples/s
+Step 4220/37465 | Loss: 2.4843 | LR: 4.89e-04 | 8349.2ms/step | 2.4 samples/s
+Step 4230/37465 | Loss: 2.3858 | LR: 4.89e-04 | 8357.1ms/step | 2.4 samples/s
+Step 4240/37465 | Loss: 2.4300 | LR: 4.89e-04 | 8369.7ms/step | 2.4 samples/s
+Step 4250/37465 | Loss: 2.3458 | LR: 4.89e-04 | 8377.7ms/step | 2.4 samples/s
+Step 4260/37465 | Loss: 2.4023 | LR: 4.89e-04 | 8367.7ms/step | 2.4 samples/s
+Step 4270/37465 | Loss: 2.4831 | LR: 4.89e-04 | 8347.0ms/step | 2.4 samples/s
+Step 4280/37465 | Loss: 2.4608 | LR: 4.89e-04 | 8362.1ms/step | 2.4 samples/s
+Step 4290/37465 | Loss: 2.5375 | LR: 4.89e-04 | 8377.8ms/step | 2.4 samples/s
+Step 4300/37465 | Loss: 2.4230 | LR: 4.89e-04 | 8363.5ms/step | 2.4 samples/s
+Step 4310/37465 | Loss: 2.6084 | LR: 4.88e-04 | 8364.0ms/step | 2.4 samples/s
+Step 4320/37465 | Loss: 2.3435 | LR: 4.88e-04 | 8351.6ms/step | 2.4 samples/s
+Step 4330/37465 | Loss: 2.2068 | LR: 4.88e-04 | 8377.3ms/step | 2.4 samples/s
+Step 4340/37465 | Loss: 2.5103 | LR: 4.88e-04 | 8364.9ms/step | 2.4 samples/s
+Step 4350/37465 | Loss: 2.6194 | LR: 4.88e-04 | 8333.5ms/step | 2.4 samples/s
+Step 4360/37465 | Loss: 2.5892 | LR: 4.88e-04 | 8381.0ms/step | 2.4 samples/s
+Step 4370/37465 | Loss: 2.4106 | LR: 4.88e-04 | 8374.3ms/step | 2.4 samples/s
+Step 4380/37465 | Loss: 2.4141 | LR: 4.88e-04 | 8380.8ms/step | 2.4 samples/s
+Step 4390/37465 | Loss: 2.5619 | LR: 4.88e-04 | 8334.0ms/step | 2.4 samples/s
+Step 4400/37465 | Loss: 2.3260 | LR: 4.88e-04 | 8380.4ms/step | 2.4 samples/s
+Step 4410/37465 | Loss: 2.5833 | LR: 4.88e-04 | 8362.6ms/step | 2.4 samples/s
+Step 4420/37465 | Loss: 2.3065 | LR: 4.88e-04 | 8388.2ms/step | 2.4 samples/s
+Step 4430/37465 | Loss: 2.5054 | LR: 4.88e-04 | 8416.8ms/step | 2.4 samples/s
+Step 4440/37465 | Loss: 2.3477 | LR: 4.88e-04 | 8370.3ms/step | 2.4 samples/s
+Step 4450/37465 | Loss: 2.4271 | LR: 4.88e-04 | 8381.1ms/step | 2.4 samples/s
+Step 4460/37465 | Loss: 2.3098 | LR: 4.88e-04 | 8334.6ms/step | 2.4 samples/s
+Step 4470/37465 | Loss: 2.3284 | LR: 4.87e-04 | 8364.4ms/step | 2.4 samples/s
+Step 4480/37465 | Loss: 2.3389 | LR: 4.87e-04 | 8353.3ms/step | 2.4 samples/s
+Step 4490/37465 | Loss: 2.4233 | LR: 4.87e-04 | 8360.4ms/step | 2.4 samples/s
+Step 4500/37465 | Loss: 2.6390 | LR: 4.87e-04 | 8391.1ms/step | 2.4 samples/s
+Validation at step 4500: Loss = 2.4759, Perplexity = 11.89
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-4500
+Step 4510/37465 | Loss: 2.3560 | LR: 4.87e-04 | 10250.6ms/step | 2.0 samples/s
+Step 4520/37465 | Loss: 2.4278 | LR: 4.87e-04 | 8351.7ms/step | 2.4 samples/s
+Step 4530/37465 | Loss: 2.5453 | LR: 4.87e-04 | 8374.4ms/step | 2.4 samples/s
+Step 4540/37465 | Loss: 2.4651 | LR: 4.87e-04 | 8386.8ms/step | 2.4 samples/s
+Step 4550/37465 | Loss: 2.4067 | LR: 4.87e-04 | 8401.2ms/step | 2.4 samples/s
+Step 4560/37465 | Loss: 2.4239 | LR: 4.87e-04 | 8392.5ms/step | 2.4 samples/s
+Step 4570/37465 | Loss: 2.2015 | LR: 4.87e-04 | 8395.1ms/step | 2.4 samples/s
+Step 4580/37465 | Loss: 2.5254 | LR: 4.87e-04 | 8394.8ms/step | 2.4 samples/s
+Step 4590/37465 | Loss: 2.4194 | LR: 4.87e-04 | 8396.8ms/step | 2.4 samples/s
+Step 4600/37465 | Loss: 2.6416 | LR: 4.87e-04 | 8384.7ms/step | 2.4 samples/s
+Step 4610/37465 | Loss: 2.3960 | LR: 4.86e-04 | 8380.4ms/step | 2.4 samples/s
+Step 4620/37465 | Loss: 2.5731 | LR: 4.86e-04 | 8385.8ms/step | 2.4 samples/s
+Step 4630/37465 | Loss: 2.5278 | LR: 4.86e-04 | 8355.9ms/step | 2.4 samples/s
+Step 4640/37465 | Loss: 2.3128 | LR: 4.86e-04 | 8368.9ms/step | 2.4 samples/s
+Step 4650/37465 | Loss: 2.2687 | LR: 4.86e-04 | 8420.4ms/step | 2.4 samples/s
+Step 4660/37465 | Loss: 2.4507 | LR: 4.86e-04 | 8403.7ms/step | 2.4 samples/s
+Step 4670/37465 | Loss: 2.4984 | LR: 4.86e-04 | 8415.4ms/step | 2.4 samples/s
+Step 4680/37465 | Loss: 2.4701 | LR: 4.86e-04 | 8359.8ms/step | 2.4 samples/s
+Step 4690/37465 | Loss: 2.6007 | LR: 4.86e-04 | 8353.6ms/step | 2.4 samples/s
+Step 4700/37465 | Loss: 2.4826 | LR: 4.86e-04 | 8402.5ms/step | 2.4 samples/s
+Step 4710/37465 | Loss: 2.2913 | LR: 4.86e-04 | 8369.6ms/step | 2.4 samples/s
+Step 4720/37465 | Loss: 2.5531 | LR: 4.86e-04 | 8378.1ms/step | 2.4 samples/s
+Step 4730/37465 | Loss: 2.3186 | LR: 4.86e-04 | 8362.9ms/step | 2.4 samples/s
+Step 4740/37465 | Loss: 2.4826 | LR: 4.86e-04 | 8392.5ms/step | 2.4 samples/s
+Step 4750/37465 | Loss: 2.4423 | LR: 4.85e-04 | 8381.7ms/step | 2.4 samples/s
+Step 4760/37465 | Loss: 2.3839 | LR: 4.85e-04 | 8358.8ms/step | 2.4 samples/s
+Step 4770/37465 | Loss: 2.3475 | LR: 4.85e-04 | 8386.0ms/step | 2.4 samples/s
+Step 4780/37465 | Loss: 2.5026 | LR: 4.85e-04 | 8373.9ms/step | 2.4 samples/s
+Step 4790/37465 | Loss: 2.6148 | LR: 4.85e-04 | 8389.5ms/step | 2.4 samples/s
+Step 4800/37465 | Loss: 2.4976 | LR: 4.85e-04 | 8363.1ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-4800
+Step 4810/37465 | Loss: 2.5211 | LR: 4.85e-04 | 9312.5ms/step | 2.1 samples/s

epfl_final/logs/l30k1_2b_dense.log.old2

@@ -0,0 +1,222 @@
+W0708 06:36:11.151000 2475 site-packages/torch/distributed/run.py:851] 
+W0708 06:36:11.151000 2475 site-packages/torch/distributed/run.py:851] *****************************************
+W0708 06:36:11.151000 2475 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0708 06:36:11.151000 2475 site-packages/torch/distributed/run.py:851] *****************************************
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_063617
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized",
+  "val_path": null,
+  "max_seq_length": 30720,
+  "num_workers": 4,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_063617",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 5,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 300,
+  "eval_steps": 500,
+  "eval_batches": 20,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b dense 30k longctx",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "dense",
+    "30k",
+    "longctx",
+    "b128",
+    "scaleladder"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 120,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Initializing model from scratch (random weights)...
+Checkpoint activations: True
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 120
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 30720
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 30720]                --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 30720, 1536]          --                        True
+│    └─Embedding: 2-1                                        [1, 30720]                [1, 30720, 1536]          233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 30720, 1536]          [1, 30720, 1536]          1,536                     True
+├─Linear: 1-2                                                [1, 30720, 1536]          [1, 30720, 151936]        233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.25
+Forward/backward pass size (MB): 98319.73
+Params size (MB): 2871.45
+Estimated Total Size (MB): 101191.43
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Dataset split: 749299 train, 39437 validation
+Training examples (packed, train split):  749299
+Validation examples (packed, val split):  39437
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              5
+Effective batch size (seqs/step):         20
+Sequence length:                          30720
+Tokens per optimizer step:                614,400
+Micro-batches per epoch (per rank):       187325
+Optimizer steps per epoch:                37465
+Total optimizer steps:                    37465
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_063704-t6a1jz9b
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 1.2b dense 30k longctx
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/t6a1jz9b
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/37465 | Loss: 12.2534 | LR: 6.67e-06 | 13041.4ms/step | 1.5 samples/s
+Step 20/37465 | Loss: 12.1201 | LR: 1.33e-05 | 6349.5ms/step | 3.1 samples/s
+Step 30/37465 | Loss: 11.6859 | LR: 2.00e-05 | 6356.2ms/step | 3.1 samples/s
+Step 40/37465 | Loss: 10.6683 | LR: 2.67e-05 | 6340.9ms/step | 3.2 samples/s
+Step 50/37465 | Loss: 10.0551 | LR: 3.33e-05 | 6325.8ms/step | 3.2 samples/s
+W0708 06:43:38.137000 2475 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2475 got signal: 15

epfl_final/logs/l30k1_2b_dense.log.reclaim_20260708_172842

@@ -0,0 +1,546 @@
+W0708 06:51:07.977000 139 site-packages/torch/distributed/run.py:851] 
+W0708 06:51:07.977000 139 site-packages/torch/distributed/run.py:851] *****************************************
+W0708 06:51:07.977000 139 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0708 06:51:07.977000 139 site-packages/torch/distributed/run.py:851] *****************************************
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized",
+  "val_path": null,
+  "max_seq_length": 30720,
+  "num_workers": 4,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 5,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 300,
+  "eval_steps": 500,
+  "eval_batches": 20,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b dense 30k longctx",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "dense",
+    "30k",
+    "longctx",
+    "b128",
+    "scaleladder"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 120,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 120
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 30720
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 30720]                --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 30720, 1536]          --                        True
+│    └─Embedding: 2-1                                        [1, 30720]                [1, 30720, 1536]          233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+���    │    └─Qwen3DecoderLayer: 3-4                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 30720, 1536]          [1, 30720, 1536]          1,536                     True
+├─Linear: 1-2                                                [1, 30720, 1536]          [1, 30720, 151936]        233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.25
+Forward/backward pass size (MB): 98319.73
+Params size (MB): 2871.45
+Estimated Total Size (MB): 101191.43
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Dataset split: 749299 train, 39437 validation
+Training examples (packed, train split):  749299
+Validation examples (packed, val split):  39437
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              5
+Effective batch size (seqs/step):         20
+Sequence length:                          30720
+Tokens per optimizer step:                614,400
+Micro-batches per epoch (per rank):       187325
+Optimizer steps per epoch:                37465
+Total optimizer steps:                    37465
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run cd66e13p
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_065216-cd66e13p
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 1.2b dense 30k longctx
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/cd66e13p
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/37465 | Loss: 12.2534 | LR: 6.67e-06 | 16466.7ms/step | 1.2 samples/s
+Step 20/37465 | Loss: 12.1201 | LR: 1.33e-05 | 6357.6ms/step | 3.1 samples/s
+Step 30/37465 | Loss: 11.6858 | LR: 2.00e-05 | 6363.5ms/step | 3.1 samples/s
+Step 40/37465 | Loss: 10.6683 | LR: 2.67e-05 | 6353.9ms/step | 3.1 samples/s
+Step 50/37465 | Loss: 10.0551 | LR: 3.33e-05 | 6330.1ms/step | 3.2 samples/s
+Step 60/37465 | Loss: 9.5722 | LR: 4.00e-05 | 6319.8ms/step | 3.2 samples/s
+Step 70/37465 | Loss: 9.1242 | LR: 4.67e-05 | 6319.8ms/step | 3.2 samples/s
+Step 80/37465 | Loss: 8.6455 | LR: 5.33e-05 | 6326.9ms/step | 3.2 samples/s
+Step 90/37465 | Loss: 8.0658 | LR: 6.00e-05 | 6329.3ms/step | 3.2 samples/s
+Step 100/37465 | Loss: 7.7199 | LR: 6.67e-05 | 6327.5ms/step | 3.2 samples/s
+Step 110/37465 | Loss: 7.3702 | LR: 7.33e-05 | 6331.3ms/step | 3.2 samples/s
+Step 120/37465 | Loss: 7.1084 | LR: 8.00e-05 | 6339.4ms/step | 3.2 samples/s
+Step 130/37465 | Loss: 6.9197 | LR: 8.67e-05 | 6338.2ms/step | 3.2 samples/s
+Step 140/37465 | Loss: 6.8033 | LR: 9.33e-05 | 6347.0ms/step | 3.2 samples/s
+Step 150/37465 | Loss: 6.8035 | LR: 1.00e-04 | 6351.2ms/step | 3.1 samples/s
+Step 160/37465 | Loss: 6.6536 | LR: 1.07e-04 | 6354.2ms/step | 3.1 samples/s
+Step 170/37465 | Loss: 6.3240 | LR: 1.13e-04 | 6361.6ms/step | 3.1 samples/s
+Step 180/37465 | Loss: 6.1787 | LR: 1.20e-04 | 6363.3ms/step | 3.1 samples/s
+Step 190/37465 | Loss: 6.2520 | LR: 1.27e-04 | 6359.6ms/step | 3.1 samples/s
+Step 200/37465 | Loss: 6.0200 | LR: 1.33e-04 | 6364.1ms/step | 3.1 samples/s
+Step 210/37465 | Loss: 6.0067 | LR: 1.40e-04 | 6360.8ms/step | 3.1 samples/s
+Step 220/37465 | Loss: 5.9344 | LR: 1.47e-04 | 6362.6ms/step | 3.1 samples/s
+Step 230/37465 | Loss: 5.9027 | LR: 1.53e-04 | 6366.6ms/step | 3.1 samples/s
+Step 240/37465 | Loss: 5.8670 | LR: 1.60e-04 | 6356.6ms/step | 3.1 samples/s
+Step 250/37465 | Loss: 5.8162 | LR: 1.67e-04 | 6368.7ms/step | 3.1 samples/s
+Step 260/37465 | Loss: 5.6788 | LR: 1.73e-04 | 6369.0ms/step | 3.1 samples/s
+Step 270/37465 | Loss: 5.7045 | LR: 1.80e-04 | 6368.3ms/step | 3.1 samples/s
+Step 280/37465 | Loss: 5.7118 | LR: 1.87e-04 | 6360.1ms/step | 3.1 samples/s
+Step 290/37465 | Loss: 5.5616 | LR: 1.93e-04 | 6365.8ms/step | 3.1 samples/s
+Step 300/37465 | Loss: 5.3729 | LR: 2.00e-04 | 6371.3ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-300
+Step 310/37465 | Loss: 5.2669 | LR: 2.07e-04 | 7332.6ms/step | 2.7 samples/s
+Step 320/37465 | Loss: 5.3585 | LR: 2.13e-04 | 6363.0ms/step | 3.1 samples/s
+Step 330/37465 | Loss: 5.5393 | LR: 2.20e-04 | 6364.3ms/step | 3.1 samples/s
+Step 340/37465 | Loss: 5.3772 | LR: 2.27e-04 | 6362.6ms/step | 3.1 samples/s
+Step 350/37465 | Loss: 5.2125 | LR: 2.33e-04 | 6367.0ms/step | 3.1 samples/s
+Step 360/37465 | Loss: 5.2798 | LR: 2.40e-04 | 6363.3ms/step | 3.1 samples/s
+Step 370/37465 | Loss: 5.1172 | LR: 2.47e-04 | 6363.6ms/step | 3.1 samples/s
+Step 380/37465 | Loss: 5.3529 | LR: 2.53e-04 | 6367.7ms/step | 3.1 samples/s
+Step 390/37465 | Loss: 5.2645 | LR: 2.60e-04 | 6367.3ms/step | 3.1 samples/s
+Step 400/37465 | Loss: 5.2519 | LR: 2.67e-04 | 6365.2ms/step | 3.1 samples/s
+Step 410/37465 | Loss: 5.0620 | LR: 2.73e-04 | 6366.8ms/step | 3.1 samples/s
+Step 420/37465 | Loss: 4.8713 | LR: 2.80e-04 | 6364.6ms/step | 3.1 samples/s
+Step 430/37465 | Loss: 4.9729 | LR: 2.87e-04 | 6365.5ms/step | 3.1 samples/s
+Step 440/37465 | Loss: 4.8967 | LR: 2.93e-04 | 6369.5ms/step | 3.1 samples/s
+Step 450/37465 | Loss: 4.8868 | LR: 3.00e-04 | 6367.4ms/step | 3.1 samples/s
+Step 460/37465 | Loss: 4.6993 | LR: 3.07e-04 | 6368.1ms/step | 3.1 samples/s
+Step 470/37465 | Loss: 4.7116 | LR: 3.13e-04 | 6370.6ms/step | 3.1 samples/s
+Step 480/37465 | Loss: 4.8928 | LR: 3.20e-04 | 6368.0ms/step | 3.1 samples/s
+Step 490/37465 | Loss: 4.6241 | LR: 3.27e-04 | 6368.7ms/step | 3.1 samples/s
+Step 500/37465 | Loss: 4.8609 | LR: 3.33e-04 | 6369.7ms/step | 3.1 samples/s
+Validation at step 500: Loss = 4.6498, Perplexity = 104.56
+Step 510/37465 | Loss: 4.4969 | LR: 3.40e-04 | 10432.4ms/step | 1.9 samples/s
+Step 520/37465 | Loss: 4.6458 | LR: 3.47e-04 | 6360.4ms/step | 3.1 samples/s
+Step 530/37465 | Loss: 4.8233 | LR: 3.53e-04 | 6367.5ms/step | 3.1 samples/s
+Step 540/37465 | Loss: 4.9458 | LR: 3.60e-04 | 6369.9ms/step | 3.1 samples/s
+Step 550/37465 | Loss: 4.4279 | LR: 3.67e-04 | 6367.6ms/step | 3.1 samples/s
+Step 560/37465 | Loss: 4.4328 | LR: 3.73e-04 | 6368.9ms/step | 3.1 samples/s
+Step 570/37465 | Loss: 4.4073 | LR: 3.80e-04 | 6368.9ms/step | 3.1 samples/s
+Step 580/37465 | Loss: 4.0842 | LR: 3.87e-04 | 6369.7ms/step | 3.1 samples/s
+Step 590/37465 | Loss: 4.4378 | LR: 3.93e-04 | 6371.7ms/step | 3.1 samples/s
+Step 600/37465 | Loss: 4.5479 | LR: 4.00e-04 | 6371.6ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-600
+Step 610/37465 | Loss: 4.2887 | LR: 4.07e-04 | 7304.6ms/step | 2.7 samples/s
+Step 620/37465 | Loss: 4.1438 | LR: 4.13e-04 | 6380.9ms/step | 3.1 samples/s
+Step 630/37465 | Loss: 4.2081 | LR: 4.20e-04 | 6371.7ms/step | 3.1 samples/s
+Step 640/37465 | Loss: 4.2991 | LR: 4.27e-04 | 6372.2ms/step | 3.1 samples/s
+Step 650/37465 | Loss: 4.2150 | LR: 4.33e-04 | 6368.2ms/step | 3.1 samples/s
+Step 660/37465 | Loss: 4.3003 | LR: 4.40e-04 | 6376.6ms/step | 3.1 samples/s
+Step 670/37465 | Loss: 4.4093 | LR: 4.47e-04 | 6375.2ms/step | 3.1 samples/s
+Step 680/37465 | Loss: 4.1509 | LR: 4.53e-04 | 6370.9ms/step | 3.1 samples/s
+Step 690/37465 | Loss: 3.9877 | LR: 4.60e-04 | 6374.5ms/step | 3.1 samples/s
+Step 700/37465 | Loss: 4.1206 | LR: 4.67e-04 | 6372.4ms/step | 3.1 samples/s
+Step 710/37465 | Loss: 4.0514 | LR: 4.73e-04 | 6374.8ms/step | 3.1 samples/s
+Step 720/37465 | Loss: 3.9697 | LR: 4.80e-04 | 6372.0ms/step | 3.1 samples/s
+Step 730/37465 | Loss: 3.5586 | LR: 4.87e-04 | 6372.2ms/step | 3.1 samples/s
+Step 740/37465 | Loss: 4.0557 | LR: 4.93e-04 | 6369.6ms/step | 3.1 samples/s
+Step 750/37465 | Loss: 3.7544 | LR: 5.00e-04 | 6375.6ms/step | 3.1 samples/s
+Step 760/37465 | Loss: 3.6661 | LR: 5.00e-04 | 6374.7ms/step | 3.1 samples/s
+Step 770/37465 | Loss: 3.6655 | LR: 5.00e-04 | 6379.4ms/step | 3.1 samples/s
+Step 780/37465 | Loss: 3.7366 | LR: 5.00e-04 | 6373.3ms/step | 3.1 samples/s
+Step 790/37465 | Loss: 3.3627 | LR: 5.00e-04 | 6377.5ms/step | 3.1 samples/s
+Step 800/37465 | Loss: 3.7698 | LR: 5.00e-04 | 6377.4ms/step | 3.1 samples/s
+Step 810/37465 | Loss: 3.8066 | LR: 5.00e-04 | 6371.7ms/step | 3.1 samples/s
+Step 820/37465 | Loss: 3.6056 | LR: 5.00e-04 | 6376.1ms/step | 3.1 samples/s
+Step 830/37465 | Loss: 3.6018 | LR: 5.00e-04 | 6376.6ms/step | 3.1 samples/s
+Step 840/37465 | Loss: 3.7742 | LR: 5.00e-04 | 6375.5ms/step | 3.1 samples/s
+Step 850/37465 | Loss: 3.4180 | LR: 5.00e-04 | 6378.2ms/step | 3.1 samples/s
+Step 860/37465 | Loss: 3.6353 | LR: 5.00e-04 | 6374.2ms/step | 3.1 samples/s
+Step 870/37465 | Loss: 3.5633 | LR: 5.00e-04 | 6373.2ms/step | 3.1 samples/s
+Step 880/37465 | Loss: 3.4691 | LR: 5.00e-04 | 6374.4ms/step | 3.1 samples/s
+Step 890/37465 | Loss: 3.5110 | LR: 5.00e-04 | 6370.8ms/step | 3.1 samples/s
+Step 900/37465 | Loss: 3.1987 | LR: 5.00e-04 | 6375.8ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-900
+Step 910/37465 | Loss: 3.2169 | LR: 5.00e-04 | 7315.5ms/step | 2.7 samples/s
+Step 920/37465 | Loss: 3.3605 | LR: 5.00e-04 | 6378.5ms/step | 3.1 samples/s
+Step 930/37465 | Loss: 3.2232 | LR: 5.00e-04 | 6380.6ms/step | 3.1 samples/s
+Step 940/37465 | Loss: 3.4859 | LR: 5.00e-04 | 6380.2ms/step | 3.1 samples/s
+Step 950/37465 | Loss: 3.7345 | LR: 5.00e-04 | 6381.3ms/step | 3.1 samples/s
+Step 960/37465 | Loss: 3.0550 | LR: 5.00e-04 | 6386.2ms/step | 3.1 samples/s
+Step 970/37465 | Loss: 3.4433 | LR: 5.00e-04 | 6389.7ms/step | 3.1 samples/s
+Step 980/37465 | Loss: 3.2395 | LR: 5.00e-04 | 6383.5ms/step | 3.1 samples/s
+Step 990/37465 | Loss: 3.2277 | LR: 5.00e-04 | 6383.7ms/step | 3.1 samples/s
+Step 1000/37465 | Loss: 3.0568 | LR: 5.00e-04 | 6387.2ms/step | 3.1 samples/s
+Validation at step 1000: Loss = 3.2705, Perplexity = 26.32
+Step 1010/37465 | Loss: 3.1221 | LR: 5.00e-04 | 7136.6ms/step | 2.8 samples/s
+Step 1020/37465 | Loss: 3.3939 | LR: 5.00e-04 | 6387.9ms/step | 3.1 samples/s
+Step 1030/37465 | Loss: 2.9741 | LR: 5.00e-04 | 6392.5ms/step | 3.1 samples/s
+Step 1040/37465 | Loss: 3.3890 | LR: 5.00e-04 | 6384.8ms/step | 3.1 samples/s
+Step 1050/37465 | Loss: 3.1908 | LR: 5.00e-04 | 6385.4ms/step | 3.1 samples/s
+Step 1060/37465 | Loss: 3.2037 | LR: 5.00e-04 | 6391.9ms/step | 3.1 samples/s
+Step 1070/37465 | Loss: 3.1875 | LR: 5.00e-04 | 6389.5ms/step | 3.1 samples/s
+Step 1080/37465 | Loss: 3.1991 | LR: 5.00e-04 | 6397.2ms/step | 3.1 samples/s
+Step 1090/37465 | Loss: 3.1953 | LR: 5.00e-04 | 6388.9ms/step | 3.1 samples/s
+Step 1100/37465 | Loss: 3.0628 | LR: 5.00e-04 | 6385.3ms/step | 3.1 samples/s
+Step 1110/37465 | Loss: 2.9822 | LR: 5.00e-04 | 6388.1ms/step | 3.1 samples/s
+Step 1120/37465 | Loss: 2.9275 | LR: 5.00e-04 | 6390.9ms/step | 3.1 samples/s
+Step 1130/37465 | Loss: 3.1709 | LR: 5.00e-04 | 6384.2ms/step | 3.1 samples/s
+Step 1140/37465 | Loss: 3.1406 | LR: 5.00e-04 | 6396.0ms/step | 3.1 samples/s
+Step 1150/37465 | Loss: 2.9701 | LR: 5.00e-04 | 6386.2ms/step | 3.1 samples/s
+Step 1160/37465 | Loss: 3.2755 | LR: 5.00e-04 | 6389.9ms/step | 3.1 samples/s
+Step 1170/37465 | Loss: 2.8357 | LR: 5.00e-04 | 6391.5ms/step | 3.1 samples/s
+Step 1180/37465 | Loss: 3.2176 | LR: 5.00e-04 | 6391.0ms/step | 3.1 samples/s
+Step 1190/37465 | Loss: 3.0387 | LR: 5.00e-04 | 6388.0ms/step | 3.1 samples/s
+Step 1200/37465 | Loss: 2.7643 | LR: 5.00e-04 | 6386.4ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-1200
+Step 1210/37465 | Loss: 3.1515 | LR: 5.00e-04 | 7315.0ms/step | 2.7 samples/s
+Step 1220/37465 | Loss: 3.2124 | LR: 5.00e-04 | 6384.4ms/step | 3.1 samples/s
+Step 1230/37465 | Loss: 2.8344 | LR: 5.00e-04 | 6379.6ms/step | 3.1 samples/s
+Step 1240/37465 | Loss: 3.0946 | LR: 5.00e-04 | 6375.6ms/step | 3.1 samples/s
+Step 1250/37465 | Loss: 2.9099 | LR: 5.00e-04 | 6378.2ms/step | 3.1 samples/s
+Step 1260/37465 | Loss: 2.9297 | LR: 5.00e-04 | 6374.3ms/step | 3.1 samples/s
+Step 1270/37465 | Loss: 3.0279 | LR: 5.00e-04 | 6376.6ms/step | 3.1 samples/s
+Step 1280/37465 | Loss: 2.8854 | LR: 5.00e-04 | 6377.8ms/step | 3.1 samples/s
+Step 1290/37465 | Loss: 2.9985 | LR: 5.00e-04 | 6378.2ms/step | 3.1 samples/s
+Step 1300/37465 | Loss: 3.3021 | LR: 5.00e-04 | 6378.0ms/step | 3.1 samples/s
+Step 1310/37465 | Loss: 3.0772 | LR: 5.00e-04 | 6379.3ms/step | 3.1 samples/s
+Step 1320/37465 | Loss: 2.9217 | LR: 5.00e-04 | 6377.9ms/step | 3.1 samples/s
+Step 1330/37465 | Loss: 3.1318 | LR: 5.00e-04 | 6378.1ms/step | 3.1 samples/s
+Step 1340/37465 | Loss: 3.0401 | LR: 5.00e-04 | 6380.0ms/step | 3.1 samples/s
+Step 1350/37465 | Loss: 2.9640 | LR: 5.00e-04 | 6379.0ms/step | 3.1 samples/s
+Step 1360/37465 | Loss: 2.8963 | LR: 5.00e-04 | 6376.5ms/step | 3.1 samples/s
+Step 1370/37465 | Loss: 3.2083 | LR: 5.00e-04 | 6380.2ms/step | 3.1 samples/s
+Step 1380/37465 | Loss: 2.9132 | LR: 5.00e-04 | 6379.1ms/step | 3.1 samples/s
+Step 1390/37465 | Loss: 2.8943 | LR: 5.00e-04 | 6374.9ms/step | 3.1 samples/s
+Step 1400/37465 | Loss: 2.9270 | LR: 5.00e-04 | 6377.8ms/step | 3.1 samples/s
+Step 1410/37465 | Loss: 3.1229 | LR: 5.00e-04 | 6381.8ms/step | 3.1 samples/s
+Step 1420/37465 | Loss: 2.7350 | LR: 5.00e-04 | 6381.4ms/step | 3.1 samples/s
+Step 1430/37465 | Loss: 3.2892 | LR: 5.00e-04 | 6377.8ms/step | 3.1 samples/s
+Step 1440/37465 | Loss: 2.9455 | LR: 5.00e-04 | 6382.3ms/step | 3.1 samples/s
+Step 1450/37465 | Loss: 3.0110 | LR: 5.00e-04 | 6375.8ms/step | 3.1 samples/s
+Step 1460/37465 | Loss: 3.0622 | LR: 5.00e-04 | 6377.9ms/step | 3.1 samples/s
+Step 1470/37465 | Loss: 2.9322 | LR: 5.00e-04 | 6379.8ms/step | 3.1 samples/s
+Step 1480/37465 | Loss: 2.7953 | LR: 5.00e-04 | 6377.0ms/step | 3.1 samples/s
+Step 1490/37465 | Loss: 2.9015 | LR: 4.99e-04 | 6379.9ms/step | 3.1 samples/s
+Step 1500/37465 | Loss: 2.8845 | LR: 4.99e-04 | 6381.4ms/step | 3.1 samples/s
+Validation at step 1500: Loss = 2.9509, Perplexity = 19.12
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-1500
+Step 1510/37465 | Loss: 2.9572 | LR: 4.99e-04 | 8029.5ms/step | 2.5 samples/s
+Step 1520/37465 | Loss: 2.8701 | LR: 4.99e-04 | 6387.5ms/step | 3.1 samples/s
+Step 1530/37465 | Loss: 2.8017 | LR: 4.99e-04 | 6387.6ms/step | 3.1 samples/s
+Step 1540/37465 | Loss: 2.7911 | LR: 4.99e-04 | 6386.7ms/step | 3.1 samples/s
+Step 1550/37465 | Loss: 3.0827 | LR: 4.99e-04 | 6385.9ms/step | 3.1 samples/s
+Step 1560/37465 | Loss: 2.9479 | LR: 4.99e-04 | 6387.4ms/step | 3.1 samples/s
+Step 1570/37465 | Loss: 2.7598 | LR: 4.99e-04 | 6390.2ms/step | 3.1 samples/s
+Step 1580/37465 | Loss: 3.1258 | LR: 4.99e-04 | 6387.3ms/step | 3.1 samples/s
+Step 1590/37465 | Loss: 2.9865 | LR: 4.99e-04 | 6389.2ms/step | 3.1 samples/s
+Step 1600/37465 | Loss: 3.2862 | LR: 4.99e-04 | 6383.2ms/step | 3.1 samples/s
+Step 1610/37465 | Loss: 2.7886 | LR: 4.99e-04 | 6386.6ms/step | 3.1 samples/s
+Step 1620/37465 | Loss: 3.0593 | LR: 4.99e-04 | 6384.5ms/step | 3.1 samples/s
+Step 1630/37465 | Loss: 2.9216 | LR: 4.99e-04 | 6390.2ms/step | 3.1 samples/s
+Step 1640/37465 | Loss: 2.8619 | LR: 4.99e-04 | 6391.2ms/step | 3.1 samples/s
+Step 1650/37465 | Loss: 2.6076 | LR: 4.99e-04 | 6387.9ms/step | 3.1 samples/s
+Step 1660/37465 | Loss: 2.7470 | LR: 4.99e-04 | 6386.2ms/step | 3.1 samples/s
+Step 1670/37465 | Loss: 2.7488 | LR: 4.99e-04 | 6394.0ms/step | 3.1 samples/s
+Step 1680/37465 | Loss: 3.0364 | LR: 4.99e-04 | 6389.6ms/step | 3.1 samples/s
+Step 1690/37465 | Loss: 2.9046 | LR: 4.99e-04 | 6398.3ms/step | 3.1 samples/s
+Step 1700/37465 | Loss: 2.9512 | LR: 4.99e-04 | 6388.5ms/step | 3.1 samples/s
+Step 1710/37465 | Loss: 2.9809 | LR: 4.99e-04 | 6382.4ms/step | 3.1 samples/s
+Step 1720/37465 | Loss: 3.0311 | LR: 4.99e-04 | 6392.7ms/step | 3.1 samples/s
+Step 1730/37465 | Loss: 2.7612 | LR: 4.99e-04 | 6390.9ms/step | 3.1 samples/s
+Step 1740/37465 | Loss: 2.9211 | LR: 4.99e-04 | 6379.4ms/step | 3.1 samples/s
+Step 1750/37465 | Loss: 2.9566 | LR: 4.99e-04 | 6387.7ms/step | 3.1 samples/s
+Step 1760/37465 | Loss: 2.8662 | LR: 4.99e-04 | 6387.3ms/step | 3.1 samples/s
+Step 1770/37465 | Loss: 2.8642 | LR: 4.99e-04 | 6385.7ms/step | 3.1 samples/s
+Step 1780/37465 | Loss: 2.7108 | LR: 4.99e-04 | 6390.8ms/step | 3.1 samples/s
+Step 1790/37465 | Loss: 2.9409 | LR: 4.99e-04 | 6388.7ms/step | 3.1 samples/s
+Step 1800/37465 | Loss: 2.6069 | LR: 4.99e-04 | 6392.8ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-1800
+Step 1810/37465 | Loss: 2.9186 | LR: 4.99e-04 | 7290.9ms/step | 2.7 samples/s
+Step 1820/37465 | Loss: 2.9307 | LR: 4.99e-04 | 6389.4ms/step | 3.1 samples/s
+Step 1830/37465 | Loss: 2.8242 | LR: 4.99e-04 | 6394.3ms/step | 3.1 samples/s
+Step 1840/37465 | Loss: 2.9409 | LR: 4.99e-04 | 6387.8ms/step | 3.1 samples/s
+Step 1850/37465 | Loss: 2.7138 | LR: 4.99e-04 | 6391.3ms/step | 3.1 samples/s
+Step 1860/37465 | Loss: 2.7519 | LR: 4.99e-04 | 6391.2ms/step | 3.1 samples/s
+Step 1870/37465 | Loss: 3.1458 | LR: 4.99e-04 | 6386.0ms/step | 3.1 samples/s
+Step 1880/37465 | Loss: 2.8918 | LR: 4.99e-04 | 6383.1ms/step | 3.1 samples/s
+Step 1890/37465 | Loss: 2.8751 | LR: 4.99e-04 | 6379.4ms/step | 3.1 samples/s
+Step 1900/37465 | Loss: 2.7494 | LR: 4.99e-04 | 6380.6ms/step | 3.1 samples/s
+Step 1910/37465 | Loss: 2.7345 | LR: 4.99e-04 | 6379.9ms/step | 3.1 samples/s
+Step 1920/37465 | Loss: 2.9475 | LR: 4.99e-04 | 6382.4ms/step | 3.1 samples/s
+Step 1930/37465 | Loss: 2.8216 | LR: 4.99e-04 | 6382.3ms/step | 3.1 samples/s
+Step 1940/37465 | Loss: 2.8334 | LR: 4.99e-04 | 6382.6ms/step | 3.1 samples/s
+Step 1950/37465 | Loss: 2.7589 | LR: 4.99e-04 | 6384.2ms/step | 3.1 samples/s
+Step 1960/37465 | Loss: 2.7681 | LR: 4.99e-04 | 6381.6ms/step | 3.1 samples/s
+Step 1970/37465 | Loss: 2.9729 | LR: 4.99e-04 | 6382.0ms/step | 3.1 samples/s
+Step 1980/37465 | Loss: 2.7409 | LR: 4.99e-04 | 6385.0ms/step | 3.1 samples/s
+Step 1990/37465 | Loss: 2.6951 | LR: 4.99e-04 | 6389.5ms/step | 3.1 samples/s
+Step 2000/37465 | Loss: 2.7914 | LR: 4.99e-04 | 6383.1ms/step | 3.1 samples/s
+Validation at step 2000: Loss = 2.7777, Perplexity = 16.08
+Step 2010/37465 | Loss: 2.7050 | LR: 4.99e-04 | 7128.8ms/step | 2.8 samples/s
+Step 2020/37465 | Loss: 2.7686 | LR: 4.99e-04 | 6384.7ms/step | 3.1 samples/s
+Step 2030/37465 | Loss: 2.6295 | LR: 4.99e-04 | 6385.0ms/step | 3.1 samples/s
+Step 2040/37465 | Loss: 2.6745 | LR: 4.98e-04 | 6379.5ms/step | 3.1 samples/s
+Step 2050/37465 | Loss: 2.7631 | LR: 4.98e-04 | 6382.7ms/step | 3.1 samples/s
+Step 2060/37465 | Loss: 2.6218 | LR: 4.98e-04 | 6384.9ms/step | 3.1 samples/s
+Step 2070/37465 | Loss: 2.6614 | LR: 4.98e-04 | 6386.5ms/step | 3.1 samples/s
+Step 2080/37465 | Loss: 2.8349 | LR: 4.98e-04 | 6382.5ms/step | 3.1 samples/s
+Step 2090/37465 | Loss: 2.8247 | LR: 4.98e-04 | 6382.2ms/step | 3.1 samples/s
+Step 2100/37465 | Loss: 2.9305 | LR: 4.98e-04 | 6379.0ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-2100
+Step 2110/37465 | Loss: 2.8710 | LR: 4.98e-04 | 7287.2ms/step | 2.7 samples/s
+Step 2120/37465 | Loss: 2.7928 | LR: 4.98e-04 | 6384.3ms/step | 3.1 samples/s
+Step 2130/37465 | Loss: 2.5135 | LR: 4.98e-04 | 6387.6ms/step | 3.1 samples/s
+Step 2140/37465 | Loss: 2.8393 | LR: 4.98e-04 | 6384.1ms/step | 3.1 samples/s
+Step 2150/37465 | Loss: 2.7712 | LR: 4.98e-04 | 6387.7ms/step | 3.1 samples/s
+Step 2160/37465 | Loss: 2.6872 | LR: 4.98e-04 | 6382.7ms/step | 3.1 samples/s
+Step 2170/37465 | Loss: 2.7883 | LR: 4.98e-04 | 6379.3ms/step | 3.1 samples/s
+Step 2180/37465 | Loss: 2.8064 | LR: 4.98e-04 | 6374.1ms/step | 3.1 samples/s
+Step 2190/37465 | Loss: 2.7624 | LR: 4.98e-04 | 6382.0ms/step | 3.1 samples/s
+Step 2200/37465 | Loss: 2.6835 | LR: 4.98e-04 | 6383.3ms/step | 3.1 samples/s
+Step 2210/37465 | Loss: 2.6176 | LR: 4.98e-04 | 6382.6ms/step | 3.1 samples/s
+Step 2220/37465 | Loss: 2.8132 | LR: 4.98e-04 | 6382.2ms/step | 3.1 samples/s
+Step 2230/37465 | Loss: 2.8084 | LR: 4.98e-04 | 6378.5ms/step | 3.1 samples/s
+Step 2240/37465 | Loss: 2.8041 | LR: 4.98e-04 | 6377.2ms/step | 3.1 samples/s
+Step 2250/37465 | Loss: 2.6921 | LR: 4.98e-04 | 6380.4ms/step | 3.1 samples/s
+Step 2260/37465 | Loss: 2.8456 | LR: 4.98e-04 | 6376.4ms/step | 3.1 samples/s
+Step 2270/37465 | Loss: 2.7451 | LR: 4.98e-04 | 6379.1ms/step | 3.1 samples/s
+Step 2280/37465 | Loss: 2.5784 | LR: 4.98e-04 | 6382.5ms/step | 3.1 samples/s
+Step 2290/37465 | Loss: 2.7031 | LR: 4.98e-04 | 6381.7ms/step | 3.1 samples/s
+Step 2300/37465 | Loss: 2.8041 | LR: 4.98e-04 | 6377.0ms/step | 3.1 samples/s
+Step 2310/37465 | Loss: 2.8593 | LR: 4.98e-04 | 6379.1ms/step | 3.1 samples/s
+Step 2320/37465 | Loss: 2.7207 | LR: 4.98e-04 | 6382.5ms/step | 3.1 samples/s
+Step 2330/37465 | Loss: 2.9899 | LR: 4.98e-04 | 6372.6ms/step | 3.1 samples/s
+Step 2340/37465 | Loss: 2.5633 | LR: 4.98e-04 | 6377.4ms/step | 3.1 samples/s
+Step 2350/37465 | Loss: 2.9726 | LR: 4.98e-04 | 6379.6ms/step | 3.1 samples/s
+Step 2360/37465 | Loss: 2.5830 | LR: 4.98e-04 | 6378.0ms/step | 3.1 samples/s
+Step 2370/37465 | Loss: 2.8398 | LR: 4.98e-04 | 6378.2ms/step | 3.1 samples/s
+Step 2380/37465 | Loss: 2.6590 | LR: 4.98e-04 | 6377.3ms/step | 3.1 samples/s
+Step 2390/37465 | Loss: 2.5337 | LR: 4.98e-04 | 6380.3ms/step | 3.1 samples/s
+Step 2400/37465 | Loss: 2.5193 | LR: 4.98e-04 | 6379.5ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-2400
+Step 2410/37465 | Loss: 2.6855 | LR: 4.97e-04 | 7312.0ms/step | 2.7 samples/s
+Step 2420/37465 | Loss: 2.8776 | LR: 4.97e-04 | 6381.3ms/step | 3.1 samples/s
+Step 2430/37465 | Loss: 2.8649 | LR: 4.97e-04 | 6384.6ms/step | 3.1 samples/s
+Step 2440/37465 | Loss: 2.8029 | LR: 4.97e-04 | 6385.3ms/step | 3.1 samples/s
+Step 2450/37465 | Loss: 2.7996 | LR: 4.97e-04 | 6390.0ms/step | 3.1 samples/s
+Step 2460/37465 | Loss: 2.7896 | LR: 4.97e-04 | 6391.3ms/step | 3.1 samples/s
+Step 2470/37465 | Loss: 2.8103 | LR: 4.97e-04 | 6381.5ms/step | 3.1 samples/s
+Step 2480/37465 | Loss: 2.6114 | LR: 4.97e-04 | 6383.0ms/step | 3.1 samples/s
+Step 2490/37465 | Loss: 2.6087 | LR: 4.97e-04 | 6371.4ms/step | 3.1 samples/s
+Step 2500/37465 | Loss: 2.7416 | LR: 4.97e-04 | 6382.6ms/step | 3.1 samples/s
+Validation at step 2500: Loss = 2.6779, Perplexity = 14.55
+Step 2510/37465 | Loss: 2.8022 | LR: 4.97e-04 | 7132.4ms/step | 2.8 samples/s
+Step 2520/37465 | Loss: 2.4969 | LR: 4.97e-04 | 6380.0ms/step | 3.1 samples/s
+Step 2530/37465 | Loss: 2.6487 | LR: 4.97e-04 | 6378.3ms/step | 3.1 samples/s
+Step 2540/37465 | Loss: 2.9364 | LR: 4.97e-04 | 6381.9ms/step | 3.1 samples/s
+Step 2550/37465 | Loss: 2.4333 | LR: 4.97e-04 | 6374.7ms/step | 3.1 samples/s
+Step 2560/37465 | Loss: 2.7193 | LR: 4.97e-04 | 6375.6ms/step | 3.1 samples/s
+Step 2570/37465 | Loss: 2.6816 | LR: 4.97e-04 | 6378.3ms/step | 3.1 samples/s
+Step 2580/37465 | Loss: 2.4674 | LR: 4.97e-04 | 6378.2ms/step | 3.1 samples/s
+Step 2590/37465 | Loss: 2.7053 | LR: 4.97e-04 | 6378.6ms/step | 3.1 samples/s
+Step 2600/37465 | Loss: 2.8359 | LR: 4.97e-04 | 6378.8ms/step | 3.1 samples/s
+Step 2610/37465 | Loss: 2.5548 | LR: 4.97e-04 | 6379.4ms/step | 3.1 samples/s
+Step 2620/37465 | Loss: 2.8504 | LR: 4.97e-04 | 6382.6ms/step | 3.1 samples/s
+Step 2630/37465 | Loss: 2.6607 | LR: 4.97e-04 | 6385.9ms/step | 3.1 samples/s
+Step 2640/37465 | Loss: 2.5939 | LR: 4.97e-04 | 6377.5ms/step | 3.1 samples/s
+Step 2650/37465 | Loss: 2.5936 | LR: 4.97e-04 | 6378.4ms/step | 3.1 samples/s
+Step 2660/37465 | Loss: 2.4835 | LR: 4.97e-04 | 6379.9ms/step | 3.1 samples/s
+Step 2670/37465 | Loss: 2.5658 | LR: 4.97e-04 | 6374.7ms/step | 3.1 samples/s
+Step 2680/37465 | Loss: 2.8609 | LR: 4.97e-04 | 6379.0ms/step | 3.1 samples/s
+Step 2690/37465 | Loss: 2.5883 | LR: 4.97e-04 | 6382.0ms/step | 3.1 samples/s
+Step 2700/37465 | Loss: 2.7509 | LR: 4.97e-04 | 6381.7ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-2700
+Step 2710/37465 | Loss: 2.5879 | LR: 4.96e-04 | 7293.1ms/step | 2.7 samples/s
+Step 2720/37465 | Loss: 2.5680 | LR: 4.96e-04 | 6377.8ms/step | 3.1 samples/s
+Step 2730/37465 | Loss: 2.6055 | LR: 4.96e-04 | 6377.1ms/step | 3.1 samples/s
+Step 2740/37465 | Loss: 2.6871 | LR: 4.96e-04 | 6382.4ms/step | 3.1 samples/s
+Step 2750/37465 | Loss: 2.5974 | LR: 4.96e-04 | 6387.7ms/step | 3.1 samples/s
+Step 2760/37465 | Loss: 2.8237 | LR: 4.96e-04 | 6378.7ms/step | 3.1 samples/s
+Step 2770/37465 | Loss: 2.6716 | LR: 4.96e-04 | 6382.1ms/step | 3.1 samples/s
+Step 2780/37465 | Loss: 2.7498 | LR: 4.96e-04 | 6380.9ms/step | 3.1 samples/s
+Step 2790/37465 | Loss: 2.4863 | LR: 4.96e-04 | 6380.2ms/step | 3.1 samples/s
+Step 2800/37465 | Loss: 2.5453 | LR: 4.96e-04 | 6383.2ms/step | 3.1 samples/s
+Step 2810/37465 | Loss: 2.5213 | LR: 4.96e-04 | 6386.4ms/step | 3.1 samples/s
+Step 2820/37465 | Loss: 2.3568 | LR: 4.96e-04 | 6391.6ms/step | 3.1 samples/s
+Step 2830/37465 | Loss: 2.4480 | LR: 4.96e-04 | 6387.9ms/step | 3.1 samples/s
+Step 2840/37465 | Loss: 2.8941 | LR: 4.96e-04 | 6383.9ms/step | 3.1 samples/s
+Step 2850/37465 | Loss: 2.5534 | LR: 4.96e-04 | 6384.4ms/step | 3.1 samples/s
+Step 2860/37465 | Loss: 2.5181 | LR: 4.96e-04 | 6387.4ms/step | 3.1 samples/s
+Step 2870/37465 | Loss: 2.7222 | LR: 4.96e-04 | 6384.2ms/step | 3.1 samples/s
+Step 2880/37465 | Loss: 2.6761 | LR: 4.96e-04 | 6383.9ms/step | 3.1 samples/s
+Step 2890/37465 | Loss: 2.6068 | LR: 4.96e-04 | 6388.8ms/step | 3.1 samples/s
+Step 2900/37465 | Loss: 2.5596 | LR: 4.96e-04 | 6387.7ms/step | 3.1 samples/s
+Step 2910/37465 | Loss: 2.6637 | LR: 4.96e-04 | 6385.2ms/step | 3.1 samples/s
+Step 2920/37465 | Loss: 2.4493 | LR: 4.96e-04 | 6381.5ms/step | 3.1 samples/s
+Step 2930/37465 | Loss: 2.5966 | LR: 4.96e-04 | 6384.4ms/step | 3.1 samples/s
+Step 2940/37465 | Loss: 2.5092 | LR: 4.96e-04 | 6381.1ms/step | 3.1 samples/s
+Step 2950/37465 | Loss: 2.5356 | LR: 4.96e-04 | 6383.8ms/step | 3.1 samples/s
+Step 2960/37465 | Loss: 2.5739 | LR: 4.96e-04 | 6386.1ms/step | 3.1 samples/s
+Step 2970/37465 | Loss: 2.6963 | LR: 4.96e-04 | 6387.8ms/step | 3.1 samples/s
+Step 2980/37465 | Loss: 2.3348 | LR: 4.95e-04 | 6385.2ms/step | 3.1 samples/s
+Step 2990/37465 | Loss: 2.7376 | LR: 4.95e-04 | 6388.7ms/step | 3.1 samples/s
+Step 3000/37465 | Loss: 2.6397 | LR: 4.95e-04 | 6387.3ms/step | 3.1 samples/s
+Validation at step 3000: Loss = 2.6081, Perplexity = 13.57
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3000
+Step 3010/37465 | Loss: 2.6149 | LR: 4.95e-04 | 8062.9ms/step | 2.5 samples/s
+Step 3020/37465 | Loss: 2.6029 | LR: 4.95e-04 | 6386.1ms/step | 3.1 samples/s
+Step 3030/37465 | Loss: 2.8015 | LR: 4.95e-04 | 6384.6ms/step | 3.1 samples/s
+Step 3040/37465 | Loss: 2.7272 | LR: 4.95e-04 | 6380.3ms/step | 3.1 samples/s
+Step 3050/37465 | Loss: 2.6205 | LR: 4.95e-04 | 6380.2ms/step | 3.1 samples/s
+Step 3060/37465 | Loss: 2.6117 | LR: 4.95e-04 | 6386.5ms/step | 3.1 samples/s
+Step 3070/37465 | Loss: 2.4390 | LR: 4.95e-04 | 6384.2ms/step | 3.1 samples/s
+Step 3080/37465 | Loss: 2.3941 | LR: 4.95e-04 | 6385.5ms/step | 3.1 samples/s
+Step 3090/37465 | Loss: 2.5643 | LR: 4.95e-04 | 6385.4ms/step | 3.1 samples/s
+Step 3100/37465 | Loss: 2.6672 | LR: 4.95e-04 | 6382.4ms/step | 3.1 samples/s
+Step 3110/37465 | Loss: 2.6058 | LR: 4.95e-04 | 6380.2ms/step | 3.1 samples/s
+Step 3120/37465 | Loss: 2.4922 | LR: 4.95e-04 | 6381.2ms/step | 3.1 samples/s
+Step 3130/37465 | Loss: 2.5960 | LR: 4.95e-04 | 6383.0ms/step | 3.1 samples/s
+Step 3140/37465 | Loss: 2.4059 | LR: 4.95e-04 | 6378.9ms/step | 3.1 samples/s
+Step 3150/37465 | Loss: 2.3778 | LR: 4.95e-04 | 6382.1ms/step | 3.1 samples/s
+Step 3160/37465 | Loss: 2.5527 | LR: 4.95e-04 | 6374.2ms/step | 3.1 samples/s
+Step 3170/37465 | Loss: 2.8536 | LR: 4.95e-04 | 6381.6ms/step | 3.1 samples/s
+Step 3180/37465 | Loss: 2.7932 | LR: 4.95e-04 | 6378.5ms/step | 3.1 samples/s
+Step 3190/37465 | Loss: 2.2729 | LR: 4.95e-04 | 6379.2ms/step | 3.1 samples/s
+Step 3200/37465 | Loss: 2.4523 | LR: 4.95e-04 | 6376.7ms/step | 3.1 samples/s
+Step 3210/37465 | Loss: 2.3870 | LR: 4.94e-04 | 6389.6ms/step | 3.1 samples/s
+Step 3220/37465 | Loss: 2.6399 | LR: 4.94e-04 | 6385.6ms/step | 3.1 samples/s
+Step 3230/37465 | Loss: 2.6069 | LR: 4.94e-04 | 6382.8ms/step | 3.1 samples/s
+Step 3240/37465 | Loss: 2.4823 | LR: 4.94e-04 | 6378.3ms/step | 3.1 samples/s
+Step 3250/37465 | Loss: 2.4919 | LR: 4.94e-04 | 6378.8ms/step | 3.1 samples/s
+Step 3260/37465 | Loss: 2.6710 | LR: 4.94e-04 | 6378.5ms/step | 3.1 samples/s
+Step 3270/37465 | Loss: 2.6290 | LR: 4.94e-04 | 6377.1ms/step | 3.1 samples/s
+Step 3280/37465 | Loss: 2.5979 | LR: 4.94e-04 | 6382.3ms/step | 3.1 samples/s
+Step 3290/37465 | Loss: 2.4953 | LR: 4.94e-04 | 6383.3ms/step | 3.1 samples/s
+Step 3300/37465 | Loss: 2.4728 | LR: 4.94e-04 | 6378.9ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300
+Step 3310/37465 | Loss: 2.6605 | LR: 4.94e-04 | 7302.7ms/step | 2.7 samples/s
+Step 3320/37465 | Loss: 2.3535 | LR: 4.94e-04 | 6384.4ms/step | 3.1 samples/s
+Step 3330/37465 | Loss: 2.6802 | LR: 4.94e-04 | 6381.3ms/step | 3.1 samples/s
+Step 3340/37465 | Loss: 2.6431 | LR: 4.94e-04 | 6383.2ms/step | 3.1 samples/s
+Step 3350/37465 | Loss: 2.9169 | LR: 4.94e-04 | 6392.3ms/step | 3.1 samples/s
+Step 3360/37465 | Loss: 2.5130 | LR: 4.94e-04 | 6380.6ms/step | 3.1 samples/s
+Step 3370/37465 | Loss: 2.5432 | LR: 4.94e-04 | 6380.1ms/step | 3.1 samples/s
+Step 3380/37465 | Loss: 2.5796 | LR: 4.94e-04 | 6382.4ms/step | 3.1 samples/s
+Step 3390/37465 | Loss: 2.4650 | LR: 4.94e-04 | 6383.2ms/step | 3.1 samples/s
+Step 3400/37465 | Loss: 2.3815 | LR: 4.94e-04 | 6386.5ms/step | 3.1 samples/s
+Step 3410/37465 | Loss: 2.4313 | LR: 4.94e-04 | 6382.9ms/step | 3.1 samples/s
+Step 3420/37465 | Loss: 2.5677 | LR: 4.94e-04 | 6383.0ms/step | 3.1 samples/s
+Step 3430/37465 | Loss: 2.7321 | LR: 4.93e-04 | 6379.4ms/step | 3.1 samples/s
+Step 3440/37465 | Loss: 2.6157 | LR: 4.93e-04 | 6381.4ms/step | 3.1 samples/s

epfl_final/logs/l30k1_2b_dense.log.rescheduled1

@@ -0,0 +1,219 @@
+W0708 05:43:00.370000 206 site-packages/torch/distributed/run.py:851] 
+W0708 05:43:00.370000 206 site-packages/torch/distributed/run.py:851] *****************************************
+W0708 05:43:00.370000 206 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0708 05:43:00.370000 206 site-packages/torch/distributed/run.py:851] *****************************************
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_054311
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized",
+  "val_path": null,
+  "max_seq_length": 30720,
+  "num_workers": 4,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_054311",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 5,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 500,
+  "eval_batches": 20,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b dense 30k longctx",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "dense",
+    "30k",
+    "longctx",
+    "b128",
+    "scaleladder"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 120,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: TrueCheckpoint activations: True
+
+Checkpoint activations: True
+Checkpoint activations: True
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 120
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 30720
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 30720]                --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 30720, 1536]          --                        True
+│    └─Embedding: 2-1                                        [1, 30720]                [1, 30720, 1536]          233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 30720, 1536]          [1, 30720, 1536]          1,536                     True
+├─Linear: 1-2                                                [1, 30720, 1536]          [1, 30720, 151936]        233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.25
+Forward/backward pass size (MB): 98319.73
+Params size (MB): 2871.45
+Estimated Total Size (MB): 101191.43
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Dataset split: 749299 train, 39437 validation
+Training examples (packed, train split):  749299
+Validation examples (packed, val split):  39437
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              5
+Effective batch size (seqs/step):         20
+Sequence length:                          30720
+Tokens per optimizer step:                614,400
+Micro-batches per epoch (per rank):       187325
+Optimizer steps per epoch:                37465
+Total optimizer steps:                    37465
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_054405-43sa4ouk
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 1.2b dense 30k longctx
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/43sa4ouk
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/37465 | Loss: 12.2534 | LR: 6.67e-06 | 16167.7ms/step | 1.2 samples/s
+Step 20/37465 | Loss: 12.1201 | LR: 1.33e-05 | 6304.5ms/step | 3.2 samples/s
+Step 30/37465 | Loss: 11.6858 | LR: 2.00e-05 | 6309.1ms/step | 3.2 samples/s
+Step 40/37465 | Loss: 10.6684 | LR: 2.67e-05 | 6298.4ms/step | 3.2 samples/s
+Step 50/37465 | Loss: 10.0551 | LR: 3.33e-05 | 6286.7ms/step | 3.2 samples/s
+Step 60/37465 | Loss: 9.5722 | LR: 4.00e-05 | 6268.7ms/step | 3.2 samples/s
+Step 70/37465 | Loss: 9.1242 | LR: 4.67e-05 | 6271.1ms/step | 3.2 samples/s
+Step 80/37465 | Loss: 8.6456 | LR: 5.33e-05 | 6278.9ms/step | 3.2 samples/s
+Step 90/37465 | Loss: 8.0658 | LR: 6.00e-05 | 6281.1ms/step | 3.2 samples/s
+Step 100/37465 | Loss: 7.7196 | LR: 6.67e-05 | 6269.8ms/step | 3.2 samples/s
+Step 110/37465 | Loss: 7.3699 | LR: 7.33e-05 | 6276.4ms/step | 3.2 samples/s
+Step 120/37465 | Loss: 7.1081 | LR: 8.00e-05 | 6290.5ms/step | 3.2 samples/s
+Step 130/37465 | Loss: 6.9196 | LR: 8.67e-05 | 6302.0ms/step | 3.2 samples/s
+Step 140/37465 | Loss: 6.8029 | LR: 9.33e-05 | 6285.6ms/step | 3.2 samples/s
+Step 150/37465 | Loss: 6.8077 | LR: 1.00e-04 | 6286.5ms/step | 3.2 samples/s
+Step 160/37465 | Loss: 6.6545 | LR: 1.07e-04 | 6303.8ms/step | 3.2 samples/s
+Step 170/37465 | Loss: 6.3287 | LR: 1.13e-04 | 6301.6ms/step | 3.2 samples/s
+Step 180/37465 | Loss: 6.1895 | LR: 1.20e-04 | 6295.4ms/step | 3.2 samples/s
+Step 190/37465 | Loss: 6.2694 | LR: 1.27e-04 | 6300.7ms/step | 3.2 samples/s
+Step 200/37465 | Loss: 6.0386 | LR: 1.33e-04 | 6304.3ms/step | 3.2 samples/s
+Step 210/37465 | Loss: 6.0358 | LR: 1.40e-04 | 6303.8ms/step | 3.2 samples/s
+Step 220/37465 | Loss: 5.9696 | LR: 1.47e-04 | 6309.1ms/step | 3.2 samples/s
+Step 230/37465 | Loss: 5.9117 | LR: 1.53e-04 | 6327.0ms/step | 3.2 samples/s
+Step 240/37465 | Loss: 5.8905 | LR: 1.60e-04 | 6389.6ms/step | 3.1 samples/s
+Step 250/37465 | Loss: 5.8312 | LR: 1.67e-04 | 6367.4ms/step | 3.1 samples/s
+Step 260/37465 | Loss: 5.6800 | LR: 1.73e-04 | 6363.2ms/step | 3.1 samples/s
+Step 270/37465 | Loss: 5.7001 | LR: 1.80e-04 | 6299.8ms/step | 3.2 samples/s
+Step 280/37465 | Loss: 5.7136 | LR: 1.87e-04 | 6306.0ms/step | 3.2 samples/s
+Step 290/37465 | Loss: 5.4759 | LR: 1.93e-04 | 6300.9ms/step | 3.2 samples/s
+Step 300/37465 | Loss: 5.3876 | LR: 2.00e-04 | 6309.8ms/step | 3.2 samples/s
+Step 310/37465 | Loss: 5.2540 | LR: 2.07e-04 | 6324.4ms/step | 3.2 samples/s
+Step 320/37465 | Loss: 5.3460 | LR: 2.13e-04 | 6306.3ms/step | 3.2 samples/s
+Step 330/37465 | Loss: 5.5442 | LR: 2.20e-04 | 6306.8ms/step | 3.2 samples/s
+Step 340/37465 | Loss: 5.3776 | LR: 2.27e-04 | 6312.8ms/step | 3.2 samples/s
+Step 350/37465 | Loss: 5.2115 | LR: 2.33e-04 | 6318.3ms/step | 3.2 samples/s

epfl_final/logs/l30k1_2b_dense.log.savestep_restart

@@ -0,0 +1,218 @@
+W0708 06:32:22.733000 139 site-packages/torch/distributed/run.py:851] 
+W0708 06:32:22.733000 139 site-packages/torch/distributed/run.py:851] *****************************************
+W0708 06:32:22.733000 139 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0708 06:32:22.733000 139 site-packages/torch/distributed/run.py:851] *****************************************
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_063234
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized",
+  "val_path": null,
+  "max_seq_length": 30720,
+  "num_workers": 4,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_063234",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 5,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 500,
+  "eval_batches": 20,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b dense 30k longctx",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "dense",
+    "30k",
+    "longctx",
+    "b128",
+    "scaleladder"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 120,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: TrueCheckpoint activations: True
+
+Initializing model from scratch (random weights)...
+Checkpoint activations: True
+Checkpoint activations: True
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 120
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 30720
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 30720]                --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 30720, 1536]          --                        True
+│    └─Embedding: 2-1                                        [1, 30720]                [1, 30720, 1536]          233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 30720, 1536]          [1, 30720, 1536]          34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 30720, 1536]          [1, 30720, 1536]          1,536                     True
+├─Linear: 1-2                                                [1, 30720, 1536]          [1, 30720, 151936]        233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.25
+Forward/backward pass size (MB): 98319.73
+Params size (MB): 2871.45
+Estimated Total Size (MB): 101191.43
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+
+
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Dataset split: 749299 train, 39437 validation
+Training examples (packed, train split):  749299
+Validation examples (packed, val split):  39437
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              5
+Effective batch size (seqs/step):         20
+Sequence length:                          30720
+Tokens per optimizer step:                614,400
+Micro-batches per epoch (per rank):       187325
+Optimizer steps per epoch:                37465
+Total optimizer steps:                    37465
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run i61v7k8r
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_063331-i61v7k8r
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 1.2b dense 30k longctx
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/i61v7k8r
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+W0708 06:35:46.667000 139 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 139 got signal: 15

epfl_final/logs/l30k1_2b_resampled.log

@@ -0,0 +1,374 @@
+W0709 00:04:52.738000 161 site-packages/torch/distributed/run.py:851] 
+W0709 00:04:52.738000 161 site-packages/torch/distributed/run.py:851] *****************************************
+W0709 00:04:52.738000 161 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0709 00:04:52.738000 161 site-packages/torch/distributed/run.py:851] *****************************************
+Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized",
+  "val_path": null,
+  "max_seq_length": 30720,
+  "num_workers": 4,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 5,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 300,
+  "eval_steps": 500,
+  "eval_batches": 20,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b resampled 30k longctx",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "resampled",
+    "30k",
+    "longctx",
+    "b128",
+    "scaleladder"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 120,
+  "sparse_mask_pool_size": 1000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120",
+  "sparse_mask_seed": 0,
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: True
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 120
+  sparse_mask_pool_size: 1000
+  sparse_max_seq_length: 30720
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+Could not generate detailed summary: Failed to run torchinfo. See above stack traces for more details. Executed layers up to: [Embedding: 2, Qwen3RMSNorm: 4, Linear: 5, Linear: 5, Linear: 5, Qwen3RMSNorm: 5, Qwen3RMSNorm: 5, Qwen3RotaryEmbedding: 5]
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Dataset split: 749299 train, 39437 validation
+Training examples (packed, train split):  749299
+Validation examples (packed, val split):  39437
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              5
+Effective batch size (seqs/step):         20
+Sequence length:                          30720
+Tokens per optimizer step:                614,400
+Micro-batches per epoch (per rank):       187325
+Optimizer steps per epoch:                37465
+Total optimizer steps:                    37465
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200...
+Overriding checkpoint LR with config LR: 0.0005
+Resumed from checkpoint at step 4200 (epoch 0, micro_step 21000)
+Resuming training from step 4200
+Resuming wandb run hn868f80
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run hn868f80
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260709_000736-hn868f80
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b resampled 30k longctx
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/hn868f80
+Starting training...
+Validation at step 4200: Loss = 2.5215, Perplexity = 12.45
+wandb: WARNING Tried to log to step 4200 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 4201: sparse_stochastic (configured sparse attention)
+Step 4210/37465 | Loss: 2.1631 | LR: 4.89e-04 | 25283.0ms/step | 0.8 samples/s
+wandb: WARNING Tried to log to step 4210 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4220/37465 | Loss: 2.5147 | LR: 4.89e-04 | 8085.2ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4220 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4230/37465 | Loss: 2.4117 | LR: 4.89e-04 | 8214.6ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 4230 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4240/37465 | Loss: 2.4558 | LR: 4.89e-04 | 8096.8ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4240 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4250/37465 | Loss: 2.3723 | LR: 4.89e-04 | 8096.8ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4250 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4260/37465 | Loss: 2.4316 | LR: 4.89e-04 | 8094.9ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4260 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4270/37465 | Loss: 2.5079 | LR: 4.89e-04 | 8092.9ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4270 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4280/37465 | Loss: 2.4872 | LR: 4.89e-04 | 8092.7ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4280 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4290/37465 | Loss: 2.5664 | LR: 4.89e-04 | 8095.6ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4290 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4300/37465 | Loss: 2.4478 | LR: 4.89e-04 | 8098.4ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4300 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4310/37465 | Loss: 2.6316 | LR: 4.88e-04 | 8092.3ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4310 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4320/37465 | Loss: 2.3660 | LR: 4.88e-04 | 8094.7ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4320 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4330/37465 | Loss: 2.2324 | LR: 4.88e-04 | 8092.0ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4330 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4340/37465 | Loss: 2.5369 | LR: 4.88e-04 | 8090.7ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4340 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4350/37465 | Loss: 2.6448 | LR: 4.88e-04 | 8231.6ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 4350 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4360/37465 | Loss: 2.6113 | LR: 4.88e-04 | 8074.1ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4360 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4370/37465 | Loss: 2.4337 | LR: 4.88e-04 | 8078.8ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4370 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4380/37465 | Loss: 2.4301 | LR: 4.88e-04 | 8086.8ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4380 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4390/37465 | Loss: 2.5853 | LR: 4.88e-04 | 8088.1ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4390 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4400/37465 | Loss: 2.3710 | LR: 4.88e-04 | 8088.4ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4400 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4410/37465 | Loss: 2.6066 | LR: 4.88e-04 | 8090.8ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4410 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4420/37465 | Loss: 2.3306 | LR: 4.88e-04 | 8089.6ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 4420 that is less than the current step 4421. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 4430/37465 | Loss: 2.5145 | LR: 4.88e-04 | 8089.5ms/step | 2.5 samples/s
+Step 4440/37465 | Loss: 2.3588 | LR: 4.88e-04 | 8090.0ms/step | 2.5 samples/s
+Step 4450/37465 | Loss: 2.4488 | LR: 4.88e-04 | 8074.7ms/step | 2.5 samples/s
+Step 4460/37465 | Loss: 2.3425 | LR: 4.88e-04 | 8086.2ms/step | 2.5 samples/s
+Step 4470/37465 | Loss: 2.3559 | LR: 4.87e-04 | 8088.1ms/step | 2.5 samples/s
+Step 4480/37465 | Loss: 2.3664 | LR: 4.87e-04 | 8088.8ms/step | 2.5 samples/s
+Step 4490/37465 | Loss: 2.4456 | LR: 4.87e-04 | 8088.2ms/step | 2.5 samples/s
+Step 4500/37465 | Loss: 2.6641 | LR: 4.87e-04 | 8087.2ms/step | 2.5 samples/s
+Validation at step 4500: Loss = 2.5009, Perplexity = 12.19
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4500
+Step 4510/37465 | Loss: 2.3789 | LR: 4.87e-04 | 9685.6ms/step | 2.1 samples/s
+Step 4520/37465 | Loss: 2.4548 | LR: 4.87e-04 | 8075.6ms/step | 2.5 samples/s
+Step 4530/37465 | Loss: 2.5635 | LR: 4.87e-04 | 8087.1ms/step | 2.5 samples/s
+Step 4540/37465 | Loss: 2.4873 | LR: 4.87e-04 | 8087.0ms/step | 2.5 samples/s
+Step 4550/37465 | Loss: 2.4339 | LR: 4.87e-04 | 8097.0ms/step | 2.5 samples/s
+Step 4560/37465 | Loss: 2.4475 | LR: 4.87e-04 | 8085.9ms/step | 2.5 samples/s
+Step 4570/37465 | Loss: 2.2232 | LR: 4.87e-04 | 8090.2ms/step | 2.5 samples/s
+Step 4580/37465 | Loss: 2.5504 | LR: 4.87e-04 | 8091.0ms/step | 2.5 samples/s
+Step 4590/37465 | Loss: 2.4426 | LR: 4.87e-04 | 8087.9ms/step | 2.5 samples/s
+Step 4600/37465 | Loss: 2.6598 | LR: 4.87e-04 | 8084.2ms/step | 2.5 samples/s
+Step 4610/37465 | Loss: 2.4243 | LR: 4.86e-04 | 8088.4ms/step | 2.5 samples/s
+Step 4620/37465 | Loss: 2.5928 | LR: 4.86e-04 | 8089.5ms/step | 2.5 samples/s
+Step 4630/37465 | Loss: 2.5539 | LR: 4.86e-04 | 8079.4ms/step | 2.5 samples/s
+Step 4640/37465 | Loss: 2.3340 | LR: 4.86e-04 | 8086.4ms/step | 2.5 samples/s
+Step 4650/37465 | Loss: 2.2886 | LR: 4.86e-04 | 8082.5ms/step | 2.5 samples/s
+Step 4660/37465 | Loss: 2.4744 | LR: 4.86e-04 | 8080.6ms/step | 2.5 samples/s
+Step 4670/37465 | Loss: 2.5130 | LR: 4.86e-04 | 8085.7ms/step | 2.5 samples/s
+Step 4680/37465 | Loss: 2.4986 | LR: 4.86e-04 | 8082.6ms/step | 2.5 samples/s
+Step 4690/37465 | Loss: 2.6264 | LR: 4.86e-04 | 8083.7ms/step | 2.5 samples/s
+Step 4700/37465 | Loss: 2.5099 | LR: 4.86e-04 | 8084.9ms/step | 2.5 samples/s
+Step 4710/37465 | Loss: 2.3189 | LR: 4.86e-04 | 8083.4ms/step | 2.5 samples/s
+Step 4720/37465 | Loss: 2.5777 | LR: 4.86e-04 | 8083.3ms/step | 2.5 samples/s
+Step 4730/37465 | Loss: 2.3423 | LR: 4.86e-04 | 8081.6ms/step | 2.5 samples/s
+Step 4740/37465 | Loss: 2.5076 | LR: 4.86e-04 | 8083.0ms/step | 2.5 samples/s
+Step 4750/37465 | Loss: 2.4668 | LR: 4.85e-04 | 8084.6ms/step | 2.5 samples/s
+Step 4760/37465 | Loss: 2.4106 | LR: 4.85e-04 | 8079.4ms/step | 2.5 samples/s
+Step 4770/37465 | Loss: 2.3726 | LR: 4.85e-04 | 8082.2ms/step | 2.5 samples/s
+Step 4780/37465 | Loss: 2.5252 | LR: 4.85e-04 | 8103.5ms/step | 2.5 samples/s
+Step 4790/37465 | Loss: 2.6367 | LR: 4.85e-04 | 8083.1ms/step | 2.5 samples/s
+Step 4800/37465 | Loss: 2.5199 | LR: 4.85e-04 | 8084.3ms/step | 2.5 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4800
+Step 4810/37465 | Loss: 2.5485 | LR: 4.85e-04 | 8989.9ms/step | 2.2 samples/s
+Step 4820/37465 | Loss: 2.5484 | LR: 4.85e-04 | 8080.0ms/step | 2.5 samples/s
+Step 4830/37465 | Loss: 2.5447 | LR: 4.85e-04 | 8070.9ms/step | 2.5 samples/s
+Step 4840/37465 | Loss: 2.3650 | LR: 4.85e-04 | 8066.4ms/step | 2.5 samples/s
+Step 4850/37465 | Loss: 2.5167 | LR: 4.85e-04 | 8067.6ms/step | 2.5 samples/s
+Step 4860/37465 | Loss: 2.2573 | LR: 4.85e-04 | 8067.0ms/step | 2.5 samples/s
+Step 4870/37465 | Loss: 2.5819 | LR: 4.85e-04 | 8069.0ms/step | 2.5 samples/s
+Step 4880/37465 | Loss: 2.8365 | LR: 4.85e-04 | 8070.2ms/step | 2.5 samples/s
+Step 4890/37465 | Loss: 2.6807 | LR: 4.84e-04 | 8069.4ms/step | 2.5 samples/s
+Step 4900/37465 | Loss: 2.3875 | LR: 4.84e-04 | 8062.9ms/step | 2.5 samples/s
+Step 4910/37465 | Loss: 2.4953 | LR: 4.84e-04 | 8066.4ms/step | 2.5 samples/s
+Step 4920/37465 | Loss: 2.6403 | LR: 4.84e-04 | 8063.9ms/step | 2.5 samples/s
+Step 4930/37465 | Loss: 2.1881 | LR: 4.84e-04 | 8092.9ms/step | 2.5 samples/s
+Step 4940/37465 | Loss: 2.3991 | LR: 4.84e-04 | 8084.3ms/step | 2.5 samples/s
+Step 4950/37465 | Loss: 2.4519 | LR: 4.84e-04 | 8083.1ms/step | 2.5 samples/s
+Step 4960/37465 | Loss: 2.4604 | LR: 4.84e-04 | 8082.6ms/step | 2.5 samples/s
+Step 4970/37465 | Loss: 2.4633 | LR: 4.84e-04 | 8083.0ms/step | 2.5 samples/s
+Step 4980/37465 | Loss: 2.3453 | LR: 4.84e-04 | 8084.9ms/step | 2.5 samples/s
+Step 4990/37465 | Loss: 2.4328 | LR: 4.84e-04 | 8081.4ms/step | 2.5 samples/s
+Step 5000/37465 | Loss: 2.6884 | LR: 4.84e-04 | 8084.0ms/step | 2.5 samples/s
+Validation at step 5000: Loss = 2.4698, Perplexity = 11.82
+Step 5010/37465 | Loss: 2.3863 | LR: 4.84e-04 | 8754.2ms/step | 2.3 samples/s
+Step 5020/37465 | Loss: 2.4304 | LR: 4.83e-04 | 8085.3ms/step | 2.5 samples/s
+Step 5030/37465 | Loss: 2.5446 | LR: 4.83e-04 | 8084.4ms/step | 2.5 samples/s
+Step 5040/37465 | Loss: 2.3558 | LR: 4.83e-04 | 8083.1ms/step | 2.5 samples/s
+Step 5050/37465 | Loss: 2.3629 | LR: 4.83e-04 | 8085.8ms/step | 2.5 samples/s
+Step 5060/37465 | Loss: 2.6030 | LR: 4.83e-04 | 8085.0ms/step | 2.5 samples/s
+Step 5070/37465 | Loss: 2.6285 | LR: 4.83e-04 | 8083.1ms/step | 2.5 samples/s
+Step 5080/37465 | Loss: 2.3657 | LR: 4.83e-04 | 8083.7ms/step | 2.5 samples/s
+Step 5090/37465 | Loss: 2.2803 | LR: 4.83e-04 | 8086.5ms/step | 2.5 samples/s
+Step 5100/37465 | Loss: 2.6280 | LR: 4.83e-04 | 8084.4ms/step | 2.5 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-5100
+Step 5110/37465 | Loss: 2.4089 | LR: 4.83e-04 | 8955.2ms/step | 2.2 samples/s
+Step 5120/37465 | Loss: 2.2645 | LR: 4.83e-04 | 8074.0ms/step | 2.5 samples/s
+Step 5130/37465 | Loss: 2.3024 | LR: 4.83e-04 | 8080.2ms/step | 2.5 samples/s
+Step 5140/37465 | Loss: 2.3108 | LR: 4.83e-04 | 8071.6ms/step | 2.5 samples/s
+Step 5150/37465 | Loss: 2.4108 | LR: 4.82e-04 | 8075.9ms/step | 2.5 samples/s
+Step 5160/37465 | Loss: 2.2551 | LR: 4.82e-04 | 8082.8ms/step | 2.5 samples/s
+Step 5170/37465 | Loss: 2.7241 | LR: 4.82e-04 | 8084.8ms/step | 2.5 samples/s
+Step 5180/37465 | Loss: 2.3629 | LR: 4.82e-04 | 8087.8ms/step | 2.5 samples/s
+Step 5190/37465 | Loss: 2.6014 | LR: 4.82e-04 | 8083.4ms/step | 2.5 samples/s
+Step 5200/37465 | Loss: 2.5887 | LR: 4.82e-04 | 8085.5ms/step | 2.5 samples/s
+Step 5210/37465 | Loss: 2.3217 | LR: 4.82e-04 | 8090.2ms/step | 2.5 samples/s
+Step 5220/37465 | Loss: 2.3421 | LR: 4.82e-04 | 8085.4ms/step | 2.5 samples/s
+Step 5230/37465 | Loss: 2.4593 | LR: 4.82e-04 | 8082.5ms/step | 2.5 samples/s
+Step 5240/37465 | Loss: 2.4859 | LR: 4.82e-04 | 8081.9ms/step | 2.5 samples/s
+Step 5250/37465 | Loss: 2.6048 | LR: 4.82e-04 | 8084.0ms/step | 2.5 samples/s
+Step 5260/37465 | Loss: 2.5825 | LR: 4.82e-04 | 8085.2ms/step | 2.5 samples/s
+Step 5270/37465 | Loss: 2.4574 | LR: 4.82e-04 | 8084.5ms/step | 2.5 samples/s
+Step 5280/37465 | Loss: 2.3719 | LR: 4.81e-04 | 8085.7ms/step | 2.5 samples/s
+Step 5290/37465 | Loss: 2.5576 | LR: 4.81e-04 | 8086.5ms/step | 2.5 samples/s
+Step 5300/37465 | Loss: 2.6705 | LR: 4.81e-04 | 8084.8ms/step | 2.5 samples/s
+Step 5310/37465 | Loss: 2.3375 | LR: 4.81e-04 | 8086.1ms/step | 2.5 samples/s
+Step 5320/37465 | Loss: 2.4033 | LR: 4.81e-04 | 8087.5ms/step | 2.5 samples/s
+Step 5330/37465 | Loss: 2.1951 | LR: 4.81e-04 | 8082.9ms/step | 2.5 samples/s
+Step 5340/37465 | Loss: 2.4334 | LR: 4.81e-04 | 8083.9ms/step | 2.5 samples/s
+Step 5350/37465 | Loss: 2.6761 | LR: 4.81e-04 | 8084.2ms/step | 2.5 samples/s
+Step 5360/37465 | Loss: 2.5290 | LR: 4.81e-04 | 8082.7ms/step | 2.5 samples/s
+Step 5370/37465 | Loss: 2.5438 | LR: 4.81e-04 | 8080.5ms/step | 2.5 samples/s
+Step 5380/37465 | Loss: 2.4727 | LR: 4.81e-04 | 8081.9ms/step | 2.5 samples/s
+Step 5390/37465 | Loss: 2.4013 | LR: 4.81e-04 | 8085.5ms/step | 2.5 samples/s
+Step 5400/37465 | Loss: 2.5290 | LR: 4.80e-04 | 8079.6ms/step | 2.5 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-5400
+Step 5410/37465 | Loss: 2.4230 | LR: 4.80e-04 | 8996.0ms/step | 2.2 samples/s
+Step 5420/37465 | Loss: 2.3855 | LR: 4.80e-04 | 8068.2ms/step | 2.5 samples/s
+Step 5430/37465 | Loss: 2.3206 | LR: 4.80e-04 | 8073.6ms/step | 2.5 samples/s
+Step 5440/37465 | Loss: 2.4543 | LR: 4.80e-04 | 8068.1ms/step | 2.5 samples/s
+Step 5450/37465 | Loss: 2.3674 | LR: 4.80e-04 | 8071.0ms/step | 2.5 samples/s
+Step 5460/37465 | Loss: 2.5938 | LR: 4.80e-04 | 8066.7ms/step | 2.5 samples/s
+Step 5470/37465 | Loss: 2.5577 | LR: 4.80e-04 | 8071.6ms/step | 2.5 samples/s
+Step 5480/37465 | Loss: 2.3778 | LR: 4.80e-04 | 8071.7ms/step | 2.5 samples/s
+Step 5490/37465 | Loss: 2.6355 | LR: 4.80e-04 | 8076.2ms/step | 2.5 samples/s
+Step 5500/37465 | Loss: 2.3604 | LR: 4.80e-04 | 8084.9ms/step | 2.5 samples/s
+Validation at step 5500: Loss = 2.4435, Perplexity = 11.51
+Step 5510/37465 | Loss: 2.2868 | LR: 4.80e-04 | 8752.7ms/step | 2.3 samples/s
+Step 5520/37465 | Loss: 2.3157 | LR: 4.79e-04 | 8081.4ms/step | 2.5 samples/s
+Step 5530/37465 | Loss: 2.4186 | LR: 4.79e-04 | 8077.2ms/step | 2.5 samples/s
+Step 5540/37465 | Loss: 2.2127 | LR: 4.79e-04 | 8078.9ms/step | 2.5 samples/s
+Step 5550/37465 | Loss: 2.2966 | LR: 4.79e-04 | 8079.7ms/step | 2.5 samples/s
+Step 5560/37465 | Loss: 2.6664 | LR: 4.79e-04 | 8083.2ms/step | 2.5 samples/s
+Step 5570/37465 | Loss: 2.2826 | LR: 4.79e-04 | 8082.9ms/step | 2.5 samples/s
+Step 5580/37465 | Loss: 2.4479 | LR: 4.79e-04 | 8083.3ms/step | 2.5 samples/s
+Step 5590/37465 | Loss: 2.4605 | LR: 4.79e-04 | 8083.7ms/step | 2.5 samples/s
+Step 5600/37465 | Loss: 2.2986 | LR: 4.79e-04 | 8081.9ms/step | 2.5 samples/s
+Step 5610/37465 | Loss: 2.3985 | LR: 4.79e-04 | 8082.0ms/step | 2.5 samples/s
+Step 5620/37465 | Loss: 2.4390 | LR: 4.79e-04 | 8083.8ms/step | 2.5 samples/s
+Step 5630/37465 | Loss: 2.5223 | LR: 4.79e-04 | 8083.9ms/step | 2.5 samples/s
+Step 5640/37465 | Loss: 2.3728 | LR: 4.78e-04 | 8082.4ms/step | 2.5 samples/s
+Step 5650/37465 | Loss: 2.2795 | LR: 4.78e-04 | 8085.3ms/step | 2.5 samples/s
+Step 5660/37465 | Loss: 2.4398 | LR: 4.78e-04 | 8082.7ms/step | 2.5 samples/s
+Step 5670/37465 | Loss: 2.5131 | LR: 4.78e-04 | 8084.9ms/step | 2.5 samples/s
+Step 5680/37465 | Loss: 2.4396 | LR: 4.78e-04 | 8082.2ms/step | 2.5 samples/s
+Step 5690/37465 | Loss: 2.3987 | LR: 4.78e-04 | 8084.2ms/step | 2.5 samples/s
+Step 5700/37465 | Loss: 2.5544 | LR: 4.78e-04 | 8082.0ms/step | 2.5 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-5700
+Step 5710/37465 | Loss: 2.4240 | LR: 4.78e-04 | 8972.8ms/step | 2.2 samples/s
+Step 5720/37465 | Loss: 2.3987 | LR: 4.78e-04 | 8075.1ms/step | 2.5 samples/s
+Step 5730/37465 | Loss: 2.4198 | LR: 4.78e-04 | 8064.6ms/step | 2.5 samples/s
+Step 5740/37465 | Loss: 2.3960 | LR: 4.78e-04 | 8068.0ms/step | 2.5 samples/s
+Step 5750/37465 | Loss: 2.4376 | LR: 4.77e-04 | 8066.7ms/step | 2.5 samples/s
+Step 5760/37465 | Loss: 2.1489 | LR: 4.77e-04 | 8065.3ms/step | 2.5 samples/s
+Step 5770/37465 | Loss: 2.4842 | LR: 4.77e-04 | 8069.4ms/step | 2.5 samples/s
+W0709 03:43:10.658000 161 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 161 got signal: 15

epfl_final/logs/l8k365m_dense_resume2.log

@@ -0,0 +1,209 @@
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run qlqhc3gy
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_045920-qlqhc3gy
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 365m dense 8k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qlqhc3gy
+Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-41000
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-41000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 44000,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m dense 8k",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "8k",
+    "scaleladder",
+    "dense"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-41000
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 768]            --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 768]            116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 768]            [2, 8192, 768]            768                       True
+├─Linear: 1-2                                                [2, 8192, 768]            [2, 8192, 151936]         116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 951.29
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 31197.23
+Params size (MB): 951.29
+Estimated Total Size (MB): 32148.66
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        1
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         20
+Sequence length:                          8192
+Tokens per optimizer step:                163,840
+Micro-batches per epoch (per rank):       1513882
+Optimizer steps per epoch:                151388
+Total optimizer steps:                    44000  (capped by max_steps; 0.29 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-41000...
+Overriding checkpoint LR with config LR: 0.0005
+Resumed from checkpoint at step 41000 (epoch 0, micro_step 410000)
+Resuming training from step 41000
+Resuming wandb run qlqhc3gy
+Starting training...
+Validation at step 41000: Loss = 2.8935, Perplexity = 18.06
+wandb: WARNING Tried to log to step 41000 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41010 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41020 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41030 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41040 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41050 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41060 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41070 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41080 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41090 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41100 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41110 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41120 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41130 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41140 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41150 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41160 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41170 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41180 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41190 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41200 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41210 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41220 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41230 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41240 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41250 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41260 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41270 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41280 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41290 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41300 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41310 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 41320 that is less than the current step 41321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.

epfl_final/logs/l8k365m_dense_resume3.log

@@ -0,0 +1,513 @@
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run qlqhc3gy
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_060003-qlqhc3gy
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 365m dense 8k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qlqhc3gy
+Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-42000
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-42000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 44000,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m dense 8k",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "8k",
+    "scaleladder",
+    "dense"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-42000
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 768]            --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 768]            116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 768]            [2, 8192, 768]            768                       True
+├─Linear: 1-2                                                [2, 8192, 768]            [2, 8192, 151936]         116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 951.29
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 31197.23
+Params size (MB): 951.29
+Estimated Total Size (MB): 32148.66
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        1
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         20
+Sequence length:                          8192
+Tokens per optimizer step:                163,840
+Micro-batches per epoch (per rank):       1513882
+Optimizer steps per epoch:                151388
+Total optimizer steps:                    44000  (capped by max_steps; 0.29 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-42000...
+Overriding checkpoint LR with config LR: 0.0005
+Resumed from checkpoint at step 42000 (epoch 0, micro_step 420000)
+Resuming training from step 42000
+Resuming wandb run qlqhc3gy
+Starting training...
+Validation at step 42000: Loss = 2.8956, Perplexity = 18.09
+wandb: WARNING Tried to log to step 42000 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42010 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42020 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42030 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42040 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42050 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42060 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42070 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42080 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42090 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42100 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42110 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42120 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42130 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42140 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42150 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42160 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42170 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42180 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42190 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42200 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42210 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42220 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42230 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42240 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42250 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42260 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42270 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42280 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42290 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42300 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42310 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42320 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42330 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42340 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42350 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42360 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42370 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42380 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42390 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42400 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42410 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42420 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42430 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42440 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42450 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42460 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42470 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42480 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42490 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42500 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42510 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42520 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42530 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42540 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42550 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42560 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42570 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42580 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42590 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42600 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42610 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42620 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42630 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42640 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42650 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42660 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42670 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42680 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42690 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42700 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42710 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42720 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42730 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42740 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42750 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42760 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42770 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42780 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42790 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42800 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42810 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42820 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42830 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42840 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42850 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42860 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42870 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42880 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42890 that is less than the current step 42891. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: updating run metadata
+wandb: uploading wandb-summary.json; uploading config.yaml
+wandb: uploading history steps 4399-4399, summary, console lines 302-303
+wandb: 
+wandb: Run history:
+wandb: eval/dense_interval_active ▁▁▁▁▁
+wandb:    eval/dense_interval_end ▁▁▁▁▁
+wandb:  eval/dense_interval_start ▁▁▁▁▁
+wandb:                  eval/loss █▂▁▁▃
+wandb:            eval/perplexity █▂▁▁▃
+wandb:  eval/uses_dense_attention ▁▁▁▁▁
+wandb: eval/uses_sparse_attention ▁▁▁▁▁
+wandb:          speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁█▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb:      speed/samples_per_sec █████▁███████████████████████████▁██████
+wandb:        speed/steps_per_sec ███████████████▁████████████████████████
+wandb:                         +4 ...
+wandb: 
+wandb: Run summary:
+wandb: eval/dense_interval_active 0
+wandb:    eval/dense_interval_end -1
+wandb:  eval/dense_interval_start -1
+wandb:                  eval/loss 2.91276
+wandb:            eval/perplexity 18.40744
+wandb:  eval/uses_dense_attention 1
+wandb: eval/uses_sparse_attention 0
+wandb:          speed/ms_per_step 2472.70107
+wandb:      speed/samples_per_sec 8.08832
+wandb:        speed/steps_per_sec 0.40442
+wandb:                         +4 ...
+wandb: 
+wandb: 🚀 View run 365m dense 8k at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qlqhc3gy
+wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20260708_060003-qlqhc3gy/logs
+Switching attention mode for step 42001: dense (dense step interval)
+Step 42010/44000 | Loss: 2.9026 | LR: 2.61e-06 | 58442.7ms/step | 0.3 samples/s
+Step 42020/44000 | Loss: 2.8940 | LR: 2.58e-06 | 2471.8ms/step | 8.1 samples/s
+Step 42030/44000 | Loss: 2.8854 | LR: 2.56e-06 | 2472.3ms/step | 8.1 samples/s
+Step 42040/44000 | Loss: 2.8688 | LR: 2.53e-06 | 2473.4ms/step | 8.1 samples/s
+Step 42050/44000 | Loss: 2.8986 | LR: 2.50e-06 | 2475.4ms/step | 8.1 samples/s
+Step 42060/44000 | Loss: 2.8971 | LR: 2.48e-06 | 2474.1ms/step | 8.1 samples/s
+Step 42070/44000 | Loss: 2.9161 | LR: 2.45e-06 | 2473.5ms/step | 8.1 samples/s
+Step 42080/44000 | Loss: 2.8754 | LR: 2.43e-06 | 2474.6ms/step | 8.1 samples/s
+Step 42090/44000 | Loss: 2.8948 | LR: 2.40e-06 | 2475.4ms/step | 8.1 samples/s
+Step 42100/44000 | Loss: 2.8972 | LR: 2.38e-06 | 2476.0ms/step | 8.1 samples/s
+Step 42110/44000 | Loss: 2.9053 | LR: 2.35e-06 | 2475.6ms/step | 8.1 samples/s
+Step 42120/44000 | Loss: 2.9192 | LR: 2.33e-06 | 2474.4ms/step | 8.1 samples/s
+Step 42130/44000 | Loss: 2.8948 | LR: 2.30e-06 | 2473.3ms/step | 8.1 samples/s
+Step 42140/44000 | Loss: 2.9002 | LR: 2.28e-06 | 2473.8ms/step | 8.1 samples/s
+Step 42150/44000 | Loss: 2.8784 | LR: 2.25e-06 | 2474.0ms/step | 8.1 samples/s
+Step 42160/44000 | Loss: 2.9180 | LR: 2.23e-06 | 2473.2ms/step | 8.1 samples/s
+Step 42170/44000 | Loss: 2.9129 | LR: 2.21e-06 | 2473.5ms/step | 8.1 samples/s
+Step 42180/44000 | Loss: 2.9178 | LR: 2.18e-06 | 2473.3ms/step | 8.1 samples/s
+Step 42190/44000 | Loss: 2.9356 | LR: 2.16e-06 | 2473.3ms/step | 8.1 samples/s
+Step 42200/44000 | Loss: 2.9144 | LR: 2.13e-06 | 2473.3ms/step | 8.1 samples/s
+Step 42210/44000 | Loss: 2.8649 | LR: 2.11e-06 | 2474.5ms/step | 8.1 samples/s
+Step 42220/44000 | Loss: 2.9082 | LR: 2.09e-06 | 2475.2ms/step | 8.1 samples/s
+Step 42230/44000 | Loss: 2.9055 | LR: 2.06e-06 | 2474.6ms/step | 8.1 samples/s
+Step 42240/44000 | Loss: 2.8562 | LR: 2.04e-06 | 2474.7ms/step | 8.1 samples/s
+Step 42250/44000 | Loss: 2.8947 | LR: 2.02e-06 | 2474.9ms/step | 8.1 samples/s
+Validation at step 42250: Loss = 2.9152, Perplexity = 18.45
+Step 42260/44000 | Loss: 2.9025 | LR: 1.99e-06 | 6679.7ms/step | 3.0 samples/s
+Step 42270/44000 | Loss: 2.9179 | LR: 1.97e-06 | 2474.1ms/step | 8.1 samples/s
+Step 42280/44000 | Loss: 2.8910 | LR: 1.95e-06 | 2474.6ms/step | 8.1 samples/s
+Step 42290/44000 | Loss: 2.9246 | LR: 1.93e-06 | 2474.4ms/step | 8.1 samples/s
+Step 42300/44000 | Loss: 2.8877 | LR: 1.90e-06 | 2473.5ms/step | 8.1 samples/s
+Step 42310/44000 | Loss: 2.9260 | LR: 1.88e-06 | 2474.1ms/step | 8.1 samples/s
+Step 42320/44000 | Loss: 2.9189 | LR: 1.86e-06 | 2473.8ms/step | 8.1 samples/s
+Step 42330/44000 | Loss: 2.9417 | LR: 1.84e-06 | 2474.1ms/step | 8.1 samples/s
+Step 42340/44000 | Loss: 2.8960 | LR: 1.82e-06 | 2474.1ms/step | 8.1 samples/s
+Step 42350/44000 | Loss: 2.8536 | LR: 1.79e-06 | 2474.2ms/step | 8.1 samples/s
+Step 42360/44000 | Loss: 2.9162 | LR: 1.77e-06 | 2474.4ms/step | 8.1 samples/s
+Step 42370/44000 | Loss: 2.9019 | LR: 1.75e-06 | 2473.8ms/step | 8.1 samples/s
+Step 42380/44000 | Loss: 2.9031 | LR: 1.73e-06 | 2473.9ms/step | 8.1 samples/s
+Step 42390/44000 | Loss: 2.9117 | LR: 1.71e-06 | 2472.9ms/step | 8.1 samples/s
+Step 42400/44000 | Loss: 2.8752 | LR: 1.69e-06 | 2472.6ms/step | 8.1 samples/s
+Step 42410/44000 | Loss: 2.8923 | LR: 1.67e-06 | 2477.5ms/step | 8.1 samples/s
+Step 42420/44000 | Loss: 2.8904 | LR: 1.64e-06 | 2478.4ms/step | 8.1 samples/s
+Step 42430/44000 | Loss: 2.8909 | LR: 1.62e-06 | 2474.9ms/step | 8.1 samples/s
+Step 42440/44000 | Loss: 2.8826 | LR: 1.60e-06 | 2473.5ms/step | 8.1 samples/s
+Step 42450/44000 | Loss: 2.8496 | LR: 1.58e-06 | 2472.3ms/step | 8.1 samples/s
+Step 42460/44000 | Loss: 2.8867 | LR: 1.56e-06 | 2473.0ms/step | 8.1 samples/s
+Step 42470/44000 | Loss: 2.8888 | LR: 1.54e-06 | 2473.8ms/step | 8.1 samples/s
+Step 42480/44000 | Loss: 2.8738 | LR: 1.52e-06 | 2474.0ms/step | 8.1 samples/s
+Step 42490/44000 | Loss: 2.9266 | LR: 1.50e-06 | 2474.3ms/step | 8.1 samples/s
+Step 42500/44000 | Loss: 2.9400 | LR: 1.48e-06 | 2473.3ms/step | 8.1 samples/s
+Validation at step 42500: Loss = 2.9134, Perplexity = 18.42
+Step 42510/44000 | Loss: 2.8661 | LR: 1.46e-06 | 6676.2ms/step | 3.0 samples/s
+Step 42520/44000 | Loss: 2.8836 | LR: 1.44e-06 | 2473.9ms/step | 8.1 samples/s
+Step 42530/44000 | Loss: 2.8974 | LR: 1.42e-06 | 2473.8ms/step | 8.1 samples/s
+Step 42540/44000 | Loss: 2.8869 | LR: 1.40e-06 | 2473.9ms/step | 8.1 samples/s
+Step 42550/44000 | Loss: 2.9057 | LR: 1.39e-06 | 2473.6ms/step | 8.1 samples/s
+Step 42560/44000 | Loss: 2.9137 | LR: 1.37e-06 | 2476.8ms/step | 8.1 samples/s
+Step 42570/44000 | Loss: 2.9186 | LR: 1.35e-06 | 2474.4ms/step | 8.1 samples/s
+Step 42580/44000 | Loss: 2.9066 | LR: 1.33e-06 | 2472.8ms/step | 8.1 samples/s
+Step 42590/44000 | Loss: 2.9172 | LR: 1.31e-06 | 2473.4ms/step | 8.1 samples/s
+Step 42600/44000 | Loss: 2.9135 | LR: 1.29e-06 | 2472.6ms/step | 8.1 samples/s
+Step 42610/44000 | Loss: 2.8956 | LR: 1.27e-06 | 2474.9ms/step | 8.1 samples/s
+Step 42620/44000 | Loss: 2.8944 | LR: 1.25e-06 | 2475.7ms/step | 8.1 samples/s
+Step 42630/44000 | Loss: 2.8900 | LR: 1.24e-06 | 2474.4ms/step | 8.1 samples/s
+Step 42640/44000 | Loss: 2.9035 | LR: 1.22e-06 | 2474.4ms/step | 8.1 samples/s
+Step 42650/44000 | Loss: 2.8893 | LR: 1.20e-06 | 2477.1ms/step | 8.1 samples/s
+Step 42660/44000 | Loss: 2.9111 | LR: 1.18e-06 | 2474.4ms/step | 8.1 samples/s
+Step 42670/44000 | Loss: 2.8768 | LR: 1.17e-06 | 2473.4ms/step | 8.1 samples/s
+Step 42680/44000 | Loss: 2.8596 | LR: 1.15e-06 | 2474.7ms/step | 8.1 samples/s
+Step 42690/44000 | Loss: 2.8910 | LR: 1.13e-06 | 2475.0ms/step | 8.1 samples/s
+Step 42700/44000 | Loss: 2.9024 | LR: 1.11e-06 | 2474.9ms/step | 8.1 samples/s
+Step 42710/44000 | Loss: 2.9106 | LR: 1.10e-06 | 2475.7ms/step | 8.1 samples/s
+Step 42720/44000 | Loss: 2.9081 | LR: 1.08e-06 | 2475.6ms/step | 8.1 samples/s
+Step 42730/44000 | Loss: 2.8979 | LR: 1.06e-06 | 2475.9ms/step | 8.1 samples/s
+Step 42740/44000 | Loss: 2.9098 | LR: 1.05e-06 | 2473.4ms/step | 8.1 samples/s
+Step 42750/44000 | Loss: 2.9176 | LR: 1.03e-06 | 2474.7ms/step | 8.1 samples/s
+Validation at step 42750: Loss = 2.9128, Perplexity = 18.41
+Step 42760/44000 | Loss: 2.8979 | LR: 1.01e-06 | 6675.5ms/step | 3.0 samples/s
+Step 42770/44000 | Loss: 2.9136 | LR: 9.97e-07 | 2473.9ms/step | 8.1 samples/s
+Step 42780/44000 | Loss: 2.8889 | LR: 9.81e-07 | 2474.6ms/step | 8.1 samples/s
+Step 42790/44000 | Loss: 2.9061 | LR: 9.65e-07 | 2475.3ms/step | 8.1 samples/s
+Step 42800/44000 | Loss: 2.9060 | LR: 9.49e-07 | 2474.3ms/step | 8.1 samples/s
+Step 42810/44000 | Loss: 2.8747 | LR: 9.33e-07 | 2474.0ms/step | 8.1 samples/s
+Step 42820/44000 | Loss: 2.9063 | LR: 9.18e-07 | 2474.3ms/step | 8.1 samples/s
+Step 42830/44000 | Loss: 2.9125 | LR: 9.02e-07 | 2475.6ms/step | 8.1 samples/s
+Step 42840/44000 | Loss: 2.9042 | LR: 8.87e-07 | 2474.3ms/step | 8.1 samples/s
+Step 42850/44000 | Loss: 2.8695 | LR: 8.72e-07 | 2475.6ms/step | 8.1 samples/s
+Step 42860/44000 | Loss: 2.8815 | LR: 8.57e-07 | 2478.8ms/step | 8.1 samples/s
+Step 42870/44000 | Loss: 2.9173 | LR: 8.42e-07 | 2474.9ms/step | 8.1 samples/s
+Step 42880/44000 | Loss: 2.9004 | LR: 8.27e-07 | 2472.8ms/step | 8.1 samples/s
+Step 42890/44000 | Loss: 2.9127 | LR: 8.12e-07 | 2472.7ms/step | 8.1 samples/s
+Step 42900/44000 | Loss: 2.9098 | LR: 7.98e-07 | 2472.7ms/step | 8.1 samples/s
+Step 42910/44000 | Loss: 2.8639 | LR: 7.83e-07 | 2472.6ms/step | 8.1 samples/s
+Step 42920/44000 | Loss: 2.8815 | LR: 7.69e-07 | 2473.5ms/step | 8.1 samples/s
+Step 42930/44000 | Loss: 2.8930 | LR: 7.55e-07 | 2473.4ms/step | 8.1 samples/s
+Step 42940/44000 | Loss: 2.8811 | LR: 7.41e-07 | 2473.3ms/step | 8.1 samples/s
+Step 42950/44000 | Loss: 2.9038 | LR: 7.27e-07 | 2473.6ms/step | 8.1 samples/s
+Step 42960/44000 | Loss: 2.8809 | LR: 7.13e-07 | 2472.9ms/step | 8.1 samples/s
+Step 42970/44000 | Loss: 2.9096 | LR: 6.99e-07 | 2472.7ms/step | 8.1 samples/s
+Step 42980/44000 | Loss: 2.9116 | LR: 6.86e-07 | 2472.9ms/step | 8.1 samples/s
+Step 42990/44000 | Loss: 2.8679 | LR: 6.72e-07 | 2473.2ms/step | 8.1 samples/s
+Step 43000/44000 | Loss: 2.8914 | LR: 6.59e-07 | 2473.3ms/step | 8.1 samples/s
+Validation at step 43000: Loss = 2.9128, Perplexity = 18.41
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-43000
+Step 43010/44000 | Loss: 2.8783 | LR: 6.46e-07 | 6981.5ms/step | 2.9 samples/s
+Step 43020/44000 | Loss: 2.9055 | LR: 6.33e-07 | 2473.4ms/step | 8.1 samples/s
+Step 43030/44000 | Loss: 2.9116 | LR: 6.20e-07 | 2473.2ms/step | 8.1 samples/s
+Step 43040/44000 | Loss: 2.8814 | LR: 6.08e-07 | 2472.8ms/step | 8.1 samples/s
+Step 43050/44000 | Loss: 2.8901 | LR: 5.95e-07 | 2473.4ms/step | 8.1 samples/s
+Step 43060/44000 | Loss: 2.8589 | LR: 5.83e-07 | 2473.9ms/step | 8.1 samples/s
+Step 43070/44000 | Loss: 2.8816 | LR: 5.70e-07 | 2474.0ms/step | 8.1 samples/s
+Step 43080/44000 | Loss: 2.8898 | LR: 5.58e-07 | 2474.0ms/step | 8.1 samples/s
+Step 43090/44000 | Loss: 2.9181 | LR: 5.46e-07 | 2474.5ms/step | 8.1 samples/s
+Step 43100/44000 | Loss: 2.8988 | LR: 5.34e-07 | 2473.4ms/step | 8.1 samples/s
+Step 43110/44000 | Loss: 2.8700 | LR: 5.22e-07 | 2472.6ms/step | 8.1 samples/s
+Step 43120/44000 | Loss: 2.8807 | LR: 5.11e-07 | 2472.0ms/step | 8.1 samples/s
+Step 43130/44000 | Loss: 2.9162 | LR: 4.99e-07 | 2473.0ms/step | 8.1 samples/s
+Step 43140/44000 | Loss: 2.9092 | LR: 4.88e-07 | 2472.3ms/step | 8.1 samples/s
+Step 43150/44000 | Loss: 2.8604 | LR: 4.76e-07 | 2472.6ms/step | 8.1 samples/s
+Step 43160/44000 | Loss: 2.8885 | LR: 4.65e-07 | 2471.5ms/step | 8.1 samples/s
+Step 43170/44000 | Loss: 2.8890 | LR: 4.54e-07 | 2472.7ms/step | 8.1 samples/s
+Step 43180/44000 | Loss: 2.8879 | LR: 4.43e-07 | 2474.7ms/step | 8.1 samples/s
+Step 43190/44000 | Loss: 2.8875 | LR: 4.33e-07 | 2473.3ms/step | 8.1 samples/s
+Step 43200/44000 | Loss: 2.9041 | LR: 4.22e-07 | 2473.3ms/step | 8.1 samples/s
+Step 43210/44000 | Loss: 2.8635 | LR: 4.12e-07 | 2473.5ms/step | 8.1 samples/s
+Step 43220/44000 | Loss: 2.9215 | LR: 4.01e-07 | 2473.8ms/step | 8.1 samples/s
+Step 43230/44000 | Loss: 2.9011 | LR: 3.91e-07 | 2473.3ms/step | 8.1 samples/s
+Step 43240/44000 | Loss: 2.9095 | LR: 3.81e-07 | 2473.7ms/step | 8.1 samples/s
+Step 43250/44000 | Loss: 2.9180 | LR: 3.71e-07 | 2473.4ms/step | 8.1 samples/s
+Validation at step 43250: Loss = 2.9128, Perplexity = 18.41
+Step 43260/44000 | Loss: 2.9016 | LR: 3.61e-07 | 6676.0ms/step | 3.0 samples/s
+Step 43270/44000 | Loss: 2.9167 | LR: 3.51e-07 | 2472.6ms/step | 8.1 samples/s
+Step 43280/44000 | Loss: 2.8757 | LR: 3.42e-07 | 2472.6ms/step | 8.1 samples/s
+Step 43290/44000 | Loss: 2.8804 | LR: 3.32e-07 | 2472.1ms/step | 8.1 samples/s
+Step 43300/44000 | Loss: 2.9320 | LR: 3.23e-07 | 2471.2ms/step | 8.1 samples/s
+Step 43310/44000 | Loss: 2.8923 | LR: 3.14e-07 | 2471.9ms/step | 8.1 samples/s
+Step 43320/44000 | Loss: 2.8882 | LR: 3.05e-07 | 2480.6ms/step | 8.1 samples/s
+Step 43330/44000 | Loss: 2.9103 | LR: 2.96e-07 | 2472.7ms/step | 8.1 samples/s
+Step 43340/44000 | Loss: 2.8948 | LR: 2.87e-07 | 2473.0ms/step | 8.1 samples/s
+Step 43350/44000 | Loss: 2.8576 | LR: 2.79e-07 | 2473.1ms/step | 8.1 samples/s
+Step 43360/44000 | Loss: 2.8828 | LR: 2.70e-07 | 2473.6ms/step | 8.1 samples/s
+Step 43370/44000 | Loss: 2.8778 | LR: 2.62e-07 | 2474.4ms/step | 8.1 samples/s
+Step 43380/44000 | Loss: 2.8832 | LR: 2.53e-07 | 2474.2ms/step | 8.1 samples/s
+Step 43390/44000 | Loss: 2.8876 | LR: 2.45e-07 | 2473.4ms/step | 8.1 samples/s
+Step 43400/44000 | Loss: 2.8796 | LR: 2.37e-07 | 2475.7ms/step | 8.1 samples/s
+Step 43410/44000 | Loss: 2.8704 | LR: 2.30e-07 | 2480.9ms/step | 8.1 samples/s
+Step 43420/44000 | Loss: 2.9239 | LR: 2.22e-07 | 2479.2ms/step | 8.1 samples/s
+Step 43430/44000 | Loss: 2.8647 | LR: 2.14e-07 | 2482.4ms/step | 8.1 samples/s
+Step 43440/44000 | Loss: 2.8877 | LR: 2.07e-07 | 2481.7ms/step | 8.1 samples/s
+Step 43450/44000 | Loss: 2.8725 | LR: 1.99e-07 | 2479.4ms/step | 8.1 samples/s
+Step 43460/44000 | Loss: 2.8947 | LR: 1.92e-07 | 2475.6ms/step | 8.1 samples/s
+Step 43470/44000 | Loss: 2.8734 | LR: 1.85e-07 | 2477.6ms/step | 8.1 samples/s
+Step 43480/44000 | Loss: 2.8769 | LR: 1.78e-07 | 2477.4ms/step | 8.1 samples/s
+Step 43490/44000 | Loss: 2.9095 | LR: 1.72e-07 | 2476.5ms/step | 8.1 samples/s
+Step 43500/44000 | Loss: 2.8874 | LR: 1.65e-07 | 2473.5ms/step | 8.1 samples/s
+Validation at step 43500: Loss = 2.9128, Perplexity = 18.41
+Step 43510/44000 | Loss: 2.8898 | LR: 1.58e-07 | 6675.9ms/step | 3.0 samples/s
+Step 43520/44000 | Loss: 2.8896 | LR: 1.52e-07 | 2471.9ms/step | 8.1 samples/s
+Step 43530/44000 | Loss: 2.8807 | LR: 1.46e-07 | 2472.7ms/step | 8.1 samples/s
+Step 43540/44000 | Loss: 2.8856 | LR: 1.40e-07 | 2472.7ms/step | 8.1 samples/s
+Step 43550/44000 | Loss: 2.9021 | LR: 1.34e-07 | 2472.8ms/step | 8.1 samples/s
+Step 43560/44000 | Loss: 2.8786 | LR: 1.28e-07 | 2473.0ms/step | 8.1 samples/s
+Step 43570/44000 | Loss: 2.8951 | LR: 1.22e-07 | 2472.2ms/step | 8.1 samples/s
+Step 43580/44000 | Loss: 2.9028 | LR: 1.16e-07 | 2472.8ms/step | 8.1 samples/s
+Step 43590/44000 | Loss: 2.9086 | LR: 1.11e-07 | 2472.4ms/step | 8.1 samples/s
+Step 43600/44000 | Loss: 2.8982 | LR: 1.06e-07 | 2472.2ms/step | 8.1 samples/s
+Step 43610/44000 | Loss: 2.8895 | LR: 1.00e-07 | 2472.7ms/step | 8.1 samples/s
+Step 43620/44000 | Loss: 2.8601 | LR: 9.52e-08 | 2473.7ms/step | 8.1 samples/s
+Step 43630/44000 | Loss: 2.9137 | LR: 9.03e-08 | 2474.1ms/step | 8.1 samples/s
+Step 43640/44000 | Loss: 2.9201 | LR: 8.55e-08 | 2479.9ms/step | 8.1 samples/s
+Step 43650/44000 | Loss: 2.8716 | LR: 8.08e-08 | 2475.0ms/step | 8.1 samples/s
+Step 43660/44000 | Loss: 2.8757 | LR: 7.62e-08 | 2473.3ms/step | 8.1 samples/s
+Step 43670/44000 | Loss: 2.9088 | LR: 7.18e-08 | 2473.4ms/step | 8.1 samples/s
+Step 43680/44000 | Loss: 2.9063 | LR: 6.75e-08 | 2476.3ms/step | 8.1 samples/s
+Step 43690/44000 | Loss: 2.9049 | LR: 6.34e-08 | 2474.2ms/step | 8.1 samples/s
+Step 43700/44000 | Loss: 2.9062 | LR: 5.94e-08 | 2472.7ms/step | 8.1 samples/s
+Step 43710/44000 | Loss: 2.8842 | LR: 5.55e-08 | 2472.9ms/step | 8.1 samples/s
+Step 43720/44000 | Loss: 2.8903 | LR: 5.17e-08 | 2472.2ms/step | 8.1 samples/s
+Step 43730/44000 | Loss: 2.9026 | LR: 4.81e-08 | 2472.5ms/step | 8.1 samples/s
+Step 43740/44000 | Loss: 2.8897 | LR: 4.46e-08 | 2473.9ms/step | 8.1 samples/s
+Step 43750/44000 | Loss: 2.8944 | LR: 4.12e-08 | 2473.0ms/step | 8.1 samples/s
+Validation at step 43750: Loss = 2.9127, Perplexity = 18.41
+Step 43760/44000 | Loss: 2.8659 | LR: 3.80e-08 | 6671.9ms/step | 3.0 samples/s
+Step 43770/44000 | Loss: 2.8963 | LR: 3.49e-08 | 2475.3ms/step | 8.1 samples/s
+Step 43780/44000 | Loss: 2.8845 | LR: 3.19e-08 | 2475.8ms/step | 8.1 samples/s
+Step 43790/44000 | Loss: 2.9188 | LR: 2.91e-08 | 2477.7ms/step | 8.1 samples/s
+Step 43800/44000 | Loss: 2.8794 | LR: 2.64e-08 | 2474.1ms/step | 8.1 samples/s
+Step 43810/44000 | Loss: 2.8787 | LR: 2.38e-08 | 2474.4ms/step | 8.1 samples/s
+Step 43820/44000 | Loss: 2.8945 | LR: 2.14e-08 | 2474.2ms/step | 8.1 samples/s
+Step 43830/44000 | Loss: 2.8698 | LR: 1.91e-08 | 2474.8ms/step | 8.1 samples/s
+Step 43840/44000 | Loss: 2.8995 | LR: 1.69e-08 | 2474.3ms/step | 8.1 samples/s
+Step 43850/44000 | Loss: 2.8989 | LR: 1.48e-08 | 2474.5ms/step | 8.1 samples/s
+Step 43860/44000 | Loss: 2.9028 | LR: 1.29e-08 | 2474.0ms/step | 8.1 samples/s
+Step 43870/44000 | Loss: 2.8806 | LR: 1.11e-08 | 2473.2ms/step | 8.1 samples/s
+Step 43880/44000 | Loss: 2.8919 | LR: 9.50e-09 | 2475.0ms/step | 8.1 samples/s
+Step 43890/44000 | Loss: 2.9090 | LR: 7.98e-09 | 2473.2ms/step | 8.1 samples/s
+Step 43900/44000 | Loss: 2.8898 | LR: 6.60e-09 | 2475.8ms/step | 8.1 samples/s
+Step 43910/44000 | Loss: 2.9074 | LR: 5.34e-09 | 2474.6ms/step | 8.1 samples/s
+Step 43920/44000 | Loss: 2.8697 | LR: 4.22e-09 | 2476.8ms/step | 8.1 samples/s
+Step 43930/44000 | Loss: 2.9084 | LR: 3.23e-09 | 2474.6ms/step | 8.1 samples/s
+Step 43940/44000 | Loss: 2.8652 | LR: 2.37e-09 | 2478.6ms/step | 8.1 samples/s
+Step 43950/44000 | Loss: 2.9014 | LR: 1.65e-09 | 2473.5ms/step | 8.1 samples/s
+Step 43960/44000 | Loss: 2.8861 | LR: 1.06e-09 | 2473.2ms/step | 8.1 samples/s
+Step 43970/44000 | Loss: 2.9254 | LR: 5.94e-10 | 2474.1ms/step | 8.1 samples/s
+Step 43980/44000 | Loss: 2.8674 | LR: 2.64e-10 | 2473.4ms/step | 8.1 samples/s
+Step 43990/44000 | Loss: 2.9117 | LR: 6.60e-11 | 2474.6ms/step | 8.1 samples/s
+Step 44000/44000 | Loss: 2.8938 | LR: 0.00e+00 | 2472.7ms/step | 8.1 samples/s
+Validation at step 44000: Loss = 2.9128, Perplexity = 18.41
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000
+Training complete!

epfl_final/logs/l8k365m_pool10.log

@@ -0,0 +1,1072 @@
+W0708 20:48:06.039000 254 site-packages/torch/distributed/run.py:851] 
+W0708 20:48:06.039000 254 site-packages/torch/distributed/run.py:851] *****************************************
+W0708 20:48:06.039000 254 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0708 20:48:06.039000 254 site-packages/torch/distributed/run.py:851] *****************************************
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 44000,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 5,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m pool10 8k nrb31 freqabl",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "8k",
+    "scaleladder",
+    "resampled",
+    "freqablation",
+    "pool10"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32",
+  "sparse_mask_seed": 0,
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: False
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 768]            --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 768]            116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 768]            [2, 8192, 768]            768                       True
+├─Linear: 1-2                                                [2, 8192, 768]            [2, 8192, 151936]         116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 951.29
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 31197.23
+Params size (MB): 951.29
+Estimated Total Size (MB): 32148.66
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        2
+Gradient accumulation steps:              5
+Effective batch size (seqs/step):         20
+Sequence length:                          8192
+Tokens per optimizer step:                163,840
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                151388
+Total optimizer steps:                    44000  (capped by max_steps; 0.29 epochs)
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run ti3a1x4c
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_205109-ti3a1x4c
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 365m pool10 8k nrb31 freqabl
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/ti3a1x4c
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 10/44000 | Loss: 12.0382 | LR: 6.67e-06 | 6783.2ms/step | 2.9 samples/s
+Step 20/44000 | Loss: 11.9252 | LR: 1.33e-05 | 671.2ms/step | 29.8 samples/s
+Step 30/44000 | Loss: 11.5508 | LR: 2.00e-05 | 670.7ms/step | 29.8 samples/s
+Step 40/44000 | Loss: 11.0227 | LR: 2.67e-05 | 668.7ms/step | 29.9 samples/s
+Step 50/44000 | Loss: 10.7142 | LR: 3.33e-05 | 667.4ms/step | 30.0 samples/s
+Step 60/44000 | Loss: 10.4834 | LR: 4.00e-05 | 666.6ms/step | 30.0 samples/s
+Step 70/44000 | Loss: 10.2641 | LR: 4.67e-05 | 667.3ms/step | 30.0 samples/s
+Step 80/44000 | Loss: 9.9467 | LR: 5.33e-05 | 665.5ms/step | 30.1 samples/s
+Step 90/44000 | Loss: 9.5576 | LR: 6.00e-05 | 670.1ms/step | 29.8 samples/s
+Step 100/44000 | Loss: 9.1273 | LR: 6.67e-05 | 669.9ms/step | 29.9 samples/s
+Step 110/44000 | Loss: 8.6009 | LR: 7.33e-05 | 669.8ms/step | 29.9 samples/s
+Step 120/44000 | Loss: 8.1706 | LR: 8.00e-05 | 670.7ms/step | 29.8 samples/s
+Step 130/44000 | Loss: 7.8000 | LR: 8.67e-05 | 669.7ms/step | 29.9 samples/s
+Step 140/44000 | Loss: 7.5213 | LR: 9.33e-05 | 671.1ms/step | 29.8 samples/s
+Step 150/44000 | Loss: 7.3526 | LR: 1.00e-04 | 671.4ms/step | 29.8 samples/s
+Step 160/44000 | Loss: 7.1745 | LR: 1.07e-04 | 672.9ms/step | 29.7 samples/s
+Step 170/44000 | Loss: 7.1121 | LR: 1.13e-04 | 672.2ms/step | 29.8 samples/s
+Step 180/44000 | Loss: 7.0473 | LR: 1.20e-04 | 673.6ms/step | 29.7 samples/s
+Step 190/44000 | Loss: 6.9656 | LR: 1.27e-04 | 674.7ms/step | 29.6 samples/s
+Step 200/44000 | Loss: 6.8609 | LR: 1.33e-04 | 674.4ms/step | 29.7 samples/s
+Step 210/44000 | Loss: 6.7756 | LR: 1.40e-04 | 675.2ms/step | 29.6 samples/s
+Step 220/44000 | Loss: 6.6990 | LR: 1.47e-04 | 675.0ms/step | 29.6 samples/s
+Step 230/44000 | Loss: 6.5972 | LR: 1.53e-04 | 676.2ms/step | 29.6 samples/s
+Step 240/44000 | Loss: 6.5657 | LR: 1.60e-04 | 676.3ms/step | 29.6 samples/s
+Step 250/44000 | Loss: 6.4418 | LR: 1.67e-04 | 675.6ms/step | 29.6 samples/s
+Validation at step 250: Loss = 6.4439, Perplexity = 628.86
+Step 260/44000 | Loss: 6.4414 | LR: 1.73e-04 | 3935.5ms/step | 5.1 samples/s
+Step 270/44000 | Loss: 6.3787 | LR: 1.80e-04 | 676.7ms/step | 29.6 samples/s
+Step 280/44000 | Loss: 6.2945 | LR: 1.87e-04 | 676.6ms/step | 29.6 samples/s
+Step 290/44000 | Loss: 6.2513 | LR: 1.93e-04 | 678.2ms/step | 29.5 samples/s
+Step 300/44000 | Loss: 6.2507 | LR: 2.00e-04 | 676.8ms/step | 29.6 samples/s
+Step 310/44000 | Loss: 6.2086 | LR: 2.07e-04 | 679.3ms/step | 29.4 samples/s
+Step 320/44000 | Loss: 6.1304 | LR: 2.13e-04 | 677.8ms/step | 29.5 samples/s
+Step 330/44000 | Loss: 6.0840 | LR: 2.20e-04 | 678.7ms/step | 29.5 samples/s
+Step 340/44000 | Loss: 6.0627 | LR: 2.27e-04 | 678.2ms/step | 29.5 samples/s
+Step 350/44000 | Loss: 5.9852 | LR: 2.33e-04 | 679.5ms/step | 29.4 samples/s
+Step 360/44000 | Loss: 5.9827 | LR: 2.40e-04 | 679.9ms/step | 29.4 samples/s
+Step 370/44000 | Loss: 5.8705 | LR: 2.47e-04 | 678.0ms/step | 29.5 samples/s
+Step 380/44000 | Loss: 5.8938 | LR: 2.53e-04 | 677.8ms/step | 29.5 samples/s
+Step 390/44000 | Loss: 5.8599 | LR: 2.60e-04 | 678.1ms/step | 29.5 samples/s
+Step 400/44000 | Loss: 5.8131 | LR: 2.67e-04 | 678.1ms/step | 29.5 samples/s
+Step 410/44000 | Loss: 5.7854 | LR: 2.73e-04 | 678.6ms/step | 29.5 samples/s
+Step 420/44000 | Loss: 5.7321 | LR: 2.80e-04 | 678.1ms/step | 29.5 samples/s
+Step 430/44000 | Loss: 5.7388 | LR: 2.87e-04 | 678.6ms/step | 29.5 samples/s
+Step 440/44000 | Loss: 5.7220 | LR: 2.93e-04 | 678.3ms/step | 29.5 samples/s
+Step 450/44000 | Loss: 5.6070 | LR: 3.00e-04 | 680.3ms/step | 29.4 samples/s
+Step 460/44000 | Loss: 5.6095 | LR: 3.07e-04 | 677.9ms/step | 29.5 samples/s
+Step 470/44000 | Loss: 5.5903 | LR: 3.13e-04 | 678.9ms/step | 29.5 samples/s
+Step 480/44000 | Loss: 5.5445 | LR: 3.20e-04 | 679.7ms/step | 29.4 samples/s
+Step 490/44000 | Loss: 5.5292 | LR: 3.27e-04 | 679.7ms/step | 29.4 samples/s
+Step 500/44000 | Loss: 5.4936 | LR: 3.33e-04 | 678.5ms/step | 29.5 samples/s
+Validation at step 500: Loss = 5.4865, Perplexity = 241.40
+Step 510/44000 | Loss: 5.3963 | LR: 3.40e-04 | 1233.9ms/step | 16.2 samples/s
+Step 520/44000 | Loss: 5.3932 | LR: 3.47e-04 | 680.1ms/step | 29.4 samples/s
+Step 530/44000 | Loss: 5.3959 | LR: 3.53e-04 | 679.0ms/step | 29.5 samples/s
+Step 540/44000 | Loss: 5.3567 | LR: 3.60e-04 | 678.1ms/step | 29.5 samples/s
+Step 550/44000 | Loss: 5.3682 | LR: 3.67e-04 | 679.1ms/step | 29.5 samples/s
+Step 560/44000 | Loss: 5.3081 | LR: 3.73e-04 | 679.1ms/step | 29.4 samples/s
+Step 570/44000 | Loss: 5.3110 | LR: 3.80e-04 | 678.3ms/step | 29.5 samples/s
+Step 580/44000 | Loss: 5.2360 | LR: 3.87e-04 | 678.2ms/step | 29.5 samples/s
+Step 590/44000 | Loss: 5.2242 | LR: 3.93e-04 | 680.0ms/step | 29.4 samples/s
+Step 600/44000 | Loss: 5.2097 | LR: 4.00e-04 | 678.8ms/step | 29.5 samples/s
+Step 610/44000 | Loss: 5.1316 | LR: 4.07e-04 | 679.2ms/step | 29.4 samples/s
+Step 620/44000 | Loss: 5.1077 | LR: 4.13e-04 | 679.4ms/step | 29.4 samples/s
+Step 630/44000 | Loss: 5.1647 | LR: 4.20e-04 | 678.6ms/step | 29.5 samples/s
+Step 640/44000 | Loss: 5.1062 | LR: 4.27e-04 | 678.7ms/step | 29.5 samples/s
+Step 650/44000 | Loss: 5.0631 | LR: 4.33e-04 | 677.8ms/step | 29.5 samples/s
+Step 660/44000 | Loss: 5.0732 | LR: 4.40e-04 | 678.6ms/step | 29.5 samples/s
+Step 670/44000 | Loss: 5.0441 | LR: 4.47e-04 | 679.0ms/step | 29.5 samples/s
+Step 680/44000 | Loss: 5.0482 | LR: 4.53e-04 | 680.4ms/step | 29.4 samples/s
+Step 690/44000 | Loss: 4.9983 | LR: 4.60e-04 | 681.3ms/step | 29.4 samples/s
+Step 700/44000 | Loss: 4.9409 | LR: 4.67e-04 | 679.7ms/step | 29.4 samples/s
+Step 710/44000 | Loss: 4.9727 | LR: 4.73e-04 | 680.2ms/step | 29.4 samples/s
+Step 720/44000 | Loss: 4.8840 | LR: 4.80e-04 | 680.4ms/step | 29.4 samples/s
+Step 730/44000 | Loss: 4.9085 | LR: 4.87e-04 | 679.2ms/step | 29.4 samples/s
+Step 740/44000 | Loss: 4.8286 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s
+Step 750/44000 | Loss: 4.7973 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Validation at step 750: Loss = 4.8014, Perplexity = 121.68
+Step 760/44000 | Loss: 4.7792 | LR: 5.00e-04 | 1233.5ms/step | 16.2 samples/s
+Step 770/44000 | Loss: 4.8051 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s
+Step 780/44000 | Loss: 4.7135 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s
+Step 790/44000 | Loss: 4.7176 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 800/44000 | Loss: 4.7575 | LR: 5.00e-04 | 681.4ms/step | 29.4 samples/s
+Step 810/44000 | Loss: 4.6824 | LR: 5.00e-04 | 678.3ms/step | 29.5 samples/s
+Step 820/44000 | Loss: 4.6420 | LR: 5.00e-04 | 678.4ms/step | 29.5 samples/s
+Step 830/44000 | Loss: 4.6077 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 840/44000 | Loss: 4.5562 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 850/44000 | Loss: 4.5649 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 860/44000 | Loss: 4.5487 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 870/44000 | Loss: 4.5591 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s
+Step 880/44000 | Loss: 4.4944 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s
+Step 890/44000 | Loss: 4.4827 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 900/44000 | Loss: 4.4674 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 910/44000 | Loss: 4.4617 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Step 920/44000 | Loss: 4.4134 | LR: 5.00e-04 | 679.3ms/step | 29.4 samples/s
+Step 930/44000 | Loss: 4.4133 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 940/44000 | Loss: 4.3863 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Step 950/44000 | Loss: 4.4044 | LR: 5.00e-04 | 677.9ms/step | 29.5 samples/s
+Step 960/44000 | Loss: 4.3391 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Step 970/44000 | Loss: 4.3446 | LR: 5.00e-04 | 678.9ms/step | 29.5 samples/s
+Step 980/44000 | Loss: 4.3478 | LR: 5.00e-04 | 677.2ms/step | 29.5 samples/s
+Step 990/44000 | Loss: 4.3144 | LR: 5.00e-04 | 678.0ms/step | 29.5 samples/s
+Step 1000/44000 | Loss: 4.3165 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Validation at step 1000: Loss = 4.3113, Perplexity = 74.54
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-1000
+Step 1010/44000 | Loss: 4.3391 | LR: 5.00e-04 | 1524.7ms/step | 13.1 samples/s
+Step 1020/44000 | Loss: 4.2807 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1030/44000 | Loss: 4.2830 | LR: 5.00e-04 | 678.1ms/step | 29.5 samples/s
+Step 1040/44000 | Loss: 4.2130 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s
+Step 1050/44000 | Loss: 4.2590 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s
+Step 1060/44000 | Loss: 4.2902 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s
+Step 1070/44000 | Loss: 4.2019 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1080/44000 | Loss: 4.2037 | LR: 5.00e-04 | 680.9ms/step | 29.4 samples/s
+Step 1090/44000 | Loss: 4.2202 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s
+Step 1100/44000 | Loss: 4.1972 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1110/44000 | Loss: 4.2082 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1120/44000 | Loss: 4.1097 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1130/44000 | Loss: 4.1362 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 1140/44000 | Loss: 4.1240 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s
+Step 1150/44000 | Loss: 4.1284 | LR: 5.00e-04 | 678.9ms/step | 29.5 samples/s
+Step 1160/44000 | Loss: 4.1355 | LR: 5.00e-04 | 679.7ms/step | 29.4 samples/s
+Step 1170/44000 | Loss: 4.1230 | LR: 5.00e-04 | 679.1ms/step | 29.5 samples/s
+Step 1180/44000 | Loss: 4.1392 | LR: 5.00e-04 | 681.1ms/step | 29.4 samples/s
+Step 1190/44000 | Loss: 4.0935 | LR: 5.00e-04 | 679.1ms/step | 29.4 samples/s
+Step 1200/44000 | Loss: 4.0989 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1210/44000 | Loss: 4.1222 | LR: 5.00e-04 | 679.0ms/step | 29.5 samples/s
+Step 1220/44000 | Loss: 4.1045 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s
+Step 1230/44000 | Loss: 4.0698 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s
+Step 1240/44000 | Loss: 4.0740 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s
+Step 1250/44000 | Loss: 4.0511 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s
+Validation at step 1250: Loss = 4.0719, Perplexity = 58.67
+Step 1260/44000 | Loss: 4.1055 | LR: 5.00e-04 | 1235.6ms/step | 16.2 samples/s
+Step 1270/44000 | Loss: 4.0299 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 1280/44000 | Loss: 4.0458 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1290/44000 | Loss: 4.0374 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1300/44000 | Loss: 4.0425 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s
+Step 1310/44000 | Loss: 3.9859 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1320/44000 | Loss: 3.9726 | LR: 5.00e-04 | 679.5ms/step | 29.4 samples/s
+Step 1330/44000 | Loss: 3.9942 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Step 1340/44000 | Loss: 4.0381 | LR: 5.00e-04 | 680.3ms/step | 29.4 samples/s
+Step 1350/44000 | Loss: 3.9667 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s
+Step 1360/44000 | Loss: 3.9881 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s
+Step 1370/44000 | Loss: 3.9616 | LR: 5.00e-04 | 679.1ms/step | 29.4 samples/s
+Step 1380/44000 | Loss: 3.9808 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Step 1390/44000 | Loss: 3.9663 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1400/44000 | Loss: 3.9699 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 1410/44000 | Loss: 3.9459 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 1420/44000 | Loss: 3.9498 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s
+Step 1430/44000 | Loss: 3.9607 | LR: 5.00e-04 | 677.7ms/step | 29.5 samples/s
+Step 1440/44000 | Loss: 3.8915 | LR: 5.00e-04 | 678.4ms/step | 29.5 samples/s
+Step 1450/44000 | Loss: 3.9141 | LR: 5.00e-04 | 678.3ms/step | 29.5 samples/s
+Step 1460/44000 | Loss: 3.9595 | LR: 5.00e-04 | 678.8ms/step | 29.5 samples/s
+Step 1470/44000 | Loss: 3.9038 | LR: 5.00e-04 | 679.2ms/step | 29.4 samples/s
+Step 1480/44000 | Loss: 3.8795 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s
+Step 1490/44000 | Loss: 3.8771 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 1500/44000 | Loss: 3.9326 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s
+Validation at step 1500: Loss = 3.9134, Perplexity = 50.07
+Step 1510/44000 | Loss: 3.9028 | LR: 5.00e-04 | 1233.6ms/step | 16.2 samples/s
+Step 1520/44000 | Loss: 3.8939 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s
+Step 1530/44000 | Loss: 3.9195 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 1540/44000 | Loss: 3.8397 | LR: 5.00e-04 | 680.2ms/step | 29.4 samples/s
+Step 1550/44000 | Loss: 3.9178 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1560/44000 | Loss: 3.8780 | LR: 5.00e-04 | 678.8ms/step | 29.5 samples/s
+Step 1570/44000 | Loss: 3.8966 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1580/44000 | Loss: 3.8516 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1590/44000 | Loss: 3.8534 | LR: 5.00e-04 | 678.3ms/step | 29.5 samples/s
+Step 1600/44000 | Loss: 3.8467 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s
+Step 1610/44000 | Loss: 3.8594 | LR: 5.00e-04 | 679.7ms/step | 29.4 samples/s
+Step 1620/44000 | Loss: 3.8355 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1630/44000 | Loss: 3.8507 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s
+Step 1640/44000 | Loss: 3.8121 | LR: 4.99e-04 | 679.5ms/step | 29.4 samples/s
+Step 1650/44000 | Loss: 3.8513 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s
+Step 1660/44000 | Loss: 3.8623 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s
+Step 1670/44000 | Loss: 3.8700 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s
+Step 1680/44000 | Loss: 3.8120 | LR: 4.99e-04 | 678.2ms/step | 29.5 samples/s
+Step 1690/44000 | Loss: 3.7932 | LR: 4.99e-04 | 678.1ms/step | 29.5 samples/s
+Step 1700/44000 | Loss: 3.8350 | LR: 4.99e-04 | 679.8ms/step | 29.4 samples/s
+Step 1710/44000 | Loss: 3.7861 | LR: 4.99e-04 | 679.7ms/step | 29.4 samples/s
+Step 1720/44000 | Loss: 3.8389 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s
+Step 1730/44000 | Loss: 3.8071 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s
+Step 1740/44000 | Loss: 3.7915 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s
+Step 1750/44000 | Loss: 3.7694 | LR: 4.99e-04 | 678.9ms/step | 29.5 samples/s
+Validation at step 1750: Loss = 3.8036, Perplexity = 44.86
+Step 1760/44000 | Loss: 3.8084 | LR: 4.99e-04 | 1234.0ms/step | 16.2 samples/s
+Step 1770/44000 | Loss: 3.8108 | LR: 4.99e-04 | 679.8ms/step | 29.4 samples/s
+Step 1780/44000 | Loss: 3.8069 | LR: 4.99e-04 | 679.3ms/step | 29.4 samples/s
+Step 1790/44000 | Loss: 3.7994 | LR: 4.99e-04 | 679.4ms/step | 29.4 samples/s
+Step 1800/44000 | Loss: 3.7784 | LR: 4.99e-04 | 680.0ms/step | 29.4 samples/s
+Step 1810/44000 | Loss: 3.7567 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s
+Step 1820/44000 | Loss: 3.7668 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s
+Step 1830/44000 | Loss: 3.7511 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s
+Step 1840/44000 | Loss: 3.7417 | LR: 4.99e-04 | 678.8ms/step | 29.5 samples/s
+Step 1850/44000 | Loss: 3.7737 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s
+Step 1860/44000 | Loss: 3.8001 | LR: 4.99e-04 | 678.0ms/step | 29.5 samples/s
+Step 1870/44000 | Loss: 3.7855 | LR: 4.99e-04 | 678.1ms/step | 29.5 samples/s
+Step 1880/44000 | Loss: 3.7448 | LR: 4.99e-04 | 680.2ms/step | 29.4 samples/s
+Step 1890/44000 | Loss: 3.7230 | LR: 4.99e-04 | 678.4ms/step | 29.5 samples/s
+Step 1900/44000 | Loss: 3.7565 | LR: 4.99e-04 | 679.6ms/step | 29.4 samples/s
+Step 1910/44000 | Loss: 3.7632 | LR: 4.99e-04 | 680.1ms/step | 29.4 samples/s
+Step 1920/44000 | Loss: 3.7395 | LR: 4.99e-04 | 679.9ms/step | 29.4 samples/s
+Step 1930/44000 | Loss: 3.7510 | LR: 4.99e-04 | 679.1ms/step | 29.5 samples/s
+Step 1940/44000 | Loss: 3.7140 | LR: 4.99e-04 | 679.0ms/step | 29.5 samples/s
+Step 1950/44000 | Loss: 3.6843 | LR: 4.99e-04 | 679.7ms/step | 29.4 samples/s
+Step 1960/44000 | Loss: 3.7313 | LR: 4.99e-04 | 680.3ms/step | 29.4 samples/s
+Step 1970/44000 | Loss: 3.6992 | LR: 4.99e-04 | 679.1ms/step | 29.5 samples/s
+Step 1980/44000 | Loss: 3.6899 | LR: 4.99e-04 | 679.1ms/step | 29.5 samples/s
+Step 1990/44000 | Loss: 3.7579 | LR: 4.99e-04 | 679.2ms/step | 29.4 samples/s
+Step 2000/44000 | Loss: 3.7184 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s
+Validation at step 2000: Loss = 3.7220, Perplexity = 41.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-2000
+Step 2010/44000 | Loss: 3.7271 | LR: 4.99e-04 | 1526.4ms/step | 13.1 samples/s
+Step 2020/44000 | Loss: 3.7128 | LR: 4.99e-04 | 679.4ms/step | 29.4 samples/s
+Step 2030/44000 | Loss: 3.6913 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s
+Step 2040/44000 | Loss: 3.7242 | LR: 4.99e-04 | 676.4ms/step | 29.6 samples/s
+Step 2050/44000 | Loss: 3.7057 | LR: 4.99e-04 | 677.0ms/step | 29.5 samples/s
+Step 2060/44000 | Loss: 3.7248 | LR: 4.99e-04 | 676.5ms/step | 29.6 samples/s
+Step 2070/44000 | Loss: 3.7039 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s
+Step 2080/44000 | Loss: 3.7238 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s
+Step 2090/44000 | Loss: 3.7226 | LR: 4.99e-04 | 677.6ms/step | 29.5 samples/s
+Step 2100/44000 | Loss: 3.6758 | LR: 4.99e-04 | 678.8ms/step | 29.5 samples/s
+Step 2110/44000 | Loss: 3.7050 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s
+Step 2120/44000 | Loss: 3.6907 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s
+Step 2130/44000 | Loss: 3.6842 | LR: 4.99e-04 | 679.1ms/step | 29.4 samples/s
+Step 2140/44000 | Loss: 3.6678 | LR: 4.99e-04 | 680.2ms/step | 29.4 samples/s
+Step 2150/44000 | Loss: 3.6774 | LR: 4.99e-04 | 679.5ms/step | 29.4 samples/s
+Step 2160/44000 | Loss: 3.6910 | LR: 4.99e-04 | 678.2ms/step | 29.5 samples/s
+Step 2170/44000 | Loss: 3.6656 | LR: 4.99e-04 | 677.4ms/step | 29.5 samples/s
+Step 2180/44000 | Loss: 3.6832 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s
+Step 2190/44000 | Loss: 3.6646 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s
+Step 2200/44000 | Loss: 3.6518 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s
+Step 2210/44000 | Loss: 3.6337 | LR: 4.99e-04 | 681.1ms/step | 29.4 samples/s
+Step 2220/44000 | Loss: 3.6290 | LR: 4.99e-04 | 679.5ms/step | 29.4 samples/s
+Step 2230/44000 | Loss: 3.6832 | LR: 4.99e-04 | 679.4ms/step | 29.4 samples/s
+Step 2240/44000 | Loss: 3.6783 | LR: 4.99e-04 | 678.9ms/step | 29.5 samples/s
+Step 2250/44000 | Loss: 3.6675 | LR: 4.99e-04 | 679.2ms/step | 29.4 samples/s
+Validation at step 2250: Loss = 3.6566, Perplexity = 38.73
+Step 2260/44000 | Loss: 3.6536 | LR: 4.98e-04 | 1235.3ms/step | 16.2 samples/s
+Step 2270/44000 | Loss: 3.6374 | LR: 4.98e-04 | 678.3ms/step | 29.5 samples/s
+Step 2280/44000 | Loss: 3.6526 | LR: 4.98e-04 | 684.8ms/step | 29.2 samples/s
+Step 2290/44000 | Loss: 3.6724 | LR: 4.98e-04 | 679.6ms/step | 29.4 samples/s
+Step 2300/44000 | Loss: 3.6020 | LR: 4.98e-04 | 678.9ms/step | 29.5 samples/s
+Step 2310/44000 | Loss: 3.6434 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Step 2320/44000 | Loss: 3.6527 | LR: 4.98e-04 | 680.1ms/step | 29.4 samples/s
+Step 2330/44000 | Loss: 3.6444 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Step 2340/44000 | Loss: 3.6432 | LR: 4.98e-04 | 679.7ms/step | 29.4 samples/s
+Step 2350/44000 | Loss: 3.6070 | LR: 4.98e-04 | 679.9ms/step | 29.4 samples/s
+Step 2360/44000 | Loss: 3.6312 | LR: 4.98e-04 | 678.6ms/step | 29.5 samples/s
+Step 2370/44000 | Loss: 3.6329 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s
+Step 2380/44000 | Loss: 3.5914 | LR: 4.98e-04 | 679.4ms/step | 29.4 samples/s
+Step 2390/44000 | Loss: 3.5777 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s
+Step 2400/44000 | Loss: 3.6346 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Step 2410/44000 | Loss: 3.6268 | LR: 4.98e-04 | 680.8ms/step | 29.4 samples/s
+Step 2420/44000 | Loss: 3.6248 | LR: 4.98e-04 | 678.7ms/step | 29.5 samples/s
+Step 2430/44000 | Loss: 3.6167 | LR: 4.98e-04 | 680.8ms/step | 29.4 samples/s
+Step 2440/44000 | Loss: 3.6231 | LR: 4.98e-04 | 679.7ms/step | 29.4 samples/s
+Step 2450/44000 | Loss: 3.5986 | LR: 4.98e-04 | 680.0ms/step | 29.4 samples/s
+Step 2460/44000 | Loss: 3.6018 | LR: 4.98e-04 | 679.5ms/step | 29.4 samples/s
+Step 2470/44000 | Loss: 3.6084 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s
+Step 2480/44000 | Loss: 3.6210 | LR: 4.98e-04 | 680.5ms/step | 29.4 samples/s
+Step 2490/44000 | Loss: 3.5856 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Step 2500/44000 | Loss: 3.5987 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Validation at step 2500: Loss = 3.6030, Perplexity = 36.71
+Step 2510/44000 | Loss: 3.6005 | LR: 4.98e-04 | 1235.9ms/step | 16.2 samples/s
+Step 2520/44000 | Loss: 3.5890 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s
+Step 2530/44000 | Loss: 3.6016 | LR: 4.98e-04 | 680.3ms/step | 29.4 samples/s
+Step 2540/44000 | Loss: 3.5777 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s
+Step 2550/44000 | Loss: 3.5435 | LR: 4.98e-04 | 677.7ms/step | 29.5 samples/s
+Step 2560/44000 | Loss: 3.5676 | LR: 4.98e-04 | 680.0ms/step | 29.4 samples/s
+Step 2570/44000 | Loss: 3.5774 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s
+Step 2580/44000 | Loss: 3.5951 | LR: 4.98e-04 | 680.0ms/step | 29.4 samples/s
+Step 2590/44000 | Loss: 3.5799 | LR: 4.98e-04 | 679.3ms/step | 29.4 samples/s
+Step 2600/44000 | Loss: 3.5484 | LR: 4.98e-04 | 678.9ms/step | 29.5 samples/s
+Step 2610/44000 | Loss: 3.5159 | LR: 4.98e-04 | 679.9ms/step | 29.4 samples/s
+Step 2620/44000 | Loss: 3.5791 | LR: 4.98e-04 | 679.9ms/step | 29.4 samples/s
+Step 2630/44000 | Loss: 3.5372 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Step 2640/44000 | Loss: 3.5582 | LR: 4.98e-04 | 679.5ms/step | 29.4 samples/s
+Step 2650/44000 | Loss: 3.5712 | LR: 4.98e-04 | 679.3ms/step | 29.4 samples/s
+Step 2660/44000 | Loss: 3.5420 | LR: 4.98e-04 | 680.6ms/step | 29.4 samples/s
+Step 2670/44000 | Loss: 3.5468 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s
+Step 2680/44000 | Loss: 3.5787 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s
+Step 2690/44000 | Loss: 3.5915 | LR: 4.98e-04 | 679.5ms/step | 29.4 samples/s
+Step 2700/44000 | Loss: 3.5287 | LR: 4.97e-04 | 680.3ms/step | 29.4 samples/s
+Step 2710/44000 | Loss: 3.5807 | LR: 4.97e-04 | 679.9ms/step | 29.4 samples/s
+Step 2720/44000 | Loss: 3.5960 | LR: 4.97e-04 | 680.0ms/step | 29.4 samples/s
+Step 2730/44000 | Loss: 3.5774 | LR: 4.97e-04 | 680.2ms/step | 29.4 samples/s
+Step 2740/44000 | Loss: 3.5472 | LR: 4.97e-04 | 680.0ms/step | 29.4 samples/s
+Step 2750/44000 | Loss: 3.5272 | LR: 4.97e-04 | 678.7ms/step | 29.5 samples/s
+Validation at step 2750: Loss = 3.5600, Perplexity = 35.16
+Step 2760/44000 | Loss: 3.5219 | LR: 4.97e-04 | 1235.3ms/step | 16.2 samples/s
+Step 2770/44000 | Loss: 3.5977 | LR: 4.97e-04 | 679.5ms/step | 29.4 samples/s
+Step 2780/44000 | Loss: 3.5897 | LR: 4.97e-04 | 679.2ms/step | 29.4 samples/s
+Step 2790/44000 | Loss: 3.5675 | LR: 4.97e-04 | 679.6ms/step | 29.4 samples/s
+Step 2800/44000 | Loss: 3.5683 | LR: 4.97e-04 | 679.1ms/step | 29.4 samples/s
+Step 2810/44000 | Loss: 3.5227 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s
+Step 2820/44000 | Loss: 3.5505 | LR: 4.97e-04 | 678.3ms/step | 29.5 samples/s
+Step 2830/44000 | Loss: 3.5359 | LR: 4.97e-04 | 678.1ms/step | 29.5 samples/s
+Step 2840/44000 | Loss: 3.5107 | LR: 4.97e-04 | 680.1ms/step | 29.4 samples/s
+Step 2850/44000 | Loss: 3.5808 | LR: 4.97e-04 | 679.7ms/step | 29.4 samples/s
+Step 2860/44000 | Loss: 3.5556 | LR: 4.97e-04 | 679.6ms/step | 29.4 samples/s
+Step 2870/44000 | Loss: 3.5299 | LR: 4.97e-04 | 681.9ms/step | 29.3 samples/s
+Step 2880/44000 | Loss: 3.4852 | LR: 4.97e-04 | 678.8ms/step | 29.5 samples/s
+Step 2890/44000 | Loss: 3.5212 | LR: 4.97e-04 | 679.0ms/step | 29.5 samples/s
+Step 2900/44000 | Loss: 3.5748 | LR: 4.97e-04 | 679.8ms/step | 29.4 samples/s
+Step 2910/44000 | Loss: 3.5205 | LR: 4.97e-04 | 679.5ms/step | 29.4 samples/s
+Step 2920/44000 | Loss: 3.5393 | LR: 4.97e-04 | 681.4ms/step | 29.4 samples/s
+Step 2930/44000 | Loss: 3.5234 | LR: 4.97e-04 | 681.2ms/step | 29.4 samples/s
+Step 2940/44000 | Loss: 3.5169 | LR: 4.97e-04 | 679.8ms/step | 29.4 samples/s
+Step 2950/44000 | Loss: 3.5483 | LR: 4.97e-04 | 679.1ms/step | 29.5 samples/s
+Step 2960/44000 | Loss: 3.5279 | LR: 4.97e-04 | 679.4ms/step | 29.4 samples/s
+Step 2970/44000 | Loss: 3.5482 | LR: 4.97e-04 | 679.9ms/step | 29.4 samples/s
+Step 2980/44000 | Loss: 3.6133 | LR: 4.97e-04 | 679.4ms/step | 29.4 samples/s
+Step 2990/44000 | Loss: 3.5098 | LR: 4.97e-04 | 679.9ms/step | 29.4 samples/s
+Step 3000/44000 | Loss: 3.5279 | LR: 4.97e-04 | 678.3ms/step | 29.5 samples/s
+Validation at step 3000: Loss = 3.5247, Perplexity = 33.94
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-3000
+Step 3010/44000 | Loss: 3.5169 | LR: 4.97e-04 | 1523.9ms/step | 13.1 samples/s
+Step 3020/44000 | Loss: 3.5089 | LR: 4.97e-04 | 679.6ms/step | 29.4 samples/s
+Step 3030/44000 | Loss: 3.5554 | LR: 4.97e-04 | 679.5ms/step | 29.4 samples/s
+Step 3040/44000 | Loss: 3.5123 | LR: 4.97e-04 | 679.2ms/step | 29.4 samples/s
+Step 3050/44000 | Loss: 3.5534 | LR: 4.97e-04 | 678.6ms/step | 29.5 samples/s
+Step 3060/44000 | Loss: 3.4931 | LR: 4.96e-04 | 678.3ms/step | 29.5 samples/s
+Step 3070/44000 | Loss: 3.5219 | LR: 4.96e-04 | 679.7ms/step | 29.4 samples/s
+Step 3080/44000 | Loss: 3.5287 | LR: 4.96e-04 | 680.1ms/step | 29.4 samples/s
+Step 3090/44000 | Loss: 3.4914 | LR: 4.96e-04 | 680.2ms/step | 29.4 samples/s
+Step 3100/44000 | Loss: 3.5144 | LR: 4.96e-04 | 680.0ms/step | 29.4 samples/s
+Step 3110/44000 | Loss: 3.4814 | LR: 4.96e-04 | 679.9ms/step | 29.4 samples/s
+Step 3120/44000 | Loss: 3.4979 | LR: 4.96e-04 | 679.5ms/step | 29.4 samples/s
+Step 3130/44000 | Loss: 3.4916 | LR: 4.96e-04 | 679.8ms/step | 29.4 samples/s
+Step 3140/44000 | Loss: 3.4857 | LR: 4.96e-04 | 678.6ms/step | 29.5 samples/s
+Step 3150/44000 | Loss: 3.4805 | LR: 4.96e-04 | 680.9ms/step | 29.4 samples/s
+Step 3160/44000 | Loss: 3.5001 | LR: 4.96e-04 | 679.5ms/step | 29.4 samples/s
+Step 3170/44000 | Loss: 3.4908 | LR: 4.96e-04 | 679.1ms/step | 29.4 samples/s
+Step 3180/44000 | Loss: 3.5278 | LR: 4.96e-04 | 678.6ms/step | 29.5 samples/s
+Step 3190/44000 | Loss: 3.5230 | LR: 4.96e-04 | 679.3ms/step | 29.4 samples/s
+Step 3200/44000 | Loss: 3.4853 | LR: 4.96e-04 | 679.8ms/step | 29.4 samples/s
+Step 3210/44000 | Loss: 3.4717 | LR: 4.96e-04 | 679.9ms/step | 29.4 samples/s
+Step 3220/44000 | Loss: 3.5071 | LR: 4.96e-04 | 679.2ms/step | 29.4 samples/s
+Step 3230/44000 | Loss: 3.4747 | LR: 4.96e-04 | 679.4ms/step | 29.4 samples/s
+Step 3240/44000 | Loss: 3.4899 | LR: 4.96e-04 | 678.4ms/step | 29.5 samples/s
+Step 3250/44000 | Loss: 3.4701 | LR: 4.96e-04 | 678.3ms/step | 29.5 samples/s
+Validation at step 3250: Loss = 3.4909, Perplexity = 32.82
+Step 3260/44000 | Loss: 3.5190 | LR: 4.96e-04 | 1243.6ms/step | 16.1 samples/s
+Step 3270/44000 | Loss: 3.4723 | LR: 4.96e-04 | 681.4ms/step | 29.4 samples/s
+Step 3280/44000 | Loss: 3.4731 | LR: 4.96e-04 | 680.0ms/step | 29.4 samples/s
+Step 3290/44000 | Loss: 3.4796 | LR: 4.96e-04 | 679.9ms/step | 29.4 samples/s
+Step 3300/44000 | Loss: 3.4657 | LR: 4.96e-04 | 679.5ms/step | 29.4 samples/s
+Step 3310/44000 | Loss: 3.4818 | LR: 4.96e-04 | 678.8ms/step | 29.5 samples/s
+Step 3320/44000 | Loss: 3.4657 | LR: 4.96e-04 | 681.1ms/step | 29.4 samples/s
+Step 3330/44000 | Loss: 3.4865 | LR: 4.96e-04 | 679.2ms/step | 29.4 samples/s
+Step 3340/44000 | Loss: 3.4610 | LR: 4.96e-04 | 679.4ms/step | 29.4 samples/s
+Step 3350/44000 | Loss: 3.4906 | LR: 4.96e-04 | 679.6ms/step | 29.4 samples/s
+Step 3360/44000 | Loss: 3.4482 | LR: 4.96e-04 | 679.8ms/step | 29.4 samples/s
+Step 3370/44000 | Loss: 3.4783 | LR: 4.95e-04 | 680.1ms/step | 29.4 samples/s
+Step 3380/44000 | Loss: 3.4658 | LR: 4.95e-04 | 678.9ms/step | 29.5 samples/s
+Step 3390/44000 | Loss: 3.4788 | LR: 4.95e-04 | 679.5ms/step | 29.4 samples/s
+Step 3400/44000 | Loss: 3.4584 | LR: 4.95e-04 | 680.0ms/step | 29.4 samples/s
+Step 3410/44000 | Loss: 3.4768 | LR: 4.95e-04 | 679.1ms/step | 29.4 samples/s
+Step 3420/44000 | Loss: 3.3836 | LR: 4.95e-04 | 680.6ms/step | 29.4 samples/s
+Step 3430/44000 | Loss: 3.4377 | LR: 4.95e-04 | 680.8ms/step | 29.4 samples/s
+Step 3440/44000 | Loss: 3.4689 | LR: 4.95e-04 | 678.5ms/step | 29.5 samples/s
+Step 3450/44000 | Loss: 3.4345 | LR: 4.95e-04 | 678.8ms/step | 29.5 samples/s
+Step 3460/44000 | Loss: 3.4805 | LR: 4.95e-04 | 678.5ms/step | 29.5 samples/s
+Step 3470/44000 | Loss: 3.4827 | LR: 4.95e-04 | 680.1ms/step | 29.4 samples/s
+Step 3480/44000 | Loss: 3.4605 | LR: 4.95e-04 | 680.3ms/step | 29.4 samples/s
+Step 3490/44000 | Loss: 3.4687 | LR: 4.95e-04 | 679.3ms/step | 29.4 samples/s
+Step 3500/44000 | Loss: 3.4217 | LR: 4.95e-04 | 680.3ms/step | 29.4 samples/s
+Validation at step 3500: Loss = 3.4627, Perplexity = 31.90
+Step 3510/44000 | Loss: 3.4708 | LR: 4.95e-04 | 1234.0ms/step | 16.2 samples/s
+Step 3520/44000 | Loss: 3.4309 | LR: 4.95e-04 | 679.2ms/step | 29.4 samples/s
+Step 3530/44000 | Loss: 3.4098 | LR: 4.95e-04 | 679.4ms/step | 29.4 samples/s
+Step 3540/44000 | Loss: 3.4374 | LR: 4.95e-04 | 679.3ms/step | 29.4 samples/s
+Step 3550/44000 | Loss: 3.4533 | LR: 4.95e-04 | 680.2ms/step | 29.4 samples/s
+Step 3560/44000 | Loss: 3.4913 | LR: 4.95e-04 | 680.6ms/step | 29.4 samples/s
+Step 3570/44000 | Loss: 3.4153 | LR: 4.95e-04 | 680.2ms/step | 29.4 samples/s
+Step 3580/44000 | Loss: 3.4416 | LR: 4.95e-04 | 678.9ms/step | 29.5 samples/s
+Step 3590/44000 | Loss: 3.4475 | LR: 4.95e-04 | 678.5ms/step | 29.5 samples/s
+Step 3600/44000 | Loss: 3.4678 | LR: 4.95e-04 | 678.7ms/step | 29.5 samples/s
+Step 3610/44000 | Loss: 3.4923 | LR: 4.95e-04 | 681.2ms/step | 29.4 samples/s
+Step 3620/44000 | Loss: 3.4472 | LR: 4.95e-04 | 678.6ms/step | 29.5 samples/s
+Step 3630/44000 | Loss: 3.4275 | LR: 4.95e-04 | 678.9ms/step | 29.5 samples/s
+Step 3640/44000 | Loss: 3.4488 | LR: 4.95e-04 | 679.7ms/step | 29.4 samples/s
+Step 3650/44000 | Loss: 3.4783 | LR: 4.94e-04 | 679.3ms/step | 29.4 samples/s
+Step 3660/44000 | Loss: 3.4170 | LR: 4.94e-04 | 678.3ms/step | 29.5 samples/s
+Step 3670/44000 | Loss: 3.4606 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s
+Step 3680/44000 | Loss: 3.4019 | LR: 4.94e-04 | 678.2ms/step | 29.5 samples/s
+Step 3690/44000 | Loss: 3.4091 | LR: 4.94e-04 | 679.1ms/step | 29.4 samples/s
+Step 3700/44000 | Loss: 3.4554 | LR: 4.94e-04 | 678.5ms/step | 29.5 samples/s
+Step 3710/44000 | Loss: 3.4502 | LR: 4.94e-04 | 679.8ms/step | 29.4 samples/s
+Step 3720/44000 | Loss: 3.4324 | LR: 4.94e-04 | 679.4ms/step | 29.4 samples/s
+Step 3730/44000 | Loss: 3.4059 | LR: 4.94e-04 | 680.7ms/step | 29.4 samples/s
+Step 3740/44000 | Loss: 3.4658 | LR: 4.94e-04 | 679.9ms/step | 29.4 samples/s
+Step 3750/44000 | Loss: 3.4212 | LR: 4.94e-04 | 682.5ms/step | 29.3 samples/s
+Validation at step 3750: Loss = 3.4375, Perplexity = 31.11
+Step 3760/44000 | Loss: 3.4491 | LR: 4.94e-04 | 1234.2ms/step | 16.2 samples/s
+Step 3770/44000 | Loss: 3.4293 | LR: 4.94e-04 | 679.2ms/step | 29.4 samples/s
+Step 3780/44000 | Loss: 3.4265 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s
+Step 3790/44000 | Loss: 3.4338 | LR: 4.94e-04 | 680.9ms/step | 29.4 samples/s
+Step 3800/44000 | Loss: 3.4025 | LR: 4.94e-04 | 680.3ms/step | 29.4 samples/s
+Step 3810/44000 | Loss: 3.3858 | LR: 4.94e-04 | 680.1ms/step | 29.4 samples/s
+Step 3820/44000 | Loss: 3.3710 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s
+Step 3830/44000 | Loss: 3.4053 | LR: 4.94e-04 | 680.0ms/step | 29.4 samples/s
+Step 3840/44000 | Loss: 3.4694 | LR: 4.94e-04 | 679.6ms/step | 29.4 samples/s
+Step 3850/44000 | Loss: 3.4273 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s
+Step 3860/44000 | Loss: 3.4149 | LR: 4.94e-04 | 679.7ms/step | 29.4 samples/s
+Step 3870/44000 | Loss: 3.4650 | LR: 4.94e-04 | 678.9ms/step | 29.5 samples/s
+Step 3880/44000 | Loss: 3.4368 | LR: 4.94e-04 | 678.9ms/step | 29.5 samples/s
+Step 3890/44000 | Loss: 3.4628 | LR: 4.94e-04 | 681.1ms/step | 29.4 samples/s
+Step 3900/44000 | Loss: 3.4075 | LR: 4.93e-04 | 679.9ms/step | 29.4 samples/s
+Step 3910/44000 | Loss: 3.4479 | LR: 4.93e-04 | 677.4ms/step | 29.5 samples/s
+Step 3920/44000 | Loss: 3.4100 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s
+Step 3930/44000 | Loss: 3.4134 | LR: 4.93e-04 | 679.7ms/step | 29.4 samples/s
+Step 3940/44000 | Loss: 3.4183 | LR: 4.93e-04 | 680.4ms/step | 29.4 samples/s
+Step 3950/44000 | Loss: 3.3578 | LR: 4.93e-04 | 681.2ms/step | 29.4 samples/s
+Step 3960/44000 | Loss: 3.4280 | LR: 4.93e-04 | 680.6ms/step | 29.4 samples/s
+Step 3970/44000 | Loss: 3.4507 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s
+Step 3980/44000 | Loss: 3.3956 | LR: 4.93e-04 | 679.0ms/step | 29.5 samples/s
+Step 3990/44000 | Loss: 3.3910 | LR: 4.93e-04 | 680.8ms/step | 29.4 samples/s
+Step 4000/44000 | Loss: 3.4093 | LR: 4.93e-04 | 679.1ms/step | 29.4 samples/s
+Validation at step 4000: Loss = 3.4173, Perplexity = 30.49
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-4000
+Step 4010/44000 | Loss: 3.4274 | LR: 4.93e-04 | 1541.5ms/step | 13.0 samples/s
+Step 4020/44000 | Loss: 3.4592 | LR: 4.93e-04 | 680.5ms/step | 29.4 samples/s
+Step 4030/44000 | Loss: 3.3700 | LR: 4.93e-04 | 679.1ms/step | 29.5 samples/s
+Step 4040/44000 | Loss: 3.3770 | LR: 4.93e-04 | 679.2ms/step | 29.4 samples/s
+Step 4050/44000 | Loss: 3.3930 | LR: 4.93e-04 | 677.3ms/step | 29.5 samples/s
+Step 4060/44000 | Loss: 3.3864 | LR: 4.93e-04 | 678.8ms/step | 29.5 samples/s
+Step 4070/44000 | Loss: 3.4125 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s
+Step 4080/44000 | Loss: 3.4098 | LR: 4.93e-04 | 681.7ms/step | 29.3 samples/s
+Step 4090/44000 | Loss: 3.4028 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s
+Step 4100/44000 | Loss: 3.3925 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s
+Step 4110/44000 | Loss: 3.4158 | LR: 4.93e-04 | 679.5ms/step | 29.4 samples/s
+Step 4120/44000 | Loss: 3.4153 | LR: 4.93e-04 | 680.1ms/step | 29.4 samples/s
+Step 4130/44000 | Loss: 3.4069 | LR: 4.93e-04 | 680.1ms/step | 29.4 samples/s
+Step 4140/44000 | Loss: 3.4074 | LR: 4.92e-04 | 680.0ms/step | 29.4 samples/s
+Step 4150/44000 | Loss: 3.3860 | LR: 4.92e-04 | 678.3ms/step | 29.5 samples/s
+Step 4160/44000 | Loss: 3.4024 | LR: 4.92e-04 | 677.9ms/step | 29.5 samples/s
+Step 4170/44000 | Loss: 3.3716 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Step 4180/44000 | Loss: 3.3577 | LR: 4.92e-04 | 679.4ms/step | 29.4 samples/s
+Step 4190/44000 | Loss: 3.4122 | LR: 4.92e-04 | 679.2ms/step | 29.4 samples/s
+Step 4200/44000 | Loss: 3.3779 | LR: 4.92e-04 | 679.2ms/step | 29.4 samples/s
+Step 4210/44000 | Loss: 3.4211 | LR: 4.92e-04 | 679.9ms/step | 29.4 samples/s
+Step 4220/44000 | Loss: 3.3901 | LR: 4.92e-04 | 680.5ms/step | 29.4 samples/s
+Step 4230/44000 | Loss: 3.4316 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Step 4240/44000 | Loss: 3.3808 | LR: 4.92e-04 | 680.2ms/step | 29.4 samples/s
+Step 4250/44000 | Loss: 3.4069 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Validation at step 4250: Loss = 3.3935, Perplexity = 29.77
+Step 4260/44000 | Loss: 3.3471 | LR: 4.92e-04 | 1234.2ms/step | 16.2 samples/s
+Step 4270/44000 | Loss: 3.3606 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s
+Step 4280/44000 | Loss: 3.4021 | LR: 4.92e-04 | 679.5ms/step | 29.4 samples/s
+Step 4290/44000 | Loss: 3.3672 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Step 4300/44000 | Loss: 3.4024 | LR: 4.92e-04 | 678.4ms/step | 29.5 samples/s
+Step 4310/44000 | Loss: 3.3616 | LR: 4.92e-04 | 679.2ms/step | 29.4 samples/s
+Step 4320/44000 | Loss: 3.3728 | LR: 4.92e-04 | 680.0ms/step | 29.4 samples/s
+Step 4330/44000 | Loss: 3.3557 | LR: 4.92e-04 | 677.8ms/step | 29.5 samples/s
+Step 4340/44000 | Loss: 3.3848 | LR: 4.92e-04 | 679.7ms/step | 29.4 samples/s
+Step 4350/44000 | Loss: 3.3405 | LR: 4.92e-04 | 679.4ms/step | 29.4 samples/s
+Step 4360/44000 | Loss: 3.3781 | LR: 4.91e-04 | 678.0ms/step | 29.5 samples/s
+Step 4370/44000 | Loss: 3.3946 | LR: 4.91e-04 | 680.6ms/step | 29.4 samples/s
+Step 4380/44000 | Loss: 3.3441 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s
+Step 4390/44000 | Loss: 3.3535 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s
+Step 4400/44000 | Loss: 3.4129 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s
+Step 4410/44000 | Loss: 3.3832 | LR: 4.91e-04 | 677.8ms/step | 29.5 samples/s
+Step 4420/44000 | Loss: 3.3845 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s
+Step 4430/44000 | Loss: 3.3751 | LR: 4.91e-04 | 679.1ms/step | 29.5 samples/s
+Step 4440/44000 | Loss: 3.3846 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s
+Step 4450/44000 | Loss: 3.3631 | LR: 4.91e-04 | 678.6ms/step | 29.5 samples/s
+Step 4460/44000 | Loss: 3.3365 | LR: 4.91e-04 | 678.0ms/step | 29.5 samples/s
+Step 4470/44000 | Loss: 3.3994 | LR: 4.91e-04 | 679.4ms/step | 29.4 samples/s
+Step 4480/44000 | Loss: 3.3678 | LR: 4.91e-04 | 680.4ms/step | 29.4 samples/s
+Step 4490/44000 | Loss: 3.3312 | LR: 4.91e-04 | 679.9ms/step | 29.4 samples/s
+Step 4500/44000 | Loss: 3.3581 | LR: 4.91e-04 | 679.9ms/step | 29.4 samples/s
+Validation at step 4500: Loss = 3.3756, Perplexity = 29.24
+Step 4510/44000 | Loss: 3.3550 | LR: 4.91e-04 | 1235.7ms/step | 16.2 samples/s
+Step 4520/44000 | Loss: 3.3489 | LR: 4.91e-04 | 680.2ms/step | 29.4 samples/s
+Step 4530/44000 | Loss: 3.3322 | LR: 4.91e-04 | 680.4ms/step | 29.4 samples/s
+Step 4540/44000 | Loss: 3.3758 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s
+Step 4550/44000 | Loss: 3.3705 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s
+Step 4560/44000 | Loss: 3.3866 | LR: 4.90e-04 | 678.9ms/step | 29.5 samples/s
+Step 4570/44000 | Loss: 3.3803 | LR: 4.90e-04 | 679.3ms/step | 29.4 samples/s
+Step 4580/44000 | Loss: 3.3635 | LR: 4.90e-04 | 679.8ms/step | 29.4 samples/s
+Step 4590/44000 | Loss: 3.3132 | LR: 4.90e-04 | 679.9ms/step | 29.4 samples/s
+Step 4600/44000 | Loss: 3.3504 | LR: 4.90e-04 | 679.7ms/step | 29.4 samples/s
+Step 4610/44000 | Loss: 3.3584 | LR: 4.90e-04 | 678.7ms/step | 29.5 samples/s
+Step 4620/44000 | Loss: 3.3225 | LR: 4.90e-04 | 679.3ms/step | 29.4 samples/s
+Step 4630/44000 | Loss: 3.3467 | LR: 4.90e-04 | 678.1ms/step | 29.5 samples/s
+Step 4640/44000 | Loss: 3.3604 | LR: 4.90e-04 | 679.9ms/step | 29.4 samples/s
+Step 4650/44000 | Loss: 3.3247 | LR: 4.90e-04 | 680.5ms/step | 29.4 samples/s
+Step 4660/44000 | Loss: 3.3196 | LR: 4.90e-04 | 679.1ms/step | 29.5 samples/s
+Step 4670/44000 | Loss: 3.3940 | LR: 4.90e-04 | 678.5ms/step | 29.5 samples/s
+Step 4680/44000 | Loss: 3.3543 | LR: 4.90e-04 | 680.0ms/step | 29.4 samples/s
+Step 4690/44000 | Loss: 3.4117 | LR: 4.90e-04 | 678.3ms/step | 29.5 samples/s
+Step 4700/44000 | Loss: 3.3923 | LR: 4.90e-04 | 679.6ms/step | 29.4 samples/s
+Step 4710/44000 | Loss: 3.3810 | LR: 4.90e-04 | 678.0ms/step | 29.5 samples/s
+Step 4720/44000 | Loss: 3.3317 | LR: 4.90e-04 | 678.5ms/step | 29.5 samples/s
+Step 4730/44000 | Loss: 3.3653 | LR: 4.90e-04 | 678.9ms/step | 29.5 samples/s
+Step 4740/44000 | Loss: 3.3559 | LR: 4.90e-04 | 678.2ms/step | 29.5 samples/s
+Step 4750/44000 | Loss: 3.3137 | LR: 4.90e-04 | 678.6ms/step | 29.5 samples/s
+Validation at step 4750: Loss = 3.3607, Perplexity = 28.81
+Step 4760/44000 | Loss: 3.3728 | LR: 4.89e-04 | 1234.3ms/step | 16.2 samples/s
+Step 4770/44000 | Loss: 3.3637 | LR: 4.89e-04 | 678.5ms/step | 29.5 samples/s
+Step 4780/44000 | Loss: 3.3674 | LR: 4.89e-04 | 679.8ms/step | 29.4 samples/s
+Step 4790/44000 | Loss: 3.3470 | LR: 4.89e-04 | 678.9ms/step | 29.5 samples/s
+Step 4800/44000 | Loss: 3.3552 | LR: 4.89e-04 | 679.3ms/step | 29.4 samples/s
+Step 4810/44000 | Loss: 3.3407 | LR: 4.89e-04 | 679.7ms/step | 29.4 samples/s
+Step 4820/44000 | Loss: 3.3467 | LR: 4.89e-04 | 680.5ms/step | 29.4 samples/s
+Step 4830/44000 | Loss: 3.3377 | LR: 4.89e-04 | 678.5ms/step | 29.5 samples/s
+Step 4840/44000 | Loss: 3.3188 | LR: 4.89e-04 | 679.7ms/step | 29.4 samples/s
+Step 4850/44000 | Loss: 3.3334 | LR: 4.89e-04 | 683.2ms/step | 29.3 samples/s
+Step 4860/44000 | Loss: 3.3322 | LR: 4.89e-04 | 681.0ms/step | 29.4 samples/s
+Step 4870/44000 | Loss: 3.3272 | LR: 4.89e-04 | 680.9ms/step | 29.4 samples/s
+Step 4880/44000 | Loss: 3.3476 | LR: 4.89e-04 | 681.2ms/step | 29.4 samples/s
+Step 4890/44000 | Loss: 3.3145 | LR: 4.89e-04 | 680.5ms/step | 29.4 samples/s
+Step 4900/44000 | Loss: 3.3838 | LR: 4.89e-04 | 685.0ms/step | 29.2 samples/s
+Step 4910/44000 | Loss: 3.3637 | LR: 4.89e-04 | 688.5ms/step | 29.1 samples/s
+Step 4920/44000 | Loss: 3.3630 | LR: 4.89e-04 | 689.4ms/step | 29.0 samples/s
+Step 4930/44000 | Loss: 3.3524 | LR: 4.89e-04 | 687.1ms/step | 29.1 samples/s
+Step 4940/44000 | Loss: 3.3524 | LR: 4.89e-04 | 683.6ms/step | 29.3 samples/s
+Step 4950/44000 | Loss: 3.3078 | LR: 4.88e-04 | 683.8ms/step | 29.2 samples/s
+Step 4960/44000 | Loss: 3.3596 | LR: 4.88e-04 | 684.7ms/step | 29.2 samples/s
+Step 4970/44000 | Loss: 3.3109 | LR: 4.88e-04 | 684.1ms/step | 29.2 samples/s
+Step 4980/44000 | Loss: 3.3340 | LR: 4.88e-04 | 683.6ms/step | 29.3 samples/s
+Step 4990/44000 | Loss: 3.3376 | LR: 4.88e-04 | 682.6ms/step | 29.3 samples/s
+Step 5000/44000 | Loss: 3.3362 | LR: 4.88e-04 | 684.5ms/step | 29.2 samples/s
+Validation at step 5000: Loss = 3.3428, Perplexity = 28.30
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-5000
+Step 5010/44000 | Loss: 3.3187 | LR: 4.88e-04 | 1530.9ms/step | 13.1 samples/s
+Step 5020/44000 | Loss: 3.3284 | LR: 4.88e-04 | 683.4ms/step | 29.3 samples/s
+Step 5030/44000 | Loss: 3.3320 | LR: 4.88e-04 | 682.6ms/step | 29.3 samples/s
+Step 5040/44000 | Loss: 3.3599 | LR: 4.88e-04 | 682.4ms/step | 29.3 samples/s
+Step 5050/44000 | Loss: 3.3305 | LR: 4.88e-04 | 681.7ms/step | 29.3 samples/s
+Step 5060/44000 | Loss: 3.3416 | LR: 4.88e-04 | 683.2ms/step | 29.3 samples/s
+Step 5070/44000 | Loss: 3.3174 | LR: 4.88e-04 | 685.3ms/step | 29.2 samples/s
+Step 5080/44000 | Loss: 3.3351 | LR: 4.88e-04 | 684.8ms/step | 29.2 samples/s
+Step 5090/44000 | Loss: 3.3322 | LR: 4.88e-04 | 683.2ms/step | 29.3 samples/s
+Step 5100/44000 | Loss: 3.3526 | LR: 4.88e-04 | 682.2ms/step | 29.3 samples/s
+Step 5110/44000 | Loss: 3.3092 | LR: 4.88e-04 | 684.6ms/step | 29.2 samples/s
+Step 5120/44000 | Loss: 3.3815 | LR: 4.88e-04 | 683.5ms/step | 29.3 samples/s
+Step 5130/44000 | Loss: 3.3388 | LR: 4.87e-04 | 683.6ms/step | 29.3 samples/s
+Step 5140/44000 | Loss: 3.3184 | LR: 4.87e-04 | 683.6ms/step | 29.3 samples/s
+Step 5150/44000 | Loss: 3.3296 | LR: 4.87e-04 | 683.3ms/step | 29.3 samples/s
+Step 5160/44000 | Loss: 3.3362 | LR: 4.87e-04 | 683.3ms/step | 29.3 samples/s
+Step 5170/44000 | Loss: 3.2838 | LR: 4.87e-04 | 684.1ms/step | 29.2 samples/s
+Step 5180/44000 | Loss: 3.3286 | LR: 4.87e-04 | 682.8ms/step | 29.3 samples/s
+Step 5190/44000 | Loss: 3.3495 | LR: 4.87e-04 | 684.0ms/step | 29.2 samples/s
+Step 5200/44000 | Loss: 3.3265 | LR: 4.87e-04 | 682.3ms/step | 29.3 samples/s
+Step 5210/44000 | Loss: 3.3450 | LR: 4.87e-04 | 682.5ms/step | 29.3 samples/s
+Step 5220/44000 | Loss: 3.3372 | LR: 4.87e-04 | 684.2ms/step | 29.2 samples/s
+Step 5230/44000 | Loss: 3.3439 | LR: 4.87e-04 | 683.1ms/step | 29.3 samples/s
+Step 5240/44000 | Loss: 3.3951 | LR: 4.87e-04 | 684.0ms/step | 29.2 samples/s
+Step 5250/44000 | Loss: 3.3348 | LR: 4.87e-04 | 684.0ms/step | 29.2 samples/s
+Validation at step 5250: Loss = 3.3283, Perplexity = 27.89
+Step 5260/44000 | Loss: 3.3578 | LR: 4.87e-04 | 1248.7ms/step | 16.0 samples/s
+Step 5270/44000 | Loss: 3.3456 | LR: 4.87e-04 | 687.0ms/step | 29.1 samples/s
+Step 5280/44000 | Loss: 3.3424 | LR: 4.87e-04 | 685.5ms/step | 29.2 samples/s
+Step 5290/44000 | Loss: 3.3156 | LR: 4.87e-04 | 683.2ms/step | 29.3 samples/s
+Step 5300/44000 | Loss: 3.3169 | LR: 4.86e-04 | 684.7ms/step | 29.2 samples/s
+Step 5310/44000 | Loss: 3.3450 | LR: 4.86e-04 | 683.3ms/step | 29.3 samples/s
+Step 5320/44000 | Loss: 3.2940 | LR: 4.86e-04 | 683.7ms/step | 29.3 samples/s
+Step 5330/44000 | Loss: 3.3424 | LR: 4.86e-04 | 684.4ms/step | 29.2 samples/s
+Step 5340/44000 | Loss: 3.3076 | LR: 4.86e-04 | 683.0ms/step | 29.3 samples/s
+Step 5350/44000 | Loss: 3.2986 | LR: 4.86e-04 | 684.6ms/step | 29.2 samples/s
+Step 5360/44000 | Loss: 3.3042 | LR: 4.86e-04 | 682.7ms/step | 29.3 samples/s
+Step 5370/44000 | Loss: 3.3298 | LR: 4.86e-04 | 684.5ms/step | 29.2 samples/s
+Step 5380/44000 | Loss: 3.3142 | LR: 4.86e-04 | 684.7ms/step | 29.2 samples/s
+Step 5390/44000 | Loss: 3.2867 | LR: 4.86e-04 | 682.6ms/step | 29.3 samples/s
+Step 5400/44000 | Loss: 3.3236 | LR: 4.86e-04 | 682.8ms/step | 29.3 samples/s
+Step 5410/44000 | Loss: 3.2990 | LR: 4.86e-04 | 683.0ms/step | 29.3 samples/s
+Step 5420/44000 | Loss: 3.3215 | LR: 4.86e-04 | 680.0ms/step | 29.4 samples/s
+Step 5430/44000 | Loss: 3.3168 | LR: 4.86e-04 | 680.1ms/step | 29.4 samples/s
+Step 5440/44000 | Loss: 3.3473 | LR: 4.86e-04 | 679.8ms/step | 29.4 samples/s
+Step 5450/44000 | Loss: 3.2748 | LR: 4.86e-04 | 680.3ms/step | 29.4 samples/s
+Step 5460/44000 | Loss: 3.2848 | LR: 4.86e-04 | 679.7ms/step | 29.4 samples/s
+Step 5470/44000 | Loss: 3.2872 | LR: 4.85e-04 | 679.0ms/step | 29.5 samples/s
+Step 5480/44000 | Loss: 3.3316 | LR: 4.85e-04 | 680.5ms/step | 29.4 samples/s
+Step 5490/44000 | Loss: 3.3487 | LR: 4.85e-04 | 679.2ms/step | 29.4 samples/s
+Step 5500/44000 | Loss: 3.3209 | LR: 4.85e-04 | 679.5ms/step | 29.4 samples/s
+Validation at step 5500: Loss = 3.3158, Perplexity = 27.55
+Step 5510/44000 | Loss: 3.2648 | LR: 4.85e-04 | 1242.4ms/step | 16.1 samples/s
+Step 5520/44000 | Loss: 3.2905 | LR: 4.85e-04 | 683.8ms/step | 29.2 samples/s
+Step 5530/44000 | Loss: 3.2811 | LR: 4.85e-04 | 682.1ms/step | 29.3 samples/s
+Step 5540/44000 | Loss: 3.2864 | LR: 4.85e-04 | 682.0ms/step | 29.3 samples/s
+Step 5550/44000 | Loss: 3.3184 | LR: 4.85e-04 | 682.8ms/step | 29.3 samples/s
+Step 5560/44000 | Loss: 3.3184 | LR: 4.85e-04 | 682.0ms/step | 29.3 samples/s
+Step 5570/44000 | Loss: 3.2975 | LR: 4.85e-04 | 682.0ms/step | 29.3 samples/s
+Step 5580/44000 | Loss: 3.3139 | LR: 4.85e-04 | 682.6ms/step | 29.3 samples/s
+Step 5590/44000 | Loss: 3.3363 | LR: 4.85e-04 | 681.7ms/step | 29.3 samples/s
+Step 5600/44000 | Loss: 3.3200 | LR: 4.85e-04 | 682.2ms/step | 29.3 samples/s
+Step 5610/44000 | Loss: 3.2936 | LR: 4.85e-04 | 680.5ms/step | 29.4 samples/s
+Step 5620/44000 | Loss: 3.2959 | LR: 4.85e-04 | 680.5ms/step | 29.4 samples/s
+Step 5630/44000 | Loss: 3.2888 | LR: 4.84e-04 | 679.6ms/step | 29.4 samples/s
+Step 5640/44000 | Loss: 3.3318 | LR: 4.84e-04 | 682.2ms/step | 29.3 samples/s
+Step 5650/44000 | Loss: 3.2412 | LR: 4.84e-04 | 682.8ms/step | 29.3 samples/s
+Step 5660/44000 | Loss: 3.2890 | LR: 4.84e-04 | 682.7ms/step | 29.3 samples/s
+Step 5670/44000 | Loss: 3.3116 | LR: 4.84e-04 | 682.6ms/step | 29.3 samples/s
+Step 5680/44000 | Loss: 3.2914 | LR: 4.84e-04 | 683.7ms/step | 29.3 samples/s
+Step 5690/44000 | Loss: 3.2913 | LR: 4.84e-04 | 682.6ms/step | 29.3 samples/s
+Step 5700/44000 | Loss: 3.2328 | LR: 4.84e-04 | 681.8ms/step | 29.3 samples/s
+Step 5710/44000 | Loss: 3.2992 | LR: 4.84e-04 | 683.0ms/step | 29.3 samples/s
+Step 5720/44000 | Loss: 3.3119 | LR: 4.84e-04 | 681.8ms/step | 29.3 samples/s
+Step 5730/44000 | Loss: 3.2860 | LR: 4.84e-04 | 682.5ms/step | 29.3 samples/s
+Step 5740/44000 | Loss: 3.2499 | LR: 4.84e-04 | 682.7ms/step | 29.3 samples/s
+Step 5750/44000 | Loss: 3.2661 | LR: 4.84e-04 | 681.6ms/step | 29.3 samples/s
+Validation at step 5750: Loss = 3.2998, Perplexity = 27.11
+Step 5760/44000 | Loss: 3.3369 | LR: 4.84e-04 | 1237.9ms/step | 16.2 samples/s
+Step 5770/44000 | Loss: 3.3070 | LR: 4.84e-04 | 680.1ms/step | 29.4 samples/s
+Step 5780/44000 | Loss: 3.2880 | LR: 4.83e-04 | 679.3ms/step | 29.4 samples/s
+Step 5790/44000 | Loss: 3.2553 | LR: 4.83e-04 | 680.5ms/step | 29.4 samples/s
+Step 5800/44000 | Loss: 3.3024 | LR: 4.83e-04 | 680.0ms/step | 29.4 samples/s
+Step 5810/44000 | Loss: 3.3459 | LR: 4.83e-04 | 680.1ms/step | 29.4 samples/s
+Step 5820/44000 | Loss: 3.3006 | LR: 4.83e-04 | 680.9ms/step | 29.4 samples/s
+Step 5830/44000 | Loss: 3.3122 | LR: 4.83e-04 | 680.9ms/step | 29.4 samples/s
+Step 5840/44000 | Loss: 3.2838 | LR: 4.83e-04 | 680.8ms/step | 29.4 samples/s
+Step 5850/44000 | Loss: 3.2904 | LR: 4.83e-04 | 681.9ms/step | 29.3 samples/s
+Step 5860/44000 | Loss: 3.2707 | LR: 4.83e-04 | 679.1ms/step | 29.5 samples/s
+Step 5870/44000 | Loss: 3.2643 | LR: 4.83e-04 | 680.5ms/step | 29.4 samples/s
+Step 5880/44000 | Loss: 3.2992 | LR: 4.83e-04 | 681.2ms/step | 29.4 samples/s
+Step 5890/44000 | Loss: 3.3205 | LR: 4.83e-04 | 681.4ms/step | 29.4 samples/s
+Step 5900/44000 | Loss: 3.3075 | LR: 4.83e-04 | 681.2ms/step | 29.4 samples/s
+Step 5910/44000 | Loss: 3.3168 | LR: 4.83e-04 | 680.2ms/step | 29.4 samples/s
+Step 5920/44000 | Loss: 3.2993 | LR: 4.83e-04 | 679.7ms/step | 29.4 samples/s
+Step 5930/44000 | Loss: 3.2549 | LR: 4.83e-04 | 680.4ms/step | 29.4 samples/s
+Step 5940/44000 | Loss: 3.2812 | LR: 4.82e-04 | 679.9ms/step | 29.4 samples/s
+Step 5950/44000 | Loss: 3.2856 | LR: 4.82e-04 | 680.8ms/step | 29.4 samples/s
+Step 5960/44000 | Loss: 3.2767 | LR: 4.82e-04 | 678.6ms/step | 29.5 samples/s
+Step 5970/44000 | Loss: 3.3060 | LR: 4.82e-04 | 680.9ms/step | 29.4 samples/s
+Step 5980/44000 | Loss: 3.2944 | LR: 4.82e-04 | 681.5ms/step | 29.3 samples/s
+Step 5990/44000 | Loss: 3.2877 | LR: 4.82e-04 | 680.4ms/step | 29.4 samples/s
+Step 6000/44000 | Loss: 3.2886 | LR: 4.82e-04 | 681.8ms/step | 29.3 samples/s
+Validation at step 6000: Loss = 3.2881, Perplexity = 26.79
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-6000
+Step 6010/44000 | Loss: 3.3164 | LR: 4.82e-04 | 1529.9ms/step | 13.1 samples/s
+Step 6020/44000 | Loss: 3.3169 | LR: 4.82e-04 | 680.4ms/step | 29.4 samples/s
+Step 6030/44000 | Loss: 3.2748 | LR: 4.82e-04 | 681.6ms/step | 29.3 samples/s
+Step 6040/44000 | Loss: 3.2842 | LR: 4.82e-04 | 680.4ms/step | 29.4 samples/s
+Step 6050/44000 | Loss: 3.2479 | LR: 4.82e-04 | 681.6ms/step | 29.3 samples/s
+Step 6060/44000 | Loss: 3.2579 | LR: 4.82e-04 | 682.3ms/step | 29.3 samples/s
+Step 6070/44000 | Loss: 3.2966 | LR: 4.82e-04 | 682.2ms/step | 29.3 samples/s
+Step 6080/44000 | Loss: 3.2917 | LR: 4.81e-04 | 681.9ms/step | 29.3 samples/s
+Step 6090/44000 | Loss: 3.3131 | LR: 4.81e-04 | 682.0ms/step | 29.3 samples/s
+Step 6100/44000 | Loss: 3.2950 | LR: 4.81e-04 | 683.6ms/step | 29.3 samples/s
+Step 6110/44000 | Loss: 3.2380 | LR: 4.81e-04 | 681.4ms/step | 29.4 samples/s
+Step 6120/44000 | Loss: 3.2746 | LR: 4.81e-04 | 682.8ms/step | 29.3 samples/s
+Step 6130/44000 | Loss: 3.2626 | LR: 4.81e-04 | 682.2ms/step | 29.3 samples/s
+Step 6140/44000 | Loss: 3.2603 | LR: 4.81e-04 | 682.0ms/step | 29.3 samples/s
+Step 6150/44000 | Loss: 3.2538 | LR: 4.81e-04 | 683.3ms/step | 29.3 samples/s
+Step 6160/44000 | Loss: 3.2764 | LR: 4.81e-04 | 683.2ms/step | 29.3 samples/s
+Step 6170/44000 | Loss: 3.2947 | LR: 4.81e-04 | 681.6ms/step | 29.3 samples/s
+Step 6180/44000 | Loss: 3.3079 | LR: 4.81e-04 | 680.8ms/step | 29.4 samples/s
+Step 6190/44000 | Loss: 3.2699 | LR: 4.81e-04 | 682.7ms/step | 29.3 samples/s
+Step 6200/44000 | Loss: 3.2474 | LR: 4.81e-04 | 681.2ms/step | 29.4 samples/s
+Step 6210/44000 | Loss: 3.2506 | LR: 4.81e-04 | 682.0ms/step | 29.3 samples/s
+Step 6220/44000 | Loss: 3.2324 | LR: 4.81e-04 | 681.5ms/step | 29.3 samples/s
+Step 6230/44000 | Loss: 3.2779 | LR: 4.80e-04 | 681.7ms/step | 29.3 samples/s
+Step 6240/44000 | Loss: 3.2363 | LR: 4.80e-04 | 680.4ms/step | 29.4 samples/s
+Step 6250/44000 | Loss: 3.2897 | LR: 4.80e-04 | 682.7ms/step | 29.3 samples/s
+Validation at step 6250: Loss = 3.2784, Perplexity = 26.53
+Step 6260/44000 | Loss: 3.2699 | LR: 4.80e-04 | 1239.7ms/step | 16.1 samples/s
+Step 6270/44000 | Loss: 3.2876 | LR: 4.80e-04 | 681.1ms/step | 29.4 samples/s
+Step 6280/44000 | Loss: 3.2835 | LR: 4.80e-04 | 681.9ms/step | 29.3 samples/s
+Step 6290/44000 | Loss: 3.2915 | LR: 4.80e-04 | 682.5ms/step | 29.3 samples/s
+Step 6300/44000 | Loss: 3.2651 | LR: 4.80e-04 | 681.2ms/step | 29.4 samples/s
+Step 6310/44000 | Loss: 3.3021 | LR: 4.80e-04 | 682.1ms/step | 29.3 samples/s
+Step 6320/44000 | Loss: 3.2442 | LR: 4.80e-04 | 682.3ms/step | 29.3 samples/s
+Step 6330/44000 | Loss: 3.3132 | LR: 4.80e-04 | 682.4ms/step | 29.3 samples/s
+Step 6340/44000 | Loss: 3.2685 | LR: 4.80e-04 | 681.7ms/step | 29.3 samples/s
+Step 6350/44000 | Loss: 3.3161 | LR: 4.80e-04 | 682.1ms/step | 29.3 samples/s
+Step 6360/44000 | Loss: 3.2853 | LR: 4.80e-04 | 681.2ms/step | 29.4 samples/s
+Step 6370/44000 | Loss: 3.2403 | LR: 4.79e-04 | 681.5ms/step | 29.3 samples/s
+Step 6380/44000 | Loss: 3.2693 | LR: 4.79e-04 | 683.7ms/step | 29.3 samples/s
+Step 6390/44000 | Loss: 3.2859 | LR: 4.79e-04 | 682.0ms/step | 29.3 samples/s
+Step 6400/44000 | Loss: 3.2454 | LR: 4.79e-04 | 681.8ms/step | 29.3 samples/s
+Step 6410/44000 | Loss: 3.2697 | LR: 4.79e-04 | 683.0ms/step | 29.3 samples/s
+Step 6420/44000 | Loss: 3.2635 | LR: 4.79e-04 | 681.2ms/step | 29.4 samples/s
+Step 6430/44000 | Loss: 3.2927 | LR: 4.79e-04 | 682.3ms/step | 29.3 samples/s
+Step 6440/44000 | Loss: 3.2558 | LR: 4.79e-04 | 681.4ms/step | 29.4 samples/s
+Step 6450/44000 | Loss: 3.2614 | LR: 4.79e-04 | 682.0ms/step | 29.3 samples/s
+Step 6460/44000 | Loss: 3.2724 | LR: 4.79e-04 | 682.3ms/step | 29.3 samples/s
+Step 6470/44000 | Loss: 3.2957 | LR: 4.79e-04 | 682.6ms/step | 29.3 samples/s
+Step 6480/44000 | Loss: 3.3140 | LR: 4.79e-04 | 682.1ms/step | 29.3 samples/s
+Step 6490/44000 | Loss: 3.2279 | LR: 4.79e-04 | 680.6ms/step | 29.4 samples/s
+Step 6500/44000 | Loss: 3.3076 | LR: 4.79e-04 | 682.7ms/step | 29.3 samples/s
+Validation at step 6500: Loss = 3.2658, Perplexity = 26.20
+Step 6510/44000 | Loss: 3.2742 | LR: 4.78e-04 | 1242.0ms/step | 16.1 samples/s
+Step 6520/44000 | Loss: 3.2622 | LR: 4.78e-04 | 681.7ms/step | 29.3 samples/s
+Step 6530/44000 | Loss: 3.2865 | LR: 4.78e-04 | 681.1ms/step | 29.4 samples/s
+Step 6540/44000 | Loss: 3.2561 | LR: 4.78e-04 | 681.4ms/step | 29.4 samples/s
+Step 6550/44000 | Loss: 3.2848 | LR: 4.78e-04 | 682.8ms/step | 29.3 samples/s
+Step 6560/44000 | Loss: 3.2322 | LR: 4.78e-04 | 682.1ms/step | 29.3 samples/s
+Step 6570/44000 | Loss: 3.2234 | LR: 4.78e-04 | 683.3ms/step | 29.3 samples/s
+Step 6580/44000 | Loss: 3.2458 | LR: 4.78e-04 | 682.2ms/step | 29.3 samples/s
+Step 6590/44000 | Loss: 3.2562 | LR: 4.78e-04 | 682.4ms/step | 29.3 samples/s
+Step 6600/44000 | Loss: 3.2259 | LR: 4.78e-04 | 682.5ms/step | 29.3 samples/s
+Step 6610/44000 | Loss: 3.2552 | LR: 4.78e-04 | 682.0ms/step | 29.3 samples/s
+Step 6620/44000 | Loss: 3.2465 | LR: 4.78e-04 | 682.3ms/step | 29.3 samples/s
+Step 6630/44000 | Loss: 3.2883 | LR: 4.78e-04 | 682.3ms/step | 29.3 samples/s
+Step 6640/44000 | Loss: 3.2659 | LR: 4.77e-04 | 683.8ms/step | 29.2 samples/s
+Step 6650/44000 | Loss: 3.2864 | LR: 4.77e-04 | 681.4ms/step | 29.4 samples/s
+Step 6660/44000 | Loss: 3.2136 | LR: 4.77e-04 | 681.9ms/step | 29.3 samples/s
+Step 6670/44000 | Loss: 3.2661 | LR: 4.77e-04 | 682.2ms/step | 29.3 samples/s
+Step 6680/44000 | Loss: 3.2368 | LR: 4.77e-04 | 681.5ms/step | 29.3 samples/s
+Step 6690/44000 | Loss: 3.2720 | LR: 4.77e-04 | 682.4ms/step | 29.3 samples/s
+Step 6700/44000 | Loss: 3.2586 | LR: 4.77e-04 | 680.6ms/step | 29.4 samples/s
+Step 6710/44000 | Loss: 3.2374 | LR: 4.77e-04 | 680.0ms/step | 29.4 samples/s
+Step 6720/44000 | Loss: 3.2537 | LR: 4.77e-04 | 682.3ms/step | 29.3 samples/s
+Step 6730/44000 | Loss: 3.3197 | LR: 4.77e-04 | 681.2ms/step | 29.4 samples/s
+Step 6740/44000 | Loss: 3.2437 | LR: 4.77e-04 | 681.2ms/step | 29.4 samples/s
+Step 6750/44000 | Loss: 3.2199 | LR: 4.77e-04 | 682.5ms/step | 29.3 samples/s
+Validation at step 6750: Loss = 3.2597, Perplexity = 26.04
+Step 6760/44000 | Loss: 3.2379 | LR: 4.77e-04 | 1240.3ms/step | 16.1 samples/s
+Step 6770/44000 | Loss: 3.2695 | LR: 4.76e-04 | 682.8ms/step | 29.3 samples/s
+Step 6780/44000 | Loss: 3.2444 | LR: 4.76e-04 | 681.2ms/step | 29.4 samples/s
+Step 6790/44000 | Loss: 3.2643 | LR: 4.76e-04 | 682.6ms/step | 29.3 samples/s
+Step 6800/44000 | Loss: 3.2226 | LR: 4.76e-04 | 682.1ms/step | 29.3 samples/s
+Step 6810/44000 | Loss: 3.2656 | LR: 4.76e-04 | 681.0ms/step | 29.4 samples/s
+Step 6820/44000 | Loss: 3.2647 | LR: 4.76e-04 | 681.9ms/step | 29.3 samples/s
+Step 6830/44000 | Loss: 3.2415 | LR: 4.76e-04 | 680.8ms/step | 29.4 samples/s
+Step 6840/44000 | Loss: 3.2216 | LR: 4.76e-04 | 681.0ms/step | 29.4 samples/s
+Step 6850/44000 | Loss: 3.2313 | LR: 4.76e-04 | 681.1ms/step | 29.4 samples/s
+Step 6860/44000 | Loss: 3.2074 | LR: 4.76e-04 | 681.7ms/step | 29.3 samples/s
+Step 6870/44000 | Loss: 3.2803 | LR: 4.76e-04 | 680.0ms/step | 29.4 samples/s
+Step 6880/44000 | Loss: 3.2353 | LR: 4.76e-04 | 680.1ms/step | 29.4 samples/s
+Step 6890/44000 | Loss: 3.2318 | LR: 4.76e-04 | 681.3ms/step | 29.4 samples/s
+Step 6900/44000 | Loss: 3.2416 | LR: 4.75e-04 | 681.1ms/step | 29.4 samples/s
+Step 6910/44000 | Loss: 3.2817 | LR: 4.75e-04 | 680.9ms/step | 29.4 samples/s
+Step 6920/44000 | Loss: 3.2866 | LR: 4.75e-04 | 681.9ms/step | 29.3 samples/s
+Step 6930/44000 | Loss: 3.2466 | LR: 4.75e-04 | 682.2ms/step | 29.3 samples/s
+Step 6940/44000 | Loss: 3.2453 | LR: 4.75e-04 | 682.6ms/step | 29.3 samples/s
+Step 6950/44000 | Loss: 3.2409 | LR: 4.75e-04 | 682.5ms/step | 29.3 samples/s
+Step 6960/44000 | Loss: 3.2616 | LR: 4.75e-04 | 682.2ms/step | 29.3 samples/s
+Step 6970/44000 | Loss: 3.2052 | LR: 4.75e-04 | 682.7ms/step | 29.3 samples/s
+Step 6980/44000 | Loss: 3.2595 | LR: 4.75e-04 | 681.7ms/step | 29.3 samples/s
+Step 6990/44000 | Loss: 3.2086 | LR: 4.75e-04 | 682.2ms/step | 29.3 samples/s
+Step 7000/44000 | Loss: 3.2390 | LR: 4.75e-04 | 681.5ms/step | 29.3 samples/s
+Validation at step 7000: Loss = 3.2479, Perplexity = 25.74
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-7000
+Step 7010/44000 | Loss: 3.2416 | LR: 4.75e-04 | 1513.6ms/step | 13.2 samples/s
+Step 7020/44000 | Loss: 3.2459 | LR: 4.75e-04 | 681.5ms/step | 29.3 samples/s
+Step 7030/44000 | Loss: 3.2243 | LR: 4.74e-04 | 679.4ms/step | 29.4 samples/s
+Step 7040/44000 | Loss: 3.2355 | LR: 4.74e-04 | 680.1ms/step | 29.4 samples/s
+Step 7050/44000 | Loss: 3.2453 | LR: 4.74e-04 | 681.1ms/step | 29.4 samples/s
+Step 7060/44000 | Loss: 3.2557 | LR: 4.74e-04 | 680.9ms/step | 29.4 samples/s
+Step 7070/44000 | Loss: 3.1812 | LR: 4.74e-04 | 679.9ms/step | 29.4 samples/s
+Step 7080/44000 | Loss: 3.2296 | LR: 4.74e-04 | 678.6ms/step | 29.5 samples/s
+Step 7090/44000 | Loss: 3.2749 | LR: 4.74e-04 | 678.8ms/step | 29.5 samples/s
+Step 7100/44000 | Loss: 3.2316 | LR: 4.74e-04 | 678.9ms/step | 29.5 samples/s
+Step 7110/44000 | Loss: 3.2163 | LR: 4.74e-04 | 677.4ms/step | 29.5 samples/s
+Step 7120/44000 | Loss: 3.2116 | LR: 4.74e-04 | 676.9ms/step | 29.5 samples/s
+Step 7130/44000 | Loss: 3.1702 | LR: 4.74e-04 | 677.5ms/step | 29.5 samples/s
+Step 7140/44000 | Loss: 3.2219 | LR: 4.74e-04 | 677.5ms/step | 29.5 samples/s
+Step 7150/44000 | Loss: 3.2445 | LR: 4.73e-04 | 678.5ms/step | 29.5 samples/s
+Step 7160/44000 | Loss: 3.2340 | LR: 4.73e-04 | 677.7ms/step | 29.5 samples/s
+Step 7170/44000 | Loss: 3.2090 | LR: 4.73e-04 | 677.9ms/step | 29.5 samples/s
+Step 7180/44000 | Loss: 3.2265 | LR: 4.73e-04 | 680.2ms/step | 29.4 samples/s
+Step 7190/44000 | Loss: 3.2487 | LR: 4.73e-04 | 679.9ms/step | 29.4 samples/s
+Step 7200/44000 | Loss: 3.2554 | LR: 4.73e-04 | 679.0ms/step | 29.5 samples/s
+Step 7210/44000 | Loss: 3.2330 | LR: 4.73e-04 | 679.6ms/step | 29.4 samples/s
+Step 7220/44000 | Loss: 3.2147 | LR: 4.73e-04 | 678.6ms/step | 29.5 samples/s
+Step 7230/44000 | Loss: 3.2292 | LR: 4.73e-04 | 679.8ms/step | 29.4 samples/s
+Step 7240/44000 | Loss: 3.2154 | LR: 4.73e-04 | 679.5ms/step | 29.4 samples/s
+Step 7250/44000 | Loss: 3.2517 | LR: 4.73e-04 | 678.5ms/step | 29.5 samples/s
+Validation at step 7250: Loss = 3.2378, Perplexity = 25.48
+Step 7260/44000 | Loss: 3.2445 | LR: 4.73e-04 | 1235.9ms/step | 16.2 samples/s
+Step 7270/44000 | Loss: 3.2137 | LR: 4.72e-04 | 679.9ms/step | 29.4 samples/s
+Step 7280/44000 | Loss: 3.2321 | LR: 4.72e-04 | 679.2ms/step | 29.4 samples/s
+Step 7290/44000 | Loss: 3.2509 | LR: 4.72e-04 | 678.9ms/step | 29.5 samples/s
+Step 7300/44000 | Loss: 3.2473 | LR: 4.72e-04 | 677.3ms/step | 29.5 samples/s
+Step 7310/44000 | Loss: 3.2488 | LR: 4.72e-04 | 678.8ms/step | 29.5 samples/s
+Step 7320/44000 | Loss: 3.2084 | LR: 4.72e-04 | 678.0ms/step | 29.5 samples/s
+Step 7330/44000 | Loss: 3.2435 | LR: 4.72e-04 | 678.0ms/step | 29.5 samples/s
+Step 7340/44000 | Loss: 3.2316 | LR: 4.72e-04 | 678.2ms/step | 29.5 samples/s
+Step 7350/44000 | Loss: 3.2156 | LR: 4.72e-04 | 678.1ms/step | 29.5 samples/s
+Step 7360/44000 | Loss: 3.2001 | LR: 4.72e-04 | 678.6ms/step | 29.5 samples/s
+Step 7370/44000 | Loss: 3.2559 | LR: 4.72e-04 | 678.2ms/step | 29.5 samples/s
+Step 7380/44000 | Loss: 3.2513 | LR: 4.72e-04 | 678.6ms/step | 29.5 samples/s
+Step 7390/44000 | Loss: 3.2778 | LR: 4.71e-04 | 678.8ms/step | 29.5 samples/s
+Step 7400/44000 | Loss: 3.2280 | LR: 4.71e-04 | 677.9ms/step | 29.5 samples/s
+Step 7410/44000 | Loss: 3.2428 | LR: 4.71e-04 | 678.4ms/step | 29.5 samples/s
+Step 7420/44000 | Loss: 3.2287 | LR: 4.71e-04 | 678.7ms/step | 29.5 samples/s
+Step 7430/44000 | Loss: 3.2288 | LR: 4.71e-04 | 679.1ms/step | 29.5 samples/s
+Step 7440/44000 | Loss: 3.2346 | LR: 4.71e-04 | 677.6ms/step | 29.5 samples/s
+Step 7450/44000 | Loss: 3.2619 | LR: 4.71e-04 | 678.1ms/step | 29.5 samples/s
+Step 7460/44000 | Loss: 3.2427 | LR: 4.71e-04 | 679.7ms/step | 29.4 samples/s
+Step 7470/44000 | Loss: 3.2438 | LR: 4.71e-04 | 678.4ms/step | 29.5 samples/s
+Step 7480/44000 | Loss: 3.2501 | LR: 4.71e-04 | 677.5ms/step | 29.5 samples/s
+Step 7490/44000 | Loss: 3.2011 | LR: 4.71e-04 | 678.5ms/step | 29.5 samples/s
+Step 7500/44000 | Loss: 3.1936 | LR: 4.71e-04 | 678.5ms/step | 29.5 samples/s
+Validation at step 7500: Loss = 3.2298, Perplexity = 25.27
+Step 7510/44000 | Loss: 3.2095 | LR: 4.70e-04 | 1239.6ms/step | 16.1 samples/s
+Step 7520/44000 | Loss: 3.2317 | LR: 4.70e-04 | 682.3ms/step | 29.3 samples/s
+Step 7530/44000 | Loss: 3.2518 | LR: 4.70e-04 | 682.0ms/step | 29.3 samples/s
+Step 7540/44000 | Loss: 3.2086 | LR: 4.70e-04 | 683.8ms/step | 29.3 samples/s
+Step 7550/44000 | Loss: 3.2021 | LR: 4.70e-04 | 682.6ms/step | 29.3 samples/s
+Step 7560/44000 | Loss: 3.2269 | LR: 4.70e-04 | 680.4ms/step | 29.4 samples/s
+Step 7570/44000 | Loss: 3.1721 | LR: 4.70e-04 | 679.8ms/step | 29.4 samples/s
+Step 7580/44000 | Loss: 3.1467 | LR: 4.70e-04 | 682.4ms/step | 29.3 samples/s
+Step 7590/44000 | Loss: 3.2108 | LR: 4.70e-04 | 681.0ms/step | 29.4 samples/s
+Step 7600/44000 | Loss: 3.2213 | LR: 4.70e-04 | 681.0ms/step | 29.4 samples/s
+Step 7610/44000 | Loss: 3.2189 | LR: 4.70e-04 | 682.2ms/step | 29.3 samples/s
+Step 7620/44000 | Loss: 3.2377 | LR: 4.70e-04 | 682.4ms/step | 29.3 samples/s
+Step 7630/44000 | Loss: 3.2080 | LR: 4.69e-04 | 680.9ms/step | 29.4 samples/s
+Step 7640/44000 | Loss: 3.1766 | LR: 4.69e-04 | 679.9ms/step | 29.4 samples/s
+Step 7650/44000 | Loss: 3.1990 | LR: 4.69e-04 | 681.9ms/step | 29.3 samples/s
+Step 7660/44000 | Loss: 3.2231 | LR: 4.69e-04 | 682.3ms/step | 29.3 samples/s
+Step 7670/44000 | Loss: 3.1943 | LR: 4.69e-04 | 683.1ms/step | 29.3 samples/s
+Step 7680/44000 | Loss: 3.2572 | LR: 4.69e-04 | 682.7ms/step | 29.3 samples/s
+Step 7690/44000 | Loss: 3.2244 | LR: 4.69e-04 | 682.1ms/step | 29.3 samples/s
+Step 7700/44000 | Loss: 3.1975 | LR: 4.69e-04 | 681.8ms/step | 29.3 samples/s
+Step 7710/44000 | Loss: 3.1890 | LR: 4.69e-04 | 681.1ms/step | 29.4 samples/s
+Step 7720/44000 | Loss: 3.2167 | LR: 4.69e-04 | 683.3ms/step | 29.3 samples/s
+Step 7730/44000 | Loss: 3.2047 | LR: 4.69e-04 | 681.9ms/step | 29.3 samples/s
+Step 7740/44000 | Loss: 3.1646 | LR: 4.68e-04 | 682.6ms/step | 29.3 samples/s
+Step 7750/44000 | Loss: 3.2065 | LR: 4.68e-04 | 684.8ms/step | 29.2 samples/s
+Validation at step 7750: Loss = 3.2211, Perplexity = 25.06
+Step 7760/44000 | Loss: 3.2048 | LR: 4.68e-04 | 1241.5ms/step | 16.1 samples/s
+Step 7770/44000 | Loss: 3.2135 | LR: 4.68e-04 | 682.3ms/step | 29.3 samples/s
+Step 7780/44000 | Loss: 3.2123 | LR: 4.68e-04 | 682.3ms/step | 29.3 samples/s
+Step 7790/44000 | Loss: 3.2149 | LR: 4.68e-04 | 681.1ms/step | 29.4 samples/s
+Step 7800/44000 | Loss: 3.2257 | LR: 4.68e-04 | 681.3ms/step | 29.4 samples/s
+Step 7810/44000 | Loss: 3.1717 | LR: 4.68e-04 | 680.5ms/step | 29.4 samples/s
+Step 7820/44000 | Loss: 3.2175 | LR: 4.68e-04 | 681.5ms/step | 29.3 samples/s
+Step 7830/44000 | Loss: 3.2034 | LR: 4.68e-04 | 681.7ms/step | 29.3 samples/s
+Step 7840/44000 | Loss: 3.1849 | LR: 4.68e-04 | 682.0ms/step | 29.3 samples/s
+Step 7850/44000 | Loss: 3.1836 | LR: 4.67e-04 | 681.3ms/step | 29.4 samples/s
+Step 7860/44000 | Loss: 3.2015 | LR: 4.67e-04 | 682.0ms/step | 29.3 samples/s
+Step 7870/44000 | Loss: 3.2070 | LR: 4.67e-04 | 683.2ms/step | 29.3 samples/s
+Step 7880/44000 | Loss: 3.2347 | LR: 4.67e-04 | 681.5ms/step | 29.3 samples/s
+Step 7890/44000 | Loss: 3.2008 | LR: 4.67e-04 | 682.0ms/step | 29.3 samples/s
+Step 7900/44000 | Loss: 3.2439 | LR: 4.67e-04 | 681.8ms/step | 29.3 samples/s
+Step 7910/44000 | Loss: 3.2194 | LR: 4.67e-04 | 682.7ms/step | 29.3 samples/s
+Step 7920/44000 | Loss: 3.2346 | LR: 4.67e-04 | 683.1ms/step | 29.3 samples/s
+Step 7930/44000 | Loss: 3.2598 | LR: 4.67e-04 | 682.1ms/step | 29.3 samples/s
+Step 7940/44000 | Loss: 3.2585 | LR: 4.67e-04 | 682.5ms/step | 29.3 samples/s
+Step 7950/44000 | Loss: 3.1714 | LR: 4.67e-04 | 682.1ms/step | 29.3 samples/s
+Step 7960/44000 | Loss: 3.2250 | LR: 4.66e-04 | 682.0ms/step | 29.3 samples/s
+Step 7970/44000 | Loss: 3.1593 | LR: 4.66e-04 | 682.2ms/step | 29.3 samples/s
+Step 7980/44000 | Loss: 3.1908 | LR: 4.66e-04 | 682.0ms/step | 29.3 samples/s
+Step 7990/44000 | Loss: 3.2133 | LR: 4.66e-04 | 682.5ms/step | 29.3 samples/s
+Step 8000/44000 | Loss: 3.2482 | LR: 4.66e-04 | 682.3ms/step | 29.3 samples/s
+Validation at step 8000: Loss = 3.2132, Perplexity = 24.86
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-8000
+Step 8010/44000 | Loss: 3.2003 | LR: 4.66e-04 | 1522.8ms/step | 13.1 samples/s
+Step 8020/44000 | Loss: 3.2183 | LR: 4.66e-04 | 682.3ms/step | 29.3 samples/s
+Step 8030/44000 | Loss: 3.2136 | LR: 4.66e-04 | 682.5ms/step | 29.3 samples/s
+Step 8040/44000 | Loss: 3.2349 | LR: 4.66e-04 | 680.9ms/step | 29.4 samples/s
+Step 8050/44000 | Loss: 3.2064 | LR: 4.66e-04 | 680.0ms/step | 29.4 samples/s
+Step 8060/44000 | Loss: 3.2226 | LR: 4.66e-04 | 680.0ms/step | 29.4 samples/s
+Step 8070/44000 | Loss: 3.1707 | LR: 4.65e-04 | 681.7ms/step | 29.3 samples/s
+Step 8080/44000 | Loss: 3.1858 | LR: 4.65e-04 | 682.5ms/step | 29.3 samples/s
+Step 8090/44000 | Loss: 3.2295 | LR: 4.65e-04 | 681.9ms/step | 29.3 samples/s
+Step 8100/44000 | Loss: 3.2361 | LR: 4.65e-04 | 680.9ms/step | 29.4 samples/s
+Step 8110/44000 | Loss: 3.2172 | LR: 4.65e-04 | 683.2ms/step | 29.3 samples/s
+Step 8120/44000 | Loss: 3.1681 | LR: 4.65e-04 | 681.7ms/step | 29.3 samples/s
+W0708 22:28:09.644000 254 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 254 got signal: 15

epfl_final/logs/l8k365m_pool100.log

@@ -0,0 +1,1072 @@
+W0708 20:48:06.039000 255 site-packages/torch/distributed/run.py:851] 
+W0708 20:48:06.039000 255 site-packages/torch/distributed/run.py:851] *****************************************
+W0708 20:48:06.039000 255 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0708 20:48:06.039000 255 site-packages/torch/distributed/run.py:851] *****************************************
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 44000,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 5,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m pool100 8k nrb31 freqabl",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "8k",
+    "scaleladder",
+    "resampled",
+    "freqablation",
+    "pool100"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 100,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_32",
+  "sparse_mask_seed": 0,
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: False
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 100
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 768]            --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 768]            116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 768]            [2, 8192, 768]            768                       True
+├─Linear: 1-2                                                [2, 8192, 768]            [2, 8192, 151936]         116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 951.29
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 31197.23
+Params size (MB): 951.29
+Estimated Total Size (MB): 32148.66
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        2
+Gradient accumulation steps:              5
+Effective batch size (seqs/step):         20
+Sequence length:                          8192
+Tokens per optimizer step:                163,840
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                151388
+Total optimizer steps:                    44000  (capped by max_steps; 0.29 epochs)
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 82re6m3i
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_205109-82re6m3i
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 365m pool100 8k nrb31 freqabl
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/82re6m3i
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 10/44000 | Loss: 12.0398 | LR: 6.67e-06 | 6736.3ms/step | 3.0 samples/s
+Step 20/44000 | Loss: 11.9261 | LR: 1.33e-05 | 677.8ms/step | 29.5 samples/s
+Step 30/44000 | Loss: 11.5485 | LR: 2.00e-05 | 680.1ms/step | 29.4 samples/s
+Step 40/44000 | Loss: 11.0225 | LR: 2.67e-05 | 674.7ms/step | 29.6 samples/s
+Step 50/44000 | Loss: 10.7137 | LR: 3.33e-05 | 671.4ms/step | 29.8 samples/s
+Step 60/44000 | Loss: 10.4833 | LR: 4.00e-05 | 675.4ms/step | 29.6 samples/s
+Step 70/44000 | Loss: 10.2636 | LR: 4.67e-05 | 672.6ms/step | 29.7 samples/s
+Step 80/44000 | Loss: 9.9451 | LR: 5.33e-05 | 673.1ms/step | 29.7 samples/s
+Step 90/44000 | Loss: 9.5554 | LR: 6.00e-05 | 677.6ms/step | 29.5 samples/s
+Step 100/44000 | Loss: 9.1247 | LR: 6.67e-05 | 675.0ms/step | 29.6 samples/s
+Step 110/44000 | Loss: 8.5965 | LR: 7.33e-05 | 674.8ms/step | 29.6 samples/s
+Step 120/44000 | Loss: 8.1657 | LR: 8.00e-05 | 672.8ms/step | 29.7 samples/s
+Step 130/44000 | Loss: 7.7923 | LR: 8.67e-05 | 673.8ms/step | 29.7 samples/s
+Step 140/44000 | Loss: 7.5092 | LR: 9.33e-05 | 675.6ms/step | 29.6 samples/s
+Step 150/44000 | Loss: 7.3410 | LR: 1.00e-04 | 674.4ms/step | 29.7 samples/s
+Step 160/44000 | Loss: 7.1668 | LR: 1.07e-04 | 676.6ms/step | 29.6 samples/s
+Step 170/44000 | Loss: 7.1116 | LR: 1.13e-04 | 677.2ms/step | 29.5 samples/s
+Step 180/44000 | Loss: 7.0514 | LR: 1.20e-04 | 678.2ms/step | 29.5 samples/s
+Step 190/44000 | Loss: 6.9679 | LR: 1.27e-04 | 678.7ms/step | 29.5 samples/s
+Step 200/44000 | Loss: 6.8446 | LR: 1.33e-04 | 680.1ms/step | 29.4 samples/s
+Step 210/44000 | Loss: 6.7549 | LR: 1.40e-04 | 677.8ms/step | 29.5 samples/s
+Step 220/44000 | Loss: 6.6656 | LR: 1.47e-04 | 680.1ms/step | 29.4 samples/s
+Step 230/44000 | Loss: 6.5696 | LR: 1.53e-04 | 679.5ms/step | 29.4 samples/s
+Step 240/44000 | Loss: 6.5532 | LR: 1.60e-04 | 680.1ms/step | 29.4 samples/s
+Step 250/44000 | Loss: 6.4335 | LR: 1.67e-04 | 680.9ms/step | 29.4 samples/s
+Validation at step 250: Loss = 6.4401, Perplexity = 626.49
+Step 260/44000 | Loss: 6.4447 | LR: 1.73e-04 | 3834.4ms/step | 5.2 samples/s
+Step 270/44000 | Loss: 6.3853 | LR: 1.80e-04 | 680.5ms/step | 29.4 samples/s
+Step 280/44000 | Loss: 6.2990 | LR: 1.87e-04 | 681.3ms/step | 29.4 samples/s
+Step 290/44000 | Loss: 6.2483 | LR: 1.93e-04 | 679.4ms/step | 29.4 samples/s
+Step 300/44000 | Loss: 6.2525 | LR: 2.00e-04 | 680.0ms/step | 29.4 samples/s
+Step 310/44000 | Loss: 6.2115 | LR: 2.07e-04 | 680.8ms/step | 29.4 samples/s
+Step 320/44000 | Loss: 6.1328 | LR: 2.13e-04 | 680.6ms/step | 29.4 samples/s
+Step 330/44000 | Loss: 6.0866 | LR: 2.20e-04 | 680.0ms/step | 29.4 samples/s
+Step 340/44000 | Loss: 6.0659 | LR: 2.27e-04 | 683.8ms/step | 29.2 samples/s
+Step 350/44000 | Loss: 5.9946 | LR: 2.33e-04 | 683.4ms/step | 29.3 samples/s
+Step 360/44000 | Loss: 5.9899 | LR: 2.40e-04 | 680.1ms/step | 29.4 samples/s
+Step 370/44000 | Loss: 5.8784 | LR: 2.47e-04 | 679.2ms/step | 29.4 samples/s
+Step 380/44000 | Loss: 5.8986 | LR: 2.53e-04 | 679.1ms/step | 29.4 samples/s
+Step 390/44000 | Loss: 5.8632 | LR: 2.60e-04 | 681.4ms/step | 29.4 samples/s
+Step 400/44000 | Loss: 5.8139 | LR: 2.67e-04 | 681.8ms/step | 29.3 samples/s
+Step 410/44000 | Loss: 5.7832 | LR: 2.73e-04 | 681.2ms/step | 29.4 samples/s
+Step 420/44000 | Loss: 5.7248 | LR: 2.80e-04 | 680.6ms/step | 29.4 samples/s
+Step 430/44000 | Loss: 5.7328 | LR: 2.87e-04 | 681.6ms/step | 29.3 samples/s
+Step 440/44000 | Loss: 5.7227 | LR: 2.93e-04 | 681.9ms/step | 29.3 samples/s
+Step 450/44000 | Loss: 5.6031 | LR: 3.00e-04 | 682.6ms/step | 29.3 samples/s
+Step 460/44000 | Loss: 5.6141 | LR: 3.07e-04 | 681.7ms/step | 29.3 samples/s
+Step 470/44000 | Loss: 5.5835 | LR: 3.13e-04 | 680.8ms/step | 29.4 samples/s
+Step 480/44000 | Loss: 5.5358 | LR: 3.20e-04 | 680.0ms/step | 29.4 samples/s
+Step 490/44000 | Loss: 5.5257 | LR: 3.27e-04 | 680.0ms/step | 29.4 samples/s
+Step 500/44000 | Loss: 5.4933 | LR: 3.33e-04 | 680.0ms/step | 29.4 samples/s
+Validation at step 500: Loss = 5.4703, Perplexity = 237.54
+Step 510/44000 | Loss: 5.3939 | LR: 3.40e-04 | 1247.2ms/step | 16.0 samples/s
+Step 520/44000 | Loss: 5.3948 | LR: 3.47e-04 | 684.7ms/step | 29.2 samples/s
+Step 530/44000 | Loss: 5.3818 | LR: 3.53e-04 | 682.7ms/step | 29.3 samples/s
+Step 540/44000 | Loss: 5.3525 | LR: 3.60e-04 | 682.7ms/step | 29.3 samples/s
+Step 550/44000 | Loss: 5.3688 | LR: 3.67e-04 | 683.7ms/step | 29.3 samples/s
+Step 560/44000 | Loss: 5.3101 | LR: 3.73e-04 | 681.4ms/step | 29.3 samples/s
+Step 570/44000 | Loss: 5.3016 | LR: 3.80e-04 | 680.8ms/step | 29.4 samples/s
+Step 580/44000 | Loss: 5.2444 | LR: 3.87e-04 | 680.8ms/step | 29.4 samples/s
+Step 590/44000 | Loss: 5.2354 | LR: 3.93e-04 | 681.7ms/step | 29.3 samples/s
+Step 600/44000 | Loss: 5.2160 | LR: 4.00e-04 | 682.2ms/step | 29.3 samples/s
+Step 610/44000 | Loss: 5.1322 | LR: 4.07e-04 | 681.8ms/step | 29.3 samples/s
+Step 620/44000 | Loss: 5.1132 | LR: 4.13e-04 | 682.4ms/step | 29.3 samples/s
+Step 630/44000 | Loss: 5.1564 | LR: 4.20e-04 | 681.6ms/step | 29.3 samples/s
+Step 640/44000 | Loss: 5.1229 | LR: 4.27e-04 | 681.7ms/step | 29.3 samples/s
+Step 650/44000 | Loss: 5.0708 | LR: 4.33e-04 | 680.7ms/step | 29.4 samples/s
+Step 660/44000 | Loss: 5.0615 | LR: 4.40e-04 | 680.9ms/step | 29.4 samples/s
+Step 670/44000 | Loss: 5.0260 | LR: 4.47e-04 | 680.7ms/step | 29.4 samples/s
+Step 680/44000 | Loss: 5.0432 | LR: 4.53e-04 | 683.5ms/step | 29.3 samples/s
+Step 690/44000 | Loss: 5.0021 | LR: 4.60e-04 | 681.8ms/step | 29.3 samples/s
+Step 700/44000 | Loss: 4.9098 | LR: 4.67e-04 | 683.3ms/step | 29.3 samples/s
+Step 710/44000 | Loss: 4.9521 | LR: 4.73e-04 | 679.8ms/step | 29.4 samples/s
+Step 720/44000 | Loss: 4.8679 | LR: 4.80e-04 | 683.1ms/step | 29.3 samples/s
+Step 730/44000 | Loss: 4.8684 | LR: 4.87e-04 | 682.5ms/step | 29.3 samples/s
+Step 740/44000 | Loss: 4.8072 | LR: 4.93e-04 | 681.6ms/step | 29.3 samples/s
+Step 750/44000 | Loss: 4.7878 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Validation at step 750: Loss = 4.8024, Perplexity = 121.80
+Step 760/44000 | Loss: 4.7690 | LR: 5.00e-04 | 1242.5ms/step | 16.1 samples/s
+Step 770/44000 | Loss: 4.7786 | LR: 5.00e-04 | 682.1ms/step | 29.3 samples/s
+Step 780/44000 | Loss: 4.6998 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s
+Step 790/44000 | Loss: 4.7071 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s
+Step 800/44000 | Loss: 4.7447 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 810/44000 | Loss: 4.6695 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s
+Step 820/44000 | Loss: 4.6305 | LR: 5.00e-04 | 683.6ms/step | 29.3 samples/s
+Step 830/44000 | Loss: 4.6075 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s
+Step 840/44000 | Loss: 4.5450 | LR: 5.00e-04 | 681.3ms/step | 29.4 samples/s
+Step 850/44000 | Loss: 4.5586 | LR: 5.00e-04 | 681.8ms/step | 29.3 samples/s
+Step 860/44000 | Loss: 4.5464 | LR: 5.00e-04 | 682.2ms/step | 29.3 samples/s
+Step 870/44000 | Loss: 4.5607 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s
+Step 880/44000 | Loss: 4.4897 | LR: 5.00e-04 | 682.5ms/step | 29.3 samples/s
+Step 890/44000 | Loss: 4.4793 | LR: 5.00e-04 | 681.4ms/step | 29.4 samples/s
+Step 900/44000 | Loss: 4.4737 | LR: 5.00e-04 | 681.3ms/step | 29.4 samples/s
+Step 910/44000 | Loss: 4.4645 | LR: 5.00e-04 | 680.8ms/step | 29.4 samples/s
+Step 920/44000 | Loss: 4.4159 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s
+Step 930/44000 | Loss: 4.4167 | LR: 5.00e-04 | 684.6ms/step | 29.2 samples/s
+Step 940/44000 | Loss: 4.3917 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 950/44000 | Loss: 4.4081 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 960/44000 | Loss: 4.3439 | LR: 5.00e-04 | 681.0ms/step | 29.4 samples/s
+Step 970/44000 | Loss: 4.3519 | LR: 5.00e-04 | 681.6ms/step | 29.3 samples/s
+Step 980/44000 | Loss: 4.3508 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s
+Step 990/44000 | Loss: 4.3235 | LR: 5.00e-04 | 681.1ms/step | 29.4 samples/s
+Step 1000/44000 | Loss: 4.3257 | LR: 5.00e-04 | 681.5ms/step | 29.3 samples/s
+Validation at step 1000: Loss = 4.3220, Perplexity = 75.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-1000
+Step 1010/44000 | Loss: 4.3460 | LR: 5.00e-04 | 1526.2ms/step | 13.1 samples/s
+Step 1020/44000 | Loss: 4.2801 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s
+Step 1030/44000 | Loss: 4.2858 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s
+Step 1040/44000 | Loss: 4.2180 | LR: 5.00e-04 | 684.1ms/step | 29.2 samples/s
+Step 1050/44000 | Loss: 4.2685 | LR: 5.00e-04 | 680.8ms/step | 29.4 samples/s
+Step 1060/44000 | Loss: 4.2945 | LR: 5.00e-04 | 681.1ms/step | 29.4 samples/s
+Step 1070/44000 | Loss: 4.2088 | LR: 5.00e-04 | 683.5ms/step | 29.3 samples/s
+Step 1080/44000 | Loss: 4.2116 | LR: 5.00e-04 | 681.3ms/step | 29.4 samples/s
+Step 1090/44000 | Loss: 4.2323 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 1100/44000 | Loss: 4.2100 | LR: 5.00e-04 | 680.2ms/step | 29.4 samples/s
+Step 1110/44000 | Loss: 4.2157 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s
+Step 1120/44000 | Loss: 4.1160 | LR: 5.00e-04 | 681.8ms/step | 29.3 samples/s
+Step 1130/44000 | Loss: 4.1421 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s
+Step 1140/44000 | Loss: 4.1314 | LR: 5.00e-04 | 683.8ms/step | 29.2 samples/s
+Step 1150/44000 | Loss: 4.1335 | LR: 5.00e-04 | 680.6ms/step | 29.4 samples/s
+Step 1160/44000 | Loss: 4.1396 | LR: 5.00e-04 | 680.8ms/step | 29.4 samples/s
+Step 1170/44000 | Loss: 4.1272 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s
+Step 1180/44000 | Loss: 4.1398 | LR: 5.00e-04 | 681.4ms/step | 29.4 samples/s
+Step 1190/44000 | Loss: 4.1006 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s
+Step 1200/44000 | Loss: 4.1044 | LR: 5.00e-04 | 680.9ms/step | 29.4 samples/s
+Step 1210/44000 | Loss: 4.1278 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s
+Step 1220/44000 | Loss: 4.1129 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s
+Step 1230/44000 | Loss: 4.0767 | LR: 5.00e-04 | 684.0ms/step | 29.2 samples/s
+Step 1240/44000 | Loss: 4.0811 | LR: 5.00e-04 | 684.2ms/step | 29.2 samples/s
+Step 1250/44000 | Loss: 4.0561 | LR: 5.00e-04 | 681.9ms/step | 29.3 samples/s
+Validation at step 1250: Loss = 4.0741, Perplexity = 58.80
+Step 1260/44000 | Loss: 4.1101 | LR: 5.00e-04 | 1243.5ms/step | 16.1 samples/s
+Step 1270/44000 | Loss: 4.0343 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s
+Step 1280/44000 | Loss: 4.0511 | LR: 5.00e-04 | 681.2ms/step | 29.4 samples/s
+Step 1290/44000 | Loss: 4.0439 | LR: 5.00e-04 | 682.5ms/step | 29.3 samples/s
+Step 1300/44000 | Loss: 4.0494 | LR: 5.00e-04 | 680.3ms/step | 29.4 samples/s
+Step 1310/44000 | Loss: 3.9890 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s
+Step 1320/44000 | Loss: 3.9759 | LR: 5.00e-04 | 684.4ms/step | 29.2 samples/s
+Step 1330/44000 | Loss: 3.9987 | LR: 5.00e-04 | 682.3ms/step | 29.3 samples/s
+Step 1340/44000 | Loss: 4.0415 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s
+Step 1350/44000 | Loss: 3.9718 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s
+Step 1360/44000 | Loss: 3.9880 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 1370/44000 | Loss: 3.9674 | LR: 5.00e-04 | 680.9ms/step | 29.4 samples/s
+Step 1380/44000 | Loss: 3.9866 | LR: 5.00e-04 | 681.2ms/step | 29.4 samples/s
+Step 1390/44000 | Loss: 3.9734 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s
+Step 1400/44000 | Loss: 3.9759 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s
+Step 1410/44000 | Loss: 3.9504 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s
+Step 1420/44000 | Loss: 3.9534 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 1430/44000 | Loss: 3.9652 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s
+Step 1440/44000 | Loss: 3.8943 | LR: 5.00e-04 | 683.2ms/step | 29.3 samples/s
+Step 1450/44000 | Loss: 3.9158 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 1460/44000 | Loss: 3.9641 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s
+Step 1470/44000 | Loss: 3.9075 | LR: 5.00e-04 | 681.9ms/step | 29.3 samples/s
+Step 1480/44000 | Loss: 3.8820 | LR: 5.00e-04 | 683.5ms/step | 29.3 samples/s
+Step 1490/44000 | Loss: 3.8792 | LR: 5.00e-04 | 681.9ms/step | 29.3 samples/s
+Step 1500/44000 | Loss: 3.9346 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Validation at step 1500: Loss = 3.9142, Perplexity = 50.11
+Step 1510/44000 | Loss: 3.9090 | LR: 5.00e-04 | 1238.7ms/step | 16.1 samples/s
+Step 1520/44000 | Loss: 3.8981 | LR: 5.00e-04 | 683.3ms/step | 29.3 samples/s
+Step 1530/44000 | Loss: 3.9228 | LR: 5.00e-04 | 682.7ms/step | 29.3 samples/s
+Step 1540/44000 | Loss: 3.8443 | LR: 5.00e-04 | 682.3ms/step | 29.3 samples/s
+Step 1550/44000 | Loss: 3.9214 | LR: 5.00e-04 | 684.9ms/step | 29.2 samples/s
+Step 1560/44000 | Loss: 3.8821 | LR: 5.00e-04 | 681.7ms/step | 29.3 samples/s
+Step 1570/44000 | Loss: 3.9016 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s
+Step 1580/44000 | Loss: 3.8545 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s
+Step 1590/44000 | Loss: 3.8580 | LR: 5.00e-04 | 682.3ms/step | 29.3 samples/s
+Step 1600/44000 | Loss: 3.8513 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s
+Step 1610/44000 | Loss: 3.8633 | LR: 5.00e-04 | 681.7ms/step | 29.3 samples/s
+Step 1620/44000 | Loss: 3.8403 | LR: 5.00e-04 | 681.2ms/step | 29.4 samples/s
+Step 1630/44000 | Loss: 3.8547 | LR: 4.99e-04 | 680.4ms/step | 29.4 samples/s
+Step 1640/44000 | Loss: 3.8142 | LR: 4.99e-04 | 682.2ms/step | 29.3 samples/s
+Step 1650/44000 | Loss: 3.8556 | LR: 4.99e-04 | 681.9ms/step | 29.3 samples/s
+Step 1660/44000 | Loss: 3.8659 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s
+Step 1670/44000 | Loss: 3.8726 | LR: 4.99e-04 | 680.3ms/step | 29.4 samples/s
+Step 1680/44000 | Loss: 3.8139 | LR: 4.99e-04 | 683.3ms/step | 29.3 samples/s
+Step 1690/44000 | Loss: 3.7974 | LR: 4.99e-04 | 680.8ms/step | 29.4 samples/s
+Step 1700/44000 | Loss: 3.8406 | LR: 4.99e-04 | 681.2ms/step | 29.4 samples/s
+Step 1710/44000 | Loss: 3.7913 | LR: 4.99e-04 | 682.3ms/step | 29.3 samples/s
+Step 1720/44000 | Loss: 3.8454 | LR: 4.99e-04 | 681.0ms/step | 29.4 samples/s
+Step 1730/44000 | Loss: 3.8109 | LR: 4.99e-04 | 681.5ms/step | 29.3 samples/s
+Step 1740/44000 | Loss: 3.7950 | LR: 4.99e-04 | 682.6ms/step | 29.3 samples/s
+Step 1750/44000 | Loss: 3.7722 | LR: 4.99e-04 | 682.2ms/step | 29.3 samples/s
+Validation at step 1750: Loss = 3.8081, Perplexity = 45.06
+Step 1760/44000 | Loss: 3.8124 | LR: 4.99e-04 | 1244.5ms/step | 16.1 samples/s
+Step 1770/44000 | Loss: 3.8140 | LR: 4.99e-04 | 683.5ms/step | 29.3 samples/s
+Step 1780/44000 | Loss: 3.8096 | LR: 4.99e-04 | 682.1ms/step | 29.3 samples/s
+Step 1790/44000 | Loss: 3.8036 | LR: 4.99e-04 | 682.3ms/step | 29.3 samples/s
+Step 1800/44000 | Loss: 3.7815 | LR: 4.99e-04 | 682.7ms/step | 29.3 samples/s
+Step 1810/44000 | Loss: 3.7622 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s
+Step 1820/44000 | Loss: 3.7738 | LR: 4.99e-04 | 684.6ms/step | 29.2 samples/s
+Step 1830/44000 | Loss: 3.7561 | LR: 4.99e-04 | 682.1ms/step | 29.3 samples/s
+Step 1840/44000 | Loss: 3.7443 | LR: 4.99e-04 | 684.2ms/step | 29.2 samples/s
+Step 1850/44000 | Loss: 3.7809 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s
+Step 1860/44000 | Loss: 3.8033 | LR: 4.99e-04 | 683.7ms/step | 29.3 samples/s
+Step 1870/44000 | Loss: 3.7895 | LR: 4.99e-04 | 684.3ms/step | 29.2 samples/s
+Step 1880/44000 | Loss: 3.7483 | LR: 4.99e-04 | 681.8ms/step | 29.3 samples/s
+Step 1890/44000 | Loss: 3.7258 | LR: 4.99e-04 | 684.1ms/step | 29.2 samples/s
+Step 1900/44000 | Loss: 3.7610 | LR: 4.99e-04 | 681.5ms/step | 29.3 samples/s
+Step 1910/44000 | Loss: 3.7677 | LR: 4.99e-04 | 681.1ms/step | 29.4 samples/s
+Step 1920/44000 | Loss: 3.7417 | LR: 4.99e-04 | 682.8ms/step | 29.3 samples/s
+Step 1930/44000 | Loss: 3.7539 | LR: 4.99e-04 | 681.8ms/step | 29.3 samples/s
+Step 1940/44000 | Loss: 3.7155 | LR: 4.99e-04 | 682.7ms/step | 29.3 samples/s
+Step 1950/44000 | Loss: 3.6881 | LR: 4.99e-04 | 686.7ms/step | 29.1 samples/s
+Step 1960/44000 | Loss: 3.7373 | LR: 4.99e-04 | 684.7ms/step | 29.2 samples/s
+Step 1970/44000 | Loss: 3.7051 | LR: 4.99e-04 | 682.5ms/step | 29.3 samples/s
+Step 1980/44000 | Loss: 3.6944 | LR: 4.99e-04 | 682.8ms/step | 29.3 samples/s
+Step 1990/44000 | Loss: 3.7627 | LR: 4.99e-04 | 682.4ms/step | 29.3 samples/s
+Step 2000/44000 | Loss: 3.7249 | LR: 4.99e-04 | 683.7ms/step | 29.3 samples/s
+Validation at step 2000: Loss = 3.7284, Perplexity = 41.61
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-2000
+Step 2010/44000 | Loss: 3.7331 | LR: 4.99e-04 | 1527.0ms/step | 13.1 samples/s
+Step 2020/44000 | Loss: 3.7163 | LR: 4.99e-04 | 683.5ms/step | 29.3 samples/s
+Step 2030/44000 | Loss: 3.6928 | LR: 4.99e-04 | 680.5ms/step | 29.4 samples/s
+Step 2040/44000 | Loss: 3.7274 | LR: 4.99e-04 | 680.8ms/step | 29.4 samples/s
+Step 2050/44000 | Loss: 3.7085 | LR: 4.99e-04 | 680.4ms/step | 29.4 samples/s
+Step 2060/44000 | Loss: 3.7294 | LR: 4.99e-04 | 683.1ms/step | 29.3 samples/s
+Step 2070/44000 | Loss: 3.7066 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s
+Step 2080/44000 | Loss: 3.7279 | LR: 4.99e-04 | 679.7ms/step | 29.4 samples/s
+Step 2090/44000 | Loss: 3.7261 | LR: 4.99e-04 | 681.6ms/step | 29.3 samples/s
+Step 2100/44000 | Loss: 3.6805 | LR: 4.99e-04 | 681.8ms/step | 29.3 samples/s
+Step 2110/44000 | Loss: 3.7071 | LR: 4.99e-04 | 682.7ms/step | 29.3 samples/s
+Step 2120/44000 | Loss: 3.6959 | LR: 4.99e-04 | 681.3ms/step | 29.4 samples/s
+Step 2130/44000 | Loss: 3.6898 | LR: 4.99e-04 | 684.0ms/step | 29.2 samples/s
+Step 2140/44000 | Loss: 3.6711 | LR: 4.99e-04 | 681.0ms/step | 29.4 samples/s
+Step 2150/44000 | Loss: 3.6808 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s
+Step 2160/44000 | Loss: 3.6943 | LR: 4.99e-04 | 684.8ms/step | 29.2 samples/s
+Step 2170/44000 | Loss: 3.6677 | LR: 4.99e-04 | 683.1ms/step | 29.3 samples/s
+Step 2180/44000 | Loss: 3.6871 | LR: 4.99e-04 | 683.6ms/step | 29.3 samples/s
+Step 2190/44000 | Loss: 3.6678 | LR: 4.99e-04 | 684.5ms/step | 29.2 samples/s
+Step 2200/44000 | Loss: 3.6547 | LR: 4.99e-04 | 682.4ms/step | 29.3 samples/s
+Step 2210/44000 | Loss: 3.6378 | LR: 4.99e-04 | 682.6ms/step | 29.3 samples/s
+Step 2220/44000 | Loss: 3.6328 | LR: 4.99e-04 | 680.6ms/step | 29.4 samples/s
+Step 2230/44000 | Loss: 3.6869 | LR: 4.99e-04 | 682.1ms/step | 29.3 samples/s
+Step 2240/44000 | Loss: 3.6823 | LR: 4.99e-04 | 682.6ms/step | 29.3 samples/s
+Step 2250/44000 | Loss: 3.6684 | LR: 4.99e-04 | 684.5ms/step | 29.2 samples/s
+Validation at step 2250: Loss = 3.6606, Perplexity = 38.88
+Step 2260/44000 | Loss: 3.6578 | LR: 4.98e-04 | 1239.2ms/step | 16.1 samples/s
+Step 2270/44000 | Loss: 3.6413 | LR: 4.98e-04 | 681.9ms/step | 29.3 samples/s
+Step 2280/44000 | Loss: 3.6586 | LR: 4.98e-04 | 682.8ms/step | 29.3 samples/s
+Step 2290/44000 | Loss: 3.6776 | LR: 4.98e-04 | 684.0ms/step | 29.2 samples/s
+Step 2300/44000 | Loss: 3.6054 | LR: 4.98e-04 | 682.7ms/step | 29.3 samples/s
+Step 2310/44000 | Loss: 3.6456 | LR: 4.98e-04 | 683.2ms/step | 29.3 samples/s
+Step 2320/44000 | Loss: 3.6547 | LR: 4.98e-04 | 683.7ms/step | 29.3 samples/s
+Step 2330/44000 | Loss: 3.6483 | LR: 4.98e-04 | 683.6ms/step | 29.3 samples/s
+Step 2340/44000 | Loss: 3.6473 | LR: 4.98e-04 | 683.1ms/step | 29.3 samples/s
+Step 2350/44000 | Loss: 3.6125 | LR: 4.98e-04 | 682.8ms/step | 29.3 samples/s
+Step 2360/44000 | Loss: 3.6355 | LR: 4.98e-04 | 682.0ms/step | 29.3 samples/s
+Step 2370/44000 | Loss: 3.6366 | LR: 4.98e-04 | 679.0ms/step | 29.5 samples/s
+Step 2380/44000 | Loss: 3.5960 | LR: 4.98e-04 | 682.2ms/step | 29.3 samples/s
+Step 2390/44000 | Loss: 3.5817 | LR: 4.98e-04 | 681.4ms/step | 29.4 samples/s
+Step 2400/44000 | Loss: 3.6369 | LR: 4.98e-04 | 681.5ms/step | 29.3 samples/s
+Step 2410/44000 | Loss: 3.6284 | LR: 4.98e-04 | 680.6ms/step | 29.4 samples/s
+Step 2420/44000 | Loss: 3.6282 | LR: 4.98e-04 | 682.3ms/step | 29.3 samples/s
+Step 2430/44000 | Loss: 3.6212 | LR: 4.98e-04 | 681.0ms/step | 29.4 samples/s
+Step 2440/44000 | Loss: 3.6275 | LR: 4.98e-04 | 682.0ms/step | 29.3 samples/s
+Step 2450/44000 | Loss: 3.6034 | LR: 4.98e-04 | 680.9ms/step | 29.4 samples/s
+Step 2460/44000 | Loss: 3.6057 | LR: 4.98e-04 | 679.2ms/step | 29.4 samples/s
+Step 2470/44000 | Loss: 3.6119 | LR: 4.98e-04 | 682.4ms/step | 29.3 samples/s
+Step 2480/44000 | Loss: 3.6256 | LR: 4.98e-04 | 679.7ms/step | 29.4 samples/s
+Step 2490/44000 | Loss: 3.5894 | LR: 4.98e-04 | 680.6ms/step | 29.4 samples/s
+Step 2500/44000 | Loss: 3.6012 | LR: 4.98e-04 | 681.7ms/step | 29.3 samples/s
+Validation at step 2500: Loss = 3.6062, Perplexity = 36.83
+Step 2510/44000 | Loss: 3.6057 | LR: 4.98e-04 | 1247.7ms/step | 16.0 samples/s
+Step 2520/44000 | Loss: 3.5934 | LR: 4.98e-04 | 682.6ms/step | 29.3 samples/s
+Step 2530/44000 | Loss: 3.6051 | LR: 4.98e-04 | 683.6ms/step | 29.3 samples/s
+Step 2540/44000 | Loss: 3.5801 | LR: 4.98e-04 | 681.8ms/step | 29.3 samples/s
+Step 2550/44000 | Loss: 3.5463 | LR: 4.98e-04 | 680.3ms/step | 29.4 samples/s
+Step 2560/44000 | Loss: 3.5690 | LR: 4.98e-04 | 681.8ms/step | 29.3 samples/s
+Step 2570/44000 | Loss: 3.5804 | LR: 4.98e-04 | 683.3ms/step | 29.3 samples/s
+Step 2580/44000 | Loss: 3.6015 | LR: 4.98e-04 | 683.3ms/step | 29.3 samples/s
+Step 2590/44000 | Loss: 3.5855 | LR: 4.98e-04 | 681.3ms/step | 29.4 samples/s
+Step 2600/44000 | Loss: 3.5511 | LR: 4.98e-04 | 682.0ms/step | 29.3 samples/s
+Step 2610/44000 | Loss: 3.5176 | LR: 4.98e-04 | 682.8ms/step | 29.3 samples/s
+Step 2620/44000 | Loss: 3.5812 | LR: 4.98e-04 | 684.3ms/step | 29.2 samples/s
+Step 2630/44000 | Loss: 3.5415 | LR: 4.98e-04 | 681.5ms/step | 29.3 samples/s
+Step 2640/44000 | Loss: 3.5606 | LR: 4.98e-04 | 680.9ms/step | 29.4 samples/s
+Step 2650/44000 | Loss: 3.5750 | LR: 4.98e-04 | 683.7ms/step | 29.3 samples/s
+Step 2660/44000 | Loss: 3.5457 | LR: 4.98e-04 | 683.1ms/step | 29.3 samples/s
+Step 2670/44000 | Loss: 3.5489 | LR: 4.98e-04 | 683.9ms/step | 29.2 samples/s
+Step 2680/44000 | Loss: 3.5814 | LR: 4.98e-04 | 680.7ms/step | 29.4 samples/s
+Step 2690/44000 | Loss: 3.5940 | LR: 4.98e-04 | 682.2ms/step | 29.3 samples/s
+Step 2700/44000 | Loss: 3.5307 | LR: 4.97e-04 | 682.5ms/step | 29.3 samples/s
+Step 2710/44000 | Loss: 3.5850 | LR: 4.97e-04 | 682.3ms/step | 29.3 samples/s
+Step 2720/44000 | Loss: 3.5988 | LR: 4.97e-04 | 684.4ms/step | 29.2 samples/s
+Step 2730/44000 | Loss: 3.5787 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s
+Step 2740/44000 | Loss: 3.5510 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s
+Step 2750/44000 | Loss: 3.5302 | LR: 4.97e-04 | 682.7ms/step | 29.3 samples/s
+Validation at step 2750: Loss = 3.5617, Perplexity = 35.22
+Step 2760/44000 | Loss: 3.5235 | LR: 4.97e-04 | 1241.4ms/step | 16.1 samples/s
+Step 2770/44000 | Loss: 3.6002 | LR: 4.97e-04 | 682.0ms/step | 29.3 samples/s
+Step 2780/44000 | Loss: 3.5929 | LR: 4.97e-04 | 683.1ms/step | 29.3 samples/s
+Step 2790/44000 | Loss: 3.5685 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s
+Step 2800/44000 | Loss: 3.5706 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s
+Step 2810/44000 | Loss: 3.5281 | LR: 4.97e-04 | 683.2ms/step | 29.3 samples/s
+Step 2820/44000 | Loss: 3.5558 | LR: 4.97e-04 | 682.6ms/step | 29.3 samples/s
+Step 2830/44000 | Loss: 3.5401 | LR: 4.97e-04 | 683.1ms/step | 29.3 samples/s
+Step 2840/44000 | Loss: 3.5146 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s
+Step 2850/44000 | Loss: 3.5859 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s
+Step 2860/44000 | Loss: 3.5575 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s
+Step 2870/44000 | Loss: 3.5329 | LR: 4.97e-04 | 681.6ms/step | 29.3 samples/s
+Step 2880/44000 | Loss: 3.4896 | LR: 4.97e-04 | 681.3ms/step | 29.4 samples/s
+Step 2890/44000 | Loss: 3.5236 | LR: 4.97e-04 | 682.2ms/step | 29.3 samples/s
+Step 2900/44000 | Loss: 3.5765 | LR: 4.97e-04 | 682.9ms/step | 29.3 samples/s
+Step 2910/44000 | Loss: 3.5248 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s
+Step 2920/44000 | Loss: 3.5438 | LR: 4.97e-04 | 679.7ms/step | 29.4 samples/s
+Step 2930/44000 | Loss: 3.5284 | LR: 4.97e-04 | 681.1ms/step | 29.4 samples/s
+Step 2940/44000 | Loss: 3.5211 | LR: 4.97e-04 | 681.1ms/step | 29.4 samples/s
+Step 2950/44000 | Loss: 3.5509 | LR: 4.97e-04 | 681.7ms/step | 29.3 samples/s
+Step 2960/44000 | Loss: 3.5302 | LR: 4.97e-04 | 682.7ms/step | 29.3 samples/s
+Step 2970/44000 | Loss: 3.5520 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s
+Step 2980/44000 | Loss: 3.6119 | LR: 4.97e-04 | 680.9ms/step | 29.4 samples/s
+Step 2990/44000 | Loss: 3.5136 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s
+Step 3000/44000 | Loss: 3.5309 | LR: 4.97e-04 | 683.2ms/step | 29.3 samples/s
+Validation at step 3000: Loss = 3.5276, Perplexity = 34.04
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-3000
+Step 3010/44000 | Loss: 3.5184 | LR: 4.97e-04 | 1522.3ms/step | 13.1 samples/s
+Step 3020/44000 | Loss: 3.5128 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s
+Step 3030/44000 | Loss: 3.5592 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s
+Step 3040/44000 | Loss: 3.5153 | LR: 4.97e-04 | 680.5ms/step | 29.4 samples/s
+Step 3050/44000 | Loss: 3.5552 | LR: 4.97e-04 | 682.6ms/step | 29.3 samples/s
+Step 3060/44000 | Loss: 3.4928 | LR: 4.96e-04 | 683.5ms/step | 29.3 samples/s
+Step 3070/44000 | Loss: 3.5235 | LR: 4.96e-04 | 681.3ms/step | 29.4 samples/s
+Step 3080/44000 | Loss: 3.5322 | LR: 4.96e-04 | 680.0ms/step | 29.4 samples/s
+Step 3090/44000 | Loss: 3.4932 | LR: 4.96e-04 | 681.9ms/step | 29.3 samples/s
+Step 3100/44000 | Loss: 3.5170 | LR: 4.96e-04 | 681.1ms/step | 29.4 samples/s
+Step 3110/44000 | Loss: 3.4852 | LR: 4.96e-04 | 681.0ms/step | 29.4 samples/s
+Step 3120/44000 | Loss: 3.4994 | LR: 4.96e-04 | 680.1ms/step | 29.4 samples/s
+Step 3130/44000 | Loss: 3.4935 | LR: 4.96e-04 | 680.4ms/step | 29.4 samples/s
+Step 3140/44000 | Loss: 3.4892 | LR: 4.96e-04 | 680.6ms/step | 29.4 samples/s
+Step 3150/44000 | Loss: 3.4834 | LR: 4.96e-04 | 680.2ms/step | 29.4 samples/s
+Step 3160/44000 | Loss: 3.5026 | LR: 4.96e-04 | 680.4ms/step | 29.4 samples/s
+Step 3170/44000 | Loss: 3.4916 | LR: 4.96e-04 | 680.2ms/step | 29.4 samples/s
+Step 3180/44000 | Loss: 3.5293 | LR: 4.96e-04 | 681.0ms/step | 29.4 samples/s
+Step 3190/44000 | Loss: 3.5257 | LR: 4.96e-04 | 681.1ms/step | 29.4 samples/s
+Step 3200/44000 | Loss: 3.4865 | LR: 4.96e-04 | 681.5ms/step | 29.3 samples/s
+Step 3210/44000 | Loss: 3.4721 | LR: 4.96e-04 | 680.4ms/step | 29.4 samples/s
+Step 3220/44000 | Loss: 3.5098 | LR: 4.96e-04 | 682.8ms/step | 29.3 samples/s
+Step 3230/44000 | Loss: 3.4769 | LR: 4.96e-04 | 681.6ms/step | 29.3 samples/s
+Step 3240/44000 | Loss: 3.4926 | LR: 4.96e-04 | 680.8ms/step | 29.4 samples/s
+Step 3250/44000 | Loss: 3.4725 | LR: 4.96e-04 | 679.6ms/step | 29.4 samples/s
+Validation at step 3250: Loss = 3.4923, Perplexity = 32.86
+Step 3260/44000 | Loss: 3.5206 | LR: 4.96e-04 | 1247.0ms/step | 16.0 samples/s
+Step 3270/44000 | Loss: 3.4743 | LR: 4.96e-04 | 682.1ms/step | 29.3 samples/s
+Step 3280/44000 | Loss: 3.4761 | LR: 4.96e-04 | 683.5ms/step | 29.3 samples/s
+Step 3290/44000 | Loss: 3.4805 | LR: 4.96e-04 | 683.7ms/step | 29.3 samples/s
+Step 3300/44000 | Loss: 3.4668 | LR: 4.96e-04 | 682.3ms/step | 29.3 samples/s
+Step 3310/44000 | Loss: 3.4832 | LR: 4.96e-04 | 681.9ms/step | 29.3 samples/s
+Step 3320/44000 | Loss: 3.4688 | LR: 4.96e-04 | 681.0ms/step | 29.4 samples/s
+Step 3330/44000 | Loss: 3.4872 | LR: 4.96e-04 | 679.4ms/step | 29.4 samples/s
+Step 3340/44000 | Loss: 3.4631 | LR: 4.96e-04 | 681.3ms/step | 29.4 samples/s
+Step 3350/44000 | Loss: 3.4930 | LR: 4.96e-04 | 682.6ms/step | 29.3 samples/s
+Step 3360/44000 | Loss: 3.4527 | LR: 4.96e-04 | 683.3ms/step | 29.3 samples/s
+Step 3370/44000 | Loss: 3.4798 | LR: 4.95e-04 | 683.8ms/step | 29.3 samples/s
+Step 3380/44000 | Loss: 3.4679 | LR: 4.95e-04 | 682.5ms/step | 29.3 samples/s
+Step 3390/44000 | Loss: 3.4798 | LR: 4.95e-04 | 684.0ms/step | 29.2 samples/s
+Step 3400/44000 | Loss: 3.4594 | LR: 4.95e-04 | 682.8ms/step | 29.3 samples/s
+Step 3410/44000 | Loss: 3.4802 | LR: 4.95e-04 | 680.9ms/step | 29.4 samples/s
+Step 3420/44000 | Loss: 3.3880 | LR: 4.95e-04 | 681.2ms/step | 29.4 samples/s
+Step 3430/44000 | Loss: 3.4410 | LR: 4.95e-04 | 679.7ms/step | 29.4 samples/s
+Step 3440/44000 | Loss: 3.4704 | LR: 4.95e-04 | 682.3ms/step | 29.3 samples/s
+Step 3450/44000 | Loss: 3.4364 | LR: 4.95e-04 | 681.3ms/step | 29.4 samples/s
+Step 3460/44000 | Loss: 3.4823 | LR: 4.95e-04 | 682.2ms/step | 29.3 samples/s
+Step 3470/44000 | Loss: 3.4830 | LR: 4.95e-04 | 681.6ms/step | 29.3 samples/s
+Step 3480/44000 | Loss: 3.4622 | LR: 4.95e-04 | 683.7ms/step | 29.3 samples/s
+Step 3490/44000 | Loss: 3.4707 | LR: 4.95e-04 | 681.4ms/step | 29.4 samples/s
+Step 3500/44000 | Loss: 3.4223 | LR: 4.95e-04 | 682.5ms/step | 29.3 samples/s
+Validation at step 3500: Loss = 3.4642, Perplexity = 31.95
+Step 3510/44000 | Loss: 3.4733 | LR: 4.95e-04 | 1252.3ms/step | 16.0 samples/s
+Step 3520/44000 | Loss: 3.4341 | LR: 4.95e-04 | 685.1ms/step | 29.2 samples/s
+Step 3530/44000 | Loss: 3.4105 | LR: 4.95e-04 | 683.0ms/step | 29.3 samples/s
+Step 3540/44000 | Loss: 3.4402 | LR: 4.95e-04 | 682.7ms/step | 29.3 samples/s
+Step 3550/44000 | Loss: 3.4576 | LR: 4.95e-04 | 682.6ms/step | 29.3 samples/s
+Step 3560/44000 | Loss: 3.4948 | LR: 4.95e-04 | 681.3ms/step | 29.4 samples/s
+Step 3570/44000 | Loss: 3.4166 | LR: 4.95e-04 | 681.1ms/step | 29.4 samples/s
+Step 3580/44000 | Loss: 3.4434 | LR: 4.95e-04 | 681.7ms/step | 29.3 samples/s
+Step 3590/44000 | Loss: 3.4437 | LR: 4.95e-04 | 682.6ms/step | 29.3 samples/s
+Step 3600/44000 | Loss: 3.4644 | LR: 4.95e-04 | 682.3ms/step | 29.3 samples/s
+Step 3610/44000 | Loss: 3.4921 | LR: 4.95e-04 | 683.2ms/step | 29.3 samples/s
+Step 3620/44000 | Loss: 3.4481 | LR: 4.95e-04 | 681.9ms/step | 29.3 samples/s
+Step 3630/44000 | Loss: 3.4289 | LR: 4.95e-04 | 681.9ms/step | 29.3 samples/s
+Step 3640/44000 | Loss: 3.4517 | LR: 4.95e-04 | 681.0ms/step | 29.4 samples/s
+Step 3650/44000 | Loss: 3.4796 | LR: 4.94e-04 | 680.6ms/step | 29.4 samples/s
+Step 3660/44000 | Loss: 3.4184 | LR: 4.94e-04 | 680.2ms/step | 29.4 samples/s
+Step 3670/44000 | Loss: 3.4607 | LR: 4.94e-04 | 680.8ms/step | 29.4 samples/s
+Step 3680/44000 | Loss: 3.4027 | LR: 4.94e-04 | 680.4ms/step | 29.4 samples/s
+Step 3690/44000 | Loss: 3.4122 | LR: 4.94e-04 | 679.8ms/step | 29.4 samples/s
+Step 3700/44000 | Loss: 3.4601 | LR: 4.94e-04 | 679.5ms/step | 29.4 samples/s
+Step 3710/44000 | Loss: 3.4568 | LR: 4.94e-04 | 681.4ms/step | 29.4 samples/s
+Step 3720/44000 | Loss: 3.4345 | LR: 4.94e-04 | 681.3ms/step | 29.4 samples/s
+Step 3730/44000 | Loss: 3.4070 | LR: 4.94e-04 | 682.4ms/step | 29.3 samples/s
+Step 3740/44000 | Loss: 3.4669 | LR: 4.94e-04 | 683.0ms/step | 29.3 samples/s
+Step 3750/44000 | Loss: 3.4222 | LR: 4.94e-04 | 681.6ms/step | 29.3 samples/s
+Validation at step 3750: Loss = 3.4390, Perplexity = 31.16
+Step 3760/44000 | Loss: 3.4512 | LR: 4.94e-04 | 1249.4ms/step | 16.0 samples/s
+Step 3770/44000 | Loss: 3.4291 | LR: 4.94e-04 | 682.9ms/step | 29.3 samples/s
+Step 3780/44000 | Loss: 3.4285 | LR: 4.94e-04 | 682.6ms/step | 29.3 samples/s
+Step 3790/44000 | Loss: 3.4337 | LR: 4.94e-04 | 683.0ms/step | 29.3 samples/s
+Step 3800/44000 | Loss: 3.4043 | LR: 4.94e-04 | 683.9ms/step | 29.2 samples/s
+Step 3810/44000 | Loss: 3.3895 | LR: 4.94e-04 | 682.0ms/step | 29.3 samples/s
+Step 3820/44000 | Loss: 3.3728 | LR: 4.94e-04 | 681.3ms/step | 29.4 samples/s
+Step 3830/44000 | Loss: 3.4081 | LR: 4.94e-04 | 683.8ms/step | 29.2 samples/s
+Step 3840/44000 | Loss: 3.4717 | LR: 4.94e-04 | 683.2ms/step | 29.3 samples/s
+Step 3850/44000 | Loss: 3.4292 | LR: 4.94e-04 | 683.5ms/step | 29.3 samples/s
+Step 3860/44000 | Loss: 3.4177 | LR: 4.94e-04 | 681.0ms/step | 29.4 samples/s
+Step 3870/44000 | Loss: 3.4681 | LR: 4.94e-04 | 682.8ms/step | 29.3 samples/s
+Step 3880/44000 | Loss: 3.4396 | LR: 4.94e-04 | 685.1ms/step | 29.2 samples/s
+Step 3890/44000 | Loss: 3.4616 | LR: 4.94e-04 | 681.3ms/step | 29.4 samples/s
+Step 3900/44000 | Loss: 3.4096 | LR: 4.93e-04 | 681.0ms/step | 29.4 samples/s
+Step 3910/44000 | Loss: 3.4503 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s
+Step 3920/44000 | Loss: 3.4124 | LR: 4.93e-04 | 679.4ms/step | 29.4 samples/s
+Step 3930/44000 | Loss: 3.4170 | LR: 4.93e-04 | 680.3ms/step | 29.4 samples/s
+Step 3940/44000 | Loss: 3.4201 | LR: 4.93e-04 | 679.7ms/step | 29.4 samples/s
+Step 3950/44000 | Loss: 3.3592 | LR: 4.93e-04 | 679.5ms/step | 29.4 samples/s
+Step 3960/44000 | Loss: 3.4302 | LR: 4.93e-04 | 679.4ms/step | 29.4 samples/s
+Step 3970/44000 | Loss: 3.4533 | LR: 4.93e-04 | 679.9ms/step | 29.4 samples/s
+Step 3980/44000 | Loss: 3.3968 | LR: 4.93e-04 | 678.2ms/step | 29.5 samples/s
+Step 3990/44000 | Loss: 3.3934 | LR: 4.93e-04 | 677.5ms/step | 29.5 samples/s
+Step 4000/44000 | Loss: 3.4114 | LR: 4.93e-04 | 677.6ms/step | 29.5 samples/s
+Validation at step 4000: Loss = 3.4178, Perplexity = 30.50
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-4000
+Step 4010/44000 | Loss: 3.4281 | LR: 4.93e-04 | 1525.1ms/step | 13.1 samples/s
+Step 4020/44000 | Loss: 3.4596 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s
+Step 4030/44000 | Loss: 3.3729 | LR: 4.93e-04 | 678.4ms/step | 29.5 samples/s
+Step 4040/44000 | Loss: 3.3798 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s
+Step 4050/44000 | Loss: 3.3952 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s
+Step 4060/44000 | Loss: 3.3878 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s
+Step 4070/44000 | Loss: 3.4152 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s
+Step 4080/44000 | Loss: 3.4150 | LR: 4.93e-04 | 678.4ms/step | 29.5 samples/s
+Step 4090/44000 | Loss: 3.4078 | LR: 4.93e-04 | 679.0ms/step | 29.5 samples/s
+Step 4100/44000 | Loss: 3.3983 | LR: 4.93e-04 | 680.2ms/step | 29.4 samples/s
+Step 4110/44000 | Loss: 3.4223 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s
+Step 4120/44000 | Loss: 3.4187 | LR: 4.93e-04 | 678.8ms/step | 29.5 samples/s
+Step 4130/44000 | Loss: 3.4097 | LR: 4.93e-04 | 679.2ms/step | 29.4 samples/s
+Step 4140/44000 | Loss: 3.4083 | LR: 4.92e-04 | 678.3ms/step | 29.5 samples/s
+Step 4150/44000 | Loss: 3.3893 | LR: 4.92e-04 | 679.5ms/step | 29.4 samples/s
+Step 4160/44000 | Loss: 3.4045 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Step 4170/44000 | Loss: 3.3740 | LR: 4.92e-04 | 677.5ms/step | 29.5 samples/s
+Step 4180/44000 | Loss: 3.3596 | LR: 4.92e-04 | 679.7ms/step | 29.4 samples/s
+Step 4190/44000 | Loss: 3.4140 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s
+Step 4200/44000 | Loss: 3.3799 | LR: 4.92e-04 | 679.7ms/step | 29.4 samples/s
+Step 4210/44000 | Loss: 3.4241 | LR: 4.92e-04 | 680.1ms/step | 29.4 samples/s
+Step 4220/44000 | Loss: 3.3931 | LR: 4.92e-04 | 679.3ms/step | 29.4 samples/s
+Step 4230/44000 | Loss: 3.4336 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Step 4240/44000 | Loss: 3.3840 | LR: 4.92e-04 | 679.9ms/step | 29.4 samples/s
+Step 4250/44000 | Loss: 3.4068 | LR: 4.92e-04 | 679.6ms/step | 29.4 samples/s
+Validation at step 4250: Loss = 3.3950, Perplexity = 29.81
+Step 4260/44000 | Loss: 3.3487 | LR: 4.92e-04 | 1233.9ms/step | 16.2 samples/s
+Step 4270/44000 | Loss: 3.3628 | LR: 4.92e-04 | 678.4ms/step | 29.5 samples/s
+Step 4280/44000 | Loss: 3.4044 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s
+Step 4290/44000 | Loss: 3.3746 | LR: 4.92e-04 | 680.6ms/step | 29.4 samples/s
+Step 4300/44000 | Loss: 3.4053 | LR: 4.92e-04 | 680.0ms/step | 29.4 samples/s
+Step 4310/44000 | Loss: 3.3666 | LR: 4.92e-04 | 680.5ms/step | 29.4 samples/s
+Step 4320/44000 | Loss: 3.3756 | LR: 4.92e-04 | 679.0ms/step | 29.5 samples/s
+Step 4330/44000 | Loss: 3.3588 | LR: 4.92e-04 | 679.5ms/step | 29.4 samples/s
+Step 4340/44000 | Loss: 3.3874 | LR: 4.92e-04 | 679.3ms/step | 29.4 samples/s
+Step 4350/44000 | Loss: 3.3448 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s
+Step 4360/44000 | Loss: 3.3811 | LR: 4.91e-04 | 678.5ms/step | 29.5 samples/s
+Step 4370/44000 | Loss: 3.3967 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s
+Step 4380/44000 | Loss: 3.3458 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s
+Step 4390/44000 | Loss: 3.3588 | LR: 4.91e-04 | 678.0ms/step | 29.5 samples/s
+Step 4400/44000 | Loss: 3.4147 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s
+Step 4410/44000 | Loss: 3.3854 | LR: 4.91e-04 | 680.7ms/step | 29.4 samples/s
+Step 4420/44000 | Loss: 3.3861 | LR: 4.91e-04 | 678.9ms/step | 29.5 samples/s
+Step 4430/44000 | Loss: 3.3763 | LR: 4.91e-04 | 678.9ms/step | 29.5 samples/s
+Step 4440/44000 | Loss: 3.3891 | LR: 4.91e-04 | 679.0ms/step | 29.5 samples/s
+Step 4450/44000 | Loss: 3.3627 | LR: 4.91e-04 | 679.6ms/step | 29.4 samples/s
+Step 4460/44000 | Loss: 3.3368 | LR: 4.91e-04 | 679.9ms/step | 29.4 samples/s
+Step 4470/44000 | Loss: 3.4033 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s
+Step 4480/44000 | Loss: 3.3707 | LR: 4.91e-04 | 679.3ms/step | 29.4 samples/s
+Step 4490/44000 | Loss: 3.3335 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s
+Step 4500/44000 | Loss: 3.3609 | LR: 4.91e-04 | 678.6ms/step | 29.5 samples/s
+Validation at step 4500: Loss = 3.3769, Perplexity = 29.28
+Step 4510/44000 | Loss: 3.3556 | LR: 4.91e-04 | 1234.8ms/step | 16.2 samples/s
+Step 4520/44000 | Loss: 3.3521 | LR: 4.91e-04 | 679.6ms/step | 29.4 samples/s
+Step 4530/44000 | Loss: 3.3342 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s
+Step 4540/44000 | Loss: 3.3765 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s
+Step 4550/44000 | Loss: 3.3709 | LR: 4.91e-04 | 678.2ms/step | 29.5 samples/s
+Step 4560/44000 | Loss: 3.3867 | LR: 4.90e-04 | 678.6ms/step | 29.5 samples/s
+Step 4570/44000 | Loss: 3.3832 | LR: 4.90e-04 | 677.5ms/step | 29.5 samples/s
+Step 4580/44000 | Loss: 3.3673 | LR: 4.90e-04 | 678.4ms/step | 29.5 samples/s
+Step 4590/44000 | Loss: 3.3171 | LR: 4.90e-04 | 678.3ms/step | 29.5 samples/s
+Step 4600/44000 | Loss: 3.3516 | LR: 4.90e-04 | 679.5ms/step | 29.4 samples/s
+Step 4610/44000 | Loss: 3.3609 | LR: 4.90e-04 | 679.1ms/step | 29.5 samples/s
+Step 4620/44000 | Loss: 3.3258 | LR: 4.90e-04 | 677.5ms/step | 29.5 samples/s
+Step 4630/44000 | Loss: 3.3486 | LR: 4.90e-04 | 678.4ms/step | 29.5 samples/s
+Step 4640/44000 | Loss: 3.3631 | LR: 4.90e-04 | 677.2ms/step | 29.5 samples/s
+Step 4650/44000 | Loss: 3.3265 | LR: 4.90e-04 | 678.7ms/step | 29.5 samples/s
+Step 4660/44000 | Loss: 3.3211 | LR: 4.90e-04 | 679.8ms/step | 29.4 samples/s
+Step 4670/44000 | Loss: 3.3957 | LR: 4.90e-04 | 680.1ms/step | 29.4 samples/s
+Step 4680/44000 | Loss: 3.3563 | LR: 4.90e-04 | 678.8ms/step | 29.5 samples/s
+Step 4690/44000 | Loss: 3.4132 | LR: 4.90e-04 | 679.6ms/step | 29.4 samples/s
+Step 4700/44000 | Loss: 3.3966 | LR: 4.90e-04 | 679.9ms/step | 29.4 samples/s
+Step 4710/44000 | Loss: 3.3839 | LR: 4.90e-04 | 679.2ms/step | 29.4 samples/s
+Step 4720/44000 | Loss: 3.3321 | LR: 4.90e-04 | 678.9ms/step | 29.5 samples/s
+Step 4730/44000 | Loss: 3.3661 | LR: 4.90e-04 | 679.5ms/step | 29.4 samples/s
+Step 4740/44000 | Loss: 3.3577 | LR: 4.90e-04 | 680.3ms/step | 29.4 samples/s
+Step 4750/44000 | Loss: 3.3139 | LR: 4.90e-04 | 679.6ms/step | 29.4 samples/s
+Validation at step 4750: Loss = 3.3614, Perplexity = 28.83
+Step 4760/44000 | Loss: 3.3747 | LR: 4.89e-04 | 1233.8ms/step | 16.2 samples/s
+Step 4770/44000 | Loss: 3.3652 | LR: 4.89e-04 | 679.9ms/step | 29.4 samples/s
+Step 4780/44000 | Loss: 3.3694 | LR: 4.89e-04 | 679.8ms/step | 29.4 samples/s
+Step 4790/44000 | Loss: 3.3512 | LR: 4.89e-04 | 680.1ms/step | 29.4 samples/s
+Step 4800/44000 | Loss: 3.3583 | LR: 4.89e-04 | 679.8ms/step | 29.4 samples/s
+Step 4810/44000 | Loss: 3.3428 | LR: 4.89e-04 | 680.2ms/step | 29.4 samples/s
+Step 4820/44000 | Loss: 3.3505 | LR: 4.89e-04 | 679.0ms/step | 29.5 samples/s
+Step 4830/44000 | Loss: 3.3398 | LR: 4.89e-04 | 679.3ms/step | 29.4 samples/s
+Step 4840/44000 | Loss: 3.3197 | LR: 4.89e-04 | 680.0ms/step | 29.4 samples/s
+Step 4850/44000 | Loss: 3.3333 | LR: 4.89e-04 | 681.2ms/step | 29.4 samples/s
+Step 4860/44000 | Loss: 3.3329 | LR: 4.89e-04 | 678.1ms/step | 29.5 samples/s
+Step 4870/44000 | Loss: 3.3272 | LR: 4.89e-04 | 678.5ms/step | 29.5 samples/s
+Step 4880/44000 | Loss: 3.3495 | LR: 4.89e-04 | 679.3ms/step | 29.4 samples/s
+Step 4890/44000 | Loss: 3.3169 | LR: 4.89e-04 | 678.7ms/step | 29.5 samples/s
+Step 4900/44000 | Loss: 3.3860 | LR: 4.89e-04 | 678.3ms/step | 29.5 samples/s
+Step 4910/44000 | Loss: 3.3662 | LR: 4.89e-04 | 678.1ms/step | 29.5 samples/s
+Step 4920/44000 | Loss: 3.3651 | LR: 4.89e-04 | 677.5ms/step | 29.5 samples/s
+Step 4930/44000 | Loss: 3.3539 | LR: 4.89e-04 | 678.1ms/step | 29.5 samples/s
+Step 4940/44000 | Loss: 3.3554 | LR: 4.89e-04 | 678.7ms/step | 29.5 samples/s
+Step 4950/44000 | Loss: 3.3109 | LR: 4.88e-04 | 680.9ms/step | 29.4 samples/s
+Step 4960/44000 | Loss: 3.3611 | LR: 4.88e-04 | 679.2ms/step | 29.4 samples/s
+Step 4970/44000 | Loss: 3.3116 | LR: 4.88e-04 | 679.0ms/step | 29.5 samples/s
+Step 4980/44000 | Loss: 3.3370 | LR: 4.88e-04 | 678.5ms/step | 29.5 samples/s
+Step 4990/44000 | Loss: 3.3416 | LR: 4.88e-04 | 679.5ms/step | 29.4 samples/s
+Step 5000/44000 | Loss: 3.3401 | LR: 4.88e-04 | 679.2ms/step | 29.4 samples/s
+Validation at step 5000: Loss = 3.3442, Perplexity = 28.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-5000
+Step 5010/44000 | Loss: 3.3201 | LR: 4.88e-04 | 1521.3ms/step | 13.1 samples/s
+Step 5020/44000 | Loss: 3.3306 | LR: 4.88e-04 | 678.3ms/step | 29.5 samples/s
+Step 5030/44000 | Loss: 3.3327 | LR: 4.88e-04 | 677.6ms/step | 29.5 samples/s
+Step 5040/44000 | Loss: 3.3609 | LR: 4.88e-04 | 678.3ms/step | 29.5 samples/s
+Step 5050/44000 | Loss: 3.3322 | LR: 4.88e-04 | 677.3ms/step | 29.5 samples/s
+Step 5060/44000 | Loss: 3.3437 | LR: 4.88e-04 | 679.3ms/step | 29.4 samples/s
+Step 5070/44000 | Loss: 3.3212 | LR: 4.88e-04 | 679.2ms/step | 29.4 samples/s
+Step 5080/44000 | Loss: 3.3374 | LR: 4.88e-04 | 679.5ms/step | 29.4 samples/s
+Step 5090/44000 | Loss: 3.3319 | LR: 4.88e-04 | 679.7ms/step | 29.4 samples/s
+Step 5100/44000 | Loss: 3.3516 | LR: 4.88e-04 | 679.6ms/step | 29.4 samples/s
+Step 5110/44000 | Loss: 3.3098 | LR: 4.88e-04 | 679.5ms/step | 29.4 samples/s
+Step 5120/44000 | Loss: 3.3821 | LR: 4.88e-04 | 678.1ms/step | 29.5 samples/s
+Step 5130/44000 | Loss: 3.3418 | LR: 4.87e-04 | 679.4ms/step | 29.4 samples/s
+Step 5140/44000 | Loss: 3.3210 | LR: 4.87e-04 | 679.5ms/step | 29.4 samples/s
+Step 5150/44000 | Loss: 3.3319 | LR: 4.87e-04 | 679.4ms/step | 29.4 samples/s
+Step 5160/44000 | Loss: 3.3373 | LR: 4.87e-04 | 679.4ms/step | 29.4 samples/s
+Step 5170/44000 | Loss: 3.2874 | LR: 4.87e-04 | 678.7ms/step | 29.5 samples/s
+Step 5180/44000 | Loss: 3.3305 | LR: 4.87e-04 | 678.4ms/step | 29.5 samples/s
+Step 5190/44000 | Loss: 3.3494 | LR: 4.87e-04 | 678.4ms/step | 29.5 samples/s
+Step 5200/44000 | Loss: 3.3278 | LR: 4.87e-04 | 678.9ms/step | 29.5 samples/s
+Step 5210/44000 | Loss: 3.3478 | LR: 4.87e-04 | 677.8ms/step | 29.5 samples/s
+Step 5220/44000 | Loss: 3.3379 | LR: 4.87e-04 | 678.1ms/step | 29.5 samples/s
+Step 5230/44000 | Loss: 3.3458 | LR: 4.87e-04 | 678.0ms/step | 29.5 samples/s
+Step 5240/44000 | Loss: 3.3979 | LR: 4.87e-04 | 677.1ms/step | 29.5 samples/s
+Step 5250/44000 | Loss: 3.3364 | LR: 4.87e-04 | 678.5ms/step | 29.5 samples/s
+Validation at step 5250: Loss = 3.3295, Perplexity = 27.93
+Step 5260/44000 | Loss: 3.3595 | LR: 4.87e-04 | 1234.7ms/step | 16.2 samples/s
+Step 5270/44000 | Loss: 3.3485 | LR: 4.87e-04 | 677.6ms/step | 29.5 samples/s
+Step 5280/44000 | Loss: 3.3431 | LR: 4.87e-04 | 678.4ms/step | 29.5 samples/s
+Step 5290/44000 | Loss: 3.3174 | LR: 4.87e-04 | 679.0ms/step | 29.5 samples/s
+Step 5300/44000 | Loss: 3.3188 | LR: 4.86e-04 | 681.3ms/step | 29.4 samples/s
+Step 5310/44000 | Loss: 3.3468 | LR: 4.86e-04 | 679.8ms/step | 29.4 samples/s
+Step 5320/44000 | Loss: 3.2971 | LR: 4.86e-04 | 678.9ms/step | 29.5 samples/s
+Step 5330/44000 | Loss: 3.3454 | LR: 4.86e-04 | 679.3ms/step | 29.4 samples/s
+Step 5340/44000 | Loss: 3.3099 | LR: 4.86e-04 | 679.7ms/step | 29.4 samples/s
+Step 5350/44000 | Loss: 3.3012 | LR: 4.86e-04 | 680.3ms/step | 29.4 samples/s
+Step 5360/44000 | Loss: 3.3069 | LR: 4.86e-04 | 679.6ms/step | 29.4 samples/s
+Step 5370/44000 | Loss: 3.3328 | LR: 4.86e-04 | 680.9ms/step | 29.4 samples/s
+Step 5380/44000 | Loss: 3.3176 | LR: 4.86e-04 | 679.3ms/step | 29.4 samples/s
+Step 5390/44000 | Loss: 3.2883 | LR: 4.86e-04 | 680.2ms/step | 29.4 samples/s
+Step 5400/44000 | Loss: 3.3253 | LR: 4.86e-04 | 679.8ms/step | 29.4 samples/s
+Step 5410/44000 | Loss: 3.2998 | LR: 4.86e-04 | 679.2ms/step | 29.4 samples/s
+Step 5420/44000 | Loss: 3.3220 | LR: 4.86e-04 | 679.0ms/step | 29.5 samples/s
+Step 5430/44000 | Loss: 3.3162 | LR: 4.86e-04 | 679.4ms/step | 29.4 samples/s
+Step 5440/44000 | Loss: 3.3488 | LR: 4.86e-04 | 678.1ms/step | 29.5 samples/s
+Step 5450/44000 | Loss: 3.2814 | LR: 4.86e-04 | 679.2ms/step | 29.4 samples/s
+Step 5460/44000 | Loss: 3.2891 | LR: 4.86e-04 | 678.3ms/step | 29.5 samples/s
+Step 5470/44000 | Loss: 3.2895 | LR: 4.85e-04 | 678.2ms/step | 29.5 samples/s
+Step 5480/44000 | Loss: 3.3356 | LR: 4.85e-04 | 678.8ms/step | 29.5 samples/s
+Step 5490/44000 | Loss: 3.3519 | LR: 4.85e-04 | 678.6ms/step | 29.5 samples/s
+Step 5500/44000 | Loss: 3.3232 | LR: 4.85e-04 | 679.5ms/step | 29.4 samples/s
+Validation at step 5500: Loss = 3.3156, Perplexity = 27.54
+Step 5510/44000 | Loss: 3.2666 | LR: 4.85e-04 | 1233.2ms/step | 16.2 samples/s
+Step 5520/44000 | Loss: 3.2916 | LR: 4.85e-04 | 678.6ms/step | 29.5 samples/s
+Step 5530/44000 | Loss: 3.2834 | LR: 4.85e-04 | 678.9ms/step | 29.5 samples/s
+Step 5540/44000 | Loss: 3.2878 | LR: 4.85e-04 | 679.0ms/step | 29.5 samples/s
+Step 5550/44000 | Loss: 3.3214 | LR: 4.85e-04 | 679.9ms/step | 29.4 samples/s
+Step 5560/44000 | Loss: 3.3214 | LR: 4.85e-04 | 678.2ms/step | 29.5 samples/s
+Step 5570/44000 | Loss: 3.3011 | LR: 4.85e-04 | 678.8ms/step | 29.5 samples/s
+Step 5580/44000 | Loss: 3.3161 | LR: 4.85e-04 | 679.8ms/step | 29.4 samples/s
+Step 5590/44000 | Loss: 3.3378 | LR: 4.85e-04 | 679.8ms/step | 29.4 samples/s
+Step 5600/44000 | Loss: 3.3209 | LR: 4.85e-04 | 678.2ms/step | 29.5 samples/s
+Step 5610/44000 | Loss: 3.2975 | LR: 4.85e-04 | 679.3ms/step | 29.4 samples/s
+Step 5620/44000 | Loss: 3.3005 | LR: 4.85e-04 | 678.3ms/step | 29.5 samples/s
+Step 5630/44000 | Loss: 3.2914 | LR: 4.84e-04 | 678.4ms/step | 29.5 samples/s
+Step 5640/44000 | Loss: 3.3362 | LR: 4.84e-04 | 678.0ms/step | 29.5 samples/s
+Step 5650/44000 | Loss: 3.2457 | LR: 4.84e-04 | 678.3ms/step | 29.5 samples/s
+Step 5660/44000 | Loss: 3.2908 | LR: 4.84e-04 | 678.8ms/step | 29.5 samples/s
+Step 5670/44000 | Loss: 3.3123 | LR: 4.84e-04 | 678.7ms/step | 29.5 samples/s
+Step 5680/44000 | Loss: 3.2936 | LR: 4.84e-04 | 680.3ms/step | 29.4 samples/s
+Step 5690/44000 | Loss: 3.2909 | LR: 4.84e-04 | 679.8ms/step | 29.4 samples/s
+Step 5700/44000 | Loss: 3.2340 | LR: 4.84e-04 | 679.9ms/step | 29.4 samples/s
+Step 5710/44000 | Loss: 3.3004 | LR: 4.84e-04 | 680.5ms/step | 29.4 samples/s
+Step 5720/44000 | Loss: 3.3141 | LR: 4.84e-04 | 679.5ms/step | 29.4 samples/s
+Step 5730/44000 | Loss: 3.2876 | LR: 4.84e-04 | 679.8ms/step | 29.4 samples/s
+Step 5740/44000 | Loss: 3.2518 | LR: 4.84e-04 | 679.4ms/step | 29.4 samples/s
+Step 5750/44000 | Loss: 3.2692 | LR: 4.84e-04 | 679.0ms/step | 29.5 samples/s
+Validation at step 5750: Loss = 3.3021, Perplexity = 27.17
+Step 5760/44000 | Loss: 3.3389 | LR: 4.84e-04 | 1232.6ms/step | 16.2 samples/s
+Step 5770/44000 | Loss: 3.3094 | LR: 4.84e-04 | 679.0ms/step | 29.5 samples/s
+Step 5780/44000 | Loss: 3.2895 | LR: 4.83e-04 | 679.7ms/step | 29.4 samples/s
+Step 5790/44000 | Loss: 3.2590 | LR: 4.83e-04 | 681.2ms/step | 29.4 samples/s
+Step 5800/44000 | Loss: 3.3042 | LR: 4.83e-04 | 678.2ms/step | 29.5 samples/s
+Step 5810/44000 | Loss: 3.3517 | LR: 4.83e-04 | 680.2ms/step | 29.4 samples/s
+Step 5820/44000 | Loss: 3.3069 | LR: 4.83e-04 | 678.7ms/step | 29.5 samples/s
+Step 5830/44000 | Loss: 3.3161 | LR: 4.83e-04 | 679.4ms/step | 29.4 samples/s
+Step 5840/44000 | Loss: 3.2882 | LR: 4.83e-04 | 679.3ms/step | 29.4 samples/s
+Step 5850/44000 | Loss: 3.2929 | LR: 4.83e-04 | 677.5ms/step | 29.5 samples/s
+Step 5860/44000 | Loss: 3.2733 | LR: 4.83e-04 | 679.7ms/step | 29.4 samples/s
+Step 5870/44000 | Loss: 3.2655 | LR: 4.83e-04 | 679.0ms/step | 29.5 samples/s
+Step 5880/44000 | Loss: 3.3012 | LR: 4.83e-04 | 678.2ms/step | 29.5 samples/s
+Step 5890/44000 | Loss: 3.3211 | LR: 4.83e-04 | 677.8ms/step | 29.5 samples/s
+Step 5900/44000 | Loss: 3.3090 | LR: 4.83e-04 | 679.8ms/step | 29.4 samples/s
+Step 5910/44000 | Loss: 3.3210 | LR: 4.83e-04 | 679.5ms/step | 29.4 samples/s
+Step 5920/44000 | Loss: 3.3013 | LR: 4.83e-04 | 679.9ms/step | 29.4 samples/s
+Step 5930/44000 | Loss: 3.2569 | LR: 4.83e-04 | 679.5ms/step | 29.4 samples/s
+Step 5940/44000 | Loss: 3.2824 | LR: 4.82e-04 | 679.2ms/step | 29.4 samples/s
+Step 5950/44000 | Loss: 3.2879 | LR: 4.82e-04 | 679.6ms/step | 29.4 samples/s
+Step 5960/44000 | Loss: 3.2794 | LR: 4.82e-04 | 678.7ms/step | 29.5 samples/s
+Step 5970/44000 | Loss: 3.3082 | LR: 4.82e-04 | 679.7ms/step | 29.4 samples/s
+Step 5980/44000 | Loss: 3.2952 | LR: 4.82e-04 | 679.8ms/step | 29.4 samples/s
+Step 5990/44000 | Loss: 3.2891 | LR: 4.82e-04 | 678.9ms/step | 29.5 samples/s
+Step 6000/44000 | Loss: 3.2904 | LR: 4.82e-04 | 679.5ms/step | 29.4 samples/s
+Validation at step 6000: Loss = 3.2899, Perplexity = 26.84
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-6000
+Step 6010/44000 | Loss: 3.3171 | LR: 4.82e-04 | 1517.4ms/step | 13.2 samples/s
+Step 6020/44000 | Loss: 3.3210 | LR: 4.82e-04 | 678.6ms/step | 29.5 samples/s
+Step 6030/44000 | Loss: 3.2763 | LR: 4.82e-04 | 678.5ms/step | 29.5 samples/s
+Step 6040/44000 | Loss: 3.2888 | LR: 4.82e-04 | 680.5ms/step | 29.4 samples/s
+Step 6050/44000 | Loss: 3.2496 | LR: 4.82e-04 | 678.9ms/step | 29.5 samples/s
+Step 6060/44000 | Loss: 3.2594 | LR: 4.82e-04 | 679.1ms/step | 29.4 samples/s
+Step 6070/44000 | Loss: 3.2980 | LR: 4.82e-04 | 680.3ms/step | 29.4 samples/s
+Step 6080/44000 | Loss: 3.2949 | LR: 4.81e-04 | 678.5ms/step | 29.5 samples/s
+Step 6090/44000 | Loss: 3.3161 | LR: 4.81e-04 | 678.7ms/step | 29.5 samples/s
+Step 6100/44000 | Loss: 3.2980 | LR: 4.81e-04 | 679.2ms/step | 29.4 samples/s
+Step 6110/44000 | Loss: 3.2402 | LR: 4.81e-04 | 679.3ms/step | 29.4 samples/s
+Step 6120/44000 | Loss: 3.2767 | LR: 4.81e-04 | 678.6ms/step | 29.5 samples/s
+Step 6130/44000 | Loss: 3.2639 | LR: 4.81e-04 | 679.2ms/step | 29.4 samples/s
+Step 6140/44000 | Loss: 3.2620 | LR: 4.81e-04 | 678.9ms/step | 29.5 samples/s
+Step 6150/44000 | Loss: 3.2545 | LR: 4.81e-04 | 678.6ms/step | 29.5 samples/s
+Step 6160/44000 | Loss: 3.2771 | LR: 4.81e-04 | 677.4ms/step | 29.5 samples/s
+Step 6170/44000 | Loss: 3.2956 | LR: 4.81e-04 | 676.4ms/step | 29.6 samples/s
+Step 6180/44000 | Loss: 3.3112 | LR: 4.81e-04 | 678.4ms/step | 29.5 samples/s
+Step 6190/44000 | Loss: 3.2732 | LR: 4.81e-04 | 678.4ms/step | 29.5 samples/s
+Step 6200/44000 | Loss: 3.2504 | LR: 4.81e-04 | 678.6ms/step | 29.5 samples/s
+Step 6210/44000 | Loss: 3.2548 | LR: 4.81e-04 | 676.9ms/step | 29.5 samples/s
+Step 6220/44000 | Loss: 3.2342 | LR: 4.81e-04 | 679.1ms/step | 29.4 samples/s
+Step 6230/44000 | Loss: 3.2800 | LR: 4.80e-04 | 680.0ms/step | 29.4 samples/s
+Step 6240/44000 | Loss: 3.2352 | LR: 4.80e-04 | 679.0ms/step | 29.5 samples/s
+Step 6250/44000 | Loss: 3.2904 | LR: 4.80e-04 | 679.4ms/step | 29.4 samples/s
+Validation at step 6250: Loss = 3.2793, Perplexity = 26.56
+Step 6260/44000 | Loss: 3.2713 | LR: 4.80e-04 | 1233.3ms/step | 16.2 samples/s
+Step 6270/44000 | Loss: 3.2917 | LR: 4.80e-04 | 679.7ms/step | 29.4 samples/s
+Step 6280/44000 | Loss: 3.2853 | LR: 4.80e-04 | 680.2ms/step | 29.4 samples/s
+Step 6290/44000 | Loss: 3.2928 | LR: 4.80e-04 | 678.7ms/step | 29.5 samples/s
+Step 6300/44000 | Loss: 3.2695 | LR: 4.80e-04 | 680.0ms/step | 29.4 samples/s
+Step 6310/44000 | Loss: 3.3031 | LR: 4.80e-04 | 679.5ms/step | 29.4 samples/s
+Step 6320/44000 | Loss: 3.2441 | LR: 4.80e-04 | 678.8ms/step | 29.5 samples/s
+Step 6330/44000 | Loss: 3.3167 | LR: 4.80e-04 | 679.1ms/step | 29.5 samples/s
+Step 6340/44000 | Loss: 3.2719 | LR: 4.80e-04 | 680.6ms/step | 29.4 samples/s
+Step 6350/44000 | Loss: 3.3179 | LR: 4.80e-04 | 679.4ms/step | 29.4 samples/s
+Step 6360/44000 | Loss: 3.2857 | LR: 4.80e-04 | 679.0ms/step | 29.5 samples/s
+Step 6370/44000 | Loss: 3.2397 | LR: 4.79e-04 | 679.6ms/step | 29.4 samples/s
+Step 6380/44000 | Loss: 3.2720 | LR: 4.79e-04 | 680.0ms/step | 29.4 samples/s
+Step 6390/44000 | Loss: 3.2851 | LR: 4.79e-04 | 679.6ms/step | 29.4 samples/s
+Step 6400/44000 | Loss: 3.2461 | LR: 4.79e-04 | 680.4ms/step | 29.4 samples/s
+Step 6410/44000 | Loss: 3.2709 | LR: 4.79e-04 | 679.2ms/step | 29.4 samples/s
+Step 6420/44000 | Loss: 3.2656 | LR: 4.79e-04 | 679.7ms/step | 29.4 samples/s
+Step 6430/44000 | Loss: 3.2963 | LR: 4.79e-04 | 679.1ms/step | 29.4 samples/s
+Step 6440/44000 | Loss: 3.2586 | LR: 4.79e-04 | 678.3ms/step | 29.5 samples/s
+Step 6450/44000 | Loss: 3.2651 | LR: 4.79e-04 | 679.1ms/step | 29.5 samples/s
+Step 6460/44000 | Loss: 3.2751 | LR: 4.79e-04 | 680.5ms/step | 29.4 samples/s
+Step 6470/44000 | Loss: 3.2956 | LR: 4.79e-04 | 679.2ms/step | 29.4 samples/s
+Step 6480/44000 | Loss: 3.3148 | LR: 4.79e-04 | 678.9ms/step | 29.5 samples/s
+Step 6490/44000 | Loss: 3.2306 | LR: 4.79e-04 | 679.1ms/step | 29.4 samples/s
+Step 6500/44000 | Loss: 3.3101 | LR: 4.79e-04 | 678.8ms/step | 29.5 samples/s
+Validation at step 6500: Loss = 3.2681, Perplexity = 26.26
+Step 6510/44000 | Loss: 3.2772 | LR: 4.78e-04 | 1233.9ms/step | 16.2 samples/s
+Step 6520/44000 | Loss: 3.2638 | LR: 4.78e-04 | 677.9ms/step | 29.5 samples/s
+Step 6530/44000 | Loss: 3.2889 | LR: 4.78e-04 | 678.3ms/step | 29.5 samples/s
+Step 6540/44000 | Loss: 3.2582 | LR: 4.78e-04 | 678.4ms/step | 29.5 samples/s
+Step 6550/44000 | Loss: 3.2878 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s
+Step 6560/44000 | Loss: 3.2331 | LR: 4.78e-04 | 678.5ms/step | 29.5 samples/s
+Step 6570/44000 | Loss: 3.2258 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s
+Step 6580/44000 | Loss: 3.2472 | LR: 4.78e-04 | 678.6ms/step | 29.5 samples/s
+Step 6590/44000 | Loss: 3.2573 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s
+Step 6600/44000 | Loss: 3.2273 | LR: 4.78e-04 | 677.9ms/step | 29.5 samples/s
+Step 6610/44000 | Loss: 3.2580 | LR: 4.78e-04 | 679.4ms/step | 29.4 samples/s
+Step 6620/44000 | Loss: 3.2500 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s
+Step 6630/44000 | Loss: 3.2876 | LR: 4.78e-04 | 678.3ms/step | 29.5 samples/s
+Step 6640/44000 | Loss: 3.2677 | LR: 4.77e-04 | 677.6ms/step | 29.5 samples/s
+Step 6650/44000 | Loss: 3.2883 | LR: 4.77e-04 | 678.5ms/step | 29.5 samples/s
+Step 6660/44000 | Loss: 3.2172 | LR: 4.77e-04 | 678.0ms/step | 29.5 samples/s
+Step 6670/44000 | Loss: 3.2684 | LR: 4.77e-04 | 678.6ms/step | 29.5 samples/s
+Step 6680/44000 | Loss: 3.2422 | LR: 4.77e-04 | 678.4ms/step | 29.5 samples/s
+Step 6690/44000 | Loss: 3.2733 | LR: 4.77e-04 | 678.5ms/step | 29.5 samples/s
+Step 6700/44000 | Loss: 3.2605 | LR: 4.77e-04 | 678.6ms/step | 29.5 samples/s
+Step 6710/44000 | Loss: 3.2407 | LR: 4.77e-04 | 678.1ms/step | 29.5 samples/s
+Step 6720/44000 | Loss: 3.2555 | LR: 4.77e-04 | 677.8ms/step | 29.5 samples/s
+Step 6730/44000 | Loss: 3.3198 | LR: 4.77e-04 | 678.6ms/step | 29.5 samples/s
+Step 6740/44000 | Loss: 3.2470 | LR: 4.77e-04 | 678.2ms/step | 29.5 samples/s
+Step 6750/44000 | Loss: 3.2217 | LR: 4.77e-04 | 679.2ms/step | 29.4 samples/s
+Validation at step 6750: Loss = 3.2617, Perplexity = 26.10
+Step 6760/44000 | Loss: 3.2398 | LR: 4.77e-04 | 1234.5ms/step | 16.2 samples/s
+Step 6770/44000 | Loss: 3.2717 | LR: 4.76e-04 | 679.8ms/step | 29.4 samples/s
+Step 6780/44000 | Loss: 3.2467 | LR: 4.76e-04 | 680.6ms/step | 29.4 samples/s
+Step 6790/44000 | Loss: 3.2670 | LR: 4.76e-04 | 679.7ms/step | 29.4 samples/s
+Step 6800/44000 | Loss: 3.2245 | LR: 4.76e-04 | 680.1ms/step | 29.4 samples/s
+Step 6810/44000 | Loss: 3.2664 | LR: 4.76e-04 | 680.6ms/step | 29.4 samples/s
+Step 6820/44000 | Loss: 3.2656 | LR: 4.76e-04 | 679.5ms/step | 29.4 samples/s
+Step 6830/44000 | Loss: 3.2434 | LR: 4.76e-04 | 679.8ms/step | 29.4 samples/s
+Step 6840/44000 | Loss: 3.2242 | LR: 4.76e-04 | 680.1ms/step | 29.4 samples/s
+Step 6850/44000 | Loss: 3.2333 | LR: 4.76e-04 | 678.3ms/step | 29.5 samples/s
+Step 6860/44000 | Loss: 3.2091 | LR: 4.76e-04 | 678.8ms/step | 29.5 samples/s
+Step 6870/44000 | Loss: 3.2832 | LR: 4.76e-04 | 679.0ms/step | 29.5 samples/s
+Step 6880/44000 | Loss: 3.2373 | LR: 4.76e-04 | 679.5ms/step | 29.4 samples/s
+Step 6890/44000 | Loss: 3.2338 | LR: 4.76e-04 | 678.7ms/step | 29.5 samples/s
+Step 6900/44000 | Loss: 3.2429 | LR: 4.75e-04 | 681.7ms/step | 29.3 samples/s
+Step 6910/44000 | Loss: 3.2843 | LR: 4.75e-04 | 679.7ms/step | 29.4 samples/s
+Step 6920/44000 | Loss: 3.2883 | LR: 4.75e-04 | 678.5ms/step | 29.5 samples/s
+Step 6930/44000 | Loss: 3.2477 | LR: 4.75e-04 | 680.7ms/step | 29.4 samples/s
+Step 6940/44000 | Loss: 3.2472 | LR: 4.75e-04 | 680.6ms/step | 29.4 samples/s
+Step 6950/44000 | Loss: 3.2425 | LR: 4.75e-04 | 680.5ms/step | 29.4 samples/s
+Step 6960/44000 | Loss: 3.2633 | LR: 4.75e-04 | 678.1ms/step | 29.5 samples/s
+Step 6970/44000 | Loss: 3.2067 | LR: 4.75e-04 | 679.0ms/step | 29.5 samples/s
+Step 6980/44000 | Loss: 3.2616 | LR: 4.75e-04 | 678.8ms/step | 29.5 samples/s
+Step 6990/44000 | Loss: 3.2119 | LR: 4.75e-04 | 679.1ms/step | 29.5 samples/s
+Step 7000/44000 | Loss: 3.2409 | LR: 4.75e-04 | 678.9ms/step | 29.5 samples/s
+Validation at step 7000: Loss = 3.2491, Perplexity = 25.77
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-7000
+Step 7010/44000 | Loss: 3.2437 | LR: 4.75e-04 | 1513.2ms/step | 13.2 samples/s
+Step 7020/44000 | Loss: 3.2494 | LR: 4.75e-04 | 679.5ms/step | 29.4 samples/s
+Step 7030/44000 | Loss: 3.2273 | LR: 4.74e-04 | 678.0ms/step | 29.5 samples/s
+Step 7040/44000 | Loss: 3.2369 | LR: 4.74e-04 | 677.2ms/step | 29.5 samples/s
+Step 7050/44000 | Loss: 3.2484 | LR: 4.74e-04 | 678.2ms/step | 29.5 samples/s
+Step 7060/44000 | Loss: 3.2558 | LR: 4.74e-04 | 677.9ms/step | 29.5 samples/s
+Step 7070/44000 | Loss: 3.1837 | LR: 4.74e-04 | 679.2ms/step | 29.4 samples/s
+Step 7080/44000 | Loss: 3.2312 | LR: 4.74e-04 | 677.3ms/step | 29.5 samples/s
+Step 7090/44000 | Loss: 3.2743 | LR: 4.74e-04 | 676.6ms/step | 29.6 samples/s
+Step 7100/44000 | Loss: 3.2342 | LR: 4.74e-04 | 676.8ms/step | 29.6 samples/s
+Step 7110/44000 | Loss: 3.2184 | LR: 4.74e-04 | 677.9ms/step | 29.5 samples/s
+Step 7120/44000 | Loss: 3.2133 | LR: 4.74e-04 | 677.0ms/step | 29.5 samples/s
+Step 7130/44000 | Loss: 3.1702 | LR: 4.74e-04 | 678.6ms/step | 29.5 samples/s
+Step 7140/44000 | Loss: 3.2252 | LR: 4.74e-04 | 678.4ms/step | 29.5 samples/s
+Step 7150/44000 | Loss: 3.2479 | LR: 4.73e-04 | 678.2ms/step | 29.5 samples/s
+Step 7160/44000 | Loss: 3.2362 | LR: 4.73e-04 | 678.0ms/step | 29.5 samples/s
+Step 7170/44000 | Loss: 3.2109 | LR: 4.73e-04 | 678.4ms/step | 29.5 samples/s
+Step 7180/44000 | Loss: 3.2289 | LR: 4.73e-04 | 678.7ms/step | 29.5 samples/s
+Step 7190/44000 | Loss: 3.2519 | LR: 4.73e-04 | 678.4ms/step | 29.5 samples/s
+Step 7200/44000 | Loss: 3.2563 | LR: 4.73e-04 | 678.0ms/step | 29.5 samples/s
+Step 7210/44000 | Loss: 3.2352 | LR: 4.73e-04 | 677.6ms/step | 29.5 samples/s
+Step 7220/44000 | Loss: 3.2157 | LR: 4.73e-04 | 677.8ms/step | 29.5 samples/s
+Step 7230/44000 | Loss: 3.2302 | LR: 4.73e-04 | 677.1ms/step | 29.5 samples/s
+Step 7240/44000 | Loss: 3.2174 | LR: 4.73e-04 | 678.0ms/step | 29.5 samples/s
+Step 7250/44000 | Loss: 3.2539 | LR: 4.73e-04 | 677.8ms/step | 29.5 samples/s
+Validation at step 7250: Loss = 3.2385, Perplexity = 25.50
+Step 7260/44000 | Loss: 3.2460 | LR: 4.73e-04 | 1233.0ms/step | 16.2 samples/s
+Step 7270/44000 | Loss: 3.2156 | LR: 4.72e-04 | 678.9ms/step | 29.5 samples/s
+Step 7280/44000 | Loss: 3.2339 | LR: 4.72e-04 | 679.9ms/step | 29.4 samples/s
+Step 7290/44000 | Loss: 3.2530 | LR: 4.72e-04 | 680.5ms/step | 29.4 samples/s
+Step 7300/44000 | Loss: 3.2496 | LR: 4.72e-04 | 680.4ms/step | 29.4 samples/s
+Step 7310/44000 | Loss: 3.2499 | LR: 4.72e-04 | 679.6ms/step | 29.4 samples/s
+Step 7320/44000 | Loss: 3.2096 | LR: 4.72e-04 | 680.8ms/step | 29.4 samples/s
+Step 7330/44000 | Loss: 3.2435 | LR: 4.72e-04 | 679.0ms/step | 29.5 samples/s
+Step 7340/44000 | Loss: 3.2338 | LR: 4.72e-04 | 680.0ms/step | 29.4 samples/s
+Step 7350/44000 | Loss: 3.2175 | LR: 4.72e-04 | 680.6ms/step | 29.4 samples/s
+Step 7360/44000 | Loss: 3.2012 | LR: 4.72e-04 | 679.2ms/step | 29.4 samples/s
+Step 7370/44000 | Loss: 3.2560 | LR: 4.72e-04 | 678.3ms/step | 29.5 samples/s
+Step 7380/44000 | Loss: 3.2539 | LR: 4.72e-04 | 678.0ms/step | 29.5 samples/s
+Step 7390/44000 | Loss: 3.2795 | LR: 4.71e-04 | 677.7ms/step | 29.5 samples/s
+Step 7400/44000 | Loss: 3.2290 | LR: 4.71e-04 | 678.1ms/step | 29.5 samples/s
+Step 7410/44000 | Loss: 3.2432 | LR: 4.71e-04 | 678.1ms/step | 29.5 samples/s
+Step 7420/44000 | Loss: 3.2311 | LR: 4.71e-04 | 680.0ms/step | 29.4 samples/s
+Step 7430/44000 | Loss: 3.2289 | LR: 4.71e-04 | 678.3ms/step | 29.5 samples/s
+Step 7440/44000 | Loss: 3.2376 | LR: 4.71e-04 | 680.8ms/step | 29.4 samples/s
+Step 7450/44000 | Loss: 3.2647 | LR: 4.71e-04 | 679.5ms/step | 29.4 samples/s
+Step 7460/44000 | Loss: 3.2444 | LR: 4.71e-04 | 680.9ms/step | 29.4 samples/s
+Step 7470/44000 | Loss: 3.2463 | LR: 4.71e-04 | 680.5ms/step | 29.4 samples/s
+Step 7480/44000 | Loss: 3.2525 | LR: 4.71e-04 | 679.5ms/step | 29.4 samples/s
+Step 7490/44000 | Loss: 3.2028 | LR: 4.71e-04 | 681.6ms/step | 29.3 samples/s
+Step 7500/44000 | Loss: 3.1953 | LR: 4.71e-04 | 679.6ms/step | 29.4 samples/s
+Validation at step 7500: Loss = 3.2309, Perplexity = 25.30
+Step 7510/44000 | Loss: 3.2101 | LR: 4.70e-04 | 1237.4ms/step | 16.2 samples/s
+Step 7520/44000 | Loss: 3.2320 | LR: 4.70e-04 | 679.5ms/step | 29.4 samples/s
+Step 7530/44000 | Loss: 3.2523 | LR: 4.70e-04 | 679.8ms/step | 29.4 samples/s
+Step 7540/44000 | Loss: 3.2104 | LR: 4.70e-04 | 680.7ms/step | 29.4 samples/s
+Step 7550/44000 | Loss: 3.2043 | LR: 4.70e-04 | 680.1ms/step | 29.4 samples/s
+Step 7560/44000 | Loss: 3.2283 | LR: 4.70e-04 | 679.4ms/step | 29.4 samples/s
+Step 7570/44000 | Loss: 3.1746 | LR: 4.70e-04 | 680.1ms/step | 29.4 samples/s
+Step 7580/44000 | Loss: 3.1486 | LR: 4.70e-04 | 679.5ms/step | 29.4 samples/s
+Step 7590/44000 | Loss: 3.2140 | LR: 4.70e-04 | 678.8ms/step | 29.5 samples/s
+Step 7600/44000 | Loss: 3.2231 | LR: 4.70e-04 | 678.4ms/step | 29.5 samples/s
+Step 7610/44000 | Loss: 3.2206 | LR: 4.70e-04 | 678.7ms/step | 29.5 samples/s
+Step 7620/44000 | Loss: 3.2409 | LR: 4.70e-04 | 678.6ms/step | 29.5 samples/s
+Step 7630/44000 | Loss: 3.2120 | LR: 4.69e-04 | 679.0ms/step | 29.5 samples/s
+Step 7640/44000 | Loss: 3.1782 | LR: 4.69e-04 | 678.9ms/step | 29.5 samples/s
+Step 7650/44000 | Loss: 3.1998 | LR: 4.69e-04 | 680.3ms/step | 29.4 samples/s
+Step 7660/44000 | Loss: 3.2252 | LR: 4.69e-04 | 679.8ms/step | 29.4 samples/s
+Step 7670/44000 | Loss: 3.1973 | LR: 4.69e-04 | 678.9ms/step | 29.5 samples/s
+Step 7680/44000 | Loss: 3.2567 | LR: 4.69e-04 | 679.4ms/step | 29.4 samples/s
+Step 7690/44000 | Loss: 3.2256 | LR: 4.69e-04 | 680.2ms/step | 29.4 samples/s
+Step 7700/44000 | Loss: 3.1972 | LR: 4.69e-04 | 680.0ms/step | 29.4 samples/s
+Step 7710/44000 | Loss: 3.1891 | LR: 4.69e-04 | 680.3ms/step | 29.4 samples/s
+Step 7720/44000 | Loss: 3.2200 | LR: 4.69e-04 | 680.3ms/step | 29.4 samples/s
+Step 7730/44000 | Loss: 3.2071 | LR: 4.69e-04 | 678.8ms/step | 29.5 samples/s
+Step 7740/44000 | Loss: 3.1694 | LR: 4.68e-04 | 680.0ms/step | 29.4 samples/s
+Step 7750/44000 | Loss: 3.2097 | LR: 4.68e-04 | 681.5ms/step | 29.3 samples/s
+Validation at step 7750: Loss = 3.2223, Perplexity = 25.09
+Step 7760/44000 | Loss: 3.2055 | LR: 4.68e-04 | 1233.8ms/step | 16.2 samples/s
+Step 7770/44000 | Loss: 3.2165 | LR: 4.68e-04 | 678.0ms/step | 29.5 samples/s
+Step 7780/44000 | Loss: 3.2132 | LR: 4.68e-04 | 679.0ms/step | 29.5 samples/s
+Step 7790/44000 | Loss: 3.2165 | LR: 4.68e-04 | 679.5ms/step | 29.4 samples/s
+Step 7800/44000 | Loss: 3.2271 | LR: 4.68e-04 | 679.9ms/step | 29.4 samples/s
+Step 7810/44000 | Loss: 3.1735 | LR: 4.68e-04 | 678.1ms/step | 29.5 samples/s
+Step 7820/44000 | Loss: 3.2187 | LR: 4.68e-04 | 677.3ms/step | 29.5 samples/s
+Step 7830/44000 | Loss: 3.2056 | LR: 4.68e-04 | 678.4ms/step | 29.5 samples/s
+Step 7840/44000 | Loss: 3.1888 | LR: 4.68e-04 | 678.7ms/step | 29.5 samples/s
+Step 7850/44000 | Loss: 3.1858 | LR: 4.67e-04 | 679.5ms/step | 29.4 samples/s
+Step 7860/44000 | Loss: 3.2051 | LR: 4.67e-04 | 680.9ms/step | 29.4 samples/s
+Step 7870/44000 | Loss: 3.2081 | LR: 4.67e-04 | 679.9ms/step | 29.4 samples/s
+Step 7880/44000 | Loss: 3.2378 | LR: 4.67e-04 | 679.6ms/step | 29.4 samples/s
+Step 7890/44000 | Loss: 3.2040 | LR: 4.67e-04 | 680.0ms/step | 29.4 samples/s
+Step 7900/44000 | Loss: 3.2436 | LR: 4.67e-04 | 680.0ms/step | 29.4 samples/s
+Step 7910/44000 | Loss: 3.2239 | LR: 4.67e-04 | 678.6ms/step | 29.5 samples/s
+Step 7920/44000 | Loss: 3.2372 | LR: 4.67e-04 | 678.4ms/step | 29.5 samples/s
+Step 7930/44000 | Loss: 3.2606 | LR: 4.67e-04 | 678.1ms/step | 29.5 samples/s
+Step 7940/44000 | Loss: 3.2614 | LR: 4.67e-04 | 678.3ms/step | 29.5 samples/s
+Step 7950/44000 | Loss: 3.1729 | LR: 4.67e-04 | 679.1ms/step | 29.4 samples/s
+Step 7960/44000 | Loss: 3.2248 | LR: 4.66e-04 | 679.1ms/step | 29.5 samples/s
+Step 7970/44000 | Loss: 3.1600 | LR: 4.66e-04 | 680.0ms/step | 29.4 samples/s
+Step 7980/44000 | Loss: 3.1919 | LR: 4.66e-04 | 678.3ms/step | 29.5 samples/s
+Step 7990/44000 | Loss: 3.2139 | LR: 4.66e-04 | 680.5ms/step | 29.4 samples/s
+Step 8000/44000 | Loss: 3.2483 | LR: 4.66e-04 | 678.2ms/step | 29.5 samples/s
+Validation at step 8000: Loss = 3.2159, Perplexity = 24.93
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-8000
+Step 8010/44000 | Loss: 3.2012 | LR: 4.66e-04 | 1521.3ms/step | 13.1 samples/s
+Step 8020/44000 | Loss: 3.2198 | LR: 4.66e-04 | 678.7ms/step | 29.5 samples/s
+Step 8030/44000 | Loss: 3.2153 | LR: 4.66e-04 | 679.2ms/step | 29.4 samples/s
+Step 8040/44000 | Loss: 3.2348 | LR: 4.66e-04 | 677.0ms/step | 29.5 samples/s
+Step 8050/44000 | Loss: 3.2063 | LR: 4.66e-04 | 676.9ms/step | 29.5 samples/s
+Step 8060/44000 | Loss: 3.2254 | LR: 4.66e-04 | 677.9ms/step | 29.5 samples/s
+Step 8070/44000 | Loss: 3.1721 | LR: 4.65e-04 | 678.9ms/step | 29.5 samples/s
+Step 8080/44000 | Loss: 3.1908 | LR: 4.65e-04 | 678.1ms/step | 29.5 samples/s
+Step 8090/44000 | Loss: 3.2304 | LR: 4.65e-04 | 679.3ms/step | 29.4 samples/s
+Step 8100/44000 | Loss: 3.2392 | LR: 4.65e-04 | 679.7ms/step | 29.4 samples/s
+Step 8110/44000 | Loss: 3.2200 | LR: 4.65e-04 | 679.5ms/step | 29.4 samples/s
+Step 8120/44000 | Loss: 3.1675 | LR: 4.65e-04 | 678.8ms/step | 29.5 samples/s
+W0708 22:28:09.644000 255 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 255 got signal: 15