EPFL ALiBi and training provenance 951-1000 of 1184

epfl_final/train_logs/eval_seed71_mixed_50462_battery.log

@@ -0,0 +1,78 @@
+[14:12:02] total jobs=22 already-done=0 gpus=[0, 1, 2, 3, 4] n=100
+[14:12:02] START gpu0 passkey_8k_mixed_seed71 try1 (~22 remaining)
+[14:12:02] START gpu1 ppl_8k_mixed_seed71_resampled_fineweb try1 (~22 remaining)
+[14:12:02] START gpu2 ppl_8k_mixed_seed71_resampled_pg19 try1 (~22 remaining)
+[14:12:02] START gpu3 ppl_8k_mixed_seed71_resampled_arxiv try1 (~22 remaining)
+[14:12:03] START gpu4 ppl_8k_mixed_seed71_fixed_fineweb try1 (~22 remaining)
+[14:13:08] DONE gpu0 passkey_8k_mixed_seed71 rc=1 NO-TSV
+[14:13:08] START gpu0 passkey_8k_mixed_seed71 try2 (~22 remaining)
+[14:13:48] DONE gpu0 passkey_8k_mixed_seed71 rc=1 NO-TSV
+[14:13:48] START gpu0 passkey_8k_mixed_seed71 try3 (~21 remaining)
+[14:14:23] DONE gpu0 passkey_8k_mixed_seed71 rc=1 NO-TSV
+[14:14:23] START gpu0 ppl_8k_mixed_seed71_fixed_pg19 try1 (~21 remaining)
+[14:29:38] DONE gpu1 ppl_8k_mixed_seed71_resampled_fineweb rc=0 ok
+[14:29:38] START gpu1 ppl_8k_mixed_seed71_fixed_arxiv try1 (~19 remaining)
+[14:29:43] DONE gpu3 ppl_8k_mixed_seed71_resampled_arxiv rc=0 ok
+[14:29:43] START gpu3 ruler_8k_niah_multikey_mixed_seed71 try1 (~18 remaining)
+[14:29:48] DONE gpu4 ppl_8k_mixed_seed71_fixed_fineweb rc=0 ok
+[14:29:48] START gpu4 ruler_8k_niah_multivalue_mixed_seed71 try1 (~17 remaining)
+[14:29:53] DONE gpu2 ppl_8k_mixed_seed71_resampled_pg19 rc=0 ok
+[14:29:54] START gpu2 ruler_8k_vt_mixed_seed71 try1 (~17 remaining)
+[14:30:24] DONE gpu3 ruler_8k_niah_multikey_mixed_seed71 rc=1 NO-TSV
+[14:30:24] START gpu3 ruler_8k_niah_multikey_mixed_seed71 try2 (~17 remaining)
+[14:30:29] DONE gpu4 ruler_8k_niah_multivalue_mixed_seed71 rc=1 NO-TSV
+[14:30:29] START gpu4 ruler_8k_niah_multivalue_mixed_seed71 try2 (~17 remaining)
+[14:30:39] DONE gpu2 ruler_8k_vt_mixed_seed71 rc=1 NO-TSV
+[14:30:39] START gpu2 ruler_8k_vt_mixed_seed71 try2 (~17 remaining)
+[14:30:59] DONE gpu3 ruler_8k_niah_multikey_mixed_seed71 rc=1 NO-TSV
+[14:30:59] START gpu3 ruler_8k_niah_multikey_mixed_seed71 try3 (~16 remaining)
+[14:31:04] DONE gpu4 ruler_8k_niah_multivalue_mixed_seed71 rc=1 NO-TSV
+[14:31:04] START gpu4 ruler_8k_niah_multivalue_mixed_seed71 try3 (~15 remaining)
+[14:31:14] DONE gpu2 ruler_8k_vt_mixed_seed71 rc=1 NO-TSV
+[14:31:14] START gpu2 ruler_8k_vt_mixed_seed71 try3 (~14 remaining)
+[14:31:34] DONE gpu3 ruler_8k_niah_multikey_mixed_seed71 rc=1 NO-TSV
+[14:31:34] START gpu3 kv_h2o_8k_mixed_seed71 try1 (~14 remaining)
+[14:31:44] DONE gpu0 ppl_8k_mixed_seed71_fixed_pg19 rc=0 ok
+[14:31:44] DONE gpu4 ruler_8k_niah_multivalue_mixed_seed71 rc=1 NO-TSV
+[14:31:45] START gpu0 kv_stream_8k_mixed_seed71 try1 (~13 remaining)
+[14:31:45] START gpu4 passkey_30k_mixed_seed71 try1 (~13 remaining)
+[14:31:50] DONE gpu2 ruler_8k_vt_mixed_seed71 rc=1 NO-TSV
+[14:31:50] START gpu2 ppl_30k_mixed_seed71_resampled_pg19 try1 (~13 remaining)
+[14:32:20] DONE gpu4 passkey_30k_mixed_seed71 rc=1 NO-TSV
+[14:32:20] START gpu4 passkey_30k_mixed_seed71 try2 (~13 remaining)
+[14:32:55] DONE gpu4 passkey_30k_mixed_seed71 rc=1 NO-TSV
+[14:32:55] START gpu4 passkey_30k_mixed_seed71 try3 (~12 remaining)
+[14:33:35] DONE gpu4 passkey_30k_mixed_seed71 rc=1 NO-TSV
+[14:33:35] START gpu4 ppl_30k_mixed_seed71_resampled_arxiv try1 (~12 remaining)
+[14:34:10] DONE gpu0 kv_stream_8k_mixed_seed71 rc=0 ok
+[14:34:10] START gpu0 ppl_30k_mixed_seed71_fixed_pg19 try1 (~11 remaining)
+[14:36:45] DONE gpu3 kv_h2o_8k_mixed_seed71 rc=0 ok
+[14:36:45] START gpu3 ppl_30k_mixed_seed71_fixed_arxiv try1 (~10 remaining)
+[14:47:00] DONE gpu1 ppl_8k_mixed_seed71_fixed_arxiv rc=0 ok
+[14:47:00] START gpu1 ruler_30k_niah_multikey_mixed_seed71 try1 (~9 remaining)
+[14:47:40] DONE gpu1 ruler_30k_niah_multikey_mixed_seed71 rc=1 NO-TSV
+[14:47:41] START gpu1 ruler_30k_niah_multikey_mixed_seed71 try2 (~9 remaining)
+[14:48:21] DONE gpu1 ruler_30k_niah_multikey_mixed_seed71 rc=1 NO-TSV
+[14:48:21] START gpu1 ruler_30k_niah_multikey_mixed_seed71 try3 (~8 remaining)
+[14:48:56] DONE gpu1 ruler_30k_niah_multikey_mixed_seed71 rc=1 NO-TSV
+[14:48:56] START gpu1 ruler_30k_niah_multivalue_mixed_seed71 try1 (~8 remaining)
+[14:49:31] DONE gpu1 ruler_30k_niah_multivalue_mixed_seed71 rc=1 NO-TSV
+[14:49:31] START gpu1 ruler_30k_niah_multivalue_mixed_seed71 try2 (~8 remaining)
+[14:50:06] DONE gpu1 ruler_30k_niah_multivalue_mixed_seed71 rc=1 NO-TSV
+[14:50:06] START gpu1 ruler_30k_niah_multivalue_mixed_seed71 try3 (~7 remaining)
+[14:50:41] DONE gpu1 ruler_30k_niah_multivalue_mixed_seed71 rc=1 NO-TSV
+[14:50:41] START gpu1 ruler_30k_vt_mixed_seed71 try1 (~7 remaining)
+[14:51:21] DONE gpu1 ruler_30k_vt_mixed_seed71 rc=1 NO-TSV
+[14:51:21] START gpu1 ruler_30k_vt_mixed_seed71 try2 (~7 remaining)
+[14:51:56] DONE gpu1 ruler_30k_vt_mixed_seed71 rc=1 NO-TSV
+[14:51:56] START gpu1 ruler_30k_vt_mixed_seed71 try3 (~6 remaining)
+[14:52:36] DONE gpu1 ruler_30k_vt_mixed_seed71 rc=1 NO-TSV
+[14:52:36] START gpu1 kv_h2o_30k_mixed_seed71 try1 (~6 remaining)
+[14:57:41] DONE gpu1 kv_h2o_30k_mixed_seed71 rc=0 ok
+[14:57:41] START gpu1 kv_stream_30k_mixed_seed71 try1 (~5 remaining)
+[15:00:01] DONE gpu1 kv_stream_30k_mixed_seed71 rc=0 ok
+[17:05:20] DONE gpu0 ppl_30k_mixed_seed71_fixed_pg19 rc=0 ok
+[17:05:25] DONE gpu2 ppl_30k_mixed_seed71_resampled_pg19 rc=0 ok
+[17:05:45] DONE gpu4 ppl_30k_mixed_seed71_resampled_arxiv rc=0 ok
+[17:08:08] DONE gpu3 ppl_30k_mixed_seed71_fixed_arxiv rc=0 ok
+[17:08:08] === 1.2B BATTERY DONE. done=14/22 failed_or_skipped=['passkey_8k_mixed_seed71', 'ruler_8k_niah_multikey_mixed_seed71', 'ruler_8k_niah_multivalue_mixed_seed71', 'ruler_8k_vt_mixed_seed71', 'passkey_30k_mixed_seed71', 'ruler_30k_niah_multikey_mixed_seed71', 'ruler_30k_niah_multivalue_mixed_seed71', 'ruler_30k_vt_mixed_seed71'] ===

epfl_final/train_logs/eval_seed71_mixed_50462_h100_retry.log

@@ -0,0 +1,18 @@
+[02:44:40] total jobs=22 already-done=14 gpus=[0, 1, 2, 3] n=100
+[02:44:40] START gpu0 passkey_8k_mixed_seed71 try1 (~8 remaining)
+[02:44:40] START gpu1 ruler_8k_niah_multikey_mixed_seed71 try1 (~8 remaining)
+[02:44:40] START gpu2 ruler_8k_niah_multivalue_mixed_seed71 try1 (~8 remaining)
+[02:44:40] START gpu3 ruler_8k_vt_mixed_seed71 try1 (~8 remaining)
+[02:58:36] DONE gpu0 passkey_8k_mixed_seed71 rc=0 ok
+[02:58:36] DONE gpu1 ruler_8k_niah_multikey_mixed_seed71 rc=0 ok
+[02:58:36] DONE gpu2 ruler_8k_niah_multivalue_mixed_seed71 rc=0 ok
+[02:58:36] DONE gpu3 ruler_8k_vt_mixed_seed71 rc=0 ok
+[02:58:36] START gpu0 passkey_30k_mixed_seed71 try1 (~4 remaining)
+[02:58:36] START gpu1 ruler_30k_niah_multikey_mixed_seed71 try1 (~4 remaining)
+[02:58:36] START gpu2 ruler_30k_niah_multivalue_mixed_seed71 try1 (~4 remaining)
+[02:58:36] START gpu3 ruler_30k_vt_mixed_seed71 try1 (~4 remaining)
+[04:46:33] DONE gpu2 ruler_30k_niah_multivalue_mixed_seed71 rc=0 ok
+[04:46:38] DONE gpu3 ruler_30k_vt_mixed_seed71 rc=0 ok
+[04:46:43] DONE gpu1 ruler_30k_niah_multikey_mixed_seed71 rc=0 ok
+[04:47:13] DONE gpu0 passkey_30k_mixed_seed71 rc=0 ok
+[04:47:13] === 1.2B BATTERY DONE. done=22/22 failed_or_skipped=[] ===

epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu_autoresume.log

@@ -0,0 +1,982 @@
+
+===== AUTO-RESUME FROM checkpoint-16500 2026-08-27T13:58:53Z =====
+W0827 13:59:01.843000 153 site-packages/torch/distributed/run.py:851] 
+W0827 13:59:01.843000 153 site-packages/torch/distributed/run.py:851] *****************************************
+W0827 13:59:01.843000 153 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0827 13:59:01.843000 153 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16500",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: TrueCheckpoint activations: True
+
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16500
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16500...
+Preserving checkpoint LR state for exact resume: current=[0.0003860555075981444], schedule_base=[0.0005]
+Resumed from checkpoint at step 16500 (epoch 0, micro_step 245000)
+Resuming training from step 16500
+Resuming wandb run 8ynn1we6
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_140233-8ynn1we6
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed71 4gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8ynn1we6
+Starting training...
+Validation at step 16500: Loss = 2.6615, Perplexity = 14.32
+wandb: WARNING Tried to log to step 16500 that is less than the current step 16561. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 16501: sparse_stochastic (configured sparse attention)
+Step 16510/50462 | Loss: 2.6293 | LR: 3.86e-04 | 52254.7ms/step | 1.1 samples/s
+wandb: WARNING Tried to log to step 16510 that is less than the current step 16561. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 16520/50462 | Loss: 2.6919 | LR: 3.86e-04 | 4020.8ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 16520 that is less than the current step 16561. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 16530/50462 | Loss: 2.6652 | LR: 3.86e-04 | 4017.4ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 16530 that is less than the current step 16561. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 16540/50462 | Loss: 2.6860 | LR: 3.86e-04 | 4027.0ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 16540 that is less than the current step 16561. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 16550/50462 | Loss: 2.6464 | LR: 3.85e-04 | 4025.4ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 16550 that is less than the current step 16561. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 16560/50462 | Loss: 2.6892 | LR: 3.85e-04 | 4027.6ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 16560 that is less than the current step 16561. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 16570/50462 | Loss: 2.6877 | LR: 3.85e-04 | 4007.7ms/step | 15.0 samples/s
+Step 16580/50462 | Loss: 2.6624 | LR: 3.85e-04 | 4031.1ms/step | 14.9 samples/s
+Step 16590/50462 | Loss: 2.6388 | LR: 3.85e-04 | 4024.0ms/step | 14.9 samples/s
+Step 16600/50462 | Loss: 2.6775 | LR: 3.85e-04 | 4027.9ms/step | 14.9 samples/s
+Step 16610/50462 | Loss: 2.6450 | LR: 3.85e-04 | 4027.2ms/step | 14.9 samples/s
+Step 16620/50462 | Loss: 2.6924 | LR: 3.84e-04 | 4034.0ms/step | 14.9 samples/s
+Step 16630/50462 | Loss: 2.6915 | LR: 3.84e-04 | 4026.6ms/step | 14.9 samples/s
+Step 16640/50462 | Loss: 2.6879 | LR: 3.84e-04 | 4031.7ms/step | 14.9 samples/s
+Step 16650/50462 | Loss: 2.6794 | LR: 3.84e-04 | 4013.6ms/step | 14.9 samples/s
+Step 16660/50462 | Loss: 2.6714 | LR: 3.84e-04 | 4017.3ms/step | 14.9 samples/s
+Step 16670/50462 | Loss: 2.7052 | LR: 3.84e-04 | 4020.0ms/step | 14.9 samples/s
+Step 16680/50462 | Loss: 2.6580 | LR: 3.84e-04 | 4025.6ms/step | 14.9 samples/s
+Step 16690/50462 | Loss: 2.6450 | LR: 3.84e-04 | 4022.3ms/step | 14.9 samples/s
+Step 16700/50462 | Loss: 2.6899 | LR: 3.83e-04 | 4015.6ms/step | 14.9 samples/s
+Step 16710/50462 | Loss: 2.6570 | LR: 3.83e-04 | 4016.1ms/step | 14.9 samples/s
+Step 16720/50462 | Loss: 2.6778 | LR: 3.83e-04 | 4010.0ms/step | 15.0 samples/s
+Step 16730/50462 | Loss: 2.6642 | LR: 3.83e-04 | 4022.3ms/step | 14.9 samples/s
+Step 16740/50462 | Loss: 2.6843 | LR: 3.83e-04 | 4027.0ms/step | 14.9 samples/s
+Step 16750/50462 | Loss: 2.7161 | LR: 3.83e-04 | 4020.0ms/step | 14.9 samples/s
+Validation at step 16750: Loss = 2.6573, Perplexity = 14.26
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16750
+Step 16760/50462 | Loss: 2.6654 | LR: 3.83e-04 | 6040.4ms/step | 9.9 samples/s
+Step 16770/50462 | Loss: 2.6647 | LR: 3.82e-04 | 4018.8ms/step | 14.9 samples/s
+Step 16780/50462 | Loss: 2.6711 | LR: 3.82e-04 | 4029.9ms/step | 14.9 samples/s
+Step 16790/50462 | Loss: 2.6705 | LR: 3.82e-04 | 4017.3ms/step | 14.9 samples/s
+Step 16800/50462 | Loss: 2.6851 | LR: 3.82e-04 | 4015.4ms/step | 14.9 samples/s
+Step 16810/50462 | Loss: 2.6862 | LR: 3.82e-04 | 4020.0ms/step | 14.9 samples/s
+Step 16820/50462 | Loss: 2.6651 | LR: 3.82e-04 | 4012.0ms/step | 15.0 samples/s
+Step 16830/50462 | Loss: 2.6882 | LR: 3.82e-04 | 4005.2ms/step | 15.0 samples/s
+Step 16840/50462 | Loss: 2.6549 | LR: 3.82e-04 | 4028.4ms/step | 14.9 samples/s
+Step 16850/50462 | Loss: 2.6570 | LR: 3.81e-04 | 4015.0ms/step | 14.9 samples/s
+Step 16860/50462 | Loss: 2.6706 | LR: 3.81e-04 | 4024.8ms/step | 14.9 samples/s
+Step 16870/50462 | Loss: 2.6821 | LR: 3.81e-04 | 4020.4ms/step | 14.9 samples/s
+Step 16880/50462 | Loss: 2.6583 | LR: 3.81e-04 | 4029.6ms/step | 14.9 samples/s
+Step 16890/50462 | Loss: 2.6633 | LR: 3.81e-04 | 4027.1ms/step | 14.9 samples/s
+Step 16900/50462 | Loss: 2.6674 | LR: 3.81e-04 | 4019.1ms/step | 14.9 samples/s
+Step 16910/50462 | Loss: 2.6569 | LR: 3.81e-04 | 4015.6ms/step | 14.9 samples/s
+Step 16920/50462 | Loss: 2.6515 | LR: 3.80e-04 | 4020.3ms/step | 14.9 samples/s
+Step 16930/50462 | Loss: 2.6445 | LR: 3.80e-04 | 4024.9ms/step | 14.9 samples/s
+Step 16940/50462 | Loss: 2.6218 | LR: 3.80e-04 | 4025.3ms/step | 14.9 samples/s
+Step 16950/50462 | Loss: 2.6282 | LR: 3.80e-04 | 4018.1ms/step | 14.9 samples/s
+Step 16960/50462 | Loss: 2.6607 | LR: 3.80e-04 | 4023.5ms/step | 14.9 samples/s
+Step 16970/50462 | Loss: 2.6688 | LR: 3.80e-04 | 4026.1ms/step | 14.9 samples/s
+Step 16980/50462 | Loss: 2.6993 | LR: 3.80e-04 | 4019.0ms/step | 14.9 samples/s
+Step 16990/50462 | Loss: 2.6659 | LR: 3.79e-04 | 4012.9ms/step | 15.0 samples/s
+Step 17000/50462 | Loss: 2.6493 | LR: 3.79e-04 | 4012.2ms/step | 15.0 samples/s
+Validation at step 17000: Loss = 2.6556, Perplexity = 14.23
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-17000
+Step 17010/50462 | Loss: 2.6451 | LR: 3.79e-04 | 5968.9ms/step | 10.1 samples/s
+Step 17020/50462 | Loss: 2.6330 | LR: 3.79e-04 | 4019.3ms/step | 14.9 samples/s
+Step 17030/50462 | Loss: 2.6810 | LR: 3.79e-04 | 4015.0ms/step | 14.9 samples/s
+Step 17040/50462 | Loss: 2.6561 | LR: 3.79e-04 | 4012.3ms/step | 15.0 samples/s
+Step 17050/50462 | Loss: 2.6845 | LR: 3.79e-04 | 4012.4ms/step | 15.0 samples/s
+Step 17060/50462 | Loss: 2.6664 | LR: 3.79e-04 | 4010.5ms/step | 15.0 samples/s
+Step 17070/50462 | Loss: 2.6509 | LR: 3.78e-04 | 4012.9ms/step | 15.0 samples/s
+Step 17080/50462 | Loss: 2.6758 | LR: 3.78e-04 | 4015.2ms/step | 14.9 samples/s
+Step 17090/50462 | Loss: 2.6680 | LR: 3.78e-04 | 4006.8ms/step | 15.0 samples/s
+Step 17100/50462 | Loss: 2.6740 | LR: 3.78e-04 | 4032.1ms/step | 14.9 samples/s
+Step 17110/50462 | Loss: 2.6678 | LR: 3.78e-04 | 4029.2ms/step | 14.9 samples/s
+Step 17120/50462 | Loss: 2.6480 | LR: 3.78e-04 | 4010.2ms/step | 15.0 samples/s
+Step 17130/50462 | Loss: 2.6582 | LR: 3.78e-04 | 4022.6ms/step | 14.9 samples/s
+Step 17140/50462 | Loss: 2.6524 | LR: 3.77e-04 | 4010.3ms/step | 15.0 samples/s
+Step 17150/50462 | Loss: 2.6673 | LR: 3.77e-04 | 4003.4ms/step | 15.0 samples/s
+Step 17160/50462 | Loss: 2.6996 | LR: 3.77e-04 | 4011.8ms/step | 15.0 samples/s
+Step 17170/50462 | Loss: 2.6588 | LR: 3.77e-04 | 4020.6ms/step | 14.9 samples/s
+Step 17180/50462 | Loss: 2.6777 | LR: 3.77e-04 | 4015.6ms/step | 14.9 samples/s
+Step 17190/50462 | Loss: 2.6482 | LR: 3.77e-04 | 4010.4ms/step | 15.0 samples/s
+Step 17200/50462 | Loss: 2.6840 | LR: 3.77e-04 | 4014.8ms/step | 14.9 samples/s
+Step 17210/50462 | Loss: 2.6602 | LR: 3.77e-04 | 4021.7ms/step | 14.9 samples/s
+Step 17220/50462 | Loss: 2.6696 | LR: 3.76e-04 | 4014.1ms/step | 14.9 samples/s
+Step 17230/50462 | Loss: 2.6546 | LR: 3.76e-04 | 4006.2ms/step | 15.0 samples/s
+Step 17240/50462 | Loss: 2.6607 | LR: 3.76e-04 | 4013.9ms/step | 14.9 samples/s
+Step 17250/50462 | Loss: 2.6734 | LR: 3.76e-04 | 4007.7ms/step | 15.0 samples/s
+Validation at step 17250: Loss = 2.6526, Perplexity = 14.19
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-17250
+Step 17260/50462 | Loss: 2.6992 | LR: 3.76e-04 | 6014.7ms/step | 10.0 samples/s
+Step 17270/50462 | Loss: 2.6663 | LR: 3.76e-04 | 4019.8ms/step | 14.9 samples/s
+Step 17280/50462 | Loss: 2.6550 | LR: 3.76e-04 | 4024.5ms/step | 14.9 samples/s
+Step 17290/50462 | Loss: 2.6799 | LR: 3.75e-04 | 4011.2ms/step | 15.0 samples/s
+Step 17300/50462 | Loss: 2.6970 | LR: 3.75e-04 | 3998.3ms/step | 15.0 samples/s
+Step 17310/50462 | Loss: 2.6446 | LR: 3.75e-04 | 4007.9ms/step | 15.0 samples/s
+Step 17320/50462 | Loss: 2.6625 | LR: 3.75e-04 | 4006.6ms/step | 15.0 samples/s
+Step 17330/50462 | Loss: 2.6877 | LR: 3.75e-04 | 4008.6ms/step | 15.0 samples/s
+Step 17340/50462 | Loss: 2.6549 | LR: 3.75e-04 | 4003.6ms/step | 15.0 samples/s
+Step 17350/50462 | Loss: 2.6828 | LR: 3.75e-04 | 4015.6ms/step | 14.9 samples/s
+Step 17360/50462 | Loss: 2.6462 | LR: 3.74e-04 | 4025.1ms/step | 14.9 samples/s
+Step 17370/50462 | Loss: 2.6614 | LR: 3.74e-04 | 4016.3ms/step | 14.9 samples/s
+Step 17380/50462 | Loss: 2.6569 | LR: 3.74e-04 | 4014.8ms/step | 14.9 samples/s
+Step 17390/50462 | Loss: 2.6891 | LR: 3.74e-04 | 4023.0ms/step | 14.9 samples/s
+Step 17400/50462 | Loss: 2.6656 | LR: 3.74e-04 | 4020.0ms/step | 14.9 samples/s
+Step 17410/50462 | Loss: 2.6557 | LR: 3.74e-04 | 4012.1ms/step | 15.0 samples/s
+Step 17420/50462 | Loss: 2.6920 | LR: 3.74e-04 | 4016.5ms/step | 14.9 samples/s
+Step 17430/50462 | Loss: 2.6578 | LR: 3.74e-04 | 4010.9ms/step | 15.0 samples/s
+Step 17440/50462 | Loss: 2.6902 | LR: 3.73e-04 | 4008.0ms/step | 15.0 samples/s
+Step 17450/50462 | Loss: 2.6838 | LR: 3.73e-04 | 4010.1ms/step | 15.0 samples/s
+Step 17460/50462 | Loss: 2.6819 | LR: 3.73e-04 | 4009.4ms/step | 15.0 samples/s
+Step 17470/50462 | Loss: 2.6615 | LR: 3.73e-04 | 4011.7ms/step | 15.0 samples/s
+Step 17480/50462 | Loss: 2.6660 | LR: 3.73e-04 | 4018.5ms/step | 14.9 samples/s
+Step 17490/50462 | Loss: 2.6446 | LR: 3.73e-04 | 4008.6ms/step | 15.0 samples/s
+Step 17500/50462 | Loss: 2.6329 | LR: 3.73e-04 | 3994.9ms/step | 15.0 samples/s
+Validation at step 17500: Loss = 2.6499, Perplexity = 14.15
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-17500
+Step 17510/50462 | Loss: 2.6536 | LR: 3.72e-04 | 5968.0ms/step | 10.1 samples/s
+Step 17520/50462 | Loss: 2.6680 | LR: 3.72e-04 | 4005.6ms/step | 15.0 samples/s
+Step 17530/50462 | Loss: 2.6795 | LR: 3.72e-04 | 4008.8ms/step | 15.0 samples/s
+Step 17540/50462 | Loss: 2.7140 | LR: 3.72e-04 | 4021.2ms/step | 14.9 samples/s
+Step 17550/50462 | Loss: 2.6292 | LR: 3.72e-04 | 4004.9ms/step | 15.0 samples/s
+Step 17560/50462 | Loss: 2.6647 | LR: 3.72e-04 | 4005.3ms/step | 15.0 samples/s
+Step 17570/50462 | Loss: 2.6608 | LR: 3.72e-04 | 4001.3ms/step | 15.0 samples/s
+Step 17580/50462 | Loss: 2.6877 | LR: 3.71e-04 | 4004.3ms/step | 15.0 samples/s
+Step 17590/50462 | Loss: 2.6272 | LR: 3.71e-04 | 4014.3ms/step | 14.9 samples/s
+Step 17600/50462 | Loss: 2.6724 | LR: 3.71e-04 | 4004.0ms/step | 15.0 samples/s
+Step 17610/50462 | Loss: 2.6710 | LR: 3.71e-04 | 4018.9ms/step | 14.9 samples/s
+Step 17620/50462 | Loss: 2.6752 | LR: 3.71e-04 | 4022.5ms/step | 14.9 samples/s
+Step 17630/50462 | Loss: 2.6687 | LR: 3.71e-04 | 4001.7ms/step | 15.0 samples/s
+Step 17640/50462 | Loss: 2.6531 | LR: 3.71e-04 | 4005.0ms/step | 15.0 samples/s
+Step 17650/50462 | Loss: 2.6423 | LR: 3.70e-04 | 4005.5ms/step | 15.0 samples/s
+Step 17660/50462 | Loss: 2.6266 | LR: 3.70e-04 | 4001.5ms/step | 15.0 samples/s
+Step 17670/50462 | Loss: 2.6497 | LR: 3.70e-04 | 4011.4ms/step | 15.0 samples/s
+Step 17680/50462 | Loss: 2.6798 | LR: 3.70e-04 | 4006.3ms/step | 15.0 samples/s
+Step 17690/50462 | Loss: 2.6602 | LR: 3.70e-04 | 4009.3ms/step | 15.0 samples/s
+Step 17700/50462 | Loss: 2.6851 | LR: 3.70e-04 | 4006.6ms/step | 15.0 samples/s
+Step 17710/50462 | Loss: 2.6588 | LR: 3.70e-04 | 4001.7ms/step | 15.0 samples/s
+Step 17720/50462 | Loss: 2.6200 | LR: 3.69e-04 | 4016.4ms/step | 14.9 samples/s
+Step 17730/50462 | Loss: 2.6722 | LR: 3.69e-04 | 4006.0ms/step | 15.0 samples/s
+Step 17740/50462 | Loss: 2.6896 | LR: 3.69e-04 | 4006.8ms/step | 15.0 samples/s
+Step 17750/50462 | Loss: 2.6811 | LR: 3.69e-04 | 4009.2ms/step | 15.0 samples/s
+Validation at step 17750: Loss = 2.6476, Perplexity = 14.12
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-17750
+Step 17760/50462 | Loss: 2.6450 | LR: 3.69e-04 | 5960.6ms/step | 10.1 samples/s
+Step 17770/50462 | Loss: 2.6769 | LR: 3.69e-04 | 4012.2ms/step | 15.0 samples/s
+Step 17780/50462 | Loss: 2.6633 | LR: 3.69e-04 | 4004.9ms/step | 15.0 samples/s
+Step 17790/50462 | Loss: 2.6709 | LR: 3.69e-04 | 4013.5ms/step | 14.9 samples/s
+Step 17800/50462 | Loss: 2.6777 | LR: 3.68e-04 | 4012.0ms/step | 15.0 samples/s
+Step 17810/50462 | Loss: 2.6514 | LR: 3.68e-04 | 4008.4ms/step | 15.0 samples/s
+Step 17820/50462 | Loss: 2.6892 | LR: 3.68e-04 | 4021.2ms/step | 14.9 samples/s
+Step 17830/50462 | Loss: 2.6440 | LR: 3.68e-04 | 4014.1ms/step | 14.9 samples/s
+Step 17840/50462 | Loss: 2.6676 | LR: 3.68e-04 | 4015.6ms/step | 14.9 samples/s
+Step 17850/50462 | Loss: 2.6506 | LR: 3.68e-04 | 4012.7ms/step | 15.0 samples/s
+Step 17860/50462 | Loss: 2.6536 | LR: 3.68e-04 | 4004.9ms/step | 15.0 samples/s
+Step 17870/50462 | Loss: 2.6483 | LR: 3.67e-04 | 4003.1ms/step | 15.0 samples/s
+Step 17880/50462 | Loss: 2.6903 | LR: 3.67e-04 | 4012.5ms/step | 15.0 samples/s
+Step 17890/50462 | Loss: 2.6619 | LR: 3.67e-04 | 4020.3ms/step | 14.9 samples/s
+Step 17900/50462 | Loss: 2.6557 | LR: 3.67e-04 | 4005.7ms/step | 15.0 samples/s
+Step 17910/50462 | Loss: 2.6472 | LR: 3.67e-04 | 3999.8ms/step | 15.0 samples/s
+Step 17920/50462 | Loss: 2.6451 | LR: 3.67e-04 | 4003.5ms/step | 15.0 samples/s
+Step 17930/50462 | Loss: 2.6656 | LR: 3.67e-04 | 3998.0ms/step | 15.0 samples/s
+Step 17940/50462 | Loss: 2.6664 | LR: 3.66e-04 | 4000.5ms/step | 15.0 samples/s
+Step 17950/50462 | Loss: 2.6666 | LR: 3.66e-04 | 3998.5ms/step | 15.0 samples/s
+Step 17960/50462 | Loss: 2.6837 | LR: 3.66e-04 | 4009.9ms/step | 15.0 samples/s
+Step 17970/50462 | Loss: 2.6650 | LR: 3.66e-04 | 4006.3ms/step | 15.0 samples/s
+Step 17980/50462 | Loss: 2.6620 | LR: 3.66e-04 | 4007.8ms/step | 15.0 samples/s
+Step 17990/50462 | Loss: 2.6599 | LR: 3.66e-04 | 4007.6ms/step | 15.0 samples/s
+Step 18000/50462 | Loss: 2.6503 | LR: 3.66e-04 | 4007.4ms/step | 15.0 samples/s
+Validation at step 18000: Loss = 2.6438, Perplexity = 14.07
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-18000
+Step 18010/50462 | Loss: 2.6753 | LR: 3.65e-04 | 5967.4ms/step | 10.1 samples/s
+Step 18020/50462 | Loss: 2.6814 | LR: 3.65e-04 | 4002.7ms/step | 15.0 samples/s
+Step 18030/50462 | Loss: 2.6602 | LR: 3.65e-04 | 3998.3ms/step | 15.0 samples/s
+Step 18040/50462 | Loss: 2.6571 | LR: 3.65e-04 | 4010.9ms/step | 15.0 samples/s
+Step 18050/50462 | Loss: 2.6486 | LR: 3.65e-04 | 4005.6ms/step | 15.0 samples/s
+Step 18060/50462 | Loss: 2.6633 | LR: 3.65e-04 | 4004.8ms/step | 15.0 samples/s
+Step 18070/50462 | Loss: 2.6459 | LR: 3.65e-04 | 4001.0ms/step | 15.0 samples/s
+Step 18080/50462 | Loss: 2.6658 | LR: 3.64e-04 | 4017.0ms/step | 14.9 samples/s
+Step 18090/50462 | Loss: 2.6908 | LR: 3.64e-04 | 4007.1ms/step | 15.0 samples/s
+Step 18100/50462 | Loss: 2.6609 | LR: 3.64e-04 | 4001.7ms/step | 15.0 samples/s
+Step 18110/50462 | Loss: 2.6363 | LR: 3.64e-04 | 4008.5ms/step | 15.0 samples/s
+Step 18120/50462 | Loss: 2.6432 | LR: 3.64e-04 | 4002.0ms/step | 15.0 samples/s
+Step 18130/50462 | Loss: 2.6444 | LR: 3.64e-04 | 3999.4ms/step | 15.0 samples/s
+Step 18140/50462 | Loss: 2.6085 | LR: 3.64e-04 | 3999.1ms/step | 15.0 samples/s
+Step 18150/50462 | Loss: 2.6517 | LR: 3.63e-04 | 3995.8ms/step | 15.0 samples/s
+Step 18160/50462 | Loss: 2.6621 | LR: 3.63e-04 | 4000.6ms/step | 15.0 samples/s
+Step 18170/50462 | Loss: 2.6607 | LR: 3.63e-04 | 4006.7ms/step | 15.0 samples/s
+Step 18180/50462 | Loss: 2.6682 | LR: 3.63e-04 | 3996.2ms/step | 15.0 samples/s
+Step 18190/50462 | Loss: 2.6633 | LR: 3.63e-04 | 4001.5ms/step | 15.0 samples/s
+Step 18200/50462 | Loss: 2.6701 | LR: 3.63e-04 | 4006.1ms/step | 15.0 samples/s
+Step 18210/50462 | Loss: 2.6703 | LR: 3.63e-04 | 4008.5ms/step | 15.0 samples/s
+Step 18220/50462 | Loss: 2.6460 | LR: 3.62e-04 | 3996.4ms/step | 15.0 samples/s
+Step 18230/50462 | Loss: 2.6789 | LR: 3.62e-04 | 4005.9ms/step | 15.0 samples/s
+Step 18240/50462 | Loss: 2.6622 | LR: 3.62e-04 | 3997.7ms/step | 15.0 samples/s
+Step 18250/50462 | Loss: 2.6573 | LR: 3.62e-04 | 4003.2ms/step | 15.0 samples/s
+Validation at step 18250: Loss = 2.6411, Perplexity = 14.03
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-18250
+Step 18260/50462 | Loss: 2.6620 | LR: 3.62e-04 | 5958.4ms/step | 10.1 samples/s
+Step 18270/50462 | Loss: 2.6432 | LR: 3.62e-04 | 3992.0ms/step | 15.0 samples/s
+Step 18280/50462 | Loss: 2.6613 | LR: 3.62e-04 | 4010.1ms/step | 15.0 samples/s
+Step 18290/50462 | Loss: 2.6709 | LR: 3.62e-04 | 3999.4ms/step | 15.0 samples/s
+Step 18300/50462 | Loss: 2.6329 | LR: 3.61e-04 | 3998.5ms/step | 15.0 samples/s
+Step 18310/50462 | Loss: 2.6482 | LR: 3.61e-04 | 4006.2ms/step | 15.0 samples/s
+Step 18320/50462 | Loss: 2.6772 | LR: 3.61e-04 | 4003.3ms/step | 15.0 samples/s
+Step 18330/50462 | Loss: 2.6398 | LR: 3.61e-04 | 4005.8ms/step | 15.0 samples/s
+Step 18340/50462 | Loss: 2.6576 | LR: 3.61e-04 | 4010.6ms/step | 15.0 samples/s
+Step 18350/50462 | Loss: 2.6663 | LR: 3.61e-04 | 4005.6ms/step | 15.0 samples/s
+Step 18360/50462 | Loss: 2.6775 | LR: 3.61e-04 | 3991.8ms/step | 15.0 samples/s
+Step 18370/50462 | Loss: 2.6902 | LR: 3.60e-04 | 4011.1ms/step | 15.0 samples/s
+Step 18380/50462 | Loss: 2.6371 | LR: 3.60e-04 | 4016.1ms/step | 14.9 samples/s
+Step 18390/50462 | Loss: 2.6625 | LR: 3.60e-04 | 3995.9ms/step | 15.0 samples/s
+Step 18400/50462 | Loss: 2.6539 | LR: 3.60e-04 | 4002.5ms/step | 15.0 samples/s
+Step 18410/50462 | Loss: 2.6815 | LR: 3.60e-04 | 4006.1ms/step | 15.0 samples/s
+Step 18420/50462 | Loss: 2.6562 | LR: 3.60e-04 | 4000.2ms/step | 15.0 samples/s
+Step 18430/50462 | Loss: 2.6422 | LR: 3.60e-04 | 4013.1ms/step | 15.0 samples/s
+Step 18440/50462 | Loss: 2.6658 | LR: 3.59e-04 | 3998.4ms/step | 15.0 samples/s
+Step 18450/50462 | Loss: 2.6452 | LR: 3.59e-04 | 4003.7ms/step | 15.0 samples/s
+Step 18460/50462 | Loss: 2.6138 | LR: 3.59e-04 | 3994.1ms/step | 15.0 samples/s
+Step 18470/50462 | Loss: 2.6414 | LR: 3.59e-04 | 4011.1ms/step | 15.0 samples/s
+Step 18480/50462 | Loss: 2.6583 | LR: 3.59e-04 | 4018.0ms/step | 14.9 samples/s
+Step 18490/50462 | Loss: 2.6656 | LR: 3.59e-04 | 4013.4ms/step | 14.9 samples/s
+Step 18500/50462 | Loss: 2.6615 | LR: 3.59e-04 | 4019.1ms/step | 14.9 samples/s
+Validation at step 18500: Loss = 2.6374, Perplexity = 13.98
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-18500
+Step 18510/50462 | Loss: 2.6654 | LR: 3.58e-04 | 5981.7ms/step | 10.0 samples/s
+Step 18520/50462 | Loss: 2.6258 | LR: 3.58e-04 | 4009.2ms/step | 15.0 samples/s
+Step 18530/50462 | Loss: 2.6690 | LR: 3.58e-04 | 4010.1ms/step | 15.0 samples/s
+
+===== AUTO-RESUME FROM checkpoint-29000 2026-08-28T21:42:37Z =====
+W0828 21:42:40.314000 415 site-packages/torch/distributed/run.py:851] 
+W0828 21:42:40.314000 415 site-packages/torch/distributed/run.py:851] *****************************************
+W0828 21:42:40.314000 415 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0828 21:42:40.314000 415 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+W0828 21:42:55.374000 415 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+W0828 21:42:55.374000 415 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 483 closing signal SIGTERM
+W0828 21:42:55.375000 415 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 484 closing signal SIGTERM
+W0828 21:42:55.375000 415 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 485 closing signal SIGTERM
+W0828 21:42:55.376000 415 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 486 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 415 got signal: 15
+
+===== AUTO-RESUME FROM checkpoint-29000 2026-08-28T21:47:43Z =====
+W0828 21:47:46.975000 1687 site-packages/torch/distributed/run.py:851] 
+W0828 21:47:46.975000 1687 site-packages/torch/distributed/run.py:851] *****************************************
+W0828 21:47:46.975000 1687 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0828 21:47:46.975000 1687 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000...
+Preserving checkpoint LR state for exact resume: current=[0.00019678850748957392], schedule_base=[0.0005]
+Resumed from checkpoint at step 29000 (epoch 0, micro_step 432495)
+Resuming training from step 29000
+Resuming wandb run 8ynn1we6
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 8ynn1we6
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_215102-8ynn1we6
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed71 4gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8ynn1we6
+Starting training...
+Validation at step 29000: Loss = 2.5472, Perplexity = 12.77
+wandb: WARNING Tried to log to step 29000 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 29001: sparse_stochastic (configured sparse attention)
+Step 29010/50462 | Loss: 2.5465 | LR: 1.97e-04 | 105113.9ms/step | 0.6 samples/s
+wandb: WARNING Tried to log to step 29010 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29020/50462 | Loss: 2.5168 | LR: 1.96e-04 | 12079.1ms/step | 5.0 samples/s
+wandb: WARNING Tried to log to step 29020 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29030/50462 | Loss: 2.5452 | LR: 1.96e-04 | 12036.5ms/step | 5.0 samples/s
+wandb: WARNING Tried to log to step 29030 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29040/50462 | Loss: 2.5690 | LR: 1.96e-04 | 11882.0ms/step | 5.0 samples/s
+wandb: WARNING Tried to log to step 29040 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29050/50462 | Loss: 2.5354 | LR: 1.96e-04 | 11867.8ms/step | 5.1 samples/s
+wandb: WARNING Tried to log to step 29050 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29060/50462 | Loss: 2.5873 | LR: 1.96e-04 | 12090.1ms/step | 5.0 samples/s
+wandb: WARNING Tried to log to step 29060 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29070/50462 | Loss: 2.5506 | LR: 1.96e-04 | 11901.4ms/step | 5.0 samples/s
+Step 29080/50462 | Loss: 2.5596 | LR: 1.96e-04 | 11892.5ms/step | 5.0 samples/s
+Step 29090/50462 | Loss: 2.5504 | LR: 1.95e-04 | 11951.9ms/step | 5.0 samples/s
+Step 29100/50462 | Loss: 2.5628 | LR: 1.95e-04 | 11959.7ms/step | 5.0 samples/s
+Step 29110/50462 | Loss: 2.5619 | LR: 1.95e-04 | 12246.3ms/step | 4.9 samples/s
+Step 29120/50462 | Loss: 2.5659 | LR: 1.95e-04 | 12002.1ms/step | 5.0 samples/s
+Step 29130/50462 | Loss: 2.5645 | LR: 1.95e-04 | 12130.9ms/step | 4.9 samples/s
+Step 29140/50462 | Loss: 2.5994 | LR: 1.95e-04 | 11504.1ms/step | 5.2 samples/s
+Step 29150/50462 | Loss: 2.5235 | LR: 1.94e-04 | 10107.7ms/step | 5.9 samples/s
+Step 29160/50462 | Loss: 2.5563 | LR: 1.94e-04 | 12134.2ms/step | 4.9 samples/s
+Step 29170/50462 | Loss: 2.5235 | LR: 1.94e-04 | 12145.4ms/step | 4.9 samples/s
+Step 29180/50462 | Loss: 2.5304 | LR: 1.94e-04 | 12145.4ms/step | 4.9 samples/s
+Step 29190/50462 | Loss: 2.5503 | LR: 1.94e-04 | 12168.1ms/step | 4.9 samples/s
+Step 29200/50462 | Loss: 2.5488 | LR: 1.94e-04 | 12164.8ms/step | 4.9 samples/s
+Step 29210/50462 | Loss: 2.5682 | LR: 1.94e-04 | 11890.7ms/step | 5.0 samples/s
+Step 29220/50462 | Loss: 2.5340 | LR: 1.93e-04 | 12108.2ms/step | 5.0 samples/s
+Step 29230/50462 | Loss: 2.5666 | LR: 1.93e-04 | 11883.1ms/step | 5.0 samples/s
+Step 29240/50462 | Loss: 2.5868 | LR: 1.93e-04 | 11905.5ms/step | 5.0 samples/s
+Step 29250/50462 | Loss: 2.5577 | LR: 1.93e-04 | 12011.6ms/step | 5.0 samples/s
+Validation at step 29250: Loss = 2.5460, Perplexity = 12.76
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29250
+Step 29260/50462 | Loss: 2.5749 | LR: 1.93e-04 | 17422.6ms/step | 3.4 samples/s
+Step 29270/50462 | Loss: 2.5511 | LR: 1.93e-04 | 11929.3ms/step | 5.0 samples/s
+Step 29280/50462 | Loss: 2.5505 | LR: 1.92e-04 | 11987.7ms/step | 5.0 samples/s
+Step 29290/50462 | Loss: 2.5447 | LR: 1.92e-04 | 12069.4ms/step | 5.0 samples/s
+Step 29300/50462 | Loss: 2.5608 | LR: 1.92e-04 | 12088.7ms/step | 5.0 samples/s
+Step 29310/50462 | Loss: 2.5273 | LR: 1.92e-04 | 11856.9ms/step | 5.1 samples/s
+Step 29320/50462 | Loss: 2.5709 | LR: 1.92e-04 | 11931.1ms/step | 5.0 samples/s
+Step 29330/50462 | Loss: 2.5504 | LR: 1.92e-04 | 12268.7ms/step | 4.9 samples/s
+Step 29340/50462 | Loss: 2.5607 | LR: 1.92e-04 | 11681.2ms/step | 5.1 samples/s
+Step 29350/50462 | Loss: 2.5866 | LR: 1.91e-04 | 12018.1ms/step | 5.0 samples/s
+Step 29360/50462 | Loss: 2.5565 | LR: 1.91e-04 | 12117.6ms/step | 5.0 samples/s
+Step 29370/50462 | Loss: 2.5664 | LR: 1.91e-04 | 11901.0ms/step | 5.0 samples/s
+Step 29380/50462 | Loss: 2.5507 | LR: 1.91e-04 | 12056.6ms/step | 5.0 samples/s
+Step 29390/50462 | Loss: 2.5611 | LR: 1.91e-04 | 11629.5ms/step | 5.2 samples/s
+Step 29400/50462 | Loss: 2.5455 | LR: 1.91e-04 | 10053.3ms/step | 6.0 samples/s
+Step 29410/50462 | Loss: 2.5759 | LR: 1.90e-04 | 11928.4ms/step | 5.0 samples/s
+Step 29420/50462 | Loss: 2.5709 | LR: 1.90e-04 | 12079.3ms/step | 5.0 samples/s
+Step 29430/50462 | Loss: 2.5373 | LR: 1.90e-04 | 12110.5ms/step | 5.0 samples/s
+Step 29440/50462 | Loss: 2.5517 | LR: 1.90e-04 | 11866.1ms/step | 5.1 samples/s
+Step 29450/50462 | Loss: 2.5325 | LR: 1.90e-04 | 11997.2ms/step | 5.0 samples/s
+Step 29460/50462 | Loss: 2.5588 | LR: 1.90e-04 | 12094.7ms/step | 5.0 samples/s
+Step 29470/50462 | Loss: 2.5466 | LR: 1.90e-04 | 11908.0ms/step | 5.0 samples/s
+Step 29480/50462 | Loss: 2.5578 | LR: 1.89e-04 | 11859.6ms/step | 5.1 samples/s
+Step 29490/50462 | Loss: 2.5761 | LR: 1.89e-04 | 11837.1ms/step | 5.1 samples/s
+Step 29500/50462 | Loss: 2.5303 | LR: 1.89e-04 | 11847.8ms/step | 5.1 samples/s
+Validation at step 29500: Loss = 2.5443, Perplexity = 12.73
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29500
+Step 29510/50462 | Loss: 2.5208 | LR: 1.89e-04 | 17079.8ms/step | 3.5 samples/s
+Step 29520/50462 | Loss: 2.5629 | LR: 1.89e-04 | 12012.4ms/step | 5.0 samples/s
+Step 29530/50462 | Loss: 2.5463 | LR: 1.89e-04 | 12009.8ms/step | 5.0 samples/s
+Step 29540/50462 | Loss: 2.5673 | LR: 1.88e-04 | 11927.5ms/step | 5.0 samples/s
+Step 29550/50462 | Loss: 2.5592 | LR: 1.88e-04 | 11898.4ms/step | 5.0 samples/s
+Step 29560/50462 | Loss: 2.5307 | LR: 1.88e-04 | 11860.9ms/step | 5.1 samples/s
+Step 29570/50462 | Loss: 2.5442 | LR: 1.88e-04 | 12234.2ms/step | 4.9 samples/s
+Step 29580/50462 | Loss: 2.5741 | LR: 1.88e-04 | 11775.7ms/step | 5.1 samples/s
+Step 29590/50462 | Loss: 2.5524 | LR: 1.88e-04 | 11898.8ms/step | 5.0 samples/s
+Step 29600/50462 | Loss: 2.5645 | LR: 1.88e-04 | 11973.1ms/step | 5.0 samples/s
+Step 29610/50462 | Loss: 2.5682 | LR: 1.87e-04 | 11787.7ms/step | 5.1 samples/s
+Step 29620/50462 | Loss: 2.5469 | LR: 1.87e-04 | 12175.9ms/step | 4.9 samples/s
+Step 29630/50462 | Loss: 2.5522 | LR: 1.87e-04 | 11873.4ms/step | 5.1 samples/s
+Step 29640/50462 | Loss: 2.5661 | LR: 1.87e-04 | 11316.4ms/step | 5.3 samples/s
+Step 29650/50462 | Loss: 2.5361 | LR: 1.87e-04 | 10391.1ms/step | 5.8 samples/s
+Step 29660/50462 | Loss: 2.5809 | LR: 1.87e-04 | 11861.9ms/step | 5.1 samples/s
+Step 29670/50462 | Loss: 2.5482 | LR: 1.86e-04 | 12006.0ms/step | 5.0 samples/s
+Step 29680/50462 | Loss: 2.5631 | LR: 1.86e-04 | 11865.3ms/step | 5.1 samples/s
+Step 29690/50462 | Loss: 2.5693 | LR: 1.86e-04 | 12021.2ms/step | 5.0 samples/s
+Step 29700/50462 | Loss: 2.5583 | LR: 1.86e-04 | 12024.8ms/step | 5.0 samples/s
+Step 29710/50462 | Loss: 2.5798 | LR: 1.86e-04 | 12133.6ms/step | 4.9 samples/s
+Step 29720/50462 | Loss: 2.5395 | LR: 1.86e-04 | 12143.1ms/step | 4.9 samples/s
+Step 29730/50462 | Loss: 2.5603 | LR: 1.86e-04 | 12217.9ms/step | 4.9 samples/s
+Step 29740/50462 | Loss: 2.5936 | LR: 1.85e-04 | 12331.1ms/step | 4.9 samples/s
+Step 29750/50462 | Loss: 2.5300 | LR: 1.85e-04 | 12040.5ms/step | 5.0 samples/s
+Validation at step 29750: Loss = 2.5448, Perplexity = 12.74
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29750
+Step 29760/50462 | Loss: 2.5724 | LR: 1.85e-04 | 16992.2ms/step | 3.5 samples/s
+Step 29770/50462 | Loss: 2.5270 | LR: 1.85e-04 | 11696.2ms/step | 5.1 samples/s
+Step 29780/50462 | Loss: 2.5747 | LR: 1.85e-04 | 11975.8ms/step | 5.0 samples/s
+Step 29790/50462 | Loss: 2.5694 | LR: 1.85e-04 | 11760.6ms/step | 5.1 samples/s
+Step 29800/50462 | Loss: 2.5570 | LR: 1.85e-04 | 11959.3ms/step | 5.0 samples/s
+Step 29810/50462 | Loss: 2.5637 | LR: 1.84e-04 | 11827.8ms/step | 5.1 samples/s
+Step 29820/50462 | Loss: 2.5684 | LR: 1.84e-04 | 12060.6ms/step | 5.0 samples/s
+Step 29830/50462 | Loss: 2.5205 | LR: 1.84e-04 | 12001.3ms/step | 5.0 samples/s
+Step 29840/50462 | Loss: 2.5729 | LR: 1.84e-04 | 12042.0ms/step | 5.0 samples/s
+Step 29850/50462 | Loss: 2.5275 | LR: 1.84e-04 | 11871.6ms/step | 5.1 samples/s
+Step 29860/50462 | Loss: 2.5761 | LR: 1.84e-04 | 11908.4ms/step | 5.0 samples/s
+Step 29870/50462 | Loss: 2.5489 | LR: 1.83e-04 | 12167.8ms/step | 4.9 samples/s
+Step 29880/50462 | Loss: 2.5391 | LR: 1.83e-04 | 12006.3ms/step | 5.0 samples/s
+Step 29890/50462 | Loss: 2.5413 | LR: 1.83e-04 | 11538.6ms/step | 5.2 samples/s
+Step 29900/50462 | Loss: 2.5227 | LR: 1.83e-04 | 10085.0ms/step | 5.9 samples/s
+Step 29910/50462 | Loss: 2.5585 | LR: 1.83e-04 | 12078.2ms/step | 5.0 samples/s
+Step 29920/50462 | Loss: 2.5693 | LR: 1.83e-04 | 12109.7ms/step | 5.0 samples/s
+Step 29930/50462 | Loss: 2.5479 | LR: 1.83e-04 | 11811.9ms/step | 5.1 samples/s
+Step 29940/50462 | Loss: 2.5505 | LR: 1.82e-04 | 11975.4ms/step | 5.0 samples/s
+Step 29950/50462 | Loss: 2.5861 | LR: 1.82e-04 | 11809.3ms/step | 5.1 samples/s
+Step 29960/50462 | Loss: 2.5470 | LR: 1.82e-04 | 11957.4ms/step | 5.0 samples/s
+Step 29970/50462 | Loss: 2.5749 | LR: 1.82e-04 | 12253.5ms/step | 4.9 samples/s
+Step 29980/50462 | Loss: 2.5730 | LR: 1.82e-04 | 12052.0ms/step | 5.0 samples/s
+Step 29990/50462 | Loss: 2.5568 | LR: 1.82e-04 | 11975.7ms/step | 5.0 samples/s
+Step 30000/50462 | Loss: 2.5687 | LR: 1.81e-04 | 11835.5ms/step | 5.1 samples/s
+Validation at step 30000: Loss = 2.5429, Perplexity = 12.72
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30000
+Step 30010/50462 | Loss: 2.5326 | LR: 1.81e-04 | 17220.9ms/step | 3.5 samples/s
+Step 30020/50462 | Loss: 2.5468 | LR: 1.81e-04 | 12197.7ms/step | 4.9 samples/s
+Step 30030/50462 | Loss: 2.5323 | LR: 1.81e-04 | 11843.4ms/step | 5.1 samples/s
+Step 30040/50462 | Loss: 2.5470 | LR: 1.81e-04 | 11926.8ms/step | 5.0 samples/s
+Step 30050/50462 | Loss: 2.5496 | LR: 1.81e-04 | 11902.5ms/step | 5.0 samples/s
+Step 30060/50462 | Loss: 2.5366 | LR: 1.81e-04 | 11780.5ms/step | 5.1 samples/s
+Step 30070/50462 | Loss: 2.5249 | LR: 1.80e-04 | 12087.3ms/step | 5.0 samples/s
+Step 30080/50462 | Loss: 2.5219 | LR: 1.80e-04 | 11951.9ms/step | 5.0 samples/s
+Step 30090/50462 | Loss: 2.5584 | LR: 1.80e-04 | 11944.2ms/step | 5.0 samples/s
+Step 30100/50462 | Loss: 2.5619 | LR: 1.80e-04 | 11819.1ms/step | 5.1 samples/s
+Step 30110/50462 | Loss: 2.5073 | LR: 1.80e-04 | 11932.7ms/step | 5.0 samples/s
+Step 30120/50462 | Loss: 2.5425 | LR: 1.80e-04 | 12126.3ms/step | 4.9 samples/s
+Step 30130/50462 | Loss: 2.5363 | LR: 1.79e-04 | 12047.0ms/step | 5.0 samples/s
+Step 30140/50462 | Loss: 2.5634 | LR: 1.79e-04 | 11740.1ms/step | 5.1 samples/s
+Step 30150/50462 | Loss: 2.5304 | LR: 1.79e-04 | 9937.1ms/step | 6.0 samples/s
+Step 30160/50462 | Loss: 2.5273 | LR: 1.79e-04 | 11837.0ms/step | 5.1 samples/s
+Step 30170/50462 | Loss: 2.5864 | LR: 1.79e-04 | 11712.8ms/step | 5.1 samples/s
+Step 30180/50462 | Loss: 2.5439 | LR: 1.79e-04 | 11808.5ms/step | 5.1 samples/s
+Step 30190/50462 | Loss: 2.5634 | LR: 1.79e-04 | 11855.4ms/step | 5.1 samples/s
+Step 30200/50462 | Loss: 2.5368 | LR: 1.78e-04 | 12035.6ms/step | 5.0 samples/s
+Step 30210/50462 | Loss: 2.5514 | LR: 1.78e-04 | 11974.5ms/step | 5.0 samples/s
+Step 30220/50462 | Loss: 2.5701 | LR: 1.78e-04 | 11786.4ms/step | 5.1 samples/s
+Step 30230/50462 | Loss: 2.5407 | LR: 1.78e-04 | 11755.1ms/step | 5.1 samples/s
+Step 30240/50462 | Loss: 2.5461 | LR: 1.78e-04 | 11943.2ms/step | 5.0 samples/s
+Step 30250/50462 | Loss: 2.5235 | LR: 1.78e-04 | 11808.0ms/step | 5.1 samples/s
+Validation at step 30250: Loss = 2.5413, Perplexity = 12.70
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30250
+Step 30260/50462 | Loss: 2.5831 | LR: 1.78e-04 | 17312.3ms/step | 3.5 samples/s
+Step 30270/50462 | Loss: 2.5569 | LR: 1.77e-04 | 12078.7ms/step | 5.0 samples/s
+Step 30280/50462 | Loss: 2.5573 | LR: 1.77e-04 | 11995.0ms/step | 5.0 samples/s
+Step 30290/50462 | Loss: 2.5618 | LR: 1.77e-04 | 11980.1ms/step | 5.0 samples/s
+Step 30300/50462 | Loss: 2.5267 | LR: 1.77e-04 | 11978.5ms/step | 5.0 samples/s
+Step 30310/50462 | Loss: 2.5490 | LR: 1.77e-04 | 11940.3ms/step | 5.0 samples/s
+Step 30320/50462 | Loss: 2.5487 | LR: 1.77e-04 | 11879.0ms/step | 5.1 samples/s
+Step 30330/50462 | Loss: 2.5439 | LR: 1.76e-04 | 12112.0ms/step | 5.0 samples/s
+Step 30340/50462 | Loss: 2.5304 | LR: 1.76e-04 | 12165.9ms/step | 4.9 samples/s
+Step 30350/50462 | Loss: 2.5490 | LR: 1.76e-04 | 11922.8ms/step | 5.0 samples/s
+Step 30360/50462 | Loss: 2.5541 | LR: 1.76e-04 | 11820.3ms/step | 5.1 samples/s
+Step 30370/50462 | Loss: 2.5562 | LR: 1.76e-04 | 12066.8ms/step | 5.0 samples/s
+Step 30380/50462 | Loss: 2.5716 | LR: 1.76e-04 | 12223.0ms/step | 4.9 samples/s
+Step 30390/50462 | Loss: 2.5660 | LR: 1.76e-04 | 11933.0ms/step | 5.0 samples/s
+Step 30400/50462 | Loss: 2.5275 | LR: 1.75e-04 | 9690.5ms/step | 6.2 samples/s
+Step 30410/50462 | Loss: 2.5692 | LR: 1.75e-04 | 12061.1ms/step | 5.0 samples/s
+Step 30420/50462 | Loss: 2.5695 | LR: 1.75e-04 | 11744.3ms/step | 5.1 samples/s
+Step 30430/50462 | Loss: 2.5585 | LR: 1.75e-04 | 11998.8ms/step | 5.0 samples/s
+Step 30440/50462 | Loss: 2.5123 | LR: 1.75e-04 | 11972.4ms/step | 5.0 samples/s
+Step 30450/50462 | Loss: 2.5462 | LR: 1.75e-04 | 11997.8ms/step | 5.0 samples/s
+Step 30460/50462 | Loss: 2.5577 | LR: 1.75e-04 | 11891.5ms/step | 5.0 samples/s
+Step 30470/50462 | Loss: 2.5401 | LR: 1.74e-04 | 11968.3ms/step | 5.0 samples/s
+Step 30480/50462 | Loss: 2.5399 | LR: 1.74e-04 | 12080.9ms/step | 5.0 samples/s
+Step 30490/50462 | Loss: 2.5532 | LR: 1.74e-04 | 12102.5ms/step | 5.0 samples/s
+Step 30500/50462 | Loss: 2.5494 | LR: 1.74e-04 | 11998.7ms/step | 5.0 samples/s
+Validation at step 30500: Loss = 2.5397, Perplexity = 12.68
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30500
+Step 30510/50462 | Loss: 2.5273 | LR: 1.74e-04 | 17115.4ms/step | 3.5 samples/s
+Step 30520/50462 | Loss: 2.5862 | LR: 1.74e-04 | 12183.1ms/step | 4.9 samples/s
+Step 30530/50462 | Loss: 2.5591 | LR: 1.73e-04 | 11849.2ms/step | 5.1 samples/s
+Step 30540/50462 | Loss: 2.5506 | LR: 1.73e-04 | 12147.0ms/step | 4.9 samples/s
+Step 30550/50462 | Loss: 2.5670 | LR: 1.73e-04 | 11823.3ms/step | 5.1 samples/s
+Step 30560/50462 | Loss: 2.5413 | LR: 1.73e-04 | 12073.3ms/step | 5.0 samples/s
+Step 30570/50462 | Loss: 2.5557 | LR: 1.73e-04 | 11849.5ms/step | 5.1 samples/s
+Step 30580/50462 | Loss: 2.5546 | LR: 1.73e-04 | 12021.4ms/step | 5.0 samples/s
+Step 30590/50462 | Loss: 2.5587 | LR: 1.73e-04 | 11929.1ms/step | 5.0 samples/s
+Step 30600/50462 | Loss: 2.5823 | LR: 1.72e-04 | 11910.2ms/step | 5.0 samples/s
+Step 30610/50462 | Loss: 2.5677 | LR: 1.72e-04 | 12256.9ms/step | 4.9 samples/s
+Step 30620/50462 | Loss: 2.5349 | LR: 1.72e-04 | 12304.2ms/step | 4.9 samples/s
+Step 30630/50462 | Loss: 2.5687 | LR: 1.72e-04 | 11793.5ms/step | 5.1 samples/s
+Step 30640/50462 | Loss: 2.5348 | LR: 1.72e-04 | 12379.2ms/step | 4.8 samples/s
+Step 30650/50462 | Loss: 2.5321 | LR: 1.72e-04 | 9512.4ms/step | 6.3 samples/s
+Step 30660/50462 | Loss: 2.5199 | LR: 1.72e-04 | 12045.2ms/step | 5.0 samples/s
+Step 30670/50462 | Loss: 2.5480 | LR: 1.71e-04 | 11896.4ms/step | 5.0 samples/s
+Step 30680/50462 | Loss: 2.5683 | LR: 1.71e-04 | 12075.6ms/step | 5.0 samples/s
+Step 30690/50462 | Loss: 2.5589 | LR: 1.71e-04 | 12092.7ms/step | 5.0 samples/s
+W0829 03:47:17.983000 1687 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+W0829 03:47:17.993000 1687 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 1755 closing signal SIGTERM
+W0829 03:47:17.999000 1687 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 1756 closing signal SIGTERM
+W0829 03:47:18.003000 1687 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 1757 closing signal SIGTERM
+W0829 03:47:18.004000 1687 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 1758 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 1687 got signal: 15

epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu_resume28500.log

@@ -0,0 +1,294 @@
+
+===== RESUME FROM CHECKPOINT-28500 ON 4GPU 2026-08-28T13:32:29Z =====
+W0828 13:32:41.922000 130 site-packages/torch/distributed/run.py:851] 
+W0828 13:32:41.922000 130 site-packages/torch/distributed/run.py:851] *****************************************
+W0828 13:32:41.922000 130 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0828 13:32:41.922000 130 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28500",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28500
+Checkpoint activations: TrueCheckpoint activations: True
+Checkpoint activations: True
+
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28500...
+Preserving checkpoint LR state for exact resume: current=[0.00020453225696237217], schedule_base=[0.0005]
+Resumed from checkpoint at step 28500 (epoch 0, micro_step 424995)
+Resuming training from step 28500
+Resuming wandb run 8ynn1we6
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 8ynn1we6
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_133629-8ynn1we6
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed71 4gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8ynn1we6
+Starting training...
+Validation at step 28500: Loss = 2.5505, Perplexity = 12.81
+wandb: WARNING Tried to log to step 28500 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 28501: sparse_stochastic (configured sparse attention)
+Step 28510/50462 | Loss: 2.5451 | LR: 2.04e-04 | 76514.7ms/step | 0.8 samples/s
+wandb: WARNING Tried to log to step 28510 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28520/50462 | Loss: 2.5720 | LR: 2.04e-04 | 3980.4ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 28520 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28530/50462 | Loss: 2.5474 | LR: 2.04e-04 | 3977.4ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 28530 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28540/50462 | Loss: 2.5644 | LR: 2.04e-04 | 3982.4ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 28540 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28550/50462 | Loss: 2.5906 | LR: 2.04e-04 | 3987.8ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28550 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28560/50462 | Loss: 2.5604 | LR: 2.04e-04 | 3985.3ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 28560 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28570/50462 | Loss: 2.5596 | LR: 2.03e-04 | 3974.8ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 28570 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28580/50462 | Loss: 2.5638 | LR: 2.03e-04 | 4002.9ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28580 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28590/50462 | Loss: 2.5860 | LR: 2.03e-04 | 3995.8ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28590 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28600/50462 | Loss: 2.5439 | LR: 2.03e-04 | 3992.3ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28600 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28610/50462 | Loss: 2.5935 | LR: 2.03e-04 | 3982.6ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 28610 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28620/50462 | Loss: 2.5405 | LR: 2.03e-04 | 3981.1ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 28620 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28630/50462 | Loss: 2.5792 | LR: 2.03e-04 | 3996.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28630 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28640/50462 | Loss: 2.5795 | LR: 2.02e-04 | 3988.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28640 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28650/50462 | Loss: 2.5703 | LR: 2.02e-04 | 3988.3ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28650 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28660/50462 | Loss: 2.5650 | LR: 2.02e-04 | 3983.2ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 28660 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28670/50462 | Loss: 2.5692 | LR: 2.02e-04 | 3988.2ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28670 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28680/50462 | Loss: 2.5620 | LR: 2.02e-04 | 3980.9ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 28680 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28690/50462 | Loss: 2.5730 | LR: 2.02e-04 | 3987.5ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28690 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28700/50462 | Loss: 2.5995 | LR: 2.01e-04 | 3992.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28700 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28710/50462 | Loss: 2.5724 | LR: 2.01e-04 | 3986.5ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 28710 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28720/50462 | Loss: 2.5222 | LR: 2.01e-04 | 3992.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 28720 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 28730/50462 | Loss: 2.5623 | LR: 2.01e-04 | 3997.3ms/step | 15.0 samples/s
+Step 28740/50462 | Loss: 2.5654 | LR: 2.01e-04 | 3992.9ms/step | 15.0 samples/s
+Step 28750/50462 | Loss: 2.5627 | LR: 2.01e-04 | 3990.6ms/step | 15.0 samples/s
+Validation at step 28750: Loss = 2.5506, Perplexity = 12.82
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28750
+Step 28760/50462 | Loss: 2.5691 | LR: 2.00e-04 | 5924.9ms/step | 10.1 samples/s
+Step 28770/50462 | Loss: 2.5540 | LR: 2.00e-04 | 3984.7ms/step | 15.1 samples/s
+Step 28780/50462 | Loss: 2.5849 | LR: 2.00e-04 | 3990.8ms/step | 15.0 samples/s
+Step 28790/50462 | Loss: 2.5539 | LR: 2.00e-04 | 3992.7ms/step | 15.0 samples/s
+Step 28800/50462 | Loss: 2.5415 | LR: 2.00e-04 | 3981.7ms/step | 15.1 samples/s
+Step 28810/50462 | Loss: 2.5533 | LR: 2.00e-04 | 3988.3ms/step | 15.0 samples/s
+Step 28820/50462 | Loss: 2.5651 | LR: 2.00e-04 | 3983.2ms/step | 15.1 samples/s
+Step 28830/50462 | Loss: 2.5387 | LR: 1.99e-04 | 3982.7ms/step | 15.1 samples/s
+Step 28840/50462 | Loss: 2.5934 | LR: 1.99e-04 | 3981.2ms/step | 15.1 samples/s
+Step 28850/50462 | Loss: 2.5488 | LR: 1.99e-04 | 3988.4ms/step | 15.0 samples/s
+Step 28860/50462 | Loss: 2.5718 | LR: 1.99e-04 | 3991.5ms/step | 15.0 samples/s
+Step 28870/50462 | Loss: 2.5898 | LR: 1.99e-04 | 3994.7ms/step | 15.0 samples/s
+Step 28880/50462 | Loss: 2.5456 | LR: 1.99e-04 | 3983.8ms/step | 15.1 samples/s
+Step 28890/50462 | Loss: 2.5258 | LR: 1.98e-04 | 3989.6ms/step | 15.0 samples/s
+Step 28900/50462 | Loss: 2.5468 | LR: 1.98e-04 | 3988.6ms/step | 15.0 samples/s
+Step 28910/50462 | Loss: 2.5241 | LR: 1.98e-04 | 3991.9ms/step | 15.0 samples/s
+Step 28920/50462 | Loss: 2.5744 | LR: 1.98e-04 | 3994.6ms/step | 15.0 samples/s
+Step 28930/50462 | Loss: 2.5797 | LR: 1.98e-04 | 3976.5ms/step | 15.1 samples/s
+Step 28940/50462 | Loss: 2.5382 | LR: 1.98e-04 | 3990.5ms/step | 15.0 samples/s
+Step 28950/50462 | Loss: 2.5434 | LR: 1.98e-04 | 3979.6ms/step | 15.1 samples/s
+Step 28960/50462 | Loss: 2.5487 | LR: 1.97e-04 | 3984.5ms/step | 15.1 samples/s
+Step 28970/50462 | Loss: 2.5581 | LR: 1.97e-04 | 3980.7ms/step | 15.1 samples/s
+Step 28980/50462 | Loss: 2.5621 | LR: 1.97e-04 | 3991.3ms/step | 15.0 samples/s
+Step 28990/50462 | Loss: 2.5556 | LR: 1.97e-04 | 3981.1ms/step | 15.1 samples/s
+Step 29000/50462 | Loss: 2.5778 | LR: 1.97e-04 | 3979.1ms/step | 15.1 samples/s
+Validation at step 29000: Loss = 2.5477, Perplexity = 12.78
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000
+Step 29010/50462 | Loss: 2.5469 | LR: 1.97e-04 | 5909.8ms/step | 10.2 samples/s
+Step 29020/50462 | Loss: 2.5159 | LR: 1.96e-04 | 3986.9ms/step | 15.0 samples/s
+Step 29030/50462 | Loss: 2.5460 | LR: 1.96e-04 | 3991.0ms/step | 15.0 samples/s
+Step 29040/50462 | Loss: 2.5689 | LR: 1.96e-04 | 3994.9ms/step | 15.0 samples/s
+Step 29050/50462 | Loss: 2.5351 | LR: 1.96e-04 | 3979.8ms/step | 15.1 samples/s
+Step 29060/50462 | Loss: 2.5869 | LR: 1.96e-04 | 3985.0ms/step | 15.1 samples/s
+Step 29070/50462 | Loss: 2.5512 | LR: 1.96e-04 | 3996.0ms/step | 15.0 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu_resume29000.log

@@ -0,0 +1,794 @@
+
+===== RESUME FROM checkpoint-29000 ON 4GPU 2026-08-28T21:41:40+00:00 =====
+W0828 21:41:43.225000 404 site-packages/torch/distributed/run.py:851] 
+W0828 21:41:43.225000 404 site-packages/torch/distributed/run.py:851] *****************************************
+W0828 21:41:43.225000 404 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0828 21:41:43.225000 404 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000
+Checkpoint activations: TrueCheckpoint activations: True
+
+Checkpoint activations: TrueCheckpoint activations: True
+
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000...
+Preserving checkpoint LR state for exact resume: current=[0.00019678850748957392], schedule_base=[0.0005]
+Resumed from checkpoint at step 29000 (epoch 0, micro_step 432495)
+Resuming training from step 29000
+Resuming wandb run 8ynn1we6
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 8ynn1we6
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_214632-8ynn1we6
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed71 4gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8ynn1we6
+Starting training...
+Validation at step 29000: Loss = 2.5472, Perplexity = 12.77
+wandb: WARNING Tried to log to step 29000 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 29001: sparse_stochastic (configured sparse attention)
+Step 29010/50462 | Loss: 2.5465 | LR: 1.97e-04 | 69761.1ms/step | 0.9 samples/s
+wandb: WARNING Tried to log to step 29010 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29020/50462 | Loss: 2.5167 | LR: 1.96e-04 | 3987.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 29020 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29030/50462 | Loss: 2.5452 | LR: 1.96e-04 | 3985.3ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 29030 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29040/50462 | Loss: 2.5690 | LR: 1.96e-04 | 3980.9ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 29040 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29050/50462 | Loss: 2.5354 | LR: 1.96e-04 | 3989.2ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 29050 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29060/50462 | Loss: 2.5873 | LR: 1.96e-04 | 3984.3ms/step | 15.1 samples/s
+wandb: WARNING Tried to log to step 29060 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 29070/50462 | Loss: 2.5505 | LR: 1.96e-04 | 3989.7ms/step | 15.0 samples/s
+Step 29080/50462 | Loss: 2.5596 | LR: 1.96e-04 | 3988.9ms/step | 15.0 samples/s
+Step 29090/50462 | Loss: 2.5504 | LR: 1.95e-04 | 3992.3ms/step | 15.0 samples/s
+Step 29100/50462 | Loss: 2.5628 | LR: 1.95e-04 | 3980.2ms/step | 15.1 samples/s
+Step 29110/50462 | Loss: 2.5619 | LR: 1.95e-04 | 3993.2ms/step | 15.0 samples/s
+Step 29120/50462 | Loss: 2.5659 | LR: 1.95e-04 | 3991.7ms/step | 15.0 samples/s
+Step 29130/50462 | Loss: 2.5645 | LR: 1.95e-04 | 3995.5ms/step | 15.0 samples/s
+Step 29140/50462 | Loss: 2.5995 | LR: 1.95e-04 | 3989.8ms/step | 15.0 samples/s
+Step 29150/50462 | Loss: 2.5235 | LR: 1.94e-04 | 3983.8ms/step | 15.1 samples/s
+Step 29160/50462 | Loss: 2.5563 | LR: 1.94e-04 | 3985.3ms/step | 15.1 samples/s
+Step 29170/50462 | Loss: 2.5234 | LR: 1.94e-04 | 3976.5ms/step | 15.1 samples/s
+Step 29180/50462 | Loss: 2.5304 | LR: 1.94e-04 | 3988.9ms/step | 15.0 samples/s
+Step 29190/50462 | Loss: 2.5504 | LR: 1.94e-04 | 3986.2ms/step | 15.1 samples/s
+Step 29200/50462 | Loss: 2.5488 | LR: 1.94e-04 | 3981.4ms/step | 15.1 samples/s
+Step 29210/50462 | Loss: 2.5682 | LR: 1.94e-04 | 3982.7ms/step | 15.1 samples/s
+Step 29220/50462 | Loss: 2.5340 | LR: 1.93e-04 | 3976.0ms/step | 15.1 samples/s
+Step 29230/50462 | Loss: 2.5665 | LR: 1.93e-04 | 3982.2ms/step | 15.1 samples/s
+Step 29240/50462 | Loss: 2.5868 | LR: 1.93e-04 | 3985.4ms/step | 15.1 samples/s
+Step 29250/50462 | Loss: 2.5577 | LR: 1.93e-04 | 3965.7ms/step | 15.1 samples/s
+Validation at step 29250: Loss = 2.5460, Perplexity = 12.76
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29250
+Step 29260/50462 | Loss: 2.5749 | LR: 1.93e-04 | 5875.1ms/step | 10.2 samples/s
+Step 29270/50462 | Loss: 2.5511 | LR: 1.93e-04 | 3991.9ms/step | 15.0 samples/s
+Step 29280/50462 | Loss: 2.5505 | LR: 1.92e-04 | 3977.1ms/step | 15.1 samples/s
+Step 29290/50462 | Loss: 2.5447 | LR: 1.92e-04 | 3978.9ms/step | 15.1 samples/s
+Step 29300/50462 | Loss: 2.5608 | LR: 1.92e-04 | 3968.9ms/step | 15.1 samples/s
+Step 29310/50462 | Loss: 2.5273 | LR: 1.92e-04 | 3989.0ms/step | 15.0 samples/s
+Step 29320/50462 | Loss: 2.5709 | LR: 1.92e-04 | 3981.8ms/step | 15.1 samples/s
+Step 29330/50462 | Loss: 2.5505 | LR: 1.92e-04 | 3980.9ms/step | 15.1 samples/s
+Step 29340/50462 | Loss: 2.5607 | LR: 1.92e-04 | 3977.6ms/step | 15.1 samples/s
+Step 29350/50462 | Loss: 2.5865 | LR: 1.91e-04 | 3962.8ms/step | 15.1 samples/s
+Step 29360/50462 | Loss: 2.5565 | LR: 1.91e-04 | 3971.0ms/step | 15.1 samples/s
+Step 29370/50462 | Loss: 2.5664 | LR: 1.91e-04 | 3986.1ms/step | 15.1 samples/s
+Step 29380/50462 | Loss: 2.5507 | LR: 1.91e-04 | 3973.2ms/step | 15.1 samples/s
+Step 29390/50462 | Loss: 2.5610 | LR: 1.91e-04 | 3987.8ms/step | 15.0 samples/s
+Step 29400/50462 | Loss: 2.5456 | LR: 1.91e-04 | 3974.4ms/step | 15.1 samples/s
+Step 29410/50462 | Loss: 2.5759 | LR: 1.90e-04 | 3983.0ms/step | 15.1 samples/s
+Step 29420/50462 | Loss: 2.5709 | LR: 1.90e-04 | 3976.9ms/step | 15.1 samples/s
+Step 29430/50462 | Loss: 2.5373 | LR: 1.90e-04 | 3991.1ms/step | 15.0 samples/s
+Step 29440/50462 | Loss: 2.5516 | LR: 1.90e-04 | 3985.2ms/step | 15.1 samples/s
+Step 29450/50462 | Loss: 2.5324 | LR: 1.90e-04 | 3992.1ms/step | 15.0 samples/s
+Step 29460/50462 | Loss: 2.5588 | LR: 1.90e-04 | 3992.7ms/step | 15.0 samples/s
+Step 29470/50462 | Loss: 2.5467 | LR: 1.90e-04 | 3989.2ms/step | 15.0 samples/s
+Step 29480/50462 | Loss: 2.5578 | LR: 1.89e-04 | 3983.6ms/step | 15.1 samples/s
+Step 29490/50462 | Loss: 2.5761 | LR: 1.89e-04 | 3976.0ms/step | 15.1 samples/s
+Step 29500/50462 | Loss: 2.5302 | LR: 1.89e-04 | 3987.8ms/step | 15.0 samples/s
+Validation at step 29500: Loss = 2.5442, Perplexity = 12.73
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29500
+Step 29510/50462 | Loss: 2.5208 | LR: 1.89e-04 | 5907.8ms/step | 10.2 samples/s
+Step 29520/50462 | Loss: 2.5628 | LR: 1.89e-04 | 3998.4ms/step | 15.0 samples/s
+Step 29530/50462 | Loss: 2.5463 | LR: 1.89e-04 | 3990.3ms/step | 15.0 samples/s
+Step 29540/50462 | Loss: 2.5673 | LR: 1.88e-04 | 3980.8ms/step | 15.1 samples/s
+Step 29550/50462 | Loss: 2.5592 | LR: 1.88e-04 | 4002.7ms/step | 15.0 samples/s
+Step 29560/50462 | Loss: 2.5307 | LR: 1.88e-04 | 3995.1ms/step | 15.0 samples/s
+Step 29570/50462 | Loss: 2.5442 | LR: 1.88e-04 | 3986.4ms/step | 15.1 samples/s
+Step 29580/50462 | Loss: 2.5741 | LR: 1.88e-04 | 3976.7ms/step | 15.1 samples/s
+Step 29590/50462 | Loss: 2.5524 | LR: 1.88e-04 | 3977.9ms/step | 15.1 samples/s
+Step 29600/50462 | Loss: 2.5645 | LR: 1.88e-04 | 3982.3ms/step | 15.1 samples/s
+Step 29610/50462 | Loss: 2.5682 | LR: 1.87e-04 | 3983.5ms/step | 15.1 samples/s
+Step 29620/50462 | Loss: 2.5469 | LR: 1.87e-04 | 3993.8ms/step | 15.0 samples/s
+Step 29630/50462 | Loss: 2.5522 | LR: 1.87e-04 | 4000.4ms/step | 15.0 samples/s
+Step 29640/50462 | Loss: 2.5661 | LR: 1.87e-04 | 3990.2ms/step | 15.0 samples/s
+Step 29650/50462 | Loss: 2.5361 | LR: 1.87e-04 | 3980.1ms/step | 15.1 samples/s
+Step 29660/50462 | Loss: 2.5809 | LR: 1.87e-04 | 3993.8ms/step | 15.0 samples/s
+Step 29670/50462 | Loss: 2.5482 | LR: 1.86e-04 | 3981.1ms/step | 15.1 samples/s
+Step 29680/50462 | Loss: 2.5631 | LR: 1.86e-04 | 3982.5ms/step | 15.1 samples/s
+Step 29690/50462 | Loss: 2.5693 | LR: 1.86e-04 | 3984.4ms/step | 15.1 samples/s
+Step 29700/50462 | Loss: 2.5583 | LR: 1.86e-04 | 3979.4ms/step | 15.1 samples/s
+Step 29710/50462 | Loss: 2.5798 | LR: 1.86e-04 | 3980.3ms/step | 15.1 samples/s
+Step 29720/50462 | Loss: 2.5395 | LR: 1.86e-04 | 3991.4ms/step | 15.0 samples/s
+Step 29730/50462 | Loss: 2.5602 | LR: 1.86e-04 | 3976.3ms/step | 15.1 samples/s
+Step 29740/50462 | Loss: 2.5936 | LR: 1.85e-04 | 3986.1ms/step | 15.1 samples/s
+Step 29750/50462 | Loss: 2.5300 | LR: 1.85e-04 | 3984.8ms/step | 15.1 samples/s
+Validation at step 29750: Loss = 2.5448, Perplexity = 12.74
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29750
+Step 29760/50462 | Loss: 2.5724 | LR: 1.85e-04 | 5897.3ms/step | 10.2 samples/s
+Step 29770/50462 | Loss: 2.5270 | LR: 1.85e-04 | 3992.1ms/step | 15.0 samples/s
+Step 29780/50462 | Loss: 2.5747 | LR: 1.85e-04 | 3987.3ms/step | 15.0 samples/s
+Step 29790/50462 | Loss: 2.5694 | LR: 1.85e-04 | 3976.7ms/step | 15.1 samples/s
+Step 29800/50462 | Loss: 2.5570 | LR: 1.85e-04 | 3996.4ms/step | 15.0 samples/s
+Step 29810/50462 | Loss: 2.5637 | LR: 1.84e-04 | 3981.2ms/step | 15.1 samples/s
+Step 29820/50462 | Loss: 2.5685 | LR: 1.84e-04 | 3994.3ms/step | 15.0 samples/s
+Step 29830/50462 | Loss: 2.5205 | LR: 1.84e-04 | 3984.2ms/step | 15.1 samples/s
+Step 29840/50462 | Loss: 2.5729 | LR: 1.84e-04 | 3986.0ms/step | 15.1 samples/s
+Step 29850/50462 | Loss: 2.5275 | LR: 1.84e-04 | 3988.5ms/step | 15.0 samples/s
+Step 29860/50462 | Loss: 2.5761 | LR: 1.84e-04 | 3990.2ms/step | 15.0 samples/s
+Step 29870/50462 | Loss: 2.5489 | LR: 1.83e-04 | 3983.9ms/step | 15.1 samples/s
+Step 29880/50462 | Loss: 2.5391 | LR: 1.83e-04 | 3982.4ms/step | 15.1 samples/s
+Step 29890/50462 | Loss: 2.5414 | LR: 1.83e-04 | 3989.4ms/step | 15.0 samples/s
+Step 29900/50462 | Loss: 2.5228 | LR: 1.83e-04 | 3988.9ms/step | 15.0 samples/s
+Step 29910/50462 | Loss: 2.5585 | LR: 1.83e-04 | 3990.3ms/step | 15.0 samples/s
+Step 29920/50462 | Loss: 2.5693 | LR: 1.83e-04 | 3982.0ms/step | 15.1 samples/s
+Step 29930/50462 | Loss: 2.5479 | LR: 1.83e-04 | 3982.7ms/step | 15.1 samples/s
+Step 29940/50462 | Loss: 2.5505 | LR: 1.82e-04 | 3982.8ms/step | 15.1 samples/s
+Step 29950/50462 | Loss: 2.5861 | LR: 1.82e-04 | 3979.0ms/step | 15.1 samples/s
+Step 29960/50462 | Loss: 2.5470 | LR: 1.82e-04 | 4001.0ms/step | 15.0 samples/s
+Step 29970/50462 | Loss: 2.5750 | LR: 1.82e-04 | 3981.0ms/step | 15.1 samples/s
+Step 29980/50462 | Loss: 2.5730 | LR: 1.82e-04 | 3984.8ms/step | 15.1 samples/s
+Step 29990/50462 | Loss: 2.5568 | LR: 1.82e-04 | 3985.4ms/step | 15.1 samples/s
+Step 30000/50462 | Loss: 2.5687 | LR: 1.81e-04 | 3994.3ms/step | 15.0 samples/s
+Validation at step 30000: Loss = 2.5429, Perplexity = 12.72
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30000
+Step 30010/50462 | Loss: 2.5326 | LR: 1.81e-04 | 5928.2ms/step | 10.1 samples/s
+Step 30020/50462 | Loss: 2.5467 | LR: 1.81e-04 | 3982.9ms/step | 15.1 samples/s
+Step 30030/50462 | Loss: 2.5323 | LR: 1.81e-04 | 3986.1ms/step | 15.1 samples/s
+Step 30040/50462 | Loss: 2.5470 | LR: 1.81e-04 | 3983.4ms/step | 15.1 samples/s
+Step 30050/50462 | Loss: 2.5496 | LR: 1.81e-04 | 3979.3ms/step | 15.1 samples/s
+Step 30060/50462 | Loss: 2.5366 | LR: 1.81e-04 | 3980.6ms/step | 15.1 samples/s
+Step 30070/50462 | Loss: 2.5250 | LR: 1.80e-04 | 3992.2ms/step | 15.0 samples/s
+Step 30080/50462 | Loss: 2.5219 | LR: 1.80e-04 | 3984.7ms/step | 15.1 samples/s
+Step 30090/50462 | Loss: 2.5584 | LR: 1.80e-04 | 3989.0ms/step | 15.0 samples/s
+Step 30100/50462 | Loss: 2.5619 | LR: 1.80e-04 | 3983.3ms/step | 15.1 samples/s
+Step 30110/50462 | Loss: 2.5072 | LR: 1.80e-04 | 3988.1ms/step | 15.0 samples/s
+Step 30120/50462 | Loss: 2.5425 | LR: 1.80e-04 | 3985.9ms/step | 15.1 samples/s
+Step 30130/50462 | Loss: 2.5363 | LR: 1.79e-04 | 3986.2ms/step | 15.1 samples/s
+Step 30140/50462 | Loss: 2.5634 | LR: 1.79e-04 | 3998.5ms/step | 15.0 samples/s
+Step 30150/50462 | Loss: 2.5305 | LR: 1.79e-04 | 3986.6ms/step | 15.1 samples/s
+Step 30160/50462 | Loss: 2.5274 | LR: 1.79e-04 | 3977.9ms/step | 15.1 samples/s
+Step 30170/50462 | Loss: 2.5864 | LR: 1.79e-04 | 3994.8ms/step | 15.0 samples/s
+Step 30180/50462 | Loss: 2.5439 | LR: 1.79e-04 | 3992.6ms/step | 15.0 samples/s
+Step 30190/50462 | Loss: 2.5634 | LR: 1.79e-04 | 3978.2ms/step | 15.1 samples/s
+Step 30200/50462 | Loss: 2.5368 | LR: 1.78e-04 | 3983.8ms/step | 15.1 samples/s
+Step 30210/50462 | Loss: 2.5514 | LR: 1.78e-04 | 4585.9ms/step | 13.1 samples/s
+Step 30220/50462 | Loss: 2.5701 | LR: 1.78e-04 | 3988.9ms/step | 15.0 samples/s
+Step 30230/50462 | Loss: 2.5407 | LR: 1.78e-04 | 3983.5ms/step | 15.1 samples/s
+Step 30240/50462 | Loss: 2.5461 | LR: 1.78e-04 | 3989.4ms/step | 15.0 samples/s
+Step 30250/50462 | Loss: 2.5235 | LR: 1.78e-04 | 3974.7ms/step | 15.1 samples/s
+Validation at step 30250: Loss = 2.5413, Perplexity = 12.70
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30250
+Step 30260/50462 | Loss: 2.5831 | LR: 1.78e-04 | 5897.1ms/step | 10.2 samples/s
+Step 30270/50462 | Loss: 2.5569 | LR: 1.77e-04 | 3995.8ms/step | 15.0 samples/s
+Step 30280/50462 | Loss: 2.5572 | LR: 1.77e-04 | 3979.3ms/step | 15.1 samples/s
+Step 30290/50462 | Loss: 2.5618 | LR: 1.77e-04 | 3984.1ms/step | 15.1 samples/s
+Step 30300/50462 | Loss: 2.5267 | LR: 1.77e-04 | 3998.0ms/step | 15.0 samples/s
+Step 30310/50462 | Loss: 2.5489 | LR: 1.77e-04 | 3985.0ms/step | 15.1 samples/s
+Step 30320/50462 | Loss: 2.5487 | LR: 1.77e-04 | 3989.3ms/step | 15.0 samples/s
+Step 30330/50462 | Loss: 2.5439 | LR: 1.76e-04 | 3989.2ms/step | 15.0 samples/s
+Step 30340/50462 | Loss: 2.5304 | LR: 1.76e-04 | 3982.6ms/step | 15.1 samples/s
+Step 30350/50462 | Loss: 2.5489 | LR: 1.76e-04 | 3976.5ms/step | 15.1 samples/s
+Step 30360/50462 | Loss: 2.5540 | LR: 1.76e-04 | 3992.5ms/step | 15.0 samples/s
+Step 30370/50462 | Loss: 2.5561 | LR: 1.76e-04 | 4583.4ms/step | 13.1 samples/s
+Step 30380/50462 | Loss: 2.5716 | LR: 1.76e-04 | 3985.3ms/step | 15.1 samples/s
+Step 30390/50462 | Loss: 2.5661 | LR: 1.76e-04 | 3992.1ms/step | 15.0 samples/s
+Step 30400/50462 | Loss: 2.5275 | LR: 1.75e-04 | 3980.7ms/step | 15.1 samples/s
+Step 30410/50462 | Loss: 2.5692 | LR: 1.75e-04 | 3991.2ms/step | 15.0 samples/s
+Step 30420/50462 | Loss: 2.5696 | LR: 1.75e-04 | 3996.4ms/step | 15.0 samples/s
+Step 30430/50462 | Loss: 2.5585 | LR: 1.75e-04 | 3984.2ms/step | 15.1 samples/s
+Step 30440/50462 | Loss: 2.5123 | LR: 1.75e-04 | 3978.7ms/step | 15.1 samples/s
+Step 30450/50462 | Loss: 2.5462 | LR: 1.75e-04 | 3994.8ms/step | 15.0 samples/s
+Step 30460/50462 | Loss: 2.5577 | LR: 1.75e-04 | 3974.3ms/step | 15.1 samples/s
+Step 30470/50462 | Loss: 2.5401 | LR: 1.74e-04 | 3993.2ms/step | 15.0 samples/s
+Step 30480/50462 | Loss: 2.5399 | LR: 1.74e-04 | 3986.8ms/step | 15.0 samples/s
+Step 30490/50462 | Loss: 2.5532 | LR: 1.74e-04 | 3983.3ms/step | 15.1 samples/s
+Step 30500/50462 | Loss: 2.5494 | LR: 1.74e-04 | 3985.2ms/step | 15.1 samples/s
+Validation at step 30500: Loss = 2.5397, Perplexity = 12.68
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30500
+Step 30510/50462 | Loss: 2.5273 | LR: 1.74e-04 | 5886.6ms/step | 10.2 samples/s
+Step 30520/50462 | Loss: 2.5863 | LR: 1.74e-04 | 3993.9ms/step | 15.0 samples/s
+Step 30530/50462 | Loss: 2.5592 | LR: 1.73e-04 | 3998.5ms/step | 15.0 samples/s
+Step 30540/50462 | Loss: 2.5506 | LR: 1.73e-04 | 3984.7ms/step | 15.1 samples/s
+Step 30550/50462 | Loss: 2.5670 | LR: 1.73e-04 | 3984.9ms/step | 15.1 samples/s
+Step 30560/50462 | Loss: 2.5413 | LR: 1.73e-04 | 3977.7ms/step | 15.1 samples/s
+Step 30570/50462 | Loss: 2.5556 | LR: 1.73e-04 | 3992.9ms/step | 15.0 samples/s
+Step 30580/50462 | Loss: 2.5546 | LR: 1.73e-04 | 3986.2ms/step | 15.1 samples/s
+Step 30590/50462 | Loss: 2.5587 | LR: 1.73e-04 | 3991.3ms/step | 15.0 samples/s
+Step 30600/50462 | Loss: 2.5823 | LR: 1.72e-04 | 3986.6ms/step | 15.1 samples/s
+Step 30610/50462 | Loss: 2.5677 | LR: 1.72e-04 | 3993.1ms/step | 15.0 samples/s
+Step 30620/50462 | Loss: 2.5349 | LR: 1.72e-04 | 3994.7ms/step | 15.0 samples/s
+Step 30630/50462 | Loss: 2.5687 | LR: 1.72e-04 | 4017.6ms/step | 14.9 samples/s
+Step 30640/50462 | Loss: 2.5348 | LR: 1.72e-04 | 4018.2ms/step | 14.9 samples/s
+Step 30650/50462 | Loss: 2.5321 | LR: 1.72e-04 | 4010.4ms/step | 15.0 samples/s
+Step 30660/50462 | Loss: 2.5199 | LR: 1.72e-04 | 4010.9ms/step | 15.0 samples/s
+Step 30670/50462 | Loss: 2.5480 | LR: 1.71e-04 | 3995.0ms/step | 15.0 samples/s
+Step 30680/50462 | Loss: 2.5683 | LR: 1.71e-04 | 3997.1ms/step | 15.0 samples/s
+Step 30690/50462 | Loss: 2.5588 | LR: 1.71e-04 | 4002.1ms/step | 15.0 samples/s
+Step 30700/50462 | Loss: 2.5392 | LR: 1.71e-04 | 4573.7ms/step | 13.1 samples/s
+Step 30710/50462 | Loss: 2.5489 | LR: 1.71e-04 | 3993.7ms/step | 15.0 samples/s
+Step 30720/50462 | Loss: 2.5524 | LR: 1.71e-04 | 4005.1ms/step | 15.0 samples/s
+Step 30730/50462 | Loss: 2.5375 | LR: 1.70e-04 | 4585.5ms/step | 13.1 samples/s
+Step 30740/50462 | Loss: 2.5377 | LR: 1.70e-04 | 3995.9ms/step | 15.0 samples/s
+Step 30750/50462 | Loss: 2.5375 | LR: 1.70e-04 | 4000.8ms/step | 15.0 samples/s
+Validation at step 30750: Loss = 2.5388, Perplexity = 12.66
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30750
+Step 30760/50462 | Loss: 2.5459 | LR: 1.70e-04 | 5909.0ms/step | 10.2 samples/s
+Step 30770/50462 | Loss: 2.5489 | LR: 1.70e-04 | 3997.7ms/step | 15.0 samples/s
+Step 30780/50462 | Loss: 2.5801 | LR: 1.70e-04 | 4002.4ms/step | 15.0 samples/s
+Step 30790/50462 | Loss: 2.5618 | LR: 1.70e-04 | 3993.9ms/step | 15.0 samples/s
+Step 30800/50462 | Loss: 2.5627 | LR: 1.69e-04 | 3996.9ms/step | 15.0 samples/s
+Step 30810/50462 | Loss: 2.5269 | LR: 1.69e-04 | 3988.8ms/step | 15.0 samples/s
+Step 30820/50462 | Loss: 2.5471 | LR: 1.69e-04 | 3998.5ms/step | 15.0 samples/s
+Step 30830/50462 | Loss: 2.5710 | LR: 1.69e-04 | 3994.2ms/step | 15.0 samples/s
+Step 30840/50462 | Loss: 2.5394 | LR: 1.69e-04 | 3983.7ms/step | 15.1 samples/s
+Step 30850/50462 | Loss: 2.5713 | LR: 1.69e-04 | 3992.7ms/step | 15.0 samples/s
+Step 30860/50462 | Loss: 2.5933 | LR: 1.69e-04 | 4005.3ms/step | 15.0 samples/s
+Step 30870/50462 | Loss: 2.5611 | LR: 1.68e-04 | 3998.3ms/step | 15.0 samples/s
+Step 30880/50462 | Loss: 2.5608 | LR: 1.68e-04 | 3995.9ms/step | 15.0 samples/s
+Step 30890/50462 | Loss: 2.5533 | LR: 1.68e-04 | 3996.9ms/step | 15.0 samples/s
+Step 30900/50462 | Loss: 2.5587 | LR: 1.68e-04 | 3994.1ms/step | 15.0 samples/s
+Step 30910/50462 | Loss: 2.5660 | LR: 1.68e-04 | 3998.7ms/step | 15.0 samples/s
+Step 30920/50462 | Loss: 2.5657 | LR: 1.68e-04 | 4000.1ms/step | 15.0 samples/s
+Step 30930/50462 | Loss: 2.5450 | LR: 1.67e-04 | 4003.1ms/step | 15.0 samples/s
+Step 30940/50462 | Loss: 2.5559 | LR: 1.67e-04 | 3990.7ms/step | 15.0 samples/s
+Step 30950/50462 | Loss: 2.5527 | LR: 1.67e-04 | 3992.3ms/step | 15.0 samples/s
+Step 30960/50462 | Loss: 2.5343 | LR: 1.67e-04 | 3991.3ms/step | 15.0 samples/s
+Step 30970/50462 | Loss: 2.5656 | LR: 1.67e-04 | 3994.5ms/step | 15.0 samples/s
+Step 30980/50462 | Loss: 2.5456 | LR: 1.67e-04 | 4001.7ms/step | 15.0 samples/s
+Step 30990/50462 | Loss: 2.5560 | LR: 1.67e-04 | 4011.8ms/step | 15.0 samples/s
+Step 31000/50462 | Loss: 2.4963 | LR: 1.66e-04 | 4019.0ms/step | 14.9 samples/s
+Validation at step 31000: Loss = 2.5375, Perplexity = 12.65
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-31000
+Step 31010/50462 | Loss: 2.5399 | LR: 1.66e-04 | 5965.3ms/step | 10.1 samples/s
+Step 31020/50462 | Loss: 2.5483 | LR: 1.66e-04 | 4024.6ms/step | 14.9 samples/s
+Step 31030/50462 | Loss: 2.5314 | LR: 1.66e-04 | 4015.7ms/step | 14.9 samples/s
+Step 31040/50462 | Loss: 2.5474 | LR: 1.66e-04 | 4009.3ms/step | 15.0 samples/s
+Step 31050/50462 | Loss: 2.5361 | LR: 1.66e-04 | 4018.4ms/step | 14.9 samples/s
+Step 31060/50462 | Loss: 2.5478 | LR: 1.66e-04 | 4009.5ms/step | 15.0 samples/s
+Step 31070/50462 | Loss: 2.5506 | LR: 1.65e-04 | 4016.6ms/step | 14.9 samples/s
+Step 31080/50462 | Loss: 2.5307 | LR: 1.65e-04 | 4019.7ms/step | 14.9 samples/s
+Step 31090/50462 | Loss: 2.5569 | LR: 1.65e-04 | 4024.1ms/step | 14.9 samples/s
+Step 31100/50462 | Loss: 2.5829 | LR: 1.65e-04 | 4021.7ms/step | 14.9 samples/s
+Step 31110/50462 | Loss: 2.5786 | LR: 1.65e-04 | 4013.8ms/step | 14.9 samples/s
+Step 31120/50462 | Loss: 2.5577 | LR: 1.65e-04 | 4013.2ms/step | 15.0 samples/s
+Step 31130/50462 | Loss: 2.5301 | LR: 1.64e-04 | 4016.7ms/step | 14.9 samples/s
+Step 31140/50462 | Loss: 2.5472 | LR: 1.64e-04 | 4023.6ms/step | 14.9 samples/s
+Step 31150/50462 | Loss: 2.5368 | LR: 1.64e-04 | 4014.8ms/step | 14.9 samples/s
+Step 31160/50462 | Loss: 2.5665 | LR: 1.64e-04 | 4009.1ms/step | 15.0 samples/s
+Step 31170/50462 | Loss: 2.5616 | LR: 1.64e-04 | 4008.6ms/step | 15.0 samples/s
+Step 31180/50462 | Loss: 2.5573 | LR: 1.64e-04 | 4026.7ms/step | 14.9 samples/s
+Step 31190/50462 | Loss: 2.5453 | LR: 1.64e-04 | 4009.0ms/step | 15.0 samples/s
+Step 31200/50462 | Loss: 2.5745 | LR: 1.63e-04 | 4026.0ms/step | 14.9 samples/s
+Step 31210/50462 | Loss: 2.5811 | LR: 1.63e-04 | 4024.5ms/step | 14.9 samples/s
+Step 31220/50462 | Loss: 2.5491 | LR: 1.63e-04 | 4030.0ms/step | 14.9 samples/s
+Step 31230/50462 | Loss: 2.5344 | LR: 1.63e-04 | 4008.4ms/step | 15.0 samples/s
+Step 31240/50462 | Loss: 2.5518 | LR: 1.63e-04 | 4014.2ms/step | 14.9 samples/s
+Step 31250/50462 | Loss: 2.5773 | LR: 1.63e-04 | 4008.5ms/step | 15.0 samples/s
+Validation at step 31250: Loss = 2.5363, Perplexity = 12.63
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-31250
+Step 31260/50462 | Loss: 2.5863 | LR: 1.63e-04 | 5937.4ms/step | 10.1 samples/s
+Step 31270/50462 | Loss: 2.5513 | LR: 1.62e-04 | 4026.0ms/step | 14.9 samples/s
+Step 31280/50462 | Loss: 2.5491 | LR: 1.62e-04 | 4012.9ms/step | 15.0 samples/s
+Step 31290/50462 | Loss: 2.5909 | LR: 1.62e-04 | 4023.2ms/step | 14.9 samples/s
+Step 31300/50462 | Loss: 2.5514 | LR: 1.62e-04 | 4021.0ms/step | 14.9 samples/s
+Step 31310/50462 | Loss: 2.5486 | LR: 1.62e-04 | 4006.4ms/step | 15.0 samples/s
+Step 31320/50462 | Loss: 2.5626 | LR: 1.62e-04 | 4008.6ms/step | 15.0 samples/s
+Step 31330/50462 | Loss: 2.5745 | LR: 1.62e-04 | 4007.9ms/step | 15.0 samples/s
+Step 31340/50462 | Loss: 2.5606 | LR: 1.61e-04 | 4007.8ms/step | 15.0 samples/s
+Step 31350/50462 | Loss: 2.5642 | LR: 1.61e-04 | 4012.3ms/step | 15.0 samples/s
+Step 31360/50462 | Loss: 2.5640 | LR: 1.61e-04 | 4020.7ms/step | 14.9 samples/s
+Step 31370/50462 | Loss: 2.5501 | LR: 1.61e-04 | 4017.1ms/step | 14.9 samples/s
+Step 31380/50462 | Loss: 2.5525 | LR: 1.61e-04 | 4005.5ms/step | 15.0 samples/s
+Step 31390/50462 | Loss: 2.5786 | LR: 1.61e-04 | 4015.4ms/step | 14.9 samples/s
+Step 31400/50462 | Loss: 2.5493 | LR: 1.60e-04 | 4002.6ms/step | 15.0 samples/s
+Step 31410/50462 | Loss: 2.5167 | LR: 1.60e-04 | 4003.3ms/step | 15.0 samples/s
+Step 31420/50462 | Loss: 2.5882 | LR: 1.60e-04 | 3998.9ms/step | 15.0 samples/s
+Step 31430/50462 | Loss: 2.5397 | LR: 1.60e-04 | 4002.4ms/step | 15.0 samples/s
+Step 31440/50462 | Loss: 2.5867 | LR: 1.60e-04 | 4008.3ms/step | 15.0 samples/s
+Step 31450/50462 | Loss: 2.5532 | LR: 1.60e-04 | 4007.2ms/step | 15.0 samples/s
+Step 31460/50462 | Loss: 2.5370 | LR: 1.60e-04 | 4012.6ms/step | 15.0 samples/s
+Step 31470/50462 | Loss: 2.5664 | LR: 1.59e-04 | 4011.0ms/step | 15.0 samples/s
+Step 31480/50462 | Loss: 2.5354 | LR: 1.59e-04 | 4021.0ms/step | 14.9 samples/s
+Step 31490/50462 | Loss: 2.5244 | LR: 1.59e-04 | 4019.4ms/step | 14.9 samples/s
+Step 31500/50462 | Loss: 2.5909 | LR: 1.59e-04 | 4014.9ms/step | 14.9 samples/s
+Validation at step 31500: Loss = 2.5352, Perplexity = 12.62
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-31500
+Step 31510/50462 | Loss: 2.5504 | LR: 1.59e-04 | 5917.3ms/step | 10.1 samples/s
+Step 31520/50462 | Loss: 2.5485 | LR: 1.59e-04 | 4003.2ms/step | 15.0 samples/s
+Step 31530/50462 | Loss: 2.5791 | LR: 1.59e-04 | 4006.0ms/step | 15.0 samples/s
+Step 31540/50462 | Loss: 2.5473 | LR: 1.58e-04 | 4007.4ms/step | 15.0 samples/s
+Step 31550/50462 | Loss: 2.5473 | LR: 1.58e-04 | 4006.6ms/step | 15.0 samples/s
+Step 31560/50462 | Loss: 2.5479 | LR: 1.58e-04 | 3996.5ms/step | 15.0 samples/s
+Step 31570/50462 | Loss: 2.5394 | LR: 1.58e-04 | 4009.2ms/step | 15.0 samples/s
+Step 31580/50462 | Loss: 2.5267 | LR: 1.58e-04 | 4006.2ms/step | 15.0 samples/s
+Step 31590/50462 | Loss: 2.5394 | LR: 1.58e-04 | 4004.9ms/step | 15.0 samples/s
+Step 31600/50462 | Loss: 2.5777 | LR: 1.58e-04 | 3998.6ms/step | 15.0 samples/s
+Step 31610/50462 | Loss: 2.5267 | LR: 1.57e-04 | 3991.1ms/step | 15.0 samples/s
+Step 31620/50462 | Loss: 2.5502 | LR: 1.57e-04 | 3996.2ms/step | 15.0 samples/s
+Step 31630/50462 | Loss: 2.5190 | LR: 1.57e-04 | 4000.0ms/step | 15.0 samples/s
+Step 31640/50462 | Loss: 2.5518 | LR: 1.57e-04 | 4005.4ms/step | 15.0 samples/s
+Step 31650/50462 | Loss: 2.5342 | LR: 1.57e-04 | 4004.9ms/step | 15.0 samples/s
+Step 31660/50462 | Loss: 2.5939 | LR: 1.57e-04 | 3998.6ms/step | 15.0 samples/s
+Step 31670/50462 | Loss: 2.5803 | LR: 1.57e-04 | 4011.0ms/step | 15.0 samples/s
+Step 31680/50462 | Loss: 2.5215 | LR: 1.56e-04 | 4011.0ms/step | 15.0 samples/s
+Step 31690/50462 | Loss: 2.5518 | LR: 1.56e-04 | 4007.5ms/step | 15.0 samples/s
+Step 31700/50462 | Loss: 2.5528 | LR: 1.56e-04 | 4000.9ms/step | 15.0 samples/s
+Step 31710/50462 | Loss: 2.5412 | LR: 1.56e-04 | 3998.4ms/step | 15.0 samples/s
+Step 31720/50462 | Loss: 2.5706 | LR: 1.56e-04 | 4004.2ms/step | 15.0 samples/s
+Step 31730/50462 | Loss: 2.5727 | LR: 1.56e-04 | 3996.7ms/step | 15.0 samples/s
+Step 31740/50462 | Loss: 2.5718 | LR: 1.55e-04 | 3997.7ms/step | 15.0 samples/s
+Step 31750/50462 | Loss: 2.5136 | LR: 1.55e-04 | 4012.2ms/step | 15.0 samples/s
+Validation at step 31750: Loss = 2.5347, Perplexity = 12.61
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-31750
+Step 31760/50462 | Loss: 2.5616 | LR: 1.55e-04 | 5919.8ms/step | 10.1 samples/s
+Step 31770/50462 | Loss: 2.5832 | LR: 1.55e-04 | 4003.3ms/step | 15.0 samples/s
+Step 31780/50462 | Loss: 2.5087 | LR: 1.55e-04 | 4010.1ms/step | 15.0 samples/s
+Step 31790/50462 | Loss: 2.5240 | LR: 1.55e-04 | 4015.0ms/step | 14.9 samples/s
+Step 31800/50462 | Loss: 2.5576 | LR: 1.55e-04 | 3992.9ms/step | 15.0 samples/s
+Step 31810/50462 | Loss: 2.5349 | LR: 1.54e-04 | 4003.6ms/step | 15.0 samples/s
+Step 31820/50462 | Loss: 2.5401 | LR: 1.54e-04 | 4001.6ms/step | 15.0 samples/s
+Step 31830/50462 | Loss: 2.5467 | LR: 1.54e-04 | 3992.7ms/step | 15.0 samples/s
+Step 31840/50462 | Loss: 2.5450 | LR: 1.54e-04 | 3990.7ms/step | 15.0 samples/s
+Step 31850/50462 | Loss: 2.5814 | LR: 1.54e-04 | 4001.8ms/step | 15.0 samples/s
+Step 31860/50462 | Loss: 2.5581 | LR: 1.54e-04 | 3992.6ms/step | 15.0 samples/s
+Step 31870/50462 | Loss: 2.6159 | LR: 1.54e-04 | 4007.7ms/step | 15.0 samples/s
+Step 31880/50462 | Loss: 2.5704 | LR: 1.53e-04 | 4004.8ms/step | 15.0 samples/s
+Step 31890/50462 | Loss: 2.5370 | LR: 1.53e-04 | 4005.4ms/step | 15.0 samples/s
+Step 31900/50462 | Loss: 2.5506 | LR: 1.53e-04 | 4000.7ms/step | 15.0 samples/s
+Step 31910/50462 | Loss: 2.5195 | LR: 1.53e-04 | 4009.0ms/step | 15.0 samples/s
+Step 31920/50462 | Loss: 2.5121 | LR: 1.53e-04 | 4003.4ms/step | 15.0 samples/s
+Step 31930/50462 | Loss: 2.5359 | LR: 1.53e-04 | 3994.4ms/step | 15.0 samples/s
+Step 31940/50462 | Loss: 2.5212 | LR: 1.53e-04 | 4000.1ms/step | 15.0 samples/s
+Step 31950/50462 | Loss: 2.5410 | LR: 1.52e-04 | 3999.9ms/step | 15.0 samples/s
+Step 31960/50462 | Loss: 2.5588 | LR: 1.52e-04 | 3991.5ms/step | 15.0 samples/s
+Step 31970/50462 | Loss: 2.5588 | LR: 1.52e-04 | 4012.3ms/step | 15.0 samples/s
+Step 31980/50462 | Loss: 2.5522 | LR: 1.52e-04 | 3995.2ms/step | 15.0 samples/s
+Step 31990/50462 | Loss: 2.5022 | LR: 1.52e-04 | 3992.8ms/step | 15.0 samples/s
+Step 32000/50462 | Loss: 2.5552 | LR: 1.52e-04 | 4003.1ms/step | 15.0 samples/s
+Validation at step 32000: Loss = 2.5339, Perplexity = 12.60
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-32000
+Step 32010/50462 | Loss: 2.5440 | LR: 1.52e-04 | 5929.5ms/step | 10.1 samples/s
+Step 32020/50462 | Loss: 2.5226 | LR: 1.51e-04 | 4005.0ms/step | 15.0 samples/s
+Step 32030/50462 | Loss: 2.5572 | LR: 1.51e-04 | 4000.9ms/step | 15.0 samples/s
+Step 32040/50462 | Loss: 2.5329 | LR: 1.51e-04 | 3989.6ms/step | 15.0 samples/s
+Step 32050/50462 | Loss: 2.5465 | LR: 1.51e-04 | 4009.7ms/step | 15.0 samples/s
+Step 32060/50462 | Loss: 2.5826 | LR: 1.51e-04 | 4001.3ms/step | 15.0 samples/s
+Step 32070/50462 | Loss: 2.5750 | LR: 1.51e-04 | 3985.6ms/step | 15.1 samples/s
+Step 32080/50462 | Loss: 2.5734 | LR: 1.51e-04 | 3998.9ms/step | 15.0 samples/s
+Step 32090/50462 | Loss: 2.5172 | LR: 1.50e-04 | 3996.0ms/step | 15.0 samples/s
+Step 32100/50462 | Loss: 2.5301 | LR: 1.50e-04 | 3999.9ms/step | 15.0 samples/s
+Step 32110/50462 | Loss: 2.5331 | LR: 1.50e-04 | 3985.5ms/step | 15.1 samples/s
+Step 32120/50462 | Loss: 2.5640 | LR: 1.50e-04 | 3988.3ms/step | 15.0 samples/s
+Step 32130/50462 | Loss: 2.5198 | LR: 1.50e-04 | 4010.1ms/step | 15.0 samples/s
+Step 32140/50462 | Loss: 2.5624 | LR: 1.50e-04 | 4000.4ms/step | 15.0 samples/s
+Step 32150/50462 | Loss: 2.5512 | LR: 1.50e-04 | 4005.9ms/step | 15.0 samples/s
+Step 32160/50462 | Loss: 2.5277 | LR: 1.49e-04 | 4001.5ms/step | 15.0 samples/s
+Step 32170/50462 | Loss: 2.5752 | LR: 1.49e-04 | 4002.6ms/step | 15.0 samples/s
+Step 32180/50462 | Loss: 2.5022 | LR: 1.49e-04 | 3992.3ms/step | 15.0 samples/s
+Step 32190/50462 | Loss: 2.5572 | LR: 1.49e-04 | 3991.8ms/step | 15.0 samples/s
+Step 32200/50462 | Loss: 2.5409 | LR: 1.49e-04 | 3996.4ms/step | 15.0 samples/s
+Step 32210/50462 | Loss: 2.5555 | LR: 1.49e-04 | 3996.5ms/step | 15.0 samples/s
+Step 32220/50462 | Loss: 2.5448 | LR: 1.49e-04 | 3997.5ms/step | 15.0 samples/s
+Step 32230/50462 | Loss: 2.5546 | LR: 1.48e-04 | 3990.6ms/step | 15.0 samples/s
+Step 32240/50462 | Loss: 2.5533 | LR: 1.48e-04 | 3999.1ms/step | 15.0 samples/s
+Step 32250/50462 | Loss: 2.5644 | LR: 1.48e-04 | 4010.7ms/step | 15.0 samples/s
+Validation at step 32250: Loss = 2.5319, Perplexity = 12.58
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-32250
+Step 32260/50462 | Loss: 2.5284 | LR: 1.48e-04 | 5913.0ms/step | 10.1 samples/s
+Step 32270/50462 | Loss: 2.5472 | LR: 1.48e-04 | 3997.5ms/step | 15.0 samples/s
+Step 32280/50462 | Loss: 2.5353 | LR: 1.48e-04 | 3993.5ms/step | 15.0 samples/s
+Step 32290/50462 | Loss: 2.5247 | LR: 1.48e-04 | 4002.8ms/step | 15.0 samples/s
+Step 32300/50462 | Loss: 2.5432 | LR: 1.47e-04 | 3987.0ms/step | 15.0 samples/s
+Step 32310/50462 | Loss: 2.5293 | LR: 1.47e-04 | 3987.1ms/step | 15.0 samples/s
+Step 32320/50462 | Loss: 2.5346 | LR: 1.47e-04 | 3986.1ms/step | 15.1 samples/s
+Step 32330/50462 | Loss: 2.5362 | LR: 1.47e-04 | 3994.2ms/step | 15.0 samples/s
+Step 32340/50462 | Loss: 2.5666 | LR: 1.47e-04 | 4001.0ms/step | 15.0 samples/s
+Step 32350/50462 | Loss: 2.5409 | LR: 1.47e-04 | 3996.1ms/step | 15.0 samples/s
+Step 32360/50462 | Loss: 2.5228 | LR: 1.47e-04 | 3999.0ms/step | 15.0 samples/s
+Step 32370/50462 | Loss: 2.5348 | LR: 1.46e-04 | 4000.5ms/step | 15.0 samples/s
+Step 32380/50462 | Loss: 2.4890 | LR: 1.46e-04 | 4008.0ms/step | 15.0 samples/s
+Step 32390/50462 | Loss: 2.5361 | LR: 1.46e-04 | 4003.5ms/step | 15.0 samples/s
+Step 32400/50462 | Loss: 2.5536 | LR: 1.46e-04 | 3995.3ms/step | 15.0 samples/s
+Step 32410/50462 | Loss: 2.5567 | LR: 1.46e-04 | 3993.3ms/step | 15.0 samples/s
+Step 32420/50462 | Loss: 2.5547 | LR: 1.46e-04 | 4002.6ms/step | 15.0 samples/s
+Step 32430/50462 | Loss: 2.5301 | LR: 1.45e-04 | 4011.6ms/step | 15.0 samples/s
+Step 32440/50462 | Loss: 2.5352 | LR: 1.45e-04 | 3998.8ms/step | 15.0 samples/s
+Step 32450/50462 | Loss: 2.5188 | LR: 1.45e-04 | 4006.0ms/step | 15.0 samples/s
+Step 32460/50462 | Loss: 2.5557 | LR: 1.45e-04 | 3988.6ms/step | 15.0 samples/s
+Step 32470/50462 | Loss: 2.5433 | LR: 1.45e-04 | 4007.9ms/step | 15.0 samples/s
+Step 32480/50462 | Loss: 2.5508 | LR: 1.45e-04 | 3999.3ms/step | 15.0 samples/s
+Step 32490/50462 | Loss: 2.4919 | LR: 1.45e-04 | 3991.8ms/step | 15.0 samples/s
+Step 32500/50462 | Loss: 2.5285 | LR: 1.44e-04 | 3998.0ms/step | 15.0 samples/s
+Validation at step 32500: Loss = 2.5312, Perplexity = 12.57
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-32500
+Step 32510/50462 | Loss: 2.5552 | LR: 1.44e-04 | 5912.4ms/step | 10.1 samples/s
+Step 32520/50462 | Loss: 2.5924 | LR: 1.44e-04 | 4000.5ms/step | 15.0 samples/s
+Step 32530/50462 | Loss: 2.5570 | LR: 1.44e-04 | 3994.9ms/step | 15.0 samples/s
+Step 32540/50462 | Loss: 2.5616 | LR: 1.44e-04 | 3996.0ms/step | 15.0 samples/s
+Step 32550/50462 | Loss: 2.5670 | LR: 1.44e-04 | 3999.5ms/step | 15.0 samples/s
+Step 32560/50462 | Loss: 2.5546 | LR: 1.44e-04 | 4004.9ms/step | 15.0 samples/s
+Step 32570/50462 | Loss: 2.5680 | LR: 1.43e-04 | 3995.8ms/step | 15.0 samples/s
+Step 32580/50462 | Loss: 2.5565 | LR: 1.43e-04 | 4005.3ms/step | 15.0 samples/s
+Step 32590/50462 | Loss: 2.5416 | LR: 1.43e-04 | 3993.8ms/step | 15.0 samples/s
+Step 32600/50462 | Loss: 2.5362 | LR: 1.43e-04 | 3999.2ms/step | 15.0 samples/s
+Step 32610/50462 | Loss: 2.5364 | LR: 1.43e-04 | 3998.6ms/step | 15.0 samples/s
+Step 32620/50462 | Loss: 2.5368 | LR: 1.43e-04 | 4001.9ms/step | 15.0 samples/s
+Step 32630/50462 | Loss: 2.5533 | LR: 1.43e-04 | 4003.5ms/step | 15.0 samples/s
+Step 32640/50462 | Loss: 2.5462 | LR: 1.42e-04 | 3996.1ms/step | 15.0 samples/s
+Step 32650/50462 | Loss: 2.5922 | LR: 1.42e-04 | 4000.4ms/step | 15.0 samples/s
+Step 32660/50462 | Loss: 2.5491 | LR: 1.42e-04 | 3994.3ms/step | 15.0 samples/s
+Step 32670/50462 | Loss: 2.5630 | LR: 1.42e-04 | 4010.8ms/step | 15.0 samples/s
+Step 32680/50462 | Loss: 2.5407 | LR: 1.42e-04 | 3996.9ms/step | 15.0 samples/s
+Step 32690/50462 | Loss: 2.5797 | LR: 1.42e-04 | 4002.4ms/step | 15.0 samples/s
+Step 32700/50462 | Loss: 2.5387 | LR: 1.42e-04 | 4001.6ms/step | 15.0 samples/s
+Step 32710/50462 | Loss: 2.5425 | LR: 1.41e-04 | 3995.6ms/step | 15.0 samples/s
+Step 32720/50462 | Loss: 2.5390 | LR: 1.41e-04 | 3998.1ms/step | 15.0 samples/s
+Step 32730/50462 | Loss: 2.5711 | LR: 1.41e-04 | 3998.5ms/step | 15.0 samples/s
+Step 32740/50462 | Loss: 2.5186 | LR: 1.41e-04 | 4000.4ms/step | 15.0 samples/s
+Step 32750/50462 | Loss: 2.5370 | LR: 1.41e-04 | 4006.8ms/step | 15.0 samples/s
+Validation at step 32750: Loss = 2.5288, Perplexity = 12.54
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-32750
+Step 32760/50462 | Loss: 2.5310 | LR: 1.41e-04 | 5914.5ms/step | 10.1 samples/s
+Step 32770/50462 | Loss: 2.5497 | LR: 1.41e-04 | 4015.5ms/step | 14.9 samples/s
+Step 32780/50462 | Loss: 2.5638 | LR: 1.41e-04 | 3999.9ms/step | 15.0 samples/s
+Step 32790/50462 | Loss: 2.5550 | LR: 1.40e-04 | 4007.3ms/step | 15.0 samples/s
+Step 32800/50462 | Loss: 2.5459 | LR: 1.40e-04 | 4003.0ms/step | 15.0 samples/s
+Step 32810/50462 | Loss: 2.5576 | LR: 1.40e-04 | 3993.3ms/step | 15.0 samples/s
+Step 32820/50462 | Loss: 2.5105 | LR: 1.40e-04 | 4001.5ms/step | 15.0 samples/s
+Step 32830/50462 | Loss: 2.5560 | LR: 1.40e-04 | 3998.5ms/step | 15.0 samples/s
+Step 32840/50462 | Loss: 2.5490 | LR: 1.40e-04 | 4007.3ms/step | 15.0 samples/s
+Step 32850/50462 | Loss: 2.5578 | LR: 1.40e-04 | 3997.7ms/step | 15.0 samples/s
+Step 32860/50462 | Loss: 2.5323 | LR: 1.39e-04 | 3985.9ms/step | 15.1 samples/s
+Step 32870/50462 | Loss: 2.5270 | LR: 1.39e-04 | 3984.9ms/step | 15.1 samples/s
+Step 32880/50462 | Loss: 2.5383 | LR: 1.39e-04 | 3996.1ms/step | 15.0 samples/s
+Step 32890/50462 | Loss: 2.5557 | LR: 1.39e-04 | 4002.9ms/step | 15.0 samples/s
+Step 32900/50462 | Loss: 2.5480 | LR: 1.39e-04 | 4004.4ms/step | 15.0 samples/s
+Step 32910/50462 | Loss: 2.5626 | LR: 1.39e-04 | 4014.0ms/step | 14.9 samples/s
+Step 32920/50462 | Loss: 2.5148 | LR: 1.39e-04 | 4004.1ms/step | 15.0 samples/s
+Step 32930/50462 | Loss: 2.5294 | LR: 1.38e-04 | 4004.5ms/step | 15.0 samples/s
+Step 32940/50462 | Loss: 2.5351 | LR: 1.38e-04 | 3990.9ms/step | 15.0 samples/s
+Step 32950/50462 | Loss: 2.5453 | LR: 1.38e-04 | 4003.4ms/step | 15.0 samples/s
+Step 32960/50462 | Loss: 2.5551 | LR: 1.38e-04 | 3997.9ms/step | 15.0 samples/s
+Step 32970/50462 | Loss: 2.5432 | LR: 1.38e-04 | 4001.7ms/step | 15.0 samples/s
+Step 32980/50462 | Loss: 2.5420 | LR: 1.38e-04 | 4003.8ms/step | 15.0 samples/s
+Step 32990/50462 | Loss: 2.5328 | LR: 1.38e-04 | 3990.5ms/step | 15.0 samples/s
+Step 33000/50462 | Loss: 2.5538 | LR: 1.37e-04 | 3999.8ms/step | 15.0 samples/s
+Validation at step 33000: Loss = 2.5285, Perplexity = 12.53
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-33000
+Step 33010/50462 | Loss: 2.5227 | LR: 1.37e-04 | 5974.9ms/step | 10.0 samples/s
+Step 33020/50462 | Loss: 2.5038 | LR: 1.37e-04 | 4010.2ms/step | 15.0 samples/s
+Step 33030/50462 | Loss: 2.5191 | LR: 1.37e-04 | 3997.6ms/step | 15.0 samples/s
+Step 33040/50462 | Loss: 2.5391 | LR: 1.37e-04 | 4008.3ms/step | 15.0 samples/s
+Step 33050/50462 | Loss: 2.5638 | LR: 1.37e-04 | 4013.8ms/step | 14.9 samples/s
+Step 33060/50462 | Loss: 2.5283 | LR: 1.37e-04 | 4002.9ms/step | 15.0 samples/s
+Step 33070/50462 | Loss: 2.5504 | LR: 1.36e-04 | 4004.9ms/step | 15.0 samples/s
+Step 33080/50462 | Loss: 2.5422 | LR: 1.36e-04 | 4013.4ms/step | 14.9 samples/s
+Step 33090/50462 | Loss: 2.5337 | LR: 1.36e-04 | 3990.7ms/step | 15.0 samples/s
+Step 33100/50462 | Loss: 2.5267 | LR: 1.36e-04 | 3997.5ms/step | 15.0 samples/s
+Step 33110/50462 | Loss: 2.5416 | LR: 1.36e-04 | 4006.2ms/step | 15.0 samples/s
+Step 33120/50462 | Loss: 2.5168 | LR: 1.36e-04 | 3997.7ms/step | 15.0 samples/s
+Step 33130/50462 | Loss: 2.5398 | LR: 1.36e-04 | 3991.7ms/step | 15.0 samples/s
+Step 33140/50462 | Loss: 2.5436 | LR: 1.35e-04 | 4007.0ms/step | 15.0 samples/s
+Step 33150/50462 | Loss: 2.5607 | LR: 1.35e-04 | 4009.8ms/step | 15.0 samples/s
+Step 33160/50462 | Loss: 2.5532 | LR: 1.35e-04 | 4002.2ms/step | 15.0 samples/s
+Step 33170/50462 | Loss: 2.5251 | LR: 1.35e-04 | 4012.1ms/step | 15.0 samples/s
+Step 33180/50462 | Loss: 2.5563 | LR: 1.35e-04 | 4002.3ms/step | 15.0 samples/s
+Step 33190/50462 | Loss: 2.5745 | LR: 1.35e-04 | 4014.5ms/step | 14.9 samples/s
+Step 33200/50462 | Loss: 2.5310 | LR: 1.35e-04 | 3997.7ms/step | 15.0 samples/s
+Step 33210/50462 | Loss: 2.5302 | LR: 1.34e-04 | 4004.6ms/step | 15.0 samples/s
+Step 33220/50462 | Loss: 2.5650 | LR: 1.34e-04 | 3996.7ms/step | 15.0 samples/s
+Step 33230/50462 | Loss: 2.5651 | LR: 1.34e-04 | 4005.1ms/step | 15.0 samples/s
+Step 33240/50462 | Loss: 2.5637 | LR: 1.34e-04 | 4000.0ms/step | 15.0 samples/s
+Step 33250/50462 | Loss: 2.5328 | LR: 1.34e-04 | 4009.8ms/step | 15.0 samples/s
+Validation at step 33250: Loss = 2.5277, Perplexity = 12.53
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-33250
+Step 33260/50462 | Loss: 2.5196 | LR: 1.34e-04 | 5943.6ms/step | 10.1 samples/s
+Step 33270/50462 | Loss: 2.5541 | LR: 1.34e-04 | 4003.7ms/step | 15.0 samples/s
+Step 33280/50462 | Loss: 2.5416 | LR: 1.33e-04 | 4000.1ms/step | 15.0 samples/s
+Step 33290/50462 | Loss: 2.5499 | LR: 1.33e-04 | 4009.1ms/step | 15.0 samples/s
+Step 33300/50462 | Loss: 2.5548 | LR: 1.33e-04 | 4002.4ms/step | 15.0 samples/s
+Step 33310/50462 | Loss: 2.5393 | LR: 1.33e-04 | 3988.0ms/step | 15.0 samples/s
+Step 33320/50462 | Loss: 2.5324 | LR: 1.33e-04 | 3999.7ms/step | 15.0 samples/s
+Step 33330/50462 | Loss: 2.5408 | LR: 1.33e-04 | 4012.2ms/step | 15.0 samples/s
+Step 33340/50462 | Loss: 2.5421 | LR: 1.33e-04 | 4005.8ms/step | 15.0 samples/s
+Step 33350/50462 | Loss: 2.5296 | LR: 1.32e-04 | 4003.1ms/step | 15.0 samples/s
+Step 33360/50462 | Loss: 2.5457 | LR: 1.32e-04 | 3992.2ms/step | 15.0 samples/s
+Step 33370/50462 | Loss: 2.5395 | LR: 1.32e-04 | 4007.9ms/step | 15.0 samples/s
+Step 33380/50462 | Loss: 2.5402 | LR: 1.32e-04 | 3993.7ms/step | 15.0 samples/s
+Step 33390/50462 | Loss: 2.5283 | LR: 1.32e-04 | 3997.9ms/step | 15.0 samples/s
+Step 33400/50462 | Loss: 2.5477 | LR: 1.32e-04 | 4001.3ms/step | 15.0 samples/s
+Step 33410/50462 | Loss: 2.5464 | LR: 1.32e-04 | 3990.9ms/step | 15.0 samples/s
+Step 33420/50462 | Loss: 2.5422 | LR: 1.32e-04 | 3992.8ms/step | 15.0 samples/s
+Step 33430/50462 | Loss: 2.5738 | LR: 1.31e-04 | 4004.7ms/step | 15.0 samples/s
+Step 33440/50462 | Loss: 2.5359 | LR: 1.31e-04 | 3997.8ms/step | 15.0 samples/s
+Step 33450/50462 | Loss: 2.5318 | LR: 1.31e-04 | 4002.0ms/step | 15.0 samples/s
+Step 33460/50462 | Loss: 2.5651 | LR: 1.31e-04 | 3989.9ms/step | 15.0 samples/s
+Step 33470/50462 | Loss: 2.5774 | LR: 1.31e-04 | 3993.9ms/step | 15.0 samples/s
+Step 33480/50462 | Loss: 2.5308 | LR: 1.31e-04 | 4001.3ms/step | 15.0 samples/s
+Step 33490/50462 | Loss: 2.5301 | LR: 1.31e-04 | 3996.6ms/step | 15.0 samples/s
+Step 33500/50462 | Loss: 2.5022 | LR: 1.30e-04 | 3989.7ms/step | 15.0 samples/s
+Validation at step 33500: Loss = 2.5274, Perplexity = 12.52
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-33500
+Step 33510/50462 | Loss: 2.5683 | LR: 1.30e-04 | 5891.9ms/step | 10.2 samples/s
+Step 33520/50462 | Loss: 2.5509 | LR: 1.30e-04 | 4004.6ms/step | 15.0 samples/s
+Step 33530/50462 | Loss: 2.5228 | LR: 1.30e-04 | 3995.4ms/step | 15.0 samples/s
+Step 33540/50462 | Loss: 2.5472 | LR: 1.30e-04 | 4001.6ms/step | 15.0 samples/s
+Step 33550/50462 | Loss: 2.5511 | LR: 1.30e-04 | 3996.5ms/step | 15.0 samples/s
+Step 33560/50462 | Loss: 2.5294 | LR: 1.30e-04 | 3996.8ms/step | 15.0 samples/s
+Step 33570/50462 | Loss: 2.5269 | LR: 1.29e-04 | 4005.3ms/step | 15.0 samples/s
+Step 33580/50462 | Loss: 2.5751 | LR: 1.29e-04 | 4016.4ms/step | 14.9 samples/s
+Step 33590/50462 | Loss: 2.5351 | LR: 1.29e-04 | 4005.0ms/step | 15.0 samples/s
+Step 33600/50462 | Loss: 2.5408 | LR: 1.29e-04 | 3999.1ms/step | 15.0 samples/s
+Step 33610/50462 | Loss: 2.5099 | LR: 1.29e-04 | 4000.8ms/step | 15.0 samples/s
+Step 33620/50462 | Loss: 2.5174 | LR: 1.29e-04 | 3995.1ms/step | 15.0 samples/s
+Step 33630/50462 | Loss: 2.5299 | LR: 1.29e-04 | 3990.9ms/step | 15.0 samples/s
+Step 33640/50462 | Loss: 2.5530 | LR: 1.28e-04 | 4005.6ms/step | 15.0 samples/s
+Step 33650/50462 | Loss: 2.5466 | LR: 1.28e-04 | 3999.2ms/step | 15.0 samples/s
+Step 33660/50462 | Loss: 2.5458 | LR: 1.28e-04 | 4008.9ms/step | 15.0 samples/s
+Step 33670/50462 | Loss: 2.5438 | LR: 1.28e-04 | 3992.8ms/step | 15.0 samples/s
+Step 33680/50462 | Loss: 2.4912 | LR: 1.28e-04 | 3995.2ms/step | 15.0 samples/s
+Step 33690/50462 | Loss: 2.5471 | LR: 1.28e-04 | 3986.9ms/step | 15.0 samples/s
+Step 33700/50462 | Loss: 2.5055 | LR: 1.28e-04 | 3994.1ms/step | 15.0 samples/s
+Step 33710/50462 | Loss: 2.5489 | LR: 1.27e-04 | 3995.0ms/step | 15.0 samples/s
+Step 33720/50462 | Loss: 2.5623 | LR: 1.27e-04 | 3993.1ms/step | 15.0 samples/s
+Step 33730/50462 | Loss: 2.5244 | LR: 1.27e-04 | 3993.8ms/step | 15.0 samples/s
+Step 33740/50462 | Loss: 2.5425 | LR: 1.27e-04 | 3992.9ms/step | 15.0 samples/s
+Step 33750/50462 | Loss: 2.5329 | LR: 1.27e-04 | 3992.5ms/step | 15.0 samples/s
+Validation at step 33750: Loss = 2.5255, Perplexity = 12.50
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-33750
+Step 33760/50462 | Loss: 2.5262 | LR: 1.27e-04 | 5897.0ms/step | 10.2 samples/s
+Step 33770/50462 | Loss: 2.5317 | LR: 1.27e-04 | 3993.7ms/step | 15.0 samples/s
+Step 33780/50462 | Loss: 2.5610 | LR: 1.27e-04 | 3995.9ms/step | 15.0 samples/s
+Step 33790/50462 | Loss: 2.5851 | LR: 1.26e-04 | 3982.2ms/step | 15.1 samples/s
+Step 33800/50462 | Loss: 2.5204 | LR: 1.26e-04 | 3995.7ms/step | 15.0 samples/s
+Step 33810/50462 | Loss: 2.5283 | LR: 1.26e-04 | 3992.2ms/step | 15.0 samples/s
+Step 33820/50462 | Loss: 2.5168 | LR: 1.26e-04 | 3986.6ms/step | 15.1 samples/s
+Step 33830/50462 | Loss: 2.5435 | LR: 1.26e-04 | 3998.1ms/step | 15.0 samples/s
+Step 33840/50462 | Loss: 2.5605 | LR: 1.26e-04 | 3999.6ms/step | 15.0 samples/s
+Step 33850/50462 | Loss: 2.5547 | LR: 1.26e-04 | 3994.2ms/step | 15.0 samples/s
+Step 33860/50462 | Loss: 2.5653 | LR: 1.25e-04 | 3998.4ms/step | 15.0 samples/s
+Step 33870/50462 | Loss: 2.5200 | LR: 1.25e-04 | 3997.5ms/step | 15.0 samples/s
+Step 33880/50462 | Loss: 2.5544 | LR: 1.25e-04 | 4000.9ms/step | 15.0 samples/s
+Step 33890/50462 | Loss: 2.5740 | LR: 1.25e-04 | 3994.1ms/step | 15.0 samples/s
+Step 33900/50462 | Loss: 2.5620 | LR: 1.25e-04 | 4001.0ms/step | 15.0 samples/s
+Step 33910/50462 | Loss: 2.5652 | LR: 1.25e-04 | 3985.8ms/step | 15.1 samples/s
+Step 33920/50462 | Loss: 2.5058 | LR: 1.25e-04 | 3985.4ms/step | 15.1 samples/s
+Step 33930/50462 | Loss: 2.5710 | LR: 1.24e-04 | 4000.5ms/step | 15.0 samples/s
+Step 33940/50462 | Loss: 2.5517 | LR: 1.24e-04 | 4001.6ms/step | 15.0 samples/s
+Step 33950/50462 | Loss: 2.5431 | LR: 1.24e-04 | 3991.1ms/step | 15.0 samples/s
+Step 33960/50462 | Loss: 2.5400 | LR: 1.24e-04 | 3992.1ms/step | 15.0 samples/s
+Step 33970/50462 | Loss: 2.5317 | LR: 1.24e-04 | 4001.0ms/step | 15.0 samples/s
+Step 33980/50462 | Loss: 2.5551 | LR: 1.24e-04 | 3993.5ms/step | 15.0 samples/s
+Step 33990/50462 | Loss: 2.5302 | LR: 1.24e-04 | 3996.2ms/step | 15.0 samples/s
+Step 34000/50462 | Loss: 2.5320 | LR: 1.24e-04 | 3995.4ms/step | 15.0 samples/s
+Validation at step 34000: Loss = 2.5257, Perplexity = 12.50
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34000
+Step 34010/50462 | Loss: 2.5450 | LR: 1.23e-04 | 5884.7ms/step | 10.2 samples/s
+Step 34020/50462 | Loss: 2.5156 | LR: 1.23e-04 | 3996.6ms/step | 15.0 samples/s
+Step 34030/50462 | Loss: 2.5530 | LR: 1.23e-04 | 3999.3ms/step | 15.0 samples/s
+Step 34040/50462 | Loss: 2.5551 | LR: 1.23e-04 | 3989.2ms/step | 15.0 samples/s
+Step 34050/50462 | Loss: 2.5222 | LR: 1.23e-04 | 3988.7ms/step | 15.0 samples/s
+Step 34060/50462 | Loss: 2.5546 | LR: 1.23e-04 | 4002.3ms/step | 15.0 samples/s
+Step 34070/50462 | Loss: 2.5347 | LR: 1.23e-04 | 3997.9ms/step | 15.0 samples/s
+Step 34080/50462 | Loss: 2.5056 | LR: 1.22e-04 | 3989.8ms/step | 15.0 samples/s
+Step 34090/50462 | Loss: 2.5670 | LR: 1.22e-04 | 3978.4ms/step | 15.1 samples/s
+Step 34100/50462 | Loss: 2.5446 | LR: 1.22e-04 | 3987.9ms/step | 15.0 samples/s
+Step 34110/50462 | Loss: 2.5539 | LR: 1.22e-04 | 3994.8ms/step | 15.0 samples/s
+Step 34120/50462 | Loss: 2.5154 | LR: 1.22e-04 | 3993.1ms/step | 15.0 samples/s
+Step 34130/50462 | Loss: 2.5160 | LR: 1.22e-04 | 4006.1ms/step | 15.0 samples/s
+Step 34140/50462 | Loss: 2.5086 | LR: 1.22e-04 | 3990.4ms/step | 15.0 samples/s
+Step 34150/50462 | Loss: 2.5210 | LR: 1.21e-04 | 3975.4ms/step | 15.1 samples/s
+Step 34160/50462 | Loss: 2.5732 | LR: 1.21e-04 | 4000.7ms/step | 15.0 samples/s
+Step 34170/50462 | Loss: 2.5560 | LR: 1.21e-04 | 4011.2ms/step | 15.0 samples/s
+Step 34180/50462 | Loss: 2.5096 | LR: 1.21e-04 | 4001.5ms/step | 15.0 samples/s
+Step 34190/50462 | Loss: 2.4965 | LR: 1.21e-04 | 3995.3ms/step | 15.0 samples/s
+Step 34200/50462 | Loss: 2.5670 | LR: 1.21e-04 | 3998.6ms/step | 15.0 samples/s
+Step 34210/50462 | Loss: 2.5240 | LR: 1.21e-04 | 3995.4ms/step | 15.0 samples/s
+Step 34220/50462 | Loss: 2.5623 | LR: 1.21e-04 | 3993.9ms/step | 15.0 samples/s
+Step 34230/50462 | Loss: 2.5258 | LR: 1.20e-04 | 4000.1ms/step | 15.0 samples/s
+Step 34240/50462 | Loss: 2.5736 | LR: 1.20e-04 | 3986.8ms/step | 15.0 samples/s
+Step 34250/50462 | Loss: 2.5252 | LR: 1.20e-04 | 3993.0ms/step | 15.0 samples/s
+Validation at step 34250: Loss = 2.5244, Perplexity = 12.48
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34250
+Step 34260/50462 | Loss: 2.5287 | LR: 1.20e-04 | 5872.7ms/step | 10.2 samples/s
+Step 34270/50462 | Loss: 2.5308 | LR: 1.20e-04 | 3989.2ms/step | 15.0 samples/s
+Step 34280/50462 | Loss: 2.5218 | LR: 1.20e-04 | 3988.6ms/step | 15.0 samples/s
+Step 34290/50462 | Loss: 2.5161 | LR: 1.20e-04 | 3994.7ms/step | 15.0 samples/s
+Step 34300/50462 | Loss: 2.5562 | LR: 1.19e-04 | 3989.2ms/step | 15.0 samples/s
+Step 34310/50462 | Loss: 2.5269 | LR: 1.19e-04 | 3996.9ms/step | 15.0 samples/s
+Step 34320/50462 | Loss: 2.5511 | LR: 1.19e-04 | 3989.2ms/step | 15.0 samples/s
+Step 34330/50462 | Loss: 2.5325 | LR: 1.19e-04 | 3987.2ms/step | 15.0 samples/s
+Step 34340/50462 | Loss: 2.5275 | LR: 1.19e-04 | 3989.2ms/step | 15.0 samples/s
+Step 34350/50462 | Loss: 2.5452 | LR: 1.19e-04 | 3983.9ms/step | 15.1 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu_resume29000.pid

@@ -0,0 +1 @@
+404

epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu_resume34250.log

@@ -0,0 +1,520 @@
+===== RESUME FROM CHECKPOINT-34250 ON 4GPU 2026-08-29T17:44:37+00:00 =====
+W0829 17:44:48.676000 128 site-packages/torch/distributed/run.py:851] 
+W0829 17:44:48.676000 128 site-packages/torch/distributed/run.py:851] *****************************************
+W0829 17:44:48.676000 128 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0829 17:44:48.676000 128 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34250",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: TrueCheckpoint activations: True
+
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34250
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34250...
+Preserving checkpoint LR state for exact resume: current=[0.00012012514190618698], schedule_base=[0.0005]
+Resumed from checkpoint at step 34250 (epoch 0, micro_step 511245)
+Resuming training from step 34250
+Resuming wandb run 8ynn1we6
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 8ynn1we6
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260829_174918-8ynn1we6
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed71 4gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8ynn1we6
+Starting training...
+Validation at step 34250: Loss = 2.5253, Perplexity = 12.50
+wandb: WARNING Tried to log to step 34250 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 34251: sparse_stochastic (configured sparse attention)
+Step 34260/50462 | Loss: 2.5295 | LR: 1.20e-04 | 79690.4ms/step | 0.8 samples/s
+wandb: WARNING Tried to log to step 34260 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34270/50462 | Loss: 2.5307 | LR: 1.20e-04 | 4005.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34270 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34280/50462 | Loss: 2.5218 | LR: 1.20e-04 | 4008.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34280 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34290/50462 | Loss: 2.5158 | LR: 1.20e-04 | 4011.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34290 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34300/50462 | Loss: 2.5569 | LR: 1.19e-04 | 4015.8ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 34300 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34310/50462 | Loss: 2.5269 | LR: 1.19e-04 | 4009.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34310 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34320/50462 | Loss: 2.5510 | LR: 1.19e-04 | 4011.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34320 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34330/50462 | Loss: 2.5320 | LR: 1.19e-04 | 4008.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34330 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34340/50462 | Loss: 2.5267 | LR: 1.19e-04 | 4009.5ms/step | 15.0 samples/s
+Step 34350/50462 | Loss: 2.5445 | LR: 1.19e-04 | 4018.6ms/step | 14.9 samples/s
+Step 34360/50462 | Loss: 2.5204 | LR: 1.19e-04 | 4005.3ms/step | 15.0 samples/s
+Step 34370/50462 | Loss: 2.5200 | LR: 1.19e-04 | 4036.4ms/step | 14.9 samples/s
+Step 34380/50462 | Loss: 2.5552 | LR: 1.18e-04 | 4008.2ms/step | 15.0 samples/s
+Step 34390/50462 | Loss: 2.5581 | LR: 1.18e-04 | 4026.4ms/step | 14.9 samples/s
+Step 34400/50462 | Loss: 2.5278 | LR: 1.18e-04 | 4016.6ms/step | 14.9 samples/s
+Step 34410/50462 | Loss: 2.5462 | LR: 1.18e-04 | 4013.9ms/step | 14.9 samples/s
+Step 34420/50462 | Loss: 2.5239 | LR: 1.18e-04 | 4023.7ms/step | 14.9 samples/s
+Step 34430/50462 | Loss: 2.5254 | LR: 1.18e-04 | 4012.9ms/step | 15.0 samples/s
+Step 34440/50462 | Loss: 2.5449 | LR: 1.18e-04 | 4011.3ms/step | 15.0 samples/s
+Step 34450/50462 | Loss: 2.5153 | LR: 1.17e-04 | 4025.0ms/step | 14.9 samples/s
+Step 34460/50462 | Loss: 2.5579 | LR: 1.17e-04 | 4021.4ms/step | 14.9 samples/s
+Step 34470/50462 | Loss: 2.5449 | LR: 1.17e-04 | 4022.0ms/step | 14.9 samples/s
+Step 34480/50462 | Loss: 2.5341 | LR: 1.17e-04 | 4025.6ms/step | 14.9 samples/s
+Step 34490/50462 | Loss: 2.5312 | LR: 1.17e-04 | 4019.8ms/step | 14.9 samples/s
+Step 34500/50462 | Loss: 2.5539 | LR: 1.17e-04 | 4017.8ms/step | 14.9 samples/s
+Validation at step 34500: Loss = 2.5244, Perplexity = 12.48
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34500
+Step 34510/50462 | Loss: 2.5325 | LR: 1.17e-04 | 5973.8ms/step | 10.0 samples/s
+Step 34520/50462 | Loss: 2.5269 | LR: 1.16e-04 | 4023.4ms/step | 14.9 samples/s
+Step 34530/50462 | Loss: 2.5297 | LR: 1.16e-04 | 4017.7ms/step | 14.9 samples/s
+Step 34540/50462 | Loss: 2.5602 | LR: 1.16e-04 | 4029.1ms/step | 14.9 samples/s
+Step 34550/50462 | Loss: 2.5648 | LR: 1.16e-04 | 4024.2ms/step | 14.9 samples/s
+Step 34560/50462 | Loss: 2.5035 | LR: 1.16e-04 | 4014.5ms/step | 14.9 samples/s
+Step 34570/50462 | Loss: 2.5666 | LR: 1.16e-04 | 4020.8ms/step | 14.9 samples/s
+Step 34580/50462 | Loss: 2.5642 | LR: 1.16e-04 | 4015.5ms/step | 14.9 samples/s
+Step 34590/50462 | Loss: 2.5220 | LR: 1.16e-04 | 4019.1ms/step | 14.9 samples/s
+Step 34600/50462 | Loss: 2.5484 | LR: 1.15e-04 | 4019.9ms/step | 14.9 samples/s
+Step 34610/50462 | Loss: 2.5392 | LR: 1.15e-04 | 4014.7ms/step | 14.9 samples/s
+Step 34620/50462 | Loss: 2.5612 | LR: 1.15e-04 | 4018.6ms/step | 14.9 samples/s
+Step 34630/50462 | Loss: 2.5521 | LR: 1.15e-04 | 4016.2ms/step | 14.9 samples/s
+Step 34640/50462 | Loss: 2.5098 | LR: 1.15e-04 | 4003.9ms/step | 15.0 samples/s
+Step 34650/50462 | Loss: 2.5224 | LR: 1.15e-04 | 4010.4ms/step | 15.0 samples/s
+Step 34660/50462 | Loss: 2.5692 | LR: 1.15e-04 | 4015.3ms/step | 14.9 samples/s
+Step 34670/50462 | Loss: 2.5276 | LR: 1.15e-04 | 4020.9ms/step | 14.9 samples/s
+Step 34680/50462 | Loss: 2.5193 | LR: 1.14e-04 | 4010.0ms/step | 15.0 samples/s
+Step 34690/50462 | Loss: 2.5741 | LR: 1.14e-04 | 4007.4ms/step | 15.0 samples/s
+Step 34700/50462 | Loss: 2.5128 | LR: 1.14e-04 | 4032.4ms/step | 14.9 samples/s
+Step 34710/50462 | Loss: 2.5240 | LR: 1.14e-04 | 4024.5ms/step | 14.9 samples/s
+Step 34720/50462 | Loss: 2.5527 | LR: 1.14e-04 | 4009.3ms/step | 15.0 samples/s
+Step 34730/50462 | Loss: 2.5113 | LR: 1.14e-04 | 4017.6ms/step | 14.9 samples/s
+Step 34740/50462 | Loss: 2.5015 | LR: 1.14e-04 | 4002.1ms/step | 15.0 samples/s
+Step 34750/50462 | Loss: 2.5349 | LR: 1.13e-04 | 4008.5ms/step | 15.0 samples/s
+Validation at step 34750: Loss = 2.5230, Perplexity = 12.47
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34750
+Step 34760/50462 | Loss: 2.5588 | LR: 1.13e-04 | 5979.6ms/step | 10.0 samples/s
+Step 34770/50462 | Loss: 2.5299 | LR: 1.13e-04 | 4005.8ms/step | 15.0 samples/s
+Step 34780/50462 | Loss: 2.5303 | LR: 1.13e-04 | 4011.3ms/step | 15.0 samples/s
+Step 34790/50462 | Loss: 2.5336 | LR: 1.13e-04 | 4007.3ms/step | 15.0 samples/s
+Step 34800/50462 | Loss: 2.5455 | LR: 1.13e-04 | 4017.7ms/step | 14.9 samples/s
+Step 34810/50462 | Loss: 2.5243 | LR: 1.13e-04 | 4009.1ms/step | 15.0 samples/s
+Step 34820/50462 | Loss: 2.5563 | LR: 1.13e-04 | 4033.8ms/step | 14.9 samples/s
+Step 34830/50462 | Loss: 2.5607 | LR: 1.12e-04 | 4016.8ms/step | 14.9 samples/s
+Step 34840/50462 | Loss: 2.5298 | LR: 1.12e-04 | 4020.1ms/step | 14.9 samples/s
+Step 34850/50462 | Loss: 2.5112 | LR: 1.12e-04 | 4023.5ms/step | 14.9 samples/s
+Step 34860/50462 | Loss: 2.5664 | LR: 1.12e-04 | 4009.2ms/step | 15.0 samples/s
+Step 34870/50462 | Loss: 2.5437 | LR: 1.12e-04 | 4017.0ms/step | 14.9 samples/s
+Step 34880/50462 | Loss: 2.5579 | LR: 1.12e-04 | 4009.8ms/step | 15.0 samples/s
+Step 34890/50462 | Loss: 2.5077 | LR: 1.12e-04 | 4021.0ms/step | 14.9 samples/s
+Step 34900/50462 | Loss: 2.5464 | LR: 1.11e-04 | 4020.6ms/step | 14.9 samples/s
+Step 34910/50462 | Loss: 2.5150 | LR: 1.11e-04 | 4034.4ms/step | 14.9 samples/s
+Step 34920/50462 | Loss: 2.5349 | LR: 1.11e-04 | 4028.4ms/step | 14.9 samples/s
+Step 34930/50462 | Loss: 2.5198 | LR: 1.11e-04 | 4024.4ms/step | 14.9 samples/s
+Step 34940/50462 | Loss: 2.5309 | LR: 1.11e-04 | 4021.5ms/step | 14.9 samples/s
+Step 34950/50462 | Loss: 2.4777 | LR: 1.11e-04 | 4019.0ms/step | 14.9 samples/s
+Step 34960/50462 | Loss: 2.5168 | LR: 1.11e-04 | 4018.5ms/step | 14.9 samples/s
+Step 34970/50462 | Loss: 2.5148 | LR: 1.11e-04 | 4004.0ms/step | 15.0 samples/s
+Step 34980/50462 | Loss: 2.5357 | LR: 1.10e-04 | 4014.6ms/step | 14.9 samples/s
+Step 34990/50462 | Loss: 2.5604 | LR: 1.10e-04 | 4025.5ms/step | 14.9 samples/s
+Step 35000/50462 | Loss: 2.5520 | LR: 1.10e-04 | 4026.4ms/step | 14.9 samples/s
+Validation at step 35000: Loss = 2.5225, Perplexity = 12.46
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-35000
+Step 35010/50462 | Loss: 2.5116 | LR: 1.10e-04 | 5924.9ms/step | 10.1 samples/s
+Step 35020/50462 | Loss: 2.5388 | LR: 1.10e-04 | 4016.1ms/step | 14.9 samples/s
+Step 35030/50462 | Loss: 2.5464 | LR: 1.10e-04 | 3998.3ms/step | 15.0 samples/s
+Step 35040/50462 | Loss: 2.5211 | LR: 1.10e-04 | 4003.2ms/step | 15.0 samples/s
+Step 35050/50462 | Loss: 2.5373 | LR: 1.09e-04 | 4010.7ms/step | 15.0 samples/s
+Step 35060/50462 | Loss: 2.5419 | LR: 1.09e-04 | 4022.3ms/step | 14.9 samples/s
+Step 35070/50462 | Loss: 2.5417 | LR: 1.09e-04 | 4004.0ms/step | 15.0 samples/s
+Step 35080/50462 | Loss: 2.5076 | LR: 1.09e-04 | 4020.1ms/step | 14.9 samples/s
+Step 35090/50462 | Loss: 2.5604 | LR: 1.09e-04 | 4004.6ms/step | 15.0 samples/s
+Step 35100/50462 | Loss: 2.5408 | LR: 1.09e-04 | 4009.0ms/step | 15.0 samples/s
+Step 35110/50462 | Loss: 2.5242 | LR: 1.09e-04 | 4012.1ms/step | 15.0 samples/s
+Step 35120/50462 | Loss: 2.5540 | LR: 1.09e-04 | 4012.6ms/step | 15.0 samples/s
+Step 35130/50462 | Loss: 2.5148 | LR: 1.08e-04 | 4006.1ms/step | 15.0 samples/s
+Step 35140/50462 | Loss: 2.5324 | LR: 1.08e-04 | 4014.9ms/step | 14.9 samples/s
+Step 35150/50462 | Loss: 2.5577 | LR: 1.08e-04 | 4012.7ms/step | 15.0 samples/s
+Step 35160/50462 | Loss: 2.5730 | LR: 1.08e-04 | 4008.1ms/step | 15.0 samples/s
+Step 35170/50462 | Loss: 2.5207 | LR: 1.08e-04 | 4011.7ms/step | 15.0 samples/s
+Step 35180/50462 | Loss: 2.5442 | LR: 1.08e-04 | 4012.0ms/step | 15.0 samples/s
+Step 35190/50462 | Loss: 2.4948 | LR: 1.08e-04 | 3999.1ms/step | 15.0 samples/s
+Step 35200/50462 | Loss: 2.5474 | LR: 1.08e-04 | 4018.2ms/step | 14.9 samples/s
+Step 35210/50462 | Loss: 2.5174 | LR: 1.07e-04 | 4021.2ms/step | 14.9 samples/s
+Step 35220/50462 | Loss: 2.5127 | LR: 1.07e-04 | 4011.0ms/step | 15.0 samples/s
+Step 35230/50462 | Loss: 2.5335 | LR: 1.07e-04 | 4018.6ms/step | 14.9 samples/s
+Step 35240/50462 | Loss: 2.5686 | LR: 1.07e-04 | 4014.5ms/step | 14.9 samples/s
+Step 35250/50462 | Loss: 2.5247 | LR: 1.07e-04 | 4008.8ms/step | 15.0 samples/s
+Validation at step 35250: Loss = 2.5219, Perplexity = 12.45
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-35250
+Step 35260/50462 | Loss: 2.5233 | LR: 1.07e-04 | 6050.4ms/step | 9.9 samples/s
+Step 35270/50462 | Loss: 2.5183 | LR: 1.07e-04 | 4014.7ms/step | 14.9 samples/s
+Step 35280/50462 | Loss: 2.4972 | LR: 1.07e-04 | 4626.5ms/step | 13.0 samples/s
+Step 35290/50462 | Loss: 2.5205 | LR: 1.06e-04 | 4008.7ms/step | 15.0 samples/s
+Step 35300/50462 | Loss: 2.5727 | LR: 1.06e-04 | 4009.5ms/step | 15.0 samples/s
+Step 35310/50462 | Loss: 2.5299 | LR: 1.06e-04 | 4016.0ms/step | 14.9 samples/s
+Step 35320/50462 | Loss: 2.5099 | LR: 1.06e-04 | 4020.5ms/step | 14.9 samples/s
+Step 35330/50462 | Loss: 2.5235 | LR: 1.06e-04 | 3998.7ms/step | 15.0 samples/s
+Step 35340/50462 | Loss: 2.5591 | LR: 1.06e-04 | 4009.1ms/step | 15.0 samples/s
+Step 35350/50462 | Loss: 2.5479 | LR: 1.06e-04 | 4006.2ms/step | 15.0 samples/s
+Step 35360/50462 | Loss: 2.5261 | LR: 1.05e-04 | 4010.2ms/step | 15.0 samples/s
+Step 35370/50462 | Loss: 2.5126 | LR: 1.05e-04 | 4000.7ms/step | 15.0 samples/s
+Step 35380/50462 | Loss: 2.5480 | LR: 1.05e-04 | 4008.4ms/step | 15.0 samples/s
+Step 35390/50462 | Loss: 2.5339 | LR: 1.05e-04 | 4009.0ms/step | 15.0 samples/s
+Step 35400/50462 | Loss: 2.5355 | LR: 1.05e-04 | 4016.5ms/step | 14.9 samples/s
+Step 35410/50462 | Loss: 2.5563 | LR: 1.05e-04 | 4030.3ms/step | 14.9 samples/s
+Step 35420/50462 | Loss: 2.5303 | LR: 1.05e-04 | 4011.8ms/step | 15.0 samples/s
+Step 35430/50462 | Loss: 2.5179 | LR: 1.05e-04 | 4017.6ms/step | 14.9 samples/s
+Step 35440/50462 | Loss: 2.5297 | LR: 1.04e-04 | 4020.5ms/step | 14.9 samples/s
+Step 35450/50462 | Loss: 2.5393 | LR: 1.04e-04 | 4010.1ms/step | 15.0 samples/s
+Step 35460/50462 | Loss: 2.5552 | LR: 1.04e-04 | 4014.1ms/step | 14.9 samples/s
+Step 35470/50462 | Loss: 2.5197 | LR: 1.04e-04 | 4014.7ms/step | 14.9 samples/s
+Step 35480/50462 | Loss: 2.5634 | LR: 1.04e-04 | 4012.9ms/step | 15.0 samples/s
+Step 35490/50462 | Loss: 2.5298 | LR: 1.04e-04 | 4013.7ms/step | 14.9 samples/s
+Step 35500/50462 | Loss: 2.4988 | LR: 1.04e-04 | 4026.9ms/step | 14.9 samples/s
+Validation at step 35500: Loss = 2.5208, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-35500
+Step 35510/50462 | Loss: 2.5346 | LR: 1.04e-04 | 5962.3ms/step | 10.1 samples/s
+Step 35520/50462 | Loss: 2.5298 | LR: 1.03e-04 | 4011.1ms/step | 15.0 samples/s
+Step 35530/50462 | Loss: 2.4897 | LR: 1.03e-04 | 4026.2ms/step | 14.9 samples/s
+Step 35540/50462 | Loss: 2.5122 | LR: 1.03e-04 | 4018.1ms/step | 14.9 samples/s
+Step 35550/50462 | Loss: 2.5125 | LR: 1.03e-04 | 4016.7ms/step | 14.9 samples/s
+Step 35560/50462 | Loss: 2.5303 | LR: 1.03e-04 | 4020.5ms/step | 14.9 samples/s
+Step 35570/50462 | Loss: 2.5376 | LR: 1.03e-04 | 4012.6ms/step | 15.0 samples/s
+Step 35580/50462 | Loss: 2.5466 | LR: 1.03e-04 | 4013.3ms/step | 15.0 samples/s
+Step 35590/50462 | Loss: 2.5081 | LR: 1.03e-04 | 4009.0ms/step | 15.0 samples/s
+Step 35600/50462 | Loss: 2.5461 | LR: 1.02e-04 | 4015.9ms/step | 14.9 samples/s
+Step 35610/50462 | Loss: 2.5229 | LR: 1.02e-04 | 4013.6ms/step | 14.9 samples/s
+Step 35620/50462 | Loss: 2.5312 | LR: 1.02e-04 | 4001.3ms/step | 15.0 samples/s
+Step 35630/50462 | Loss: 2.5188 | LR: 1.02e-04 | 4015.8ms/step | 14.9 samples/s
+Step 35640/50462 | Loss: 2.5152 | LR: 1.02e-04 | 4020.4ms/step | 14.9 samples/s
+Step 35650/50462 | Loss: 2.5260 | LR: 1.02e-04 | 4012.2ms/step | 15.0 samples/s
+Step 35660/50462 | Loss: 2.5310 | LR: 1.02e-04 | 4009.3ms/step | 15.0 samples/s
+Step 35670/50462 | Loss: 2.5373 | LR: 1.02e-04 | 4014.3ms/step | 14.9 samples/s
+Step 35680/50462 | Loss: 2.5261 | LR: 1.01e-04 | 4029.4ms/step | 14.9 samples/s
+Step 35690/50462 | Loss: 2.5123 | LR: 1.01e-04 | 4026.2ms/step | 14.9 samples/s
+Step 35700/50462 | Loss: 2.5476 | LR: 1.01e-04 | 4028.5ms/step | 14.9 samples/s
+Step 35710/50462 | Loss: 2.4814 | LR: 1.01e-04 | 4019.7ms/step | 14.9 samples/s
+Step 35720/50462 | Loss: 2.5257 | LR: 1.01e-04 | 4000.6ms/step | 15.0 samples/s
+Step 35730/50462 | Loss: 2.5289 | LR: 1.01e-04 | 4005.9ms/step | 15.0 samples/s
+Step 35740/50462 | Loss: 2.5200 | LR: 1.01e-04 | 4012.5ms/step | 15.0 samples/s
+Step 35750/50462 | Loss: 2.5227 | LR: 1.00e-04 | 4005.6ms/step | 15.0 samples/s
+Validation at step 35750: Loss = 2.5209, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-35750
+Step 35760/50462 | Loss: 2.5489 | LR: 1.00e-04 | 6039.7ms/step | 9.9 samples/s
+Step 35770/50462 | Loss: 2.5250 | LR: 1.00e-04 | 4005.4ms/step | 15.0 samples/s
+Step 35780/50462 | Loss: 2.5408 | LR: 1.00e-04 | 3999.1ms/step | 15.0 samples/s
+Step 35790/50462 | Loss: 2.4973 | LR: 1.00e-04 | 4005.4ms/step | 15.0 samples/s
+Step 35800/50462 | Loss: 2.5341 | LR: 9.99e-05 | 4017.2ms/step | 14.9 samples/s
+Step 35810/50462 | Loss: 2.5531 | LR: 9.97e-05 | 4003.5ms/step | 15.0 samples/s
+Step 35820/50462 | Loss: 2.5459 | LR: 9.96e-05 | 4008.9ms/step | 15.0 samples/s
+Step 35830/50462 | Loss: 2.5224 | LR: 9.95e-05 | 4020.4ms/step | 14.9 samples/s
+Step 35840/50462 | Loss: 2.5282 | LR: 9.94e-05 | 4017.6ms/step | 14.9 samples/s
+Step 35850/50462 | Loss: 2.5165 | LR: 9.92e-05 | 4011.7ms/step | 15.0 samples/s
+Step 35860/50462 | Loss: 2.5354 | LR: 9.91e-05 | 4020.8ms/step | 14.9 samples/s
+Step 35870/50462 | Loss: 2.5510 | LR: 9.90e-05 | 4022.1ms/step | 14.9 samples/s
+Step 35880/50462 | Loss: 2.5354 | LR: 9.88e-05 | 4017.0ms/step | 14.9 samples/s
+Step 35890/50462 | Loss: 2.5194 | LR: 9.87e-05 | 4027.3ms/step | 14.9 samples/s
+Step 35900/50462 | Loss: 2.5286 | LR: 9.86e-05 | 4002.3ms/step | 15.0 samples/s
+Step 35910/50462 | Loss: 2.5463 | LR: 9.85e-05 | 4010.4ms/step | 15.0 samples/s
+Step 35920/50462 | Loss: 2.5333 | LR: 9.83e-05 | 4001.0ms/step | 15.0 samples/s
+Step 35930/50462 | Loss: 2.5047 | LR: 9.82e-05 | 4788.9ms/step | 12.5 samples/s
+Step 35940/50462 | Loss: 2.5115 | LR: 9.81e-05 | 4010.6ms/step | 15.0 samples/s
+Step 35950/50462 | Loss: 2.5324 | LR: 9.80e-05 | 4000.4ms/step | 15.0 samples/s
+Step 35960/50462 | Loss: 2.5399 | LR: 9.78e-05 | 4578.2ms/step | 13.1 samples/s
+Step 35970/50462 | Loss: 2.5408 | LR: 9.77e-05 | 4017.4ms/step | 14.9 samples/s
+Step 35980/50462 | Loss: 2.5010 | LR: 9.76e-05 | 4012.7ms/step | 15.0 samples/s
+Step 35990/50462 | Loss: 2.4870 | LR: 9.75e-05 | 4009.5ms/step | 15.0 samples/s
+Step 36000/50462 | Loss: 2.5162 | LR: 9.73e-05 | 4016.4ms/step | 14.9 samples/s
+Validation at step 36000: Loss = 2.5207, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-36000
+Step 36010/50462 | Loss: 2.5543 | LR: 9.72e-05 | 6012.2ms/step | 10.0 samples/s
+Step 36020/50462 | Loss: 2.5155 | LR: 9.71e-05 | 4010.3ms/step | 15.0 samples/s
+Step 36030/50462 | Loss: 2.5246 | LR: 9.70e-05 | 4592.2ms/step | 13.1 samples/s
+Step 36040/50462 | Loss: 2.5354 | LR: 9.68e-05 | 4011.1ms/step | 15.0 samples/s
+Step 36050/50462 | Loss: 2.5228 | LR: 9.67e-05 | 4013.1ms/step | 15.0 samples/s
+Step 36060/50462 | Loss: 2.5228 | LR: 9.66e-05 | 4009.2ms/step | 15.0 samples/s
+Step 36070/50462 | Loss: 2.5632 | LR: 9.65e-05 | 4026.1ms/step | 14.9 samples/s
+Step 36080/50462 | Loss: 2.5276 | LR: 9.63e-05 | 4011.1ms/step | 15.0 samples/s
+Step 36090/50462 | Loss: 2.5277 | LR: 9.62e-05 | 3999.8ms/step | 15.0 samples/s
+Step 36100/50462 | Loss: 2.5209 | LR: 9.61e-05 | 4007.4ms/step | 15.0 samples/s
+Step 36110/50462 | Loss: 2.5169 | LR: 9.60e-05 | 4015.4ms/step | 14.9 samples/s
+Step 36120/50462 | Loss: 2.5280 | LR: 9.58e-05 | 4009.9ms/step | 15.0 samples/s
+Step 36130/50462 | Loss: 2.5401 | LR: 9.57e-05 | 4006.8ms/step | 15.0 samples/s
+Step 36140/50462 | Loss: 2.5506 | LR: 9.56e-05 | 4006.0ms/step | 15.0 samples/s
+Step 36150/50462 | Loss: 2.5169 | LR: 9.55e-05 | 4002.5ms/step | 15.0 samples/s
+Step 36160/50462 | Loss: 2.5164 | LR: 9.53e-05 | 4020.7ms/step | 14.9 samples/s
+Step 36170/50462 | Loss: 2.5375 | LR: 9.52e-05 | 4011.4ms/step | 15.0 samples/s
+Step 36180/50462 | Loss: 2.5221 | LR: 9.51e-05 | 4007.1ms/step | 15.0 samples/s
+Step 36190/50462 | Loss: 2.5787 | LR: 9.50e-05 | 4024.1ms/step | 14.9 samples/s
+Step 36200/50462 | Loss: 2.5372 | LR: 9.49e-05 | 4000.8ms/step | 15.0 samples/s
+Step 36210/50462 | Loss: 2.5342 | LR: 9.47e-05 | 4002.6ms/step | 15.0 samples/s
+Step 36220/50462 | Loss: 2.5146 | LR: 9.46e-05 | 4004.9ms/step | 15.0 samples/s
+Step 36230/50462 | Loss: 2.5496 | LR: 9.45e-05 | 4001.2ms/step | 15.0 samples/s
+Step 36240/50462 | Loss: 2.5357 | LR: 9.44e-05 | 4008.1ms/step | 15.0 samples/s
+Step 36250/50462 | Loss: 2.5324 | LR: 9.42e-05 | 4004.5ms/step | 15.0 samples/s
+Validation at step 36250: Loss = 2.5197, Perplexity = 12.42
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-36250
+Step 36260/50462 | Loss: 2.4829 | LR: 9.41e-05 | 5979.5ms/step | 10.0 samples/s
+Step 36270/50462 | Loss: 2.5536 | LR: 9.40e-05 | 4015.5ms/step | 14.9 samples/s
+Step 36280/50462 | Loss: 2.5448 | LR: 9.39e-05 | 4012.5ms/step | 15.0 samples/s
+Step 36290/50462 | Loss: 2.5206 | LR: 9.37e-05 | 4005.7ms/step | 15.0 samples/s
+Step 36300/50462 | Loss: 2.5289 | LR: 9.36e-05 | 4007.7ms/step | 15.0 samples/s
+Step 36310/50462 | Loss: 2.5285 | LR: 9.35e-05 | 4007.8ms/step | 15.0 samples/s
+Step 36320/50462 | Loss: 2.5442 | LR: 9.34e-05 | 4015.3ms/step | 14.9 samples/s
+Step 36330/50462 | Loss: 2.5320 | LR: 9.32e-05 | 4002.4ms/step | 15.0 samples/s
+Step 36340/50462 | Loss: 2.5322 | LR: 9.31e-05 | 4014.7ms/step | 14.9 samples/s
+Step 36350/50462 | Loss: 2.5404 | LR: 9.30e-05 | 3998.5ms/step | 15.0 samples/s
+Step 36360/50462 | Loss: 2.5360 | LR: 9.29e-05 | 3997.9ms/step | 15.0 samples/s
+Step 36370/50462 | Loss: 2.5276 | LR: 9.28e-05 | 4009.5ms/step | 15.0 samples/s
+Step 36380/50462 | Loss: 2.5442 | LR: 9.26e-05 | 4009.0ms/step | 15.0 samples/s
+Step 36390/50462 | Loss: 2.5160 | LR: 9.25e-05 | 4011.2ms/step | 15.0 samples/s
+Step 36400/50462 | Loss: 2.5252 | LR: 9.24e-05 | 4011.2ms/step | 15.0 samples/s
+Step 36410/50462 | Loss: 2.5149 | LR: 9.23e-05 | 4014.4ms/step | 14.9 samples/s
+Step 36420/50462 | Loss: 2.5040 | LR: 9.21e-05 | 4013.6ms/step | 14.9 samples/s
+Step 36430/50462 | Loss: 2.5291 | LR: 9.20e-05 | 4012.1ms/step | 15.0 samples/s
+Step 36440/50462 | Loss: 2.5311 | LR: 9.19e-05 | 3999.6ms/step | 15.0 samples/s
+Step 36450/50462 | Loss: 2.5197 | LR: 9.18e-05 | 4000.0ms/step | 15.0 samples/s
+Step 36460/50462 | Loss: 2.5115 | LR: 9.17e-05 | 4010.0ms/step | 15.0 samples/s
+Step 36470/50462 | Loss: 2.5109 | LR: 9.15e-05 | 4019.0ms/step | 14.9 samples/s
+Step 36480/50462 | Loss: 2.5090 | LR: 9.14e-05 | 4012.0ms/step | 15.0 samples/s
+Step 36490/50462 | Loss: 2.5172 | LR: 9.13e-05 | 4016.6ms/step | 14.9 samples/s
+Step 36500/50462 | Loss: 2.5063 | LR: 9.12e-05 | 4013.1ms/step | 15.0 samples/s
+Validation at step 36500: Loss = 2.5191, Perplexity = 12.42
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-36500
+Step 36510/50462 | Loss: 2.5201 | LR: 9.10e-05 | 5965.9ms/step | 10.1 samples/s
+Step 36520/50462 | Loss: 2.5520 | LR: 9.09e-05 | 3998.6ms/step | 15.0 samples/s
+Step 36530/50462 | Loss: 2.5210 | LR: 9.08e-05 | 4003.7ms/step | 15.0 samples/s
+Step 36540/50462 | Loss: 2.5408 | LR: 9.07e-05 | 4011.9ms/step | 15.0 samples/s
+Step 36550/50462 | Loss: 2.5400 | LR: 9.06e-05 | 4009.7ms/step | 15.0 samples/s
+Step 36560/50462 | Loss: 2.5334 | LR: 9.04e-05 | 4017.7ms/step | 14.9 samples/s
+Step 36570/50462 | Loss: 2.4692 | LR: 9.03e-05 | 4017.9ms/step | 14.9 samples/s
+Step 36580/50462 | Loss: 2.5159 | LR: 9.02e-05 | 4011.0ms/step | 15.0 samples/s
+Step 36590/50462 | Loss: 2.5443 | LR: 9.01e-05 | 3991.4ms/step | 15.0 samples/s
+Step 36600/50462 | Loss: 2.5868 | LR: 8.99e-05 | 4000.3ms/step | 15.0 samples/s
+Step 36610/50462 | Loss: 2.5105 | LR: 8.98e-05 | 4004.4ms/step | 15.0 samples/s
+Step 36620/50462 | Loss: 2.5493 | LR: 8.97e-05 | 4018.8ms/step | 14.9 samples/s
+Step 36630/50462 | Loss: 2.5100 | LR: 8.96e-05 | 4014.5ms/step | 14.9 samples/s
+Step 36640/50462 | Loss: 2.5401 | LR: 8.95e-05 | 4000.1ms/step | 15.0 samples/s
+Step 36650/50462 | Loss: 2.5508 | LR: 8.93e-05 | 4010.1ms/step | 15.0 samples/s
+Step 36660/50462 | Loss: 2.5201 | LR: 8.92e-05 | 4006.3ms/step | 15.0 samples/s
+Step 36670/50462 | Loss: 2.5645 | LR: 8.91e-05 | 4009.9ms/step | 15.0 samples/s
+Step 36680/50462 | Loss: 2.5091 | LR: 8.90e-05 | 4006.8ms/step | 15.0 samples/s
+Step 36690/50462 | Loss: 2.5254 | LR: 8.89e-05 | 4001.2ms/step | 15.0 samples/s
+Step 36700/50462 | Loss: 2.5203 | LR: 8.87e-05 | 4005.7ms/step | 15.0 samples/s
+Step 36710/50462 | Loss: 2.5155 | LR: 8.86e-05 | 4017.2ms/step | 14.9 samples/s
+Step 36720/50462 | Loss: 2.5342 | LR: 8.85e-05 | 4005.9ms/step | 15.0 samples/s
+Step 36730/50462 | Loss: 2.5213 | LR: 8.84e-05 | 4007.6ms/step | 15.0 samples/s
+Step 36740/50462 | Loss: 2.5368 | LR: 8.83e-05 | 4014.3ms/step | 14.9 samples/s
+Step 36750/50462 | Loss: 2.4980 | LR: 8.81e-05 | 4011.1ms/step | 15.0 samples/s
+Validation at step 36750: Loss = 2.5186, Perplexity = 12.41
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-36750
+Step 36760/50462 | Loss: 2.5350 | LR: 8.80e-05 | 5960.2ms/step | 10.1 samples/s
+Step 36770/50462 | Loss: 2.5130 | LR: 8.79e-05 | 4000.3ms/step | 15.0 samples/s
+Step 36780/50462 | Loss: 2.5424 | LR: 8.78e-05 | 4004.6ms/step | 15.0 samples/s
+Step 36790/50462 | Loss: 2.5619 | LR: 8.77e-05 | 4008.2ms/step | 15.0 samples/s
+Step 36800/50462 | Loss: 2.5399 | LR: 8.75e-05 | 4007.4ms/step | 15.0 samples/s
+Step 36810/50462 | Loss: 2.4872 | LR: 8.74e-05 | 4010.8ms/step | 15.0 samples/s
+Step 36820/50462 | Loss: 2.5151 | LR: 8.73e-05 | 4011.5ms/step | 15.0 samples/s
+Step 36830/50462 | Loss: 2.5531 | LR: 8.72e-05 | 4007.2ms/step | 15.0 samples/s
+Step 36840/50462 | Loss: 2.5234 | LR: 8.71e-05 | 4015.4ms/step | 14.9 samples/s
+Step 36850/50462 | Loss: 2.5282 | LR: 8.69e-05 | 4014.1ms/step | 14.9 samples/s
+Step 36860/50462 | Loss: 2.5357 | LR: 8.68e-05 | 4013.8ms/step | 14.9 samples/s
+Step 36870/50462 | Loss: 2.4795 | LR: 8.67e-05 | 3994.2ms/step | 15.0 samples/s
+Step 36880/50462 | Loss: 2.5289 | LR: 8.66e-05 | 4006.1ms/step | 15.0 samples/s
+Step 36890/50462 | Loss: 2.5420 | LR: 8.65e-05 | 4012.2ms/step | 15.0 samples/s
+Step 36900/50462 | Loss: 2.5646 | LR: 8.63e-05 | 4012.3ms/step | 15.0 samples/s
+Step 36910/50462 | Loss: 2.5077 | LR: 8.62e-05 | 4006.7ms/step | 15.0 samples/s
+Step 36920/50462 | Loss: 2.5406 | LR: 8.61e-05 | 4006.6ms/step | 15.0 samples/s
+Step 36930/50462 | Loss: 2.5513 | LR: 8.60e-05 | 4009.5ms/step | 15.0 samples/s
+Step 36940/50462 | Loss: 2.5333 | LR: 8.59e-05 | 4000.8ms/step | 15.0 samples/s
+Step 36950/50462 | Loss: 2.5341 | LR: 8.57e-05 | 3996.6ms/step | 15.0 samples/s
+Step 36960/50462 | Loss: 2.5054 | LR: 8.56e-05 | 4020.8ms/step | 14.9 samples/s
+Step 36970/50462 | Loss: 2.4908 | LR: 8.55e-05 | 4015.0ms/step | 14.9 samples/s
+Step 36980/50462 | Loss: 2.5400 | LR: 8.54e-05 | 4011.5ms/step | 15.0 samples/s
+Step 36990/50462 | Loss: 2.5302 | LR: 8.53e-05 | 4016.5ms/step | 14.9 samples/s
+Step 37000/50462 | Loss: 2.5417 | LR: 8.51e-05 | 4009.3ms/step | 15.0 samples/s
+Validation at step 37000: Loss = 2.5185, Perplexity = 12.41
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37000
+Step 37010/50462 | Loss: 2.5209 | LR: 8.50e-05 | 6067.3ms/step | 9.9 samples/s
+Step 37020/50462 | Loss: 2.4995 | LR: 8.49e-05 | 4021.3ms/step | 14.9 samples/s
+Step 37030/50462 | Loss: 2.5161 | LR: 8.48e-05 | 4025.2ms/step | 14.9 samples/s
+Step 37040/50462 | Loss: 2.5463 | LR: 8.47e-05 | 4000.8ms/step | 15.0 samples/s
+Step 37050/50462 | Loss: 2.5419 | LR: 8.46e-05 | 4011.3ms/step | 15.0 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu_resume34250.pid

@@ -0,0 +1 @@
+128

epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu_resume37000.log

@@ -0,0 +1,750 @@
+===== RESUME FROM CHECKPOINT-37000 ON 4GPU 2026-08-30T00:25:17+00:00 =====
+W0830 00:25:28.036000 143 site-packages/torch/distributed/run.py:851] 
+W0830 00:25:28.036000 143 site-packages/torch/distributed/run.py:851] *****************************************
+W0830 00:25:28.036000 143 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0830 00:25:28.036000 143 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37000
+Checkpoint activations: TrueCheckpoint activations: TrueCheckpoint activations: True
+
+
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37000...
+Preserving checkpoint LR state for exact resume: current=[8.514374998761301e-05], schedule_base=[0.0005]
+Resumed from checkpoint at step 37000 (epoch 0, micro_step 552495)
+Resuming training from step 37000
+Resuming wandb run 8ynn1we6
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260830_002911-8ynn1we6
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed71 4gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8ynn1we6
+Starting training...
+Validation at step 37000: Loss = 2.5187, Perplexity = 12.41
+wandb: WARNING Tried to log to step 37000 that is less than the current step 37041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 37001: sparse_stochastic (configured sparse attention)
+Step 37010/50462 | Loss: 2.5209 | LR: 8.50e-05 | 88560.5ms/step | 0.7 samples/s
+wandb: WARNING Tried to log to step 37010 that is less than the current step 37041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 37020/50462 | Loss: 2.4994 | LR: 8.49e-05 | 4008.4ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 37020 that is less than the current step 37041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 37030/50462 | Loss: 2.5164 | LR: 8.48e-05 | 4012.4ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 37030 that is less than the current step 37041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 37040/50462 | Loss: 2.5460 | LR: 8.47e-05 | 4015.4ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 37040 that is less than the current step 37041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 37050/50462 | Loss: 2.5402 | LR: 8.46e-05 | 4023.2ms/step | 14.9 samples/s
+Step 37060/50462 | Loss: 2.5259 | LR: 8.44e-05 | 4021.0ms/step | 14.9 samples/s
+Step 37070/50462 | Loss: 2.5400 | LR: 8.43e-05 | 4019.5ms/step | 14.9 samples/s
+Step 37080/50462 | Loss: 2.5154 | LR: 8.42e-05 | 4014.8ms/step | 14.9 samples/s
+Step 37090/50462 | Loss: 2.4991 | LR: 8.41e-05 | 4015.8ms/step | 14.9 samples/s
+Step 37100/50462 | Loss: 2.5339 | LR: 8.40e-05 | 4002.4ms/step | 15.0 samples/s
+Step 37110/50462 | Loss: 2.5480 | LR: 8.38e-05 | 4007.3ms/step | 15.0 samples/s
+Step 37120/50462 | Loss: 2.5428 | LR: 8.37e-05 | 4016.9ms/step | 14.9 samples/s
+Step 37130/50462 | Loss: 2.4922 | LR: 8.36e-05 | 4015.0ms/step | 14.9 samples/s
+Step 37140/50462 | Loss: 2.5174 | LR: 8.35e-05 | 4015.5ms/step | 14.9 samples/s
+Step 37150/50462 | Loss: 2.5209 | LR: 8.34e-05 | 4024.3ms/step | 14.9 samples/s
+Step 37160/50462 | Loss: 2.5147 | LR: 8.33e-05 | 4010.5ms/step | 15.0 samples/s
+Step 37170/50462 | Loss: 2.5313 | LR: 8.31e-05 | 4013.3ms/step | 15.0 samples/s
+Step 37180/50462 | Loss: 2.5433 | LR: 8.30e-05 | 4005.7ms/step | 15.0 samples/s
+Step 37190/50462 | Loss: 2.5234 | LR: 8.29e-05 | 4007.3ms/step | 15.0 samples/s
+Step 37200/50462 | Loss: 2.5430 | LR: 8.28e-05 | 4016.2ms/step | 14.9 samples/s
+Step 37210/50462 | Loss: 2.5253 | LR: 8.27e-05 | 4016.7ms/step | 14.9 samples/s
+Step 37220/50462 | Loss: 2.5080 | LR: 8.25e-05 | 4004.9ms/step | 15.0 samples/s
+Step 37230/50462 | Loss: 2.5245 | LR: 8.24e-05 | 4015.3ms/step | 14.9 samples/s
+Step 37240/50462 | Loss: 2.5479 | LR: 8.23e-05 | 4022.2ms/step | 14.9 samples/s
+Step 37250/50462 | Loss: 2.5347 | LR: 8.22e-05 | 3999.2ms/step | 15.0 samples/s
+Validation at step 37250: Loss = 2.5186, Perplexity = 12.41
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37250
+Step 37260/50462 | Loss: 2.5185 | LR: 8.21e-05 | 5974.0ms/step | 10.0 samples/s
+Step 37270/50462 | Loss: 2.5397 | LR: 8.20e-05 | 4007.0ms/step | 15.0 samples/s
+Step 37280/50462 | Loss: 2.5277 | LR: 8.18e-05 | 4014.6ms/step | 14.9 samples/s
+Step 37290/50462 | Loss: 2.5529 | LR: 8.17e-05 | 4023.6ms/step | 14.9 samples/s
+Step 37300/50462 | Loss: 2.5347 | LR: 8.16e-05 | 4005.6ms/step | 15.0 samples/s
+Step 37310/50462 | Loss: 2.5321 | LR: 8.15e-05 | 4010.9ms/step | 15.0 samples/s
+Step 37320/50462 | Loss: 2.5161 | LR: 8.14e-05 | 4022.1ms/step | 14.9 samples/s
+Step 37330/50462 | Loss: 2.5420 | LR: 8.13e-05 | 4015.8ms/step | 14.9 samples/s
+Step 37340/50462 | Loss: 2.5075 | LR: 8.11e-05 | 4007.2ms/step | 15.0 samples/s
+Step 37350/50462 | Loss: 2.5490 | LR: 8.10e-05 | 4018.0ms/step | 14.9 samples/s
+Step 37360/50462 | Loss: 2.4989 | LR: 8.09e-05 | 4015.8ms/step | 14.9 samples/s
+Step 37370/50462 | Loss: 2.5496 | LR: 8.08e-05 | 4011.7ms/step | 15.0 samples/s
+Step 37380/50462 | Loss: 2.5320 | LR: 8.07e-05 | 4022.4ms/step | 14.9 samples/s
+Step 37390/50462 | Loss: 2.5320 | LR: 8.06e-05 | 4013.0ms/step | 15.0 samples/s
+Step 37400/50462 | Loss: 2.5390 | LR: 8.04e-05 | 3999.2ms/step | 15.0 samples/s
+Step 37410/50462 | Loss: 2.5283 | LR: 8.03e-05 | 4006.2ms/step | 15.0 samples/s
+Step 37420/50462 | Loss: 2.5621 | LR: 8.02e-05 | 4008.7ms/step | 15.0 samples/s
+Step 37430/50462 | Loss: 2.4928 | LR: 8.01e-05 | 4023.6ms/step | 14.9 samples/s
+Step 37440/50462 | Loss: 2.4970 | LR: 8.00e-05 | 4015.3ms/step | 14.9 samples/s
+Step 37450/50462 | Loss: 2.4874 | LR: 7.99e-05 | 4012.5ms/step | 15.0 samples/s
+Step 37460/50462 | Loss: 2.5284 | LR: 7.98e-05 | 4003.2ms/step | 15.0 samples/s
+Step 37470/50462 | Loss: 2.5782 | LR: 7.96e-05 | 4014.5ms/step | 14.9 samples/s
+Step 37480/50462 | Loss: 2.5119 | LR: 7.95e-05 | 4004.4ms/step | 15.0 samples/s
+Step 37490/50462 | Loss: 2.5167 | LR: 7.94e-05 | 4009.0ms/step | 15.0 samples/s
+Step 37500/50462 | Loss: 2.5421 | LR: 7.93e-05 | 4022.6ms/step | 14.9 samples/s
+Validation at step 37500: Loss = 2.5174, Perplexity = 12.40
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37500
+Step 37510/50462 | Loss: 2.5200 | LR: 7.92e-05 | 5983.4ms/step | 10.0 samples/s
+Step 37520/50462 | Loss: 2.5330 | LR: 7.91e-05 | 4009.3ms/step | 15.0 samples/s
+Step 37530/50462 | Loss: 2.5239 | LR: 7.89e-05 | 4021.3ms/step | 14.9 samples/s
+Step 37540/50462 | Loss: 2.5474 | LR: 7.88e-05 | 4009.3ms/step | 15.0 samples/s
+Step 37550/50462 | Loss: 2.5640 | LR: 7.87e-05 | 4002.1ms/step | 15.0 samples/s
+Step 37560/50462 | Loss: 2.5490 | LR: 7.86e-05 | 4005.6ms/step | 15.0 samples/s
+Step 37570/50462 | Loss: 2.5234 | LR: 7.85e-05 | 4020.8ms/step | 14.9 samples/s
+Step 37580/50462 | Loss: 2.4794 | LR: 7.84e-05 | 4009.1ms/step | 15.0 samples/s
+Step 37590/50462 | Loss: 2.5330 | LR: 7.83e-05 | 4008.6ms/step | 15.0 samples/s
+Step 37600/50462 | Loss: 2.5372 | LR: 7.81e-05 | 4014.1ms/step | 14.9 samples/s
+Step 37610/50462 | Loss: 2.5169 | LR: 7.80e-05 | 3998.5ms/step | 15.0 samples/s
+Step 37620/50462 | Loss: 2.4926 | LR: 7.79e-05 | 4007.2ms/step | 15.0 samples/s
+Step 37630/50462 | Loss: 2.5177 | LR: 7.78e-05 | 4009.7ms/step | 15.0 samples/s
+Step 37640/50462 | Loss: 2.5319 | LR: 7.77e-05 | 4007.7ms/step | 15.0 samples/s
+Step 37650/50462 | Loss: 2.5355 | LR: 7.76e-05 | 4009.1ms/step | 15.0 samples/s
+Step 37660/50462 | Loss: 2.5330 | LR: 7.75e-05 | 4001.7ms/step | 15.0 samples/s
+Step 37670/50462 | Loss: 2.5413 | LR: 7.73e-05 | 4009.0ms/step | 15.0 samples/s
+Step 37680/50462 | Loss: 2.5254 | LR: 7.72e-05 | 4007.4ms/step | 15.0 samples/s
+Step 37690/50462 | Loss: 2.5601 | LR: 7.71e-05 | 4011.0ms/step | 15.0 samples/s
+Step 37700/50462 | Loss: 2.5274 | LR: 7.70e-05 | 4015.3ms/step | 14.9 samples/s
+Step 37710/50462 | Loss: 2.5339 | LR: 7.69e-05 | 4017.6ms/step | 14.9 samples/s
+Step 37720/50462 | Loss: 2.5433 | LR: 7.68e-05 | 4002.3ms/step | 15.0 samples/s
+Step 37730/50462 | Loss: 2.5434 | LR: 7.67e-05 | 4006.3ms/step | 15.0 samples/s
+Step 37740/50462 | Loss: 2.5303 | LR: 7.65e-05 | 4011.6ms/step | 15.0 samples/s
+Step 37750/50462 | Loss: 2.5163 | LR: 7.64e-05 | 4023.7ms/step | 14.9 samples/s
+Validation at step 37750: Loss = 2.5170, Perplexity = 12.39
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37750
+Step 37760/50462 | Loss: 2.5241 | LR: 7.63e-05 | 5948.0ms/step | 10.1 samples/s
+Step 37770/50462 | Loss: 2.5658 | LR: 7.62e-05 | 4011.1ms/step | 15.0 samples/s
+Step 37780/50462 | Loss: 2.5006 | LR: 7.61e-05 | 4009.0ms/step | 15.0 samples/s
+Step 37790/50462 | Loss: 2.5109 | LR: 7.60e-05 | 4019.7ms/step | 14.9 samples/s
+Step 37800/50462 | Loss: 2.5383 | LR: 7.59e-05 | 4020.7ms/step | 14.9 samples/s
+Step 37810/50462 | Loss: 2.5317 | LR: 7.57e-05 | 4015.3ms/step | 14.9 samples/s
+Step 37820/50462 | Loss: 2.5175 | LR: 7.56e-05 | 4009.0ms/step | 15.0 samples/s
+Step 37830/50462 | Loss: 2.5493 | LR: 7.55e-05 | 4009.4ms/step | 15.0 samples/s
+Step 37840/50462 | Loss: 2.5404 | LR: 7.54e-05 | 4002.2ms/step | 15.0 samples/s
+Step 37850/50462 | Loss: 2.5368 | LR: 7.53e-05 | 4006.6ms/step | 15.0 samples/s
+Step 37860/50462 | Loss: 2.5151 | LR: 7.52e-05 | 4027.0ms/step | 14.9 samples/s
+Step 37870/50462 | Loss: 2.5120 | LR: 7.51e-05 | 4003.2ms/step | 15.0 samples/s
+Step 37880/50462 | Loss: 2.4851 | LR: 7.50e-05 | 4008.4ms/step | 15.0 samples/s
+Step 37890/50462 | Loss: 2.5321 | LR: 7.48e-05 | 4016.0ms/step | 14.9 samples/s
+Step 37900/50462 | Loss: 2.5021 | LR: 7.47e-05 | 4010.1ms/step | 15.0 samples/s
+Step 37910/50462 | Loss: 2.5287 | LR: 7.46e-05 | 4003.1ms/step | 15.0 samples/s
+Step 37920/50462 | Loss: 2.5196 | LR: 7.45e-05 | 4011.0ms/step | 15.0 samples/s
+Step 37930/50462 | Loss: 2.5343 | LR: 7.44e-05 | 4013.4ms/step | 14.9 samples/s
+Step 37940/50462 | Loss: 2.5294 | LR: 7.43e-05 | 4004.5ms/step | 15.0 samples/s
+Step 37950/50462 | Loss: 2.4690 | LR: 7.42e-05 | 4014.6ms/step | 14.9 samples/s
+Step 37960/50462 | Loss: 2.5227 | LR: 7.41e-05 | 4009.8ms/step | 15.0 samples/s
+Step 37970/50462 | Loss: 2.5276 | LR: 7.39e-05 | 4017.4ms/step | 14.9 samples/s
+Step 37980/50462 | Loss: 2.5424 | LR: 7.38e-05 | 4013.4ms/step | 14.9 samples/s
+Step 37990/50462 | Loss: 2.5276 | LR: 7.37e-05 | 3998.6ms/step | 15.0 samples/s
+Step 38000/50462 | Loss: 2.5303 | LR: 7.36e-05 | 4003.7ms/step | 15.0 samples/s
+Validation at step 38000: Loss = 2.5168, Perplexity = 12.39
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-38000
+Step 38010/50462 | Loss: 2.5436 | LR: 7.35e-05 | 5989.7ms/step | 10.0 samples/s
+Step 38020/50462 | Loss: 2.5508 | LR: 7.34e-05 | 4003.7ms/step | 15.0 samples/s
+Step 38030/50462 | Loss: 2.5237 | LR: 7.33e-05 | 4006.4ms/step | 15.0 samples/s
+Step 38040/50462 | Loss: 2.5327 | LR: 7.32e-05 | 4012.7ms/step | 15.0 samples/s
+Step 38050/50462 | Loss: 2.5654 | LR: 7.30e-05 | 4573.6ms/step | 13.1 samples/s
+Step 38060/50462 | Loss: 2.5513 | LR: 7.29e-05 | 4016.9ms/step | 14.9 samples/s
+Step 38070/50462 | Loss: 2.5554 | LR: 7.28e-05 | 4013.8ms/step | 14.9 samples/s
+Step 38080/50462 | Loss: 2.5710 | LR: 7.27e-05 | 4004.5ms/step | 15.0 samples/s
+Step 38090/50462 | Loss: 2.4927 | LR: 7.26e-05 | 4001.5ms/step | 15.0 samples/s
+Step 38100/50462 | Loss: 2.5545 | LR: 7.25e-05 | 4001.9ms/step | 15.0 samples/s
+Step 38110/50462 | Loss: 2.5445 | LR: 7.24e-05 | 3996.8ms/step | 15.0 samples/s
+Step 38120/50462 | Loss: 2.5296 | LR: 7.23e-05 | 4007.2ms/step | 15.0 samples/s
+Step 38130/50462 | Loss: 2.5132 | LR: 7.22e-05 | 4016.1ms/step | 14.9 samples/s
+Step 38140/50462 | Loss: 2.5305 | LR: 7.20e-05 | 4009.1ms/step | 15.0 samples/s
+Step 38150/50462 | Loss: 2.5286 | LR: 7.19e-05 | 4010.0ms/step | 15.0 samples/s
+Step 38160/50462 | Loss: 2.5233 | LR: 7.18e-05 | 4007.6ms/step | 15.0 samples/s
+Step 38170/50462 | Loss: 2.5372 | LR: 7.17e-05 | 4008.4ms/step | 15.0 samples/s
+Step 38180/50462 | Loss: 2.5496 | LR: 7.16e-05 | 4006.9ms/step | 15.0 samples/s
+Step 38190/50462 | Loss: 2.5129 | LR: 7.15e-05 | 4001.8ms/step | 15.0 samples/s
+Step 38200/50462 | Loss: 2.5115 | LR: 7.14e-05 | 4016.4ms/step | 14.9 samples/s
+Step 38210/50462 | Loss: 2.5442 | LR: 7.13e-05 | 4001.1ms/step | 15.0 samples/s
+Step 38220/50462 | Loss: 2.4845 | LR: 7.12e-05 | 4000.1ms/step | 15.0 samples/s
+Step 38230/50462 | Loss: 2.5413 | LR: 7.10e-05 | 4011.6ms/step | 15.0 samples/s
+Step 38240/50462 | Loss: 2.5150 | LR: 7.09e-05 | 4019.6ms/step | 14.9 samples/s
+Step 38250/50462 | Loss: 2.5721 | LR: 7.08e-05 | 4013.4ms/step | 15.0 samples/s
+Validation at step 38250: Loss = 2.5166, Perplexity = 12.39
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-38250
+Step 38260/50462 | Loss: 2.5265 | LR: 7.07e-05 | 6002.0ms/step | 10.0 samples/s
+Step 38270/50462 | Loss: 2.5039 | LR: 7.06e-05 | 3991.4ms/step | 15.0 samples/s
+Step 38280/50462 | Loss: 2.5187 | LR: 7.05e-05 | 4007.0ms/step | 15.0 samples/s
+Step 38290/50462 | Loss: 2.5397 | LR: 7.04e-05 | 4011.1ms/step | 15.0 samples/s
+Step 38300/50462 | Loss: 2.5738 | LR: 7.03e-05 | 4009.2ms/step | 15.0 samples/s
+Step 38310/50462 | Loss: 2.5500 | LR: 7.02e-05 | 3992.4ms/step | 15.0 samples/s
+Step 38320/50462 | Loss: 2.5473 | LR: 7.01e-05 | 4017.2ms/step | 14.9 samples/s
+Step 38330/50462 | Loss: 2.5522 | LR: 6.99e-05 | 3996.7ms/step | 15.0 samples/s
+Step 38340/50462 | Loss: 2.5050 | LR: 6.98e-05 | 3988.7ms/step | 15.0 samples/s
+Step 38350/50462 | Loss: 2.5426 | LR: 6.97e-05 | 4002.1ms/step | 15.0 samples/s
+Step 38360/50462 | Loss: 2.5391 | LR: 6.96e-05 | 3992.2ms/step | 15.0 samples/s
+Step 38370/50462 | Loss: 2.5113 | LR: 6.95e-05 | 4003.4ms/step | 15.0 samples/s
+Step 38380/50462 | Loss: 2.5209 | LR: 6.94e-05 | 4005.0ms/step | 15.0 samples/s
+Step 38390/50462 | Loss: 2.5366 | LR: 6.93e-05 | 4001.4ms/step | 15.0 samples/s
+Step 38400/50462 | Loss: 2.5408 | LR: 6.92e-05 | 4003.6ms/step | 15.0 samples/s
+Step 38410/50462 | Loss: 2.5018 | LR: 6.91e-05 | 4001.8ms/step | 15.0 samples/s
+Step 38420/50462 | Loss: 2.5188 | LR: 6.90e-05 | 4590.6ms/step | 13.1 samples/s
+Step 38430/50462 | Loss: 2.5445 | LR: 6.89e-05 | 4004.6ms/step | 15.0 samples/s
+Step 38440/50462 | Loss: 2.5409 | LR: 6.87e-05 | 3999.5ms/step | 15.0 samples/s
+Step 38450/50462 | Loss: 2.5168 | LR: 6.86e-05 | 4002.1ms/step | 15.0 samples/s
+Step 38460/50462 | Loss: 2.5426 | LR: 6.85e-05 | 4003.1ms/step | 15.0 samples/s
+Step 38470/50462 | Loss: 2.4927 | LR: 6.84e-05 | 4002.8ms/step | 15.0 samples/s
+Step 38480/50462 | Loss: 2.5251 | LR: 6.83e-05 | 4000.6ms/step | 15.0 samples/s
+Step 38490/50462 | Loss: 2.5102 | LR: 6.82e-05 | 4002.5ms/step | 15.0 samples/s
+Step 38500/50462 | Loss: 2.5297 | LR: 6.81e-05 | 4002.4ms/step | 15.0 samples/s
+Validation at step 38500: Loss = 2.5156, Perplexity = 12.37
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-38500
+Step 38510/50462 | Loss: 2.5386 | LR: 6.80e-05 | 5951.1ms/step | 10.1 samples/s
+Step 38520/50462 | Loss: 2.5286 | LR: 6.79e-05 | 4013.1ms/step | 15.0 samples/s
+Step 38530/50462 | Loss: 2.5254 | LR: 6.78e-05 | 4004.9ms/step | 15.0 samples/s
+Step 38540/50462 | Loss: 2.5558 | LR: 6.77e-05 | 4000.5ms/step | 15.0 samples/s
+Step 38550/50462 | Loss: 2.5272 | LR: 6.76e-05 | 4011.4ms/step | 15.0 samples/s
+Step 38560/50462 | Loss: 2.5052 | LR: 6.74e-05 | 4015.1ms/step | 14.9 samples/s
+Step 38570/50462 | Loss: 2.5224 | LR: 6.73e-05 | 3994.2ms/step | 15.0 samples/s
+Step 38580/50462 | Loss: 2.5193 | LR: 6.72e-05 | 4005.1ms/step | 15.0 samples/s
+Step 38590/50462 | Loss: 2.5426 | LR: 6.71e-05 | 4005.6ms/step | 15.0 samples/s
+Step 38600/50462 | Loss: 2.5274 | LR: 6.70e-05 | 4001.6ms/step | 15.0 samples/s
+Step 38610/50462 | Loss: 2.5188 | LR: 6.69e-05 | 4009.7ms/step | 15.0 samples/s
+Step 38620/50462 | Loss: 2.5197 | LR: 6.68e-05 | 4004.1ms/step | 15.0 samples/s
+Step 38630/50462 | Loss: 2.5226 | LR: 6.67e-05 | 4003.8ms/step | 15.0 samples/s
+Step 38640/50462 | Loss: 2.5265 | LR: 6.66e-05 | 3998.0ms/step | 15.0 samples/s
+Step 38650/50462 | Loss: 2.5219 | LR: 6.65e-05 | 3999.3ms/step | 15.0 samples/s
+Step 38660/50462 | Loss: 2.4942 | LR: 6.64e-05 | 4000.2ms/step | 15.0 samples/s
+Step 38670/50462 | Loss: 2.5160 | LR: 6.63e-05 | 3997.9ms/step | 15.0 samples/s
+Step 38680/50462 | Loss: 2.5272 | LR: 6.62e-05 | 4010.0ms/step | 15.0 samples/s
+Step 38690/50462 | Loss: 2.5409 | LR: 6.60e-05 | 3995.9ms/step | 15.0 samples/s
+Step 38700/50462 | Loss: 2.5198 | LR: 6.59e-05 | 4018.2ms/step | 14.9 samples/s
+Step 38710/50462 | Loss: 2.5478 | LR: 6.58e-05 | 4587.0ms/step | 13.1 samples/s
+Step 38720/50462 | Loss: 2.5239 | LR: 6.57e-05 | 4001.9ms/step | 15.0 samples/s
+Step 38730/50462 | Loss: 2.5330 | LR: 6.56e-05 | 4006.8ms/step | 15.0 samples/s
+Step 38740/50462 | Loss: 2.5448 | LR: 6.55e-05 | 4001.4ms/step | 15.0 samples/s
+Step 38750/50462 | Loss: 2.5414 | LR: 6.54e-05 | 4008.8ms/step | 15.0 samples/s
+Validation at step 38750: Loss = 2.5156, Perplexity = 12.37
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-38750
+Step 38760/50462 | Loss: 2.5087 | LR: 6.53e-05 | 5975.8ms/step | 10.0 samples/s
+Step 38770/50462 | Loss: 2.5608 | LR: 6.52e-05 | 4577.5ms/step | 13.1 samples/s
+Step 38780/50462 | Loss: 2.5204 | LR: 6.51e-05 | 4010.7ms/step | 15.0 samples/s
+Step 38790/50462 | Loss: 2.5456 | LR: 6.50e-05 | 4014.4ms/step | 14.9 samples/s
+Step 38800/50462 | Loss: 2.5340 | LR: 6.49e-05 | 4011.2ms/step | 15.0 samples/s
+Step 38810/50462 | Loss: 2.5234 | LR: 6.48e-05 | 4003.9ms/step | 15.0 samples/s
+Step 38820/50462 | Loss: 2.5060 | LR: 6.47e-05 | 4003.1ms/step | 15.0 samples/s
+Step 38830/50462 | Loss: 2.5149 | LR: 6.46e-05 | 4001.2ms/step | 15.0 samples/s
+Step 38840/50462 | Loss: 2.5306 | LR: 6.45e-05 | 4016.2ms/step | 14.9 samples/s
+Step 38850/50462 | Loss: 2.5183 | LR: 6.43e-05 | 4008.1ms/step | 15.0 samples/s
+Step 38860/50462 | Loss: 2.5169 | LR: 6.42e-05 | 4003.5ms/step | 15.0 samples/s
+Step 38870/50462 | Loss: 2.5164 | LR: 6.41e-05 | 3991.1ms/step | 15.0 samples/s
+Step 38880/50462 | Loss: 2.5213 | LR: 6.40e-05 | 4004.3ms/step | 15.0 samples/s
+Step 38890/50462 | Loss: 2.5466 | LR: 6.39e-05 | 3988.8ms/step | 15.0 samples/s
+Step 38900/50462 | Loss: 2.5454 | LR: 6.38e-05 | 4000.0ms/step | 15.0 samples/s
+Step 38910/50462 | Loss: 2.5072 | LR: 6.37e-05 | 4007.2ms/step | 15.0 samples/s
+Step 38920/50462 | Loss: 2.5254 | LR: 6.36e-05 | 4001.9ms/step | 15.0 samples/s
+Step 38930/50462 | Loss: 2.5420 | LR: 6.35e-05 | 4010.4ms/step | 15.0 samples/s
+Step 38940/50462 | Loss: 2.4852 | LR: 6.34e-05 | 4007.4ms/step | 15.0 samples/s
+Step 38950/50462 | Loss: 2.5488 | LR: 6.33e-05 | 4008.1ms/step | 15.0 samples/s
+Step 38960/50462 | Loss: 2.4928 | LR: 6.32e-05 | 4004.9ms/step | 15.0 samples/s
+Step 38970/50462 | Loss: 2.5180 | LR: 6.31e-05 | 4014.2ms/step | 14.9 samples/s
+Step 38980/50462 | Loss: 2.5604 | LR: 6.30e-05 | 3999.0ms/step | 15.0 samples/s
+Step 38990/50462 | Loss: 2.5140 | LR: 6.29e-05 | 4000.7ms/step | 15.0 samples/s
+Step 39000/50462 | Loss: 2.5415 | LR: 6.28e-05 | 4008.3ms/step | 15.0 samples/s
+Validation at step 39000: Loss = 2.5153, Perplexity = 12.37
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-39000
+Step 39010/50462 | Loss: 2.5268 | LR: 6.27e-05 | 5968.9ms/step | 10.1 samples/s
+Step 39020/50462 | Loss: 2.4951 | LR: 6.26e-05 | 4003.1ms/step | 15.0 samples/s
+Step 39030/50462 | Loss: 2.5821 | LR: 6.25e-05 | 3999.4ms/step | 15.0 samples/s
+Step 39040/50462 | Loss: 2.5043 | LR: 6.23e-05 | 4009.1ms/step | 15.0 samples/s
+Step 39050/50462 | Loss: 2.4923 | LR: 6.22e-05 | 4007.6ms/step | 15.0 samples/s
+Step 39060/50462 | Loss: 2.5380 | LR: 6.21e-05 | 4004.9ms/step | 15.0 samples/s
+Step 39070/50462 | Loss: 2.5371 | LR: 6.20e-05 | 3988.1ms/step | 15.0 samples/s
+Step 39080/50462 | Loss: 2.5212 | LR: 6.19e-05 | 4006.2ms/step | 15.0 samples/s
+Step 39090/50462 | Loss: 2.5256 | LR: 6.18e-05 | 4008.7ms/step | 15.0 samples/s
+Step 39100/50462 | Loss: 2.5145 | LR: 6.17e-05 | 4001.5ms/step | 15.0 samples/s
+Step 39110/50462 | Loss: 2.5222 | LR: 6.16e-05 | 4003.8ms/step | 15.0 samples/s
+Step 39120/50462 | Loss: 2.5528 | LR: 6.15e-05 | 4003.7ms/step | 15.0 samples/s
+Step 39130/50462 | Loss: 2.5391 | LR: 6.14e-05 | 4000.8ms/step | 15.0 samples/s
+Step 39140/50462 | Loss: 2.5232 | LR: 6.13e-05 | 4001.8ms/step | 15.0 samples/s
+Step 39150/50462 | Loss: 2.5392 | LR: 6.12e-05 | 4010.7ms/step | 15.0 samples/s
+Step 39160/50462 | Loss: 2.5816 | LR: 6.11e-05 | 4006.3ms/step | 15.0 samples/s
+Step 39170/50462 | Loss: 2.5041 | LR: 6.10e-05 | 4001.2ms/step | 15.0 samples/s
+Step 39180/50462 | Loss: 2.5259 | LR: 6.09e-05 | 4013.6ms/step | 14.9 samples/s
+Step 39190/50462 | Loss: 2.5302 | LR: 6.08e-05 | 4010.9ms/step | 15.0 samples/s
+Step 39200/50462 | Loss: 2.5225 | LR: 6.07e-05 | 4009.2ms/step | 15.0 samples/s
+Step 39210/50462 | Loss: 2.5333 | LR: 6.06e-05 | 4004.9ms/step | 15.0 samples/s
+Step 39220/50462 | Loss: 2.5238 | LR: 6.05e-05 | 4006.6ms/step | 15.0 samples/s
+Step 39230/50462 | Loss: 2.5580 | LR: 6.04e-05 | 4005.6ms/step | 15.0 samples/s
+Step 39240/50462 | Loss: 2.5330 | LR: 6.03e-05 | 4005.1ms/step | 15.0 samples/s
+Step 39250/50462 | Loss: 2.5298 | LR: 6.02e-05 | 4008.3ms/step | 15.0 samples/s
+Validation at step 39250: Loss = 2.5147, Perplexity = 12.36
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-39250
+Step 39260/50462 | Loss: 2.5355 | LR: 6.01e-05 | 5946.9ms/step | 10.1 samples/s
+Step 39270/50462 | Loss: 2.5374 | LR: 6.00e-05 | 4011.4ms/step | 15.0 samples/s
+Step 39280/50462 | Loss: 2.5800 | LR: 5.99e-05 | 4001.8ms/step | 15.0 samples/s
+Step 39290/50462 | Loss: 2.5258 | LR: 5.98e-05 | 4002.2ms/step | 15.0 samples/s
+Step 39300/50462 | Loss: 2.5512 | LR: 5.97e-05 | 4005.1ms/step | 15.0 samples/s
+Step 39310/50462 | Loss: 2.5680 | LR: 5.96e-05 | 4012.6ms/step | 15.0 samples/s
+Step 39320/50462 | Loss: 2.5396 | LR: 5.95e-05 | 4005.5ms/step | 15.0 samples/s
+Step 39330/50462 | Loss: 2.4976 | LR: 5.94e-05 | 4007.1ms/step | 15.0 samples/s
+Step 39340/50462 | Loss: 2.5320 | LR: 5.93e-05 | 4014.7ms/step | 14.9 samples/s
+Step 39350/50462 | Loss: 2.5123 | LR: 5.91e-05 | 4008.1ms/step | 15.0 samples/s
+Step 39360/50462 | Loss: 2.5155 | LR: 5.90e-05 | 4018.9ms/step | 14.9 samples/s
+Step 39370/50462 | Loss: 2.5245 | LR: 5.89e-05 | 4006.9ms/step | 15.0 samples/s
+Step 39380/50462 | Loss: 2.5099 | LR: 5.88e-05 | 4000.3ms/step | 15.0 samples/s
+Step 39390/50462 | Loss: 2.5313 | LR: 5.87e-05 | 4008.8ms/step | 15.0 samples/s
+Step 39400/50462 | Loss: 2.5152 | LR: 5.86e-05 | 4012.8ms/step | 15.0 samples/s
+Step 39410/50462 | Loss: 2.5459 | LR: 5.85e-05 | 4014.3ms/step | 14.9 samples/s
+Step 39420/50462 | Loss: 2.5242 | LR: 5.84e-05 | 3995.2ms/step | 15.0 samples/s
+Step 39430/50462 | Loss: 2.5140 | LR: 5.83e-05 | 4006.9ms/step | 15.0 samples/s
+Step 39440/50462 | Loss: 2.5428 | LR: 5.82e-05 | 4015.4ms/step | 14.9 samples/s
+Step 39450/50462 | Loss: 2.5944 | LR: 5.81e-05 | 4006.8ms/step | 15.0 samples/s
+Step 39460/50462 | Loss: 2.5178 | LR: 5.80e-05 | 4017.0ms/step | 14.9 samples/s
+Step 39470/50462 | Loss: 2.5450 | LR: 5.79e-05 | 3998.8ms/step | 15.0 samples/s
+Step 39480/50462 | Loss: 2.5051 | LR: 5.78e-05 | 4020.4ms/step | 14.9 samples/s
+Step 39490/50462 | Loss: 2.4963 | LR: 5.77e-05 | 4021.2ms/step | 14.9 samples/s
+Step 39500/50462 | Loss: 2.5452 | LR: 5.76e-05 | 4013.7ms/step | 14.9 samples/s
+Validation at step 39500: Loss = 2.5140, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-39500
+Step 39510/50462 | Loss: 2.5403 | LR: 5.75e-05 | 5975.0ms/step | 10.0 samples/s
+Step 39520/50462 | Loss: 2.5472 | LR: 5.74e-05 | 4003.1ms/step | 15.0 samples/s
+Step 39530/50462 | Loss: 2.5334 | LR: 5.73e-05 | 4010.2ms/step | 15.0 samples/s
+Step 39540/50462 | Loss: 2.5224 | LR: 5.72e-05 | 4021.7ms/step | 14.9 samples/s
+Step 39550/50462 | Loss: 2.5452 | LR: 5.71e-05 | 4019.2ms/step | 14.9 samples/s
+Step 39560/50462 | Loss: 2.5217 | LR: 5.70e-05 | 3999.4ms/step | 15.0 samples/s
+Step 39570/50462 | Loss: 2.5778 | LR: 5.69e-05 | 4000.3ms/step | 15.0 samples/s
+Step 39580/50462 | Loss: 2.5182 | LR: 5.68e-05 | 4008.9ms/step | 15.0 samples/s
+Step 39590/50462 | Loss: 2.5446 | LR: 5.67e-05 | 4017.3ms/step | 14.9 samples/s
+Step 39600/50462 | Loss: 2.5355 | LR: 5.66e-05 | 4001.4ms/step | 15.0 samples/s
+Step 39610/50462 | Loss: 2.5099 | LR: 5.65e-05 | 4001.9ms/step | 15.0 samples/s
+Step 39620/50462 | Loss: 2.4746 | LR: 5.64e-05 | 4008.0ms/step | 15.0 samples/s
+Step 39630/50462 | Loss: 2.5512 | LR: 5.63e-05 | 4012.7ms/step | 15.0 samples/s
+Step 39640/50462 | Loss: 2.5394 | LR: 5.62e-05 | 4018.7ms/step | 14.9 samples/s
+Step 39650/50462 | Loss: 2.5413 | LR: 5.61e-05 | 4014.3ms/step | 14.9 samples/s
+Step 39660/50462 | Loss: 2.5323 | LR: 5.60e-05 | 4017.1ms/step | 14.9 samples/s
+Step 39670/50462 | Loss: 2.5298 | LR: 5.59e-05 | 4006.0ms/step | 15.0 samples/s
+Step 39680/50462 | Loss: 2.5260 | LR: 5.58e-05 | 4002.2ms/step | 15.0 samples/s
+Step 39690/50462 | Loss: 2.5366 | LR: 5.57e-05 | 3997.4ms/step | 15.0 samples/s
+Step 39700/50462 | Loss: 2.5185 | LR: 5.56e-05 | 4009.3ms/step | 15.0 samples/s
+Step 39710/50462 | Loss: 2.5375 | LR: 5.55e-05 | 4009.0ms/step | 15.0 samples/s
+Step 39720/50462 | Loss: 2.5514 | LR: 5.54e-05 | 4014.6ms/step | 14.9 samples/s
+Step 39730/50462 | Loss: 2.5145 | LR: 5.53e-05 | 4003.4ms/step | 15.0 samples/s
+Step 39740/50462 | Loss: 2.5117 | LR: 5.52e-05 | 4010.6ms/step | 15.0 samples/s
+Step 39750/50462 | Loss: 2.5405 | LR: 5.51e-05 | 4012.6ms/step | 15.0 samples/s
+Validation at step 39750: Loss = 2.5142, Perplexity = 12.36
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-39750
+Step 39760/50462 | Loss: 2.5424 | LR: 5.50e-05 | 6040.2ms/step | 9.9 samples/s
+Step 39770/50462 | Loss: 2.4993 | LR: 5.49e-05 | 4016.8ms/step | 14.9 samples/s
+Step 39780/50462 | Loss: 2.5170 | LR: 5.48e-05 | 4014.7ms/step | 14.9 samples/s
+Step 39790/50462 | Loss: 2.5228 | LR: 5.47e-05 | 4009.8ms/step | 15.0 samples/s
+Step 39800/50462 | Loss: 2.5443 | LR: 5.46e-05 | 4001.9ms/step | 15.0 samples/s
+Step 39810/50462 | Loss: 2.4825 | LR: 5.45e-05 | 4009.5ms/step | 15.0 samples/s
+Step 39820/50462 | Loss: 2.5189 | LR: 5.44e-05 | 4015.0ms/step | 14.9 samples/s
+Step 39830/50462 | Loss: 2.5114 | LR: 5.43e-05 | 4011.0ms/step | 15.0 samples/s
+Step 39840/50462 | Loss: 2.5643 | LR: 5.42e-05 | 4013.9ms/step | 14.9 samples/s
+Step 39850/50462 | Loss: 2.5269 | LR: 5.41e-05 | 4012.2ms/step | 15.0 samples/s
+Step 39860/50462 | Loss: 2.5235 | LR: 5.40e-05 | 4013.5ms/step | 14.9 samples/s
+Step 39870/50462 | Loss: 2.5342 | LR: 5.39e-05 | 4014.5ms/step | 14.9 samples/s
+Step 39880/50462 | Loss: 2.5516 | LR: 5.38e-05 | 4016.3ms/step | 14.9 samples/s
+Step 39890/50462 | Loss: 2.4943 | LR: 5.38e-05 | 3998.2ms/step | 15.0 samples/s
+Step 39900/50462 | Loss: 2.5347 | LR: 5.37e-05 | 4007.6ms/step | 15.0 samples/s
+Step 39910/50462 | Loss: 2.4953 | LR: 5.36e-05 | 4024.9ms/step | 14.9 samples/s
+Step 39920/50462 | Loss: 2.5496 | LR: 5.35e-05 | 4006.5ms/step | 15.0 samples/s
+Step 39930/50462 | Loss: 2.5228 | LR: 5.34e-05 | 4002.6ms/step | 15.0 samples/s
+Step 39940/50462 | Loss: 2.5518 | LR: 5.33e-05 | 4013.9ms/step | 14.9 samples/s
+Step 39950/50462 | Loss: 2.5025 | LR: 5.32e-05 | 4008.0ms/step | 15.0 samples/s
+Step 39960/50462 | Loss: 2.5485 | LR: 5.31e-05 | 4013.8ms/step | 14.9 samples/s
+Step 39970/50462 | Loss: 2.5016 | LR: 5.30e-05 | 3999.7ms/step | 15.0 samples/s
+Step 39980/50462 | Loss: 2.5409 | LR: 5.29e-05 | 4007.2ms/step | 15.0 samples/s
+Step 39990/50462 | Loss: 2.5106 | LR: 5.28e-05 | 3993.4ms/step | 15.0 samples/s
+Step 40000/50462 | Loss: 2.5219 | LR: 5.27e-05 | 4006.8ms/step | 15.0 samples/s
+Validation at step 40000: Loss = 2.5144, Perplexity = 12.36
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-40000
+Step 40010/50462 | Loss: 2.5225 | LR: 5.26e-05 | 5978.1ms/step | 10.0 samples/s
+Step 40020/50462 | Loss: 2.5309 | LR: 5.25e-05 | 4015.9ms/step | 14.9 samples/s
+Step 40030/50462 | Loss: 2.5096 | LR: 5.24e-05 | 4008.3ms/step | 15.0 samples/s
+Step 40040/50462 | Loss: 2.5430 | LR: 5.23e-05 | 4016.0ms/step | 14.9 samples/s
+Step 40050/50462 | Loss: 2.5232 | LR: 5.22e-05 | 4004.6ms/step | 15.0 samples/s
+Step 40060/50462 | Loss: 2.5315 | LR: 5.21e-05 | 4019.3ms/step | 14.9 samples/s
+Step 40070/50462 | Loss: 2.5196 | LR: 5.20e-05 | 3990.4ms/step | 15.0 samples/s
+Step 40080/50462 | Loss: 2.5333 | LR: 5.19e-05 | 4003.2ms/step | 15.0 samples/s
+Step 40090/50462 | Loss: 2.5457 | LR: 5.18e-05 | 4000.8ms/step | 15.0 samples/s
+Step 40100/50462 | Loss: 2.5135 | LR: 5.17e-05 | 3998.7ms/step | 15.0 samples/s
+Step 40110/50462 | Loss: 2.5633 | LR: 5.16e-05 | 4005.9ms/step | 15.0 samples/s
+Step 40120/50462 | Loss: 2.5690 | LR: 5.15e-05 | 4015.2ms/step | 14.9 samples/s
+Step 40130/50462 | Loss: 2.5397 | LR: 5.14e-05 | 4017.3ms/step | 14.9 samples/s
+Step 40140/50462 | Loss: 2.5211 | LR: 5.13e-05 | 4019.6ms/step | 14.9 samples/s
+Step 40150/50462 | Loss: 2.5124 | LR: 5.12e-05 | 4017.4ms/step | 14.9 samples/s
+Step 40160/50462 | Loss: 2.5518 | LR: 5.11e-05 | 4014.8ms/step | 14.9 samples/s
+Step 40170/50462 | Loss: 2.4963 | LR: 5.10e-05 | 4003.6ms/step | 15.0 samples/s
+Step 40180/50462 | Loss: 2.5508 | LR: 5.09e-05 | 4010.8ms/step | 15.0 samples/s
+Step 40190/50462 | Loss: 2.5099 | LR: 5.09e-05 | 4004.3ms/step | 15.0 samples/s
+Step 40200/50462 | Loss: 2.5201 | LR: 5.08e-05 | 4004.1ms/step | 15.0 samples/s
+Step 40210/50462 | Loss: 2.4976 | LR: 5.07e-05 | 4005.2ms/step | 15.0 samples/s
+Step 40220/50462 | Loss: 2.5117 | LR: 5.06e-05 | 4005.2ms/step | 15.0 samples/s
+Step 40230/50462 | Loss: 2.5380 | LR: 5.05e-05 | 4005.2ms/step | 15.0 samples/s
+Step 40240/50462 | Loss: 2.5342 | LR: 5.04e-05 | 4008.8ms/step | 15.0 samples/s
+Step 40250/50462 | Loss: 2.5363 | LR: 5.03e-05 | 4003.6ms/step | 15.0 samples/s
+Validation at step 40250: Loss = 2.5140, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-40250
+Step 40260/50462 | Loss: 2.5399 | LR: 5.02e-05 | 6007.2ms/step | 10.0 samples/s
+Step 40270/50462 | Loss: 2.5376 | LR: 5.01e-05 | 4024.9ms/step | 14.9 samples/s
+Step 40280/50462 | Loss: 2.5174 | LR: 5.00e-05 | 4011.5ms/step | 15.0 samples/s
+Step 40290/50462 | Loss: 2.5202 | LR: 4.99e-05 | 4016.7ms/step | 14.9 samples/s
+Step 40300/50462 | Loss: 2.5231 | LR: 4.98e-05 | 4000.3ms/step | 15.0 samples/s
+Step 40310/50462 | Loss: 2.5439 | LR: 4.97e-05 | 4009.4ms/step | 15.0 samples/s
+Step 40320/50462 | Loss: 2.5283 | LR: 4.96e-05 | 4002.1ms/step | 15.0 samples/s
+Step 40330/50462 | Loss: 2.5249 | LR: 4.95e-05 | 4014.1ms/step | 14.9 samples/s
+Step 40340/50462 | Loss: 2.5171 | LR: 4.94e-05 | 4000.6ms/step | 15.0 samples/s
+Step 40350/50462 | Loss: 2.4994 | LR: 4.93e-05 | 4007.6ms/step | 15.0 samples/s
+Step 40360/50462 | Loss: 2.5192 | LR: 4.92e-05 | 4003.8ms/step | 15.0 samples/s
+Step 40370/50462 | Loss: 2.4739 | LR: 4.91e-05 | 4000.2ms/step | 15.0 samples/s
+Step 40380/50462 | Loss: 2.5332 | LR: 4.90e-05 | 4018.2ms/step | 14.9 samples/s
+Step 40390/50462 | Loss: 2.5144 | LR: 4.90e-05 | 4014.3ms/step | 14.9 samples/s
+Step 40400/50462 | Loss: 2.5392 | LR: 4.89e-05 | 4011.1ms/step | 15.0 samples/s
+Step 40410/50462 | Loss: 2.5479 | LR: 4.88e-05 | 4019.2ms/step | 14.9 samples/s
+Step 40420/50462 | Loss: 2.5107 | LR: 4.87e-05 | 4001.6ms/step | 15.0 samples/s
+Step 40430/50462 | Loss: 2.5407 | LR: 4.86e-05 | 4012.6ms/step | 15.0 samples/s
+Step 40440/50462 | Loss: 2.5588 | LR: 4.85e-05 | 4010.6ms/step | 15.0 samples/s
+Step 40450/50462 | Loss: 2.5069 | LR: 4.84e-05 | 4016.1ms/step | 14.9 samples/s
+Step 40460/50462 | Loss: 2.5123 | LR: 4.83e-05 | 4013.3ms/step | 15.0 samples/s
+Step 40470/50462 | Loss: 2.5297 | LR: 4.82e-05 | 4008.5ms/step | 15.0 samples/s
+Step 40480/50462 | Loss: 2.5401 | LR: 4.81e-05 | 4004.4ms/step | 15.0 samples/s
+Step 40490/50462 | Loss: 2.5168 | LR: 4.80e-05 | 4008.5ms/step | 15.0 samples/s
+Step 40500/50462 | Loss: 2.5467 | LR: 4.79e-05 | 4016.3ms/step | 14.9 samples/s
+Validation at step 40500: Loss = 2.5138, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-40500
+Step 40510/50462 | Loss: 2.5449 | LR: 4.78e-05 | 6018.6ms/step | 10.0 samples/s
+Step 40520/50462 | Loss: 2.5054 | LR: 4.77e-05 | 4016.1ms/step | 14.9 samples/s
+Step 40530/50462 | Loss: 2.5126 | LR: 4.76e-05 | 4009.9ms/step | 15.0 samples/s
+Step 40540/50462 | Loss: 2.5323 | LR: 4.76e-05 | 4013.4ms/step | 15.0 samples/s
+Step 40550/50462 | Loss: 2.5121 | LR: 4.75e-05 | 4013.6ms/step | 14.9 samples/s
+Step 40560/50462 | Loss: 2.5098 | LR: 4.74e-05 | 4013.8ms/step | 14.9 samples/s
+Step 40570/50462 | Loss: 2.5412 | LR: 4.73e-05 | 4013.2ms/step | 15.0 samples/s
+Step 40580/50462 | Loss: 2.5172 | LR: 4.72e-05 | 4014.5ms/step | 14.9 samples/s
+Step 40590/50462 | Loss: 2.5357 | LR: 4.71e-05 | 4016.8ms/step | 14.9 samples/s
+Step 40600/50462 | Loss: 2.5051 | LR: 4.70e-05 | 4016.5ms/step | 14.9 samples/s
+Step 40610/50462 | Loss: 2.5517 | LR: 4.69e-05 | 3996.6ms/step | 15.0 samples/s
+Step 40620/50462 | Loss: 2.5123 | LR: 4.68e-05 | 4005.0ms/step | 15.0 samples/s
+Step 40630/50462 | Loss: 2.5527 | LR: 4.67e-05 | 4010.3ms/step | 15.0 samples/s
+Step 40640/50462 | Loss: 2.5468 | LR: 4.66e-05 | 4003.2ms/step | 15.0 samples/s
+Step 40650/50462 | Loss: 2.5537 | LR: 4.65e-05 | 4005.6ms/step | 15.0 samples/s
+Step 40660/50462 | Loss: 2.5046 | LR: 4.64e-05 | 4015.7ms/step | 14.9 samples/s
+Step 40670/50462 | Loss: 2.5486 | LR: 4.64e-05 | 4005.3ms/step | 15.0 samples/s
+Step 40680/50462 | Loss: 2.5265 | LR: 4.63e-05 | 4006.0ms/step | 15.0 samples/s
+Step 40690/50462 | Loss: 2.5485 | LR: 4.62e-05 | 4017.1ms/step | 14.9 samples/s
+Step 40700/50462 | Loss: 2.5038 | LR: 4.61e-05 | 4007.0ms/step | 15.0 samples/s
+Step 40710/50462 | Loss: 2.4995 | LR: 4.60e-05 | 4005.6ms/step | 15.0 samples/s
+Step 40720/50462 | Loss: 2.5343 | LR: 4.59e-05 | 4007.6ms/step | 15.0 samples/s
+Step 40730/50462 | Loss: 2.5376 | LR: 4.58e-05 | 4006.0ms/step | 15.0 samples/s
+Step 40740/50462 | Loss: 2.5358 | LR: 4.57e-05 | 4007.9ms/step | 15.0 samples/s
+Step 40750/50462 | Loss: 2.5441 | LR: 4.56e-05 | 4018.8ms/step | 14.9 samples/s
+Validation at step 40750: Loss = 2.5142, Perplexity = 12.36
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-40750
+Step 40760/50462 | Loss: 2.5616 | LR: 4.55e-05 | 6113.1ms/step | 9.8 samples/s
+Step 40770/50462 | Loss: 2.5284 | LR: 4.54e-05 | 4006.6ms/step | 15.0 samples/s
+Step 40780/50462 | Loss: 2.5351 | LR: 4.54e-05 | 4013.7ms/step | 14.9 samples/s
+Step 40790/50462 | Loss: 2.5405 | LR: 4.53e-05 | 4011.4ms/step | 15.0 samples/s
+Step 40800/50462 | Loss: 2.5533 | LR: 4.52e-05 | 4011.3ms/step | 15.0 samples/s
+Step 40810/50462 | Loss: 2.5353 | LR: 4.51e-05 | 3999.1ms/step | 15.0 samples/s
+Step 40820/50462 | Loss: 2.5097 | LR: 4.50e-05 | 4003.0ms/step | 15.0 samples/s
+Step 40830/50462 | Loss: 2.5362 | LR: 4.49e-05 | 4021.2ms/step | 14.9 samples/s
+Step 40840/50462 | Loss: 2.4932 | LR: 4.48e-05 | 3998.5ms/step | 15.0 samples/s
+Step 40850/50462 | Loss: 2.5236 | LR: 4.47e-05 | 4009.0ms/step | 15.0 samples/s
+Step 40860/50462 | Loss: 2.5116 | LR: 4.46e-05 | 4000.3ms/step | 15.0 samples/s
+Step 40870/50462 | Loss: 2.5206 | LR: 4.45e-05 | 4013.8ms/step | 14.9 samples/s
+Step 40880/50462 | Loss: 2.5283 | LR: 4.45e-05 | 4022.5ms/step | 14.9 samples/s
+Step 40890/50462 | Loss: 2.5058 | LR: 4.44e-05 | 4011.7ms/step | 15.0 samples/s
+Step 40900/50462 | Loss: 2.5314 | LR: 4.43e-05 | 4013.5ms/step | 14.9 samples/s
+Step 40910/50462 | Loss: 2.5079 | LR: 4.42e-05 | 4001.1ms/step | 15.0 samples/s
+Step 40920/50462 | Loss: 2.5369 | LR: 4.41e-05 | 4017.9ms/step | 14.9 samples/s
+Step 40930/50462 | Loss: 2.5192 | LR: 4.40e-05 | 4004.8ms/step | 15.0 samples/s
+Step 40940/50462 | Loss: 2.5180 | LR: 4.39e-05 | 4008.0ms/step | 15.0 samples/s
+Step 40950/50462 | Loss: 2.5770 | LR: 4.38e-05 | 4024.4ms/step | 14.9 samples/s
+Step 40960/50462 | Loss: 2.5337 | LR: 4.37e-05 | 4004.6ms/step | 15.0 samples/s
+Step 40970/50462 | Loss: 2.5261 | LR: 4.36e-05 | 4016.9ms/step | 14.9 samples/s
+Step 40980/50462 | Loss: 2.5395 | LR: 4.36e-05 | 4010.8ms/step | 15.0 samples/s
+Step 40990/50462 | Loss: 2.5225 | LR: 4.35e-05 | 4024.8ms/step | 14.9 samples/s
+Step 41000/50462 | Loss: 2.5120 | LR: 4.34e-05 | 4018.7ms/step | 14.9 samples/s
+Validation at step 41000: Loss = 2.5138, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41000
+Step 41010/50462 | Loss: 2.5142 | LR: 4.33e-05 | 6083.0ms/step | 9.9 samples/s
+Step 41020/50462 | Loss: 2.5342 | LR: 4.32e-05 | 4014.6ms/step | 14.9 samples/s
+Step 41030/50462 | Loss: 2.4986 | LR: 4.31e-05 | 4014.1ms/step | 14.9 samples/s
+Step 41040/50462 | Loss: 2.5457 | LR: 4.30e-05 | 4018.3ms/step | 14.9 samples/s
+Step 41050/50462 | Loss: 2.5463 | LR: 4.29e-05 | 4008.5ms/step | 15.0 samples/s
+Step 41060/50462 | Loss: 2.5428 | LR: 4.28e-05 | 4010.4ms/step | 15.0 samples/s
+Step 41070/50462 | Loss: 2.5159 | LR: 4.28e-05 | 3997.6ms/step | 15.0 samples/s
+Step 41080/50462 | Loss: 2.5320 | LR: 4.27e-05 | 4013.1ms/step | 15.0 samples/s
+Step 41090/50462 | Loss: 2.5029 | LR: 4.26e-05 | 4016.8ms/step | 14.9 samples/s
+Step 41100/50462 | Loss: 2.5280 | LR: 4.25e-05 | 4012.7ms/step | 15.0 samples/s
+Step 41110/50462 | Loss: 2.5120 | LR: 4.24e-05 | 4004.6ms/step | 15.0 samples/s
+Step 41120/50462 | Loss: 2.5210 | LR: 4.23e-05 | 4009.0ms/step | 15.0 samples/s
+Step 41130/50462 | Loss: 2.5365 | LR: 4.22e-05 | 3998.4ms/step | 15.0 samples/s
+Step 41140/50462 | Loss: 2.5316 | LR: 4.21e-05 | 4011.5ms/step | 15.0 samples/s
+Step 41150/50462 | Loss: 2.5744 | LR: 4.21e-05 | 4007.3ms/step | 15.0 samples/s
+Step 41160/50462 | Loss: 2.5142 | LR: 4.20e-05 | 4011.6ms/step | 15.0 samples/s
+Step 41170/50462 | Loss: 2.5081 | LR: 4.19e-05 | 4008.9ms/step | 15.0 samples/s
+Step 41180/50462 | Loss: 2.4817 | LR: 4.18e-05 | 4007.7ms/step | 15.0 samples/s
+Step 41190/50462 | Loss: 2.5155 | LR: 4.17e-05 | 4012.6ms/step | 15.0 samples/s
+Step 41200/50462 | Loss: 2.5207 | LR: 4.16e-05 | 4011.1ms/step | 15.0 samples/s
+Step 41210/50462 | Loss: 2.5278 | LR: 4.15e-05 | 4015.0ms/step | 14.9 samples/s
+Step 41220/50462 | Loss: 2.5038 | LR: 4.14e-05 | 4008.0ms/step | 15.0 samples/s
+Step 41230/50462 | Loss: 2.5054 | LR: 4.14e-05 | 4021.4ms/step | 14.9 samples/s
+Step 41240/50462 | Loss: 2.4993 | LR: 4.13e-05 | 4002.6ms/step | 15.0 samples/s
+Step 41250/50462 | Loss: 2.4888 | LR: 4.12e-05 | 4014.7ms/step | 14.9 samples/s
+Validation at step 41250: Loss = 2.5137, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41250
+Step 41260/50462 | Loss: 2.5110 | LR: 4.11e-05 | 6007.1ms/step | 10.0 samples/s
+Step 41270/50462 | Loss: 2.5246 | LR: 4.10e-05 | 4025.7ms/step | 14.9 samples/s
+Step 41280/50462 | Loss: 2.4996 | LR: 4.09e-05 | 4016.9ms/step | 14.9 samples/s
+Step 41290/50462 | Loss: 2.5258 | LR: 4.08e-05 | 4002.0ms/step | 15.0 samples/s
+Step 41300/50462 | Loss: 2.5121 | LR: 4.07e-05 | 4015.8ms/step | 14.9 samples/s
+Step 41310/50462 | Loss: 2.4976 | LR: 4.07e-05 | 4001.5ms/step | 15.0 samples/s
+Step 41320/50462 | Loss: 2.5293 | LR: 4.06e-05 | 4005.9ms/step | 15.0 samples/s
+Step 41330/50462 | Loss: 2.5348 | LR: 4.05e-05 | 4009.0ms/step | 15.0 samples/s
+Step 41340/50462 | Loss: 2.5139 | LR: 4.04e-05 | 4010.9ms/step | 15.0 samples/s
+Step 41350/50462 | Loss: 2.5209 | LR: 4.03e-05 | 4010.9ms/step | 15.0 samples/s
+Step 41360/50462 | Loss: 2.5241 | LR: 4.02e-05 | 3997.1ms/step | 15.0 samples/s
+Step 41370/50462 | Loss: 2.5372 | LR: 4.01e-05 | 4012.6ms/step | 15.0 samples/s
+Step 41380/50462 | Loss: 2.5285 | LR: 4.01e-05 | 4006.5ms/step | 15.0 samples/s
+Step 41390/50462 | Loss: 2.5382 | LR: 4.00e-05 | 4001.6ms/step | 15.0 samples/s
+Step 41400/50462 | Loss: 2.5235 | LR: 3.99e-05 | 4012.8ms/step | 15.0 samples/s
+Step 41410/50462 | Loss: 2.5132 | LR: 3.98e-05 | 4010.1ms/step | 15.0 samples/s
+Step 41420/50462 | Loss: 2.5363 | LR: 3.97e-05 | 4019.4ms/step | 14.9 samples/s
+Step 41430/50462 | Loss: 2.5527 | LR: 3.96e-05 | 4014.1ms/step | 14.9 samples/s
+Step 41440/50462 | Loss: 2.5370 | LR: 3.95e-05 | 4008.4ms/step | 15.0 samples/s
+Step 41450/50462 | Loss: 2.5460 | LR: 3.95e-05 | 4019.8ms/step | 14.9 samples/s
+Step 41460/50462 | Loss: 2.5152 | LR: 3.94e-05 | 4009.3ms/step | 15.0 samples/s
+Step 41470/50462 | Loss: 2.5329 | LR: 3.93e-05 | 4002.4ms/step | 15.0 samples/s
+Step 41480/50462 | Loss: 2.5015 | LR: 3.92e-05 | 4020.1ms/step | 14.9 samples/s
+Step 41490/50462 | Loss: 2.5214 | LR: 3.91e-05 | 4009.2ms/step | 15.0 samples/s
+Step 41500/50462 | Loss: 2.5022 | LR: 3.90e-05 | 4006.1ms/step | 15.0 samples/s
+Validation at step 41500: Loss = 2.5127, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41500
+Step 41510/50462 | Loss: 2.5158 | LR: 3.90e-05 | 6015.6ms/step | 10.0 samples/s
+Step 41520/50462 | Loss: 2.5272 | LR: 3.89e-05 | 4000.3ms/step | 15.0 samples/s
+Step 41530/50462 | Loss: 2.5400 | LR: 3.88e-05 | 3998.2ms/step | 15.0 samples/s
+Step 41540/50462 | Loss: 2.5086 | LR: 3.87e-05 | 4008.2ms/step | 15.0 samples/s
+Step 41550/50462 | Loss: 2.5544 | LR: 3.86e-05 | 4015.9ms/step | 14.9 samples/s
+Step 41560/50462 | Loss: 2.5107 | LR: 3.85e-05 | 4008.5ms/step | 15.0 samples/s
+Step 41570/50462 | Loss: 2.5127 | LR: 3.84e-05 | 4018.9ms/step | 14.9 samples/s
+Step 41580/50462 | Loss: 2.5236 | LR: 3.84e-05 | 4005.6ms/step | 15.0 samples/s
+Step 41590/50462 | Loss: 2.5112 | LR: 3.83e-05 | 4009.0ms/step | 15.0 samples/s
+Step 41600/50462 | Loss: 2.5354 | LR: 3.82e-05 | 3996.0ms/step | 15.0 samples/s
+Step 41610/50462 | Loss: 2.5519 | LR: 3.81e-05 | 4012.0ms/step | 15.0 samples/s
+Step 41620/50462 | Loss: 2.5336 | LR: 3.80e-05 | 4022.8ms/step | 14.9 samples/s
+Step 41630/50462 | Loss: 2.5274 | LR: 3.79e-05 | 4008.2ms/step | 15.0 samples/s
+Step 41640/50462 | Loss: 2.5526 | LR: 3.79e-05 | 4017.5ms/step | 14.9 samples/s
+Step 41650/50462 | Loss: 2.5316 | LR: 3.78e-05 | 4013.2ms/step | 15.0 samples/s
+Step 41660/50462 | Loss: 2.5129 | LR: 3.77e-05 | 4008.6ms/step | 15.0 samples/s
+Step 41670/50462 | Loss: 2.5274 | LR: 3.76e-05 | 4005.7ms/step | 15.0 samples/s
+Step 41680/50462 | Loss: 2.5415 | LR: 3.75e-05 | 4013.5ms/step | 14.9 samples/s
+Step 41690/50462 | Loss: 2.5204 | LR: 3.74e-05 | 4023.1ms/step | 14.9 samples/s
+Step 41700/50462 | Loss: 2.5329 | LR: 3.74e-05 | 4021.8ms/step | 14.9 samples/s
+Step 41710/50462 | Loss: 2.5468 | LR: 3.73e-05 | 4015.7ms/step | 14.9 samples/s
+Step 41720/50462 | Loss: 2.5391 | LR: 3.72e-05 | 4011.2ms/step | 15.0 samples/s
+Step 41730/50462 | Loss: 2.4665 | LR: 3.71e-05 | 4015.2ms/step | 14.9 samples/s
+Step 41740/50462 | Loss: 2.5286 | LR: 3.70e-05 | 4008.1ms/step | 15.0 samples/s
+Step 41750/50462 | Loss: 2.4952 | LR: 3.69e-05 | 4019.0ms/step | 14.9 samples/s
+Validation at step 41750: Loss = 2.5134, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750
+Step 41760/50462 | Loss: 2.5262 | LR: 3.69e-05 | 5985.5ms/step | 10.0 samples/s
+Step 41770/50462 | Loss: 2.5425 | LR: 3.68e-05 | 4010.0ms/step | 15.0 samples/s
+Step 41780/50462 | Loss: 2.5098 | LR: 3.67e-05 | 4015.9ms/step | 14.9 samples/s
+Step 41790/50462 | Loss: 2.5031 | LR: 3.66e-05 | 4013.3ms/step | 15.0 samples/s
+Step 41800/50462 | Loss: 2.5011 | LR: 3.65e-05 | 4015.4ms/step | 14.9 samples/s
+Step 41810/50462 | Loss: 2.5073 | LR: 3.64e-05 | 4013.6ms/step | 14.9 samples/s
+Step 41820/50462 | Loss: 2.5483 | LR: 3.64e-05 | 4017.3ms/step | 14.9 samples/s
+Step 41830/50462 | Loss: 2.5514 | LR: 3.63e-05 | 4005.4ms/step | 15.0 samples/s
+Step 41840/50462 | Loss: 2.5439 | LR: 3.62e-05 | 4003.0ms/step | 15.0 samples/s
+Step 41850/50462 | Loss: 2.5347 | LR: 3.61e-05 | 4015.6ms/step | 14.9 samples/s
+Step 41860/50462 | Loss: 2.5376 | LR: 3.60e-05 | 4010.9ms/step | 15.0 samples/s
+Step 41870/50462 | Loss: 2.5340 | LR: 3.60e-05 | 4007.8ms/step | 15.0 samples/s
+Step 41880/50462 | Loss: 2.5312 | LR: 3.59e-05 | 4014.3ms/step | 14.9 samples/s
+Step 41890/50462 | Loss: 2.5281 | LR: 3.58e-05 | 4012.0ms/step | 15.0 samples/s
+Step 41900/50462 | Loss: 2.4941 | LR: 3.57e-05 | 4003.7ms/step | 15.0 samples/s
+Step 41910/50462 | Loss: 2.5519 | LR: 3.56e-05 | 4010.2ms/step | 15.0 samples/s
+Step 41920/50462 | Loss: 2.5021 | LR: 3.55e-05 | 4012.3ms/step | 15.0 samples/s
+Step 41930/50462 | Loss: 2.5074 | LR: 3.55e-05 | 4009.7ms/step | 15.0 samples/s
+Step 41940/50462 | Loss: 2.5100 | LR: 3.54e-05 | 4008.2ms/step | 15.0 samples/s
+Step 41950/50462 | Loss: 2.5295 | LR: 3.53e-05 | 4006.8ms/step | 15.0 samples/s
+Step 41960/50462 | Loss: 2.4884 | LR: 3.52e-05 | 4016.7ms/step | 14.9 samples/s
+Step 41970/50462 | Loss: 2.5388 | LR: 3.51e-05 | 3997.9ms/step | 15.0 samples/s
+Step 41980/50462 | Loss: 2.4969 | LR: 3.51e-05 | 4018.1ms/step | 14.9 samples/s
+Step 41990/50462 | Loss: 2.5505 | LR: 3.50e-05 | 4016.9ms/step | 14.9 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu_resume41750.log

@@ -0,0 +1,79 @@
+===== RESUME FROM CHECKPOINT-41750 ON 4GPU 2026-08-30T15:20:30+00:00 =====
+W0830 15:20:40.657000 143 site-packages/torch/distributed/run.py:851] 
+W0830 15:20:40.657000 143 site-packages/torch/distributed/run.py:851] *****************************************
+W0830 15:20:40.657000 143 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0830 15:20:40.657000 143 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: TrueCheckpoint activations: True
+
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750
+Checkpoint activations: True
+Checkpoint activations: True

epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu_resume44500_fixed.log

@@ -0,0 +1,295 @@
+===== RESUME FROM CHECKPOINT-44500 ON 4GPU, DDP CURSOR FIX 2026-08-30T19:12:57+00:00 =====
+W0830 19:13:07.139000 134 site-packages/torch/distributed/run.py:851] 
+W0830 19:13:07.139000 134 site-packages/torch/distributed/run.py:851] *****************************************
+W0830 19:13:07.139000 134 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0830 19:13:07.139000 134 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-44500",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "resume_micro_step_override": 667500,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: TrueCheckpoint activations: True
+
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-44500
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-44500...
+Preserving checkpoint LR state for exact resume: current=[1.753587702534784e-05], schedule_base=[0.0005]
+Resumed from checkpoint at step 44500 (epoch 0, micro_step 445000)
+Overriding resume micro-step 445000 -> 667500 for DDP migration
+Resuming training from step 44500
+Starting training...
+Validation at step 44500: Loss = 2.5133, Perplexity = 12.35
+Switching attention mode for step 44501: sparse_stochastic (configured sparse attention)
+Step 44510/50462 | Loss: 2.5427 | LR: 1.75e-05 | 97826.4ms/step | 0.6 samples/s
+Step 44520/50462 | Loss: 2.5066 | LR: 1.74e-05 | 3993.0ms/step | 15.0 samples/s
+Step 44530/50462 | Loss: 2.5160 | LR: 1.74e-05 | 4008.5ms/step | 15.0 samples/s
+Step 44540/50462 | Loss: 2.5259 | LR: 1.73e-05 | 4009.0ms/step | 15.0 samples/s
+Step 44550/50462 | Loss: 2.5739 | LR: 1.72e-05 | 4004.3ms/step | 15.0 samples/s
+Step 44560/50462 | Loss: 2.5245 | LR: 1.72e-05 | 4004.0ms/step | 15.0 samples/s
+Step 44570/50462 | Loss: 2.5113 | LR: 1.71e-05 | 4011.6ms/step | 15.0 samples/s
+Step 44580/50462 | Loss: 2.5597 | LR: 1.71e-05 | 3990.0ms/step | 15.0 samples/s
+Step 44590/50462 | Loss: 2.5493 | LR: 1.70e-05 | 4016.2ms/step | 14.9 samples/s
+Step 44600/50462 | Loss: 2.5169 | LR: 1.70e-05 | 4000.1ms/step | 15.0 samples/s
+Step 44610/50462 | Loss: 2.5440 | LR: 1.69e-05 | 4006.1ms/step | 15.0 samples/s
+Step 44620/50462 | Loss: 2.4826 | LR: 1.68e-05 | 4000.4ms/step | 15.0 samples/s
+Step 44630/50462 | Loss: 2.4975 | LR: 1.68e-05 | 3993.0ms/step | 15.0 samples/s
+Step 44640/50462 | Loss: 2.5329 | LR: 1.67e-05 | 3987.7ms/step | 15.0 samples/s
+Step 44650/50462 | Loss: 2.5176 | LR: 1.67e-05 | 3993.2ms/step | 15.0 samples/s
+Step 44660/50462 | Loss: 2.5070 | LR: 1.66e-05 | 3995.1ms/step | 15.0 samples/s
+Step 44670/50462 | Loss: 2.5389 | LR: 1.66e-05 | 3980.0ms/step | 15.1 samples/s
+Step 44680/50462 | Loss: 2.5110 | LR: 1.65e-05 | 3986.7ms/step | 15.1 samples/s
+Step 44690/50462 | Loss: 2.5318 | LR: 1.64e-05 | 4002.3ms/step | 15.0 samples/s
+Step 44700/50462 | Loss: 2.4748 | LR: 1.64e-05 | 4005.7ms/step | 15.0 samples/s
+Step 44710/50462 | Loss: 2.5290 | LR: 1.63e-05 | 4004.8ms/step | 15.0 samples/s
+Step 44720/50462 | Loss: 2.5403 | LR: 1.63e-05 | 3985.3ms/step | 15.1 samples/s
+Step 44730/50462 | Loss: 2.5151 | LR: 1.62e-05 | 3998.7ms/step | 15.0 samples/s
+Step 44740/50462 | Loss: 2.4857 | LR: 1.62e-05 | 4005.8ms/step | 15.0 samples/s
+Step 44750/50462 | Loss: 2.5483 | LR: 1.61e-05 | 3987.3ms/step | 15.0 samples/s
+Validation at step 44750: Loss = 2.5136, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-44750
+Step 44760/50462 | Loss: 2.5358 | LR: 1.61e-05 | 6012.0ms/step | 10.0 samples/s
+Step 44770/50462 | Loss: 2.5280 | LR: 1.60e-05 | 3997.1ms/step | 15.0 samples/s
+Step 44780/50462 | Loss: 2.5248 | LR: 1.59e-05 | 4003.4ms/step | 15.0 samples/s
+Step 44790/50462 | Loss: 2.5002 | LR: 1.59e-05 | 3999.9ms/step | 15.0 samples/s
+Step 44800/50462 | Loss: 2.5317 | LR: 1.58e-05 | 4002.9ms/step | 15.0 samples/s
+Step 44810/50462 | Loss: 2.5399 | LR: 1.58e-05 | 4004.6ms/step | 15.0 samples/s
+Step 44820/50462 | Loss: 2.5348 | LR: 1.57e-05 | 3993.9ms/step | 15.0 samples/s
+Step 44830/50462 | Loss: 2.5566 | LR: 1.57e-05 | 4010.1ms/step | 15.0 samples/s
+Step 44840/50462 | Loss: 2.5517 | LR: 1.56e-05 | 3994.8ms/step | 15.0 samples/s
+Step 44850/50462 | Loss: 2.4796 | LR: 1.56e-05 | 3996.4ms/step | 15.0 samples/s
+Step 44860/50462 | Loss: 2.5489 | LR: 1.55e-05 | 4014.5ms/step | 14.9 samples/s
+Step 44870/50462 | Loss: 2.5290 | LR: 1.54e-05 | 4000.0ms/step | 15.0 samples/s
+Step 44880/50462 | Loss: 2.5385 | LR: 1.54e-05 | 4007.2ms/step | 15.0 samples/s
+Step 44890/50462 | Loss: 2.5322 | LR: 1.53e-05 | 4009.2ms/step | 15.0 samples/s
+Step 44900/50462 | Loss: 2.5348 | LR: 1.53e-05 | 4005.0ms/step | 15.0 samples/s
+Step 44910/50462 | Loss: 2.5024 | LR: 1.52e-05 | 3999.8ms/step | 15.0 samples/s
+Step 44920/50462 | Loss: 2.5525 | LR: 1.52e-05 | 3985.6ms/step | 15.1 samples/s
+Step 44930/50462 | Loss: 2.4924 | LR: 1.51e-05 | 3996.1ms/step | 15.0 samples/s
+Step 44940/50462 | Loss: 2.5054 | LR: 1.51e-05 | 3990.4ms/step | 15.0 samples/s
+Step 44950/50462 | Loss: 2.5189 | LR: 1.50e-05 | 3999.9ms/step | 15.0 samples/s
+Step 44960/50462 | Loss: 2.5249 | LR: 1.50e-05 | 4001.2ms/step | 15.0 samples/s
+Step 44970/50462 | Loss: 2.5190 | LR: 1.49e-05 | 4007.7ms/step | 15.0 samples/s
+Step 44980/50462 | Loss: 2.4942 | LR: 1.49e-05 | 4004.1ms/step | 15.0 samples/s
+Step 44990/50462 | Loss: 2.4747 | LR: 1.48e-05 | 3996.6ms/step | 15.0 samples/s
+Step 45000/50462 | Loss: 2.5174 | LR: 1.47e-05 | 3992.0ms/step | 15.0 samples/s
+Validation at step 45000: Loss = 2.5114, Perplexity = 12.32
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-45000
+Step 45010/50462 | Loss: 2.4860 | LR: 1.47e-05 | 5941.4ms/step | 10.1 samples/s
+Step 45020/50462 | Loss: 2.5232 | LR: 1.46e-05 | 3988.0ms/step | 15.0 samples/s
+Step 45030/50462 | Loss: 2.5301 | LR: 1.46e-05 | 4006.7ms/step | 15.0 samples/s
+Step 45040/50462 | Loss: 2.5204 | LR: 1.45e-05 | 3996.8ms/step | 15.0 samples/s
+Step 45050/50462 | Loss: 2.5135 | LR: 1.45e-05 | 4007.3ms/step | 15.0 samples/s
+Step 45060/50462 | Loss: 2.5465 | LR: 1.44e-05 | 4005.1ms/step | 15.0 samples/s
+Step 45070/50462 | Loss: 2.5414 | LR: 1.44e-05 | 3987.4ms/step | 15.0 samples/s
+Step 45080/50462 | Loss: 2.5172 | LR: 1.43e-05 | 3997.2ms/step | 15.0 samples/s
+Step 45090/50462 | Loss: 2.5415 | LR: 1.43e-05 | 3992.1ms/step | 15.0 samples/s
+Step 45100/50462 | Loss: 2.4984 | LR: 1.42e-05 | 3991.6ms/step | 15.0 samples/s
+Step 45110/50462 | Loss: 2.5473 | LR: 1.42e-05 | 3995.4ms/step | 15.0 samples/s
+Step 45120/50462 | Loss: 2.5422 | LR: 1.41e-05 | 3993.9ms/step | 15.0 samples/s
+Step 45130/50462 | Loss: 2.5251 | LR: 1.41e-05 | 3996.2ms/step | 15.0 samples/s
+Step 45140/50462 | Loss: 2.5249 | LR: 1.40e-05 | 4000.1ms/step | 15.0 samples/s
+Step 45150/50462 | Loss: 2.5308 | LR: 1.40e-05 | 4005.2ms/step | 15.0 samples/s
+Step 45160/50462 | Loss: 2.5310 | LR: 1.39e-05 | 3999.1ms/step | 15.0 samples/s
+Step 45170/50462 | Loss: 2.5388 | LR: 1.39e-05 | 3986.6ms/step | 15.1 samples/s
+Step 45180/50462 | Loss: 2.5236 | LR: 1.38e-05 | 3989.9ms/step | 15.0 samples/s
+Step 45190/50462 | Loss: 2.5019 | LR: 1.37e-05 | 3995.8ms/step | 15.0 samples/s
+Step 45200/50462 | Loss: 2.5266 | LR: 1.37e-05 | 3997.2ms/step | 15.0 samples/s
+Step 45210/50462 | Loss: 2.5140 | LR: 1.36e-05 | 3990.3ms/step | 15.0 samples/s
+Step 45220/50462 | Loss: 2.5275 | LR: 1.36e-05 | 3999.0ms/step | 15.0 samples/s
+Step 45230/50462 | Loss: 2.5180 | LR: 1.35e-05 | 4005.0ms/step | 15.0 samples/s
+Step 45240/50462 | Loss: 2.5187 | LR: 1.35e-05 | 3992.9ms/step | 15.0 samples/s
+Step 45250/50462 | Loss: 2.5337 | LR: 1.34e-05 | 4006.4ms/step | 15.0 samples/s
+Validation at step 45250: Loss = 2.5126, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-45250
+Step 45260/50462 | Loss: 2.5346 | LR: 1.34e-05 | 5940.0ms/step | 10.1 samples/s
+Step 45270/50462 | Loss: 2.5370 | LR: 1.33e-05 | 3996.3ms/step | 15.0 samples/s
+Step 45280/50462 | Loss: 2.4974 | LR: 1.33e-05 | 3990.9ms/step | 15.0 samples/s
+Step 45290/50462 | Loss: 2.5217 | LR: 1.32e-05 | 3993.0ms/step | 15.0 samples/s
+Step 45300/50462 | Loss: 2.5605 | LR: 1.32e-05 | 3996.3ms/step | 15.0 samples/s
+Step 45310/50462 | Loss: 2.5515 | LR: 1.31e-05 | 3996.5ms/step | 15.0 samples/s
+Step 45320/50462 | Loss: 2.5251 | LR: 1.31e-05 | 4003.7ms/step | 15.0 samples/s
+Step 45330/50462 | Loss: 2.5269 | LR: 1.30e-05 | 3997.1ms/step | 15.0 samples/s
+Step 45340/50462 | Loss: 2.5303 | LR: 1.30e-05 | 3993.1ms/step | 15.0 samples/s
+Step 45350/50462 | Loss: 2.4866 | LR: 1.29e-05 | 4004.3ms/step | 15.0 samples/s
+Step 45360/50462 | Loss: 2.5355 | LR: 1.29e-05 | 4006.4ms/step | 15.0 samples/s
+Step 45370/50462 | Loss: 2.5409 | LR: 1.28e-05 | 4004.0ms/step | 15.0 samples/s
+Step 45380/50462 | Loss: 2.5192 | LR: 1.28e-05 | 3988.9ms/step | 15.0 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu_resume45250.log

@@ -0,0 +1,763 @@
+===== RESUME FROM CHECKPOINT-45250 ON 4GPU 2026-08-31T00:49:34+00:00 =====
+W0831 00:49:36.652000 208 site-packages/torch/distributed/run.py:851] 
+W0831 00:49:36.652000 208 site-packages/torch/distributed/run.py:851] *****************************************
+W0831 00:49:36.652000 208 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0831 00:49:36.652000 208 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-45250",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "resume_micro_step_override": null,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-45250
+Checkpoint activations: TrueCheckpoint activations: True
+Checkpoint activations: TrueCheckpoint activations: True
+
+
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-45250...
+Preserving checkpoint LR state for exact resume: current=[1.3438976469445447e-05], schedule_base=[0.0005]
+Resumed from checkpoint at step 45250 (epoch 0, micro_step 678750)
+Resuming training from step 45250
+Starting training...
+Validation at step 45250: Loss = 2.5123, Perplexity = 12.33
+Switching attention mode for step 45251: sparse_stochastic (configured sparse attention)
+Step 45260/50462 | Loss: 2.5368 | LR: 1.34e-05 | 101699.2ms/step | 0.6 samples/s
+Step 45270/50462 | Loss: 2.5359 | LR: 1.33e-05 | 4025.6ms/step | 14.9 samples/s
+Step 45280/50462 | Loss: 2.4981 | LR: 1.33e-05 | 4011.8ms/step | 15.0 samples/s
+Step 45290/50462 | Loss: 2.5205 | LR: 1.32e-05 | 4024.3ms/step | 14.9 samples/s
+Step 45300/50462 | Loss: 2.5596 | LR: 1.32e-05 | 4030.7ms/step | 14.9 samples/s
+Step 45310/50462 | Loss: 2.5513 | LR: 1.31e-05 | 4024.7ms/step | 14.9 samples/s
+Step 45320/50462 | Loss: 2.5258 | LR: 1.31e-05 | 4019.5ms/step | 14.9 samples/s
+Step 45330/50462 | Loss: 2.5262 | LR: 1.30e-05 | 4025.4ms/step | 14.9 samples/s
+Step 45340/50462 | Loss: 2.5318 | LR: 1.30e-05 | 4009.4ms/step | 15.0 samples/s
+Step 45350/50462 | Loss: 2.4845 | LR: 1.29e-05 | 4027.7ms/step | 14.9 samples/s
+Step 45360/50462 | Loss: 2.5359 | LR: 1.29e-05 | 3997.1ms/step | 15.0 samples/s
+Step 45370/50462 | Loss: 2.5414 | LR: 1.28e-05 | 4012.9ms/step | 15.0 samples/s
+Step 45380/50462 | Loss: 2.5206 | LR: 1.28e-05 | 4016.8ms/step | 14.9 samples/s
+Step 45390/50462 | Loss: 2.5585 | LR: 1.27e-05 | 4025.6ms/step | 14.9 samples/s
+Step 45400/50462 | Loss: 2.4893 | LR: 1.27e-05 | 4019.0ms/step | 14.9 samples/s
+Step 45410/50462 | Loss: 2.5299 | LR: 1.26e-05 | 4024.4ms/step | 14.9 samples/s
+Step 45420/50462 | Loss: 2.4974 | LR: 1.26e-05 | 4010.1ms/step | 15.0 samples/s
+Step 45430/50462 | Loss: 2.5219 | LR: 1.25e-05 | 4020.5ms/step | 14.9 samples/s
+Step 45440/50462 | Loss: 2.5310 | LR: 1.25e-05 | 4011.2ms/step | 15.0 samples/s
+Step 45450/50462 | Loss: 2.5160 | LR: 1.24e-05 | 4021.3ms/step | 14.9 samples/s
+Step 45460/50462 | Loss: 2.5319 | LR: 1.24e-05 | 4024.2ms/step | 14.9 samples/s
+Step 45470/50462 | Loss: 2.5174 | LR: 1.23e-05 | 4014.4ms/step | 14.9 samples/s
+Step 45480/50462 | Loss: 2.5672 | LR: 1.23e-05 | 4004.3ms/step | 15.0 samples/s
+Step 45490/50462 | Loss: 2.5189 | LR: 1.22e-05 | 4011.1ms/step | 15.0 samples/s
+Step 45500/50462 | Loss: 2.5297 | LR: 1.22e-05 | 4009.9ms/step | 15.0 samples/s
+Validation at step 45500: Loss = 2.5123, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-45500
+Step 45510/50462 | Loss: 2.5202 | LR: 1.21e-05 | 5996.6ms/step | 10.0 samples/s
+Step 45520/50462 | Loss: 2.5182 | LR: 1.21e-05 | 4016.7ms/step | 14.9 samples/s
+Step 45530/50462 | Loss: 2.5375 | LR: 1.20e-05 | 4012.6ms/step | 15.0 samples/s
+Step 45540/50462 | Loss: 2.5255 | LR: 1.20e-05 | 4021.4ms/step | 14.9 samples/s
+Step 45550/50462 | Loss: 2.5122 | LR: 1.19e-05 | 4011.9ms/step | 15.0 samples/s
+Step 45560/50462 | Loss: 2.4956 | LR: 1.19e-05 | 4010.4ms/step | 15.0 samples/s
+Step 45570/50462 | Loss: 2.5218 | LR: 1.19e-05 | 4014.8ms/step | 14.9 samples/s
+Step 45580/50462 | Loss: 2.5062 | LR: 1.18e-05 | 3994.2ms/step | 15.0 samples/s
+Step 45590/50462 | Loss: 2.5698 | LR: 1.18e-05 | 4013.5ms/step | 14.9 samples/s
+Step 45600/50462 | Loss: 2.5142 | LR: 1.17e-05 | 4009.3ms/step | 15.0 samples/s
+Step 45610/50462 | Loss: 2.5318 | LR: 1.17e-05 | 4018.1ms/step | 14.9 samples/s
+Step 45620/50462 | Loss: 2.5070 | LR: 1.16e-05 | 4002.1ms/step | 15.0 samples/s
+Step 45630/50462 | Loss: 2.5207 | LR: 1.16e-05 | 4016.5ms/step | 14.9 samples/s
+Step 45640/50462 | Loss: 2.5103 | LR: 1.15e-05 | 4002.6ms/step | 15.0 samples/s
+Step 45650/50462 | Loss: 2.5143 | LR: 1.15e-05 | 4013.0ms/step | 15.0 samples/s
+Step 45660/50462 | Loss: 2.5294 | LR: 1.14e-05 | 4020.0ms/step | 14.9 samples/s
+Step 45670/50462 | Loss: 2.5381 | LR: 1.14e-05 | 4003.1ms/step | 15.0 samples/s
+Step 45680/50462 | Loss: 2.5405 | LR: 1.13e-05 | 4006.3ms/step | 15.0 samples/s
+Step 45690/50462 | Loss: 2.5180 | LR: 1.13e-05 | 3998.2ms/step | 15.0 samples/s
+Step 45700/50462 | Loss: 2.5020 | LR: 1.12e-05 | 4005.6ms/step | 15.0 samples/s
+Step 45710/50462 | Loss: 2.5563 | LR: 1.12e-05 | 4000.2ms/step | 15.0 samples/s
+Step 45720/50462 | Loss: 2.5394 | LR: 1.11e-05 | 4006.0ms/step | 15.0 samples/s
+Step 45730/50462 | Loss: 2.5201 | LR: 1.11e-05 | 4016.9ms/step | 14.9 samples/s
+Step 45740/50462 | Loss: 2.5099 | LR: 1.10e-05 | 4007.4ms/step | 15.0 samples/s
+Step 45750/50462 | Loss: 2.5036 | LR: 1.10e-05 | 4008.9ms/step | 15.0 samples/s
+Validation at step 45750: Loss = 2.5118, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-45750
+Step 45760/50462 | Loss: 2.5242 | LR: 1.10e-05 | 5901.3ms/step | 10.2 samples/s
+Step 45770/50462 | Loss: 2.4949 | LR: 1.09e-05 | 4013.1ms/step | 15.0 samples/s
+Step 45780/50462 | Loss: 2.5236 | LR: 1.09e-05 | 4026.1ms/step | 14.9 samples/s
+Step 45790/50462 | Loss: 2.5004 | LR: 1.08e-05 | 4008.6ms/step | 15.0 samples/s
+Step 45800/50462 | Loss: 2.5344 | LR: 1.08e-05 | 4022.9ms/step | 14.9 samples/s
+Step 45810/50462 | Loss: 2.5581 | LR: 1.07e-05 | 4009.7ms/step | 15.0 samples/s
+Step 45820/50462 | Loss: 2.5464 | LR: 1.07e-05 | 4006.7ms/step | 15.0 samples/s
+Step 45830/50462 | Loss: 2.5372 | LR: 1.06e-05 | 4005.4ms/step | 15.0 samples/s
+Step 45840/50462 | Loss: 2.5349 | LR: 1.06e-05 | 4011.5ms/step | 15.0 samples/s
+Step 45850/50462 | Loss: 2.4884 | LR: 1.05e-05 | 4008.2ms/step | 15.0 samples/s
+Step 45860/50462 | Loss: 2.5203 | LR: 1.05e-05 | 4013.9ms/step | 14.9 samples/s
+Step 45870/50462 | Loss: 2.5179 | LR: 1.05e-05 | 4003.9ms/step | 15.0 samples/s
+Step 45880/50462 | Loss: 2.5325 | LR: 1.04e-05 | 4004.3ms/step | 15.0 samples/s
+Step 45890/50462 | Loss: 2.5105 | LR: 1.04e-05 | 4009.0ms/step | 15.0 samples/s
+Step 45900/50462 | Loss: 2.5273 | LR: 1.03e-05 | 4015.0ms/step | 14.9 samples/s
+Step 45910/50462 | Loss: 2.5172 | LR: 1.03e-05 | 4015.5ms/step | 14.9 samples/s
+Step 45920/50462 | Loss: 2.5540 | LR: 1.02e-05 | 4014.0ms/step | 14.9 samples/s
+Step 45930/50462 | Loss: 2.5379 | LR: 1.02e-05 | 4015.4ms/step | 14.9 samples/s
+Step 45940/50462 | Loss: 2.5450 | LR: 1.01e-05 | 4000.2ms/step | 15.0 samples/s
+Step 45950/50462 | Loss: 2.5303 | LR: 1.01e-05 | 4010.3ms/step | 15.0 samples/s
+Step 45960/50462 | Loss: 2.5260 | LR: 1.01e-05 | 4011.2ms/step | 15.0 samples/s
+Step 45970/50462 | Loss: 2.5312 | LR: 1.00e-05 | 4000.2ms/step | 15.0 samples/s
+Step 45980/50462 | Loss: 2.5251 | LR: 9.96e-06 | 4011.0ms/step | 15.0 samples/s
+Step 45990/50462 | Loss: 2.5455 | LR: 9.92e-06 | 4000.8ms/step | 15.0 samples/s
+Step 46000/50462 | Loss: 2.5508 | LR: 9.87e-06 | 4000.3ms/step | 15.0 samples/s
+Validation at step 46000: Loss = 2.5132, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-46000
+Step 46010/50462 | Loss: 2.5278 | LR: 9.83e-06 | 5933.4ms/step | 10.1 samples/s
+Step 46020/50462 | Loss: 2.5029 | LR: 9.79e-06 | 4003.7ms/step | 15.0 samples/s
+Step 46030/50462 | Loss: 2.5030 | LR: 9.74e-06 | 3996.1ms/step | 15.0 samples/s
+Step 46040/50462 | Loss: 2.5045 | LR: 9.70e-06 | 4016.1ms/step | 14.9 samples/s
+Step 46050/50462 | Loss: 2.5077 | LR: 9.65e-06 | 4000.4ms/step | 15.0 samples/s
+Step 46060/50462 | Loss: 2.5410 | LR: 9.61e-06 | 3991.2ms/step | 15.0 samples/s
+Step 46070/50462 | Loss: 2.4809 | LR: 9.57e-06 | 3988.9ms/step | 15.0 samples/s
+Step 46080/50462 | Loss: 2.5241 | LR: 9.52e-06 | 3998.9ms/step | 15.0 samples/s
+Step 46090/50462 | Loss: 2.5302 | LR: 9.48e-06 | 3998.9ms/step | 15.0 samples/s
+Step 46100/50462 | Loss: 2.5446 | LR: 9.44e-06 | 3996.8ms/step | 15.0 samples/s
+Step 46110/50462 | Loss: 2.4847 | LR: 9.40e-06 | 4002.9ms/step | 15.0 samples/s
+Step 46120/50462 | Loss: 2.5110 | LR: 9.35e-06 | 3994.9ms/step | 15.0 samples/s
+Step 46130/50462 | Loss: 2.5144 | LR: 9.31e-06 | 4007.8ms/step | 15.0 samples/s
+Step 46140/50462 | Loss: 2.4898 | LR: 9.27e-06 | 4012.4ms/step | 15.0 samples/s
+Step 46150/50462 | Loss: 2.4866 | LR: 9.22e-06 | 4007.1ms/step | 15.0 samples/s
+Step 46160/50462 | Loss: 2.5484 | LR: 9.18e-06 | 4000.3ms/step | 15.0 samples/s
+Step 46170/50462 | Loss: 2.5189 | LR: 9.14e-06 | 4008.9ms/step | 15.0 samples/s
+Step 46180/50462 | Loss: 2.5228 | LR: 9.10e-06 | 3983.9ms/step | 15.1 samples/s
+Step 46190/50462 | Loss: 2.5407 | LR: 9.06e-06 | 3990.4ms/step | 15.0 samples/s
+Step 46200/50462 | Loss: 2.5730 | LR: 9.01e-06 | 3988.1ms/step | 15.0 samples/s
+Step 46210/50462 | Loss: 2.5385 | LR: 8.97e-06 | 3998.4ms/step | 15.0 samples/s
+Step 46220/50462 | Loss: 2.5310 | LR: 8.93e-06 | 4003.9ms/step | 15.0 samples/s
+Step 46230/50462 | Loss: 2.5155 | LR: 8.89e-06 | 4001.7ms/step | 15.0 samples/s
+Step 46240/50462 | Loss: 2.5544 | LR: 8.85e-06 | 3992.7ms/step | 15.0 samples/s
+Step 46250/50462 | Loss: 2.5115 | LR: 8.80e-06 | 4002.9ms/step | 15.0 samples/s
+Validation at step 46250: Loss = 2.5128, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-46250
+Step 46260/50462 | Loss: 2.5251 | LR: 8.76e-06 | 5946.6ms/step | 10.1 samples/s
+Step 46270/50462 | Loss: 2.5531 | LR: 8.72e-06 | 3994.1ms/step | 15.0 samples/s
+Step 46280/50462 | Loss: 2.5307 | LR: 8.68e-06 | 4005.8ms/step | 15.0 samples/s
+Step 46290/50462 | Loss: 2.4974 | LR: 8.64e-06 | 4007.5ms/step | 15.0 samples/s
+Step 46300/50462 | Loss: 2.5009 | LR: 8.60e-06 | 3987.8ms/step | 15.0 samples/s
+Step 46310/50462 | Loss: 2.4842 | LR: 8.56e-06 | 3994.8ms/step | 15.0 samples/s
+Step 46320/50462 | Loss: 2.5273 | LR: 8.52e-06 | 3994.7ms/step | 15.0 samples/s
+Step 46330/50462 | Loss: 2.5298 | LR: 8.47e-06 | 3991.6ms/step | 15.0 samples/s
+Step 46340/50462 | Loss: 2.5100 | LR: 8.43e-06 | 3997.4ms/step | 15.0 samples/s
+Step 46350/50462 | Loss: 2.5211 | LR: 8.39e-06 | 3991.9ms/step | 15.0 samples/s
+Step 46360/50462 | Loss: 2.5574 | LR: 8.35e-06 | 3984.2ms/step | 15.1 samples/s
+Step 46370/50462 | Loss: 2.5282 | LR: 8.31e-06 | 4004.8ms/step | 15.0 samples/s
+Step 46380/50462 | Loss: 2.5359 | LR: 8.27e-06 | 4010.6ms/step | 15.0 samples/s
+Step 46390/50462 | Loss: 2.5364 | LR: 8.23e-06 | 3999.2ms/step | 15.0 samples/s
+Step 46400/50462 | Loss: 2.5585 | LR: 8.19e-06 | 3995.6ms/step | 15.0 samples/s
+Step 46410/50462 | Loss: 2.5640 | LR: 8.15e-06 | 4002.0ms/step | 15.0 samples/s
+Step 46420/50462 | Loss: 2.5306 | LR: 8.11e-06 | 3989.4ms/step | 15.0 samples/s
+Step 46430/50462 | Loss: 2.5326 | LR: 8.07e-06 | 4002.4ms/step | 15.0 samples/s
+Step 46440/50462 | Loss: 2.5387 | LR: 8.03e-06 | 4002.4ms/step | 15.0 samples/s
+Step 46450/50462 | Loss: 2.5558 | LR: 7.99e-06 | 4001.4ms/step | 15.0 samples/s
+Step 46460/50462 | Loss: 2.5572 | LR: 7.95e-06 | 3994.5ms/step | 15.0 samples/s
+Step 46470/50462 | Loss: 2.5727 | LR: 7.91e-06 | 3998.3ms/step | 15.0 samples/s
+Step 46480/50462 | Loss: 2.4877 | LR: 7.87e-06 | 3993.0ms/step | 15.0 samples/s
+Step 46490/50462 | Loss: 2.5340 | LR: 7.83e-06 | 3997.9ms/step | 15.0 samples/s
+Step 46500/50462 | Loss: 2.5122 | LR: 7.80e-06 | 3999.7ms/step | 15.0 samples/s
+Validation at step 46500: Loss = 2.5122, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-46500
+Step 46510/50462 | Loss: 2.5380 | LR: 7.76e-06 | 6083.4ms/step | 9.9 samples/s
+Step 46520/50462 | Loss: 2.4708 | LR: 7.72e-06 | 4003.0ms/step | 15.0 samples/s
+Step 46530/50462 | Loss: 2.5013 | LR: 7.68e-06 | 3998.4ms/step | 15.0 samples/s
+Step 46540/50462 | Loss: 2.5430 | LR: 7.64e-06 | 4003.0ms/step | 15.0 samples/s
+Step 46550/50462 | Loss: 2.5047 | LR: 7.60e-06 | 3988.2ms/step | 15.0 samples/s
+Step 46560/50462 | Loss: 2.5322 | LR: 7.56e-06 | 4009.2ms/step | 15.0 samples/s
+Step 46570/50462 | Loss: 2.5246 | LR: 7.52e-06 | 4002.1ms/step | 15.0 samples/s
+Step 46580/50462 | Loss: 2.5257 | LR: 7.49e-06 | 4000.6ms/step | 15.0 samples/s
+Step 46590/50462 | Loss: 2.5137 | LR: 7.45e-06 | 4004.2ms/step | 15.0 samples/s
+Step 46600/50462 | Loss: 2.5553 | LR: 7.41e-06 | 4003.3ms/step | 15.0 samples/s
+Step 46610/50462 | Loss: 2.5410 | LR: 7.37e-06 | 3987.7ms/step | 15.0 samples/s
+Step 46620/50462 | Loss: 2.5374 | LR: 7.33e-06 | 3993.4ms/step | 15.0 samples/s
+Step 46630/50462 | Loss: 2.5337 | LR: 7.29e-06 | 4001.4ms/step | 15.0 samples/s
+Step 46640/50462 | Loss: 2.5204 | LR: 7.26e-06 | 4574.1ms/step | 13.1 samples/s
+Step 46650/50462 | Loss: 2.5448 | LR: 7.22e-06 | 4009.1ms/step | 15.0 samples/s
+Step 46660/50462 | Loss: 2.5346 | LR: 7.18e-06 | 4003.4ms/step | 15.0 samples/s
+Step 46670/50462 | Loss: 2.5003 | LR: 7.14e-06 | 4008.5ms/step | 15.0 samples/s
+Step 46680/50462 | Loss: 2.5138 | LR: 7.11e-06 | 4006.0ms/step | 15.0 samples/s
+Step 46690/50462 | Loss: 2.5009 | LR: 7.07e-06 | 4011.8ms/step | 15.0 samples/s
+Step 46700/50462 | Loss: 2.5195 | LR: 7.03e-06 | 4018.2ms/step | 14.9 samples/s
+Step 46710/50462 | Loss: 2.5356 | LR: 6.99e-06 | 4007.7ms/step | 15.0 samples/s
+Step 46720/50462 | Loss: 2.5264 | LR: 6.96e-06 | 3992.8ms/step | 15.0 samples/s
+Step 46730/50462 | Loss: 2.5032 | LR: 6.92e-06 | 4002.1ms/step | 15.0 samples/s
+Step 46740/50462 | Loss: 2.5106 | LR: 6.88e-06 | 3995.6ms/step | 15.0 samples/s
+Step 46750/50462 | Loss: 2.5438 | LR: 6.85e-06 | 3992.8ms/step | 15.0 samples/s
+Validation at step 46750: Loss = 2.5125, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-46750
+Step 46760/50462 | Loss: 2.5145 | LR: 6.81e-06 | 5954.3ms/step | 10.1 samples/s
+Step 46770/50462 | Loss: 2.5129 | LR: 6.77e-06 | 4008.1ms/step | 15.0 samples/s
+Step 46780/50462 | Loss: 2.5372 | LR: 6.74e-06 | 4708.5ms/step | 12.7 samples/s
+Step 46790/50462 | Loss: 2.5017 | LR: 6.70e-06 | 4631.1ms/step | 13.0 samples/s
+Step 46800/50462 | Loss: 2.5298 | LR: 6.66e-06 | 3996.1ms/step | 15.0 samples/s
+Step 46810/50462 | Loss: 2.4904 | LR: 6.63e-06 | 3999.0ms/step | 15.0 samples/s
+Step 46820/50462 | Loss: 2.5282 | LR: 6.59e-06 | 4001.5ms/step | 15.0 samples/s
+Step 46830/50462 | Loss: 2.5507 | LR: 6.56e-06 | 3990.7ms/step | 15.0 samples/s
+Step 46840/50462 | Loss: 2.5404 | LR: 6.52e-06 | 4005.4ms/step | 15.0 samples/s
+Step 46850/50462 | Loss: 2.4975 | LR: 6.48e-06 | 4000.1ms/step | 15.0 samples/s
+Step 46860/50462 | Loss: 2.5182 | LR: 6.45e-06 | 3991.7ms/step | 15.0 samples/s
+Step 46870/50462 | Loss: 2.5124 | LR: 6.41e-06 | 3998.9ms/step | 15.0 samples/s
+Step 46880/50462 | Loss: 2.4942 | LR: 6.38e-06 | 4000.7ms/step | 15.0 samples/s
+Step 46890/50462 | Loss: 2.5230 | LR: 6.34e-06 | 3995.5ms/step | 15.0 samples/s
+Step 46900/50462 | Loss: 2.5307 | LR: 6.31e-06 | 3986.8ms/step | 15.0 samples/s
+Step 46910/50462 | Loss: 2.5287 | LR: 6.27e-06 | 4000.6ms/step | 15.0 samples/s
+Step 46920/50462 | Loss: 2.5250 | LR: 6.24e-06 | 3999.5ms/step | 15.0 samples/s
+Step 46930/50462 | Loss: 2.5134 | LR: 6.20e-06 | 3999.2ms/step | 15.0 samples/s
+Step 46940/50462 | Loss: 2.5161 | LR: 6.17e-06 | 3991.2ms/step | 15.0 samples/s
+Step 46950/50462 | Loss: 2.5417 | LR: 6.13e-06 | 4013.4ms/step | 14.9 samples/s
+Step 46960/50462 | Loss: 2.5209 | LR: 6.10e-06 | 4003.2ms/step | 15.0 samples/s
+Step 46970/50462 | Loss: 2.5438 | LR: 6.06e-06 | 3986.5ms/step | 15.1 samples/s
+Step 46980/50462 | Loss: 2.5620 | LR: 6.03e-06 | 4000.6ms/step | 15.0 samples/s
+Step 46990/50462 | Loss: 2.5244 | LR: 5.99e-06 | 4006.3ms/step | 15.0 samples/s
+Step 47000/50462 | Loss: 2.5387 | LR: 5.96e-06 | 4004.2ms/step | 15.0 samples/s
+Validation at step 47000: Loss = 2.5130, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-47000
+Step 47010/50462 | Loss: 2.5351 | LR: 5.93e-06 | 6028.7ms/step | 10.0 samples/s
+Step 47020/50462 | Loss: 2.5337 | LR: 5.89e-06 | 4004.5ms/step | 15.0 samples/s
+Step 47030/50462 | Loss: 2.5196 | LR: 5.86e-06 | 3991.4ms/step | 15.0 samples/s
+Step 47040/50462 | Loss: 2.5165 | LR: 5.82e-06 | 3996.2ms/step | 15.0 samples/s
+Step 47050/50462 | Loss: 2.5025 | LR: 5.79e-06 | 4000.4ms/step | 15.0 samples/s
+Step 47060/50462 | Loss: 2.5447 | LR: 5.76e-06 | 3994.7ms/step | 15.0 samples/s
+Step 47070/50462 | Loss: 2.5410 | LR: 5.72e-06 | 4004.4ms/step | 15.0 samples/s
+Step 47080/50462 | Loss: 2.5424 | LR: 5.69e-06 | 3998.6ms/step | 15.0 samples/s
+Step 47090/50462 | Loss: 2.5380 | LR: 5.65e-06 | 4006.9ms/step | 15.0 samples/s
+Step 47100/50462 | Loss: 2.5425 | LR: 5.62e-06 | 4007.6ms/step | 15.0 samples/s
+Step 47110/50462 | Loss: 2.5413 | LR: 5.59e-06 | 3996.4ms/step | 15.0 samples/s
+Step 47120/50462 | Loss: 2.4844 | LR: 5.56e-06 | 3998.1ms/step | 15.0 samples/s
+Step 47130/50462 | Loss: 2.4800 | LR: 5.52e-06 | 4007.3ms/step | 15.0 samples/s
+Step 47140/50462 | Loss: 2.5175 | LR: 5.49e-06 | 3999.9ms/step | 15.0 samples/s
+Step 47150/50462 | Loss: 2.5589 | LR: 5.46e-06 | 3988.4ms/step | 15.0 samples/s
+Step 47160/50462 | Loss: 2.5193 | LR: 5.42e-06 | 3998.6ms/step | 15.0 samples/s
+Step 47170/50462 | Loss: 2.4999 | LR: 5.39e-06 | 3988.2ms/step | 15.0 samples/s
+Step 47180/50462 | Loss: 2.5362 | LR: 5.36e-06 | 3993.0ms/step | 15.0 samples/s
+Step 47190/50462 | Loss: 2.5419 | LR: 5.33e-06 | 3999.2ms/step | 15.0 samples/s
+Step 47200/50462 | Loss: 2.5204 | LR: 5.29e-06 | 4001.9ms/step | 15.0 samples/s
+Step 47210/50462 | Loss: 2.4978 | LR: 5.26e-06 | 3999.9ms/step | 15.0 samples/s
+Step 47220/50462 | Loss: 2.4918 | LR: 5.23e-06 | 4020.5ms/step | 14.9 samples/s
+Step 47230/50462 | Loss: 2.5015 | LR: 5.20e-06 | 4017.5ms/step | 14.9 samples/s
+Step 47240/50462 | Loss: 2.5507 | LR: 5.16e-06 | 3997.5ms/step | 15.0 samples/s
+Step 47250/50462 | Loss: 2.5496 | LR: 5.13e-06 | 4006.9ms/step | 15.0 samples/s
+Validation at step 47250: Loss = 2.5132, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-47250
+Step 47260/50462 | Loss: 2.5523 | LR: 5.10e-06 | 5995.1ms/step | 10.0 samples/s
+Step 47270/50462 | Loss: 2.5278 | LR: 5.07e-06 | 4004.4ms/step | 15.0 samples/s
+Step 47280/50462 | Loss: 2.4885 | LR: 5.04e-06 | 4010.9ms/step | 15.0 samples/s
+Step 47290/50462 | Loss: 2.4885 | LR: 5.01e-06 | 3996.1ms/step | 15.0 samples/s
+Step 47300/50462 | Loss: 2.5002 | LR: 4.97e-06 | 3980.0ms/step | 15.1 samples/s
+Step 47310/50462 | Loss: 2.5438 | LR: 4.94e-06 | 4002.8ms/step | 15.0 samples/s
+Step 47320/50462 | Loss: 2.5220 | LR: 4.91e-06 | 4004.4ms/step | 15.0 samples/s
+Step 47330/50462 | Loss: 2.5702 | LR: 4.88e-06 | 4004.0ms/step | 15.0 samples/s
+Step 47340/50462 | Loss: 2.5698 | LR: 4.85e-06 | 4008.0ms/step | 15.0 samples/s
+Step 47350/50462 | Loss: 2.5231 | LR: 4.82e-06 | 3998.8ms/step | 15.0 samples/s
+Step 47360/50462 | Loss: 2.5627 | LR: 4.79e-06 | 3991.5ms/step | 15.0 samples/s
+Step 47370/50462 | Loss: 2.5437 | LR: 4.76e-06 | 4007.6ms/step | 15.0 samples/s
+Step 47380/50462 | Loss: 2.5016 | LR: 4.73e-06 | 4001.9ms/step | 15.0 samples/s
+Step 47390/50462 | Loss: 2.5323 | LR: 4.70e-06 | 4011.3ms/step | 15.0 samples/s
+Step 47400/50462 | Loss: 2.5563 | LR: 4.67e-06 | 4013.6ms/step | 14.9 samples/s
+Step 47410/50462 | Loss: 2.5295 | LR: 4.64e-06 | 4009.7ms/step | 15.0 samples/s
+Step 47420/50462 | Loss: 2.5129 | LR: 4.61e-06 | 4010.0ms/step | 15.0 samples/s
+Step 47430/50462 | Loss: 2.5014 | LR: 4.58e-06 | 3994.8ms/step | 15.0 samples/s
+Step 47440/50462 | Loss: 2.5406 | LR: 4.55e-06 | 4011.1ms/step | 15.0 samples/s
+Step 47450/50462 | Loss: 2.5231 | LR: 4.52e-06 | 4003.6ms/step | 15.0 samples/s
+Step 47460/50462 | Loss: 2.5397 | LR: 4.49e-06 | 4002.8ms/step | 15.0 samples/s
+Step 47470/50462 | Loss: 2.5279 | LR: 4.46e-06 | 4008.5ms/step | 15.0 samples/s
+Step 47480/50462 | Loss: 2.5029 | LR: 4.43e-06 | 4002.5ms/step | 15.0 samples/s
+Step 47490/50462 | Loss: 2.5237 | LR: 4.40e-06 | 4004.0ms/step | 15.0 samples/s
+Step 47500/50462 | Loss: 2.5863 | LR: 4.37e-06 | 4008.4ms/step | 15.0 samples/s
+Validation at step 47500: Loss = 2.5127, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-47500
+Step 47510/50462 | Loss: 2.4824 | LR: 4.34e-06 | 5975.4ms/step | 10.0 samples/s
+Step 47520/50462 | Loss: 2.5174 | LR: 4.31e-06 | 3997.7ms/step | 15.0 samples/s
+Step 47530/50462 | Loss: 2.4965 | LR: 4.28e-06 | 4010.5ms/step | 15.0 samples/s
+Step 47540/50462 | Loss: 2.5283 | LR: 4.25e-06 | 4004.3ms/step | 15.0 samples/s
+Step 47550/50462 | Loss: 2.5063 | LR: 4.22e-06 | 3999.7ms/step | 15.0 samples/s
+Step 47560/50462 | Loss: 2.5555 | LR: 4.19e-06 | 4006.3ms/step | 15.0 samples/s
+Step 47570/50462 | Loss: 2.5353 | LR: 4.16e-06 | 4018.2ms/step | 14.9 samples/s
+Step 47580/50462 | Loss: 2.5487 | LR: 4.13e-06 | 4008.1ms/step | 15.0 samples/s
+Step 47590/50462 | Loss: 2.5727 | LR: 4.11e-06 | 4010.9ms/step | 15.0 samples/s
+Step 47600/50462 | Loss: 2.5249 | LR: 4.08e-06 | 4021.4ms/step | 14.9 samples/s
+Step 47610/50462 | Loss: 2.5468 | LR: 4.05e-06 | 4006.2ms/step | 15.0 samples/s
+Step 47620/50462 | Loss: 2.5273 | LR: 4.02e-06 | 4010.8ms/step | 15.0 samples/s
+Step 47630/50462 | Loss: 2.5288 | LR: 3.99e-06 | 3990.9ms/step | 15.0 samples/s
+Step 47640/50462 | Loss: 2.5308 | LR: 3.97e-06 | 4000.2ms/step | 15.0 samples/s
+Step 47650/50462 | Loss: 2.5401 | LR: 3.94e-06 | 4014.2ms/step | 14.9 samples/s
+Step 47660/50462 | Loss: 2.5250 | LR: 3.91e-06 | 4020.0ms/step | 14.9 samples/s
+Step 47670/50462 | Loss: 2.5381 | LR: 3.88e-06 | 4008.1ms/step | 15.0 samples/s
+Step 47680/50462 | Loss: 2.5457 | LR: 3.85e-06 | 4010.9ms/step | 15.0 samples/s
+Step 47690/50462 | Loss: 2.5307 | LR: 3.83e-06 | 4003.4ms/step | 15.0 samples/s
+Step 47700/50462 | Loss: 2.5451 | LR: 3.80e-06 | 4005.6ms/step | 15.0 samples/s
+Step 47710/50462 | Loss: 2.5331 | LR: 3.77e-06 | 4015.1ms/step | 14.9 samples/s
+Step 47720/50462 | Loss: 2.5642 | LR: 3.74e-06 | 4002.0ms/step | 15.0 samples/s
+Step 47730/50462 | Loss: 2.5052 | LR: 3.72e-06 | 4012.0ms/step | 15.0 samples/s
+Step 47740/50462 | Loss: 2.5014 | LR: 3.69e-06 | 4006.7ms/step | 15.0 samples/s
+Step 47750/50462 | Loss: 2.5385 | LR: 3.66e-06 | 4002.5ms/step | 15.0 samples/s
+Validation at step 47750: Loss = 2.5124, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-47750
+Step 47760/50462 | Loss: 2.5368 | LR: 3.64e-06 | 5944.3ms/step | 10.1 samples/s
+Step 47770/50462 | Loss: 2.5309 | LR: 3.61e-06 | 4004.2ms/step | 15.0 samples/s
+Step 47780/50462 | Loss: 2.5170 | LR: 3.58e-06 | 3998.1ms/step | 15.0 samples/s
+Step 47790/50462 | Loss: 2.5410 | LR: 3.56e-06 | 4016.9ms/step | 14.9 samples/s
+Step 47800/50462 | Loss: 2.5190 | LR: 3.53e-06 | 4008.9ms/step | 15.0 samples/s
+Step 47810/50462 | Loss: 2.5089 | LR: 3.50e-06 | 4001.0ms/step | 15.0 samples/s
+Step 47820/50462 | Loss: 2.5277 | LR: 3.48e-06 | 4002.2ms/step | 15.0 samples/s
+Step 47830/50462 | Loss: 2.5149 | LR: 3.45e-06 | 4005.1ms/step | 15.0 samples/s
+Step 47840/50462 | Loss: 2.5289 | LR: 3.42e-06 | 4000.4ms/step | 15.0 samples/s
+Step 47850/50462 | Loss: 2.5237 | LR: 3.40e-06 | 3994.5ms/step | 15.0 samples/s
+Step 47860/50462 | Loss: 2.5079 | LR: 3.37e-06 | 3998.7ms/step | 15.0 samples/s
+Step 47870/50462 | Loss: 2.5234 | LR: 3.35e-06 | 3996.1ms/step | 15.0 samples/s
+Step 47880/50462 | Loss: 2.5270 | LR: 3.32e-06 | 4003.4ms/step | 15.0 samples/s
+Step 47890/50462 | Loss: 2.4999 | LR: 3.30e-06 | 4011.4ms/step | 15.0 samples/s
+Step 47900/50462 | Loss: 2.5344 | LR: 3.27e-06 | 4000.2ms/step | 15.0 samples/s
+Step 47910/50462 | Loss: 2.5350 | LR: 3.24e-06 | 3996.7ms/step | 15.0 samples/s
+Step 47920/50462 | Loss: 2.5175 | LR: 3.22e-06 | 3999.9ms/step | 15.0 samples/s
+Step 47930/50462 | Loss: 2.4893 | LR: 3.19e-06 | 3996.4ms/step | 15.0 samples/s
+Step 47940/50462 | Loss: 2.5493 | LR: 3.17e-06 | 3999.4ms/step | 15.0 samples/s
+Step 47950/50462 | Loss: 2.5609 | LR: 3.14e-06 | 3999.6ms/step | 15.0 samples/s
+Step 47960/50462 | Loss: 2.5253 | LR: 3.12e-06 | 3988.1ms/step | 15.0 samples/s
+Step 47970/50462 | Loss: 2.5461 | LR: 3.09e-06 | 4000.7ms/step | 15.0 samples/s
+Step 47980/50462 | Loss: 2.5247 | LR: 3.07e-06 | 4004.3ms/step | 15.0 samples/s
+Step 47990/50462 | Loss: 2.5076 | LR: 3.04e-06 | 3991.2ms/step | 15.0 samples/s
+Step 48000/50462 | Loss: 2.5407 | LR: 3.02e-06 | 4003.1ms/step | 15.0 samples/s
+Validation at step 48000: Loss = 2.5126, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-48000
+Step 48010/50462 | Loss: 2.4993 | LR: 3.00e-06 | 5964.9ms/step | 10.1 samples/s
+Step 48020/50462 | Loss: 2.5387 | LR: 2.97e-06 | 4007.5ms/step | 15.0 samples/s
+Step 48030/50462 | Loss: 2.5155 | LR: 2.95e-06 | 3997.7ms/step | 15.0 samples/s
+Step 48040/50462 | Loss: 2.5068 | LR: 2.92e-06 | 4000.8ms/step | 15.0 samples/s
+Step 48050/50462 | Loss: 2.5055 | LR: 2.90e-06 | 4005.9ms/step | 15.0 samples/s
+Step 48060/50462 | Loss: 2.5351 | LR: 2.87e-06 | 4008.0ms/step | 15.0 samples/s
+Step 48070/50462 | Loss: 2.5198 | LR: 2.85e-06 | 3998.6ms/step | 15.0 samples/s
+Step 48080/50462 | Loss: 2.5055 | LR: 2.83e-06 | 4005.1ms/step | 15.0 samples/s
+Step 48090/50462 | Loss: 2.5069 | LR: 2.80e-06 | 3998.2ms/step | 15.0 samples/s
+Step 48100/50462 | Loss: 2.5675 | LR: 2.78e-06 | 4003.2ms/step | 15.0 samples/s
+Step 48110/50462 | Loss: 2.5275 | LR: 2.76e-06 | 4002.3ms/step | 15.0 samples/s
+Step 48120/50462 | Loss: 2.5117 | LR: 2.73e-06 | 4001.2ms/step | 15.0 samples/s
+Step 48130/50462 | Loss: 2.5193 | LR: 2.71e-06 | 3996.5ms/step | 15.0 samples/s
+Step 48140/50462 | Loss: 2.5220 | LR: 2.69e-06 | 3998.9ms/step | 15.0 samples/s
+Step 48150/50462 | Loss: 2.5201 | LR: 2.66e-06 | 4016.2ms/step | 14.9 samples/s
+Step 48160/50462 | Loss: 2.5473 | LR: 2.64e-06 | 4002.8ms/step | 15.0 samples/s
+Step 48170/50462 | Loss: 2.4980 | LR: 2.62e-06 | 4001.3ms/step | 15.0 samples/s
+Step 48180/50462 | Loss: 2.5069 | LR: 2.60e-06 | 3998.1ms/step | 15.0 samples/s
+Step 48190/50462 | Loss: 2.4959 | LR: 2.57e-06 | 4016.4ms/step | 14.9 samples/s
+Step 48200/50462 | Loss: 2.5215 | LR: 2.55e-06 | 3995.2ms/step | 15.0 samples/s
+Step 48210/50462 | Loss: 2.5387 | LR: 2.53e-06 | 4003.5ms/step | 15.0 samples/s
+Step 48220/50462 | Loss: 2.5207 | LR: 2.51e-06 | 4002.6ms/step | 15.0 samples/s
+Step 48230/50462 | Loss: 2.5337 | LR: 2.48e-06 | 3995.1ms/step | 15.0 samples/s
+Step 48240/50462 | Loss: 2.5079 | LR: 2.46e-06 | 3993.5ms/step | 15.0 samples/s
+Step 48250/50462 | Loss: 2.5472 | LR: 2.44e-06 | 3998.8ms/step | 15.0 samples/s
+Validation at step 48250: Loss = 2.5124, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-48250
+Step 48260/50462 | Loss: 2.5223 | LR: 2.42e-06 | 5934.8ms/step | 10.1 samples/s
+Step 48270/50462 | Loss: 2.5049 | LR: 2.39e-06 | 4001.1ms/step | 15.0 samples/s
+Step 48280/50462 | Loss: 2.5123 | LR: 2.37e-06 | 4007.0ms/step | 15.0 samples/s
+Step 48290/50462 | Loss: 2.5174 | LR: 2.35e-06 | 4001.9ms/step | 15.0 samples/s
+Step 48300/50462 | Loss: 2.5331 | LR: 2.33e-06 | 4002.5ms/step | 15.0 samples/s
+Step 48310/50462 | Loss: 2.5251 | LR: 2.31e-06 | 3991.5ms/step | 15.0 samples/s
+Step 48320/50462 | Loss: 2.5274 | LR: 2.29e-06 | 4003.4ms/step | 15.0 samples/s
+Step 48330/50462 | Loss: 2.5474 | LR: 2.27e-06 | 4000.5ms/step | 15.0 samples/s
+Step 48340/50462 | Loss: 2.5404 | LR: 2.24e-06 | 4001.8ms/step | 15.0 samples/s
+Step 48350/50462 | Loss: 2.5426 | LR: 2.22e-06 | 3989.9ms/step | 15.0 samples/s
+Step 48360/50462 | Loss: 2.5190 | LR: 2.20e-06 | 4008.1ms/step | 15.0 samples/s
+Step 48370/50462 | Loss: 2.5255 | LR: 2.18e-06 | 4007.2ms/step | 15.0 samples/s
+Step 48380/50462 | Loss: 2.4985 | LR: 2.16e-06 | 4004.8ms/step | 15.0 samples/s
+Step 48390/50462 | Loss: 2.5549 | LR: 2.14e-06 | 4003.0ms/step | 15.0 samples/s
+Step 48400/50462 | Loss: 2.5403 | LR: 2.12e-06 | 3989.3ms/step | 15.0 samples/s
+Step 48410/50462 | Loss: 2.5282 | LR: 2.10e-06 | 3985.3ms/step | 15.1 samples/s
+Step 48420/50462 | Loss: 2.5113 | LR: 2.08e-06 | 3999.6ms/step | 15.0 samples/s
+Step 48430/50462 | Loss: 2.5322 | LR: 2.06e-06 | 4004.0ms/step | 15.0 samples/s
+Step 48440/50462 | Loss: 2.5752 | LR: 2.04e-06 | 3995.6ms/step | 15.0 samples/s
+Step 48450/50462 | Loss: 2.5069 | LR: 2.02e-06 | 4010.2ms/step | 15.0 samples/s
+Step 48460/50462 | Loss: 2.5351 | LR: 2.00e-06 | 4008.5ms/step | 15.0 samples/s
+Step 48470/50462 | Loss: 2.5160 | LR: 1.98e-06 | 4006.4ms/step | 15.0 samples/s
+Step 48480/50462 | Loss: 2.5524 | LR: 1.96e-06 | 3993.4ms/step | 15.0 samples/s
+Step 48490/50462 | Loss: 2.5433 | LR: 1.94e-06 | 3993.4ms/step | 15.0 samples/s
+Step 48500/50462 | Loss: 2.5015 | LR: 1.92e-06 | 4002.5ms/step | 15.0 samples/s
+Validation at step 48500: Loss = 2.5115, Perplexity = 12.32
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-48500
+Step 48510/50462 | Loss: 2.5224 | LR: 1.90e-06 | 6001.5ms/step | 10.0 samples/s
+Step 48520/50462 | Loss: 2.5237 | LR: 1.88e-06 | 3999.2ms/step | 15.0 samples/s
+Step 48530/50462 | Loss: 2.5149 | LR: 1.86e-06 | 3996.0ms/step | 15.0 samples/s
+Step 48540/50462 | Loss: 2.5610 | LR: 1.84e-06 | 3992.7ms/step | 15.0 samples/s
+Step 48550/50462 | Loss: 2.5211 | LR: 1.82e-06 | 4001.1ms/step | 15.0 samples/s
+Step 48560/50462 | Loss: 2.5464 | LR: 1.80e-06 | 3997.6ms/step | 15.0 samples/s
+Step 48570/50462 | Loss: 2.5199 | LR: 1.78e-06 | 3987.7ms/step | 15.0 samples/s
+Step 48580/50462 | Loss: 2.5222 | LR: 1.77e-06 | 4004.5ms/step | 15.0 samples/s
+Step 48590/50462 | Loss: 2.5220 | LR: 1.75e-06 | 3996.2ms/step | 15.0 samples/s
+Step 48600/50462 | Loss: 2.5240 | LR: 1.73e-06 | 3996.7ms/step | 15.0 samples/s
+Step 48610/50462 | Loss: 2.5330 | LR: 1.71e-06 | 4016.2ms/step | 14.9 samples/s
+Step 48620/50462 | Loss: 2.5127 | LR: 1.69e-06 | 3994.0ms/step | 15.0 samples/s
+Step 48630/50462 | Loss: 2.5327 | LR: 1.67e-06 | 4010.3ms/step | 15.0 samples/s
+Step 48640/50462 | Loss: 2.5192 | LR: 1.66e-06 | 4000.3ms/step | 15.0 samples/s
+Step 48650/50462 | Loss: 2.5419 | LR: 1.64e-06 | 3997.4ms/step | 15.0 samples/s
+Step 48660/50462 | Loss: 2.5292 | LR: 1.62e-06 | 4016.2ms/step | 14.9 samples/s
+Step 48670/50462 | Loss: 2.5366 | LR: 1.60e-06 | 4005.8ms/step | 15.0 samples/s
+Step 48680/50462 | Loss: 2.4884 | LR: 1.58e-06 | 3991.2ms/step | 15.0 samples/s
+Step 48690/50462 | Loss: 2.5398 | LR: 1.57e-06 | 3997.6ms/step | 15.0 samples/s
+Step 48700/50462 | Loss: 2.4958 | LR: 1.55e-06 | 4000.4ms/step | 15.0 samples/s
+Step 48710/50462 | Loss: 2.4942 | LR: 1.53e-06 | 3986.5ms/step | 15.1 samples/s
+Step 48720/50462 | Loss: 2.5129 | LR: 1.51e-06 | 4002.1ms/step | 15.0 samples/s
+Step 48730/50462 | Loss: 2.5524 | LR: 1.50e-06 | 4009.5ms/step | 15.0 samples/s
+Step 48740/50462 | Loss: 2.5374 | LR: 1.48e-06 | 4004.7ms/step | 15.0 samples/s
+Step 48750/50462 | Loss: 2.5286 | LR: 1.46e-06 | 3995.8ms/step | 15.0 samples/s
+Validation at step 48750: Loss = 2.5124, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-48750
+Step 48760/50462 | Loss: 2.5189 | LR: 1.44e-06 | 5951.1ms/step | 10.1 samples/s
+Step 48770/50462 | Loss: 2.5050 | LR: 1.43e-06 | 3994.9ms/step | 15.0 samples/s
+Step 48780/50462 | Loss: 2.5144 | LR: 1.41e-06 | 4002.3ms/step | 15.0 samples/s
+Step 48790/50462 | Loss: 2.5250 | LR: 1.39e-06 | 4000.6ms/step | 15.0 samples/s
+Step 48800/50462 | Loss: 2.5422 | LR: 1.38e-06 | 4005.0ms/step | 15.0 samples/s
+Step 48810/50462 | Loss: 2.5146 | LR: 1.36e-06 | 3988.2ms/step | 15.0 samples/s
+Step 48820/50462 | Loss: 2.5229 | LR: 1.34e-06 | 3989.7ms/step | 15.0 samples/s
+Step 48830/50462 | Loss: 2.5259 | LR: 1.33e-06 | 3996.7ms/step | 15.0 samples/s
+Step 48840/50462 | Loss: 2.5367 | LR: 1.31e-06 | 4004.1ms/step | 15.0 samples/s
+Step 48850/50462 | Loss: 2.5498 | LR: 1.30e-06 | 3985.3ms/step | 15.1 samples/s
+Step 48860/50462 | Loss: 2.5108 | LR: 1.28e-06 | 3991.5ms/step | 15.0 samples/s
+Step 48870/50462 | Loss: 2.5302 | LR: 1.26e-06 | 3997.3ms/step | 15.0 samples/s
+Step 48880/50462 | Loss: 2.4858 | LR: 1.25e-06 | 4001.2ms/step | 15.0 samples/s
+Step 48890/50462 | Loss: 2.5145 | LR: 1.23e-06 | 4009.7ms/step | 15.0 samples/s
+Step 48900/50462 | Loss: 2.5356 | LR: 1.22e-06 | 3993.6ms/step | 15.0 samples/s
+Step 48910/50462 | Loss: 2.5292 | LR: 1.20e-06 | 4003.7ms/step | 15.0 samples/s
+Step 48920/50462 | Loss: 2.5086 | LR: 1.19e-06 | 3996.3ms/step | 15.0 samples/s
+Step 48930/50462 | Loss: 2.5322 | LR: 1.17e-06 | 4000.2ms/step | 15.0 samples/s
+Step 48940/50462 | Loss: 2.5600 | LR: 1.16e-06 | 4006.7ms/step | 15.0 samples/s
+Step 48950/50462 | Loss: 2.5004 | LR: 1.14e-06 | 4008.5ms/step | 15.0 samples/s
+Step 48960/50462 | Loss: 2.5393 | LR: 1.13e-06 | 4005.8ms/step | 15.0 samples/s
+Step 48970/50462 | Loss: 2.5527 | LR: 1.11e-06 | 4005.3ms/step | 15.0 samples/s
+Step 48980/50462 | Loss: 2.4964 | LR: 1.10e-06 | 4009.9ms/step | 15.0 samples/s
+Step 48990/50462 | Loss: 2.5001 | LR: 1.08e-06 | 3991.7ms/step | 15.0 samples/s
+Step 49000/50462 | Loss: 2.4974 | LR: 1.07e-06 | 3999.0ms/step | 15.0 samples/s
+Validation at step 49000: Loss = 2.5119, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-49000
+Step 49010/50462 | Loss: 2.5145 | LR: 1.05e-06 | 5951.4ms/step | 10.1 samples/s
+Step 49020/50462 | Loss: 2.5306 | LR: 1.04e-06 | 3989.9ms/step | 15.0 samples/s
+Step 49030/50462 | Loss: 2.5387 | LR: 1.02e-06 | 3996.4ms/step | 15.0 samples/s
+Step 49040/50462 | Loss: 2.5355 | LR: 1.01e-06 | 3992.3ms/step | 15.0 samples/s
+Step 49050/50462 | Loss: 2.5116 | LR: 9.95e-07 | 3998.5ms/step | 15.0 samples/s
+Step 49060/50462 | Loss: 2.5105 | LR: 9.81e-07 | 4002.8ms/step | 15.0 samples/s
+Step 49070/50462 | Loss: 2.5096 | LR: 9.67e-07 | 3996.1ms/step | 15.0 samples/s
+Step 49080/50462 | Loss: 2.4991 | LR: 9.53e-07 | 4008.7ms/step | 15.0 samples/s
+Step 49090/50462 | Loss: 2.5473 | LR: 9.39e-07 | 4001.2ms/step | 15.0 samples/s
+Step 49100/50462 | Loss: 2.5605 | LR: 9.25e-07 | 3994.9ms/step | 15.0 samples/s
+Step 49110/50462 | Loss: 2.5609 | LR: 9.12e-07 | 3995.2ms/step | 15.0 samples/s
+Step 49120/50462 | Loss: 2.5264 | LR: 8.99e-07 | 3991.5ms/step | 15.0 samples/s
+Step 49130/50462 | Loss: 2.5395 | LR: 8.85e-07 | 4002.9ms/step | 15.0 samples/s
+Step 49140/50462 | Loss: 2.5189 | LR: 8.72e-07 | 3998.9ms/step | 15.0 samples/s
+Step 49150/50462 | Loss: 2.5358 | LR: 8.59e-07 | 3997.8ms/step | 15.0 samples/s
+Step 49160/50462 | Loss: 2.5454 | LR: 8.46e-07 | 4000.5ms/step | 15.0 samples/s
+Step 49170/50462 | Loss: 2.5231 | LR: 8.33e-07 | 3987.3ms/step | 15.0 samples/s
+Step 49180/50462 | Loss: 2.5428 | LR: 8.20e-07 | 4000.2ms/step | 15.0 samples/s
+Step 49190/50462 | Loss: 2.5098 | LR: 8.07e-07 | 3994.4ms/step | 15.0 samples/s
+Step 49200/50462 | Loss: 2.5238 | LR: 7.95e-07 | 4017.5ms/step | 14.9 samples/s
+Step 49210/50462 | Loss: 2.4944 | LR: 7.82e-07 | 3995.9ms/step | 15.0 samples/s
+Step 49220/50462 | Loss: 2.5123 | LR: 7.70e-07 | 4009.4ms/step | 15.0 samples/s
+Step 49230/50462 | Loss: 2.5427 | LR: 7.57e-07 | 3991.3ms/step | 15.0 samples/s
+Step 49240/50462 | Loss: 2.5185 | LR: 7.45e-07 | 3996.2ms/step | 15.0 samples/s
+Step 49250/50462 | Loss: 2.5129 | LR: 7.33e-07 | 4001.3ms/step | 15.0 samples/s
+Validation at step 49250: Loss = 2.5121, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-49250
+Step 49260/50462 | Loss: 2.5413 | LR: 7.21e-07 | 5939.0ms/step | 10.1 samples/s
+Step 49270/50462 | Loss: 2.5167 | LR: 7.09e-07 | 4000.0ms/step | 15.0 samples/s
+Step 49280/50462 | Loss: 2.5413 | LR: 6.97e-07 | 4001.6ms/step | 15.0 samples/s
+Step 49290/50462 | Loss: 2.5261 | LR: 6.85e-07 | 4004.7ms/step | 15.0 samples/s
+Step 49300/50462 | Loss: 2.5075 | LR: 6.74e-07 | 4001.6ms/step | 15.0 samples/s
+Step 49310/50462 | Loss: 2.5506 | LR: 6.62e-07 | 3989.7ms/step | 15.0 samples/s
+Step 49320/50462 | Loss: 2.5084 | LR: 6.51e-07 | 4004.8ms/step | 15.0 samples/s
+Step 49330/50462 | Loss: 2.5215 | LR: 6.39e-07 | 3997.5ms/step | 15.0 samples/s
+Step 49340/50462 | Loss: 2.5675 | LR: 6.28e-07 | 4002.0ms/step | 15.0 samples/s
+Step 49350/50462 | Loss: 2.5026 | LR: 6.17e-07 | 4004.4ms/step | 15.0 samples/s
+Step 49360/50462 | Loss: 2.5429 | LR: 6.06e-07 | 4018.9ms/step | 14.9 samples/s
+Step 49370/50462 | Loss: 2.4956 | LR: 5.95e-07 | 4009.7ms/step | 15.0 samples/s
+Step 49380/50462 | Loss: 2.5128 | LR: 5.84e-07 | 4007.0ms/step | 15.0 samples/s
+Step 49390/50462 | Loss: 2.4993 | LR: 5.73e-07 | 3998.4ms/step | 15.0 samples/s
+Step 49400/50462 | Loss: 2.5325 | LR: 5.63e-07 | 4007.1ms/step | 15.0 samples/s
+Step 49410/50462 | Loss: 2.5199 | LR: 5.52e-07 | 4000.2ms/step | 15.0 samples/s
+Step 49420/50462 | Loss: 2.5240 | LR: 5.42e-07 | 4009.4ms/step | 15.0 samples/s
+Step 49430/50462 | Loss: 2.5285 | LR: 5.31e-07 | 4008.9ms/step | 15.0 samples/s
+Step 49440/50462 | Loss: 2.5525 | LR: 5.21e-07 | 4000.4ms/step | 15.0 samples/s
+Step 49450/50462 | Loss: 2.5148 | LR: 5.11e-07 | 4005.5ms/step | 15.0 samples/s
+Step 49460/50462 | Loss: 2.5115 | LR: 5.01e-07 | 3998.1ms/step | 15.0 samples/s
+Step 49470/50462 | Loss: 2.5184 | LR: 4.91e-07 | 4004.8ms/step | 15.0 samples/s
+Step 49480/50462 | Loss: 2.5508 | LR: 4.81e-07 | 4004.0ms/step | 15.0 samples/s
+Step 49490/50462 | Loss: 2.5180 | LR: 4.72e-07 | 3999.5ms/step | 15.0 samples/s
+Step 49500/50462 | Loss: 2.5122 | LR: 4.62e-07 | 4003.2ms/step | 15.0 samples/s
+Validation at step 49500: Loss = 2.5122, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-49500
+Step 49510/50462 | Loss: 2.5061 | LR: 4.52e-07 | 5952.8ms/step | 10.1 samples/s
+Step 49520/50462 | Loss: 2.5349 | LR: 4.43e-07 | 3982.9ms/step | 15.1 samples/s
+Step 49530/50462 | Loss: 2.5227 | LR: 4.34e-07 | 3994.8ms/step | 15.0 samples/s
+Step 49540/50462 | Loss: 2.5393 | LR: 4.24e-07 | 3992.5ms/step | 15.0 samples/s
+Step 49550/50462 | Loss: 2.5133 | LR: 4.15e-07 | 4001.8ms/step | 15.0 samples/s
+Step 49560/50462 | Loss: 2.5532 | LR: 4.06e-07 | 4000.4ms/step | 15.0 samples/s
+Step 49570/50462 | Loss: 2.5144 | LR: 3.97e-07 | 4003.4ms/step | 15.0 samples/s
+Step 49580/50462 | Loss: 2.5471 | LR: 3.88e-07 | 3995.5ms/step | 15.0 samples/s
+Step 49590/50462 | Loss: 2.5259 | LR: 3.79e-07 | 4007.1ms/step | 15.0 samples/s
+Step 49600/50462 | Loss: 2.5006 | LR: 3.71e-07 | 4006.2ms/step | 15.0 samples/s
+Step 49610/50462 | Loss: 2.5044 | LR: 3.62e-07 | 3995.4ms/step | 15.0 samples/s
+Step 49620/50462 | Loss: 2.5509 | LR: 3.54e-07 | 3998.3ms/step | 15.0 samples/s
+Step 49630/50462 | Loss: 2.5172 | LR: 3.45e-07 | 3993.6ms/step | 15.0 samples/s
+Step 49640/50462 | Loss: 2.5235 | LR: 3.37e-07 | 3998.8ms/step | 15.0 samples/s
+Step 49650/50462 | Loss: 2.5511 | LR: 3.29e-07 | 3996.6ms/step | 15.0 samples/s
+Step 49660/50462 | Loss: 2.5428 | LR: 3.21e-07 | 4004.2ms/step | 15.0 samples/s
+Step 49670/50462 | Loss: 2.5190 | LR: 3.13e-07 | 3999.9ms/step | 15.0 samples/s
+Step 49680/50462 | Loss: 2.5041 | LR: 3.05e-07 | 3999.8ms/step | 15.0 samples/s
+Step 49690/50462 | Loss: 2.5244 | LR: 2.97e-07 | 4002.2ms/step | 15.0 samples/s
+Step 49700/50462 | Loss: 2.5103 | LR: 2.90e-07 | 3992.8ms/step | 15.0 samples/s
+Step 49710/50462 | Loss: 2.4948 | LR: 2.82e-07 | 4017.6ms/step | 14.9 samples/s
+Step 49720/50462 | Loss: 2.5431 | LR: 2.75e-07 | 3999.4ms/step | 15.0 samples/s
+Step 49730/50462 | Loss: 2.5340 | LR: 2.67e-07 | 4012.1ms/step | 15.0 samples/s
+Step 49740/50462 | Loss: 2.5245 | LR: 2.60e-07 | 3998.9ms/step | 15.0 samples/s
+Step 49750/50462 | Loss: 2.5413 | LR: 2.53e-07 | 3988.9ms/step | 15.0 samples/s
+Validation at step 49750: Loss = 2.5124, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-49750
+Step 49760/50462 | Loss: 2.5273 | LR: 2.46e-07 | 5949.9ms/step | 10.1 samples/s
+Step 49770/50462 | Loss: 2.5268 | LR: 2.39e-07 | 4011.7ms/step | 15.0 samples/s
+Step 49780/50462 | Loss: 2.5176 | LR: 2.32e-07 | 4006.5ms/step | 15.0 samples/s
+Step 49790/50462 | Loss: 2.5247 | LR: 2.25e-07 | 4000.6ms/step | 15.0 samples/s
+Step 49800/50462 | Loss: 2.5029 | LR: 2.19e-07 | 4004.5ms/step | 15.0 samples/s
+Step 49810/50462 | Loss: 2.5368 | LR: 2.12e-07 | 3992.8ms/step | 15.0 samples/s
+Step 49820/50462 | Loss: 2.5324 | LR: 2.06e-07 | 4005.8ms/step | 15.0 samples/s
+Step 49830/50462 | Loss: 2.5138 | LR: 1.99e-07 | 4019.5ms/step | 14.9 samples/s
+Step 49840/50462 | Loss: 2.5321 | LR: 1.93e-07 | 4011.3ms/step | 15.0 samples/s
+Step 49850/50462 | Loss: 2.5359 | LR: 1.87e-07 | 4010.4ms/step | 15.0 samples/s
+Step 49860/50462 | Loss: 2.5227 | LR: 1.81e-07 | 4011.7ms/step | 15.0 samples/s
+Step 49870/50462 | Loss: 2.5461 | LR: 1.75e-07 | 4009.2ms/step | 15.0 samples/s
+Step 49880/50462 | Loss: 2.5242 | LR: 1.69e-07 | 4003.2ms/step | 15.0 samples/s
+Step 49890/50462 | Loss: 2.5372 | LR: 1.63e-07 | 4000.6ms/step | 15.0 samples/s
+Step 49900/50462 | Loss: 2.5612 | LR: 1.58e-07 | 4009.6ms/step | 15.0 samples/s
+Step 49910/50462 | Loss: 2.5080 | LR: 1.52e-07 | 4013.6ms/step | 14.9 samples/s
+Step 49920/50462 | Loss: 2.4789 | LR: 1.47e-07 | 4002.2ms/step | 15.0 samples/s
+Step 49930/50462 | Loss: 2.5398 | LR: 1.41e-07 | 4000.8ms/step | 15.0 samples/s
+Step 49940/50462 | Loss: 2.5112 | LR: 1.36e-07 | 4010.0ms/step | 15.0 samples/s
+Step 49950/50462 | Loss: 2.5258 | LR: 1.31e-07 | 4009.4ms/step | 15.0 samples/s
+Step 49960/50462 | Loss: 2.5171 | LR: 1.26e-07 | 3990.0ms/step | 15.0 samples/s
+Step 49970/50462 | Loss: 2.5060 | LR: 1.21e-07 | 4003.4ms/step | 15.0 samples/s
+Step 49980/50462 | Loss: 2.5344 | LR: 1.16e-07 | 3994.6ms/step | 15.0 samples/s
+Step 49990/50462 | Loss: 2.4861 | LR: 1.11e-07 | 4006.1ms/step | 15.0 samples/s
+Step 50000/50462 | Loss: 2.5163 | LR: 1.07e-07 | 4019.3ms/step | 14.9 samples/s
+Validation at step 50000: Loss = 2.5127, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-50000
+Step 50010/50462 | Loss: 2.5315 | LR: 1.02e-07 | 5996.7ms/step | 10.0 samples/s
+Step 50020/50462 | Loss: 2.5056 | LR: 9.75e-08 | 3998.8ms/step | 15.0 samples/s
+Step 50030/50462 | Loss: 2.5432 | LR: 9.32e-08 | 4006.5ms/step | 15.0 samples/s
+Step 50040/50462 | Loss: 2.5155 | LR: 8.89e-08 | 3999.4ms/step | 15.0 samples/s
+Step 50050/50462 | Loss: 2.4929 | LR: 8.47e-08 | 4006.5ms/step | 15.0 samples/s
+Step 50060/50462 | Loss: 2.5170 | LR: 8.07e-08 | 4011.0ms/step | 15.0 samples/s
+Step 50070/50462 | Loss: 2.5340 | LR: 7.67e-08 | 3993.4ms/step | 15.0 samples/s
+Step 50080/50462 | Loss: 2.5207 | LR: 7.28e-08 | 3999.2ms/step | 15.0 samples/s
+Step 50090/50462 | Loss: 2.5241 | LR: 6.91e-08 | 4000.7ms/step | 15.0 samples/s
+Step 50100/50462 | Loss: 2.5258 | LR: 6.54e-08 | 4004.0ms/step | 15.0 samples/s
+Step 50110/50462 | Loss: 2.4582 | LR: 6.19e-08 | 3995.3ms/step | 15.0 samples/s
+Step 50120/50462 | Loss: 2.4748 | LR: 5.84e-08 | 3999.2ms/step | 15.0 samples/s
+Step 50130/50462 | Loss: 2.5260 | LR: 5.50e-08 | 4004.3ms/step | 15.0 samples/s
+Step 50140/50462 | Loss: 2.5276 | LR: 5.18e-08 | 3990.8ms/step | 15.0 samples/s
+Step 50150/50462 | Loss: 2.5215 | LR: 4.86e-08 | 4008.1ms/step | 15.0 samples/s
+Step 50160/50462 | Loss: 2.5096 | LR: 4.55e-08 | 3999.6ms/step | 15.0 samples/s
+Step 50170/50462 | Loss: 2.5483 | LR: 4.26e-08 | 4001.2ms/step | 15.0 samples/s
+Step 50180/50462 | Loss: 2.5151 | LR: 3.97e-08 | 4007.4ms/step | 15.0 samples/s
+Step 50190/50462 | Loss: 2.5116 | LR: 3.69e-08 | 4008.3ms/step | 15.0 samples/s
+Step 50200/50462 | Loss: 2.5186 | LR: 3.43e-08 | 4015.0ms/step | 14.9 samples/s
+Step 50210/50462 | Loss: 2.5216 | LR: 3.17e-08 | 4008.2ms/step | 15.0 samples/s
+Step 50220/50462 | Loss: 2.5358 | LR: 2.92e-08 | 3997.3ms/step | 15.0 samples/s
+Step 50230/50462 | Loss: 2.5622 | LR: 2.69e-08 | 3990.9ms/step | 15.0 samples/s
+Step 50240/50462 | Loss: 2.5394 | LR: 2.46e-08 | 4005.6ms/step | 15.0 samples/s
+Step 50250/50462 | Loss: 2.5495 | LR: 2.24e-08 | 3992.3ms/step | 15.0 samples/s
+Validation at step 50250: Loss = 2.5126, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-50250
+Step 50260/50462 | Loss: 2.5268 | LR: 2.04e-08 | 5911.6ms/step | 10.1 samples/s
+Step 50270/50462 | Loss: 2.5361 | LR: 1.84e-08 | 4014.4ms/step | 14.9 samples/s
+Step 50280/50462 | Loss: 2.5271 | LR: 1.65e-08 | 3995.1ms/step | 15.0 samples/s
+Step 50290/50462 | Loss: 2.5042 | LR: 1.48e-08 | 3994.5ms/step | 15.0 samples/s
+Step 50300/50462 | Loss: 2.5200 | LR: 1.31e-08 | 4014.4ms/step | 14.9 samples/s
+Step 50310/50462 | Loss: 2.5336 | LR: 1.15e-08 | 3997.2ms/step | 15.0 samples/s
+Step 50320/50462 | Loss: 2.4981 | LR: 1.01e-08 | 4003.0ms/step | 15.0 samples/s
+Step 50330/50462 | Loss: 2.5235 | LR: 8.70e-09 | 3997.7ms/step | 15.0 samples/s
+Step 50340/50462 | Loss: 2.5343 | LR: 7.43e-09 | 3994.8ms/step | 15.0 samples/s
+Step 50350/50462 | Loss: 2.4980 | LR: 6.26e-09 | 3996.7ms/step | 15.0 samples/s
+Step 50360/50462 | Loss: 2.5792 | LR: 5.19e-09 | 4003.7ms/step | 15.0 samples/s
+Step 50370/50462 | Loss: 2.5257 | LR: 4.23e-09 | 4002.4ms/step | 15.0 samples/s
+Step 50380/50462 | Loss: 2.5256 | LR: 3.36e-09 | 4002.8ms/step | 15.0 samples/s
+Step 50390/50462 | Loss: 2.5326 | LR: 2.59e-09 | 4005.5ms/step | 15.0 samples/s
+Step 50400/50462 | Loss: 2.5101 | LR: 1.92e-09 | 4004.1ms/step | 15.0 samples/s
+Step 50410/50462 | Loss: 2.5254 | LR: 1.35e-09 | 4004.4ms/step | 15.0 samples/s
+Step 50420/50462 | Loss: 2.5357 | LR: 8.81e-10 | 4020.0ms/step | 14.9 samples/s
+Step 50430/50462 | Loss: 2.5317 | LR: 5.11e-10 | 3989.8ms/step | 15.0 samples/s
+Step 50440/50462 | Loss: 2.5260 | LR: 2.42e-10 | 4004.2ms/step | 15.0 samples/s
+Step 50450/50462 | Loss: 2.4975 | LR: 7.19e-11 | 3993.5ms/step | 15.0 samples/s
+Step 50460/50462 | Loss: 2.5308 | LR: 2.00e-12 | 4002.8ms/step | 15.0 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-50462
+Training complete!

epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_resume28500.log

@@ -0,0 +1,582 @@
+
+===== RESUME FROM CHECKPOINT-28500 2026-08-28T05:32:53Z =====
+W0828 05:33:03.659000 49 site-packages/torch/distributed/run.py:851] 
+W0828 05:33:03.659000 49 site-packages/torch/distributed/run.py:851] *****************************************
+W0828 05:33:03.659000 49 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0828 05:33:03.659000 49 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28500",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28500
+Checkpoint activations: TrueCheckpoint activations: TrueCheckpoint activations: TrueCheckpoint activations: True
+
+
+
+Checkpoint activations: True
+Checkpoint activations: True
+[rank3]: Traceback (most recent call last):
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
+[rank3]:     return pwd.getpwuid(os.getuid())[0]
+[rank3]:            ~~~~~~~~~~~~^^^^^^^^^^^^^
+[rank3]: KeyError: 'getpwuid(): uid not found: 285818'
+
+[rank3]: The above exception was the direct cause of the following exception:
+
+[rank3]: Traceback (most recent call last):
+[rank3]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+[rank3]:     main()
+[rank3]:     ~~~~^^
+[rank3]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+[rank3]:     train(config)
+[rank3]:     ~~~~~^^^^^^^^
+[rank3]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1104, in train
+[rank3]:     model = torch.compile(model)
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
+[rank3]:     return torch._dynamo.optimize(
+[rank3]:            ~~~~~~~~~~~~~~~~~~~~~~^
+[rank3]:         backend=backend,
+[rank3]:         ^^^^^^^^^^^^^^^^
+[rank3]:     ...<3 lines>...
+[rank3]:         guard_filter_fn=guard_filter_fn,
+[rank3]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank3]:     )(model)  # type: ignore[return-value]
+[rank3]:     ^
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
+[rank3]:     return _optimize(rebuild_ctx, *args, **kwargs)
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
+[rank3]:     backend = get_compiler_fn(backend)
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
+[rank3]:     from .repro.after_dynamo import wrap_backend_debug
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
+[rank3]:     from torch._dynamo.debug_utils import (
+[rank3]:     ...<15 lines>...
+[rank3]:     )
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
+[rank3]:     from torch._dynamo.testing import rand_strided
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
+[rank3]:     from torch._dynamo.backends.debugging import aot_eager
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
+[rank3]:     from functorch.compile import min_cut_rematerialization_partition
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
+[rank3]:     from torch._functorch.aot_autograd import (
+[rank3]:     ...<10 lines>...
+[rank3]:     )
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
+[rank3]:     from ._aot_autograd.autograd_cache import (  # noqa: F401
+[rank3]:     ...<4 lines>...
+[rank3]:     )
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
+[rank3]:     from torch._inductor.codecache import (
+[rank3]:     ...<11 lines>...
+[rank3]:     )
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
+[rank3]:     _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
+[rank3]:                                ~~~~~~~~~~~~~~~~~^^
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
+[rank3]:     sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
+[rank3]:                                                       ~~~~~~~~~~~~~~~^^
+[rank3]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
+[rank3]:     raise OSError('No username set in the environment') from e
+[rank3]: OSError: No username set in the environment
+[rank2]: Traceback (most recent call last):
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
+[rank2]:     return pwd.getpwuid(os.getuid())[0]
+[rank2]:            ~~~~~~~~~~~~^^^^^^^^^^^^^
+[rank2]: KeyError: 'getpwuid(): uid not found: 285818'
+
+[rank2]: The above exception was the direct cause of the following exception:
+
+[rank2]: Traceback (most recent call last):
+[rank2]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+[rank2]:     main()
+[rank2]:     ~~~~^^
+[rank2]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+[rank2]:     train(config)
+[rank2]:     ~~~~~^^^^^^^^
+[rank2]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1104, in train
+[rank2]:     model = torch.compile(model)
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
+[rank2]:     return torch._dynamo.optimize(
+[rank2]:            ~~~~~~~~~~~~~~~~~~~~~~^
+[rank2]:         backend=backend,
+[rank2]:         ^^^^^^^^^^^^^^^^
+[rank2]:     ...<3 lines>...
+[rank2]:         guard_filter_fn=guard_filter_fn,
+[rank2]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank2]:     )(model)  # type: ignore[return-value]
+[rank2]:     ^
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
+[rank2]:     return _optimize(rebuild_ctx, *args, **kwargs)
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
+[rank2]:     backend = get_compiler_fn(backend)
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
+[rank2]:     from .repro.after_dynamo import wrap_backend_debug
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
+[rank2]:     from torch._dynamo.debug_utils import (
+[rank2]:     ...<15 lines>...
+[rank2]:     )
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
+[rank2]:     from torch._dynamo.testing import rand_strided
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
+[rank2]:     from torch._dynamo.backends.debugging import aot_eager
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
+[rank2]:     from functorch.compile import min_cut_rematerialization_partition
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
+[rank2]:     from torch._functorch.aot_autograd import (
+[rank2]:     ...<10 lines>...
+[rank2]:     )
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
+[rank2]:     from ._aot_autograd.autograd_cache import (  # noqa: F401
+[rank2]:     ...<4 lines>...
+[rank2]:     )
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
+[rank2]:     from torch._inductor.codecache import (
+[rank2]:     ...<11 lines>...
+[rank2]:     )
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
+[rank2]:     _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
+[rank2]:                                ~~~~~~~~~~~~~~~~~^^
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
+[rank2]:     sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
+[rank2]:                                                       ~~~~~~~~~~~~~~~^^
+[rank2]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
+[rank2]:     raise OSError('No username set in the environment') from e
+[rank2]: OSError: No username set in the environment
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+[rank4]: Traceback (most recent call last):
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
+[rank4]:     return pwd.getpwuid(os.getuid())[0]
+[rank4]:            ~~~~~~~~~~~~^^^^^^^^^^^^^
+[rank4]: KeyError: 'getpwuid(): uid not found: 285818'
+
+[rank4]: The above exception was the direct cause of the following exception:
+
+[rank4]: Traceback (most recent call last):
+[rank4]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+[rank4]:     main()
+[rank4]:     ~~~~^^
+[rank4]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+[rank4]:     train(config)
+[rank4]:     ~~~~~^^^^^^^^
+[rank4]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1104, in train
+[rank4]:     model = torch.compile(model)
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
+[rank4]:     return torch._dynamo.optimize(
+[rank4]:            ~~~~~~~~~~~~~~~~~~~~~~^
+[rank4]:         backend=backend,
+[rank4]:         ^^^^^^^^^^^^^^^^
+[rank4]:     ...<3 lines>...
+[rank4]:         guard_filter_fn=guard_filter_fn,
+[rank4]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank4]:     )(model)  # type: ignore[return-value]
+[rank4]:     ^
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
+[rank4]:     return _optimize(rebuild_ctx, *args, **kwargs)
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
+[rank4]:     backend = get_compiler_fn(backend)
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
+[rank4]:     from .repro.after_dynamo import wrap_backend_debug
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
+[rank4]:     from torch._dynamo.debug_utils import (
+[rank4]:     ...<15 lines>...
+[rank4]:     )
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
+[rank4]:     from torch._dynamo.testing import rand_strided
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
+[rank4]:     from torch._dynamo.backends.debugging import aot_eager
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
+[rank4]:     from functorch.compile import min_cut_rematerialization_partition
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
+[rank4]:     from torch._functorch.aot_autograd import (
+[rank4]:     ...<10 lines>...
+[rank4]:     )
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
+[rank4]:     from ._aot_autograd.autograd_cache import (  # noqa: F401
+[rank4]:     ...<4 lines>...
+[rank4]:     )
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
+[rank4]:     from torch._inductor.codecache import (
+[rank4]:     ...<11 lines>...
+[rank4]:     )
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
+[rank4]:     _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
+[rank4]:                                ~~~~~~~~~~~~~~~~~^^
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
+[rank4]:     sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
+[rank4]:                                                       ~~~~~~~~~~~~~~~^^
+[rank4]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
+[rank4]:     raise OSError('No username set in the environment') from e
+[rank4]: OSError: No username set in the environment
+[rank5]: Traceback (most recent call last):
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
+[rank5]:     return pwd.getpwuid(os.getuid())[0]
+[rank5]:            ~~~~~~~~~~~~^^^^^^^^^^^^^
+[rank5]: KeyError: 'getpwuid(): uid not found: 285818'
+
+[rank5]: The above exception was the direct cause of the following exception:
+
+[rank5]: Traceback (most recent call last):
+[rank5]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+[rank5]:     main()
+[rank5]:     ~~~~^^
+[rank5]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+[rank5]:     train(config)
+[rank5]:     ~~~~~^^^^^^^^
+[rank5]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1104, in train
+[rank5]:     model = torch.compile(model)
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
+[rank5]:     return torch._dynamo.optimize(
+[rank5]:            ~~~~~~~~~~~~~~~~~~~~~~^
+[rank5]:         backend=backend,
+[rank5]:         ^^^^^^^^^^^^^^^^
+[rank5]:     ...<3 lines>...
+[rank5]:         guard_filter_fn=guard_filter_fn,
+[rank5]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank5]:     )(model)  # type: ignore[return-value]
+[rank5]:     ^
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
+[rank5]:     return _optimize(rebuild_ctx, *args, **kwargs)
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
+[rank5]:     backend = get_compiler_fn(backend)
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
+[rank5]:     from .repro.after_dynamo import wrap_backend_debug
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
+[rank5]:     from torch._dynamo.debug_utils import (
+[rank5]:     ...<15 lines>...
+[rank5]:     )
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
+[rank5]:     from torch._dynamo.testing import rand_strided
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
+[rank5]:     from torch._dynamo.backends.debugging import aot_eager
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
+[rank5]:     from functorch.compile import min_cut_rematerialization_partition
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
+[rank5]:     from torch._functorch.aot_autograd import (
+[rank5]:     ...<10 lines>...
+[rank5]:     )
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
+[rank5]:     from ._aot_autograd.autograd_cache import (  # noqa: F401
+[rank5]:     ...<4 lines>...
+[rank5]:     )
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
+[rank5]:     from torch._inductor.codecache import (
+[rank5]:     ...<11 lines>...
+[rank5]:     )
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
+[rank5]:     _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
+[rank5]:                                ~~~~~~~~~~~~~~~~~^^
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
+[rank5]:     sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
+[rank5]:                                                       ~~~~~~~~~~~~~~~^^
+[rank5]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
+[rank5]:     raise OSError('No username set in the environment') from e
+[rank5]: OSError: No username set in the environment
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+Could not generate detailed summary: Failed to run torchinfo. See above stack traces for more details. Executed layers up to: [Embedding: 2, Qwen3RMSNorm: 4, Linear: 5, Linear: 5, Linear: 5, Qwen3RMSNorm: 5, Qwen3RMSNorm: 5, Qwen3RotaryEmbedding: 5]
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+[rank0]:     main()
+[rank0]:     ~~~~^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+[rank0]:     train(config)
+[rank0]:     ~~~~~^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1104, in train
+[rank0]:     model = torch.compile(model)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
+[rank0]:     return torch._dynamo.optimize(
+[rank0]:            ~~~~~~~~~~~~~~~~~~~~~~^
+[rank0]:         backend=backend,
+[rank0]:         ^^^^^^^^^^^^^^^^
+[rank0]:     ...<3 lines>...
+[rank0]:         guard_filter_fn=guard_filter_fn,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )(model)  # type: ignore[return-value]
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
+[rank0]:     return _optimize(rebuild_ctx, *args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
+[rank0]:     backend = get_compiler_fn(backend)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
+[rank0]:     from .repro.after_dynamo import wrap_backend_debug
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
+[rank0]:     from torch._dynamo.debug_utils import (
+[rank0]:     ...<15 lines>...
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
+[rank0]:     from torch._dynamo.testing import rand_strided
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
+[rank0]:     from torch._dynamo.backends.debugging import aot_eager
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
+[rank0]:     from functorch.compile import min_cut_rematerialization_partition
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
+[rank0]:     from torch._functorch.aot_autograd import (
+[rank0]:     ...<10 lines>...
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
+[rank0]:     from ._aot_autograd.autograd_cache import (  # noqa: F401
+[rank0]:     ...<4 lines>...
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
+[rank0]:     from torch._inductor.codecache import (
+[rank0]:     ...<11 lines>...
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 1194, in <module>
+[rank0]:     @CacheArtifactFactory.register
+[rank0]:      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/compiler/_cache.py", line 74, in register
+[rank0]:     raise AssertionError(
+[rank0]:         f"Artifact of type={artifact_type_key} already registered in mega-cache artifact factory"
+[rank0]:     )
+[rank0]: AssertionError: Artifact of type=inductor already registered in mega-cache artifact factory
+[rank1]: Traceback (most recent call last):
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
+[rank1]:     return pwd.getpwuid(os.getuid())[0]
+[rank1]:            ~~~~~~~~~~~~^^^^^^^^^^^^^
+[rank1]: KeyError: 'getpwuid(): uid not found: 285818'
+
+[rank1]: The above exception was the direct cause of the following exception:
+
+[rank1]: Traceback (most recent call last):
+[rank1]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+[rank1]:     main()
+[rank1]:     ~~~~^^
+[rank1]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+[rank1]:     train(config)
+[rank1]:     ~~~~~^^^^^^^^
+[rank1]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1104, in train
+[rank1]:     model = torch.compile(model)
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
+[rank1]:     return torch._dynamo.optimize(
+[rank1]:            ~~~~~~~~~~~~~~~~~~~~~~^
+[rank1]:         backend=backend,
+[rank1]:         ^^^^^^^^^^^^^^^^
+[rank1]:     ...<3 lines>...
+[rank1]:         guard_filter_fn=guard_filter_fn,
+[rank1]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank1]:     )(model)  # type: ignore[return-value]
+[rank1]:     ^
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
+[rank1]:     return _optimize(rebuild_ctx, *args, **kwargs)
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
+[rank1]:     backend = get_compiler_fn(backend)
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
+[rank1]:     from .repro.after_dynamo import wrap_backend_debug
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
+[rank1]:     from torch._dynamo.debug_utils import (
+[rank1]:     ...<15 lines>...
+[rank1]:     )
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
+[rank1]:     from torch._dynamo.testing import rand_strided
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
+[rank1]:     from torch._dynamo.backends.debugging import aot_eager
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
+[rank1]:     from functorch.compile import min_cut_rematerialization_partition
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
+[rank1]:     from torch._functorch.aot_autograd import (
+[rank1]:     ...<10 lines>...
+[rank1]:     )
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
+[rank1]:     from ._aot_autograd.autograd_cache import (  # noqa: F401
+[rank1]:     ...<4 lines>...
+[rank1]:     )
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
+[rank1]:     from torch._inductor.codecache import (
+[rank1]:     ...<11 lines>...
+[rank1]:     )
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
+[rank1]:     _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
+[rank1]:                                ~~~~~~~~~~~~~~~~~^^
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
+[rank1]:     sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
+[rank1]:                                                       ~~~~~~~~~~~~~~~^^
+[rank1]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
+[rank1]:     raise OSError('No username set in the environment') from e
+[rank1]: OSError: No username set in the environment
+[rank0]:[W828 05:36:13.612804296 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+W0828 05:36:17.937000 49 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 117 closing signal SIGTERM
+W0828 05:36:17.939000 49 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 118 closing signal SIGTERM
+W0828 05:36:17.940000 49 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 120 closing signal SIGTERM
+W0828 05:36:17.940000 49 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 121 closing signal SIGTERM
+W0828 05:36:17.940000 49 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 122 closing signal SIGTERM
+E0828 05:36:20.026000 49 site-packages/torch/distributed/elastic/multiprocessing/api.py:986] failed (exitcode: 1) local_rank: 2 (pid: 119) of binary: /mloscratch/homes/navasard/conda/bin/python3.13
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 317, in launch_agent
+    raise ChildFailedError(
+    ...<2 lines>...
+    )
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+train.py FAILED
+------------------------------------------------------------
+Failures:
+[1]:
+  time      : 2026-08-28_05:36:20
+  host      : h100-workspace-6-gpu-0-0
+  rank      : 0 (local_rank: 0)
+  exitcode  : 1 (pid: 117)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[2]:
+  time      : 2026-08-28_05:36:20
+  host      : h100-workspace-6-gpu-0-0
+  rank      : 1 (local_rank: 1)
+  exitcode  : 1 (pid: 118)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[3]:
+  time      : 2026-08-28_05:36:20
+  host      : h100-workspace-6-gpu-0-0
+  rank      : 3 (local_rank: 3)
+  exitcode  : 1 (pid: 120)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[4]:
+  time      : 2026-08-28_05:36:20
+  host      : h100-workspace-6-gpu-0-0
+  rank      : 4 (local_rank: 4)
+  exitcode  : 1 (pid: 121)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[5]:
+  time      : 2026-08-28_05:36:20
+  host      : h100-workspace-6-gpu-0-0
+  rank      : 5 (local_rank: 5)
+  exitcode  : 1 (pid: 122)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2026-08-28_05:36:17
+  host      : h100-workspace-6-gpu-0-0
+  rank      : 2 (local_rank: 2)
+  exitcode  : 1 (pid: 119)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================

epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_resume34250.log

@@ -0,0 +1,336 @@
+===== RESUME FROM CHECKPOINT-34250 ON 6GPU 2026-08-29T04:02:34+00:00 =====
+W0829 04:02:47.578000 83 site-packages/torch/distributed/run.py:851] 
+W0829 04:02:47.578000 83 site-packages/torch/distributed/run.py:851] *****************************************
+W0829 04:02:47.578000 83 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0829 04:02:47.578000 83 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34250",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: TrueCheckpoint activations: True
+
+Checkpoint activations: True
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34250
+Checkpoint activations: TrueCheckpoint activations: True
+
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        6
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504628
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34250...
+Preserving checkpoint LR state for exact resume: current=[0.00012012514190618698], schedule_base=[0.0005]
+Resumed from checkpoint at step 34250 (epoch 0, micro_step 511245)
+Resuming training from step 34250
+Resuming wandb run 8ynn1we6
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+[rank0]:     main()
+[rank0]:     ~~~~^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+[rank0]:     train(config)
+[rank0]:     ~~~~~^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1275, in train
+[rank0]:     wandb.init(
+[rank0]:     ~~~~~~~~~~^
+[rank0]:         project=config.wandb_project,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     ...<4 lines>...
+[rank0]:         resume="must",
+[rank0]:         ^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/wandb_init.py", line 1603, in init
+[rank0]:     get_sentry().reraise(e)
+[rank0]:     ~~~~~~~~~~~~~~~~~~~~^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/analytics/sentry.py", line 190, in reraise
+[rank0]:     raise exc.with_traceback(tb)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/wandb_init.py", line 1524, in init
+[rank0]:     wi.maybe_login(init_settings)
+[rank0]:     ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/wandb_init.py", line 200, in maybe_login
+[rank0]:     wandb_login._login(
+[rank0]:     ~~~~~~~~~~~~~~~~~~^
+[rank0]:         host=run_settings.base_url,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     ...<3 lines>...
+[rank0]:         update_api_key=explicit_key is None,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/wandb_login.py", line 190, in _login
+[rank0]:     auth = _find_or_prompt_for_key(
+[rank0]:         settings,
+[rank0]:     ...<4 lines>...
+[rank0]:         input_timeout=timeout,
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/wandb_login.py", line 266, in _find_or_prompt_for_key
+[rank0]:     auth = wbauth.authenticate_session(
+[rank0]:         host=host,
+[rank0]:     ...<5 lines>...
+[rank0]:         relogin=relogin,
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/lib/wbauth/authenticate.py", line 139, in authenticate_session
+[rank0]:     raise UsageError(
+[rank0]:         "No API key configured. Use `wandb login` to log in."
+[rank0]:     ) from None
+[rank0]: wandb.errors.errors.UsageError: No API key configured. Use `wandb login` to log in.
+[rank0]:[W829 04:07:34.101247650 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+W0829 04:07:43.749000 83 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 152 closing signal SIGTERM
+W0829 04:07:43.750000 83 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 153 closing signal SIGTERM
+W0829 04:07:43.760000 83 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 154 closing signal SIGTERM
+W0829 04:07:43.763000 83 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 155 closing signal SIGTERM
+W0829 04:07:43.776000 83 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 156 closing signal SIGTERM
+E0829 04:07:43.801000 83 site-packages/torch/distributed/elastic/multiprocessing/api.py:986] failed (exitcode: 1) local_rank: 0 (pid: 151) of binary: /mloscratch/homes/navasard/conda/bin/python3.13
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 317, in launch_agent
+    raise ChildFailedError(
+    ...<2 lines>...
+    )
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+train.py FAILED
+------------------------------------------------------------
+Failures:
+[1]:
+  time      : 2026-08-29_04:07:43
+  host      : h100-workspace-6-gpu-0-9
+  rank      : 1 (local_rank: 1)
+  exitcode  : -15 (pid: 152)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 152
+[2]:
+  time      : 2026-08-29_04:07:43
+  host      : h100-workspace-6-gpu-0-9
+  rank      : 2 (local_rank: 2)
+  exitcode  : -15 (pid: 153)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 153
+[3]:
+  time      : 2026-08-29_04:07:43
+  host      : h100-workspace-6-gpu-0-9
+  rank      : 3 (local_rank: 3)
+  exitcode  : -15 (pid: 154)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 154
+[4]:
+  time      : 2026-08-29_04:07:43
+  host      : h100-workspace-6-gpu-0-9
+  rank      : 4 (local_rank: 4)
+  exitcode  : -15 (pid: 155)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 155
+[5]:
+  time      : 2026-08-29_04:07:43
+  host      : h100-workspace-6-gpu-0-9
+  rank      : 5 (local_rank: 5)
+  exitcode  : -15 (pid: 156)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 156
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2026-08-29_04:07:43
+  host      : h100-workspace-6-gpu-0-9
+  rank      : 0 (local_rank: 0)
+  exitcode  : 1 (pid: 151)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================

epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_resume34250.pid

@@ -0,0 +1 @@
+83

epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_resume41750.log

@@ -0,0 +1,336 @@
+===== RESUME FROM CHECKPOINT-41750 ON 6GPU 2026-08-30T15:22:11+00:00 =====
+W0830 15:22:21.873000 59 site-packages/torch/distributed/run.py:851] 
+W0830 15:22:21.873000 59 site-packages/torch/distributed/run.py:851] *****************************************
+W0830 15:22:21.873000 59 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0830 15:22:21.873000 59 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750
+Checkpoint activations: TrueCheckpoint activations: True
+
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        6
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504628
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750...
+Preserving checkpoint LR state for exact resume: current=[3.694238556405591e-05], schedule_base=[0.0005]
+Resumed from checkpoint at step 41750 (epoch 0, micro_step 623745)
+Resuming training from step 41750
+Resuming wandb run 8ynn1we6
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+[rank0]:     main()
+[rank0]:     ~~~~^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+[rank0]:     train(config)
+[rank0]:     ~~~~~^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1275, in train
+[rank0]:     wandb.init(
+[rank0]:     ~~~~~~~~~~^
+[rank0]:         project=config.wandb_project,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     ...<4 lines>...
+[rank0]:         resume="must",
+[rank0]:         ^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/wandb_init.py", line 1603, in init
+[rank0]:     get_sentry().reraise(e)
+[rank0]:     ~~~~~~~~~~~~~~~~~~~~^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/analytics/sentry.py", line 190, in reraise
+[rank0]:     raise exc.with_traceback(tb)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/wandb_init.py", line 1524, in init
+[rank0]:     wi.maybe_login(init_settings)
+[rank0]:     ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/wandb_init.py", line 200, in maybe_login
+[rank0]:     wandb_login._login(
+[rank0]:     ~~~~~~~~~~~~~~~~~~^
+[rank0]:         host=run_settings.base_url,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     ...<3 lines>...
+[rank0]:         update_api_key=explicit_key is None,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/wandb_login.py", line 190, in _login
+[rank0]:     auth = _find_or_prompt_for_key(
+[rank0]:         settings,
+[rank0]:     ...<4 lines>...
+[rank0]:         input_timeout=timeout,
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/wandb_login.py", line 266, in _find_or_prompt_for_key
+[rank0]:     auth = wbauth.authenticate_session(
+[rank0]:         host=host,
+[rank0]:     ...<5 lines>...
+[rank0]:         relogin=relogin,
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/wandb/sdk/lib/wbauth/authenticate.py", line 139, in authenticate_session
+[rank0]:     raise UsageError(
+[rank0]:         "No API key configured. Use `wandb login` to log in."
+[rank0]:     ) from None
+[rank0]: wandb.errors.errors.UsageError: No API key configured. Use `wandb login` to log in.
+[rank0]:[W830 15:26:40.938359884 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+W0830 15:26:48.919000 59 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 128 closing signal SIGTERM
+W0830 15:26:48.923000 59 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 129 closing signal SIGTERM
+W0830 15:26:48.930000 59 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 130 closing signal SIGTERM
+W0830 15:26:48.933000 59 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 131 closing signal SIGTERM
+W0830 15:26:48.936000 59 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 132 closing signal SIGTERM
+E0830 15:26:48.951000 59 site-packages/torch/distributed/elastic/multiprocessing/api.py:986] failed (exitcode: 1) local_rank: 0 (pid: 127) of binary: /mloscratch/homes/navasard/conda/bin/python3.13
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 317, in launch_agent
+    raise ChildFailedError(
+    ...<2 lines>...
+    )
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+train.py FAILED
+------------------------------------------------------------
+Failures:
+[1]:
+  time      : 2026-08-30_15:26:48
+  host      : h100-workspace-6-gpu-0-16
+  rank      : 1 (local_rank: 1)
+  exitcode  : -15 (pid: 128)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 128
+[2]:
+  time      : 2026-08-30_15:26:48
+  host      : h100-workspace-6-gpu-0-16
+  rank      : 2 (local_rank: 2)
+  exitcode  : -15 (pid: 129)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 129
+[3]:
+  time      : 2026-08-30_15:26:48
+  host      : h100-workspace-6-gpu-0-16
+  rank      : 3 (local_rank: 3)
+  exitcode  : -15 (pid: 130)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 130
+[4]:
+  time      : 2026-08-30_15:26:48
+  host      : h100-workspace-6-gpu-0-16
+  rank      : 4 (local_rank: 4)
+  exitcode  : -15 (pid: 131)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 131
+[5]:
+  time      : 2026-08-30_15:26:48
+  host      : h100-workspace-6-gpu-0-16
+  rank      : 5 (local_rank: 5)
+  exitcode  : -15 (pid: 132)
+  error_file: <N/A>
+  traceback : Signal 15 (SIGTERM) received by PID 132
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2026-08-30_15:26:48
+  host      : h100-workspace-6-gpu-0-16
+  rank      : 0 (local_rank: 0)
+  exitcode  : 1 (pid: 127)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================

epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_resume41750_fixed.log

@@ -0,0 +1,520 @@
+===== RESUME FROM CHECKPOINT-41750 ON 6GPU, DDP CURSOR FIX 2026-08-30T15:50:20+00:00 =====
+W0830 15:50:23.094000 3212 site-packages/torch/distributed/run.py:851] 
+W0830 15:50:23.094000 3212 site-packages/torch/distributed/run.py:851] *****************************************
+W0830 15:50:23.094000 3212 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0830 15:50:23.094000 3212 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "resume_micro_step_override": 417500,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        6
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504628
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750...
+Preserving checkpoint LR state for exact resume: current=[3.694238556405591e-05], schedule_base=[0.0005]
+Resumed from checkpoint at step 41750 (epoch 0, micro_step 504628)
+Overriding resume micro-step 504628 -> 417500 for DDP migration
+Resuming training from step 41750
+Starting training...
+Validation at step 41750: Loss = 2.5216, Perplexity = 12.45
+Switching attention mode for step 41751: sparse_stochastic (configured sparse attention)
+Step 41760/50462 | Loss: 2.5086 | LR: 3.69e-05 | 56020.9ms/step | 1.1 samples/s
+Step 41770/50462 | Loss: 2.4860 | LR: 3.68e-05 | 2723.2ms/step | 22.0 samples/s
+Step 41780/50462 | Loss: 2.5298 | LR: 3.67e-05 | 2725.6ms/step | 22.0 samples/s
+Step 41790/50462 | Loss: 2.5016 | LR: 3.66e-05 | 2725.5ms/step | 22.0 samples/s
+Step 41800/50462 | Loss: 2.5404 | LR: 3.65e-05 | 2736.2ms/step | 21.9 samples/s
+Step 41810/50462 | Loss: 2.5405 | LR: 3.64e-05 | 2727.0ms/step | 22.0 samples/s
+Step 41820/50462 | Loss: 2.5193 | LR: 3.64e-05 | 2728.1ms/step | 22.0 samples/s
+Step 41830/50462 | Loss: 2.5108 | LR: 3.63e-05 | 2732.9ms/step | 22.0 samples/s
+Step 41840/50462 | Loss: 2.5370 | LR: 3.62e-05 | 2731.8ms/step | 22.0 samples/s
+Step 41850/50462 | Loss: 2.5383 | LR: 3.61e-05 | 2724.9ms/step | 22.0 samples/s
+Step 41860/50462 | Loss: 2.5218 | LR: 3.60e-05 | 2727.7ms/step | 22.0 samples/s
+Step 41870/50462 | Loss: 2.5459 | LR: 3.60e-05 | 2726.4ms/step | 22.0 samples/s
+Step 41880/50462 | Loss: 2.4835 | LR: 3.59e-05 | 2726.1ms/step | 22.0 samples/s
+Step 41890/50462 | Loss: 2.5340 | LR: 3.58e-05 | 2726.0ms/step | 22.0 samples/s
+Step 41900/50462 | Loss: 2.5488 | LR: 3.57e-05 | 2730.7ms/step | 22.0 samples/s
+Step 41910/50462 | Loss: 2.5311 | LR: 3.56e-05 | 2724.8ms/step | 22.0 samples/s
+Step 41920/50462 | Loss: 2.5674 | LR: 3.55e-05 | 2729.6ms/step | 22.0 samples/s
+Step 41930/50462 | Loss: 2.5345 | LR: 3.55e-05 | 3248.1ms/step | 18.5 samples/s
+Step 41940/50462 | Loss: 2.5198 | LR: 3.54e-05 | 2718.4ms/step | 22.1 samples/s
+Step 41950/50462 | Loss: 2.5468 | LR: 3.53e-05 | 2724.9ms/step | 22.0 samples/s
+Step 41960/50462 | Loss: 2.5154 | LR: 3.52e-05 | 2715.3ms/step | 22.1 samples/s
+Step 41970/50462 | Loss: 2.4999 | LR: 3.51e-05 | 2723.2ms/step | 22.0 samples/s
+Step 41980/50462 | Loss: 2.5251 | LR: 3.51e-05 | 2732.9ms/step | 22.0 samples/s
+Step 41990/50462 | Loss: 2.5310 | LR: 3.50e-05 | 2728.3ms/step | 22.0 samples/s
+Step 42000/50462 | Loss: 2.4920 | LR: 3.49e-05 | 2730.6ms/step | 22.0 samples/s
+Validation at step 42000: Loss = 2.5205, Perplexity = 12.43
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-42000
+Step 42010/50462 | Loss: 2.5288 | LR: 3.48e-05 | 4554.4ms/step | 13.2 samples/s
+Step 42020/50462 | Loss: 2.5287 | LR: 3.47e-05 | 2728.4ms/step | 22.0 samples/s
+Step 42030/50462 | Loss: 2.5712 | LR: 3.47e-05 | 2730.3ms/step | 22.0 samples/s
+Step 42040/50462 | Loss: 2.5535 | LR: 3.46e-05 | 2738.5ms/step | 21.9 samples/s
+Step 42050/50462 | Loss: 2.5031 | LR: 3.45e-05 | 2725.6ms/step | 22.0 samples/s
+Step 42060/50462 | Loss: 2.5244 | LR: 3.44e-05 | 2738.7ms/step | 21.9 samples/s
+Step 42070/50462 | Loss: 2.5497 | LR: 3.43e-05 | 2741.7ms/step | 21.9 samples/s
+Step 42080/50462 | Loss: 2.5297 | LR: 3.43e-05 | 2756.4ms/step | 21.8 samples/s
+Step 42090/50462 | Loss: 2.4813 | LR: 3.42e-05 | 2730.4ms/step | 22.0 samples/s
+Step 42100/50462 | Loss: 2.5298 | LR: 3.41e-05 | 2730.2ms/step | 22.0 samples/s
+Step 42110/50462 | Loss: 2.5463 | LR: 3.40e-05 | 2727.7ms/step | 22.0 samples/s
+Step 42120/50462 | Loss: 2.5573 | LR: 3.39e-05 | 2733.1ms/step | 22.0 samples/s
+Step 42130/50462 | Loss: 2.5346 | LR: 3.39e-05 | 2739.0ms/step | 21.9 samples/s
+Step 42140/50462 | Loss: 2.5040 | LR: 3.38e-05 | 2735.7ms/step | 21.9 samples/s
+Step 42150/50462 | Loss: 2.4818 | LR: 3.37e-05 | 2735.9ms/step | 21.9 samples/s
+Step 42160/50462 | Loss: 2.5915 | LR: 3.36e-05 | 2732.1ms/step | 22.0 samples/s
+Step 42170/50462 | Loss: 2.5185 | LR: 3.35e-05 | 2734.2ms/step | 21.9 samples/s
+Step 42180/50462 | Loss: 2.5723 | LR: 3.35e-05 | 2737.0ms/step | 21.9 samples/s
+Step 42190/50462 | Loss: 2.5800 | LR: 3.34e-05 | 2734.8ms/step | 21.9 samples/s
+Step 42200/50462 | Loss: 2.5437 | LR: 3.33e-05 | 2738.5ms/step | 21.9 samples/s
+Step 42210/50462 | Loss: 2.4921 | LR: 3.32e-05 | 2737.2ms/step | 21.9 samples/s
+Step 42220/50462 | Loss: 2.5887 | LR: 3.32e-05 | 2730.1ms/step | 22.0 samples/s
+Step 42230/50462 | Loss: 2.5081 | LR: 3.31e-05 | 2737.7ms/step | 21.9 samples/s
+Step 42240/50462 | Loss: 2.4761 | LR: 3.30e-05 | 2733.5ms/step | 21.9 samples/s
+Step 42250/50462 | Loss: 2.5265 | LR: 3.29e-05 | 2736.6ms/step | 21.9 samples/s
+Validation at step 42250: Loss = 2.5212, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-42250
+Step 42260/50462 | Loss: 2.4917 | LR: 3.28e-05 | 4060.4ms/step | 14.8 samples/s
+Step 42270/50462 | Loss: 2.5085 | LR: 3.28e-05 | 2738.8ms/step | 21.9 samples/s
+Step 42280/50462 | Loss: 2.5428 | LR: 3.27e-05 | 2746.7ms/step | 21.8 samples/s
+Step 42290/50462 | Loss: 2.5257 | LR: 3.26e-05 | 2754.7ms/step | 21.8 samples/s
+Step 42300/50462 | Loss: 2.5349 | LR: 3.25e-05 | 2744.3ms/step | 21.9 samples/s
+Step 42310/50462 | Loss: 2.5383 | LR: 3.24e-05 | 2732.5ms/step | 22.0 samples/s
+Step 42320/50462 | Loss: 2.5212 | LR: 3.24e-05 | 2737.9ms/step | 21.9 samples/s
+Step 42330/50462 | Loss: 2.5539 | LR: 3.23e-05 | 2735.5ms/step | 21.9 samples/s
+Step 42340/50462 | Loss: 2.5381 | LR: 3.22e-05 | 2727.6ms/step | 22.0 samples/s
+Step 42350/50462 | Loss: 2.5457 | LR: 3.21e-05 | 2733.8ms/step | 21.9 samples/s
+Step 42360/50462 | Loss: 2.5168 | LR: 3.21e-05 | 2730.6ms/step | 22.0 samples/s
+Step 42370/50462 | Loss: 2.5593 | LR: 3.20e-05 | 2738.7ms/step | 21.9 samples/s
+Step 42380/50462 | Loss: 2.5564 | LR: 3.19e-05 | 2733.3ms/step | 22.0 samples/s
+Step 42390/50462 | Loss: 2.5128 | LR: 3.18e-05 | 2732.5ms/step | 22.0 samples/s
+Step 42400/50462 | Loss: 2.5363 | LR: 3.18e-05 | 2729.2ms/step | 22.0 samples/s
+Step 42410/50462 | Loss: 2.5288 | LR: 3.17e-05 | 2730.5ms/step | 22.0 samples/s
+Step 42420/50462 | Loss: 2.5520 | LR: 3.16e-05 | 2731.5ms/step | 22.0 samples/s
+Step 42430/50462 | Loss: 2.5563 | LR: 3.15e-05 | 2732.6ms/step | 22.0 samples/s
+Step 42440/50462 | Loss: 2.5373 | LR: 3.14e-05 | 2730.8ms/step | 22.0 samples/s
+Step 42450/50462 | Loss: 2.5193 | LR: 3.14e-05 | 2728.6ms/step | 22.0 samples/s
+Step 42460/50462 | Loss: 2.5160 | LR: 3.13e-05 | 2735.9ms/step | 21.9 samples/s
+Step 42470/50462 | Loss: 2.5369 | LR: 3.12e-05 | 2732.6ms/step | 22.0 samples/s
+Step 42480/50462 | Loss: 2.5071 | LR: 3.11e-05 | 2736.5ms/step | 21.9 samples/s
+Step 42490/50462 | Loss: 2.5167 | LR: 3.11e-05 | 2733.9ms/step | 21.9 samples/s
+Step 42500/50462 | Loss: 2.5511 | LR: 3.10e-05 | 2733.3ms/step | 22.0 samples/s
+Validation at step 42500: Loss = 2.5193, Perplexity = 12.42
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-42500
+Step 42510/50462 | Loss: 2.5018 | LR: 3.09e-05 | 4127.5ms/step | 14.5 samples/s
+Step 42520/50462 | Loss: 2.5685 | LR: 3.08e-05 | 2739.8ms/step | 21.9 samples/s
+Step 42530/50462 | Loss: 2.5188 | LR: 3.08e-05 | 2729.9ms/step | 22.0 samples/s
+Step 42540/50462 | Loss: 2.5315 | LR: 3.07e-05 | 2729.2ms/step | 22.0 samples/s
+Step 42550/50462 | Loss: 2.5558 | LR: 3.06e-05 | 2736.0ms/step | 21.9 samples/s
+Step 42560/50462 | Loss: 2.5161 | LR: 3.05e-05 | 2737.5ms/step | 21.9 samples/s
+Step 42570/50462 | Loss: 2.4898 | LR: 3.05e-05 | 2731.9ms/step | 22.0 samples/s
+Step 42580/50462 | Loss: 2.5034 | LR: 3.04e-05 | 2733.7ms/step | 21.9 samples/s
+Step 42590/50462 | Loss: 2.5425 | LR: 3.03e-05 | 2735.3ms/step | 21.9 samples/s
+Step 42600/50462 | Loss: 2.5022 | LR: 3.02e-05 | 2729.4ms/step | 22.0 samples/s
+Step 42610/50462 | Loss: 2.5361 | LR: 3.02e-05 | 2729.7ms/step | 22.0 samples/s
+Step 42620/50462 | Loss: 2.5173 | LR: 3.01e-05 | 2734.2ms/step | 21.9 samples/s
+Step 42630/50462 | Loss: 2.5356 | LR: 3.00e-05 | 2745.5ms/step | 21.9 samples/s
+Step 42640/50462 | Loss: 2.5185 | LR: 2.99e-05 | 2743.4ms/step | 21.9 samples/s
+Step 42650/50462 | Loss: 2.5386 | LR: 2.99e-05 | 2735.5ms/step | 21.9 samples/s
+Step 42660/50462 | Loss: 2.5202 | LR: 2.98e-05 | 2734.7ms/step | 21.9 samples/s
+Step 42670/50462 | Loss: 2.5243 | LR: 2.97e-05 | 2734.2ms/step | 21.9 samples/s
+Step 42680/50462 | Loss: 2.5409 | LR: 2.96e-05 | 2739.1ms/step | 21.9 samples/s
+Step 42690/50462 | Loss: 2.5322 | LR: 2.96e-05 | 2737.2ms/step | 21.9 samples/s
+Step 42700/50462 | Loss: 2.5255 | LR: 2.95e-05 | 2729.4ms/step | 22.0 samples/s
+Step 42710/50462 | Loss: 2.5258 | LR: 2.94e-05 | 2741.8ms/step | 21.9 samples/s
+Step 42720/50462 | Loss: 2.5349 | LR: 2.93e-05 | 2733.7ms/step | 21.9 samples/s
+Step 42730/50462 | Loss: 2.5121 | LR: 2.93e-05 | 2733.1ms/step | 22.0 samples/s
+Step 42740/50462 | Loss: 2.5238 | LR: 2.92e-05 | 2733.9ms/step | 21.9 samples/s
+Step 42750/50462 | Loss: 2.5124 | LR: 2.91e-05 | 2734.5ms/step | 21.9 samples/s
+Validation at step 42750: Loss = 2.5207, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-42750
+Step 42760/50462 | Loss: 2.5714 | LR: 2.90e-05 | 4036.8ms/step | 14.9 samples/s
+Step 42770/50462 | Loss: 2.4918 | LR: 2.90e-05 | 2731.3ms/step | 22.0 samples/s
+Step 42780/50462 | Loss: 2.5021 | LR: 2.89e-05 | 2732.6ms/step | 22.0 samples/s
+Step 42790/50462 | Loss: 2.5253 | LR: 2.88e-05 | 2732.8ms/step | 22.0 samples/s
+Step 42800/50462 | Loss: 2.5356 | LR: 2.87e-05 | 2736.9ms/step | 21.9 samples/s
+Step 42810/50462 | Loss: 2.5110 | LR: 2.87e-05 | 2720.8ms/step | 22.1 samples/s
+Step 42820/50462 | Loss: 2.5755 | LR: 2.86e-05 | 2730.8ms/step | 22.0 samples/s
+Step 42830/50462 | Loss: 2.4892 | LR: 2.85e-05 | 2734.6ms/step | 21.9 samples/s
+Step 42840/50462 | Loss: 2.5342 | LR: 2.84e-05 | 2729.4ms/step | 22.0 samples/s
+Step 42850/50462 | Loss: 2.5264 | LR: 2.84e-05 | 2731.6ms/step | 22.0 samples/s
+Step 42860/50462 | Loss: 2.5449 | LR: 2.83e-05 | 2729.5ms/step | 22.0 samples/s
+Step 42870/50462 | Loss: 2.5485 | LR: 2.82e-05 | 2734.4ms/step | 21.9 samples/s
+Step 42880/50462 | Loss: 2.5169 | LR: 2.82e-05 | 2734.8ms/step | 21.9 samples/s
+Step 42890/50462 | Loss: 2.5571 | LR: 2.81e-05 | 2732.4ms/step | 22.0 samples/s
+Step 42900/50462 | Loss: 2.5465 | LR: 2.80e-05 | 2732.0ms/step | 22.0 samples/s
+Step 42910/50462 | Loss: 2.4815 | LR: 2.79e-05 | 2727.2ms/step | 22.0 samples/s
+Step 42920/50462 | Loss: 2.5433 | LR: 2.79e-05 | 2731.8ms/step | 22.0 samples/s
+Step 42930/50462 | Loss: 2.5105 | LR: 2.78e-05 | 2726.6ms/step | 22.0 samples/s
+Step 42940/50462 | Loss: 2.5140 | LR: 2.77e-05 | 2730.7ms/step | 22.0 samples/s
+Step 42950/50462 | Loss: 2.5175 | LR: 2.76e-05 | 2736.4ms/step | 21.9 samples/s
+Step 42960/50462 | Loss: 2.5325 | LR: 2.76e-05 | 2729.7ms/step | 22.0 samples/s
+Step 42970/50462 | Loss: 2.5278 | LR: 2.75e-05 | 2729.6ms/step | 22.0 samples/s
+Step 42980/50462 | Loss: 2.5653 | LR: 2.74e-05 | 2731.4ms/step | 22.0 samples/s
+Step 42990/50462 | Loss: 2.5530 | LR: 2.74e-05 | 2730.0ms/step | 22.0 samples/s
+Step 43000/50462 | Loss: 2.5365 | LR: 2.73e-05 | 2737.1ms/step | 21.9 samples/s
+Validation at step 43000: Loss = 2.5211, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-43000
+Step 43010/50462 | Loss: 2.5312 | LR: 2.72e-05 | 4043.7ms/step | 14.8 samples/s
+Step 43020/50462 | Loss: 2.5102 | LR: 2.71e-05 | 2733.6ms/step | 21.9 samples/s
+Step 43030/50462 | Loss: 2.5540 | LR: 2.71e-05 | 2732.1ms/step | 22.0 samples/s
+Step 43040/50462 | Loss: 2.5030 | LR: 2.70e-05 | 2741.1ms/step | 21.9 samples/s
+Step 43050/50462 | Loss: 2.5243 | LR: 2.69e-05 | 2732.5ms/step | 22.0 samples/s
+Step 43060/50462 | Loss: 2.5229 | LR: 2.69e-05 | 2737.3ms/step | 21.9 samples/s
+Step 43070/50462 | Loss: 2.5397 | LR: 2.68e-05 | 2724.5ms/step | 22.0 samples/s
+Step 43080/50462 | Loss: 2.5471 | LR: 2.67e-05 | 2736.0ms/step | 21.9 samples/s
+Step 43090/50462 | Loss: 2.5069 | LR: 2.66e-05 | 2732.7ms/step | 22.0 samples/s
+Step 43100/50462 | Loss: 2.5243 | LR: 2.66e-05 | 2732.9ms/step | 22.0 samples/s
+Step 43110/50462 | Loss: 2.5180 | LR: 2.65e-05 | 2728.8ms/step | 22.0 samples/s
+Step 43120/50462 | Loss: 2.5301 | LR: 2.64e-05 | 2734.4ms/step | 21.9 samples/s
+Step 43130/50462 | Loss: 2.5530 | LR: 2.64e-05 | 2731.2ms/step | 22.0 samples/s
+Step 43140/50462 | Loss: 2.5405 | LR: 2.63e-05 | 2737.2ms/step | 21.9 samples/s
+Step 43150/50462 | Loss: 2.5171 | LR: 2.62e-05 | 2726.8ms/step | 22.0 samples/s
+Step 43160/50462 | Loss: 2.5167 | LR: 2.61e-05 | 2729.8ms/step | 22.0 samples/s
+Step 43170/50462 | Loss: 2.5610 | LR: 2.61e-05 | 2727.0ms/step | 22.0 samples/s
+Step 43180/50462 | Loss: 2.4815 | LR: 2.60e-05 | 2731.9ms/step | 22.0 samples/s
+Step 43190/50462 | Loss: 2.5634 | LR: 2.59e-05 | 2728.5ms/step | 22.0 samples/s
+Step 43200/50462 | Loss: 2.5176 | LR: 2.59e-05 | 2726.1ms/step | 22.0 samples/s
+Step 43210/50462 | Loss: 2.5244 | LR: 2.58e-05 | 2725.0ms/step | 22.0 samples/s
+Step 43220/50462 | Loss: 2.5249 | LR: 2.57e-05 | 2736.4ms/step | 21.9 samples/s
+Step 43230/50462 | Loss: 2.4846 | LR: 2.57e-05 | 2732.3ms/step | 22.0 samples/s
+Step 43240/50462 | Loss: 2.5038 | LR: 2.56e-05 | 2733.3ms/step | 22.0 samples/s
+Step 43250/50462 | Loss: 2.5087 | LR: 2.55e-05 | 2732.6ms/step | 22.0 samples/s
+Validation at step 43250: Loss = 2.5206, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-43250
+Step 43260/50462 | Loss: 2.5565 | LR: 2.54e-05 | 4050.6ms/step | 14.8 samples/s
+Step 43270/50462 | Loss: 2.5277 | LR: 2.54e-05 | 2731.2ms/step | 22.0 samples/s
+Step 43280/50462 | Loss: 2.5534 | LR: 2.53e-05 | 2732.5ms/step | 22.0 samples/s
+Step 43290/50462 | Loss: 2.5267 | LR: 2.52e-05 | 2728.2ms/step | 22.0 samples/s
+Step 43300/50462 | Loss: 2.5202 | LR: 2.52e-05 | 2729.3ms/step | 22.0 samples/s
+Step 43310/50462 | Loss: 2.5352 | LR: 2.51e-05 | 2729.4ms/step | 22.0 samples/s
+Step 43320/50462 | Loss: 2.5204 | LR: 2.50e-05 | 2737.8ms/step | 21.9 samples/s
+Step 43330/50462 | Loss: 2.5061 | LR: 2.50e-05 | 2736.7ms/step | 21.9 samples/s
+Step 43340/50462 | Loss: 2.5357 | LR: 2.49e-05 | 2730.0ms/step | 22.0 samples/s
+Step 43350/50462 | Loss: 2.5538 | LR: 2.48e-05 | 2729.3ms/step | 22.0 samples/s
+Step 43360/50462 | Loss: 2.5033 | LR: 2.48e-05 | 2744.4ms/step | 21.9 samples/s
+Step 43370/50462 | Loss: 2.5397 | LR: 2.47e-05 | 2738.6ms/step | 21.9 samples/s
+Step 43380/50462 | Loss: 2.5544 | LR: 2.46e-05 | 2735.1ms/step | 21.9 samples/s
+Step 43390/50462 | Loss: 2.5524 | LR: 2.46e-05 | 2740.3ms/step | 21.9 samples/s
+Step 43400/50462 | Loss: 2.5053 | LR: 2.45e-05 | 2734.0ms/step | 21.9 samples/s
+Step 43410/50462 | Loss: 2.4862 | LR: 2.44e-05 | 2733.3ms/step | 22.0 samples/s
+Step 43420/50462 | Loss: 2.5584 | LR: 2.44e-05 | 2728.3ms/step | 22.0 samples/s
+Step 43430/50462 | Loss: 2.4927 | LR: 2.43e-05 | 2740.2ms/step | 21.9 samples/s
+Step 43440/50462 | Loss: 2.5411 | LR: 2.42e-05 | 2731.5ms/step | 22.0 samples/s
+Step 43450/50462 | Loss: 2.5032 | LR: 2.41e-05 | 2729.2ms/step | 22.0 samples/s
+Step 43460/50462 | Loss: 2.5194 | LR: 2.41e-05 | 2730.8ms/step | 22.0 samples/s
+Step 43470/50462 | Loss: 2.5251 | LR: 2.40e-05 | 2730.3ms/step | 22.0 samples/s
+Step 43480/50462 | Loss: 2.5118 | LR: 2.39e-05 | 2724.4ms/step | 22.0 samples/s
+Step 43490/50462 | Loss: 2.5068 | LR: 2.39e-05 | 2733.6ms/step | 21.9 samples/s
+Step 43500/50462 | Loss: 2.5315 | LR: 2.38e-05 | 2734.5ms/step | 21.9 samples/s
+Validation at step 43500: Loss = 2.5203, Perplexity = 12.43
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-43500
+Step 43510/50462 | Loss: 2.5352 | LR: 2.37e-05 | 4024.2ms/step | 14.9 samples/s
+Step 43520/50462 | Loss: 2.5761 | LR: 2.37e-05 | 2726.8ms/step | 22.0 samples/s
+Step 43530/50462 | Loss: 2.5340 | LR: 2.36e-05 | 2723.7ms/step | 22.0 samples/s
+Step 43540/50462 | Loss: 2.5022 | LR: 2.35e-05 | 2734.7ms/step | 21.9 samples/s
+Step 43550/50462 | Loss: 2.5263 | LR: 2.35e-05 | 2731.5ms/step | 22.0 samples/s
+Step 43560/50462 | Loss: 2.5125 | LR: 2.34e-05 | 2731.7ms/step | 22.0 samples/s
+Step 43570/50462 | Loss: 2.5487 | LR: 2.33e-05 | 2732.2ms/step | 22.0 samples/s
+Step 43580/50462 | Loss: 2.5087 | LR: 2.33e-05 | 2728.2ms/step | 22.0 samples/s
+Step 43590/50462 | Loss: 2.5231 | LR: 2.32e-05 | 2728.1ms/step | 22.0 samples/s
+Step 43600/50462 | Loss: 2.5579 | LR: 2.31e-05 | 2733.0ms/step | 22.0 samples/s
+Step 43610/50462 | Loss: 2.5060 | LR: 2.31e-05 | 2730.2ms/step | 22.0 samples/s
+Step 43620/50462 | Loss: 2.4852 | LR: 2.30e-05 | 2730.4ms/step | 22.0 samples/s
+Step 43630/50462 | Loss: 2.5019 | LR: 2.29e-05 | 2723.4ms/step | 22.0 samples/s
+Step 43640/50462 | Loss: 2.4561 | LR: 2.29e-05 | 2722.7ms/step | 22.0 samples/s
+Step 43650/50462 | Loss: 2.5473 | LR: 2.28e-05 | 2721.8ms/step | 22.0 samples/s
+Step 43660/50462 | Loss: 2.4923 | LR: 2.27e-05 | 2734.5ms/step | 21.9 samples/s
+Step 43670/50462 | Loss: 2.5366 | LR: 2.27e-05 | 2725.4ms/step | 22.0 samples/s
+Step 43680/50462 | Loss: 2.5558 | LR: 2.26e-05 | 2736.2ms/step | 21.9 samples/s
+Step 43690/50462 | Loss: 2.5077 | LR: 2.25e-05 | 2728.8ms/step | 22.0 samples/s
+Step 43700/50462 | Loss: 2.5384 | LR: 2.25e-05 | 2735.4ms/step | 21.9 samples/s
+Step 43710/50462 | Loss: 2.5066 | LR: 2.24e-05 | 2730.3ms/step | 22.0 samples/s
+Step 43720/50462 | Loss: 2.5451 | LR: 2.24e-05 | 2735.4ms/step | 21.9 samples/s
+Step 43730/50462 | Loss: 2.5614 | LR: 2.23e-05 | 2729.7ms/step | 22.0 samples/s
+Step 43740/50462 | Loss: 2.5140 | LR: 2.22e-05 | 2729.1ms/step | 22.0 samples/s
+Step 43750/50462 | Loss: 2.5743 | LR: 2.22e-05 | 2726.5ms/step | 22.0 samples/s
+Validation at step 43750: Loss = 2.5195, Perplexity = 12.42
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-43750
+Step 43760/50462 | Loss: 2.5136 | LR: 2.21e-05 | 4025.1ms/step | 14.9 samples/s
+Step 43770/50462 | Loss: 2.5023 | LR: 2.20e-05 | 2728.1ms/step | 22.0 samples/s
+Step 43780/50462 | Loss: 2.5044 | LR: 2.20e-05 | 2728.7ms/step | 22.0 samples/s
+Step 43790/50462 | Loss: 2.5415 | LR: 2.19e-05 | 2732.5ms/step | 22.0 samples/s
+Step 43800/50462 | Loss: 2.5168 | LR: 2.18e-05 | 2727.0ms/step | 22.0 samples/s
+Step 43810/50462 | Loss: 2.4990 | LR: 2.18e-05 | 2730.7ms/step | 22.0 samples/s
+Step 43820/50462 | Loss: 2.4838 | LR: 2.17e-05 | 2735.4ms/step | 21.9 samples/s
+Step 43830/50462 | Loss: 2.4991 | LR: 2.16e-05 | 2734.4ms/step | 21.9 samples/s
+Step 43840/50462 | Loss: 2.5263 | LR: 2.16e-05 | 2723.1ms/step | 22.0 samples/s
+Step 43850/50462 | Loss: 2.4865 | LR: 2.15e-05 | 2731.8ms/step | 22.0 samples/s
+Step 43860/50462 | Loss: 2.5206 | LR: 2.14e-05 | 2736.4ms/step | 21.9 samples/s
+Step 43870/50462 | Loss: 2.5400 | LR: 2.14e-05 | 2732.7ms/step | 22.0 samples/s
+Step 43880/50462 | Loss: 2.5567 | LR: 2.13e-05 | 2732.2ms/step | 22.0 samples/s
+Step 43890/50462 | Loss: 2.5372 | LR: 2.13e-05 | 2734.7ms/step | 21.9 samples/s
+Step 43900/50462 | Loss: 2.5271 | LR: 2.12e-05 | 2730.5ms/step | 22.0 samples/s
+Step 43910/50462 | Loss: 2.5319 | LR: 2.11e-05 | 2728.1ms/step | 22.0 samples/s
+Step 43920/50462 | Loss: 2.4903 | LR: 2.11e-05 | 2732.0ms/step | 22.0 samples/s
+Step 43930/50462 | Loss: 2.5894 | LR: 2.10e-05 | 2730.5ms/step | 22.0 samples/s
+Step 43940/50462 | Loss: 2.4965 | LR: 2.09e-05 | 2736.0ms/step | 21.9 samples/s
+Step 43950/50462 | Loss: 2.4954 | LR: 2.09e-05 | 2745.8ms/step | 21.9 samples/s
+Step 43960/50462 | Loss: 2.5055 | LR: 2.08e-05 | 2739.9ms/step | 21.9 samples/s
+Step 43970/50462 | Loss: 2.5150 | LR: 2.07e-05 | 2741.1ms/step | 21.9 samples/s
+Step 43980/50462 | Loss: 2.4968 | LR: 2.07e-05 | 2737.9ms/step | 21.9 samples/s
+Step 43990/50462 | Loss: 2.5287 | LR: 2.06e-05 | 2732.2ms/step | 22.0 samples/s
+Step 44000/50462 | Loss: 2.5126 | LR: 2.06e-05 | 2745.1ms/step | 21.9 samples/s
+Validation at step 44000: Loss = 2.5195, Perplexity = 12.42
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-44000
+Step 44010/50462 | Loss: 2.4940 | LR: 2.05e-05 | 4056.0ms/step | 14.8 samples/s
+Step 44020/50462 | Loss: 2.4906 | LR: 2.04e-05 | 2736.9ms/step | 21.9 samples/s
+Step 44030/50462 | Loss: 2.5250 | LR: 2.04e-05 | 2730.1ms/step | 22.0 samples/s
+Step 44040/50462 | Loss: 2.5458 | LR: 2.03e-05 | 2730.9ms/step | 22.0 samples/s
+Step 44050/50462 | Loss: 2.5389 | LR: 2.02e-05 | 2727.3ms/step | 22.0 samples/s
+Step 44060/50462 | Loss: 2.5179 | LR: 2.02e-05 | 2727.1ms/step | 22.0 samples/s
+Step 44070/50462 | Loss: 2.5533 | LR: 2.01e-05 | 2730.3ms/step | 22.0 samples/s
+Step 44080/50462 | Loss: 2.5435 | LR: 2.01e-05 | 2735.9ms/step | 21.9 samples/s
+Step 44090/50462 | Loss: 2.5072 | LR: 2.00e-05 | 2741.2ms/step | 21.9 samples/s
+Step 44100/50462 | Loss: 2.5250 | LR: 1.99e-05 | 2727.8ms/step | 22.0 samples/s
+Step 44110/50462 | Loss: 2.4819 | LR: 1.99e-05 | 2725.3ms/step | 22.0 samples/s
+Step 44120/50462 | Loss: 2.5354 | LR: 1.98e-05 | 2741.6ms/step | 21.9 samples/s
+Step 44130/50462 | Loss: 2.5266 | LR: 1.97e-05 | 2732.2ms/step | 22.0 samples/s
+Step 44140/50462 | Loss: 2.5285 | LR: 1.97e-05 | 2733.3ms/step | 22.0 samples/s
+Step 44150/50462 | Loss: 2.4852 | LR: 1.96e-05 | 2741.5ms/step | 21.9 samples/s
+Step 44160/50462 | Loss: 2.5450 | LR: 1.96e-05 | 2731.8ms/step | 22.0 samples/s
+Step 44170/50462 | Loss: 2.5304 | LR: 1.95e-05 | 2736.2ms/step | 21.9 samples/s
+Step 44180/50462 | Loss: 2.5373 | LR: 1.94e-05 | 2724.4ms/step | 22.0 samples/s
+Step 44190/50462 | Loss: 2.5167 | LR: 1.94e-05 | 2738.6ms/step | 21.9 samples/s
+Step 44200/50462 | Loss: 2.5149 | LR: 1.93e-05 | 2740.1ms/step | 21.9 samples/s
+Step 44210/50462 | Loss: 2.5433 | LR: 1.93e-05 | 2736.2ms/step | 21.9 samples/s
+Step 44220/50462 | Loss: 2.5345 | LR: 1.92e-05 | 2738.9ms/step | 21.9 samples/s
+Step 44230/50462 | Loss: 2.5264 | LR: 1.91e-05 | 2737.7ms/step | 21.9 samples/s
+Step 44240/50462 | Loss: 2.4999 | LR: 1.91e-05 | 2736.1ms/step | 21.9 samples/s
+Step 44250/50462 | Loss: 2.5557 | LR: 1.90e-05 | 2739.3ms/step | 21.9 samples/s
+Validation at step 44250: Loss = 2.5207, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-44250
+Step 44260/50462 | Loss: 2.5687 | LR: 1.90e-05 | 4640.7ms/step | 12.9 samples/s
+Step 44270/50462 | Loss: 2.5269 | LR: 1.89e-05 | 2730.3ms/step | 22.0 samples/s
+Step 44280/50462 | Loss: 2.5233 | LR: 1.88e-05 | 2735.2ms/step | 21.9 samples/s
+Step 44290/50462 | Loss: 2.4921 | LR: 1.88e-05 | 2738.0ms/step | 21.9 samples/s
+Step 44300/50462 | Loss: 2.4836 | LR: 1.87e-05 | 2738.1ms/step | 21.9 samples/s
+Step 44310/50462 | Loss: 2.5067 | LR: 1.87e-05 | 2739.1ms/step | 21.9 samples/s
+Step 44320/50462 | Loss: 2.4805 | LR: 1.86e-05 | 2739.8ms/step | 21.9 samples/s
+Step 44330/50462 | Loss: 2.5108 | LR: 1.85e-05 | 2737.9ms/step | 21.9 samples/s
+Step 44340/50462 | Loss: 2.5251 | LR: 1.85e-05 | 2736.6ms/step | 21.9 samples/s
+Step 44350/50462 | Loss: 2.5207 | LR: 1.84e-05 | 2738.8ms/step | 21.9 samples/s
+Step 44360/50462 | Loss: 2.5171 | LR: 1.84e-05 | 2736.8ms/step | 21.9 samples/s
+Step 44370/50462 | Loss: 2.5610 | LR: 1.83e-05 | 2738.0ms/step | 21.9 samples/s
+Step 44380/50462 | Loss: 2.5507 | LR: 1.82e-05 | 2737.2ms/step | 21.9 samples/s
+Step 44390/50462 | Loss: 2.5341 | LR: 1.82e-05 | 2741.7ms/step | 21.9 samples/s
+Step 44400/50462 | Loss: 2.5162 | LR: 1.81e-05 | 2743.0ms/step | 21.9 samples/s
+Step 44410/50462 | Loss: 2.5546 | LR: 1.81e-05 | 2733.5ms/step | 22.0 samples/s
+Step 44420/50462 | Loss: 2.4824 | LR: 1.80e-05 | 2732.1ms/step | 22.0 samples/s
+Step 44430/50462 | Loss: 2.5417 | LR: 1.79e-05 | 2739.2ms/step | 21.9 samples/s
+Step 44440/50462 | Loss: 2.5131 | LR: 1.79e-05 | 2734.3ms/step | 21.9 samples/s
+Step 44450/50462 | Loss: 2.5297 | LR: 1.78e-05 | 2737.6ms/step | 21.9 samples/s
+Step 44460/50462 | Loss: 2.5527 | LR: 1.78e-05 | 2734.1ms/step | 21.9 samples/s
+Step 44470/50462 | Loss: 2.5223 | LR: 1.77e-05 | 2734.4ms/step | 21.9 samples/s
+Step 44480/50462 | Loss: 2.5416 | LR: 1.77e-05 | 2734.7ms/step | 21.9 samples/s
+Step 44490/50462 | Loss: 2.5401 | LR: 1.76e-05 | 2741.2ms/step | 21.9 samples/s
+Step 44500/50462 | Loss: 2.4832 | LR: 1.75e-05 | 2731.1ms/step | 22.0 samples/s
+Validation at step 44500: Loss = 2.5188, Perplexity = 12.41
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-44500
+Step 44510/50462 | Loss: 2.5349 | LR: 1.75e-05 | 4029.9ms/step | 14.9 samples/s
+Step 44520/50462 | Loss: 2.5118 | LR: 1.74e-05 | 2729.0ms/step | 22.0 samples/s
+Step 44530/50462 | Loss: 2.5405 | LR: 1.74e-05 | 2738.7ms/step | 21.9 samples/s
+Step 44540/50462 | Loss: 2.5486 | LR: 1.73e-05 | 2729.7ms/step | 22.0 samples/s
+Step 44550/50462 | Loss: 2.5361 | LR: 1.72e-05 | 2735.5ms/step | 21.9 samples/s
+Step 44560/50462 | Loss: 2.5530 | LR: 1.72e-05 | 2737.6ms/step | 21.9 samples/s
+Step 44570/50462 | Loss: 2.5121 | LR: 1.71e-05 | 2735.3ms/step | 21.9 samples/s
+Step 44580/50462 | Loss: 2.5581 | LR: 1.71e-05 | 2739.6ms/step | 21.9 samples/s
+Step 44590/50462 | Loss: 2.5213 | LR: 1.70e-05 | 2736.8ms/step | 21.9 samples/s
+Step 44600/50462 | Loss: 2.5308 | LR: 1.70e-05 | 3298.0ms/step | 18.2 samples/s
+Step 44610/50462 | Loss: 2.5637 | LR: 1.69e-05 | 2730.8ms/step | 22.0 samples/s
+Step 44620/50462 | Loss: 2.4685 | LR: 1.68e-05 | 2736.0ms/step | 21.9 samples/s
+Step 44630/50462 | Loss: 2.5105 | LR: 1.68e-05 | 2729.6ms/step | 22.0 samples/s
+Step 44640/50462 | Loss: 2.5276 | LR: 1.67e-05 | 2726.2ms/step | 22.0 samples/s
+Step 44650/50462 | Loss: 2.5107 | LR: 1.67e-05 | 2740.2ms/step | 21.9 samples/s
+Step 44660/50462 | Loss: 2.5093 | LR: 1.66e-05 | 2731.4ms/step | 22.0 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_resume41750_nowandb.log

@@ -0,0 +1,206 @@
+===== RESUME FROM CHECKPOINT-41750 ON 6GPU, WANDB OFF 2026-08-30T15:30:20+00:00 =====
+W0830 15:30:23.076000 1268 site-packages/torch/distributed/run.py:851] 
+W0830 15:30:23.076000 1268 site-packages/torch/distributed/run.py:851] *****************************************
+W0830 15:30:23.076000 1268 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0830 15:30:23.076000 1268 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: TrueCheckpoint activations: True
+
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        6
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504628
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750...
+Preserving checkpoint LR state for exact resume: current=[3.694238556405591e-05], schedule_base=[0.0005]
+Resumed from checkpoint at step 41750 (epoch 0, micro_step 623745)
+Resuming training from step 41750
+Starting training...
+Validation at step 41750: Loss = 2.5214, Perplexity = 12.45
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750
+Training complete!

epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_resume44500.log

@@ -0,0 +1,218 @@
+===== RESUME FROM CHECKPOINT-44500 ON 6GPU 2026-08-30T18:43:45+00:00 =====
+W0830 18:43:47.125000 123 site-packages/torch/distributed/run.py:851] 
+W0830 18:43:47.125000 123 site-packages/torch/distributed/run.py:851] *****************************************
+W0830 18:43:47.125000 123 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0830 18:43:47.125000 123 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-44500",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 71,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "resume_micro_step_override": null,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed71 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed71"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 71,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: True
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-44500
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        6
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       504628
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-44500...
+Preserving checkpoint LR state for exact resume: current=[1.753587702534784e-05], schedule_base=[0.0005]
+Resumed from checkpoint at step 44500 (epoch 0, micro_step 445000)
+Resuming training from step 44500
+Starting training...
+Validation at step 44500: Loss = 2.5206, Perplexity = 12.44
+Switching attention mode for step 44501: sparse_stochastic (configured sparse attention)
+Step 44510/50462 | Loss: 2.5355 | LR: 1.75e-05 | 58255.0ms/step | 1.0 samples/s
+Step 44520/50462 | Loss: 2.5104 | LR: 1.74e-05 | 2740.6ms/step | 21.9 samples/s
+Step 44530/50462 | Loss: 2.5406 | LR: 1.74e-05 | 2738.5ms/step | 21.9 samples/s
+Step 44540/50462 | Loss: 2.5488 | LR: 1.73e-05 | 2740.4ms/step | 21.9 samples/s
+Step 44550/50462 | Loss: 2.5363 | LR: 1.72e-05 | 2746.3ms/step | 21.8 samples/s
+Step 44560/50462 | Loss: 2.5526 | LR: 1.72e-05 | 2742.0ms/step | 21.9 samples/s
+Step 44570/50462 | Loss: 2.5119 | LR: 1.71e-05 | 2748.0ms/step | 21.8 samples/s
+Step 44580/50462 | Loss: 2.5587 | LR: 1.71e-05 | 2749.3ms/step | 21.8 samples/s
+Step 44590/50462 | Loss: 2.5203 | LR: 1.70e-05 | 2739.0ms/step | 21.9 samples/s
+Step 44600/50462 | Loss: 2.5312 | LR: 1.70e-05 | 2747.7ms/step | 21.8 samples/s
+Step 44610/50462 | Loss: 2.5633 | LR: 1.69e-05 | 2744.7ms/step | 21.9 samples/s
+Step 44620/50462 | Loss: 2.4670 | LR: 1.68e-05 | 2740.7ms/step | 21.9 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_autoresume.log

@@ -0,0 +1,1178 @@
+
+===== AUTO-RESUME seed72 FROM checkpoint-8250 2026-08-27T16:56:30Z =====
+W0827 16:56:32.778000 151 site-packages/torch/distributed/run.py:851] 
+W0827 16:56:32.778000 151 site-packages/torch/distributed/run.py:851] *****************************************
+W0827 16:56:32.778000 151 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0827 16:56:32.778000 151 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8250",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8250
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8250...
+Preserving checkpoint LR state for exact resume: current=[0.00047244094585975404], schedule_base=[0.0005]
+Resumed from checkpoint at step 8250 (epoch 0, micro_step 91245)
+Resuming training from step 8250
+Resuming wandb run z1i60278
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run z1i60278
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_165838-z1i60278
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed72 6gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/z1i60278
+Starting training...
+Validation at step 8250: Loss = 2.8611, Perplexity = 17.48
+wandb: WARNING Tried to log to step 8250 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 8251: sparse_stochastic (configured sparse attention)
+Step 8260/50462 | Loss: 2.7394 | LR: 4.72e-04 | 25210.2ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 8260 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8270/50462 | Loss: 2.7790 | LR: 4.72e-04 | 3991.4ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 8270 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8280/50462 | Loss: 2.7315 | LR: 4.72e-04 | 4020.8ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8280 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8290/50462 | Loss: 2.7680 | LR: 4.72e-04 | 4015.8ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8290 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8300/50462 | Loss: 2.7817 | LR: 4.72e-04 | 4016.7ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8300 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8310/50462 | Loss: 2.7250 | LR: 4.72e-04 | 4009.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 8310 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8320/50462 | Loss: 2.7829 | LR: 4.72e-04 | 4016.9ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8320 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8330/50462 | Loss: 2.7627 | LR: 4.72e-04 | 4009.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 8330 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8340/50462 | Loss: 2.7749 | LR: 4.72e-04 | 4038.2ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8340 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8350/50462 | Loss: 2.7762 | LR: 4.72e-04 | 4017.8ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8350 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8360/50462 | Loss: 2.7303 | LR: 4.72e-04 | 4024.9ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8360 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8370/50462 | Loss: 2.7964 | LR: 4.72e-04 | 4015.3ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8370 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8380/50462 | Loss: 2.7636 | LR: 4.71e-04 | 4016.7ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8380 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8390/50462 | Loss: 2.7618 | LR: 4.71e-04 | 4010.3ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 8390 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8400/50462 | Loss: 2.7211 | LR: 4.71e-04 | 4020.6ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8400 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8410/50462 | Loss: 2.7478 | LR: 4.71e-04 | 4009.5ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 8410 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8420/50462 | Loss: 2.7156 | LR: 4.71e-04 | 4014.7ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8420 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8430/50462 | Loss: 2.7761 | LR: 4.71e-04 | 4013.9ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8430 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8440/50462 | Loss: 2.7260 | LR: 4.71e-04 | 4007.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 8440 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8450/50462 | Loss: 2.7464 | LR: 4.71e-04 | 4007.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 8450 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8460/50462 | Loss: 2.7657 | LR: 4.71e-04 | 4015.8ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8460 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8470/50462 | Loss: 2.7412 | LR: 4.71e-04 | 4017.0ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8470 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8480/50462 | Loss: 2.7585 | LR: 4.71e-04 | 4021.3ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8480 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8490/50462 | Loss: 2.7542 | LR: 4.71e-04 | 3999.6ms/step | 15.0 samples/s
+Step 8500/50462 | Loss: 2.7541 | LR: 4.71e-04 | 4006.5ms/step | 15.0 samples/s
+Validation at step 8500: Loss = 2.8565, Perplexity = 17.40
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8500
+Step 8510/50462 | Loss: 2.7523 | LR: 4.71e-04 | 5919.3ms/step | 10.1 samples/s
+Step 8520/50462 | Loss: 2.7336 | LR: 4.70e-04 | 4008.2ms/step | 15.0 samples/s
+Step 8530/50462 | Loss: 2.7116 | LR: 4.70e-04 | 4008.3ms/step | 15.0 samples/s
+Step 8540/50462 | Loss: 2.7526 | LR: 4.70e-04 | 4029.1ms/step | 14.9 samples/s
+Step 8550/50462 | Loss: 2.7305 | LR: 4.70e-04 | 4015.4ms/step | 14.9 samples/s
+Step 8560/50462 | Loss: 2.7267 | LR: 4.70e-04 | 4018.8ms/step | 14.9 samples/s
+Step 8570/50462 | Loss: 2.7103 | LR: 4.70e-04 | 4002.2ms/step | 15.0 samples/s
+Step 8580/50462 | Loss: 2.7249 | LR: 4.70e-04 | 3995.4ms/step | 15.0 samples/s
+Step 8590/50462 | Loss: 2.7633 | LR: 4.70e-04 | 4011.9ms/step | 15.0 samples/s
+Step 8600/50462 | Loss: 2.7553 | LR: 4.70e-04 | 4015.2ms/step | 14.9 samples/s
+Step 8610/50462 | Loss: 2.7673 | LR: 4.70e-04 | 4012.2ms/step | 15.0 samples/s
+Step 8620/50462 | Loss: 2.7474 | LR: 4.70e-04 | 4012.7ms/step | 15.0 samples/s
+Step 8630/50462 | Loss: 2.7741 | LR: 4.70e-04 | 4013.2ms/step | 15.0 samples/s
+Step 8640/50462 | Loss: 2.7566 | LR: 4.70e-04 | 4021.0ms/step | 14.9 samples/s
+Step 8650/50462 | Loss: 2.7237 | LR: 4.69e-04 | 4015.9ms/step | 14.9 samples/s
+Step 8660/50462 | Loss: 2.7136 | LR: 4.69e-04 | 4014.4ms/step | 14.9 samples/s
+Step 8670/50462 | Loss: 2.8036 | LR: 4.69e-04 | 4024.7ms/step | 14.9 samples/s
+Step 8680/50462 | Loss: 2.8222 | LR: 4.69e-04 | 4029.9ms/step | 14.9 samples/s
+Step 8690/50462 | Loss: 2.8280 | LR: 4.69e-04 | 4020.4ms/step | 14.9 samples/s
+Step 8700/50462 | Loss: 2.8465 | LR: 4.69e-04 | 4024.2ms/step | 14.9 samples/s
+Step 8710/50462 | Loss: 2.8272 | LR: 4.69e-04 | 4015.9ms/step | 14.9 samples/s
+Step 8720/50462 | Loss: 2.8590 | LR: 4.69e-04 | 4022.5ms/step | 14.9 samples/s
+Step 8730/50462 | Loss: 2.8163 | LR: 4.69e-04 | 4018.4ms/step | 14.9 samples/s
+Step 8740/50462 | Loss: 2.8061 | LR: 4.69e-04 | 4008.2ms/step | 15.0 samples/s
+Step 8750/50462 | Loss: 2.8217 | LR: 4.69e-04 | 4016.9ms/step | 14.9 samples/s
+Validation at step 8750: Loss = 2.8455, Perplexity = 17.21
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8750
+Step 8760/50462 | Loss: 2.8314 | LR: 4.69e-04 | 5884.7ms/step | 10.2 samples/s
+Step 8770/50462 | Loss: 2.8724 | LR: 4.69e-04 | 4015.4ms/step | 14.9 samples/s
+Step 8780/50462 | Loss: 2.8087 | LR: 4.68e-04 | 4021.4ms/step | 14.9 samples/s
+Step 8790/50462 | Loss: 2.7953 | LR: 4.68e-04 | 4009.5ms/step | 15.0 samples/s
+Step 8800/50462 | Loss: 2.8772 | LR: 4.68e-04 | 4021.5ms/step | 14.9 samples/s
+Step 8810/50462 | Loss: 2.8346 | LR: 4.68e-04 | 4009.5ms/step | 15.0 samples/s
+Step 8820/50462 | Loss: 2.8667 | LR: 4.68e-04 | 4011.9ms/step | 15.0 samples/s
+Step 8830/50462 | Loss: 2.8352 | LR: 4.68e-04 | 4018.1ms/step | 14.9 samples/s
+Step 8840/50462 | Loss: 2.8236 | LR: 4.68e-04 | 4006.4ms/step | 15.0 samples/s
+Step 8850/50462 | Loss: 2.8225 | LR: 4.68e-04 | 4020.6ms/step | 14.9 samples/s
+Step 8860/50462 | Loss: 2.8112 | LR: 4.68e-04 | 4011.9ms/step | 15.0 samples/s
+Step 8870/50462 | Loss: 2.8148 | LR: 4.68e-04 | 4022.2ms/step | 14.9 samples/s
+Step 8880/50462 | Loss: 2.7695 | LR: 4.68e-04 | 4017.1ms/step | 14.9 samples/s
+Step 8890/50462 | Loss: 2.8272 | LR: 4.68e-04 | 4013.3ms/step | 15.0 samples/s
+Step 8900/50462 | Loss: 2.8745 | LR: 4.68e-04 | 4007.8ms/step | 15.0 samples/s
+Step 8910/50462 | Loss: 2.8406 | LR: 4.67e-04 | 4009.2ms/step | 15.0 samples/s
+Step 8920/50462 | Loss: 2.7976 | LR: 4.67e-04 | 4011.9ms/step | 15.0 samples/s
+Step 8930/50462 | Loss: 2.7960 | LR: 4.67e-04 | 4014.9ms/step | 14.9 samples/s
+Step 8940/50462 | Loss: 2.8124 | LR: 4.67e-04 | 4015.2ms/step | 14.9 samples/s
+Step 8950/50462 | Loss: 2.8485 | LR: 4.67e-04 | 4019.0ms/step | 14.9 samples/s
+Step 8960/50462 | Loss: 2.8009 | LR: 4.67e-04 | 4017.8ms/step | 14.9 samples/s
+Step 8970/50462 | Loss: 2.8190 | LR: 4.67e-04 | 4016.7ms/step | 14.9 samples/s
+Step 8980/50462 | Loss: 2.8171 | LR: 4.67e-04 | 4007.4ms/step | 15.0 samples/s
+Step 8990/50462 | Loss: 2.8261 | LR: 4.67e-04 | 4018.3ms/step | 14.9 samples/s
+Step 9000/50462 | Loss: 2.8473 | LR: 4.67e-04 | 4007.5ms/step | 15.0 samples/s
+Validation at step 9000: Loss = 2.8357, Perplexity = 17.04
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-9000
+Step 9010/50462 | Loss: 2.8311 | LR: 4.67e-04 | 5880.5ms/step | 10.2 samples/s
+Step 9020/50462 | Loss: 2.8378 | LR: 4.67e-04 | 4011.7ms/step | 15.0 samples/s
+Step 9030/50462 | Loss: 2.8148 | LR: 4.67e-04 | 4005.1ms/step | 15.0 samples/s
+Step 9040/50462 | Loss: 2.7948 | LR: 4.66e-04 | 4015.1ms/step | 14.9 samples/s
+Step 9050/50462 | Loss: 2.8323 | LR: 4.66e-04 | 4024.6ms/step | 14.9 samples/s
+Step 9060/50462 | Loss: 2.8213 | LR: 4.66e-04 | 4019.5ms/step | 14.9 samples/s
+Step 9070/50462 | Loss: 2.8456 | LR: 4.66e-04 | 4018.8ms/step | 14.9 samples/s
+Step 9080/50462 | Loss: 2.8299 | LR: 4.66e-04 | 4008.9ms/step | 15.0 samples/s
+Step 9090/50462 | Loss: 2.7960 | LR: 4.66e-04 | 4004.9ms/step | 15.0 samples/s
+Step 9100/50462 | Loss: 2.7983 | LR: 4.66e-04 | 4015.7ms/step | 14.9 samples/s
+Step 9110/50462 | Loss: 2.8215 | LR: 4.66e-04 | 4012.0ms/step | 15.0 samples/s
+Step 9120/50462 | Loss: 2.7971 | LR: 4.66e-04 | 4013.7ms/step | 14.9 samples/s
+Step 9130/50462 | Loss: 2.7712 | LR: 4.66e-04 | 4028.7ms/step | 14.9 samples/s
+Step 9140/50462 | Loss: 2.8209 | LR: 4.66e-04 | 4012.0ms/step | 15.0 samples/s
+Step 9150/50462 | Loss: 2.8109 | LR: 4.66e-04 | 4009.6ms/step | 15.0 samples/s
+Step 9160/50462 | Loss: 2.8104 | LR: 4.66e-04 | 4021.8ms/step | 14.9 samples/s
+Step 9170/50462 | Loss: 2.7905 | LR: 4.65e-04 | 4007.8ms/step | 15.0 samples/s
+Step 9180/50462 | Loss: 2.8504 | LR: 4.65e-04 | 4033.6ms/step | 14.9 samples/s
+Step 9190/50462 | Loss: 2.8212 | LR: 4.65e-04 | 4004.8ms/step | 15.0 samples/s
+Step 9200/50462 | Loss: 2.8102 | LR: 4.65e-04 | 4023.9ms/step | 14.9 samples/s
+Step 9210/50462 | Loss: 2.8253 | LR: 4.65e-04 | 4020.1ms/step | 14.9 samples/s
+Step 9220/50462 | Loss: 2.8055 | LR: 4.65e-04 | 4013.8ms/step | 14.9 samples/s
+Step 9230/50462 | Loss: 2.7812 | LR: 4.65e-04 | 4015.6ms/step | 14.9 samples/s
+Step 9240/50462 | Loss: 2.8088 | LR: 4.65e-04 | 4023.0ms/step | 14.9 samples/s
+Step 9250/50462 | Loss: 2.8111 | LR: 4.65e-04 | 4014.6ms/step | 14.9 samples/s
+Validation at step 9250: Loss = 2.8286, Perplexity = 16.92
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-9250
+Step 9260/50462 | Loss: 2.8137 | LR: 4.65e-04 | 5883.7ms/step | 10.2 samples/s
+Step 9270/50462 | Loss: 2.7981 | LR: 4.65e-04 | 4009.0ms/step | 15.0 samples/s
+Step 9280/50462 | Loss: 2.7995 | LR: 4.65e-04 | 3999.5ms/step | 15.0 samples/s
+Step 9290/50462 | Loss: 2.7876 | LR: 4.64e-04 | 4025.9ms/step | 14.9 samples/s
+Step 9300/50462 | Loss: 2.8177 | LR: 4.64e-04 | 4021.1ms/step | 14.9 samples/s
+Step 9310/50462 | Loss: 2.8394 | LR: 4.64e-04 | 4004.6ms/step | 15.0 samples/s
+Step 9320/50462 | Loss: 2.8334 | LR: 4.64e-04 | 4027.7ms/step | 14.9 samples/s
+Step 9330/50462 | Loss: 2.8372 | LR: 4.64e-04 | 4022.2ms/step | 14.9 samples/s
+Step 9340/50462 | Loss: 2.7949 | LR: 4.64e-04 | 4023.2ms/step | 14.9 samples/s
+Step 9350/50462 | Loss: 2.7918 | LR: 4.64e-04 | 4008.0ms/step | 15.0 samples/s
+Step 9360/50462 | Loss: 2.8171 | LR: 4.64e-04 | 4012.9ms/step | 15.0 samples/s
+Step 9370/50462 | Loss: 2.8048 | LR: 4.64e-04 | 4011.2ms/step | 15.0 samples/s
+Step 9380/50462 | Loss: 2.8022 | LR: 4.64e-04 | 4005.3ms/step | 15.0 samples/s
+Step 9390/50462 | Loss: 2.7919 | LR: 4.64e-04 | 4014.2ms/step | 14.9 samples/s
+Step 9400/50462 | Loss: 2.7971 | LR: 4.64e-04 | 4009.3ms/step | 15.0 samples/s
+Step 9410/50462 | Loss: 2.8152 | LR: 4.63e-04 | 4019.5ms/step | 14.9 samples/s
+Step 9420/50462 | Loss: 2.8324 | LR: 4.63e-04 | 4004.6ms/step | 15.0 samples/s
+Step 9430/50462 | Loss: 2.8018 | LR: 4.63e-04 | 4020.3ms/step | 14.9 samples/s
+Step 9440/50462 | Loss: 2.8072 | LR: 4.63e-04 | 4019.4ms/step | 14.9 samples/s
+Step 9450/50462 | Loss: 2.8193 | LR: 4.63e-04 | 4015.0ms/step | 14.9 samples/s
+Step 9460/50462 | Loss: 2.8045 | LR: 4.63e-04 | 4022.0ms/step | 14.9 samples/s
+Step 9470/50462 | Loss: 2.8386 | LR: 4.63e-04 | 4018.2ms/step | 14.9 samples/s
+Step 9480/50462 | Loss: 2.8338 | LR: 4.63e-04 | 4019.8ms/step | 14.9 samples/s
+Step 9490/50462 | Loss: 2.7984 | LR: 4.63e-04 | 4011.7ms/step | 15.0 samples/s
+Step 9500/50462 | Loss: 2.8064 | LR: 4.63e-04 | 4014.3ms/step | 14.9 samples/s
+Validation at step 9500: Loss = 2.8192, Perplexity = 16.76
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-9500
+Step 9510/50462 | Loss: 2.8204 | LR: 4.63e-04 | 5898.9ms/step | 10.2 samples/s
+Step 9520/50462 | Loss: 2.8014 | LR: 4.63e-04 | 4012.8ms/step | 15.0 samples/s
+Step 9530/50462 | Loss: 2.7959 | LR: 4.62e-04 | 4010.5ms/step | 15.0 samples/s
+Step 9540/50462 | Loss: 2.8346 | LR: 4.62e-04 | 4005.9ms/step | 15.0 samples/s
+Step 9550/50462 | Loss: 2.7763 | LR: 4.62e-04 | 4011.1ms/step | 15.0 samples/s
+Step 9560/50462 | Loss: 2.7748 | LR: 4.62e-04 | 4019.2ms/step | 14.9 samples/s
+Step 9570/50462 | Loss: 2.8238 | LR: 4.62e-04 | 4013.3ms/step | 15.0 samples/s
+Step 9580/50462 | Loss: 2.7983 | LR: 4.62e-04 | 4021.7ms/step | 14.9 samples/s
+Step 9590/50462 | Loss: 2.8049 | LR: 4.62e-04 | 4018.6ms/step | 14.9 samples/s
+Step 9600/50462 | Loss: 2.7710 | LR: 4.62e-04 | 4009.2ms/step | 15.0 samples/s
+Step 9610/50462 | Loss: 2.7836 | LR: 4.62e-04 | 4015.4ms/step | 14.9 samples/s
+Step 9620/50462 | Loss: 2.7863 | LR: 4.62e-04 | 4017.3ms/step | 14.9 samples/s
+Step 9630/50462 | Loss: 2.8027 | LR: 4.62e-04 | 4017.6ms/step | 14.9 samples/s
+Step 9640/50462 | Loss: 2.7988 | LR: 4.62e-04 | 4010.6ms/step | 15.0 samples/s
+Step 9650/50462 | Loss: 2.8145 | LR: 4.61e-04 | 4020.0ms/step | 14.9 samples/s
+Step 9660/50462 | Loss: 2.8111 | LR: 4.61e-04 | 4013.2ms/step | 15.0 samples/s
+Step 9670/50462 | Loss: 2.8097 | LR: 4.61e-04 | 4017.9ms/step | 14.9 samples/s
+Step 9680/50462 | Loss: 2.7806 | LR: 4.61e-04 | 4017.8ms/step | 14.9 samples/s
+Step 9690/50462 | Loss: 2.8079 | LR: 4.61e-04 | 4012.1ms/step | 15.0 samples/s
+Step 9700/50462 | Loss: 2.8260 | LR: 4.61e-04 | 4023.9ms/step | 14.9 samples/s
+Step 9710/50462 | Loss: 2.7815 | LR: 4.61e-04 | 4016.6ms/step | 14.9 samples/s
+Step 9720/50462 | Loss: 2.8069 | LR: 4.61e-04 | 4017.2ms/step | 14.9 samples/s
+Step 9730/50462 | Loss: 2.8046 | LR: 4.61e-04 | 4016.4ms/step | 14.9 samples/s
+Step 9740/50462 | Loss: 2.7809 | LR: 4.61e-04 | 4022.8ms/step | 14.9 samples/s
+Step 9750/50462 | Loss: 2.7793 | LR: 4.61e-04 | 4003.0ms/step | 15.0 samples/s
+Validation at step 9750: Loss = 2.8154, Perplexity = 16.70
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-9750
+Step 9760/50462 | Loss: 2.8017 | LR: 4.61e-04 | 5874.9ms/step | 10.2 samples/s
+Step 9770/50462 | Loss: 2.7777 | LR: 4.60e-04 | 4014.6ms/step | 14.9 samples/s
+Step 9780/50462 | Loss: 2.7877 | LR: 4.60e-04 | 4011.5ms/step | 15.0 samples/s
+Step 9790/50462 | Loss: 2.7952 | LR: 4.60e-04 | 4003.3ms/step | 15.0 samples/s
+Step 9800/50462 | Loss: 2.7946 | LR: 4.60e-04 | 4023.7ms/step | 14.9 samples/s
+Step 9810/50462 | Loss: 2.7983 | LR: 4.60e-04 | 4010.8ms/step | 15.0 samples/s
+Step 9820/50462 | Loss: 2.7709 | LR: 4.60e-04 | 4000.0ms/step | 15.0 samples/s
+Step 9830/50462 | Loss: 2.7854 | LR: 4.60e-04 | 4016.0ms/step | 14.9 samples/s
+Step 9840/50462 | Loss: 2.7656 | LR: 4.60e-04 | 4010.2ms/step | 15.0 samples/s
+Step 9850/50462 | Loss: 2.8045 | LR: 4.60e-04 | 3999.0ms/step | 15.0 samples/s
+Step 9860/50462 | Loss: 2.7647 | LR: 4.60e-04 | 4006.5ms/step | 15.0 samples/s
+Step 9870/50462 | Loss: 2.7849 | LR: 4.60e-04 | 4002.6ms/step | 15.0 samples/s
+Step 9880/50462 | Loss: 2.8040 | LR: 4.60e-04 | 4011.2ms/step | 15.0 samples/s
+Step 9890/50462 | Loss: 2.7795 | LR: 4.59e-04 | 4005.4ms/step | 15.0 samples/s
+Step 9900/50462 | Loss: 2.7902 | LR: 4.59e-04 | 4005.3ms/step | 15.0 samples/s
+Step 9910/50462 | Loss: 2.8048 | LR: 4.59e-04 | 4003.8ms/step | 15.0 samples/s
+Step 9920/50462 | Loss: 2.8062 | LR: 4.59e-04 | 4017.1ms/step | 14.9 samples/s
+Step 9930/50462 | Loss: 2.7986 | LR: 4.59e-04 | 4018.9ms/step | 14.9 samples/s
+Step 9940/50462 | Loss: 2.7825 | LR: 4.59e-04 | 4012.5ms/step | 15.0 samples/s
+Step 9950/50462 | Loss: 2.7723 | LR: 4.59e-04 | 4013.2ms/step | 15.0 samples/s
+Step 9960/50462 | Loss: 2.7699 | LR: 4.59e-04 | 4020.7ms/step | 14.9 samples/s
+Step 9970/50462 | Loss: 2.7890 | LR: 4.59e-04 | 4020.8ms/step | 14.9 samples/s
+Step 9980/50462 | Loss: 2.7719 | LR: 4.59e-04 | 4005.5ms/step | 15.0 samples/s
+Step 9990/50462 | Loss: 2.8018 | LR: 4.59e-04 | 4009.5ms/step | 15.0 samples/s
+Step 10000/50462 | Loss: 2.8052 | LR: 4.58e-04 | 4000.1ms/step | 15.0 samples/s
+Validation at step 10000: Loss = 2.8076, Perplexity = 16.57
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-10000
+Step 10010/50462 | Loss: 2.7814 | LR: 4.58e-04 | 5878.6ms/step | 10.2 samples/s
+Step 10020/50462 | Loss: 2.8112 | LR: 4.58e-04 | 4012.9ms/step | 15.0 samples/s
+Step 10030/50462 | Loss: 2.7894 | LR: 4.58e-04 | 4002.4ms/step | 15.0 samples/s
+Step 10040/50462 | Loss: 2.8066 | LR: 4.58e-04 | 4009.1ms/step | 15.0 samples/s
+Step 10050/50462 | Loss: 2.8003 | LR: 4.58e-04 | 4011.7ms/step | 15.0 samples/s
+Step 10060/50462 | Loss: 2.7596 | LR: 4.58e-04 | 4001.0ms/step | 15.0 samples/s
+Step 10070/50462 | Loss: 2.7947 | LR: 4.58e-04 | 4009.9ms/step | 15.0 samples/s
+Step 10080/50462 | Loss: 2.7670 | LR: 4.58e-04 | 4011.4ms/step | 15.0 samples/s
+Step 10090/50462 | Loss: 2.7960 | LR: 4.58e-04 | 4004.5ms/step | 15.0 samples/s
+Step 10100/50462 | Loss: 2.7781 | LR: 4.58e-04 | 4015.6ms/step | 14.9 samples/s
+Step 10110/50462 | Loss: 2.8010 | LR: 4.58e-04 | 4004.4ms/step | 15.0 samples/s
+Step 10120/50462 | Loss: 2.7916 | LR: 4.57e-04 | 4008.9ms/step | 15.0 samples/s
+Step 10130/50462 | Loss: 2.7793 | LR: 4.57e-04 | 4002.4ms/step | 15.0 samples/s
+Step 10140/50462 | Loss: 2.7984 | LR: 4.57e-04 | 3999.9ms/step | 15.0 samples/s
+Step 10150/50462 | Loss: 2.7354 | LR: 4.57e-04 | 4003.4ms/step | 15.0 samples/s
+Step 10160/50462 | Loss: 2.7629 | LR: 4.57e-04 | 4004.8ms/step | 15.0 samples/s
+Step 10170/50462 | Loss: 2.7799 | LR: 4.57e-04 | 4004.7ms/step | 15.0 samples/s
+Step 10180/50462 | Loss: 2.7932 | LR: 4.57e-04 | 4012.8ms/step | 15.0 samples/s
+Step 10190/50462 | Loss: 2.7606 | LR: 4.57e-04 | 4014.2ms/step | 14.9 samples/s
+Step 10200/50462 | Loss: 2.7596 | LR: 4.57e-04 | 4013.4ms/step | 14.9 samples/s
+Step 10210/50462 | Loss: 2.8070 | LR: 4.57e-04 | 4017.8ms/step | 14.9 samples/s
+Step 10220/50462 | Loss: 2.7870 | LR: 4.57e-04 | 4020.3ms/step | 14.9 samples/s
+Step 10230/50462 | Loss: 2.8149 | LR: 4.56e-04 | 4018.6ms/step | 14.9 samples/s
+Step 10240/50462 | Loss: 2.8051 | LR: 4.56e-04 | 4009.7ms/step | 15.0 samples/s
+Step 10250/50462 | Loss: 2.7936 | LR: 4.56e-04 | 4005.4ms/step | 15.0 samples/s
+Validation at step 10250: Loss = 2.8017, Perplexity = 16.47
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-10250
+Step 10260/50462 | Loss: 2.7959 | LR: 4.56e-04 | 5905.0ms/step | 10.2 samples/s
+Step 10270/50462 | Loss: 2.7910 | LR: 4.56e-04 | 4003.2ms/step | 15.0 samples/s
+Step 10280/50462 | Loss: 2.7763 | LR: 4.56e-04 | 4013.8ms/step | 14.9 samples/s
+Step 10290/50462 | Loss: 2.7785 | LR: 4.56e-04 | 4014.9ms/step | 14.9 samples/s
+Step 10300/50462 | Loss: 2.7828 | LR: 4.56e-04 | 4007.6ms/step | 15.0 samples/s
+Step 10310/50462 | Loss: 2.7958 | LR: 4.56e-04 | 4011.4ms/step | 15.0 samples/s
+Step 10320/50462 | Loss: 2.7848 | LR: 4.56e-04 | 4008.6ms/step | 15.0 samples/s
+Step 10330/50462 | Loss: 2.7910 | LR: 4.56e-04 | 4014.5ms/step | 14.9 samples/s
+Step 10340/50462 | Loss: 2.7757 | LR: 4.55e-04 | 4010.6ms/step | 15.0 samples/s
+Step 10350/50462 | Loss: 2.7573 | LR: 4.55e-04 | 4007.0ms/step | 15.0 samples/s
+Step 10360/50462 | Loss: 2.7826 | LR: 4.55e-04 | 4005.7ms/step | 15.0 samples/s
+Step 10370/50462 | Loss: 2.7906 | LR: 4.55e-04 | 4013.8ms/step | 14.9 samples/s
+Step 10380/50462 | Loss: 2.8095 | LR: 4.55e-04 | 4008.1ms/step | 15.0 samples/s
+Step 10390/50462 | Loss: 2.7638 | LR: 4.55e-04 | 4017.4ms/step | 14.9 samples/s
+Step 10400/50462 | Loss: 2.7800 | LR: 4.55e-04 | 4016.9ms/step | 14.9 samples/s
+Step 10410/50462 | Loss: 2.7536 | LR: 4.55e-04 | 4005.4ms/step | 15.0 samples/s
+Step 10420/50462 | Loss: 2.7792 | LR: 4.55e-04 | 4007.4ms/step | 15.0 samples/s
+Step 10430/50462 | Loss: 2.7685 | LR: 4.55e-04 | 3999.4ms/step | 15.0 samples/s
+Step 10440/50462 | Loss: 2.7627 | LR: 4.55e-04 | 4006.5ms/step | 15.0 samples/s
+Step 10450/50462 | Loss: 2.8025 | LR: 4.54e-04 | 4011.8ms/step | 15.0 samples/s
+Step 10460/50462 | Loss: 2.7610 | LR: 4.54e-04 | 3999.2ms/step | 15.0 samples/s
+Step 10470/50462 | Loss: 2.7901 | LR: 4.54e-04 | 4004.7ms/step | 15.0 samples/s
+Step 10480/50462 | Loss: 2.7906 | LR: 4.54e-04 | 4018.9ms/step | 14.9 samples/s
+Step 10490/50462 | Loss: 2.7912 | LR: 4.54e-04 | 4011.5ms/step | 15.0 samples/s
+Step 10500/50462 | Loss: 2.7794 | LR: 4.54e-04 | 3998.6ms/step | 15.0 samples/s
+Validation at step 10500: Loss = 2.7957, Perplexity = 16.37
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-10500
+Step 10510/50462 | Loss: 2.7576 | LR: 4.54e-04 | 5884.5ms/step | 10.2 samples/s
+Step 10520/50462 | Loss: 2.7676 | LR: 4.54e-04 | 4014.0ms/step | 14.9 samples/s
+Step 10530/50462 | Loss: 2.7977 | LR: 4.54e-04 | 4012.0ms/step | 15.0 samples/s
+Step 10540/50462 | Loss: 2.7865 | LR: 4.54e-04 | 4020.6ms/step | 14.9 samples/s
+Step 10550/50462 | Loss: 2.8009 | LR: 4.54e-04 | 4022.8ms/step | 14.9 samples/s
+Step 10560/50462 | Loss: 2.7620 | LR: 4.53e-04 | 4016.7ms/step | 14.9 samples/s
+Step 10570/50462 | Loss: 2.7682 | LR: 4.53e-04 | 4022.1ms/step | 14.9 samples/s
+Step 10580/50462 | Loss: 2.7385 | LR: 4.53e-04 | 4009.0ms/step | 15.0 samples/s
+Step 10590/50462 | Loss: 2.7896 | LR: 4.53e-04 | 4009.3ms/step | 15.0 samples/s
+Step 10600/50462 | Loss: 2.7033 | LR: 4.53e-04 | 4010.1ms/step | 15.0 samples/s
+Step 10610/50462 | Loss: 2.7826 | LR: 4.53e-04 | 4001.9ms/step | 15.0 samples/s
+Step 10620/50462 | Loss: 2.7800 | LR: 4.53e-04 | 4010.0ms/step | 15.0 samples/s
+Step 10630/50462 | Loss: 2.7960 | LR: 4.53e-04 | 4014.3ms/step | 14.9 samples/s
+Step 10640/50462 | Loss: 2.8079 | LR: 4.53e-04 | 4016.8ms/step | 14.9 samples/s
+Step 10650/50462 | Loss: 2.8131 | LR: 4.53e-04 | 4008.5ms/step | 15.0 samples/s
+Step 10660/50462 | Loss: 2.7649 | LR: 4.53e-04 | 4016.3ms/step | 14.9 samples/s
+Step 10670/50462 | Loss: 2.7893 | LR: 4.52e-04 | 4015.3ms/step | 14.9 samples/s
+Step 10680/50462 | Loss: 2.7893 | LR: 4.52e-04 | 4006.5ms/step | 15.0 samples/s
+Step 10690/50462 | Loss: 2.7935 | LR: 4.52e-04 | 4012.1ms/step | 15.0 samples/s
+Step 10700/50462 | Loss: 2.7568 | LR: 4.52e-04 | 4009.1ms/step | 15.0 samples/s
+Step 10710/50462 | Loss: 2.7938 | LR: 4.52e-04 | 4006.9ms/step | 15.0 samples/s
+Step 10720/50462 | Loss: 2.7717 | LR: 4.52e-04 | 4011.7ms/step | 15.0 samples/s
+Step 10730/50462 | Loss: 2.7429 | LR: 4.52e-04 | 4008.9ms/step | 15.0 samples/s
+Step 10740/50462 | Loss: 2.8088 | LR: 4.52e-04 | 4009.2ms/step | 15.0 samples/s
+Step 10750/50462 | Loss: 2.7259 | LR: 4.52e-04 | 4014.8ms/step | 14.9 samples/s
+Validation at step 10750: Loss = 2.7898, Perplexity = 16.28
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-10750
+Step 10760/50462 | Loss: 2.7912 | LR: 4.52e-04 | 5872.7ms/step | 10.2 samples/s
+Step 10770/50462 | Loss: 2.7486 | LR: 4.52e-04 | 4014.0ms/step | 14.9 samples/s
+Step 10780/50462 | Loss: 2.7776 | LR: 4.51e-04 | 4009.5ms/step | 15.0 samples/s
+Step 10790/50462 | Loss: 2.7759 | LR: 4.51e-04 | 4012.7ms/step | 15.0 samples/s
+Step 10800/50462 | Loss: 2.7516 | LR: 4.51e-04 | 4012.6ms/step | 15.0 samples/s
+Step 10810/50462 | Loss: 2.7922 | LR: 4.51e-04 | 4009.0ms/step | 15.0 samples/s
+Step 10820/50462 | Loss: 2.7786 | LR: 4.51e-04 | 4005.9ms/step | 15.0 samples/s
+Step 10830/50462 | Loss: 2.7668 | LR: 4.51e-04 | 4004.2ms/step | 15.0 samples/s
+Step 10840/50462 | Loss: 2.7570 | LR: 4.51e-04 | 4001.8ms/step | 15.0 samples/s
+Step 10850/50462 | Loss: 2.7805 | LR: 4.51e-04 | 4005.3ms/step | 15.0 samples/s
+Step 10860/50462 | Loss: 2.7984 | LR: 4.51e-04 | 4007.3ms/step | 15.0 samples/s
+Step 10870/50462 | Loss: 2.7621 | LR: 4.51e-04 | 4013.3ms/step | 15.0 samples/s
+Step 10880/50462 | Loss: 2.7772 | LR: 4.50e-04 | 4003.8ms/step | 15.0 samples/s
+Step 10890/50462 | Loss: 2.7853 | LR: 4.50e-04 | 4014.3ms/step | 14.9 samples/s
+Step 10900/50462 | Loss: 2.7896 | LR: 4.50e-04 | 4015.2ms/step | 14.9 samples/s
+Step 10910/50462 | Loss: 2.7881 | LR: 4.50e-04 | 4021.2ms/step | 14.9 samples/s
+Step 10920/50462 | Loss: 2.7565 | LR: 4.50e-04 | 4003.5ms/step | 15.0 samples/s
+Step 10930/50462 | Loss: 2.7736 | LR: 4.50e-04 | 4013.0ms/step | 15.0 samples/s
+Step 10940/50462 | Loss: 2.7647 | LR: 4.50e-04 | 4014.8ms/step | 14.9 samples/s
+Step 10950/50462 | Loss: 2.7889 | LR: 4.50e-04 | 4014.8ms/step | 14.9 samples/s
+Step 10960/50462 | Loss: 2.7584 | LR: 4.50e-04 | 3999.7ms/step | 15.0 samples/s
+Step 10970/50462 | Loss: 2.7345 | LR: 4.50e-04 | 4005.9ms/step | 15.0 samples/s
+Step 10980/50462 | Loss: 2.7710 | LR: 4.50e-04 | 4012.8ms/step | 15.0 samples/s
+Step 10990/50462 | Loss: 2.7515 | LR: 4.49e-04 | 4005.4ms/step | 15.0 samples/s
+Step 11000/50462 | Loss: 2.7825 | LR: 4.49e-04 | 4008.6ms/step | 15.0 samples/s
+Validation at step 11000: Loss = 2.7824, Perplexity = 16.16
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-11000
+Step 11010/50462 | Loss: 2.7783 | LR: 4.49e-04 | 5915.7ms/step | 10.1 samples/s
+Step 11020/50462 | Loss: 2.7471 | LR: 4.49e-04 | 4017.0ms/step | 14.9 samples/s
+Step 11030/50462 | Loss: 2.7573 | LR: 4.49e-04 | 4017.0ms/step | 14.9 samples/s
+Step 11040/50462 | Loss: 2.7911 | LR: 4.49e-04 | 4014.4ms/step | 14.9 samples/s
+Step 11050/50462 | Loss: 2.7354 | LR: 4.49e-04 | 4003.5ms/step | 15.0 samples/s
+Step 11060/50462 | Loss: 2.7843 | LR: 4.49e-04 | 4008.4ms/step | 15.0 samples/s
+Step 11070/50462 | Loss: 2.7618 | LR: 4.49e-04 | 4006.2ms/step | 15.0 samples/s
+Step 11080/50462 | Loss: 2.7698 | LR: 4.49e-04 | 4009.6ms/step | 15.0 samples/s
+Step 11090/50462 | Loss: 2.7671 | LR: 4.48e-04 | 4008.7ms/step | 15.0 samples/s
+Step 11100/50462 | Loss: 2.7409 | LR: 4.48e-04 | 4022.7ms/step | 14.9 samples/s
+Step 11110/50462 | Loss: 2.7672 | LR: 4.48e-04 | 4009.6ms/step | 15.0 samples/s
+Step 11120/50462 | Loss: 2.7942 | LR: 4.48e-04 | 4008.8ms/step | 15.0 samples/s
+Step 11130/50462 | Loss: 2.7714 | LR: 4.48e-04 | 4017.4ms/step | 14.9 samples/s
+Step 11140/50462 | Loss: 2.7960 | LR: 4.48e-04 | 4004.7ms/step | 15.0 samples/s
+Step 11150/50462 | Loss: 2.7485 | LR: 4.48e-04 | 4013.4ms/step | 15.0 samples/s
+Step 11160/50462 | Loss: 2.7854 | LR: 4.48e-04 | 4003.0ms/step | 15.0 samples/s
+Step 11170/50462 | Loss: 2.7743 | LR: 4.48e-04 | 4000.9ms/step | 15.0 samples/s
+Step 11180/50462 | Loss: 2.7590 | LR: 4.48e-04 | 4011.2ms/step | 15.0 samples/s
+Step 11190/50462 | Loss: 2.8113 | LR: 4.48e-04 | 4004.9ms/step | 15.0 samples/s
+Step 11200/50462 | Loss: 2.7258 | LR: 4.47e-04 | 4022.2ms/step | 14.9 samples/s
+Step 11210/50462 | Loss: 2.7655 | LR: 4.47e-04 | 4009.7ms/step | 15.0 samples/s
+Step 11220/50462 | Loss: 2.7546 | LR: 4.47e-04 | 4005.6ms/step | 15.0 samples/s
+Step 11230/50462 | Loss: 2.7765 | LR: 4.47e-04 | 4020.3ms/step | 14.9 samples/s
+Step 11240/50462 | Loss: 2.7836 | LR: 4.47e-04 | 4007.3ms/step | 15.0 samples/s
+Step 11250/50462 | Loss: 2.7469 | LR: 4.47e-04 | 4009.8ms/step | 15.0 samples/s
+Validation at step 11250: Loss = 2.7799, Perplexity = 16.12
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-11250
+Step 11260/50462 | Loss: 2.7781 | LR: 4.47e-04 | 5936.5ms/step | 10.1 samples/s
+Step 11270/50462 | Loss: 2.7304 | LR: 4.47e-04 | 4002.8ms/step | 15.0 samples/s
+Step 11280/50462 | Loss: 2.7732 | LR: 4.47e-04 | 4005.5ms/step | 15.0 samples/s
+Step 11290/50462 | Loss: 2.7801 | LR: 4.47e-04 | 4001.9ms/step | 15.0 samples/s
+Step 11300/50462 | Loss: 2.7644 | LR: 4.46e-04 | 4007.1ms/step | 15.0 samples/s
+Step 11310/50462 | Loss: 2.7284 | LR: 4.46e-04 | 4019.2ms/step | 14.9 samples/s
+Step 11320/50462 | Loss: 2.7261 | LR: 4.46e-04 | 4016.7ms/step | 14.9 samples/s
+Step 11330/50462 | Loss: 2.7186 | LR: 4.46e-04 | 4009.4ms/step | 15.0 samples/s
+Step 11340/50462 | Loss: 2.7897 | LR: 4.46e-04 | 4008.4ms/step | 15.0 samples/s
+Step 11350/50462 | Loss: 2.7893 | LR: 4.46e-04 | 4010.3ms/step | 15.0 samples/s
+Step 11360/50462 | Loss: 2.7743 | LR: 4.46e-04 | 4017.9ms/step | 14.9 samples/s
+Step 11370/50462 | Loss: 2.7765 | LR: 4.46e-04 | 4017.3ms/step | 14.9 samples/s
+Step 11380/50462 | Loss: 2.7437 | LR: 4.46e-04 | 4011.3ms/step | 15.0 samples/s
+Step 11390/50462 | Loss: 2.7268 | LR: 4.46e-04 | 4011.8ms/step | 15.0 samples/s
+Step 11400/50462 | Loss: 2.7342 | LR: 4.45e-04 | 4001.2ms/step | 15.0 samples/s
+Step 11410/50462 | Loss: 2.7648 | LR: 4.45e-04 | 4018.2ms/step | 14.9 samples/s
+Step 11420/50462 | Loss: 2.7680 | LR: 4.45e-04 | 4004.9ms/step | 15.0 samples/s
+Step 11430/50462 | Loss: 2.7492 | LR: 4.45e-04 | 4011.5ms/step | 15.0 samples/s
+Step 11440/50462 | Loss: 2.7593 | LR: 4.45e-04 | 4006.3ms/step | 15.0 samples/s
+Step 11450/50462 | Loss: 2.7530 | LR: 4.45e-04 | 4007.6ms/step | 15.0 samples/s
+Step 11460/50462 | Loss: 2.7743 | LR: 4.45e-04 | 4009.2ms/step | 15.0 samples/s
+Step 11470/50462 | Loss: 2.7438 | LR: 4.45e-04 | 4005.1ms/step | 15.0 samples/s
+Step 11480/50462 | Loss: 2.7526 | LR: 4.45e-04 | 4001.5ms/step | 15.0 samples/s
+Step 11490/50462 | Loss: 2.7598 | LR: 4.45e-04 | 4003.0ms/step | 15.0 samples/s
+Step 11500/50462 | Loss: 2.7705 | LR: 4.44e-04 | 4011.6ms/step | 15.0 samples/s
+Validation at step 11500: Loss = 2.7733, Perplexity = 16.01
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-11500
+Step 11510/50462 | Loss: 2.7679 | LR: 4.44e-04 | 5882.1ms/step | 10.2 samples/s
+Step 11520/50462 | Loss: 2.7397 | LR: 4.44e-04 | 4008.5ms/step | 15.0 samples/s
+Step 11530/50462 | Loss: 2.7976 | LR: 4.44e-04 | 4012.6ms/step | 15.0 samples/s
+Step 11540/50462 | Loss: 2.7662 | LR: 4.44e-04 | 4017.5ms/step | 14.9 samples/s
+Step 11550/50462 | Loss: 2.7960 | LR: 4.44e-04 | 4026.0ms/step | 14.9 samples/s
+Step 11560/50462 | Loss: 2.7346 | LR: 4.44e-04 | 4011.0ms/step | 15.0 samples/s
+Step 11570/50462 | Loss: 2.7898 | LR: 4.44e-04 | 4020.8ms/step | 14.9 samples/s
+Step 11580/50462 | Loss: 2.7288 | LR: 4.44e-04 | 4021.6ms/step | 14.9 samples/s
+Step 11590/50462 | Loss: 2.7489 | LR: 4.44e-04 | 4009.9ms/step | 15.0 samples/s
+Step 11600/50462 | Loss: 2.7626 | LR: 4.43e-04 | 4023.6ms/step | 14.9 samples/s
+Step 11610/50462 | Loss: 2.7244 | LR: 4.43e-04 | 4008.5ms/step | 15.0 samples/s
+Step 11620/50462 | Loss: 2.7654 | LR: 4.43e-04 | 4014.3ms/step | 14.9 samples/s
+Step 11630/50462 | Loss: 2.7503 | LR: 4.43e-04 | 4012.2ms/step | 15.0 samples/s
+Step 11640/50462 | Loss: 2.7536 | LR: 4.43e-04 | 4008.1ms/step | 15.0 samples/s
+Step 11650/50462 | Loss: 2.7874 | LR: 4.43e-04 | 4009.7ms/step | 15.0 samples/s
+Step 11660/50462 | Loss: 2.7643 | LR: 4.43e-04 | 4029.2ms/step | 14.9 samples/s
+Step 11670/50462 | Loss: 2.7569 | LR: 4.43e-04 | 4006.3ms/step | 15.0 samples/s
+Step 11680/50462 | Loss: 2.7619 | LR: 4.43e-04 | 4009.1ms/step | 15.0 samples/s
+Step 11690/50462 | Loss: 2.7696 | LR: 4.43e-04 | 4024.1ms/step | 14.9 samples/s
+Step 11700/50462 | Loss: 2.7473 | LR: 4.42e-04 | 4008.6ms/step | 15.0 samples/s
+Step 11710/50462 | Loss: 2.7309 | LR: 4.42e-04 | 4015.9ms/step | 14.9 samples/s
+Step 11720/50462 | Loss: 2.7448 | LR: 4.42e-04 | 4017.9ms/step | 14.9 samples/s
+Step 11730/50462 | Loss: 2.7703 | LR: 4.42e-04 | 4018.0ms/step | 14.9 samples/s
+Step 11740/50462 | Loss: 2.7744 | LR: 4.42e-04 | 4025.2ms/step | 14.9 samples/s
+Step 11750/50462 | Loss: 2.7255 | LR: 4.42e-04 | 4016.0ms/step | 14.9 samples/s
+Validation at step 11750: Loss = 2.7701, Perplexity = 15.96
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-11750
+Step 11760/50462 | Loss: 2.7440 | LR: 4.42e-04 | 5875.5ms/step | 10.2 samples/s
+Step 11770/50462 | Loss: 2.7667 | LR: 4.42e-04 | 4002.7ms/step | 15.0 samples/s
+Step 11780/50462 | Loss: 2.7306 | LR: 4.42e-04 | 4006.8ms/step | 15.0 samples/s
+Step 11790/50462 | Loss: 2.7286 | LR: 4.42e-04 | 4010.0ms/step | 15.0 samples/s
+Step 11800/50462 | Loss: 2.7841 | LR: 4.41e-04 | 4003.7ms/step | 15.0 samples/s
+Step 11810/50462 | Loss: 2.7286 | LR: 4.41e-04 | 4017.4ms/step | 14.9 samples/s
+Step 11820/50462 | Loss: 2.7874 | LR: 4.41e-04 | 4009.7ms/step | 15.0 samples/s
+Step 11830/50462 | Loss: 2.7410 | LR: 4.41e-04 | 4012.0ms/step | 15.0 samples/s
+Step 11840/50462 | Loss: 2.7584 | LR: 4.41e-04 | 4011.7ms/step | 15.0 samples/s
+Step 11850/50462 | Loss: 2.7384 | LR: 4.41e-04 | 4011.2ms/step | 15.0 samples/s
+Step 11860/50462 | Loss: 2.7941 | LR: 4.41e-04 | 4021.8ms/step | 14.9 samples/s
+Step 11870/50462 | Loss: 2.7258 | LR: 4.41e-04 | 4046.0ms/step | 14.8 samples/s
+Step 11880/50462 | Loss: 2.7724 | LR: 4.41e-04 | 4039.2ms/step | 14.9 samples/s
+Step 11890/50462 | Loss: 2.7354 | LR: 4.41e-04 | 4046.5ms/step | 14.8 samples/s
+Step 11900/50462 | Loss: 2.7815 | LR: 4.40e-04 | 4027.8ms/step | 14.9 samples/s
+Step 11910/50462 | Loss: 2.7686 | LR: 4.40e-04 | 4036.6ms/step | 14.9 samples/s
+Step 11920/50462 | Loss: 2.7426 | LR: 4.40e-04 | 4013.7ms/step | 14.9 samples/s
+Step 11930/50462 | Loss: 2.7456 | LR: 4.40e-04 | 4025.7ms/step | 14.9 samples/s
+Step 11940/50462 | Loss: 2.7818 | LR: 4.40e-04 | 4016.7ms/step | 14.9 samples/s
+Step 11950/50462 | Loss: 2.7521 | LR: 4.40e-04 | 4022.3ms/step | 14.9 samples/s
+Step 11960/50462 | Loss: 2.7350 | LR: 4.40e-04 | 4022.2ms/step | 14.9 samples/s
+Step 11970/50462 | Loss: 2.7267 | LR: 4.40e-04 | 4025.4ms/step | 14.9 samples/s
+Step 11980/50462 | Loss: 2.7184 | LR: 4.40e-04 | 4018.3ms/step | 14.9 samples/s
+Step 11990/50462 | Loss: 2.7589 | LR: 4.40e-04 | 4011.1ms/step | 15.0 samples/s
+Step 12000/50462 | Loss: 2.7454 | LR: 4.39e-04 | 4028.9ms/step | 14.9 samples/s
+Validation at step 12000: Loss = 2.7675, Perplexity = 15.92
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-12000
+Step 12010/50462 | Loss: 2.7620 | LR: 4.39e-04 | 6115.2ms/step | 9.8 samples/s
+Step 12020/50462 | Loss: 2.7640 | LR: 4.39e-04 | 4025.0ms/step | 14.9 samples/s
+Step 12030/50462 | Loss: 2.7462 | LR: 4.39e-04 | 4022.9ms/step | 14.9 samples/s
+Step 12040/50462 | Loss: 2.7621 | LR: 4.39e-04 | 4025.6ms/step | 14.9 samples/s
+Step 12050/50462 | Loss: 2.7131 | LR: 4.39e-04 | 4021.7ms/step | 14.9 samples/s
+Step 12060/50462 | Loss: 2.7423 | LR: 4.39e-04 | 4003.1ms/step | 15.0 samples/s
+Step 12070/50462 | Loss: 2.7533 | LR: 4.39e-04 | 4021.1ms/step | 14.9 samples/s
+Step 12080/50462 | Loss: 2.7407 | LR: 4.39e-04 | 4027.2ms/step | 14.9 samples/s
+Step 12090/50462 | Loss: 2.7486 | LR: 4.39e-04 | 4021.3ms/step | 14.9 samples/s
+Step 12100/50462 | Loss: 2.7575 | LR: 4.38e-04 | 4024.1ms/step | 14.9 samples/s
+Step 12110/50462 | Loss: 2.7310 | LR: 4.38e-04 | 4019.5ms/step | 14.9 samples/s
+Step 12120/50462 | Loss: 2.7506 | LR: 4.38e-04 | 4019.5ms/step | 14.9 samples/s
+Step 12130/50462 | Loss: 2.7616 | LR: 4.38e-04 | 4018.2ms/step | 14.9 samples/s
+Step 12140/50462 | Loss: 2.7201 | LR: 4.38e-04 | 4019.9ms/step | 14.9 samples/s
+Step 12150/50462 | Loss: 2.7378 | LR: 4.38e-04 | 4012.1ms/step | 15.0 samples/s
+Step 12160/50462 | Loss: 2.7408 | LR: 4.38e-04 | 4019.2ms/step | 14.9 samples/s
+Step 12170/50462 | Loss: 2.7371 | LR: 4.38e-04 | 4013.5ms/step | 14.9 samples/s
+Step 12180/50462 | Loss: 2.7587 | LR: 4.38e-04 | 4012.9ms/step | 15.0 samples/s
+Step 12190/50462 | Loss: 2.7977 | LR: 4.37e-04 | 4016.0ms/step | 14.9 samples/s
+Step 12200/50462 | Loss: 2.7372 | LR: 4.37e-04 | 4019.4ms/step | 14.9 samples/s
+Step 12210/50462 | Loss: 2.7489 | LR: 4.37e-04 | 4020.0ms/step | 14.9 samples/s
+Step 12220/50462 | Loss: 2.7113 | LR: 4.37e-04 | 4016.4ms/step | 14.9 samples/s
+Step 12230/50462 | Loss: 2.7808 | LR: 4.37e-04 | 4009.6ms/step | 15.0 samples/s
+Step 12240/50462 | Loss: 2.7325 | LR: 4.37e-04 | 4009.6ms/step | 15.0 samples/s
+Step 12250/50462 | Loss: 2.7323 | LR: 4.37e-04 | 4012.3ms/step | 15.0 samples/s
+Validation at step 12250: Loss = 2.7616, Perplexity = 15.83
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-12250
+Step 12260/50462 | Loss: 2.7367 | LR: 4.37e-04 | 5988.3ms/step | 10.0 samples/s
+Step 12270/50462 | Loss: 2.7291 | LR: 4.37e-04 | 4025.9ms/step | 14.9 samples/s
+Step 12280/50462 | Loss: 2.7079 | LR: 4.37e-04 | 4008.2ms/step | 15.0 samples/s
+Step 12290/50462 | Loss: 2.7046 | LR: 4.36e-04 | 4017.8ms/step | 14.9 samples/s
+Step 12300/50462 | Loss: 2.7455 | LR: 4.36e-04 | 4019.8ms/step | 14.9 samples/s
+Step 12310/50462 | Loss: 2.7560 | LR: 4.36e-04 | 4021.6ms/step | 14.9 samples/s
+Step 12320/50462 | Loss: 2.7670 | LR: 4.36e-04 | 4024.2ms/step | 14.9 samples/s
+Step 12330/50462 | Loss: 2.7517 | LR: 4.36e-04 | 4010.5ms/step | 15.0 samples/s
+Step 12340/50462 | Loss: 2.7417 | LR: 4.36e-04 | 4010.9ms/step | 15.0 samples/s
+Step 12350/50462 | Loss: 2.7360 | LR: 4.36e-04 | 4005.8ms/step | 15.0 samples/s
+Step 12360/50462 | Loss: 2.7193 | LR: 4.36e-04 | 4028.1ms/step | 14.9 samples/s
+Step 12370/50462 | Loss: 2.7488 | LR: 4.36e-04 | 4019.4ms/step | 14.9 samples/s
+Step 12380/50462 | Loss: 2.7572 | LR: 4.35e-04 | 4028.7ms/step | 14.9 samples/s
+Step 12390/50462 | Loss: 2.7249 | LR: 4.35e-04 | 4001.8ms/step | 15.0 samples/s
+Step 12400/50462 | Loss: 2.7383 | LR: 4.35e-04 | 4008.5ms/step | 15.0 samples/s
+Step 12410/50462 | Loss: 2.7476 | LR: 4.35e-04 | 4020.7ms/step | 14.9 samples/s
+Step 12420/50462 | Loss: 2.7198 | LR: 4.35e-04 | 4016.1ms/step | 14.9 samples/s
+Step 12430/50462 | Loss: 2.7702 | LR: 4.35e-04 | 4019.3ms/step | 14.9 samples/s
+Step 12440/50462 | Loss: 2.7654 | LR: 4.35e-04 | 4017.4ms/step | 14.9 samples/s
+Step 12450/50462 | Loss: 2.7430 | LR: 4.35e-04 | 4016.9ms/step | 14.9 samples/s
+Step 12460/50462 | Loss: 2.7761 | LR: 4.35e-04 | 4009.3ms/step | 15.0 samples/s
+Step 12470/50462 | Loss: 2.7799 | LR: 4.35e-04 | 4006.7ms/step | 15.0 samples/s
+Step 12480/50462 | Loss: 2.7625 | LR: 4.34e-04 | 4010.9ms/step | 15.0 samples/s
+Step 12490/50462 | Loss: 2.7638 | LR: 4.34e-04 | 4016.7ms/step | 14.9 samples/s
+Step 12500/50462 | Loss: 2.7333 | LR: 4.34e-04 | 4008.0ms/step | 15.0 samples/s
+Validation at step 12500: Loss = 2.7564, Perplexity = 15.74
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-12500
+Step 12510/50462 | Loss: 2.7882 | LR: 4.34e-04 | 5945.0ms/step | 10.1 samples/s
+Step 12520/50462 | Loss: 2.7345 | LR: 4.34e-04 | 4008.0ms/step | 15.0 samples/s
+Step 12530/50462 | Loss: 2.7241 | LR: 4.34e-04 | 4004.9ms/step | 15.0 samples/s
+Step 12540/50462 | Loss: 2.7476 | LR: 4.34e-04 | 4017.5ms/step | 14.9 samples/s
+Step 12550/50462 | Loss: 2.7224 | LR: 4.34e-04 | 4017.3ms/step | 14.9 samples/s
+Step 12560/50462 | Loss: 2.7463 | LR: 4.34e-04 | 4023.3ms/step | 14.9 samples/s
+Step 12570/50462 | Loss: 2.7292 | LR: 4.33e-04 | 4005.4ms/step | 15.0 samples/s
+Step 12580/50462 | Loss: 2.7384 | LR: 4.33e-04 | 4008.9ms/step | 15.0 samples/s
+Step 12590/50462 | Loss: 2.7390 | LR: 4.33e-04 | 4007.1ms/step | 15.0 samples/s
+Step 12600/50462 | Loss: 2.7657 | LR: 4.33e-04 | 4000.8ms/step | 15.0 samples/s
+Step 12610/50462 | Loss: 2.7621 | LR: 4.33e-04 | 4007.8ms/step | 15.0 samples/s
+Step 12620/50462 | Loss: 2.7417 | LR: 4.33e-04 | 4013.8ms/step | 14.9 samples/s
+Step 12630/50462 | Loss: 2.7345 | LR: 4.33e-04 | 4004.5ms/step | 15.0 samples/s
+Step 12640/50462 | Loss: 2.7272 | LR: 4.33e-04 | 4014.0ms/step | 14.9 samples/s
+Step 12650/50462 | Loss: 2.7412 | LR: 4.33e-04 | 4009.2ms/step | 15.0 samples/s
+Step 12660/50462 | Loss: 2.7278 | LR: 4.32e-04 | 4001.3ms/step | 15.0 samples/s
+Step 12670/50462 | Loss: 2.7532 | LR: 4.32e-04 | 3996.7ms/step | 15.0 samples/s
+Step 12680/50462 | Loss: 2.7499 | LR: 4.32e-04 | 4004.9ms/step | 15.0 samples/s
+Step 12690/50462 | Loss: 2.7222 | LR: 4.32e-04 | 4005.5ms/step | 15.0 samples/s
+Step 12700/50462 | Loss: 2.7320 | LR: 4.32e-04 | 3997.9ms/step | 15.0 samples/s
+Step 12710/50462 | Loss: 2.7021 | LR: 4.32e-04 | 4014.1ms/step | 14.9 samples/s
+Step 12720/50462 | Loss: 2.7415 | LR: 4.32e-04 | 4011.7ms/step | 15.0 samples/s
+Step 12730/50462 | Loss: 2.7394 | LR: 4.32e-04 | 4013.9ms/step | 14.9 samples/s
+Step 12740/50462 | Loss: 2.7133 | LR: 4.32e-04 | 4013.8ms/step | 14.9 samples/s
+Step 12750/50462 | Loss: 2.7106 | LR: 4.31e-04 | 4009.0ms/step | 15.0 samples/s
+Validation at step 12750: Loss = 2.7530, Perplexity = 15.69
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-12750
+Step 12760/50462 | Loss: 2.7268 | LR: 4.31e-04 | 5897.8ms/step | 10.2 samples/s
+Step 12770/50462 | Loss: 2.7491 | LR: 4.31e-04 | 4000.2ms/step | 15.0 samples/s
+Step 12780/50462 | Loss: 2.7323 | LR: 4.31e-04 | 4006.7ms/step | 15.0 samples/s
+Step 12790/50462 | Loss: 2.7085 | LR: 4.31e-04 | 4009.7ms/step | 15.0 samples/s
+Step 12800/50462 | Loss: 2.7448 | LR: 4.31e-04 | 4005.1ms/step | 15.0 samples/s
+Step 12810/50462 | Loss: 2.7395 | LR: 4.31e-04 | 4010.7ms/step | 15.0 samples/s
+Step 12820/50462 | Loss: 2.7227 | LR: 4.31e-04 | 4011.5ms/step | 15.0 samples/s
+Step 12830/50462 | Loss: 2.7101 | LR: 4.31e-04 | 4009.5ms/step | 15.0 samples/s
+Step 12840/50462 | Loss: 2.7227 | LR: 4.31e-04 | 4010.5ms/step | 15.0 samples/s
+Step 12850/50462 | Loss: 2.7403 | LR: 4.30e-04 | 4019.2ms/step | 14.9 samples/s
+Step 12860/50462 | Loss: 2.7390 | LR: 4.30e-04 | 4008.9ms/step | 15.0 samples/s
+Step 12870/50462 | Loss: 2.7295 | LR: 4.30e-04 | 4016.7ms/step | 14.9 samples/s
+Step 12880/50462 | Loss: 2.7335 | LR: 4.30e-04 | 4013.1ms/step | 15.0 samples/s
+Step 12890/50462 | Loss: 2.7365 | LR: 4.30e-04 | 4016.3ms/step | 14.9 samples/s
+Step 12900/50462 | Loss: 2.7064 | LR: 4.30e-04 | 4003.5ms/step | 15.0 samples/s
+Step 12910/50462 | Loss: 2.7282 | LR: 4.30e-04 | 4007.5ms/step | 15.0 samples/s
+Step 12920/50462 | Loss: 2.7631 | LR: 4.30e-04 | 4014.1ms/step | 14.9 samples/s
+
+===== AUTO-RESUME seed72 FROM checkpoint-23250 2026-08-28T21:39:47Z =====
+W0828 21:40:02.014000 165 site-packages/torch/distributed/run.py:851] 
+W0828 21:40:02.014000 165 site-packages/torch/distributed/run.py:851] *****************************************
+W0828 21:40:02.014000 165 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0828 21:40:02.014000 165 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23250",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23250
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23250...
+Preserving checkpoint LR state for exact resume: current=[0.00028708498997505533], schedule_base=[0.0005]
+Resumed from checkpoint at step 23250 (epoch 0, micro_step 316245)
+Resuming training from step 23250
+Resuming wandb run z1i60278
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run z1i60278
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_214524-z1i60278
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed72 6gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/z1i60278
+Starting training...
+Validation at step 23250: Loss = 2.6326, Perplexity = 13.91
+wandb: WARNING Tried to log to step 23250 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 23251: sparse_stochastic (configured sparse attention)
+Step 23260/50462 | Loss: 2.6253 | LR: 2.87e-04 | 80261.3ms/step | 0.7 samples/s
+wandb: WARNING Tried to log to step 23260 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23270/50462 | Loss: 2.5993 | LR: 2.87e-04 | 4070.3ms/step | 14.7 samples/s
+wandb: WARNING Tried to log to step 23270 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23280/50462 | Loss: 2.5940 | LR: 2.87e-04 | 4070.5ms/step | 14.7 samples/s
+wandb: WARNING Tried to log to step 23280 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23290/50462 | Loss: 2.6180 | LR: 2.86e-04 | 4084.0ms/step | 14.7 samples/s
+wandb: WARNING Tried to log to step 23290 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23300/50462 | Loss: 2.6354 | LR: 2.86e-04 | 4075.9ms/step | 14.7 samples/s
+wandb: WARNING Tried to log to step 23300 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23310/50462 | Loss: 2.5866 | LR: 2.86e-04 | 4067.4ms/step | 14.8 samples/s
+wandb: WARNING Tried to log to step 23310 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23320/50462 | Loss: 2.6033 | LR: 2.86e-04 | 4075.2ms/step | 14.7 samples/s
+wandb: WARNING Tried to log to step 23320 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23330/50462 | Loss: 2.6093 | LR: 2.86e-04 | 4076.5ms/step | 14.7 samples/s
+wandb: WARNING Tried to log to step 23330 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23340/50462 | Loss: 2.5829 | LR: 2.86e-04 | 4077.9ms/step | 14.7 samples/s
+wandb: WARNING Tried to log to step 23340 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23350/50462 | Loss: 2.6551 | LR: 2.86e-04 | 4092.4ms/step | 14.7 samples/s
+wandb: WARNING Tried to log to step 23350 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23360/50462 | Loss: 2.5932 | LR: 2.85e-04 | 7608.3ms/step | 7.9 samples/s
+wandb: WARNING Tried to log to step 23360 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23370/50462 | Loss: 2.6252 | LR: 2.85e-04 | 12642.8ms/step | 4.7 samples/s
+wandb: WARNING Tried to log to step 23370 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23380/50462 | Loss: 2.5743 | LR: 2.85e-04 | 12113.3ms/step | 5.0 samples/s
+wandb: WARNING Tried to log to step 23380 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23390/50462 | Loss: 2.6239 | LR: 2.85e-04 | 11988.2ms/step | 5.0 samples/s
+wandb: WARNING Tried to log to step 23390 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23400/50462 | Loss: 2.5765 | LR: 2.85e-04 | 11945.6ms/step | 5.0 samples/s
+wandb: WARNING Tried to log to step 23400 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 23410/50462 | Loss: 2.6190 | LR: 2.85e-04 | 11844.8ms/step | 5.1 samples/s
+Step 23420/50462 | Loss: 2.5710 | LR: 2.84e-04 | 12096.3ms/step | 5.0 samples/s
+Step 23430/50462 | Loss: 2.6226 | LR: 2.84e-04 | 11941.3ms/step | 5.0 samples/s
+Step 23440/50462 | Loss: 2.6118 | LR: 2.84e-04 | 11778.9ms/step | 5.1 samples/s
+Step 23450/50462 | Loss: 2.6270 | LR: 2.84e-04 | 12013.1ms/step | 5.0 samples/s
+Step 23460/50462 | Loss: 2.6307 | LR: 2.84e-04 | 11944.9ms/step | 5.0 samples/s
+Step 23470/50462 | Loss: 2.6120 | LR: 2.84e-04 | 12128.2ms/step | 4.9 samples/s
+Step 23480/50462 | Loss: 2.6121 | LR: 2.83e-04 | 12104.6ms/step | 5.0 samples/s
+Step 23490/50462 | Loss: 2.6154 | LR: 2.83e-04 | 12127.0ms/step | 4.9 samples/s
+Step 23500/50462 | Loss: 2.6215 | LR: 2.83e-04 | 12148.3ms/step | 4.9 samples/s
+Validation at step 23500: Loss = 2.6301, Perplexity = 13.88
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23500
+Step 23510/50462 | Loss: 2.6426 | LR: 2.83e-04 | 17277.9ms/step | 3.5 samples/s
+Step 23520/50462 | Loss: 2.6220 | LR: 2.83e-04 | 12106.3ms/step | 5.0 samples/s
+Step 23530/50462 | Loss: 2.5945 | LR: 2.83e-04 | 12221.3ms/step | 4.9 samples/s
+Step 23540/50462 | Loss: 2.6364 | LR: 2.83e-04 | 12031.3ms/step | 5.0 samples/s
+Step 23550/50462 | Loss: 2.6192 | LR: 2.82e-04 | 12154.0ms/step | 4.9 samples/s
+Step 23560/50462 | Loss: 2.6352 | LR: 2.82e-04 | 12085.8ms/step | 5.0 samples/s
+Step 23570/50462 | Loss: 2.6031 | LR: 2.82e-04 | 11980.8ms/step | 5.0 samples/s
+Step 23580/50462 | Loss: 2.6069 | LR: 2.82e-04 | 11999.9ms/step | 5.0 samples/s
+Step 23590/50462 | Loss: 2.6162 | LR: 2.82e-04 | 11854.0ms/step | 5.1 samples/s
+Step 23600/50462 | Loss: 2.6115 | LR: 2.82e-04 | 12081.8ms/step | 5.0 samples/s
+Step 23610/50462 | Loss: 2.6253 | LR: 2.81e-04 | 9508.5ms/step | 6.3 samples/s
+Step 23620/50462 | Loss: 2.6078 | LR: 2.81e-04 | 12246.2ms/step | 4.9 samples/s
+Step 23630/50462 | Loss: 2.6459 | LR: 2.81e-04 | 11993.7ms/step | 5.0 samples/s
+Step 23640/50462 | Loss: 2.6063 | LR: 2.81e-04 | 12061.3ms/step | 5.0 samples/s
+Step 23650/50462 | Loss: 2.6141 | LR: 2.81e-04 | 12042.5ms/step | 5.0 samples/s
+Step 23660/50462 | Loss: 2.5993 | LR: 2.81e-04 | 12004.0ms/step | 5.0 samples/s
+Step 23670/50462 | Loss: 2.6249 | LR: 2.81e-04 | 11903.6ms/step | 5.0 samples/s
+Step 23680/50462 | Loss: 2.6420 | LR: 2.80e-04 | 12011.4ms/step | 5.0 samples/s
+Step 23690/50462 | Loss: 2.6115 | LR: 2.80e-04 | 12237.4ms/step | 4.9 samples/s
+Step 23700/50462 | Loss: 2.6102 | LR: 2.80e-04 | 11687.4ms/step | 5.1 samples/s
+Step 23710/50462 | Loss: 2.6101 | LR: 2.80e-04 | 11889.5ms/step | 5.0 samples/s
+Step 23720/50462 | Loss: 2.6356 | LR: 2.80e-04 | 12187.7ms/step | 4.9 samples/s
+Step 23730/50462 | Loss: 2.6235 | LR: 2.80e-04 | 12027.9ms/step | 5.0 samples/s
+Step 23740/50462 | Loss: 2.6183 | LR: 2.79e-04 | 11960.0ms/step | 5.0 samples/s
+Step 23750/50462 | Loss: 2.6015 | LR: 2.79e-04 | 12163.0ms/step | 4.9 samples/s
+Validation at step 23750: Loss = 2.6280, Perplexity = 13.85
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23750
+Step 23760/50462 | Loss: 2.6270 | LR: 2.79e-04 | 17182.5ms/step | 3.5 samples/s
+Step 23770/50462 | Loss: 2.6084 | LR: 2.79e-04 | 11838.3ms/step | 5.1 samples/s
+Step 23780/50462 | Loss: 2.6061 | LR: 2.79e-04 | 12140.5ms/step | 4.9 samples/s
+Step 23790/50462 | Loss: 2.5800 | LR: 2.79e-04 | 11911.8ms/step | 5.0 samples/s
+Step 23800/50462 | Loss: 2.5625 | LR: 2.78e-04 | 11878.8ms/step | 5.1 samples/s
+Step 23810/50462 | Loss: 2.6035 | LR: 2.78e-04 | 12143.5ms/step | 4.9 samples/s
+Step 23820/50462 | Loss: 2.5922 | LR: 2.78e-04 | 11855.3ms/step | 5.1 samples/s
+Step 23830/50462 | Loss: 2.6080 | LR: 2.78e-04 | 12000.6ms/step | 5.0 samples/s
+Step 23840/50462 | Loss: 2.5943 | LR: 2.78e-04 | 11885.3ms/step | 5.0 samples/s
+Step 23850/50462 | Loss: 2.6118 | LR: 2.78e-04 | 11897.3ms/step | 5.0 samples/s
+Step 23860/50462 | Loss: 2.6130 | LR: 2.78e-04 | 9456.2ms/step | 6.3 samples/s
+Step 23870/50462 | Loss: 2.6200 | LR: 2.77e-04 | 11828.7ms/step | 5.1 samples/s
+Step 23880/50462 | Loss: 2.6045 | LR: 2.77e-04 | 11933.2ms/step | 5.0 samples/s
+Step 23890/50462 | Loss: 2.6396 | LR: 2.77e-04 | 12110.2ms/step | 5.0 samples/s
+Step 23900/50462 | Loss: 2.6033 | LR: 2.77e-04 | 11972.5ms/step | 5.0 samples/s
+Step 23910/50462 | Loss: 2.6191 | LR: 2.77e-04 | 11834.6ms/step | 5.1 samples/s
+Step 23920/50462 | Loss: 2.5841 | LR: 2.77e-04 | 11788.6ms/step | 5.1 samples/s
+Step 23930/50462 | Loss: 2.5913 | LR: 2.76e-04 | 12339.2ms/step | 4.9 samples/s
+Step 23940/50462 | Loss: 2.6208 | LR: 2.76e-04 | 11801.0ms/step | 5.1 samples/s
+Step 23950/50462 | Loss: 2.5853 | LR: 2.76e-04 | 11890.6ms/step | 5.0 samples/s
+Step 23960/50462 | Loss: 2.6300 | LR: 2.76e-04 | 11911.9ms/step | 5.0 samples/s
+Step 23970/50462 | Loss: 2.6000 | LR: 2.76e-04 | 11882.7ms/step | 5.0 samples/s
+Step 23980/50462 | Loss: 2.5970 | LR: 2.76e-04 | 12110.1ms/step | 5.0 samples/s
+Step 23990/50462 | Loss: 2.5902 | LR: 2.75e-04 | 11856.5ms/step | 5.1 samples/s
+Step 24000/50462 | Loss: 2.5884 | LR: 2.75e-04 | 11915.1ms/step | 5.0 samples/s
+Validation at step 24000: Loss = 2.6265, Perplexity = 13.83
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-24000
+Step 24010/50462 | Loss: 2.6338 | LR: 2.75e-04 | 17281.5ms/step | 3.5 samples/s
+Step 24020/50462 | Loss: 2.6027 | LR: 2.75e-04 | 11859.8ms/step | 5.1 samples/s
+Step 24030/50462 | Loss: 2.6219 | LR: 2.75e-04 | 11986.4ms/step | 5.0 samples/s
+Step 24040/50462 | Loss: 2.6319 | LR: 2.75e-04 | 11926.7ms/step | 5.0 samples/s
+Step 24050/50462 | Loss: 2.6178 | LR: 2.75e-04 | 11995.7ms/step | 5.0 samples/s
+Step 24060/50462 | Loss: 2.5977 | LR: 2.74e-04 | 12094.5ms/step | 5.0 samples/s
+Step 24070/50462 | Loss: 2.6420 | LR: 2.74e-04 | 12367.1ms/step | 4.9 samples/s
+Step 24080/50462 | Loss: 2.5995 | LR: 2.74e-04 | 12079.6ms/step | 5.0 samples/s
+Step 24090/50462 | Loss: 2.5821 | LR: 2.74e-04 | 12224.9ms/step | 4.9 samples/s
+Step 24100/50462 | Loss: 2.6065 | LR: 2.74e-04 | 12132.9ms/step | 4.9 samples/s
+Step 24110/50462 | Loss: 2.6058 | LR: 2.74e-04 | 9639.6ms/step | 6.2 samples/s
+Step 24120/50462 | Loss: 2.6061 | LR: 2.73e-04 | 11938.3ms/step | 5.0 samples/s
+Step 24130/50462 | Loss: 2.6218 | LR: 2.73e-04 | 11748.1ms/step | 5.1 samples/s
+Step 24140/50462 | Loss: 2.6104 | LR: 2.73e-04 | 11806.0ms/step | 5.1 samples/s
+Step 24150/50462 | Loss: 2.6216 | LR: 2.73e-04 | 11893.2ms/step | 5.0 samples/s
+Step 24160/50462 | Loss: 2.5975 | LR: 2.73e-04 | 11797.4ms/step | 5.1 samples/s
+Step 24170/50462 | Loss: 2.5691 | LR: 2.73e-04 | 12064.2ms/step | 5.0 samples/s
+Step 24180/50462 | Loss: 2.6040 | LR: 2.72e-04 | 11933.7ms/step | 5.0 samples/s
+Step 24190/50462 | Loss: 2.6052 | LR: 2.72e-04 | 12030.9ms/step | 5.0 samples/s
+Step 24200/50462 | Loss: 2.6000 | LR: 2.72e-04 | 12020.9ms/step | 5.0 samples/s
+Step 24210/50462 | Loss: 2.5935 | LR: 2.72e-04 | 11891.3ms/step | 5.0 samples/s
+Step 24220/50462 | Loss: 2.5901 | LR: 2.72e-04 | 11890.3ms/step | 5.0 samples/s
+Step 24230/50462 | Loss: 2.5935 | LR: 2.72e-04 | 12220.5ms/step | 4.9 samples/s
+Step 24240/50462 | Loss: 2.6079 | LR: 2.72e-04 | 11877.2ms/step | 5.1 samples/s
+Step 24250/50462 | Loss: 2.6220 | LR: 2.71e-04 | 12187.8ms/step | 4.9 samples/s
+Validation at step 24250: Loss = 2.6239, Perplexity = 13.79
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-24250
+Step 24260/50462 | Loss: 2.6042 | LR: 2.71e-04 | 17048.3ms/step | 3.5 samples/s
+Step 24270/50462 | Loss: 2.5951 | LR: 2.71e-04 | 12094.9ms/step | 5.0 samples/s
+Step 24280/50462 | Loss: 2.5874 | LR: 2.71e-04 | 12029.4ms/step | 5.0 samples/s
+Step 24290/50462 | Loss: 2.6175 | LR: 2.71e-04 | 11879.0ms/step | 5.1 samples/s
+Step 24300/50462 | Loss: 2.6419 | LR: 2.71e-04 | 11819.8ms/step | 5.1 samples/s
+Step 24310/50462 | Loss: 2.5845 | LR: 2.70e-04 | 11832.2ms/step | 5.1 samples/s
+Step 24320/50462 | Loss: 2.6221 | LR: 2.70e-04 | 12292.5ms/step | 4.9 samples/s
+Step 24330/50462 | Loss: 2.5635 | LR: 2.70e-04 | 11997.6ms/step | 5.0 samples/s
+Step 24340/50462 | Loss: 2.6121 | LR: 2.70e-04 | 11967.0ms/step | 5.0 samples/s
+Step 24350/50462 | Loss: 2.5734 | LR: 2.70e-04 | 12077.9ms/step | 5.0 samples/s
+Step 24360/50462 | Loss: 2.6030 | LR: 2.70e-04 | 9483.4ms/step | 6.3 samples/s
+Step 24370/50462 | Loss: 2.6186 | LR: 2.70e-04 | 12161.8ms/step | 4.9 samples/s
+Step 24380/50462 | Loss: 2.6137 | LR: 2.69e-04 | 12198.6ms/step | 4.9 samples/s
+Step 24390/50462 | Loss: 2.6172 | LR: 2.69e-04 | 11936.2ms/step | 5.0 samples/s
+Step 24400/50462 | Loss: 2.6392 | LR: 2.69e-04 | 11776.2ms/step | 5.1 samples/s
+Step 24410/50462 | Loss: 2.5887 | LR: 2.69e-04 | 11866.7ms/step | 5.1 samples/s
+Step 24420/50462 | Loss: 2.6059 | LR: 2.69e-04 | 11838.4ms/step | 5.1 samples/s
+Step 24430/50462 | Loss: 2.6020 | LR: 2.69e-04 | 12061.9ms/step | 5.0 samples/s
+Step 24440/50462 | Loss: 2.5962 | LR: 2.68e-04 | 12016.5ms/step | 5.0 samples/s
+Step 24450/50462 | Loss: 2.6325 | LR: 2.68e-04 | 11918.1ms/step | 5.0 samples/s
+Step 24460/50462 | Loss: 2.5924 | LR: 2.68e-04 | 11831.5ms/step | 5.1 samples/s
+Step 24470/50462 | Loss: 2.5972 | LR: 2.68e-04 | 11982.6ms/step | 5.0 samples/s
+Step 24480/50462 | Loss: 2.6074 | LR: 2.68e-04 | 12642.1ms/step | 4.7 samples/s
+Step 24490/50462 | Loss: 2.5694 | LR: 2.68e-04 | 12046.6ms/step | 5.0 samples/s
+Step 24500/50462 | Loss: 2.6091 | LR: 2.67e-04 | 12059.1ms/step | 5.0 samples/s
+Validation at step 24500: Loss = 2.6215, Perplexity = 13.76
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-24500
+Step 24510/50462 | Loss: 2.6294 | LR: 2.67e-04 | 17120.7ms/step | 3.5 samples/s
+Step 24520/50462 | Loss: 2.5950 | LR: 2.67e-04 | 11895.7ms/step | 5.0 samples/s
+Step 24530/50462 | Loss: 2.5876 | LR: 2.67e-04 | 11586.1ms/step | 5.2 samples/s
+Step 24540/50462 | Loss: 2.6070 | LR: 2.67e-04 | 11928.6ms/step | 5.0 samples/s
+Step 24550/50462 | Loss: 2.6119 | LR: 2.67e-04 | 12012.3ms/step | 5.0 samples/s
+Step 24560/50462 | Loss: 2.6301 | LR: 2.67e-04 | 11872.3ms/step | 5.1 samples/s
+Step 24570/50462 | Loss: 2.5883 | LR: 2.66e-04 | 11877.8ms/step | 5.1 samples/s
+Step 24580/50462 | Loss: 2.6169 | LR: 2.66e-04 | 11667.7ms/step | 5.1 samples/s
+Step 24590/50462 | Loss: 2.5993 | LR: 2.66e-04 | 11986.7ms/step | 5.0 samples/s
+Step 24600/50462 | Loss: 2.5934 | LR: 2.66e-04 | 12383.3ms/step | 4.8 samples/s
+Step 24610/50462 | Loss: 2.6329 | LR: 2.66e-04 | 9442.0ms/step | 6.4 samples/s
+Step 24620/50462 | Loss: 2.6142 | LR: 2.66e-04 | 12208.7ms/step | 4.9 samples/s
+Step 24630/50462 | Loss: 2.5341 | LR: 2.65e-04 | 12087.7ms/step | 5.0 samples/s
+Step 24640/50462 | Loss: 2.5782 | LR: 2.65e-04 | 11995.2ms/step | 5.0 samples/s
+Step 24650/50462 | Loss: 2.6035 | LR: 2.65e-04 | 11987.3ms/step | 5.0 samples/s
+Step 24660/50462 | Loss: 2.6002 | LR: 2.65e-04 | 11928.6ms/step | 5.0 samples/s
+Step 24670/50462 | Loss: 2.6280 | LR: 2.65e-04 | 12468.0ms/step | 4.8 samples/s
+Step 24680/50462 | Loss: 2.6387 | LR: 2.65e-04 | 11915.8ms/step | 5.0 samples/s
+Step 24690/50462 | Loss: 2.5985 | LR: 2.64e-04 | 12109.5ms/step | 5.0 samples/s
+Step 24700/50462 | Loss: 2.5447 | LR: 2.64e-04 | 12129.3ms/step | 4.9 samples/s
+Step 24710/50462 | Loss: 2.6018 | LR: 2.64e-04 | 11922.3ms/step | 5.0 samples/s
+Step 24720/50462 | Loss: 2.5804 | LR: 2.64e-04 | 11839.3ms/step | 5.1 samples/s
+Step 24730/50462 | Loss: 2.5811 | LR: 2.64e-04 | 12104.3ms/step | 5.0 samples/s
+Step 24740/50462 | Loss: 2.5862 | LR: 2.64e-04 | 12188.6ms/step | 4.9 samples/s
+Step 24750/50462 | Loss: 2.6143 | LR: 2.64e-04 | 11888.8ms/step | 5.0 samples/s
+Validation at step 24750: Loss = 2.6193, Perplexity = 13.73
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-24750
+Step 24760/50462 | Loss: 2.5857 | LR: 2.63e-04 | 17248.4ms/step | 3.5 samples/s
+Step 24770/50462 | Loss: 2.5990 | LR: 2.63e-04 | 11850.8ms/step | 5.1 samples/s
+Step 24780/50462 | Loss: 2.5961 | LR: 2.63e-04 | 11824.7ms/step | 5.1 samples/s
+Step 24790/50462 | Loss: 2.6206 | LR: 2.63e-04 | 12014.0ms/step | 5.0 samples/s
+Step 24800/50462 | Loss: 2.5585 | LR: 2.63e-04 | 11987.8ms/step | 5.0 samples/s
+Step 24810/50462 | Loss: 2.5824 | LR: 2.63e-04 | 11995.2ms/step | 5.0 samples/s
+Step 24820/50462 | Loss: 2.5922 | LR: 2.62e-04 | 11928.5ms/step | 5.0 samples/s
+Step 24830/50462 | Loss: 2.6009 | LR: 2.62e-04 | 12112.2ms/step | 5.0 samples/s
+Step 24840/50462 | Loss: 2.6099 | LR: 2.62e-04 | 12171.5ms/step | 4.9 samples/s
+Step 24850/50462 | Loss: 2.5924 | LR: 2.62e-04 | 11969.1ms/step | 5.0 samples/s
+Step 24860/50462 | Loss: 2.5817 | LR: 2.62e-04 | 9445.9ms/step | 6.4 samples/s
+Step 24870/50462 | Loss: 2.6098 | LR: 2.62e-04 | 11944.1ms/step | 5.0 samples/s
+Step 24880/50462 | Loss: 2.5697 | LR: 2.61e-04 | 12182.9ms/step | 4.9 samples/s
+Step 24890/50462 | Loss: 2.6025 | LR: 2.61e-04 | 11843.5ms/step | 5.1 samples/s
+Step 24900/50462 | Loss: 2.6044 | LR: 2.61e-04 | 12171.4ms/step | 4.9 samples/s
+Step 24910/50462 | Loss: 2.6097 | LR: 2.61e-04 | 11828.4ms/step | 5.1 samples/s
+Step 24920/50462 | Loss: 2.6129 | LR: 2.61e-04 | 12051.1ms/step | 5.0 samples/s
+Step 24930/50462 | Loss: 2.6081 | LR: 2.61e-04 | 11853.0ms/step | 5.1 samples/s
+Step 24940/50462 | Loss: 2.6027 | LR: 2.61e-04 | 12009.5ms/step | 5.0 samples/s
+Step 24950/50462 | Loss: 2.5778 | LR: 2.60e-04 | 11948.4ms/step | 5.0 samples/s
+Step 24960/50462 | Loss: 2.5854 | LR: 2.60e-04 | 11925.8ms/step | 5.0 samples/s
+Step 24970/50462 | Loss: 2.5831 | LR: 2.60e-04 | 12251.1ms/step | 4.9 samples/s
+Step 24980/50462 | Loss: 2.5815 | LR: 2.60e-04 | 12275.8ms/step | 4.9 samples/s
+Step 24990/50462 | Loss: 2.6082 | LR: 2.60e-04 | 11795.7ms/step | 5.1 samples/s
+Step 25000/50462 | Loss: 2.6009 | LR: 2.60e-04 | 12407.4ms/step | 4.8 samples/s
+Validation at step 25000: Loss = 2.6164, Perplexity = 13.69
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25000
+Step 25010/50462 | Loss: 2.6280 | LR: 2.59e-04 | 16953.9ms/step | 3.5 samples/s
+Step 25020/50462 | Loss: 2.5512 | LR: 2.59e-04 | 11947.6ms/step | 5.0 samples/s
+Step 25030/50462 | Loss: 2.5974 | LR: 2.59e-04 | 12531.8ms/step | 4.8 samples/s
+Step 25040/50462 | Loss: 2.5898 | LR: 2.59e-04 | 12009.5ms/step | 5.0 samples/s
+Step 25050/50462 | Loss: 2.6044 | LR: 2.59e-04 | 9394.8ms/step | 6.4 samples/s
+Step 25060/50462 | Loss: 2.5819 | LR: 2.59e-04 | 4007.8ms/step | 15.0 samples/s
+Step 25070/50462 | Loss: 2.6098 | LR: 2.58e-04 | 4014.2ms/step | 14.9 samples/s
+Step 25080/50462 | Loss: 2.6072 | LR: 2.58e-04 | 4019.8ms/step | 14.9 samples/s
+Step 25090/50462 | Loss: 2.5939 | LR: 2.58e-04 | 4010.8ms/step | 15.0 samples/s
+Step 25100/50462 | Loss: 2.5698 | LR: 2.58e-04 | 4016.8ms/step | 14.9 samples/s
+Step 25110/50462 | Loss: 2.5632 | LR: 2.58e-04 | 4017.5ms/step | 14.9 samples/s
+Step 25120/50462 | Loss: 2.5901 | LR: 2.58e-04 | 4010.2ms/step | 15.0 samples/s
+Step 25130/50462 | Loss: 2.6156 | LR: 2.58e-04 | 4005.3ms/step | 15.0 samples/s
+Step 25140/50462 | Loss: 2.6016 | LR: 2.57e-04 | 4006.9ms/step | 15.0 samples/s
+Step 25150/50462 | Loss: 2.5955 | LR: 2.57e-04 | 4010.3ms/step | 15.0 samples/s
+Step 25160/50462 | Loss: 2.5763 | LR: 2.57e-04 | 4008.5ms/step | 15.0 samples/s
+Step 25170/50462 | Loss: 2.6167 | LR: 2.57e-04 | 4015.3ms/step | 14.9 samples/s
+Step 25180/50462 | Loss: 2.5570 | LR: 2.57e-04 | 4011.2ms/step | 15.0 samples/s
+Step 25190/50462 | Loss: 2.5725 | LR: 2.57e-04 | 4016.8ms/step | 14.9 samples/s
+Step 25200/50462 | Loss: 2.6080 | LR: 2.56e-04 | 4009.8ms/step | 15.0 samples/s
+Step 25210/50462 | Loss: 2.5904 | LR: 2.56e-04 | 4006.5ms/step | 15.0 samples/s
+Step 25220/50462 | Loss: 2.5918 | LR: 2.56e-04 | 4011.7ms/step | 15.0 samples/s
+Step 25230/50462 | Loss: 2.5817 | LR: 2.56e-04 | 4018.0ms/step | 14.9 samples/s
+Step 25240/50462 | Loss: 2.5974 | LR: 2.56e-04 | 4010.5ms/step | 15.0 samples/s
+Step 25250/50462 | Loss: 2.5872 | LR: 2.56e-04 | 4015.5ms/step | 14.9 samples/s
+Validation at step 25250: Loss = 2.6152, Perplexity = 13.67

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume23250.log

@@ -0,0 +1,117 @@
+
+===== RESUME FROM checkpoint-23250 ON 4GPU 2026-08-28T21:41:40+00:00 =====
+W0828 21:41:44.087000 267 site-packages/torch/distributed/run.py:851] 
+W0828 21:41:44.087000 267 site-packages/torch/distributed/run.py:851] *****************************************
+W0828 21:41:44.087000 267 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0828 21:41:44.087000 267 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23250",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23250
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+W0828 21:42:07.009000 267 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+W0828 21:42:07.011000 267 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 335 closing signal SIGTERM
+W0828 21:42:07.011000 267 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 336 closing signal SIGTERM
+W0828 21:42:07.012000 267 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 337 closing signal SIGTERM
+W0828 21:42:07.021000 267 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 338 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 267 got signal: 15

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume23250.pid

@@ -0,0 +1 @@
+267

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume25000.log

@@ -0,0 +1,882 @@
+===== RESUME FROM CHECKPOINT-25000 ON 4GPU 2026-08-29T04:04:09+00:00 =====
+W0829 04:04:20.274000 127 site-packages/torch/distributed/run.py:851] 
+W0829 04:04:20.274000 127 site-packages/torch/distributed/run.py:851] *****************************************
+W0829 04:04:20.274000 127 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0829 04:04:20.274000 127 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: TrueCheckpoint activations: TrueCheckpoint activations: True
+
+
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25000
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25000...
+Preserving checkpoint LR state for exact resume: current=[0.00025957183284290125], schedule_base=[0.0005]
+Resumed from checkpoint at step 25000 (epoch 0, micro_step 342495)
+Resuming training from step 25000
+Resuming wandb run z1i60278
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run z1i60278
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260829_040843-z1i60278
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed72 6gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/z1i60278
+Starting training...
+Validation at step 25000: Loss = 2.6170, Perplexity = 13.69
+wandb: WARNING Tried to log to step 25000 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 25001: sparse_stochastic (configured sparse attention)
+Step 25010/50462 | Loss: 2.6280 | LR: 2.59e-04 | 58173.4ms/step | 1.0 samples/s
+wandb: WARNING Tried to log to step 25010 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25020/50462 | Loss: 2.5513 | LR: 2.59e-04 | 4009.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25020 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25030/50462 | Loss: 2.5974 | LR: 2.59e-04 | 3993.8ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25030 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25040/50462 | Loss: 2.5901 | LR: 2.59e-04 | 3996.7ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25040 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25050/50462 | Loss: 2.6022 | LR: 2.59e-04 | 4009.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25050 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25060/50462 | Loss: 2.5829 | LR: 2.59e-04 | 4006.2ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25060 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25070/50462 | Loss: 2.6101 | LR: 2.58e-04 | 4003.8ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25070 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25080/50462 | Loss: 2.6068 | LR: 2.58e-04 | 4006.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25080 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25090/50462 | Loss: 2.5944 | LR: 2.58e-04 | 4010.3ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25090 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25100/50462 | Loss: 2.5705 | LR: 2.58e-04 | 4008.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25100 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25110/50462 | Loss: 2.5632 | LR: 2.58e-04 | 4003.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25110 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25120/50462 | Loss: 2.5900 | LR: 2.58e-04 | 4001.4ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25120 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25130/50462 | Loss: 2.6161 | LR: 2.58e-04 | 4004.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25130 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25140/50462 | Loss: 2.6008 | LR: 2.57e-04 | 4007.2ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25140 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25150/50462 | Loss: 2.5963 | LR: 2.57e-04 | 4002.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25150 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25160/50462 | Loss: 2.5763 | LR: 2.57e-04 | 4002.2ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25160 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25170/50462 | Loss: 2.6165 | LR: 2.57e-04 | 4006.9ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25170 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25180/50462 | Loss: 2.5574 | LR: 2.57e-04 | 4009.8ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25180 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25190/50462 | Loss: 2.5740 | LR: 2.57e-04 | 4005.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25190 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25200/50462 | Loss: 2.6076 | LR: 2.56e-04 | 4009.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25200 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25210/50462 | Loss: 2.5915 | LR: 2.56e-04 | 3997.3ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25210 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25220/50462 | Loss: 2.5918 | LR: 2.56e-04 | 4000.3ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25220 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25230/50462 | Loss: 2.5812 | LR: 2.56e-04 | 4001.9ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25230 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25240/50462 | Loss: 2.5987 | LR: 2.56e-04 | 4007.8ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 25240 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 25250/50462 | Loss: 2.5879 | LR: 2.56e-04 | 4001.6ms/step | 15.0 samples/s
+Validation at step 25250: Loss = 2.6153, Perplexity = 13.67
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25250
+Step 25260/50462 | Loss: 2.5977 | LR: 2.55e-04 | 5877.2ms/step | 10.2 samples/s
+Step 25270/50462 | Loss: 2.5752 | LR: 2.55e-04 | 3997.3ms/step | 15.0 samples/s
+Step 25280/50462 | Loss: 2.6060 | LR: 2.55e-04 | 4012.0ms/step | 15.0 samples/s
+Step 25290/50462 | Loss: 2.5910 | LR: 2.55e-04 | 4006.7ms/step | 15.0 samples/s
+Step 25300/50462 | Loss: 2.6100 | LR: 2.55e-04 | 4007.1ms/step | 15.0 samples/s
+Step 25310/50462 | Loss: 2.6083 | LR: 2.55e-04 | 4003.0ms/step | 15.0 samples/s
+Step 25320/50462 | Loss: 2.5730 | LR: 2.55e-04 | 4017.6ms/step | 14.9 samples/s
+Step 25330/50462 | Loss: 2.6052 | LR: 2.54e-04 | 4011.0ms/step | 15.0 samples/s
+Step 25340/50462 | Loss: 2.6152 | LR: 2.54e-04 | 4012.1ms/step | 15.0 samples/s
+Step 25350/50462 | Loss: 2.5869 | LR: 2.54e-04 | 3998.7ms/step | 15.0 samples/s
+Step 25360/50462 | Loss: 2.5738 | LR: 2.54e-04 | 4011.2ms/step | 15.0 samples/s
+Step 25370/50462 | Loss: 2.6424 | LR: 2.54e-04 | 4005.3ms/step | 15.0 samples/s
+Step 25380/50462 | Loss: 2.6114 | LR: 2.54e-04 | 4004.3ms/step | 15.0 samples/s
+Step 25390/50462 | Loss: 2.5956 | LR: 2.53e-04 | 4010.1ms/step | 15.0 samples/s
+Step 25400/50462 | Loss: 2.5914 | LR: 2.53e-04 | 4000.5ms/step | 15.0 samples/s
+Step 25410/50462 | Loss: 2.6131 | LR: 2.53e-04 | 4006.6ms/step | 15.0 samples/s
+Step 25420/50462 | Loss: 2.6089 | LR: 2.53e-04 | 4009.9ms/step | 15.0 samples/s
+Step 25430/50462 | Loss: 2.5995 | LR: 2.53e-04 | 4003.3ms/step | 15.0 samples/s
+Step 25440/50462 | Loss: 2.6123 | LR: 2.53e-04 | 4013.5ms/step | 14.9 samples/s
+Step 25450/50462 | Loss: 2.5886 | LR: 2.52e-04 | 4000.3ms/step | 15.0 samples/s
+Step 25460/50462 | Loss: 2.5843 | LR: 2.52e-04 | 3999.2ms/step | 15.0 samples/s
+Step 25470/50462 | Loss: 2.5661 | LR: 2.52e-04 | 4014.6ms/step | 14.9 samples/s
+Step 25480/50462 | Loss: 2.5716 | LR: 2.52e-04 | 4007.7ms/step | 15.0 samples/s
+Step 25490/50462 | Loss: 2.5875 | LR: 2.52e-04 | 4008.1ms/step | 15.0 samples/s
+Step 25500/50462 | Loss: 2.5796 | LR: 2.52e-04 | 4008.9ms/step | 15.0 samples/s
+Validation at step 25500: Loss = 2.6125, Perplexity = 13.63
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25500
+Step 25510/50462 | Loss: 2.5794 | LR: 2.52e-04 | 5851.1ms/step | 10.3 samples/s
+Step 25520/50462 | Loss: 2.5684 | LR: 2.51e-04 | 4010.6ms/step | 15.0 samples/s
+Step 25530/50462 | Loss: 2.6146 | LR: 2.51e-04 | 4005.3ms/step | 15.0 samples/s
+Step 25540/50462 | Loss: 2.5745 | LR: 2.51e-04 | 4018.4ms/step | 14.9 samples/s
+Step 25550/50462 | Loss: 2.5861 | LR: 2.51e-04 | 4007.7ms/step | 15.0 samples/s
+Step 25560/50462 | Loss: 2.6104 | LR: 2.51e-04 | 4017.7ms/step | 14.9 samples/s
+Step 25570/50462 | Loss: 2.5718 | LR: 2.51e-04 | 4018.8ms/step | 14.9 samples/s
+Step 25580/50462 | Loss: 2.5818 | LR: 2.50e-04 | 4003.7ms/step | 15.0 samples/s
+Step 25590/50462 | Loss: 2.5771 | LR: 2.50e-04 | 4005.5ms/step | 15.0 samples/s
+Step 25600/50462 | Loss: 2.5913 | LR: 2.50e-04 | 4008.1ms/step | 15.0 samples/s
+Step 25610/50462 | Loss: 2.5877 | LR: 2.50e-04 | 4006.7ms/step | 15.0 samples/s
+Step 25620/50462 | Loss: 2.5774 | LR: 2.50e-04 | 4007.5ms/step | 15.0 samples/s
+Step 25630/50462 | Loss: 2.5982 | LR: 2.50e-04 | 4014.2ms/step | 14.9 samples/s
+Step 25640/50462 | Loss: 2.6228 | LR: 2.49e-04 | 4011.7ms/step | 15.0 samples/s
+Step 25650/50462 | Loss: 2.5732 | LR: 2.49e-04 | 4007.7ms/step | 15.0 samples/s
+Step 25660/50462 | Loss: 2.5501 | LR: 2.49e-04 | 4004.7ms/step | 15.0 samples/s
+Step 25670/50462 | Loss: 2.6400 | LR: 2.49e-04 | 4014.1ms/step | 14.9 samples/s
+Step 25680/50462 | Loss: 2.5958 | LR: 2.49e-04 | 4007.3ms/step | 15.0 samples/s
+Step 25690/50462 | Loss: 2.5751 | LR: 2.49e-04 | 4015.3ms/step | 14.9 samples/s
+Step 25700/50462 | Loss: 2.5882 | LR: 2.49e-04 | 4020.7ms/step | 14.9 samples/s
+Step 25710/50462 | Loss: 2.5797 | LR: 2.48e-04 | 3989.1ms/step | 15.0 samples/s
+Step 25720/50462 | Loss: 2.6001 | LR: 2.48e-04 | 4006.1ms/step | 15.0 samples/s
+Step 25730/50462 | Loss: 2.5964 | LR: 2.48e-04 | 4011.8ms/step | 15.0 samples/s
+Step 25740/50462 | Loss: 2.6219 | LR: 2.48e-04 | 4006.0ms/step | 15.0 samples/s
+Step 25750/50462 | Loss: 2.5926 | LR: 2.48e-04 | 4009.1ms/step | 15.0 samples/s
+Validation at step 25750: Loss = 2.6111, Perplexity = 13.61
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25750
+Step 25760/50462 | Loss: 2.6253 | LR: 2.48e-04 | 5851.1ms/step | 10.3 samples/s
+Step 25770/50462 | Loss: 2.6164 | LR: 2.47e-04 | 4013.8ms/step | 14.9 samples/s
+Step 25780/50462 | Loss: 2.5851 | LR: 2.47e-04 | 4008.2ms/step | 15.0 samples/s
+Step 25790/50462 | Loss: 2.6205 | LR: 2.47e-04 | 4016.5ms/step | 14.9 samples/s
+Step 25800/50462 | Loss: 2.5798 | LR: 2.47e-04 | 3998.3ms/step | 15.0 samples/s
+Step 25810/50462 | Loss: 2.5610 | LR: 2.47e-04 | 4017.9ms/step | 14.9 samples/s
+Step 25820/50462 | Loss: 2.6066 | LR: 2.47e-04 | 4001.8ms/step | 15.0 samples/s
+Step 25830/50462 | Loss: 2.6160 | LR: 2.46e-04 | 3995.7ms/step | 15.0 samples/s
+Step 25840/50462 | Loss: 2.6181 | LR: 2.46e-04 | 4001.2ms/step | 15.0 samples/s
+Step 25850/50462 | Loss: 2.6284 | LR: 2.46e-04 | 3997.9ms/step | 15.0 samples/s
+Step 25860/50462 | Loss: 2.6206 | LR: 2.46e-04 | 4009.1ms/step | 15.0 samples/s
+Step 25870/50462 | Loss: 2.5538 | LR: 2.46e-04 | 3995.3ms/step | 15.0 samples/s
+Step 25880/50462 | Loss: 2.5933 | LR: 2.46e-04 | 3998.8ms/step | 15.0 samples/s
+Step 25890/50462 | Loss: 2.5947 | LR: 2.46e-04 | 4009.0ms/step | 15.0 samples/s
+Step 25900/50462 | Loss: 2.5718 | LR: 2.45e-04 | 4007.4ms/step | 15.0 samples/s
+Step 25910/50462 | Loss: 2.5829 | LR: 2.45e-04 | 3994.6ms/step | 15.0 samples/s
+Step 25920/50462 | Loss: 2.5713 | LR: 2.45e-04 | 4015.9ms/step | 14.9 samples/s
+Step 25930/50462 | Loss: 2.5822 | LR: 2.45e-04 | 4021.0ms/step | 14.9 samples/s
+Step 25940/50462 | Loss: 2.5811 | LR: 2.45e-04 | 4012.9ms/step | 15.0 samples/s
+Step 25950/50462 | Loss: 2.5883 | LR: 2.45e-04 | 4004.9ms/step | 15.0 samples/s
+Step 25960/50462 | Loss: 2.5899 | LR: 2.44e-04 | 3999.4ms/step | 15.0 samples/s
+Step 25970/50462 | Loss: 2.5756 | LR: 2.44e-04 | 4004.6ms/step | 15.0 samples/s
+Step 25980/50462 | Loss: 2.6009 | LR: 2.44e-04 | 4000.0ms/step | 15.0 samples/s
+Step 25990/50462 | Loss: 2.6221 | LR: 2.44e-04 | 4004.7ms/step | 15.0 samples/s
+Step 26000/50462 | Loss: 2.6309 | LR: 2.44e-04 | 4006.9ms/step | 15.0 samples/s
+Validation at step 26000: Loss = 2.6093, Perplexity = 13.59
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-26000
+Step 26010/50462 | Loss: 2.5757 | LR: 2.44e-04 | 5875.6ms/step | 10.2 samples/s
+Step 26020/50462 | Loss: 2.6059 | LR: 2.43e-04 | 4577.6ms/step | 13.1 samples/s
+Step 26030/50462 | Loss: 2.6089 | LR: 2.43e-04 | 4011.9ms/step | 15.0 samples/s
+Step 26040/50462 | Loss: 2.5799 | LR: 2.43e-04 | 4003.3ms/step | 15.0 samples/s
+Step 26050/50462 | Loss: 2.5462 | LR: 2.43e-04 | 4002.0ms/step | 15.0 samples/s
+Step 26060/50462 | Loss: 2.5989 | LR: 2.43e-04 | 3997.1ms/step | 15.0 samples/s
+Step 26070/50462 | Loss: 2.6032 | LR: 2.43e-04 | 4014.6ms/step | 14.9 samples/s
+Step 26080/50462 | Loss: 2.5678 | LR: 2.43e-04 | 3999.7ms/step | 15.0 samples/s
+Step 26090/50462 | Loss: 2.5768 | LR: 2.42e-04 | 4004.9ms/step | 15.0 samples/s
+Step 26100/50462 | Loss: 2.5668 | LR: 2.42e-04 | 4004.5ms/step | 15.0 samples/s
+Step 26110/50462 | Loss: 2.5622 | LR: 2.42e-04 | 4007.7ms/step | 15.0 samples/s
+Step 26120/50462 | Loss: 2.5803 | LR: 2.42e-04 | 4008.1ms/step | 15.0 samples/s
+Step 26130/50462 | Loss: 2.5797 | LR: 2.42e-04 | 3996.9ms/step | 15.0 samples/s
+Step 26140/50462 | Loss: 2.5677 | LR: 2.42e-04 | 4011.0ms/step | 15.0 samples/s
+Step 26150/50462 | Loss: 2.5590 | LR: 2.41e-04 | 4010.4ms/step | 15.0 samples/s
+Step 26160/50462 | Loss: 2.5888 | LR: 2.41e-04 | 4022.6ms/step | 14.9 samples/s
+Step 26170/50462 | Loss: 2.5621 | LR: 2.41e-04 | 4007.9ms/step | 15.0 samples/s
+Step 26180/50462 | Loss: 2.5942 | LR: 2.41e-04 | 4011.6ms/step | 15.0 samples/s
+Step 26190/50462 | Loss: 2.5780 | LR: 2.41e-04 | 3992.6ms/step | 15.0 samples/s
+Step 26200/50462 | Loss: 2.5669 | LR: 2.41e-04 | 4010.1ms/step | 15.0 samples/s
+Step 26210/50462 | Loss: 2.5655 | LR: 2.40e-04 | 4007.8ms/step | 15.0 samples/s
+Step 26220/50462 | Loss: 2.6029 | LR: 2.40e-04 | 4005.1ms/step | 15.0 samples/s
+Step 26230/50462 | Loss: 2.5883 | LR: 2.40e-04 | 4001.4ms/step | 15.0 samples/s
+Step 26240/50462 | Loss: 2.6144 | LR: 2.40e-04 | 4014.2ms/step | 14.9 samples/s
+Step 26250/50462 | Loss: 2.5718 | LR: 2.40e-04 | 4003.5ms/step | 15.0 samples/s
+Validation at step 26250: Loss = 2.6074, Perplexity = 13.56
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-26250
+Step 26260/50462 | Loss: 2.5904 | LR: 2.40e-04 | 5849.1ms/step | 10.3 samples/s
+Step 26270/50462 | Loss: 2.5928 | LR: 2.40e-04 | 3998.3ms/step | 15.0 samples/s
+Step 26280/50462 | Loss: 2.5835 | LR: 2.39e-04 | 4008.7ms/step | 15.0 samples/s
+Step 26290/50462 | Loss: 2.5884 | LR: 2.39e-04 | 4001.1ms/step | 15.0 samples/s
+Step 26300/50462 | Loss: 2.5676 | LR: 2.39e-04 | 3998.7ms/step | 15.0 samples/s
+Step 26310/50462 | Loss: 2.6005 | LR: 2.39e-04 | 4005.8ms/step | 15.0 samples/s
+Step 26320/50462 | Loss: 2.5575 | LR: 2.39e-04 | 4011.5ms/step | 15.0 samples/s
+Step 26330/50462 | Loss: 2.5735 | LR: 2.39e-04 | 4004.5ms/step | 15.0 samples/s
+Step 26340/50462 | Loss: 2.5866 | LR: 2.38e-04 | 4006.8ms/step | 15.0 samples/s
+Step 26350/50462 | Loss: 2.5961 | LR: 2.38e-04 | 3999.2ms/step | 15.0 samples/s
+Step 26360/50462 | Loss: 2.5831 | LR: 2.38e-04 | 4012.3ms/step | 15.0 samples/s
+Step 26370/50462 | Loss: 2.5877 | LR: 2.38e-04 | 4002.2ms/step | 15.0 samples/s
+Step 26380/50462 | Loss: 2.6219 | LR: 2.38e-04 | 4014.0ms/step | 14.9 samples/s
+Step 26390/50462 | Loss: 2.6313 | LR: 2.38e-04 | 4015.9ms/step | 14.9 samples/s
+Step 26400/50462 | Loss: 2.5812 | LR: 2.37e-04 | 4003.1ms/step | 15.0 samples/s
+Step 26410/50462 | Loss: 2.5905 | LR: 2.37e-04 | 4584.8ms/step | 13.1 samples/s
+Step 26420/50462 | Loss: 2.5765 | LR: 2.37e-04 | 4569.2ms/step | 13.1 samples/s
+Step 26430/50462 | Loss: 2.6128 | LR: 2.37e-04 | 4006.3ms/step | 15.0 samples/s
+Step 26440/50462 | Loss: 2.6147 | LR: 2.37e-04 | 4005.5ms/step | 15.0 samples/s
+Step 26450/50462 | Loss: 2.5937 | LR: 2.37e-04 | 3990.9ms/step | 15.0 samples/s
+Step 26460/50462 | Loss: 2.5672 | LR: 2.37e-04 | 3984.6ms/step | 15.1 samples/s
+Step 26470/50462 | Loss: 2.6128 | LR: 2.36e-04 | 3993.2ms/step | 15.0 samples/s
+Step 26480/50462 | Loss: 2.5686 | LR: 2.36e-04 | 4012.1ms/step | 15.0 samples/s
+Step 26490/50462 | Loss: 2.5954 | LR: 2.36e-04 | 3999.3ms/step | 15.0 samples/s
+Step 26500/50462 | Loss: 2.5816 | LR: 2.36e-04 | 4015.5ms/step | 14.9 samples/s
+Validation at step 26500: Loss = 2.6047, Perplexity = 13.53
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-26500
+Step 26510/50462 | Loss: 2.5851 | LR: 2.36e-04 | 5867.0ms/step | 10.2 samples/s
+Step 26520/50462 | Loss: 2.5968 | LR: 2.36e-04 | 4000.7ms/step | 15.0 samples/s
+Step 26530/50462 | Loss: 2.5959 | LR: 2.35e-04 | 4004.5ms/step | 15.0 samples/s
+Step 26540/50462 | Loss: 2.5926 | LR: 2.35e-04 | 4014.6ms/step | 14.9 samples/s
+Step 26550/50462 | Loss: 2.6270 | LR: 2.35e-04 | 4009.0ms/step | 15.0 samples/s
+Step 26560/50462 | Loss: 2.5743 | LR: 2.35e-04 | 4003.2ms/step | 15.0 samples/s
+Step 26570/50462 | Loss: 2.6094 | LR: 2.35e-04 | 4000.0ms/step | 15.0 samples/s
+Step 26580/50462 | Loss: 2.5688 | LR: 2.35e-04 | 4015.5ms/step | 14.9 samples/s
+Step 26590/50462 | Loss: 2.5596 | LR: 2.34e-04 | 4001.9ms/step | 15.0 samples/s
+Step 26600/50462 | Loss: 2.6046 | LR: 2.34e-04 | 3998.3ms/step | 15.0 samples/s
+Step 26610/50462 | Loss: 2.5984 | LR: 2.34e-04 | 4011.3ms/step | 15.0 samples/s
+Step 26620/50462 | Loss: 2.5994 | LR: 2.34e-04 | 4010.7ms/step | 15.0 samples/s
+Step 26630/50462 | Loss: 2.5994 | LR: 2.34e-04 | 4017.3ms/step | 14.9 samples/s
+Step 26640/50462 | Loss: 2.5783 | LR: 2.34e-04 | 4006.5ms/step | 15.0 samples/s
+Step 26650/50462 | Loss: 2.6084 | LR: 2.34e-04 | 4017.5ms/step | 14.9 samples/s
+Step 26660/50462 | Loss: 2.6024 | LR: 2.33e-04 | 4007.2ms/step | 15.0 samples/s
+Step 26670/50462 | Loss: 2.5304 | LR: 2.33e-04 | 4010.7ms/step | 15.0 samples/s
+Step 26680/50462 | Loss: 2.6146 | LR: 2.33e-04 | 4579.8ms/step | 13.1 samples/s
+Step 26690/50462 | Loss: 2.5709 | LR: 2.33e-04 | 4006.9ms/step | 15.0 samples/s
+Step 26700/50462 | Loss: 2.5713 | LR: 2.33e-04 | 4014.6ms/step | 14.9 samples/s
+Step 26710/50462 | Loss: 2.5760 | LR: 2.33e-04 | 4006.6ms/step | 15.0 samples/s
+Step 26720/50462 | Loss: 2.5753 | LR: 2.32e-04 | 4012.9ms/step | 15.0 samples/s
+Step 26730/50462 | Loss: 2.5968 | LR: 2.32e-04 | 4008.7ms/step | 15.0 samples/s
+Step 26740/50462 | Loss: 2.5981 | LR: 2.32e-04 | 3997.3ms/step | 15.0 samples/s
+Step 26750/50462 | Loss: 2.5618 | LR: 2.32e-04 | 4001.7ms/step | 15.0 samples/s
+Validation at step 26750: Loss = 2.6021, Perplexity = 13.49
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-26750
+Step 26760/50462 | Loss: 2.5784 | LR: 2.32e-04 | 5883.2ms/step | 10.2 samples/s
+Step 26770/50462 | Loss: 2.5562 | LR: 2.32e-04 | 4007.5ms/step | 15.0 samples/s
+Step 26780/50462 | Loss: 2.5895 | LR: 2.31e-04 | 3995.0ms/step | 15.0 samples/s
+Step 26790/50462 | Loss: 2.6099 | LR: 2.31e-04 | 4007.4ms/step | 15.0 samples/s
+Step 26800/50462 | Loss: 2.5785 | LR: 2.31e-04 | 4005.5ms/step | 15.0 samples/s
+Step 26810/50462 | Loss: 2.5922 | LR: 2.31e-04 | 4006.7ms/step | 15.0 samples/s
+Step 26820/50462 | Loss: 2.5990 | LR: 2.31e-04 | 4005.6ms/step | 15.0 samples/s
+Step 26830/50462 | Loss: 2.5730 | LR: 2.31e-04 | 4001.7ms/step | 15.0 samples/s
+Step 26840/50462 | Loss: 2.5676 | LR: 2.31e-04 | 4005.5ms/step | 15.0 samples/s
+Step 26850/50462 | Loss: 2.5644 | LR: 2.30e-04 | 4015.5ms/step | 14.9 samples/s
+Step 26860/50462 | Loss: 2.6272 | LR: 2.30e-04 | 4004.7ms/step | 15.0 samples/s
+Step 26870/50462 | Loss: 2.5952 | LR: 2.30e-04 | 4014.7ms/step | 14.9 samples/s
+Step 26880/50462 | Loss: 2.5862 | LR: 2.30e-04 | 3990.4ms/step | 15.0 samples/s
+Step 26890/50462 | Loss: 2.5861 | LR: 2.30e-04 | 4005.7ms/step | 15.0 samples/s
+Step 26900/50462 | Loss: 2.5933 | LR: 2.30e-04 | 3986.1ms/step | 15.1 samples/s
+Step 26910/50462 | Loss: 2.5948 | LR: 2.29e-04 | 3998.3ms/step | 15.0 samples/s
+Step 26920/50462 | Loss: 2.5595 | LR: 2.29e-04 | 4020.8ms/step | 14.9 samples/s
+Step 26930/50462 | Loss: 2.5562 | LR: 2.29e-04 | 4005.7ms/step | 15.0 samples/s
+Step 26940/50462 | Loss: 2.5914 | LR: 2.29e-04 | 3998.7ms/step | 15.0 samples/s
+Step 26950/50462 | Loss: 2.5688 | LR: 2.29e-04 | 4019.0ms/step | 14.9 samples/s
+Step 26960/50462 | Loss: 2.5958 | LR: 2.29e-04 | 4020.6ms/step | 14.9 samples/s
+Step 26970/50462 | Loss: 2.5795 | LR: 2.28e-04 | 4011.3ms/step | 15.0 samples/s
+Step 26980/50462 | Loss: 2.5710 | LR: 2.28e-04 | 4015.7ms/step | 14.9 samples/s
+Step 26990/50462 | Loss: 2.5767 | LR: 2.28e-04 | 3993.9ms/step | 15.0 samples/s
+Step 27000/50462 | Loss: 2.5847 | LR: 2.28e-04 | 4011.3ms/step | 15.0 samples/s
+Validation at step 27000: Loss = 2.5992, Perplexity = 13.45
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-27000
+Step 27010/50462 | Loss: 2.5624 | LR: 2.28e-04 | 5895.5ms/step | 10.2 samples/s
+Step 27020/50462 | Loss: 2.5521 | LR: 2.28e-04 | 4010.3ms/step | 15.0 samples/s
+Step 27030/50462 | Loss: 2.5952 | LR: 2.28e-04 | 3990.2ms/step | 15.0 samples/s
+Step 27040/50462 | Loss: 2.5870 | LR: 2.27e-04 | 3998.5ms/step | 15.0 samples/s
+Step 27050/50462 | Loss: 2.5833 | LR: 2.27e-04 | 4004.0ms/step | 15.0 samples/s
+Step 27060/50462 | Loss: 2.5873 | LR: 2.27e-04 | 4008.3ms/step | 15.0 samples/s
+Step 27070/50462 | Loss: 2.5696 | LR: 2.27e-04 | 4011.6ms/step | 15.0 samples/s
+Step 27080/50462 | Loss: 2.5794 | LR: 2.27e-04 | 4005.2ms/step | 15.0 samples/s
+Step 27090/50462 | Loss: 2.5960 | LR: 2.27e-04 | 4007.3ms/step | 15.0 samples/s
+Step 27100/50462 | Loss: 2.5724 | LR: 2.26e-04 | 4003.8ms/step | 15.0 samples/s
+Step 27110/50462 | Loss: 2.5812 | LR: 2.26e-04 | 4019.4ms/step | 14.9 samples/s
+Step 27120/50462 | Loss: 2.5819 | LR: 2.26e-04 | 4011.2ms/step | 15.0 samples/s
+Step 27130/50462 | Loss: 2.5860 | LR: 2.26e-04 | 4013.3ms/step | 15.0 samples/s
+Step 27140/50462 | Loss: 2.5859 | LR: 2.26e-04 | 4000.8ms/step | 15.0 samples/s
+Step 27150/50462 | Loss: 2.5593 | LR: 2.26e-04 | 4002.3ms/step | 15.0 samples/s
+Step 27160/50462 | Loss: 2.5824 | LR: 2.25e-04 | 4015.2ms/step | 14.9 samples/s
+Step 27170/50462 | Loss: 2.5691 | LR: 2.25e-04 | 4002.4ms/step | 15.0 samples/s
+Step 27180/50462 | Loss: 2.5856 | LR: 2.25e-04 | 3998.7ms/step | 15.0 samples/s
+Step 27190/50462 | Loss: 2.5932 | LR: 2.25e-04 | 4007.8ms/step | 15.0 samples/s
+Step 27200/50462 | Loss: 2.5553 | LR: 2.25e-04 | 4004.2ms/step | 15.0 samples/s
+Step 27210/50462 | Loss: 2.5560 | LR: 2.25e-04 | 4014.2ms/step | 14.9 samples/s
+Step 27220/50462 | Loss: 2.5873 | LR: 2.25e-04 | 4016.1ms/step | 14.9 samples/s
+Step 27230/50462 | Loss: 2.5729 | LR: 2.24e-04 | 4012.7ms/step | 15.0 samples/s
+Step 27240/50462 | Loss: 2.5773 | LR: 2.24e-04 | 4002.4ms/step | 15.0 samples/s
+Step 27250/50462 | Loss: 2.5795 | LR: 2.24e-04 | 4003.6ms/step | 15.0 samples/s
+Validation at step 27250: Loss = 2.5992, Perplexity = 13.45
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-27250
+Step 27260/50462 | Loss: 2.5288 | LR: 2.24e-04 | 5839.3ms/step | 10.3 samples/s
+Step 27270/50462 | Loss: 2.5535 | LR: 2.24e-04 | 4008.9ms/step | 15.0 samples/s
+Step 27280/50462 | Loss: 2.6165 | LR: 2.24e-04 | 4014.0ms/step | 14.9 samples/s
+Step 27290/50462 | Loss: 2.5651 | LR: 2.23e-04 | 4008.2ms/step | 15.0 samples/s
+Step 27300/50462 | Loss: 2.5612 | LR: 2.23e-04 | 4003.0ms/step | 15.0 samples/s
+Step 27310/50462 | Loss: 2.5902 | LR: 2.23e-04 | 3991.2ms/step | 15.0 samples/s
+Step 27320/50462 | Loss: 2.5794 | LR: 2.23e-04 | 4000.4ms/step | 15.0 samples/s
+Step 27330/50462 | Loss: 2.5482 | LR: 2.23e-04 | 3983.2ms/step | 15.1 samples/s
+Step 27340/50462 | Loss: 2.5846 | LR: 2.23e-04 | 3998.2ms/step | 15.0 samples/s
+Step 27350/50462 | Loss: 2.5797 | LR: 2.23e-04 | 4003.6ms/step | 15.0 samples/s
+Step 27360/50462 | Loss: 2.5544 | LR: 2.22e-04 | 3997.8ms/step | 15.0 samples/s
+Step 27370/50462 | Loss: 2.6152 | LR: 2.22e-04 | 4017.1ms/step | 14.9 samples/s
+Step 27380/50462 | Loss: 2.5583 | LR: 2.22e-04 | 4019.5ms/step | 14.9 samples/s
+Step 27390/50462 | Loss: 2.5687 | LR: 2.22e-04 | 3997.0ms/step | 15.0 samples/s
+Step 27400/50462 | Loss: 2.5816 | LR: 2.22e-04 | 4010.0ms/step | 15.0 samples/s
+Step 27410/50462 | Loss: 2.5670 | LR: 2.22e-04 | 4009.4ms/step | 15.0 samples/s
+Step 27420/50462 | Loss: 2.5594 | LR: 2.21e-04 | 4024.2ms/step | 14.9 samples/s
+Step 27430/50462 | Loss: 2.5573 | LR: 2.21e-04 | 4011.0ms/step | 15.0 samples/s
+Step 27440/50462 | Loss: 2.5756 | LR: 2.21e-04 | 4012.6ms/step | 15.0 samples/s
+Step 27450/50462 | Loss: 2.5559 | LR: 2.21e-04 | 4015.7ms/step | 14.9 samples/s
+Step 27460/50462 | Loss: 2.5533 | LR: 2.21e-04 | 4019.2ms/step | 14.9 samples/s
+Step 27470/50462 | Loss: 2.5973 | LR: 2.21e-04 | 4006.1ms/step | 15.0 samples/s
+Step 27480/50462 | Loss: 2.5765 | LR: 2.20e-04 | 4016.0ms/step | 14.9 samples/s
+Step 27490/50462 | Loss: 2.5768 | LR: 2.20e-04 | 4016.9ms/step | 14.9 samples/s
+Step 27500/50462 | Loss: 2.6068 | LR: 2.20e-04 | 4010.5ms/step | 15.0 samples/s
+Validation at step 27500: Loss = 2.5974, Perplexity = 13.43
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-27500
+Step 27510/50462 | Loss: 2.5780 | LR: 2.20e-04 | 5861.9ms/step | 10.2 samples/s
+Step 27520/50462 | Loss: 2.5644 | LR: 2.20e-04 | 4005.0ms/step | 15.0 samples/s
+Step 27530/50462 | Loss: 2.5793 | LR: 2.20e-04 | 4015.0ms/step | 14.9 samples/s
+Step 27540/50462 | Loss: 2.5713 | LR: 2.20e-04 | 4009.4ms/step | 15.0 samples/s
+Step 27550/50462 | Loss: 2.5615 | LR: 2.19e-04 | 4009.2ms/step | 15.0 samples/s
+Step 27560/50462 | Loss: 2.6155 | LR: 2.19e-04 | 4015.3ms/step | 14.9 samples/s
+Step 27570/50462 | Loss: 2.5759 | LR: 2.19e-04 | 4001.8ms/step | 15.0 samples/s
+Step 27580/50462 | Loss: 2.5546 | LR: 2.19e-04 | 3998.3ms/step | 15.0 samples/s
+Step 27590/50462 | Loss: 2.5781 | LR: 2.19e-04 | 4012.8ms/step | 15.0 samples/s
+Step 27600/50462 | Loss: 2.5667 | LR: 2.19e-04 | 4011.0ms/step | 15.0 samples/s
+Step 27610/50462 | Loss: 2.6037 | LR: 2.18e-04 | 4014.5ms/step | 14.9 samples/s
+Step 27620/50462 | Loss: 2.5669 | LR: 2.18e-04 | 4004.2ms/step | 15.0 samples/s
+Step 27630/50462 | Loss: 2.5614 | LR: 2.18e-04 | 4003.0ms/step | 15.0 samples/s
+Step 27640/50462 | Loss: 2.5466 | LR: 2.18e-04 | 4015.4ms/step | 14.9 samples/s
+Step 27650/50462 | Loss: 2.5726 | LR: 2.18e-04 | 4015.6ms/step | 14.9 samples/s
+Step 27660/50462 | Loss: 2.5701 | LR: 2.18e-04 | 4012.6ms/step | 15.0 samples/s
+Step 27670/50462 | Loss: 2.5745 | LR: 2.17e-04 | 3993.7ms/step | 15.0 samples/s
+Step 27680/50462 | Loss: 2.5743 | LR: 2.17e-04 | 4004.5ms/step | 15.0 samples/s
+Step 27690/50462 | Loss: 2.5834 | LR: 2.17e-04 | 4003.8ms/step | 15.0 samples/s
+Step 27700/50462 | Loss: 2.5588 | LR: 2.17e-04 | 4002.1ms/step | 15.0 samples/s
+Step 27710/50462 | Loss: 2.5688 | LR: 2.17e-04 | 4014.7ms/step | 14.9 samples/s
+Step 27720/50462 | Loss: 2.5925 | LR: 2.17e-04 | 4004.3ms/step | 15.0 samples/s
+Step 27730/50462 | Loss: 2.5488 | LR: 2.17e-04 | 4019.5ms/step | 14.9 samples/s
+Step 27740/50462 | Loss: 2.5534 | LR: 2.16e-04 | 4006.9ms/step | 15.0 samples/s
+Step 27750/50462 | Loss: 2.5913 | LR: 2.16e-04 | 4009.3ms/step | 15.0 samples/s
+Validation at step 27750: Loss = 2.5953, Perplexity = 13.40
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-27750
+Step 27760/50462 | Loss: 2.5951 | LR: 2.16e-04 | 5840.3ms/step | 10.3 samples/s
+Step 27770/50462 | Loss: 2.5425 | LR: 2.16e-04 | 4010.4ms/step | 15.0 samples/s
+Step 27780/50462 | Loss: 2.6086 | LR: 2.16e-04 | 4005.9ms/step | 15.0 samples/s
+Step 27790/50462 | Loss: 2.5732 | LR: 2.16e-04 | 4022.7ms/step | 14.9 samples/s
+Step 27800/50462 | Loss: 2.5568 | LR: 2.15e-04 | 4005.1ms/step | 15.0 samples/s
+Step 27810/50462 | Loss: 2.5846 | LR: 2.15e-04 | 4001.1ms/step | 15.0 samples/s
+Step 27820/50462 | Loss: 2.5673 | LR: 2.15e-04 | 3998.4ms/step | 15.0 samples/s
+Step 27830/50462 | Loss: 2.5607 | LR: 2.15e-04 | 3996.8ms/step | 15.0 samples/s
+Step 27840/50462 | Loss: 2.5557 | LR: 2.15e-04 | 4004.3ms/step | 15.0 samples/s
+Step 27850/50462 | Loss: 2.5431 | LR: 2.15e-04 | 3997.8ms/step | 15.0 samples/s
+Step 27860/50462 | Loss: 2.5941 | LR: 2.15e-04 | 4001.4ms/step | 15.0 samples/s
+Step 27870/50462 | Loss: 2.5677 | LR: 2.14e-04 | 4001.7ms/step | 15.0 samples/s
+Step 27880/50462 | Loss: 2.5614 | LR: 2.14e-04 | 4003.0ms/step | 15.0 samples/s
+Step 27890/50462 | Loss: 2.5649 | LR: 2.14e-04 | 4014.0ms/step | 14.9 samples/s
+Step 27900/50462 | Loss: 2.5661 | LR: 2.14e-04 | 4003.6ms/step | 15.0 samples/s
+Step 27910/50462 | Loss: 2.5921 | LR: 2.14e-04 | 4012.8ms/step | 15.0 samples/s
+Step 27920/50462 | Loss: 2.5863 | LR: 2.14e-04 | 4020.9ms/step | 14.9 samples/s
+Step 27930/50462 | Loss: 2.5459 | LR: 2.13e-04 | 4013.5ms/step | 14.9 samples/s
+Step 27940/50462 | Loss: 2.5811 | LR: 2.13e-04 | 4022.5ms/step | 14.9 samples/s
+Step 27950/50462 | Loss: 2.5672 | LR: 2.13e-04 | 4029.0ms/step | 14.9 samples/s
+Step 27960/50462 | Loss: 2.5672 | LR: 2.13e-04 | 4018.7ms/step | 14.9 samples/s
+Step 27970/50462 | Loss: 2.5740 | LR: 2.13e-04 | 4034.9ms/step | 14.9 samples/s
+Step 27980/50462 | Loss: 2.5634 | LR: 2.13e-04 | 4027.4ms/step | 14.9 samples/s
+Step 27990/50462 | Loss: 2.5721 | LR: 2.12e-04 | 4015.7ms/step | 14.9 samples/s
+Step 28000/50462 | Loss: 2.5381 | LR: 2.12e-04 | 4016.5ms/step | 14.9 samples/s
+Validation at step 28000: Loss = 2.5933, Perplexity = 13.37
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-28000
+Step 28010/50462 | Loss: 2.5490 | LR: 2.12e-04 | 5894.6ms/step | 10.2 samples/s
+Step 28020/50462 | Loss: 2.5311 | LR: 2.12e-04 | 4004.6ms/step | 15.0 samples/s
+Step 28030/50462 | Loss: 2.5633 | LR: 2.12e-04 | 4011.9ms/step | 15.0 samples/s
+Step 28040/50462 | Loss: 2.5374 | LR: 2.12e-04 | 4011.1ms/step | 15.0 samples/s
+Step 28050/50462 | Loss: 2.5810 | LR: 2.12e-04 | 4003.1ms/step | 15.0 samples/s
+Step 28060/50462 | Loss: 2.5808 | LR: 2.11e-04 | 4009.9ms/step | 15.0 samples/s
+Step 28070/50462 | Loss: 2.5684 | LR: 2.11e-04 | 3999.6ms/step | 15.0 samples/s
+Step 28080/50462 | Loss: 2.5412 | LR: 2.11e-04 | 3997.5ms/step | 15.0 samples/s
+Step 28090/50462 | Loss: 2.5767 | LR: 2.11e-04 | 4003.5ms/step | 15.0 samples/s
+Step 28100/50462 | Loss: 2.5991 | LR: 2.11e-04 | 4009.9ms/step | 15.0 samples/s
+Step 28110/50462 | Loss: 2.5731 | LR: 2.11e-04 | 3998.6ms/step | 15.0 samples/s
+Step 28120/50462 | Loss: 2.5864 | LR: 2.10e-04 | 4004.2ms/step | 15.0 samples/s
+Step 28130/50462 | Loss: 2.5934 | LR: 2.10e-04 | 4001.6ms/step | 15.0 samples/s
+Step 28140/50462 | Loss: 2.6212 | LR: 2.10e-04 | 3998.4ms/step | 15.0 samples/s
+Step 28150/50462 | Loss: 2.6062 | LR: 2.10e-04 | 4002.6ms/step | 15.0 samples/s
+Step 28160/50462 | Loss: 2.5651 | LR: 2.10e-04 | 3999.5ms/step | 15.0 samples/s
+Step 28170/50462 | Loss: 2.5805 | LR: 2.10e-04 | 4003.9ms/step | 15.0 samples/s
+Step 28180/50462 | Loss: 2.5935 | LR: 2.10e-04 | 4016.4ms/step | 14.9 samples/s
+Step 28190/50462 | Loss: 2.6050 | LR: 2.09e-04 | 4005.4ms/step | 15.0 samples/s
+Step 28200/50462 | Loss: 2.5570 | LR: 2.09e-04 | 4002.3ms/step | 15.0 samples/s
+Step 28210/50462 | Loss: 2.5839 | LR: 2.09e-04 | 4011.5ms/step | 15.0 samples/s
+Step 28220/50462 | Loss: 2.5695 | LR: 2.09e-04 | 4012.0ms/step | 15.0 samples/s
+Step 28230/50462 | Loss: 2.5700 | LR: 2.09e-04 | 4009.3ms/step | 15.0 samples/s
+Step 28240/50462 | Loss: 2.6183 | LR: 2.09e-04 | 4005.4ms/step | 15.0 samples/s
+Step 28250/50462 | Loss: 2.5520 | LR: 2.08e-04 | 4018.6ms/step | 14.9 samples/s
+Validation at step 28250: Loss = 2.5917, Perplexity = 13.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-28250
+Step 28260/50462 | Loss: 2.5770 | LR: 2.08e-04 | 5885.3ms/step | 10.2 samples/s
+Step 28270/50462 | Loss: 2.5498 | LR: 2.08e-04 | 4017.2ms/step | 14.9 samples/s
+Step 28280/50462 | Loss: 2.5914 | LR: 2.08e-04 | 4009.6ms/step | 15.0 samples/s
+Step 28290/50462 | Loss: 2.5939 | LR: 2.08e-04 | 4004.7ms/step | 15.0 samples/s
+Step 28300/50462 | Loss: 2.6334 | LR: 2.08e-04 | 4022.3ms/step | 14.9 samples/s
+Step 28310/50462 | Loss: 2.6075 | LR: 2.07e-04 | 4008.0ms/step | 15.0 samples/s
+Step 28320/50462 | Loss: 2.5609 | LR: 2.07e-04 | 4011.1ms/step | 15.0 samples/s
+Step 28330/50462 | Loss: 2.5892 | LR: 2.07e-04 | 4006.7ms/step | 15.0 samples/s
+Step 28340/50462 | Loss: 2.5694 | LR: 2.07e-04 | 4006.1ms/step | 15.0 samples/s
+Step 28350/50462 | Loss: 2.5958 | LR: 2.07e-04 | 3993.1ms/step | 15.0 samples/s
+Step 28360/50462 | Loss: 2.5802 | LR: 2.07e-04 | 4008.3ms/step | 15.0 samples/s
+Step 28370/50462 | Loss: 2.5942 | LR: 2.07e-04 | 3988.8ms/step | 15.0 samples/s
+Step 28380/50462 | Loss: 2.5433 | LR: 2.06e-04 | 4009.6ms/step | 15.0 samples/s
+Step 28390/50462 | Loss: 2.5795 | LR: 2.06e-04 | 3999.4ms/step | 15.0 samples/s
+Step 28400/50462 | Loss: 2.5751 | LR: 2.06e-04 | 4017.9ms/step | 14.9 samples/s
+Step 28410/50462 | Loss: 2.5886 | LR: 2.06e-04 | 4007.0ms/step | 15.0 samples/s
+Step 28420/50462 | Loss: 2.5385 | LR: 2.06e-04 | 4012.1ms/step | 15.0 samples/s
+Step 28430/50462 | Loss: 2.5518 | LR: 2.06e-04 | 4010.0ms/step | 15.0 samples/s
+Step 28440/50462 | Loss: 2.5702 | LR: 2.05e-04 | 3998.0ms/step | 15.0 samples/s
+Step 28450/50462 | Loss: 2.5451 | LR: 2.05e-04 | 4002.5ms/step | 15.0 samples/s
+Step 28460/50462 | Loss: 2.5790 | LR: 2.05e-04 | 4010.5ms/step | 15.0 samples/s
+Step 28470/50462 | Loss: 2.5813 | LR: 2.05e-04 | 4009.9ms/step | 15.0 samples/s
+Step 28480/50462 | Loss: 2.5294 | LR: 2.05e-04 | 4002.4ms/step | 15.0 samples/s
+Step 28490/50462 | Loss: 2.5458 | LR: 2.05e-04 | 4005.4ms/step | 15.0 samples/s
+Step 28500/50462 | Loss: 2.5619 | LR: 2.05e-04 | 4011.9ms/step | 15.0 samples/s
+Validation at step 28500: Loss = 2.5907, Perplexity = 13.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-28500
+Step 28510/50462 | Loss: 2.5414 | LR: 2.04e-04 | 5870.4ms/step | 10.2 samples/s
+Step 28520/50462 | Loss: 2.5574 | LR: 2.04e-04 | 4006.7ms/step | 15.0 samples/s
+Step 28530/50462 | Loss: 2.5582 | LR: 2.04e-04 | 3999.3ms/step | 15.0 samples/s
+Step 28540/50462 | Loss: 2.5456 | LR: 2.04e-04 | 4014.6ms/step | 14.9 samples/s
+Step 28550/50462 | Loss: 2.6087 | LR: 2.04e-04 | 4030.4ms/step | 14.9 samples/s
+Step 28560/50462 | Loss: 2.5712 | LR: 2.04e-04 | 4014.0ms/step | 14.9 samples/s
+Step 28570/50462 | Loss: 2.5804 | LR: 2.03e-04 | 4028.7ms/step | 14.9 samples/s
+Step 28580/50462 | Loss: 2.5765 | LR: 2.03e-04 | 4002.1ms/step | 15.0 samples/s
+Step 28590/50462 | Loss: 2.5983 | LR: 2.03e-04 | 4007.4ms/step | 15.0 samples/s
+Step 28600/50462 | Loss: 2.5975 | LR: 2.03e-04 | 4007.6ms/step | 15.0 samples/s
+Step 28610/50462 | Loss: 2.5641 | LR: 2.03e-04 | 4011.3ms/step | 15.0 samples/s
+Step 28620/50462 | Loss: 2.5739 | LR: 2.03e-04 | 4000.8ms/step | 15.0 samples/s
+Step 28630/50462 | Loss: 2.5955 | LR: 2.03e-04 | 4007.6ms/step | 15.0 samples/s
+Step 28640/50462 | Loss: 2.5457 | LR: 2.02e-04 | 4002.4ms/step | 15.0 samples/s
+Step 28650/50462 | Loss: 2.5545 | LR: 2.02e-04 | 4010.9ms/step | 15.0 samples/s
+Step 28660/50462 | Loss: 2.6446 | LR: 2.02e-04 | 4015.7ms/step | 14.9 samples/s
+Step 28670/50462 | Loss: 2.5769 | LR: 2.02e-04 | 4000.8ms/step | 15.0 samples/s
+Step 28680/50462 | Loss: 2.5671 | LR: 2.02e-04 | 4003.6ms/step | 15.0 samples/s
+Step 28690/50462 | Loss: 2.5597 | LR: 2.02e-04 | 3999.5ms/step | 15.0 samples/s
+Step 28700/50462 | Loss: 2.5702 | LR: 2.01e-04 | 4006.2ms/step | 15.0 samples/s
+Step 28710/50462 | Loss: 2.5655 | LR: 2.01e-04 | 4004.9ms/step | 15.0 samples/s
+Step 28720/50462 | Loss: 2.5658 | LR: 2.01e-04 | 4011.5ms/step | 15.0 samples/s
+Step 28730/50462 | Loss: 2.5536 | LR: 2.01e-04 | 4003.1ms/step | 15.0 samples/s
+Step 28740/50462 | Loss: 2.5602 | LR: 2.01e-04 | 4012.9ms/step | 15.0 samples/s
+Step 28750/50462 | Loss: 2.5693 | LR: 2.01e-04 | 4001.2ms/step | 15.0 samples/s
+Validation at step 28750: Loss = 2.5895, Perplexity = 13.32
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-28750
+Step 28760/50462 | Loss: 2.5592 | LR: 2.00e-04 | 5866.8ms/step | 10.2 samples/s
+Step 28770/50462 | Loss: 2.5974 | LR: 2.00e-04 | 4006.1ms/step | 15.0 samples/s
+Step 28780/50462 | Loss: 2.5860 | LR: 2.00e-04 | 4008.9ms/step | 15.0 samples/s
+Step 28790/50462 | Loss: 2.5383 | LR: 2.00e-04 | 4008.2ms/step | 15.0 samples/s
+Step 28800/50462 | Loss: 2.5707 | LR: 2.00e-04 | 4012.9ms/step | 15.0 samples/s
+Step 28810/50462 | Loss: 2.5191 | LR: 2.00e-04 | 4005.3ms/step | 15.0 samples/s
+Step 28820/50462 | Loss: 2.5949 | LR: 2.00e-04 | 4000.0ms/step | 15.0 samples/s
+Step 28830/50462 | Loss: 2.5262 | LR: 1.99e-04 | 4002.3ms/step | 15.0 samples/s
+Step 28840/50462 | Loss: 2.5327 | LR: 1.99e-04 | 4004.2ms/step | 15.0 samples/s
+Step 28850/50462 | Loss: 2.5982 | LR: 1.99e-04 | 4000.7ms/step | 15.0 samples/s
+Step 28860/50462 | Loss: 2.5656 | LR: 1.99e-04 | 4000.2ms/step | 15.0 samples/s
+Step 28870/50462 | Loss: 2.5700 | LR: 1.99e-04 | 4008.4ms/step | 15.0 samples/s
+Step 28880/50462 | Loss: 2.5216 | LR: 1.99e-04 | 4000.5ms/step | 15.0 samples/s
+Step 28890/50462 | Loss: 2.5482 | LR: 1.98e-04 | 4013.8ms/step | 14.9 samples/s
+Step 28900/50462 | Loss: 2.5608 | LR: 1.98e-04 | 4007.8ms/step | 15.0 samples/s
+Step 28910/50462 | Loss: 2.5663 | LR: 1.98e-04 | 4005.6ms/step | 15.0 samples/s
+Step 28920/50462 | Loss: 2.5897 | LR: 1.98e-04 | 4006.6ms/step | 15.0 samples/s
+Step 28930/50462 | Loss: 2.5831 | LR: 1.98e-04 | 4012.4ms/step | 15.0 samples/s
+Step 28940/50462 | Loss: 2.5523 | LR: 1.98e-04 | 3991.1ms/step | 15.0 samples/s
+Step 28950/50462 | Loss: 2.5823 | LR: 1.98e-04 | 4002.7ms/step | 15.0 samples/s
+Step 28960/50462 | Loss: 2.5692 | LR: 1.97e-04 | 4004.9ms/step | 15.0 samples/s
+Step 28970/50462 | Loss: 2.5690 | LR: 1.97e-04 | 4007.8ms/step | 15.0 samples/s
+Step 28980/50462 | Loss: 2.5435 | LR: 1.97e-04 | 4013.4ms/step | 14.9 samples/s
+Step 28990/50462 | Loss: 2.5316 | LR: 1.97e-04 | 3998.6ms/step | 15.0 samples/s
+Step 29000/50462 | Loss: 2.5825 | LR: 1.97e-04 | 3992.7ms/step | 15.0 samples/s
+Validation at step 29000: Loss = 2.5891, Perplexity = 13.32
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-29000
+Step 29010/50462 | Loss: 2.5562 | LR: 1.97e-04 | 5869.4ms/step | 10.2 samples/s
+Step 29020/50462 | Loss: 2.5386 | LR: 1.96e-04 | 4005.9ms/step | 15.0 samples/s
+Step 29030/50462 | Loss: 2.5817 | LR: 1.96e-04 | 4005.5ms/step | 15.0 samples/s
+Step 29040/50462 | Loss: 2.5680 | LR: 1.96e-04 | 3999.7ms/step | 15.0 samples/s
+Step 29050/50462 | Loss: 2.5984 | LR: 1.96e-04 | 4000.1ms/step | 15.0 samples/s
+Step 29060/50462 | Loss: 2.5597 | LR: 1.96e-04 | 4006.0ms/step | 15.0 samples/s
+Step 29070/50462 | Loss: 2.5676 | LR: 1.96e-04 | 4003.8ms/step | 15.0 samples/s
+Step 29080/50462 | Loss: 2.5703 | LR: 1.96e-04 | 4000.9ms/step | 15.0 samples/s
+Step 29090/50462 | Loss: 2.5794 | LR: 1.95e-04 | 4007.5ms/step | 15.0 samples/s
+Step 29100/50462 | Loss: 2.5499 | LR: 1.95e-04 | 4002.6ms/step | 15.0 samples/s
+Step 29110/50462 | Loss: 2.5474 | LR: 1.95e-04 | 4003.2ms/step | 15.0 samples/s
+Step 29120/50462 | Loss: 2.5794 | LR: 1.95e-04 | 4016.3ms/step | 14.9 samples/s
+Step 29130/50462 | Loss: 2.5724 | LR: 1.95e-04 | 4019.9ms/step | 14.9 samples/s
+Step 29140/50462 | Loss: 2.5585 | LR: 1.95e-04 | 4005.7ms/step | 15.0 samples/s
+Step 29150/50462 | Loss: 2.5533 | LR: 1.94e-04 | 4000.3ms/step | 15.0 samples/s
+Step 29160/50462 | Loss: 2.5557 | LR: 1.94e-04 | 4012.1ms/step | 15.0 samples/s
+Step 29170/50462 | Loss: 2.5831 | LR: 1.94e-04 | 4001.0ms/step | 15.0 samples/s
+Step 29180/50462 | Loss: 2.5694 | LR: 1.94e-04 | 4003.3ms/step | 15.0 samples/s
+Step 29190/50462 | Loss: 2.5534 | LR: 1.94e-04 | 4001.7ms/step | 15.0 samples/s
+Step 29200/50462 | Loss: 2.5445 | LR: 1.94e-04 | 4008.4ms/step | 15.0 samples/s
+Step 29210/50462 | Loss: 2.5859 | LR: 1.94e-04 | 3998.2ms/step | 15.0 samples/s
+Step 29220/50462 | Loss: 2.5480 | LR: 1.93e-04 | 4018.2ms/step | 14.9 samples/s
+Step 29230/50462 | Loss: 2.5323 | LR: 1.93e-04 | 4021.0ms/step | 14.9 samples/s
+Step 29240/50462 | Loss: 2.5375 | LR: 1.93e-04 | 4000.4ms/step | 15.0 samples/s
+Step 29250/50462 | Loss: 2.5777 | LR: 1.93e-04 | 4009.3ms/step | 15.0 samples/s
+Validation at step 29250: Loss = 2.5859, Perplexity = 13.27
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-29250
+Step 29260/50462 | Loss: 2.5489 | LR: 1.93e-04 | 5873.3ms/step | 10.2 samples/s
+Step 29270/50462 | Loss: 2.5634 | LR: 1.93e-04 | 4014.9ms/step | 14.9 samples/s
+Step 29280/50462 | Loss: 2.5499 | LR: 1.92e-04 | 4003.7ms/step | 15.0 samples/s
+Step 29290/50462 | Loss: 2.5549 | LR: 1.92e-04 | 4014.3ms/step | 14.9 samples/s
+Step 29300/50462 | Loss: 2.5512 | LR: 1.92e-04 | 4001.5ms/step | 15.0 samples/s
+Step 29310/50462 | Loss: 2.5813 | LR: 1.92e-04 | 3991.9ms/step | 15.0 samples/s
+Step 29320/50462 | Loss: 2.5834 | LR: 1.92e-04 | 3994.5ms/step | 15.0 samples/s
+Step 29330/50462 | Loss: 2.5751 | LR: 1.92e-04 | 4006.5ms/step | 15.0 samples/s
+Step 29340/50462 | Loss: 2.5664 | LR: 1.92e-04 | 3997.5ms/step | 15.0 samples/s
+Step 29350/50462 | Loss: 2.5649 | LR: 1.91e-04 | 4003.8ms/step | 15.0 samples/s
+Step 29360/50462 | Loss: 2.5791 | LR: 1.91e-04 | 4004.5ms/step | 15.0 samples/s
+Step 29370/50462 | Loss: 2.5702 | LR: 1.91e-04 | 3996.9ms/step | 15.0 samples/s
+Step 29380/50462 | Loss: 2.5543 | LR: 1.91e-04 | 4007.5ms/step | 15.0 samples/s
+Step 29390/50462 | Loss: 2.5705 | LR: 1.91e-04 | 4006.1ms/step | 15.0 samples/s
+Step 29400/50462 | Loss: 2.5745 | LR: 1.91e-04 | 3992.8ms/step | 15.0 samples/s
+Step 29410/50462 | Loss: 2.5687 | LR: 1.90e-04 | 3993.1ms/step | 15.0 samples/s
+Step 29420/50462 | Loss: 2.5918 | LR: 1.90e-04 | 4009.4ms/step | 15.0 samples/s
+Step 29430/50462 | Loss: 2.5579 | LR: 1.90e-04 | 4004.4ms/step | 15.0 samples/s
+Step 29440/50462 | Loss: 2.5625 | LR: 1.90e-04 | 3993.8ms/step | 15.0 samples/s
+Step 29450/50462 | Loss: 2.5681 | LR: 1.90e-04 | 4017.3ms/step | 14.9 samples/s
+Step 29460/50462 | Loss: 2.5599 | LR: 1.90e-04 | 4000.2ms/step | 15.0 samples/s
+Step 29470/50462 | Loss: 2.5667 | LR: 1.90e-04 | 4003.5ms/step | 15.0 samples/s
+Step 29480/50462 | Loss: 2.5556 | LR: 1.89e-04 | 4010.0ms/step | 15.0 samples/s
+Step 29490/50462 | Loss: 2.5405 | LR: 1.89e-04 | 4003.3ms/step | 15.0 samples/s
+Step 29500/50462 | Loss: 2.5687 | LR: 1.89e-04 | 3999.3ms/step | 15.0 samples/s
+Validation at step 29500: Loss = 2.5844, Perplexity = 13.25
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-29500
+Step 29510/50462 | Loss: 2.5743 | LR: 1.89e-04 | 5852.2ms/step | 10.3 samples/s
+Step 29520/50462 | Loss: 2.5636 | LR: 1.89e-04 | 4017.4ms/step | 14.9 samples/s
+Step 29530/50462 | Loss: 2.5431 | LR: 1.89e-04 | 4001.7ms/step | 15.0 samples/s
+Step 29540/50462 | Loss: 2.5577 | LR: 1.88e-04 | 3995.3ms/step | 15.0 samples/s
+Step 29550/50462 | Loss: 2.5452 | LR: 1.88e-04 | 4003.8ms/step | 15.0 samples/s
+Step 29560/50462 | Loss: 2.5712 | LR: 1.88e-04 | 4009.4ms/step | 15.0 samples/s
+Step 29570/50462 | Loss: 2.5476 | LR: 1.88e-04 | 4011.2ms/step | 15.0 samples/s
+Step 29580/50462 | Loss: 2.5774 | LR: 1.88e-04 | 4002.4ms/step | 15.0 samples/s
+Step 29590/50462 | Loss: 2.6001 | LR: 1.88e-04 | 4005.2ms/step | 15.0 samples/s
+Step 29600/50462 | Loss: 2.5495 | LR: 1.88e-04 | 3991.3ms/step | 15.0 samples/s
+Step 29610/50462 | Loss: 2.5557 | LR: 1.87e-04 | 4007.3ms/step | 15.0 samples/s
+Step 29620/50462 | Loss: 2.5748 | LR: 1.87e-04 | 4004.3ms/step | 15.0 samples/s
+Step 29630/50462 | Loss: 2.5855 | LR: 1.87e-04 | 4002.6ms/step | 15.0 samples/s
+Step 29640/50462 | Loss: 2.5613 | LR: 1.87e-04 | 4000.3ms/step | 15.0 samples/s
+Step 29650/50462 | Loss: 2.5708 | LR: 1.87e-04 | 3994.7ms/step | 15.0 samples/s
+Step 29660/50462 | Loss: 2.5740 | LR: 1.87e-04 | 3997.5ms/step | 15.0 samples/s
+Step 29670/50462 | Loss: 2.5639 | LR: 1.86e-04 | 3997.7ms/step | 15.0 samples/s
+Step 29680/50462 | Loss: 2.5530 | LR: 1.86e-04 | 4003.8ms/step | 15.0 samples/s
+Step 29690/50462 | Loss: 2.5658 | LR: 1.86e-04 | 4004.6ms/step | 15.0 samples/s
+Step 29700/50462 | Loss: 2.5620 | LR: 1.86e-04 | 4008.5ms/step | 15.0 samples/s
+Step 29710/50462 | Loss: 2.5647 | LR: 1.86e-04 | 3995.1ms/step | 15.0 samples/s
+Step 29720/50462 | Loss: 2.5907 | LR: 1.86e-04 | 4014.5ms/step | 14.9 samples/s
+Step 29730/50462 | Loss: 2.5404 | LR: 1.86e-04 | 4011.6ms/step | 15.0 samples/s
+Step 29740/50462 | Loss: 2.5723 | LR: 1.85e-04 | 4012.1ms/step | 15.0 samples/s
+Step 29750/50462 | Loss: 2.5618 | LR: 1.85e-04 | 4006.5ms/step | 15.0 samples/s
+Validation at step 29750: Loss = 2.5834, Perplexity = 13.24
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-29750
+Step 29760/50462 | Loss: 2.5587 | LR: 1.85e-04 | 5853.0ms/step | 10.3 samples/s
+Step 29770/50462 | Loss: 2.5630 | LR: 1.85e-04 | 4004.5ms/step | 15.0 samples/s
+Step 29780/50462 | Loss: 2.5772 | LR: 1.85e-04 | 3996.0ms/step | 15.0 samples/s
+Step 29790/50462 | Loss: 2.5865 | LR: 1.85e-04 | 3999.8ms/step | 15.0 samples/s
+Step 29800/50462 | Loss: 2.5858 | LR: 1.85e-04 | 4010.1ms/step | 15.0 samples/s
+Step 29810/50462 | Loss: 2.5574 | LR: 1.84e-04 | 4001.6ms/step | 15.0 samples/s
+Step 29820/50462 | Loss: 2.5411 | LR: 1.84e-04 | 4003.8ms/step | 15.0 samples/s
+Step 29830/50462 | Loss: 2.5659 | LR: 1.84e-04 | 4001.9ms/step | 15.0 samples/s
+Step 29840/50462 | Loss: 2.5662 | LR: 1.84e-04 | 4012.3ms/step | 15.0 samples/s
+Step 29850/50462 | Loss: 2.5259 | LR: 1.84e-04 | 4009.5ms/step | 15.0 samples/s
+Step 29860/50462 | Loss: 2.5950 | LR: 1.84e-04 | 4007.7ms/step | 15.0 samples/s
+Step 29870/50462 | Loss: 2.5322 | LR: 1.83e-04 | 3994.1ms/step | 15.0 samples/s
+Step 29880/50462 | Loss: 2.5846 | LR: 1.83e-04 | 3996.7ms/step | 15.0 samples/s
+Step 29890/50462 | Loss: 2.6032 | LR: 1.83e-04 | 3995.2ms/step | 15.0 samples/s
+Step 29900/50462 | Loss: 2.5548 | LR: 1.83e-04 | 4006.2ms/step | 15.0 samples/s
+Step 29910/50462 | Loss: 2.5736 | LR: 1.83e-04 | 3996.3ms/step | 15.0 samples/s
+Step 29920/50462 | Loss: 2.5823 | LR: 1.83e-04 | 4006.2ms/step | 15.0 samples/s
+Step 29930/50462 | Loss: 2.5870 | LR: 1.83e-04 | 4002.7ms/step | 15.0 samples/s
+Step 29940/50462 | Loss: 2.5801 | LR: 1.82e-04 | 4006.5ms/step | 15.0 samples/s
+Step 29950/50462 | Loss: 2.5625 | LR: 1.82e-04 | 4016.4ms/step | 14.9 samples/s
+Step 29960/50462 | Loss: 2.5707 | LR: 1.82e-04 | 4005.7ms/step | 15.0 samples/s
+Step 29970/50462 | Loss: 2.5408 | LR: 1.82e-04 | 4021.1ms/step | 14.9 samples/s
+Step 29980/50462 | Loss: 2.5530 | LR: 1.82e-04 | 3996.7ms/step | 15.0 samples/s
+Step 29990/50462 | Loss: 2.5622 | LR: 1.82e-04 | 4002.1ms/step | 15.0 samples/s
+Step 30000/50462 | Loss: 2.5579 | LR: 1.81e-04 | 4003.4ms/step | 15.0 samples/s
+Validation at step 30000: Loss = 2.5832, Perplexity = 13.24
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-30000
+Step 30010/50462 | Loss: 2.5898 | LR: 1.81e-04 | 5852.7ms/step | 10.3 samples/s
+Step 30020/50462 | Loss: 2.5715 | LR: 1.81e-04 | 4015.9ms/step | 14.9 samples/s
+Step 30030/50462 | Loss: 2.5664 | LR: 1.81e-04 | 4003.1ms/step | 15.0 samples/s
+Step 30040/50462 | Loss: 2.5294 | LR: 1.81e-04 | 4006.7ms/step | 15.0 samples/s
+Step 30050/50462 | Loss: 2.5646 | LR: 1.81e-04 | 4012.6ms/step | 15.0 samples/s
+Step 30060/50462 | Loss: 2.5616 | LR: 1.81e-04 | 4012.7ms/step | 15.0 samples/s
+Step 30070/50462 | Loss: 2.5653 | LR: 1.80e-04 | 4006.8ms/step | 15.0 samples/s
+Step 30080/50462 | Loss: 2.5485 | LR: 1.80e-04 | 4005.0ms/step | 15.0 samples/s
+Step 30090/50462 | Loss: 2.5562 | LR: 1.80e-04 | 4010.4ms/step | 15.0 samples/s
+Step 30100/50462 | Loss: 2.5918 | LR: 1.80e-04 | 4006.0ms/step | 15.0 samples/s
+Step 30110/50462 | Loss: 2.5876 | LR: 1.80e-04 | 3999.9ms/step | 15.0 samples/s
+Step 30120/50462 | Loss: 2.5459 | LR: 1.80e-04 | 4007.6ms/step | 15.0 samples/s
+Step 30130/50462 | Loss: 2.5566 | LR: 1.79e-04 | 4009.4ms/step | 15.0 samples/s
+Step 30140/50462 | Loss: 2.5404 | LR: 1.79e-04 | 4022.0ms/step | 14.9 samples/s
+Step 30150/50462 | Loss: 2.5443 | LR: 1.79e-04 | 4006.7ms/step | 15.0 samples/s
+Step 30160/50462 | Loss: 2.5797 | LR: 1.79e-04 | 4004.0ms/step | 15.0 samples/s
+Step 30170/50462 | Loss: 2.5887 | LR: 1.79e-04 | 3998.9ms/step | 15.0 samples/s
+Step 30180/50462 | Loss: 2.5581 | LR: 1.79e-04 | 4003.6ms/step | 15.0 samples/s
+Step 30190/50462 | Loss: 2.5326 | LR: 1.79e-04 | 4004.4ms/step | 15.0 samples/s
+Step 30200/50462 | Loss: 2.5533 | LR: 1.78e-04 | 4006.7ms/step | 15.0 samples/s
+Step 30210/50462 | Loss: 2.5682 | LR: 1.78e-04 | 4001.0ms/step | 15.0 samples/s
+Step 30220/50462 | Loss: 2.5202 | LR: 1.78e-04 | 4004.0ms/step | 15.0 samples/s
+Step 30230/50462 | Loss: 2.5500 | LR: 1.78e-04 | 3993.5ms/step | 15.0 samples/s
+Step 30240/50462 | Loss: 2.5903 | LR: 1.78e-04 | 4010.8ms/step | 15.0 samples/s
+Step 30250/50462 | Loss: 2.5532 | LR: 1.78e-04 | 4005.9ms/step | 15.0 samples/s
+Validation at step 30250: Loss = 2.5810, Perplexity = 13.21
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-30250
+Step 30260/50462 | Loss: 2.5700 | LR: 1.78e-04 | 5862.8ms/step | 10.2 samples/s
+Step 30270/50462 | Loss: 2.5551 | LR: 1.77e-04 | 3996.2ms/step | 15.0 samples/s
+Step 30280/50462 | Loss: 2.5504 | LR: 1.77e-04 | 4012.7ms/step | 15.0 samples/s
+Step 30290/50462 | Loss: 2.5582 | LR: 1.77e-04 | 4014.0ms/step | 14.9 samples/s
+Step 30300/50462 | Loss: 2.5591 | LR: 1.77e-04 | 4000.4ms/step | 15.0 samples/s
+Step 30310/50462 | Loss: 2.5591 | LR: 1.77e-04 | 4007.8ms/step | 15.0 samples/s
+Step 30320/50462 | Loss: 2.5626 | LR: 1.77e-04 | 4014.1ms/step | 14.9 samples/s
+Step 30330/50462 | Loss: 2.5666 | LR: 1.76e-04 | 3997.4ms/step | 15.0 samples/s
+Step 30340/50462 | Loss: 2.5596 | LR: 1.76e-04 | 4011.8ms/step | 15.0 samples/s
+Step 30350/50462 | Loss: 2.5726 | LR: 1.76e-04 | 4016.6ms/step | 14.9 samples/s
+Step 30360/50462 | Loss: 2.5712 | LR: 1.76e-04 | 4009.3ms/step | 15.0 samples/s
+Step 30370/50462 | Loss: 2.5384 | LR: 1.76e-04 | 4020.0ms/step | 14.9 samples/s
+Step 30380/50462 | Loss: 2.5522 | LR: 1.76e-04 | 4008.0ms/step | 15.0 samples/s
+Step 30390/50462 | Loss: 2.5477 | LR: 1.76e-04 | 4016.0ms/step | 14.9 samples/s
+Step 30400/50462 | Loss: 2.5332 | LR: 1.75e-04 | 4006.4ms/step | 15.0 samples/s
+Step 30410/50462 | Loss: 2.5274 | LR: 1.75e-04 | 4002.5ms/step | 15.0 samples/s
+Step 30420/50462 | Loss: 2.5765 | LR: 1.75e-04 | 4006.4ms/step | 15.0 samples/s
+Step 30430/50462 | Loss: 2.5598 | LR: 1.75e-04 | 4014.6ms/step | 14.9 samples/s
+Step 30440/50462 | Loss: 2.5764 | LR: 1.75e-04 | 4006.7ms/step | 15.0 samples/s
+Step 30450/50462 | Loss: 2.5843 | LR: 1.75e-04 | 4017.0ms/step | 14.9 samples/s
+Step 30460/50462 | Loss: 2.5639 | LR: 1.75e-04 | 4005.8ms/step | 15.0 samples/s
+Step 30470/50462 | Loss: 2.5510 | LR: 1.74e-04 | 4002.6ms/step | 15.0 samples/s
+Step 30480/50462 | Loss: 2.5552 | LR: 1.74e-04 | 4018.6ms/step | 14.9 samples/s
+Step 30490/50462 | Loss: 2.5640 | LR: 1.74e-04 | 4012.3ms/step | 15.0 samples/s
+Step 30500/50462 | Loss: 2.5819 | LR: 1.74e-04 | 4031.2ms/step | 14.9 samples/s
+Validation at step 30500: Loss = 2.5793, Perplexity = 13.19
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-30500
+Step 30510/50462 | Loss: 2.5373 | LR: 1.74e-04 | 5904.7ms/step | 10.2 samples/s
+Step 30520/50462 | Loss: 2.5769 | LR: 1.74e-04 | 4021.2ms/step | 14.9 samples/s
+Step 30530/50462 | Loss: 2.5310 | LR: 1.73e-04 | 4027.2ms/step | 14.9 samples/s
+Step 30540/50462 | Loss: 2.5460 | LR: 1.73e-04 | 4018.7ms/step | 14.9 samples/s
+Step 30550/50462 | Loss: 2.5802 | LR: 1.73e-04 | 4010.8ms/step | 15.0 samples/s
+Step 30560/50462 | Loss: 2.5552 | LR: 1.73e-04 | 4018.1ms/step | 14.9 samples/s
+Step 30570/50462 | Loss: 2.5901 | LR: 1.73e-04 | 4021.2ms/step | 14.9 samples/s
+Step 30580/50462 | Loss: 2.5939 | LR: 1.73e-04 | 4012.2ms/step | 15.0 samples/s
+Step 30590/50462 | Loss: 2.5396 | LR: 1.73e-04 | 4015.6ms/step | 14.9 samples/s
+Step 30600/50462 | Loss: 2.5539 | LR: 1.72e-04 | 4019.5ms/step | 14.9 samples/s
+Step 30610/50462 | Loss: 2.5610 | LR: 1.72e-04 | 4025.5ms/step | 14.9 samples/s
+Step 30620/50462 | Loss: 2.5506 | LR: 1.72e-04 | 4017.3ms/step | 14.9 samples/s
+Step 30630/50462 | Loss: 2.5696 | LR: 1.72e-04 | 4014.5ms/step | 14.9 samples/s
+Step 30640/50462 | Loss: 2.5991 | LR: 1.72e-04 | 4013.0ms/step | 15.0 samples/s
+Step 30650/50462 | Loss: 2.5531 | LR: 1.72e-04 | 3997.6ms/step | 15.0 samples/s
+Step 30660/50462 | Loss: 2.5453 | LR: 1.72e-04 | 4012.7ms/step | 15.0 samples/s
+Step 30670/50462 | Loss: 2.5743 | LR: 1.71e-04 | 3997.0ms/step | 15.0 samples/s
+Step 30680/50462 | Loss: 2.5364 | LR: 1.71e-04 | 4004.3ms/step | 15.0 samples/s
+Step 30690/50462 | Loss: 2.5624 | LR: 1.71e-04 | 4001.6ms/step | 15.0 samples/s
+Step 30700/50462 | Loss: 2.5896 | LR: 1.71e-04 | 4008.3ms/step | 15.0 samples/s
+Step 30710/50462 | Loss: 2.5467 | LR: 1.71e-04 | 4004.0ms/step | 15.0 samples/s
+Step 30720/50462 | Loss: 2.5763 | LR: 1.71e-04 | 4004.2ms/step | 15.0 samples/s
+Step 30730/50462 | Loss: 2.5569 | LR: 1.70e-04 | 3999.6ms/step | 15.0 samples/s
+Step 30740/50462 | Loss: 2.5467 | LR: 1.70e-04 | 4016.9ms/step | 14.9 samples/s
+Step 30750/50462 | Loss: 2.5480 | LR: 1.70e-04 | 4001.2ms/step | 15.0 samples/s
+Validation at step 30750: Loss = 2.5788, Perplexity = 13.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-30750
+Step 30760/50462 | Loss: 2.5538 | LR: 1.70e-04 | 5899.9ms/step | 10.2 samples/s
+Step 30770/50462 | Loss: 2.5540 | LR: 1.70e-04 | 4001.1ms/step | 15.0 samples/s
+Step 30780/50462 | Loss: 2.5516 | LR: 1.70e-04 | 4011.6ms/step | 15.0 samples/s
+Step 30790/50462 | Loss: 2.5753 | LR: 1.70e-04 | 4005.6ms/step | 15.0 samples/s
+Step 30800/50462 | Loss: 2.5549 | LR: 1.69e-04 | 4004.0ms/step | 15.0 samples/s
+Step 30810/50462 | Loss: 2.5904 | LR: 1.69e-04 | 4010.8ms/step | 15.0 samples/s
+Step 30820/50462 | Loss: 2.5860 | LR: 1.69e-04 | 3992.8ms/step | 15.0 samples/s
+Step 30830/50462 | Loss: 2.5678 | LR: 1.69e-04 | 4005.5ms/step | 15.0 samples/s
+Step 30840/50462 | Loss: 2.5455 | LR: 1.69e-04 | 3996.7ms/step | 15.0 samples/s
+Step 30850/50462 | Loss: 2.5766 | LR: 1.69e-04 | 3995.5ms/step | 15.0 samples/s
+Step 30860/50462 | Loss: 2.5614 | LR: 1.69e-04 | 4006.8ms/step | 15.0 samples/s
+Step 30870/50462 | Loss: 2.5543 | LR: 1.68e-04 | 3988.2ms/step | 15.0 samples/s
+Step 30880/50462 | Loss: 2.5550 | LR: 1.68e-04 | 3999.8ms/step | 15.0 samples/s
+Step 30890/50462 | Loss: 2.5567 | LR: 1.68e-04 | 3996.6ms/step | 15.0 samples/s
+Step 30900/50462 | Loss: 2.5522 | LR: 1.68e-04 | 3998.5ms/step | 15.0 samples/s
+Step 30910/50462 | Loss: 2.5210 | LR: 1.68e-04 | 4025.7ms/step | 14.9 samples/s
+Step 30920/50462 | Loss: 2.5636 | LR: 1.68e-04 | 4008.1ms/step | 15.0 samples/s
+Step 30930/50462 | Loss: 2.5697 | LR: 1.67e-04 | 4009.0ms/step | 15.0 samples/s
+Step 30940/50462 | Loss: 2.5319 | LR: 1.67e-04 | 3999.9ms/step | 15.0 samples/s
+Step 30950/50462 | Loss: 2.5696 | LR: 1.67e-04 | 4001.9ms/step | 15.0 samples/s
+Step 30960/50462 | Loss: 2.5869 | LR: 1.67e-04 | 4001.9ms/step | 15.0 samples/s
+Step 30970/50462 | Loss: 2.5536 | LR: 1.67e-04 | 4010.4ms/step | 15.0 samples/s
+Step 30980/50462 | Loss: 2.5406 | LR: 1.67e-04 | 3999.4ms/step | 15.0 samples/s
+Step 30990/50462 | Loss: 2.5526 | LR: 1.67e-04 | 4005.8ms/step | 15.0 samples/s
+Step 31000/50462 | Loss: 2.5669 | LR: 1.66e-04 | 4003.8ms/step | 15.0 samples/s
+Validation at step 31000: Loss = 2.5774, Perplexity = 13.16
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31000

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume25000.pid

@@ -0,0 +1 @@
+127

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume31000.log

@@ -0,0 +1,207 @@
+===== RESUME FROM CHECKPOINT-31000 ON 4GPU 2026-08-29T13:46:09+00:00 =====
+W0829 13:46:22.374000 150 site-packages/torch/distributed/run.py:851] 
+W0829 13:46:22.374000 150 site-packages/torch/distributed/run.py:851] *****************************************
+W0829 13:46:22.374000 150 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0829 13:46:22.374000 150 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31000
+Checkpoint activations: TrueCheckpoint activations: True
+Checkpoint activations: True
+
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31000...
+Preserving checkpoint LR state for exact resume: current=[0.00016642121244498845], schedule_base=[0.0005]
+Resumed from checkpoint at step 31000 (epoch 0, micro_step 432495)
+Resuming training from step 31000
+Resuming wandb run z1i60278
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260829_135046-z1i60278
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed72 6gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/z1i60278
+Starting training...
+Validation at step 31000: Loss = 2.5770, Perplexity = 13.16

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume31000_after_seed71.log

@@ -0,0 +1,327 @@
+W0831 07:05:14.718000 2440 site-packages/torch/distributed/run.py:851] 
+W0831 07:05:14.718000 2440 site-packages/torch/distributed/run.py:851] *****************************************
+W0831 07:05:14.718000 2440 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0831 07:05:14.718000 2440 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "resume_micro_step_override": null,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31000
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31000...
+Preserving checkpoint LR state for exact resume: current=[0.00016642121244498845], schedule_base=[0.0005]
+Resumed from checkpoint at step 31000 (epoch 0, micro_step 432495)
+Resuming training from step 31000
+Starting training...
+Validation at step 31000: Loss = 2.5770, Perplexity = 13.16
+Switching attention mode for step 31001: sparse_stochastic (configured sparse attention)
+Step 31010/50462 | Loss: 2.6035 | LR: 1.66e-04 | 61105.0ms/step | 1.0 samples/s
+Step 31020/50462 | Loss: 2.5678 | LR: 1.66e-04 | 3998.9ms/step | 15.0 samples/s
+Step 31030/50462 | Loss: 2.5906 | LR: 1.66e-04 | 4027.2ms/step | 14.9 samples/s
+Step 31040/50462 | Loss: 2.5997 | LR: 1.66e-04 | 4013.8ms/step | 14.9 samples/s
+Step 31050/50462 | Loss: 2.5656 | LR: 1.66e-04 | 4000.2ms/step | 15.0 samples/s
+Step 31060/50462 | Loss: 2.5531 | LR: 1.66e-04 | 4012.6ms/step | 15.0 samples/s
+Step 31070/50462 | Loss: 2.5388 | LR: 1.65e-04 | 4008.1ms/step | 15.0 samples/s
+Step 31080/50462 | Loss: 2.5638 | LR: 1.65e-04 | 4005.1ms/step | 15.0 samples/s
+Step 31090/50462 | Loss: 2.5432 | LR: 1.65e-04 | 4009.3ms/step | 15.0 samples/s
+Step 31100/50462 | Loss: 2.5370 | LR: 1.65e-04 | 4012.0ms/step | 15.0 samples/s
+Step 31110/50462 | Loss: 2.5470 | LR: 1.65e-04 | 4013.9ms/step | 14.9 samples/s
+Step 31120/50462 | Loss: 2.5730 | LR: 1.65e-04 | 4008.8ms/step | 15.0 samples/s
+Step 31130/50462 | Loss: 2.5857 | LR: 1.64e-04 | 4017.9ms/step | 14.9 samples/s
+Step 31140/50462 | Loss: 2.5592 | LR: 1.64e-04 | 4019.7ms/step | 14.9 samples/s
+Step 31150/50462 | Loss: 2.5864 | LR: 1.64e-04 | 4018.8ms/step | 14.9 samples/s
+Step 31160/50462 | Loss: 2.5647 | LR: 1.64e-04 | 4028.8ms/step | 14.9 samples/s
+Step 31170/50462 | Loss: 2.5515 | LR: 1.64e-04 | 4021.0ms/step | 14.9 samples/s
+Step 31180/50462 | Loss: 2.5742 | LR: 1.64e-04 | 4026.6ms/step | 14.9 samples/s
+Step 31190/50462 | Loss: 2.5710 | LR: 1.64e-04 | 4032.2ms/step | 14.9 samples/s
+Step 31200/50462 | Loss: 2.5559 | LR: 1.63e-04 | 4013.4ms/step | 14.9 samples/s
+Step 31210/50462 | Loss: 2.5369 | LR: 1.63e-04 | 4014.2ms/step | 14.9 samples/s
+Step 31220/50462 | Loss: 2.5508 | LR: 1.63e-04 | 4010.2ms/step | 15.0 samples/s
+Step 31230/50462 | Loss: 2.5793 | LR: 1.63e-04 | 4014.7ms/step | 14.9 samples/s
+Step 31240/50462 | Loss: 2.5661 | LR: 1.63e-04 | 4004.4ms/step | 15.0 samples/s
+Step 31250/50462 | Loss: 2.5734 | LR: 1.63e-04 | 4009.4ms/step | 15.0 samples/s
+Validation at step 31250: Loss = 2.5758, Perplexity = 13.14
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31250
+Step 31260/50462 | Loss: 2.5742 | LR: 1.63e-04 | 6047.9ms/step | 9.9 samples/s
+Step 31270/50462 | Loss: 2.5381 | LR: 1.62e-04 | 4028.7ms/step | 14.9 samples/s
+Step 31280/50462 | Loss: 2.5159 | LR: 1.62e-04 | 4020.6ms/step | 14.9 samples/s
+Step 31290/50462 | Loss: 2.5595 | LR: 1.62e-04 | 4014.5ms/step | 14.9 samples/s
+Step 31300/50462 | Loss: 2.5547 | LR: 1.62e-04 | 4012.3ms/step | 15.0 samples/s
+Step 31310/50462 | Loss: 2.5649 | LR: 1.62e-04 | 4023.7ms/step | 14.9 samples/s
+Step 31320/50462 | Loss: 2.5358 | LR: 1.62e-04 | 4027.6ms/step | 14.9 samples/s
+Step 31330/50462 | Loss: 2.5974 | LR: 1.62e-04 | 4008.5ms/step | 15.0 samples/s
+Step 31340/50462 | Loss: 2.5361 | LR: 1.61e-04 | 4018.4ms/step | 14.9 samples/s
+Step 31350/50462 | Loss: 2.5485 | LR: 1.61e-04 | 4010.0ms/step | 15.0 samples/s
+Step 31360/50462 | Loss: 2.5678 | LR: 1.61e-04 | 4006.5ms/step | 15.0 samples/s
+Step 31370/50462 | Loss: 2.5331 | LR: 1.61e-04 | 4019.5ms/step | 14.9 samples/s
+Step 31380/50462 | Loss: 2.5382 | LR: 1.61e-04 | 4012.7ms/step | 15.0 samples/s
+Step 31390/50462 | Loss: 2.5651 | LR: 1.61e-04 | 4006.8ms/step | 15.0 samples/s
+Step 31400/50462 | Loss: 2.5798 | LR: 1.60e-04 | 4016.8ms/step | 14.9 samples/s
+Step 31410/50462 | Loss: 2.5503 | LR: 1.60e-04 | 4006.3ms/step | 15.0 samples/s
+Step 31420/50462 | Loss: 2.5301 | LR: 1.60e-04 | 4013.4ms/step | 14.9 samples/s
+Step 31430/50462 | Loss: 2.5511 | LR: 1.60e-04 | 4012.1ms/step | 15.0 samples/s
+Step 31440/50462 | Loss: 2.5432 | LR: 1.60e-04 | 4016.0ms/step | 14.9 samples/s
+Step 31450/50462 | Loss: 2.5751 | LR: 1.60e-04 | 4014.3ms/step | 14.9 samples/s
+Step 31460/50462 | Loss: 2.5741 | LR: 1.60e-04 | 4017.0ms/step | 14.9 samples/s
+Step 31470/50462 | Loss: 2.5360 | LR: 1.59e-04 | 4010.5ms/step | 15.0 samples/s
+Step 31480/50462 | Loss: 2.5519 | LR: 1.59e-04 | 4005.4ms/step | 15.0 samples/s
+Step 31490/50462 | Loss: 2.5458 | LR: 1.59e-04 | 4015.9ms/step | 14.9 samples/s
+Step 31500/50462 | Loss: 2.5668 | LR: 1.59e-04 | 4037.4ms/step | 14.9 samples/s
+Validation at step 31500: Loss = 2.5750, Perplexity = 13.13
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31500
+Step 31510/50462 | Loss: 2.5371 | LR: 1.59e-04 | 6012.4ms/step | 10.0 samples/s
+Step 31520/50462 | Loss: 2.5491 | LR: 1.59e-04 | 4025.4ms/step | 14.9 samples/s
+Step 31530/50462 | Loss: 2.5706 | LR: 1.59e-04 | 4034.8ms/step | 14.9 samples/s
+Step 31540/50462 | Loss: 2.5593 | LR: 1.58e-04 | 4012.6ms/step | 15.0 samples/s
+Step 31550/50462 | Loss: 2.5576 | LR: 1.58e-04 | 4016.0ms/step | 14.9 samples/s
+Step 31560/50462 | Loss: 2.5764 | LR: 1.58e-04 | 4009.0ms/step | 15.0 samples/s
+Step 31570/50462 | Loss: 2.5556 | LR: 1.58e-04 | 4015.6ms/step | 14.9 samples/s
+Step 31580/50462 | Loss: 2.5347 | LR: 1.58e-04 | 4009.6ms/step | 15.0 samples/s
+Step 31590/50462 | Loss: 2.5329 | LR: 1.58e-04 | 4012.7ms/step | 15.0 samples/s
+Step 31600/50462 | Loss: 2.5405 | LR: 1.58e-04 | 4013.3ms/step | 15.0 samples/s
+Step 31610/50462 | Loss: 2.5719 | LR: 1.57e-04 | 4016.2ms/step | 14.9 samples/s
+Step 31620/50462 | Loss: 2.5914 | LR: 1.57e-04 | 4018.1ms/step | 14.9 samples/s
+Step 31630/50462 | Loss: 2.5642 | LR: 1.57e-04 | 4009.4ms/step | 15.0 samples/s
+Step 31640/50462 | Loss: 2.5463 | LR: 1.57e-04 | 4003.8ms/step | 15.0 samples/s
+Step 31650/50462 | Loss: 2.5856 | LR: 1.57e-04 | 4015.0ms/step | 14.9 samples/s
+Step 31660/50462 | Loss: 2.5614 | LR: 1.57e-04 | 4015.3ms/step | 14.9 samples/s
+Step 31670/50462 | Loss: 2.5793 | LR: 1.57e-04 | 4012.9ms/step | 15.0 samples/s
+Step 31680/50462 | Loss: 2.5611 | LR: 1.56e-04 | 4015.5ms/step | 14.9 samples/s
+Step 31690/50462 | Loss: 2.5311 | LR: 1.56e-04 | 4011.4ms/step | 15.0 samples/s
+Step 31700/50462 | Loss: 2.5421 | LR: 1.56e-04 | 4013.0ms/step | 15.0 samples/s
+Step 31710/50462 | Loss: 2.5401 | LR: 1.56e-04 | 4025.4ms/step | 14.9 samples/s
+Step 31720/50462 | Loss: 2.5728 | LR: 1.56e-04 | 4017.1ms/step | 14.9 samples/s
+Step 31730/50462 | Loss: 2.5336 | LR: 1.56e-04 | 4011.4ms/step | 15.0 samples/s
+Step 31740/50462 | Loss: 2.5610 | LR: 1.55e-04 | 4007.0ms/step | 15.0 samples/s
+Step 31750/50462 | Loss: 2.5631 | LR: 1.55e-04 | 3998.1ms/step | 15.0 samples/s
+Validation at step 31750: Loss = 2.5734, Perplexity = 13.11
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31750
+Step 31760/50462 | Loss: 2.5546 | LR: 1.55e-04 | 5975.4ms/step | 10.0 samples/s
+Step 31770/50462 | Loss: 2.5649 | LR: 1.55e-04 | 4008.2ms/step | 15.0 samples/s
+Step 31780/50462 | Loss: 2.5759 | LR: 1.55e-04 | 4006.8ms/step | 15.0 samples/s
+Step 31790/50462 | Loss: 2.5568 | LR: 1.55e-04 | 4009.1ms/step | 15.0 samples/s
+Step 31800/50462 | Loss: 2.5649 | LR: 1.55e-04 | 4017.8ms/step | 14.9 samples/s
+Step 31810/50462 | Loss: 2.5793 | LR: 1.54e-04 | 4001.1ms/step | 15.0 samples/s
+Step 31820/50462 | Loss: 2.5531 | LR: 1.54e-04 | 4024.7ms/step | 14.9 samples/s
+Step 31830/50462 | Loss: 2.5500 | LR: 1.54e-04 | 4008.1ms/step | 15.0 samples/s
+Step 31840/50462 | Loss: 2.5620 | LR: 1.54e-04 | 4003.4ms/step | 15.0 samples/s
+Step 31850/50462 | Loss: 2.5554 | LR: 1.54e-04 | 4010.8ms/step | 15.0 samples/s
+Step 31860/50462 | Loss: 2.5489 | LR: 1.54e-04 | 4009.1ms/step | 15.0 samples/s
+Step 31870/50462 | Loss: 2.5928 | LR: 1.54e-04 | 4024.6ms/step | 14.9 samples/s
+Step 31880/50462 | Loss: 2.5517 | LR: 1.53e-04 | 4006.6ms/step | 15.0 samples/s
+Step 31890/50462 | Loss: 2.5431 | LR: 1.53e-04 | 4012.5ms/step | 15.0 samples/s
+Step 31900/50462 | Loss: 2.5612 | LR: 1.53e-04 | 4014.8ms/step | 14.9 samples/s
+Step 31910/50462 | Loss: 2.5636 | LR: 1.53e-04 | 4016.9ms/step | 14.9 samples/s
+Step 31920/50462 | Loss: 2.5749 | LR: 1.53e-04 | 4012.9ms/step | 15.0 samples/s
+Step 31930/50462 | Loss: 2.5876 | LR: 1.53e-04 | 4012.4ms/step | 15.0 samples/s
+Step 31940/50462 | Loss: 2.5796 | LR: 1.53e-04 | 4010.2ms/step | 15.0 samples/s
+Step 31950/50462 | Loss: 2.5504 | LR: 1.52e-04 | 4018.4ms/step | 14.9 samples/s
+Step 31960/50462 | Loss: 2.5420 | LR: 1.52e-04 | 4010.1ms/step | 15.0 samples/s
+Step 31970/50462 | Loss: 2.5584 | LR: 1.52e-04 | 4004.4ms/step | 15.0 samples/s
+Step 31980/50462 | Loss: 2.5581 | LR: 1.52e-04 | 4006.1ms/step | 15.0 samples/s
+Step 31990/50462 | Loss: 2.5690 | LR: 1.52e-04 | 4006.0ms/step | 15.0 samples/s
+Step 32000/50462 | Loss: 2.5451 | LR: 1.52e-04 | 4006.9ms/step | 15.0 samples/s
+Validation at step 32000: Loss = 2.5734, Perplexity = 13.11
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-32000
+Step 32010/50462 | Loss: 2.5614 | LR: 1.52e-04 | 5965.8ms/step | 10.1 samples/s
+Step 32020/50462 | Loss: 2.6000 | LR: 1.51e-04 | 4002.9ms/step | 15.0 samples/s
+Step 32030/50462 | Loss: 2.5384 | LR: 1.51e-04 | 4014.5ms/step | 14.9 samples/s
+Step 32040/50462 | Loss: 2.5731 | LR: 1.51e-04 | 3999.8ms/step | 15.0 samples/s
+Step 32050/50462 | Loss: 2.5513 | LR: 1.51e-04 | 3997.0ms/step | 15.0 samples/s
+Step 32060/50462 | Loss: 2.5432 | LR: 1.51e-04 | 4016.3ms/step | 14.9 samples/s
+Step 32070/50462 | Loss: 2.5161 | LR: 1.51e-04 | 4001.9ms/step | 15.0 samples/s
+Step 32080/50462 | Loss: 2.5781 | LR: 1.51e-04 | 4005.9ms/step | 15.0 samples/s
+Step 32090/50462 | Loss: 2.5826 | LR: 1.50e-04 | 4000.6ms/step | 15.0 samples/s
+Step 32100/50462 | Loss: 2.5483 | LR: 1.50e-04 | 4013.5ms/step | 14.9 samples/s
+Step 32110/50462 | Loss: 2.5902 | LR: 1.50e-04 | 4001.9ms/step | 15.0 samples/s
+Step 32120/50462 | Loss: 2.5687 | LR: 1.50e-04 | 4000.9ms/step | 15.0 samples/s
+Step 32130/50462 | Loss: 2.5527 | LR: 1.50e-04 | 4004.0ms/step | 15.0 samples/s
+Step 32140/50462 | Loss: 2.5433 | LR: 1.50e-04 | 4001.6ms/step | 15.0 samples/s
+Step 32150/50462 | Loss: 2.5858 | LR: 1.50e-04 | 3997.2ms/step | 15.0 samples/s
+Step 32160/50462 | Loss: 2.5717 | LR: 1.49e-04 | 4007.4ms/step | 15.0 samples/s
+Step 32170/50462 | Loss: 2.5025 | LR: 1.49e-04 | 4008.8ms/step | 15.0 samples/s
+Step 32180/50462 | Loss: 2.5461 | LR: 1.49e-04 | 4008.1ms/step | 15.0 samples/s
+Step 32190/50462 | Loss: 2.5264 | LR: 1.49e-04 | 4019.2ms/step | 14.9 samples/s
+Step 32200/50462 | Loss: 2.5578 | LR: 1.49e-04 | 4010.1ms/step | 15.0 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume32000.log

@@ -0,0 +1,227 @@
+W0831 14:13:32.052000 149 site-packages/torch/distributed/run.py:851] 
+W0831 14:13:32.052000 149 site-packages/torch/distributed/run.py:851] *****************************************
+W0831 14:13:32.052000 149 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0831 14:13:32.052000 149 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-32000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "resume_micro_step_override": null,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-32000
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-32000...
+Preserving checkpoint LR state for exact resume: current=[0.0001517080469136949], schedule_base=[0.0005]
+Resumed from checkpoint at step 32000 (epoch 0, micro_step 447495)
+Resuming training from step 32000
+Starting training...
+Validation at step 32000: Loss = 2.5732, Perplexity = 13.11
+Switching attention mode for step 32001: sparse_stochastic (configured sparse attention)
+Step 32010/50462 | Loss: 2.5616 | LR: 1.52e-04 | 73617.5ms/step | 0.8 samples/s
+Step 32020/50462 | Loss: 2.5990 | LR: 1.51e-04 | 3996.7ms/step | 15.0 samples/s
+Step 32030/50462 | Loss: 2.5394 | LR: 1.51e-04 | 3988.2ms/step | 15.0 samples/s
+Step 32040/50462 | Loss: 2.5724 | LR: 1.51e-04 | 3983.6ms/step | 15.1 samples/s
+Step 32050/50462 | Loss: 2.5523 | LR: 1.51e-04 | 4007.7ms/step | 15.0 samples/s
+Step 32060/50462 | Loss: 2.5437 | LR: 1.51e-04 | 3979.2ms/step | 15.1 samples/s
+Step 32070/50462 | Loss: 2.5160 | LR: 1.51e-04 | 3994.4ms/step | 15.0 samples/s
+Step 32080/50462 | Loss: 2.5789 | LR: 1.51e-04 | 3983.0ms/step | 15.1 samples/s
+Step 32090/50462 | Loss: 2.5823 | LR: 1.50e-04 | 3994.7ms/step | 15.0 samples/s
+Step 32100/50462 | Loss: 2.5478 | LR: 1.50e-04 | 3999.4ms/step | 15.0 samples/s
+Step 32110/50462 | Loss: 2.5888 | LR: 1.50e-04 | 3979.3ms/step | 15.1 samples/s
+Step 32120/50462 | Loss: 2.5682 | LR: 1.50e-04 | 4000.8ms/step | 15.0 samples/s
+Step 32130/50462 | Loss: 2.5514 | LR: 1.50e-04 | 3989.1ms/step | 15.0 samples/s
+Step 32140/50462 | Loss: 2.5443 | LR: 1.50e-04 | 4016.1ms/step | 14.9 samples/s
+Step 32150/50462 | Loss: 2.5860 | LR: 1.50e-04 | 3992.1ms/step | 15.0 samples/s
+Step 32160/50462 | Loss: 2.5720 | LR: 1.49e-04 | 4003.8ms/step | 15.0 samples/s
+Step 32170/50462 | Loss: 2.5021 | LR: 1.49e-04 | 3996.2ms/step | 15.0 samples/s
+Step 32180/50462 | Loss: 2.5466 | LR: 1.49e-04 | 4005.5ms/step | 15.0 samples/s
+Step 32190/50462 | Loss: 2.5260 | LR: 1.49e-04 | 3999.7ms/step | 15.0 samples/s
+Step 32200/50462 | Loss: 2.5563 | LR: 1.49e-04 | 4008.2ms/step | 15.0 samples/s
+Step 32210/50462 | Loss: 2.5656 | LR: 1.49e-04 | 4007.9ms/step | 15.0 samples/s
+Step 32220/50462 | Loss: 2.5311 | LR: 1.49e-04 | 3993.7ms/step | 15.0 samples/s
+Step 32230/50462 | Loss: 2.5459 | LR: 1.48e-04 | 4001.9ms/step | 15.0 samples/s
+Step 32240/50462 | Loss: 2.5277 | LR: 1.48e-04 | 4002.6ms/step | 15.0 samples/s
+Step 32250/50462 | Loss: 2.5761 | LR: 1.48e-04 | 3996.2ms/step | 15.0 samples/s
+Validation at step 32250: Loss = 2.5724, Perplexity = 13.10
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-32250
+Step 32260/50462 | Loss: 2.5388 | LR: 1.48e-04 | 5980.9ms/step | 10.0 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume32250.log

@@ -0,0 +1,518 @@
+W0831 16:14:58.309000 155 site-packages/torch/distributed/run.py:851] 
+W0831 16:14:58.309000 155 site-packages/torch/distributed/run.py:851] *****************************************
+W0831 16:14:58.309000 155 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0831 16:14:58.309000 155 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-32250",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "resume_micro_step_override": null,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Checkpoint activations: True
+Checkpoint activations: True
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-32250
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-32250...
+Preserving checkpoint LR state for exact resume: current=[0.00014808880918230983], schedule_base=[0.0005]
+Resumed from checkpoint at step 32250 (epoch 0, micro_step 451245)
+Resuming training from step 32250
+Starting training...
+Validation at step 32250: Loss = 2.5719, Perplexity = 13.09
+Switching attention mode for step 32251: sparse_stochastic (configured sparse attention)
+Step 32260/50462 | Loss: 2.5393 | LR: 1.48e-04 | 70657.0ms/step | 0.8 samples/s
+Step 32270/50462 | Loss: 2.5336 | LR: 1.48e-04 | 4005.3ms/step | 15.0 samples/s
+Step 32280/50462 | Loss: 2.5597 | LR: 1.48e-04 | 4017.5ms/step | 14.9 samples/s
+Step 32290/50462 | Loss: 2.5244 | LR: 1.48e-04 | 4017.8ms/step | 14.9 samples/s
+Step 32300/50462 | Loss: 2.5595 | LR: 1.47e-04 | 4030.5ms/step | 14.9 samples/s
+Step 32310/50462 | Loss: 2.5472 | LR: 1.47e-04 | 4036.5ms/step | 14.9 samples/s
+Step 32320/50462 | Loss: 2.5284 | LR: 1.47e-04 | 4040.7ms/step | 14.8 samples/s
+Step 32330/50462 | Loss: 2.5404 | LR: 1.47e-04 | 4018.5ms/step | 14.9 samples/s
+Step 32340/50462 | Loss: 2.5410 | LR: 1.47e-04 | 4014.2ms/step | 14.9 samples/s
+Step 32350/50462 | Loss: 2.5653 | LR: 1.47e-04 | 4017.1ms/step | 14.9 samples/s
+Step 32360/50462 | Loss: 2.5436 | LR: 1.47e-04 | 4023.7ms/step | 14.9 samples/s
+Step 32370/50462 | Loss: 2.5626 | LR: 1.46e-04 | 4031.9ms/step | 14.9 samples/s
+Step 32380/50462 | Loss: 2.5720 | LR: 1.46e-04 | 4050.3ms/step | 14.8 samples/s
+Step 32390/50462 | Loss: 2.5375 | LR: 1.46e-04 | 4028.4ms/step | 14.9 samples/s
+Step 32400/50462 | Loss: 2.5387 | LR: 1.46e-04 | 4015.7ms/step | 14.9 samples/s
+Step 32410/50462 | Loss: 2.5567 | LR: 1.46e-04 | 4025.5ms/step | 14.9 samples/s
+Step 32420/50462 | Loss: 2.5529 | LR: 1.46e-04 | 4012.1ms/step | 15.0 samples/s
+Step 32430/50462 | Loss: 2.5645 | LR: 1.45e-04 | 4025.7ms/step | 14.9 samples/s
+Step 32440/50462 | Loss: 2.5573 | LR: 1.45e-04 | 4026.0ms/step | 14.9 samples/s
+Step 32450/50462 | Loss: 2.5932 | LR: 1.45e-04 | 4024.5ms/step | 14.9 samples/s
+Step 32460/50462 | Loss: 2.5207 | LR: 1.45e-04 | 4022.7ms/step | 14.9 samples/s
+Step 32470/50462 | Loss: 2.4971 | LR: 1.45e-04 | 4037.1ms/step | 14.9 samples/s
+Step 32480/50462 | Loss: 2.5816 | LR: 1.45e-04 | 4024.1ms/step | 14.9 samples/s
+Step 32490/50462 | Loss: 2.5110 | LR: 1.45e-04 | 4033.8ms/step | 14.9 samples/s
+Step 32500/50462 | Loss: 2.5299 | LR: 1.44e-04 | 4033.7ms/step | 14.9 samples/s
+Validation at step 32500: Loss = 2.5716, Perplexity = 13.09
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-32500
+Step 32510/50462 | Loss: 2.5554 | LR: 1.44e-04 | 6029.7ms/step | 10.0 samples/s
+Step 32520/50462 | Loss: 2.5664 | LR: 1.44e-04 | 4024.2ms/step | 14.9 samples/s
+Step 32530/50462 | Loss: 2.5498 | LR: 1.44e-04 | 4017.8ms/step | 14.9 samples/s
+Step 32540/50462 | Loss: 2.5474 | LR: 1.44e-04 | 4047.4ms/step | 14.8 samples/s
+Step 32550/50462 | Loss: 2.6017 | LR: 1.44e-04 | 4056.8ms/step | 14.8 samples/s
+Step 32560/50462 | Loss: 2.5128 | LR: 1.44e-04 | 4029.9ms/step | 14.9 samples/s
+Step 32570/50462 | Loss: 2.5797 | LR: 1.43e-04 | 4022.7ms/step | 14.9 samples/s
+Step 32580/50462 | Loss: 2.5503 | LR: 1.43e-04 | 4042.8ms/step | 14.8 samples/s
+Step 32590/50462 | Loss: 2.5618 | LR: 1.43e-04 | 4054.3ms/step | 14.8 samples/s
+Step 32600/50462 | Loss: 2.5547 | LR: 1.43e-04 | 4035.7ms/step | 14.9 samples/s
+Step 32610/50462 | Loss: 2.5401 | LR: 1.43e-04 | 4047.1ms/step | 14.8 samples/s
+Step 32620/50462 | Loss: 2.5547 | LR: 1.43e-04 | 4033.7ms/step | 14.9 samples/s
+Step 32630/50462 | Loss: 2.5653 | LR: 1.43e-04 | 4036.5ms/step | 14.9 samples/s
+Step 32640/50462 | Loss: 2.5404 | LR: 1.42e-04 | 4036.2ms/step | 14.9 samples/s
+Step 32650/50462 | Loss: 2.5301 | LR: 1.42e-04 | 4034.3ms/step | 14.9 samples/s
+Step 32660/50462 | Loss: 2.5530 | LR: 1.42e-04 | 4029.9ms/step | 14.9 samples/s
+Step 32670/50462 | Loss: 2.5494 | LR: 1.42e-04 | 4031.5ms/step | 14.9 samples/s
+Step 32680/50462 | Loss: 2.5423 | LR: 1.42e-04 | 4038.5ms/step | 14.9 samples/s
+Step 32690/50462 | Loss: 2.5548 | LR: 1.42e-04 | 4017.1ms/step | 14.9 samples/s
+Step 32700/50462 | Loss: 2.5422 | LR: 1.42e-04 | 4023.6ms/step | 14.9 samples/s
+Step 32710/50462 | Loss: 2.5326 | LR: 1.41e-04 | 4030.1ms/step | 14.9 samples/s
+Step 32720/50462 | Loss: 2.5688 | LR: 1.41e-04 | 4030.9ms/step | 14.9 samples/s
+Step 32730/50462 | Loss: 2.5298 | LR: 1.41e-04 | 4026.6ms/step | 14.9 samples/s
+Step 32740/50462 | Loss: 2.5386 | LR: 1.41e-04 | 4028.6ms/step | 14.9 samples/s
+Step 32750/50462 | Loss: 2.5613 | LR: 1.41e-04 | 4044.4ms/step | 14.8 samples/s
+Validation at step 32750: Loss = 2.5690, Perplexity = 13.05
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-32750
+Step 32760/50462 | Loss: 2.5669 | LR: 1.41e-04 | 6056.6ms/step | 9.9 samples/s
+Step 32770/50462 | Loss: 2.5578 | LR: 1.41e-04 | 4028.5ms/step | 14.9 samples/s
+Step 32780/50462 | Loss: 2.5888 | LR: 1.41e-04 | 4036.6ms/step | 14.9 samples/s
+Step 32790/50462 | Loss: 2.5094 | LR: 1.40e-04 | 4039.8ms/step | 14.9 samples/s
+Step 32800/50462 | Loss: 2.5632 | LR: 1.40e-04 | 4037.5ms/step | 14.9 samples/s
+Step 32810/50462 | Loss: 2.4969 | LR: 1.40e-04 | 4023.8ms/step | 14.9 samples/s
+Step 32820/50462 | Loss: 2.5710 | LR: 1.40e-04 | 4022.2ms/step | 14.9 samples/s
+Step 32830/50462 | Loss: 2.5960 | LR: 1.40e-04 | 4025.0ms/step | 14.9 samples/s
+Step 32840/50462 | Loss: 2.5663 | LR: 1.40e-04 | 4031.2ms/step | 14.9 samples/s
+Step 32850/50462 | Loss: 2.5385 | LR: 1.40e-04 | 4037.4ms/step | 14.9 samples/s
+Step 32860/50462 | Loss: 2.5480 | LR: 1.39e-04 | 4025.3ms/step | 14.9 samples/s
+Step 32870/50462 | Loss: 2.5197 | LR: 1.39e-04 | 4028.7ms/step | 14.9 samples/s
+Step 32880/50462 | Loss: 2.5651 | LR: 1.39e-04 | 4031.4ms/step | 14.9 samples/s
+Step 32890/50462 | Loss: 2.5368 | LR: 1.39e-04 | 4028.9ms/step | 14.9 samples/s
+Step 32900/50462 | Loss: 2.5569 | LR: 1.39e-04 | 4033.3ms/step | 14.9 samples/s
+Step 32910/50462 | Loss: 2.5813 | LR: 1.39e-04 | 4034.3ms/step | 14.9 samples/s
+Step 32920/50462 | Loss: 2.5390 | LR: 1.39e-04 | 4026.0ms/step | 14.9 samples/s
+Step 32930/50462 | Loss: 2.5284 | LR: 1.38e-04 | 4023.1ms/step | 14.9 samples/s
+Step 32940/50462 | Loss: 2.5529 | LR: 1.38e-04 | 4036.8ms/step | 14.9 samples/s
+Step 32950/50462 | Loss: 2.5461 | LR: 1.38e-04 | 4042.5ms/step | 14.8 samples/s
+Step 32960/50462 | Loss: 2.5488 | LR: 1.38e-04 | 4029.9ms/step | 14.9 samples/s
+Step 32970/50462 | Loss: 2.5310 | LR: 1.38e-04 | 4019.5ms/step | 14.9 samples/s
+Step 32980/50462 | Loss: 2.5355 | LR: 1.38e-04 | 4032.8ms/step | 14.9 samples/s
+Step 32990/50462 | Loss: 2.5606 | LR: 1.38e-04 | 4033.4ms/step | 14.9 samples/s
+Step 33000/50462 | Loss: 2.5550 | LR: 1.37e-04 | 4036.5ms/step | 14.9 samples/s
+Validation at step 33000: Loss = 2.5683, Perplexity = 13.04
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-33000
+Step 33010/50462 | Loss: 2.5822 | LR: 1.37e-04 | 6043.0ms/step | 9.9 samples/s
+Step 33020/50462 | Loss: 2.5363 | LR: 1.37e-04 | 4046.9ms/step | 14.8 samples/s
+Step 33030/50462 | Loss: 2.5543 | LR: 1.37e-04 | 4045.1ms/step | 14.8 samples/s
+Step 33040/50462 | Loss: 2.5750 | LR: 1.37e-04 | 4034.1ms/step | 14.9 samples/s
+Step 33050/50462 | Loss: 2.5661 | LR: 1.37e-04 | 4033.8ms/step | 14.9 samples/s
+Step 33060/50462 | Loss: 2.5531 | LR: 1.37e-04 | 4030.1ms/step | 14.9 samples/s
+Step 33070/50462 | Loss: 2.5316 | LR: 1.36e-04 | 4037.7ms/step | 14.9 samples/s
+Step 33080/50462 | Loss: 2.5713 | LR: 1.36e-04 | 4039.5ms/step | 14.9 samples/s
+Step 33090/50462 | Loss: 2.5458 | LR: 1.36e-04 | 4030.8ms/step | 14.9 samples/s
+Step 33100/50462 | Loss: 2.5204 | LR: 1.36e-04 | 4034.9ms/step | 14.9 samples/s
+Step 33110/50462 | Loss: 2.5450 | LR: 1.36e-04 | 4058.3ms/step | 14.8 samples/s
+Step 33120/50462 | Loss: 2.5518 | LR: 1.36e-04 | 4044.6ms/step | 14.8 samples/s
+Step 33130/50462 | Loss: 2.5608 | LR: 1.36e-04 | 4035.4ms/step | 14.9 samples/s
+Step 33140/50462 | Loss: 2.5776 | LR: 1.35e-04 | 4042.8ms/step | 14.8 samples/s
+Step 33150/50462 | Loss: 2.5539 | LR: 1.35e-04 | 4044.7ms/step | 14.8 samples/s
+Step 33160/50462 | Loss: 2.5265 | LR: 1.35e-04 | 4044.4ms/step | 14.8 samples/s
+Step 33170/50462 | Loss: 2.5785 | LR: 1.35e-04 | 4044.4ms/step | 14.8 samples/s
+Step 33180/50462 | Loss: 2.5602 | LR: 1.35e-04 | 4044.3ms/step | 14.8 samples/s
+Step 33190/50462 | Loss: 2.5636 | LR: 1.35e-04 | 4029.6ms/step | 14.9 samples/s
+Step 33200/50462 | Loss: 2.5621 | LR: 1.35e-04 | 4051.5ms/step | 14.8 samples/s
+Step 33210/50462 | Loss: 2.5224 | LR: 1.34e-04 | 4028.7ms/step | 14.9 samples/s
+Step 33220/50462 | Loss: 2.5631 | LR: 1.34e-04 | 4031.1ms/step | 14.9 samples/s
+Step 33230/50462 | Loss: 2.5819 | LR: 1.34e-04 | 4039.7ms/step | 14.9 samples/s
+Step 33240/50462 | Loss: 2.5797 | LR: 1.34e-04 | 4039.4ms/step | 14.9 samples/s
+Step 33250/50462 | Loss: 2.5487 | LR: 1.34e-04 | 4021.5ms/step | 14.9 samples/s
+Validation at step 33250: Loss = 2.5677, Perplexity = 13.04
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-33250
+Step 33260/50462 | Loss: 2.5272 | LR: 1.34e-04 | 6013.5ms/step | 10.0 samples/s
+Step 33270/50462 | Loss: 2.5385 | LR: 1.34e-04 | 4011.7ms/step | 15.0 samples/s
+Step 33280/50462 | Loss: 2.5483 | LR: 1.33e-04 | 4015.9ms/step | 14.9 samples/s
+Step 33290/50462 | Loss: 2.5677 | LR: 1.33e-04 | 4018.1ms/step | 14.9 samples/s
+Step 33300/50462 | Loss: 2.5717 | LR: 1.33e-04 | 4006.5ms/step | 15.0 samples/s
+Step 33310/50462 | Loss: 2.5301 | LR: 1.33e-04 | 4022.5ms/step | 14.9 samples/s
+Step 33320/50462 | Loss: 2.5491 | LR: 1.33e-04 | 4009.7ms/step | 15.0 samples/s
+Step 33330/50462 | Loss: 2.5506 | LR: 1.33e-04 | 3997.1ms/step | 15.0 samples/s
+Step 33340/50462 | Loss: 2.5387 | LR: 1.33e-04 | 3998.1ms/step | 15.0 samples/s
+Step 33350/50462 | Loss: 2.5662 | LR: 1.32e-04 | 4006.4ms/step | 15.0 samples/s
+Step 33360/50462 | Loss: 2.5738 | LR: 1.32e-04 | 3995.2ms/step | 15.0 samples/s
+Step 33370/50462 | Loss: 2.5278 | LR: 1.32e-04 | 4004.1ms/step | 15.0 samples/s
+Step 33380/50462 | Loss: 2.5576 | LR: 1.32e-04 | 4010.1ms/step | 15.0 samples/s
+Step 33390/50462 | Loss: 2.5440 | LR: 1.32e-04 | 4016.9ms/step | 14.9 samples/s
+Step 33400/50462 | Loss: 2.5546 | LR: 1.32e-04 | 4005.7ms/step | 15.0 samples/s
+Step 33410/50462 | Loss: 2.5446 | LR: 1.32e-04 | 4010.3ms/step | 15.0 samples/s
+Step 33420/50462 | Loss: 2.5797 | LR: 1.32e-04 | 4018.9ms/step | 14.9 samples/s
+Step 33430/50462 | Loss: 2.5752 | LR: 1.31e-04 | 4017.7ms/step | 14.9 samples/s
+Step 33440/50462 | Loss: 2.5479 | LR: 1.31e-04 | 4019.3ms/step | 14.9 samples/s
+Step 33450/50462 | Loss: 2.5599 | LR: 1.31e-04 | 4024.0ms/step | 14.9 samples/s
+Step 33460/50462 | Loss: 2.5556 | LR: 1.31e-04 | 4022.6ms/step | 14.9 samples/s
+Step 33470/50462 | Loss: 2.5325 | LR: 1.31e-04 | 4021.0ms/step | 14.9 samples/s
+Step 33480/50462 | Loss: 2.5340 | LR: 1.31e-04 | 4024.0ms/step | 14.9 samples/s
+Step 33490/50462 | Loss: 2.5430 | LR: 1.31e-04 | 4013.9ms/step | 14.9 samples/s
+Step 33500/50462 | Loss: 2.5463 | LR: 1.30e-04 | 4033.3ms/step | 14.9 samples/s
+Validation at step 33500: Loss = 2.5662, Perplexity = 13.02
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-33500
+Step 33510/50462 | Loss: 2.5245 | LR: 1.30e-04 | 6014.1ms/step | 10.0 samples/s
+Step 33520/50462 | Loss: 2.5437 | LR: 1.30e-04 | 4020.7ms/step | 14.9 samples/s
+Step 33530/50462 | Loss: 2.5687 | LR: 1.30e-04 | 4019.4ms/step | 14.9 samples/s
+Step 33540/50462 | Loss: 2.5436 | LR: 1.30e-04 | 4019.6ms/step | 14.9 samples/s
+Step 33550/50462 | Loss: 2.5439 | LR: 1.30e-04 | 4016.9ms/step | 14.9 samples/s
+Step 33560/50462 | Loss: 2.5359 | LR: 1.30e-04 | 4021.6ms/step | 14.9 samples/s
+Step 33570/50462 | Loss: 2.5198 | LR: 1.29e-04 | 4026.3ms/step | 14.9 samples/s
+Step 33580/50462 | Loss: 2.5612 | LR: 1.29e-04 | 4011.9ms/step | 15.0 samples/s
+Step 33590/50462 | Loss: 2.5676 | LR: 1.29e-04 | 4023.6ms/step | 14.9 samples/s
+Step 33600/50462 | Loss: 2.5529 | LR: 1.29e-04 | 4008.8ms/step | 15.0 samples/s
+Step 33610/50462 | Loss: 2.5195 | LR: 1.29e-04 | 4017.3ms/step | 14.9 samples/s
+Step 33620/50462 | Loss: 2.5270 | LR: 1.29e-04 | 4009.4ms/step | 15.0 samples/s
+Step 33630/50462 | Loss: 2.5279 | LR: 1.29e-04 | 4005.3ms/step | 15.0 samples/s
+Step 33640/50462 | Loss: 2.5419 | LR: 1.28e-04 | 4004.5ms/step | 15.0 samples/s
+Step 33650/50462 | Loss: 2.5309 | LR: 1.28e-04 | 3999.0ms/step | 15.0 samples/s
+Step 33660/50462 | Loss: 2.5662 | LR: 1.28e-04 | 4003.9ms/step | 15.0 samples/s
+Step 33670/50462 | Loss: 2.5999 | LR: 1.28e-04 | 4721.8ms/step | 12.7 samples/s
+Step 33680/50462 | Loss: 2.5321 | LR: 1.28e-04 | 4020.3ms/step | 14.9 samples/s
+Step 33690/50462 | Loss: 2.5649 | LR: 1.28e-04 | 4030.4ms/step | 14.9 samples/s
+Step 33700/50462 | Loss: 2.5559 | LR: 1.28e-04 | 4022.6ms/step | 14.9 samples/s
+Step 33710/50462 | Loss: 2.5668 | LR: 1.27e-04 | 4032.5ms/step | 14.9 samples/s
+Step 33720/50462 | Loss: 2.5690 | LR: 1.27e-04 | 5592.8ms/step | 10.7 samples/s
+Step 33730/50462 | Loss: 2.5538 | LR: 1.27e-04 | 4028.7ms/step | 14.9 samples/s
+Step 33740/50462 | Loss: 2.5556 | LR: 1.27e-04 | 4006.5ms/step | 15.0 samples/s
+Step 33750/50462 | Loss: 2.5465 | LR: 1.27e-04 | 4011.2ms/step | 15.0 samples/s
+Validation at step 33750: Loss = 2.5661, Perplexity = 13.01
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-33750
+Step 33760/50462 | Loss: 2.5552 | LR: 1.27e-04 | 5960.7ms/step | 10.1 samples/s
+Step 33770/50462 | Loss: 2.5481 | LR: 1.27e-04 | 4009.3ms/step | 15.0 samples/s
+Step 33780/50462 | Loss: 2.5128 | LR: 1.27e-04 | 4013.4ms/step | 14.9 samples/s
+Step 33790/50462 | Loss: 2.5485 | LR: 1.26e-04 | 3997.5ms/step | 15.0 samples/s
+Step 33800/50462 | Loss: 2.5430 | LR: 1.26e-04 | 4007.9ms/step | 15.0 samples/s
+Step 33810/50462 | Loss: 2.5412 | LR: 1.26e-04 | 4010.0ms/step | 15.0 samples/s
+Step 33820/50462 | Loss: 2.5120 | LR: 1.26e-04 | 4671.8ms/step | 12.8 samples/s
+Step 33830/50462 | Loss: 2.5494 | LR: 1.26e-04 | 4595.8ms/step | 13.1 samples/s
+Step 33840/50462 | Loss: 2.5395 | LR: 1.26e-04 | 4011.6ms/step | 15.0 samples/s
+Step 33850/50462 | Loss: 2.5193 | LR: 1.26e-04 | 4015.4ms/step | 14.9 samples/s
+Step 33860/50462 | Loss: 2.5599 | LR: 1.25e-04 | 4016.7ms/step | 14.9 samples/s
+Step 33870/50462 | Loss: 2.5590 | LR: 1.25e-04 | 4019.4ms/step | 14.9 samples/s
+Step 33880/50462 | Loss: 2.5251 | LR: 1.25e-04 | 4009.0ms/step | 15.0 samples/s
+Step 33890/50462 | Loss: 2.5433 | LR: 1.25e-04 | 4010.5ms/step | 15.0 samples/s
+Step 33900/50462 | Loss: 2.5708 | LR: 1.25e-04 | 4021.4ms/step | 14.9 samples/s
+Step 33910/50462 | Loss: 2.5377 | LR: 1.25e-04 | 4020.0ms/step | 14.9 samples/s
+Step 33920/50462 | Loss: 2.5628 | LR: 1.25e-04 | 4011.1ms/step | 15.0 samples/s
+Step 33930/50462 | Loss: 2.5429 | LR: 1.24e-04 | 4024.4ms/step | 14.9 samples/s
+Step 33940/50462 | Loss: 2.5658 | LR: 1.24e-04 | 4014.1ms/step | 14.9 samples/s
+Step 33950/50462 | Loss: 2.5435 | LR: 1.24e-04 | 4006.8ms/step | 15.0 samples/s
+Step 33960/50462 | Loss: 2.5523 | LR: 1.24e-04 | 4013.8ms/step | 14.9 samples/s
+Step 33970/50462 | Loss: 2.5416 | LR: 1.24e-04 | 4010.4ms/step | 15.0 samples/s
+Step 33980/50462 | Loss: 2.5483 | LR: 1.24e-04 | 4006.6ms/step | 15.0 samples/s
+Step 33990/50462 | Loss: 2.5312 | LR: 1.24e-04 | 4016.8ms/step | 14.9 samples/s
+Step 34000/50462 | Loss: 2.5636 | LR: 1.24e-04 | 4016.8ms/step | 14.9 samples/s
+Validation at step 34000: Loss = 2.5669, Perplexity = 13.02
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-34000
+Step 34010/50462 | Loss: 2.5350 | LR: 1.23e-04 | 5977.3ms/step | 10.0 samples/s
+Step 34020/50462 | Loss: 2.5549 | LR: 1.23e-04 | 4002.1ms/step | 15.0 samples/s
+Step 34030/50462 | Loss: 2.5295 | LR: 1.23e-04 | 4007.0ms/step | 15.0 samples/s
+Step 34040/50462 | Loss: 2.5661 | LR: 1.23e-04 | 4026.4ms/step | 14.9 samples/s
+Step 34050/50462 | Loss: 2.5493 | LR: 1.23e-04 | 4015.9ms/step | 14.9 samples/s
+Step 34060/50462 | Loss: 2.5507 | LR: 1.23e-04 | 4007.3ms/step | 15.0 samples/s
+Step 34070/50462 | Loss: 2.5823 | LR: 1.23e-04 | 4023.4ms/step | 14.9 samples/s
+Step 34080/50462 | Loss: 2.5277 | LR: 1.22e-04 | 4014.3ms/step | 14.9 samples/s
+Step 34090/50462 | Loss: 2.5371 | LR: 1.22e-04 | 4013.5ms/step | 14.9 samples/s
+Step 34100/50462 | Loss: 2.5651 | LR: 1.22e-04 | 4015.6ms/step | 14.9 samples/s
+Step 34110/50462 | Loss: 2.5252 | LR: 1.22e-04 | 4014.1ms/step | 14.9 samples/s
+Step 34120/50462 | Loss: 2.5540 | LR: 1.22e-04 | 4011.7ms/step | 15.0 samples/s
+Step 34130/50462 | Loss: 2.5435 | LR: 1.22e-04 | 3988.0ms/step | 15.0 samples/s
+Step 34140/50462 | Loss: 2.5183 | LR: 1.22e-04 | 4001.4ms/step | 15.0 samples/s
+Step 34150/50462 | Loss: 2.5648 | LR: 1.21e-04 | 3998.5ms/step | 15.0 samples/s
+Step 34160/50462 | Loss: 2.5213 | LR: 1.21e-04 | 4002.7ms/step | 15.0 samples/s
+Step 34170/50462 | Loss: 2.5628 | LR: 1.21e-04 | 3999.2ms/step | 15.0 samples/s
+Step 34180/50462 | Loss: 2.5346 | LR: 1.21e-04 | 4010.8ms/step | 15.0 samples/s
+Step 34190/50462 | Loss: 2.5199 | LR: 1.21e-04 | 4000.1ms/step | 15.0 samples/s
+Step 34200/50462 | Loss: 2.5358 | LR: 1.21e-04 | 3982.9ms/step | 15.1 samples/s
+Step 34210/50462 | Loss: 2.5399 | LR: 1.21e-04 | 3996.9ms/step | 15.0 samples/s
+Step 34220/50462 | Loss: 2.5516 | LR: 1.21e-04 | 3997.6ms/step | 15.0 samples/s
+Step 34230/50462 | Loss: 2.5260 | LR: 1.20e-04 | 4005.8ms/step | 15.0 samples/s
+Step 34240/50462 | Loss: 2.5297 | LR: 1.20e-04 | 4003.6ms/step | 15.0 samples/s
+Step 34250/50462 | Loss: 2.5481 | LR: 1.20e-04 | 3997.0ms/step | 15.0 samples/s
+Validation at step 34250: Loss = 2.5643, Perplexity = 12.99
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-34250
+Step 34260/50462 | Loss: 2.5565 | LR: 1.20e-04 | 5963.1ms/step | 10.1 samples/s
+Step 34270/50462 | Loss: 2.5499 | LR: 1.20e-04 | 3982.4ms/step | 15.1 samples/s
+Step 34280/50462 | Loss: 2.5395 | LR: 1.20e-04 | 3996.9ms/step | 15.0 samples/s
+Step 34290/50462 | Loss: 2.5406 | LR: 1.20e-04 | 3993.1ms/step | 15.0 samples/s
+Step 34300/50462 | Loss: 2.5442 | LR: 1.19e-04 | 3996.8ms/step | 15.0 samples/s
+Step 34310/50462 | Loss: 2.5467 | LR: 1.19e-04 | 4008.9ms/step | 15.0 samples/s
+Step 34320/50462 | Loss: 2.5254 | LR: 1.19e-04 | 3999.9ms/step | 15.0 samples/s
+Step 34330/50462 | Loss: 2.5447 | LR: 1.19e-04 | 3991.3ms/step | 15.0 samples/s
+Step 34340/50462 | Loss: 2.5364 | LR: 1.19e-04 | 3990.2ms/step | 15.0 samples/s
+Step 34350/50462 | Loss: 2.5445 | LR: 1.19e-04 | 3993.5ms/step | 15.0 samples/s
+Step 34360/50462 | Loss: 2.5558 | LR: 1.19e-04 | 4007.0ms/step | 15.0 samples/s
+Step 34370/50462 | Loss: 2.5330 | LR: 1.19e-04 | 4005.8ms/step | 15.0 samples/s
+Step 34380/50462 | Loss: 2.5040 | LR: 1.18e-04 | 3987.6ms/step | 15.0 samples/s
+Step 34390/50462 | Loss: 2.5213 | LR: 1.18e-04 | 3997.3ms/step | 15.0 samples/s
+Step 34400/50462 | Loss: 2.5390 | LR: 1.18e-04 | 4008.5ms/step | 15.0 samples/s
+Step 34410/50462 | Loss: 2.5187 | LR: 1.18e-04 | 4001.6ms/step | 15.0 samples/s
+Step 34420/50462 | Loss: 2.5608 | LR: 1.18e-04 | 3997.6ms/step | 15.0 samples/s
+Step 34430/50462 | Loss: 2.5532 | LR: 1.18e-04 | 4001.0ms/step | 15.0 samples/s
+Step 34440/50462 | Loss: 2.5689 | LR: 1.18e-04 | 4005.0ms/step | 15.0 samples/s
+Step 34450/50462 | Loss: 2.5383 | LR: 1.17e-04 | 3998.9ms/step | 15.0 samples/s
+Step 34460/50462 | Loss: 2.5491 | LR: 1.17e-04 | 3994.5ms/step | 15.0 samples/s
+Step 34470/50462 | Loss: 2.5456 | LR: 1.17e-04 | 4008.4ms/step | 15.0 samples/s
+Step 34480/50462 | Loss: 2.5165 | LR: 1.17e-04 | 3997.0ms/step | 15.0 samples/s
+Step 34490/50462 | Loss: 2.5507 | LR: 1.17e-04 | 4005.4ms/step | 15.0 samples/s
+Step 34500/50462 | Loss: 2.5321 | LR: 1.17e-04 | 4010.7ms/step | 15.0 samples/s
+Validation at step 34500: Loss = 2.5640, Perplexity = 12.99
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-34500
+Step 34510/50462 | Loss: 2.4987 | LR: 1.17e-04 | 5934.9ms/step | 10.1 samples/s
+Step 34520/50462 | Loss: 2.5402 | LR: 1.16e-04 | 4005.6ms/step | 15.0 samples/s
+Step 34530/50462 | Loss: 2.5655 | LR: 1.16e-04 | 3996.7ms/step | 15.0 samples/s
+Step 34540/50462 | Loss: 2.5058 | LR: 1.16e-04 | 3995.8ms/step | 15.0 samples/s
+Step 34550/50462 | Loss: 2.5404 | LR: 1.16e-04 | 4000.6ms/step | 15.0 samples/s
+Step 34560/50462 | Loss: 2.5700 | LR: 1.16e-04 | 3995.9ms/step | 15.0 samples/s
+Step 34570/50462 | Loss: 2.5744 | LR: 1.16e-04 | 3982.7ms/step | 15.1 samples/s
+Step 34580/50462 | Loss: 2.5205 | LR: 1.16e-04 | 3998.9ms/step | 15.0 samples/s
+Step 34590/50462 | Loss: 2.5718 | LR: 1.16e-04 | 3996.8ms/step | 15.0 samples/s
+Step 34600/50462 | Loss: 2.5342 | LR: 1.15e-04 | 4004.3ms/step | 15.0 samples/s
+Step 34610/50462 | Loss: 2.5435 | LR: 1.15e-04 | 4005.6ms/step | 15.0 samples/s
+Step 34620/50462 | Loss: 2.5535 | LR: 1.15e-04 | 4003.5ms/step | 15.0 samples/s
+Step 34630/50462 | Loss: 2.5442 | LR: 1.15e-04 | 3996.7ms/step | 15.0 samples/s
+Step 34640/50462 | Loss: 2.5738 | LR: 1.15e-04 | 4003.2ms/step | 15.0 samples/s
+Step 34650/50462 | Loss: 2.5378 | LR: 1.15e-04 | 3999.1ms/step | 15.0 samples/s
+Step 34660/50462 | Loss: 2.5398 | LR: 1.15e-04 | 4008.3ms/step | 15.0 samples/s
+Step 34670/50462 | Loss: 2.5361 | LR: 1.15e-04 | 4001.5ms/step | 15.0 samples/s
+Step 34680/50462 | Loss: 2.5530 | LR: 1.14e-04 | 3999.7ms/step | 15.0 samples/s
+Step 34690/50462 | Loss: 2.5447 | LR: 1.14e-04 | 4001.7ms/step | 15.0 samples/s
+Step 34700/50462 | Loss: 2.5522 | LR: 1.14e-04 | 4000.2ms/step | 15.0 samples/s
+Step 34710/50462 | Loss: 2.5242 | LR: 1.14e-04 | 3997.0ms/step | 15.0 samples/s
+Step 34720/50462 | Loss: 2.5051 | LR: 1.14e-04 | 4002.1ms/step | 15.0 samples/s
+Step 34730/50462 | Loss: 2.5451 | LR: 1.14e-04 | 3999.1ms/step | 15.0 samples/s
+Step 34740/50462 | Loss: 2.5381 | LR: 1.14e-04 | 3996.3ms/step | 15.0 samples/s
+Step 34750/50462 | Loss: 2.5455 | LR: 1.13e-04 | 3996.8ms/step | 15.0 samples/s
+Validation at step 34750: Loss = 2.5628, Perplexity = 12.97
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-34750
+Step 34760/50462 | Loss: 2.5593 | LR: 1.13e-04 | 5935.6ms/step | 10.1 samples/s
+Step 34770/50462 | Loss: 2.5321 | LR: 1.13e-04 | 3987.3ms/step | 15.0 samples/s
+Step 34780/50462 | Loss: 2.5469 | LR: 1.13e-04 | 3988.7ms/step | 15.0 samples/s
+Step 34790/50462 | Loss: 2.5611 | LR: 1.13e-04 | 4003.6ms/step | 15.0 samples/s
+Step 34800/50462 | Loss: 2.5615 | LR: 1.13e-04 | 3995.1ms/step | 15.0 samples/s
+Step 34810/50462 | Loss: 2.5491 | LR: 1.13e-04 | 4006.8ms/step | 15.0 samples/s
+Step 34820/50462 | Loss: 2.5479 | LR: 1.13e-04 | 4002.0ms/step | 15.0 samples/s
+Step 34830/50462 | Loss: 2.5275 | LR: 1.12e-04 | 4003.2ms/step | 15.0 samples/s
+Step 34840/50462 | Loss: 2.5619 | LR: 1.12e-04 | 3998.1ms/step | 15.0 samples/s
+Step 34850/50462 | Loss: 2.5452 | LR: 1.12e-04 | 3995.5ms/step | 15.0 samples/s
+Step 34860/50462 | Loss: 2.5611 | LR: 1.12e-04 | 3989.6ms/step | 15.0 samples/s
+Step 34870/50462 | Loss: 2.5358 | LR: 1.12e-04 | 3998.3ms/step | 15.0 samples/s
+Step 34880/50462 | Loss: 2.5056 | LR: 1.12e-04 | 3995.1ms/step | 15.0 samples/s
+Step 34890/50462 | Loss: 2.5163 | LR: 1.12e-04 | 3994.4ms/step | 15.0 samples/s
+Step 34900/50462 | Loss: 2.5610 | LR: 1.11e-04 | 3996.2ms/step | 15.0 samples/s
+Step 34910/50462 | Loss: 2.5751 | LR: 1.11e-04 | 4001.7ms/step | 15.0 samples/s
+Step 34920/50462 | Loss: 2.5213 | LR: 1.11e-04 | 4006.8ms/step | 15.0 samples/s
+Step 34930/50462 | Loss: 2.5446 | LR: 1.11e-04 | 4011.0ms/step | 15.0 samples/s
+Step 34940/50462 | Loss: 2.5022 | LR: 1.11e-04 | 4004.1ms/step | 15.0 samples/s
+Step 34950/50462 | Loss: 2.5642 | LR: 1.11e-04 | 4000.9ms/step | 15.0 samples/s
+Step 34960/50462 | Loss: 2.4960 | LR: 1.11e-04 | 3989.8ms/step | 15.0 samples/s
+Step 34970/50462 | Loss: 2.5584 | LR: 1.11e-04 | 4003.1ms/step | 15.0 samples/s
+Step 34980/50462 | Loss: 2.5278 | LR: 1.10e-04 | 4002.8ms/step | 15.0 samples/s
+Step 34990/50462 | Loss: 2.5234 | LR: 1.10e-04 | 3992.9ms/step | 15.0 samples/s
+Step 35000/50462 | Loss: 2.5243 | LR: 1.10e-04 | 4001.1ms/step | 15.0 samples/s
+Validation at step 35000: Loss = 2.5624, Perplexity = 12.97
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-35000
+Step 35010/50462 | Loss: 2.5180 | LR: 1.10e-04 | 5912.6ms/step | 10.1 samples/s
+Step 35020/50462 | Loss: 2.5465 | LR: 1.10e-04 | 3991.9ms/step | 15.0 samples/s
+Step 35030/50462 | Loss: 2.5656 | LR: 1.10e-04 | 3991.6ms/step | 15.0 samples/s
+Step 35040/50462 | Loss: 2.5369 | LR: 1.10e-04 | 3999.2ms/step | 15.0 samples/s
+Step 35050/50462 | Loss: 2.5768 | LR: 1.09e-04 | 4003.5ms/step | 15.0 samples/s
+Step 35060/50462 | Loss: 2.5564 | LR: 1.09e-04 | 4002.3ms/step | 15.0 samples/s
+Step 35070/50462 | Loss: 2.5342 | LR: 1.09e-04 | 4005.8ms/step | 15.0 samples/s
+Step 35080/50462 | Loss: 2.5061 | LR: 1.09e-04 | 4000.3ms/step | 15.0 samples/s
+Step 35090/50462 | Loss: 2.5801 | LR: 1.09e-04 | 4005.1ms/step | 15.0 samples/s
+Step 35100/50462 | Loss: 2.5262 | LR: 1.09e-04 | 4013.0ms/step | 15.0 samples/s
+Step 35110/50462 | Loss: 2.5617 | LR: 1.09e-04 | 3995.2ms/step | 15.0 samples/s
+Step 35120/50462 | Loss: 2.5388 | LR: 1.09e-04 | 4000.4ms/step | 15.0 samples/s
+Step 35130/50462 | Loss: 2.5407 | LR: 1.08e-04 | 3997.0ms/step | 15.0 samples/s
+Step 35140/50462 | Loss: 2.5578 | LR: 1.08e-04 | 4004.7ms/step | 15.0 samples/s
+Step 35150/50462 | Loss: 2.5413 | LR: 1.08e-04 | 4000.5ms/step | 15.0 samples/s
+Step 35160/50462 | Loss: 2.5038 | LR: 1.08e-04 | 4012.2ms/step | 15.0 samples/s
+Step 35170/50462 | Loss: 2.5282 | LR: 1.08e-04 | 4014.0ms/step | 14.9 samples/s
+Step 35180/50462 | Loss: 2.5409 | LR: 1.08e-04 | 4014.4ms/step | 14.9 samples/s
+Step 35190/50462 | Loss: 2.5412 | LR: 1.08e-04 | 4001.6ms/step | 15.0 samples/s
+Step 35200/50462 | Loss: 2.5201 | LR: 1.08e-04 | 4003.7ms/step | 15.0 samples/s
+Step 35210/50462 | Loss: 2.5461 | LR: 1.07e-04 | 3999.8ms/step | 15.0 samples/s
+Step 35220/50462 | Loss: 2.5475 | LR: 1.07e-04 | 4000.2ms/step | 15.0 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume35000.log

@@ -0,0 +1,207 @@
+W0901 13:25:19.064000 178 site-packages/torch/distributed/run.py:851] 
+W0901 13:25:19.064000 178 site-packages/torch/distributed/run.py:851] *****************************************
+W0901 13:25:19.064000 178 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0901 13:25:19.064000 178 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-35000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "resume_micro_step_override": null,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 4gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-35000
+Checkpoint activations: TrueCheckpoint activations: TrueCheckpoint activations: True
+
+
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-35000...
+Preserving checkpoint LR state for exact resume: current=[0.00011014996888775508], schedule_base=[0.0005]
+Resumed from checkpoint at step 35000 (epoch 0, micro_step 492495)
+Resuming training from step 35000
+Starting training...
+Validation at step 35000: Loss = 2.5626, Perplexity = 12.97
+Switching attention mode for step 35001: sparse_stochastic (configured sparse attention)
+Step 35010/50462 | Loss: 2.5181 | LR: 1.10e-04 | 61635.7ms/step | 1.0 samples/s
+Step 35020/50462 | Loss: 2.5458 | LR: 1.10e-04 | 3986.3ms/step | 15.1 samples/s
+Step 35030/50462 | Loss: 2.5658 | LR: 1.10e-04 | 4008.8ms/step | 15.0 samples/s
+Step 35040/50462 | Loss: 2.5370 | LR: 1.10e-04 | 4012.3ms/step | 15.0 samples/s
+Step 35050/50462 | Loss: 2.5761 | LR: 1.09e-04 | 4009.9ms/step | 15.0 samples/s
+Step 35060/50462 | Loss: 2.5562 | LR: 1.09e-04 | 4010.3ms/step | 15.0 samples/s
+Step 35070/50462 | Loss: 2.5348 | LR: 1.09e-04 | 4006.2ms/step | 15.0 samples/s
+Step 35080/50462 | Loss: 2.5057 | LR: 1.09e-04 | 4001.1ms/step | 15.0 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume7750.log

@@ -0,0 +1,296 @@
+
+===== RESUME FROM CHECKPOINT-7750 2026-08-27T14:55:05Z =====
+W0827 14:55:14.471000 146 site-packages/torch/distributed/run.py:851] 
+W0827 14:55:14.471000 146 site-packages/torch/distributed/run.py:851] *****************************************
+W0827 14:55:14.471000 146 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0827 14:55:14.471000 146 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-7750",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-7750
+Checkpoint activations: TrueCheckpoint activations: TrueCheckpoint activations: TrueCheckpoint activations: True
+
+
+
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-7750...
+Preserving checkpoint LR state for exact resume: current=[0.0004759348015865587], schedule_base=[0.0005]
+Resumed from checkpoint at step 7750 (epoch 0, micro_step 83745)
+Resuming training from step 7750
+Resuming wandb run z1i60278
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_145841-z1i60278
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed72 6gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/z1i60278
+Starting training...
+Validation at step 7750: Loss = 2.8691, Perplexity = 17.62
+wandb: WARNING Tried to log to step 7750 that is less than the current step 7831. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 7751: sparse_stochastic (configured sparse attention)
+Step 7760/50462 | Loss: 2.7914 | LR: 4.76e-04 | 25899.6ms/step | 2.3 samples/s
+wandb: WARNING Tried to log to step 7760 that is less than the current step 7831. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 7770/50462 | Loss: 2.7507 | LR: 4.76e-04 | 4004.4ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 7770 that is less than the current step 7831. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 7780/50462 | Loss: 2.7673 | LR: 4.76e-04 | 3995.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 7780 that is less than the current step 7831. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 7790/50462 | Loss: 2.7822 | LR: 4.76e-04 | 4004.3ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 7790 that is less than the current step 7831. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 7800/50462 | Loss: 2.7468 | LR: 4.76e-04 | 4005.4ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 7800 that is less than the current step 7831. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 7810/50462 | Loss: 2.7738 | LR: 4.76e-04 | 3994.8ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 7810 that is less than the current step 7831. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 7820/50462 | Loss: 2.7233 | LR: 4.75e-04 | 4002.3ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 7820 that is less than the current step 7831. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 7830/50462 | Loss: 2.7590 | LR: 4.75e-04 | 4003.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 7830 that is less than the current step 7831. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 7840/50462 | Loss: 2.7556 | LR: 4.75e-04 | 4002.6ms/step | 15.0 samples/s
+Step 7850/50462 | Loss: 2.7362 | LR: 4.75e-04 | 3989.7ms/step | 15.0 samples/s
+Step 7860/50462 | Loss: 2.7583 | LR: 4.75e-04 | 3994.7ms/step | 15.0 samples/s
+Step 7870/50462 | Loss: 2.7743 | LR: 4.75e-04 | 3997.9ms/step | 15.0 samples/s
+Step 7880/50462 | Loss: 2.7498 | LR: 4.75e-04 | 4005.2ms/step | 15.0 samples/s
+Step 7890/50462 | Loss: 2.7816 | LR: 4.75e-04 | 4010.2ms/step | 15.0 samples/s
+Step 7900/50462 | Loss: 2.7888 | LR: 4.75e-04 | 4005.6ms/step | 15.0 samples/s
+Step 7910/50462 | Loss: 2.7650 | LR: 4.75e-04 | 4002.7ms/step | 15.0 samples/s
+Step 7920/50462 | Loss: 2.7621 | LR: 4.75e-04 | 3991.1ms/step | 15.0 samples/s
+Step 7930/50462 | Loss: 2.7807 | LR: 4.75e-04 | 4007.9ms/step | 15.0 samples/s
+Step 7940/50462 | Loss: 2.7984 | LR: 4.75e-04 | 4011.4ms/step | 15.0 samples/s
+Step 7950/50462 | Loss: 2.7641 | LR: 4.75e-04 | 4000.8ms/step | 15.0 samples/s
+Step 7960/50462 | Loss: 2.7369 | LR: 4.74e-04 | 3997.0ms/step | 15.0 samples/s
+Step 7970/50462 | Loss: 2.7641 | LR: 4.74e-04 | 3995.6ms/step | 15.0 samples/s
+Step 7980/50462 | Loss: 2.7728 | LR: 4.74e-04 | 4003.6ms/step | 15.0 samples/s
+Step 7990/50462 | Loss: 2.7265 | LR: 4.74e-04 | 4007.8ms/step | 15.0 samples/s
+Step 8000/50462 | Loss: 2.7443 | LR: 4.74e-04 | 4001.8ms/step | 15.0 samples/s
+Validation at step 8000: Loss = 2.8638, Perplexity = 17.53
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8000
+Step 8010/50462 | Loss: 2.7515 | LR: 4.74e-04 | 6012.3ms/step | 10.0 samples/s
+Step 8020/50462 | Loss: 2.7622 | LR: 4.74e-04 | 4003.6ms/step | 15.0 samples/s
+Step 8030/50462 | Loss: 2.8073 | LR: 4.74e-04 | 4003.0ms/step | 15.0 samples/s
+Step 8040/50462 | Loss: 2.7614 | LR: 4.74e-04 | 4003.4ms/step | 15.0 samples/s
+Step 8050/50462 | Loss: 2.7447 | LR: 4.74e-04 | 4010.3ms/step | 15.0 samples/s
+Step 8060/50462 | Loss: 2.7401 | LR: 4.74e-04 | 4008.6ms/step | 15.0 samples/s
+Step 8070/50462 | Loss: 2.7527 | LR: 4.74e-04 | 4005.1ms/step | 15.0 samples/s
+Step 8080/50462 | Loss: 2.7649 | LR: 4.74e-04 | 4014.3ms/step | 14.9 samples/s
+Step 8090/50462 | Loss: 2.7734 | LR: 4.74e-04 | 4008.0ms/step | 15.0 samples/s
+Step 8100/50462 | Loss: 2.7701 | LR: 4.74e-04 | 3995.3ms/step | 15.0 samples/s
+Step 8110/50462 | Loss: 2.7608 | LR: 4.73e-04 | 3993.8ms/step | 15.0 samples/s
+Step 8120/50462 | Loss: 2.7585 | LR: 4.73e-04 | 4007.6ms/step | 15.0 samples/s
+Step 8130/50462 | Loss: 2.7957 | LR: 4.73e-04 | 4004.9ms/step | 15.0 samples/s
+Step 8140/50462 | Loss: 2.7964 | LR: 4.73e-04 | 4012.8ms/step | 15.0 samples/s
+Step 8150/50462 | Loss: 2.7593 | LR: 4.73e-04 | 3998.0ms/step | 15.0 samples/s
+Step 8160/50462 | Loss: 2.7185 | LR: 4.73e-04 | 3998.7ms/step | 15.0 samples/s
+Step 8170/50462 | Loss: 2.7757 | LR: 4.73e-04 | 4021.6ms/step | 14.9 samples/s
+Step 8180/50462 | Loss: 2.7548 | LR: 4.73e-04 | 4012.5ms/step | 15.0 samples/s
+Step 8190/50462 | Loss: 2.7218 | LR: 4.73e-04 | 4010.9ms/step | 15.0 samples/s
+Step 8200/50462 | Loss: 2.7554 | LR: 4.73e-04 | 4005.7ms/step | 15.0 samples/s
+Step 8210/50462 | Loss: 2.7607 | LR: 4.73e-04 | 4002.2ms/step | 15.0 samples/s
+Step 8220/50462 | Loss: 2.7540 | LR: 4.73e-04 | 4012.3ms/step | 15.0 samples/s
+Step 8230/50462 | Loss: 2.7572 | LR: 4.73e-04 | 3990.4ms/step | 15.0 samples/s
+Step 8240/50462 | Loss: 2.7266 | LR: 4.73e-04 | 4010.1ms/step | 15.0 samples/s
+Step 8250/50462 | Loss: 2.7733 | LR: 4.72e-04 | 4004.9ms/step | 15.0 samples/s
+Validation at step 8250: Loss = 2.8603, Perplexity = 17.47
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8250
+Step 8260/50462 | Loss: 2.7394 | LR: 4.72e-04 | 6048.9ms/step | 9.9 samples/s
+Step 8270/50462 | Loss: 2.7803 | LR: 4.72e-04 | 4005.6ms/step | 15.0 samples/s
+Step 8280/50462 | Loss: 2.7308 | LR: 4.72e-04 | 4000.9ms/step | 15.0 samples/s
+Step 8290/50462 | Loss: 2.7676 | LR: 4.72e-04 | 4004.1ms/step | 15.0 samples/s
+Step 8300/50462 | Loss: 2.7822 | LR: 4.72e-04 | 4002.9ms/step | 15.0 samples/s
+Step 8310/50462 | Loss: 2.7256 | LR: 4.72e-04 | 4012.6ms/step | 15.0 samples/s
+Step 8320/50462 | Loss: 2.7814 | LR: 4.72e-04 | 4007.4ms/step | 15.0 samples/s
+Step 8330/50462 | Loss: 2.7609 | LR: 4.72e-04 | 4010.8ms/step | 15.0 samples/s
+Step 8340/50462 | Loss: 2.7771 | LR: 4.72e-04 | 3999.0ms/step | 15.0 samples/s
+Step 8350/50462 | Loss: 2.7753 | LR: 4.72e-04 | 4022.0ms/step | 14.9 samples/s
+Step 8360/50462 | Loss: 2.7315 | LR: 4.72e-04 | 4008.8ms/step | 15.0 samples/s
+Step 8370/50462 | Loss: 2.7979 | LR: 4.72e-04 | 4017.9ms/step | 14.9 samples/s
+Step 8380/50462 | Loss: 2.7634 | LR: 4.71e-04 | 4003.9ms/step | 15.0 samples/s
+Step 8390/50462 | Loss: 2.7608 | LR: 4.71e-04 | 4010.8ms/step | 15.0 samples/s
+Step 8400/50462 | Loss: 2.7217 | LR: 4.71e-04 | 3994.6ms/step | 15.0 samples/s
+Step 8410/50462 | Loss: 2.7481 | LR: 4.71e-04 | 4005.1ms/step | 15.0 samples/s
+Step 8420/50462 | Loss: 2.7151 | LR: 4.71e-04 | 4010.8ms/step | 15.0 samples/s
+Step 8430/50462 | Loss: 2.7745 | LR: 4.71e-04 | 4019.6ms/step | 14.9 samples/s
+Step 8440/50462 | Loss: 2.7241 | LR: 4.71e-04 | 4022.9ms/step | 14.9 samples/s
+Step 8450/50462 | Loss: 2.7473 | LR: 4.71e-04 | 4001.4ms/step | 15.0 samples/s
+Step 8460/50462 | Loss: 2.7661 | LR: 4.71e-04 | 4001.8ms/step | 15.0 samples/s
+Step 8470/50462 | Loss: 2.7409 | LR: 4.71e-04 | 4012.3ms/step | 15.0 samples/s
+Step 8480/50462 | Loss: 2.7592 | LR: 4.71e-04 | 4005.1ms/step | 15.0 samples/s
+Step 8490/50462 | Loss: 2.7538 | LR: 4.71e-04 | 4011.2ms/step | 15.0 samples/s

epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume8250.log

@@ -0,0 +1,302 @@
+
+===== RESUME FROM CHECKPOINT-8250 2026-08-27T16:28:14Z =====
+W0827 16:28:22.105000 144 site-packages/torch/distributed/run.py:851] 
+W0827 16:28:22.105000 144 site-packages/torch/distributed/run.py:851] *****************************************
+W0827 16:28:22.105000 144 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0827 16:28:22.105000 144 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8250",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+
+===== RETRY RESUME FROM CHECKPOINT-8250 2026-08-27T16:36:13Z =====
+W0827 16:36:22.392000 153 site-packages/torch/distributed/run.py:851] 
+W0827 16:36:22.392000 153 site-packages/torch/distributed/run.py:851] *****************************************
+W0827 16:36:22.392000 153 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0827 16:36:22.392000 153 site-packages/torch/distributed/run.py:851] *****************************************
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8250",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 50462,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 250,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 72,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "1.2b mixed 8k seed72 6gpu",
+  "wandb_tags": [
+    "qwen3",
+    "1.2b",
+    "8k",
+    "p2",
+    "mixed",
+    "seed72"
+  ],
+  "hidden_size": 1536,
+  "intermediate_size": 6144,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 4,
+  "num_hidden_layers": null,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_train_position_scale_set": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 35,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+  "sparse_mask_seed": 72,
+  "sparse_mixed_nrb_set": [
+    63,
+    47,
+    31,
+    21,
+    11
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8250
+Checkpoint activations: TrueCheckpoint activations: TrueCheckpoint activations: True
+
+
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1536
+  num_hidden_layers: 28
+  num_attention_heads: 12
+  num_key_value_heads: 4
+  head_dim: 128
+  intermediate_size: 6144
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  rope_train_position_scale_set: None
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 35
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 1,202,352,640
+  Trainable parameters: 1,202,352,640
+  Non-trainable parameters: 0
+  Model size (BF16): 2.40 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 1536]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 1536]           233,373,696               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 1536]           [1, 8192, 1536]           34,606,336                True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 1536]           [1, 8192, 1536]           1,536                     True
+├─Linear: 1-2                                                [1, 8192, 1536]           [1, 8192, 151936]         233,373,696               True
+================================================================================================================================================================
+Total params: 1,435,726,336
+Trainable params: 1,435,726,336
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 1.44
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 26218.59
+Params size (MB): 2871.45
+Estimated Total Size (MB): 29090.11
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462  (capped by max_steps; 1.00 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8250...
+Preserving checkpoint LR state for exact resume: current=[0.00047244094585975404], schedule_base=[0.0005]
+Resumed from checkpoint at step 8250 (epoch 0, micro_step 91245)
+Resuming training from step 8250
+Resuming wandb run z1i60278
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run z1i60278
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_163945-z1i60278
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 1.2b mixed 8k seed72 6gpu
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/z1i60278
+Starting training...
+Validation at step 8250: Loss = 2.8611, Perplexity = 17.48
+wandb: WARNING Tried to log to step 8250 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 8251: sparse_stochastic (configured sparse attention)
+Step 8260/50462 | Loss: 2.7394 | LR: 4.72e-04 | 27071.2ms/step | 2.2 samples/s
+wandb: WARNING Tried to log to step 8260 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8270/50462 | Loss: 2.7790 | LR: 4.72e-04 | 4034.1ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8270 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8280/50462 | Loss: 2.7315 | LR: 4.72e-04 | 4058.6ms/step | 14.8 samples/s
+wandb: WARNING Tried to log to step 8280 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8290/50462 | Loss: 2.7680 | LR: 4.72e-04 | 4040.9ms/step | 14.8 samples/s
+wandb: WARNING Tried to log to step 8290 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8300/50462 | Loss: 2.7817 | LR: 4.72e-04 | 4038.2ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8300 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8310/50462 | Loss: 2.7250 | LR: 4.72e-04 | 4035.8ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8310 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8320/50462 | Loss: 2.7829 | LR: 4.72e-04 | 4041.2ms/step | 14.8 samples/s
+wandb: WARNING Tried to log to step 8320 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 8330/50462 | Loss: 2.7627 | LR: 4.72e-04 | 4033.5ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 8330 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.

epfl_final/train_logs/p3_mixed71_locked_h100.log

@@ -0,0 +1,21 @@
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.
+  warnings.warn(
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/eval/moba_eval.py", line 405, in <module>
+    main()
+    ~~~~^^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/eval/moba_eval.py", line 355, in main
+    exact, token_accuracy = score_ruler(
+                            ~~~~~~~~~~~^
+        model, depth_examples, args.batch_size, device, dtype
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
+    return func(*args, **kwargs)
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/eval/moba_eval.py", line 209, in score_ruler
+    logits = model(input_ids=batch, attention_mask=None)["logits"].float()
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 37.09 GiB. GPU 0 has a total capacity of 79.18 GiB of which 20.53 GiB is free. Including non-PyTorch memory, this process has 58.63 GiB memory in use. Of the allocated memory 57.91 GiB is allocated by PyTorch, and 59.67 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)
+Checkpoint activations: True

epfl_final/train_logs/p3_mixed71_locked_h100_batch1.log

@@ -0,0 +1,16 @@
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.
+  warnings.warn(
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.
+  warnings.warn(
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True
+Checkpoint activations: True

epfl_final/train_logs/p4_a100_safe_queue.log

@@ -0,0 +1,74 @@
+SAFE_JOBS 36
+START 0 365m_dense_seed44_fineweb_val_seed42
+START 1 365m_dense_seed44_pg19_ood_windows
+START 2 365m_dense_seed44_arxiv_ood_windows
+START 3 365m_mixed_seed44_fineweb_val_seed42
+START 4 365m_mixed_seed44_pg19_ood_windows
+TIMEOUT_SKIP 0 365m_dense_seed44_fineweb_val_seed42
+TIMEOUT_SKIP 1 365m_dense_seed44_pg19_ood_windows
+TIMEOUT_SKIP 2 365m_dense_seed44_arxiv_ood_windows
+TIMEOUT_SKIP 3 365m_mixed_seed44_fineweb_val_seed42
+TIMEOUT_SKIP 4 365m_mixed_seed44_pg19_ood_windows
+START 0 365m_mixed_seed44_arxiv_ood_windows
+START 1 365m_dense_seed45_fineweb_val_seed42
+START 2 365m_dense_seed45_pg19_ood_windows
+START 3 365m_dense_seed45_arxiv_ood_windows
+START 4 365m_mixed_seed45_fineweb_val_seed42
+TIMEOUT_SKIP 0 365m_mixed_seed44_arxiv_ood_windows
+TIMEOUT_SKIP 1 365m_dense_seed45_fineweb_val_seed42
+TIMEOUT_SKIP 2 365m_dense_seed45_pg19_ood_windows
+TIMEOUT_SKIP 3 365m_dense_seed45_arxiv_ood_windows
+TIMEOUT_SKIP 4 365m_mixed_seed45_fineweb_val_seed42
+START 0 365m_mixed_seed45_pg19_ood_windows
+START 1 365m_mixed_seed45_arxiv_ood_windows
+START 2 365m_dense_seed46_fineweb_val_seed42
+START 3 365m_dense_seed46_pg19_ood_windows
+START 4 365m_dense_seed46_arxiv_ood_windows
+TIMEOUT_SKIP 0 365m_mixed_seed45_pg19_ood_windows
+TIMEOUT_SKIP 1 365m_mixed_seed45_arxiv_ood_windows
+TIMEOUT_SKIP 2 365m_dense_seed46_fineweb_val_seed42
+TIMEOUT_SKIP 3 365m_dense_seed46_pg19_ood_windows
+TIMEOUT_SKIP 4 365m_dense_seed46_arxiv_ood_windows
+START 0 365m_mixed_seed46_fineweb_val_seed42
+START 1 365m_mixed_seed46_pg19_ood_windows
+START 2 365m_mixed_seed46_arxiv_ood_windows
+START 3 365m_dense_seed47_fineweb_val_seed42
+START 4 365m_dense_seed47_pg19_ood_windows
+TIMEOUT_SKIP 0 365m_mixed_seed46_fineweb_val_seed42
+TIMEOUT_SKIP 1 365m_mixed_seed46_pg19_ood_windows
+TIMEOUT_SKIP 2 365m_mixed_seed46_arxiv_ood_windows
+TIMEOUT_SKIP 3 365m_dense_seed47_fineweb_val_seed42
+TIMEOUT_SKIP 4 365m_dense_seed47_pg19_ood_windows
+START 0 365m_dense_seed47_arxiv_ood_windows
+START 1 365m_mixed_seed47_fineweb_val_seed42
+START 2 365m_mixed_seed47_pg19_ood_windows
+START 3 365m_mixed_seed47_arxiv_ood_windows
+START 4 1p2b_dense_seed70_16384_fineweb_val_seed42
+TIMEOUT_SKIP 0 365m_dense_seed47_arxiv_ood_windows
+TIMEOUT_SKIP 1 365m_mixed_seed47_fineweb_val_seed42
+TIMEOUT_SKIP 2 365m_mixed_seed47_pg19_ood_windows
+TIMEOUT_SKIP 3 365m_mixed_seed47_arxiv_ood_windows
+TIMEOUT_SKIP 4 1p2b_dense_seed70_16384_fineweb_val_seed42
+START 0 1p2b_dense_seed70_16384_pg19_ood_windows
+START 1 1p2b_dense_seed70_16384_arxiv_ood_windows
+START 2 1p2b_mixed_seed70_16384_fineweb_val_seed42
+START 3 1p2b_mixed_seed70_16384_pg19_ood_windows
+START 4 1p2b_mixed_seed70_16384_arxiv_ood_windows
+TIMEOUT_SKIP 0 1p2b_dense_seed70_16384_pg19_ood_windows
+TIMEOUT_SKIP 1 1p2b_dense_seed70_16384_arxiv_ood_windows
+TIMEOUT_SKIP 2 1p2b_mixed_seed70_16384_fineweb_val_seed42
+TIMEOUT_SKIP 3 1p2b_mixed_seed70_16384_pg19_ood_windows
+TIMEOUT_SKIP 4 1p2b_mixed_seed70_16384_arxiv_ood_windows
+START 0 1p2b_dense_seed71_16384_fineweb_val_seed42
+START 1 1p2b_dense_seed71_16384_pg19_ood_windows
+START 2 1p2b_dense_seed71_16384_arxiv_ood_windows
+START 3 1p2b_mixed_seed71_16384_fineweb_val_seed42
+START 4 1p2b_mixed_seed71_16384_pg19_ood_windows
+TIMEOUT_SKIP 0 1p2b_dense_seed71_16384_fineweb_val_seed42
+TIMEOUT_SKIP 1 1p2b_dense_seed71_16384_pg19_ood_windows
+TIMEOUT_SKIP 2 1p2b_dense_seed71_16384_arxiv_ood_windows
+TIMEOUT_SKIP 3 1p2b_mixed_seed71_16384_fineweb_val_seed42
+TIMEOUT_SKIP 4 1p2b_mixed_seed71_16384_pg19_ood_windows
+START 0 1p2b_mixed_seed71_16384_arxiv_ood_windows
+TIMEOUT_SKIP 0 1p2b_mixed_seed71_16384_arxiv_ood_windows
+ALL_DONE

epfl_final/train_logs/p4_extra_1p2b_h100_retry.log

@@ -0,0 +1,43 @@
+jobs 52
+filtered_1p2b_jobs 20
+START 0 ppl_1p2b_dense_seed71_16384_fineweb_val_seed42
+START 1 ppl_1p2b_dense_seed71_16384_pg19_ood_windows
+START 2 ppl_1p2b_dense_seed71_16384_arxiv_ood_windows
+START 3 ppl_1p2b_dense_seed71_30720_fineweb_val_seed42
+DONE 0 ppl_1p2b_dense_seed71_16384_fineweb_val_seed42 0
+DONE 1 ppl_1p2b_dense_seed71_16384_pg19_ood_windows 0
+START 0 ppl_1p2b_dense_seed71_30720_pg19_ood_windows
+START 1 ppl_1p2b_dense_seed71_30720_arxiv_ood_windows
+DONE 2 ppl_1p2b_dense_seed71_16384_arxiv_ood_windows 0
+START 2 passkey_1p2b_dense_seed71
+DONE 3 ppl_1p2b_dense_seed71_30720_fineweb_val_seed42 0
+START 3 niah_multikey_1p2b_dense_seed71
+DONE 2 passkey_1p2b_dense_seed71 0
+START 2 niah_multivalue_1p2b_dense_seed71
+DONE 1 ppl_1p2b_dense_seed71_30720_arxiv_ood_windows 0
+START 1 vt_1p2b_dense_seed71
+DONE 0 ppl_1p2b_dense_seed71_30720_pg19_ood_windows 0
+START 0 ppl_1p2b_mixed_seed71_16384_fineweb_val_seed42
+DONE 0 ppl_1p2b_mixed_seed71_16384_fineweb_val_seed42 0
+START 0 ppl_1p2b_mixed_seed71_16384_pg19_ood_windows
+DONE 3 niah_multikey_1p2b_dense_seed71 0
+START 3 ppl_1p2b_mixed_seed71_16384_arxiv_ood_windows
+DONE 2 niah_multivalue_1p2b_dense_seed71 0
+START 2 ppl_1p2b_mixed_seed71_30720_fineweb_val_seed42
+DONE 0 ppl_1p2b_mixed_seed71_16384_pg19_ood_windows 0
+START 0 ppl_1p2b_mixed_seed71_30720_pg19_ood_windows
+DONE 3 ppl_1p2b_mixed_seed71_16384_arxiv_ood_windows 0
+START 3 ppl_1p2b_mixed_seed71_30720_arxiv_ood_windows
+DONE 1 vt_1p2b_dense_seed71 0
+START 1 passkey_1p2b_mixed_seed71
+DONE 1 passkey_1p2b_mixed_seed71 0
+START 1 niah_multikey_1p2b_mixed_seed71
+DONE 2 ppl_1p2b_mixed_seed71_30720_fineweb_val_seed42 0
+START 2 niah_multivalue_1p2b_mixed_seed71
+DONE 0 ppl_1p2b_mixed_seed71_30720_pg19_ood_windows 0
+START 0 vt_1p2b_mixed_seed71
+DONE 3 ppl_1p2b_mixed_seed71_30720_arxiv_ood_windows 0
+DONE 1 niah_multikey_1p2b_mixed_seed71 0
+DONE 2 niah_multivalue_1p2b_mixed_seed71 0
+DONE 0 vt_1p2b_mixed_seed71 0
+ALL_DONE

epfl_final/train_logs/p4_extra_365m.log

@@ -0,0 +1,57 @@
+jobs 40
+filtered_365_jobs 27
+START 0 niah_multikey_365m_mixed_seed44
+START 1 niah_multivalue_365m_mixed_seed44
+START 2 vt_365m_mixed_seed44
+START 3 passkey_365m_dense_seed45
+START 4 niah_multikey_365m_dense_seed45
+DONE 1 niah_multivalue_365m_mixed_seed44 0
+START 1 niah_multivalue_365m_dense_seed45
+DONE 0 niah_multikey_365m_mixed_seed44 0
+START 0 vt_365m_dense_seed45
+DONE 2 vt_365m_mixed_seed44 0
+START 2 passkey_365m_mixed_seed45
+DONE 4 niah_multikey_365m_dense_seed45 0
+START 4 niah_multikey_365m_mixed_seed45
+DONE 3 passkey_365m_dense_seed45 0
+START 3 niah_multivalue_365m_mixed_seed45
+DONE 2 passkey_365m_mixed_seed45 0
+START 2 vt_365m_mixed_seed45
+DONE 1 niah_multivalue_365m_dense_seed45 0
+START 1 passkey_365m_dense_seed46
+DONE 0 vt_365m_dense_seed45 0
+START 0 niah_multikey_365m_dense_seed46
+DONE 4 niah_multikey_365m_mixed_seed45 0
+DONE 3 niah_multivalue_365m_mixed_seed45 0
+START 3 niah_multivalue_365m_dense_seed46
+START 4 vt_365m_dense_seed46
+DONE 2 vt_365m_mixed_seed45 0
+START 2 passkey_365m_mixed_seed46
+DONE 1 passkey_365m_dense_seed46 0
+START 1 niah_multikey_365m_mixed_seed46
+DONE 3 niah_multivalue_365m_dense_seed46 0
+START 3 niah_multivalue_365m_mixed_seed46
+DONE 0 niah_multikey_365m_dense_seed46 0
+START 0 vt_365m_mixed_seed46
+DONE 4 vt_365m_dense_seed46 0
+START 4 passkey_365m_dense_seed47
+DONE 2 passkey_365m_mixed_seed46 0
+START 2 niah_multikey_365m_dense_seed47
+DONE 1 niah_multikey_365m_mixed_seed46 0
+START 1 niah_multivalue_365m_dense_seed47
+DONE 0 vt_365m_mixed_seed46 0
+START 0 vt_365m_dense_seed47
+DONE 3 niah_multivalue_365m_mixed_seed46 0
+START 3 passkey_365m_mixed_seed47
+DONE 4 passkey_365m_dense_seed47 0
+START 4 niah_multikey_365m_mixed_seed47
+DONE 2 niah_multikey_365m_dense_seed47 0
+START 2 niah_multivalue_365m_mixed_seed47
+DONE 1 niah_multivalue_365m_dense_seed47 0
+DONE 4 niah_multikey_365m_mixed_seed47 0
+START 1 vt_365m_mixed_seed47
+DONE 3 passkey_365m_mixed_seed47 0
+DONE 0 vt_365m_dense_seed47 0
+DONE 2 niah_multivalue_365m_mixed_seed47 0
+DONE 1 vt_365m_mixed_seed47 0
+ALL_DONE

epfl_final/train_logs/p4_extra_365m_a100_0_4.log

@@ -0,0 +1,67 @@
+jobs 52
+filtered_365_jobs 32
+START 0 passkey_365m_dense_seed44
+START 1 niah_multikey_365m_dense_seed44
+START 2 niah_multivalue_365m_dense_seed44
+START 3 vt_365m_dense_seed44
+START 4 passkey_365m_mixed_seed44
+DONE 4 passkey_365m_mixed_seed44 0
+START 4 niah_multikey_365m_mixed_seed44
+DONE 1 niah_multikey_365m_dense_seed44 0
+START 1 niah_multivalue_365m_mixed_seed44
+DONE 2 niah_multivalue_365m_dense_seed44 0
+START 2 vt_365m_mixed_seed44
+DONE 0 passkey_365m_dense_seed44 0
+START 0 passkey_365m_dense_seed45
+DONE 3 vt_365m_dense_seed44 0
+START 3 niah_multikey_365m_dense_seed45
+DONE 1 niah_multivalue_365m_mixed_seed44 0
+START 1 niah_multivalue_365m_dense_seed45
+DONE 2 vt_365m_mixed_seed44 0
+START 2 vt_365m_dense_seed45
+DONE 4 niah_multikey_365m_mixed_seed44 0
+START 4 passkey_365m_mixed_seed45
+DONE 3 niah_multikey_365m_dense_seed45 0
+START 3 niah_multikey_365m_mixed_seed45
+DONE 0 passkey_365m_dense_seed45 0
+START 0 niah_multivalue_365m_mixed_seed45
+DONE 2 vt_365m_dense_seed45 0
+START 2 vt_365m_mixed_seed45
+DONE 1 niah_multivalue_365m_dense_seed45 0
+START 1 passkey_365m_dense_seed46
+DONE 4 passkey_365m_mixed_seed45 0
+START 4 niah_multikey_365m_dense_seed46
+DONE 3 niah_multikey_365m_mixed_seed45 0
+DONE 0 niah_multivalue_365m_mixed_seed45 0
+START 0 niah_multivalue_365m_dense_seed46
+START 3 vt_365m_dense_seed46
+DONE 2 vt_365m_mixed_seed45 0
+START 2 passkey_365m_mixed_seed46
+DONE 1 passkey_365m_dense_seed46 0
+START 1 niah_multikey_365m_mixed_seed46
+DONE 4 niah_multikey_365m_dense_seed46 0
+START 4 niah_multivalue_365m_mixed_seed46
+DONE 3 vt_365m_dense_seed46 0
+START 3 vt_365m_mixed_seed46
+DONE 0 niah_multivalue_365m_dense_seed46 0
+START 0 passkey_365m_dense_seed47
+DONE 2 passkey_365m_mixed_seed46 0
+START 2 niah_multikey_365m_dense_seed47
+DONE 4 niah_multivalue_365m_mixed_seed46 0
+START 4 niah_multivalue_365m_dense_seed47
+DONE 1 niah_multikey_365m_mixed_seed46 0
+START 1 vt_365m_dense_seed47
+DONE 3 vt_365m_mixed_seed46 0
+START 3 passkey_365m_mixed_seed47
+DONE 0 passkey_365m_dense_seed47 0
+START 0 niah_multikey_365m_mixed_seed47
+DONE 3 passkey_365m_mixed_seed47 0
+START 3 niah_multivalue_365m_mixed_seed47
+DONE 1 vt_365m_dense_seed47 0
+START 1 vt_365m_mixed_seed47
+DONE 4 niah_multivalue_365m_dense_seed47 0
+DONE 2 niah_multikey_365m_dense_seed47 0
+DONE 0 niah_multikey_365m_mixed_seed47 0
+DONE 3 niah_multivalue_365m_mixed_seed47 0
+DONE 1 vt_365m_mixed_seed47 0
+ALL_DONE

epfl_final/train_logs/p4_gap_a100.log

@@ -0,0 +1,19 @@
+jobs 36
+indices 12,13,14,15,16,17,18,19 jobs 8
+START 0 365m_dense
+START 1 365m_dense
+START 2 365m_dense
+START 3 365m_dense
+START 4 365m_mixed
+DONE 4 365m_mixed 0
+START 4 365m_mixed
+DONE 3 365m_dense 0
+START 3 365m_mixed
+DONE 0 365m_dense 0
+START 0 365m_mixed
+DONE 2 365m_dense 0
+DONE 1 365m_dense 0
+DONE 0 365m_mixed 0
+DONE 4 365m_mixed 0
+DONE 3 365m_mixed 0
+ALL_DONE

epfl_final/train_logs/p4_gap_a10080.log

@@ -0,0 +1,5 @@
+large_jobs 26
+START 0 1p2b_dense_seed70
+START 1 1p2b_dense_seed70
+START 2 1p2b_dense_seed70
+START 3 1p2b_dense_seed70

epfl_final/train_logs/p4_gap_a10080_retry.log

@@ -0,0 +1,15 @@
+large_jobs 26
+START 0 1p2b_dense_seed70
+START 1 1p2b_dense_seed70
+START 2 1p2b_dense_seed70
+START 3 1p2b_dense_seed70
+DONE 2 1p2b_dense_seed70 0
+START 2 1p2b_dense_seed70
+DONE 0 1p2b_dense_seed70 0
+START 0 1p2b_dense_seed70
+DONE 1 1p2b_dense_seed70 0
+START 1 1p2b_mixed_seed70
+DONE 1 1p2b_mixed_seed70 0
+START 1 1p2b_mixed_seed70
+DONE 3 1p2b_dense_seed70 0
+START 3 1p2b_mixed_seed70

epfl_final/train_logs/p4_gap_h100.log

@@ -0,0 +1,18 @@
+jobs 36
+indices 0,1,2,3,4,5,6,7,8,9,10,11,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35 jobs 28
+START 0 1p2b_dense_seed70
+START 1 1p2b_dense_seed70
+START 2 1p2b_dense_seed70
+START 3 1p2b_dense_seed70
+DONE 0 1p2b_dense_seed70 -15
+DONE 1 1p2b_dense_seed70 -15
+DONE 2 1p2b_dense_seed70 -15
+DONE 3 1p2b_dense_seed70 -15
+START 0 1p2b_dense_seed70
+START 1 1p2b_dense_seed70
+START 2 1p2b_mixed_seed70
+START 3 1p2b_mixed_seed70
+DONE 2 1p2b_mixed_seed70 0
+START 2 1p2b_mixed_seed70
+DONE 3 1p2b_mixed_seed70 0
+START 3 1p2b_mixed_seed70

epfl_final/train_logs/p4_lost_h100_a10080_recovery.log

@@ -0,0 +1,14 @@
+START_LOST_RECOVERY 2026-09-02T03:13:37+00:00
+large_jobs 21
+START 0 1p2b_dense_seed70
+START 1 1p2b_dense_seed70
+START 2 1p2b_mixed_seed70
+START 3 1p2b_mixed_seed70
+DONE 3 1p2b_mixed_seed70 0
+START 3 1p2b_mixed_seed70
+DONE 2 1p2b_mixed_seed70 0
+START 2 1p2b_dense
+DONE 0 1p2b_dense_seed70 0
+START 0 1p2b_dense
+DONE 1 1p2b_dense_seed70 0
+START 1 1p2b_dense

epfl_final/train_logs/p4_safe16_a.log

@@ -0,0 +1,14 @@
+SAFE16_A 6
+START 0 1p2b_dense_seed70_16384_fineweb_val_seed42
+START 1 1p2b_dense_seed70_16384_arxiv_ood_windows
+START 2 1p2b_mixed_seed70_16384_pg19_ood_windows
+START 3 1p2b_dense_seed71_16384_fineweb_val_seed42
+START 4 1p2b_dense_seed71_16384_arxiv_ood_windows
+TIMEOUT_SKIP 0 1p2b_dense_seed70_16384_fineweb_val_seed42
+TIMEOUT_SKIP 1 1p2b_dense_seed70_16384_arxiv_ood_windows
+TIMEOUT_SKIP 2 1p2b_mixed_seed70_16384_pg19_ood_windows
+TIMEOUT_SKIP 3 1p2b_dense_seed71_16384_fineweb_val_seed42
+TIMEOUT_SKIP 4 1p2b_dense_seed71_16384_arxiv_ood_windows
+START 0 1p2b_mixed_seed71_16384_pg19_ood_windows
+TIMEOUT_SKIP 0 1p2b_mixed_seed71_16384_pg19_ood_windows
+ALL_DONE

epfl_final/train_logs/queue_seed72_after_seed71.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+seed71_log=/mloscratch/homes/navasard/sparse/train_logs/p2_1p2b_mixed_seed71_4gpu_resume45250.log
+seed72_log=/mloscratch/homes/navasard/sparse/train_logs/p2_1p2b_mixed_seed72_4gpu_resume31000_after_seed71.log
+repo=/mloscratch/homes/navasard/sparse/SparseAtt
+seed71_pattern='torchrun.*p2_1p2b_mixed_seed71_4gpu_resume28500.json'
+seed72_pattern='torchrun.*p2_1p2b_mixed_seed72_4gpu_autoresume_current.json'
+while ! grep -q '^Training complete' "$seed71_log"; do sleep 60; done
+while pgrep -f "$seed71_pattern" >/dev/null; do sleep 15; done
+if pgrep -f "$seed72_pattern" >/dev/null; then exit 0; fi
+cd "$repo"
+nohup env USER=navasard LOGNAME=navasard HOME=/mloscratch/homes/navasard TORCHINDUCTOR_CACHE_DIR=/mloscratch/homes/navasard/.cache/torchinductor /mloscratch/homes/navasard/conda/bin/torchrun --standalone --nproc_per_node=4 train.py --config configs/p2_1p2b_mixed_seed72_4gpu_autoresume_current.json >> "$seed72_log" 2>&1 < /dev/null &

epfl_final/train_logs/run_p3_mixed71_locked_h100.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd /mloscratch/homes/navasard/sparse/SparseAtt
+py=/mloscratch/homes/navasard/conda/bin/python
+ckpt=/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-50462
+tok=/mloscratch/homes/navasard/qwen3_06b_base
+out=eval/results/P3_SCALE1P2B_LOCKED_20260827
+common=(--mode ruler --checkpoint "$ckpt" --tokenizer-path "$tok" --task niah_multikey --depths 0.1 0.3 0.5 0.7 0.9 --n-per-depth 40 --value-width 1 --example-seed 0 --device cuda --dtype bf16)
+"$py" -m eval.moba_eval "${common[@]}" --label p3_1p2b_mixed_s71_onekey_8k_r1 --top-k 64 16 8 4 --eval-len 8192 --batch-size 1 --n-keys 1 --evidence-repeats 1 --tsv-output "$out/p3_1p2b_mixed_s71_onekey_8k_r1.tsv"
+"$py" -m eval.moba_eval "${common[@]}" --label p3_1p2b_mixed_s71_twokey_8k_r1 --top-k 64 16 8 4 --eval-len 8192 --batch-size 1 --n-keys 2 --evidence-repeats 1 --tsv-output "$out/p3_1p2b_mixed_s71_twokey_8k_r1.tsv"
+"$py" -m eval.moba_eval "${common[@]}" --label p3_1p2b_mixed_s71_onekey_16k_r3 --top-k 128 32 16 8 --eval-len 16384 --batch-size 1 --n-keys 1 --evidence-repeats 3 --tsv-output "$out/p3_1p2b_mixed_s71_onekey_16k_r3.tsv"

epfl_final/train_logs/scale_matched_24k30k_h100.log

@@ -0,0 +1,100 @@
+QUEUE total=75 pending=71
+[1/71] START phase1_365m_30k__365m_dense_seed43__pg19
+[1/71] OK phase1_365m_30k__365m_dense_seed43__pg19 sec=95.8
+[2/71] START phase1_365m_30k__365m_dense_seed43__arxiv
+[2/71] OK phase1_365m_30k__365m_dense_seed43__arxiv sec=93.6
+[3/71] START phase1_365m_30k__365m_dense_seed44__fineweb
+[3/71] OK phase1_365m_30k__365m_dense_seed44__fineweb sec=97.5
+[4/71] START phase1_365m_30k__365m_dense_seed44__pg19
+[4/71] OK phase1_365m_30k__365m_dense_seed44__pg19 sec=95.2
+[5/71] START phase1_365m_30k__365m_dense_seed44__arxiv
+[5/71] OK phase1_365m_30k__365m_dense_seed44__arxiv sec=91.5
+[6/71] START phase1_365m_30k__365m_dense_seed45__fineweb
+[6/71] OK phase1_365m_30k__365m_dense_seed45__fineweb sec=96.4
+[7/71] START phase1_365m_30k__365m_dense_seed45__pg19
+[7/71] OK phase1_365m_30k__365m_dense_seed45__pg19 sec=93.9
+[8/71] START phase1_365m_30k__365m_dense_seed45__arxiv
+[8/71] OK phase1_365m_30k__365m_dense_seed45__arxiv sec=94.4
+[9/71] START phase1_365m_30k__365m_dense_seed46__fineweb
+[9/71] OK phase1_365m_30k__365m_dense_seed46__fineweb sec=96.1
+[10/71] START phase1_365m_30k__365m_dense_seed46__pg19
+[10/71] OK phase1_365m_30k__365m_dense_seed46__pg19 sec=96.6
+[11/71] START phase1_365m_30k__365m_dense_seed46__arxiv
+[11/71] OK phase1_365m_30k__365m_dense_seed46__arxiv sec=96.0
+[12/71] START phase1_365m_30k__365m_dense_seed47__fineweb
+[12/71] OK phase1_365m_30k__365m_dense_seed47__fineweb sec=100.6
+[13/71] START phase1_365m_30k__365m_dense_seed47__pg19
+[13/71] OK phase1_365m_30k__365m_dense_seed47__pg19 sec=95.2
+[14/71] START phase1_365m_30k__365m_dense_seed47__arxiv
+[14/71] OK phase1_365m_30k__365m_dense_seed47__arxiv sec=93.6
+[15/71] START phase1_365m_30k__365m_mixed_seed42__fineweb
+[15/71] OK phase1_365m_30k__365m_mixed_seed42__fineweb sec=97.5
+[16/71] START phase1_365m_30k__365m_mixed_seed42__pg19
+[16/71] OK phase1_365m_30k__365m_mixed_seed42__pg19 sec=95.4
+[17/71] START phase1_365m_30k__365m_mixed_seed42__arxiv
+[17/71] OK phase1_365m_30k__365m_mixed_seed42__arxiv sec=92.0
+[18/71] START phase1_365m_30k__365m_mixed_seed43__fineweb
+[18/71] OK phase1_365m_30k__365m_mixed_seed43__fineweb sec=99.5
+[19/71] START phase1_365m_30k__365m_mixed_seed43__pg19
+[19/71] OK phase1_365m_30k__365m_mixed_seed43__pg19 sec=91.4
+[20/71] START phase1_365m_30k__365m_mixed_seed43__arxiv
+[20/71] OK phase1_365m_30k__365m_mixed_seed43__arxiv sec=93.9
+[21/71] START phase1_365m_30k__365m_mixed_seed44__fineweb
+[21/71] OK phase1_365m_30k__365m_mixed_seed44__fineweb sec=97.1
+[22/71] START phase1_365m_30k__365m_mixed_seed44__pg19
+[22/71] OK phase1_365m_30k__365m_mixed_seed44__pg19 sec=93.9
+[23/71] START phase1_365m_30k__365m_mixed_seed44__arxiv
+[23/71] OK phase1_365m_30k__365m_mixed_seed44__arxiv sec=92.9
+[24/71] START phase1_365m_30k__365m_mixed_seed45__fineweb
+[24/71] OK phase1_365m_30k__365m_mixed_seed45__fineweb sec=103.0
+[25/71] START phase1_365m_30k__365m_mixed_seed45__pg19
+[25/71] OK phase1_365m_30k__365m_mixed_seed45__pg19 sec=92.9
+[26/71] START phase1_365m_30k__365m_mixed_seed45__arxiv
+[26/71] OK phase1_365m_30k__365m_mixed_seed45__arxiv sec=96.1
+[27/71] START phase1_365m_30k__365m_mixed_seed46__fineweb
+[27/71] OK phase1_365m_30k__365m_mixed_seed46__fineweb sec=93.9
+[28/71] START phase1_365m_30k__365m_mixed_seed46__pg19
+[28/71] OK phase1_365m_30k__365m_mixed_seed46__pg19 sec=94.2
+[29/71] START phase1_365m_30k__365m_mixed_seed46__arxiv
+[29/71] OK phase1_365m_30k__365m_mixed_seed46__arxiv sec=94.6
+[30/71] START phase1_365m_30k__365m_mixed_seed47__fineweb
+[30/71] OK phase1_365m_30k__365m_mixed_seed47__fineweb sec=96.4
+[31/71] START phase1_365m_30k__365m_mixed_seed47__pg19
+[31/71] OK phase1_365m_30k__365m_mixed_seed47__pg19 sec=95.2
+[32/71] START phase1_365m_30k__365m_mixed_seed47__arxiv
+[32/71] OK phase1_365m_30k__365m_mixed_seed47__arxiv sec=94.1
+[33/71] START phase2_1p2b_24k__1p2b_dense_seed70__fineweb
+[33/71] OK phase2_1p2b_24k__1p2b_dense_seed70__fineweb sec=304.3
+[34/71] START phase2_1p2b_24k__1p2b_dense_seed70__pg19
+[34/71] OK phase2_1p2b_24k__1p2b_dense_seed70__pg19 sec=144.9
+[35/71] START phase2_1p2b_24k__1p2b_dense_seed70__arxiv
+[35/71] OK phase2_1p2b_24k__1p2b_dense_seed70__arxiv sec=142.5
+[36/71] START phase2_1p2b_24k__1p2b_dense_seed71__fineweb
+[36/71] OK phase2_1p2b_24k__1p2b_dense_seed71__fineweb sec=151.8
+[37/71] START phase2_1p2b_24k__1p2b_dense_seed71__pg19
+[37/71] OK phase2_1p2b_24k__1p2b_dense_seed71__pg19 sec=142.4
+[38/71] START phase2_1p2b_24k__1p2b_dense_seed71__arxiv
+[38/71] OK phase2_1p2b_24k__1p2b_dense_seed71__arxiv sec=141.8
+[39/71] START phase2_1p2b_24k__1p2b_legacy_dense__fineweb
+[39/71] OK phase2_1p2b_24k__1p2b_legacy_dense__fineweb sec=155.7
+[40/71] START phase2_1p2b_24k__1p2b_legacy_dense__pg19
+[40/71] OK phase2_1p2b_24k__1p2b_legacy_dense__pg19 sec=140.8
+[41/71] START phase2_1p2b_24k__1p2b_legacy_dense__arxiv
+[41/71] OK phase2_1p2b_24k__1p2b_legacy_dense__arxiv sec=141.4
+[42/71] START phase2_1p2b_24k__1p2b_legacy_fixed50__fineweb
+[42/71] OK phase2_1p2b_24k__1p2b_legacy_fixed50__fineweb sec=162.8
+[43/71] START phase2_1p2b_24k__1p2b_legacy_fixed50__pg19
+[43/71] OK phase2_1p2b_24k__1p2b_legacy_fixed50__pg19 sec=143.6
+[44/71] START phase2_1p2b_24k__1p2b_legacy_fixed50__arxiv
+[44/71] OK phase2_1p2b_24k__1p2b_legacy_fixed50__arxiv sec=142.7
+[45/71] START phase2_1p2b_24k__1p2b_legacy_mixed_seed42__fineweb
+[45/71] OK phase2_1p2b_24k__1p2b_legacy_mixed_seed42__fineweb sec=149.8
+[46/71] START phase2_1p2b_24k__1p2b_legacy_mixed_seed42__pg19
+[46/71] OK phase2_1p2b_24k__1p2b_legacy_mixed_seed42__pg19 sec=142.7
+[47/71] START phase2_1p2b_24k__1p2b_legacy_mixed_seed42__arxiv
+[47/71] OK phase2_1p2b_24k__1p2b_legacy_mixed_seed42__arxiv sec=141.9
+[48/71] START phase2_1p2b_24k__1p2b_legacy_mixed_seed43__fineweb
+[48/71] OK phase2_1p2b_24k__1p2b_legacy_mixed_seed43__fineweb sec=155.2
+[49/71] START phase2_1p2b_24k__1p2b_legacy_mixed_seed43__pg19
+[49/71] OK phase2_1p2b_24k__1p2b_legacy_mixed_seed43__pg19 sec=141.1
+[50/71] START phase2_1p2b_24k__1p2b_legacy_mixed_seed43__arxiv

epfl_final/train_logs/scale_matched_24k30k_h100_resume.log

@@ -0,0 +1,46 @@
+QUEUE total=75 pending=22
+[1/22] START phase2_1p2b_24k__1p2b_legacy_mixed_seed43__arxiv
+[1/22] OK phase2_1p2b_24k__1p2b_legacy_mixed_seed43__arxiv sec=190.3
+[2/22] START phase2_1p2b_24k__1p2b_legacy_resampled50__fineweb
+[2/22] OK phase2_1p2b_24k__1p2b_legacy_resampled50__fineweb sec=155.3
+[3/22] START phase2_1p2b_24k__1p2b_legacy_resampled50__pg19
+[3/22] OK phase2_1p2b_24k__1p2b_legacy_resampled50__pg19 sec=141.6
+[4/22] START phase2_1p2b_24k__1p2b_legacy_resampled50__arxiv
+[4/22] OK phase2_1p2b_24k__1p2b_legacy_resampled50__arxiv sec=146.2
+[5/22] START phase2_1p2b_24k__1p2b_legacy_resampled65__fineweb
+[5/22] OK phase2_1p2b_24k__1p2b_legacy_resampled65__fineweb sec=150.8
+[6/22] START phase2_1p2b_24k__1p2b_legacy_resampled65__pg19
+[6/22] OK phase2_1p2b_24k__1p2b_legacy_resampled65__pg19 sec=141.2
+[7/22] START phase2_1p2b_24k__1p2b_legacy_resampled65__arxiv
+[7/22] OK phase2_1p2b_24k__1p2b_legacy_resampled65__arxiv sec=140.3
+[8/22] START phase2_1p2b_24k__1p2b_mixed_seed70__fineweb
+[8/22] OK phase2_1p2b_24k__1p2b_mixed_seed70__fineweb sec=196.7
+[9/22] START phase2_1p2b_24k__1p2b_mixed_seed70__pg19
+[9/22] OK phase2_1p2b_24k__1p2b_mixed_seed70__pg19 sec=147.1
+[10/22] START phase2_1p2b_24k__1p2b_mixed_seed70__arxiv
+[10/22] OK phase2_1p2b_24k__1p2b_mixed_seed70__arxiv sec=147.8
+[11/22] START phase2_1p2b_24k__1p2b_mixed_seed71__fineweb
+[11/22] OK phase2_1p2b_24k__1p2b_mixed_seed71__fineweb sec=156.4
+[12/22] START phase2_1p2b_24k__1p2b_mixed_seed71__pg19
+[12/22] OK phase2_1p2b_24k__1p2b_mixed_seed71__pg19 sec=146.0
+[13/22] START phase2_1p2b_24k__1p2b_mixed_seed71__arxiv
+[13/22] OK phase2_1p2b_24k__1p2b_mixed_seed71__arxiv sec=144.8
+[14/22] START phase3_mixed71_30k__1p2b_mixed_seed71__fineweb
+[14/22] OK phase3_mixed71_30k__1p2b_mixed_seed71__fineweb sec=300.6
+[15/22] START phase3_mixed71_30k__1p2b_mixed_seed71__pg19
+[15/22] OK phase3_mixed71_30k__1p2b_mixed_seed71__pg19 sec=148.8
+[16/22] START phase3_mixed71_30k__1p2b_mixed_seed71__arxiv
+[16/22] OK phase3_mixed71_30k__1p2b_mixed_seed71__arxiv sec=147.4
+[17/22] START phase4_3b_24k__3b_dense__fineweb
+[17/22] OK phase4_3b_24k__3b_dense__fineweb sec=464.6
+[18/22] START phase4_3b_24k__3b_dense__pg19
+[18/22] OK phase4_3b_24k__3b_dense__pg19 sec=260.4
+[19/22] START phase4_3b_24k__3b_dense__arxiv
+[19/22] OK phase4_3b_24k__3b_dense__arxiv sec=256.9
+[20/22] START phase4_3b_24k__3b_mixed__fineweb
+[20/22] OK phase4_3b_24k__3b_mixed__fineweb sec=282.6
+[21/22] START phase4_3b_24k__3b_mixed__pg19
+[21/22] OK phase4_3b_24k__3b_mixed__pg19 sec=261.3
+[22/22] START phase4_3b_24k__3b_mixed__arxiv
+[22/22] OK phase4_3b_24k__3b_mixed__arxiv sec=260.9
+DONE ok=22 fail=0 failures=[]